From 35bba26a1c7ef22bc80892cadf126ff80867e929 Mon Sep 17 00:00:00 2001 From: Mitzi Morris Date: Mon, 22 Dec 2025 14:24:06 -0500 Subject: [PATCH 01/81] added generic observation processes and unit tests --- pyrenew/observation/__init__.py | 23 ++ pyrenew/observation/base.py | 324 +++++++++++++++++ pyrenew/observation/count_observations.py | 327 +++++++++++++++++ pyrenew/observation/measurements.py | 133 +++++++ pyrenew/observation/noise.py | 344 ++++++++++++++++++ pyrenew/randomvariable/__init__.py | 8 + pyrenew/randomvariable/hierarchical.py | 336 ++++++++++++++++++ test/conftest.py | 406 ++++++++++++++++++++++ test/test_observation_counts.py | 378 ++++++++++++++++++++ test/test_observation_measurements.py | 234 +++++++++++++ test/test_randomvariable_hierarchical.py | 179 ++++++++++ 11 files changed, 2692 insertions(+) create mode 100644 pyrenew/observation/base.py create mode 100644 pyrenew/observation/count_observations.py create mode 100644 pyrenew/observation/measurements.py create mode 100644 pyrenew/observation/noise.py create mode 100644 pyrenew/randomvariable/hierarchical.py create mode 100644 test/conftest.py create mode 100644 test/test_observation_counts.py create mode 100644 test/test_observation_measurements.py create mode 100644 test/test_randomvariable_hierarchical.py diff --git a/pyrenew/observation/__init__.py b/pyrenew/observation/__init__.py index b0e04e69..46cfbb67 100644 --- a/pyrenew/observation/__init__.py +++ b/pyrenew/observation/__init__.py @@ -1,9 +1,32 @@ # numpydoc ignore=GL08 +from pyrenew.observation.base import BaseObservationProcess +from pyrenew.observation.count_observations import Counts, CountsBySite +from pyrenew.observation.measurements import Measurements from pyrenew.observation.negativebinomial import NegativeBinomialObservation +from pyrenew.observation.noise import ( + CountNoise, + HierarchicalNormalNoise, + MeasurementNoise, + NegativeBinomialNoise, + PoissonNoise, +) from pyrenew.observation.poisson import PoissonObservation __all__ = [ + # Existing (kept for backward compatibility) "NegativeBinomialObservation", "PoissonObservation", + # New base classes + "BaseObservationProcess", + # New noise models + "CountNoise", + "PoissonNoise", + "NegativeBinomialNoise", + "MeasurementNoise", + "HierarchicalNormalNoise", + # New observation processes + "Counts", + "CountsBySite", + "Measurements", ] diff --git a/pyrenew/observation/base.py b/pyrenew/observation/base.py new file mode 100644 index 00000000..d84125a1 --- /dev/null +++ b/pyrenew/observation/base.py @@ -0,0 +1,324 @@ +# numpydoc ignore=GL08 +""" +Abstract base class for observation processes. + +Provides common functionality for observation processes that use convolution +with temporal distributions to connect infections to observed data. +""" + +from __future__ import annotations + +from abc import abstractmethod + +import jax.numpy as jnp +import numpyro +from jax.typing import ArrayLike + +from pyrenew.convolve import compute_delay_ascertained_incidence +from pyrenew.metaclass import RandomVariable + + +class BaseObservationProcess(RandomVariable): + """ + Abstract base class for observation processes that use convolution + with temporal distributions. + + This class provides common functionality for connecting infections + to observed data (e.g., hospital admissions, wastewater concentrations) + through temporal convolution operations. + + Key features provided: + + - PMF validation (sum to 1, non-negative) + - Minimum observation day calculation + - Convolution wrapper with timeline alignment + - Deterministic quantity tracking + + Notes + ----- + Computing expected observations on day t requires infection history + from previous days (determined by the temporal PMF length). + The first ``len(pmf) - 1`` days have insufficient history and return NaN. + + See Also + -------- + pyrenew.convolve.compute_delay_ascertained_incidence : + Underlying convolution function + pyrenew.metaclass.RandomVariable : + Base class for all random variables + """ + + def __init__(self, temporal_pmf_rv: RandomVariable) -> None: + """ + Initialize base observation process. + + Parameters + ---------- + temporal_pmf_rv : RandomVariable + The temporal distribution PMF (e.g., delay or shedding distribution). + Must sample to a 1D array that sums to ~1.0 with non-negative values. + Subclasses may have additional parameters. + + Notes + ----- + Subclasses should call ``super().__init__(temporal_pmf_rv)`` + in their constructors and may add additional parameters. + """ + self.temporal_pmf_rv = temporal_pmf_rv + + @abstractmethod + def validate(self) -> None: + """ + Validate observation process parameters. + + Subclasses must implement this method to validate all parameters. + Typically this involves calling ``_validate_pmf()`` for the PMF + and adding any additional parameter-specific validation. + + Raises + ------ + ValueError + If any parameters fail validation. + """ + pass + + @abstractmethod + def get_required_lookback(self) -> int: + """ + Return the number of days this observation process needs to look back. + + This determines the minimum n_initialization_points required by the + latent process when this observation is included in a multi-signal model. + + Returns + ------- + int + Number of days of infection history required. + Typically the length of the delay or shedding PMF. + + Notes + ----- + This is used by model builders to automatically compute + n_initialization_points as: + ``max(gen_int_length, max(all lookbacks)) - 1`` + """ + pass + + @abstractmethod + def infection_resolution(self) -> str: + """ + Return the resolution of infections this observation uses. + + Returns one of: + + - ``"jurisdiction"``: Uses jurisdiction-level aggregated infections + - ``"site"``: Uses site-level disaggregated infections + + Returns + ------- + str + Either ``"jurisdiction"`` or ``"site"`` + + Examples + -------- + >>> # Aggregated count observations use jurisdiction-level + >>> hosp_obs.infection_resolution() # Returns "jurisdiction" + >>> + >>> # Wastewater uses site-level + >>> ww_obs.infection_resolution() # Returns "site" + + Notes + ----- + This is used by multi-signal models to route the correct infection + output to each observation process. + """ + pass + + def _validate_pmf( + self, + pmf: ArrayLike, + param_name: str, + atol: float = 1e-6, + ) -> None: + """ + Validate that an array is a valid probability mass function. + + Checks: + + - Non-empty array + - Sums to 1.0 (within tolerance) + - All non-negative values + + Parameters + ---------- + pmf : ArrayLike + The PMF array to validate + param_name : str + Name of the parameter (for error messages) + atol : float, default 1e-6 + Absolute tolerance for sum-to-one check + + Raises + ------ + ValueError + If PMF is empty, doesn't sum to 1.0 (within tolerance), + or contains negative values. + """ + if pmf.size == 0: + raise ValueError(f"{param_name} must return non-empty array") + + pmf_sum = jnp.sum(pmf) + if not jnp.isclose(pmf_sum, 1.0, atol=atol): + raise ValueError( + f"{param_name} must sum to 1.0 (±{atol}), got {float(pmf_sum):.6f}" + ) + + if jnp.any(pmf < 0): + raise ValueError(f"{param_name} must have non-negative values") + + def get_minimum_observation_day(self) -> int: + """ + Get the first day with valid (non-NaN) convolution results. + + Due to the convolution operation requiring a history window, + the first ``len(pmf) - 1`` days will have NaN values in the + output. This method returns the index of the first valid day. + + Returns + ------- + int + Day index (0-based) of first valid observation. + Equal to ``len(pmf) - 1``. + """ + pmf = self.temporal_pmf_rv() + return int(len(pmf) - 1) + + def _convolve_with_alignment( + self, + latent_incidence: ArrayLike, + pmf: ArrayLike, + p_observed: float = 1.0, + ) -> tuple[ArrayLike, int]: + """ + Convolve latent incidence with PMF while maintaining timeline alignment. + + This is a wrapper around ``compute_delay_ascertained_incidence`` that + always uses ``pad=True`` to ensure day t in the output corresponds to + day t in the input. The first ``len(pmf) - 1`` days will be NaN. + + Parameters + ---------- + latent_incidence : ArrayLike + Latent incidence time series (infections, prevalence, etc.). + Shape: (n_days,) + pmf : ArrayLike + Delay or shedding PMF. Shape: (n_pmf,) + p_observed : float, default 1.0 + Observation probability multiplier. Scales the convolution result. + + Returns + ------- + tuple[ArrayLike, int] + - convolved_array : ArrayLike + Convolved time series with same length as input. + First ``len(pmf) - 1`` days are NaN. + Shape: (n_days,) + - offset : int + Always 0 when pad=True (maintained for API compatibility) + + Notes + ----- + For t < len(pmf)-1, there is insufficient history, so output[t] = NaN. + + See Also + -------- + pyrenew.convolve.compute_delay_ascertained_incidence : + Underlying function + """ + return compute_delay_ascertained_incidence( + latent_incidence=latent_incidence, + delay_incidence_to_observation_pmf=pmf, + p_observed_given_incident=p_observed, + pad=True, # Maintains timeline alignment + ) + + def _deterministic(self, name: str, value: ArrayLike) -> None: + """ + Track a deterministic quantity in the numpyro execution trace. + + This is a convenience wrapper around ``numpyro.deterministic`` for + tracking intermediate quantities (e.g., latent admissions, expected + concentrations) that are useful for diagnostics and model checking. + These quantities are stored in MCMC samples and can be used for + model diagnostics and posterior predictive checks. + + Parameters + ---------- + name : str + Name for the tracked quantity. Will appear in MCMC samples. + value : ArrayLike + Value to track. Can be any shape. + """ + numpyro.deterministic(name, value) + + @abstractmethod + def _expected_signal( + self, + infections: ArrayLike, + ) -> ArrayLike: + """ + Transform infections to expected observation values. + + This is the core transformation that each observation process must + implement. It converts infections (from the infection process) + to expected values for the observation model. + + Parameters + ---------- + infections : ArrayLike + Infections from the infection process. + Shape: (n_days,) for jurisdiction-level observations + Shape: (n_days, n_sites) for site-level observations + + Returns + ------- + ArrayLike + Expected observation values (counts, log-concentrations, etc.). + Same shape as input, with first len(pmf)-1 days as NaN. + + Notes + ----- + The transformation is observation-specific: + + - Count observations: ascertainment x delay convolution -> expected counts + - Wastewater: shedding convolution -> genome scaling -> dilution -> log + + See Also + -------- + sample : Uses this method then applies noise model + """ + pass + + @abstractmethod + def sample(self, **kwargs) -> ArrayLike: + """ + Sample from the observation process. + + Subclasses must implement this method to define the specific + observation model. Typically calls ``_expected_signal`` first, + then applies the noise model. + + Parameters + ---------- + **kwargs + Subclass-specific parameters. At minimum, should include: + + - infections from the infection process + - Observed data (or None for prior predictive sampling) + + Returns + ------- + ArrayLike + Observed or sampled values from the observation process. + """ + pass diff --git a/pyrenew/observation/count_observations.py b/pyrenew/observation/count_observations.py new file mode 100644 index 00000000..437c37e6 --- /dev/null +++ b/pyrenew/observation/count_observations.py @@ -0,0 +1,327 @@ +# numpydoc ignore=GL08 +""" +Count observations with composable noise models. + +Ascertainment x delay convolution with pluggable noise (Poisson, Negative Binomial, etc.). +""" + +from __future__ import annotations + +import jax +import jax.numpy as jnp +from jax.typing import ArrayLike + +from pyrenew.metaclass import RandomVariable +from pyrenew.observation.base import BaseObservationProcess +from pyrenew.observation.noise import CountNoise + + +class _CountBase(BaseObservationProcess): + """ + Internal base for count observation processes. + + Implements ascertainment x delay convolution with pluggable noise model. + """ + + def __init__( + self, + ascertainment_rate_rv: RandomVariable, + delay_distribution_rv: RandomVariable, + noise: CountNoise, + ) -> None: + """ + Initialize count observation base. + + Parameters + ---------- + ascertainment_rate_rv : RandomVariable + Ascertainment rate in [0, 1] (e.g., IHR, IER). + delay_distribution_rv : RandomVariable + Delay distribution PMF (must sum to ~1.0). + noise : CountNoise + Noise model for count observations (Poisson, NegBin, etc.). + """ + super().__init__(temporal_pmf_rv=delay_distribution_rv) + self.ascertainment_rate_rv = ascertainment_rate_rv + self.noise = noise + + def validate(self) -> None: + """ + Validate observation parameters. + + Raises + ------ + ValueError + If delay PMF invalid, ascertainment rate outside [0,1], + or noise params invalid. + """ + delay_pmf = self.temporal_pmf_rv() + self._validate_pmf(delay_pmf, "delay_distribution_rv") + + ascertainment_rate = self.ascertainment_rate_rv() + if jnp.any(ascertainment_rate < 0) or jnp.any(ascertainment_rate > 1): + raise ValueError( + "ascertainment_rate_rv must be in [0, 1], " + "got value(s) outside this range" + ) + + self.noise.validate() + + def get_required_lookback(self) -> int: + """ + Return delay PMF length. + + Returns + ------- + int + Length of delay distribution PMF. + """ + return len(self.temporal_pmf_rv()) + + def infection_resolution(self) -> str: + """ + Return required infection resolution. + + Returns + ------- + str + "jurisdiction" for aggregated, "site" for disaggregated. + """ + raise NotImplementedError("Subclasses must implement infection_resolution()") + + def _expected_signal( + self, + infections: ArrayLike, + ) -> ArrayLike: + """ + Compute expected counts via ascertainment x delay convolution. + + Parameters + ---------- + infections : ArrayLike + Infections from the infection process. + Shape: (n_days,) for jurisdiction-level + Shape: (n_days, n_sites) for site-level + + Returns + ------- + ArrayLike + Expected counts with timeline alignment. + Same shape as input. + First len(delay_pmf)-1 days are NaN. + """ + delay_pmf = self.temporal_pmf_rv() + ascertainment_rate = self.ascertainment_rate_rv() + + is_1d = infections.ndim == 1 + if is_1d: + infections = infections[:, jnp.newaxis] + + def convolve_col(col): # numpydoc ignore=GL08 + return self._convolve_with_alignment(col, delay_pmf, ascertainment_rate)[0] + + expected_counts = jax.vmap(convolve_col, in_axes=1, out_axes=1)(infections) + + return expected_counts[:, 0] if is_1d else expected_counts + + +class Counts(_CountBase): + """ + Aggregated count observation for jurisdiction-level data. + + Maps jurisdiction-level infections to aggregated counts through + ascertainment x delay convolution with composable noise model. + + Parameters + ---------- + ascertainment_rate_rv : RandomVariable + Ascertainment rate in [0, 1] (e.g., IHR, IER). + delay_distribution_rv : RandomVariable + Delay distribution PMF (must sum to ~1.0). + noise : CountNoise + Noise model (PoissonNoise, NegativeBinomialNoise, etc.). + + Notes + ----- + Output preserves input timeline. First len(delay_pmf)-1 days return + -1 or ~0 (depending on noise model) due to NaN padding. + + Examples + -------- + >>> from pyrenew.deterministic import DeterministicVariable, DeterministicPMF + >>> from pyrenew.observation import Counts, NegativeBinomialNoise + >>> import jax.numpy as jnp + >>> import numpyro + >>> + >>> delay_pmf = jnp.array([0.2, 0.5, 0.3]) + >>> counts_obs = Counts( + ... ascertainment_rate_rv=DeterministicVariable("ihr", 0.01), + ... delay_distribution_rv=DeterministicPMF("delay", delay_pmf), + ... noise=NegativeBinomialNoise(DeterministicVariable("conc", 10.0)), + ... ) + >>> + >>> with numpyro.handlers.seed(rng_seed=42): + ... infections = jnp.ones(30) * 1000 + ... sampled_counts = counts_obs.sample(infections=infections, counts=None) + """ + + def infection_resolution(self) -> str: + """ + Return "jurisdiction" for aggregated observations. + + Returns + ------- + str + The string "jurisdiction". + """ + return "jurisdiction" + + def sample( + self, + infections: ArrayLike, + counts: ArrayLike | None = None, + times: ArrayLike | None = None, + ) -> ArrayLike: + """ + Sample aggregated counts with dense or sparse observations. + + Validation is performed before JAX tracing at runtime, + prior to calling this method. + + Parameters + ---------- + infections : ArrayLike + Jurisdiction-level infections from the infection process. + Shape: (n_days,) + counts : ArrayLike | None + Observed counts. Dense: (n_days,), Sparse: (n_obs,), None: prior. + times : ArrayLike | None + Day indices for sparse observations. None for dense observations. + + Returns + ------- + ArrayLike + Observed or sampled counts. + Dense: (n_days,), Sparse: (n_obs,) + """ + expected_counts = self._expected_signal(infections) + self._deterministic("expected_counts", expected_counts) + expected_counts_safe = jnp.nan_to_num(expected_counts, nan=0.0) + + # Only use sparse indexing when conditioning on observations + if times is not None and counts is not None: + expected_obs = expected_counts_safe[times] + else: + expected_obs = expected_counts_safe + + return self.noise.sample( + name="counts", + expected=expected_obs, + obs=counts, + ) + + +class CountsBySite(_CountBase): + """ + Disaggregated count observation for site-specific data. + + Maps site-level infections to site-specific counts through + ascertainment x delay convolution with composable noise model. + + Parameters + ---------- + ascertainment_rate_rv : RandomVariable + Ascertainment rate in [0, 1]. + delay_distribution_rv : RandomVariable + Delay distribution PMF (must sum to ~1.0). + noise : CountNoise + Noise model (PoissonNoise, NegativeBinomialNoise, etc.). + + Notes + ----- + Output preserves input timeline. First len(delay_pmf)-1 days are NaN. + + Examples + -------- + >>> from pyrenew.deterministic import DeterministicVariable, DeterministicPMF + >>> from pyrenew.observation import CountsBySite, PoissonNoise + >>> import jax.numpy as jnp + >>> import numpyro + >>> + >>> delay_pmf = jnp.array([0.3, 0.4, 0.3]) + >>> counts_obs = CountsBySite( + ... ascertainment_rate_rv=DeterministicVariable("ihr", 0.02), + ... delay_distribution_rv=DeterministicPMF("delay", delay_pmf), + ... noise=PoissonNoise(), + ... ) + >>> + >>> with numpyro.handlers.seed(rng_seed=42): + ... infections = jnp.ones((30, 3)) * 500 # 30 days, 3 sites + ... times = jnp.array([10, 15, 10, 15]) + ... subpop_indices = jnp.array([0, 0, 1, 1]) + ... sampled = counts_obs.sample( + ... infections=infections, + ... subpop_indices=subpop_indices, + ... times=times, + ... counts=None, + ... ) + """ + + def infection_resolution(self) -> str: + """ + Return "site" for disaggregated observations. + + Returns + ------- + str + The string "site". + """ + return "site" + + def sample( + self, + infections: ArrayLike, + subpop_indices: ArrayLike, + times: ArrayLike, + counts: ArrayLike | None = None, + ) -> ArrayLike: + """ + Sample disaggregated counts with flexible indexing. + + Validation is performed before JAX tracing at runtime, + prior to calling this method. + + Parameters + ---------- + infections : ArrayLike + Site-level infections from the infection process. + Shape: (n_days, n_sites) + subpop_indices : ArrayLike + Subpopulation index for each observation (0-indexed). + Shape: (n_obs,) + times : ArrayLike + Day index for each observation (0-indexed). + Shape: (n_obs,) + counts : ArrayLike | None + Observed counts (n_obs,), or None for prior sampling. + + Returns + ------- + ArrayLike + Observed or sampled counts. + Shape: (n_obs,) + """ + # Compute expected counts for all sites + expected_counts_all = self._expected_signal(infections) + + self._deterministic("expected_counts_by_site", expected_counts_all) + + # Replace NaN padding with 0 for distribution creation + expected_counts_safe = jnp.nan_to_num(expected_counts_all, nan=0.0) + expected_obs = expected_counts_safe[times, subpop_indices] + + return self.noise.sample( + name="counts_by_site", + expected=expected_obs, + obs=counts, + ) diff --git a/pyrenew/observation/measurements.py b/pyrenew/observation/measurements.py new file mode 100644 index 00000000..65a95fa9 --- /dev/null +++ b/pyrenew/observation/measurements.py @@ -0,0 +1,133 @@ +# numpydoc ignore=GL08 +""" +Continuous measurement observation processes. + +Abstract base for any population-level continuous measurements (wastewater, +air quality, serology, etc.) with signal-specific processing. +""" + +from jax.typing import ArrayLike + +from pyrenew.metaclass import RandomVariable +from pyrenew.observation.base import BaseObservationProcess +from pyrenew.observation.noise import MeasurementNoise + + +class Measurements(BaseObservationProcess): + """ + Abstract base for continuous measurement observations. + + Subclasses implement signal-specific transformations from infections + to expected measurement values, then add measurement noise. + + Parameters + ---------- + temporal_pmf_rv : RandomVariable + Temporal distribution PMF (e.g., shedding kinetics for wastewater). + noise : MeasurementNoise + Noise model for continuous measurements + (e.g., HierarchicalNormalNoise). + + Notes + ----- + Subclasses must implement ``_expected_signal()`` according to their + specific signal processing (e.g., wastewater shedding kinetics, + dilution factors, etc.). + + See Also + -------- + pyrenew.observation.noise.HierarchicalNormalNoise : + Suitable noise model for site-level measurements + pyrenew.observation.base.BaseObservationProcess : + Parent class with common observation utilities + """ + + def __init__( + self, + temporal_pmf_rv: RandomVariable, + noise: MeasurementNoise, + ) -> None: + """ + Initialize measurement observation base. + + Parameters + ---------- + temporal_pmf_rv : RandomVariable + Temporal distribution PMF (e.g., shedding kinetics). + noise : MeasurementNoise + Noise model (e.g., HierarchicalNormalNoise with site effects). + """ + super().__init__(temporal_pmf_rv=temporal_pmf_rv) + self.noise = noise + + def infection_resolution(self) -> str: + """ + Return "site" for measurement observations. + + Measurement observations require site-level infections + because each measurement site corresponds to a specific catchment. + + Returns + ------- + str + ``"site"`` + """ + return "site" + + def sample( + self, + infections: ArrayLike, + subpop_indices: ArrayLike, + site_indices: ArrayLike, + times: ArrayLike, + concentrations: ArrayLike | None, + n_sites: int, + ) -> ArrayLike: + """ + Sample measurements from observed sites. + + This method does not perform runtime validation of index values + (times, subpop_indices, site_indices). Validate observation data + before sampling. + + Transforms infections to expected values via signal-specific processing + (``_expected_signal``), then applies noise model. + + Parameters + ---------- + infections : ArrayLike + Infections from the infection process. + Shape: (n_days, n_sites) + subpop_indices : ArrayLike + Subpopulation index for each observation (0-indexed). + Shape: (n_obs,) + site_indices : ArrayLike + Site index for each observation (0-indexed). + Shape: (n_obs,) + times : ArrayLike + Day index for each observation (0-indexed). + Shape: (n_obs,) + concentrations : ArrayLike | None + Observed measurements (n_obs,), or None for prior sampling. + n_sites : int + Total number of measurement sites. + + Returns + ------- + ArrayLike + Observed or sampled measurements. + Shape: (n_obs,) + """ + expected_values = self._expected_signal(infections) + + self._deterministic("expected_log_conc", expected_values) + + expected_obs = expected_values[times, subpop_indices] + + return self.noise.sample( + name="concentrations", + expected=expected_obs, + obs=concentrations, + site_indices=site_indices, + n_sites=n_sites, + ) diff --git a/pyrenew/observation/noise.py b/pyrenew/observation/noise.py new file mode 100644 index 00000000..7264b726 --- /dev/null +++ b/pyrenew/observation/noise.py @@ -0,0 +1,344 @@ +# numpydoc ignore=GL08 +""" +Noise models for observation processes. + +Provides composable noise strategies for count and measurement observations, +separating the noise distribution from the observation structure. +""" + +from __future__ import annotations + +from abc import ABC, abstractmethod + +import jax.numpy as jnp +import numpyro +import numpyro.distributions as dist +from jax.typing import ArrayLike + +from pyrenew.metaclass import RandomVariable + +_EPSILON = 1e-10 + + +class CountNoise(ABC): + """ + Abstract base for count observation noise models. + + Defines how discrete count observations are distributed around expected values. + """ + + @abstractmethod + def sample( + self, + name: str, + expected: ArrayLike, + obs: ArrayLike | None = None, + ) -> ArrayLike: + """ + Sample count observations given expected counts. + + Parameters + ---------- + name : str + Numpyro sample site name. + expected : ArrayLike + Expected count values (non-negative). + obs : ArrayLike | None + Observed counts for conditioning, or None for prior sampling. + + Returns + ------- + ArrayLike + Sampled or conditioned counts, same shape as expected. + """ + pass + + @abstractmethod + def validate(self) -> None: + """ + Validate noise model parameters. + + Raises + ------ + ValueError + If parameters are invalid. + """ + pass + + +class PoissonNoise(CountNoise): + """ + Poisson noise for equidispersed counts (variance = mean). + """ + + def __init__(self) -> None: + """Initialize Poisson noise (no parameters).""" + pass + + def validate(self) -> None: + """Validate Poisson noise (always valid).""" + pass + + def sample( + self, + name: str, + expected: ArrayLike, + obs: ArrayLike | None = None, + ) -> ArrayLike: + """ + Sample from Poisson distribution. + + Parameters + ---------- + name : str + Numpyro sample site name. + expected : ArrayLike + Expected count values. + obs : ArrayLike | None + Observed counts for conditioning. + + Returns + ------- + ArrayLike + Poisson-distributed counts. + """ + return numpyro.sample( + name, + dist.Poisson(rate=expected + _EPSILON), + obs=obs, + ) + + +class NegativeBinomialNoise(CountNoise): + """ + Negative Binomial noise for overdispersed counts (variance > mean). + + Uses NB2 parameterization. Higher concentration reduces overdispersion. + + Parameters + ---------- + concentration_rv : RandomVariable + Concentration parameter (must be > 0). + Higher values reduce overdispersion. + + Notes + ----- + The NB2 parameterization has variance = mean + mean^2 / concentration. + As concentration -> infinity, this approaches Poisson. + """ + + def __init__(self, concentration_rv: RandomVariable) -> None: + """ + Initialize Negative Binomial noise. + + Parameters + ---------- + concentration_rv : RandomVariable + Concentration parameter (must be > 0). + Higher values reduce overdispersion. + """ + self.concentration_rv = concentration_rv + + def validate(self) -> None: + """ + Validate concentration is positive. + + Raises + ------ + ValueError + If concentration <= 0. + """ + concentration = self.concentration_rv() + if jnp.any(concentration <= 0): + raise ValueError( + f"NegativeBinomialNoise: concentration must be positive, " + f"got {float(concentration)}" + ) + + def sample( + self, + name: str, + expected: ArrayLike, + obs: ArrayLike | None = None, + ) -> ArrayLike: + """ + Sample from Negative Binomial distribution. + + Parameters + ---------- + name : str + Numpyro sample site name. + expected : ArrayLike + Expected count values. + obs : ArrayLike | None + Observed counts for conditioning. + + Returns + ------- + ArrayLike + Negative Binomial-distributed counts. + """ + concentration = self.concentration_rv() + return numpyro.sample( + name, + dist.NegativeBinomial2( + mean=expected + _EPSILON, + concentration=concentration, + ), + obs=obs, + ) + + +class MeasurementNoise(ABC): + """ + Abstract base for continuous measurement noise models. + + Defines how continuous observations are distributed around expected values. + """ + + @abstractmethod + def sample( + self, + name: str, + expected: ArrayLike, + obs: ArrayLike | None = None, + **kwargs, + ) -> ArrayLike: + """ + Sample continuous observations given expected values. + + Parameters + ---------- + name : str + Numpyro sample site name. + expected : ArrayLike + Expected measurement values. + obs : ArrayLike | None + Observed measurements for conditioning, or None for prior sampling. + **kwargs + Additional context (e.g., site indices). + + Returns + ------- + ArrayLike + Sampled or conditioned measurements, same shape as expected. + """ + pass + + @abstractmethod + def validate(self) -> None: + """ + Validate noise model parameters. + + Raises + ------ + ValueError + If parameters are invalid. + """ + pass + + +class HierarchicalNormalNoise(MeasurementNoise): + """ + Normal noise with hierarchical site-level effects. + + Observation model: ``obs ~ Normal(expected + site_mode, site_sd)`` + where site_mode and site_sd are hierarchically modeled. + + Parameters + ---------- + site_mode_prior_rv : RandomVariable + Hierarchical prior for site-level modes (log-scale biases). + Must support ``sample(n_groups=...)`` interface. + site_sd_prior_rv : RandomVariable + Hierarchical prior for site-level SDs (must be > 0). + Must support ``sample(n_groups=...)`` interface. + + Notes + ----- + Expects data already on log scale for wastewater applications. + + See Also + -------- + pyrenew.randomvariable.HierarchicalNormalPrior : + Suitable prior for site_mode_prior_rv + pyrenew.randomvariable.GammaGroupSdPrior : + Suitable prior for site_sd_prior_rv + """ + + def __init__( + self, + site_mode_prior_rv: RandomVariable, + site_sd_prior_rv: RandomVariable, + ) -> None: + """ + Initialize hierarchical Normal noise. + + Parameters + ---------- + site_mode_prior_rv : RandomVariable + Hierarchical prior for site-level modes (log-scale biases). + Must support ``sample(n_groups=...)`` interface. + site_sd_prior_rv : RandomVariable + Hierarchical prior for site-level SDs (must be > 0). + Must support ``sample(n_groups=...)`` interface. + """ + self.site_mode_prior_rv = site_mode_prior_rv + self.site_sd_prior_rv = site_sd_prior_rv + + def validate(self) -> None: + """ + Validate noise parameters. + + Notes + ----- + Full validation requires n_groups, which is only available during sample(). + """ + pass + + def sample( + self, + name: str, + expected: ArrayLike, + obs: ArrayLike | None = None, + *, + site_indices: ArrayLike, + n_sites: int, + ) -> ArrayLike: + """ + Sample from Normal distribution with site-level hierarchical effects. + + Parameters + ---------- + name : str + Numpyro sample site name. + expected : ArrayLike + Expected log-scale measurement values. + Shape: (n_obs,) + obs : ArrayLike | None + Observed log-scale measurements for conditioning. + Shape: (n_obs,) + site_indices : ArrayLike + Site index for each observation (0-indexed). + Shape: (n_obs,) + n_sites : int + Total number of sites. + + Returns + ------- + ArrayLike + Normal distributed measurements with hierarchical site effects. + Shape: (n_obs,) + + Raises + ------ + ValueError + If site_sd samples non-positive values. + """ + site_mode = self.site_mode_prior_rv.sample(n_groups=n_sites) + site_sd = self.site_sd_prior_rv.sample(n_groups=n_sites) + + loc = expected + site_mode[site_indices] + scale = site_sd[site_indices] + + return numpyro.sample(name, dist.Normal(loc=loc, scale=scale), obs=obs) diff --git a/pyrenew/randomvariable/__init__.py b/pyrenew/randomvariable/__init__.py index c599d101..75594c58 100644 --- a/pyrenew/randomvariable/__init__.py +++ b/pyrenew/randomvariable/__init__.py @@ -5,6 +5,11 @@ DynamicDistributionalVariable, StaticDistributionalVariable, ) +from pyrenew.randomvariable.hierarchical import ( + GammaGroupSdPrior, + HierarchicalNormalPrior, + StudentTGroupModePrior, +) from pyrenew.randomvariable.transformedvariable import TransformedVariable __all__ = [ @@ -12,4 +17,7 @@ "StaticDistributionalVariable", "DynamicDistributionalVariable", "TransformedVariable", + "HierarchicalNormalPrior", + "GammaGroupSdPrior", + "StudentTGroupModePrior", ] diff --git a/pyrenew/randomvariable/hierarchical.py b/pyrenew/randomvariable/hierarchical.py new file mode 100644 index 00000000..e97126d4 --- /dev/null +++ b/pyrenew/randomvariable/hierarchical.py @@ -0,0 +1,336 @@ +# numpydoc ignore=GL08 +""" +Hierarchical prior distributions for group-level random effects. + +These classes provide random variables that sample from hierarchical +distributions with a `sample(n_groups=...)` interface, enabling +dynamic group sizes at sample time with proper numpyro plate contexts. +""" + +import jax.numpy as jnp +import numpyro +import numpyro.distributions as dist + +from pyrenew.metaclass import RandomVariable + + +class HierarchicalNormalPrior(RandomVariable): + """ + Zero-centered Normal prior for group-level effects. + + Samples n_groups values from Normal(0, sd) within a numpyro plate context. + + Parameters + ---------- + name : str + Unique name for the sampled parameter in numpyro. + sd_rv : RandomVariable + RandomVariable returning the standard deviation. + + Notes + ----- + This class is designed for hierarchical models where group effects + are assumed to be drawn from a common distribution centered at zero. + The number of groups is specified at sample time, allowing dynamic + group sizes. + + Examples + -------- + >>> from pyrenew.deterministic import DeterministicVariable + >>> from pyrenew.randomvariable import HierarchicalNormalPrior + >>> import numpyro + >>> + >>> sd_rv = DeterministicVariable("sd", 0.5) + >>> prior = HierarchicalNormalPrior("site_effects", sd_rv) + >>> + >>> with numpyro.handlers.seed(rng_seed=42): + ... effects = prior.sample(n_groups=5) + >>> effects.shape + (5,) + """ + + def __init__( + self, + name: str, + sd_rv: RandomVariable, + ) -> None: + """ + Default constructor for HierarchicalNormalPrior. + + Parameters + ---------- + name : str + Unique name for the sampled parameter in numpyro. + sd_rv : RandomVariable + RandomVariable returning the standard deviation. + + Returns + ------- + None + """ + if not isinstance(sd_rv, RandomVariable): + raise TypeError( + f"sd_rv must be a RandomVariable, got {type(sd_rv).__name__}. " + "Use DeterministicVariable(name, value) to wrap a fixed value." + ) + + self.name = name + self.sd_rv = sd_rv + + def validate(self): + """Validate the random variable (no-op for this class).""" + pass + + def sample(self, n_groups: int, **kwargs): + """ + Sample group-level effects. + + Parameters + ---------- + n_groups : int + Number of groups. + **kwargs + Additional keyword arguments (unused). + + Returns + ------- + ArrayLike + Array of shape (n_groups,) containing sampled effects. + """ + sd = self.sd_rv() + + with numpyro.plate(f"n_{self.name}", n_groups): + effects = numpyro.sample( + self.name, + dist.Normal(0.0, sd), + ) + return effects + + +class GammaGroupSdPrior(RandomVariable): + """ + Gamma prior for group-level standard deviations, bounded away from zero. + + Samples n_groups positive values from Gamma(concentration, rate) + sd_min + within a numpyro plate context. + + Parameters + ---------- + name : str + Unique name for the sampled parameter in numpyro. + sd_mean_rv : RandomVariable + RandomVariable returning the mean of the Gamma distribution. + sd_concentration_rv : RandomVariable + RandomVariable returning the concentration (shape) parameter of Gamma. + sd_min : float, default=0.05 + Minimum SD value (lower bound). + + Notes + ----- + This class parameterizes Gamma by mean and concentration rather than + shape and rate, which is often more interpretable. The rate is computed as + concentration / mean. + + The sd_min floor prevents numerical issues when standard deviations + approach zero. + + Examples + -------- + >>> from pyrenew.deterministic import DeterministicVariable + >>> from pyrenew.randomvariable import GammaGroupSdPrior + >>> import numpyro + >>> + >>> mean_rv = DeterministicVariable("sd_mean", 0.3) + >>> conc_rv = DeterministicVariable("sd_conc", 4.0) + >>> prior = GammaGroupSdPrior("site_sd", mean_rv, conc_rv, sd_min=0.05) + >>> + >>> with numpyro.handlers.seed(rng_seed=42): + ... sds = prior.sample(n_groups=5) + >>> all(sds >= 0.05) + True + """ + + def __init__( + self, + name: str, + sd_mean_rv: RandomVariable, + sd_concentration_rv: RandomVariable, + sd_min: float = 0.05, + ) -> None: + """ + Default constructor for GammaGroupSdPrior. + + Parameters + ---------- + name : str + Unique name for the sampled parameter in numpyro. + sd_mean_rv : RandomVariable + RandomVariable returning the mean of the Gamma distribution. + sd_concentration_rv : RandomVariable + RandomVariable returning the concentration (shape) parameter. + sd_min : float, default=0.05 + Minimum SD value (lower bound). + + Returns + ------- + None + """ + if not isinstance(sd_mean_rv, RandomVariable): + raise TypeError( + f"sd_mean_rv must be a RandomVariable, got {type(sd_mean_rv).__name__}. " + "Use DeterministicVariable(name, value) to wrap a fixed value." + ) + if not isinstance(sd_concentration_rv, RandomVariable): + raise TypeError( + f"sd_concentration_rv must be a RandomVariable, got {type(sd_concentration_rv).__name__}. " + "Use DeterministicVariable(name, value) to wrap a fixed value." + ) + if sd_min < 0: + raise ValueError(f"sd_min must be non-negative, got {sd_min}") + + self.name = name + self.sd_mean_rv = sd_mean_rv + self.sd_concentration_rv = sd_concentration_rv + self.sd_min = sd_min + + def validate(self): + """Validate the random variable (no-op for this class).""" + pass + + def sample(self, n_groups: int, **kwargs): + """ + Sample group-level standard deviations. + + Parameters + ---------- + n_groups : int + Number of groups. + **kwargs + Additional keyword arguments (unused). + + Returns + ------- + ArrayLike + Array of shape (n_groups,) with values >= sd_min. + """ + sd_mean = self.sd_mean_rv() + concentration = self.sd_concentration_rv() + rate = concentration / sd_mean + + with numpyro.plate(f"n_{self.name}", n_groups): + raw_sd = numpyro.sample( + f"{self.name}_raw", + dist.Gamma(concentration, rate), + ) + + group_sd = numpyro.deterministic( + self.name, + jnp.maximum(raw_sd, self.sd_min), + ) + return group_sd + + +class StudentTGroupModePrior(RandomVariable): + """ + Zero-centered Student-t prior for group-level modes (robust alternative to Normal). + + Samples n_groups values from StudentT(df, 0, sd) within a numpyro plate context. + This is useful when group effects may have heavier tails than a Normal distribution. + + Parameters + ---------- + name : str + Unique name for the sampled parameter in numpyro. + sd_rv : RandomVariable + RandomVariable returning the scale parameter. + df_rv : RandomVariable + RandomVariable returning the degrees of freedom. + + Notes + ----- + The Student-t distribution approaches the Normal distribution as df -> infinity. + Lower df values give heavier tails, making the prior more robust to outliers. + Common choices include df=3 (heavy tails) or df=7 (moderate tails). + + Examples + -------- + >>> from pyrenew.deterministic import DeterministicVariable + >>> from pyrenew.randomvariable import StudentTGroupModePrior + >>> import numpyro + >>> + >>> sd_rv = DeterministicVariable("scale", 0.5) + >>> df_rv = DeterministicVariable("df", 4.0) + >>> prior = StudentTGroupModePrior("site_modes", sd_rv, df_rv) + >>> + >>> with numpyro.handlers.seed(rng_seed=42): + ... modes = prior.sample(n_groups=5) + >>> modes.shape + (5,) + """ + + def __init__( + self, + name: str, + sd_rv: RandomVariable, + df_rv: RandomVariable, + ) -> None: + """ + Default constructor for StudentTGroupModePrior. + + Parameters + ---------- + name : str + Unique name for the sampled parameter in numpyro. + sd_rv : RandomVariable + RandomVariable returning the scale parameter. + df_rv : RandomVariable + RandomVariable returning the degrees of freedom. + + Returns + ------- + None + """ + if not isinstance(sd_rv, RandomVariable): + raise TypeError( + f"sd_rv must be a RandomVariable, got {type(sd_rv).__name__}. " + "Use DeterministicVariable(name, value) to wrap a fixed value." + ) + if not isinstance(df_rv, RandomVariable): + raise TypeError( + f"df_rv must be a RandomVariable, got {type(df_rv).__name__}. " + "Use DeterministicVariable(name, value) to wrap a fixed value." + ) + + self.name = name + self.sd_rv = sd_rv + self.df_rv = df_rv + + def validate(self): + """Validate the random variable (no-op for this class).""" + pass + + def sample(self, n_groups: int, **kwargs): + """ + Sample group-level modes. + + Parameters + ---------- + n_groups : int + Number of groups. + **kwargs + Additional keyword arguments (unused). + + Returns + ------- + ArrayLike + Array of shape (n_groups,) containing sampled modes. + """ + sd = self.sd_rv() + df = self.df_rv() + + with numpyro.plate(f"n_{self.name}", n_groups): + effects = numpyro.sample( + self.name, + dist.StudentT(df=df, loc=0.0, scale=sd), + ) + return effects diff --git a/test/conftest.py b/test/conftest.py new file mode 100644 index 00000000..73854684 --- /dev/null +++ b/test/conftest.py @@ -0,0 +1,406 @@ +""" +Shared pytest fixtures for PyRenew tests. + +This module provides reusable fixtures for creating observation processes, +test data, and common configurations used across multiple test files. +""" + +import jax.numpy as jnp +import pytest + +from pyrenew.deterministic import DeterministicPMF, DeterministicVariable +from pyrenew.observation import Counts, NegativeBinomialNoise +from pyrenew.randomvariable import GammaGroupSdPrior, HierarchicalNormalPrior + +# ============================================================================= +# PMF Fixtures +# ============================================================================= + + +@pytest.fixture +def simple_delay_pmf(): + """ + Simple 1-day delay PMF (no delay). + + Returns + ------- + jnp.ndarray + A single-element PMF array representing no delay. + """ + return jnp.array([1.0]) + + +@pytest.fixture +def short_delay_pmf(): + """ + Short 2-day delay PMF. + + Returns + ------- + jnp.ndarray + A 2-element PMF array. + """ + return jnp.array([0.5, 0.5]) + + +@pytest.fixture +def medium_delay_pmf(): + """ + Medium 4-day delay PMF. + + Returns + ------- + jnp.ndarray + A 4-element PMF array. + """ + return jnp.array([0.1, 0.3, 0.4, 0.2]) + + +@pytest.fixture +def realistic_delay_pmf(): + """ + Realistic 10-day delay PMF (shifted gamma-like). + + Returns + ------- + jnp.ndarray + A 10-element PMF array with gamma-like shape. + """ + return jnp.array([0.01, 0.05, 0.10, 0.15, 0.20, 0.20, 0.15, 0.08, 0.04, 0.02]) + + +@pytest.fixture +def long_delay_pmf(): + """ + Long 10-day delay PMF for edge case testing. + + Returns + ------- + jnp.ndarray + A 10-element PMF array. + """ + return jnp.array([0.05, 0.1, 0.15, 0.2, 0.2, 0.15, 0.1, 0.03, 0.01, 0.01]) + + +@pytest.fixture +def simple_shedding_pmf(): + """ + Simple 1-day shedding PMF (no delay). + + Returns + ------- + jnp.ndarray + A single-element PMF array representing no shedding delay. + """ + return jnp.array([1.0]) + + +@pytest.fixture +def short_shedding_pmf(): + """ + Short 3-day shedding PMF. + + Returns + ------- + jnp.ndarray + A 3-element PMF array. + """ + return jnp.array([0.3, 0.4, 0.3]) + + +@pytest.fixture +def medium_shedding_pmf(): + """ + Medium 5-day shedding PMF. + + Returns + ------- + jnp.ndarray + A 5-element PMF array. + """ + return jnp.array([0.1, 0.3, 0.3, 0.2, 0.1]) + + +# ============================================================================= +# Hierarchical Prior Fixtures +# ============================================================================= + + +@pytest.fixture +def site_mode_prior(): + """ + Standard hierarchical normal prior for site modes. + + Returns + ------- + HierarchicalNormalPrior + A hierarchical normal prior with standard deviation 0.5. + """ + return HierarchicalNormalPrior( + name="ww_site_mode", sd_rv=DeterministicVariable("mode_sd", 0.5) + ) + + +@pytest.fixture +def site_mode_prior_tight(): + """ + Tight hierarchical normal prior for deterministic-like behavior. + + Returns + ------- + HierarchicalNormalPrior + A hierarchical normal prior with small standard deviation 0.01. + """ + return HierarchicalNormalPrior( + name="ww_site_mode", sd_rv=DeterministicVariable("mode_sd_tight", 0.01) + ) + + +@pytest.fixture +def site_sd_prior(): + """ + Standard gamma prior for site standard deviations. + + Returns + ------- + GammaGroupSdPrior + A gamma prior for group standard deviations. + """ + return GammaGroupSdPrior( + name="ww_site_sd", + sd_mean_rv=DeterministicVariable("sd_mean", 0.3), + sd_concentration_rv=DeterministicVariable("sd_concentration", 4.0), + sd_min=0.10, + ) + + +@pytest.fixture +def site_sd_prior_tight(): + """ + Tight gamma prior for deterministic-like behavior. + + Returns + ------- + GammaGroupSdPrior + A gamma prior with small mean for tight behavior. + """ + return GammaGroupSdPrior( + name="ww_site_sd", + sd_mean_rv=DeterministicVariable("sd_mean_tight", 0.01), + sd_concentration_rv=DeterministicVariable("sd_concentration_tight", 4.0), + sd_min=0.005, + ) + + +# ============================================================================= +# Counts Process Fixtures +# ============================================================================= + + +@pytest.fixture +def counts_process(simple_delay_pmf): + """ + Standard Counts observation process with simple delay. + + Returns + ------- + Counts + A Counts observation process with no delay. + """ + return Counts( + ascertainment_rate_rv=DeterministicVariable("ihr", 0.01), + delay_distribution_rv=DeterministicPMF("delay", simple_delay_pmf), + noise=NegativeBinomialNoise(DeterministicVariable("conc", 10.0)), + ) + + +@pytest.fixture +def counts_process_medium_delay(medium_delay_pmf): + """ + Counts observation process with medium delay. + + Returns + ------- + Counts + A Counts observation process with 4-day delay. + """ + return Counts( + ascertainment_rate_rv=DeterministicVariable("ihr", 0.01), + delay_distribution_rv=DeterministicPMF("delay", medium_delay_pmf), + noise=NegativeBinomialNoise(DeterministicVariable("conc", 50.0)), + ) + + +@pytest.fixture +def counts_process_realistic(realistic_delay_pmf): + """ + Counts observation process with realistic delay and ascertainment. + + Returns + ------- + Counts + A Counts observation process with realistic parameters. + """ + return Counts( + ascertainment_rate_rv=DeterministicVariable("ihr", 0.005), + delay_distribution_rv=DeterministicPMF("delay", realistic_delay_pmf), + noise=NegativeBinomialNoise(DeterministicVariable("conc", 100.0)), + ) + + +class CountsProcessFactory: + """Factory for creating Counts processes with custom parameters.""" + + @staticmethod + def create( + delay_pmf=None, + ascertainment_rate=0.01, + concentration=10.0, + ): + """ + Create a Counts process with specified parameters. + + Returns + ------- + Counts + A Counts observation process with the specified parameters. + """ + if delay_pmf is None: + delay_pmf = jnp.array([1.0]) + return Counts( + ascertainment_rate_rv=DeterministicVariable("ihr", ascertainment_rate), + delay_distribution_rv=DeterministicPMF("delay", delay_pmf), + noise=NegativeBinomialNoise(DeterministicVariable("conc", concentration)), + ) + + +@pytest.fixture +def counts_factory(): + """ + Factory fixture for creating custom Counts processes. + + Returns + ------- + CountsProcessFactory + A factory for creating Counts processes. + """ + return CountsProcessFactory() + + +# ============================================================================= +# Infection Fixtures +# ============================================================================= + + +@pytest.fixture +def constant_infections(): + """ + Constant infections array (30 days, 100 infections/day). + + Returns + ------- + jnp.ndarray + A 1D array of shape (30,) with constant value 100. + """ + return jnp.ones(30) * 100 + + +@pytest.fixture +def constant_infections_2d(): + """ + Constant infections array for 2 subpopulations. + + Returns + ------- + jnp.ndarray + A 2D array of shape (30, 2) with constant value 100. + """ + return jnp.ones((30, 2)) * 100 + + +def make_infections(n_days, n_subpops=None, value=100.0): + """ + Create infection arrays for testing. + + Parameters + ---------- + n_days : int + Number of days + n_subpops : int, optional + Number of subpopulations (None for 1D array) + value : float + Constant infection value + + Returns + ------- + jnp.ndarray + Infections array + """ + if n_subpops is None: + return jnp.ones(n_days) * value + return jnp.ones((n_days, n_subpops)) * value + + +def make_spike_infections(n_days, spike_day, spike_value=1000.0, n_subpops=None): + """ + Create spike infection arrays for testing. + + Parameters + ---------- + n_days : int + Number of days + spike_day : int + Day of the spike + spike_value : float + Value at spike + n_subpops : int, optional + Number of subpopulations + + Returns + ------- + jnp.ndarray + Infections array with spike + """ + if n_subpops is None: + infections = jnp.zeros(n_days) + return infections.at[spike_day].set(spike_value) + infections = jnp.zeros((n_days, n_subpops)) + return infections.at[spike_day, :].set(spike_value) + + +def create_mock_infections( + n_days: int, + peak_day: int = 10, + peak_value: float = 1000.0, + shape: str = "spike", +) -> jnp.ndarray: + """ + Create mock infection time series for testing. + + Parameters + ---------- + n_days : int + Number of days + peak_day : int + Day of peak infections + peak_value : float + Peak infection value + shape : str + Shape of the curve: "spike", "constant", or "decay" + + Returns + ------- + jnp.ndarray + Array of infections of shape (n_days,) + """ + if shape == "spike": + infections = jnp.zeros(n_days) + infections = infections.at[peak_day].set(peak_value) + elif shape == "constant": + infections = jnp.ones(n_days) * peak_value + elif shape == "decay": + infections = peak_value * jnp.exp(-jnp.arange(n_days) / 20.0) + else: + raise ValueError(f"Unknown shape: {shape}") + + return infections diff --git a/test/test_observation_counts.py b/test/test_observation_counts.py new file mode 100644 index 00000000..cd87d403 --- /dev/null +++ b/test/test_observation_counts.py @@ -0,0 +1,378 @@ +""" +Unit tests for Counts (aggregated count observations). + +These tests validate the count observation process implementation. +""" + +import jax.numpy as jnp +import numpyro +import numpyro.distributions as dist +import pytest + +from pyrenew.deterministic import DeterministicPMF, DeterministicVariable +from pyrenew.observation import ( + Counts, + CountsBySite, + NegativeBinomialNoise, + PoissonNoise, +) +from pyrenew.randomvariable import DistributionalVariable + + +def create_mock_infections( + n_days: int, + peak_day: int = 10, + peak_value: float = 1000.0, + shape: str = "spike", +) -> jnp.ndarray: + """ + Create mock infection time series for testing. + + Parameters + ---------- + n_days : int + Number of days + peak_day : int + Day of peak infections + peak_value : float + Peak infection value + shape : str + Shape of the curve: "spike", "constant", or "decay" + + Returns + ------- + jnp.ndarray + Array of infections of shape (n_days,) + """ + if shape == "spike": + infections = jnp.zeros(n_days) + infections = infections.at[peak_day].set(peak_value) + elif shape == "constant": + infections = jnp.ones(n_days) * peak_value + elif shape == "decay": + infections = peak_value * jnp.exp(-jnp.arange(n_days) / 20.0) + else: + raise ValueError(f"Unknown shape: {shape}") + + return infections + + +class TestCountsBasics: + """Test basic functionality of aggregated count observation process.""" + + def test_sample_returns_correct_shape(self, counts_process): + """Test that sample returns correct shape.""" + infections = jnp.ones(30) * 100 + + with numpyro.handlers.seed(rng_seed=42): + counts = counts_process.sample( + infections=infections, + counts=None, + ) + + assert counts.shape[0] > 0 + assert counts.ndim == 1 + + def test_delay_convolution(self, counts_factory, short_delay_pmf): + """Test that delay is properly applied.""" + process = counts_factory.create(delay_pmf=short_delay_pmf) + + infections = jnp.zeros(30) + infections = infections.at[10].set(1000) + + with numpyro.handlers.seed(rng_seed=42): + counts = process.sample( + infections=infections, + counts=None, + ) + + # Timeline alignment: output length equals input length + assert counts.shape[0] == len(infections) + # First len(delay_pmf)-1 days are NaN (appear as -1 after NegativeBinomial sampling) + assert jnp.all(counts[1:] >= 0) + assert jnp.sum(counts[counts >= 0]) > 0 + + def test_ascertainment_scaling(self, counts_factory, simple_delay_pmf): + """Test that ascertainment rate properly scales counts.""" + infections = jnp.ones(20) * 100 + + results = [] + for rate_value in [0.01, 0.02, 0.05]: + process = counts_factory.create( + delay_pmf=simple_delay_pmf, + ascertainment_rate=rate_value, + ) + + with numpyro.handlers.seed(rng_seed=42): + counts = process.sample( + infections=infections, + counts=None, + ) + results.append(jnp.mean(counts)) + + # Higher ascertainment rate should lead to more counts + assert results[1] > results[0] + assert results[2] > results[1] + + def test_negative_binomial_observation(self, counts_factory, simple_delay_pmf): + """Test that negative binomial observation is used.""" + process = counts_factory.create( + delay_pmf=simple_delay_pmf, + concentration=5.0, + ) + + infections = jnp.ones(20) * 100 + + samples = [] + for seed in range(5): + with numpyro.handlers.seed(rng_seed=seed): + counts = process.sample( + infections=infections, + counts=None, + ) + samples.append(jnp.sum(counts)) + + # Should have some variability due to negative binomial sampling + assert jnp.std(jnp.array(samples)) > 0 + + +class TestCountsWithPriors: + """Test aggregated count observation with uncertain parameters.""" + + def test_with_stochastic_ascertainment(self, short_shedding_pmf): + """Test with uncertain ascertainment rate parameter.""" + delay = DeterministicPMF("delay", jnp.array([0.2, 0.5, 0.3])) + ascertainment = DistributionalVariable("ihr", dist.Beta(2, 100)) + concentration = DeterministicVariable("conc", 10.0) + + process = Counts( + ascertainment_rate_rv=ascertainment, + delay_distribution_rv=delay, + noise=NegativeBinomialNoise(concentration), + ) + + infections = jnp.ones(20) * 100 + + with numpyro.handlers.seed(rng_seed=42): + counts = process.sample( + infections=infections, + counts=None, + ) + + assert counts.shape[0] > 0 + # Skip NaN padding + valid_counts = counts[2:] + assert jnp.all(valid_counts >= 0) + + def test_with_stochastic_concentration(self, simple_delay_pmf): + """Test with uncertain concentration parameter.""" + delay = DeterministicPMF("delay", simple_delay_pmf) + ascertainment = DeterministicVariable("ihr", 0.01) + concentration = DistributionalVariable("conc", dist.HalfNormal(10.0)) + + process = Counts( + ascertainment_rate_rv=ascertainment, + delay_distribution_rv=delay, + noise=NegativeBinomialNoise(concentration), + ) + + infections = jnp.ones(20) * 100 + + with numpyro.handlers.seed(rng_seed=42): + counts = process.sample( + infections=infections, + counts=None, + ) + + assert counts.shape[0] > 0 + assert jnp.all(counts >= 0) + + +class TestCountsEdgeCases: + """Test edge cases and error handling.""" + + def test_zero_infections(self, counts_process): + """Test with zero infections.""" + infections = jnp.zeros(20) + + with numpyro.handlers.seed(rng_seed=42): + counts = counts_process.sample( + infections=infections, + counts=None, + ) + + assert counts.shape[0] > 0 + + def test_small_infections(self, counts_process): + """Test with small infection values.""" + infections = jnp.ones(20) * 10 + + with numpyro.handlers.seed(rng_seed=42): + counts = counts_process.sample( + infections=infections, + counts=None, + ) + + assert counts.shape[0] > 0 + assert jnp.all(counts >= 0) + + def test_long_delay_distribution(self, counts_factory, long_delay_pmf): + """Test with longer delay distribution.""" + process = counts_factory.create(delay_pmf=long_delay_pmf) + + infections = create_mock_infections(40, peak_day=20, shape="spike") + + with numpyro.handlers.seed(rng_seed=42): + counts = process.sample( + infections=infections, + counts=None, + ) + + # Timeline alignment maintained + assert counts.shape[0] == infections.shape[0] + # Skip NaN padding: 10-day delay -> first 9 days are NaN + valid_counts = counts[9:] + assert jnp.sum(valid_counts) > 0 + + +class TestCountsSparseObservations: + """Test sparse observation support.""" + + def test_sparse_observations(self, counts_process): + """Test with sparse (irregular) observations.""" + n_days = 30 + infections = jnp.ones(n_days) * 100 + + # Sparse observations: only days 5, 10, 15, 20 + times = jnp.array([5, 10, 15, 20]) + counts_data = jnp.array([10, 12, 8, 15]) + + with numpyro.handlers.seed(rng_seed=42): + counts = counts_process.sample( + infections=infections, + counts=counts_data, + times=times, + ) + + assert counts.shape == times.shape + assert jnp.allclose(counts, counts_data) + + def test_sparse_vs_dense_sampling(self, counts_process): + """Test that sparse sampling gives different output shape than dense.""" + n_days = 30 + infections = jnp.ones(n_days) * 100 + + # Dense: prior sampling (counts=None, no times) + with numpyro.handlers.seed(rng_seed=42): + dense_counts = counts_process.sample( + infections=infections, + counts=None, + ) + + # Sparse with observed data: only some days + times = jnp.array([5, 10, 15, 20]) + sparse_obs_data = jnp.array([10, 12, 8, 15]) + with numpyro.handlers.seed(rng_seed=42): + sparse_counts = counts_process.sample( + infections=infections, + counts=sparse_obs_data, + times=times, + ) + + # Dense prior produces full length output + assert dense_counts.shape == (n_days,) + + # Sparse observations produce output matching times shape + assert sparse_counts.shape == times.shape + assert jnp.allclose(sparse_counts, sparse_obs_data) + + def test_prior_sampling_ignores_times(self, counts_process): + """Test that times parameter is ignored when counts=None (prior sampling).""" + n_days = 30 + infections = jnp.ones(n_days) * 100 + times = jnp.array([5, 10, 15, 20]) + + # When counts=None, times is ignored - output is dense + with numpyro.handlers.seed(rng_seed=42): + prior_with_times = counts_process.sample( + infections=infections, + counts=None, + times=times, + ) + + with numpyro.handlers.seed(rng_seed=42): + prior_without_times = counts_process.sample( + infections=infections, + counts=None, + ) + + # Both should produce dense output of shape (n_days,) + assert prior_with_times.shape == (n_days,) + assert prior_without_times.shape == (n_days,) + # With same seed, outputs should be identical + assert jnp.allclose(prior_with_times, prior_without_times) + + +class TestCountsBySite: + """Test CountsBySite for disaggregated observations.""" + + def test_sample_returns_correct_shape(self): + """Test that CountsBySite sample returns correct shape.""" + delay_pmf = jnp.array([0.3, 0.4, 0.3]) + process = CountsBySite( + ascertainment_rate_rv=DeterministicVariable("ihr", 0.02), + delay_distribution_rv=DeterministicPMF("delay", delay_pmf), + noise=PoissonNoise(), + ) + + infections = jnp.ones((30, 3)) * 500 # 30 days, 3 sites + times = jnp.array([10, 15, 10, 15]) + subpop_indices = jnp.array([0, 0, 1, 1]) + + with numpyro.handlers.seed(rng_seed=42): + counts = process.sample( + infections=infections, + subpop_indices=subpop_indices, + times=times, + counts=None, + ) + + assert counts.shape == times.shape + + def test_infection_resolution(self): + """Test that CountsBySite returns 'site' resolution.""" + delay_pmf = jnp.array([1.0]) + process = CountsBySite( + ascertainment_rate_rv=DeterministicVariable("ihr", 0.01), + delay_distribution_rv=DeterministicPMF("delay", delay_pmf), + noise=PoissonNoise(), + ) + + assert process.infection_resolution() == "site" + + +class TestPoissonNoise: + """Test PoissonNoise model.""" + + def test_poisson_counts(self, simple_delay_pmf): + """Test Counts with Poisson noise.""" + process = Counts( + ascertainment_rate_rv=DeterministicVariable("ihr", 0.01), + delay_distribution_rv=DeterministicPMF("delay", simple_delay_pmf), + noise=PoissonNoise(), + ) + + infections = jnp.ones(20) * 1000 + + with numpyro.handlers.seed(rng_seed=42): + counts = process.sample( + infections=infections, + counts=None, + ) + + assert counts.shape[0] == 20 + assert jnp.all(counts >= 0) + + +if __name__ == "__main__": + pytest.main([__file__, "-v"]) diff --git a/test/test_observation_measurements.py b/test/test_observation_measurements.py new file mode 100644 index 00000000..2ca11a22 --- /dev/null +++ b/test/test_observation_measurements.py @@ -0,0 +1,234 @@ +""" +Unit tests for Measurements (continuous measurement observations). + +These tests validate the measurement observation process base class implementation. +""" + +import jax.numpy as jnp +import numpyro +import pytest + +from pyrenew.deterministic import DeterministicPMF, DeterministicVariable +from pyrenew.observation import HierarchicalNormalNoise, Measurements +from pyrenew.observation.base import BaseObservationProcess +from pyrenew.randomvariable import GammaGroupSdPrior, HierarchicalNormalPrior + + +class ConcreteMeasurements(Measurements): + """Concrete implementation of Measurements for testing.""" + + def __init__(self, temporal_pmf_rv, noise, log10_scale=9.0): + """Initialize the concrete measurements for testing.""" + super().__init__(temporal_pmf_rv=temporal_pmf_rv, noise=noise) + self.log10_scale = log10_scale + + def validate(self) -> None: + """Validate parameters.""" + pmf = self.temporal_pmf_rv() + self._validate_pmf(pmf, "temporal_pmf_rv") + + def get_required_lookback(self) -> int: + """ + Return temporal PMF length. + + Returns + ------- + int + Length of the temporal PMF. + """ + return len(self.temporal_pmf_rv()) + + def _expected_signal(self, infections): + """ + Simple expected signal: log(convolution * scale). + + Returns + ------- + jnp.ndarray + Log-transformed expected signal. + """ + pmf = self.temporal_pmf_rv() + + # Handle 2D infections (n_days, n_sites) + if infections.ndim == 1: + infections = infections[:, jnp.newaxis] + + def convolve_col(col): # numpydoc ignore=GL08 + return self._convolve_with_alignment(col, pmf, 1.0)[0] + + import jax + + expected = jax.vmap(convolve_col, in_axes=1, out_axes=1)(infections) + + # Apply log10 scaling (simplified from wastewater model) + log_expected = jnp.log(expected + 1e-10) + self.log10_scale * jnp.log(10) + + return log_expected + + +class TestMeasurementsBase: + """Test Measurements abstract base class.""" + + def test_is_base_observation_process(self): + """Test that Measurements inherits from BaseObservationProcess.""" + assert issubclass(Measurements, BaseObservationProcess) + + def test_infection_resolution_is_site(self): + """Test that Measurements returns 'site' resolution.""" + shedding_pmf = jnp.array([0.3, 0.4, 0.3]) + site_mode_prior = HierarchicalNormalPrior( + name="mode", sd_rv=DeterministicVariable("sd", 0.5) + ) + site_sd_prior = GammaGroupSdPrior( + name="sd", + sd_mean_rv=DeterministicVariable("mean", 0.3), + sd_concentration_rv=DeterministicVariable("conc", 4.0), + ) + noise = HierarchicalNormalNoise(site_mode_prior, site_sd_prior) + + process = ConcreteMeasurements( + temporal_pmf_rv=DeterministicPMF("shedding", shedding_pmf), + noise=noise, + ) + + assert process.infection_resolution() == "site" + + +class TestHierarchicalNormalNoise: + """Test HierarchicalNormalNoise model.""" + + def test_sample_shape(self): + """Test that HierarchicalNormalNoise produces correct shape.""" + site_mode_prior = HierarchicalNormalPrior( + name="mode", sd_rv=DeterministicVariable("mode_sd", 0.5) + ) + site_sd_prior = GammaGroupSdPrior( + name="sd", + sd_mean_rv=DeterministicVariable("sd_mean", 0.3), + sd_concentration_rv=DeterministicVariable("sd_conc", 4.0), + ) + noise = HierarchicalNormalNoise(site_mode_prior, site_sd_prior) + + expected = jnp.array([1.0, 2.0, 3.0, 4.0]) + site_indices = jnp.array([0, 0, 1, 1]) + + with numpyro.handlers.seed(rng_seed=42): + samples = noise.sample( + name="test", + expected=expected, + obs=None, + site_indices=site_indices, + n_sites=2, + ) + + assert samples.shape == expected.shape + + def test_sample_with_observations(self): + """Test that HierarchicalNormalNoise conditions on observations.""" + site_mode_prior = HierarchicalNormalPrior( + name="mode", sd_rv=DeterministicVariable("mode_sd", 0.5) + ) + site_sd_prior = GammaGroupSdPrior( + name="sd", + sd_mean_rv=DeterministicVariable("sd_mean", 0.3), + sd_concentration_rv=DeterministicVariable("sd_conc", 4.0), + ) + noise = HierarchicalNormalNoise(site_mode_prior, site_sd_prior) + + expected = jnp.array([1.0, 2.0, 3.0, 4.0]) + obs = jnp.array([1.1, 2.1, 3.1, 4.1]) + site_indices = jnp.array([0, 0, 1, 1]) + + with numpyro.handlers.seed(rng_seed=42): + samples = noise.sample( + name="test", + expected=expected, + obs=obs, + site_indices=site_indices, + n_sites=2, + ) + + # When obs is provided, samples should equal obs + assert jnp.allclose(samples, obs) + + +class TestConcreteMeasurements: + """Test concrete Measurements implementation.""" + + def test_sample_shape(self): + """Test that sample returns correct shape.""" + shedding_pmf = jnp.array([0.3, 0.4, 0.3]) + site_mode_prior = HierarchicalNormalPrior( + name="mode", sd_rv=DeterministicVariable("mode_sd", 0.5) + ) + site_sd_prior = GammaGroupSdPrior( + name="sd", + sd_mean_rv=DeterministicVariable("sd_mean", 0.3), + sd_concentration_rv=DeterministicVariable("sd_conc", 4.0), + ) + noise = HierarchicalNormalNoise(site_mode_prior, site_sd_prior) + + process = ConcreteMeasurements( + temporal_pmf_rv=DeterministicPMF("shedding", shedding_pmf), + noise=noise, + ) + + # 30 days, 2 sites + infections = jnp.ones((30, 2)) * 1000 + subpop_indices = jnp.array([0, 0, 1, 1]) + site_indices = jnp.array([0, 0, 1, 1]) + times = jnp.array([10, 15, 10, 15]) + + with numpyro.handlers.seed(rng_seed=42): + samples = process.sample( + infections=infections, + subpop_indices=subpop_indices, + site_indices=site_indices, + times=times, + concentrations=None, + n_sites=2, + ) + + assert samples.shape == times.shape + + def test_expected_signal_stored(self): + """Test that expected_log_conc is stored as deterministic.""" + shedding_pmf = jnp.array([0.5, 0.5]) + site_mode_prior = HierarchicalNormalPrior( + name="mode", sd_rv=DeterministicVariable("mode_sd", 0.01) + ) + site_sd_prior = GammaGroupSdPrior( + name="sd", + sd_mean_rv=DeterministicVariable("sd_mean", 0.01), + sd_concentration_rv=DeterministicVariable("sd_conc", 4.0), + sd_min=0.001, + ) + noise = HierarchicalNormalNoise(site_mode_prior, site_sd_prior) + + process = ConcreteMeasurements( + temporal_pmf_rv=DeterministicPMF("shedding", shedding_pmf), + noise=noise, + ) + + infections = jnp.ones((20, 2)) * 1000 + subpop_indices = jnp.array([0, 1]) + site_indices = jnp.array([0, 1]) + times = jnp.array([10, 10]) + + with numpyro.handlers.seed(rng_seed=42): + trace = numpyro.handlers.trace( + lambda: process.sample( + infections=infections, + subpop_indices=subpop_indices, + site_indices=site_indices, + times=times, + concentrations=None, + n_sites=2, + ) + ).get_trace() + + assert "expected_log_conc" in trace + + +if __name__ == "__main__": + pytest.main([__file__, "-v"]) diff --git a/test/test_randomvariable_hierarchical.py b/test/test_randomvariable_hierarchical.py new file mode 100644 index 00000000..30384800 --- /dev/null +++ b/test/test_randomvariable_hierarchical.py @@ -0,0 +1,179 @@ +"""Unit tests for hierarchical prior distributions.""" + +import jax.numpy as jnp +import numpyro +import numpyro.distributions as dist +import pytest + +from pyrenew.deterministic import DeterministicVariable +from pyrenew.randomvariable import ( + DistributionalVariable, + GammaGroupSdPrior, + HierarchicalNormalPrior, + StudentTGroupModePrior, +) + + +class TestHierarchicalNormalPrior: + """Test HierarchicalNormalPrior.""" + + def test_sample_shape(self): + """Test that sample returns correct shape.""" + prior = HierarchicalNormalPrior( + "effect", sd_rv=DeterministicVariable("sd", 1.0) + ) + + with numpyro.handlers.seed(rng_seed=42): + samples = prior.sample(n_groups=5) + + assert samples.shape == (5,) + + def test_smaller_sd_produces_tighter_distribution(self): + """Test that smaller sd produces samples closer to zero.""" + prior_tight = HierarchicalNormalPrior( + "a", sd_rv=DeterministicVariable("sd_tight", 0.1) + ) + prior_wide = HierarchicalNormalPrior( + "b", sd_rv=DeterministicVariable("sd_wide", 10.0) + ) + + n_samples = 1000 + with numpyro.handlers.seed(rng_seed=42): + samples_tight = prior_tight.sample(n_groups=n_samples) + with numpyro.handlers.seed(rng_seed=43): + samples_wide = prior_wide.sample(n_groups=n_samples) + + # Tight prior should have smaller standard deviation + assert jnp.std(samples_tight) < jnp.std(samples_wide) + + def test_rejects_non_random_variable_sd(self): + """Test that non-RandomVariable sd_rv is rejected.""" + with pytest.raises(TypeError, match="sd_rv must be a RandomVariable"): + HierarchicalNormalPrior("effect", sd_rv=1.0) + + def test_accepts_distributional_variable_for_sd(self): + """Test that DistributionalVariable can be used for sd_rv.""" + sd_rv = DistributionalVariable("sd", dist.HalfNormal(1.0)) + prior = HierarchicalNormalPrior("effect", sd_rv=sd_rv) + + with numpyro.handlers.seed(rng_seed=42): + samples = prior.sample(n_groups=5) + + assert samples.shape == (5,) + + +class TestGammaGroupSdPrior: + """Test GammaGroupSdPrior.""" + + def test_sample_shape(self): + """Test that sample returns correct shape.""" + prior = GammaGroupSdPrior( + "sd", + sd_mean_rv=DeterministicVariable("sd_mean", 0.5), + sd_concentration_rv=DeterministicVariable("sd_conc", 4.0), + ) + + with numpyro.handlers.seed(rng_seed=42): + samples = prior.sample(n_groups=5) + + assert samples.shape == (5,) + + def test_respects_sd_min(self): + """Test that sd_min is enforced as lower bound.""" + prior = GammaGroupSdPrior( + "sd", + sd_mean_rv=DeterministicVariable("sd_mean", 0.1), + sd_concentration_rv=DeterministicVariable("sd_conc", 4.0), + sd_min=0.5, + ) + + with numpyro.handlers.seed(rng_seed=42): + samples = prior.sample(n_groups=100) + + assert jnp.all(samples >= 0.5) + + def test_rejects_non_random_variable_params(self): + """Test that non-RandomVariable parameters are rejected.""" + with pytest.raises(TypeError, match="sd_mean_rv must be a RandomVariable"): + GammaGroupSdPrior( + "sd", + sd_mean_rv=0.5, + sd_concentration_rv=DeterministicVariable("sd_conc", 4.0), + ) + + with pytest.raises( + TypeError, match="sd_concentration_rv must be a RandomVariable" + ): + GammaGroupSdPrior( + "sd", + sd_mean_rv=DeterministicVariable("sd_mean", 0.5), + sd_concentration_rv=4.0, + ) + + def test_rejects_negative_sd_min(self): + """Test that negative sd_min is rejected.""" + with pytest.raises(ValueError, match="sd_min must be non-negative"): + GammaGroupSdPrior( + "sd", + sd_mean_rv=DeterministicVariable("sd_mean", 0.5), + sd_concentration_rv=DeterministicVariable("sd_conc", 4.0), + sd_min=-0.1, + ) + + +class TestStudentTGroupModePrior: + """Test StudentTGroupModePrior.""" + + def test_sample_shape(self): + """Test that sample returns correct shape.""" + prior = StudentTGroupModePrior( + "mode", + sd_rv=DeterministicVariable("sd", 1.0), + df_rv=DeterministicVariable("df", 4.0), + ) + + with numpyro.handlers.seed(rng_seed=42): + samples = prior.sample(n_groups=5) + + assert samples.shape == (5,) + + def test_heavier_tails_than_normal(self): + """Test Student-t produces more extreme values than Normal.""" + # df=2 gives very heavy tails + student_prior = StudentTGroupModePrior( + "s", + sd_rv=DeterministicVariable("sd_s", 1.0), + df_rv=DeterministicVariable("df", 2.0), + ) + normal_prior = HierarchicalNormalPrior( + "n", sd_rv=DeterministicVariable("sd_n", 1.0) + ) + + n_samples = 5000 + with numpyro.handlers.seed(rng_seed=42): + student_samples = student_prior.sample(n_groups=n_samples) + with numpyro.handlers.seed(rng_seed=42): + normal_samples = normal_prior.sample(n_groups=n_samples) + + # Student-t should have more extreme values (higher max absolute value) + assert jnp.max(jnp.abs(student_samples)) > jnp.max(jnp.abs(normal_samples)) + + def test_rejects_non_random_variable_params(self): + """Test that non-RandomVariable parameters are rejected.""" + with pytest.raises(TypeError, match="sd_rv must be a RandomVariable"): + StudentTGroupModePrior( + "mode", + sd_rv=1.0, + df_rv=DeterministicVariable("df", 4.0), + ) + + with pytest.raises(TypeError, match="df_rv must be a RandomVariable"): + StudentTGroupModePrior( + "mode", + sd_rv=DeterministicVariable("sd", 1.0), + df_rv=4.0, + ) + + +if __name__ == "__main__": + pytest.main([__file__, "-v"]) From 89d7aab5fcca52ecb52432e8413aace4df5d7f11 Mon Sep 17 00:00:00 2001 From: Mitzi Morris Date: Tue, 23 Dec 2025 10:56:50 -0500 Subject: [PATCH 02/81] adding tutorials --- docs/tutorials/.pages | 2 + .../observation_processes_counts.qmd | 425 +++++++++++++ .../observation_processes_measurements.qmd | 569 ++++++++++++++++++ pyrenew/observation/__init__.py | 19 + pyrenew/observation/base.py | 8 + pyrenew/observation/noise.py | 12 + 6 files changed, 1035 insertions(+) create mode 100644 docs/tutorials/observation_processes_counts.qmd create mode 100644 docs/tutorials/observation_processes_measurements.qmd diff --git a/docs/tutorials/.pages b/docs/tutorials/.pages index 875214fc..4f1079b3 100644 --- a/docs/tutorials/.pages +++ b/docs/tutorials/.pages @@ -3,5 +3,7 @@ nav: - basic_renewal_model.md - custom_randomvariables.md - hospital_admissions_model.md + - observation_processes_counts.md + - observation_processes_measurements.md - day_of_the_week.md - periodic_effects.md diff --git a/docs/tutorials/observation_processes_counts.qmd b/docs/tutorials/observation_processes_counts.qmd new file mode 100644 index 00000000..83279331 --- /dev/null +++ b/docs/tutorials/observation_processes_counts.qmd @@ -0,0 +1,425 @@ +--- +title: "Observation processes for count data" +format: gfm +engine: jupyter +--- + +This tutorial demonstrates how to use the `Counts` observation process to model count data such as hospital admissions, emergency department visits, or deaths. + +```{python} +# | label: setup +# | output: false +import jax.numpy as jnp +import numpy as np +import numpyro +import matplotlib.pyplot as plt + +from pyrenew.observation import Counts, NegativeBinomialNoise, PoissonNoise +from pyrenew.deterministic import DeterministicVariable, DeterministicPMF +from pyrenew import datasets +``` + +## Overview + +Count observation processes model the lag between infections and an observed outcome such as hospital admissions, emergency department visits, confirmed cases, or deaths. +Observed data can be aggregated at the jurisdiction level or available as disaggregated counts from reporting entities, which are modeled by classes `Counts` and `CountsBySite`, respectively. + +Count observation processes transform infections into expected observed counts by applying an ascertainment rate and convolving with a delay distribution. + +The expected observations on day $t$ are: + +$$\lambda_t = \alpha \cdot \sum_{d=0}^{D} I_{t-d} \cdot p_d$$ + +where: + +- $I_{t-d}$ is the number of incident (new) infections on day $t-d$ (i.e., $d$ days before day $t$) +- $\alpha$ is the ascertainment rate (e.g., infection-hospitalization ratio) +- $p_d$ is the delay distribution from infection to observation, conditional on an infection leading to an observation +- $D$ is the maximum delay + +Discrete observations are generated by sampling from a noise distribution—either Poisson or negative binomial—to model reporting variability. +Poisson assumes variance equals the mean; negative binomial accommodates the overdispersion common in surveillance data. + +**Note on terminology:** In real-world inference, infections are *latent* (unobserved) and must be estimated from observed data like hospital admissions. In this tutorial, we simulate the observation process by specifying infections directly and showing how they produce hospital admissions through convolution and sampling. + +## Hospital admissions example + +For hospital admissions data, we construct a `Counts` observation process. +The delay is the key mechanism: infections from $d$ days ago ($I_{t-d}$) contribute to today's hospital admissions ($\lambda_t$) weighted by the probability ($p_d$) that an infection leads to hospitalization after exactly $d$ days. The convolution sums these contributions across all past days. + +The process generates hospital admissions by sampling from a negative binomial distribution: +$$Y_t \sim \text{NegativeBinomial}(\mu = \lambda_t, \text{concentration} = \phi)$$ + +The concentration parameter $\phi$ (sometimes called $k$ or the dispersion parameter) controls overdispersion: as $\phi \to \infty$, the distribution approaches Poisson; smaller values allow greater overdispersion. + +We use the negative binomial distribution because real-world hospital admission counts exhibit overdispersion—the variance exceeds the mean. +The Poisson distribution assumes variance equals the mean, which is too restrictive. The negative binomial adds an overdispersion term: +$$\text{Var}[Y_t] = \mu + \frac{\mu^2}{\phi}$$ + +In this example, we use fixed parameter values for illustration; in practice, these parameters would be estimated from data using weakly informative priors. + +## Infection-to-hospitalization delay distribution + +The delay distribution specifies the probability that an infected person is hospitalized $d$ days after infection, conditional on the infection leading to a hospitalization. +For example, if `hosp_delay_pmf[5] = 0.2`, then 20% of infections that result in hospitalization will appear as hospital admissions 5 days after infection. + +We load a delay distribution from PyRenew's datasets: + +```{python} +# | label: load-delay +inf_hosp_int = datasets.load_infection_admission_interval() +hosp_delay_pmf = jnp.array(inf_hosp_int["probability_mass"].to_numpy()) + +delay_rv = DeterministicPMF("inf_to_hosp_delay", hosp_delay_pmf) + +# Summary statistics +days = np.arange(len(hosp_delay_pmf)) +mean_delay = float(np.sum(days * hosp_delay_pmf)) +mode_delay = int(np.argmax(hosp_delay_pmf)) +print(f"Mode delay: {mode_delay} days, Mean delay: {mean_delay:.1f} days") +``` + +```{python} +# | label: fig-delay-distribution +# | fig-cap: Infection-to-hospitalization delay distribution +fig, ax = plt.subplots(figsize=(8, 4)) +ax.bar(days, hosp_delay_pmf, color="steelblue", alpha=0.7, edgecolor="black") +ax.axvline( + mode_delay, + color="purple", + linestyle="-", + linewidth=2, + label=f"Mode: {mode_delay}", +) +ax.axvline( + mean_delay, + color="red", + linestyle="--", + linewidth=2, + label=f"Mean: {mean_delay:.1f}", +) +ax.set_xlabel("Days from infection to hospitalization") +ax.set_ylabel("Probability") +ax.set_title("Infection-to-Hospitalization Delay Distribution") +ax.legend() +plt.tight_layout() +plt.show() +``` + +## Creating a Counts observation process + +A `Counts` object takes the following arguments: + +- **`ascertainment_rate_rv`**: the probability an infection results in an observation (e.g., IHR) +- **`delay_distribution_rv`**: delay distribution from infection to observation (PMF) +- **`noise`**: noise model (`PoissonNoise()` or `NegativeBinomialNoise(concentration_rv)`) + +```{python} +# | label: create-counts-process +# Infection-hospitalization ratio (1% of infections lead to hospitalization) +ihr_rv = DeterministicVariable("ihr", 0.01) + +# Overdispersion parameter for negative binomial +concentration_rv = DeterministicVariable("concentration", 10.0) + +# Create the observation process +hosp_process = Counts( + ascertainment_rate_rv=ihr_rv, + delay_distribution_rv=delay_rv, + noise=NegativeBinomialNoise(concentration_rv), +) + +print(f"Required lookback: {hosp_process.get_required_lookback()} days") +``` + +### Timeline alignment and lookback period + +The observation process convolves infections with a delay distribution, maintaining alignment between input and output: day $t$ in the output corresponds to day $t$ in the input. + +Hospital admissions depend on infections from prior days (the length of our delay distribution minus one). The method `get_required_lookback()` returns this length; the first valid observation day is at index `lookback - 1`. Earlier days are marked invalid. + +```{python} +# | label: helper-function +def first_valid_observation_day(obs_process) -> int: + """Return the first day index with complete infection history for convolution.""" + return obs_process.get_required_lookback() - 1 +``` + +## Simulating hospital admissions from infections + +To demonstrate how the observation process works, we simulate admissions from a spike of infections on a single day. + +```{python} +# | label: simulate-spike +n_days = 100 +lookback = hosp_process.get_required_lookback() + +# First valid observation day +day_one = lookback - 1 + +# Create infections with a spike +infection_spike_day = day_one + 10 +infections = jnp.zeros(n_days) +infections = infections.at[infection_spike_day].set(2000) + +# Sample hospital admissions +with numpyro.handlers.seed(rng_seed=42): + hosp_admissions = hosp_process.sample( + infections=infections, + counts=None, # Sample from prior (no observed data) + ) +``` + +```{python} +# | label: fig-spike-infections +# | fig-cap: Input infections with a single-day spike +# Plot relative to first valid observation day +spike_day = infection_spike_day - day_one +n_plot_days = n_days - day_one + +fig, ax = plt.subplots(figsize=(8, 4)) +ax.plot( + np.arange(n_plot_days), + np.array(infections[day_one:]), + "-o", + color="darkblue", +) +ax.axvline(spike_day, color="darkred", linestyle="--", alpha=0.7) +ax.annotate( + f"Infection spike\n(day {spike_day})", + xy=(spike_day, 1800), + xytext=(spike_day + 5, 1800), + fontsize=10, + color="darkred", +) +ax.set_xlabel("Day") +ax.set_ylabel("Daily Infections") +ax.set_title("Infections (Input)") +plt.tight_layout() +plt.show() +``` + +Because all infections occur on a single day, we can see how they spread into hospital admissions over subsequent days according to the delay distribution: + +```{python} +# | label: fig-spike-admissions +# | fig-cap: Hospital admissions from a single-day infection spike +fig, ax = plt.subplots(figsize=(8, 4)) +ax.plot( + np.arange(n_plot_days), + np.array(hosp_admissions[day_one:]), + "-o", + color="purple", +) +ax.axvline( + spike_day, + color="darkred", + linestyle="--", + alpha=0.5, + label="Infection spike", +) +ax.axvline( + spike_day + mode_delay, + color="purple", + linestyle="--", + alpha=0.5, + label="Peak admissions", +) +ax.set_xlabel("Day") +ax.set_ylabel("Hospital Admissions") +ax.set_title("Hospital Admissions (Output)") +ax.legend() +plt.tight_layout() +plt.show() +``` + +The admissions peak occurs `{python} mode_delay` days after the infection spike, matching the mode of the delay distribution. + +## Observation noise + +The negative binomial distribution adds stochastic variation. Sampling multiple times from the same infections shows the range of possible observations: + +```{python} +# | label: sample-realizations +n_samples = 50 +samples = [] + +for seed in range(n_samples): + with numpyro.handlers.seed(rng_seed=seed): + sample = hosp_process.sample(infections=infections, counts=None) + samples.append(np.array(sample[day_one:])) + +samples = np.array(samples) +sample_mean = samples.mean(axis=0) +``` + +```{python} +# | label: fig-sampled-admissions +# | fig-cap: Multiple realizations showing observation noise +fig, ax = plt.subplots(figsize=(8, 4)) + +# Plot all samples in light orange +for i in range(1, n_samples): + ax.plot( + np.arange(n_plot_days), + samples[i], + color="orange", + alpha=0.15, + linewidth=0.5, + ) + +# Highlight one sample +ax.plot( + np.arange(n_plot_days), + samples[0], + color="steelblue", + linewidth=1, + label="One realization", +) + +# Sample mean +ax.plot( + np.arange(n_plot_days), + sample_mean, + color="darkred", + linewidth=1.2, + label="Sample mean", +) + +ax.axvline(spike_day, color="darkblue", linestyle="--", alpha=0.5) +ax.set_xlabel("Day") +ax.set_ylabel("Hospital Admissions") +ax.set_title(f"Observation Noise: {n_samples} Samples from Same Infections") +ax.legend() +plt.tight_layout() +plt.show() +``` + +```{python} +# | label: timeline-stats +# Print timeline statistics +print("Timeline Analysis:") +print( + f" Infection spike on day {spike_day}: {infections[infection_spike_day]:.0f} people" +) +print(f" Mode delay from infection to hospitalization: {mode_delay} days") +print( + f" Expected hospitalization peak: day {spike_day + mode_delay} (= {spike_day} + {mode_delay})" +) +``` + +## Effect of the ascertainment rate + +The ascertainment rate directly scales expected hospital admissions. We compare IHR values of 0.5% and 2.5%: + +```{python} +# | label: compare-ihr +ihr_values = [0.005, 0.025] +infections_decay = 3000 * jnp.exp(-jnp.arange(n_days) / 20.0) + +results = {} +for ihr_val in ihr_values: + process = Counts( + ascertainment_rate_rv=DeterministicVariable("ihr", ihr_val), + delay_distribution_rv=delay_rv, + noise=NegativeBinomialNoise(concentration_rv), + ) + # Average over multiple samples to show the effect on the mean + samples = [] + for seed in range(20): + with numpyro.handlers.seed(rng_seed=seed): + sample = process.sample(infections=infections_decay, counts=None) + samples.append(np.array(sample[day_one:])) + results[ihr_val] = np.mean(samples, axis=0) +``` + +```{python} +# | label: fig-ihr-comparison +# | fig-cap: Effect of different infection-hospitalization rates +fig, ax = plt.subplots(figsize=(8, 4)) +colors = ["steelblue", "darkred"] + +for (ihr_val, mean_sample), color in zip(results.items(), colors): + ax.plot( + np.arange(n_plot_days), + mean_sample, + color=color, + linewidth=1.5, + label=f"IHR = {ihr_val:.1%}", + ) + +ax.set_xlabel("Day") +ax.set_ylabel("Hospital Admissions (mean of samples)") +ax.set_title("Effect of IHR on Hospital Admissions") +ax.legend() +plt.tight_layout() +plt.show() +``` + +## Negative binomial concentration parameter + +The concentration parameter $\phi$ controls overdispersion: + +- Higher $\phi$ → less overdispersion (approaches Poisson) +- Lower $\phi$ → more overdispersion (noisier data) + +```{python} +# | label: fig-concentration-comparison +# | fig-cap: Effect of concentration parameter on variability +infections_constant = 2000 * jnp.ones(n_days) +concentration_values = [1.0, 10.0, 100.0] +n_replicates = 10 + +fig, axes = plt.subplots(1, 3, figsize=(12, 4), sharey=True) + +for ax, conc_val in zip(axes, concentration_values): + process = Counts( + ascertainment_rate_rv=ihr_rv, + delay_distribution_rv=delay_rv, + noise=NegativeBinomialNoise(DeterministicVariable("conc", conc_val)), + ) + + for seed in range(n_replicates): + with numpyro.handlers.seed(rng_seed=seed): + sample = process.sample( + infections=infections_constant, counts=None + ) + ax.plot( + np.arange(n_plot_days), + np.array(sample[day_one:]), + color="steelblue", + alpha=0.5, + linewidth=0.8, + ) + + ax.set_xlabel("Day") + ax.set_title(f"φ = {int(conc_val)}") + +axes[0].set_ylabel("Hospital Admissions") +fig.suptitle("Effect of Concentration Parameter on Variability") +plt.tight_layout() +plt.show() +``` + +## Swapping noise models + +To use Poisson noise instead of negative binomial, change the noise model: + +```{python} +# | label: poisson-noise +hosp_process_poisson = Counts( + ascertainment_rate_rv=ihr_rv, + delay_distribution_rv=delay_rv, + noise=PoissonNoise(), +) + +with numpyro.handlers.seed(rng_seed=42): + poisson_sample = hosp_process_poisson.sample( + infections=infections, + counts=None, + ) + +print( + f"Sampled {len(poisson_sample)} days of hospital admissions with Poisson noise" +) +``` diff --git a/docs/tutorials/observation_processes_measurements.qmd b/docs/tutorials/observation_processes_measurements.qmd new file mode 100644 index 00000000..96138d0a --- /dev/null +++ b/docs/tutorials/observation_processes_measurements.qmd @@ -0,0 +1,569 @@ +--- +title: "Observation processes for continuous measurements" +format: gfm +engine: jupyter +--- + +This tutorial demonstrates how to use the `Measurements` observation process to model continuous measurement data such as wastewater viral concentrations. + +```{python} +# | label: setup +# | output: false +import jax +import jax.numpy as jnp +import numpy as np +import numpyro +import matplotlib.pyplot as plt + +from pyrenew.observation import Measurements, HierarchicalNormalNoise +from pyrenew.randomvariable import HierarchicalNormalPrior, GammaGroupSdPrior +from pyrenew.deterministic import DeterministicVariable, DeterministicPMF +``` + +## Overview + +Measurement observation processes model continuous signals derived from infections, such as viral RNA concentrations in wastewater. Unlike count observations (hospital admissions, deaths), measurements are continuous values that span orders of magnitude. + +The expected measurement on day $t$ is: + +$$\lambda_t = \frac{G}{V} \cdot \sum_{d=0}^{D} I_{t-d} \cdot p_d$$ + +where: + +- $I_{t-d}$ is the number of incident (new) infections on day $t-d$ +- $G$ is a scaling factor (e.g., genome copies shed per infection) +- $V$ is a normalization factor (e.g., wastewater volume per person per day) +- $p_d$ is the temporal distribution (e.g., viral shedding kinetics) +- $D$ is the maximum duration + +Observed log-concentrations are generated by sampling from a normal distribution: + +$$\log(C_t) \sim \text{Normal}(\mu = \log(\lambda_t), \sigma)$$ + +The log-normal distribution is appropriate for concentration data because concentrations are strictly positive and often span several orders of magnitude. + +### Comparison with count observations + +The core convolution $\sum_{d=0}^{D} I_{t-d} \cdot p_d$ is the same as for count observations. The key differences are: + +| Aspect | Counts | Measurements | +|--------|--------|--------------| +| Scaling factor | Ascertainment rate $\alpha \in [0,1]$ | Domain-specific (e.g., $G/V$) | +| Temporal PMF | Delay distribution | Shedding/decay kinetics | +| Output space | Expected counts (linear) | Log-concentrations | +| Noise model | Poisson or Negative Binomial | Normal on log scale | +| Site structure | Optional (`CountsBySite`) | Inherent (hierarchical effects) | + +**Key features of measurement data:** + +- **Multiple sites**: Each jurisdiction has several measurement sites (e.g., wastewater treatment plants). +- **Irregular sampling**: Sites measure on different schedules. +- **Site-level variability**: Lab protocols and sampling methods vary across sites. +- **Temporal lag**: Signal peaks several days after infection. + +**Note on terminology:** In real-world inference, infections are *latent* (unobserved) and must be estimated from observed data. In this tutorial, we simulate the observation process by specifying infections directly and showing how they produce measurements through convolution and sampling. + +## Subclassing Measurements for wastewater + +The `Measurements` class is abstract—you must subclass it and implement `_expected_signal()` for your specific signal type. Here we create a `Wastewater` class for viral concentration measurements: + +```{python} +# | label: wastewater-class +from jax.typing import ArrayLike +from pyrenew.metaclass import RandomVariable +from pyrenew.observation.noise import MeasurementNoise + + +class Wastewater(Measurements): + """ + Wastewater viral concentration observation process. + + Transforms site-level infections into expected log-concentrations + via shedding kinetics convolution and genome/volume scaling. + """ + + def __init__( + self, + shedding_kinetics_rv: RandomVariable, + log10_genome_per_infection_rv: RandomVariable, + ml_per_person_per_day: float, + noise: MeasurementNoise, + ) -> None: + """ + Initialize wastewater observation process. + + Parameters + ---------- + shedding_kinetics_rv : RandomVariable + Viral shedding PMF (fraction shed each day post-infection). + log10_genome_per_infection_rv : RandomVariable + Log10 genome copies shed per infection. + ml_per_person_per_day : float + Wastewater volume per person per day (mL). + noise : MeasurementNoise + Noise model (e.g., HierarchicalNormalNoise). + """ + super().__init__(temporal_pmf_rv=shedding_kinetics_rv, noise=noise) + self.log10_genome_per_infection_rv = log10_genome_per_infection_rv + self.ml_per_person_per_day = ml_per_person_per_day + + def validate(self) -> None: + """Validate parameters.""" + shedding_pmf = self.temporal_pmf_rv() + self._validate_pmf(shedding_pmf, "shedding_kinetics_rv") + self.noise.validate() + + def get_required_lookback(self) -> int: + """Return shedding PMF length.""" + return len(self.temporal_pmf_rv()) + + def _expected_signal(self, infections: ArrayLike) -> ArrayLike: + """ + Compute expected log-concentration from infections. + + Applies shedding kinetics convolution, then scales by + genome copies and volume to get concentration. + """ + shedding_pmf = self.temporal_pmf_rv() + log10_genome = self.log10_genome_per_infection_rv() + + # Convolve each site's infections with shedding kinetics + def convolve_site(site_infections): + convolved, _ = self._convolve_with_alignment( + site_infections, shedding_pmf, p_observed=1.0 + ) + return convolved + + # Apply to all sites (infections shape: n_days x n_sites) + shedding_signal = jax.vmap(convolve_site, in_axes=1, out_axes=1)( + infections + ) + + # Convert to concentration: genomes per mL + genome_copies = 10**log10_genome + concentration = ( + shedding_signal * genome_copies / self.ml_per_person_per_day + ) + + # Return log-concentration (what we model) + return jnp.log(concentration) +``` + +## Viral shedding kinetics + +The shedding PMF describes what fraction of total viral shedding occurs on each day after infection: + +```{python} +# | label: shedding-pmf +# Peak shedding ~3 days after infection, continues for ~10 days +shedding_pmf = jnp.array( + [0.0, 0.05, 0.15, 0.25, 0.20, 0.15, 0.10, 0.05, 0.03, 0.02] +) +print(f"PMF sums to: {shedding_pmf.sum():.2f}") + +shedding_rv = DeterministicPMF("viral_shedding", shedding_pmf) + +# Summary statistics +days = np.arange(len(shedding_pmf)) +mean_shedding_day = float(np.sum(days * shedding_pmf)) +mode_shedding_day = int(np.argmax(shedding_pmf)) +print(f"Mode: {mode_shedding_day} days, Mean: {mean_shedding_day:.1f} days") +``` + +```{python} +# | label: fig-shedding +# | fig-cap: Viral shedding kinetics distribution +fig, ax = plt.subplots(figsize=(8, 4)) +ax.bar(days, shedding_pmf, color="steelblue", alpha=0.7, edgecolor="black") +ax.axvline( + mode_shedding_day, + color="purple", + linestyle="-", + linewidth=2, + label=f"Mode: {mode_shedding_day}", +) +ax.axvline( + mean_shedding_day, + color="red", + linestyle="--", + linewidth=2, + label=f"Mean: {mean_shedding_day:.1f}", +) +ax.set_xlabel("Days after infection") +ax.set_ylabel("Fraction of total shedding") +ax.set_title("Viral Shedding Kinetics") +ax.legend() +plt.tight_layout() +plt.show() +``` + +## Genome copies and wastewater volume + +```{python} +# | label: scaling-params +# Log10 genome copies shed per infection (typical: 8-10) +log10_genome_rv = DeterministicVariable("log10_genome", 9.0) + +# Wastewater volume per person per day (mL) +ml_per_person_per_day = 1000.0 +``` + +## Noise model with site-level effects + +Different sites have systematic differences in measured concentrations due to lab protocols, sampling methods, and flow estimation. We model this with hierarchical site effects: + +```{python} +# | label: noise-model +# Site-level mode: systematic differences between sites +site_mode_prior = HierarchicalNormalPrior( + name="ww_site_mode", + sd_rv=DeterministicVariable("mode_sd", 0.5), +) + +# Site-level SD: measurement variability within each site +site_sd_prior = GammaGroupSdPrior( + name="ww_site_sd", + sd_mean_rv=DeterministicVariable("sd_mean", 0.3), + sd_concentration_rv=DeterministicVariable("sd_concentration", 4.0), + sd_min=0.10, +) + +# Create the noise model +ww_noise = HierarchicalNormalNoise( + site_mode_prior_rv=site_mode_prior, + site_sd_prior_rv=site_sd_prior, +) +``` + +## Creating the observation process + +```{python} +# | label: create-process +ww_process = Wastewater( + shedding_kinetics_rv=shedding_rv, + log10_genome_per_infection_rv=log10_genome_rv, + ml_per_person_per_day=ml_per_person_per_day, + noise=ww_noise, +) + +print(f"Required lookback: {ww_process.get_required_lookback()} days") +``` + +### Timeline alignment and lookback period + +The observation process convolves infections with a shedding distribution, maintaining alignment between input and output: day $t$ in the output corresponds to day $t$ in the input. + +Wastewater concentrations depend on infections from prior days (the length of our shedding distribution minus one). The method `get_required_lookback()` returns this length; the first valid observation day is at index `lookback - 1`. Earlier days are marked invalid with NaN. + +```{python} +# | label: helper-function +def first_valid_observation_day(obs_process) -> int: + """Return the first day index with complete infection history for convolution.""" + return obs_process.get_required_lookback() - 1 +``` + +## Simulating wastewater observations from infections + +To demonstrate how the observation process works, we simulate concentrations from a spike of infections on a single day. + +```{python} +# | label: simulate-spike +n_days = 50 + +# First valid observation day +day_one = first_valid_observation_day(ww_process) + +# Create infections with a spike (shape: n_days x n_sites) +infection_spike_day = day_one + 10 +infections = jnp.zeros((n_days, 1)) # 1 subpopulation +infections = infections.at[infection_spike_day, 0].set(2000.0) + +# For plotting +spike_day = infection_spike_day - day_one +n_plot_days = n_days - day_one + +# Observation times and indices +observation_days = jnp.arange(day_one, 40, dtype=jnp.int32) +n_obs = len(observation_days) + +with numpyro.handlers.seed(rng_seed=42): + ww_obs = ww_process.sample( + infections=infections, + subpop_indices=jnp.zeros(n_obs, dtype=jnp.int32), + site_indices=jnp.zeros(n_obs, dtype=jnp.int32), + times=observation_days, + concentrations=None, # Sample from prior + n_sites=1, + ) +``` + +```{python} +# | label: fig-spike-infections +# | fig-cap: Input infections with a single-day spike +fig, ax = plt.subplots(figsize=(8, 4)) +ax.plot( + np.arange(n_plot_days), + np.array(infections[day_one:, 0]), + "-o", + color="darkblue", +) +ax.axvline(spike_day, color="darkred", linestyle="--", alpha=0.7) +ax.annotate( + f"Infection spike\n(day {spike_day})", + xy=(spike_day, 1800), + xytext=(spike_day + 3, 1800), + fontsize=10, + color="darkred", +) +ax.set_xlabel("Day") +ax.set_ylabel("Daily Infections") +ax.set_title("Infections (Input)") +plt.tight_layout() +plt.show() +``` + +Because all infections occur on a single day, we can see how they spread into wastewater concentrations over subsequent days according to the shedding kinetics. + +## Observation noise + +The log-normal noise model adds stochastic variation. Sampling multiple times from the same infections shows the range of possible observations: + +```{python} +# | label: sample-realizations +n_samples = 50 +samples_by_day = {int(d - day_one): [] for d in observation_days} + +for seed in range(n_samples): + with numpyro.handlers.seed(rng_seed=seed): + ww_sample = ww_process.sample( + infections=infections, + subpop_indices=jnp.zeros(n_obs, dtype=jnp.int32), + site_indices=jnp.zeros(n_obs, dtype=jnp.int32), + times=observation_days, + concentrations=None, + n_sites=1, + ) + for day_idx, conc in zip(observation_days, ww_sample): + samples_by_day[int(day_idx) - day_one].append(float(conc)) + +# Convert to arrays for plotting +plot_days = sorted(samples_by_day.keys()) +all_samples = np.array( + [[samples_by_day[d][i] for d in plot_days] for i in range(n_samples)] +) +sample_mean = all_samples.mean(axis=0) +``` + +```{python} +# | label: fig-sampled-concentrations +# | fig-cap: Multiple realizations showing observation noise +fig, ax = plt.subplots(figsize=(8, 4)) + +# Plot all samples +for i in range(1, n_samples): + ax.plot( + plot_days, all_samples[i], color="orange", alpha=0.15, linewidth=0.5 + ) + +# Highlight one sample +ax.plot( + plot_days, + all_samples[0], + color="steelblue", + linewidth=1, + label="One realization", +) + +# Sample mean +ax.plot( + plot_days, sample_mean, color="darkred", linewidth=1.2, label="Sample mean" +) + +ax.axvline(spike_day, color="darkblue", linestyle="--", alpha=0.5) +ax.set_xlabel("Day") +ax.set_ylabel("Log Viral Concentration") +ax.set_title(f"Observation Noise: {n_samples} Samples from Same Infections") +ax.legend() +plt.tight_layout() +plt.show() +``` + +```{python} +# | label: timeline-stats +print("Timeline Analysis:") +print( + f" Infection spike on day {spike_day}: {infections[infection_spike_day, 0]:.0f} people" +) +print( + f" Mode delay from infection to concentration peak: {mode_shedding_day} days" +) +print( + f" Expected concentration peak: day {spike_day + mode_shedding_day} (= {spike_day} + {mode_shedding_day})" +) +``` + +## Site-level variability + +Different sites measuring the same underlying infections will show systematic differences: + +```{python} +# | label: multi-site +n_sites = 4 +infections_constant = jnp.ones((40, 1)) * 1500.0 + +# Each site samples at multiple time points +times_per_site = 10 +observation_times = jnp.tile(jnp.arange(10, 30, 2, dtype=jnp.int32), n_sites) +site_ids = jnp.repeat(jnp.arange(n_sites, dtype=jnp.int32), times_per_site) +subpop_ids = jnp.zeros(n_sites * times_per_site, dtype=jnp.int32) + +with numpyro.handlers.seed(rng_seed=42): + ww_multi_site = ww_process.sample( + infections=infections_constant, + subpop_indices=subpop_ids, + site_indices=site_ids, + times=observation_times, + concentrations=None, + n_sites=n_sites, + ) +``` + +```{python} +# | label: fig-multi-site +# | fig-cap: Multiple sites observing the same infections +fig, ax = plt.subplots(figsize=(8, 4)) + +colors = ["steelblue", "coral", "green", "purple"] +for site in range(n_sites): + mask = np.array(site_ids) == site + ax.plot( + np.array(observation_times)[mask], + np.array(ww_multi_site)[mask], + "-o", + color=colors[site], + label=f"Site {site}", + ) + +ax.set_xlabel("Day") +ax.set_ylabel("Log Viral Concentration") +ax.set_title("Multiple Sites Observing Same Infections") +ax.legend() +plt.tight_layout() +plt.show() +``` + +Each site has a different baseline concentration (site mode) and different measurement variability (site SD). The hierarchical model learns these site-specific effects during inference. + +## Effect of genome shedding parameter + +Higher genome shedding means higher observed concentrations: + +```{python} +# | label: compare-genome +genome_values = [8.5, 9.0, 9.5, 10.0] +infections_decay = 2000.0 * jnp.exp(-jnp.arange(40) / 15.0) +infections_decay = infections_decay.reshape(-1, 1) + +observation_days_cmp = jnp.arange(10, 30, dtype=jnp.int32) +n_obs_cmp = len(observation_days_cmp) + +results = {} +for genome_val in genome_values: + process_temp = Wastewater( + shedding_kinetics_rv=shedding_rv, + log10_genome_per_infection_rv=DeterministicVariable( + "log10_genome", genome_val + ), + ml_per_person_per_day=ml_per_person_per_day, + noise=ww_noise, + ) + + with numpyro.handlers.seed(rng_seed=42): + ww_temp = process_temp.sample( + infections=infections_decay, + subpop_indices=jnp.zeros(n_obs_cmp, dtype=jnp.int32), + site_indices=jnp.zeros(n_obs_cmp, dtype=jnp.int32), + times=observation_days_cmp, + concentrations=None, + n_sites=1, + ) + results[genome_val] = np.array(ww_temp) +``` + +```{python} +# | label: fig-genome-effect +# | fig-cap: Effect of genome shedding parameter on concentrations +fig, ax = plt.subplots(figsize=(8, 4)) + +colors = ["steelblue", "coral", "green", "purple"] +for (genome_val, conc), color in zip(results.items(), colors): + ax.plot( + np.array(observation_days_cmp), + conc, + "-o", + color=color, + label=f"log10 = {genome_val}", + markersize=4, + ) + +ax.set_xlabel("Day") +ax.set_ylabel("Log Viral Concentration") +ax.set_title("Effect of Genome Shedding Parameter") +ax.legend() +plt.tight_layout() +plt.show() +``` + +## Multiple subpopulations + +In hierarchical models, each measurement site serves a distinct subpopulation. Different subpopulations can have different infection levels: + +```{python} +# | label: multi-subpop +# Two subpopulations with different infection patterns +n_days_mp = 40 +infections_subpop1 = 1000.0 * jnp.exp( + -jnp.arange(n_days_mp) / 20.0 +) # Slow decay +infections_subpop2 = 2000.0 * jnp.exp( + -jnp.arange(n_days_mp) / 10.0 +) # Fast decay +infections_multi = jnp.stack([infections_subpop1, infections_subpop2], axis=1) + +# Two sites, each observing a different subpopulation +obs_days_mp = jnp.tile(jnp.arange(10, 30, 2, dtype=jnp.int32), 2) +subpop_ids_mp = jnp.array([0] * 10 + [1] * 10, dtype=jnp.int32) +site_ids_mp = jnp.array([0] * 10 + [1] * 10, dtype=jnp.int32) + +with numpyro.handlers.seed(rng_seed=42): + ww_multi_subpop = ww_process.sample( + infections=infections_multi, + subpop_indices=subpop_ids_mp, + site_indices=site_ids_mp, + times=obs_days_mp, + concentrations=None, + n_sites=2, + ) +``` + +```{python} +# | label: fig-multi-subpop +# | fig-cap: Sites observing different subpopulations with different infection dynamics +fig, ax = plt.subplots(figsize=(8, 4)) + +for subpop in [0, 1]: + mask = np.array(subpop_ids_mp) == subpop + ax.plot( + np.array(obs_days_mp)[mask], + np.array(ww_multi_subpop)[mask], + "-o", + label=f"Subpop {subpop}", + ) + +ax.set_xlabel("Day") +ax.set_ylabel("Log Viral Concentration") +ax.set_title("Multiple Subpopulations with Different Infection Dynamics") +ax.legend() +plt.tight_layout() +plt.show() +``` diff --git a/pyrenew/observation/__init__.py b/pyrenew/observation/__init__.py index 46cfbb67..dec3fadf 100644 --- a/pyrenew/observation/__init__.py +++ b/pyrenew/observation/__init__.py @@ -1,4 +1,23 @@ # numpydoc ignore=GL08 +""" +Observation processes for connecting infections to observed data. + +Architecture +------------ +``BaseObservationProcess`` is the abstract base. Concrete subclasses: + +- ``Counts``: Jurisdiction-level counts (admissions, deaths) +- ``CountsBySite``: Site-specific disaggregated counts +- ``Measurements``: Continuous site-level signals (e.g., wastewater) + +All observation processes implement: + +- ``_expected_signal(infections)``: transforms infections to expected values +- ``sample()``: calls ``_expected_signal()`` then applies noise model + +Noise models (``CountNoise``, ``MeasurementNoise``) are composable—pass them +to observation constructors to control the output distribution. +""" from pyrenew.observation.base import BaseObservationProcess from pyrenew.observation.count_observations import Counts, CountsBySite diff --git a/pyrenew/observation/base.py b/pyrenew/observation/base.py index d84125a1..5ee0c7e5 100644 --- a/pyrenew/observation/base.py +++ b/pyrenew/observation/base.py @@ -34,6 +34,14 @@ class BaseObservationProcess(RandomVariable): - Convolution wrapper with timeline alignment - Deterministic quantity tracking + Subclasses must implement: + + - ``validate()``: Validate parameters (call ``_validate_pmf()`` for PMFs) + - ``get_required_lookback()``: Return PMF length for initialization + - ``infection_resolution()``: Return ``"jurisdiction"`` or ``"site"`` + - ``_expected_signal()``: Transform infections to expected values + - ``sample()``: Apply noise model to expected signal + Notes ----- Computing expected observations on day t requires infection history diff --git a/pyrenew/observation/noise.py b/pyrenew/observation/noise.py index 7264b726..10f1e337 100644 --- a/pyrenew/observation/noise.py +++ b/pyrenew/observation/noise.py @@ -4,6 +4,18 @@ Provides composable noise strategies for count and measurement observations, separating the noise distribution from the observation structure. + +Count Noise +----------- +- ``PoissonNoise``: Equidispersed counts (variance = mean). No parameters. +- ``NegativeBinomialNoise``: Overdispersed counts (variance > mean). + Takes ``concentration_rv`` (higher = less overdispersion). + +Measurement Noise +----------------- +- ``HierarchicalNormalNoise``: Normal noise with hierarchical site effects. + Takes ``site_mode_prior_rv`` and ``site_sd_prior_rv`` for site-level + bias and variability. """ from __future__ import annotations From a2e4630431e85a0f4b67fa183f26284d1370351d Mon Sep 17 00:00:00 2001 From: Mitzi Morris Date: Tue, 23 Dec 2025 12:07:52 -0500 Subject: [PATCH 03/81] improve test coverage --- test/test_observation_counts.py | 163 +++++++++++++++++++++++ test/test_observation_measurements.py | 14 ++ test/test_observation_poisson.py | 7 + test/test_randomvariable_hierarchical.py | 25 ++++ 4 files changed, 209 insertions(+) diff --git a/test/test_observation_counts.py b/test/test_observation_counts.py index cd87d403..58f0b33e 100644 --- a/test/test_observation_counts.py +++ b/test/test_observation_counts.py @@ -16,6 +16,7 @@ NegativeBinomialNoise, PoissonNoise, ) +from pyrenew.observation.count_observations import _CountBase from pyrenew.randomvariable import DistributionalVariable @@ -374,5 +375,167 @@ def test_poisson_counts(self, simple_delay_pmf): assert jnp.all(counts >= 0) +class TestCountBaseInternalMethods: + """Test internal _CountBase methods for coverage.""" + + def test_count_base_infection_resolution_raises(self, simple_delay_pmf): + """Test that _CountBase.infection_resolution() raises NotImplementedError.""" + + # Create a subclass that doesn't override infection_resolution + class IncompleteCountProcess(_CountBase): + """Incomplete count process for testing.""" + + def sample(self, **kwargs): + """Sample method stub.""" + pass + + process = IncompleteCountProcess( + ascertainment_rate_rv=DeterministicVariable("ihr", 0.01), + delay_distribution_rv=DeterministicPMF("delay", simple_delay_pmf), + noise=NegativeBinomialNoise(DeterministicVariable("conc", 10.0)), + ) + with pytest.raises( + NotImplementedError, match="Subclasses must implement infection_resolution" + ): + process.infection_resolution() + + +class TestValidationMethods: + """Test validation methods for coverage.""" + + def test_validate_calls_all_validations(self, simple_delay_pmf): + """Test that validate() calls all necessary validations.""" + process = Counts( + ascertainment_rate_rv=DeterministicVariable("ihr", 0.01), + delay_distribution_rv=DeterministicPMF("delay", simple_delay_pmf), + noise=NegativeBinomialNoise(DeterministicVariable("conc", 10.0)), + ) + # Should not raise + process.validate() + + def test_validate_invalid_ascertainment_rate_negative(self, simple_delay_pmf): + """Test that validate raises for negative ascertainment rate.""" + process = Counts( + ascertainment_rate_rv=DeterministicVariable("ihr", -0.1), + delay_distribution_rv=DeterministicPMF("delay", simple_delay_pmf), + noise=NegativeBinomialNoise(DeterministicVariable("conc", 10.0)), + ) + with pytest.raises(ValueError, match="ascertainment_rate_rv must be in"): + process.validate() + + def test_validate_invalid_ascertainment_rate_greater_than_one( + self, simple_delay_pmf + ): + """Test that validate raises for ascertainment rate > 1.""" + process = Counts( + ascertainment_rate_rv=DeterministicVariable("ihr", 1.5), + delay_distribution_rv=DeterministicPMF("delay", simple_delay_pmf), + noise=NegativeBinomialNoise(DeterministicVariable("conc", 10.0)), + ) + with pytest.raises(ValueError, match="ascertainment_rate_rv must be in"): + process.validate() + + def test_get_required_lookback(self, simple_delay_pmf, long_delay_pmf): + """Test get_required_lookback returns PMF length.""" + process_short = Counts( + ascertainment_rate_rv=DeterministicVariable("ihr", 0.01), + delay_distribution_rv=DeterministicPMF("delay", simple_delay_pmf), + noise=NegativeBinomialNoise(DeterministicVariable("conc", 10.0)), + ) + assert process_short.get_required_lookback() == 1 + + process_long = Counts( + ascertainment_rate_rv=DeterministicVariable("ihr", 0.01), + delay_distribution_rv=DeterministicPMF("delay", long_delay_pmf), + noise=NegativeBinomialNoise(DeterministicVariable("conc", 10.0)), + ) + assert process_long.get_required_lookback() == 10 + + def test_infection_resolution_counts(self, simple_delay_pmf): + """Test that Counts returns 'jurisdiction' resolution.""" + process = Counts( + ascertainment_rate_rv=DeterministicVariable("ihr", 0.01), + delay_distribution_rv=DeterministicPMF("delay", simple_delay_pmf), + noise=NegativeBinomialNoise(DeterministicVariable("conc", 10.0)), + ) + assert process.infection_resolution() == "jurisdiction" + + +class TestNoiseValidation: + """Test noise model validation methods.""" + + def test_poisson_noise_validate(self): + """Test PoissonNoise validate method.""" + noise = PoissonNoise() + # Should not raise - Poisson has no parameters to validate + noise.validate() + + def test_negative_binomial_noise_validate_success(self): + """Test NegativeBinomialNoise validate with valid concentration.""" + noise = NegativeBinomialNoise(DeterministicVariable("conc", 10.0)) + # Should not raise + noise.validate() + + def test_negative_binomial_noise_validate_zero_concentration(self): + """Test NegativeBinomialNoise validate with zero concentration.""" + noise = NegativeBinomialNoise(DeterministicVariable("conc", 0.0)) + with pytest.raises(ValueError, match="concentration must be positive"): + noise.validate() + + def test_negative_binomial_noise_validate_negative_concentration(self): + """Test NegativeBinomialNoise validate with negative concentration.""" + noise = NegativeBinomialNoise(DeterministicVariable("conc", -1.0)) + with pytest.raises(ValueError, match="concentration must be positive"): + noise.validate() + + +class TestBaseObservationProcessValidation: + """Test base observation process PMF validation.""" + + def test_validate_pmf_empty_array(self, simple_delay_pmf): + """Test that _validate_pmf raises for empty array.""" + process = Counts( + ascertainment_rate_rv=DeterministicVariable("ihr", 0.01), + delay_distribution_rv=DeterministicPMF("delay", simple_delay_pmf), + noise=NegativeBinomialNoise(DeterministicVariable("conc", 10.0)), + ) + empty_pmf = jnp.array([]) + with pytest.raises(ValueError, match="must return non-empty array"): + process._validate_pmf(empty_pmf, "test_pmf") + + def test_validate_pmf_sum_not_one(self, simple_delay_pmf): + """Test that _validate_pmf raises for PMF not summing to 1.""" + process = Counts( + ascertainment_rate_rv=DeterministicVariable("ihr", 0.01), + delay_distribution_rv=DeterministicPMF("delay", simple_delay_pmf), + noise=NegativeBinomialNoise(DeterministicVariable("conc", 10.0)), + ) + bad_pmf = jnp.array([0.3, 0.3, 0.3]) # sums to 0.9 + with pytest.raises(ValueError, match="must sum to 1.0"): + process._validate_pmf(bad_pmf, "test_pmf") + + def test_validate_pmf_negative_values(self, simple_delay_pmf): + """Test that _validate_pmf raises for negative values.""" + process = Counts( + ascertainment_rate_rv=DeterministicVariable("ihr", 0.01), + delay_distribution_rv=DeterministicPMF("delay", simple_delay_pmf), + noise=NegativeBinomialNoise(DeterministicVariable("conc", 10.0)), + ) + bad_pmf = jnp.array([1.5, -0.5]) # sums to 1.0 but has negative + with pytest.raises(ValueError, match="must have non-negative values"): + process._validate_pmf(bad_pmf, "test_pmf") + + def test_get_minimum_observation_day(self): + """Test get_minimum_observation_day returns correct value.""" + delay_pmf = jnp.array([0.2, 0.5, 0.3]) # length 3 + process = Counts( + ascertainment_rate_rv=DeterministicVariable("ihr", 0.01), + delay_distribution_rv=DeterministicPMF("delay", delay_pmf), + noise=NegativeBinomialNoise(DeterministicVariable("conc", 10.0)), + ) + # First valid day should be len(pmf) - 1 = 2 + assert process.get_minimum_observation_day() == 2 + + if __name__ == "__main__": pytest.main([__file__, "-v"]) diff --git a/test/test_observation_measurements.py b/test/test_observation_measurements.py index 2ca11a22..5e69fd3d 100644 --- a/test/test_observation_measurements.py +++ b/test/test_observation_measurements.py @@ -97,6 +97,20 @@ def test_infection_resolution_is_site(self): class TestHierarchicalNormalNoise: """Test HierarchicalNormalNoise model.""" + def test_validate(self): + """Test HierarchicalNormalNoise validate method.""" + site_mode_prior = HierarchicalNormalPrior( + name="mode", sd_rv=DeterministicVariable("mode_sd", 0.5) + ) + site_sd_prior = GammaGroupSdPrior( + name="sd", + sd_mean_rv=DeterministicVariable("sd_mean", 0.3), + sd_concentration_rv=DeterministicVariable("sd_conc", 4.0), + ) + noise = HierarchicalNormalNoise(site_mode_prior, site_sd_prior) + # Should not raise - validation is deferred to sample time + noise.validate() + def test_sample_shape(self): """Test that HierarchicalNormalNoise produces correct shape.""" site_mode_prior = HierarchicalNormalPrior( diff --git a/test/test_observation_poisson.py b/test/test_observation_poisson.py index b9d975be..8b9c0716 100644 --- a/test/test_observation_poisson.py +++ b/test/test_observation_poisson.py @@ -20,3 +20,10 @@ def test_poisson_obs(): sim_pois = pois(mu=rates) testing.assert_array_equal(sim_pois, jnp.ceil(sim_pois)) + + +def test_poisson_validate(): + """ + Check that PoissonObservation.validate() runs without error. + """ + PoissonObservation.validate() diff --git a/test/test_randomvariable_hierarchical.py b/test/test_randomvariable_hierarchical.py index 30384800..89a58c3a 100644 --- a/test/test_randomvariable_hierarchical.py +++ b/test/test_randomvariable_hierarchical.py @@ -46,6 +46,13 @@ def test_smaller_sd_produces_tighter_distribution(self): # Tight prior should have smaller standard deviation assert jnp.std(samples_tight) < jnp.std(samples_wide) + def test_validate(self): + """Test that validate() runs without error.""" + prior = HierarchicalNormalPrior( + "effect", sd_rv=DeterministicVariable("sd", 1.0) + ) + prior.validate() # Should not raise + def test_rejects_non_random_variable_sd(self): """Test that non-RandomVariable sd_rv is rejected.""" with pytest.raises(TypeError, match="sd_rv must be a RandomVariable"): @@ -120,6 +127,15 @@ def test_rejects_negative_sd_min(self): sd_min=-0.1, ) + def test_validate(self): + """Test that validate() runs without error.""" + prior = GammaGroupSdPrior( + "sd", + sd_mean_rv=DeterministicVariable("sd_mean", 0.5), + sd_concentration_rv=DeterministicVariable("sd_conc", 4.0), + ) + prior.validate() # Should not raise + class TestStudentTGroupModePrior: """Test StudentTGroupModePrior.""" @@ -174,6 +190,15 @@ def test_rejects_non_random_variable_params(self): df_rv=4.0, ) + def test_validate(self): + """Test that validate() runs without error.""" + prior = StudentTGroupModePrior( + "mode", + sd_rv=DeterministicVariable("sd", 1.0), + df_rv=DeterministicVariable("df", 4.0), + ) + prior.validate() # Should not raise + if __name__ == "__main__": pytest.main([__file__, "-v"]) From 24096bc824dd03ab52e9e24cacda1a7782954900 Mon Sep 17 00:00:00 2001 From: Mitzi Morris Date: Tue, 23 Dec 2025 12:27:17 -0500 Subject: [PATCH 04/81] improve unit test coverage --- pyrenew/observation/base.py | 10 +++++----- pyrenew/observation/noise.py | 8 ++++---- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/pyrenew/observation/base.py b/pyrenew/observation/base.py index 5ee0c7e5..8468566a 100644 --- a/pyrenew/observation/base.py +++ b/pyrenew/observation/base.py @@ -88,7 +88,7 @@ def validate(self) -> None: ValueError If any parameters fail validation. """ - pass + pass # pragma: no cover @abstractmethod def get_required_lookback(self) -> int: @@ -110,7 +110,7 @@ def get_required_lookback(self) -> int: n_initialization_points as: ``max(gen_int_length, max(all lookbacks)) - 1`` """ - pass + pass # pragma: no cover @abstractmethod def infection_resolution(self) -> str: @@ -140,7 +140,7 @@ def infection_resolution(self) -> str: This is used by multi-signal models to route the correct infection output to each observation process. """ - pass + pass # pragma: no cover def _validate_pmf( self, @@ -305,7 +305,7 @@ def _expected_signal( -------- sample : Uses this method then applies noise model """ - pass + pass # pragma: no cover @abstractmethod def sample(self, **kwargs) -> ArrayLike: @@ -329,4 +329,4 @@ def sample(self, **kwargs) -> ArrayLike: ArrayLike Observed or sampled values from the observation process. """ - pass + pass # pragma: no cover diff --git a/pyrenew/observation/noise.py b/pyrenew/observation/noise.py index 10f1e337..4065061e 100644 --- a/pyrenew/observation/noise.py +++ b/pyrenew/observation/noise.py @@ -63,7 +63,7 @@ def sample( ArrayLike Sampled or conditioned counts, same shape as expected. """ - pass + pass # pragma: no cover @abstractmethod def validate(self) -> None: @@ -75,7 +75,7 @@ def validate(self) -> None: ValueError If parameters are invalid. """ - pass + pass # pragma: no cover class PoissonNoise(CountNoise): @@ -235,7 +235,7 @@ def sample( ArrayLike Sampled or conditioned measurements, same shape as expected. """ - pass + pass # pragma: no cover @abstractmethod def validate(self) -> None: @@ -247,7 +247,7 @@ def validate(self) -> None: ValueError If parameters are invalid. """ - pass + pass # pragma: no cover class HierarchicalNormalNoise(MeasurementNoise): From 671d9d0075234fee8640e323ea4f46314090fa12 Mon Sep 17 00:00:00 2001 From: Mitzi Morris Date: Fri, 26 Dec 2025 16:05:05 -0500 Subject: [PATCH 05/81] consistent names: 'subpop' (not site), 'sensor' for site/lab measurements, 'aggregate' instead of 'jurisdiction' --- .../observation_processes_counts.qmd | 28 ++--- .../observation_processes_measurements.qmd | 108 ++++++++-------- pyrenew/observation/__init__.py | 23 ++-- pyrenew/observation/base.py | 28 +++-- pyrenew/observation/count_observations.py | 93 ++++++++------ pyrenew/observation/measurements.py | 57 +++++---- pyrenew/observation/noise.py | 82 +++++++----- test/conftest.py | 20 +-- test/test_observation_counts.py | 118 +++++++++--------- test/test_observation_measurements.py | 77 ++++++------ 10 files changed, 345 insertions(+), 289 deletions(-) diff --git a/docs/tutorials/observation_processes_counts.qmd b/docs/tutorials/observation_processes_counts.qmd index 83279331..b3b415c7 100644 --- a/docs/tutorials/observation_processes_counts.qmd +++ b/docs/tutorials/observation_processes_counts.qmd @@ -22,7 +22,7 @@ from pyrenew import datasets ## Overview Count observation processes model the lag between infections and an observed outcome such as hospital admissions, emergency department visits, confirmed cases, or deaths. -Observed data can be aggregated at the jurisdiction level or available as disaggregated counts from reporting entities, which are modeled by classes `Counts` and `CountsBySite`, respectively. +Observed data can be aggregated or available as subpopulation-level counts, which are modeled by classes `Counts` and `CountsBySubpop`, respectively. Count observation processes transform infections into expected observed counts by applying an ascertainment rate and convolving with a delay distribution. @@ -129,20 +129,20 @@ hosp_process = Counts( noise=NegativeBinomialNoise(concentration_rv), ) -print(f"Required lookback: {hosp_process.get_required_lookback()} days") +print(f"Required lookback: {hosp_process.lookback_days()} days") ``` ### Timeline alignment and lookback period The observation process convolves infections with a delay distribution, maintaining alignment between input and output: day $t$ in the output corresponds to day $t$ in the input. -Hospital admissions depend on infections from prior days (the length of our delay distribution minus one). The method `get_required_lookback()` returns this length; the first valid observation day is at index `lookback - 1`. Earlier days are marked invalid. +Hospital admissions depend on infections from prior days (the length of our delay distribution minus one). The method `lookback_days()` returns this length; the first valid observation day is at index `lookback - 1`. Earlier days are marked invalid. ```{python} # | label: helper-function def first_valid_observation_day(obs_process) -> int: """Return the first day index with complete infection history for convolution.""" - return obs_process.get_required_lookback() - 1 + return obs_process.lookback_days() - 1 ``` ## Simulating hospital admissions from infections @@ -152,7 +152,7 @@ To demonstrate how the observation process works, we simulate admissions from a ```{python} # | label: simulate-spike n_days = 100 -lookback = hosp_process.get_required_lookback() +lookback = hosp_process.lookback_days() # First valid observation day day_one = lookback - 1 @@ -207,7 +207,7 @@ Because all infections occur on a single day, we can see how they spread into ho fig, ax = plt.subplots(figsize=(8, 4)) ax.plot( np.arange(n_plot_days), - np.array(hosp_admissions[day_one:]), + np.array(hosp_admissions.observed[day_one:]), "-o", color="purple", ) @@ -246,8 +246,8 @@ samples = [] for seed in range(n_samples): with numpyro.handlers.seed(rng_seed=seed): - sample = hosp_process.sample(infections=infections, counts=None) - samples.append(np.array(sample[day_one:])) + result = hosp_process.sample(infections=infections, counts=None) + samples.append(np.array(result.observed[day_one:])) samples = np.array(samples) sample_mean = samples.mean(axis=0) @@ -328,8 +328,8 @@ for ihr_val in ihr_values: samples = [] for seed in range(20): with numpyro.handlers.seed(rng_seed=seed): - sample = process.sample(infections=infections_decay, counts=None) - samples.append(np.array(sample[day_one:])) + result = process.sample(infections=infections_decay, counts=None) + samples.append(np.array(result.observed[day_one:])) results[ihr_val] = np.mean(samples, axis=0) ``` @@ -381,12 +381,12 @@ for ax, conc_val in zip(axes, concentration_values): for seed in range(n_replicates): with numpyro.handlers.seed(rng_seed=seed): - sample = process.sample( + result = process.sample( infections=infections_constant, counts=None ) ax.plot( np.arange(n_plot_days), - np.array(sample[day_one:]), + np.array(result.observed[day_one:]), color="steelblue", alpha=0.5, linewidth=0.8, @@ -414,12 +414,12 @@ hosp_process_poisson = Counts( ) with numpyro.handlers.seed(rng_seed=42): - poisson_sample = hosp_process_poisson.sample( + poisson_result = hosp_process_poisson.sample( infections=infections, counts=None, ) print( - f"Sampled {len(poisson_sample)} days of hospital admissions with Poisson noise" + f"Sampled {len(poisson_result.observed)} days of hospital admissions with Poisson noise" ) ``` diff --git a/docs/tutorials/observation_processes_measurements.qmd b/docs/tutorials/observation_processes_measurements.qmd index 96138d0a..613e1176 100644 --- a/docs/tutorials/observation_processes_measurements.qmd +++ b/docs/tutorials/observation_processes_measurements.qmd @@ -52,7 +52,7 @@ The core convolution $\sum_{d=0}^{D} I_{t-d} \cdot p_d$ is the same as for count | Temporal PMF | Delay distribution | Shedding/decay kinetics | | Output space | Expected counts (linear) | Log-concentrations | | Noise model | Poisson or Negative Binomial | Normal on log scale | -| Site structure | Optional (`CountsBySite`) | Inherent (hierarchical effects) | +| Subpop structure | Optional (`CountsBySubpop`) | Inherent (hierarchical effects) | **Key features of measurement data:** @@ -113,7 +113,7 @@ class Wastewater(Measurements): self._validate_pmf(shedding_pmf, "shedding_kinetics_rv") self.noise.validate() - def get_required_lookback(self) -> int: + def lookback_days(self) -> int: """Return shedding PMF length.""" return len(self.temporal_pmf_rv()) @@ -134,7 +134,7 @@ class Wastewater(Measurements): ) return convolved - # Apply to all sites (infections shape: n_days x n_sites) + # Apply to all subpops (infections shape: n_days x n_subpops) shedding_signal = jax.vmap(convolve_site, in_axes=1, out_axes=1)( infections ) @@ -208,21 +208,21 @@ log10_genome_rv = DeterministicVariable("log10_genome", 9.0) ml_per_person_per_day = 1000.0 ``` -## Noise model with site-level effects +## Noise model with sensor-level effects -Different sites have systematic differences in measured concentrations due to lab protocols, sampling methods, and flow estimation. We model this with hierarchical site effects: +A measurement depends on both the thing being measured (e.g., wastewater from a treatment plant) and the calibration of the lab/instruments used to obtain that measurement. We call this combination a "sensor"—the WWTP/lab pair that determines the measurement characteristics. Different sensors have systematic biases and variabilities that we model with hierarchical effects. ```{python} # | label: noise-model -# Site-level mode: systematic differences between sites -site_mode_prior = HierarchicalNormalPrior( - name="ww_site_mode", +# Sensor-level mode: systematic differences between sensors +sensor_mode_prior = HierarchicalNormalPrior( + name="ww_sensor_mode", sd_rv=DeterministicVariable("mode_sd", 0.5), ) -# Site-level SD: measurement variability within each site -site_sd_prior = GammaGroupSdPrior( - name="ww_site_sd", +# Sensor-level SD: measurement variability within each sensor +sensor_sd_prior = GammaGroupSdPrior( + name="ww_sensor_sd", sd_mean_rv=DeterministicVariable("sd_mean", 0.3), sd_concentration_rv=DeterministicVariable("sd_concentration", 4.0), sd_min=0.10, @@ -230,8 +230,8 @@ site_sd_prior = GammaGroupSdPrior( # Create the noise model ww_noise = HierarchicalNormalNoise( - site_mode_prior_rv=site_mode_prior, - site_sd_prior_rv=site_sd_prior, + sensor_mode_prior_rv=sensor_mode_prior, + sensor_sd_prior_rv=sensor_sd_prior, ) ``` @@ -246,20 +246,20 @@ ww_process = Wastewater( noise=ww_noise, ) -print(f"Required lookback: {ww_process.get_required_lookback()} days") +print(f"Required lookback: {ww_process.lookback_days()} days") ``` ### Timeline alignment and lookback period The observation process convolves infections with a shedding distribution, maintaining alignment between input and output: day $t$ in the output corresponds to day $t$ in the input. -Wastewater concentrations depend on infections from prior days (the length of our shedding distribution minus one). The method `get_required_lookback()` returns this length; the first valid observation day is at index `lookback - 1`. Earlier days are marked invalid with NaN. +Wastewater concentrations depend on infections from prior days (the length of our shedding distribution minus one). The method `lookback_days()` returns this length; the first valid observation day is at index `lookback - 1`. Earlier days are marked invalid with NaN. ```{python} # | label: helper-function def first_valid_observation_day(obs_process) -> int: """Return the first day index with complete infection history for convolution.""" - return obs_process.get_required_lookback() - 1 + return obs_process.lookback_days() - 1 ``` ## Simulating wastewater observations from infections @@ -273,7 +273,7 @@ n_days = 50 # First valid observation day day_one = first_valid_observation_day(ww_process) -# Create infections with a spike (shape: n_days x n_sites) +# Create infections with a spike (shape: n_days x n_subpops) infection_spike_day = day_one + 10 infections = jnp.zeros((n_days, 1)) # 1 subpopulation infections = infections.at[infection_spike_day, 0].set(2000.0) @@ -290,10 +290,10 @@ with numpyro.handlers.seed(rng_seed=42): ww_obs = ww_process.sample( infections=infections, subpop_indices=jnp.zeros(n_obs, dtype=jnp.int32), - site_indices=jnp.zeros(n_obs, dtype=jnp.int32), + sensor_indices=jnp.zeros(n_obs, dtype=jnp.int32), times=observation_days, concentrations=None, # Sample from prior - n_sites=1, + n_sensors=1, ) ``` @@ -335,15 +335,15 @@ samples_by_day = {int(d - day_one): [] for d in observation_days} for seed in range(n_samples): with numpyro.handlers.seed(rng_seed=seed): - ww_sample = ww_process.sample( + ww_result = ww_process.sample( infections=infections, subpop_indices=jnp.zeros(n_obs, dtype=jnp.int32), - site_indices=jnp.zeros(n_obs, dtype=jnp.int32), + sensor_indices=jnp.zeros(n_obs, dtype=jnp.int32), times=observation_days, concentrations=None, - n_sites=1, + n_sensors=1, ) - for day_idx, conc in zip(observation_days, ww_sample): + for day_idx, conc in zip(observation_days, ww_result.observed): samples_by_day[int(day_idx) - day_one].append(float(conc)) # Convert to arrays for plotting @@ -402,57 +402,57 @@ print( ) ``` -## Site-level variability +## Sensor-level variability -Different sites measuring the same underlying infections will show systematic differences: +Different sensors measuring the same underlying infections will show systematic differences: ```{python} -# | label: multi-site -n_sites = 4 +# | label: multi-sensor +num_sensors = 4 infections_constant = jnp.ones((40, 1)) * 1500.0 -# Each site samples at multiple time points -times_per_site = 10 -observation_times = jnp.tile(jnp.arange(10, 30, 2, dtype=jnp.int32), n_sites) -site_ids = jnp.repeat(jnp.arange(n_sites, dtype=jnp.int32), times_per_site) -subpop_ids = jnp.zeros(n_sites * times_per_site, dtype=jnp.int32) +# Each sensor samples at multiple time points +times_per_sensor = 10 +observation_times = jnp.tile(jnp.arange(10, 30, 2, dtype=jnp.int32), num_sensors) +sensor_ids = jnp.repeat(jnp.arange(num_sensors, dtype=jnp.int32), times_per_sensor) +subpop_ids = jnp.zeros(num_sensors * times_per_sensor, dtype=jnp.int32) with numpyro.handlers.seed(rng_seed=42): - ww_multi_site = ww_process.sample( + ww_multi_sensor = ww_process.sample( infections=infections_constant, subpop_indices=subpop_ids, - site_indices=site_ids, + sensor_indices=sensor_ids, times=observation_times, concentrations=None, - n_sites=n_sites, + n_sensors=num_sensors, ) ``` ```{python} -# | label: fig-multi-site -# | fig-cap: Multiple sites observing the same infections +# | label: fig-multi-sensor +# | fig-cap: Multiple sensors observing the same infections fig, ax = plt.subplots(figsize=(8, 4)) colors = ["steelblue", "coral", "green", "purple"] -for site in range(n_sites): - mask = np.array(site_ids) == site +for sensor in range(num_sensors): + mask = np.array(sensor_ids) == sensor ax.plot( np.array(observation_times)[mask], - np.array(ww_multi_site)[mask], + np.array(ww_multi_sensor.observed)[mask], "-o", - color=colors[site], - label=f"Site {site}", + color=colors[sensor], + label=f"Sensor {sensor}", ) ax.set_xlabel("Day") ax.set_ylabel("Log Viral Concentration") -ax.set_title("Multiple Sites Observing Same Infections") +ax.set_title("Multiple Sensors Observing Same Infections") ax.legend() plt.tight_layout() plt.show() ``` -Each site has a different baseline concentration (site mode) and different measurement variability (site SD). The hierarchical model learns these site-specific effects during inference. +Each sensor has a different baseline concentration (sensor mode) and different measurement variability (sensor SD). The hierarchical model learns these sensor-specific effects during inference. ## Effect of genome shedding parameter @@ -482,12 +482,12 @@ for genome_val in genome_values: ww_temp = process_temp.sample( infections=infections_decay, subpop_indices=jnp.zeros(n_obs_cmp, dtype=jnp.int32), - site_indices=jnp.zeros(n_obs_cmp, dtype=jnp.int32), + sensor_indices=jnp.zeros(n_obs_cmp, dtype=jnp.int32), times=observation_days_cmp, concentrations=None, - n_sites=1, + n_sensors=1, ) - results[genome_val] = np.array(ww_temp) + results[genome_val] = np.array(ww_temp.observed) ``` ```{python} @@ -516,7 +516,7 @@ plt.show() ## Multiple subpopulations -In hierarchical models, each measurement site serves a distinct subpopulation. Different subpopulations can have different infection levels: +In hierarchical models, each sensor serves a distinct subpopulation (catchment area). Different subpopulations can have different infection levels: ```{python} # | label: multi-subpop @@ -530,32 +530,32 @@ infections_subpop2 = 2000.0 * jnp.exp( ) # Fast decay infections_multi = jnp.stack([infections_subpop1, infections_subpop2], axis=1) -# Two sites, each observing a different subpopulation +# Two sensors, each observing a different subpopulation obs_days_mp = jnp.tile(jnp.arange(10, 30, 2, dtype=jnp.int32), 2) subpop_ids_mp = jnp.array([0] * 10 + [1] * 10, dtype=jnp.int32) -site_ids_mp = jnp.array([0] * 10 + [1] * 10, dtype=jnp.int32) +sensor_ids_mp = jnp.array([0] * 10 + [1] * 10, dtype=jnp.int32) with numpyro.handlers.seed(rng_seed=42): ww_multi_subpop = ww_process.sample( infections=infections_multi, subpop_indices=subpop_ids_mp, - site_indices=site_ids_mp, + sensor_indices=sensor_ids_mp, times=obs_days_mp, concentrations=None, - n_sites=2, + n_sensors=2, ) ``` ```{python} # | label: fig-multi-subpop -# | fig-cap: Sites observing different subpopulations with different infection dynamics +# | fig-cap: Sensors observing different subpopulations with different infection dynamics fig, ax = plt.subplots(figsize=(8, 4)) for subpop in [0, 1]: mask = np.array(subpop_ids_mp) == subpop ax.plot( np.array(obs_days_mp)[mask], - np.array(ww_multi_subpop)[mask], + np.array(ww_multi_subpop.observed)[mask], "-o", label=f"Subpop {subpop}", ) diff --git a/pyrenew/observation/__init__.py b/pyrenew/observation/__init__.py index dec3fadf..8a0cdeab 100644 --- a/pyrenew/observation/__init__.py +++ b/pyrenew/observation/__init__.py @@ -6,21 +6,22 @@ ------------ ``BaseObservationProcess`` is the abstract base. Concrete subclasses: -- ``Counts``: Jurisdiction-level counts (admissions, deaths) -- ``CountsBySite``: Site-specific disaggregated counts -- ``Measurements``: Continuous site-level signals (e.g., wastewater) +- ``Counts``: Aggregate counts (admissions, deaths) +- ``CountsBySubpop``: Subpopulation-level counts +- ``Measurements``: Continuous subpopulation-level signals (e.g., wastewater) All observation processes implement: -- ``_expected_signal(infections)``: transforms infections to expected values -- ``sample()``: calls ``_expected_signal()`` then applies noise model +- ``sample()``: Sample observations given infections +- ``infection_resolution()``: returns ``"aggregate"`` or ``"subpop"`` +- ``lookback_days()``: returns required infection history length Noise models (``CountNoise``, ``MeasurementNoise``) are composable—pass them to observation constructors to control the output distribution. """ from pyrenew.observation.base import BaseObservationProcess -from pyrenew.observation.count_observations import Counts, CountsBySite +from pyrenew.observation.count_observations import Counts, CountsBySubpop from pyrenew.observation.measurements import Measurements from pyrenew.observation.negativebinomial import NegativeBinomialObservation from pyrenew.observation.noise import ( @@ -31,21 +32,23 @@ PoissonNoise, ) from pyrenew.observation.poisson import PoissonObservation +from pyrenew.observation.types import ObservationSample __all__ = [ # Existing (kept for backward compatibility) "NegativeBinomialObservation", "PoissonObservation", - # New base classes + # Base classes and types "BaseObservationProcess", - # New noise models + "ObservationSample", + # Noise models "CountNoise", "PoissonNoise", "NegativeBinomialNoise", "MeasurementNoise", "HierarchicalNormalNoise", - # New observation processes + # Observation processes "Counts", - "CountsBySite", + "CountsBySubpop", "Measurements", ] diff --git a/pyrenew/observation/base.py b/pyrenew/observation/base.py index 8468566a..635d386f 100644 --- a/pyrenew/observation/base.py +++ b/pyrenew/observation/base.py @@ -37,8 +37,8 @@ class BaseObservationProcess(RandomVariable): Subclasses must implement: - ``validate()``: Validate parameters (call ``_validate_pmf()`` for PMFs) - - ``get_required_lookback()``: Return PMF length for initialization - - ``infection_resolution()``: Return ``"jurisdiction"`` or ``"site"`` + - ``lookback_days()``: Return PMF length for initialization + - ``infection_resolution()``: Return ``"aggregate"`` or ``"subpop"`` - ``_expected_signal()``: Transform infections to expected values - ``sample()``: Apply noise model to expected signal @@ -91,7 +91,7 @@ def validate(self) -> None: pass # pragma: no cover @abstractmethod - def get_required_lookback(self) -> int: + def lookback_days(self) -> int: """ Return the number of days this observation process needs to look back. @@ -115,25 +115,27 @@ def get_required_lookback(self) -> int: @abstractmethod def infection_resolution(self) -> str: """ - Return the resolution of infections this observation uses. + Return whether this observation uses aggregate or subpop infections. Returns one of: - - ``"jurisdiction"``: Uses jurisdiction-level aggregated infections - - ``"site"``: Uses site-level disaggregated infections + - ``"aggregate"``: Uses a single aggregated infection trajectory. + Shape: ``(n_days,)`` + - ``"subpop"``: Uses subpopulation-level infection trajectories. + Shape: ``(n_days, n_subpops)``, indexed via ``subpop_indices``. Returns ------- str - Either ``"jurisdiction"`` or ``"site"`` + Either ``"aggregate"`` or ``"subpop"`` Examples -------- - >>> # Aggregated count observations use jurisdiction-level - >>> hosp_obs.infection_resolution() # Returns "jurisdiction" + >>> # Aggregated count observations + >>> hosp_obs.infection_resolution() # Returns "aggregate" >>> - >>> # Wastewater uses site-level - >>> ww_obs.infection_resolution() # Returns "site" + >>> # Subpopulation-level observations (wastewater, subpop-specific counts) + >>> ww_obs.infection_resolution() # Returns "subpop" Notes ----- @@ -285,8 +287,8 @@ def _expected_signal( ---------- infections : ArrayLike Infections from the infection process. - Shape: (n_days,) for jurisdiction-level observations - Shape: (n_days, n_sites) for site-level observations + Shape: (n_days,) for aggregate observations + Shape: (n_days, n_subpops) for subpop-level observations Returns ------- diff --git a/pyrenew/observation/count_observations.py b/pyrenew/observation/count_observations.py index 437c37e6..7a883932 100644 --- a/pyrenew/observation/count_observations.py +++ b/pyrenew/observation/count_observations.py @@ -14,6 +14,7 @@ from pyrenew.metaclass import RandomVariable from pyrenew.observation.base import BaseObservationProcess from pyrenew.observation.noise import CountNoise +from pyrenew.observation.types import ObservationSample class _CountBase(BaseObservationProcess): @@ -67,7 +68,7 @@ def validate(self) -> None: self.noise.validate() - def get_required_lookback(self) -> int: + def lookback_days(self) -> int: """ Return delay PMF length. @@ -85,7 +86,7 @@ def infection_resolution(self) -> str: Returns ------- str - "jurisdiction" for aggregated, "site" for disaggregated. + "aggregate" or "subpop". """ raise NotImplementedError("Subclasses must implement infection_resolution()") @@ -100,8 +101,8 @@ def _expected_signal( ---------- infections : ArrayLike Infections from the infection process. - Shape: (n_days,) for jurisdiction-level - Shape: (n_days, n_sites) for site-level + Shape: (n_days,) for aggregate + Shape: (n_days, n_subpops) for subpop-level Returns ------- @@ -127,10 +128,10 @@ def convolve_col(col): # numpydoc ignore=GL08 class Counts(_CountBase): """ - Aggregated count observation for jurisdiction-level data. + Aggregated count observation. - Maps jurisdiction-level infections to aggregated counts through - ascertainment x delay convolution with composable noise model. + Maps aggregate infections to counts through ascertainment x delay + convolution with composable noise model. Parameters ---------- @@ -167,21 +168,29 @@ class Counts(_CountBase): def infection_resolution(self) -> str: """ - Return "jurisdiction" for aggregated observations. + Return "aggregate" for aggregated observations. Returns ------- str - The string "jurisdiction". + The string "aggregate". """ - return "jurisdiction" + return "aggregate" + + def __repr__(self) -> str: + """Return string representation.""" + return ( + f"Counts(ascertainment_rate_rv={self.ascertainment_rate_rv!r}, " + f"delay_distribution_rv={self.temporal_pmf_rv!r}, " + f"noise={self.noise!r})" + ) def sample( self, infections: ArrayLike, counts: ArrayLike | None = None, times: ArrayLike | None = None, - ) -> ArrayLike: + ) -> ObservationSample: """ Sample aggregated counts with dense or sparse observations. @@ -191,7 +200,7 @@ def sample( Parameters ---------- infections : ArrayLike - Jurisdiction-level infections from the infection process. + Aggregate infections from the infection process. Shape: (n_days,) counts : ArrayLike | None Observed counts. Dense: (n_days,), Sparse: (n_obs,), None: prior. @@ -200,9 +209,9 @@ def sample( Returns ------- - ArrayLike - Observed or sampled counts. - Dense: (n_days,), Sparse: (n_obs,) + ObservationSample + Named tuple with `observed` (sampled/conditioned counts) and + `expected` (expected counts before noise). """ expected_counts = self._expected_signal(infections) self._deterministic("expected_counts", expected_counts) @@ -214,18 +223,20 @@ def sample( else: expected_obs = expected_counts_safe - return self.noise.sample( + observed = self.noise.sample( name="counts", expected=expected_obs, obs=counts, ) + return ObservationSample(observed=observed, expected=expected_counts) + -class CountsBySite(_CountBase): +class CountsBySubpop(_CountBase): """ - Disaggregated count observation for site-specific data. + Subpopulation-level count observation. - Maps site-level infections to site-specific counts through + Maps subpopulation-level infections to counts through ascertainment x delay convolution with composable noise model. Parameters @@ -244,19 +255,19 @@ class CountsBySite(_CountBase): Examples -------- >>> from pyrenew.deterministic import DeterministicVariable, DeterministicPMF - >>> from pyrenew.observation import CountsBySite, PoissonNoise + >>> from pyrenew.observation import CountsBySubpop, PoissonNoise >>> import jax.numpy as jnp >>> import numpyro >>> >>> delay_pmf = jnp.array([0.3, 0.4, 0.3]) - >>> counts_obs = CountsBySite( + >>> counts_obs = CountsBySubpop( ... ascertainment_rate_rv=DeterministicVariable("ihr", 0.02), ... delay_distribution_rv=DeterministicPMF("delay", delay_pmf), ... noise=PoissonNoise(), ... ) >>> >>> with numpyro.handlers.seed(rng_seed=42): - ... infections = jnp.ones((30, 3)) * 500 # 30 days, 3 sites + ... infections = jnp.ones((30, 3)) * 500 # 30 days, 3 subpops ... times = jnp.array([10, 15, 10, 15]) ... subpop_indices = jnp.array([0, 0, 1, 1]) ... sampled = counts_obs.sample( @@ -267,16 +278,24 @@ class CountsBySite(_CountBase): ... ) """ + def __repr__(self) -> str: + """Return string representation.""" + return ( + f"CountsBySubpop(ascertainment_rate_rv={self.ascertainment_rate_rv!r}, " + f"delay_distribution_rv={self.temporal_pmf_rv!r}, " + f"noise={self.noise!r})" + ) + def infection_resolution(self) -> str: """ - Return "site" for disaggregated observations. + Return "subpop" for subpopulation-level observations. Returns ------- str - The string "site". + The string "subpop". """ - return "site" + return "subpop" def sample( self, @@ -284,9 +303,9 @@ def sample( subpop_indices: ArrayLike, times: ArrayLike, counts: ArrayLike | None = None, - ) -> ArrayLike: + ) -> ObservationSample: """ - Sample disaggregated counts with flexible indexing. + Sample subpopulation-level counts with flexible indexing. Validation is performed before JAX tracing at runtime, prior to calling this method. @@ -294,8 +313,8 @@ def sample( Parameters ---------- infections : ArrayLike - Site-level infections from the infection process. - Shape: (n_days, n_sites) + Subpopulation-level infections from the infection process. + Shape: (n_days, n_subpops) subpop_indices : ArrayLike Subpopulation index for each observation (0-indexed). Shape: (n_obs,) @@ -307,21 +326,23 @@ def sample( Returns ------- - ArrayLike - Observed or sampled counts. - Shape: (n_obs,) + ObservationSample + Named tuple with `observed` (sampled/conditioned counts) and + `expected` (expected counts before noise, shape: n_days x n_subpops). """ - # Compute expected counts for all sites + # Compute expected counts for all subpops expected_counts_all = self._expected_signal(infections) - self._deterministic("expected_counts_by_site", expected_counts_all) + self._deterministic("expected_counts_by_subpop", expected_counts_all) # Replace NaN padding with 0 for distribution creation expected_counts_safe = jnp.nan_to_num(expected_counts_all, nan=0.0) expected_obs = expected_counts_safe[times, subpop_indices] - return self.noise.sample( - name="counts_by_site", + observed = self.noise.sample( + name="counts_by_subpop", expected=expected_obs, obs=counts, ) + + return ObservationSample(observed=observed, expected=expected_counts_all) diff --git a/pyrenew/observation/measurements.py b/pyrenew/observation/measurements.py index 65a95fa9..33e38980 100644 --- a/pyrenew/observation/measurements.py +++ b/pyrenew/observation/measurements.py @@ -11,6 +11,7 @@ from pyrenew.metaclass import RandomVariable from pyrenew.observation.base import BaseObservationProcess from pyrenew.observation.noise import MeasurementNoise +from pyrenew.observation.types import ObservationSample class Measurements(BaseObservationProcess): @@ -37,7 +38,7 @@ class Measurements(BaseObservationProcess): See Also -------- pyrenew.observation.noise.HierarchicalNormalNoise : - Suitable noise model for site-level measurements + Suitable noise model for sensor-level measurements pyrenew.observation.base.BaseObservationProcess : Parent class with common observation utilities """ @@ -55,39 +56,47 @@ def __init__( temporal_pmf_rv : RandomVariable Temporal distribution PMF (e.g., shedding kinetics). noise : MeasurementNoise - Noise model (e.g., HierarchicalNormalNoise with site effects). + Noise model (e.g., HierarchicalNormalNoise with sensor effects). """ super().__init__(temporal_pmf_rv=temporal_pmf_rv) self.noise = noise + def __repr__(self) -> str: + """Return string representation.""" + return ( + f"{self.__class__.__name__}(" + f"temporal_pmf_rv={self.temporal_pmf_rv!r}, " + f"noise={self.noise!r})" + ) + def infection_resolution(self) -> str: """ - Return "site" for measurement observations. + Return "subpop" for measurement observations. - Measurement observations require site-level infections - because each measurement site corresponds to a specific catchment. + Measurement observations require subpopulation-level infections + because each measurement corresponds to a specific catchment area. Returns ------- str - ``"site"`` + ``"subpop"`` """ - return "site" + return "subpop" def sample( self, infections: ArrayLike, subpop_indices: ArrayLike, - site_indices: ArrayLike, + sensor_indices: ArrayLike, times: ArrayLike, concentrations: ArrayLike | None, - n_sites: int, - ) -> ArrayLike: + n_sensors: int, + ) -> ObservationSample: """ - Sample measurements from observed sites. + Sample measurements from observed sensors. This method does not perform runtime validation of index values - (times, subpop_indices, site_indices). Validate observation data + (times, subpop_indices, sensor_indices). Validate observation data before sampling. Transforms infections to expected values via signal-specific processing @@ -97,26 +106,26 @@ def sample( ---------- infections : ArrayLike Infections from the infection process. - Shape: (n_days, n_sites) + Shape: (n_days, n_subpops) subpop_indices : ArrayLike Subpopulation index for each observation (0-indexed). Shape: (n_obs,) - site_indices : ArrayLike - Site index for each observation (0-indexed). + sensor_indices : ArrayLike + Sensor index for each observation (0-indexed). Shape: (n_obs,) times : ArrayLike Day index for each observation (0-indexed). Shape: (n_obs,) concentrations : ArrayLike | None Observed measurements (n_obs,), or None for prior sampling. - n_sites : int - Total number of measurement sites. + n_sensors : int + Total number of measurement sensors. Returns ------- - ArrayLike - Observed or sampled measurements. - Shape: (n_obs,) + ObservationSample + Named tuple with `observed` (sampled/conditioned measurements) and + `expected` (expected values before noise, shape: n_days x n_subpops). """ expected_values = self._expected_signal(infections) @@ -124,10 +133,12 @@ def sample( expected_obs = expected_values[times, subpop_indices] - return self.noise.sample( + observed = self.noise.sample( name="concentrations", expected=expected_obs, obs=concentrations, - site_indices=site_indices, - n_sites=n_sites, + sensor_indices=sensor_indices, + n_sensors=n_sensors, ) + + return ObservationSample(observed=observed, expected=expected_values) diff --git a/pyrenew/observation/noise.py b/pyrenew/observation/noise.py index 4065061e..647b1781 100644 --- a/pyrenew/observation/noise.py +++ b/pyrenew/observation/noise.py @@ -13,8 +13,8 @@ Measurement Noise ----------------- -- ``HierarchicalNormalNoise``: Normal noise with hierarchical site effects. - Takes ``site_mode_prior_rv`` and ``site_sd_prior_rv`` for site-level +- ``HierarchicalNormalNoise``: Normal noise with hierarchical sensor effects. + Takes ``sensor_mode_prior_rv`` and ``sensor_sd_prior_rv`` for sensor-level bias and variability. """ @@ -87,6 +87,10 @@ def __init__(self) -> None: """Initialize Poisson noise (no parameters).""" pass + def __repr__(self) -> str: + """Return string representation.""" + return "PoissonNoise()" + def validate(self) -> None: """Validate Poisson noise (always valid).""" pass @@ -151,6 +155,10 @@ def __init__(self, concentration_rv: RandomVariable) -> None: """ self.concentration_rv = concentration_rv + def __repr__(self) -> str: + """Return string representation.""" + return f"NegativeBinomialNoise(concentration_rv={self.concentration_rv!r})" + def validate(self) -> None: """ Validate concentration is positive. @@ -228,7 +236,7 @@ def sample( obs : ArrayLike | None Observed measurements for conditioning, or None for prior sampling. **kwargs - Additional context (e.g., site indices). + Additional context (e.g., sensor indices). Returns ------- @@ -252,18 +260,18 @@ def validate(self) -> None: class HierarchicalNormalNoise(MeasurementNoise): """ - Normal noise with hierarchical site-level effects. + Normal noise with hierarchical sensor-level effects. - Observation model: ``obs ~ Normal(expected + site_mode, site_sd)`` - where site_mode and site_sd are hierarchically modeled. + Observation model: ``obs ~ Normal(expected + sensor_mode, sensor_sd)`` + where sensor_mode and sensor_sd are hierarchically modeled. Parameters ---------- - site_mode_prior_rv : RandomVariable - Hierarchical prior for site-level modes (log-scale biases). + sensor_mode_prior_rv : RandomVariable + Hierarchical prior for sensor-level modes (log-scale biases). Must support ``sample(n_groups=...)`` interface. - site_sd_prior_rv : RandomVariable - Hierarchical prior for site-level SDs (must be > 0). + sensor_sd_prior_rv : RandomVariable + Hierarchical prior for sensor-level SDs (must be > 0). Must support ``sample(n_groups=...)`` interface. Notes @@ -273,30 +281,38 @@ class HierarchicalNormalNoise(MeasurementNoise): See Also -------- pyrenew.randomvariable.HierarchicalNormalPrior : - Suitable prior for site_mode_prior_rv + Suitable prior for sensor_mode_prior_rv pyrenew.randomvariable.GammaGroupSdPrior : - Suitable prior for site_sd_prior_rv + Suitable prior for sensor_sd_prior_rv """ def __init__( self, - site_mode_prior_rv: RandomVariable, - site_sd_prior_rv: RandomVariable, + sensor_mode_prior_rv: RandomVariable, + sensor_sd_prior_rv: RandomVariable, ) -> None: """ Initialize hierarchical Normal noise. Parameters ---------- - site_mode_prior_rv : RandomVariable - Hierarchical prior for site-level modes (log-scale biases). + sensor_mode_prior_rv : RandomVariable + Hierarchical prior for sensor-level modes (log-scale biases). Must support ``sample(n_groups=...)`` interface. - site_sd_prior_rv : RandomVariable - Hierarchical prior for site-level SDs (must be > 0). + sensor_sd_prior_rv : RandomVariable + Hierarchical prior for sensor-level SDs (must be > 0). Must support ``sample(n_groups=...)`` interface. """ - self.site_mode_prior_rv = site_mode_prior_rv - self.site_sd_prior_rv = site_sd_prior_rv + self.sensor_mode_prior_rv = sensor_mode_prior_rv + self.sensor_sd_prior_rv = sensor_sd_prior_rv + + def __repr__(self) -> str: + """Return string representation.""" + return ( + f"HierarchicalNormalNoise(" + f"sensor_mode_prior_rv={self.sensor_mode_prior_rv!r}, " + f"sensor_sd_prior_rv={self.sensor_sd_prior_rv!r})" + ) def validate(self) -> None: """ @@ -314,11 +330,11 @@ def sample( expected: ArrayLike, obs: ArrayLike | None = None, *, - site_indices: ArrayLike, - n_sites: int, + sensor_indices: ArrayLike, + n_sensors: int, ) -> ArrayLike: """ - Sample from Normal distribution with site-level hierarchical effects. + Sample from Normal distribution with sensor-level hierarchical effects. Parameters ---------- @@ -330,27 +346,27 @@ def sample( obs : ArrayLike | None Observed log-scale measurements for conditioning. Shape: (n_obs,) - site_indices : ArrayLike - Site index for each observation (0-indexed). + sensor_indices : ArrayLike + Sensor index for each observation (0-indexed). Shape: (n_obs,) - n_sites : int - Total number of sites. + n_sensors : int + Total number of sensors. Returns ------- ArrayLike - Normal distributed measurements with hierarchical site effects. + Normal distributed measurements with hierarchical sensor effects. Shape: (n_obs,) Raises ------ ValueError - If site_sd samples non-positive values. + If sensor_sd samples non-positive values. """ - site_mode = self.site_mode_prior_rv.sample(n_groups=n_sites) - site_sd = self.site_sd_prior_rv.sample(n_groups=n_sites) + sensor_mode = self.sensor_mode_prior_rv.sample(n_groups=n_sensors) + sensor_sd = self.sensor_sd_prior_rv.sample(n_groups=n_sensors) - loc = expected + site_mode[site_indices] - scale = site_sd[site_indices] + loc = expected + sensor_mode[sensor_indices] + scale = sensor_sd[sensor_indices] return numpyro.sample(name, dist.Normal(loc=loc, scale=scale), obs=obs) diff --git a/test/conftest.py b/test/conftest.py index 73854684..6f55681a 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -127,9 +127,9 @@ def medium_shedding_pmf(): @pytest.fixture -def site_mode_prior(): +def sensor_mode_prior(): """ - Standard hierarchical normal prior for site modes. + Standard hierarchical normal prior for sensor modes. Returns ------- @@ -137,12 +137,12 @@ def site_mode_prior(): A hierarchical normal prior with standard deviation 0.5. """ return HierarchicalNormalPrior( - name="ww_site_mode", sd_rv=DeterministicVariable("mode_sd", 0.5) + name="ww_sensor_mode", sd_rv=DeterministicVariable("mode_sd", 0.5) ) @pytest.fixture -def site_mode_prior_tight(): +def sensor_mode_prior_tight(): """ Tight hierarchical normal prior for deterministic-like behavior. @@ -152,14 +152,14 @@ def site_mode_prior_tight(): A hierarchical normal prior with small standard deviation 0.01. """ return HierarchicalNormalPrior( - name="ww_site_mode", sd_rv=DeterministicVariable("mode_sd_tight", 0.01) + name="ww_sensor_mode", sd_rv=DeterministicVariable("mode_sd_tight", 0.01) ) @pytest.fixture -def site_sd_prior(): +def sensor_sd_prior(): """ - Standard gamma prior for site standard deviations. + Standard gamma prior for sensor standard deviations. Returns ------- @@ -167,7 +167,7 @@ def site_sd_prior(): A gamma prior for group standard deviations. """ return GammaGroupSdPrior( - name="ww_site_sd", + name="ww_sensor_sd", sd_mean_rv=DeterministicVariable("sd_mean", 0.3), sd_concentration_rv=DeterministicVariable("sd_concentration", 4.0), sd_min=0.10, @@ -175,7 +175,7 @@ def site_sd_prior(): @pytest.fixture -def site_sd_prior_tight(): +def sensor_sd_prior_tight(): """ Tight gamma prior for deterministic-like behavior. @@ -185,7 +185,7 @@ def site_sd_prior_tight(): A gamma prior with small mean for tight behavior. """ return GammaGroupSdPrior( - name="ww_site_sd", + name="ww_sensor_sd", sd_mean_rv=DeterministicVariable("sd_mean_tight", 0.01), sd_concentration_rv=DeterministicVariable("sd_concentration_tight", 4.0), sd_min=0.005, diff --git a/test/test_observation_counts.py b/test/test_observation_counts.py index 58f0b33e..ddb4d684 100644 --- a/test/test_observation_counts.py +++ b/test/test_observation_counts.py @@ -12,7 +12,7 @@ from pyrenew.deterministic import DeterministicPMF, DeterministicVariable from pyrenew.observation import ( Counts, - CountsBySite, + CountsBySubpop, NegativeBinomialNoise, PoissonNoise, ) @@ -66,13 +66,14 @@ def test_sample_returns_correct_shape(self, counts_process): infections = jnp.ones(30) * 100 with numpyro.handlers.seed(rng_seed=42): - counts = counts_process.sample( + result = counts_process.sample( infections=infections, counts=None, ) - assert counts.shape[0] > 0 - assert counts.ndim == 1 + assert result.observed.shape[0] > 0 + assert result.observed.ndim == 1 + assert result.expected.shape == infections.shape def test_delay_convolution(self, counts_factory, short_delay_pmf): """Test that delay is properly applied.""" @@ -82,16 +83,16 @@ def test_delay_convolution(self, counts_factory, short_delay_pmf): infections = infections.at[10].set(1000) with numpyro.handlers.seed(rng_seed=42): - counts = process.sample( + result = process.sample( infections=infections, counts=None, ) # Timeline alignment: output length equals input length - assert counts.shape[0] == len(infections) + assert result.observed.shape[0] == len(infections) # First len(delay_pmf)-1 days are NaN (appear as -1 after NegativeBinomial sampling) - assert jnp.all(counts[1:] >= 0) - assert jnp.sum(counts[counts >= 0]) > 0 + assert jnp.all(result.observed[1:] >= 0) + assert jnp.sum(result.observed[result.observed >= 0]) > 0 def test_ascertainment_scaling(self, counts_factory, simple_delay_pmf): """Test that ascertainment rate properly scales counts.""" @@ -105,11 +106,11 @@ def test_ascertainment_scaling(self, counts_factory, simple_delay_pmf): ) with numpyro.handlers.seed(rng_seed=42): - counts = process.sample( + result = process.sample( infections=infections, counts=None, ) - results.append(jnp.mean(counts)) + results.append(jnp.mean(result.observed)) # Higher ascertainment rate should lead to more counts assert results[1] > results[0] @@ -127,11 +128,11 @@ def test_negative_binomial_observation(self, counts_factory, simple_delay_pmf): samples = [] for seed in range(5): with numpyro.handlers.seed(rng_seed=seed): - counts = process.sample( + result = process.sample( infections=infections, counts=None, ) - samples.append(jnp.sum(counts)) + samples.append(jnp.sum(result.observed)) # Should have some variability due to negative binomial sampling assert jnp.std(jnp.array(samples)) > 0 @@ -155,14 +156,14 @@ def test_with_stochastic_ascertainment(self, short_shedding_pmf): infections = jnp.ones(20) * 100 with numpyro.handlers.seed(rng_seed=42): - counts = process.sample( + result = process.sample( infections=infections, counts=None, ) - assert counts.shape[0] > 0 + assert result.observed.shape[0] > 0 # Skip NaN padding - valid_counts = counts[2:] + valid_counts = result.observed[2:] assert jnp.all(valid_counts >= 0) def test_with_stochastic_concentration(self, simple_delay_pmf): @@ -180,13 +181,13 @@ def test_with_stochastic_concentration(self, simple_delay_pmf): infections = jnp.ones(20) * 100 with numpyro.handlers.seed(rng_seed=42): - counts = process.sample( + result = process.sample( infections=infections, counts=None, ) - assert counts.shape[0] > 0 - assert jnp.all(counts >= 0) + assert result.observed.shape[0] > 0 + assert jnp.all(result.observed >= 0) class TestCountsEdgeCases: @@ -197,25 +198,25 @@ def test_zero_infections(self, counts_process): infections = jnp.zeros(20) with numpyro.handlers.seed(rng_seed=42): - counts = counts_process.sample( + result = counts_process.sample( infections=infections, counts=None, ) - assert counts.shape[0] > 0 + assert result.observed.shape[0] > 0 def test_small_infections(self, counts_process): """Test with small infection values.""" infections = jnp.ones(20) * 10 with numpyro.handlers.seed(rng_seed=42): - counts = counts_process.sample( + result = counts_process.sample( infections=infections, counts=None, ) - assert counts.shape[0] > 0 - assert jnp.all(counts >= 0) + assert result.observed.shape[0] > 0 + assert jnp.all(result.observed >= 0) def test_long_delay_distribution(self, counts_factory, long_delay_pmf): """Test with longer delay distribution.""" @@ -224,15 +225,15 @@ def test_long_delay_distribution(self, counts_factory, long_delay_pmf): infections = create_mock_infections(40, peak_day=20, shape="spike") with numpyro.handlers.seed(rng_seed=42): - counts = process.sample( + result = process.sample( infections=infections, counts=None, ) # Timeline alignment maintained - assert counts.shape[0] == infections.shape[0] + assert result.observed.shape[0] == infections.shape[0] # Skip NaN padding: 10-day delay -> first 9 days are NaN - valid_counts = counts[9:] + valid_counts = result.observed[9:] assert jnp.sum(valid_counts) > 0 @@ -249,14 +250,14 @@ def test_sparse_observations(self, counts_process): counts_data = jnp.array([10, 12, 8, 15]) with numpyro.handlers.seed(rng_seed=42): - counts = counts_process.sample( + result = counts_process.sample( infections=infections, counts=counts_data, times=times, ) - assert counts.shape == times.shape - assert jnp.allclose(counts, counts_data) + assert result.observed.shape == times.shape + assert jnp.allclose(result.observed, counts_data) def test_sparse_vs_dense_sampling(self, counts_process): """Test that sparse sampling gives different output shape than dense.""" @@ -265,7 +266,7 @@ def test_sparse_vs_dense_sampling(self, counts_process): # Dense: prior sampling (counts=None, no times) with numpyro.handlers.seed(rng_seed=42): - dense_counts = counts_process.sample( + dense_result = counts_process.sample( infections=infections, counts=None, ) @@ -274,18 +275,18 @@ def test_sparse_vs_dense_sampling(self, counts_process): times = jnp.array([5, 10, 15, 20]) sparse_obs_data = jnp.array([10, 12, 8, 15]) with numpyro.handlers.seed(rng_seed=42): - sparse_counts = counts_process.sample( + sparse_result = counts_process.sample( infections=infections, counts=sparse_obs_data, times=times, ) # Dense prior produces full length output - assert dense_counts.shape == (n_days,) + assert dense_result.observed.shape == (n_days,) # Sparse observations produce output matching times shape - assert sparse_counts.shape == times.shape - assert jnp.allclose(sparse_counts, sparse_obs_data) + assert sparse_result.observed.shape == times.shape + assert jnp.allclose(sparse_result.observed, sparse_obs_data) def test_prior_sampling_ignores_times(self, counts_process): """Test that times parameter is ignored when counts=None (prior sampling).""" @@ -295,61 +296,62 @@ def test_prior_sampling_ignores_times(self, counts_process): # When counts=None, times is ignored - output is dense with numpyro.handlers.seed(rng_seed=42): - prior_with_times = counts_process.sample( + result_with_times = counts_process.sample( infections=infections, counts=None, times=times, ) with numpyro.handlers.seed(rng_seed=42): - prior_without_times = counts_process.sample( + result_without_times = counts_process.sample( infections=infections, counts=None, ) # Both should produce dense output of shape (n_days,) - assert prior_with_times.shape == (n_days,) - assert prior_without_times.shape == (n_days,) + assert result_with_times.observed.shape == (n_days,) + assert result_without_times.observed.shape == (n_days,) # With same seed, outputs should be identical - assert jnp.allclose(prior_with_times, prior_without_times) + assert jnp.allclose(result_with_times.observed, result_without_times.observed) -class TestCountsBySite: - """Test CountsBySite for disaggregated observations.""" +class TestCountsBySubpop: + """Test CountsBySubpop for subpopulation-level observations.""" def test_sample_returns_correct_shape(self): - """Test that CountsBySite sample returns correct shape.""" + """Test that CountsBySubpop sample returns correct shape.""" delay_pmf = jnp.array([0.3, 0.4, 0.3]) - process = CountsBySite( + process = CountsBySubpop( ascertainment_rate_rv=DeterministicVariable("ihr", 0.02), delay_distribution_rv=DeterministicPMF("delay", delay_pmf), noise=PoissonNoise(), ) - infections = jnp.ones((30, 3)) * 500 # 30 days, 3 sites + infections = jnp.ones((30, 3)) * 500 # 30 days, 3 subpops times = jnp.array([10, 15, 10, 15]) subpop_indices = jnp.array([0, 0, 1, 1]) with numpyro.handlers.seed(rng_seed=42): - counts = process.sample( + result = process.sample( infections=infections, subpop_indices=subpop_indices, times=times, counts=None, ) - assert counts.shape == times.shape + assert result.observed.shape == times.shape + assert result.expected.shape == infections.shape def test_infection_resolution(self): - """Test that CountsBySite returns 'site' resolution.""" + """Test that CountsBySubpop returns 'subpop' resolution.""" delay_pmf = jnp.array([1.0]) - process = CountsBySite( + process = CountsBySubpop( ascertainment_rate_rv=DeterministicVariable("ihr", 0.01), delay_distribution_rv=DeterministicPMF("delay", delay_pmf), noise=PoissonNoise(), ) - assert process.infection_resolution() == "site" + assert process.infection_resolution() == "subpop" class TestPoissonNoise: @@ -366,13 +368,13 @@ def test_poisson_counts(self, simple_delay_pmf): infections = jnp.ones(20) * 1000 with numpyro.handlers.seed(rng_seed=42): - counts = process.sample( + result = process.sample( infections=infections, counts=None, ) - assert counts.shape[0] == 20 - assert jnp.all(counts >= 0) + assert result.observed.shape[0] == 20 + assert jnp.all(result.observed >= 0) class TestCountBaseInternalMethods: @@ -435,30 +437,30 @@ def test_validate_invalid_ascertainment_rate_greater_than_one( with pytest.raises(ValueError, match="ascertainment_rate_rv must be in"): process.validate() - def test_get_required_lookback(self, simple_delay_pmf, long_delay_pmf): - """Test get_required_lookback returns PMF length.""" + def test_lookback_days(self, simple_delay_pmf, long_delay_pmf): + """Test lookback_days returns PMF length.""" process_short = Counts( ascertainment_rate_rv=DeterministicVariable("ihr", 0.01), delay_distribution_rv=DeterministicPMF("delay", simple_delay_pmf), noise=NegativeBinomialNoise(DeterministicVariable("conc", 10.0)), ) - assert process_short.get_required_lookback() == 1 + assert process_short.lookback_days() == 1 process_long = Counts( ascertainment_rate_rv=DeterministicVariable("ihr", 0.01), delay_distribution_rv=DeterministicPMF("delay", long_delay_pmf), noise=NegativeBinomialNoise(DeterministicVariable("conc", 10.0)), ) - assert process_long.get_required_lookback() == 10 + assert process_long.lookback_days() == 10 def test_infection_resolution_counts(self, simple_delay_pmf): - """Test that Counts returns 'jurisdiction' resolution.""" + """Test that Counts returns 'aggregate' resolution.""" process = Counts( ascertainment_rate_rv=DeterministicVariable("ihr", 0.01), delay_distribution_rv=DeterministicPMF("delay", simple_delay_pmf), noise=NegativeBinomialNoise(DeterministicVariable("conc", 10.0)), ) - assert process.infection_resolution() == "jurisdiction" + assert process.infection_resolution() == "aggregate" class TestNoiseValidation: diff --git a/test/test_observation_measurements.py b/test/test_observation_measurements.py index 5e69fd3d..e5c014a2 100644 --- a/test/test_observation_measurements.py +++ b/test/test_observation_measurements.py @@ -27,7 +27,7 @@ def validate(self) -> None: pmf = self.temporal_pmf_rv() self._validate_pmf(pmf, "temporal_pmf_rv") - def get_required_lookback(self) -> int: + def lookback_days(self) -> int: """ Return temporal PMF length. @@ -49,7 +49,7 @@ def _expected_signal(self, infections): """ pmf = self.temporal_pmf_rv() - # Handle 2D infections (n_days, n_sites) + # Handle 2D infections (n_days, n_subpops) if infections.ndim == 1: infections = infections[:, jnp.newaxis] @@ -73,25 +73,25 @@ def test_is_base_observation_process(self): """Test that Measurements inherits from BaseObservationProcess.""" assert issubclass(Measurements, BaseObservationProcess) - def test_infection_resolution_is_site(self): - """Test that Measurements returns 'site' resolution.""" + def test_infection_resolution_is_subpop(self): + """Test that Measurements returns 'subpop' resolution.""" shedding_pmf = jnp.array([0.3, 0.4, 0.3]) - site_mode_prior = HierarchicalNormalPrior( + sensor_mode_prior = HierarchicalNormalPrior( name="mode", sd_rv=DeterministicVariable("sd", 0.5) ) - site_sd_prior = GammaGroupSdPrior( + sensor_sd_prior = GammaGroupSdPrior( name="sd", sd_mean_rv=DeterministicVariable("mean", 0.3), sd_concentration_rv=DeterministicVariable("conc", 4.0), ) - noise = HierarchicalNormalNoise(site_mode_prior, site_sd_prior) + noise = HierarchicalNormalNoise(sensor_mode_prior, sensor_sd_prior) process = ConcreteMeasurements( temporal_pmf_rv=DeterministicPMF("shedding", shedding_pmf), noise=noise, ) - assert process.infection_resolution() == "site" + assert process.infection_resolution() == "subpop" class TestHierarchicalNormalNoise: @@ -99,67 +99,67 @@ class TestHierarchicalNormalNoise: def test_validate(self): """Test HierarchicalNormalNoise validate method.""" - site_mode_prior = HierarchicalNormalPrior( + sensor_mode_prior = HierarchicalNormalPrior( name="mode", sd_rv=DeterministicVariable("mode_sd", 0.5) ) - site_sd_prior = GammaGroupSdPrior( + sensor_sd_prior = GammaGroupSdPrior( name="sd", sd_mean_rv=DeterministicVariable("sd_mean", 0.3), sd_concentration_rv=DeterministicVariable("sd_conc", 4.0), ) - noise = HierarchicalNormalNoise(site_mode_prior, site_sd_prior) + noise = HierarchicalNormalNoise(sensor_mode_prior, sensor_sd_prior) # Should not raise - validation is deferred to sample time noise.validate() def test_sample_shape(self): """Test that HierarchicalNormalNoise produces correct shape.""" - site_mode_prior = HierarchicalNormalPrior( + sensor_mode_prior = HierarchicalNormalPrior( name="mode", sd_rv=DeterministicVariable("mode_sd", 0.5) ) - site_sd_prior = GammaGroupSdPrior( + sensor_sd_prior = GammaGroupSdPrior( name="sd", sd_mean_rv=DeterministicVariable("sd_mean", 0.3), sd_concentration_rv=DeterministicVariable("sd_conc", 4.0), ) - noise = HierarchicalNormalNoise(site_mode_prior, site_sd_prior) + noise = HierarchicalNormalNoise(sensor_mode_prior, sensor_sd_prior) expected = jnp.array([1.0, 2.0, 3.0, 4.0]) - site_indices = jnp.array([0, 0, 1, 1]) + sensor_indices = jnp.array([0, 0, 1, 1]) with numpyro.handlers.seed(rng_seed=42): samples = noise.sample( name="test", expected=expected, obs=None, - site_indices=site_indices, - n_sites=2, + sensor_indices=sensor_indices, + n_sensors=2, ) assert samples.shape == expected.shape def test_sample_with_observations(self): """Test that HierarchicalNormalNoise conditions on observations.""" - site_mode_prior = HierarchicalNormalPrior( + sensor_mode_prior = HierarchicalNormalPrior( name="mode", sd_rv=DeterministicVariable("mode_sd", 0.5) ) - site_sd_prior = GammaGroupSdPrior( + sensor_sd_prior = GammaGroupSdPrior( name="sd", sd_mean_rv=DeterministicVariable("sd_mean", 0.3), sd_concentration_rv=DeterministicVariable("sd_conc", 4.0), ) - noise = HierarchicalNormalNoise(site_mode_prior, site_sd_prior) + noise = HierarchicalNormalNoise(sensor_mode_prior, sensor_sd_prior) expected = jnp.array([1.0, 2.0, 3.0, 4.0]) obs = jnp.array([1.1, 2.1, 3.1, 4.1]) - site_indices = jnp.array([0, 0, 1, 1]) + sensor_indices = jnp.array([0, 0, 1, 1]) with numpyro.handlers.seed(rng_seed=42): samples = noise.sample( name="test", expected=expected, obs=obs, - site_indices=site_indices, - n_sites=2, + sensor_indices=sensor_indices, + n_sensors=2, ) # When obs is provided, samples should equal obs @@ -172,52 +172,53 @@ class TestConcreteMeasurements: def test_sample_shape(self): """Test that sample returns correct shape.""" shedding_pmf = jnp.array([0.3, 0.4, 0.3]) - site_mode_prior = HierarchicalNormalPrior( + sensor_mode_prior = HierarchicalNormalPrior( name="mode", sd_rv=DeterministicVariable("mode_sd", 0.5) ) - site_sd_prior = GammaGroupSdPrior( + sensor_sd_prior = GammaGroupSdPrior( name="sd", sd_mean_rv=DeterministicVariable("sd_mean", 0.3), sd_concentration_rv=DeterministicVariable("sd_conc", 4.0), ) - noise = HierarchicalNormalNoise(site_mode_prior, site_sd_prior) + noise = HierarchicalNormalNoise(sensor_mode_prior, sensor_sd_prior) process = ConcreteMeasurements( temporal_pmf_rv=DeterministicPMF("shedding", shedding_pmf), noise=noise, ) - # 30 days, 2 sites + # 30 days, 2 subpops infections = jnp.ones((30, 2)) * 1000 subpop_indices = jnp.array([0, 0, 1, 1]) - site_indices = jnp.array([0, 0, 1, 1]) + sensor_indices = jnp.array([0, 0, 1, 1]) times = jnp.array([10, 15, 10, 15]) with numpyro.handlers.seed(rng_seed=42): - samples = process.sample( + result = process.sample( infections=infections, subpop_indices=subpop_indices, - site_indices=site_indices, + sensor_indices=sensor_indices, times=times, concentrations=None, - n_sites=2, + n_sensors=2, ) - assert samples.shape == times.shape + assert result.observed.shape == times.shape + assert result.expected.shape == infections.shape def test_expected_signal_stored(self): """Test that expected_log_conc is stored as deterministic.""" shedding_pmf = jnp.array([0.5, 0.5]) - site_mode_prior = HierarchicalNormalPrior( + sensor_mode_prior = HierarchicalNormalPrior( name="mode", sd_rv=DeterministicVariable("mode_sd", 0.01) ) - site_sd_prior = GammaGroupSdPrior( + sensor_sd_prior = GammaGroupSdPrior( name="sd", sd_mean_rv=DeterministicVariable("sd_mean", 0.01), sd_concentration_rv=DeterministicVariable("sd_conc", 4.0), sd_min=0.001, ) - noise = HierarchicalNormalNoise(site_mode_prior, site_sd_prior) + noise = HierarchicalNormalNoise(sensor_mode_prior, sensor_sd_prior) process = ConcreteMeasurements( temporal_pmf_rv=DeterministicPMF("shedding", shedding_pmf), @@ -226,7 +227,7 @@ def test_expected_signal_stored(self): infections = jnp.ones((20, 2)) * 1000 subpop_indices = jnp.array([0, 1]) - site_indices = jnp.array([0, 1]) + sensor_indices = jnp.array([0, 1]) times = jnp.array([10, 10]) with numpyro.handlers.seed(rng_seed=42): @@ -234,10 +235,10 @@ def test_expected_signal_stored(self): lambda: process.sample( infections=infections, subpop_indices=subpop_indices, - site_indices=site_indices, + sensor_indices=sensor_indices, times=times, concentrations=None, - n_sites=2, + n_sensors=2, ) ).get_trace() From 57d2fbaacd229768d7989f1a31448de16401f34d Mon Sep 17 00:00:00 2001 From: Mitzi Morris Date: Fri, 26 Dec 2025 16:05:17 -0500 Subject: [PATCH 06/81] consistent names: 'subpop' (not site), 'sensor' for site/lab measurements, 'aggregate' instead of 'jurisdiction' --- docs/tutorials/observation_processes_measurements.qmd | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/docs/tutorials/observation_processes_measurements.qmd b/docs/tutorials/observation_processes_measurements.qmd index 613e1176..21dbff21 100644 --- a/docs/tutorials/observation_processes_measurements.qmd +++ b/docs/tutorials/observation_processes_measurements.qmd @@ -413,8 +413,12 @@ infections_constant = jnp.ones((40, 1)) * 1500.0 # Each sensor samples at multiple time points times_per_sensor = 10 -observation_times = jnp.tile(jnp.arange(10, 30, 2, dtype=jnp.int32), num_sensors) -sensor_ids = jnp.repeat(jnp.arange(num_sensors, dtype=jnp.int32), times_per_sensor) +observation_times = jnp.tile( + jnp.arange(10, 30, 2, dtype=jnp.int32), num_sensors +) +sensor_ids = jnp.repeat( + jnp.arange(num_sensors, dtype=jnp.int32), times_per_sensor +) subpop_ids = jnp.zeros(num_sensors * times_per_sensor, dtype=jnp.int32) with numpyro.handlers.seed(rng_seed=42): From 7efb52413894cdbd861fd1c8c83680f5ce3a2628 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 26 Dec 2025 21:05:27 +0000 Subject: [PATCH 07/81] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- pyrenew/observation/__init__.py | 3 ++- pyrenew/observation/count_observations.py | 2 +- pyrenew/observation/measurements.py | 2 +- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/pyrenew/observation/__init__.py b/pyrenew/observation/__init__.py index 8a0cdeab..7c8972d8 100644 --- a/pyrenew/observation/__init__.py +++ b/pyrenew/observation/__init__.py @@ -20,6 +20,8 @@ to observation constructors to control the output distribution. """ +from pyrenew.observation.types import ObservationSample + from pyrenew.observation.base import BaseObservationProcess from pyrenew.observation.count_observations import Counts, CountsBySubpop from pyrenew.observation.measurements import Measurements @@ -32,7 +34,6 @@ PoissonNoise, ) from pyrenew.observation.poisson import PoissonObservation -from pyrenew.observation.types import ObservationSample __all__ = [ # Existing (kept for backward compatibility) diff --git a/pyrenew/observation/count_observations.py b/pyrenew/observation/count_observations.py index 7a883932..135ccca3 100644 --- a/pyrenew/observation/count_observations.py +++ b/pyrenew/observation/count_observations.py @@ -10,11 +10,11 @@ import jax import jax.numpy as jnp from jax.typing import ArrayLike +from pyrenew.observation.types import ObservationSample from pyrenew.metaclass import RandomVariable from pyrenew.observation.base import BaseObservationProcess from pyrenew.observation.noise import CountNoise -from pyrenew.observation.types import ObservationSample class _CountBase(BaseObservationProcess): diff --git a/pyrenew/observation/measurements.py b/pyrenew/observation/measurements.py index 33e38980..6826dff8 100644 --- a/pyrenew/observation/measurements.py +++ b/pyrenew/observation/measurements.py @@ -7,11 +7,11 @@ """ from jax.typing import ArrayLike +from pyrenew.observation.types import ObservationSample from pyrenew.metaclass import RandomVariable from pyrenew.observation.base import BaseObservationProcess from pyrenew.observation.noise import MeasurementNoise -from pyrenew.observation.types import ObservationSample class Measurements(BaseObservationProcess): From 8a7947f383637810a6dd8d503dd562d858eefca8 Mon Sep 17 00:00:00 2001 From: Mitzi Morris Date: Mon, 29 Dec 2025 10:13:46 -0500 Subject: [PATCH 08/81] add observation types --- pyrenew/observation/types.py | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) create mode 100644 pyrenew/observation/types.py diff --git a/pyrenew/observation/types.py b/pyrenew/observation/types.py new file mode 100644 index 00000000..b494e2e6 --- /dev/null +++ b/pyrenew/observation/types.py @@ -0,0 +1,28 @@ +# numpydoc ignore=GL08 +""" +Return types for observation processes. + +Named tuples providing structured access to observation process outputs. +""" + +from typing import NamedTuple + +from jax.typing import ArrayLike + + +class ObservationSample(NamedTuple): + """ + Return type for observation process sample() methods. + + Attributes + ---------- + observed : ArrayLike + Sampled or conditioned observations. Shape depends on the + observation process and indexing. + expected : ArrayLike + Expected values before noise is applied. Useful for + diagnostics and posterior predictive checks. + """ + + observed: ArrayLike + expected: ArrayLike From 6dff8cdf0de75f6f7e18df5c797f44153b299e07 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 29 Dec 2025 15:14:34 +0000 Subject: [PATCH 09/81] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- pyrenew/observation/__init__.py | 3 +-- pyrenew/observation/count_observations.py | 2 +- pyrenew/observation/measurements.py | 2 +- 3 files changed, 3 insertions(+), 4 deletions(-) diff --git a/pyrenew/observation/__init__.py b/pyrenew/observation/__init__.py index 7c8972d8..8a0cdeab 100644 --- a/pyrenew/observation/__init__.py +++ b/pyrenew/observation/__init__.py @@ -20,8 +20,6 @@ to observation constructors to control the output distribution. """ -from pyrenew.observation.types import ObservationSample - from pyrenew.observation.base import BaseObservationProcess from pyrenew.observation.count_observations import Counts, CountsBySubpop from pyrenew.observation.measurements import Measurements @@ -34,6 +32,7 @@ PoissonNoise, ) from pyrenew.observation.poisson import PoissonObservation +from pyrenew.observation.types import ObservationSample __all__ = [ # Existing (kept for backward compatibility) diff --git a/pyrenew/observation/count_observations.py b/pyrenew/observation/count_observations.py index 135ccca3..7a883932 100644 --- a/pyrenew/observation/count_observations.py +++ b/pyrenew/observation/count_observations.py @@ -10,11 +10,11 @@ import jax import jax.numpy as jnp from jax.typing import ArrayLike -from pyrenew.observation.types import ObservationSample from pyrenew.metaclass import RandomVariable from pyrenew.observation.base import BaseObservationProcess from pyrenew.observation.noise import CountNoise +from pyrenew.observation.types import ObservationSample class _CountBase(BaseObservationProcess): diff --git a/pyrenew/observation/measurements.py b/pyrenew/observation/measurements.py index 6826dff8..33e38980 100644 --- a/pyrenew/observation/measurements.py +++ b/pyrenew/observation/measurements.py @@ -7,11 +7,11 @@ """ from jax.typing import ArrayLike -from pyrenew.observation.types import ObservationSample from pyrenew.metaclass import RandomVariable from pyrenew.observation.base import BaseObservationProcess from pyrenew.observation.noise import MeasurementNoise +from pyrenew.observation.types import ObservationSample class Measurements(BaseObservationProcess): From 90e1ad04a52a554f774e736afc4458bf22146e17 Mon Sep 17 00:00:00 2001 From: Mitzi Morris Date: Fri, 2 Jan 2026 15:33:57 -0500 Subject: [PATCH 10/81] Update docs/tutorials/observation_processes_counts.qmd Co-authored-by: Dylan H. Morris --- docs/tutorials/observation_processes_counts.qmd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/tutorials/observation_processes_counts.qmd b/docs/tutorials/observation_processes_counts.qmd index b3b415c7..0bda5fb8 100644 --- a/docs/tutorials/observation_processes_counts.qmd +++ b/docs/tutorials/observation_processes_counts.qmd @@ -33,7 +33,7 @@ $$\lambda_t = \alpha \cdot \sum_{d=0}^{D} I_{t-d} \cdot p_d$$ where: - $I_{t-d}$ is the number of incident (new) infections on day $t-d$ (i.e., $d$ days before day $t$) -- $\alpha$ is the ascertainment rate (e.g., infection-hospitalization ratio) +- $\alpha$ is the rate of ascertained counts per infection (e.g., infection-to-hospital admission rate). This can model a mix of biological effects (e.g. some percentage of infections lead to hospital admissions, but not all) and reporting effects (e.g. some percentage of admissions that occur are reported, but not all). - $p_d$ is the delay distribution from infection to observation, conditional on an infection leading to an observation - $D$ is the maximum delay From c752be5ed03440b23807ed4091f0d38cc3e20ffb Mon Sep 17 00:00:00 2001 From: Mitzi Morris Date: Fri, 2 Jan 2026 15:34:24 -0500 Subject: [PATCH 11/81] Update docs/tutorials/observation_processes_counts.qmd Co-authored-by: Dylan H. Morris --- docs/tutorials/observation_processes_counts.qmd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/tutorials/observation_processes_counts.qmd b/docs/tutorials/observation_processes_counts.qmd index 0bda5fb8..2c574e83 100644 --- a/docs/tutorials/observation_processes_counts.qmd +++ b/docs/tutorials/observation_processes_counts.qmd @@ -24,7 +24,7 @@ from pyrenew import datasets Count observation processes model the lag between infections and an observed outcome such as hospital admissions, emergency department visits, confirmed cases, or deaths. Observed data can be aggregated or available as subpopulation-level counts, which are modeled by classes `Counts` and `CountsBySubpop`, respectively. -Count observation processes transform infections into expected observed counts by applying an ascertainment rate and convolving with a delay distribution. +Count observation processes transform infections into predicted counts by applying an event probability and/or ascertainment rate and convolving with a delay distribution. The expected observations on day $t$ are: From f136bd742c6390f2b7e640c68ec56b5218db70ac Mon Sep 17 00:00:00 2001 From: Mitzi Morris Date: Fri, 2 Jan 2026 15:34:37 -0500 Subject: [PATCH 12/81] Update docs/tutorials/observation_processes_counts.qmd Co-authored-by: Dylan H. Morris --- docs/tutorials/observation_processes_counts.qmd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/tutorials/observation_processes_counts.qmd b/docs/tutorials/observation_processes_counts.qmd index 2c574e83..b8da8147 100644 --- a/docs/tutorials/observation_processes_counts.qmd +++ b/docs/tutorials/observation_processes_counts.qmd @@ -26,7 +26,7 @@ Observed data can be aggregated or available as subpopulation-level counts, whic Count observation processes transform infections into predicted counts by applying an event probability and/or ascertainment rate and convolving with a delay distribution. -The expected observations on day $t$ are: +The predicted observations on day $t$ are: $$\lambda_t = \alpha \cdot \sum_{d=0}^{D} I_{t-d} \cdot p_d$$ From f93d081cc3b1073cd350ec9ed088918b3e401b35 Mon Sep 17 00:00:00 2001 From: Mitzi Morris Date: Fri, 2 Jan 2026 15:34:56 -0500 Subject: [PATCH 13/81] Update docs/tutorials/observation_processes_counts.qmd Co-authored-by: Dylan H. Morris --- docs/tutorials/observation_processes_counts.qmd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/tutorials/observation_processes_counts.qmd b/docs/tutorials/observation_processes_counts.qmd index b8da8147..b263d8ab 100644 --- a/docs/tutorials/observation_processes_counts.qmd +++ b/docs/tutorials/observation_processes_counts.qmd @@ -37,7 +37,7 @@ where: - $p_d$ is the delay distribution from infection to observation, conditional on an infection leading to an observation - $D$ is the maximum delay -Discrete observations are generated by sampling from a noise distribution—either Poisson or negative binomial—to model reporting variability. +Discrete observations are generated by sampling from a noise distribution—e.g. Poisson or negative binomial—to model reporting variability. Poisson assumes variance equals the mean; negative binomial accommodates the overdispersion common in surveillance data. **Note on terminology:** In real-world inference, infections are *latent* (unobserved) and must be estimated from observed data like hospital admissions. In this tutorial, we simulate the observation process by specifying infections directly and showing how they produce hospital admissions through convolution and sampling. From 648b2c76b9dbe9f9365d3fe410ca0b03cb3f32ca Mon Sep 17 00:00:00 2001 From: Mitzi Morris Date: Fri, 2 Jan 2026 15:35:21 -0500 Subject: [PATCH 14/81] Update docs/tutorials/observation_processes_counts.qmd Co-authored-by: Dylan H. Morris --- docs/tutorials/observation_processes_counts.qmd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/tutorials/observation_processes_counts.qmd b/docs/tutorials/observation_processes_counts.qmd index b263d8ab..426a9c53 100644 --- a/docs/tutorials/observation_processes_counts.qmd +++ b/docs/tutorials/observation_processes_counts.qmd @@ -40,7 +40,7 @@ where: Discrete observations are generated by sampling from a noise distribution—e.g. Poisson or negative binomial—to model reporting variability. Poisson assumes variance equals the mean; negative binomial accommodates the overdispersion common in surveillance data. -**Note on terminology:** In real-world inference, infections are *latent* (unobserved) and must be estimated from observed data like hospital admissions. In this tutorial, we simulate the observation process by specifying infections directly and showing how they produce hospital admissions through convolution and sampling. +**Note on terminology:** In real-world inference, incident infections are typically a *latent* (unobserved) quantity and must be estimated from observed data like hospital admissions. In this tutorial, we simulate the observation process by specifying infections directly and showing how they produce hospital admissions through convolution and sampling. ## Hospital admissions example From 27f057789f5481c58726875cc860a44df6966ed6 Mon Sep 17 00:00:00 2001 From: Mitzi Morris Date: Tue, 6 Jan 2026 22:07:27 -0500 Subject: [PATCH 15/81] Update docs/tutorials/observation_processes_measurements.qmd Co-authored-by: Dylan H. Morris --- docs/tutorials/observation_processes_measurements.qmd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/tutorials/observation_processes_measurements.qmd b/docs/tutorials/observation_processes_measurements.qmd index 21dbff21..37d8f6b2 100644 --- a/docs/tutorials/observation_processes_measurements.qmd +++ b/docs/tutorials/observation_processes_measurements.qmd @@ -22,7 +22,7 @@ from pyrenew.deterministic import DeterministicVariable, DeterministicPMF ## Overview -Measurement observation processes model continuous signals derived from infections, such as viral RNA concentrations in wastewater. Unlike count observations (hospital admissions, deaths), measurements are continuous values that span orders of magnitude. +Measurement observation processes model continuous signals derived from infections, such as viral RNA concentrations in wastewater. Unlike count observations (hospital admissions, deaths), measurements are continuous values and in some cases may be negative. The expected measurement on day $t$ is: From d9749c7299617dfb1da58daabd34fa71d76fe14c Mon Sep 17 00:00:00 2001 From: Mitzi Morris Date: Tue, 6 Jan 2026 22:07:44 -0500 Subject: [PATCH 16/81] Update docs/tutorials/observation_processes_measurements.qmd Co-authored-by: Dylan H. Morris --- docs/tutorials/observation_processes_measurements.qmd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/tutorials/observation_processes_measurements.qmd b/docs/tutorials/observation_processes_measurements.qmd index 37d8f6b2..61d794b2 100644 --- a/docs/tutorials/observation_processes_measurements.qmd +++ b/docs/tutorials/observation_processes_measurements.qmd @@ -31,7 +31,7 @@ $$\lambda_t = \frac{G}{V} \cdot \sum_{d=0}^{D} I_{t-d} \cdot p_d$$ where: - $I_{t-d}$ is the number of incident (new) infections on day $t-d$ -- $G$ is a scaling factor (e.g., genome copies shed per infection) +- $G$ is a scaling factor (here, genome copies shed per infection) - $V$ is a normalization factor (e.g., wastewater volume per person per day) - $p_d$ is the temporal distribution (e.g., viral shedding kinetics) - $D$ is the maximum duration From e2a955b9b75effa3d4ba4ee7c0e7220581e2b59a Mon Sep 17 00:00:00 2001 From: Mitzi Morris Date: Tue, 6 Jan 2026 22:07:58 -0500 Subject: [PATCH 17/81] Update docs/tutorials/observation_processes_measurements.qmd Co-authored-by: Dylan H. Morris --- docs/tutorials/observation_processes_measurements.qmd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/tutorials/observation_processes_measurements.qmd b/docs/tutorials/observation_processes_measurements.qmd index 61d794b2..ab9e1948 100644 --- a/docs/tutorials/observation_processes_measurements.qmd +++ b/docs/tutorials/observation_processes_measurements.qmd @@ -24,7 +24,7 @@ from pyrenew.deterministic import DeterministicVariable, DeterministicPMF Measurement observation processes model continuous signals derived from infections, such as viral RNA concentrations in wastewater. Unlike count observations (hospital admissions, deaths), measurements are continuous values and in some cases may be negative. -The expected measurement on day $t$ is: +As an example, we'll consider the wastewater viral genome measurement from the [Pyrenew-HEW](https://github.com/cdcgov/pyrenew-hew) family of models. The predicted measurement on day $t$ is: $$\lambda_t = \frac{G}{V} \cdot \sum_{d=0}^{D} I_{t-d} \cdot p_d$$ From d30b39113d0512325be447557c307ce5046aba2d Mon Sep 17 00:00:00 2001 From: Mitzi Morris Date: Tue, 6 Jan 2026 22:08:14 -0500 Subject: [PATCH 18/81] Update docs/tutorials/observation_processes_measurements.qmd Co-authored-by: Dylan H. Morris --- docs/tutorials/observation_processes_measurements.qmd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/tutorials/observation_processes_measurements.qmd b/docs/tutorials/observation_processes_measurements.qmd index ab9e1948..408e2ef9 100644 --- a/docs/tutorials/observation_processes_measurements.qmd +++ b/docs/tutorials/observation_processes_measurements.qmd @@ -32,7 +32,7 @@ where: - $I_{t-d}$ is the number of incident (new) infections on day $t-d$ - $G$ is a scaling factor (here, genome copies shed per infection) -- $V$ is a normalization factor (e.g., wastewater volume per person per day) +- $V$ is another scaling factor (here, wastewater volume per person per day) - $p_d$ is the temporal distribution (e.g., viral shedding kinetics) - $D$ is the maximum duration From a655f993b93cbd31a6f54dbc83913b84525902eb Mon Sep 17 00:00:00 2001 From: Mitzi Morris Date: Tue, 6 Jan 2026 22:08:33 -0500 Subject: [PATCH 19/81] Update docs/tutorials/observation_processes_measurements.qmd Co-authored-by: Dylan H. Morris --- docs/tutorials/observation_processes_measurements.qmd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/tutorials/observation_processes_measurements.qmd b/docs/tutorials/observation_processes_measurements.qmd index 408e2ef9..eab28e52 100644 --- a/docs/tutorials/observation_processes_measurements.qmd +++ b/docs/tutorials/observation_processes_measurements.qmd @@ -33,7 +33,7 @@ where: - $I_{t-d}$ is the number of incident (new) infections on day $t-d$ - $G$ is a scaling factor (here, genome copies shed per infection) - $V$ is another scaling factor (here, wastewater volume per person per day) -- $p_d$ is the temporal distribution (e.g., viral shedding kinetics) +- $p_d$ is a delay distribution (here, normalized viral genome shedding by time-since-infection) - $D$ is the maximum duration Observed log-concentrations are generated by sampling from a normal distribution: From e37a5724f9fbb15f50b42ecc6096e618acecc6c4 Mon Sep 17 00:00:00 2001 From: Mitzi Morris Date: Tue, 6 Jan 2026 22:08:45 -0500 Subject: [PATCH 20/81] Update docs/tutorials/observation_processes_measurements.qmd Co-authored-by: Dylan H. Morris --- docs/tutorials/observation_processes_measurements.qmd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/tutorials/observation_processes_measurements.qmd b/docs/tutorials/observation_processes_measurements.qmd index eab28e52..81ddc8af 100644 --- a/docs/tutorials/observation_processes_measurements.qmd +++ b/docs/tutorials/observation_processes_measurements.qmd @@ -34,7 +34,7 @@ where: - $G$ is a scaling factor (here, genome copies shed per infection) - $V$ is another scaling factor (here, wastewater volume per person per day) - $p_d$ is a delay distribution (here, normalized viral genome shedding by time-since-infection) -- $D$ is the maximum duration +- $D$ is the maximum delay considered. Observed log-concentrations are generated by sampling from a normal distribution: From 3457f62d1bef6259055e52436cfcb27613e02bab Mon Sep 17 00:00:00 2001 From: Mitzi Morris Date: Tue, 6 Jan 2026 22:09:02 -0500 Subject: [PATCH 21/81] Update pyrenew/observation/noise.py Co-authored-by: Dylan H. Morris --- pyrenew/observation/noise.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyrenew/observation/noise.py b/pyrenew/observation/noise.py index 647b1781..1ca68129 100644 --- a/pyrenew/observation/noise.py +++ b/pyrenew/observation/noise.py @@ -9,7 +9,7 @@ ----------- - ``PoissonNoise``: Equidispersed counts (variance = mean). No parameters. - ``NegativeBinomialNoise``: Overdispersed counts (variance > mean). - Takes ``concentration_rv`` (higher = less overdispersion). + Takes ``concentration_rv`` (higher concentration = less overdispersed, more Poisson-like). Measurement Noise ----------------- From 25e4dd8d1e6e7df837fbca80033f2b0f9dc1d948 Mon Sep 17 00:00:00 2001 From: Mitzi Morris Date: Tue, 6 Jan 2026 22:09:22 -0500 Subject: [PATCH 22/81] Update pyrenew/observation/noise.py Co-authored-by: Dylan H. Morris --- pyrenew/observation/noise.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyrenew/observation/noise.py b/pyrenew/observation/noise.py index 1ca68129..5e5eb248 100644 --- a/pyrenew/observation/noise.py +++ b/pyrenew/observation/noise.py @@ -8,7 +8,7 @@ Count Noise ----------- - ``PoissonNoise``: Equidispersed counts (variance = mean). No parameters. -- ``NegativeBinomialNoise``: Overdispersed counts (variance > mean). +- ``NegativeBinomialNoise``: Overdispersed counts relative to Poisson (variance > mean). Takes ``concentration_rv`` (higher concentration = less overdispersed, more Poisson-like). Measurement Noise From 4bd8f2e9c0ff556c2ce4a2078ceffe6a3da84344 Mon Sep 17 00:00:00 2001 From: Mitzi Morris Date: Wed, 7 Jan 2026 08:39:58 -0500 Subject: [PATCH 23/81] change expected to predicted --- .../observation_processes_measurements.qmd | 8 +-- pyrenew/observation/base.py | 20 ++++---- pyrenew/observation/count_observations.py | 42 ++++++++-------- pyrenew/observation/measurements.py | 20 ++++---- pyrenew/observation/noise.py | 50 +++++++++---------- pyrenew/observation/types.py | 6 +-- test/test_observation_counts.py | 4 +- test/test_observation_measurements.py | 30 +++++------ 8 files changed, 90 insertions(+), 90 deletions(-) diff --git a/docs/tutorials/observation_processes_measurements.qmd b/docs/tutorials/observation_processes_measurements.qmd index 81ddc8af..321ab226 100644 --- a/docs/tutorials/observation_processes_measurements.qmd +++ b/docs/tutorials/observation_processes_measurements.qmd @@ -65,7 +65,7 @@ The core convolution $\sum_{d=0}^{D} I_{t-d} \cdot p_d$ is the same as for count ## Subclassing Measurements for wastewater -The `Measurements` class is abstract—you must subclass it and implement `_expected_signal()` for your specific signal type. Here we create a `Wastewater` class for viral concentration measurements: +The `Measurements` class is abstract—you must subclass it and implement `_predicted_obs()` for your specific signal type. Here we create a `Wastewater` class for viral concentration measurements: ```{python} # | label: wastewater-class @@ -78,7 +78,7 @@ class Wastewater(Measurements): """ Wastewater viral concentration observation process. - Transforms site-level infections into expected log-concentrations + Transforms site-level infections into predicted log-concentrations via shedding kinetics convolution and genome/volume scaling. """ @@ -117,9 +117,9 @@ class Wastewater(Measurements): """Return shedding PMF length.""" return len(self.temporal_pmf_rv()) - def _expected_signal(self, infections: ArrayLike) -> ArrayLike: + def _predicted_obs(self, infections: ArrayLike) -> ArrayLike: """ - Compute expected log-concentration from infections. + Compute predicted log-concentration from infections. Applies shedding kinetics convolution, then scales by genome copies and volume to get concentration. diff --git a/pyrenew/observation/base.py b/pyrenew/observation/base.py index 635d386f..3ee71b83 100644 --- a/pyrenew/observation/base.py +++ b/pyrenew/observation/base.py @@ -39,12 +39,12 @@ class BaseObservationProcess(RandomVariable): - ``validate()``: Validate parameters (call ``_validate_pmf()`` for PMFs) - ``lookback_days()``: Return PMF length for initialization - ``infection_resolution()``: Return ``"aggregate"`` or ``"subpop"`` - - ``_expected_signal()``: Transform infections to expected values - - ``sample()``: Apply noise model to expected signal + - ``_predicted_obs()``: Transform infections to predicted values + - ``sample()``: Apply noise model to predicted observations Notes ----- - Computing expected observations on day t requires infection history + Computing predicted observations on day t requires infection history from previous days (determined by the temporal PMF length). The first ``len(pmf) - 1`` days have insufficient history and return NaN. @@ -257,7 +257,7 @@ def _deterministic(self, name: str, value: ArrayLike) -> None: Track a deterministic quantity in the numpyro execution trace. This is a convenience wrapper around ``numpyro.deterministic`` for - tracking intermediate quantities (e.g., latent admissions, expected + tracking intermediate quantities (e.g., latent admissions, predicted concentrations) that are useful for diagnostics and model checking. These quantities are stored in MCMC samples and can be used for model diagnostics and posterior predictive checks. @@ -272,16 +272,16 @@ def _deterministic(self, name: str, value: ArrayLike) -> None: numpyro.deterministic(name, value) @abstractmethod - def _expected_signal( + def _predicted_obs( self, infections: ArrayLike, ) -> ArrayLike: """ - Transform infections to expected observation values. + Transform infections to predicted observation values. This is the core transformation that each observation process must implement. It converts infections (from the infection process) - to expected values for the observation model. + to predicted values for the observation model. Parameters ---------- @@ -293,14 +293,14 @@ def _expected_signal( Returns ------- ArrayLike - Expected observation values (counts, log-concentrations, etc.). + Predicted observation values (counts, log-concentrations, etc.). Same shape as input, with first len(pmf)-1 days as NaN. Notes ----- The transformation is observation-specific: - - Count observations: ascertainment x delay convolution -> expected counts + - Count observations: ascertainment x delay convolution -> predicted counts - Wastewater: shedding convolution -> genome scaling -> dilution -> log See Also @@ -315,7 +315,7 @@ def sample(self, **kwargs) -> ArrayLike: Sample from the observation process. Subclasses must implement this method to define the specific - observation model. Typically calls ``_expected_signal`` first, + observation model. Typically calls ``_predicted_obs`` first, then applies the noise model. Parameters diff --git a/pyrenew/observation/count_observations.py b/pyrenew/observation/count_observations.py index 7a883932..6d89d0f5 100644 --- a/pyrenew/observation/count_observations.py +++ b/pyrenew/observation/count_observations.py @@ -90,12 +90,12 @@ def infection_resolution(self) -> str: """ raise NotImplementedError("Subclasses must implement infection_resolution()") - def _expected_signal( + def _predicted_obs( self, infections: ArrayLike, ) -> ArrayLike: """ - Compute expected counts via ascertainment x delay convolution. + Compute predicted counts via ascertainment x delay convolution. Parameters ---------- @@ -107,7 +107,7 @@ def _expected_signal( Returns ------- ArrayLike - Expected counts with timeline alignment. + Predicted counts with timeline alignment. Same shape as input. First len(delay_pmf)-1 days are NaN. """ @@ -121,9 +121,9 @@ def _expected_signal( def convolve_col(col): # numpydoc ignore=GL08 return self._convolve_with_alignment(col, delay_pmf, ascertainment_rate)[0] - expected_counts = jax.vmap(convolve_col, in_axes=1, out_axes=1)(infections) + predicted_counts = jax.vmap(convolve_col, in_axes=1, out_axes=1)(infections) - return expected_counts[:, 0] if is_1d else expected_counts + return predicted_counts[:, 0] if is_1d else predicted_counts class Counts(_CountBase): @@ -211,25 +211,25 @@ def sample( ------- ObservationSample Named tuple with `observed` (sampled/conditioned counts) and - `expected` (expected counts before noise). + `predicted` (predicted counts before noise). """ - expected_counts = self._expected_signal(infections) - self._deterministic("expected_counts", expected_counts) - expected_counts_safe = jnp.nan_to_num(expected_counts, nan=0.0) + predicted_counts = self._predicted_obs(infections) + self._deterministic("predicted_counts", predicted_counts) + predicted_counts_safe = jnp.nan_to_num(predicted_counts, nan=0.0) # Only use sparse indexing when conditioning on observations if times is not None and counts is not None: - expected_obs = expected_counts_safe[times] + predicted_obs = predicted_counts_safe[times] else: - expected_obs = expected_counts_safe + predicted_obs = predicted_counts_safe observed = self.noise.sample( name="counts", - expected=expected_obs, + predicted=predicted_obs, obs=counts, ) - return ObservationSample(observed=observed, expected=expected_counts) + return ObservationSample(observed=observed, predicted=predicted_counts) class CountsBySubpop(_CountBase): @@ -328,21 +328,21 @@ def sample( ------- ObservationSample Named tuple with `observed` (sampled/conditioned counts) and - `expected` (expected counts before noise, shape: n_days x n_subpops). + `predicted` (predicted counts before noise, shape: n_days x n_subpops). """ - # Compute expected counts for all subpops - expected_counts_all = self._expected_signal(infections) + # Compute predicted counts for all subpops + predicted_counts_all = self._predicted_obs(infections) - self._deterministic("expected_counts_by_subpop", expected_counts_all) + self._deterministic("predicted_counts_by_subpop", predicted_counts_all) # Replace NaN padding with 0 for distribution creation - expected_counts_safe = jnp.nan_to_num(expected_counts_all, nan=0.0) - expected_obs = expected_counts_safe[times, subpop_indices] + predicted_counts_safe = jnp.nan_to_num(predicted_counts_all, nan=0.0) + predicted_obs = predicted_counts_safe[times, subpop_indices] observed = self.noise.sample( name="counts_by_subpop", - expected=expected_obs, + predicted=predicted_obs, obs=counts, ) - return ObservationSample(observed=observed, expected=expected_counts_all) + return ObservationSample(observed=observed, predicted=predicted_counts_all) diff --git a/pyrenew/observation/measurements.py b/pyrenew/observation/measurements.py index 33e38980..d9c17d0a 100644 --- a/pyrenew/observation/measurements.py +++ b/pyrenew/observation/measurements.py @@ -19,7 +19,7 @@ class Measurements(BaseObservationProcess): Abstract base for continuous measurement observations. Subclasses implement signal-specific transformations from infections - to expected measurement values, then add measurement noise. + to predicted measurement values, then add measurement noise. Parameters ---------- @@ -31,7 +31,7 @@ class Measurements(BaseObservationProcess): Notes ----- - Subclasses must implement ``_expected_signal()`` according to their + Subclasses must implement ``_predicted_obs()`` according to their specific signal processing (e.g., wastewater shedding kinetics, dilution factors, etc.). @@ -99,8 +99,8 @@ def sample( (times, subpop_indices, sensor_indices). Validate observation data before sampling. - Transforms infections to expected values via signal-specific processing - (``_expected_signal``), then applies noise model. + Transforms infections to predicted values via signal-specific processing + (``_predicted_obs``), then applies noise model. Parameters ---------- @@ -125,20 +125,20 @@ def sample( ------- ObservationSample Named tuple with `observed` (sampled/conditioned measurements) and - `expected` (expected values before noise, shape: n_days x n_subpops). + `predicted` (predicted values before noise, shape: n_days x n_subpops). """ - expected_values = self._expected_signal(infections) + predicted_values = self._predicted_obs(infections) - self._deterministic("expected_log_conc", expected_values) + self._deterministic("predicted_log_conc", predicted_values) - expected_obs = expected_values[times, subpop_indices] + predicted_obs = predicted_values[times, subpop_indices] observed = self.noise.sample( name="concentrations", - expected=expected_obs, + predicted=predicted_obs, obs=concentrations, sensor_indices=sensor_indices, n_sensors=n_sensors, ) - return ObservationSample(observed=observed, expected=expected_values) + return ObservationSample(observed=observed, predicted=predicted_values) diff --git a/pyrenew/observation/noise.py b/pyrenew/observation/noise.py index 5e5eb248..1de84f42 100644 --- a/pyrenew/observation/noise.py +++ b/pyrenew/observation/noise.py @@ -36,32 +36,32 @@ class CountNoise(ABC): """ Abstract base for count observation noise models. - Defines how discrete count observations are distributed around expected values. + Defines how discrete count observations are distributed around predicted values. """ @abstractmethod def sample( self, name: str, - expected: ArrayLike, + predicted: ArrayLike, obs: ArrayLike | None = None, ) -> ArrayLike: """ - Sample count observations given expected counts. + Sample count observations given predicted counts. Parameters ---------- name : str Numpyro sample site name. - expected : ArrayLike - Expected count values (non-negative). + predicted : ArrayLike + Predicted count values (non-negative). obs : ArrayLike | None Observed counts for conditioning, or None for prior sampling. Returns ------- ArrayLike - Sampled or conditioned counts, same shape as expected. + Sampled or conditioned counts, same shape as predicted. """ pass # pragma: no cover @@ -98,7 +98,7 @@ def validate(self) -> None: def sample( self, name: str, - expected: ArrayLike, + predicted: ArrayLike, obs: ArrayLike | None = None, ) -> ArrayLike: """ @@ -108,8 +108,8 @@ def sample( ---------- name : str Numpyro sample site name. - expected : ArrayLike - Expected count values. + predicted : ArrayLike + Predicted count values. obs : ArrayLike | None Observed counts for conditioning. @@ -120,7 +120,7 @@ def sample( """ return numpyro.sample( name, - dist.Poisson(rate=expected + _EPSILON), + dist.Poisson(rate=predicted + _EPSILON), obs=obs, ) @@ -178,7 +178,7 @@ def validate(self) -> None: def sample( self, name: str, - expected: ArrayLike, + predicted: ArrayLike, obs: ArrayLike | None = None, ) -> ArrayLike: """ @@ -188,8 +188,8 @@ def sample( ---------- name : str Numpyro sample site name. - expected : ArrayLike - Expected count values. + predicted : ArrayLike + Predicted count values. obs : ArrayLike | None Observed counts for conditioning. @@ -202,7 +202,7 @@ def sample( return numpyro.sample( name, dist.NegativeBinomial2( - mean=expected + _EPSILON, + mean=predicted + _EPSILON, concentration=concentration, ), obs=obs, @@ -213,26 +213,26 @@ class MeasurementNoise(ABC): """ Abstract base for continuous measurement noise models. - Defines how continuous observations are distributed around expected values. + Defines how continuous observations are distributed around predicted values. """ @abstractmethod def sample( self, name: str, - expected: ArrayLike, + predicted: ArrayLike, obs: ArrayLike | None = None, **kwargs, ) -> ArrayLike: """ - Sample continuous observations given expected values. + Sample continuous observations given predicted values. Parameters ---------- name : str Numpyro sample site name. - expected : ArrayLike - Expected measurement values. + predicted : ArrayLike + Predicted measurement values. obs : ArrayLike | None Observed measurements for conditioning, or None for prior sampling. **kwargs @@ -241,7 +241,7 @@ def sample( Returns ------- ArrayLike - Sampled or conditioned measurements, same shape as expected. + Sampled or conditioned measurements, same shape as predicted. """ pass # pragma: no cover @@ -262,7 +262,7 @@ class HierarchicalNormalNoise(MeasurementNoise): """ Normal noise with hierarchical sensor-level effects. - Observation model: ``obs ~ Normal(expected + sensor_mode, sensor_sd)`` + Observation model: ``obs ~ Normal(predicted + sensor_mode, sensor_sd)`` where sensor_mode and sensor_sd are hierarchically modeled. Parameters @@ -327,7 +327,7 @@ def validate(self) -> None: def sample( self, name: str, - expected: ArrayLike, + predicted: ArrayLike, obs: ArrayLike | None = None, *, sensor_indices: ArrayLike, @@ -340,8 +340,8 @@ def sample( ---------- name : str Numpyro sample site name. - expected : ArrayLike - Expected log-scale measurement values. + predicted : ArrayLike + Predicted log-scale measurement values. Shape: (n_obs,) obs : ArrayLike | None Observed log-scale measurements for conditioning. @@ -366,7 +366,7 @@ def sample( sensor_mode = self.sensor_mode_prior_rv.sample(n_groups=n_sensors) sensor_sd = self.sensor_sd_prior_rv.sample(n_groups=n_sensors) - loc = expected + sensor_mode[sensor_indices] + loc = predicted + sensor_mode[sensor_indices] scale = sensor_sd[sensor_indices] return numpyro.sample(name, dist.Normal(loc=loc, scale=scale), obs=obs) diff --git a/pyrenew/observation/types.py b/pyrenew/observation/types.py index b494e2e6..f8a1163c 100644 --- a/pyrenew/observation/types.py +++ b/pyrenew/observation/types.py @@ -19,10 +19,10 @@ class ObservationSample(NamedTuple): observed : ArrayLike Sampled or conditioned observations. Shape depends on the observation process and indexing. - expected : ArrayLike - Expected values before noise is applied. Useful for + predicted : ArrayLike + Predicted values before noise is applied. Useful for diagnostics and posterior predictive checks. """ observed: ArrayLike - expected: ArrayLike + predicted: ArrayLike diff --git a/test/test_observation_counts.py b/test/test_observation_counts.py index ddb4d684..57bdb37a 100644 --- a/test/test_observation_counts.py +++ b/test/test_observation_counts.py @@ -73,7 +73,7 @@ def test_sample_returns_correct_shape(self, counts_process): assert result.observed.shape[0] > 0 assert result.observed.ndim == 1 - assert result.expected.shape == infections.shape + assert result.predicted.shape == infections.shape def test_delay_convolution(self, counts_factory, short_delay_pmf): """Test that delay is properly applied.""" @@ -340,7 +340,7 @@ def test_sample_returns_correct_shape(self): ) assert result.observed.shape == times.shape - assert result.expected.shape == infections.shape + assert result.predicted.shape == infections.shape def test_infection_resolution(self): """Test that CountsBySubpop returns 'subpop' resolution.""" diff --git a/test/test_observation_measurements.py b/test/test_observation_measurements.py index e5c014a2..a3af44a1 100644 --- a/test/test_observation_measurements.py +++ b/test/test_observation_measurements.py @@ -38,14 +38,14 @@ def lookback_days(self) -> int: """ return len(self.temporal_pmf_rv()) - def _expected_signal(self, infections): + def _predicted_obs(self, infections): """ - Simple expected signal: log(convolution * scale). + Simple predicted signal: log(convolution * scale). Returns ------- jnp.ndarray - Log-transformed expected signal. + Log-transformed predicted signal. """ pmf = self.temporal_pmf_rv() @@ -58,12 +58,12 @@ def convolve_col(col): # numpydoc ignore=GL08 import jax - expected = jax.vmap(convolve_col, in_axes=1, out_axes=1)(infections) + predicted = jax.vmap(convolve_col, in_axes=1, out_axes=1)(infections) # Apply log10 scaling (simplified from wastewater model) - log_expected = jnp.log(expected + 1e-10) + self.log10_scale * jnp.log(10) + log_predicted = jnp.log(predicted + 1e-10) + self.log10_scale * jnp.log(10) - return log_expected + return log_predicted class TestMeasurementsBase: @@ -123,19 +123,19 @@ def test_sample_shape(self): ) noise = HierarchicalNormalNoise(sensor_mode_prior, sensor_sd_prior) - expected = jnp.array([1.0, 2.0, 3.0, 4.0]) + predicted = jnp.array([1.0, 2.0, 3.0, 4.0]) sensor_indices = jnp.array([0, 0, 1, 1]) with numpyro.handlers.seed(rng_seed=42): samples = noise.sample( name="test", - expected=expected, + predicted=predicted, obs=None, sensor_indices=sensor_indices, n_sensors=2, ) - assert samples.shape == expected.shape + assert samples.shape == predicted.shape def test_sample_with_observations(self): """Test that HierarchicalNormalNoise conditions on observations.""" @@ -149,14 +149,14 @@ def test_sample_with_observations(self): ) noise = HierarchicalNormalNoise(sensor_mode_prior, sensor_sd_prior) - expected = jnp.array([1.0, 2.0, 3.0, 4.0]) + predicted = jnp.array([1.0, 2.0, 3.0, 4.0]) obs = jnp.array([1.1, 2.1, 3.1, 4.1]) sensor_indices = jnp.array([0, 0, 1, 1]) with numpyro.handlers.seed(rng_seed=42): samples = noise.sample( name="test", - expected=expected, + predicted=predicted, obs=obs, sensor_indices=sensor_indices, n_sensors=2, @@ -204,10 +204,10 @@ def test_sample_shape(self): ) assert result.observed.shape == times.shape - assert result.expected.shape == infections.shape + assert result.predicted.shape == infections.shape - def test_expected_signal_stored(self): - """Test that expected_log_conc is stored as deterministic.""" + def test_predicted_obs_stored(self): + """Test that predicted_log_conc is stored as deterministic.""" shedding_pmf = jnp.array([0.5, 0.5]) sensor_mode_prior = HierarchicalNormalPrior( name="mode", sd_rv=DeterministicVariable("mode_sd", 0.01) @@ -242,7 +242,7 @@ def test_expected_signal_stored(self): ) ).get_trace() - assert "expected_log_conc" in trace + assert "predicted_log_conc" in trace if __name__ == "__main__": From 4e429bccbf76b8f1403ed80318ce311d23fc67bd Mon Sep 17 00:00:00 2001 From: Mitzi Morris Date: Wed, 7 Jan 2026 10:17:29 -0500 Subject: [PATCH 24/81] standardize on param name obs for sample method --- .../observation_processes_counts.qmd | 12 +++--- .../observation_processes_measurements.qmd | 10 ++--- pyrenew/observation/count_observations.py | 18 ++++----- pyrenew/observation/measurements.py | 6 +-- test/test_observation_counts.py | 38 +++++++++---------- test/test_observation_measurements.py | 4 +- 6 files changed, 43 insertions(+), 45 deletions(-) diff --git a/docs/tutorials/observation_processes_counts.qmd b/docs/tutorials/observation_processes_counts.qmd index 426a9c53..ad50e074 100644 --- a/docs/tutorials/observation_processes_counts.qmd +++ b/docs/tutorials/observation_processes_counts.qmd @@ -166,7 +166,7 @@ infections = infections.at[infection_spike_day].set(2000) with numpyro.handlers.seed(rng_seed=42): hosp_admissions = hosp_process.sample( infections=infections, - counts=None, # Sample from prior (no observed data) + obs=None, # Sample from prior (no observed data) ) ``` @@ -246,7 +246,7 @@ samples = [] for seed in range(n_samples): with numpyro.handlers.seed(rng_seed=seed): - result = hosp_process.sample(infections=infections, counts=None) + result = hosp_process.sample(infections=infections, obs=None) samples.append(np.array(result.observed[day_one:])) samples = np.array(samples) @@ -328,7 +328,7 @@ for ihr_val in ihr_values: samples = [] for seed in range(20): with numpyro.handlers.seed(rng_seed=seed): - result = process.sample(infections=infections_decay, counts=None) + result = process.sample(infections=infections_decay, obs=None) samples.append(np.array(result.observed[day_one:])) results[ihr_val] = np.mean(samples, axis=0) ``` @@ -381,9 +381,7 @@ for ax, conc_val in zip(axes, concentration_values): for seed in range(n_replicates): with numpyro.handlers.seed(rng_seed=seed): - result = process.sample( - infections=infections_constant, counts=None - ) + result = process.sample(infections=infections_constant, obs=None) ax.plot( np.arange(n_plot_days), np.array(result.observed[day_one:]), @@ -416,7 +414,7 @@ hosp_process_poisson = Counts( with numpyro.handlers.seed(rng_seed=42): poisson_result = hosp_process_poisson.sample( infections=infections, - counts=None, + obs=None, ) print( diff --git a/docs/tutorials/observation_processes_measurements.qmd b/docs/tutorials/observation_processes_measurements.qmd index 321ab226..7d0b458c 100644 --- a/docs/tutorials/observation_processes_measurements.qmd +++ b/docs/tutorials/observation_processes_measurements.qmd @@ -292,7 +292,7 @@ with numpyro.handlers.seed(rng_seed=42): subpop_indices=jnp.zeros(n_obs, dtype=jnp.int32), sensor_indices=jnp.zeros(n_obs, dtype=jnp.int32), times=observation_days, - concentrations=None, # Sample from prior + obs=None, n_sensors=1, ) ``` @@ -340,7 +340,7 @@ for seed in range(n_samples): subpop_indices=jnp.zeros(n_obs, dtype=jnp.int32), sensor_indices=jnp.zeros(n_obs, dtype=jnp.int32), times=observation_days, - concentrations=None, + obs=None, n_sensors=1, ) for day_idx, conc in zip(observation_days, ww_result.observed): @@ -427,7 +427,7 @@ with numpyro.handlers.seed(rng_seed=42): subpop_indices=subpop_ids, sensor_indices=sensor_ids, times=observation_times, - concentrations=None, + obs=None, n_sensors=num_sensors, ) ``` @@ -488,7 +488,7 @@ for genome_val in genome_values: subpop_indices=jnp.zeros(n_obs_cmp, dtype=jnp.int32), sensor_indices=jnp.zeros(n_obs_cmp, dtype=jnp.int32), times=observation_days_cmp, - concentrations=None, + obs=None, n_sensors=1, ) results[genome_val] = np.array(ww_temp.observed) @@ -545,7 +545,7 @@ with numpyro.handlers.seed(rng_seed=42): subpop_indices=subpop_ids_mp, sensor_indices=sensor_ids_mp, times=obs_days_mp, - concentrations=None, + obs=None, n_sensors=2, ) ``` diff --git a/pyrenew/observation/count_observations.py b/pyrenew/observation/count_observations.py index 6d89d0f5..48a8d574 100644 --- a/pyrenew/observation/count_observations.py +++ b/pyrenew/observation/count_observations.py @@ -163,7 +163,7 @@ class Counts(_CountBase): >>> >>> with numpyro.handlers.seed(rng_seed=42): ... infections = jnp.ones(30) * 1000 - ... sampled_counts = counts_obs.sample(infections=infections, counts=None) + ... sampled_counts = counts_obs.sample(infections=infections, obs=None) """ def infection_resolution(self) -> str: @@ -188,7 +188,7 @@ def __repr__(self) -> str: def sample( self, infections: ArrayLike, - counts: ArrayLike | None = None, + obs: ArrayLike | None = None, times: ArrayLike | None = None, ) -> ObservationSample: """ @@ -202,7 +202,7 @@ def sample( infections : ArrayLike Aggregate infections from the infection process. Shape: (n_days,) - counts : ArrayLike | None + obs : ArrayLike | None Observed counts. Dense: (n_days,), Sparse: (n_obs,), None: prior. times : ArrayLike | None Day indices for sparse observations. None for dense observations. @@ -218,7 +218,7 @@ def sample( predicted_counts_safe = jnp.nan_to_num(predicted_counts, nan=0.0) # Only use sparse indexing when conditioning on observations - if times is not None and counts is not None: + if times is not None and obs is not None: predicted_obs = predicted_counts_safe[times] else: predicted_obs = predicted_counts_safe @@ -226,7 +226,7 @@ def sample( observed = self.noise.sample( name="counts", predicted=predicted_obs, - obs=counts, + obs=obs, ) return ObservationSample(observed=observed, predicted=predicted_counts) @@ -274,7 +274,7 @@ class CountsBySubpop(_CountBase): ... infections=infections, ... subpop_indices=subpop_indices, ... times=times, - ... counts=None, + ... obs=None, ... ) """ @@ -302,7 +302,7 @@ def sample( infections: ArrayLike, subpop_indices: ArrayLike, times: ArrayLike, - counts: ArrayLike | None = None, + obs: ArrayLike | None = None, ) -> ObservationSample: """ Sample subpopulation-level counts with flexible indexing. @@ -321,7 +321,7 @@ def sample( times : ArrayLike Day index for each observation (0-indexed). Shape: (n_obs,) - counts : ArrayLike | None + obs : ArrayLike | None Observed counts (n_obs,), or None for prior sampling. Returns @@ -342,7 +342,7 @@ def sample( observed = self.noise.sample( name="counts_by_subpop", predicted=predicted_obs, - obs=counts, + obs=obs, ) return ObservationSample(observed=observed, predicted=predicted_counts_all) diff --git a/pyrenew/observation/measurements.py b/pyrenew/observation/measurements.py index d9c17d0a..82043e65 100644 --- a/pyrenew/observation/measurements.py +++ b/pyrenew/observation/measurements.py @@ -89,7 +89,7 @@ def sample( subpop_indices: ArrayLike, sensor_indices: ArrayLike, times: ArrayLike, - concentrations: ArrayLike | None, + obs: ArrayLike | None, n_sensors: int, ) -> ObservationSample: """ @@ -116,7 +116,7 @@ def sample( times : ArrayLike Day index for each observation (0-indexed). Shape: (n_obs,) - concentrations : ArrayLike | None + obs : ArrayLike | None Observed measurements (n_obs,), or None for prior sampling. n_sensors : int Total number of measurement sensors. @@ -136,7 +136,7 @@ def sample( observed = self.noise.sample( name="concentrations", predicted=predicted_obs, - obs=concentrations, + obs=obs, sensor_indices=sensor_indices, n_sensors=n_sensors, ) diff --git a/test/test_observation_counts.py b/test/test_observation_counts.py index 57bdb37a..bcfbd54f 100644 --- a/test/test_observation_counts.py +++ b/test/test_observation_counts.py @@ -68,7 +68,7 @@ def test_sample_returns_correct_shape(self, counts_process): with numpyro.handlers.seed(rng_seed=42): result = counts_process.sample( infections=infections, - counts=None, + obs=None, ) assert result.observed.shape[0] > 0 @@ -85,7 +85,7 @@ def test_delay_convolution(self, counts_factory, short_delay_pmf): with numpyro.handlers.seed(rng_seed=42): result = process.sample( infections=infections, - counts=None, + obs=None, ) # Timeline alignment: output length equals input length @@ -108,7 +108,7 @@ def test_ascertainment_scaling(self, counts_factory, simple_delay_pmf): with numpyro.handlers.seed(rng_seed=42): result = process.sample( infections=infections, - counts=None, + obs=None, ) results.append(jnp.mean(result.observed)) @@ -130,7 +130,7 @@ def test_negative_binomial_observation(self, counts_factory, simple_delay_pmf): with numpyro.handlers.seed(rng_seed=seed): result = process.sample( infections=infections, - counts=None, + obs=None, ) samples.append(jnp.sum(result.observed)) @@ -158,7 +158,7 @@ def test_with_stochastic_ascertainment(self, short_shedding_pmf): with numpyro.handlers.seed(rng_seed=42): result = process.sample( infections=infections, - counts=None, + obs=None, ) assert result.observed.shape[0] > 0 @@ -183,7 +183,7 @@ def test_with_stochastic_concentration(self, simple_delay_pmf): with numpyro.handlers.seed(rng_seed=42): result = process.sample( infections=infections, - counts=None, + obs=None, ) assert result.observed.shape[0] > 0 @@ -200,7 +200,7 @@ def test_zero_infections(self, counts_process): with numpyro.handlers.seed(rng_seed=42): result = counts_process.sample( infections=infections, - counts=None, + obs=None, ) assert result.observed.shape[0] > 0 @@ -212,7 +212,7 @@ def test_small_infections(self, counts_process): with numpyro.handlers.seed(rng_seed=42): result = counts_process.sample( infections=infections, - counts=None, + obs=None, ) assert result.observed.shape[0] > 0 @@ -227,7 +227,7 @@ def test_long_delay_distribution(self, counts_factory, long_delay_pmf): with numpyro.handlers.seed(rng_seed=42): result = process.sample( infections=infections, - counts=None, + obs=None, ) # Timeline alignment maintained @@ -252,7 +252,7 @@ def test_sparse_observations(self, counts_process): with numpyro.handlers.seed(rng_seed=42): result = counts_process.sample( infections=infections, - counts=counts_data, + obs=counts_data, times=times, ) @@ -264,11 +264,11 @@ def test_sparse_vs_dense_sampling(self, counts_process): n_days = 30 infections = jnp.ones(n_days) * 100 - # Dense: prior sampling (counts=None, no times) + # Dense: prior sampling (obs=None, no times) with numpyro.handlers.seed(rng_seed=42): dense_result = counts_process.sample( infections=infections, - counts=None, + obs=None, ) # Sparse with observed data: only some days @@ -277,7 +277,7 @@ def test_sparse_vs_dense_sampling(self, counts_process): with numpyro.handlers.seed(rng_seed=42): sparse_result = counts_process.sample( infections=infections, - counts=sparse_obs_data, + obs=sparse_obs_data, times=times, ) @@ -289,23 +289,23 @@ def test_sparse_vs_dense_sampling(self, counts_process): assert jnp.allclose(sparse_result.observed, sparse_obs_data) def test_prior_sampling_ignores_times(self, counts_process): - """Test that times parameter is ignored when counts=None (prior sampling).""" + """Test that times parameter is ignored when obs=None (prior sampling).""" n_days = 30 infections = jnp.ones(n_days) * 100 times = jnp.array([5, 10, 15, 20]) - # When counts=None, times is ignored - output is dense + # When obs=None, times is ignored - output is dense with numpyro.handlers.seed(rng_seed=42): result_with_times = counts_process.sample( infections=infections, - counts=None, + obs=None, times=times, ) with numpyro.handlers.seed(rng_seed=42): result_without_times = counts_process.sample( infections=infections, - counts=None, + obs=None, ) # Both should produce dense output of shape (n_days,) @@ -336,7 +336,7 @@ def test_sample_returns_correct_shape(self): infections=infections, subpop_indices=subpop_indices, times=times, - counts=None, + obs=None, ) assert result.observed.shape == times.shape @@ -370,7 +370,7 @@ def test_poisson_counts(self, simple_delay_pmf): with numpyro.handlers.seed(rng_seed=42): result = process.sample( infections=infections, - counts=None, + obs=None, ) assert result.observed.shape[0] == 20 diff --git a/test/test_observation_measurements.py b/test/test_observation_measurements.py index a3af44a1..ae5b70c2 100644 --- a/test/test_observation_measurements.py +++ b/test/test_observation_measurements.py @@ -199,7 +199,7 @@ def test_sample_shape(self): subpop_indices=subpop_indices, sensor_indices=sensor_indices, times=times, - concentrations=None, + obs=None, n_sensors=2, ) @@ -237,7 +237,7 @@ def test_predicted_obs_stored(self): subpop_indices=subpop_indices, sensor_indices=sensor_indices, times=times, - concentrations=None, + obs=None, n_sensors=2, ) ).get_trace() From d05249a510f27f8e677ef41ca32c90fca8d63f8d Mon Sep 17 00:00:00 2001 From: Mitzi Morris Date: Wed, 7 Jan 2026 10:26:03 -0500 Subject: [PATCH 25/81] standardize sample - all methods must have arg obs --- pyrenew/observation/base.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/pyrenew/observation/base.py b/pyrenew/observation/base.py index 3ee71b83..f415671a 100644 --- a/pyrenew/observation/base.py +++ b/pyrenew/observation/base.py @@ -310,7 +310,11 @@ def _predicted_obs( pass # pragma: no cover @abstractmethod - def sample(self, **kwargs) -> ArrayLike: + def sample( + self, + obs: ArrayLike | None = None, + **kwargs, + ) -> ArrayLike: """ Sample from the observation process. @@ -320,11 +324,10 @@ def sample(self, **kwargs) -> ArrayLike: Parameters ---------- + obs : ArrayLike | None + Observed data for conditioning, or None for prior predictive sampling. **kwargs - Subclass-specific parameters. At minimum, should include: - - - infections from the infection process - - Observed data (or None for prior predictive sampling) + Subclass-specific parameters (e.g., infections from the infection process). Returns ------- From 613a8f1748567975291a238c7d79c0a534a0e611 Mon Sep 17 00:00:00 2001 From: Mitzi Morris Date: Wed, 7 Jan 2026 11:34:20 -0500 Subject: [PATCH 26/81] in order to use np TruncatedDistribution, change use of Gamma to TruncatedNormal --- .../observation_processes_measurements.qmd | 18 +++- pyrenew/observation/noise.py | 2 +- pyrenew/randomvariable/__init__.py | 4 +- pyrenew/randomvariable/hierarchical.py | 89 ++++++++++--------- test/conftest.py | 26 +++--- test/test_observation_measurements.py | 38 ++++---- test/test_randomvariable_hierarchical.py | 48 +++++----- 7 files changed, 118 insertions(+), 107 deletions(-) diff --git a/docs/tutorials/observation_processes_measurements.qmd b/docs/tutorials/observation_processes_measurements.qmd index 7d0b458c..9b32567e 100644 --- a/docs/tutorials/observation_processes_measurements.qmd +++ b/docs/tutorials/observation_processes_measurements.qmd @@ -16,7 +16,10 @@ import numpyro import matplotlib.pyplot as plt from pyrenew.observation import Measurements, HierarchicalNormalNoise -from pyrenew.randomvariable import HierarchicalNormalPrior, GammaGroupSdPrior +from pyrenew.randomvariable import ( + HierarchicalNormalPrior, + TruncatedNormalGroupSdPrior, +) from pyrenew.deterministic import DeterministicVariable, DeterministicPMF ``` @@ -212,6 +215,13 @@ ml_per_person_per_day = 1000.0 A measurement depends on both the thing being measured (e.g., wastewater from a treatment plant) and the calibration of the lab/instruments used to obtain that measurement. We call this combination a "sensor"—the WWTP/lab pair that determines the measurement characteristics. Different sensors have systematic biases and variabilities that we model with hierarchical effects. +For the sensor-level standard deviations, we use a truncated normal prior. This is a common choice for SD parameters because: + +- It naturally constrains values to be positive +- The `loc` parameter represents our prior expectation for the SD +- The `scale` parameter controls how uncertain we are about the SD +- The `sd_min` floor prevents numerical issues when SDs approach zero + ```{python} # | label: noise-model # Sensor-level mode: systematic differences between sensors @@ -221,10 +231,10 @@ sensor_mode_prior = HierarchicalNormalPrior( ) # Sensor-level SD: measurement variability within each sensor -sensor_sd_prior = GammaGroupSdPrior( +sensor_sd_prior = TruncatedNormalGroupSdPrior( name="ww_sensor_sd", - sd_mean_rv=DeterministicVariable("sd_mean", 0.3), - sd_concentration_rv=DeterministicVariable("sd_concentration", 4.0), + loc_rv=DeterministicVariable("sd_loc", 0.3), + scale_rv=DeterministicVariable("sd_scale", 0.15), sd_min=0.10, ) diff --git a/pyrenew/observation/noise.py b/pyrenew/observation/noise.py index 1de84f42..47d2d4ad 100644 --- a/pyrenew/observation/noise.py +++ b/pyrenew/observation/noise.py @@ -282,7 +282,7 @@ class HierarchicalNormalNoise(MeasurementNoise): -------- pyrenew.randomvariable.HierarchicalNormalPrior : Suitable prior for sensor_mode_prior_rv - pyrenew.randomvariable.GammaGroupSdPrior : + pyrenew.randomvariable.TruncatedNormalGroupSdPrior : Suitable prior for sensor_sd_prior_rv """ diff --git a/pyrenew/randomvariable/__init__.py b/pyrenew/randomvariable/__init__.py index 75594c58..09812b9c 100644 --- a/pyrenew/randomvariable/__init__.py +++ b/pyrenew/randomvariable/__init__.py @@ -6,9 +6,9 @@ StaticDistributionalVariable, ) from pyrenew.randomvariable.hierarchical import ( - GammaGroupSdPrior, HierarchicalNormalPrior, StudentTGroupModePrior, + TruncatedNormalGroupSdPrior, ) from pyrenew.randomvariable.transformedvariable import TransformedVariable @@ -18,6 +18,6 @@ "DynamicDistributionalVariable", "TransformedVariable", "HierarchicalNormalPrior", - "GammaGroupSdPrior", + "TruncatedNormalGroupSdPrior", "StudentTGroupModePrior", ] diff --git a/pyrenew/randomvariable/hierarchical.py b/pyrenew/randomvariable/hierarchical.py index e97126d4..45528d87 100644 --- a/pyrenew/randomvariable/hierarchical.py +++ b/pyrenew/randomvariable/hierarchical.py @@ -7,7 +7,6 @@ dynamic group sizes at sample time with proper numpyro plate contexts. """ -import jax.numpy as jnp import numpyro import numpyro.distributions as dist @@ -107,42 +106,52 @@ def sample(self, n_groups: int, **kwargs): return effects -class GammaGroupSdPrior(RandomVariable): +class TruncatedNormalGroupSdPrior(RandomVariable): """ - Gamma prior for group-level standard deviations, bounded away from zero. + Truncated Normal prior for group-level standard deviations. - Samples n_groups positive values from Gamma(concentration, rate) + sd_min - within a numpyro plate context. + Samples n_groups positive values from TruncatedNormal(loc, scale, low=sd_min) + within a numpyro plate context. Uses NumPyro's native truncated distribution + support for proper likelihood computation. Parameters ---------- name : str Unique name for the sampled parameter in numpyro. - sd_mean_rv : RandomVariable - RandomVariable returning the mean of the Gamma distribution. - sd_concentration_rv : RandomVariable - RandomVariable returning the concentration (shape) parameter of Gamma. + loc_rv : RandomVariable + RandomVariable returning the location (mean) of the underlying Normal. + scale_rv : RandomVariable + RandomVariable returning the scale (std) of the underlying Normal. sd_min : float, default=0.05 - Minimum SD value (lower bound). + Minimum SD value (left truncation point). Notes ----- - This class parameterizes Gamma by mean and concentration rather than - shape and rate, which is often more interpretable. The rate is computed as - concentration / mean. + Truncated normal is a common choice for standard deviation priors because: - The sd_min floor prevents numerical issues when standard deviations - approach zero. + - It naturally constrains values to be positive (via left truncation) + - The location parameter represents the expected SD value + - The scale parameter controls uncertainty about the SD + - It integrates well with NumPyro's truncation support for proper likelihood + + Parameter guidance: + + - ``loc``: Set to your prior expectation for the SD (e.g., 0.3 for + moderate sensor variability on log scale) + - ``scale``: Controls uncertainty; smaller values give tighter priors + around ``loc`` + - ``sd_min``: Prevents numerical issues when SDs approach zero; typical + values are 0.01-0.1 depending on the scale of the data Examples -------- >>> from pyrenew.deterministic import DeterministicVariable - >>> from pyrenew.randomvariable import GammaGroupSdPrior + >>> from pyrenew.randomvariable import TruncatedNormalGroupSdPrior >>> import numpyro >>> - >>> mean_rv = DeterministicVariable("sd_mean", 0.3) - >>> conc_rv = DeterministicVariable("sd_conc", 4.0) - >>> prior = GammaGroupSdPrior("site_sd", mean_rv, conc_rv, sd_min=0.05) + >>> loc_rv = DeterministicVariable("sd_loc", 0.3) + >>> scale_rv = DeterministicVariable("sd_scale", 0.15) + >>> prior = TruncatedNormalGroupSdPrior("site_sd", loc_rv, scale_rv, sd_min=0.05) >>> >>> with numpyro.handlers.seed(rng_seed=42): ... sds = prior.sample(n_groups=5) @@ -153,44 +162,44 @@ class GammaGroupSdPrior(RandomVariable): def __init__( self, name: str, - sd_mean_rv: RandomVariable, - sd_concentration_rv: RandomVariable, + loc_rv: RandomVariable, + scale_rv: RandomVariable, sd_min: float = 0.05, ) -> None: """ - Default constructor for GammaGroupSdPrior. + Default constructor for TruncatedNormalGroupSdPrior. Parameters ---------- name : str Unique name for the sampled parameter in numpyro. - sd_mean_rv : RandomVariable - RandomVariable returning the mean of the Gamma distribution. - sd_concentration_rv : RandomVariable - RandomVariable returning the concentration (shape) parameter. + loc_rv : RandomVariable + RandomVariable returning the location (mean) of the underlying Normal. + scale_rv : RandomVariable + RandomVariable returning the scale (std) of the underlying Normal. sd_min : float, default=0.05 - Minimum SD value (lower bound). + Minimum SD value (left truncation point). Returns ------- None """ - if not isinstance(sd_mean_rv, RandomVariable): + if not isinstance(loc_rv, RandomVariable): raise TypeError( - f"sd_mean_rv must be a RandomVariable, got {type(sd_mean_rv).__name__}. " + f"loc_rv must be a RandomVariable, got {type(loc_rv).__name__}. " "Use DeterministicVariable(name, value) to wrap a fixed value." ) - if not isinstance(sd_concentration_rv, RandomVariable): + if not isinstance(scale_rv, RandomVariable): raise TypeError( - f"sd_concentration_rv must be a RandomVariable, got {type(sd_concentration_rv).__name__}. " + f"scale_rv must be a RandomVariable, got {type(scale_rv).__name__}. " "Use DeterministicVariable(name, value) to wrap a fixed value." ) if sd_min < 0: raise ValueError(f"sd_min must be non-negative, got {sd_min}") self.name = name - self.sd_mean_rv = sd_mean_rv - self.sd_concentration_rv = sd_concentration_rv + self.loc_rv = loc_rv + self.scale_rv = scale_rv self.sd_min = sd_min def validate(self): @@ -213,19 +222,13 @@ def sample(self, n_groups: int, **kwargs): ArrayLike Array of shape (n_groups,) with values >= sd_min. """ - sd_mean = self.sd_mean_rv() - concentration = self.sd_concentration_rv() - rate = concentration / sd_mean + loc = self.loc_rv() + scale = self.scale_rv() with numpyro.plate(f"n_{self.name}", n_groups): - raw_sd = numpyro.sample( - f"{self.name}_raw", - dist.Gamma(concentration, rate), - ) - - group_sd = numpyro.deterministic( + group_sd = numpyro.sample( self.name, - jnp.maximum(raw_sd, self.sd_min), + dist.TruncatedNormal(loc=loc, scale=scale, low=self.sd_min), ) return group_sd diff --git a/test/conftest.py b/test/conftest.py index 6f55681a..d159106a 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -10,7 +10,7 @@ from pyrenew.deterministic import DeterministicPMF, DeterministicVariable from pyrenew.observation import Counts, NegativeBinomialNoise -from pyrenew.randomvariable import GammaGroupSdPrior, HierarchicalNormalPrior +from pyrenew.randomvariable import HierarchicalNormalPrior, TruncatedNormalGroupSdPrior # ============================================================================= # PMF Fixtures @@ -159,17 +159,17 @@ def sensor_mode_prior_tight(): @pytest.fixture def sensor_sd_prior(): """ - Standard gamma prior for sensor standard deviations. + Standard truncated normal prior for sensor standard deviations. Returns ------- - GammaGroupSdPrior - A gamma prior for group standard deviations. + TruncatedNormalGroupSdPrior + A truncated normal prior for group standard deviations. """ - return GammaGroupSdPrior( + return TruncatedNormalGroupSdPrior( name="ww_sensor_sd", - sd_mean_rv=DeterministicVariable("sd_mean", 0.3), - sd_concentration_rv=DeterministicVariable("sd_concentration", 4.0), + loc_rv=DeterministicVariable("sd_loc", 0.3), + scale_rv=DeterministicVariable("sd_scale", 0.15), sd_min=0.10, ) @@ -177,17 +177,17 @@ def sensor_sd_prior(): @pytest.fixture def sensor_sd_prior_tight(): """ - Tight gamma prior for deterministic-like behavior. + Tight truncated normal prior for deterministic-like behavior. Returns ------- - GammaGroupSdPrior - A gamma prior with small mean for tight behavior. + TruncatedNormalGroupSdPrior + A truncated normal prior with small scale for tight behavior. """ - return GammaGroupSdPrior( + return TruncatedNormalGroupSdPrior( name="ww_sensor_sd", - sd_mean_rv=DeterministicVariable("sd_mean_tight", 0.01), - sd_concentration_rv=DeterministicVariable("sd_concentration_tight", 4.0), + loc_rv=DeterministicVariable("sd_loc_tight", 0.01), + scale_rv=DeterministicVariable("sd_scale_tight", 0.005), sd_min=0.005, ) diff --git a/test/test_observation_measurements.py b/test/test_observation_measurements.py index ae5b70c2..4287404d 100644 --- a/test/test_observation_measurements.py +++ b/test/test_observation_measurements.py @@ -11,7 +11,7 @@ from pyrenew.deterministic import DeterministicPMF, DeterministicVariable from pyrenew.observation import HierarchicalNormalNoise, Measurements from pyrenew.observation.base import BaseObservationProcess -from pyrenew.randomvariable import GammaGroupSdPrior, HierarchicalNormalPrior +from pyrenew.randomvariable import HierarchicalNormalPrior, TruncatedNormalGroupSdPrior class ConcreteMeasurements(Measurements): @@ -79,10 +79,10 @@ def test_infection_resolution_is_subpop(self): sensor_mode_prior = HierarchicalNormalPrior( name="mode", sd_rv=DeterministicVariable("sd", 0.5) ) - sensor_sd_prior = GammaGroupSdPrior( + sensor_sd_prior = TruncatedNormalGroupSdPrior( name="sd", - sd_mean_rv=DeterministicVariable("mean", 0.3), - sd_concentration_rv=DeterministicVariable("conc", 4.0), + loc_rv=DeterministicVariable("loc", 0.3), + scale_rv=DeterministicVariable("scale", 0.15), ) noise = HierarchicalNormalNoise(sensor_mode_prior, sensor_sd_prior) @@ -102,10 +102,10 @@ def test_validate(self): sensor_mode_prior = HierarchicalNormalPrior( name="mode", sd_rv=DeterministicVariable("mode_sd", 0.5) ) - sensor_sd_prior = GammaGroupSdPrior( + sensor_sd_prior = TruncatedNormalGroupSdPrior( name="sd", - sd_mean_rv=DeterministicVariable("sd_mean", 0.3), - sd_concentration_rv=DeterministicVariable("sd_conc", 4.0), + loc_rv=DeterministicVariable("sd_loc", 0.3), + scale_rv=DeterministicVariable("sd_scale", 0.15), ) noise = HierarchicalNormalNoise(sensor_mode_prior, sensor_sd_prior) # Should not raise - validation is deferred to sample time @@ -116,10 +116,10 @@ def test_sample_shape(self): sensor_mode_prior = HierarchicalNormalPrior( name="mode", sd_rv=DeterministicVariable("mode_sd", 0.5) ) - sensor_sd_prior = GammaGroupSdPrior( + sensor_sd_prior = TruncatedNormalGroupSdPrior( name="sd", - sd_mean_rv=DeterministicVariable("sd_mean", 0.3), - sd_concentration_rv=DeterministicVariable("sd_conc", 4.0), + loc_rv=DeterministicVariable("sd_loc", 0.3), + scale_rv=DeterministicVariable("sd_scale", 0.15), ) noise = HierarchicalNormalNoise(sensor_mode_prior, sensor_sd_prior) @@ -142,10 +142,10 @@ def test_sample_with_observations(self): sensor_mode_prior = HierarchicalNormalPrior( name="mode", sd_rv=DeterministicVariable("mode_sd", 0.5) ) - sensor_sd_prior = GammaGroupSdPrior( + sensor_sd_prior = TruncatedNormalGroupSdPrior( name="sd", - sd_mean_rv=DeterministicVariable("sd_mean", 0.3), - sd_concentration_rv=DeterministicVariable("sd_conc", 4.0), + loc_rv=DeterministicVariable("sd_loc", 0.3), + scale_rv=DeterministicVariable("sd_scale", 0.15), ) noise = HierarchicalNormalNoise(sensor_mode_prior, sensor_sd_prior) @@ -175,10 +175,10 @@ def test_sample_shape(self): sensor_mode_prior = HierarchicalNormalPrior( name="mode", sd_rv=DeterministicVariable("mode_sd", 0.5) ) - sensor_sd_prior = GammaGroupSdPrior( + sensor_sd_prior = TruncatedNormalGroupSdPrior( name="sd", - sd_mean_rv=DeterministicVariable("sd_mean", 0.3), - sd_concentration_rv=DeterministicVariable("sd_conc", 4.0), + loc_rv=DeterministicVariable("sd_loc", 0.3), + scale_rv=DeterministicVariable("sd_scale", 0.15), ) noise = HierarchicalNormalNoise(sensor_mode_prior, sensor_sd_prior) @@ -212,10 +212,10 @@ def test_predicted_obs_stored(self): sensor_mode_prior = HierarchicalNormalPrior( name="mode", sd_rv=DeterministicVariable("mode_sd", 0.01) ) - sensor_sd_prior = GammaGroupSdPrior( + sensor_sd_prior = TruncatedNormalGroupSdPrior( name="sd", - sd_mean_rv=DeterministicVariable("sd_mean", 0.01), - sd_concentration_rv=DeterministicVariable("sd_conc", 4.0), + loc_rv=DeterministicVariable("sd_loc", 0.01), + scale_rv=DeterministicVariable("sd_scale", 0.005), sd_min=0.001, ) noise = HierarchicalNormalNoise(sensor_mode_prior, sensor_sd_prior) diff --git a/test/test_randomvariable_hierarchical.py b/test/test_randomvariable_hierarchical.py index 89a58c3a..8d017a73 100644 --- a/test/test_randomvariable_hierarchical.py +++ b/test/test_randomvariable_hierarchical.py @@ -8,9 +8,9 @@ from pyrenew.deterministic import DeterministicVariable from pyrenew.randomvariable import ( DistributionalVariable, - GammaGroupSdPrior, HierarchicalNormalPrior, StudentTGroupModePrior, + TruncatedNormalGroupSdPrior, ) @@ -69,15 +69,15 @@ def test_accepts_distributional_variable_for_sd(self): assert samples.shape == (5,) -class TestGammaGroupSdPrior: - """Test GammaGroupSdPrior.""" +class TestTruncatedNormalGroupSdPrior: + """Test TruncatedNormalGroupSdPrior.""" def test_sample_shape(self): """Test that sample returns correct shape.""" - prior = GammaGroupSdPrior( + prior = TruncatedNormalGroupSdPrior( "sd", - sd_mean_rv=DeterministicVariable("sd_mean", 0.5), - sd_concentration_rv=DeterministicVariable("sd_conc", 4.0), + loc_rv=DeterministicVariable("sd_loc", 0.5), + scale_rv=DeterministicVariable("sd_scale", 0.15), ) with numpyro.handlers.seed(rng_seed=42): @@ -87,10 +87,10 @@ def test_sample_shape(self): def test_respects_sd_min(self): """Test that sd_min is enforced as lower bound.""" - prior = GammaGroupSdPrior( + prior = TruncatedNormalGroupSdPrior( "sd", - sd_mean_rv=DeterministicVariable("sd_mean", 0.1), - sd_concentration_rv=DeterministicVariable("sd_conc", 4.0), + loc_rv=DeterministicVariable("sd_loc", 0.1), + scale_rv=DeterministicVariable("sd_scale", 0.05), sd_min=0.5, ) @@ -101,38 +101,36 @@ def test_respects_sd_min(self): def test_rejects_non_random_variable_params(self): """Test that non-RandomVariable parameters are rejected.""" - with pytest.raises(TypeError, match="sd_mean_rv must be a RandomVariable"): - GammaGroupSdPrior( + with pytest.raises(TypeError, match="loc_rv must be a RandomVariable"): + TruncatedNormalGroupSdPrior( "sd", - sd_mean_rv=0.5, - sd_concentration_rv=DeterministicVariable("sd_conc", 4.0), + loc_rv=0.5, + scale_rv=DeterministicVariable("sd_scale", 0.15), ) - with pytest.raises( - TypeError, match="sd_concentration_rv must be a RandomVariable" - ): - GammaGroupSdPrior( + with pytest.raises(TypeError, match="scale_rv must be a RandomVariable"): + TruncatedNormalGroupSdPrior( "sd", - sd_mean_rv=DeterministicVariable("sd_mean", 0.5), - sd_concentration_rv=4.0, + loc_rv=DeterministicVariable("sd_loc", 0.5), + scale_rv=0.15, ) def test_rejects_negative_sd_min(self): """Test that negative sd_min is rejected.""" with pytest.raises(ValueError, match="sd_min must be non-negative"): - GammaGroupSdPrior( + TruncatedNormalGroupSdPrior( "sd", - sd_mean_rv=DeterministicVariable("sd_mean", 0.5), - sd_concentration_rv=DeterministicVariable("sd_conc", 4.0), + loc_rv=DeterministicVariable("sd_loc", 0.5), + scale_rv=DeterministicVariable("sd_scale", 0.15), sd_min=-0.1, ) def test_validate(self): """Test that validate() runs without error.""" - prior = GammaGroupSdPrior( + prior = TruncatedNormalGroupSdPrior( "sd", - sd_mean_rv=DeterministicVariable("sd_mean", 0.5), - sd_concentration_rv=DeterministicVariable("sd_conc", 4.0), + loc_rv=DeterministicVariable("sd_loc", 0.5), + scale_rv=DeterministicVariable("sd_scale", 0.15), ) prior.validate() # Should not raise From 910fa70848a55230861cedd9137109e6cbb44008 Mon Sep 17 00:00:00 2001 From: Mitzi Morris Date: Wed, 7 Jan 2026 12:38:09 -0500 Subject: [PATCH 27/81] Remove the provided helper classes for sampling priors (i.e. pyrenew/randomvariable/hierarchical.py), per reviewer comments --- .../observation_processes_measurements.qmd | 34 +- pyrenew/observation/noise.py | 61 ++-- pyrenew/randomvariable/__init__.py | 8 - pyrenew/randomvariable/hierarchical.py | 339 ------------------ test/conftest.py | 55 ++- test/test_observation_measurements.py | 78 ++-- test/test_randomvariable_hierarchical.py | 202 ----------- 7 files changed, 96 insertions(+), 681 deletions(-) delete mode 100644 pyrenew/randomvariable/hierarchical.py delete mode 100644 test/test_randomvariable_hierarchical.py diff --git a/docs/tutorials/observation_processes_measurements.qmd b/docs/tutorials/observation_processes_measurements.qmd index 9b32567e..edfb807b 100644 --- a/docs/tutorials/observation_processes_measurements.qmd +++ b/docs/tutorials/observation_processes_measurements.qmd @@ -15,11 +15,10 @@ import numpy as np import numpyro import matplotlib.pyplot as plt +import numpyro.distributions as dist + from pyrenew.observation import Measurements, HierarchicalNormalNoise -from pyrenew.randomvariable import ( - HierarchicalNormalPrior, - TruncatedNormalGroupSdPrior, -) +from pyrenew.randomvariable import DistributionalVariable from pyrenew.deterministic import DeterministicVariable, DeterministicPMF ``` @@ -215,33 +214,22 @@ ml_per_person_per_day = 1000.0 A measurement depends on both the thing being measured (e.g., wastewater from a treatment plant) and the calibration of the lab/instruments used to obtain that measurement. We call this combination a "sensor"—the WWTP/lab pair that determines the measurement characteristics. Different sensors have systematic biases and variabilities that we model with hierarchical effects. -For the sensor-level standard deviations, we use a truncated normal prior. This is a common choice for SD parameters because: - -- It naturally constrains values to be positive -- The `loc` parameter represents our prior expectation for the SD -- The `scale` parameter controls how uncertain we are about the SD -- The `sd_min` floor prevents numerical issues when SDs approach zero +The `HierarchicalNormalNoise` model samples sensor-level parameters within a plate context, so any `RandomVariable` can be used. For the sensor-level standard deviations, we use a truncated normal prior—a common choice for SD parameters because it naturally constrains values to be positive. ```{python} # | label: noise-model -# Sensor-level mode: systematic differences between sensors -sensor_mode_prior = HierarchicalNormalPrior( - name="ww_sensor_mode", - sd_rv=DeterministicVariable("mode_sd", 0.5), -) +# Sensor-level mode: systematic differences between sensors (zero-centered normal) +sensor_mode_rv = DistributionalVariable("ww_sensor_mode", dist.Normal(0, 0.5)) -# Sensor-level SD: measurement variability within each sensor -sensor_sd_prior = TruncatedNormalGroupSdPrior( - name="ww_sensor_sd", - loc_rv=DeterministicVariable("sd_loc", 0.3), - scale_rv=DeterministicVariable("sd_scale", 0.15), - sd_min=0.10, +# Sensor-level SD: measurement variability within each sensor (truncated normal) +sensor_sd_rv = DistributionalVariable( + "ww_sensor_sd", dist.TruncatedNormal(loc=0.3, scale=0.15, low=0.10) ) # Create the noise model ww_noise = HierarchicalNormalNoise( - sensor_mode_prior_rv=sensor_mode_prior, - sensor_sd_prior_rv=sensor_sd_prior, + sensor_mode_rv=sensor_mode_rv, + sensor_sd_rv=sensor_sd_rv, ) ``` diff --git a/pyrenew/observation/noise.py b/pyrenew/observation/noise.py index 47d2d4ad..9ce282e7 100644 --- a/pyrenew/observation/noise.py +++ b/pyrenew/observation/noise.py @@ -263,55 +263,63 @@ class HierarchicalNormalNoise(MeasurementNoise): Normal noise with hierarchical sensor-level effects. Observation model: ``obs ~ Normal(predicted + sensor_mode, sensor_sd)`` - where sensor_mode and sensor_sd are hierarchically modeled. + where sensor_mode and sensor_sd are sampled per-sensor within a plate. Parameters ---------- - sensor_mode_prior_rv : RandomVariable - Hierarchical prior for sensor-level modes (log-scale biases). - Must support ``sample(n_groups=...)`` interface. - sensor_sd_prior_rv : RandomVariable - Hierarchical prior for sensor-level SDs (must be > 0). - Must support ``sample(n_groups=...)`` interface. + sensor_mode_rv : RandomVariable + Prior for sensor-level modes (log-scale biases). + Sampled once per sensor within a plate context. + Example: ``DistributionalVariable("mode", dist.Normal(0, 0.5))`` + sensor_sd_rv : RandomVariable + Prior for sensor-level SDs (should be > 0). + Sampled once per sensor within a plate context. + Example: ``DistributionalVariable("sd", dist.TruncatedNormal(0.3, 0.15, low=0.05))`` Notes ----- Expects data already on log scale for wastewater applications. - See Also + The sensor-level parameters are sampled within a numpyro plate context, + so any standard RandomVariable can be used (no special interface required). + + Examples -------- - pyrenew.randomvariable.HierarchicalNormalPrior : - Suitable prior for sensor_mode_prior_rv - pyrenew.randomvariable.TruncatedNormalGroupSdPrior : - Suitable prior for sensor_sd_prior_rv + >>> from pyrenew.randomvariable import DistributionalVariable + >>> import numpyro.distributions as dist + >>> + >>> noise = HierarchicalNormalNoise( + ... sensor_mode_rv=DistributionalVariable("mode", dist.Normal(0, 0.5)), + ... sensor_sd_rv=DistributionalVariable("sd", dist.TruncatedNormal(0.3, 0.15, low=0.05)), + ... ) """ def __init__( self, - sensor_mode_prior_rv: RandomVariable, - sensor_sd_prior_rv: RandomVariable, + sensor_mode_rv: RandomVariable, + sensor_sd_rv: RandomVariable, ) -> None: """ Initialize hierarchical Normal noise. Parameters ---------- - sensor_mode_prior_rv : RandomVariable - Hierarchical prior for sensor-level modes (log-scale biases). - Must support ``sample(n_groups=...)`` interface. - sensor_sd_prior_rv : RandomVariable - Hierarchical prior for sensor-level SDs (must be > 0). - Must support ``sample(n_groups=...)`` interface. + sensor_mode_rv : RandomVariable + Prior for sensor-level modes (log-scale biases). + Sampled once per sensor within a plate context. + sensor_sd_rv : RandomVariable + Prior for sensor-level SDs (should be > 0). + Sampled once per sensor within a plate context. """ - self.sensor_mode_prior_rv = sensor_mode_prior_rv - self.sensor_sd_prior_rv = sensor_sd_prior_rv + self.sensor_mode_rv = sensor_mode_rv + self.sensor_sd_rv = sensor_sd_rv def __repr__(self) -> str: """Return string representation.""" return ( f"HierarchicalNormalNoise(" - f"sensor_mode_prior_rv={self.sensor_mode_prior_rv!r}, " - f"sensor_sd_prior_rv={self.sensor_sd_prior_rv!r})" + f"sensor_mode_rv={self.sensor_mode_rv!r}, " + f"sensor_sd_rv={self.sensor_sd_rv!r})" ) def validate(self) -> None: @@ -363,8 +371,9 @@ def sample( ValueError If sensor_sd samples non-positive values. """ - sensor_mode = self.sensor_mode_prior_rv.sample(n_groups=n_sensors) - sensor_sd = self.sensor_sd_prior_rv.sample(n_groups=n_sensors) + with numpyro.plate("sensor", n_sensors): + sensor_mode = self.sensor_mode_rv() + sensor_sd = self.sensor_sd_rv() loc = predicted + sensor_mode[sensor_indices] scale = sensor_sd[sensor_indices] diff --git a/pyrenew/randomvariable/__init__.py b/pyrenew/randomvariable/__init__.py index 09812b9c..c599d101 100644 --- a/pyrenew/randomvariable/__init__.py +++ b/pyrenew/randomvariable/__init__.py @@ -5,11 +5,6 @@ DynamicDistributionalVariable, StaticDistributionalVariable, ) -from pyrenew.randomvariable.hierarchical import ( - HierarchicalNormalPrior, - StudentTGroupModePrior, - TruncatedNormalGroupSdPrior, -) from pyrenew.randomvariable.transformedvariable import TransformedVariable __all__ = [ @@ -17,7 +12,4 @@ "StaticDistributionalVariable", "DynamicDistributionalVariable", "TransformedVariable", - "HierarchicalNormalPrior", - "TruncatedNormalGroupSdPrior", - "StudentTGroupModePrior", ] diff --git a/pyrenew/randomvariable/hierarchical.py b/pyrenew/randomvariable/hierarchical.py deleted file mode 100644 index 45528d87..00000000 --- a/pyrenew/randomvariable/hierarchical.py +++ /dev/null @@ -1,339 +0,0 @@ -# numpydoc ignore=GL08 -""" -Hierarchical prior distributions for group-level random effects. - -These classes provide random variables that sample from hierarchical -distributions with a `sample(n_groups=...)` interface, enabling -dynamic group sizes at sample time with proper numpyro plate contexts. -""" - -import numpyro -import numpyro.distributions as dist - -from pyrenew.metaclass import RandomVariable - - -class HierarchicalNormalPrior(RandomVariable): - """ - Zero-centered Normal prior for group-level effects. - - Samples n_groups values from Normal(0, sd) within a numpyro plate context. - - Parameters - ---------- - name : str - Unique name for the sampled parameter in numpyro. - sd_rv : RandomVariable - RandomVariable returning the standard deviation. - - Notes - ----- - This class is designed for hierarchical models where group effects - are assumed to be drawn from a common distribution centered at zero. - The number of groups is specified at sample time, allowing dynamic - group sizes. - - Examples - -------- - >>> from pyrenew.deterministic import DeterministicVariable - >>> from pyrenew.randomvariable import HierarchicalNormalPrior - >>> import numpyro - >>> - >>> sd_rv = DeterministicVariable("sd", 0.5) - >>> prior = HierarchicalNormalPrior("site_effects", sd_rv) - >>> - >>> with numpyro.handlers.seed(rng_seed=42): - ... effects = prior.sample(n_groups=5) - >>> effects.shape - (5,) - """ - - def __init__( - self, - name: str, - sd_rv: RandomVariable, - ) -> None: - """ - Default constructor for HierarchicalNormalPrior. - - Parameters - ---------- - name : str - Unique name for the sampled parameter in numpyro. - sd_rv : RandomVariable - RandomVariable returning the standard deviation. - - Returns - ------- - None - """ - if not isinstance(sd_rv, RandomVariable): - raise TypeError( - f"sd_rv must be a RandomVariable, got {type(sd_rv).__name__}. " - "Use DeterministicVariable(name, value) to wrap a fixed value." - ) - - self.name = name - self.sd_rv = sd_rv - - def validate(self): - """Validate the random variable (no-op for this class).""" - pass - - def sample(self, n_groups: int, **kwargs): - """ - Sample group-level effects. - - Parameters - ---------- - n_groups : int - Number of groups. - **kwargs - Additional keyword arguments (unused). - - Returns - ------- - ArrayLike - Array of shape (n_groups,) containing sampled effects. - """ - sd = self.sd_rv() - - with numpyro.plate(f"n_{self.name}", n_groups): - effects = numpyro.sample( - self.name, - dist.Normal(0.0, sd), - ) - return effects - - -class TruncatedNormalGroupSdPrior(RandomVariable): - """ - Truncated Normal prior for group-level standard deviations. - - Samples n_groups positive values from TruncatedNormal(loc, scale, low=sd_min) - within a numpyro plate context. Uses NumPyro's native truncated distribution - support for proper likelihood computation. - - Parameters - ---------- - name : str - Unique name for the sampled parameter in numpyro. - loc_rv : RandomVariable - RandomVariable returning the location (mean) of the underlying Normal. - scale_rv : RandomVariable - RandomVariable returning the scale (std) of the underlying Normal. - sd_min : float, default=0.05 - Minimum SD value (left truncation point). - - Notes - ----- - Truncated normal is a common choice for standard deviation priors because: - - - It naturally constrains values to be positive (via left truncation) - - The location parameter represents the expected SD value - - The scale parameter controls uncertainty about the SD - - It integrates well with NumPyro's truncation support for proper likelihood - - Parameter guidance: - - - ``loc``: Set to your prior expectation for the SD (e.g., 0.3 for - moderate sensor variability on log scale) - - ``scale``: Controls uncertainty; smaller values give tighter priors - around ``loc`` - - ``sd_min``: Prevents numerical issues when SDs approach zero; typical - values are 0.01-0.1 depending on the scale of the data - - Examples - -------- - >>> from pyrenew.deterministic import DeterministicVariable - >>> from pyrenew.randomvariable import TruncatedNormalGroupSdPrior - >>> import numpyro - >>> - >>> loc_rv = DeterministicVariable("sd_loc", 0.3) - >>> scale_rv = DeterministicVariable("sd_scale", 0.15) - >>> prior = TruncatedNormalGroupSdPrior("site_sd", loc_rv, scale_rv, sd_min=0.05) - >>> - >>> with numpyro.handlers.seed(rng_seed=42): - ... sds = prior.sample(n_groups=5) - >>> all(sds >= 0.05) - True - """ - - def __init__( - self, - name: str, - loc_rv: RandomVariable, - scale_rv: RandomVariable, - sd_min: float = 0.05, - ) -> None: - """ - Default constructor for TruncatedNormalGroupSdPrior. - - Parameters - ---------- - name : str - Unique name for the sampled parameter in numpyro. - loc_rv : RandomVariable - RandomVariable returning the location (mean) of the underlying Normal. - scale_rv : RandomVariable - RandomVariable returning the scale (std) of the underlying Normal. - sd_min : float, default=0.05 - Minimum SD value (left truncation point). - - Returns - ------- - None - """ - if not isinstance(loc_rv, RandomVariable): - raise TypeError( - f"loc_rv must be a RandomVariable, got {type(loc_rv).__name__}. " - "Use DeterministicVariable(name, value) to wrap a fixed value." - ) - if not isinstance(scale_rv, RandomVariable): - raise TypeError( - f"scale_rv must be a RandomVariable, got {type(scale_rv).__name__}. " - "Use DeterministicVariable(name, value) to wrap a fixed value." - ) - if sd_min < 0: - raise ValueError(f"sd_min must be non-negative, got {sd_min}") - - self.name = name - self.loc_rv = loc_rv - self.scale_rv = scale_rv - self.sd_min = sd_min - - def validate(self): - """Validate the random variable (no-op for this class).""" - pass - - def sample(self, n_groups: int, **kwargs): - """ - Sample group-level standard deviations. - - Parameters - ---------- - n_groups : int - Number of groups. - **kwargs - Additional keyword arguments (unused). - - Returns - ------- - ArrayLike - Array of shape (n_groups,) with values >= sd_min. - """ - loc = self.loc_rv() - scale = self.scale_rv() - - with numpyro.plate(f"n_{self.name}", n_groups): - group_sd = numpyro.sample( - self.name, - dist.TruncatedNormal(loc=loc, scale=scale, low=self.sd_min), - ) - return group_sd - - -class StudentTGroupModePrior(RandomVariable): - """ - Zero-centered Student-t prior for group-level modes (robust alternative to Normal). - - Samples n_groups values from StudentT(df, 0, sd) within a numpyro plate context. - This is useful when group effects may have heavier tails than a Normal distribution. - - Parameters - ---------- - name : str - Unique name for the sampled parameter in numpyro. - sd_rv : RandomVariable - RandomVariable returning the scale parameter. - df_rv : RandomVariable - RandomVariable returning the degrees of freedom. - - Notes - ----- - The Student-t distribution approaches the Normal distribution as df -> infinity. - Lower df values give heavier tails, making the prior more robust to outliers. - Common choices include df=3 (heavy tails) or df=7 (moderate tails). - - Examples - -------- - >>> from pyrenew.deterministic import DeterministicVariable - >>> from pyrenew.randomvariable import StudentTGroupModePrior - >>> import numpyro - >>> - >>> sd_rv = DeterministicVariable("scale", 0.5) - >>> df_rv = DeterministicVariable("df", 4.0) - >>> prior = StudentTGroupModePrior("site_modes", sd_rv, df_rv) - >>> - >>> with numpyro.handlers.seed(rng_seed=42): - ... modes = prior.sample(n_groups=5) - >>> modes.shape - (5,) - """ - - def __init__( - self, - name: str, - sd_rv: RandomVariable, - df_rv: RandomVariable, - ) -> None: - """ - Default constructor for StudentTGroupModePrior. - - Parameters - ---------- - name : str - Unique name for the sampled parameter in numpyro. - sd_rv : RandomVariable - RandomVariable returning the scale parameter. - df_rv : RandomVariable - RandomVariable returning the degrees of freedom. - - Returns - ------- - None - """ - if not isinstance(sd_rv, RandomVariable): - raise TypeError( - f"sd_rv must be a RandomVariable, got {type(sd_rv).__name__}. " - "Use DeterministicVariable(name, value) to wrap a fixed value." - ) - if not isinstance(df_rv, RandomVariable): - raise TypeError( - f"df_rv must be a RandomVariable, got {type(df_rv).__name__}. " - "Use DeterministicVariable(name, value) to wrap a fixed value." - ) - - self.name = name - self.sd_rv = sd_rv - self.df_rv = df_rv - - def validate(self): - """Validate the random variable (no-op for this class).""" - pass - - def sample(self, n_groups: int, **kwargs): - """ - Sample group-level modes. - - Parameters - ---------- - n_groups : int - Number of groups. - **kwargs - Additional keyword arguments (unused). - - Returns - ------- - ArrayLike - Array of shape (n_groups,) containing sampled modes. - """ - sd = self.sd_rv() - df = self.df_rv() - - with numpyro.plate(f"n_{self.name}", n_groups): - effects = numpyro.sample( - self.name, - dist.StudentT(df=df, loc=0.0, scale=sd), - ) - return effects diff --git a/test/conftest.py b/test/conftest.py index d159106a..bc95fd10 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -6,11 +6,12 @@ """ import jax.numpy as jnp +import numpyro.distributions as dist import pytest from pyrenew.deterministic import DeterministicPMF, DeterministicVariable from pyrenew.observation import Counts, NegativeBinomialNoise -from pyrenew.randomvariable import HierarchicalNormalPrior, TruncatedNormalGroupSdPrior +from pyrenew.randomvariable import DistributionalVariable # ============================================================================= # PMF Fixtures @@ -122,73 +123,63 @@ def medium_shedding_pmf(): # ============================================================================= -# Hierarchical Prior Fixtures +# Sensor Prior Fixtures # ============================================================================= @pytest.fixture -def sensor_mode_prior(): +def sensor_mode_rv(): """ - Standard hierarchical normal prior for sensor modes. + Standard normal prior for sensor modes. Returns ------- - HierarchicalNormalPrior - A hierarchical normal prior with standard deviation 0.5. + DistributionalVariable + A normal prior with standard deviation 0.5. """ - return HierarchicalNormalPrior( - name="ww_sensor_mode", sd_rv=DeterministicVariable("mode_sd", 0.5) - ) + return DistributionalVariable("ww_sensor_mode", dist.Normal(0, 0.5)) @pytest.fixture -def sensor_mode_prior_tight(): +def sensor_mode_rv_tight(): """ - Tight hierarchical normal prior for deterministic-like behavior. + Tight normal prior for deterministic-like behavior. Returns ------- - HierarchicalNormalPrior - A hierarchical normal prior with small standard deviation 0.01. + DistributionalVariable + A normal prior with small standard deviation 0.01. """ - return HierarchicalNormalPrior( - name="ww_sensor_mode", sd_rv=DeterministicVariable("mode_sd_tight", 0.01) - ) + return DistributionalVariable("ww_sensor_mode", dist.Normal(0, 0.01)) @pytest.fixture -def sensor_sd_prior(): +def sensor_sd_rv(): """ Standard truncated normal prior for sensor standard deviations. Returns ------- - TruncatedNormalGroupSdPrior - A truncated normal prior for group standard deviations. - """ - return TruncatedNormalGroupSdPrior( - name="ww_sensor_sd", - loc_rv=DeterministicVariable("sd_loc", 0.3), - scale_rv=DeterministicVariable("sd_scale", 0.15), - sd_min=0.10, + DistributionalVariable + A truncated normal prior for sensor standard deviations. + """ + return DistributionalVariable( + "ww_sensor_sd", dist.TruncatedNormal(0.3, 0.15, low=0.10) ) @pytest.fixture -def sensor_sd_prior_tight(): +def sensor_sd_rv_tight(): """ Tight truncated normal prior for deterministic-like behavior. Returns ------- - TruncatedNormalGroupSdPrior + DistributionalVariable A truncated normal prior with small scale for tight behavior. """ - return TruncatedNormalGroupSdPrior( - name="ww_sensor_sd", - loc_rv=DeterministicVariable("sd_loc_tight", 0.01), - scale_rv=DeterministicVariable("sd_scale_tight", 0.005), - sd_min=0.005, + return DistributionalVariable( + "ww_sensor_sd", dist.TruncatedNormal(0.01, 0.005, low=0.005) ) diff --git a/test/test_observation_measurements.py b/test/test_observation_measurements.py index 4287404d..32e053eb 100644 --- a/test/test_observation_measurements.py +++ b/test/test_observation_measurements.py @@ -6,12 +6,13 @@ import jax.numpy as jnp import numpyro +import numpyro.distributions as dist import pytest -from pyrenew.deterministic import DeterministicPMF, DeterministicVariable +from pyrenew.deterministic import DeterministicPMF from pyrenew.observation import HierarchicalNormalNoise, Measurements from pyrenew.observation.base import BaseObservationProcess -from pyrenew.randomvariable import HierarchicalNormalPrior, TruncatedNormalGroupSdPrior +from pyrenew.randomvariable import DistributionalVariable class ConcreteMeasurements(Measurements): @@ -76,15 +77,11 @@ def test_is_base_observation_process(self): def test_infection_resolution_is_subpop(self): """Test that Measurements returns 'subpop' resolution.""" shedding_pmf = jnp.array([0.3, 0.4, 0.3]) - sensor_mode_prior = HierarchicalNormalPrior( - name="mode", sd_rv=DeterministicVariable("sd", 0.5) + sensor_mode_rv = DistributionalVariable("mode", dist.Normal(0, 0.5)) + sensor_sd_rv = DistributionalVariable( + "sd", dist.TruncatedNormal(0.3, 0.15, low=0.05) ) - sensor_sd_prior = TruncatedNormalGroupSdPrior( - name="sd", - loc_rv=DeterministicVariable("loc", 0.3), - scale_rv=DeterministicVariable("scale", 0.15), - ) - noise = HierarchicalNormalNoise(sensor_mode_prior, sensor_sd_prior) + noise = HierarchicalNormalNoise(sensor_mode_rv, sensor_sd_rv) process = ConcreteMeasurements( temporal_pmf_rv=DeterministicPMF("shedding", shedding_pmf), @@ -99,29 +96,21 @@ class TestHierarchicalNormalNoise: def test_validate(self): """Test HierarchicalNormalNoise validate method.""" - sensor_mode_prior = HierarchicalNormalPrior( - name="mode", sd_rv=DeterministicVariable("mode_sd", 0.5) - ) - sensor_sd_prior = TruncatedNormalGroupSdPrior( - name="sd", - loc_rv=DeterministicVariable("sd_loc", 0.3), - scale_rv=DeterministicVariable("sd_scale", 0.15), + sensor_mode_rv = DistributionalVariable("mode", dist.Normal(0, 0.5)) + sensor_sd_rv = DistributionalVariable( + "sd", dist.TruncatedNormal(0.3, 0.15, low=0.05) ) - noise = HierarchicalNormalNoise(sensor_mode_prior, sensor_sd_prior) + noise = HierarchicalNormalNoise(sensor_mode_rv, sensor_sd_rv) # Should not raise - validation is deferred to sample time noise.validate() def test_sample_shape(self): """Test that HierarchicalNormalNoise produces correct shape.""" - sensor_mode_prior = HierarchicalNormalPrior( - name="mode", sd_rv=DeterministicVariable("mode_sd", 0.5) - ) - sensor_sd_prior = TruncatedNormalGroupSdPrior( - name="sd", - loc_rv=DeterministicVariable("sd_loc", 0.3), - scale_rv=DeterministicVariable("sd_scale", 0.15), + sensor_mode_rv = DistributionalVariable("mode", dist.Normal(0, 0.5)) + sensor_sd_rv = DistributionalVariable( + "sd", dist.TruncatedNormal(0.3, 0.15, low=0.05) ) - noise = HierarchicalNormalNoise(sensor_mode_prior, sensor_sd_prior) + noise = HierarchicalNormalNoise(sensor_mode_rv, sensor_sd_rv) predicted = jnp.array([1.0, 2.0, 3.0, 4.0]) sensor_indices = jnp.array([0, 0, 1, 1]) @@ -139,15 +128,11 @@ def test_sample_shape(self): def test_sample_with_observations(self): """Test that HierarchicalNormalNoise conditions on observations.""" - sensor_mode_prior = HierarchicalNormalPrior( - name="mode", sd_rv=DeterministicVariable("mode_sd", 0.5) + sensor_mode_rv = DistributionalVariable("mode", dist.Normal(0, 0.5)) + sensor_sd_rv = DistributionalVariable( + "sd", dist.TruncatedNormal(0.3, 0.15, low=0.05) ) - sensor_sd_prior = TruncatedNormalGroupSdPrior( - name="sd", - loc_rv=DeterministicVariable("sd_loc", 0.3), - scale_rv=DeterministicVariable("sd_scale", 0.15), - ) - noise = HierarchicalNormalNoise(sensor_mode_prior, sensor_sd_prior) + noise = HierarchicalNormalNoise(sensor_mode_rv, sensor_sd_rv) predicted = jnp.array([1.0, 2.0, 3.0, 4.0]) obs = jnp.array([1.1, 2.1, 3.1, 4.1]) @@ -172,15 +157,11 @@ class TestConcreteMeasurements: def test_sample_shape(self): """Test that sample returns correct shape.""" shedding_pmf = jnp.array([0.3, 0.4, 0.3]) - sensor_mode_prior = HierarchicalNormalPrior( - name="mode", sd_rv=DeterministicVariable("mode_sd", 0.5) - ) - sensor_sd_prior = TruncatedNormalGroupSdPrior( - name="sd", - loc_rv=DeterministicVariable("sd_loc", 0.3), - scale_rv=DeterministicVariable("sd_scale", 0.15), + sensor_mode_rv = DistributionalVariable("mode", dist.Normal(0, 0.5)) + sensor_sd_rv = DistributionalVariable( + "sd", dist.TruncatedNormal(0.3, 0.15, low=0.05) ) - noise = HierarchicalNormalNoise(sensor_mode_prior, sensor_sd_prior) + noise = HierarchicalNormalNoise(sensor_mode_rv, sensor_sd_rv) process = ConcreteMeasurements( temporal_pmf_rv=DeterministicPMF("shedding", shedding_pmf), @@ -209,16 +190,11 @@ def test_sample_shape(self): def test_predicted_obs_stored(self): """Test that predicted_log_conc is stored as deterministic.""" shedding_pmf = jnp.array([0.5, 0.5]) - sensor_mode_prior = HierarchicalNormalPrior( - name="mode", sd_rv=DeterministicVariable("mode_sd", 0.01) - ) - sensor_sd_prior = TruncatedNormalGroupSdPrior( - name="sd", - loc_rv=DeterministicVariable("sd_loc", 0.01), - scale_rv=DeterministicVariable("sd_scale", 0.005), - sd_min=0.001, + sensor_mode_rv = DistributionalVariable("mode", dist.Normal(0, 0.01)) + sensor_sd_rv = DistributionalVariable( + "sd", dist.TruncatedNormal(0.01, 0.005, low=0.001) ) - noise = HierarchicalNormalNoise(sensor_mode_prior, sensor_sd_prior) + noise = HierarchicalNormalNoise(sensor_mode_rv, sensor_sd_rv) process = ConcreteMeasurements( temporal_pmf_rv=DeterministicPMF("shedding", shedding_pmf), diff --git a/test/test_randomvariable_hierarchical.py b/test/test_randomvariable_hierarchical.py deleted file mode 100644 index 8d017a73..00000000 --- a/test/test_randomvariable_hierarchical.py +++ /dev/null @@ -1,202 +0,0 @@ -"""Unit tests for hierarchical prior distributions.""" - -import jax.numpy as jnp -import numpyro -import numpyro.distributions as dist -import pytest - -from pyrenew.deterministic import DeterministicVariable -from pyrenew.randomvariable import ( - DistributionalVariable, - HierarchicalNormalPrior, - StudentTGroupModePrior, - TruncatedNormalGroupSdPrior, -) - - -class TestHierarchicalNormalPrior: - """Test HierarchicalNormalPrior.""" - - def test_sample_shape(self): - """Test that sample returns correct shape.""" - prior = HierarchicalNormalPrior( - "effect", sd_rv=DeterministicVariable("sd", 1.0) - ) - - with numpyro.handlers.seed(rng_seed=42): - samples = prior.sample(n_groups=5) - - assert samples.shape == (5,) - - def test_smaller_sd_produces_tighter_distribution(self): - """Test that smaller sd produces samples closer to zero.""" - prior_tight = HierarchicalNormalPrior( - "a", sd_rv=DeterministicVariable("sd_tight", 0.1) - ) - prior_wide = HierarchicalNormalPrior( - "b", sd_rv=DeterministicVariable("sd_wide", 10.0) - ) - - n_samples = 1000 - with numpyro.handlers.seed(rng_seed=42): - samples_tight = prior_tight.sample(n_groups=n_samples) - with numpyro.handlers.seed(rng_seed=43): - samples_wide = prior_wide.sample(n_groups=n_samples) - - # Tight prior should have smaller standard deviation - assert jnp.std(samples_tight) < jnp.std(samples_wide) - - def test_validate(self): - """Test that validate() runs without error.""" - prior = HierarchicalNormalPrior( - "effect", sd_rv=DeterministicVariable("sd", 1.0) - ) - prior.validate() # Should not raise - - def test_rejects_non_random_variable_sd(self): - """Test that non-RandomVariable sd_rv is rejected.""" - with pytest.raises(TypeError, match="sd_rv must be a RandomVariable"): - HierarchicalNormalPrior("effect", sd_rv=1.0) - - def test_accepts_distributional_variable_for_sd(self): - """Test that DistributionalVariable can be used for sd_rv.""" - sd_rv = DistributionalVariable("sd", dist.HalfNormal(1.0)) - prior = HierarchicalNormalPrior("effect", sd_rv=sd_rv) - - with numpyro.handlers.seed(rng_seed=42): - samples = prior.sample(n_groups=5) - - assert samples.shape == (5,) - - -class TestTruncatedNormalGroupSdPrior: - """Test TruncatedNormalGroupSdPrior.""" - - def test_sample_shape(self): - """Test that sample returns correct shape.""" - prior = TruncatedNormalGroupSdPrior( - "sd", - loc_rv=DeterministicVariable("sd_loc", 0.5), - scale_rv=DeterministicVariable("sd_scale", 0.15), - ) - - with numpyro.handlers.seed(rng_seed=42): - samples = prior.sample(n_groups=5) - - assert samples.shape == (5,) - - def test_respects_sd_min(self): - """Test that sd_min is enforced as lower bound.""" - prior = TruncatedNormalGroupSdPrior( - "sd", - loc_rv=DeterministicVariable("sd_loc", 0.1), - scale_rv=DeterministicVariable("sd_scale", 0.05), - sd_min=0.5, - ) - - with numpyro.handlers.seed(rng_seed=42): - samples = prior.sample(n_groups=100) - - assert jnp.all(samples >= 0.5) - - def test_rejects_non_random_variable_params(self): - """Test that non-RandomVariable parameters are rejected.""" - with pytest.raises(TypeError, match="loc_rv must be a RandomVariable"): - TruncatedNormalGroupSdPrior( - "sd", - loc_rv=0.5, - scale_rv=DeterministicVariable("sd_scale", 0.15), - ) - - with pytest.raises(TypeError, match="scale_rv must be a RandomVariable"): - TruncatedNormalGroupSdPrior( - "sd", - loc_rv=DeterministicVariable("sd_loc", 0.5), - scale_rv=0.15, - ) - - def test_rejects_negative_sd_min(self): - """Test that negative sd_min is rejected.""" - with pytest.raises(ValueError, match="sd_min must be non-negative"): - TruncatedNormalGroupSdPrior( - "sd", - loc_rv=DeterministicVariable("sd_loc", 0.5), - scale_rv=DeterministicVariable("sd_scale", 0.15), - sd_min=-0.1, - ) - - def test_validate(self): - """Test that validate() runs without error.""" - prior = TruncatedNormalGroupSdPrior( - "sd", - loc_rv=DeterministicVariable("sd_loc", 0.5), - scale_rv=DeterministicVariable("sd_scale", 0.15), - ) - prior.validate() # Should not raise - - -class TestStudentTGroupModePrior: - """Test StudentTGroupModePrior.""" - - def test_sample_shape(self): - """Test that sample returns correct shape.""" - prior = StudentTGroupModePrior( - "mode", - sd_rv=DeterministicVariable("sd", 1.0), - df_rv=DeterministicVariable("df", 4.0), - ) - - with numpyro.handlers.seed(rng_seed=42): - samples = prior.sample(n_groups=5) - - assert samples.shape == (5,) - - def test_heavier_tails_than_normal(self): - """Test Student-t produces more extreme values than Normal.""" - # df=2 gives very heavy tails - student_prior = StudentTGroupModePrior( - "s", - sd_rv=DeterministicVariable("sd_s", 1.0), - df_rv=DeterministicVariable("df", 2.0), - ) - normal_prior = HierarchicalNormalPrior( - "n", sd_rv=DeterministicVariable("sd_n", 1.0) - ) - - n_samples = 5000 - with numpyro.handlers.seed(rng_seed=42): - student_samples = student_prior.sample(n_groups=n_samples) - with numpyro.handlers.seed(rng_seed=42): - normal_samples = normal_prior.sample(n_groups=n_samples) - - # Student-t should have more extreme values (higher max absolute value) - assert jnp.max(jnp.abs(student_samples)) > jnp.max(jnp.abs(normal_samples)) - - def test_rejects_non_random_variable_params(self): - """Test that non-RandomVariable parameters are rejected.""" - with pytest.raises(TypeError, match="sd_rv must be a RandomVariable"): - StudentTGroupModePrior( - "mode", - sd_rv=1.0, - df_rv=DeterministicVariable("df", 4.0), - ) - - with pytest.raises(TypeError, match="df_rv must be a RandomVariable"): - StudentTGroupModePrior( - "mode", - sd_rv=DeterministicVariable("sd", 1.0), - df_rv=4.0, - ) - - def test_validate(self): - """Test that validate() runs without error.""" - prior = StudentTGroupModePrior( - "mode", - sd_rv=DeterministicVariable("sd", 1.0), - df_rv=DeterministicVariable("df", 4.0), - ) - prior.validate() # Should not raise - - -if __name__ == "__main__": - pytest.main([__file__, "-v"]) From 70199552b3eb70641fa0ebdabc555609faef3634 Mon Sep 17 00:00:00 2001 From: Mitzi Morris Date: Wed, 7 Jan 2026 14:48:11 -0500 Subject: [PATCH 28/81] updated observation process measurements per reviewer comments --- .../observation_processes_measurements.qmd | 314 ++++++++++-------- 1 file changed, 179 insertions(+), 135 deletions(-) diff --git a/docs/tutorials/observation_processes_measurements.qmd b/docs/tutorials/observation_processes_measurements.qmd index edfb807b..ab5af73e 100644 --- a/docs/tutorials/observation_processes_measurements.qmd +++ b/docs/tutorials/observation_processes_measurements.qmd @@ -4,7 +4,7 @@ format: gfm engine: jupyter --- -This tutorial demonstrates how to use the `Measurements` observation process to model continuous measurement data such as wastewater viral concentrations. +This tutorial demonstrates how to use the `Measurements` observation process to model continuous measurement data. We first explain the general framework, then illustrate with a wastewater viral concentration example. ```{python} # | label: setup @@ -22,52 +22,155 @@ from pyrenew.randomvariable import DistributionalVariable from pyrenew.deterministic import DeterministicVariable, DeterministicPMF ``` +# Part 1: The Measurements Framework + ## Overview -Measurement observation processes model continuous signals derived from infections, such as viral RNA concentrations in wastewater. Unlike count observations (hospital admissions, deaths), measurements are continuous values and in some cases may be negative. +The `Measurements` class models continuous signals derived from infections. Unlike count observations (hospital admissions, deaths), measurements are continuous values that may span orders of magnitude or even be negative (e.g., log-transformed data). -As an example, we'll consider the wastewater viral genome measurement from the [Pyrenew-HEW](https://github.com/cdcgov/pyrenew-hew) family of models. The predicted measurement on day $t$ is: +**Examples of measurement data:** -$$\lambda_t = \frac{G}{V} \cdot \sum_{d=0}^{D} I_{t-d} \cdot p_d$$ +- Wastewater viral concentrations +- Air quality pathogen levels +- Serological assay results +- Environmental sensor readings -where: +### The general pattern + +All measurement observation processes follow the same pattern: -- $I_{t-d}$ is the number of incident (new) infections on day $t-d$ -- $G$ is a scaling factor (here, genome copies shed per infection) -- $V$ is another scaling factor (here, wastewater volume per person per day) -- $p_d$ is a delay distribution (here, normalized viral genome shedding by time-since-infection) -- $D$ is the maximum delay considered. +$$\text{observed} \sim \text{Noise}\bigl(\text{predicted}(\text{infections})\bigr)$$ -Observed log-concentrations are generated by sampling from a normal distribution: +where: + +1. **`_predicted_obs(infections)`**: Transforms infections into predicted measurement values (you implement this) +2. **Noise model**: Adds stochastic variation around predictions (provided by PyRenew) -$$\log(C_t) \sim \text{Normal}(\mu = \log(\lambda_t), \sigma)$$ +The `Measurements` base class provides: -The log-normal distribution is appropriate for concentration data because concentrations are strictly positive and often span several orders of magnitude. +- Convolution utilities for temporal delays +- Timeline alignment between infections and observations +- Integration with hierarchical noise models +- Support for multiple sensors and subpopulations ### Comparison with count observations -The core convolution $\sum_{d=0}^{D} I_{t-d} \cdot p_d$ is the same as for count observations. The key differences are: +The core convolution structure is shared with count observations, but key aspects differ: | Aspect | Counts | Measurements | |--------|--------|--------------| -| Scaling factor | Ascertainment rate $\alpha \in [0,1]$ | Domain-specific (e.g., $G/V$) | -| Temporal PMF | Delay distribution | Shedding/decay kinetics | -| Output space | Expected counts (linear) | Log-concentrations | -| Noise model | Poisson or Negative Binomial | Normal on log scale | -| Subpop structure | Optional (`CountsBySubpop`) | Inherent (hierarchical effects) | +| Output type | Discrete counts | Continuous values | +| Output space | Linear (expected counts) | Often log-transformed | +| Noise model | Poisson or Negative Binomial | Normal (often on log scale) | +| Scaling | Ascertainment rate $\alpha \in [0,1]$ | Domain-specific | +| Subpop structure | Optional (`CountsBySubpop`) | Inherent (sensor/site effects) | + +## The noise model + +Measurement data typically exhibits **sensor-level variability**: different instruments, labs, or sampling locations have systematic biases and different precision levels. + +`HierarchicalNormalNoise` models this with two per-sensor parameters: + +- **Sensor mode**: Systematic bias (additive shift) +- **Sensor SD**: Measurement precision (noise level) + +``` +observed ~ Normal(predicted + sensor_mode[sensor], sensor_sd[sensor]) +``` + +The noise model samples sensor-level parameters within a plate, so any `RandomVariable` can be used as a prior: + +```{python} +# | label: noise-model-general +# Sensor modes: zero-centered, allowing positive or negative bias +sensor_mode_rv = DistributionalVariable("sensor_mode", dist.Normal(0, 0.5)) + +# Sensor SDs: must be positive, truncated normal is a common choice +sensor_sd_rv = DistributionalVariable( + "sensor_sd", dist.TruncatedNormal(loc=0.3, scale=0.15, low=0.05) +) + +# Create noise model +noise = HierarchicalNormalNoise( + sensor_mode_rv=sensor_mode_rv, + sensor_sd_rv=sensor_sd_rv, +) +``` + +## The indexing system -**Key features of measurement data:** +Measurement observations use three index arrays to map observations to their context: + +| Index array | Purpose | +|-------------|---------| +| `times` | Day index for each observation | +| `subpop_indices` | Which infection trajectory (subpopulation) generated each observation | +| `sensor_indices` | Which sensor made each observation (determines noise parameters) | + +This flexible indexing supports: + +- **Irregular sampling**: Observations don't need to be daily +- **Multiple sensors per subpopulation**: Different labs analyzing the same source +- **Multiple subpopulations per sensor**: One sensor serving multiple areas (less common) + +## Subclassing Measurements + +To create a measurement process for your domain, subclass `Measurements` and implement: + +1. **`_predicted_obs(infections)`**: Transform infections to predicted values +2. **`validate()`**: Check parameter validity +3. **`lookback_days()`**: Return the temporal PMF length + +```python +class MyMeasurement(Measurements): + def __init__(self, temporal_pmf_rv, noise, my_scaling_param): + super().__init__(temporal_pmf_rv=temporal_pmf_rv, noise=noise) + self.my_scaling_param = my_scaling_param + + def _predicted_obs(self, infections): + # Your domain-specific transformation here + pmf = self.temporal_pmf_rv() + # ... convolve, scale, transform ... + return predicted_values + + def validate(self): + pmf = self.temporal_pmf_rv() + self._validate_pmf(pmf, "temporal_pmf_rv") + + def lookback_days(self): + return len(self.temporal_pmf_rv()) +``` + +--- + +# Part 2: Wastewater Example + +We now illustrate the framework with a wastewater viral concentration model, based on the [PyRenew-HEW](https://github.com/cdcgov/pyrenew-hew) family of models. + +## The wastewater signal + +Wastewater treatment plants measure viral RNA concentrations in sewage. The predicted concentration depends on: + +- **Infections**: People shed virus into wastewater +- **Shedding kinetics**: Viral shedding peaks a few days after infection +- **Scaling factors**: Genome copies per infection, wastewater volume + +The predicted log-concentration on day $t$ is: + +$$\log(\lambda_t) = \log\left(\frac{G}{V} \cdot \sum_{d=0}^{D} I_{t-d} \cdot p_d\right)$$ + +where: -- **Multiple sites**: Each jurisdiction has several measurement sites (e.g., wastewater treatment plants). -- **Irregular sampling**: Sites measure on different schedules. -- **Site-level variability**: Lab protocols and sampling methods vary across sites. -- **Temporal lag**: Signal peaks several days after infection. +- $I_{t-d}$ is infections on day $t-d$ +- $p_d$ is the shedding kinetics PMF (fraction shed on day $d$ post-infection) +- $G$ is genome copies shed per infection +- $V$ is wastewater volume per person per day -**Note on terminology:** In real-world inference, infections are *latent* (unobserved) and must be estimated from observed data. In this tutorial, we simulate the observation process by specifying infections directly and showing how they produce measurements through convolution and sampling. +Observations are log-concentrations with normal noise: -## Subclassing Measurements for wastewater +$$y_t \sim \text{Normal}(\log(\lambda_t) + \text{sensor\_mode}, \text{sensor\_sd})$$ -The `Measurements` class is abstract—you must subclass it and implement `_predicted_obs()` for your specific signal type. Here we create a `Wastewater` class for viral concentration measurements: +## Implementing the Wastewater class ```{python} # | label: wastewater-class @@ -151,7 +254,9 @@ class Wastewater(Measurements): return jnp.log(concentration) ``` -## Viral shedding kinetics +## Configuring wastewater-specific parameters + +### Viral shedding kinetics The shedding PMF describes what fraction of total viral shedding occurs on each day after infection: @@ -199,41 +304,40 @@ plt.tight_layout() plt.show() ``` -## Genome copies and wastewater volume +### Genome copies and wastewater volume ```{python} # | label: scaling-params -# Log10 genome copies shed per infection (typical: 8-10) +# Log10 genome copies shed per infection (typical range: 8-10) log10_genome_rv = DeterministicVariable("log10_genome", 9.0) # Wastewater volume per person per day (mL) ml_per_person_per_day = 1000.0 ``` -## Noise model with sensor-level effects - -A measurement depends on both the thing being measured (e.g., wastewater from a treatment plant) and the calibration of the lab/instruments used to obtain that measurement. We call this combination a "sensor"—the WWTP/lab pair that determines the measurement characteristics. Different sensors have systematic biases and variabilities that we model with hierarchical effects. +### Sensor-level noise -The `HierarchicalNormalNoise` model samples sensor-level parameters within a plate context, so any `RandomVariable` can be used. For the sensor-level standard deviations, we use a truncated normal prior—a common choice for SD parameters because it naturally constrains values to be positive. +For wastewater, a "sensor" is a WWTP/lab pair—the combination of treatment plant and laboratory that determines measurement characteristics: ```{python} -# | label: noise-model -# Sensor-level mode: systematic differences between sensors (zero-centered normal) -sensor_mode_rv = DistributionalVariable("ww_sensor_mode", dist.Normal(0, 0.5)) +# | label: ww-noise-model +# Sensor-level mode: systematic differences between WWTP/lab pairs +ww_sensor_mode_rv = DistributionalVariable( + "ww_sensor_mode", dist.Normal(0, 0.5) +) -# Sensor-level SD: measurement variability within each sensor (truncated normal) -sensor_sd_rv = DistributionalVariable( +# Sensor-level SD: measurement variability within each WWTP/lab pair +ww_sensor_sd_rv = DistributionalVariable( "ww_sensor_sd", dist.TruncatedNormal(loc=0.3, scale=0.15, low=0.10) ) -# Create the noise model ww_noise = HierarchicalNormalNoise( - sensor_mode_rv=sensor_mode_rv, - sensor_sd_rv=sensor_sd_rv, + sensor_mode_rv=ww_sensor_mode_rv, + sensor_sd_rv=ww_sensor_sd_rv, ) ``` -## Creating the observation process +## Creating the wastewater observation process ```{python} # | label: create-process @@ -247,28 +351,26 @@ ww_process = Wastewater( print(f"Required lookback: {ww_process.lookback_days()} days") ``` -### Timeline alignment and lookback period +## Demonstrations -The observation process convolves infections with a shedding distribution, maintaining alignment between input and output: day $t$ in the output corresponds to day $t$ in the input. +### Timeline alignment -Wastewater concentrations depend on infections from prior days (the length of our shedding distribution minus one). The method `lookback_days()` returns this length; the first valid observation day is at index `lookback - 1`. Earlier days are marked invalid with NaN. +The observation process maintains alignment: day $t$ in output corresponds to day $t$ in input. The first `lookback_days() - 1` days have incomplete history and are marked invalid. ```{python} # | label: helper-function def first_valid_observation_day(obs_process) -> int: - """Return the first day index with complete infection history for convolution.""" + """Return the first day index with complete infection history.""" return obs_process.lookback_days() - 1 ``` -## Simulating wastewater observations from infections +### Simulating from an infection spike -To demonstrate how the observation process works, we simulate concentrations from a spike of infections on a single day. +To see how infections spread into concentrations via shedding kinetics, we simulate from a single-day spike: ```{python} # | label: simulate-spike n_days = 50 - -# First valid observation day day_one = first_valid_observation_day(ww_process) # Create infections with a spike (shape: n_days x n_subpops) @@ -320,11 +422,9 @@ plt.tight_layout() plt.show() ``` -Because all infections occur on a single day, we can see how they spread into wastewater concentrations over subsequent days according to the shedding kinetics. - -## Observation noise +### Observation noise -The log-normal noise model adds stochastic variation. Sampling multiple times from the same infections shows the range of possible observations: +Sampling multiple times from the same infections shows the range of possible observations: ```{python} # | label: sample-realizations @@ -386,23 +486,9 @@ plt.tight_layout() plt.show() ``` -```{python} -# | label: timeline-stats -print("Timeline Analysis:") -print( - f" Infection spike on day {spike_day}: {infections[infection_spike_day, 0]:.0f} people" -) -print( - f" Mode delay from infection to concentration peak: {mode_shedding_day} days" -) -print( - f" Expected concentration peak: day {spike_day + mode_shedding_day} (= {spike_day} + {mode_shedding_day})" -) -``` - -## Sensor-level variability +### Sensor-level variability -Different sensors measuring the same underlying infections will show systematic differences: +Different sensors (WWTP/lab pairs) measuring the same underlying infections show systematic differences: ```{python} # | label: multi-sensor @@ -454,71 +540,11 @@ plt.tight_layout() plt.show() ``` -Each sensor has a different baseline concentration (sensor mode) and different measurement variability (sensor SD). The hierarchical model learns these sensor-specific effects during inference. - -## Effect of genome shedding parameter - -Higher genome shedding means higher observed concentrations: - -```{python} -# | label: compare-genome -genome_values = [8.5, 9.0, 9.5, 10.0] -infections_decay = 2000.0 * jnp.exp(-jnp.arange(40) / 15.0) -infections_decay = infections_decay.reshape(-1, 1) - -observation_days_cmp = jnp.arange(10, 30, dtype=jnp.int32) -n_obs_cmp = len(observation_days_cmp) - -results = {} -for genome_val in genome_values: - process_temp = Wastewater( - shedding_kinetics_rv=shedding_rv, - log10_genome_per_infection_rv=DeterministicVariable( - "log10_genome", genome_val - ), - ml_per_person_per_day=ml_per_person_per_day, - noise=ww_noise, - ) - - with numpyro.handlers.seed(rng_seed=42): - ww_temp = process_temp.sample( - infections=infections_decay, - subpop_indices=jnp.zeros(n_obs_cmp, dtype=jnp.int32), - sensor_indices=jnp.zeros(n_obs_cmp, dtype=jnp.int32), - times=observation_days_cmp, - obs=None, - n_sensors=1, - ) - results[genome_val] = np.array(ww_temp.observed) -``` - -```{python} -# | label: fig-genome-effect -# | fig-cap: Effect of genome shedding parameter on concentrations -fig, ax = plt.subplots(figsize=(8, 4)) +Each sensor has a different baseline (sensor mode) and measurement variability (sensor SD). During inference, these sensor-specific effects are learned from data. -colors = ["steelblue", "coral", "green", "purple"] -for (genome_val, conc), color in zip(results.items(), colors): - ax.plot( - np.array(observation_days_cmp), - conc, - "-o", - color=color, - label=f"log10 = {genome_val}", - markersize=4, - ) +### Multiple subpopulations -ax.set_xlabel("Day") -ax.set_ylabel("Log Viral Concentration") -ax.set_title("Effect of Genome Shedding Parameter") -ax.legend() -plt.tight_layout() -plt.show() -``` - -## Multiple subpopulations - -In hierarchical models, each sensor serves a distinct subpopulation (catchment area). Different subpopulations can have different infection levels: +In hierarchical models, each sensor may serve a distinct catchment area (subpopulation) with different infection dynamics: ```{python} # | label: multi-subpop @@ -550,7 +576,7 @@ with numpyro.handlers.seed(rng_seed=42): ```{python} # | label: fig-multi-subpop -# | fig-cap: Sensors observing different subpopulations with different infection dynamics +# | fig-cap: Sensors observing different subpopulations fig, ax = plt.subplots(figsize=(8, 4)) for subpop in [0, 1]: @@ -569,3 +595,21 @@ ax.legend() plt.tight_layout() plt.show() ``` + +--- + +## Summary + +The `Measurements` framework provides: + +1. **A consistent interface** for continuous observation processes +2. **Hierarchical noise models** that capture sensor-level variability +3. **Flexible indexing** for irregular, multi-sensor, multi-subpopulation data +4. **Convolution utilities** with proper timeline alignment + +To use it for your domain: + +1. Subclass `Measurements` +2. Implement `_predicted_obs()` with your signal transformation +3. Configure appropriate priors for sensor-level effects +4. Use the indexing system to map observations to their context From 2455242322a06a57021d27a1357fc2530d3c9729 Mon Sep 17 00:00:00 2001 From: Mitzi Morris Date: Wed, 7 Jan 2026 15:02:01 -0500 Subject: [PATCH 29/81] more unit tests --- test/test_observation_counts.py | 17 +++++++++++++++ test/test_observation_measurements.py | 31 +++++++++++++++++++++++++++ 2 files changed, 48 insertions(+) diff --git a/test/test_observation_counts.py b/test/test_observation_counts.py index bcfbd54f..33f4eade 100644 --- a/test/test_observation_counts.py +++ b/test/test_observation_counts.py @@ -463,6 +463,23 @@ def test_infection_resolution_counts(self, simple_delay_pmf): assert process.infection_resolution() == "aggregate" +class TestNoiseRepr: + """Test noise model __repr__ methods.""" + + def test_poisson_noise_repr(self): + """Test PoissonNoise __repr__ method.""" + noise = PoissonNoise() + assert repr(noise) == "PoissonNoise()" + + def test_negative_binomial_noise_repr(self): + """Test NegativeBinomialNoise __repr__ method.""" + conc_rv = DeterministicVariable("conc", 10.0) + noise = NegativeBinomialNoise(conc_rv) + repr_str = repr(noise) + assert "NegativeBinomialNoise" in repr_str + assert "concentration_rv" in repr_str + + class TestNoiseValidation: """Test noise model validation methods.""" diff --git a/test/test_observation_measurements.py b/test/test_observation_measurements.py index 32e053eb..b4ed3488 100644 --- a/test/test_observation_measurements.py +++ b/test/test_observation_measurements.py @@ -94,6 +94,18 @@ def test_infection_resolution_is_subpop(self): class TestHierarchicalNormalNoise: """Test HierarchicalNormalNoise model.""" + def test_repr(self): + """Test HierarchicalNormalNoise __repr__ method.""" + sensor_mode_rv = DistributionalVariable("mode", dist.Normal(0, 0.5)) + sensor_sd_rv = DistributionalVariable( + "sd", dist.TruncatedNormal(0.3, 0.15, low=0.05) + ) + noise = HierarchicalNormalNoise(sensor_mode_rv, sensor_sd_rv) + repr_str = repr(noise) + assert "HierarchicalNormalNoise" in repr_str + assert "sensor_mode_rv" in repr_str + assert "sensor_sd_rv" in repr_str + def test_validate(self): """Test HierarchicalNormalNoise validate method.""" sensor_mode_rv = DistributionalVariable("mode", dist.Normal(0, 0.5)) @@ -154,6 +166,25 @@ def test_sample_with_observations(self): class TestConcreteMeasurements: """Test concrete Measurements implementation.""" + def test_repr(self): + """Test Measurements __repr__ method.""" + shedding_pmf = jnp.array([0.3, 0.4, 0.3]) + sensor_mode_rv = DistributionalVariable("mode", dist.Normal(0, 0.5)) + sensor_sd_rv = DistributionalVariable( + "sd", dist.TruncatedNormal(0.3, 0.15, low=0.05) + ) + noise = HierarchicalNormalNoise(sensor_mode_rv, sensor_sd_rv) + + process = ConcreteMeasurements( + temporal_pmf_rv=DeterministicPMF("shedding", shedding_pmf), + noise=noise, + ) + + repr_str = repr(process) + assert "ConcreteMeasurements" in repr_str + assert "temporal_pmf_rv" in repr_str + assert "noise" in repr_str + def test_sample_shape(self): """Test that sample returns correct shape.""" shedding_pmf = jnp.array([0.3, 0.4, 0.3]) From 101e40168fb57b8e9237ff02cc4e9bba5d52d23a Mon Sep 17 00:00:00 2001 From: Mitzi Morris Date: Thu, 8 Jan 2026 14:54:26 -0500 Subject: [PATCH 30/81] use plotnine for plots in observation_processes tutorial --- .../observation_processes_counts.qmd | 614 ++++++++++++------ mkdocs.yml | 2 +- pyproject.toml | 1 + uv.lock | 73 +++ 4 files changed, 502 insertions(+), 188 deletions(-) diff --git a/docs/tutorials/observation_processes_counts.qmd b/docs/tutorials/observation_processes_counts.qmd index ad50e074..71055d96 100644 --- a/docs/tutorials/observation_processes_counts.qmd +++ b/docs/tutorials/observation_processes_counts.qmd @@ -1,23 +1,37 @@ --- title: "Observation processes for count data" -format: gfm +format: + gfm: + fig-width: 16 + fig-height: 10 engine: jupyter --- This tutorial demonstrates how to use the `Counts` observation process to model count data such as hospital admissions, emergency department visits, or deaths. +
+Show code ```{python} # | label: setup # | output: false import jax.numpy as jnp import numpy as np import numpyro +import plotnine as p9 +import pandas as pd +from pathlib import Path +import sys +import warnings +from plotnine.exceptions import PlotnineWarning +warnings.filterwarnings("ignore", category=PlotnineWarning) + import matplotlib.pyplot as plt from pyrenew.observation import Counts, NegativeBinomialNoise, PoissonNoise from pyrenew.deterministic import DeterministicVariable, DeterministicPMF from pyrenew import datasets ``` +
## Overview @@ -48,12 +62,14 @@ For hospital admissions data, we construct a `Counts` observation process. The delay is the key mechanism: infections from $d$ days ago ($I_{t-d}$) contribute to today's hospital admissions ($\lambda_t$) weighted by the probability ($p_d$) that an infection leads to hospitalization after exactly $d$ days. The convolution sums these contributions across all past days. The process generates hospital admissions by sampling from a negative binomial distribution: + $$Y_t \sim \text{NegativeBinomial}(\mu = \lambda_t, \text{concentration} = \phi)$$ The concentration parameter $\phi$ (sometimes called $k$ or the dispersion parameter) controls overdispersion: as $\phi \to \infty$, the distribution approaches Poisson; smaller values allow greater overdispersion. We use the negative binomial distribution because real-world hospital admission counts exhibit overdispersion—the variance exceeds the mean. The Poisson distribution assumes variance equals the mean, which is too restrictive. The negative binomial adds an overdispersion term: + $$\text{Var}[Y_t] = \mu + \frac{\mu^2}{\phi}$$ In this example, we use fixed parameter values for illustration; in practice, these parameters would be estimated from data using weakly informative priors. @@ -63,47 +79,68 @@ In this example, we use fixed parameter values for illustration; in practice, th The delay distribution specifies the probability that an infected person is hospitalized $d$ days after infection, conditional on the infection leading to a hospitalization. For example, if `hosp_delay_pmf[5] = 0.2`, then 20% of infections that result in hospitalization will appear as hospital admissions 5 days after infection. -We load a delay distribution from PyRenew's datasets: +We load a delay distribution from PyRenew's datasets which peaks around day 8-9 post-infection, compute summary statistics, and plot it. ```{python} -# | label: load-delay +# | label: delay-distribution inf_hosp_int = datasets.load_infection_admission_interval() hosp_delay_pmf = jnp.array(inf_hosp_int["probability_mass"].to_numpy()) - delay_rv = DeterministicPMF("inf_to_hosp_delay", hosp_delay_pmf) # Summary statistics days = np.arange(len(hosp_delay_pmf)) mean_delay = float(np.sum(days * hosp_delay_pmf)) mode_delay = int(np.argmax(hosp_delay_pmf)) -print(f"Mode delay: {mode_delay} days, Mean delay: {mean_delay:.1f} days") +sd = float(np.sqrt(np.sum(days**2 * hosp_delay_pmf) - mean_delay**2)) +print(f"mode delay: {mode_delay}, mean delay: {mean_delay:.1f}, sd: {sd:.1f}") ``` +
+Show code ```{python} -# | label: fig-delay-distribution -# | fig-cap: Infection-to-hospitalization delay distribution -fig, ax = plt.subplots(figsize=(8, 4)) -ax.bar(days, hosp_delay_pmf, color="steelblue", alpha=0.7, edgecolor="black") -ax.axvline( - mode_delay, - color="purple", - linestyle="-", - linewidth=2, - label=f"Mode: {mode_delay}", +# | label: plot-delay-distribution +delay_df = pd.DataFrame( + {"days": days, "probability": np.array(hosp_delay_pmf)} ) -ax.axvline( - mean_delay, - color="red", - linestyle="--", - linewidth=2, - label=f"Mean: {mean_delay:.1f}", + +plot_delay = ( + p9.ggplot(delay_df, p9.aes(x="days", y="probability")) + + p9.geom_col(fill="steelblue", alpha=0.7, color="black") + + p9.geom_vline( + xintercept=mode_delay, color="purple", linetype="solid", size=1 + ) + + p9.geom_vline( + xintercept=mean_delay, color="red", linetype="dashed", size=1 + ) + + p9.labs( + x="Days from infection to hospitalization", + y="Probability", + title="Infection-to-Hospitalization Delay Distribution", + ) + + p9.theme_grey() + + p9.theme(plot_title=p9.element_text(size=14, weight="bold")) + + p9.annotate( + "text", + x=mode_delay + 8, + y=max(delay_df["probability"]) * 0.95, + label=f"Mode: {mode_delay} days", + color="purple", + size=10, + ) + + p9.annotate( + "text", + x=mean_delay + 8, + y=max(delay_df["probability"]) * 0.8, + label=f"Mean: {mean_delay:.1f} days", + color="red", + size=10, + ) ) -ax.set_xlabel("Days from infection to hospitalization") -ax.set_ylabel("Probability") -ax.set_title("Infection-to-Hospitalization Delay Distribution") -ax.legend() -plt.tight_layout() -plt.show() +``` +
+```{python} +# | label: viz-delay-distribution +plot_delay ``` ## Creating a Counts observation process @@ -114,6 +151,12 @@ A `Counts` object takes the following arguments: - **`delay_distribution_rv`**: delay distribution from infection to observation (PMF) - **`noise`**: noise model (`PoissonNoise()` or `NegativeBinomialNoise(concentration_rv)`) +For hospital admissions, the ascertainment rate is specifically called the infection-hospitalization rate (IHR). +In this example, the percentage of infections which lead to hospitalization is treated as a fixed value, +which will allow us to see how different values affect the model. +The concentration parameter for the negative binomial noise model is also fixed. +In practice, both of these parameters would be given a somewhat informative prior and then inferred. + ```{python} # | label: create-counts-process # Infection-hospitalization ratio (1% of infections lead to hospitalization) @@ -128,8 +171,6 @@ hosp_process = Counts( delay_distribution_rv=delay_rv, noise=NegativeBinomialNoise(concentration_rv), ) - -print(f"Required lookback: {hosp_process.lookback_days()} days") ``` ### Timeline alignment and lookback period @@ -140,159 +181,254 @@ Hospital admissions depend on infections from prior days (the length of our dela ```{python} # | label: helper-function +print(f"Required lookback: {hosp_process.lookback_days()} days") + def first_valid_observation_day(obs_process) -> int: """Return the first day index with complete infection history for convolution.""" return obs_process.lookback_days() - 1 ``` -## Simulating hospital admissions from infections +## Simulating observed hospital admissions given a single day's worth of infections + +To demonstrate how a `Counts` observation process works, we examine how infections occurring on a single day result in observed hospital admissions. -To demonstrate how the observation process works, we simulate admissions from a spike of infections on a single day. ```{python} # | label: simulate-spike n_days = 100 -lookback = hosp_process.lookback_days() - -# First valid observation day -day_one = lookback - 1 +day_one = first_valid_observation_day(hosp_process) # Create infections with a spike infection_spike_day = day_one + 10 infections = jnp.zeros(n_days) infections = infections.at[infection_spike_day].set(2000) - -# Sample hospital admissions -with numpyro.handlers.seed(rng_seed=42): - hosp_admissions = hosp_process.sample( - infections=infections, - obs=None, # Sample from prior (no observed data) - ) ``` +We plot the infections starting from day_one (the first valid observation day, after the lookback period). +
+Show code ```{python} -# | label: fig-spike-infections -# | fig-cap: Input infections with a single-day spike +# | label: plot-infections # Plot relative to first valid observation day -spike_day = infection_spike_day - day_one n_plot_days = n_days - day_one +rel_spike_day = infection_spike_day - day_one -fig, ax = plt.subplots(figsize=(8, 4)) -ax.plot( - np.arange(n_plot_days), - np.array(infections[day_one:]), - "-o", - color="darkblue", +infections_df = pd.DataFrame( + { + "day": np.arange(n_plot_days), + "count": np.array(infections[day_one:]), + } ) -ax.axvline(spike_day, color="darkred", linestyle="--", alpha=0.7) -ax.annotate( - f"Infection spike\n(day {spike_day})", - xy=(spike_day, 1800), - xytext=(spike_day + 5, 1800), - fontsize=10, - color="darkred", + +max_infection_count = float(jnp.max(infections[day_one:])) + +plot_infections = ( + p9.ggplot(infections_df, p9.aes(x="day", y="count")) + + p9.geom_line(color="darkblue", size=1) + + p9.geom_point(color="darkblue", size=2) + + p9.geom_vline( + xintercept=rel_spike_day, + color="darkred", + linetype="dashed", + alpha=0.5, + ) + + p9.labs(x="Day", y="Daily Infections", title="Infections (Input)") + + p9.theme_grey() + + p9.theme(plot_title=p9.element_text(size=13, weight="bold")) + + p9.annotate( + "text", + x=rel_spike_day + 2, + y=max_infection_count * 0.9, + label=f"Infection spike (day {rel_spike_day})", + color="darkred", + size=10, + ) ) -ax.set_xlabel("Day") -ax.set_ylabel("Daily Infections") -ax.set_title("Infections (Input)") -plt.tight_layout() -plt.show() ``` +
+```{python} +# | label: viz-infections +plot_infections +``` + +Because all infections occur on a single day, this allows us to see how one day's worth of infections result in hospital admissions spread over subsequent days according to the delay distribution. -Because all infections occur on a single day, we can see how they spread into hospital admissions over subsequent days according to the delay distribution: +## Predicted admissions without observation noise. + +First, we compute the predicted admissions from the convolution alone, without observation noise. This is the mean of the distribution from which samples are drawn. ```{python} -# | label: fig-spike-admissions -# | fig-cap: Hospital admissions from a single-day infection spike -fig, ax = plt.subplots(figsize=(8, 4)) -ax.plot( - np.arange(n_plot_days), - np.array(hosp_admissions.observed[day_one:]), - "-o", - color="purple", +# | label: predicted-no-noise +# Compute predicted admissions (convolution only, no observation noise) +from pyrenew.convolve import compute_delay_ascertained_incidence + +# Scale infections by IHR (ascertainment rate) +infections_scaled = infections * float(ihr_rv.sample()) +predicted_admissions, offset = compute_delay_ascertained_incidence( + p_observed_given_incident=1.0, + latent_incidence=infections_scaled, + delay_incidence_to_observation_pmf=hosp_delay_pmf, + pad=True, ) -ax.axvline( - spike_day, - color="darkred", - linestyle="--", - alpha=0.5, - label="Infection spike", +``` + +
+Show code +```{python} +# | label: plot-predicted-no-noise +# Relative peak day for plotting +peak_day = rel_spike_day + mode_delay + +# Plot predicted admissions (x-axis: day_one = first valid observation day) +predicted_df = pd.DataFrame( + { + "day": np.arange(n_plot_days), + "admissions": np.array(predicted_admissions[day_one:]), + } ) -ax.axvline( - spike_day + mode_delay, - color="purple", - linestyle="--", - alpha=0.5, - label="Peak admissions", + +max_predicted = float(predicted_df["admissions"].max()) +plot_predicted = ( + p9.ggplot(predicted_df, p9.aes(x="day", y="admissions")) + + p9.geom_line(color="purple", size=1) + + p9.geom_point(color="purple", size=1.5) + + p9.geom_vline( + xintercept=rel_spike_day, + color="darkred", + linetype="dashed", + alpha=0.5, + ) + + p9.geom_vline( + xintercept=peak_day, + color="purple", + linetype="dashed", + alpha=0.5, + ) + + p9.labs( + x="Day", + y="Hospital Admissions", + title="Predicted Hospital Admissions (Deterministic)", + ) + + p9.theme_grey() + + p9.annotate( + "text", + x=rel_spike_day, + y=max_predicted * 1.05, + label=f"Infection spike\n(day {rel_spike_day})", + color="darkred", + size=9, + ha="center", + ) + + p9.annotate( + "text", + x=peak_day, + y=max_predicted * 1.05, + label=f"Peak\n(day {peak_day})", + color="purple", + size=9, + ha="center", + ) ) -ax.set_xlabel("Day") -ax.set_ylabel("Hospital Admissions") -ax.set_title("Hospital Admissions (Output)") -ax.legend() -plt.tight_layout() -plt.show() +``` +
+```{python} +# | label: viz-predicted +plot_predicted ``` -The admissions peak occurs `{python} mode_delay` days after the infection spike, matching the mode of the delay distribution. +The predicted admissions mirror the delay distribution, shifted by the infection spike day and scaled by the IHR. -## Observation noise + +## Observation Noise (Negative Binomial) The negative binomial distribution adds stochastic variation. Sampling multiple times from the same infections shows the range of possible observations: ```{python} # | label: sample-realizations +# Sample 50 realizations of hospital admissions from the same infection spike n_samples = 50 -samples = [] +samples_list = [] for seed in range(n_samples): with numpyro.handlers.seed(rng_seed=seed): - result = hosp_process.sample(infections=infections, obs=None) - samples.append(np.array(result.observed[day_one:])) + hosp_sample = hosp_process.sample(infections=infections, obs=None) + + for i, val in enumerate(hosp_sample.observed[day_one:]): + samples_list.append( + { + "day": i, + "admissions": float(val), + "sample": seed, + "type": "sampled", + } + ) -samples = np.array(samples) -sample_mean = samples.mean(axis=0) +# Add predicted values +for i, val in enumerate(predicted_admissions[day_one:]): + samples_list.append( + { + "day": i, + "admissions": float(val), + "sample": -1, + "type": "predicted", + } + ) ``` +
+Show code ```{python} -# | label: fig-sampled-admissions -# | fig-cap: Multiple realizations showing observation noise -fig, ax = plt.subplots(figsize=(8, 4)) - -# Plot all samples in light orange -for i in range(1, n_samples): - ax.plot( - np.arange(n_plot_days), - samples[i], +# | label: plot-realizations +samples_df = pd.DataFrame(samples_list) +sampled_df = samples_df[samples_df["type"] == "sampled"] +predicted_noise_df = samples_df[samples_df["type"] == "predicted"] + +# Separate one sample to highlight +highlight_sample = 0 +other_samples_df = sampled_df[sampled_df["sample"] != highlight_sample] +highlight_df = sampled_df[sampled_df["sample"] == highlight_sample] + +plot_50_samples = ( + p9.ggplot() + + p9.geom_line( + p9.aes(x="day", y="admissions", group="sample"), + data=other_samples_df, color="orange", alpha=0.15, - linewidth=0.5, + size=0.5, ) - -# Highlight one sample -ax.plot( - np.arange(n_plot_days), - samples[0], - color="steelblue", - linewidth=1, - label="One realization", -) - -# Sample mean -ax.plot( - np.arange(n_plot_days), - sample_mean, - color="darkred", - linewidth=1.2, - label="Sample mean", + + p9.geom_line( + p9.aes(x="day", y="admissions"), + data=highlight_df, + color="steelblue", + size=1, + ) + + p9.geom_line( + p9.aes(x="day", y="admissions"), + data=predicted_noise_df, + color="darkred", + size=1.2, + ) + + p9.geom_vline( + xintercept=rel_spike_day, + color="darkblue", + linetype="dashed", + alpha=0.5, + ) + + p9.labs( + x="Day", + y="Hospital Admissions", + title=f"Observation Noise: {n_samples} Samples from Same Infections", + subtitle="Blue: one realization | Orange: other samples | Dark red: predicted", + ) + + p9.theme_grey() ) +``` +
-ax.axvline(spike_day, color="darkblue", linestyle="--", alpha=0.5) -ax.set_xlabel("Day") -ax.set_ylabel("Hospital Admissions") -ax.set_title(f"Observation Noise: {n_samples} Samples from Same Infections") -ax.legend() -plt.tight_layout() -plt.show() +```{python} +# | label: viz-realizations +plot_50_samples ``` ```{python} @@ -300,60 +436,72 @@ plt.show() # Print timeline statistics print("Timeline Analysis:") print( - f" Infection spike on day {spike_day}: {infections[infection_spike_day]:.0f} people" + f" Infection spike on day {rel_spike_day}: {infections[infection_spike_day]:.0f} people" ) print(f" Mode delay from infection to hospitalization: {mode_delay} days") print( - f" Expected hospitalization peak: day {spike_day + mode_delay} (= {spike_day} + {mode_delay})" + f" Predicted hospitalization peak: day {rel_spike_day + mode_delay} (= {rel_spike_day} + {mode_delay})" ) ``` ## Effect of the ascertainment rate -The ascertainment rate directly scales expected hospital admissions. We compare IHR values of 0.5% and 2.5%: +The ascertainment rate (here, the infection-hospitalization rate or IHR) directly scales the number of predicted hospital admissions. +We compare two contrasting IHR values: **0.5%** and **2.5%**. ```{python} # | label: compare-ihr +# Two contrasting IHR values ihr_values = [0.005, 0.025] -infections_decay = 3000 * jnp.exp(-jnp.arange(n_days) / 20.0) +peak_value = 3000 # Peak infections +infections_decay = peak_value * jnp.exp(-jnp.arange(n_days) / 20.0) -results = {} +# Compute predicted hospital admissions (no noise) for each IHR +results_list = [] for ihr_val in ihr_values: - process = Counts( - ascertainment_rate_rv=DeterministicVariable("ihr", ihr_val), - delay_distribution_rv=delay_rv, - noise=NegativeBinomialNoise(concentration_rv), + infections_scaled = infections_decay * ihr_val + predicted_hosp, _ = compute_delay_ascertained_incidence( + p_observed_given_incident=1.0, + latent_incidence=infections_scaled, + delay_incidence_to_observation_pmf=hosp_delay_pmf, + pad=True, ) - # Average over multiple samples to show the effect on the mean - samples = [] - for seed in range(20): - with numpyro.handlers.seed(rng_seed=seed): - result = process.sample(infections=infections_decay, obs=None) - samples.append(np.array(result.observed[day_one:])) - results[ihr_val] = np.mean(samples, axis=0) + + for i, admit in enumerate(predicted_hosp[day_one:]): + results_list.append( + { + "day": i, + "admissions": float(admit), + "IHR": f"IHR = {ihr_val:.1%}", + } + ) ``` + +
+Show code ```{python} -# | label: fig-ihr-comparison -# | fig-cap: Effect of different infection-hospitalization rates -fig, ax = plt.subplots(figsize=(8, 4)) -colors = ["steelblue", "darkred"] - -for (ihr_val, mean_sample), color in zip(results.items(), colors): - ax.plot( - np.arange(n_plot_days), - mean_sample, - color=color, - linewidth=1.5, - label=f"IHR = {ihr_val:.1%}", +# | label: plot-ihr-comparisons +results_df = pd.DataFrame(results_list) + +plot_ihr = ( + p9.ggplot(results_df, p9.aes(x="day", y="admissions", color="IHR")) + + p9.geom_line(size=1) + + p9.scale_color_manual(values=["steelblue", "darkred"]) + + p9.labs( + x="Day", + y="Predicted Hospital Admissions", + title="Effect of IHR on Predicted Hospital Admissions", + color="Infection-Hospitalization\nrate", ) + + p9.theme_grey() +) +``` +
-ax.set_xlabel("Day") -ax.set_ylabel("Hospital Admissions (mean of samples)") -ax.set_title("Effect of IHR on Hospital Admissions") -ax.legend() -plt.tight_layout() -plt.show() +```{python} +# | label: viz-ihr-comparisons +plot_ihr ``` ## Negative binomial concentration parameter @@ -363,40 +511,81 @@ The concentration parameter $\phi$ controls overdispersion: - Higher $\phi$ → less overdispersion (approaches Poisson) - Lower $\phi$ → more overdispersion (noisier data) +We compare three concentration values spanning two orders of magnitude: + +- **φ = 1**: high overdispersion (noisy) +- **φ = 10**: moderate overdispersion +- **φ = 100**: nearly Poisson (minimal noise) + ```{python} -# | label: fig-concentration-comparison -# | fig-cap: Effect of concentration parameter on variability -infections_constant = 2000 * jnp.ones(n_days) +# | label: concentration-comparisons +# Use constant infections +peak_value = 2000 +infections_constant = peak_value * jnp.ones(n_days) + +# Concentration values spanning two orders of magnitude concentration_values = [1.0, 10.0, 100.0] n_replicates = 10 -fig, axes = plt.subplots(1, 3, figsize=(12, 4), sharey=True) - -for ax, conc_val in zip(axes, concentration_values): - process = Counts( +# Collect results +conc_results = [] +for conc_val in concentration_values: + conc_rv_temp = DeterministicVariable("conc", conc_val) + process_temp = Counts( ascertainment_rate_rv=ihr_rv, delay_distribution_rv=delay_rv, - noise=NegativeBinomialNoise(DeterministicVariable("conc", conc_val)), + noise=NegativeBinomialNoise(conc_rv_temp), ) for seed in range(n_replicates): with numpyro.handlers.seed(rng_seed=seed): - result = process.sample(infections=infections_constant, obs=None) - ax.plot( - np.arange(n_plot_days), - np.array(result.observed[day_one:]), - color="steelblue", - alpha=0.5, - linewidth=0.8, - ) + hosp_temp = process_temp.sample( + infections=infections_constant, + obs=None, + ) + + # Use relative days + for i, admit in enumerate(hosp_temp.observed[day_one:]): + conc_results.append( + { + "day": i, + "admissions": float(admit), + "concentration": f"φ = {int(conc_val)}", + "replicate": seed, + } + ) +``` - ax.set_xlabel("Day") - ax.set_title(f"φ = {int(conc_val)}") +
+Show code +```{python} +# | label: plot-concentration-comparisons +conc_df = pd.DataFrame(conc_results) + +# Convert to ordered categorical +conc_df["concentration"] = pd.Categorical( + conc_df["concentration"], + categories=["φ = 1", "φ = 10", "φ = 100"], + ordered=True, +) -axes[0].set_ylabel("Hospital Admissions") -fig.suptitle("Effect of Concentration Parameter on Variability") -plt.tight_layout() -plt.show() +plot_concentration = ( + p9.ggplot(conc_df, p9.aes(x="day", y="admissions", group="replicate")) + + p9.geom_line(alpha=0.5, size=0.8, color="steelblue") + + p9.facet_wrap("~ concentration", ncol=3) + + p9.labs( + x="Day", + y="Hospital Admissions", + title="Effect of Concentration Parameter on Variability", + ) + + p9.theme_grey() +) +``` +
+ +```{python} +# | label: viz-concentration-comparisons +plot_concentration ``` ## Swapping noise models @@ -421,3 +610,54 @@ print( f"Sampled {len(poisson_result.observed)} days of hospital admissions with Poisson noise" ) ``` + +We can visualize the Poisson noise model using the same constant infection scenario as the concentration comparison above. Since Poisson assumes variance equals the mean, it produces less variability than the negative binomial with low concentration values. + +```{python} +# | label: poisson-realizations +# Sample multiple realizations with Poisson noise +n_replicates_poisson = 10 + +poisson_results = [] +for seed in range(n_replicates_poisson): + with numpyro.handlers.seed(rng_seed=seed): + poisson_temp = hosp_process_poisson.sample( + infections=infections_constant, + obs=None, + ) + + for i, admit in enumerate(poisson_temp.observed[day_one:]): + poisson_results.append( + { + "day": i, + "admissions": float(admit), + "replicate": seed, + } + ) +``` + +
+Show code +```{python} +# | label: plot-poisson-realizations +poisson_df = pd.DataFrame(poisson_results) + +plot_poisson = ( + p9.ggplot(poisson_df, p9.aes(x="day", y="admissions", group="replicate")) + + p9.geom_line(alpha=0.5, size=0.8, color="steelblue") + + p9.labs( + x="Day", + y="Hospital Admissions", + title="Poisson Noise Model (Variance = Mean)", + ) + + p9.theme_grey() + + p9.ylim(0, 105) +) +``` +
+ +To see the reduction in noise, it is ncessary to keep the y-axis on the same scale as in the previous plot. +```{python} +# | label: viz-poisson-realizations +plot_poisson +``` diff --git a/mkdocs.yml b/mkdocs.yml index a4da9382..1c2745eb 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -64,9 +64,9 @@ markdown_extensions: - callouts extra_javascript: - - javascripts/katex.js - https://unpkg.com/katex@0/dist/katex.min.js - https://unpkg.com/katex@0/dist/contrib/auto-render.min.js + - javascripts/katex.js extra_css: - https://unpkg.com/katex@0/dist/katex.min.css diff --git a/pyproject.toml b/pyproject.toml index 01d7800a..514e6d3d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -28,6 +28,7 @@ dev = [ "nbconvert>=7.16.6", "pytest>=8.4.2", "pytest-cov>=6.3.0", + "plotnine>=0.14.0", "pytest-mpl>=0.17.0", "scipy>=1.16.1", ] diff --git a/uv.lock b/uv.lock index f2477f56..6a0bfd17 100644 --- a/uv.lock +++ b/uv.lock @@ -949,6 +949,21 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/7a/f0/8282d9641415e9e33df173516226b404d367a0fc55e1a60424a152913abc/mistune-3.1.4-py3-none-any.whl", hash = "sha256:93691da911e5d9d2e23bc54472892aff676df27a75274962ff9edc210364266d", size = 53481, upload-time = "2025-08-29T07:20:42.218Z" }, ] +[[package]] +name = "mizani" +version = "0.14.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "numpy" }, + { name = "pandas" }, + { name = "scipy" }, + { name = "tzdata", marker = "sys_platform == 'emscripten' or sys_platform == 'win32'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/6d/19/98f2bd61e5441b687e0a5d3b36041981cc032451f2d11472021b040d27fd/mizani-0.14.3.tar.gz", hash = "sha256:c2fb886b3c9e8109be5b8fd21e1130fba1f0a20230a987146240221209fc0ddd", size = 772470, upload-time = "2025-10-30T20:16:53.268Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/5c/d2/4ffcaa27c8a4b4f9ad456da4821c76dfbdfada23e8210cd4d80e1eb3236a/mizani-0.14.3-py3-none-any.whl", hash = "sha256:6d2ca9b1b8366ff85668f0cc1b6095f1e702e26e66f132c4f02a949efa32a688", size = 133433, upload-time = "2025-10-30T20:16:51.218Z" }, +] + [[package]] name = "mkdocs" version = "1.6.1" @@ -1348,6 +1363,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/cc/20/ff623b09d963f88bfde16306a54e12ee5ea43e9b597108672ff3a408aad6/pathspec-0.12.1-py3-none-any.whl", hash = "sha256:a0d503e138a4c123b27490a4f7beda6a01c6f288df0e4a8b79c7eb0dc7b4cc08", size = 31191, upload-time = "2023-12-10T22:30:43.14Z" }, ] +[[package]] +name = "patsy" +version = "1.0.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "numpy" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/be/44/ed13eccdd0519eff265f44b670d46fbb0ec813e2274932dc1c0e48520f7d/patsy-1.0.2.tar.gz", hash = "sha256:cdc995455f6233e90e22de72c37fcadb344e7586fb83f06696f54d92f8ce74c0", size = 399942, upload-time = "2025-10-20T16:17:37.535Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f1/70/ba4b949bdc0490ab78d545459acd7702b211dfccf7eb89bbc1060f52818d/patsy-1.0.2-py2.py3-none-any.whl", hash = "sha256:37bfddbc58fcf0362febb5f54f10743f8b21dd2aa73dec7e7ef59d1b02ae668a", size = 233301, upload-time = "2025-10-20T16:17:36.563Z" }, +] + [[package]] name = "pexpect" version = "4.9.0" @@ -1427,6 +1454,23 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/73/cb/ac7874b3e5d58441674fb70742e6c374b28b0c7cb988d37d991cde47166c/platformdirs-4.5.0-py3-none-any.whl", hash = "sha256:e578a81bb873cbb89a41fcc904c7ef523cc18284b7e3b3ccf06aca1403b7ebd3", size = 18651, upload-time = "2025-10-08T17:44:47.223Z" }, ] +[[package]] +name = "plotnine" +version = "0.15.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "matplotlib" }, + { name = "mizani" }, + { name = "numpy" }, + { name = "pandas" }, + { name = "scipy" }, + { name = "statsmodels" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/11/14/3adedabe6b8710caee34e4ac9f4edc48218a381594ee1980c323b8866577/plotnine-0.15.2.tar.gz", hash = "sha256:ec2e4cdf2d022eb0dab63ef4aa0017ce0d84c60bd99d55093e72637fddf757e6", size = 6787690, upload-time = "2025-12-12T10:41:37.249Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/58/27/4e6ffe2f095fbfd6285343aa6114903a4cf011564b4f1f2bb706341472df/plotnine-0.15.2-py3-none-any.whl", hash = "sha256:7dc508bc51625b9b9f945e274d8ee4463cf30b280749190a5b707e6828003fa6", size = 1332822, upload-time = "2025-12-12T10:41:34.203Z" }, +] + [[package]] name = "pluggy" version = "1.6.0" @@ -1582,6 +1626,7 @@ dev = [ { name = "mkdocstrings" }, { name = "mkdocstrings-python" }, { name = "nbconvert" }, + { name = "plotnine" }, { name = "pytest" }, { name = "pytest-cov" }, { name = "pytest-mpl" }, @@ -1609,6 +1654,7 @@ dev = [ { name = "mkdocstrings", specifier = ">=0.30.0" }, { name = "mkdocstrings-python", specifier = ">=1.18.2" }, { name = "nbconvert", specifier = ">=7.16.6" }, + { name = "plotnine", specifier = ">=0.14.0" }, { name = "pytest", specifier = ">=8.4.2" }, { name = "pytest-cov", specifier = ">=6.3.0" }, { name = "pytest-mpl", specifier = ">=0.17.0" }, @@ -1971,6 +2017,33 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/f1/7b/ce1eafaf1a76852e2ec9b22edecf1daa58175c090266e9f6c64afcd81d91/stack_data-0.6.3-py3-none-any.whl", hash = "sha256:d5558e0c25a4cb0853cddad3d77da9891a08cb85dd9f9f91b9f8cd66e511e695", size = 24521, upload-time = "2023-09-30T13:58:03.53Z" }, ] +[[package]] +name = "statsmodels" +version = "0.14.6" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "numpy" }, + { name = "packaging" }, + { name = "pandas" }, + { name = "patsy" }, + { name = "scipy" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/0d/81/e8d74b34f85285f7335d30c5e3c2d7c0346997af9f3debf9a0a9a63de184/statsmodels-0.14.6.tar.gz", hash = "sha256:4d17873d3e607d398b85126cd4ed7aad89e4e9d89fc744cdab1af3189a996c2a", size = 20689085, upload-time = "2025-12-05T23:08:39.522Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/81/59/a5aad5b0cc266f5be013db8cde563ac5d2a025e7efc0c328d83b50c72992/statsmodels-0.14.6-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:47ee7af083623d2091954fa71c7549b8443168f41b7c5dce66510274c50fd73e", size = 10072009, upload-time = "2025-12-05T23:11:14.021Z" }, + { url = "https://files.pythonhosted.org/packages/53/dd/d8cfa7922fc6dc3c56fa6c59b348ea7de829a94cd73208c6f8202dd33f17/statsmodels-0.14.6-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:aa60d82e29fcd0a736e86feb63a11d2380322d77a9369a54be8b0965a3985f71", size = 9980018, upload-time = "2025-12-05T23:11:30.907Z" }, + { url = "https://files.pythonhosted.org/packages/ee/77/0ec96803eba444efd75dba32f2ef88765ae3e8f567d276805391ec2c98c6/statsmodels-0.14.6-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:89ee7d595f5939cc20bf946faedcb5137d975f03ae080f300ebb4398f16a5bd4", size = 10060269, upload-time = "2025-12-05T23:11:46.338Z" }, + { url = "https://files.pythonhosted.org/packages/10/b9/fd41f1f6af13a1a1212a06bb377b17762feaa6d656947bf666f76300fc05/statsmodels-0.14.6-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:730f3297b26749b216a06e4327fe0be59b8d05f7d594fb6caff4287b69654589", size = 10324155, upload-time = "2025-12-05T23:12:01.805Z" }, + { url = "https://files.pythonhosted.org/packages/ee/0f/a6900e220abd2c69cd0a07e3ad26c71984be6061415a60e0f17b152ecf08/statsmodels-0.14.6-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:f1c08befa85e93acc992b72a390ddb7bd876190f1360e61d10cf43833463bc9c", size = 10349765, upload-time = "2025-12-05T23:12:18.018Z" }, + { url = "https://files.pythonhosted.org/packages/98/08/b79f0c614f38e566eebbdcff90c0bcacf3c6ba7a5bbb12183c09c29ca400/statsmodels-0.14.6-cp313-cp313-win_amd64.whl", hash = "sha256:8021271a79f35b842c02a1794465a651a9d06ec2080f76ebc3b7adce77d08233", size = 9540043, upload-time = "2025-12-05T23:12:33.887Z" }, + { url = "https://files.pythonhosted.org/packages/71/de/09540e870318e0c7b58316561d417be45eff731263b4234fdd2eee3511a8/statsmodels-0.14.6-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:00781869991f8f02ad3610da6627fd26ebe262210287beb59761982a8fa88cae", size = 10069403, upload-time = "2025-12-05T23:12:48.424Z" }, + { url = "https://files.pythonhosted.org/packages/ab/f0/63c1bfda75dc53cee858006e1f46bd6d6f883853bea1b97949d0087766ca/statsmodels-0.14.6-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:73f305fbf31607b35ce919fae636ab8b80d175328ed38fdc6f354e813b86ee37", size = 9989253, upload-time = "2025-12-05T23:13:05.274Z" }, + { url = "https://files.pythonhosted.org/packages/c1/98/b0dfb4f542b2033a3341aa5f1bdd97024230a4ad3670c5b0839d54e3dcab/statsmodels-0.14.6-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e443e7077a6e2d3faeea72f5a92c9f12c63722686eb80bb40a0f04e4a7e267ad", size = 10090802, upload-time = "2025-12-05T23:13:20.653Z" }, + { url = "https://files.pythonhosted.org/packages/34/0e/2408735aca9e764643196212f9069912100151414dd617d39ffc72d77eee/statsmodels-0.14.6-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3414e40c073d725007a6603a18247ab7af3467e1af4a5e5a24e4c27bc26673b4", size = 10337587, upload-time = "2025-12-05T23:13:37.597Z" }, + { url = "https://files.pythonhosted.org/packages/0f/36/4d44f7035ab3c0b2b6a4c4ebb98dedf36246ccbc1b3e2f51ebcd7ac83abb/statsmodels-0.14.6-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:a518d3f9889ef920116f9fa56d0338069e110f823926356946dae83bc9e33e19", size = 10363350, upload-time = "2025-12-05T23:13:53.08Z" }, + { url = "https://files.pythonhosted.org/packages/26/33/f1652d0c59fa51de18492ee2345b65372550501ad061daa38f950be390b6/statsmodels-0.14.6-cp314-cp314-win_amd64.whl", hash = "sha256:151b73e29f01fe619dbce7f66d61a356e9d1fe5e906529b78807df9189c37721", size = 9588010, upload-time = "2025-12-05T23:14:07.28Z" }, +] + [[package]] name = "tinycss2" version = "1.4.0" From bd5546f608a8a308db128ff54e95e4171724ba55 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 8 Jan 2026 19:54:40 +0000 Subject: [PATCH 31/81] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- docs/tutorials/observation_processes_counts.qmd | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/docs/tutorials/observation_processes_counts.qmd b/docs/tutorials/observation_processes_counts.qmd index 71055d96..59153e41 100644 --- a/docs/tutorials/observation_processes_counts.qmd +++ b/docs/tutorials/observation_processes_counts.qmd @@ -23,6 +23,7 @@ from pathlib import Path import sys import warnings from plotnine.exceptions import PlotnineWarning + warnings.filterwarnings("ignore", category=PlotnineWarning) import matplotlib.pyplot as plt @@ -92,7 +93,9 @@ days = np.arange(len(hosp_delay_pmf)) mean_delay = float(np.sum(days * hosp_delay_pmf)) mode_delay = int(np.argmax(hosp_delay_pmf)) sd = float(np.sqrt(np.sum(days**2 * hosp_delay_pmf) - mean_delay**2)) -print(f"mode delay: {mode_delay}, mean delay: {mean_delay:.1f}, sd: {sd:.1f}") +print( + f"mode delay: {mode_delay}, mean delay: {mean_delay:.1f}, sd: {sd:.1f}" +) ```
@@ -183,6 +186,7 @@ Hospital admissions depend on infections from prior days (the length of our dela # | label: helper-function print(f"Required lookback: {hosp_process.lookback_days()} days") + def first_valid_observation_day(obs_process) -> int: """Return the first day index with complete infection history for convolution.""" return obs_process.lookback_days() - 1 From f3a583dc5a58258362ddaee86b5e8a019fdeabf7 Mon Sep 17 00:00:00 2001 From: Mitzi Morris Date: Thu, 8 Jan 2026 21:16:33 -0500 Subject: [PATCH 32/81] update observation_processes tutorials to use plotnine --- .../observation_processes_counts.qmd | 68 +-- .../observation_processes_measurements.qmd | 395 +++++++++++------- 2 files changed, 266 insertions(+), 197 deletions(-) diff --git a/docs/tutorials/observation_processes_counts.qmd b/docs/tutorials/observation_processes_counts.qmd index 71055d96..6715279a 100644 --- a/docs/tutorials/observation_processes_counts.qmd +++ b/docs/tutorials/observation_processes_counts.qmd @@ -4,13 +4,17 @@ format: gfm: fig-width: 16 fig-height: 10 + html: + toc: true + embed-resources: true + self-contained-math: true + code-fold: true + code-tools: true engine: jupyter --- This tutorial demonstrates how to use the `Counts` observation process to model count data such as hospital admissions, emergency department visits, or deaths. -
-Show code ```{python} # | label: setup # | output: false @@ -23,6 +27,7 @@ from pathlib import Path import sys import warnings from plotnine.exceptions import PlotnineWarning + warnings.filterwarnings("ignore", category=PlotnineWarning) import matplotlib.pyplot as plt @@ -31,7 +36,6 @@ from pyrenew.observation import Counts, NegativeBinomialNoise, PoissonNoise from pyrenew.deterministic import DeterministicVariable, DeterministicPMF from pyrenew import datasets ``` -
## Overview @@ -92,11 +96,11 @@ days = np.arange(len(hosp_delay_pmf)) mean_delay = float(np.sum(days * hosp_delay_pmf)) mode_delay = int(np.argmax(hosp_delay_pmf)) sd = float(np.sqrt(np.sum(days**2 * hosp_delay_pmf) - mean_delay**2)) -print(f"mode delay: {mode_delay}, mean delay: {mean_delay:.1f}, sd: {sd:.1f}") +print( + f"mode delay: {mode_delay}, mean delay: {mean_delay:.1f}, sd: {sd:.1f}" +) ``` -
-Show code ```{python} # | label: plot-delay-distribution delay_df = pd.DataFrame( @@ -136,10 +140,6 @@ plot_delay = ( size=10, ) ) -``` -
-```{python} -# | label: viz-delay-distribution plot_delay ``` @@ -183,6 +183,7 @@ Hospital admissions depend on infections from prior days (the length of our dela # | label: helper-function print(f"Required lookback: {hosp_process.lookback_days()} days") + def first_valid_observation_day(obs_process) -> int: """Return the first day index with complete infection history for convolution.""" return obs_process.lookback_days() - 1 @@ -205,8 +206,6 @@ infections = infections.at[infection_spike_day].set(2000) ``` We plot the infections starting from day_one (the first valid observation day, after the lookback period). -
-Show code ```{python} # | label: plot-infections # Plot relative to first valid observation day @@ -244,10 +243,6 @@ plot_infections = ( size=10, ) ) -``` -
-```{python} -# | label: viz-infections plot_infections ``` @@ -272,8 +267,6 @@ predicted_admissions, offset = compute_delay_ascertained_incidence( ) ``` -
-Show code ```{python} # | label: plot-predicted-no-noise # Relative peak day for plotting @@ -329,10 +322,6 @@ plot_predicted = ( ha="center", ) ) -``` -
-```{python} -# | label: viz-predicted plot_predicted ``` @@ -375,8 +364,6 @@ for i, val in enumerate(predicted_admissions[day_one:]): ) ``` -
-Show code ```{python} # | label: plot-realizations samples_df = pd.DataFrame(samples_list) @@ -423,11 +410,6 @@ plot_50_samples = ( ) + p9.theme_grey() ) -``` -
- -```{python} -# | label: viz-realizations plot_50_samples ``` @@ -478,8 +460,6 @@ for ihr_val in ihr_values: ``` -
-Show code ```{python} # | label: plot-ihr-comparisons results_df = pd.DataFrame(results_list) @@ -496,11 +476,6 @@ plot_ihr = ( ) + p9.theme_grey() ) -``` -
- -```{python} -# | label: viz-ihr-comparisons plot_ihr ``` @@ -556,8 +531,6 @@ for conc_val in concentration_values: ) ``` -
-Show code ```{python} # | label: plot-concentration-comparisons conc_df = pd.DataFrame(conc_results) @@ -580,11 +553,6 @@ plot_concentration = ( ) + p9.theme_grey() ) -``` -
- -```{python} -# | label: viz-concentration-comparisons plot_concentration ``` @@ -613,6 +581,8 @@ print( We can visualize the Poisson noise model using the same constant infection scenario as the concentration comparison above. Since Poisson assumes variance equals the mean, it produces less variability than the negative binomial with low concentration values. +To see the reduction in noise, it is necessary to keep the y-axis on the same scale as in the previous plot. + ```{python} # | label: poisson-realizations # Sample multiple realizations with Poisson noise @@ -634,12 +604,6 @@ for seed in range(n_replicates_poisson): "replicate": seed, } ) -``` - -
-Show code -```{python} -# | label: plot-poisson-realizations poisson_df = pd.DataFrame(poisson_results) plot_poisson = ( @@ -653,11 +617,5 @@ plot_poisson = ( + p9.theme_grey() + p9.ylim(0, 105) ) -``` -
- -To see the reduction in noise, it is ncessary to keep the y-axis on the same scale as in the previous plot. -```{python} -# | label: viz-poisson-realizations plot_poisson ``` diff --git a/docs/tutorials/observation_processes_measurements.qmd b/docs/tutorials/observation_processes_measurements.qmd index ab5af73e..66c65aa4 100644 --- a/docs/tutorials/observation_processes_measurements.qmd +++ b/docs/tutorials/observation_processes_measurements.qmd @@ -1,6 +1,15 @@ --- title: "Observation processes for continuous measurements" -format: gfm +format: + gfm: + fig-width: 16 + fig-height: 10 + html: + toc: true + embed-resources: true + self-contained-math: true + code-fold: true + code-tools: true engine: jupyter --- @@ -14,6 +23,8 @@ import jax.numpy as jnp import numpy as np import numpyro import matplotlib.pyplot as plt +import pandas as pd +import plotnine as p9 import numpyro.distributions as dist @@ -22,9 +33,7 @@ from pyrenew.randomvariable import DistributionalVariable from pyrenew.deterministic import DeterministicVariable, DeterministicPMF ``` -# Part 1: The Measurements Framework - -## Overview +# The Measurements Framework The `Measurements` class models continuous signals derived from infections. Unlike count observations (hospital admissions, deaths), measurements are continuous values that may span orders of magnitude or even be negative (e.g., log-transformed data). @@ -35,7 +44,7 @@ The `Measurements` class models continuous signals derived from infections. Unli - Serological assay results - Environmental sensor readings -### The general pattern +## The general pattern All measurement observation processes follow the same pattern: @@ -53,7 +62,7 @@ The `Measurements` base class provides: - Integration with hierarchical noise models - Support for multiple sensors and subpopulations -### Comparison with count observations +## Comparison with count observations The core convolution structure is shared with count observations, but key aspects differ: @@ -141,15 +150,16 @@ class MyMeasurement(Measurements): return len(self.temporal_pmf_rv()) ``` ---- -# Part 2: Wastewater Example +# Wastewater Example -We now illustrate the framework with a wastewater viral concentration model, based on the [PyRenew-HEW](https://github.com/cdcgov/pyrenew-hew) family of models. +To illustrate the framework, we specify a wastewater viral concentration observation process, +based on the [PyRenew-HEW](https://github.com/cdcgov/pyrenew-hew) family of models. ## The wastewater signal -Wastewater treatment plants measure viral RNA concentrations in sewage. The predicted concentration depends on: +Wastewater treatment plants measure viral RNA concentrations in sewage. +The predicted concentration depends on: - **Infections**: People shed virus into wastewater - **Shedding kinetics**: Viral shedding peaks a few days after infection @@ -278,30 +288,45 @@ print(f"Mode: {mode_shedding_day} days, Mean: {mean_shedding_day:.1f} days") ``` ```{python} -# | label: fig-shedding -# | fig-cap: Viral shedding kinetics distribution -fig, ax = plt.subplots(figsize=(8, 4)) -ax.bar(days, shedding_pmf, color="steelblue", alpha=0.7, edgecolor="black") -ax.axvline( - mode_shedding_day, - color="purple", - linestyle="-", - linewidth=2, - label=f"Mode: {mode_shedding_day}", +# | label: plot-shedding +# Visualize the shedding distribution +shedding_df = pd.DataFrame( + {"days": days, "probability": np.array(shedding_pmf)} ) -ax.axvline( - mean_shedding_day, - color="red", - linestyle="--", - linewidth=2, - label=f"Mean: {mean_shedding_day:.1f}", + +( + p9.ggplot(shedding_df, p9.aes(x="days", y="probability")) + + p9.geom_col(fill="steelblue", alpha=0.7, color="black") + + p9.geom_vline( + xintercept=mode_shedding_day, color="purple", linetype="solid", size=1 + ) + + p9.geom_vline( + xintercept=mean_shedding_day, color="red", linetype="dashed", size=1 + ) + + p9.labs( + x="Days after infection", + y="Fraction of total shedding", + title="Viral Shedding Kinetics", + ) + + p9.theme_grey() + + p9.theme(plot_title=p9.element_text(size=14, weight="bold")) + + p9.annotate( + "text", + x=mode_shedding_day + 2, + y=max(shedding_df["probability"]) * 0.95, + label=f"Mode: {mode_shedding_day} days", + color="purple", + size=10, + ) + + p9.annotate( + "text", + x=mean_shedding_day + 2, + y=max(shedding_df["probability"]) * 0.8, + label=f"Mean: {mean_shedding_day:.1f} days", + color="red", + size=10, + ) ) -ax.set_xlabel("Days after infection") -ax.set_ylabel("Fraction of total shedding") -ax.set_title("Viral Shedding Kinetics") -ax.legend() -plt.tight_layout() -plt.show() ``` ### Genome copies and wastewater volume @@ -364,7 +389,7 @@ def first_valid_observation_day(obs_process) -> int: return obs_process.lookback_days() - 1 ``` -### Simulating from an infection spike +### Simulating from observations from a single-day infection spike To see how infections spread into concentrations via shedding kinetics, we simulate from a single-day spike: @@ -379,7 +404,7 @@ infections = jnp.zeros((n_days, 1)) # 1 subpopulation infections = infections.at[infection_spike_day, 0].set(2000.0) # For plotting -spike_day = infection_spike_day - day_one +rel_spike_day = infection_spike_day - day_one n_plot_days = n_days - day_one # Observation times and indices @@ -396,30 +421,43 @@ with numpyro.handlers.seed(rng_seed=42): n_sensors=1, ) ``` +We plot the resulting observations starting from the first valid observation day. ```{python} -# | label: fig-spike-infections -# | fig-cap: Input infections with a single-day spike -fig, ax = plt.subplots(figsize=(8, 4)) -ax.plot( - np.arange(n_plot_days), - np.array(infections[day_one:, 0]), - "-o", - color="darkblue", +# | label: plot-spike-infections +infections_df = pd.DataFrame( + { + "day": np.arange(n_plot_days), + "infections": np.array(infections[day_one:, 0]), + } ) -ax.axvline(spike_day, color="darkred", linestyle="--", alpha=0.7) -ax.annotate( - f"Infection spike\n(day {spike_day})", - xy=(spike_day, 1800), - xytext=(spike_day + 3, 1800), - fontsize=10, - color="darkred", + +max_infection_count = float(jnp.max(infections[day_one:])) + +plot_infections = ( + p9.ggplot(infections_df, p9.aes(x="day", y="infections")) + + p9.geom_line(color="darkblue", size=1) + + p9.geom_point(color="darkblue", size=2) + + p9.geom_vline( + xintercept=rel_spike_day, + color="darkred", + linetype="dashed", + alpha=0.5, + ) + + p9.labs(x="Day", y="Daily Infections", title="Infections (Input)") + + p9.theme_grey() + + p9.theme(plot_title=p9.element_text(size=13, weight="bold")) + + p9.annotate( + "text", + x=rel_spike_day, + y=max_infection_count * 1.05, + label=f"Infection spike\n(day {rel_spike_day})", + color="darkred", + size=10, + ha="center", + ) ) -ax.set_xlabel("Day") -ax.set_ylabel("Daily Infections") -ax.set_title("Infections (Input)") -plt.tight_layout() -plt.show() +plot_infections ``` ### Observation noise @@ -429,7 +467,7 @@ Sampling multiple times from the same infections shows the range of possible obs ```{python} # | label: sample-realizations n_samples = 50 -samples_by_day = {int(d - day_one): [] for d in observation_days} +ww_samples_list = [] for seed in range(n_samples): with numpyro.handlers.seed(rng_seed=seed): @@ -442,109 +480,174 @@ for seed in range(n_samples): n_sensors=1, ) for day_idx, conc in zip(observation_days, ww_result.observed): - samples_by_day[int(day_idx) - day_one].append(float(conc)) + ww_samples_list.append( + { + "day": int(day_idx) - day_one, + "log_concentration": float(conc), + "sample": seed, + } + ) -# Convert to arrays for plotting -plot_days = sorted(samples_by_day.keys()) -all_samples = np.array( - [[samples_by_day[d][i] for d in plot_days] for i in range(n_samples)] -) -sample_mean = all_samples.mean(axis=0) +ww_samples_df = pd.DataFrame(ww_samples_list) ``` ```{python} -# | label: fig-sampled-concentrations -# | fig-cap: Multiple realizations showing observation noise -fig, ax = plt.subplots(figsize=(8, 4)) - -# Plot all samples -for i in range(1, n_samples): - ax.plot( - plot_days, all_samples[i], color="orange", alpha=0.15, linewidth=0.5 - ) - -# Highlight one sample -ax.plot( - plot_days, - all_samples[0], - color="steelblue", - linewidth=1, - label="One realization", +# | label: plot-sampled-concentrations +# Compute mean across samples for each day +mean_by_day = ( + ww_samples_df.groupby("day")["log_concentration"].mean().reset_index() ) - -# Sample mean -ax.plot( - plot_days, sample_mean, color="darkred", linewidth=1.2, label="Sample mean" +mean_by_day["sample"] = -1 + +# Relative peak day for plotting (using mode, not mean, since distribution is skewed) +peak_day = rel_spike_day + mode_shedding_day + +# Separate one sample to highlight +highlight_sample = 0 +other_samples_df = ww_samples_df[ww_samples_df["sample"] != highlight_sample] +highlight_df = ww_samples_df[ww_samples_df["sample"] == highlight_sample] + +# For annotation positioning +max_conc = ww_samples_df["log_concentration"].max() + +( + p9.ggplot() + + p9.geom_line( + p9.aes(x="day", y="log_concentration", group="sample"), + data=other_samples_df, + color="orange", + alpha=0.15, + size=0.5, + ) + + p9.geom_line( + p9.aes(x="day", y="log_concentration"), + data=highlight_df, + color="steelblue", + size=1, + ) + + p9.geom_line( + p9.aes(x="day", y="log_concentration"), + data=mean_by_day, + color="darkred", + size=1.2, + ) + + p9.geom_vline( + xintercept=rel_spike_day, + color="darkblue", + linetype="dashed", + alpha=0.5, + ) + + p9.geom_vline( + xintercept=peak_day, + color="darkred", + linetype="dotted", + alpha=0.7, + ) + + p9.annotate( + "text", + x=rel_spike_day, + y=max_conc * 1.05, + label=f"Infection spike\n(day {rel_spike_day})", + color="darkblue", + size=9, + ha="center", + ) + + p9.annotate( + "text", + x=peak_day, + y=max_conc * 0.98, + label=f"Expected peak\n(day {peak_day})", + color="darkred", + size=9, + ha="center", + ) + + p9.labs( + x="Day", + y="Log Viral Concentration", + title=f"Observation Noise: {n_samples} Samples from Same Infections", + subtitle="Blue: one realization | Orange: other samples | Dark red: sample mean", + ) + + p9.theme_grey() ) - -ax.axvline(spike_day, color="darkblue", linestyle="--", alpha=0.5) -ax.set_xlabel("Day") -ax.set_ylabel("Log Viral Concentration") -ax.set_title(f"Observation Noise: {n_samples} Samples from Same Infections") -ax.legend() -plt.tight_layout() -plt.show() ``` ### Sensor-level variability -Different sensors (WWTP/lab pairs) measuring the same underlying infections show systematic differences: +The previous plot showed variability from repeatedly sampling the entire observation process (resampling sensor parameters and noise each time). In practice, we have multiple physical sensors, each with fixed but unknown characteristics. + +This plot shows four sensors observing the **same infection spike**. Each sensor has: + +- A **sensor mode** (systematic bias): shifts all observations up or down +- A **sensor SD** (measurement precision): determines noise level around predictions + +These parameters are sampled once per sensor, then held fixed across all observations from that sensor. ```{python} # | label: multi-sensor num_sensors = 4 -infections_constant = jnp.ones((40, 1)) * 1500.0 -# Each sensor samples at multiple time points -times_per_sensor = 10 -observation_times = jnp.tile( - jnp.arange(10, 30, 2, dtype=jnp.int32), num_sensors -) +# Use the same observation times and infections as the sampled-concentrations plot +sensor_obs_times = jnp.tile(observation_days, num_sensors) sensor_ids = jnp.repeat( - jnp.arange(num_sensors, dtype=jnp.int32), times_per_sensor + jnp.arange(num_sensors, dtype=jnp.int32), len(observation_days) ) -subpop_ids = jnp.zeros(num_sensors * times_per_sensor, dtype=jnp.int32) +subpop_ids = jnp.zeros(num_sensors * len(observation_days), dtype=jnp.int32) with numpyro.handlers.seed(rng_seed=42): ww_multi_sensor = ww_process.sample( - infections=infections_constant, + infections=infections, # Same spike as before subpop_indices=subpop_ids, sensor_indices=sensor_ids, - times=observation_times, + times=sensor_obs_times, obs=None, n_sensors=num_sensors, ) + +# Create DataFrame for plotting (using relative days) +multi_sensor_df = pd.DataFrame( + { + "day": np.array(sensor_obs_times) - day_one, + "log_concentration": np.array(ww_multi_sensor.observed), + "sensor": [f"Sensor {i}" for i in np.array(sensor_ids)], + } +) ``` ```{python} -# | label: fig-multi-sensor -# | fig-cap: Multiple sensors observing the same infections -fig, ax = plt.subplots(figsize=(8, 4)) - -colors = ["steelblue", "coral", "green", "purple"] -for sensor in range(num_sensors): - mask = np.array(sensor_ids) == sensor - ax.plot( - np.array(observation_times)[mask], - np.array(ww_multi_sensor.observed)[mask], - "-o", - color=colors[sensor], - label=f"Sensor {sensor}", +# | label: plot-multi-sensor +# Use same y-axis range as sampled-concentrations plot for comparison +y_min = ww_samples_df["log_concentration"].min() +y_max = ww_samples_df["log_concentration"].max() + +( + p9.ggplot( + multi_sensor_df, p9.aes(x="day", y="log_concentration", color="sensor") ) - -ax.set_xlabel("Day") -ax.set_ylabel("Log Viral Concentration") -ax.set_title("Multiple Sensors Observing Same Infections") -ax.legend() -plt.tight_layout() -plt.show() + + p9.geom_line(size=1) + + p9.geom_point(size=2) + + p9.ylim(y_min, y_max * 1.05) + + p9.labs( + x="Day", + y="Log Viral Concentration", + title="Four Sensors Observing the Same Infection Spike", + color="Sensor", + ) + + p9.theme_grey() +) ``` -Each sensor has a different baseline (sensor mode) and measurement variability (sensor SD). During inference, these sensor-specific effects are learned from data. +Compare this to the previous plot: here, each colored line represents a distinct physical sensor with its own systematic bias. The vertical spread between sensors reflects differences in sensor modes, while the noise within each line reflects each sensor's measurement precision. During inference, these sensor-specific effects are learned from data. ### Multiple subpopulations -In hierarchical models, each sensor may serve a distinct catchment area (subpopulation) with different infection dynamics: +In regional surveillance, each wastewater treatment plant serves a distinct **catchment area** (subpopulation) with its own infection dynamics. The `subpop_indices` array maps each observation to the appropriate infection trajectory. + +This example shows two subpopulations with different epidemic curves: + +- **Subpopulation 0**: Slow decay (e.g., large urban area with sustained transmission) +- **Subpopulation 1**: Fast decay (e.g., smaller community with rapid burnout) + +Each subpopulation is observed by its own sensor. The observed concentrations reflect both the underlying infection differences AND the sensor-specific measurement characteristics. ```{python} # | label: multi-subpop @@ -572,30 +675,38 @@ with numpyro.handlers.seed(rng_seed=42): obs=None, n_sensors=2, ) + +# Create DataFrame for plotting +multi_subpop_df = pd.DataFrame( + { + "day": np.array(obs_days_mp), + "log_concentration": np.array(ww_multi_subpop.observed), + "subpopulation": [f"Subpop {i}" for i in np.array(subpop_ids_mp)], + } +) ``` ```{python} -# | label: fig-multi-subpop -# | fig-cap: Sensors observing different subpopulations -fig, ax = plt.subplots(figsize=(8, 4)) - -for subpop in [0, 1]: - mask = np.array(subpop_ids_mp) == subpop - ax.plot( - np.array(obs_days_mp)[mask], - np.array(ww_multi_subpop.observed)[mask], - "-o", - label=f"Subpop {subpop}", +# | label: plot-multi-subpop +( + p9.ggplot( + multi_subpop_df, + p9.aes(x="day", y="log_concentration", color="subpopulation"), ) - -ax.set_xlabel("Day") -ax.set_ylabel("Log Viral Concentration") -ax.set_title("Multiple Subpopulations with Different Infection Dynamics") -ax.legend() -plt.tight_layout() -plt.show() + + p9.geom_line(size=1) + + p9.geom_point(size=2) + + p9.labs( + x="Day", + y="Log Viral Concentration", + title="Two Subpopulations with Different Infection Dynamics", + color="Subpopulation", + ) + + p9.theme_grey() +) ``` +The diverging trajectories reflect the different underlying infection curves. Subpopulation 1 starts higher but decays faster, while Subpopulation 0 maintains more sustained levels. In a full model, you would jointly infer the infection trajectories for each subpopulation while accounting for sensor-specific biases. + --- ## Summary From 9306edcd6bb23be8b8e04cee622fab634ac67de1 Mon Sep 17 00:00:00 2001 From: Mitzi Morris Date: Thu, 8 Jan 2026 21:40:08 -0500 Subject: [PATCH 33/81] minor tutorials edits --- .../observation_processes_measurements.qmd | 30 +++++++++---------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/docs/tutorials/observation_processes_measurements.qmd b/docs/tutorials/observation_processes_measurements.qmd index 66c65aa4..ddf89a8d 100644 --- a/docs/tutorials/observation_processes_measurements.qmd +++ b/docs/tutorials/observation_processes_measurements.qmd @@ -33,7 +33,7 @@ from pyrenew.randomvariable import DistributionalVariable from pyrenew.deterministic import DeterministicVariable, DeterministicPMF ``` -# The Measurements Framework +## The Measurements Class The `Measurements` class models continuous signals derived from infections. Unlike count observations (hospital admissions, deaths), measurements are continuous values that may span orders of magnitude or even be negative (e.g., log-transformed data). @@ -44,7 +44,7 @@ The `Measurements` class models continuous signals derived from infections. Unli - Serological assay results - Environmental sensor readings -## The general pattern +### The general pattern All measurement observation processes follow the same pattern: @@ -62,7 +62,7 @@ The `Measurements` base class provides: - Integration with hierarchical noise models - Support for multiple sensors and subpopulations -## Comparison with count observations +### Comparison with count observations The core convolution structure is shared with count observations, but key aspects differ: @@ -74,7 +74,7 @@ The core convolution structure is shared with count observations, but key aspect | Scaling | Ascertainment rate $\alpha \in [0,1]$ | Domain-specific | | Subpop structure | Optional (`CountsBySubpop`) | Inherent (sensor/site effects) | -## The noise model +### The noise model Measurement data typically exhibits **sensor-level variability**: different instruments, labs, or sampling locations have systematic biases and different precision levels. @@ -106,7 +106,7 @@ noise = HierarchicalNormalNoise( ) ``` -## The indexing system +### The indexing system Measurement observations use three index arrays to map observations to their context: @@ -122,7 +122,7 @@ This flexible indexing supports: - **Multiple sensors per subpopulation**: Different labs analyzing the same source - **Multiple subpopulations per sensor**: One sensor serving multiple areas (less common) -## Subclassing Measurements +### Subclassing Measurements To create a measurement process for your domain, subclass `Measurements` and implement: @@ -151,12 +151,12 @@ class MyMeasurement(Measurements): ``` -# Wastewater Example +## Measurement Example: Wastewater To illustrate the framework, we specify a wastewater viral concentration observation process, based on the [PyRenew-HEW](https://github.com/cdcgov/pyrenew-hew) family of models. -## The wastewater signal +**The wastewater signal** Wastewater treatment plants measure viral RNA concentrations in sewage. The predicted concentration depends on: @@ -180,7 +180,7 @@ Observations are log-concentrations with normal noise: $$y_t \sim \text{Normal}(\log(\lambda_t) + \text{sensor\_mode}, \text{sensor\_sd})$$ -## Implementing the Wastewater class +### Implementing the Wastewater class ```{python} # | label: wastewater-class @@ -264,9 +264,9 @@ class Wastewater(Measurements): return jnp.log(concentration) ``` -## Configuring wastewater-specific parameters +#### Configuring wastewater-specific parameters -### Viral shedding kinetics +**Viral shedding kinetics** The shedding PMF describes what fraction of total viral shedding occurs on each day after infection: @@ -329,7 +329,7 @@ shedding_df = pd.DataFrame( ) ``` -### Genome copies and wastewater volume +**Genome copies and wastewater volume** ```{python} # | label: scaling-params @@ -362,7 +362,7 @@ ww_noise = HierarchicalNormalNoise( ) ``` -## Creating the wastewater observation process +### Creating the wastewater observation process ```{python} # | label: create-process @@ -376,7 +376,7 @@ ww_process = Wastewater( print(f"Required lookback: {ww_process.lookback_days()} days") ``` -## Demonstrations +## Simulations ### Timeline alignment @@ -711,7 +711,7 @@ The diverging trajectories reflect the different underlying infection curves. Su ## Summary -The `Measurements` framework provides: +The `Measurements` class provides: 1. **A consistent interface** for continuous observation processes 2. **Hierarchical noise models** that capture sensor-level variability From d103d0ff1c6a65b3c7c3778c1d0a450dcf6495e5 Mon Sep 17 00:00:00 2001 From: Mitzi Morris Date: Tue, 13 Jan 2026 18:30:21 -0500 Subject: [PATCH 34/81] change plate management strategy --- .../observation_processes_measurements.qmd | 38 ++++--- pyrenew/observation/__init__.py | 2 + pyrenew/observation/noise.py | 106 ++++++++++++------ test/test_observation_measurements.py | 87 +++++++++----- 4 files changed, 157 insertions(+), 76 deletions(-) diff --git a/docs/tutorials/observation_processes_measurements.qmd b/docs/tutorials/observation_processes_measurements.qmd index ddf89a8d..f0bf03ff 100644 --- a/docs/tutorials/observation_processes_measurements.qmd +++ b/docs/tutorials/observation_processes_measurements.qmd @@ -28,7 +28,11 @@ import plotnine as p9 import numpyro.distributions as dist -from pyrenew.observation import Measurements, HierarchicalNormalNoise +from pyrenew.observation import ( + Measurements, + HierarchicalNormalNoise, + VectorizedRV, +) from pyrenew.randomvariable import DistributionalVariable from pyrenew.deterministic import DeterministicVariable, DeterministicPMF ``` @@ -87,16 +91,22 @@ Measurement data typically exhibits **sensor-level variability**: different inst observed ~ Normal(predicted + sensor_mode[sensor], sensor_sd[sensor]) ``` -The noise model samples sensor-level parameters within a plate, so any `RandomVariable` can be used as a prior: +The sensor-level RVs must implement `sample(n_groups=...)`. Use `VectorizedRV` to wrap simple distributions: ```{python} # | label: noise-model-general # Sensor modes: zero-centered, allowing positive or negative bias -sensor_mode_rv = DistributionalVariable("sensor_mode", dist.Normal(0, 0.5)) +sensor_mode_rv = VectorizedRV( + DistributionalVariable("sensor_mode", dist.Normal(0, 0.5)), + plate_name="sensor_mode", +) # Sensor SDs: must be positive, truncated normal is a common choice -sensor_sd_rv = DistributionalVariable( - "sensor_sd", dist.TruncatedNormal(loc=0.3, scale=0.15, low=0.05) +sensor_sd_rv = VectorizedRV( + DistributionalVariable( + "sensor_sd", dist.TruncatedNormal(loc=0.3, scale=0.15, low=0.05) + ), + plate_name="sensor_sd", ) # Create noise model @@ -347,13 +357,17 @@ For wastewater, a "sensor" is a WWTP/lab pair—the combination of treatment pla ```{python} # | label: ww-noise-model # Sensor-level mode: systematic differences between WWTP/lab pairs -ww_sensor_mode_rv = DistributionalVariable( - "ww_sensor_mode", dist.Normal(0, 0.5) +ww_sensor_mode_rv = VectorizedRV( + DistributionalVariable("ww_sensor_mode", dist.Normal(0, 0.5)), + plate_name="ww_sensor_mode", ) # Sensor-level SD: measurement variability within each WWTP/lab pair -ww_sensor_sd_rv = DistributionalVariable( - "ww_sensor_sd", dist.TruncatedNormal(loc=0.3, scale=0.15, low=0.10) +ww_sensor_sd_rv = VectorizedRV( + DistributionalVariable( + "ww_sensor_sd", dist.TruncatedNormal(loc=0.3, scale=0.15, low=0.10) + ), + plate_name="ww_sensor_sd", ) ww_noise = HierarchicalNormalNoise( @@ -421,6 +435,7 @@ with numpyro.handlers.seed(rng_seed=42): n_sensors=1, ) ``` + We plot the resulting observations starting from the first valid observation day. ```{python} @@ -615,17 +630,12 @@ multi_sensor_df = pd.DataFrame( ```{python} # | label: plot-multi-sensor -# Use same y-axis range as sampled-concentrations plot for comparison -y_min = ww_samples_df["log_concentration"].min() -y_max = ww_samples_df["log_concentration"].max() - ( p9.ggplot( multi_sensor_df, p9.aes(x="day", y="log_concentration", color="sensor") ) + p9.geom_line(size=1) + p9.geom_point(size=2) - + p9.ylim(y_min, y_max * 1.05) + p9.labs( x="Day", y="Log Viral Concentration", diff --git a/pyrenew/observation/__init__.py b/pyrenew/observation/__init__.py index 8a0cdeab..42f0aafd 100644 --- a/pyrenew/observation/__init__.py +++ b/pyrenew/observation/__init__.py @@ -30,6 +30,7 @@ MeasurementNoise, NegativeBinomialNoise, PoissonNoise, + VectorizedRV, ) from pyrenew.observation.poisson import PoissonObservation from pyrenew.observation.types import ObservationSample @@ -47,6 +48,7 @@ "NegativeBinomialNoise", "MeasurementNoise", "HierarchicalNormalNoise", + "VectorizedRV", # Observation processes "Counts", "CountsBySubpop", diff --git a/pyrenew/observation/noise.py b/pyrenew/observation/noise.py index 9ce282e7..01c833fe 100644 --- a/pyrenew/observation/noise.py +++ b/pyrenew/observation/noise.py @@ -14,8 +14,12 @@ Measurement Noise ----------------- - ``HierarchicalNormalNoise``: Normal noise with hierarchical sensor effects. - Takes ``sensor_mode_prior_rv`` and ``sensor_sd_prior_rv`` for sensor-level + Takes ``sensor_mode_rv`` and ``sensor_sd_rv`` for sensor-level bias and variability. + +Utilities +--------- +- ``VectorizedRV``: Wrapper that adds ``n_groups`` support to simple RVs. """ from __future__ import annotations @@ -32,6 +36,57 @@ _EPSILON = 1e-10 +class VectorizedRV(RandomVariable): + """ + Wrapper that adds n_groups support to simple RandomVariables. + + Uses numpyro.plate to vectorize sampling, enabling simple RVs + to work with noise models expecting the group-level interface. + + Parameters + ---------- + rv : RandomVariable + The underlying RandomVariable to wrap. + plate_name : str + Name for the numpyro plate used for vectorization. + """ + + def __init__(self, rv: RandomVariable, plate_name: str) -> None: + """ + Initialize VectorizedRV wrapper. + + Parameters + ---------- + rv : RandomVariable + The underlying RandomVariable to wrap. + plate_name : str + Name for the numpyro plate used for vectorization. + """ + self.rv = rv + self.plate_name = plate_name + + def validate(self): + """Validate the underlying RV.""" + self.rv.validate() + + def sample(self, n_groups: int, **kwargs): + """ + Sample n_groups values using numpyro.plate. + + Parameters + ---------- + n_groups : int + Number of group-level values to sample. + + Returns + ------- + ArrayLike + Array of shape (n_groups,). + """ + with numpyro.plate(self.plate_name, n_groups): + return self.rv(**kwargs) + + class CountNoise(ABC): """ Abstract base for count observation noise models. @@ -263,35 +318,20 @@ class HierarchicalNormalNoise(MeasurementNoise): Normal noise with hierarchical sensor-level effects. Observation model: ``obs ~ Normal(predicted + sensor_mode, sensor_sd)`` - where sensor_mode and sensor_sd are sampled per-sensor within a plate. + where sensor_mode and sensor_sd are sampled per-sensor. Parameters ---------- sensor_mode_rv : RandomVariable - Prior for sensor-level modes (log-scale biases). - Sampled once per sensor within a plate context. - Example: ``DistributionalVariable("mode", dist.Normal(0, 0.5))`` + Prior for sensor-level modes. + Must implement ``sample(n_groups=...) -> ArrayLike``. sensor_sd_rv : RandomVariable - Prior for sensor-level SDs (should be > 0). - Sampled once per sensor within a plate context. - Example: ``DistributionalVariable("sd", dist.TruncatedNormal(0.3, 0.15, low=0.05))`` + Prior for sensor-level SDs (must be > 0). + Must implement ``sample(n_groups=...) -> ArrayLike``. Notes ----- - Expects data already on log scale for wastewater applications. - - The sensor-level parameters are sampled within a numpyro plate context, - so any standard RandomVariable can be used (no special interface required). - - Examples - -------- - >>> from pyrenew.randomvariable import DistributionalVariable - >>> import numpyro.distributions as dist - >>> - >>> noise = HierarchicalNormalNoise( - ... sensor_mode_rv=DistributionalVariable("mode", dist.Normal(0, 0.5)), - ... sensor_sd_rv=DistributionalVariable("sd", dist.TruncatedNormal(0.3, 0.15, low=0.05)), - ... ) + Use ``VectorizedRV`` to wrap simple RVs that lack this interface. """ def __init__( @@ -305,11 +345,11 @@ def __init__( Parameters ---------- sensor_mode_rv : RandomVariable - Prior for sensor-level modes (log-scale biases). - Sampled once per sensor within a plate context. + Prior for sensor-level modes. + Must implement ``sample(n_groups=...) -> ArrayLike``. sensor_sd_rv : RandomVariable - Prior for sensor-level SDs (should be > 0). - Sampled once per sensor within a plate context. + Prior for sensor-level SDs (must be > 0). + Must implement ``sample(n_groups=...) -> ArrayLike``. """ self.sensor_mode_rv = sensor_mode_rv self.sensor_sd_rv = sensor_sd_rv @@ -349,10 +389,10 @@ def sample( name : str Numpyro sample site name. predicted : ArrayLike - Predicted log-scale measurement values. + Predicted measurement values. Shape: (n_obs,) obs : ArrayLike | None - Observed log-scale measurements for conditioning. + Observed measurements for conditioning. Shape: (n_obs,) sensor_indices : ArrayLike Sensor index for each observation (0-indexed). @@ -365,15 +405,9 @@ def sample( ArrayLike Normal distributed measurements with hierarchical sensor effects. Shape: (n_obs,) - - Raises - ------ - ValueError - If sensor_sd samples non-positive values. """ - with numpyro.plate("sensor", n_sensors): - sensor_mode = self.sensor_mode_rv() - sensor_sd = self.sensor_sd_rv() + sensor_mode = self.sensor_mode_rv(n_groups=n_sensors) + sensor_sd = self.sensor_sd_rv(n_groups=n_sensors) loc = predicted + sensor_mode[sensor_indices] scale = sensor_sd[sensor_indices] diff --git a/test/test_observation_measurements.py b/test/test_observation_measurements.py index b4ed3488..2b52f712 100644 --- a/test/test_observation_measurements.py +++ b/test/test_observation_measurements.py @@ -10,7 +10,11 @@ import pytest from pyrenew.deterministic import DeterministicPMF -from pyrenew.observation import HierarchicalNormalNoise, Measurements +from pyrenew.observation import ( + HierarchicalNormalNoise, + Measurements, + VectorizedRV, +) from pyrenew.observation.base import BaseObservationProcess from pyrenew.randomvariable import DistributionalVariable @@ -77,9 +81,13 @@ def test_is_base_observation_process(self): def test_infection_resolution_is_subpop(self): """Test that Measurements returns 'subpop' resolution.""" shedding_pmf = jnp.array([0.3, 0.4, 0.3]) - sensor_mode_rv = DistributionalVariable("mode", dist.Normal(0, 0.5)) - sensor_sd_rv = DistributionalVariable( - "sd", dist.TruncatedNormal(0.3, 0.15, low=0.05) + sensor_mode_rv = VectorizedRV( + DistributionalVariable("mode", dist.Normal(0, 0.5)), + plate_name="sensor_mode", + ) + sensor_sd_rv = VectorizedRV( + DistributionalVariable("sd", dist.TruncatedNormal(0.3, 0.15, low=0.05)), + plate_name="sensor_sd", ) noise = HierarchicalNormalNoise(sensor_mode_rv, sensor_sd_rv) @@ -96,9 +104,13 @@ class TestHierarchicalNormalNoise: def test_repr(self): """Test HierarchicalNormalNoise __repr__ method.""" - sensor_mode_rv = DistributionalVariable("mode", dist.Normal(0, 0.5)) - sensor_sd_rv = DistributionalVariable( - "sd", dist.TruncatedNormal(0.3, 0.15, low=0.05) + sensor_mode_rv = VectorizedRV( + DistributionalVariable("mode", dist.Normal(0, 0.5)), + plate_name="sensor_mode", + ) + sensor_sd_rv = VectorizedRV( + DistributionalVariable("sd", dist.TruncatedNormal(0.3, 0.15, low=0.05)), + plate_name="sensor_sd", ) noise = HierarchicalNormalNoise(sensor_mode_rv, sensor_sd_rv) repr_str = repr(noise) @@ -108,9 +120,13 @@ def test_repr(self): def test_validate(self): """Test HierarchicalNormalNoise validate method.""" - sensor_mode_rv = DistributionalVariable("mode", dist.Normal(0, 0.5)) - sensor_sd_rv = DistributionalVariable( - "sd", dist.TruncatedNormal(0.3, 0.15, low=0.05) + sensor_mode_rv = VectorizedRV( + DistributionalVariable("mode", dist.Normal(0, 0.5)), + plate_name="sensor_mode", + ) + sensor_sd_rv = VectorizedRV( + DistributionalVariable("sd", dist.TruncatedNormal(0.3, 0.15, low=0.05)), + plate_name="sensor_sd", ) noise = HierarchicalNormalNoise(sensor_mode_rv, sensor_sd_rv) # Should not raise - validation is deferred to sample time @@ -118,9 +134,13 @@ def test_validate(self): def test_sample_shape(self): """Test that HierarchicalNormalNoise produces correct shape.""" - sensor_mode_rv = DistributionalVariable("mode", dist.Normal(0, 0.5)) - sensor_sd_rv = DistributionalVariable( - "sd", dist.TruncatedNormal(0.3, 0.15, low=0.05) + sensor_mode_rv = VectorizedRV( + DistributionalVariable("mode", dist.Normal(0, 0.5)), + plate_name="sensor_mode", + ) + sensor_sd_rv = VectorizedRV( + DistributionalVariable("sd", dist.TruncatedNormal(0.3, 0.15, low=0.05)), + plate_name="sensor_sd", ) noise = HierarchicalNormalNoise(sensor_mode_rv, sensor_sd_rv) @@ -140,9 +160,13 @@ def test_sample_shape(self): def test_sample_with_observations(self): """Test that HierarchicalNormalNoise conditions on observations.""" - sensor_mode_rv = DistributionalVariable("mode", dist.Normal(0, 0.5)) - sensor_sd_rv = DistributionalVariable( - "sd", dist.TruncatedNormal(0.3, 0.15, low=0.05) + sensor_mode_rv = VectorizedRV( + DistributionalVariable("mode", dist.Normal(0, 0.5)), + plate_name="sensor_mode", + ) + sensor_sd_rv = VectorizedRV( + DistributionalVariable("sd", dist.TruncatedNormal(0.3, 0.15, low=0.05)), + plate_name="sensor_sd", ) noise = HierarchicalNormalNoise(sensor_mode_rv, sensor_sd_rv) @@ -169,9 +193,13 @@ class TestConcreteMeasurements: def test_repr(self): """Test Measurements __repr__ method.""" shedding_pmf = jnp.array([0.3, 0.4, 0.3]) - sensor_mode_rv = DistributionalVariable("mode", dist.Normal(0, 0.5)) - sensor_sd_rv = DistributionalVariable( - "sd", dist.TruncatedNormal(0.3, 0.15, low=0.05) + sensor_mode_rv = VectorizedRV( + DistributionalVariable("mode", dist.Normal(0, 0.5)), + plate_name="sensor_mode", + ) + sensor_sd_rv = VectorizedRV( + DistributionalVariable("sd", dist.TruncatedNormal(0.3, 0.15, low=0.05)), + plate_name="sensor_sd", ) noise = HierarchicalNormalNoise(sensor_mode_rv, sensor_sd_rv) @@ -188,9 +216,13 @@ def test_repr(self): def test_sample_shape(self): """Test that sample returns correct shape.""" shedding_pmf = jnp.array([0.3, 0.4, 0.3]) - sensor_mode_rv = DistributionalVariable("mode", dist.Normal(0, 0.5)) - sensor_sd_rv = DistributionalVariable( - "sd", dist.TruncatedNormal(0.3, 0.15, low=0.05) + sensor_mode_rv = VectorizedRV( + DistributionalVariable("mode", dist.Normal(0, 0.5)), + plate_name="sensor_mode", + ) + sensor_sd_rv = VectorizedRV( + DistributionalVariable("sd", dist.TruncatedNormal(0.3, 0.15, low=0.05)), + plate_name="sensor_sd", ) noise = HierarchicalNormalNoise(sensor_mode_rv, sensor_sd_rv) @@ -199,7 +231,6 @@ def test_sample_shape(self): noise=noise, ) - # 30 days, 2 subpops infections = jnp.ones((30, 2)) * 1000 subpop_indices = jnp.array([0, 0, 1, 1]) sensor_indices = jnp.array([0, 0, 1, 1]) @@ -221,9 +252,13 @@ def test_sample_shape(self): def test_predicted_obs_stored(self): """Test that predicted_log_conc is stored as deterministic.""" shedding_pmf = jnp.array([0.5, 0.5]) - sensor_mode_rv = DistributionalVariable("mode", dist.Normal(0, 0.01)) - sensor_sd_rv = DistributionalVariable( - "sd", dist.TruncatedNormal(0.01, 0.005, low=0.001) + sensor_mode_rv = VectorizedRV( + DistributionalVariable("mode", dist.Normal(0, 0.01)), + plate_name="sensor_mode", + ) + sensor_sd_rv = VectorizedRV( + DistributionalVariable("sd", dist.TruncatedNormal(0.01, 0.005, low=0.001)), + plate_name="sensor_sd", ) noise = HierarchicalNormalNoise(sensor_mode_rv, sensor_sd_rv) From 5f9cb345c705a0e485bada74f76028165934e82f Mon Sep 17 00:00:00 2001 From: Mitzi Morris Date: Wed, 14 Jan 2026 14:20:11 -0500 Subject: [PATCH 35/81] adding hierarchical latent infections, model builder, and unit tests --- pyrenew/latent/__init__.py | 34 ++ pyrenew/latent/base.py | 392 ++++++++++++++++ pyrenew/latent/hierarchical_infections.py | 307 +++++++++++++ pyrenew/latent/hierarchical_priors.py | 208 +++++++++ pyrenew/latent/partitioned_infections.py | 293 ++++++++++++ pyrenew/latent/temporal_processes.py | 527 ++++++++++++++++++++++ pyrenew/model/__init__.py | 9 +- pyrenew/model/model_builder.py | 256 +++++++++++ pyrenew/model/multisignal_model.py | 370 +++++++++++++++ test/test_base_latent_process.py | 59 +++ test/test_hierarchical_infections.py | 130 ++++++ test/test_hierarchical_priors.py | 179 ++++++++ test/test_model_builder.py | 152 +++++++ test/test_partitioned_infections.py | 125 +++++ test/test_temporal_processes.py | 125 +++++ 15 files changed, 3165 insertions(+), 1 deletion(-) create mode 100644 pyrenew/latent/base.py create mode 100644 pyrenew/latent/hierarchical_infections.py create mode 100644 pyrenew/latent/hierarchical_priors.py create mode 100644 pyrenew/latent/partitioned_infections.py create mode 100644 pyrenew/latent/temporal_processes.py create mode 100644 pyrenew/model/model_builder.py create mode 100644 pyrenew/model/multisignal_model.py create mode 100644 test/test_base_latent_process.py create mode 100644 test/test_hierarchical_infections.py create mode 100644 test/test_hierarchical_priors.py create mode 100644 test/test_model_builder.py create mode 100644 test/test_partitioned_infections.py create mode 100644 test/test_temporal_processes.py diff --git a/pyrenew/latent/__init__.py b/pyrenew/latent/__init__.py index 0d8a553a..573bbe8d 100644 --- a/pyrenew/latent/__init__.py +++ b/pyrenew/latent/__init__.py @@ -1,5 +1,16 @@ # numpydoc ignore=GL08 +from pyrenew.latent.base import ( + BaseLatentInfectionProcess, + LatentSample, + PopulationStructure, +) +from pyrenew.latent.hierarchical_infections import HierarchicalInfections +from pyrenew.latent.hierarchical_priors import ( + GammaGroupSdPrior, + HierarchicalNormalPrior, + StudentTGroupModePrior, +) from pyrenew.latent.hospitaladmissions import HospitalAdmissions from pyrenew.latent.infection_functions import ( compute_infections_from_rt, @@ -17,6 +28,13 @@ ) from pyrenew.latent.infections import Infections from pyrenew.latent.infectionswithfeedback import InfectionsWithFeedback +from pyrenew.latent.partitioned_infections import PartitionedInfections +from pyrenew.latent.temporal_processes import ( + AR1, + DifferencedAR1, + RandomWalk, + TemporalProcess, +) __all__ = [ "HospitalAdmissions", @@ -30,4 +48,20 @@ "InitializeInfectionsZeroPad", "InfectionInitializationProcess", "InfectionsWithFeedback", + # Base classes and types + "BaseLatentInfectionProcess", + "LatentSample", + "PopulationStructure", + # Hierarchical infection processes + "HierarchicalInfections", + "PartitionedInfections", + # Hierarchical priors + "HierarchicalNormalPrior", + "GammaGroupSdPrior", + "StudentTGroupModePrior", + # Temporal processes + "TemporalProcess", + "AR1", + "DifferencedAR1", + "RandomWalk", ] diff --git a/pyrenew/latent/base.py b/pyrenew/latent/base.py new file mode 100644 index 00000000..ac9a8a95 --- /dev/null +++ b/pyrenew/latent/base.py @@ -0,0 +1,392 @@ +""" +Base class for latent infection processes with subpopulation structure. +""" + +from __future__ import annotations + +from abc import abstractmethod +from dataclasses import dataclass +from typing import NamedTuple + +import jax +import jax.numpy as jnp +from jax.typing import ArrayLike +from pyrenew.metaclass import RandomVariable + + +class LatentSample(NamedTuple): + """ + Output from latent infection process sampling. + + Attributes + ---------- + aggregate : ArrayLike + Total infections aggregated across all subpopulations. + Shape: (n_total_days,) + all_subpops : ArrayLike + Infections for all subpopulations. + Shape: (n_total_days, K) + observed : ArrayLike + Infections for observed subpopulations only. + Shape: (n_total_days, K_obs) + unobserved : ArrayLike + Infections for unobserved subpopulations only. + Shape: (n_total_days, K_unobs) + """ + + aggregate: ArrayLike + all_subpops: ArrayLike + observed: ArrayLike + unobserved: ArrayLike + + +@dataclass +class PopulationStructure: + """ + Parsed and validated population structure for a jurisdiction. + + Attributes + ---------- + obs_fractions : ArrayLike + Population fractions for observed subpopulations + unobs_fractions : ArrayLike + Population fractions for unobserved subpopulations + K : int + Total number of subpopulations + K_obs : int + Number of observed subpopulations + K_unobs : int + Number of unobserved subpopulations + """ + + obs_fractions: ArrayLike + unobs_fractions: ArrayLike + K: int + K_obs: int + K_unobs: int + + +class BaseLatentInfectionProcess(RandomVariable): + """ + Base class for latent infection processes with subpopulation structure. + + Provides common functionality for hierarchical and partitioned infection models: + - Population fraction validation and parsing (at sample time) + - Splitting subpopulations into observed/unobserved + - Standard 4-tuple output structure + + All subclasses return infections as a ``LatentSample`` named tuple with fields: + (aggregate, all_subpops, observed, unobserved) + + The constructor specifies model structure (generation interval, priors, + temporal processes). Population structure (obs_fractions, unobs_fractions) is + provided at sample time, allowing a single model to be fit to multiple + jurisdictions with different population structures. + + Parameters + ---------- + gen_int_rv : RandomVariable + Generation interval PMF + n_initialization_points : int, optional + Number of initialization days before day 0. If not specified, automatically + computed as max(21, 2*len(gen_int)-1). The ModelBuilder will compute and pass + the correct value based on all observation processes. + + Notes + ----- + Population structure (obs_fractions, unobs_fractions) is passed to the sample() + method, not the constructor. This allows a single model instance to be fit to + multiple datasets with different jurisdiction structures. + + When using ModelBuilder, n_initialization_points is computed automatically from + all observation processes. When manually creating latent processes, you can either + specify it explicitly or let it default to max(21, 2*len(gen_int)-1). + + Subclasses must implement the `sample` method which returns a 4-tuple with + shapes: (n_total_days,), (n_total_days, K), (n_total_days, K_obs), + (n_total_days, K_unobs). + """ + + def __init__( + self, + *, + gen_int_rv: RandomVariable, + n_initialization_points: int = None, + ) -> None: + """ + Initialize base latent infection process. + + Parameters + ---------- + gen_int_rv : RandomVariable + Generation interval PMF + n_initialization_points : int, optional + Number of initialization days before day 0. If not specified, + defaults to max(21, 2*len(gen_int)-1). + """ + if gen_int_rv is None: + raise ValueError("gen_int_rv is required") + self.gen_int_rv = gen_int_rv + + # Compute n_initialization_points if not provided + # Use max(21, 2*len(gen_int)-1) for conservative default + if n_initialization_points is None: + gen_int_length = len(self.gen_int_rv()) + n_initialization_points = max(21, 2 * gen_int_length - 1) + + if n_initialization_points < 0: + raise ValueError( + f"n_initialization_points must be non-negative, " + f"got {n_initialization_points}" + ) + self.n_initialization_points = n_initialization_points + + @staticmethod + def _parse_and_validate_fractions( + obs_fractions: ArrayLike = None, + unobs_fractions: ArrayLike = None, + ) -> PopulationStructure: + """ + Parse and validate population fraction parameters. + + Returns + ------- + PopulationStructure + Parsed and validated population structure + + Raises + ------ + ValueError + If parameterization is invalid or fractions don't sum to 1.0 + """ + if obs_fractions is None or unobs_fractions is None: + raise ValueError("Both obs_fractions and unobs_fractions must be provided") + + obs_fractions = jnp.asarray(obs_fractions) + unobs_fractions = jnp.asarray(unobs_fractions) + + if obs_fractions.ndim != 1 or unobs_fractions.ndim != 1: + raise ValueError("Fractions must be 1D arrays") + + K_obs = len(obs_fractions) + K_unobs = len(unobs_fractions) + K = K_obs + K_unobs + + if K_obs == 0: + raise ValueError("Must have at least one observed subpopulation") + + # Only validate when values are concrete (not during JAX tracing) + try: + if jnp.any(obs_fractions < 0) or jnp.any(unobs_fractions < 0): + raise ValueError("All population fractions must be non-negative") + + total = jnp.sum(obs_fractions) + jnp.sum(unobs_fractions) + if not jnp.isclose(total, 1.0, atol=1e-6): + raise ValueError( + f"Population fractions must sum to 1.0, got {float(total):.6f}" + ) + except jax.errors.TracerBoolConversionError: + # Skip validation during JAX tracing - values were validated before MCMC + pass + + return PopulationStructure( + obs_fractions=obs_fractions, + unobs_fractions=unobs_fractions, + K=K, + K_obs=K_obs, + K_unobs=K_unobs, + ) + + @staticmethod + def _split_subpopulations( + infections_all: ArrayLike, + pop: PopulationStructure, + ) -> tuple[ArrayLike, ArrayLike]: + """ + Split all subpopulations into observed and unobserved. + + Parameters + ---------- + infections_all : ArrayLike + Infections for all subpopulations, shape (n_days, K) + pop : PopulationStructure + Population structure with K_obs for splitting + + Returns + ------- + tuple[ArrayLike, ArrayLike] + (infections_observed, infections_unobserved) with shapes + (n_days, K_obs) and (n_days, K_unobs) + """ + infections_observed = infections_all[:, : pop.K_obs] + infections_unobserved = infections_all[:, pop.K_obs :] + return infections_observed, infections_unobserved + + @staticmethod + def _validate_output_shapes( + infections_aggregate: ArrayLike, + infections_all: ArrayLike, + infections_observed: ArrayLike, + infections_unobserved: ArrayLike, + n_total_days: int, + pop: PopulationStructure, + ) -> None: + """ + Validate that output shapes match expected dimensions. + + Parameters + ---------- + infections_aggregate : ArrayLike + Aggregate infections (sum across subpopulations) + infections_all : ArrayLike + All subpopulation infections + infections_observed : ArrayLike + Observed subpopulation infections + infections_unobserved : ArrayLike + Unobserved subpopulation infections + n_total_days : int + Expected number of days (n_initialization_points + n_days_post_init) + pop : PopulationStructure + Population structure with K, K_obs, K_unobs + + Raises + ------ + ValueError + If any output shape is incorrect + """ + expected = { + "infections_aggregate": (n_total_days,), + "infections_all": (n_total_days, pop.K), + "infections_observed": (n_total_days, pop.K_obs), + "infections_unobserved": (n_total_days, pop.K_unobs), + } + + actual = { + "infections_aggregate": infections_aggregate.shape, + "infections_all": infections_all.shape, + "infections_observed": infections_observed.shape, + "infections_unobserved": infections_unobserved.shape, + } + + for name, expected_shape in expected.items(): + if actual[name] != expected_shape: + raise ValueError( + f"{name} has incorrect shape. " + f"Expected {expected_shape}, got {actual[name]}" + ) + + @staticmethod + def _validate_I0(I0: ArrayLike) -> None: + """ + Validate that I0 values are valid infection prevalences. + + I0 represents initial infection prevalence as a proportion of the + population. Values must be in the interval (0, 1]. + + Parameters + ---------- + I0 : ArrayLike + Initial infection prevalence (scalar or array) + + Raises + ------ + ValueError + If any I0 value is not in the interval (0, 1] + + Notes + ----- + Validation is skipped during JAX tracing (e.g., when using Predictive) + since traced values cannot be used in Python boolean operations. + """ + import jax + + I0 = jnp.asarray(I0) + + # Skip validation during JAX tracing (e.g., Predictive sampling) + # Traced arrays cannot be used in Python boolean operations + try: + # This will fail if I0 is a tracer + is_invalid_low = bool(jnp.any(I0 <= 0)) + is_invalid_high = bool(jnp.any(I0 > 1)) + except jax.errors.TracerBoolConversionError: + # During tracing, skip validation + return + + if is_invalid_low: + raise ValueError( + f"I0 must be positive (got min={float(jnp.min(I0)):.6f}). " + "I0 represents infection prevalence as a proportion of the population." + ) + if is_invalid_high: + raise ValueError( + f"I0 must be <= 1 (got max={float(jnp.max(I0)):.6f}). " + "I0 represents infection prevalence as a proportion of the population." + ) + + def get_required_lookback(self) -> int: + """ + Return the generation interval length for builder pattern support. + + This method is used by ModelBuilder to compute n_initialization_points + from all model components. Returns the generation interval PMF length. + + Returns + ------- + int + Length of generation interval PMF + """ + return len(self.gen_int_rv()) + + @abstractmethod + def validate(self) -> None: + """ + Validate latent process parameters. + + Subclasses must implement this method to validate all parameters specific + to their implementation (e.g., temporal process parameters, I0 parameters). + + Common validation (n_initialization_points, gen_int_rv) is performed in + __init__. Population structure validation is performed at sample time. + + Raises + ------ + ValueError + If any parameters fail validation + """ + pass + + @abstractmethod + def sample( + self, + n_days_post_init: int, + *, + obs_fractions: ArrayLike = None, + unobs_fractions: ArrayLike = None, + **kwargs, + ) -> LatentSample: + """ + Sample latent infections for all subpopulations. + + Parameters + ---------- + n_days_post_init : int + Number of days to simulate after initialization period + obs_fractions : ArrayLike + Population fractions for observed subpopulations. + unobs_fractions : ArrayLike + Population fractions for unobserved subpopulations. + **kwargs + Additional parameters required by specific implementations + + Returns + ------- + LatentSample + Named tuple with fields: + - aggregate: shape (n_total_days,) + - all_subpops: shape (n_total_days, K) + - observed: shape (n_total_days, K_obs) + - unobserved: shape (n_total_days, K_unobs) + + where n_total_days = n_initialization_points + n_days_post_init + """ + pass diff --git a/pyrenew/latent/hierarchical_infections.py b/pyrenew/latent/hierarchical_infections.py new file mode 100644 index 00000000..883e9a08 --- /dev/null +++ b/pyrenew/latent/hierarchical_infections.py @@ -0,0 +1,307 @@ +""" +Hierarchical latent infection process with subpopulation-specific renewal models. +""" + +from __future__ import annotations + +from functools import partial + +import jax +import jax.numpy as jnp +import numpyro +from jax.typing import ArrayLike +from pyrenew.deterministic import DeterministicVariable +from pyrenew.latent.infection_functions import compute_infections_from_rt +from pyrenew.latent.infection_initialization_method import ( + InitializeInfectionsExponentialGrowth, +) +from pyrenew.latent.infection_initialization_process import ( + InfectionInitializationProcess, +) +from pyrenew.math import r_approx_from_R +from pyrenew.metaclass import RandomVariable + +from pyrenew.latent.base import ( + BaseLatentInfectionProcess, + LatentSample, +) +from pyrenew.latent.temporal_processes import TemporalProcess + + +class HierarchicalInfections(BaseLatentInfectionProcess): + """ + Multi-subpopulation renewal model with hierarchical Rt structure. + + Each subpopulation has its own renewal equation with Rt deviating from a + shared baseline. Suitable when transmission dynamics vary substantially + across subpopulations. + + Mathematical form: + - Baseline Rt: log[R_baseline(t)] ~ TemporalProcess + - Subpopulation Rt: log R_k(t) = log[R_baseline(t)] + delta_k(t) + - Deviations: delta_k(t) ~ TemporalProcess with sum-to-zero constraint + - Renewal per subpop: I_k(t) = R_k(t) * sum_tau I_k(t-tau) * g(tau) + - Aggregate total: I_aggregate(t) = sum_k p_k * I_k(t) + + The constructor specifies model structure (priors, temporal processes). + Population structure (fractions, K, K_obs) is provided at sample time, + allowing a single model to be fit to multiple jurisdictions. + + Parameters + ---------- + gen_int_rv : RandomVariable + Generation interval PMF + I0_rv : RandomVariable + Initial infection prevalence (proportion of population) at first + observation time. Must return values in the interval (0, 1). + Returns scalar (same for all subpops) or (K,) array (per-subpop). + Full I0 matrix generated via exponential backprojection during sampling. + initial_log_rt_rv : RandomVariable + Initial value for log(Rt) at time 0. Can be estimated from data + or given a prior distribution. + baseline_temporal : TemporalProcess + Temporal process for baseline Rt dynamics + subpop_temporal : TemporalProcess + Temporal process for subpopulation deviations + n_initialization_points : int, optional + Number of initialization days before day 0. If not specified, defaults to + max(21, 2*len(gen_int)-1). When using ModelBuilder, this is computed + automatically from all observation processes. + + Notes + ----- + Sum-to-zero constraint on deviations ensures R_baseline(t) is the geometric + mean of subpopulation Rt values, providing identifiability. + + When using ModelBuilder (recommended), n_initialization_points is computed + automatically from all observation processes and should not be specified manually. + """ + + def __init__( + self, + *, + gen_int_rv: RandomVariable, + I0_rv: RandomVariable, + baseline_temporal: TemporalProcess, + subpop_temporal: TemporalProcess, + initial_log_rt_rv: RandomVariable, + n_initialization_points: int = None, + ) -> None: + """ + Initialize hierarchical infections process. + + Parameters + ---------- + gen_int_rv : RandomVariable + Generation interval PMF + I0_rv : RandomVariable + Initial infection prevalence (proportion of population) + baseline_temporal : TemporalProcess + Temporal process for baseline Rt dynamics + subpop_temporal : TemporalProcess + Temporal process for subpopulation deviations + initial_log_rt_rv : RandomVariable + Initial value for log(Rt) at time 0. + n_initialization_points : int, optional + Number of initialization days before day 0. + + Raises + ------ + ValueError + If required parameters are missing or invalid + """ + super().__init__( + gen_int_rv=gen_int_rv, + n_initialization_points=n_initialization_points, + ) + + if I0_rv is None: + raise ValueError("I0_rv is required") + self.I0_rv = I0_rv + + # Validate I0 at construction time if it's deterministic + if isinstance(I0_rv, DeterministicVariable): + self._validate_I0(I0_rv.value) + + # Validate initial log Rt + if initial_log_rt_rv is None: + raise ValueError("initial_log_rt_rv is required") + self.initial_log_rt_rv = initial_log_rt_rv + + if baseline_temporal is None: + raise ValueError("baseline_temporal is required") + self.baseline_temporal = baseline_temporal + + if subpop_temporal is None: + raise ValueError("subpop_temporal is required") + self.subpop_temporal = subpop_temporal + + def validate(self) -> None: + """ + Validate hierarchical infections parameters. + + Checks that temporal processes and I0_rv can be sampled. + + Raises + ------ + ValueError + If any parameters fail validation + """ + gen_int_pmf = self.gen_int_rv() + if gen_int_pmf.size == 0: + raise ValueError("gen_int_rv must return non-empty array") + + pmf_sum = jnp.sum(gen_int_pmf) + if not jnp.isclose(pmf_sum, 1.0, atol=1e-6): + raise ValueError(f"gen_int_rv must sum to 1.0, got {float(pmf_sum):.6f}") + + if jnp.any(gen_int_pmf < 0): + raise ValueError("gen_int_rv must have non-negative values") + + def sample( + self, + n_days_post_init: int, + *, + obs_fractions: ArrayLike = None, + unobs_fractions: ArrayLike = None, + **kwargs, + ) -> LatentSample: + """ + Sample hierarchical infections for all subpopulations. + + Generates baseline Rt, subpopulation deviations with sum-to-zero + constraint, initial infections, and runs K independent renewal processes. + + Parameters + ---------- + n_days_post_init : int + Number of days to simulate after initialization period + obs_fractions : ArrayLike + Population fractions for observed subpopulations. + unobs_fractions : ArrayLike + Population fractions for unobserved subpopulations. + **kwargs + Additional arguments (unused, for compatibility) + + Returns + ------- + LatentSample + Named tuple with fields: + - aggregate: shape (n_total_days,) + - all_subpops: shape (n_total_days, K) + - observed: shape (n_total_days, K_obs) + - unobserved: shape (n_total_days, K_unobs) + """ + # Parse and validate population structure + pop = self._parse_and_validate_fractions( + obs_fractions=obs_fractions, + unobs_fractions=unobs_fractions, + ) + + n_total_days = self.n_initialization_points + n_days_post_init + + initial_log_rt = self.initial_log_rt_rv() + + log_rt_baseline = self.baseline_temporal.sample( + n_timepoints=n_total_days, + initial_value=initial_log_rt, + name_prefix="log_rt_baseline", + ) + + deviations_raw = self.subpop_temporal.sample( + n_timepoints=n_total_days, + n_processes=pop.K, + initial_value=jnp.zeros(pop.K), + name_prefix="subpop_deviations", + ) + + # Sum-to-zero constraint ensures identifiability + mean_deviation = jnp.mean(deviations_raw, axis=1, keepdims=True) + deviations = deviations_raw - mean_deviation + + log_rt_subpop = log_rt_baseline[:, jnp.newaxis] + deviations + rt_subpop = jnp.exp(log_rt_subpop) + + gen_int = self.gen_int_rv() + + I0_raw = self.I0_rv() + I0 = jnp.asarray(I0_raw) + + # Validate I0 at runtime (for stochastic RVs) + self._validate_I0(I0) + + if I0.ndim == 0: + I0_subpop = jnp.full(pop.K, I0) + else: + I0_subpop = I0 + + initial_r_subpop = jax.vmap( + partial(r_approx_from_R, g=gen_int, n_newton_steps=4) + )(rt_subpop[0, :]) + + I0_all = jnp.zeros((self.n_initialization_points, pop.K)) + + for k in range(pop.K): + i0_rv_k = DeterministicVariable(f"I0_subpop_{k}", I0_subpop[k]) + r_rv_k = DeterministicVariable(f"initial_r_subpop_{k}", initial_r_subpop[k]) + + init_proc_k = InfectionInitializationProcess( + f"I0_initialization_subpop_{k}", + i0_rv_k, + InitializeInfectionsExponentialGrowth( + self.n_initialization_points, + r_rv_k, + t_pre_init=0, + ), + ) + I0_all = I0_all.at[:, k].set(init_proc_k()) + + gen_int_reversed = jnp.flip(gen_int) + recent_I0_all = I0_all[-gen_int.size :, :] + + all_fractions = jnp.concatenate([pop.obs_fractions, pop.unobs_fractions]) + + post_init_infections_all = jnp.zeros((n_days_post_init, pop.K)) + + for k in range(pop.K): + post_init_k = compute_infections_from_rt( + I0=recent_I0_all[:, k], + Rt=rt_subpop[self.n_initialization_points :, k], + reversed_generation_interval_pmf=gen_int_reversed, + ) + post_init_infections_all = post_init_infections_all.at[:, k].set( + post_init_k + ) + + infections_all = jnp.vstack([I0_all, post_init_infections_all]) + + infections_aggregate = jnp.sum( + infections_all * all_fractions[jnp.newaxis, :], axis=1 + ) + + infections_observed, infections_unobserved = self._split_subpopulations( + infections_all, pop + ) + + self._validate_output_shapes( + infections_aggregate, + infections_all, + infections_observed, + infections_unobserved, + n_total_days, + pop, + ) + + # Record key quantities for diagnostics and posterior analysis + numpyro.deterministic("log_rt_baseline", log_rt_baseline) + numpyro.deterministic("rt_baseline", jnp.exp(log_rt_baseline)) + numpyro.deterministic("rt_subpop", rt_subpop) + numpyro.deterministic("subpop_deviations", deviations) + numpyro.deterministic("infections_aggregate", infections_aggregate) + + return LatentSample( + aggregate=infections_aggregate, + all_subpops=infections_all, + observed=infections_observed, + unobserved=infections_unobserved, + ) diff --git a/pyrenew/latent/hierarchical_priors.py b/pyrenew/latent/hierarchical_priors.py new file mode 100644 index 00000000..eab3dd36 --- /dev/null +++ b/pyrenew/latent/hierarchical_priors.py @@ -0,0 +1,208 @@ +""" +Hierarchical prior distributions for group-level random effects. + +All classes in this module implement the group-level RV interface: +``sample(n_groups, **kwargs) -> ArrayLike`` with shape ``(n_groups,)``. +""" + +import jax.numpy as jnp +import numpyro +import numpyro.distributions as dist +from pyrenew.metaclass import RandomVariable + + +class HierarchicalNormalPrior(RandomVariable): + """ + Zero-centered Normal prior for group-level effects. + + Samples n_groups values from Normal(0, sd). + + Parameters + ---------- + name : str + Unique name for the sampled parameter in numpyro. + sd_rv : RandomVariable + RandomVariable returning the standard deviation. + """ + + def __init__( + self, + name: str, + sd_rv: RandomVariable, + ) -> None: + if not isinstance(sd_rv, RandomVariable): + raise TypeError( + f"sd_rv must be a RandomVariable, got {type(sd_rv).__name__}. " + "Use DeterministicVariable(name, value) to wrap a fixed value." + ) + + self.name = name + self.sd_rv = sd_rv + + def validate(self): + pass + + def sample(self, n_groups: int, **kwargs): + """ + Sample group-level effects. + + Parameters + ---------- + n_groups : int + Number of groups. + + Returns + ------- + ArrayLike + Array of shape (n_groups,). + """ + sd = self.sd_rv() + + with numpyro.plate(f"n_{self.name}", n_groups): + effects = numpyro.sample( + self.name, + dist.Normal(0.0, sd), + ) + return effects + + +class GammaGroupSdPrior(RandomVariable): + """ + Gamma prior for group-level standard deviations, bounded away from zero. + + Samples n_groups positive values from Gamma(concentration, rate) + sd_min. + + Parameters + ---------- + name : str + Unique name for the sampled parameter in numpyro. + sd_mean_rv : RandomVariable + RandomVariable returning the mean of the Gamma distribution. + sd_concentration_rv : RandomVariable + RandomVariable returning the concentration (shape) parameter of Gamma. + sd_min : float, default=0.05 + Minimum SD value (lower bound). + """ + + def __init__( + self, + name: str, + sd_mean_rv: RandomVariable, + sd_concentration_rv: RandomVariable, + sd_min: float = 0.05, + ) -> None: + if not isinstance(sd_mean_rv, RandomVariable): + raise TypeError( + f"sd_mean_rv must be a RandomVariable, got {type(sd_mean_rv).__name__}. " + "Use DeterministicVariable(name, value) to wrap a fixed value." + ) + if not isinstance(sd_concentration_rv, RandomVariable): + raise TypeError( + f"sd_concentration_rv must be a RandomVariable, got {type(sd_concentration_rv).__name__}. " + "Use DeterministicVariable(name, value) to wrap a fixed value." + ) + if sd_min < 0: + raise ValueError(f"sd_min must be non-negative, got {sd_min}") + + self.name = name + self.sd_mean_rv = sd_mean_rv + self.sd_concentration_rv = sd_concentration_rv + self.sd_min = sd_min + + def validate(self): + pass + + def sample(self, n_groups: int, **kwargs): + """ + Sample group-level standard deviations. + + Parameters + ---------- + n_groups : int + Number of groups. + + Returns + ------- + ArrayLike + Array of shape (n_groups,) with values >= sd_min. + """ + sd_mean = self.sd_mean_rv() + concentration = self.sd_concentration_rv() + rate = concentration / sd_mean + + with numpyro.plate(f"n_{self.name}", n_groups): + raw_sd = numpyro.sample( + f"{self.name}_raw", + dist.Gamma(concentration, rate), + ) + + group_sd = numpyro.deterministic( + self.name, + jnp.maximum(raw_sd, self.sd_min), + ) + return group_sd + + +class StudentTGroupModePrior(RandomVariable): + """ + Zero-centered Student-t prior for group-level modes (robust alternative to Normal). + + Samples n_groups values from StudentT(df, 0, sd). + + Parameters + ---------- + name : str + Unique name for the sampled parameter in numpyro. + sd_rv : RandomVariable + RandomVariable returning the scale parameter. + df_rv : RandomVariable + RandomVariable returning the degrees of freedom. + """ + + def __init__( + self, + name: str, + sd_rv: RandomVariable, + df_rv: RandomVariable, + ) -> None: + if not isinstance(sd_rv, RandomVariable): + raise TypeError( + f"sd_rv must be a RandomVariable, got {type(sd_rv).__name__}. " + "Use DeterministicVariable(name, value) to wrap a fixed value." + ) + if not isinstance(df_rv, RandomVariable): + raise TypeError( + f"df_rv must be a RandomVariable, got {type(df_rv).__name__}. " + "Use DeterministicVariable(name, value) to wrap a fixed value." + ) + + self.name = name + self.sd_rv = sd_rv + self.df_rv = df_rv + + def validate(self): + pass + + def sample(self, n_groups: int, **kwargs): + """ + Sample group-level modes. + + Parameters + ---------- + n_groups : int + Number of groups. + + Returns + ------- + ArrayLike + Array of shape (n_groups,). + """ + sd = self.sd_rv() + df = self.df_rv() + + with numpyro.plate(f"n_{self.name}", n_groups): + effects = numpyro.sample( + self.name, + dist.StudentT(df=df, loc=0.0, scale=sd), + ) + return effects diff --git a/pyrenew/latent/partitioned_infections.py b/pyrenew/latent/partitioned_infections.py new file mode 100644 index 00000000..00f37ad7 --- /dev/null +++ b/pyrenew/latent/partitioned_infections.py @@ -0,0 +1,293 @@ +""" +Partitioned latent infection process with single renewal model and allocation. +""" + +from __future__ import annotations + +import jax.numpy as jnp +import numpyro +from jax.typing import ArrayLike +from pyrenew.deterministic import DeterministicVariable +from pyrenew.latent.infection_functions import compute_infections_from_rt +from pyrenew.latent.infection_initialization_method import ( + InitializeInfectionsExponentialGrowth, +) +from pyrenew.latent.infection_initialization_process import ( + InfectionInitializationProcess, +) +from pyrenew.math import r_approx_from_R +from pyrenew.metaclass import RandomVariable + +from pyrenew.latent.base import ( + BaseLatentInfectionProcess, + LatentSample, +) +from pyrenew.latent.temporal_processes import TemporalProcess + + +class PartitionedInfections(BaseLatentInfectionProcess): + """ + Single aggregate renewal model with spatial allocation. + + Computes one renewal equation for total infections, then allocates to + subpopulations via time-varying distribution. More efficient when spatial + variation is in infection distribution rather than transmission dynamics. + + Mathematical form: + - Aggregate Rt: log[R_aggregate(t)] ~ TemporalProcess + - Single renewal: I_total(t) = R_aggregate(t) * sum_tau I_total(t-tau) * g(tau) + - Baseline allocation: pi_k^base = population fractions + - Allocation deviations: delta_k(t) ~ TemporalProcess, delta_1(t) = 0 (reference) + - Softmax allocation: pi_k(t) = (pi_k^base * exp(delta_k)) / sum_j(...) + - Subpop infections: I_k(t) = pi_k(t) * I_total(t) + + The constructor specifies model structure (priors, temporal processes). + Population structure (fractions, K, K_obs) is provided at sample time, + allowing a single model to be fit to multiple jurisdictions. + + Parameters + ---------- + gen_int_rv : RandomVariable + Generation interval PMF + I0_rv : RandomVariable + Initial infection prevalence (proportion of population) at first + observation time. Must return value in the interval (0, 1]. + Returns scalar. Full I0 vector generated via exponential backprojection during sampling. + initial_log_rt_rv : RandomVariable + Initial value for log(Rt) at time 0. Can be estimated from data + or given a prior distribution. + rt_temporal : TemporalProcess + Temporal process for aggregate-level Rt + allocation_temporal : TemporalProcess + Temporal process for allocation deviations (K-1 processes, reference fixed to 0) + n_initialization_points : int, optional + Number of initialization days before day 0. If not specified, defaults to + max(21, 2*len(gen_int)-1). When using ModelBuilder, this is computed + automatically from all observation processes. + + Notes + ----- + Reference category (subpopulation 0) has delta_0(t) = 0 for identifiability. + Choose the largest or most representative subpopulation as index 0. + """ + + def __init__( + self, + *, + gen_int_rv: RandomVariable, + I0_rv: RandomVariable, + rt_temporal: TemporalProcess, + allocation_temporal: TemporalProcess, + initial_log_rt_rv: RandomVariable, + n_initialization_points: int = None, + ) -> None: + """ + Initialize partitioned infections process. + + Parameters + ---------- + gen_int_rv : RandomVariable + Generation interval PMF + I0_rv : RandomVariable + Initial infection prevalence (proportion of population) + rt_temporal : TemporalProcess + Temporal process for aggregate-level Rt + allocation_temporal : TemporalProcess + Temporal process for allocation deviations + initial_log_rt_rv : RandomVariable + Initial value for log(Rt) at time 0 + n_initialization_points : int, optional + Number of initialization days before day 0 + + Raises + ------ + ValueError + If required parameters are missing or invalid + """ + super().__init__( + gen_int_rv=gen_int_rv, + n_initialization_points=n_initialization_points, + ) + + if I0_rv is None: + raise ValueError("I0_rv is required") + self.I0_rv = I0_rv + + # Validate I0 at construction time if it's deterministic + if isinstance(I0_rv, DeterministicVariable): + self._validate_I0(I0_rv.value) + + if initial_log_rt_rv is None: + raise ValueError("initial_log_rt_rv is required") + self.initial_log_rt_rv = initial_log_rt_rv + + if rt_temporal is None: + raise ValueError("rt_temporal is required") + self.rt_temporal = rt_temporal + + if allocation_temporal is None: + raise ValueError("allocation_temporal is required") + self.allocation_temporal = allocation_temporal + + def validate(self) -> None: + """ + Validate partitioned infections parameters. + + Checks that temporal processes and I0_rv can be sampled. + + Raises + ------ + ValueError + If any parameters fail validation + """ + gen_int_pmf = self.gen_int_rv() + if gen_int_pmf.size == 0: + raise ValueError("gen_int_rv must return non-empty array") + + pmf_sum = jnp.sum(gen_int_pmf) + if not jnp.isclose(pmf_sum, 1.0, atol=1e-6): + raise ValueError(f"gen_int_rv must sum to 1.0, got {float(pmf_sum):.6f}") + + if jnp.any(gen_int_pmf < 0): + raise ValueError("gen_int_rv must have non-negative values") + + def sample( + self, + n_days_post_init: int, + *, + obs_fractions: ArrayLike = None, + unobs_fractions: ArrayLike = None, + **kwargs, + ) -> LatentSample: + """ + Sample partitioned infections with single renewal model and allocation. + + Generates aggregate-level Rt, runs one renewal equation for total + infections, then allocates to subpopulations via time-varying softmax. + + Parameters + ---------- + n_days_post_init : int + Number of days to simulate after initialization period + obs_fractions : ArrayLike + Population fractions for observed subpopulations. + unobs_fractions : ArrayLike + Population fractions for unobserved subpopulations. + **kwargs + Additional arguments (unused, for compatibility) + + Returns + ------- + LatentSample + Named tuple with fields: + - aggregate: shape (n_total_days,) - total infections + - all_subpops: shape (n_total_days, K) - allocated to all subpops + - observed: shape (n_total_days, K_obs) + - unobserved: shape (n_total_days, K_unobs) + """ + # Parse and validate population structure + pop = self._parse_and_validate_fractions( + obs_fractions=obs_fractions, + unobs_fractions=unobs_fractions, + ) + + n_total_days = self.n_initialization_points + n_days_post_init + + initial_log_rt = self.initial_log_rt_rv() + + log_rt_aggregate = self.rt_temporal.sample( + n_timepoints=n_total_days, + initial_value=initial_log_rt, + innovation_sd=1.0, + name_prefix="log_rt_aggregate", + ) + rt_aggregate = jnp.exp(log_rt_aggregate) + + # Reference category (subpop 0) fixed to delta_0(t) = 0 for identifiability + if pop.K > 1: + deviations_nonref = self.allocation_temporal.sample( + n_timepoints=n_total_days, + n_processes=pop.K - 1, + initial_value=jnp.zeros(pop.K - 1), + innovation_sd=1.0, + name_prefix="allocation_deviations", + ) + deviations = jnp.concatenate( + [jnp.zeros((n_total_days, 1)), deviations_nonref], axis=1 + ) + else: + deviations = jnp.zeros((n_total_days, 1)) + + baseline_allocation = jnp.concatenate([pop.obs_fractions, pop.unobs_fractions]) + + # Softmax: pi_k(t) = (baseline_k * exp(delta_k(t))) / sum_j(baseline_j * exp(delta_j(t))) + unnormalized = baseline_allocation[jnp.newaxis, :] * jnp.exp(deviations) + allocation_proportions = unnormalized / jnp.sum( + unnormalized, axis=1, keepdims=True + ) + + gen_int = self.gen_int_rv() + I0_total_raw = self.I0_rv() + I0_total = jnp.asarray(I0_total_raw) + + # Validate I0 at runtime (for stochastic RVs) + self._validate_I0(I0_total) + + initial_r = r_approx_from_R(rt_aggregate[0], gen_int, n_newton_steps=4) + + i0_rv = DeterministicVariable("i0_prevalence", I0_total) + r_rv = DeterministicVariable("initial_r", initial_r) + + init_proc = InfectionInitializationProcess( + "I0_initialization", + i0_rv, + InitializeInfectionsExponentialGrowth( + self.n_initialization_points, + r_rv, + t_pre_init=0, + ), + ) + + I0_total = init_proc() + + gen_int_reversed = jnp.flip(gen_int) + recent_I0_total = I0_total[-gen_int.size :] + + post_init_infections_total = compute_infections_from_rt( + I0=recent_I0_total, + Rt=rt_aggregate[self.n_initialization_points :], + reversed_generation_interval_pmf=gen_int_reversed, + ) + + infections_total = jnp.concatenate([I0_total, post_init_infections_total]) + + infections_all = allocation_proportions * infections_total[:, jnp.newaxis] + + infections_aggregate = infections_total + + infections_observed, infections_unobserved = self._split_subpopulations( + infections_all, pop + ) + + self._validate_output_shapes( + infections_aggregate, + infections_all, + infections_observed, + infections_unobserved, + n_total_days, + pop, + ) + + # Record key quantities for diagnostics and posterior analysis + numpyro.deterministic("log_rt_aggregate", log_rt_aggregate) + numpyro.deterministic("rt_aggregate", rt_aggregate) + numpyro.deterministic("allocation_deviations", deviations) + numpyro.deterministic("allocation_proportions", allocation_proportions) + numpyro.deterministic("infections_aggregate", infections_aggregate) + + return LatentSample( + aggregate=infections_aggregate, + all_subpops=infections_all, + observed=infections_observed, + unobserved=infections_unobserved, + ) diff --git a/pyrenew/latent/temporal_processes.py b/pyrenew/latent/temporal_processes.py new file mode 100644 index 00000000..aac818fd --- /dev/null +++ b/pyrenew/latent/temporal_processes.py @@ -0,0 +1,527 @@ +# numpydoc ignore=GL08 +""" +Temporal processes for latent infection models. + +Provides time-series processes for modeling Rt dynamics and subpopulation +deviations. All processes support both single trajectory and vectorized +(multiple parallel trajectories) sampling. + +Temporal Processes +------------------ +- ``AR1``: Autoregressive process with mean reversion. Keeps Rt bounded + near a baseline. +- ``DifferencedAR1``: AR(1) on first differences. Allows persistent trends + while stabilizing the growth rate. +- ``RandomWalk``: No mean reversion. Rt can drift without bound. + +All implementations satisfy the ``TemporalProcess`` protocol. +""" + +from __future__ import annotations + +from typing import Optional, Protocol, Union, runtime_checkable + +import jax.numpy as jnp +import numpyro +import numpyro.distributions as dist +from jax.typing import ArrayLike +from pyrenew.process import ARProcess, DifferencedProcess +from pyrenew.process import RandomWalk as PyRenewRandomWalk +from pyrenew.randomvariable import DistributionalVariable + + +@runtime_checkable +class TemporalProcess(Protocol): + """ + Protocol for temporal processes generating time-varying parameters. + + Used for jurisdiction-level Rt dynamics, subpopulation deviations, or + allocation trajectories. Supports both single trajectory and vectorized + (multiple parallel trajectories) sampling through a unified interface. + """ + + def sample( + self, + n_timepoints: int, + initial_value: Optional[Union[float, ArrayLike]] = None, + innovation_sd: float = 1.0, + n_processes: Optional[int] = None, + name_prefix: str = "temporal", + ) -> ArrayLike: + """ + Sample temporal trajectory or trajectories. + + Parameters + ---------- + n_timepoints : int + Number of time points to generate + initial_value : float or ArrayLike, optional + Initial value(s) for the process(es). + - If n_processes is None: scalar initial value (default: 0.0) + - If n_processes is K: scalar (broadcast) or array of shape (K,) + innovation_sd : float, default 1.0 + Standard deviation of innovations + n_processes : int, optional + Number of parallel processes. If None, samples a single trajectory. + name_prefix : str, default "temporal" + Prefix for numpyro sample site names to avoid collisions + + Returns + ------- + ArrayLike + - If n_processes is None: trajectory of shape (n_timepoints,) + - If n_processes is K: trajectories of shape (n_timepoints, K) + """ + ... + + +class AR1(TemporalProcess): + """ + AR(1) autoregressive process for log(Rt). + + Each value depends on the previous value plus noise, with reversion + toward a mean level. Keeps Rt bounded near a baseline — values that + drift away are "pulled back" over time. + + Parameters + ---------- + autoreg : float + Autoregressive coefficient. Must satisfy |autoreg| < 1 for stationarity. + innovation_sd : float, default 1.0 + Standard deviation of noise at each time step. Larger values produce + more volatile trajectories; smaller values produce smoother ones. + """ + + def __init__(self, autoreg: float, innovation_sd: float = 1.0): + """ + Initialize AR(1) process. + + Parameters + ---------- + autoreg : float + Autoregressive coefficient. Must satisfy |autoreg| < 1. + innovation_sd : float, default 1.0 + Standard deviation of innovations + + Raises + ------ + ValueError + If |autoreg| >= 1 or innovation_sd <= 0 + """ + if abs(autoreg) >= 1.0: + raise ValueError( + f"AR coefficient must satisfy |autoreg| < 1 for stationarity. " + f"Got autoreg={autoreg}" + ) + if innovation_sd <= 0: + raise ValueError(f"innovation_sd must be positive, got {innovation_sd}") + self.autoreg = autoreg + self.innovation_sd = innovation_sd + self.ar_process = ARProcess() + + def __repr__(self) -> str: + """Return string representation.""" + return f"AR1(autoreg={self.autoreg}, innovation_sd={self.innovation_sd})" + + def sample( + self, + n_timepoints: int, + initial_value: Optional[Union[float, ArrayLike]] = None, + innovation_sd: Optional[float] = None, + n_processes: Optional[int] = None, + name_prefix: str = "ar1", + ) -> ArrayLike: + """ + Sample AR(1) trajectory or trajectories. + + Parameters + ---------- + n_timepoints : int + Number of time points to generate + initial_value : float or ArrayLike, optional + Initial value(s). Defaults to 0.0. + innovation_sd : float, optional + Override constructor's innovation_sd if provided. + n_processes : int, optional + Number of parallel processes. If None, samples single trajectory. + name_prefix : str, default "ar1" + Prefix for numpyro sample sites + + Returns + ------- + ArrayLike + Trajectory (n_timepoints,) or trajectories (n_timepoints, K) + """ + if innovation_sd is None: + innovation_sd = self.innovation_sd + + if n_processes is None: + return self._sample_single( + n_timepoints, initial_value, innovation_sd, name_prefix + ) + else: + return self._sample_vectorized( + n_timepoints, n_processes, initial_value, innovation_sd, name_prefix + ) + + def _sample_single( + self, + n_timepoints: int, + initial_value: Optional[float], + innovation_sd: float, + name_prefix: str, + ) -> ArrayLike: + """Sample a single AR(1) trajectory.""" + if initial_value is None: + initial_value = 0.0 + + stationary_sd = innovation_sd / jnp.sqrt(1 - self.autoreg**2) + init_state = numpyro.sample( + f"{name_prefix}_init", dist.Normal(initial_value, stationary_sd) + ) + + trajectory = self.ar_process( + n=n_timepoints, + init_vals=jnp.array([[init_state]]), + autoreg=jnp.array([[self.autoreg]]), + noise_sd=innovation_sd, + noise_name=f"{name_prefix}_noise", + ) + + return jnp.squeeze(trajectory) + + def _sample_vectorized( + self, + n_timepoints: int, + n_processes: int, + initial_values: Optional[ArrayLike], + innovation_sd: float, + name_prefix: str, + ) -> ArrayLike: + """Sample multiple AR(1) trajectories in parallel.""" + if initial_values is None: + initial_values = jnp.zeros(n_processes) + elif jnp.isscalar(initial_values): + initial_values = jnp.full(n_processes, initial_values) + + stationary_sd = innovation_sd / jnp.sqrt(1 - self.autoreg**2) + + with numpyro.plate(f"{name_prefix}_init_plate", n_processes): + init_states = numpyro.sample( + f"{name_prefix}_init", + dist.Normal(initial_values, stationary_sd), + ) + + trajectories = self.ar_process( + n=n_timepoints, + init_vals=init_states[jnp.newaxis, :], + autoreg=jnp.full((1, n_processes), self.autoreg), + noise_sd=innovation_sd, + noise_name=f"{name_prefix}_noise", + ) + + return trajectories + + +class DifferencedAR1(TemporalProcess): + """ + AR(1) process on first differences of log(Rt). + + Each *change* in value depends on the previous change plus noise, with + the rate of change reverting toward a mean. Unlike AR(1), this allows + Rt to trend persistently upward or downward while the growth rate stabilizes. + + Parameters + ---------- + autoreg : float + Autoregressive coefficient for differences. Must satisfy |autoreg| < 1. + innovation_sd : float, default 1.0 + Standard deviation of noise added to changes. Larger values produce + more erratic growth rates; smaller values produce smoother trends. + """ + + def __init__(self, autoreg: float, innovation_sd: float = 1.0): + """ + Initialize differenced AR(1) process. + + Parameters + ---------- + autoreg : float + Autoregressive coefficient for differences. Must satisfy |autoreg| < 1. + innovation_sd : float, default 1.0 + Standard deviation of innovations + + Raises + ------ + ValueError + If |autoreg| >= 1 or innovation_sd <= 0 + """ + if abs(autoreg) >= 1.0: + raise ValueError( + f"AR coefficient must satisfy |autoreg| < 1 for stationarity. " + f"Got autoreg={autoreg}" + ) + if innovation_sd <= 0: + raise ValueError(f"innovation_sd must be positive, got {innovation_sd}") + self.autoreg = autoreg + self.innovation_sd = innovation_sd + self.process = DifferencedProcess( + fundamental_process=ARProcess(), differencing_order=1 + ) + + def __repr__(self) -> str: + """Return string representation.""" + return f"DifferencedAR1(autoreg={self.autoreg}, innovation_sd={self.innovation_sd})" + + def sample( + self, + n_timepoints: int, + initial_value: Optional[Union[float, ArrayLike]] = None, + innovation_sd: Optional[float] = None, + n_processes: Optional[int] = None, + name_prefix: str = "diff_ar1", + ) -> ArrayLike: + """ + Sample differenced AR(1) trajectory or trajectories. + + Parameters + ---------- + n_timepoints : int + Number of time points to generate + initial_value : float or ArrayLike, optional + Initial value(s). Defaults to 0.0. + innovation_sd : float, optional + Override constructor's innovation_sd if provided. + n_processes : int, optional + Number of parallel processes. If None, samples single trajectory. + name_prefix : str, default "diff_ar1" + Prefix for numpyro sample sites + + Returns + ------- + ArrayLike + Trajectory (n_timepoints,) or trajectories (n_timepoints, K) + """ + if innovation_sd is None: + innovation_sd = self.innovation_sd + + if n_processes is None: + return self._sample_single( + n_timepoints, initial_value, innovation_sd, name_prefix + ) + else: + return self._sample_vectorized( + n_timepoints, n_processes, initial_value, innovation_sd, name_prefix + ) + + def _sample_single( + self, + n_timepoints: int, + initial_value: Optional[float], + innovation_sd: float, + name_prefix: str, + ) -> ArrayLike: + """Sample a single differenced AR(1) trajectory.""" + if initial_value is None: + initial_value = 0.0 + + stationary_sd = innovation_sd / jnp.sqrt(1 - self.autoreg**2) + init_rate_of_change = numpyro.sample( + f"{name_prefix}_init_rate", dist.Normal(0, stationary_sd) + ) + + trajectory = self.process( + n=n_timepoints + 1, + init_vals=jnp.array([[initial_value]]), + autoreg=jnp.array([[self.autoreg]]), + noise_sd=jnp.array([[innovation_sd]]), + fundamental_process_init_vals=jnp.array([[init_rate_of_change]]), + noise_name=f"{name_prefix}_noise", + ) + + return jnp.squeeze(trajectory[:n_timepoints]) + + def _sample_vectorized( + self, + n_timepoints: int, + n_processes: int, + initial_values: Optional[ArrayLike], + innovation_sd: float, + name_prefix: str, + ) -> ArrayLike: + """Sample multiple differenced AR(1) trajectories in parallel.""" + if initial_values is None: + initial_values = jnp.zeros(n_processes) + elif jnp.isscalar(initial_values): + initial_values = jnp.full(n_processes, initial_values) + + stationary_sd = innovation_sd / jnp.sqrt(1 - self.autoreg**2) + + with numpyro.plate(f"{name_prefix}_init_rate_plate", n_processes): + init_rates = numpyro.sample( + f"{name_prefix}_init_rate", + dist.Normal(0, stationary_sd), + ) + + trajectories = self.process( + n=n_timepoints + 1, + init_vals=initial_values[jnp.newaxis, :], + autoreg=jnp.full((1, n_processes), self.autoreg), + noise_sd=jnp.full((1, n_processes), innovation_sd), + fundamental_process_init_vals=init_rates[jnp.newaxis, :], + noise_name=f"{name_prefix}_noise", + ) + + return trajectories[:n_timepoints, :] + + +class RandomWalk(TemporalProcess): + """ + Random walk process for log(Rt). + + Each value equals the previous value plus noise, with no reversion + toward a mean. Allows Rt to drift without bound — suitable when you + have no prior expectation that Rt will return to a baseline. + + Parameters + ---------- + innovation_sd : float, default 1.0 + Standard deviation of noise at each time step. Larger values produce + faster drift; smaller values produce more gradual changes. + + Notes + ----- + Unlike AR(1), variance grows over time — the process can wander arbitrarily + far from its starting point. For long time horizons, consider AR(1) if you + want Rt to stay bounded near a baseline. + """ + + def __init__(self, innovation_sd: float = 1.0): + """ + Initialize random walk process. + + Parameters + ---------- + innovation_sd : float, default 1.0 + Standard deviation of innovations + + Raises + ------ + ValueError + If innovation_sd <= 0 + """ + if innovation_sd <= 0: + raise ValueError(f"innovation_sd must be positive, got {innovation_sd}") + self.innovation_sd = innovation_sd + + def __repr__(self) -> str: + """Return string representation.""" + return f"RandomWalk(innovation_sd={self.innovation_sd})" + + def sample( + self, + n_timepoints: int, + initial_value: Optional[Union[float, ArrayLike]] = None, + innovation_sd: Optional[float] = None, + n_processes: Optional[int] = None, + name_prefix: str = "rw", + ) -> ArrayLike: + """ + Sample random walk trajectory or trajectories. + + Parameters + ---------- + n_timepoints : int + Number of time points to generate + initial_value : float or ArrayLike, optional + Initial value(s). Defaults to 0.0. + innovation_sd : float, optional + Override constructor's innovation_sd if provided. + n_processes : int, optional + Number of parallel processes. If None, samples single trajectory. + name_prefix : str, default "rw" + Prefix for numpyro sample sites + + Returns + ------- + ArrayLike + Trajectory (n_timepoints,) or trajectories (n_timepoints, K) + """ + if innovation_sd is None: + innovation_sd = self.innovation_sd + + if n_processes is None: + return self._sample_single( + n_timepoints, initial_value, innovation_sd, name_prefix + ) + else: + return self._sample_vectorized( + n_timepoints, n_processes, initial_value, innovation_sd, name_prefix + ) + + def _sample_single( + self, + n_timepoints: int, + initial_value: Optional[float], + innovation_sd: float, + name_prefix: str, + ) -> ArrayLike: + """Sample a single random walk trajectory using PyRenew's RandomWalk.""" + if initial_value is None: + initial_value = 0.0 + + step_rv = DistributionalVariable( + name=f"{name_prefix}_step", + distribution=dist.Normal(0, innovation_sd), + ) + + rw = PyRenewRandomWalk(step_rv=step_rv) + + trajectory = rw.sample( + init_vals=jnp.array([initial_value]), + n=n_timepoints, + ) + + return jnp.squeeze(trajectory) + + def _sample_vectorized( + self, + n_timepoints: int, + n_processes: int, + initial_values: Optional[ArrayLike], + innovation_sd: float, + name_prefix: str, + ) -> ArrayLike: + """Sample multiple random walk trajectories in parallel.""" + if initial_values is None: + initial_values = jnp.zeros(n_processes) + elif jnp.isscalar(initial_values): + initial_values = jnp.full(n_processes, initial_values) + + # Non-centered parameterization to avoid funnel problems + with numpyro.plate(f"{name_prefix}_time", n_timepoints - 1): + with numpyro.plate(f"{name_prefix}_proc", n_processes): + increments_raw = numpyro.sample( + f"{name_prefix}_increments_raw", + dist.Normal(0, 1), + ) + + increments = numpyro.deterministic( + f"{name_prefix}_increments", + increments_raw * innovation_sd, + ) + + # Transpose: (n_processes, n_timepoints-1) -> (n_timepoints-1, n_processes) + increments = increments.T + + cumulative = jnp.cumsum(increments, axis=0) + + trajectories = jnp.concatenate( + [ + initial_values[jnp.newaxis, :], + initial_values[jnp.newaxis, :] + cumulative, + ], + axis=0, + ) + + return trajectories diff --git a/pyrenew/model/__init__.py b/pyrenew/model/__init__.py index d30e0cc2..9f34582e 100644 --- a/pyrenew/model/__init__.py +++ b/pyrenew/model/__init__.py @@ -1,6 +1,13 @@ # numpydoc ignore=GL08 from pyrenew.model.admissionsmodel import HospitalAdmissionsModel +from pyrenew.model.model_builder import ModelBuilder +from pyrenew.model.multisignal_model import MultiSignalModel from pyrenew.model.rtinfectionsrenewalmodel import RtInfectionsRenewalModel -__all__ = ["RtInfectionsRenewalModel", "HospitalAdmissionsModel"] +__all__ = [ + "RtInfectionsRenewalModel", + "HospitalAdmissionsModel", + "ModelBuilder", + "MultiSignalModel", +] diff --git a/pyrenew/model/model_builder.py b/pyrenew/model/model_builder.py new file mode 100644 index 00000000..8e8e25ee --- /dev/null +++ b/pyrenew/model/model_builder.py @@ -0,0 +1,256 @@ +""" +Model builder for multi-signal renewal models. + +Automatically computes n_initialization_points from observation processes +and constructs properly configured models. +""" + +from __future__ import annotations + +from typing import Any, Dict, Type + +from pyrenew.latent.base import BaseLatentInfectionProcess +from pyrenew.model.multisignal_model import MultiSignalModel +from pyrenew.observation.base import BaseObservationProcess + +# Parameters that should be passed at sample time, not configure time +_SAMPLE_TIME_PARAMS = { + "obs_fractions", + "unobs_fractions", + "K", + "K_obs", + "subpop_fractions", +} + + +class ModelBuilder: + """ + Builder for multi-signal renewal models. + + Automatically computes n_initialization_points from observation + processes and constructs a properly configured model. + + The builder pattern ensures that: + 1. n_initialization_points is computed correctly from all components + 2. Validation happens at build time (fail-fast) + 3. Latent infections are routed to the correct observations + 4. The API is clean and easy to use + + Population structure (obs_fractions, unobs_fractions, K, K_obs, subpop_fractions) + is passed at sample/fit time, not at configure time. This allows a single model + to be fit to multiple jurisdictions with different population structures. + """ + + def __init__(self): + """ + Initialize a new model builder. + + The builder starts empty and is configured by calling: + 1. configure_latent() - set up the latent infection process + 2. add_observation() - add one or more observation processes + 3. build() - construct the final model + """ + self.latent_class: Type[BaseLatentInfectionProcess] | None = None + self.latent_params: Dict[str, Any] = {} + self.observations: Dict[str, BaseObservationProcess] = {} + + def configure_latent( + self, + latent_class: Type[BaseLatentInfectionProcess], + **params, + ) -> "ModelBuilder": + """ + Configure the latent infection process. + + Parameters + ---------- + latent_class : Type[BaseLatentInfectionProcess] + Class to use for latent infections (e.g., HierarchicalInfections, + PartitionedInfections, or a custom implementation) + **params + Parameters for latent class constructor (model structure). + DO NOT include n_initialization_points - it will be computed + automatically from observation processes. + DO NOT include population structure params (obs_fractions, + unobs_fractions, K, K_obs, subpop_fractions) - these are passed + at sample/fit time to allow fitting to multiple jurisdictions. + + Returns + ------- + ModelBuilder + Self, for method chaining + + Raises + ------ + ValueError + If n_initialization_points or population structure params are included + RuntimeError + If latent has already been configured + """ + if self.latent_class is not None: + raise RuntimeError( + "Latent process already configured. Create a new builder for " + "a different configuration." + ) + + if "n_initialization_points" in params: + raise ValueError( + "Do not specify n_initialization_points - it will be computed " + "automatically from observation processes and generation interval. " + "Use ModelBuilder.build() to create the model with the correct value." + ) + + # Check for population structure params that should be at sample time + sample_time_found = _SAMPLE_TIME_PARAMS.intersection(params.keys()) + if sample_time_found: + raise ValueError( + f"Do not specify {sample_time_found} at configure time. " + f"Population structure is passed at sample/fit time to allow " + f"fitting the same model to multiple jurisdictions." + ) + + self.latent_class = latent_class + self.latent_params = params + return self + + def add_observation( + self, + obs_process: BaseObservationProcess, + name: str, + ) -> "ModelBuilder": + """ + Add an observation process to the model. + + Parameters + ---------- + obs_process : BaseObservationProcess + Configured observation process instance (e.g., CountObservation, + WastewaterObservation, CountObservationBySubpop) + name : str + Unique identifier for this observation. This name is used when passing + observation data to model.sample(). + + Returns + ------- + ModelBuilder + Self, for method chaining + + Raises + ------ + ValueError + If an observation with this name already exists + """ + if name in self.observations: + raise ValueError( + f"Observation '{name}' already added. " + f"Each observation must have a unique name." + ) + + self.observations[name] = obs_process + return self + + def compute_n_initialization_points(self) -> int: + """ + Compute required n_initialization_points from all components. + + Formula: n_initialization_points = max(all lookbacks) + + Where lookbacks include: + - Generation interval length (from latent process) + - All observation delay/shedding PMF lengths + + Useful for inspection before building the model. + + Returns + ------- + int + Minimum n_initialization_points needed to satisfy all components + + Raises + ------ + ValueError + If latent process not configured or gen_int_rv missing + If any observation process doesn't implement get_required_lookback() + """ + if self.latent_class is None: + raise ValueError( + "Must call configure_latent() before computing n_initialization_points" + ) + + # Get generation interval length from latent params + gen_int_rv = self.latent_params.get("gen_int_rv") + if gen_int_rv is None: + raise ValueError("gen_int_rv is required in latent process parameters") + + # Start with generation interval lookback + lookbacks = [len(gen_int_rv())] + + # Add lookback from each observation process + for name, obs_process in self.observations.items(): + try: + lookback = obs_process.lookback_days() + lookbacks.append(lookback) + except (NotImplementedError, AttributeError) as e: + raise ValueError( + f"Observation process '{name}' must implement " + f"lookback_days(). Error: {e}" + ) + + # Formula: max(all lookbacks) + # For generation interval (1-indexed): L-element PMF has max lag L days → need L init points + # For delay distributions (0-indexed): L-element PMF has max delay L-1 days + # We need at least max(all PMF lengths) to satisfy the renewal equation extraction + n_init = max(lookbacks) + + return n_init + + def build(self): + """ + Build the multi-signal model with computed n_initialization_points. + + This method: + 1. Computes n_initialization_points from all components + 2. Constructs the latent process with the computed value + 3. Creates a MultiSignalModel with automatic infection routing + 4. Validates that observation/latent types are compatible + + Can be called multiple times to create multiple model instances. + + Returns + ------- + MultiSignalModel + Configured model ready for sampling + + Raises + ------ + ValueError + If latent process not configured + TypeError + If latent process constructor fails + """ + if self.latent_class is None: + raise ValueError("Must call configure_latent() before build()") + + # Compute n_initialization_points + n_init = self.compute_n_initialization_points() + + # Construct latent process with computed n_initialization_points + latent_params = self.latent_params.copy() + latent_params["n_initialization_points"] = n_init + + try: + latent_process = self.latent_class(**latent_params) + except TypeError as e: + raise TypeError( + f"Error constructing {self.latent_class.__name__} with " + f"computed n_initialization_points={n_init}. " + f"Original error: {e}" + ) + + # Build model + model = MultiSignalModel( + latent_process=latent_process, + observations=self.observations, + ) + + return model diff --git a/pyrenew/model/multisignal_model.py b/pyrenew/model/multisignal_model.py new file mode 100644 index 00000000..d1989630 --- /dev/null +++ b/pyrenew/model/multisignal_model.py @@ -0,0 +1,370 @@ +""" +Multi-signal renewal model. + +Combines a latent infection process with multiple observation processes. +""" + +from __future__ import annotations + +from typing import Any, Dict, Optional + +import jax.numpy as jnp +import jax.random as random +import numpyro +import numpyro.handlers +from numpyro.infer import MCMC, NUTS +from pyrenew.metaclass import Model + +from pyrenew.latent.base import BaseLatentInfectionProcess +from pyrenew.observation.base import BaseObservationProcess + + +class MultiSignalModel(Model): + """ + Multi-signal renewal model. + + Combines a latent infection process (e.g., HierarchicalInfections, + PartitionedInfections) with multiple observation processes (e.g., + CountObservation, WastewaterObservation). + + Built via ModelBuilder to ensure n_initialization_points is computed + correctly from all components. Can also be constructed manually for + advanced use cases. + + Parameters + ---------- + latent_process : BaseLatentInfectionProcess + Latent infection process generating infections at jurisdiction and/or + subpopulation levels + observations : Dict[str, BaseObservationProcess] + Dictionary mapping names to observation process instances. Names are + used when passing observation data to sample(). + + Notes + ----- + The model automatically routes latent infections to observations based on + each observation's infection_resolution() method: + - "aggregate" → receives aggregate infections from latent process + - "subpop" → receives observed subpopulation infections from latent process + """ + + def __init__( + self, + latent_process: BaseLatentInfectionProcess, + observations: Dict[str, BaseObservationProcess], + ): + """ + Initialize multi-signal model. + + Parameters + ---------- + latent_process + Configured latent infection process + observations + Dictionary mapping observation names to observation process instances + + Raises + ------ + ValueError + If validation fails (e.g., observation requires subpopulations + but latent process doesn't support them) + """ + self.latent = latent_process + self.observations = observations + self._validate() + + def _validate(self) -> None: + """ + Validate that observation processes are compatible with latent process. + + Checks that each observation implements infection_resolution(). + + Raises + ------ + ValueError + If an observation doesn't implement infection_resolution() + """ + for name, obs in self.observations.items(): + try: + obs.infection_resolution() + except (NotImplementedError, AttributeError): + raise ValueError( + f"Observation '{name}' must implement infection_resolution()" + ) + + def validate(self) -> None: + """ + Validate the model configuration. + + This method is required by PyRenew's Model base class. + Validation is performed in __init__ via _validate(). + + Raises + ------ + ValueError + If validation fails + """ + self._validate() + + def validate_data( + self, + n_days_post_init: int, + obs_fractions=None, + unobs_fractions=None, + **observation_data: Dict[str, Any], + ) -> None: + """ + Validate observation data before running MCMC. + + This method should be called with concrete (non-traced) values + before running inference. It performs validation that cannot be + done during JAX tracing. + + Parameters + ---------- + n_days_post_init : int + Number of days to simulate after initialization period + obs_fractions : ArrayLike + Population fractions for observed subpopulations + unobs_fractions : ArrayLike + Population fractions for unobserved subpopulations + **observation_data + Data for each observation process, keyed by observation name. + Each value should be a dict of kwargs for that observation's sample(). + + Raises + ------ + ValueError + If times indices are out of bounds or negative + If data shapes are inconsistent + """ + from pyrenew.latent.base import BaseLatentInfectionProcess + + # Parse population structure to get K_obs for validation + pop = BaseLatentInfectionProcess._parse_and_validate_fractions( + obs_fractions=obs_fractions, + unobs_fractions=unobs_fractions, + ) + + n_total_days = self.latent.n_initialization_points + n_days_post_init + + for name, obs_data in observation_data.items(): + if name not in self.observations: + raise ValueError( + f"Unknown observation '{name}'. " + f"Available: {list(self.observations.keys())}" + ) + + # Validate times if present (sparse observations) + times = obs_data.get("times") + if times is not None: + times = jnp.asarray(times) + if jnp.any(times < 0): + raise ValueError( + f"Observation '{name}': times cannot contain negative values" + ) + max_time = jnp.max(times) + if max_time >= n_total_days: + raise ValueError( + f"Observation '{name}': times contains {int(max_time)} " + f">= {n_total_days} (n_total_days = " + f"{self.latent.n_initialization_points} init + " + f"{n_days_post_init} post-init)" + ) + + # Validate obs matches times length + obs = obs_data.get("obs") + if obs is not None and len(obs) != len(times): + raise ValueError( + f"Observation '{name}': obs length {len(obs)} " + f"must match times length {len(times)}" + ) + + # Validate subpop_indices if present (site-level observations) + subpop_indices = obs_data.get("subpop_indices") + if subpop_indices is not None: + subpop_indices = jnp.asarray(subpop_indices) + if jnp.any(subpop_indices < 0): + raise ValueError( + f"Observation '{name}': subpop_indices cannot be negative" + ) + max_idx = jnp.max(subpop_indices) + if max_idx >= pop.K_obs: + raise ValueError( + f"Observation '{name}': subpop_indices contains " + f"{int(max_idx)} >= {pop.K_obs} (K_obs)" + ) + + def sample( + self, + n_days_post_init: int, + population_size: float, + *, + obs_fractions=None, + unobs_fractions=None, + **observation_data, + ): + """ + Sample from the joint generative model. + + This is the model function called by NumPyro during inference. + + Parameters + ---------- + n_days_post_init : int + Number of days to simulate after initialization period + population_size : float + Total population size. Used to convert infection proportions + (from latent process) to infection counts (for observation processes). + obs_fractions : ArrayLike + Population fractions for observed subpopulations. + unobs_fractions : ArrayLike + Population fractions for unobserved subpopulations. + **observation_data + Data for each observation process, keyed by observation name + (the names provided to builder.add_observation()). + Each value should be a dict of kwargs for that observation's sample(). + + Returns + ------- + tuple + Four arrays of infection counts (scaled by population_size): + - inf_aggregate: shape (n_total_days,) - aggregate infections + - inf_all: shape (n_total_days, K) - all subpopulation infections + - inf_obs: shape (n_total_days, K_obs) - observed subpopulation infections + - inf_unobs: shape (n_total_days, K_unobs) - unobserved subpopulation infections + + where n_total_days = n_initialization_points + n_days_post_init + """ + # Generate latent infections (proportions) + latent_sample = self.latent.sample( + n_days_post_init=n_days_post_init, + obs_fractions=obs_fractions, + unobs_fractions=unobs_fractions, + ) + + # Scale from proportions to counts + inf_aggregate = latent_sample.aggregate * population_size + inf_all = latent_sample.all_subpops * population_size + inf_obs = latent_sample.observed * population_size + inf_unobs = latent_sample.unobserved * population_size + + # Record scaled infections for posterior analysis + numpyro.deterministic("latent_infections", inf_aggregate) + numpyro.deterministic("latent_infections_by_subpop", inf_all) + + # Map infection resolution to infection arrays + latent_map = { + "aggregate": inf_aggregate, + "subpop": inf_obs, + } + + # Apply each observation process + for name, obs_process in self.observations.items(): + # Get the appropriate latent infections based on observation type + resolution = obs_process.infection_resolution() + latent_infections = latent_map[resolution] + + # Get observation-specific data + obs_data = observation_data.get(name, {}) + + # Sample from observation process + obs_process.sample( + infections=latent_infections, + **obs_data, + ) + + # Return scaled infection counts + return inf_aggregate, inf_all, inf_obs, inf_unobs + + def fit( + self, + n_days_post_init: int, + population_size: float, + *, + obs_fractions=None, + unobs_fractions=None, + num_warmup: int = 500, + num_samples: int = 500, + num_chains: int = 1, + rng_key: random.PRNGKey = None, + reparam_config: Optional[Dict[str, Any]] = None, + progress_bar: bool = True, + **observation_data, + ): + """ + Fit the model to observed data via MCMC. + + Validates observation data, runs NUTS sampler, and returns + posterior samples. + + Parameters + ---------- + n_days_post_init : int + Number of days to simulate after initialization period + population_size : float + Total population size for scaling infections + obs_fractions : ArrayLike + Population fractions for observed subpopulations. + unobs_fractions : ArrayLike + Population fractions for unobserved subpopulations. + num_warmup : int + Number of MCMC warmup iterations (default: 500) + num_samples : int + Number of MCMC samples to draw (default: 500) + num_chains : int + Number of MCMC chains (default: 1) + rng_key : jax.random.PRNGKey, optional + Random key for reproducibility. Defaults to PRNGKey(0). + reparam_config : dict, optional + Reparameterization configuration for numpyro.handlers.reparam. + Maps sample site names to reparameterizers (e.g., LocScaleReparam). + Use non-centered parameterization (centered=0) for sparse data, + centered (centered=1) for abundant data. Default None (no reparam). + progress_bar : bool + Whether to show progress bar during MCMC (default: True). + Set to False if experiencing tqdm widget errors in Jupyter notebooks + with parallel chains. + **observation_data + Data for each observation process, keyed by observation name. + Each value should be a dict with 'counts', 'times', etc. + + Returns + ------- + numpyro.infer.MCMC + The MCMC object after sampling. Use `mcmc.get_samples()` to get + posterior samples as a dict, or access other MCMC diagnostics. + """ + self.validate_data( + n_days_post_init=n_days_post_init, + obs_fractions=obs_fractions, + unobs_fractions=unobs_fractions, + **observation_data, + ) + + if rng_key is None: + rng_key = random.PRNGKey(0) + + # Apply reparameterization if config provided + sample_fn = self.sample + if reparam_config is not None: + sample_fn = numpyro.handlers.reparam(self.sample, config=reparam_config) + + kernel = NUTS(sample_fn) + mcmc = MCMC( + kernel, + num_warmup=num_warmup, + num_samples=num_samples, + num_chains=num_chains, + progress_bar=progress_bar, + ) + mcmc.run( + rng_key, + n_days_post_init=n_days_post_init, + population_size=population_size, + obs_fractions=obs_fractions, + unobs_fractions=unobs_fractions, + **observation_data, + ) + + return mcmc diff --git a/test/test_base_latent_process.py b/test/test_base_latent_process.py new file mode 100644 index 00000000..90e1c42c --- /dev/null +++ b/test/test_base_latent_process.py @@ -0,0 +1,59 @@ +""" +Unit tests for BaseLatentInfectionProcess. +""" + +import jax.numpy as jnp +import pytest + +from pyrenew.latent.base import BaseLatentInfectionProcess + + +class TestPopulationStructureParsing: + """Test _parse_and_validate_fractions static method.""" + + def test_parse_obs_unobs_fractions(self): + """Test parsing obs_fractions + unobs_fractions.""" + pop = BaseLatentInfectionProcess._parse_and_validate_fractions( + obs_fractions=jnp.array([0.3, 0.25]), + unobs_fractions=jnp.array([0.45]), + ) + + assert pop.K == 3 + assert pop.K_obs == 2 + assert pop.K_unobs == 1 + + def test_rejects_fractions_not_summing_to_one(self): + """Test that fractions not summing to 1 raises error.""" + with pytest.raises(ValueError, match="must sum to 1.0"): + BaseLatentInfectionProcess._parse_and_validate_fractions( + obs_fractions=jnp.array([0.3, 0.25]), + unobs_fractions=jnp.array([0.40]), + ) + + def test_rejects_negative_fractions(self): + """Test that negative fractions raise error.""" + with pytest.raises(ValueError, match="must be non-negative"): + BaseLatentInfectionProcess._parse_and_validate_fractions( + obs_fractions=jnp.array([0.3, -0.1]), + unobs_fractions=jnp.array([0.8]), + ) + + def test_rejects_missing_fractions(self): + """Test that missing fractions raise error.""" + with pytest.raises( + ValueError, match="Both obs_fractions and unobs_fractions must be provided" + ): + BaseLatentInfectionProcess._parse_and_validate_fractions( + obs_fractions=jnp.array([0.3, 0.25]), + ) + + with pytest.raises( + ValueError, match="Both obs_fractions and unobs_fractions must be provided" + ): + BaseLatentInfectionProcess._parse_and_validate_fractions( + unobs_fractions=jnp.array([0.45]), + ) + + +if __name__ == "__main__": + pytest.main([__file__, "-v"]) diff --git a/test/test_hierarchical_infections.py b/test/test_hierarchical_infections.py new file mode 100644 index 00000000..60dd26be --- /dev/null +++ b/test/test_hierarchical_infections.py @@ -0,0 +1,130 @@ +""" +Unit tests for HierarchicalInfections. +""" + +import jax.numpy as jnp +import numpyro +import pytest +from pyrenew.deterministic import DeterministicPMF, DeterministicVariable + +from pyrenew.latent import HierarchicalInfections, RandomWalk + + +@pytest.fixture +def gen_int_rv(): + return DeterministicPMF("gen_int", jnp.array([0.2, 0.5, 0.3])) + + +@pytest.fixture +def process(gen_int_rv): + """Create a HierarchicalInfections with the new keyword-only API.""" + return HierarchicalInfections( + gen_int_rv=gen_int_rv, + I0_rv=DeterministicVariable("I0", 0.001), + initial_log_rt_rv=DeterministicVariable("initial_log_rt", 0.0), + baseline_temporal=RandomWalk(), + subpop_temporal=RandomWalk(), + ) + + +class TestHierarchicalInfectionsSample: + """Test sample method with population structure at sample time.""" + + def test_sample_returns_correct_shapes(self, process): + """Test that sample returns correct output shapes.""" + with numpyro.handlers.seed(rng_seed=42): + result = process.sample( + n_days_post_init=30, + obs_fractions=jnp.array([0.3, 0.25]), + unobs_fractions=jnp.array([0.45]), + ) + + inf_juris, inf_all, inf_obs, inf_unobs = result + n_total = process.n_initialization_points + 30 + + assert inf_juris.shape == (n_total,) + assert inf_all.shape == (n_total, 3) + assert inf_obs.shape == (n_total, 2) + assert inf_unobs.shape == (n_total, 1) + + def test_same_model_different_jurisdictions(self, process): + """Test that one model can fit different population structures.""" + # Jurisdiction A: 2 observed, 1 unobserved + with numpyro.handlers.seed(rng_seed=42): + _, inf_all_a, _, _ = process.sample( + n_days_post_init=30, + obs_fractions=jnp.array([0.3, 0.25]), + unobs_fractions=jnp.array([0.45]), + ) + + # Jurisdiction B: 4 observed, 1 unobserved (different structure) + with numpyro.handlers.seed(rng_seed=42): + _, inf_all_b, _, _ = process.sample( + n_days_post_init=30, + obs_fractions=jnp.array([0.15, 0.20, 0.25, 0.10]), + unobs_fractions=jnp.array([0.30]), + ) + + assert inf_all_a.shape[1] == 3 # K=3 + assert inf_all_b.shape[1] == 5 # K=5 + + def test_jurisdiction_total_is_weighted_sum(self, process): + """Test that jurisdiction total equals weighted sum of subpopulations.""" + obs_fracs = jnp.array([0.3, 0.25]) + unobs_fracs = jnp.array([0.45]) + + with numpyro.handlers.seed(rng_seed=42): + inf_juris, inf_all, _, _ = process.sample( + n_days_post_init=30, + obs_fractions=obs_fracs, + unobs_fractions=unobs_fracs, + ) + + all_fractions = jnp.concatenate([obs_fracs, unobs_fracs]) + expected = jnp.sum(inf_all * all_fractions[jnp.newaxis, :], axis=1) + + assert jnp.allclose(inf_juris, expected, atol=1e-6) + + def test_deviations_sum_to_zero(self, process): + """Test that subpopulation deviations sum to zero (identifiability).""" + with numpyro.handlers.seed(rng_seed=42): + with numpyro.handlers.trace() as trace: + process.sample( + n_days_post_init=30, + obs_fractions=jnp.array([0.3, 0.25]), + unobs_fractions=jnp.array([0.45]), + ) + + deviations = trace["subpop_deviations"]["value"] + deviation_sums = jnp.sum(deviations, axis=1) + + assert jnp.allclose(deviation_sums, 0.0, atol=1e-6) + + +class TestHierarchicalInfectionsValidation: + """Test validation of inputs.""" + + def test_rejects_invalid_I0(self, gen_int_rv): + """Test that invalid I0 values are rejected.""" + with pytest.raises(ValueError, match="I0 must be positive"): + HierarchicalInfections( + gen_int_rv=gen_int_rv, + I0_rv=DeterministicVariable("I0", -0.1), + initial_log_rt_rv=DeterministicVariable("initial_log_rt", 0.0), + baseline_temporal=RandomWalk(), + subpop_temporal=RandomWalk(), + ) + + def test_rejects_fractions_not_summing_to_one(self, process): + """Test that fractions not summing to 1 raises error at sample time.""" + with pytest.raises(ValueError, match="must sum to 1.0"): + with numpyro.handlers.seed(rng_seed=42): + process.sample( + n_days_post_init=30, + obs_fractions=jnp.array([0.3, 0.25]), + unobs_fractions=jnp.array([0.40]), # Sum is 0.95 + ) + + +if __name__ == "__main__": + pytest.main([__file__, "-v"]) diff --git a/test/test_hierarchical_priors.py b/test/test_hierarchical_priors.py new file mode 100644 index 00000000..af1f508b --- /dev/null +++ b/test/test_hierarchical_priors.py @@ -0,0 +1,179 @@ +"""Unit tests for hierarchical prior distributions.""" + +import jax.numpy as jnp +import numpyro +import numpyro.distributions as dist +import pytest +from pyrenew.deterministic import DeterministicVariable +from pyrenew.randomvariable import DistributionalVariable + +from pyrenew.latent import ( + GammaGroupSdPrior, + HierarchicalNormalPrior, + StudentTGroupModePrior, +) + + +class TestHierarchicalNormalPrior: + """Test HierarchicalNormalPrior.""" + + def test_sample_shape(self): + """Test that sample returns correct shape.""" + prior = HierarchicalNormalPrior( + "effect", sd_rv=DeterministicVariable("sd", 1.0) + ) + + with numpyro.handlers.seed(rng_seed=42): + samples = prior.sample(n_groups=5) + + assert samples.shape == (5,) + + def test_smaller_sd_produces_tighter_distribution(self): + """Test that smaller sd produces samples closer to zero.""" + prior_tight = HierarchicalNormalPrior( + "a", sd_rv=DeterministicVariable("sd_tight", 0.1) + ) + prior_wide = HierarchicalNormalPrior( + "b", sd_rv=DeterministicVariable("sd_wide", 10.0) + ) + + n_samples = 1000 + with numpyro.handlers.seed(rng_seed=42): + samples_tight = prior_tight.sample(n_groups=n_samples) + with numpyro.handlers.seed(rng_seed=43): + samples_wide = prior_wide.sample(n_groups=n_samples) + + # Tight prior should have smaller standard deviation + assert jnp.std(samples_tight) < jnp.std(samples_wide) + + def test_rejects_non_random_variable_sd(self): + """Test that non-RandomVariable sd_rv is rejected.""" + with pytest.raises(TypeError, match="sd_rv must be a RandomVariable"): + HierarchicalNormalPrior("effect", sd_rv=1.0) + + def test_accepts_distributional_variable_for_sd(self): + """Test that DistributionalVariable can be used for sd_rv.""" + sd_rv = DistributionalVariable("sd", dist.HalfNormal(1.0)) + prior = HierarchicalNormalPrior("effect", sd_rv=sd_rv) + + with numpyro.handlers.seed(rng_seed=42): + samples = prior.sample(n_groups=5) + + assert samples.shape == (5,) + + +class TestGammaGroupSdPrior: + """Test GammaGroupSdPrior.""" + + def test_sample_shape(self): + """Test that sample returns correct shape.""" + prior = GammaGroupSdPrior( + "sd", + sd_mean_rv=DeterministicVariable("sd_mean", 0.5), + sd_concentration_rv=DeterministicVariable("sd_conc", 4.0), + ) + + with numpyro.handlers.seed(rng_seed=42): + samples = prior.sample(n_groups=5) + + assert samples.shape == (5,) + + def test_respects_sd_min(self): + """Test that sd_min is enforced as lower bound.""" + prior = GammaGroupSdPrior( + "sd", + sd_mean_rv=DeterministicVariable("sd_mean", 0.1), + sd_concentration_rv=DeterministicVariable("sd_conc", 4.0), + sd_min=0.5, + ) + + with numpyro.handlers.seed(rng_seed=42): + samples = prior.sample(n_groups=100) + + assert jnp.all(samples >= 0.5) + + def test_rejects_non_random_variable_params(self): + """Test that non-RandomVariable parameters are rejected.""" + with pytest.raises(TypeError, match="sd_mean_rv must be a RandomVariable"): + GammaGroupSdPrior( + "sd", + sd_mean_rv=0.5, + sd_concentration_rv=DeterministicVariable("sd_conc", 4.0), + ) + + with pytest.raises( + TypeError, match="sd_concentration_rv must be a RandomVariable" + ): + GammaGroupSdPrior( + "sd", + sd_mean_rv=DeterministicVariable("sd_mean", 0.5), + sd_concentration_rv=4.0, + ) + + def test_rejects_negative_sd_min(self): + """Test that negative sd_min is rejected.""" + with pytest.raises(ValueError, match="sd_min must be non-negative"): + GammaGroupSdPrior( + "sd", + sd_mean_rv=DeterministicVariable("sd_mean", 0.5), + sd_concentration_rv=DeterministicVariable("sd_conc", 4.0), + sd_min=-0.1, + ) + + +class TestStudentTGroupModePrior: + """Test StudentTGroupModePrior.""" + + def test_sample_shape(self): + """Test that sample returns correct shape.""" + prior = StudentTGroupModePrior( + "mode", + sd_rv=DeterministicVariable("sd", 1.0), + df_rv=DeterministicVariable("df", 4.0), + ) + + with numpyro.handlers.seed(rng_seed=42): + samples = prior.sample(n_groups=5) + + assert samples.shape == (5,) + + def test_heavier_tails_than_normal(self): + """Test Student-t produces more extreme values than Normal.""" + # df=2 gives very heavy tails + student_prior = StudentTGroupModePrior( + "s", + sd_rv=DeterministicVariable("sd_s", 1.0), + df_rv=DeterministicVariable("df", 2.0), + ) + normal_prior = HierarchicalNormalPrior( + "n", sd_rv=DeterministicVariable("sd_n", 1.0) + ) + + n_samples = 5000 + with numpyro.handlers.seed(rng_seed=42): + student_samples = student_prior.sample(n_groups=n_samples) + with numpyro.handlers.seed(rng_seed=42): + normal_samples = normal_prior.sample(n_groups=n_samples) + + # Student-t should have more extreme values (higher max absolute value) + assert jnp.max(jnp.abs(student_samples)) > jnp.max(jnp.abs(normal_samples)) + + def test_rejects_non_random_variable_params(self): + """Test that non-RandomVariable parameters are rejected.""" + with pytest.raises(TypeError, match="sd_rv must be a RandomVariable"): + StudentTGroupModePrior( + "mode", + sd_rv=1.0, + df_rv=DeterministicVariable("df", 4.0), + ) + + with pytest.raises(TypeError, match="df_rv must be a RandomVariable"): + StudentTGroupModePrior( + "mode", + sd_rv=DeterministicVariable("sd", 1.0), + df_rv=4.0, + ) + + +if __name__ == "__main__": + pytest.main([__file__, "-v"]) diff --git a/test/test_model_builder.py b/test/test_model_builder.py new file mode 100644 index 00000000..3b737aab --- /dev/null +++ b/test/test_model_builder.py @@ -0,0 +1,152 @@ +""" +Tests for ModelBuilder and MultiSignalModel. +""" + +import jax.numpy as jnp +import numpyro +import pytest +from pyrenew.deterministic import DeterministicPMF, DeterministicVariable + +from pyrenew.latent import HierarchicalInfections, RandomWalk +from pyrenew.model import ModelBuilder, MultiSignalModel +from pyrenew.observation import Counts, NegativeBinomialNoise + +# Standard population structure for tests +OBS_FRACTIONS = jnp.array([0.3, 0.25]) +UNOBS_FRACTIONS = jnp.array([0.45]) + + +@pytest.fixture +def simple_builder(): + """Create a configured builder (no population structure at configure time).""" + builder = ModelBuilder() + gen_int = DeterministicPMF("gen_int", jnp.array([0.2, 0.5, 0.3])) + + builder.configure_latent( + HierarchicalInfections, + gen_int_rv=gen_int, + I0_rv=DeterministicVariable("I0", 0.001), + initial_log_rt_rv=DeterministicVariable("initial_log_rt", 0.0), + baseline_temporal=RandomWalk(), + subpop_temporal=RandomWalk(), + ) + + delay = DeterministicPMF("delay", jnp.array([0.1, 0.3, 0.4, 0.2])) + obs = Counts( + ascertainment_rate_rv=DeterministicVariable("ihr", 0.01), + delay_distribution_rv=delay, + noise=NegativeBinomialNoise(DeterministicVariable("conc", 10.0)), + ) + builder.add_observation(obs, "hospital") + + return builder + + +class TestModelBuilderConfiguration: + """Test ModelBuilder configuration.""" + + def test_rejects_population_structure_at_configure_time(self): + """Test that population structure params are rejected at configure time.""" + builder = ModelBuilder() + gen_int = DeterministicPMF("gen_int", jnp.array([0.2, 0.5, 0.3])) + + with pytest.raises(ValueError, match="Do not specify"): + builder.configure_latent( + HierarchicalInfections, + gen_int_rv=gen_int, + I0_rv=DeterministicVariable("I0", 0.001), + initial_log_rt_rv=DeterministicVariable("initial_log_rt", 0.0), + baseline_temporal=RandomWalk(), + subpop_temporal=RandomWalk(), + obs_fractions=jnp.array([0.5, 0.5]), # Should fail + ) + + def test_build_creates_model(self, simple_builder): + """Test that build() creates a MultiSignalModel.""" + model = simple_builder.build() + assert isinstance(model, MultiSignalModel) + + +class TestMultiSignalModelSampling: + """Test MultiSignalModel sampling with population structure at sample time.""" + + def test_sample_with_population_structure(self, simple_builder): + """Test that sample() works with population structure at sample time.""" + model = simple_builder.build() + n_days = 30 + n_total = model.latent.n_initialization_points + n_days + + with numpyro.handlers.seed(rng_seed=42): + inf_aggregate, inf_all, inf_obs, inf_unobs = model.sample( + n_days_post_init=n_days, + population_size=1_000_000, + obs_fractions=OBS_FRACTIONS, + unobs_fractions=UNOBS_FRACTIONS, + hospital={"obs": None}, + ) + + assert inf_aggregate.shape == (n_total,) + assert inf_all.shape == (n_total, 3) # K=3 + assert inf_obs.shape == (n_total, 2) # K_obs=2 + + def test_fit_with_population_structure(self, simple_builder): + """Test that fit() works with population structure at sample time.""" + model = simple_builder.build() + n_days = 10 + n_total = model.latent.n_initialization_points + n_days + + times = jnp.array([5, 6, 7, 8, 9]) + obs = jnp.array([10, 12, 15, 14, 11]) + + mcmc = model.fit( + n_days_post_init=n_days, + population_size=1_000_000, + obs_fractions=OBS_FRACTIONS, + unobs_fractions=UNOBS_FRACTIONS, + num_warmup=5, + num_samples=5, + hospital={"obs": obs, "times": times}, + ) + + samples = mcmc.get_samples() + assert "latent_infections" in samples + assert samples["latent_infections"].shape == (5, n_total) + + +class TestMultiSignalModelValidation: + """Test data validation.""" + + def test_validate_data_requires_population_structure(self, simple_builder): + """Test that validate_data requires population structure.""" + model = simple_builder.build() + + # Should work with population structure + model.validate_data( + n_days_post_init=30, + obs_fractions=OBS_FRACTIONS, + unobs_fractions=UNOBS_FRACTIONS, + hospital={ + "obs": jnp.array([10, 20]), + "times": jnp.array([5, 10]), + }, + ) + + def test_validate_data_rejects_out_of_bounds_times(self, simple_builder): + """Test that times exceeding n_total_days raises error.""" + model = simple_builder.build() + n_total = model.latent.n_initialization_points + 30 + + with pytest.raises(ValueError, match="times contains"): + model.validate_data( + n_days_post_init=30, + obs_fractions=OBS_FRACTIONS, + unobs_fractions=UNOBS_FRACTIONS, + hospital={ + "obs": jnp.array([10]), + "times": jnp.array([n_total + 10]), + }, + ) + + +if __name__ == "__main__": + pytest.main([__file__, "-v"]) diff --git a/test/test_partitioned_infections.py b/test/test_partitioned_infections.py new file mode 100644 index 00000000..b043bf31 --- /dev/null +++ b/test/test_partitioned_infections.py @@ -0,0 +1,125 @@ +""" +Unit tests for PartitionedInfections. +""" + +import jax.numpy as jnp +import numpyro +import pytest +from pyrenew.deterministic import DeterministicPMF, DeterministicVariable + +from pyrenew.latent import PartitionedInfections, RandomWalk + + +@pytest.fixture +def gen_int_rv(): + return DeterministicPMF("gen_int", jnp.array([0.2, 0.5, 0.3])) + + +@pytest.fixture +def process(gen_int_rv): + """Create a PartitionedInfections with the new keyword-only API.""" + return PartitionedInfections( + gen_int_rv=gen_int_rv, + I0_rv=DeterministicVariable("I0", 0.001), + initial_log_rt_rv=DeterministicVariable("initial_log_rt", 0.0), + rt_temporal=RandomWalk(), + allocation_temporal=RandomWalk(), + ) + + +class TestPartitionedInfectionsSample: + """Test sample method with population structure at sample time.""" + + def test_sample_returns_correct_shapes(self, process): + """Test that sample returns correct output shapes.""" + with numpyro.handlers.seed(rng_seed=42): + result = process.sample( + n_days_post_init=30, + obs_fractions=jnp.array([0.3, 0.25]), + unobs_fractions=jnp.array([0.45]), + ) + + inf_juris, inf_all, inf_obs, inf_unobs = result + n_total = process.n_initialization_points + 30 + + assert inf_juris.shape == (n_total,) + assert inf_all.shape == (n_total, 3) + assert inf_obs.shape == (n_total, 2) + assert inf_unobs.shape == (n_total, 1) + + def test_same_model_different_jurisdictions(self, process): + """Test that one model can fit different population structures.""" + # Jurisdiction A: 2 observed, 1 unobserved + with numpyro.handlers.seed(rng_seed=42): + _, inf_all_a, _, _ = process.sample( + n_days_post_init=30, + obs_fractions=jnp.array([0.3, 0.25]), + unobs_fractions=jnp.array([0.45]), + ) + + # Jurisdiction B: 4 observed, 1 unobserved + with numpyro.handlers.seed(rng_seed=42): + _, inf_all_b, _, _ = process.sample( + n_days_post_init=30, + obs_fractions=jnp.array([0.15, 0.20, 0.25, 0.10]), + unobs_fractions=jnp.array([0.30]), + ) + + assert inf_all_a.shape[1] == 3 # K=3 + assert inf_all_b.shape[1] == 5 # K=5 + + def test_allocated_infections_sum_to_total(self, process): + """Test that allocated infections sum to jurisdiction total.""" + with numpyro.handlers.seed(rng_seed=42): + inf_juris, inf_all, _, _ = process.sample( + n_days_post_init=30, + obs_fractions=jnp.array([0.3, 0.25]), + unobs_fractions=jnp.array([0.45]), + ) + + infections_sum = jnp.sum(inf_all, axis=1) + assert jnp.allclose(inf_juris, infections_sum, atol=1e-6) + + def test_allocation_proportions_sum_to_one(self, process): + """Test that allocation proportions sum to 1 at each timepoint.""" + with numpyro.handlers.seed(rng_seed=42): + with numpyro.handlers.trace() as trace: + process.sample( + n_days_post_init=30, + obs_fractions=jnp.array([0.3, 0.25]), + unobs_fractions=jnp.array([0.45]), + ) + + proportions = trace["allocation_proportions"]["value"] + proportion_sums = jnp.sum(proportions, axis=1) + + assert jnp.allclose(proportion_sums, 1.0, atol=1e-6) + + +class TestPartitionedInfectionsValidation: + """Test validation of inputs.""" + + def test_rejects_invalid_I0(self, gen_int_rv): + """Test that invalid I0 values are rejected.""" + with pytest.raises(ValueError, match="I0 must be positive"): + PartitionedInfections( + gen_int_rv=gen_int_rv, + I0_rv=DeterministicVariable("I0", -0.1), + initial_log_rt_rv=DeterministicVariable("initial_log_rt", 0.0), + rt_temporal=RandomWalk(), + allocation_temporal=RandomWalk(), + ) + + def test_rejects_fractions_not_summing_to_one(self, process): + """Test that fractions not summing to 1 raises error at sample time.""" + with pytest.raises(ValueError, match="must sum to 1.0"): + with numpyro.handlers.seed(rng_seed=42): + process.sample( + n_days_post_init=30, + obs_fractions=jnp.array([0.3, 0.25]), + unobs_fractions=jnp.array([0.40]), # Sum is 0.95 + ) + + +if __name__ == "__main__": + pytest.main([__file__, "-v"]) diff --git a/test/test_temporal_processes.py b/test/test_temporal_processes.py new file mode 100644 index 00000000..de43be35 --- /dev/null +++ b/test/test_temporal_processes.py @@ -0,0 +1,125 @@ +""" +Unit tests for temporal processes innovation_sd behavior. +""" + +import jax.numpy as jnp +import numpyro +import pytest + +from pyrenew.latent import AR1, DifferencedAR1, RandomWalk + + +class TestTemporalProcessInnovationSD: + """Test that temporal processes correctly use innovation_sd parameter.""" + + def test_random_walk_smaller_innovation_sd_produces_smoother_trajectory( + self, + ): + """Verify that smaller innovation_sd produces less volatile trajectories.""" + n_timepoints = 100 + + with numpyro.handlers.seed(rng_seed=42): + rw_small = RandomWalk(innovation_sd=0.1) + trajectory_small = rw_small.sample(n_timepoints=n_timepoints) + + with numpyro.handlers.seed(rng_seed=42): + rw_large = RandomWalk(innovation_sd=1.0) + trajectory_large = rw_large.sample(n_timepoints=n_timepoints) + + # Smaller innovation_sd should produce smaller step sizes + steps_small = jnp.abs(jnp.diff(trajectory_small)) + steps_large = jnp.abs(jnp.diff(trajectory_large)) + + assert jnp.mean(steps_small) < jnp.mean(steps_large) + assert jnp.max(steps_small) < jnp.max(steps_large) + + def test_ar1_smaller_innovation_sd_produces_lower_variance(self): + """Verify AR1 with smaller innovation_sd produces lower variance trajectories.""" + n_timepoints = 100 + autoreg = 0.7 + + with numpyro.handlers.seed(rng_seed=42): + ar_small = AR1(autoreg=autoreg, innovation_sd=0.2) + trajectory_small = ar_small.sample(n_timepoints=n_timepoints) + + with numpyro.handlers.seed(rng_seed=42): + ar_large = AR1(autoreg=autoreg, innovation_sd=1.0) + trajectory_large = ar_large.sample(n_timepoints=n_timepoints) + + # After burn-in, smaller innovation_sd should have lower variance + burn_in = 20 + var_small = jnp.var(trajectory_small[burn_in:]) + var_large = jnp.var(trajectory_large[burn_in:]) + + assert var_small < var_large + + def test_differenced_ar1_smaller_innovation_sd_produces_smoother_changes( + self, + ): + """Verify DifferencedAR1 with smaller innovation_sd produces smoother changes.""" + n_timepoints = 100 + autoreg = 0.6 + + with numpyro.handlers.seed(rng_seed=42): + dar_small = DifferencedAR1(autoreg=autoreg, innovation_sd=0.15) + trajectory_small = dar_small.sample(n_timepoints=n_timepoints) + + with numpyro.handlers.seed(rng_seed=42): + dar_large = DifferencedAR1(autoreg=autoreg, innovation_sd=0.8) + trajectory_large = dar_large.sample(n_timepoints=n_timepoints) + + # Growth rates (differences) should have lower variance + diffs_small = jnp.diff(trajectory_small) + diffs_large = jnp.diff(trajectory_large) + + assert jnp.std(diffs_small) < jnp.std(diffs_large) + + def test_explicit_innovation_sd_overrides_constructor_value(self): + """Verify that explicit innovation_sd in sample() overrides constructor value.""" + rw = RandomWalk(innovation_sd=0.1) + + with numpyro.handlers.seed(rng_seed=42): + traj_constructor = rw.sample(n_timepoints=50) + + with numpyro.handlers.seed(rng_seed=42): + traj_override = rw.sample(n_timepoints=50, innovation_sd=1.0) + + # Trajectories should be substantially different + assert not jnp.allclose(traj_constructor, traj_override, rtol=0.1) + + def test_vectorized_sampling_respects_innovation_sd(self): + """Verify vectorized sampling uses the innovation_sd parameter.""" + n_processes = 5 + n_timepoints = 50 + + with numpyro.handlers.seed(rng_seed=42): + rw_small = RandomWalk(innovation_sd=0.2) + trajs_small = rw_small.sample( + n_timepoints=n_timepoints, n_processes=n_processes + ) + + with numpyro.handlers.seed(rng_seed=42): + rw_large = RandomWalk(innovation_sd=1.0) + trajs_large = rw_large.sample( + n_timepoints=n_timepoints, n_processes=n_processes + ) + + # Check that smaller innovation_sd produces smaller step sizes across all processes + steps_small = jnp.abs(jnp.diff(trajs_small, axis=0)) + steps_large = jnp.abs(jnp.diff(trajs_large, axis=0)) + + assert jnp.mean(steps_small) < jnp.mean(steps_large) + + def test_validation_rejects_non_positive_innovation_sd(self): + """Verify that non-positive innovation_sd values are rejected.""" + with pytest.raises(ValueError, match="innovation_sd must be positive"): + RandomWalk(innovation_sd=0.0) + + with pytest.raises(ValueError, match="innovation_sd must be positive"): + RandomWalk(innovation_sd=-0.5) + + with pytest.raises(ValueError, match="innovation_sd must be positive"): + AR1(autoreg=0.5, innovation_sd=-0.1) + + with pytest.raises(ValueError, match="innovation_sd must be positive"): + DifferencedAR1(autoreg=0.5, innovation_sd=0.0) From b2acc98511b8bc559b98b872fd0b371507ca8559 Mon Sep 17 00:00:00 2001 From: Mitzi Morris Date: Wed, 14 Jan 2026 16:52:06 -0500 Subject: [PATCH 36/81] tutorial - latent hierarchical infections --- .../latent_hierarchical_infections.qmd | 828 ++++++++++++++++++ 1 file changed, 828 insertions(+) create mode 100644 docs/tutorials/latent_hierarchical_infections.qmd diff --git a/docs/tutorials/latent_hierarchical_infections.qmd b/docs/tutorials/latent_hierarchical_infections.qmd new file mode 100644 index 00000000..8bd2e746 --- /dev/null +++ b/docs/tutorials/latent_hierarchical_infections.qmd @@ -0,0 +1,828 @@ +--- +title: 'Hierarchical Latent Infections: Understanding Temporal Process Choices' +format: + html: + toc: true + toc-depth: 3 + embed-resources: true + self-contained-math: true + code-fold: true + code-tools: true +engine: jupyter +jupyter: + jupytext: + text_representation: + extension: .qmd + format_name: quarto + format_version: '1.0' + jupytext_version: 1.18.1 + kernelspec: + display_name: Python 3 (ipykernel) + language: python + name: python3 +--- + +```{python} +#| label: setup +#| output: false +import jax.numpy as jnp +import jax.random as random +import numpy as np +import numpyro +from numpyro.infer import Predictive +import pandas as pd +import plotnine as p9 +import warnings +warnings.filterwarnings("ignore") +``` + +```{python} +from pyrenew.latent import ( + HierarchicalInfections, + AR1, + DifferencedAR1, + RandomWalk, +) +from pyrenew.deterministic import DeterministicPMF, DeterministicVariable +``` + +## The Modeling Problem + +You have a jurisdiction (e.g., a state or metropolitan area) with multiple subpopulations. Some subpopulations have direct observations; others do not. Disaggregated data from many subpopulations is often both sparse and noisy. You want to model transmission dynamics that: + +- Share information across subpopulations (pooling strength) +- Allow local transmission to deviate from the jurisdiction average +- Produce estimates for both observed and unobserved areas + +`HierarchicalInfections` addresses this by decomposing the effective reproduction number $R(t)$ into: + +$$\log R_k(t) = \log R_{\text{baseline}}(t) + \delta_k(t)$$ + +where: + +- $R_{\text{baseline}}(t)$ is the jurisdiction-level reproduction number (shared across all subpopulations) +- $\delta_k(t)$ is the deviation for subpopulation $k$, constrained so that $\sum_k \delta_k(t) = 0$ + +The sum-to-zero constraint ensures $R_{\text{baseline}}(t)$ represents the geometric mean of subpopulation $R$ values—a natural "center" for the hierarchy. + +Each subpopulation runs its own renewal equation with its own $R_k(t)$, then results are aggregated for jurisdiction-level quantities (like total hospital admissions). + +## Model Inputs + +`HierarchicalInfections` requires several inputs that shape the infection dynamics. Understanding what each does helps you configure the model appropriately. + +### A Note on RandomVariables + +All inputs to `HierarchicalInfections` are **RandomVariables**—objects that can be sampled. In this tutorial, we use `DeterministicVariable` and `DeterministicPMF` (fixed values) for clarity. In real inference, you would typically use `DistributionalVariable` with priors for quantities you want to estimate (like I0 or initial Rt). + +For example: +```python +# Fixed value (for illustration) +I0_rv = DeterministicVariable("I0", 0.001) + +# With a prior (for inference) +from pyrenew.randomvariable import DistributionalVariable +import numpyro.distributions as dist +I0_rv = DistributionalVariable("I0", dist.Beta(1, 1000)) +``` + +### Population Structure + +We define subpopulations as fractions of the total jurisdiction population: + +```{python} +# 5 observed subpopulations +obs_fractions = jnp.array([0.10, 0.14, 0.21, 0.22, 0.07]) + +# 1 unobserved subpopulation +unobs_fractions = jnp.array([0.26]) + +# Verify fractions sum to 1 +total = float(jnp.sum(obs_fractions) + jnp.sum(unobs_fractions)) +print(f"Total population fraction: {total:.2f}") +print(f"Observed coverage: {float(jnp.sum(obs_fractions)):.0%}") +print(f"Unobserved coverage: {float(jnp.sum(unobs_fractions)):.0%}") +``` + +Note: Population structure is passed at **sample time**, not when constructing the process. This allows the same model specification to be fit to different jurisdictions. + +### Generation Interval + +The **generation interval** is the probability distribution over the time between infection of a primary case and infection of a secondary case. It determines the "memory" of the renewal process—how current infections depend on past infections. + +```{python} +# Typical COVID-like generation interval +gen_int_pmf = jnp.array([0.16, 0.32, 0.25, 0.14, 0.07, 0.04, 0.02]) +gen_int_rv = DeterministicPMF("gen_int", gen_int_pmf) + +days = np.arange(len(gen_int_pmf)) +mean_gen_time = float(np.sum(days * gen_int_pmf)) +print(f"Mean generation time: {mean_gen_time:.1f} days") +``` + +**What it controls:** The generation interval affects how quickly infection dynamics respond to changes in Rt: + +- **Shorter generation interval** → faster response, more volatile dynamics +- **Longer generation interval** → slower response, smoother dynamics + +The generation interval is typically fixed based on pathogen biology, not estimated from data. + +```{python} +# Compare generation intervals: short vs long +gi_short = jnp.array([0.4, 0.4, 0.2]) # Mean ~1 day +gi_long = jnp.array([0.05, 0.1, 0.2, 0.3, 0.2, 0.1, 0.05]) # Mean ~3.5 days + +gi_short_rv = DeterministicPMF("gi_short", gi_short) +gi_long_rv = DeterministicPMF("gi_long", gi_long) + +print(f"Short GI mean: {float(np.sum(np.arange(len(gi_short)) * gi_short)):.1f} days") +print(f"Long GI mean: {float(np.sum(np.arange(len(gi_long)) * gi_long)):.1f} days") +``` + +To illustrate, we sample infection trajectories using each generation interval while holding other inputs fixed (I0 = 0.001, DifferencedAR1 baseline, AR1 deviations) and using the same random seed so the underlying Rt trajectory is identical. Solid lines use `initial_log_rt = 0` (Rt starts at 1); dashed lines show `initial_log_rt = ±0.2` (Rt starts growing or declining) with the standard GI: + +```{python} +#| fig-cap: Effect of generation interval and initial_log_rt on infection dynamics. Solid lines start at Rt=1; dashed lines show initial growth (Rt≈1.22) or decline (Rt≈0.82). + +# Create models with different generation intervals +baseline_temporal = DifferencedAR1(autoreg=0.9, innovation_sd=0.05) +subpop_temporal = AR1(autoreg=0.8, innovation_sd=0.05) + +n_days_demo = 28 +gi_data = [] + +# GI comparison at initial_log_rt = 0 +gi_configs = { + "Short GI (~1 day)": gi_short_rv, + "Standard GI (~2 days)": gen_int_rv, + "Long GI (~3.5 days)": gi_long_rv, +} + +for name, gi_rv in gi_configs.items(): + model = HierarchicalInfections( + gen_int_rv=gi_rv, + I0_rv=DeterministicVariable("I0", 0.001), + initial_log_rt_rv=DeterministicVariable("initial_log_rt", 0.0), + baseline_temporal=baseline_temporal, + subpop_temporal=subpop_temporal, + ) + with numpyro.handlers.seed(rng_seed=42): + inf_agg, _, _, _ = model.sample( + n_days_post_init=n_days_demo, + obs_fractions=obs_fractions, + unobs_fractions=unobs_fractions, + ) + n_init = model.n_initialization_points + inf = np.array(inf_agg)[n_init:] + for d in range(len(inf)): + gi_data.append({ + "day": d, + "infections": float(inf[d]), + "config": name, + "linetype": "solid", + }) + +# initial_log_rt comparison with standard GI +for init_rt_name, init_rt_val in [("Rt≈1.22 (growing)", 0.2), ("Rt≈0.82 (declining)", -0.2)]: + model = HierarchicalInfections( + gen_int_rv=gen_int_rv, + I0_rv=DeterministicVariable("I0", 0.001), + initial_log_rt_rv=DeterministicVariable("initial_log_rt", init_rt_val), + baseline_temporal=baseline_temporal, + subpop_temporal=subpop_temporal, + ) + with numpyro.handlers.seed(rng_seed=42): + inf_agg, _, _, _ = model.sample( + n_days_post_init=n_days_demo, + obs_fractions=obs_fractions, + unobs_fractions=unobs_fractions, + ) + n_init = model.n_initialization_points + inf = np.array(inf_agg)[n_init:] + for d in range(len(inf)): + gi_data.append({ + "day": d, + "infections": float(inf[d]), + "config": f"Standard GI, {init_rt_name}", + "linetype": "dashed", + }) + +gi_df = pd.DataFrame(gi_data) + +( + p9.ggplot(gi_df, p9.aes(x="day", y="infections", color="config", linetype="linetype")) + + p9.geom_line(size=1) + + p9.scale_linetype_manual( + values={"solid": "solid", "dashed": "dashed"}, + labels={"solid": "initial Rt = 1", "dashed": "initial Rt ≠ 1"}, + ) + + p9.labs( + x="Days", + y="Infections (proportion)", + title="Effect of Generation Interval and Initial Rt", + color="Configuration", + linetype="Initial Rt", + ) + + p9.theme_minimal() + + p9.theme(figure_size=(10, 5), legend_position="right") +) +``` + +```{python} +#| fig-cap: Zoomed view of days 14-21 showing separation between trajectories. + +gi_df_zoom = gi_df[(gi_df["day"] >= 14) & (gi_df["day"] <= 21)] + +( + p9.ggplot(gi_df_zoom, p9.aes(x="day", y="infections", color="config", linetype="linetype")) + + p9.geom_line(size=1) + + p9.scale_linetype_manual( + values={"solid": "solid", "dashed": "dashed"}, + labels={"solid": "initial Rt = 1", "dashed": "initial Rt ≠ 1"}, + ) + + p9.labs( + x="Days", + y="Infections (proportion)", + title="Zoomed: Days 14-21", + color="Configuration", + linetype="Initial Rt", + ) + + p9.theme_minimal() + + p9.theme(figure_size=(10, 5), legend_position="right") +) +``` + +### Initial Conditions: I0 and initial_log_rt + +Parameters `I0` and `initial_log_rt` set the starting point for the infection process. + +The previous plot shows the effect of `initial_log_rt`: the solid lines use `initial_log_rt = 0` (Rt starts at 1), while the dashed lines use `initial_log_rt = ±0.2` (Rt starts above or below 1). Notice that the solid lines stay flat for the first ~15 days — when Rt = 1, each infected person infects approximately one other, so infection levels remain stable. The dashed lines show immediate growth or decline because Rt starts away from 1. + +The flat period for solid lines also reflects DifferencedAR1 with high autoregression (0.9): this process models the rate of change of log(Rt), starting from zero, and changes accumulate gradually. If you need more initial dynamics, set `initial_log_rt ≠ 0`, use `RandomWalk` instead, or increase `innovation_sd`. + +`I0` is the proportion of the population infected at time zero. It seeds the renewal equation and scales the entire trajectory: + +```{python} +# Default values +I0_rv = DeterministicVariable("I0", 0.001) # 0.1% initially infected +initial_log_rt_rv = DeterministicVariable("initial_log_rt", 0.0) # Rt starts at 1.0 + +print(f"I0 = 0.001 means {0.001:.1%} of population initially infected") +print(f"initial_log_rt = 0.0 means Rt starts at {np.exp(0.0):.1f}") +``` + +In the previous plot, `I0` was specified as: `I0_rv=DeterministicVariable("I0", 0.001)`, i.e., $1%$ of the population. We overplot trajectories for $10%$ and $0.1%$ as well. + +```{python} +#| fig-cap: Effect of I0 on infection scale. Higher I0 shifts the entire trajectory up proportionally. + +i0_configs = { + "I0 = 0.0001 (0.01%)": 0.0001, + "I0 = 0.001 (0.1%)": 0.001, + "I0 = 0.01 (1%)": 0.01, +} + +i0_results = {} +for name, i0_val in i0_configs.items(): + model = HierarchicalInfections( + gen_int_rv=gen_int_rv, + I0_rv=DeterministicVariable("I0", i0_val), + initial_log_rt_rv=DeterministicVariable("initial_log_rt", 0.0), + baseline_temporal=baseline_temporal, + subpop_temporal=subpop_temporal, + ) + with numpyro.handlers.seed(rng_seed=42): + inf_agg, _, _, _ = model.sample( + n_days_post_init=n_days_demo, + obs_fractions=obs_fractions, + unobs_fractions=unobs_fractions, + ) + n_init = model.n_initialization_points + i0_results[name] = np.array(inf_agg)[n_init:] + +i0_data = [] +for name, inf in i0_results.items(): + for d in range(len(inf)): + i0_data.append({"day": d, "infections": float(inf[d]), "I0": name}) +i0_df = pd.DataFrame(i0_data) + +( + p9.ggplot(i0_df, p9.aes(x="day", y="infections", color="I0")) + + p9.geom_line(size=1) + + p9.scale_y_log10() + + p9.labs( + x="Days", + y="Infections (proportion, log scale)", + title="Effect of Initial Infection Prevalence (I0)", + subtitle="Same Rt trajectory, different starting prevalence", + ) + + p9.theme_minimal() + + p9.theme(figure_size=(10, 5)) +) +``` + +### Temporal Processes + +The temporal process determines how $R_{\text{baseline}}(t)$ and the deviations $\delta_k(t)$ evolve over time. PyRenew provides three options: + +| Process | What It Does | Epidemiological Interpretation | +|---------|--------------|-------------------------------| +| `RandomWalk` | Each value = previous + noise | Maximum flexibility; Rt can drift anywhere | +| `AR1` | Each value reverts toward a mean | Rt stays bounded near a baseline; deviations decay | +| `DifferencedAR1` | The *rate of change* reverts toward zero | Rt can trend persistently, but growth rate stabilizes | + +**Choosing a temporal process:** + +| If you believe... | Consider | +|-------------------|----------| +| Rt should stay near 1 (endemic equilibrium) | `AR1` for baseline | +| Rt may trend up/down over your horizon | `DifferencedAR1` for baseline | +| Local differences are transient | `AR1` for deviations | +| Local differences persist | `RandomWalk` for deviations | + +**Parameters:** + +- **`autoreg`** (AR1, DifferencedAR1): How strongly the process reverts. Values near 1 = slow reversion; near 0 = fast. +- **`innovation_sd`**: Daily volatility. Larger values = more variable trajectories. + +## Configuring the Latent Process + +Now let's assemble all the inputs: + +```{python} +# Baseline Rt: DifferencedAR1 allows trends while stabilizing growth rate +baseline_temporal = DifferencedAR1(autoreg=0.9, innovation_sd=0.05) + +# Subpopulation deviations: AR1 shrinks differences back toward zero +subpop_temporal = AR1(autoreg=0.8, innovation_sd=0.05) +``` + +**Why these choices?** + +- `DifferencedAR1` for baseline: We don't want to force Rt back to 1; epidemics can grow or decline persistently. But we do want the *rate of change* to stabilize. +- `AR1` for deviations: We expect local differences to be somewhat transient. If one area has a temporary spike, it should eventually return toward the jurisdiction average. + +### Creating the Process + +```{python} +latent_infections = HierarchicalInfections( + gen_int_rv=gen_int_rv, + I0_rv=I0_rv, + initial_log_rt_rv=initial_log_rt_rv, + baseline_temporal=baseline_temporal, + subpop_temporal=subpop_temporal, +) + +print(f"Generation interval length: {latent_infections.get_required_lookback()} days") +print(f"Initialization points: {latent_infections.n_initialization_points} days") +``` + +## Prior Predictive Checks: What Does the Model Believe? + +Now we examine what infection dynamics this configuration considers plausible—*before* seeing any data. + +### Single Sample + +```{python} +n_days = 28 # 4 weeks of simulation + +with numpyro.handlers.seed(rng_seed=42): + result = latent_infections.sample( + n_days_post_init=n_days, + obs_fractions=obs_fractions, + unobs_fractions=unobs_fractions, + ) + +infections_agg, infections_all, infections_obs, infections_unobs = result + +n_total = infections_agg.shape[0] +n_init = n_total - n_days + +print("Output shapes:") +print(f" Aggregate (jurisdiction total): {infections_agg.shape}") +print(f" All subpopulations: {infections_all.shape}") +print(f" Observed subpopulations: {infections_obs.shape}") +print(f" Unobserved subpopulations: {infections_unobs.shape}") +print(f"\nTimeline: {n_init} initialization days + {n_days} simulation days = {n_total} total") +``` + +### Many Samples: The Prior Predictive Distribution + +A single sample tells us little. We need many samples to understand the *distribution* of trajectories the model considers plausible. + +```{python} +def sample_model(): + """Wrapper for prior predictive sampling.""" + return latent_infections.sample( + n_days_post_init=n_days, + obs_fractions=obs_fractions, + unobs_fractions=unobs_fractions, + ) + +n_samples = 200 +predictive = Predictive(sample_model, num_samples=n_samples) +prior_samples = predictive(random.PRNGKey(42)) + +print(f"Rt baseline samples: {prior_samples['rt_baseline'].shape}") +print(f"Infection samples: {prior_samples['infections_aggregate'].shape}") +``` + +### Visualizing the Prior: Baseline Rt Trajectories + +```{python} +# Extract post-initialization Rt trajectories +rt_samples = np.array(prior_samples["rt_baseline"])[:, n_init:] + +# Build long-format dataframe for plotting +rt_data = [] +for i in range(n_samples): + for d in range(n_days): + rt_data.append({"day": d, "rt": float(rt_samples[i, d]), "sample": i}) +rt_df = pd.DataFrame(rt_data) + +# Compute summary statistics +rt_median = np.median(rt_samples, axis=0) +rt_mean = np.mean(rt_samples, axis=0) +rt_q05 = np.percentile(rt_samples, 5, axis=0) +rt_q95 = np.percentile(rt_samples, 95, axis=0) +``` + +```{python} +#| fig-cap: Prior predictive distribution of baseline Rt. Each light blue line is one sample from the prior. The model considers all these trajectories plausible before seeing data. + +# Cap display for readability (some trajectories explode) +rt_cap = 5.0 +n_extreme = np.sum(np.any(rt_samples > rt_cap, axis=1)) + +( + p9.ggplot(rt_df, p9.aes(x="day", y="rt", group="sample")) + + p9.geom_line(alpha=0.1, size=0.4, color="steelblue") + + p9.geom_hline(yintercept=1.0, color="red", linetype="dashed", alpha=0.7) + + p9.coord_cartesian(ylim=(0, rt_cap)) + + p9.labs( + x="Days", + y="Rt (baseline)", + title=f"Prior Predictive: Baseline Rt ({n_samples} samples)", + subtitle=f"Red dashed line: Rt = 1. {n_extreme} samples exceed Rt = {rt_cap} (not shown in full).", + ) + + p9.theme_minimal() +) +``` + +### Interpreting the Prior Predictive + +```{python} +print("Rt at end of simulation (day 28):") +print(f" 5th percentile: {rt_q05[-1]:.2f}") +print(f" Median: {rt_median[-1]:.2f}") +print(f" Mean: {rt_mean[-1]:.2f}") +print(f" 95th percentile: {rt_q95[-1]:.2f}") +print(f" Maximum: {np.max(rt_samples[:, -1]):.1f}") +``` + +**What this tells us:** + +The median stays near 1, but the mean is pulled up by explosive trajectories. This is a consequence of modeling on the log scale: + +- Half the trajectories have Rt < 1 (bounded below by 0) +- Half have Rt > 1 (unbounded above—can reach 10, 50, 100+) +- The explosive trajectories dominate the mean + +**Is this a problem?** It depends on your data. With abundant data, these extreme trajectories will be ruled out by the likelihood. With sparse data, they may not be—and your posterior could include implausible dynamics. + +## Comparing Temporal Process Choices + +The prior predictive distribution changes dramatically based on your temporal process choices. Let's compare the three options for baseline Rt. + +```{python} +temporal_configs = { + "AR(1)": AR1(autoreg=0.9, innovation_sd=0.05), + "DifferencedAR1": DifferencedAR1(autoreg=0.9, innovation_sd=0.05), + "RandomWalk": RandomWalk(innovation_sd=0.05), +} + +process_samples = {} +for name, temporal in temporal_configs.items(): + model = HierarchicalInfections( + gen_int_rv=gen_int_rv, + I0_rv=I0_rv, + initial_log_rt_rv=initial_log_rt_rv, + baseline_temporal=temporal, + subpop_temporal=AR1(autoreg=0.8, innovation_sd=0.05), + ) + + def make_sampler(m): + return lambda: m.sample( + n_days_post_init=n_days, + obs_fractions=obs_fractions, + unobs_fractions=unobs_fractions, + ) + + samples = Predictive(make_sampler(model), num_samples=n_samples)( + random.PRNGKey(42) + ) + process_samples[name] = np.array(samples["rt_baseline"])[:, n_init:] +``` + +```{python} +# Build comparison dataframe +comparison_data = [] +for name, rt in process_samples.items(): + for i in range(n_samples): + for d in range(n_days): + comparison_data.append({ + "day": d, + "rt": float(rt[i, d]), + "sample": i, + "process": name, + }) +comparison_df = pd.DataFrame(comparison_data) +comparison_df["process"] = pd.Categorical( + comparison_df["process"], + categories=["AR(1)", "DifferencedAR1", "RandomWalk"], + ordered=True, +) +``` + +```{python} +#| fig-cap: Effect of temporal process choice on prior Rt trajectories. AR(1) stays bounded near 1; DifferencedAR1 and RandomWalk can drift further. + +( + p9.ggplot(comparison_df, p9.aes(x="day", y="rt", group="sample")) + + p9.geom_line(alpha=0.1, size=0.4, color="steelblue") + + p9.geom_hline(yintercept=1.0, color="red", linetype="dashed", alpha=0.7) + + p9.facet_wrap("~ process", ncol=3) + + p9.coord_cartesian(ylim=(0, 3.0)) + + p9.labs( + x="Days", + y="Rt (baseline)", + title="Prior Predictive by Temporal Process", + subtitle="Same innovation_sd=0.05; AR processes have autoreg=0.9", + ) + + p9.theme_minimal() + + p9.theme(figure_size=(12, 4)) +) +``` + +```{python} +print("Rt at day 28 by temporal process:") +print(f"{'Process':<18} {'Median':>8} {'Mean':>8} {'95th %':>8} {'Max':>8}") +print("-" * 52) +for name in ["AR(1)", "DifferencedAR1", "RandomWalk"]: + rt = process_samples[name][:, -1] + print(f"{name:<18} {np.median(rt):>8.2f} {np.mean(rt):>8.2f} " + f"{np.percentile(rt, 95):>8.2f} {np.max(rt):>8.1f}") +``` + +**Key observations:** + +- **AR(1)** produces the tightest distribution—Rt reverts toward 1 +- **DifferencedAR1** allows more drift but growth rate stabilizes +- **RandomWalk** has the widest spread—no mean reversion at all + +## Subpopulation Deviations + +The same choice applies to subpopulation deviations $\delta_k(t)$. Let's compare AR(1) vs RandomWalk: + +```{python} +subpop_configs = { + "AR(1) deviations": AR1(autoreg=0.8, innovation_sd=0.05), + "RandomWalk deviations": RandomWalk(innovation_sd=0.05), +} + +deviation_samples = {} +for name, subpop_temporal in subpop_configs.items(): + model = HierarchicalInfections( + gen_int_rv=gen_int_rv, + I0_rv=I0_rv, + initial_log_rt_rv=initial_log_rt_rv, + baseline_temporal=DifferencedAR1(autoreg=0.9, innovation_sd=0.05), + subpop_temporal=subpop_temporal, + ) + + def make_sampler(m): + return lambda: m.sample( + n_days_post_init=n_days, + obs_fractions=obs_fractions, + unobs_fractions=unobs_fractions, + ) + + deviation_samples[name] = Predictive(make_sampler(model), num_samples=n_samples)( + random.PRNGKey(42) + ) +``` + +```{python} +# Plot deviations for first subpopulation +dev_data = [] +for name in subpop_configs.keys(): + devs = np.array(deviation_samples[name]["subpop_deviations"])[:, n_init:, 0] + for i in range(n_samples): + for d in range(n_days): + dev_data.append({ + "day": d, + "deviation": float(devs[i, d]), + "sample": i, + "process": name, + }) +dev_df = pd.DataFrame(dev_data) +``` + +```{python} +#| fig-cap: Subpopulation deviations under different temporal processes. AR(1) deviations decay toward zero; RandomWalk deviations can diverge. + +( + p9.ggplot(dev_df, p9.aes(x="day", y="deviation", group="sample")) + + p9.geom_line(alpha=0.15, size=0.4, color="purple") + + p9.geom_hline(yintercept=0, color="black", linetype="dotted") + + p9.facet_wrap("~ process", ncol=2) + + p9.labs( + x="Days", + y="Deviation δ₁(t)", + title="Subpopulation Deviation Trajectories (first subpop)", + subtitle=f"{n_samples} prior samples; baseline uses DifferencedAR1", + ) + + p9.theme_minimal() + + p9.theme(figure_size=(10, 4)) +) +``` + +**Interpretation:** + +- **AR(1)** deviations stay close to zero—subpopulations track the jurisdiction average +- **RandomWalk** deviations can grow—subpopulations can diverge substantially + +Choose based on your scientific question: Do you expect local differences to persist or decay? + +## Tuning Hyperparameters + +The `autoreg` and `innovation_sd` parameters control how constrained the prior is. Let's compare a "weakly informative" vs "more informative" configuration: + +```{python} +configs = { + "Weakly informative\n(autoreg=0.9, sd=0.05)": { + "baseline": DifferencedAR1(autoreg=0.9, innovation_sd=0.05), + "subpop": AR1(autoreg=0.8, innovation_sd=0.05), + }, + "More informative\n(autoreg=0.7, sd=0.02)": { + "baseline": DifferencedAR1(autoreg=0.7, innovation_sd=0.02), + "subpop": AR1(autoreg=0.6, innovation_sd=0.02), + }, +} + +tuning_samples = {} +for name, cfg in configs.items(): + model = HierarchicalInfections( + gen_int_rv=gen_int_rv, + I0_rv=I0_rv, + initial_log_rt_rv=initial_log_rt_rv, + baseline_temporal=cfg["baseline"], + subpop_temporal=cfg["subpop"], + ) + + def make_sampler(m): + return lambda: m.sample( + n_days_post_init=n_days, + obs_fractions=obs_fractions, + unobs_fractions=unobs_fractions, + ) + + tuning_samples[name] = Predictive(make_sampler(model), num_samples=n_samples)( + random.PRNGKey(42) + ) +``` + +```{python} +# Build comparison dataframe +tuning_data = [] +for name, samples in tuning_samples.items(): + rt = np.array(samples["rt_baseline"])[:, n_init:] + for i in range(n_samples): + for d in range(n_days): + tuning_data.append({ + "day": d, + "rt": float(rt[i, d]), + "sample": i, + "config": name, + }) +tuning_df = pd.DataFrame(tuning_data) +``` + +```{python} +#| fig-cap: Effect of hyperparameter choices on prior Rt distribution. Tighter parameters produce more constrained trajectories. + +( + p9.ggplot(tuning_df, p9.aes(x="day", y="rt", group="sample")) + + p9.geom_line(alpha=0.1, size=0.4, color="steelblue") + + p9.geom_hline(yintercept=1.0, color="red", linetype="dashed", alpha=0.7) + + p9.facet_wrap("~ config", ncol=2) + + p9.coord_cartesian(ylim=(0, 3.0)) + + p9.labs( + x="Days", + y="Rt (baseline)", + title="Effect of Hyperparameter Tuning", + ) + + p9.theme_minimal() + + p9.theme(figure_size=(10, 4)) +) +``` + +**Guidance:** + +- **Lower `autoreg`** → faster reversion → tighter distribution +- **Lower `innovation_sd`** → smaller daily changes → smoother trajectories +- With sparse data, more informative priors prevent explosive estimates +- With abundant data, weakly informative priors let the data speak + +## Verifying Model Structure + +Before using `HierarchicalInfections` in a full model, verify the hierarchical structure is working correctly: + +```{python} +# Check that deviations sum to zero (identifiability constraint) +with numpyro.handlers.seed(rng_seed=123): + with numpyro.handlers.trace() as trace: + latent_infections.sample( + n_days_post_init=n_days, + obs_fractions=obs_fractions, + unobs_fractions=unobs_fractions, + ) + +deviations = trace["subpop_deviations"]["value"] +deviation_sums = jnp.sum(deviations, axis=1) +print(f"Max |sum of deviations|: {float(jnp.max(jnp.abs(deviation_sums))):.2e}") +print("(Should be ~0 due to sum-to-zero constraint)") +``` + +```{python} +# Check that aggregate = weighted sum of subpopulations +with numpyro.handlers.seed(rng_seed=123): + inf_agg, inf_all, _, _ = latent_infections.sample( + n_days_post_init=n_days, + obs_fractions=obs_fractions, + unobs_fractions=unobs_fractions, + ) + +all_fractions = jnp.concatenate([obs_fractions, unobs_fractions]) +computed_agg = jnp.sum(inf_all * all_fractions[None, :], axis=1) +max_diff = float(jnp.max(jnp.abs(computed_agg - inf_agg))) + +print(f"Max difference (aggregate vs weighted sum): {max_diff:.2e}") +print("Hierarchical structure verified!" if max_diff < 1e-5 else "ERROR: Mismatch detected") +``` + +## Summary + +### What We Covered + +1. **`HierarchicalInfections`** models subpopulation structure with shared baseline Rt and local deviations + +2. **Model inputs** each affect the dynamics differently: + - **Generation interval**: Determines response speed (fixed by pathogen biology) + - **I0**: Scales the entire infection trajectory + - **initial_log_rt**: Sets initial growth/decline direction + - **Temporal processes**: Control how Rt evolves over time + +3. **All inputs are RandomVariables**: In inference, use `DistributionalVariable` with priors for quantities you want to estimate + +4. **Prior predictive checks** reveal model beliefs before seeing data—critical when data is sparse or noisy + +### Decision Guide + +| Input | Consideration | +|-------|---------------| +| Generation interval | Fixed by pathogen; shorter = faster dynamics | +| I0 | Often poorly identified; prior matters | +| initial_log_rt | Sets starting direction; can often be estimated | +| Temporal process | AR1 for regularization; RandomWalk for flexibility | +| `autoreg`, `innovation_sd` | Lower values = tighter prior; adjust if trajectories explode | + +### Next Steps + +- **`model_architecture.qmd`**: Learn how to combine `HierarchicalInfections` with observation processes using `ModelBuilder` +- **Observation process tutorials**: `observation_processes_counts.qmd` and `observation_processes_measurements.qmd` + +### Code Summary + +```python +from pyrenew.latent import HierarchicalInfections, AR1, DifferencedAR1 +from pyrenew.deterministic import DeterministicPMF, DeterministicVariable + +# Configure the latent process +latent = HierarchicalInfections( + gen_int_rv=DeterministicPMF("gen_int", gen_int_pmf), + I0_rv=DeterministicVariable("I0", 0.001), + initial_log_rt_rv=DeterministicVariable("initial_log_rt", 0.0), + baseline_temporal=DifferencedAR1(autoreg=0.9, innovation_sd=0.05), + subpop_temporal=AR1(autoreg=0.8, innovation_sd=0.05), +) + +# Sample (population structure provided at sample time) +infections_agg, infections_all, infections_obs, infections_unobs = latent.sample( + n_days_post_init=28, + obs_fractions=obs_fractions, + unobs_fractions=unobs_fractions, +) +``` + +In a full model, you would pass `HierarchicalInfections` to `ModelBuilder` along with observation processes—the builder handles initialization, routing, and assembly automatically. From 35e57f5e256429803fd00af5595e3efa9f6f3b4b Mon Sep 17 00:00:00 2001 From: Mitzi Morris Date: Wed, 14 Jan 2026 20:53:13 -0500 Subject: [PATCH 37/81] Fix numpydoc validation and typo errors - Fix typo: "obervation" -> "observation" in building_multisignal_models.qmd - Reorder docstring parameters to match actual signatures in partitioned_infections.py and hierarchical_infections.py - Add missing docstrings to __init__ and validate methods in hierarchical_priors.py - Add Returns sections to _sample_single and _sample_vectorized methods in temporal_processes.py - Add docstrings with Returns sections to test fixtures Co-Authored-By: Claude Opus 4.5 --- .../tutorials/building_multisignal_models.qmd | 1054 ++++++ .../latent_hierarchical_infections.qmd | 188 +- docs/tutorials/model_builder_helper.qmd | 974 +++++ pyrenew/datasets/__init__.py | 4 + pyrenew/datasets/hospital_admissions.py | 79 + .../hospital_admissions_data/2023-11-06.csv | 311 ++ .../hospital_admissions_data/README.md | 36 + .../hospital_admissions_data/__init__.py | 2 + pyrenew/datasets/wastewater_nwss.py | 141 + .../datasets/wastewater_nwss_data/README.md | 53 + .../datasets/wastewater_nwss_data/__init__.py | 2 + .../wastewater_nwss_data/fake_nwss.csv | 3286 +++++++++++++++++ pyrenew/latent/base.py | 1 + pyrenew/latent/hierarchical_infections.py | 18 +- pyrenew/latent/hierarchical_priors.py | 40 + pyrenew/latent/partitioned_infections.py | 18 +- pyrenew/latent/temporal_processes.py | 55 +- pyrenew/model/multisignal_model.py | 2 +- test/test_hierarchical_infections.py | 19 +- test/test_hierarchical_priors.py | 4 +- test/test_model_builder.py | 11 +- test/test_partitioned_infections.py | 19 +- 22 files changed, 6225 insertions(+), 92 deletions(-) create mode 100644 docs/tutorials/building_multisignal_models.qmd create mode 100644 docs/tutorials/model_builder_helper.qmd create mode 100644 pyrenew/datasets/hospital_admissions.py create mode 100644 pyrenew/datasets/hospital_admissions_data/2023-11-06.csv create mode 100644 pyrenew/datasets/hospital_admissions_data/README.md create mode 100644 pyrenew/datasets/hospital_admissions_data/__init__.py create mode 100644 pyrenew/datasets/wastewater_nwss.py create mode 100644 pyrenew/datasets/wastewater_nwss_data/README.md create mode 100644 pyrenew/datasets/wastewater_nwss_data/__init__.py create mode 100644 pyrenew/datasets/wastewater_nwss_data/fake_nwss.csv diff --git a/docs/tutorials/building_multisignal_models.qmd b/docs/tutorials/building_multisignal_models.qmd new file mode 100644 index 00000000..0fb17488 --- /dev/null +++ b/docs/tutorials/building_multisignal_models.qmd @@ -0,0 +1,1054 @@ +--- +title: Building Multi-Signal Renewal Models +format: + gfm: + fig-width: 16 + fig-height: 10 + html: + toc: true + embed-resources: true + self-contained-math: true + code-fold: true + code-tools: true +engine: jupyter +jupyter: + jupytext: + text_representation: + extension: .qmd + format_name: quarto + format_version: '1.0' + jupytext_version: 1.18.1 + kernelspec: + display_name: Python 3 (ipykernel) + language: python + name: python3 +--- + +```{python} +# | label: setup +# | output: false +import numpyro +from numpyro.infer.reparam import LocScaleReparam + +# to run samplers in parallel you must run `set_host_device_count` before importing jax +numpyro.set_host_device_count(4) +numpyro.enable_x64() +``` + +```{python} +import arviz as az +import jax +import jax.numpy as jnp +import jax.random as random +import numpy as np +import plotnine as p9 +import pandas as pd +import time +import warnings + +warnings.filterwarnings("ignore") + + +def make_rng_key(): + """Generate a time-based random seed.""" + seed = int(time.time() * 1000) % (2**32) + return random.PRNGKey(seed) +``` + +```{python} +import jax +from jax.typing import ArrayLike + +from pyrenew import datasets +from pyrenew.deterministic import ( + DeterministicPMF, + DeterministicVariable, +) +from pyrenew.metaclass import RandomVariable + +from pyrenew.latent import ( + HierarchicalInfections, + AR1, + RandomWalk, + GammaGroupSdPrior, + HierarchicalNormalPrior, +) +from pyrenew.model import ModelBuilder + +from pyrenew.observation import ( + Counts, + HierarchicalNormalNoise, + Measurements, + MeasurementNoise, + NegativeBinomialNoise, +) +``` + +## Overview + +Renewal models in PyRenew combine two types of components: + +1. **Latent infection process**: Generates unobserved infections via the renewal equation, driven by a time-varying reproduction number $R(t)$ + +2. **Observation processes**: Transform latent infections into observable signals (hospital admissions, wastewater concentrations, etc.) by applying delays, ascertainment, and noise + +A **multi-signal model** combines multiple observation processes—each representing a different data stream, e.g., hospital admissions, wastewater concentrations, which stem from the same underlying latent infection process. By jointly modeling these signals, we can improve estimation and prediction of the time-varying reproduction number $R(t)$. Such a model must: + +- Generate a single coherent infection trajectory (or set of trajectories for subpopulations) +- Route those infections to each observation process appropriately +- Handle the initialization period required by delay distributions + +The `ModelBuilder` class handles this plumbing. You specify: + +1. A **latent process** (e.g., `HierarchicalInfections`) that defines how infections evolve +2. One or more **observation processes** (e.g., `Counts`, `Measurements`) that define how infections become data + +The builder computes initialization requirements, wires components together, and produces a model ready for inference. + +### Related Tutorials + +Before diving into multi-signal models, you may want to review these foundational tutorials: + +- **[Hierarchical Latent Infections](latent_hierarchical_infections.qmd)**: Understanding temporal process choices for $R(t)$ +- **[Observation Processes: Counts](observation_processes_counts.qmd)**: Modeling count data (admissions, deaths) +- **[Observation Processes: Measurements](observation_processes_measurements.qmd)**: Modeling continuous data (wastewater) + +This tutorial shows how to combine these components into a complete multi-signal model. + +### What This Tutorial Covers + +This tutorial demonstrates building a multi-signal renewal model using: + +- `HierarchicalInfections` — subpopulations share a jurisdiction-level baseline $R(t)$ with subpopulation-specific deviations +- `Counts` — hospital admissions (jurisdiction-level) +- A custom `Wastewater` class — viral concentrations (subpopulation-level) + +## Model Structure + +Before diving into implementation, let's visualize how data flows through a multi-signal renewal model. The diagram below shows the generative process: latent infections flow from the renewal equation to observation processes, which transform them into predicted observations. + +```mermaid +flowchart TB + + subgraph Latent["Latent Infection Process"] + L[HierarchicalInfections] + end + + subgraph Infections["Infection Arrays"] + J["infections_aggregate (T,)"] + S["infections_subpop (T, K_obs)"] + end + + subgraph Obs["Observation Processes"] + C[Counts] + W[Wastewater] + end + + subgraph Data["Observed Data"] + HA[Hospital Admissions] + WW[Concentrations] + end + + L --> J + L --> S + J -->|"aggregate"| C + S -->|"subpop"| W + C -->|"ascertainment × delay"| HA + W -->|"shedding × genome × dilution"| WW +``` + +### Infection Resolution + +Different observation processes observe different levels of the model hierarchy. Each observation process declares an **infection resolution** that determines what infection data it receives: + +| Resolution | Receives | Example signals | +|----------------------|-------------------|-------------------------------| +| `"aggregate"` | Aggregated infections (sum across all subpopulations) | Hospital admissions, case counts | +| `"subpop"` | Infection matrix with one column per observed subpopulation | Wastewater, site-specific surveillance | + +The `ModelBuilder` routes latent infections to observation processes based on each process's declared resolution. + +With this structure in mind, we'll now define each component following the generative direction: first the latent infection process, then the observation processes. + +## Latent Infection Process + +Latent infection processes implement the renewal equation to generate infection trajectories. All latent processes share common components: + +- **Generation interval**: PMF for secondary infection timing +- **Initial infections (I0)**: Starting condition for the renewal process +- **Temporal dynamics**: How $R(t)$ evolves over time + +### Generation Interval + +The generation interval PMF specifies the probability that a secondary infection occurs $d$ days after the primary infection. + +```{python} +covid_gen_int = [0.16, 0.32, 0.25, 0.14, 0.07, 0.04, 0.02] +gen_int_pmf = jnp.array(covid_gen_int) +gen_int_rv = DeterministicPMF("gen_int", gen_int_pmf) + +# Mean generation time +days = np.arange(len(gen_int_pmf)) +print(f"Generation interval length: {len(gen_int_pmf)} days") +``` + +### I0: Initial Infections + +The initial infections RV `I0_rv` specifies the **proportion of the population infected** at the first observation time. This must be a value in the interval (0, 1]. + +```{python} +I0_rv = DeterministicVariable("I0", 0.01) +``` + +### Initial Log Rt + +We set the initial log(Rt) to 0.0, meaning Rt starts at 1.0. + +```{python} +initial_log_rt_rv = DeterministicVariable("initial_log_rt", 0.0) +``` + +### Temporal Processes for $R(t)$ + +We configure two temporal processes: + +- **Jurisdiction-level** (`baseline_temporal`): AR(1) process for the baseline $R(t)$ +- **Subpopulation-level** (`subpop_temporal`): RandomWalk for subpopulation deviations + +The RandomWalk allows flexible evolution of subpopulation-specific transmission without mean reversion. + +```{python} +# AR1 provides mean-reverting behavior for baseline Rt +baseline_temporal = AR1(autoreg=0.9, innovation_sd=0.05) + +# RandomWalk allows flexible subpopulation deviations +subpop_temporal = RandomWalk(innovation_sd=0.025) +``` + +## Observation Processes + +Observation processes transform latent infections into observable signals and define the statistical model linking predictions to data. Each observation process: + +- Declares what **infection resolution** it needs (`"aggregate"` or `"subpop"`) +- Applies signal-specific transformations (ascertainment, delay convolution, shedding kinetics) +- Defines the noise model + +### Hospital Admissions + +In this example we use a dataset consisting of hospital admissions for COVID-19 across California +for the first 10 months of 2023 (as reported to the CDC). + +```{python} +# Load daily hospital admissions for California +ca_hosp_data = datasets.load_hospital_data_for_state("CA", "2023-11-06.csv") + +hosp_admits = ca_hosp_data["daily_admits"] +population_size = ca_hosp_data["population"] +n_hosp_days = ca_hosp_data["n_days"] + +print("State: California") +print(f"Population: {population_size:,}") +print(f"Date range: {ca_hosp_data['dates'][0]} to {ca_hosp_data['dates'][-1]}") +print(f"Number of days: {n_hosp_days}") +print( + f"Admissions range: {int(hosp_admits.min())} to {int(hosp_admits.max())}" +) +``` + +The hospital admissions data is aggregated at the jurisdiction level, therefore we specify a `Counts` observation process. + +```{python} +# Infection-to-hospitalization delay (COVID-19, from literature) +inf_to_hosp_pmf = jnp.array( + [ + 0, + 0.00469, + 0.01452, + 0.02786, + 0.04237, + 0.05581, + 0.06657, + 0.07379, + 0.07729, + 0.07737, + 0.07465, + 0.06988, + 0.06377, + 0.05696, + 0.04996, + 0.04315, + 0.03677, + 0.03097, + 0.02583, + 0.02135, + 0.01751, + 0.01427, + 0.01156, + 0.00931, + 0.00746, + 0.00596, + 0.00474, + 0.00375, + 0.00296, + 0.00233, + 0.00183, + 0.00143, + 0.00107, + 0.00077, + 0.00054, + 0.00036, + 0.00024, + 0.00015, + 0.00009, + 0.00005, + 0.00003, + 0.00002, + 0.00001, + ] +) +hosp_delay_rv = DeterministicPMF("inf_to_hosp_delay", inf_to_hosp_pmf) + +# IHR: ~1% of infections lead to hospitalization +ihr_rv = DeterministicVariable("ihr", 0.01) + +# Negative binomial concentration (moderate overdispersion) +hosp_concentration_rv = DeterministicVariable("hosp_concentration", 10.0) + +# Create hospital observation process +hosp_obs = Counts( + ascertainment_rate_rv=ihr_rv, + delay_distribution_rv=hosp_delay_rv, + noise=NegativeBinomialNoise(hosp_concentration_rv), +) + +print("Hospital observation:") +print(f" Infection resolution: {hosp_obs.infection_resolution()}") +print(f" Delay PMF length: {len(inf_to_hosp_pmf)} days") +``` + +### Wastewater Concentrations + +#### Wastewater Observation Process + +The `Measurements` base class handles continuous observation processes. Domain-specific +implementations subclass it and implement `_predicted_obs()` to transform infections +into predicted values. See `observation_processes_measurements.qmd` for a detailed tutorial. + +```{python} +class Wastewater(Measurements): + """ + Wastewater viral concentration observation process. + + Transforms site-level infections into predicted log-concentrations + via shedding kinetics convolution and genome/volume scaling. + """ + + def __init__( + self, + shedding_kinetics_rv: RandomVariable, + log10_genome_per_infection_rv: RandomVariable, + ml_per_person_per_day: float, + noise: MeasurementNoise, + ) -> None: + super().__init__(temporal_pmf_rv=shedding_kinetics_rv, noise=noise) + self.log10_genome_per_infection_rv = log10_genome_per_infection_rv + self.ml_per_person_per_day = ml_per_person_per_day + + def validate(self) -> None: + shedding_pmf = self.temporal_pmf_rv() + self._validate_pmf(shedding_pmf, "shedding_kinetics_rv") + self.noise.validate() + + def lookback_days(self) -> int: + return len(self.temporal_pmf_rv()) + + def _predicted_obs(self, infections: ArrayLike) -> ArrayLike: + shedding_pmf = self.temporal_pmf_rv() + log10_genome = self.log10_genome_per_infection_rv() + + def convolve_site(site_infections): + convolved, _ = self._convolve_with_alignment( + site_infections, shedding_pmf, p_observed=1.0 + ) + return convolved + + shedding_signal = jax.vmap(convolve_site, in_axes=1, out_axes=1)( + infections + ) + genome_copies = 10**log10_genome + concentration = ( + shedding_signal * genome_copies / self.ml_per_person_per_day + ) + return jnp.log(concentration) +``` + +#### Wastewater Data + +For the wastewater data, we use a simulated dataset for California with realistic noise patterns +that covers the same time period. + +```{python} +# Load wastewater data for California +ca_ww_data = datasets.load_wastewater_data_for_state("CA", "fake_nwss.csv") + +ww_conc = ca_ww_data["observed_conc"] # log concentrations +ww_site_ids = ca_ww_data["site_ids"] +ww_time_indices = ca_ww_data["time_indices"] +ww_n_sites = ca_ww_data["n_sites"] +ww_n_obs = ca_ww_data["n_obs"] +ww_wwtp_names = ca_ww_data["wwtp_names"] + +print("State: California") +print(f"Number of sites: {ww_n_sites}") +print(f"Number of observations: {ww_n_obs}") +print(f"Date range: {ca_ww_data['dates'][0]} to {ca_ww_data['dates'][-1]}") +print( + f"Time index range: {int(ww_time_indices.min())} to {int(ww_time_indices.max())}" +) +print("\nSites:") +for i, name in enumerate(ww_wwtp_names[:5]): + print(f" {i}: {name}") +if ww_n_sites > 5: + print(f" ... and {ww_n_sites - 5} more") +``` + +Wastewater observations are site-level: each measurement is associated with a specific measurement site. The Wastewater observation process uses LogNormalNoise, which takes hierarchical priors for the site-level mode and standard deviation parameters. This enables partial pooling across measurement sites. + +Here we specify HierarchicalNormalPrior for the site-level mode and GammaGroupSdPrior for the standard deviation. + +```{python} +# Viral shedding kinetics PMF (days post-infection) +shedding_pmf = jnp.array( + [ + 0.0, + 0.02, + 0.08, + 0.15, + 0.20, + 0.18, + 0.14, + 0.10, + 0.06, + 0.04, + 0.02, + 0.01, + ] +) +shedding_pmf = shedding_pmf / shedding_pmf.sum() # normalize +shedding_rv = DeterministicPMF("shedding_kinetics", shedding_pmf) + +# Log10 genomes shed per infection +log10_genome_rv = DeterministicVariable("log10_genome_per_inf", 9.0) + +# Wastewater volume per person per day (mL) +ml_per_person_per_day = 1000.0 + +# Hierarchical priors for site-level effects +site_mode_prior = HierarchicalNormalPrior( + "ww_site_mode", sd_rv=DeterministicVariable("site_mode_sd", 0.5) +) +site_sd_prior = GammaGroupSdPrior( + "ww_site_sd", + sd_mean_rv=DeterministicVariable("site_sd_mean", 0.3), + sd_concentration_rv=DeterministicVariable("site_sd_conc", 4.0), +) + +# Create wastewater observation process +ww_obs = Wastewater( + shedding_kinetics_rv=shedding_rv, + log10_genome_per_infection_rv=log10_genome_rv, + ml_per_person_per_day=ml_per_person_per_day, + noise=HierarchicalNormalNoise(site_mode_prior, site_sd_prior), +) + +print("Wastewater observation:") +print(f" Infection resolution: {ww_obs.infection_resolution()}") +print(f" Shedding PMF length: {len(shedding_pmf)} days") +``` + +## Model Building + +We instantiate a `ModelBuilder` object which handles the composition of the latent infection process and the observation process. + +```{python} +# Build the multi-signal model +builder = ModelBuilder() +``` + +The `ModelBuilder` object has 3 key methods: + +- `configure_latent` +- `add_observation` +- `build` + +Methods `configure_latent` and `add_observation` can be called in any order. Method `build` is called once all processes have been specified in the model. + +### Configuring the Latent Process + +We use `configure_latent` to specify the **model structure**: generation interval, initial infections, and temporal dynamics. + +```{python} +print("Latent process configuration:") +print(f" Generation interval length: {len(gen_int_rv())} days") + +builder.configure_latent( + HierarchicalInfections, + gen_int_rv=gen_int_rv, + I0_rv=I0_rv, + initial_log_rt_rv=initial_log_rt_rv, + baseline_temporal=baseline_temporal, + subpop_temporal=subpop_temporal, +) +``` + +### Specifying the Observation Processes, Data + +The name passed to add_observation() becomes the keyword used to pass that observation's data to model.fit() (e.g., hospital={...}, wastewater={...}). + +```{python} +builder.add_observation(hosp_obs, "hospital") +builder.add_observation(ww_obs, "wastewater") +model = builder.build() + +n_init = model.latent.n_initialization_points +print("Model built successfully") +print(f" n_initialization_points: {n_init}") +print(f" Latent process: {type(model.latent).__name__}") +print(f" Observation processes: {list(model.observations.keys())}") +``` + +## Fitting the Model to Data: `model.fit()` + +When you call `model.fit()`, you supply two types of information: + +- **Observation data** — one data dictionary per registered observation process +- **Population structure** — how the jurisdiction is divided into subpopulations + +### Observation Data by Signal Type + +Each observation process is registered with a name via `add_observation()`. That name becomes the keyword argument for passing data to `model.fit()`: + +```python +builder.add_observation(hosp_obs, "hospital") # → hospital={...} +builder.add_observation(ww_obs, "wastewater") # → wastewater={...} +``` + +#### Jurisdiction-level signals + +The jurisdiction-level hospital admissions data is specified as a `Counts` observations process. + +```python +hospital={ + "obs": jnp.array([...]), # observed values +} +``` + +*Note*: this example uses dense data, i.e., there is an observation for every day in the timeline. Were the data sparse, it would be necessary to also provide an array `times` which supplies the time index for each observation. + +#### Subpopulation-level signal + +The subpopulation-level wastewater data is specified as a `Wastewater` observations process. + +```python +wastewater={ + "obs": jnp.array([...]), # observed log concentrations + "times": jnp.array([...]), # time index for each observation + "subpop_indices": jnp.array([...]), # which catchment (selects infection trajectory) + "sensor_indices": jnp.array([...]), # which WWTP/lab pair (selects noise parameters) + "n_sensors": int, # total number of WWTP/lab pairs +} +``` + +A **subpopulation** is a portion of the jurisdiction's population defined by a catchment area. A **sensor** is a measurement source — typically a WWTP/lab pair — that produces observations. Multiple sensors can observe the same subpopulation (e.g., different labs processing samples from the same catchment), so `subpop_indices` and `sensor_indices` may differ. + +- `subpop_indices` links each observation to the appropriate infection trajectory +- `sensor_indices` selects which sensor's noise parameters (mode and sd) to apply + +**Example**: A jurisdiction has 3 catchments (subpopulations 0, 1, 2). Catchment 0 has samples processed by two labs (sensors 0 and 1). Observations from catchment 0 all have `subpop_indices=0` but may have `sensor_indices=0` or `sensor_indices=1` depending on which lab processed the sample. + +### Population Structure + +Population structure is specified via two arrays: + +```python +model.fit( + obs_fractions=jnp.array([0.10, 0.14, 0.21, 0.22, 0.07]), # 5 observed + unobs_fractions=jnp.array([0.26]), # 1 unobserved + ... +) +``` + +This specifies 5 observed subpopulations (e.g., wastewater catchments) and 1 unobserved (remainder of jurisdiction). The fractions must sum to 1.0. + + +### Example `model.fit()` Call + +```python +model.fit( + n_days_post_init=n_days, + population_size=population_size, + # Population structure (choose one parameterization) + obs_fractions=obs_subpop_fractions, + unobs_fractions=unobs_subpop_fractions, + # Observation data (keyed by names from add_observation) + hospital={"obs": hosp_counts, "times": hosp_times}, + wastewater={ + "obs": ww_conc, + "times": ww_times, + "subpop_indices": ww_subpop_indices, + "sensor_indices": ww_sensor_indices, + "n_sensors": n_ww_sensors, + }, +) +``` + +## Running the Model + +First we declare the population structure. + +```{python} +# 5 observed subpopulations (wastewater catchments) +obs_subpop_fractions = jnp.array([0.10, 0.14, 0.21, 0.22, 0.07]) + +# 1 unobserved subpopulation (remainder of jurisdiction) +unobs_subpop_fractions = jnp.array([0.26]) + +K_obs = len(obs_subpop_fractions) + +print(f"Observed subpopulations (K_obs): {K_obs}") +print(f"Unobserved subpopulations: {len(unobs_subpop_fractions)}") +print(f"Observed coverage: {float(jnp.sum(obs_subpop_fractions)):.0%}") +print( + f"Total coverage: {float(jnp.sum(obs_subpop_fractions) + jnp.sum(unobs_subpop_fractions)):.0%}" +) +``` + +It is necessary to align the data with the model's time indexing. + +- Times are relative to the start of the simulation (0 = first day of initialization period) +- Observation times should be offset by `n_initialization_points` + +To do this, we define function `prepare_observation-data`. + +```{python} +def prepare_observation_data(n_days_fit, n_init, hosp_admits, ww_data, K_obs): + """ + Prepare observation data for fitting. + + Returns hospital and wastewater data dicts aligned with model time indexing. + + Parameters + ---------- + n_days_fit : int + Number of days to include in fit + n_init : int + Number of initialization points (for time offset) + hosp_admits : array + Hospital admissions time series + ww_data : dict + Wastewater data dictionary + K_obs : int + Number of observed subpopulations (for mapping sites to subpops) + """ + # Hospital: jurisdiction-level, sparse times + hosp_times = jnp.arange(n_days_fit) + n_init + hosp_counts = hosp_admits[:n_days_fit] + + # Wastewater: sensor-level, sparse times + # Filter to observations within the fitting window + ww_mask = ww_data["time_indices"] < n_days_fit + ww_times = ww_data["time_indices"][ww_mask] + n_init + ww_conc = ww_data["observed_conc"][ww_mask] + ww_sensors = ww_data["site_ids"][ww_mask] + + # Map wastewater sensors to subpopulation indices + # For this demo, we cycle through the K_obs observed subpopulations + n_ww_sensors = ww_data["n_sites"] + sensor_to_subpop = {i: i % K_obs for i in range(n_ww_sensors)} + ww_subpop_indices = jnp.array( + [sensor_to_subpop[int(s)] for s in ww_sensors] + ) + + return { + "hospital": { + "obs": hosp_counts, + "times": hosp_times, + }, + "wastewater": { + "obs": ww_conc, + "times": ww_times, + "subpop_indices": ww_subpop_indices, + "sensor_indices": ww_sensors, + "n_sensors": n_ww_sensors, + }, + } +``` + +### Fit: 90 Days + +Putting this altogether, we align the data with the model time and call `model.fit()`. +We run 4 sampler chains. + +```{python} +# Clear JAX caches to avoid interference from earlier cells +jax.clear_caches() + +n_days_90 = 90 +obs_data_90 = prepare_observation_data( + n_days_90, n_init, hosp_admits, ca_ww_data, K_obs +) + +# Try non-centered reparameterization for hierarchical parameters +reparam_config = { + "log_rt_baseline_noise": LocScaleReparam(centered=0), + "subpop_deviations_increments": LocScaleReparam(centered=0), + "ww_site_mode": LocScaleReparam(centered=0), +} + +print(f"Fitting model with {n_days_90} days of data...") +print(" This may take a few minutes...") + +start_time = time.time() +mcmc_90 = model.fit( + n_days_post_init=n_days_90, + population_size=population_size, + obs_fractions=obs_subpop_fractions, + unobs_fractions=unobs_subpop_fractions, + num_warmup=500, + num_samples=500, + num_chains=4, + rng_key=make_rng_key(), + reparam_config=reparam_config, + **obs_data_90, + progress_bar=False, +) +# JAX uses asynchronous dispatch, so we must block until sampling completes +# to get accurate timing +samples_90 = mcmc_90.get_samples() +jax.block_until_ready(samples_90) +elapsed_90 = time.time() - start_time +print(f"Elapsed time: {elapsed_90:.1f} seconds") +``` + +```{python} +lat_inf_90 = samples_90["latent_infections"] +print(f"Posterior samples shape: {lat_inf_90.shape}") +print(f"Mean latent infections: {np.mean(lat_inf_90):,.0f}") +``` + +We check that the chains have converged and the number of effective samples. + +```{python} +# ArviZ diagnostics for 90-day fit +def filter_samples_for_arviz(samples, n_init, num_chains=2): + """Slice time-series samples to exclude initialization period and reshape for chains.""" + num_samples_per_chain = len(list(samples.values())[0]) // num_chains + filtered = {} + for k, v in samples.items(): + if v.ndim == 2 and v.shape[1] > n_init: + # Time-series: slice to post-init period, reshape to (chains, draws, time) + filtered[k] = v[:, n_init:].reshape( + num_chains, num_samples_per_chain, -1 + ) + elif v.ndim == 1: + # Scalar: reshape to (chains, draws) + filtered[k] = v.reshape(num_chains, num_samples_per_chain) + else: + # Multi-dim: reshape to (chains, draws, ...) + filtered[k] = v.reshape( + num_chains, num_samples_per_chain, *v.shape[1:] + ) + return filtered + + +samples_90_filtered = filter_samples_for_arviz( + samples_90, n_init, num_chains=2 +) +idata_90 = az.from_dict(posterior=samples_90_filtered) +az.summary(idata_90, var_names=["~latent_infections", "~expected"]) +``` + +We extract the posterior quantiles and print summary statistics. + +```{python} +def extract_posterior_quantiles(samples, n_init, n_days): + """Extract posterior quantiles for latent infections.""" + latent_inf = np.array(samples["latent_infections"])[ + :, n_init : n_init + n_days + ] + return { + "q05": np.percentile(latent_inf, 5, axis=0), + "q50": np.percentile(latent_inf, 50, axis=0), + "q95": np.percentile(latent_inf, 95, axis=0), + } + + +quantiles_90 = extract_posterior_quantiles(samples_90, n_init, n_days_90) + +# Summary statistics +ci_width_90 = quantiles_90["q95"] - quantiles_90["q05"] +print(f"Posterior summary for {n_days_90} days:") +print(f" Mean 90% CI width: {ci_width_90.mean():,.0f} infections") +print(f" Median infections (day 45): {quantiles_90['q50'][45]:,.0f}") +``` + +Finally, we visualize the posterior latent infections alongside observed hospitalizations. Note that **hospital admissions lag behind infections** by the infection-to-hospitalization delay (mode ~10 days in our delay PMF). When comparing the two panels, peaks in the infection curve should precede corresponding peaks in hospitalizations by roughly 10-14 days. + +```{python} +# Visualize posterior latent infections and observed hospitalizations (90 days) +# Create separate dataframes for faceted plot +infections_df_90 = pd.DataFrame( + { + "day": np.arange(n_days_90), + "median": quantiles_90["q50"], + "q05": quantiles_90["q05"], + "q95": quantiles_90["q95"], + "signal": "Latent Infections", + } +) + +# Add 14-day moving average to smooth noisy daily admissions +hosp_raw_90 = np.array(hosp_admits[:n_days_90], dtype=float) +hosp_ma_90 = ( + pd.Series(hosp_raw_90).rolling(window=14, center=True).mean().values +) + +hosp_df_90 = pd.DataFrame( + { + "day": np.arange(n_days_90), + "median": hosp_ma_90, + "raw": hosp_raw_90, + "q05": np.nan, + "q95": np.nan, + "signal": "Hospital Admissions (14-day MA)", + } +) + +plot_df_90 = pd.concat([infections_df_90, hosp_df_90], ignore_index=True) +plot_df_90["signal"] = pd.Categorical( + plot_df_90["signal"], + categories=["Latent Infections", "Hospital Admissions (14-day MA)"], + ordered=True, +) + +( + p9.ggplot(plot_df_90, p9.aes(x="day")) + + p9.geom_ribbon( + p9.aes(ymin="q05", ymax="q95"), + fill="steelblue", + alpha=0.3, + ) + + p9.geom_point( + p9.aes(y="raw"), + color="gray", + alpha=0.3, + size=1, + ) + + p9.geom_line( + p9.aes(y="median"), + color="darkblue", + size=1, + ) + + p9.facet_wrap("~signal", ncol=1, scales="free_y") + + p9.labs( + x="Day", + y="Count", + title="Posterior Latent Infections vs Observed Hospitalizations (90 days)", + ) + + p9.theme_grey() + + p9.theme(figure_size=(12, 8)) +) +``` + +### Fit: 180 Days + +```{python} +# Clear JAX caches to avoid interference +jax.clear_caches() + +n_days_180 = 180 +obs_data_180 = prepare_observation_data( + n_days_180, n_init, hosp_admits, ca_ww_data, K_obs +) + +print(f"Fitting model with {n_days_180} days of data...") +print(" This may take a few minutes...") + +start_time = time.time() +mcmc_180 = model.fit( + n_days_post_init=n_days_180, + population_size=population_size, + obs_fractions=obs_subpop_fractions, + unobs_fractions=unobs_subpop_fractions, + num_warmup=1000, + num_samples=500, + num_chains=4, + rng_key=make_rng_key(), + **obs_data_180, + progress_bar=False, +) +# Block until sampling completes for accurate timing +samples_180 = mcmc_180.get_samples() +jax.block_until_ready(samples_180) +elapsed_180 = time.time() - start_time +print(f"Elapsed time: {elapsed_180:.1f} seconds") +``` + +```{python} +lat_inf_180 = samples_180["latent_infections"] +print(f"Posterior samples shape: {lat_inf_180.shape}") +print(f"Mean latent infections: {np.mean(lat_inf_180):,.0f}") +``` + +We check the model fit, as before. + +```{python} +# ArviZ diagnostics for 180-day fit +samples_180_filtered = filter_samples_for_arviz( + samples_180, n_init, num_chains=2 +) +idata_180 = az.from_dict(posterior=samples_180_filtered) +az.summary(idata_180, var_names=["~latent_infections", "~expected"]) +``` + +```{python} +quantiles_180 = extract_posterior_quantiles(samples_180, n_init, n_days_180) + +ci_width_180 = quantiles_180["q95"] - quantiles_180["q05"] +print(f"Posterior summary for {n_days_180} days:") +print(f" Mean 90% CI width: {ci_width_180.mean():,.0f} infections") +print(f" Median infections (day 90): {quantiles_180['q50'][90]:,.0f}") +``` + +```{python} +# Visualize posterior latent infections and observed hospitalizations (180 days) +infections_df_180 = pd.DataFrame( + { + "day": np.arange(n_days_180), + "median": quantiles_180["q50"], + "q05": quantiles_180["q05"], + "q95": quantiles_180["q95"], + "signal": "Latent Infections", + } +) + +# Add 14-day moving average to smooth noisy daily admissions +hosp_raw_180 = np.array(hosp_admits[:n_days_180], dtype=float) +hosp_ma_180 = ( + pd.Series(hosp_raw_180).rolling(window=14, center=True).mean().values +) + +hosp_df_180 = pd.DataFrame( + { + "day": np.arange(n_days_180), + "median": hosp_ma_180, + "raw": hosp_raw_180, + "q05": np.nan, + "q95": np.nan, + "signal": "Hospital Admissions (14-day MA)", + } +) + +plot_df_180 = pd.concat([infections_df_180, hosp_df_180], ignore_index=True) +plot_df_180["signal"] = pd.Categorical( + plot_df_180["signal"], + categories=["Latent Infections", "Hospital Admissions (14-day MA)"], + ordered=True, +) + +( + p9.ggplot(plot_df_180, p9.aes(x="day")) + + p9.geom_ribbon( + p9.aes(ymin="q05", ymax="q95"), + fill="steelblue", + alpha=0.3, + ) + + p9.geom_point( + p9.aes(y="raw"), + color="gray", + alpha=0.3, + size=1, + ) + + p9.geom_line( + p9.aes(y="median"), + color="darkblue", + size=1, + ) + + p9.facet_wrap("~signal", ncol=1, scales="free_y") + + p9.labs( + x="Day", + y="Count", + title="Posterior Latent Infections vs Observed Hospitalizations (180 days)", + ) + + p9.theme_grey() + + p9.theme(figure_size=(12, 8)) +) +``` + +### Comparing 90-Day vs 180-Day Fits + +Comparing the two fits reveals where uncertainty reduction occurs—and why it matters for forecasting. + +```{python} +# Compare CI widths for the overlapping 90-day period +ci_width_90_overlap = quantiles_90["q95"] - quantiles_90["q05"] +ci_width_180_overlap = ( + quantiles_180["q95"][:n_days_90] - quantiles_180["q05"][:n_days_90] +) + +# Compute difference in CI widths +ci_diff = ci_width_90_overlap - ci_width_180_overlap +ci_ratio = ci_width_90_overlap / ci_width_180_overlap + +print("CI Width Comparison (first 90 days):") +print( + f" 90-day fit mean CI width: {ci_width_90_overlap.mean():,.0f} infections" +) +print( + f" 180-day fit mean CI width: {ci_width_180_overlap.mean():,.0f} infections" +) +print(f" Mean difference: {ci_diff.mean():,.0f} infections") +print(f" Mean ratio (90/180): {ci_ratio.mean():.2f}x") +print( + f"\nThe 180-day fit has {(1 - ci_width_180_overlap.mean() / ci_width_90_overlap.mean()) * 100:.1f}% narrower CIs on average" +) + +# Per-day comparison +print("\nCI width by time period:") +for start, end, label in [ + (0, 30, "Days 0-30"), + (30, 60, "Days 30-60"), + (60, 90, "Days 60-90"), +]: + mean_90 = ci_width_90_overlap[start:end].mean() + mean_180 = ci_width_180_overlap[start:end].mean() + reduction = (1 - mean_180 / mean_90) * 100 + print( + f" {label}: 90-day={mean_90:,.0f}, 180-day={mean_180:,.0f}, reduction={reduction:.1f}%" + ) +``` + +Notice that the uncertainty reduction is concentrated in days 60-90—the final month of the 90-day window. Earlier periods (days 0-60) show little change because both fits have sufficient future data to constrain those estimates. + +This pattern has a direct implication for forecasting: **renewal models are most uncertain at the edge of the observation window**. Future observations constrain past latent infections through the renewal equation, but when predicting beyond available data, this constraint disappears. The high uncertainty in days 60-90 of the 90-day fit is exactly what we'd expect when forecasting 30 days ahead—there's no future signal to anchor the estimates. + +## Summary + +This tutorial demonstrated composing a multi-signal renewal model using `ModelBuilder`: + +1. **Configure latent process** (`configure_latent`): generation interval, initial infections, temporal dynamics +2. **Add observation processes** (`add_observation`): each declares its infection resolution and gets a name for data binding +3. **Build and fit** (`build`, `model.fit`): the model routes infections to observations based on resolution and runs NUTS inference + +### Key Concepts + +- **Two-part structure**: Renewal models separate latent infection dynamics from observation processes +- **Infection resolution**: Observation processes declare whether they need aggregate or subpop-level infections +- **Data routing**: `ModelBuilder` automatically routes infection trajectories to the appropriate observation processes +- **Time alignment**: Observations must be offset by `n_initialization_points` to align with model time + +### Next Steps + +- Explore different temporal processes for $R(t)$ in the [Hierarchical Latent Infections](latent_hierarchical_infections.qmd) tutorial +- Learn about count-based observation models in [Observation Processes: Counts](observation_processes_counts.qmd) +- Learn about continuous measurement models in [Observation Processes: Measurements](observation_processes_measurements.qmd) diff --git a/docs/tutorials/latent_hierarchical_infections.qmd b/docs/tutorials/latent_hierarchical_infections.qmd index 8bd2e746..ca362a6b 100644 --- a/docs/tutorials/latent_hierarchical_infections.qmd +++ b/docs/tutorials/latent_hierarchical_infections.qmd @@ -23,8 +23,8 @@ jupyter: --- ```{python} -#| label: setup -#| output: false +# | label: setup +# | output: false import jax.numpy as jnp import jax.random as random import numpy as np @@ -33,10 +33,12 @@ from numpyro.infer import Predictive import pandas as pd import plotnine as p9 import warnings + warnings.filterwarnings("ignore") ``` ```{python} +# | label: imports from pyrenew.latent import ( HierarchicalInfections, AR1, @@ -91,6 +93,7 @@ I0_rv = DistributionalVariable("I0", dist.Beta(1, 1000)) We define subpopulations as fractions of the total jurisdiction population: ```{python} +# | label: population-structure # 5 observed subpopulations obs_fractions = jnp.array([0.10, 0.14, 0.21, 0.22, 0.07]) @@ -111,6 +114,7 @@ Note: Population structure is passed at **sample time**, not when constructing t The **generation interval** is the probability distribution over the time between infection of a primary case and infection of a secondary case. It determines the "memory" of the renewal process—how current infections depend on past infections. ```{python} +# | label: gen-interval-define # Typical COVID-like generation interval gen_int_pmf = jnp.array([0.16, 0.32, 0.25, 0.14, 0.07, 0.04, 0.02]) gen_int_rv = DeterministicPMF("gen_int", gen_int_pmf) @@ -128,6 +132,7 @@ print(f"Mean generation time: {mean_gen_time:.1f} days") The generation interval is typically fixed based on pathogen biology, not estimated from data. ```{python} +# | label: gen-interval-compare # Compare generation intervals: short vs long gi_short = jnp.array([0.4, 0.4, 0.2]) # Mean ~1 day gi_long = jnp.array([0.05, 0.1, 0.2, 0.3, 0.2, 0.1, 0.05]) # Mean ~3.5 days @@ -135,14 +140,19 @@ gi_long = jnp.array([0.05, 0.1, 0.2, 0.3, 0.2, 0.1, 0.05]) # Mean ~3.5 days gi_short_rv = DeterministicPMF("gi_short", gi_short) gi_long_rv = DeterministicPMF("gi_long", gi_long) -print(f"Short GI mean: {float(np.sum(np.arange(len(gi_short)) * gi_short)):.1f} days") -print(f"Long GI mean: {float(np.sum(np.arange(len(gi_long)) * gi_long)):.1f} days") +print( + f"Short GI mean: {float(np.sum(np.arange(len(gi_short)) * gi_short)):.1f} days" +) +print( + f"Long GI mean: {float(np.sum(np.arange(len(gi_long)) * gi_long)):.1f} days" +) ``` To illustrate, we sample infection trajectories using each generation interval while holding other inputs fixed (I0 = 0.001, DifferencedAR1 baseline, AR1 deviations) and using the same random seed so the underlying Rt trajectory is identical. Solid lines use `initial_log_rt = 0` (Rt starts at 1); dashed lines show `initial_log_rt = ±0.2` (Rt starts growing or declining) with the standard GI: ```{python} -#| fig-cap: Effect of generation interval and initial_log_rt on infection dynamics. Solid lines start at Rt=1; dashed lines show initial growth (Rt≈1.22) or decline (Rt≈0.82). +# | label: fig-gi-effect +# | fig-cap: Effect of generation interval and initial_log_rt on infection dynamics. Solid lines start at Rt=1; dashed lines show initial growth (Rt≈1.22) or decline (Rt≈0.82). # Create models with different generation intervals baseline_temporal = DifferencedAR1(autoreg=0.9, innovation_sd=0.05) @@ -175,15 +185,20 @@ for name, gi_rv in gi_configs.items(): n_init = model.n_initialization_points inf = np.array(inf_agg)[n_init:] for d in range(len(inf)): - gi_data.append({ - "day": d, - "infections": float(inf[d]), - "config": name, - "linetype": "solid", - }) + gi_data.append( + { + "day": d, + "infections": float(inf[d]), + "config": name, + "linetype": "solid", + } + ) # initial_log_rt comparison with standard GI -for init_rt_name, init_rt_val in [("Rt≈1.22 (growing)", 0.2), ("Rt≈0.82 (declining)", -0.2)]: +for init_rt_name, init_rt_val in [ + ("Rt≈1.22 (growing)", 0.2), + ("Rt≈0.82 (declining)", -0.2), +]: model = HierarchicalInfections( gen_int_rv=gen_int_rv, I0_rv=DeterministicVariable("I0", 0.001), @@ -200,17 +215,22 @@ for init_rt_name, init_rt_val in [("Rt≈1.22 (growing)", 0.2), ("Rt≈0.82 (dec n_init = model.n_initialization_points inf = np.array(inf_agg)[n_init:] for d in range(len(inf)): - gi_data.append({ - "day": d, - "infections": float(inf[d]), - "config": f"Standard GI, {init_rt_name}", - "linetype": "dashed", - }) + gi_data.append( + { + "day": d, + "infections": float(inf[d]), + "config": f"Standard GI, {init_rt_name}", + "linetype": "dashed", + } + ) gi_df = pd.DataFrame(gi_data) ( - p9.ggplot(gi_df, p9.aes(x="day", y="infections", color="config", linetype="linetype")) + p9.ggplot( + gi_df, + p9.aes(x="day", y="infections", color="config", linetype="linetype"), + ) + p9.geom_line(size=1) + p9.scale_linetype_manual( values={"solid": "solid", "dashed": "dashed"}, @@ -229,12 +249,16 @@ gi_df = pd.DataFrame(gi_data) ``` ```{python} -#| fig-cap: Zoomed view of days 14-21 showing separation between trajectories. +# | label: fig-gi-effect-zoom +# | fig-cap: Zoomed view of days 14-21 showing separation between trajectories. gi_df_zoom = gi_df[(gi_df["day"] >= 14) & (gi_df["day"] <= 21)] ( - p9.ggplot(gi_df_zoom, p9.aes(x="day", y="infections", color="config", linetype="linetype")) + p9.ggplot( + gi_df_zoom, + p9.aes(x="day", y="infections", color="config", linetype="linetype"), + ) + p9.geom_line(size=1) + p9.scale_linetype_manual( values={"solid": "solid", "dashed": "dashed"}, @@ -263,9 +287,12 @@ The flat period for solid lines also reflects DifferencedAR1 with high autoregre `I0` is the proportion of the population infected at time zero. It seeds the renewal equation and scales the entire trajectory: ```{python} +# | label: initial-conditions # Default values I0_rv = DeterministicVariable("I0", 0.001) # 0.1% initially infected -initial_log_rt_rv = DeterministicVariable("initial_log_rt", 0.0) # Rt starts at 1.0 +initial_log_rt_rv = DeterministicVariable( + "initial_log_rt", 0.0 +) # Rt starts at 1.0 print(f"I0 = 0.001 means {0.001:.1%} of population initially infected") print(f"initial_log_rt = 0.0 means Rt starts at {np.exp(0.0):.1f}") @@ -274,7 +301,8 @@ print(f"initial_log_rt = 0.0 means Rt starts at {np.exp(0.0):.1f}") In the previous plot, `I0` was specified as: `I0_rv=DeterministicVariable("I0", 0.001)`, i.e., $1%$ of the population. We overplot trajectories for $10%$ and $0.1%$ as well. ```{python} -#| fig-cap: Effect of I0 on infection scale. Higher I0 shifts the entire trajectory up proportionally. +# | label: fig-i0-effect +# | fig-cap: Effect of I0 on infection scale. Higher I0 shifts the entire trajectory up proportionally. i0_configs = { "I0 = 0.0001 (0.01%)": 0.0001, @@ -350,6 +378,7 @@ The temporal process determines how $R_{\text{baseline}}(t)$ and the deviations Now let's assemble all the inputs: ```{python} +# | label: temporal-config # Baseline Rt: DifferencedAR1 allows trends while stabilizing growth rate baseline_temporal = DifferencedAR1(autoreg=0.9, innovation_sd=0.05) @@ -365,6 +394,7 @@ subpop_temporal = AR1(autoreg=0.8, innovation_sd=0.05) ### Creating the Process ```{python} +# | label: create-process latent_infections = HierarchicalInfections( gen_int_rv=gen_int_rv, I0_rv=I0_rv, @@ -373,8 +403,12 @@ latent_infections = HierarchicalInfections( subpop_temporal=subpop_temporal, ) -print(f"Generation interval length: {latent_infections.get_required_lookback()} days") -print(f"Initialization points: {latent_infections.n_initialization_points} days") +print( + f"Generation interval length: {latent_infections.get_required_lookback()} days" +) +print( + f"Initialization points: {latent_infections.n_initialization_points} days" +) ``` ## Prior Predictive Checks: What Does the Model Believe? @@ -384,6 +418,7 @@ Now we examine what infection dynamics this configuration considers plausible— ### Single Sample ```{python} +# | label: single-sample n_days = 28 # 4 weeks of simulation with numpyro.handlers.seed(rng_seed=42): @@ -403,7 +438,9 @@ print(f" Aggregate (jurisdiction total): {infections_agg.shape}") print(f" All subpopulations: {infections_all.shape}") print(f" Observed subpopulations: {infections_obs.shape}") print(f" Unobserved subpopulations: {infections_unobs.shape}") -print(f"\nTimeline: {n_init} initialization days + {n_days} simulation days = {n_total} total") +print( + f"\nTimeline: {n_init} initialization days + {n_days} simulation days = {n_total} total" +) ``` ### Many Samples: The Prior Predictive Distribution @@ -411,6 +448,7 @@ print(f"\nTimeline: {n_init} initialization days + {n_days} simulation days = {n A single sample tells us little. We need many samples to understand the *distribution* of trajectories the model considers plausible. ```{python} +# | label: prior-predictive-sample def sample_model(): """Wrapper for prior predictive sampling.""" return latent_infections.sample( @@ -419,6 +457,7 @@ def sample_model(): unobs_fractions=unobs_fractions, ) + n_samples = 200 predictive = Predictive(sample_model, num_samples=n_samples) prior_samples = predictive(random.PRNGKey(42)) @@ -430,6 +469,7 @@ print(f"Infection samples: {prior_samples['infections_aggregate'].shape}") ### Visualizing the Prior: Baseline Rt Trajectories ```{python} +# | label: rt-trajectories-data # Extract post-initialization Rt trajectories rt_samples = np.array(prior_samples["rt_baseline"])[:, n_init:] @@ -448,7 +488,8 @@ rt_q95 = np.percentile(rt_samples, 95, axis=0) ``` ```{python} -#| fig-cap: Prior predictive distribution of baseline Rt. Each light blue line is one sample from the prior. The model considers all these trajectories plausible before seeing data. +# | label: fig-prior-rt +# | fig-cap: Prior predictive distribution of baseline Rt. Each light blue line is one sample from the prior. The model considers all these trajectories plausible before seeing data. # Cap display for readability (some trajectories explode) rt_cap = 5.0 @@ -472,6 +513,7 @@ n_extreme = np.sum(np.any(rt_samples > rt_cap, axis=1)) ### Interpreting the Prior Predictive ```{python} +# | label: rt-summary-stats print("Rt at end of simulation (day 28):") print(f" 5th percentile: {rt_q05[-1]:.2f}") print(f" Median: {rt_median[-1]:.2f}") @@ -495,6 +537,7 @@ The median stays near 1, but the mean is pulled up by explosive trajectories. Th The prior predictive distribution changes dramatically based on your temporal process choices. Let's compare the three options for baseline Rt. ```{python} +# | label: temporal-process-samples temporal_configs = { "AR(1)": AR1(autoreg=0.9, innovation_sd=0.05), "DifferencedAR1": DifferencedAR1(autoreg=0.9, innovation_sd=0.05), @@ -525,17 +568,20 @@ for name, temporal in temporal_configs.items(): ``` ```{python} +# | label: temporal-comparison-data # Build comparison dataframe comparison_data = [] for name, rt in process_samples.items(): for i in range(n_samples): for d in range(n_days): - comparison_data.append({ - "day": d, - "rt": float(rt[i, d]), - "sample": i, - "process": name, - }) + comparison_data.append( + { + "day": d, + "rt": float(rt[i, d]), + "sample": i, + "process": name, + } + ) comparison_df = pd.DataFrame(comparison_data) comparison_df["process"] = pd.Categorical( comparison_df["process"], @@ -545,7 +591,8 @@ comparison_df["process"] = pd.Categorical( ``` ```{python} -#| fig-cap: Effect of temporal process choice on prior Rt trajectories. AR(1) stays bounded near 1; DifferencedAR1 and RandomWalk can drift further. +# | label: fig-temporal-comparison +# | fig-cap: Effect of temporal process choice on prior Rt trajectories. AR(1) stays bounded near 1; DifferencedAR1 and RandomWalk can drift further. ( p9.ggplot(comparison_df, p9.aes(x="day", y="rt", group="sample")) @@ -565,13 +612,16 @@ comparison_df["process"] = pd.Categorical( ``` ```{python} +# | label: temporal-summary-table print("Rt at day 28 by temporal process:") print(f"{'Process':<18} {'Median':>8} {'Mean':>8} {'95th %':>8} {'Max':>8}") print("-" * 52) for name in ["AR(1)", "DifferencedAR1", "RandomWalk"]: rt = process_samples[name][:, -1] - print(f"{name:<18} {np.median(rt):>8.2f} {np.mean(rt):>8.2f} " - f"{np.percentile(rt, 95):>8.2f} {np.max(rt):>8.1f}") + print( + f"{name:<18} {np.median(rt):>8.2f} {np.mean(rt):>8.2f} " + f"{np.percentile(rt, 95):>8.2f} {np.max(rt):>8.1f}" + ) ``` **Key observations:** @@ -585,6 +635,7 @@ for name in ["AR(1)", "DifferencedAR1", "RandomWalk"]: The same choice applies to subpopulation deviations $\delta_k(t)$. Let's compare AR(1) vs RandomWalk: ```{python} +# | label: subpop-deviation-samples subpop_configs = { "AR(1) deviations": AR1(autoreg=0.8, innovation_sd=0.05), "RandomWalk deviations": RandomWalk(innovation_sd=0.05), @@ -607,29 +658,35 @@ for name, subpop_temporal in subpop_configs.items(): unobs_fractions=unobs_fractions, ) - deviation_samples[name] = Predictive(make_sampler(model), num_samples=n_samples)( - random.PRNGKey(42) - ) + deviation_samples[name] = Predictive( + make_sampler(model), num_samples=n_samples + )(random.PRNGKey(42)) ``` ```{python} +# | label: deviation-data # Plot deviations for first subpopulation dev_data = [] for name in subpop_configs.keys(): - devs = np.array(deviation_samples[name]["subpop_deviations"])[:, n_init:, 0] + devs = np.array(deviation_samples[name]["subpop_deviations"])[ + :, n_init:, 0 + ] for i in range(n_samples): for d in range(n_days): - dev_data.append({ - "day": d, - "deviation": float(devs[i, d]), - "sample": i, - "process": name, - }) + dev_data.append( + { + "day": d, + "deviation": float(devs[i, d]), + "sample": i, + "process": name, + } + ) dev_df = pd.DataFrame(dev_data) ``` ```{python} -#| fig-cap: Subpopulation deviations under different temporal processes. AR(1) deviations decay toward zero; RandomWalk deviations can diverge. +# | label: fig-subpop-deviations +# | fig-cap: Subpopulation deviations under different temporal processes. AR(1) deviations decay toward zero; RandomWalk deviations can diverge. ( p9.ggplot(dev_df, p9.aes(x="day", y="deviation", group="sample")) @@ -659,6 +716,7 @@ Choose based on your scientific question: Do you expect local differences to per The `autoreg` and `innovation_sd` parameters control how constrained the prior is. Let's compare a "weakly informative" vs "more informative" configuration: ```{python} +# | label: tuning-samples configs = { "Weakly informative\n(autoreg=0.9, sd=0.05)": { "baseline": DifferencedAR1(autoreg=0.9, innovation_sd=0.05), @@ -687,29 +745,33 @@ for name, cfg in configs.items(): unobs_fractions=unobs_fractions, ) - tuning_samples[name] = Predictive(make_sampler(model), num_samples=n_samples)( - random.PRNGKey(42) - ) + tuning_samples[name] = Predictive( + make_sampler(model), num_samples=n_samples + )(random.PRNGKey(42)) ``` ```{python} +# | label: tuning-data # Build comparison dataframe tuning_data = [] for name, samples in tuning_samples.items(): rt = np.array(samples["rt_baseline"])[:, n_init:] for i in range(n_samples): for d in range(n_days): - tuning_data.append({ - "day": d, - "rt": float(rt[i, d]), - "sample": i, - "config": name, - }) + tuning_data.append( + { + "day": d, + "rt": float(rt[i, d]), + "sample": i, + "config": name, + } + ) tuning_df = pd.DataFrame(tuning_data) ``` ```{python} -#| fig-cap: Effect of hyperparameter choices on prior Rt distribution. Tighter parameters produce more constrained trajectories. +# | label: fig-tuning-effect +# | fig-cap: Effect of hyperparameter choices on prior Rt distribution. Tighter parameters produce more constrained trajectories. ( p9.ggplot(tuning_df, p9.aes(x="day", y="rt", group="sample")) @@ -739,6 +801,7 @@ tuning_df = pd.DataFrame(tuning_data) Before using `HierarchicalInfections` in a full model, verify the hierarchical structure is working correctly: ```{python} +# | label: verify-deviations # Check that deviations sum to zero (identifiability constraint) with numpyro.handlers.seed(rng_seed=123): with numpyro.handlers.trace() as trace: @@ -750,11 +813,14 @@ with numpyro.handlers.seed(rng_seed=123): deviations = trace["subpop_deviations"]["value"] deviation_sums = jnp.sum(deviations, axis=1) -print(f"Max |sum of deviations|: {float(jnp.max(jnp.abs(deviation_sums))):.2e}") +print( + f"Max |sum of deviations|: {float(jnp.max(jnp.abs(deviation_sums))):.2e}" +) print("(Should be ~0 due to sum-to-zero constraint)") ``` ```{python} +# | label: verify-aggregate # Check that aggregate = weighted sum of subpopulations with numpyro.handlers.seed(rng_seed=123): inf_agg, inf_all, _, _ = latent_infections.sample( @@ -768,7 +834,11 @@ computed_agg = jnp.sum(inf_all * all_fractions[None, :], axis=1) max_diff = float(jnp.max(jnp.abs(computed_agg - inf_agg))) print(f"Max difference (aggregate vs weighted sum): {max_diff:.2e}") -print("Hierarchical structure verified!" if max_diff < 1e-5 else "ERROR: Mismatch detected") +print( + "Hierarchical structure verified!" + if max_diff < 1e-5 + else "ERROR: Mismatch detected" +) ``` ## Summary diff --git a/docs/tutorials/model_builder_helper.qmd b/docs/tutorials/model_builder_helper.qmd new file mode 100644 index 00000000..e2471cef --- /dev/null +++ b/docs/tutorials/model_builder_helper.qmd @@ -0,0 +1,974 @@ +--- +title: 'Model Building: Composing Latent and Observation Processes' +format: + gfm: + fig-width: 16 + fig-height: 10 + html: + toc: true + embed-resources: true + self-contained-math: true + code-fold: true + code-tools: true +engine: jupyter +jupyter: + jupytext: + text_representation: + extension: .qmd + format_name: quarto + format_version: '1.0' + jupytext_version: 1.18.1 + kernelspec: + display_name: Python 3 (ipykernel) + language: python + name: python3 +--- + +```{python} +# | label: setup +# | output: false +import numpyro +from numpyro.infer.reparam import LocScaleReparam + +# to run samplers in parallel you must run `set_host_device_count` before importing jax +numpyro.set_host_device_count(4) +numpyro.enable_x64() +``` + +```{python} +import arviz as az +import jax +import jax.numpy as jnp +import jax.random as random +import numpy as np +import plotnine as p9 +import pandas as pd +import time +import warnings + +warnings.filterwarnings("ignore") + + +def make_rng_key(): + """Generate a time-based random seed.""" + seed = int(time.time() * 1000) % (2**32) + return random.PRNGKey(seed) +``` + +```{python} +import jax +from jax.typing import ArrayLike + +from pyrenew import datasets +from pyrenew.deterministic import ( + DeterministicPMF, + DeterministicVariable, +) +from pyrenew.metaclass import RandomVariable + +from pyrenew.latent import ( + HierarchicalInfections, + AR1, + RandomWalk, + GammaGroupSdPrior, + HierarchicalNormalPrior, +) +from pyrenew.model import ModelBuilder + +from pyrenew.observation import ( + Counts, + HierarchicalNormalNoise, + Measurements, + MeasurementNoise, + NegativeBinomialNoise, +) +``` + +## Overview + +The `pyrenew` package provides a modular architecture for building multi-signal renewal models by composing: + +- **Latent infection process**: Generates unobserved infection trajectories via the renewal equation +- **Observation processes**: Transform latent infections into observable signals (hospital admissions, wastewater, etc.) + +This notebook demonstrates building a renewal model using the following components: + +- `HierarchicalInfections` — subpopulations share a jurisdiction-level baseline $R(t)$ with subpopulation-specific deviations +- `Counts` — hospital admissions (jurisdiction-level) +- `Wastewater` — viral concentrations (site-level) + +The `ModelBuilder` class composes these components and handles initialization requirements automatically. + +## Observation Processes for Hospital Admissions and Wastewater Concentrations + +Observation processes transform latent infections into observable signals and define the likelihood. Each observation process: + +- Declares what **infection resolution** it needs (`"aggregate"` or `"site"`) +- Applies signal-specific transformations (ascertainment, delay convolution, shedding kinetics) +- Defines the noise model for the likelihood + +### Hospital Admissions + +In this example we use a dataset consisting of hospital admissions for COVID-19 across California +for the first 10 months of 2023 (as reported to the CDC). + +```{python} +# Load daily hospital admissions for California +ca_hosp_data = datasets.load_hospital_data_for_state("CA", "2023-11-06.csv") + +hosp_admits = ca_hosp_data["daily_admits"] +population_size = ca_hosp_data["population"] +n_hosp_days = ca_hosp_data["n_days"] + +print("State: California") +print(f"Population: {population_size:,}") +print(f"Date range: {ca_hosp_data['dates'][0]} to {ca_hosp_data['dates'][-1]}") +print(f"Number of days: {n_hosp_days}") +print( + f"Admissions range: {int(hosp_admits.min())} to {int(hosp_admits.max())}" +) +``` + +The hospital admissions data is aggregated at the jurisdiction level, therefore we specify a `Counts` observation process. + +```{python} +# Infection-to-hospitalization delay (COVID-19, from literature) +inf_to_hosp_pmf = jnp.array( + [ + 0, + 0.00469, + 0.01452, + 0.02786, + 0.04237, + 0.05581, + 0.06657, + 0.07379, + 0.07729, + 0.07737, + 0.07465, + 0.06988, + 0.06377, + 0.05696, + 0.04996, + 0.04315, + 0.03677, + 0.03097, + 0.02583, + 0.02135, + 0.01751, + 0.01427, + 0.01156, + 0.00931, + 0.00746, + 0.00596, + 0.00474, + 0.00375, + 0.00296, + 0.00233, + 0.00183, + 0.00143, + 0.00107, + 0.00077, + 0.00054, + 0.00036, + 0.00024, + 0.00015, + 0.00009, + 0.00005, + 0.00003, + 0.00002, + 0.00001, + ] +) +hosp_delay_rv = DeterministicPMF("inf_to_hosp_delay", inf_to_hosp_pmf) + +# IHR: ~1% of infections lead to hospitalization +ihr_rv = DeterministicVariable("ihr", 0.01) + +# Negative binomial concentration (moderate overdispersion) +hosp_concentration_rv = DeterministicVariable("hosp_concentration", 10.0) + +# Create hospital observation process +hosp_obs = Counts( + ascertainment_rate_rv=ihr_rv, + delay_distribution_rv=hosp_delay_rv, + noise=NegativeBinomialNoise(hosp_concentration_rv), +) + +print("Hospital observation:") +print(f" Infection resolution: {hosp_obs.infection_resolution()}") +print(f" Delay PMF length: {len(inf_to_hosp_pmf)} days") +``` + +### Wastewater Concentrations + +#### Wastewater Observation Process + +The `Measurements` base class handles continuous observation processes. Domain-specific +implementations subclass it and implement `_predicted_obs()` to transform infections +into predicted values. See `observation_processes_measurements.qmd` for a detailed tutorial. + +```{python} +class Wastewater(Measurements): + """ + Wastewater viral concentration observation process. + + Transforms site-level infections into predicted log-concentrations + via shedding kinetics convolution and genome/volume scaling. + """ + + def __init__( + self, + shedding_kinetics_rv: RandomVariable, + log10_genome_per_infection_rv: RandomVariable, + ml_per_person_per_day: float, + noise: MeasurementNoise, + ) -> None: + super().__init__(temporal_pmf_rv=shedding_kinetics_rv, noise=noise) + self.log10_genome_per_infection_rv = log10_genome_per_infection_rv + self.ml_per_person_per_day = ml_per_person_per_day + + def validate(self) -> None: + shedding_pmf = self.temporal_pmf_rv() + self._validate_pmf(shedding_pmf, "shedding_kinetics_rv") + self.noise.validate() + + def lookback_days(self) -> int: + return len(self.temporal_pmf_rv()) + + def _predicted_obs(self, infections: ArrayLike) -> ArrayLike: + shedding_pmf = self.temporal_pmf_rv() + log10_genome = self.log10_genome_per_infection_rv() + + def convolve_site(site_infections): + convolved, _ = self._convolve_with_alignment( + site_infections, shedding_pmf, p_observed=1.0 + ) + return convolved + + shedding_signal = jax.vmap(convolve_site, in_axes=1, out_axes=1)( + infections + ) + genome_copies = 10**log10_genome + concentration = ( + shedding_signal * genome_copies / self.ml_per_person_per_day + ) + return jnp.log(concentration) +``` + +#### Wastewater Data + +For the wastewater data, we use a simulated dataset for California with realistic noise patterns +that covers the same time period. + +```{python} +# Load wastewater data for California +ca_ww_data = datasets.load_wastewater_data_for_state("CA", "fake_nwss.csv") + +ww_conc = ca_ww_data["observed_conc"] # log concentrations +ww_site_ids = ca_ww_data["site_ids"] +ww_time_indices = ca_ww_data["time_indices"] +ww_n_sites = ca_ww_data["n_sites"] +ww_n_obs = ca_ww_data["n_obs"] +ww_wwtp_names = ca_ww_data["wwtp_names"] + +print("State: California") +print(f"Number of sites: {ww_n_sites}") +print(f"Number of observations: {ww_n_obs}") +print(f"Date range: {ca_ww_data['dates'][0]} to {ca_ww_data['dates'][-1]}") +print( + f"Time index range: {int(ww_time_indices.min())} to {int(ww_time_indices.max())}" +) +print("\nSites:") +for i, name in enumerate(ww_wwtp_names[:5]): + print(f" {i}: {name}") +if ww_n_sites > 5: + print(f" ... and {ww_n_sites - 5} more") +``` + +Wastewater observations are site-level: each measurement is associated with a specific measurement site. The Wastewater observation process uses LogNormalNoise, which takes hierarchical priors for the site-level mode and standard deviation parameters. This enables partial pooling across measurement sites. + +Here we specify HierarchicalNormalPrior for the site-level mode and GammaGroupSdPrior for the standard deviation. + +```{python} +# Viral shedding kinetics PMF (days post-infection) +shedding_pmf = jnp.array( + [ + 0.0, + 0.02, + 0.08, + 0.15, + 0.20, + 0.18, + 0.14, + 0.10, + 0.06, + 0.04, + 0.02, + 0.01, + ] +) +shedding_pmf = shedding_pmf / shedding_pmf.sum() # normalize +shedding_rv = DeterministicPMF("shedding_kinetics", shedding_pmf) + +# Log10 genomes shed per infection +log10_genome_rv = DeterministicVariable("log10_genome_per_inf", 9.0) + +# Wastewater volume per person per day (mL) +ml_per_person_per_day = 1000.0 + +# Hierarchical priors for site-level effects +site_mode_prior = HierarchicalNormalPrior( + "ww_site_mode", sd_rv=DeterministicVariable("site_mode_sd", 0.5) +) +site_sd_prior = GammaGroupSdPrior( + "ww_site_sd", + sd_mean_rv=DeterministicVariable("site_sd_mean", 0.3), + sd_concentration_rv=DeterministicVariable("site_sd_conc", 4.0), +) + +# Create wastewater observation process +ww_obs = Wastewater( + shedding_kinetics_rv=shedding_rv, + log10_genome_per_infection_rv=log10_genome_rv, + ml_per_person_per_day=ml_per_person_per_day, + noise=HierarchicalNormalNoise(site_mode_prior, site_sd_prior), +) + +print("Wastewater observation:") +print(f" Infection resolution: {ww_obs.infection_resolution()}") +print(f" Shedding PMF length: {len(shedding_pmf)} days") +``` + +## Latent Infection Process + +Latent infection processes implement the renewal equation to generate infection trajectories. All latent processes share common components: + +- **Generation interval**: PMF for secondary infection timing +- **Initial infections (I0)**: Starting condition for the renewal process +- **Temporal dynamics**: How $R(t)$ evolves over time + +### Generation Interval + +The generation interval PMF specifies the probability that a secondary infection occurs $d$ days after the primary infection. + +```{python} +covid_gen_int = [0.16, 0.32, 0.25, 0.14, 0.07, 0.04, 0.02] +gen_int_pmf = jnp.array(covid_gen_int) +gen_int_rv = DeterministicPMF("gen_int", gen_int_pmf) + +# Mean generation time +days = np.arange(len(gen_int_pmf)) +print(f"Generation interval length: {len(gen_int_pmf)} days") +``` + +### I0: Initial Infections + +The initial infections RV `I0_rv` specifies the **proportion of the population infected** at the first observation time. This must be a value in the interval (0, 1\]. + +```{python} +I0_rv = DeterministicVariable("I0", 0.01) +``` + +### Initial Log Rt + +We set the initial log(Rt) to 0.0, meaning Rt starts at 1.0. + +```{python} +initial_log_rt_rv = DeterministicVariable("initial_log_rt", 0.0) +``` + +### Temporal Processes for $R(t)$ + +We configure two temporal processes: + +- **Jurisdiction-level** (`baseline_temporal`): AR(1) process for the baseline $R(t)$ +- **Subpopulation-level** (`subpop_temporal`): RandomWalk for subpopulation deviations + +The RandomWalk allows flexible evolution of subpopulation-specific transmission without mean reversion. + +```{python} +# AR1 provides mean-reverting behavior for baseline Rt +baseline_temporal = AR1(autoreg=0.9, innovation_sd=0.05) + +# RandomWalk allows flexible subpopulation deviations +subpop_temporal = RandomWalk(innovation_sd=0.025) +``` + +## Model Building + +We instantiate a `ModelBuilder` object which handles the composition of the latent infection process and the observation process. + +```{python} +# Build the multi-signal model +builder = ModelBuilder() +``` + +The `ModelBuilder` object has 3 key methods: + +- `configure_latent` +- `add_observation` +- `build` + +Methods `configure_latent` and `add_observation` can be called in any order. Method `build` is called once all processes have been specified in the model. + +### Configuring the Latent Process + +We use `configure_latent` to specify the **model structure**: generation interval, initial infections, and temporal dynamics. + +```{python} +print("Latent process configuration:") +print(f" Generation interval length: {len(gen_int_rv())} days") + +builder.configure_latent( + HierarchicalInfections, + gen_int_rv=gen_int_rv, + I0_rv=I0_rv, + initial_log_rt_rv=initial_log_rt_rv, + baseline_temporal=baseline_temporal, + subpop_temporal=subpop_temporal, +) +``` + +### Specifying the Observation Processes, Data + +The name passed to add_observation() becomes the keyword used to pass that observation's data to model.fit() (e.g., hospital={...}, wastewater={...}). + +```{python} +builder.add_observation(hosp_obs, "hospital") +builder.add_observation(ww_obs, "wastewater") +model = builder.build() + +n_init = model.latent.n_initialization_points +print("Model built successfully!") +print(f" n_initialization_points: {n_init}") +print(f" Latent process: {type(model.latent).__name__}") +print(f" Observation processes: {list(model.observations.keys())}") +``` + +## Fitting the Model to Data: `model.fit()` + +When you call `model.fit()`, you supply two types of information: + +- **Observation data** — one data dictionary per registered observation process +- **Population structure** — how the jurisdiction is divided into subpopulations + +### Data Routing - Mapping Signals to Model Levels + +This is a hierarchical model that estimates parameters at two levels: + +- **Jurisdiction level**: a baseline $R(t)$ trajectory and aggregated infections across the entire jurisdiction +- **Subpopulation level**: deviations from the baseline $R(t)$ for each subpopulation, producing separate infection trajectories + +Different signals observe different levels of this hierarchy. Each observation process declares an **infection resolution** that determines what infection data it receives: + +| Resolution | Receives | Example signals | +|----------------------|-------------------|-------------------------------| +| `"aggregate"` | Aggregated infections (sum across all subpopulations) | Hospital admissions, case counts | +| `"site"` | Infection matrix with one column per observed subpopulation | Wastewater, site-specific surveillance | + +The model routes latent infections to observation processes based on each process's declared resolution: + +```mermaid +flowchart TB + + subgraph Latent["Latent Infection Process"] + L[HierarchicalInfections] + end + + subgraph Infections["Infection Arrays"] + J["infections_aggregate (T,)"] + S["infections_subpop (T, K_obs)"] + end + + subgraph Obs["Observation Processes"] + C[Counts] + W[Wastewater] + end + + subgraph Data["Observed Data"] + HA[Hospital Admissions] + WW[Concentrations] + end + + L --> J + L --> S + J -->|"aggregate"| C + S -->|"site"| W + C -->|"ascertainment × delay"| HA + W -->|"shedding × genome × dilution"| WW +``` + +### Observation Data by Signal Type + +Each observation process is registered with a name via `add_observation()`. That name becomes the keyword argument for passing data to `model.fit()`: + +```python +builder.add_observation(hosp_obs, "hospital") # → hospital={...} +builder.add_observation(ww_obs, "wastewater") # → wastewater={...} +``` + +#### Jurisdiction-level signals + +The jurisdiction-level hospital admissions data is specified as a `Counts` observations process. + +```python +hospital={ + "obs": jnp.array([...]), # observed values +} +``` + +*Note*: this example uses dense data, i.e., there is an obervation for every day in the timeline. Were the data sparse, it would be necessary to also provide an array `times` which supplies the time index for each observation. + +#### Subpopulation-level signal + +The subpopulation-level wastewater data is specified as a `Wastewater` observations process. + +```python +wastewater={ + "obs": jnp.array([...]), # observed log concentrations + "times": jnp.array([...]), # time index for each observation + "subpop_indices": jnp.array([...]), # which catchment (selects infection trajectory) + "sensor_indices": jnp.array([...]), # which WWTP/lab pair (selects noise parameters) + "n_sensors": int, # total number of WWTP/lab pairs +} +``` + +A **subpopulation** is a portion of the jurisdiction's population defined by a catchment area. A **sensor** is a measurement source — typically a WWTP/lab pair — that produces observations. Multiple sensors can observe the same subpopulation (e.g., different labs processing samples from the same catchment), so `subpop_indices` and `sensor_indices` may differ. + +- `subpop_indices` links each observation to the appropriate infection trajectory +- `sensor_indices` selects which sensor's noise parameters (mode and sd) to apply + +**Example**: A jurisdiction has 3 catchments (subpopulations 0, 1, 2). Catchment 0 has samples processed by two labs (sensors 0 and 1). Observations from catchment 0 all have `subpop_indices=0` but may have `sensor_indices=0` or `sensor_indices=1` depending on which lab processed the sample. + +### Population Structure + +Population structure is specified via two arrays: + +```python +model.fit( + obs_fractions=jnp.array([0.10, 0.14, 0.21, 0.22, 0.07]), # 5 observed + unobs_fractions=jnp.array([0.26]), # 1 unobserved + ... +) +``` + +This specifies 5 observed subpopulations (e.g., wastewater catchments) and 1 unobserved (remainder of jurisdiction). The fractions must sum to 1.0. + + +### Example `model.fit()` Call + +```python +model.fit( + n_days_post_init=n_days, + population_size=population_size, + # Population structure (choose one parameterization) + obs_fractions=obs_subpop_fractions, + unobs_fractions=unobs_subpop_fractions, + # Observation data (keyed by names from add_observation) + hospital={"obs": hosp_counts, "times": hosp_times}, + wastewater={ + "obs": ww_conc, + "times": ww_times, + "subpop_indices": ww_subpop_indices, + "sensor_indices": ww_sensor_indices, + "n_sensors": n_ww_sensors, + }, +) +``` + +## Running the Model + +First we declare the population structure. + +```{python} +# 5 observed subpopulations (wastewater catchments) +obs_subpop_fractions = jnp.array([0.10, 0.14, 0.21, 0.22, 0.07]) + +# 1 unobserved subpopulation (remainder of jurisdiction) +unobs_subpop_fractions = jnp.array([0.26]) + +K_obs = len(obs_subpop_fractions) + +print(f"Observed subpopulations (K_obs): {K_obs}") +print(f"Unobserved subpopulations: {len(unobs_subpop_fractions)}") +print(f"Observed coverage: {float(jnp.sum(obs_subpop_fractions)):.0%}") +print( + f"Total coverage: {float(jnp.sum(obs_subpop_fractions) + jnp.sum(unobs_subpop_fractions)):.0%}" +) +``` + +It is necessary to align the data with the model's time indexing. + +- Times are relative to the start of the simulation (0 = first day of initialization period) +- Observation times should be offset by `n_initialization_points` + +To do this, we define function `prepare_observation-data`. + +```{python} +def prepare_observation_data(n_days_fit, n_init, hosp_admits, ww_data, K_obs): + """ + Prepare observation data for fitting. + + Returns hospital and wastewater data dicts aligned with model time indexing. + + Parameters + ---------- + n_days_fit : int + Number of days to include in fit + n_init : int + Number of initialization points (for time offset) + hosp_admits : array + Hospital admissions time series + ww_data : dict + Wastewater data dictionary + K_obs : int + Number of observed subpopulations (for mapping sites to subpops) + """ + # Hospital: jurisdiction-level, sparse times + hosp_times = jnp.arange(n_days_fit) + n_init + hosp_counts = hosp_admits[:n_days_fit] + + # Wastewater: sensor-level, sparse times + # Filter to observations within the fitting window + ww_mask = ww_data["time_indices"] < n_days_fit + ww_times = ww_data["time_indices"][ww_mask] + n_init + ww_conc = ww_data["observed_conc"][ww_mask] + ww_sensors = ww_data["site_ids"][ww_mask] + + # Map wastewater sensors to subpopulation indices + # For this demo, we cycle through the K_obs observed subpopulations + n_ww_sensors = ww_data["n_sites"] + sensor_to_subpop = {i: i % K_obs for i in range(n_ww_sensors)} + ww_subpop_indices = jnp.array( + [sensor_to_subpop[int(s)] for s in ww_sensors] + ) + + return { + "hospital": { + "obs": hosp_counts, + "times": hosp_times, + }, + "wastewater": { + "obs": ww_conc, + "times": ww_times, + "subpop_indices": ww_subpop_indices, + "sensor_indices": ww_sensors, + "n_sensors": n_ww_sensors, + }, + } +``` + +### Fit: 90 Days + +Putting this altogether, we align the data with the model time and call `model.fit()`. +We run 4 sampler chains. + +```{python} +# Clear JAX caches to avoid interference from earlier cells +jax.clear_caches() + +n_days_90 = 90 +obs_data_90 = prepare_observation_data( + n_days_90, n_init, hosp_admits, ca_ww_data, K_obs +) + +# Try non-centered reparameterization for hierarchical parameters +reparam_config = { + "log_rt_baseline_noise": LocScaleReparam(centered=0), + "subpop_deviations_increments": LocScaleReparam(centered=0), + "ww_site_mode": LocScaleReparam(centered=0), +} + +print(f"Fitting model with {n_days_90} days of data...") +print(" This may take a few minutes...") + +start_time = time.time() +mcmc_90 = model.fit( + n_days_post_init=n_days_90, + population_size=population_size, + obs_fractions=obs_subpop_fractions, + unobs_fractions=unobs_subpop_fractions, + num_warmup=500, + num_samples=500, + num_chains=4, + rng_key=make_rng_key(), + reparam_config=reparam_config, + **obs_data_90, + progress_bar=False, +) +elapsed_90 = time.time() - start_time +print(f"Elapsed time: {elapsed_90:.1f} seconds") +``` + +```{python} +samples_90 = mcmc_90.get_samples() +lat_inf_90 = samples_90["latent_infections"] +print(f"\nPosterior samples shape: {lat_inf_90.shape}") +print(f"Mean latent infections: {np.mean(lat_inf_90):,.0f}") +``` + +We check that the chains have converged and the number of effective samples. + +```{python} +# ArviZ diagnostics for 90-day fit +def filter_samples_for_arviz(samples, n_init, num_chains=2): + """Slice time-series samples to exclude initialization period and reshape for chains.""" + num_samples_per_chain = len(list(samples.values())[0]) // num_chains + filtered = {} + for k, v in samples.items(): + if v.ndim == 2 and v.shape[1] > n_init: + # Time-series: slice to post-init period, reshape to (chains, draws, time) + filtered[k] = v[:, n_init:].reshape( + num_chains, num_samples_per_chain, -1 + ) + elif v.ndim == 1: + # Scalar: reshape to (chains, draws) + filtered[k] = v.reshape(num_chains, num_samples_per_chain) + else: + # Multi-dim: reshape to (chains, draws, ...) + filtered[k] = v.reshape( + num_chains, num_samples_per_chain, *v.shape[1:] + ) + return filtered + + +samples_90_filtered = filter_samples_for_arviz( + samples_90, n_init, num_chains=2 +) +idata_90 = az.from_dict(posterior=samples_90_filtered) +az.summary(idata_90, var_names=["~latent_infections", "~expected"]) +``` + +We extract the posterior quantiles and print summary statistics. + +```{python} +def extract_posterior_quantiles(samples, n_init, n_days): + """Extract posterior quantiles for latent infections.""" + latent_inf = np.array(samples["latent_infections"])[ + :, n_init : n_init + n_days + ] + return { + "q05": np.percentile(latent_inf, 5, axis=0), + "q50": np.percentile(latent_inf, 50, axis=0), + "q95": np.percentile(latent_inf, 95, axis=0), + } + + +quantiles_90 = extract_posterior_quantiles(samples_90, n_init, n_days_90) + +# Summary statistics +ci_width_90 = quantiles_90["q95"] - quantiles_90["q05"] +print(f"Posterior summary for {n_days_90} days:") +print(f" Mean 90% CI width: {ci_width_90.mean():,.0f} infections") +print(f" Median infections (day 45): {quantiles_90['q50'][45]:,.0f}") +``` + +Finally, we visualize the posterior latent infections over the 90 day interval. + +```{python} +# Visualize posterior latent infections (90 days) +posterior_df_90 = pd.DataFrame( + { + "day": np.arange(n_days_90), + "median": quantiles_90["q50"], + "q05": quantiles_90["q05"], + "q95": quantiles_90["q95"], + } +) + +obs_plot_df_90 = pd.DataFrame( + { + "day": np.arange(n_days_90), + "admissions": np.array(hosp_admits[:n_days_90]), + } +) + +( + p9.ggplot() + + p9.geom_ribbon( + p9.aes(x="day", ymin="q05", ymax="q95"), + data=posterior_df_90, + fill="steelblue", + alpha=0.3, + ) + + p9.geom_line( + p9.aes(x="day", y="median"), + data=posterior_df_90, + color="darkblue", + size=1, + ) + + p9.geom_point( + p9.aes(x="day", y="admissions"), + data=obs_plot_df_90, + color="darkorange", + size=1, + alpha=0.6, + ) + + p9.labs( + x="Day", + y="Count", + title="Posterior Latent Infections (90 days, multi-signal fit)", + subtitle="Blue: posterior infections (90% CI) | Orange: observed hospital admissions", + ) + + p9.theme_grey() +) +``` + +### Fit: 180 Days + +```{python} +# Clear JAX caches to avoid interference +jax.clear_caches() + +n_days_180 = 180 +obs_data_180 = prepare_observation_data( + n_days_180, n_init, hosp_admits, ca_ww_data, K_obs +) + +print(f"Fitting model with {n_days_180} days of data...") +print(" This may take a few minutes...") + +start_time = time.time() +mcmc_180 = model.fit( + n_days_post_init=n_days_180, + population_size=population_size, + obs_fractions=obs_subpop_fractions, + unobs_fractions=unobs_subpop_fractions, + num_warmup=1000, + num_samples=500, + num_chains=4, + rng_key=make_rng_key(), + **obs_data_180, + progress_bar=False, +) +elapsed_180 = time.time() - start_time +print(f"Elapsed time: {elapsed_180:.1f} seconds") +``` + +```{python} +samples_180 = mcmc_180.get_samples() +lat_inf_180 = samples_180["latent_infections"] +print(f"\nPosterior samples shape: {lat_inf_180.shape}") +print(f"Mean latent infections: {np.mean(lat_inf_180):,.0f}") +``` + +We check the model fit, as before. + +```{python} +# ArviZ diagnostics for 180-day fit +samples_180_filtered = filter_samples_for_arviz( + samples_180, n_init, num_chains=2 +) +idata_180 = az.from_dict(posterior=samples_180_filtered) +az.summary(idata_180, var_names=["~latent_infections", "~expected"]) +``` + +```{python} +quantiles_180 = extract_posterior_quantiles(samples_180, n_init, n_days_180) + +ci_width_180 = quantiles_180["q95"] - quantiles_180["q05"] +print(f"Posterior summary for {n_days_180} days:") +print(f" Mean 90% CI width: {ci_width_180.mean():,.0f} infections") +print(f" Median infections (day 90): {quantiles_180['q50'][90]:,.0f}") +``` + +```{python} +# Visualize posterior latent infections (180 days) +posterior_df_180 = pd.DataFrame( + { + "day": np.arange(n_days_180), + "median": quantiles_180["q50"], + "q05": quantiles_180["q05"], + "q95": quantiles_180["q95"], + } +) + +obs_plot_df_180 = pd.DataFrame( + { + "day": np.arange(n_days_180), + "admissions": np.array(hosp_admits[:n_days_180]), + } +) + +( + p9.ggplot() + + p9.geom_ribbon( + p9.aes(x="day", ymin="q05", ymax="q95"), + data=posterior_df_180, + fill="steelblue", + alpha=0.3, + ) + + p9.geom_line( + p9.aes(x="day", y="median"), + data=posterior_df_180, + color="darkblue", + size=1, + ) + + p9.geom_point( + p9.aes(x="day", y="admissions"), + data=obs_plot_df_180, + color="darkorange", + size=1, + alpha=0.6, + ) + + p9.labs( + x="Day", + y="Count", + title="Posterior Latent Infections (180 days, multi-signal fit)", + subtitle="Blue: posterior infections (90% CI) | Orange: observed hospital admissions", + ) + + p9.theme_grey() +) +``` + +### Comparing 90-Day vs 180-Day Fits + +We can compare how the posterior estimates differ between fitting to 90 days vs 180 days of data. The key insight is how the credible intervals change in the overlapping period (first 90 days) when we have more data informing the model. + +```{python} +# Compare CI widths for the overlapping 90-day period +ci_width_90_overlap = quantiles_90["q95"] - quantiles_90["q05"] +ci_width_180_overlap = ( + quantiles_180["q95"][:n_days_90] - quantiles_180["q05"][:n_days_90] +) + +# Compute difference in CI widths +ci_diff = ci_width_90_overlap - ci_width_180_overlap +ci_ratio = ci_width_90_overlap / ci_width_180_overlap + +print("CI Width Comparison (first 90 days):") +print( + f" 90-day fit mean CI width: {ci_width_90_overlap.mean():,.0f} infections" +) +print( + f" 180-day fit mean CI width: {ci_width_180_overlap.mean():,.0f} infections" +) +print(f" Mean difference: {ci_diff.mean():,.0f} infections") +print(f" Mean ratio (90/180): {ci_ratio.mean():.2f}x") +print( + f"\nThe 180-day fit has {(1 - ci_width_180_overlap.mean() / ci_width_90_overlap.mean()) * 100:.1f}% narrower CIs on average" +) + +# Per-day comparison +print("\nCI width by time period:") +for start, end, label in [ + (0, 30, "Days 0-30"), + (30, 60, "Days 30-60"), + (60, 90, "Days 60-90"), +]: + mean_90 = ci_width_90_overlap[start:end].mean() + mean_180 = ci_width_180_overlap[start:end].mean() + reduction = (1 - mean_180 / mean_90) * 100 + print( + f" {label}: 90-day={mean_90:,.0f}, 180-day={mean_180:,.0f}, reduction={reduction:.1f}%" + ) +``` + +## Recap + +This notebook demonstrated composing a multi-signal renewal model using `ModelBuilder`: + +1. **Configure latent process** (`configure_latent`): generation interval, initial infections, temporal dynamics +2. **Add observation processes** (`add_observation`): each declares its infection resolution and gets a name for data binding +3. **Build and fit** (`build`, `model.fit`): the model routes infections to observations based on resolution and runs NUTS inference diff --git a/pyrenew/datasets/__init__.py b/pyrenew/datasets/__init__.py index 9fdbac4a..91927f71 100644 --- a/pyrenew/datasets/__init__.py +++ b/pyrenew/datasets/__init__.py @@ -1,13 +1,17 @@ # numpydoc ignore=GL08 from pyrenew.datasets.generation_interval import load_generation_interval +from pyrenew.datasets.hospital_admissions import load_hospital_data_for_state from pyrenew.datasets.infection_admission_interval import ( load_infection_admission_interval, ) from pyrenew.datasets.wastewater import load_wastewater +from pyrenew.datasets.wastewater_nwss import load_wastewater_data_for_state __all__ = [ "load_wastewater", "load_infection_admission_interval", "load_generation_interval", + "load_hospital_data_for_state", + "load_wastewater_data_for_state", ] diff --git a/pyrenew/datasets/hospital_admissions.py b/pyrenew/datasets/hospital_admissions.py new file mode 100644 index 00000000..e3ffffbb --- /dev/null +++ b/pyrenew/datasets/hospital_admissions.py @@ -0,0 +1,79 @@ +# numpydoc ignore=ES01,SA01,EX01 +""" +Load hospital admissions data for use in tutorials and examples. + +This module provides functions to load COVID-19 hospital admissions +data from the CDC's cfa-forecast-renewal-ww project. +""" + +from importlib.resources import files + +import jax.numpy as jnp +import polars as pl + + +def load_hospital_data_for_state( + state_abbr: str = "CA", + filename: str = "2023-11-06.csv", +) -> dict: + """ + Load hospital admissions data for a specific state. + + Parameters + ---------- + state_abbr : str + State abbreviation (e.g., "CA"). Default is "CA". + filename : str + CSV filename. Default is "2023-11-06.csv". + + Returns + ------- + dict + Dictionary containing: + + - daily_admits: JAX array of daily hospital admissions + - population: Population size (scalar) + - dates: List of date strings + - n_days: Number of days + + Notes + ----- + Data source: CDC cfa-forecast-renewal-ww repository. + License: Public Domain (CC0 1.0 Universal) - U.S. Government work. + + Examples + -------- + >>> from pyrenew import datasets + >>> ca_data = datasets.load_hospital_data_for_state("CA") + >>> ca_data["n_days"] + 311 + """ + data_path = files("pyrenew.datasets.hospital_admissions_data") / filename + df = pl.read_csv(source=data_path) + + df = ( + df.with_columns(pl.col("date").str.to_date()) + .filter(pl.col("location") == state_abbr) + .sort("date") + ) + + if len(df) == 0: + # Try filtering by ABBR column if it exists + df = pl.read_csv(source=data_path) + df = df.with_columns(pl.col("date").str.to_date()) + if "ABBR" in df.columns: + df = df.filter(pl.col("ABBR") == state_abbr).sort("date") + + if len(df) == 0: + raise ValueError(f"No data found for state {state_abbr} in {filename}") + + daily_admits = jnp.array(df["daily_hosp_admits"].to_numpy()) + population = int(df["pop"][0]) + dates = df["date"].to_list() + + return { + "daily_admits": daily_admits, + "population": population, + "dates": dates, + "n_days": len(daily_admits), + } diff --git a/pyrenew/datasets/hospital_admissions_data/2023-11-06.csv b/pyrenew/datasets/hospital_admissions_data/2023-11-06.csv new file mode 100644 index 00000000..fd8ed9dd --- /dev/null +++ b/pyrenew/datasets/hospital_admissions_data/2023-11-06.csv @@ -0,0 +1,311 @@ +"date","location","daily_hosp_admits","pop","week" +2023-01-01,"CA",2369,39512223,2023-01-04 +2023-01-02,"CA",1177,39512223,2023-01-04 +2023-01-03,"CA",1098,39512223,2023-01-04 +2023-01-04,"CA",1252,39512223,2023-01-04 +2023-01-05,"CA",634,39512223,2023-01-04 +2023-01-06,"CA",2097,39512223,2023-01-04 +2023-01-07,"CA",1152,39512223,2023-01-04 +2023-01-08,"CA",1327,39512223,2023-01-11 +2023-01-09,"CA",2072,39512223,2023-01-11 +2023-01-10,"CA",356,39512223,2023-01-11 +2023-01-11,"CA",1625,39512223,2023-01-11 +2023-01-12,"CA",165,39512223,2023-01-11 +2023-01-13,"CA",2458,39512223,2023-01-11 +2023-01-14,"CA",1903,39512223,2023-01-11 +2023-01-15,"CA",899,39512223,2023-01-18 +2023-01-16,"CA",601,39512223,2023-01-18 +2023-01-17,"CA",1307,39512223,2023-01-18 +2023-01-18,"CA",932,39512223,2023-01-18 +2023-01-19,"CA",997,39512223,2023-01-18 +2023-01-20,"CA",2015,39512223,2023-01-18 +2023-01-21,"CA",621,39512223,2023-01-18 +2023-01-22,"CA",1541,39512223,2023-01-25 +2023-01-23,"CA",860,39512223,2023-01-25 +2023-01-24,"CA",259,39512223,2023-01-25 +2023-01-25,"CA",1338,39512223,2023-01-25 +2023-01-26,"CA",481,39512223,2023-01-25 +2023-01-27,"CA",1322,39512223,2023-01-25 +2023-01-28,"CA",2072,39512223,2023-01-25 +2023-01-29,"CA",299,39512223,2023-02-01 +2023-01-30,"CA",2454,39512223,2023-02-01 +2023-01-31,"CA",2362,39512223,2023-02-01 +2023-02-01,"CA",1316,39512223,2023-02-01 +2023-02-02,"CA",836,39512223,2023-02-01 +2023-02-03,"CA",982,39512223,2023-02-01 +2023-02-04,"CA",2194,39512223,2023-02-01 +2023-02-05,"CA",1372,39512223,2023-02-08 +2023-02-06,"CA",2245,39512223,2023-02-08 +2023-02-07,"CA",1540,39512223,2023-02-08 +2023-02-08,"CA",2274,39512223,2023-02-08 +2023-02-09,"CA",2552,39512223,2023-02-08 +2023-02-10,"CA",727,39512223,2023-02-08 +2023-02-11,"CA",945,39512223,2023-02-08 +2023-02-12,"CA",626,39512223,2023-02-15 +2023-02-13,"CA",262,39512223,2023-02-15 +2023-02-14,"CA",2438,39512223,2023-02-15 +2023-02-15,"CA",130,39512223,2023-02-15 +2023-02-16,"CA",1396,39512223,2023-02-15 +2023-02-17,"CA",1027,39512223,2023-02-15 +2023-02-18,"CA",374,39512223,2023-02-15 +2023-02-19,"CA",1015,39512223,2023-02-22 +2023-02-20,"CA",1894,39512223,2023-02-22 +2023-02-21,"CA",650,39512223,2023-02-22 +2023-02-22,"CA",2088,39512223,2023-02-22 +2023-02-23,"CA",517,39512223,2023-02-22 +2023-02-24,"CA",33,39512223,2023-02-22 +2023-02-25,"CA",945,39512223,2023-02-22 +2023-02-26,"CA",103,39512223,2023-03-01 +2023-02-27,"CA",1252,39512223,2023-03-01 +2023-02-28,"CA",2157,39512223,2023-03-01 +2023-03-01,"CA",1353,39512223,2023-03-01 +2023-03-02,"CA",1693,39512223,2023-03-01 +2023-03-03,"CA",76,39512223,2023-03-01 +2023-03-04,"CA",1801,39512223,2023-03-01 +2023-03-05,"CA",1515,39512223,2023-03-08 +2023-03-06,"CA",16,39512223,2023-03-08 +2023-03-07,"CA",1381,39512223,2023-03-08 +2023-03-08,"CA",1349,39512223,2023-03-08 +2023-03-09,"CA",988,39512223,2023-03-08 +2023-03-10,"CA",82,39512223,2023-03-08 +2023-03-11,"CA",1944,39512223,2023-03-08 +2023-03-12,"CA",1905,39512223,2023-03-15 +2023-03-13,"CA",914,39512223,2023-03-15 +2023-03-14,"CA",325,39512223,2023-03-15 +2023-03-15,"CA",1646,39512223,2023-03-15 +2023-03-16,"CA",2344,39512223,2023-03-15 +2023-03-17,"CA",149,39512223,2023-03-15 +2023-03-18,"CA",1593,39512223,2023-03-15 +2023-03-19,"CA",100,39512223,2023-03-22 +2023-03-20,"CA",999,39512223,2023-03-22 +2023-03-21,"CA",2346,39512223,2023-03-22 +2023-03-22,"CA",2450,39512223,2023-03-22 +2023-03-23,"CA",91,39512223,2023-03-22 +2023-03-24,"CA",1078,39512223,2023-03-22 +2023-03-25,"CA",2336,39512223,2023-03-22 +2023-03-26,"CA",720,39512223,2023-03-29 +2023-03-27,"CA",1782,39512223,2023-03-29 +2023-03-28,"CA",2108,39512223,2023-03-29 +2023-03-29,"CA",1309,39512223,2023-03-29 +2023-03-30,"CA",1644,39512223,2023-03-29 +2023-03-31,"CA",2425,39512223,2023-03-29 +2023-04-01,"CA",1877,39512223,2023-03-29 +2023-04-02,"CA",427,39512223,2023-04-05 +2023-04-03,"CA",2490,39512223,2023-04-05 +2023-04-04,"CA",1136,39512223,2023-04-05 +2023-04-05,"CA",584,39512223,2023-04-05 +2023-04-06,"CA",951,39512223,2023-04-05 +2023-04-07,"CA",1537,39512223,2023-04-05 +2023-04-08,"CA",1165,39512223,2023-04-05 +2023-04-09,"CA",1742,39512223,2023-04-12 +2023-04-10,"CA",1925,39512223,2023-04-12 +2023-04-11,"CA",311,39512223,2023-04-12 +2023-04-12,"CA",474,39512223,2023-04-12 +2023-04-13,"CA",1578,39512223,2023-04-12 +2023-04-14,"CA",2090,39512223,2023-04-12 +2023-04-15,"CA",760,39512223,2023-04-12 +2023-04-16,"CA",1889,39512223,2023-04-19 +2023-04-17,"CA",1275,39512223,2023-04-19 +2023-04-18,"CA",1693,39512223,2023-04-19 +2023-04-19,"CA",25,39512223,2023-04-19 +2023-04-20,"CA",1907,39512223,2023-04-19 +2023-04-21,"CA",703,39512223,2023-04-19 +2023-04-22,"CA",2574,39512223,2023-04-19 +2023-04-23,"CA",623,39512223,2023-04-26 +2023-04-24,"CA",2310,39512223,2023-04-26 +2023-04-25,"CA",943,39512223,2023-04-26 +2023-04-26,"CA",299,39512223,2023-04-26 +2023-04-27,"CA",2417,39512223,2023-04-26 +2023-04-28,"CA",2560,39512223,2023-04-26 +2023-04-29,"CA",754,39512223,2023-04-26 +2023-04-30,"CA",799,39512223,2023-05-03 +2023-05-01,"CA",1954,39512223,2023-05-03 +2023-05-02,"CA",1248,39512223,2023-05-03 +2023-05-03,"CA",294,39512223,2023-05-03 +2023-05-04,"CA",607,39512223,2023-05-03 +2023-05-05,"CA",546,39512223,2023-05-03 +2023-05-06,"CA",2474,39512223,2023-05-03 +2023-05-07,"CA",471,39512223,2023-05-10 +2023-05-08,"CA",2236,39512223,2023-05-10 +2023-05-09,"CA",254,39512223,2023-05-10 +2023-05-10,"CA",268,39512223,2023-05-10 +2023-05-11,"CA",640,39512223,2023-05-10 +2023-05-12,"CA",2089,39512223,2023-05-10 +2023-05-13,"CA",664,39512223,2023-05-10 +2023-05-14,"CA",1122,39512223,2023-05-17 +2023-05-15,"CA",348,39512223,2023-05-17 +2023-05-16,"CA",2155,39512223,2023-05-17 +2023-05-17,"CA",317,39512223,2023-05-17 +2023-05-18,"CA",1982,39512223,2023-05-17 +2023-05-19,"CA",162,39512223,2023-05-17 +2023-05-20,"CA",636,39512223,2023-05-17 +2023-05-21,"CA",1631,39512223,2023-05-24 +2023-05-22,"CA",1568,39512223,2023-05-24 +2023-05-23,"CA",906,39512223,2023-05-24 +2023-05-24,"CA",1721,39512223,2023-05-24 +2023-05-25,"CA",796,39512223,2023-05-24 +2023-05-26,"CA",1162,39512223,2023-05-24 +2023-05-27,"CA",1571,39512223,2023-05-24 +2023-05-28,"CA",78,39512223,2023-05-31 +2023-05-29,"CA",526,39512223,2023-05-31 +2023-05-30,"CA",2531,39512223,2023-05-31 +2023-05-31,"CA",2230,39512223,2023-05-31 +2023-06-01,"CA",1151,39512223,2023-05-31 +2023-06-02,"CA",1579,39512223,2023-05-31 +2023-06-03,"CA",337,39512223,2023-05-31 +2023-06-04,"CA",539,39512223,2023-06-07 +2023-06-05,"CA",1566,39512223,2023-06-07 +2023-06-06,"CA",2260,39512223,2023-06-07 +2023-06-07,"CA",1611,39512223,2023-06-07 +2023-06-08,"CA",201,39512223,2023-06-07 +2023-06-09,"CA",1425,39512223,2023-06-07 +2023-06-10,"CA",62,39512223,2023-06-07 +2023-06-11,"CA",1293,39512223,2023-06-14 +2023-06-12,"CA",2015,39512223,2023-06-14 +2023-06-13,"CA",1406,39512223,2023-06-14 +2023-06-14,"CA",573,39512223,2023-06-14 +2023-06-15,"CA",1793,39512223,2023-06-14 +2023-06-16,"CA",540,39512223,2023-06-14 +2023-06-17,"CA",2,39512223,2023-06-14 +2023-06-18,"CA",671,39512223,2023-06-21 +2023-06-19,"CA",1151,39512223,2023-06-21 +2023-06-20,"CA",771,39512223,2023-06-21 +2023-06-21,"CA",741,39512223,2023-06-21 +2023-06-22,"CA",2317,39512223,2023-06-21 +2023-06-23,"CA",1036,39512223,2023-06-21 +2023-06-24,"CA",705,39512223,2023-06-21 +2023-06-25,"CA",670,39512223,2023-06-28 +2023-06-26,"CA",1587,39512223,2023-06-28 +2023-06-27,"CA",444,39512223,2023-06-28 +2023-06-28,"CA",1887,39512223,2023-06-28 +2023-06-29,"CA",1848,39512223,2023-06-28 +2023-06-30,"CA",1034,39512223,2023-06-28 +2023-07-01,"CA",1225,39512223,2023-06-28 +2023-07-02,"CA",1769,39512223,2023-07-05 +2023-07-03,"CA",1945,39512223,2023-07-05 +2023-07-04,"CA",181,39512223,2023-07-05 +2023-07-05,"CA",751,39512223,2023-07-05 +2023-07-06,"CA",1308,39512223,2023-07-05 +2023-07-07,"CA",1788,39512223,2023-07-05 +2023-07-08,"CA",463,39512223,2023-07-05 +2023-07-09,"CA",1337,39512223,2023-07-12 +2023-07-10,"CA",2132,39512223,2023-07-12 +2023-07-11,"CA",464,39512223,2023-07-12 +2023-07-12,"CA",984,39512223,2023-07-12 +2023-07-13,"CA",225,39512223,2023-07-12 +2023-07-14,"CA",1126,39512223,2023-07-12 +2023-07-15,"CA",2219,39512223,2023-07-12 +2023-07-16,"CA",489,39512223,2023-07-19 +2023-07-17,"CA",2004,39512223,2023-07-19 +2023-07-18,"CA",1591,39512223,2023-07-19 +2023-07-19,"CA",1850,39512223,2023-07-19 +2023-07-20,"CA",1709,39512223,2023-07-19 +2023-07-21,"CA",1231,39512223,2023-07-19 +2023-07-22,"CA",1109,39512223,2023-07-19 +2023-07-23,"CA",1954,39512223,2023-07-26 +2023-07-24,"CA",153,39512223,2023-07-26 +2023-07-25,"CA",1934,39512223,2023-07-26 +2023-07-26,"CA",2288,39512223,2023-07-26 +2023-07-27,"CA",49,39512223,2023-07-26 +2023-07-28,"CA",156,39512223,2023-07-26 +2023-07-29,"CA",2322,39512223,2023-07-26 +2023-07-30,"CA",1591,39512223,2023-08-02 +2023-07-31,"CA",2218,39512223,2023-08-02 +2023-08-01,"CA",292,39512223,2023-08-02 +2023-08-02,"CA",148,39512223,2023-08-02 +2023-08-03,"CA",2237,39512223,2023-08-02 +2023-08-04,"CA",1863,39512223,2023-08-02 +2023-08-05,"CA",1110,39512223,2023-08-02 +2023-08-06,"CA",2298,39512223,2023-08-09 +2023-08-07,"CA",169,39512223,2023-08-09 +2023-08-08,"CA",907,39512223,2023-08-09 +2023-08-09,"CA",732,39512223,2023-08-09 +2023-08-10,"CA",2505,39512223,2023-08-09 +2023-08-11,"CA",919,39512223,2023-08-09 +2023-08-12,"CA",2163,39512223,2023-08-09 +2023-08-13,"CA",1803,39512223,2023-08-16 +2023-08-14,"CA",1939,39512223,2023-08-16 +2023-08-15,"CA",1237,39512223,2023-08-16 +2023-08-16,"CA",1316,39512223,2023-08-16 +2023-08-17,"CA",985,39512223,2023-08-16 +2023-08-18,"CA",2486,39512223,2023-08-16 +2023-08-19,"CA",1851,39512223,2023-08-16 +2023-08-20,"CA",625,39512223,2023-08-23 +2023-08-21,"CA",542,39512223,2023-08-23 +2023-08-22,"CA",1493,39512223,2023-08-23 +2023-08-23,"CA",1900,39512223,2023-08-23 +2023-08-24,"CA",917,39512223,2023-08-23 +2023-08-25,"CA",1129,39512223,2023-08-23 +2023-08-26,"CA",1226,39512223,2023-08-23 +2023-08-27,"CA",2136,39512223,2023-08-30 +2023-08-28,"CA",1975,39512223,2023-08-30 +2023-08-29,"CA",2439,39512223,2023-08-30 +2023-08-30,"CA",398,39512223,2023-08-30 +2023-08-31,"CA",2316,39512223,2023-08-30 +2023-09-01,"CA",1717,39512223,2023-08-30 +2023-09-02,"CA",1546,39512223,2023-08-30 +2023-09-03,"CA",1625,39512223,2023-09-06 +2023-09-04,"CA",2085,39512223,2023-09-06 +2023-09-05,"CA",2211,39512223,2023-09-06 +2023-09-06,"CA",1819,39512223,2023-09-06 +2023-09-07,"CA",2094,39512223,2023-09-06 +2023-09-08,"CA",859,39512223,2023-09-06 +2023-09-09,"CA",1488,39512223,2023-09-06 +2023-09-10,"CA",868,39512223,2023-09-13 +2023-09-11,"CA",376,39512223,2023-09-13 +2023-09-12,"CA",280,39512223,2023-09-13 +2023-09-13,"CA",981,39512223,2023-09-13 +2023-09-14,"CA",1428,39512223,2023-09-13 +2023-09-15,"CA",715,39512223,2023-09-13 +2023-09-16,"CA",1249,39512223,2023-09-13 +2023-09-17,"CA",2254,39512223,2023-09-20 +2023-09-18,"CA",1372,39512223,2023-09-20 +2023-09-19,"CA",1825,39512223,2023-09-20 +2023-09-20,"CA",605,39512223,2023-09-20 +2023-09-21,"CA",1828,39512223,2023-09-20 +2023-09-22,"CA",647,39512223,2023-09-20 +2023-09-23,"CA",1900,39512223,2023-09-20 +2023-09-24,"CA",1797,39512223,2023-09-27 +2023-09-25,"CA",159,39512223,2023-09-27 +2023-09-26,"CA",928,39512223,2023-09-27 +2023-09-27,"CA",978,39512223,2023-09-27 +2023-09-28,"CA",653,39512223,2023-09-27 +2023-09-29,"CA",1402,39512223,2023-09-27 +2023-09-30,"CA",1078,39512223,2023-09-27 +2023-10-01,"CA",2562,39512223,2023-10-04 +2023-10-02,"CA",2435,39512223,2023-10-04 +2023-10-03,"CA",1647,39512223,2023-10-04 +2023-10-04,"CA",1846,39512223,2023-10-04 +2023-10-05,"CA",1162,39512223,2023-10-04 +2023-10-06,"CA",1433,39512223,2023-10-04 +2023-10-07,"CA",620,39512223,2023-10-04 +2023-10-08,"CA",2250,39512223,2023-10-11 +2023-10-09,"CA",2121,39512223,2023-10-11 +2023-10-10,"CA",335,39512223,2023-10-11 +2023-10-11,"CA",1271,39512223,2023-10-11 +2023-10-12,"CA",2179,39512223,2023-10-11 +2023-10-13,"CA",1174,39512223,2023-10-11 +2023-10-14,"CA",1085,39512223,2023-10-11 +2023-10-15,"CA",1044,39512223,2023-10-18 +2023-10-16,"CA",682,39512223,2023-10-18 +2023-10-17,"CA",983,39512223,2023-10-18 +2023-10-18,"CA",529,39512223,2023-10-18 +2023-10-19,"CA",82,39512223,2023-10-18 +2023-10-20,"CA",1235,39512223,2023-10-18 +2023-10-21,"CA",2321,39512223,2023-10-18 +2023-10-22,"CA",94,39512223,2023-10-25 +2023-10-23,"CA",2378,39512223,2023-10-25 +2023-10-24,"CA",933,39512223,2023-10-25 +2023-10-25,"CA",1714,39512223,2023-10-25 +2023-10-26,"CA",2227,39512223,2023-10-25 +2023-10-27,"CA",2035,39512223,2023-10-25 +2023-10-28,"CA",2352,39512223,2023-10-25 +2023-10-29,"CA",954,39512223,2023-11-01 +2023-10-30,"CA",1860,39512223,2023-11-01 +2023-10-31,"CA",473,39512223,2023-11-01 +2023-11-01,"CA",1370,39512223,2023-11-01 +2023-11-02,"CA",829,39512223,2023-11-01 +2023-11-03,"CA",1255,39512223,2023-11-01 +2023-11-04,"CA",1390,39512223,2023-11-01 +2023-11-05,"CA",2156,39512223,2023-11-08 +2023-11-06,"CA",601,39512223,2023-11-08 diff --git a/pyrenew/datasets/hospital_admissions_data/README.md b/pyrenew/datasets/hospital_admissions_data/README.md new file mode 100644 index 00000000..a6db74b1 --- /dev/null +++ b/pyrenew/datasets/hospital_admissions_data/README.md @@ -0,0 +1,36 @@ +# Hospital Admissions Data + +## Provenance + +- **Source Repository**: [CDC cfa-forecast-renewal-ww](https://github.com/cdcgov/cfa-forecast-renewal-ww) +- **Original Location**: `cfaforecastrenewalww/inst/testdata/` +- **License**: Public Domain (CC0 1.0 Universal) - U.S. Government work +- **Date Extracted**: January 2025 + +## Files + +### `2023-11-06.csv` + +Vintaged snapshot of COVID-19 hospital admissions data as it would have been available on 2023-11-06. + +- **Coverage**: California (CA) only +- **Date range**: 2023-01-01 to 2023-11-06 (~310 days) +- **Size**: 12 KB, 311 rows +- **Use case**: Tutorials, single-jurisdiction model development + +### Schema + +| Column | Type | Description | +|--------|------|-------------| +| `date` | string | Date in ISO 8601 format (YYYY-MM-DD) | +| `location` | string | State 2-letter abbreviation | +| `daily_hosp_admits` | integer | Daily COVID-19 hospital admissions count | +| `pop` | integer | State population | + +## Usage + +```python +from pyrenew import datasets + +ca_data = datasets.load_hospital_data_for_state("CA") +``` diff --git a/pyrenew/datasets/hospital_admissions_data/__init__.py b/pyrenew/datasets/hospital_admissions_data/__init__.py new file mode 100644 index 00000000..bbc65f63 --- /dev/null +++ b/pyrenew/datasets/hospital_admissions_data/__init__.py @@ -0,0 +1,2 @@ +# numpydoc ignore=GL08 +"""Hospital admissions data package.""" diff --git a/pyrenew/datasets/wastewater_nwss.py b/pyrenew/datasets/wastewater_nwss.py new file mode 100644 index 00000000..c72a4fd8 --- /dev/null +++ b/pyrenew/datasets/wastewater_nwss.py @@ -0,0 +1,141 @@ +# numpydoc ignore=ES01,SA01,EX01 +""" +Load wastewater NWSS data for use in tutorials and examples. + +This module provides functions to load synthetic wastewater surveillance +data in NWSS format from the CDC's cfa-forecast-renewal-ww project. +""" + +from importlib.resources import files + +import jax.numpy as jnp +import polars as pl + + +def load_wastewater_data_for_state( + state_abbr: str = "CA", + filename: str = "fake_nwss.csv", + substitute_below_lod: bool = True, + standardize_log_units: bool = True, +) -> dict: + """ + Load wastewater data for a specific state. + + Parameters + ---------- + state_abbr : str + State abbreviation (e.g., "CA"). Default is "CA". + filename : str + CSV filename. Default is "fake_nwss.csv". + substitute_below_lod : bool + If True, replace concentrations below LOD with 0.5*LOD. + Default is True. + standardize_log_units : bool + If True, convert log10 units to linear scale. + Default is True. + + Returns + ------- + dict + Dictionary containing: + + - observed_conc: JAX array of log concentrations (log copies/mL) + - observed_conc_linear: JAX array of linear concentrations (copies/mL) + - site_ids: JAX array of site indices + - time_indices: JAX array of time indices (days from start) + - wwtp_names: List of unique WWTP names + - dates: List of unique dates + - n_sites: Number of unique sites + - n_obs: Number of observations + - raw_df: Polars DataFrame (for debugging) + + Notes + ----- + Data source: CDC cfa-forecast-renewal-ww repository. + License: Public Domain (CC0 1.0 Universal) - U.S. Government work. + + The data is synthetic and contains deliberately added noise for + public release. + + Examples + -------- + >>> from pyrenew import datasets + >>> ca_ww = datasets.load_wastewater_data_for_state("CA") + >>> ca_ww["n_sites"] + 5 + """ + data_path = files("pyrenew.datasets.wastewater_nwss_data") / filename + df = pl.read_csv( + source=data_path, + schema_overrides={"county_names": pl.String}, + ) + df = df.with_columns(pl.col("sample_collect_date").str.to_date()) + + # Quality filtering for real NWSS data + if "sample_location" in df.columns: + df = df.filter(pl.col("sample_location") == "wwtp") + if "sample_matrix" in df.columns: + df = df.filter(pl.col("sample_matrix") != "primary sludge") + if "pcr_target" in df.columns: + df = df.filter(pl.col("pcr_target") == "sars-cov-2") + + # Filter to requested state + df = df.filter(pl.col("wwtp_jurisdiction") == state_abbr) + if len(df) == 0: + raise ValueError(f"No wastewater data found for state {state_abbr}") + + if standardize_log_units: + # Convert log10 units to linear, then standardize to copies/mL + # NWSS data is in copies/L, so divide by 1000 to get copies/mL + df = df.with_columns( + pl.when(pl.col("pcr_target_units").str.contains("(?i)log10")) + .then(10 ** pl.col("pcr_target_avg_conc") / 1000) + .otherwise(pl.col("pcr_target_avg_conc") / 1000) + .alias("conc_linear"), + pl.when(pl.col("pcr_target_units").str.contains("(?i)log10")) + .then(10 ** pl.col("lod_sewage") / 1000) + .otherwise(pl.col("lod_sewage") / 1000) + .alias("lod_linear"), + ) + else: + df = df.with_columns( + (pl.col("pcr_target_avg_conc") / 1000).alias("conc_linear"), + (pl.col("lod_sewage") / 1000).alias("lod_linear"), + ) + + # Substitute below-LOD values with 0.5*LOD + if substitute_below_lod: + df = df.with_columns( + pl.when(pl.col("conc_linear") < pl.col("lod_linear")) + .then(0.5 * pl.col("lod_linear")) + .otherwise(pl.col("conc_linear")) + .alias("conc_linear") + ) + + df = df.sort("sample_collect_date") + unique_sites = sorted(df["wwtp_name"].unique().to_list()) + site_to_idx = {site: idx for idx, site in enumerate(unique_sites)} + min_date = df["sample_collect_date"].min() + df = df.with_columns( + ((pl.col("sample_collect_date") - min_date).dt.total_days()).alias("time_idx") + ) + df = df.with_columns( + pl.col("wwtp_name").replace_strict(site_to_idx, default=None).alias("site_idx") + ) + + observed_conc_linear = jnp.array(df["conc_linear"].to_numpy()) + observed_log_conc = jnp.log(observed_conc_linear + 1e-8) + site_ids = jnp.array(df["site_idx"].to_numpy(), dtype=jnp.int32) + time_indices = jnp.array(df["time_idx"].to_numpy(), dtype=jnp.int32) + + return { + "observed_conc": observed_log_conc, + "observed_conc_linear": observed_conc_linear, + "site_ids": site_ids, + "time_indices": time_indices, + "wwtp_names": unique_sites, + "dates": sorted(df["sample_collect_date"].unique().to_list()), + "n_sites": len(unique_sites), + "n_obs": len(df), + "raw_df": df, + } diff --git a/pyrenew/datasets/wastewater_nwss_data/README.md b/pyrenew/datasets/wastewater_nwss_data/README.md new file mode 100644 index 00000000..3472031f --- /dev/null +++ b/pyrenew/datasets/wastewater_nwss_data/README.md @@ -0,0 +1,53 @@ +# Wastewater NWSS Data + +## Provenance + +- **Source Repository**: [CDC cfa-forecast-renewal-ww](https://github.com/cdcgov/cfa-forecast-renewal-ww) +- **Original Location**: `cfaforecastrenewalww/inst/testdata/` +- **License**: Public Domain (CC0 1.0 Universal) - U.S. Government work +- **Date Extracted**: January 2025 + +## Files + +### `fake_nwss.csv` + +Synthetic wastewater surveillance data in NWSS (National Wastewater Surveillance System) format. Contains deliberately added noise for public release. + +- **Jurisdictions**: CA, WA, NM (real states) plus XX, YY, ZZ (fictional) +- **WWTPs**: CA (5), WA (4), NM (2), others (4 each) +- **Date range**: 2023-01-01 to 2023-11-06 (~310 days) +- **Size**: 487 KB, 3,286 rows +- **Granularity**: Site-lab-date level (multiple labs per WWTP) +- **Use case**: Tutorials, multi-signal model development + +### Schema + +| Column | Type | Description | +|--------|------|-------------| +| `wwtp_jurisdiction` | string | State/territory abbreviation | +| `wwtp_name` | string | Wastewater treatment plant identifier | +| `county_names` | string | County code | +| `lab_id` | integer | Laboratory identifier | +| `population_served` | integer | Population served by this WWTP | +| `sample_location` | string | Sample collection point (e.g., "wwtp") | +| `sample_matrix` | string | Sample type (e.g., "raw wastewater") | +| `pcr_target_units` | string | Measurement units | +| `pcr_target` | string | Target pathogen (always "sars-cov-2") | +| `pcr_target_avg_conc` | float | Viral RNA concentration | +| `lod_sewage` | float | Limit of detection for this sample | +| `pcr_target_below_lod` | integer | Below detection limit flag (0=above, 1=below) | +| `sample_collect_date` | string | Sample collection date (YYYY-MM-DD) | +| `quality_flag` | string | Data quality flags | + +## Usage + +```python +from pyrenew import datasets + +ca_ww = datasets.load_wastewater_data_for_state("CA") +``` + +## Notes + +- Some measurements are in linear scale, others in log10 scale (check `pcr_target_units`) +- The loader function handles unit standardization and below-LOD substitution diff --git a/pyrenew/datasets/wastewater_nwss_data/__init__.py b/pyrenew/datasets/wastewater_nwss_data/__init__.py new file mode 100644 index 00000000..6c03debf --- /dev/null +++ b/pyrenew/datasets/wastewater_nwss_data/__init__.py @@ -0,0 +1,2 @@ +# numpydoc ignore=GL08 +"""Wastewater NWSS data package.""" diff --git a/pyrenew/datasets/wastewater_nwss_data/fake_nwss.csv b/pyrenew/datasets/wastewater_nwss_data/fake_nwss.csv new file mode 100644 index 00000000..73898c38 --- /dev/null +++ b/pyrenew/datasets/wastewater_nwss_data/fake_nwss.csv @@ -0,0 +1,3286 @@ +wwtp_jurisdiction,junk1,junk2,wwtp_name,county_names,lab_id,population_served,sample_location,sample_matrix,pcr_target_units,pcr_target,pcr_target_avg_conc,pcr_target_flowpop_lin,lod_sewage,pcr_target_below_lod,sample_collect_date,quality_flag +CA,A,YES,1,04971,1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,8.054115836944005,55.04513393435594,7.443448737491519,0,2023-01-01,NA +CA,A,YES,2,31809,1,30000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,7.8698405584687325,85.47494922884091,1.6125732477322323,0,2023-01-01,temperature not assessed upon arrival at the laboratory +CA,A,YES,4,12078,4,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,701.3091712313847,404.0086944333856,13.881587487408053,0,2023-01-01,> max temp and/or hold time +WA,B,Yes,6,31928,1,2000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,244.42352923698678,44.04065263553125,20.924849355634137,0,2023-01-01,NA +WA,B,Yes,7,07528,1,100000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,29.19971609624074,104.3650894301212,2.5731702128827867,0,2023-01-01,NA +WA,B,Yes,9,14396,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,257.5597420887679,226.42429315424206,10.130901996571637,0,2023-01-01,NA +NM,C,yes,10,27604,5,1000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,2.1236739877669955,40.53356746877808,0.4530527192221311,0,2023-01-01,NA +NM,C,yes,11,91578,6,10000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,2.1399793267291494,651.7571697903521,1.0887289571287273,0,2023-01-01,NA +CA,A,YES,1,04971,1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,318.5878534041641,32.51171069583552,1.9747551214710093,0,2023-01-02,no +CA,A,YES,2,31809,1,30000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,9.897117221594996,90.91045487095828,5.86592310408781,0,2023-01-02,no +CA,A,YES,4,12078,3,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,7.332069630582176,67.05837186719428,1.6708185559623274,0,2023-01-02,NA +CA,A,YES,4,12078,4,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,9.065560011477302,815.3348623734826,20.646824858007953,0,2023-01-02,NA +WA,B,Yes,6,31928,1,2000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,240.85122185566547,386.36572390749814,1.8340868130391876,0,2023-01-02,temperature not assessed upon arrival at the laboratory +WA,B,Yes,7,07528,1,100000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,544.0669587294154,25.439665117925003,2.382577502458453,0,2023-01-02,> max temp and/or hold time +WA,B,Yes,8,97385,2,50000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,77.42087123963306,13.356856415392215,5.520118250210656,0,2023-01-02,NA +WA,B,Yes,9,14396,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,456.3402279933906,6.459774947213775,1.071485091360489,0,2023-01-02,NA +NM,C,yes,11,91578,6,10000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,1.0622568511245791,32.127546724221176,0.15424185094347556,0,2023-01-02,NA +CA,A,YES,1,04971,1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,8.045477208790313,40.27750071592156,3.3258959869434412,0,2023-01-03,no +CA,A,YES,2,31809,1,30000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,276.04358187989425,10.60490126457747,13.406836622913755,0,2023-01-03,NA +CA,A,YES,4,12078,3,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,10.802883747691222,99.6879994674735,4.1228747821196965,0,2023-01-03,NA +CA,A,YES,4,12078,4,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,52.71543846184618,432.96599083032794,1.6987881432802636,0,2023-01-03,NA +CA,A,YES,5,34687,4,1000000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,6.885239603500818,229.74987074032768,10.82238345503649,0,2023-01-03,NA +WA,B,Yes,6,31928,1,2000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,28.351182973827616,266.95375529779506,4.623974785920688,0,2023-01-03,no +WA,B,Yes,7,07528,1,100000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,525.1236659332544,668.4903593460351,9.08483758036911,0,2023-01-03,NA +WA,B,Yes,8,97385,2,50000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,692.340538837637,5.063398393209281,21.38753865100015,0,2023-01-03,NA +WA,B,Yes,9,14396,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,39.93782388613959,11.69833491729379,4.959358574631709,0,2023-01-03,no +NM,C,yes,10,27604,5,1000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,1.0644811979504432,41.46761347133586,0.6577316500311907,0,2023-01-03,NA +NM,C,yes,11,91578,6,10000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,1.435179031423122,179.0196107307954,0.7831147395280831,0,2023-01-03,NA +ZZ,F,NO,15,98436,4,8472,wwtp,raw wastewater,copies/g dry sludge,sars-cov-2,48.8854542150505,785.5101763320581,11.481962595257894,0,2023-01-03,NA +CA,A,YES,1,04971,1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,1.7936991416491015,7.336498654648253,3.587398283298203,1,2023-01-04,NA +CA,A,YES,2,31809,1,30000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,5.639017094726396,59.20611278144642,9.425485444619234,0,2023-01-04,no +CA,A,YES,4,12078,3,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,141.3480424620658,193.41645063823643,3.7106659118350334,0,2023-01-04,NA +CA,A,YES,4,12078,4,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,8.860362666247195,89.9942777741431,8.713846998511125,0,2023-01-04,NA +WA,B,Yes,6,31928,1,2000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,20.354768660533676,7.798149164474054,15.21557641776394,0,2023-01-04,NA +WA,B,Yes,7,07528,1,100000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,7.81439055868008,690.6142401259643,3.5420380746438065,0,2023-01-04,NA +WA,B,Yes,8,97385,2,50000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,38.632604782682094,5.455289144363855,1.2897570562193899,0,2023-01-04,below loq +WA,B,Yes,9,14396,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,9.690787019938725,44.85481154601836,14.390826441300998,0,2023-01-04,NA +NM,C,yes,10,27604,5,1000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,2.0371437125447556,16.643658778352314,0.1018983983031235,0,2023-01-04,NA +NM,C,yes,11,91578,6,10000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,-0.14059865717500838,8.477074200774332,0.16043133848897279,1,2023-01-04,no +CA,A,YES,1,04971,1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,0.6741893206279964,20.52304747265837,1.3483786412559928,1,2023-01-05,no +CA,A,YES,2,31809,1,30000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,9.213432438093243,467.54870027586674,19.104719267310756,0,2023-01-05,no +CA,A,YES,4,12078,3,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,105.40816229939371,332.0959006655594,17.309586785263622,0,2023-01-05,no +CA,A,YES,4,12078,4,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,0.9693694950545131,8.274179546191354,1.9387389901090262,1,2023-01-05,no +WA,B,Yes,6,31928,1,2000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,293.372952930381,83.3920227790324,1.1330846161238484,0,2023-01-05,no +NM,C,yes,10,27604,5,1000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,0.05078059327057802,814.7988536447767,0.3518105889345592,1,2023-01-05,NA +NM,C,yes,11,91578,6,10000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,1.0543294986626754,77.89632981660215,0.7233458885830267,0,2023-01-05,NA +CA,A,YES,1,04971,1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,11.917227178713146,18.02822613418176,2.7315453798288956,0,2023-01-06,NA +CA,A,YES,2,31809,1,30000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,51.94043404667794,99.73155755071176,12.01622056070663,0,2023-01-06,y +CA,A,YES,4,12078,3,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,62.57132662203255,8.865505807767626,2.941917084707746,0,2023-01-06,NA +CA,A,YES,4,12078,4,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,32.244377683448256,220.47509528547852,11.775924849564442,0,2023-01-06,yes +WA,B,Yes,6,31928,1,2000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,661.1820611975214,9.235676726791054,3.908097932696782,0,2023-01-06,y +WA,B,Yes,7,07528,1,100000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,8.78196358452969,105.21353472717601,14.86860612818895,0,2023-01-06,NA +WA,B,Yes,9,14396,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,65.8794464791104,78.75667911237066,4.550292350271927,0,2023-01-06,result is not quantifiable +NM,C,yes,10,27604,5,1000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,0.7905438180592764,197.65608933880296,1.0915738137232576,1,2023-01-06,NA +NM,C,yes,11,91578,6,10000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,-0.18539681942192834,70.67471715756251,0.11563317624205283,1,2023-01-06,NA +CA,A,YES,1,04971,1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,521.4482122070327,177.2417430876511,1.0293360181042364,0,2023-01-07,NA +CA,A,YES,4,12078,3,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,339.87932515251276,760.3278706484415,19.170134988503573,0,2023-01-07,NA +CA,A,YES,4,12078,4,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,24.267629166826836,734.2385186628919,5.6382099008339335,0,2023-01-07,NA +WA,B,Yes,6,31928,1,2000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,177.4833931184256,43.96399978434991,1.3693087230193897,0,2023-01-07,NA +WA,B,Yes,7,07528,1,100000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,14.27431798158416,6.517668307294351,3.4008803156596836,0,2023-01-07,NA +WA,B,Yes,8,97385,2,50000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,137.20250183458862,308.88085874008095,1.8914415671146554,0,2023-01-07,no +NM,C,yes,10,27604,5,1000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,1.5574348710725703,36.94506053036365,0.015389978853004065,0,2023-01-07,NA +NM,C,yes,11,91578,6,10000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,1.421700131005064,18.303770404633134,1.2845080250231884,0,2023-01-07,NA +ZZ,F,NO,14,09634,3,8472,wwtp,raw wastewater,copies/g dry sludge,sars-cov-2,62.55157839741537,210.8539351550834,7.65256236552534,0,2023-01-07,NA +CA,A,YES,1,04971,1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,131.30679750544832,359.54013723174427,1.7967577817836142,0,2023-01-08,NA +CA,A,YES,2,31809,1,30000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,690.6411503137712,7.362639187526764,4.746511552418034,0,2023-01-08,no +CA,A,YES,4,12078,3,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,5.803335900572163,87.02906434858058,13.709835921006245,0,2023-01-08,NA +CA,A,YES,4,12078,4,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,152.32124636799452,32.25317457398982,14.221600706427989,0,2023-01-08,NA +WA,B,Yes,6,31928,1,2000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,6.408874139696166,12.407714498649254,8.69955762720997,0,2023-01-08,NA +WA,B,Yes,7,07528,1,100000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,6.452718763896252,79.10056984823339,1.46519908277987,0,2023-01-08,no +WA,B,Yes,8,97385,2,50000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,821.2964027322233,287.3360386844041,4.634604139930713,0,2023-01-08,NA +WA,B,Yes,9,14396,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,16.598803750091392,23.08901523781061,19.659377267354607,0,2023-01-08,NA +NM,C,yes,11,91578,6,10000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,1.2016890011336943,173.53867556998992,1.2759719511813978,0,2023-01-08,temperature not assessed upon arrival at the laboratory +CA,A,YES,1,04971,1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,10.145581141014883,38.471620515982444,4.204687765288971,0,2023-01-09,NA +CA,A,YES,2,31809,1,30000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,332.68665374034987,5.186752497468046,7.039870865907318,0,2023-01-09,no +CA,A,YES,4,12078,3,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,909.2718230213907,95.62537352340955,1.211119201600031,0,2023-01-09,no +CA,A,YES,4,12078,4,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,471.71923193370566,44.59725966999692,3.35533588059495,0,2023-01-09,NA +WA,B,Yes,6,31928,1,2000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,34.87247567690326,248.9636961803126,1.4960641475089012,0,2023-01-09,no +WA,B,Yes,7,07528,1,100000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,11.495592763655702,39.13385631233105,3.9814621547285354,0,2023-01-09,> max temp and/or hold time +WA,B,Yes,9,14396,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,65.61180498764986,136.0683646239188,4.314202773212136,0,2023-01-09,result is not quantifiable +NM,C,yes,10,27604,5,1000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,2.885415702225869,351.03049549927937,1.1103951101380294,0,2023-01-09,y +NM,C,yes,11,91578,6,10000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,2.780835821161669,159.95578342572534,0.6588162693304195,0,2023-01-09,NA +CA,A,YES,1,04971,1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,25.78457079057178,5.312387660424927,8.871589067317199,0,2023-01-10,NA +CA,A,YES,2,31809,1,30000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,9.927658223343428,388.0424407801104,18.9200986030612,0,2023-01-10,NA +CA,A,YES,4,12078,3,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,58.731333743180485,9.137731244097838,1.5513755587507887,0,2023-01-10,NA +CA,A,YES,5,34687,3,1000000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,6.400543374601076,191.5471770338321,1.3813037709408231,0,2023-01-10,NA +CA,A,YES,4,12078,4,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,615.6672698290056,391.83983760118497,22.694932696240812,0,2023-01-10,no +CA,A,YES,5,34687,4,1000000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,896.5532914720403,151.96565798603766,3.227220996362399,0,2023-01-10,no +WA,B,Yes,6,31928,1,2000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,12.337628290713585,81.97642740200946,10.31261223992233,0,2023-01-10,NA +WA,B,Yes,8,97385,2,50000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,10.656916381741485,22.895855658755295,1.0831517913830018,0,2023-01-10,NA +WA,B,Yes,9,14396,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,10.505560111445298,6.037383599591004,21.011120222890597,1,2023-01-10,NA +NM,C,yes,11,91578,6,10000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,2.8559477414250343,10.007681049446875,0.9733984415971332,0,2023-01-10,NA +CA,A,YES,1,04971,1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,12.286683603569443,355.21081413629145,24.573367207138887,1,2023-01-11,no +CA,A,YES,2,31809,1,30000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,6.811702797589217,71.10128114633655,4.9183009345096265,0,2023-01-11,NA +CA,A,YES,4,12078,3,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,5.628004007672834,59.63648010473265,6.072753827051153,0,2023-01-11,NA +CA,A,YES,4,12078,4,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,6.277535074850094,11.71195910581476,12.555070149700189,1,2023-01-11,NA +WA,B,Yes,6,31928,1,2000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,117.70121730042058,49.73018704466841,4.52632345989394,0,2023-01-11,NA +WA,B,Yes,7,07528,1,100000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,22.513746889067654,29.006369154559778,3.6456603526056863,0,2023-01-11,NA +WA,B,Yes,8,97385,2,50000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,6.699328104770948,53.68909630207695,18.258572955498895,0,2023-01-11,no +WA,B,Yes,9,14396,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,37.02286990746543,6.218667479820192,6.137656519134712,0,2023-01-11,NA +NM,C,yes,10,27604,5,1000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,1.9190310586915884,18.153400325556547,0.9659718998696941,0,2023-01-11,NA +NM,C,yes,11,91578,6,10000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,1.2764780702330636,8.637070751610635,0.46449944968220436,0,2023-01-11,NA +XX,D,ye,15,98436,10,1000000000,wwtp,raw wastewater,copies/l wastewater,FLUAV,12.694081385868751,7.329719106319113,1.0494477977485082,0,2023-01-11,NA +ZZ,F,NO,13,65174,2,8472,wwtp,raw wastewater,copies/g dry sludge,sars-cov-2,456.63354567581274,47.08633818823485,8.466017464898515,0,2023-01-11,NA +CA,A,YES,1,04971,1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,5.805642914082082,24.45507289087341,2.313825261175917,0,2023-01-12,NA +CA,A,YES,2,31809,1,30000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,19.44718987764599,40.203764632495684,1.0905164782503574,0,2023-01-12,no +CA,A,YES,4,12078,3,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,406.1682285142044,182.75552755410317,11.065053145208116,0,2023-01-12,NA +CA,A,YES,4,12078,4,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,66.32166767073055,16.749457921748586,5.45899388909348,0,2023-01-12,NA +WA,B,Yes,6,31928,1,2000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,8.509899618392536,103.06717296936746,2.3869633498675435,0,2023-01-12,NA +WA,B,Yes,7,07528,1,100000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,432.0649042180794,527.9723470794902,1.004104510783298,0,2023-01-12,NA +WA,B,Yes,8,97385,2,50000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,42.181350795398465,116.77287742885648,9.719964389544671,0,2023-01-12,NA +WA,B,Yes,9,14396,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,299.74807000174496,648.1111978177291,13.009797047536873,0,2023-01-12,y +NM,C,yes,11,91578,6,10000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,1.309822094411865,19.204585969094797,0.8375484672231703,0,2023-01-12,NA +CA,A,YES,1,04971,1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,164.05594243893796,395.07512339116994,6.521506611074547,0,2023-01-13,NA +CA,A,YES,2,31809,1,30000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,7.067189807481014,8.143104172197086,1.31190507988112,0,2023-01-13,NA +CA,A,YES,4,12078,4,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,20.668281743022384,6.767658180542428,5.350108435953022,0,2023-01-13,no +WA,B,Yes,6,31928,1,2000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,96.0092791625505,10.946085495005368,5.45077171046834,0,2023-01-13,NA +WA,B,Yes,7,07528,1,100000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,1.7295987093645164,26.87041153519793,3.459197418729033,1,2023-01-13,NA +WA,B,Yes,9,14396,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,9.034912516100114,22.351243840903013,6.19113281635205,0,2023-01-13,NA +NM,C,yes,10,27604,5,1000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,0.3250037772000478,47.458536978029834,0.626033772864029,1,2023-01-13,no +NM,C,yes,11,91578,6,10000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,2.404008614288464,66.19495041727531,0.8609710536047737,0,2023-01-13,NA +YY,E,No,12,32541,1,13,upstream,primary sludge,copies/g dry sludge,sars-cov-2,0.988911760903357,913.7650175746188,1.977823521806714,1,2023-01-13,NA +CA,A,YES,2,31809,1,30000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,139.9026637757234,19.35698296935395,2.6019121973694945,0,2023-01-14,below loq +CA,A,YES,4,12078,3,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,685.5529370076822,32.5081314819729,3.5254470765823758,0,2023-01-14,NA +CA,A,YES,4,12078,4,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,125.55114410035812,16.769759239273125,10.443100783639888,0,2023-01-14,NA +WA,B,Yes,6,31928,1,2000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,532.3347609753455,423.9301289145801,1.5906633424573682,0,2023-01-14,NA +WA,B,Yes,7,07528,1,100000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,141.22139840745226,19.457990196094233,1.1894997182825309,0,2023-01-14,NA +WA,B,Yes,8,97385,2,50000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,491.3249448660555,57.31987839122089,21.635482672228033,0,2023-01-14,NA +WA,B,Yes,9,14396,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,663.291603218385,415.5687611832833,1.085896864015427,0,2023-01-14,NA +NM,C,yes,10,27604,5,1000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,2.0077470638584316,28.769392752861677,1.2871716307244851,0,2023-01-14,NA +NM,C,yes,11,91578,6,10000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,2.4169543327984786,72.75685852787738,0.5125753166842177,0,2023-01-14,NA +CA,A,YES,1,04971,1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,2.8135112007183563,43.47218034808254,5.6270224014367125,1,2023-01-15,NA +CA,A,YES,2,31809,1,30000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,2.893614726205544,21.83701015300171,5.787229452411088,1,2023-01-15,NA +CA,A,YES,4,12078,3,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,11.158014543873952,19.259977815897816,1.6244820620583074,0,2023-01-15,temperature not assessed upon arrival at the laboratory +CA,A,YES,4,12078,4,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,177.41317655177008,177.32874626625346,8.404587197692639,0,2023-01-15,NA +WA,B,Yes,6,31928,1,2000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,541.8182179417457,10.901547157426906,1.5837425856449976,0,2023-01-15,NA +WA,B,Yes,7,07528,1,100000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,103.0228091253949,848.558656969157,6.5956000522480345,0,2023-01-15,NA +WA,B,Yes,8,97385,2,50000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,60.30230501674837,7.215824222592436,9.743599391588965,0,2023-01-15,NA +WA,B,Yes,9,14396,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,145.42306914087774,121.64141586332515,3.866020629940402,0,2023-01-15,result is not quantifiable +NM,C,yes,10,27604,5,1000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,2.403108102691187,14.04002632865471,1.3852148072601094,0,2023-01-15,temperature not assessed upon arrival at the laboratory +NM,C,yes,11,91578,6,10000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,1.0346387310851741,99.34613229408048,0.12024822764162499,0,2023-01-15,temperature not assessed upon arrival at the laboratory +XX,D,ye,13,65174,10,1000000000,wwtp,raw wastewater,copies/l wastewater,FLUAV,56.69014470402034,393.7601646806122,3.383593646869905,0,2023-01-15,NA +CA,A,YES,1,04971,1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,632.8401179640316,12.224419039706119,2.4220839813602884,0,2023-01-16,NA +CA,A,YES,2,31809,1,30000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,36.437595766791006,34.051180914187746,1.6254244912916882,0,2023-01-16,no +CA,A,YES,4,12078,3,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,297.3302317904181,401.17364384539303,9.87258502299475,0,2023-01-16,NA +CA,A,YES,4,12078,4,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,996.3301269518645,716.9768503635777,7.262153111508785,0,2023-01-16,no +WA,B,Yes,6,31928,1,2000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,224.30359201433757,388.66246571436386,1.2963794216212479,0,2023-01-16,no +WA,B,Yes,7,07528,1,100000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,86.18583858723275,18.98330712581169,8.454855376322739,0,2023-01-16,NA +WA,B,Yes,8,97385,2,50000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,39.787407589139285,9.321144696730963,12.862416580432402,0,2023-01-16,NA +WA,B,Yes,9,14396,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,70.43920442807422,10.107684327570373,3.3261656700476827,0,2023-01-16,NA +NM,C,yes,11,91578,6,10000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,0.8321668281694808,34.74190472778599,1.133196823833462,1,2023-01-16,below loq +XX,D,ye,15,98436,10,1000000000,wwtp,raw wastewater,copies/l wastewater,FLUAV,68.76978974336204,8.945084051899727,17.13221995466942,0,2023-01-16,NA +ZZ,F,NO,12,32541,1,8472,wwtp,raw wastewater,copies/g dry sludge,sars-cov-2,135.51672834177882,426.12127776912934,8.442434121409349,0,2023-01-16,NA +CA,A,YES,1,04971,1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,236.3829208832575,70.41303977777574,22.48688884133083,0,2023-01-17,no +CA,A,YES,2,31809,1,30000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,14.905461421620513,654.4029988440628,5.658387022894478,0,2023-01-17,no +CA,A,YES,5,34687,3,1000000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,14.96946780095108,7.52064900643304,1.3927662713952733,0,2023-01-17,no +CA,A,YES,4,12078,4,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,25.69083947943441,26.266272232406603,8.625654472821527,0,2023-01-17,NA +CA,A,YES,5,34687,4,1000000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,13.56862327150862,44.52048579470506,1.0049832763717805,0,2023-01-17,NA +WA,B,Yes,6,31928,1,2000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,78.17670563937182,40.49338892272207,3.2470445686183287,0,2023-01-17,NA +WA,B,Yes,7,07528,1,100000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,13.256884947615685,196.88346637585602,5.546666029155501,0,2023-01-17,y +WA,B,Yes,8,97385,2,50000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,2.987293679567218,115.24536163359764,5.974587359134436,1,2023-01-17,NA +WA,B,Yes,9,14396,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,2.8789863912248874,11.109698926579298,5.757972782449775,1,2023-01-17,NA +NM,C,yes,10,27604,5,1000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,1.143331880577684,30.683035959603508,0.7685363082971409,0,2023-01-17,NA +NM,C,yes,11,91578,6,10000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,2.9352758756073856,101.20949274243777,0.817111810674234,0,2023-01-17,no +CA,A,YES,1,04971,1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,289.68812960182566,356.71546463539164,5.989158818609282,0,2023-01-18,NA +CA,A,YES,2,31809,1,30000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,8.141488677344599,10.106787638088505,1.786335366416635,0,2023-01-18,NA +CA,A,YES,4,12078,3,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,348.18954712932003,566.3514558041516,9.564428529714396,0,2023-01-18,NA +WA,B,Yes,6,31928,1,2000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,70.31256445299974,444.63422358497667,1.5519225823961627,0,2023-01-18,no +WA,B,Yes,7,07528,1,100000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,237.28164446475813,216.03447864094858,18.301737767620455,0,2023-01-18,no +WA,B,Yes,8,97385,2,50000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,313.30894461015595,356.35107727233486,18.25696449276984,0,2023-01-18,> max temp and/or hold time +WA,B,Yes,9,14396,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,21.098923501980277,5.399623633367948,15.878066541258196,0,2023-01-18,NA +NM,C,yes,10,27604,5,1000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,2.3432421925110125,19.167766197915537,0.7435611458771272,0,2023-01-18,NA +NM,C,yes,11,91578,6,10000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,0.8541439983467669,42.17035049722412,1.1551739940107482,1,2023-01-18,NA +XX,D,ye,13,65174,10,1000000000,wwtp,raw wastewater,copies/l wastewater,FLUAV,3.0807037766533982,494.5700077765639,6.1614075533067965,1,2023-01-18,NA +YY,E,No,13,65174,2,13,upstream,primary sludge,copies/g dry sludge,sars-cov-2,414.3276152103089,601.7222517889131,2.122932988478446,0,2023-01-18,NA +CA,A,YES,1,04971,1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,7.79602795207444,19.92412929908597,1.4943600748960477,0,2023-01-19,NA +CA,A,YES,2,31809,1,30000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,306.13317174325095,464.775983801532,6.1760747272047585,0,2023-01-19,NA +CA,A,YES,4,12078,3,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,0.5027486094028235,462.13509119759544,1.005497218805647,1,2023-01-19,NA +WA,B,Yes,6,31928,1,2000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,215.49555011237982,289.71627111028477,21.433474267868558,0,2023-01-19,NA +WA,B,Yes,7,07528,1,100000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,1.7136822572700443,783.5772172028877,3.4273645145400886,1,2023-01-19,yes +WA,B,Yes,8,97385,2,50000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,29.574774722513,20.45645719529841,5.443076278674264,0,2023-01-19,NA +WA,B,Yes,9,14396,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,568.7106216236186,7.879249044194013,3.4905652020646643,0,2023-01-19,NA +NM,C,yes,10,27604,5,1000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,1.2236735182375196,16.81647738230983,0.18414728261815422,0,2023-01-19,NA +NM,C,yes,11,91578,6,10000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,2.0150771179599927,29.10049041164487,1.1291418930163737,0,2023-01-19,NA +CA,A,YES,1,04971,1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,103.2642046732852,5.814870566547347,11.602114739709226,0,2023-01-20,NA +CA,A,YES,2,31809,1,30000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,122.41214198032966,259.49349703379596,1.598403779569477,0,2023-01-20,NA +CA,A,YES,4,12078,3,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,81.86193115818583,41.86244658297052,20.232306962509405,0,2023-01-20,NA +CA,A,YES,4,12078,4,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,1.4251806032831906,286.24515648806636,2.8503612065663813,1,2023-01-20,no +WA,B,Yes,6,31928,1,2000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,201.23303389756353,16.451382197185772,20.021318134243305,0,2023-01-20,NA +WA,B,Yes,7,07528,1,100000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,756.7769895417883,160.70977903472937,5.548038483776634,0,2023-01-20,NA +WA,B,Yes,8,97385,2,50000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,104.14929192418167,69.4847962921983,19.109134845799755,0,2023-01-20,y +WA,B,Yes,9,14396,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,68.08611267500726,113.68409882934158,2.865003211968211,0,2023-01-20,NA +NM,C,yes,10,27604,5,1000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,1.843175810609984,49.523107462612565,0.4251905826721891,0,2023-01-20,no +NM,C,yes,11,91578,6,10000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,0.9145073336895877,168.02553578904363,0.007447682191439161,0,2023-01-20,NA +XX,D,ye,12,32541,10,1000000000,wwtp,raw wastewater,copies/l wastewater,FLUAV,803.4571253171747,136.55109830877655,4.7265355889827605,0,2023-01-20,temperature not assessed upon arrival at the laboratory +ZZ,F,NO,13,65174,2,8472,wwtp,raw wastewater,copies/g dry sludge,sars-cov-2,5.11580838822335,193.54000239526147,2.424713521902825,0,2023-01-20,NA +CA,A,YES,1,04971,1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,260.509256741051,749.4630810279345,6.307817618056269,0,2023-01-21,NA +CA,A,YES,2,31809,1,30000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,97.72824871596048,138.94851800465568,2.126669537839003,0,2023-01-21,NA +CA,A,YES,4,12078,4,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,134.998794085979,70.4671732544781,1.4395557466800735,0,2023-01-21,NA +WA,B,Yes,6,31928,1,2000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,5.158667463224353,232.2343659194519,3.611688522332628,0,2023-01-21,NA +WA,B,Yes,7,07528,1,100000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,7.742367365272937,11.541731418626943,6.054905700117429,0,2023-01-21,NA +WA,B,Yes,8,97385,2,50000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,13.682976182206449,26.597886093805492,4.7718765369549,0,2023-01-21,NA +WA,B,Yes,9,14396,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,406.83485287388635,24.02879102615582,1.5046238102057574,0,2023-01-21,no +NM,C,yes,10,27604,5,1000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,1.6137411930745136,183.0026740748744,1.1722579697564341,0,2023-01-21,NA +CA,A,YES,1,04971,1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,40.45204295439143,13.660265247606043,13.910212931525614,0,2023-01-22,NA +CA,A,YES,2,31809,1,30000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,321.62912370113264,30.23251957964165,5.276804697923777,0,2023-01-22,NA +CA,A,YES,4,12078,3,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,155.0613518200385,44.980610482006554,22.137322660998443,0,2023-01-22,NA +CA,A,YES,4,12078,4,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,10.759587967953939,54.71240692270756,21.519175935907878,1,2023-01-22,NA +WA,B,Yes,6,31928,1,2000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,269.083673256451,179.6342712848203,2.2292711832368153,0,2023-01-22,temperature not assessed upon arrival at the laboratory +WA,B,Yes,7,07528,1,100000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,0.5456469474753993,265.2781010391292,1.0912938949507986,1,2023-01-22,NA +WA,B,Yes,8,97385,2,50000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,238.10661965953364,805.6989709825211,3.098589615159992,0,2023-01-22,NA +WA,B,Yes,9,14396,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,466.72046938122975,977.874236607463,1.9765069242332014,0,2023-01-22,NA +NM,C,yes,10,27604,5,1000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,2.337993063809855,901.1432450089196,0.4391614700824327,0,2023-01-22,no +NM,C,yes,11,91578,6,10000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,2.2522941675025807,100.29423835344085,0.3948295896377767,0,2023-01-22,NA +XX,D,ye,12,32541,10,1000000000,wwtp,raw wastewater,copies/l wastewater,FLUAV,882.8295001437226,640.7885699797939,2.577651108783669,0,2023-01-22,no +XX,D,ye,13,65174,10,1000000000,wwtp,raw wastewater,copies/l wastewater,FLUAV,404.72664655904003,860.3990497627694,11.291638913382435,0,2023-01-22,NA +XX,D,ye,15,98436,10,1000000000,wwtp,raw wastewater,copies/l wastewater,FLUAV,21.437380096872687,120.20376617792512,9.087572955168046,0,2023-01-22,no +YY,E,No,14,09634,3,13,upstream,primary sludge,copies/g dry sludge,sars-cov-2,52.4904982368234,43.34837692043038,1.8407282030349492,0,2023-01-22,NA +CA,A,YES,1,04971,1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,17.666520139477633,10.127875083418793,19.803401555625793,0,2023-01-23,NA +CA,A,YES,2,31809,1,30000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,36.1060134606152,76.39784543054549,4.627565454833862,0,2023-01-23,NA +CA,A,YES,4,12078,3,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,5.5485088848553765,138.4158068122119,3.6685179725953256,0,2023-01-23,NA +CA,A,YES,4,12078,4,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,394.7123649531898,23.570814953547,5.588583688070053,0,2023-01-23,yes +WA,B,Yes,6,31928,1,2000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,290.6173745250686,304.1201720079703,2.477717183618308,0,2023-01-23,NA +WA,B,Yes,7,07528,1,100000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,320.3183862969877,395.0088214447861,8.157449812486812,0,2023-01-23,below loq +WA,B,Yes,8,97385,2,50000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,646.8810771253503,95.61209718916973,19.822252780052455,0,2023-01-23,NA +WA,B,Yes,9,14396,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,311.01625335114124,5.204735562398545,8.098384475075749,0,2023-01-23,no +NM,C,yes,10,27604,5,1000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,1.6667585450872147,58.783791392361124,0.7582909704443117,0,2023-01-23,NA +NM,C,yes,11,91578,6,10000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,0.6349732285274168,800.5142198789868,0.936003224191398,1,2023-01-23,NA +CA,A,YES,1,04971,1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,328.77787163502705,83.36518324074643,1.2192699469400203,0,2023-01-24,NA +CA,A,YES,2,31809,1,30000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,284.8536793579486,30.053574914421993,3.7727797575594777,0,2023-01-24,NA +CA,A,YES,4,12078,3,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,8.020263842534687,9.530036329157536,4.8581445181548935,0,2023-01-24,NA +CA,A,YES,5,34687,3,1000000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,27.591834380698437,580.8611473751317,7.009454448581375,0,2023-01-24,no +CA,A,YES,4,12078,4,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,13.409126779536637,364.3961722043428,23.35194035694984,0,2023-01-24,NA +CA,A,YES,5,34687,4,1000000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,23.791378883513556,569.4005964736075,15.608223318116881,0,2023-01-24,NA +WA,B,Yes,6,31928,1,2000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,135.07819512192518,37.357950613455586,1.0260045196785377,0,2023-01-24,NA +WA,B,Yes,7,07528,1,100000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,23.509761569662132,6.773093031915997,6.368645627704384,0,2023-01-24,no +WA,B,Yes,8,97385,2,50000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,8.90867326762054,31.22554152879125,1.6403981321468637,0,2023-01-24,NA +WA,B,Yes,9,14396,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,516.2856797551622,41.587346226683295,4.3657634509158605,0,2023-01-24,NA +NM,C,yes,10,27604,5,1000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,2.292145216000574,124.31395694627734,0.8806018005528857,0,2023-01-24,NA +NM,C,yes,11,91578,6,10000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,2.6632456647484126,183.56031882405125,0.3822003774740463,0,2023-01-24,NA +YY,E,No,12,32541,1,13,upstream,primary sludge,copies/g dry sludge,sars-cov-2,20.308589653879682,24.81607391430652,17.92523966687062,0,2023-01-24,no +CA,A,YES,1,04971,1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,138.33302679956105,360.73327421400006,21.9041912020723,0,2023-01-25,NA +CA,A,YES,4,12078,3,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,135.0965517596343,303.3282531353774,2.5437786616786906,0,2023-01-25,> max temp and/or hold time +CA,A,YES,4,12078,4,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,618.6090940077504,180.93925916697256,3.3630470829531594,0,2023-01-25,NA +WA,B,Yes,6,31928,1,2000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,277.5791282020149,9.880496307667432,5.090985261949685,0,2023-01-25,NA +WA,B,Yes,7,07528,1,100000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,49.77573075461759,6.512690230016329,1.1726964723402862,0,2023-01-25,no +WA,B,Yes,8,97385,2,50000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,6.0193935745246545,6.08939374475535,1.5321350896998966,0,2023-01-25,NA +WA,B,Yes,9,14396,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,548.4853958412981,817.5041793707269,11.98644542805899,0,2023-01-25,no +NM,C,yes,10,27604,5,1000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,1.480748537667162,31.448261185770857,1.1445946800508937,0,2023-01-25,no +NM,C,yes,11,91578,6,10000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,2.005172368484683,308.6894594474974,0.5266772836942144,0,2023-01-25,y +YY,E,No,12,32541,1,13,upstream,primary sludge,copies/g dry sludge,sars-cov-2,10.111626933394229,151.79571706844175,20.223253866788458,1,2023-01-25,NA +YY,E,No,15,98436,4,13,upstream,primary sludge,copies/g dry sludge,sars-cov-2,481.4358736157335,30.499865983027433,1.518807296004087,0,2023-01-25,result is not quantifiable +CA,A,YES,1,04971,1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,9.328578762817422,8.84809728269201,3.7340129757787976,0,2023-01-26,NA +CA,A,YES,2,31809,1,30000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,355.9679157333589,62.04743740549308,6.123814827844655,0,2023-01-26,NA +CA,A,YES,4,12078,3,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,32.49705444635645,23.88886046416477,5.100924666713219,0,2023-01-26,y +CA,A,YES,4,12078,4,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,51.362042353567304,45.2045845093206,9.803990573607825,0,2023-01-26,NA +WA,B,Yes,6,31928,1,2000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,6.799807848977796,8.67156408874369,1.627840855546493,0,2023-01-26,NA +WA,B,Yes,7,07528,1,100000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,333.52527371174165,720.4585624429182,5.365392848364999,0,2023-01-26,NA +WA,B,Yes,8,97385,2,50000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,26.181175405439532,724.4917420104347,12.852963089065417,0,2023-01-26,NA +WA,B,Yes,9,14396,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,29.47443345287017,623.5438410012663,3.7282945527925393,0,2023-01-26,NA +NM,C,yes,10,27604,5,1000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,2.9664432395583265,10.754979253186622,0.4415560387972022,0,2023-01-26,NA +NM,C,yes,11,91578,6,10000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,1.487929301198781,14.400617832464908,0.6915995313334659,0,2023-01-26,no +XX,D,ye,14,09634,10,1000000000,wwtp,raw wastewater,copies/l wastewater,FLUAV,34.3804254619772,420.02586967410815,5.037196334303159,0,2023-01-26,NA +CA,A,YES,1,04971,1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,6.624234663444247,676.3758121890099,13.248469326888493,1,2023-01-27,NA +CA,A,YES,2,31809,1,30000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,12.209407160343636,7.232133382092917,2.1601098912818673,0,2023-01-27,NA +CA,A,YES,4,12078,3,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,314.6375014637295,20.408566190230182,4.639711180604959,0,2023-01-27,NA +CA,A,YES,4,12078,4,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,9.425757644516645,80.14931063230614,19.628753309506365,0,2023-01-27,NA +WA,B,Yes,6,31928,1,2000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,26.945486801049366,478.1334514488874,9.26756794348223,0,2023-01-27,no +WA,B,Yes,7,07528,1,100000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,365.11403162240805,24.1085549809469,3.226935992785829,0,2023-01-27,no +WA,B,Yes,8,97385,2,50000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,396.21285625895035,23.282108069247435,11.383299456708617,0,2023-01-27,no +NM,C,yes,10,27604,5,1000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,2.5842191686971283,25.597940718539054,0.3989386997493572,0,2023-01-27,NA +NM,C,yes,11,91578,6,10000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,1.8075596249629944,478.42352367736515,0.1899299935386595,0,2023-01-27,NA +ZZ,F,NO,14,09634,3,8472,wwtp,raw wastewater,copies/g dry sludge,sars-cov-2,9.672377940423436,6.428477048138668,4.591763639719992,0,2023-01-27,no +CA,A,YES,1,04971,1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,217.18311574688119,210.68691679343732,1.083315522601008,0,2023-01-28,> max temp and/or hold time +CA,A,YES,2,31809,1,30000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,51.59143891731625,11.916425004334544,15.757811618162995,0,2023-01-28,NA +CA,A,YES,4,12078,3,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,299.94997972292424,74.89757745870757,20.711695926081383,0,2023-01-28,NA +CA,A,YES,4,12078,4,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,0.5684788354997882,435.598195141075,1.1369576709995763,1,2023-01-28,NA +WA,B,Yes,6,31928,1,2000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,149.72148140646118,64.08095433494701,4.943832960435076,0,2023-01-28,NA +WA,B,Yes,7,07528,1,100000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,657.0660361809229,37.48438675004846,3.3211668971862696,0,2023-01-28,NA +WA,B,Yes,8,97385,2,50000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,718.1161199756948,163.33976423764327,6.269041721580137,0,2023-01-28,NA +WA,B,Yes,9,14396,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,38.32790168254147,623.5758786648391,22.580147948169653,0,2023-01-28,NA +NM,C,yes,10,27604,5,1000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,1.0346842586094893,343.14621025625723,0.11433932555462889,0,2023-01-28,NA +NM,C,yes,11,91578,6,10000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,2.37709697209051,41.272380300827685,1.0102363278120257,0,2023-01-28,NA +YY,E,No,14,09634,3,13,upstream,primary sludge,copies/g dry sludge,sars-cov-2,730.0147753108231,55.529689312050664,1.3159731786348885,0,2023-01-28,no +ZZ,F,NO,13,65174,2,8472,wwtp,raw wastewater,copies/g dry sludge,sars-cov-2,207.31955275313456,23.65128861950823,11.51907119348711,0,2023-01-28,no +CA,A,YES,1,04971,1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,289.6888018012041,12.186631110177862,5.699175937236678,0,2023-01-29,no +CA,A,YES,2,31809,1,30000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,21.329204225856117,385.0533558978367,10.583829822675794,0,2023-01-29,NA +CA,A,YES,4,12078,3,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,53.11179734854606,23.24124229097715,3.2398640591945953,0,2023-01-29,no +WA,B,Yes,6,31928,1,2000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,301.31166839716354,148.24387063960305,1.087629233391115,0,2023-01-29,NA +WA,B,Yes,7,07528,1,100000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,15.479445618816932,8.136414652575812,2.6649134665521994,0,2023-01-29,y +WA,B,Yes,8,97385,2,50000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,14.705164177513977,495.25725951441257,21.170110466868074,0,2023-01-29,NA +WA,B,Yes,9,14396,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,143.248333281356,432.386337894861,2.487612286674729,0,2023-01-29,yes +NM,C,yes,10,27604,5,1000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,0.20041525681581643,69.04134018020609,0.5014452524797977,1,2023-01-29,NA +NM,C,yes,11,91578,6,10000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,2.5581268199589866,7.613930458539535,0.7010876893135397,0,2023-01-29,NA +CA,A,YES,1,04971,1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,2.4337899540452472,541.6026112926763,4.8675799080904945,1,2023-01-30,NA +CA,A,YES,4,12078,3,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,31.57216795771621,64.28640207766581,7.092061280498464,0,2023-01-30,NA +CA,A,YES,4,12078,4,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,22.591060831366768,26.722281332326776,5.808664988355861,0,2023-01-30,NA +WA,B,Yes,6,31928,1,2000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,21.473073860544346,95.16273227196038,1.5150120356366488,0,2023-01-30,NA +WA,B,Yes,7,07528,1,100000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,197.71560614442018,386.5913071762268,10.958507298542244,0,2023-01-30,NA +WA,B,Yes,8,97385,2,50000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,446.3323037045921,145.2491436823739,17.156970453045762,0,2023-01-30,NA +NM,C,yes,10,27604,5,1000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,2.893090891489194,63.642010489879496,0.3593943293178801,0,2023-01-30,NA +NM,C,yes,11,91578,6,10000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,0.8783642285197524,10.53940055434671,0.3008368551794367,0,2023-01-30,> max temp and/or hold time +XX,D,ye,12,32541,10,1000000000,wwtp,raw wastewater,copies/l wastewater,FLUAV,79.70782874204784,17.671732544338127,1.9559914962536087,0,2023-01-30,NA +CA,A,YES,1,04971,1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,13.82721164208279,6.021428455246692,1.0250892949476675,0,2023-01-31,NA +CA,A,YES,2,31809,1,30000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,1.835298210003826,9.100682890365364,3.670596420007652,1,2023-01-31,y +CA,A,YES,4,12078,3,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,1.672435573532772,7.64320342925198,3.344871147065544,1,2023-01-31,NA +CA,A,YES,5,34687,3,1000000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,6.448425465052048,26.641340333632552,15.023179396429876,0,2023-01-31,NA +CA,A,YES,4,12078,4,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,46.46574530469409,31.93189285934334,10.63677237194646,0,2023-01-31,NA +CA,A,YES,5,34687,4,1000000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,5.387921707544142,84.79424485209003,10.775843415088284,1,2023-01-31,below loq +WA,B,Yes,6,31928,1,2000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,31.133401616727127,568.0570635934372,2.859741665677186,0,2023-01-31,NA +WA,B,Yes,7,07528,1,100000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,678.2869411407843,53.58106806376892,12.889546726634627,0,2023-01-31,NA +WA,B,Yes,8,97385,2,50000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,612.1672577203768,28.43368588202369,13.604990112452253,0,2023-01-31,NA +WA,B,Yes,9,14396,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,849.3779358753316,6.409224577273975,24.22686551854024,0,2023-01-31,NA +NM,C,yes,10,27604,5,1000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,2.5379202952160353,39.339055282510536,1.2125602497941623,0,2023-01-31,no +NM,C,yes,11,91578,6,10000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,1.0727064699006164,22.575325140881382,0.22748404840855116,0,2023-01-31,NA +XX,D,ye,14,09634,10,1000000000,wwtp,raw wastewater,copies/l wastewater,FLUAV,391.888870161768,140.42276334509128,4.721303128622876,0,2023-01-31,NA +XX,D,ye,15,98436,10,1000000000,wwtp,raw wastewater,copies/l wastewater,FLUAV,92.87369611467575,26.305207506256373,10.835275255844284,0,2023-01-31,NA +CA,A,YES,1,04971,1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,66.73897564847593,6.38105523650712,3.8713058962796816,0,2023-02-01,NA +CA,A,YES,2,31809,1,30000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,188.61607843839352,46.19107449073388,1.239523788927882,0,2023-02-01,NA +CA,A,YES,4,12078,3,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,208.53271839797665,694.0037736798627,5.226708828886194,0,2023-02-01,NA +CA,A,YES,4,12078,4,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,73.476874788484,440.98584091397856,4.426868836864933,0,2023-02-01,NA +WA,B,Yes,6,31928,1,2000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,257.43834697052165,208.80187090085778,3.6797231858698405,0,2023-02-01,NA +WA,B,Yes,7,07528,1,100000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,955.3517728519003,141.43022160641425,4.764679986673232,0,2023-02-01,NA +WA,B,Yes,8,97385,2,50000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,68.97609646759618,498.110389601371,5.135349852059051,0,2023-02-01,NA +WA,B,Yes,9,14396,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,4.223326353869056,44.61407496300907,8.446652707738112,1,2023-02-01,NA +NM,C,yes,10,27604,5,1000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,0.102596266322803,947.6088251324358,0.4036262619867842,1,2023-02-01,NA +ZZ,F,NO,12,32541,1,8472,wwtp,raw wastewater,copies/g dry sludge,sars-cov-2,25.540771964804886,808.133160849064,4.558116694393982,0,2023-02-01,NA +ZZ,F,NO,14,09634,3,8472,wwtp,raw wastewater,copies/g dry sludge,sars-cov-2,84.23735448471687,12.541185989167685,11.701213170815196,0,2023-02-01,yes +CA,A,YES,1,04971,1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,13.047102928555004,51.050394457828354,20.101885729743387,0,2023-02-02,below loq +CA,A,YES,4,12078,4,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,13.55723029618432,9.944896930988406,15.63562992133033,0,2023-02-02,NA +WA,B,Yes,6,31928,1,2000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,5.6439838340373605,134.32277764467858,7.856527967929066,0,2023-02-02,no +WA,B,Yes,7,07528,1,100000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,191.22411483095232,16.477580532489196,13.9144867220376,0,2023-02-02,no +WA,B,Yes,8,97385,2,50000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,2.5195035918167683,453.979213280071,5.039007183633537,1,2023-02-02,result is not quantifiable +WA,B,Yes,9,14396,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,37.914159978131806,26.050805383285905,3.3186565309966913,0,2023-02-02,NA +NM,C,yes,10,27604,5,1000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,2.9413198948781702,322.6762550255326,1.245188607472911,0,2023-02-02,NA +NM,C,yes,11,91578,6,10000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,2.4128066466997202,96.41583338847116,0.09677275641250235,0,2023-02-02,NA +XX,D,ye,13,65174,10,1000000000,wwtp,raw wastewater,copies/l wastewater,FLUAV,341.8510038167221,952.3035479561282,10.27615425581342,0,2023-02-02,temperature not assessed upon arrival at the laboratory +YY,E,No,15,98436,4,13,upstream,primary sludge,copies/g dry sludge,sars-cov-2,7.133673799941099,54.85634227500369,6.050891649875597,0,2023-02-02,NA +CA,A,YES,2,31809,1,30000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,757.0573443619562,200.38345545665464,2.5626400078432092,0,2023-02-03,no +CA,A,YES,4,12078,3,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,18.41310152071136,34.804157183358846,2.9385001424293486,0,2023-02-03,NA +CA,A,YES,4,12078,4,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,11.306257022433117,8.688553699112347,16.30036057079285,0,2023-02-03,NA +WA,B,Yes,6,31928,1,2000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,152.20054977036764,132.4258669111194,20.336568256032432,0,2023-02-03,NA +WA,B,Yes,7,07528,1,100000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,917.9992475147316,26.198936877962097,19.49689599505458,0,2023-02-03,NA +WA,B,Yes,8,97385,2,50000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,11.691373103239751,66.77928913375712,1.4236894708754788,0,2023-02-03,temperature not assessed upon arrival at the laboratory +WA,B,Yes,9,14396,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,14.657741626252527,329.1452964917317,2.881426507331583,0,2023-02-03,NA +NM,C,yes,10,27604,5,1000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,2.63145526552871,5.4225789572658405,1.0253510815905353,0,2023-02-03,NA +NM,C,yes,11,91578,6,10000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,1.919555735563699,40.980532016287626,0.20842236567227088,0,2023-02-03,NA +XX,D,ye,12,32541,10,1000000000,wwtp,raw wastewater,copies/l wastewater,FLUAV,9.255631927427881,177.64107074714565,9.638119621319293,0,2023-02-03,NA +YY,E,No,13,65174,2,13,upstream,primary sludge,copies/g dry sludge,sars-cov-2,2.6032578243761786,924.83945106976,5.206515648752357,1,2023-02-03,no +ZZ,F,NO,12,32541,1,8472,wwtp,raw wastewater,copies/g dry sludge,sars-cov-2,980.3346685982729,541.5162006690184,2.715020621781234,0,2023-02-03,NA +ZZ,F,NO,13,65174,2,8472,wwtp,raw wastewater,copies/g dry sludge,sars-cov-2,9.87950269943077,14.502760975383495,4.267051083211353,0,2023-02-03,NA +CA,A,YES,1,04971,1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,25.71421568374014,238.04162383662378,10.183439487702529,0,2023-02-04,no +CA,A,YES,2,31809,1,30000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,487.248639541455,41.91247925009177,2.2927868555741258,0,2023-02-04,NA +CA,A,YES,4,12078,3,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,854.7881696976922,12.514281776092561,2.522963315695632,0,2023-02-04,NA +CA,A,YES,4,12078,4,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,96.78507582948151,7.615539810795813,1.5871102032286302,0,2023-02-04,NA +WA,B,Yes,6,31928,1,2000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,22.41267337829998,560.3655986299398,20.80987934302243,0,2023-02-04,NA +WA,B,Yes,7,07528,1,100000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,2.077655152688752,304.7435354640638,4.155310305377504,1,2023-02-04,NA +WA,B,Yes,8,97385,2,50000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,13.645220493328909,8.400301728103852,4.396813407116788,0,2023-02-04,NA +WA,B,Yes,9,14396,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,8.525360081020944,32.2712815392429,14.285514150686454,0,2023-02-04,no +NM,C,yes,10,27604,5,1000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,1.1984263686289522,250.5405711864802,0.9502600786005623,0,2023-02-04,NA +NM,C,yes,11,91578,6,10000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,0.10815536893350523,102.15710926817935,0.40918536459748645,1,2023-02-04,NA +XX,D,ye,12,32541,10,1000000000,wwtp,raw wastewater,copies/l wastewater,FLUAV,6.893554189617403,89.40184366041713,7.374746965403133,0,2023-02-04,no +CA,A,YES,1,04971,1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,689.8299461380375,88.05279343129827,3.073115833003546,0,2023-02-05,NA +CA,A,YES,4,12078,3,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,5.697621222918042,323.63889750588254,3.5917156404986654,0,2023-02-05,temperature not assessed upon arrival at the laboratory +CA,A,YES,4,12078,4,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,37.67655006669244,72.9683305233421,14.964856624870379,0,2023-02-05,NA +WA,B,Yes,6,31928,1,2000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,11.426687900057091,5.428435138898313,1.7767031915939124,0,2023-02-05,NA +WA,B,Yes,7,07528,1,100000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,362.00179859055584,78.43234977707188,3.0882382154765975,0,2023-02-05,NA +WA,B,Yes,8,97385,2,50000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,367.3555035252702,367.580224915589,3.891212284787295,0,2023-02-05,no +WA,B,Yes,9,14396,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,91.05034120233795,307.0591598404405,1.8426035872331117,0,2023-02-05,NA +NM,C,yes,10,27604,5,1000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,0.6486532266552174,856.294091366936,0.94968322231919855,1,2023-02-05,NA +NM,C,yes,11,91578,6,10000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,2.267563318248261,128.37634308763091,0.4334954699771545,0,2023-02-05,> max temp and/or hold time +YY,E,No,15,98436,4,13,upstream,primary sludge,copies/g dry sludge,sars-cov-2,43.51361807880693,348.58351428401477,5.66814556133341,0,2023-02-05,NA +CA,A,YES,1,04971,1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,487.1800743189516,26.80129255300873,13.729088207939094,0,2023-02-06,no +CA,A,YES,2,31809,1,30000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,18.313384417993927,156.02446006488861,4.340232292086359,0,2023-02-06,no +CA,A,YES,4,12078,3,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,6.749889852835903,11.032782852623546,1.7719265545339626,0,2023-02-06,no +CA,A,YES,4,12078,4,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,41.56934752978162,140.3156934178459,11.932798995077206,0,2023-02-06,NA +WA,B,Yes,6,31928,1,2000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,666.5607815248727,28.818359823642385,11.094828707941558,0,2023-02-06,NA +WA,B,Yes,7,07528,1,100000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,106.03623854459045,45.6325461531413,2.6145533186874212,0,2023-02-06,NA +WA,B,Yes,9,14396,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,205.06565525715305,6.077360065661159,2.8503741157220603,0,2023-02-06,no +NM,C,yes,10,27604,5,1000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,2.058593289408069,29.332446303741694,1.0127021012002795,0,2023-02-06,NA +NM,C,yes,11,91578,6,10000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,0.8577636471126031,208.83574776457928,1.2890701076005573,0,2023-02-06,NA +ZZ,F,NO,14,09634,3,8472,wwtp,raw wastewater,copies/g dry sludge,sars-cov-2,555.9169131090603,808.802189633116,5.733693854097275,0,2023-02-06,below loq +CA,A,YES,1,04971,1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,22.145805633858085,53.19228646653339,1.796493213395071,0,2023-02-07,y +CA,A,YES,2,31809,1,30000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,950.6634872825501,659.3661533790163,3.0556026411256902,0,2023-02-07,NA +CA,A,YES,4,12078,3,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,166.2955332025867,241.50062531616834,2.579510711822856,0,2023-02-07,NA +CA,A,YES,5,34687,3,1000000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,155.40602919052714,32.046009045815204,18.863957982039132,0,2023-02-07,NA +CA,A,YES,5,34687,4,1000000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,206.77874649475413,183.28262230424306,1.4683649240357233,0,2023-02-07,temperature not assessed upon arrival at the laboratory +WA,B,Yes,6,31928,1,2000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,36.469248641609774,29.543568799825657,2.114283464440757,0,2023-02-07,no +WA,B,Yes,8,97385,2,50000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,22.022334116785338,107.04278462449535,9.121124565340267,0,2023-02-07,NA +WA,B,Yes,9,14396,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,410.27409498421474,41.380242600640535,6.24075198143086,0,2023-02-07,NA +NM,C,yes,10,27604,5,1000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,0.83184270632383,420.90049742307593,0.49513981933024576,0,2023-02-07,yes +NM,C,yes,11,91578,6,10000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,2.1543317546093776,16.7590713602393,0.8372679660648364,0,2023-02-07,NA +CA,A,YES,1,04971,1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,77.42659147330004,6.05283506543072,2.349344073250793,0,2023-02-08,NA +CA,A,YES,2,31809,1,30000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,179.30789811696954,31.642211026259535,1.705674515325716,0,2023-02-08,NA +CA,A,YES,4,12078,3,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,41.22393527247582,239.60794281224048,2.5170843394669875,0,2023-02-08,NA +CA,A,YES,4,12078,4,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,83.21898933944021,545.6380254123185,20.712043538546645,0,2023-02-08,NA +WA,B,Yes,6,31928,1,2000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,52.21090787857168,20.898651744638073,5.634775619774309,0,2023-02-08,NA +WA,B,Yes,7,07528,1,100000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,275.0359828254286,542.3252812208773,10.403386930465565,0,2023-02-08,NA +WA,B,Yes,8,97385,2,50000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,1.3866677814403519,859.8939124291788,2.7733355628807037,1,2023-02-08,NA +WA,B,Yes,9,14396,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,577.4170779169992,593.9444444469693,14.507738773896499,0,2023-02-08,NA +NM,C,yes,10,27604,5,1000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,1.5803489299391544,199.99749472015387,0.924278089010746,0,2023-02-08,NA +NM,C,yes,11,91578,6,10000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,2.8900083835911192,693.689630374196,0.2958298063738046,0,2023-02-08,NA +CA,A,YES,1,04971,1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,812.9826106403942,36.26527894451485,9.318355394326483,0,2023-02-09,NA +CA,A,YES,2,31809,1,30000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,58.81496709419618,106.12022504455213,1.6002230106207778,0,2023-02-09,NA +CA,A,YES,4,12078,3,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,343.3431146412499,5.2824260648441035,2.3682424690322543,0,2023-02-09,NA +CA,A,YES,4,12078,4,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,171.43529648525683,119.01545720314303,2.2030530541532998,0,2023-02-09,NA +WA,B,Yes,6,31928,1,2000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,175.09298648009462,45.63914307590408,11.421617712844483,0,2023-02-09,NA +WA,B,Yes,7,07528,1,100000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,11.128947279213488,320.28560924790276,4.597474543500214,0,2023-02-09,no +WA,B,Yes,8,97385,2,50000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,124.2072445232873,247.33608554176058,1.5522000618540654,0,2023-02-09,NA +WA,B,Yes,9,14396,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,695.0059354522365,443.5532232315415,1.7527968676901575,0,2023-02-09,no +NM,C,yes,10,27604,5,1000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,0.9792318637749995,166.3738201916728,1.2802618594389807,1,2023-02-09,NA +NM,C,yes,11,91578,6,10000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,2.7309414036927997,240.20559592957258,0.3677788298311789,0,2023-02-09,NA +ZZ,F,NO,15,98436,4,8472,wwtp,raw wastewater,copies/g dry sludge,sars-cov-2,5.354270394091138,6.091304691687146,9.4510707743411,0,2023-02-09,NA +CA,A,YES,1,04971,1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,612.8199257092496,264.1565679134045,15.825119680643422,0,2023-02-10,NA +CA,A,YES,2,31809,1,30000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,25.068353989081764,50.0360958940333,22.128478717635208,0,2023-02-10,temperature not assessed upon arrival at the laboratory +CA,A,YES,4,12078,3,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,34.41752828189917,711.188002245272,1.4981778284877636,0,2023-02-10,NA +CA,A,YES,4,12078,4,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,20.122215416166394,180.6336568057654,5.255159084782216,0,2023-02-10,y +WA,B,Yes,6,31928,1,2000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,74.64798716531446,208.29080423748468,18.425343741512823,0,2023-02-10,NA +WA,B,Yes,7,07528,1,100000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,186.66745436312686,6.483236640978133,13.726380854494069,0,2023-02-10,NA +WA,B,Yes,8,97385,2,50000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,52.73500854807239,7.401569976963654,6.781785174795238,0,2023-02-10,NA +WA,B,Yes,9,14396,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,6.734919684982307,7.5459281864131365,13.469839369964614,1,2023-02-10,NA +NM,C,yes,10,27604,5,1000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,2.217883349300206,865.2267833512358,1.0060218392362998,0,2023-02-10,result is not quantifiable +NM,C,yes,11,91578,6,10000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,0.6585610419352896,8.534975385086046,0.9595910375992709,1,2023-02-10,no +CA,A,YES,1,04971,1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,314.7503986972577,355.80233751648694,12.63157077178409,0,2023-02-11,NA +CA,A,YES,2,31809,1,30000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,990.293899521586,632.6046303636238,4.981723547495451,0,2023-02-11,no +CA,A,YES,4,12078,3,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,40.37592283448573,8.643644183933917,1.5387259632940757,0,2023-02-11,NA +CA,A,YES,4,12078,4,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,96.49415443725236,17.610640342157396,1.98779600518458,0,2023-02-11,NA +WA,B,Yes,7,07528,1,100000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,19.852903882184243,9.350051289981593,23.981455675522536,0,2023-02-11,no +WA,B,Yes,8,97385,2,50000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,773.0257203484232,184.78035421131548,7.711455920129094,0,2023-02-11,NA +WA,B,Yes,9,14396,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,10.567557509692255,320.33545715281474,21.13511501938451,1,2023-02-11,NA +NM,C,yes,10,27604,5,1000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,2.1218191425620607,108.80888021324046,0.8123397334503972,0,2023-02-11,NA +ZZ,F,NO,13,65174,2,8472,wwtp,raw wastewater,copies/g dry sludge,sars-cov-2,98.56120607383399,105.47838717817903,12.532443569901066,0,2023-02-11,yes +CA,A,YES,2,31809,1,30000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,74.51655901889292,130.11577712144495,1.3595678562994715,0,2023-02-12,NA +CA,A,YES,4,12078,3,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,215.14603278230834,65.52484024894848,6.697337153094882,0,2023-02-12,NA +CA,A,YES,4,12078,4,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,74.31925342133682,161.7395412023269,12.662826954420847,0,2023-02-12,NA +WA,B,Yes,6,31928,1,2000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,421.72072251194186,563.6680830535669,4.678075101950242,0,2023-02-12,no +WA,B,Yes,7,07528,1,100000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,5.639828625431392,181.14037175474104,4.045415878992869,0,2023-02-12,NA +WA,B,Yes,8,97385,2,50000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,32.57290476214622,87.79570136496618,1.7099287048268954,0,2023-02-12,yes +WA,B,Yes,9,14396,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,20.56961440402346,35.47381997189633,10.958372698389978,0,2023-02-12,no +NM,C,yes,10,27604,5,1000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,2.7106199374743167,172.43078957093235,1.1374964536254133,0,2023-02-12,NA +NM,C,yes,11,91578,6,10000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,2.462552111121334,490.80322072405033,0.19742319554213886,0,2023-02-12,no +CA,A,YES,1,04971,1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,81.25881347500219,653.9280096040354,2.6648826415257783,0,2023-02-13,NA +CA,A,YES,2,31809,1,30000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,5.395411487557759,38.84213917064014,1.8958261608896645,0,2023-02-13,NA +CA,A,YES,4,12078,3,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,209.1094824154509,23.524237707806382,8.137972534741653,0,2023-02-13,yes +CA,A,YES,4,12078,4,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,177.16156686350635,12.252234182374197,1.6174961110324442,0,2023-02-13,> max temp and/or hold time +WA,B,Yes,6,31928,1,2000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,278.17211478829796,27.3960873379447,1.331808333535425,0,2023-02-13,NA +WA,B,Yes,7,07528,1,100000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,3.7319199370098715,18.553773453864867,7.463839874019743,1,2023-02-13,NA +WA,B,Yes,8,97385,2,50000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,11.847983775680005,29.47894443470502,11.828682416214651,0,2023-02-13,no +WA,B,Yes,9,14396,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,521.2964079072428,11.227856507337526,3.017533732170826,0,2023-02-13,result is not quantifiable +NM,C,yes,10,27604,5,1000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,1.446367431104645,312.2584376354574,1.129455517956333,0,2023-02-13,no +NM,C,yes,11,91578,6,10000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,1.2217211032603674,599.6263649041317,0.0571878782858908,0,2023-02-13,temperature not assessed upon arrival at the laboratory +YY,E,No,13,65174,2,13,upstream,primary sludge,copies/g dry sludge,sars-cov-2,447.35463144094143,20.659681230914632,16.69156854551457,0,2023-02-13,NA +CA,A,YES,1,04971,1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,2.50735666511393,333.1255535218914,5.01471333022786,1,2023-02-14,NA +CA,A,YES,2,31809,1,30000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,937.3949039579877,29.958018337706235,12.339335825863003,0,2023-02-14,NA +CA,A,YES,4,12078,3,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,670.6211184607217,280.7963060482757,4.29525458734857,0,2023-02-14,NA +CA,A,YES,4,12078,4,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,14.7833143140836,747.6107215278645,2.1505469190720947,0,2023-02-14,NA +CA,A,YES,5,34687,4,1000000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,36.3060279145444,146.18038775681904,4.6746096492268405,0,2023-02-14,no +WA,B,Yes,6,31928,1,2000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,31.99336511056609,25.07148468062684,9.068368050626622,0,2023-02-14,NA +WA,B,Yes,7,07528,1,100000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,840.6498215878155,6.7367829895379625,18.840587824147992,0,2023-02-14,NA +WA,B,Yes,8,97385,2,50000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,78.63149083020775,120.22155468207438,1.497532403535465,0,2023-02-14,NA +WA,B,Yes,9,14396,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,85.49147571760332,111.35377793707082,1.7770300173057934,0,2023-02-14,NA +NM,C,yes,10,27604,5,1000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,1.221399960068312,48.57641066448648,0.6229325396058031,0,2023-02-14,no +XX,D,ye,13,65174,10,1000000000,wwtp,raw wastewater,copies/l wastewater,FLUAV,603.8662595055229,73.81653689586987,1.597500410002734,0,2023-02-14,NA +XX,D,ye,15,98436,10,1000000000,wwtp,raw wastewater,copies/l wastewater,FLUAV,20.888749907285764,373.287549139169,24.00740101489395,0,2023-02-14,NA +YY,E,No,12,32541,1,13,upstream,primary sludge,copies/g dry sludge,sars-cov-2,5.286663714892587,20.0029813551299,1.0466216487692521,0,2023-02-14,NA +CA,A,YES,1,04971,1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,5.052700248787683,15.74826735545662,10.105400497575365,1,2023-02-15,no +CA,A,YES,2,31809,1,30000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,1.9352873893478986,60.52387303757406,3.870574778695797,1,2023-02-15,NA +CA,A,YES,4,12078,3,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,1.2842785703500454,15.983567608218957,2.568557140700091,1,2023-02-15,NA +CA,A,YES,4,12078,4,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,926.2736155542457,78.09869378954305,4.617464034106609,0,2023-02-15,NA +WA,B,Yes,6,31928,1,2000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,951.7652669873343,5.648094592120505,2.0522406592637883,0,2023-02-15,NA +WA,B,Yes,8,97385,2,50000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,107.34332857568265,55.11322470918748,22.90898687455756,0,2023-02-15,NA +WA,B,Yes,9,14396,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,137.54496302444423,79.67635678295045,1.5464088453180767,0,2023-02-15,NA +NM,C,yes,10,27604,5,1000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,0.7194720136818241,60.70034297118233,1.0205020093458053,1,2023-02-15,NA +NM,C,yes,11,91578,6,10000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,2.513197412630603,24.20735928613179,0.917511088033159,0,2023-02-15,NA +ZZ,F,NO,14,09634,3,8472,wwtp,raw wastewater,copies/g dry sludge,sars-cov-2,10.974171814623093,196.64847576389664,10.970368150392106,0,2023-02-15,yes +CA,A,YES,1,04971,1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,6.804398813682943,89.56254239127894,13.608797627365886,1,2023-02-16,NA +CA,A,YES,2,31809,1,30000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,71.30523320972318,514.3788429754185,1.301444470125622,0,2023-02-16,no +CA,A,YES,4,12078,4,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,0.7129218814336417,79.41054123174462,1.4258437628672833,1,2023-02-16,NA +WA,B,Yes,7,07528,1,100000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,247.9346152512209,62.690070085456796,9.756379214605827,0,2023-02-16,NA +WA,B,Yes,8,97385,2,50000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,229.6668748655777,523.4652870766724,17.79501819555069,0,2023-02-16,NA +WA,B,Yes,9,14396,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,74.76349542178585,216.0187867481684,4.960643116660774,0,2023-02-16,NA +NM,C,yes,10,27604,5,1000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,1.6078897112089805,231.466427409066,0.896783121677392,0,2023-02-16,NA +CA,A,YES,1,04971,1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,14.382556035028882,97.66500858852643,19.278770342928855,0,2023-02-17,NA +CA,A,YES,2,31809,1,30000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,148.79587342560575,70.03519617413565,2.0681671603404626,0,2023-02-17,no +CA,A,YES,4,12078,3,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,373.3957858548941,58.321648736481706,9.5168866792719,0,2023-02-17,NA +CA,A,YES,4,12078,4,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,69.96024832334722,934.9199936704985,2.428371848526688,0,2023-02-17,NA +WA,B,Yes,6,31928,1,2000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,338.40465021930845,5.694040859500567,24.211322702483272,0,2023-02-17,NA +WA,B,Yes,7,07528,1,100000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,8.874069917072676,68.23262260396105,9.336072463948186,0,2023-02-17,NA +WA,B,Yes,8,97385,2,50000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,97.760488028523,11.195296953677103,2.858427553957841,0,2023-02-17,NA +WA,B,Yes,9,14396,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,206.03755957505484,167.07250344667585,16.639227707234344,0,2023-02-17,NA +NM,C,yes,10,27604,5,1000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,1.4723001466134007,5.779227619791328,1.0073333470657468,0,2023-02-17,NA +ZZ,F,NO,13,65174,2,8472,wwtp,raw wastewater,copies/g dry sludge,sars-cov-2,3.4524376286360603,97.94643970382845,6.904875257272121,1,2023-02-17,no +CA,A,YES,1,04971,1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,9.475867017783791,246.73122089405535,18.951734035567583,1,2023-02-18,NA +CA,A,YES,2,31809,1,30000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,0.5108120441987868,9.022630079050423,1.0216240883975736,1,2023-02-18,NA +CA,A,YES,4,12078,3,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,20.184926664672165,78.63624546051453,1.1076634668362468,0,2023-02-18,no +CA,A,YES,4,12078,4,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,146.75359140523335,21.11498460899427,9.621866380642274,0,2023-02-18,no +WA,B,Yes,6,31928,1,2000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,38.98563161981508,33.558375698952865,15.445967914968106,0,2023-02-18,no +WA,B,Yes,7,07528,1,100000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,2.891382993385002,70.05784945520003,5.782765986770004,1,2023-02-18,NA +WA,B,Yes,8,97385,2,50000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,848.5065702481577,729.2342315675392,11.02093785964551,0,2023-02-18,NA +WA,B,Yes,9,14396,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,6.03136309909273,615.8848515659726,8.948666606790095,0,2023-02-18,NA +NM,C,yes,10,27604,5,1000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,1.639394892508407,231.0575441335458,0.6030825982265398,0,2023-02-18,NA +NM,C,yes,11,91578,6,10000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,2.9609059650615306,335.9459477829114,1.1348647749037513,0,2023-02-18,NA +CA,A,YES,2,31809,1,30000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,20.160594685608125,22.600771548372215,3.390728483551403,0,2023-02-19,no +CA,A,YES,4,12078,3,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,29.655738090674486,106.00019845899403,5.929483441053241,0,2023-02-19,y +WA,B,Yes,6,31928,1,2000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,404.35855572337414,26.083038440250597,3.0042637396147818,0,2023-02-19,NA +WA,B,Yes,7,07528,1,100000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,79.97864196009634,54.434921251423816,3.650589315100694,0,2023-02-19,no +WA,B,Yes,8,97385,2,50000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,16.50778866263999,18.296165891169654,20.986638153001977,0,2023-02-19,NA +NM,C,yes,10,27604,5,1000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,-0.0727433361020844,598.9170521670416,0.2282866595618968,1,2023-02-19,NA +NM,C,yes,11,91578,6,10000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,2.0608509293645128,223.04084460217635,0.8141227776840516,0,2023-02-19,NA +ZZ,F,NO,12,32541,1,8472,wwtp,raw wastewater,copies/g dry sludge,sars-cov-2,643.4869872696589,5.688777144880834,5.316157370467915,0,2023-02-19,> max temp and/or hold time +CA,A,YES,1,04971,1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,66.40146942636754,21.070560662823272,18.06404763401332,0,2023-02-20,NA +CA,A,YES,2,31809,1,30000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,707.3062970567546,28.682977566339932,12.083412623285835,0,2023-02-20,NA +CA,A,YES,4,12078,3,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,34.68781732675998,9.499033562696464,2.9413455274546685,0,2023-02-20,NA +CA,A,YES,4,12078,4,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,11.691595963414354,600.0035216992512,1.1364982692267527,0,2023-02-20,no +WA,B,Yes,6,31928,1,2000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,242.81364858996707,38.81710409648819,13.007714759961274,0,2023-02-20,NA +WA,B,Yes,7,07528,1,100000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,20.735184183433628,5.2715010947272365,17.07513663922713,0,2023-02-20,NA +WA,B,Yes,8,97385,2,50000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,709.0355360889283,712.8212752776209,2.284157856762076,0,2023-02-20,no +WA,B,Yes,9,14396,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,387.88931212789464,5.3855052925274975,1.0829834479726788,0,2023-02-20,NA +NM,C,yes,10,27604,5,1000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,2.3423627380433287,34.16583095894536,0.7768183518841436,0,2023-02-20,no +NM,C,yes,11,91578,6,10000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,1.446524457373096,398.1329993947551,1.0797008632439722,0,2023-02-20,NA +CA,A,YES,1,04971,1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,316.15662584248014,58.139235913183434,17.60328449458461,0,2023-02-21,NA +CA,A,YES,2,31809,1,30000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,945.6278147571411,19.587123060665945,5.827776761828387,0,2023-02-21,NA +CA,A,YES,4,12078,3,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,11.751380650249304,27.73154003333902,14.766252620080733,0,2023-02-21,NA +CA,A,YES,4,12078,4,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,35.70170755070612,332.7832436305292,2.7352553654792673,0,2023-02-21,no +CA,A,YES,5,34687,4,1000000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,0.91981564366402635,5.3899500110707645,1.8396312873280527,1,2023-02-21,NA +WA,B,Yes,6,31928,1,2000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,8.430851554331136,25.0406208007693,16.861703108662272,1,2023-02-21,NA +WA,B,Yes,7,07528,1,100000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,23.612069119147947,244.7397096523221,3.0087210472919703,0,2023-02-21,NA +WA,B,Yes,9,14396,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,29.026762828851798,30.11574618355358,2.0207345280466376,0,2023-02-21,no +NM,C,yes,10,27604,5,1000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,1.1829975069700929,263.21076865469865,0.30550454536117444,0,2023-02-21,NA +NM,C,yes,11,91578,6,10000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,1.7206183992625057,48.80636563084332,0.06449951725767869,0,2023-02-21,yes +CA,A,YES,1,04971,1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,97.43244440599466,166.25369812836306,3.073760381121263,0,2023-02-22,NA +CA,A,YES,2,31809,1,30000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,112.09852587468147,615.8718263421359,1.5460914490061697,0,2023-02-22,no +CA,A,YES,4,12078,3,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,34.544023681908016,164.20380110691042,22.600847159992487,0,2023-02-22,NA +CA,A,YES,4,12078,4,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,294.16792829310066,26.815602289518647,1.027936232116507,0,2023-02-22,NA +WA,B,Yes,6,31928,1,2000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,32.92332481271343,74.40943392735927,13.256448761245595,0,2023-02-22,no +WA,B,Yes,8,97385,2,50000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,115.08714931046507,19.5401219888647,5.900498537748314,0,2023-02-22,NA +NM,C,yes,10,27604,5,1000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,0.8832508037455562,27.159189533307664,1.267416502947841,0,2023-02-22,NA +NM,C,yes,11,91578,6,10000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,1.404593762310653,195.7373966043554,0.10079922306648723,0,2023-02-22,NA +XX,D,ye,14,09634,10,1000000000,wwtp,raw wastewater,copies/l wastewater,FLUAV,5.943205742281485,35.85674215492902,21.16260089753815,0,2023-02-22,NA +YY,E,No,14,09634,3,13,upstream,primary sludge,copies/g dry sludge,sars-cov-2,55.36847248791359,5.325337361832822,8.1040803604102,0,2023-02-22,NA +ZZ,F,NO,15,98436,4,8472,wwtp,raw wastewater,copies/g dry sludge,sars-cov-2,27.680975173456222,199.17938688580583,9.77673775845998,0,2023-02-22,NA +CA,A,YES,1,04971,1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,387.1038296292479,81.13681422221168,4.537393024553714,0,2023-02-23,NA +CA,A,YES,2,31809,1,30000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,14.259275701356149,39.090634850792306,1.6836949930209062,0,2023-02-23,NA +CA,A,YES,4,12078,3,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,90.14606118521341,97.3106471895911,2.0628197367307095,0,2023-02-23,NA +CA,A,YES,4,12078,4,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,13.988374881165887,42.448744393005995,16.472659883629504,0,2023-02-23,NA +WA,B,Yes,6,31928,1,2000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,29.04071020749954,679.8678525007043,13.900827737002743,0,2023-02-23,NA +WA,B,Yes,7,07528,1,100000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,5.169248792658407,37.52840148876902,7.626665024082168,0,2023-02-23,no +WA,B,Yes,8,97385,2,50000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,108.3418255271921,141.33735983396926,1.129357433824155,0,2023-02-23,NA +WA,B,Yes,9,14396,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,929.6224729077021,716.5820395919653,4.394683935597443,0,2023-02-23,no +NM,C,yes,11,91578,6,10000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,2.346817414418601,26.705028663072103,0.2301666473607025,0,2023-02-23,NA +ZZ,F,NO,12,32541,1,8472,wwtp,raw wastewater,copies/g dry sludge,sars-cov-2,34.42383486625209,112.59838281050702,4.55895449667454,0,2023-02-23,NA +CA,A,YES,1,04971,1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,10.859558090606194,467.0747242248433,23.913992786438158,0,2023-02-24,no +CA,A,YES,2,31809,1,30000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,23.771280144369904,988.8859103796827,24.474069180740113,0,2023-02-24,NA +CA,A,YES,4,12078,3,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,904.3690188816917,7.374260983279588,1.265157619811638,0,2023-02-24,no +CA,A,YES,4,12078,4,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,37.902686233067605,345.8782990879349,6.995083143513485,0,2023-02-24,NA +WA,B,Yes,6,31928,1,2000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,11.810749386814294,27.057706299663575,23.621498773628588,1,2023-02-24,NA +WA,B,Yes,8,97385,2,50000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,0.6486511843266478,27.874678397939704,1.2973023686532956,1,2023-02-24,NA +NM,C,yes,10,27604,5,1000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,2.3163271579421916,93.50154558575514,0.49602667596985195,0,2023-02-24,NA +NM,C,yes,11,91578,6,10000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,0.7799087919533654,911.5753066006426,0.5430949287835113,0,2023-02-24,temperature not assessed upon arrival at the laboratory +XX,D,ye,15,98436,10,1000000000,wwtp,raw wastewater,copies/l wastewater,FLUAV,130.62485930434323,6.542478137460546,23.934118638259893,0,2023-02-24,NA +CA,A,YES,1,04971,1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,128.54700185544306,205.41718468186758,3.8516288229128457,0,2023-02-25,NA +CA,A,YES,2,31809,1,30000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,119.5790878096687,8.169905190906652,9.119662308925404,0,2023-02-25,NA +CA,A,YES,4,12078,3,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,23.463825427129432,473.5851376032156,22.943329577823516,0,2023-02-25,NA +CA,A,YES,4,12078,4,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,42.98447119111173,67.56123793722364,5.407381181504064,0,2023-02-25,NA +WA,B,Yes,6,31928,1,2000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,43.011730368426605,28.301965846090543,5.600195710245945,0,2023-02-25,yes +WA,B,Yes,7,07528,1,100000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,11.007370762852444,373.50606885712034,12.47726264866389,0,2023-02-25,NA +WA,B,Yes,8,97385,2,50000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,117.8071881312598,765.0788528846027,12.846710556950349,0,2023-02-25,NA +WA,B,Yes,9,14396,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,112.15233616285764,107.30328168469553,1.403094955856213,0,2023-02-25,NA +NM,C,yes,10,27604,5,1000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,2.1532164987571236,175.2759059956653,0.7504881921872932,0,2023-02-25,NA +NM,C,yes,11,91578,6,10000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,0.8767313114455,5.019459525594294,0.12962278024237,0,2023-02-25,NA +CA,A,YES,1,04971,1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,22.112115170593434,79.0814556891465,7.82263396630389,0,2023-02-26,NA +CA,A,YES,2,31809,1,30000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,44.950472982339264,565.81078500947865,5.877299554035761,0,2023-02-26,NA +CA,A,YES,4,12078,3,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,26.222211538218698,40.40800042509997,1.514468978375766,0,2023-02-26,NA +CA,A,YES,4,12078,4,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,8.231360797851224,26.616218514023316,9.07636708084877,0,2023-02-26,no +WA,B,Yes,6,31928,1,2000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,15.23741314609261,37.88541115713193,17.823704546433635,0,2023-02-26,NA +WA,B,Yes,7,07528,1,100000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,33.96574276293469,591.2422003061563,21.7076332361784,0,2023-02-26,y +WA,B,Yes,8,97385,2,50000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,57.51497824796495,567.4880517387049,6.273113493197985,0,2023-02-26,NA +WA,B,Yes,9,14396,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,560.8382047719646,386.1777868759051,3.95251328396741,0,2023-02-26,NA +NM,C,yes,10,27604,5,1000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,2.8431479805823763,232.74950535627903,0.5667863789945409,0,2023-02-26,no +NM,C,yes,11,91578,6,10000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,1.1775674655735837,64.8460492809882,1.3541257709644852,0,2023-02-26,no +XX,D,ye,12,32541,10,1000000000,wwtp,raw wastewater,copies/l wastewater,FLUAV,619.2718309237938,12.321697259506582,2.287974642412762,0,2023-02-26,no +ZZ,F,NO,15,98436,4,8472,wwtp,raw wastewater,copies/g dry sludge,sars-cov-2,24.40671232256105,7.015319494607072,2.28897468183092,0,2023-02-26,no +CA,A,YES,1,04971,1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,30.137088470279455,494.0285027347975,3.0394003409320747,0,2023-02-27,NA +CA,A,YES,2,31809,1,30000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,346.88480645002824,29.153345555677355,6.8506876767237195,0,2023-02-27,NA +CA,A,YES,4,12078,3,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,21.260422831276216,145.11101166531353,4.372196502681745,0,2023-02-27,no +CA,A,YES,4,12078,4,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,258.0585440319528,7.549154753798491,10.819758561537075,0,2023-02-27,yes +WA,B,Yes,6,31928,1,2000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,34.137595827146306,37.07416218857916,6.16944287318849,0,2023-02-27,NA +WA,B,Yes,7,07528,1,100000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,12.944566632148454,114.44373622871511,24.257767746995423,0,2023-02-27,y +WA,B,Yes,8,97385,2,50000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,130.75385704627033,915.8212844171684,10.99284507461069,0,2023-02-27,no +WA,B,Yes,9,14396,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,116.24607081688374,57.90612418148004,12.744355773971424,0,2023-02-27,no +NM,C,yes,10,27604,5,1000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,0.005595987013185169,44.979667352843165,0.30662598267716634,1,2023-02-27,NA +NM,C,yes,11,91578,6,10000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,1.8084750711354933,793.6732721357164,1.0953155515166886,0,2023-02-27,NA +XX,D,ye,13,65174,10,1000000000,wwtp,raw wastewater,copies/l wastewater,FLUAV,10.044711123415624,35.234552561956214,6.215182247905489,0,2023-02-27,no +ZZ,F,NO,13,65174,2,8472,wwtp,raw wastewater,copies/g dry sludge,sars-cov-2,541.4693103569356,11.980790126112248,12.41203228598927,0,2023-02-27,NA +ZZ,F,NO,15,98436,4,8472,wwtp,raw wastewater,copies/g dry sludge,sars-cov-2,46.3017516255929,401.34065964227017,2.8771743421414504,0,2023-02-27,NA +CA,A,YES,1,04971,1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,543.1175903211706,615.6357607608519,4.409013104344052,0,2023-02-28,NA +CA,A,YES,2,31809,1,30000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,85.5619902645178,61.16045764553125,17.69091988168579,0,2023-02-28,NA +CA,A,YES,4,12078,3,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,0.6181858725929332,385.8952720073726,1.2363717451858665,1,2023-02-28,NA +CA,A,YES,5,34687,3,1000000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,130.10020066515787,81.37108472334596,1.6666854514132112,0,2023-02-28,no +CA,A,YES,4,12078,4,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,1.553954507639502,12.338739479627472,3.107909015279004,1,2023-02-28,NA +CA,A,YES,5,34687,4,1000000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,128.71749941402584,67.91444361127051,1.340638983554489,0,2023-02-28,NA +WA,B,Yes,6,31928,1,2000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,18.072438873574498,46.39689163176238,2.3856336000270364,0,2023-02-28,NA +WA,B,Yes,7,07528,1,100000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,776.8817927675503,605.8866364907634,2.4271991209794184,0,2023-02-28,NA +WA,B,Yes,8,97385,2,50000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,575.5730316193439,6.374169082600377,3.1532445476376303,0,2023-02-28,NA +WA,B,Yes,9,14396,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,0.6719868433150867,101.191083702492,1.3439736866301735,1,2023-02-28,NA +NM,C,yes,10,27604,5,1000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,1.8678757673687947,432.9844752199857,0.3307709049708068,0,2023-02-28,result is not quantifiable +NM,C,yes,11,91578,6,10000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,1.4936766562442374,225.21137830761992,0.5979644444040744,0,2023-02-28,> max temp and/or hold time +CA,A,YES,1,04971,1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,13.251298657542206,94.02223533504265,10.272813536625328,0,2023-03-01,NA +CA,A,YES,2,31809,1,30000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,27.641997164162607,5.885755335469401,11.103281157878275,0,2023-03-01,NA +CA,A,YES,4,12078,3,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,1.0964427217175736,10.64581672514099,2.1928854434351472,1,2023-03-01,NA +CA,A,YES,4,12078,4,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,27.06330655232275,164.0900723087505,5.620495308551656,0,2023-03-01,yes +WA,B,Yes,6,31928,1,2000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,19.73634411730031,989.2744700103444,11.20522870451425,0,2023-03-01,NA +WA,B,Yes,7,07528,1,100000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,7.976812872218228,47.314415431077194,2.9222343503083867,0,2023-03-01,NA +WA,B,Yes,8,97385,2,50000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,4.15853139341677,19.17618359511787,8.31706278683354,1,2023-03-01,NA +WA,B,Yes,9,14396,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,134.37588668681397,9.215290659066747,6.185416457115013,0,2023-03-01,NA +NM,C,yes,10,27604,5,1000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,2.243364369985031,64.30433149336938,0.151329038323999,0,2023-03-01,NA +NM,C,yes,11,91578,6,10000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,1.5200024747816658,14.806253538768813,0.7770802799905268,0,2023-03-01,NA +XX,D,ye,13,65174,10,1000000000,wwtp,raw wastewater,copies/l wastewater,FLUAV,2.17809668770798,12.901608802198007,4.35619337541596,1,2023-03-01,result is not quantifiable +ZZ,F,NO,13,65174,2,8472,wwtp,raw wastewater,copies/g dry sludge,sars-cov-2,20.042419609818673,23.134311576134916,3.0539194219217776,0,2023-03-01,y +CA,A,YES,1,04971,1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,6.080179057593754,7.901916299409575,5.366037941447647,0,2023-03-02,NA +CA,A,YES,2,31809,1,30000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,387.28243228948486,23.746695618457178,1.9040197991862475,0,2023-03-02,NA +CA,A,YES,4,12078,3,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,234.55384639831667,288.66270708710994,11.66091913371317,0,2023-03-02,> max temp and/or hold time +CA,A,YES,4,12078,4,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,256.00581559351707,34.98412759741163,2.160436218313202,0,2023-03-02,NA +WA,B,Yes,6,31928,1,2000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,540.1977643993337,20.170602201570407,1.938434520317956,0,2023-03-02,NA +WA,B,Yes,7,07528,1,100000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,6.150421962369688,715.9632004648928,12.300843924739375,1,2023-03-02,no +WA,B,Yes,8,97385,2,50000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,820.6963591939166,531.6833019877936,21.147996077303855,0,2023-03-02,NA +WA,B,Yes,9,14396,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,6.378505672957012,69.91021982186116,2.360122286989393,0,2023-03-02,NA +NM,C,yes,10,27604,5,1000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,1.0391115249281708,764.0875208455874,0.15867112951071605,0,2023-03-02,NA +ZZ,F,NO,12,32541,1,8472,wwtp,raw wastewater,copies/g dry sludge,sars-cov-2,511.7636628721869,281.0090468039806,14.462586308479416,0,2023-03-02,NA +CA,A,YES,1,04971,1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,31.088129916068763,5.6395919830043235,1.1282655157766144,0,2023-03-03,NA +CA,A,YES,2,31809,1,30000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,487.73104802072436,84.29561876624399,1.2397533322835683,0,2023-03-03,NA +CA,A,YES,4,12078,3,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,639.8905046167573,590.6370181521473,9.95762126006784,0,2023-03-03,NA +CA,A,YES,4,12078,4,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,20.634736890993377,66.56427348801235,2.272923082780746,0,2023-03-03,no +WA,B,Yes,7,07528,1,100000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,2.356085652054681,9.704598289962506,4.712171304109362,1,2023-03-03,NA +WA,B,Yes,8,97385,2,50000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,66.4141538875427,336.8803247983554,12.606261613708172,0,2023-03-03,no +NM,C,yes,10,27604,5,1000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,1.727915604906232,65.57369812338612,0.8969882975440178,0,2023-03-03,no +NM,C,yes,11,91578,6,10000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,2.7560900384656293,29.438978948273437,0.3098567983120942,0,2023-03-03,NA +XX,D,ye,15,98436,10,1000000000,wwtp,raw wastewater,copies/l wastewater,FLUAV,7.2534338613074265,97.15900174066766,2.193239806280088,0,2023-03-03,NA +YY,E,No,14,09634,3,13,upstream,primary sludge,copies/g dry sludge,sars-cov-2,53.0808598383249,185.50934354950854,2.237706241420627,0,2023-03-03,no +CA,A,YES,1,04971,1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,158.55850221202064,369.6975606751693,7.462109862894364,0,2023-03-04,NA +CA,A,YES,2,31809,1,30000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,20.2891763830902,963.2346018442207,2.238006509841936,0,2023-03-04,NA +CA,A,YES,4,12078,3,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,0.7980968204314328,182.5527435589447,1.5961936408628656,1,2023-03-04,NA +CA,A,YES,4,12078,4,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,664.7637958633509,583.4604648778959,15.780375594624859,0,2023-03-04,NA +WA,B,Yes,6,31928,1,2000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,37.98634280959712,8.829276085955533,2.0103281770498067,0,2023-03-04,NA +WA,B,Yes,7,07528,1,100000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,126.68664054876504,65.9004524077722,8.033146055245622,0,2023-03-04,NA +WA,B,Yes,8,97385,2,50000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,609.2873798183433,682.5951395055189,2.951329684226727,0,2023-03-04,result is not quantifiable +WA,B,Yes,9,14396,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,55.48096986853254,797.688853660138,21.132838031995462,0,2023-03-04,NA +NM,C,yes,10,27604,5,1000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,2.8440815785106186,268.1219705432606,0.41076340189033306,0,2023-03-04,NA +NM,C,yes,11,91578,6,10000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,1.622150077901525,43.11335886334251,0.22990452689370564,0,2023-03-04,NA +XX,D,ye,12,32541,10,1000000000,wwtp,raw wastewater,copies/l wastewater,FLUAV,5.721831327423401,141.12883722434464,1.2490657216384413,0,2023-03-04,NA +XX,D,ye,15,98436,10,1000000000,wwtp,raw wastewater,copies/l wastewater,FLUAV,121.65732906983988,44.32796012384547,3.6101735869168006,0,2023-03-04,NA +YY,E,No,15,98436,4,13,upstream,primary sludge,copies/g dry sludge,sars-cov-2,32.08771834830813,965.8801370063932,8.01383503824439,0,2023-03-04,NA +CA,A,YES,1,04971,1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,10.754474756704594,255.4304579860096,1.8140997306783964,0,2023-03-05,no +CA,A,YES,2,31809,1,30000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,10.196977342754817,219.5321422055375,20.393954685509634,1,2023-03-05,NA +CA,A,YES,4,12078,4,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,105.94947155961388,412.28575941770464,1.514415625279379,0,2023-03-05,NA +WA,B,Yes,6,31928,1,2000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,17.749761285487708,89.54948751524978,16.23452164317336,0,2023-03-05,NA +WA,B,Yes,7,07528,1,100000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,262.4013127219605,6.123293057995181,5.87538347456332,0,2023-03-05,NA +WA,B,Yes,8,97385,2,50000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,61.64656692897097,7.4667795113923905,3.468847532097106,0,2023-03-05,NA +WA,B,Yes,9,14396,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,260.07719685845444,937.3799728940805,6.950134989220923,0,2023-03-05,NA +NM,C,yes,10,27604,5,1000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,1.6858887823113662,5.939766859958962,0.9722122471057923,0,2023-03-05,no +XX,D,ye,13,65174,10,1000000000,wwtp,raw wastewater,copies/l wastewater,FLUAV,20.622349106252877,784.4847634001611,5.329832916642551,0,2023-03-05,NA +CA,A,YES,1,04971,1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,708.0987990107047,47.13318907166913,12.138428684072297,0,2023-03-06,NA +CA,A,YES,2,31809,1,30000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,166.27778468585075,6.704180736391196,2.9569887678886433,0,2023-03-06,NA +CA,A,YES,4,12078,3,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,171.262370903392,5.178892129960095,7.6234706034318735,0,2023-03-06,no +CA,A,YES,4,12078,4,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,800.5393389045784,6.511081282134894,3.572164238031614,0,2023-03-06,NA +WA,B,Yes,6,31928,1,2000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,52.91477174049062,708.8394604682445,12.223884690244677,0,2023-03-06,no +WA,B,Yes,7,07528,1,100000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,7.675313764011645,574.9331217191386,1.7960476616478152,0,2023-03-06,NA +WA,B,Yes,8,97385,2,50000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,11.24110834882159,38.80803895585481,11.134765333053068,0,2023-03-06,NA +WA,B,Yes,9,14396,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,386.3718485856084,59.52802925537838,3.4891688933973004,0,2023-03-06,no +NM,C,yes,10,27604,5,1000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,1.4983300327604037,37.28221301535899,0.0504959357037284,0,2023-03-06,no +NM,C,yes,11,91578,6,10000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,1.1105879856637835,96.42507484342136,0.8459054045110621,0,2023-03-06,NA +XX,D,ye,13,65174,10,1000000000,wwtp,raw wastewater,copies/l wastewater,FLUAV,9.95864097213848,147.3339873871709,17.522062159515226,0,2023-03-06,NA +CA,A,YES,1,04971,1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,199.9498140556388,11.84964334047132,14.718438267591178,0,2023-03-07,NA +CA,A,YES,2,31809,1,30000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,12.776397509492606,170.65313298110112,1.9406092463754114,0,2023-03-07,NA +CA,A,YES,4,12078,3,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,159.59510292136193,330.6761285137837,7.858918233856867,0,2023-03-07,no +CA,A,YES,5,34687,3,1000000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,48.38145539142506,269.2561623905531,1.7334650879066913,0,2023-03-07,NA +CA,A,YES,4,12078,4,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,89.64676414548772,19.23015744546207,11.888225770976586,0,2023-03-07,NA +CA,A,YES,5,34687,4,1000000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,10.801250955309504,211.80186275662714,5.668064994180599,0,2023-03-07,NA +WA,B,Yes,6,31928,1,2000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,3.331635676828732,12.42504001654992,6.663271353657464,1,2023-03-07,NA +WA,B,Yes,7,07528,1,100000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,60.02925612720606,953.9203569928756,1.9755224804730762,0,2023-03-07,NA +WA,B,Yes,8,97385,2,50000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,3.0961189331904264,145.80595693679248,6.192237866380853,1,2023-03-07,NA +WA,B,Yes,9,14396,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,5.212216432357146,21.264947301016928,1.545096233089596,0,2023-03-07,NA +NM,C,yes,10,27604,5,1000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,2.8838760753504773,15.46153313874573,0.11000438798829677,0,2023-03-07,NA +NM,C,yes,11,91578,6,10000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,2.518836016259436,730.4543968915425,0.27567691362752067,0,2023-03-07,NA +XX,D,ye,13,65174,10,1000000000,wwtp,raw wastewater,copies/l wastewater,FLUAV,13.844289844174835,7.96558167953937,9.465273653529986,0,2023-03-07,NA +ZZ,F,NO,14,09634,3,8472,wwtp,raw wastewater,copies/g dry sludge,sars-cov-2,49.45148321180406,8.920770676161167,4.046800888705728,0,2023-03-07,result is not quantifiable +CA,A,YES,1,04971,1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,802.3550464348742,93.8516436328436,1.7587376183625947,0,2023-03-08,NA +CA,A,YES,2,31809,1,30000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,2.159952547896308,28.466246255036687,4.319905095792616,1,2023-03-08,below loq +CA,A,YES,4,12078,4,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,232.73483961295574,358.7088840119746,1.9046654156512355,0,2023-03-08,no +WA,B,Yes,6,31928,1,2000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,171.53107787253276,61.2855673241315,1.0405184197279778,0,2023-03-08,NA +WA,B,Yes,7,07528,1,100000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,114.46735014828246,47.876108578585466,10.34259542701581,0,2023-03-08,no +WA,B,Yes,8,97385,2,50000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,63.86667489906438,545.917430737053,9.918897103472322,0,2023-03-08,no +WA,B,Yes,9,14396,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,61.244872584285595,309.0336564656152,11.671419070329925,0,2023-03-08,NA +NM,C,yes,10,27604,5,1000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,0.7565391353219517,5.153199587148225,0.9851042098721342,0,2023-03-08,NA +NM,C,yes,11,91578,6,10000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,1.0630752927414222,122.83769706181525,0.48640608893637927,0,2023-03-08,NA +ZZ,F,NO,12,32541,1,8472,wwtp,raw wastewater,copies/g dry sludge,sars-cov-2,105.0828108267289,345.9553078858336,8.595852981958881,0,2023-03-08,NA +CA,A,YES,1,04971,1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,565.5905896213546,10.326294021590266,7.371266777590111,0,2023-03-09,NA +CA,A,YES,4,12078,3,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,469.9823099610116,464.34409632261026,1.1712470116181852,0,2023-03-09,NA +CA,A,YES,4,12078,4,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,323.9137925394194,42.34021338734792,1.0838633980705725,0,2023-03-09,below loq +WA,B,Yes,6,31928,1,2000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,259.6496779886425,6.911801266572399,7.932910321294406,0,2023-03-09,NA +WA,B,Yes,8,97385,2,50000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,578.8966079683211,223.3565849925504,21.498945188837876,0,2023-03-09,NA +WA,B,Yes,9,14396,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,5.204118089750765,199.08774347279748,7.382827235613202,0,2023-03-09,no +NM,C,yes,10,27604,5,1000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,1.844154976779564,209.59504458005313,1.1378408523577748,0,2023-03-09,NA +NM,C,yes,11,91578,6,10000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,1.6383985512558712,17.596567858020265,0.4448796975590207,0,2023-03-09,NA +YY,E,No,15,98436,4,13,upstream,primary sludge,copies/g dry sludge,sars-cov-2,40.52569506764774,597.0461070668506,24.611684055472185,0,2023-03-09,NA +CA,A,YES,1,04971,1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,115.39074259807938,235.9585444770106,20.74287530120597,0,2023-03-10,NA +CA,A,YES,2,31809,1,30000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,89.02990216869499,16.925788538932313,22.610331041005825,0,2023-03-10,NA +CA,A,YES,4,12078,3,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,67.00459584147991,525.4351042892039,15.221872955596583,0,2023-03-10,NA +CA,A,YES,4,12078,4,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,22.756377746450454,276.67160206650755,2.9082271311034376,0,2023-03-10,NA +WA,B,Yes,6,31928,1,2000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,380.27475471881877,313.4671980769839,1.6153613178561694,0,2023-03-10,below loq +WA,B,Yes,7,07528,1,100000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,285.57390464234277,44.154441562154425,13.934770292319422,0,2023-03-10,NA +WA,B,Yes,8,97385,2,50000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,57.43177440612398,11.259515363518478,2.911966989171706,0,2023-03-10,> max temp and/or hold time +WA,B,Yes,9,14396,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,7.239302452608382,62.931420295175826,19.836324921305557,0,2023-03-10,no +NM,C,yes,10,27604,5,1000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,2.6030379164137893,11.05276606414447,0.489674123257978,0,2023-03-10,no +NM,C,yes,11,91578,6,10000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,2.4813635945997454,20.38099797885449,0.596586134062397,0,2023-03-10,no +XX,D,ye,12,32541,10,1000000000,wwtp,raw wastewater,copies/l wastewater,FLUAV,245.50156711035,522.9653391177062,3.641492565163893,0,2023-03-10,NA +ZZ,F,NO,12,32541,1,8472,wwtp,raw wastewater,copies/g dry sludge,sars-cov-2,559.8455976946192,879.4456019812421,1.4372698151744994,0,2023-03-10,no +CA,A,YES,1,04971,1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,458.2185722722606,81.68467683478113,1.567411435737819,0,2023-03-11,NA +CA,A,YES,2,31809,1,30000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,55.235690285138965,215.34485562578317,7.685079911428711,0,2023-03-11,NA +CA,A,YES,4,12078,3,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,5.406826800928464,15.168377128354015,10.813653601856927,1,2023-03-11,NA +CA,A,YES,4,12078,4,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,21.9756570087032,62.653535552088655,3.56946166559274,0,2023-03-11,result is not quantifiable +WA,B,Yes,6,31928,1,2000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,2.390514397851157,458.38269024565403,4.781028795702314,1,2023-03-11,NA +WA,B,Yes,8,97385,2,50000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,98.70913753253285,50.10212833124913,2.9115445215398714,0,2023-03-11,NA +WA,B,Yes,9,14396,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,23.87792051249501,18.242748377402133,3.551038992756499,0,2023-03-11,NA +NM,C,yes,10,27604,5,1000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,1.070377955537653,31.620155355594186,0.41858269516857316,0,2023-03-11,NA +NM,C,yes,11,91578,6,10000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,2.2923017477715026,658.8482334048955,0.9501279086318479,0,2023-03-11,NA +XX,D,ye,12,32541,10,1000000000,wwtp,raw wastewater,copies/l wastewater,FLUAV,203.69078928696254,224.565546299968,11.479071124886952,0,2023-03-11,no +YY,E,No,12,32541,1,13,upstream,primary sludge,copies/g dry sludge,sars-cov-2,658.7435725779864,174.6947199493657,1.8202812850905217,0,2023-03-11,no +CA,A,YES,1,04971,1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,166.94136827508945,5.539390170211123,7.808403195655328,0,2023-03-12,NA +CA,A,YES,2,31809,1,30000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,38.097816458513016,359.9083814029601,12.184099743365918,0,2023-03-12,NA +CA,A,YES,4,12078,3,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,65.61317595388444,7.65300102269132,1.7127518989253088,0,2023-03-12,NA +CA,A,YES,4,12078,4,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,837.5191638172039,590.3695105119665,3.9875164898391353,0,2023-03-12,NA +WA,B,Yes,6,31928,1,2000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,71.83034919697306,221.07819619424095,10.919114626007389,0,2023-03-12,result is not quantifiable +WA,B,Yes,7,07528,1,100000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,956.3555665398779,13.543869367290185,5.33486615691988,0,2023-03-12,NA +WA,B,Yes,8,97385,2,50000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,54.290699317116996,266.3683873273148,2.7221797773072467,0,2023-03-12,NA +WA,B,Yes,9,14396,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,11.185927461857894,24.369799059007654,22.371854923715787,1,2023-03-12,NA +NM,C,yes,10,27604,5,1000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,2.3784404982854386,170.8842741297347,0.849998262917388,0,2023-03-12,NA +NM,C,yes,11,91578,6,10000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,1.016659056729215,75.5043719699276,0.8281478466671701,0,2023-03-12,no +YY,E,No,15,98436,4,13,upstream,primary sludge,copies/g dry sludge,sars-cov-2,778.0238464588273,355.4013686945431,2.5603510524959283,0,2023-03-12,NA +ZZ,F,NO,13,65174,2,8472,wwtp,raw wastewater,copies/g dry sludge,sars-cov-2,125.58711763027108,45.51689176015359,2.914302253429205,0,2023-03-12,no +CA,A,YES,1,04971,1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,792.0931373110479,8.762107070715258,8.618496350940111,0,2023-03-13,NA +CA,A,YES,2,31809,1,30000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,159.0281480318035,68.20496172500553,10.428022760773471,0,2023-03-13,NA +CA,A,YES,4,12078,3,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,3.2983799017662783,306.0640508728707,6.5967598035325565,1,2023-03-13,no +CA,A,YES,4,12078,4,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,9.98339095161482,34.864716117518945,5.753836506429399,0,2023-03-13,no +WA,B,Yes,6,31928,1,2000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,743.4291658538455,5.196126844280465,5.321293787348545,0,2023-03-13,y +WA,B,Yes,7,07528,1,100000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,18.356768329311393,29.62237679124599,20.19612400874314,0,2023-03-13,yes +WA,B,Yes,8,97385,2,50000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,0.7922780187296575,269.37008786260884,1.584556037459315,1,2023-03-13,NA +WA,B,Yes,9,14396,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,876.9254257039869,272.58217800808154,7.795230180461602,0,2023-03-13,NA +NM,C,yes,10,27604,5,1000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,1.0770069803782623,769.29321512096806,0.6047048016464983,0,2023-03-13,NA +NM,C,yes,11,91578,6,10000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,1.4566462896534202,459.52707764671754,0.4822649262086211,0,2023-03-13,NA +XX,D,ye,15,98436,10,1000000000,wwtp,raw wastewater,copies/l wastewater,FLUAV,9.868216226742039,25.49901923062265,1.9760983842279234,0,2023-03-13,no +YY,E,No,12,32541,1,13,upstream,primary sludge,copies/g dry sludge,sars-cov-2,721.4422282670149,80.47655006843375,5.688116624209073,0,2023-03-13,NA +ZZ,F,NO,12,32541,1,8472,wwtp,raw wastewater,copies/g dry sludge,sars-cov-2,537.390257628136,6.989255606255752,7.648755225601417,0,2023-03-13,NA +ZZ,F,NO,15,98436,4,8472,wwtp,raw wastewater,copies/g dry sludge,sars-cov-2,8.69602602747301,53.66908111727376,10.681918430839763,0,2023-03-13,NA +CA,A,YES,1,04971,1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,4.374859641075281,57.47609650597885,8.749719282150561,1,2023-03-14,NA +CA,A,YES,2,31809,1,30000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,64.32185520236284,27.525742632583487,5.949257859119654,0,2023-03-14,NA +CA,A,YES,4,12078,3,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,94.18597636039269,17.31264694329305,1.650928910441701,0,2023-03-14,NA +CA,A,YES,5,34687,3,1000000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,5.154605568793737,45.54115125728395,1.3511942769922272,0,2023-03-14,NA +CA,A,YES,4,12078,4,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,8.065478695114393,34.53514417696152,12.342663245457915,0,2023-03-14,no +WA,B,Yes,6,31928,1,2000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,5.7764748265636,71.57390310027671,3.0665124249881885,0,2023-03-14,NA +WA,B,Yes,7,07528,1,100000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,11.48573709483161,21.99476838964128,2.194460242109824,0,2023-03-14,NA +WA,B,Yes,9,14396,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,355.14102579646215,104.58969984147521,24.70710080460524,0,2023-03-14,no +NM,C,yes,10,27604,5,1000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,1.8328398850112908,79.02239127853267,0.4863590129888156,0,2023-03-14,NA +NM,C,yes,11,91578,6,10000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,-0.29984502857005907,37.83263018880683,0.0011849670939221375,1,2023-03-14,NA +CA,A,YES,1,04971,1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,82.7827970998116,15.184501160544444,2.1389600988131803,0,2023-03-15,NA +CA,A,YES,2,31809,1,30000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,145.01237086618718,9.243972100308385,18.120945623976642,0,2023-03-15,NA +CA,A,YES,4,12078,3,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,201.37225586866074,133.49089612731532,20.40373805687369,0,2023-03-15,NA +CA,A,YES,4,12078,4,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,416.6376945073775,456.1133923164395,1.5711181571692232,0,2023-03-15,NA +WA,B,Yes,6,31928,1,2000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,284.3287479765909,28.52592986642656,22.186837444801466,0,2023-03-15,NA +WA,B,Yes,7,07528,1,100000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,8.135231557270036,54.923213551712514,1.111408555761005,0,2023-03-15,no +WA,B,Yes,8,97385,2,50000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,42.94670960686573,16.828683429572283,2.3721970864090842,0,2023-03-15,yes +WA,B,Yes,9,14396,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,13.767800328631848,11.962807425491464,6.175614834322937,0,2023-03-15,NA +NM,C,yes,10,27604,5,1000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,1.7065046481386805,11.672690095065569,0.6560135449180345,0,2023-03-15,NA +NM,C,yes,11,91578,6,10000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,-0.284092152142929,408.2523211827641,0.016937843521052185,1,2023-03-15,NA +YY,E,No,15,98436,4,13,upstream,primary sludge,copies/g dry sludge,sars-cov-2,5.071630630794381,114.6523272070523,4.058206601459443,0,2023-03-15,no +CA,A,YES,1,04971,1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,8.672654771899738,5.454841947027963,14.751477956518597,0,2023-03-16,NA +CA,A,YES,2,31809,1,30000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,151.190078222533,14.350251564232256,6.047182393249267,0,2023-03-16,NA +CA,A,YES,4,12078,3,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,746.8793810154516,8.94825593533036,8.238967662018773,0,2023-03-16,NA +CA,A,YES,4,12078,4,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,242.4215232054661,52.08075329406804,9.563141213236626,0,2023-03-16,NA +WA,B,Yes,6,31928,1,2000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,0.6745538529692778,37.600587307668086,1.3491077059385557,1,2023-03-16,NA +WA,B,Yes,8,97385,2,50000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,115.47646177048624,10.779420909122653,6.900927870555157,0,2023-03-16,result is not quantifiable +WA,B,Yes,9,14396,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,7.0492224913476695,441.4552022003286,10.44343344344846,0,2023-03-16,no +NM,C,yes,10,27604,5,1000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,1.7459086933780763,203.88899926378238,0.7586215325572967,0,2023-03-16,no +NM,C,yes,11,91578,6,10000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,1.012696823150673,36.42141033040396,1.3137268188146543,1,2023-03-16,NA +YY,E,No,13,65174,2,13,upstream,primary sludge,copies/g dry sludge,sars-cov-2,7.812046887665094,16.04185007257759,1.315709226715126,0,2023-03-16,NA +ZZ,F,NO,15,98436,4,8472,wwtp,raw wastewater,copies/g dry sludge,sars-cov-2,12.915631119573327,29.68503028376494,1.1527275517532478,0,2023-03-16,result is not quantifiable +CA,A,YES,1,04971,1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,29.246057749763843,41.93822503967432,1.7268634206436253,0,2023-03-17,NA +CA,A,YES,2,31809,1,30000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,418.5911633098013,11.349017968173655,18.623302053560778,0,2023-03-17,no +CA,A,YES,4,12078,3,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,32.47815102300223,45.495067959943505,6.67530593539439,0,2023-03-17,NA +CA,A,YES,4,12078,4,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,68.09484703366523,8.885580298373203,20.226112273482762,0,2023-03-17,NA +WA,B,Yes,6,31928,1,2000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,181.1970812008741,22.629917891822238,7.535959001159097,0,2023-03-17,no +WA,B,Yes,7,07528,1,100000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,12.045113240083257,76.51631358314398,4.309565587049435,0,2023-03-17,NA +WA,B,Yes,8,97385,2,50000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,37.43333461476963,43.71153066900172,6.467857006972642,0,2023-03-17,NA +NM,C,yes,11,91578,6,10000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,1.2656865624062994,31.79927921332804,0.5141618181695529,0,2023-03-17,NA +CA,A,YES,1,04971,1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,87.32050764429154,10.643799430398044,1.0545988000977582,0,2023-03-18,no +CA,A,YES,2,31809,1,30000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,236.27561467458713,11.346468637492807,1.0939118802364765,0,2023-03-18,NA +CA,A,YES,4,12078,3,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,24.57804612951748,13.962209618461099,7.386635370336105,0,2023-03-18,yes +CA,A,YES,4,12078,4,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,54.66123404859118,870.8917137446232,2.178376076261856,0,2023-03-18,NA +WA,B,Yes,6,31928,1,2000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,711.048704802888,15.912840127350169,12.26792564818291,0,2023-03-18,temperature not assessed upon arrival at the laboratory +WA,B,Yes,7,07528,1,100000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,52.44966939413459,41.58990286737903,7.521859898636171,0,2023-03-18,NA +WA,B,Yes,8,97385,2,50000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,28.477645977122325,320.93061534958946,13.27809985047202,0,2023-03-18,NA +WA,B,Yes,9,14396,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,15.03983031601969,166.12743752326423,14.102217641346385,0,2023-03-18,y +NM,C,yes,10,27604,5,1000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,1.1405868739354204,10.20683468723425,1.397275334725503,0,2023-03-18,NA +NM,C,yes,11,91578,6,10000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,1.5558413155724884,222.65473477418013,0.40373049898109475,0,2023-03-18,NA +XX,D,ye,13,65174,10,1000000000,wwtp,raw wastewater,copies/l wastewater,FLUAV,154.35689955294305,6.003268796897057,1.675676546858301,0,2023-03-18,NA +ZZ,F,NO,15,98436,4,8472,wwtp,raw wastewater,copies/g dry sludge,sars-cov-2,422.2602624817239,6.537008649073889,2.3841167229677063,0,2023-03-18,yes +CA,A,YES,1,04971,1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,0.5448361754382418,37.791788520690055,1.0896723508764836,1,2023-03-19,no +CA,A,YES,2,31809,1,30000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,57.40730587372734,6.523915972616801,1.2118490483690498,0,2023-03-19,NA +CA,A,YES,4,12078,3,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,112.28214599344166,5.252415834766401,20.9737217035241,0,2023-03-19,NA +WA,B,Yes,6,31928,1,2000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,602.437884751041,65.33820058960393,1.5164469049741234,0,2023-03-19,NA +WA,B,Yes,7,07528,1,100000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,1.0493017919728487,11.991998709886959,2.0986035839456973,1,2023-03-19,no +WA,B,Yes,8,97385,2,50000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,14.393362385182053,14.987570362790258,1.4018843284146667,0,2023-03-19,below loq +WA,B,Yes,9,14396,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,1.157377819215666,137.87988106317957,2.314755638431332,1,2023-03-19,NA +NM,C,yes,11,91578,6,10000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,0.747987181593174,14.92382571087265,0.9399303566055986,0,2023-03-19,NA +CA,A,YES,1,04971,1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,33.394745995145854,39.31634484551162,17.627843578230518,0,2023-03-20,NA +CA,A,YES,2,31809,1,30000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,9.52196414544461,147.77911723662717,1.6118943318815933,0,2023-03-20,no +CA,A,YES,4,12078,3,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,37.83188701131832,6.665076693388046,10.451909952591972,0,2023-03-20,NA +CA,A,YES,4,12078,4,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,330.6576067631166,10.748361280291071,14.501830465184495,0,2023-03-20,NA +WA,B,Yes,6,31928,1,2000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,228.32952542074057,7.82611746629333,6.080454033221211,0,2023-03-20,NA +WA,B,Yes,7,07528,1,100000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,302.9757143430111,73.62313410846662,14.76843661131516,0,2023-03-20,no +WA,B,Yes,8,97385,2,50000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,45.25748961757045,13.832287404477702,6.254063796138068,0,2023-03-20,NA +WA,B,Yes,9,14396,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,6.129455681347205,11.698155553402666,7.879565971805341,0,2023-03-20,NA +NM,C,yes,10,27604,5,1000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,1.1165284969060818,8.618499503896963,0.6303833952591443,0,2023-03-20,NA +NM,C,yes,11,91578,6,10000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,1.9560351435159358,7.438255034098769,0.075732766168991,0,2023-03-20,NA +YY,E,No,12,32541,1,13,upstream,primary sludge,copies/g dry sludge,sars-cov-2,465.303120157964,35.97048660814587,1.1708959359098903,0,2023-03-20,NA +CA,A,YES,1,04971,1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,72.91747979089877,19.917617521251366,13.47828358468021,0,2023-03-21,NA +CA,A,YES,2,31809,1,30000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,313.12689977402124,234.4836289389294,3.2489684484867616,0,2023-03-21,NA +CA,A,YES,4,12078,3,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,13.107769565670337,502.84632420253075,10.998370103008456,0,2023-03-21,NA +CA,A,YES,5,34687,3,1000000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,360.50471273431145,170.68100593650615,1.064561694471887,0,2023-03-21,NA +CA,A,YES,4,12078,4,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,172.02279554286577,16.389552616378825,1.220782363881426,0,2023-03-21,NA +WA,B,Yes,6,31928,1,2000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,19.65457220860151,5.869112201798803,2.7275385951921374,0,2023-03-21,NA +WA,B,Yes,7,07528,1,100000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,10.427298780866836,984.1298485363877,2.7310004809923707,0,2023-03-21,NA +WA,B,Yes,8,97385,2,50000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,128.2976956587911,192.31079137697452,22.64456253054075,0,2023-03-21,no +WA,B,Yes,9,14396,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,102.22377931229167,14.672695025758012,7.56476906147389,0,2023-03-21,NA +NM,C,yes,10,27604,5,1000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,0.9218065773274359,333.03823621581836,0.35576039809380383,0,2023-03-21,NA +CA,A,YES,1,04971,1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,996.1543331368099,9.02367826861503,1.2487248394080674,0,2023-03-22,NA +CA,A,YES,2,31809,1,30000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,28.17317301009663,68.19109898222287,1.4592920731543813,0,2023-03-22,no +CA,A,YES,4,12078,3,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,187.2979168351214,33.57970793645007,1.7222110475356054,0,2023-03-22,NA +CA,A,YES,4,12078,4,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,954.7525028816011,5.701453495837322,4.79201569728624,0,2023-03-22,no +WA,B,Yes,6,31928,1,2000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,164.65310289124676,730.3550505701782,8.864299185891197,0,2023-03-22,y +WA,B,Yes,7,07528,1,100000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,7.205023912719112,217.8595172006536,1.609053483034028,0,2023-03-22,NA +WA,B,Yes,8,97385,2,50000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,112.2541781344915,274.39449478828794,1.8739647149606258,0,2023-03-22,NA +WA,B,Yes,9,14396,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,12.840671326577535,141.80292033508474,18.895579088497218,0,2023-03-22,no +NM,C,yes,10,27604,5,1000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,1.6681673910296049,30.567115427868693,0.5623121544422687,0,2023-03-22,NA +NM,C,yes,11,91578,6,10000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,2.0086319652993643,51.234172119776375,0.17314343138760419,0,2023-03-22,NA +CA,A,YES,1,04971,1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,13.017321446475458,271.37718198121854,18.40013057788529,0,2023-03-23,temperature not assessed upon arrival at the laboratory +CA,A,YES,2,31809,1,30000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,12.28532546514711,39.6411003234242,2.334156796233545,0,2023-03-23,temperature not assessed upon arrival at the laboratory +CA,A,YES,4,12078,3,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,6.372810317545694,259.42962167841773,1.636533240568632,0,2023-03-23,no +CA,A,YES,4,12078,4,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,356.2624415052372,63.86701951262568,24.487303216378642,0,2023-03-23,result is not quantifiable +WA,B,Yes,6,31928,1,2000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,57.12229448416939,47.33230911495545,2.0752983845585478,0,2023-03-23,NA +WA,B,Yes,7,07528,1,100000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,952.02496221291,6.169410663772091,6.069574341988132,0,2023-03-23,NA +WA,B,Yes,8,97385,2,50000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,30.383111693070234,41.13844340094726,3.100305592096694,0,2023-03-23,NA +WA,B,Yes,9,14396,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,917.1907283739225,451.15560607275324,1.2791564289211648,0,2023-03-23,> max temp and/or hold time +NM,C,yes,10,27604,5,1000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,2.4320723687444628,813.6970080351902,0.5423803891722885,0,2023-03-23,no +NM,C,yes,11,91578,6,10000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,1.2186646237188727,357.3229187839913,0.7503414037883832,0,2023-03-23,NA +ZZ,F,NO,13,65174,2,8472,wwtp,raw wastewater,copies/g dry sludge,sars-cov-2,17.77012486270121,9.48241112751736,1.8752507121891457,0,2023-03-23,NA +ZZ,F,NO,15,98436,4,8472,wwtp,raw wastewater,copies/g dry sludge,sars-cov-2,16.69859560320987,969.319661596872,7.326203786400412,0,2023-03-23,below loq +CA,A,YES,1,04971,1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,57.19019343505626,10.285016207169908,22.79070573429898,0,2023-03-24,NA +CA,A,YES,4,12078,3,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,1.7376589193059462,967.3495267419621,3.4753178386118924,1,2023-03-24,no +CA,A,YES,4,12078,4,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,111.27166881702163,28.229952677729866,3.6216335717949595,0,2023-03-24,NA +WA,B,Yes,6,31928,1,2000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,18.995312252126684,22.258125279766134,3.039191399772452,0,2023-03-24,NA +WA,B,Yes,9,14396,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,84.76152159547529,134.01229597177553,1.0275912470560846,0,2023-03-24,NA +NM,C,yes,10,27604,5,1000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,0.9080500355697724,696.2142607043737,0.25105430589969824,0,2023-03-24,NA +NM,C,yes,11,91578,6,10000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,1.6927620546399462,420.194216131754,0.3426093350197461,0,2023-03-24,yes +CA,A,YES,1,04971,1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,218.27243814724895,312.00042051144555,20.229522333360077,0,2023-03-25,NA +CA,A,YES,2,31809,1,30000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,107.345665149764,515.9267700080054,1.470878840010913,0,2023-03-25,NA +CA,A,YES,4,12078,3,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,306.0684483908621,690.7429884465517,1.2472233358245872,0,2023-03-25,no +CA,A,YES,4,12078,4,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,0.5670101256865503,57.76235080693807,1.1340202513731006,1,2023-03-25,no +WA,B,Yes,6,31928,1,2000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,7.234204292279638,109.45436700055618,3.435598191380963,0,2023-03-25,no +WA,B,Yes,8,97385,2,50000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,38.52235263081736,777.3075990048837,14.728996531543899,0,2023-03-25,NA +WA,B,Yes,9,14396,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,474.04142777356236,12.102775609092566,4.962162065039853,0,2023-03-25,NA +NM,C,yes,10,27604,5,1000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,1.475351287403206,655.6370665812346,1.1120620138031492,0,2023-03-25,no +NM,C,yes,11,91578,6,10000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,2.463990881932199,649.0397208296008,1.3686791025529064,0,2023-03-25,NA +CA,A,YES,1,04971,1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,8.807374893730517,9.54018147503281,6.126507865610079,0,2023-03-26,NA +CA,A,YES,2,31809,1,30000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,23.992158576801295,6.373300968944614,1.521159136379448,0,2023-03-26,no +CA,A,YES,4,12078,3,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,47.52635632317084,181.881147912835,4.51645456715964,0,2023-03-26,NA +CA,A,YES,4,12078,4,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,66.12395141119353,129.66724813174386,8.132719979488547,0,2023-03-26,NA +WA,B,Yes,6,31928,1,2000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,5.656280519207332,98.92801300163985,12.594099870545627,0,2023-03-26,> max temp and/or hold time +WA,B,Yes,7,07528,1,100000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,23.956173284947624,75.01528595730731,14.819946123688243,0,2023-03-26,NA +WA,B,Yes,8,97385,2,50000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,43.06360214366511,81.6194646552056,2.1108575682177664,0,2023-03-26,NA +WA,B,Yes,9,14396,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,35.39760093547524,5.161523767680968,4.47058547308224,0,2023-03-26,y +NM,C,yes,10,27604,5,1000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,1.2548976256452007,203.42350103503762,0.7330238609212207,0,2023-03-26,NA +NM,C,yes,11,91578,6,10000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,2.0524242201251885,658.8376334456489,0.3035913003326015,0,2023-03-26,y +YY,E,No,12,32541,1,13,upstream,primary sludge,copies/g dry sludge,sars-cov-2,6.186967637837796,5.653437263417586,3.176220230720233,0,2023-03-26,NA +ZZ,F,NO,13,65174,2,8472,wwtp,raw wastewater,copies/g dry sludge,sars-cov-2,11.88220309531992,736.2853370081403,15.95434833305698,0,2023-03-26,NA +CA,A,YES,1,04971,1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,73.4268981868111,131.858283598738,3.4136857352345937,0,2023-03-27,NA +CA,A,YES,2,31809,1,30000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,163.13505553003236,593.1086259599754,15.773299121731014,0,2023-03-27,NA +CA,A,YES,4,12078,3,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,85.65401382193214,785.0530114973682,3.544452565324648,0,2023-03-27,temperature not assessed upon arrival at the laboratory +CA,A,YES,4,12078,4,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,187.9685878653195,7.1038412317222015,8.671424162848957,0,2023-03-27,NA +WA,B,Yes,6,31928,1,2000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,64.71314419233782,8.460165039118673,1.140055827278722,0,2023-03-27,NA +WA,B,Yes,7,07528,1,100000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,435.17056673604725,450.89257248527866,4.092584947130413,0,2023-03-27,NA +WA,B,Yes,9,14396,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,229.9112667285273,654.0125717777411,22.20932235768555,0,2023-03-27,no +NM,C,yes,10,27604,5,1000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,2.880356891852241,36.47758589399812,0.02842637124098375,0,2023-03-27,NA +NM,C,yes,11,91578,6,10000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,1.15437661131711,519.5254454148705,0.8683312283166003,0,2023-03-27,y +XX,D,ye,14,09634,10,1000000000,wwtp,raw wastewater,copies/l wastewater,FLUAV,3.8146868432017,30.897846842187118,7.6293736864034,1,2023-03-27,NA +YY,E,No,14,09634,3,13,upstream,primary sludge,copies/g dry sludge,sars-cov-2,5.188815762563365,15.646629999598368,2.0905499272838144,0,2023-03-27,NA +ZZ,F,NO,15,98436,4,8472,wwtp,raw wastewater,copies/g dry sludge,sars-cov-2,250.63839007114578,157.9402743172809,4.876193156274159,0,2023-03-27,y +CA,A,YES,1,04971,1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,6.682849019380353,282.9619057651956,1.0279225570177555,0,2023-03-28,NA +CA,A,YES,2,31809,1,30000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,213.46549262740135,14.512192256662717,2.5478249258982113,0,2023-03-28,NA +CA,A,YES,4,12078,3,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,91.85644941948355,7.196072391440902,17.642393700732026,0,2023-03-28,NA +CA,A,YES,5,34687,3,1000000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,7.469344247949058,652.0884766190582,1.6197019495186475,0,2023-03-28,NA +CA,A,YES,4,12078,4,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,226.99942738128578,50.93264171588683,2.6064533430635564,0,2023-03-28,NA +WA,B,Yes,8,97385,2,50000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,665.8514811378952,755.0423012322362,7.938225085174666,0,2023-03-28,NA +WA,B,Yes,9,14396,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,48.59211674562865,5.2251562326755145,17.601907520097537,0,2023-03-28,NA +NM,C,yes,10,27604,5,1000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,0.8566400384244849,211.58277546977004,1.157670034088466,1,2023-03-28,NA +NM,C,yes,11,91578,6,10000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,2.6479352590870193,57.70831070637592,0.6618860957312795,0,2023-03-28,NA +XX,D,ye,14,09634,10,1000000000,wwtp,raw wastewater,copies/l wastewater,FLUAV,149.27265666664286,46.34460042299492,5.582793719547046,0,2023-03-28,NA +ZZ,F,NO,12,32541,1,8472,wwtp,raw wastewater,copies/g dry sludge,sars-cov-2,977.1807953659874,12.53762415630151,7.683993277382071,0,2023-03-28,NA +CA,A,YES,1,04971,1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,75.62091664975829,21.40665496542848,13.264487438945563,0,2023-03-29,NA +CA,A,YES,2,31809,1,30000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,11.640484757373864,11.998540843913457,23.280969514747728,1,2023-03-29,NA +CA,A,YES,4,12078,4,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,113.73720836239401,689.1429047358099,6.2691009398844955,0,2023-03-29,NA +WA,B,Yes,6,31928,1,2000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,9.442671313031157,24.07732912344634,1.7767435381810266,0,2023-03-29,NA +WA,B,Yes,7,07528,1,100000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,31.374599281976742,10.236396073217696,5.520480739668748,0,2023-03-29,NA +WA,B,Yes,9,14396,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,576.5807544895031,60.46527738529,1.0903716745653873,0,2023-03-29,no +NM,C,yes,10,27604,5,1000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,1.5577447940461984,5.209059640059037,0.48220564594029025,0,2023-03-29,NA +NM,C,yes,11,91578,6,10000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,0.940607790472693,21.38394296561431,1.0015868878211114,0,2023-03-29,no +CA,A,YES,1,04971,1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,1.6324382532577906,819.4494291923656,3.264876506515581,1,2023-03-30,NA +CA,A,YES,2,31809,1,30000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,295.8762293645139,210.7452994240196,4.393247593749634,0,2023-03-30,NA +CA,A,YES,4,12078,3,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,115.83107833427295,261.62955050387285,2.773349989678891,0,2023-03-30,no +CA,A,YES,4,12078,4,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,311.30927035041526,359.20498471250187,1.0982649002560176,0,2023-03-30,NA +WA,B,Yes,6,31928,1,2000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,4.198291721915954,181.6276930973596,8.396583443831908,1,2023-03-30,NA +WA,B,Yes,7,07528,1,100000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,162.61942438771246,13.967393226659778,6.679199009904495,0,2023-03-30,NA +WA,B,Yes,8,97385,2,50000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,993.08251211322,451.9138054530801,2.6574128319761585,0,2023-03-30,NA +WA,B,Yes,9,14396,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,767.4509235267698,527.8894486960496,1.53821933637687,0,2023-03-30,NA +NM,C,yes,10,27604,5,1000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,0.9584380113906208,192.16632039797727,0.7851083590703626,0,2023-03-30,NA +NM,C,yes,11,91578,6,10000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,2.6730970202778264,58.49478013990041,0.29938504843655683,0,2023-03-30,NA +CA,A,YES,1,04971,1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,35.20532643073089,9.248725349979356,15.886088075205711,0,2023-03-31,NA +CA,A,YES,2,31809,1,30000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,23.272235002450206,251.209699858953,5.787394830010697,0,2023-03-31,NA +CA,A,YES,4,12078,3,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,40.3450457130837,43.140416280741945,2.401611349890003,0,2023-03-31,no +CA,A,YES,4,12078,4,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,594.2310981869983,240.48372233726306,1.427556274291066,0,2023-03-31,NA +WA,B,Yes,6,31928,1,2000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,3.9271169468506946,11.785732509393805,7.854233893701389,1,2023-03-31,NA +WA,B,Yes,7,07528,1,100000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,139.87865385486384,35.33839148486886,2.3945305890941455,0,2023-03-31,result is not quantifiable +WA,B,Yes,8,97385,2,50000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,8.709861124050093,273.64973133042974,6.907347630820922,0,2023-03-31,no +WA,B,Yes,9,14396,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,458.1604827140203,30.040551865401653,17.72887875047263,0,2023-03-31,NA +NM,C,yes,10,27604,5,1000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,1.2284040385101573,46.802110878164655,0.3969893846645802,0,2023-03-31,NA +NM,C,yes,11,91578,6,10000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,1.830424965912257,22.302548730182078,0.472855116934369,0,2023-03-31,NA +CA,A,YES,1,04971,1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,390.3741886882092,194.96633941221833,11.890457486370696,0,2023-04-01,NA +CA,A,YES,2,31809,1,30000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,677.777855128173,615.4807390980077,3.173075903139985,0,2023-04-01,NA +CA,A,YES,4,12078,3,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,5.535614015188141,303.4536495625179,2.240777235926934,0,2023-04-01,> max temp and/or hold time +CA,A,YES,4,12078,4,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,102.6499394427105,11.483475758272078,2.729616665672996,0,2023-04-01,temperature not assessed upon arrival at the laboratory +WA,B,Yes,6,31928,1,2000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,22.035526932123773,532.2928583348399,12.362716854022691,0,2023-04-01,NA +WA,B,Yes,7,07528,1,100000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,17.24232435262091,618.754418043131,10.266321621420415,0,2023-04-01,NA +WA,B,Yes,8,97385,2,50000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,375.1614950900082,118.9832474766205,1.2451104611579882,0,2023-04-01,NA +WA,B,Yes,9,14396,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,10.207469407197692,45.2633982257767,3.454524009409514,0,2023-04-01,no +NM,C,yes,11,91578,6,10000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,1.0283951710812824,21.703675808591377,0.11000523977801965,0,2023-04-01,NA +ZZ,F,NO,13,65174,2,8472,wwtp,raw wastewater,copies/g dry sludge,sars-cov-2,113.55153439549117,829.9229756073087,1.5817763749255775,0,2023-04-01,NA +CA,A,YES,1,04971,1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,9.059006455958267,10.381219237878499,18.118012911916534,1,2023-04-02,NA +CA,A,YES,2,31809,1,30000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,902.0031704675641,131.6142552918608,15.989720423775033,0,2023-04-02,no +CA,A,YES,4,12078,3,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,104.58872120685022,134.5517625017887,13.24140470741661,0,2023-04-02,NA +CA,A,YES,4,12078,4,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,9.06251770567413,7.481460176674117,1.016188578692917,0,2023-04-02,NA +WA,B,Yes,7,07528,1,100000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,61.19700925428539,390.96087912923343,1.126274580376372,0,2023-04-02,NA +WA,B,Yes,8,97385,2,50000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,133.09184702959703,809.3285296795337,1.4247325104833497,0,2023-04-02,NA +WA,B,Yes,9,14396,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,11.876502051636832,5.092561435270632,23.753004103273664,1,2023-04-02,NA +NM,C,yes,10,27604,5,1000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,1.8159284753134648,65.62728696383928,0.41625290153073263,0,2023-04-02,NA +NM,C,yes,11,91578,6,10000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,0.8759789176607697,14.134626581447831,0.37928209738656654,0,2023-04-02,no +YY,E,No,14,09634,3,13,upstream,primary sludge,copies/g dry sludge,sars-cov-2,892.5818822141072,597.1179775685647,1.7028082205099477,0,2023-04-02,no +ZZ,F,NO,14,09634,3,8472,wwtp,raw wastewater,copies/g dry sludge,sars-cov-2,6.415075406285594,691.4913909499812,2.6254389786583334,0,2023-04-02,result is not quantifiable +CA,A,YES,1,04971,1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,590.6204299820803,39.18326309363071,7.69840360708392,0,2023-04-03,NA +CA,A,YES,2,31809,1,30000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,2.631327104091619,6.2404841531535435,5.262654208183238,1,2023-04-03,NA +CA,A,YES,4,12078,3,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,61.40731305235723,47.69985938770851,3.3531032253677284,0,2023-04-03,no +CA,A,YES,4,12078,4,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,30.957677001713034,18.523321801049295,1.1622494476454377,0,2023-04-03,NA +WA,B,Yes,6,31928,1,2000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,52.854928898274046,34.83801358738989,2.0095851671085034,0,2023-04-03,NA +WA,B,Yes,7,07528,1,100000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,122.99160529962944,383.6699933199615,1.771693536707636,0,2023-04-03,NA +WA,B,Yes,8,97385,2,50000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,7.628436263214652,72.5113716240953,22.265583004160288,0,2023-04-03,NA +WA,B,Yes,9,14396,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,47.51985938214286,10.19962809411392,1.2029726257463753,0,2023-04-03,no +NM,C,yes,10,27604,5,1000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,0.8248913799990276,994.8913263067676,0.07850423572469814,0,2023-04-03,NA +ZZ,F,NO,15,98436,4,8472,wwtp,raw wastewater,copies/g dry sludge,sars-cov-2,0.6874122777675692,38.76866667444012,1.3748245555351384,1,2023-04-03,NA +CA,A,YES,1,04971,1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,282.5664507150931,99.44932473136679,5.496039422038657,0,2023-04-04,NA +CA,A,YES,2,31809,1,30000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,8.044405520601384,79.34261124258812,1.7966125018670736,0,2023-04-04,NA +CA,A,YES,4,12078,3,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,666.839185169513,83.17414267468308,3.7722753502525497,0,2023-04-04,NA +CA,A,YES,4,12078,4,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,33.55461428327824,48.54673289792136,24.655547465902714,0,2023-04-04,no +CA,A,YES,5,34687,4,1000000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,16.933691061375384,124.24622806242756,6.400005025784543,0,2023-04-04,NA +WA,B,Yes,6,31928,1,2000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,395.1139812730705,185.28633402259942,1.001562047386236,0,2023-04-04,no +WA,B,Yes,7,07528,1,100000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,273.97360842181405,54.92525291302878,23.104653273077734,0,2023-04-04,no +WA,B,Yes,8,97385,2,50000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,24.57186104436064,406.23102538643815,15.911118887002894,0,2023-04-04,NA +WA,B,Yes,9,14396,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,30.288662854491605,586.0733488729976,6.66564823095393,0,2023-04-04,no +NM,C,yes,10,27604,5,1000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,0.7813343484832055,28.262508357405633,0.41276006632862217,0,2023-04-04,NA +NM,C,yes,11,91578,6,10000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,2.5841551733243957,6.383875271867941,0.6604158868151668,0,2023-04-04,no +XX,D,ye,13,65174,10,1000000000,wwtp,raw wastewater,copies/l wastewater,FLUAV,12.24097679759742,259.835774879926,6.288169534860324,0,2023-04-04,no +CA,A,YES,1,04971,1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,56.97307164298722,224.66257416062479,16.17596585536359,0,2023-04-05,NA +CA,A,YES,2,31809,1,30000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,656.5186313137227,262.2870636845951,9.413836214999804,0,2023-04-05,NA +CA,A,YES,4,12078,3,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,281.1968791055341,26.382145664834148,14.269835324979727,0,2023-04-05,NA +WA,B,Yes,6,31928,1,2000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,162.3029335579853,190.31816555935592,13.890885193606142,0,2023-04-05,no +WA,B,Yes,7,07528,1,100000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,92.60940351013274,9.573634427819217,20.300804021995056,0,2023-04-05,NA +WA,B,Yes,8,97385,2,50000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,236.9413365799493,10.810678660381582,1.1963309078654965,0,2023-04-05,NA +WA,B,Yes,9,14396,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,13.859821831723332,5.011486146719104,2.915460265795362,0,2023-04-05,NA +NM,C,yes,10,27604,5,1000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,0.877964887214645,137.86288228712337,1.3009280926844788,0,2023-04-05,NA +NM,C,yes,11,91578,6,10000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,1.206329039074767,125.86265159892605,0.13583909249172982,0,2023-04-05,NA +XX,D,ye,14,09634,10,1000000000,wwtp,raw wastewater,copies/l wastewater,FLUAV,141.0083181632142,42.050694411795675,1.2056073206531754,0,2023-04-05,NA +YY,E,No,12,32541,1,13,upstream,primary sludge,copies/g dry sludge,sars-cov-2,8.144157949838752,169.114630237328,16.288315899677503,1,2023-04-05,result is not quantifiable +CA,A,YES,1,04971,1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,784.899042119334,8.014518989701141,1.7527365477504029,0,2023-04-06,y +CA,A,YES,2,31809,1,30000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,686.3739121168077,520.7600338812931,1.035478213127512,0,2023-04-06,NA +CA,A,YES,4,12078,3,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,43.98289272819491,246.17292256584577,7.1031679448104725,0,2023-04-06,NA +CA,A,YES,4,12078,4,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,10.290987568805631,154.3023899425953,18.203789941116487,0,2023-04-06,temperature not assessed upon arrival at the laboratory +WA,B,Yes,6,31928,1,2000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,14.811490243479987,153.5776136831542,3.567332910465447,0,2023-04-06,NA +WA,B,Yes,7,07528,1,100000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,310.3037333418046,415.0895832394591,5.655518525854642,0,2023-04-06,NA +WA,B,Yes,8,97385,2,50000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,54.16668213393648,31.230577136063825,1.7814287155048465,0,2023-04-06,no +WA,B,Yes,9,14396,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,150.47218012962094,134.7708971958526,2.7450708219113484,0,2023-04-06,NA +NM,C,yes,10,27604,5,1000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,-0.08468168105296528,167.881279757593,0.2163483146110159,1,2023-04-06,NA +NM,C,yes,11,91578,6,10000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,2.6601309025395805,52.69835054701574,0.30088095555892774,0,2023-04-06,NA +XX,D,ye,15,98436,10,1000000000,wwtp,raw wastewater,copies/l wastewater,FLUAV,189.53955135429854,55.03975651892324,2.089502131319954,0,2023-04-06,result is not quantifiable +YY,E,No,15,98436,4,13,upstream,primary sludge,copies/g dry sludge,sars-cov-2,28.44680575494976,110.11677746001274,18.54001408001815,0,2023-04-06,NA +ZZ,F,NO,15,98436,4,8472,wwtp,raw wastewater,copies/g dry sludge,sars-cov-2,15.421219635348143,44.5465336286134,2.23838377880263,0,2023-04-06,NA +CA,A,YES,1,04971,1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,157.90319031704828,163.90882472260637,7.625228932162711,0,2023-04-07,NA +CA,A,YES,2,31809,1,30000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,97.22649124671644,44.19433824984998,13.792100458386194,0,2023-04-07,NA +CA,A,YES,4,12078,3,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,742.5301161811412,96.08187004252646,9.563721710577955,0,2023-04-07,NA +CA,A,YES,4,12078,4,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,2.0734235714452525,237.5448258708884,4.146847142890505,1,2023-04-07,NA +WA,B,Yes,7,07528,1,100000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,26.591199012501473,245.26811495172652,8.044375025053975,0,2023-04-07,NA +NM,C,yes,10,27604,5,1000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,1.1877605601436416,5.494219691665465,0.71801639586818,0,2023-04-07,NA +NM,C,yes,11,91578,6,10000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,0.90893612691211,88.93943915043704,0.918834486417499,0,2023-04-07,NA +YY,E,No,15,98436,4,13,upstream,primary sludge,copies/g dry sludge,sars-cov-2,29.550402383983954,138.05347414608397,10.087206793189631,0,2023-04-07,NA +CA,A,YES,1,04971,1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,18.741526594607375,6.286134977955453,17.90925875056528,0,2023-04-08,NA +CA,A,YES,2,31809,1,30000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,10.95544846226623,623.814563326829,1.0293773503157166,0,2023-04-08,NA +CA,A,YES,4,12078,4,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,33.12918380831049,182.23242552557838,8.09494836959698,0,2023-04-08,NA +WA,B,Yes,6,31928,1,2000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,17.136366411063207,21.315640844407163,6.9185139639129,0,2023-04-08,NA +WA,B,Yes,7,07528,1,100000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,5.128511242236564,5.281547217787373,10.257022484473127,1,2023-04-08,no +WA,B,Yes,8,97385,2,50000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,32.19031427500802,7.309612769675325,9.441944671146873,0,2023-04-08,NA +WA,B,Yes,9,14396,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,9.940517816982343,61.251441078010764,20.46870828258581,0,2023-04-08,NA +NM,C,yes,10,27604,5,1000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,1.988085110539044,35.12573439298503,1.1900892215698746,0,2023-04-08,NA +NM,C,yes,11,91578,6,10000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,2.721213357522774,160.52755976629774,0.04755359979858107,0,2023-04-08,NA +CA,A,YES,1,04971,1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,8.166963197060474,177.94206519240126,6.618345313066703,0,2023-04-09,NA +CA,A,YES,2,31809,1,30000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,35.45993972342169,8.782671193805756,4.058230294305921,0,2023-04-09,no +CA,A,YES,4,12078,3,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,13.443515035138622,112.4095320568888,3.7663030252902274,0,2023-04-09,NA +CA,A,YES,4,12078,4,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,9.939894313577529,7.042971287123867,3.3933854298476804,0,2023-04-09,no +WA,B,Yes,6,31928,1,2000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,345.4451451656114,252.91117545767608,3.7285623790989244,0,2023-04-09,no +WA,B,Yes,7,07528,1,100000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,10.251599505709105,176.32960024160525,5.2113048708093785,0,2023-04-09,NA +WA,B,Yes,8,97385,2,50000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,219.33327973276917,603.8714293824142,14.336688842087373,0,2023-04-09,NA +WA,B,Yes,9,14396,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,7.386628538775972,152.15326808391075,14.773257077551945,1,2023-04-09,below loq +NM,C,yes,10,27604,5,1000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,1.4400818720278807,877.7463129615602,0.4718051830855049,0,2023-04-09,NA +NM,C,yes,11,91578,6,10000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,2.99143111388315,93.34884540577853,1.2526475927219374,0,2023-04-09,NA +CA,A,YES,1,04971,1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,210.509691295845,624.5578615281554,17.273425323465588,0,2023-04-10,NA +CA,A,YES,2,31809,1,30000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,7.981315561174949,23.5008393519363,15.962631122349897,1,2023-04-10,NA +CA,A,YES,4,12078,3,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,280.89462552241105,63.812339046785254,3.3159000389611912,0,2023-04-10,NA +CA,A,YES,4,12078,4,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,5.966880202506393,12.019470260487012,11.933760405012785,1,2023-04-10,no +WA,B,Yes,6,31928,1,2000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,212.792744025519,8.641291054872614,19.541754477321188,0,2023-04-10,no +WA,B,Yes,7,07528,1,100000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,407.0690347711487,572.9397991612419,6.106600781824925,0,2023-04-10,NA +WA,B,Yes,8,97385,2,50000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,748.0140924187915,54.17746384125275,7.915502347281119,0,2023-04-10,NA +WA,B,Yes,9,14396,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,31.194882177750348,29.752836382850525,24.31877190845533,0,2023-04-10,no +NM,C,yes,10,27604,5,1000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,2.130917536411431,200.10127217216137,1.2554158041900918,0,2023-04-10,NA +NM,C,yes,11,91578,6,10000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,0.5442098394960652,5.235970725204678,0.8452398351600463,1,2023-04-10,no +YY,E,No,13,65174,2,13,upstream,primary sludge,copies/g dry sludge,sars-cov-2,568.5349439951062,9.032124404231801,6.168961041623295,0,2023-04-10,NA +CA,A,YES,1,04971,1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,260.17284607415934,327.2116063875679,1.4571587493996616,0,2023-04-11,> max temp and/or hold time +CA,A,YES,2,31809,1,30000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,358.55680488694117,8.957589110390598,1.7569015144119247,0,2023-04-11,NA +CA,A,YES,4,12078,3,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,406.3571721031981,36.864848287358505,16.294351953601957,0,2023-04-11,no +CA,A,YES,5,34687,3,1000000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,203.39061378638357,10.844214941466674,7.3445374451332475,0,2023-04-11,NA +CA,A,YES,4,12078,4,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,90.95001272827207,6.521336946477955,10.628731818735114,0,2023-04-11,NA +CA,A,YES,5,34687,4,1000000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,23.022500399652195,15.98885833376444,3.372872193228345,0,2023-04-11,NA +WA,B,Yes,6,31928,1,2000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,615.708771482301,10.135763825325489,4.685961340492325,0,2023-04-11,no +WA,B,Yes,9,14396,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,856.7944959452399,42.743068785412426,1.3621775676569894,0,2023-04-11,NA +NM,C,yes,10,27604,5,1000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,1.566919860727531,190.36538584518604,0.9379017625838088,0,2023-04-11,no +NM,C,yes,11,91578,6,10000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,0.7070201646950343,49.24820297105888,1.3411337598480602,0,2023-04-11,NA +XX,D,ye,12,32541,10,1000000000,wwtp,raw wastewater,copies/l wastewater,FLUAV,60.521081381691936,255.0753427869043,1.027922545462062,0,2023-04-11,NA +CA,A,YES,1,04971,1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,5.236194326347224,533.1325503441246,6.9827168044249746,0,2023-04-12,no +CA,A,YES,2,31809,1,30000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,173.7741754320949,92.49540081415049,7.51063601193581,0,2023-04-12,NA +CA,A,YES,4,12078,3,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,858.3921294290185,37.546161748685506,2.6179545510764006,0,2023-04-12,NA +CA,A,YES,4,12078,4,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,263.4972474779873,32.654446175887905,17.04319310830586,0,2023-04-12,NA +WA,B,Yes,6,31928,1,2000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,0.8456355914108287,559.3976421163684,1.6912711828216573,1,2023-04-12,NA +WA,B,Yes,7,07528,1,100000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,10.663283101159395,9.818850502214255,17.411733308140732,0,2023-04-12,NA +NM,C,yes,10,27604,5,1000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,1.7327672781433205,10.553875947373328,0.8390867291582444,0,2023-04-12,> max temp and/or hold time +NM,C,yes,11,91578,6,10000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,2.3668723550445976,49.788526386792086,1.23394135854962,0,2023-04-12,NA +YY,E,No,14,09634,3,13,upstream,primary sludge,copies/g dry sludge,sars-cov-2,20.59993907975133,60.826298845929806,10.409717401982743,0,2023-04-12,temperature not assessed upon arrival at the laboratory +CA,A,YES,2,31809,1,30000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,36.456793445491606,133.4512864441959,2.375645011570705,0,2023-04-13,y +CA,A,YES,4,12078,3,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,5.068797859436806,468.7450638551754,1.1052477073450997,0,2023-04-13,NA +CA,A,YES,4,12078,4,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,13.771251472672594,223.4474870717352,1.583510367075877,0,2023-04-13,NA +WA,B,Yes,6,31928,1,2000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,36.229840491977434,5.135799331018692,14.868361534317362,0,2023-04-13,NA +WA,B,Yes,7,07528,1,100000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,423.80803371533824,855.5827435678136,10.87464033537561,0,2023-04-13,NA +WA,B,Yes,8,97385,2,50000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,35.23487612225432,7.435625948970397,2.647883691412648,0,2023-04-13,NA +WA,B,Yes,9,14396,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,123.74672714461803,117.55612791022045,1.3853878162251763,0,2023-04-13,NA +NM,C,yes,10,27604,5,1000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,1.2082654469046061,30.764177147366887,0.7688326627692044,0,2023-04-13,no +NM,C,yes,11,91578,6,10000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,1.6605960167796854,26.436376681889204,1.3065795707026098,0,2023-04-13,NA +YY,E,No,14,09634,3,13,upstream,primary sludge,copies/g dry sludge,sars-cov-2,275.9744969282582,21.377571481712348,1.010752110202523,0,2023-04-13,no +CA,A,YES,1,04971,1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,7.927165967825946,52.418847144635976,1.9441287429439593,0,2023-04-14,NA +CA,A,YES,2,31809,1,30000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,345.0026009254806,610.9094157788714,21.65532245034835,0,2023-04-14,NA +CA,A,YES,4,12078,3,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,54.848557710253914,10.778600344944579,7.0418453780458234,0,2023-04-14,NA +CA,A,YES,4,12078,4,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,90.80459905549722,458.30345159200976,14.255520480805934,0,2023-04-14,NA +WA,B,Yes,6,31928,1,2000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,12.19918549690248,867.137211880195,24.39837099380496,1,2023-04-14,no +WA,B,Yes,7,07528,1,100000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,510.23596515904586,88.49005281787859,1.2078826773626496,0,2023-04-14,> max temp and/or hold time +WA,B,Yes,8,97385,2,50000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,0.6403686891705347,39.879257885907286,1.2807373783410694,1,2023-04-14,no +WA,B,Yes,9,14396,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,9.235689635328649,15.075804230677768,3.858557330572206,0,2023-04-14,no +NM,C,yes,10,27604,5,1000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,0.7353618396165423,75.71212528265467,1.073746787008171,0,2023-04-14,NA +NM,C,yes,11,91578,6,10000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,0.29820943051127224,5.970127846185665,0.5992394261752534,1,2023-04-14,NA +XX,D,ye,13,65174,10,1000000000,wwtp,raw wastewater,copies/l wastewater,FLUAV,18.50771349156326,6.865548277964143,1.4564122026922453,0,2023-04-14,temperature not assessed upon arrival at the laboratory +CA,A,YES,1,04971,1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,8.498960817154165,6.087806077690802,3.7936991995095233,0,2023-04-15,result is not quantifiable +CA,A,YES,4,12078,3,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,112.03237309292703,997.4318247637414,3.34361692518484,0,2023-04-15,NA +CA,A,YES,4,12078,4,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,287.5741900028177,423.6972701578573,1.4350865331338685,0,2023-04-15,no +WA,B,Yes,6,31928,1,2000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,156.14758749256313,50.93273759615766,6.332029487304983,0,2023-04-15,no +WA,B,Yes,7,07528,1,100000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,584.9495823822291,266.6015781169515,18.674273295970064,0,2023-04-15,NA +WA,B,Yes,8,97385,2,50000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,337.74159169207536,6.0430450612580815,15.368369161646344,0,2023-04-15,NA +WA,B,Yes,9,14396,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,5.748251202931816,496.80250214986035,16.88011734288282,0,2023-04-15,NA +NM,C,yes,11,91578,6,10000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,1.2462283024205552,87.58114737405828,0.33310987430756855,0,2023-04-15,NA +YY,E,No,14,09634,3,13,upstream,primary sludge,copies/g dry sludge,sars-cov-2,944.06667727823606,7.206019361456571,2.6178037103464837,0,2023-04-15,NA +CA,A,YES,1,04971,1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,161.50883552236422,89.71084760429939,10.168423259482495,0,2023-04-16,NA +CA,A,YES,2,31809,1,30000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,407.0664727349443,190.82759804800338,2.1732174927564163,0,2023-04-16,NA +CA,A,YES,4,12078,3,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,40.4696085533005,574.726610762378,2.136833852975637,0,2023-04-16,no +CA,A,YES,4,12078,4,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,77.76803004100108,11.598447367637489,2.155812915197988,0,2023-04-16,NA +WA,B,Yes,6,31928,1,2000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,5.983885870929228,8.643478324248626,5.049567206405092,0,2023-04-16,no +WA,B,Yes,7,07528,1,100000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,199.3719151313511,37.95426709020617,1.282954534721593,0,2023-04-16,NA +WA,B,Yes,8,97385,2,50000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,742.3159420994318,119.96543086817792,16.101745965449375,0,2023-04-16,NA +WA,B,Yes,9,14396,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,913.2399637271544,46.276083423789125,1.3941584923529649,0,2023-04-16,below loq +NM,C,yes,10,27604,5,1000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,0.5950777326007958,86.73939982911452,0.8961077282647769,1,2023-04-16,NA +NM,C,yes,11,91578,6,10000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,2.06359058493283,254.07422677371417,1.1323271208132573,0,2023-04-16,NA +CA,A,YES,1,04971,1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,24.00532686771313,22.516942505563232,21.438766889349207,0,2023-04-17,NA +CA,A,YES,2,31809,1,30000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,726.3991549581713,135.67180416410423,19.777441676963882,0,2023-04-17,yes +CA,A,YES,4,12078,3,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,73.71514994002796,250.18833821853778,4.655085368654688,0,2023-04-17,NA +CA,A,YES,4,12078,4,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,212.0499053192169,45.70888002158401,3.8323957435389917,0,2023-04-17,NA +WA,B,Yes,7,07528,1,100000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,36.91496996190142,10.181505770466218,1.1696587670041114,0,2023-04-17,NA +WA,B,Yes,8,97385,2,50000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,13.512082844840123,5.092843082069778,6.0443829012413826,0,2023-04-17,NA +WA,B,Yes,9,14396,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,15.872450803078664,39.76769498859967,4.792835186121277,0,2023-04-17,NA +NM,C,yes,10,27604,5,1000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,0.908731451548176,29.31256850765844,0.251487901369854,0,2023-04-17,NA +NM,C,yes,11,91578,6,10000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,2.2757745663987134,301.68947550489986,0.9170896350419486,0,2023-04-17,NA +XX,D,ye,15,98436,10,1000000000,wwtp,raw wastewater,copies/l wastewater,FLUAV,59.440063125912026,12.829456701048255,18.53837422465829,0,2023-04-17,NA +CA,A,YES,1,04971,1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,348.3430684735925,932.7512717430938,14.691412369816117,0,2023-04-18,NA +CA,A,YES,2,31809,1,30000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,43.96389147827063,38.480136789613034,14.138988990024531,0,2023-04-18,no +CA,A,YES,4,12078,3,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,13.88590534616682,342.31577683221275,2.7378015670030256,0,2023-04-18,NA +CA,A,YES,5,34687,3,1000000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,208.92445271460898,8.600500193064741,11.118884992419307,0,2023-04-18,no +CA,A,YES,4,12078,4,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,6.94019823559976,68.41697875014958,1.9055291744456355,0,2023-04-18,NA +WA,B,Yes,6,31928,1,2000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,13.4503632605554,254.5920922936517,2.6663471761631388,0,2023-04-18,> max temp and/or hold time +WA,B,Yes,7,07528,1,100000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,807.7392465244916,614.2415974460437,2.286769765716975,0,2023-04-18,temperature not assessed upon arrival at the laboratory +WA,B,Yes,9,14396,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,10.875544854966329,6.053766521066263,21.751089709932657,1,2023-04-18,no +NM,C,yes,10,27604,5,1000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,1.4382452846572475,273.3319108650942,0.6413824638384955,0,2023-04-18,> max temp and/or hold time +NM,C,yes,11,91578,6,10000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,1.2827799745155766,11.048393673034747,0.606054305742153,0,2023-04-18,no +CA,A,YES,2,31809,1,30000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,774.7874894453848,578.7270854760085,17.652621929971264,0,2023-04-19,NA +CA,A,YES,4,12078,3,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,39.87209606119167,455.3794883219907,3.6141314578966,0,2023-04-19,no +CA,A,YES,4,12078,4,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,0.8533681658379263,6.2881898288526985,1.7067363316758526,1,2023-04-19,NA +WA,B,Yes,6,31928,1,2000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,29.60183330479927,31.83194169451296,2.4089308437328163,0,2023-04-19,NA +WA,B,Yes,7,07528,1,100000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,385.1344412943056,14.478873211245618,8.011947625255246,0,2023-04-19,NA +WA,B,Yes,8,97385,2,50000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,73.91484148955173,212.71143837360657,3.9514052984561774,0,2023-04-19,NA +NM,C,yes,11,91578,6,10000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,2.3520362646560886,23.54649657151886,1.1598250939846282,0,2023-04-19,result is not quantifiable +CA,A,YES,1,04971,1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,568.4551169471282,921.4530009141888,3.7972934877373823,0,2023-04-20,NA +CA,A,YES,2,31809,1,30000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,806.547589143575,14.994390482282718,6.750795610258758,0,2023-04-20,NA +CA,A,YES,4,12078,3,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,40.63167718674632,17.45569937552425,2.15511720890374,0,2023-04-20,NA +CA,A,YES,4,12078,4,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,58.4431880225128,472.1210584034307,3.4646010229846405,0,2023-04-20,NA +WA,B,Yes,6,31928,1,2000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,10.757643754868367,11.520459777562253,7.317821054770153,0,2023-04-20,NA +WA,B,Yes,7,07528,1,100000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,5.9496975108143895,5.474188636301651,11.899395021628779,1,2023-04-20,NA +WA,B,Yes,8,97385,2,50000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,908.6021800642861,16.483099788200132,1.555071776820289,0,2023-04-20,NA +WA,B,Yes,9,14396,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,644.8842998330048,69.35506398978191,1.6938206997456873,0,2023-04-20,NA +NM,C,yes,10,27604,5,1000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,2.5334149625788744,27.261158148393246,0.37509153020391683,0,2023-04-20,yes +NM,C,yes,11,91578,6,10000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,2.941097591722281,679.062851968885,0.17291375899896033,0,2023-04-20,no +CA,A,YES,2,31809,1,30000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,101.15457702347922,298.9739075596262,5.264925508606661,0,2023-04-21,NA +CA,A,YES,4,12078,3,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,7.55927910341746,5.329766717444594,15.11855820683492,1,2023-04-21,no +CA,A,YES,4,12078,4,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,295.0191891574346,422.8627541338376,18.209965662118226,0,2023-04-21,no +WA,B,Yes,6,31928,1,2000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,59.559794658800776,29.904944570019108,1.0617176709785725,0,2023-04-21,NA +WA,B,Yes,7,07528,1,100000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,247.59165333276297,8.324596449026497,17.129344994037808,0,2023-04-21,NA +WA,B,Yes,8,97385,2,50000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,57.77226390074322,75.79684638401724,1.2486808612905065,0,2023-04-21,NA +WA,B,Yes,9,14396,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,27.91482353238682,26.100411002431038,6.08262970393365,0,2023-04-21,NA +NM,C,yes,10,27604,5,1000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,2.779110769501942,112.43778301886368,0.8877713432926129,0,2023-04-21,NA +NM,C,yes,11,91578,6,10000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,1.4378321970710723,35.383774517479615,0.5853837074404942,0,2023-04-21,> max temp and/or hold time +XX,D,ye,13,65174,10,1000000000,wwtp,raw wastewater,copies/l wastewater,FLUAV,10.59635486519744,64.80369788335476,7.676105734217961,0,2023-04-21,NA +ZZ,F,NO,14,09634,3,8472,wwtp,raw wastewater,copies/g dry sludge,sars-cov-2,71.21869289502744,64.33093604601123,6.72586369003842,0,2023-04-21,NA +CA,A,YES,1,04971,1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,466.6239673139718,219.33386254534756,3.664561808956291,0,2023-04-22,NA +CA,A,YES,4,12078,3,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,59.51861739707618,684.3346843782483,7.779887130863979,0,2023-04-22,no +CA,A,YES,4,12078,4,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,165.2922469404885,877.9425951443376,10.497201658873347,0,2023-04-22,NA +WA,B,Yes,6,31928,1,2000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,145.97438394249232,10.660303599195432,2.849215690422492,0,2023-04-22,no +WA,B,Yes,7,07528,1,100000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,11.915879076145051,9.453228808724623,23.831758152290103,1,2023-04-22,no +WA,B,Yes,8,97385,2,50000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,19.39331606016249,205.11370668937406,5.561320915090677,0,2023-04-22,NA +WA,B,Yes,9,14396,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,17.448034732148663,96.33420914521396,21.088515987791205,0,2023-04-22,NA +NM,C,yes,10,27604,5,1000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,2.5671182394754624,412.4273434061986,0.963801868393986,0,2023-04-22,NA +NM,C,yes,11,91578,6,10000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,0.7466774140958314,941.4454140130463,1.2450075180193803,0,2023-04-22,NA +YY,E,No,15,98436,4,13,upstream,primary sludge,copies/g dry sludge,sars-cov-2,106.55875600423985,289.3561016811968,3.0190568337992536,0,2023-04-22,NA +CA,A,YES,1,04971,1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,265.93446063582985,5.607476230855946,10.197305945712793,0,2023-04-23,NA +CA,A,YES,2,31809,1,30000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,7.933859824227034,57.924267231524674,16.349396700287905,0,2023-04-23,NA +CA,A,YES,4,12078,3,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,306.716276603354,7.106070449235682,8.063096325263846,0,2023-04-23,NA +CA,A,YES,4,12078,4,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,5.7286766628454036,6.779905346699579,10.968482081244277,0,2023-04-23,NA +WA,B,Yes,6,31928,1,2000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,146.70236042004032,6.8535270810878455,13.005670127403205,0,2023-04-23,no +WA,B,Yes,7,07528,1,100000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,272.2439857253243,8.048514132881587,10.401522145955388,0,2023-04-23,NA +WA,B,Yes,9,14396,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,158.51992667301744,656.3601447637814,2.194692330189646,0,2023-04-23,NA +NM,C,yes,10,27604,5,1000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,2.9694905749137868,230.36090532042041,0.8886398335621462,0,2023-04-23,no +NM,C,yes,11,91578,6,10000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,0.9853264727448121,52.3367275498159,1.1415405377124621,0,2023-04-23,no +ZZ,F,NO,12,32541,1,8472,wwtp,raw wastewater,copies/g dry sludge,sars-cov-2,126.70636487097767,27.091078868267534,1.8130008671178994,0,2023-04-23,> max temp and/or hold time +ZZ,F,NO,15,98436,4,8472,wwtp,raw wastewater,copies/g dry sludge,sars-cov-2,202.85589279337665,238.0645044353233,21.07363752699312,0,2023-04-23,NA +CA,A,YES,2,31809,1,30000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,546.2003090392154,14.490361702307968,1.3065851736404275,0,2023-04-24,y +CA,A,YES,4,12078,3,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,39.679812621724764,66.957830126471407,1.577139506665979,0,2023-04-24,NA +CA,A,YES,4,12078,4,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,5.001308020393117,30.5427521764067,1.276197561202989,0,2023-04-24,temperature not assessed upon arrival at the laboratory +WA,B,Yes,6,31928,1,2000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,414.00886661055074,114.32925175080292,15.064028828347864,0,2023-04-24,NA +WA,B,Yes,8,97385,2,50000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,22.084385136755607,757.9400271638747,15.970967418189986,0,2023-04-24,NA +WA,B,Yes,9,14396,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,419.6124654495961,26.78539625776327,7.402992630022404,0,2023-04-24,NA +NM,C,yes,10,27604,5,1000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,2.7237786649441036,5.371053942839231,0.9670568755731058,0,2023-04-24,NA +NM,C,yes,11,91578,6,10000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,0.982673105654751,658.8051892997945,1.1845849746079036,0,2023-04-24,NA +CA,A,YES,1,04971,1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,11.963077032320644,13.399709308986886,7.498009327668975,0,2023-04-25,NA +CA,A,YES,2,31809,1,30000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,12.769511481447745,18.569695911188614,7.889490440456316,0,2023-04-25,NA +CA,A,YES,4,12078,3,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,402.5991942368238,14.904477059006823,1.1965216993894612,0,2023-04-25,no +CA,A,YES,5,34687,3,1000000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,20.710669975083402,348.86604366503803,16.512048932158514,0,2023-04-25,NA +CA,A,YES,4,12078,4,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,9.125275349863035,223.81019103758862,2.0164338562299036,0,2023-04-25,no +CA,A,YES,5,34687,4,1000000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,258.21853046745724,11.857382727308176,1.3864765605864577,0,2023-04-25,y +WA,B,Yes,6,31928,1,2000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,61.524173281211475,662.3507885573871,9.460303540576735,0,2023-04-25,NA +WA,B,Yes,7,07528,1,100000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,497.18806946389896,637.5170932275527,1.8595980986798208,0,2023-04-25,NA +WA,B,Yes,8,97385,2,50000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,55.87414826717504,88.42594736326919,16.08146154265202,0,2023-04-25,NA +WA,B,Yes,9,14396,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,89.56842239127599,64.22847365518726,10.634424557095306,0,2023-04-25,NA +NM,C,yes,10,27604,5,1000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,1.3376795067847622,13.303674086686142,0.9418230659180908,0,2023-04-25,NA +NM,C,yes,11,91578,6,10000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,2.907029589954394,106.01484596999951,0.6401694449687754,0,2023-04-25,NA +YY,E,No,13,65174,2,13,upstream,primary sludge,copies/g dry sludge,sars-cov-2,21.875653019734887,33.11192470972023,1.07229367976199,0,2023-04-25,NA +CA,A,YES,1,04971,1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,66.21163639175714,114.48596232947426,16.12191225075471,0,2023-04-26,NA +CA,A,YES,2,31809,1,30000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,6.629022509031395,9.595463056578318,13.25804501806279,1,2023-04-26,NA +CA,A,YES,4,12078,3,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,217.23853819488096,10.31363779327352,3.052167508140189,0,2023-04-26,NA +CA,A,YES,4,12078,4,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,147.8106706816599,9.652858417257933,1.5527913057744365,0,2023-04-26,NA +WA,B,Yes,7,07528,1,100000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,6.296221235584299,7.115636788104664,10.387813111522187,0,2023-04-26,NA +WA,B,Yes,8,97385,2,50000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,27.46919277375994,456.47292187041234,4.411504863466449,0,2023-04-26,NA +NM,C,yes,10,27604,5,1000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,1.4664156180624237,659.6567691595407,0.33705174460432563,0,2023-04-26,NA +YY,E,No,13,65174,2,13,upstream,primary sludge,copies/g dry sludge,sars-cov-2,60.70509365653131,101.47081384689383,1.1615174290288617,0,2023-04-26,below loq +CA,A,YES,1,04971,1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,208.04588021116533,103.55688479513707,2.691806067533322,0,2023-04-27,NA +CA,A,YES,2,31809,1,30000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,53.274180656979944,820.6814423366915,11.919816966529458,0,2023-04-27,NA +CA,A,YES,4,12078,3,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,575.1924668955329,11.920612056191986,8.524966641800178,0,2023-04-27,no +CA,A,YES,4,12078,4,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,0.801535748563595,21.94619927781503,1.60307149712719,1,2023-04-27,NA +WA,B,Yes,6,31928,1,2000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,8.325609824673485,270.8679649084886,5.321569026385628,0,2023-04-27,NA +WA,B,Yes,7,07528,1,100000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,5.382393185970981,19.014623933202657,1.4033406539427198,0,2023-04-27,NA +WA,B,Yes,8,97385,2,50000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,91.25817134557516,331.62674903296255,1.4324917708791804,0,2023-04-27,NA +WA,B,Yes,9,14396,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,280.81347867539483,6.520918067681479,2.775740578136018,0,2023-04-27,NA +NM,C,yes,10,27604,5,1000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,1.2553339133621488,26.884360791717008,1.149406491947094,0,2023-04-27,NA +NM,C,yes,11,91578,6,10000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,1.5322295469537799,169.48498742663392,0.8582923829102322,0,2023-04-27,NA +YY,E,No,12,32541,1,13,upstream,primary sludge,copies/g dry sludge,sars-cov-2,79.44765142043451,222.308652960011,23.64480588964088,0,2023-04-27,NA +ZZ,F,NO,12,32541,1,8472,wwtp,raw wastewater,copies/g dry sludge,sars-cov-2,9.669034640536697,22.33185092635469,19.338069281073395,1,2023-04-27,NA +ZZ,F,NO,15,98436,4,8472,wwtp,raw wastewater,copies/g dry sludge,sars-cov-2,73.08836970536736,38.1529044790854,22.798485293123537,0,2023-04-27,NA +CA,A,YES,1,04971,1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,6.999970333652621,291.4020800306635,1.7604127194300565,0,2023-04-28,NA +CA,A,YES,2,31809,1,30000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,371.0363863962996,53.260900973447896,2.5671821277396587,0,2023-04-28,> max temp and/or hold time +CA,A,YES,4,12078,3,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,844.9449301454397,502.2418617689685,1.631595402698108,0,2023-04-28,> max temp and/or hold time +CA,A,YES,4,12078,4,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,38.34077595815648,280.4878000645076,1.1387535898468688,0,2023-04-28,result is not quantifiable +WA,B,Yes,6,31928,1,2000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,511.7817037076505,429.95146517260093,2.986924182627039,0,2023-04-28,no +WA,B,Yes,7,07528,1,100000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,70.34241432333175,40.298566499919026,1.9687390811274348,0,2023-04-28,result is not quantifiable +WA,B,Yes,8,97385,2,50000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,13.058763427320049,833.8737705892236,2.736195264833119,0,2023-04-28,NA +WA,B,Yes,9,14396,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,5.396061369342731,22.71059618985424,7.186845957444637,0,2023-04-28,NA +NM,C,yes,10,27604,5,1000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,2.1708674061943305,6.989954766345039,0.6195850219436966,0,2023-04-28,no +XX,D,ye,14,09634,10,1000000000,wwtp,raw wastewater,copies/l wastewater,FLUAV,850.4514179664667,335.642098053034,4.393836808862517,0,2023-04-28,NA +CA,A,YES,1,04971,1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,560.0658426399126,100.00078493635858,11.271197957537696,0,2023-04-29,NA +CA,A,YES,2,31809,1,30000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,488.1045363416398,24.620903291639102,2.0183846522814695,0,2023-04-29,temperature not assessed upon arrival at the laboratory +CA,A,YES,4,12078,3,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,641.3257808273241,325.7785282678551,2.270722328822303,0,2023-04-29,NA +WA,B,Yes,6,31928,1,2000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,64.80996664030987,7.188380813577274,3.1961157943147254,0,2023-04-29,NA +WA,B,Yes,8,97385,2,50000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,7.18554509694876,5.3870090990504425,14.37109019389752,1,2023-04-29,NA +WA,B,Yes,9,14396,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,5.97086222214866,12.52765858411627,4.811423536639933,0,2023-04-29,no +NM,C,yes,10,27604,5,1000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,1.4177799625198153,6.286229313895517,0.7070417098917016,0,2023-04-29,NA +NM,C,yes,11,91578,6,10000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,2.369719720530965,72.22293718435151,1.0266707417994965,0,2023-04-29,NA +YY,E,No,14,09634,3,13,upstream,primary sludge,copies/g dry sludge,sars-cov-2,413.6396632378768,24.20334514929526,3.5206417060830546,0,2023-04-29,NA +CA,A,YES,1,04971,1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,3.32450934958347,235.98845261349894,6.64901869916694,1,2023-04-30,NA +CA,A,YES,2,31809,1,30000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,432.84917078058385,6.319487844386457,3.8899588593964083,0,2023-04-30,below loq +CA,A,YES,4,12078,3,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,18.45668065328468,5.644103603863737,13.720615491710742,0,2023-04-30,no +CA,A,YES,4,12078,4,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,14.00983196989704,850.5684832964836,22.078730981681296,0,2023-04-30,no +WA,B,Yes,6,31928,1,2000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,0.824104984255467,77.63224155256582,1.648209968510934,1,2023-04-30,yes +WA,B,Yes,7,07528,1,100000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,827.4937747266428,5.428272347592286,4.356713600577065,0,2023-04-30,NA +WA,B,Yes,8,97385,2,50000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,698.7745753713236,698.803125916308,1.4319671221737449,0,2023-04-30,no +WA,B,Yes,9,14396,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,5.358059856342436,33.07476108777929,5.06766761755592,0,2023-04-30,NA +NM,C,yes,10,27604,5,1000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,2.6813715780406335,15.05178904848824,0.6896286272687399,0,2023-04-30,temperature not assessed upon arrival at the laboratory +NM,C,yes,11,91578,6,10000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,1.3182920676305083,8.521220101289806,0.1617059898946307,0,2023-04-30,no +CA,A,YES,1,04971,1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,126.39909080021351,157.10491649611788,3.067204511505263,0,2023-05-01,NA +CA,A,YES,4,12078,3,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,180.07925754475085,7.279911081740505,1.5229264195104935,0,2023-05-01,NA +CA,A,YES,4,12078,4,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,11.66831183899276,46.16363058138029,13.962847005760967,0,2023-05-01,NA +WA,B,Yes,6,31928,1,2000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,26.69352868165472,649.1344404868371,8.844742015201016,0,2023-05-01,no +WA,B,Yes,7,07528,1,100000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,4.125173737123623,10.927394438026713,8.250347474247246,1,2023-05-01,NA +WA,B,Yes,8,97385,2,50000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,2.069070292431558,72.79373184044034,4.138140584863116,1,2023-05-01,NA +WA,B,Yes,9,14396,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,284.89234776819956,259.5699931303272,16.005252726491367,0,2023-05-01,NA +NM,C,yes,10,27604,5,1000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,2.3517199062499174,14.360895813156974,1.3296240203832916,0,2023-05-01,yes +NM,C,yes,11,91578,6,10000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,2.5386329288283753,295.43051467768817,0.7309938220791863,0,2023-05-01,NA +ZZ,F,NO,12,32541,1,8472,wwtp,raw wastewater,copies/g dry sludge,sars-cov-2,5.094366390743734,16.672166739194523,16.302630258421793,0,2023-05-01,no +ZZ,F,NO,13,65174,2,8472,wwtp,raw wastewater,copies/g dry sludge,sars-cov-2,359.22207666562974,91.00063333575055,1.1828553936228068,0,2023-05-01,yes +CA,A,YES,1,04971,1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,13.167240820226265,9.647033797931563,11.831287615485943,0,2023-05-02,no +CA,A,YES,2,31809,1,30000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,194.45846088210806,205.4973725477047,2.8874883364653505,0,2023-05-02,no +CA,A,YES,4,12078,3,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,58.45601230437733,15.617335284955422,9.725898991741676,0,2023-05-02,no +CA,A,YES,4,12078,4,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,1.1159132292358338,17.81804116754442,2.2318264584716676,1,2023-05-02,y +CA,A,YES,5,34687,4,1000000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,10.793151927887916,74.9405348976295,5.1902510946435445,0,2023-05-02,NA +WA,B,Yes,6,31928,1,2000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,13.592826008421673,6.611016298249312,7.401449649309691,0,2023-05-02,NA +WA,B,Yes,7,07528,1,100000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,172.37057991253843,19.933849268276305,22.042262043503793,0,2023-05-02,no +WA,B,Yes,8,97385,2,50000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,20.18807652956671,649.1425764540309,5.588368409462939,0,2023-05-02,NA +NM,C,yes,10,27604,5,1000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,2.9515590864751067,545.9635747566659,0.8247335456963463,0,2023-05-02,NA +NM,C,yes,11,91578,6,10000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,1.6397095225841265,16.745792519220874,0.7778295030992146,0,2023-05-02,NA +CA,A,YES,1,04971,1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,75.45453801139261,174.94212865353848,2.1148952660069806,0,2023-05-03,NA +CA,A,YES,2,31809,1,30000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,798.9596874998856,86.14722796498161,1.4098484908518343,0,2023-05-03,no +CA,A,YES,4,12078,3,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,4.4230710158255615,61.276040709658425,8.846142031651123,1,2023-05-03,y +CA,A,YES,4,12078,4,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,6.033512992915349,283.12071476660077,1.1524204864120626,0,2023-05-03,NA +WA,B,Yes,6,31928,1,2000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,361.82174869275235,412.1726882019008,3.392967811549041,0,2023-05-03,NA +WA,B,Yes,7,07528,1,100000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,11.70915083854198,5.4517105541978195,3.998170760729833,0,2023-05-03,NA +WA,B,Yes,9,14396,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,5.682255156783968,6.697510679194023,8.949831096621802,0,2023-05-03,NA +NM,C,yes,10,27604,5,1000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,2.8065377188350977,141.18746211251073,0.11029372080973322,0,2023-05-03,no +NM,C,yes,11,91578,6,10000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,1.6045755223818543,308.83979426045056,0.2874724726472874,0,2023-05-03,no +ZZ,F,NO,12,32541,1,8472,wwtp,raw wastewater,copies/g dry sludge,sars-cov-2,16.479547177592877,133.90982813149452,22.68334622677171,0,2023-05-03,NA +CA,A,YES,1,04971,1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,15.586661702644017,486.2775228649737,15.35829976263435,0,2023-05-04,NA +CA,A,YES,2,31809,1,30000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,13.869096496141317,371.89552269272076,5.788525413467363,0,2023-05-04,NA +CA,A,YES,4,12078,3,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,323.2076513997515,397.9307153994006,8.641007325263786,0,2023-05-04,NA +WA,B,Yes,6,31928,1,2000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,18.74289910238316,33.31537370201077,18.825149889096664,0,2023-05-04,NA +WA,B,Yes,7,07528,1,100000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,87.56096558150305,210.6593961449894,19.355792201443624,0,2023-05-04,NA +WA,B,Yes,8,97385,2,50000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,52.63655482296249,853.6217859408173,14.75972634458482,0,2023-05-04,NA +WA,B,Yes,9,14396,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,1.0908618823912202,9.601131820134302,2.1817237647824403,1,2023-05-04,no +NM,C,yes,11,91578,6,10000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,0.9236604871123559,63.72798769754329,1.3647248749582441,0,2023-05-04,> max temp and/or hold time +XX,D,ye,13,65174,10,1000000000,wwtp,raw wastewater,copies/l wastewater,FLUAV,329.0310724879845,147.48107985659834,15.179999526067473,0,2023-05-04,NA +XX,D,ye,14,09634,10,1000000000,wwtp,raw wastewater,copies/l wastewater,FLUAV,2.3325059816367673,156.0210348452466,4.6650119632735345,1,2023-05-04,NA +ZZ,F,NO,12,32541,1,8472,wwtp,raw wastewater,copies/g dry sludge,sars-cov-2,6.974594799887846,38.691175090662064,1.0999467854799458,0,2023-05-04,NA +CA,A,YES,1,04971,1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,741.2838545573932,6.512754463154311,18.960706734517032,0,2023-05-05,NA +CA,A,YES,2,31809,1,30000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,11.175276552688192,13.847475497747844,10.31159381756311,0,2023-05-05,NA +CA,A,YES,4,12078,3,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,353.2196122637675,11.076161672400303,8.306734923461548,0,2023-05-05,no +CA,A,YES,4,12078,4,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,38.685208699574964,152.87902704525837,1.7687885507785361,0,2023-05-05,NA +WA,B,Yes,6,31928,1,2000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,13.662184846720443,5.678396098094882,18.780388831150216,0,2023-05-05,y +WA,B,Yes,7,07528,1,100000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,516.2452340163961,197.2847081127134,2.3450897766945604,0,2023-05-05,yes +WA,B,Yes,8,97385,2,50000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,190.0890489367773,75.11816457143435,2.9705270402936423,0,2023-05-05,NA +NM,C,yes,10,27604,5,1000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,0.994311687737122,406.842475936896,0.13516827114263688,0,2023-05-05,no +NM,C,yes,11,91578,6,10000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,1.6097605683829808,60.0350054120879,0.8733447425219474,0,2023-05-05,NA +XX,D,ye,12,32541,10,1000000000,wwtp,raw wastewater,copies/l wastewater,FLUAV,248.2688759987132,14.829613550991244,1.782822424751423,0,2023-05-05,no +YY,E,No,13,65174,2,13,upstream,primary sludge,copies/g dry sludge,sars-cov-2,11.682221377701298,13.542731459362583,1.509893392242752,0,2023-05-05,no +ZZ,F,NO,15,98436,4,8472,wwtp,raw wastewater,copies/g dry sludge,sars-cov-2,39.78230528490684,14.023513374605809,4.308838293737673,0,2023-05-05,NA +CA,A,YES,1,04971,1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,8.48954291484976,39.35578831757263,2.0469024097072297,0,2023-05-06,below loq +CA,A,YES,2,31809,1,30000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,3.0539834050350025,15.148250438030558,6.107966810070005,1,2023-05-06,NA +CA,A,YES,4,12078,3,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,3.1581856888921447,39.377965667019836,6.316371377784289,1,2023-05-06,no +CA,A,YES,4,12078,4,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,20.871524659166692,29.45033010567461,1.971245204586767,0,2023-05-06,no +WA,B,Yes,6,31928,1,2000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,18.657603638747016,880.1788810601715,8.115614147593085,0,2023-05-06,result is not quantifiable +WA,B,Yes,7,07528,1,100000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,58.11789037373,41.069861023623936,2.861250908142004,0,2023-05-06,below loq +WA,B,Yes,8,97385,2,50000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,392.40289205674054,59.34956624736275,9.6707308731978,0,2023-05-06,no +WA,B,Yes,9,14396,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,0.5426373123151723,104.21961163671229,1.0852746246303446,1,2023-05-06,NA +NM,C,yes,10,27604,5,1000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,2.1876565716043546,110.02870458460215,0.6014996503675683,0,2023-05-06,no +XX,D,ye,12,32541,10,1000000000,wwtp,raw wastewater,copies/l wastewater,FLUAV,11.43292290286606,27.506365272225754,13.759174550953487,0,2023-05-06,NA +XX,D,ye,13,65174,10,1000000000,wwtp,raw wastewater,copies/l wastewater,FLUAV,18.747462442743863,19.946895779862693,6.063474053678778,0,2023-05-06,NA +CA,A,YES,1,04971,1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,58.809195500389826,152.84300810289068,3.3717870985386202,0,2023-05-07,y +CA,A,YES,2,31809,1,30000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,683.4899880001176,35.58329693674288,1.8396085882704407,0,2023-05-07,NA +CA,A,YES,4,12078,3,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,9.713375570415083,198.90980468351847,1.5988946509832203,0,2023-05-07,NA +CA,A,YES,4,12078,4,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,8.968834536113757,325.85665461325084,16.96107512552476,0,2023-05-07,NA +WA,B,Yes,6,31928,1,2000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,8.63832294978206,223.73394586163096,1.15097615979303,0,2023-05-07,NA +WA,B,Yes,7,07528,1,100000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,12.688701856368318,13.700831847692132,1.136715437320594,0,2023-05-07,NA +WA,B,Yes,9,14396,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,7.9375420614033985,35.53685806500322,2.843405936463906,0,2023-05-07,NA +NM,C,yes,10,27604,5,1000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,1.9472627478277935,18.988839218789366,1.1774012303820522,0,2023-05-07,NA +ZZ,F,NO,15,98436,4,8472,wwtp,raw wastewater,copies/g dry sludge,sars-cov-2,16.779834316342004,28.0722399216928,15.65463712014979,0,2023-05-07,NA +CA,A,YES,1,04971,1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,130.27569233977755,439.6737693684355,23.251322844723248,0,2023-05-08,NA +CA,A,YES,2,31809,1,30000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,361.40112906072727,61.22342197409292,2.576477774840847,0,2023-05-08,NA +CA,A,YES,4,12078,3,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,32.39996799701914,28.82829484546444,4.0052456437600075,0,2023-05-08,NA +CA,A,YES,4,12078,4,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,981.1537183969033,56.872459904927226,6.327830340071864,0,2023-05-08,y +WA,B,Yes,6,31928,1,2000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,948.0332822374847,119.12293194091615,3.9237460646628537,0,2023-05-08,NA +WA,B,Yes,7,07528,1,100000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,181.4727165945616,12.78211127132978,3.090225890113611,0,2023-05-08,NA +WA,B,Yes,8,97385,2,50000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,6.235097734426509,54.60141209914501,1.256086550622904,0,2023-05-08,NA +WA,B,Yes,9,14396,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,157.56654965758685,345.8669487554043,11.677685306674013,0,2023-05-08,NA +NM,C,yes,10,27604,5,1000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,2.213811515799708,110.26595043811086,0.34809492625701544,0,2023-05-08,yes +NM,C,yes,11,91578,6,10000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,0.839409857789925,6.0737698525946096,0.7609343773047139,0,2023-05-08,NA +XX,D,ye,12,32541,10,1000000000,wwtp,raw wastewater,copies/l wastewater,FLUAV,41.921410512945954,105.30362489461278,3.6400012197255998,0,2023-05-08,no +ZZ,F,NO,12,32541,1,8472,wwtp,raw wastewater,copies/g dry sludge,sars-cov-2,449.02431591294146,187.4924694541619,1.1981959121509067,0,2023-05-08,NA +CA,A,YES,4,12078,3,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,98.06797389660449,181.97194459847933,1.2716100953697005,0,2023-05-09,temperature not assessed upon arrival at the laboratory +CA,A,YES,5,34687,3,1000000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,77.14820809183166,38.91736205274438,1.4736650396075555,0,2023-05-09,NA +CA,A,YES,4,12078,4,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,245.5138293428475,164.2946506646567,20.19720452417,0,2023-05-09,NA +CA,A,YES,5,34687,4,1000000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,156.4988208352484,70.89738715513944,7.359126277521304,0,2023-05-09,NA +WA,B,Yes,6,31928,1,2000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,4.864758820217579,8.53583206474093,9.729517640435159,1,2023-05-09,no +WA,B,Yes,7,07528,1,100000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,47.15517728386585,26.762066148870552,20.85864963610736,0,2023-05-09,NA +WA,B,Yes,8,97385,2,50000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,32.14412432226787,11.652339440842086,18.08013099489817,0,2023-05-09,NA +NM,C,yes,10,27604,5,1000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,2.1551480430457852,42.125682768141,0.5341347284309143,0,2023-05-09,NA +NM,C,yes,11,91578,6,10000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,2.0330957113603785,69.72372775796367,0.7489104741935365,0,2023-05-09,NA +XX,D,ye,15,98436,10,1000000000,wwtp,raw wastewater,copies/l wastewater,FLUAV,200.40669661042503,71.87017168243563,14.558700816918398,0,2023-05-09,NA +YY,E,No,12,32541,1,13,upstream,primary sludge,copies/g dry sludge,sars-cov-2,14.203805318442566,598.1663393564133,8.932650809999693,0,2023-05-09,NA +ZZ,F,NO,12,32541,1,8472,wwtp,raw wastewater,copies/g dry sludge,sars-cov-2,56.78812704308477,523.7215636578719,1.32218861626182,0,2023-05-09,NA +CA,A,YES,2,31809,1,30000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,37.41443871089028,488.5071841160147,6.288618476744448,0,2023-05-10,NA +CA,A,YES,4,12078,3,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,527.3317932016605,797.739683766454,1.2187930442349697,0,2023-05-10,result is not quantifiable +CA,A,YES,4,12078,4,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,10.360323733417687,12.317566209683534,1.019241933572425,0,2023-05-10,NA +WA,B,Yes,7,07528,1,100000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,72.85325077217195,11.411298160491015,15.265403207738379,0,2023-05-10,NA +WA,B,Yes,8,97385,2,50000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,85.33539173719784,507.56408911990434,1.2245979644892824,0,2023-05-10,no +WA,B,Yes,9,14396,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,31.91187496794521,103.67627382303348,21.780300092317898,0,2023-05-10,NA +NM,C,yes,10,27604,5,1000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,2.256087680867849,716.8324822039457,0.1624529144404517,0,2023-05-10,NA +NM,C,yes,11,91578,6,10000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,1.6946341906499007,153.16098499686217,0.8807744661738537,0,2023-05-10,NA +XX,D,ye,14,09634,10,1000000000,wwtp,raw wastewater,copies/l wastewater,FLUAV,550.8416991192717,220.78692628989512,1.9839118171639367,0,2023-05-10,NA +CA,A,YES,1,04971,1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,303.5259117307216,618.6713389578282,13.139319300179181,0,2023-05-11,no +CA,A,YES,4,12078,3,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,14.157814898817636,145.50583381018333,22.964901658773055,0,2023-05-11,NA +CA,A,YES,4,12078,4,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,24.589538905387833,5.265624781223151,1.8268296547535512,0,2023-05-11,no +WA,B,Yes,6,31928,1,2000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,3.2747882985805314,132.74505034721957,6.549576597161063,1,2023-05-11,NA +WA,B,Yes,7,07528,1,100000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,767.6927853645723,17.446844663230237,6.076995664662751,0,2023-05-11,no +WA,B,Yes,8,97385,2,50000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,38.19039495843281,370.8668858669339,7.1230665054547,0,2023-05-11,NA +WA,B,Yes,9,14396,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,41.48044540948232,720.9077048625485,22.71658315627536,0,2023-05-11,NA +NM,C,yes,10,27604,5,1000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,0.8832715233524635,329.5936831023259,1.298288819136148,0,2023-05-11,NA +NM,C,yes,11,91578,6,10000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,-0.01763037982498822,16.65580503404038,0.283399615838993,1,2023-05-11,NA +YY,E,No,13,65174,2,13,upstream,primary sludge,copies/g dry sludge,sars-cov-2,4.2674466851261865,107.37397914674295,8.534893370252373,1,2023-05-11,NA +CA,A,YES,1,04971,1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,10.540565710845435,5.703475703147575,21.08113142169087,1,2023-05-12,NA +CA,A,YES,2,31809,1,30000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,6.176377152174395,8.977679302540572,14.16075939728461,0,2023-05-12,no +CA,A,YES,4,12078,3,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,5.919047393158567,6.1100449574325,11.838094786317134,1,2023-05-12,no +CA,A,YES,4,12078,4,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,89.01507878773202,90.89799674311114,18.04901604920424,0,2023-05-12,NA +WA,B,Yes,6,31928,1,2000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,84.05133484366335,169.769693874769,8.330007810549052,0,2023-05-12,NA +WA,B,Yes,7,07528,1,100000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,80.30837048290714,64.50397368036465,1.7180404273157999,0,2023-05-12,NA +WA,B,Yes,8,97385,2,50000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,10.105348153839827,319.00157343516423,17.44609141322325,0,2023-05-12,NA +WA,B,Yes,9,14396,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,45.23805430212266,63.268854860535626,2.00038865752301,0,2023-05-12,NA +NM,C,yes,10,27604,5,1000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,2.7727142243940746,59.19572435742954,1.293431219229691,0,2023-05-12,NA +NM,C,yes,11,91578,6,10000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,2.513870024088659,390.06918434304947,1.3715217165676923,0,2023-05-12,NA +YY,E,No,15,98436,4,13,upstream,primary sludge,copies/g dry sludge,sars-cov-2,118.49650899461251,755.5811420789239,3.2084720002359366,0,2023-05-12,NA +CA,A,YES,1,04971,1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,86.93925031392622,61.31553298100391,10.500151659860256,0,2023-05-13,NA +CA,A,YES,2,31809,1,30000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,80.71946716865867,48.19330701311014,6.2000010031220585,0,2023-05-13,NA +CA,A,YES,4,12078,3,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,7.685255306842818,141.11393191180213,9.361953642604531,0,2023-05-13,NA +CA,A,YES,4,12078,4,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,2.84828471109639,86.44569869581473,5.69656942219278,1,2023-05-13,no +WA,B,Yes,6,31928,1,2000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,53.35432921810096,76.09744425125925,3.2217477893878095,0,2023-05-13,NA +WA,B,Yes,7,07528,1,100000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,44.05353332111659,6.383799670237423,15.335222288649632,0,2023-05-13,no +WA,B,Yes,8,97385,2,50000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,20.55823156009283,759.1218225293253,14.462911970732762,0,2023-05-13,NA +WA,B,Yes,9,14396,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,8.766971067118003,53.63153133026417,1.3551263755757346,0,2023-05-13,NA +NM,C,yes,10,27604,5,1000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,0.13992624809277923,67.4151650674816,0.44095624375676046,1,2023-05-13,NA +NM,C,yes,11,91578,6,10000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,2.4996948486577666,37.62228090742334,0.922143074652857,0,2023-05-13,NA +CA,A,YES,1,04971,1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,279.89140422897793,192.9022700619561,20.297971158600266,0,2023-05-14,NA +CA,A,YES,2,31809,1,30000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,95.72865997906074,360.9305932737754,1.8166472786359893,0,2023-05-14,NA +WA,B,Yes,6,31928,1,2000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,17.235530400186455,11.779515497397066,1.129976611539121,0,2023-05-14,yes +WA,B,Yes,7,07528,1,100000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,566.4508540992459,25.072809968821453,1.5430419674286542,0,2023-05-14,NA +WA,B,Yes,8,97385,2,50000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,184.34488325197583,30.50313170331635,3.5099003371340842,0,2023-05-14,NA +WA,B,Yes,9,14396,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,210.25185574870903,17.61984870946503,3.968916190055598,0,2023-05-14,NA +NM,C,yes,10,27604,5,1000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,2.187349112546333,63.03431492831927,1.0403607361695468,0,2023-05-14,NA +NM,C,yes,11,91578,6,10000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,0.7276552961029552,159.9749967513903,0.3889270374039342,0,2023-05-14,NA +YY,E,No,15,98436,4,13,upstream,primary sludge,copies/g dry sludge,sars-cov-2,224.89046221967178,12.571410317774037,1.941650621102451,0,2023-05-14,NA +CA,A,YES,1,04971,1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,601.9975510440462,201.68876689786737,19.109713023668867,0,2023-05-15,temperature not assessed upon arrival at the laboratory +CA,A,YES,2,31809,1,30000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,137.52745890879558,61.84841375647781,2.541117757157155,0,2023-05-15,NA +CA,A,YES,4,12078,3,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,92.76228205183253,224.8689656855861,4.281389532706772,0,2023-05-15,NA +CA,A,YES,4,12078,4,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,6.496335623626201,308.70920355142766,12.586762232582274,0,2023-05-15,no +WA,B,Yes,6,31928,1,2000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,1.920274836196945,36.17410598623839,3.84054967239389,1,2023-05-15,NA +WA,B,Yes,7,07528,1,100000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,90.66270866804192,688.2992784003794,8.705428392425011,0,2023-05-15,NA +WA,B,Yes,8,97385,2,50000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,6.207504081379959,8.145467700184918,1.7294984743859478,0,2023-05-15,NA +WA,B,Yes,9,14396,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,12.84516449429904,259.84301461088285,17.68759057168942,0,2023-05-15,NA +NM,C,yes,10,27604,5,1000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,2.061315101812813,6.788622441013434,0.2741676437894175,0,2023-05-15,NA +CA,A,YES,1,04971,1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,5.399639999551275,32.131818045827,2.5192556599572704,0,2023-05-16,NA +CA,A,YES,2,31809,1,30000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,1.0235557830273876,43.76413723367602,2.047111566054775,1,2023-05-16,y +CA,A,YES,4,12078,3,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,13.387766204253701,277.4527851775533,1.2126745870033109,0,2023-05-16,no +CA,A,YES,5,34687,3,1000000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,187.96946367831058,9.145998926904923,5.776146985459205,0,2023-05-16,NA +CA,A,YES,4,12078,4,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,5.295518334926672,183.71463183977792,2.0008245980858104,0,2023-05-16,NA +CA,A,YES,5,34687,4,1000000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,41.122978067821876,331.4796985381598,9.73235755340418,0,2023-05-16,NA +WA,B,Yes,6,31928,1,2000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,16.695131362918914,10.359659802230343,4.501303872754907,0,2023-05-16,NA +WA,B,Yes,8,97385,2,50000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,379.60719267211874,181.27518393367768,1.2118963868273491,0,2023-05-16,NA +WA,B,Yes,9,14396,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,547.4886863228286,267.72195797462825,2.1648210003607096,0,2023-05-16,no +NM,C,yes,10,27604,5,1000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,1.584619295354954,340.182472180515,0.35370921650938636,0,2023-05-16,yes +NM,C,yes,11,91578,6,10000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,2.0522546658718404,6.429704448885619,1.0638227873978812,0,2023-05-16,NA +YY,E,No,12,32541,1,13,upstream,primary sludge,copies/g dry sludge,sars-cov-2,92.55554447468218,23.049969073329304,11.273647997186174,0,2023-05-16,NA +ZZ,F,NO,13,65174,2,8472,wwtp,raw wastewater,copies/g dry sludge,sars-cov-2,458.38772461500673,220.28882526822002,5.09503330805962,0,2023-05-16,NA +CA,A,YES,1,04971,1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,38.82271578473764,377.3869222461186,2.863484552043326,0,2023-05-17,NA +CA,A,YES,2,31809,1,30000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,533.82351620114,201.42108675455356,4.093114662273232,0,2023-05-17,NA +CA,A,YES,4,12078,3,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,31.037287283097793,84.27945039529483,6.60680139509126,0,2023-05-17,NA +CA,A,YES,4,12078,4,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,16.554535831429487,41.703645760139906,15.608276062813811,0,2023-05-17,NA +WA,B,Yes,7,07528,1,100000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,61.66912487355205,99.38564447215045,3.853657968874788,0,2023-05-17,no +WA,B,Yes,8,97385,2,50000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,66.28883930178358,146.38678359820517,1.043387202856135,0,2023-05-17,NA +NM,C,yes,10,27604,5,1000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,1.7072513163600087,252.59310689235096,0.2302776602657698,0,2023-05-17,NA +NM,C,yes,11,91578,6,10000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,2.7542050251615855,72.11520528842085,0.30954828118112226,0,2023-05-17,NA +CA,A,YES,1,04971,1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,146.38831947568443,172.829198192071,16.482343278862913,0,2023-05-18,NA +CA,A,YES,2,31809,1,30000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,170.85233999728462,824.0343047927632,19.6829976099525,0,2023-05-18,NA +CA,A,YES,4,12078,3,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,7.9759428813570645,34.47343438519275,1.7095585022603272,0,2023-05-18,no +CA,A,YES,4,12078,4,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,98.62745715877868,337.88697048484835,1.0217936875566813,0,2023-05-18,NA +WA,B,Yes,6,31928,1,2000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,11.942679211434587,5.023072392959766,23.885358422869174,1,2023-05-18,NA +WA,B,Yes,7,07528,1,100000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,15.57074686720341,66.6084569864403,1.0102680931909986,0,2023-05-18,NA +WA,B,Yes,8,97385,2,50000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,115.19549892752735,11.947496419935781,6.937774673620826,0,2023-05-18,no +WA,B,Yes,9,14396,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,458.9667691766944,251.60448122958704,1.0843813554143147,0,2023-05-18,NA +NM,C,yes,10,27604,5,1000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,1.7262490973599864,50.30404214251418,1.1249683398208015,0,2023-05-18,no +NM,C,yes,11,91578,6,10000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,0.9365110531554846,17.15031289097754,1.0809558264919896,0,2023-05-18,> max temp and/or hold time +XX,D,ye,13,65174,10,1000000000,wwtp,raw wastewater,copies/l wastewater,FLUAV,210.80761851824843,287.35172223030366,11.934813779736402,0,2023-05-18,NA +ZZ,F,NO,14,09634,3,8472,wwtp,raw wastewater,copies/g dry sludge,sars-cov-2,117.31670738617805,6.7324995979285776,1.6006026423009314,0,2023-05-18,NA +CA,A,YES,1,04971,1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,5.002846167617119,16.50332136529176,10.005692335234238,1,2023-05-19,NA +CA,A,YES,2,31809,1,30000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,326.704266388954,122.16568638209436,21.297482952603538,0,2023-05-19,NA +CA,A,YES,4,12078,3,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,342.3795291297284,387.38743382059675,17.481051890464474,0,2023-05-19,yes +CA,A,YES,4,12078,4,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,97.9774273823614,48.03146276657064,21.14791392988001,0,2023-05-19,y +WA,B,Yes,6,31928,1,2000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,714.8342642953518,235.96193355710287,7.0795660658003365,0,2023-05-19,NA +WA,B,Yes,7,07528,1,100000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,58.514475210232284,91.41389635339901,6.281084954651419,0,2023-05-19,NA +WA,B,Yes,8,97385,2,50000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,389.20984975977086,9.441802101067266,2.7749218085374316,0,2023-05-19,no +WA,B,Yes,9,14396,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,726.4147623903082,11.55853160602998,1.0202411109188378,0,2023-05-19,NA +NM,C,yes,10,27604,5,1000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,1.47716638850826,102.4861834271557,0.45063035944366436,0,2023-05-19,NA +NM,C,yes,11,91578,6,10000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,1.9463969624931354,531.8437995782391,1.0707841044927238,0,2023-05-19,NA +XX,D,ye,14,09634,10,1000000000,wwtp,raw wastewater,copies/l wastewater,FLUAV,0.6544447770136405,237.2453060303681,1.308889554027281,1,2023-05-19,no +ZZ,F,NO,13,65174,2,8472,wwtp,raw wastewater,copies/g dry sludge,sars-cov-2,5.571504410881499,132.47972817095504,4.538088873799303,0,2023-05-19,NA +CA,A,YES,1,04971,1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,865.8282311572634,176.39108944087667,6.302458811424311,0,2023-05-20,no +CA,A,YES,2,31809,1,30000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,43.27012656860755,35.65428403389397,21.601541410835775,0,2023-05-20,NA +CA,A,YES,4,12078,3,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,11.176882799062037,114.49353270803404,2.6892557683893807,0,2023-05-20,no +CA,A,YES,4,12078,4,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,64.09693070446394,14.045765575504465,23.499639399314944,0,2023-05-20,NA +WA,B,Yes,6,31928,1,2000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,23.17371196027208,297.86042718993485,1.6758473699931842,0,2023-05-20,NA +WA,B,Yes,7,07528,1,100000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,972.280654799704,23.812419066904784,2.429494382129149,0,2023-05-20,yes +WA,B,Yes,8,97385,2,50000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,6.224001936494755,11.768823647538094,1.854639846797819,0,2023-05-20,NA +WA,B,Yes,9,14396,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,61.976473249966254,19.83300420202743,5.51233798991324,0,2023-05-20,> max temp and/or hold time +NM,C,yes,10,27604,5,1000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,1.8126858771910022,99.4704369426305,0.37831636117792217,0,2023-05-20,NA +NM,C,yes,11,91578,6,10000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,0.42135747552725544,36.218266362726645,0.7223874711912367,1,2023-05-20,NA +CA,A,YES,4,12078,3,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,25.176064587169314,8.749725675189737,2.972238149968307,0,2023-05-21,NA +WA,B,Yes,6,31928,1,2000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,207.43738029840407,202.4754836026557,11.998745582439804,0,2023-05-21,NA +WA,B,Yes,7,07528,1,100000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,377.73790317071206,104.85055810273369,8.269438247389818,0,2023-05-21,no +WA,B,Yes,8,97385,2,50000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,1.700260252672952,596.1879120370844,3.400520505345904,1,2023-05-21,no +WA,B,Yes,9,14396,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,447.9017539107904,19.476964168560965,1.1981223994391035,0,2023-05-21,no +NM,C,yes,10,27604,5,1000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,0.8361058589334085,251.62088669283023,1.3340876872346699,0,2023-05-21,below loq +NM,C,yes,11,91578,6,10000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,2.4134725757443474,54.397842822300255,0.6634579459126085,0,2023-05-21,NA +XX,D,ye,14,09634,10,1000000000,wwtp,raw wastewater,copies/l wastewater,FLUAV,10.529050520304677,187.92614562312716,21.058101040609355,1,2023-05-21,NA +ZZ,F,NO,13,65174,2,8472,wwtp,raw wastewater,copies/g dry sludge,sars-cov-2,20.221795493143,9.18249545716989,13.502515544071704,0,2023-05-21,y +CA,A,YES,1,04971,1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,224.07955465704714,11.129784849158769,4.19206971903038,0,2023-05-22,NA +CA,A,YES,2,31809,1,30000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,17.227265721236904,16.310306791247932,3.8318192444096533,0,2023-05-22,no +CA,A,YES,4,12078,3,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,5.416210971962392,158.93929503159652,22.63014239011,0,2023-05-22,NA +CA,A,YES,4,12078,4,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,690.3106551816034,126.24830605679335,1.5738251300994728,0,2023-05-22,NA +WA,B,Yes,6,31928,1,2000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,60.43232693939857,9.252286612271343,2.8299955511694965,0,2023-05-22,NA +WA,B,Yes,7,07528,1,100000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,257.3259781819269,134.68426273691202,4.1632297907193765,0,2023-05-22,NA +WA,B,Yes,8,97385,2,50000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,503.1231310317763,14.934335221462257,12.714965557029682,0,2023-05-22,NA +NM,C,yes,10,27604,5,1000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,2.479271463446173,620.1731397388947,0.5052738168418361,0,2023-05-22,NA +NM,C,yes,11,91578,6,10000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,2.8515420599499866,358.7257262274633,0.5522092591357017,0,2023-05-22,NA +XX,D,ye,15,98436,10,1000000000,wwtp,raw wastewater,copies/l wastewater,FLUAV,6.92430328345902,8.32637682440617,13.84860656691804,1,2023-05-22,NA +CA,A,YES,1,04971,1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,56.435957943803174,73.85178932618149,18.18520482551056,0,2023-05-23,NA +CA,A,YES,2,31809,1,30000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,161.6044909954615,123.73649218406017,3.384065937696514,0,2023-05-23,y +CA,A,YES,4,12078,3,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,10.479848556861334,61.804290624058126,6.717100142360066,0,2023-05-23,NA +CA,A,YES,5,34687,3,1000000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,25.4518426449602,416.0125171300613,1.5321394978884033,0,2023-05-23,NA +CA,A,YES,5,34687,4,1000000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,11.009050182354105,16.136309499503383,5.195299066956687,0,2023-05-23,NA +WA,B,Yes,6,31928,1,2000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,205.7110952452072,64.03836991156005,17.558536165241293,0,2023-05-23,NA +WA,B,Yes,7,07528,1,100000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,943.5097989713016,22.581348882518306,1.5557746700474782,0,2023-05-23,NA +NM,C,yes,10,27604,5,1000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,2.3654620298956432,986.6868929949203,0.8976708226206057,0,2023-05-23,NA +NM,C,yes,11,91578,6,10000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,1.452505402225905,36.767085819680275,0.3895632432815658,0,2023-05-23,no +XX,D,ye,13,65174,10,1000000000,wwtp,raw wastewater,copies/l wastewater,FLUAV,79.51439331851611,116.41006735740858,1.425510980027306,0,2023-05-23,no +YY,E,No,15,98436,4,13,upstream,primary sludge,copies/g dry sludge,sars-cov-2,14.120612150036221,10.741090077316265,1.8167211342504983,0,2023-05-23,NA +CA,A,YES,1,04971,1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,13.157355805362196,153.7932692351134,1.7738913709877437,0,2023-05-24,> max temp and/or hold time +CA,A,YES,4,12078,3,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,678.6659932379766,316.4186639963307,14.43938837529447,0,2023-05-24,> max temp and/or hold time +CA,A,YES,4,12078,4,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,353.161636752946,908.4946554755223,1.3593237787583339,0,2023-05-24,NA +WA,B,Yes,7,07528,1,100000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,1.3046726332691492,14.3868594416138,2.6093452665382983,1,2023-05-24,NA +WA,B,Yes,8,97385,2,50000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,7.667605903532735,89.84240879127282,1.1433547113784146,0,2023-05-24,NA +WA,B,Yes,9,14396,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,0.5663851050805276,60.22345435443492,1.1327702101610553,1,2023-05-24,NA +NM,C,yes,10,27604,5,1000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,2.630794991864196,11.354910558745384,0.4394786512526222,0,2023-05-24,temperature not assessed upon arrival at the laboratory +NM,C,yes,11,91578,6,10000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,1.3659314242448546,564.994895593238,0.8180575550493451,0,2023-05-24,NA +XX,D,ye,14,09634,10,1000000000,wwtp,raw wastewater,copies/l wastewater,FLUAV,466.82126726335855,375.24762154527144,11.354567670230887,0,2023-05-24,no +XX,D,ye,15,98436,10,1000000000,wwtp,raw wastewater,copies/l wastewater,FLUAV,5.928705041844908,30.778312441424227,4.898101180132712,0,2023-05-24,NA +CA,A,YES,1,04971,1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,5.827507281480294,5.614635812615721,17.492705146845655,0,2023-05-25,NA +CA,A,YES,2,31809,1,30000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,13.265199701080418,31.76781092867949,1.6080240462660593,0,2023-05-25,no +CA,A,YES,4,12078,4,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,645.6671326734727,589.4555183412973,7.807287819761643,0,2023-05-25,NA +WA,B,Yes,6,31928,1,2000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,349.5167938668359,25.86408497094788,15.825950140538298,0,2023-05-25,NA +WA,B,Yes,7,07528,1,100000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,548.6871718439456,803.649667019898,1.0128926168559382,0,2023-05-25,NA +WA,B,Yes,8,97385,2,50000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,3.829662447173397,312.3559209687922,7.659324894346794,1,2023-05-25,NA +WA,B,Yes,9,14396,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,9.42944798022602,848.0390132364623,19.558386048783497,0,2023-05-25,NA +NM,C,yes,10,27604,5,1000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,1.96330969549419,759.0108581672927,1.363036717235462,0,2023-05-25,NA +NM,C,yes,11,91578,6,10000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,1.7902229704332997,9.584140924318472,0.09765845791197622,0,2023-05-25,NA +XX,D,ye,14,09634,10,1000000000,wwtp,raw wastewater,copies/l wastewater,FLUAV,11.21521171365811,10.970808917565472,1.1312616681199354,0,2023-05-25,no +YY,E,No,14,09634,3,13,upstream,primary sludge,copies/g dry sludge,sars-cov-2,7.193708758757231,5.438973019979676,6.0027614817347335,0,2023-05-25,NA +ZZ,F,NO,14,09634,3,8472,wwtp,raw wastewater,copies/g dry sludge,sars-cov-2,589.8304754443087,5.666264516833046,17.20577582410742,0,2023-05-25,no +CA,A,YES,1,04971,1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,991.974730854174,43.804695804123156,13.012253586755072,0,2023-05-26,NA +CA,A,YES,2,31809,1,30000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,378.1602834322613,115.22029745676069,1.9134601811942624,0,2023-05-26,NA +CA,A,YES,4,12078,3,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,131.47962047171535,7.95167619177637,3.2539325590800914,0,2023-05-26,no +CA,A,YES,4,12078,4,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,4.871339657308557,276.835630332145,9.742679314617114,1,2023-05-26,NA +WA,B,Yes,6,31928,1,2000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,167.7634954092627,6.0900000862294705,19.694815188868027,0,2023-05-26,NA +WA,B,Yes,7,07528,1,100000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,529.8337410202203,9.257286224278664,2.999637328639855,0,2023-05-26,NA +WA,B,Yes,8,97385,2,50000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,1.4642815174766652,12.846074209754176,2.9285630349533305,1,2023-05-26,temperature not assessed upon arrival at the laboratory +WA,B,Yes,9,14396,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,530.9828578883064,28.04638542003842,1.9591868482759995,0,2023-05-26,temperature not assessed upon arrival at the laboratory +NM,C,yes,10,27604,5,1000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,2.0189747099278117,10.646618118320449,0.7889289448609544,0,2023-05-26,NA +NM,C,yes,11,91578,6,10000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,2.7772207276456293,35.9539774374944,0.46530360395174264,0,2023-05-26,NA +CA,A,YES,2,31809,1,30000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,99.57409850571689,13.574340638667277,14.079707711456251,0,2023-05-27,NA +CA,A,YES,4,12078,3,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,986.7938542706584,337.7414654496874,2.1272876736818094,0,2023-05-27,no +CA,A,YES,4,12078,4,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,365.1770960675602,62.42673834659085,2.9582669741327647,0,2023-05-27,no +WA,B,Yes,6,31928,1,2000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,1.0605107171198536,156.97081213040195,2.121021434239707,1,2023-05-27,NA +WA,B,Yes,7,07528,1,100000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,38.345292718282764,15.155838212038033,1.7875345085445717,0,2023-05-27,NA +WA,B,Yes,8,97385,2,50000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,495.0485653504598,377.71226434092694,11.313420636300823,0,2023-05-27,no +WA,B,Yes,9,14396,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,13.426268200807078,57.322657589918315,13.152248812699622,0,2023-05-27,NA +NM,C,yes,11,91578,6,10000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,2.0642273546961536,15.417255018459796,0.4673947760383755,0,2023-05-27,no +YY,E,No,12,32541,1,13,upstream,primary sludge,copies/g dry sludge,sars-cov-2,7.3164172061154895,557.1742679189549,1.9524293333390506,0,2023-05-27,NA +CA,A,YES,1,04971,1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,3.8722209834809966,11.874904724137725,7.744441966961993,1,2023-05-28,below loq +CA,A,YES,2,31809,1,30000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,21.936760903134747,19.794408077824063,1.1226962892471302,0,2023-05-28,NA +CA,A,YES,4,12078,3,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,40.355277797323424,74.58141950163451,3.3607064342505204,0,2023-05-28,NA +CA,A,YES,4,12078,4,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,7.144707824488737,79.09343780565374,14.289415648977474,1,2023-05-28,> max temp and/or hold time +WA,B,Yes,6,31928,1,2000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,55.853159062227604,14.153578694682196,3.065119460530618,0,2023-05-28,> max temp and/or hold time +WA,B,Yes,7,07528,1,100000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,8.070145798836876,584.105481233125,2.180590108263856,0,2023-05-28,NA +WA,B,Yes,8,97385,2,50000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,33.29576778231581,12.231001844402927,6.537464487887805,0,2023-05-28,NA +WA,B,Yes,9,14396,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,3.7124315408858783,74.40141061408208,7.424863081771757,1,2023-05-28,no +NM,C,yes,10,27604,5,1000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,1.029972037423926,18.47967392886831,0.3405261310890953,0,2023-05-28,NA +NM,C,yes,11,91578,6,10000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,1.6402786292281981,29.39195712530409,1.021584954359083,0,2023-05-28,NA +XX,D,ye,15,98436,10,1000000000,wwtp,raw wastewater,copies/l wastewater,FLUAV,6.7353571136104655,20.18522634249294,4.7166385892979905,0,2023-05-28,y +CA,A,YES,1,04971,1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,134.56052411216203,68.35616771771406,4.597724848363917,0,2023-05-29,NA +CA,A,YES,2,31809,1,30000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,38.32774357283795,239.03622818680398,2.4594250664667268,0,2023-05-29,NA +CA,A,YES,4,12078,3,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,37.18596863004385,333.3327361472142,23.514420672209603,0,2023-05-29,below loq +WA,B,Yes,6,31928,1,2000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,6.066718246921654,11.888644589169925,24.419671805509402,0,2023-05-29,NA +WA,B,Yes,7,07528,1,100000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,546.7645325569501,30.2974890176503,13.407256988971396,0,2023-05-29,no +WA,B,Yes,8,97385,2,50000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,326.3635938769475,66.14295827663285,1.0995961395590341,0,2023-05-29,NA +WA,B,Yes,9,14396,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,208.9471435907558,47.574825124400824,1.4811143205074937,0,2023-05-29,no +NM,C,yes,10,27604,5,1000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,0.9867017544669734,495.91035241684705,1.0015188640956445,0,2023-05-29,no +NM,C,yes,11,91578,6,10000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,2.4208474598233396,236.12772663733344,0.13853508931180694,0,2023-05-29,NA +XX,D,ye,13,65174,10,1000000000,wwtp,raw wastewater,copies/l wastewater,FLUAV,101.5355234047676,65.96363830818757,1.7456421268735094,0,2023-05-29,NA +CA,A,YES,1,04971,1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,5.221786941619451,6.320443859538269,1.3314827910739935,0,2023-05-30,NA +CA,A,YES,2,31809,1,30000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,5.4767140165053885,456.7943792752749,11.167911237568681,0,2023-05-30,no +CA,A,YES,4,12078,3,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,378.4095712252915,17.198805020550573,2.937680568360228,0,2023-05-30,NA +CA,A,YES,5,34687,3,1000000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,312.6415446018471,42.71621020543016,1.3528602411149588,0,2023-05-30,no +CA,A,YES,4,12078,4,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,266.734588365075,44.11942178084142,1.5323376762292367,0,2023-05-30,no +CA,A,YES,5,34687,4,1000000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,119.61929568297784,6.52683576212407,1.2073754919590003,0,2023-05-30,no +WA,B,Yes,6,31928,1,2000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,1.8852302477603475,7.937877360545863,3.770460495520695,1,2023-05-30,no +WA,B,Yes,7,07528,1,100000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,167.8438037873305,6.218423548969813,1.9473666862941972,0,2023-05-30,NA +WA,B,Yes,8,97385,2,50000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,379.87529571674537,510.9568071417825,9.814214778442148,0,2023-05-30,no +WA,B,Yes,9,14396,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,159.1555547971404,11.408958476852641,4.63802897536972,0,2023-05-30,no +NM,C,yes,10,27604,5,1000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,1.4244503997867242,636.6102338319649,1.3323751716457437,0,2023-05-30,NA +NM,C,yes,11,91578,6,10000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,2.348900539073965,265.00410017408274,1.1208406918094636,0,2023-05-30,NA +CA,A,YES,1,04971,1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,35.06678316756629,913.5850921445049,1.581489785807367,0,2023-05-31,no +CA,A,YES,2,31809,1,30000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,6.788670108930944,32.55140313025375,2.1148840441344183,0,2023-05-31,no +CA,A,YES,4,12078,3,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,356.63090139635034,5.161968779103229,4.9108181978990695,0,2023-05-31,NA +WA,B,Yes,6,31928,1,2000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,7.612295819363786,87.49678097601002,4.165463805121776,0,2023-05-31,NA +WA,B,Yes,7,07528,1,100000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,34.864450448943174,110.15511485909215,14.200678210066382,0,2023-05-31,NA +WA,B,Yes,8,97385,2,50000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,810.1698709521145,6.1924220586359775,2.0396679250337595,0,2023-05-31,NA +WA,B,Yes,9,14396,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,79.68138298025984,534.5467810030315,7.123387275963566,0,2023-05-31,NA +NM,C,yes,10,27604,5,1000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,2.267094910693879,250.04743788629807,1.1203952035060318,0,2023-05-31,yes +NM,C,yes,11,91578,6,10000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,2.7647472858786264,6.697650855796543,0.7337328822890496,0,2023-05-31,NA +YY,E,No,12,32541,1,13,upstream,primary sludge,copies/g dry sludge,sars-cov-2,1.508570766117019,249.12826657532884,3.017141532234038,1,2023-05-31,no +YY,E,No,15,98436,4,13,upstream,primary sludge,copies/g dry sludge,sars-cov-2,0.5016624541325644,233.46805700096917,1.0033249082651288,1,2023-05-31,no +ZZ,F,NO,13,65174,2,8472,wwtp,raw wastewater,copies/g dry sludge,sars-cov-2,1.587440160403535,10.826080746808305,3.17488032080707,1,2023-05-31,NA +CA,A,YES,1,04971,1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,13.972497746966296,14.097585562510597,9.951165894532037,0,2023-06-01,NA +CA,A,YES,2,31809,1,30000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,1.515951171779795,719.0218563150918,3.03190234355959,1,2023-06-01,NA +CA,A,YES,4,12078,3,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,34.909237143175204,438.10319466849637,1.3356094954826512,0,2023-06-01,NA +CA,A,YES,4,12078,4,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,150.61814398332416,864.2637625218005,6.29583678487963,0,2023-06-01,NA +WA,B,Yes,6,31928,1,2000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,0.727634762225632,17.279227732343113,1.455269524451264,1,2023-06-01,no +WA,B,Yes,7,07528,1,100000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,763.5676275813757,10.97850518974091,1.0368043157678366,0,2023-06-01,NA +WA,B,Yes,8,97385,2,50000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,23.698391231596837,70.26352210580887,4.07599234105755,0,2023-06-01,no +WA,B,Yes,9,14396,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,359.62260382232,22.118748611621715,3.455018984597424,0,2023-06-01,NA +NM,C,yes,10,27604,5,1000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,1.688179851478964,5.5537513950694635,0.5978474845941443,0,2023-06-01,NA +YY,E,No,14,09634,3,13,upstream,primary sludge,copies/g dry sludge,sars-cov-2,285.20356230514494,216.35311504250504,18.90268738220765,0,2023-06-01,NA +ZZ,F,NO,13,65174,2,8472,wwtp,raw wastewater,copies/g dry sludge,sars-cov-2,550.9440558422646,23.46526231208123,1.1513467002791757,0,2023-06-01,below loq +ZZ,F,NO,14,09634,3,8472,wwtp,raw wastewater,copies/g dry sludge,sars-cov-2,34.837547209052516,28.51469798716336,1.3485567981938167,0,2023-06-01,NA +CA,A,YES,2,31809,1,30000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,5.296707135123079,21.680451295324353,1.7775894748199987,0,2023-06-02,NA +CA,A,YES,4,12078,3,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,43.64208253448855,14.49551958430133,1.6330255395129818,0,2023-06-02,NA +CA,A,YES,4,12078,4,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,836.8859951463978,517.9705193798546,10.206687698574452,0,2023-06-02,NA +WA,B,Yes,6,31928,1,2000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,9.581694540998116,35.339916865405414,2.212154984091819,0,2023-06-02,no +WA,B,Yes,7,07528,1,100000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,19.00877280010673,39.04269708050778,2.1440993763137417,0,2023-06-02,> max temp and/or hold time +WA,B,Yes,8,97385,2,50000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,54.479634030446505,186.7864698323426,1.2682035243195369,0,2023-06-02,no +WA,B,Yes,9,14396,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,9.905724325888672,5.812244610696417,1.882614647443275,0,2023-06-02,y +NM,C,yes,10,27604,5,1000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,1.2432119505761525,17.58209191177872,0.8550715687252866,0,2023-06-02,NA +NM,C,yes,11,91578,6,10000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,1.815074206467147,150.94430679649136,0.05102253671692746,0,2023-06-02,NA +XX,D,ye,12,32541,10,1000000000,wwtp,raw wastewater,copies/l wastewater,FLUAV,15.583975896526454,35.09575240906895,8.52197005996017,0,2023-06-02,NA +XX,D,ye,15,98436,10,1000000000,wwtp,raw wastewater,copies/l wastewater,FLUAV,166.5735936980469,6.007480448932445,1.5947377304376358,0,2023-06-02,NA +ZZ,F,NO,13,65174,2,8472,wwtp,raw wastewater,copies/g dry sludge,sars-cov-2,6.483390620419577,6.037862384258953,12.966781240839154,1,2023-06-02,NA +CA,A,YES,1,04971,1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,8.233667313492495,11.157350694770544,6.377286299468939,0,2023-06-03,NA +CA,A,YES,2,31809,1,30000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,62.62875801068984,48.16834973602615,8.880278724493799,0,2023-06-03,NA +CA,A,YES,4,12078,3,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,41.996889218785476,49.18066052573325,3.7841308685879853,0,2023-06-03,NA +CA,A,YES,4,12078,4,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,62.24539243563918,6.467022556505043,7.136041076672717,0,2023-06-03,NA +WA,B,Yes,6,31928,1,2000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,201.1190019381625,803.5687448561665,2.910816655821985,0,2023-06-03,no +WA,B,Yes,7,07528,1,100000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,9.317970433407892,79.58635354300762,17.686328657184458,0,2023-06-03,no +WA,B,Yes,8,97385,2,50000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,32.69782210936447,202.251393605709,24.76359191204226,0,2023-06-03,NA +WA,B,Yes,9,14396,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,192.85464151594854,27.485843949453407,1.0771288495798297,0,2023-06-03,NA +NM,C,yes,10,27604,5,1000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,1.4865833084094948,143.88884643564074,0.7337568668258807,0,2023-06-03,> max temp and/or hold time +ZZ,F,NO,14,09634,3,8472,wwtp,raw wastewater,copies/g dry sludge,sars-cov-2,712.9684328408249,71.67527384237853,5.5823669887677125,0,2023-06-03,NA +ZZ,F,NO,15,98436,4,8472,wwtp,raw wastewater,copies/g dry sludge,sars-cov-2,373.7809535728842,892.4585610944084,2.1202805308129706,0,2023-06-03,NA +CA,A,YES,1,04971,1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,132.28786729786012,471.13289773261994,14.410948003082948,0,2023-06-04,NA +CA,A,YES,2,31809,1,30000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,608.1624186407511,182.16934793933143,19.792049842872213,0,2023-06-04,NA +CA,A,YES,4,12078,3,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,40.80286855358536,6.69256491712004,2.7374797916921185,0,2023-06-04,NA +CA,A,YES,4,12078,4,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,952.1059272072401,289.6967453790776,19.32159666054628,0,2023-06-04,y +WA,B,Yes,6,31928,1,2000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,61.497654027123,7.689517568693169,19.451344022838473,0,2023-06-04,NA +WA,B,Yes,7,07528,1,100000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,35.20450218876082,259.1377842843348,14.778420024247932,0,2023-06-04,NA +WA,B,Yes,8,97385,2,50000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,399.7662075827858,13.356542199642641,10.07272661715923,0,2023-06-04,NA +WA,B,Yes,9,14396,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,5.930362983278411,9.301301229524176,11.860725966556823,1,2023-06-04,NA +NM,C,yes,11,91578,6,10000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,2.854972211928456,6.80583155596977,0.27278464095540755,0,2023-06-04,NA +CA,A,YES,2,31809,1,30000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,526.136999955998,15.730854338540544,4.089646781455904,0,2023-06-05,NA +CA,A,YES,4,12078,3,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,6.844054311840791,399.0874489680176,8.578828924279405,0,2023-06-05,NA +CA,A,YES,4,12078,4,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,0.9296375391970718,498.4942102258899,1.8592750783941436,1,2023-06-05,no +WA,B,Yes,6,31928,1,2000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,176.46542197081473,29.774537553392246,22.70633993464678,0,2023-06-05,NA +WA,B,Yes,7,07528,1,100000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,55.09997165158521,435.69652136071284,20.884488594928698,0,2023-06-05,NA +WA,B,Yes,8,97385,2,50000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,20.545348506220172,12.23726359501874,21.294893666235208,0,2023-06-05,NA +WA,B,Yes,9,14396,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,439.28891851672006,24.043649607263855,1.2711770861894394,0,2023-06-05,NA +NM,C,yes,10,27604,5,1000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,0.8875332369367497,25.842034740804994,1.3903810684866273,0,2023-06-05,NA +NM,C,yes,11,91578,6,10000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,1.8385540176452062,182.88487967734133,0.9324301082147635,0,2023-06-05,NA +XX,D,ye,13,65174,10,1000000000,wwtp,raw wastewater,copies/l wastewater,FLUAV,426.5908635212914,706.0485744070917,9.96379440464718,0,2023-06-05,NA +CA,A,YES,2,31809,1,30000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,11.900402320004803,97.29911073758969,2.4005833423474856,0,2023-06-06,result is not quantifiable +CA,A,YES,4,12078,3,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,10.682256464117884,568.6887163026446,21.36451292823577,1,2023-06-06,NA +CA,A,YES,5,34687,3,1000000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,81.85957802207197,83.23412626543427,1.3769070744124532,0,2023-06-06,NA +CA,A,YES,4,12078,4,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,36.4058925791464,103.58390912401481,17.60766003242909,0,2023-06-06,y +CA,A,YES,5,34687,4,1000000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,63.94551785554418,15.142031209569508,9.670893898383353,0,2023-06-06,no +WA,B,Yes,6,31928,1,2000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,10.913569591085322,923.0893848666866,2.805014712206869,0,2023-06-06,NA +WA,B,Yes,7,07528,1,100000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,116.96956128708953,385.00922420219285,2.8307271099282527,0,2023-06-06,NA +WA,B,Yes,8,97385,2,50000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,76.4782921017086,569.897751390256,2.162429472820231,0,2023-06-06,NA +WA,B,Yes,9,14396,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,71.12925953653036,20.11666132138429,17.04042056836225,0,2023-06-06,y +NM,C,yes,10,27604,5,1000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,0.17997388303781894,34.142966604232114,0.48100387870180017,1,2023-06-06,below loq +NM,C,yes,11,91578,6,10000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,1.983422810565965,615.1604056597208,1.0187510479982729,0,2023-06-06,no +XX,D,ye,14,09634,10,1000000000,wwtp,raw wastewater,copies/l wastewater,FLUAV,132.4987004617313,7.457542086002927,3.850330322224466,0,2023-06-06,NA +CA,A,YES,2,31809,1,30000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,13.600956632682392,25.468650715100587,2.1603784156008996,0,2023-06-07,NA +CA,A,YES,4,12078,3,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,254.2680917642392,74.7197956895409,1.580777428402586,0,2023-06-07,NA +CA,A,YES,4,12078,4,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,140.94060637208315,31.41196530123296,2.8915801519667164,0,2023-06-07,y +WA,B,Yes,6,31928,1,2000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,115.14437982438227,259.62731002001436,3.9533099562772613,0,2023-06-07,y +WA,B,Yes,7,07528,1,100000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,18.077506889821134,7.312712445502879,1.4098629475335127,0,2023-06-07,NA +WA,B,Yes,8,97385,2,50000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,6.678251444820117,313.16051846518445,1.380460822205801,0,2023-06-07,temperature not assessed upon arrival at the laboratory +NM,C,yes,10,27604,5,1000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,2.0358023647105266,466.24152666522247,0.6920410070791201,0,2023-06-07,NA +NM,C,yes,11,91578,6,10000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,0.975472920103439,481.0035747455745,0.4159823289141282,0,2023-06-07,NA +YY,E,No,12,32541,1,13,upstream,primary sludge,copies/g dry sludge,sars-cov-2,9.568992150208635,118.15140896883022,2.1581517007215854,0,2023-06-07,no +YY,E,No,14,09634,3,13,upstream,primary sludge,copies/g dry sludge,sars-cov-2,30.926231489878194,26.258052652986226,3.9176038065078678,0,2023-06-07,NA +CA,A,YES,1,04971,1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,11.883770711884882,159.78548776673784,8.191422399765973,0,2023-06-08,NA +CA,A,YES,2,31809,1,30000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,126.89996827381593,482.7721858378786,18.793764830715944,0,2023-06-08,NA +CA,A,YES,4,12078,3,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,943.3173009704533,915.5855832440617,6.131705549868085,0,2023-06-08,NA +CA,A,YES,4,12078,4,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,21.07469011246089,28.415295751450465,1.7074938991039184,0,2023-06-08,NA +WA,B,Yes,6,31928,1,2000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,508.63097888114214,94.68757126886175,2.3541902565484705,0,2023-06-08,no +WA,B,Yes,7,07528,1,100000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,109.6680712298033,863.3328518462861,13.742428389953183,0,2023-06-08,no +WA,B,Yes,8,97385,2,50000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,35.139587424846226,368.58733888552797,15.568194273004142,0,2023-06-08,NA +NM,C,yes,10,27604,5,1000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,0.22234682385734467,269.75788812737926,0.5233768195213259,1,2023-06-08,NA +YY,E,No,12,32541,1,13,upstream,primary sludge,copies/g dry sludge,sars-cov-2,229.46070830973937,126.1243274727436,2.5726370982422115,0,2023-06-08,yes +CA,A,YES,1,04971,1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,3.062617732302472,20.095139119442187,6.125235464604944,1,2023-06-09,no +CA,A,YES,2,31809,1,30000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,14.99482175162784,27.642011895134065,2.124766466611782,0,2023-06-09,NA +CA,A,YES,4,12078,3,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,11.789867095225627,687.8972397954828,2.089113452787408,0,2023-06-09,NA +CA,A,YES,4,12078,4,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,41.91950171517967,14.275640351758339,2.969261759832721,0,2023-06-09,NA +WA,B,Yes,7,07528,1,100000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,17.42630355157437,6.296463518944592,2.5709912899294305,0,2023-06-09,NA +WA,B,Yes,8,97385,2,50000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,44.003281443487644,195.71617623138138,1.8764866213109237,0,2023-06-09,NA +WA,B,Yes,9,14396,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,25.009858255387,381.88514295630955,9.648044176517505,0,2023-06-09,NA +CA,A,YES,1,04971,1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,60.104711478529254,5.608352642439963,7.698766320975435,0,2023-06-10,temperature not assessed upon arrival at the laboratory +CA,A,YES,2,31809,1,30000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,283.72081502729395,61.17307478977554,1.9065543992621838,0,2023-06-10,no +WA,B,Yes,6,31928,1,2000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,18.782601692338034,48.753391987420954,7.565954378759333,0,2023-06-10,NA +WA,B,Yes,7,07528,1,100000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,10.010433900160992,133.7631996077972,8.590093445320287,0,2023-06-10,NA +WA,B,Yes,8,97385,2,50000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,28.89087391037751,272.2018064308127,21.58855170631366,0,2023-06-10,NA +WA,B,Yes,9,14396,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,111.11916460797119,91.86809124335616,2.1479185639774268,0,2023-06-10,NA +NM,C,yes,10,27604,5,1000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,1.5625441357302254,219.64139618292072,0.5399611413393856,0,2023-06-10,NA +CA,A,YES,1,04971,1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,121.00486682545225,41.39264660940628,17.858118605185556,0,2023-06-11,NA +CA,A,YES,2,31809,1,30000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,23.44553278245965,528.998961467369,1.5363655321803267,0,2023-06-11,NA +CA,A,YES,4,12078,3,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,588.5074755501734,76.30126411457383,1.6871129964446332,0,2023-06-11,NA +CA,A,YES,4,12078,4,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,73.67077163216811,83.52714931578771,1.8318110350598291,0,2023-06-11,NA +WA,B,Yes,6,31928,1,2000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,0.6693982665750879,10.292607930286017,1.3387965331501759,1,2023-06-11,temperature not assessed upon arrival at the laboratory +WA,B,Yes,8,97385,2,50000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,42.550114392622625,681.7586675214868,4.648718058870017,0,2023-06-11,NA +WA,B,Yes,9,14396,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,32.76545867332434,13.479016682337873,17.824594600676868,0,2023-06-11,> max temp and/or hold time +NM,C,yes,10,27604,5,1000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,0.8733548539688414,6.675014915544343,0.6171885055370274,0,2023-06-11,NA +CA,A,YES,1,04971,1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,285.1635796114686,104.64656733085897,4.308534565625693,0,2023-06-12,NA +CA,A,YES,2,31809,1,30000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,5.233186119306241,7.618266032282597,23.061502623610284,0,2023-06-12,temperature not assessed upon arrival at the laboratory +CA,A,YES,4,12078,3,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,237.82564783004125,25.348732999615176,22.067398786458455,0,2023-06-12,no +CA,A,YES,4,12078,4,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,1.0898653483344545,310.52225163605294,2.179730696668909,1,2023-06-12,NA +WA,B,Yes,6,31928,1,2000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,389.38191718475406,7.008811392579709,20.113979348349748,0,2023-06-12,NA +WA,B,Yes,7,07528,1,100000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,352.2319245936304,475.19404667448543,2.4129736696837196,0,2023-06-12,no +WA,B,Yes,9,14396,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,729.015437060445,155.60368602802595,19.888816687860707,0,2023-06-12,NA +NM,C,yes,10,27604,5,1000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,0.99030485467691,552.6546764206508,0.7824865050612662,0,2023-06-12,NA +ZZ,F,NO,15,98436,4,8472,wwtp,raw wastewater,copies/g dry sludge,sars-cov-2,355.17335552924874,282.6157002195035,11.989751891524934,0,2023-06-12,> max temp and/or hold time +CA,A,YES,1,04971,1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,327.3311918923494,55.86301140342441,4.117629955751652,0,2023-06-13,NA +CA,A,YES,2,31809,1,30000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,12.645851740232095,12.250317573578705,2.7556767544995036,0,2023-06-13,NA +CA,A,YES,4,12078,3,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,38.577844880152924,208.79175751641623,1.3242995768691284,0,2023-06-13,NA +CA,A,YES,5,34687,3,1000000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,430.54759923790124,155.37376305710086,5.839301650531405,0,2023-06-13,NA +CA,A,YES,4,12078,4,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,24.957791548171297,5.096551697428779,2.3363615092088788,0,2023-06-13,NA +CA,A,YES,5,34687,4,1000000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,224.93943929076912,6.587789139525146,13.85145302819917,0,2023-06-13,y +WA,B,Yes,6,31928,1,2000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,153.78256820392357,706.7575380511578,2.6355720853319444,0,2023-06-13,NA +WA,B,Yes,7,07528,1,100000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,35.56895130202935,869.6243511299224,1.1159277964630634,0,2023-06-13,> max temp and/or hold time +WA,B,Yes,8,97385,2,50000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,11.770452943110248,550.2062120897756,20.65010434655936,0,2023-06-13,no +WA,B,Yes,9,14396,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,2.5011947447358183,254.85062849047918,5.0023894894716365,1,2023-06-13,NA +NM,C,yes,10,27604,5,1000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,1.8631339738384503,315.8740492521134,0.6689151188799155,0,2023-06-13,y +CA,A,YES,1,04971,1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,290.0672745142428,144.404870087013,4.451449957611034,0,2023-06-14,NA +CA,A,YES,2,31809,1,30000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,16.028982312614566,51.34796045413794,3.2534928211240675,0,2023-06-14,NA +CA,A,YES,4,12078,3,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,476.86259349282665,94.87257794986905,2.346696669484884,0,2023-06-14,NA +CA,A,YES,4,12078,4,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,8.073682558792928,617.4310528096794,4.411020559274714,0,2023-06-14,NA +WA,B,Yes,6,31928,1,2000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,6.854654178250884,48.698481971886935,13.507424191121693,0,2023-06-14,NA +WA,B,Yes,7,07528,1,100000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,7.330677080461552,6.5541242649198885,12.575126666509739,0,2023-06-14,result is not quantifiable +WA,B,Yes,8,97385,2,50000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,530.8186158484637,552.5916505571174,10.13650176743089,0,2023-06-14,NA +NM,C,yes,10,27604,5,1000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,1.020238662681467,686.1767397826148,0.8363593834168237,0,2023-06-14,no +CA,A,YES,1,04971,1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,47.08489285130636,610.0219388403381,5.917075486246098,0,2023-06-15,NA +CA,A,YES,4,12078,3,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,128.07236382595138,11.088441552719832,4.786570568124821,0,2023-06-15,NA +CA,A,YES,4,12078,4,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,51.940207929063405,66.61736732068331,1.5319561812298486,0,2023-06-15,NA +WA,B,Yes,7,07528,1,100000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,394.35460880823456,74.83382130364674,1.2552475089365511,0,2023-06-15,NA +WA,B,Yes,8,97385,2,50000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,88.37531813769647,86.5633549910392,1.0227723330132934,0,2023-06-15,NA +WA,B,Yes,9,14396,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,6.118990619360798,12.338876242870473,17.61455195530809,0,2023-06-15,y +NM,C,yes,10,27604,5,1000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,2.0997474128107205,37.09554813270325,1.1295356203660243,0,2023-06-15,result is not quantifiable +ZZ,F,NO,13,65174,2,8472,wwtp,raw wastewater,copies/g dry sludge,sars-cov-2,876.5439157117215,5.4985824773766625,1.1105540378198477,0,2023-06-15,below loq +CA,A,YES,1,04971,1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,96.40070617203892,9.540281193184393,2.460538013144499,0,2023-06-16,NA +CA,A,YES,2,31809,1,30000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,30.745887653535338,40.39990930219769,1.2584581808865403,0,2023-06-16,NA +CA,A,YES,4,12078,3,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,326.1686291754285,9.027917226699811,1.7426155385998634,0,2023-06-16,NA +CA,A,YES,4,12078,4,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,0.5593016709940662,997.3048093491204,1.1186033419881325,1,2023-06-16,no +WA,B,Yes,6,31928,1,2000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,206.42663647470448,113.39032598840141,2.8484698537192648,0,2023-06-16,NA +WA,B,Yes,8,97385,2,50000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,199.39179407633213,18.637236456939593,3.6198657874814373,0,2023-06-16,NA +WA,B,Yes,9,14396,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,67.21571665198742,60.454562200299954,1.253875575772465,0,2023-06-16,NA +NM,C,yes,10,27604,5,1000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,0.5315909549165555,13.432250803008639,0.8326209505805368,1,2023-06-16,NA +CA,A,YES,1,04971,1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,24.243481229112778,10.742240797003861,12.503952415457558,0,2023-06-17,NA +CA,A,YES,2,31809,1,30000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,129.02244789511738,872.8264222783479,1.2937972283220098,0,2023-06-17,NA +CA,A,YES,4,12078,3,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,19.202142281370364,281.7738490632108,18.49878979060863,0,2023-06-17,no +CA,A,YES,4,12078,4,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,267.1772320770139,283.3219094070632,4.7952413049252085,0,2023-06-17,below loq +WA,B,Yes,6,31928,1,2000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,707.2777943480771,20.010889073161017,4.417778919847446,0,2023-06-17,NA +WA,B,Yes,7,07528,1,100000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,249.18297437869177,75.63524890757763,1.0134570427090221,0,2023-06-17,NA +WA,B,Yes,8,97385,2,50000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,12.511607239973387,31.643297365810483,13.890598604129396,0,2023-06-17,NA +WA,B,Yes,9,14396,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,725.2914339140082,389.8173416338911,17.656540386041577,0,2023-06-17,NA +NM,C,yes,10,27604,5,1000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,2.6035949997497414,52.55743023107519,1.3034855888862011,0,2023-06-17,no +XX,D,ye,12,32541,10,1000000000,wwtp,raw wastewater,copies/l wastewater,FLUAV,3.551696321115769,363.81072843299665,7.103392642231538,1,2023-06-17,yes +YY,E,No,12,32541,1,13,upstream,primary sludge,copies/g dry sludge,sars-cov-2,0.8719804376559966,199.258319353223,1.7439608753119933,1,2023-06-17,NA +CA,A,YES,1,04971,1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,493.18207031912567,198.36030538085393,11.390638088057084,0,2023-06-18,no +CA,A,YES,4,12078,3,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,10.239433466885268,186.49099476058652,3.815655671269626,0,2023-06-18,NA +CA,A,YES,4,12078,4,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,762.4383045645158,7.93007959004813,6.55945934270004,0,2023-06-18,NA +WA,B,Yes,6,31928,1,2000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,10.946442971256268,18.84947505608084,19.360184424947917,0,2023-06-18,temperature not assessed upon arrival at the laboratory +WA,B,Yes,7,07528,1,100000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,138.48873849463055,6.210036367710981,23.02270432984408,0,2023-06-18,NA +WA,B,Yes,8,97385,2,50000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,68.18129501708528,636.4360708669876,3.976490040389193,0,2023-06-18,> max temp and/or hold time +WA,B,Yes,9,14396,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,5.092428372926272,16.749984199040675,11.513200876027977,0,2023-06-18,below loq +NM,C,yes,10,27604,5,1000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,0.8132566987145542,283.4539101729675,1.2749129796385201,0,2023-06-18,NA +ZZ,F,NO,13,65174,2,8472,wwtp,raw wastewater,copies/g dry sludge,sars-cov-2,11.919390909306907,18.25160800895212,5.987155710738186,0,2023-06-18,NA +CA,A,YES,2,31809,1,30000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,16.413278186494708,557.0287627462893,11.124221027496533,0,2023-06-19,NA +CA,A,YES,4,12078,4,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,352.1447868959927,205.73655578096108,1.0720537335709446,0,2023-06-19,NA +WA,B,Yes,6,31928,1,2000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,374.0508766582659,90.65592422139603,10.86256029300896,0,2023-06-19,NA +WA,B,Yes,8,97385,2,50000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,24.413282644546193,6.541034095251058,1.5185342188897863,0,2023-06-19,NA +WA,B,Yes,9,14396,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,7.96247128952236,5.1127695060926746,20.7011423623064,0,2023-06-19,NA +NM,C,yes,10,27604,5,1000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,2.114615439274312,301.8907701648655,1.2934485863661191,0,2023-06-19,NA +YY,E,No,15,98436,4,13,upstream,primary sludge,copies/g dry sludge,sars-cov-2,176.20526497799898,118.24237960363114,2.62778584693363,0,2023-06-19,no +CA,A,YES,1,04971,1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,47.75410667127288,79.12621599215444,1.1900373986193729,0,2023-06-20,NA +CA,A,YES,2,31809,1,30000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,9.023033899452097,420.46672596680327,13.114104684698134,0,2023-06-20,no +CA,A,YES,5,34687,3,1000000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,120.9820317206792,19.331675462979547,1.3783793404269271,0,2023-06-20,NA +CA,A,YES,4,12078,4,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,42.81274813058854,615.0390138682561,2.9807606475266275,0,2023-06-20,NA +CA,A,YES,5,34687,4,1000000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,78.30046637852257,77.11992688210815,1.3940077611922677,0,2023-06-20,no +WA,B,Yes,6,31928,1,2000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,359.71523993282176,18.04105087748291,3.420390813494028,0,2023-06-20,NA +WA,B,Yes,7,07528,1,100000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,134.9184555785401,6.015218783136929,18.091983979090053,0,2023-06-20,NA +WA,B,Yes,8,97385,2,50000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,40.58464947505558,16.72207900135216,4.996880888324421,0,2023-06-20,NA +WA,B,Yes,9,14396,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,8.712438388020946,41.57281884427929,2.3936630325035706,0,2023-06-20,NA +NM,C,yes,10,27604,5,1000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,2.7055329675294355,323.8979782096462,1.3370017135812675,0,2023-06-20,y +YY,E,No,15,98436,4,13,upstream,primary sludge,copies/g dry sludge,sars-cov-2,14.230612504706135,8.525170199153212,5.0783112615298664,0,2023-06-20,NA +CA,A,YES,1,04971,1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,195.9009104592535,15.002948692432616,9.991368118457647,0,2023-06-21,no +CA,A,YES,2,31809,1,30000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,1.6322364991686873,51.227410727669735,3.2644729983373746,1,2023-06-21,NA +CA,A,YES,4,12078,3,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,8.693123787825622,10.50273468997773,17.386247575651243,1,2023-06-21,y +CA,A,YES,4,12078,4,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,355.4555689696,14.227900598002725,1.7821634279790772,0,2023-06-21,NA +WA,B,Yes,6,31928,1,2000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,4.736655109835559,429.94574969881927,9.473310219671118,1,2023-06-21,NA +WA,B,Yes,7,07528,1,100000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,76.31153414614397,54.212895716393824,4.264312604579561,0,2023-06-21,NA +WA,B,Yes,8,97385,2,50000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,56.87447362903516,77.8773615882523,2.9558917972820824,0,2023-06-21,NA +WA,B,Yes,9,14396,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,507.67862177300105,43.84437371969059,6.637964823671057,0,2023-06-21,NA +NM,C,yes,10,27604,5,1000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,1.0957946669000724,132.79246838405473,0.24721385261018308,0,2023-06-21,NA +XX,D,ye,13,65174,10,1000000000,wwtp,raw wastewater,copies/l wastewater,FLUAV,535.0612124923401,14.88779989022269,1.0175370486570117,0,2023-06-21,NA +ZZ,F,NO,12,32541,1,8472,wwtp,raw wastewater,copies/g dry sludge,sars-cov-2,92.04823957784674,105.27608813768465,6.354727925909786,0,2023-06-21,result is not quantifiable +CA,A,YES,1,04971,1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,4.0966415673043555,113.83743480845315,8.193283134608711,1,2023-06-22,NA +CA,A,YES,2,31809,1,30000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,755.6838964661431,25.841406253783493,1.0669738415142123,0,2023-06-22,NA +CA,A,YES,4,12078,3,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,5.2735262579433675,5.024803706471183,4.553443879760379,0,2023-06-22,NA +CA,A,YES,4,12078,4,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,7.156672565160005,623.973695366069,22.51487421049501,0,2023-06-22,NA +WA,B,Yes,6,31928,1,2000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,8.992182457854184,14.809806208201698,23.595774142103373,0,2023-06-22,y +WA,B,Yes,7,07528,1,100000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,1.4331672247577871,61.16376930720631,2.8663344495155743,1,2023-06-22,no +WA,B,Yes,9,14396,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,13.017216666448709,13.87179699720965,3.0531750791318997,0,2023-06-22,NA +NM,C,yes,10,27604,5,1000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,1.797251673030202,124.24819547058351,0.06283733342548625,0,2023-06-22,NA +CA,A,YES,4,12078,3,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,5.135728220290362,312.59403384743507,10.271456440580724,1,2023-06-23,result is not quantifiable +CA,A,YES,4,12078,4,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,67.08078916346847,31.16735525249866,2.215895170346125,0,2023-06-23,NA +WA,B,Yes,6,31928,1,2000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,40.26537589282479,5.067980493229841,5.882660757539129,0,2023-06-23,temperature not assessed upon arrival at the laboratory +WA,B,Yes,7,07528,1,100000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,29.834157181516616,6.920032520834492,5.7332918756321956,0,2023-06-23,no +WA,B,Yes,8,97385,2,50000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,0.5687362779708394,281.72745082116097,1.1374725559416787,1,2023-06-23,NA +WA,B,Yes,9,14396,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,6.819582019032745,67.34153167788038,4.6077769154798105,0,2023-06-23,NA +NM,C,yes,10,27604,5,1000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,1.8487678253931676,181.0674901555772,0.5803354062340011,0,2023-06-23,NA +CA,A,YES,1,04971,1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,22.573421285399924,949.051520476134,3.535377539088544,0,2023-06-24,NA +CA,A,YES,2,31809,1,30000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,36.01724900850814,66.45012333656123,4.813987441054439,0,2023-06-24,NA +CA,A,YES,4,12078,3,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,1.3701263498336447,12.995674882375479,2.7402526996672893,1,2023-06-24,no +CA,A,YES,4,12078,4,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,39.0601815465359,146.26379364286205,7.3734661718433445,0,2023-06-24,NA +WA,B,Yes,6,31928,1,2000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,35.41768613729615,19.636905021236487,3.9432958243875627,0,2023-06-24,NA +WA,B,Yes,7,07528,1,100000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,236.90829424349357,587.4427466962778,3.085508730195386,0,2023-06-24,NA +WA,B,Yes,9,14396,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,13.415781230257416,7.213108871712538,7.362647293832722,0,2023-06-24,NA +NM,C,yes,10,27604,5,1000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,2.7502898996938634,328.0745989915488,0.3057106491806088,0,2023-06-24,NA +ZZ,F,NO,14,09634,3,8472,wwtp,raw wastewater,copies/g dry sludge,sars-cov-2,105.80960096490658,11.562744615665789,1.892417700394112,0,2023-06-24,NA +CA,A,YES,2,31809,1,30000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,360.53486840122713,58.04753166732912,24.48676277190495,0,2023-06-25,no +CA,A,YES,4,12078,3,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,57.001978822594914,24.08694105052301,1.0740039329689395,0,2023-06-25,no +CA,A,YES,4,12078,4,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,404.99895139434614,464.8077795841589,1.8381548839451483,0,2023-06-25,NA +WA,B,Yes,7,07528,1,100000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,39.30576416239907,13.877391380207115,3.5712683822510995,0,2023-06-25,NA +WA,B,Yes,8,97385,2,50000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,256.64517687093763,131.22630703182492,2.4270669053691187,0,2023-06-25,no +WA,B,Yes,9,14396,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,23.08971854717595,29.39148566330611,11.739269555410605,0,2023-06-25,NA +NM,C,yes,10,27604,5,1000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,2.0281958725645226,414.6658256512925,0.03742905822700909,0,2023-06-25,NA +YY,E,No,14,09634,3,13,upstream,primary sludge,copies/g dry sludge,sars-cov-2,50.43147030955524,19.246992883249067,1.278092430524568,0,2023-06-25,no +CA,A,YES,1,04971,1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,347.2523830430573,351.4116188477411,1.6635794733977365,0,2023-06-26,NA +CA,A,YES,2,31809,1,30000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,32.93097374839386,264.9210308309461,9.60879112807307,0,2023-06-26,NA +CA,A,YES,4,12078,3,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,120.4089807074322,379.0345638302019,2.073322534112873,0,2023-06-26,NA +CA,A,YES,4,12078,4,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,5.666734590454033,85.1577688336309,18.748850145837707,0,2023-06-26,NA +WA,B,Yes,6,31928,1,2000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,1.5402819170272406,499.0401982824324,3.080563834054481,1,2023-06-26,NA +WA,B,Yes,7,07528,1,100000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,288.1637919185413,105.40843484814033,4.219076069016244,0,2023-06-26,no +WA,B,Yes,8,97385,2,50000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,468.4055980259085,262.3001824338929,2.288263443343838,0,2023-06-26,no +WA,B,Yes,9,14396,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,25.772206663073405,548.9215665578598,23.805388556193233,0,2023-06-26,NA +NM,C,yes,10,27604,5,1000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,1.8623296058461054,34.99450696431354,0.352523027551261,0,2023-06-26,NA +YY,E,No,12,32541,1,13,upstream,primary sludge,copies/g dry sludge,sars-cov-2,343.96101779487975,892.075321397832,1.5274963739633212,0,2023-06-26,NA +YY,E,No,15,98436,4,13,upstream,primary sludge,copies/g dry sludge,sars-cov-2,7.640105529076845,38.40294952200265,1.3191710103020269,0,2023-06-26,NA +CA,A,YES,1,04971,1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,25.858403322348636,28.242645711496458,19.453488209921662,0,2023-06-27,NA +CA,A,YES,2,31809,1,30000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,6.527701927414233,97.41680701824544,1.1779758750978262,0,2023-06-27,NA +CA,A,YES,4,12078,3,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,145.2537352652396,51.51766661928613,14.797983985323617,0,2023-06-27,NA +CA,A,YES,5,34687,3,1000000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,308.7304353844812,726.8952835119471,3.2389757395493923,0,2023-06-27,NA +CA,A,YES,5,34687,4,1000000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,162.67518982289297,52.163829077902925,9.20442696528458,0,2023-06-27,yes +WA,B,Yes,6,31928,1,2000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,6.580445849806655,514.4973499555462,2.632942593288797,0,2023-06-27,NA +WA,B,Yes,7,07528,1,100000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,561.6685264021344,41.938125086921914,10.794632304113174,0,2023-06-27,no +WA,B,Yes,8,97385,2,50000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,860.7507307414419,7.919437326687023,3.0485024200049144,0,2023-06-27,NA +WA,B,Yes,9,14396,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,6.842042023414739,26.058051029784174,6.305415906974324,0,2023-06-27,no +NM,C,yes,10,27604,5,1000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,1.7511935228716389,6.683664832791913,0.22621961001700228,0,2023-06-27,no +YY,E,No,14,09634,3,13,upstream,primary sludge,copies/g dry sludge,sars-cov-2,450.0801336718615,20.431200098282122,10.862714948262134,0,2023-06-27,NA +YY,E,No,15,98436,4,13,upstream,primary sludge,copies/g dry sludge,sars-cov-2,99.08870644987405,25.502120275088153,12.062691408859417,0,2023-06-27,NA +CA,A,YES,1,04971,1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,123.69060013469947,61.01617619804707,8.720083252613676,0,2023-06-28,result is not quantifiable +CA,A,YES,2,31809,1,30000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,419.5831583043475,34.13489458550083,1.2849580658789475,0,2023-06-28,no +CA,A,YES,4,12078,3,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,10.19717305340525,7.199868196918408,7.9395260535548005,0,2023-06-28,result is not quantifiable +WA,B,Yes,6,31928,1,2000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,6.970411553977127,62.109602951048046,13.940823107954254,1,2023-06-28,no +WA,B,Yes,7,07528,1,100000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,1.3181976021646793,505.0195162610059,2.6363952043293586,1,2023-06-28,y +WA,B,Yes,8,97385,2,50000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,10.632058162242386,5.570716655002732,4.146333689829419,0,2023-06-28,NA +WA,B,Yes,9,14396,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,150.57668842337895,477.5711913410926,1.3460807304855096,0,2023-06-28,NA +NM,C,yes,10,27604,5,1000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,0.9033928155809902,161.43382929294324,0.44263177142665533,0,2023-06-28,NA +XX,D,ye,12,32541,10,1000000000,wwtp,raw wastewater,copies/l wastewater,FLUAV,121.1832095034962,264.05871371249475,14.254201931375112,0,2023-06-28,no +ZZ,F,NO,15,98436,4,8472,wwtp,raw wastewater,copies/g dry sludge,sars-cov-2,134.32700543224544,880.1088259873993,5.621333870877718,0,2023-06-28,no +CA,A,YES,1,04971,1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,10.019380902282103,12.758316383889646,3.589310259088568,0,2023-06-29,NA +CA,A,YES,2,31809,1,30000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,91.69756387532672,128.66140050586372,14.70479749208368,0,2023-06-29,NA +CA,A,YES,4,12078,3,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,139.53594836684155,106.25971844335412,10.217041855811175,0,2023-06-29,NA +WA,B,Yes,7,07528,1,100000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,16.819192352482006,6.070556723228853,3.9123650072669274,0,2023-06-29,yes +WA,B,Yes,9,14396,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,21.126614604830475,59.87436626025763,1.4226935223249029,0,2023-06-29,NA +NM,C,yes,10,27604,5,1000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,0.5358503621367183,443.97048479687,0.8368803578006996,1,2023-06-29,NA +CA,A,YES,1,04971,1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,935.1444950109765,12.244724479173952,1.7206828159565872,0,2023-06-30,NA +CA,A,YES,2,31809,1,30000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,47.4629536695786,97.82043438787936,22.995244828101182,0,2023-06-30,NA +CA,A,YES,4,12078,3,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,797.1064905835335,165.06325746313107,1.8848399967633076,0,2023-06-30,no +CA,A,YES,4,12078,4,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,260.0967527466577,8.514264526642851,19.950241007885037,0,2023-06-30,NA +WA,B,Yes,6,31928,1,2000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,59.606529040235486,13.554890260760516,1.0991749590798126,0,2023-06-30,no +WA,B,Yes,7,07528,1,100000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,123.06335772231505,635.4874928373218,1.5822877992657438,0,2023-06-30,NA +WA,B,Yes,9,14396,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,881.6070844290954,46.9554165391308,12.58741764853157,0,2023-06-30,NA +NM,C,yes,10,27604,5,1000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,2.5357034746103144,28.78557982021822,0.9666688324633175,0,2023-06-30,y +XX,D,ye,14,09634,10,1000000000,wwtp,raw wastewater,copies/l wastewater,FLUAV,80.1355922819374,66.67531865379503,3.7774497343840188,0,2023-06-30,no +CA,A,YES,1,04971,1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,73.32340920826366,287.4803390320116,2.0635163304229067,0,2023-07-01,NA +CA,A,YES,2,31809,1,30000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,6.4782058806812906,6.677409850309035,2.7587163215254455,0,2023-07-01,NA +CA,A,YES,4,12078,3,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,394.820425160619,10.737720690994108,10.016948287136834,0,2023-07-01,no +CA,A,YES,4,12078,4,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,26.5855880947208,450.4488909674384,1.6664221402164379,0,2023-07-01,> max temp and/or hold time +WA,B,Yes,6,31928,1,2000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,153.8329133244241,780.1954930519188,7.401264069928704,0,2023-07-01,NA +WA,B,Yes,7,07528,1,100000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,81.47913521782827,273.41766712791224,8.229475996770763,0,2023-07-01,NA +WA,B,Yes,8,97385,2,50000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,37.499458980375174,662.4381225095182,6.641178921768606,0,2023-07-01,NA +NM,C,yes,10,27604,5,1000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,1.167443414751288,27.128075644561925,0.5353775674270493,0,2023-07-01,NA +CA,A,YES,1,04971,1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,0.6849727543487996,523.395297367423,1.3699455086975991,1,2023-07-02,NA +CA,A,YES,2,31809,1,30000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,22.628337118962513,139.740109475956,5.753838485743138,0,2023-07-02,NA +CA,A,YES,4,12078,3,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,201.3080764182664,27.459737280572067,6.269831521960649,0,2023-07-02,NA +CA,A,YES,4,12078,4,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,173.8478495365214,168.39542754887853,10.47424942993786,0,2023-07-02,NA +WA,B,Yes,6,31928,1,2000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,52.894157673521846,68.62193593354726,14.340681581393689,0,2023-07-02,NA +WA,B,Yes,7,07528,1,100000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,620.1195118994862,792.7971470940889,1.2977074229133851,0,2023-07-02,NA +WA,B,Yes,8,97385,2,50000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,77.8722936583329,93.13241511939898,6.983720628634834,0,2023-07-02,NA +WA,B,Yes,9,14396,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,508.24537856214647,53.92116524292217,3.046084398056569,0,2023-07-02,temperature not assessed upon arrival at the laboratory +XX,D,ye,12,32541,10,1000000000,wwtp,raw wastewater,copies/l wastewater,FLUAV,253.33711308398824,27.903078533313465,5.527711625888085,0,2023-07-02,no +YY,E,No,12,32541,1,13,upstream,primary sludge,copies/g dry sludge,sars-cov-2,34.10417962535124,7.970811703101307,22.989497987213454,0,2023-07-02,no +ZZ,F,NO,13,65174,2,8472,wwtp,raw wastewater,copies/g dry sludge,sars-cov-2,47.257333084367744,336.21179188521495,10.599278147601924,0,2023-07-02,NA +CA,A,YES,1,04971,1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,511.61411497329766,62.63682789928954,3.5404402095624605,0,2023-07-03,no +CA,A,YES,2,31809,1,30000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,12.079188456705287,274.53273208614485,5.159820029652694,0,2023-07-03,no +CA,A,YES,4,12078,3,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,8.620905662401379,94.18590327765146,3.245520132421641,0,2023-07-03,no +CA,A,YES,4,12078,4,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,805.0150596209197,231.84544435593202,12.702953864616344,0,2023-07-03,NA +WA,B,Yes,6,31928,1,2000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,12.90274086211937,51.73050385640109,16.77301103168526,0,2023-07-03,NA +WA,B,Yes,8,97385,2,50000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,80.41021519837274,82.3179377927694,12.966163958950181,0,2023-07-03,NA +WA,B,Yes,9,14396,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,0.7254631359668282,134.53923557184794,1.4509262719336564,1,2023-07-03,NA +NM,C,yes,10,27604,5,1000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,1.0942372531148883,5.019826003581729,0.1745549185646807,0,2023-07-03,result is not quantifiable +XX,D,ye,13,65174,10,1000000000,wwtp,raw wastewater,copies/l wastewater,FLUAV,94.70899048057963,16.950965000573742,1.4846722158905687,0,2023-07-03,NA +CA,A,YES,1,04971,1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,541.1218770752313,203.98987755029955,13.794798866774372,0,2023-07-04,NA +CA,A,YES,2,31809,1,30000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,964.80251664486,5.979174028952507,1.7372752734722674,0,2023-07-04,NA +CA,A,YES,4,12078,3,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,2.2464255786250718,156.4078657032445,4.4928511572501435,1,2023-07-04,NA +CA,A,YES,5,34687,3,1000000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,5.764812504115516,16.010407671971542,2.522489568055847,0,2023-07-04,NA +CA,A,YES,4,12078,4,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,38.77522629708733,551.0091137853318,5.177804735926379,0,2023-07-04,NA +CA,A,YES,5,34687,4,1000000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,92.45319518796069,211.00627981764913,1.0630241799353948,0,2023-07-04,y +WA,B,Yes,6,31928,1,2000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,5.71730667710701,128.30161526042383,1.062009612397215,0,2023-07-04,NA +WA,B,Yes,7,07528,1,100000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,9.022295226804387,38.267147500222585,1.096554063900201,0,2023-07-04,NA +WA,B,Yes,8,97385,2,50000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,7.1855617422750395,238.64321994623378,24.948522449367708,0,2023-07-04,no +WA,B,Yes,9,14396,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,872.500768528626,23.209781316233617,3.4714224852702955,0,2023-07-04,NA +NM,C,yes,10,27604,5,1000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,2.205753418835823,729.3038344539345,0.6481232022218308,0,2023-07-04,NA +CA,A,YES,1,04971,1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,24.388026570905993,981.6321747909421,6.177275606147465,0,2023-07-05,NA +CA,A,YES,2,31809,1,30000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,1.467196440896244,44.00637812140559,2.934392881792488,1,2023-07-05,NA +CA,A,YES,4,12078,3,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,128.17503571814547,25.165812619467378,15.17053341652229,0,2023-07-05,NA +CA,A,YES,4,12078,4,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,5.451822114409701,20.205016414519534,10.903644228819402,1,2023-07-05,NA +WA,B,Yes,6,31928,1,2000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,11.73034715944187,84.68716716139197,5.966234129206577,0,2023-07-05,y +WA,B,Yes,7,07528,1,100000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,522.8385479253805,562.2846787794747,2.09709201378588,0,2023-07-05,NA +WA,B,Yes,8,97385,2,50000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,6.4313523920190665,36.943126024186554,12.862704784038133,1,2023-07-05,temperature not assessed upon arrival at the laboratory +WA,B,Yes,9,14396,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,12.864683829500173,22.67078490153635,7.54645635035426,0,2023-07-05,NA +NM,C,yes,10,27604,5,1000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,1.0298742399578338,12.342390581073865,0.8969216590753566,0,2023-07-05,NA +XX,D,ye,12,32541,10,1000000000,wwtp,raw wastewater,copies/l wastewater,FLUAV,73.44500631493693,249.26700291515954,14.974617073455041,0,2023-07-05,NA +XX,D,ye,14,09634,10,1000000000,wwtp,raw wastewater,copies/l wastewater,FLUAV,88.0957200243151,49.35655188234011,1.7431022134926317,0,2023-07-05,NA +CA,A,YES,1,04971,1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,28.11953184261253,99.94044777851639,2.794650281477016,0,2023-07-06,NA +CA,A,YES,2,31809,1,30000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,30.972396985100428,355.1743926214724,16.592770105492104,0,2023-07-06,NA +CA,A,YES,4,12078,3,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,110.88732137150963,46.57913353205974,6.799222658245407,0,2023-07-06,NA +CA,A,YES,4,12078,4,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,10.810892868038199,45.479450095491096,6.350961749916545,0,2023-07-06,NA +WA,B,Yes,6,31928,1,2000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,431.95980409977653,249.1233884171567,2.306193950373202,0,2023-07-06,NA +WA,B,Yes,7,07528,1,100000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,14.325913434796774,574.5090079837199,3.957075095058098,0,2023-07-06,NA +WA,B,Yes,8,97385,2,50000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,5.438830094490895,34.791333425308,6.3323059600616185,0,2023-07-06,no +WA,B,Yes,9,14396,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,563.740963601475,499.7263855499281,1.4060925996297213,0,2023-07-06,no +NM,C,yes,10,27604,5,1000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,1.7706822675849168,265.23716818162796,0.03442370094880236,0,2023-07-06,temperature not assessed upon arrival at the laboratory +XX,D,ye,12,32541,10,1000000000,wwtp,raw wastewater,copies/l wastewater,FLUAV,2.5847369933086686,29.66124820755884,5.169473986617337,1,2023-07-06,NA +XX,D,ye,13,65174,10,1000000000,wwtp,raw wastewater,copies/l wastewater,FLUAV,26.63346976043322,8.67257834087213,23.985556361989577,0,2023-07-06,NA +CA,A,YES,1,04971,1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,625.5325312472202,319.7882667185706,1.9388145516085369,0,2023-07-07,NA +CA,A,YES,2,31809,1,30000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,47.97279393350384,6.991200607959283,1.9455734103948752,0,2023-07-07,NA +CA,A,YES,4,12078,3,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,0.6097012913996386,327.3817841745552,1.2194025827992772,1,2023-07-07,no +CA,A,YES,4,12078,4,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,8.544613002047246,60.00665993245423,2.087722036346646,0,2023-07-07,NA +WA,B,Yes,6,31928,1,2000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,425.50824230123834,6.138754643428855,2.0057416967729966,0,2023-07-07,no +WA,B,Yes,8,97385,2,50000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,408.2131108694158,24.88359197995236,1.310224579040975,0,2023-07-07,NA +NM,C,yes,10,27604,5,1000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,2.14317304353045,48.17837093966867,0.4604364302803231,0,2023-07-07,NA +YY,E,No,12,32541,1,13,upstream,primary sludge,copies/g dry sludge,sars-cov-2,463.82171291832583,252.32382020955413,5.4561223884982555,0,2023-07-07,NA +CA,A,YES,1,04971,1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,12.351905330391771,210.4768920078485,24.703810660783542,1,2023-07-08,NA +CA,A,YES,2,31809,1,30000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,343.79973653813323,9.32316508132063,1.3377158337599355,0,2023-07-08,NA +CA,A,YES,4,12078,3,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,581.463890424258,136.32987898727944,1.114863081095604,0,2023-07-08,NA +CA,A,YES,4,12078,4,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,74.60890573734395,103.23745886051009,20.135359095930284,0,2023-07-08,NA +WA,B,Yes,6,31928,1,2000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,602.7127270151725,6.004293640181676,7.663232666804708,0,2023-07-08,no +WA,B,Yes,7,07528,1,100000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,297.0581854990987,87.00971840561509,3.655792091289433,0,2023-07-08,no +WA,B,Yes,8,97385,2,50000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,35.719003467421444,8.001521621226889,8.229038028298687,0,2023-07-08,no +WA,B,Yes,9,14396,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,1.7600391847403285,57.84305345949273,3.520078369480657,1,2023-07-08,NA +NM,C,yes,10,27604,5,1000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,2.1306073878869287,91.64527014911167,0.7190565489447202,0,2023-07-08,NA +XX,D,ye,12,32541,10,1000000000,wwtp,raw wastewater,copies/l wastewater,FLUAV,348.82188241797206,889.217266262356,5.259852934836894,0,2023-07-08,y +YY,E,No,14,09634,3,13,upstream,primary sludge,copies/g dry sludge,sars-cov-2,64.33551230566621,632.2985114943104,1.0665990601799857,0,2023-07-08,no +CA,A,YES,2,31809,1,30000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,814.6729448971579,336.05459647921606,1.8202442211066545,0,2023-07-09,NA +CA,A,YES,4,12078,3,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,17.803513416152697,36.11234936089526,1.1964883403527566,0,2023-07-09,NA +CA,A,YES,4,12078,4,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,233.7981150339279,70.0817301721835,1.942927603951384,0,2023-07-09,NA +WA,B,Yes,6,31928,1,2000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,612.4352581138744,29.843983187984794,1.2299931978878929,0,2023-07-09,temperature not assessed upon arrival at the laboratory +WA,B,Yes,7,07528,1,100000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,315.6748254322807,134.41602251060337,5.803950062245853,0,2023-07-09,NA +WA,B,Yes,8,97385,2,50000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,123.65065711858405,39.63490926700756,3.9503809820070024,0,2023-07-09,NA +WA,B,Yes,9,14396,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,634.770707659623,35.18521269132205,2.4758387584201498,0,2023-07-09,NA +NM,C,yes,10,27604,5,1000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,1.6042037818604424,12.161913275339137,0.18617333656848944,0,2023-07-09,NA +CA,A,YES,1,04971,1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,2.218836283267302,73.69835255749938,4.437672566534604,1,2023-07-10,NA +CA,A,YES,2,31809,1,30000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,750.4006625631381,10.987394412929502,4.4171387871148005,0,2023-07-10,result is not quantifiable +CA,A,YES,4,12078,3,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,262.8860756367084,53.071899700697266,1.027000195027039,0,2023-07-10,NA +CA,A,YES,4,12078,4,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,358.0377903236937,225.4261899225438,3.9951340659237347,0,2023-07-10,NA +WA,B,Yes,6,31928,1,2000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,45.9915006417231,49.86104211584604,4.123890548280911,0,2023-07-10,> max temp and/or hold time +WA,B,Yes,7,07528,1,100000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,7.6602368357954616,39.80057291750455,15.320473671590923,1,2023-07-10,NA +WA,B,Yes,8,97385,2,50000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,3.418148370054517,509.31244504896495,6.836296740109034,1,2023-07-10,no +WA,B,Yes,9,14396,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,132.73999383105556,300.9897037628022,9.965656080620311,0,2023-07-10,NA +NM,C,yes,10,27604,5,1000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,1.8835014744010268,710.6242742600398,0.25699016613869846,0,2023-07-10,NA +ZZ,F,NO,13,65174,2,8472,wwtp,raw wastewater,copies/g dry sludge,sars-cov-2,186.63211464763253,6.448344206809252,8.678303120380205,0,2023-07-10,no +CA,A,YES,1,04971,1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,7.717464574326779,16.383796256828802,20.529840694664493,0,2023-07-11,NA +CA,A,YES,2,31809,1,30000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,24.00880480813905,70.22797057025903,3.4655121624057426,0,2023-07-11,result is not quantifiable +CA,A,YES,4,12078,3,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,16.739955666498375,312.73707079100194,3.035946335104121,0,2023-07-11,> max temp and/or hold time +CA,A,YES,5,34687,3,1000000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,0.931657731939781,70.43588600329262,1.863315463879562,1,2023-07-11,NA +CA,A,YES,5,34687,4,1000000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,48.51465102658729,803.8174893268102,1.183470875692719,0,2023-07-11,NA +WA,B,Yes,6,31928,1,2000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,5.1713765252420965,58.38233223227598,11.457025196809154,0,2023-07-11,yes +WA,B,Yes,7,07528,1,100000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,6.502173144813194,193.10186158070346,12.309034225780419,0,2023-07-11,no +WA,B,Yes,8,97385,2,50000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,741.01402495434,72.46487149825553,11.48150388753033,0,2023-07-11,NA +WA,B,Yes,9,14396,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,527.1209458543116,872.595872515886,1.374753289006661,0,2023-07-11,NA +NM,C,yes,10,27604,5,1000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,1.6039125356778468,12.188942844163753,1.3120061798142253,0,2023-07-11,no +CA,A,YES,1,04971,1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,7.944992394163731,837.4203115687657,1.5976617777047457,0,2023-07-12,no +CA,A,YES,2,31809,1,30000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,2.353228452672452,987.1544036992674,4.706456905344904,1,2023-07-12,no +CA,A,YES,4,12078,3,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,11.396949210275352,34.44058334128715,7.468245802137186,0,2023-07-12,no +CA,A,YES,4,12078,4,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,13.00250556256381,317.8975971043451,16.314870169073192,0,2023-07-12,NA +WA,B,Yes,7,07528,1,100000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,408.76322610566086,992.0951990148831,12.419737437951905,0,2023-07-12,no +WA,B,Yes,8,97385,2,50000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,19.636444571235103,268.17966791176303,6.588590792565492,0,2023-07-12,no +WA,B,Yes,9,14396,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,455.13644102804267,7.293477578141697,10.528096080942063,0,2023-07-12,NA +NM,C,yes,10,27604,5,1000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,1.6298608661646992,173.86684834859736,0.30464524380989333,0,2023-07-12,NA +CA,A,YES,1,04971,1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,41.51428591660013,81.41641651292139,1.584614391186242,0,2023-07-13,no +CA,A,YES,2,31809,1,30000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,72.54495511424763,230.61234580545207,19.02097722321938,0,2023-07-13,temperature not assessed upon arrival at the laboratory +CA,A,YES,4,12078,3,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,124.43408364890477,230.1286855378592,3.5217228195229016,0,2023-07-13,NA +CA,A,YES,4,12078,4,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,0.7344417610142682,38.882240648787075,1.4688835220285363,1,2023-07-13,no +WA,B,Yes,7,07528,1,100000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,728.9839767856851,389.70662879942984,13.009210467392592,0,2023-07-13,NA +NM,C,yes,10,27604,5,1000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,1.9933259983291647,36.60256515326269,0.29711714902320474,0,2023-07-13,no +XX,D,ye,13,65174,10,1000000000,wwtp,raw wastewater,copies/l wastewater,FLUAV,124.0738871020695,258.9940426039666,19.097747416186202,0,2023-07-13,NA +CA,A,YES,1,04971,1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,0.93826933952755,44.69264572935639,1.8765386790551,1,2023-07-14,no +CA,A,YES,2,31809,1,30000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,55.52731665842245,5.895663003002946,22.51125104773069,0,2023-07-14,NA +CA,A,YES,4,12078,3,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,164.08102342886792,308.02984168192944,2.4975765827769427,0,2023-07-14,NA +CA,A,YES,4,12078,4,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,45.14404398527416,119.93273812538193,4.745574413654393,0,2023-07-14,no +WA,B,Yes,6,31928,1,2000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,8.044557502955271,14.756592287965516,20.99737213635358,0,2023-07-14,NA +WA,B,Yes,7,07528,1,100000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,1.1133885489551105,7.71278967281845,2.226777097910221,1,2023-07-14,NA +WA,B,Yes,8,97385,2,50000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,83.10478438773698,623.0606796022838,17.0193196034174,0,2023-07-14,NA +WA,B,Yes,9,14396,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,22.202168411106786,743.803278964884,5.8296330179847065,0,2023-07-14,y +NM,C,yes,10,27604,5,1000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,2.185250461847421,11.011194730903643,1.247676260823159,0,2023-07-14,NA +CA,A,YES,1,04971,1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,795.655576312077,708.0869856489144,23.358835684662168,0,2023-07-15,NA +CA,A,YES,4,12078,3,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,6.707720784480135,362.72722244038005,1.9658402755819568,0,2023-07-15,y +CA,A,YES,4,12078,4,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,78.5145417591039,19.872184032401016,5.933856526543314,0,2023-07-15,NA +WA,B,Yes,6,31928,1,2000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,48.145194368623656,152.0628129617066,5.441635315770852,0,2023-07-15,NA +WA,B,Yes,7,07528,1,100000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,101.06928551938962,47.10168137117944,2.101542040267802,0,2023-07-15,no +WA,B,Yes,8,97385,2,50000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,69.31315305819089,182.00764014746503,18.15433067907769,0,2023-07-15,NA +WA,B,Yes,9,14396,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,8.142074611877137,5.805846186768318,1.2860421542955445,0,2023-07-15,NA +NM,C,yes,10,27604,5,1000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,1.5815131185445985,65.9818477228319,0.6665179534596019,0,2023-07-15,> max temp and/or hold time +CA,A,YES,1,04971,1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,6.551935780466369,13.333930645682143,13.103871560932738,1,2023-07-16,no +CA,A,YES,2,31809,1,30000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,3.044330667966795,7.32693770316833,6.08866133593359,1,2023-07-16,NA +CA,A,YES,4,12078,3,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,770.7282120328325,56.95659592164818,17.30392034646923,0,2023-07-16,NA +CA,A,YES,4,12078,4,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,95.66168209559994,9.0079802045799,1.9141521256105298,0,2023-07-16,NA +WA,B,Yes,6,31928,1,2000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,17.53093797199904,9.109771037353486,1.412627070427465,0,2023-07-16,no +WA,B,Yes,7,07528,1,100000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,73.75315379886283,302.9004350515071,12.134994101840299,0,2023-07-16,NA +WA,B,Yes,8,97385,2,50000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,6.155250822860765,66.46328090667852,8.734357406664934,0,2023-07-16,no +NM,C,yes,10,27604,5,1000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,1.041509542176899,229.37451077843812,0.9579136299379967,0,2023-07-16,NA +YY,E,No,15,98436,4,13,upstream,primary sludge,copies/g dry sludge,sars-cov-2,8.492050618662649,420.8303385238243,16.984101237325298,1,2023-07-16,NA +CA,A,YES,1,04971,1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,5.882346411121828,39.322479253267844,11.764692822243656,1,2023-07-17,> max temp and/or hold time +CA,A,YES,2,31809,1,30000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,9.033386126282563,108.05455560068502,8.777395398940188,0,2023-07-17,NA +CA,A,YES,4,12078,4,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,211.3941266521302,6.4292960956101455,10.973809774102156,0,2023-07-17,NA +WA,B,Yes,7,07528,1,100000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,173.09189944929298,41.11433051978895,10.159206211965113,0,2023-07-17,NA +WA,B,Yes,8,97385,2,50000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,588.0695150705761,8.633242750827845,3.5355390464383674,0,2023-07-17,NA +WA,B,Yes,9,14396,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,105.21452258280651,10.65218739155434,24.343530859116047,0,2023-07-17,y +NM,C,yes,10,27604,5,1000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,2.5837419662659045,239.43027677959032,0.2760732676271765,0,2023-07-17,NA +XX,D,ye,15,98436,10,1000000000,wwtp,raw wastewater,copies/l wastewater,FLUAV,10.08711490288661,121.55540654601802,20.17422980577322,1,2023-07-17,NA +CA,A,YES,1,04971,1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,241.68723320654166,111.96442647581554,19.774130720183855,0,2023-07-18,NA +CA,A,YES,2,31809,1,30000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,537.9248750681627,822.5197847312589,22.020642895941357,0,2023-07-18,NA +CA,A,YES,4,12078,3,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,16.226026094539854,611.7953876096274,13.514793961665061,0,2023-07-18,temperature not assessed upon arrival at the laboratory +CA,A,YES,5,34687,4,1000000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,36.62244984087406,430.9258004560411,2.759467631177355,0,2023-07-18,no +WA,B,Yes,6,31928,1,2000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,63.22908977775509,436.962517804828,1.009390111624778,0,2023-07-18,NA +WA,B,Yes,7,07528,1,100000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,750.8610767918537,256.8698596470396,1.8428080930087778,0,2023-07-18,NA +WA,B,Yes,8,97385,2,50000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,601.7329980312988,110.95795974497264,1.3633029722287286,0,2023-07-18,no +WA,B,Yes,9,14396,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,1.4767467385567152,234.06219282230043,2.9534934771134305,1,2023-07-18,NA +CA,A,YES,1,04971,1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,43.394035193624006,39.46900061049787,5.129002133042138,0,2023-07-19,NA +CA,A,YES,2,31809,1,30000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,12.144038358853544,130.19728200548795,16.189449004203126,0,2023-07-19,NA +CA,A,YES,4,12078,3,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,65.42278306616743,224.07515066658064,5.575897845450405,0,2023-07-19,NA +CA,A,YES,4,12078,4,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,2.0094458540818056,746.467128002413,4.018891708163611,1,2023-07-19,NA +WA,B,Yes,6,31928,1,2000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,475.0985020274341,25.94211370112956,9.867254826761892,0,2023-07-19,NA +WA,B,Yes,8,97385,2,50000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,10.644001481437686,36.573517141692264,1.1271975957925,0,2023-07-19,no +WA,B,Yes,9,14396,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,14.516678699373639,141.83595631243017,1.5557635594648263,0,2023-07-19,NA +NM,C,yes,10,27604,5,1000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,1.7596387160318898,118.68572725080334,0.6839470365147324,0,2023-07-19,NA +ZZ,F,NO,12,32541,1,8472,wwtp,raw wastewater,copies/g dry sludge,sars-cov-2,19.879466600947246,169.81487827120927,3.6896504606912592,0,2023-07-19,NA +CA,A,YES,2,31809,1,30000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,5.0225912503747265,419.06460655529287,2.617504050003626,0,2023-07-20,NA +CA,A,YES,4,12078,3,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,93.16860177145843,300.7999900496941,5.984573058392557,0,2023-07-20,no +CA,A,YES,4,12078,4,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,42.79508070595576,80.41527350366636,8.058769680438006,0,2023-07-20,NA +WA,B,Yes,6,31928,1,2000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,128.24910665912122,387.5257651979432,4.303074273201593,0,2023-07-20,temperature not assessed upon arrival at the laboratory +WA,B,Yes,7,07528,1,100000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,537.1873457887094,5.304618012230567,18.14733567138234,0,2023-07-20,> max temp and/or hold time +WA,B,Yes,8,97385,2,50000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,119.93842043929429,114.92858920274679,14.032907908478409,0,2023-07-20,NA +WA,B,Yes,9,14396,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,17.18420005425334,249.64294557907263,2.4579698189272334,0,2023-07-20,NA +NM,C,yes,10,27604,5,1000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,2.7543753621461144,7.987570144254859,1.1471414867309968,0,2023-07-20,no +XX,D,ye,12,32541,10,1000000000,wwtp,raw wastewater,copies/l wastewater,FLUAV,70.97089695594413,100.77192193153839,2.0160185123712093,0,2023-07-20,NA +ZZ,F,NO,13,65174,2,8472,wwtp,raw wastewater,copies/g dry sludge,sars-cov-2,472.72050106819825,422.81727722072645,1.1035821955921314,0,2023-07-20,NA +CA,A,YES,1,04971,1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,577.5598577313433,163.07337147206727,24.214569522151354,0,2023-07-21,no +CA,A,YES,2,31809,1,30000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,266.897260359998,223.4530678946011,22.06620546840118,0,2023-07-21,NA +CA,A,YES,4,12078,3,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,405.9456914943519,278.3879406999624,4.534869060789636,0,2023-07-21,NA +CA,A,YES,4,12078,4,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,216.85011118147364,11.271527034164327,5.186482146010251,0,2023-07-21,temperature not assessed upon arrival at the laboratory +WA,B,Yes,6,31928,1,2000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,74.39118804514018,614.1825810621549,1.028601411528418,0,2023-07-21,result is not quantifiable +WA,B,Yes,7,07528,1,100000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,387.3704205231332,464.6320905644908,12.200819044143627,0,2023-07-21,NA +WA,B,Yes,8,97385,2,50000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,8.22510757052565,779.8759706479922,1.7575766448674532,0,2023-07-21,NA +NM,C,yes,10,27604,5,1000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,-0.05119827491285073,26.97305319474855,0.24983172075113044,1,2023-07-21,NA +XX,D,ye,13,65174,10,1000000000,wwtp,raw wastewater,copies/l wastewater,FLUAV,504.32812637403566,5.83155458514248,2.420854667589885,0,2023-07-21,NA +XX,D,ye,15,98436,10,1000000000,wwtp,raw wastewater,copies/l wastewater,FLUAV,563.9541838219717,130.74024183188638,6.379283749680133,0,2023-07-21,no +ZZ,F,NO,15,98436,4,8472,wwtp,raw wastewater,copies/g dry sludge,sars-cov-2,34.423170413279976,284.7916691874159,2.875191943404999,0,2023-07-21,NA +CA,A,YES,1,04971,1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,11.032938404219838,67.90345189717881,7.6546364904309225,0,2023-07-22,NA +CA,A,YES,2,31809,1,30000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,86.06362261142885,25.49200522379626,9.323497288504242,0,2023-07-22,NA +CA,A,YES,4,12078,3,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,4.215090567519388,15.991860059954055,8.430181135038776,1,2023-07-22,temperature not assessed upon arrival at the laboratory +CA,A,YES,4,12078,4,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,1.211994686232142,13.025174350814847,2.423989372464284,1,2023-07-22,NA +WA,B,Yes,6,31928,1,2000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,131.90106228947778,101.83769940905518,9.82648318242834,0,2023-07-22,no +WA,B,Yes,7,07528,1,100000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,92.60776366705345,7.161859873871535,4.347341025077192,0,2023-07-22,NA +WA,B,Yes,8,97385,2,50000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,246.705412559002,38.15815691645869,8.669886521405825,0,2023-07-22,no +WA,B,Yes,9,14396,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,192.08745255336368,6.099214804938713,4.629318364001758,0,2023-07-22,NA +NM,C,yes,10,27604,5,1000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,2.1479632656729564,386.096158413021,0.6962755053899595,0,2023-07-22,NA +XX,D,ye,12,32541,10,1000000000,wwtp,raw wastewater,copies/l wastewater,FLUAV,923.4490724658855,29.578513120980205,4.855949844833705,0,2023-07-22,NA +YY,E,No,14,09634,3,13,upstream,primary sludge,copies/g dry sludge,sars-cov-2,10.798486582741294,757.1323226902997,21.596973165482588,1,2023-07-22,NA +CA,A,YES,2,31809,1,30000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,349.66652388334256,5.973007269086334,3.1100791023289265,0,2023-07-23,no +CA,A,YES,4,12078,3,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,4.8442883977067765,198.30799799749943,9.688576795413553,1,2023-07-23,NA +CA,A,YES,4,12078,4,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,430.2736317767041,6.2208215486945395,13.798099441223014,0,2023-07-23,result is not quantifiable +WA,B,Yes,7,07528,1,100000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,518.9170735925353,854.1811095960346,17.49815503215251,0,2023-07-23,no +WA,B,Yes,9,14396,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,6.886271912337521,84.91545507777077,8.84182279211746,0,2023-07-23,below loq +NM,C,yes,10,27604,5,1000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,1.0908938669681871,5.45394528060175,1.1451872047714888,0,2023-07-23,no +XX,D,ye,15,98436,10,1000000000,wwtp,raw wastewater,copies/l wastewater,FLUAV,230.41056809693274,402.98400962142017,2.838318575520138,0,2023-07-23,NA +CA,A,YES,1,04971,1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,421.5581390603822,50.35463879070723,8.61281044196096,0,2023-07-24,y +CA,A,YES,2,31809,1,30000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,13.931704961820937,33.50447991282662,7.00103565994306,0,2023-07-24,NA +CA,A,YES,4,12078,3,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,70.68597390973369,5.149639862998421,1.1925513163121158,0,2023-07-24,NA +CA,A,YES,4,12078,4,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,12.410294366260937,51.76608303310153,13.589369036717457,0,2023-07-24,NA +WA,B,Yes,6,31928,1,2000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,5.126477751661954,11.940671189758673,2.689051997922301,0,2023-07-24,NA +WA,B,Yes,7,07528,1,100000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,35.444322398416176,822.1245737359479,1.3704801396102488,0,2023-07-24,NA +WA,B,Yes,9,14396,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,19.678224499034048,15.670833550807297,5.566075820590167,0,2023-07-24,NA +NM,C,yes,10,27604,5,1000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,1.2896805336781794,8.997528693520168,1.164342497798732,0,2023-07-24,NA +CA,A,YES,1,04971,1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,23.610997025519655,5.371442831117368,16.501544238849036,0,2023-07-25,NA +CA,A,YES,4,12078,3,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,11.131632377366053,495.84583303313366,8.18280331494476,0,2023-07-25,NA +CA,A,YES,5,34687,3,1000000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,210.61745521349232,45.49156443572035,22.77546631120279,0,2023-07-25,NA +CA,A,YES,4,12078,4,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,35.572668333924135,666.6232906241524,3.173564589181408,0,2023-07-25,NA +CA,A,YES,5,34687,4,1000000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,46.034555134965295,6.9041689710217256,9.083707699348228,0,2023-07-25,NA +WA,B,Yes,6,31928,1,2000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,230.26210868762553,35.81418044984833,1.6929321809334588,0,2023-07-25,NA +WA,B,Yes,7,07528,1,100000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,48.1848695828013,362.5466632522034,11.04271122445443,0,2023-07-25,NA +WA,B,Yes,8,97385,2,50000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,56.84958442586768,17.272331132916392,1.4266905228561024,0,2023-07-25,NA +WA,B,Yes,9,14396,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,19.726375677269136,15.557311260099098,3.981193447017898,0,2023-07-25,NA +NM,C,yes,10,27604,5,1000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,1.7810502996272863,25.67138528128818,0.7728682348177237,0,2023-07-25,NA +YY,E,No,13,65174,2,13,upstream,primary sludge,copies/g dry sludge,sars-cov-2,10.047924691736624,11.555097645033824,12.015321395023786,0,2023-07-25,NA +CA,A,YES,1,04971,1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,7.4285751551321955,19.457111516742547,7.053586036128444,0,2023-07-26,temperature not assessed upon arrival at the laboratory +CA,A,YES,2,31809,1,30000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,2.022878459375225,367.9357253557759,4.04575691875045,1,2023-07-26,NA +CA,A,YES,4,12078,4,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,605.2738581265361,13.228818256721318,2.200025417772529,0,2023-07-26,NA +WA,B,Yes,6,31928,1,2000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,918.4555018609133,47.42556023969875,1.561819227217992,0,2023-07-26,> max temp and/or hold time +WA,B,Yes,7,07528,1,100000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,9.870542766401227,22.99169488312762,8.284825151691335,0,2023-07-26,no +WA,B,Yes,8,97385,2,50000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,45.5517567571074,25.743014750312746,1.2046951089329523,0,2023-07-26,NA +WA,B,Yes,9,14396,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,987.3083082079538,9.290153493600062,1.3437724561565907,0,2023-07-26,no +NM,C,yes,10,27604,5,1000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,2.6981669463675733,579.0400872135936,1.1842199545573608,0,2023-07-26,NA +ZZ,F,NO,13,65174,2,8472,wwtp,raw wastewater,copies/g dry sludge,sars-cov-2,683.1767472521793,959.9423931079921,1.8497728700884701,0,2023-07-26,NA +CA,A,YES,1,04971,1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,5.055534203854524,176.16206933310832,16.59448203988224,0,2023-07-27,> max temp and/or hold time +CA,A,YES,2,31809,1,30000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,18.34817059311526,30.198938215788566,4.084743957726863,0,2023-07-27,no +CA,A,YES,4,12078,3,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,25.393595958767254,42.04473143028779,20.468704754309428,0,2023-07-27,NA +WA,B,Yes,6,31928,1,2000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,10.260974833426713,47.098409991349065,1.238724105245997,0,2023-07-27,NA +WA,B,Yes,7,07528,1,100000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,61.439719294020065,658.5149958042917,13.38389749484834,0,2023-07-27,no +WA,B,Yes,8,97385,2,50000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,69.44356844778974,421.4201693986762,2.3585611383696983,0,2023-07-27,NA +WA,B,Yes,9,14396,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,17.20340881764183,411.1027015651457,1.1114410453881693,0,2023-07-27,no +NM,C,yes,10,27604,5,1000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,1.4660105951254385,29.833140201273768,1.283289265605,0,2023-07-27,no +XX,D,ye,14,09634,10,1000000000,wwtp,raw wastewater,copies/l wastewater,FLUAV,1.1144662421971845,6.3929155639487805,2.228932484394369,1,2023-07-27,NA +YY,E,No,15,98436,4,13,upstream,primary sludge,copies/g dry sludge,sars-cov-2,74.96824753548066,359.24262787918093,2.087121568031942,0,2023-07-27,temperature not assessed upon arrival at the laboratory +CA,A,YES,1,04971,1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,6.757122856022711,482.460205052603,10.015110580420062,0,2023-07-28,NA +CA,A,YES,4,12078,3,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,523.004904845772,77.67660235298708,12.556092808646966,0,2023-07-28,NA +CA,A,YES,4,12078,4,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,11.965686775859371,42.07324912883769,23.931373551718742,1,2023-07-28,NA +WA,B,Yes,6,31928,1,2000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,26.869646565182435,193.4692064575791,9.6319546219826,0,2023-07-28,no +WA,B,Yes,7,07528,1,100000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,43.11517432171161,8.71907065410205,18.625036578631903,0,2023-07-28,temperature not assessed upon arrival at the laboratory +WA,B,Yes,8,97385,2,50000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,780.0696724720883,130.89778247621842,2.7780691144942384,0,2023-07-28,NA +NM,C,yes,10,27604,5,1000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,0.17922483080784876,178.41650913675426,0.48025482647182993,1,2023-07-28,temperature not assessed upon arrival at the laboratory +XX,D,ye,14,09634,10,1000000000,wwtp,raw wastewater,copies/l wastewater,FLUAV,93.87893829512821,335.2062089623892,3.7080859082445232,0,2023-07-28,NA +CA,A,YES,1,04971,1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,18.784296198219966,24.456558953248408,1.2534572274074924,0,2023-07-29,NA +CA,A,YES,2,31809,1,30000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,726.1220317055539,28.831370274962772,1.6912704261075246,0,2023-07-29,NA +CA,A,YES,4,12078,4,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,11.575306369723137,109.52895431551734,15.21304891369186,0,2023-07-29,NA +WA,B,Yes,6,31928,1,2000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,24.388756391739967,26.720527097323632,3.199955392493262,0,2023-07-29,NA +WA,B,Yes,7,07528,1,100000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,481.1296333761557,989.5569134642733,13.032247132356186,0,2023-07-29,NA +WA,B,Yes,8,97385,2,50000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,693.8417724731394,790.0467927326773,1.89720897724016,0,2023-07-29,no +NM,C,yes,10,27604,5,1000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,0.7118998726735573,81.83300294458198,0.5326940922739297,0,2023-07-29,NA +XX,D,ye,12,32541,10,1000000000,wwtp,raw wastewater,copies/l wastewater,FLUAV,8.737336974037659,389.52437870189783,6.544890674740152,0,2023-07-29,NA +YY,E,No,14,09634,3,13,upstream,primary sludge,copies/g dry sludge,sars-cov-2,217.51626595412512,16.637332328320927,11.644643906398457,0,2023-07-29,NA +YY,E,No,15,98436,4,13,upstream,primary sludge,copies/g dry sludge,sars-cov-2,8.391443390627675,21.816466090463543,3.2620521602127983,0,2023-07-29,no +CA,A,YES,1,04971,1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,43.089152719699264,435.73082145092775,3.6013387596174287,0,2023-07-30,NA +CA,A,YES,2,31809,1,30000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,293.07747474889976,640.4133163904908,2.550628153414425,0,2023-07-30,NA +CA,A,YES,4,12078,3,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,112.6451622683913,55.690792692252025,7.422539146040794,0,2023-07-30,NA +WA,B,Yes,6,31928,1,2000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,83.05580532971635,13.75681947578676,3.468712824091568,0,2023-07-30,no +WA,B,Yes,7,07528,1,100000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,35.92465897535307,6.113370963494267,2.0107761649688167,0,2023-07-30,NA +WA,B,Yes,8,97385,2,50000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,181.06509947668224,12.659505068382138,2.518004628254172,0,2023-07-30,NA +WA,B,Yes,9,14396,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,150.53654307805084,325.2854402693327,3.9992681270118213,0,2023-07-30,NA +NM,C,yes,10,27604,5,1000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,0.7054785764378129,22.157444364143263,0.9116653199364343,0,2023-07-30,NA +ZZ,F,NO,14,09634,3,8472,wwtp,raw wastewater,copies/g dry sludge,sars-cov-2,108.36145123957724,86.89442270686818,11.39560045291121,0,2023-07-30,NA +CA,A,YES,4,12078,4,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,246.6032919435724,287.6997057887416,8.944527262332093,0,2023-07-31,NA +WA,B,Yes,6,31928,1,2000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,35.0804527865654,9.815623595834188,4.506925619391673,0,2023-07-31,temperature not assessed upon arrival at the laboratory +WA,B,Yes,7,07528,1,100000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,47.030914535691025,129.80799855966578,1.3235553513674838,0,2023-07-31,NA +WA,B,Yes,8,97385,2,50000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,2.7111607469026726,42.07629323981842,5.422321493805345,1,2023-07-31,NA +WA,B,Yes,9,14396,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,8.5644405503772,716.9936997202635,3.322350533295187,0,2023-07-31,NA +NM,C,yes,10,27604,5,1000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,2.0518639222874664,106.4044310689322,0.9368685615044754,0,2023-07-31,yes +YY,E,No,14,09634,3,13,upstream,primary sludge,copies/g dry sludge,sars-cov-2,3.074861961872334,27.031670850394182,6.149723923744668,1,2023-07-31,result is not quantifiable +CA,A,YES,1,04971,1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,5.982668029827039,71.39805501927842,13.681632332433592,0,2023-08-01,NA +CA,A,YES,2,31809,1,30000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,303.015158055623,339.7455181400534,4.660378211910215,0,2023-08-01,NA +CA,A,YES,4,12078,3,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,16.24632418137527,539.1877904601992,11.755171264652056,0,2023-08-01,no +CA,A,YES,5,34687,3,1000000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,599.3820574248351,273.0288163965346,1.6131250103135448,0,2023-08-01,y +CA,A,YES,4,12078,4,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,118.03992962940161,128.80524355785863,2.1903224377883364,0,2023-08-01,NA +CA,A,YES,5,34687,4,1000000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,10.189133119631624,68.87700797893143,11.297255204351748,0,2023-08-01,no +WA,B,Yes,6,31928,1,2000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,15.592550369843522,16.27795028653531,1.3230353037470421,0,2023-08-01,NA +WA,B,Yes,7,07528,1,100000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,716.1763495974984,20.27189355355651,11.220105425189498,0,2023-08-01,NA +WA,B,Yes,8,97385,2,50000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,85.60667181953598,82.31974912944196,3.3031764400461694,0,2023-08-01,NA +WA,B,Yes,9,14396,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,450.9262821458641,53.7502517972468,1.0729790148397356,0,2023-08-01,NA +NM,C,yes,10,27604,5,1000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,2.36791362753137,44.87393750816078,0.5413630134151237,0,2023-08-01,result is not quantifiable +XX,D,ye,12,32541,10,1000000000,wwtp,raw wastewater,copies/l wastewater,FLUAV,205.38702868053082,399.0007566622886,2.708092710837074,0,2023-08-01,NA +XX,D,ye,15,98436,10,1000000000,wwtp,raw wastewater,copies/l wastewater,FLUAV,43.14571988113972,118.28744838665774,16.9769537344861,0,2023-08-01,NA +CA,A,YES,1,04971,1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,29.89881095401264,644.8783031094963,2.1379496979228128,0,2023-08-02,NA +CA,A,YES,2,31809,1,30000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,8.186167928932948,394.9724924521999,3.707995046001653,0,2023-08-02,> max temp and/or hold time +CA,A,YES,4,12078,4,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,12.170372509019106,318.8893881369137,24.340745018038213,1,2023-08-02,NA +WA,B,Yes,6,31928,1,2000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,61.40776120705407,306.820905000891,6.367177273734145,0,2023-08-02,no +WA,B,Yes,7,07528,1,100000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,48.59629266678792,10.85427701445372,1.8291309007478727,0,2023-08-02,NA +WA,B,Yes,9,14396,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,118.74761913373607,37.40420679540226,10.14468370963547,0,2023-08-02,no +NM,C,yes,10,27604,5,1000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,1.490027103269226,5.067015045749847,0.5489066064132921,0,2023-08-02,NA +YY,E,No,15,98436,4,13,upstream,primary sludge,copies/g dry sludge,sars-cov-2,42.04476991549186,325.9380082818229,2.425107733760121,0,2023-08-02,NA +CA,A,YES,1,04971,1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,15.245052164050056,14.601654948065194,9.130847682327424,0,2023-08-03,NA +CA,A,YES,2,31809,1,30000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,28.279628909864392,16.735452337914275,15.49792768069906,0,2023-08-03,no +CA,A,YES,4,12078,3,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,5.394507993401724,7.318649408462229,2.068053759693033,0,2023-08-03,NA +CA,A,YES,4,12078,4,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,559.611448701473,295.1955902450851,1.7182769404791116,0,2023-08-03,no +WA,B,Yes,6,31928,1,2000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,54.86459826169503,109.74105034032581,19.512330040985805,0,2023-08-03,NA +WA,B,Yes,7,07528,1,100000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,422.30532314703635,896.24649708455,4.842478528490544,0,2023-08-03,NA +WA,B,Yes,8,97385,2,50000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,709.6846424252865,13.64311829474672,8.333672140923492,0,2023-08-03,NA +WA,B,Yes,9,14396,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,19.569510877394052,210.54941598732407,22.693754511360808,0,2023-08-03,no +NM,C,yes,10,27604,5,1000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,2.410808354353311,11.615967931815392,0.755495903351384,0,2023-08-03,no +CA,A,YES,1,04971,1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,445.18973523111055,46.58244883444312,1.69920384070378,0,2023-08-04,NA +CA,A,YES,2,31809,1,30000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,105.68271227819572,5.714044703782743,2.7817956882960715,0,2023-08-04,NA +CA,A,YES,4,12078,3,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,664.3227869998262,16.488534852363777,2.3779864402641393,0,2023-08-04,NA +CA,A,YES,4,12078,4,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,860.136571198302,34.78123378571127,20.158468136558824,0,2023-08-04,no +WA,B,Yes,7,07528,1,100000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,195.5846328836593,101.03447072107053,1.1277045609481247,0,2023-08-04,no +WA,B,Yes,8,97385,2,50000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,642.8127922668028,881.9735410060316,4.554575222819634,0,2023-08-04,NA +WA,B,Yes,9,14396,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,153.88225875925394,845.5560405496877,3.92411194136486,0,2023-08-04,NA +NM,C,yes,10,27604,5,1000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,1.5618679384954453,14.026741047240066,0.9101024937788217,0,2023-08-04,no +ZZ,F,NO,15,98436,4,8472,wwtp,raw wastewater,copies/g dry sludge,sars-cov-2,137.42857743819567,83.27254839131045,12.127263966420896,0,2023-08-04,temperature not assessed upon arrival at the laboratory +CA,A,YES,1,04971,1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,56.64386164261395,8.884665829339795,5.669307914328272,0,2023-08-05,NA +CA,A,YES,2,31809,1,30000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,310.41131633214593,13.900424476349487,4.193320265882613,0,2023-08-05,NA +CA,A,YES,4,12078,3,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,66.73533042657527,201.94219172452438,5.555692745960596,0,2023-08-05,NA +CA,A,YES,4,12078,4,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,99.44260825530226,7.507257005132422,20.78476980672308,0,2023-08-05,NA +WA,B,Yes,6,31928,1,2000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,344.82023192989226,29.91248398658464,11.593194698942327,0,2023-08-05,no +WA,B,Yes,7,07528,1,100000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,43.506825387979426,835.0557327517798,16.20126377502058,0,2023-08-05,NA +WA,B,Yes,8,97385,2,50000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,8.057314815169914,36.3726138374796,21.029956471540032,0,2023-08-05,NA +WA,B,Yes,9,14396,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,0.9458951422454772,84.20342107698912,1.8917902844909544,1,2023-08-05,NA +NM,C,yes,10,27604,5,1000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,0.959469186055834,21.991334951854444,0.1237061645273106,0,2023-08-05,NA +CA,A,YES,2,31809,1,30000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,78.17453789973618,56.090228252421475,2.210850689544784,0,2023-08-06,no +CA,A,YES,4,12078,3,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,6.57066470470062,12.342880249439874,5.013772210792979,0,2023-08-06,temperature not assessed upon arrival at the laboratory +CA,A,YES,4,12078,4,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,208.66887747331415,942.4172132133331,6.930758092021518,0,2023-08-06,NA +WA,B,Yes,6,31928,1,2000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,510.5978868626329,25.740680663632325,5.486129249767782,0,2023-08-06,NA +WA,B,Yes,7,07528,1,100000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,959.8816387141374,35.89664655296101,2.6801398818610838,0,2023-08-06,NA +WA,B,Yes,8,97385,2,50000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,803.058662597642,20.33453061479167,5.669344535538341,0,2023-08-06,no +WA,B,Yes,9,14396,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,26.52573139239787,594.3197774401526,21.212798466869295,0,2023-08-06,NA +NM,C,yes,10,27604,5,1000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,1.5172420872841843,964.3411698335758,0.44376776207618857,0,2023-08-06,NA +XX,D,ye,13,65174,10,1000000000,wwtp,raw wastewater,copies/l wastewater,FLUAV,944.5552973111352,94.59366857818313,15.187647319159383,0,2023-08-06,no +XX,D,ye,14,09634,10,1000000000,wwtp,raw wastewater,copies/l wastewater,FLUAV,21.687447988086134,40.033703572755435,12.236921135810197,0,2023-08-06,NA +XX,D,ye,15,98436,10,1000000000,wwtp,raw wastewater,copies/l wastewater,FLUAV,285.2317977204103,20.3901775074221,8.418250484956513,0,2023-08-06,NA +YY,E,No,12,32541,1,13,upstream,primary sludge,copies/g dry sludge,sars-cov-2,9.76752281538091,146.3428825559688,1.084729442891947,0,2023-08-06,below loq +ZZ,F,NO,13,65174,2,8472,wwtp,raw wastewater,copies/g dry sludge,sars-cov-2,64.89526905834938,914.9629465728397,7.231917260081632,0,2023-08-06,NA +CA,A,YES,1,04971,1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,25.482224251359003,994.4385402519111,2.869294462719715,0,2023-08-07,NA +CA,A,YES,2,31809,1,30000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,2.9146177115473964,76.63341768548509,5.829235423094793,1,2023-08-07,no +CA,A,YES,4,12078,4,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,109.08994189893532,6.875008365746965,23.225017484203427,0,2023-08-07,NA +WA,B,Yes,6,31928,1,2000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,28.868053917653118,169.48296376806599,7.970682783262692,0,2023-08-07,no +WA,B,Yes,7,07528,1,100000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,12.045847462263154,411.04806738167974,24.09169492452631,1,2023-08-07,NA +WA,B,Yes,8,97385,2,50000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,32.100826571581784,157.72216380509755,1.1003112715966439,0,2023-08-07,NA +WA,B,Yes,9,14396,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,929.1921951736591,20.91807099616335,1.4551659518078803,0,2023-08-07,NA +NM,C,yes,10,27604,5,1000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,2.8374080788950238,11.193954612574963,0.19166715569739182,0,2023-08-07,NA +YY,E,No,12,32541,1,13,upstream,primary sludge,copies/g dry sludge,sars-cov-2,9.757533389558919,10.324515391238647,7.695903126280461,0,2023-08-07,NA +CA,A,YES,1,04971,1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,43.06626276789402,6.304198094739177,19.154037997448203,0,2023-08-08,NA +CA,A,YES,2,31809,1,30000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,10.950776866206521,785.9975485294962,19.501502038716477,0,2023-08-08,no +CA,A,YES,4,12078,3,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,20.9229537998017,81.42974638609589,1.4841507364848099,0,2023-08-08,no +CA,A,YES,5,34687,3,1000000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,989.4411817956612,15.57693983588628,3.063510998152033,0,2023-08-08,result is not quantifiable +CA,A,YES,4,12078,4,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,9.53920795675469,8.37315100720299,14.723137279033994,0,2023-08-08,no +CA,A,YES,5,34687,4,1000000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,372.13978502278394,25.62414703813156,1.2460822277499688,0,2023-08-08,NA +WA,B,Yes,6,31928,1,2000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,4.774678912302408,96.25169246591742,9.549357824604815,1,2023-08-08,NA +WA,B,Yes,8,97385,2,50000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,14.599289830479231,158.40317672488013,15.75508105360311,0,2023-08-08,NA +WA,B,Yes,9,14396,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,1.1187305313155609,59.14976110857724,2.2374610626311218,1,2023-08-08,NA +NM,C,yes,10,27604,5,1000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,0.9212160717450832,946.9310894094828,1.1479778273903374,0,2023-08-08,NA +XX,D,ye,14,09634,10,1000000000,wwtp,raw wastewater,copies/l wastewater,FLUAV,287.0561376044087,38.05066844093249,6.667666654777346,0,2023-08-08,NA +ZZ,F,NO,13,65174,2,8472,wwtp,raw wastewater,copies/g dry sludge,sars-cov-2,6.594174621029365,38.31597630192922,6.6516030919398625,0,2023-08-08,NA +CA,A,YES,2,31809,1,30000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,36.58738721505971,24.729216820403384,4.556239281642803,0,2023-08-09,NA +CA,A,YES,4,12078,3,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,23.380158092200155,690.7196674990745,14.695136203854734,0,2023-08-09,> max temp and/or hold time +CA,A,YES,4,12078,4,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,9.430287855700232,98.67861874928543,6.427163845167934,0,2023-08-09,NA +WA,B,Yes,6,31928,1,2000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,2.6004917741878115,12.292218744316562,5.200983548375623,1,2023-08-09,NA +WA,B,Yes,7,07528,1,100000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,67.84342972789176,6.693660043830929,1.1397458933349964,0,2023-08-09,NA +WA,B,Yes,8,97385,2,50000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,26.88238052610876,160.55119957546154,8.814953251156307,0,2023-08-09,no +WA,B,Yes,9,14396,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,1.96668817873219,540.1546018039382,3.93337635746438,1,2023-08-09,NA +NM,C,yes,10,27604,5,1000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,1.1322636796893362,15.247463624841483,0.11320810648986335,0,2023-08-09,NA +CA,A,YES,1,04971,1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,229.18755885572608,18.445161479616353,13.17444100390749,0,2023-08-10,no +CA,A,YES,2,31809,1,30000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,15.101998516798645,6.00087161481564,3.2747091950997804,0,2023-08-10,NA +CA,A,YES,4,12078,3,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,3.5137222122124943,846.7057556611683,7.0274444244249885,1,2023-08-10,temperature not assessed upon arrival at the laboratory +CA,A,YES,4,12078,4,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,57.646969364933994,97.96118059268086,7.085905074338222,0,2023-08-10,NA +WA,B,Yes,6,31928,1,2000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,10.88903118919192,35.72542305982524,1.8863060828753513,0,2023-08-10,no +WA,B,Yes,7,07528,1,100000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,90.87799805152652,327.109327342467,4.729553329890909,0,2023-08-10,NA +WA,B,Yes,8,97385,2,50000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,37.735038538929686,31.13935150498589,4.071972278424945,0,2023-08-10,no +WA,B,Yes,9,14396,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,7.240546236025829,6.497582489305786,4.6943160532879915,0,2023-08-10,NA +NM,C,yes,10,27604,5,1000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,2.028198212721811,155.18893038104613,1.1346719657740452,0,2023-08-10,NA +CA,A,YES,1,04971,1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,17.87839963583158,353.7913947879838,2.386031352430203,0,2023-08-11,NA +CA,A,YES,2,31809,1,30000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,48.07329389170977,749.7926569129905,1.5203552578467876,0,2023-08-11,NA +CA,A,YES,4,12078,3,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,16.531917330400987,52.20011647889214,1.703341145071786,0,2023-08-11,no +CA,A,YES,4,12078,4,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,5.757250894998428,487.92192672993053,3.749671870317186,0,2023-08-11,no +WA,B,Yes,6,31928,1,2000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,19.835092331283192,561.1530211490855,4.020428148735447,0,2023-08-11,NA +WA,B,Yes,7,07528,1,100000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,7.180196118874672,28.894022848243257,5.204695896143978,0,2023-08-11,NA +WA,B,Yes,8,97385,2,50000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,106.49121949244278,383.21558804057565,3.787459713054729,0,2023-08-11,NA +WA,B,Yes,9,14396,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,5.545133079666732,579.6957490084072,15.246338211255718,0,2023-08-11,NA +NM,C,yes,10,27604,5,1000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,2.468921896037719,78.05896889950841,0.7878387428726441,0,2023-08-11,no +YY,E,No,15,98436,4,13,upstream,primary sludge,copies/g dry sludge,sars-cov-2,17.569965551620353,49.94593093295727,19.319493840373116,0,2023-08-11,NA +CA,A,YES,1,04971,1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,341.5319622331677,22.535336369918106,2.9009542439153364,0,2023-08-12,NA +CA,A,YES,2,31809,1,30000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,324.80470135073665,153.7669368849218,1.8141346817987787,0,2023-08-12,NA +CA,A,YES,4,12078,4,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,351.81194547539064,128.75768791860088,1.9097447464442745,0,2023-08-12,NA +WA,B,Yes,6,31928,1,2000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,16.08721384294355,33.30550000784778,1.1171000196953182,0,2023-08-12,NA +WA,B,Yes,7,07528,1,100000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,786.3846287692694,22.30306509379012,2.220453200688894,0,2023-08-12,NA +WA,B,Yes,8,97385,2,50000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,188.40167100371477,381.18199580545524,12.643595698971913,0,2023-08-12,NA +WA,B,Yes,9,14396,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,524.9837750702088,599.3325785458084,4.525253824919206,0,2023-08-12,NA +NM,C,yes,10,27604,5,1000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,1.2403665132550972,75.28911455318081,0.7377640684504787,0,2023-08-12,no +XX,D,ye,14,09634,10,1000000000,wwtp,raw wastewater,copies/l wastewater,FLUAV,2.640379172134827,363.4560857795932,5.280758344269654,1,2023-08-12,yes +CA,A,YES,1,04971,1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,4.671492064147434,70.30665174981034,9.342984128294868,1,2023-08-13,NA +CA,A,YES,2,31809,1,30000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,50.10491001084156,24.763964371380652,14.2125618076529,0,2023-08-13,NA +CA,A,YES,4,12078,3,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,530.1023323808713,460.2171319878419,3.6592007659798154,0,2023-08-13,NA +CA,A,YES,4,12078,4,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,38.516471713458365,67.55807467980863,5.736968972927896,0,2023-08-13,no +WA,B,Yes,6,31928,1,2000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,14.630581861685869,41.78764076589132,1.2672776982555698,0,2023-08-13,NA +WA,B,Yes,7,07528,1,100000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,472.43799713310676,83.95171427551291,1.397969694686829,0,2023-08-13,NA +WA,B,Yes,8,97385,2,50000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,35.26436074013556,22.054417686543484,7.351717062950148,0,2023-08-13,no +WA,B,Yes,9,14396,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,977.6744018211551,919.6661998391734,2.386788491456303,0,2023-08-13,NA +NM,C,yes,10,27604,5,1000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,1.990276264955384,112.93092071747631,1.0255643229326417,0,2023-08-13,NA +YY,E,No,13,65174,2,13,upstream,primary sludge,copies/g dry sludge,sars-cov-2,17.068314747122773,16.59892568786431,21.428646529058966,0,2023-08-13,NA +ZZ,F,NO,15,98436,4,8472,wwtp,raw wastewater,copies/g dry sludge,sars-cov-2,64.05508207354157,38.15548080834952,14.118797546468123,0,2023-08-13,no +CA,A,YES,1,04971,1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,236.9049470981896,10.503899757854246,1.8661087177927753,0,2023-08-14,NA +CA,A,YES,2,31809,1,30000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,21.3593493566267,979.6148827987079,10.199919721876482,0,2023-08-14,no +CA,A,YES,4,12078,3,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,17.227047254752026,245.43931623029175,6.1794061206855275,0,2023-08-14,no +CA,A,YES,4,12078,4,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,401.2753154776294,257.64138835931163,2.218582971236311,0,2023-08-14,NA +WA,B,Yes,6,31928,1,2000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,156.09253659813467,98.52231274648375,13.13109305779592,0,2023-08-14,NA +WA,B,Yes,7,07528,1,100000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,19.62916468048353,72.31301773731008,3.2720063370596124,0,2023-08-14,NA +WA,B,Yes,8,97385,2,50000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,136.8506185149455,8.552031610553357,19.974278660296427,0,2023-08-14,NA +WA,B,Yes,9,14396,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,67.29490785554128,15.730286769423293,3.538489775910998,0,2023-08-14,no +NM,C,yes,10,27604,5,1000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,1.1007844940657217,202.24459760378195,0.7232533071446867,0,2023-08-14,NA +ZZ,F,NO,12,32541,1,8472,wwtp,raw wastewater,copies/g dry sludge,sars-cov-2,19.716650486891098,263.02133599565866,9.883629176505147,0,2023-08-14,NA +CA,A,YES,1,04971,1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,0.6768424203702207,262.5342829784635,1.3536848407404414,1,2023-08-15,no +CA,A,YES,2,31809,1,30000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,11.903723592187118,71.97988230139589,21.330160772389217,0,2023-08-15,NA +CA,A,YES,4,12078,3,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,12.15848195824239,52.80483964548394,9.656226749052312,0,2023-08-15,no +CA,A,YES,5,34687,3,1000000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,119.45368009510078,168.88077412720432,12.38244997942478,0,2023-08-15,NA +CA,A,YES,4,12078,4,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,950.6644184453914,239.23197168978385,3.727102834975412,0,2023-08-15,no +CA,A,YES,5,34687,4,1000000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,6.372422557568877,25.646135875550524,14.21564234186357,0,2023-08-15,NA +WA,B,Yes,6,31928,1,2000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,222.18214999470277,109.96211642101737,16.360318106486528,0,2023-08-15,NA +WA,B,Yes,7,07528,1,100000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,8.59025678522661,758.2537780724726,1.2700870249594154,0,2023-08-15,NA +WA,B,Yes,8,97385,2,50000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,89.27098917744034,57.81610800727803,2.4655668609828405,0,2023-08-15,NA +WA,B,Yes,9,14396,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,15.018014139737593,14.202750307522718,4.863654184703253,0,2023-08-15,NA +CA,A,YES,1,04971,1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,13.750449354745616,97.53703067606361,7.9317837944845575,0,2023-08-16,NA +CA,A,YES,2,31809,1,30000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,71.3996163844299,565.744758290746,15.767492127580145,0,2023-08-16,NA +CA,A,YES,4,12078,3,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,10.431516019095051,5.7190547873987105,18.844077274657444,0,2023-08-16,NA +CA,A,YES,4,12078,4,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,5.42440562301125,32.627026951232544,10.8488112460225,1,2023-08-16,NA +WA,B,Yes,6,31928,1,2000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,31.633310028904923,439.9267264379598,1.3088376038873997,0,2023-08-16,no +WA,B,Yes,7,07528,1,100000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,37.79469499184592,162.90786522763028,6.667442198713768,0,2023-08-16,NA +WA,B,Yes,8,97385,2,50000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,59.884387835785965,34.11242730837711,2.841212316610816,0,2023-08-16,NA +WA,B,Yes,9,14396,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,20.701496885128975,7.802502557104931,2.983584598591913,0,2023-08-16,NA +NM,C,yes,10,27604,5,1000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,1.4281791345123291,7.915264728894823,1.3858148011708302,0,2023-08-16,NA +CA,A,YES,1,04971,1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,4.22681496611314,130.38668828033036,8.45362993222628,1,2023-08-17,NA +CA,A,YES,2,31809,1,30000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,5.406518624233262,569.6226412740097,11.589359910286262,0,2023-08-17,NA +CA,A,YES,4,12078,3,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,3.4106518047656484,577.8763528561315,6.821303609531297,1,2023-08-17,NA +CA,A,YES,4,12078,4,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,57.976680749531056,49.55839987997479,22.10700470814893,0,2023-08-17,NA +WA,B,Yes,6,31928,1,2000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,22.310984551658432,163.81327886295642,19.64512344050914,0,2023-08-17,NA +WA,B,Yes,7,07528,1,100000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,882.1436669917961,24.82003508044034,1.563263128393175,0,2023-08-17,NA +WA,B,Yes,8,97385,2,50000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,151.57077331423298,230.2503656353763,1.74775107264067,0,2023-08-17,NA +WA,B,Yes,9,14396,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,660.4267385583253,7.071336700810739,16.961542916185742,0,2023-08-17,no +NM,C,yes,10,27604,5,1000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,0.8523719515077052,818.705875191483,0.6782085815547221,0,2023-08-17,NA +YY,E,No,13,65174,2,13,upstream,primary sludge,copies/g dry sludge,sars-cov-2,16.097599868811564,769.3673637634419,8.298221463509858,0,2023-08-17,NA +CA,A,YES,1,04971,1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,31.75325288550397,237.1484455656829,13.23501170557511,0,2023-08-18,no +CA,A,YES,2,31809,1,30000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,13.3344648337198,242.17330714372997,16.03532869235851,0,2023-08-18,NA +CA,A,YES,4,12078,3,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,7.399222209591935,6.747956878068289,2.7290298487415083,0,2023-08-18,NA +CA,A,YES,4,12078,4,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,808.6801035515025,980.8235388878362,2.283833971896064,0,2023-08-18,NA +WA,B,Yes,6,31928,1,2000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,87.40143956825595,6.377668604070174,5.66034014045319,0,2023-08-18,below loq +WA,B,Yes,7,07528,1,100000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,966.7982985283151,19.980060576406125,5.904119917944044,0,2023-08-18,NA +WA,B,Yes,8,97385,2,50000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,30.13548661161582,21.809952469339486,5.006006299206098,0,2023-08-18,NA +WA,B,Yes,9,14396,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,457.0438603859718,35.22902172255401,8.762204446756947,0,2023-08-18,no +NM,C,yes,10,27604,5,1000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,0.028732022478877888,5.818962500354877,0.3297620181428591,1,2023-08-18,NA +XX,D,ye,14,09634,10,1000000000,wwtp,raw wastewater,copies/l wastewater,FLUAV,49.662398573555734,222.94174901148332,8.547529478084769,0,2023-08-18,no +YY,E,No,14,09634,3,13,upstream,primary sludge,copies/g dry sludge,sars-cov-2,16.508241018452086,6.162479225695746,1.375328954639339,0,2023-08-18,NA +ZZ,F,NO,12,32541,1,8472,wwtp,raw wastewater,copies/g dry sludge,sars-cov-2,558.3030933494869,475.362394539252,2.297148530288737,0,2023-08-18,y +CA,A,YES,1,04971,1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,667.8359155793845,30.681900301625824,1.9258804785929342,0,2023-08-19,NA +CA,A,YES,2,31809,1,30000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,63.58541652865025,120.15704335483372,14.827121264887648,0,2023-08-19,no +CA,A,YES,4,12078,3,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,27.241652498022177,264.73575909199496,17.579769569662503,0,2023-08-19,no +CA,A,YES,4,12078,4,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,6.719736259162623,12.953134504354272,1.1792683346283608,0,2023-08-19,no +WA,B,Yes,6,31928,1,2000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,1.0593379675658476,813.705354552676,2.1186759351316953,1,2023-08-19,yes +WA,B,Yes,7,07528,1,100000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,19.120611866712046,78.595051478207,12.146643633436451,0,2023-08-19,NA +WA,B,Yes,8,97385,2,50000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,2.40182457660564,153.21715366659382,4.80364915321128,1,2023-08-19,NA +XX,D,ye,15,98436,10,1000000000,wwtp,raw wastewater,copies/l wastewater,FLUAV,1.6980201674614759,51.50402898942396,3.3960403349229518,1,2023-08-19,no +CA,A,YES,1,04971,1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,61.413781917387254,15.875257689664238,24.332944170594313,0,2023-08-20,NA +CA,A,YES,2,31809,1,30000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,261.24330008159126,18.762872233100506,23.476416223878275,0,2023-08-20,NA +CA,A,YES,4,12078,4,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,730.262111129662,146.64304565106204,2.7523899294958123,0,2023-08-20,NA +WA,B,Yes,6,31928,1,2000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,10.114617381374126,16.0718131245347,1.8419619132443998,0,2023-08-20,NA +WA,B,Yes,7,07528,1,100000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,177.65708577575276,466.9996569490327,10.620264168888937,0,2023-08-20,NA +WA,B,Yes,8,97385,2,50000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,203.30125309476637,54.1268738991868,11.82193396810585,0,2023-08-20,NA +WA,B,Yes,9,14396,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,113.4725048930823,132.34930594509993,2.078640711412774,0,2023-08-20,NA +NM,C,yes,10,27604,5,1000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,2.718538864727082,34.6963406384159,0.14745275746331743,0,2023-08-20,NA +XX,D,ye,12,32541,10,1000000000,wwtp,raw wastewater,copies/l wastewater,FLUAV,39.21024658876213,236.595223455924,1.8472185676808772,0,2023-08-20,y +YY,E,No,12,32541,1,13,upstream,primary sludge,copies/g dry sludge,sars-cov-2,477.22430808583096,126.94262569872676,1.5653402497017324,0,2023-08-20,NA +CA,A,YES,1,04971,1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,127.03367781841675,7.099144687453064,20.461066572372655,0,2023-08-21,NA +CA,A,YES,2,31809,1,30000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,153.10238066080672,31.5888895428649,2.2928166379641106,0,2023-08-21,NA +CA,A,YES,4,12078,3,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,506.6042065354576,42.77798115871329,1.1857617089361108,0,2023-08-21,NA +WA,B,Yes,6,31928,1,2000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,775.3004804909477,106.62416714036044,4.847754689652473,0,2023-08-21,NA +WA,B,Yes,7,07528,1,100000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,444.58108974016056,8.130869943607303,4.8602054966002175,0,2023-08-21,NA +WA,B,Yes,9,14396,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,97.28385525975452,61.47506484360955,6.904560367890778,0,2023-08-21,no +NM,C,yes,10,27604,5,1000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,2.593859117695902,7.2245727613919275,0.49412200449614696,0,2023-08-21,> max temp and/or hold time +YY,E,No,15,98436,4,13,upstream,primary sludge,copies/g dry sludge,sars-cov-2,12.729102275281958,22.277210602160903,1.9823849372931954,0,2023-08-21,NA +ZZ,F,NO,14,09634,3,8472,wwtp,raw wastewater,copies/g dry sludge,sars-cov-2,336.89252600768543,359.5597410173625,18.428126422963572,0,2023-08-21,no +CA,A,YES,1,04971,1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,26.30279892188309,21.490421548808033,23.74006180005436,0,2023-08-22,NA +CA,A,YES,2,31809,1,30000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,96.90469534608877,13.530099253387254,1.0783629112301665,0,2023-08-22,NA +CA,A,YES,5,34687,3,1000000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,9.313368716835797,107.25738462999824,1.0127893809128778,0,2023-08-22,NA +CA,A,YES,4,12078,4,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,61.48655860499735,16.23734732692948,23.670124231577127,0,2023-08-22,NA +WA,B,Yes,6,31928,1,2000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,1.0380678540666828,314.2459969197762,2.0761357081333656,1,2023-08-22,no +WA,B,Yes,7,07528,1,100000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,4.7679099273565,189.66367334914315,9.535819854713,1,2023-08-22,NA +WA,B,Yes,8,97385,2,50000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,694.3657937886617,67.787606373973,15.639912021744358,0,2023-08-22,NA +WA,B,Yes,9,14396,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,2.601746510157122,21.55567564517636,5.203493020314244,1,2023-08-22,no +XX,D,ye,15,98436,10,1000000000,wwtp,raw wastewater,copies/l wastewater,FLUAV,4.958352670982755,18.14205767830565,9.91670534196551,1,2023-08-22,NA +CA,A,YES,2,31809,1,30000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,876.4395221526388,26.939708704775256,3.7100845623466077,0,2023-08-23,y +CA,A,YES,4,12078,3,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,20.361030616297313,6.916270070127149,2.855555440853578,0,2023-08-23,NA +CA,A,YES,4,12078,4,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,17.97245854752631,12.408087426670384,8.738132279816174,0,2023-08-23,no +WA,B,Yes,6,31928,1,2000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,8.41818312290283,6.225239998108919,16.83636624580566,1,2023-08-23,no +WA,B,Yes,7,07528,1,100000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,5.981345998226674,52.279558239784066,3.0342093388920257,0,2023-08-23,NA +WA,B,Yes,8,97385,2,50000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,451.3697651479735,698.5585100534294,4.353014993681636,0,2023-08-23,NA +WA,B,Yes,9,14396,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,235.43197042904762,672.7787610861362,1.6790310770891739,0,2023-08-23,NA +NM,C,yes,10,27604,5,1000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,2.556094091726618,12.74952476467688,0.03714508155829687,0,2023-08-23,NA +YY,E,No,12,32541,1,13,upstream,primary sludge,copies/g dry sludge,sars-cov-2,5.388118089893717,47.21748397253332,4.476637487815586,0,2023-08-23,> max temp and/or hold time +ZZ,F,NO,12,32541,1,8472,wwtp,raw wastewater,copies/g dry sludge,sars-cov-2,11.70990818378548,472.0316890953147,23.41981636757096,1,2023-08-23,NA +CA,A,YES,2,31809,1,30000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,405.3573519653681,99.46635932365233,6.812103722285051,0,2023-08-24,NA +CA,A,YES,4,12078,3,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,7.0988512094836755,8.992698057302745,1.3681577085802403,0,2023-08-24,NA +CA,A,YES,4,12078,4,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,27.626178416606862,43.517820155677846,7.473818745518004,0,2023-08-24,no +WA,B,Yes,6,31928,1,2000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,215.32130742097635,98.48018507876697,1.4074279186806538,0,2023-08-24,NA +WA,B,Yes,7,07528,1,100000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,8.299461914433133,45.33196545740755,13.969326073765522,0,2023-08-24,NA +WA,B,Yes,8,97385,2,50000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,45.69862077614117,74.30936811422417,4.995946560220474,0,2023-08-24,NA +WA,B,Yes,9,14396,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,13.892998329286918,64.96191575350063,14.122849852466057,0,2023-08-24,NA +NM,C,yes,10,27604,5,1000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,2.01864325675708,171.58537056139346,0.2649785746663753,0,2023-08-24,NA +CA,A,YES,1,04971,1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,121.05669770858235,14.212953467533138,2.99557871212313,0,2023-08-25,NA +CA,A,YES,2,31809,1,30000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,176.00416510260447,116.1461492707102,21.15923893994668,0,2023-08-25,NA +CA,A,YES,4,12078,3,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,345.96617756740824,443.5680824826337,1.570541261845567,0,2023-08-25,yes +WA,B,Yes,6,31928,1,2000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,14.221303114054285,9.737133857567505,6.212413617219244,0,2023-08-25,NA +WA,B,Yes,8,97385,2,50000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,192.50679622857263,9.23424125731099,1.5071633477790083,0,2023-08-25,no +WA,B,Yes,9,14396,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,1.6700244004547962,17.26167828197841,3.3400488009095923,1,2023-08-25,NA +NM,C,yes,10,27604,5,1000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,0.8631544289053168,85.53949657265971,1.164184424569298,1,2023-08-25,no +XX,D,ye,14,09634,10,1000000000,wwtp,raw wastewater,copies/l wastewater,FLUAV,23.38414843660148,188.51136529168156,19.47944980817053,0,2023-08-25,below loq +CA,A,YES,2,31809,1,30000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,74.11308662726181,941.0306707923485,3.379319730884111,0,2023-08-26,NA +CA,A,YES,4,12078,3,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,32.33710933824061,55.808893286789306,1.4019867722748334,0,2023-08-26,no +CA,A,YES,3,84531,4,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,975.6941069484476,5.016745388676012,17.627670115515627,0,2023-08-26,NA +CA,A,YES,4,12078,4,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,10.256250115338316,64.80848166110793,1.619590887213091,0,2023-08-26,NA +WA,B,Yes,6,31928,1,2000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,845.8237431967651,36.2675735898427,12.205589007152874,0,2023-08-26,no +WA,B,Yes,7,07528,1,100000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,218.02873680868845,31.65188782685804,1.571821774843201,0,2023-08-26,no +WA,B,Yes,8,97385,2,50000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,8.015315375505672,9.131254355281762,16.030630751011344,1,2023-08-26,NA +WA,B,Yes,9,14396,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,396.0138468536225,50.0506805029922,1.7284873328094683,0,2023-08-26,no +NM,C,yes,10,27604,5,1000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,1.0566722286239654,186.0125662937437,1.1541284659034543,0,2023-08-26,no +YY,E,No,13,65174,2,13,upstream,primary sludge,copies/g dry sludge,sars-cov-2,114.57184241427856,64.89039706384061,5.206056297643236,0,2023-08-26,NA +CA,A,YES,1,04971,1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,109.00779424472073,495.6208223070091,2.3091115346315756,0,2023-08-27,no +CA,A,YES,3,84531,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,52.53525037635115,13.858475168623146,4.611076081411231,0,2023-08-27,NA +CA,A,YES,4,12078,3,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,111.47713974227425,416.7923244640025,2.2937814945365353,0,2023-08-27,NA +CA,A,YES,3,84531,4,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,16.944598830067516,148.22510992685,7.931539039475292,0,2023-08-27,NA +CA,A,YES,4,12078,4,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,25.200973680504116,63.19318873784766,8.428723219141508,0,2023-08-27,NA +WA,B,Yes,6,31928,1,2000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,44.45842492554502,77.93539969927019,1.5404348785329505,0,2023-08-27,yes +WA,B,Yes,7,07528,1,100000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,1.9605894207639316,39.990367961450396,3.9211788415278632,1,2023-08-27,y +WA,B,Yes,8,97385,2,50000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,62.84227114376742,346.9948466140951,1.8485091493476775,0,2023-08-27,result is not quantifiable +NM,C,yes,10,27604,5,1000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,1.0946969622274811,337.89928776880095,0.5400903995347485,0,2023-08-27,NA +ZZ,F,NO,14,09634,3,8472,wwtp,raw wastewater,copies/g dry sludge,sars-cov-2,9.850355733987014,15.773989418241909,19.700711467974028,1,2023-08-27,NA +ZZ,F,NO,15,98436,4,8472,wwtp,raw wastewater,copies/g dry sludge,sars-cov-2,123.42292179364728,16.525825984905236,4.022542985114692,0,2023-08-27,NA +CA,A,YES,1,04971,1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,2.6729273348013622,8.752613815741094,5.3458546696027245,1,2023-08-28,NA +CA,A,YES,2,31809,1,30000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,203.91199020954105,28.744704743086213,1.423774384332217,0,2023-08-28,NA +CA,A,YES,3,84531,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,384.5284515039396,277.71588550308985,1.0874034821548364,0,2023-08-28,NA +CA,A,YES,4,12078,3,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,426.982819656184,872.993920152534,6.012191509803236,0,2023-08-28,NA +CA,A,YES,3,84531,4,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,14.336452140069214,31.314873207902604,2.997924964140215,0,2023-08-28,NA +CA,A,YES,4,12078,4,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,630.1843931774775,662.8490451447499,1.280957508684957,0,2023-08-28,NA +WA,B,Yes,6,31928,1,2000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,10.298584093426232,31.368052480149522,4.120651290004484,0,2023-08-28,no +WA,B,Yes,7,07528,1,100000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,258.8401172572234,197.64804211567056,20.449081028908058,0,2023-08-28,NA +WA,B,Yes,8,97385,2,50000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,8.953354059547262,134.94881191364675,8.203429026407477,0,2023-08-28,NA +WA,B,Yes,9,14396,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,360.18623076230966,67.48086806685087,6.652808360147601,0,2023-08-28,NA +NM,C,yes,10,27604,5,1000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,1.08688956747112,246.06615489835366,0.7932305955785415,0,2023-08-28,no +CA,A,YES,1,04971,1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,23.083558937890217,678.3996529565197,2.3853349770620484,0,2023-08-29,NA +CA,A,YES,2,31809,1,30000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,7.514766992235714,365.4853736472118,2.78393316692157,0,2023-08-29,NA +CA,A,YES,3,84531,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,29.911071548006323,671.0571601172005,7.550778524710741,0,2023-08-29,NA +CA,A,YES,4,12078,3,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,287.62680456507144,240.5223863069383,11.174693765916885,0,2023-08-29,no +CA,A,YES,5,34687,3,1000000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,93.36607181853765,69.52235553794377,21.641146780724988,0,2023-08-29,NA +CA,A,YES,3,84531,4,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,36.17741265373531,10.260030206806325,12.119556539763312,0,2023-08-29,NA +CA,A,YES,4,12078,4,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,67.12863878338517,449.76801470563555,3.5436526221978437,0,2023-08-29,NA +CA,A,YES,5,34687,4,1000000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,34.220006382322275,307.5580052927976,1.5959180866470606,0,2023-08-29,NA +WA,B,Yes,7,07528,1,100000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,66.41861926136549,9.9974449364002,18.435118276295444,0,2023-08-29,NA +WA,B,Yes,8,97385,2,50000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,845.5215588850326,197.7977208156787,1.528985483373644,0,2023-08-29,NA +NM,C,yes,10,27604,5,1000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,2.7479105983459386,14.950139886181232,0.6922452514126216,0,2023-08-29,no +YY,E,No,13,65174,2,13,upstream,primary sludge,copies/g dry sludge,sars-cov-2,5.175680501052945,35.08224825142978,10.35136100210589,1,2023-08-29,yes +CA,A,YES,1,04971,1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,53.48263727151335,21.262408576525846,2.8500736421369344,0,2023-08-30,NA +CA,A,YES,2,31809,1,30000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,7.115774757024073,549.9654124672761,14.231549514048146,1,2023-08-30,NA +CA,A,YES,3,84531,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,583.115674339175,64.32264979939596,7.898854515608286,0,2023-08-30,no +CA,A,YES,4,12078,3,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,216.05192486106424,130.15619234505024,9.318865334820485,0,2023-08-30,NA +CA,A,YES,3,84531,4,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,18.45275892439423,154.0771781115166,15.615375342322556,0,2023-08-30,NA +CA,A,YES,4,12078,4,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,64.35571632331445,127.43442788185827,7.219272897400687,0,2023-08-30,NA +WA,B,Yes,6,31928,1,2000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,27.76907611550683,337.7445277783476,11.21568076476297,0,2023-08-30,NA +WA,B,Yes,7,07528,1,100000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,896.0005438302749,318.2377723329214,3.6237583566904767,0,2023-08-30,NA +WA,B,Yes,8,97385,2,50000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,324.0523165943549,571.4191331470736,3.330197241422567,0,2023-08-30,NA +WA,B,Yes,9,14396,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,15.186447683979834,643.1219458801303,2.552462598856978,0,2023-08-30,NA +NM,C,yes,10,27604,5,1000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,2.4885149727634364,12.257056546531048,0.6039381915469029,0,2023-08-30,NA +XX,D,ye,13,65174,10,1000000000,wwtp,raw wastewater,copies/l wastewater,FLUAV,54.37860740584349,80.78522989660938,6.586606119901073,0,2023-08-30,NA +CA,A,YES,1,04971,1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,592.4451254098414,72.18722222591065,12.061499723477331,0,2023-08-31,NA +CA,A,YES,3,84531,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,115.81052665234976,234.4936130121587,1.389452325695656,0,2023-08-31,no +CA,A,YES,4,12078,3,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,140.03564238357285,68.70175771998532,1.761544983720526,0,2023-08-31,NA +CA,A,YES,4,12078,4,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,104.90403678279746,324.6737278147639,10.318658689015285,0,2023-08-31,NA +WA,B,Yes,6,31928,1,2000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,178.0967543419759,8.746321814517836,21.475882770345862,0,2023-08-31,NA +WA,B,Yes,8,97385,2,50000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,112.63872136605683,89.58478175599917,9.018952051607036,0,2023-08-31,NA +WA,B,Yes,9,14396,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,208.42653872700927,26.97337782024099,1.4349171310554374,0,2023-08-31,NA +NM,C,yes,10,27604,5,1000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,0.8989649925942519,5.62785778104256,1.199994988258233,1,2023-08-31,NA +XX,D,ye,12,32541,10,1000000000,wwtp,raw wastewater,copies/l wastewater,FLUAV,318.7838213817547,21.46210551634529,4.3154013851987125,0,2023-08-31,no +ZZ,F,NO,14,09634,3,8472,wwtp,raw wastewater,copies/g dry sludge,sars-cov-2,789.8819770388337,17.47204292255899,1.3043552849508282,0,2023-08-31,no +CA,A,YES,1,04971,1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,748.2893178235532,25.84322755333694,16.723603429453362,0,2023-09-01,yes +CA,A,YES,2,31809,1,30000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,269.4346181883466,187.17828823900814,14.314491656812645,0,2023-09-01,NA +CA,A,YES,3,84531,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,120.83375534704902,20.775236296416725,17.09004881133646,0,2023-09-01,temperature not assessed upon arrival at the laboratory +CA,A,YES,4,12078,3,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,71.77048272933497,9.58952409705518,2.1673294027339844,0,2023-09-01,NA +CA,A,YES,3,84531,4,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,530.5772759408212,106.48438486096293,1.5614452402146148,0,2023-09-01,NA +CA,A,YES,4,12078,4,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,862.2251630128826,655.855303617898,1.209552368544324,0,2023-09-01,NA +WA,B,Yes,6,31928,1,2000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,77.9118734728638,20.784604644517014,2.650876967721488,0,2023-09-01,NA +WA,B,Yes,9,14396,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,12.359823172259661,108.07101199928319,1.1052863364141858,0,2023-09-01,> max temp and/or hold time +CA,A,YES,1,04971,1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,39.57219130798205,769.0657781019385,3.9230363057513404,0,2023-09-02,below loq +CA,A,YES,2,31809,1,30000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,19.75356127430844,182.22910385402272,2.9364508787846253,0,2023-09-02,NA +CA,A,YES,3,84531,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,1.0765963245612948,21.687638369436854,2.1531926491225897,1,2023-09-02,NA +CA,A,YES,4,12078,3,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,264.6435066083946,195.9760707448207,11.728276763349779,0,2023-09-02,no +CA,A,YES,3,84531,4,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,17.538329924755374,229.0001253842372,16.05504167087367,0,2023-09-02,NA +CA,A,YES,4,12078,4,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,168.85975212386782,14.44722582802326,15.994578343141253,0,2023-09-02,NA +WA,B,Yes,6,31928,1,2000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,0.7090021395246358,466.67783751757605,1.4180042790492715,1,2023-09-02,NA +WA,B,Yes,7,07528,1,100000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,34.82511248417436,421.625426504808,4.727934989649429,0,2023-09-02,NA +WA,B,Yes,8,97385,2,50000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,7.203000541719041,5.122262438995919,2.262339722867181,0,2023-09-02,NA +WA,B,Yes,9,14396,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,62.95545178985512,5.016908364825607,2.345633515413886,0,2023-09-02,no +NM,C,yes,10,27604,5,1000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,0.009290543402472325,997.0536203126686,0.3103205390664535,1,2023-09-02,below loq +YY,E,No,14,09634,3,13,upstream,primary sludge,copies/g dry sludge,sars-cov-2,40.12271986487103,835.0193306826875,10.74666524657234,0,2023-09-02,no +ZZ,F,NO,13,65174,2,8472,wwtp,raw wastewater,copies/g dry sludge,sars-cov-2,56.79199712220917,11.726210229367643,6.224802988094461,0,2023-09-02,NA +CA,A,YES,1,04971,1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,430.2273711266606,49.10700461580872,12.024286675454327,0,2023-09-03,NA +CA,A,YES,2,31809,1,30000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,3.140423453540527,44.57699799251904,6.280846907081054,1,2023-09-03,NA +CA,A,YES,3,84531,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,109.33707641767815,563.6346962634074,6.1676072248482035,0,2023-09-03,below loq +CA,A,YES,4,12078,3,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,51.416813206666305,9.741599462978815,4.066622644953145,0,2023-09-03,NA +CA,A,YES,3,84531,4,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,2.234278943036694,578.2961630367103,4.468557886073388,1,2023-09-03,no +WA,B,Yes,6,31928,1,2000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,12.067327231705665,676.03113157472,24.13465446341133,1,2023-09-03,NA +WA,B,Yes,7,07528,1,100000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,379.19622751304973,84.15017609215634,2.0665921063140757,0,2023-09-03,NA +WA,B,Yes,8,97385,2,50000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,178.979628490837,17.205740224938236,4.814397123431832,0,2023-09-03,NA +WA,B,Yes,9,14396,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,71.29831297685892,67.45691454103022,5.885020570288048,0,2023-09-03,NA +YY,E,No,14,09634,3,13,upstream,primary sludge,copies/g dry sludge,sars-cov-2,380.4419399247757,92.48674990569818,7.9273681605146065,0,2023-09-03,NA +ZZ,F,NO,14,09634,3,8472,wwtp,raw wastewater,copies/g dry sludge,sars-cov-2,11.7316754984645,295.2547373576655,4.873570575688488,0,2023-09-03,NA +CA,A,YES,1,04971,1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,786.0080000784335,106.71467557046878,12.403173656961675,0,2023-09-04,no +CA,A,YES,2,31809,1,30000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,146.40757393837086,5.6021890882388465,4.731356803824381,0,2023-09-04,NA +CA,A,YES,4,12078,3,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,11.014299017908412,9.122926837628256,2.4645400936270576,0,2023-09-04,no +CA,A,YES,4,12078,4,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,30.966555856995438,37.6860124951643,3.3637788302328704,0,2023-09-04,no +WA,B,Yes,6,31928,1,2000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,5.43166046424192,348.5493210623553,1.2958325769843038,0,2023-09-04,NA +WA,B,Yes,7,07528,1,100000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,211.3252856797089,35.33902512666274,4.1970217098910485,0,2023-09-04,NA +WA,B,Yes,9,14396,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,244.3869152712867,19.628533387980333,1.6008559540872358,0,2023-09-04,NA +NM,C,yes,10,27604,5,1000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,0.8906333929852905,6.051817816131931,0.5161209999542375,0,2023-09-04,NA +XX,D,ye,14,09634,10,1000000000,wwtp,raw wastewater,copies/l wastewater,FLUAV,374.79682338073434,693.8023194740917,2.9144725202571315,0,2023-09-04,NA +YY,E,No,13,65174,2,13,upstream,primary sludge,copies/g dry sludge,sars-cov-2,292.051378120117,719.7515128324092,21.29413260222399,0,2023-09-04,NA +CA,A,YES,1,04971,1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,8.213217766641202,62.75875762506459,16.426435533282405,1,2023-09-05,NA +CA,A,YES,2,31809,1,30000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,12.391538239002914,121.77325451487717,1.0772511681253278,0,2023-09-05,NA +CA,A,YES,3,84531,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,45.213714472791736,29.49455978609377,2.3769870936228963,0,2023-09-05,NA +CA,A,YES,5,34687,3,1000000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,19.74685852404313,811.4006926523665,24.51123500577057,0,2023-09-05,NA +CA,A,YES,3,84531,4,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,20.132484745424044,34.734526652325584,5.530921858339393,0,2023-09-05,result is not quantifiable +CA,A,YES,4,12078,4,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,42.00130694761647,6.115210103860079,5.345578394822269,0,2023-09-05,NA +CA,A,YES,5,34687,4,1000000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,55.40531249209961,447.60839905834854,6.831921979279036,0,2023-09-05,NA +WA,B,Yes,6,31928,1,2000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,108.54820822221978,18.104253284754552,5.1322936139140785,0,2023-09-05,NA +WA,B,Yes,7,07528,1,100000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,18.18375572481969,86.96818269882104,2.5327142830989757,0,2023-09-05,> max temp and/or hold time +WA,B,Yes,8,97385,2,50000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,569.4689515288138,475.32633449439624,1.9613036850390617,0,2023-09-05,NA +WA,B,Yes,9,14396,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,98.17086940614291,22.58809023680244,4.473339776754265,0,2023-09-05,no +CA,A,YES,1,04971,1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,569.0592734875645,49.8948087609619,16.741583747541483,0,2023-09-06,NA +CA,A,YES,2,31809,1,30000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,260.2456716058832,54.0904670429179,7.1907003070173685,0,2023-09-06,> max temp and/or hold time +CA,A,YES,3,84531,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,888.7942817773062,21.450426847575212,4.291430262637199,0,2023-09-06,no +CA,A,YES,4,12078,3,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,195.87529438836057,50.38792491907603,3.973626153895206,0,2023-09-06,NA +CA,A,YES,3,84531,4,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,23.796201520626628,89.71022155615749,4.668382649811058,0,2023-09-06,NA +CA,A,YES,4,12078,4,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,6.4627814160698405,609.2429086366412,7.348901968435019,0,2023-09-06,NA +WA,B,Yes,6,31928,1,2000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,9.01400003633199,420.27317384189496,6.673366975994265,0,2023-09-06,NA +WA,B,Yes,7,07528,1,100000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,0.7242794500365215,664.4173133824515,1.448558900073043,1,2023-09-06,NA +WA,B,Yes,8,97385,2,50000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,5.203548845711328,5.799512527492022,10.407097691422656,1,2023-09-06,NA +WA,B,Yes,9,14396,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,26.273477365195667,41.882609669572645,2.7014730609439828,0,2023-09-06,result is not quantifiable +XX,D,ye,12,32541,10,1000000000,wwtp,raw wastewater,copies/l wastewater,FLUAV,6.071206051002803,417.4206551875899,1.043185977695066,0,2023-09-06,no +XX,D,ye,15,98436,10,1000000000,wwtp,raw wastewater,copies/l wastewater,FLUAV,19.280456726615277,501.27607876736533,8.179115555666378,0,2023-09-06,below loq +ZZ,F,NO,13,65174,2,8472,wwtp,raw wastewater,copies/g dry sludge,sars-cov-2,28.01947973423674,33.88286641999909,16.875780061729348,0,2023-09-06,no +ZZ,F,NO,14,09634,3,8472,wwtp,raw wastewater,copies/g dry sludge,sars-cov-2,39.84675651832644,604.4916737134605,4.616793458565574,0,2023-09-06,NA +CA,A,YES,1,04971,1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,125.16062540964309,15.744586170221984,3.413231366066114,0,2023-09-07,NA +CA,A,YES,2,31809,1,30000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,2.03574872068184,941.8539891612145,4.07149744136368,1,2023-09-07,temperature not assessed upon arrival at the laboratory +CA,A,YES,3,84531,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,268.2615245913104,11.464638631278572,3.3510304233960753,0,2023-09-07,no +CA,A,YES,4,12078,3,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,670.4594334513204,91.73270335995056,1.7831830123125219,0,2023-09-07,no +CA,A,YES,3,84531,4,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,95.31807342833757,52.20110488126039,5.621490155721226,0,2023-09-07,y +CA,A,YES,4,12078,4,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,5.526282550111774,69.0991353428843,11.052565100223548,1,2023-09-07,NA +WA,B,Yes,6,31928,1,2000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,68.97239780491303,38.01974089489456,9.404688748552074,0,2023-09-07,NA +WA,B,Yes,7,07528,1,100000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,525.757420380718,997.2677562889236,5.399495360825986,0,2023-09-07,NA +WA,B,Yes,8,97385,2,50000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,5.8520079987222395,295.14565187298376,7.12828927066237,0,2023-09-07,NA +WA,B,Yes,9,14396,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,57.11744186210839,16.259692902134955,1.0041871900405717,0,2023-09-07,y +NM,C,yes,10,27604,5,1000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,2.4393194142310732,5.64545916019006,0.41718335600365775,0,2023-09-07,NA +XX,D,ye,14,09634,10,1000000000,wwtp,raw wastewater,copies/l wastewater,FLUAV,19.560058059797647,182.15648687225786,3.1323110868458732,0,2023-09-07,yes +XX,D,ye,15,98436,10,1000000000,wwtp,raw wastewater,copies/l wastewater,FLUAV,6.047399563411604,85.99863414855055,10.167875860172252,0,2023-09-07,NA +ZZ,F,NO,12,32541,1,8472,wwtp,raw wastewater,copies/g dry sludge,sars-cov-2,336.4017773640832,8.962216251836132,1.834938407348662,0,2023-09-07,NA +CA,A,YES,1,04971,1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,13.416099669710217,112.80358085277192,4.2191715782747234,0,2023-09-08,y +CA,A,YES,2,31809,1,30000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,12.35470483115196,260.6789416900791,24.70940966230392,1,2023-09-08,no +CA,A,YES,3,84531,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,685.1373497699898,34.079146981953514,18.355239343220926,0,2023-09-08,NA +CA,A,YES,4,12078,3,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,23.28879422835718,20.357919162156755,1.1887886442646176,0,2023-09-08,NA +CA,A,YES,3,84531,4,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,210.6720860609845,109.20076252950167,1.1024012200773223,0,2023-09-08,NA +CA,A,YES,4,12078,4,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,30.554570994273394,146.18335402985502,3.6842514608887416,0,2023-09-08,result is not quantifiable +WA,B,Yes,6,31928,1,2000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,55.05569222631711,68.55062944685439,2.5837262487304695,0,2023-09-08,NA +WA,B,Yes,7,07528,1,100000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,5.443465445200407,991.9650537980673,9.332403447386636,0,2023-09-08,NA +WA,B,Yes,8,97385,2,50000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,60.114494331533876,7.01186131764175,1.060183405857427,0,2023-09-08,NA +WA,B,Yes,9,14396,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,11.771903218511845,24.65795255464619,13.16691020448697,0,2023-09-08,NA +NM,C,yes,10,27604,5,1000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,0.9168491753768885,776.2576667378628,0.46153892772471683,0,2023-09-08,NA +YY,E,No,14,09634,3,13,upstream,primary sludge,copies/g dry sludge,sars-cov-2,484.2895489686721,17.134194602366936,2.7814060117765926,0,2023-09-08,NA +CA,A,YES,1,04971,1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,527.5990713281764,502.8502582656528,8.387939118671651,0,2023-09-09,NA +CA,A,YES,2,31809,1,30000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,78.47346448457697,89.04542972050675,3.7618940732239037,0,2023-09-09,no +CA,A,YES,3,84531,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,5.928170347534162,9.16972576503793,11.856340695068324,1,2023-09-09,NA +CA,A,YES,4,12078,3,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,106.53281947212461,7.115932066081365,1.472358546638881,0,2023-09-09,no +CA,A,YES,3,84531,4,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,1.0080902495561197,23.23748232553108,2.0161804991122394,1,2023-09-09,NA +CA,A,YES,4,12078,4,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,11.747780514809891,9.215766584683582,3.346266999403597,0,2023-09-09,NA +WA,B,Yes,6,31928,1,2000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,14.347383353273045,503.1798288563729,3.000230068352586,0,2023-09-09,NA +WA,B,Yes,7,07528,1,100000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,175.41751997072893,229.01184139917856,1.122279380004004,0,2023-09-09,below loq +WA,B,Yes,8,97385,2,50000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,18.52221641500781,447.30562441519214,1.9032431604930327,0,2023-09-09,NA +NM,C,yes,10,27604,5,1000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,0.8986360403293976,154.9251124891775,0.14255255895718694,0,2023-09-09,NA +XX,D,ye,13,65174,10,1000000000,wwtp,raw wastewater,copies/l wastewater,FLUAV,230.04933288748234,45.38393518191321,3.5775430938506894,0,2023-09-09,no +ZZ,F,NO,13,65174,2,8472,wwtp,raw wastewater,copies/g dry sludge,sars-cov-2,79.43208651713678,145.94909290148374,2.196471015585315,0,2023-09-09,no +CA,A,YES,1,04971,1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,12.521220019300424,20.76965227783997,5.3834170132267305,0,2023-09-10,NA +CA,A,YES,2,31809,1,30000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,17.58715101097612,12.83397982530517,9.024293572135813,0,2023-09-10,NA +CA,A,YES,3,84531,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,512.1009244327516,801.131424450119,1.152934282696263,0,2023-09-10,NA +CA,A,YES,4,12078,3,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,120.70002154247815,390.0095950052722,2.9393046329562877,0,2023-09-10,NA +CA,A,YES,3,84531,4,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,33.09745859979751,88.0984675005074,19.30350545190957,0,2023-09-10,yes +CA,A,YES,4,12078,4,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,153.78075555645515,70.46670636007109,2.0801735944517024,0,2023-09-10,result is not quantifiable +WA,B,Yes,6,31928,1,2000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,50.38679202041178,10.440269191841129,3.6209161243520365,0,2023-09-10,NA +WA,B,Yes,8,97385,2,50000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,19.441119082411728,27.353014976951812,3.976032523776446,0,2023-09-10,NA +WA,B,Yes,9,14396,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,50.73848803694325,355.9063885472663,1.9205736310791068,0,2023-09-10,yes +NM,C,yes,10,27604,5,1000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,0.44005475663120985,13.866496675230948,0.741084752295191,1,2023-09-10,NA +CA,A,YES,1,04971,1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,56.9534624514444,366.1366633850493,1.292548620565424,0,2023-09-11,NA +CA,A,YES,2,31809,1,30000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,5.0966126392390345,240.093309426411,4.37020913372028,0,2023-09-11,NA +CA,A,YES,3,84531,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,99.86720751992983,10.9075131215385,6.246847884499809,0,2023-09-11,NA +CA,A,YES,3,84531,4,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,115.00319257173182,104.84945091739647,5.914995444575997,0,2023-09-11,no +WA,B,Yes,6,31928,1,2000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,7.162957885677333,29.57055122661636,1.8165351074426492,0,2023-09-11,NA +WA,B,Yes,7,07528,1,100000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,1.4110832410828409,380.3485070548362,2.8221664821656818,1,2023-09-11,NA +WA,B,Yes,8,97385,2,50000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,7.327125663023487,318.9832365431544,24.41155368659637,0,2023-09-11,NA +NM,C,yes,10,27604,5,1000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,1.7035260050787888,142.01179949024075,0.7113749578743687,0,2023-09-11,NA +XX,D,ye,15,98436,10,1000000000,wwtp,raw wastewater,copies/l wastewater,FLUAV,40.454228923757924,9.341962171841942,5.311132323665777,0,2023-09-11,temperature not assessed upon arrival at the laboratory +ZZ,F,NO,14,09634,3,8472,wwtp,raw wastewater,copies/g dry sludge,sars-cov-2,6.743804895608624,105.53979075726559,2.8508711573320333,0,2023-09-11,NA +CA,A,YES,1,04971,1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,901.0494424708787,461.91170796163334,8.579360686641467,0,2023-09-12,NA +CA,A,YES,2,31809,1,30000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,65.27244626668926,506.05965221931496,23.06300976042336,0,2023-09-12,NA +CA,A,YES,3,84531,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,696.5337997332985,288.76589833544875,19.78523736584671,0,2023-09-12,NA +CA,A,YES,4,12078,3,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,22.93825568238866,205.6113269404251,4.355115879356507,0,2023-09-12,NA +CA,A,YES,5,34687,3,1000000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,22.044383332327712,104.59155185353832,2.668449102560198,0,2023-09-12,NA +CA,A,YES,3,84531,4,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,1.3137728200640493,25.97814175965599,2.6275456401280985,1,2023-09-12,no +CA,A,YES,4,12078,4,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,5.543546049553814,17.096054317289024,8.226210306017569,0,2023-09-12,NA +CA,A,YES,5,34687,4,1000000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,138.91535050178177,82.86451161329475,9.133905510977868,0,2023-09-12,yes +WA,B,Yes,6,31928,1,2000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,12.226368729829534,521.6183575741151,18.049391356612787,0,2023-09-12,result is not quantifiable +WA,B,Yes,9,14396,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,29.620091476208373,25.210228281526383,13.686401983024062,0,2023-09-12,NA +NM,C,yes,10,27604,5,1000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,1.8766720446620844,726.2311281456256,0.2558777167177798,0,2023-09-12,NA +XX,D,ye,12,32541,10,1000000000,wwtp,raw wastewater,copies/l wastewater,FLUAV,841.8049511459294,7.685428614397752,2.4650724544135625,0,2023-09-12,no +XX,D,ye,14,09634,10,1000000000,wwtp,raw wastewater,copies/l wastewater,FLUAV,502.3880020093616,52.57992313831526,17.005828583693976,0,2023-09-12,no +CA,A,YES,1,04971,1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,603.0385591829224,135.6580883620457,4.139170907451966,0,2023-09-13,NA +CA,A,YES,2,31809,1,30000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,22.237577152775437,401.9403918677182,19.787889372051975,0,2023-09-13,yes +CA,A,YES,4,12078,3,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,31.011091214856222,966.1418207121662,13.445781201228822,0,2023-09-13,NA +CA,A,YES,3,84531,4,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,176.56799228854456,9.856970493457414,8.697769771555773,0,2023-09-13,NA +CA,A,YES,4,12078,4,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,136.22918111097988,12.824333782212003,23.3899953225163,0,2023-09-13,NA +WA,B,Yes,6,31928,1,2000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,9.52250302662332,12.309747495464435,3.1258157129247337,0,2023-09-13,NA +WA,B,Yes,7,07528,1,100000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,317.06857174613333,28.937258226467367,3.587994701609066,0,2023-09-13,NA +WA,B,Yes,8,97385,2,50000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,150.2207063323918,825.2697517544991,16.179023426700656,0,2023-09-13,NA +WA,B,Yes,9,14396,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,293.32533006557884,326.7757230297949,13.145887919698414,0,2023-09-13,temperature not assessed upon arrival at the laboratory +NM,C,yes,10,27604,5,1000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,2.206287749928249,12.678598244318016,0.5533204739985695,0,2023-09-13,no +ZZ,F,NO,12,32541,1,8472,wwtp,raw wastewater,copies/g dry sludge,sars-cov-2,6.759775049434626,13.40975673312541,2.4916769038418742,0,2023-09-13,NA +CA,A,YES,1,04971,1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,2.7178157495953896,594.1185475832327,5.435631499190779,1,2023-09-14,no +CA,A,YES,2,31809,1,30000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,50.03140994688281,39.44621369622172,16.074000358249478,0,2023-09-14,NA +CA,A,YES,3,84531,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,6.042434526272639,17.96477256898295,12.084869052545278,1,2023-09-14,NA +CA,A,YES,4,12078,3,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,270.2556918948978,772.3136263757754,10.71087958562735,0,2023-09-14,NA +WA,B,Yes,6,31928,1,2000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,28.999636551725438,7.112110226684085,18.683558601460902,0,2023-09-14,NA +WA,B,Yes,7,07528,1,100000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,242.90574957188858,441.49582348827323,15.76269950552462,0,2023-09-14,NA +WA,B,Yes,9,14396,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,1.9500871596517524,123.00732124310856,3.9001743193035048,1,2023-09-14,NA +NM,C,yes,10,27604,5,1000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,2.8454444415409426,418.61803702666384,0.5406164700102142,0,2023-09-14,NA +ZZ,F,NO,13,65174,2,8472,wwtp,raw wastewater,copies/g dry sludge,sars-cov-2,6.069282489349167,20.525310445756737,8.848302737682596,0,2023-09-14,no +CA,A,YES,1,04971,1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,627.1858702693393,640.4499704489788,9.847493729048772,0,2023-09-15,no +CA,A,YES,2,31809,1,30000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,300.12201613719,9.13753490350026,4.7419849443985616,0,2023-09-15,NA +CA,A,YES,4,12078,3,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,1.4973163391200663,306.3932463794785,2.9946326782401327,1,2023-09-15,yes +CA,A,YES,3,84531,4,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,230.13764014508246,107.68233325353903,4.060576452260714,0,2023-09-15,NA +CA,A,YES,4,12078,4,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,11.297211715445465,261.220863465785,4.55816430474349,0,2023-09-15,no +WA,B,Yes,7,07528,1,100000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,659.6519077775376,154.76241976588497,2.963945321898027,0,2023-09-15,NA +WA,B,Yes,8,97385,2,50000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,24.43289897335104,54.4910442943567,3.7862987450286094,0,2023-09-15,NA +WA,B,Yes,9,14396,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,121.61991083088714,46.66901552690469,1.5712555961867025,0,2023-09-15,NA +NM,C,yes,10,27604,5,1000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,1.9618414032348217,639.4249431666166,0.3978812299862965,0,2023-09-15,NA +ZZ,F,NO,12,32541,1,8472,wwtp,raw wastewater,copies/g dry sludge,sars-cov-2,14.102204272137506,984.7060029667207,1.8440824070294632,0,2023-09-15,no +ZZ,F,NO,13,65174,2,8472,wwtp,raw wastewater,copies/g dry sludge,sars-cov-2,105.99735034894364,43.06748608757233,2.4110883649611536,0,2023-09-15,no +CA,A,YES,1,04971,1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,30.859634741461225,358.8292028381085,3.9280047220217194,0,2023-09-16,NA +CA,A,YES,2,31809,1,30000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,401.39145497144455,434.7427435314635,2.134992864334295,0,2023-09-16,NA +CA,A,YES,3,84531,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,435.18550861096793,47.37998927877417,2.592626358648773,0,2023-09-16,NA +CA,A,YES,4,12078,3,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,274.1310453605152,187.9368273876609,1.8170810349301558,0,2023-09-16,NA +CA,A,YES,3,84531,4,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,395.1561777373086,18.7945882821773,6.513522254402335,0,2023-09-16,no +CA,A,YES,4,12078,4,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,30.11301833861387,42.62591316659547,6.546255907865505,0,2023-09-16,NA +WA,B,Yes,7,07528,1,100000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,40.82224527616124,925.1411007017958,1.7585024036077894,0,2023-09-16,NA +WA,B,Yes,8,97385,2,50000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,70.21647024578209,96.90393218873007,4.671718311983423,0,2023-09-16,NA +WA,B,Yes,9,14396,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,50.75390229248807,26.805787063321805,1.2463680418046597,0,2023-09-16,NA +NM,C,yes,10,27604,5,1000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,1.5231145448452659,10.75111979253397,1.1381690644815359,0,2023-09-16,NA +ZZ,F,NO,13,65174,2,8472,wwtp,raw wastewater,copies/g dry sludge,sars-cov-2,23.447222157974128,455.5509942181424,1.2584251963530146,0,2023-09-16,NA +CA,A,YES,2,31809,1,30000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,5.124328061730982,858.3698531829724,10.248656123461965,1,2023-09-17,NA +CA,A,YES,3,84531,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,754.1507219369661,153.44263880512628,1.5984730416961492,0,2023-09-17,NA +CA,A,YES,4,12078,3,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,623.7242192141716,5.592627892087036,1.701161118881908,0,2023-09-17,no +CA,A,YES,3,84531,4,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,76.06906355042437,584.3058071281038,4.487847386099529,0,2023-09-17,no +CA,A,YES,4,12078,4,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,25.195452712044077,5.0580789742507495,5.123044358872136,0,2023-09-17,y +WA,B,Yes,7,07528,1,100000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,18.13796488449633,7.470078421183283,1.1016057011105906,0,2023-09-17,yes +WA,B,Yes,8,97385,2,50000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,16.03817049795714,120.3608266601489,2.013031385657321,0,2023-09-17,NA +WA,B,Yes,9,14396,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,25.423492038559306,698.4937844836131,1.2658622406219853,0,2023-09-17,no +NM,C,yes,10,27604,5,1000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,1.147533852110829,13.495236467046974,0.4704001570768611,0,2023-09-17,NA +ZZ,F,NO,15,98436,4,8472,wwtp,raw wastewater,copies/g dry sludge,sars-cov-2,12.454472909542101,26.244781970139844,2.558091093066221,0,2023-09-17,NA +CA,A,YES,1,04971,1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,15.899032256424775,564.6450526067346,3.1924705121443107,0,2023-09-18,NA +CA,A,YES,2,31809,1,30000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,0.5641751920521331,208.7532835329705,1.1283503841042661,1,2023-09-18,NA +CA,A,YES,3,84531,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,12.08350810017529,609.2131164367498,24.16701620035058,1,2023-09-18,NA +CA,A,YES,4,12078,3,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,158.29173947891587,145.91010752381382,10.880390014694674,0,2023-09-18,NA +CA,A,YES,3,84531,4,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,41.05244068678272,41.273562796522526,10.39642921391872,0,2023-09-18,NA +CA,A,YES,4,12078,4,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,10.368509978708842,73.34180431530028,13.554946356358688,0,2023-09-18,NA +WA,B,Yes,6,31928,1,2000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,9.038999091792924,6.494058971301816,5.192272677063447,0,2023-09-18,temperature not assessed upon arrival at the laboratory +WA,B,Yes,7,07528,1,100000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,31.337762988991308,206.1987105064303,1.355964836748163,0,2023-09-18,no +WA,B,Yes,8,97385,2,50000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,216.98007931079516,5.452676034619082,6.332371926542281,0,2023-09-18,NA +WA,B,Yes,9,14396,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,31.86396005250983,268.450426832543,3.533681021155041,0,2023-09-18,NA +NM,C,yes,10,27604,5,1000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,1.6624158485598437,349.8323866970989,0.09681419791914433,0,2023-09-18,NA +YY,E,No,12,32541,1,13,upstream,primary sludge,copies/g dry sludge,sars-cov-2,869.8862706586108,75.19807868043753,1.2395373844013466,0,2023-09-18,NA +YY,E,No,15,98436,4,13,upstream,primary sludge,copies/g dry sludge,sars-cov-2,922.9115148474556,660.1431713924558,1.5067063420985314,0,2023-09-18,NA +CA,A,YES,1,04971,1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,163.00694208865636,26.41527898111204,3.642606142908498,0,2023-09-19,NA +CA,A,YES,2,31809,1,30000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,905.5141939506323,36.41631052169025,5.629397584516005,0,2023-09-19,NA +CA,A,YES,3,84531,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,219.91883803237423,75.20349970014401,1.5576135326080256,0,2023-09-19,y +CA,A,YES,4,12078,3,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,165.9073301360782,55.28195609009838,8.717230116924922,0,2023-09-19,NA +CA,A,YES,5,34687,3,1000000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,0.5122944767552731,73.63502061663446,1.0245889535105461,1,2023-09-19,NA +CA,A,YES,3,84531,4,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,26.442704204964137,228.80343238736606,8.802839341089758,0,2023-09-19,NA +CA,A,YES,4,12078,4,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,223.4129354375486,138.35409964264724,23.981304217763874,0,2023-09-19,NA +CA,A,YES,5,34687,4,1000000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,195.112865867627,730.1865455720992,4.205915024549816,0,2023-09-19,NA +WA,B,Yes,7,07528,1,100000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,39.73572801128709,67.75775960349195,5.999784427986614,0,2023-09-19,NA +WA,B,Yes,9,14396,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,77.12493148152373,556.8459046605676,1.09012843716842,0,2023-09-19,NA +NM,C,yes,10,27604,5,1000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,1.9253028023754963,21.588013372477345,1.1365218981220306,0,2023-09-19,NA +XX,D,ye,13,65174,10,1000000000,wwtp,raw wastewater,copies/l wastewater,FLUAV,17.976432542843934,202.84608742290547,1.5191165443255417,0,2023-09-19,NA +ZZ,F,NO,14,09634,3,8472,wwtp,raw wastewater,copies/g dry sludge,sars-cov-2,223.39901839121097,713.5445145109429,2.8096776304763913,0,2023-09-19,NA +CA,A,YES,1,04971,1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,765.5408539523569,14.41215467320515,5.134440972765299,0,2023-09-20,NA +CA,A,YES,2,31809,1,30000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,32.03138902904595,14.291454496086965,21.750935727284062,0,2023-09-20,NA +CA,A,YES,3,84531,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,66.67007985779541,5.1893370844269455,2.0337834665254544,0,2023-09-20,> max temp and/or hold time +CA,A,YES,4,12078,3,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,0.6514278972287366,6.977577371797697,1.3028557944574732,1,2023-09-20,NA +CA,A,YES,3,84531,4,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,298.3636762597554,955.0479543441802,1.6816635707057863,0,2023-09-20,NA +CA,A,YES,4,12078,4,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,4.992181212686645,46.24693935643921,9.98436242537329,1,2023-09-20,NA +WA,B,Yes,6,31928,1,2000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,66.66555316883107,5.861154974446485,22.063204828896303,0,2023-09-20,NA +WA,B,Yes,7,07528,1,100000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,29.834576084555795,7.489234718173136,1.227909324424047,0,2023-09-20,NA +WA,B,Yes,8,97385,2,50000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,8.310041298136628,73.78539209232302,1.811308540706124,0,2023-09-20,NA +WA,B,Yes,9,14396,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,891.7997712566988,231.9433549268617,21.324133130079847,0,2023-09-20,no +CA,A,YES,2,31809,1,30000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,13.158660603343469,11.848202926995205,15.9843003832825,0,2023-09-21,no +CA,A,YES,3,84531,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,117.36392021689579,75.17912747889912,14.211676821888307,0,2023-09-21,NA +CA,A,YES,4,12078,3,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,74.27612525176858,8.20507467240631,3.491236295169116,0,2023-09-21,NA +CA,A,YES,3,84531,4,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,3.1511117008765823,933.013637706733,6.302223401753165,1,2023-09-21,NA +CA,A,YES,4,12078,4,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,42.83993233569971,5.592995282575416,18.283598052695787,0,2023-09-21,temperature not assessed upon arrival at the laboratory +WA,B,Yes,6,31928,1,2000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,14.021412604773127,7.000543339679783,13.778872574215725,0,2023-09-21,NA +WA,B,Yes,7,07528,1,100000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,481.92240466391627,145.5380957875945,21.2845208771929,0,2023-09-21,NA +WA,B,Yes,8,97385,2,50000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,6.715159495803706,522.0217438517983,13.430318991607413,1,2023-09-21,NA +WA,B,Yes,9,14396,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,598.7761301909878,77.29300873856852,11.890243384486608,0,2023-09-21,no +NM,C,yes,10,27604,5,1000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,1.1266103198901292,9.127208331494755,1.0465820742061096,0,2023-09-21,NA +XX,D,ye,13,65174,10,1000000000,wwtp,raw wastewater,copies/l wastewater,FLUAV,186.7405814430018,130.44527558807707,1.8635571605521872,0,2023-09-21,no +XX,D,ye,14,09634,10,1000000000,wwtp,raw wastewater,copies/l wastewater,FLUAV,7.831683989328578,88.73688812446485,2.0432388785908464,0,2023-09-21,NA +YY,E,No,14,09634,3,13,upstream,primary sludge,copies/g dry sludge,sars-cov-2,11.140698852114427,96.25931818406534,22.281397704228855,1,2023-09-21,yes +CA,A,YES,1,04971,1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,7.3055257413574735,118.07007497423882,2.0505872967575423,0,2023-09-22,NA +CA,A,YES,2,31809,1,30000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,267.86907646268435,12.121669155145032,23.321045100446533,0,2023-09-22,NA +CA,A,YES,3,84531,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,5.906247733955276,492.8212436581897,11.812495467910551,1,2023-09-22,no +CA,A,YES,4,12078,3,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,8.241052893392862,7.943906975151102,16.482105786785723,1,2023-09-22,NA +CA,A,YES,3,84531,4,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,12.755052244944414,238.72987341973166,15.527018757416068,0,2023-09-22,NA +CA,A,YES,4,12078,4,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,8.110036746967317,27.31577175009725,1.5835462351255556,0,2023-09-22,> max temp and/or hold time +WA,B,Yes,6,31928,1,2000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,186.45623243551162,79.27449546437143,12.242454817723667,0,2023-09-22,NA +WA,B,Yes,7,07528,1,100000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,7.311228039730669,35.74185387118438,4.44038706481002,0,2023-09-22,NA +WA,B,Yes,8,97385,2,50000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,140.46085090977772,60.27575341716619,1.0034512324148095,0,2023-09-22,NA +WA,B,Yes,9,14396,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,94.22141817802886,15.051688038650608,5.374429094299932,0,2023-09-22,NA +NM,C,yes,10,27604,5,1000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,2.158982938299587,9.313469419085513,0.975422992140235,0,2023-09-22,NA +CA,A,YES,1,04971,1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,9.615214324184283,98.28676416676558,22.216317904643557,0,2023-09-23,> max temp and/or hold time +CA,A,YES,2,31809,1,30000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,589.3986281978644,8.679133272140561,1.8098973454090526,0,2023-09-23,NA +CA,A,YES,3,84531,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,37.39048546074088,18.685814104050912,12.850001955744203,0,2023-09-23,NA +CA,A,YES,4,12078,3,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,43.76232441134541,10.164161127127485,12.482619362899898,0,2023-09-23,NA +CA,A,YES,3,84531,4,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,23.217931853486764,89.47016678434515,8.416339623993995,0,2023-09-23,NA +CA,A,YES,4,12078,4,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,124.22847114573283,628.6235519362383,1.2642699947706546,0,2023-09-23,NA +WA,B,Yes,6,31928,1,2000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,385.7144117750784,19.122014212075385,6.796198356949728,0,2023-09-23,NA +WA,B,Yes,8,97385,2,50000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,122.86920120852614,12.118305805132154,1.3095842369835462,0,2023-09-23,NA +NM,C,yes,10,27604,5,1000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,1.2604096759466097,76.92000128669947,0.6407394664269102,0,2023-09-23,NA +CA,A,YES,1,04971,1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,31.556363874652213,28.002580514447082,3.3700121744830933,0,2023-09-24,NA +CA,A,YES,2,31809,1,30000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,360.54843693920793,470.26117697733577,5.606447239255045,0,2023-09-24,NA +CA,A,YES,3,84531,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,6.071314365181604,981.1203214557684,4.998796740742051,0,2023-09-24,NA +CA,A,YES,4,12078,3,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,566.7666578013052,59.070074117639074,7.508477127847408,0,2023-09-24,NA +CA,A,YES,3,84531,4,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,813.3088033937094,28.24450545026727,2.063151085209026,0,2023-09-24,NA +CA,A,YES,4,12078,4,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,113.97985183410167,144.7627628917344,2.0539290996208246,0,2023-09-24,NA +WA,B,Yes,6,31928,1,2000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,7.020972184015425,89.21413873583612,2.5120647283373807,0,2023-09-24,> max temp and/or hold time +WA,B,Yes,7,07528,1,100000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,74.60265135002568,145.28254128458858,3.26001319033406,0,2023-09-24,NA +WA,B,Yes,8,97385,2,50000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,477.84024698321844,42.73506869943282,13.92995860191306,0,2023-09-24,NA +NM,C,yes,10,27604,5,1000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,1.0878754599856533,176.24909304262266,1.1426858251406937,0,2023-09-24,no +XX,D,ye,15,98436,10,1000000000,wwtp,raw wastewater,copies/l wastewater,FLUAV,26.905453106940413,113.13698548396529,4.647740235578791,0,2023-09-24,NA +CA,A,YES,2,31809,1,30000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,114.14099468369378,35.91044341925311,13.687935681006486,0,2023-09-25,no +CA,A,YES,3,84531,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,9.220679999764629,50.90915290814995,2.9866980228498874,0,2023-09-25,NA +CA,A,YES,4,12078,3,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,70.87809379380387,190.00122004407942,16.38319293800713,0,2023-09-25,NA +CA,A,YES,3,84531,4,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,203.12695288545802,6.00190364537894,1.3926822751837495,0,2023-09-25,NA +CA,A,YES,4,12078,4,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,663.7176234826474,224.3156076234778,19.1854317057789,0,2023-09-25,below loq +WA,B,Yes,6,31928,1,2000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,42.444677904591536,582.916096604578,2.598417138292784,0,2023-09-25,NA +WA,B,Yes,8,97385,2,50000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,950.3150020785197,12.580653468395676,1.670466354276355,0,2023-09-25,NA +WA,B,Yes,9,14396,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,4.889032963255966,13.102325779020566,9.778065926511932,1,2023-09-25,NA +NM,C,yes,10,27604,5,1000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,1.1086449013137685,463.60724308942036,0.7148204765967774,0,2023-09-25,no +CA,A,YES,1,04971,1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,85.78745849492218,975.8485468745371,1.5410159137588204,0,2023-09-26,no +CA,A,YES,2,31809,1,30000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,9.150488676209262,12.540172173003441,1.9074647370440232,0,2023-09-26,temperature not assessed upon arrival at the laboratory +CA,A,YES,3,84531,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,2.2499979472637692,5.542076442621802,4.4999958945275385,1,2023-09-26,> max temp and/or hold time +CA,A,YES,4,12078,3,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,211.46535478634382,29.953658449638638,3.6904666390759293,0,2023-09-26,NA +CA,A,YES,3,84531,4,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,8.250180670789337,133.8527388427675,5.128959599959575,0,2023-09-26,NA +CA,A,YES,4,12078,4,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,10.53145582712634,465.38892829773954,9.054361831881396,0,2023-09-26,NA +CA,A,YES,5,34687,4,1000000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,5.391070234800884,929.8155106852355,4.541582182010096,0,2023-09-26,NA +WA,B,Yes,7,07528,1,100000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,0.7642637031878313,427.8139497512402,1.5285274063756626,1,2023-09-26,NA +WA,B,Yes,8,97385,2,50000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,10.367492988529134,267.7100006795026,1.8877613699178106,0,2023-09-26,no +WA,B,Yes,9,14396,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,620.0420443084761,161.7127204685984,6.762417687997335,0,2023-09-26,no +XX,D,ye,14,09634,10,1000000000,wwtp,raw wastewater,copies/l wastewater,FLUAV,185.28361724530987,9.736445543866681,12.179650555363896,0,2023-09-26,no +CA,A,YES,1,04971,1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,954.7987735948498,62.703028308156135,1.1257510937636845,0,2023-09-27,no +CA,A,YES,2,31809,1,30000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,6.162318139303052,100.03188428173453,1.02910426769529,0,2023-09-27,NA +CA,A,YES,3,84531,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,35.61555429763078,11.08132187994924,1.8462746901145295,0,2023-09-27,yes +CA,A,YES,3,84531,4,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,137.49700829913627,22.173473915902825,10.040806265175325,0,2023-09-27,NA +CA,A,YES,4,12078,4,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,6.379531276928821,36.67846798883373,4.321196355409046,0,2023-09-27,no +WA,B,Yes,6,31928,1,2000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,703.2490997129202,40.31647367576254,2.6756321338279543,0,2023-09-27,NA +WA,B,Yes,7,07528,1,100000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,462.3139194793472,137.82972840836823,1.3190950559810797,0,2023-09-27,NA +WA,B,Yes,8,97385,2,50000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,463.3614312316438,9.452998553164475,2.281879990940991,0,2023-09-27,NA +WA,B,Yes,9,14396,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,50.52356366306579,5.9639524573694,12.618601666986986,0,2023-09-27,NA +NM,C,yes,10,27604,5,1000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,2.440898459470167,21.662439446543726,0.11158832206158237,0,2023-09-27,no +YY,E,No,13,65174,2,13,upstream,primary sludge,copies/g dry sludge,sars-cov-2,153.12242831097095,308.256127965897,19.03053639573866,0,2023-09-27,NA +CA,A,YES,1,04971,1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,47.157479753344234,6.313948334153875,5.600355902607118,0,2023-09-28,no +CA,A,YES,2,31809,1,30000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,70.21973467488006,447.1875755540949,1.4805569683914932,0,2023-09-28,y +CA,A,YES,3,84531,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,13.672572480670611,968.9419362678033,3.3272351753637106,0,2023-09-28,NA +CA,A,YES,4,12078,3,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,7.07931153839453,42.14947276666262,11.719942977672483,0,2023-09-28,NA +CA,A,YES,3,84531,4,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,17.774222622960316,13.603449560565462,2.450885148411109,0,2023-09-28,NA +CA,A,YES,4,12078,4,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,30.941604143704264,267.63886670627926,19.237314824078485,0,2023-09-28,no +WA,B,Yes,9,14396,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,9.587812640351965,129.6144052832534,12.51699014908505,0,2023-09-28,no +NM,C,yes,10,27604,5,1000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,1.9699460970806106,8.522301578312781,0.6628074177945588,0,2023-09-28,temperature not assessed upon arrival at the laboratory +YY,E,No,14,09634,3,13,upstream,primary sludge,copies/g dry sludge,sars-cov-2,371.9795342089151,458.7670615484376,3.852203925639722,0,2023-09-28,NA +CA,A,YES,1,04971,1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,15.378172379650767,43.82671974549931,2.9286051626342475,0,2023-09-29,NA +CA,A,YES,2,31809,1,30000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,480.3029646825651,35.532865070896015,11.744278594389417,0,2023-09-29,NA +CA,A,YES,3,84531,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,7.14864872101037,465.6332659230167,9.05575862381514,0,2023-09-29,NA +CA,A,YES,4,12078,3,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,218.4735343607085,6.413567737727083,1.6131721737909337,0,2023-09-29,NA +CA,A,YES,3,84531,4,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,0.6645901961000065,270.7550893217722,1.329180392200013,1,2023-09-29,NA +CA,A,YES,4,12078,4,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,333.588588043735,197.26383929321725,5.2879672745399775,0,2023-09-29,NA +WA,B,Yes,6,31928,1,2000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,17.65272729639427,431.60429318073045,1.0578660486581493,0,2023-09-29,NA +WA,B,Yes,7,07528,1,100000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,437.8495360827869,205.29034456915676,2.5364457591472522,0,2023-09-29,NA +WA,B,Yes,8,97385,2,50000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,259.62476221296896,65.01878639775279,3.5983552533169902,0,2023-09-29,no +WA,B,Yes,9,14396,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,1.738545227109971,66.44295029778412,3.477090454219942,1,2023-09-29,NA +NM,C,yes,10,27604,5,1000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,2.2465545808364014,11.01037346073883,1.2504673426989044,0,2023-09-29,NA +XX,D,ye,15,98436,10,1000000000,wwtp,raw wastewater,copies/l wastewater,FLUAV,27.426072668225096,148.1952222949359,1.6586494536492085,0,2023-09-29,NA +CA,A,YES,1,04971,1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,265.74559264543973,632.1814334477729,1.2170482720748443,0,2023-09-30,no +CA,A,YES,2,31809,1,30000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,12.053441426000306,95.82955650517901,7.355491929373887,0,2023-09-30,no +CA,A,YES,3,84531,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,2.24818819745346,590.6495140506445,4.49637639490692,1,2023-09-30,temperature not assessed upon arrival at the laboratory +CA,A,YES,4,12078,3,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,131.70152437486388,329.6406118397978,9.804620498770095,0,2023-09-30,NA +CA,A,YES,3,84531,4,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,5.694805168024263,11.044263015030353,5.7101126058158895,0,2023-09-30,NA +CA,A,YES,4,12078,4,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,35.793847922739396,70.52077478740188,11.96751242919476,0,2023-09-30,NA +WA,B,Yes,6,31928,1,2000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,9.673471207314114,51.489536833801864,4.055811843828344,0,2023-09-30,NA +WA,B,Yes,7,07528,1,100000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,627.3481081758306,67.52115582422896,1.5244221387164485,0,2023-09-30,no +WA,B,Yes,8,97385,2,50000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,24.946577712397836,579.4763370161098,3.2074833238823177,0,2023-09-30,no +WA,B,Yes,9,14396,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,812.6357861639331,13.53044237204365,5.360688488910962,0,2023-09-30,NA +NM,C,yes,10,27604,5,1000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,1.2944443690471588,243.9897226729591,0.23921643958778607,0,2023-09-30,NA +YY,E,No,12,32541,1,13,upstream,primary sludge,copies/g dry sludge,sars-cov-2,437.18245198245785,95.19289351173602,12.898248406766136,0,2023-09-30,y +YY,E,No,15,98436,4,13,upstream,primary sludge,copies/g dry sludge,sars-cov-2,13.133514471188034,523.7099843002505,22.55960981679513,0,2023-09-30,NA +CA,A,YES,3,84531,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,66.49371593153035,15.235060202154155,10.84492587846565,0,2023-10-01,NA +CA,A,YES,4,12078,3,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,199.03601450021895,48.24715561675021,1.8444513028015532,0,2023-10-01,NA +CA,A,YES,3,84531,4,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,458.306410172384,40.01874650771095,17.98269063570498,0,2023-10-01,NA +CA,A,YES,4,12078,4,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,139.06621172850657,13.809881485616762,20.36366690044936,0,2023-10-01,y +WA,B,Yes,9,14396,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,89.94695773804438,867.2472748766627,20.834674268341548,0,2023-10-01,NA +NM,C,yes,10,27604,5,1000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,0.7719059962064264,251.9251007120738,0.03515675372305361,0,2023-10-01,no +ZZ,F,NO,13,65174,2,8472,wwtp,raw wastewater,copies/g dry sludge,sars-cov-2,22.884519428772002,417.5940820303677,2.99709872161177,0,2023-10-01,y +CA,A,YES,3,84531,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,509.4986392243626,966.1325100808231,10.005681911914005,0,2023-10-02,yes +CA,A,YES,4,12078,3,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,3.0922195484474155,30.146968914272172,6.184439096894831,1,2023-10-02,NA +CA,A,YES,3,84531,4,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,115.29228033492146,946.7680480293168,8.192163058664878,0,2023-10-02,below loq +CA,A,YES,4,12078,4,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,19.373047744646126,635.9751700373812,2.4429328725699233,0,2023-10-02,> max temp and/or hold time +WA,B,Yes,8,97385,2,50000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,215.58950669901932,515.4319280558805,18.720373499930734,0,2023-10-02,NA +WA,B,Yes,9,14396,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,60.80932072602497,751.761561011576,1.0143530352806562,0,2023-10-02,NA +NM,C,yes,10,27604,5,1000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,2.1631820944649323,54.79417116892696,1.18256121173432,0,2023-10-02,NA +YY,E,No,14,09634,3,13,upstream,primary sludge,copies/g dry sludge,sars-cov-2,16.64110837053129,538.0374688498371,1.5915975971913003,0,2023-10-02,NA +CA,A,YES,3,84531,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,244.42972290899493,509.0965445468101,3.826052118700248,0,2023-10-03,NA +CA,A,YES,4,12078,3,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,796.86680327611,15.077124128078676,13.774662684200747,0,2023-10-03,no +CA,A,YES,5,34687,3,1000000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,10.10931822832897,100.64251661134968,3.6447241058947077,0,2023-10-03,NA +CA,A,YES,3,84531,4,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,7.574430031501879,6.478789068777146,1.294899514234036,0,2023-10-03,NA +CA,A,YES,5,34687,4,1000000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,277.9909575369758,33.84350183818802,5.410353108745165,0,2023-10-03,no +WA,B,Yes,8,97385,2,50000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,92.92781904129124,222.55265400805212,1.34703303804376,0,2023-10-03,yes +WA,B,Yes,9,14396,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,150.1203744212265,808.7870429779051,6.510256818905695,0,2023-10-03,below loq +ZZ,F,NO,15,98436,4,8472,wwtp,raw wastewater,copies/g dry sludge,sars-cov-2,106.1014806321398,595.3379127934736,1.009329905107801,0,2023-10-03,NA +CA,A,YES,4,12078,3,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,425.7941127032567,147.42410415204753,11.746803736774377,0,2023-10-04,NA +CA,A,YES,3,84531,4,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,50.44101826490875,21.92765976231842,8.41812919371939,0,2023-10-04,NA +CA,A,YES,4,12078,4,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,1.5867545670111491,9.725344984431333,3.1735091340222983,1,2023-10-04,NA +WA,B,Yes,8,97385,2,50000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,42.60801036223503,462.7272565675213,8.115997406608045,0,2023-10-04,no +WA,B,Yes,9,14396,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,32.0643051889199,283.7348224137569,11.696890522830913,0,2023-10-04,NA +NM,C,yes,10,27604,5,1000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,0.932165600728596,200.61773061431515,0.8697163168474171,0,2023-10-04,NA +XX,D,ye,15,98436,10,1000000000,wwtp,raw wastewater,copies/l wastewater,FLUAV,63.36852460260965,388.02047081638364,2.028991614773172,0,2023-10-04,NA +CA,A,YES,3,84531,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,170.51718502349667,8.207298818930841,9.612164105904183,0,2023-10-05,NA +CA,A,YES,4,12078,3,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,545.5520416938306,240.07149574947834,17.91560101875043,0,2023-10-05,NA +CA,A,YES,3,84531,4,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,2.847622709748941,209.17199745315637,5.695245419497882,1,2023-10-05,NA +CA,A,YES,4,12078,4,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,15.046236982558538,33.91499043852539,15.601088565276516,0,2023-10-05,NA +WA,B,Yes,8,97385,2,50000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,41.843747024089865,96.62740073806953,6.39029585416629,0,2023-10-05,> max temp and/or hold time +WA,B,Yes,9,14396,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,187.4672297176034,5.255488582026274,1.0084797102876852,0,2023-10-05,NA +XX,D,ye,12,32541,10,1000000000,wwtp,raw wastewater,copies/l wastewater,FLUAV,59.21544330011755,5.219469324573686,17.66517039030799,0,2023-10-05,no +ZZ,F,NO,14,09634,3,8472,wwtp,raw wastewater,copies/g dry sludge,sars-cov-2,7.5308736068437625,814.5623020935992,8.573361750232271,0,2023-10-05,temperature not assessed upon arrival at the laboratory +CA,A,YES,3,84531,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,552.1067172104347,5.827496620388416,1.6054787246194608,0,2023-10-06,NA +CA,A,YES,4,12078,3,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,80.0964930575536,75.4061118864051,11.800598611149846,0,2023-10-06,no +CA,A,YES,3,84531,4,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,516.2371308563871,15.699042218304001,3.1238083233728,0,2023-10-06,NA +CA,A,YES,4,12078,4,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,222.0449535412146,351.33813267659326,19.76519636722731,0,2023-10-06,NA +WA,B,Yes,8,97385,2,50000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,203.76470899279204,474.6710903268399,6.42914724144003,0,2023-10-06,no +WA,B,Yes,9,14396,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,84.87278466607732,30.509841200488566,16.282178272816456,0,2023-10-06,NA +NM,C,yes,10,27604,5,1000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,1.3793220956970882,19.275791018935816,0.5841938994338673,0,2023-10-06,NA +ZZ,F,NO,13,65174,2,8472,wwtp,raw wastewater,copies/g dry sludge,sars-cov-2,15.923917934940867,868.0989752772641,15.443210635967363,0,2023-10-06,no +ZZ,F,NO,14,09634,3,8472,wwtp,raw wastewater,copies/g dry sludge,sars-cov-2,29.225067348702897,147.3721024200546,19.632876481097327,0,2023-10-06,result is not quantifiable +CA,A,YES,3,84531,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,5.557118078207582,151.94466862947516,12.902612893832606,0,2023-10-07,no +CA,A,YES,4,12078,3,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,61.179145697395086,10.127090862042,1.2867775165992612,0,2023-10-07,NA +CA,A,YES,3,84531,4,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,117.04153520553125,54.575542140000586,18.380920108198477,0,2023-10-07,NA +CA,A,YES,4,12078,4,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,6.016523979395135,22.548366524657805,7.93397923515608,0,2023-10-07,NA +WA,B,Yes,8,97385,2,50000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,56.39240843997941,8.388325849957104,3.749640255671547,0,2023-10-07,result is not quantifiable +WA,B,Yes,9,14396,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,34.72479326458965,497.28028395626234,8.95334587412118,0,2023-10-07,no +XX,D,ye,15,98436,10,1000000000,wwtp,raw wastewater,copies/l wastewater,FLUAV,20.882512316077573,102.26712804115729,18.54492480140478,0,2023-10-07,below loq +CA,A,YES,3,84531,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,10.508855169065688,34.10402455096962,4.875167163760601,0,2023-10-08,NA +CA,A,YES,3,84531,4,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,10.749599451332713,9.597579224508529,5.539251259990351,0,2023-10-08,NA +CA,A,YES,4,12078,4,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,25.724074976712398,45.16251189433572,2.974925462833576,0,2023-10-08,NA +WA,B,Yes,8,97385,2,50000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,853.8349356373734,31.81054076027622,24.252005008716317,0,2023-10-08,NA +WA,B,Yes,9,14396,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,12.264871266683702,9.726118732379241,7.048412891176777,0,2023-10-08,no +NM,C,yes,10,27604,5,1000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,2.22831472524295,52.26733747856825,1.12892134848061,0,2023-10-08,NA +YY,E,No,14,09634,3,13,upstream,primary sludge,copies/g dry sludge,sars-cov-2,85.65961375858862,87.50414949491378,3.5161997719432363,0,2023-10-08,result is not quantifiable +CA,A,YES,3,84531,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,27.77432548785992,858.4473984478545,2.186843517523021,0,2023-10-09,NA +CA,A,YES,3,84531,4,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,598.3951629810593,506.9210276579006,2.534444432123457,0,2023-10-09,no +CA,A,YES,4,12078,4,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,542.0600074655836,34.736799388515806,21.78409955975368,0,2023-10-09,NA +NM,C,yes,10,27604,5,1000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,1.5761519157034882,911.009836101548,1.2906727491503849,0,2023-10-09,NA +YY,E,No,13,65174,2,13,upstream,primary sludge,copies/g dry sludge,sars-cov-2,98.21301402022664,65.36807614428987,13.124110237600412,0,2023-10-09,NA +CA,A,YES,3,84531,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,11.990252632118173,506.85851970699514,22.694291389108418,0,2023-10-10,below loq +CA,A,YES,4,12078,3,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,12.07822320749399,6.6754660177788665,1.527312722305752,0,2023-10-10,NA +CA,A,YES,5,34687,3,1000000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,206.0890130822997,563.6701579726736,22.417459505654655,0,2023-10-10,NA +CA,A,YES,4,12078,4,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,8.827629865078057,27.141143146421644,22.74692593275052,0,2023-10-10,no +CA,A,YES,5,34687,4,1000000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,106.7503871808489,158.53915488992675,23.236384209749293,0,2023-10-10,yes +WA,B,Yes,8,97385,2,50000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,187.57469607722751,13.062607606257759,2.3544183792662965,0,2023-10-10,NA +WA,B,Yes,9,14396,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,10.409353484171133,20.324159553278403,2.1035893582037777,0,2023-10-10,no +NM,C,yes,10,27604,5,1000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,2.2843118121746437,29.79467753601584,0.08182608089372499,0,2023-10-10,no +YY,E,No,13,65174,2,13,upstream,primary sludge,copies/g dry sludge,sars-cov-2,114.01861217659278,44.98664426874732,2.6876953682622893,0,2023-10-10,NA +CA,A,YES,3,84531,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,663.2734171858697,10.677942366953156,8.959274078659726,0,2023-10-11,NA +CA,A,YES,4,12078,3,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,131.7833970068489,119.60530968140968,1.4542665332491322,0,2023-10-11,NA +CA,A,YES,3,84531,4,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,8.063117528710123,32.58338015893037,5.148147321826049,0,2023-10-11,NA +CA,A,YES,4,12078,4,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,8.565017967767252,68.06480612581274,4.528508915573428,0,2023-10-11,NA +WA,B,Yes,8,97385,2,50000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,14.118948492291432,5.5537078012660315,8.617411164567168,0,2023-10-11,below loq +WA,B,Yes,9,14396,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,481.5879615142811,8.38959308050087,1.0948457285147692,0,2023-10-11,no +NM,C,yes,10,27604,5,1000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,1.4397264322449435,7.356000064399919,0.4678975298686844,0,2023-10-11,no +CA,A,YES,3,84531,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,38.020652531387434,63.48119041280152,2.1186852701099914,0,2023-10-12,no +CA,A,YES,3,84531,4,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,21.341047830253036,218.88972364963215,4.467046810026822,0,2023-10-12,NA +CA,A,YES,4,12078,4,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,971.3120887776709,15.053038080001487,2.507115118268606,0,2023-10-12,no +WA,B,Yes,8,97385,2,50000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,25.231384201205923,190.71046208455218,1.962476193313247,0,2023-10-12,NA +WA,B,Yes,9,14396,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,39.27056319028819,22.175973558824193,4.675360330440523,0,2023-10-12,NA +XX,D,ye,12,32541,10,1000000000,wwtp,raw wastewater,copies/l wastewater,FLUAV,36.494964694765265,156.69522299381546,16.01553521662312,0,2023-10-12,no +XX,D,ye,13,65174,10,1000000000,wwtp,raw wastewater,copies/l wastewater,FLUAV,178.47675597804286,19.82343921995668,4.976382084080779,0,2023-10-12,NA +CA,A,YES,3,84531,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,18.18958050348871,155.8686367557096,17.966080948016497,0,2023-10-13,NA +CA,A,YES,4,12078,3,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,768.1183895694751,821.8787089884058,1.5356466209070216,0,2023-10-13,no +CA,A,YES,3,84531,4,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,70.36156803282712,8.561568178744963,7.817915122585046,0,2023-10-13,NA +CA,A,YES,4,12078,4,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,514.0444230881371,5.097355951567906,12.378544783715427,0,2023-10-13,NA +WA,B,Yes,9,14396,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,493.1807957339792,9.230123707760841,4.756798019876201,0,2023-10-13,NA +NM,C,yes,10,27604,5,1000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,2.296432143600992,21.031562268044375,0.575137965337441,0,2023-10-13,no +CA,A,YES,3,84531,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,444.7765144161529,354.6767000530684,15.813943213219234,0,2023-10-14,NA +CA,A,YES,4,12078,3,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,831.3638951426606,18.73167908393349,20.78817742863092,0,2023-10-14,NA +CA,A,YES,3,84531,4,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,10.338419058479285,200.98171909516114,17.079502270529343,0,2023-10-14,> max temp and/or hold time +WA,B,Yes,9,14396,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,7.070138924447792,18.622333430370695,14.140277848895584,1,2023-10-14,NA +NM,C,yes,10,27604,5,1000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,2.323167577526934,7.534409446935117,1.1065199036976288,0,2023-10-14,NA +YY,E,No,13,65174,2,13,upstream,primary sludge,copies/g dry sludge,sars-cov-2,425.8335660706917,150.26090776990105,4.75235475998427,0,2023-10-14,NA +CA,A,YES,3,84531,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,5.289026997867349,5.909666287558022,10.578053995734699,1,2023-10-15,result is not quantifiable +CA,A,YES,4,12078,3,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,835.4953116613802,13.67747557841826,1.9320736502046731,0,2023-10-15,below loq +CA,A,YES,3,84531,4,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,287.30468975513685,612.3138496532684,6.638613246420889,0,2023-10-15,NA +CA,A,YES,4,12078,4,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,0.7189999430558236,28.65466801461557,1.4379998861116472,1,2023-10-15,temperature not assessed upon arrival at the laboratory +WA,B,Yes,8,97385,2,50000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,6.329085813799978,15.738436730228898,12.658171627599955,1,2023-10-15,yes +WA,B,Yes,9,14396,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,473.8838737776612,42.308310550453726,11.931234873961118,0,2023-10-15,NA +NM,C,yes,10,27604,5,1000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,1.0623789475452994,35.90059707798774,0.48515781339315184,0,2023-10-15,NA +XX,D,ye,12,32541,10,1000000000,wwtp,raw wastewater,copies/l wastewater,FLUAV,9.838213549938805,171.94735560844063,3.7344010705537523,0,2023-10-15,NA +ZZ,F,NO,15,98436,4,8472,wwtp,raw wastewater,copies/g dry sludge,sars-cov-2,269.04774731186535,52.212859539783864,24.509626781922755,0,2023-10-15,below loq +CA,A,YES,3,84531,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,34.964297624296584,942.6248267945231,2.0176834482935866,0,2023-10-16,NA +CA,A,YES,3,84531,4,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,6.907761011191645,764.2355815946142,5.102738023612888,0,2023-10-16,no +CA,A,YES,4,12078,4,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,90.78763189092989,7.472361321043849,21.167902954814384,0,2023-10-16,NA +WA,B,Yes,8,97385,2,50000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,47.98991940717253,13.769259646301014,2.011818618700013,0,2023-10-16,NA +WA,B,Yes,9,14396,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,127.91341178261992,69.064600994677,4.329233089304012,0,2023-10-16,no +NM,C,yes,10,27604,5,1000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,2.951397655803481,316.6449504879748,1.3074940657542586,0,2023-10-16,NA +ZZ,F,NO,13,65174,2,8472,wwtp,raw wastewater,copies/g dry sludge,sars-cov-2,113.59560950476138,691.4040102554357,5.852184389026455,0,2023-10-16,no +CA,A,YES,3,84531,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,39.874448481740394,641.0085993329824,6.1753394778118755,0,2023-10-17,no +CA,A,YES,4,12078,3,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,0.6152240655505071,309.1674447015484,1.2304481311010143,1,2023-10-17,NA +CA,A,YES,5,34687,3,1000000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,518.0249672943361,54.91940137753456,1.3015160750831454,0,2023-10-17,NA +CA,A,YES,4,12078,4,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,32.49691525864569,92.75658600382631,3.1538397447463677,0,2023-10-17,NA +CA,A,YES,5,34687,4,1000000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,608.4254668620891,175.37466628136374,2.730008123699104,0,2023-10-17,NA +WA,B,Yes,8,97385,2,50000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,78.36970650850955,14.251845133610155,8.429632571503626,0,2023-10-17,no +NM,C,yes,10,27604,5,1000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,1.7233024884174808,163.48714684650386,0.9958442103666133,0,2023-10-17,NA +CA,A,YES,3,84531,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,292.6869298175846,12.505990622948287,4.009485161469891,0,2023-10-18,NA +CA,A,YES,4,12078,3,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,0.817378877961846,6.518168165614757,1.634757755923692,1,2023-10-18,yes +CA,A,YES,3,84531,4,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,53.030851844483806,140.5334778970173,10.867406378824402,0,2023-10-18,NA +CA,A,YES,4,12078,4,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,57.89600163784824,9.95468921442796,1.8302479711097623,0,2023-10-18,NA +WA,B,Yes,8,97385,2,50000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,29.412281448935417,6.143481137463285,1.8433619481520604,0,2023-10-18,NA +NM,C,yes,10,27604,5,1000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,2.5783851260394464,261.6915902157961,0.978713270314126,0,2023-10-18,NA +CA,A,YES,3,84531,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,213.35258068958808,42.05595602820387,2.0350607152609643,0,2023-10-19,NA +CA,A,YES,4,12078,3,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,13.50607758196845,45.15793786949418,3.0553103762400298,0,2023-10-19,temperature not assessed upon arrival at the laboratory +CA,A,YES,4,12078,4,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,127.71917258570824,522.4373687729158,18.511210948317686,0,2023-10-19,NA +WA,B,Yes,8,97385,2,50000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,9.98926177077853,12.262451858244894,3.5787431348251717,0,2023-10-19,NA +WA,B,Yes,9,14396,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,603.388784336611,207.82019823436622,17.963883853541887,0,2023-10-19,NA +NM,C,yes,10,27604,5,1000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,1.0544844751504978,63.76882652641839,0.8713477623648845,0,2023-10-19,NA +CA,A,YES,3,84531,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,371.37600701787204,7.016909186610781,11.161963492593715,0,2023-10-20,NA +CA,A,YES,4,12078,3,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,44.41903844600761,105.74379676301754,15.701504038009565,0,2023-10-20,NA +CA,A,YES,3,84531,4,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,996.030897449756,532.994223754511,23.868798786152826,0,2023-10-20,yes +CA,A,YES,4,12078,4,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,5.308416365276997,169.91720585782465,16.711984889932864,0,2023-10-20,NA +WA,B,Yes,8,97385,2,50000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,36.628662185722085,760.6327725504028,4.768728832170876,0,2023-10-20,NA +WA,B,Yes,9,14396,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,266.3778178386391,119.4574892471819,1.186721947459849,0,2023-10-20,result is not quantifiable +NM,C,yes,10,27604,5,1000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,0.9591834329210125,8.955841474358499,0.45011272874944175,0,2023-10-20,NA +ZZ,F,NO,14,09634,3,8472,wwtp,raw wastewater,copies/g dry sludge,sars-cov-2,334.26454326412875,84.50727995779566,2.5448210261884157,0,2023-10-20,no +CA,A,YES,3,84531,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,881.9425049233201,18.34611526942993,12.306178530286193,0,2023-10-21,NA +CA,A,YES,4,12078,3,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,698.4952691415095,293.4596198943334,10.432976578556037,0,2023-10-21,temperature not assessed upon arrival at the laboratory +CA,A,YES,3,84531,4,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,17.558919627044183,404.15202774521543,4.9641642969629105,0,2023-10-21,no +CA,A,YES,4,12078,4,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,460.4856693596631,22.187875277045435,1.0096818420901097,0,2023-10-21,no +WA,B,Yes,8,97385,2,50000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,709.2997542824913,74.62411662340395,12.027109262117163,0,2023-10-21,NA +NM,C,yes,10,27604,5,1000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,1.4326060086617947,16.450507665226073,0.7957900583501931,0,2023-10-21,NA +ZZ,F,NO,13,65174,2,8472,wwtp,raw wastewater,copies/g dry sludge,sars-cov-2,11.277010424795156,6.652413566939346,22.554020849590312,1,2023-10-21,NA +CA,A,YES,3,84531,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,90.92540499384846,91.19075416307263,5.00812944604182,0,2023-10-22,no +CA,A,YES,3,84531,4,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,233.74224921560173,34.21907949847487,2.2950298200484025,0,2023-10-22,NA +CA,A,YES,4,12078,4,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,13.953757794137852,47.64123815215977,20.744774712885185,0,2023-10-22,NA +WA,B,Yes,8,97385,2,50000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,29.37615695458643,21.32353365304303,24.091745173289443,0,2023-10-22,NA +WA,B,Yes,9,14396,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,34.60123935920891,86.55039327262894,1.746845097265944,0,2023-10-22,no +NM,C,yes,10,27604,5,1000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,2.7480353542311233,140.45400519411635,1.2950399004471143,0,2023-10-22,NA +CA,A,YES,3,84531,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,5.1957581268404285,31.740342545095242,10.391516253680857,1,2023-10-23,result is not quantifiable +CA,A,YES,4,12078,3,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,571.7233730620007,61.5580416378177,12.215821349251746,0,2023-10-23,NA +CA,A,YES,3,84531,4,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,52.25518203822464,44.10997146781117,4.875760127257766,0,2023-10-23,no +WA,B,Yes,8,97385,2,50000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,5.069009669122406,312.4628890948918,2.8906215172644414,0,2023-10-23,NA +WA,B,Yes,9,14396,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,9.012503361857162,131.788880911984,18.025006723714323,1,2023-10-23,NA +NM,C,yes,10,27604,5,1000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,1.1082975470742826,300.4134833785745,1.1156696951268212,0,2023-10-23,no +CA,A,YES,3,84531,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,309.45075397965945,64.66287637434947,22.557679957367,0,2023-10-24,NA +CA,A,YES,5,34687,3,1000000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,2.76090258095095,37.70967410350615,5.5218051619019,1,2023-10-24,yes +CA,A,YES,3,84531,4,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,71.3088633548584,250.45303944975612,2.7755499950332636,0,2023-10-24,NA +CA,A,YES,4,12078,4,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,29.667120199120983,31.21637045526279,7.226346554564121,0,2023-10-24,NA +CA,A,YES,5,34687,4,1000000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,87.45380790460942,6.861979449729165,7.862762685921604,0,2023-10-24,NA +WA,B,Yes,8,97385,2,50000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,377.1866824467282,53.1831159463672,10.887618473014689,0,2023-10-24,temperature not assessed upon arrival at the laboratory +WA,B,Yes,9,14396,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,100.33135117141322,935.8984934341171,1.104547165455358,0,2023-10-24,NA +NM,C,yes,10,27604,5,1000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,2.447357451632415,993.3049975674772,0.6964236539834632,0,2023-10-24,no +YY,E,No,14,09634,3,13,upstream,primary sludge,copies/g dry sludge,sars-cov-2,7.876483345830302,600.8077486803925,6.542890033523782,0,2023-10-24,NA +CA,A,YES,3,84531,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,5.297762944187445,66.20056690250102,16.282743464093173,0,2023-10-25,NA +CA,A,YES,4,12078,3,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,195.91191041260308,290.3665167377153,5.5177050960528184,0,2023-10-25,NA +CA,A,YES,3,84531,4,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,5.47625848293571,952.9421341352993,1.9998292009158356,0,2023-10-25,no +CA,A,YES,4,12078,4,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,19.29051370757508,19.419171942994353,1.2563711246328657,0,2023-10-25,NA +WA,B,Yes,8,97385,2,50000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,681.5040502626623,7.0117077661785014,3.852579978785611,0,2023-10-25,NA +WA,B,Yes,9,14396,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,8.705326567473751,500.6064201342458,7.388921007891184,0,2023-10-25,NA +NM,C,yes,10,27604,5,1000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,1.26942115663288,8.995025095420347,1.2401015229816585,0,2023-10-25,> max temp and/or hold time +XX,D,ye,15,98436,10,1000000000,wwtp,raw wastewater,copies/l wastewater,FLUAV,6.428228368624624,198.56343615193552,1.0266461386453336,0,2023-10-25,NA +ZZ,F,NO,15,98436,4,8472,wwtp,raw wastewater,copies/g dry sludge,sars-cov-2,42.92907788204936,805.887785002305,1.1724784203148468,0,2023-10-25,NA +CA,A,YES,3,84531,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,739.6002492749661,83.94212948183987,3.126052978669926,0,2023-10-26,NA +CA,A,YES,3,84531,4,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,721.5120665044292,589.2979074291899,5.598324463852309,0,2023-10-26,NA +CA,A,YES,4,12078,4,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,209.6663803866837,398.77366861863317,1.1622040955046566,0,2023-10-26,NA +WA,B,Yes,8,97385,2,50000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,172.35772258650223,85.59644730246501,18.75817332003118,0,2023-10-26,NA +WA,B,Yes,9,14396,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,23.53691085291629,679.5261838838637,1.5328592280460662,0,2023-10-26,NA +NM,C,yes,10,27604,5,1000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,2.3924379144472154,5.7610450138235505,0.7506544627129327,0,2023-10-26,no +XX,D,ye,14,09634,10,1000000000,wwtp,raw wastewater,copies/l wastewater,FLUAV,1.140998244207903,83.13074347049407,2.281996488415806,1,2023-10-26,NA +ZZ,F,NO,15,98436,4,8472,wwtp,raw wastewater,copies/g dry sludge,sars-cov-2,58.958967756575724,217.0814582381969,6.731468772528137,0,2023-10-26,no +CA,A,YES,4,12078,3,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,1.788096450288146,781.9842754351879,3.576192900576292,1,2023-10-27,NA +CA,A,YES,3,84531,4,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,13.029781811123328,38.41041913635569,13.543257773318713,0,2023-10-27,NA +WA,B,Yes,8,97385,2,50000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,11.749268768284553,7.895339330564046,7.320231659001802,0,2023-10-27,no +WA,B,Yes,9,14396,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,251.07084085112646,10.931221205892783,1.148475330539271,0,2023-10-27,NA +NM,C,yes,10,27604,5,1000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,1.1102710186452773,542.6389353833116,1.3195624061201137,0,2023-10-27,NA +XX,D,ye,13,65174,10,1000000000,wwtp,raw wastewater,copies/l wastewater,FLUAV,2.9626077565551148,15.742030099352489,5.9252155131102295,1,2023-10-27,NA +CA,A,YES,3,84531,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,23.151357588493724,12.85901710353788,9.497656434414187,0,2023-10-28,NA +CA,A,YES,4,12078,3,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,10.128274268275781,136.6203139538166,20.256548536551563,1,2023-10-28,NA +CA,A,YES,3,84531,4,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,0.5309558335251195,21.499993526558665,1.061911667050239,1,2023-10-28,NA +CA,A,YES,4,12078,4,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,691.7570647807544,784.7243574747406,16.46625356463624,0,2023-10-28,below loq +WA,B,Yes,9,14396,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,24.34123626792319,20.80316131189279,1.2752781453619355,0,2023-10-28,y +NM,C,yes,10,27604,5,1000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,2.1116601897510496,18.642171228660473,0.8922795323498025,0,2023-10-28,NA +XX,D,ye,14,09634,10,1000000000,wwtp,raw wastewater,copies/l wastewater,FLUAV,24.41487986435696,22.66144922848546,3.317744367366803,0,2023-10-28,NA +CA,A,YES,3,84531,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,580.5611124011879,272.3514115460085,2.831896459437785,0,2023-10-29,NA +CA,A,YES,4,12078,3,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,45.81199912874592,144.3482380365069,1.3314936241432842,0,2023-10-29,NA +CA,A,YES,3,84531,4,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,23.217279629578062,24.72135487735582,5.815106208956857,0,2023-10-29,NA +WA,B,Yes,8,97385,2,50000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,170.89055791054858,51.569700948204755,5.604883533987637,0,2023-10-29,NA +WA,B,Yes,9,14396,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,2.393795746246259,46.993642641220326,4.787591492492518,1,2023-10-29,no +NM,C,yes,10,27604,5,1000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,1.263817265872908,165.05810728692956,1.1040384534075787,0,2023-10-29,NA +YY,E,No,15,98436,4,13,upstream,primary sludge,copies/g dry sludge,sars-cov-2,459.54128721766295,524.3957445798312,14.762828753023754,0,2023-10-29,no +CA,A,YES,3,84531,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,769.7345508091349,815.2777657853723,13.64851103358145,0,2023-10-30,y +CA,A,YES,4,12078,3,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,33.78582222711176,5.029160208468522,3.2029295831144675,0,2023-10-30,no +CA,A,YES,3,84531,4,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,22.983090454428666,297.0109976498183,24.405735080988762,0,2023-10-30,NA +CA,A,YES,4,12078,4,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,9.984746426574418,355.8014579187878,11.1482373630055,0,2023-10-30,NA +WA,B,Yes,8,97385,2,50000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,6.752539906990167,5.600656476576822,2.9906492093018584,0,2023-10-30,NA +WA,B,Yes,9,14396,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,5.035310492720919,33.126685324204594,19.6514526523485,0,2023-10-30,no +NM,C,yes,10,27604,5,1000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,1.2484154397811835,14.676253632432395,1.1470626654234641,0,2023-10-30,> max temp and/or hold time +ZZ,F,NO,14,09634,3,8472,wwtp,raw wastewater,copies/g dry sludge,sars-cov-2,36.94002793537277,508.1924770040036,8.706057221494842,0,2023-10-30,NA +CA,A,YES,3,84531,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,269.26720584969314,16.67380736422907,2.6969377942786568,0,2023-10-31,NA +CA,A,YES,4,12078,3,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,899.9166179825222,183.52784086245305,12.641026403363474,0,2023-10-31,NA +CA,A,YES,5,34687,3,1000000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,6.394103512602598,573.0906631170873,12.788207025205196,1,2023-10-31,NA +CA,A,YES,3,84531,4,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,30.06847239697734,75.69458011229106,2.969114104493046,0,2023-10-31,NA +CA,A,YES,5,34687,4,1000000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,0.5302541969725236,129.378000830073,1.0605083939450473,1,2023-10-31,NA +WA,B,Yes,8,97385,2,50000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,504.75228370348555,369.5377132902841,9.387637614836358,0,2023-10-31,NA +WA,B,Yes,9,14396,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,583.5991365062471,5.529402768771192,14.452899224937328,0,2023-10-31,y +XX,D,ye,13,65174,10,1000000000,wwtp,raw wastewater,copies/l wastewater,FLUAV,10.051240027085301,119.16473266852991,20.102480054170602,1,2023-10-31,NA +CA,A,YES,3,84531,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,28.69703924526375,11.464713320177978,10.731973839165256,0,2023-11-01,NA +CA,A,YES,4,12078,3,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,42.923515876257945,249.02731312055343,2.2749360533587684,0,2023-11-01,NA +CA,A,YES,3,84531,4,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,228.89090730823332,52.21791522323911,12.187661915377944,0,2023-11-01,no +CA,A,YES,4,12078,4,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,332.12831507522054,121.66825807427423,7.861778142216846,0,2023-11-01,y +WA,B,Yes,8,97385,2,50000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,6.562497749214034,24.22970644310019,3.867880211440783,0,2023-11-01,NA +WA,B,Yes,9,14396,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,46.6438871417938,18.795635494280692,1.0699955012597828,0,2023-11-01,NA +NM,C,yes,10,27604,5,1000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,0.8204175500149606,16.34123446859702,0.03914264758524947,0,2023-11-01,temperature not assessed upon arrival at the laboratory +CA,A,YES,3,84531,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,95.47047432725927,288.6295570361401,1.3690630660390806,0,2023-11-02,NA +CA,A,YES,4,12078,3,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,26.00997906148354,214.99061120351823,19.22811980611785,0,2023-11-02,y +CA,A,YES,4,12078,4,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,1.0414183344111156,71.86994693017198,2.0828366688222313,1,2023-11-02,NA +WA,B,Yes,8,97385,2,50000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,354.14187444993246,178.36221651011323,21.79070109459368,0,2023-11-02,no +WA,B,Yes,9,14396,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,3.7807274080380058,116.667897580627,7.5614548160760116,1,2023-11-02,no +NM,C,yes,10,27604,5,1000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,2.3902822572381686,33.72288275476988,0.7757594851468794,0,2023-11-02,no +CA,A,YES,3,84531,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,0.6773663898890263,200.96973535388744,1.3547327797780526,1,2023-11-03,NA +CA,A,YES,4,12078,3,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,121.52269246280062,13.769447562652998,5.849950742033164,0,2023-11-03,no +CA,A,YES,3,84531,4,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,16.84398544774963,383.31080426744256,4.086622753380791,0,2023-11-03,yes +CA,A,YES,4,12078,4,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,39.000009246879166,130.0008048991091,9.846731535460002,0,2023-11-03,NA +WA,B,Yes,8,97385,2,50000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,10.532784430810398,28.57516435962721,14.642323982256304,0,2023-11-03,NA +WA,B,Yes,9,14396,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,63.258855605650936,7.33243729767299,14.635115488163226,0,2023-11-03,NA +NM,C,yes,10,27604,5,1000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,1.468526574945611,156.51548885880672,1.0738855193706154,0,2023-11-03,no +XX,D,ye,13,65174,10,1000000000,wwtp,raw wastewater,copies/l wastewater,FLUAV,233.699612361303,89.1419812021984,19.823576435504965,0,2023-11-03,NA +CA,A,YES,3,84531,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,6.250746368711096,12.080731555409574,12.501492737422192,1,2023-11-04,no +CA,A,YES,4,12078,3,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,60.734476847453664,99.66198503857115,3.7493705219237743,0,2023-11-04,NA +CA,A,YES,3,84531,4,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,0.7799320699735204,467.3744394399941,1.5598641399470408,1,2023-11-04,temperature not assessed upon arrival at the laboratory +CA,A,YES,4,12078,4,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,41.353064095648925,211.98303591158268,24.692401665883022,0,2023-11-04,NA +WA,B,Yes,8,97385,2,50000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,94.2678381257388,246.33825157228895,7.020286492550806,0,2023-11-04,NA +WA,B,Yes,9,14396,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,138.04075862539347,43.157070438302654,12.281487212140066,0,2023-11-04,no +NM,C,yes,10,27604,5,1000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,1.758385091950957,11.36416749982378,0.035947124101505544,0,2023-11-04,NA +ZZ,F,NO,14,09634,3,8472,wwtp,raw wastewater,copies/g dry sludge,sars-cov-2,117.02328638635942,14.523259510070451,1.0067504990671456,0,2023-11-04,NA +CA,A,YES,3,84531,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,44.88602489025539,8.272052534180705,1.2868955707053842,0,2023-11-05,NA +CA,A,YES,4,12078,3,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,48.58861525749648,846.4470200747313,1.4227710519469416,0,2023-11-05,NA +CA,A,YES,3,84531,4,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,8.311333643471128,10.980412346399168,6.158218214457452,0,2023-11-05,NA +CA,A,YES,4,12078,4,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,16.618376997399402,66.21018929400746,3.961933654951957,0,2023-11-05,NA +WA,B,Yes,8,97385,2,50000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,1.222659822594126,389.8313052784647,2.445319645188252,1,2023-11-05,NA +WA,B,Yes,9,14396,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,104.96850856807576,260.57750538917736,2.9562985989831194,0,2023-11-05,NA +XX,D,ye,14,09634,10,1000000000,wwtp,raw wastewater,copies/l wastewater,FLUAV,937.0771105159462,110.7558617499176,10.11368872182712,0,2023-11-05,result is not quantifiable +CA,A,YES,3,84531,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,1.250101091272628,9.76231345246801,2.500202182545256,1,2023-11-06,NA +CA,A,YES,3,84531,4,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,36.33283874016626,155.07444948869752,21.18515612958183,0,2023-11-06,NA +CA,A,YES,4,12078,4,200000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,49.60200830858396,14.96239623181222,7.805525767505929,0,2023-11-06,NA +WA,B,Yes,8,97385,2,50000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,99.45035538605052,11.63330921074691,1.0801383077051412,0,2023-11-06,result is not quantifiable +WA,B,Yes,9,14396,2,600000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,2.572009643690199,79.23499776086382,5.144019287380398,1,2023-11-06,NA +NM,C,yes,10,27604,5,1000,wwtp,post grit removal,log10 copies/l wastewater,sars-cov-2,0.7525598001813789,889.6181725911234,0.32341560339452874,0,2023-11-06,NA +CA,A,YES,1,"04971,36910",1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,8.054115836944005,55.04513393435594,7.443448737491519,0,2023-01-01,NA +CA,A,YES,1,"04971,36910",1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,318.5878534041641,32.51171069583552,1.9747551214710093,0,2023-01-02,no +CA,A,YES,1,"04971,36910",1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,8.045477208790313,40.27750071592156,3.3258959869434412,0,2023-01-03,no +CA,A,YES,1,"04971,36910",1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,1.7936991416491015,7.336498654648253,3.587398283298203,1,2023-01-04,NA +CA,A,YES,1,"04971,36910",1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,0.6741893206279964,20.52304747265837,1.3483786412559928,1,2023-01-05,no +CA,A,YES,1,"04971,36910",1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,11.917227178713146,18.02822613418176,2.7315453798288956,0,2023-01-06,NA +CA,A,YES,1,"04971,36910",1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,521.4482122070327,177.2417430876511,1.0293360181042364,0,2023-01-07,NA +CA,A,YES,1,"04971,36910",1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,131.30679750544832,359.54013723174427,1.7967577817836142,0,2023-01-08,NA +CA,A,YES,1,"04971,36910",1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,10.145581141014883,38.471620515982444,4.204687765288971,0,2023-01-09,NA +CA,A,YES,1,"04971,36910",1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,25.78457079057178,5.312387660424927,8.871589067317199,0,2023-01-10,NA +CA,A,YES,1,"04971,36910",1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,12.286683603569443,355.21081413629145,24.573367207138887,1,2023-01-11,no +CA,A,YES,1,"04971,36910",1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,5.805642914082082,24.45507289087341,2.313825261175917,0,2023-01-12,NA +CA,A,YES,1,"04971,36910",1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,164.05594243893796,395.07512339116994,6.521506611074547,0,2023-01-13,NA +CA,A,YES,1,"04971,36910",1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,2.8135112007183563,43.47218034808254,5.6270224014367125,1,2023-01-15,NA +CA,A,YES,1,"04971,36910",1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,632.8401179640316,12.224419039706119,2.4220839813602884,0,2023-01-16,NA +CA,A,YES,1,"04971,36910",1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,236.3829208832575,70.41303977777574,22.48688884133083,0,2023-01-17,no +CA,A,YES,1,"04971,36910",1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,289.68812960182566,356.71546463539164,5.989158818609282,0,2023-01-18,NA +CA,A,YES,1,"04971,36910",1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,7.79602795207444,19.92412929908597,1.4943600748960477,0,2023-01-19,NA +CA,A,YES,1,"04971,36910",1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,103.2642046732852,5.814870566547347,11.602114739709226,0,2023-01-20,NA +CA,A,YES,1,"04971,36910",1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,260.509256741051,749.4630810279345,6.307817618056269,0,2023-01-21,NA +CA,A,YES,1,"04971,36910",1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,40.45204295439143,13.660265247606043,13.910212931525614,0,2023-01-22,NA +CA,A,YES,1,"04971,36910",1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,17.666520139477633,10.127875083418793,19.803401555625793,0,2023-01-23,NA +CA,A,YES,1,"04971,36910",1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,328.77787163502705,83.36518324074643,1.2192699469400203,0,2023-01-24,NA +CA,A,YES,1,"04971,36910",1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,138.33302679956105,360.73327421400006,21.9041912020723,0,2023-01-25,NA +CA,A,YES,1,"04971,36910",1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,9.328578762817422,8.84809728269201,3.7340129757787976,0,2023-01-26,NA +CA,A,YES,1,"04971,36910",1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,6.624234663444247,676.3758121890099,13.248469326888493,1,2023-01-27,NA +CA,A,YES,1,"04971,36910",1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,217.18311574688119,210.68691679343732,1.083315522601008,0,2023-01-28,> max temp and/or hold time +CA,A,YES,1,"04971,36910",1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,289.6888018012041,12.186631110177862,5.699175937236678,0,2023-01-29,no +CA,A,YES,1,"04971,36910",1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,2.4337899540452472,541.6026112926763,4.8675799080904945,1,2023-01-30,NA +CA,A,YES,1,"04971,36910",1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,13.82721164208279,6.021428455246692,1.0250892949476675,0,2023-01-31,NA +CA,A,YES,1,"04971,36910",1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,66.73897564847593,6.38105523650712,3.8713058962796816,0,2023-02-01,NA +CA,A,YES,1,"04971,36910",1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,13.047102928555004,51.050394457828354,20.101885729743387,0,2023-02-02,below loq +CA,A,YES,1,"04971,36910",1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,25.71421568374014,238.04162383662378,10.183439487702529,0,2023-02-04,no +CA,A,YES,1,"04971,36910",1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,689.8299461380375,88.05279343129827,3.073115833003546,0,2023-02-05,NA +CA,A,YES,1,"04971,36910",1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,487.1800743189516,26.80129255300873,13.729088207939094,0,2023-02-06,no +CA,A,YES,1,"04971,36910",1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,22.145805633858085,53.19228646653339,1.796493213395071,0,2023-02-07,y +CA,A,YES,1,"04971,36910",1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,77.42659147330004,6.05283506543072,2.349344073250793,0,2023-02-08,NA +CA,A,YES,1,"04971,36910",1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,812.9826106403942,36.26527894451485,9.318355394326483,0,2023-02-09,NA +CA,A,YES,1,"04971,36910",1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,612.8199257092496,264.1565679134045,15.825119680643422,0,2023-02-10,NA +CA,A,YES,1,"04971,36910",1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,314.7503986972577,355.80233751648694,12.63157077178409,0,2023-02-11,NA +CA,A,YES,1,"04971,36910",1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,81.25881347500219,653.9280096040354,2.6648826415257783,0,2023-02-13,NA +CA,A,YES,1,"04971,36910",1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,2.50735666511393,333.1255535218914,5.01471333022786,1,2023-02-14,NA +CA,A,YES,1,"04971,36910",1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,5.052700248787683,15.74826735545662,10.105400497575365,1,2023-02-15,no +CA,A,YES,1,"04971,36910",1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,6.804398813682943,89.56254239127894,13.608797627365886,1,2023-02-16,NA +CA,A,YES,1,"04971,36910",1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,14.382556035028882,97.66500858852643,19.278770342928855,0,2023-02-17,NA +CA,A,YES,1,"04971,36910",1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,9.475867017783791,246.73122089405535,18.951734035567583,1,2023-02-18,NA +CA,A,YES,1,"04971,36910",1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,66.40146942636754,21.070560662823272,18.06404763401332,0,2023-02-20,NA +CA,A,YES,1,"04971,36910",1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,316.15662584248014,58.139235913183434,17.60328449458461,0,2023-02-21,NA +CA,A,YES,1,"04971,36910",1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,97.43244440599466,166.25369812836306,3.073760381121263,0,2023-02-22,NA +CA,A,YES,1,"04971,36910",1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,387.1038296292479,81.13681422221168,4.537393024553714,0,2023-02-23,NA +CA,A,YES,1,"04971,36910",1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,10.859558090606194,467.0747242248433,23.913992786438158,0,2023-02-24,no +CA,A,YES,1,"04971,36910",1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,128.54700185544306,205.41718468186758,3.8516288229128457,0,2023-02-25,NA +CA,A,YES,1,"04971,36910",1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,22.112115170593434,79.0814556891465,7.82263396630389,0,2023-02-26,NA +CA,A,YES,1,"04971,36910",1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,30.137088470279455,494.0285027347975,3.0394003409320747,0,2023-02-27,NA +CA,A,YES,1,"04971,36910",1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,543.1175903211706,615.6357607608519,4.409013104344052,0,2023-02-28,NA +CA,A,YES,1,"04971,36910",1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,13.251298657542206,94.02223533504265,10.272813536625328,0,2023-03-01,NA +CA,A,YES,1,"04971,36910",1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,6.080179057593754,7.901916299409575,5.366037941447647,0,2023-03-02,NA +CA,A,YES,1,"04971,36910",1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,31.088129916068763,5.6395919830043235,1.1282655157766144,0,2023-03-03,NA +CA,A,YES,1,"04971,36910",1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,158.55850221202064,369.6975606751693,7.462109862894364,0,2023-03-04,NA +CA,A,YES,1,"04971,36910",1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,10.754474756704594,255.4304579860096,1.8140997306783964,0,2023-03-05,no +CA,A,YES,1,"04971,36910",1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,708.0987990107047,47.13318907166913,12.138428684072297,0,2023-03-06,NA +CA,A,YES,1,"04971,36910",1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,199.9498140556388,11.84964334047132,14.718438267591178,0,2023-03-07,NA +CA,A,YES,1,"04971,36910",1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,802.3550464348742,93.8516436328436,1.7587376183625947,0,2023-03-08,NA +CA,A,YES,1,"04971,36910",1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,565.5905896213546,10.326294021590266,7.371266777590111,0,2023-03-09,NA +CA,A,YES,1,"04971,36910",1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,115.39074259807938,235.9585444770106,20.74287530120597,0,2023-03-10,NA +CA,A,YES,1,"04971,36910",1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,458.2185722722606,81.68467683478113,1.567411435737819,0,2023-03-11,NA +CA,A,YES,1,"04971,36910",1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,166.94136827508945,5.539390170211123,7.808403195655328,0,2023-03-12,NA +CA,A,YES,1,"04971,36910",1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,792.0931373110479,8.762107070715258,8.618496350940111,0,2023-03-13,NA +CA,A,YES,1,"04971,36910",1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,4.374859641075281,57.47609650597885,8.749719282150561,1,2023-03-14,NA +CA,A,YES,1,"04971,36910",1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,82.7827970998116,15.184501160544444,2.1389600988131803,0,2023-03-15,NA +CA,A,YES,1,"04971,36910",1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,8.672654771899738,5.454841947027963,14.751477956518597,0,2023-03-16,NA +CA,A,YES,1,"04971,36910",1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,29.246057749763843,41.93822503967432,1.7268634206436253,0,2023-03-17,NA +CA,A,YES,1,"04971,36910",1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,87.32050764429154,10.643799430398044,1.0545988000977582,0,2023-03-18,no +CA,A,YES,1,"04971,36910",1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,0.5448361754382418,37.791788520690055,1.0896723508764836,1,2023-03-19,no +CA,A,YES,1,"04971,36910",1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,33.394745995145854,39.31634484551162,17.627843578230518,0,2023-03-20,NA +CA,A,YES,1,"04971,36910",1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,72.91747979089877,19.917617521251366,13.47828358468021,0,2023-03-21,NA +CA,A,YES,1,"04971,36910",1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,996.1543331368099,9.02367826861503,1.2487248394080674,0,2023-03-22,NA +CA,A,YES,1,"04971,36910",1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,13.017321446475458,271.37718198121854,18.40013057788529,0,2023-03-23,temperature not assessed upon arrival at the laboratory +CA,A,YES,1,"04971,36910",1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,57.19019343505626,10.285016207169908,22.79070573429898,0,2023-03-24,NA +CA,A,YES,1,"04971,36910",1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,218.27243814724895,312.00042051144555,20.229522333360077,0,2023-03-25,NA +CA,A,YES,1,"04971,36910",1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,8.807374893730517,9.54018147503281,6.126507865610079,0,2023-03-26,NA +CA,A,YES,1,"04971,36910",1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,73.4268981868111,131.858283598738,3.4136857352345937,0,2023-03-27,NA +CA,A,YES,1,"04971,36910",1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,6.682849019380353,282.9619057651956,1.0279225570177555,0,2023-03-28,NA +CA,A,YES,1,"04971,36910",1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,75.62091664975829,21.40665496542848,13.264487438945563,0,2023-03-29,NA +CA,A,YES,1,"04971,36910",1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,1.6324382532577906,819.4494291923656,3.264876506515581,1,2023-03-30,NA +CA,A,YES,1,"04971,36910",1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,35.20532643073089,9.248725349979356,15.886088075205711,0,2023-03-31,NA +CA,A,YES,1,"04971,36910",1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,390.3741886882092,194.96633941221833,11.890457486370696,0,2023-04-01,NA +CA,A,YES,1,"04971,36910",1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,9.059006455958267,10.381219237878499,18.118012911916534,1,2023-04-02,NA +CA,A,YES,1,"04971,36910",1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,590.6204299820803,39.18326309363071,7.69840360708392,0,2023-04-03,NA +CA,A,YES,1,"04971,36910",1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,282.5664507150931,99.44932473136679,5.496039422038657,0,2023-04-04,NA +CA,A,YES,1,"04971,36910",1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,56.97307164298722,224.66257416062479,16.17596585536359,0,2023-04-05,NA +CA,A,YES,1,"04971,36910",1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,784.899042119334,8.014518989701141,1.7527365477504029,0,2023-04-06,y +CA,A,YES,1,"04971,36910",1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,157.90319031704828,163.90882472260637,7.625228932162711,0,2023-04-07,NA +CA,A,YES,1,"04971,36910",1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,18.741526594607375,6.286134977955453,17.90925875056528,0,2023-04-08,NA +CA,A,YES,1,"04971,36910",1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,8.166963197060474,177.94206519240126,6.618345313066703,0,2023-04-09,NA +CA,A,YES,1,"04971,36910",1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,210.509691295845,624.5578615281554,17.273425323465588,0,2023-04-10,NA +CA,A,YES,1,"04971,36910",1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,260.17284607415934,327.2116063875679,1.4571587493996616,0,2023-04-11,> max temp and/or hold time +CA,A,YES,1,"04971,36910",1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,5.236194326347224,533.1325503441246,6.9827168044249746,0,2023-04-12,no +CA,A,YES,1,"04971,36910",1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,7.927165967825946,52.418847144635976,1.9441287429439593,0,2023-04-14,NA +CA,A,YES,1,"04971,36910",1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,8.498960817154165,6.087806077690802,3.7936991995095233,0,2023-04-15,result is not quantifiable +CA,A,YES,1,"04971,36910",1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,161.50883552236422,89.71084760429939,10.168423259482495,0,2023-04-16,NA +CA,A,YES,1,"04971,36910",1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,24.00532686771313,22.516942505563232,21.438766889349207,0,2023-04-17,NA +CA,A,YES,1,"04971,36910",1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,348.3430684735925,932.7512717430938,14.691412369816117,0,2023-04-18,NA +CA,A,YES,1,"04971,36910",1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,568.4551169471282,921.4530009141888,3.7972934877373823,0,2023-04-20,NA +CA,A,YES,1,"04971,36910",1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,466.6239673139718,219.33386254534756,3.664561808956291,0,2023-04-22,NA +CA,A,YES,1,"04971,36910",1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,265.93446063582985,5.607476230855946,10.197305945712793,0,2023-04-23,NA +CA,A,YES,1,"04971,36910",1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,11.963077032320644,13.399709308986886,7.498009327668975,0,2023-04-25,NA +CA,A,YES,1,"04971,36910",1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,66.21163639175714,114.48596232947426,16.12191225075471,0,2023-04-26,NA +CA,A,YES,1,"04971,36910",1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,208.04588021116533,103.55688479513707,2.691806067533322,0,2023-04-27,NA +CA,A,YES,1,"04971,36910",1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,6.999970333652621,291.4020800306635,1.7604127194300565,0,2023-04-28,NA +CA,A,YES,1,"04971,36910",1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,560.0658426399126,100.00078493635858,11.271197957537696,0,2023-04-29,NA +CA,A,YES,1,"04971,36910",1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,3.32450934958347,235.98845261349894,6.64901869916694,1,2023-04-30,NA +CA,A,YES,1,"04971,36910",1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,126.39909080021351,157.10491649611788,3.067204511505263,0,2023-05-01,NA +CA,A,YES,1,"04971,36910",1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,13.167240820226265,9.647033797931563,11.831287615485943,0,2023-05-02,no +CA,A,YES,1,"04971,36910",1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,75.45453801139261,174.94212865353848,2.1148952660069806,0,2023-05-03,NA +CA,A,YES,1,"04971,36910",1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,15.586661702644017,486.2775228649737,15.35829976263435,0,2023-05-04,NA +CA,A,YES,1,"04971,36910",1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,741.2838545573932,6.512754463154311,18.960706734517032,0,2023-05-05,NA +CA,A,YES,1,"04971,36910",1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,8.48954291484976,39.35578831757263,2.0469024097072297,0,2023-05-06,below loq +CA,A,YES,1,"04971,36910",1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,58.809195500389826,152.84300810289068,3.3717870985386202,0,2023-05-07,y +CA,A,YES,1,"04971,36910",1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,130.27569233977755,439.6737693684355,23.251322844723248,0,2023-05-08,NA +CA,A,YES,1,"04971,36910",1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,303.5259117307216,618.6713389578282,13.139319300179181,0,2023-05-11,no +CA,A,YES,1,"04971,36910",1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,10.540565710845435,5.703475703147575,21.08113142169087,1,2023-05-12,NA +CA,A,YES,1,"04971,36910",1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,86.93925031392622,61.31553298100391,10.500151659860256,0,2023-05-13,NA +CA,A,YES,1,"04971,36910",1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,279.89140422897793,192.9022700619561,20.297971158600266,0,2023-05-14,NA +CA,A,YES,1,"04971,36910",1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,601.9975510440462,201.68876689786737,19.109713023668867,0,2023-05-15,temperature not assessed upon arrival at the laboratory +CA,A,YES,1,"04971,36910",1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,5.399639999551275,32.131818045827,2.5192556599572704,0,2023-05-16,NA +CA,A,YES,1,"04971,36910",1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,38.82271578473764,377.3869222461186,2.863484552043326,0,2023-05-17,NA +CA,A,YES,1,"04971,36910",1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,146.38831947568443,172.829198192071,16.482343278862913,0,2023-05-18,NA +CA,A,YES,1,"04971,36910",1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,5.002846167617119,16.50332136529176,10.005692335234238,1,2023-05-19,NA +CA,A,YES,1,"04971,36910",1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,865.8282311572634,176.39108944087667,6.302458811424311,0,2023-05-20,no +CA,A,YES,1,"04971,36910",1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,224.07955465704714,11.129784849158769,4.19206971903038,0,2023-05-22,NA +CA,A,YES,1,"04971,36910",1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,56.435957943803174,73.85178932618149,18.18520482551056,0,2023-05-23,NA +CA,A,YES,1,"04971,36910",1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,13.157355805362196,153.7932692351134,1.7738913709877437,0,2023-05-24,> max temp and/or hold time +CA,A,YES,1,"04971,36910",1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,5.827507281480294,5.614635812615721,17.492705146845655,0,2023-05-25,NA +CA,A,YES,1,"04971,36910",1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,991.974730854174,43.804695804123156,13.012253586755072,0,2023-05-26,NA +CA,A,YES,1,"04971,36910",1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,3.8722209834809966,11.874904724137725,7.744441966961993,1,2023-05-28,below loq +CA,A,YES,1,"04971,36910",1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,134.56052411216203,68.35616771771406,4.597724848363917,0,2023-05-29,NA +CA,A,YES,1,"04971,36910",1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,5.221786941619451,6.320443859538269,1.3314827910739935,0,2023-05-30,NA +CA,A,YES,1,"04971,36910",1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,35.06678316756629,913.5850921445049,1.581489785807367,0,2023-05-31,no +CA,A,YES,1,"04971,36910",1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,13.972497746966296,14.097585562510597,9.951165894532037,0,2023-06-01,NA +CA,A,YES,1,"04971,36910",1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,8.233667313492495,11.157350694770544,6.377286299468939,0,2023-06-03,NA +CA,A,YES,1,"04971,36910",1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,132.28786729786012,471.13289773261994,14.410948003082948,0,2023-06-04,NA +CA,A,YES,1,"04971,36910",1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,11.883770711884882,159.78548776673784,8.191422399765973,0,2023-06-08,NA +CA,A,YES,1,"04971,36910",1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,3.062617732302472,20.095139119442187,6.125235464604944,1,2023-06-09,no +CA,A,YES,1,"04971,36910",1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,60.104711478529254,5.608352642439963,7.698766320975435,0,2023-06-10,temperature not assessed upon arrival at the laboratory +CA,A,YES,1,"04971,36910",1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,121.00486682545225,41.39264660940628,17.858118605185556,0,2023-06-11,NA +CA,A,YES,1,"04971,36910",1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,285.1635796114686,104.64656733085897,4.308534565625693,0,2023-06-12,NA +CA,A,YES,1,"04971,36910",1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,327.3311918923494,55.86301140342441,4.117629955751652,0,2023-06-13,NA +CA,A,YES,1,"04971,36910",1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,290.0672745142428,144.404870087013,4.451449957611034,0,2023-06-14,NA +CA,A,YES,1,"04971,36910",1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,47.08489285130636,610.0219388403381,5.917075486246098,0,2023-06-15,NA +CA,A,YES,1,"04971,36910",1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,96.40070617203892,9.540281193184393,2.460538013144499,0,2023-06-16,NA +CA,A,YES,1,"04971,36910",1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,24.243481229112778,10.742240797003861,12.503952415457558,0,2023-06-17,NA +CA,A,YES,1,"04971,36910",1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,493.18207031912567,198.36030538085393,11.390638088057084,0,2023-06-18,no +CA,A,YES,1,"04971,36910",1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,47.75410667127288,79.12621599215444,1.1900373986193729,0,2023-06-20,NA +CA,A,YES,1,"04971,36910",1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,195.9009104592535,15.002948692432616,9.991368118457647,0,2023-06-21,no +CA,A,YES,1,"04971,36910",1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,4.0966415673043555,113.83743480845315,8.193283134608711,1,2023-06-22,NA +CA,A,YES,1,"04971,36910",1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,22.573421285399924,949.051520476134,3.535377539088544,0,2023-06-24,NA +CA,A,YES,1,"04971,36910",1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,347.2523830430573,351.4116188477411,1.6635794733977365,0,2023-06-26,NA +CA,A,YES,1,"04971,36910",1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,25.858403322348636,28.242645711496458,19.453488209921662,0,2023-06-27,NA +CA,A,YES,1,"04971,36910",1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,123.69060013469947,61.01617619804707,8.720083252613676,0,2023-06-28,result is not quantifiable +CA,A,YES,1,"04971,36910",1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,10.019380902282103,12.758316383889646,3.589310259088568,0,2023-06-29,NA +CA,A,YES,1,"04971,36910",1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,935.1444950109765,12.244724479173952,1.7206828159565872,0,2023-06-30,NA +CA,A,YES,1,"04971,36910",1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,73.32340920826366,287.4803390320116,2.0635163304229067,0,2023-07-01,NA +CA,A,YES,1,"04971,36910",1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,0.6849727543487996,523.395297367423,1.3699455086975991,1,2023-07-02,NA +CA,A,YES,1,"04971,36910",1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,511.61411497329766,62.63682789928954,3.5404402095624605,0,2023-07-03,no +CA,A,YES,1,"04971,36910",1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,541.1218770752313,203.98987755029955,13.794798866774372,0,2023-07-04,NA +CA,A,YES,1,"04971,36910",1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,24.388026570905993,981.6321747909421,6.177275606147465,0,2023-07-05,NA +CA,A,YES,1,"04971,36910",1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,28.11953184261253,99.94044777851639,2.794650281477016,0,2023-07-06,NA +CA,A,YES,1,"04971,36910",1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,625.5325312472202,319.7882667185706,1.9388145516085369,0,2023-07-07,NA +CA,A,YES,1,"04971,36910",1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,12.351905330391771,210.4768920078485,24.703810660783542,1,2023-07-08,NA +CA,A,YES,1,"04971,36910",1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,2.218836283267302,73.69835255749938,4.437672566534604,1,2023-07-10,NA +CA,A,YES,1,"04971,36910",1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,7.717464574326779,16.383796256828802,20.529840694664493,0,2023-07-11,NA +CA,A,YES,1,"04971,36910",1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,7.944992394163731,837.4203115687657,1.5976617777047457,0,2023-07-12,no +CA,A,YES,1,"04971,36910",1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,41.51428591660013,81.41641651292139,1.584614391186242,0,2023-07-13,no +CA,A,YES,1,"04971,36910",1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,0.93826933952755,44.69264572935639,1.8765386790551,1,2023-07-14,no +CA,A,YES,1,"04971,36910",1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,795.655576312077,708.0869856489144,23.358835684662168,0,2023-07-15,NA +CA,A,YES,1,"04971,36910",1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,6.551935780466369,13.333930645682143,13.103871560932738,1,2023-07-16,no +CA,A,YES,1,"04971,36910",1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,5.882346411121828,39.322479253267844,11.764692822243656,1,2023-07-17,> max temp and/or hold time +CA,A,YES,1,"04971,36910",1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,241.68723320654166,111.96442647581554,19.774130720183855,0,2023-07-18,NA +CA,A,YES,1,"04971,36910",1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,43.394035193624006,39.46900061049787,5.129002133042138,0,2023-07-19,NA +CA,A,YES,1,"04971,36910",1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,577.5598577313433,163.07337147206727,24.214569522151354,0,2023-07-21,no +CA,A,YES,1,"04971,36910",1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,11.032938404219838,67.90345189717881,7.6546364904309225,0,2023-07-22,NA +CA,A,YES,1,"04971,36910",1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,421.5581390603822,50.35463879070723,8.61281044196096,0,2023-07-24,y +CA,A,YES,1,"04971,36910",1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,23.610997025519655,5.371442831117368,16.501544238849036,0,2023-07-25,NA +CA,A,YES,1,"04971,36910",1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,7.4285751551321955,19.457111516742547,7.053586036128444,0,2023-07-26,temperature not assessed upon arrival at the laboratory +CA,A,YES,1,"04971,36910",1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,5.055534203854524,176.16206933310832,16.59448203988224,0,2023-07-27,> max temp and/or hold time +CA,A,YES,1,"04971,36910",1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,6.757122856022711,482.460205052603,10.015110580420062,0,2023-07-28,NA +CA,A,YES,1,"04971,36910",1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,18.784296198219966,24.456558953248408,1.2534572274074924,0,2023-07-29,NA +CA,A,YES,1,"04971,36910",1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,43.089152719699264,435.73082145092775,3.6013387596174287,0,2023-07-30,NA +CA,A,YES,1,"04971,36910",1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,5.982668029827039,71.39805501927842,13.681632332433592,0,2023-08-01,NA +CA,A,YES,1,"04971,36910",1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,29.89881095401264,644.8783031094963,2.1379496979228128,0,2023-08-02,NA +CA,A,YES,1,"04971,36910",1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,15.245052164050056,14.601654948065194,9.130847682327424,0,2023-08-03,NA +CA,A,YES,1,"04971,36910",1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,445.18973523111055,46.58244883444312,1.69920384070378,0,2023-08-04,NA +CA,A,YES,1,"04971,36910",1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,56.64386164261395,8.884665829339795,5.669307914328272,0,2023-08-05,NA +CA,A,YES,1,"04971,36910",1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,25.482224251359003,994.4385402519111,2.869294462719715,0,2023-08-07,NA +CA,A,YES,1,"04971,36910",1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,43.06626276789402,6.304198094739177,19.154037997448203,0,2023-08-08,NA +CA,A,YES,1,"04971,36910",1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,229.18755885572608,18.445161479616353,13.17444100390749,0,2023-08-10,no +CA,A,YES,1,"04971,36910",1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,17.87839963583158,353.7913947879838,2.386031352430203,0,2023-08-11,NA +CA,A,YES,1,"04971,36910",1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,341.5319622331677,22.535336369918106,2.9009542439153364,0,2023-08-12,NA +CA,A,YES,1,"04971,36910",1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,4.671492064147434,70.30665174981034,9.342984128294868,1,2023-08-13,NA +CA,A,YES,1,"04971,36910",1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,236.9049470981896,10.503899757854246,1.8661087177927753,0,2023-08-14,NA +CA,A,YES,1,"04971,36910",1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,0.6768424203702207,262.5342829784635,1.3536848407404414,1,2023-08-15,no +CA,A,YES,1,"04971,36910",1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,13.750449354745616,97.53703067606361,7.9317837944845575,0,2023-08-16,NA +CA,A,YES,1,"04971,36910",1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,4.22681496611314,130.38668828033036,8.45362993222628,1,2023-08-17,NA +CA,A,YES,1,"04971,36910",1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,31.75325288550397,237.1484455656829,13.23501170557511,0,2023-08-18,no +CA,A,YES,1,"04971,36910",1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,667.8359155793845,30.681900301625824,1.9258804785929342,0,2023-08-19,NA +CA,A,YES,1,"04971,36910",1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,61.413781917387254,15.875257689664238,24.332944170594313,0,2023-08-20,NA +CA,A,YES,1,"04971,36910",1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,127.03367781841675,7.099144687453064,20.461066572372655,0,2023-08-21,NA +CA,A,YES,1,"04971,36910",1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,26.30279892188309,21.490421548808033,23.74006180005436,0,2023-08-22,NA +CA,A,YES,1,"04971,36910",1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,121.05669770858235,14.212953467533138,2.99557871212313,0,2023-08-25,NA +CA,A,YES,1,"04971,36910",1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,109.00779424472073,495.6208223070091,2.3091115346315756,0,2023-08-27,no +CA,A,YES,1,"04971,36910",1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,2.6729273348013622,8.752613815741094,5.3458546696027245,1,2023-08-28,NA +CA,A,YES,1,"04971,36910",1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,23.083558937890217,678.3996529565197,2.3853349770620484,0,2023-08-29,NA +CA,A,YES,1,"04971,36910",1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,53.48263727151335,21.262408576525846,2.8500736421369344,0,2023-08-30,NA +CA,A,YES,1,"04971,36910",1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,592.4451254098414,72.18722222591065,12.061499723477331,0,2023-08-31,NA +CA,A,YES,1,"04971,36910",1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,748.2893178235532,25.84322755333694,16.723603429453362,0,2023-09-01,yes +CA,A,YES,1,"04971,36910",1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,39.57219130798205,769.0657781019385,3.9230363057513404,0,2023-09-02,below loq +CA,A,YES,1,"04971,36910",1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,430.2273711266606,49.10700461580872,12.024286675454327,0,2023-09-03,NA +CA,A,YES,1,"04971,36910",1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,786.0080000784335,106.71467557046878,12.403173656961675,0,2023-09-04,no +CA,A,YES,1,"04971,36910",1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,8.213217766641202,62.75875762506459,16.426435533282405,1,2023-09-05,NA +CA,A,YES,1,"04971,36910",1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,569.0592734875645,49.8948087609619,16.741583747541483,0,2023-09-06,NA +CA,A,YES,1,"04971,36910",1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,125.16062540964309,15.744586170221984,3.413231366066114,0,2023-09-07,NA +CA,A,YES,1,"04971,36910",1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,13.416099669710217,112.80358085277192,4.2191715782747234,0,2023-09-08,y +CA,A,YES,1,"04971,36910",1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,527.5990713281764,502.8502582656528,8.387939118671651,0,2023-09-09,NA +CA,A,YES,1,"04971,36910",1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,12.521220019300424,20.76965227783997,5.3834170132267305,0,2023-09-10,NA +CA,A,YES,1,"04971,36910",1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,56.9534624514444,366.1366633850493,1.292548620565424,0,2023-09-11,NA +CA,A,YES,1,"04971,36910",1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,901.0494424708787,461.91170796163334,8.579360686641467,0,2023-09-12,NA +CA,A,YES,1,"04971,36910",1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,603.0385591829224,135.6580883620457,4.139170907451966,0,2023-09-13,NA +CA,A,YES,1,"04971,36910",1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,2.7178157495953896,594.1185475832327,5.435631499190779,1,2023-09-14,no +CA,A,YES,1,"04971,36910",1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,627.1858702693393,640.4499704489788,9.847493729048772,0,2023-09-15,no +CA,A,YES,1,"04971,36910",1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,30.859634741461225,358.8292028381085,3.9280047220217194,0,2023-09-16,NA +CA,A,YES,1,"04971,36910",1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,15.899032256424775,564.6450526067346,3.1924705121443107,0,2023-09-18,NA +CA,A,YES,1,"04971,36910",1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,163.00694208865636,26.41527898111204,3.642606142908498,0,2023-09-19,NA +CA,A,YES,1,"04971,36910",1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,765.5408539523569,14.41215467320515,5.134440972765299,0,2023-09-20,NA +CA,A,YES,1,"04971,36910",1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,7.3055257413574735,118.07007497423882,2.0505872967575423,0,2023-09-22,NA +CA,A,YES,1,"04971,36910",1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,9.615214324184283,98.28676416676558,22.216317904643557,0,2023-09-23,> max temp and/or hold time +CA,A,YES,1,"04971,36910",1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,31.556363874652213,28.002580514447082,3.3700121744830933,0,2023-09-24,NA +CA,A,YES,1,"04971,36910",1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,85.78745849492218,975.8485468745371,1.5410159137588204,0,2023-09-26,no +CA,A,YES,1,"04971,36910",1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,954.7987735948498,62.703028308156135,1.1257510937636845,0,2023-09-27,no +CA,A,YES,1,"04971,36910",1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,47.157479753344234,6.313948334153875,5.600355902607118,0,2023-09-28,no +CA,A,YES,1,"04971,36910",1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,15.378172379650767,43.82671974549931,2.9286051626342475,0,2023-09-29,NA +CA,A,YES,1,"04971,36910",1,10000,wwtp,raw wastewater,copies/l wastewater,sars-cov-2,265.74559264543973,632.1814334477729,1.2170482720748443,0,2023-09-30,no diff --git a/pyrenew/latent/base.py b/pyrenew/latent/base.py index ac9a8a95..c23e9ebe 100644 --- a/pyrenew/latent/base.py +++ b/pyrenew/latent/base.py @@ -11,6 +11,7 @@ import jax import jax.numpy as jnp from jax.typing import ArrayLike + from pyrenew.metaclass import RandomVariable diff --git a/pyrenew/latent/hierarchical_infections.py b/pyrenew/latent/hierarchical_infections.py index 883e9a08..51c195c4 100644 --- a/pyrenew/latent/hierarchical_infections.py +++ b/pyrenew/latent/hierarchical_infections.py @@ -10,7 +10,12 @@ import jax.numpy as jnp import numpyro from jax.typing import ArrayLike + from pyrenew.deterministic import DeterministicVariable +from pyrenew.latent.base import ( + BaseLatentInfectionProcess, + LatentSample, +) from pyrenew.latent.infection_functions import compute_infections_from_rt from pyrenew.latent.infection_initialization_method import ( InitializeInfectionsExponentialGrowth, @@ -18,15 +23,10 @@ from pyrenew.latent.infection_initialization_process import ( InfectionInitializationProcess, ) +from pyrenew.latent.temporal_processes import TemporalProcess from pyrenew.math import r_approx_from_R from pyrenew.metaclass import RandomVariable -from pyrenew.latent.base import ( - BaseLatentInfectionProcess, - LatentSample, -) -from pyrenew.latent.temporal_processes import TemporalProcess - class HierarchicalInfections(BaseLatentInfectionProcess): """ @@ -56,13 +56,13 @@ class HierarchicalInfections(BaseLatentInfectionProcess): observation time. Must return values in the interval (0, 1). Returns scalar (same for all subpops) or (K,) array (per-subpop). Full I0 matrix generated via exponential backprojection during sampling. - initial_log_rt_rv : RandomVariable - Initial value for log(Rt) at time 0. Can be estimated from data - or given a prior distribution. baseline_temporal : TemporalProcess Temporal process for baseline Rt dynamics subpop_temporal : TemporalProcess Temporal process for subpopulation deviations + initial_log_rt_rv : RandomVariable + Initial value for log(Rt) at time 0. Can be estimated from data + or given a prior distribution. n_initialization_points : int, optional Number of initialization days before day 0. If not specified, defaults to max(21, 2*len(gen_int)-1). When using ModelBuilder, this is computed diff --git a/pyrenew/latent/hierarchical_priors.py b/pyrenew/latent/hierarchical_priors.py index eab3dd36..c2d6488e 100644 --- a/pyrenew/latent/hierarchical_priors.py +++ b/pyrenew/latent/hierarchical_priors.py @@ -8,6 +8,7 @@ import jax.numpy as jnp import numpyro import numpyro.distributions as dist + from pyrenew.metaclass import RandomVariable @@ -30,6 +31,16 @@ def __init__( name: str, sd_rv: RandomVariable, ) -> None: + """ + Initialize hierarchical normal prior. + + Parameters + ---------- + name : str + Unique name for the sampled parameter. + sd_rv : RandomVariable + RandomVariable returning the standard deviation. + """ if not isinstance(sd_rv, RandomVariable): raise TypeError( f"sd_rv must be a RandomVariable, got {type(sd_rv).__name__}. " @@ -40,6 +51,7 @@ def __init__( self.sd_rv = sd_rv def validate(self): + """Validate the random variable (no-op for this class).""" pass def sample(self, n_groups: int, **kwargs): @@ -91,6 +103,20 @@ def __init__( sd_concentration_rv: RandomVariable, sd_min: float = 0.05, ) -> None: + """ + Initialize gamma group SD prior. + + Parameters + ---------- + name : str + Unique name for the sampled parameter. + sd_mean_rv : RandomVariable + RandomVariable returning the mean of the Gamma distribution. + sd_concentration_rv : RandomVariable + RandomVariable returning the concentration parameter. + sd_min : float, default=0.05 + Minimum SD value (lower bound). + """ if not isinstance(sd_mean_rv, RandomVariable): raise TypeError( f"sd_mean_rv must be a RandomVariable, got {type(sd_mean_rv).__name__}. " @@ -110,6 +136,7 @@ def __init__( self.sd_min = sd_min def validate(self): + """Validate the random variable (no-op for this class).""" pass def sample(self, n_groups: int, **kwargs): @@ -165,6 +192,18 @@ def __init__( sd_rv: RandomVariable, df_rv: RandomVariable, ) -> None: + """ + Initialize Student-t group mode prior. + + Parameters + ---------- + name : str + Unique name for the sampled parameter. + sd_rv : RandomVariable + RandomVariable returning the scale parameter. + df_rv : RandomVariable + RandomVariable returning the degrees of freedom. + """ if not isinstance(sd_rv, RandomVariable): raise TypeError( f"sd_rv must be a RandomVariable, got {type(sd_rv).__name__}. " @@ -181,6 +220,7 @@ def __init__( self.df_rv = df_rv def validate(self): + """Validate the random variable (no-op for this class).""" pass def sample(self, n_groups: int, **kwargs): diff --git a/pyrenew/latent/partitioned_infections.py b/pyrenew/latent/partitioned_infections.py index 00f37ad7..f28ae517 100644 --- a/pyrenew/latent/partitioned_infections.py +++ b/pyrenew/latent/partitioned_infections.py @@ -7,7 +7,12 @@ import jax.numpy as jnp import numpyro from jax.typing import ArrayLike + from pyrenew.deterministic import DeterministicVariable +from pyrenew.latent.base import ( + BaseLatentInfectionProcess, + LatentSample, +) from pyrenew.latent.infection_functions import compute_infections_from_rt from pyrenew.latent.infection_initialization_method import ( InitializeInfectionsExponentialGrowth, @@ -15,15 +20,10 @@ from pyrenew.latent.infection_initialization_process import ( InfectionInitializationProcess, ) +from pyrenew.latent.temporal_processes import TemporalProcess from pyrenew.math import r_approx_from_R from pyrenew.metaclass import RandomVariable -from pyrenew.latent.base import ( - BaseLatentInfectionProcess, - LatentSample, -) -from pyrenew.latent.temporal_processes import TemporalProcess - class PartitionedInfections(BaseLatentInfectionProcess): """ @@ -53,13 +53,13 @@ class PartitionedInfections(BaseLatentInfectionProcess): Initial infection prevalence (proportion of population) at first observation time. Must return value in the interval (0, 1]. Returns scalar. Full I0 vector generated via exponential backprojection during sampling. - initial_log_rt_rv : RandomVariable - Initial value for log(Rt) at time 0. Can be estimated from data - or given a prior distribution. rt_temporal : TemporalProcess Temporal process for aggregate-level Rt allocation_temporal : TemporalProcess Temporal process for allocation deviations (K-1 processes, reference fixed to 0) + initial_log_rt_rv : RandomVariable + Initial value for log(Rt) at time 0. Can be estimated from data + or given a prior distribution. n_initialization_points : int, optional Number of initialization days before day 0. If not specified, defaults to max(21, 2*len(gen_int)-1). When using ModelBuilder, this is computed diff --git a/pyrenew/latent/temporal_processes.py b/pyrenew/latent/temporal_processes.py index aac818fd..d08b675f 100644 --- a/pyrenew/latent/temporal_processes.py +++ b/pyrenew/latent/temporal_processes.py @@ -25,6 +25,7 @@ import numpyro import numpyro.distributions as dist from jax.typing import ArrayLike + from pyrenew.process import ARProcess, DifferencedProcess from pyrenew.process import RandomWalk as PyRenewRandomWalk from pyrenew.randomvariable import DistributionalVariable @@ -171,7 +172,14 @@ def _sample_single( innovation_sd: float, name_prefix: str, ) -> ArrayLike: - """Sample a single AR(1) trajectory.""" + """ + Sample a single AR(1) trajectory. + + Returns + ------- + ArrayLike + Trajectory of shape (n_timepoints,). + """ if initial_value is None: initial_value = 0.0 @@ -198,7 +206,14 @@ def _sample_vectorized( innovation_sd: float, name_prefix: str, ) -> ArrayLike: - """Sample multiple AR(1) trajectories in parallel.""" + """ + Sample multiple AR(1) trajectories in parallel. + + Returns + ------- + ArrayLike + Trajectories of shape (n_timepoints, n_processes). + """ if initial_values is None: initial_values = jnp.zeros(n_processes) elif jnp.isscalar(initial_values): @@ -321,7 +336,14 @@ def _sample_single( innovation_sd: float, name_prefix: str, ) -> ArrayLike: - """Sample a single differenced AR(1) trajectory.""" + """ + Sample a single differenced AR(1) trajectory. + + Returns + ------- + ArrayLike + Trajectory of shape (n_timepoints,). + """ if initial_value is None: initial_value = 0.0 @@ -349,7 +371,14 @@ def _sample_vectorized( innovation_sd: float, name_prefix: str, ) -> ArrayLike: - """Sample multiple differenced AR(1) trajectories in parallel.""" + """ + Sample multiple differenced AR(1) trajectories in parallel. + + Returns + ------- + ArrayLike + Trajectories of shape (n_timepoints, n_processes). + """ if initial_values is None: initial_values = jnp.zeros(n_processes) elif jnp.isscalar(initial_values): @@ -466,7 +495,14 @@ def _sample_single( innovation_sd: float, name_prefix: str, ) -> ArrayLike: - """Sample a single random walk trajectory using PyRenew's RandomWalk.""" + """ + Sample a single random walk trajectory using PyRenew's RandomWalk. + + Returns + ------- + ArrayLike + Trajectory of shape (n_timepoints,). + """ if initial_value is None: initial_value = 0.0 @@ -492,7 +528,14 @@ def _sample_vectorized( innovation_sd: float, name_prefix: str, ) -> ArrayLike: - """Sample multiple random walk trajectories in parallel.""" + """ + Sample multiple random walk trajectories in parallel. + + Returns + ------- + ArrayLike + Trajectories of shape (n_timepoints, n_processes). + """ if initial_values is None: initial_values = jnp.zeros(n_processes) elif jnp.isscalar(initial_values): diff --git a/pyrenew/model/multisignal_model.py b/pyrenew/model/multisignal_model.py index d1989630..dc480e86 100644 --- a/pyrenew/model/multisignal_model.py +++ b/pyrenew/model/multisignal_model.py @@ -13,9 +13,9 @@ import numpyro import numpyro.handlers from numpyro.infer import MCMC, NUTS -from pyrenew.metaclass import Model from pyrenew.latent.base import BaseLatentInfectionProcess +from pyrenew.metaclass import Model from pyrenew.observation.base import BaseObservationProcess diff --git a/test/test_hierarchical_infections.py b/test/test_hierarchical_infections.py index 60dd26be..7230ca6e 100644 --- a/test/test_hierarchical_infections.py +++ b/test/test_hierarchical_infections.py @@ -5,19 +5,34 @@ import jax.numpy as jnp import numpyro import pytest -from pyrenew.deterministic import DeterministicPMF, DeterministicVariable +from pyrenew.deterministic import DeterministicPMF, DeterministicVariable from pyrenew.latent import HierarchicalInfections, RandomWalk @pytest.fixture def gen_int_rv(): + """ + Create a generation interval random variable. + + Returns + ------- + DeterministicPMF + Generation interval PMF. + """ return DeterministicPMF("gen_int", jnp.array([0.2, 0.5, 0.3])) @pytest.fixture def process(gen_int_rv): - """Create a HierarchicalInfections with the new keyword-only API.""" + """ + Create a HierarchicalInfections with the new keyword-only API. + + Returns + ------- + HierarchicalInfections + Configured infection process. + """ return HierarchicalInfections( gen_int_rv=gen_int_rv, I0_rv=DeterministicVariable("I0", 0.001), diff --git a/test/test_hierarchical_priors.py b/test/test_hierarchical_priors.py index af1f508b..e72a7a48 100644 --- a/test/test_hierarchical_priors.py +++ b/test/test_hierarchical_priors.py @@ -4,14 +4,14 @@ import numpyro import numpyro.distributions as dist import pytest -from pyrenew.deterministic import DeterministicVariable -from pyrenew.randomvariable import DistributionalVariable +from pyrenew.deterministic import DeterministicVariable from pyrenew.latent import ( GammaGroupSdPrior, HierarchicalNormalPrior, StudentTGroupModePrior, ) +from pyrenew.randomvariable import DistributionalVariable class TestHierarchicalNormalPrior: diff --git a/test/test_model_builder.py b/test/test_model_builder.py index 3b737aab..16a426c6 100644 --- a/test/test_model_builder.py +++ b/test/test_model_builder.py @@ -5,8 +5,8 @@ import jax.numpy as jnp import numpyro import pytest -from pyrenew.deterministic import DeterministicPMF, DeterministicVariable +from pyrenew.deterministic import DeterministicPMF, DeterministicVariable from pyrenew.latent import HierarchicalInfections, RandomWalk from pyrenew.model import ModelBuilder, MultiSignalModel from pyrenew.observation import Counts, NegativeBinomialNoise @@ -18,7 +18,14 @@ @pytest.fixture def simple_builder(): - """Create a configured builder (no population structure at configure time).""" + """ + Create a configured builder (no population structure at configure time). + + Returns + ------- + ModelBuilder + Configured model builder. + """ builder = ModelBuilder() gen_int = DeterministicPMF("gen_int", jnp.array([0.2, 0.5, 0.3])) diff --git a/test/test_partitioned_infections.py b/test/test_partitioned_infections.py index b043bf31..95cd9cf3 100644 --- a/test/test_partitioned_infections.py +++ b/test/test_partitioned_infections.py @@ -5,19 +5,34 @@ import jax.numpy as jnp import numpyro import pytest -from pyrenew.deterministic import DeterministicPMF, DeterministicVariable +from pyrenew.deterministic import DeterministicPMF, DeterministicVariable from pyrenew.latent import PartitionedInfections, RandomWalk @pytest.fixture def gen_int_rv(): + """ + Create a generation interval random variable. + + Returns + ------- + DeterministicPMF + Generation interval PMF. + """ return DeterministicPMF("gen_int", jnp.array([0.2, 0.5, 0.3])) @pytest.fixture def process(gen_int_rv): - """Create a PartitionedInfections with the new keyword-only API.""" + """ + Create a PartitionedInfections with the new keyword-only API. + + Returns + ------- + PartitionedInfections + Configured infection process. + """ return PartitionedInfections( gen_int_rv=gen_int_rv, I0_rv=DeterministicVariable("I0", 0.001), From 096e3e9b0d0a63ba5f704b55c1fb8432d8d0a1b5 Mon Sep 17 00:00:00 2001 From: Mitzi Morris Date: Wed, 14 Jan 2026 20:59:16 -0500 Subject: [PATCH 38/81] label all code blocks --- .../tutorials/building_multisignal_models.qmd | 29 + docs/tutorials/model_builder_helper.qmd | 974 ------------------ 2 files changed, 29 insertions(+), 974 deletions(-) delete mode 100644 docs/tutorials/model_builder_helper.qmd diff --git a/docs/tutorials/building_multisignal_models.qmd b/docs/tutorials/building_multisignal_models.qmd index 0fb17488..680f5ca5 100644 --- a/docs/tutorials/building_multisignal_models.qmd +++ b/docs/tutorials/building_multisignal_models.qmd @@ -36,6 +36,7 @@ numpyro.enable_x64() ``` ```{python} +# | label: imports-base import arviz as az import jax import jax.numpy as jnp @@ -56,6 +57,7 @@ def make_rng_key(): ``` ```{python} +# | label: imports-pyrenew import jax from jax.typing import ArrayLike @@ -183,6 +185,7 @@ Latent infection processes implement the renewal equation to generate infection The generation interval PMF specifies the probability that a secondary infection occurs $d$ days after the primary infection. ```{python} +# | label: gen-interval covid_gen_int = [0.16, 0.32, 0.25, 0.14, 0.07, 0.04, 0.02] gen_int_pmf = jnp.array(covid_gen_int) gen_int_rv = DeterministicPMF("gen_int", gen_int_pmf) @@ -197,6 +200,7 @@ print(f"Generation interval length: {len(gen_int_pmf)} days") The initial infections RV `I0_rv` specifies the **proportion of the population infected** at the first observation time. This must be a value in the interval (0, 1]. ```{python} +# | label: initial-infections I0_rv = DeterministicVariable("I0", 0.01) ``` @@ -205,6 +209,7 @@ I0_rv = DeterministicVariable("I0", 0.01) We set the initial log(Rt) to 0.0, meaning Rt starts at 1.0. ```{python} +# | label: initial-log-rt initial_log_rt_rv = DeterministicVariable("initial_log_rt", 0.0) ``` @@ -218,6 +223,7 @@ We configure two temporal processes: The RandomWalk allows flexible evolution of subpopulation-specific transmission without mean reversion. ```{python} +# | label: temporal-processes # AR1 provides mean-reverting behavior for baseline Rt baseline_temporal = AR1(autoreg=0.9, innovation_sd=0.05) @@ -239,6 +245,7 @@ In this example we use a dataset consisting of hospital admissions for COVID-19 for the first 10 months of 2023 (as reported to the CDC). ```{python} +# | label: load-hospital-data # Load daily hospital admissions for California ca_hosp_data = datasets.load_hospital_data_for_state("CA", "2023-11-06.csv") @@ -258,6 +265,7 @@ print( The hospital admissions data is aggregated at the jurisdiction level, therefore we specify a `Counts` observation process. ```{python} +# | label: hospital-obs-process # Infection-to-hospitalization delay (COVID-19, from literature) inf_to_hosp_pmf = jnp.array( [ @@ -335,6 +343,7 @@ implementations subclass it and implement `_predicted_obs()` to transform infect into predicted values. See `observation_processes_measurements.qmd` for a detailed tutorial. ```{python} +# | label: wastewater-class class Wastewater(Measurements): """ Wastewater viral concentration observation process. @@ -388,6 +397,7 @@ For the wastewater data, we use a simulated dataset for California with realisti that covers the same time period. ```{python} +# | label: load-wastewater-data # Load wastewater data for California ca_ww_data = datasets.load_wastewater_data_for_state("CA", "fake_nwss.csv") @@ -417,6 +427,7 @@ Wastewater observations are site-level: each measurement is associated with a sp Here we specify HierarchicalNormalPrior for the site-level mode and GammaGroupSdPrior for the standard deviation. ```{python} +# | label: wastewater-obs-process # Viral shedding kinetics PMF (days post-infection) shedding_pmf = jnp.array( [ @@ -471,6 +482,7 @@ print(f" Shedding PMF length: {len(shedding_pmf)} days") We instantiate a `ModelBuilder` object which handles the composition of the latent infection process and the observation process. ```{python} +# | label: model-builder-init # Build the multi-signal model builder = ModelBuilder() ``` @@ -488,6 +500,7 @@ Methods `configure_latent` and `add_observation` can be called in any order. Met We use `configure_latent` to specify the **model structure**: generation interval, initial infections, and temporal dynamics. ```{python} +# | label: configure-latent print("Latent process configuration:") print(f" Generation interval length: {len(gen_int_rv())} days") @@ -506,6 +519,7 @@ builder.configure_latent( The name passed to add_observation() becomes the keyword used to pass that observation's data to model.fit() (e.g., hospital={...}, wastewater={...}). ```{python} +# | label: add-observations-build builder.add_observation(hosp_obs, "hospital") builder.add_observation(ww_obs, "wastewater") model = builder.build() @@ -607,6 +621,7 @@ model.fit( First we declare the population structure. ```{python} +# | label: population-structure # 5 observed subpopulations (wastewater catchments) obs_subpop_fractions = jnp.array([0.10, 0.14, 0.21, 0.22, 0.07]) @@ -631,6 +646,7 @@ It is necessary to align the data with the model's time indexing. To do this, we define function `prepare_observation-data`. ```{python} +# | label: prepare-observation-data def prepare_observation_data(n_days_fit, n_init, hosp_admits, ww_data, K_obs): """ Prepare observation data for fitting. @@ -690,6 +706,7 @@ Putting this altogether, we align the data with the model time and call `model.f We run 4 sampler chains. ```{python} +# | label: fit-90-days # Clear JAX caches to avoid interference from earlier cells jax.clear_caches() @@ -731,6 +748,7 @@ print(f"Elapsed time: {elapsed_90:.1f} seconds") ``` ```{python} +# | label: posterior-shape-90 lat_inf_90 = samples_90["latent_infections"] print(f"Posterior samples shape: {lat_inf_90.shape}") print(f"Mean latent infections: {np.mean(lat_inf_90):,.0f}") @@ -739,6 +757,7 @@ print(f"Mean latent infections: {np.mean(lat_inf_90):,.0f}") We check that the chains have converged and the number of effective samples. ```{python} +# | label: arviz-diagnostics-90 # ArviZ diagnostics for 90-day fit def filter_samples_for_arviz(samples, n_init, num_chains=2): """Slice time-series samples to exclude initialization period and reshape for chains.""" @@ -771,6 +790,7 @@ az.summary(idata_90, var_names=["~latent_infections", "~expected"]) We extract the posterior quantiles and print summary statistics. ```{python} +# | label: extract-quantiles-90 def extract_posterior_quantiles(samples, n_init, n_days): """Extract posterior quantiles for latent infections.""" latent_inf = np.array(samples["latent_infections"])[ @@ -795,6 +815,8 @@ print(f" Median infections (day 45): {quantiles_90['q50'][45]:,.0f}") Finally, we visualize the posterior latent infections alongside observed hospitalizations. Note that **hospital admissions lag behind infections** by the infection-to-hospitalization delay (mode ~10 days in our delay PMF). When comparing the two panels, peaks in the infection curve should precede corresponding peaks in hospitalizations by roughly 10-14 days. ```{python} +# | label: fig-posterior-90 +# | fig-cap: Posterior latent infections and observed hospitalizations (90 days). # Visualize posterior latent infections and observed hospitalizations (90 days) # Create separate dataframes for faceted plot infections_df_90 = pd.DataFrame( @@ -863,6 +885,7 @@ plot_df_90["signal"] = pd.Categorical( ### Fit: 180 Days ```{python} +# | label: fit-180-days # Clear JAX caches to avoid interference jax.clear_caches() @@ -895,6 +918,7 @@ print(f"Elapsed time: {elapsed_180:.1f} seconds") ``` ```{python} +# | label: posterior-shape-180 lat_inf_180 = samples_180["latent_infections"] print(f"Posterior samples shape: {lat_inf_180.shape}") print(f"Mean latent infections: {np.mean(lat_inf_180):,.0f}") @@ -903,6 +927,7 @@ print(f"Mean latent infections: {np.mean(lat_inf_180):,.0f}") We check the model fit, as before. ```{python} +# | label: arviz-diagnostics-180 # ArviZ diagnostics for 180-day fit samples_180_filtered = filter_samples_for_arviz( samples_180, n_init, num_chains=2 @@ -912,6 +937,7 @@ az.summary(idata_180, var_names=["~latent_infections", "~expected"]) ``` ```{python} +# | label: extract-quantiles-180 quantiles_180 = extract_posterior_quantiles(samples_180, n_init, n_days_180) ci_width_180 = quantiles_180["q95"] - quantiles_180["q05"] @@ -921,6 +947,8 @@ print(f" Median infections (day 90): {quantiles_180['q50'][90]:,.0f}") ``` ```{python} +# | label: fig-posterior-180 +# | fig-cap: Posterior latent infections and observed hospitalizations (180 days). # Visualize posterior latent infections and observed hospitalizations (180 days) infections_df_180 = pd.DataFrame( { @@ -990,6 +1018,7 @@ plot_df_180["signal"] = pd.Categorical( Comparing the two fits reveals where uncertainty reduction occurs—and why it matters for forecasting. ```{python} +# | label: compare-fits # Compare CI widths for the overlapping 90-day period ci_width_90_overlap = quantiles_90["q95"] - quantiles_90["q05"] ci_width_180_overlap = ( diff --git a/docs/tutorials/model_builder_helper.qmd b/docs/tutorials/model_builder_helper.qmd deleted file mode 100644 index e2471cef..00000000 --- a/docs/tutorials/model_builder_helper.qmd +++ /dev/null @@ -1,974 +0,0 @@ ---- -title: 'Model Building: Composing Latent and Observation Processes' -format: - gfm: - fig-width: 16 - fig-height: 10 - html: - toc: true - embed-resources: true - self-contained-math: true - code-fold: true - code-tools: true -engine: jupyter -jupyter: - jupytext: - text_representation: - extension: .qmd - format_name: quarto - format_version: '1.0' - jupytext_version: 1.18.1 - kernelspec: - display_name: Python 3 (ipykernel) - language: python - name: python3 ---- - -```{python} -# | label: setup -# | output: false -import numpyro -from numpyro.infer.reparam import LocScaleReparam - -# to run samplers in parallel you must run `set_host_device_count` before importing jax -numpyro.set_host_device_count(4) -numpyro.enable_x64() -``` - -```{python} -import arviz as az -import jax -import jax.numpy as jnp -import jax.random as random -import numpy as np -import plotnine as p9 -import pandas as pd -import time -import warnings - -warnings.filterwarnings("ignore") - - -def make_rng_key(): - """Generate a time-based random seed.""" - seed = int(time.time() * 1000) % (2**32) - return random.PRNGKey(seed) -``` - -```{python} -import jax -from jax.typing import ArrayLike - -from pyrenew import datasets -from pyrenew.deterministic import ( - DeterministicPMF, - DeterministicVariable, -) -from pyrenew.metaclass import RandomVariable - -from pyrenew.latent import ( - HierarchicalInfections, - AR1, - RandomWalk, - GammaGroupSdPrior, - HierarchicalNormalPrior, -) -from pyrenew.model import ModelBuilder - -from pyrenew.observation import ( - Counts, - HierarchicalNormalNoise, - Measurements, - MeasurementNoise, - NegativeBinomialNoise, -) -``` - -## Overview - -The `pyrenew` package provides a modular architecture for building multi-signal renewal models by composing: - -- **Latent infection process**: Generates unobserved infection trajectories via the renewal equation -- **Observation processes**: Transform latent infections into observable signals (hospital admissions, wastewater, etc.) - -This notebook demonstrates building a renewal model using the following components: - -- `HierarchicalInfections` — subpopulations share a jurisdiction-level baseline $R(t)$ with subpopulation-specific deviations -- `Counts` — hospital admissions (jurisdiction-level) -- `Wastewater` — viral concentrations (site-level) - -The `ModelBuilder` class composes these components and handles initialization requirements automatically. - -## Observation Processes for Hospital Admissions and Wastewater Concentrations - -Observation processes transform latent infections into observable signals and define the likelihood. Each observation process: - -- Declares what **infection resolution** it needs (`"aggregate"` or `"site"`) -- Applies signal-specific transformations (ascertainment, delay convolution, shedding kinetics) -- Defines the noise model for the likelihood - -### Hospital Admissions - -In this example we use a dataset consisting of hospital admissions for COVID-19 across California -for the first 10 months of 2023 (as reported to the CDC). - -```{python} -# Load daily hospital admissions for California -ca_hosp_data = datasets.load_hospital_data_for_state("CA", "2023-11-06.csv") - -hosp_admits = ca_hosp_data["daily_admits"] -population_size = ca_hosp_data["population"] -n_hosp_days = ca_hosp_data["n_days"] - -print("State: California") -print(f"Population: {population_size:,}") -print(f"Date range: {ca_hosp_data['dates'][0]} to {ca_hosp_data['dates'][-1]}") -print(f"Number of days: {n_hosp_days}") -print( - f"Admissions range: {int(hosp_admits.min())} to {int(hosp_admits.max())}" -) -``` - -The hospital admissions data is aggregated at the jurisdiction level, therefore we specify a `Counts` observation process. - -```{python} -# Infection-to-hospitalization delay (COVID-19, from literature) -inf_to_hosp_pmf = jnp.array( - [ - 0, - 0.00469, - 0.01452, - 0.02786, - 0.04237, - 0.05581, - 0.06657, - 0.07379, - 0.07729, - 0.07737, - 0.07465, - 0.06988, - 0.06377, - 0.05696, - 0.04996, - 0.04315, - 0.03677, - 0.03097, - 0.02583, - 0.02135, - 0.01751, - 0.01427, - 0.01156, - 0.00931, - 0.00746, - 0.00596, - 0.00474, - 0.00375, - 0.00296, - 0.00233, - 0.00183, - 0.00143, - 0.00107, - 0.00077, - 0.00054, - 0.00036, - 0.00024, - 0.00015, - 0.00009, - 0.00005, - 0.00003, - 0.00002, - 0.00001, - ] -) -hosp_delay_rv = DeterministicPMF("inf_to_hosp_delay", inf_to_hosp_pmf) - -# IHR: ~1% of infections lead to hospitalization -ihr_rv = DeterministicVariable("ihr", 0.01) - -# Negative binomial concentration (moderate overdispersion) -hosp_concentration_rv = DeterministicVariable("hosp_concentration", 10.0) - -# Create hospital observation process -hosp_obs = Counts( - ascertainment_rate_rv=ihr_rv, - delay_distribution_rv=hosp_delay_rv, - noise=NegativeBinomialNoise(hosp_concentration_rv), -) - -print("Hospital observation:") -print(f" Infection resolution: {hosp_obs.infection_resolution()}") -print(f" Delay PMF length: {len(inf_to_hosp_pmf)} days") -``` - -### Wastewater Concentrations - -#### Wastewater Observation Process - -The `Measurements` base class handles continuous observation processes. Domain-specific -implementations subclass it and implement `_predicted_obs()` to transform infections -into predicted values. See `observation_processes_measurements.qmd` for a detailed tutorial. - -```{python} -class Wastewater(Measurements): - """ - Wastewater viral concentration observation process. - - Transforms site-level infections into predicted log-concentrations - via shedding kinetics convolution and genome/volume scaling. - """ - - def __init__( - self, - shedding_kinetics_rv: RandomVariable, - log10_genome_per_infection_rv: RandomVariable, - ml_per_person_per_day: float, - noise: MeasurementNoise, - ) -> None: - super().__init__(temporal_pmf_rv=shedding_kinetics_rv, noise=noise) - self.log10_genome_per_infection_rv = log10_genome_per_infection_rv - self.ml_per_person_per_day = ml_per_person_per_day - - def validate(self) -> None: - shedding_pmf = self.temporal_pmf_rv() - self._validate_pmf(shedding_pmf, "shedding_kinetics_rv") - self.noise.validate() - - def lookback_days(self) -> int: - return len(self.temporal_pmf_rv()) - - def _predicted_obs(self, infections: ArrayLike) -> ArrayLike: - shedding_pmf = self.temporal_pmf_rv() - log10_genome = self.log10_genome_per_infection_rv() - - def convolve_site(site_infections): - convolved, _ = self._convolve_with_alignment( - site_infections, shedding_pmf, p_observed=1.0 - ) - return convolved - - shedding_signal = jax.vmap(convolve_site, in_axes=1, out_axes=1)( - infections - ) - genome_copies = 10**log10_genome - concentration = ( - shedding_signal * genome_copies / self.ml_per_person_per_day - ) - return jnp.log(concentration) -``` - -#### Wastewater Data - -For the wastewater data, we use a simulated dataset for California with realistic noise patterns -that covers the same time period. - -```{python} -# Load wastewater data for California -ca_ww_data = datasets.load_wastewater_data_for_state("CA", "fake_nwss.csv") - -ww_conc = ca_ww_data["observed_conc"] # log concentrations -ww_site_ids = ca_ww_data["site_ids"] -ww_time_indices = ca_ww_data["time_indices"] -ww_n_sites = ca_ww_data["n_sites"] -ww_n_obs = ca_ww_data["n_obs"] -ww_wwtp_names = ca_ww_data["wwtp_names"] - -print("State: California") -print(f"Number of sites: {ww_n_sites}") -print(f"Number of observations: {ww_n_obs}") -print(f"Date range: {ca_ww_data['dates'][0]} to {ca_ww_data['dates'][-1]}") -print( - f"Time index range: {int(ww_time_indices.min())} to {int(ww_time_indices.max())}" -) -print("\nSites:") -for i, name in enumerate(ww_wwtp_names[:5]): - print(f" {i}: {name}") -if ww_n_sites > 5: - print(f" ... and {ww_n_sites - 5} more") -``` - -Wastewater observations are site-level: each measurement is associated with a specific measurement site. The Wastewater observation process uses LogNormalNoise, which takes hierarchical priors for the site-level mode and standard deviation parameters. This enables partial pooling across measurement sites. - -Here we specify HierarchicalNormalPrior for the site-level mode and GammaGroupSdPrior for the standard deviation. - -```{python} -# Viral shedding kinetics PMF (days post-infection) -shedding_pmf = jnp.array( - [ - 0.0, - 0.02, - 0.08, - 0.15, - 0.20, - 0.18, - 0.14, - 0.10, - 0.06, - 0.04, - 0.02, - 0.01, - ] -) -shedding_pmf = shedding_pmf / shedding_pmf.sum() # normalize -shedding_rv = DeterministicPMF("shedding_kinetics", shedding_pmf) - -# Log10 genomes shed per infection -log10_genome_rv = DeterministicVariable("log10_genome_per_inf", 9.0) - -# Wastewater volume per person per day (mL) -ml_per_person_per_day = 1000.0 - -# Hierarchical priors for site-level effects -site_mode_prior = HierarchicalNormalPrior( - "ww_site_mode", sd_rv=DeterministicVariable("site_mode_sd", 0.5) -) -site_sd_prior = GammaGroupSdPrior( - "ww_site_sd", - sd_mean_rv=DeterministicVariable("site_sd_mean", 0.3), - sd_concentration_rv=DeterministicVariable("site_sd_conc", 4.0), -) - -# Create wastewater observation process -ww_obs = Wastewater( - shedding_kinetics_rv=shedding_rv, - log10_genome_per_infection_rv=log10_genome_rv, - ml_per_person_per_day=ml_per_person_per_day, - noise=HierarchicalNormalNoise(site_mode_prior, site_sd_prior), -) - -print("Wastewater observation:") -print(f" Infection resolution: {ww_obs.infection_resolution()}") -print(f" Shedding PMF length: {len(shedding_pmf)} days") -``` - -## Latent Infection Process - -Latent infection processes implement the renewal equation to generate infection trajectories. All latent processes share common components: - -- **Generation interval**: PMF for secondary infection timing -- **Initial infections (I0)**: Starting condition for the renewal process -- **Temporal dynamics**: How $R(t)$ evolves over time - -### Generation Interval - -The generation interval PMF specifies the probability that a secondary infection occurs $d$ days after the primary infection. - -```{python} -covid_gen_int = [0.16, 0.32, 0.25, 0.14, 0.07, 0.04, 0.02] -gen_int_pmf = jnp.array(covid_gen_int) -gen_int_rv = DeterministicPMF("gen_int", gen_int_pmf) - -# Mean generation time -days = np.arange(len(gen_int_pmf)) -print(f"Generation interval length: {len(gen_int_pmf)} days") -``` - -### I0: Initial Infections - -The initial infections RV `I0_rv` specifies the **proportion of the population infected** at the first observation time. This must be a value in the interval (0, 1\]. - -```{python} -I0_rv = DeterministicVariable("I0", 0.01) -``` - -### Initial Log Rt - -We set the initial log(Rt) to 0.0, meaning Rt starts at 1.0. - -```{python} -initial_log_rt_rv = DeterministicVariable("initial_log_rt", 0.0) -``` - -### Temporal Processes for $R(t)$ - -We configure two temporal processes: - -- **Jurisdiction-level** (`baseline_temporal`): AR(1) process for the baseline $R(t)$ -- **Subpopulation-level** (`subpop_temporal`): RandomWalk for subpopulation deviations - -The RandomWalk allows flexible evolution of subpopulation-specific transmission without mean reversion. - -```{python} -# AR1 provides mean-reverting behavior for baseline Rt -baseline_temporal = AR1(autoreg=0.9, innovation_sd=0.05) - -# RandomWalk allows flexible subpopulation deviations -subpop_temporal = RandomWalk(innovation_sd=0.025) -``` - -## Model Building - -We instantiate a `ModelBuilder` object which handles the composition of the latent infection process and the observation process. - -```{python} -# Build the multi-signal model -builder = ModelBuilder() -``` - -The `ModelBuilder` object has 3 key methods: - -- `configure_latent` -- `add_observation` -- `build` - -Methods `configure_latent` and `add_observation` can be called in any order. Method `build` is called once all processes have been specified in the model. - -### Configuring the Latent Process - -We use `configure_latent` to specify the **model structure**: generation interval, initial infections, and temporal dynamics. - -```{python} -print("Latent process configuration:") -print(f" Generation interval length: {len(gen_int_rv())} days") - -builder.configure_latent( - HierarchicalInfections, - gen_int_rv=gen_int_rv, - I0_rv=I0_rv, - initial_log_rt_rv=initial_log_rt_rv, - baseline_temporal=baseline_temporal, - subpop_temporal=subpop_temporal, -) -``` - -### Specifying the Observation Processes, Data - -The name passed to add_observation() becomes the keyword used to pass that observation's data to model.fit() (e.g., hospital={...}, wastewater={...}). - -```{python} -builder.add_observation(hosp_obs, "hospital") -builder.add_observation(ww_obs, "wastewater") -model = builder.build() - -n_init = model.latent.n_initialization_points -print("Model built successfully!") -print(f" n_initialization_points: {n_init}") -print(f" Latent process: {type(model.latent).__name__}") -print(f" Observation processes: {list(model.observations.keys())}") -``` - -## Fitting the Model to Data: `model.fit()` - -When you call `model.fit()`, you supply two types of information: - -- **Observation data** — one data dictionary per registered observation process -- **Population structure** — how the jurisdiction is divided into subpopulations - -### Data Routing - Mapping Signals to Model Levels - -This is a hierarchical model that estimates parameters at two levels: - -- **Jurisdiction level**: a baseline $R(t)$ trajectory and aggregated infections across the entire jurisdiction -- **Subpopulation level**: deviations from the baseline $R(t)$ for each subpopulation, producing separate infection trajectories - -Different signals observe different levels of this hierarchy. Each observation process declares an **infection resolution** that determines what infection data it receives: - -| Resolution | Receives | Example signals | -|----------------------|-------------------|-------------------------------| -| `"aggregate"` | Aggregated infections (sum across all subpopulations) | Hospital admissions, case counts | -| `"site"` | Infection matrix with one column per observed subpopulation | Wastewater, site-specific surveillance | - -The model routes latent infections to observation processes based on each process's declared resolution: - -```mermaid -flowchart TB - - subgraph Latent["Latent Infection Process"] - L[HierarchicalInfections] - end - - subgraph Infections["Infection Arrays"] - J["infections_aggregate (T,)"] - S["infections_subpop (T, K_obs)"] - end - - subgraph Obs["Observation Processes"] - C[Counts] - W[Wastewater] - end - - subgraph Data["Observed Data"] - HA[Hospital Admissions] - WW[Concentrations] - end - - L --> J - L --> S - J -->|"aggregate"| C - S -->|"site"| W - C -->|"ascertainment × delay"| HA - W -->|"shedding × genome × dilution"| WW -``` - -### Observation Data by Signal Type - -Each observation process is registered with a name via `add_observation()`. That name becomes the keyword argument for passing data to `model.fit()`: - -```python -builder.add_observation(hosp_obs, "hospital") # → hospital={...} -builder.add_observation(ww_obs, "wastewater") # → wastewater={...} -``` - -#### Jurisdiction-level signals - -The jurisdiction-level hospital admissions data is specified as a `Counts` observations process. - -```python -hospital={ - "obs": jnp.array([...]), # observed values -} -``` - -*Note*: this example uses dense data, i.e., there is an obervation for every day in the timeline. Were the data sparse, it would be necessary to also provide an array `times` which supplies the time index for each observation. - -#### Subpopulation-level signal - -The subpopulation-level wastewater data is specified as a `Wastewater` observations process. - -```python -wastewater={ - "obs": jnp.array([...]), # observed log concentrations - "times": jnp.array([...]), # time index for each observation - "subpop_indices": jnp.array([...]), # which catchment (selects infection trajectory) - "sensor_indices": jnp.array([...]), # which WWTP/lab pair (selects noise parameters) - "n_sensors": int, # total number of WWTP/lab pairs -} -``` - -A **subpopulation** is a portion of the jurisdiction's population defined by a catchment area. A **sensor** is a measurement source — typically a WWTP/lab pair — that produces observations. Multiple sensors can observe the same subpopulation (e.g., different labs processing samples from the same catchment), so `subpop_indices` and `sensor_indices` may differ. - -- `subpop_indices` links each observation to the appropriate infection trajectory -- `sensor_indices` selects which sensor's noise parameters (mode and sd) to apply - -**Example**: A jurisdiction has 3 catchments (subpopulations 0, 1, 2). Catchment 0 has samples processed by two labs (sensors 0 and 1). Observations from catchment 0 all have `subpop_indices=0` but may have `sensor_indices=0` or `sensor_indices=1` depending on which lab processed the sample. - -### Population Structure - -Population structure is specified via two arrays: - -```python -model.fit( - obs_fractions=jnp.array([0.10, 0.14, 0.21, 0.22, 0.07]), # 5 observed - unobs_fractions=jnp.array([0.26]), # 1 unobserved - ... -) -``` - -This specifies 5 observed subpopulations (e.g., wastewater catchments) and 1 unobserved (remainder of jurisdiction). The fractions must sum to 1.0. - - -### Example `model.fit()` Call - -```python -model.fit( - n_days_post_init=n_days, - population_size=population_size, - # Population structure (choose one parameterization) - obs_fractions=obs_subpop_fractions, - unobs_fractions=unobs_subpop_fractions, - # Observation data (keyed by names from add_observation) - hospital={"obs": hosp_counts, "times": hosp_times}, - wastewater={ - "obs": ww_conc, - "times": ww_times, - "subpop_indices": ww_subpop_indices, - "sensor_indices": ww_sensor_indices, - "n_sensors": n_ww_sensors, - }, -) -``` - -## Running the Model - -First we declare the population structure. - -```{python} -# 5 observed subpopulations (wastewater catchments) -obs_subpop_fractions = jnp.array([0.10, 0.14, 0.21, 0.22, 0.07]) - -# 1 unobserved subpopulation (remainder of jurisdiction) -unobs_subpop_fractions = jnp.array([0.26]) - -K_obs = len(obs_subpop_fractions) - -print(f"Observed subpopulations (K_obs): {K_obs}") -print(f"Unobserved subpopulations: {len(unobs_subpop_fractions)}") -print(f"Observed coverage: {float(jnp.sum(obs_subpop_fractions)):.0%}") -print( - f"Total coverage: {float(jnp.sum(obs_subpop_fractions) + jnp.sum(unobs_subpop_fractions)):.0%}" -) -``` - -It is necessary to align the data with the model's time indexing. - -- Times are relative to the start of the simulation (0 = first day of initialization period) -- Observation times should be offset by `n_initialization_points` - -To do this, we define function `prepare_observation-data`. - -```{python} -def prepare_observation_data(n_days_fit, n_init, hosp_admits, ww_data, K_obs): - """ - Prepare observation data for fitting. - - Returns hospital and wastewater data dicts aligned with model time indexing. - - Parameters - ---------- - n_days_fit : int - Number of days to include in fit - n_init : int - Number of initialization points (for time offset) - hosp_admits : array - Hospital admissions time series - ww_data : dict - Wastewater data dictionary - K_obs : int - Number of observed subpopulations (for mapping sites to subpops) - """ - # Hospital: jurisdiction-level, sparse times - hosp_times = jnp.arange(n_days_fit) + n_init - hosp_counts = hosp_admits[:n_days_fit] - - # Wastewater: sensor-level, sparse times - # Filter to observations within the fitting window - ww_mask = ww_data["time_indices"] < n_days_fit - ww_times = ww_data["time_indices"][ww_mask] + n_init - ww_conc = ww_data["observed_conc"][ww_mask] - ww_sensors = ww_data["site_ids"][ww_mask] - - # Map wastewater sensors to subpopulation indices - # For this demo, we cycle through the K_obs observed subpopulations - n_ww_sensors = ww_data["n_sites"] - sensor_to_subpop = {i: i % K_obs for i in range(n_ww_sensors)} - ww_subpop_indices = jnp.array( - [sensor_to_subpop[int(s)] for s in ww_sensors] - ) - - return { - "hospital": { - "obs": hosp_counts, - "times": hosp_times, - }, - "wastewater": { - "obs": ww_conc, - "times": ww_times, - "subpop_indices": ww_subpop_indices, - "sensor_indices": ww_sensors, - "n_sensors": n_ww_sensors, - }, - } -``` - -### Fit: 90 Days - -Putting this altogether, we align the data with the model time and call `model.fit()`. -We run 4 sampler chains. - -```{python} -# Clear JAX caches to avoid interference from earlier cells -jax.clear_caches() - -n_days_90 = 90 -obs_data_90 = prepare_observation_data( - n_days_90, n_init, hosp_admits, ca_ww_data, K_obs -) - -# Try non-centered reparameterization for hierarchical parameters -reparam_config = { - "log_rt_baseline_noise": LocScaleReparam(centered=0), - "subpop_deviations_increments": LocScaleReparam(centered=0), - "ww_site_mode": LocScaleReparam(centered=0), -} - -print(f"Fitting model with {n_days_90} days of data...") -print(" This may take a few minutes...") - -start_time = time.time() -mcmc_90 = model.fit( - n_days_post_init=n_days_90, - population_size=population_size, - obs_fractions=obs_subpop_fractions, - unobs_fractions=unobs_subpop_fractions, - num_warmup=500, - num_samples=500, - num_chains=4, - rng_key=make_rng_key(), - reparam_config=reparam_config, - **obs_data_90, - progress_bar=False, -) -elapsed_90 = time.time() - start_time -print(f"Elapsed time: {elapsed_90:.1f} seconds") -``` - -```{python} -samples_90 = mcmc_90.get_samples() -lat_inf_90 = samples_90["latent_infections"] -print(f"\nPosterior samples shape: {lat_inf_90.shape}") -print(f"Mean latent infections: {np.mean(lat_inf_90):,.0f}") -``` - -We check that the chains have converged and the number of effective samples. - -```{python} -# ArviZ diagnostics for 90-day fit -def filter_samples_for_arviz(samples, n_init, num_chains=2): - """Slice time-series samples to exclude initialization period and reshape for chains.""" - num_samples_per_chain = len(list(samples.values())[0]) // num_chains - filtered = {} - for k, v in samples.items(): - if v.ndim == 2 and v.shape[1] > n_init: - # Time-series: slice to post-init period, reshape to (chains, draws, time) - filtered[k] = v[:, n_init:].reshape( - num_chains, num_samples_per_chain, -1 - ) - elif v.ndim == 1: - # Scalar: reshape to (chains, draws) - filtered[k] = v.reshape(num_chains, num_samples_per_chain) - else: - # Multi-dim: reshape to (chains, draws, ...) - filtered[k] = v.reshape( - num_chains, num_samples_per_chain, *v.shape[1:] - ) - return filtered - - -samples_90_filtered = filter_samples_for_arviz( - samples_90, n_init, num_chains=2 -) -idata_90 = az.from_dict(posterior=samples_90_filtered) -az.summary(idata_90, var_names=["~latent_infections", "~expected"]) -``` - -We extract the posterior quantiles and print summary statistics. - -```{python} -def extract_posterior_quantiles(samples, n_init, n_days): - """Extract posterior quantiles for latent infections.""" - latent_inf = np.array(samples["latent_infections"])[ - :, n_init : n_init + n_days - ] - return { - "q05": np.percentile(latent_inf, 5, axis=0), - "q50": np.percentile(latent_inf, 50, axis=0), - "q95": np.percentile(latent_inf, 95, axis=0), - } - - -quantiles_90 = extract_posterior_quantiles(samples_90, n_init, n_days_90) - -# Summary statistics -ci_width_90 = quantiles_90["q95"] - quantiles_90["q05"] -print(f"Posterior summary for {n_days_90} days:") -print(f" Mean 90% CI width: {ci_width_90.mean():,.0f} infections") -print(f" Median infections (day 45): {quantiles_90['q50'][45]:,.0f}") -``` - -Finally, we visualize the posterior latent infections over the 90 day interval. - -```{python} -# Visualize posterior latent infections (90 days) -posterior_df_90 = pd.DataFrame( - { - "day": np.arange(n_days_90), - "median": quantiles_90["q50"], - "q05": quantiles_90["q05"], - "q95": quantiles_90["q95"], - } -) - -obs_plot_df_90 = pd.DataFrame( - { - "day": np.arange(n_days_90), - "admissions": np.array(hosp_admits[:n_days_90]), - } -) - -( - p9.ggplot() - + p9.geom_ribbon( - p9.aes(x="day", ymin="q05", ymax="q95"), - data=posterior_df_90, - fill="steelblue", - alpha=0.3, - ) - + p9.geom_line( - p9.aes(x="day", y="median"), - data=posterior_df_90, - color="darkblue", - size=1, - ) - + p9.geom_point( - p9.aes(x="day", y="admissions"), - data=obs_plot_df_90, - color="darkorange", - size=1, - alpha=0.6, - ) - + p9.labs( - x="Day", - y="Count", - title="Posterior Latent Infections (90 days, multi-signal fit)", - subtitle="Blue: posterior infections (90% CI) | Orange: observed hospital admissions", - ) - + p9.theme_grey() -) -``` - -### Fit: 180 Days - -```{python} -# Clear JAX caches to avoid interference -jax.clear_caches() - -n_days_180 = 180 -obs_data_180 = prepare_observation_data( - n_days_180, n_init, hosp_admits, ca_ww_data, K_obs -) - -print(f"Fitting model with {n_days_180} days of data...") -print(" This may take a few minutes...") - -start_time = time.time() -mcmc_180 = model.fit( - n_days_post_init=n_days_180, - population_size=population_size, - obs_fractions=obs_subpop_fractions, - unobs_fractions=unobs_subpop_fractions, - num_warmup=1000, - num_samples=500, - num_chains=4, - rng_key=make_rng_key(), - **obs_data_180, - progress_bar=False, -) -elapsed_180 = time.time() - start_time -print(f"Elapsed time: {elapsed_180:.1f} seconds") -``` - -```{python} -samples_180 = mcmc_180.get_samples() -lat_inf_180 = samples_180["latent_infections"] -print(f"\nPosterior samples shape: {lat_inf_180.shape}") -print(f"Mean latent infections: {np.mean(lat_inf_180):,.0f}") -``` - -We check the model fit, as before. - -```{python} -# ArviZ diagnostics for 180-day fit -samples_180_filtered = filter_samples_for_arviz( - samples_180, n_init, num_chains=2 -) -idata_180 = az.from_dict(posterior=samples_180_filtered) -az.summary(idata_180, var_names=["~latent_infections", "~expected"]) -``` - -```{python} -quantiles_180 = extract_posterior_quantiles(samples_180, n_init, n_days_180) - -ci_width_180 = quantiles_180["q95"] - quantiles_180["q05"] -print(f"Posterior summary for {n_days_180} days:") -print(f" Mean 90% CI width: {ci_width_180.mean():,.0f} infections") -print(f" Median infections (day 90): {quantiles_180['q50'][90]:,.0f}") -``` - -```{python} -# Visualize posterior latent infections (180 days) -posterior_df_180 = pd.DataFrame( - { - "day": np.arange(n_days_180), - "median": quantiles_180["q50"], - "q05": quantiles_180["q05"], - "q95": quantiles_180["q95"], - } -) - -obs_plot_df_180 = pd.DataFrame( - { - "day": np.arange(n_days_180), - "admissions": np.array(hosp_admits[:n_days_180]), - } -) - -( - p9.ggplot() - + p9.geom_ribbon( - p9.aes(x="day", ymin="q05", ymax="q95"), - data=posterior_df_180, - fill="steelblue", - alpha=0.3, - ) - + p9.geom_line( - p9.aes(x="day", y="median"), - data=posterior_df_180, - color="darkblue", - size=1, - ) - + p9.geom_point( - p9.aes(x="day", y="admissions"), - data=obs_plot_df_180, - color="darkorange", - size=1, - alpha=0.6, - ) - + p9.labs( - x="Day", - y="Count", - title="Posterior Latent Infections (180 days, multi-signal fit)", - subtitle="Blue: posterior infections (90% CI) | Orange: observed hospital admissions", - ) - + p9.theme_grey() -) -``` - -### Comparing 90-Day vs 180-Day Fits - -We can compare how the posterior estimates differ between fitting to 90 days vs 180 days of data. The key insight is how the credible intervals change in the overlapping period (first 90 days) when we have more data informing the model. - -```{python} -# Compare CI widths for the overlapping 90-day period -ci_width_90_overlap = quantiles_90["q95"] - quantiles_90["q05"] -ci_width_180_overlap = ( - quantiles_180["q95"][:n_days_90] - quantiles_180["q05"][:n_days_90] -) - -# Compute difference in CI widths -ci_diff = ci_width_90_overlap - ci_width_180_overlap -ci_ratio = ci_width_90_overlap / ci_width_180_overlap - -print("CI Width Comparison (first 90 days):") -print( - f" 90-day fit mean CI width: {ci_width_90_overlap.mean():,.0f} infections" -) -print( - f" 180-day fit mean CI width: {ci_width_180_overlap.mean():,.0f} infections" -) -print(f" Mean difference: {ci_diff.mean():,.0f} infections") -print(f" Mean ratio (90/180): {ci_ratio.mean():.2f}x") -print( - f"\nThe 180-day fit has {(1 - ci_width_180_overlap.mean() / ci_width_90_overlap.mean()) * 100:.1f}% narrower CIs on average" -) - -# Per-day comparison -print("\nCI width by time period:") -for start, end, label in [ - (0, 30, "Days 0-30"), - (30, 60, "Days 30-60"), - (60, 90, "Days 60-90"), -]: - mean_90 = ci_width_90_overlap[start:end].mean() - mean_180 = ci_width_180_overlap[start:end].mean() - reduction = (1 - mean_180 / mean_90) * 100 - print( - f" {label}: 90-day={mean_90:,.0f}, 180-day={mean_180:,.0f}, reduction={reduction:.1f}%" - ) -``` - -## Recap - -This notebook demonstrated composing a multi-signal renewal model using `ModelBuilder`: - -1. **Configure latent process** (`configure_latent`): generation interval, initial infections, temporal dynamics -2. **Add observation processes** (`add_observation`): each declares its infection resolution and gets a name for data binding -3. **Build and fit** (`build`, `model.fit`): the model routes infections to observations based on resolution and runs NUTS inference From 642ad53ec4db99d756d16b323b7f0647a0291805 Mon Sep 17 00:00:00 2001 From: Mitzi Morris Date: Thu, 15 Jan 2026 10:19:44 -0500 Subject: [PATCH 39/81] site builds with mkdocs --- docs/tutorials/.pages | 4 +++- docs/tutorials/latent_hierarchical_infections.qmd | 3 +++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/docs/tutorials/.pages b/docs/tutorials/.pages index 4f1079b3..3de734dd 100644 --- a/docs/tutorials/.pages +++ b/docs/tutorials/.pages @@ -2,8 +2,10 @@ nav: - getting_started.md - basic_renewal_model.md - custom_randomvariables.md - - hospital_admissions_model.md + - building_multisignal_models.md - observation_processes_counts.md - observation_processes_measurements.md + - latent_hierarchical_infections.md + - hospital_admissions_model.md - day_of_the_week.md - periodic_effects.md diff --git a/docs/tutorials/latent_hierarchical_infections.qmd b/docs/tutorials/latent_hierarchical_infections.qmd index ca362a6b..bb00389f 100644 --- a/docs/tutorials/latent_hierarchical_infections.qmd +++ b/docs/tutorials/latent_hierarchical_infections.qmd @@ -1,6 +1,9 @@ --- title: 'Hierarchical Latent Infections: Understanding Temporal Process Choices' format: + gfm: + fig-width: 16 + fig-height: 10 html: toc: true toc-depth: 3 From f5bbcff5093b45a9297b99fbf728fd5e422049ee Mon Sep 17 00:00:00 2001 From: Mitzi Morris Date: Thu, 15 Jan 2026 10:20:54 -0500 Subject: [PATCH 40/81] Update docs/tutorials/observation_processes_counts.qmd Co-authored-by: Dylan H. Morris --- docs/tutorials/observation_processes_counts.qmd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/tutorials/observation_processes_counts.qmd b/docs/tutorials/observation_processes_counts.qmd index 6715279a..52fbe7a6 100644 --- a/docs/tutorials/observation_processes_counts.qmd +++ b/docs/tutorials/observation_processes_counts.qmd @@ -83,7 +83,7 @@ In this example, we use fixed parameter values for illustration; in practice, th The delay distribution specifies the probability that an infected person is hospitalized $d$ days after infection, conditional on the infection leading to a hospitalization. For example, if `hosp_delay_pmf[5] = 0.2`, then 20% of infections that result in hospitalization will appear as hospital admissions 5 days after infection. -We load a delay distribution from PyRenew's datasets which peaks around day 8-9 post-infection, compute summary statistics, and plot it. +We load a delay distribution from PyRenew's example datasets, compute summary statistics, and plot it. The distribution peaks around day 8-9 post-infection. ```{python} # | label: delay-distribution From edd657365111e92339c8d05c77b8f0afb2cdf8ec Mon Sep 17 00:00:00 2001 From: Mitzi Morris Date: Thu, 15 Jan 2026 10:21:07 -0500 Subject: [PATCH 41/81] Update docs/tutorials/observation_processes_measurements.qmd Co-authored-by: Dylan H. Morris --- docs/tutorials/observation_processes_measurements.qmd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/tutorials/observation_processes_measurements.qmd b/docs/tutorials/observation_processes_measurements.qmd index f0bf03ff..2ef50f91 100644 --- a/docs/tutorials/observation_processes_measurements.qmd +++ b/docs/tutorials/observation_processes_measurements.qmd @@ -80,7 +80,7 @@ The core convolution structure is shared with count observations, but key aspect ### The noise model -Measurement data typically exhibits **sensor-level variability**: different instruments, labs, or sampling locations have systematic biases and different precision levels. +Measurement data typically exhibits **sensor-level variability**: different instruments, labs, or sampling locations may have systematic biases and different precision levels. `HierarchicalNormalNoise` models this with two per-sensor parameters: From f97f5c3d2ff726478435b4b016773dc68800c453 Mon Sep 17 00:00:00 2001 From: Mitzi Morris Date: Thu, 15 Jan 2026 13:34:33 -0500 Subject: [PATCH 42/81] user-specified signal name --- .../observation_processes_counts.qmd | 8 ++- .../observation_processes_measurements.qmd | 12 +++- pyrenew/observation/base.py | 47 +++++++++---- pyrenew/observation/count_observations.py | 70 ++++++------------- pyrenew/observation/measurements.py | 15 ++-- test/conftest.py | 5 ++ test/test_observation_counts.py | 16 +++++ test/test_observation_measurements.py | 12 ++-- 8 files changed, 109 insertions(+), 76 deletions(-) diff --git a/docs/tutorials/observation_processes_counts.qmd b/docs/tutorials/observation_processes_counts.qmd index 52fbe7a6..8fa90e4b 100644 --- a/docs/tutorials/observation_processes_counts.qmd +++ b/docs/tutorials/observation_processes_counts.qmd @@ -147,15 +147,18 @@ plot_delay A `Counts` object takes the following arguments: +- **`name`**: unique, meaningful identifier for this observation process (e.g., `"hospital"`, `"deaths"`) - **`ascertainment_rate_rv`**: the probability an infection results in an observation (e.g., IHR) - **`delay_distribution_rv`**: delay distribution from infection to observation (PMF) - **`noise`**: noise model (`PoissonNoise()` or `NegativeBinomialNoise(concentration_rv)`) +Observation processes are components in multi-signal models, where each signal must have a unique name. This name prefixes all numpyro sample sites (e.g., `"hospital"` creates sites `"hospital_obs"` and `"hospital_predicted"`), ensuring distinct identifiers in the inference trace. + For hospital admissions, the ascertainment rate is specifically called the infection-hospitalization rate (IHR). In this example, the percentage of infections which lead to hospitalization is treated as a fixed value, which will allow us to see how different values affect the model. The concentration parameter for the negative binomial noise model is also fixed. -In practice, both of these parameters would be given a somewhat informative prior and then inferred. +In practice, both of these parameters would be given a somewhat informative prior and then inferred. ```{python} # | label: create-counts-process @@ -167,6 +170,7 @@ concentration_rv = DeterministicVariable("concentration", 10.0) # Create the observation process hosp_process = Counts( + name="hospital", ascertainment_rate_rv=ihr_rv, delay_distribution_rv=delay_rv, noise=NegativeBinomialNoise(concentration_rv), @@ -507,6 +511,7 @@ conc_results = [] for conc_val in concentration_values: conc_rv_temp = DeterministicVariable("conc", conc_val) process_temp = Counts( + name="hospital", ascertainment_rate_rv=ihr_rv, delay_distribution_rv=delay_rv, noise=NegativeBinomialNoise(conc_rv_temp), @@ -563,6 +568,7 @@ To use Poisson noise instead of negative binomial, change the noise model: ```{python} # | label: poisson-noise hosp_process_poisson = Counts( + name="hospital", ascertainment_rate_rv=ihr_rv, delay_distribution_rv=delay_rv, noise=PoissonNoise(), diff --git a/docs/tutorials/observation_processes_measurements.qmd b/docs/tutorials/observation_processes_measurements.qmd index 2ef50f91..47b414a5 100644 --- a/docs/tutorials/observation_processes_measurements.qmd +++ b/docs/tutorials/observation_processes_measurements.qmd @@ -140,10 +140,12 @@ To create a measurement process for your domain, subclass `Measurements` and imp 2. **`validate()`**: Check parameter validity 3. **`lookback_days()`**: Return the temporal PMF length +The `Measurements` base class requires a `name` parameter. Observation processes are components in multi-signal models, where each signal must have a unique, meaningful name (e.g., `"wastewater"`, `"air_quality"`). This name prefixes all numpyro sample sites, ensuring distinct identifiers in the inference trace. + ```python class MyMeasurement(Measurements): - def __init__(self, temporal_pmf_rv, noise, my_scaling_param): - super().__init__(temporal_pmf_rv=temporal_pmf_rv, noise=noise) + def __init__(self, name, temporal_pmf_rv, noise, my_scaling_param): + super().__init__(name=name, temporal_pmf_rv=temporal_pmf_rv, noise=noise) self.my_scaling_param = my_scaling_param def _predicted_obs(self, infections): @@ -209,6 +211,7 @@ class Wastewater(Measurements): def __init__( self, + name: str, shedding_kinetics_rv: RandomVariable, log10_genome_per_infection_rv: RandomVariable, ml_per_person_per_day: float, @@ -219,6 +222,8 @@ class Wastewater(Measurements): Parameters ---------- + name : str + Unique name for this observation process. shedding_kinetics_rv : RandomVariable Viral shedding PMF (fraction shed each day post-infection). log10_genome_per_infection_rv : RandomVariable @@ -228,7 +233,7 @@ class Wastewater(Measurements): noise : MeasurementNoise Noise model (e.g., HierarchicalNormalNoise). """ - super().__init__(temporal_pmf_rv=shedding_kinetics_rv, noise=noise) + super().__init__(name=name, temporal_pmf_rv=shedding_kinetics_rv, noise=noise) self.log10_genome_per_infection_rv = log10_genome_per_infection_rv self.ml_per_person_per_day = ml_per_person_per_day @@ -381,6 +386,7 @@ ww_noise = HierarchicalNormalNoise( ```{python} # | label: create-process ww_process = Wastewater( + name="wastewater", shedding_kinetics_rv=shedding_rv, log10_genome_per_infection_rv=log10_genome_rv, ml_per_person_per_day=ml_per_person_per_day, diff --git a/pyrenew/observation/base.py b/pyrenew/observation/base.py index f415671a..3dfaa642 100644 --- a/pyrenew/observation/base.py +++ b/pyrenew/observation/base.py @@ -56,12 +56,16 @@ class BaseObservationProcess(RandomVariable): Base class for all random variables """ - def __init__(self, temporal_pmf_rv: RandomVariable) -> None: + def __init__(self, name: str, temporal_pmf_rv: RandomVariable) -> None: """ Initialize base observation process. Parameters ---------- + name : str + Unique name for this observation process. Used to prefix all + numpyro sample and deterministic site names, enabling multiple + observations of the same type in a single model. temporal_pmf_rv : RandomVariable The temporal distribution PMF (e.g., delay or shedding distribution). Must sample to a 1D array that sums to ~1.0 with non-negative values. @@ -69,9 +73,10 @@ def __init__(self, temporal_pmf_rv: RandomVariable) -> None: Notes ----- - Subclasses should call ``super().__init__(temporal_pmf_rv)`` + Subclasses should call ``super().__init__(name, temporal_pmf_rv)`` in their constructors and may add additional parameters. """ + self.name = name self.temporal_pmf_rv = temporal_pmf_rv @abstractmethod @@ -129,14 +134,6 @@ def infection_resolution(self) -> str: str Either ``"aggregate"`` or ``"subpop"`` - Examples - -------- - >>> # Aggregated count observations - >>> hosp_obs.infection_resolution() # Returns "aggregate" - >>> - >>> # Subpopulation-level observations (wastewater, subpop-specific counts) - >>> ww_obs.infection_resolution() # Returns "subpop" - Notes ----- This is used by multi-signal models to route the correct infection @@ -252,7 +249,7 @@ def _convolve_with_alignment( pad=True, # Maintains timeline alignment ) - def _deterministic(self, name: str, value: ArrayLike) -> None: + def _deterministic(self, suffix: str, value: ArrayLike) -> None: """ Track a deterministic quantity in the numpyro execution trace. @@ -262,14 +259,36 @@ def _deterministic(self, name: str, value: ArrayLike) -> None: These quantities are stored in MCMC samples and can be used for model diagnostics and posterior predictive checks. + The site name is prefixed with ``self.name`` to ensure uniqueness + when multiple observations of the same type are used. + Parameters ---------- - name : str - Name for the tracked quantity. Will appear in MCMC samples. + suffix : str + Suffix for the site name. Will be prefixed with ``self.name``. + For example, suffix="predicted" becomes "{name}_predicted". value : ArrayLike Value to track. Can be any shape. """ - numpyro.deterministic(name, value) + site_name = f"{self.name}_{suffix}" + numpyro.deterministic(site_name, value) + + def _sample_site_name(self, suffix: str) -> str: + """ + Generate a prefixed sample site name. + + Parameters + ---------- + suffix : str + Suffix for the site name (e.g., "obs"). + + Returns + ------- + str + Full site name with ``self.name`` prefix. + For example, suffix="obs" returns "{name}_obs". + """ + return f"{self.name}_{suffix}" @abstractmethod def _predicted_obs( diff --git a/pyrenew/observation/count_observations.py b/pyrenew/observation/count_observations.py index 48a8d574..cbecb58a 100644 --- a/pyrenew/observation/count_observations.py +++ b/pyrenew/observation/count_observations.py @@ -26,6 +26,7 @@ class _CountBase(BaseObservationProcess): def __init__( self, + name: str, ascertainment_rate_rv: RandomVariable, delay_distribution_rv: RandomVariable, noise: CountNoise, @@ -35,6 +36,9 @@ def __init__( Parameters ---------- + name : str + Unique name for this observation process. Used to prefix all + numpyro sample and deterministic site names. ascertainment_rate_rv : RandomVariable Ascertainment rate in [0, 1] (e.g., IHR, IER). delay_distribution_rv : RandomVariable @@ -42,7 +46,7 @@ def __init__( noise : CountNoise Noise model for count observations (Poisson, NegBin, etc.). """ - super().__init__(temporal_pmf_rv=delay_distribution_rv) + super().__init__(name=name, temporal_pmf_rv=delay_distribution_rv) self.ascertainment_rate_rv = ascertainment_rate_rv self.noise = noise @@ -135,6 +139,10 @@ class Counts(_CountBase): Parameters ---------- + name : str + Unique name for this observation process. Used to prefix all + numpyro sample and deterministic site names (e.g., "hospital" + produces sites "hospital_obs", "hospital_predicted"). ascertainment_rate_rv : RandomVariable Ascertainment rate in [0, 1] (e.g., IHR, IER). delay_distribution_rv : RandomVariable @@ -146,24 +154,6 @@ class Counts(_CountBase): ----- Output preserves input timeline. First len(delay_pmf)-1 days return -1 or ~0 (depending on noise model) due to NaN padding. - - Examples - -------- - >>> from pyrenew.deterministic import DeterministicVariable, DeterministicPMF - >>> from pyrenew.observation import Counts, NegativeBinomialNoise - >>> import jax.numpy as jnp - >>> import numpyro - >>> - >>> delay_pmf = jnp.array([0.2, 0.5, 0.3]) - >>> counts_obs = Counts( - ... ascertainment_rate_rv=DeterministicVariable("ihr", 0.01), - ... delay_distribution_rv=DeterministicPMF("delay", delay_pmf), - ... noise=NegativeBinomialNoise(DeterministicVariable("conc", 10.0)), - ... ) - >>> - >>> with numpyro.handlers.seed(rng_seed=42): - ... infections = jnp.ones(30) * 1000 - ... sampled_counts = counts_obs.sample(infections=infections, obs=None) """ def infection_resolution(self) -> str: @@ -180,7 +170,8 @@ def infection_resolution(self) -> str: def __repr__(self) -> str: """Return string representation.""" return ( - f"Counts(ascertainment_rate_rv={self.ascertainment_rate_rv!r}, " + f"Counts(name={self.name!r}, " + f"ascertainment_rate_rv={self.ascertainment_rate_rv!r}, " f"delay_distribution_rv={self.temporal_pmf_rv!r}, " f"noise={self.noise!r})" ) @@ -214,7 +205,7 @@ def sample( `predicted` (predicted counts before noise). """ predicted_counts = self._predicted_obs(infections) - self._deterministic("predicted_counts", predicted_counts) + self._deterministic("predicted", predicted_counts) predicted_counts_safe = jnp.nan_to_num(predicted_counts, nan=0.0) # Only use sparse indexing when conditioning on observations @@ -224,7 +215,7 @@ def sample( predicted_obs = predicted_counts_safe observed = self.noise.sample( - name="counts", + name=self._sample_site_name("obs"), predicted=predicted_obs, obs=obs, ) @@ -241,6 +232,9 @@ class CountsBySubpop(_CountBase): Parameters ---------- + name : str + Unique name for this observation process. Used to prefix all + numpyro sample and deterministic site names. ascertainment_rate_rv : RandomVariable Ascertainment rate in [0, 1]. delay_distribution_rv : RandomVariable @@ -251,37 +245,13 @@ class CountsBySubpop(_CountBase): Notes ----- Output preserves input timeline. First len(delay_pmf)-1 days are NaN. - - Examples - -------- - >>> from pyrenew.deterministic import DeterministicVariable, DeterministicPMF - >>> from pyrenew.observation import CountsBySubpop, PoissonNoise - >>> import jax.numpy as jnp - >>> import numpyro - >>> - >>> delay_pmf = jnp.array([0.3, 0.4, 0.3]) - >>> counts_obs = CountsBySubpop( - ... ascertainment_rate_rv=DeterministicVariable("ihr", 0.02), - ... delay_distribution_rv=DeterministicPMF("delay", delay_pmf), - ... noise=PoissonNoise(), - ... ) - >>> - >>> with numpyro.handlers.seed(rng_seed=42): - ... infections = jnp.ones((30, 3)) * 500 # 30 days, 3 subpops - ... times = jnp.array([10, 15, 10, 15]) - ... subpop_indices = jnp.array([0, 0, 1, 1]) - ... sampled = counts_obs.sample( - ... infections=infections, - ... subpop_indices=subpop_indices, - ... times=times, - ... obs=None, - ... ) """ def __repr__(self) -> str: """Return string representation.""" return ( - f"CountsBySubpop(ascertainment_rate_rv={self.ascertainment_rate_rv!r}, " + f"CountsBySubpop(name={self.name!r}, " + f"ascertainment_rate_rv={self.ascertainment_rate_rv!r}, " f"delay_distribution_rv={self.temporal_pmf_rv!r}, " f"noise={self.noise!r})" ) @@ -333,14 +303,14 @@ def sample( # Compute predicted counts for all subpops predicted_counts_all = self._predicted_obs(infections) - self._deterministic("predicted_counts_by_subpop", predicted_counts_all) + self._deterministic("predicted", predicted_counts_all) # Replace NaN padding with 0 for distribution creation predicted_counts_safe = jnp.nan_to_num(predicted_counts_all, nan=0.0) predicted_obs = predicted_counts_safe[times, subpop_indices] observed = self.noise.sample( - name="counts_by_subpop", + name=self._sample_site_name("obs"), predicted=predicted_obs, obs=obs, ) diff --git a/pyrenew/observation/measurements.py b/pyrenew/observation/measurements.py index 82043e65..2d090bed 100644 --- a/pyrenew/observation/measurements.py +++ b/pyrenew/observation/measurements.py @@ -23,6 +23,9 @@ class Measurements(BaseObservationProcess): Parameters ---------- + name : str + Unique name for this observation process. Used to prefix all + numpyro sample and deterministic site names. temporal_pmf_rv : RandomVariable Temporal distribution PMF (e.g., shedding kinetics for wastewater). noise : MeasurementNoise @@ -45,6 +48,7 @@ class Measurements(BaseObservationProcess): def __init__( self, + name: str, temporal_pmf_rv: RandomVariable, noise: MeasurementNoise, ) -> None: @@ -53,18 +57,21 @@ def __init__( Parameters ---------- + name : str + Unique name for this observation process. Used to prefix all + numpyro sample and deterministic site names. temporal_pmf_rv : RandomVariable Temporal distribution PMF (e.g., shedding kinetics). noise : MeasurementNoise Noise model (e.g., HierarchicalNormalNoise with sensor effects). """ - super().__init__(temporal_pmf_rv=temporal_pmf_rv) + super().__init__(name=name, temporal_pmf_rv=temporal_pmf_rv) self.noise = noise def __repr__(self) -> str: """Return string representation.""" return ( - f"{self.__class__.__name__}(" + f"{self.__class__.__name__}(name={self.name!r}, " f"temporal_pmf_rv={self.temporal_pmf_rv!r}, " f"noise={self.noise!r})" ) @@ -129,12 +136,12 @@ def sample( """ predicted_values = self._predicted_obs(infections) - self._deterministic("predicted_log_conc", predicted_values) + self._deterministic("predicted", predicted_values) predicted_obs = predicted_values[times, subpop_indices] observed = self.noise.sample( - name="concentrations", + name=self._sample_site_name("obs"), predicted=predicted_obs, obs=obs, sensor_indices=sensor_indices, diff --git a/test/conftest.py b/test/conftest.py index bc95fd10..381ee056 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -199,6 +199,7 @@ def counts_process(simple_delay_pmf): A Counts observation process with no delay. """ return Counts( + name="test_counts", ascertainment_rate_rv=DeterministicVariable("ihr", 0.01), delay_distribution_rv=DeterministicPMF("delay", simple_delay_pmf), noise=NegativeBinomialNoise(DeterministicVariable("conc", 10.0)), @@ -216,6 +217,7 @@ def counts_process_medium_delay(medium_delay_pmf): A Counts observation process with 4-day delay. """ return Counts( + name="test_counts", ascertainment_rate_rv=DeterministicVariable("ihr", 0.01), delay_distribution_rv=DeterministicPMF("delay", medium_delay_pmf), noise=NegativeBinomialNoise(DeterministicVariable("conc", 50.0)), @@ -233,6 +235,7 @@ def counts_process_realistic(realistic_delay_pmf): A Counts observation process with realistic parameters. """ return Counts( + name="test_counts", ascertainment_rate_rv=DeterministicVariable("ihr", 0.005), delay_distribution_rv=DeterministicPMF("delay", realistic_delay_pmf), noise=NegativeBinomialNoise(DeterministicVariable("conc", 100.0)), @@ -244,6 +247,7 @@ class CountsProcessFactory: @staticmethod def create( + name="test_counts", delay_pmf=None, ascertainment_rate=0.01, concentration=10.0, @@ -259,6 +263,7 @@ def create( if delay_pmf is None: delay_pmf = jnp.array([1.0]) return Counts( + name=name, ascertainment_rate_rv=DeterministicVariable("ihr", ascertainment_rate), delay_distribution_rv=DeterministicPMF("delay", delay_pmf), noise=NegativeBinomialNoise(DeterministicVariable("conc", concentration)), diff --git a/test/test_observation_counts.py b/test/test_observation_counts.py index 33f4eade..4de76ebf 100644 --- a/test/test_observation_counts.py +++ b/test/test_observation_counts.py @@ -148,6 +148,7 @@ def test_with_stochastic_ascertainment(self, short_shedding_pmf): concentration = DeterministicVariable("conc", 10.0) process = Counts( + name="test", ascertainment_rate_rv=ascertainment, delay_distribution_rv=delay, noise=NegativeBinomialNoise(concentration), @@ -173,6 +174,7 @@ def test_with_stochastic_concentration(self, simple_delay_pmf): concentration = DistributionalVariable("conc", dist.HalfNormal(10.0)) process = Counts( + name="test", ascertainment_rate_rv=ascertainment, delay_distribution_rv=delay, noise=NegativeBinomialNoise(concentration), @@ -322,6 +324,7 @@ def test_sample_returns_correct_shape(self): """Test that CountsBySubpop sample returns correct shape.""" delay_pmf = jnp.array([0.3, 0.4, 0.3]) process = CountsBySubpop( + name="test", ascertainment_rate_rv=DeterministicVariable("ihr", 0.02), delay_distribution_rv=DeterministicPMF("delay", delay_pmf), noise=PoissonNoise(), @@ -346,6 +349,7 @@ def test_infection_resolution(self): """Test that CountsBySubpop returns 'subpop' resolution.""" delay_pmf = jnp.array([1.0]) process = CountsBySubpop( + name="test", ascertainment_rate_rv=DeterministicVariable("ihr", 0.01), delay_distribution_rv=DeterministicPMF("delay", delay_pmf), noise=PoissonNoise(), @@ -360,6 +364,7 @@ class TestPoissonNoise: def test_poisson_counts(self, simple_delay_pmf): """Test Counts with Poisson noise.""" process = Counts( + name="test", ascertainment_rate_rv=DeterministicVariable("ihr", 0.01), delay_distribution_rv=DeterministicPMF("delay", simple_delay_pmf), noise=PoissonNoise(), @@ -392,6 +397,7 @@ def sample(self, **kwargs): pass process = IncompleteCountProcess( + name="test", ascertainment_rate_rv=DeterministicVariable("ihr", 0.01), delay_distribution_rv=DeterministicPMF("delay", simple_delay_pmf), noise=NegativeBinomialNoise(DeterministicVariable("conc", 10.0)), @@ -408,6 +414,7 @@ class TestValidationMethods: def test_validate_calls_all_validations(self, simple_delay_pmf): """Test that validate() calls all necessary validations.""" process = Counts( + name="test", ascertainment_rate_rv=DeterministicVariable("ihr", 0.01), delay_distribution_rv=DeterministicPMF("delay", simple_delay_pmf), noise=NegativeBinomialNoise(DeterministicVariable("conc", 10.0)), @@ -418,6 +425,7 @@ def test_validate_calls_all_validations(self, simple_delay_pmf): def test_validate_invalid_ascertainment_rate_negative(self, simple_delay_pmf): """Test that validate raises for negative ascertainment rate.""" process = Counts( + name="test", ascertainment_rate_rv=DeterministicVariable("ihr", -0.1), delay_distribution_rv=DeterministicPMF("delay", simple_delay_pmf), noise=NegativeBinomialNoise(DeterministicVariable("conc", 10.0)), @@ -430,6 +438,7 @@ def test_validate_invalid_ascertainment_rate_greater_than_one( ): """Test that validate raises for ascertainment rate > 1.""" process = Counts( + name="test", ascertainment_rate_rv=DeterministicVariable("ihr", 1.5), delay_distribution_rv=DeterministicPMF("delay", simple_delay_pmf), noise=NegativeBinomialNoise(DeterministicVariable("conc", 10.0)), @@ -440,6 +449,7 @@ def test_validate_invalid_ascertainment_rate_greater_than_one( def test_lookback_days(self, simple_delay_pmf, long_delay_pmf): """Test lookback_days returns PMF length.""" process_short = Counts( + name="test_short", ascertainment_rate_rv=DeterministicVariable("ihr", 0.01), delay_distribution_rv=DeterministicPMF("delay", simple_delay_pmf), noise=NegativeBinomialNoise(DeterministicVariable("conc", 10.0)), @@ -447,6 +457,7 @@ def test_lookback_days(self, simple_delay_pmf, long_delay_pmf): assert process_short.lookback_days() == 1 process_long = Counts( + name="test_long", ascertainment_rate_rv=DeterministicVariable("ihr", 0.01), delay_distribution_rv=DeterministicPMF("delay", long_delay_pmf), noise=NegativeBinomialNoise(DeterministicVariable("conc", 10.0)), @@ -456,6 +467,7 @@ def test_lookback_days(self, simple_delay_pmf, long_delay_pmf): def test_infection_resolution_counts(self, simple_delay_pmf): """Test that Counts returns 'aggregate' resolution.""" process = Counts( + name="test", ascertainment_rate_rv=DeterministicVariable("ihr", 0.01), delay_distribution_rv=DeterministicPMF("delay", simple_delay_pmf), noise=NegativeBinomialNoise(DeterministicVariable("conc", 10.0)), @@ -514,6 +526,7 @@ class TestBaseObservationProcessValidation: def test_validate_pmf_empty_array(self, simple_delay_pmf): """Test that _validate_pmf raises for empty array.""" process = Counts( + name="test", ascertainment_rate_rv=DeterministicVariable("ihr", 0.01), delay_distribution_rv=DeterministicPMF("delay", simple_delay_pmf), noise=NegativeBinomialNoise(DeterministicVariable("conc", 10.0)), @@ -525,6 +538,7 @@ def test_validate_pmf_empty_array(self, simple_delay_pmf): def test_validate_pmf_sum_not_one(self, simple_delay_pmf): """Test that _validate_pmf raises for PMF not summing to 1.""" process = Counts( + name="test", ascertainment_rate_rv=DeterministicVariable("ihr", 0.01), delay_distribution_rv=DeterministicPMF("delay", simple_delay_pmf), noise=NegativeBinomialNoise(DeterministicVariable("conc", 10.0)), @@ -536,6 +550,7 @@ def test_validate_pmf_sum_not_one(self, simple_delay_pmf): def test_validate_pmf_negative_values(self, simple_delay_pmf): """Test that _validate_pmf raises for negative values.""" process = Counts( + name="test", ascertainment_rate_rv=DeterministicVariable("ihr", 0.01), delay_distribution_rv=DeterministicPMF("delay", simple_delay_pmf), noise=NegativeBinomialNoise(DeterministicVariable("conc", 10.0)), @@ -548,6 +563,7 @@ def test_get_minimum_observation_day(self): """Test get_minimum_observation_day returns correct value.""" delay_pmf = jnp.array([0.2, 0.5, 0.3]) # length 3 process = Counts( + name="test", ascertainment_rate_rv=DeterministicVariable("ihr", 0.01), delay_distribution_rv=DeterministicPMF("delay", delay_pmf), noise=NegativeBinomialNoise(DeterministicVariable("conc", 10.0)), diff --git a/test/test_observation_measurements.py b/test/test_observation_measurements.py index 2b52f712..1ef1674e 100644 --- a/test/test_observation_measurements.py +++ b/test/test_observation_measurements.py @@ -22,9 +22,9 @@ class ConcreteMeasurements(Measurements): """Concrete implementation of Measurements for testing.""" - def __init__(self, temporal_pmf_rv, noise, log10_scale=9.0): + def __init__(self, name, temporal_pmf_rv, noise, log10_scale=9.0): """Initialize the concrete measurements for testing.""" - super().__init__(temporal_pmf_rv=temporal_pmf_rv, noise=noise) + super().__init__(name=name, temporal_pmf_rv=temporal_pmf_rv, noise=noise) self.log10_scale = log10_scale def validate(self) -> None: @@ -92,6 +92,7 @@ def test_infection_resolution_is_subpop(self): noise = HierarchicalNormalNoise(sensor_mode_rv, sensor_sd_rv) process = ConcreteMeasurements( + name="test", temporal_pmf_rv=DeterministicPMF("shedding", shedding_pmf), noise=noise, ) @@ -204,6 +205,7 @@ def test_repr(self): noise = HierarchicalNormalNoise(sensor_mode_rv, sensor_sd_rv) process = ConcreteMeasurements( + name="test", temporal_pmf_rv=DeterministicPMF("shedding", shedding_pmf), noise=noise, ) @@ -227,6 +229,7 @@ def test_sample_shape(self): noise = HierarchicalNormalNoise(sensor_mode_rv, sensor_sd_rv) process = ConcreteMeasurements( + name="test", temporal_pmf_rv=DeterministicPMF("shedding", shedding_pmf), noise=noise, ) @@ -250,7 +253,7 @@ def test_sample_shape(self): assert result.predicted.shape == infections.shape def test_predicted_obs_stored(self): - """Test that predicted_log_conc is stored as deterministic.""" + """Test that predicted values are stored as deterministic.""" shedding_pmf = jnp.array([0.5, 0.5]) sensor_mode_rv = VectorizedRV( DistributionalVariable("mode", dist.Normal(0, 0.01)), @@ -263,6 +266,7 @@ def test_predicted_obs_stored(self): noise = HierarchicalNormalNoise(sensor_mode_rv, sensor_sd_rv) process = ConcreteMeasurements( + name="test", temporal_pmf_rv=DeterministicPMF("shedding", shedding_pmf), noise=noise, ) @@ -284,7 +288,7 @@ def test_predicted_obs_stored(self): ) ).get_trace() - assert "predicted_log_conc" in trace + assert "test_predicted" in trace if __name__ == "__main__": From df7d082ed3048785c443abed56dd152a3c73cd6d Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 15 Jan 2026 18:34:50 +0000 Subject: [PATCH 43/81] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- docs/tutorials/observation_processes_measurements.qmd | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/tutorials/observation_processes_measurements.qmd b/docs/tutorials/observation_processes_measurements.qmd index 47b414a5..1ff75dfe 100644 --- a/docs/tutorials/observation_processes_measurements.qmd +++ b/docs/tutorials/observation_processes_measurements.qmd @@ -233,7 +233,9 @@ class Wastewater(Measurements): noise : MeasurementNoise Noise model (e.g., HierarchicalNormalNoise). """ - super().__init__(name=name, temporal_pmf_rv=shedding_kinetics_rv, noise=noise) + super().__init__( + name=name, temporal_pmf_rv=shedding_kinetics_rv, noise=noise + ) self.log10_genome_per_infection_rv = log10_genome_per_infection_rv self.ml_per_person_per_day = ml_per_person_per_day From 060bf0766bbfe4e755cdc0eeb040bdb15a63440b Mon Sep 17 00:00:00 2001 From: Mitzi Morris Date: Thu, 15 Jan 2026 13:34:53 -0500 Subject: [PATCH 44/81] lint fix --- docs/tutorials/observation_processes_measurements.qmd | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/tutorials/observation_processes_measurements.qmd b/docs/tutorials/observation_processes_measurements.qmd index 47b414a5..1ff75dfe 100644 --- a/docs/tutorials/observation_processes_measurements.qmd +++ b/docs/tutorials/observation_processes_measurements.qmd @@ -233,7 +233,9 @@ class Wastewater(Measurements): noise : MeasurementNoise Noise model (e.g., HierarchicalNormalNoise). """ - super().__init__(name=name, temporal_pmf_rv=shedding_kinetics_rv, noise=noise) + super().__init__( + name=name, temporal_pmf_rv=shedding_kinetics_rv, noise=noise + ) self.log10_genome_per_infection_rv = log10_genome_per_infection_rv self.ml_per_person_per_day = ml_per_person_per_day From 50cee63dc1e8bf440db2e9dcdaa4e0db8a946f03 Mon Sep 17 00:00:00 2001 From: Mitzi Morris Date: Thu, 15 Jan 2026 14:41:38 -0500 Subject: [PATCH 45/81] user-specified signal names --- .../tutorials/building_multisignal_models.qmd | 41 ++++++++++++++----- pyrenew/datasets/hospital_admissions.py | 7 ---- pyrenew/datasets/wastewater_nwss.py | 7 ---- pyrenew/model/model_builder.py | 14 ++++--- pyrenew/model/multisignal_model.py | 2 +- test/test_model_builder.py | 3 +- 6 files changed, 41 insertions(+), 33 deletions(-) diff --git a/docs/tutorials/building_multisignal_models.qmd b/docs/tutorials/building_multisignal_models.qmd index 680f5ca5..28ba95bc 100644 --- a/docs/tutorials/building_multisignal_models.qmd +++ b/docs/tutorials/building_multisignal_models.qmd @@ -142,13 +142,13 @@ flowchart TB end subgraph Obs["Observation Processes"] - C[Counts] - W[Wastewater] + C["Counts (name='hospital')"] + W["Wastewater (name='wastewater')"] end subgraph Data["Observed Data"] - HA[Hospital Admissions] - WW[Concentrations] + HA["hospital_obs"] + WW["wastewater_obs"] end L --> J @@ -235,10 +235,24 @@ subpop_temporal = RandomWalk(innovation_sd=0.025) Observation processes transform latent infections into observable signals and define the statistical model linking predictions to data. Each observation process: +- Has a unique **name** that identifies the signal in model outputs - Declares what **infection resolution** it needs (`"aggregate"` or `"subpop"`) - Applies signal-specific transformations (ascertainment, delay convolution, shedding kinetics) - Defines the noise model +### Signal Naming + +Each observation process requires a `name` parameter—a short, meaningful identifier like `"hospital"` or `"wastewater"`. This name serves as the single identifier for the signal throughout the model: + +- **Numpyro sites**: Prefixes all sample and deterministic sites (e.g., `hospital_obs`, `hospital_predicted`) +- **Data binding**: Becomes the keyword argument for passing data to `model.fit()` (e.g., `hospital={...}`) + +This unified naming provides several benefits: + +- **Interpretable outputs**: When examining MCMC samples or posterior diagnostics, site names like `hospital_predicted` immediately indicate which signal each quantity refers to +- **Multiple signals of the same type**: You can include multiple count observations (e.g., hospital admissions and deaths) by giving each a distinct name +- **Clearer debugging**: Error messages and trace inspection show meaningful signal names rather than generic identifiers + ### Hospital Admissions In this example we use a dataset consisting of hospital admissions for COVID-19 across California @@ -324,6 +338,7 @@ hosp_concentration_rv = DeterministicVariable("hosp_concentration", 10.0) # Create hospital observation process hosp_obs = Counts( + name="hospital", ascertainment_rate_rv=ihr_rv, delay_distribution_rv=hosp_delay_rv, noise=NegativeBinomialNoise(hosp_concentration_rv), @@ -354,12 +369,15 @@ class Wastewater(Measurements): def __init__( self, + name: str, shedding_kinetics_rv: RandomVariable, log10_genome_per_infection_rv: RandomVariable, ml_per_person_per_day: float, noise: MeasurementNoise, ) -> None: - super().__init__(temporal_pmf_rv=shedding_kinetics_rv, noise=noise) + super().__init__( + name=name, temporal_pmf_rv=shedding_kinetics_rv, noise=noise + ) self.log10_genome_per_infection_rv = log10_genome_per_infection_rv self.ml_per_person_per_day = ml_per_person_per_day @@ -466,6 +484,7 @@ site_sd_prior = GammaGroupSdPrior( # Create wastewater observation process ww_obs = Wastewater( + name="wastewater", shedding_kinetics_rv=shedding_rv, log10_genome_per_infection_rv=log10_genome_rv, ml_per_person_per_day=ml_per_person_per_day, @@ -516,12 +535,12 @@ builder.configure_latent( ### Specifying the Observation Processes, Data -The name passed to add_observation() becomes the keyword used to pass that observation's data to model.fit() (e.g., hospital={...}, wastewater={...}). +Each observation process's `name` attribute becomes the keyword used to pass that observation's data to `model.fit()` (e.g., `hospital={...}`, `wastewater={...}`). ```{python} # | label: add-observations-build -builder.add_observation(hosp_obs, "hospital") -builder.add_observation(ww_obs, "wastewater") +builder.add_observation(hosp_obs) # Uses hosp_obs.name = "hospital" +builder.add_observation(ww_obs) # Uses ww_obs.name = "wastewater" model = builder.build() n_init = model.latent.n_initialization_points @@ -540,11 +559,11 @@ When you call `model.fit()`, you supply two types of information: ### Observation Data by Signal Type -Each observation process is registered with a name via `add_observation()`. That name becomes the keyword argument for passing data to `model.fit()`: +Each observation process's `name` attribute becomes the keyword argument for passing data to `model.fit()`: ```python -builder.add_observation(hosp_obs, "hospital") # → hospital={...} -builder.add_observation(ww_obs, "wastewater") # → wastewater={...} +builder.add_observation(hosp_obs) # hosp_obs.name="hospital" → hospital={...} +builder.add_observation(ww_obs) # ww_obs.name="wastewater" → wastewater={...} ``` #### Jurisdiction-level signals diff --git a/pyrenew/datasets/hospital_admissions.py b/pyrenew/datasets/hospital_admissions.py index e3ffffbb..7a233fa3 100644 --- a/pyrenew/datasets/hospital_admissions.py +++ b/pyrenew/datasets/hospital_admissions.py @@ -40,13 +40,6 @@ def load_hospital_data_for_state( ----- Data source: CDC cfa-forecast-renewal-ww repository. License: Public Domain (CC0 1.0 Universal) - U.S. Government work. - - Examples - -------- - >>> from pyrenew import datasets - >>> ca_data = datasets.load_hospital_data_for_state("CA") - >>> ca_data["n_days"] - 311 """ data_path = files("pyrenew.datasets.hospital_admissions_data") / filename df = pl.read_csv(source=data_path) diff --git a/pyrenew/datasets/wastewater_nwss.py b/pyrenew/datasets/wastewater_nwss.py index c72a4fd8..4b36b3f5 100644 --- a/pyrenew/datasets/wastewater_nwss.py +++ b/pyrenew/datasets/wastewater_nwss.py @@ -56,13 +56,6 @@ def load_wastewater_data_for_state( The data is synthetic and contains deliberately added noise for public release. - - Examples - -------- - >>> from pyrenew import datasets - >>> ca_ww = datasets.load_wastewater_data_for_state("CA") - >>> ca_ww["n_sites"] - 5 """ data_path = files("pyrenew.datasets.wastewater_nwss_data") / filename df = pl.read_csv( diff --git a/pyrenew/model/model_builder.py b/pyrenew/model/model_builder.py index 8e8e25ee..eb88683a 100644 --- a/pyrenew/model/model_builder.py +++ b/pyrenew/model/model_builder.py @@ -116,19 +116,20 @@ def configure_latent( def add_observation( self, obs_process: BaseObservationProcess, - name: str, ) -> "ModelBuilder": """ Add an observation process to the model. + The observation process's ``name`` attribute is used as the unique + identifier for this observation. This name is used when passing + observation data to ``model.sample()`` and ``model.fit()``, and also + prefixes all numpyro sample sites created by the process. + Parameters ---------- obs_process : BaseObservationProcess - Configured observation process instance (e.g., CountObservation, - WastewaterObservation, CountObservationBySubpop) - name : str - Unique identifier for this observation. This name is used when passing - observation data to model.sample(). + Configured observation process instance (e.g., Counts, + Wastewater, CountsBySubpop). Must have a ``name`` attribute. Returns ------- @@ -140,6 +141,7 @@ def add_observation( ValueError If an observation with this name already exists """ + name = obs_process.name if name in self.observations: raise ValueError( f"Observation '{name}' already added. " diff --git a/pyrenew/model/multisignal_model.py b/pyrenew/model/multisignal_model.py index dc480e86..c45e4232 100644 --- a/pyrenew/model/multisignal_model.py +++ b/pyrenew/model/multisignal_model.py @@ -222,7 +222,7 @@ def sample( Population fractions for unobserved subpopulations. **observation_data Data for each observation process, keyed by observation name - (the names provided to builder.add_observation()). + (the ``name`` attribute of each observation process). Each value should be a dict of kwargs for that observation's sample(). Returns diff --git a/test/test_model_builder.py b/test/test_model_builder.py index 16a426c6..825c3751 100644 --- a/test/test_model_builder.py +++ b/test/test_model_builder.py @@ -40,11 +40,12 @@ def simple_builder(): delay = DeterministicPMF("delay", jnp.array([0.1, 0.3, 0.4, 0.2])) obs = Counts( + name="hospital", ascertainment_rate_rv=DeterministicVariable("ihr", 0.01), delay_distribution_rv=delay, noise=NegativeBinomialNoise(DeterministicVariable("conc", 10.0)), ) - builder.add_observation(obs, "hospital") + builder.add_observation(obs) return builder From b6399387ad73a725645fd17da1ed54bffafff764 Mon Sep 17 00:00:00 2001 From: Mitzi Morris Date: Thu, 15 Jan 2026 16:01:18 -0500 Subject: [PATCH 46/81] more unit tests --- pyrenew/latent/temporal_processes.py | 2 +- test/test_model_builder.py | 190 +++++++++++++++++++++ test/test_temporal_processes.py | 242 +++++++++++++++++++++++++++ 3 files changed, 433 insertions(+), 1 deletion(-) diff --git a/pyrenew/latent/temporal_processes.py b/pyrenew/latent/temporal_processes.py index d08b675f..f34659f1 100644 --- a/pyrenew/latent/temporal_processes.py +++ b/pyrenew/latent/temporal_processes.py @@ -401,7 +401,7 @@ def _sample_vectorized( noise_name=f"{name_prefix}_noise", ) - return trajectories[:n_timepoints, :] + return trajectories[:n_timepoints, :].squeeze(axis=1) class RandomWalk(TemporalProcess): diff --git a/test/test_model_builder.py b/test/test_model_builder.py index 825c3751..cc2c3ff0 100644 --- a/test/test_model_builder.py +++ b/test/test_model_builder.py @@ -69,11 +69,96 @@ def test_rejects_population_structure_at_configure_time(self): obs_fractions=jnp.array([0.5, 0.5]), # Should fail ) + def test_rejects_n_initialization_points_at_configure_time(self): + """Test that n_initialization_points is rejected at configure time.""" + builder = ModelBuilder() + gen_int = DeterministicPMF("gen_int", jnp.array([0.2, 0.5, 0.3])) + + with pytest.raises(ValueError, match="Do not specify n_initialization_points"): + builder.configure_latent( + HierarchicalInfections, + gen_int_rv=gen_int, + I0_rv=DeterministicVariable("I0", 0.001), + initial_log_rt_rv=DeterministicVariable("initial_log_rt", 0.0), + baseline_temporal=RandomWalk(), + subpop_temporal=RandomWalk(), + n_initialization_points=10, # Should fail + ) + + def test_rejects_reconfiguring_latent(self): + """Test that configuring latent twice raises RuntimeError.""" + builder = ModelBuilder() + gen_int = DeterministicPMF("gen_int", jnp.array([0.2, 0.5, 0.3])) + + builder.configure_latent( + HierarchicalInfections, + gen_int_rv=gen_int, + I0_rv=DeterministicVariable("I0", 0.001), + initial_log_rt_rv=DeterministicVariable("initial_log_rt", 0.0), + baseline_temporal=RandomWalk(), + subpop_temporal=RandomWalk(), + ) + + with pytest.raises(RuntimeError, match="already configured"): + builder.configure_latent( + HierarchicalInfections, + gen_int_rv=gen_int, + I0_rv=DeterministicVariable("I0", 0.001), + initial_log_rt_rv=DeterministicVariable("initial_log_rt", 0.0), + baseline_temporal=RandomWalk(), + subpop_temporal=RandomWalk(), + ) + + def test_rejects_duplicate_observation_name(self, simple_builder): + """Test that adding duplicate observation name raises ValueError.""" + delay = DeterministicPMF("delay2", jnp.array([0.5, 0.5])) + obs = Counts( + name="hospital", # Same name as existing observation + ascertainment_rate_rv=DeterministicVariable("ihr2", 0.02), + delay_distribution_rv=delay, + noise=NegativeBinomialNoise(DeterministicVariable("conc2", 20.0)), + ) + + with pytest.raises(ValueError, match="already added"): + simple_builder.add_observation(obs) + def test_build_creates_model(self, simple_builder): """Test that build() creates a MultiSignalModel.""" model = simple_builder.build() assert isinstance(model, MultiSignalModel) + def test_build_without_latent_raises_error(self): + """Test that build() without configure_latent raises ValueError.""" + builder = ModelBuilder() + + with pytest.raises(ValueError, match="Must call configure_latent"): + builder.build() + + def test_compute_n_initialization_points_without_latent_raises(self): + """Test that compute_n_initialization_points without latent raises.""" + builder = ModelBuilder() + + with pytest.raises(ValueError, match="Must call configure_latent"): + builder.compute_n_initialization_points() + + def test_compute_n_initialization_points_without_gen_int_raises(self): + """Test that compute_n_initialization_points without gen_int_rv raises.""" + builder = ModelBuilder() + # Configure latent without gen_int_rv + builder.latent_class = HierarchicalInfections + builder.latent_params = {} # Missing gen_int_rv + + with pytest.raises(ValueError, match="gen_int_rv is required"): + builder.compute_n_initialization_points() + + def test_compute_n_initialization_points_returns_correct_value( + self, simple_builder + ): + """Test that compute_n_initialization_points returns max of lookbacks.""" + n_init = simple_builder.compute_n_initialization_points() + # gen_int has 3 elements, delay has 4 elements, so max is 4 + assert n_init == 4 + class TestMultiSignalModelSampling: """Test MultiSignalModel sampling with population structure at sample time.""" @@ -155,6 +240,111 @@ def test_validate_data_rejects_out_of_bounds_times(self, simple_builder): }, ) + def test_validate_data_rejects_negative_times(self, simple_builder): + """Test that negative times raises error.""" + model = simple_builder.build() + + with pytest.raises(ValueError, match="cannot contain negative"): + model.validate_data( + n_days_post_init=30, + obs_fractions=OBS_FRACTIONS, + unobs_fractions=UNOBS_FRACTIONS, + hospital={ + "obs": jnp.array([10]), + "times": jnp.array([-1]), + }, + ) + + def test_validate_data_rejects_unknown_observation(self, simple_builder): + """Test that unknown observation name raises error.""" + model = simple_builder.build() + + with pytest.raises(ValueError, match="Unknown observation"): + model.validate_data( + n_days_post_init=30, + obs_fractions=OBS_FRACTIONS, + unobs_fractions=UNOBS_FRACTIONS, + unknown_obs={ + "obs": jnp.array([10]), + "times": jnp.array([5]), + }, + ) + + def test_validate_data_rejects_mismatched_obs_times_length(self, simple_builder): + """Test that mismatched obs and times lengths raises error.""" + model = simple_builder.build() + + with pytest.raises(ValueError, match="obs length.*must match times length"): + model.validate_data( + n_days_post_init=30, + obs_fractions=OBS_FRACTIONS, + unobs_fractions=UNOBS_FRACTIONS, + hospital={ + "obs": jnp.array([10, 20, 30]), # 3 elements + "times": jnp.array([5, 10]), # 2 elements + }, + ) + + def test_validate_method_calls_internal_validate(self, simple_builder): + """Test that validate() method calls _validate().""" + model = simple_builder.build() + # Should not raise + model.validate() + + +class TestMultiSignalModelObservationValidation: + """Test observation process validation in MultiSignalModel.""" + + def test_rejects_observation_without_infection_resolution(self): + """Test that observations must implement infection_resolution().""" + from pyrenew.observation.base import BaseObservationProcess + + class BadObservation(BaseObservationProcess): + """Observation that raises NotImplementedError for infection_resolution.""" + + def __init__(self): + """Initialize without temporal_pmf_rv.""" + self.name = "bad" + self.temporal_pmf_rv = None + + def sample(self, **kwargs): + """Sample stub.""" + pass + + def validate(self): + """Validate stub.""" + pass + + def lookback_days(self): + """Return lookback.""" + return 1 + + def infection_resolution(self): + """Raise NotImplementedError to simulate missing implementation.""" + raise NotImplementedError("Not implemented") + + def _predicted_obs(self, infections): + """Predicted obs stub.""" + return infections + + builder = ModelBuilder() + gen_int = DeterministicPMF("gen_int", jnp.array([0.2, 0.5, 0.3])) + + builder.configure_latent( + HierarchicalInfections, + gen_int_rv=gen_int, + I0_rv=DeterministicVariable("I0", 0.001), + initial_log_rt_rv=DeterministicVariable("initial_log_rt", 0.0), + baseline_temporal=RandomWalk(), + subpop_temporal=RandomWalk(), + ) + + bad_obs = BadObservation() + builder.add_observation(bad_obs) + + with pytest.raises(ValueError, match="must implement infection_resolution"): + builder.build() + if __name__ == "__main__": pytest.main([__file__, "-v"]) diff --git a/test/test_temporal_processes.py b/test/test_temporal_processes.py index de43be35..7c9b585d 100644 --- a/test/test_temporal_processes.py +++ b/test/test_temporal_processes.py @@ -9,6 +9,248 @@ from pyrenew.latent import AR1, DifferencedAR1, RandomWalk +class TestTemporalProcessRepr: + """Test __repr__ methods for temporal processes.""" + + def test_ar1_repr(self): + """Test AR1 __repr__ method.""" + ar1 = AR1(autoreg=0.7, innovation_sd=0.5) + repr_str = repr(ar1) + assert "AR1" in repr_str + assert "autoreg=0.7" in repr_str + assert "innovation_sd=0.5" in repr_str + + def test_differenced_ar1_repr(self): + """Test DifferencedAR1 __repr__ method.""" + dar1 = DifferencedAR1(autoreg=0.6, innovation_sd=0.3) + repr_str = repr(dar1) + assert "DifferencedAR1" in repr_str + assert "autoreg=0.6" in repr_str + assert "innovation_sd=0.3" in repr_str + + def test_random_walk_repr(self): + """Test RandomWalk __repr__ method.""" + rw = RandomWalk(innovation_sd=0.2) + repr_str = repr(rw) + assert "RandomWalk" in repr_str + assert "innovation_sd=0.2" in repr_str + + +class TestAR1Validation: + """Test AR1 validation.""" + + def test_ar1_rejects_autoreg_equal_one(self): + """Test that AR1 rejects autoreg=1.""" + with pytest.raises(ValueError, match="must satisfy"): + AR1(autoreg=1.0) + + def test_ar1_rejects_autoreg_negative_one(self): + """Test that AR1 rejects autoreg=-1.""" + with pytest.raises(ValueError, match="must satisfy"): + AR1(autoreg=-1.0) + + def test_ar1_rejects_autoreg_greater_than_one(self): + """Test that AR1 rejects autoreg > 1.""" + with pytest.raises(ValueError, match="must satisfy"): + AR1(autoreg=1.5) + + +class TestDifferencedAR1Validation: + """Test DifferencedAR1 validation.""" + + def test_differenced_ar1_rejects_autoreg_equal_one(self): + """Test that DifferencedAR1 rejects autoreg=1.""" + with pytest.raises(ValueError, match="must satisfy"): + DifferencedAR1(autoreg=1.0) + + def test_differenced_ar1_rejects_autoreg_negative_one(self): + """Test that DifferencedAR1 rejects autoreg=-1.""" + with pytest.raises(ValueError, match="must satisfy"): + DifferencedAR1(autoreg=-1.0) + + +class TestAR1VectorizedSampling: + """Test AR1 vectorized sampling.""" + + def test_ar1_vectorized_sample_shape(self): + """Test that AR1 vectorized sample returns correct shape.""" + n_timepoints = 30 + n_processes = 5 + + ar1 = AR1(autoreg=0.7, innovation_sd=0.3) + + with numpyro.handlers.seed(rng_seed=42): + trajectories = ar1.sample( + n_timepoints=n_timepoints, + n_processes=n_processes, + ) + + assert trajectories.shape == (n_timepoints, n_processes) + + def test_ar1_vectorized_with_initial_values_array(self): + """Test AR1 vectorized with array of initial values.""" + n_timepoints = 30 + n_processes = 4 + initial_values = jnp.array([0.0, 1.0, -1.0, 2.0]) + + ar1 = AR1(autoreg=0.7, innovation_sd=0.3) + + with numpyro.handlers.seed(rng_seed=42): + trajectories = ar1.sample( + n_timepoints=n_timepoints, + n_processes=n_processes, + initial_value=initial_values, + ) + + assert trajectories.shape == (n_timepoints, n_processes) + + def test_ar1_vectorized_with_scalar_initial_value(self): + """Test AR1 vectorized with scalar initial value (broadcast).""" + n_timepoints = 30 + n_processes = 3 + + ar1 = AR1(autoreg=0.7, innovation_sd=0.3) + + with numpyro.handlers.seed(rng_seed=42): + trajectories = ar1.sample( + n_timepoints=n_timepoints, + n_processes=n_processes, + initial_value=1.0, + ) + + assert trajectories.shape == (n_timepoints, n_processes) + + +class TestDifferencedAR1Sampling: + """Test DifferencedAR1 sampling methods.""" + + def test_differenced_ar1_single_sample_shape(self): + """Test that DifferencedAR1 single sample returns correct shape.""" + n_timepoints = 30 + + dar1 = DifferencedAR1(autoreg=0.6, innovation_sd=0.3) + + with numpyro.handlers.seed(rng_seed=42): + trajectory = dar1.sample(n_timepoints=n_timepoints) + + assert trajectory.shape == (n_timepoints,) + + def test_differenced_ar1_single_with_initial_value(self): + """Test DifferencedAR1 single sample with initial value.""" + n_timepoints = 30 + + dar1 = DifferencedAR1(autoreg=0.6, innovation_sd=0.3) + + with numpyro.handlers.seed(rng_seed=42): + trajectory = dar1.sample( + n_timepoints=n_timepoints, + initial_value=1.5, + ) + + assert trajectory.shape == (n_timepoints,) + + def test_differenced_ar1_vectorized_sample_shape(self): + """Test that DifferencedAR1 vectorized sample returns correct shape.""" + n_timepoints = 30 + n_processes = 5 + + dar1 = DifferencedAR1(autoreg=0.6, innovation_sd=0.3) + + with numpyro.handlers.seed(rng_seed=42): + trajectories = dar1.sample( + n_timepoints=n_timepoints, + n_processes=n_processes, + ) + + assert trajectories.shape == (n_timepoints, n_processes) + + def test_differenced_ar1_vectorized_with_initial_values_array(self): + """Test DifferencedAR1 vectorized with array of initial values.""" + n_timepoints = 30 + n_processes = 4 + initial_values = jnp.array([0.0, 1.0, -1.0, 2.0]) + + dar1 = DifferencedAR1(autoreg=0.6, innovation_sd=0.3) + + with numpyro.handlers.seed(rng_seed=42): + trajectories = dar1.sample( + n_timepoints=n_timepoints, + n_processes=n_processes, + initial_value=initial_values, + ) + + assert trajectories.shape == (n_timepoints, n_processes) + + def test_differenced_ar1_vectorized_with_scalar_initial_value(self): + """Test DifferencedAR1 vectorized with scalar initial value.""" + n_timepoints = 30 + n_processes = 3 + + dar1 = DifferencedAR1(autoreg=0.6, innovation_sd=0.3) + + with numpyro.handlers.seed(rng_seed=42): + trajectories = dar1.sample( + n_timepoints=n_timepoints, + n_processes=n_processes, + initial_value=1.0, + ) + + assert trajectories.shape == (n_timepoints, n_processes) + + def test_differenced_ar1_override_innovation_sd(self): + """Test DifferencedAR1 with overridden innovation_sd.""" + dar1 = DifferencedAR1(autoreg=0.6, innovation_sd=0.1) + + with numpyro.handlers.seed(rng_seed=42): + trajectory = dar1.sample( + n_timepoints=30, + innovation_sd=0.5, # Override constructor value + ) + + assert trajectory.shape == (30,) + + +class TestRandomWalkVectorizedSampling: + """Test RandomWalk vectorized sampling.""" + + def test_random_walk_vectorized_with_initial_values_array(self): + """Test RandomWalk vectorized with array of initial values.""" + n_timepoints = 30 + n_processes = 4 + initial_values = jnp.array([0.0, 1.0, -1.0, 2.0]) + + rw = RandomWalk(innovation_sd=0.3) + + with numpyro.handlers.seed(rng_seed=42): + trajectories = rw.sample( + n_timepoints=n_timepoints, + n_processes=n_processes, + initial_value=initial_values, + ) + + assert trajectories.shape == (n_timepoints, n_processes) + # First row should be close to initial values (may vary by implementation) + assert jnp.allclose(trajectories[0, :], initial_values) + + def test_random_walk_vectorized_with_scalar_initial_value(self): + """Test RandomWalk vectorized with scalar initial value.""" + n_timepoints = 30 + n_processes = 3 + + rw = RandomWalk(innovation_sd=0.3) + + with numpyro.handlers.seed(rng_seed=42): + trajectories = rw.sample( + n_timepoints=n_timepoints, + n_processes=n_processes, + initial_value=1.0, + ) + + assert trajectories.shape == (n_timepoints, n_processes) + # First row should be all 1.0 + assert jnp.allclose(trajectories[0, :], 1.0) + + class TestTemporalProcessInnovationSD: """Test that temporal processes correctly use innovation_sd parameter.""" From 49e1f34f82b95e382fda0b694688867c37cdd3a8 Mon Sep 17 00:00:00 2001 From: Mitzi Morris Date: Thu, 15 Jan 2026 16:50:59 -0500 Subject: [PATCH 47/81] more unit tests; lint fix --- pyrenew/latent/base.py | 4 +- pyrenew/latent/hierarchical_infections.py | 6 +- pyrenew/latent/partitioned_infections.py | 6 +- pyrenew/latent/temporal_processes.py | 4 +- pyrenew/observation/__init__.py | 4 +- pyrenew/observation/noise.py | 14 +- pyrenew/regression.py | 2 +- test/test_base_latent_process.py | 146 +++++++++++++++++- test/test_hierarchical_infections.py | 75 ++++++++++ test/test_hierarchical_priors.py | 28 ++++ test/test_model_builder.py | 174 +++++++++++++++++++++- test/test_observation_counts.py | 32 ++++ test/test_observation_measurements.py | 14 ++ test/test_partitioned_infections.py | 78 ++++++++++ 14 files changed, 564 insertions(+), 23 deletions(-) diff --git a/pyrenew/latent/base.py b/pyrenew/latent/base.py index c23e9ebe..d65f129d 100644 --- a/pyrenew/latent/base.py +++ b/pyrenew/latent/base.py @@ -354,7 +354,7 @@ def validate(self) -> None: ValueError If any parameters fail validation """ - pass + pass # pragma: no cover @abstractmethod def sample( @@ -390,4 +390,4 @@ def sample( where n_total_days = n_initialization_points + n_days_post_init """ - pass + pass # pragma: no cover diff --git a/pyrenew/latent/hierarchical_infections.py b/pyrenew/latent/hierarchical_infections.py index 51c195c4..b3c2d855 100644 --- a/pyrenew/latent/hierarchical_infections.py +++ b/pyrenew/latent/hierarchical_infections.py @@ -148,14 +148,14 @@ def validate(self) -> None: If any parameters fail validation """ gen_int_pmf = self.gen_int_rv() - if gen_int_pmf.size == 0: + if gen_int_pmf.size == 0: # pragma: no cover raise ValueError("gen_int_rv must return non-empty array") pmf_sum = jnp.sum(gen_int_pmf) - if not jnp.isclose(pmf_sum, 1.0, atol=1e-6): + if not jnp.isclose(pmf_sum, 1.0, atol=1e-6): # pragma: no cover raise ValueError(f"gen_int_rv must sum to 1.0, got {float(pmf_sum):.6f}") - if jnp.any(gen_int_pmf < 0): + if jnp.any(gen_int_pmf < 0): # pragma: no cover raise ValueError("gen_int_rv must have non-negative values") def sample( diff --git a/pyrenew/latent/partitioned_infections.py b/pyrenew/latent/partitioned_infections.py index f28ae517..d49612c9 100644 --- a/pyrenew/latent/partitioned_infections.py +++ b/pyrenew/latent/partitioned_infections.py @@ -141,14 +141,14 @@ def validate(self) -> None: If any parameters fail validation """ gen_int_pmf = self.gen_int_rv() - if gen_int_pmf.size == 0: + if gen_int_pmf.size == 0: # pragma: no cover raise ValueError("gen_int_rv must return non-empty array") pmf_sum = jnp.sum(gen_int_pmf) - if not jnp.isclose(pmf_sum, 1.0, atol=1e-6): + if not jnp.isclose(pmf_sum, 1.0, atol=1e-6): # pragma: no cover raise ValueError(f"gen_int_rv must sum to 1.0, got {float(pmf_sum):.6f}") - if jnp.any(gen_int_pmf < 0): + if jnp.any(gen_int_pmf < 0): # pragma: no cover raise ValueError("gen_int_rv must have non-negative values") def sample( diff --git a/pyrenew/latent/temporal_processes.py b/pyrenew/latent/temporal_processes.py index f34659f1..70d82789 100644 --- a/pyrenew/latent/temporal_processes.py +++ b/pyrenew/latent/temporal_processes.py @@ -6,8 +6,8 @@ deviations. All processes support both single trajectory and vectorized (multiple parallel trajectories) sampling. -Temporal Processes ------------------- +**Temporal Processes** + - ``AR1``: Autoregressive process with mean reversion. Keeps Rt bounded near a baseline. - ``DifferencedAR1``: AR(1) on first differences. Allows persistent trends diff --git a/pyrenew/observation/__init__.py b/pyrenew/observation/__init__.py index 42f0aafd..c65f0917 100644 --- a/pyrenew/observation/__init__.py +++ b/pyrenew/observation/__init__.py @@ -2,8 +2,8 @@ """ Observation processes for connecting infections to observed data. -Architecture ------------- +**Architecture** + ``BaseObservationProcess`` is the abstract base. Concrete subclasses: - ``Counts``: Aggregate counts (admissions, deaths) diff --git a/pyrenew/observation/noise.py b/pyrenew/observation/noise.py index 01c833fe..349d481d 100644 --- a/pyrenew/observation/noise.py +++ b/pyrenew/observation/noise.py @@ -5,20 +5,20 @@ Provides composable noise strategies for count and measurement observations, separating the noise distribution from the observation structure. -Count Noise ------------ +**Count Noise** + - ``PoissonNoise``: Equidispersed counts (variance = mean). No parameters. - ``NegativeBinomialNoise``: Overdispersed counts relative to Poisson (variance > mean). Takes ``concentration_rv`` (higher concentration = less overdispersed, more Poisson-like). -Measurement Noise ------------------ +**Measurement Noise** + - ``HierarchicalNormalNoise``: Normal noise with hierarchical sensor effects. Takes ``sensor_mode_rv`` and ``sensor_sd_rv`` for sensor-level bias and variability. -Utilities ---------- +**Utilities** + - ``VectorizedRV``: Wrapper that adds ``n_groups`` support to simple RVs. """ @@ -65,7 +65,7 @@ def __init__(self, rv: RandomVariable, plate_name: str) -> None: self.rv = rv self.plate_name = plate_name - def validate(self): + def validate(self): # pragma: no cover """Validate the underlying RV.""" self.rv.validate() diff --git a/pyrenew/regression.py b/pyrenew/regression.py index 32029b9e..257be338 100755 --- a/pyrenew/regression.py +++ b/pyrenew/regression.py @@ -157,7 +157,7 @@ def sample(self, predictor_values: ArrayLike) -> GLMPredictionSample: Sample generalized linear model Parameters - ----------- + ---------- predictor_values Matrix of predictor variables (covariates) for the regression problem. Each row should represent the diff --git a/test/test_base_latent_process.py b/test/test_base_latent_process.py index 90e1c42c..77cd37a8 100644 --- a/test/test_base_latent_process.py +++ b/test/test_base_latent_process.py @@ -5,7 +5,9 @@ import jax.numpy as jnp import pytest -from pyrenew.latent.base import BaseLatentInfectionProcess +from pyrenew.deterministic import DeterministicPMF, DeterministicVariable +from pyrenew.latent import HierarchicalInfections, RandomWalk +from pyrenew.latent.base import BaseLatentInfectionProcess, LatentSample class TestPopulationStructureParsing: @@ -54,6 +56,148 @@ def test_rejects_missing_fractions(self): unobs_fractions=jnp.array([0.45]), ) + def test_rejects_2d_fractions(self): + """Test that 2D fraction arrays raise error.""" + with pytest.raises(ValueError, match="Fractions must be 1D arrays"): + BaseLatentInfectionProcess._parse_and_validate_fractions( + obs_fractions=jnp.array([[0.3, 0.25]]), # 2D array + unobs_fractions=jnp.array([0.45]), + ) + + def test_rejects_no_observed_subpopulations(self): + """Test that empty observed subpopulations raise error.""" + with pytest.raises( + ValueError, match="Must have at least one observed subpopulation" + ): + BaseLatentInfectionProcess._parse_and_validate_fractions( + obs_fractions=jnp.array([]), # No observed subpops + unobs_fractions=jnp.array([1.0]), + ) + + +class TestBaseLatentInfectionProcessInit: + """Test BaseLatentInfectionProcess initialization.""" + + def test_rejects_missing_gen_int_rv(self): + """Test that None gen_int_rv is rejected.""" + with pytest.raises(ValueError, match="gen_int_rv is required"): + # Create a minimal concrete subclass for testing + class ConcreteLatent(BaseLatentInfectionProcess): + def validate(self): + pass + + def sample(self, n_days_post_init, **kwargs): + pass + + ConcreteLatent(gen_int_rv=None) + + def test_rejects_negative_n_initialization_points(self): + """Test that negative n_initialization_points is rejected.""" + with pytest.raises( + ValueError, match="n_initialization_points must be non-negative" + ): + + class ConcreteLatent(BaseLatentInfectionProcess): + def validate(self): + pass + + def sample(self, n_days_post_init, **kwargs): + pass + + ConcreteLatent( + gen_int_rv=DeterministicPMF("gen_int", jnp.array([0.2, 0.5, 0.3])), + n_initialization_points=-1, + ) + + +class TestGetRequiredLookback: + """Test get_required_lookback method.""" + + def test_get_required_lookback_returns_gen_int_length(self): + """Test that get_required_lookback returns generation interval length.""" + gen_int = DeterministicPMF("gen_int", jnp.array([0.2, 0.5, 0.3])) + process = HierarchicalInfections( + gen_int_rv=gen_int, + I0_rv=DeterministicVariable("I0", 0.001), + initial_log_rt_rv=DeterministicVariable("initial_log_rt", 0.0), + baseline_temporal=RandomWalk(), + subpop_temporal=RandomWalk(), + ) + + assert process.get_required_lookback() == 3 + + +class TestValidateOutputShapes: + """Test _validate_output_shapes method.""" + + def test_validate_output_shapes_raises_on_mismatch(self): + """Test that shape validation raises on incorrect shapes.""" + from pyrenew.latent.base import PopulationStructure + + pop = PopulationStructure( + obs_fractions=jnp.array([0.5, 0.5]), + unobs_fractions=jnp.array([]), + K=2, + K_obs=2, + K_unobs=0, + ) + + # Create arrays with wrong shapes + infections_aggregate = jnp.ones(10) # Correct: (10,) + infections_all = jnp.ones((10, 2)) # Correct: (10, 2) + infections_observed = jnp.ones((10, 3)) # WRONG: should be (10, 2) + infections_unobserved = jnp.ones((10, 0)) # Correct: (10, 0) + + with pytest.raises(ValueError, match="has incorrect shape"): + BaseLatentInfectionProcess._validate_output_shapes( + infections_aggregate, + infections_all, + infections_observed, + infections_unobserved, + n_total_days=10, + pop=pop, + ) + + +class TestValidateI0: + """Test _validate_I0 method.""" + + def test_validate_I0_rejects_values_greater_than_one(self): + """Test that I0 values > 1 are rejected.""" + with pytest.raises(ValueError, match="I0 must be <= 1"): + BaseLatentInfectionProcess._validate_I0(jnp.array(1.5)) + + def test_validate_I0_rejects_zero(self): + """Test that I0 = 0 is rejected.""" + with pytest.raises(ValueError, match="I0 must be positive"): + BaseLatentInfectionProcess._validate_I0(jnp.array(0.0)) + + def test_validate_I0_accepts_valid_values(self): + """Test that valid I0 values are accepted.""" + # Should not raise + BaseLatentInfectionProcess._validate_I0(jnp.array(0.001)) + BaseLatentInfectionProcess._validate_I0(jnp.array(1.0)) + BaseLatentInfectionProcess._validate_I0(jnp.array([0.001, 0.002])) + + +class TestLatentSample: + """Test LatentSample named tuple.""" + + def test_latent_sample_unpacking(self): + """Test that LatentSample can be unpacked correctly.""" + sample = LatentSample( + aggregate=jnp.ones(10), + all_subpops=jnp.ones((10, 2)), + observed=jnp.ones((10, 1)), + unobserved=jnp.ones((10, 1)), + ) + + agg, all_s, obs, unobs = sample + assert agg.shape == (10,) + assert all_s.shape == (10, 2) + assert obs.shape == (10, 1) + assert unobs.shape == (10, 1) + if __name__ == "__main__": pytest.main([__file__, "-v"]) diff --git a/test/test_hierarchical_infections.py b/test/test_hierarchical_infections.py index 7230ca6e..82790235 100644 --- a/test/test_hierarchical_infections.py +++ b/test/test_hierarchical_infections.py @@ -119,6 +119,50 @@ def test_deviations_sum_to_zero(self, process): class TestHierarchicalInfectionsValidation: """Test validation of inputs.""" + def test_rejects_missing_I0_rv(self, gen_int_rv): + """Test that None I0_rv is rejected.""" + with pytest.raises(ValueError, match="I0_rv is required"): + HierarchicalInfections( + gen_int_rv=gen_int_rv, + I0_rv=None, + initial_log_rt_rv=DeterministicVariable("initial_log_rt", 0.0), + baseline_temporal=RandomWalk(), + subpop_temporal=RandomWalk(), + ) + + def test_rejects_missing_initial_log_rt_rv(self, gen_int_rv): + """Test that None initial_log_rt_rv is rejected.""" + with pytest.raises(ValueError, match="initial_log_rt_rv is required"): + HierarchicalInfections( + gen_int_rv=gen_int_rv, + I0_rv=DeterministicVariable("I0", 0.001), + initial_log_rt_rv=None, + baseline_temporal=RandomWalk(), + subpop_temporal=RandomWalk(), + ) + + def test_rejects_missing_baseline_temporal(self, gen_int_rv): + """Test that None baseline_temporal is rejected.""" + with pytest.raises(ValueError, match="baseline_temporal is required"): + HierarchicalInfections( + gen_int_rv=gen_int_rv, + I0_rv=DeterministicVariable("I0", 0.001), + initial_log_rt_rv=DeterministicVariable("initial_log_rt", 0.0), + baseline_temporal=None, + subpop_temporal=RandomWalk(), + ) + + def test_rejects_missing_subpop_temporal(self, gen_int_rv): + """Test that None subpop_temporal is rejected.""" + with pytest.raises(ValueError, match="subpop_temporal is required"): + HierarchicalInfections( + gen_int_rv=gen_int_rv, + I0_rv=DeterministicVariable("I0", 0.001), + initial_log_rt_rv=DeterministicVariable("initial_log_rt", 0.0), + baseline_temporal=RandomWalk(), + subpop_temporal=None, + ) + def test_rejects_invalid_I0(self, gen_int_rv): """Test that invalid I0 values are rejected.""" with pytest.raises(ValueError, match="I0 must be positive"): @@ -140,6 +184,37 @@ def test_rejects_fractions_not_summing_to_one(self, process): unobs_fractions=jnp.array([0.40]), # Sum is 0.95 ) + def test_validate_method(self, process): + """Test that validate() method runs without error.""" + process.validate() + + +class TestHierarchicalInfectionsPerSubpopI0: + """Test per-subpopulation I0 values.""" + + def test_per_subpop_I0_array(self, gen_int_rv): + """Test with per-subpopulation I0 values (array instead of scalar).""" + process = HierarchicalInfections( + gen_int_rv=gen_int_rv, + I0_rv=DeterministicVariable("I0", jnp.array([0.001, 0.002, 0.0015])), + initial_log_rt_rv=DeterministicVariable("initial_log_rt", 0.0), + baseline_temporal=RandomWalk(), + subpop_temporal=RandomWalk(), + ) + + with numpyro.handlers.seed(rng_seed=42): + result = process.sample( + n_days_post_init=30, + obs_fractions=jnp.array([0.3, 0.25]), + unobs_fractions=jnp.array([0.45]), + ) + + inf_juris, inf_all, inf_obs, inf_unobs = result + n_total = process.n_initialization_points + 30 + + assert inf_juris.shape == (n_total,) + assert inf_all.shape == (n_total, 3) + if __name__ == "__main__": pytest.main([__file__, "-v"]) diff --git a/test/test_hierarchical_priors.py b/test/test_hierarchical_priors.py index e72a7a48..70f9a50f 100644 --- a/test/test_hierarchical_priors.py +++ b/test/test_hierarchical_priors.py @@ -61,6 +61,14 @@ def test_accepts_distributional_variable_for_sd(self): assert samples.shape == (5,) + def test_validate_method(self): + """Test that validate() method runs without error.""" + prior = HierarchicalNormalPrior( + "effect", sd_rv=DeterministicVariable("sd", 1.0) + ) + # Should not raise - validate is a no-op + prior.validate() + class TestGammaGroupSdPrior: """Test GammaGroupSdPrior.""" @@ -120,6 +128,16 @@ def test_rejects_negative_sd_min(self): sd_min=-0.1, ) + def test_validate_method(self): + """Test that validate() method runs without error.""" + prior = GammaGroupSdPrior( + "sd", + sd_mean_rv=DeterministicVariable("sd_mean", 0.5), + sd_concentration_rv=DeterministicVariable("sd_conc", 4.0), + ) + # Should not raise - validate is a no-op + prior.validate() + class TestStudentTGroupModePrior: """Test StudentTGroupModePrior.""" @@ -174,6 +192,16 @@ def test_rejects_non_random_variable_params(self): df_rv=4.0, ) + def test_validate_method(self): + """Test that validate() method runs without error.""" + prior = StudentTGroupModePrior( + "mode", + sd_rv=DeterministicVariable("sd", 1.0), + df_rv=DeterministicVariable("df", 4.0), + ) + # Should not raise - validate is a no-op + prior.validate() + if __name__ == "__main__": pytest.main([__file__, "-v"]) diff --git a/test/test_model_builder.py b/test/test_model_builder.py index cc2c3ff0..bab89446 100644 --- a/test/test_model_builder.py +++ b/test/test_model_builder.py @@ -182,6 +182,35 @@ def test_sample_with_population_structure(self, simple_builder): assert inf_all.shape == (n_total, 3) # K=3 assert inf_obs.shape == (n_total, 2) # K_obs=2 + def test_fit_with_reparam_config(self, simple_builder): + """Test that fit() works with reparameterization config.""" + from numpyro.infer.reparam import LocScaleReparam + + model = simple_builder.build() + n_days = 10 + + times = jnp.array([5, 6, 7, 8, 9]) + obs = jnp.array([10, 12, 15, 14, 11]) + + # Test with reparam_config + reparam_config = { + "log_rt_baseline_innovations": LocScaleReparam(centered=0), + } + + mcmc = model.fit( + n_days_post_init=n_days, + population_size=1_000_000, + obs_fractions=OBS_FRACTIONS, + unobs_fractions=UNOBS_FRACTIONS, + num_warmup=2, + num_samples=2, + reparam_config=reparam_config, + hospital={"obs": obs, "times": times}, + ) + + samples = mcmc.get_samples() + assert "latent_infections" in samples + def test_fit_with_population_structure(self, simple_builder): """Test that fit() works with population structure at sample time.""" model = simple_builder.build() @@ -291,6 +320,133 @@ def test_validate_method_calls_internal_validate(self, simple_builder): # Should not raise model.validate() + def test_validate_data_rejects_negative_subpop_indices(self, simple_builder): + """Test that negative subpop_indices raises error.""" + model = simple_builder.build() + + with pytest.raises(ValueError, match="subpop_indices cannot be negative"): + model.validate_data( + n_days_post_init=30, + obs_fractions=OBS_FRACTIONS, + unobs_fractions=UNOBS_FRACTIONS, + hospital={ + "subpop_indices": jnp.array([-1, 0, 1]), + "times": jnp.array([5, 6, 7]), + }, + ) + + def test_validate_data_rejects_out_of_bounds_subpop_indices(self, simple_builder): + """Test that subpop_indices >= K_obs raises error.""" + model = simple_builder.build() + + # K_obs is 2 (from OBS_FRACTIONS = [0.3, 0.25]) + with pytest.raises(ValueError, match="subpop_indices contains"): + model.validate_data( + n_days_post_init=30, + obs_fractions=OBS_FRACTIONS, + unobs_fractions=UNOBS_FRACTIONS, + hospital={ + "subpop_indices": jnp.array([0, 1, 5]), # 5 >= 2 + "times": jnp.array([5, 6, 7]), + }, + ) + + +class TestModelBuilderErrorHandling: + """Test ModelBuilder error handling.""" + + def test_compute_n_init_rejects_obs_without_lookback_days(self): + """Test that observation without lookback_days() raises error.""" + from pyrenew.observation.base import BaseObservationProcess + + class BadObservationNoLookback(BaseObservationProcess): + """Observation that has no lookback_days method.""" + + def __init__(self): + """Initialize without temporal_pmf_rv.""" + self.name = "bad" + self.temporal_pmf_rv = None + + def sample(self, **kwargs): + """Sample stub.""" + pass + + def validate(self): + """Validate stub.""" + pass + + def lookback_days(self): + """Raise NotImplementedError to simulate missing implementation.""" + raise NotImplementedError("Not implemented") + + def infection_resolution(self): + """ + Return aggregate. + + Returns + ------- + str + The string "aggregate". + """ + return "aggregate" + + def _predicted_obs(self, infections): + """ + Predicted obs stub. + + Returns + ------- + ArrayLike + The infections array unchanged. + """ + return infections + + builder = ModelBuilder() + gen_int = DeterministicPMF("gen_int", jnp.array([0.2, 0.5, 0.3])) + + builder.configure_latent( + HierarchicalInfections, + gen_int_rv=gen_int, + I0_rv=DeterministicVariable("I0", 0.001), + initial_log_rt_rv=DeterministicVariable("initial_log_rt", 0.0), + baseline_temporal=RandomWalk(), + subpop_temporal=RandomWalk(), + ) + + bad_obs = BadObservationNoLookback() + builder.add_observation(bad_obs) + + with pytest.raises(ValueError, match="must implement lookback_days"): + builder.compute_n_initialization_points() + + def test_build_raises_on_construction_error(self): + """Test that build() raises TypeError on latent construction failure.""" + builder = ModelBuilder() + gen_int = DeterministicPMF("gen_int", jnp.array([0.2, 0.5, 0.3])) + + # Configure with an invalid parameter that will cause construction to fail + builder.configure_latent( + HierarchicalInfections, + gen_int_rv=gen_int, + I0_rv=DeterministicVariable("I0", 0.001), + initial_log_rt_rv=DeterministicVariable("initial_log_rt", 0.0), + baseline_temporal=RandomWalk(), + subpop_temporal=RandomWalk(), + invalid_extra_param="this will cause TypeError", # Invalid param + ) + + delay = DeterministicPMF("delay", jnp.array([0.5, 0.5])) + obs = Counts( + name="hospital", + ascertainment_rate_rv=DeterministicVariable("ihr", 0.01), + delay_distribution_rv=delay, + noise=NegativeBinomialNoise(DeterministicVariable("conc", 10.0)), + ) + builder.add_observation(obs) + + with pytest.raises(TypeError, match="Error constructing"): + builder.build() + class TestMultiSignalModelObservationValidation: """Test observation process validation in MultiSignalModel.""" @@ -316,7 +472,14 @@ def validate(self): pass def lookback_days(self): - """Return lookback.""" + """ + Return lookback. + + Returns + ------- + int + The lookback value of 1. + """ return 1 def infection_resolution(self): @@ -324,7 +487,14 @@ def infection_resolution(self): raise NotImplementedError("Not implemented") def _predicted_obs(self, infections): - """Predicted obs stub.""" + """ + Predicted obs stub. + + Returns + ------- + ArrayLike + The infections array unchanged. + """ return infections builder = ModelBuilder() diff --git a/test/test_observation_counts.py b/test/test_observation_counts.py index 4de76ebf..7658f513 100644 --- a/test/test_observation_counts.py +++ b/test/test_observation_counts.py @@ -492,6 +492,38 @@ def test_negative_binomial_noise_repr(self): assert "concentration_rv" in repr_str +class TestCountsRepr: + """Test Counts and CountsBySubpop __repr__ methods.""" + + def test_counts_repr(self, simple_delay_pmf): + """Test Counts __repr__ method.""" + process = Counts( + name="test", + ascertainment_rate_rv=DeterministicVariable("ihr", 0.01), + delay_distribution_rv=DeterministicPMF("delay", simple_delay_pmf), + noise=NegativeBinomialNoise(DeterministicVariable("conc", 10.0)), + ) + repr_str = repr(process) + assert "Counts" in repr_str + assert "name='test'" in repr_str + assert "ascertainment_rate_rv" in repr_str + assert "delay_distribution_rv" in repr_str + assert "noise" in repr_str + + def test_counts_by_subpop_repr(self, simple_delay_pmf): + """Test CountsBySubpop __repr__ method.""" + process = CountsBySubpop( + name="test_subpop", + ascertainment_rate_rv=DeterministicVariable("ihr", 0.02), + delay_distribution_rv=DeterministicPMF("delay", simple_delay_pmf), + noise=PoissonNoise(), + ) + repr_str = repr(process) + assert "CountsBySubpop" in repr_str + assert "name='test_subpop'" in repr_str + assert "ascertainment_rate_rv" in repr_str + + class TestNoiseValidation: """Test noise model validation methods.""" diff --git a/test/test_observation_measurements.py b/test/test_observation_measurements.py index 1ef1674e..91c6b14a 100644 --- a/test/test_observation_measurements.py +++ b/test/test_observation_measurements.py @@ -100,6 +100,20 @@ def test_infection_resolution_is_subpop(self): assert process.infection_resolution() == "subpop" +class TestVectorizedRV: + """Test VectorizedRV wrapper class.""" + + def test_init_and_sample(self): + """Test VectorizedRV initialization and sampling.""" + rv = DistributionalVariable("test", dist.Normal(0, 1.0)) + vectorized = VectorizedRV(rv, plate_name="test_plate") + + with numpyro.handlers.seed(rng_seed=42): + samples = vectorized.sample(n_groups=5) + + assert samples.shape == (5,) + + class TestHierarchicalNormalNoise: """Test HierarchicalNormalNoise model.""" diff --git a/test/test_partitioned_infections.py b/test/test_partitioned_infections.py index 95cd9cf3..558eb344 100644 --- a/test/test_partitioned_infections.py +++ b/test/test_partitioned_infections.py @@ -114,6 +114,50 @@ def test_allocation_proportions_sum_to_one(self, process): class TestPartitionedInfectionsValidation: """Test validation of inputs.""" + def test_rejects_missing_I0_rv(self, gen_int_rv): + """Test that None I0_rv is rejected.""" + with pytest.raises(ValueError, match="I0_rv is required"): + PartitionedInfections( + gen_int_rv=gen_int_rv, + I0_rv=None, + initial_log_rt_rv=DeterministicVariable("initial_log_rt", 0.0), + rt_temporal=RandomWalk(), + allocation_temporal=RandomWalk(), + ) + + def test_rejects_missing_initial_log_rt_rv(self, gen_int_rv): + """Test that None initial_log_rt_rv is rejected.""" + with pytest.raises(ValueError, match="initial_log_rt_rv is required"): + PartitionedInfections( + gen_int_rv=gen_int_rv, + I0_rv=DeterministicVariable("I0", 0.001), + initial_log_rt_rv=None, + rt_temporal=RandomWalk(), + allocation_temporal=RandomWalk(), + ) + + def test_rejects_missing_rt_temporal(self, gen_int_rv): + """Test that None rt_temporal is rejected.""" + with pytest.raises(ValueError, match="rt_temporal is required"): + PartitionedInfections( + gen_int_rv=gen_int_rv, + I0_rv=DeterministicVariable("I0", 0.001), + initial_log_rt_rv=DeterministicVariable("initial_log_rt", 0.0), + rt_temporal=None, + allocation_temporal=RandomWalk(), + ) + + def test_rejects_missing_allocation_temporal(self, gen_int_rv): + """Test that None allocation_temporal is rejected.""" + with pytest.raises(ValueError, match="allocation_temporal is required"): + PartitionedInfections( + gen_int_rv=gen_int_rv, + I0_rv=DeterministicVariable("I0", 0.001), + initial_log_rt_rv=DeterministicVariable("initial_log_rt", 0.0), + rt_temporal=RandomWalk(), + allocation_temporal=None, + ) + def test_rejects_invalid_I0(self, gen_int_rv): """Test that invalid I0 values are rejected.""" with pytest.raises(ValueError, match="I0 must be positive"): @@ -135,6 +179,40 @@ def test_rejects_fractions_not_summing_to_one(self, process): unobs_fractions=jnp.array([0.40]), # Sum is 0.95 ) + def test_validate_method(self, process): + """Test that validate() method runs without error.""" + process.validate() + + +class TestPartitionedInfectionsSingleSubpop: + """Test partitioned infections with single subpopulation (K=1).""" + + def test_single_subpop_sample(self, gen_int_rv): + """Test sampling with a single observed subpopulation.""" + process = PartitionedInfections( + gen_int_rv=gen_int_rv, + I0_rv=DeterministicVariable("I0", 0.001), + initial_log_rt_rv=DeterministicVariable("initial_log_rt", 0.0), + rt_temporal=RandomWalk(), + allocation_temporal=RandomWalk(), + ) + + with numpyro.handlers.seed(rng_seed=42): + result = process.sample( + n_days_post_init=30, + obs_fractions=jnp.array([1.0]), + unobs_fractions=jnp.array([]), + ) + + inf_juris, inf_all, inf_obs, inf_unobs = result + n_total = process.n_initialization_points + 30 + + # K=1 case: no allocation deviations needed + assert inf_juris.shape == (n_total,) + assert inf_all.shape == (n_total, 1) + assert inf_obs.shape == (n_total, 1) + assert inf_unobs.shape == (n_total, 0) + if __name__ == "__main__": pytest.main([__file__, "-v"]) From 8704bfa41c732bf3a4a6e74fe58b5f71a6ce8961 Mon Sep 17 00:00:00 2001 From: Mitzi Morris Date: Thu, 15 Jan 2026 17:25:08 -0500 Subject: [PATCH 48/81] dataset data loaders have some tests; don't hold to codecov requirements --- codecov.yml | 11 ++ test/test_datasets_nwss_hosp.py | 189 ++++++++++++++++++++++++++++++++ 2 files changed, 200 insertions(+) create mode 100644 codecov.yml create mode 100644 test/test_datasets_nwss_hosp.py diff --git a/codecov.yml b/codecov.yml new file mode 100644 index 00000000..d8bbdc6e --- /dev/null +++ b/codecov.yml @@ -0,0 +1,11 @@ +coverage: + status: + project: + default: + target: auto + patch: + default: + target: auto + +ignore: + - "pyrenew/datasets/**" diff --git a/test/test_datasets_nwss_hosp.py b/test/test_datasets_nwss_hosp.py new file mode 100644 index 00000000..2132d7f6 --- /dev/null +++ b/test/test_datasets_nwss_hosp.py @@ -0,0 +1,189 @@ +""" +Unit tests for wastewater NWSS and hospital admissions dataset loaders. + +Tests cover the load_wastewater_data_for_state and load_hospital_data_for_state +functions with various parameters and edge cases. +""" + +import jax.numpy as jnp +import polars as pl +import pytest + +from pyrenew.datasets import ( + load_hospital_data_for_state, + load_wastewater_data_for_state, +) + + +class TestLoadWastewaterDataForState: + """Test load_wastewater_data_for_state function.""" + + def test_load_default_state_ca(self): + """Test loading wastewater data for default state (CA).""" + data = load_wastewater_data_for_state() + + assert "observed_conc" in data + assert "observed_conc_linear" in data + assert "site_ids" in data + assert "time_indices" in data + assert "wwtp_names" in data + assert "dates" in data + assert "n_sites" in data + assert "n_obs" in data + assert "raw_df" in data + + assert isinstance(data["observed_conc"], jnp.ndarray) + assert isinstance(data["observed_conc_linear"], jnp.ndarray) + assert isinstance(data["site_ids"], jnp.ndarray) + assert isinstance(data["time_indices"], jnp.ndarray) + assert isinstance(data["wwtp_names"], list) + assert isinstance(data["dates"], list) + assert isinstance(data["n_sites"], int) + assert isinstance(data["n_obs"], int) + assert isinstance(data["raw_df"], pl.DataFrame) + + assert data["n_obs"] > 0 + assert data["n_sites"] > 0 + assert len(data["observed_conc"]) == data["n_obs"] + assert len(data["site_ids"]) == data["n_obs"] + + def test_load_state_wa(self): + """Test loading wastewater data for WA state.""" + data = load_wastewater_data_for_state(state_abbr="WA") + + assert data["n_obs"] > 0 + assert data["n_sites"] > 0 + assert len(data["wwtp_names"]) == data["n_sites"] + + def test_load_state_nm(self): + """Test loading wastewater data for NM state (has log10 units).""" + data = load_wastewater_data_for_state(state_abbr="NM") + + assert data["n_obs"] > 0 + # NM has log10 units, so standardize_log_units should convert them + assert jnp.all(data["observed_conc_linear"] > 0) + + def test_invalid_state_raises_error(self): + """Test that invalid state raises ValueError.""" + with pytest.raises(ValueError, match="No wastewater data found"): + load_wastewater_data_for_state(state_abbr="XX") + + def test_substitute_below_lod_true(self): + """Test below-LOD substitution when enabled.""" + data = load_wastewater_data_for_state( + state_abbr="CA", + substitute_below_lod=True, + ) + + # All linear concentrations should be positive + assert jnp.all(data["observed_conc_linear"] > 0) + + def test_substitute_below_lod_false(self): + """Test behavior when below-LOD substitution is disabled.""" + data = load_wastewater_data_for_state( + state_abbr="CA", + substitute_below_lod=False, + ) + + # Should still have valid data + assert data["n_obs"] > 0 + + def test_standardize_log_units_false(self): + """Test behavior when log unit standardization is disabled.""" + data = load_wastewater_data_for_state( + state_abbr="CA", + standardize_log_units=False, + ) + + assert data["n_obs"] > 0 + assert jnp.all(data["observed_conc_linear"] >= 0) + + def test_standardize_log_units_true_with_log10_data(self): + """Test log unit standardization with data that has log10 units.""" + # NM has log10 units in the test data + data = load_wastewater_data_for_state( + state_abbr="NM", + standardize_log_units=True, + ) + + assert data["n_obs"] > 0 + # Linear concentrations should be converted from log10 + assert jnp.all(data["observed_conc_linear"] > 0) + + def test_site_ids_are_valid_indices(self): + """Test that site_ids are valid indices into wwtp_names.""" + data = load_wastewater_data_for_state() + + assert jnp.all(data["site_ids"] >= 0) + assert jnp.all(data["site_ids"] < data["n_sites"]) + + def test_time_indices_are_non_negative(self): + """Test that time indices are non-negative.""" + data = load_wastewater_data_for_state() + + assert jnp.all(data["time_indices"] >= 0) + + def test_observed_conc_is_log_transformed(self): + """Test that observed_conc is log of observed_conc_linear.""" + data = load_wastewater_data_for_state() + + # observed_conc should be approximately log(observed_conc_linear + epsilon) + # There's an epsilon of 1e-8 added before log transform + expected_log = jnp.log(data["observed_conc_linear"] + 1e-8) + assert jnp.allclose(data["observed_conc"], expected_log, rtol=1e-5) + + +class TestLoadHospitalDataForState: + """Test load_hospital_data_for_state function.""" + + def test_load_default_state_ca(self): + """Test loading hospital data for default state (CA).""" + data = load_hospital_data_for_state() + + assert "daily_admits" in data + assert "population" in data + assert "dates" in data + assert "n_days" in data + + assert isinstance(data["daily_admits"], jnp.ndarray) + assert isinstance(data["population"], int) + assert isinstance(data["dates"], list) + assert isinstance(data["n_days"], int) + + assert data["n_days"] > 0 + assert data["population"] > 0 + assert len(data["daily_admits"]) == data["n_days"] + assert len(data["dates"]) == data["n_days"] + + def test_daily_admits_are_non_negative(self): + """Test that daily admissions are non-negative.""" + data = load_hospital_data_for_state() + + assert jnp.all(data["daily_admits"] >= 0) + + def test_population_is_reasonable(self): + """Test that population is a reasonable value.""" + data = load_hospital_data_for_state() + + # California population should be in the tens of millions + assert data["population"] > 1_000_000 + assert data["population"] < 100_000_000 + + def test_dates_are_sorted(self): + """Test that dates are sorted chronologically.""" + data = load_hospital_data_for_state() + + dates = data["dates"] + for i in range(1, len(dates)): + assert dates[i] > dates[i - 1] + + def test_n_days_matches_array_length(self): + """Test that n_days matches the length of daily_admits array.""" + data = load_hospital_data_for_state() + + assert data["n_days"] == len(data["daily_admits"]) + assert data["n_days"] == len(data["dates"]) + + +if __name__ == "__main__": + pytest.main([__file__, "-v"]) From 65d0b952ecb3122013e979ba835df89b9c4af25d Mon Sep 17 00:00:00 2001 From: Mitzi Morris Date: Thu, 15 Jan 2026 17:34:57 -0500 Subject: [PATCH 49/81] cleanup dataset loaders --- codecov.yml | 11 ---- pyrenew/datasets/hospital_admissions.py | 7 --- pyrenew/datasets/wastewater_nwss.py | 50 ++------------- .../datasets/wastewater_nwss_data/README.md | 4 +- test/test_datasets_nwss_hosp.py | 62 ++++--------------- 5 files changed, 19 insertions(+), 115 deletions(-) delete mode 100644 codecov.yml diff --git a/codecov.yml b/codecov.yml deleted file mode 100644 index d8bbdc6e..00000000 --- a/codecov.yml +++ /dev/null @@ -1,11 +0,0 @@ -coverage: - status: - project: - default: - target: auto - patch: - default: - target: auto - -ignore: - - "pyrenew/datasets/**" diff --git a/pyrenew/datasets/hospital_admissions.py b/pyrenew/datasets/hospital_admissions.py index 7a233fa3..046fbcb1 100644 --- a/pyrenew/datasets/hospital_admissions.py +++ b/pyrenew/datasets/hospital_admissions.py @@ -50,13 +50,6 @@ def load_hospital_data_for_state( .sort("date") ) - if len(df) == 0: - # Try filtering by ABBR column if it exists - df = pl.read_csv(source=data_path) - df = df.with_columns(pl.col("date").str.to_date()) - if "ABBR" in df.columns: - df = df.filter(pl.col("ABBR") == state_abbr).sort("date") - if len(df) == 0: raise ValueError(f"No data found for state {state_abbr} in {filename}") diff --git a/pyrenew/datasets/wastewater_nwss.py b/pyrenew/datasets/wastewater_nwss.py index 4b36b3f5..0878f3b7 100644 --- a/pyrenew/datasets/wastewater_nwss.py +++ b/pyrenew/datasets/wastewater_nwss.py @@ -15,8 +15,6 @@ def load_wastewater_data_for_state( state_abbr: str = "CA", filename: str = "fake_nwss.csv", - substitute_below_lod: bool = True, - standardize_log_units: bool = True, ) -> dict: """ Load wastewater data for a specific state. @@ -27,12 +25,6 @@ def load_wastewater_data_for_state( State abbreviation (e.g., "CA"). Default is "CA". filename : str CSV filename. Default is "fake_nwss.csv". - substitute_below_lod : bool - If True, replace concentrations below LOD with 0.5*LOD. - Default is True. - standardize_log_units : bool - If True, convert log10 units to linear scale. - Default is True. Returns ------- @@ -55,7 +47,8 @@ def load_wastewater_data_for_state( License: Public Domain (CC0 1.0 Universal) - U.S. Government work. The data is synthetic and contains deliberately added noise for - public release. + public release. Concentrations are in copies/L and are converted + to copies/mL (divided by 1000). """ data_path = files("pyrenew.datasets.wastewater_nwss_data") / filename df = pl.read_csv( @@ -64,46 +57,15 @@ def load_wastewater_data_for_state( ) df = df.with_columns(pl.col("sample_collect_date").str.to_date()) - # Quality filtering for real NWSS data - if "sample_location" in df.columns: - df = df.filter(pl.col("sample_location") == "wwtp") - if "sample_matrix" in df.columns: - df = df.filter(pl.col("sample_matrix") != "primary sludge") - if "pcr_target" in df.columns: - df = df.filter(pl.col("pcr_target") == "sars-cov-2") - # Filter to requested state df = df.filter(pl.col("wwtp_jurisdiction") == state_abbr) if len(df) == 0: raise ValueError(f"No wastewater data found for state {state_abbr}") - if standardize_log_units: - # Convert log10 units to linear, then standardize to copies/mL - # NWSS data is in copies/L, so divide by 1000 to get copies/mL - df = df.with_columns( - pl.when(pl.col("pcr_target_units").str.contains("(?i)log10")) - .then(10 ** pl.col("pcr_target_avg_conc") / 1000) - .otherwise(pl.col("pcr_target_avg_conc") / 1000) - .alias("conc_linear"), - pl.when(pl.col("pcr_target_units").str.contains("(?i)log10")) - .then(10 ** pl.col("lod_sewage") / 1000) - .otherwise(pl.col("lod_sewage") / 1000) - .alias("lod_linear"), - ) - else: - df = df.with_columns( - (pl.col("pcr_target_avg_conc") / 1000).alias("conc_linear"), - (pl.col("lod_sewage") / 1000).alias("lod_linear"), - ) - - # Substitute below-LOD values with 0.5*LOD - if substitute_below_lod: - df = df.with_columns( - pl.when(pl.col("conc_linear") < pl.col("lod_linear")) - .then(0.5 * pl.col("lod_linear")) - .otherwise(pl.col("conc_linear")) - .alias("conc_linear") - ) + # Convert copies/L to copies/mL + df = df.with_columns( + (pl.col("pcr_target_avg_conc") / 1000).alias("conc_linear"), + ) df = df.sort("sample_collect_date") unique_sites = sorted(df["wwtp_name"].unique().to_list()) diff --git a/pyrenew/datasets/wastewater_nwss_data/README.md b/pyrenew/datasets/wastewater_nwss_data/README.md index 3472031f..1a3a242d 100644 --- a/pyrenew/datasets/wastewater_nwss_data/README.md +++ b/pyrenew/datasets/wastewater_nwss_data/README.md @@ -49,5 +49,5 @@ ca_ww = datasets.load_wastewater_data_for_state("CA") ## Notes -- Some measurements are in linear scale, others in log10 scale (check `pcr_target_units`) -- The loader function handles unit standardization and below-LOD substitution +- Concentrations are in copies/L and are converted to copies/mL by the loader +- The synthetic data uses consistent units (copies/L wastewater) diff --git a/test/test_datasets_nwss_hosp.py b/test/test_datasets_nwss_hosp.py index 2132d7f6..682b5c60 100644 --- a/test/test_datasets_nwss_hosp.py +++ b/test/test_datasets_nwss_hosp.py @@ -55,60 +55,10 @@ def test_load_state_wa(self): assert data["n_sites"] > 0 assert len(data["wwtp_names"]) == data["n_sites"] - def test_load_state_nm(self): - """Test loading wastewater data for NM state (has log10 units).""" - data = load_wastewater_data_for_state(state_abbr="NM") - - assert data["n_obs"] > 0 - # NM has log10 units, so standardize_log_units should convert them - assert jnp.all(data["observed_conc_linear"] > 0) - def test_invalid_state_raises_error(self): """Test that invalid state raises ValueError.""" with pytest.raises(ValueError, match="No wastewater data found"): - load_wastewater_data_for_state(state_abbr="XX") - - def test_substitute_below_lod_true(self): - """Test below-LOD substitution when enabled.""" - data = load_wastewater_data_for_state( - state_abbr="CA", - substitute_below_lod=True, - ) - - # All linear concentrations should be positive - assert jnp.all(data["observed_conc_linear"] > 0) - - def test_substitute_below_lod_false(self): - """Test behavior when below-LOD substitution is disabled.""" - data = load_wastewater_data_for_state( - state_abbr="CA", - substitute_below_lod=False, - ) - - # Should still have valid data - assert data["n_obs"] > 0 - - def test_standardize_log_units_false(self): - """Test behavior when log unit standardization is disabled.""" - data = load_wastewater_data_for_state( - state_abbr="CA", - standardize_log_units=False, - ) - - assert data["n_obs"] > 0 - assert jnp.all(data["observed_conc_linear"] >= 0) - - def test_standardize_log_units_true_with_log10_data(self): - """Test log unit standardization with data that has log10 units.""" - # NM has log10 units in the test data - data = load_wastewater_data_for_state( - state_abbr="NM", - standardize_log_units=True, - ) - - assert data["n_obs"] > 0 - # Linear concentrations should be converted from log10 - assert jnp.all(data["observed_conc_linear"] > 0) + load_wastewater_data_for_state(state_abbr="INVALID") def test_site_ids_are_valid_indices(self): """Test that site_ids are valid indices into wwtp_names.""" @@ -132,6 +82,11 @@ def test_observed_conc_is_log_transformed(self): expected_log = jnp.log(data["observed_conc_linear"] + 1e-8) assert jnp.allclose(data["observed_conc"], expected_log, rtol=1e-5) + def test_concentrations_are_positive(self): + """Test that linear concentrations are positive.""" + data = load_wastewater_data_for_state() + assert jnp.all(data["observed_conc_linear"] > 0) + class TestLoadHospitalDataForState: """Test load_hospital_data_for_state function.""" @@ -184,6 +139,11 @@ def test_n_days_matches_array_length(self): assert data["n_days"] == len(data["daily_admits"]) assert data["n_days"] == len(data["dates"]) + def test_invalid_state_raises_error(self): + """Test that invalid state raises ValueError.""" + with pytest.raises(ValueError, match="No data found for state"): + load_hospital_data_for_state(state_abbr="XX") + if __name__ == "__main__": pytest.main([__file__, "-v"]) From e497d6d61ffd372a342cf5f15024536962f75715 Mon Sep 17 00:00:00 2001 From: Mitzi Morris Date: Fri, 16 Jan 2026 11:14:51 -0500 Subject: [PATCH 50/81] collapse code blocks in tutorials --- docs/tutorials/building_multisignal_models.qmd | 5 ----- docs/tutorials/latent_hierarchical_infections.qmd | 6 ------ docs/tutorials/observation_processes_counts.qmd | 5 ----- docs/tutorials/observation_processes_measurements.qmd | 5 ----- mkdocs.yml | 1 + 5 files changed, 1 insertion(+), 21 deletions(-) diff --git a/docs/tutorials/building_multisignal_models.qmd b/docs/tutorials/building_multisignal_models.qmd index 28ba95bc..b89a529b 100644 --- a/docs/tutorials/building_multisignal_models.qmd +++ b/docs/tutorials/building_multisignal_models.qmd @@ -4,12 +4,7 @@ format: gfm: fig-width: 16 fig-height: 10 - html: - toc: true - embed-resources: true - self-contained-math: true code-fold: true - code-tools: true engine: jupyter jupyter: jupytext: diff --git a/docs/tutorials/latent_hierarchical_infections.qmd b/docs/tutorials/latent_hierarchical_infections.qmd index bb00389f..c25c55fd 100644 --- a/docs/tutorials/latent_hierarchical_infections.qmd +++ b/docs/tutorials/latent_hierarchical_infections.qmd @@ -4,13 +4,7 @@ format: gfm: fig-width: 16 fig-height: 10 - html: - toc: true - toc-depth: 3 - embed-resources: true - self-contained-math: true code-fold: true - code-tools: true engine: jupyter jupyter: jupytext: diff --git a/docs/tutorials/observation_processes_counts.qmd b/docs/tutorials/observation_processes_counts.qmd index 8fa90e4b..99ca09e0 100644 --- a/docs/tutorials/observation_processes_counts.qmd +++ b/docs/tutorials/observation_processes_counts.qmd @@ -4,12 +4,7 @@ format: gfm: fig-width: 16 fig-height: 10 - html: - toc: true - embed-resources: true - self-contained-math: true code-fold: true - code-tools: true engine: jupyter --- diff --git a/docs/tutorials/observation_processes_measurements.qmd b/docs/tutorials/observation_processes_measurements.qmd index 1ff75dfe..510cd528 100644 --- a/docs/tutorials/observation_processes_measurements.qmd +++ b/docs/tutorials/observation_processes_measurements.qmd @@ -4,12 +4,7 @@ format: gfm: fig-width: 16 fig-height: 10 - html: - toc: true - embed-resources: true - self-contained-math: true code-fold: true - code-tools: true engine: jupyter --- diff --git a/mkdocs.yml b/mkdocs.yml index 1c2745eb..e00bf26e 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -50,6 +50,7 @@ markdown_extensions: - pymdownx.snippets - pymdownx.arithmatex: generic: true + - pymdownx.details - pymdownx.superfences: custom_fences: - name: mermaid From 7a5f1438d0c9e2402c277054025439af345ba8c5 Mon Sep 17 00:00:00 2001 From: Mitzi Morris Date: Tue, 27 Jan 2026 12:23:48 -0500 Subject: [PATCH 51/81] code cleanup --- pyrenew/observation/count_observations.py | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/pyrenew/observation/count_observations.py b/pyrenew/observation/count_observations.py index cbecb58a..2c8bcf81 100644 --- a/pyrenew/observation/count_observations.py +++ b/pyrenew/observation/count_observations.py @@ -206,13 +206,12 @@ def sample( """ predicted_counts = self._predicted_obs(infections) self._deterministic("predicted", predicted_counts) - predicted_counts_safe = jnp.nan_to_num(predicted_counts, nan=0.0) # Only use sparse indexing when conditioning on observations if times is not None and obs is not None: - predicted_obs = predicted_counts_safe[times] + predicted_obs = predicted_counts[times] else: - predicted_obs = predicted_counts_safe + predicted_obs = predicted_counts observed = self.noise.sample( name=self._sample_site_name("obs"), @@ -301,13 +300,10 @@ def sample( `predicted` (predicted counts before noise, shape: n_days x n_subpops). """ # Compute predicted counts for all subpops - predicted_counts_all = self._predicted_obs(infections) - - self._deterministic("predicted", predicted_counts_all) + predicted_counts = self._predicted_obs(infections) - # Replace NaN padding with 0 for distribution creation - predicted_counts_safe = jnp.nan_to_num(predicted_counts_all, nan=0.0) - predicted_obs = predicted_counts_safe[times, subpop_indices] + self._deterministic("predicted", predicted_counts) + predicted_obs = predicted_counts[times, subpop_indices] observed = self.noise.sample( name=self._sample_site_name("obs"), @@ -315,4 +311,4 @@ def sample( obs=obs, ) - return ObservationSample(observed=observed, predicted=predicted_counts_all) + return ObservationSample(observed=observed, predicted=predicted_counts) From aaee3dbab30adaabbb230c5c8c8cb8c131bd64dc Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 29 Jan 2026 22:53:08 +0000 Subject: [PATCH 52/81] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- docs/tutorials/observation_processes_counts.qmd | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/tutorials/observation_processes_counts.qmd b/docs/tutorials/observation_processes_counts.qmd index 889e8fe1..24326ed5 100644 --- a/docs/tutorials/observation_processes_counts.qmd +++ b/docs/tutorials/observation_processes_counts.qmd @@ -341,6 +341,7 @@ The predicted admissions mirror the delay distribution, shifted by the infection The negative binomial distribution adds stochastic variation. Sampling multiple times from the same infections shows the range of possible observations: ```{python} + # | label: sample-realizations # Sample 50 realizations of hospital admissions from the same infection spike n_samples = 50 @@ -505,6 +506,7 @@ We compare three concentration values spanning two orders of magnitude: - **φ = 100**: nearly Poisson (minimal noise) ```{python} + # | label: concentration-comparisons # Use constant infections peak_value = 2000 @@ -602,6 +604,7 @@ We can visualize the Poisson noise model using the same constant infection scena To see the reduction in noise, it is necessary to keep the y-axis on the same scale as in the previous plot. ```{python} + # | label: poisson-realizations # Sample multiple realizations with Poisson noise n_replicates_poisson = 10 From adf9fda951c59a95683d4e410c0a63db5d1701b8 Mon Sep 17 00:00:00 2001 From: Mitzi Morris Date: Thu, 29 Jan 2026 17:55:01 -0500 Subject: [PATCH 53/81] simplify this PR --- .../observation_processes_counts.qmd | 21 -- pyrenew/latent/partitioned_infections.py | 293 ------------------ test/test_partitioned_infections.py | 218 ------------- 3 files changed, 532 deletions(-) delete mode 100644 pyrenew/latent/partitioned_infections.py delete mode 100644 test/test_partitioned_infections.py diff --git a/docs/tutorials/observation_processes_counts.qmd b/docs/tutorials/observation_processes_counts.qmd index 889e8fe1..c39b9cfe 100644 --- a/docs/tutorials/observation_processes_counts.qmd +++ b/docs/tutorials/observation_processes_counts.qmd @@ -4,16 +4,7 @@ format: gfm: fig-width: 16 fig-height: 10 -<<<<<<< HEAD code-fold: true -======= - html: - toc: true - embed-resources: true - self-contained-math: true - code-fold: true - code-tools: true ->>>>>>> 247baba3a881150741bdc8f46087eed2d6e677e2 engine: jupyter --- @@ -350,11 +341,7 @@ for seed in range(n_samples): with numpyro.handlers.seed(rng_seed=seed): hosp_sample = hosp_process.sample(infections=infections, obs=None) -<<<<<<< HEAD - for i, val in enumerate(hosp_sample.observed[day_one:]): -======= for i, val in enumerate(hosp_sample.observed): ->>>>>>> 247baba3a881150741bdc8f46087eed2d6e677e2 samples_list.append( { "day": i, @@ -533,11 +520,7 @@ for conc_val in concentration_values: ) # Use relative days -<<<<<<< HEAD - for i, admit in enumerate(hosp_temp.observed[day_one:]): -======= for i, admit in enumerate(hosp_temp.observed): ->>>>>>> 247baba3a881150741bdc8f46087eed2d6e677e2 conc_results.append( { "day": i, @@ -614,11 +597,7 @@ for seed in range(n_replicates_poisson): obs=None, ) -<<<<<<< HEAD - for i, admit in enumerate(poisson_temp.observed[day_one:]): -======= for i, admit in enumerate(poisson_temp.observed): ->>>>>>> 247baba3a881150741bdc8f46087eed2d6e677e2 poisson_results.append( { "day": i, diff --git a/pyrenew/latent/partitioned_infections.py b/pyrenew/latent/partitioned_infections.py deleted file mode 100644 index d49612c9..00000000 --- a/pyrenew/latent/partitioned_infections.py +++ /dev/null @@ -1,293 +0,0 @@ -""" -Partitioned latent infection process with single renewal model and allocation. -""" - -from __future__ import annotations - -import jax.numpy as jnp -import numpyro -from jax.typing import ArrayLike - -from pyrenew.deterministic import DeterministicVariable -from pyrenew.latent.base import ( - BaseLatentInfectionProcess, - LatentSample, -) -from pyrenew.latent.infection_functions import compute_infections_from_rt -from pyrenew.latent.infection_initialization_method import ( - InitializeInfectionsExponentialGrowth, -) -from pyrenew.latent.infection_initialization_process import ( - InfectionInitializationProcess, -) -from pyrenew.latent.temporal_processes import TemporalProcess -from pyrenew.math import r_approx_from_R -from pyrenew.metaclass import RandomVariable - - -class PartitionedInfections(BaseLatentInfectionProcess): - """ - Single aggregate renewal model with spatial allocation. - - Computes one renewal equation for total infections, then allocates to - subpopulations via time-varying distribution. More efficient when spatial - variation is in infection distribution rather than transmission dynamics. - - Mathematical form: - - Aggregate Rt: log[R_aggregate(t)] ~ TemporalProcess - - Single renewal: I_total(t) = R_aggregate(t) * sum_tau I_total(t-tau) * g(tau) - - Baseline allocation: pi_k^base = population fractions - - Allocation deviations: delta_k(t) ~ TemporalProcess, delta_1(t) = 0 (reference) - - Softmax allocation: pi_k(t) = (pi_k^base * exp(delta_k)) / sum_j(...) - - Subpop infections: I_k(t) = pi_k(t) * I_total(t) - - The constructor specifies model structure (priors, temporal processes). - Population structure (fractions, K, K_obs) is provided at sample time, - allowing a single model to be fit to multiple jurisdictions. - - Parameters - ---------- - gen_int_rv : RandomVariable - Generation interval PMF - I0_rv : RandomVariable - Initial infection prevalence (proportion of population) at first - observation time. Must return value in the interval (0, 1]. - Returns scalar. Full I0 vector generated via exponential backprojection during sampling. - rt_temporal : TemporalProcess - Temporal process for aggregate-level Rt - allocation_temporal : TemporalProcess - Temporal process for allocation deviations (K-1 processes, reference fixed to 0) - initial_log_rt_rv : RandomVariable - Initial value for log(Rt) at time 0. Can be estimated from data - or given a prior distribution. - n_initialization_points : int, optional - Number of initialization days before day 0. If not specified, defaults to - max(21, 2*len(gen_int)-1). When using ModelBuilder, this is computed - automatically from all observation processes. - - Notes - ----- - Reference category (subpopulation 0) has delta_0(t) = 0 for identifiability. - Choose the largest or most representative subpopulation as index 0. - """ - - def __init__( - self, - *, - gen_int_rv: RandomVariable, - I0_rv: RandomVariable, - rt_temporal: TemporalProcess, - allocation_temporal: TemporalProcess, - initial_log_rt_rv: RandomVariable, - n_initialization_points: int = None, - ) -> None: - """ - Initialize partitioned infections process. - - Parameters - ---------- - gen_int_rv : RandomVariable - Generation interval PMF - I0_rv : RandomVariable - Initial infection prevalence (proportion of population) - rt_temporal : TemporalProcess - Temporal process for aggregate-level Rt - allocation_temporal : TemporalProcess - Temporal process for allocation deviations - initial_log_rt_rv : RandomVariable - Initial value for log(Rt) at time 0 - n_initialization_points : int, optional - Number of initialization days before day 0 - - Raises - ------ - ValueError - If required parameters are missing or invalid - """ - super().__init__( - gen_int_rv=gen_int_rv, - n_initialization_points=n_initialization_points, - ) - - if I0_rv is None: - raise ValueError("I0_rv is required") - self.I0_rv = I0_rv - - # Validate I0 at construction time if it's deterministic - if isinstance(I0_rv, DeterministicVariable): - self._validate_I0(I0_rv.value) - - if initial_log_rt_rv is None: - raise ValueError("initial_log_rt_rv is required") - self.initial_log_rt_rv = initial_log_rt_rv - - if rt_temporal is None: - raise ValueError("rt_temporal is required") - self.rt_temporal = rt_temporal - - if allocation_temporal is None: - raise ValueError("allocation_temporal is required") - self.allocation_temporal = allocation_temporal - - def validate(self) -> None: - """ - Validate partitioned infections parameters. - - Checks that temporal processes and I0_rv can be sampled. - - Raises - ------ - ValueError - If any parameters fail validation - """ - gen_int_pmf = self.gen_int_rv() - if gen_int_pmf.size == 0: # pragma: no cover - raise ValueError("gen_int_rv must return non-empty array") - - pmf_sum = jnp.sum(gen_int_pmf) - if not jnp.isclose(pmf_sum, 1.0, atol=1e-6): # pragma: no cover - raise ValueError(f"gen_int_rv must sum to 1.0, got {float(pmf_sum):.6f}") - - if jnp.any(gen_int_pmf < 0): # pragma: no cover - raise ValueError("gen_int_rv must have non-negative values") - - def sample( - self, - n_days_post_init: int, - *, - obs_fractions: ArrayLike = None, - unobs_fractions: ArrayLike = None, - **kwargs, - ) -> LatentSample: - """ - Sample partitioned infections with single renewal model and allocation. - - Generates aggregate-level Rt, runs one renewal equation for total - infections, then allocates to subpopulations via time-varying softmax. - - Parameters - ---------- - n_days_post_init : int - Number of days to simulate after initialization period - obs_fractions : ArrayLike - Population fractions for observed subpopulations. - unobs_fractions : ArrayLike - Population fractions for unobserved subpopulations. - **kwargs - Additional arguments (unused, for compatibility) - - Returns - ------- - LatentSample - Named tuple with fields: - - aggregate: shape (n_total_days,) - total infections - - all_subpops: shape (n_total_days, K) - allocated to all subpops - - observed: shape (n_total_days, K_obs) - - unobserved: shape (n_total_days, K_unobs) - """ - # Parse and validate population structure - pop = self._parse_and_validate_fractions( - obs_fractions=obs_fractions, - unobs_fractions=unobs_fractions, - ) - - n_total_days = self.n_initialization_points + n_days_post_init - - initial_log_rt = self.initial_log_rt_rv() - - log_rt_aggregate = self.rt_temporal.sample( - n_timepoints=n_total_days, - initial_value=initial_log_rt, - innovation_sd=1.0, - name_prefix="log_rt_aggregate", - ) - rt_aggregate = jnp.exp(log_rt_aggregate) - - # Reference category (subpop 0) fixed to delta_0(t) = 0 for identifiability - if pop.K > 1: - deviations_nonref = self.allocation_temporal.sample( - n_timepoints=n_total_days, - n_processes=pop.K - 1, - initial_value=jnp.zeros(pop.K - 1), - innovation_sd=1.0, - name_prefix="allocation_deviations", - ) - deviations = jnp.concatenate( - [jnp.zeros((n_total_days, 1)), deviations_nonref], axis=1 - ) - else: - deviations = jnp.zeros((n_total_days, 1)) - - baseline_allocation = jnp.concatenate([pop.obs_fractions, pop.unobs_fractions]) - - # Softmax: pi_k(t) = (baseline_k * exp(delta_k(t))) / sum_j(baseline_j * exp(delta_j(t))) - unnormalized = baseline_allocation[jnp.newaxis, :] * jnp.exp(deviations) - allocation_proportions = unnormalized / jnp.sum( - unnormalized, axis=1, keepdims=True - ) - - gen_int = self.gen_int_rv() - I0_total_raw = self.I0_rv() - I0_total = jnp.asarray(I0_total_raw) - - # Validate I0 at runtime (for stochastic RVs) - self._validate_I0(I0_total) - - initial_r = r_approx_from_R(rt_aggregate[0], gen_int, n_newton_steps=4) - - i0_rv = DeterministicVariable("i0_prevalence", I0_total) - r_rv = DeterministicVariable("initial_r", initial_r) - - init_proc = InfectionInitializationProcess( - "I0_initialization", - i0_rv, - InitializeInfectionsExponentialGrowth( - self.n_initialization_points, - r_rv, - t_pre_init=0, - ), - ) - - I0_total = init_proc() - - gen_int_reversed = jnp.flip(gen_int) - recent_I0_total = I0_total[-gen_int.size :] - - post_init_infections_total = compute_infections_from_rt( - I0=recent_I0_total, - Rt=rt_aggregate[self.n_initialization_points :], - reversed_generation_interval_pmf=gen_int_reversed, - ) - - infections_total = jnp.concatenate([I0_total, post_init_infections_total]) - - infections_all = allocation_proportions * infections_total[:, jnp.newaxis] - - infections_aggregate = infections_total - - infections_observed, infections_unobserved = self._split_subpopulations( - infections_all, pop - ) - - self._validate_output_shapes( - infections_aggregate, - infections_all, - infections_observed, - infections_unobserved, - n_total_days, - pop, - ) - - # Record key quantities for diagnostics and posterior analysis - numpyro.deterministic("log_rt_aggregate", log_rt_aggregate) - numpyro.deterministic("rt_aggregate", rt_aggregate) - numpyro.deterministic("allocation_deviations", deviations) - numpyro.deterministic("allocation_proportions", allocation_proportions) - numpyro.deterministic("infections_aggregate", infections_aggregate) - - return LatentSample( - aggregate=infections_aggregate, - all_subpops=infections_all, - observed=infections_observed, - unobserved=infections_unobserved, - ) diff --git a/test/test_partitioned_infections.py b/test/test_partitioned_infections.py deleted file mode 100644 index 558eb344..00000000 --- a/test/test_partitioned_infections.py +++ /dev/null @@ -1,218 +0,0 @@ -""" -Unit tests for PartitionedInfections. -""" - -import jax.numpy as jnp -import numpyro -import pytest - -from pyrenew.deterministic import DeterministicPMF, DeterministicVariable -from pyrenew.latent import PartitionedInfections, RandomWalk - - -@pytest.fixture -def gen_int_rv(): - """ - Create a generation interval random variable. - - Returns - ------- - DeterministicPMF - Generation interval PMF. - """ - return DeterministicPMF("gen_int", jnp.array([0.2, 0.5, 0.3])) - - -@pytest.fixture -def process(gen_int_rv): - """ - Create a PartitionedInfections with the new keyword-only API. - - Returns - ------- - PartitionedInfections - Configured infection process. - """ - return PartitionedInfections( - gen_int_rv=gen_int_rv, - I0_rv=DeterministicVariable("I0", 0.001), - initial_log_rt_rv=DeterministicVariable("initial_log_rt", 0.0), - rt_temporal=RandomWalk(), - allocation_temporal=RandomWalk(), - ) - - -class TestPartitionedInfectionsSample: - """Test sample method with population structure at sample time.""" - - def test_sample_returns_correct_shapes(self, process): - """Test that sample returns correct output shapes.""" - with numpyro.handlers.seed(rng_seed=42): - result = process.sample( - n_days_post_init=30, - obs_fractions=jnp.array([0.3, 0.25]), - unobs_fractions=jnp.array([0.45]), - ) - - inf_juris, inf_all, inf_obs, inf_unobs = result - n_total = process.n_initialization_points + 30 - - assert inf_juris.shape == (n_total,) - assert inf_all.shape == (n_total, 3) - assert inf_obs.shape == (n_total, 2) - assert inf_unobs.shape == (n_total, 1) - - def test_same_model_different_jurisdictions(self, process): - """Test that one model can fit different population structures.""" - # Jurisdiction A: 2 observed, 1 unobserved - with numpyro.handlers.seed(rng_seed=42): - _, inf_all_a, _, _ = process.sample( - n_days_post_init=30, - obs_fractions=jnp.array([0.3, 0.25]), - unobs_fractions=jnp.array([0.45]), - ) - - # Jurisdiction B: 4 observed, 1 unobserved - with numpyro.handlers.seed(rng_seed=42): - _, inf_all_b, _, _ = process.sample( - n_days_post_init=30, - obs_fractions=jnp.array([0.15, 0.20, 0.25, 0.10]), - unobs_fractions=jnp.array([0.30]), - ) - - assert inf_all_a.shape[1] == 3 # K=3 - assert inf_all_b.shape[1] == 5 # K=5 - - def test_allocated_infections_sum_to_total(self, process): - """Test that allocated infections sum to jurisdiction total.""" - with numpyro.handlers.seed(rng_seed=42): - inf_juris, inf_all, _, _ = process.sample( - n_days_post_init=30, - obs_fractions=jnp.array([0.3, 0.25]), - unobs_fractions=jnp.array([0.45]), - ) - - infections_sum = jnp.sum(inf_all, axis=1) - assert jnp.allclose(inf_juris, infections_sum, atol=1e-6) - - def test_allocation_proportions_sum_to_one(self, process): - """Test that allocation proportions sum to 1 at each timepoint.""" - with numpyro.handlers.seed(rng_seed=42): - with numpyro.handlers.trace() as trace: - process.sample( - n_days_post_init=30, - obs_fractions=jnp.array([0.3, 0.25]), - unobs_fractions=jnp.array([0.45]), - ) - - proportions = trace["allocation_proportions"]["value"] - proportion_sums = jnp.sum(proportions, axis=1) - - assert jnp.allclose(proportion_sums, 1.0, atol=1e-6) - - -class TestPartitionedInfectionsValidation: - """Test validation of inputs.""" - - def test_rejects_missing_I0_rv(self, gen_int_rv): - """Test that None I0_rv is rejected.""" - with pytest.raises(ValueError, match="I0_rv is required"): - PartitionedInfections( - gen_int_rv=gen_int_rv, - I0_rv=None, - initial_log_rt_rv=DeterministicVariable("initial_log_rt", 0.0), - rt_temporal=RandomWalk(), - allocation_temporal=RandomWalk(), - ) - - def test_rejects_missing_initial_log_rt_rv(self, gen_int_rv): - """Test that None initial_log_rt_rv is rejected.""" - with pytest.raises(ValueError, match="initial_log_rt_rv is required"): - PartitionedInfections( - gen_int_rv=gen_int_rv, - I0_rv=DeterministicVariable("I0", 0.001), - initial_log_rt_rv=None, - rt_temporal=RandomWalk(), - allocation_temporal=RandomWalk(), - ) - - def test_rejects_missing_rt_temporal(self, gen_int_rv): - """Test that None rt_temporal is rejected.""" - with pytest.raises(ValueError, match="rt_temporal is required"): - PartitionedInfections( - gen_int_rv=gen_int_rv, - I0_rv=DeterministicVariable("I0", 0.001), - initial_log_rt_rv=DeterministicVariable("initial_log_rt", 0.0), - rt_temporal=None, - allocation_temporal=RandomWalk(), - ) - - def test_rejects_missing_allocation_temporal(self, gen_int_rv): - """Test that None allocation_temporal is rejected.""" - with pytest.raises(ValueError, match="allocation_temporal is required"): - PartitionedInfections( - gen_int_rv=gen_int_rv, - I0_rv=DeterministicVariable("I0", 0.001), - initial_log_rt_rv=DeterministicVariable("initial_log_rt", 0.0), - rt_temporal=RandomWalk(), - allocation_temporal=None, - ) - - def test_rejects_invalid_I0(self, gen_int_rv): - """Test that invalid I0 values are rejected.""" - with pytest.raises(ValueError, match="I0 must be positive"): - PartitionedInfections( - gen_int_rv=gen_int_rv, - I0_rv=DeterministicVariable("I0", -0.1), - initial_log_rt_rv=DeterministicVariable("initial_log_rt", 0.0), - rt_temporal=RandomWalk(), - allocation_temporal=RandomWalk(), - ) - - def test_rejects_fractions_not_summing_to_one(self, process): - """Test that fractions not summing to 1 raises error at sample time.""" - with pytest.raises(ValueError, match="must sum to 1.0"): - with numpyro.handlers.seed(rng_seed=42): - process.sample( - n_days_post_init=30, - obs_fractions=jnp.array([0.3, 0.25]), - unobs_fractions=jnp.array([0.40]), # Sum is 0.95 - ) - - def test_validate_method(self, process): - """Test that validate() method runs without error.""" - process.validate() - - -class TestPartitionedInfectionsSingleSubpop: - """Test partitioned infections with single subpopulation (K=1).""" - - def test_single_subpop_sample(self, gen_int_rv): - """Test sampling with a single observed subpopulation.""" - process = PartitionedInfections( - gen_int_rv=gen_int_rv, - I0_rv=DeterministicVariable("I0", 0.001), - initial_log_rt_rv=DeterministicVariable("initial_log_rt", 0.0), - rt_temporal=RandomWalk(), - allocation_temporal=RandomWalk(), - ) - - with numpyro.handlers.seed(rng_seed=42): - result = process.sample( - n_days_post_init=30, - obs_fractions=jnp.array([1.0]), - unobs_fractions=jnp.array([]), - ) - - inf_juris, inf_all, inf_obs, inf_unobs = result - n_total = process.n_initialization_points + 30 - - # K=1 case: no allocation deviations needed - assert inf_juris.shape == (n_total,) - assert inf_all.shape == (n_total, 1) - assert inf_obs.shape == (n_total, 1) - assert inf_unobs.shape == (n_total, 0) - - -if __name__ == "__main__": - pytest.main([__file__, "-v"]) From 0b1a57f1465b71e2fe2cd892524a2c8d8f17ebc3 Mon Sep 17 00:00:00 2001 From: Mitzi Morris Date: Thu, 29 Jan 2026 17:55:36 -0500 Subject: [PATCH 54/81] lint fix --- docs/tutorials/observation_processes_counts.qmd | 3 --- pyrenew/latent/__init__.py | 3 ++- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/docs/tutorials/observation_processes_counts.qmd b/docs/tutorials/observation_processes_counts.qmd index ea44241d..c39b9cfe 100644 --- a/docs/tutorials/observation_processes_counts.qmd +++ b/docs/tutorials/observation_processes_counts.qmd @@ -332,7 +332,6 @@ The predicted admissions mirror the delay distribution, shifted by the infection The negative binomial distribution adds stochastic variation. Sampling multiple times from the same infections shows the range of possible observations: ```{python} - # | label: sample-realizations # Sample 50 realizations of hospital admissions from the same infection spike n_samples = 50 @@ -493,7 +492,6 @@ We compare three concentration values spanning two orders of magnitude: - **φ = 100**: nearly Poisson (minimal noise) ```{python} - # | label: concentration-comparisons # Use constant infections peak_value = 2000 @@ -587,7 +585,6 @@ We can visualize the Poisson noise model using the same constant infection scena To see the reduction in noise, it is necessary to keep the y-axis on the same scale as in the previous plot. ```{python} - # | label: poisson-realizations # Sample multiple realizations with Poisson noise n_replicates_poisson = 10 diff --git a/pyrenew/latent/__init__.py b/pyrenew/latent/__init__.py index 573bbe8d..c3609b39 100644 --- a/pyrenew/latent/__init__.py +++ b/pyrenew/latent/__init__.py @@ -1,5 +1,7 @@ # numpydoc ignore=GL08 +from pyrenew.latent.partitioned_infections import PartitionedInfections + from pyrenew.latent.base import ( BaseLatentInfectionProcess, LatentSample, @@ -28,7 +30,6 @@ ) from pyrenew.latent.infections import Infections from pyrenew.latent.infectionswithfeedback import InfectionsWithFeedback -from pyrenew.latent.partitioned_infections import PartitionedInfections from pyrenew.latent.temporal_processes import ( AR1, DifferencedAR1, From c704b1e3da82fa9b91a0bd10173831b34c5a22eb Mon Sep 17 00:00:00 2001 From: Mitzi Morris Date: Thu, 29 Jan 2026 18:03:31 -0500 Subject: [PATCH 55/81] simplify this PR - ignore ParitionedInfections --- pyrenew/latent/__init__.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/pyrenew/latent/__init__.py b/pyrenew/latent/__init__.py index c3609b39..433d13e9 100644 --- a/pyrenew/latent/__init__.py +++ b/pyrenew/latent/__init__.py @@ -1,7 +1,5 @@ # numpydoc ignore=GL08 -from pyrenew.latent.partitioned_infections import PartitionedInfections - from pyrenew.latent.base import ( BaseLatentInfectionProcess, LatentSample, @@ -55,7 +53,6 @@ "PopulationStructure", # Hierarchical infection processes "HierarchicalInfections", - "PartitionedInfections", # Hierarchical priors "HierarchicalNormalPrior", "GammaGroupSdPrior", From 0c9ad50ec3c0b81c92085d59e2570ab845626b15 Mon Sep 17 00:00:00 2001 From: Mitzi Morris Date: Thu, 29 Jan 2026 22:33:34 -0500 Subject: [PATCH 56/81] code cleanup --- pyrenew/latent/base.py | 28 +++++------- pyrenew/latent/hierarchical_infections.py | 53 ++++++++--------------- pyrenew/latent/temporal_processes.py | 15 ++----- pyrenew/model/multisignal_model.py | 2 - 4 files changed, 32 insertions(+), 66 deletions(-) diff --git a/pyrenew/latent/base.py b/pyrenew/latent/base.py index d65f129d..a05872d7 100644 --- a/pyrenew/latent/base.py +++ b/pyrenew/latent/base.py @@ -299,30 +299,24 @@ def _validate_I0(I0: ArrayLike) -> None: Validation is skipped during JAX tracing (e.g., when using Predictive) since traced values cannot be used in Python boolean operations. """ - import jax - I0 = jnp.asarray(I0) # Skip validation during JAX tracing (e.g., Predictive sampling) # Traced arrays cannot be used in Python boolean operations try: - # This will fail if I0 is a tracer - is_invalid_low = bool(jnp.any(I0 <= 0)) - is_invalid_high = bool(jnp.any(I0 > 1)) + if jnp.any(I0 <= 0): + raise ValueError( + f"I0 must be positive (got min={float(jnp.min(I0)):.6f}). " + "I0 represents infection prevalence as a proportion of the population." + ) + if jnp.any(I0 > 1): + raise ValueError( + f"I0 must be <= 1 (got max={float(jnp.max(I0)):.6f}). " + "I0 represents infection prevalence as a proportion of the population." + ) except jax.errors.TracerBoolConversionError: # During tracing, skip validation - return - - if is_invalid_low: - raise ValueError( - f"I0 must be positive (got min={float(jnp.min(I0)):.6f}). " - "I0 represents infection prevalence as a proportion of the population." - ) - if is_invalid_high: - raise ValueError( - f"I0 must be <= 1 (got max={float(jnp.max(I0)):.6f}). " - "I0 represents infection prevalence as a proportion of the population." - ) + pass def get_required_lookback(self) -> int: """ diff --git a/pyrenew/latent/hierarchical_infections.py b/pyrenew/latent/hierarchical_infections.py index b3c2d855..53edec09 100644 --- a/pyrenew/latent/hierarchical_infections.py +++ b/pyrenew/latent/hierarchical_infections.py @@ -17,12 +17,6 @@ LatentSample, ) from pyrenew.latent.infection_functions import compute_infections_from_rt -from pyrenew.latent.infection_initialization_method import ( - InitializeInfectionsExponentialGrowth, -) -from pyrenew.latent.infection_initialization_process import ( - InfectionInitializationProcess, -) from pyrenew.latent.temporal_processes import TemporalProcess from pyrenew.math import r_approx_from_R from pyrenew.metaclass import RandomVariable @@ -224,10 +218,7 @@ def sample( gen_int = self.gen_int_rv() - I0_raw = self.I0_rv() - I0 = jnp.asarray(I0_raw) - - # Validate I0 at runtime (for stochastic RVs) + I0 = jnp.asarray(self.I0_rv()) self._validate_I0(I0) if I0.ndim == 0: @@ -239,39 +230,29 @@ def sample( partial(r_approx_from_R, g=gen_int, n_newton_steps=4) )(rt_subpop[0, :]) - I0_all = jnp.zeros((self.n_initialization_points, pop.K)) - - for k in range(pop.K): - i0_rv_k = DeterministicVariable(f"I0_subpop_{k}", I0_subpop[k]) - r_rv_k = DeterministicVariable(f"initial_r_subpop_{k}", initial_r_subpop[k]) - - init_proc_k = InfectionInitializationProcess( - f"I0_initialization_subpop_{k}", - i0_rv_k, - InitializeInfectionsExponentialGrowth( - self.n_initialization_points, - r_rv_k, - t_pre_init=0, - ), - ) - I0_all = I0_all.at[:, k].set(init_proc_k()) + # Vectorized exponential growth initialization for all subpopulations + # Formula: I0_subpop[k] * exp(initial_r_subpop[k] * t) for t in [0, n_init) + time_indices = jnp.arange(self.n_initialization_points) + I0_all = I0_subpop[jnp.newaxis, :] * jnp.exp( + initial_r_subpop[jnp.newaxis, :] * time_indices[:, jnp.newaxis] + ) + numpyro.deterministic("I0_init_all_subpops", I0_all) gen_int_reversed = jnp.flip(gen_int) recent_I0_all = I0_all[-gen_int.size :, :] all_fractions = jnp.concatenate([pop.obs_fractions, pop.unobs_fractions]) - post_init_infections_all = jnp.zeros((n_days_post_init, pop.K)) - - for k in range(pop.K): - post_init_k = compute_infections_from_rt( - I0=recent_I0_all[:, k], - Rt=rt_subpop[self.n_initialization_points :, k], + # Vectorized renewal equation for all subpopulations via vmap + post_init_infections_all = jax.vmap( + lambda I0_col, Rt_col: compute_infections_from_rt( + I0=I0_col, + Rt=Rt_col, reversed_generation_interval_pmf=gen_int_reversed, - ) - post_init_infections_all = post_init_infections_all.at[:, k].set( - post_init_k - ) + ), + in_axes=1, + out_axes=1, + )(recent_I0_all, rt_subpop[self.n_initialization_points :, :]) infections_all = jnp.vstack([I0_all, post_init_infections_all]) diff --git a/pyrenew/latent/temporal_processes.py b/pyrenew/latent/temporal_processes.py index 70d82789..5c4ea484 100644 --- a/pyrenew/latent/temporal_processes.py +++ b/pyrenew/latent/temporal_processes.py @@ -549,22 +549,15 @@ def _sample_vectorized( dist.Normal(0, 1), ) + # Transpose: (n_processes, n_timepoints-1) -> (n_timepoints-1, n_processes) increments = numpyro.deterministic( f"{name_prefix}_increments", - increments_raw * innovation_sd, + (increments_raw * innovation_sd).T, ) - # Transpose: (n_processes, n_timepoints-1) -> (n_timepoints-1, n_processes) - increments = increments.T - cumulative = jnp.cumsum(increments, axis=0) - trajectories = jnp.concatenate( - [ - initial_values[jnp.newaxis, :], - initial_values[jnp.newaxis, :] + cumulative, - ], + return jnp.concatenate( + [initial_values[jnp.newaxis, :], initial_values + cumulative], axis=0, ) - - return trajectories diff --git a/pyrenew/model/multisignal_model.py b/pyrenew/model/multisignal_model.py index c45e4232..6692b06d 100644 --- a/pyrenew/model/multisignal_model.py +++ b/pyrenew/model/multisignal_model.py @@ -138,8 +138,6 @@ def validate_data( If times indices are out of bounds or negative If data shapes are inconsistent """ - from pyrenew.latent.base import BaseLatentInfectionProcess - # Parse population structure to get K_obs for validation pop = BaseLatentInfectionProcess._parse_and_validate_fractions( obs_fractions=obs_fractions, From 00eb0a56970bdc516e905e81b8f54f0e8e5ec2a5 Mon Sep 17 00:00:00 2001 From: Mitzi Morris Date: Fri, 30 Jan 2026 13:48:26 -0500 Subject: [PATCH 57/81] refactoring per issue #662 - option 2 --- .../tutorials/building_multisignal_models.qmd | 78 +++++++------ .../observation_processes_counts.qmd | 14 ++- .../observation_processes_measurements.qmd | 10 +- pyrenew/latent/temporal_processes.py | 69 ++++++++++-- pyrenew/model/multisignal_model.py | 104 +++++++++++++++--- pyrenew/observation/base.py | 8 +- pyrenew/observation/count_observations.py | 100 +++++++++-------- pyrenew/observation/measurements.py | 40 ++++--- pyrenew/observation/noise.py | 63 ++++++++--- test/test_model_builder.py | 28 +++-- test/test_observation_counts.py | 96 +++++----------- test/test_observation_measurements.py | 39 +++++-- 12 files changed, 421 insertions(+), 228 deletions(-) diff --git a/docs/tutorials/building_multisignal_models.qmd b/docs/tutorials/building_multisignal_models.qmd index b89a529b..db2821c5 100644 --- a/docs/tutorials/building_multisignal_models.qmd +++ b/docs/tutorials/building_multisignal_models.qmd @@ -382,7 +382,7 @@ class Wastewater(Measurements): self.noise.validate() def lookback_days(self) -> int: - return len(self.temporal_pmf_rv()) + return len(self.temporal_pmf_rv()) - 1 def _predicted_obs(self, infections: ArrayLike) -> ArrayLike: shedding_pmf = self.temporal_pmf_rv() @@ -552,6 +552,19 @@ When you call `model.fit()`, you supply two types of information: - **Observation data** — one data dictionary per registered observation process - **Population structure** — how the jurisdiction is divided into subpopulations +### Shared Time Axis + +All observation data uses a **shared time axis** `[0, n_total)` where `n_total = n_init + n_days`. This shared axis aligns observations with the internal infection vectors: + +- Index 0 corresponds to the first day of the initialization period +- Index `n_init` corresponds to the first day of actual observations +- Index `n_total - 1` corresponds to the last observation day + +The model provides helper methods to align your data with this shared axis: + +- `model.pad_observations(obs)` — prepends `n_init` NaN values to dense observation vectors +- `model.shift_times(times)` — adds `n_init` to sparse time indices + ### Observation Data by Signal Type Each observation process's `name` attribute becomes the keyword argument for passing data to `model.fit()`: @@ -561,32 +574,34 @@ builder.add_observation(hosp_obs) # hosp_obs.name="hospital" → hospital={... builder.add_observation(ww_obs) # ww_obs.name="wastewater" → wastewater={...} ``` -#### Jurisdiction-level signals +#### Jurisdiction-level signals (dense) -The jurisdiction-level hospital admissions data is specified as a `Counts` observations process. +The jurisdiction-level hospital admissions data is specified as a `Counts` observations process with dense data padded to length `n_total`: ```python hospital={ - "obs": jnp.array([...]), # observed values + "obs": model.pad_observations(hosp_counts), # shape: (n_total,), NaN-padded } ``` -*Note*: this example uses dense data, i.e., there is an observation for every day in the timeline. Were the data sparse, it would be necessary to also provide an array `times` which supplies the time index for each observation. +The `pad_observations` method prepends `n_init` NaN values. NaN marks the initialization period where predictions exist but observations do not. You can also use NaN to mark missing data within the observation period. -#### Subpopulation-level signal +#### Subpopulation-level signals (sparse) -The subpopulation-level wastewater data is specified as a `Wastewater` observations process. +The subpopulation-level wastewater data is specified as a `Wastewater` observations process with sparse indexing on the shared time axis: ```python wastewater={ - "obs": jnp.array([...]), # observed log concentrations - "times": jnp.array([...]), # time index for each observation - "subpop_indices": jnp.array([...]), # which catchment (selects infection trajectory) - "sensor_indices": jnp.array([...]), # which WWTP/lab pair (selects noise parameters) - "n_sensors": int, # total number of WWTP/lab pairs + "obs": jnp.array([...]), # observed log concentrations (n_obs,) + "times": model.shift_times(ww_times), # time indices on shared axis + "subpop_indices": jnp.array([...]), # which catchment (selects infection trajectory) + "sensor_indices": jnp.array([...]), # which WWTP/lab pair (selects noise parameters) + "n_sensors": int, # total number of WWTP/lab pairs } ``` +The `shift_times` method adds `n_init` to convert from natural coordinates (0 = first observation day) to the shared time axis. + A **subpopulation** is a portion of the jurisdiction's population defined by a catchment area. A **sensor** is a measurement source — typically a WWTP/lab pair — that produces observations. Multiple sensors can observe the same subpopulation (e.g., different labs processing samples from the same catchment), so `subpop_indices` and `sensor_indices` may differ. - `subpop_indices` links each observation to the appropriate infection trajectory @@ -615,14 +630,14 @@ This specifies 5 observed subpopulations (e.g., wastewater catchments) and 1 uno model.fit( n_days_post_init=n_days, population_size=population_size, - # Population structure (choose one parameterization) + # Population structure obs_fractions=obs_subpop_fractions, unobs_fractions=unobs_subpop_fractions, # Observation data (keyed by names from add_observation) - hospital={"obs": hosp_counts, "times": hosp_times}, + hospital={"obs": model.pad_observations(hosp_counts)}, wastewater={ "obs": ww_conc, - "times": ww_times, + "times": model.shift_times(ww_times), "subpop_indices": ww_subpop_indices, "sensor_indices": ww_sensor_indices, "n_sensors": n_ww_sensors, @@ -652,27 +667,23 @@ print( ) ``` -It is necessary to align the data with the model's time indexing. - -- Times are relative to the start of the simulation (0 = first day of initialization period) -- Observation times should be offset by `n_initialization_points` - -To do this, we define function `prepare_observation-data`. +We define a function to prepare observation data using the model's helper methods to align with the shared time axis. ```{python} # | label: prepare-observation-data -def prepare_observation_data(n_days_fit, n_init, hosp_admits, ww_data, K_obs): +def prepare_observation_data(model, n_days_fit, hosp_admits, ww_data, K_obs): """ Prepare observation data for fitting. - Returns hospital and wastewater data dicts aligned with model time indexing. + Uses model.pad_observations() and model.shift_times() to align data + with the shared time axis. Parameters ---------- + model : MultiSignalModel + The model (provides padding/shifting helpers) n_days_fit : int Number of days to include in fit - n_init : int - Number of initialization points (for time offset) hosp_admits : array Hospital admissions time series ww_data : dict @@ -680,14 +691,12 @@ def prepare_observation_data(n_days_fit, n_init, hosp_admits, ww_data, K_obs): K_obs : int Number of observed subpopulations (for mapping sites to subpops) """ - # Hospital: jurisdiction-level, sparse times - hosp_times = jnp.arange(n_days_fit) + n_init - hosp_counts = hosp_admits[:n_days_fit] + # Hospital: dense, NaN-padded to length n_total + hosp_obs = model.pad_observations(hosp_admits[:n_days_fit]) - # Wastewater: sensor-level, sparse times - # Filter to observations within the fitting window + # Wastewater: sparse, times shifted to shared axis ww_mask = ww_data["time_indices"] < n_days_fit - ww_times = ww_data["time_indices"][ww_mask] + n_init + ww_times = model.shift_times(ww_data["time_indices"][ww_mask]) ww_conc = ww_data["observed_conc"][ww_mask] ww_sensors = ww_data["site_ids"][ww_mask] @@ -701,8 +710,7 @@ def prepare_observation_data(n_days_fit, n_init, hosp_admits, ww_data, K_obs): return { "hospital": { - "obs": hosp_counts, - "times": hosp_times, + "obs": hosp_obs, }, "wastewater": { "obs": ww_conc, @@ -726,7 +734,7 @@ jax.clear_caches() n_days_90 = 90 obs_data_90 = prepare_observation_data( - n_days_90, n_init, hosp_admits, ca_ww_data, K_obs + model, n_days_90, hosp_admits, ca_ww_data, K_obs ) # Try non-centered reparameterization for hierarchical parameters @@ -905,7 +913,7 @@ jax.clear_caches() n_days_180 = 180 obs_data_180 = prepare_observation_data( - n_days_180, n_init, hosp_admits, ca_ww_data, K_obs + model, n_days_180, hosp_admits, ca_ww_data, K_obs ) print(f"Fitting model with {n_days_180} days of data...") diff --git a/docs/tutorials/observation_processes_counts.qmd b/docs/tutorials/observation_processes_counts.qmd index c39b9cfe..dabc79e8 100644 --- a/docs/tutorials/observation_processes_counts.qmd +++ b/docs/tutorials/observation_processes_counts.qmd @@ -176,7 +176,7 @@ hosp_process = Counts( The observation process convolves infections with a delay distribution, maintaining alignment between input and output: day $t$ in the output corresponds to day $t$ in the input. -Hospital admissions depend on infections from prior days (the length of our delay distribution minus one). The method `lookback_days()` returns this length; the first valid observation day is at index `lookback - 1`. Earlier days are marked invalid. +Hospital admissions depend on infections from prior days. A delay PMF of length $L$ covers delays 0 to $L-1$, requiring $L-1$ days of prior infection history. The method `lookback_days()` returns $L-1$; the first valid observation day is at index `lookback_days()`. Earlier days are marked invalid. ```{python} # | label: helper-function @@ -185,7 +185,7 @@ print(f"Required lookback: {hosp_process.lookback_days()} days") def first_valid_observation_day(obs_process) -> int: """Return the first day index with complete infection history for convolution.""" - return obs_process.lookback_days() - 1 + return obs_process.lookback_days() ``` ## Simulating observed hospital admissions given a single day's worth of infections @@ -341,7 +341,8 @@ for seed in range(n_samples): with numpyro.handlers.seed(rng_seed=seed): hosp_sample = hosp_process.sample(infections=infections, obs=None) - for i, val in enumerate(hosp_sample.observed): + # Slice from day_one to align with valid observation period + for i, val in enumerate(hosp_sample.observed[day_one:]): samples_list.append( { "day": i, @@ -519,8 +520,8 @@ for conc_val in concentration_values: obs=None, ) - # Use relative days - for i, admit in enumerate(hosp_temp.observed): + # Slice from day_one to align with valid observation period + for i, admit in enumerate(hosp_temp.observed[day_one:]): conc_results.append( { "day": i, @@ -597,7 +598,8 @@ for seed in range(n_replicates_poisson): obs=None, ) - for i, admit in enumerate(poisson_temp.observed): + # Slice from day_one to align with valid observation period + for i, admit in enumerate(poisson_temp.observed[day_one:]): poisson_results.append( { "day": i, diff --git a/docs/tutorials/observation_processes_measurements.qmd b/docs/tutorials/observation_processes_measurements.qmd index 510cd528..aeebd370 100644 --- a/docs/tutorials/observation_processes_measurements.qmd +++ b/docs/tutorials/observation_processes_measurements.qmd @@ -154,7 +154,7 @@ class MyMeasurement(Measurements): self._validate_pmf(pmf, "temporal_pmf_rv") def lookback_days(self): - return len(self.temporal_pmf_rv()) + return len(self.temporal_pmf_rv()) - 1 ``` @@ -241,8 +241,8 @@ class Wastewater(Measurements): self.noise.validate() def lookback_days(self) -> int: - """Return shedding PMF length.""" - return len(self.temporal_pmf_rv()) + """Return required lookback (PMF length minus 1).""" + return len(self.temporal_pmf_rv()) - 1 def _predicted_obs(self, infections: ArrayLike) -> ArrayLike: """ @@ -397,13 +397,13 @@ print(f"Required lookback: {ww_process.lookback_days()} days") ### Timeline alignment -The observation process maintains alignment: day $t$ in output corresponds to day $t$ in input. The first `lookback_days() - 1` days have incomplete history and are marked invalid. +The observation process maintains alignment: day $t$ in output corresponds to day $t$ in input. A temporal PMF of length $L$ covers lags 0 to $L-1$, requiring $L-1$ days of prior history. The method `lookback_days()` returns $L-1$; the first valid observation day is at index `lookback_days()`. ```{python} # | label: helper-function def first_valid_observation_day(obs_process) -> int: """Return the first day index with complete infection history.""" - return obs_process.lookback_days() - 1 + return obs_process.lookback_days() ``` ### Simulating from observations from a single-day infection spike diff --git a/pyrenew/latent/temporal_processes.py b/pyrenew/latent/temporal_processes.py index 5c4ea484..3854b0a2 100644 --- a/pyrenew/latent/temporal_processes.py +++ b/pyrenew/latent/temporal_processes.py @@ -3,18 +3,59 @@ Temporal processes for latent infection models. Provides time-series processes for modeling Rt dynamics and subpopulation -deviations. All processes support both single trajectory and vectorized -(multiple parallel trajectories) sampling. - -**Temporal Processes** - +deviations in hierarchical infection models. All processes support both +single trajectory and vectorized (multiple parallel trajectories) sampling +through a unified ``TemporalProcess`` protocol. + +Relationship to pyrenew.process +------------------------------- +This module provides **high-level, domain-specific wrappers** around the +low-level building blocks in :mod:`pyrenew.process`. The key differences: + +.. list-table:: + :header-rows: 1 + + * - Aspect + - ``pyrenew.process`` + - ``pyrenew.latent.temporal_processes`` + * - Abstraction level + - Low-level composable primitives + - High-level domain-specific API + * - Interface + - Varied signatures per class + - Unified ``TemporalProcess`` protocol + * - Target use + - General time-series modeling + - Rt dynamics, hierarchical infections + * - Vectorization + - Caller manages array shapes + - Automatic via ``n_processes`` parameter + * - Validation + - Minimal constraints + - Enforces stationarity (``|autoreg| < 1``) + +**When to use which:** + +- Use ``pyrenew.process`` classes (``ARProcess``, ``DifferencedProcess``, + ``RandomWalk``) when building novel statistical models or when you need + fine-grained control over array shapes and numpyro sampling semantics. + +- Use this module's classes (``AR1``, ``DifferencedAR1``, ``RandomWalk``) + when modeling Rt trajectories in hierarchical infection models. These + provide a consistent interface, automatic vectorization, and enforce + epidemiologically-sensible constraints. + +Temporal Processes +------------------ - ``AR1``: Autoregressive process with mean reversion. Keeps Rt bounded - near a baseline. + near a baseline. Wraps :class:`pyrenew.process.ARProcess`. - ``DifferencedAR1``: AR(1) on first differences. Allows persistent trends - while stabilizing the growth rate. + while stabilizing the growth rate. Wraps :class:`pyrenew.process.DifferencedProcess`. - ``RandomWalk``: No mean reversion. Rt can drift without bound. + Wraps :class:`pyrenew.process.RandomWalk`. -All implementations satisfy the ``TemporalProcess`` protocol. +All implementations satisfy the ``TemporalProcess`` protocol and can be +used interchangeably in hierarchical infection models. """ from __future__ import annotations @@ -84,6 +125,10 @@ class AR1(TemporalProcess): toward a mean level. Keeps Rt bounded near a baseline — values that drift away are "pulled back" over time. + This class wraps :class:`pyrenew.process.ARProcess` with a simplified, + protocol-compliant interface that handles vectorization automatically + and enforces stationarity constraints. + Parameters ---------- autoreg : float @@ -246,6 +291,10 @@ class DifferencedAR1(TemporalProcess): the rate of change reverting toward a mean. Unlike AR(1), this allows Rt to trend persistently upward or downward while the growth rate stabilizes. + This class wraps :class:`pyrenew.process.DifferencedProcess` with + :class:`pyrenew.process.ARProcess` as the fundamental process, providing + a simplified, protocol-compliant interface. + Parameters ---------- autoreg : float @@ -412,6 +461,10 @@ class RandomWalk(TemporalProcess): toward a mean. Allows Rt to drift without bound — suitable when you have no prior expectation that Rt will return to a baseline. + This class wraps :class:`pyrenew.process.RandomWalk` with a simplified, + protocol-compliant interface. The vectorized mode uses a non-centered + parameterization to avoid funnel problems in inference. + Parameters ---------- innovation_sd : float, default 1.0 diff --git a/pyrenew/model/multisignal_model.py b/pyrenew/model/multisignal_model.py index 6692b06d..219a3109 100644 --- a/pyrenew/model/multisignal_model.py +++ b/pyrenew/model/multisignal_model.py @@ -106,6 +106,61 @@ def validate(self) -> None: """ self._validate() + def pad_observations( + self, + obs: jnp.ndarray, + axis: int = 0, + ) -> jnp.ndarray: + """ + Pad observations with NaN for the initialization period. + + Observation data uses a shared time axis [0, n_total) where + n_total = n_init + n_days. This method prepends n_init NaN values + to align user data (starting at day 0 of observations) with the + shared axis. + + Parameters + ---------- + obs : ArrayLike + Observations in natural coordinates (index 0 = first observation day). + Integer arrays are converted to float (required for NaN). + axis : int, default 0 + Axis along which to pad (typically 0 for time axis). + + Returns + ------- + jnp.ndarray + Padded observations. First n_init values are NaN. + """ + n_init = self.latent.n_initialization_points + obs = jnp.asarray(obs, dtype=float) + pad_shape = list(obs.shape) + pad_shape[axis] = n_init + padding = jnp.full(pad_shape, jnp.nan) + return jnp.concatenate([padding, obs], axis=axis) + + def shift_times(self, times: jnp.ndarray) -> jnp.ndarray: + """ + Shift time indices from natural coordinates to shared time axis. + + Observation data uses a shared time axis [0, n_total) where + n_total = n_init + n_days. User-provided times in natural coordinates + (0 = first observation day) must be shifted by n_init to align with + the shared axis. + + Parameters + ---------- + times : ArrayLike + Time indices in natural coordinates (0 = first observation day). + + Returns + ------- + jnp.ndarray + Time indices on the shared axis [n_init, n_total). + """ + n_init = self.latent.n_initialization_points + return jnp.asarray(times) + n_init + def validate_data( self, n_days_post_init: int, @@ -116,9 +171,14 @@ def validate_data( """ Validate observation data before running MCMC. - This method should be called with concrete (non-traced) values - before running inference. It performs validation that cannot be - done during JAX tracing. + All observation data uses a shared time axis [0, n_total) where + n_total = n_init + n_days_post_init. Dense observations must have + length n_total with NaN padding for the initialization period. + Sparse observations provide times indices on this shared axis. + + This method must be called with concrete (non-traced) values + before running inference. Validation using Python control flow + (if/raise) cannot be done during JAX tracing. Parameters ---------- @@ -136,15 +196,16 @@ def validate_data( ------ ValueError If times indices are out of bounds or negative + If dense obs length doesn't match n_total If data shapes are inconsistent """ - # Parse population structure to get K_obs for validation pop = BaseLatentInfectionProcess._parse_and_validate_fractions( obs_fractions=obs_fractions, unobs_fractions=unobs_fractions, ) - n_total_days = self.latent.n_initialization_points + n_days_post_init + n_init = self.latent.n_initialization_points + n_total = n_init + n_days_post_init for name, obs_data in observation_data.items(): if name not in self.observations: @@ -153,32 +214,39 @@ def validate_data( f"Available: {list(self.observations.keys())}" ) - # Validate times if present (sparse observations) + obs = obs_data.get("obs") times = obs_data.get("times") + if times is not None: + # Sparse observations: times on shared axis [0, n_total) times = jnp.asarray(times) if jnp.any(times < 0): - raise ValueError( - f"Observation '{name}': times cannot contain negative values" - ) + raise ValueError(f"Observation '{name}': times cannot be negative") max_time = jnp.max(times) - if max_time >= n_total_days: + if max_time >= n_total: raise ValueError( - f"Observation '{name}': times contains {int(max_time)} " - f">= {n_total_days} (n_total_days = " - f"{self.latent.n_initialization_points} init + " - f"{n_days_post_init} post-init)" + f"Observation '{name}': times index {int(max_time)} " + f">= n_total ({n_total} = {n_init} init + " + f"{n_days_post_init} days). " + f"Times must be on shared axis [0, {n_total})." ) - - # Validate obs matches times length - obs = obs_data.get("obs") if obs is not None and len(obs) != len(times): raise ValueError( f"Observation '{name}': obs length {len(obs)} " f"must match times length {len(times)}" ) + elif obs is not None: + # Dense observations: length must equal n_total + obs = jnp.asarray(obs) + if obs.shape[0] != n_total: + raise ValueError( + f"Observation '{name}': obs length {obs.shape[0]} " + f"must equal n_total ({n_total} = {n_init} init + " + f"{n_days_post_init} days). " + f"Pad with NaN for initialization period." + ) - # Validate subpop_indices if present (site-level observations) + # Validate subpop_indices if present subpop_indices = obs_data.get("subpop_indices") if subpop_indices is not None: subpop_indices = jnp.asarray(subpop_indices) diff --git a/pyrenew/observation/base.py b/pyrenew/observation/base.py index 4ba0117f..764aeb66 100644 --- a/pyrenew/observation/base.py +++ b/pyrenew/observation/base.py @@ -107,13 +107,15 @@ def lookback_days(self) -> int: ------- int Number of days of infection history required. - Typically the length of the delay or shedding PMF. Notes ----- + Delay/shedding PMFs are 0-indexed (effect can occur on day 0), so a + PMF of length L covers lags 0 to L-1, requiring L-1 initialization + points. Implementations should return ``len(pmf) - 1``. + This is used by model builders to automatically compute - n_initialization_points as: - ``max(gen_int_length, max(all lookbacks)) - 1`` + n_initialization_points as ``max(gen_int_length, max(all lookbacks))``. """ pass # pragma: no cover diff --git a/pyrenew/observation/count_observations.py b/pyrenew/observation/count_observations.py index 45aa2f8a..090970a7 100644 --- a/pyrenew/observation/count_observations.py +++ b/pyrenew/observation/count_observations.py @@ -74,14 +74,17 @@ def validate(self) -> None: def lookback_days(self) -> int: """ - Return delay PMF length. + Return required lookback days for this observation. + + Delay PMFs are 0-indexed (delay can be 0), so a PMF of length L + covers delays 0 to L-1, requiring L-1 initialization points. Returns ------- int - Length of delay distribution PMF. + Length of delay distribution PMF minus 1. """ - return len(self.temporal_pmf_rv()) + return len(self.temporal_pmf_rv()) - 1 def infection_resolution(self) -> str: """ @@ -118,20 +121,17 @@ def _predicted_obs( delay_pmf = self.temporal_pmf_rv() ascertainment_rate = self.ascertainment_rate_rv() - is_1d = infections.ndim == 1 - if is_1d: - predicted_counts = self._convolve_with_alignment( + if infections.ndim == 1: + return self._convolve_with_alignment( infections, delay_pmf, ascertainment_rate )[0] - else: - predicted_counts = jax.vmap( - lambda col: self._convolve_with_alignment( - col, delay_pmf, ascertainment_rate - )[0], - in_axes=1, - out_axes=1, - )(infections) - return predicted_counts + return jax.vmap( + lambda col: self._convolve_with_alignment( + col, delay_pmf, ascertainment_rate + )[0], + in_axes=1, + out_axes=1, + )(infections) class Counts(_CountBase): @@ -179,46 +179,57 @@ def sample( self, infections: ArrayLike, obs: ArrayLike | None = None, - times: ArrayLike | None = None, ) -> ObservationSample: """ - Sample aggregated counts with dense or sparse observations. + Sample aggregated counts. - Validation is performed before JAX tracing at runtime, - prior to calling this method. + Both infections and obs use a shared time axis [0, n_total) where + n_total = n_init + n_days. NaN in obs marks unobserved timepoints + (initialization period or missing data). Parameters ---------- infections : ArrayLike Aggregate infections from the infection process. - Shape: (n_days,) + Shape: (n_total,) where n_total = n_init + n_days. obs : ArrayLike | None - Observed counts. Dense: (n_days,), Sparse: (n_obs,), None: prior. - times : ArrayLike | None - Day indices relative to the infections vector for sparse observations. None for dense observations. + Observed counts on shared time axis. Shape: (n_total,). + Use NaN for initialization period and any missing observations. + None for prior predictive sampling. Returns ------- ObservationSample Named tuple with `observed` (sampled/conditioned counts) and - `predicted` (predicted counts before noise). + `predicted` (predicted counts before noise, shape: n_total). """ predicted_counts = self._predicted_obs(infections) self._deterministic("predicted", predicted_counts) - # Only use sparse indexing when conditioning on observations - if times is not None and obs is not None: - predicted_obs = predicted_counts[times] + # Compute mask: True where observation contributes to likelihood. + # NaN in predictions (initialization period) or obs (missing data) + # are excluded via obs_mask. + valid_pred = ~jnp.isnan(predicted_counts) + if obs is not None: + valid_obs = ~jnp.isnan(obs) + obs_mask = valid_pred & valid_obs else: - observable = ~jnp.isnan(predicted_counts) - predicted_obs = predicted_counts[observable] - if obs is not None: - obs = obs[observable] + obs_mask = valid_pred + + # JAX evaluates log_prob for all array elements even when obs_mask + # excludes them from the likelihood sum. Replace NaN with safe values + # to avoid NaN propagation in JAX's computation graph. These values + # do not affect inference since obs_mask=False excludes them. + safe_predicted = jnp.where(jnp.isnan(predicted_counts), 1.0, predicted_counts) + safe_obs = None + if obs is not None: + safe_obs = jnp.where(jnp.isnan(obs), 0.0, obs) observed = self.noise.sample( name=self._sample_site_name("obs"), - predicted=predicted_obs, - obs=obs, + predicted=safe_predicted, + obs=safe_obs, + obs_mask=obs_mask, ) return ObservationSample(observed=observed, predicted=predicted_counts) @@ -267,27 +278,28 @@ def infection_resolution(self) -> str: def sample( self, infections: ArrayLike, - subpop_indices: ArrayLike, times: ArrayLike, + subpop_indices: ArrayLike, obs: ArrayLike | None = None, ) -> ObservationSample: """ - Sample subpopulation-level counts with flexible indexing. + Sample subpopulation-level counts. - Validation is performed before JAX tracing at runtime, - prior to calling this method. + Times are on the shared time axis [0, n_total) where + n_total = n_init + n_days. This method performs direct indexing + without any offset adjustment. Parameters ---------- infections : ArrayLike Subpopulation-level infections from the infection process. - Shape: (n_days, n_subpops) + Shape: (n_total, n_subpops) + times : ArrayLike + Day index for each observation on the shared time axis. + Must be in range [0, n_total). Shape: (n_obs,) subpop_indices : ArrayLike Subpopulation index for each observation (0-indexed). Shape: (n_obs,) - times : ArrayLike - Day index for each observation (0-indexed). - Shape: (n_obs,) obs : ArrayLike | None Observed counts (n_obs,), or None for prior sampling. @@ -295,12 +307,12 @@ def sample( ------- ObservationSample Named tuple with `observed` (sampled/conditioned counts) and - `predicted` (predicted counts before noise, shape: n_days x n_subpops). + `predicted` (predicted counts before noise, shape: n_total x n_subpops). """ - # Compute predicted counts for all subpops predicted_counts = self._predicted_obs(infections) - self._deterministic("predicted", predicted_counts) + + # Direct indexing on shared time axis - no offset needed predicted_obs = predicted_counts[times, subpop_indices] observed = self.noise.sample( diff --git a/pyrenew/observation/measurements.py b/pyrenew/observation/measurements.py index 2d090bed..ab1b8ff1 100644 --- a/pyrenew/observation/measurements.py +++ b/pyrenew/observation/measurements.py @@ -76,6 +76,20 @@ def __repr__(self) -> str: f"noise={self.noise!r})" ) + def lookback_days(self) -> int: + """ + Return required lookback days for this observation. + + Temporal PMFs are 0-indexed (effect can occur on day 0), so a PMF + of length L covers lags 0 to L-1, requiring L-1 initialization points. + + Returns + ------- + int + Length of temporal PMF minus 1. + """ + return len(self.temporal_pmf_rv()) - 1 + def infection_resolution(self) -> str: """ Return "subpop" for measurement observations. @@ -93,18 +107,18 @@ def infection_resolution(self) -> str: def sample( self, infections: ArrayLike, + times: ArrayLike, subpop_indices: ArrayLike, sensor_indices: ArrayLike, - times: ArrayLike, - obs: ArrayLike | None, n_sensors: int, + obs: ArrayLike | None = None, ) -> ObservationSample: """ Sample measurements from observed sensors. - This method does not perform runtime validation of index values - (times, subpop_indices, sensor_indices). Validate observation data - before sampling. + Times are on the shared time axis [0, n_total) where + n_total = n_init + n_days. This method performs direct indexing + without any offset adjustment. Transforms infections to predicted values via signal-specific processing (``_predicted_obs``), then applies noise model. @@ -113,31 +127,31 @@ def sample( ---------- infections : ArrayLike Infections from the infection process. - Shape: (n_days, n_subpops) + Shape: (n_total, n_subpops) + times : ArrayLike + Day index for each observation on the shared time axis. + Must be in range [0, n_total). Shape: (n_obs,) subpop_indices : ArrayLike Subpopulation index for each observation (0-indexed). Shape: (n_obs,) sensor_indices : ArrayLike Sensor index for each observation (0-indexed). Shape: (n_obs,) - times : ArrayLike - Day index for each observation (0-indexed). - Shape: (n_obs,) - obs : ArrayLike | None - Observed measurements (n_obs,), or None for prior sampling. n_sensors : int Total number of measurement sensors. + obs : ArrayLike | None + Observed measurements (n_obs,), or None for prior sampling. Returns ------- ObservationSample Named tuple with `observed` (sampled/conditioned measurements) and - `predicted` (predicted values before noise, shape: n_days x n_subpops). + `predicted` (predicted values before noise, shape: n_total x n_subpops). """ predicted_values = self._predicted_obs(infections) - self._deterministic("predicted", predicted_values) + # Direct indexing on shared time axis - no offset needed predicted_obs = predicted_values[times, subpop_indices] observed = self.noise.sample( diff --git a/pyrenew/observation/noise.py b/pyrenew/observation/noise.py index 349d481d..2f86d416 100644 --- a/pyrenew/observation/noise.py +++ b/pyrenew/observation/noise.py @@ -100,6 +100,7 @@ def sample( name: str, predicted: ArrayLike, obs: ArrayLike | None = None, + obs_mask: ArrayLike | None = None, ) -> ArrayLike: """ Sample count observations given predicted counts. @@ -112,11 +113,22 @@ def sample( Predicted count values (non-negative). obs : ArrayLike | None Observed counts for conditioning, or None for prior sampling. + obs_mask : ArrayLike | None + Boolean mask indicating which observations to include in the + likelihood. If None, all observations are included. If provided, + observations where mask is False are excluded from the likelihood. Returns ------- ArrayLike Sampled or conditioned counts, same shape as predicted. + + Notes + ----- + Implementations use ``numpyro.handlers.mask`` rather than the + ``obs_mask`` parameter of ``numpyro.sample``. This avoids creating + latent variables for masked entries, which would fail with NUTS + for discrete distributions. """ pass # pragma: no cover @@ -155,6 +167,7 @@ def sample( name: str, predicted: ArrayLike, obs: ArrayLike | None = None, + obs_mask: ArrayLike | None = None, ) -> ArrayLike: """ Sample from Poisson distribution. @@ -167,17 +180,27 @@ def sample( Predicted count values. obs : ArrayLike | None Observed counts for conditioning. + obs_mask : ArrayLike | None + Boolean mask indicating which observations to include in the + likelihood. If None, all observations are included. Returns ------- ArrayLike Poisson-distributed counts. """ - return numpyro.sample( - name, - dist.Poisson(rate=predicted + _EPSILON), - obs=obs, - ) + if obs_mask is None: + return numpyro.sample( + name, + dist.Poisson(rate=predicted + _EPSILON), + obs=obs, + ) + with numpyro.handlers.mask(mask=obs_mask): + return numpyro.sample( + name, + dist.Poisson(rate=predicted + _EPSILON), + obs=obs, + ) class NegativeBinomialNoise(CountNoise): @@ -235,6 +258,7 @@ def sample( name: str, predicted: ArrayLike, obs: ArrayLike | None = None, + obs_mask: ArrayLike | None = None, ) -> ArrayLike: """ Sample from Negative Binomial distribution. @@ -247,6 +271,9 @@ def sample( Predicted count values. obs : ArrayLike | None Observed counts for conditioning. + obs_mask : ArrayLike | None + Boolean mask indicating which observations to include in the + likelihood. If None, all observations are included. Returns ------- @@ -254,14 +281,24 @@ def sample( Negative Binomial-distributed counts. """ concentration = self.concentration_rv() - return numpyro.sample( - name, - dist.NegativeBinomial2( - mean=predicted + _EPSILON, - concentration=concentration, - ), - obs=obs, - ) + if obs_mask is None: + return numpyro.sample( + name, + dist.NegativeBinomial2( + mean=predicted + _EPSILON, + concentration=concentration, + ), + obs=obs, + ) + with numpyro.handlers.mask(mask=obs_mask): + return numpyro.sample( + name, + dist.NegativeBinomial2( + mean=predicted + _EPSILON, + concentration=concentration, + ), + obs=obs, + ) class MeasurementNoise(ABC): diff --git a/test/test_model_builder.py b/test/test_model_builder.py index bab89446..8ff08eb9 100644 --- a/test/test_model_builder.py +++ b/test/test_model_builder.py @@ -156,8 +156,10 @@ def test_compute_n_initialization_points_returns_correct_value( ): """Test that compute_n_initialization_points returns max of lookbacks.""" n_init = simple_builder.compute_n_initialization_points() - # gen_int has 3 elements, delay has 4 elements, so max is 4 - assert n_init == 4 + # gen_int has 3 elements (1-indexed) -> 3 + # delay has 4 elements (0-indexed) -> lookback_days = 3 + # max(3, 3) = 3 + assert n_init == 3 class TestMultiSignalModelSampling: @@ -189,8 +191,11 @@ def test_fit_with_reparam_config(self, simple_builder): model = simple_builder.build() n_days = 10 - times = jnp.array([5, 6, 7, 8, 9]) - obs = jnp.array([10, 12, 15, 14, 11]) + # Create dense observations with NaN padding for initialization period + obs_values = jnp.array([10.0, 12.0, 15.0, 14.0, 11.0]) + obs = model.pad_observations(obs_values) + # Pad with NaN for remaining days + obs = jnp.concatenate([obs, jnp.full(n_days - len(obs_values), jnp.nan)]) # Test with reparam_config reparam_config = { @@ -205,7 +210,7 @@ def test_fit_with_reparam_config(self, simple_builder): num_warmup=2, num_samples=2, reparam_config=reparam_config, - hospital={"obs": obs, "times": times}, + hospital={"obs": obs}, ) samples = mcmc.get_samples() @@ -217,8 +222,11 @@ def test_fit_with_population_structure(self, simple_builder): n_days = 10 n_total = model.latent.n_initialization_points + n_days - times = jnp.array([5, 6, 7, 8, 9]) - obs = jnp.array([10, 12, 15, 14, 11]) + # Create dense observations with NaN padding for initialization period + obs_values = jnp.array([10.0, 12.0, 15.0, 14.0, 11.0]) + obs = model.pad_observations(obs_values) + # Pad with NaN for remaining days + obs = jnp.concatenate([obs, jnp.full(n_days - len(obs_values), jnp.nan)]) mcmc = model.fit( n_days_post_init=n_days, @@ -227,7 +235,7 @@ def test_fit_with_population_structure(self, simple_builder): unobs_fractions=UNOBS_FRACTIONS, num_warmup=5, num_samples=5, - hospital={"obs": obs, "times": times}, + hospital={"obs": obs}, ) samples = mcmc.get_samples() @@ -258,7 +266,7 @@ def test_validate_data_rejects_out_of_bounds_times(self, simple_builder): model = simple_builder.build() n_total = model.latent.n_initialization_points + 30 - with pytest.raises(ValueError, match="times contains"): + with pytest.raises(ValueError, match="times index"): model.validate_data( n_days_post_init=30, obs_fractions=OBS_FRACTIONS, @@ -273,7 +281,7 @@ def test_validate_data_rejects_negative_times(self, simple_builder): """Test that negative times raises error.""" model = simple_builder.build() - with pytest.raises(ValueError, match="cannot contain negative"): + with pytest.raises(ValueError, match="times cannot be negative"): model.validate_data( n_days_post_init=30, obs_fractions=OBS_FRACTIONS, diff --git a/test/test_observation_counts.py b/test/test_observation_counts.py index a22ea275..80f5b02c 100644 --- a/test/test_observation_counts.py +++ b/test/test_observation_counts.py @@ -88,14 +88,13 @@ def test_delay_convolution(self, counts_factory, short_delay_pmf): obs=None, ) - # Timeline alignment: predicted length equals input length + # Timeline alignment: both predicted and observed have same length as input assert result.predicted.shape[0] == len(infections) - # observed excludes NaN entries (first len(pmf) - 1) - assert result.observed.shape[0] == len(infections) - (len(short_delay_pmf) - 1) - # first len(pmf) - 1 entries NaN, others not + assert result.observed.shape[0] == len(infections) + # First len(pmf) - 1 entries in predicted are NaN (initialization period) assert jnp.all(jnp.isnan(result.predicted[:1])) assert jnp.all(~jnp.isnan(result.predicted[1:])) - assert jnp.all(~jnp.isnan(result.observed)) + # Observed is sampled for all entries (masked entries don't affect likelihood) assert jnp.all(result.observed >= 0) def test_ascertainment_scaling(self, counts_factory, simple_delay_pmf): @@ -240,82 +239,44 @@ def test_long_delay_distribution(self, counts_factory, long_delay_pmf): assert jnp.all(result.observed >= 0) -class TestCountsSparseObservations: - """Test sparse observation support.""" +class TestCountsDenseObservations: + """Test dense observation support with NaN padding.""" - def test_sparse_observations(self, counts_process): - """Test with sparse (irregular) observations.""" + def test_dense_observations_with_nan_padding(self, counts_process): + """Test with dense observations including NaN padding.""" n_days = 30 infections = jnp.ones(n_days) * 100 - # Sparse observations: only days 5, 10, 15, 20 - times = jnp.array([5, 10, 15, 20]) - counts_data = jnp.array([10, 12, 8, 15]) + # Dense observations with NaN for "missing" days + obs = jnp.ones(n_days) * 10.0 + obs = obs.at[:5].set(jnp.nan) # First 5 days "missing" with numpyro.handlers.seed(rng_seed=42): result = counts_process.sample( infections=infections, - obs=counts_data, - times=times, - ) - - assert result.observed.shape == times.shape - assert jnp.allclose(result.observed, counts_data) - - def test_sparse_vs_dense_sampling(self, counts_process): - """Test that sparse sampling gives different output shape than dense.""" - n_days = 30 - infections = jnp.ones(n_days) * 100 - - # Dense: prior sampling (obs=None, no times) - with numpyro.handlers.seed(rng_seed=42): - dense_result = counts_process.sample( - infections=infections, - obs=None, - ) - - # Sparse with observed data: only some days - times = jnp.array([5, 10, 15, 20]) - sparse_obs_data = jnp.array([10, 12, 8, 15]) - with numpyro.handlers.seed(rng_seed=42): - sparse_result = counts_process.sample( - infections=infections, - obs=sparse_obs_data, - times=times, + obs=obs, ) - # Dense prior produces full length output - assert dense_result.observed.shape == (n_days,) - - # Sparse observations produce output matching times shape - assert sparse_result.observed.shape == times.shape - assert jnp.allclose(sparse_result.observed, sparse_obs_data) + # With masking, observed has same shape as input (masked entries + # are sampled but don't contribute to likelihood) + assert result.observed.shape[0] == n_days + assert result.predicted.shape[0] == n_days - def test_prior_sampling_ignores_times(self, counts_process): - """Test that times parameter is ignored when obs=None (prior sampling).""" + def test_prior_sampling_dense(self, counts_process): + """Test prior sampling produces dense output.""" n_days = 30 infections = jnp.ones(n_days) * 100 - times = jnp.array([5, 10, 15, 20]) - - # When obs=None, times is ignored - output is dense - with numpyro.handlers.seed(rng_seed=42): - result_with_times = counts_process.sample( - infections=infections, - obs=None, - times=times, - ) with numpyro.handlers.seed(rng_seed=42): - result_without_times = counts_process.sample( + result = counts_process.sample( infections=infections, obs=None, ) - # Both should produce dense output of shape (n_days,) - assert result_with_times.observed.shape == (n_days,) - assert result_without_times.observed.shape == (n_days,) - # With same seed, outputs should be identical - assert jnp.allclose(result_with_times.observed, result_without_times.observed) + # Prior sampling: observed excludes NaN predictions (init period) + assert result.observed.shape[0] == n_days # simple_delay_pmf has no init + assert result.predicted.shape == (n_days,) + assert jnp.all(~jnp.isnan(result.observed)) class TestCountsBySubpop: @@ -332,14 +293,15 @@ def test_sample_returns_correct_shape(self): ) infections = jnp.ones((30, 3)) * 500 # 30 days, 3 subpops + # Times on shared axis (must be >= len(delay_pmf) - 1 to avoid NaN) times = jnp.array([10, 15, 10, 15]) subpop_indices = jnp.array([0, 0, 1, 1]) with numpyro.handlers.seed(rng_seed=42): result = process.sample( infections=infections, - subpop_indices=subpop_indices, times=times, + subpop_indices=subpop_indices, obs=None, ) @@ -448,14 +410,15 @@ def test_validate_invalid_ascertainment_rate_greater_than_one( process.validate() def test_lookback_days(self, simple_delay_pmf, long_delay_pmf): - """Test lookback_days returns PMF length.""" + """Test lookback_days returns PMF length minus 1 (0-indexed delays).""" process_short = Counts( name="test_short", ascertainment_rate_rv=DeterministicVariable("ihr", 0.01), delay_distribution_rv=DeterministicPMF("delay", simple_delay_pmf), noise=NegativeBinomialNoise(DeterministicVariable("conc", 10.0)), ) - assert process_short.lookback_days() == 1 + # simple_delay_pmf has length 1, lookback = 1 - 1 = 0 + assert process_short.lookback_days() == 0 process_long = Counts( name="test_long", @@ -463,7 +426,8 @@ def test_lookback_days(self, simple_delay_pmf, long_delay_pmf): delay_distribution_rv=DeterministicPMF("delay", long_delay_pmf), noise=NegativeBinomialNoise(DeterministicVariable("conc", 10.0)), ) - assert process_long.lookback_days() == 10 + # long_delay_pmf has length 10, lookback = 10 - 1 = 9 + assert process_long.lookback_days() == 9 def test_infection_resolution_counts(self, simple_delay_pmf): """Test that Counts returns 'aggregate' resolution.""" diff --git a/test/test_observation_measurements.py b/test/test_observation_measurements.py index d0592d50..14c09f44 100644 --- a/test/test_observation_measurements.py +++ b/test/test_observation_measurements.py @@ -34,14 +34,17 @@ def validate(self) -> None: def lookback_days(self) -> int: """ - Return temporal PMF length. + Return required lookback days for this observation. + + Temporal PMFs are 0-indexed (effect can occur on day 0), so a PMF + of length L covers lags 0 to L-1, requiring L-1 initialization points. Returns ------- int - Length of the temporal PMF. + Length of temporal PMF minus 1. """ - return len(self.temporal_pmf_rv()) + return len(self.temporal_pmf_rv()) - 1 def _predicted_obs(self, infections): """ @@ -205,6 +208,28 @@ def test_sample_with_observations(self): class TestConcreteMeasurements: """Test concrete Measurements implementation.""" + def test_lookback_days(self): + """Test lookback_days returns len(pmf) - 1.""" + # PMF of length 3 should return 2 (covers lags 0, 1, 2) + shedding_pmf = jnp.array([0.3, 0.4, 0.3]) + sensor_mode_rv = VectorizedRV( + DistributionalVariable("mode", dist.Normal(0, 0.5)), + plate_name="sensor_mode", + ) + sensor_sd_rv = VectorizedRV( + DistributionalVariable("sd", dist.TruncatedNormal(0.3, 0.15, low=0.05)), + plate_name="sensor_sd", + ) + noise = HierarchicalNormalNoise(sensor_mode_rv, sensor_sd_rv) + + process = ConcreteMeasurements( + name="test", + temporal_pmf_rv=DeterministicPMF("shedding", shedding_pmf), + noise=noise, + ) + + assert process.lookback_days() == 2 # len(3) - 1 = 2 + def test_repr(self): """Test Measurements __repr__ method.""" shedding_pmf = jnp.array([0.3, 0.4, 0.3]) @@ -256,11 +281,11 @@ def test_sample_shape(self): with numpyro.handlers.seed(rng_seed=42): result = process.sample( infections=infections, + times=times, subpop_indices=subpop_indices, sensor_indices=sensor_indices, - times=times, - obs=None, n_sensors=2, + obs=None, ) assert result.observed.shape == times.shape @@ -294,11 +319,11 @@ def test_predicted_obs_stored(self): trace = numpyro.handlers.trace( lambda: process.sample( infections=infections, + times=times, subpop_indices=subpop_indices, sensor_indices=sensor_indices, - times=times, - obs=None, n_sensors=2, + obs=None, ) ).get_trace() From 2a6c02f7ccc9fda70fc3b0b6bad2af939e75f1c8 Mon Sep 17 00:00:00 2001 From: Mitzi Morris Date: Sun, 1 Feb 2026 14:54:42 -0500 Subject: [PATCH 58/81] changes per coderabbit review --- docs/tutorials/.pages | 6 ++---- docs/tutorials/building_multisignal_models.qmd | 2 +- docs/tutorials/latent_hierarchical_infections.qmd | 14 +++++++------- pyrenew/datasets/hospital_admissions.py | 2 +- pyrenew/latent/temporal_processes.py | 6 +++--- 5 files changed, 14 insertions(+), 16 deletions(-) diff --git a/docs/tutorials/.pages b/docs/tutorials/.pages index acb9ee8b..81b29109 100644 --- a/docs/tutorials/.pages +++ b/docs/tutorials/.pages @@ -1,13 +1,11 @@ nav: - getting_started.md - - basic_renewal_model.md - - custom_randomvariables.md - building_multisignal_models.md - observation_processes_counts.md - observation_processes_measurements.md - latent_hierarchical_infections.md + - custom_randomvariables.md + - basic_renewal_model.md - hospital_admissions_model.md - - observation_processes_counts.md - - observation_processes_measurements.md - day_of_the_week.md - periodic_effects.md diff --git a/docs/tutorials/building_multisignal_models.qmd b/docs/tutorials/building_multisignal_models.qmd index db2821c5..d0788911 100644 --- a/docs/tutorials/building_multisignal_models.qmd +++ b/docs/tutorials/building_multisignal_models.qmd @@ -781,7 +781,7 @@ We check that the chains have converged and the number of effective samples. ```{python} # | label: arviz-diagnostics-90 # ArviZ diagnostics for 90-day fit -def filter_samples_for_arviz(samples, n_init, num_chains=2): +def filter_samples_for_arviz(samples, n_init, num_chains=4): """Slice time-series samples to exclude initialization period and reshape for chains.""" num_samples_per_chain = len(list(samples.values())[0]) // num_chains filtered = {} diff --git a/docs/tutorials/latent_hierarchical_infections.qmd b/docs/tutorials/latent_hierarchical_infections.qmd index c25c55fd..5d756c25 100644 --- a/docs/tutorials/latent_hierarchical_infections.qmd +++ b/docs/tutorials/latent_hierarchical_infections.qmd @@ -295,7 +295,7 @@ print(f"I0 = 0.001 means {0.001:.1%} of population initially infected") print(f"initial_log_rt = 0.0 means Rt starts at {np.exp(0.0):.1f}") ``` -In the previous plot, `I0` was specified as: `I0_rv=DeterministicVariable("I0", 0.001)`, i.e., $1%$ of the population. We overplot trajectories for $10%$ and $0.1%$ as well. +In the previous plot, `I0` was specified as: `I0_rv=DeterministicVariable("I0", 0.001)`, i.e., $0.1%$ of the population. We overplot trajectories for $10%$ and $0.01%$ as well. ```{python} # | label: fig-i0-effect @@ -864,11 +864,6 @@ print( | Temporal process | AR1 for regularization; RandomWalk for flexibility | | `autoreg`, `innovation_sd` | Lower values = tighter prior; adjust if trajectories explode | -### Next Steps - -- **`model_architecture.qmd`**: Learn how to combine `HierarchicalInfections` with observation processes using `ModelBuilder` -- **Observation process tutorials**: `observation_processes_counts.qmd` and `observation_processes_measurements.qmd` - ### Code Summary ```python @@ -892,4 +887,9 @@ infections_agg, infections_all, infections_obs, infections_unobs = latent.sample ) ``` -In a full model, you would pass `HierarchicalInfections` to `ModelBuilder` along with observation processes—the builder handles initialization, routing, and assembly automatically. +When building the full multisignal model, you would instantiate a `ModelBuilder` object +and call the `configure_latent` method +to specify the latent process model structure: generation interval, initial infections, and temporal dynamics. +See tutorial: + +- [Building Multi-Signal Renewal Models](building_multisignal_models.qmd) diff --git a/pyrenew/datasets/hospital_admissions.py b/pyrenew/datasets/hospital_admissions.py index 046fbcb1..5cb01b58 100644 --- a/pyrenew/datasets/hospital_admissions.py +++ b/pyrenew/datasets/hospital_admissions.py @@ -33,7 +33,7 @@ def load_hospital_data_for_state( - daily_admits: JAX array of daily hospital admissions - population: Population size (scalar) - - dates: List of date strings + - dates: List of datetime.date objects - n_days: Number of days Notes diff --git a/pyrenew/latent/temporal_processes.py b/pyrenew/latent/temporal_processes.py index 3854b0a2..a8621de8 100644 --- a/pyrenew/latent/temporal_processes.py +++ b/pyrenew/latent/temporal_processes.py @@ -405,7 +405,7 @@ def _sample_single( n=n_timepoints + 1, init_vals=jnp.array([[initial_value]]), autoreg=jnp.array([[self.autoreg]]), - noise_sd=jnp.array([[innovation_sd]]), + noise_sd=innovation_sd, fundamental_process_init_vals=jnp.array([[init_rate_of_change]]), noise_name=f"{name_prefix}_noise", ) @@ -445,12 +445,12 @@ def _sample_vectorized( n=n_timepoints + 1, init_vals=initial_values[jnp.newaxis, :], autoreg=jnp.full((1, n_processes), self.autoreg), - noise_sd=jnp.full((1, n_processes), innovation_sd), + noise_sd=innovation_sd, fundamental_process_init_vals=init_rates[jnp.newaxis, :], noise_name=f"{name_prefix}_noise", ) - return trajectories[:n_timepoints, :].squeeze(axis=1) + return trajectories[:n_timepoints, :] class RandomWalk(TemporalProcess): From 7bb4dff2be6f718f23421192249493dcd02b69a2 Mon Sep 17 00:00:00 2001 From: Mitzi Morris Date: Mon, 2 Feb 2026 15:41:10 -0500 Subject: [PATCH 59/81] changes per code review --- .../tutorials/building_multisignal_models.qmd | 1 - pyrenew/latent/temporal_processes.py | 69 +++++++++---------- pyrenew/model/__init__.py | 2 +- pyrenew/model/model_builder.py | 4 +- pyrenew/model/multisignal_model.py | 9 ++- test/test_hierarchical_infections.py | 2 +- test/test_model_builder.py | 4 +- test/test_temporal_processes.py | 33 --------- 8 files changed, 45 insertions(+), 79 deletions(-) diff --git a/docs/tutorials/building_multisignal_models.qmd b/docs/tutorials/building_multisignal_models.qmd index d0788911..66379c23 100644 --- a/docs/tutorials/building_multisignal_models.qmd +++ b/docs/tutorials/building_multisignal_models.qmd @@ -53,7 +53,6 @@ def make_rng_key(): ```{python} # | label: imports-pyrenew -import jax from jax.typing import ArrayLike from pyrenew import datasets diff --git a/pyrenew/latent/temporal_processes.py b/pyrenew/latent/temporal_processes.py index a8621de8..25562931 100644 --- a/pyrenew/latent/temporal_processes.py +++ b/pyrenew/latent/temporal_processes.py @@ -32,7 +32,7 @@ - Automatic via ``n_processes`` parameter * - Validation - Minimal constraints - - Enforces stationarity (``|autoreg| < 1``) + - Validates positive innovation_sd **When to use which:** @@ -60,7 +60,7 @@ from __future__ import annotations -from typing import Optional, Protocol, Union, runtime_checkable +from typing import Protocol, runtime_checkable import jax.numpy as jnp import numpyro @@ -85,9 +85,9 @@ class TemporalProcess(Protocol): def sample( self, n_timepoints: int, - initial_value: Optional[Union[float, ArrayLike]] = None, + initial_value: float | ArrayLike | None = None, innovation_sd: float = 1.0, - n_processes: Optional[int] = None, + n_processes: int | None = None, name_prefix: str = "temporal", ) -> ArrayLike: """ @@ -126,13 +126,13 @@ class AR1(TemporalProcess): drift away are "pulled back" over time. This class wraps :class:`pyrenew.process.ARProcess` with a simplified, - protocol-compliant interface that handles vectorization automatically - and enforces stationarity constraints. + protocol-compliant interface that handles vectorization automatically. Parameters ---------- autoreg : float - Autoregressive coefficient. Must satisfy |autoreg| < 1 for stationarity. + Autoregressive coefficient. For stationarity, |autoreg| < 1, but + this is not enforced (use priors to constrain if needed). innovation_sd : float, default 1.0 Standard deviation of noise at each time step. Larger values produce more volatile trajectories; smaller values produce smoother ones. @@ -145,20 +145,16 @@ def __init__(self, autoreg: float, innovation_sd: float = 1.0): Parameters ---------- autoreg : float - Autoregressive coefficient. Must satisfy |autoreg| < 1. + Autoregressive coefficient. For stationarity, |autoreg| < 1, + but this is not enforced (use priors to constrain if needed). innovation_sd : float, default 1.0 Standard deviation of innovations Raises ------ ValueError - If |autoreg| >= 1 or innovation_sd <= 0 + If innovation_sd <= 0 """ - if abs(autoreg) >= 1.0: - raise ValueError( - f"AR coefficient must satisfy |autoreg| < 1 for stationarity. " - f"Got autoreg={autoreg}" - ) if innovation_sd <= 0: raise ValueError(f"innovation_sd must be positive, got {innovation_sd}") self.autoreg = autoreg @@ -172,9 +168,9 @@ def __repr__(self) -> str: def sample( self, n_timepoints: int, - initial_value: Optional[Union[float, ArrayLike]] = None, - innovation_sd: Optional[float] = None, - n_processes: Optional[int] = None, + initial_value: float | ArrayLike | None = None, + innovation_sd: float | None = None, + n_processes: int | None = None, name_prefix: str = "ar1", ) -> ArrayLike: """ @@ -213,7 +209,7 @@ def sample( def _sample_single( self, n_timepoints: int, - initial_value: Optional[float], + initial_value: float | None, innovation_sd: float, name_prefix: str, ) -> ArrayLike: @@ -247,7 +243,7 @@ def _sample_vectorized( self, n_timepoints: int, n_processes: int, - initial_values: Optional[ArrayLike], + initial_values: ArrayLike | None, innovation_sd: float, name_prefix: str, ) -> ArrayLike: @@ -298,7 +294,9 @@ class DifferencedAR1(TemporalProcess): Parameters ---------- autoreg : float - Autoregressive coefficient for differences. Must satisfy |autoreg| < 1. + Autoregressive coefficient for differences. For stationarity, + |autoreg| < 1, but this is not enforced (use priors to constrain + if needed). innovation_sd : float, default 1.0 Standard deviation of noise added to changes. Larger values produce more erratic growth rates; smaller values produce smoother trends. @@ -311,20 +309,17 @@ def __init__(self, autoreg: float, innovation_sd: float = 1.0): Parameters ---------- autoreg : float - Autoregressive coefficient for differences. Must satisfy |autoreg| < 1. + Autoregressive coefficient for differences. For stationarity, + |autoreg| < 1, but this is not enforced (use priors to constrain + if needed). innovation_sd : float, default 1.0 Standard deviation of innovations Raises ------ ValueError - If |autoreg| >= 1 or innovation_sd <= 0 + If innovation_sd <= 0 """ - if abs(autoreg) >= 1.0: - raise ValueError( - f"AR coefficient must satisfy |autoreg| < 1 for stationarity. " - f"Got autoreg={autoreg}" - ) if innovation_sd <= 0: raise ValueError(f"innovation_sd must be positive, got {innovation_sd}") self.autoreg = autoreg @@ -340,9 +335,9 @@ def __repr__(self) -> str: def sample( self, n_timepoints: int, - initial_value: Optional[Union[float, ArrayLike]] = None, - innovation_sd: Optional[float] = None, - n_processes: Optional[int] = None, + initial_value: float | ArrayLike | None = None, + innovation_sd: float | None = None, + n_processes: int | None = None, name_prefix: str = "diff_ar1", ) -> ArrayLike: """ @@ -381,7 +376,7 @@ def sample( def _sample_single( self, n_timepoints: int, - initial_value: Optional[float], + initial_value: float | None, innovation_sd: float, name_prefix: str, ) -> ArrayLike: @@ -416,7 +411,7 @@ def _sample_vectorized( self, n_timepoints: int, n_processes: int, - initial_values: Optional[ArrayLike], + initial_values: ArrayLike | None, innovation_sd: float, name_prefix: str, ) -> ArrayLike: @@ -503,9 +498,9 @@ def __repr__(self) -> str: def sample( self, n_timepoints: int, - initial_value: Optional[Union[float, ArrayLike]] = None, - innovation_sd: Optional[float] = None, - n_processes: Optional[int] = None, + initial_value: float | ArrayLike | None = None, + innovation_sd: float | None = None, + n_processes: int | None = None, name_prefix: str = "rw", ) -> ArrayLike: """ @@ -544,7 +539,7 @@ def sample( def _sample_single( self, n_timepoints: int, - initial_value: Optional[float], + initial_value: float | None, innovation_sd: float, name_prefix: str, ) -> ArrayLike: @@ -577,7 +572,7 @@ def _sample_vectorized( self, n_timepoints: int, n_processes: int, - initial_values: Optional[ArrayLike], + initial_values: ArrayLike | None, innovation_sd: float, name_prefix: str, ) -> ArrayLike: diff --git a/pyrenew/model/__init__.py b/pyrenew/model/__init__.py index 9f34582e..0321e5f1 100644 --- a/pyrenew/model/__init__.py +++ b/pyrenew/model/__init__.py @@ -6,8 +6,8 @@ from pyrenew.model.rtinfectionsrenewalmodel import RtInfectionsRenewalModel __all__ = [ - "RtInfectionsRenewalModel", "HospitalAdmissionsModel", "ModelBuilder", "MultiSignalModel", + "RtInfectionsRenewalModel", ] diff --git a/pyrenew/model/model_builder.py b/pyrenew/model/model_builder.py index eb88683a..28897c1d 100644 --- a/pyrenew/model/model_builder.py +++ b/pyrenew/model/model_builder.py @@ -196,7 +196,7 @@ def compute_n_initialization_points(self) -> int: raise ValueError( f"Observation process '{name}' must implement " f"lookback_days(). Error: {e}" - ) + ) from e # Formula: max(all lookbacks) # For generation interval (1-indexed): L-element PMF has max lag L days → need L init points @@ -247,7 +247,7 @@ def build(self): f"Error constructing {self.latent_class.__name__} with " f"computed n_initialization_points={n_init}. " f"Original error: {e}" - ) + ) from e # Build model model = MultiSignalModel( diff --git a/pyrenew/model/multisignal_model.py b/pyrenew/model/multisignal_model.py index 219a3109..1018e0bd 100644 --- a/pyrenew/model/multisignal_model.py +++ b/pyrenew/model/multisignal_model.py @@ -6,7 +6,7 @@ from __future__ import annotations -from typing import Any, Dict, Optional +from typing import Any, Dict import jax.numpy as jnp import jax.random as random @@ -329,6 +329,11 @@ def sample( for name, obs_process in self.observations.items(): # Get the appropriate latent infections based on observation type resolution = obs_process.infection_resolution() + if resolution not in latent_map: + raise ValueError( + f"Observation '{name}' returned invalid infection_resolution " + f"'{resolution}'. Expected 'aggregate' or 'subpop'." + ) latent_infections = latent_map[resolution] # Get observation-specific data @@ -354,7 +359,7 @@ def fit( num_samples: int = 500, num_chains: int = 1, rng_key: random.PRNGKey = None, - reparam_config: Optional[Dict[str, Any]] = None, + reparam_config: Dict[str, Any] | None = None, progress_bar: bool = True, **observation_data, ): diff --git a/test/test_hierarchical_infections.py b/test/test_hierarchical_infections.py index 82790235..385a2d63 100644 --- a/test/test_hierarchical_infections.py +++ b/test/test_hierarchical_infections.py @@ -209,7 +209,7 @@ def test_per_subpop_I0_array(self, gen_int_rv): unobs_fractions=jnp.array([0.45]), ) - inf_juris, inf_all, inf_obs, inf_unobs = result + inf_juris, inf_all, _inf_obs, _inf_unobs = result n_total = process.n_initialization_points + 30 assert inf_juris.shape == (n_total,) diff --git a/test/test_model_builder.py b/test/test_model_builder.py index 8ff08eb9..4fc4ad4e 100644 --- a/test/test_model_builder.py +++ b/test/test_model_builder.py @@ -172,7 +172,7 @@ def test_sample_with_population_structure(self, simple_builder): n_total = model.latent.n_initialization_points + n_days with numpyro.handlers.seed(rng_seed=42): - inf_aggregate, inf_all, inf_obs, inf_unobs = model.sample( + inf_aggregate, inf_all, inf_obs, _inf_unobs = model.sample( n_days_post_init=n_days, population_size=1_000_000, obs_fractions=OBS_FRACTIONS, @@ -311,7 +311,7 @@ def test_validate_data_rejects_mismatched_obs_times_length(self, simple_builder) """Test that mismatched obs and times lengths raises error.""" model = simple_builder.build() - with pytest.raises(ValueError, match="obs length.*must match times length"): + with pytest.raises(ValueError, match=r"obs length.*must match times length"): model.validate_data( n_days_post_init=30, obs_fractions=OBS_FRACTIONS, diff --git a/test/test_temporal_processes.py b/test/test_temporal_processes.py index 7c9b585d..0c90863f 100644 --- a/test/test_temporal_processes.py +++ b/test/test_temporal_processes.py @@ -36,39 +36,6 @@ def test_random_walk_repr(self): assert "innovation_sd=0.2" in repr_str -class TestAR1Validation: - """Test AR1 validation.""" - - def test_ar1_rejects_autoreg_equal_one(self): - """Test that AR1 rejects autoreg=1.""" - with pytest.raises(ValueError, match="must satisfy"): - AR1(autoreg=1.0) - - def test_ar1_rejects_autoreg_negative_one(self): - """Test that AR1 rejects autoreg=-1.""" - with pytest.raises(ValueError, match="must satisfy"): - AR1(autoreg=-1.0) - - def test_ar1_rejects_autoreg_greater_than_one(self): - """Test that AR1 rejects autoreg > 1.""" - with pytest.raises(ValueError, match="must satisfy"): - AR1(autoreg=1.5) - - -class TestDifferencedAR1Validation: - """Test DifferencedAR1 validation.""" - - def test_differenced_ar1_rejects_autoreg_equal_one(self): - """Test that DifferencedAR1 rejects autoreg=1.""" - with pytest.raises(ValueError, match="must satisfy"): - DifferencedAR1(autoreg=1.0) - - def test_differenced_ar1_rejects_autoreg_negative_one(self): - """Test that DifferencedAR1 rejects autoreg=-1.""" - with pytest.raises(ValueError, match="must satisfy"): - DifferencedAR1(autoreg=-1.0) - - class TestAR1VectorizedSampling: """Test AR1 vectorized sampling.""" From 6083be162e54559374c79161701a0dca00764ffa Mon Sep 17 00:00:00 2001 From: Mitzi Morris Date: Wed, 4 Feb 2026 14:14:31 -0500 Subject: [PATCH 60/81] Update pyrenew/observation/count_observations.py Co-authored-by: Dylan H. Morris --- pyrenew/observation/count_observations.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyrenew/observation/count_observations.py b/pyrenew/observation/count_observations.py index 090970a7..b80d2734 100644 --- a/pyrenew/observation/count_observations.py +++ b/pyrenew/observation/count_observations.py @@ -223,7 +223,7 @@ def sample( safe_predicted = jnp.where(jnp.isnan(predicted_counts), 1.0, predicted_counts) safe_obs = None if obs is not None: - safe_obs = jnp.where(jnp.isnan(obs), 0.0, obs) + safe_obs = jnp.where(jnp.isnan(obs), safe_predicted, obs) observed = self.noise.sample( name=self._sample_site_name("obs"), From 9ed3cd16a4a7decaac74aa1df2a9f19b41d474e4 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 4 Feb 2026 19:14:49 +0000 Subject: [PATCH 61/81] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- pyrenew/model/model_builder.py | 14 +++++++------- pyrenew/model/multisignal_model.py | 8 ++++---- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/pyrenew/model/model_builder.py b/pyrenew/model/model_builder.py index 28897c1d..ff8c9686 100644 --- a/pyrenew/model/model_builder.py +++ b/pyrenew/model/model_builder.py @@ -7,7 +7,7 @@ from __future__ import annotations -from typing import Any, Dict, Type +from typing import Any from pyrenew.latent.base import BaseLatentInfectionProcess from pyrenew.model.multisignal_model import MultiSignalModel @@ -50,15 +50,15 @@ def __init__(self): 2. add_observation() - add one or more observation processes 3. build() - construct the final model """ - self.latent_class: Type[BaseLatentInfectionProcess] | None = None - self.latent_params: Dict[str, Any] = {} - self.observations: Dict[str, BaseObservationProcess] = {} + self.latent_class: type[BaseLatentInfectionProcess] | None = None + self.latent_params: dict[str, Any] = {} + self.observations: dict[str, BaseObservationProcess] = {} def configure_latent( self, - latent_class: Type[BaseLatentInfectionProcess], + latent_class: type[BaseLatentInfectionProcess], **params, - ) -> "ModelBuilder": + ) -> ModelBuilder: """ Configure the latent infection process. @@ -116,7 +116,7 @@ def configure_latent( def add_observation( self, obs_process: BaseObservationProcess, - ) -> "ModelBuilder": + ) -> ModelBuilder: """ Add an observation process to the model. diff --git a/pyrenew/model/multisignal_model.py b/pyrenew/model/multisignal_model.py index 1018e0bd..99562b8d 100644 --- a/pyrenew/model/multisignal_model.py +++ b/pyrenew/model/multisignal_model.py @@ -6,7 +6,7 @@ from __future__ import annotations -from typing import Any, Dict +from typing import Any import jax.numpy as jnp import jax.random as random @@ -51,7 +51,7 @@ class MultiSignalModel(Model): def __init__( self, latent_process: BaseLatentInfectionProcess, - observations: Dict[str, BaseObservationProcess], + observations: dict[str, BaseObservationProcess], ): """ Initialize multi-signal model. @@ -166,7 +166,7 @@ def validate_data( n_days_post_init: int, obs_fractions=None, unobs_fractions=None, - **observation_data: Dict[str, Any], + **observation_data: dict[str, Any], ) -> None: """ Validate observation data before running MCMC. @@ -359,7 +359,7 @@ def fit( num_samples: int = 500, num_chains: int = 1, rng_key: random.PRNGKey = None, - reparam_config: Dict[str, Any] | None = None, + reparam_config: dict[str, Any] | None = None, progress_bar: bool = True, **observation_data, ): From 7812560d85b02c776e400518301409069ace7438 Mon Sep 17 00:00:00 2001 From: Mitzi Morris Date: Wed, 4 Feb 2026 14:19:07 -0500 Subject: [PATCH 62/81] Update pyrenew/latent/temporal_processes.py Co-authored-by: Dylan H. Morris --- pyrenew/latent/temporal_processes.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyrenew/latent/temporal_processes.py b/pyrenew/latent/temporal_processes.py index 25562931..4d9a9841 100644 --- a/pyrenew/latent/temporal_processes.py +++ b/pyrenew/latent/temporal_processes.py @@ -281,7 +281,7 @@ def _sample_vectorized( class DifferencedAR1(TemporalProcess): """ - AR(1) process on first differences of log(Rt). + AR(1) process on first differences. Each *change* in value depends on the previous change plus noise, with the rate of change reverting toward a mean. Unlike AR(1), this allows From 5eaa6abffeefb3a8ead16ca2687611abbea46f8b Mon Sep 17 00:00:00 2001 From: Mitzi Morris Date: Wed, 4 Feb 2026 14:19:26 -0500 Subject: [PATCH 63/81] Update pyrenew/latent/temporal_processes.py Co-authored-by: Dylan H. Morris --- pyrenew/latent/temporal_processes.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyrenew/latent/temporal_processes.py b/pyrenew/latent/temporal_processes.py index 4d9a9841..84ba2da5 100644 --- a/pyrenew/latent/temporal_processes.py +++ b/pyrenew/latent/temporal_processes.py @@ -119,7 +119,7 @@ def sample( class AR1(TemporalProcess): """ - AR(1) autoregressive process for log(Rt). + AR(1) process. Each value depends on the previous value plus noise, with reversion toward a mean level. Keeps Rt bounded near a baseline — values that From cbffbe5620a969c7a148316fca25dbc3cd3d0cb5 Mon Sep 17 00:00:00 2001 From: Mitzi Morris Date: Wed, 4 Feb 2026 15:33:14 -0500 Subject: [PATCH 64/81] changes per code review --- pyrenew/latent/temporal_processes.py | 81 +++++++---------------- pyrenew/observation/count_observations.py | 12 ++-- pyrenew/observation/noise.py | 22 +++--- test/test_temporal_processes.py | 25 ------- 4 files changed, 41 insertions(+), 99 deletions(-) diff --git a/pyrenew/latent/temporal_processes.py b/pyrenew/latent/temporal_processes.py index 84ba2da5..b423c4f9 100644 --- a/pyrenew/latent/temporal_processes.py +++ b/pyrenew/latent/temporal_processes.py @@ -86,7 +86,6 @@ def sample( self, n_timepoints: int, initial_value: float | ArrayLike | None = None, - innovation_sd: float = 1.0, n_processes: int | None = None, name_prefix: str = "temporal", ) -> ArrayLike: @@ -101,8 +100,6 @@ def sample( Initial value(s) for the process(es). - If n_processes is None: scalar initial value (default: 0.0) - If n_processes is K: scalar (broadcast) or array of shape (K,) - innovation_sd : float, default 1.0 - Standard deviation of innovations n_processes : int, optional Number of parallel processes. If None, samples a single trajectory. name_prefix : str, default "temporal" @@ -169,7 +166,6 @@ def sample( self, n_timepoints: int, initial_value: float | ArrayLike | None = None, - innovation_sd: float | None = None, n_processes: int | None = None, name_prefix: str = "ar1", ) -> ArrayLike: @@ -182,8 +178,6 @@ def sample( Number of time points to generate initial_value : float or ArrayLike, optional Initial value(s). Defaults to 0.0. - innovation_sd : float, optional - Override constructor's innovation_sd if provided. n_processes : int, optional Number of parallel processes. If None, samples single trajectory. name_prefix : str, default "ar1" @@ -194,23 +188,17 @@ def sample( ArrayLike Trajectory (n_timepoints,) or trajectories (n_timepoints, K) """ - if innovation_sd is None: - innovation_sd = self.innovation_sd - if n_processes is None: - return self._sample_single( - n_timepoints, initial_value, innovation_sd, name_prefix - ) + return self._sample_single(n_timepoints, initial_value, name_prefix) else: return self._sample_vectorized( - n_timepoints, n_processes, initial_value, innovation_sd, name_prefix + n_timepoints, n_processes, initial_value, name_prefix ) def _sample_single( self, n_timepoints: int, initial_value: float | None, - innovation_sd: float, name_prefix: str, ) -> ArrayLike: """ @@ -224,27 +212,26 @@ def _sample_single( if initial_value is None: initial_value = 0.0 - stationary_sd = innovation_sd / jnp.sqrt(1 - self.autoreg**2) + stationary_sd = self.innovation_sd / jnp.sqrt(1 - self.autoreg**2) init_state = numpyro.sample( f"{name_prefix}_init", dist.Normal(initial_value, stationary_sd) ) trajectory = self.ar_process( n=n_timepoints, - init_vals=jnp.array([[init_state]]), - autoreg=jnp.array([[self.autoreg]]), - noise_sd=innovation_sd, + init_vals=jnp.array([init_state]), + autoreg=jnp.array([self.autoreg]), + noise_sd=self.innovation_sd, noise_name=f"{name_prefix}_noise", ) - return jnp.squeeze(trajectory) + return trajectory def _sample_vectorized( self, n_timepoints: int, n_processes: int, initial_values: ArrayLike | None, - innovation_sd: float, name_prefix: str, ) -> ArrayLike: """ @@ -260,7 +247,7 @@ def _sample_vectorized( elif jnp.isscalar(initial_values): initial_values = jnp.full(n_processes, initial_values) - stationary_sd = innovation_sd / jnp.sqrt(1 - self.autoreg**2) + stationary_sd = self.innovation_sd / jnp.sqrt(1 - self.autoreg**2) with numpyro.plate(f"{name_prefix}_init_plate", n_processes): init_states = numpyro.sample( @@ -272,7 +259,7 @@ def _sample_vectorized( n=n_timepoints, init_vals=init_states[jnp.newaxis, :], autoreg=jnp.full((1, n_processes), self.autoreg), - noise_sd=innovation_sd, + noise_sd=self.innovation_sd, noise_name=f"{name_prefix}_noise", ) @@ -336,7 +323,6 @@ def sample( self, n_timepoints: int, initial_value: float | ArrayLike | None = None, - innovation_sd: float | None = None, n_processes: int | None = None, name_prefix: str = "diff_ar1", ) -> ArrayLike: @@ -349,8 +335,6 @@ def sample( Number of time points to generate initial_value : float or ArrayLike, optional Initial value(s). Defaults to 0.0. - innovation_sd : float, optional - Override constructor's innovation_sd if provided. n_processes : int, optional Number of parallel processes. If None, samples single trajectory. name_prefix : str, default "diff_ar1" @@ -361,23 +345,17 @@ def sample( ArrayLike Trajectory (n_timepoints,) or trajectories (n_timepoints, K) """ - if innovation_sd is None: - innovation_sd = self.innovation_sd - if n_processes is None: - return self._sample_single( - n_timepoints, initial_value, innovation_sd, name_prefix - ) + return self._sample_single(n_timepoints, initial_value, name_prefix) else: return self._sample_vectorized( - n_timepoints, n_processes, initial_value, innovation_sd, name_prefix + n_timepoints, n_processes, initial_value, name_prefix ) def _sample_single( self, n_timepoints: int, initial_value: float | None, - innovation_sd: float, name_prefix: str, ) -> ArrayLike: """ @@ -391,28 +369,27 @@ def _sample_single( if initial_value is None: initial_value = 0.0 - stationary_sd = innovation_sd / jnp.sqrt(1 - self.autoreg**2) + stationary_sd = self.innovation_sd / jnp.sqrt(1 - self.autoreg**2) init_rate_of_change = numpyro.sample( f"{name_prefix}_init_rate", dist.Normal(0, stationary_sd) ) trajectory = self.process( n=n_timepoints + 1, - init_vals=jnp.array([[initial_value]]), - autoreg=jnp.array([[self.autoreg]]), - noise_sd=innovation_sd, - fundamental_process_init_vals=jnp.array([[init_rate_of_change]]), + init_vals=jnp.array([initial_value]), + autoreg=jnp.array([self.autoreg]), + noise_sd=self.innovation_sd, + fundamental_process_init_vals=jnp.array([init_rate_of_change]), noise_name=f"{name_prefix}_noise", ) - return jnp.squeeze(trajectory[:n_timepoints]) + return trajectory[:n_timepoints] def _sample_vectorized( self, n_timepoints: int, n_processes: int, initial_values: ArrayLike | None, - innovation_sd: float, name_prefix: str, ) -> ArrayLike: """ @@ -428,7 +405,7 @@ def _sample_vectorized( elif jnp.isscalar(initial_values): initial_values = jnp.full(n_processes, initial_values) - stationary_sd = innovation_sd / jnp.sqrt(1 - self.autoreg**2) + stationary_sd = self.innovation_sd / jnp.sqrt(1 - self.autoreg**2) with numpyro.plate(f"{name_prefix}_init_rate_plate", n_processes): init_rates = numpyro.sample( @@ -440,7 +417,7 @@ def _sample_vectorized( n=n_timepoints + 1, init_vals=initial_values[jnp.newaxis, :], autoreg=jnp.full((1, n_processes), self.autoreg), - noise_sd=innovation_sd, + noise_sd=self.innovation_sd, fundamental_process_init_vals=init_rates[jnp.newaxis, :], noise_name=f"{name_prefix}_noise", ) @@ -499,7 +476,6 @@ def sample( self, n_timepoints: int, initial_value: float | ArrayLike | None = None, - innovation_sd: float | None = None, n_processes: int | None = None, name_prefix: str = "rw", ) -> ArrayLike: @@ -512,8 +488,6 @@ def sample( Number of time points to generate initial_value : float or ArrayLike, optional Initial value(s). Defaults to 0.0. - innovation_sd : float, optional - Override constructor's innovation_sd if provided. n_processes : int, optional Number of parallel processes. If None, samples single trajectory. name_prefix : str, default "rw" @@ -524,23 +498,17 @@ def sample( ArrayLike Trajectory (n_timepoints,) or trajectories (n_timepoints, K) """ - if innovation_sd is None: - innovation_sd = self.innovation_sd - if n_processes is None: - return self._sample_single( - n_timepoints, initial_value, innovation_sd, name_prefix - ) + return self._sample_single(n_timepoints, initial_value, name_prefix) else: return self._sample_vectorized( - n_timepoints, n_processes, initial_value, innovation_sd, name_prefix + n_timepoints, n_processes, initial_value, name_prefix ) def _sample_single( self, n_timepoints: int, initial_value: float | None, - innovation_sd: float, name_prefix: str, ) -> ArrayLike: """ @@ -556,7 +524,7 @@ def _sample_single( step_rv = DistributionalVariable( name=f"{name_prefix}_step", - distribution=dist.Normal(0, innovation_sd), + distribution=dist.Normal(0, self.innovation_sd), ) rw = PyRenewRandomWalk(step_rv=step_rv) @@ -566,14 +534,13 @@ def _sample_single( n=n_timepoints, ) - return jnp.squeeze(trajectory) + return trajectory def _sample_vectorized( self, n_timepoints: int, n_processes: int, initial_values: ArrayLike | None, - innovation_sd: float, name_prefix: str, ) -> ArrayLike: """ @@ -600,7 +567,7 @@ def _sample_vectorized( # Transpose: (n_processes, n_timepoints-1) -> (n_timepoints-1, n_processes) increments = numpyro.deterministic( f"{name_prefix}_increments", - (increments_raw * innovation_sd).T, + (increments_raw * self.innovation_sd).T, ) cumulative = jnp.cumsum(increments, axis=0) diff --git a/pyrenew/observation/count_observations.py b/pyrenew/observation/count_observations.py index b80d2734..80912143 100644 --- a/pyrenew/observation/count_observations.py +++ b/pyrenew/observation/count_observations.py @@ -208,18 +208,18 @@ def sample( # Compute mask: True where observation contributes to likelihood. # NaN in predictions (initialization period) or obs (missing data) - # are excluded via obs_mask. + # are excluded via mask. valid_pred = ~jnp.isnan(predicted_counts) if obs is not None: valid_obs = ~jnp.isnan(obs) - obs_mask = valid_pred & valid_obs + mask = valid_pred & valid_obs else: - obs_mask = valid_pred + mask = valid_pred - # JAX evaluates log_prob for all array elements even when obs_mask + # JAX evaluates log_prob for all array elements even when mask # excludes them from the likelihood sum. Replace NaN with safe values # to avoid NaN propagation in JAX's computation graph. These values - # do not affect inference since obs_mask=False excludes them. + # do not affect inference since mask=False excludes them. safe_predicted = jnp.where(jnp.isnan(predicted_counts), 1.0, predicted_counts) safe_obs = None if obs is not None: @@ -229,7 +229,7 @@ def sample( name=self._sample_site_name("obs"), predicted=safe_predicted, obs=safe_obs, - obs_mask=obs_mask, + mask=mask, ) return ObservationSample(observed=observed, predicted=predicted_counts) diff --git a/pyrenew/observation/noise.py b/pyrenew/observation/noise.py index 2f86d416..15c8c0ad 100644 --- a/pyrenew/observation/noise.py +++ b/pyrenew/observation/noise.py @@ -100,7 +100,7 @@ def sample( name: str, predicted: ArrayLike, obs: ArrayLike | None = None, - obs_mask: ArrayLike | None = None, + mask: ArrayLike | None = None, ) -> ArrayLike: """ Sample count observations given predicted counts. @@ -113,7 +113,7 @@ def sample( Predicted count values (non-negative). obs : ArrayLike | None Observed counts for conditioning, or None for prior sampling. - obs_mask : ArrayLike | None + mask : ArrayLike | None Boolean mask indicating which observations to include in the likelihood. If None, all observations are included. If provided, observations where mask is False are excluded from the likelihood. @@ -126,7 +126,7 @@ def sample( Notes ----- Implementations use ``numpyro.handlers.mask`` rather than the - ``obs_mask`` parameter of ``numpyro.sample``. This avoids creating + ``mask`` parameter of ``numpyro.sample``. This avoids creating latent variables for masked entries, which would fail with NUTS for discrete distributions. """ @@ -167,7 +167,7 @@ def sample( name: str, predicted: ArrayLike, obs: ArrayLike | None = None, - obs_mask: ArrayLike | None = None, + mask: ArrayLike | None = None, ) -> ArrayLike: """ Sample from Poisson distribution. @@ -180,7 +180,7 @@ def sample( Predicted count values. obs : ArrayLike | None Observed counts for conditioning. - obs_mask : ArrayLike | None + mask : ArrayLike | None Boolean mask indicating which observations to include in the likelihood. If None, all observations are included. @@ -189,13 +189,13 @@ def sample( ArrayLike Poisson-distributed counts. """ - if obs_mask is None: + if mask is None: return numpyro.sample( name, dist.Poisson(rate=predicted + _EPSILON), obs=obs, ) - with numpyro.handlers.mask(mask=obs_mask): + with numpyro.handlers.mask(mask=mask): return numpyro.sample( name, dist.Poisson(rate=predicted + _EPSILON), @@ -258,7 +258,7 @@ def sample( name: str, predicted: ArrayLike, obs: ArrayLike | None = None, - obs_mask: ArrayLike | None = None, + mask: ArrayLike | None = None, ) -> ArrayLike: """ Sample from Negative Binomial distribution. @@ -271,7 +271,7 @@ def sample( Predicted count values. obs : ArrayLike | None Observed counts for conditioning. - obs_mask : ArrayLike | None + mask : ArrayLike | None Boolean mask indicating which observations to include in the likelihood. If None, all observations are included. @@ -281,7 +281,7 @@ def sample( Negative Binomial-distributed counts. """ concentration = self.concentration_rv() - if obs_mask is None: + if mask is None: return numpyro.sample( name, dist.NegativeBinomial2( @@ -290,7 +290,7 @@ def sample( ), obs=obs, ) - with numpyro.handlers.mask(mask=obs_mask): + with numpyro.handlers.mask(mask=mask): return numpyro.sample( name, dist.NegativeBinomial2( diff --git a/test/test_temporal_processes.py b/test/test_temporal_processes.py index 0c90863f..8763d9cb 100644 --- a/test/test_temporal_processes.py +++ b/test/test_temporal_processes.py @@ -164,18 +164,6 @@ def test_differenced_ar1_vectorized_with_scalar_initial_value(self): assert trajectories.shape == (n_timepoints, n_processes) - def test_differenced_ar1_override_innovation_sd(self): - """Test DifferencedAR1 with overridden innovation_sd.""" - dar1 = DifferencedAR1(autoreg=0.6, innovation_sd=0.1) - - with numpyro.handlers.seed(rng_seed=42): - trajectory = dar1.sample( - n_timepoints=30, - innovation_sd=0.5, # Override constructor value - ) - - assert trajectory.shape == (30,) - class TestRandomWalkVectorizedSampling: """Test RandomWalk vectorized sampling.""" @@ -283,19 +271,6 @@ def test_differenced_ar1_smaller_innovation_sd_produces_smoother_changes( assert jnp.std(diffs_small) < jnp.std(diffs_large) - def test_explicit_innovation_sd_overrides_constructor_value(self): - """Verify that explicit innovation_sd in sample() overrides constructor value.""" - rw = RandomWalk(innovation_sd=0.1) - - with numpyro.handlers.seed(rng_seed=42): - traj_constructor = rw.sample(n_timepoints=50) - - with numpyro.handlers.seed(rng_seed=42): - traj_override = rw.sample(n_timepoints=50, innovation_sd=1.0) - - # Trajectories should be substantially different - assert not jnp.allclose(traj_constructor, traj_override, rtol=0.1) - def test_vectorized_sampling_respects_innovation_sd(self): """Verify vectorized sampling uses the innovation_sd parameter.""" n_processes = 5 From 87ccf5b0f8d48ea7022deda82848c07ad906eee4 Mon Sep 17 00:00:00 2001 From: Mitzi Morris Date: Wed, 4 Feb 2026 15:58:22 -0500 Subject: [PATCH 65/81] change model_builder to pyrenew_builder --- .../tutorials/building_multisignal_models.qmd | 16 ++++----- .../latent_hierarchical_infections.qmd | 2 +- pyrenew/latent/base.py | 6 ++-- pyrenew/latent/hierarchical_infections.py | 4 +-- pyrenew/model/__init__.py | 4 +-- pyrenew/model/multisignal_model.py | 2 +- .../{model_builder.py => pyrenew_builder.py} | 2 +- ...del_builder.py => test_pyrenew_builder.py} | 34 +++++++++---------- 8 files changed, 35 insertions(+), 35 deletions(-) rename pyrenew/model/{model_builder.py => pyrenew_builder.py} (99%) rename test/{test_model_builder.py => test_pyrenew_builder.py} (96%) diff --git a/docs/tutorials/building_multisignal_models.qmd b/docs/tutorials/building_multisignal_models.qmd index 66379c23..a95be919 100644 --- a/docs/tutorials/building_multisignal_models.qmd +++ b/docs/tutorials/building_multisignal_models.qmd @@ -69,7 +69,7 @@ from pyrenew.latent import ( GammaGroupSdPrior, HierarchicalNormalPrior, ) -from pyrenew.model import ModelBuilder +from pyrenew.model import PyrenewBuilder from pyrenew.observation import ( Counts, @@ -94,7 +94,7 @@ A **multi-signal model** combines multiple observation processes—each represen - Route those infections to each observation process appropriately - Handle the initialization period required by delay distributions -The `ModelBuilder` class handles this plumbing. You specify: +The `PyrenewBuilder` class handles this plumbing. You specify: 1. A **latent process** (e.g., `HierarchicalInfections`) that defines how infections evolve 2. One or more **observation processes** (e.g., `Counts`, `Measurements`) that define how infections become data @@ -162,7 +162,7 @@ Different observation processes observe different levels of the model hierarchy. | `"aggregate"` | Aggregated infections (sum across all subpopulations) | Hospital admissions, case counts | | `"subpop"` | Infection matrix with one column per observed subpopulation | Wastewater, site-specific surveillance | -The `ModelBuilder` routes latent infections to observation processes based on each process's declared resolution. +The `PyrenewBuilder` routes latent infections to observation processes based on each process's declared resolution. With this structure in mind, we'll now define each component following the generative direction: first the latent infection process, then the observation processes. @@ -492,15 +492,15 @@ print(f" Shedding PMF length: {len(shedding_pmf)} days") ## Model Building -We instantiate a `ModelBuilder` object which handles the composition of the latent infection process and the observation process. +We instantiate a `PyrenewBuilder` object which handles the composition of the latent infection process and the observation process. ```{python} # | label: model-builder-init # Build the multi-signal model -builder = ModelBuilder() +builder = PyrenewBuilder() ``` -The `ModelBuilder` object has 3 key methods: +The `PyrenewBuilder` object has 3 key methods: - `configure_latent` - `add_observation` @@ -1084,7 +1084,7 @@ This pattern has a direct implication for forecasting: **renewal models are most ## Summary -This tutorial demonstrated composing a multi-signal renewal model using `ModelBuilder`: +This tutorial demonstrated composing a multi-signal renewal model using `PyrenewBuilder`: 1. **Configure latent process** (`configure_latent`): generation interval, initial infections, temporal dynamics 2. **Add observation processes** (`add_observation`): each declares its infection resolution and gets a name for data binding @@ -1094,7 +1094,7 @@ This tutorial demonstrated composing a multi-signal renewal model using `ModelBu - **Two-part structure**: Renewal models separate latent infection dynamics from observation processes - **Infection resolution**: Observation processes declare whether they need aggregate or subpop-level infections -- **Data routing**: `ModelBuilder` automatically routes infection trajectories to the appropriate observation processes +- **Data routing**: `PyrenewBuilder` automatically routes infection trajectories to the appropriate observation processes - **Time alignment**: Observations must be offset by `n_initialization_points` to align with model time ### Next Steps diff --git a/docs/tutorials/latent_hierarchical_infections.qmd b/docs/tutorials/latent_hierarchical_infections.qmd index 5d756c25..57a84cbb 100644 --- a/docs/tutorials/latent_hierarchical_infections.qmd +++ b/docs/tutorials/latent_hierarchical_infections.qmd @@ -887,7 +887,7 @@ infections_agg, infections_all, infections_obs, infections_unobs = latent.sample ) ``` -When building the full multisignal model, you would instantiate a `ModelBuilder` object +When building the full multisignal model, you would instantiate a `PyrenewBuilder` object and call the `configure_latent` method to specify the latent process model structure: generation interval, initial infections, and temporal dynamics. See tutorial: diff --git a/pyrenew/latent/base.py b/pyrenew/latent/base.py index a05872d7..743b3c6a 100644 --- a/pyrenew/latent/base.py +++ b/pyrenew/latent/base.py @@ -90,7 +90,7 @@ class BaseLatentInfectionProcess(RandomVariable): Generation interval PMF n_initialization_points : int, optional Number of initialization days before day 0. If not specified, automatically - computed as max(21, 2*len(gen_int)-1). The ModelBuilder will compute and pass + computed as max(21, 2*len(gen_int)-1). The PyrenewBuilder will compute and pass the correct value based on all observation processes. Notes @@ -99,7 +99,7 @@ class BaseLatentInfectionProcess(RandomVariable): method, not the constructor. This allows a single model instance to be fit to multiple datasets with different jurisdiction structures. - When using ModelBuilder, n_initialization_points is computed automatically from + When using PyrenewBuilder, n_initialization_points is computed automatically from all observation processes. When manually creating latent processes, you can either specify it explicitly or let it default to max(21, 2*len(gen_int)-1). @@ -322,7 +322,7 @@ def get_required_lookback(self) -> int: """ Return the generation interval length for builder pattern support. - This method is used by ModelBuilder to compute n_initialization_points + This method is used by PyrenewBuilder to compute n_initialization_points from all model components. Returns the generation interval PMF length. Returns diff --git a/pyrenew/latent/hierarchical_infections.py b/pyrenew/latent/hierarchical_infections.py index 53edec09..6a279a03 100644 --- a/pyrenew/latent/hierarchical_infections.py +++ b/pyrenew/latent/hierarchical_infections.py @@ -59,7 +59,7 @@ class HierarchicalInfections(BaseLatentInfectionProcess): or given a prior distribution. n_initialization_points : int, optional Number of initialization days before day 0. If not specified, defaults to - max(21, 2*len(gen_int)-1). When using ModelBuilder, this is computed + max(21, 2*len(gen_int)-1). When using PyrenewBuilder, this is computed automatically from all observation processes. Notes @@ -67,7 +67,7 @@ class HierarchicalInfections(BaseLatentInfectionProcess): Sum-to-zero constraint on deviations ensures R_baseline(t) is the geometric mean of subpopulation Rt values, providing identifiability. - When using ModelBuilder (recommended), n_initialization_points is computed + When using PyrenewBuilder (recommended), n_initialization_points is computed automatically from all observation processes and should not be specified manually. """ diff --git a/pyrenew/model/__init__.py b/pyrenew/model/__init__.py index 0321e5f1..0949cf05 100644 --- a/pyrenew/model/__init__.py +++ b/pyrenew/model/__init__.py @@ -1,13 +1,13 @@ # numpydoc ignore=GL08 from pyrenew.model.admissionsmodel import HospitalAdmissionsModel -from pyrenew.model.model_builder import ModelBuilder +from pyrenew.model.pyrenew_builder import PyrenewBuilder from pyrenew.model.multisignal_model import MultiSignalModel from pyrenew.model.rtinfectionsrenewalmodel import RtInfectionsRenewalModel __all__ = [ "HospitalAdmissionsModel", - "ModelBuilder", + "PyrenewBuilder", "MultiSignalModel", "RtInfectionsRenewalModel", ] diff --git a/pyrenew/model/multisignal_model.py b/pyrenew/model/multisignal_model.py index 99562b8d..308b0998 100644 --- a/pyrenew/model/multisignal_model.py +++ b/pyrenew/model/multisignal_model.py @@ -27,7 +27,7 @@ class MultiSignalModel(Model): PartitionedInfections) with multiple observation processes (e.g., CountObservation, WastewaterObservation). - Built via ModelBuilder to ensure n_initialization_points is computed + Built via PyrenewBuilder to ensure n_initialization_points is computed correctly from all components. Can also be constructed manually for advanced use cases. diff --git a/pyrenew/model/model_builder.py b/pyrenew/model/pyrenew_builder.py similarity index 99% rename from pyrenew/model/model_builder.py rename to pyrenew/model/pyrenew_builder.py index ff8c9686..07747245 100644 --- a/pyrenew/model/model_builder.py +++ b/pyrenew/model/pyrenew_builder.py @@ -23,7 +23,7 @@ } -class ModelBuilder: +class PyrenewBuilder: """ Builder for multi-signal renewal models. diff --git a/test/test_model_builder.py b/test/test_pyrenew_builder.py similarity index 96% rename from test/test_model_builder.py rename to test/test_pyrenew_builder.py index 4fc4ad4e..bfe92489 100644 --- a/test/test_model_builder.py +++ b/test/test_pyrenew_builder.py @@ -1,5 +1,5 @@ """ -Tests for ModelBuilder and MultiSignalModel. +Tests for PyrenewBuilder and MultiSignalModel. """ import jax.numpy as jnp @@ -8,7 +8,7 @@ from pyrenew.deterministic import DeterministicPMF, DeterministicVariable from pyrenew.latent import HierarchicalInfections, RandomWalk -from pyrenew.model import ModelBuilder, MultiSignalModel +from pyrenew.model import PyrenewBuilder, MultiSignalModel from pyrenew.observation import Counts, NegativeBinomialNoise # Standard population structure for tests @@ -23,10 +23,10 @@ def simple_builder(): Returns ------- - ModelBuilder + PyrenewBuilder Configured model builder. """ - builder = ModelBuilder() + builder = PyrenewBuilder() gen_int = DeterministicPMF("gen_int", jnp.array([0.2, 0.5, 0.3])) builder.configure_latent( @@ -50,12 +50,12 @@ def simple_builder(): return builder -class TestModelBuilderConfiguration: - """Test ModelBuilder configuration.""" +class TestPyrenewBuilderConfiguration: + """Test PyrenewBuilder configuration.""" def test_rejects_population_structure_at_configure_time(self): """Test that population structure params are rejected at configure time.""" - builder = ModelBuilder() + builder = PyrenewBuilder() gen_int = DeterministicPMF("gen_int", jnp.array([0.2, 0.5, 0.3])) with pytest.raises(ValueError, match="Do not specify"): @@ -71,7 +71,7 @@ def test_rejects_population_structure_at_configure_time(self): def test_rejects_n_initialization_points_at_configure_time(self): """Test that n_initialization_points is rejected at configure time.""" - builder = ModelBuilder() + builder = PyrenewBuilder() gen_int = DeterministicPMF("gen_int", jnp.array([0.2, 0.5, 0.3])) with pytest.raises(ValueError, match="Do not specify n_initialization_points"): @@ -87,7 +87,7 @@ def test_rejects_n_initialization_points_at_configure_time(self): def test_rejects_reconfiguring_latent(self): """Test that configuring latent twice raises RuntimeError.""" - builder = ModelBuilder() + builder = PyrenewBuilder() gen_int = DeterministicPMF("gen_int", jnp.array([0.2, 0.5, 0.3])) builder.configure_latent( @@ -129,21 +129,21 @@ def test_build_creates_model(self, simple_builder): def test_build_without_latent_raises_error(self): """Test that build() without configure_latent raises ValueError.""" - builder = ModelBuilder() + builder = PyrenewBuilder() with pytest.raises(ValueError, match="Must call configure_latent"): builder.build() def test_compute_n_initialization_points_without_latent_raises(self): """Test that compute_n_initialization_points without latent raises.""" - builder = ModelBuilder() + builder = PyrenewBuilder() with pytest.raises(ValueError, match="Must call configure_latent"): builder.compute_n_initialization_points() def test_compute_n_initialization_points_without_gen_int_raises(self): """Test that compute_n_initialization_points without gen_int_rv raises.""" - builder = ModelBuilder() + builder = PyrenewBuilder() # Configure latent without gen_int_rv builder.latent_class = HierarchicalInfections builder.latent_params = {} # Missing gen_int_rv @@ -360,8 +360,8 @@ def test_validate_data_rejects_out_of_bounds_subpop_indices(self, simple_builder ) -class TestModelBuilderErrorHandling: - """Test ModelBuilder error handling.""" +class TestPyrenewBuilderErrorHandling: + """Test PyrenewBuilder error handling.""" def test_compute_n_init_rejects_obs_without_lookback_days(self): """Test that observation without lookback_days() raises error.""" @@ -409,7 +409,7 @@ def _predicted_obs(self, infections): """ return infections - builder = ModelBuilder() + builder = PyrenewBuilder() gen_int = DeterministicPMF("gen_int", jnp.array([0.2, 0.5, 0.3])) builder.configure_latent( @@ -429,7 +429,7 @@ def _predicted_obs(self, infections): def test_build_raises_on_construction_error(self): """Test that build() raises TypeError on latent construction failure.""" - builder = ModelBuilder() + builder = PyrenewBuilder() gen_int = DeterministicPMF("gen_int", jnp.array([0.2, 0.5, 0.3])) # Configure with an invalid parameter that will cause construction to fail @@ -505,7 +505,7 @@ def _predicted_obs(self, infections): """ return infections - builder = ModelBuilder() + builder = PyrenewBuilder() gen_int = DeterministicPMF("gen_int", jnp.array([0.2, 0.5, 0.3])) builder.configure_latent( From b921371a7e1418cb71d2d7037c26423ca0587e3d Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 4 Feb 2026 20:58:35 +0000 Subject: [PATCH 66/81] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- pyrenew/model/__init__.py | 2 +- test/test_pyrenew_builder.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pyrenew/model/__init__.py b/pyrenew/model/__init__.py index 0949cf05..302e6424 100644 --- a/pyrenew/model/__init__.py +++ b/pyrenew/model/__init__.py @@ -1,8 +1,8 @@ # numpydoc ignore=GL08 from pyrenew.model.admissionsmodel import HospitalAdmissionsModel -from pyrenew.model.pyrenew_builder import PyrenewBuilder from pyrenew.model.multisignal_model import MultiSignalModel +from pyrenew.model.pyrenew_builder import PyrenewBuilder from pyrenew.model.rtinfectionsrenewalmodel import RtInfectionsRenewalModel __all__ = [ diff --git a/test/test_pyrenew_builder.py b/test/test_pyrenew_builder.py index bfe92489..4aea16db 100644 --- a/test/test_pyrenew_builder.py +++ b/test/test_pyrenew_builder.py @@ -8,7 +8,7 @@ from pyrenew.deterministic import DeterministicPMF, DeterministicVariable from pyrenew.latent import HierarchicalInfections, RandomWalk -from pyrenew.model import PyrenewBuilder, MultiSignalModel +from pyrenew.model import MultiSignalModel, PyrenewBuilder from pyrenew.observation import Counts, NegativeBinomialNoise # Standard population structure for tests From 7a4b5153a27ac5e094bd2f879361866d92b5f305 Mon Sep 17 00:00:00 2001 From: Mitzi Morris Date: Wed, 4 Feb 2026 16:48:59 -0500 Subject: [PATCH 67/81] lint fix --- pyrenew/model/pyrenew_builder.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/pyrenew/model/pyrenew_builder.py b/pyrenew/model/pyrenew_builder.py index 07747245..635c3ce3 100644 --- a/pyrenew/model/pyrenew_builder.py +++ b/pyrenew/model/pyrenew_builder.py @@ -58,7 +58,7 @@ def configure_latent( self, latent_class: type[BaseLatentInfectionProcess], **params, - ) -> ModelBuilder: + ) -> PyrenewBuilder: """ Configure the latent infection process. @@ -77,7 +77,7 @@ def configure_latent( Returns ------- - ModelBuilder + PyrenewBuilder Self, for method chaining Raises @@ -97,7 +97,7 @@ def configure_latent( raise ValueError( "Do not specify n_initialization_points - it will be computed " "automatically from observation processes and generation interval. " - "Use ModelBuilder.build() to create the model with the correct value." + "Use PyrenewBuilder.build() to create the model with the correct value." ) # Check for population structure params that should be at sample time @@ -116,7 +116,7 @@ def configure_latent( def add_observation( self, obs_process: BaseObservationProcess, - ) -> ModelBuilder: + ) -> PyrenewBuilder: """ Add an observation process to the model. @@ -133,7 +133,7 @@ def add_observation( Returns ------- - ModelBuilder + PyrenewBuilder Self, for method chaining Raises From c8a03b74ed9f0204ca5a9aa6f68761655ab31293 Mon Sep 17 00:00:00 2001 From: Mitzi Morris Date: Thu, 5 Feb 2026 08:13:26 -0500 Subject: [PATCH 68/81] Update pyrenew/observation/noise.py Co-authored-by: Dylan H. Morris --- pyrenew/observation/noise.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyrenew/observation/noise.py b/pyrenew/observation/noise.py index 15c8c0ad..47b86838 100644 --- a/pyrenew/observation/noise.py +++ b/pyrenew/observation/noise.py @@ -126,7 +126,7 @@ def sample( Notes ----- Implementations use ``numpyro.handlers.mask`` rather than the - ``mask`` parameter of ``numpyro.sample``. This avoids creating + ``obs_mask`` parameter of ``numpyro.sample``. This avoids creating latent variables for masked entries, which would fail with NUTS for discrete distributions. """ From d981036f724b8f1ffd7209397b3438438cac9d9a Mon Sep 17 00:00:00 2001 From: Mitzi Morris Date: Thu, 5 Feb 2026 12:25:33 -0500 Subject: [PATCH 69/81] temporal processes always use 2D arrays for timepoints --- pyrenew/latent/hierarchical_infections.py | 2 +- pyrenew/latent/temporal_processes.py | 235 ++++------------------ test/test_temporal_processes.py | 16 +- 3 files changed, 44 insertions(+), 209 deletions(-) diff --git a/pyrenew/latent/hierarchical_infections.py b/pyrenew/latent/hierarchical_infections.py index 6a279a03..3a0ee946 100644 --- a/pyrenew/latent/hierarchical_infections.py +++ b/pyrenew/latent/hierarchical_infections.py @@ -213,7 +213,7 @@ def sample( mean_deviation = jnp.mean(deviations_raw, axis=1, keepdims=True) deviations = deviations_raw - mean_deviation - log_rt_subpop = log_rt_baseline[:, jnp.newaxis] + deviations + log_rt_subpop = log_rt_baseline + deviations rt_subpop = jnp.exp(log_rt_subpop) gen_int = self.gen_int_rv() diff --git a/pyrenew/latent/temporal_processes.py b/pyrenew/latent/temporal_processes.py index b423c4f9..4865b85b 100644 --- a/pyrenew/latent/temporal_processes.py +++ b/pyrenew/latent/temporal_processes.py @@ -3,9 +3,9 @@ Temporal processes for latent infection models. Provides time-series processes for modeling Rt dynamics and subpopulation -deviations in hierarchical infection models. All processes support both -single trajectory and vectorized (multiple parallel trajectories) sampling -through a unified ``TemporalProcess`` protocol. +deviations in hierarchical infection models. All processes return 2D arrays +of shape (n_timepoints, n_processes) through a unified ``TemporalProcess`` +protocol. Relationship to pyrenew.process ------------------------------- @@ -68,8 +68,6 @@ from jax.typing import ArrayLike from pyrenew.process import ARProcess, DifferencedProcess -from pyrenew.process import RandomWalk as PyRenewRandomWalk -from pyrenew.randomvariable import DistributionalVariable @runtime_checkable @@ -78,15 +76,15 @@ class TemporalProcess(Protocol): Protocol for temporal processes generating time-varying parameters. Used for jurisdiction-level Rt dynamics, subpopulation deviations, or - allocation trajectories. Supports both single trajectory and vectorized - (multiple parallel trajectories) sampling through a unified interface. + allocation trajectories. All processes return 2D arrays of shape + (n_timepoints, n_processes) for consistent handling. """ def sample( self, n_timepoints: int, initial_value: float | ArrayLike | None = None, - n_processes: int | None = None, + n_processes: int = 1, name_prefix: str = "temporal", ) -> ArrayLike: """ @@ -98,18 +96,17 @@ def sample( Number of time points to generate initial_value : float or ArrayLike, optional Initial value(s) for the process(es). - - If n_processes is None: scalar initial value (default: 0.0) - - If n_processes is K: scalar (broadcast) or array of shape (K,) - n_processes : int, optional - Number of parallel processes. If None, samples a single trajectory. + Scalar (broadcast to all processes) or array of shape (n_processes,). + Defaults to 0.0. + n_processes : int, default 1 + Number of parallel processes. name_prefix : str, default "temporal" Prefix for numpyro sample site names to avoid collisions Returns ------- ArrayLike - - If n_processes is None: trajectory of shape (n_timepoints,) - - If n_processes is K: trajectories of shape (n_timepoints, K) + Trajectories of shape (n_timepoints, n_processes) """ ... @@ -166,7 +163,7 @@ def sample( self, n_timepoints: int, initial_value: float | ArrayLike | None = None, - n_processes: int | None = None, + n_processes: int = 1, name_prefix: str = "ar1", ) -> ArrayLike: """ @@ -178,81 +175,27 @@ def sample( Number of time points to generate initial_value : float or ArrayLike, optional Initial value(s). Defaults to 0.0. - n_processes : int, optional - Number of parallel processes. If None, samples single trajectory. + n_processes : int, default 1 + Number of parallel processes. name_prefix : str, default "ar1" Prefix for numpyro sample sites Returns ------- ArrayLike - Trajectory (n_timepoints,) or trajectories (n_timepoints, K) - """ - if n_processes is None: - return self._sample_single(n_timepoints, initial_value, name_prefix) - else: - return self._sample_vectorized( - n_timepoints, n_processes, initial_value, name_prefix - ) - - def _sample_single( - self, - n_timepoints: int, - initial_value: float | None, - name_prefix: str, - ) -> ArrayLike: - """ - Sample a single AR(1) trajectory. - - Returns - ------- - ArrayLike - Trajectory of shape (n_timepoints,). + Trajectories of shape (n_timepoints, n_processes) """ if initial_value is None: - initial_value = 0.0 - - stationary_sd = self.innovation_sd / jnp.sqrt(1 - self.autoreg**2) - init_state = numpyro.sample( - f"{name_prefix}_init", dist.Normal(initial_value, stationary_sd) - ) - - trajectory = self.ar_process( - n=n_timepoints, - init_vals=jnp.array([init_state]), - autoreg=jnp.array([self.autoreg]), - noise_sd=self.innovation_sd, - noise_name=f"{name_prefix}_noise", - ) - - return trajectory - - def _sample_vectorized( - self, - n_timepoints: int, - n_processes: int, - initial_values: ArrayLike | None, - name_prefix: str, - ) -> ArrayLike: - """ - Sample multiple AR(1) trajectories in parallel. - - Returns - ------- - ArrayLike - Trajectories of shape (n_timepoints, n_processes). - """ - if initial_values is None: - initial_values = jnp.zeros(n_processes) - elif jnp.isscalar(initial_values): - initial_values = jnp.full(n_processes, initial_values) + initial_value = jnp.zeros(n_processes) + elif jnp.isscalar(initial_value): + initial_value = jnp.full(n_processes, initial_value) stationary_sd = self.innovation_sd / jnp.sqrt(1 - self.autoreg**2) with numpyro.plate(f"{name_prefix}_init_plate", n_processes): init_states = numpyro.sample( f"{name_prefix}_init", - dist.Normal(initial_values, stationary_sd), + dist.Normal(initial_value, stationary_sd), ) trajectories = self.ar_process( @@ -323,7 +266,7 @@ def sample( self, n_timepoints: int, initial_value: float | ArrayLike | None = None, - n_processes: int | None = None, + n_processes: int = 1, name_prefix: str = "diff_ar1", ) -> ArrayLike: """ @@ -335,75 +278,20 @@ def sample( Number of time points to generate initial_value : float or ArrayLike, optional Initial value(s). Defaults to 0.0. - n_processes : int, optional - Number of parallel processes. If None, samples single trajectory. + n_processes : int, default 1 + Number of parallel processes. name_prefix : str, default "diff_ar1" Prefix for numpyro sample sites Returns ------- ArrayLike - Trajectory (n_timepoints,) or trajectories (n_timepoints, K) - """ - if n_processes is None: - return self._sample_single(n_timepoints, initial_value, name_prefix) - else: - return self._sample_vectorized( - n_timepoints, n_processes, initial_value, name_prefix - ) - - def _sample_single( - self, - n_timepoints: int, - initial_value: float | None, - name_prefix: str, - ) -> ArrayLike: - """ - Sample a single differenced AR(1) trajectory. - - Returns - ------- - ArrayLike - Trajectory of shape (n_timepoints,). + Trajectories of shape (n_timepoints, n_processes) """ if initial_value is None: - initial_value = 0.0 - - stationary_sd = self.innovation_sd / jnp.sqrt(1 - self.autoreg**2) - init_rate_of_change = numpyro.sample( - f"{name_prefix}_init_rate", dist.Normal(0, stationary_sd) - ) - - trajectory = self.process( - n=n_timepoints + 1, - init_vals=jnp.array([initial_value]), - autoreg=jnp.array([self.autoreg]), - noise_sd=self.innovation_sd, - fundamental_process_init_vals=jnp.array([init_rate_of_change]), - noise_name=f"{name_prefix}_noise", - ) - - return trajectory[:n_timepoints] - - def _sample_vectorized( - self, - n_timepoints: int, - n_processes: int, - initial_values: ArrayLike | None, - name_prefix: str, - ) -> ArrayLike: - """ - Sample multiple differenced AR(1) trajectories in parallel. - - Returns - ------- - ArrayLike - Trajectories of shape (n_timepoints, n_processes). - """ - if initial_values is None: - initial_values = jnp.zeros(n_processes) - elif jnp.isscalar(initial_values): - initial_values = jnp.full(n_processes, initial_values) + initial_value = jnp.zeros(n_processes) + elif jnp.isscalar(initial_value): + initial_value = jnp.full(n_processes, initial_value) stationary_sd = self.innovation_sd / jnp.sqrt(1 - self.autoreg**2) @@ -415,7 +303,7 @@ def _sample_vectorized( trajectories = self.process( n=n_timepoints + 1, - init_vals=initial_values[jnp.newaxis, :], + init_vals=initial_value[jnp.newaxis, :], autoreg=jnp.full((1, n_processes), self.autoreg), noise_sd=self.innovation_sd, fundamental_process_init_vals=init_rates[jnp.newaxis, :], @@ -476,7 +364,7 @@ def sample( self, n_timepoints: int, initial_value: float | ArrayLike | None = None, - n_processes: int | None = None, + n_processes: int = 1, name_prefix: str = "rw", ) -> ArrayLike: """ @@ -488,73 +376,20 @@ def sample( Number of time points to generate initial_value : float or ArrayLike, optional Initial value(s). Defaults to 0.0. - n_processes : int, optional - Number of parallel processes. If None, samples single trajectory. + n_processes : int, default 1 + Number of parallel processes. name_prefix : str, default "rw" Prefix for numpyro sample sites Returns ------- ArrayLike - Trajectory (n_timepoints,) or trajectories (n_timepoints, K) - """ - if n_processes is None: - return self._sample_single(n_timepoints, initial_value, name_prefix) - else: - return self._sample_vectorized( - n_timepoints, n_processes, initial_value, name_prefix - ) - - def _sample_single( - self, - n_timepoints: int, - initial_value: float | None, - name_prefix: str, - ) -> ArrayLike: - """ - Sample a single random walk trajectory using PyRenew's RandomWalk. - - Returns - ------- - ArrayLike - Trajectory of shape (n_timepoints,). + Trajectories of shape (n_timepoints, n_processes) """ if initial_value is None: - initial_value = 0.0 - - step_rv = DistributionalVariable( - name=f"{name_prefix}_step", - distribution=dist.Normal(0, self.innovation_sd), - ) - - rw = PyRenewRandomWalk(step_rv=step_rv) - - trajectory = rw.sample( - init_vals=jnp.array([initial_value]), - n=n_timepoints, - ) - - return trajectory - - def _sample_vectorized( - self, - n_timepoints: int, - n_processes: int, - initial_values: ArrayLike | None, - name_prefix: str, - ) -> ArrayLike: - """ - Sample multiple random walk trajectories in parallel. - - Returns - ------- - ArrayLike - Trajectories of shape (n_timepoints, n_processes). - """ - if initial_values is None: - initial_values = jnp.zeros(n_processes) - elif jnp.isscalar(initial_values): - initial_values = jnp.full(n_processes, initial_values) + initial_value = jnp.zeros(n_processes) + elif jnp.isscalar(initial_value): + initial_value = jnp.full(n_processes, initial_value) # Non-centered parameterization to avoid funnel problems with numpyro.plate(f"{name_prefix}_time", n_timepoints - 1): @@ -573,6 +408,6 @@ def _sample_vectorized( cumulative = jnp.cumsum(increments, axis=0) return jnp.concatenate( - [initial_values[jnp.newaxis, :], initial_values + cumulative], + [initial_value[jnp.newaxis, :], initial_value + cumulative], axis=0, ) diff --git a/test/test_temporal_processes.py b/test/test_temporal_processes.py index 8763d9cb..12d75e2e 100644 --- a/test/test_temporal_processes.py +++ b/test/test_temporal_processes.py @@ -100,7 +100,7 @@ def test_differenced_ar1_single_sample_shape(self): with numpyro.handlers.seed(rng_seed=42): trajectory = dar1.sample(n_timepoints=n_timepoints) - assert trajectory.shape == (n_timepoints,) + assert trajectory.shape == (n_timepoints, 1) def test_differenced_ar1_single_with_initial_value(self): """Test DifferencedAR1 single sample with initial value.""" @@ -114,7 +114,7 @@ def test_differenced_ar1_single_with_initial_value(self): initial_value=1.5, ) - assert trajectory.shape == (n_timepoints,) + assert trajectory.shape == (n_timepoints, 1) def test_differenced_ar1_vectorized_sample_shape(self): """Test that DifferencedAR1 vectorized sample returns correct shape.""" @@ -224,8 +224,8 @@ def test_random_walk_smaller_innovation_sd_produces_smoother_trajectory( trajectory_large = rw_large.sample(n_timepoints=n_timepoints) # Smaller innovation_sd should produce smaller step sizes - steps_small = jnp.abs(jnp.diff(trajectory_small)) - steps_large = jnp.abs(jnp.diff(trajectory_large)) + steps_small = jnp.abs(jnp.diff(trajectory_small[:, 0])) + steps_large = jnp.abs(jnp.diff(trajectory_large[:, 0])) assert jnp.mean(steps_small) < jnp.mean(steps_large) assert jnp.max(steps_small) < jnp.max(steps_large) @@ -245,8 +245,8 @@ def test_ar1_smaller_innovation_sd_produces_lower_variance(self): # After burn-in, smaller innovation_sd should have lower variance burn_in = 20 - var_small = jnp.var(trajectory_small[burn_in:]) - var_large = jnp.var(trajectory_large[burn_in:]) + var_small = jnp.var(trajectory_small[burn_in:, 0]) + var_large = jnp.var(trajectory_large[burn_in:, 0]) assert var_small < var_large @@ -266,8 +266,8 @@ def test_differenced_ar1_smaller_innovation_sd_produces_smoother_changes( trajectory_large = dar_large.sample(n_timepoints=n_timepoints) # Growth rates (differences) should have lower variance - diffs_small = jnp.diff(trajectory_small) - diffs_large = jnp.diff(trajectory_large) + diffs_small = jnp.diff(trajectory_small[:, 0]) + diffs_large = jnp.diff(trajectory_large[:, 0]) assert jnp.std(diffs_small) < jnp.std(diffs_large) From 72da4278bbf013c409b776f8f3666eab3802bf6c Mon Sep 17 00:00:00 2001 From: Mitzi Morris Date: Thu, 5 Feb 2026 14:38:01 -0500 Subject: [PATCH 70/81] API changes per code review - latent infections doesn't track obs/unobs subpops --- .../tutorials/building_multisignal_models.qmd | 91 +++++---- .../latent_hierarchical_infections.qmd | 75 +++---- pyrenew/latent/base.py | 185 ++++++------------ pyrenew/latent/hierarchical_infections.py | 29 +-- pyrenew/model/multisignal_model.py | 55 ++---- pyrenew/model/pyrenew_builder.py | 16 +- test/test_base_latent_process.py | 64 ++---- test/test_hierarchical_infections.py | 43 ++-- test/test_observation_counts.py | 51 +++++ test/test_observation_measurements.py | 72 +++++++ test/test_pyrenew_builder.py | 46 ++--- 11 files changed, 349 insertions(+), 378 deletions(-) diff --git a/docs/tutorials/building_multisignal_models.qmd b/docs/tutorials/building_multisignal_models.qmd index a95be919..7a4b844c 100644 --- a/docs/tutorials/building_multisignal_models.qmd +++ b/docs/tutorials/building_multisignal_models.qmd @@ -132,7 +132,7 @@ flowchart TB subgraph Infections["Infection Arrays"] J["infections_aggregate (T,)"] - S["infections_subpop (T, K_obs)"] + S["infections_all_subpops (T, K)"] end subgraph Obs["Observation Processes"] @@ -148,7 +148,7 @@ flowchart TB L --> J L --> S J -->|"aggregate"| C - S -->|"subpop"| W + S -->|"subpop + subpop_indices"| W C -->|"ascertainment × delay"| HA W -->|"shedding × genome × dilution"| WW ``` @@ -159,11 +159,13 @@ Different observation processes observe different levels of the model hierarchy. | Resolution | Receives | Example signals | |----------------------|-------------------|-------------------------------| -| `"aggregate"` | Aggregated infections (sum across all subpopulations) | Hospital admissions, case counts | -| `"subpop"` | Infection matrix with one column per observed subpopulation | Wastewater, site-specific surveillance | +| `"aggregate"` | Aggregated infections (sum across all subpopulations), shape `(T,)` | Hospital admissions, case counts | +| `"subpop"` | Infection matrix for all K subpopulations, shape `(T, K)` | Wastewater, site-specific surveillance | The `PyrenewBuilder` routes latent infections to observation processes based on each process's declared resolution. +For subpopulation-level observations, the observation process selects which subpopulations it observes using `subpop_indices` provided at sample/fit time. This allows flexible observation patterns—for example, wastewater samples might only cover 5 of 6 subpopulations (catchment areas), while the 6th represents areas without wastewater monitoring. + With this structure in mind, we'll now define each component following the generative direction: first the latent infection process, then the observation processes. ## Latent Infection Process @@ -593,7 +595,7 @@ The subpopulation-level wastewater data is specified as a `Wastewater` observati wastewater={ "obs": jnp.array([...]), # observed log concentrations (n_obs,) "times": model.shift_times(ww_times), # time indices on shared axis - "subpop_indices": jnp.array([...]), # which catchment (selects infection trajectory) + "subpop_indices": jnp.array([...]), # which subpopulation (selects infection column) "sensor_indices": jnp.array([...]), # which WWTP/lab pair (selects noise parameters) "n_sensors": int, # total number of WWTP/lab pairs } @@ -601,26 +603,29 @@ wastewater={ The `shift_times` method adds `n_init` to convert from natural coordinates (0 = first observation day) to the shared time axis. -A **subpopulation** is a portion of the jurisdiction's population defined by a catchment area. A **sensor** is a measurement source — typically a WWTP/lab pair — that produces observations. Multiple sensors can observe the same subpopulation (e.g., different labs processing samples from the same catchment), so `subpop_indices` and `sensor_indices` may differ. +**Understanding `subpop_indices`**: The latent process generates infections for all K subpopulations as a matrix of shape `(T, K)`. Each observation selects which column (subpopulation) it came from using `subpop_indices`. This is how observation processes "know" which subpopulations they observe—the user specifies this mapping at sample/fit time. + +A **subpopulation** is a portion of the jurisdiction's population (e.g., a catchment area). A **sensor** is a measurement source — typically a WWTP/lab pair — that produces observations. Multiple sensors can observe the same subpopulation (e.g., different labs processing samples from the same catchment), so `subpop_indices` and `sensor_indices` may differ. -- `subpop_indices` links each observation to the appropriate infection trajectory +- `subpop_indices` links each observation to the appropriate infection column (0-indexed into the K subpopulations) - `sensor_indices` selects which sensor's noise parameters (mode and sd) to apply -**Example**: A jurisdiction has 3 catchments (subpopulations 0, 1, 2). Catchment 0 has samples processed by two labs (sensors 0 and 1). Observations from catchment 0 all have `subpop_indices=0` but may have `sensor_indices=0` or `sensor_indices=1` depending on which lab processed the sample. +**Example**: A jurisdiction has 6 subpopulations (indices 0-5), where 5 have wastewater monitoring and 1 does not. The `subpop_fractions` array has 6 elements. If subpopulation 2 lacks wastewater monitoring, wastewater observations would have `subpop_indices` values only in {0, 1, 3, 4, 5}—never 2. The monitored subpopulations need not be contiguous. The latent process still generates infections for all 6 subpopulations; the wastewater observation just doesn't see subpopulation 2. ### Population Structure -Population structure is specified via two arrays: +Population structure is specified via a single array of fractions for all K subpopulations: ```python model.fit( - obs_fractions=jnp.array([0.10, 0.14, 0.21, 0.22, 0.07]), # 5 observed - unobs_fractions=jnp.array([0.26]), # 1 unobserved + subpop_fractions=jnp.array([0.10, 0.14, 0.21, 0.22, 0.07, 0.26]), # 6 subpopulations ... ) ``` -This specifies 5 observed subpopulations (e.g., wastewater catchments) and 1 unobserved (remainder of jurisdiction). The fractions must sum to 1.0. +This specifies 6 subpopulations with their population fractions. The fractions must sum to 1.0. The latent process generates infections for all 6 subpopulations. + +Which subpopulations each observation process "sees" is determined by the `subpop_indices` in the observation data, not by the population structure. For example, if wastewater monitoring covers only 5 of the 6 subpopulations (say, all except subpopulation 2), the wastewater observation data would have `subpop_indices` values in {0, 1, 3, 4, 5} but never 2. The monitored subpopulations can be any subset of {0, ..., K-1}. ### Example `model.fit()` Call @@ -629,15 +634,14 @@ This specifies 5 observed subpopulations (e.g., wastewater catchments) and 1 uno model.fit( n_days_post_init=n_days, population_size=population_size, - # Population structure - obs_fractions=obs_subpop_fractions, - unobs_fractions=unobs_subpop_fractions, + # Population structure (all K subpopulations) + subpop_fractions=subpop_fractions, # Observation data (keyed by names from add_observation) hospital={"obs": model.pad_observations(hosp_counts)}, wastewater={ "obs": ww_conc, "times": model.shift_times(ww_times), - "subpop_indices": ww_subpop_indices, + "subpop_indices": ww_subpop_indices, # selects which of K subpops each obs came from "sensor_indices": ww_sensor_indices, "n_sensors": n_ww_sensors, }, @@ -646,31 +650,39 @@ model.fit( ## Running the Model -First we declare the population structure. +First we declare the population structure. We have 6 subpopulations, where 5 have wastewater monitoring and 1 does not. The subpopulations with wastewater monitoring need not be contiguous indices—they could be any subset of {0, 1, ..., K-1}. ```{python} # | label: population-structure -# 5 observed subpopulations (wastewater catchments) -obs_subpop_fractions = jnp.array([0.10, 0.14, 0.21, 0.22, 0.07]) +# All 6 subpopulations with their population fractions +subpop_fractions = jnp.array([0.10, 0.14, 0.21, 0.22, 0.07, 0.26]) -# 1 unobserved subpopulation (remainder of jurisdiction) -unobs_subpop_fractions = jnp.array([0.26]) +K = len(subpop_fractions) -K_obs = len(obs_subpop_fractions) +# Which subpopulations have wastewater monitoring? +# These indices will be used in subpop_indices for wastewater observations. +# They can be any subset of {0, 1, ..., K-1}, not necessarily contiguous. +ww_monitored_subpops = jnp.array( + [0, 1, 2, 3, 4] +) # subpop 5 has no wastewater monitoring -print(f"Observed subpopulations (K_obs): {K_obs}") -print(f"Unobserved subpopulations: {len(unobs_subpop_fractions)}") -print(f"Observed coverage: {float(jnp.sum(obs_subpop_fractions)):.0%}") +print(f"Total subpopulations (K): {K}") print( - f"Total coverage: {float(jnp.sum(obs_subpop_fractions) + jnp.sum(unobs_subpop_fractions)):.0%}" + f"Subpopulations with wastewater monitoring: {list(ww_monitored_subpops)}" ) +print( + f"Wastewater coverage: {float(jnp.sum(subpop_fractions[ww_monitored_subpops])):.0%}" +) +print(f"Total population: {float(jnp.sum(subpop_fractions)):.0%}") ``` We define a function to prepare observation data using the model's helper methods to align with the shared time axis. ```{python} # | label: prepare-observation-data -def prepare_observation_data(model, n_days_fit, hosp_admits, ww_data, K_obs): +def prepare_observation_data( + model, n_days_fit, hosp_admits, ww_data, ww_monitored_subpops +): """ Prepare observation data for fitting. @@ -687,8 +699,9 @@ def prepare_observation_data(model, n_days_fit, hosp_admits, ww_data, K_obs): Hospital admissions time series ww_data : dict Wastewater data dictionary - K_obs : int - Number of observed subpopulations (for mapping sites to subpops) + ww_monitored_subpops : array + Indices of subpopulations that have wastewater monitoring. + These are the valid values for subpop_indices in wastewater data. """ # Hospital: dense, NaN-padded to length n_total hosp_obs = model.pad_observations(hosp_admits[:n_days_fit]) @@ -700,9 +713,15 @@ def prepare_observation_data(model, n_days_fit, hosp_admits, ww_data, K_obs): ww_sensors = ww_data["site_ids"][ww_mask] # Map wastewater sensors to subpopulation indices - # For this demo, we cycle through the K_obs observed subpopulations + # Each sensor is assigned to one of the monitored subpopulations. + # In practice, this mapping comes from your data (which WWTP serves which catchment). + # For this demo, we cycle sensors through the monitored subpopulations. n_ww_sensors = ww_data["n_sites"] - sensor_to_subpop = {i: i % K_obs for i in range(n_ww_sensors)} + n_monitored = len(ww_monitored_subpops) + sensor_to_subpop = { + i: int(ww_monitored_subpops[i % n_monitored]) + for i in range(n_ww_sensors) + } ww_subpop_indices = jnp.array( [sensor_to_subpop[int(s)] for s in ww_sensors] ) @@ -733,7 +752,7 @@ jax.clear_caches() n_days_90 = 90 obs_data_90 = prepare_observation_data( - model, n_days_90, hosp_admits, ca_ww_data, K_obs + model, n_days_90, hosp_admits, ca_ww_data, ww_monitored_subpops ) # Try non-centered reparameterization for hierarchical parameters @@ -750,8 +769,7 @@ start_time = time.time() mcmc_90 = model.fit( n_days_post_init=n_days_90, population_size=population_size, - obs_fractions=obs_subpop_fractions, - unobs_fractions=unobs_subpop_fractions, + subpop_fractions=subpop_fractions, num_warmup=500, num_samples=500, num_chains=4, @@ -912,7 +930,7 @@ jax.clear_caches() n_days_180 = 180 obs_data_180 = prepare_observation_data( - model, n_days_180, hosp_admits, ca_ww_data, K_obs + model, n_days_180, hosp_admits, ca_ww_data, ww_monitored_subpops ) print(f"Fitting model with {n_days_180} days of data...") @@ -922,8 +940,7 @@ start_time = time.time() mcmc_180 = model.fit( n_days_post_init=n_days_180, population_size=population_size, - obs_fractions=obs_subpop_fractions, - unobs_fractions=unobs_subpop_fractions, + subpop_fractions=subpop_fractions, num_warmup=1000, num_samples=500, num_chains=4, diff --git a/docs/tutorials/latent_hierarchical_infections.qmd b/docs/tutorials/latent_hierarchical_infections.qmd index 57a84cbb..f08de3b9 100644 --- a/docs/tutorials/latent_hierarchical_infections.qmd +++ b/docs/tutorials/latent_hierarchical_infections.qmd @@ -91,17 +91,14 @@ We define subpopulations as fractions of the total jurisdiction population: ```{python} # | label: population-structure -# 5 observed subpopulations -obs_fractions = jnp.array([0.10, 0.14, 0.21, 0.22, 0.07]) - -# 1 unobserved subpopulation -unobs_fractions = jnp.array([0.26]) - -# Verify fractions sum to 1 -total = float(jnp.sum(obs_fractions) + jnp.sum(unobs_fractions)) -print(f"Total population fraction: {total:.2f}") -print(f"Observed coverage: {float(jnp.sum(obs_fractions)):.0%}") -print(f"Unobserved coverage: {float(jnp.sum(unobs_fractions)):.0%}") +# 6 subpopulations with their population fractions +# Which subpopulations are "observed" or "unobserved" is determined by each +# observation process (via subpop_indices), not by the latent process. +subpop_fractions = jnp.array([0.10, 0.14, 0.21, 0.22, 0.07, 0.26]) + +K = len(subpop_fractions) +print(f"Number of subpopulations (K): {K}") +print(f"Total population fraction: {float(jnp.sum(subpop_fractions)):.2f}") ``` Note: Population structure is passed at **sample time**, not when constructing the process. This allows the same model specification to be fit to different jurisdictions. @@ -174,10 +171,9 @@ for name, gi_rv in gi_configs.items(): subpop_temporal=subpop_temporal, ) with numpyro.handlers.seed(rng_seed=42): - inf_agg, _, _, _ = model.sample( + inf_agg, _ = model.sample( n_days_post_init=n_days_demo, - obs_fractions=obs_fractions, - unobs_fractions=unobs_fractions, + subpop_fractions=subpop_fractions, ) n_init = model.n_initialization_points inf = np.array(inf_agg)[n_init:] @@ -204,10 +200,9 @@ for init_rt_name, init_rt_val in [ subpop_temporal=subpop_temporal, ) with numpyro.handlers.seed(rng_seed=42): - inf_agg, _, _, _ = model.sample( + inf_agg, _ = model.sample( n_days_post_init=n_days_demo, - obs_fractions=obs_fractions, - unobs_fractions=unobs_fractions, + subpop_fractions=subpop_fractions, ) n_init = model.n_initialization_points inf = np.array(inf_agg)[n_init:] @@ -317,10 +312,9 @@ for name, i0_val in i0_configs.items(): subpop_temporal=subpop_temporal, ) with numpyro.handlers.seed(rng_seed=42): - inf_agg, _, _, _ = model.sample( + inf_agg, _ = model.sample( n_days_post_init=n_days_demo, - obs_fractions=obs_fractions, - unobs_fractions=unobs_fractions, + subpop_fractions=subpop_fractions, ) n_init = model.n_initialization_points i0_results[name] = np.array(inf_agg)[n_init:] @@ -421,11 +415,10 @@ n_days = 28 # 4 weeks of simulation with numpyro.handlers.seed(rng_seed=42): result = latent_infections.sample( n_days_post_init=n_days, - obs_fractions=obs_fractions, - unobs_fractions=unobs_fractions, + subpop_fractions=subpop_fractions, ) -infections_agg, infections_all, infections_obs, infections_unobs = result +infections_agg, infections_all = result n_total = infections_agg.shape[0] n_init = n_total - n_days @@ -433,8 +426,6 @@ n_init = n_total - n_days print("Output shapes:") print(f" Aggregate (jurisdiction total): {infections_agg.shape}") print(f" All subpopulations: {infections_all.shape}") -print(f" Observed subpopulations: {infections_obs.shape}") -print(f" Unobserved subpopulations: {infections_unobs.shape}") print( f"\nTimeline: {n_init} initialization days + {n_days} simulation days = {n_total} total" ) @@ -450,8 +441,7 @@ def sample_model(): """Wrapper for prior predictive sampling.""" return latent_infections.sample( n_days_post_init=n_days, - obs_fractions=obs_fractions, - unobs_fractions=unobs_fractions, + subpop_fractions=subpop_fractions, ) @@ -468,7 +458,7 @@ print(f"Infection samples: {prior_samples['infections_aggregate'].shape}") ```{python} # | label: rt-trajectories-data # Extract post-initialization Rt trajectories -rt_samples = np.array(prior_samples["rt_baseline"])[:, n_init:] +rt_samples = np.array(prior_samples["rt_baseline"])[:, n_init:, 0] # Build long-format dataframe for plotting rt_data = [] @@ -554,14 +544,13 @@ for name, temporal in temporal_configs.items(): def make_sampler(m): return lambda: m.sample( n_days_post_init=n_days, - obs_fractions=obs_fractions, - unobs_fractions=unobs_fractions, + subpop_fractions=subpop_fractions, ) samples = Predictive(make_sampler(model), num_samples=n_samples)( random.PRNGKey(42) ) - process_samples[name] = np.array(samples["rt_baseline"])[:, n_init:] + process_samples[name] = np.array(samples["rt_baseline"])[:, n_init:, 0] ``` ```{python} @@ -651,8 +640,7 @@ for name, subpop_temporal in subpop_configs.items(): def make_sampler(m): return lambda: m.sample( n_days_post_init=n_days, - obs_fractions=obs_fractions, - unobs_fractions=unobs_fractions, + subpop_fractions=subpop_fractions, ) deviation_samples[name] = Predictive( @@ -738,8 +726,7 @@ for name, cfg in configs.items(): def make_sampler(m): return lambda: m.sample( n_days_post_init=n_days, - obs_fractions=obs_fractions, - unobs_fractions=unobs_fractions, + subpop_fractions=subpop_fractions, ) tuning_samples[name] = Predictive( @@ -752,7 +739,7 @@ for name, cfg in configs.items(): # Build comparison dataframe tuning_data = [] for name, samples in tuning_samples.items(): - rt = np.array(samples["rt_baseline"])[:, n_init:] + rt = np.array(samples["rt_baseline"])[:, n_init:, 0] for i in range(n_samples): for d in range(n_days): tuning_data.append( @@ -804,8 +791,7 @@ with numpyro.handlers.seed(rng_seed=123): with numpyro.handlers.trace() as trace: latent_infections.sample( n_days_post_init=n_days, - obs_fractions=obs_fractions, - unobs_fractions=unobs_fractions, + subpop_fractions=subpop_fractions, ) deviations = trace["subpop_deviations"]["value"] @@ -820,14 +806,12 @@ print("(Should be ~0 due to sum-to-zero constraint)") # | label: verify-aggregate # Check that aggregate = weighted sum of subpopulations with numpyro.handlers.seed(rng_seed=123): - inf_agg, inf_all, _, _ = latent_infections.sample( + inf_agg, inf_all = latent_infections.sample( n_days_post_init=n_days, - obs_fractions=obs_fractions, - unobs_fractions=unobs_fractions, + subpop_fractions=subpop_fractions, ) -all_fractions = jnp.concatenate([obs_fractions, unobs_fractions]) -computed_agg = jnp.sum(inf_all * all_fractions[None, :], axis=1) +computed_agg = jnp.sum(inf_all * subpop_fractions[None, :], axis=1) max_diff = float(jnp.max(jnp.abs(computed_agg - inf_agg))) print(f"Max difference (aggregate vs weighted sum): {max_diff:.2e}") @@ -880,10 +864,9 @@ latent = HierarchicalInfections( ) # Sample (population structure provided at sample time) -infections_agg, infections_all, infections_obs, infections_unobs = latent.sample( +infections_agg, infections_all = latent.sample( n_days_post_init=28, - obs_fractions=obs_fractions, - unobs_fractions=unobs_fractions, + subpop_fractions=subpop_fractions, ) ``` diff --git a/pyrenew/latent/base.py b/pyrenew/latent/base.py index 743b3c6a..ba6884a9 100644 --- a/pyrenew/latent/base.py +++ b/pyrenew/latent/base.py @@ -8,9 +8,9 @@ from dataclasses import dataclass from typing import NamedTuple -import jax import jax.numpy as jnp from jax.typing import ArrayLike +from numpyro.util import not_jax_tracer from pyrenew.metaclass import RandomVariable @@ -27,18 +27,10 @@ class LatentSample(NamedTuple): all_subpops : ArrayLike Infections for all subpopulations. Shape: (n_total_days, K) - observed : ArrayLike - Infections for observed subpopulations only. - Shape: (n_total_days, K_obs) - unobserved : ArrayLike - Infections for unobserved subpopulations only. - Shape: (n_total_days, K_unobs) """ aggregate: ArrayLike all_subpops: ArrayLike - observed: ArrayLike - unobserved: ArrayLike @dataclass @@ -48,23 +40,14 @@ class PopulationStructure: Attributes ---------- - obs_fractions : ArrayLike - Population fractions for observed subpopulations - unobs_fractions : ArrayLike - Population fractions for unobserved subpopulations + fractions : ArrayLike + Population fractions for all subpopulations. Shape: (K,) K : int Total number of subpopulations - K_obs : int - Number of observed subpopulations - K_unobs : int - Number of unobserved subpopulations """ - obs_fractions: ArrayLike - unobs_fractions: ArrayLike + fractions: ArrayLike K: int - K_obs: int - K_unobs: int class BaseLatentInfectionProcess(RandomVariable): @@ -73,16 +56,15 @@ class BaseLatentInfectionProcess(RandomVariable): Provides common functionality for hierarchical and partitioned infection models: - Population fraction validation and parsing (at sample time) - - Splitting subpopulations into observed/unobserved - - Standard 4-tuple output structure + - Standard output structure via LatentSample All subclasses return infections as a ``LatentSample`` named tuple with fields: - (aggregate, all_subpops, observed, unobserved) + (aggregate, all_subpops). Observation processes are responsible for selecting + which subpopulations they observe via indexing. The constructor specifies model structure (generation interval, priors, - temporal processes). Population structure (obs_fractions, unobs_fractions) is - provided at sample time, allowing a single model to be fit to multiple - jurisdictions with different population structures. + temporal processes). Population structure (subpop_fractions) is provided at + sample time, allowing a single model to be fit to multiple jurisdictions. Parameters ---------- @@ -95,17 +77,13 @@ class BaseLatentInfectionProcess(RandomVariable): Notes ----- - Population structure (obs_fractions, unobs_fractions) is passed to the sample() - method, not the constructor. This allows a single model instance to be fit to - multiple datasets with different jurisdiction structures. + Population structure (subpop_fractions) is passed to the sample() method, not + the constructor. This allows a single model instance to be fit to multiple + datasets with different jurisdiction structures. When using PyrenewBuilder, n_initialization_points is computed automatically from all observation processes. When manually creating latent processes, you can either specify it explicitly or let it default to max(21, 2*len(gen_int)-1). - - Subclasses must implement the `sample` method which returns a 4-tuple with - shapes: (n_total_days,), (n_total_days, K), (n_total_days, K_obs), - (n_total_days, K_unobs). """ def __init__( @@ -144,12 +122,17 @@ def __init__( @staticmethod def _parse_and_validate_fractions( - obs_fractions: ArrayLike = None, - unobs_fractions: ArrayLike = None, + subpop_fractions: ArrayLike = None, ) -> PopulationStructure: """ Parse and validate population fraction parameters. + Parameters + ---------- + subpop_fractions : ArrayLike + Population fractions for all subpopulations. Must be a 1D array + with at least one element. Values must be non-negative and sum to 1. + Returns ------- PopulationStructure @@ -158,77 +141,44 @@ def _parse_and_validate_fractions( Raises ------ ValueError - If parameterization is invalid or fractions don't sum to 1.0 + If fractions are invalid or don't sum to 1.0 """ - if obs_fractions is None or unobs_fractions is None: - raise ValueError("Both obs_fractions and unobs_fractions must be provided") + if subpop_fractions is None: + raise ValueError("subpop_fractions must be provided") - obs_fractions = jnp.asarray(obs_fractions) - unobs_fractions = jnp.asarray(unobs_fractions) + fractions = jnp.asarray(subpop_fractions) - if obs_fractions.ndim != 1 or unobs_fractions.ndim != 1: - raise ValueError("Fractions must be 1D arrays") + if fractions.ndim != 1: + raise ValueError("subpop_fractions must be a 1D array") - K_obs = len(obs_fractions) - K_unobs = len(unobs_fractions) - K = K_obs + K_unobs + K = len(fractions) - if K_obs == 0: - raise ValueError("Must have at least one observed subpopulation") + if K == 0: + raise ValueError("Must have at least one subpopulation") - # Only validate when values are concrete (not during JAX tracing) - try: - if jnp.any(obs_fractions < 0) or jnp.any(unobs_fractions < 0): - raise ValueError("All population fractions must be non-negative") + # Only validate when results are concrete (not during JAX tracing) + # Check the result of the comparison, not just the input, because + # JIT compilation can trace operations on concrete arrays + neg_check = jnp.any(fractions < 0) + if not_jax_tracer(neg_check) and neg_check: + raise ValueError("All population fractions must be non-negative") - total = jnp.sum(obs_fractions) + jnp.sum(unobs_fractions) - if not jnp.isclose(total, 1.0, atol=1e-6): - raise ValueError( - f"Population fractions must sum to 1.0, got {float(total):.6f}" - ) - except jax.errors.TracerBoolConversionError: - # Skip validation during JAX tracing - values were validated before MCMC - pass + total = jnp.sum(fractions) + sum_check = jnp.isclose(total, 1.0, atol=1e-6) + if not_jax_tracer(sum_check) and not sum_check: + raise ValueError( + f"Population fractions must sum to 1.0, got {float(total):.6f}" + ) return PopulationStructure( - obs_fractions=obs_fractions, - unobs_fractions=unobs_fractions, + fractions=fractions, K=K, - K_obs=K_obs, - K_unobs=K_unobs, ) - @staticmethod - def _split_subpopulations( - infections_all: ArrayLike, - pop: PopulationStructure, - ) -> tuple[ArrayLike, ArrayLike]: - """ - Split all subpopulations into observed and unobserved. - - Parameters - ---------- - infections_all : ArrayLike - Infections for all subpopulations, shape (n_days, K) - pop : PopulationStructure - Population structure with K_obs for splitting - - Returns - ------- - tuple[ArrayLike, ArrayLike] - (infections_observed, infections_unobserved) with shapes - (n_days, K_obs) and (n_days, K_unobs) - """ - infections_observed = infections_all[:, : pop.K_obs] - infections_unobserved = infections_all[:, pop.K_obs :] - return infections_observed, infections_unobserved - @staticmethod def _validate_output_shapes( infections_aggregate: ArrayLike, infections_all: ArrayLike, - infections_observed: ArrayLike, - infections_unobserved: ArrayLike, n_total_days: int, pop: PopulationStructure, ) -> None: @@ -241,14 +191,10 @@ def _validate_output_shapes( Aggregate infections (sum across subpopulations) infections_all : ArrayLike All subpopulation infections - infections_observed : ArrayLike - Observed subpopulation infections - infections_unobserved : ArrayLike - Unobserved subpopulation infections n_total_days : int Expected number of days (n_initialization_points + n_days_post_init) pop : PopulationStructure - Population structure with K, K_obs, K_unobs + Population structure with K Raises ------ @@ -258,15 +204,11 @@ def _validate_output_shapes( expected = { "infections_aggregate": (n_total_days,), "infections_all": (n_total_days, pop.K), - "infections_observed": (n_total_days, pop.K_obs), - "infections_unobserved": (n_total_days, pop.K_unobs), } actual = { "infections_aggregate": infections_aggregate.shape, "infections_all": infections_all.shape, - "infections_observed": infections_observed.shape, - "infections_unobserved": infections_unobserved.shape, } for name, expected_shape in expected.items(): @@ -301,22 +243,21 @@ def _validate_I0(I0: ArrayLike) -> None: """ I0 = jnp.asarray(I0) - # Skip validation during JAX tracing (e.g., Predictive sampling) - # Traced arrays cannot be used in Python boolean operations - try: - if jnp.any(I0 <= 0): - raise ValueError( - f"I0 must be positive (got min={float(jnp.min(I0)):.6f}). " - "I0 represents infection prevalence as a proportion of the population." - ) - if jnp.any(I0 > 1): - raise ValueError( - f"I0 must be <= 1 (got max={float(jnp.max(I0)):.6f}). " - "I0 represents infection prevalence as a proportion of the population." - ) - except jax.errors.TracerBoolConversionError: - # During tracing, skip validation - pass + # Only validate when results are concrete (not during JAX tracing) + # Check the result of the comparison, not just the input, because + # JIT compilation can trace operations on concrete arrays + pos_check = jnp.any(I0 <= 0) + if not_jax_tracer(pos_check) and pos_check: + raise ValueError( + f"I0 must be positive (got min={float(jnp.min(I0)):.6f}). " + "I0 represents infection prevalence as a proportion of the population." + ) + max_check = jnp.any(I0 > 1) + if not_jax_tracer(max_check) and max_check: + raise ValueError( + f"I0 must be <= 1 (got max={float(jnp.max(I0)):.6f}). " + "I0 represents infection prevalence as a proportion of the population." + ) def get_required_lookback(self) -> int: """ @@ -355,8 +296,7 @@ def sample( self, n_days_post_init: int, *, - obs_fractions: ArrayLike = None, - unobs_fractions: ArrayLike = None, + subpop_fractions: ArrayLike = None, **kwargs, ) -> LatentSample: """ @@ -366,10 +306,9 @@ def sample( ---------- n_days_post_init : int Number of days to simulate after initialization period - obs_fractions : ArrayLike - Population fractions for observed subpopulations. - unobs_fractions : ArrayLike - Population fractions for unobserved subpopulations. + subpop_fractions : ArrayLike + Population fractions for all subpopulations. Shape: (K,). + Must sum to 1.0. **kwargs Additional parameters required by specific implementations @@ -379,8 +318,6 @@ def sample( Named tuple with fields: - aggregate: shape (n_total_days,) - all_subpops: shape (n_total_days, K) - - observed: shape (n_total_days, K_obs) - - unobserved: shape (n_total_days, K_unobs) where n_total_days = n_initialization_points + n_days_post_init """ diff --git a/pyrenew/latent/hierarchical_infections.py b/pyrenew/latent/hierarchical_infections.py index 3a0ee946..a10086b9 100644 --- a/pyrenew/latent/hierarchical_infections.py +++ b/pyrenew/latent/hierarchical_infections.py @@ -38,7 +38,7 @@ class HierarchicalInfections(BaseLatentInfectionProcess): - Aggregate total: I_aggregate(t) = sum_k p_k * I_k(t) The constructor specifies model structure (priors, temporal processes). - Population structure (fractions, K, K_obs) is provided at sample time, + Population structure (subpop_fractions) is provided at sample time, allowing a single model to be fit to multiple jurisdictions. Parameters @@ -156,8 +156,7 @@ def sample( self, n_days_post_init: int, *, - obs_fractions: ArrayLike = None, - unobs_fractions: ArrayLike = None, + subpop_fractions: ArrayLike = None, **kwargs, ) -> LatentSample: """ @@ -170,10 +169,9 @@ def sample( ---------- n_days_post_init : int Number of days to simulate after initialization period - obs_fractions : ArrayLike - Population fractions for observed subpopulations. - unobs_fractions : ArrayLike - Population fractions for unobserved subpopulations. + subpop_fractions : ArrayLike + Population fractions for all subpopulations. Shape: (K,). + Must sum to 1.0. **kwargs Additional arguments (unused, for compatibility) @@ -183,13 +181,10 @@ def sample( Named tuple with fields: - aggregate: shape (n_total_days,) - all_subpops: shape (n_total_days, K) - - observed: shape (n_total_days, K_obs) - - unobserved: shape (n_total_days, K_unobs) """ # Parse and validate population structure pop = self._parse_and_validate_fractions( - obs_fractions=obs_fractions, - unobs_fractions=unobs_fractions, + subpop_fractions=subpop_fractions, ) n_total_days = self.n_initialization_points + n_days_post_init @@ -241,8 +236,6 @@ def sample( gen_int_reversed = jnp.flip(gen_int) recent_I0_all = I0_all[-gen_int.size :, :] - all_fractions = jnp.concatenate([pop.obs_fractions, pop.unobs_fractions]) - # Vectorized renewal equation for all subpopulations via vmap post_init_infections_all = jax.vmap( lambda I0_col, Rt_col: compute_infections_from_rt( @@ -257,18 +250,12 @@ def sample( infections_all = jnp.vstack([I0_all, post_init_infections_all]) infections_aggregate = jnp.sum( - infections_all * all_fractions[jnp.newaxis, :], axis=1 - ) - - infections_observed, infections_unobserved = self._split_subpopulations( - infections_all, pop + infections_all * pop.fractions[jnp.newaxis, :], axis=1 ) self._validate_output_shapes( infections_aggregate, infections_all, - infections_observed, - infections_unobserved, n_total_days, pop, ) @@ -283,6 +270,4 @@ def sample( return LatentSample( aggregate=infections_aggregate, all_subpops=infections_all, - observed=infections_observed, - unobserved=infections_unobserved, ) diff --git a/pyrenew/model/multisignal_model.py b/pyrenew/model/multisignal_model.py index 308b0998..518cdf1a 100644 --- a/pyrenew/model/multisignal_model.py +++ b/pyrenew/model/multisignal_model.py @@ -45,7 +45,7 @@ class MultiSignalModel(Model): The model automatically routes latent infections to observations based on each observation's infection_resolution() method: - "aggregate" → receives aggregate infections from latent process - - "subpop" → receives observed subpopulation infections from latent process + - "subpop" → receives all subpopulation infections; observation selects via indices """ def __init__( @@ -164,8 +164,7 @@ def shift_times(self, times: jnp.ndarray) -> jnp.ndarray: def validate_data( self, n_days_post_init: int, - obs_fractions=None, - unobs_fractions=None, + subpop_fractions=None, **observation_data: dict[str, Any], ) -> None: """ @@ -184,10 +183,8 @@ def validate_data( ---------- n_days_post_init : int Number of days to simulate after initialization period - obs_fractions : ArrayLike - Population fractions for observed subpopulations - unobs_fractions : ArrayLike - Population fractions for unobserved subpopulations + subpop_fractions : ArrayLike + Population fractions for all subpopulations. Shape: (K,). **observation_data Data for each observation process, keyed by observation name. Each value should be a dict of kwargs for that observation's sample(). @@ -200,8 +197,7 @@ def validate_data( If data shapes are inconsistent """ pop = BaseLatentInfectionProcess._parse_and_validate_fractions( - obs_fractions=obs_fractions, - unobs_fractions=unobs_fractions, + subpop_fractions=subpop_fractions, ) n_init = self.latent.n_initialization_points @@ -255,10 +251,10 @@ def validate_data( f"Observation '{name}': subpop_indices cannot be negative" ) max_idx = jnp.max(subpop_indices) - if max_idx >= pop.K_obs: + if max_idx >= pop.K: raise ValueError( f"Observation '{name}': subpop_indices contains " - f"{int(max_idx)} >= {pop.K_obs} (K_obs)" + f"{int(max_idx)} >= {pop.K} (K)" ) def sample( @@ -266,8 +262,7 @@ def sample( n_days_post_init: int, population_size: float, *, - obs_fractions=None, - unobs_fractions=None, + subpop_fractions=None, **observation_data, ): """ @@ -282,10 +277,8 @@ def sample( population_size : float Total population size. Used to convert infection proportions (from latent process) to infection counts (for observation processes). - obs_fractions : ArrayLike - Population fractions for observed subpopulations. - unobs_fractions : ArrayLike - Population fractions for unobserved subpopulations. + subpop_fractions : ArrayLike + Population fractions for all subpopulations. Shape: (K,). **observation_data Data for each observation process, keyed by observation name (the ``name`` attribute of each observation process). @@ -294,26 +287,21 @@ def sample( Returns ------- tuple - Four arrays of infection counts (scaled by population_size): + Two arrays of infection counts (scaled by population_size): - inf_aggregate: shape (n_total_days,) - aggregate infections - inf_all: shape (n_total_days, K) - all subpopulation infections - - inf_obs: shape (n_total_days, K_obs) - observed subpopulation infections - - inf_unobs: shape (n_total_days, K_unobs) - unobserved subpopulation infections where n_total_days = n_initialization_points + n_days_post_init """ # Generate latent infections (proportions) latent_sample = self.latent.sample( n_days_post_init=n_days_post_init, - obs_fractions=obs_fractions, - unobs_fractions=unobs_fractions, + subpop_fractions=subpop_fractions, ) # Scale from proportions to counts inf_aggregate = latent_sample.aggregate * population_size inf_all = latent_sample.all_subpops * population_size - inf_obs = latent_sample.observed * population_size - inf_unobs = latent_sample.unobserved * population_size # Record scaled infections for posterior analysis numpyro.deterministic("latent_infections", inf_aggregate) @@ -322,7 +310,7 @@ def sample( # Map infection resolution to infection arrays latent_map = { "aggregate": inf_aggregate, - "subpop": inf_obs, + "subpop": inf_all, } # Apply each observation process @@ -346,15 +334,14 @@ def sample( ) # Return scaled infection counts - return inf_aggregate, inf_all, inf_obs, inf_unobs + return inf_aggregate, inf_all def fit( self, n_days_post_init: int, population_size: float, *, - obs_fractions=None, - unobs_fractions=None, + subpop_fractions=None, num_warmup: int = 500, num_samples: int = 500, num_chains: int = 1, @@ -375,10 +362,8 @@ def fit( Number of days to simulate after initialization period population_size : float Total population size for scaling infections - obs_fractions : ArrayLike - Population fractions for observed subpopulations. - unobs_fractions : ArrayLike - Population fractions for unobserved subpopulations. + subpop_fractions : ArrayLike + Population fractions for all subpopulations. Shape: (K,). num_warmup : int Number of MCMC warmup iterations (default: 500) num_samples : int @@ -408,8 +393,7 @@ def fit( """ self.validate_data( n_days_post_init=n_days_post_init, - obs_fractions=obs_fractions, - unobs_fractions=unobs_fractions, + subpop_fractions=subpop_fractions, **observation_data, ) @@ -433,8 +417,7 @@ def fit( rng_key, n_days_post_init=n_days_post_init, population_size=population_size, - obs_fractions=obs_fractions, - unobs_fractions=unobs_fractions, + subpop_fractions=subpop_fractions, **observation_data, ) diff --git a/pyrenew/model/pyrenew_builder.py b/pyrenew/model/pyrenew_builder.py index 635c3ce3..ca9688a8 100644 --- a/pyrenew/model/pyrenew_builder.py +++ b/pyrenew/model/pyrenew_builder.py @@ -15,10 +15,6 @@ # Parameters that should be passed at sample time, not configure time _SAMPLE_TIME_PARAMS = { - "obs_fractions", - "unobs_fractions", - "K", - "K_obs", "subpop_fractions", } @@ -36,9 +32,9 @@ class PyrenewBuilder: 3. Latent infections are routed to the correct observations 4. The API is clean and easy to use - Population structure (obs_fractions, unobs_fractions, K, K_obs, subpop_fractions) - is passed at sample/fit time, not at configure time. This allows a single model - to be fit to multiple jurisdictions with different population structures. + Population structure (subpop_fractions) is passed at sample/fit time, not at + configure time. This allows a single model to be fit to multiple jurisdictions + with different population structures. """ def __init__(self): @@ -71,9 +67,9 @@ def configure_latent( Parameters for latent class constructor (model structure). DO NOT include n_initialization_points - it will be computed automatically from observation processes. - DO NOT include population structure params (obs_fractions, - unobs_fractions, K, K_obs, subpop_fractions) - these are passed - at sample/fit time to allow fitting to multiple jurisdictions. + DO NOT include population structure params (subpop_fractions) - + these are passed at sample/fit time to allow fitting to multiple + jurisdictions. Returns ------- diff --git a/test/test_base_latent_process.py b/test/test_base_latent_process.py index 77cd37a8..d0aa8ef6 100644 --- a/test/test_base_latent_process.py +++ b/test/test_base_latent_process.py @@ -13,65 +13,46 @@ class TestPopulationStructureParsing: """Test _parse_and_validate_fractions static method.""" - def test_parse_obs_unobs_fractions(self): - """Test parsing obs_fractions + unobs_fractions.""" + def test_parse_subpop_fractions(self): + """Test parsing subpop_fractions.""" pop = BaseLatentInfectionProcess._parse_and_validate_fractions( - obs_fractions=jnp.array([0.3, 0.25]), - unobs_fractions=jnp.array([0.45]), + subpop_fractions=jnp.array([0.3, 0.25, 0.45]), ) assert pop.K == 3 - assert pop.K_obs == 2 - assert pop.K_unobs == 1 + assert jnp.allclose(pop.fractions, jnp.array([0.3, 0.25, 0.45])) def test_rejects_fractions_not_summing_to_one(self): """Test that fractions not summing to 1 raises error.""" with pytest.raises(ValueError, match="must sum to 1.0"): BaseLatentInfectionProcess._parse_and_validate_fractions( - obs_fractions=jnp.array([0.3, 0.25]), - unobs_fractions=jnp.array([0.40]), + subpop_fractions=jnp.array([0.3, 0.25, 0.40]), # Sum is 0.95 ) def test_rejects_negative_fractions(self): """Test that negative fractions raise error.""" with pytest.raises(ValueError, match="must be non-negative"): BaseLatentInfectionProcess._parse_and_validate_fractions( - obs_fractions=jnp.array([0.3, -0.1]), - unobs_fractions=jnp.array([0.8]), + subpop_fractions=jnp.array([0.3, -0.1, 0.8]), ) def test_rejects_missing_fractions(self): """Test that missing fractions raise error.""" - with pytest.raises( - ValueError, match="Both obs_fractions and unobs_fractions must be provided" - ): - BaseLatentInfectionProcess._parse_and_validate_fractions( - obs_fractions=jnp.array([0.3, 0.25]), - ) - - with pytest.raises( - ValueError, match="Both obs_fractions and unobs_fractions must be provided" - ): - BaseLatentInfectionProcess._parse_and_validate_fractions( - unobs_fractions=jnp.array([0.45]), - ) + with pytest.raises(ValueError, match="subpop_fractions must be provided"): + BaseLatentInfectionProcess._parse_and_validate_fractions() def test_rejects_2d_fractions(self): """Test that 2D fraction arrays raise error.""" - with pytest.raises(ValueError, match="Fractions must be 1D arrays"): + with pytest.raises(ValueError, match="must be a 1D array"): BaseLatentInfectionProcess._parse_and_validate_fractions( - obs_fractions=jnp.array([[0.3, 0.25]]), # 2D array - unobs_fractions=jnp.array([0.45]), + subpop_fractions=jnp.array([[0.3, 0.25, 0.45]]), # 2D array ) - def test_rejects_no_observed_subpopulations(self): - """Test that empty observed subpopulations raise error.""" - with pytest.raises( - ValueError, match="Must have at least one observed subpopulation" - ): + def test_rejects_empty_subpopulations(self): + """Test that empty subpopulations raise error.""" + with pytest.raises(ValueError, match="Must have at least one subpopulation"): BaseLatentInfectionProcess._parse_and_validate_fractions( - obs_fractions=jnp.array([]), # No observed subpops - unobs_fractions=jnp.array([1.0]), + subpop_fractions=jnp.array([]), ) @@ -135,25 +116,18 @@ def test_validate_output_shapes_raises_on_mismatch(self): from pyrenew.latent.base import PopulationStructure pop = PopulationStructure( - obs_fractions=jnp.array([0.5, 0.5]), - unobs_fractions=jnp.array([]), + fractions=jnp.array([0.5, 0.5]), K=2, - K_obs=2, - K_unobs=0, ) # Create arrays with wrong shapes infections_aggregate = jnp.ones(10) # Correct: (10,) - infections_all = jnp.ones((10, 2)) # Correct: (10, 2) - infections_observed = jnp.ones((10, 3)) # WRONG: should be (10, 2) - infections_unobserved = jnp.ones((10, 0)) # Correct: (10, 0) + infections_all = jnp.ones((10, 3)) # WRONG: should be (10, 2) with pytest.raises(ValueError, match="has incorrect shape"): BaseLatentInfectionProcess._validate_output_shapes( infections_aggregate, infections_all, - infections_observed, - infections_unobserved, n_total_days=10, pop=pop, ) @@ -188,15 +162,11 @@ def test_latent_sample_unpacking(self): sample = LatentSample( aggregate=jnp.ones(10), all_subpops=jnp.ones((10, 2)), - observed=jnp.ones((10, 1)), - unobserved=jnp.ones((10, 1)), ) - agg, all_s, obs, unobs = sample + agg, all_s = sample assert agg.shape == (10,) assert all_s.shape == (10, 2) - assert obs.shape == (10, 1) - assert unobs.shape == (10, 1) if __name__ == "__main__": diff --git a/test/test_hierarchical_infections.py b/test/test_hierarchical_infections.py index 385a2d63..a9c43a2c 100644 --- a/test/test_hierarchical_infections.py +++ b/test/test_hierarchical_infections.py @@ -50,34 +50,29 @@ def test_sample_returns_correct_shapes(self, process): with numpyro.handlers.seed(rng_seed=42): result = process.sample( n_days_post_init=30, - obs_fractions=jnp.array([0.3, 0.25]), - unobs_fractions=jnp.array([0.45]), + subpop_fractions=jnp.array([0.3, 0.25, 0.45]), ) - inf_juris, inf_all, inf_obs, inf_unobs = result + inf_juris, inf_all = result n_total = process.n_initialization_points + 30 assert inf_juris.shape == (n_total,) assert inf_all.shape == (n_total, 3) - assert inf_obs.shape == (n_total, 2) - assert inf_unobs.shape == (n_total, 1) def test_same_model_different_jurisdictions(self, process): """Test that one model can fit different population structures.""" - # Jurisdiction A: 2 observed, 1 unobserved + # Jurisdiction A: 3 subpopulations with numpyro.handlers.seed(rng_seed=42): - _, inf_all_a, _, _ = process.sample( + _, inf_all_a = process.sample( n_days_post_init=30, - obs_fractions=jnp.array([0.3, 0.25]), - unobs_fractions=jnp.array([0.45]), + subpop_fractions=jnp.array([0.3, 0.25, 0.45]), ) - # Jurisdiction B: 4 observed, 1 unobserved (different structure) + # Jurisdiction B: 5 subpopulations (different structure) with numpyro.handlers.seed(rng_seed=42): - _, inf_all_b, _, _ = process.sample( + _, inf_all_b = process.sample( n_days_post_init=30, - obs_fractions=jnp.array([0.15, 0.20, 0.25, 0.10]), - unobs_fractions=jnp.array([0.30]), + subpop_fractions=jnp.array([0.15, 0.20, 0.25, 0.10, 0.30]), ) assert inf_all_a.shape[1] == 3 # K=3 @@ -85,18 +80,15 @@ def test_same_model_different_jurisdictions(self, process): def test_jurisdiction_total_is_weighted_sum(self, process): """Test that jurisdiction total equals weighted sum of subpopulations.""" - obs_fracs = jnp.array([0.3, 0.25]) - unobs_fracs = jnp.array([0.45]) + fractions = jnp.array([0.3, 0.25, 0.45]) with numpyro.handlers.seed(rng_seed=42): - inf_juris, inf_all, _, _ = process.sample( + inf_juris, inf_all = process.sample( n_days_post_init=30, - obs_fractions=obs_fracs, - unobs_fractions=unobs_fracs, + subpop_fractions=fractions, ) - all_fractions = jnp.concatenate([obs_fracs, unobs_fracs]) - expected = jnp.sum(inf_all * all_fractions[jnp.newaxis, :], axis=1) + expected = jnp.sum(inf_all * fractions[jnp.newaxis, :], axis=1) assert jnp.allclose(inf_juris, expected, atol=1e-6) @@ -106,8 +98,7 @@ def test_deviations_sum_to_zero(self, process): with numpyro.handlers.trace() as trace: process.sample( n_days_post_init=30, - obs_fractions=jnp.array([0.3, 0.25]), - unobs_fractions=jnp.array([0.45]), + subpop_fractions=jnp.array([0.3, 0.25, 0.45]), ) deviations = trace["subpop_deviations"]["value"] @@ -180,8 +171,7 @@ def test_rejects_fractions_not_summing_to_one(self, process): with numpyro.handlers.seed(rng_seed=42): process.sample( n_days_post_init=30, - obs_fractions=jnp.array([0.3, 0.25]), - unobs_fractions=jnp.array([0.40]), # Sum is 0.95 + subpop_fractions=jnp.array([0.3, 0.25, 0.40]), # Sum is 0.95 ) def test_validate_method(self, process): @@ -205,11 +195,10 @@ def test_per_subpop_I0_array(self, gen_int_rv): with numpyro.handlers.seed(rng_seed=42): result = process.sample( n_days_post_init=30, - obs_fractions=jnp.array([0.3, 0.25]), - unobs_fractions=jnp.array([0.45]), + subpop_fractions=jnp.array([0.3, 0.25, 0.45]), ) - inf_juris, inf_all, _inf_obs, _inf_unobs = result + inf_juris, inf_all = result n_total = process.n_initialization_points + 30 assert inf_juris.shape == (n_total,) diff --git a/test/test_observation_counts.py b/test/test_observation_counts.py index 80f5b02c..e43b5272 100644 --- a/test/test_observation_counts.py +++ b/test/test_observation_counts.py @@ -320,6 +320,57 @@ def test_infection_resolution(self): assert process.infection_resolution() == "subpop" + def test_non_contiguous_subpop_indices(self): + """Test that non-contiguous subpop_indices work correctly. + + This verifies that observation processes can observe any subset + of subpopulations, not just contiguous indices starting from 0. + For example, with K=5 subpopulations, observations might only + cover indices {0, 2, 4} while indices {1, 3} are unobserved. + """ + delay_pmf = jnp.array([0.3, 0.4, 0.3]) + process = CountsBySubpop( + name="test", + ascertainment_rate_rv=DeterministicVariable("ihr", 1.0), + delay_distribution_rv=DeterministicPMF("delay", delay_pmf), + noise=PoissonNoise(), + ) + + # 5 subpopulations with distinct infection levels + # Subpop 0: 100, Subpop 1: 200, Subpop 2: 300, Subpop 3: 400, Subpop 4: 500 + n_days = 20 + infections = jnp.zeros((n_days, 5)) + for k in range(5): + infections = infections.at[:, k].set((k + 1) * 100.0) + + # Observe only subpops 0, 2, 4 (non-contiguous, skipping 1 and 3) + times = jnp.array([10, 10, 10]) + subpop_indices = jnp.array([0, 2, 4]) # Non-contiguous! + + with numpyro.handlers.seed(rng_seed=42): + result = process.sample( + infections=infections, + times=times, + subpop_indices=subpop_indices, + obs=None, + ) + + # Verify correct shape + assert result.observed.shape == (3,) + + # Verify the predicted values correspond to the correct subpopulations + # predicted[t, k] should reflect infections from subpop k + # At time 10, predicted counts should be proportional to infection levels + predicted_at_obs = result.predicted[10, subpop_indices] + + # Subpop 0 has 100 infections, subpop 2 has 300, subpop 4 has 500 + # So predicted[10, 0] < predicted[10, 2] < predicted[10, 4] + assert predicted_at_obs[0] < predicted_at_obs[1] < predicted_at_obs[2] + + # Verify the ratios match the infection ratios (100:300:500 = 1:3:5) + assert jnp.isclose(predicted_at_obs[1] / predicted_at_obs[0], 3.0, atol=0.01) + assert jnp.isclose(predicted_at_obs[2] / predicted_at_obs[0], 5.0, atol=0.01) + class TestPoissonNoise: """Test PoissonNoise model.""" diff --git a/test/test_observation_measurements.py b/test/test_observation_measurements.py index 14c09f44..671526e4 100644 --- a/test/test_observation_measurements.py +++ b/test/test_observation_measurements.py @@ -330,6 +330,78 @@ def test_predicted_obs_stored(self): assert "test_obs" in trace assert "test_predicted" in trace + def test_non_contiguous_subpop_indices(self): + """Test that non-contiguous subpop_indices work correctly. + + This verifies that observation processes can observe any subset + of subpopulations, not just contiguous indices starting from 0. + For example, with K=5 subpopulations, observations might only + cover indices {0, 2, 4} while indices {1, 3} are unobserved. + """ + shedding_pmf = jnp.array([0.5, 0.5]) + sensor_mode_rv = VectorizedRV( + DistributionalVariable("mode", dist.Normal(0, 0.01)), + plate_name="sensor_mode", + ) + sensor_sd_rv = VectorizedRV( + DistributionalVariable("sd", dist.TruncatedNormal(0.01, 0.005, low=0.001)), + plate_name="sensor_sd", + ) + noise = HierarchicalNormalNoise(sensor_mode_rv, sensor_sd_rv) + + process = ConcreteMeasurements( + name="test", + temporal_pmf_rv=DeterministicPMF("shedding", shedding_pmf), + noise=noise, + ) + + # 5 subpopulations with distinct infection levels + # Subpop 0: 100, Subpop 1: 200, Subpop 2: 300, Subpop 3: 400, Subpop 4: 500 + n_days = 20 + infections = jnp.zeros((n_days, 5)) + for k in range(5): + infections = infections.at[:, k].set((k + 1) * 100.0) + + # Observe only subpops 0, 2, 4 (non-contiguous, skipping 1 and 3) + # Each observation from a different sensor + times = jnp.array([10, 10, 10]) + subpop_indices = jnp.array([0, 2, 4]) # Non-contiguous! + sensor_indices = jnp.array([0, 1, 2]) + + with numpyro.handlers.seed(rng_seed=42): + result = process.sample( + infections=infections, + times=times, + subpop_indices=subpop_indices, + sensor_indices=sensor_indices, + n_sensors=3, + obs=None, + ) + + # Verify correct shape + assert result.observed.shape == (3,) + + # Verify the predicted values correspond to the correct subpopulations + # predicted[t, k] should reflect infections from subpop k + # At time 10, predicted values should be proportional to infection levels + # Note: ConcreteMeasurements returns LOG-scale values, so linear ratios + # become differences in log space + predicted_at_obs = result.predicted[10, subpop_indices] + + # Subpop 0 has 100 infections, subpop 2 has 300, subpop 4 has 500 + # In log space: log(300) - log(100) = log(3), log(500) - log(100) = log(5) + assert predicted_at_obs[0] < predicted_at_obs[1] < predicted_at_obs[2] + + # Verify the differences match log of the infection ratios + # diff[1] - diff[0] should equal log(3) ≈ 1.099 + # diff[2] - diff[0] should equal log(5) ≈ 1.609 + assert jnp.isclose( + predicted_at_obs[1] - predicted_at_obs[0], jnp.log(3.0), atol=0.01 + ) + assert jnp.isclose( + predicted_at_obs[2] - predicted_at_obs[0], jnp.log(5.0), atol=0.01 + ) + if __name__ == "__main__": pytest.main([__file__, "-v"]) diff --git a/test/test_pyrenew_builder.py b/test/test_pyrenew_builder.py index 4aea16db..403208f6 100644 --- a/test/test_pyrenew_builder.py +++ b/test/test_pyrenew_builder.py @@ -11,9 +11,8 @@ from pyrenew.model import MultiSignalModel, PyrenewBuilder from pyrenew.observation import Counts, NegativeBinomialNoise -# Standard population structure for tests -OBS_FRACTIONS = jnp.array([0.3, 0.25]) -UNOBS_FRACTIONS = jnp.array([0.45]) +# Standard population structure for tests (3 subpopulations) +SUBPOP_FRACTIONS = jnp.array([0.3, 0.25, 0.45]) @pytest.fixture @@ -66,7 +65,7 @@ def test_rejects_population_structure_at_configure_time(self): initial_log_rt_rv=DeterministicVariable("initial_log_rt", 0.0), baseline_temporal=RandomWalk(), subpop_temporal=RandomWalk(), - obs_fractions=jnp.array([0.5, 0.5]), # Should fail + subpop_fractions=jnp.array([0.5, 0.5]), # Should fail ) def test_rejects_n_initialization_points_at_configure_time(self): @@ -172,17 +171,15 @@ def test_sample_with_population_structure(self, simple_builder): n_total = model.latent.n_initialization_points + n_days with numpyro.handlers.seed(rng_seed=42): - inf_aggregate, inf_all, inf_obs, _inf_unobs = model.sample( + inf_aggregate, inf_all = model.sample( n_days_post_init=n_days, population_size=1_000_000, - obs_fractions=OBS_FRACTIONS, - unobs_fractions=UNOBS_FRACTIONS, + subpop_fractions=SUBPOP_FRACTIONS, hospital={"obs": None}, ) assert inf_aggregate.shape == (n_total,) assert inf_all.shape == (n_total, 3) # K=3 - assert inf_obs.shape == (n_total, 2) # K_obs=2 def test_fit_with_reparam_config(self, simple_builder): """Test that fit() works with reparameterization config.""" @@ -205,8 +202,7 @@ def test_fit_with_reparam_config(self, simple_builder): mcmc = model.fit( n_days_post_init=n_days, population_size=1_000_000, - obs_fractions=OBS_FRACTIONS, - unobs_fractions=UNOBS_FRACTIONS, + subpop_fractions=SUBPOP_FRACTIONS, num_warmup=2, num_samples=2, reparam_config=reparam_config, @@ -231,8 +227,7 @@ def test_fit_with_population_structure(self, simple_builder): mcmc = model.fit( n_days_post_init=n_days, population_size=1_000_000, - obs_fractions=OBS_FRACTIONS, - unobs_fractions=UNOBS_FRACTIONS, + subpop_fractions=SUBPOP_FRACTIONS, num_warmup=5, num_samples=5, hospital={"obs": obs}, @@ -253,8 +248,7 @@ def test_validate_data_requires_population_structure(self, simple_builder): # Should work with population structure model.validate_data( n_days_post_init=30, - obs_fractions=OBS_FRACTIONS, - unobs_fractions=UNOBS_FRACTIONS, + subpop_fractions=SUBPOP_FRACTIONS, hospital={ "obs": jnp.array([10, 20]), "times": jnp.array([5, 10]), @@ -269,8 +263,7 @@ def test_validate_data_rejects_out_of_bounds_times(self, simple_builder): with pytest.raises(ValueError, match="times index"): model.validate_data( n_days_post_init=30, - obs_fractions=OBS_FRACTIONS, - unobs_fractions=UNOBS_FRACTIONS, + subpop_fractions=SUBPOP_FRACTIONS, hospital={ "obs": jnp.array([10]), "times": jnp.array([n_total + 10]), @@ -284,8 +277,7 @@ def test_validate_data_rejects_negative_times(self, simple_builder): with pytest.raises(ValueError, match="times cannot be negative"): model.validate_data( n_days_post_init=30, - obs_fractions=OBS_FRACTIONS, - unobs_fractions=UNOBS_FRACTIONS, + subpop_fractions=SUBPOP_FRACTIONS, hospital={ "obs": jnp.array([10]), "times": jnp.array([-1]), @@ -299,8 +291,7 @@ def test_validate_data_rejects_unknown_observation(self, simple_builder): with pytest.raises(ValueError, match="Unknown observation"): model.validate_data( n_days_post_init=30, - obs_fractions=OBS_FRACTIONS, - unobs_fractions=UNOBS_FRACTIONS, + subpop_fractions=SUBPOP_FRACTIONS, unknown_obs={ "obs": jnp.array([10]), "times": jnp.array([5]), @@ -314,8 +305,7 @@ def test_validate_data_rejects_mismatched_obs_times_length(self, simple_builder) with pytest.raises(ValueError, match=r"obs length.*must match times length"): model.validate_data( n_days_post_init=30, - obs_fractions=OBS_FRACTIONS, - unobs_fractions=UNOBS_FRACTIONS, + subpop_fractions=SUBPOP_FRACTIONS, hospital={ "obs": jnp.array([10, 20, 30]), # 3 elements "times": jnp.array([5, 10]), # 2 elements @@ -335,8 +325,7 @@ def test_validate_data_rejects_negative_subpop_indices(self, simple_builder): with pytest.raises(ValueError, match="subpop_indices cannot be negative"): model.validate_data( n_days_post_init=30, - obs_fractions=OBS_FRACTIONS, - unobs_fractions=UNOBS_FRACTIONS, + subpop_fractions=SUBPOP_FRACTIONS, hospital={ "subpop_indices": jnp.array([-1, 0, 1]), "times": jnp.array([5, 6, 7]), @@ -344,17 +333,16 @@ def test_validate_data_rejects_negative_subpop_indices(self, simple_builder): ) def test_validate_data_rejects_out_of_bounds_subpop_indices(self, simple_builder): - """Test that subpop_indices >= K_obs raises error.""" + """Test that subpop_indices >= K raises error.""" model = simple_builder.build() - # K_obs is 2 (from OBS_FRACTIONS = [0.3, 0.25]) + # K is 3 (from SUBPOP_FRACTIONS = [0.3, 0.25, 0.45]) with pytest.raises(ValueError, match="subpop_indices contains"): model.validate_data( n_days_post_init=30, - obs_fractions=OBS_FRACTIONS, - unobs_fractions=UNOBS_FRACTIONS, + subpop_fractions=SUBPOP_FRACTIONS, hospital={ - "subpop_indices": jnp.array([0, 1, 5]), # 5 >= 2 + "subpop_indices": jnp.array([0, 1, 5]), # 5 >= 3 "times": jnp.array([5, 6, 7]), }, ) From 5c2f26d772a8fe38d1b5dce1d519d479b28a2dc3 Mon Sep 17 00:00:00 2001 From: Mitzi Morris Date: Thu, 5 Feb 2026 15:24:08 -0500 Subject: [PATCH 71/81] no default n_initialization_points for low-level classes --- .../latent_hierarchical_infections.qmd | 42 +++++++++++++++---- pyrenew/latent/base.py | 42 ++++++++++--------- pyrenew/latent/hierarchical_infections.py | 13 +++--- test/test_base_latent_process.py | 11 ++--- test/test_hierarchical_infections.py | 21 ++++++++++ 5 files changed, 90 insertions(+), 39 deletions(-) diff --git a/docs/tutorials/latent_hierarchical_infections.qmd b/docs/tutorials/latent_hierarchical_infections.qmd index f08de3b9..23cc4f55 100644 --- a/docs/tutorials/latent_hierarchical_infections.qmd +++ b/docs/tutorials/latent_hierarchical_infections.qmd @@ -162,6 +162,10 @@ gi_configs = { "Long GI (~3.5 days)": gi_long_rv, } +# Use consistent n_initialization_points across all GI configs +# so the temporal process trajectories are comparable (same seed → same Rt) +n_init_gi = max(len(rv()) for rv in gi_configs.values()) + for name, gi_rv in gi_configs.items(): model = HierarchicalInfections( gen_int_rv=gi_rv, @@ -169,6 +173,7 @@ for name, gi_rv in gi_configs.items(): initial_log_rt_rv=DeterministicVariable("initial_log_rt", 0.0), baseline_temporal=baseline_temporal, subpop_temporal=subpop_temporal, + n_initialization_points=n_init_gi, ) with numpyro.handlers.seed(rng_seed=42): inf_agg, _ = model.sample( @@ -198,6 +203,7 @@ for init_rt_name, init_rt_val in [ initial_log_rt_rv=DeterministicVariable("initial_log_rt", init_rt_val), baseline_temporal=baseline_temporal, subpop_temporal=subpop_temporal, + n_initialization_points=n_init_gi, ) with numpyro.handlers.seed(rng_seed=42): inf_agg, _ = model.sample( @@ -221,19 +227,24 @@ gi_df = pd.DataFrame(gi_data) ( p9.ggplot( gi_df, - p9.aes(x="day", y="infections", color="config", linetype="linetype"), + p9.aes(x="day", y="infections", color="config", linetype="config"), ) + p9.geom_line(size=1) + p9.scale_linetype_manual( - values={"solid": "solid", "dashed": "dashed"}, - labels={"solid": "initial Rt = 1", "dashed": "initial Rt ≠ 1"}, + values={ + "Short GI (~1 day)": "solid", + "Standard GI (~2 days)": "solid", + "Long GI (~3.5 days)": "solid", + "Standard GI, Rt≈1.22 (growing)": "dashed", + "Standard GI, Rt≈0.82 (declining)": "dashed", + } ) + p9.labs( x="Days", y="Infections (proportion)", title="Effect of Generation Interval and Initial Rt", color="Configuration", - linetype="Initial Rt", + linetype="Configuration", ) + p9.theme_minimal() + p9.theme(figure_size=(10, 5), legend_position="right") @@ -249,19 +260,24 @@ gi_df_zoom = gi_df[(gi_df["day"] >= 14) & (gi_df["day"] <= 21)] ( p9.ggplot( gi_df_zoom, - p9.aes(x="day", y="infections", color="config", linetype="linetype"), + p9.aes(x="day", y="infections", color="config", linetype="config"), ) + p9.geom_line(size=1) + p9.scale_linetype_manual( - values={"solid": "solid", "dashed": "dashed"}, - labels={"solid": "initial Rt = 1", "dashed": "initial Rt ≠ 1"}, + values={ + "Short GI (~1 day)": "solid", + "Standard GI (~2 days)": "solid", + "Long GI (~3.5 days)": "solid", + "Standard GI, Rt≈1.22 (growing)": "dashed", + "Standard GI, Rt≈0.82 (declining)": "dashed", + } ) + p9.labs( x="Days", y="Infections (proportion)", title="Zoomed: Days 14-21", color="Configuration", - linetype="Initial Rt", + linetype="Configuration", ) + p9.theme_minimal() + p9.theme(figure_size=(10, 5), legend_position="right") @@ -310,6 +326,7 @@ for name, i0_val in i0_configs.items(): initial_log_rt_rv=DeterministicVariable("initial_log_rt", 0.0), baseline_temporal=baseline_temporal, subpop_temporal=subpop_temporal, + n_initialization_points=len(gen_int_pmf), ) with numpyro.handlers.seed(rng_seed=42): inf_agg, _ = model.sample( @@ -392,6 +409,7 @@ latent_infections = HierarchicalInfections( initial_log_rt_rv=initial_log_rt_rv, baseline_temporal=baseline_temporal, subpop_temporal=subpop_temporal, + n_initialization_points=len(gen_int_pmf), ) print( @@ -486,6 +504,7 @@ n_extreme = np.sum(np.any(rt_samples > rt_cap, axis=1)) p9.ggplot(rt_df, p9.aes(x="day", y="rt", group="sample")) + p9.geom_line(alpha=0.1, size=0.4, color="steelblue") + p9.geom_hline(yintercept=1.0, color="red", linetype="dashed", alpha=0.7) + + p9.scale_x_continuous(expand=(0, 0)) + p9.coord_cartesian(ylim=(0, rt_cap)) + p9.labs( x="Days", @@ -539,6 +558,7 @@ for name, temporal in temporal_configs.items(): initial_log_rt_rv=initial_log_rt_rv, baseline_temporal=temporal, subpop_temporal=AR1(autoreg=0.8, innovation_sd=0.05), + n_initialization_points=len(gen_int_pmf), ) def make_sampler(m): @@ -585,6 +605,7 @@ comparison_df["process"] = pd.Categorical( + p9.geom_line(alpha=0.1, size=0.4, color="steelblue") + p9.geom_hline(yintercept=1.0, color="red", linetype="dashed", alpha=0.7) + p9.facet_wrap("~ process", ncol=3) + + p9.scale_x_continuous(expand=(0, 0)) + p9.coord_cartesian(ylim=(0, 3.0)) + p9.labs( x="Days", @@ -635,6 +656,7 @@ for name, subpop_temporal in subpop_configs.items(): initial_log_rt_rv=initial_log_rt_rv, baseline_temporal=DifferencedAR1(autoreg=0.9, innovation_sd=0.05), subpop_temporal=subpop_temporal, + n_initialization_points=len(gen_int_pmf), ) def make_sampler(m): @@ -678,6 +700,7 @@ dev_df = pd.DataFrame(dev_data) + p9.geom_line(alpha=0.15, size=0.4, color="purple") + p9.geom_hline(yintercept=0, color="black", linetype="dotted") + p9.facet_wrap("~ process", ncol=2) + + p9.scale_x_continuous(expand=(0, 0)) + p9.labs( x="Days", y="Deviation δ₁(t)", @@ -721,6 +744,7 @@ for name, cfg in configs.items(): initial_log_rt_rv=initial_log_rt_rv, baseline_temporal=cfg["baseline"], subpop_temporal=cfg["subpop"], + n_initialization_points=len(gen_int_pmf), ) def make_sampler(m): @@ -762,6 +786,7 @@ tuning_df = pd.DataFrame(tuning_data) + p9.geom_line(alpha=0.1, size=0.4, color="steelblue") + p9.geom_hline(yintercept=1.0, color="red", linetype="dashed", alpha=0.7) + p9.facet_wrap("~ config", ncol=2) + + p9.scale_x_continuous(expand=(0, 0)) + p9.coord_cartesian(ylim=(0, 3.0)) + p9.labs( x="Days", @@ -861,6 +886,7 @@ latent = HierarchicalInfections( initial_log_rt_rv=DeterministicVariable("initial_log_rt", 0.0), baseline_temporal=DifferencedAR1(autoreg=0.9, innovation_sd=0.05), subpop_temporal=AR1(autoreg=0.8, innovation_sd=0.05), + n_initialization_points=len(gen_int_pmf), ) # Sample (population structure provided at sample time) diff --git a/pyrenew/latent/base.py b/pyrenew/latent/base.py index ba6884a9..2d07d027 100644 --- a/pyrenew/latent/base.py +++ b/pyrenew/latent/base.py @@ -70,10 +70,10 @@ class BaseLatentInfectionProcess(RandomVariable): ---------- gen_int_rv : RandomVariable Generation interval PMF - n_initialization_points : int, optional - Number of initialization days before day 0. If not specified, automatically - computed as max(21, 2*len(gen_int)-1). The PyrenewBuilder will compute and pass - the correct value based on all observation processes. + n_initialization_points : int + Number of initialization days before day 0. Must be at least + ``len(gen_int_rv())`` to provide enough history for the renewal + equation convolution. Notes ----- @@ -81,16 +81,16 @@ class BaseLatentInfectionProcess(RandomVariable): the constructor. This allows a single model instance to be fit to multiple datasets with different jurisdiction structures. - When using PyrenewBuilder, n_initialization_points is computed automatically from - all observation processes. When manually creating latent processes, you can either - specify it explicitly or let it default to max(21, 2*len(gen_int)-1). + When using PyrenewBuilder (recommended), n_initialization_points is computed + automatically from all observation processes. When constructing latent processes + directly, you must specify n_initialization_points explicitly. """ def __init__( self, *, gen_int_rv: RandomVariable, - n_initialization_points: int = None, + n_initialization_points: int, ) -> None: """ Initialize base latent infection process. @@ -99,24 +99,26 @@ def __init__( ---------- gen_int_rv : RandomVariable Generation interval PMF - n_initialization_points : int, optional - Number of initialization days before day 0. If not specified, - defaults to max(21, 2*len(gen_int)-1). + n_initialization_points : int + Number of initialization days before day 0. Must be at least + ``len(gen_int_rv())`` to provide enough history for the renewal + equation convolution. + + Raises + ------ + ValueError + If gen_int_rv is None or n_initialization_points is insufficient. """ if gen_int_rv is None: raise ValueError("gen_int_rv is required") self.gen_int_rv = gen_int_rv - # Compute n_initialization_points if not provided - # Use max(21, 2*len(gen_int)-1) for conservative default - if n_initialization_points is None: - gen_int_length = len(self.gen_int_rv()) - n_initialization_points = max(21, 2 * gen_int_length - 1) - - if n_initialization_points < 0: + gen_int_length = len(self.gen_int_rv()) + if n_initialization_points < gen_int_length: raise ValueError( - f"n_initialization_points must be non-negative, " - f"got {n_initialization_points}" + f"n_initialization_points must be at least the generation " + f"interval length ({gen_int_length}), got " + f"{n_initialization_points}" ) self.n_initialization_points = n_initialization_points diff --git a/pyrenew/latent/hierarchical_infections.py b/pyrenew/latent/hierarchical_infections.py index a10086b9..10c3f965 100644 --- a/pyrenew/latent/hierarchical_infections.py +++ b/pyrenew/latent/hierarchical_infections.py @@ -57,9 +57,10 @@ class HierarchicalInfections(BaseLatentInfectionProcess): initial_log_rt_rv : RandomVariable Initial value for log(Rt) at time 0. Can be estimated from data or given a prior distribution. - n_initialization_points : int, optional - Number of initialization days before day 0. If not specified, defaults to - max(21, 2*len(gen_int)-1). When using PyrenewBuilder, this is computed + n_initialization_points : int + Number of initialization days before day 0. Must be at least + ``len(gen_int_rv())`` to provide enough history for the renewal + equation convolution. When using PyrenewBuilder, this is computed automatically from all observation processes. Notes @@ -68,7 +69,7 @@ class HierarchicalInfections(BaseLatentInfectionProcess): mean of subpopulation Rt values, providing identifiability. When using PyrenewBuilder (recommended), n_initialization_points is computed - automatically from all observation processes and should not be specified manually. + automatically from all observation processes. """ def __init__( @@ -79,7 +80,7 @@ def __init__( baseline_temporal: TemporalProcess, subpop_temporal: TemporalProcess, initial_log_rt_rv: RandomVariable, - n_initialization_points: int = None, + n_initialization_points: int, ) -> None: """ Initialize hierarchical infections process. @@ -96,7 +97,7 @@ def __init__( Temporal process for subpopulation deviations initial_log_rt_rv : RandomVariable Initial value for log(Rt) at time 0. - n_initialization_points : int, optional + n_initialization_points : int Number of initialization days before day 0. Raises diff --git a/test/test_base_latent_process.py b/test/test_base_latent_process.py index d0aa8ef6..8cb52735 100644 --- a/test/test_base_latent_process.py +++ b/test/test_base_latent_process.py @@ -70,12 +70,12 @@ def validate(self): def sample(self, n_days_post_init, **kwargs): pass - ConcreteLatent(gen_int_rv=None) + ConcreteLatent(gen_int_rv=None, n_initialization_points=3) - def test_rejects_negative_n_initialization_points(self): - """Test that negative n_initialization_points is rejected.""" + def test_rejects_insufficient_n_initialization_points(self): + """Test that n_initialization_points < gen_int length is rejected.""" with pytest.raises( - ValueError, match="n_initialization_points must be non-negative" + ValueError, match="n_initialization_points must be at least" ): class ConcreteLatent(BaseLatentInfectionProcess): @@ -87,7 +87,7 @@ def sample(self, n_days_post_init, **kwargs): ConcreteLatent( gen_int_rv=DeterministicPMF("gen_int", jnp.array([0.2, 0.5, 0.3])), - n_initialization_points=-1, + n_initialization_points=2, # gen_int has length 3 ) @@ -103,6 +103,7 @@ def test_get_required_lookback_returns_gen_int_length(self): initial_log_rt_rv=DeterministicVariable("initial_log_rt", 0.0), baseline_temporal=RandomWalk(), subpop_temporal=RandomWalk(), + n_initialization_points=3, ) assert process.get_required_lookback() == 3 diff --git a/test/test_hierarchical_infections.py b/test/test_hierarchical_infections.py index a9c43a2c..871c98a0 100644 --- a/test/test_hierarchical_infections.py +++ b/test/test_hierarchical_infections.py @@ -39,6 +39,7 @@ def process(gen_int_rv): initial_log_rt_rv=DeterministicVariable("initial_log_rt", 0.0), baseline_temporal=RandomWalk(), subpop_temporal=RandomWalk(), + n_initialization_points=3, ) @@ -119,6 +120,7 @@ def test_rejects_missing_I0_rv(self, gen_int_rv): initial_log_rt_rv=DeterministicVariable("initial_log_rt", 0.0), baseline_temporal=RandomWalk(), subpop_temporal=RandomWalk(), + n_initialization_points=3, ) def test_rejects_missing_initial_log_rt_rv(self, gen_int_rv): @@ -130,6 +132,7 @@ def test_rejects_missing_initial_log_rt_rv(self, gen_int_rv): initial_log_rt_rv=None, baseline_temporal=RandomWalk(), subpop_temporal=RandomWalk(), + n_initialization_points=3, ) def test_rejects_missing_baseline_temporal(self, gen_int_rv): @@ -141,6 +144,7 @@ def test_rejects_missing_baseline_temporal(self, gen_int_rv): initial_log_rt_rv=DeterministicVariable("initial_log_rt", 0.0), baseline_temporal=None, subpop_temporal=RandomWalk(), + n_initialization_points=3, ) def test_rejects_missing_subpop_temporal(self, gen_int_rv): @@ -152,6 +156,7 @@ def test_rejects_missing_subpop_temporal(self, gen_int_rv): initial_log_rt_rv=DeterministicVariable("initial_log_rt", 0.0), baseline_temporal=RandomWalk(), subpop_temporal=None, + n_initialization_points=3, ) def test_rejects_invalid_I0(self, gen_int_rv): @@ -163,6 +168,21 @@ def test_rejects_invalid_I0(self, gen_int_rv): initial_log_rt_rv=DeterministicVariable("initial_log_rt", 0.0), baseline_temporal=RandomWalk(), subpop_temporal=RandomWalk(), + n_initialization_points=3, + ) + + def test_rejects_insufficient_n_initialization_points(self, gen_int_rv): + """Test that n_initialization_points < gen_int length is rejected.""" + with pytest.raises( + ValueError, match="n_initialization_points must be at least" + ): + HierarchicalInfections( + gen_int_rv=gen_int_rv, + I0_rv=DeterministicVariable("I0", 0.001), + initial_log_rt_rv=DeterministicVariable("initial_log_rt", 0.0), + baseline_temporal=RandomWalk(), + subpop_temporal=RandomWalk(), + n_initialization_points=2, # gen_int has length 3 ) def test_rejects_fractions_not_summing_to_one(self, process): @@ -190,6 +210,7 @@ def test_per_subpop_I0_array(self, gen_int_rv): initial_log_rt_rv=DeterministicVariable("initial_log_rt", 0.0), baseline_temporal=RandomWalk(), subpop_temporal=RandomWalk(), + n_initialization_points=3, ) with numpyro.handlers.seed(rng_seed=42): From ba3de783a4693ef38ae9f692c55b58d0099f287a Mon Sep 17 00:00:00 2001 From: Mitzi Morris Date: Thu, 5 Feb 2026 15:32:06 -0500 Subject: [PATCH 72/81] wordsmithing --- .../tutorials/building_multisignal_models.qmd | 31 +++++++++++-------- 1 file changed, 18 insertions(+), 13 deletions(-) diff --git a/docs/tutorials/building_multisignal_models.qmd b/docs/tutorials/building_multisignal_models.qmd index 7a4b844c..9438e438 100644 --- a/docs/tutorials/building_multisignal_models.qmd +++ b/docs/tutorials/building_multisignal_models.qmd @@ -121,36 +121,41 @@ This tutorial demonstrates building a multi-signal renewal model using: ## Model Structure -Before diving into implementation, let's visualize how data flows through a multi-signal renewal model. The diagram below shows the generative process: latent infections flow from the renewal equation to observation processes, which transform them into predicted observations. +In this tutorial, we build a model that jointly fits two data streams to a shared latent infection process: + +- **Hospital admissions** — jurisdiction-level counts that reflect *total* infections across all subpopulations, delayed and underascertained +- **Wastewater concentrations** — site-level measurements from a subset of subpopulations (catchment areas), reflecting viral shedding and dilution + +The diagram below shows how data flows through the model. The latent process generates infection trajectories for all subpopulations. Each observation process receives the infections it needs — aggregated totals or per-subpopulation arrays — and transforms them into predicted observations via delays, ascertainment, shedding kinetics, and noise. ```mermaid flowchart TB subgraph Latent["Latent Infection Process"] - L[HierarchicalInfections] + L["Renewal equation
(HierarchicalInfections)"] end - subgraph Infections["Infection Arrays"] - J["infections_aggregate (T,)"] - S["infections_all_subpops (T, K)"] + subgraph Infections["Infection Trajectories"] + J["Jurisdiction total
(summed across subpopulations)"] + S["Per-subpopulation infections
(all K subpopulations)"] end subgraph Obs["Observation Processes"] - C["Counts (name='hospital')"] - W["Wastewater (name='wastewater')"] + C["Hospital admissions
(Counts)"] + W["Wastewater concentrations
(Measurements)"] end subgraph Data["Observed Data"] - HA["hospital_obs"] - WW["wastewater_obs"] + HA["Reported admissions"] + WW["Measured viral concentrations"] end L --> J L --> S - J -->|"aggregate"| C - S -->|"subpop + subpop_indices"| W - C -->|"ascertainment × delay"| HA - W -->|"shedding × genome × dilution"| WW + J --> C + S -->|"select monitored subpopulations"| W + C -->|"delay + ascertainment + noise"| HA + W -->|"shedding + dilution + noise"| WW ``` ### Infection Resolution From e0fd3438693f19c19f6d487cb0f632c3adcd7f6a Mon Sep 17 00:00:00 2001 From: Mitzi Morris Date: Thu, 5 Feb 2026 15:54:58 -0500 Subject: [PATCH 73/81] changes per code review --- pyrenew/datasets/hospital_admissions_data/README.md | 4 ++-- pyrenew/datasets/wastewater_nwss_data/README.md | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/pyrenew/datasets/hospital_admissions_data/README.md b/pyrenew/datasets/hospital_admissions_data/README.md index a6db74b1..a5e5c9a7 100644 --- a/pyrenew/datasets/hospital_admissions_data/README.md +++ b/pyrenew/datasets/hospital_admissions_data/README.md @@ -2,8 +2,8 @@ ## Provenance -- **Source Repository**: [CDC cfa-forecast-renewal-ww](https://github.com/cdcgov/cfa-forecast-renewal-ww) -- **Original Location**: `cfaforecastrenewalww/inst/testdata/` +- **Source Repository**: [CDCgov/wastewater-informed-covid-forecasting](https://github.com/CDCgov/wastewater-informed-covid-forecasting) +- **Original Location**: [`cfaforecastrenewalww/inst/testdata/`](https://github.com/CDCgov/wastewater-informed-covid-forecasting/blob/7c30b603f5e57f2b728f23506fe7714d576c31c9/cfaforecastrenewalww/inst/testdata/2023-11-06.csv) - **License**: Public Domain (CC0 1.0 Universal) - U.S. Government work - **Date Extracted**: January 2025 diff --git a/pyrenew/datasets/wastewater_nwss_data/README.md b/pyrenew/datasets/wastewater_nwss_data/README.md index 1a3a242d..1a8c80f6 100644 --- a/pyrenew/datasets/wastewater_nwss_data/README.md +++ b/pyrenew/datasets/wastewater_nwss_data/README.md @@ -2,8 +2,8 @@ ## Provenance -- **Source Repository**: [CDC cfa-forecast-renewal-ww](https://github.com/cdcgov/cfa-forecast-renewal-ww) -- **Original Location**: `cfaforecastrenewalww/inst/testdata/` +- **Source Repository**: [CDCgov/wastewater-informed-covid-forecasting](https://github.com/CDCgov/wastewater-informed-covid-forecasting) +- **Original Location**: [`cfaforecastrenewalww/inst/testdata/`](https://github.com/CDCgov/wastewater-informed-covid-forecasting/blob/7c30b603f5e57f2b728f23506fe7714d576c31c9/cfaforecastrenewalww/inst/testdata/fake_nwss.csv) - **License**: Public Domain (CC0 1.0 Universal) - U.S. Government work - **Date Extracted**: January 2025 From 470ce7ad1df932b220f9e22107e92c2b543a05c9 Mon Sep 17 00:00:00 2001 From: Mitzi Morris Date: Thu, 5 Feb 2026 23:34:31 -0500 Subject: [PATCH 74/81] Update pyrenew/observation/noise.py Co-authored-by: Dylan H. Morris --- pyrenew/observation/noise.py | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/pyrenew/observation/noise.py b/pyrenew/observation/noise.py index 47b86838..4ea2b763 100644 --- a/pyrenew/observation/noise.py +++ b/pyrenew/observation/noise.py @@ -189,13 +189,7 @@ def sample( ArrayLike Poisson-distributed counts. """ - if mask is None: - return numpyro.sample( - name, - dist.Poisson(rate=predicted + _EPSILON), - obs=obs, - ) - with numpyro.handlers.mask(mask=mask): + with numpyro.handlers.mask(mask=True if mask is None else mask): return numpyro.sample( name, dist.Poisson(rate=predicted + _EPSILON), From 25335a2f1d48a4c2c49dcc219ee7c944d8d9179d Mon Sep 17 00:00:00 2001 From: Mitzi Morris Date: Thu, 5 Feb 2026 23:35:02 -0500 Subject: [PATCH 75/81] Update pyrenew/observation/noise.py Co-authored-by: Dylan H. Morris --- pyrenew/observation/noise.py | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/pyrenew/observation/noise.py b/pyrenew/observation/noise.py index 4ea2b763..7d0f0f0e 100644 --- a/pyrenew/observation/noise.py +++ b/pyrenew/observation/noise.py @@ -275,16 +275,7 @@ def sample( Negative Binomial-distributed counts. """ concentration = self.concentration_rv() - if mask is None: - return numpyro.sample( - name, - dist.NegativeBinomial2( - mean=predicted + _EPSILON, - concentration=concentration, - ), - obs=obs, - ) - with numpyro.handlers.mask(mask=mask): + with numpyro.handlers.mask(mask=True if mask is None else mask): return numpyro.sample( name, dist.NegativeBinomial2( From 587d432f0ad6576b989aa121351de8607dd9bb99 Mon Sep 17 00:00:00 2001 From: Mitzi Morris Date: Thu, 5 Feb 2026 23:53:25 -0500 Subject: [PATCH 76/81] Update pyrenew/model/pyrenew_builder.py Co-authored-by: Dylan H. Morris --- pyrenew/model/pyrenew_builder.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/pyrenew/model/pyrenew_builder.py b/pyrenew/model/pyrenew_builder.py index ca9688a8..1682123f 100644 --- a/pyrenew/model/pyrenew_builder.py +++ b/pyrenew/model/pyrenew_builder.py @@ -233,8 +233,7 @@ def build(self): n_init = self.compute_n_initialization_points() # Construct latent process with computed n_initialization_points - latent_params = self.latent_params.copy() - latent_params["n_initialization_points"] = n_init + latent_params = {**self.latent_params, "n_initialization_points": n_init} try: latent_process = self.latent_class(**latent_params) From f5b8545e3236a830eb656141ed83fbed032dc759 Mon Sep 17 00:00:00 2001 From: Mitzi Morris Date: Thu, 5 Feb 2026 23:53:44 -0500 Subject: [PATCH 77/81] Update pyrenew/model/pyrenew_builder.py Co-authored-by: Dylan H. Morris --- pyrenew/model/pyrenew_builder.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyrenew/model/pyrenew_builder.py b/pyrenew/model/pyrenew_builder.py index 1682123f..9d99505d 100644 --- a/pyrenew/model/pyrenew_builder.py +++ b/pyrenew/model/pyrenew_builder.py @@ -197,7 +197,7 @@ def compute_n_initialization_points(self) -> int: # Formula: max(all lookbacks) # For generation interval (1-indexed): L-element PMF has max lag L days → need L init points # For delay distributions (0-indexed): L-element PMF has max delay L-1 days - # We need at least max(all PMF lengths) to satisfy the renewal equation extraction + # We need at least max(all lookbacks) to satisfy the renewal equation extraction n_init = max(lookbacks) return n_init From 951d128e322f7e1302956ef5341acde47d81f374 Mon Sep 17 00:00:00 2001 From: Mitzi Morris Date: Thu, 5 Feb 2026 23:59:58 -0500 Subject: [PATCH 78/81] Code review changes: rename processes, reuse RandomWalk/PMF validation, remove fit(), remove sample() return - Rename baseline_temporal -> baseline_rt_process, subpop_temporal -> subpop_rt_deviation_process - Refactor RandomWalk temporal process to wrap pyrenew.process.RandomWalk - Replace custom PMF validation with validate_discrete_dist_vector - Remove MultiSignalModel.fit() in favor of Model.run() - Remove return value from MultiSignalModel.sample() (use Predictive/deterministic sites) - Remove unreachable lookback_days defensive check (enforced by ABC) - Update tests and tutorials Co-Authored-By: Claude Opus 4.6 --- .../tutorials/building_multisignal_models.qmd | 100 +++++------ .../latent_hierarchical_infections.qmd | 60 +++---- pyrenew/latent/base.py | 28 ++-- pyrenew/latent/hierarchical_infections.py | 81 ++++----- pyrenew/latent/temporal_processes.py | 41 +++-- pyrenew/model/multisignal_model.py | 147 +++------------- pyrenew/model/pyrenew_builder.py | 9 +- test/test_base_latent_process.py | 7 +- test/test_hierarchical_infections.py | 46 ++--- test/test_pyrenew_builder.py | 158 ++++-------------- 10 files changed, 237 insertions(+), 440 deletions(-) diff --git a/docs/tutorials/building_multisignal_models.qmd b/docs/tutorials/building_multisignal_models.qmd index 9438e438..1a09163b 100644 --- a/docs/tutorials/building_multisignal_models.qmd +++ b/docs/tutorials/building_multisignal_models.qmd @@ -23,7 +23,6 @@ jupyter: # | label: setup # | output: false import numpyro -from numpyro.infer.reparam import LocScaleReparam # to run samplers in parallel you must run `set_host_device_count` before importing jax numpyro.set_host_device_count(4) @@ -137,7 +136,7 @@ flowchart TB subgraph Infections["Infection Trajectories"] J["Jurisdiction total
(summed across subpopulations)"] - S["Per-subpopulation infections
(all K subpopulations)"] + S["Per-subpopulation infections
(all subpopulations)"] end subgraph Obs["Observation Processes"] @@ -165,7 +164,7 @@ Different observation processes observe different levels of the model hierarchy. | Resolution | Receives | Example signals | |----------------------|-------------------|-------------------------------| | `"aggregate"` | Aggregated infections (sum across all subpopulations), shape `(T,)` | Hospital admissions, case counts | -| `"subpop"` | Infection matrix for all K subpopulations, shape `(T, K)` | Wastewater, site-specific surveillance | +| `"subpop"` | Infection matrix for all subpopulations, shape `(T, n_subpops)` | Wastewater, site-specific surveillance | The `PyrenewBuilder` routes latent infections to observation processes based on each process's declared resolution. @@ -218,18 +217,18 @@ initial_log_rt_rv = DeterministicVariable("initial_log_rt", 0.0) We configure two temporal processes: -- **Jurisdiction-level** (`baseline_temporal`): AR(1) process for the baseline $R(t)$ -- **Subpopulation-level** (`subpop_temporal`): RandomWalk for subpopulation deviations +- **Jurisdiction-level** (`baseline_rt_process`): AR(1) process for the baseline $R(t)$ +- **Subpopulation-level** (`subpop_rt_deviation_process`): RandomWalk for subpopulation deviations The RandomWalk allows flexible evolution of subpopulation-specific transmission without mean reversion. ```{python} # | label: temporal-processes # AR1 provides mean-reverting behavior for baseline Rt -baseline_temporal = AR1(autoreg=0.9, innovation_sd=0.05) +baseline_rt_process = AR1(autoreg=0.9, innovation_sd=0.05) # RandomWalk allows flexible subpopulation deviations -subpop_temporal = RandomWalk(innovation_sd=0.025) +subpop_rt_deviation_process = RandomWalk(innovation_sd=0.025) ``` ## Observation Processes @@ -246,7 +245,7 @@ Observation processes transform latent infections into observable signals and de Each observation process requires a `name` parameter—a short, meaningful identifier like `"hospital"` or `"wastewater"`. This name serves as the single identifier for the signal throughout the model: - **Numpyro sites**: Prefixes all sample and deterministic sites (e.g., `hospital_obs`, `hospital_predicted`) -- **Data binding**: Becomes the keyword argument for passing data to `model.fit()` (e.g., `hospital={...}`) +- **Data binding**: Becomes the keyword argument for passing data to `model.run()` (e.g., `hospital={...}`) This unified naming provides several benefits: @@ -529,14 +528,14 @@ builder.configure_latent( gen_int_rv=gen_int_rv, I0_rv=I0_rv, initial_log_rt_rv=initial_log_rt_rv, - baseline_temporal=baseline_temporal, - subpop_temporal=subpop_temporal, + baseline_rt_process=baseline_rt_process, + subpop_rt_deviation_process=subpop_rt_deviation_process, ) ``` ### Specifying the Observation Processes, Data -Each observation process's `name` attribute becomes the keyword used to pass that observation's data to `model.fit()` (e.g., `hospital={...}`, `wastewater={...}`). +Each observation process's `name` attribute becomes the keyword used to pass that observation's data to `model.run()` (e.g., `hospital={...}`, `wastewater={...}`). ```{python} # | label: add-observations-build @@ -551,9 +550,9 @@ print(f" Latent process: {type(model.latent).__name__}") print(f" Observation processes: {list(model.observations.keys())}") ``` -## Fitting the Model to Data: `model.fit()` +## Fitting the Model to Data: `model.run()` -When you call `model.fit()`, you supply two types of information: +When you call `model.run()`, you supply two types of information: - **Observation data** — one data dictionary per registered observation process - **Population structure** — how the jurisdiction is divided into subpopulations @@ -573,7 +572,7 @@ The model provides helper methods to align your data with this shared axis: ### Observation Data by Signal Type -Each observation process's `name` attribute becomes the keyword argument for passing data to `model.fit()`: +Each observation process's `name` attribute becomes the keyword argument for passing data to `model.run()`: ```python builder.add_observation(hosp_obs) # hosp_obs.name="hospital" → hospital={...} @@ -608,70 +607,73 @@ wastewater={ The `shift_times` method adds `n_init` to convert from natural coordinates (0 = first observation day) to the shared time axis. -**Understanding `subpop_indices`**: The latent process generates infections for all K subpopulations as a matrix of shape `(T, K)`. Each observation selects which column (subpopulation) it came from using `subpop_indices`. This is how observation processes "know" which subpopulations they observe—the user specifies this mapping at sample/fit time. +**Understanding `subpop_indices`**: The latent process generates infections for all subpopulations as a matrix of shape `(T, n_subpops)`. Each observation selects which column (subpopulation) it came from using `subpop_indices`. This is how observation processes "know" which subpopulations they observe—the user specifies this mapping at sample/run time. A **subpopulation** is a portion of the jurisdiction's population (e.g., a catchment area). A **sensor** is a measurement source — typically a WWTP/lab pair — that produces observations. Multiple sensors can observe the same subpopulation (e.g., different labs processing samples from the same catchment), so `subpop_indices` and `sensor_indices` may differ. -- `subpop_indices` links each observation to the appropriate infection column (0-indexed into the K subpopulations) +- `subpop_indices` links each observation to the appropriate infection column (0-indexed into the subpopulations) - `sensor_indices` selects which sensor's noise parameters (mode and sd) to apply **Example**: A jurisdiction has 6 subpopulations (indices 0-5), where 5 have wastewater monitoring and 1 does not. The `subpop_fractions` array has 6 elements. If subpopulation 2 lacks wastewater monitoring, wastewater observations would have `subpop_indices` values only in {0, 1, 3, 4, 5}—never 2. The monitored subpopulations need not be contiguous. The latent process still generates infections for all 6 subpopulations; the wastewater observation just doesn't see subpopulation 2. ### Population Structure -Population structure is specified via a single array of fractions for all K subpopulations: +Population structure is specified via a single array of fractions for all subpopulations: ```python -model.fit( - subpop_fractions=jnp.array([0.10, 0.14, 0.21, 0.22, 0.07, 0.26]), # 6 subpopulations - ... +model.run( + ..., + subpop_fractions=jnp.array([...]), # one fraction per subpopulation, must sum to 1 ) ``` This specifies 6 subpopulations with their population fractions. The fractions must sum to 1.0. The latent process generates infections for all 6 subpopulations. -Which subpopulations each observation process "sees" is determined by the `subpop_indices` in the observation data, not by the population structure. For example, if wastewater monitoring covers only 5 of the 6 subpopulations (say, all except subpopulation 2), the wastewater observation data would have `subpop_indices` values in {0, 1, 3, 4, 5} but never 2. The monitored subpopulations can be any subset of {0, ..., K-1}. +Which subpopulations each observation process "sees" is determined by the `subpop_indices` in the observation data, not by the population structure. For example, if wastewater monitoring covers only 5 of the 6 subpopulations (say, all except subpopulation 2), the wastewater observation data would have `subpop_indices` values in {0, 1, 3, 4, 5} but never 2. The monitored subpopulations can be any subset of {0, ..., n_subpops-1}. -### Example `model.fit()` Call +### Example `model.run()` Call ```python -model.fit( +model.run( + num_warmup=500, + num_samples=500, + mcmc_args={"num_chains": 4, "progress_bar": False}, + # Model arguments (passed through to sample()) n_days_post_init=n_days, population_size=population_size, - # Population structure (all K subpopulations) subpop_fractions=subpop_fractions, - # Observation data (keyed by names from add_observation) hospital={"obs": model.pad_observations(hosp_counts)}, wastewater={ "obs": ww_conc, "times": model.shift_times(ww_times), - "subpop_indices": ww_subpop_indices, # selects which of K subpops each obs came from + "subpop_indices": ww_subpop_indices, "sensor_indices": ww_sensor_indices, "n_sensors": n_ww_sensors, }, ) +samples = model.mcmc.get_samples() ``` ## Running the Model -First we declare the population structure. We have 6 subpopulations, where 5 have wastewater monitoring and 1 does not. The subpopulations with wastewater monitoring need not be contiguous indices—they could be any subset of {0, 1, ..., K-1}. +First we declare the population structure. We have 6 subpopulations, where 5 have wastewater monitoring and 1 does not. The subpopulations with wastewater monitoring need not be contiguous indices—they could be any subset of {0, 1, ..., n_subpops-1}. ```{python} # | label: population-structure # All 6 subpopulations with their population fractions subpop_fractions = jnp.array([0.10, 0.14, 0.21, 0.22, 0.07, 0.26]) -K = len(subpop_fractions) +n_subpops = len(subpop_fractions) # Which subpopulations have wastewater monitoring? # These indices will be used in subpop_indices for wastewater observations. -# They can be any subset of {0, 1, ..., K-1}, not necessarily contiguous. +# They can be any subset of {0, 1, ..., n_subpops-1}, not necessarily contiguous. ww_monitored_subpops = jnp.array( [0, 1, 2, 3, 4] ) # subpop 5 has no wastewater monitoring -print(f"Total subpopulations (K): {K}") +print(f"Total subpopulations: {n_subpops}") print( f"Subpopulations with wastewater monitoring: {list(ww_monitored_subpops)}" ) @@ -747,7 +749,7 @@ def prepare_observation_data( ### Fit: 90 Days -Putting this altogether, we align the data with the model time and call `model.fit()`. +Putting this altogether, we align the data with the model time and call `model.run()`. We run 4 sampler chains. ```{python} @@ -760,32 +762,23 @@ obs_data_90 = prepare_observation_data( model, n_days_90, hosp_admits, ca_ww_data, ww_monitored_subpops ) -# Try non-centered reparameterization for hierarchical parameters -reparam_config = { - "log_rt_baseline_noise": LocScaleReparam(centered=0), - "subpop_deviations_increments": LocScaleReparam(centered=0), - "ww_site_mode": LocScaleReparam(centered=0), -} - print(f"Fitting model with {n_days_90} days of data...") print(" This may take a few minutes...") start_time = time.time() -mcmc_90 = model.fit( - n_days_post_init=n_days_90, - population_size=population_size, - subpop_fractions=subpop_fractions, +model.run( num_warmup=500, num_samples=500, - num_chains=4, rng_key=make_rng_key(), - reparam_config=reparam_config, + mcmc_args={"num_chains": 4, "progress_bar": False}, + n_days_post_init=n_days_90, + population_size=population_size, + subpop_fractions=subpop_fractions, **obs_data_90, - progress_bar=False, ) # JAX uses asynchronous dispatch, so we must block until sampling completes # to get accurate timing -samples_90 = mcmc_90.get_samples() +samples_90 = model.mcmc.get_samples() jax.block_until_ready(samples_90) elapsed_90 = time.time() - start_time print(f"Elapsed time: {elapsed_90:.1f} seconds") @@ -942,19 +935,18 @@ print(f"Fitting model with {n_days_180} days of data...") print(" This may take a few minutes...") start_time = time.time() -mcmc_180 = model.fit( - n_days_post_init=n_days_180, - population_size=population_size, - subpop_fractions=subpop_fractions, +model.run( num_warmup=1000, num_samples=500, - num_chains=4, rng_key=make_rng_key(), + mcmc_args={"num_chains": 4, "progress_bar": False}, + n_days_post_init=n_days_180, + population_size=population_size, + subpop_fractions=subpop_fractions, **obs_data_180, - progress_bar=False, ) # Block until sampling completes for accurate timing -samples_180 = mcmc_180.get_samples() +samples_180 = model.mcmc.get_samples() jax.block_until_ready(samples_180) elapsed_180 = time.time() - start_time print(f"Elapsed time: {elapsed_180:.1f} seconds") @@ -1110,7 +1102,7 @@ This tutorial demonstrated composing a multi-signal renewal model using `Pyrenew 1. **Configure latent process** (`configure_latent`): generation interval, initial infections, temporal dynamics 2. **Add observation processes** (`add_observation`): each declares its infection resolution and gets a name for data binding -3. **Build and fit** (`build`, `model.fit`): the model routes infections to observations based on resolution and runs NUTS inference +3. **Build and run** (`build`, `model.run`): the model routes infections to observations based on resolution and runs NUTS inference ### Key Concepts diff --git a/docs/tutorials/latent_hierarchical_infections.qmd b/docs/tutorials/latent_hierarchical_infections.qmd index 23cc4f55..f6b2505e 100644 --- a/docs/tutorials/latent_hierarchical_infections.qmd +++ b/docs/tutorials/latent_hierarchical_infections.qmd @@ -96,8 +96,8 @@ We define subpopulations as fractions of the total jurisdiction population: # observation process (via subpop_indices), not by the latent process. subpop_fractions = jnp.array([0.10, 0.14, 0.21, 0.22, 0.07, 0.26]) -K = len(subpop_fractions) -print(f"Number of subpopulations (K): {K}") +n_subpops = len(subpop_fractions) +print(f"Number of subpopulations: {n_subpops}") print(f"Total population fraction: {float(jnp.sum(subpop_fractions)):.2f}") ``` @@ -149,8 +149,8 @@ To illustrate, we sample infection trajectories using each generation interval w # | fig-cap: Effect of generation interval and initial_log_rt on infection dynamics. Solid lines start at Rt=1; dashed lines show initial growth (Rt≈1.22) or decline (Rt≈0.82). # Create models with different generation intervals -baseline_temporal = DifferencedAR1(autoreg=0.9, innovation_sd=0.05) -subpop_temporal = AR1(autoreg=0.8, innovation_sd=0.05) +baseline_rt_process = DifferencedAR1(autoreg=0.9, innovation_sd=0.05) +subpop_rt_deviation_process = AR1(autoreg=0.8, innovation_sd=0.05) n_days_demo = 28 gi_data = [] @@ -171,8 +171,8 @@ for name, gi_rv in gi_configs.items(): gen_int_rv=gi_rv, I0_rv=DeterministicVariable("I0", 0.001), initial_log_rt_rv=DeterministicVariable("initial_log_rt", 0.0), - baseline_temporal=baseline_temporal, - subpop_temporal=subpop_temporal, + baseline_rt_process=baseline_rt_process, + subpop_rt_deviation_process=subpop_rt_deviation_process, n_initialization_points=n_init_gi, ) with numpyro.handlers.seed(rng_seed=42): @@ -201,8 +201,8 @@ for init_rt_name, init_rt_val in [ gen_int_rv=gen_int_rv, I0_rv=DeterministicVariable("I0", 0.001), initial_log_rt_rv=DeterministicVariable("initial_log_rt", init_rt_val), - baseline_temporal=baseline_temporal, - subpop_temporal=subpop_temporal, + baseline_rt_process=baseline_rt_process, + subpop_rt_deviation_process=subpop_rt_deviation_process, n_initialization_points=n_init_gi, ) with numpyro.handlers.seed(rng_seed=42): @@ -324,8 +324,8 @@ for name, i0_val in i0_configs.items(): gen_int_rv=gen_int_rv, I0_rv=DeterministicVariable("I0", i0_val), initial_log_rt_rv=DeterministicVariable("initial_log_rt", 0.0), - baseline_temporal=baseline_temporal, - subpop_temporal=subpop_temporal, + baseline_rt_process=baseline_rt_process, + subpop_rt_deviation_process=subpop_rt_deviation_process, n_initialization_points=len(gen_int_pmf), ) with numpyro.handlers.seed(rng_seed=42): @@ -388,10 +388,10 @@ Now let's assemble all the inputs: ```{python} # | label: temporal-config # Baseline Rt: DifferencedAR1 allows trends while stabilizing growth rate -baseline_temporal = DifferencedAR1(autoreg=0.9, innovation_sd=0.05) +baseline_rt_process = DifferencedAR1(autoreg=0.9, innovation_sd=0.05) # Subpopulation deviations: AR1 shrinks differences back toward zero -subpop_temporal = AR1(autoreg=0.8, innovation_sd=0.05) +subpop_rt_deviation_process = AR1(autoreg=0.8, innovation_sd=0.05) ``` **Why these choices?** @@ -407,8 +407,8 @@ latent_infections = HierarchicalInfections( gen_int_rv=gen_int_rv, I0_rv=I0_rv, initial_log_rt_rv=initial_log_rt_rv, - baseline_temporal=baseline_temporal, - subpop_temporal=subpop_temporal, + baseline_rt_process=baseline_rt_process, + subpop_rt_deviation_process=subpop_rt_deviation_process, n_initialization_points=len(gen_int_pmf), ) @@ -467,8 +467,8 @@ n_samples = 200 predictive = Predictive(sample_model, num_samples=n_samples) prior_samples = predictive(random.PRNGKey(42)) -print(f"Rt baseline samples: {prior_samples['rt_baseline'].shape}") -print(f"Infection samples: {prior_samples['infections_aggregate'].shape}") +print(f"Rt baseline samples: {prior_samples['latent_infections/rt_baseline'].shape}") +print(f"Infection samples: {prior_samples['latent_infections/infections_aggregate'].shape}") ``` ### Visualizing the Prior: Baseline Rt Trajectories @@ -476,7 +476,7 @@ print(f"Infection samples: {prior_samples['infections_aggregate'].shape}") ```{python} # | label: rt-trajectories-data # Extract post-initialization Rt trajectories -rt_samples = np.array(prior_samples["rt_baseline"])[:, n_init:, 0] +rt_samples = np.array(prior_samples["latent_infections/rt_baseline"])[:, n_init:, 0] # Build long-format dataframe for plotting rt_data = [] @@ -556,8 +556,8 @@ for name, temporal in temporal_configs.items(): gen_int_rv=gen_int_rv, I0_rv=I0_rv, initial_log_rt_rv=initial_log_rt_rv, - baseline_temporal=temporal, - subpop_temporal=AR1(autoreg=0.8, innovation_sd=0.05), + baseline_rt_process=temporal, + subpop_rt_deviation_process=AR1(autoreg=0.8, innovation_sd=0.05), n_initialization_points=len(gen_int_pmf), ) @@ -570,7 +570,7 @@ for name, temporal in temporal_configs.items(): samples = Predictive(make_sampler(model), num_samples=n_samples)( random.PRNGKey(42) ) - process_samples[name] = np.array(samples["rt_baseline"])[:, n_init:, 0] + process_samples[name] = np.array(samples["latent_infections/rt_baseline"])[:, n_init:, 0] ``` ```{python} @@ -649,13 +649,13 @@ subpop_configs = { } deviation_samples = {} -for name, subpop_temporal in subpop_configs.items(): +for name, subpop_rt_deviation_process in subpop_configs.items(): model = HierarchicalInfections( gen_int_rv=gen_int_rv, I0_rv=I0_rv, initial_log_rt_rv=initial_log_rt_rv, - baseline_temporal=DifferencedAR1(autoreg=0.9, innovation_sd=0.05), - subpop_temporal=subpop_temporal, + baseline_rt_process=DifferencedAR1(autoreg=0.9, innovation_sd=0.05), + subpop_rt_deviation_process=subpop_rt_deviation_process, n_initialization_points=len(gen_int_pmf), ) @@ -675,7 +675,7 @@ for name, subpop_temporal in subpop_configs.items(): # Plot deviations for first subpopulation dev_data = [] for name in subpop_configs.keys(): - devs = np.array(deviation_samples[name]["subpop_deviations"])[ + devs = np.array(deviation_samples[name]["latent_infections/subpop_deviations"])[ :, n_init:, 0 ] for i in range(n_samples): @@ -742,8 +742,8 @@ for name, cfg in configs.items(): gen_int_rv=gen_int_rv, I0_rv=I0_rv, initial_log_rt_rv=initial_log_rt_rv, - baseline_temporal=cfg["baseline"], - subpop_temporal=cfg["subpop"], + baseline_rt_process=cfg["baseline"], + subpop_rt_deviation_process=cfg["subpop"], n_initialization_points=len(gen_int_pmf), ) @@ -763,7 +763,7 @@ for name, cfg in configs.items(): # Build comparison dataframe tuning_data = [] for name, samples in tuning_samples.items(): - rt = np.array(samples["rt_baseline"])[:, n_init:, 0] + rt = np.array(samples["latent_infections/rt_baseline"])[:, n_init:, 0] for i in range(n_samples): for d in range(n_days): tuning_data.append( @@ -819,7 +819,7 @@ with numpyro.handlers.seed(rng_seed=123): subpop_fractions=subpop_fractions, ) -deviations = trace["subpop_deviations"]["value"] +deviations = trace["latent_infections/subpop_deviations"]["value"] deviation_sums = jnp.sum(deviations, axis=1) print( f"Max |sum of deviations|: {float(jnp.max(jnp.abs(deviation_sums))):.2e}" @@ -884,8 +884,8 @@ latent = HierarchicalInfections( gen_int_rv=DeterministicPMF("gen_int", gen_int_pmf), I0_rv=DeterministicVariable("I0", 0.001), initial_log_rt_rv=DeterministicVariable("initial_log_rt", 0.0), - baseline_temporal=DifferencedAR1(autoreg=0.9, innovation_sd=0.05), - subpop_temporal=AR1(autoreg=0.8, innovation_sd=0.05), + baseline_rt_process=DifferencedAR1(autoreg=0.9, innovation_sd=0.05), + subpop_rt_deviation_process=AR1(autoreg=0.8, innovation_sd=0.05), n_initialization_points=len(gen_int_pmf), ) diff --git a/pyrenew/latent/base.py b/pyrenew/latent/base.py index 2d07d027..9a0dac2f 100644 --- a/pyrenew/latent/base.py +++ b/pyrenew/latent/base.py @@ -26,7 +26,7 @@ class LatentSample(NamedTuple): Shape: (n_total_days,) all_subpops : ArrayLike Infections for all subpopulations. - Shape: (n_total_days, K) + Shape: (n_total_days, n_subpops) """ aggregate: ArrayLike @@ -41,13 +41,16 @@ class PopulationStructure: Attributes ---------- fractions : ArrayLike - Population fractions for all subpopulations. Shape: (K,) - K : int - Total number of subpopulations + Population fractions for all subpopulations. + Shape: (n_subpops,) """ fractions: ArrayLike - K: int + + @property + def n_subpops(self) -> int: + """Total number of subpopulations.""" + return len(self.fractions) class BaseLatentInfectionProcess(RandomVariable): @@ -153,9 +156,7 @@ def _parse_and_validate_fractions( if fractions.ndim != 1: raise ValueError("subpop_fractions must be a 1D array") - K = len(fractions) - - if K == 0: + if len(fractions) == 0: raise ValueError("Must have at least one subpopulation") # Only validate when results are concrete (not during JAX tracing) @@ -174,7 +175,6 @@ def _parse_and_validate_fractions( return PopulationStructure( fractions=fractions, - K=K, ) @staticmethod @@ -196,7 +196,7 @@ def _validate_output_shapes( n_total_days : int Expected number of days (n_initialization_points + n_days_post_init) pop : PopulationStructure - Population structure with K + Population structure Raises ------ @@ -205,7 +205,7 @@ def _validate_output_shapes( """ expected = { "infections_aggregate": (n_total_days,), - "infections_all": (n_total_days, pop.K), + "infections_all": (n_total_days, pop.n_subpops), } actual = { @@ -309,8 +309,8 @@ def sample( n_days_post_init : int Number of days to simulate after initialization period subpop_fractions : ArrayLike - Population fractions for all subpopulations. Shape: (K,). - Must sum to 1.0. + Population fractions for all subpopulations. + Shape: (n_subpops,). Must sum to 1.0. **kwargs Additional parameters required by specific implementations @@ -319,7 +319,7 @@ def sample( LatentSample Named tuple with fields: - aggregate: shape (n_total_days,) - - all_subpops: shape (n_total_days, K) + - all_subpops: shape (n_total_days, n_subpops) where n_total_days = n_initialization_points + n_days_post_init """ diff --git a/pyrenew/latent/hierarchical_infections.py b/pyrenew/latent/hierarchical_infections.py index 10c3f965..da11a4ee 100644 --- a/pyrenew/latent/hierarchical_infections.py +++ b/pyrenew/latent/hierarchical_infections.py @@ -12,6 +12,7 @@ from jax.typing import ArrayLike from pyrenew.deterministic import DeterministicVariable +from pyrenew.distutil import validate_discrete_dist_vector from pyrenew.latent.base import ( BaseLatentInfectionProcess, LatentSample, @@ -48,11 +49,11 @@ class HierarchicalInfections(BaseLatentInfectionProcess): I0_rv : RandomVariable Initial infection prevalence (proportion of population) at first observation time. Must return values in the interval (0, 1). - Returns scalar (same for all subpops) or (K,) array (per-subpop). + Returns scalar (same for all subpops) or (n_subpops,) array (per-subpop). Full I0 matrix generated via exponential backprojection during sampling. - baseline_temporal : TemporalProcess + baseline_rt_process : TemporalProcess Temporal process for baseline Rt dynamics - subpop_temporal : TemporalProcess + subpop_rt_deviation_process : TemporalProcess Temporal process for subpopulation deviations initial_log_rt_rv : RandomVariable Initial value for log(Rt) at time 0. Can be estimated from data @@ -77,10 +78,11 @@ def __init__( *, gen_int_rv: RandomVariable, I0_rv: RandomVariable, - baseline_temporal: TemporalProcess, - subpop_temporal: TemporalProcess, + baseline_rt_process: TemporalProcess, + subpop_rt_deviation_process: TemporalProcess, initial_log_rt_rv: RandomVariable, n_initialization_points: int, + name: str = "latent_infections", ) -> None: """ Initialize hierarchical infections process. @@ -91,14 +93,18 @@ def __init__( Generation interval PMF I0_rv : RandomVariable Initial infection prevalence (proportion of population) - baseline_temporal : TemporalProcess + baseline_rt_process : TemporalProcess Temporal process for baseline Rt dynamics - subpop_temporal : TemporalProcess + subpop_rt_deviation_process : TemporalProcess Temporal process for subpopulation deviations initial_log_rt_rv : RandomVariable Initial value for log(Rt) at time 0. n_initialization_points : int Number of initialization days before day 0. + name : str + Name prefix for numpyro sample sites. All deterministic + quantities are recorded under this scope (e.g., + ``"{name}/rt_baseline"``). Default: ``"latent_infections"``. Raises ------ @@ -109,6 +115,7 @@ def __init__( gen_int_rv=gen_int_rv, n_initialization_points=n_initialization_points, ) + self.name = name if I0_rv is None: raise ValueError("I0_rv is required") @@ -123,35 +130,26 @@ def __init__( raise ValueError("initial_log_rt_rv is required") self.initial_log_rt_rv = initial_log_rt_rv - if baseline_temporal is None: - raise ValueError("baseline_temporal is required") - self.baseline_temporal = baseline_temporal + if baseline_rt_process is None: + raise ValueError("baseline_rt_process is required") + self.baseline_rt_process = baseline_rt_process - if subpop_temporal is None: - raise ValueError("subpop_temporal is required") - self.subpop_temporal = subpop_temporal + if subpop_rt_deviation_process is None: + raise ValueError("subpop_rt_deviation_process is required") + self.subpop_rt_deviation_process = subpop_rt_deviation_process def validate(self) -> None: """ Validate hierarchical infections parameters. - Checks that temporal processes and I0_rv can be sampled. + Checks that the generation interval is a valid PMF. Raises ------ ValueError - If any parameters fail validation + If gen_int_rv does not return a valid discrete distribution """ - gen_int_pmf = self.gen_int_rv() - if gen_int_pmf.size == 0: # pragma: no cover - raise ValueError("gen_int_rv must return non-empty array") - - pmf_sum = jnp.sum(gen_int_pmf) - if not jnp.isclose(pmf_sum, 1.0, atol=1e-6): # pragma: no cover - raise ValueError(f"gen_int_rv must sum to 1.0, got {float(pmf_sum):.6f}") - - if jnp.any(gen_int_pmf < 0): # pragma: no cover - raise ValueError("gen_int_rv must have non-negative values") + validate_discrete_dist_vector(self.gen_int_rv()) def sample( self, @@ -164,14 +162,14 @@ def sample( Sample hierarchical infections for all subpopulations. Generates baseline Rt, subpopulation deviations with sum-to-zero - constraint, initial infections, and runs K independent renewal processes. + constraint, initial infections, and runs n_subpops independent renewal processes. Parameters ---------- n_days_post_init : int Number of days to simulate after initialization period subpop_fractions : ArrayLike - Population fractions for all subpopulations. Shape: (K,). + Population fractions for all subpopulations. Shape: (n_subpops,). Must sum to 1.0. **kwargs Additional arguments (unused, for compatibility) @@ -181,7 +179,7 @@ def sample( LatentSample Named tuple with fields: - aggregate: shape (n_total_days,) - - all_subpops: shape (n_total_days, K) + - all_subpops: shape (n_total_days, n_subpops) """ # Parse and validate population structure pop = self._parse_and_validate_fractions( @@ -192,16 +190,16 @@ def sample( initial_log_rt = self.initial_log_rt_rv() - log_rt_baseline = self.baseline_temporal.sample( + log_rt_baseline = self.baseline_rt_process.sample( n_timepoints=n_total_days, initial_value=initial_log_rt, name_prefix="log_rt_baseline", ) - deviations_raw = self.subpop_temporal.sample( + deviations_raw = self.subpop_rt_deviation_process.sample( n_timepoints=n_total_days, - n_processes=pop.K, - initial_value=jnp.zeros(pop.K), + n_processes=pop.n_subpops, + initial_value=jnp.zeros(pop.n_subpops), name_prefix="subpop_deviations", ) @@ -218,7 +216,7 @@ def sample( self._validate_I0(I0) if I0.ndim == 0: - I0_subpop = jnp.full(pop.K, I0) + I0_subpop = jnp.full(pop.n_subpops, I0) else: I0_subpop = I0 @@ -232,7 +230,6 @@ def sample( I0_all = I0_subpop[jnp.newaxis, :] * jnp.exp( initial_r_subpop[jnp.newaxis, :] * time_indices[:, jnp.newaxis] ) - numpyro.deterministic("I0_init_all_subpops", I0_all) gen_int_reversed = jnp.flip(gen_int) recent_I0_all = I0_all[-gen_int.size :, :] @@ -262,11 +259,17 @@ def sample( ) # Record key quantities for diagnostics and posterior analysis - numpyro.deterministic("log_rt_baseline", log_rt_baseline) - numpyro.deterministic("rt_baseline", jnp.exp(log_rt_baseline)) - numpyro.deterministic("rt_subpop", rt_subpop) - numpyro.deterministic("subpop_deviations", deviations) - numpyro.deterministic("infections_aggregate", infections_aggregate) + with numpyro.handlers.scope(prefix=self.name): + numpyro.deterministic("I0_init_all_subpops", I0_all) + numpyro.deterministic("log_rt_baseline", log_rt_baseline) + numpyro.deterministic( + "rt_baseline", jnp.exp(log_rt_baseline) + ) + numpyro.deterministic("rt_subpop", rt_subpop) + numpyro.deterministic("subpop_deviations", deviations) + numpyro.deterministic( + "infections_aggregate", infections_aggregate + ) return LatentSample( aggregate=infections_aggregate, diff --git a/pyrenew/latent/temporal_processes.py b/pyrenew/latent/temporal_processes.py index 4865b85b..4ba1bdaf 100644 --- a/pyrenew/latent/temporal_processes.py +++ b/pyrenew/latent/temporal_processes.py @@ -68,6 +68,8 @@ from jax.typing import ArrayLike from pyrenew.process import ARProcess, DifferencedProcess +from pyrenew.process.randomwalk import RandomWalk as ProcessRandomWalk +from pyrenew.randomvariable import DistributionalVariable @runtime_checkable @@ -302,7 +304,7 @@ def sample( ) trajectories = self.process( - n=n_timepoints + 1, + n=n_timepoints, init_vals=initial_value[jnp.newaxis, :], autoreg=jnp.full((1, n_processes), self.autoreg), noise_sd=self.innovation_sd, @@ -310,7 +312,7 @@ def sample( noise_name=f"{name_prefix}_noise", ) - return trajectories[:n_timepoints, :] + return trajectories class RandomWalk(TemporalProcess): @@ -322,8 +324,7 @@ class RandomWalk(TemporalProcess): have no prior expectation that Rt will return to a baseline. This class wraps :class:`pyrenew.process.RandomWalk` with a simplified, - protocol-compliant interface. The vectorized mode uses a non-centered - parameterization to avoid funnel problems in inference. + protocol-compliant interface that handles vectorization automatically. Parameters ---------- @@ -336,6 +337,10 @@ class RandomWalk(TemporalProcess): Unlike AR(1), variance grows over time — the process can wander arbitrarily far from its starting point. For long time horizons, consider AR(1) if you want Rt to stay bounded near a baseline. + + For non-centered parameterization (to avoid funnel problems in inference), + apply ``LocScaleReparam(centered=0)`` to the step sample site + (``{name_prefix}_step``) via ``numpyro.handlers.reparam``. """ def __init__(self, innovation_sd: float = 1.0): @@ -391,23 +396,17 @@ def sample( elif jnp.isscalar(initial_value): initial_value = jnp.full(n_processes, initial_value) - # Non-centered parameterization to avoid funnel problems - with numpyro.plate(f"{name_prefix}_time", n_timepoints - 1): - with numpyro.plate(f"{name_prefix}_proc", n_processes): - increments_raw = numpyro.sample( - f"{name_prefix}_increments_raw", - dist.Normal(0, 1), - ) - - # Transpose: (n_processes, n_timepoints-1) -> (n_timepoints-1, n_processes) - increments = numpyro.deterministic( - f"{name_prefix}_increments", - (increments_raw * self.innovation_sd).T, + rw = ProcessRandomWalk( + step_rv=DistributionalVariable( + name=f"{name_prefix}_step", + distribution=dist.Normal( + jnp.zeros(n_processes), + self.innovation_sd, + ), + ), ) - cumulative = jnp.cumsum(increments, axis=0) - - return jnp.concatenate( - [initial_value[jnp.newaxis, :], initial_value + cumulative], - axis=0, + return rw.sample( + init_vals=initial_value[jnp.newaxis, :], + n=n_timepoints, ) diff --git a/pyrenew/model/multisignal_model.py b/pyrenew/model/multisignal_model.py index 518cdf1a..47c9cd7d 100644 --- a/pyrenew/model/multisignal_model.py +++ b/pyrenew/model/multisignal_model.py @@ -9,10 +9,8 @@ from typing import Any import jax.numpy as jnp -import jax.random as random import numpyro import numpyro.handlers -from numpyro.infer import MCMC, NUTS from pyrenew.latent.base import BaseLatentInfectionProcess from pyrenew.metaclass import Model @@ -71,40 +69,34 @@ def __init__( """ self.latent = latent_process self.observations = observations - self._validate() + self.validate() - def _validate(self) -> None: + _SUPPORTED_RESOLUTIONS = {"aggregate", "subpop"} + + def validate(self) -> None: """ Validate that observation processes are compatible with latent process. - Checks that each observation implements infection_resolution(). + Checks that each observation implements infection_resolution() + and returns a supported resolution. Raises ------ ValueError If an observation doesn't implement infection_resolution() + or returns an unsupported resolution. """ for name, obs in self.observations.items(): - try: - obs.infection_resolution() - except (NotImplementedError, AttributeError): + if not hasattr(obs, "infection_resolution"): raise ValueError( f"Observation '{name}' must implement infection_resolution()" ) - - def validate(self) -> None: - """ - Validate the model configuration. - - This method is required by PyRenew's Model base class. - Validation is performed in __init__ via _validate(). - - Raises - ------ - ValueError - If validation fails - """ - self._validate() + resolution = obs.infection_resolution() + if resolution not in self._SUPPORTED_RESOLUTIONS: + raise ValueError( + f"Observation '{name}' returned invalid infection_resolution " + f"'{resolution}'. Expected one of {self._SUPPORTED_RESOLUTIONS}." + ) def pad_observations( self, @@ -184,7 +176,7 @@ def validate_data( n_days_post_init : int Number of days to simulate after initialization period subpop_fractions : ArrayLike - Population fractions for all subpopulations. Shape: (K,). + Population fractions for all subpopulations. Shape: (n_subpops,). **observation_data Data for each observation process, keyed by observation name. Each value should be a dict of kwargs for that observation's sample(). @@ -251,10 +243,10 @@ def validate_data( f"Observation '{name}': subpop_indices cannot be negative" ) max_idx = jnp.max(subpop_indices) - if max_idx >= pop.K: + if max_idx >= pop.n_subpops: raise ValueError( f"Observation '{name}': subpop_indices contains " - f"{int(max_idx)} >= {pop.K} (K)" + f"{int(max_idx)} >= {pop.n_subpops} (n_subpops)" ) def sample( @@ -278,7 +270,7 @@ def sample( Total population size. Used to convert infection proportions (from latent process) to infection counts (for observation processes). subpop_fractions : ArrayLike - Population fractions for all subpopulations. Shape: (K,). + Population fractions for all subpopulations. Shape: (n_subpops,). **observation_data Data for each observation process, keyed by observation name (the ``name`` attribute of each observation process). @@ -286,12 +278,11 @@ def sample( Returns ------- - tuple - Two arrays of infection counts (scaled by population_size): - - inf_aggregate: shape (n_total_days,) - aggregate infections - - inf_all: shape (n_total_days, K) - all subpopulation infections - - where n_total_days = n_initialization_points + n_days_post_init + None + All quantities are recorded as NumPyro deterministic sites + (``latent_infections``, ``latent_infections_by_subpop``) and + observation sites. Use ``numpyro.infer.Predictive`` for forward + sampling. """ # Generate latent infections (proportions) latent_sample = self.latent.sample( @@ -320,7 +311,7 @@ def sample( if resolution not in latent_map: raise ValueError( f"Observation '{name}' returned invalid infection_resolution " - f"'{resolution}'. Expected 'aggregate' or 'subpop'." + f"'{resolution}'. Expected one of {self._SUPPORTED_RESOLUTIONS}." ) latent_infections = latent_map[resolution] @@ -333,92 +324,4 @@ def sample( **obs_data, ) - # Return scaled infection counts - return inf_aggregate, inf_all - - def fit( - self, - n_days_post_init: int, - population_size: float, - *, - subpop_fractions=None, - num_warmup: int = 500, - num_samples: int = 500, - num_chains: int = 1, - rng_key: random.PRNGKey = None, - reparam_config: dict[str, Any] | None = None, - progress_bar: bool = True, - **observation_data, - ): - """ - Fit the model to observed data via MCMC. - - Validates observation data, runs NUTS sampler, and returns - posterior samples. - - Parameters - ---------- - n_days_post_init : int - Number of days to simulate after initialization period - population_size : float - Total population size for scaling infections - subpop_fractions : ArrayLike - Population fractions for all subpopulations. Shape: (K,). - num_warmup : int - Number of MCMC warmup iterations (default: 500) - num_samples : int - Number of MCMC samples to draw (default: 500) - num_chains : int - Number of MCMC chains (default: 1) - rng_key : jax.random.PRNGKey, optional - Random key for reproducibility. Defaults to PRNGKey(0). - reparam_config : dict, optional - Reparameterization configuration for numpyro.handlers.reparam. - Maps sample site names to reparameterizers (e.g., LocScaleReparam). - Use non-centered parameterization (centered=0) for sparse data, - centered (centered=1) for abundant data. Default None (no reparam). - progress_bar : bool - Whether to show progress bar during MCMC (default: True). - Set to False if experiencing tqdm widget errors in Jupyter notebooks - with parallel chains. - **observation_data - Data for each observation process, keyed by observation name. - Each value should be a dict with 'counts', 'times', etc. - - Returns - ------- - numpyro.infer.MCMC - The MCMC object after sampling. Use `mcmc.get_samples()` to get - posterior samples as a dict, or access other MCMC diagnostics. - """ - self.validate_data( - n_days_post_init=n_days_post_init, - subpop_fractions=subpop_fractions, - **observation_data, - ) - - if rng_key is None: - rng_key = random.PRNGKey(0) - - # Apply reparameterization if config provided - sample_fn = self.sample - if reparam_config is not None: - sample_fn = numpyro.handlers.reparam(self.sample, config=reparam_config) - - kernel = NUTS(sample_fn) - mcmc = MCMC( - kernel, - num_warmup=num_warmup, - num_samples=num_samples, - num_chains=num_chains, - progress_bar=progress_bar, - ) - mcmc.run( - rng_key, - n_days_post_init=n_days_post_init, - population_size=population_size, - subpop_fractions=subpop_fractions, - **observation_data, - ) - - return mcmc + return None diff --git a/pyrenew/model/pyrenew_builder.py b/pyrenew/model/pyrenew_builder.py index 9d99505d..f8f0eac0 100644 --- a/pyrenew/model/pyrenew_builder.py +++ b/pyrenew/model/pyrenew_builder.py @@ -185,14 +185,7 @@ def compute_n_initialization_points(self) -> int: # Add lookback from each observation process for name, obs_process in self.observations.items(): - try: - lookback = obs_process.lookback_days() - lookbacks.append(lookback) - except (NotImplementedError, AttributeError) as e: - raise ValueError( - f"Observation process '{name}' must implement " - f"lookback_days(). Error: {e}" - ) from e + lookbacks.append(obs_process.lookback_days()) # Formula: max(all lookbacks) # For generation interval (1-indexed): L-element PMF has max lag L days → need L init points diff --git a/test/test_base_latent_process.py b/test/test_base_latent_process.py index 8cb52735..56a5ea4c 100644 --- a/test/test_base_latent_process.py +++ b/test/test_base_latent_process.py @@ -19,7 +19,7 @@ def test_parse_subpop_fractions(self): subpop_fractions=jnp.array([0.3, 0.25, 0.45]), ) - assert pop.K == 3 + assert pop.n_subpops == 3 assert jnp.allclose(pop.fractions, jnp.array([0.3, 0.25, 0.45])) def test_rejects_fractions_not_summing_to_one(self): @@ -101,8 +101,8 @@ def test_get_required_lookback_returns_gen_int_length(self): gen_int_rv=gen_int, I0_rv=DeterministicVariable("I0", 0.001), initial_log_rt_rv=DeterministicVariable("initial_log_rt", 0.0), - baseline_temporal=RandomWalk(), - subpop_temporal=RandomWalk(), + baseline_rt_process=RandomWalk(), + subpop_rt_deviation_process=RandomWalk(), n_initialization_points=3, ) @@ -118,7 +118,6 @@ def test_validate_output_shapes_raises_on_mismatch(self): pop = PopulationStructure( fractions=jnp.array([0.5, 0.5]), - K=2, ) # Create arrays with wrong shapes diff --git a/test/test_hierarchical_infections.py b/test/test_hierarchical_infections.py index 871c98a0..92056ef2 100644 --- a/test/test_hierarchical_infections.py +++ b/test/test_hierarchical_infections.py @@ -37,8 +37,8 @@ def process(gen_int_rv): gen_int_rv=gen_int_rv, I0_rv=DeterministicVariable("I0", 0.001), initial_log_rt_rv=DeterministicVariable("initial_log_rt", 0.0), - baseline_temporal=RandomWalk(), - subpop_temporal=RandomWalk(), + baseline_rt_process=RandomWalk(), + subpop_rt_deviation_process=RandomWalk(), n_initialization_points=3, ) @@ -102,7 +102,7 @@ def test_deviations_sum_to_zero(self, process): subpop_fractions=jnp.array([0.3, 0.25, 0.45]), ) - deviations = trace["subpop_deviations"]["value"] + deviations = trace["latent_infections/subpop_deviations"]["value"] deviation_sums = jnp.sum(deviations, axis=1) assert jnp.allclose(deviation_sums, 0.0, atol=1e-6) @@ -118,8 +118,8 @@ def test_rejects_missing_I0_rv(self, gen_int_rv): gen_int_rv=gen_int_rv, I0_rv=None, initial_log_rt_rv=DeterministicVariable("initial_log_rt", 0.0), - baseline_temporal=RandomWalk(), - subpop_temporal=RandomWalk(), + baseline_rt_process=RandomWalk(), + subpop_rt_deviation_process=RandomWalk(), n_initialization_points=3, ) @@ -130,32 +130,32 @@ def test_rejects_missing_initial_log_rt_rv(self, gen_int_rv): gen_int_rv=gen_int_rv, I0_rv=DeterministicVariable("I0", 0.001), initial_log_rt_rv=None, - baseline_temporal=RandomWalk(), - subpop_temporal=RandomWalk(), + baseline_rt_process=RandomWalk(), + subpop_rt_deviation_process=RandomWalk(), n_initialization_points=3, ) - def test_rejects_missing_baseline_temporal(self, gen_int_rv): - """Test that None baseline_temporal is rejected.""" - with pytest.raises(ValueError, match="baseline_temporal is required"): + def test_rejects_missing_baseline_rt_process(self, gen_int_rv): + """Test that None baseline_rt_process is rejected.""" + with pytest.raises(ValueError, match="baseline_rt_process is required"): HierarchicalInfections( gen_int_rv=gen_int_rv, I0_rv=DeterministicVariable("I0", 0.001), initial_log_rt_rv=DeterministicVariable("initial_log_rt", 0.0), - baseline_temporal=None, - subpop_temporal=RandomWalk(), + baseline_rt_process=None, + subpop_rt_deviation_process=RandomWalk(), n_initialization_points=3, ) - def test_rejects_missing_subpop_temporal(self, gen_int_rv): - """Test that None subpop_temporal is rejected.""" - with pytest.raises(ValueError, match="subpop_temporal is required"): + def test_rejects_missing_subpop_rt_deviation_process(self, gen_int_rv): + """Test that None subpop_rt_deviation_process is rejected.""" + with pytest.raises(ValueError, match="subpop_rt_deviation_process is required"): HierarchicalInfections( gen_int_rv=gen_int_rv, I0_rv=DeterministicVariable("I0", 0.001), initial_log_rt_rv=DeterministicVariable("initial_log_rt", 0.0), - baseline_temporal=RandomWalk(), - subpop_temporal=None, + baseline_rt_process=RandomWalk(), + subpop_rt_deviation_process=None, n_initialization_points=3, ) @@ -166,8 +166,8 @@ def test_rejects_invalid_I0(self, gen_int_rv): gen_int_rv=gen_int_rv, I0_rv=DeterministicVariable("I0", -0.1), initial_log_rt_rv=DeterministicVariable("initial_log_rt", 0.0), - baseline_temporal=RandomWalk(), - subpop_temporal=RandomWalk(), + baseline_rt_process=RandomWalk(), + subpop_rt_deviation_process=RandomWalk(), n_initialization_points=3, ) @@ -180,8 +180,8 @@ def test_rejects_insufficient_n_initialization_points(self, gen_int_rv): gen_int_rv=gen_int_rv, I0_rv=DeterministicVariable("I0", 0.001), initial_log_rt_rv=DeterministicVariable("initial_log_rt", 0.0), - baseline_temporal=RandomWalk(), - subpop_temporal=RandomWalk(), + baseline_rt_process=RandomWalk(), + subpop_rt_deviation_process=RandomWalk(), n_initialization_points=2, # gen_int has length 3 ) @@ -208,8 +208,8 @@ def test_per_subpop_I0_array(self, gen_int_rv): gen_int_rv=gen_int_rv, I0_rv=DeterministicVariable("I0", jnp.array([0.001, 0.002, 0.0015])), initial_log_rt_rv=DeterministicVariable("initial_log_rt", 0.0), - baseline_temporal=RandomWalk(), - subpop_temporal=RandomWalk(), + baseline_rt_process=RandomWalk(), + subpop_rt_deviation_process=RandomWalk(), n_initialization_points=3, ) diff --git a/test/test_pyrenew_builder.py b/test/test_pyrenew_builder.py index 403208f6..55b0fb06 100644 --- a/test/test_pyrenew_builder.py +++ b/test/test_pyrenew_builder.py @@ -33,8 +33,8 @@ def simple_builder(): gen_int_rv=gen_int, I0_rv=DeterministicVariable("I0", 0.001), initial_log_rt_rv=DeterministicVariable("initial_log_rt", 0.0), - baseline_temporal=RandomWalk(), - subpop_temporal=RandomWalk(), + baseline_rt_process=RandomWalk(), + subpop_rt_deviation_process=RandomWalk(), ) delay = DeterministicPMF("delay", jnp.array([0.1, 0.3, 0.4, 0.2])) @@ -63,8 +63,8 @@ def test_rejects_population_structure_at_configure_time(self): gen_int_rv=gen_int, I0_rv=DeterministicVariable("I0", 0.001), initial_log_rt_rv=DeterministicVariable("initial_log_rt", 0.0), - baseline_temporal=RandomWalk(), - subpop_temporal=RandomWalk(), + baseline_rt_process=RandomWalk(), + subpop_rt_deviation_process=RandomWalk(), subpop_fractions=jnp.array([0.5, 0.5]), # Should fail ) @@ -79,8 +79,8 @@ def test_rejects_n_initialization_points_at_configure_time(self): gen_int_rv=gen_int, I0_rv=DeterministicVariable("I0", 0.001), initial_log_rt_rv=DeterministicVariable("initial_log_rt", 0.0), - baseline_temporal=RandomWalk(), - subpop_temporal=RandomWalk(), + baseline_rt_process=RandomWalk(), + subpop_rt_deviation_process=RandomWalk(), n_initialization_points=10, # Should fail ) @@ -94,8 +94,8 @@ def test_rejects_reconfiguring_latent(self): gen_int_rv=gen_int, I0_rv=DeterministicVariable("I0", 0.001), initial_log_rt_rv=DeterministicVariable("initial_log_rt", 0.0), - baseline_temporal=RandomWalk(), - subpop_temporal=RandomWalk(), + baseline_rt_process=RandomWalk(), + subpop_rt_deviation_process=RandomWalk(), ) with pytest.raises(RuntimeError, match="already configured"): @@ -104,8 +104,8 @@ def test_rejects_reconfiguring_latent(self): gen_int_rv=gen_int, I0_rv=DeterministicVariable("I0", 0.001), initial_log_rt_rv=DeterministicVariable("initial_log_rt", 0.0), - baseline_temporal=RandomWalk(), - subpop_temporal=RandomWalk(), + baseline_rt_process=RandomWalk(), + subpop_rt_deviation_process=RandomWalk(), ) def test_rejects_duplicate_observation_name(self, simple_builder): @@ -171,49 +171,21 @@ def test_sample_with_population_structure(self, simple_builder): n_total = model.latent.n_initialization_points + n_days with numpyro.handlers.seed(rng_seed=42): - inf_aggregate, inf_all = model.sample( - n_days_post_init=n_days, - population_size=1_000_000, - subpop_fractions=SUBPOP_FRACTIONS, - hospital={"obs": None}, - ) - + with numpyro.handlers.trace() as tr: + model.sample( + n_days_post_init=n_days, + population_size=1_000_000, + subpop_fractions=SUBPOP_FRACTIONS, + hospital={"obs": None}, + ) + + inf_aggregate = tr["latent_infections"]["value"] + inf_all = tr["latent_infections_by_subpop"]["value"] assert inf_aggregate.shape == (n_total,) assert inf_all.shape == (n_total, 3) # K=3 - def test_fit_with_reparam_config(self, simple_builder): - """Test that fit() works with reparameterization config.""" - from numpyro.infer.reparam import LocScaleReparam - - model = simple_builder.build() - n_days = 10 - - # Create dense observations with NaN padding for initialization period - obs_values = jnp.array([10.0, 12.0, 15.0, 14.0, 11.0]) - obs = model.pad_observations(obs_values) - # Pad with NaN for remaining days - obs = jnp.concatenate([obs, jnp.full(n_days - len(obs_values), jnp.nan)]) - - # Test with reparam_config - reparam_config = { - "log_rt_baseline_innovations": LocScaleReparam(centered=0), - } - - mcmc = model.fit( - n_days_post_init=n_days, - population_size=1_000_000, - subpop_fractions=SUBPOP_FRACTIONS, - num_warmup=2, - num_samples=2, - reparam_config=reparam_config, - hospital={"obs": obs}, - ) - - samples = mcmc.get_samples() - assert "latent_infections" in samples - - def test_fit_with_population_structure(self, simple_builder): - """Test that fit() works with population structure at sample time.""" + def test_run_with_population_structure(self, simple_builder): + """Test that run() works with population structure at sample time.""" model = simple_builder.build() n_days = 10 n_total = model.latent.n_initialization_points + n_days @@ -224,16 +196,16 @@ def test_fit_with_population_structure(self, simple_builder): # Pad with NaN for remaining days obs = jnp.concatenate([obs, jnp.full(n_days - len(obs_values), jnp.nan)]) - mcmc = model.fit( + model.run( + num_warmup=5, + num_samples=5, n_days_post_init=n_days, population_size=1_000_000, subpop_fractions=SUBPOP_FRACTIONS, - num_warmup=5, - num_samples=5, hospital={"obs": obs}, ) - samples = mcmc.get_samples() + samples = model.mcmc.get_samples() assert "latent_infections" in samples assert samples["latent_infections"].shape == (5, n_total) @@ -351,70 +323,6 @@ def test_validate_data_rejects_out_of_bounds_subpop_indices(self, simple_builder class TestPyrenewBuilderErrorHandling: """Test PyrenewBuilder error handling.""" - def test_compute_n_init_rejects_obs_without_lookback_days(self): - """Test that observation without lookback_days() raises error.""" - from pyrenew.observation.base import BaseObservationProcess - - class BadObservationNoLookback(BaseObservationProcess): - """Observation that has no lookback_days method.""" - - def __init__(self): - """Initialize without temporal_pmf_rv.""" - self.name = "bad" - self.temporal_pmf_rv = None - - def sample(self, **kwargs): - """Sample stub.""" - pass - - def validate(self): - """Validate stub.""" - pass - - def lookback_days(self): - """Raise NotImplementedError to simulate missing implementation.""" - raise NotImplementedError("Not implemented") - - def infection_resolution(self): - """ - Return aggregate. - - Returns - ------- - str - The string "aggregate". - """ - return "aggregate" - - def _predicted_obs(self, infections): - """ - Predicted obs stub. - - Returns - ------- - ArrayLike - The infections array unchanged. - """ - return infections - - builder = PyrenewBuilder() - gen_int = DeterministicPMF("gen_int", jnp.array([0.2, 0.5, 0.3])) - - builder.configure_latent( - HierarchicalInfections, - gen_int_rv=gen_int, - I0_rv=DeterministicVariable("I0", 0.001), - initial_log_rt_rv=DeterministicVariable("initial_log_rt", 0.0), - baseline_temporal=RandomWalk(), - subpop_temporal=RandomWalk(), - ) - - bad_obs = BadObservationNoLookback() - builder.add_observation(bad_obs) - - with pytest.raises(ValueError, match="must implement lookback_days"): - builder.compute_n_initialization_points() - def test_build_raises_on_construction_error(self): """Test that build() raises TypeError on latent construction failure.""" builder = PyrenewBuilder() @@ -426,8 +334,8 @@ def test_build_raises_on_construction_error(self): gen_int_rv=gen_int, I0_rv=DeterministicVariable("I0", 0.001), initial_log_rt_rv=DeterministicVariable("initial_log_rt", 0.0), - baseline_temporal=RandomWalk(), - subpop_temporal=RandomWalk(), + baseline_rt_process=RandomWalk(), + subpop_rt_deviation_process=RandomWalk(), invalid_extra_param="this will cause TypeError", # Invalid param ) @@ -479,8 +387,8 @@ def lookback_days(self): return 1 def infection_resolution(self): - """Raise NotImplementedError to simulate missing implementation.""" - raise NotImplementedError("Not implemented") + """Return an invalid resolution to simulate bad implementation.""" + return "invalid_resolution" def _predicted_obs(self, infections): """ @@ -501,14 +409,14 @@ def _predicted_obs(self, infections): gen_int_rv=gen_int, I0_rv=DeterministicVariable("I0", 0.001), initial_log_rt_rv=DeterministicVariable("initial_log_rt", 0.0), - baseline_temporal=RandomWalk(), - subpop_temporal=RandomWalk(), + baseline_rt_process=RandomWalk(), + subpop_rt_deviation_process=RandomWalk(), ) bad_obs = BadObservation() builder.add_observation(bad_obs) - with pytest.raises(ValueError, match="must implement infection_resolution"): + with pytest.raises(ValueError, match="invalid infection_resolution"): builder.build() From 59ed28eff7df419403200fdc5cfeb39ec5cb6850 Mon Sep 17 00:00:00 2001 From: Mitzi Morris Date: Fri, 6 Feb 2026 00:14:28 -0500 Subject: [PATCH 79/81] changes per code review --- .../latent_hierarchical_infections.qmd | 22 +++++++++++++------ pyrenew/latent/base.py | 9 +++++++- pyrenew/latent/hierarchical_infections.py | 8 ++----- pyrenew/latent/temporal_processes.py | 8 +++---- pyrenew/model/pyrenew_builder.py | 11 +--------- test/test_pyrenew_builder.py | 11 ++++++++-- 6 files changed, 39 insertions(+), 30 deletions(-) diff --git a/docs/tutorials/latent_hierarchical_infections.qmd b/docs/tutorials/latent_hierarchical_infections.qmd index f6b2505e..df061323 100644 --- a/docs/tutorials/latent_hierarchical_infections.qmd +++ b/docs/tutorials/latent_hierarchical_infections.qmd @@ -467,8 +467,12 @@ n_samples = 200 predictive = Predictive(sample_model, num_samples=n_samples) prior_samples = predictive(random.PRNGKey(42)) -print(f"Rt baseline samples: {prior_samples['latent_infections/rt_baseline'].shape}") -print(f"Infection samples: {prior_samples['latent_infections/infections_aggregate'].shape}") +print( + f"Rt baseline samples: {prior_samples['latent_infections/rt_baseline'].shape}" +) +print( + f"Infection samples: {prior_samples['latent_infections/infections_aggregate'].shape}" +) ``` ### Visualizing the Prior: Baseline Rt Trajectories @@ -476,7 +480,9 @@ print(f"Infection samples: {prior_samples['latent_infections/infections_aggregat ```{python} # | label: rt-trajectories-data # Extract post-initialization Rt trajectories -rt_samples = np.array(prior_samples["latent_infections/rt_baseline"])[:, n_init:, 0] +rt_samples = np.array(prior_samples["latent_infections/rt_baseline"])[ + :, n_init:, 0 +] # Build long-format dataframe for plotting rt_data = [] @@ -570,7 +576,9 @@ for name, temporal in temporal_configs.items(): samples = Predictive(make_sampler(model), num_samples=n_samples)( random.PRNGKey(42) ) - process_samples[name] = np.array(samples["latent_infections/rt_baseline"])[:, n_init:, 0] + process_samples[name] = np.array(samples["latent_infections/rt_baseline"])[ + :, n_init:, 0 + ] ``` ```{python} @@ -675,9 +683,9 @@ for name, subpop_rt_deviation_process in subpop_configs.items(): # Plot deviations for first subpopulation dev_data = [] for name in subpop_configs.keys(): - devs = np.array(deviation_samples[name]["latent_infections/subpop_deviations"])[ - :, n_init:, 0 - ] + devs = np.array( + deviation_samples[name]["latent_infections/subpop_deviations"] + )[:, n_init:, 0] for i in range(n_samples): for d in range(n_days): dev_data.append( diff --git a/pyrenew/latent/base.py b/pyrenew/latent/base.py index 9a0dac2f..e443f4e6 100644 --- a/pyrenew/latent/base.py +++ b/pyrenew/latent/base.py @@ -49,7 +49,14 @@ class PopulationStructure: @property def n_subpops(self) -> int: - """Total number of subpopulations.""" + """ + Total number of subpopulations. + + Returns + ------- + int + The number of subpopulations. + """ return len(self.fractions) diff --git a/pyrenew/latent/hierarchical_infections.py b/pyrenew/latent/hierarchical_infections.py index da11a4ee..eff96b05 100644 --- a/pyrenew/latent/hierarchical_infections.py +++ b/pyrenew/latent/hierarchical_infections.py @@ -262,14 +262,10 @@ def sample( with numpyro.handlers.scope(prefix=self.name): numpyro.deterministic("I0_init_all_subpops", I0_all) numpyro.deterministic("log_rt_baseline", log_rt_baseline) - numpyro.deterministic( - "rt_baseline", jnp.exp(log_rt_baseline) - ) + numpyro.deterministic("rt_baseline", jnp.exp(log_rt_baseline)) numpyro.deterministic("rt_subpop", rt_subpop) numpyro.deterministic("subpop_deviations", deviations) - numpyro.deterministic( - "infections_aggregate", infections_aggregate - ) + numpyro.deterministic("infections_aggregate", infections_aggregate) return LatentSample( aggregate=infections_aggregate, diff --git a/pyrenew/latent/temporal_processes.py b/pyrenew/latent/temporal_processes.py index 4ba1bdaf..30ad5618 100644 --- a/pyrenew/latent/temporal_processes.py +++ b/pyrenew/latent/temporal_processes.py @@ -7,8 +7,8 @@ of shape (n_timepoints, n_processes) through a unified ``TemporalProcess`` protocol. -Relationship to pyrenew.process -------------------------------- +**Relationship to pyrenew.process:** + This module provides **high-level, domain-specific wrappers** around the low-level building blocks in :mod:`pyrenew.process`. The key differences: @@ -45,8 +45,8 @@ provide a consistent interface, automatic vectorization, and enforce epidemiologically-sensible constraints. -Temporal Processes ------------------- +**Temporal processes provided:** + - ``AR1``: Autoregressive process with mean reversion. Keeps Rt bounded near a baseline. Wraps :class:`pyrenew.process.ARProcess`. - ``DifferencedAR1``: AR(1) on first differences. Allows persistent trends diff --git a/pyrenew/model/pyrenew_builder.py b/pyrenew/model/pyrenew_builder.py index f8f0eac0..c960bd15 100644 --- a/pyrenew/model/pyrenew_builder.py +++ b/pyrenew/model/pyrenew_builder.py @@ -216,8 +216,6 @@ def build(self): ------ ValueError If latent process not configured - TypeError - If latent process constructor fails """ if self.latent_class is None: raise ValueError("Must call configure_latent() before build()") @@ -228,14 +226,7 @@ def build(self): # Construct latent process with computed n_initialization_points latent_params = {**self.latent_params, "n_initialization_points": n_init} - try: - latent_process = self.latent_class(**latent_params) - except TypeError as e: - raise TypeError( - f"Error constructing {self.latent_class.__name__} with " - f"computed n_initialization_points={n_init}. " - f"Original error: {e}" - ) from e + latent_process = self.latent_class(**latent_params) # Build model model = MultiSignalModel( diff --git a/test/test_pyrenew_builder.py b/test/test_pyrenew_builder.py index 55b0fb06..5a947e30 100644 --- a/test/test_pyrenew_builder.py +++ b/test/test_pyrenew_builder.py @@ -348,7 +348,7 @@ def test_build_raises_on_construction_error(self): ) builder.add_observation(obs) - with pytest.raises(TypeError, match="Error constructing"): + with pytest.raises(TypeError, match="unexpected keyword argument"): builder.build() @@ -387,7 +387,14 @@ def lookback_days(self): return 1 def infection_resolution(self): - """Return an invalid resolution to simulate bad implementation.""" + """ + Return an invalid resolution to simulate bad implementation. + + Returns + ------- + str + An invalid resolution string. + """ return "invalid_resolution" def _predicted_obs(self, infections): From fe8260379effc56be787821965a0b99ec33fcfb6 Mon Sep 17 00:00:00 2001 From: Mitzi Morris Date: Fri, 6 Feb 2026 10:04:51 -0500 Subject: [PATCH 80/81] changes to tutorials per code review --- .../tutorials/building_multisignal_models.qmd | 80 ++++++++++--------- docs/tutorials/custom_randomvariables.qmd | 58 ++++++-------- .../latent_hierarchical_infections.qmd | 4 +- 3 files changed, 67 insertions(+), 75 deletions(-) diff --git a/docs/tutorials/building_multisignal_models.qmd b/docs/tutorials/building_multisignal_models.qmd index 1a09163b..46d5e853 100644 --- a/docs/tutorials/building_multisignal_models.qmd +++ b/docs/tutorials/building_multisignal_models.qmd @@ -20,8 +20,8 @@ jupyter: --- ```{python} -# | label: setup -# | output: false +#| label: setup +#| output: false import numpyro # to run samplers in parallel you must run `set_host_device_count` before importing jax @@ -30,12 +30,13 @@ numpyro.enable_x64() ``` ```{python} -# | label: imports-base +#| label: imports-base import arviz as az import jax import jax.numpy as jnp import jax.random as random import numpy as np +import numpyro.distributions as dist import plotnine as p9 import pandas as pd import time @@ -51,7 +52,7 @@ def make_rng_key(): ``` ```{python} -# | label: imports-pyrenew +#| label: imports-pyrenew from jax.typing import ArrayLike from pyrenew import datasets @@ -60,6 +61,7 @@ from pyrenew.deterministic import ( DeterministicVariable, ) from pyrenew.metaclass import RandomVariable +from pyrenew.randomvariable import DistributionalVariable from pyrenew.latent import ( HierarchicalInfections, @@ -149,8 +151,8 @@ flowchart TB WW["Measured viral concentrations"] end - L --> J L --> S + S -->|"weighted sum"| J J --> C S -->|"select monitored subpopulations"| W C -->|"delay + ascertainment + noise"| HA @@ -185,7 +187,7 @@ Latent infection processes implement the renewal equation to generate infection The generation interval PMF specifies the probability that a secondary infection occurs $d$ days after the primary infection. ```{python} -# | label: gen-interval +#| label: gen-interval covid_gen_int = [0.16, 0.32, 0.25, 0.14, 0.07, 0.04, 0.02] gen_int_pmf = jnp.array(covid_gen_int) gen_int_rv = DeterministicPMF("gen_int", gen_int_pmf) @@ -197,20 +199,20 @@ print(f"Generation interval length: {len(gen_int_pmf)} days") ### I0: Initial Infections -The initial infections RV `I0_rv` specifies the **proportion of the population infected** at the first observation time. This must be a value in the interval (0, 1]. +The initial infections RV `I0_rv` specifies the **proportion of the population infected** at the first observation time. This must be a value in the interval (0, 1]. We use a Beta prior centered near a small value: ```{python} -# | label: initial-infections -I0_rv = DeterministicVariable("I0", 0.01) +#| label: initial-infections +I0_rv = DistributionalVariable("I0", dist.Beta(1, 100)) ``` ### Initial Log Rt -We set the initial log(Rt) to 0.0, meaning Rt starts at 1.0. +We place a prior on the initial log(Rt), centered at 0.0 (Rt = 1.0) with moderate uncertainty: ```{python} -# | label: initial-log-rt -initial_log_rt_rv = DeterministicVariable("initial_log_rt", 0.0) +#| label: initial-log-rt +initial_log_rt_rv = DistributionalVariable("initial_log_rt", dist.Normal(0.0, 0.5)) ``` ### Temporal Processes for $R(t)$ @@ -223,7 +225,7 @@ We configure two temporal processes: The RandomWalk allows flexible evolution of subpopulation-specific transmission without mean reversion. ```{python} -# | label: temporal-processes +#| label: temporal-processes # AR1 provides mean-reverting behavior for baseline Rt baseline_rt_process = AR1(autoreg=0.9, innovation_sd=0.05) @@ -259,7 +261,7 @@ In this example we use a dataset consisting of hospital admissions for COVID-19 for the first 10 months of 2023 (as reported to the CDC). ```{python} -# | label: load-hospital-data +#| label: load-hospital-data # Load daily hospital admissions for California ca_hosp_data = datasets.load_hospital_data_for_state("CA", "2023-11-06.csv") @@ -279,7 +281,7 @@ print( The hospital admissions data is aggregated at the jurisdiction level, therefore we specify a `Counts` observation process. ```{python} -# | label: hospital-obs-process +#| label: hospital-obs-process # Infection-to-hospitalization delay (COVID-19, from literature) inf_to_hosp_pmf = jnp.array( [ @@ -358,7 +360,7 @@ implementations subclass it and implement `_predicted_obs()` to transform infect into predicted values. See `observation_processes_measurements.qmd` for a detailed tutorial. ```{python} -# | label: wastewater-class +#| label: wastewater-class class Wastewater(Measurements): """ Wastewater viral concentration observation process. @@ -415,7 +417,7 @@ For the wastewater data, we use a simulated dataset for California with realisti that covers the same time period. ```{python} -# | label: load-wastewater-data +#| label: load-wastewater-data # Load wastewater data for California ca_ww_data = datasets.load_wastewater_data_for_state("CA", "fake_nwss.csv") @@ -445,7 +447,7 @@ Wastewater observations are site-level: each measurement is associated with a sp Here we specify HierarchicalNormalPrior for the site-level mode and GammaGroupSdPrior for the standard deviation. ```{python} -# | label: wastewater-obs-process +#| label: wastewater-obs-process # Viral shedding kinetics PMF (days post-infection) shedding_pmf = jnp.array( [ @@ -501,7 +503,7 @@ print(f" Shedding PMF length: {len(shedding_pmf)} days") We instantiate a `PyrenewBuilder` object which handles the composition of the latent infection process and the observation process. ```{python} -# | label: model-builder-init +#| label: model-builder-init # Build the multi-signal model builder = PyrenewBuilder() ``` @@ -519,7 +521,7 @@ Methods `configure_latent` and `add_observation` can be called in any order. Met We use `configure_latent` to specify the **model structure**: generation interval, initial infections, and temporal dynamics. ```{python} -# | label: configure-latent +#| label: configure-latent print("Latent process configuration:") print(f" Generation interval length: {len(gen_int_rv())} days") @@ -538,7 +540,7 @@ builder.configure_latent( Each observation process's `name` attribute becomes the keyword used to pass that observation's data to `model.run()` (e.g., `hospital={...}`, `wastewater={...}`). ```{python} -# | label: add-observations-build +#| label: add-observations-build builder.add_observation(hosp_obs) # Uses hosp_obs.name = "hospital" builder.add_observation(ww_obs) # Uses ww_obs.name = "wastewater" model = builder.build() @@ -660,7 +662,7 @@ samples = model.mcmc.get_samples() First we declare the population structure. We have 6 subpopulations, where 5 have wastewater monitoring and 1 does not. The subpopulations with wastewater monitoring need not be contiguous indices—they could be any subset of {0, 1, ..., n_subpops-1}. ```{python} -# | label: population-structure +#| label: population-structure # All 6 subpopulations with their population fractions subpop_fractions = jnp.array([0.10, 0.14, 0.21, 0.22, 0.07, 0.26]) @@ -686,7 +688,7 @@ print(f"Total population: {float(jnp.sum(subpop_fractions)):.0%}") We define a function to prepare observation data using the model's helper methods to align with the shared time axis. ```{python} -# | label: prepare-observation-data +#| label: prepare-observation-data def prepare_observation_data( model, n_days_fit, hosp_admits, ww_data, ww_monitored_subpops ): @@ -753,7 +755,7 @@ Putting this altogether, we align the data with the model time and call `model.r We run 4 sampler chains. ```{python} -# | label: fit-90-days +#| label: fit-90-days # Clear JAX caches to avoid interference from earlier cells jax.clear_caches() @@ -785,7 +787,7 @@ print(f"Elapsed time: {elapsed_90:.1f} seconds") ``` ```{python} -# | label: posterior-shape-90 +#| label: posterior-shape-90 lat_inf_90 = samples_90["latent_infections"] print(f"Posterior samples shape: {lat_inf_90.shape}") print(f"Mean latent infections: {np.mean(lat_inf_90):,.0f}") @@ -794,7 +796,7 @@ print(f"Mean latent infections: {np.mean(lat_inf_90):,.0f}") We check that the chains have converged and the number of effective samples. ```{python} -# | label: arviz-diagnostics-90 +#| label: arviz-diagnostics-90 # ArviZ diagnostics for 90-day fit def filter_samples_for_arviz(samples, n_init, num_chains=4): """Slice time-series samples to exclude initialization period and reshape for chains.""" @@ -827,7 +829,7 @@ az.summary(idata_90, var_names=["~latent_infections", "~expected"]) We extract the posterior quantiles and print summary statistics. ```{python} -# | label: extract-quantiles-90 +#| label: extract-quantiles-90 def extract_posterior_quantiles(samples, n_init, n_days): """Extract posterior quantiles for latent infections.""" latent_inf = np.array(samples["latent_infections"])[ @@ -852,8 +854,8 @@ print(f" Median infections (day 45): {quantiles_90['q50'][45]:,.0f}") Finally, we visualize the posterior latent infections alongside observed hospitalizations. Note that **hospital admissions lag behind infections** by the infection-to-hospitalization delay (mode ~10 days in our delay PMF). When comparing the two panels, peaks in the infection curve should precede corresponding peaks in hospitalizations by roughly 10-14 days. ```{python} -# | label: fig-posterior-90 -# | fig-cap: Posterior latent infections and observed hospitalizations (90 days). +#| label: fig-posterior-90 +#| fig-cap: Posterior latent infections and observed hospitalizations (90 days). # Visualize posterior latent infections and observed hospitalizations (90 days) # Create separate dataframes for faceted plot infections_df_90 = pd.DataFrame( @@ -886,7 +888,7 @@ hosp_df_90 = pd.DataFrame( plot_df_90 = pd.concat([infections_df_90, hosp_df_90], ignore_index=True) plot_df_90["signal"] = pd.Categorical( plot_df_90["signal"], - categories=["Latent Infections", "Hospital Admissions (14-day MA)"], + categories=["Hospital Admissions (14-day MA)", "Latent Infections"], ordered=True, ) @@ -909,6 +911,7 @@ plot_df_90["signal"] = pd.Categorical( size=1, ) + p9.facet_wrap("~signal", ncol=1, scales="free_y") + + p9.scale_y_log10() + p9.labs( x="Day", y="Count", @@ -922,7 +925,7 @@ plot_df_90["signal"] = pd.Categorical( ### Fit: 180 Days ```{python} -# | label: fit-180-days +#| label: fit-180-days # Clear JAX caches to avoid interference jax.clear_caches() @@ -953,7 +956,7 @@ print(f"Elapsed time: {elapsed_180:.1f} seconds") ``` ```{python} -# | label: posterior-shape-180 +#| label: posterior-shape-180 lat_inf_180 = samples_180["latent_infections"] print(f"Posterior samples shape: {lat_inf_180.shape}") print(f"Mean latent infections: {np.mean(lat_inf_180):,.0f}") @@ -962,7 +965,7 @@ print(f"Mean latent infections: {np.mean(lat_inf_180):,.0f}") We check the model fit, as before. ```{python} -# | label: arviz-diagnostics-180 +#| label: arviz-diagnostics-180 # ArviZ diagnostics for 180-day fit samples_180_filtered = filter_samples_for_arviz( samples_180, n_init, num_chains=2 @@ -972,7 +975,7 @@ az.summary(idata_180, var_names=["~latent_infections", "~expected"]) ``` ```{python} -# | label: extract-quantiles-180 +#| label: extract-quantiles-180 quantiles_180 = extract_posterior_quantiles(samples_180, n_init, n_days_180) ci_width_180 = quantiles_180["q95"] - quantiles_180["q05"] @@ -982,8 +985,8 @@ print(f" Median infections (day 90): {quantiles_180['q50'][90]:,.0f}") ``` ```{python} -# | label: fig-posterior-180 -# | fig-cap: Posterior latent infections and observed hospitalizations (180 days). +#| label: fig-posterior-180 +#| fig-cap: Posterior latent infections and observed hospitalizations (180 days). # Visualize posterior latent infections and observed hospitalizations (180 days) infections_df_180 = pd.DataFrame( { @@ -1015,7 +1018,7 @@ hosp_df_180 = pd.DataFrame( plot_df_180 = pd.concat([infections_df_180, hosp_df_180], ignore_index=True) plot_df_180["signal"] = pd.Categorical( plot_df_180["signal"], - categories=["Latent Infections", "Hospital Admissions (14-day MA)"], + categories=["Hospital Admissions (14-day MA)", "Latent Infections"], ordered=True, ) @@ -1038,6 +1041,7 @@ plot_df_180["signal"] = pd.Categorical( size=1, ) + p9.facet_wrap("~signal", ncol=1, scales="free_y") + + p9.scale_y_log10() + p9.labs( x="Day", y="Count", @@ -1053,7 +1057,7 @@ plot_df_180["signal"] = pd.Categorical( Comparing the two fits reveals where uncertainty reduction occurs—and why it matters for forecasting. ```{python} -# | label: compare-fits +#| label: compare-fits # Compare CI widths for the overlapping 90-day period ci_width_90_overlap = quantiles_90["q95"] - quantiles_90["q05"] ci_width_180_overlap = ( diff --git a/docs/tutorials/custom_randomvariables.qmd b/docs/tutorials/custom_randomvariables.qmd index 033ecc73..9e478fbc 100644 --- a/docs/tutorials/custom_randomvariables.qmd +++ b/docs/tutorials/custom_randomvariables.qmd @@ -2,33 +2,21 @@ title: Defining Custom PyRenew RandomVariables format: gfm engine: jupyter +jupyter: + jupytext: + text_representation: + extension: .qmd + format_name: quarto + format_version: '1.0' + jupytext_version: 1.18.1 + kernelspec: + display_name: Python 3 (ipykernel) + language: python + name: python3 --- -This tutorial illustrates how define new `RandomVariable` classes in order to extend existing models and build new ones, -according to [PyRenew principles](../developer_documentation.md){pyrenew-principles}. - -An example of a custom `RandomVariable` class is the `InfectionsWithFeedback` class which models -the number of infections at time $t$ as a function of the number of infections at time $t - \tau$ as follows: - -$$I(t) = \mathcal{R}(t) \sum_{\tau = 1}^{T_g} I(t-\tau) g(\tau)$$ - -where $\mathcal{R}(t)$ is the reproduction number at time $t$, -and $g(\tau)$ -is the distribution of times from incident infection to secondary infection, i.e., the infectiousness profile. - -The reproduction number at time $t$ is a function of the unadjusted reproduction number at time $t$ -$\mathcal{R}^\mathrm{u}(t)$ and a damping factor which provides feedback from recent transmission: - -$$\mathcal{R}(t) = \mathcal{R}^u(t)\exp\left(-\gamma(t)\sum_{\tau=1}^{T_f}I(t - \tau)f(\tau)\right)$$ - -where $\gamma \geq 0$ scales the feedback strength -and the function $f(\tau)$ is the time-scale over which past infections influence -the current time-varying reproductive number $\mathcal{R}(t)$. -Because $\gamma$ is constrained to be positive, this provides a negative feedback mechanism. - - ```{python} -# | label: setup +#| label: setup import jax import jax.numpy as jnp import numpy as np @@ -68,7 +56,7 @@ set to 0.01 to create moderate damping. on $\log(\mathcal{R}^\mathrm{u}(t))$, (as in the basic renewal model). ```{python} -# | label: model-components +#| label: model-components gen_int_array = jnp.array([0.25, 0.25, 0.25, 0.25]) gen_int = DeterministicPMF(name="gen_int", value=gen_int_array) feedback_strength = DeterministicVariable(name="feedback_strength", value=0.01) @@ -117,7 +105,7 @@ class MyRt(RandomVariable): We build the model from these components and then simulate data from it without observed infections. ```{python} -# | label: build1 +#| label: build1 model0 = RtInfectionsRenewalModel( gen_int_rv=gen_int, I0_rv=I0, @@ -128,15 +116,15 @@ model0 = RtInfectionsRenewalModel( ``` ```{python} -# | label: simulate1 +#| label: simulate1 # Sampling and fitting model 0 (with no obs for infections) with numpyro.handlers.seed(rng_seed=223): model0_samp = model0.sample(n_datapoints=30) ``` ```{python} -# | label: fig-simulate1 -# | fig-cap: Simulated infections with no observation process +#| label: fig-simulate1 +#| fig-cap: Simulated infections with no observation process import matplotlib.pyplot as plt fig, ax = plt.subplots() @@ -188,7 +176,7 @@ but omits the tedious checks on its inputs. Although returning a `namedtuple` is not strictly required, they are the recommended return type, as they make the code more readable. The following code-chunk shows how to create a `namedtuple` for the `InfectionsWithFeedback` class: ```{python} -# | label: data-class +#| label: data-class from collections import namedtuple # Creating a tuple to store the output @@ -202,8 +190,8 @@ InfFeedbackSample = namedtuple( The next step is to create the actual class. The bulk of its implementation lies in the function `pyrenew.latent.compute_infections_from_rt_with_feedback()`. We will also use the `pyrenew.arrayutils.pad_edges_to_match()` function to ensure the passed vectors match their lengths. The following code-chunk shows most of the implementation of the `InfectionsWithFeedback` class: ```{python} -# | label: new-model-def -# | code-line-numbers: true +#| label: new-model-def +#| code-line-numbers: true # Creating the class from pyrenew.metaclass import RandomVariable from pyrenew.latent import compute_infections_from_rt_with_feedback @@ -296,7 +284,7 @@ To check our work, we build a second model using our new class `InfFeedback` and Since class `InfFeedback` exactly follows the `InfectionsWithFeedback` class, if correctly implemented, the two models should provide exactly the same predictions. ```{python} -# | label: simulation2 +#| label: simulation2 latent_infections2 = InfFeedback( infection_feedback_strength=feedback_strength, infection_feedback_pmf=gen_int, @@ -318,8 +306,8 @@ with numpyro.handlers.seed(rng_seed=223): Comparing `model0` with `model1`, these two should match: ```{python} -# | label: fig-model0-vs-model1 -# | fig-cap: Comparing latent infections from model 0 and model 1 +#| label: fig-model0-vs-model1 +#| fig-cap: Comparing latent infections from model 0 and model 1 import matplotlib.pyplot as plt fig, ax = plt.subplots(ncols=2) diff --git a/docs/tutorials/latent_hierarchical_infections.qmd b/docs/tutorials/latent_hierarchical_infections.qmd index df061323..567ceef6 100644 --- a/docs/tutorials/latent_hierarchical_infections.qmd +++ b/docs/tutorials/latent_hierarchical_infections.qmd @@ -552,7 +552,7 @@ The prior predictive distribution changes dramatically based on your temporal pr # | label: temporal-process-samples temporal_configs = { "AR(1)": AR1(autoreg=0.9, innovation_sd=0.05), - "DifferencedAR1": DifferencedAR1(autoreg=0.9, innovation_sd=0.05), + "DifferencedAR1": DifferencedAR1(autoreg=0.5, innovation_sd=0.05), "RandomWalk": RandomWalk(innovation_sd=0.05), } @@ -619,7 +619,7 @@ comparison_df["process"] = pd.Categorical( x="Days", y="Rt (baseline)", title="Prior Predictive by Temporal Process", - subtitle="Same innovation_sd=0.05; AR processes have autoreg=0.9", + subtitle="Same innovation_sd=0.05; AR(1) autoreg=0.9, DifferencedAR1 autoreg=0.5", ) + p9.theme_minimal() + p9.theme(figure_size=(12, 4)) From ca90f420eb35d99c79c657a922bcc5bca67e69a3 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 6 Feb 2026 15:06:04 +0000 Subject: [PATCH 81/81] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- .../tutorials/building_multisignal_models.qmd | 66 ++++++++++--------- docs/tutorials/custom_randomvariables.qmd | 24 +++---- 2 files changed, 46 insertions(+), 44 deletions(-) diff --git a/docs/tutorials/building_multisignal_models.qmd b/docs/tutorials/building_multisignal_models.qmd index 46d5e853..f4501677 100644 --- a/docs/tutorials/building_multisignal_models.qmd +++ b/docs/tutorials/building_multisignal_models.qmd @@ -20,8 +20,8 @@ jupyter: --- ```{python} -#| label: setup -#| output: false +# | label: setup +# | output: false import numpyro # to run samplers in parallel you must run `set_host_device_count` before importing jax @@ -30,7 +30,7 @@ numpyro.enable_x64() ``` ```{python} -#| label: imports-base +# | label: imports-base import arviz as az import jax import jax.numpy as jnp @@ -52,7 +52,7 @@ def make_rng_key(): ``` ```{python} -#| label: imports-pyrenew +# | label: imports-pyrenew from jax.typing import ArrayLike from pyrenew import datasets @@ -187,7 +187,7 @@ Latent infection processes implement the renewal equation to generate infection The generation interval PMF specifies the probability that a secondary infection occurs $d$ days after the primary infection. ```{python} -#| label: gen-interval +# | label: gen-interval covid_gen_int = [0.16, 0.32, 0.25, 0.14, 0.07, 0.04, 0.02] gen_int_pmf = jnp.array(covid_gen_int) gen_int_rv = DeterministicPMF("gen_int", gen_int_pmf) @@ -202,7 +202,7 @@ print(f"Generation interval length: {len(gen_int_pmf)} days") The initial infections RV `I0_rv` specifies the **proportion of the population infected** at the first observation time. This must be a value in the interval (0, 1]. We use a Beta prior centered near a small value: ```{python} -#| label: initial-infections +# | label: initial-infections I0_rv = DistributionalVariable("I0", dist.Beta(1, 100)) ``` @@ -211,8 +211,10 @@ I0_rv = DistributionalVariable("I0", dist.Beta(1, 100)) We place a prior on the initial log(Rt), centered at 0.0 (Rt = 1.0) with moderate uncertainty: ```{python} -#| label: initial-log-rt -initial_log_rt_rv = DistributionalVariable("initial_log_rt", dist.Normal(0.0, 0.5)) +# | label: initial-log-rt +initial_log_rt_rv = DistributionalVariable( + "initial_log_rt", dist.Normal(0.0, 0.5) +) ``` ### Temporal Processes for $R(t)$ @@ -225,7 +227,7 @@ We configure two temporal processes: The RandomWalk allows flexible evolution of subpopulation-specific transmission without mean reversion. ```{python} -#| label: temporal-processes +# | label: temporal-processes # AR1 provides mean-reverting behavior for baseline Rt baseline_rt_process = AR1(autoreg=0.9, innovation_sd=0.05) @@ -261,7 +263,7 @@ In this example we use a dataset consisting of hospital admissions for COVID-19 for the first 10 months of 2023 (as reported to the CDC). ```{python} -#| label: load-hospital-data +# | label: load-hospital-data # Load daily hospital admissions for California ca_hosp_data = datasets.load_hospital_data_for_state("CA", "2023-11-06.csv") @@ -281,7 +283,7 @@ print( The hospital admissions data is aggregated at the jurisdiction level, therefore we specify a `Counts` observation process. ```{python} -#| label: hospital-obs-process +# | label: hospital-obs-process # Infection-to-hospitalization delay (COVID-19, from literature) inf_to_hosp_pmf = jnp.array( [ @@ -360,7 +362,7 @@ implementations subclass it and implement `_predicted_obs()` to transform infect into predicted values. See `observation_processes_measurements.qmd` for a detailed tutorial. ```{python} -#| label: wastewater-class +# | label: wastewater-class class Wastewater(Measurements): """ Wastewater viral concentration observation process. @@ -417,7 +419,7 @@ For the wastewater data, we use a simulated dataset for California with realisti that covers the same time period. ```{python} -#| label: load-wastewater-data +# | label: load-wastewater-data # Load wastewater data for California ca_ww_data = datasets.load_wastewater_data_for_state("CA", "fake_nwss.csv") @@ -447,7 +449,7 @@ Wastewater observations are site-level: each measurement is associated with a sp Here we specify HierarchicalNormalPrior for the site-level mode and GammaGroupSdPrior for the standard deviation. ```{python} -#| label: wastewater-obs-process +# | label: wastewater-obs-process # Viral shedding kinetics PMF (days post-infection) shedding_pmf = jnp.array( [ @@ -503,7 +505,7 @@ print(f" Shedding PMF length: {len(shedding_pmf)} days") We instantiate a `PyrenewBuilder` object which handles the composition of the latent infection process and the observation process. ```{python} -#| label: model-builder-init +# | label: model-builder-init # Build the multi-signal model builder = PyrenewBuilder() ``` @@ -521,7 +523,7 @@ Methods `configure_latent` and `add_observation` can be called in any order. Met We use `configure_latent` to specify the **model structure**: generation interval, initial infections, and temporal dynamics. ```{python} -#| label: configure-latent +# | label: configure-latent print("Latent process configuration:") print(f" Generation interval length: {len(gen_int_rv())} days") @@ -540,7 +542,7 @@ builder.configure_latent( Each observation process's `name` attribute becomes the keyword used to pass that observation's data to `model.run()` (e.g., `hospital={...}`, `wastewater={...}`). ```{python} -#| label: add-observations-build +# | label: add-observations-build builder.add_observation(hosp_obs) # Uses hosp_obs.name = "hospital" builder.add_observation(ww_obs) # Uses ww_obs.name = "wastewater" model = builder.build() @@ -662,7 +664,7 @@ samples = model.mcmc.get_samples() First we declare the population structure. We have 6 subpopulations, where 5 have wastewater monitoring and 1 does not. The subpopulations with wastewater monitoring need not be contiguous indices—they could be any subset of {0, 1, ..., n_subpops-1}. ```{python} -#| label: population-structure +# | label: population-structure # All 6 subpopulations with their population fractions subpop_fractions = jnp.array([0.10, 0.14, 0.21, 0.22, 0.07, 0.26]) @@ -688,7 +690,7 @@ print(f"Total population: {float(jnp.sum(subpop_fractions)):.0%}") We define a function to prepare observation data using the model's helper methods to align with the shared time axis. ```{python} -#| label: prepare-observation-data +# | label: prepare-observation-data def prepare_observation_data( model, n_days_fit, hosp_admits, ww_data, ww_monitored_subpops ): @@ -755,7 +757,7 @@ Putting this altogether, we align the data with the model time and call `model.r We run 4 sampler chains. ```{python} -#| label: fit-90-days +# | label: fit-90-days # Clear JAX caches to avoid interference from earlier cells jax.clear_caches() @@ -787,7 +789,7 @@ print(f"Elapsed time: {elapsed_90:.1f} seconds") ``` ```{python} -#| label: posterior-shape-90 +# | label: posterior-shape-90 lat_inf_90 = samples_90["latent_infections"] print(f"Posterior samples shape: {lat_inf_90.shape}") print(f"Mean latent infections: {np.mean(lat_inf_90):,.0f}") @@ -796,7 +798,7 @@ print(f"Mean latent infections: {np.mean(lat_inf_90):,.0f}") We check that the chains have converged and the number of effective samples. ```{python} -#| label: arviz-diagnostics-90 +# | label: arviz-diagnostics-90 # ArviZ diagnostics for 90-day fit def filter_samples_for_arviz(samples, n_init, num_chains=4): """Slice time-series samples to exclude initialization period and reshape for chains.""" @@ -829,7 +831,7 @@ az.summary(idata_90, var_names=["~latent_infections", "~expected"]) We extract the posterior quantiles and print summary statistics. ```{python} -#| label: extract-quantiles-90 +# | label: extract-quantiles-90 def extract_posterior_quantiles(samples, n_init, n_days): """Extract posterior quantiles for latent infections.""" latent_inf = np.array(samples["latent_infections"])[ @@ -854,8 +856,8 @@ print(f" Median infections (day 45): {quantiles_90['q50'][45]:,.0f}") Finally, we visualize the posterior latent infections alongside observed hospitalizations. Note that **hospital admissions lag behind infections** by the infection-to-hospitalization delay (mode ~10 days in our delay PMF). When comparing the two panels, peaks in the infection curve should precede corresponding peaks in hospitalizations by roughly 10-14 days. ```{python} -#| label: fig-posterior-90 -#| fig-cap: Posterior latent infections and observed hospitalizations (90 days). +# | label: fig-posterior-90 +# | fig-cap: Posterior latent infections and observed hospitalizations (90 days). # Visualize posterior latent infections and observed hospitalizations (90 days) # Create separate dataframes for faceted plot infections_df_90 = pd.DataFrame( @@ -925,7 +927,7 @@ plot_df_90["signal"] = pd.Categorical( ### Fit: 180 Days ```{python} -#| label: fit-180-days +# | label: fit-180-days # Clear JAX caches to avoid interference jax.clear_caches() @@ -956,7 +958,7 @@ print(f"Elapsed time: {elapsed_180:.1f} seconds") ``` ```{python} -#| label: posterior-shape-180 +# | label: posterior-shape-180 lat_inf_180 = samples_180["latent_infections"] print(f"Posterior samples shape: {lat_inf_180.shape}") print(f"Mean latent infections: {np.mean(lat_inf_180):,.0f}") @@ -965,7 +967,7 @@ print(f"Mean latent infections: {np.mean(lat_inf_180):,.0f}") We check the model fit, as before. ```{python} -#| label: arviz-diagnostics-180 +# | label: arviz-diagnostics-180 # ArviZ diagnostics for 180-day fit samples_180_filtered = filter_samples_for_arviz( samples_180, n_init, num_chains=2 @@ -975,7 +977,7 @@ az.summary(idata_180, var_names=["~latent_infections", "~expected"]) ``` ```{python} -#| label: extract-quantiles-180 +# | label: extract-quantiles-180 quantiles_180 = extract_posterior_quantiles(samples_180, n_init, n_days_180) ci_width_180 = quantiles_180["q95"] - quantiles_180["q05"] @@ -985,8 +987,8 @@ print(f" Median infections (day 90): {quantiles_180['q50'][90]:,.0f}") ``` ```{python} -#| label: fig-posterior-180 -#| fig-cap: Posterior latent infections and observed hospitalizations (180 days). +# | label: fig-posterior-180 +# | fig-cap: Posterior latent infections and observed hospitalizations (180 days). # Visualize posterior latent infections and observed hospitalizations (180 days) infections_df_180 = pd.DataFrame( { @@ -1057,7 +1059,7 @@ plot_df_180["signal"] = pd.Categorical( Comparing the two fits reveals where uncertainty reduction occurs—and why it matters for forecasting. ```{python} -#| label: compare-fits +# | label: compare-fits # Compare CI widths for the overlapping 90-day period ci_width_90_overlap = quantiles_90["q95"] - quantiles_90["q05"] ci_width_180_overlap = ( diff --git a/docs/tutorials/custom_randomvariables.qmd b/docs/tutorials/custom_randomvariables.qmd index 9e478fbc..53ece1ee 100644 --- a/docs/tutorials/custom_randomvariables.qmd +++ b/docs/tutorials/custom_randomvariables.qmd @@ -16,7 +16,7 @@ jupyter: --- ```{python} -#| label: setup +# | label: setup import jax import jax.numpy as jnp import numpy as np @@ -56,7 +56,7 @@ set to 0.01 to create moderate damping. on $\log(\mathcal{R}^\mathrm{u}(t))$, (as in the basic renewal model). ```{python} -#| label: model-components +# | label: model-components gen_int_array = jnp.array([0.25, 0.25, 0.25, 0.25]) gen_int = DeterministicPMF(name="gen_int", value=gen_int_array) feedback_strength = DeterministicVariable(name="feedback_strength", value=0.01) @@ -105,7 +105,7 @@ class MyRt(RandomVariable): We build the model from these components and then simulate data from it without observed infections. ```{python} -#| label: build1 +# | label: build1 model0 = RtInfectionsRenewalModel( gen_int_rv=gen_int, I0_rv=I0, @@ -116,15 +116,15 @@ model0 = RtInfectionsRenewalModel( ``` ```{python} -#| label: simulate1 +# | label: simulate1 # Sampling and fitting model 0 (with no obs for infections) with numpyro.handlers.seed(rng_seed=223): model0_samp = model0.sample(n_datapoints=30) ``` ```{python} -#| label: fig-simulate1 -#| fig-cap: Simulated infections with no observation process +# | label: fig-simulate1 +# | fig-cap: Simulated infections with no observation process import matplotlib.pyplot as plt fig, ax = plt.subplots() @@ -176,7 +176,7 @@ but omits the tedious checks on its inputs. Although returning a `namedtuple` is not strictly required, they are the recommended return type, as they make the code more readable. The following code-chunk shows how to create a `namedtuple` for the `InfectionsWithFeedback` class: ```{python} -#| label: data-class +# | label: data-class from collections import namedtuple # Creating a tuple to store the output @@ -190,8 +190,8 @@ InfFeedbackSample = namedtuple( The next step is to create the actual class. The bulk of its implementation lies in the function `pyrenew.latent.compute_infections_from_rt_with_feedback()`. We will also use the `pyrenew.arrayutils.pad_edges_to_match()` function to ensure the passed vectors match their lengths. The following code-chunk shows most of the implementation of the `InfectionsWithFeedback` class: ```{python} -#| label: new-model-def -#| code-line-numbers: true +# | label: new-model-def +# | code-line-numbers: true # Creating the class from pyrenew.metaclass import RandomVariable from pyrenew.latent import compute_infections_from_rt_with_feedback @@ -284,7 +284,7 @@ To check our work, we build a second model using our new class `InfFeedback` and Since class `InfFeedback` exactly follows the `InfectionsWithFeedback` class, if correctly implemented, the two models should provide exactly the same predictions. ```{python} -#| label: simulation2 +# | label: simulation2 latent_infections2 = InfFeedback( infection_feedback_strength=feedback_strength, infection_feedback_pmf=gen_int, @@ -306,8 +306,8 @@ with numpyro.handlers.seed(rng_seed=223): Comparing `model0` with `model1`, these two should match: ```{python} -#| label: fig-model0-vs-model1 -#| fig-cap: Comparing latent infections from model 0 and model 1 +# | label: fig-model0-vs-model1 +# | fig-cap: Comparing latent infections from model 0 and model 1 import matplotlib.pyplot as plt fig, ax = plt.subplots(ncols=2)