From a453ccbd690543d35360a328cae9398c829f064c Mon Sep 17 00:00:00 2001 From: Farhan Date: Sun, 28 Jun 2026 20:32:25 +0500 Subject: [PATCH 01/56] feat(compiler): add incremental compile cache (REFLEX_COMPILE_CACHE) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add an experimental, flag-gated incremental frontend compile cache that recompiles only the pages whose source actually changed and reuses the rest. Two layers, both off by default and enabled by REFLEX_COMPILE_CACHE: - In-process per-page cache (page_cache.py): a Salsa-style dependency graph records the exact set of source files each page reads, so editing one file invalidates only the pages that depend on it. Pages are keyed by a small genuinely-global epoch (Reflex version + rxconfig + lockfile) plus the content hashes of their dependency set. Speeds up repeat compiles within a single process. - On-disk manifest (disk_cache.py): persists each page's serializable contribution and dependency hashes to .web/reflex_compile_cache.json so a fresh process — notably a `reflex run` hot-reload worker, which respawns on every edit — recompiles only changed pages and reuses the rest. Falls back to a full compile on any unsafe condition. REFLEX_COMPILE_CACHE_VERIFY runs a full compile alongside the cached one and asserts byte-identical output, falling back on mismatch — the backstop for gaps a static dependency graph cannot see (runtime importlib imports, data read at module-import time). Supporting changes required for safe page reuse: deterministic compile-time ref-name generation, and own-before-mutate page metadata injection. --- .../src/reflex_base/environment.py | 12 + .../src/reflex_base/plugins/compiler.py | 59 +- .../reflex-base/src/reflex_base/vars/base.py | 20 +- reflex/compiler/compiler.py | 235 +++++- reflex/compiler/disk_cache.py | 453 +++++++++++ reflex/compiler/page_cache.py | 726 ++++++++++++++++++ reflex/compiler/plugins/builtin.py | 2 +- reflex/compiler/utils.py | 12 +- tests/units/compiler/test_disk_cache.py | 287 +++++++ tests/units/compiler/test_page_cache.py | 115 +++ tests/units/vars/test_base.py | 22 + 11 files changed, 1914 insertions(+), 29 deletions(-) create mode 100644 reflex/compiler/disk_cache.py create mode 100644 reflex/compiler/page_cache.py create mode 100644 tests/units/compiler/test_disk_cache.py create mode 100644 tests/units/compiler/test_page_cache.py diff --git a/packages/reflex-base/src/reflex_base/environment.py b/packages/reflex-base/src/reflex_base/environment.py index 99521d1a339..cca5de00ba9 100644 --- a/packages/reflex-base/src/reflex_base/environment.py +++ b/packages/reflex-base/src/reflex_base/environment.py @@ -611,6 +611,18 @@ class EnvironmentVariables: # If this env var is set to "yes", App.compile will be a no-op REFLEX_SKIP_COMPILE: EnvVar[bool] = env_var(False, internal=True) + # Experimental: incremental compile cache. A fresh compile process (e.g. a + # reflex-run hot-reload worker) reuses each page's compiled output from an + # on-disk manifest and recompiles only the pages whose source changed, + # backed by an in-process per-page cache for repeat compiles in one process. + # See reflex/compiler/disk_cache.py and reflex/compiler/page_cache.py. + REFLEX_COMPILE_CACHE: EnvVar[bool] = env_var(False) + + # When the compile cache reuses pages, also run a full compile and assert + # byte-identical output, falling back to the full result on any mismatch. + # Doubles compile time; for validating the cache on an app. + REFLEX_COMPILE_CACHE_VERIFY: EnvVar[bool] = env_var(False) + # Inherited by uvicorn/granian reload workers so the backend can distinguish # dev reload-capable worker boots from other backend starts. Never set in prod. REFLEX_DEV_BACKEND_RELOAD_ACTIVE: EnvVar[bool] = env_var(False, internal=True) diff --git a/packages/reflex-base/src/reflex_base/plugins/compiler.py b/packages/reflex-base/src/reflex_base/plugins/compiler.py index c8ffe6d2711..dd02fabda08 100644 --- a/packages/reflex-base/src/reflex_base/plugins/compiler.py +++ b/packages/reflex-base/src/reflex_base/plugins/compiler.py @@ -6,6 +6,7 @@ import dataclasses import inspect from collections.abc import Callable, Sequence +from contextlib import AbstractContextManager from contextvars import ContextVar, Token from types import TracebackType from typing import TYPE_CHECKING, Any, ClassVar, Protocol, TypeAlias, TypeVar, cast @@ -35,6 +36,15 @@ _BaseComponentT = TypeVar("_BaseComponentT", bound=BaseComponent) +#: Optional per-page source-read recorder, installed by an incremental compile +#: cache. When set, :meth:`CompileContext.compile` wraps each page's evaluation +#: in ``page_source_recorder()``; the context manager yields a set that collects +#: the source files (e.g. markdown/data) read during that page's eval, which is +#: then stored on the page's ``source_files``. ``None`` (default) disables +#: recording, so the compile path is unchanged when no cache is active. +page_source_recorder: Callable[[], AbstractContextManager[set[str]]] | None = None + + class PageDefinition(Protocol): """Protocol for page-like objects compiled by :class:`CompileContext`.""" @@ -690,6 +700,18 @@ class PageContext(BaseContext): output_path: str | None = None output_code: str | None = None source_module: str | None = None + # Source files (e.g. markdown/data) read while evaluating this page, when a + # per-page read recorder is installed (see ``page_source_recorder``). Lets an + # incremental cache depend on the exact non-import inputs a page consumed, so + # editing one page's data invalidates only that page. Empty when no recorder + # is active. + source_files: set[str] = dataclasses.field(default_factory=set) + # Auto-memo components first registered while compiling THIS page, keyed by + # ``(tag, source_module)``. Lets an incremental cache attribute memo + # contributions per page so a skipped page can re-register them. + memo_contributions: dict[tuple[str, str | None], Any] = dataclasses.field( + default_factory=dict + ) # Stack of ``id(component)`` for components whose subtree is # memoize-suppressed. Populated by ``MemoizeStatefulPlugin`` when it # encounters a ``MemoizationLeaf``-style snapshot boundary and popped on @@ -794,6 +816,7 @@ def compile( """ from reflex.compiler import compiler from reflex.state import all_base_state_classes + from reflex_base.vars.base import reset_unique_variable_names self.ensure_context_attached() self.compiled_pages.clear() @@ -803,15 +826,32 @@ def compile( self.memoize_wrappers.clear() self.auto_memo_components.clear() + # Reset the deterministic ref-name generator so a second in-process + # compile reproduces the same auto-generated names as the first (these + # names feed auto-memo content hashes, so drift breaks reproducibility). + reset_unique_variable_names() + + recorder = page_source_recorder for page in self.pages: page_fn = page.component n_states_before = len(all_base_state_classes) - page_ctx = self.hooks.eval_page( - page_fn, - page=page, - compile_context=self, - **kwargs, - ) + if recorder is not None: + with recorder() as read_set: + page_ctx = self.hooks.eval_page( + page_fn, + page=page, + compile_context=self, + **kwargs, + ) + if page_ctx is not None: + page_ctx.source_files = read_set + else: + page_ctx = self.hooks.eval_page( + page_fn, + page=page, + compile_context=self, + **kwargs, + ) if page_ctx is None: page_name = getattr(page_fn, "__name__", repr(page_fn)) msg = ( @@ -836,6 +876,7 @@ def compile( self.compiled_pages.values(), strict=True, ): + memo_before = set(self.auto_memo_components) with page_ctx: page_ctx.root_component = self.hooks.compile_component( page_ctx.root_component, @@ -848,6 +889,12 @@ def compile( compile_context=self, **kwargs, ) + # Attribute newly-registered auto-memo components to this page. + page_ctx.memo_contributions = { + key: value + for key, value in self.auto_memo_components.items() + if key not in memo_before + } page_ctx.frontend_imports = page_ctx.merged_imports(collapse=True) self.all_imports = merge_imports( diff --git a/packages/reflex-base/src/reflex_base/vars/base.py b/packages/reflex-base/src/reflex_base/vars/base.py index f7212bef821..15030b3e44a 100644 --- a/packages/reflex-base/src/reflex_base/vars/base.py +++ b/packages/reflex-base/src/reflex_base/vars/base.py @@ -9,6 +9,7 @@ import functools import inspect import json +import random import re import string import uuid @@ -44,7 +45,6 @@ from reflex_base.constants.state import FIELD_MARKER from reflex_base.utils import console, exceptions, imports, serializers, types from reflex_base.utils.compat import annotations_from_namespace -from reflex_base.utils.decorator import once from reflex_base.utils.exceptions import ( ComputedVarSignatureError, UntypedComputedVarError, @@ -3211,12 +3211,20 @@ def get_uuid_string_var() -> Var: # Set of unique variable names. USED_VARIABLES = set() +_UNIQUE_NAME_RNG = random.Random(42) -@once -def _rng(): - import random - return random.Random(42) +def reset_unique_variable_names() -> None: + """Reset the deterministic unique-name generator to its initial state. + + ``get_unique_variable_name`` draws from a seeded RNG and dedups against + ``USED_VARIABLES``; both persist process-wide, so a second in-process compile + would draw *different* ref names than the first (the RNG state and the used + set carry over). Resetting them before each compile makes the generated names + reproducible across compiles — names only need to be unique within a compile. + """ + USED_VARIABLES.clear() + _UNIQUE_NAME_RNG.seed(42) def get_unique_variable_name() -> str: @@ -3225,7 +3233,7 @@ def get_unique_variable_name() -> str: Returns: The unique variable name. """ - name = "".join([_rng().choice(string.ascii_lowercase) for _ in range(8)]) + name = "".join([_UNIQUE_NAME_RNG.choice(string.ascii_lowercase) for _ in range(8)]) if name not in USED_VARIABLES: USED_VARIABLES.add(name) return name diff --git a/reflex/compiler/compiler.py b/reflex/compiler/compiler.py index f0185d793d3..de5433d3697 100644 --- a/reflex/compiler/compiler.py +++ b/reflex/compiler/compiler.py @@ -63,6 +63,24 @@ def _set_progress_total( progress.update(task, total=total) +def make_compile_progress(use_rich: bool) -> Progress | console.PoorProgress: + """Build a compile progress bar. + + Args: + use_rich: Whether to use a rich progress bar (else a plain fallback). + + Returns: + A progress bar suitable for tracking a compile. + """ + if use_rich: + return Progress( + *Progress.get_default_columns()[:-1], + MofNCompleteColumn(), + TimeElapsedColumn(), + ) + return console.PoorProgress() + + def _apply_common_imports( imports: dict[str, list[ImportVar]], ): @@ -954,7 +972,7 @@ def compile_unevaluated_page( meta_args["description"] = page.description # Add meta information to the component. - utils.add_meta( + component = utils.add_meta( component, **meta_args, ) @@ -1090,6 +1108,91 @@ def _resolve_radix_themes_plugin( return plugin_chain, radix_plugin +def _normalize_imports_for_compare(all_imports: Any) -> dict[str, list[str]]: + """Render an import dict to a comparable, order-independent form. + + Args: + all_imports: The parsed import dict to normalize. + + Returns: + A mapping of library to its sorted, stringified import fields. + """ + return {lib: sorted(str(v) for v in fields) for lib, fields in all_imports.items()} + + +def _diff_compile_contexts( + incremental: CompileContext, full: CompileContext +) -> list[str]: + """Return the aggregate fields where the two compile contexts diverge. + + Args: + incremental: The cache-assisted compile context. + full: A no-cache full compile of the same app. + + Returns: + A list of human-readable divergence labels (empty when identical). + """ + diffs: list[str] = [] + inc_pages = {r: pc.output_code for r, pc in incremental.compiled_pages.items()} + full_pages = {r: pc.output_code for r, pc in full.compiled_pages.items()} + if inc_pages.keys() != full_pages.keys(): + diffs.append(f"routes:{sorted(set(inc_pages) ^ set(full_pages))}") + diffs.extend( + f"page:{r}" + for r in inc_pages.keys() & full_pages.keys() + if inc_pages[r] != full_pages[r] + ) + if _normalize_imports_for_compare( + incremental.all_imports + ) != _normalize_imports_for_compare(full.all_imports): + diffs.append("all_imports") + if sorted(map(str, incremental.auto_memo_components)) != sorted( + map(str, full.auto_memo_components) + ): + diffs.append("auto_memo_components") + if sorted(incremental.stateful_routes) != sorted(full.stateful_routes): + diffs.append("stateful_routes") + if sorted(map(str, incremental.app_wrap_components)) != sorted( + map(str, full.app_wrap_components) + ): + diffs.append("app_wrap_components") + return diffs + + +def _full_compile_context( + app: App, compiler_plugins: Sequence[Plugin] +) -> CompileContext: + """Compile every page with no cache reuse (for verify-mode comparison). + + Resets the shared bundling/memo globals first so it starts from the same + clean state the primary compile did. + + Args: + app: The app to compile. + compiler_plugins: The resolved compiler plugins. + + Returns: + A fully compiled context over all pages. + """ + from reflex_base.components.dynamic import bundle_library, reset_bundled_libraries + + reset_bundled_libraries() + reset_memo_component_classes() + for plugin in compiler_plugins: + for dependency in plugin.get_frontend_dependencies(): + bundle_library(dependency) + ctx = CompileContext( + app=app, + pages=list(app._unevaluated_pages.values()), + hooks=CompilerHooks( + plugins=default_page_plugins(style=app.style, plugins=compiler_plugins) + ), + ) + with ctx: + ctx.compile() + return ctx + + def compile_app( app: App, *, @@ -1138,20 +1241,30 @@ def compile_app( app._add_optional_endpoints() return False - progress = ( - Progress( - *Progress.get_default_columns()[:-1], - MofNCompleteColumn(), - TimeElapsedColumn(), - ) - if use_rich - else console.PoorProgress() - ) - fixed_steps = 7 + cache_on = not dry_run and environment.REFLEX_COMPILE_CACHE.get() + compiler_plugins, radix_themes_plugin = _resolve_radix_themes_plugin( app, config.plugins, ) + + # Experimental incremental compile cache: in a fresh process, recompile only + # the pages whose source changed and reuse the rest from the on-disk + # manifest. Falls back to a full compile on any unsafe condition. + if cache_on: + from reflex.compiler import disk_cache, page_cache + + page_cache.enable_read_tracking() + if disk_cache.try_incremental_rebuild( + app, + compiler_plugins=compiler_plugins, + prerender_routes=prerender_routes, + use_rich=use_rich, + ): + return True + + progress = make_compile_progress(use_rich) + fixed_steps = 7 reset_bundled_libraries() # Drop cached memo wrapper classes so each compile recomputes a memo's # ``library`` from the current module layout (handles a module flipping to @@ -1163,9 +1276,36 @@ def compile_app( base_total = (len(app._unevaluated_pages) * 2) + fixed_steps + len(config.plugins) progress.start() task = progress.add_task("Compiling:", total=base_total) + all_pages = list(app._unevaluated_pages.values()) + + # In-process per-page cache with a Salsa-style dependency graph. Reuse the + # compiled context of pages whose recorded dependency set is byte-unchanged + # (and whose global epoch matches); compile only the rest, then re-merge + # cached pages' contributions into the app-wide aggregates. In-process only + # (contributions hold live objects), so it helps repeat compiles in one + # process; cross-process reuse is handled by the disk cache above. + in_process_cache = cache_on + route_to_component = {p.route: p.component for p in all_pages} + state_index: dict[str, Any] = {} + epoch = "" + hasher: Any = None + hit_routes: list[str] = [] + if in_process_cache: + from reflex.compiler import page_cache + + state_index, _ = page_cache.state_dependency_index() + epoch = page_cache.global_epoch() + hasher = page_cache.make_hasher() + hit_routes = [ + page.route + for page in all_pages + if page_cache.validate_page(page.route, epoch, hasher) is not None + ] + + hit_set = set(hit_routes) compile_ctx = CompileContext( app=app, - pages=list(app._unevaluated_pages.values()), + pages=[p for p in all_pages if p.route not in hit_set], hooks=CompilerHooks( plugins=default_page_plugins(style=app.style, plugins=compiler_plugins) ), @@ -1177,6 +1317,72 @@ def compile_app( render_progress=lambda: progress.advance(task), ) + if in_process_cache: + from reflex.compiler import page_cache + + # Cache freshly-compiled (miss) pages with the dependency set they read. + for route in list(compile_ctx.compiled_pages): + page_ctx_fresh = compile_ctx.compiled_pages[route] + dep_hashes = page_cache.page_dependency_hashes( + page_ctx_fresh, + route_to_component[route], + state_index, + hasher, + ) + page_cache.store_page( + route, + epoch, + dep_hashes, + page_ctx_fresh, + route in compile_ctx.stateful_routes, + ) + # Re-merge cached (hit) pages' contributions into the aggregates. + for route in hit_routes: + cached = page_cache.validate_page(route, epoch, hasher) + if cached is None: # defensive: invalidated concurrently + continue + page_ctx, is_stateful = cached + compile_ctx.compiled_pages[route] = page_ctx + compile_ctx.all_imports = utils.merge_imports( + compile_ctx.all_imports, page_ctx.frontend_imports + ) + compile_ctx.app_wrap_components.update(page_ctx.app_wrap_components) + compile_ctx.auto_memo_components.update(page_ctx.memo_contributions) + if is_stateful: + compile_ctx.stateful_routes[route] = None + # Restore deterministic page order (route order of the app). + compile_ctx.compiled_pages = { + p.route: compile_ctx.compiled_pages[p.route] + for p in all_pages + if p.route in compile_ctx.compiled_pages + } + if hit_routes: + console.info( + "Incremental compile: recompiled " + f"{len(all_pages) - len(hit_routes)} page(s)." + ) + + # Verify mode: prove the cache-assisted output matches a full compile, and + # fall back to the full result on any divergence so a cache bug can never + # ship. Doubles compile time; opt-in for validation. + if hit_routes and environment.REFLEX_COMPILE_CACHE_VERIFY.get(): + from reflex.compiler import page_cache + + full_ctx = _full_compile_context(app, compiler_plugins) + diffs = _diff_compile_contexts(compile_ctx, full_ctx) + if diffs: + console.warn( + "compile cache verify FAILED " + f"({len(diffs)} divergence(s): {diffs[:8]}); using the full " + "compile and clearing the page cache." + ) + page_cache.clear_page_store() + compile_ctx = full_ctx + else: + console.debug( + "compile cache verify: incremental output matches full compile" + ) + for route, page_ctx in compile_ctx.compiled_pages.items(): app._check_routes_conflict(route) if not isinstance(page_ctx.root_component, Component): @@ -1407,4 +1613,9 @@ def add_save_task( for output_path, code in output_mapping.items(): utils.write_file(output_path, code) + if cache_on: + from reflex.compiler import disk_cache + + disk_cache.write_manifest(compile_ctx, all_pages, all_imports) + return True diff --git a/reflex/compiler/disk_cache.py b/reflex/compiler/disk_cache.py new file mode 100644 index 00000000000..4c4cbf3bc9d --- /dev/null +++ b/reflex/compiler/disk_cache.py @@ -0,0 +1,453 @@ +"""Experimental disk-persisted incremental compile cache (flag-gated). + +Enabled by ``REFLEX_COMPILE_CACHE``. When off (default), nothing changes. + +The in-process page cache (``page_cache._PAGE_STORE``) reuses compiled pages +within one process, so it never fires in the ``reflex run`` edit loop — the +reloader respawns a fresh worker subprocess on every ``.py`` change, starting +with an empty in-memory store. This cache persists each page's *serializable* +contribution to disk so the fresh worker can reuse it. + +**What is persisted** (``.web/reflex_compile_cache.json``): per page, the +``{path: hash}`` of its **dependency set** (the exact source files it read — +own module, markdown/data, component modules in its tree, and the state files it +uses; see ``page_cache.page_dependency_hashes``), its rendered ``output_code`` +and path, its ``frontend_imports`` (``ImportVar`` is a frozen dataclass of +primitives), its app-wrap key-set, and whether evaluating it registered new +state. App-wide: the genuinely-global ``epoch`` (reflex version + +config/lockfiles), the reflex version, and the merged imports. The manifest +deliberately stores **no rendered memo files**: a hit page's memo files are +already on disk from the prior compile, and a miss page re-renders its own on +recompile — so writing the manifest is just string/key serialization, never a +second memo render. + +**The fast path** (``try_incremental_rebuild``). On a fresh compile it reuses the +manifest when the global inputs match (reflex version, route set, and the global +epoch). Each page is then a hit iff **every file in its recorded dependency set +is byte-unchanged** — so editing one markdown doc or one shared view recompiles +exactly the pages that depend on it, not all of them. The on-disk app-wide files +(app root, contexts, theme, stylesheet, …) stay valid because the epoch and route +set are unchanged. Then: + +- A *stateless* hit page is skipped entirely (its frontend file is reused and + evaluating it would register nothing). +- A *stateful* hit page is re-evaluated for the backend only (to re-register its + state classes), reusing its frontend file. ``is_stateful`` is true exactly + when the page's first eval grew the state registry, so this is precisely the + set whose state would otherwise go missing. +- A *miss* page (source changed) is fully recompiled and its files rewritten. + +After recompiling misses, two guards must hold or the whole thing falls back to a +full compile (return False): each miss page's app-wrap key-set and stateful flag +must be unchanged (otherwise the reused on-disk app root would be wrong). Any +state edit, shared-file edit, route add/remove, or version change also falls back +to a full compile. ``REFLEX_COMPILE_CACHE_VERIFY`` is the backstop for an app. +""" + +from __future__ import annotations + +import dataclasses +import json +from typing import TYPE_CHECKING, Any + +from reflex_base.plugins import CompileContext, CompilerHooks +from reflex_base.utils.imports import ImportVar, merge_imports + +from reflex.compiler import page_cache +from reflex.compiler.plugins import default_page_plugins +from reflex.utils import console, prerequisites + +if TYPE_CHECKING: + from collections.abc import Callable, Sequence + from pathlib import Path + + from reflex_base.plugins import PageDefinition + from reflex_base.utils.imports import ParsedImportDict + + from reflex.app import App + +#: Bump when the manifest layout changes (old manifests are then ignored). +_SCHEMA = 2 +#: Manifest filename under the web directory. +_MANIFEST_FILE = "reflex_compile_cache.json" + + +def _manifest_path() -> Path: + return prerequisites.get_web_dir() / _MANIFEST_FILE + + +def _serialize_imports(imports: ParsedImportDict) -> dict[str, list[dict[str, Any]]]: + """Serialize a parsed import dict to JSON-able primitives. + + Args: + imports: The parsed import dict to serialize. + + Returns: + A JSON-serializable representation. + """ + return {lib: [dataclasses.asdict(iv) for iv in ivs] for lib, ivs in imports.items()} + + +def _deserialize_imports(data: dict[str, list[dict[str, Any]]]) -> ParsedImportDict: + """Rebuild a parsed import dict from its serialized form. + + Args: + data: The serialized import dict. + + Returns: + The reconstructed parsed import dict. + """ + return {lib: [ImportVar(**iv) for iv in ivs] for lib, ivs in data.items()} + + +def _wrap_key_strs(keys: Any) -> list[str]: + """Render app-wrap ``(priority, name)`` keys as sorted stable strings. + + Args: + keys: An iterable of ``(priority, name)`` app-wrap keys. + + Returns: + A sorted list of ``"priority:name"`` strings. + """ + return sorted(f"{p}:{n}" for p, n in keys) + + +def load_manifest() -> dict[str, Any] | None: + """Load the persisted compile manifest, or None if absent/unusable. + + Returns: + The parsed manifest dict, or None. + """ + path = _manifest_path() + if not path.exists(): + return None + try: + data = json.loads(path.read_text(encoding="utf-8")) + except (OSError, ValueError): + return None + if not isinstance(data, dict) or data.get("schema") != _SCHEMA: + return None + return data + + +def write_manifest( + compile_ctx: CompileContext, + pages: Sequence[PageDefinition], + install_imports: ParsedImportDict, + root: Path | None = None, +) -> None: + """Persist a manifest of the just-completed full compile. + + Best-effort: any failure leaves no manifest (the next compile is full), it + never breaks the build. + + Args: + compile_ctx: The completed compile context (all pages compiled). + pages: The full list of page definitions that were compiled. + install_imports: The **complete** frontend import set the full compile + installed — page imports merged with the app-root (app-wrap, e.g. + the Toaster/``sonner`` provider) and memo-component imports. An + incremental rebuild reuses the on-disk app-wide files, so it must + install from this complete set, not just the per-page union. + root: Project root for fingerprinting. Defaults to cwd. + """ + try: + state_index, _ = page_cache.state_dependency_index(root) + hasher = page_cache.make_hasher() + epoch = page_cache.global_epoch(root) + + pages_data: dict[str, Any] = {} + for page in pages: + page_ctx = compile_ctx.compiled_pages.get(page.route) + if ( + page_ctx is None + or page_ctx.output_code is None + or page_ctx.output_path is None + ): + return # incomplete compile -> do not write a partial manifest + pages_data[page.route] = { + "dep_hashes": page_cache.page_dependency_hashes( + page_ctx, page.component, state_index, hasher, root + ), + "output_path": page_ctx.output_path, + "output_code": page_ctx.output_code, + "frontend_imports": _serialize_imports(page_ctx.frontend_imports), + "app_wrap_keys": _wrap_key_strs(page_ctx.app_wrap_components.keys()), + "is_stateful": page.route in compile_ctx.stateful_routes, + } + + manifest = { + "schema": _SCHEMA, + "reflex_version": page_cache._reflex_version(), + "epoch": epoch, + "all_imports": _serialize_imports(install_imports), + "pages": pages_data, + } + path = _manifest_path() + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(manifest), encoding="utf-8") + except Exception as exc: # best-effort: never break the build + console.debug(f"disk compile cache: manifest write skipped ({exc!r})") + + +def globals_match( + manifest: dict[str, Any], + *, + routes: set[str], + epoch: str, +) -> bool: + """Whether the manifest's genuinely-global inputs match the current compile. + + The fast rebuild needs the route set unchanged (adding/removing a route + changes the shared nav on every page) and the global epoch unchanged (Reflex + version + config/lockfiles). Everything else is decided per page via its + dependency set, so a shared-component or markdown edit no longer blocks the + fast path — only the pages that actually depend on the changed file miss. + + Args: + manifest: The loaded manifest. + routes: The current set of page routes. + epoch: The current global epoch (see :func:`page_cache.global_epoch`). + + Returns: + True iff the global inputs match. + """ + return ( + manifest.get("reflex_version") == page_cache._reflex_version() + and set(manifest.get("pages", {})) == routes + and manifest.get("epoch") == epoch + ) + + +def partition_pages( + pages: Sequence[PageDefinition], + manifest: dict[str, Any], + hasher: Callable[[str], str | None], +) -> list[PageDefinition]: + """Return the pages whose dependency set changed since the manifest. + + Globals are assumed already matched (see :func:`globals_match`), so a page is + a hit iff every file in its recorded dependency set is byte-unchanged. + + Args: + pages: The current page definitions. + manifest: The loaded manifest. + hasher: A memoized path -> content-hash function. + + Returns: + The list of miss pages (a dependency changed) to recompile. + """ + manifest_pages = manifest["pages"] + return [ + page + for page in pages + if not page_cache.deps_unchanged( + manifest_pages[page.route]["dep_hashes"], hasher + ) + ] + + +def try_incremental_rebuild( + app: App, + *, + compiler_plugins: Any, + prerender_routes: bool, + root: Path | None = None, + use_rich: bool = True, +) -> bool: + """Attempt a disk-cache-assisted partial rebuild; report whether it ran. + + Returns False (so the caller does a full compile) whenever anything is + unsafe to reuse: no/old manifest, a changed global input, a route change, or + a miss page that altered its app-wrap set or stateful flag. + + On success, reports (at info level) how many pages were recompiled vs reused + and, while recompiling, shows a progress bar over the changed pages so a hot + reload makes the incremental work visible. + + Args: + app: The app being compiled. + compiler_plugins: The resolved compiler plugins for this compile. + prerender_routes: Whether to prerender routes. + root: Project root for fingerprinting. Defaults to cwd. + use_rich: Whether to use a rich progress bar (else a plain fallback). + + Returns: + True if the partial rebuild completed (the caller should return), else + False (the caller should run a full compile). + """ + manifest = load_manifest() + if manifest is None: + return False + + pages = list(app._unevaluated_pages.values()) + routes = {p.route for p in pages} + hasher = page_cache.make_hasher() + epoch = page_cache.global_epoch(root) + + if not globals_match(manifest, routes=routes, epoch=epoch): + return False + + miss_pages = partition_pages(pages, manifest, hasher) + miss_routes = {p.route for p in miss_pages} + + # Recompile only the source-changed pages. + miss_ctx = None + if miss_pages: + from reflex_base.components.dynamic import ( + bundle_library, + reset_bundled_libraries, + ) + from reflex_base.components.memo import reset_memo_component_classes + + from reflex.compiler.compiler import make_compile_progress + + # Match the full compile's clean bundling/memo state before compiling. + reset_bundled_libraries() + reset_memo_component_classes() + for plugin in compiler_plugins: + for dependency in plugin.get_frontend_dependencies(): + bundle_library(dependency) + miss_ctx = CompileContext( + app=app, + pages=miss_pages, + hooks=CompilerHooks( + plugins=default_page_plugins(style=app.style, plugins=compiler_plugins) + ), + ) + # Progress over the changed pages (evaluate + render each), so a hot + # reload shows how much is being recompiled. + progress = make_compile_progress(use_rich) + progress.start() + task = progress.add_task( + "Recompiling changed pages:", total=len(miss_pages) * 2 + ) + try: + with miss_ctx: + miss_ctx.compile( + evaluate_progress=lambda: progress.advance(task), + render_progress=lambda: progress.advance(task), + ) + finally: + progress.stop() + # Guard: a miss must not change the app-wrap set or its stateful flag, or + # the reused on-disk app root / state marker would be wrong. + for page in miss_pages: + page_ctx = miss_ctx.compiled_pages.get(page.route) + if ( + page_ctx is None + or page_ctx.output_code is None + or page_ctx.output_path is None + ): + return False + entry = manifest["pages"][page.route] + if ( + _wrap_key_strs(page_ctx.app_wrap_components.keys()) + != entry["app_wrap_keys"] + ): + return False + if (page.route in miss_ctx.stateful_routes) != entry["is_stateful"]: + return False + + from reflex.compiler import compiler + + # Write changed pages + their memo files; reuse everything else on disk. + install_imports = _deserialize_imports(manifest["all_imports"]) + if miss_ctx is not None: + for page in miss_pages: + page_ctx = miss_ctx.compiled_pages[page.route] + # Both are guaranteed non-None by the guard loop above. + output_path = page_ctx.output_path + output_code = page_ctx.output_code + if output_path is None or output_code is None: + return False + compiler.utils.write_file( + compiler.utils.resolve_path_of_web_dir(output_path), + output_code, + ) + memo_defs = list(page_ctx.memo_contributions.values()) + memo_files, memo_imports = compiler.compile_memo_components(memo_defs) + for mpath, mcode in memo_files: + compiler.utils.write_file( + compiler.utils.resolve_path_of_web_dir(mpath), mcode + ) + install_imports = merge_imports( + install_imports, page_ctx.frontend_imports, memo_imports + ) + + # Re-register state for stateful hit pages (skipping eval would drop their + # state classes); stateless hits need nothing. + stateful_routes: dict[str, None] = {} + with console.timing("Evaluate Pages (Backend)"): + for page in pages: + if page.route in miss_routes: + if miss_ctx is not None and page.route in miss_ctx.stateful_routes: + stateful_routes[page.route] = None + continue + if manifest["pages"][page.route]["is_stateful"]: + app._compile_page(page.route, save_page=False) + stateful_routes[page.route] = None + + app._stateful_pages.update(stateful_routes) + app._write_stateful_pages_marker() + app._add_optional_endpoints() + app._validate_var_dependencies() + + # Frontend packages + routing scaffolding (cheap, idempotent). + from reflex.utils import frontend_skeleton + + with console.timing("Install Frontend Packages"): + app._get_frontend_packages(install_imports) + frontend_skeleton.update_react_router_config(prerender_routes=prerender_routes) + frontend_skeleton.update_entry_client() + + # Refresh the manifest for the next process. + _update_manifest_for_misses(manifest, miss_ctx, miss_pages) + + if miss_pages: + changed = ", ".join(sorted(p.route for p in miss_pages)[:8]) + if len(miss_pages) > 8: + changed += ", ..." + console.info( + f"Incremental compile: recompiled {len(miss_pages)} page(s) ({changed})." + ) + else: + console.info("Incremental compile: no page changed.") + return True + + +def _update_manifest_for_misses( + manifest: dict[str, Any], + miss_ctx: CompileContext | None, + miss_pages: Sequence[PageDefinition], +) -> None: + """Update the on-disk manifest entries for the recompiled pages. + + Args: + manifest: The loaded manifest (mutated and rewritten). + miss_ctx: The compile context of the recompiled pages, if any. + miss_pages: The recompiled page definitions. + """ + if miss_ctx is None or not miss_pages: + return + try: + state_index, _ = page_cache.state_dependency_index() + hasher = page_cache.make_hasher() + all_imports = _deserialize_imports(manifest["all_imports"]) + for page in miss_pages: + page_ctx = miss_ctx.compiled_pages[page.route] + manifest["pages"][page.route] = { + "dep_hashes": page_cache.page_dependency_hashes( + page_ctx, page.component, state_index, hasher + ), + "output_path": page_ctx.output_path, + "output_code": page_ctx.output_code, + "frontend_imports": _serialize_imports(page_ctx.frontend_imports), + "app_wrap_keys": _wrap_key_strs(page_ctx.app_wrap_components.keys()), + "is_stateful": page.route in miss_ctx.stateful_routes, + } + all_imports = merge_imports(all_imports, page_ctx.frontend_imports) + manifest["all_imports"] = _serialize_imports(all_imports) + _manifest_path().write_text(json.dumps(manifest), encoding="utf-8") + except Exception as exc: # best-effort + console.debug(f"disk compile cache: manifest refresh skipped ({exc!r})") diff --git a/reflex/compiler/page_cache.py b/reflex/compiler/page_cache.py new file mode 100644 index 00000000000..97116337a7a --- /dev/null +++ b/reflex/compiler/page_cache.py @@ -0,0 +1,726 @@ +"""Experimental incremental compile cache (flag-gated). + +Enabled by ``REFLEX_COMPILE_CACHE``. When off (the default), the compiler +behaves exactly as before. + +**In-process per-page cache, with a Salsa-style dependency graph.** Each page +records the *exact set of source files it actually read*, so a change +invalidates only the pages that depend on it (not all pages). A page's +dependency set is the union of: + +- the **transitive first-party ``.py`` import closure** of its defining module + (``page_py_dependencies`` — captures function-based views and shared helpers + that never appear as nodes in the rendered tree, e.g. a ``def hero()`` view, so + editing one invalidates exactly the pages whose closure imports it), +- the **source files read while evaluating it** (markdown/data — captured by + ``record_reads`` via the per-page read recorder; this is what lets editing one + ``.md`` doc page recompile only that page), +- the **component modules in its rendered tree** (``component_module_files`` — + belt-and-suspenders for components injected at runtime rather than statically + imported), +- the **fine-grained state files** it references (``used_state_files``). + +A page is reused iff every file in its dependency set is byte-unchanged and the +small genuinely-global ``global_epoch`` (Reflex version + ``rxconfig`` + lockfile) +is unchanged; adding/removing a route is handled separately (it changes shared +nav). Per-page dependency sets also track files *outside* the project root (e.g. +the docs site reads markdown from a sibling directory), so an external-source +edit still invalidates exactly the dependent pages. Cached pages' contributions +are re-merged into the app-wide aggregates by ``compile_app``; the store holds +live objects (``PageContext``), so it is in-process only. The cross-process +counterpart that survives a hot-reload worker respawn lives in +``reflex/compiler/disk_cache.py``. ``REFLEX_COMPILE_CACHE_VERIFY`` proves the +cache-assisted output matches a full compile and falls back on any mismatch — +the backstop for the residual gaps the static graph cannot see: runtime +``importlib`` imports and data files read at *module-import* time (outside the +per-page eval window). +""" + +from __future__ import annotations + +import builtins +import contextlib +import hashlib +import re +from collections.abc import Callable +from contextvars import ContextVar +from importlib import metadata +from pathlib import Path +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from reflex_base.components.component import BaseComponent + from reflex_base.plugins import PageContext + +#: Directories never worth hashing (build artifacts, deps, caches). +_SKIP_DIRS = {".web", ".venv", "venv", "node_modules", "__pycache__", ".git", "assets"} + +#: Genuinely-global files: a change here can affect every page's output, so it +#: bumps ``global_epoch`` rather than any single page's dependency set. +_GLOBAL_FILES = ("rxconfig.py", "reflex.lock", "uv.lock", "package.json") + +#: Matches a JS state-context identifier in compiled page output. +STATE_CONTEXT_RE = re.compile(r"reflex___state[A-Za-z0-9_]+") + + +def _sha(*parts: bytes | str) -> str: + h = hashlib.sha256() + for p in parts: + h.update(p.encode() if isinstance(p, str) else p) + h.update(b"\x1f") + return h.hexdigest() + + +def _reflex_version() -> str: + try: + return metadata.version("reflex") + except Exception: + return "unknown" + + +# --- Per-page read tracking (the Salsa "input read" seam) ------------------- +# A page's markdown/data dependencies are read lazily while the page is +# evaluated (e.g. ``Path(doc).read_text()`` inside the page callable). We record +# those reads per page so the cache depends on the exact files consumed. Patches +# are installed once (idempotent) and only record while a recording set is active +# on the current task (set by ``record_reads``), so the overhead is a contextvar +# read when no cache is running. + +_active_reads: ContextVar[set[str] | None] = ContextVar("_active_reads", default=None) +_patched = False +_recorder_root: Path | None = None + +#: Path parts that mark a dependency/build location whose reads are never a +#: page's own source dependency (a change there flows through the version/epoch). +_EXCLUDE_PARTS = _SKIP_DIRS | {"site-packages", "dist-packages"} + +#: Suffixes of content files a page may read from *outside* the project root +#: (e.g. the docs app reads its markdown from a sibling directory). Reads under +#: the project root are tracked regardless of suffix; reads elsewhere only if +#: they look like content, so stdlib/site reads of other types are ignored. +_CONTENT_SUFFIXES = { + ".md", + ".mdx", + ".py", + ".json", + ".yaml", + ".yml", + ".toml", + ".txt", + ".csv", + ".html", + ".rst", +} + + +def _record_read(path: object) -> None: + target = _active_reads.get() + if target is None: + return + try: + resolved = Path(path).resolve() # type: ignore[arg-type] + except (OSError, TypeError, ValueError): + return + if any(part in _EXCLUDE_PARTS for part in resolved.parts): + return + root = _recorder_root + under_root = root is not None and root in resolved.parents + if not under_root and resolved.suffix.lower() not in _CONTENT_SUFFIXES: + return + target.add(str(resolved)) + + +def enable_read_tracking(root: Path | None = None) -> None: + """Install per-page source-read tracking and register the recorder hook. + + Idempotent. Patches ``Path.read_text``/``read_bytes`` and ``open`` to record + reads (only while a recording set is active) and points the reflex-base + compile loop at :func:`record_reads`. Called when an incremental cache flag + is on; a no-op to the compile otherwise. + + Args: + root: Project root; only reads under it are recorded. Defaults to cwd. + """ + global _patched, _recorder_root + _recorder_root = (root or Path.cwd()).resolve() + + from reflex_base.plugins import compiler as _bc + + _bc.page_source_recorder = record_reads + + if _patched: + return + _patched = True + + orig_read_text = Path.read_text + orig_read_bytes = Path.read_bytes + orig_open = builtins.open + + def read_text(self: Path, *args: object, **kwargs: object): + _record_read(self) + return orig_read_text(self, *args, **kwargs) # type: ignore[arg-type] + + def read_bytes(self: Path): + _record_read(self) + return orig_read_bytes(self) + + def open_(file: object, mode: str = "r", *args: object, **kwargs: object): + if "r" in mode and not ("w" in mode or "a" in mode or "x" in mode): + _record_read(file) + return orig_open(file, mode, *args, **kwargs) # type: ignore[arg-type] + + Path.read_text = read_text # type: ignore[method-assign,assignment] + Path.read_bytes = read_bytes # type: ignore[method-assign,assignment] + builtins.open = open_ # type: ignore[assignment] + + +@contextlib.contextmanager +def record_reads(): + """Record source-file reads on the current task into a fresh set. + + Yields: + The set of resolved source-file path strings read within the block. + """ + reads: set[str] = set() + token = _active_reads.set(reads) + try: + yield reads + finally: + _active_reads.reset(token) + + +def global_epoch(root: Path | None = None) -> str: + """Fingerprint the genuinely-global inputs (Reflex version + config/lockfiles). + + These can affect every page's output but belong to no single page, so they + gate the whole cache rather than any one page's dependency set. Kept small on + purpose — per-file edits flow through per-page dependency sets instead. + + Args: + root: Project root. Defaults to cwd. + + Returns: + A hex digest of the global inputs. + """ + root = (root or Path.cwd()).resolve() + parts: list[str] = [f"reflex={_reflex_version()}"] + for name in _GLOBAL_FILES: + path = root / name + try: + parts.append(f"{name}={hashlib.sha256(path.read_bytes()).hexdigest()}") + except OSError: + parts.append(f"{name}=") + return _sha(*parts) + + +# In-process per-page cache. Each page is keyed by the genuinely-global epoch +# plus the content hashes of its exact dependency set, so editing one file +# misses only the pages that depend on it. Contributions to the app-wide +# aggregates include live Python objects (root_component, memo defs), so the +# store is in-process only. + + +def _module_file(component: object) -> Path | None: + import sys + + mod = sys.modules.get(getattr(component, "__module__", "") or "") + file = getattr(mod, "__file__", None) + return Path(file) if file else None + + +def page_module_files(components: object) -> set[Path]: + """Resolve the set of module files that define the given page components. + + Args: + components: An iterable of page component callables/objects. + + Returns: + The set of resolved module file paths. + """ + files = set() + for comp in components: # type: ignore[attr-defined] + f = _module_file(comp) + if f is not None: + files.add(f.resolve()) + return files + + +def component_module_files( + root_component: object, root: Path | None = None +) -> set[Path]: + """Resolve the first-party module files of every component in a tree. + + Walks the rendered component tree and collects the defining module file of + each component class under ``root``. This is the precise, barrel-immune way + (vs. static imports) to capture which shared views/templates a page renders: + editing one invalidates exactly the pages whose tree contains it. + + Args: + root_component: The page's root component (its rendered tree). + root: Project root; only files under it are returned. Defaults to cwd. + + Returns: + The set of resolved first-party module files the tree depends on. + """ + root = (root or Path.cwd()).resolve() + files: set[Path] = set() + seen: set[int] = set() + stack: list[object] = [root_component] + while stack: + comp = stack.pop() + if id(comp) in seen: + continue + seen.add(id(comp)) + f = _module_file(type(comp)) + if f is not None: + rf = f.resolve() + if root in rf.parents: + files.add(rf) + children = getattr(comp, "children", None) + if children: + stack.extend(children) + return files + + +#: Cache of the first-party import graph, keyed by project root. +_import_graph_cache: dict[Path, dict[str, set[str]]] = {} + + +def _resolve_module_file(name: str) -> str | None: + import sys + + mod = sys.modules.get(name) + file = getattr(mod, "__file__", None) + return str(Path(file).resolve()) if file else None + + +def _import_from_targets(node: object, modname: str) -> list[str]: + """Resolve a ``from ... import ...`` node to candidate module names. + + Handles relative imports via the importing module's package. Returns the + base module and each ``base.name`` (a name may be a submodule or an attribute + — both candidates are resolved against ``sys.modules`` by the caller). + + Args: + node: An ``ast.ImportFrom`` node. + modname: The dotted name of the module containing the import. + + Returns: + Candidate dotted module names to resolve. + """ + import ast + + if not isinstance(node, ast.ImportFrom): + return [] + if node.level: # relative import: walk up from the importing package + base_pkg = modname.rsplit(".", node.level)[0] if "." in modname else "" + base = f"{base_pkg}.{node.module}" if node.module else base_pkg + else: + base = node.module or "" + if not base: + return [] + return [base, *(f"{base}.{a.name}" for a in node.names)] + + +def build_import_graph(root: Path | None = None) -> dict[str, set[str]]: + """Build the first-party import graph (file -> files it imports). + + Parses every already-imported first-party module's source for ``import`` and + ``from`` statements and resolves them to files under ``root`` via + ``sys.modules``. Cached per root for the duration of the process. This is the + sound basis for per-page ``.py`` dependencies: a function (e.g. a view like + ``hero()``) can only affect a page if its module is transitively imported by + the page's module, so it appears in the page's import closure even though it + is never a node in the rendered tree. + + Args: + root: Project root. Defaults to cwd. + + Returns: + A mapping of resolved file path -> the set of first-party files it imports. + """ + import ast + import sys + + root = (root or Path.cwd()).resolve() + cached = _import_graph_cache.get(root) + if cached is not None: + return cached + + file_to_mod: dict[str, str] = {} + for name, mod in list(sys.modules.items()): + file = getattr(mod, "__file__", None) + if not file: + continue + rf = Path(file).resolve() + if root in rf.parents: + file_to_mod[str(rf)] = name + + graph: dict[str, set[str]] = {} + for file, modname in file_to_mod.items(): + deps: set[str] = set() + try: + tree = ast.parse(Path(file).read_bytes()) + except (OSError, SyntaxError, ValueError): + graph[file] = deps + continue + for node in ast.walk(tree): + names: list[str] = [] + if isinstance(node, ast.Import): + names = [a.name for a in node.names] + elif isinstance(node, ast.ImportFrom): + names = _import_from_targets(node, modname) + for n in names: + target = _resolve_module_file(n) + if target is not None and target in file_to_mod: + deps.add(target) + graph[file] = deps + _import_graph_cache[root] = graph + return graph + + +def clear_import_graph() -> None: + """Drop the cached import graph (e.g. after modules are reloaded).""" + _import_graph_cache.clear() + + +def page_py_dependencies( + component: BaseComponent | object, root: Path | None = None +) -> set[str]: + """Return the transitive first-party ``.py`` files a page's code depends on. + + Starts from the page callable's *real* defining file (``__code__`` filename, + which is correct even when ``__module__`` was reassigned, as the docs app does + for generated doc pages) plus its module file, and walks the import graph. + Captures function-based views and shared helpers that the rendered-tree walk + cannot see. + + Args: + component: The page component or callable. + root: Project root. Defaults to cwd. + + Returns: + The set of resolved first-party ``.py`` dependency file paths. + """ + root = (root or Path.cwd()).resolve() + graph = build_import_graph(root) + + start: set[str] = set() + code = getattr(component, "__code__", None) + filename = getattr(code, "co_filename", None) + if filename: + rf = Path(filename).resolve() + if root in rf.parents: + start.add(str(rf)) + own = _module_file(component) + if own is not None: + rf = own.resolve() + if root in rf.parents: + start.add(str(rf)) + + seen: set[str] = set() + stack = list(start) + while stack: + cur = stack.pop() + if cur in seen: + continue + seen.add(cur) + stack.extend(graph.get(cur, ())) + return seen + + +def make_hasher() -> Callable[[str], str | None]: + """Return a content-hasher that memoizes each path within one compile. + + Shared component files appear in many pages' dependency sets; hashing each + file at most once keeps per-page validation cheap. + + Returns: + A function mapping a path string to its content hash (None if unreadable). + """ + cache: dict[str, str | None] = {} + + def hasher(path: str) -> str | None: + if path not in cache: + try: + cache[path] = hashlib.sha256(Path(path).read_bytes()).hexdigest() + except OSError: + cache[path] = None + return cache[path] + + return hasher + + +def _subclasses(root_cls: type) -> list[type]: + seen: set[type] = set() + out: list[type] = [] + stack = [root_cls] + while stack: + cls = stack.pop() + if cls in seen: + continue + seen.add(cls) + out.append(cls) + stack.extend(cls.__subclasses__()) + return out + + +def state_dependency_index( + root: Path | None = None, +) -> tuple[dict[str, Path], set[Path]]: + """Build the state-context -> file index for fine-grained invalidation. + + A page references the JS state-context identifier + ``format_state_name(state.get_full_name())`` in its output; mapping that to + the state's module file lets a state edit invalidate only its dependents. + Only *pure* state modules under ``root`` (no Component defined in them) are + fine-grained; mixed state/component modules stay coarse (in the shared + fingerprint) so a component edit there is never missed. + + Args: + root: Project root. Defaults to cwd. + + Returns: + ``(identifier_to_file, fine_state_files)``. + """ + root = (root or Path.cwd()).resolve() + from reflex_base.components.component import Component + from reflex_base.utils import format as fmt + + from reflex.state import BaseState + + def under_root(comp: object) -> Path | None: + f = _module_file(comp) + if f is None: + return None + rf = f.resolve() + return rf if root in rf.parents else None + + component_files = {rf for cls in _subclasses(Component) if (rf := under_root(cls))} + id_to_file: dict[str, Path] = {} + state_files: set[Path] = set() + for cls in _subclasses(BaseState): + rf = under_root(cls) + if rf is None: + continue + with contextlib.suppress(Exception): + id_to_file[fmt.format_state_name(cls.get_full_name())] = rf + state_files.add(rf) + fine = state_files - component_files + return {i: f for i, f in id_to_file.items() if f in fine}, fine + + +def file_hashes(files: set[Path]) -> dict[str, str]: + """Map each file (as a string) to a hash of its current content. + + Args: + files: The files to hash. + + Returns: + A mapping of file path string to content hash. + """ + out: dict[str, str] = {} + for f in files: + with contextlib.suppress(OSError): + out[str(f)] = hashlib.sha256(f.read_bytes()).hexdigest() + return out + + +def used_state_files( + output_code: str, + memo_components: object, + id_to_file: dict[str, Path], +) -> set[Path]: + """Return the fine-grained state files a compiled page depends on. + + Stateful subtrees are auto-memoized into separate components, so a page's + own ``output_code`` may not name the state it uses — the state lives in the + memo components it *owns* (its ``memo_contributions``). Each stateful memo + is owned by exactly one page, which regenerates it whenever it recompiles, + so scanning ``output_code`` plus the page's own memo components captures the + full dependency set. If a memo can't be introspected, depend on every fine + state file (conservative — never stale). + + Args: + output_code: The page's compiled JS. + memo_components: The page's own memo component subtrees (renderable). + id_to_file: The state-context identifier -> file index. + + Returns: + The set of fine-grained state files this page depends on. + """ + chunks = [output_code or ""] + try: + chunks.extend(str(comp.render()) for comp in memo_components) # type: ignore[attr-defined] + except Exception: + return set(id_to_file.values()) # conservative: depend on all + found: set[Path] = set() + for chunk in chunks: + for ident in STATE_CONTEXT_RE.findall(chunk): + if ident in id_to_file: + found.add(id_to_file[ident]) + return found + + +def page_dependency_files( + page_ctx: PageContext, + component: BaseComponent | object, + state_index: dict[str, Path], + root: Path | None = None, +) -> set[str]: + """Return the full set of source files a compiled page depends on. + + The union of: the page's own defining module, the files read while it was + evaluated (markdown/data, via the read recorder), the component modules in + its rendered tree, and the fine-grained state files it references. A change + to any file *outside* this set cannot change the page's output, except the + genuinely-global inputs tracked by :func:`global_epoch` and route additions + (shared nav). This is the dependency set that makes invalidation precise. + + Args: + page_ctx: The compiled page context (tree, output, recorded reads). + component: The page's component/callable (for its defining module). + state_index: The state-context identifier -> file index. + root: Project root; only files under it are included. Defaults to cwd. + + Returns: + The set of resolved dependency file path strings. + """ + root = (root or Path.cwd()).resolve() + files: set[Path] = set() + files |= component_module_files(page_ctx.root_component, root) + files |= used_state_files( + page_ctx.output_code or "", + [m.component for m in page_ctx.memo_contributions.values()], + state_index, + ) + deps = {str(f) for f in files} + # Transitive first-party .py imports (captures function-based views/helpers + # that never appear as nodes in the rendered tree). + deps |= page_py_dependencies(component, root) + # Files read while evaluating the page (markdown/data). + deps |= set(page_ctx.source_files) + return deps + + +def page_dependency_hashes( + page_ctx: PageContext, + component: BaseComponent | object, + state_index: dict[str, Path], + hasher: Callable[[str], str | None], + root: Path | None = None, +) -> dict[str, str]: + """Hash a page's dependency set into a ``{path: hash}`` map. + + Args: + page_ctx: The compiled page context. + component: The page's component/callable. + state_index: The state-context identifier -> file index. + hasher: A memoized path -> content-hash function (see :func:`make_hasher`). + root: Project root. Defaults to cwd. + + Returns: + ``{path: content_hash}`` for every readable dependency file. + """ + out: dict[str, str] = {} + for path in page_dependency_files(page_ctx, component, state_index, root): + digest = hasher(path) + if digest is not None: + out[path] = digest + return out + + +def deps_unchanged( + dep_hashes: dict[str, str], hasher: Callable[[str], str | None] +) -> bool: + """Whether every file in a stored dependency set still has the same content. + + Args: + dep_hashes: A page's stored ``{path: hash}`` dependency map. + hasher: A memoized path -> content-hash function. + + Returns: + True iff every dependency file is byte-unchanged. + """ + return all(hasher(path) == digest for path, digest in dep_hashes.items()) + + +def page_source_fingerprint(component: BaseComponent | object) -> str: + """Fingerprint a page from its own module source (no build required). + + Args: + component: The page component or factory callable. + + Returns: + A hex digest of the page's defining module source. + """ + import inspect + import sys + + module_name = getattr(component, "__module__", None) + parts = [repr(getattr(component, "__qualname__", repr(component)))] + mod = sys.modules.get(module_name) if module_name else None + file = getattr(mod, "__file__", None) + if file and Path(file).exists(): + with contextlib.suppress(OSError): + parts.append(hashlib.sha256(Path(file).read_bytes()).hexdigest()) + elif callable(component): + with contextlib.suppress(OSError, TypeError): + parts.append(inspect.getsource(component)) + return _sha(*parts) + + +#: route -> (global_epoch, dep_hashes, PageContext, is_stateful) +_PAGE_STORE: dict[str, tuple[str, dict[str, str], PageContext, bool]] = {} + + +def validate_page( + route: str, + epoch: str, + hasher: Callable[[str], str | None], +) -> tuple[PageContext, bool] | None: + """Return the cached page iff its dependency set and the global epoch match. + + A page is valid when the genuinely-global epoch is unchanged and every file + in its recorded dependency set is byte-unchanged. Editing a file invalidates + exactly the pages whose dependency set contains it. + + Args: + route: The page route. + epoch: The current global epoch (see :func:`global_epoch`). + hasher: A memoized path -> content-hash function (see :func:`make_hasher`). + + Returns: + ``(PageContext, is_stateful)`` on a valid hit, else None. + """ + entry = _PAGE_STORE.get(route) + if entry is None: + return None + stored_epoch, dep_hashes, page_ctx, is_stateful = entry + if stored_epoch != epoch or not deps_unchanged(dep_hashes, hasher): + return None + return page_ctx, is_stateful + + +def store_page( + route: str, + epoch: str, + dep_hashes: dict[str, str], + page_ctx: PageContext, + is_stateful: bool, +) -> None: + """Record a freshly-compiled page with the dependency set it relies on. + + Args: + route: The page route. + epoch: The global epoch it compiled under. + dep_hashes: ``{path: hash}`` for the page's dependency set. + page_ctx: The compiled PageContext to cache. + is_stateful: Whether the page registered new state during evaluation. + """ + _PAGE_STORE[route] = (epoch, dep_hashes, page_ctx, is_stateful) + + +def clear_page_store() -> None: + """Drop all in-process per-page cache entries.""" + _PAGE_STORE.clear() diff --git a/reflex/compiler/plugins/builtin.py b/reflex/compiler/plugins/builtin.py index 4081b4b9d79..e7c18a97301 100644 --- a/reflex/compiler/plugins/builtin.py +++ b/reflex/compiler/plugins/builtin.py @@ -195,7 +195,7 @@ def eval_page( if (description := getattr(page, "description", None)) is not None: meta_args["description"] = description - utils.add_meta(component, **meta_args) + component = utils.add_meta(component, **meta_args) except Exception as err: if hasattr(err, "add_note"): err.add_note(f"Happened while evaluating page {page.route!r}") diff --git a/reflex/compiler/utils.py b/reflex/compiler/utils.py index fb537e4d176..6a58d5a917f 100644 --- a/reflex/compiler/utils.py +++ b/reflex/compiler/utils.py @@ -801,10 +801,14 @@ def add_meta( children.append(Description.create(content=description)) children.append(Image.create(content=image)) - page.children.extend(children) - page.children.extend(meta_tags) - - return page + # Own-before-mutate: the page root may be a shared instance (the construction + # cache reuses identical static subtrees across pages), so build a fresh copy + # with the metadata appended instead of mutating ``children`` in place — + # otherwise repeated reuse accumulates duplicate /<meta> tags. + new_page = copy.copy(page) + new_page.children = [*page.children, *children, *meta_tags] + new_page._clear_compile_caches() + return new_page def resolve_path_of_web_dir(path: str | Path) -> Path: diff --git a/tests/units/compiler/test_disk_cache.py b/tests/units/compiler/test_disk_cache.py new file mode 100644 index 00000000000..62b3330a38d --- /dev/null +++ b/tests/units/compiler/test_disk_cache.py @@ -0,0 +1,287 @@ +"""Tests for the experimental disk-persisted incremental compile cache.""" + +import dataclasses +from collections.abc import Callable +from typing import Any + +from reflex_base.components.component import Component +from reflex_base.plugins import CompileContext, CompilerHooks +from reflex_base.utils.imports import ImportVar + +import reflex as rx +from reflex.compiler import disk_cache, page_cache +from reflex.compiler.plugins import default_page_plugins + + +@dataclasses.dataclass(slots=True) +class _FakePage: + route: str + component: Callable[[], Component] + title: Any = None + description: Any = None + image: str = "" + meta: tuple[dict[str, Any], ...] = () + _source_module: str | None = None + + +def _footer() -> Component: + return rx.el.footer(rx.el.span("© Reflex"), class_name="footer") + + +def _page_a() -> Component: + return rx.el.div(rx.el.h1("Page A"), _footer()) + + +def _page_b() -> Component: + return rx.el.div(rx.el.h1("Page B"), _footer()) + + +def _page_b_edited() -> Component: + return rx.el.div(rx.el.h1("Page B (edited)"), rx.el.p("new body"), _footer()) + + +def _page_c() -> Component: + return rx.el.div(rx.el.h1("Page C"), _footer()) + + +def _compile(pages: list[_FakePage]) -> CompileContext: + ctx = CompileContext( + pages=pages, + hooks=CompilerHooks(plugins=default_page_plugins()), + ) + with ctx: + ctx.compile() + return ctx + + +def test_imports_round_trip(): + imports = { + "react": [ImportVar("useEffect"), ImportVar("Fragment", is_default=False)], + "@emotion/react": [ImportVar("jsx", alias="j", install=False)], + } + restored = disk_cache._deserialize_imports(disk_cache._serialize_imports(imports)) + assert restored == imports + + +def test_wrap_key_strs_is_sorted_and_stable(): + keys = [(200, "StrictMode"), (0, "AppWrap"), (45, "ColorMode")] + assert disk_cache._wrap_key_strs(keys) == [ + "0:AppWrap", + "200:StrictMode", + "45:ColorMode", + ] + + +def _manifest(pages: dict[str, dict], **overrides) -> dict: + base = { + "schema": disk_cache._SCHEMA, + "reflex_version": page_cache._reflex_version(), + "epoch": "EPOCH", + "all_imports": {}, + "pages": pages, + } + base.update(overrides) + return base + + +def test_globals_match(): + m = _manifest({"/a": {}, "/b": {}}) + routes = {"/a", "/b"} + assert disk_cache.globals_match(m, routes=routes, epoch="EPOCH") + # a changed route set -> no match + assert not disk_cache.globals_match(m, routes={"/a"}, epoch="EPOCH") + # a changed global epoch -> no match + assert not disk_cache.globals_match(m, routes=routes, epoch="OTHER") + # a stale reflex version -> no match + assert not disk_cache.globals_match( + {**m, "reflex_version": "0.0.0-old"}, routes=routes, epoch="EPOCH" + ) + + +def test_partition_pages_detects_changed_source(): + pages = [ + _FakePage(route="/a", component=_page_a), + _FakePage(route="/b", component=_page_b), + ] + # /a depends on x.py, /b depends on y.py (each at a recorded content hash). + m = _manifest({ + "/a": {"dep_hashes": {"/proj/x.py": "h-x"}}, + "/b": {"dep_hashes": {"/proj/y.py": "h-y"}}, + }) + # The hasher reports /a's dep unchanged and /b's dep changed. + current = {"/proj/x.py": "h-x", "/proj/y.py": "h-y-new"} + miss = disk_cache.partition_pages(pages, m, lambda p: current.get(p)) + assert {p.route for p in miss} == {"/b"} + + +def test_write_and_load_manifest(tmp_path, monkeypatch): + web = tmp_path / ".web" + web.mkdir() + monkeypatch.setattr(disk_cache.prerequisites, "get_web_dir", lambda: web) + + pages = [ + _FakePage(route="/a", component=_page_a), + _FakePage(route="/b", component=_page_b), + _FakePage(route="/c", component=_page_c), + ] + ctx = _compile(pages) + disk_cache.write_manifest(ctx, pages, ctx.all_imports, root=tmp_path) + + manifest = disk_cache.load_manifest() + assert manifest is not None + assert manifest["schema"] == disk_cache._SCHEMA + assert set(manifest["pages"]) == {"/a", "/b", "/c"} + # the cached output is exactly what the compile produced for each page + for route in ("/a", "/b", "/c"): + assert ( + manifest["pages"][route]["output_code"] + == ctx.compiled_pages[route].output_code + ) + # these static pages register no new state + assert manifest["pages"][route]["is_stateful"] is False + # imports round-trip cleanly + restored = disk_cache._deserialize_imports( + manifest["pages"]["/a"]["frontend_imports"] + ) + assert restored == ctx.compiled_pages["/a"].frontend_imports + + +def test_unchanged_pages_compile_identically(tmp_path, monkeypatch): + """The reuse correctness property: an unchanged page recompiles byte-for-byte. + + The disk cache reuses a hit page's stored ``output_code`` verbatim, so it is + correct iff a fresh compile of that page yields identical output. Compile A, + B, C; then compile A, B(edited), C; A and C must be byte-identical, and the + manifest's cached A/C output must equal the fresh recompile. + """ + web = tmp_path / ".web" + web.mkdir() + monkeypatch.setattr(disk_cache.prerequisites, "get_web_dir", lambda: web) + + pages = [ + _FakePage(route="/a", component=_page_a), + _FakePage(route="/b", component=_page_b), + _FakePage(route="/c", component=_page_c), + ] + ctx1 = _compile(pages) + disk_cache.write_manifest(ctx1, pages, ctx1.all_imports, root=tmp_path) + manifest = disk_cache.load_manifest() + assert manifest is not None + + # "Edit" page B; recompile the whole app cleanly. + pages_edited = [ + _FakePage(route="/a", component=_page_a), + _FakePage(route="/b", component=_page_b_edited), + _FakePage(route="/c", component=_page_c), + ] + ctx2 = _compile(pages_edited) + + # Unchanged pages are byte-identical across compiles -> safe to reuse. + assert ( + ctx2.compiled_pages["/a"].output_code == ctx1.compiled_pages["/a"].output_code + ) + assert ( + ctx2.compiled_pages["/c"].output_code == ctx1.compiled_pages["/c"].output_code + ) + # B changed. + assert ( + ctx2.compiled_pages["/b"].output_code != ctx1.compiled_pages["/b"].output_code + ) + # The cached output we'd reuse for A/C equals a clean recompile of them. + assert ( + manifest["pages"]["/a"]["output_code"] == ctx2.compiled_pages["/a"].output_code + ) + assert ( + manifest["pages"]["/c"]["output_code"] == ctx2.compiled_pages["/c"].output_code + ) + + +def _stub_externals(app, monkeypatch): + """Stub the side-effecting steps the fast path runs on a real app.""" + import reflex.utils.frontend_skeleton as fs + + monkeypatch.setattr(app, "_get_frontend_packages", lambda *a, **k: None) + monkeypatch.setattr(app, "_add_optional_endpoints", lambda *a, **k: None) + monkeypatch.setattr(app, "_validate_var_dependencies", lambda *a, **k: None) + monkeypatch.setattr(app, "_write_stateful_pages_marker", lambda *a, **k: None) + monkeypatch.setattr(fs, "update_react_router_config", lambda **k: None) + monkeypatch.setattr(fs, "update_entry_client", lambda *a, **k: None) + + +def test_incremental_rebuild_all_hits(tmp_path, monkeypatch): + web = tmp_path / ".web" + web.mkdir() + monkeypatch.setattr(disk_cache.prerequisites, "get_web_dir", lambda: web) + + app = rx.App() + app.add_page(_page_a, route="/a") + app.add_page(_page_c, route="/c") + pages = list(app._unevaluated_pages.values()) + ctx = _compile(pages) + disk_cache.write_manifest(ctx, pages, ctx.all_imports, root=tmp_path) + _stub_externals(app, monkeypatch) + + # Nothing changed -> every page is a hit -> fast path runs. + assert ( + disk_cache.try_incremental_rebuild( + app, compiler_plugins=[], prerender_routes=False, root=tmp_path + ) + is True + ) + + +def test_incremental_rebuild_one_miss_writes_only_that_page(tmp_path, monkeypatch): + import json + + from reflex.compiler import utils as compiler_utils + + web = tmp_path / ".web" + web.mkdir() + monkeypatch.setattr(disk_cache.prerequisites, "get_web_dir", lambda: web) + + app = rx.App() + app.add_page(_page_a, route="/a") + app.add_page(_page_c, route="/c") + pages = list(app._unevaluated_pages.values()) + ctx = _compile(pages) + disk_cache.write_manifest(ctx, pages, ctx.all_imports, root=tmp_path) + _stub_externals(app, monkeypatch) + + # Simulate an edit to the first page: rewrite its manifest dependency set to + # reference a file whose recorded hash no longer matches the current content, + # so partitioning sees its dependency set as changed -> a miss (only it). + edited_route = pages[0].route + manifest_path = web / disk_cache._MANIFEST_FILE + manifest = json.loads(manifest_path.read_text()) + manifest["pages"][edited_route]["dep_hashes"] = { + str(tmp_path / "view.py"): "stale-hash" + } + manifest_path.write_text(json.dumps(manifest)) + + assert ( + disk_cache.try_incremental_rebuild( + app, compiler_plugins=[], prerender_routes=False, root=tmp_path + ) + is True + ) + + # The edited page was recompiled and written; its content matches a clean + # compile of that page. + out_path = compiler_utils.resolve_path_of_web_dir( + ctx.compiled_pages[edited_route].output_path + ) + assert out_path.exists() + assert ( + out_path.read_text(encoding="utf-8") + == ctx.compiled_pages[edited_route].output_code + ) + + +def test_load_manifest_rejects_wrong_schema(tmp_path, monkeypatch): + import json + + web = tmp_path / ".web" + web.mkdir() + monkeypatch.setattr(disk_cache.prerequisites, "get_web_dir", lambda: web) + (web / disk_cache._MANIFEST_FILE).write_text(json.dumps({"schema": 999})) + assert disk_cache.load_manifest() is None diff --git a/tests/units/compiler/test_page_cache.py b/tests/units/compiler/test_page_cache.py new file mode 100644 index 00000000000..0750224390c --- /dev/null +++ b/tests/units/compiler/test_page_cache.py @@ -0,0 +1,115 @@ +"""Tests for the experimental incremental compile cache (in-process page cache).""" + +from reflex.compiler import page_cache + + +def test_global_epoch_tracks_global_files(tmp_path): + (tmp_path / "rxconfig.py").write_text("config = 1\n") + (tmp_path / "other.py").write_text("x = 1\n") + epoch = page_cache.global_epoch(root=tmp_path) + # editing a non-global file does NOT change the epoch + (tmp_path / "other.py").write_text("x = 2\n") + assert page_cache.global_epoch(root=tmp_path) == epoch + # editing a genuinely-global file (rxconfig.py) DOES + (tmp_path / "rxconfig.py").write_text("config = 2\n") + assert page_cache.global_epoch(root=tmp_path) != epoch + + +def _dummy_page(): # a page-like callable defined in this module + return None + + +def test_page_module_files_resolves(tmp_path): + files = page_cache.page_module_files([_dummy_page]) + assert any(p.name == "test_page_cache.py" for p in files) + + +def test_used_state_files_from_output_and_memos(tmp_path): + from types import SimpleNamespace + + sfile = (tmp_path / "state.py").resolve() + mfile = (tmp_path / "mstate.py").resolve() + id_to_file = { + "reflex___state____state____app_____s": sfile, + "reflex___state____state____app_____m": mfile, + } + out = 'jsx("div",{},reflex___state____state____app_____s.x_rx_state_)' + assert page_cache.used_state_files(out, [], id_to_file) == {sfile} + assert page_cache.used_state_files("no state", [], id_to_file) == set() + # state hidden inside an auto-memoized component is still captured + memo = SimpleNamespace( + render=lambda: {"contents": "reflex___state____state____app_____m.y_rx_state_"} + ) + assert page_cache.used_state_files(out, [memo], id_to_file) == {sfile, mfile} + # un-introspectable memo -> conservative (all fine files) + boom = SimpleNamespace(render=lambda: (_ for _ in ()).throw(RuntimeError())) + assert page_cache.used_state_files(out, [boom], id_to_file) == {sfile, mfile} + + +def test_validate_page_fine_grained_deps(): + page_cache.clear_page_store() + ctx = object() + dep = "/proj/state.py" + # page depends on file `dep` at content hash H1, under global epoch "e1" + page_cache.store_page("/x", "e1", {dep: "H1"}, ctx, True) + # all deps unchanged + epoch matches -> hit + assert page_cache.validate_page("/x", "e1", lambda p: {dep: "H1"}.get(p)) == ( + ctx, + True, + ) + # a dependency file changed -> miss + assert page_cache.validate_page("/x", "e1", lambda p: {dep: "H2"}.get(p)) is None + # the genuinely-global epoch changed -> miss + assert page_cache.validate_page("/x", "e2", lambda p: {dep: "H1"}.get(p)) is None + + +def test_validate_page_with_no_deps_only_tracks_epoch(): + page_cache.clear_page_store() + ctx = object() + # page depends on NO files + page_cache.store_page("/x", "e1", {}, ctx, False) + # some unrelated file changed -> page is still a hit (it depends on nothing) + assert page_cache.validate_page("/x", "e1", lambda p: "Z") == (ctx, False) + # the global epoch changed -> miss + assert page_cache.validate_page("/x", "e2", lambda p: "Z") is None + # no stored entry -> miss + page_cache.clear_page_store() + assert page_cache.validate_page("/x", "e1", lambda p: None) is None + + +def _fake_ctx(pages, imports=None, memo=None, stateful=None, wraps=None): + from types import SimpleNamespace + + return SimpleNamespace( + compiled_pages={r: SimpleNamespace(output_code=c) for r, c in pages.items()}, + all_imports=imports or {}, + auto_memo_components=memo or {}, + stateful_routes=stateful or {}, + app_wrap_components=wraps or {}, + ) + + +def test_verify_diff_identical(): + from reflex.compiler import compiler + + a = _fake_ctx({"/": "CODE", "/x": "Y"}) + b = _fake_ctx({"/": "CODE", "/x": "Y"}) + assert compiler._diff_compile_contexts(a, b) == [] + + +def test_verify_diff_detects_page_change(): + from reflex.compiler import compiler + + a = _fake_ctx({"/": "OLD"}) + b = _fake_ctx({"/": "NEW"}) + assert "page:/" in compiler._diff_compile_contexts(a, b) + + +def test_verify_diff_detects_missing_route_and_memo(): + from reflex.compiler import compiler + + a = _fake_ctx({"/": "C"}, memo={("Memo", None): 1}) + b = _fake_ctx({"/": "C", "/x": "C"}, memo={}) + diffs = compiler._diff_compile_contexts(a, b) + assert any(d.startswith("routes:") for d in diffs) + assert "auto_memo_components" in diffs diff --git a/tests/units/vars/test_base.py b/tests/units/vars/test_base.py index a8075357fa7..72d34cc5ae2 100644 --- a/tests/units/vars/test_base.py +++ b/tests/units/vars/test_base.py @@ -53,3 +53,25 @@ def cv(self) -> int: replaced = cv._replace(_var_type=float) assert replaced._var_type is float + + +def test_reset_unique_variable_names_is_deterministic(): + """Resetting the unique-name generator reproduces the same name sequence. + + Auto-memo content hashes embed these names, so a second in-process compile + must regenerate identical ones (the RNG state and used-name set otherwise + persist process-wide and drift). + """ + from reflex_base.vars.base import ( + get_unique_variable_name, + reset_unique_variable_names, + ) + + reset_unique_variable_names() + first = [get_unique_variable_name() for _ in range(8)] + # Without a reset, the sequence keeps advancing (and dedups against the + # already-used set), so it differs. + assert [get_unique_variable_name() for _ in range(8)] != first + # After a reset it reproduces the original sequence exactly. + reset_unique_variable_names() + assert [get_unique_variable_name() for _ in range(8)] == first From 8360450cb01990745d0afe9e2d8bd60e109b5ea8 Mon Sep 17 00:00:00 2001 From: Farhan <www.mfarvirus@gmail.com> Date: Sun, 28 Jun 2026 21:59:43 +0500 Subject: [PATCH 02/56] feat(compiler): warm fork-per-compile hot-reload daemon Add a persistent compile daemon (REFLEX_COMPILE_CACHE) that imports the world once and forks a throwaway child per source change, instead of the reloader respawning a worker that cold-imports on every edit. The child re-imports first-party code fresh and runs the incremental rebuild, so correctness matches a respawn while the cold import is paid once. Supporting changes that make the daemon safe and complete: - write_file now writes atomically (temp + os.replace) so a reader (vite, a concurrent compile) never sees a half-written file, even when a forked child is killed mid-compile. - _run_dev launches the daemon and sets REFLEX_SKIP_COMPILE on the backend so it only evaluates pages to register state. - The daemon watches what the compiler reads (incl. sibling-dir markdown from the manifest); uvicorn reload_includes also covers *.md/*.mdx, so markdown edits finally trigger a reload. - Drop the per-rebuild console.info now that progress is shown inline. --- reflex/compiler/disk_cache.py | 9 - reflex/compiler/utils.py | 12 +- reflex/reflex.py | 16 + reflex/utils/compile_daemon.py | 560 ++++++++++++++++++++ reflex/utils/exec.py | 3 + tests/units/compiler/test_compiler_utils.py | 30 +- tests/units/utils/test_compile_daemon.py | 225 ++++++++ 7 files changed, 844 insertions(+), 11 deletions(-) create mode 100644 reflex/utils/compile_daemon.py create mode 100644 tests/units/utils/test_compile_daemon.py diff --git a/reflex/compiler/disk_cache.py b/reflex/compiler/disk_cache.py index 4c4cbf3bc9d..7d6228fe11c 100644 --- a/reflex/compiler/disk_cache.py +++ b/reflex/compiler/disk_cache.py @@ -404,15 +404,6 @@ def try_incremental_rebuild( # Refresh the manifest for the next process. _update_manifest_for_misses(manifest, miss_ctx, miss_pages) - if miss_pages: - changed = ", ".join(sorted(p.route for p in miss_pages)[:8]) - if len(miss_pages) > 8: - changed += ", ..." - console.info( - f"Incremental compile: recompiled {len(miss_pages)} page(s) ({changed})." - ) - else: - console.info("Incremental compile: no page changed.") return True diff --git a/reflex/compiler/utils.py b/reflex/compiler/utils.py index 6a58d5a917f..9c38dcb1149 100644 --- a/reflex/compiler/utils.py +++ b/reflex/compiler/utils.py @@ -838,7 +838,17 @@ def write_file(path: str | Path, code: str): path.parent.mkdir(parents=True, exist_ok=True) if path.exists() and path.read_text(encoding="utf-8") == code: return - path.write_text(code, encoding="utf-8") + # Write atomically (temp file + os.replace) so a reader watching this tree — + # e.g. the vite dev server, or a concurrent compile — never observes a + # half-written file, even if the writing process is killed mid-write (the + # compile daemon forks throwaway children that may be terminated mid-compile). + tmp = path.with_name(f"{path.name}.{os.getpid()}.tmp") + try: + tmp.write_text(code, encoding="utf-8") + tmp.replace(path) + finally: + if tmp.exists(): + tmp.unlink(missing_ok=True) _MEMO_MANIFEST_FILENAME = ".memo-manifest.json" diff --git a/reflex/reflex.py b/reflex/reflex.py index 3c340fccea2..61cc3c5415e 100644 --- a/reflex/reflex.py +++ b/reflex/reflex.py @@ -207,6 +207,22 @@ def _run_dev( running_mode.has_backend(), )) + # When the compile cache is enabled, a persistent warm compile daemon owns + # .web regeneration (fork-per-compile, no per-edit cold import) instead of + # the backend worker recompiling on every respawn. The backend then only + # evaluates pages to register state — REFLEX_SKIP_COMPILE makes it skip the + # frontend write, and is inherited by every respawned reload worker. + from reflex_base.environment import environment + + if running_mode.has_frontend() and environment.REFLEX_COMPILE_CACHE.get(): + from reflex.utils import compile_daemon + + commands.append(( + compile_daemon.run_compile_daemon, + exec.should_prerender_routes(), + )) + environment.REFLEX_SKIP_COMPILE.set(True) + # Start the frontend and backend. with processes.run_concurrently_context(*commands): # In dev mode, run the backend on the main thread. diff --git a/reflex/utils/compile_daemon.py b/reflex/utils/compile_daemon.py new file mode 100644 index 00000000000..603ebe6d6f0 --- /dev/null +++ b/reflex/utils/compile_daemon.py @@ -0,0 +1,560 @@ +"""Persistent warm compile daemon for fast dev hot reloads (fork-per-compile). + +Active in ``reflex run`` dev when ``REFLEX_COMPILE_CACHE`` is set. Instead of the +granian/uvicorn reloader respawning a worker that cold-imports reflex + +reflex-base + the app's heavy deps (pandas/plotly/…) on every ``.py`` change, +this daemon imports everything **once** and stays warm. On each source change it +``fork()``s a throwaway child that: + +1. purges the user's first-party modules from ``sys.modules`` and resets the + cross-module state/page registries (so it is, for first-party code, a clean + interpreter — no stale ``__subclasses__`` zombies, no duplicate-substate + shadowing, no ``add_page`` re-registration error); +2. re-imports the user app fresh (third-party deps stay warm, inherited via + copy-on-write fork) and runs ``compiler.compile_app`` with the cache on, which + recompiles only the dependency-changed pages via + ``disk_cache.try_incremental_rebuild`` — the child *is* the fresh worker that + path was built for, but warm; +3. writes only the changed ``.web`` files (atomically) and ``os._exit``s. + +Because every compile runs in a child that is discarded, reload corruption can +never accumulate: correctness is the same as today's respawn, but the multi-second +cold import is paid once instead of on every edit. + +The daemon owns the file watcher, so it watches exactly what the compiler reads — +``.py``/``.md``/``.mdx`` under the app source roots **plus** every external/sibling +content file recorded in the compile manifest's per-page dependency sets (where a +docs app's markdown lives) plus ``rxconfig``/lockfiles/``assets`` — fixing the +long-standing gap where markdown edits (and sibling-dir reads) never triggered a +reload because the reloaders watch only ``*.py`` under the app dir. + +The watcher is a single-threaded poll loop on purpose: ``fork()`` from a +multi-threaded process is unsafe, so the daemon must hold no background threads. + +On Windows (no ``os.fork``) each compile runs in a fresh spawned subprocess +instead — correct, but without copy-on-write warmth (parity with today's latency); +the watcher/markdown fix applies identically. +""" + +from __future__ import annotations + +import contextlib +import importlib +import os +import signal +import subprocess +import sys +import threading +import time +from pathlib import Path + +from reflex_base.environment import environment + +from reflex.utils import console + +#: Seconds between poll passes over the watched file set. +_POLL_INTERVAL = 0.25 +#: After a change is seen, wait this long and re-snapshot so a burst of saves +#: (e.g. format-on-save touching many files) collapses into a single compile. +_DEBOUNCE = 0.1 +#: Watchdog: kill a compile child that runs longer than this (a hung/deadlocked +#: child must never wedge the daemon). Generous enough for a real full compile. +_COMPILE_TIMEOUT = 300.0 +#: Source suffixes edited under the app roots that should trigger a recompile. +_WATCH_SUFFIXES = (".py", ".md", ".mdx") +#: Directories never worth walking while building the watch snapshot. +_SKIP_DIRS = {".web", ".venv", "venv", "node_modules", "__pycache__", ".git"} + + +def run_compile_daemon(prerender_routes: bool = False) -> None: + """Supervise the compile daemon as its own (fork-safe) subprocess. + + Runs on a ``reflex run`` worker thread alongside the frontend. Launching the + daemon as a separate process keeps it single-threaded so its per-edit + ``fork()`` is safe, and isolates its environment from the backend (which is + told to skip frontend compilation via ``REFLEX_SKIP_COMPILE``). + + Args: + prerender_routes: Whether the daemon should prerender routes when compiling. + """ + env = {**os.environ} + # The daemon DOES compile; ensure the cache is on and the skip flag is off. + env.pop(environment.REFLEX_SKIP_COMPILE.name, None) + env[environment.REFLEX_COMPILE_CACHE.name] = "1" + if prerender_routes: + env["REFLEX_PRERENDER_ROUTES"] = "1" + proc = subprocess.Popen( + [sys.executable, "-m", "reflex.utils.compile_daemon"], env=env + ) + + def _terminate() -> None: + if proc.poll() is None: + proc.terminate() + try: + proc.wait(timeout=3) + except subprocess.TimeoutExpired: + proc.kill() + + # Backstop: the daemon must never outlive reflex-run, even if this thread is + # interrupted before its finally runs. The daemon also self-exits if its + # parent dies (see _serve), so the two together prevent an orphan. + import atexit + + atexit.register(_terminate) + try: + proc.wait() + finally: + _terminate() + + +def _reload_roots() -> list[Path]: + """Resolve the directories/files that hold the user's first-party source. + + Returns: + The resolved reload roots (the same set the backend reloader watches). + """ + from reflex.utils import exec as exec_utils + + return [Path(p).resolve() for p in exec_utils.get_reload_paths()] + + +def _under_roots(path: Path, roots: list[Path]) -> bool: + """Whether ``path`` is one of, or lives under, the reload roots. + + Args: + path: The resolved path to test. + roots: The resolved reload roots. + + Returns: + True if the path is covered by a reload root. + """ + return any(path == root or root in path.parents for root in roots) + + +def _iter_source_files(root: Path): + """Yield watchable source files under ``root`` (skipping build/dep dirs). + + Args: + root: A reload root directory (or file). + + Yields: + Resolved source file paths with a watched suffix. + """ + if root.is_file(): + if root.suffix in _WATCH_SUFFIXES: + yield root.resolve() + return + for path in root.rglob("*"): + if path.is_dir(): + continue + rel_parts = path.relative_to(root).parts[:-1] + if any(part in _SKIP_DIRS or part.startswith(".") for part in rel_parts): + continue + if path.suffix in _WATCH_SUFFIXES: + yield path.resolve() + + +def _external_dependency_files(roots: list[Path]) -> set[Path]: + """External content files the compiler read, taken from the disk manifest. + + The manifest records each page's full dependency set (own module, markdown, + component/state modules). Any dependency that lives *outside* the reload + roots — e.g. a docs app's markdown in a sibling directory — is invisible to + ``get_reload_paths`` and must be watched explicitly so editing it rebuilds. + + Args: + roots: The resolved reload roots. + + Returns: + Resolved external dependency file paths to watch. + """ + from reflex.compiler import disk_cache + + manifest = disk_cache.load_manifest() + if not manifest: + return set() + out: set[Path] = set() + for page in manifest.get("pages", {}).values(): + for dep in page.get("dep_hashes", {}): + path = Path(dep) + if not _under_roots(path, roots): + out.add(path) + return out + + +def _global_files(root: Path) -> set[Path]: + """Genuinely-global files whose change forces a full rebuild / daemon restart. + + Args: + root: The project root. + + Returns: + Resolved paths of rxconfig + lockfiles + package.json that exist. + """ + from reflex.compiler import page_cache + + return { + (root / name).resolve() + for name in page_cache._GLOBAL_FILES + if (root / name).exists() + } + + +def _mtime_ns(path: Path) -> int | None: + """Return the file's modification time in ns, or None if it can't be read. + + Args: + path: The file to stat. + + Returns: + The ``st_mtime_ns`` value, or None on error. + """ + try: + return path.stat().st_mtime_ns + except OSError: + return None + + +def _watch_paths(roots: list[Path], root: Path, external_files: set[Path]) -> set[Path]: + """Build the full set of files to watch this tick. + + ``external_files`` (the per-page external content deps from the manifest) is + passed in rather than re-read here, so the manifest is parsed once per compile + instead of on every poll tick. + + Args: + roots: The resolved reload roots. + root: The project root. + external_files: External content dependency files (from the manifest). + + Returns: + The set of paths to snapshot. + """ + paths: set[Path] = set(external_files) + for r in roots: + paths.update(_iter_source_files(r)) + paths.update(_global_files(root)) + assets = root / "assets" + if assets.is_dir(): + paths.update(p.resolve() for p in assets.rglob("*") if p.is_file()) + return paths + + +def _snapshot(paths: set[Path]) -> dict[Path, int]: + """Snapshot ``{path: mtime_ns}`` for the given files. + + Args: + paths: The files to stat. + + Returns: + A mapping of file path to its modification time (unreadable files omitted). + """ + return {p: m for p in paths if (m := _mtime_ns(p)) is not None} + + +def _first_party_module_names(roots: list[Path]) -> set[str]: + """Names of all loaded modules belonging to the user's first-party packages. + + First-party top-level package names are inferred from the *regular* modules + whose ``__file__`` resolves under a reload root (a plain attribute read — no + namespace-package ``__path__`` recalculation, which is lazy and would break + while ``sys.modules`` is being mutated). Every loaded module sharing one of + those top-level names is then first-party, which captures namespace packages + (they have no ``__file__``) purely by name string. + + Args: + roots: The resolved reload roots. + + Returns: + The set of ``sys.modules`` keys to purge. + """ + top_level: set[str] = set() + for name, mod in list(sys.modules.items()): + file = getattr(mod, "__file__", None) + if not file: + continue + try: + rf = Path(file).resolve() + except OSError: + continue + if _under_roots(rf, roots): + top_level.add(name.partition(".")[0]) + if not top_level: + return set() + return {name for name in sys.modules if name.partition(".")[0] in top_level} + + +def _reset_first_party(roots: list[Path]) -> None: + """Make this interpreter clean w.r.t. first-party code before re-importing. + + Purges the user's first-party modules from ``sys.modules`` and clears the + cross-module registries/caches that would otherwise pin the old class + objects (the ``__subclasses__`` zombie / duplicate-substate / stale-page + hazards). Third-party modules (reflex, reflex-base, pandas, …) are left + imported and warm. + + Args: + roots: The resolved reload roots whose modules are first-party. + """ + for name in _first_party_module_names(roots): + sys.modules.pop(name, None) + # The import-system finder caches were inherited from the warm parent via + # fork and are now stale (they reference the purged modules); without this a + # re-import can resolve a stale spec for a since-changed module. + importlib.invalidate_caches() + + from reflex_base.registry import RegistrationContext + + from reflex.compiler import page_cache + from reflex.page import DECORATED_PAGES + from reflex.state import BaseState, all_base_state_classes + + ctx = RegistrationContext.ensure_context() + ctx.base_states.clear() + ctx.base_state_substates.clear() + ctx.event_handlers.clear() + all_base_state_classes.clear() + for cached in ( + BaseState.get_parent_state, + BaseState.get_root_state, + BaseState.get_name, + BaseState.get_full_name, + BaseState.get_class_substate, + ): + cached.cache_clear() + DECORATED_PAGES.clear() + # In-process caches hold live objects from the previous import; drop them so + # the fresh compile can't reuse a tree built from now-stale classes. Cross- + # compile reuse comes from the on-disk manifest, not these. + page_cache.clear_import_graph() + page_cache.clear_page_store() + _reset_model_metadata() + + +def _reset_model_metadata() -> None: + """Clear the SQLAlchemy/SQLModel table + model registries. + + ``rx.Model`` subclasses (including ones a docs demo ``exec``s) register their + table in a process-global ``MetaData`` that lives in the framework, which the + forked child inherits warm and populated. Re-evaluating a page that defines + such a model would then raise ``Table '...' is already defined``. A fresh + respawn never hits this (empty registry); resetting here restores that + fresh-process contract. Best-effort: apps without a DB layer have nothing to + clear. + """ + with contextlib.suppress(Exception): + import sqlmodel + + sqlmodel.SQLModel.metadata.clear() + with contextlib.suppress(Exception): + from reflex.model import Model + + Model.metadata.clear() + with contextlib.suppress(Exception): + from reflex.model import ModelRegistry + + ModelRegistry.models.clear() + ModelRegistry._metadata = None + + +def _child_compile(roots: list[Path], prerender_routes: bool) -> None: + """Reset first-party state, re-import the app fresh, and compile incrementally. + + Runs in a forked child (POSIX) or a one-shot subprocess (Windows). Must not + return normally on error — the caller maps the exit code to success/failure. + + Args: + roots: The resolved reload roots. + prerender_routes: Whether to prerender routes during compile. + """ + from reflex.utils import prerequisites + + _reset_first_party(roots) + prerequisites.get_compiled_app( + reload=False, + prerender_routes=prerender_routes, + use_rich=True, + trigger="hot_reload", + ) + + +def _await_child(pid: int) -> bool: + """Reap a forked compile child, killing it if it exceeds the watchdog timeout. + + Args: + pid: The forked child's pid. + + Returns: + True if it exited 0; False on failure, timeout, or signal (a + signal-killed child — e.g. Ctrl-C during shutdown — is a quiet False). + """ + deadline = time.monotonic() + _COMPILE_TIMEOUT + while True: + done, status = os.waitpid(pid, os.WNOHANG) + if done == pid: + return os.waitstatus_to_exitcode(status) == 0 + if time.monotonic() > deadline: + with contextlib.suppress(OSError): + os.kill(pid, signal.SIGKILL) + with contextlib.suppress(OSError): + os.waitpid(pid, 0) + console.error("Compile child timed out; killed it, keeping last build.") + return False + time.sleep(0.02) + + +def _can_fork() -> bool: + """Whether forking is safe right now (POSIX and the process is single-threaded). + + Forking a multi-threaded process and then running Python (not exec) inherits + locks held by threads that don't exist in the child — a classic deadlock. The + user app, imported warm in the parent, may have started a background thread + at import time, so this is checked per compile. + + Returns: + True if a per-compile ``fork()`` is safe. + """ + return hasattr(os, "fork") and threading.active_count() == 1 + + +def _compile_once(roots: list[Path], prerender_routes: bool) -> bool: + """Run one incremental compile in an isolated child; report success. + + Uses a copy-on-write ``fork()`` when safe (warm), else a fresh subprocess + (Windows, or when the warm parent is no longer single-threaded). + + Args: + roots: The resolved reload roots. + prerender_routes: Whether to prerender routes during compile. + + Returns: + True if the child compiled successfully, else False. + """ + if _can_fork(): + pid = os.fork() + if pid == 0: # child + code = 0 + try: + _child_compile(roots, prerender_routes) + except BaseException: # report any failure, never crash the daemon + import traceback + + traceback.print_exc() + code = 1 + finally: + os._exit(code) + return _await_child(pid) + + # No fork (Windows) or unsafe to fork: a fresh (cold) subprocess compiles. + try: + proc = subprocess.run( + [sys.executable, "-m", "reflex.utils.compile_daemon", "--once"], + check=False, + timeout=_COMPILE_TIMEOUT, + ) + except subprocess.TimeoutExpired: + console.error("Compile subprocess timed out; keeping the last good build.") + return False + return proc.returncode == 0 + + +def _serve() -> None: + """Run the warm compile daemon: initial compile, then watch-and-recompile.""" + from reflex.utils import prerequisites + + root = Path.cwd() + prerender_routes = bool(os.environ.get("REFLEX_PRERENDER_ROUTES")) + roots = _reload_roots() + parent_pid = os.getppid() + + # Warm import + initial compile (writes .web + the manifest); keeps the app + # and its third-party deps resident for copy-on-write children. A failure + # here (e.g. the app is mid-edit and broken) must NOT kill the daemon — fall + # through to the watch loop so the next edit that fixes it recompiles. + try: + with console.timing("Compile daemon: initial compile"): + prerequisites.get_compiled_app( + reload=False, + prerender_routes=prerender_routes, + use_rich=True, + trigger="initial", + ) + except BaseException: # tolerate a broken initial state; keep watching + import traceback + + traceback.print_exc() + console.error("Initial compile failed; watching for a fix.") + + # External content deps (e.g. a docs app's sibling-dir markdown) come from + # the manifest; recompute only after a compile, not on every poll tick. + external = _external_dependency_files(roots) + global_files = _global_files(root) + snapshot = _snapshot(_watch_paths(roots, root, external)) + console.info("Compile daemon ready (warm); watching for changes.") + + while True: + time.sleep(_POLL_INTERVAL) + # Never outlive reflex-run: if our parent died we were reparented. + if os.getppid() != parent_pid: + return + current = _snapshot(_watch_paths(roots, root, external)) + changed = { + p + for p in current.keys() | snapshot.keys() + if current.get(p) != snapshot.get(p) + } + if not changed: + continue + + # A change to a genuinely-global input (rxconfig/lockfiles, or a reflex + # upgrade) can't be applied to the warm parent (it imported the old + # version); re-exec the daemon so the new world is actually loaded. + if changed & global_files: + console.info("Global config changed; restarting compile daemon.") + os.execv( + sys.executable, [sys.executable, "-m", "reflex.utils.compile_daemon"] + ) + + time.sleep(_DEBOUNCE) # absorb the rest of a burst of saves + roots = _reload_roots() + ok = _compile_once(roots, prerender_routes) + if not ok: + console.error("Compile failed; keeping the last good build.") + # Re-snapshot AFTER compiling so writes the compile itself made are the + # new baseline; refresh external deps + globals from the new manifest so + # a newly-referenced content dir becomes watched. + external = _external_dependency_files(roots) + global_files = _global_files(root) + snapshot = _snapshot(_watch_paths(roots, root, external)) + + +def main(argv: list[str] | None = None) -> int: + """Entry point for ``python -m reflex.utils.compile_daemon``. + + Args: + argv: Command-line arguments (defaults to ``sys.argv[1:]``). + + Returns: + Process exit code. + """ + argv = sys.argv[1:] if argv is None else argv + if "--once" in argv: + try: + _child_compile( + _reload_roots(), bool(os.environ.get("REFLEX_PRERENDER_ROUTES")) + ) + except BaseException: # report any failure, never crash + import traceback + + traceback.print_exc() + return 1 + return 0 + try: + _serve() + except KeyboardInterrupt: + return 0 # clean shutdown (Ctrl-C); no traceback + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/reflex/utils/exec.py b/reflex/utils/exec.py index 6e0543d93db..fbdef7d2499 100644 --- a/reflex/utils/exec.py +++ b/reflex/utils/exec.py @@ -601,6 +601,9 @@ def run_uvicorn_backend(host: str, port: int, loglevel: LogLevel): log_level=loglevel.value, reload=True, reload_dirs=list(map(str, get_reload_paths())), + # uvicorn's reload filter defaults to *.py only, so markdown/data edits + # would never trigger a reload; include the content suffixes too. + reload_includes=["*.py", "*.md", "*.mdx"], reload_delay=0.1, ) diff --git a/tests/units/compiler/test_compiler_utils.py b/tests/units/compiler/test_compiler_utils.py index db7eff1a495..7ffd7679f25 100644 --- a/tests/units/compiler/test_compiler_utils.py +++ b/tests/units/compiler/test_compiler_utils.py @@ -4,7 +4,7 @@ import pytest -from reflex.compiler.utils import compile_state +from reflex.compiler.utils import compile_state, write_file from reflex.constants.state import FIELD_MARKER from reflex.state import State from reflex.vars.base import computed_var @@ -48,3 +48,31 @@ async def test_compile_state_resolves_async_computed_vars_with_running_event_loo assert values[f"a{FIELD_MARKER}"] == 1 assert values[f"b{FIELD_MARKER}"] == 2 assert values[f"async_value{FIELD_MARKER}"] == "resolved" + + +def test_write_file_creates_and_updates(tmp_path): + path = tmp_path / "sub" / "page.jsx" + write_file(path, "v1") + assert path.read_text() == "v1" + write_file(path, "v2") + assert path.read_text() == "v2" + + +def test_write_file_atomic_leaves_no_temp_files(tmp_path): + path = tmp_path / "page.jsx" + write_file(path, "content") + # The temp file used for the atomic replace must not linger. + assert [p.name for p in tmp_path.iterdir()] == ["page.jsx"] + + +def test_write_file_skips_byte_identical_write(tmp_path): + """An identical write must not touch the file (so vite isn't told to HMR).""" + path = tmp_path / "page.jsx" + write_file(path, "same") + before = path.stat().st_mtime_ns + import os + + os.utime(path, ns=(before + 1_000_000_000, before + 1_000_000_000)) + bumped = path.stat().st_mtime_ns + write_file(path, "same") # identical -> no rewrite + assert path.stat().st_mtime_ns == bumped diff --git a/tests/units/utils/test_compile_daemon.py b/tests/units/utils/test_compile_daemon.py new file mode 100644 index 00000000000..9d7947a8236 --- /dev/null +++ b/tests/units/utils/test_compile_daemon.py @@ -0,0 +1,225 @@ +"""Tests for the warm fork-per-compile dev compile daemon.""" + +import os + +import pytest + +from reflex.compiler import disk_cache +from reflex.utils import compile_daemon + + +def test_iter_source_files_picks_content_skips_build_dirs(tmp_path): + (tmp_path / "page.py").write_text("x = 1\n") + (tmp_path / "doc.md").write_text("# doc\n") + (tmp_path / "guide.mdx").write_text("mdx\n") + (tmp_path / "data.txt").write_text("ignored\n") # not a watched suffix + web = tmp_path / ".web" + web.mkdir() + (web / "build.js").write_text("// artifact\n") + pycache = tmp_path / "__pycache__" + pycache.mkdir() + (pycache / "stale.py").write_text("# compiled\n") + + found = {p.name for p in compile_daemon._iter_source_files(tmp_path)} + assert found == {"page.py", "doc.md", "guide.mdx"} + + +def test_external_dependency_files_includes_sibling_markdown(tmp_path, monkeypatch): + """A page's markdown read from a sibling dir (outside the app root) is watched. + + This is the regression for markdown edits never triggering a reload: such + files live outside ``get_reload_paths`` and are only known via the compile + manifest's per-page dependency sets. + """ + app_root = tmp_path / "app" + app_root.mkdir() + own_module = app_root / "page.py" + own_module.write_text("x = 1\n") + sibling_md = tmp_path / "docs" / "guide.md" + sibling_md.parent.mkdir() + sibling_md.write_text("# guide\n") + + manifest = { + "pages": { + "/g": { + "dep_hashes": { + str(own_module): "h1", # under the app root -> not external + str(sibling_md): "h2", # sibling dir -> must be watched + } + } + } + } + monkeypatch.setattr(disk_cache, "load_manifest", lambda: manifest) + + external = compile_daemon._external_dependency_files([app_root.resolve()]) + assert sibling_md.resolve() in external + assert own_module.resolve() not in external + + +def test_external_dependency_files_empty_without_manifest(tmp_path, monkeypatch): + monkeypatch.setattr(disk_cache, "load_manifest", lambda: None) + assert compile_daemon._external_dependency_files([tmp_path]) == set() + + +def test_snapshot_detects_external_markdown_change(tmp_path, monkeypatch): + """The watch snapshot includes sibling markdown and reflects its mtime change.""" + app_root = tmp_path / "app" + app_root.mkdir() + (app_root / "page.py").write_text("x = 1\n") + sibling_md = tmp_path / "docs" / "guide.md" + sibling_md.parent.mkdir() + sibling_md.write_text("# v1\n") + monkeypatch.setattr( + disk_cache, + "load_manifest", + lambda: {"pages": {"/g": {"dep_hashes": {str(sibling_md): "h"}}}}, + ) + + roots = [app_root.resolve()] + external = compile_daemon._external_dependency_files(roots) + snap1 = compile_daemon._snapshot( + compile_daemon._watch_paths(roots, tmp_path, external) + ) + assert sibling_md.resolve() in snap1 + assert (app_root / "page.py").resolve() in snap1 + + # Force a distinct mtime (deterministic, independent of fs mtime resolution). + bumped = snap1[sibling_md.resolve()] + 1_000_000_000 + os.utime(sibling_md, ns=(bumped, bumped)) + snap2 = compile_daemon._snapshot( + compile_daemon._watch_paths(roots, tmp_path, external) + ) + assert snap2[sibling_md.resolve()] != snap1[sibling_md.resolve()] + + +def test_first_party_module_names_includes_namespace_packages(tmp_path, monkeypatch): + """Namespace packages (no ``__file__``) are captured for purge by name. + + Regression: a namespace package left in ``sys.modules`` after its regular + siblings were purged broke re-import with a ``KeyError`` on the parent path. + They are now identified by sharing a first-party top-level name (derived from + a regular sibling's ``__file__``), never by their lazy ``__path__``. + """ + import importlib + import sys + + nspkg = tmp_path / "ns_under_test" + nspkg.mkdir() # no __init__.py -> namespace package + (nspkg / "leaf.py").write_text("Y = 1\n") # regular submodule with __file__ + monkeypatch.syspath_prepend(str(tmp_path)) + try: + pkg = importlib.import_module("ns_under_test") + importlib.import_module("ns_under_test.leaf") + assert getattr(pkg, "__file__", None) is None # confirm it's a namespace pkg + + names = compile_daemon._first_party_module_names([tmp_path.resolve()]) + assert "ns_under_test" in names # namespace pkg captured by name + assert "ns_under_test.leaf" in names # regular module captured via __file__ + finally: + sys.modules.pop("ns_under_test", None) + sys.modules.pop("ns_under_test.leaf", None) + + +@pytest.mark.skipif(not hasattr(os, "fork"), reason="requires os.fork (POSIX)") +def test_reset_model_metadata_allows_table_redefinition(): + """After reset, an ``rx.Model`` table can be redefined without conflict. + + Regression: the forked child inherits the warm, populated SQLAlchemy + ``MetaData``; re-evaluating a page that defines a model raised + ``Table '...' is already defined``. Run in a fork so clearing the global + metadata can't affect the test process. + """ + read_fd, write_fd = os.pipe() + pid = os.fork() + if pid == 0: # child + os.close(read_fd) + result = b"E" + try: + import warnings + + warnings.simplefilter("ignore") + import reflex as rx + + def _define(): + class DaemonResetUser(rx.Model, table=True): + name: str + + _define() + compile_daemon._reset_model_metadata() + _define() # must NOT raise "Table 'daemonresetuser' is already defined" + result = b"1" + except Exception: + import traceback + + traceback.print_exc() + finally: + os.write(write_fd, result) + os.close(write_fd) + os._exit(0) + + os.close(write_fd) + out = os.read(read_fd, 1) + os.close(read_fd) + os.waitpid(pid, 0) + assert out == b"1" + + +@pytest.mark.skipif(not hasattr(os, "fork"), reason="requires os.fork (POSIX)") +def test_reset_first_party_purges_modules_and_registries(tmp_path): + """``_reset_first_party`` purges first-party modules and clears registries. + + Runs in a forked child so the global-registry reset can't corrupt the test + process — exactly how the daemon uses it (a throwaway child per compile). + """ + mod_file = tmp_path / "fp_module.py" + mod_file.write_text("VALUE = 1\n") + + read_fd, write_fd = os.pipe() + pid = os.fork() + if pid == 0: # child + os.close(read_fd) + result = b"E" + try: + import importlib.util + import sys + + spec = importlib.util.spec_from_file_location( + "fp_module_under_test", mod_file + ) + assert spec is not None + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) # type: ignore[union-attr] + sys.modules["fp_module_under_test"] = module + + from reflex_base.registry import RegistrationContext + + from reflex.page import DECORATED_PAGES + from reflex.state import all_base_state_classes + + RegistrationContext.ensure_context().base_states["sentinel"] = object() # type: ignore[assignment] + all_base_state_classes["sentinel"] = None + DECORATED_PAGES["sentinel_app"].append((lambda: None, {})) + + compile_daemon._reset_first_party([tmp_path.resolve()]) + + purged = "fp_module_under_test" not in sys.modules + cleared = ( + not RegistrationContext.ensure_context().base_states + and not all_base_state_classes + and not DECORATED_PAGES + ) + result = b"1" if (purged and cleared) else b"0" + except Exception: + import traceback + + traceback.print_exc() + finally: + os.write(write_fd, result) + os.close(write_fd) + os._exit(0) + + os.close(write_fd) + out = os.read(read_fd, 1) + os.close(read_fd) + os.waitpid(pid, 0) + assert out == b"1" From 7078d9ead0a056f1cf4f8cca83222eda538fb305 Mon Sep 17 00:00:00 2001 From: Farhan <www.mfarvirus@gmail.com> Date: Mon, 29 Jun 2026 01:23:12 +0500 Subject: [PATCH 03/56] perf(docs): build component prop-docs lazily at page eval multi_docs built every component's prop tables at module import, so the whole library reference was reconstructed on every import of the docs tree -- re-run on every dev hot-reload reimport, cold start, and backend respawn. Move the build into the page render closures (matching the non-library doc path) so a page builds its prop tables only when it is actually compiled. Docs cold import 9.7s -> 1.9s; hot-reload reimport 8.3s -> 0.6s. --- docs/app/reflex_docs/pages/docs/component.py | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/docs/app/reflex_docs/pages/docs/component.py b/docs/app/reflex_docs/pages/docs/component.py index 31954f6bcf8..e2847d8fb5c 100644 --- a/docs/app/reflex_docs/pages/docs/component.py +++ b/docs/app/reflex_docs/pages/docs/component.py @@ -921,16 +921,9 @@ def multi_docs( title: str, ll_component_list: list | None = None, ): - components = [ - component_docs(component_tuple, previews) - for component_tuple in component_list[1:] - ] ll_actual_path = actual_path.replace(".md", "-ll.md") ll_doc_exists = os.path.exists(ll_actual_path) ll_list = ll_component_list if ll_component_list is not None else component_list - ll_components = [ - component_docs(component_tuple, previews) for component_tuple in ll_list[1:] - ] active_class_name = "font-small bg-secondary-2 p-2 text-secondary-11 rounded-xl shadow-large w-28 cursor-default border border-secondary-4 text-center" @@ -982,6 +975,13 @@ def links(current_page, ll_doc_exists, path): @docpage(set_path=path, t=title) def out(): + # Built here (page eval) rather than at module import, so importing the + # docs tree — and every hot-reload reimport — doesn't construct every + # component's prop tables up front; only a compiled page builds its own. + components = [ + component_docs(component_tuple, previews) + for component_tuple in component_list[1:] + ] toc = get_docgen_toc(actual_path) doc_content = Path(actual_path).read_text(encoding="utf-8") # Append API Reference headings for the component list @@ -1010,6 +1010,9 @@ def out(): @docpage(set_path=path + "low", t=title + " (Low Level)") def ll(): + ll_components = [ + component_docs(component_tuple, previews) for component_tuple in ll_list[1:] + ] ll_virtual = virtual_path.replace(".md", "-ll.md") toc = get_docgen_toc(ll_actual_path) doc_content = Path(ll_actual_path).read_text(encoding="utf-8") From 5ba66143f4a85864e0287395d386074a0cbe15a4 Mon Sep 17 00:00:00 2001 From: Farhan <www.mfarvirus@gmail.com> Date: Mon, 29 Jun 2026 01:28:26 +0500 Subject: [PATCH 04/56] perf(compiler): trim compile-daemon hot reload - disk_cache: stop re-evaluating stateful HIT pages during an incremental rebuild. The compiling process never serves (the daemon, the initial compile, and CLI compiles all exit; the serving backend re-evaluates the marked stateful pages itself), so re-running their render pipeline was pure waste. The stateful-pages marker stays complete -- hits recorded from the manifest, misses from the fresh compile. - compile_daemon: poll faster (0.25s -> 0.05s) but cheaply -- stat the known file set each tick and rglob only every 1s for added/removed files, cutting detection latency without burning idle CPU. - compile_daemon: log per-edit timing (reset / reimport / compile). --- reflex/compiler/disk_cache.py | 22 +++++----- reflex/utils/compile_daemon.py | 44 +++++++++++++++----- tests/units/compiler/test_disk_cache.py | 53 ++++++++++++++++++++++--- 3 files changed, 93 insertions(+), 26 deletions(-) diff --git a/reflex/compiler/disk_cache.py b/reflex/compiler/disk_cache.py index 7d6228fe11c..0f505d0abfb 100644 --- a/reflex/compiler/disk_cache.py +++ b/reflex/compiler/disk_cache.py @@ -375,18 +375,20 @@ def try_incremental_rebuild( install_imports, page_ctx.frontend_imports, memo_imports ) - # Re-register state for stateful hit pages (skipping eval would drop their - # state classes); stateless hits need nothing. + # Record which routes are stateful — miss pages from this compile, hit pages + # from the manifest — so the stateful-pages marker is complete. We do NOT + # re-evaluate hit pages to register their state in this process: it only + # produces .web and exits (the daemon, the initial compile, and CLI compiles + # never serve), and the serving backend re-evaluates the marked stateful + # pages itself. Re-evaluating them here re-ran the full render pipeline for + # every unchanged stateful page on every edit, for nothing. stateful_routes: dict[str, None] = {} - with console.timing("Evaluate Pages (Backend)"): - for page in pages: - if page.route in miss_routes: - if miss_ctx is not None and page.route in miss_ctx.stateful_routes: - stateful_routes[page.route] = None - continue - if manifest["pages"][page.route]["is_stateful"]: - app._compile_page(page.route, save_page=False) + for page in pages: + if page.route in miss_routes: + if miss_ctx is not None and page.route in miss_ctx.stateful_routes: stateful_routes[page.route] = None + elif manifest["pages"][page.route]["is_stateful"]: + stateful_routes[page.route] = None app._stateful_pages.update(stateful_routes) app._write_stateful_pages_marker() diff --git a/reflex/utils/compile_daemon.py b/reflex/utils/compile_daemon.py index 603ebe6d6f0..6c5d46d097a 100644 --- a/reflex/utils/compile_daemon.py +++ b/reflex/utils/compile_daemon.py @@ -52,11 +52,16 @@ from reflex.utils import console -#: Seconds between poll passes over the watched file set. -_POLL_INTERVAL = 0.25 +#: Seconds between poll passes. Each pass only re-``stat``s the known file set +#: (cheap), so this can be small for snappy detection of edits to existing files. +_POLL_INTERVAL = 0.05 #: After a change is seen, wait this long and re-snapshot so a burst of saves #: (e.g. format-on-save touching many files) collapses into a single compile. -_DEBOUNCE = 0.1 +_DEBOUNCE = 0.05 +#: How often to re-walk the tree (``rglob``) to discover added/removed files. +#: Between rescans we only ``stat`` the known set; adds are rarer and tolerate a +#: little latency, so this keeps idle CPU low while polling fast for edits. +_RESCAN_INTERVAL = 1.0 #: Watchdog: kill a compile child that runs longer than this (a hung/deadlocked #: child must never wedge the daemon). Generous enough for a real full compile. _COMPILE_TIMEOUT = 300.0 @@ -369,12 +374,18 @@ def _child_compile(roots: list[Path], prerender_routes: bool) -> None: """ from reflex.utils import prerequisites + # Timed in three steps so every hot reload reports where it spent its time + # (resetting state vs re-importing first-party code vs compiling). + t0 = time.perf_counter() _reset_first_party(roots) - prerequisites.get_compiled_app( - reload=False, - prerender_routes=prerender_routes, - use_rich=True, - trigger="hot_reload", + t1 = time.perf_counter() + app, _ = prerequisites.get_and_validate_app(reload=False) + t2 = time.perf_counter() + app._compile(prerender_routes=prerender_routes, use_rich=True, trigger="hot_reload") + t3 = time.perf_counter() + console.info( + f"Hot reload {t3 - t0:.2f}s (reset {t1 - t0:.2f}s, " + f"reimport {t2 - t1:.2f}s, compile {t3 - t2:.2f}s)" ) @@ -489,7 +500,11 @@ def _serve() -> None: # the manifest; recompute only after a compile, not on every poll tick. external = _external_dependency_files(roots) global_files = _global_files(root) - snapshot = _snapshot(_watch_paths(roots, root, external)) + # `paths` is the watched set, refreshed by an rglob rescan; each tick only + # re-stats it (cheap), so the poll can be fast without burning idle CPU. + paths = _watch_paths(roots, root, external) + snapshot = _snapshot(paths) + last_rescan = time.monotonic() console.info("Compile daemon ready (warm); watching for changes.") while True: @@ -497,7 +512,12 @@ def _serve() -> None: # Never outlive reflex-run: if our parent died we were reparented. if os.getppid() != parent_pid: return - current = _snapshot(_watch_paths(roots, root, external)) + # Cheap stat of the known set every tick; re-walk the tree occasionally + # to discover added/removed files. + if time.monotonic() - last_rescan >= _RESCAN_INTERVAL: + paths = _watch_paths(roots, root, external) + last_rescan = time.monotonic() + current = _snapshot(paths) changed = { p for p in current.keys() | snapshot.keys() @@ -525,7 +545,9 @@ def _serve() -> None: # a newly-referenced content dir becomes watched. external = _external_dependency_files(roots) global_files = _global_files(root) - snapshot = _snapshot(_watch_paths(roots, root, external)) + paths = _watch_paths(roots, root, external) + snapshot = _snapshot(paths) + last_rescan = time.monotonic() def main(argv: list[str] | None = None) -> int: diff --git a/tests/units/compiler/test_disk_cache.py b/tests/units/compiler/test_disk_cache.py index 62b3330a38d..ebd54be385e 100644 --- a/tests/units/compiler/test_disk_cache.py +++ b/tests/units/compiler/test_disk_cache.py @@ -1,7 +1,7 @@ """Tests for the experimental disk-persisted incremental compile cache.""" import dataclasses -from collections.abc import Callable +from collections.abc import Callable, Sequence from typing import Any from reflex_base.components.component import Component @@ -44,7 +44,7 @@ def _page_c() -> Component: return rx.el.div(rx.el.h1("Page C"), _footer()) -def _compile(pages: list[_FakePage]) -> CompileContext: +def _compile(pages: Sequence[Any]) -> CompileContext: ctx = CompileContext( pages=pages, hooks=CompilerHooks(plugins=default_page_plugins()), @@ -267,9 +267,9 @@ def test_incremental_rebuild_one_miss_writes_only_that_page(tmp_path, monkeypatc # The edited page was recompiled and written; its content matches a clean # compile of that page. - out_path = compiler_utils.resolve_path_of_web_dir( - ctx.compiled_pages[edited_route].output_path - ) + output_path = ctx.compiled_pages[edited_route].output_path + assert output_path is not None + out_path = compiler_utils.resolve_path_of_web_dir(output_path) assert out_path.exists() assert ( out_path.read_text(encoding="utf-8") @@ -277,6 +277,49 @@ def test_incremental_rebuild_one_miss_writes_only_that_page(tmp_path, monkeypatc ) +def test_stateful_hit_is_marked_but_not_reevaluated(tmp_path, monkeypatch): + """A stateful HIT page is recorded in the marker but never re-evaluated. + + The compile process only produces .web and exits; the serving backend + re-evaluates the marked stateful pages itself, so re-evaluating them during + the incremental rebuild was pure waste. + """ + import json + + web = tmp_path / ".web" + web.mkdir() + monkeypatch.setattr(disk_cache.prerequisites, "get_web_dir", lambda: web) + + app = rx.App() + app.add_page(_page_a, route="/a") + pages = list(app._unevaluated_pages.values()) + route = pages[0].route + ctx = _compile(pages) + disk_cache.write_manifest(ctx, pages, ctx.all_imports, root=tmp_path) + # Mark the page as a stateful HIT page in the manifest. + manifest_path = web / disk_cache._MANIFEST_FILE + manifest = json.loads(manifest_path.read_text()) + manifest["pages"][route]["is_stateful"] = True + manifest_path.write_text(json.dumps(manifest)) + _stub_externals(app, monkeypatch) + + reevaluated: list[str] = [] + monkeypatch.setattr( + app, "_compile_page", lambda route, **k: reevaluated.append(route) + ) + + assert ( + disk_cache.try_incremental_rebuild( + app, compiler_plugins=[], prerender_routes=False, root=tmp_path + ) + is True + ) + # Not re-evaluated... + assert reevaluated == [] + # ...but still recorded as stateful so the backend's marker is complete. + assert route in app._stateful_pages + + def test_load_manifest_rejects_wrong_schema(tmp_path, monkeypatch): import json From fe060560e3c725bebabebeb8d332061b0384ed10 Mon Sep 17 00:00:00 2001 From: Farhan <www.mfarvirus@gmail.com> Date: Mon, 29 Jun 2026 22:17:43 +0500 Subject: [PATCH 05/56] perf(compiler): shrink compile-cache manifest, drop in-process cache MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Profiling the hot-reload compile on the docs app (418 pages) showed the disk-cache rebuild spent ~1.1s of its ~1.6s on manifest I/O: the manifest stored every page's rendered output_code (and output_path/frontend_imports), ballooning it to 46MB, yet those fields are never read back — only dep_hashes, app_wrap_keys, is_stateful, and the merged all_imports are consumed. Storing just the bookkeeping that is read drops the manifest to 14MB. The in-process page cache (_PAGE_STORE / validate_page / store_page) is never reached by the warm daemon (its fork child cleared it and the disk path returns first), and forcing it measured ~55x slower than the disk path because it re-runs the whole app-level pipeline (memo render, stylesheet, plugins) every edit. Remove it, the verify mode that only guarded it (REFLEX_COMPILE_CACHE_VERIFY), and the dead page_source_fingerprint. Manifest schema bumped 2->3 so stale fat manifests are ignored. --- .../src/reflex_base/environment.py | 8 +- reflex/compiler/compiler.py | 178 +----------------- reflex/compiler/disk_cache.py | 52 +++-- reflex/compiler/page_cache.py | 118 ++---------- reflex/utils/compile_daemon.py | 7 +- tests/units/compiler/test_disk_cache.py | 39 ++-- tests/units/compiler/test_page_cache.py | 71 +------ 7 files changed, 60 insertions(+), 413 deletions(-) diff --git a/packages/reflex-base/src/reflex_base/environment.py b/packages/reflex-base/src/reflex_base/environment.py index cca5de00ba9..169eac33176 100644 --- a/packages/reflex-base/src/reflex_base/environment.py +++ b/packages/reflex-base/src/reflex_base/environment.py @@ -613,16 +613,10 @@ class EnvironmentVariables: # Experimental: incremental compile cache. A fresh compile process (e.g. a # reflex-run hot-reload worker) reuses each page's compiled output from an - # on-disk manifest and recompiles only the pages whose source changed, - # backed by an in-process per-page cache for repeat compiles in one process. + # on-disk manifest and recompiles only the pages whose source changed. # See reflex/compiler/disk_cache.py and reflex/compiler/page_cache.py. REFLEX_COMPILE_CACHE: EnvVar[bool] = env_var(False) - # When the compile cache reuses pages, also run a full compile and assert - # byte-identical output, falling back to the full result on any mismatch. - # Doubles compile time; for validating the cache on an app. - REFLEX_COMPILE_CACHE_VERIFY: EnvVar[bool] = env_var(False) - # Inherited by uvicorn/granian reload workers so the backend can distinguish # dev reload-capable worker boots from other backend starts. Never set in prod. REFLEX_DEV_BACKEND_RELOAD_ACTIVE: EnvVar[bool] = env_var(False, internal=True) diff --git a/reflex/compiler/compiler.py b/reflex/compiler/compiler.py index de5433d3697..2b368cd5c7b 100644 --- a/reflex/compiler/compiler.py +++ b/reflex/compiler/compiler.py @@ -1108,91 +1108,6 @@ def _resolve_radix_themes_plugin( return plugin_chain, radix_plugin -def _normalize_imports_for_compare(all_imports: Any) -> dict[str, list[str]]: - """Render an import dict to a comparable, order-independent form. - - Args: - all_imports: The parsed import dict to normalize. - - Returns: - A mapping of library to its sorted, stringified import fields. - """ - return {lib: sorted(str(v) for v in fields) for lib, fields in all_imports.items()} - - -def _diff_compile_contexts( - incremental: CompileContext, full: CompileContext -) -> list[str]: - """Return the aggregate fields where the two compile contexts diverge. - - Args: - incremental: The cache-assisted compile context. - full: A no-cache full compile of the same app. - - Returns: - A list of human-readable divergence labels (empty when identical). - """ - diffs: list[str] = [] - inc_pages = {r: pc.output_code for r, pc in incremental.compiled_pages.items()} - full_pages = {r: pc.output_code for r, pc in full.compiled_pages.items()} - if inc_pages.keys() != full_pages.keys(): - diffs.append(f"routes:{sorted(set(inc_pages) ^ set(full_pages))}") - diffs.extend( - f"page:{r}" - for r in inc_pages.keys() & full_pages.keys() - if inc_pages[r] != full_pages[r] - ) - if _normalize_imports_for_compare( - incremental.all_imports - ) != _normalize_imports_for_compare(full.all_imports): - diffs.append("all_imports") - if sorted(map(str, incremental.auto_memo_components)) != sorted( - map(str, full.auto_memo_components) - ): - diffs.append("auto_memo_components") - if sorted(incremental.stateful_routes) != sorted(full.stateful_routes): - diffs.append("stateful_routes") - if sorted(map(str, incremental.app_wrap_components)) != sorted( - map(str, full.app_wrap_components) - ): - diffs.append("app_wrap_components") - return diffs - - -def _full_compile_context( - app: App, compiler_plugins: Sequence[Plugin] -) -> CompileContext: - """Compile every page with no cache reuse (for verify-mode comparison). - - Resets the shared bundling/memo globals first so it starts from the same - clean state the primary compile did. - - Args: - app: The app to compile. - compiler_plugins: The resolved compiler plugins. - - Returns: - A fully compiled context over all pages. - """ - from reflex_base.components.dynamic import bundle_library, reset_bundled_libraries - - reset_bundled_libraries() - reset_memo_component_classes() - for plugin in compiler_plugins: - for dependency in plugin.get_frontend_dependencies(): - bundle_library(dependency) - ctx = CompileContext( - app=app, - pages=list(app._unevaluated_pages.values()), - hooks=CompilerHooks( - plugins=default_page_plugins(style=app.style, plugins=compiler_plugins) - ), - ) - with ctx: - ctx.compile() - return ctx - - def compile_app( app: App, *, @@ -1278,34 +1193,9 @@ def compile_app( task = progress.add_task("Compiling:", total=base_total) all_pages = list(app._unevaluated_pages.values()) - # In-process per-page cache with a Salsa-style dependency graph. Reuse the - # compiled context of pages whose recorded dependency set is byte-unchanged - # (and whose global epoch matches); compile only the rest, then re-merge - # cached pages' contributions into the app-wide aggregates. In-process only - # (contributions hold live objects), so it helps repeat compiles in one - # process; cross-process reuse is handled by the disk cache above. - in_process_cache = cache_on - route_to_component = {p.route: p.component for p in all_pages} - state_index: dict[str, Any] = {} - epoch = "" - hasher: Any = None - hit_routes: list[str] = [] - if in_process_cache: - from reflex.compiler import page_cache - - state_index, _ = page_cache.state_dependency_index() - epoch = page_cache.global_epoch() - hasher = page_cache.make_hasher() - hit_routes = [ - page.route - for page in all_pages - if page_cache.validate_page(page.route, epoch, hasher) is not None - ] - - hit_set = set(hit_routes) compile_ctx = CompileContext( app=app, - pages=[p for p in all_pages if p.route not in hit_set], + pages=all_pages, hooks=CompilerHooks( plugins=default_page_plugins(style=app.style, plugins=compiler_plugins) ), @@ -1317,72 +1207,6 @@ def compile_app( render_progress=lambda: progress.advance(task), ) - if in_process_cache: - from reflex.compiler import page_cache - - # Cache freshly-compiled (miss) pages with the dependency set they read. - for route in list(compile_ctx.compiled_pages): - page_ctx_fresh = compile_ctx.compiled_pages[route] - dep_hashes = page_cache.page_dependency_hashes( - page_ctx_fresh, - route_to_component[route], - state_index, - hasher, - ) - page_cache.store_page( - route, - epoch, - dep_hashes, - page_ctx_fresh, - route in compile_ctx.stateful_routes, - ) - # Re-merge cached (hit) pages' contributions into the aggregates. - for route in hit_routes: - cached = page_cache.validate_page(route, epoch, hasher) - if cached is None: # defensive: invalidated concurrently - continue - page_ctx, is_stateful = cached - compile_ctx.compiled_pages[route] = page_ctx - compile_ctx.all_imports = utils.merge_imports( - compile_ctx.all_imports, page_ctx.frontend_imports - ) - compile_ctx.app_wrap_components.update(page_ctx.app_wrap_components) - compile_ctx.auto_memo_components.update(page_ctx.memo_contributions) - if is_stateful: - compile_ctx.stateful_routes[route] = None - # Restore deterministic page order (route order of the app). - compile_ctx.compiled_pages = { - p.route: compile_ctx.compiled_pages[p.route] - for p in all_pages - if p.route in compile_ctx.compiled_pages - } - if hit_routes: - console.info( - "Incremental compile: recompiled " - f"{len(all_pages) - len(hit_routes)} page(s)." - ) - - # Verify mode: prove the cache-assisted output matches a full compile, and - # fall back to the full result on any divergence so a cache bug can never - # ship. Doubles compile time; opt-in for validation. - if hit_routes and environment.REFLEX_COMPILE_CACHE_VERIFY.get(): - from reflex.compiler import page_cache - - full_ctx = _full_compile_context(app, compiler_plugins) - diffs = _diff_compile_contexts(compile_ctx, full_ctx) - if diffs: - console.warn( - "compile cache verify FAILED " - f"({len(diffs)} divergence(s): {diffs[:8]}); using the full " - "compile and clearing the page cache." - ) - page_cache.clear_page_store() - compile_ctx = full_ctx - else: - console.debug( - "compile cache verify: incremental output matches full compile" - ) - for route, page_ctx in compile_ctx.compiled_pages.items(): app._check_routes_conflict(route) if not isinstance(page_ctx.root_component, Component): diff --git a/reflex/compiler/disk_cache.py b/reflex/compiler/disk_cache.py index 0f505d0abfb..c69800ee1e7 100644 --- a/reflex/compiler/disk_cache.py +++ b/reflex/compiler/disk_cache.py @@ -2,24 +2,23 @@ Enabled by ``REFLEX_COMPILE_CACHE``. When off (default), nothing changes. -The in-process page cache (``page_cache._PAGE_STORE``) reuses compiled pages -within one process, so it never fires in the ``reflex run`` edit loop — the -reloader respawns a fresh worker subprocess on every ``.py`` change, starting -with an empty in-memory store. This cache persists each page's *serializable* -contribution to disk so the fresh worker can reuse it. +In the ``reflex run`` edit loop every compile runs in a fresh process (the warm +daemon forks a throwaway child per change), so there is no in-memory state to +reuse across edits. This cache persists the small amount of bookkeeping a fresh +process needs to recompile only what changed and reuse everything else already +on disk. **What is persisted** (``.web/reflex_compile_cache.json``): per page, the ``{path: hash}`` of its **dependency set** (the exact source files it read — own module, markdown/data, component modules in its tree, and the state files it -uses; see ``page_cache.page_dependency_hashes``), its rendered ``output_code`` -and path, its ``frontend_imports`` (``ImportVar`` is a frozen dataclass of -primitives), its app-wrap key-set, and whether evaluating it registered new -state. App-wide: the genuinely-global ``epoch`` (reflex version + -config/lockfiles), the reflex version, and the merged imports. The manifest -deliberately stores **no rendered memo files**: a hit page's memo files are -already on disk from the prior compile, and a miss page re-renders its own on -recompile — so writing the manifest is just string/key serialization, never a -second memo render. +uses; see ``page_cache.page_dependency_hashes``), its app-wrap key-set, and +whether evaluating it registered new state. App-wide: the genuinely-global +``epoch`` (reflex version + config/lockfiles), the reflex version, and the merged +frontend imports. The manifest stores **no rendered output**: a page's compiled +``.js`` (and its memo files) already live in ``.web`` from the prior compile — a +hit reuses those files untouched, a miss rewrites its own — so the manifest is +pure bookkeeping, never a second render. (This is why it stays small: storing the +rendered ``output_code`` would balloon it for no gain, as it is never read back.) **The fast path** (``try_incremental_rebuild``). On a fresh compile it reuses the manifest when the global inputs match (reflex version, route set, and the global @@ -29,19 +28,18 @@ (app root, contexts, theme, stylesheet, …) stay valid because the epoch and route set are unchanged. Then: -- A *stateless* hit page is skipped entirely (its frontend file is reused and - evaluating it would register nothing). -- A *stateful* hit page is re-evaluated for the backend only (to re-register its - state classes), reusing its frontend file. ``is_stateful`` is true exactly - when the page's first eval grew the state registry, so this is precisely the - set whose state would otherwise go missing. +- A hit page is left exactly as-is: its frontend ``.js`` is already on disk and + is neither rewritten nor re-evaluated. - A *miss* page (source changed) is fully recompiled and its files rewritten. +- Stateful pages are recorded in the stateful-pages marker — misses from this + compile, hits from the manifest's ``is_stateful`` flag — so the serving backend + (which re-evaluates them to register their state) sees the complete set. This + process only writes ``.web`` and exits; it never re-evaluates a hit page. After recompiling misses, two guards must hold or the whole thing falls back to a full compile (return False): each miss page's app-wrap key-set and stateful flag -must be unchanged (otherwise the reused on-disk app root would be wrong). Any -state edit, shared-file edit, route add/remove, or version change also falls back -to a full compile. ``REFLEX_COMPILE_CACHE_VERIFY`` is the backstop for an app. +must be unchanged (otherwise the reused on-disk app root would be wrong). A route +add/remove or a global/version change also falls back to a full compile. """ from __future__ import annotations @@ -67,7 +65,7 @@ from reflex.app import App #: Bump when the manifest layout changes (old manifests are then ignored). -_SCHEMA = 2 +_SCHEMA = 3 #: Manifest filename under the web directory. _MANIFEST_FILE = "reflex_compile_cache.json" @@ -169,9 +167,6 @@ def write_manifest( "dep_hashes": page_cache.page_dependency_hashes( page_ctx, page.component, state_index, hasher, root ), - "output_path": page_ctx.output_path, - "output_code": page_ctx.output_code, - "frontend_imports": _serialize_imports(page_ctx.frontend_imports), "app_wrap_keys": _wrap_key_strs(page_ctx.app_wrap_components.keys()), "is_stateful": page.route in compile_ctx.stateful_routes, } @@ -433,9 +428,6 @@ def _update_manifest_for_misses( "dep_hashes": page_cache.page_dependency_hashes( page_ctx, page.component, state_index, hasher ), - "output_path": page_ctx.output_path, - "output_code": page_ctx.output_code, - "frontend_imports": _serialize_imports(page_ctx.frontend_imports), "app_wrap_keys": _wrap_key_strs(page_ctx.app_wrap_components.keys()), "is_stateful": page.route in miss_ctx.stateful_routes, } diff --git a/reflex/compiler/page_cache.py b/reflex/compiler/page_cache.py index 97116337a7a..4f78b4521e4 100644 --- a/reflex/compiler/page_cache.py +++ b/reflex/compiler/page_cache.py @@ -1,12 +1,11 @@ -"""Experimental incremental compile cache (flag-gated). +"""Per-page dependency graph for the incremental compile cache (flag-gated). Enabled by ``REFLEX_COMPILE_CACHE``. When off (the default), the compiler behaves exactly as before. -**In-process per-page cache, with a Salsa-style dependency graph.** Each page -records the *exact set of source files it actually read*, so a change -invalidates only the pages that depend on it (not all pages). A page's -dependency set is the union of: +**A Salsa-style dependency graph.** Each page records the *exact set of source +files it actually read*, so a change invalidates only the pages that depend on it +(not all pages). A page's dependency set is the union of: - the **transitive first-party ``.py`` import closure** of its defining module (``page_py_dependencies`` — captures function-based views and shared helpers @@ -20,20 +19,20 @@ imported), - the **fine-grained state files** it references (``used_state_files``). -A page is reused iff every file in its dependency set is byte-unchanged and the -small genuinely-global ``global_epoch`` (Reflex version + ``rxconfig`` + lockfile) -is unchanged; adding/removing a route is handled separately (it changes shared -nav). Per-page dependency sets also track files *outside* the project root (e.g. -the docs site reads markdown from a sibling directory), so an external-source -edit still invalidates exactly the dependent pages. Cached pages' contributions -are re-merged into the app-wide aggregates by ``compile_app``; the store holds -live objects (``PageContext``), so it is in-process only. The cross-process -counterpart that survives a hot-reload worker respawn lives in -``reflex/compiler/disk_cache.py``. ``REFLEX_COMPILE_CACHE_VERIFY`` proves the -cache-assisted output matches a full compile and falls back on any mismatch — -the backstop for the residual gaps the static graph cannot see: runtime -``importlib`` imports and data files read at *module-import* time (outside the -per-page eval window). +A page is unchanged iff every file in its dependency set is byte-unchanged and +the small genuinely-global ``global_epoch`` (Reflex version + ``rxconfig`` + +lockfile) is unchanged; adding/removing a route is handled separately (it changes +shared nav). Per-page dependency sets also track files *outside* the project root +(e.g. the docs site reads markdown from a sibling directory), so an +external-source edit still invalidates exactly the dependent pages. These content +hashes and dependency sets are what ``reflex/compiler/disk_cache.py`` persists and +compares each compile to recompile only the pages whose source changed. + +Two residual gaps the static graph cannot see: runtime ``importlib`` imports and +data files read at *module-import* time (outside the per-page eval window). An +edit reached only through one of those would not invalidate its page; such +patterns are rare in page modules, and a global/version change still forces a +full recompile. """ from __future__ import annotations @@ -643,84 +642,3 @@ def deps_unchanged( True iff every dependency file is byte-unchanged. """ return all(hasher(path) == digest for path, digest in dep_hashes.items()) - - -def page_source_fingerprint(component: BaseComponent | object) -> str: - """Fingerprint a page from its own module source (no build required). - - Args: - component: The page component or factory callable. - - Returns: - A hex digest of the page's defining module source. - """ - import inspect - import sys - - module_name = getattr(component, "__module__", None) - parts = [repr(getattr(component, "__qualname__", repr(component)))] - mod = sys.modules.get(module_name) if module_name else None - file = getattr(mod, "__file__", None) - if file and Path(file).exists(): - with contextlib.suppress(OSError): - parts.append(hashlib.sha256(Path(file).read_bytes()).hexdigest()) - elif callable(component): - with contextlib.suppress(OSError, TypeError): - parts.append(inspect.getsource(component)) - return _sha(*parts) - - -#: route -> (global_epoch, dep_hashes, PageContext, is_stateful) -_PAGE_STORE: dict[str, tuple[str, dict[str, str], PageContext, bool]] = {} - - -def validate_page( - route: str, - epoch: str, - hasher: Callable[[str], str | None], -) -> tuple[PageContext, bool] | None: - """Return the cached page iff its dependency set and the global epoch match. - - A page is valid when the genuinely-global epoch is unchanged and every file - in its recorded dependency set is byte-unchanged. Editing a file invalidates - exactly the pages whose dependency set contains it. - - Args: - route: The page route. - epoch: The current global epoch (see :func:`global_epoch`). - hasher: A memoized path -> content-hash function (see :func:`make_hasher`). - - Returns: - ``(PageContext, is_stateful)`` on a valid hit, else None. - """ - entry = _PAGE_STORE.get(route) - if entry is None: - return None - stored_epoch, dep_hashes, page_ctx, is_stateful = entry - if stored_epoch != epoch or not deps_unchanged(dep_hashes, hasher): - return None - return page_ctx, is_stateful - - -def store_page( - route: str, - epoch: str, - dep_hashes: dict[str, str], - page_ctx: PageContext, - is_stateful: bool, -) -> None: - """Record a freshly-compiled page with the dependency set it relies on. - - Args: - route: The page route. - epoch: The global epoch it compiled under. - dep_hashes: ``{path: hash}`` for the page's dependency set. - page_ctx: The compiled PageContext to cache. - is_stateful: Whether the page registered new state during evaluation. - """ - _PAGE_STORE[route] = (epoch, dep_hashes, page_ctx, is_stateful) - - -def clear_page_store() -> None: - """Drop all in-process per-page cache entries.""" - _PAGE_STORE.clear() diff --git a/reflex/utils/compile_daemon.py b/reflex/utils/compile_daemon.py index 6c5d46d097a..6d18638a3fb 100644 --- a/reflex/utils/compile_daemon.py +++ b/reflex/utils/compile_daemon.py @@ -328,11 +328,10 @@ def _reset_first_party(roots: list[Path]) -> None: ): cached.cache_clear() DECORATED_PAGES.clear() - # In-process caches hold live objects from the previous import; drop them so - # the fresh compile can't reuse a tree built from now-stale classes. Cross- - # compile reuse comes from the on-disk manifest, not these. + # The import graph caches each module's parsed import edges; a changed file + # may import differently now, so drop it to force a re-parse. Cross-compile + # page reuse comes from the on-disk manifest. page_cache.clear_import_graph() - page_cache.clear_page_store() _reset_model_metadata() diff --git a/tests/units/compiler/test_disk_cache.py b/tests/units/compiler/test_disk_cache.py index ebd54be385e..660052df758 100644 --- a/tests/units/compiler/test_disk_cache.py +++ b/tests/units/compiler/test_disk_cache.py @@ -131,28 +131,27 @@ def test_write_and_load_manifest(tmp_path, monkeypatch): assert manifest is not None assert manifest["schema"] == disk_cache._SCHEMA assert set(manifest["pages"]) == {"/a", "/b", "/c"} - # the cached output is exactly what the compile produced for each page for route in ("/a", "/b", "/c"): - assert ( - manifest["pages"][route]["output_code"] - == ctx.compiled_pages[route].output_code - ) + entry = manifest["pages"][route] + # the manifest is pure bookkeeping: dep set + app-wrap keys + stateful flag + assert set(entry) == {"dep_hashes", "app_wrap_keys", "is_stateful"} # these static pages register no new state - assert manifest["pages"][route]["is_stateful"] is False - # imports round-trip cleanly - restored = disk_cache._deserialize_imports( - manifest["pages"]["/a"]["frontend_imports"] - ) - assert restored == ctx.compiled_pages["/a"].frontend_imports + assert entry["is_stateful"] is False + # rendered output is never persisted (it already lives in .web, and is + # never read back from the manifest) -> keeps the manifest small + assert "output_code" not in entry + assert "frontend_imports" not in entry + # the app-wide merged imports round-trip cleanly + restored = disk_cache._deserialize_imports(manifest["all_imports"]) + assert restored == ctx.all_imports def test_unchanged_pages_compile_identically(tmp_path, monkeypatch): """The reuse correctness property: an unchanged page recompiles byte-for-byte. - The disk cache reuses a hit page's stored ``output_code`` verbatim, so it is - correct iff a fresh compile of that page yields identical output. Compile A, - B, C; then compile A, B(edited), C; A and C must be byte-identical, and the - manifest's cached A/C output must equal the fresh recompile. + The disk cache leaves a hit page's already-on-disk ``.web`` file untouched, so + reuse is correct iff a fresh compile of that page yields identical output. + Compile A, B, C; then compile A, B(edited), C; A and C must be byte-identical. """ web = tmp_path / ".web" web.mkdir() @@ -164,9 +163,6 @@ def test_unchanged_pages_compile_identically(tmp_path, monkeypatch): _FakePage(route="/c", component=_page_c), ] ctx1 = _compile(pages) - disk_cache.write_manifest(ctx1, pages, ctx1.all_imports, root=tmp_path) - manifest = disk_cache.load_manifest() - assert manifest is not None # "Edit" page B; recompile the whole app cleanly. pages_edited = [ @@ -187,13 +183,6 @@ def test_unchanged_pages_compile_identically(tmp_path, monkeypatch): assert ( ctx2.compiled_pages["/b"].output_code != ctx1.compiled_pages["/b"].output_code ) - # The cached output we'd reuse for A/C equals a clean recompile of them. - assert ( - manifest["pages"]["/a"]["output_code"] == ctx2.compiled_pages["/a"].output_code - ) - assert ( - manifest["pages"]["/c"]["output_code"] == ctx2.compiled_pages["/c"].output_code - ) def _stub_externals(app, monkeypatch): diff --git a/tests/units/compiler/test_page_cache.py b/tests/units/compiler/test_page_cache.py index 0750224390c..62250c34a25 100644 --- a/tests/units/compiler/test_page_cache.py +++ b/tests/units/compiler/test_page_cache.py @@ -1,4 +1,4 @@ -"""Tests for the experimental incremental compile cache (in-process page cache).""" +"""Tests for the per-page dependency graph used by the incremental compile cache.""" from reflex.compiler import page_cache @@ -44,72 +44,3 @@ def test_used_state_files_from_output_and_memos(tmp_path): # un-introspectable memo -> conservative (all fine files) boom = SimpleNamespace(render=lambda: (_ for _ in ()).throw(RuntimeError())) assert page_cache.used_state_files(out, [boom], id_to_file) == {sfile, mfile} - - -def test_validate_page_fine_grained_deps(): - page_cache.clear_page_store() - ctx = object() - dep = "/proj/state.py" - # page depends on file `dep` at content hash H1, under global epoch "e1" - page_cache.store_page("/x", "e1", {dep: "H1"}, ctx, True) - # all deps unchanged + epoch matches -> hit - assert page_cache.validate_page("/x", "e1", lambda p: {dep: "H1"}.get(p)) == ( - ctx, - True, - ) - # a dependency file changed -> miss - assert page_cache.validate_page("/x", "e1", lambda p: {dep: "H2"}.get(p)) is None - # the genuinely-global epoch changed -> miss - assert page_cache.validate_page("/x", "e2", lambda p: {dep: "H1"}.get(p)) is None - - -def test_validate_page_with_no_deps_only_tracks_epoch(): - page_cache.clear_page_store() - ctx = object() - # page depends on NO files - page_cache.store_page("/x", "e1", {}, ctx, False) - # some unrelated file changed -> page is still a hit (it depends on nothing) - assert page_cache.validate_page("/x", "e1", lambda p: "Z") == (ctx, False) - # the global epoch changed -> miss - assert page_cache.validate_page("/x", "e2", lambda p: "Z") is None - # no stored entry -> miss - page_cache.clear_page_store() - assert page_cache.validate_page("/x", "e1", lambda p: None) is None - - -def _fake_ctx(pages, imports=None, memo=None, stateful=None, wraps=None): - from types import SimpleNamespace - - return SimpleNamespace( - compiled_pages={r: SimpleNamespace(output_code=c) for r, c in pages.items()}, - all_imports=imports or {}, - auto_memo_components=memo or {}, - stateful_routes=stateful or {}, - app_wrap_components=wraps or {}, - ) - - -def test_verify_diff_identical(): - from reflex.compiler import compiler - - a = _fake_ctx({"/": "CODE", "/x": "Y"}) - b = _fake_ctx({"/": "CODE", "/x": "Y"}) - assert compiler._diff_compile_contexts(a, b) == [] - - -def test_verify_diff_detects_page_change(): - from reflex.compiler import compiler - - a = _fake_ctx({"/": "OLD"}) - b = _fake_ctx({"/": "NEW"}) - assert "page:/" in compiler._diff_compile_contexts(a, b) - - -def test_verify_diff_detects_missing_route_and_memo(): - from reflex.compiler import compiler - - a = _fake_ctx({"/": "C"}, memo={("Memo", None): 1}) - b = _fake_ctx({"/": "C", "/x": "C"}, memo={}) - diffs = compiler._diff_compile_contexts(a, b) - assert any(d.startswith("routes:") for d in diffs) - assert "auto_memo_components" in diffs From 9f1b1c294d7bc53995336bb67fc5321db0b749e4 Mon Sep 17 00:00:00 2001 From: Farhan <www.mfarvirus@gmail.com> Date: Tue, 30 Jun 2026 00:28:20 +0500 Subject: [PATCH 06/56] refactor(compiler): drop unused page_cache helpers Remove page_module_files and file_hashes, now-dead after the compile-cache manifest rework, along with their tests. --- reflex/compiler/page_cache.py | 33 ------------------------- tests/units/compiler/test_page_cache.py | 9 ------- 2 files changed, 42 deletions(-) diff --git a/reflex/compiler/page_cache.py b/reflex/compiler/page_cache.py index 4f78b4521e4..2bd3da8b466 100644 --- a/reflex/compiler/page_cache.py +++ b/reflex/compiler/page_cache.py @@ -227,23 +227,6 @@ def _module_file(component: object) -> Path | None: return Path(file) if file else None -def page_module_files(components: object) -> set[Path]: - """Resolve the set of module files that define the given page components. - - Args: - components: An iterable of page component callables/objects. - - Returns: - The set of resolved module file paths. - """ - files = set() - for comp in components: # type: ignore[attr-defined] - f = _module_file(comp) - if f is not None: - files.add(f.resolve()) - return files - - def component_module_files( root_component: object, root: Path | None = None ) -> set[Path]: @@ -509,22 +492,6 @@ def under_root(comp: object) -> Path | None: return {i: f for i, f in id_to_file.items() if f in fine}, fine -def file_hashes(files: set[Path]) -> dict[str, str]: - """Map each file (as a string) to a hash of its current content. - - Args: - files: The files to hash. - - Returns: - A mapping of file path string to content hash. - """ - out: dict[str, str] = {} - for f in files: - with contextlib.suppress(OSError): - out[str(f)] = hashlib.sha256(f.read_bytes()).hexdigest() - return out - - def used_state_files( output_code: str, memo_components: object, diff --git a/tests/units/compiler/test_page_cache.py b/tests/units/compiler/test_page_cache.py index 62250c34a25..b6b33f5e3ef 100644 --- a/tests/units/compiler/test_page_cache.py +++ b/tests/units/compiler/test_page_cache.py @@ -15,15 +15,6 @@ def test_global_epoch_tracks_global_files(tmp_path): assert page_cache.global_epoch(root=tmp_path) != epoch -def _dummy_page(): # a page-like callable defined in this module - return None - - -def test_page_module_files_resolves(tmp_path): - files = page_cache.page_module_files([_dummy_page]) - assert any(p.name == "test_page_cache.py" for p in files) - - def test_used_state_files_from_output_and_memos(tmp_path): from types import SimpleNamespace From 43d5351f1a5c86c3b589a631eb29d9a16d29d148 Mon Sep 17 00:00:00 2001 From: Farhan <www.mfarvirus@gmail.com> Date: Tue, 30 Jun 2026 01:12:44 +0500 Subject: [PATCH 07/56] fix(compiler): track app-level config and memo imports in compile cache Two correctness fixes for the incremental compile cache, plus a pass to trim verbose comments and docstrings across the cache modules. - global_epoch now fingerprints the app-level config files: the app entrypoint module plus the config-only modules it imports (theme, app-wraps, stylesheets), found by walking the import graph with page modules as barriers so per-page incrementality is preserved. An edit to app-wide config previously left every page a hit, so the reused on-disk app root / contexts / theme stayed stale. - The incremental manifest refresh now persists the complete install import set (page imports merged with the memo-component imports the rebuild generated, and threading root through dependency discovery), so a later all-hit compile installs every package the reused memo files need instead of dropping newly-introduced memo packages. --- docs/app/reflex_docs/pages/docs/component.py | 4 +- .../src/reflex_base/plugins/compiler.py | 21 +- .../reflex-base/src/reflex_base/vars/base.py | 7 +- reflex/compiler/disk_cache.py | 143 ++++---- reflex/compiler/page_cache.py | 335 +++++++++++------- reflex/compiler/utils.py | 11 +- reflex/reflex.py | 7 +- reflex/utils/compile_daemon.py | 83 ++--- tests/units/compiler/test_disk_cache.py | 39 +- tests/units/compiler/test_page_cache.py | 195 ++++++++++ tests/units/utils/test_compile_daemon.py | 3 +- 11 files changed, 551 insertions(+), 297 deletions(-) diff --git a/docs/app/reflex_docs/pages/docs/component.py b/docs/app/reflex_docs/pages/docs/component.py index e2847d8fb5c..7d0310fae52 100644 --- a/docs/app/reflex_docs/pages/docs/component.py +++ b/docs/app/reflex_docs/pages/docs/component.py @@ -975,9 +975,7 @@ def links(current_page, ll_doc_exists, path): @docpage(set_path=path, t=title) def out(): - # Built here (page eval) rather than at module import, so importing the - # docs tree — and every hot-reload reimport — doesn't construct every - # component's prop tables up front; only a compiled page builds its own. + # Build prop docs during page eval so imports stay cheap. components = [ component_docs(component_tuple, previews) for component_tuple in component_list[1:] diff --git a/packages/reflex-base/src/reflex_base/plugins/compiler.py b/packages/reflex-base/src/reflex_base/plugins/compiler.py index dd02fabda08..97e4f0d9e36 100644 --- a/packages/reflex-base/src/reflex_base/plugins/compiler.py +++ b/packages/reflex-base/src/reflex_base/plugins/compiler.py @@ -36,12 +36,7 @@ _BaseComponentT = TypeVar("_BaseComponentT", bound=BaseComponent) -#: Optional per-page source-read recorder, installed by an incremental compile -#: cache. When set, :meth:`CompileContext.compile` wraps each page's evaluation -#: in ``page_source_recorder()``; the context manager yields a set that collects -#: the source files (e.g. markdown/data) read during that page's eval, which is -#: then stored on the page's ``source_files``. ``None`` (default) disables -#: recording, so the compile path is unchanged when no cache is active. +#: Optional recorder for source files read during each page evaluation. page_source_recorder: Callable[[], AbstractContextManager[set[str]]] | None = None @@ -700,15 +695,9 @@ class PageContext(BaseContext): output_path: str | None = None output_code: str | None = None source_module: str | None = None - # Source files (e.g. markdown/data) read while evaluating this page, when a - # per-page read recorder is installed (see ``page_source_recorder``). Lets an - # incremental cache depend on the exact non-import inputs a page consumed, so - # editing one page's data invalidates only that page. Empty when no recorder - # is active. + # Source files read while evaluating this page, when a recorder is installed. source_files: set[str] = dataclasses.field(default_factory=set) - # Auto-memo components first registered while compiling THIS page, keyed by - # ``(tag, source_module)``. Lets an incremental cache attribute memo - # contributions per page so a skipped page can re-register them. + # Auto-memo components first registered while compiling this page. memo_contributions: dict[tuple[str, str | None], Any] = dataclasses.field( default_factory=dict ) @@ -826,9 +815,7 @@ def compile( self.memoize_wrappers.clear() self.auto_memo_components.clear() - # Reset the deterministic ref-name generator so a second in-process - # compile reproduces the same auto-generated names as the first (these - # names feed auto-memo content hashes, so drift breaks reproducibility). + # Keep generated ref names stable across in-process compiles. reset_unique_variable_names() recorder = page_source_recorder diff --git a/packages/reflex-base/src/reflex_base/vars/base.py b/packages/reflex-base/src/reflex_base/vars/base.py index 15030b3e44a..ff33bd00edd 100644 --- a/packages/reflex-base/src/reflex_base/vars/base.py +++ b/packages/reflex-base/src/reflex_base/vars/base.py @@ -3217,11 +3217,8 @@ def get_uuid_string_var() -> Var: def reset_unique_variable_names() -> None: """Reset the deterministic unique-name generator to its initial state. - ``get_unique_variable_name`` draws from a seeded RNG and dedups against - ``USED_VARIABLES``; both persist process-wide, so a second in-process compile - would draw *different* ref names than the first (the RNG state and the used - set carry over). Resetting them before each compile makes the generated names - reproducible across compiles — names only need to be unique within a compile. + Names only need to be unique within one compile, so resetting before each + compile makes auto-generated ref names reproducible. """ USED_VARIABLES.clear() _UNIQUE_NAME_RNG.seed(42) diff --git a/reflex/compiler/disk_cache.py b/reflex/compiler/disk_cache.py index c69800ee1e7..b047afd0489 100644 --- a/reflex/compiler/disk_cache.py +++ b/reflex/compiler/disk_cache.py @@ -1,45 +1,9 @@ -"""Experimental disk-persisted incremental compile cache (flag-gated). - -Enabled by ``REFLEX_COMPILE_CACHE``. When off (default), nothing changes. - -In the ``reflex run`` edit loop every compile runs in a fresh process (the warm -daemon forks a throwaway child per change), so there is no in-memory state to -reuse across edits. This cache persists the small amount of bookkeeping a fresh -process needs to recompile only what changed and reuse everything else already -on disk. - -**What is persisted** (``.web/reflex_compile_cache.json``): per page, the -``{path: hash}`` of its **dependency set** (the exact source files it read — -own module, markdown/data, component modules in its tree, and the state files it -uses; see ``page_cache.page_dependency_hashes``), its app-wrap key-set, and -whether evaluating it registered new state. App-wide: the genuinely-global -``epoch`` (reflex version + config/lockfiles), the reflex version, and the merged -frontend imports. The manifest stores **no rendered output**: a page's compiled -``.js`` (and its memo files) already live in ``.web`` from the prior compile — a -hit reuses those files untouched, a miss rewrites its own — so the manifest is -pure bookkeeping, never a second render. (This is why it stays small: storing the -rendered ``output_code`` would balloon it for no gain, as it is never read back.) - -**The fast path** (``try_incremental_rebuild``). On a fresh compile it reuses the -manifest when the global inputs match (reflex version, route set, and the global -epoch). Each page is then a hit iff **every file in its recorded dependency set -is byte-unchanged** — so editing one markdown doc or one shared view recompiles -exactly the pages that depend on it, not all of them. The on-disk app-wide files -(app root, contexts, theme, stylesheet, …) stay valid because the epoch and route -set are unchanged. Then: - -- A hit page is left exactly as-is: its frontend ``.js`` is already on disk and - is neither rewritten nor re-evaluated. -- A *miss* page (source changed) is fully recompiled and its files rewritten. -- Stateful pages are recorded in the stateful-pages marker — misses from this - compile, hits from the manifest's ``is_stateful`` flag — so the serving backend - (which re-evaluates them to register their state) sees the complete set. This - process only writes ``.web`` and exits; it never re-evaluates a hit page. - -After recompiling misses, two guards must hold or the whole thing falls back to a -full compile (return False): each miss page's app-wrap key-set and stateful flag -must be unchanged (otherwise the reused on-disk app root would be wrong). A route -add/remove or a global/version change also falls back to a full compile. +"""Disk-persisted incremental compile cache for ``REFLEX_COMPILE_CACHE``. + +The manifest stores only bookkeeping: per-page dependency hashes, app-wrap keys, +statefulness, and app-wide frontend imports. Rendered files stay in ``.web``. +When global inputs and routes still match, unchanged pages are reused from disk +and only dependency-changed pages are recompiled. """ from __future__ import annotations @@ -59,7 +23,7 @@ from collections.abc import Callable, Sequence from pathlib import Path - from reflex_base.plugins import PageDefinition + from reflex_base.plugins import PageContext, PageDefinition from reflex_base.utils.imports import ParsedImportDict from reflex.app import App @@ -110,6 +74,37 @@ def _wrap_key_strs(keys: Any) -> list[str]: return sorted(f"{p}:{n}" for p, n in keys) +def _manifest_page_entry( + page_ctx: PageContext, + component: Any, + state_index: dict[str, Path], + hasher: Callable[[str], str | None], + *, + is_stateful: bool, + root: Path | None = None, +) -> dict[str, Any]: + """Build the manifest entry for one compiled page. + + Args: + page_ctx: The compiled page context. + component: The page component/callable used for dependency discovery. + state_index: The state-context identifier -> file index. + hasher: A memoized path -> content-hash function. + is_stateful: Whether the page registered state during compile. + root: Project root for dependency discovery. Defaults to cwd. + + Returns: + The JSON-able manifest entry for the page. + """ + return { + "dep_hashes": page_cache.page_dependency_hashes( + page_ctx, component, state_index, hasher, root + ), + "app_wrap_keys": _wrap_key_strs(page_ctx.app_wrap_components.keys()), + "is_stateful": is_stateful, + } + + def load_manifest() -> dict[str, Any] | None: """Load the persisted compile manifest, or None if absent/unusable. @@ -143,7 +138,7 @@ def write_manifest( compile_ctx: The completed compile context (all pages compiled). pages: The full list of page definitions that were compiled. install_imports: The **complete** frontend import set the full compile - installed — page imports merged with the app-root (app-wrap, e.g. + installed: page imports merged with the app-root (app-wrap, e.g. the Toaster/``sonner`` provider) and memo-component imports. An incremental rebuild reuses the on-disk app-wide files, so it must install from this complete set, not just the per-page union. @@ -152,7 +147,7 @@ def write_manifest( try: state_index, _ = page_cache.state_dependency_index(root) hasher = page_cache.make_hasher() - epoch = page_cache.global_epoch(root) + epoch = page_cache.global_epoch(root, pages=pages) pages_data: dict[str, Any] = {} for page in pages: @@ -163,13 +158,14 @@ def write_manifest( or page_ctx.output_path is None ): return # incomplete compile -> do not write a partial manifest - pages_data[page.route] = { - "dep_hashes": page_cache.page_dependency_hashes( - page_ctx, page.component, state_index, hasher, root - ), - "app_wrap_keys": _wrap_key_strs(page_ctx.app_wrap_components.keys()), - "is_stateful": page.route in compile_ctx.stateful_routes, - } + pages_data[page.route] = _manifest_page_entry( + page_ctx, + page.component, + state_index, + hasher, + is_stateful=page.route in compile_ctx.stateful_routes, + root=root, + ) manifest = { "schema": _SCHEMA, @@ -195,9 +191,11 @@ def globals_match( The fast rebuild needs the route set unchanged (adding/removing a route changes the shared nav on every page) and the global epoch unchanged (Reflex - version + config/lockfiles). Everything else is decided per page via its - dependency set, so a shared-component or markdown edit no longer blocks the - fast path — only the pages that actually depend on the changed file miss. + version + config/lockfiles + the app-level config files: the entrypoint and + the theme/app-wrap/stylesheet modules it imports, which configure the app-wide + files reused on disk). Everything else is decided per page via its dependency + set, so a shared-component or markdown edit no longer blocks the fast path. + Only the pages that depend on the changed file miss. Args: manifest: The loaded manifest. @@ -205,7 +203,7 @@ def globals_match( epoch: The current global epoch (see :func:`page_cache.global_epoch`). Returns: - True iff the global inputs match. + True if the global inputs match. """ return ( manifest.get("reflex_version") == page_cache._reflex_version() @@ -222,7 +220,7 @@ def partition_pages( """Return the pages whose dependency set changed since the manifest. Globals are assumed already matched (see :func:`globals_match`), so a page is - a hit iff every file in its recorded dependency set is byte-unchanged. + a hit if every file in its recorded dependency set is byte-unchanged. Args: pages: The current page definitions. @@ -278,7 +276,7 @@ def try_incremental_rebuild( pages = list(app._unevaluated_pages.values()) routes = {p.route for p in pages} hasher = page_cache.make_hasher() - epoch = page_cache.global_epoch(root) + epoch = page_cache.global_epoch(root, pages=pages) if not globals_match(manifest, routes=routes, epoch=epoch): return False @@ -370,8 +368,8 @@ def try_incremental_rebuild( install_imports, page_ctx.frontend_imports, memo_imports ) - # Record which routes are stateful — miss pages from this compile, hit pages - # from the manifest — so the stateful-pages marker is complete. We do NOT + # Record which routes are stateful: miss pages from this compile, hit pages + # from the manifest, so the stateful-pages marker is complete. We do NOT # re-evaluate hit pages to register their state in this process: it only # produces .web and exits (the daemon, the initial compile, and CLI compiles # never serve), and the serving backend re-evaluates the marked stateful @@ -399,7 +397,7 @@ def try_incremental_rebuild( frontend_skeleton.update_entry_client() # Refresh the manifest for the next process. - _update_manifest_for_misses(manifest, miss_ctx, miss_pages) + _update_manifest_for_misses(manifest, miss_ctx, miss_pages, install_imports, root) return True @@ -408,6 +406,8 @@ def _update_manifest_for_misses( manifest: dict[str, Any], miss_ctx: CompileContext | None, miss_pages: Sequence[PageDefinition], + all_imports: ParsedImportDict, + root: Path | None = None, ) -> None: """Update the on-disk manifest entries for the recompiled pages. @@ -415,23 +415,24 @@ def _update_manifest_for_misses( manifest: The loaded manifest (mutated and rewritten). miss_ctx: The compile context of the recompiled pages, if any. miss_pages: The recompiled page definitions. + all_imports: The complete frontend import set after recompiling misses. + root: Project root for dependency discovery. Defaults to cwd. """ if miss_ctx is None or not miss_pages: return try: - state_index, _ = page_cache.state_dependency_index() + state_index, _ = page_cache.state_dependency_index(root) hasher = page_cache.make_hasher() - all_imports = _deserialize_imports(manifest["all_imports"]) for page in miss_pages: page_ctx = miss_ctx.compiled_pages[page.route] - manifest["pages"][page.route] = { - "dep_hashes": page_cache.page_dependency_hashes( - page_ctx, page.component, state_index, hasher - ), - "app_wrap_keys": _wrap_key_strs(page_ctx.app_wrap_components.keys()), - "is_stateful": page.route in miss_ctx.stateful_routes, - } - all_imports = merge_imports(all_imports, page_ctx.frontend_imports) + manifest["pages"][page.route] = _manifest_page_entry( + page_ctx, + page.component, + state_index, + hasher, + is_stateful=page.route in miss_ctx.stateful_routes, + root=root, + ) manifest["all_imports"] = _serialize_imports(all_imports) _manifest_path().write_text(json.dumps(manifest), encoding="utf-8") except Exception as exc: # best-effort diff --git a/reflex/compiler/page_cache.py b/reflex/compiler/page_cache.py index 2bd3da8b466..e9fdf234631 100644 --- a/reflex/compiler/page_cache.py +++ b/reflex/compiler/page_cache.py @@ -1,38 +1,9 @@ -"""Per-page dependency graph for the incremental compile cache (flag-gated). - -Enabled by ``REFLEX_COMPILE_CACHE``. When off (the default), the compiler -behaves exactly as before. - -**A Salsa-style dependency graph.** Each page records the *exact set of source -files it actually read*, so a change invalidates only the pages that depend on it -(not all pages). A page's dependency set is the union of: - -- the **transitive first-party ``.py`` import closure** of its defining module - (``page_py_dependencies`` — captures function-based views and shared helpers - that never appear as nodes in the rendered tree, e.g. a ``def hero()`` view, so - editing one invalidates exactly the pages whose closure imports it), -- the **source files read while evaluating it** (markdown/data — captured by - ``record_reads`` via the per-page read recorder; this is what lets editing one - ``.md`` doc page recompile only that page), -- the **component modules in its rendered tree** (``component_module_files`` — - belt-and-suspenders for components injected at runtime rather than statically - imported), -- the **fine-grained state files** it references (``used_state_files``). - -A page is unchanged iff every file in its dependency set is byte-unchanged and -the small genuinely-global ``global_epoch`` (Reflex version + ``rxconfig`` + -lockfile) is unchanged; adding/removing a route is handled separately (it changes -shared nav). Per-page dependency sets also track files *outside* the project root -(e.g. the docs site reads markdown from a sibling directory), so an -external-source edit still invalidates exactly the dependent pages. These content -hashes and dependency sets are what ``reflex/compiler/disk_cache.py`` persists and -compares each compile to recompile only the pages whose source changed. - -Two residual gaps the static graph cannot see: runtime ``importlib`` imports and -data files read at *module-import* time (outside the per-page eval window). An -edit reached only through one of those would not invalidate its page; such -patterns are rare in page modules, and a global/version change still forces a -full recompile. +"""Per-page dependency discovery for the incremental compile cache. + +Each page records the first-party Python import closure of its page callable, +files read during page evaluation, component modules in its rendered tree, and +fine-grained state files it references. ``disk_cache`` persists hashes of that +set so only pages whose dependencies changed need to be recompiled. """ from __future__ import annotations @@ -41,7 +12,7 @@ import contextlib import hashlib import re -from collections.abc import Callable +from collections.abc import Callable, Sequence from contextvars import ContextVar from importlib import metadata from pathlib import Path @@ -77,13 +48,9 @@ def _reflex_version() -> str: return "unknown" -# --- Per-page read tracking (the Salsa "input read" seam) ------------------- # A page's markdown/data dependencies are read lazily while the page is -# evaluated (e.g. ``Path(doc).read_text()`` inside the page callable). We record -# those reads per page so the cache depends on the exact files consumed. Patches -# are installed once (idempotent) and only record while a recording set is active -# on the current task (set by ``record_reads``), so the overhead is a contextvar -# read when no cache is running. +# evaluated. Track those reads only while ``record_reads`` is active, so the +# idle overhead is one contextvar read. _active_reads: ContextVar[set[str] | None] = ContextVar("_active_reads", default=None) _patched = False @@ -188,15 +155,54 @@ def record_reads(): _active_reads.reset(token) -def global_epoch(root: Path | None = None) -> str: - """Fingerprint the genuinely-global inputs (Reflex version + config/lockfiles). +def _app_entrypoint_file(root: Path | None = None) -> Path | None: + """Resolve the user's app entrypoint module file (where ``rx.App`` is built). + + App-wide inputs like theme, app wraps, the toaster, stylesheets, and head + components are configured here, not in any single page's + dependency set. This is the root of :func:`app_dependency_files`, which walks + its imports (stopping at page modules) to find the config-only modules whose + change must invalidate the reused on-disk app-wide files. + + Args: + root: Project root; only a file under it is returned. Defaults to cwd. + + Returns: + The resolved entrypoint file path under ``root``, or None if it can't be + determined (no app module imported, or it lives outside ``root``). + """ + import sys + + try: + from reflex.config import get_config + + module = get_config().module + except Exception: + return None + file = getattr(sys.modules.get(module), "__file__", None) + if not file: + return None + rf = Path(file).resolve() + root = (root or Path.cwd()).resolve() + return rf if root in rf.parents else None + + +def global_epoch( + root: Path | None = None, *, pages: Sequence[object] | None = None +) -> str: + """Fingerprint the genuinely-global inputs. These can affect every page's output but belong to no single page, so they - gate the whole cache rather than any one page's dependency set. Kept small on - purpose — per-file edits flow through per-page dependency sets instead. + gate the whole cache rather than any one page's dependency set: the Reflex + version, the config/lockfiles, and the app-level config files: the app + entrypoint module plus the config-only modules it imports (theme, app-wraps, + stylesheets, head components; see :func:`app_dependency_files`). Kept small + on purpose; per-file edits flow through per-page dependency sets instead. Args: root: Project root. Defaults to cwd. + pages: The current page definitions, used as barriers so page modules + (tracked per page) are excluded from the app-level config files. Returns: A hex digest of the global inputs. @@ -209,16 +215,16 @@ def global_epoch(root: Path | None = None) -> str: parts.append(f"{name}={hashlib.sha256(path.read_bytes()).hexdigest()}") except OSError: parts.append(f"{name}=<absent>") + # Sorted for a deterministic digest regardless of set iteration order. + for path_str in sorted(app_dependency_files(pages, root)): + try: + digest = hashlib.sha256(Path(path_str).read_bytes()).hexdigest() + except OSError: + digest = "<absent>" + parts.append(f"app:{path_str}={digest}") return _sha(*parts) -# In-process per-page cache. Each page is keyed by the genuinely-global epoch -# plus the content hashes of its exact dependency set, so editing one file -# misses only the pages that depend on it. Contributions to the app-wide -# aggregates include live Python objects (root_component, memo defs), so the -# store is in-process only. - - def _module_file(component: object) -> Path | None: import sys @@ -232,10 +238,9 @@ def component_module_files( ) -> set[Path]: """Resolve the first-party module files of every component in a tree. - Walks the rendered component tree and collects the defining module file of - each component class under ``root``. This is the precise, barrel-immune way - (vs. static imports) to capture which shared views/templates a page renders: - editing one invalidates exactly the pages whose tree contains it. + Walks the rendered component tree and collects each component class module + under ``root``. This catches component dependencies that static imports may + miss. Args: root_component: The page's root component (its rendered tree). @@ -276,12 +281,36 @@ def _resolve_module_file(name: str) -> str | None: return str(Path(file).resolve()) if file else None +def _loaded_first_party_modules(root: Path) -> dict[str, str]: + """Map first-party module files to module names. + + Args: + root: Resolved project root. + + Returns: + A mapping of resolved file path -> module name for loaded modules under + ``root``. + """ + import sys + + file_to_mod: dict[str, str] = {} + for name, mod in list(sys.modules.items()): + file = getattr(mod, "__file__", None) + if not file: + continue + rf = Path(file).resolve() + if root in rf.parents: + file_to_mod[str(rf)] = name + return file_to_mod + + def _import_from_targets(node: object, modname: str) -> list[str]: """Resolve a ``from ... import ...`` node to candidate module names. Handles relative imports via the importing module's package. Returns the - base module and each ``base.name`` (a name may be a submodule or an attribute - — both candidates are resolved against ``sys.modules`` by the caller). + base module and each ``base.name``. A name may be a submodule or an + attribute; both candidates are resolved against ``sys.modules`` by the + caller. Args: node: An ``ast.ImportFrom`` node. @@ -304,16 +333,45 @@ def _import_from_targets(node: object, modname: str) -> list[str]: return [base, *(f"{base}.{a.name}" for a in node.names)] +def _module_import_edges( + file: str, modname: str, file_to_mod: dict[str, str] +) -> set[str]: + """Return first-party files imported by one module. + + Args: + file: The resolved module file path. + modname: The module's dotted name. + file_to_mod: Loaded first-party module files. + + Returns: + The resolved first-party files imported by ``file``. + """ + import ast + + deps: set[str] = set() + try: + tree = ast.parse(Path(file).read_bytes()) + except (OSError, SyntaxError, ValueError): + return deps + for node in ast.walk(tree): + names: list[str] = [] + if isinstance(node, ast.Import): + names = [a.name for a in node.names] + elif isinstance(node, ast.ImportFrom): + names = _import_from_targets(node, modname) + for name in names: + target = _resolve_module_file(name) + if target is not None and target in file_to_mod: + deps.add(target) + return deps + + def build_import_graph(root: Path | None = None) -> dict[str, set[str]]: """Build the first-party import graph (file -> files it imports). - Parses every already-imported first-party module's source for ``import`` and - ``from`` statements and resolves them to files under ``root`` via - ``sys.modules``. Cached per root for the duration of the process. This is the - sound basis for per-page ``.py`` dependencies: a function (e.g. a view like - ``hero()``) can only affect a page if its module is transitively imported by - the page's module, so it appears in the page's import closure even though it - is never a node in the rendered tree. + Parses already-imported first-party modules and resolves their imports to + files under ``root`` via ``sys.modules``. Cached per root for the duration of + the process. Args: root: Project root. Defaults to cwd. @@ -321,42 +379,16 @@ def build_import_graph(root: Path | None = None) -> dict[str, set[str]]: Returns: A mapping of resolved file path -> the set of first-party files it imports. """ - import ast - import sys - root = (root or Path.cwd()).resolve() cached = _import_graph_cache.get(root) if cached is not None: return cached - file_to_mod: dict[str, str] = {} - for name, mod in list(sys.modules.items()): - file = getattr(mod, "__file__", None) - if not file: - continue - rf = Path(file).resolve() - if root in rf.parents: - file_to_mod[str(rf)] = name - - graph: dict[str, set[str]] = {} - for file, modname in file_to_mod.items(): - deps: set[str] = set() - try: - tree = ast.parse(Path(file).read_bytes()) - except (OSError, SyntaxError, ValueError): - graph[file] = deps - continue - for node in ast.walk(tree): - names: list[str] = [] - if isinstance(node, ast.Import): - names = [a.name for a in node.names] - elif isinstance(node, ast.ImportFrom): - names = _import_from_targets(node, modname) - for n in names: - target = _resolve_module_file(n) - if target is not None and target in file_to_mod: - deps.add(target) - graph[file] = deps + file_to_mod = _loaded_first_party_modules(root) + graph = { + file: _module_import_edges(file, modname, file_to_mod) + for file, modname in file_to_mod.items() + } _import_graph_cache[root] = graph return graph @@ -366,16 +398,41 @@ def clear_import_graph() -> None: _import_graph_cache.clear() +def _component_source_files(component: object, root: Path) -> set[str]: + """The component callable's own defining files under ``root``. + + The callable's *real* code filename (``__code__``, correct even when + ``__module__`` was reassigned, as the docs app does for generated pages) plus + its module file. These are the roots a page's import closure walks from, and + the barriers the app-config walk stops at. + + Args: + component: The page component or callable. + root: Resolved project root; only files under it are returned. + + Returns: + The set of resolved defining file path strings under ``root``. + """ + out: set[str] = set() + code = getattr(component, "__code__", None) + filename = getattr(code, "co_filename", None) + own = _module_file(component) + for path in (filename, own): + if path: + rf = Path(path).resolve() + if root in rf.parents: + out.add(str(rf)) + return out + + def page_py_dependencies( component: BaseComponent | object, root: Path | None = None ) -> set[str]: """Return the transitive first-party ``.py`` files a page's code depends on. - Starts from the page callable's *real* defining file (``__code__`` filename, - which is correct even when ``__module__`` was reassigned, as the docs app does - for generated doc pages) plus its module file, and walks the import graph. - Captures function-based views and shared helpers that the rendered-tree walk - cannot see. + Starts from the page callable's code filename plus its module file, then + walks the import graph. This captures function-based views and shared + helpers that the rendered-tree walk cannot see. Args: component: The page component or callable. @@ -387,21 +444,8 @@ def page_py_dependencies( root = (root or Path.cwd()).resolve() graph = build_import_graph(root) - start: set[str] = set() - code = getattr(component, "__code__", None) - filename = getattr(code, "co_filename", None) - if filename: - rf = Path(filename).resolve() - if root in rf.parents: - start.add(str(rf)) - own = _module_file(component) - if own is not None: - rf = own.resolve() - if root in rf.parents: - start.add(str(rf)) - seen: set[str] = set() - stack = list(start) + stack = list(_component_source_files(component, root)) while stack: cur = stack.pop() if cur in seen: @@ -411,6 +455,57 @@ def page_py_dependencies( return seen +def app_dependency_files( + pages: Sequence[object] | None = None, root: Path | None = None +) -> set[str]: + """First-party files whose change affects app-level config (not any page). + + Walks the first-party import graph from the app entrypoint + (:func:`_app_entrypoint_file`), treating each page-defining module as a + barrier (not entered), so the result is the entrypoint plus the config-only + modules it imports, such as theme, app-wraps, stylesheets, head components, and + never page modules or their deep dependencies, which are tracked per page. A + config module shared with a page is still captured (it is reached from the + entrypoint directly, not through the barrier). + + These configure the app-wide files an incremental rebuild reuses on disk + (app root, contexts, theme, stylesheet), so they are folded into + :func:`global_epoch`: editing one forces a full recompile instead of leaving + those files stale. + + Args: + pages: The current page definitions, used as traversal barriers. When + None (no page set available), no barriers apply. + root: Project root. Defaults to cwd. + + Returns: + The set of resolved app-config dependency file path strings, or empty if + the entrypoint can't be resolved. + """ + root = (root or Path.cwd()).resolve() + entrypoint = _app_entrypoint_file(root) + if entrypoint is None: + return set() + file_to_mod = _loaded_first_party_modules(root) + barriers: set[str] = set() + for page in pages or (): + barriers |= _component_source_files(getattr(page, "component", None), root) + + start = str(entrypoint) + seen = {start} + stack = [start] + while stack: + cur = stack.pop() + modname = file_to_mod.get(cur) + if modname is None: + continue + for dep in _module_import_edges(cur, modname, file_to_mod): + if dep not in seen and dep not in barriers: + seen.add(dep) + stack.append(dep) + return seen + + def make_hasher() -> Callable[[str], str | None]: """Return a content-hasher that memoizes each path within one compile. @@ -500,12 +595,12 @@ def used_state_files( """Return the fine-grained state files a compiled page depends on. Stateful subtrees are auto-memoized into separate components, so a page's - own ``output_code`` may not name the state it uses — the state lives in the - memo components it *owns* (its ``memo_contributions``). Each stateful memo + own ``output_code`` may not name the state it uses; the state lives in the + memo components it owns (its ``memo_contributions``). Each stateful memo is owned by exactly one page, which regenerates it whenever it recompiles, so scanning ``output_code`` plus the page's own memo components captures the full dependency set. If a memo can't be introspected, depend on every fine - state file (conservative — never stale). + state file. Args: output_code: The page's compiled JS. @@ -606,6 +701,6 @@ def deps_unchanged( hasher: A memoized path -> content-hash function. Returns: - True iff every dependency file is byte-unchanged. + True if every dependency file is byte-unchanged. """ return all(hasher(path) == digest for path, digest in dep_hashes.items()) diff --git a/reflex/compiler/utils.py b/reflex/compiler/utils.py index 9c38dcb1149..bb768023fd5 100644 --- a/reflex/compiler/utils.py +++ b/reflex/compiler/utils.py @@ -801,10 +801,8 @@ def add_meta( children.append(Description.create(content=description)) children.append(Image.create(content=image)) - # Own-before-mutate: the page root may be a shared instance (the construction - # cache reuses identical static subtrees across pages), so build a fresh copy - # with the metadata appended instead of mutating ``children`` in place — - # otherwise repeated reuse accumulates duplicate <title>/<meta> tags. + # Page roots may be shared by the construction cache; copy before appending + # metadata so repeated reuse does not accumulate duplicate tags. new_page = copy.copy(page) new_page.children = [*page.children, *children, *meta_tags] new_page._clear_compile_caches() @@ -838,10 +836,7 @@ def write_file(path: str | Path, code: str): path.parent.mkdir(parents=True, exist_ok=True) if path.exists() and path.read_text(encoding="utf-8") == code: return - # Write atomically (temp file + os.replace) so a reader watching this tree — - # e.g. the vite dev server, or a concurrent compile — never observes a - # half-written file, even if the writing process is killed mid-write (the - # compile daemon forks throwaway children that may be terminated mid-compile). + # Write atomically so readers never observe a half-written file. tmp = path.with_name(f"{path.name}.{os.getpid()}.tmp") try: tmp.write_text(code, encoding="utf-8") diff --git a/reflex/reflex.py b/reflex/reflex.py index 61cc3c5415e..24389510982 100644 --- a/reflex/reflex.py +++ b/reflex/reflex.py @@ -207,11 +207,8 @@ def _run_dev( running_mode.has_backend(), )) - # When the compile cache is enabled, a persistent warm compile daemon owns - # .web regeneration (fork-per-compile, no per-edit cold import) instead of - # the backend worker recompiling on every respawn. The backend then only - # evaluates pages to register state — REFLEX_SKIP_COMPILE makes it skip the - # frontend write, and is inherited by every respawned reload worker. + # The compile daemon owns .web regeneration; backend reload workers only + # evaluate pages to register state. from reflex_base.environment import environment if running_mode.has_frontend() and environment.REFLEX_COMPILE_CACHE.get(): diff --git a/reflex/utils/compile_daemon.py b/reflex/utils/compile_daemon.py index 6d18638a3fb..8d1d20525db 100644 --- a/reflex/utils/compile_daemon.py +++ b/reflex/utils/compile_daemon.py @@ -1,39 +1,9 @@ -"""Persistent warm compile daemon for fast dev hot reloads (fork-per-compile). - -Active in ``reflex run`` dev when ``REFLEX_COMPILE_CACHE`` is set. Instead of the -granian/uvicorn reloader respawning a worker that cold-imports reflex + -reflex-base + the app's heavy deps (pandas/plotly/…) on every ``.py`` change, -this daemon imports everything **once** and stays warm. On each source change it -``fork()``s a throwaway child that: - -1. purges the user's first-party modules from ``sys.modules`` and resets the - cross-module state/page registries (so it is, for first-party code, a clean - interpreter — no stale ``__subclasses__`` zombies, no duplicate-substate - shadowing, no ``add_page`` re-registration error); -2. re-imports the user app fresh (third-party deps stay warm, inherited via - copy-on-write fork) and runs ``compiler.compile_app`` with the cache on, which - recompiles only the dependency-changed pages via - ``disk_cache.try_incremental_rebuild`` — the child *is* the fresh worker that - path was built for, but warm; -3. writes only the changed ``.web`` files (atomically) and ``os._exit``s. - -Because every compile runs in a child that is discarded, reload corruption can -never accumulate: correctness is the same as today's respawn, but the multi-second -cold import is paid once instead of on every edit. - -The daemon owns the file watcher, so it watches exactly what the compiler reads — -``.py``/``.md``/``.mdx`` under the app source roots **plus** every external/sibling -content file recorded in the compile manifest's per-page dependency sets (where a -docs app's markdown lives) plus ``rxconfig``/lockfiles/``assets`` — fixing the -long-standing gap where markdown edits (and sibling-dir reads) never triggered a -reload because the reloaders watch only ``*.py`` under the app dir. - -The watcher is a single-threaded poll loop on purpose: ``fork()`` from a -multi-threaded process is unsafe, so the daemon must hold no background threads. - -On Windows (no ``os.fork``) each compile runs in a fresh spawned subprocess -instead — correct, but without copy-on-write warmth (parity with today's latency); -the watcher/markdown fix applies identically. +"""Warm compile daemon for ``REFLEX_COMPILE_CACHE`` dev hot reloads. + +The daemon imports the app once, then compiles each change in an isolated child. +On POSIX it forks so third-party imports stay warm; otherwise it falls back to a +fresh subprocess. It also owns the watch loop so markdown and external content +dependencies recorded in the compile manifest trigger rebuilds. """ from __future__ import annotations @@ -58,7 +28,7 @@ #: After a change is seen, wait this long and re-snapshot so a burst of saves #: (e.g. format-on-save touching many files) collapses into a single compile. _DEBOUNCE = 0.05 -#: How often to re-walk the tree (``rglob``) to discover added/removed files. +#: How often to re-walk the tree to discover added/removed files. #: Between rescans we only ``stat`` the known set; adds are rarer and tolerate a #: little latency, so this keeps idle CPU low while polling fast for edits. _RESCAN_INTERVAL = 1.0 @@ -149,14 +119,17 @@ def _iter_source_files(root: Path): if root.suffix in _WATCH_SUFFIXES: yield root.resolve() return - for path in root.rglob("*"): - if path.is_dir(): - continue - rel_parts = path.relative_to(root).parts[:-1] - if any(part in _SKIP_DIRS or part.startswith(".") for part in rel_parts): - continue - if path.suffix in _WATCH_SUFFIXES: - yield path.resolve() + for dirpath, dirnames, filenames in os.walk(root): + dirnames[:] = [ + name + for name in dirnames + if name not in _SKIP_DIRS and not name.startswith(".") + ] + base = Path(dirpath) + for name in filenames: + path = base / name + if path.suffix in _WATCH_SUFFIXES: + yield path.resolve() def _external_dependency_files(roots: list[Path]) -> set[Path]: @@ -164,7 +137,7 @@ def _external_dependency_files(roots: list[Path]) -> set[Path]: The manifest records each page's full dependency set (own module, markdown, component/state modules). Any dependency that lives *outside* the reload - roots — e.g. a docs app's markdown in a sibling directory — is invisible to + roots, such as a docs app's markdown in a sibling directory, is invisible to ``get_reload_paths`` and must be watched explicitly so editing it rebuilds. Args: @@ -261,7 +234,7 @@ def _first_party_module_names(roots: list[Path]) -> set[str]: """Names of all loaded modules belonging to the user's first-party packages. First-party top-level package names are inferred from the *regular* modules - whose ``__file__`` resolves under a reload root (a plain attribute read — no + whose ``__file__`` resolves under a reload root (a plain attribute read, no namespace-package ``__path__`` recalculation, which is lazy and would break while ``sys.modules`` is being mutated). Every loaded module sharing one of those top-level names is then first-party, which captures namespace packages @@ -293,10 +266,8 @@ def _reset_first_party(roots: list[Path]) -> None: """Make this interpreter clean w.r.t. first-party code before re-importing. Purges the user's first-party modules from ``sys.modules`` and clears the - cross-module registries/caches that would otherwise pin the old class - objects (the ``__subclasses__`` zombie / duplicate-substate / stale-page - hazards). Third-party modules (reflex, reflex-base, pandas, …) are left - imported and warm. + cross-module registries/caches that would otherwise pin old class objects. + Third-party modules are left imported and warm. Args: roots: The resolved reload roots whose modules are first-party. @@ -365,7 +336,7 @@ def _child_compile(roots: list[Path], prerender_routes: bool) -> None: """Reset first-party state, re-import the app fresh, and compile incrementally. Runs in a forked child (POSIX) or a one-shot subprocess (Windows). Must not - return normally on error — the caller maps the exit code to success/failure. + return normally on error; the caller maps the exit code to success/failure. Args: roots: The resolved reload roots. @@ -396,7 +367,7 @@ def _await_child(pid: int) -> bool: Returns: True if it exited 0; False on failure, timeout, or signal (a - signal-killed child — e.g. Ctrl-C during shutdown — is a quiet False). + signal-killed child, such as Ctrl-C during shutdown, is a quiet False). """ deadline = time.monotonic() + _COMPILE_TIMEOUT while True: @@ -417,7 +388,7 @@ def _can_fork() -> bool: """Whether forking is safe right now (POSIX and the process is single-threaded). Forking a multi-threaded process and then running Python (not exec) inherits - locks held by threads that don't exist in the child — a classic deadlock. The + locks held by threads that don't exist in the child. The user app, imported warm in the parent, may have started a background thread at import time, so this is checked per compile. @@ -479,7 +450,7 @@ def _serve() -> None: # Warm import + initial compile (writes .web + the manifest); keeps the app # and its third-party deps resident for copy-on-write children. A failure - # here (e.g. the app is mid-edit and broken) must NOT kill the daemon — fall + # here (e.g. the app is mid-edit and broken) must NOT kill the daemon; fall # through to the watch loop so the next edit that fixes it recompiles. try: with console.timing("Compile daemon: initial compile"): @@ -499,7 +470,7 @@ def _serve() -> None: # the manifest; recompute only after a compile, not on every poll tick. external = _external_dependency_files(roots) global_files = _global_files(root) - # `paths` is the watched set, refreshed by an rglob rescan; each tick only + # `paths` is the watched set, refreshed by a tree rescan; each tick only # re-stats it (cheap), so the poll can be fast without burning idle CPU. paths = _watch_paths(roots, root, external) snapshot = _snapshot(paths) diff --git a/tests/units/compiler/test_disk_cache.py b/tests/units/compiler/test_disk_cache.py index 660052df758..aa9c4b80b24 100644 --- a/tests/units/compiler/test_disk_cache.py +++ b/tests/units/compiler/test_disk_cache.py @@ -1,8 +1,10 @@ """Tests for the experimental disk-persisted incremental compile cache.""" import dataclasses +import json from collections.abc import Callable, Sequence -from typing import Any +from types import SimpleNamespace +from typing import Any, cast from reflex_base.components.component import Component from reflex_base.plugins import CompileContext, CompilerHooks @@ -150,7 +152,7 @@ def test_unchanged_pages_compile_identically(tmp_path, monkeypatch): """The reuse correctness property: an unchanged page recompiles byte-for-byte. The disk cache leaves a hit page's already-on-disk ``.web`` file untouched, so - reuse is correct iff a fresh compile of that page yields identical output. + reuse is correct if a fresh compile of that page yields identical output. Compile A, B, C; then compile A, B(edited), C; A and C must be byte-identical. """ web = tmp_path / ".web" @@ -220,8 +222,6 @@ def test_incremental_rebuild_all_hits(tmp_path, monkeypatch): def test_incremental_rebuild_one_miss_writes_only_that_page(tmp_path, monkeypatch): - import json - from reflex.compiler import utils as compiler_utils web = tmp_path / ".web" @@ -273,8 +273,6 @@ def test_stateful_hit_is_marked_but_not_reevaluated(tmp_path, monkeypatch): re-evaluates the marked stateful pages itself, so re-evaluating them during the incremental rebuild was pure waste. """ - import json - web = tmp_path / ".web" web.mkdir() monkeypatch.setattr(disk_cache.prerequisites, "get_web_dir", lambda: web) @@ -303,17 +301,38 @@ def test_stateful_hit_is_marked_but_not_reevaluated(tmp_path, monkeypatch): ) is True ) - # Not re-evaluated... assert reevaluated == [] - # ...but still recorded as stateful so the backend's marker is complete. assert route in app._stateful_pages def test_load_manifest_rejects_wrong_schema(tmp_path, monkeypatch): - import json - web = tmp_path / ".web" web.mkdir() monkeypatch.setattr(disk_cache.prerequisites, "get_web_dir", lambda: web) (web / disk_cache._MANIFEST_FILE).write_text(json.dumps({"schema": 999})) assert disk_cache.load_manifest() is None + + +def test_update_manifest_for_misses_keeps_complete_imports(tmp_path, monkeypatch): + web = tmp_path / ".web" + web.mkdir() + monkeypatch.setattr(disk_cache.prerequisites, "get_web_dir", lambda: web) + monkeypatch.setattr( + page_cache, "state_dependency_index", lambda root=None: ({}, set()) + ) + monkeypatch.setattr(page_cache, "page_dependency_hashes", lambda *a, **k: {}) + + page = _FakePage(route="/a", component=_page_a) + page_ctx = SimpleNamespace(app_wrap_components={}, frontend_imports={}) + miss_ctx = SimpleNamespace(compiled_pages={"/a": page_ctx}, stateful_routes=set()) + complete_imports = {"memo-lib": [ImportVar("MemoThing")]} + manifest = _manifest({ + "/a": {"dep_hashes": {}, "app_wrap_keys": [], "is_stateful": False} + }) + + disk_cache._update_manifest_for_misses( + manifest, cast(Any, miss_ctx), [page], complete_imports, root=tmp_path + ) + + written = json.loads((web / disk_cache._MANIFEST_FILE).read_text()) + assert disk_cache._deserialize_imports(written["all_imports"]) == complete_imports diff --git a/tests/units/compiler/test_page_cache.py b/tests/units/compiler/test_page_cache.py index b6b33f5e3ef..dcd908788d1 100644 --- a/tests/units/compiler/test_page_cache.py +++ b/tests/units/compiler/test_page_cache.py @@ -15,6 +15,201 @@ def test_global_epoch_tracks_global_files(tmp_path): assert page_cache.global_epoch(root=tmp_path) != epoch +def test_global_epoch_tracks_app_entrypoint(tmp_path, monkeypatch): + """Editing the app entrypoint (theme/app_wraps live there) bumps the epoch. + + App-wide config is configured where ``rx.App`` is built, not in any page's + dependency set, so without this an edit to it would leave every page a hit + and the reused on-disk app root / contexts / theme stale. + """ + (tmp_path / "rxconfig.py").write_text("config = 1\n") + entrypoint = (tmp_path / "myapp.py").resolve() + entrypoint.write_text("app = rx.App(theme=light)\n") + monkeypatch.setattr( + page_cache, "_app_entrypoint_file", lambda root=None: entrypoint + ) + epoch = page_cache.global_epoch(root=tmp_path) + # editing the app entrypoint DOES change the epoch + entrypoint.write_text("app = rx.App(theme=dark)\n") + assert page_cache.global_epoch(root=tmp_path) != epoch + + +def test_app_entrypoint_file_resolution(tmp_path, monkeypatch): + import sys + from types import ModuleType, SimpleNamespace + + monkeypatch.setattr( + "reflex.config.get_config", lambda: SimpleNamespace(module="fake_entry_mod") + ) + mod = ModuleType("fake_entry_mod") + monkeypatch.setitem(sys.modules, "fake_entry_mod", mod) + + # no __file__ on the module -> None + assert page_cache._app_entrypoint_file(root=tmp_path) is None + + # a file under the project root -> resolved + entry = tmp_path / "myapp" / "myapp.py" + entry.parent.mkdir() + entry.write_text("app = 1\n") + mod.__file__ = str(entry) + assert page_cache._app_entrypoint_file(root=tmp_path) == entry.resolve() + + # a file outside the project root -> None (not a project input) + outside = tmp_path.parent / "elsewhere.py" + mod.__file__ = str(outside) + assert page_cache._app_entrypoint_file(root=tmp_path) is None + + +def test_app_dependency_files_keeps_config_excludes_pages(tmp_path, monkeypatch): + """The entrypoint's config closure, with page modules as traversal barriers. + + entry imports a config-only ``theme.py`` and a page; the page imports a view + and (also) theme. The result must keep entry + theme (theme is reached from + the entrypoint directly, even though a page imports it too) and exclude the + page module and its view (tracked per page). + """ + from types import SimpleNamespace + + entry = tmp_path / "myapp.py" + theme = tmp_path / "theme.py" + page = tmp_path / "pages" / "index.py" + view = tmp_path / "components" / "hero.py" + graph = { + str(entry): {str(theme), str(page)}, + str(page): {str(view), str(theme)}, + str(theme): set(), + str(view): set(), + } + monkeypatch.setattr(page_cache, "_app_entrypoint_file", lambda root=None: entry) + monkeypatch.setattr( + page_cache, + "_loaded_first_party_modules", + lambda root: { + str(entry): "myapp", + str(theme): "theme", + str(page): "pages.index", + str(view): "components.hero", + }, + ) + monkeypatch.setattr( + page_cache, + "_module_import_edges", + lambda file, modname, file_to_mod: graph[file], + ) + monkeypatch.setattr( + page_cache, "_component_source_files", lambda comp, root: {str(page)} + ) + + result = page_cache.app_dependency_files( + [SimpleNamespace(component=object())], root=tmp_path + ) + assert result == {str(entry), str(theme)} + + +def test_global_epoch_excludes_page_modules(tmp_path, monkeypatch): + """A page-module edit keeps the epoch (incremental); an app-config edit bumps it.""" + from types import SimpleNamespace + + (tmp_path / "rxconfig.py").write_text("c = 1\n") + entry = tmp_path / "myapp.py" + entry.write_text("app = 1\n") + theme = tmp_path / "theme.py" + theme.write_text("t = 1\n") + page = tmp_path / "index.py" + page.write_text("p = 1\n") + graph = {str(entry): {str(theme), str(page)}, str(page): set(), str(theme): set()} + monkeypatch.setattr(page_cache, "_app_entrypoint_file", lambda root=None: entry) + monkeypatch.setattr( + page_cache, + "_loaded_first_party_modules", + lambda root: { + str(entry): "myapp", + str(theme): "theme", + str(page): "index", + }, + ) + monkeypatch.setattr( + page_cache, + "_module_import_edges", + lambda file, modname, file_to_mod: graph[file], + ) + monkeypatch.setattr( + page_cache, "_component_source_files", lambda comp, root: {str(page)} + ) + pages = [SimpleNamespace(component=object())] + + epoch = page_cache.global_epoch(root=tmp_path, pages=pages) + # editing a page module does NOT change the epoch (tracked per page instead) + page.write_text("p = 2\n") + assert page_cache.global_epoch(root=tmp_path, pages=pages) == epoch + # editing app-level config the entrypoint imports (theme) DOES + theme.write_text("t = 2\n") + assert page_cache.global_epoch(root=tmp_path, pages=pages) != epoch + + +def test_global_epoch_hashes_only_app_dependency_closure(tmp_path, monkeypatch): + """Unrelated first-party modules do not make the app epoch coarse.""" + from types import SimpleNamespace + + entry = tmp_path / "myapp.py" + theme = tmp_path / "theme.py" + unrelated = tmp_path / "unrelated.py" + for path, code in ( + (entry, "import theme\n"), + (theme, "t = 1\n"), + (unrelated, "x = 1\n"), + ): + path.write_text(code) + graph = {str(entry): {str(theme)}, str(theme): set(), str(unrelated): set()} + monkeypatch.setattr(page_cache, "_app_entrypoint_file", lambda root=None: entry) + monkeypatch.setattr( + page_cache, + "_loaded_first_party_modules", + lambda root: { + str(entry): "myapp", + str(theme): "theme", + str(unrelated): "unrelated", + }, + ) + monkeypatch.setattr( + page_cache, + "_module_import_edges", + lambda file, modname, file_to_mod: graph[file], + ) + + epoch = page_cache.global_epoch( + root=tmp_path, pages=[SimpleNamespace(component=object())] + ) + unrelated.write_text("x = 2\n") + assert ( + page_cache.global_epoch( + root=tmp_path, pages=[SimpleNamespace(component=object())] + ) + == epoch + ) + theme.write_text("t = 2\n") + assert ( + page_cache.global_epoch( + root=tmp_path, pages=[SimpleNamespace(component=object())] + ) + != epoch + ) + + +def test_app_dependency_files_skips_graph_without_entrypoint(tmp_path, monkeypatch): + monkeypatch.setattr(page_cache, "_app_entrypoint_file", lambda root=None: None) + + def fail_loaded_first_party_modules(root): + msg = "module map should not be built without an entrypoint" + raise AssertionError(msg) + + monkeypatch.setattr( + page_cache, "_loaded_first_party_modules", fail_loaded_first_party_modules + ) + + assert page_cache.app_dependency_files(root=tmp_path) == set() + + def test_used_state_files_from_output_and_memos(tmp_path): from types import SimpleNamespace diff --git a/tests/units/utils/test_compile_daemon.py b/tests/units/utils/test_compile_daemon.py index 9d7947a8236..cb2bccaf3b6 100644 --- a/tests/units/utils/test_compile_daemon.py +++ b/tests/units/utils/test_compile_daemon.py @@ -168,8 +168,7 @@ class DaemonResetUser(rx.Model, table=True): def test_reset_first_party_purges_modules_and_registries(tmp_path): """``_reset_first_party`` purges first-party modules and clears registries. - Runs in a forked child so the global-registry reset can't corrupt the test - process — exactly how the daemon uses it (a throwaway child per compile). + Runs in a forked child so the registry reset cannot affect the test process. """ mod_file = tmp_path / "fp_module.py" mod_file.write_text("VALUE = 1\n") From 19b2df47d8d565105afd075a728fb9eefa83439a Mon Sep 17 00:00:00 2001 From: Farhan <www.mfarvirus@gmail.com> Date: Tue, 30 Jun 2026 01:23:53 +0500 Subject: [PATCH 08/56] test(compiler): skip model-metadata daemon test without sqlmodel test_reset_model_metadata_allows_table_redefinition builds an rx.Model(table=True), which needs the SQLModel stack. The "without db dependencies" unit-test job uninstalls it, so the first model definition raised TypeError before the test could exercise the reset. Guard with pytest.importorskip("sqlmodel"), matching the repo's other db-dependent tests, so it skips there instead of failing. --- tests/units/utils/test_compile_daemon.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/units/utils/test_compile_daemon.py b/tests/units/utils/test_compile_daemon.py index cb2bccaf3b6..93e3e75e38a 100644 --- a/tests/units/utils/test_compile_daemon.py +++ b/tests/units/utils/test_compile_daemon.py @@ -129,6 +129,10 @@ def test_reset_model_metadata_allows_table_redefinition(): ``Table '...' is already defined``. Run in a fork so clearing the global metadata can't affect the test process. """ + # ``rx.Model(table=True)`` needs the SQLModel stack; the db-less unit-test + # job uninstalls it, so skip there rather than fail on the first definition. + pytest.importorskip("sqlmodel") + read_fd, write_fd = os.pipe() pid = os.fork() if pid == 0: # child From aac67b2d8a34b6c0d89d4d2a9da93449fb1cbfd8 Mon Sep 17 00:00:00 2001 From: Farhan <www.mfarvirus@gmail.com> Date: Tue, 30 Jun 2026 01:55:19 +0500 Subject: [PATCH 09/56] docs(changelog): add news fragments for the incremental compile cache Add the news fragments the changelog check requires for the packaged source touched by this PR: a feature entry for the reflex package's REFLEX_COMPILE_CACHE flag and a misc entry for the reflex-base recorder hook / reproducible ref names that support it. --- news/6688.feature.md | 1 + packages/reflex-base/news/6688.misc.md | 1 + 2 files changed, 2 insertions(+) create mode 100644 news/6688.feature.md create mode 100644 packages/reflex-base/news/6688.misc.md diff --git a/news/6688.feature.md b/news/6688.feature.md new file mode 100644 index 00000000000..5ae88d4e82b --- /dev/null +++ b/news/6688.feature.md @@ -0,0 +1 @@ +Added an experimental disk-persisted incremental compile cache, enabled by the `REFLEX_COMPILE_CACHE` environment variable. When on, a fresh compile reuses the previous build already on disk in `.web` and recompiles only the pages whose source changed, tracked via a per-page dependency graph (Python import closure, files read during page evaluation, component modules, and referenced state). App-wide inputs (Reflex version, config/lockfiles, and the app entrypoint's config modules such as theme/app-wraps/stylesheets) gate the whole cache, falling back to a full compile when they change. `reflex run` dev additionally gains a warm fork-per-compile daemon so hot reloads skip the cold reimport and rebuild only what changed. Off by default — the compile path is unchanged when the flag is unset. diff --git a/packages/reflex-base/news/6688.misc.md b/packages/reflex-base/news/6688.misc.md new file mode 100644 index 00000000000..d0cfb39144c --- /dev/null +++ b/packages/reflex-base/news/6688.misc.md @@ -0,0 +1 @@ +Added an optional per-page source-read recorder hook (`page_source_recorder` in the compiler plugin) used by the incremental compile cache to track the exact files each page reads during evaluation, and made auto-generated unique ref names reproducible across in-process compiles so memo content hashes stay stable. No behavior change unless the compile cache is enabled. From 519f885117f8b95325de7da0626fc01764085d01 Mon Sep 17 00:00:00 2001 From: Farhan <www.mfarvirus@gmail.com> Date: Tue, 30 Jun 2026 02:39:46 +0500 Subject: [PATCH 10/56] feat(compiler): track runtime module imports in compile cache Patch __import__ and importlib.import_module while a page recorder is active so modules imported during page eval are recorded as source dependencies. Resolves names (including relative imports), skips stdlib and out-of-project files, and caches module-file lookups. Lets the incremental compile cache invalidate pages that depend on first-party modules pulled in at import time. --- reflex/compiler/page_cache.py | 223 ++++++++++++++++++++++-- tests/units/compiler/test_page_cache.py | 110 ++++++++++++ 2 files changed, 323 insertions(+), 10 deletions(-) diff --git a/reflex/compiler/page_cache.py b/reflex/compiler/page_cache.py index e9fdf234631..058338bd38b 100644 --- a/reflex/compiler/page_cache.py +++ b/reflex/compiler/page_cache.py @@ -11,7 +11,11 @@ import builtins import contextlib import hashlib +import importlib +import importlib.util +import os import re +import sys from collections.abc import Callable, Sequence from contextvars import ContextVar from importlib import metadata @@ -55,6 +59,8 @@ def _reflex_version() -> str: _active_reads: ContextVar[set[str] | None] = ContextVar("_active_reads", default=None) _patched = False _recorder_root: Path | None = None +_recorder_root_str: str | None = None +_recorder_raw_root_str: str | None = None #: Path parts that mark a dependency/build location whose reads are never a #: page's own source dependency (a change there flows through the version/epoch). @@ -78,6 +84,14 @@ def _reflex_version() -> str: ".rst", } +_PYTHON_PREFIXES = tuple( + os.path.abspath(os.fsdecode(prefix)) # noqa: PTH100 + for prefix in {sys.base_exec_prefix, sys.base_prefix, sys.exec_prefix, sys.prefix} + if prefix +) +_MODULE_FILE_CACHE_MISSING: object = object() +_module_file_cache: dict[tuple[str, str], str | None] = {} + def _record_read(path: object) -> None: target = _active_reads.get() @@ -96,6 +110,168 @@ def _record_read(path: object) -> None: target.add(str(resolved)) +def _is_path_within(path: str, root: str) -> bool: + """Return whether ``path`` is contained within ``root`` without filesystem IO. + + Args: + path: Absolute path to test. + root: Absolute root path. + + Returns: + True when ``path`` is nested below ``root``. + """ + normalized_path = os.path.normcase(path) + normalized_root = os.path.normcase(root) + if normalized_path == normalized_root: + return False + try: + return os.path.commonpath((normalized_path, normalized_root)) == normalized_root + except ValueError: + return False + + +def _is_python_install_path(path: str) -> bool: + """Return whether ``path`` is under this interpreter's install roots. + + Args: + path: Absolute path to test. + + Returns: + True when ``path`` is under the interpreter or virtualenv prefix. + """ + return any(_is_path_within(path, prefix) for prefix in _PYTHON_PREFIXES) + + +def _recordable_module_file(file: object) -> str | None: + """Resolve a module file only when it can be a first-party dependency. + + Args: + file: The imported module's ``__file__`` value. + + Returns: + The resolved module file path to record, or None when it is outside the + project root or otherwise not recordable. + """ + root = _recorder_root + root_str = _recorder_root_str + if root is None or root_str is None: + return None + if not isinstance(file, (str, bytes, os.PathLike)): + return None + try: + raw_path = os.path.abspath(os.fsdecode(file)) # noqa: PTH100 + except (OSError, TypeError, ValueError): + return None + + cache_key = (root_str, raw_path) + cached = _module_file_cache.get(cache_key, _MODULE_FILE_CACHE_MISSING) + if cached is not _MODULE_FILE_CACHE_MISSING: + return cached if isinstance(cached, str) else None + + path = Path(raw_path) + under_project_path = _is_path_within(raw_path, root_str) or ( + _recorder_raw_root_str is not None + and _is_path_within(raw_path, _recorder_raw_root_str) + ) + resolved_str = None + if not any(part in _EXCLUDE_PARTS for part in path.parts) and ( + under_project_path or not _is_python_install_path(raw_path) + ): + try: + resolved = path.resolve() + except (OSError, TypeError, ValueError): + pass + else: + if not any(part in _EXCLUDE_PARTS for part in resolved.parts): + resolved_str = str(resolved) if root in resolved.parents else None + _module_file_cache[cache_key] = resolved_str + return resolved_str + + +def _record_module_file(module: object, target: set[str] | None = None) -> None: + """Record the source file for an imported module, if it has one. + + Args: + module: The imported module object. + target: The active read set. Defaults to the current recorder context. + """ + if target is None: + target = _active_reads.get() + if target is None: + return + file = getattr(module, "__file__", None) + if not file: + return + if resolved_file := _recordable_module_file(file): + target.add(resolved_file) + + +def _absolute_import_name( + name: str, globals_: dict[str, object] | None, level: int +) -> str: + """Resolve an import name relative to the caller package. + + Args: + name: The import name passed to ``__import__``. + globals_: The caller globals passed to ``__import__``. + level: The relative import level. + + Returns: + The absolute module name when it can be resolved, else ``name``. + """ + if not level: + return name + package = None + if globals_ is not None: + package = globals_.get("__package__") + if not package and (module := globals_.get("__name__")): + package = ( + module + if isinstance(module, str) and globals_.get("__path__") is not None + else module.rpartition(".")[0] + if isinstance(module, str) + else None + ) + if not isinstance(package, str): + return name + with contextlib.suppress(Exception): + return importlib.util.resolve_name(f"{'.' * level}{name}", package) + return name + + +def _record_imported_modules( + name: str, + result: object, + fromlist: Sequence[str] | None = None, + target: set[str] | None = None, +) -> None: + """Record source files for modules imported while a recorder is active. + + Args: + name: The absolute requested module name. + result: The object returned by ``__import__`` or ``import_module``. + fromlist: The ``fromlist`` passed to ``__import__``. + target: The active read set. Defaults to the current recorder context. + """ + if target is None: + target = _active_reads.get() + if target is None: + return + _record_module_file(result, target) + result_id = id(result) + if (module := sys.modules.get(name)) and id(module) != result_id: + _record_module_file(module, target) + if not fromlist: + return + for item in fromlist: + if ( + item != "*" + and (module := sys.modules.get(f"{name}.{item}")) + and id(module) != result_id + ): + _record_module_file(module, target) + + def enable_read_tracking(root: Path | None = None) -> None: """Install per-page source-read tracking and register the recorder hook. @@ -107,8 +283,19 @@ def enable_read_tracking(root: Path | None = None) -> None: Args: root: Project root; only reads under it are recorded. Defaults to cwd. """ - global _patched, _recorder_root - _recorder_root = (root or Path.cwd()).resolve() + global _patched, _recorder_raw_root_str, _recorder_root, _recorder_root_str + raw_root = root or Path.cwd() + resolved_root = raw_root.resolve() + raw_root_str = os.path.abspath(os.fsdecode(raw_root)) # noqa: PTH100 + if ( + _recorder_root != resolved_root + or _recorder_raw_root_str != raw_root_str + or _recorder_root_str != str(resolved_root) + ): + _module_file_cache.clear() + _recorder_root = resolved_root + _recorder_root_str = str(resolved_root) + _recorder_raw_root_str = raw_root_str from reflex_base.plugins import compiler as _bc @@ -121,6 +308,8 @@ def enable_read_tracking(root: Path | None = None) -> None: orig_read_text = Path.read_text orig_read_bytes = Path.read_bytes orig_open = builtins.open + orig_import = builtins.__import__ + orig_import_module = importlib.import_module def read_text(self: Path, *args: object, **kwargs: object): _record_read(self) @@ -135,9 +324,31 @@ def open_(file: object, mode: str = "r", *args: object, **kwargs: object): _record_read(file) return orig_open(file, mode, *args, **kwargs) # type: ignore[arg-type] + def import_( + name: str, + globals_: dict[str, object] | None = None, + locals_: dict[str, object] | None = None, + fromlist: Sequence[str] | None = (), + level: int = 0, + ): + result = orig_import(name, globals_, locals_, fromlist, level) + if (target := _active_reads.get()) is not None: + _record_imported_modules( + _absolute_import_name(name, globals_, level), result, fromlist, target + ) + return result + + def import_module(name: str, package: str | None = None): + result = orig_import_module(name, package) + if (target := _active_reads.get()) is not None: + _record_imported_modules(result.__name__, result, target=target) + return result + Path.read_text = read_text # type: ignore[method-assign,assignment] Path.read_bytes = read_bytes # type: ignore[method-assign,assignment] builtins.open = open_ # type: ignore[assignment] + builtins.__import__ = import_ # type: ignore[assignment] + importlib.import_module = import_module @contextlib.contextmanager @@ -171,8 +382,6 @@ def _app_entrypoint_file(root: Path | None = None) -> Path | None: The resolved entrypoint file path under ``root``, or None if it can't be determined (no app module imported, or it lives outside ``root``). """ - import sys - try: from reflex.config import get_config @@ -226,8 +435,6 @@ def global_epoch( def _module_file(component: object) -> Path | None: - import sys - mod = sys.modules.get(getattr(component, "__module__", "") or "") file = getattr(mod, "__file__", None) return Path(file) if file else None @@ -274,8 +481,6 @@ def component_module_files( def _resolve_module_file(name: str) -> str | None: - import sys - mod = sys.modules.get(name) file = getattr(mod, "__file__", None) return str(Path(file).resolve()) if file else None @@ -291,8 +496,6 @@ def _loaded_first_party_modules(root: Path) -> dict[str, str]: A mapping of resolved file path -> module name for loaded modules under ``root``. """ - import sys - file_to_mod: dict[str, str] = {} for name, mod in list(sys.modules.items()): file = getattr(mod, "__file__", None) diff --git a/tests/units/compiler/test_page_cache.py b/tests/units/compiler/test_page_cache.py index dcd908788d1..e5ba80ed171 100644 --- a/tests/units/compiler/test_page_cache.py +++ b/tests/units/compiler/test_page_cache.py @@ -210,6 +210,116 @@ def fail_loaded_first_party_modules(root): assert page_cache.app_dependency_files(root=tmp_path) == set() +def test_record_reads_tracks_executed_importlib_import(tmp_path, monkeypatch): + import importlib + import sys + + module_name = "runtime_import_dep_for_page_cache" + module_file = tmp_path / f"{module_name}.py" + module_file.write_text("VALUE = 1\n") + monkeypatch.syspath_prepend(str(tmp_path)) + monkeypatch.delitem(sys.modules, module_name, raising=False) + importlib.invalidate_caches() + page_cache.enable_read_tracking(root=tmp_path) + + try: + with page_cache.record_reads() as reads: + importlib.import_module(module_name) + finally: + sys.modules.pop(module_name, None) + + assert str(module_file.resolve()) in reads + + +def test_record_reads_tracks_executed_builtin_import(tmp_path, monkeypatch): + import importlib + import sys + + package = tmp_path / "runtime_import_pkg" + package.mkdir() + (package / "__init__.py").write_text("") + child = package / "child.py" + child.write_text("VALUE = 1\n") + monkeypatch.syspath_prepend(str(tmp_path)) + monkeypatch.delitem(sys.modules, "runtime_import_pkg", raising=False) + monkeypatch.delitem(sys.modules, "runtime_import_pkg.child", raising=False) + importlib.invalidate_caches() + page_cache.enable_read_tracking(root=tmp_path) + + try: + with page_cache.record_reads() as reads: + __import__("runtime_import_pkg.child") + finally: + sys.modules.pop("runtime_import_pkg", None) + sys.modules.pop("runtime_import_pkg.child", None) + + assert str(child.resolve()) in reads + + +def test_record_reads_ignores_unexecuted_import(tmp_path, monkeypatch): + import importlib + import sys + + module_name = "uncalled_runtime_import_dep" + module_file = tmp_path / f"{module_name}.py" + module_file.write_text("VALUE = 1\n") + monkeypatch.syspath_prepend(str(tmp_path)) + monkeypatch.delitem(sys.modules, module_name, raising=False) + importlib.invalidate_caches() + page_cache.enable_read_tracking(root=tmp_path) + + def import_if_called(): + return importlib.import_module(module_name) + + with page_cache.record_reads() as reads: + pass + + assert import_if_called + assert str(module_file.resolve()) not in reads + assert module_name not in sys.modules + + +def test_record_reads_imports_only_project_modules(tmp_path): + import importlib + + page_cache.enable_read_tracking(root=tmp_path) + + with page_cache.record_reads() as reads: + importlib.import_module("json") + + assert reads == set() + + +def test_record_reads_tracks_symlinked_project_import(tmp_path, monkeypatch): + import importlib + import sys + + import pytest + + root = tmp_path / "app" + root.mkdir() + module_name = "symlinked_runtime_dep" + module_file = root / f"{module_name}.py" + module_file.write_text("VALUE = 1\n") + linked_root = tmp_path / "linked_app" + try: + linked_root.symlink_to(root, target_is_directory=True) + except OSError as exc: + pytest.skip(f"symlinks unavailable: {exc}") + monkeypatch.syspath_prepend(str(linked_root)) + monkeypatch.delitem(sys.modules, module_name, raising=False) + importlib.invalidate_caches() + page_cache.enable_read_tracking(root=root) + + try: + with page_cache.record_reads() as reads: + importlib.import_module(module_name) + finally: + sys.modules.pop(module_name, None) + + assert str(module_file.resolve()) in reads + + def test_used_state_files_from_output_and_memos(tmp_path): from types import SimpleNamespace From f79b9db7b4d76a191146bb647274bd43577b1ec1 Mon Sep 17 00:00:00 2001 From: Farhan <www.mfarvirus@gmail.com> Date: Tue, 30 Jun 2026 20:41:49 +0500 Subject: [PATCH 11/56] refactor(compiler): simplify page_cache module-file recording Replace the os.path string-comparison helpers used for runtime import tracking with pathlib.Path operations, dropping the redundant cached root-string globals (_recorder_root_str, _recorder_raw_root_str) and the sentinel-based module-file cache. Make the read-set target an explicit argument instead of falling back to the recorder context. --- reflex/compiler/page_cache.py | 91 ++++++++----------------- tests/units/compiler/test_page_cache.py | 73 ++++++++++---------- 2 files changed, 67 insertions(+), 97 deletions(-) diff --git a/reflex/compiler/page_cache.py b/reflex/compiler/page_cache.py index 058338bd38b..85291c19e3d 100644 --- a/reflex/compiler/page_cache.py +++ b/reflex/compiler/page_cache.py @@ -13,7 +13,6 @@ import hashlib import importlib import importlib.util -import os import re import sys from collections.abc import Callable, Sequence @@ -59,8 +58,6 @@ def _reflex_version() -> str: _active_reads: ContextVar[set[str] | None] = ContextVar("_active_reads", default=None) _patched = False _recorder_root: Path | None = None -_recorder_root_str: str | None = None -_recorder_raw_root_str: str | None = None #: Path parts that mark a dependency/build location whose reads are never a #: page's own source dependency (a change there flows through the version/epoch). @@ -85,12 +82,11 @@ def _reflex_version() -> str: } _PYTHON_PREFIXES = tuple( - os.path.abspath(os.fsdecode(prefix)) # noqa: PTH100 + Path(prefix).resolve() for prefix in {sys.base_exec_prefix, sys.base_prefix, sys.exec_prefix, sys.prefix} if prefix ) -_MODULE_FILE_CACHE_MISSING: object = object() -_module_file_cache: dict[tuple[str, str], str | None] = {} +_module_file_cache: dict[tuple[Path, str], str | None] = {} def _record_read(path: object) -> None: @@ -110,8 +106,8 @@ def _record_read(path: object) -> None: target.add(str(resolved)) -def _is_path_within(path: str, root: str) -> bool: - """Return whether ``path`` is contained within ``root`` without filesystem IO. +def _is_inside(path: Path, root: Path) -> bool: + """Return whether ``path`` is nested below ``root``. Args: path: Absolute path to test. @@ -120,17 +116,10 @@ def _is_path_within(path: str, root: str) -> bool: Returns: True when ``path`` is nested below ``root``. """ - normalized_path = os.path.normcase(path) - normalized_root = os.path.normcase(root) - if normalized_path == normalized_root: - return False - try: - return os.path.commonpath((normalized_path, normalized_root)) == normalized_root - except ValueError: - return False + return root in path.parents -def _is_python_install_path(path: str) -> bool: +def _is_python_install_file(path: Path) -> bool: """Return whether ``path`` is under this interpreter's install roots. Args: @@ -139,7 +128,7 @@ def _is_python_install_path(path: str) -> bool: Returns: True when ``path`` is under the interpreter or virtualenv prefix. """ - return any(_is_path_within(path, prefix) for prefix in _PYTHON_PREFIXES) + return any(prefix in path.parents for prefix in _PYTHON_PREFIXES) def _recordable_module_file(file: object) -> str | None: @@ -152,30 +141,24 @@ def _recordable_module_file(file: object) -> str | None: The resolved module file path to record, or None when it is outside the project root or otherwise not recordable. """ - root = _recorder_root - root_str = _recorder_root_str - if root is None or root_str is None: - return None - if not isinstance(file, (str, bytes, os.PathLike)): - return None try: - raw_path = os.path.abspath(os.fsdecode(file)) # noqa: PTH100 + path = Path(file).absolute() # type: ignore[arg-type] except (OSError, TypeError, ValueError): return None - cache_key = (root_str, raw_path) - cached = _module_file_cache.get(cache_key, _MODULE_FILE_CACHE_MISSING) - if cached is not _MODULE_FILE_CACHE_MISSING: - return cached if isinstance(cached, str) else None - - path = Path(raw_path) - under_project_path = _is_path_within(raw_path, root_str) or ( - _recorder_raw_root_str is not None - and _is_path_within(raw_path, _recorder_raw_root_str) - ) resolved_str = None + root = _recorder_root + if root is None: + return None + + cache_key = (root, str(path)) + try: + return _module_file_cache[cache_key] + except KeyError: + pass + if not any(part in _EXCLUDE_PARTS for part in path.parts) and ( - under_project_path or not _is_python_install_path(raw_path) + _is_inside(path, root) or not _is_python_install_file(path) ): try: resolved = path.resolve() @@ -183,22 +166,18 @@ def _recordable_module_file(file: object) -> str | None: pass else: if not any(part in _EXCLUDE_PARTS for part in resolved.parts): - resolved_str = str(resolved) if root in resolved.parents else None + resolved_str = str(resolved) if _is_inside(resolved, root) else None _module_file_cache[cache_key] = resolved_str return resolved_str -def _record_module_file(module: object, target: set[str] | None = None) -> None: +def _record_module_file(module: object, target: set[str]) -> None: """Record the source file for an imported module, if it has one. Args: module: The imported module object. - target: The active read set. Defaults to the current recorder context. + target: The active read set. """ - if target is None: - target = _active_reads.get() - if target is None: - return file = getattr(module, "__file__", None) if not file: return @@ -242,21 +221,17 @@ def _absolute_import_name( def _record_imported_modules( name: str, result: object, + target: set[str], fromlist: Sequence[str] | None = None, - target: set[str] | None = None, ) -> None: """Record source files for modules imported while a recorder is active. Args: name: The absolute requested module name. result: The object returned by ``__import__`` or ``import_module``. + target: The active read set. fromlist: The ``fromlist`` passed to ``__import__``. - target: The active read set. Defaults to the current recorder context. """ - if target is None: - target = _active_reads.get() - if target is None: - return _record_module_file(result, target) result_id = id(result) if (module := sys.modules.get(name)) and id(module) != result_id: @@ -283,19 +258,11 @@ def enable_read_tracking(root: Path | None = None) -> None: Args: root: Project root; only reads under it are recorded. Defaults to cwd. """ - global _patched, _recorder_raw_root_str, _recorder_root, _recorder_root_str - raw_root = root or Path.cwd() - resolved_root = raw_root.resolve() - raw_root_str = os.path.abspath(os.fsdecode(raw_root)) # noqa: PTH100 - if ( - _recorder_root != resolved_root - or _recorder_raw_root_str != raw_root_str - or _recorder_root_str != str(resolved_root) - ): + global _patched, _recorder_root + resolved_root = (root or Path.cwd()).resolve() + if _recorder_root != resolved_root: _module_file_cache.clear() _recorder_root = resolved_root - _recorder_root_str = str(resolved_root) - _recorder_raw_root_str = raw_root_str from reflex_base.plugins import compiler as _bc @@ -334,14 +301,14 @@ def import_( result = orig_import(name, globals_, locals_, fromlist, level) if (target := _active_reads.get()) is not None: _record_imported_modules( - _absolute_import_name(name, globals_, level), result, fromlist, target + _absolute_import_name(name, globals_, level), result, target, fromlist ) return result def import_module(name: str, package: str | None = None): result = orig_import_module(name, package) if (target := _active_reads.get()) is not None: - _record_imported_modules(result.__name__, result, target=target) + _record_imported_modules(result.__name__, result, target) return result Path.read_text = read_text # type: ignore[method-assign,assignment] diff --git a/tests/units/compiler/test_page_cache.py b/tests/units/compiler/test_page_cache.py index e5ba80ed171..31e28d22e7e 100644 --- a/tests/units/compiler/test_page_cache.py +++ b/tests/units/compiler/test_page_cache.py @@ -1,8 +1,37 @@ """Tests for the per-page dependency graph used by the incremental compile cache.""" +import importlib +import sys + from reflex.compiler import page_cache +def _prepare_runtime_module(root, monkeypatch, module_name, *, import_root=None): + """Create a temporary module and make it importable. + + Args: + root: Directory where the module file is written. + monkeypatch: Pytest monkeypatch fixture. + module_name: Name of the module to create. + import_root: Directory added to ``sys.path``. Defaults to ``root``. + + Returns: + The created module file path. + """ + module_file = root / f"{module_name}.py" + module_file.write_text("VALUE = 1\n") + monkeypatch.syspath_prepend(str(import_root or root)) + sys.modules.pop(module_name, None) + importlib.invalidate_caches() + return module_file + + +def _forget_modules(*module_names): + """Remove temporary modules imported by a test.""" + for module_name in module_names: + sys.modules.pop(module_name, None) + + def test_global_epoch_tracks_global_files(tmp_path): (tmp_path / "rxconfig.py").write_text("config = 1\n") (tmp_path / "other.py").write_text("x = 1\n") @@ -211,38 +240,27 @@ def fail_loaded_first_party_modules(root): def test_record_reads_tracks_executed_importlib_import(tmp_path, monkeypatch): - import importlib - import sys - module_name = "runtime_import_dep_for_page_cache" - module_file = tmp_path / f"{module_name}.py" - module_file.write_text("VALUE = 1\n") - monkeypatch.syspath_prepend(str(tmp_path)) - monkeypatch.delitem(sys.modules, module_name, raising=False) - importlib.invalidate_caches() + module_file = _prepare_runtime_module(tmp_path, monkeypatch, module_name) page_cache.enable_read_tracking(root=tmp_path) try: with page_cache.record_reads() as reads: importlib.import_module(module_name) finally: - sys.modules.pop(module_name, None) + _forget_modules(module_name) assert str(module_file.resolve()) in reads def test_record_reads_tracks_executed_builtin_import(tmp_path, monkeypatch): - import importlib - import sys - package = tmp_path / "runtime_import_pkg" package.mkdir() (package / "__init__.py").write_text("") child = package / "child.py" child.write_text("VALUE = 1\n") monkeypatch.syspath_prepend(str(tmp_path)) - monkeypatch.delitem(sys.modules, "runtime_import_pkg", raising=False) - monkeypatch.delitem(sys.modules, "runtime_import_pkg.child", raising=False) + _forget_modules("runtime_import_pkg", "runtime_import_pkg.child") importlib.invalidate_caches() page_cache.enable_read_tracking(root=tmp_path) @@ -250,22 +268,14 @@ def test_record_reads_tracks_executed_builtin_import(tmp_path, monkeypatch): with page_cache.record_reads() as reads: __import__("runtime_import_pkg.child") finally: - sys.modules.pop("runtime_import_pkg", None) - sys.modules.pop("runtime_import_pkg.child", None) + _forget_modules("runtime_import_pkg", "runtime_import_pkg.child") assert str(child.resolve()) in reads def test_record_reads_ignores_unexecuted_import(tmp_path, monkeypatch): - import importlib - import sys - module_name = "uncalled_runtime_import_dep" - module_file = tmp_path / f"{module_name}.py" - module_file.write_text("VALUE = 1\n") - monkeypatch.syspath_prepend(str(tmp_path)) - monkeypatch.delitem(sys.modules, module_name, raising=False) - importlib.invalidate_caches() + module_file = _prepare_runtime_module(tmp_path, monkeypatch, module_name) page_cache.enable_read_tracking(root=tmp_path) def import_if_called(): @@ -280,8 +290,6 @@ def import_if_called(): def test_record_reads_imports_only_project_modules(tmp_path): - import importlib - page_cache.enable_read_tracking(root=tmp_path) with page_cache.record_reads() as reads: @@ -291,31 +299,26 @@ def test_record_reads_imports_only_project_modules(tmp_path): def test_record_reads_tracks_symlinked_project_import(tmp_path, monkeypatch): - import importlib - import sys - import pytest root = tmp_path / "app" root.mkdir() module_name = "symlinked_runtime_dep" - module_file = root / f"{module_name}.py" - module_file.write_text("VALUE = 1\n") linked_root = tmp_path / "linked_app" try: linked_root.symlink_to(root, target_is_directory=True) except OSError as exc: pytest.skip(f"symlinks unavailable: {exc}") - monkeypatch.syspath_prepend(str(linked_root)) - monkeypatch.delitem(sys.modules, module_name, raising=False) - importlib.invalidate_caches() + module_file = _prepare_runtime_module( + root, monkeypatch, module_name, import_root=linked_root + ) page_cache.enable_read_tracking(root=root) try: with page_cache.record_reads() as reads: importlib.import_module(module_name) finally: - sys.modules.pop(module_name, None) + _forget_modules(module_name) assert str(module_file.resolve()) in reads From 451be1e96a1e1f19c86978fc5961608fcc8c6147 Mon Sep 17 00:00:00 2001 From: Farhan <www.mfarvirus@gmail.com> Date: Wed, 1 Jul 2026 21:35:44 +0500 Subject: [PATCH 12/56] fix(compiler): guard read-tracker recursion on pathlib lazy imports The compile-cache read tracker patches builtins.__import__ to record a page's runtime imports. While recording, _recordable_module_file resolves a module file via Path(...).absolute(), which on CPython 3.12 lazily runs `import ntpath` to compare path flavours. That import re-entered the patched hook -> resolved a path again -> imported again, recursing until the stack overflowed (32 unit tests failed only on ubuntu 3.12). Suspend read/import tracking (_suspend_tracking) while the tracker's own path/module bookkeeping runs, so imports and reads it triggers are neither recorded nor able to re-enter. Add a version-independent regression test that mimics the 3.12 lazy import. --- reflex/compiler/page_cache.py | 91 +++++++++++++++---------- tests/units/compiler/test_page_cache.py | 27 ++++++++ 2 files changed, 82 insertions(+), 36 deletions(-) diff --git a/reflex/compiler/page_cache.py b/reflex/compiler/page_cache.py index 85291c19e3d..14a17036d63 100644 --- a/reflex/compiler/page_cache.py +++ b/reflex/compiler/page_cache.py @@ -89,21 +89,36 @@ def _reflex_version() -> str: _module_file_cache: dict[tuple[Path, str], str | None] = {} +@contextlib.contextmanager +def _suspend_tracking(): + """Ignore read/import hooks caused by dependency bookkeeping. + + Yields: + None. + """ + token = _active_reads.set(None) + try: + yield + finally: + _active_reads.reset(token) + + def _record_read(path: object) -> None: target = _active_reads.get() if target is None: return - try: - resolved = Path(path).resolve() # type: ignore[arg-type] - except (OSError, TypeError, ValueError): - return - if any(part in _EXCLUDE_PARTS for part in resolved.parts): - return - root = _recorder_root - under_root = root is not None and root in resolved.parents - if not under_root and resolved.suffix.lower() not in _CONTENT_SUFFIXES: - return - target.add(str(resolved)) + with _suspend_tracking(): + try: + resolved = Path(path).resolve() # type: ignore[arg-type] + except (OSError, TypeError, ValueError): + return + if any(part in _EXCLUDE_PARTS for part in resolved.parts): + return + root = _recorder_root + under_root = root is not None and root in resolved.parents + if not under_root and resolved.suffix.lower() not in _CONTENT_SUFFIXES: + return + target.add(str(resolved)) def _is_inside(path: Path, root: Path) -> bool: @@ -141,34 +156,35 @@ def _recordable_module_file(file: object) -> str | None: The resolved module file path to record, or None when it is outside the project root or otherwise not recordable. """ - try: - path = Path(file).absolute() # type: ignore[arg-type] - except (OSError, TypeError, ValueError): - return None - - resolved_str = None - root = _recorder_root - if root is None: - return None + with _suspend_tracking(): + try: + path = Path(file).absolute() # type: ignore[arg-type] + except (OSError, TypeError, ValueError): + return None - cache_key = (root, str(path)) - try: - return _module_file_cache[cache_key] - except KeyError: - pass + resolved_str = None + root = _recorder_root + if root is None: + return None - if not any(part in _EXCLUDE_PARTS for part in path.parts) and ( - _is_inside(path, root) or not _is_python_install_file(path) - ): + cache_key = (root, str(path)) try: - resolved = path.resolve() - except (OSError, TypeError, ValueError): + return _module_file_cache[cache_key] + except KeyError: pass - else: - if not any(part in _EXCLUDE_PARTS for part in resolved.parts): - resolved_str = str(resolved) if _is_inside(resolved, root) else None - _module_file_cache[cache_key] = resolved_str - return resolved_str + + if not any(part in _EXCLUDE_PARTS for part in path.parts) and ( + _is_inside(path, root) or not _is_python_install_file(path) + ): + try: + resolved = path.resolve() + except (OSError, TypeError, ValueError): + pass + else: + if not any(part in _EXCLUDE_PARTS for part in resolved.parts): + resolved_str = str(resolved) if _is_inside(resolved, root) else None + _module_file_cache[cache_key] = resolved_str + return resolved_str def _record_module_file(module: object, target: set[str]) -> None: @@ -301,7 +317,10 @@ def import_( result = orig_import(name, globals_, locals_, fromlist, level) if (target := _active_reads.get()) is not None: _record_imported_modules( - _absolute_import_name(name, globals_, level), result, target, fromlist + _absolute_import_name(name, globals_, level), + result, + target, + fromlist, ) return result diff --git a/tests/units/compiler/test_page_cache.py b/tests/units/compiler/test_page_cache.py index 31e28d22e7e..7dd872488a5 100644 --- a/tests/units/compiler/test_page_cache.py +++ b/tests/units/compiler/test_page_cache.py @@ -2,6 +2,7 @@ import importlib import sys +from pathlib import Path from reflex.compiler import page_cache @@ -289,6 +290,32 @@ def import_if_called(): assert module_name not in sys.modules +def test_record_reads_no_recursion_when_recorder_import_triggers_import( + tmp_path, monkeypatch +): + """Recorder-internal imports must not re-enter dependency tracking.""" + module_name = "recorder_reentry_dep" + module_file = _prepare_runtime_module(tmp_path, monkeypatch, module_name) + page_cache.enable_read_tracking(root=tmp_path) + + real_absolute = Path.absolute + + def absolute_with_lazy_import(self: Path): + __import__("ntpath") + return real_absolute(self) + + monkeypatch.setattr(Path, "absolute", absolute_with_lazy_import) + + try: + with page_cache.record_reads() as reads: + importlib.import_module(module_name) + finally: + _forget_modules(module_name) + + assert str(module_file.resolve()) in reads + assert not any("ntpath" in read for read in reads) + + def test_record_reads_imports_only_project_modules(tmp_path): page_cache.enable_read_tracking(root=tmp_path) From 724cd3de4bb4addbfaf0940ac0839f4ad12662e8 Mon Sep 17 00:00:00 2001 From: Farhan <www.mfarvirus@gmail.com> Date: Wed, 1 Jul 2026 21:58:36 +0500 Subject: [PATCH 13/56] feat(compiler): track dynamic app-import reads for the compile cache Config the app module pulls in dynamically (importlib calls, files read at import time) was invisible to app_dependency_files, which only walked the static import graph from the entrypoint. Editing such config could leave reused app-wide output stale. Record files read/imported while the app module loads (record_app_import, wired into prerequisites.get_app under REFLEX_COMPILE_CACHE) and fold that dynamic set into app_dependency_files, subtracting per-page static closures so ordinary page edits still invalidate per-page. Extract the shared graph walk into _walk_import_closure. --- reflex/compiler/page_cache.py | 97 +++++++++++++++++-------- reflex/utils/prerequisites.py | 34 +++++---- tests/units/compiler/test_page_cache.py | 78 ++++++++++++++++++++ 3 files changed, 166 insertions(+), 43 deletions(-) diff --git a/reflex/compiler/page_cache.py b/reflex/compiler/page_cache.py index 14a17036d63..3a162112bb6 100644 --- a/reflex/compiler/page_cache.py +++ b/reflex/compiler/page_cache.py @@ -87,6 +87,7 @@ def _reflex_version() -> str: if prefix ) _module_file_cache: dict[tuple[Path, str], str | None] = {} +_app_import_reads: dict[Path, set[str]] = {} @contextlib.contextmanager @@ -352,6 +353,29 @@ def record_reads(): _active_reads.reset(token) +@contextlib.contextmanager +def record_app_import(root: Path | None = None): + """Record files read/imported while importing the app module. + + Args: + root: Project root. Defaults to cwd. + + Yields: + None. + """ + root = (root or Path.cwd()).resolve() + recorded: set[str] = set() + success = False + try: + with record_reads() as reads: + yield + recorded = set(reads) + success = True + finally: + if success: + _app_import_reads[root] = recorded + + def _app_entrypoint_file(root: Path | None = None) -> Path | None: """Resolve the user's app entrypoint module file (where ``rx.App`` is built). @@ -583,8 +607,36 @@ def build_import_graph(root: Path | None = None) -> dict[str, set[str]]: def clear_import_graph() -> None: - """Drop the cached import graph (e.g. after modules are reloaded).""" + """Drop cached import graphs (e.g. after modules are reloaded).""" _import_graph_cache.clear() + _app_import_reads.clear() + + +def _walk_import_closure( + graph: dict[str, set[str]], + starts: set[str], + barriers: set[str] | None = None, +) -> set[str]: + """Walk a first-party import graph from ``starts``. + + Args: + graph: Mapping of source file to imported source files. + starts: Source files where traversal begins. + barriers: Files included elsewhere and not traversed into. + + Returns: + The reachable source files, excluding barriers. + """ + barriers = barriers or set() + seen: set[str] = set() + stack = list(starts - barriers) + while stack: + cur = stack.pop() + if cur in seen: + continue + seen.add(cur) + stack.extend(dep for dep in graph.get(cur, ()) if dep not in barriers) + return seen def _component_source_files(component: object, root: Path) -> set[str]: @@ -632,16 +684,7 @@ def page_py_dependencies( """ root = (root or Path.cwd()).resolve() graph = build_import_graph(root) - - seen: set[str] = set() - stack = list(_component_source_files(component, root)) - while stack: - cur = stack.pop() - if cur in seen: - continue - seen.add(cur) - stack.extend(graph.get(cur, ())) - return seen + return _walk_import_closure(graph, _component_source_files(component, root)) def app_dependency_files( @@ -651,11 +694,11 @@ def app_dependency_files( Walks the first-party import graph from the app entrypoint (:func:`_app_entrypoint_file`), treating each page-defining module as a - barrier (not entered), so the result is the entrypoint plus the config-only - modules it imports, such as theme, app-wraps, stylesheets, head components, and - never page modules or their deep dependencies, which are tracked per page. A - config module shared with a page is still captured (it is reached from the - entrypoint directly, not through the barrier). + barrier (not entered), then folds in files read/imported while the app module + loaded. Static page dependency closures are removed from that dynamic set so + regular page edits still invalidate per-page rather than globally. A config + module shared with a page is still captured when it is reached from the + entrypoint directly. These configure the app-wide files an incremental rebuild reuses on disk (app root, contexts, theme, stylesheet), so they are folded into @@ -675,24 +718,18 @@ def app_dependency_files( entrypoint = _app_entrypoint_file(root) if entrypoint is None: return set() - file_to_mod = _loaded_first_party_modules(root) + graph = build_import_graph(root) barriers: set[str] = set() + page_deps: set[str] = set() for page in pages or (): - barriers |= _component_source_files(getattr(page, "component", None), root) + starts = _component_source_files(getattr(page, "component", None), root) + barriers |= starts + page_deps |= _walk_import_closure(graph, starts) start = str(entrypoint) - seen = {start} - stack = [start] - while stack: - cur = stack.pop() - modname = file_to_mod.get(cur) - if modname is None: - continue - for dep in _module_import_edges(cur, modname, file_to_mod): - if dep not in seen and dep not in barriers: - seen.add(dep) - stack.append(dep) - return seen + static_deps = _walk_import_closure(graph, {start}, barriers) + dynamic_deps = _app_import_reads.get(root, set()) - page_deps + return static_deps | dynamic_deps def make_hasher() -> Callable[[str], str | None]: diff --git a/reflex/utils/prerequisites.py b/reflex/utils/prerequisites.py index a8c84da6aaa..df243182ec3 100644 --- a/reflex/utils/prerequisites.py +++ b/reflex/utils/prerequisites.py @@ -194,22 +194,30 @@ def get_app(reload: bool = False) -> ModuleType: module = config.module sys.path.insert(0, getcwd()) # noqa: PTH109 - app = ( - __import__(module, fromlist=(constants.CompileVars.APP,)) - if not config.app_module - else config.app_module - ) - if reload: - from reflex.page import DECORATED_PAGES - from reflex.state import reload_state_module + if environment.REFLEX_COMPILE_CACHE.get(): + from reflex.compiler import page_cache + + page_cache.enable_read_tracking() + recorder = page_cache.record_app_import() + else: + recorder = contextlib.nullcontext() + with recorder: + app = ( + __import__(module, fromlist=(constants.CompileVars.APP,)) + if not config.app_module + else config.app_module + ) + if reload: + from reflex.page import DECORATED_PAGES + from reflex.state import reload_state_module - # Reset rx.State subclasses to avoid conflict when reloading. - reload_state_module(module=module) + # Reset rx.State subclasses to avoid conflict when reloading. + reload_state_module(module=module) - DECORATED_PAGES.clear() + DECORATED_PAGES.clear() - # Reload the app module. - importlib.reload(app) + # Reload the app module. + importlib.reload(app) except Exception as ex: telemetry.send_error(ex, context="frontend") raise diff --git a/tests/units/compiler/test_page_cache.py b/tests/units/compiler/test_page_cache.py index 7dd872488a5..45d84e4e7c1 100644 --- a/tests/units/compiler/test_page_cache.py +++ b/tests/units/compiler/test_page_cache.py @@ -240,6 +240,84 @@ def fail_loaded_first_party_modules(root): assert page_cache.app_dependency_files(root=tmp_path) == set() +def test_app_dependency_files_tracks_dynamic_import_config_read(tmp_path, monkeypatch): + from types import SimpleNamespace + + from reflex.utils import prerequisites + + entry = tmp_path / "myapp.py" + dynamic = tmp_path / "dynamic_theme.py" + config_file = tmp_path / "theme.json" + entry.write_text( + "import importlib\ntheme = importlib.import_module('dynamic_theme').THEME\n" + ) + dynamic.write_text( + "from pathlib import Path\nTHEME = Path('theme.json').read_text()\n" + ) + config_file.write_text('"light"\n') + config = SimpleNamespace( + module="myapp", + app_module=None, + _app_name_is_valid=True, + ) + monkeypatch.chdir(tmp_path) + monkeypatch.syspath_prepend(str(tmp_path)) + monkeypatch.setenv("REFLEX_COMPILE_CACHE", "1") + monkeypatch.setattr(prerequisites, "get_config", lambda: config) + monkeypatch.setattr("reflex.config.get_config", lambda: config) + _forget_modules("myapp", "dynamic_theme") + + try: + prerequisites.get_app() + deps = page_cache.app_dependency_files(root=tmp_path) + epoch = page_cache.global_epoch(root=tmp_path) + + config_file.write_text('"dark"\n') + finally: + _forget_modules("myapp", "dynamic_theme") + + assert str(dynamic.resolve()) in deps + assert str(config_file.resolve()) in deps + assert page_cache.global_epoch(root=tmp_path) != epoch + + +def test_app_dependency_files_subtracts_pages_from_app_import_reads( + tmp_path, monkeypatch +): + from types import SimpleNamespace + + from reflex.utils import prerequisites + + entry = tmp_path / "myapp.py" + theme = tmp_path / "theme.py" + page = tmp_path / "page.py" + view = tmp_path / "view.py" + entry.write_text("import theme\nfrom page import index\napp_theme = theme.THEME\n") + theme.write_text("THEME = 'light'\n") + page.write_text("from view import render\n\ndef index():\n return render()\n") + view.write_text("def render():\n return 'view'\n") + config = SimpleNamespace( + module="myapp", + app_module=None, + _app_name_is_valid=True, + ) + monkeypatch.chdir(tmp_path) + monkeypatch.syspath_prepend(str(tmp_path)) + monkeypatch.setenv("REFLEX_COMPILE_CACHE", "1") + monkeypatch.setattr(prerequisites, "get_config", lambda: config) + monkeypatch.setattr("reflex.config.get_config", lambda: config) + _forget_modules("myapp", "theme", "page", "view") + + try: + prerequisites.get_app() + pages = [SimpleNamespace(component=sys.modules["page"].index)] + deps = page_cache.app_dependency_files(pages, root=tmp_path) + finally: + _forget_modules("myapp", "theme", "page", "view") + + assert deps == {str(entry.resolve()), str(theme.resolve())} + + def test_record_reads_tracks_executed_importlib_import(tmp_path, monkeypatch): module_name = "runtime_import_dep_for_page_cache" module_file = _prepare_runtime_module(tmp_path, monkeypatch, module_name) From 21e7d3ea9ee99772ec457b7cf3809289fc7053fc Mon Sep 17 00:00:00 2001 From: Farhan <www.mfarvirus@gmail.com> Date: Fri, 3 Jul 2026 05:01:05 +0500 Subject: [PATCH 14/56] fix(compiler): close staleness gaps in the incremental compile cache Four reviewer-confirmed ways the incremental path could serve stale .web output: - Path.open() calls io.open directly, bypassing the builtins.open patch, so data files read through it were never recorded as page dependencies. Patch Path.open itself. - The contexts file was never re-emitted, keeping old state defaults and client-storage config after a state-module edit. Re-emit it on every incremental rebuild (write_file already skips byte-identical writes). - assets/ was never copied (excluded from dependency tracking and the epoch), so an assets-only edit produced an all-hit rebuild with a stale .web/public. Run the same mtime-incremental copy as the full compile. - Memo output files are grouped one file per source module, but a miss rewrote them from only its own auto-memo contributions, dropping user @rx.memo exports and hit-sibling exports still imported by reused pages. Recompile memo-contributing same-module siblings together (tracked by a new has_memos manifest flag, schema 3 -> 4) and include user memos that share a rewritten file or whose module changed. --- reflex/compiler/disk_cache.py | 186 ++++++++++++++++- reflex/compiler/page_cache.py | 14 +- tests/units/compiler/test_disk_cache.py | 262 +++++++++++++++++++++--- tests/units/compiler/test_page_cache.py | 24 +++ 4 files changed, 449 insertions(+), 37 deletions(-) diff --git a/reflex/compiler/disk_cache.py b/reflex/compiler/disk_cache.py index b047afd0489..7e2cfc785c1 100644 --- a/reflex/compiler/disk_cache.py +++ b/reflex/compiler/disk_cache.py @@ -10,18 +10,20 @@ import dataclasses import json +import sys +from pathlib import Path from typing import TYPE_CHECKING, Any +from reflex_base import constants from reflex_base.plugins import CompileContext, CompilerHooks from reflex_base.utils.imports import ImportVar, merge_imports from reflex.compiler import page_cache from reflex.compiler.plugins import default_page_plugins -from reflex.utils import console, prerequisites +from reflex.utils import console, path_ops, prerequisites if TYPE_CHECKING: from collections.abc import Callable, Sequence - from pathlib import Path from reflex_base.plugins import PageContext, PageDefinition from reflex_base.utils.imports import ParsedImportDict @@ -29,7 +31,7 @@ from reflex.app import App #: Bump when the manifest layout changes (old manifests are then ignored). -_SCHEMA = 3 +_SCHEMA = 4 #: Manifest filename under the web directory. _MANIFEST_FILE = "reflex_compile_cache.json" @@ -102,6 +104,11 @@ def _manifest_page_entry( ), "app_wrap_keys": _wrap_key_strs(page_ctx.app_wrap_components.keys()), "is_stateful": is_stateful, + # Whether the page contributed auto memos: pages sharing a source + # module share one memo output file, so a memo-contributing hit page + # must be recompiled alongside a same-module miss (see + # ``_with_module_siblings``). + "has_memos": bool(page_ctx.memo_contributions), } @@ -240,6 +247,123 @@ def partition_pages( ] +def _with_module_siblings( + miss_pages: list[PageDefinition], + pages: Sequence[PageDefinition], + manifest: dict[str, Any], +) -> list[PageDefinition]: + """Expand the miss set with memo-contributing same-module hit pages. + + Auto-memo output is grouped into one file per source module, so rewriting + that file needs the contributions of *all* the module's pages that have + any. Hit pages that contributed no memos (per the manifest) have nothing + in that file and are left reused. + + Args: + miss_pages: The dependency-changed pages. + pages: All current page definitions (in compile order). + manifest: The loaded manifest. + + Returns: + The expanded miss list, in ``pages`` order. + """ + miss_modules = { + module + for page in miss_pages + if (module := getattr(page, "_source_module", None)) is not None + } + if not miss_modules: + return miss_pages + miss_routes = {page.route for page in miss_pages} + return [ + page + for page in pages + if page.route in miss_routes + or ( + getattr(page, "_source_module", None) in miss_modules + and manifest["pages"][page.route]["has_memos"] + ) + ] + + +def _changed_dependency_files( + manifest: dict[str, Any], hasher: Callable[[str], str | None] +) -> set[str]: + """Return every recorded dependency file whose content changed. + + Args: + manifest: The loaded manifest. + hasher: A memoized path -> content-hash function. + + Returns: + The set of changed dependency file paths. + """ + return { + path + for entry in manifest["pages"].values() + for path, digest in entry["dep_hashes"].items() + if hasher(path) != digest + } + + +def _module_source_file(module_name: str | None) -> str | None: + """Resolve a loaded module's source file path. + + Args: + module_name: The dotted module name. + + Returns: + The resolved file path string, or None. + """ + file = getattr(sys.modules.get(module_name or ""), "__file__", None) + if not file: + return None + try: + return str(Path(file).resolve()) + except OSError: + return None + + +def _complete_memo_defs( + contributions: dict[tuple[str, str | None], Any], + changed_files: set[str], +) -> list[Any]: + """Return the full definition set for the memo files being rewritten. + + Memo output is grouped one file per source module, so a rewrite must carry + every definition landing in that file: user ``@rx.memo`` definitions from + the global registry that share a module with a recompiled contribution, + plus user memos whose own module file changed (an edited memo body must be + re-emitted even though only its importer pages missed). + + Args: + contributions: The recompiled pages' auto-memo contributions. + changed_files: The dependency files whose content changed. + + Returns: + The memo definitions to compile, user memos first (matching the full + compile's emit order). + """ + from reflex_base.components.memo import MEMOS + from reflex_base.utils import memo_paths + + dirty_segments = { + segments + for definition in contributions.values() + if ( + segments := memo_paths.module_to_mirrored_segments(definition.source_module) + ) + is not None + } + user_memos = [ + memo + for memo in MEMOS.values() + if memo_paths.module_to_mirrored_segments(memo.source_module) in dirty_segments + or _module_source_file(memo.source_module) in changed_files + ] + return [*user_memos, *contributions.values()] + + def try_incremental_rebuild( app: App, *, @@ -254,6 +378,12 @@ def try_incremental_rebuild( unsafe to reuse: no/old manifest, a changed global input, a route change, or a miss page that altered its app-wrap set or stateful flag. + App-wide outputs that per-page dependency sets do not cover are always + re-emitted rather than tracked: the contexts file (state defaults and + client-storage config change with any state module) and the ``assets`` + copy (assets are excluded from dependency tracking). Both are cheap and + idempotent. + On success, reports (at info level) how many pages were recompiled vs reused and, while recompiling, shows a progress bar over the changed pages so a hot reload makes the incremental work visible. @@ -282,6 +412,8 @@ def try_incremental_rebuild( return False miss_pages = partition_pages(pages, manifest, hasher) + if miss_pages: + miss_pages = _with_module_siblings(miss_pages, pages, manifest) miss_routes = {p.route for p in miss_pages} # Recompile only the source-changed pages. @@ -347,6 +479,7 @@ def try_incremental_rebuild( # Write changed pages + their memo files; reuse everything else on disk. install_imports = _deserialize_imports(manifest["all_imports"]) if miss_ctx is not None: + memo_contributions: dict[tuple[str, str | None], Any] = {} for page in miss_pages: page_ctx = miss_ctx.compiled_pages[page.route] # Both are guaranteed non-None by the guard loop above. @@ -358,15 +491,21 @@ def try_incremental_rebuild( compiler.utils.resolve_path_of_web_dir(output_path), output_code, ) - memo_defs = list(page_ctx.memo_contributions.values()) - memo_files, memo_imports = compiler.compile_memo_components(memo_defs) - for mpath, mcode in memo_files: - compiler.utils.write_file( - compiler.utils.resolve_path_of_web_dir(mpath), mcode - ) - install_imports = merge_imports( - install_imports, page_ctx.frontend_imports, memo_imports + memo_contributions.update(page_ctx.memo_contributions) + install_imports = merge_imports(install_imports, page_ctx.frontend_imports) + # Memo output files are grouped per source module, so compile them once + # with the complete definition set (all recompiled pages' contributions + # plus the user memos sharing those files or whose module changed). + memo_files, memo_imports = compiler.compile_memo_components( + _complete_memo_defs( + memo_contributions, _changed_dependency_files(manifest, hasher) ) + ) + for mpath, mcode in memo_files: + compiler.utils.write_file( + compiler.utils.resolve_path_of_web_dir(mpath), mcode + ) + install_imports = merge_imports(install_imports, memo_imports) # Record which routes are stateful: miss pages from this compile, hit pages # from the manifest, so the stateful-pages marker is complete. We do NOT @@ -388,6 +527,31 @@ def try_incremental_rebuild( app._add_optional_endpoints() app._validate_var_dependencies() + # App-wide outputs that are cheap to re-emit and not gated by the epoch: + # contexts (state defaults/client-storage change with any state module, + # which per-page dependency sets do not force a regenerate of) and the + # assets copy (assets are excluded from dependency tracking entirely). + from reflex_components_radix.plugin import RadixThemesPlugin + + theme = next( + ( + plugin.get_theme() + for plugin in compiler_plugins + if isinstance(plugin, RadixThemesPlugin) + ), + None, + ) + context_path, context_code = compiler.compile_contexts(app._state, theme) + compiler.utils.write_file( + compiler.utils.resolve_path_of_web_dir(context_path), context_code + ) + assets_src = (root or Path.cwd()) / constants.Dirs.APP_ASSETS + if assets_src.is_dir(): + path_ops.update_directory_tree( + src=assets_src, + dest=prerequisites.get_web_dir() / constants.Dirs.PUBLIC, + ) + # Frontend packages + routing scaffolding (cheap, idempotent). from reflex.utils import frontend_skeleton diff --git a/reflex/compiler/page_cache.py b/reflex/compiler/page_cache.py index 3a162112bb6..38bcbb2b798 100644 --- a/reflex/compiler/page_cache.py +++ b/reflex/compiler/page_cache.py @@ -291,10 +291,14 @@ def enable_read_tracking(root: Path | None = None) -> None: orig_read_text = Path.read_text orig_read_bytes = Path.read_bytes + orig_path_open = Path.open orig_open = builtins.open orig_import = builtins.__import__ orig_import_module = importlib.import_module + def _is_read_mode(mode: str) -> bool: + return "r" in mode and not ("w" in mode or "a" in mode or "x" in mode) + def read_text(self: Path, *args: object, **kwargs: object): _record_read(self) return orig_read_text(self, *args, **kwargs) # type: ignore[arg-type] @@ -303,8 +307,15 @@ def read_bytes(self: Path): _record_read(self) return orig_read_bytes(self) + # ``Path.open`` calls ``io.open`` directly (not ``builtins.open``), so it + # needs its own patch for reads through it to be recorded. + def path_open(self: Path, mode: str = "r", *args: object, **kwargs: object): + if _is_read_mode(mode): + _record_read(self) + return orig_path_open(self, mode, *args, **kwargs) # type: ignore[arg-type] + def open_(file: object, mode: str = "r", *args: object, **kwargs: object): - if "r" in mode and not ("w" in mode or "a" in mode or "x" in mode): + if _is_read_mode(mode): _record_read(file) return orig_open(file, mode, *args, **kwargs) # type: ignore[arg-type] @@ -333,6 +344,7 @@ def import_module(name: str, package: str | None = None): Path.read_text = read_text # type: ignore[method-assign,assignment] Path.read_bytes = read_bytes # type: ignore[method-assign,assignment] + Path.open = path_open # type: ignore[method-assign,assignment] builtins.open = open_ # type: ignore[assignment] builtins.__import__ = import_ # type: ignore[assignment] importlib.import_module = import_module diff --git a/tests/units/compiler/test_disk_cache.py b/tests/units/compiler/test_disk_cache.py index aa9c4b80b24..fb62ad25b2b 100644 --- a/tests/units/compiler/test_disk_cache.py +++ b/tests/units/compiler/test_disk_cache.py @@ -3,6 +3,7 @@ import dataclasses import json from collections.abc import Callable, Sequence +from pathlib import Path from types import SimpleNamespace from typing import Any, cast @@ -26,6 +27,23 @@ class _FakePage: _source_module: str | None = None +def _use_tmp_web_dir(tmp_path, monkeypatch): + """Point every ``get_web_dir`` binding (module attr + env) at a tmp web dir. + + Args: + tmp_path: The test's tmp directory. + monkeypatch: The pytest monkeypatch fixture. + + Returns: + The created web directory path. + """ + web = tmp_path / ".web" + web.mkdir() + monkeypatch.setattr(disk_cache.prerequisites, "get_web_dir", lambda: web) + monkeypatch.setenv("REFLEX_WEB_WORKDIR", str(web)) + return web + + def _footer() -> Component: return rx.el.footer(rx.el.span("© Reflex"), class_name="footer") @@ -117,9 +135,7 @@ def test_partition_pages_detects_changed_source(): def test_write_and_load_manifest(tmp_path, monkeypatch): - web = tmp_path / ".web" - web.mkdir() - monkeypatch.setattr(disk_cache.prerequisites, "get_web_dir", lambda: web) + _use_tmp_web_dir(tmp_path, monkeypatch) pages = [ _FakePage(route="/a", component=_page_a), @@ -135,10 +151,16 @@ def test_write_and_load_manifest(tmp_path, monkeypatch): assert set(manifest["pages"]) == {"/a", "/b", "/c"} for route in ("/a", "/b", "/c"): entry = manifest["pages"][route] - # the manifest is pure bookkeeping: dep set + app-wrap keys + stateful flag - assert set(entry) == {"dep_hashes", "app_wrap_keys", "is_stateful"} - # these static pages register no new state + # the manifest is pure bookkeeping: dep set + app-wrap keys + flags + assert set(entry) == { + "dep_hashes", + "app_wrap_keys", + "is_stateful", + "has_memos", + } + # these static pages register no new state and contribute no memos assert entry["is_stateful"] is False + assert entry["has_memos"] is False # rendered output is never persisted (it already lives in .web, and is # never read back from the manifest) -> keeps the manifest small assert "output_code" not in entry @@ -155,9 +177,7 @@ def test_unchanged_pages_compile_identically(tmp_path, monkeypatch): reuse is correct if a fresh compile of that page yields identical output. Compile A, B, C; then compile A, B(edited), C; A and C must be byte-identical. """ - web = tmp_path / ".web" - web.mkdir() - monkeypatch.setattr(disk_cache.prerequisites, "get_web_dir", lambda: web) + _use_tmp_web_dir(tmp_path, monkeypatch) pages = [ _FakePage(route="/a", component=_page_a), @@ -187,22 +207,30 @@ def test_unchanged_pages_compile_identically(tmp_path, monkeypatch): ) +_CONTEXTS_STUB = "// contexts stub" + + def _stub_externals(app, monkeypatch): """Stub the side-effecting steps the fast path runs on a real app.""" import reflex.utils.frontend_skeleton as fs + from reflex.compiler import utils as compiler_utils monkeypatch.setattr(app, "_get_frontend_packages", lambda *a, **k: None) monkeypatch.setattr(app, "_add_optional_endpoints", lambda *a, **k: None) monkeypatch.setattr(app, "_validate_var_dependencies", lambda *a, **k: None) monkeypatch.setattr(app, "_write_stateful_pages_marker", lambda *a, **k: None) + # Serializing the real root state tree would pick up unrelated state + # classes from other collected test modules. + monkeypatch.setattr( + "reflex.compiler.compiler.compile_contexts", + lambda state, theme: (compiler_utils.get_context_path(), _CONTEXTS_STUB), + ) monkeypatch.setattr(fs, "update_react_router_config", lambda **k: None) monkeypatch.setattr(fs, "update_entry_client", lambda *a, **k: None) def test_incremental_rebuild_all_hits(tmp_path, monkeypatch): - web = tmp_path / ".web" - web.mkdir() - monkeypatch.setattr(disk_cache.prerequisites, "get_web_dir", lambda: web) + _use_tmp_web_dir(tmp_path, monkeypatch) app = rx.App() app.add_page(_page_a, route="/a") @@ -224,9 +252,7 @@ def test_incremental_rebuild_all_hits(tmp_path, monkeypatch): def test_incremental_rebuild_one_miss_writes_only_that_page(tmp_path, monkeypatch): from reflex.compiler import utils as compiler_utils - web = tmp_path / ".web" - web.mkdir() - monkeypatch.setattr(disk_cache.prerequisites, "get_web_dir", lambda: web) + web = _use_tmp_web_dir(tmp_path, monkeypatch) app = rx.App() app.add_page(_page_a, route="/a") @@ -264,6 +290,11 @@ def test_incremental_rebuild_one_miss_writes_only_that_page(tmp_path, monkeypatc out_path.read_text(encoding="utf-8") == ctx.compiled_pages[edited_route].output_code ) + # The same-module hit page contributed no memos, so it is reused, not + # recompiled (its output was never written to this fresh web dir). + hit_output_path = ctx.compiled_pages[pages[1].route].output_path + assert hit_output_path is not None + assert not compiler_utils.resolve_path_of_web_dir(hit_output_path).exists() def test_stateful_hit_is_marked_but_not_reevaluated(tmp_path, monkeypatch): @@ -273,9 +304,7 @@ def test_stateful_hit_is_marked_but_not_reevaluated(tmp_path, monkeypatch): re-evaluates the marked stateful pages itself, so re-evaluating them during the incremental rebuild was pure waste. """ - web = tmp_path / ".web" - web.mkdir() - monkeypatch.setattr(disk_cache.prerequisites, "get_web_dir", lambda: web) + web = _use_tmp_web_dir(tmp_path, monkeypatch) app = rx.App() app.add_page(_page_a, route="/a") @@ -305,25 +334,208 @@ def test_stateful_hit_is_marked_but_not_reevaluated(tmp_path, monkeypatch): assert route in app._stateful_pages +def test_incremental_rebuild_regenerates_contexts(tmp_path, monkeypatch): + """State defaults/client-storage are baked into the contexts file and a + state-module edit never bumps the epoch, so the incremental path must + always re-emit it. + """ + from reflex.compiler import utils as compiler_utils + + _use_tmp_web_dir(tmp_path, monkeypatch) + + app = rx.App() + app.add_page(_page_a, route="/a") + pages = list(app._unevaluated_pages.values()) + ctx = _compile(pages) + disk_cache.write_manifest(ctx, pages, ctx.all_imports, root=tmp_path) + _stub_externals(app, monkeypatch) + + assert ( + disk_cache.try_incremental_rebuild( + app, compiler_plugins=[], prerender_routes=False, root=tmp_path + ) + is True + ) + + out_path = compiler_utils.resolve_path_of_web_dir(compiler_utils.get_context_path()) + assert out_path.exists() + assert out_path.read_text(encoding="utf-8") == _CONTEXTS_STUB + + +def test_incremental_rebuild_copies_assets(tmp_path, monkeypatch): + """An assets-only edit is an all-hit rebuild, so the incremental path must + run the same assets -> public copy as the full compile. + """ + web = _use_tmp_web_dir(tmp_path, monkeypatch) + assets = tmp_path / "assets" + assets.mkdir() + (assets / "logo.svg").write_text("<svg/>") + + app = rx.App() + app.add_page(_page_a, route="/a") + pages = list(app._unevaluated_pages.values()) + ctx = _compile(pages) + disk_cache.write_manifest(ctx, pages, ctx.all_imports, root=tmp_path) + _stub_externals(app, monkeypatch) + + assert ( + disk_cache.try_incremental_rebuild( + app, compiler_plugins=[], prerender_routes=False, root=tmp_path + ) + is True + ) + + assert (web / "public" / "logo.svg").read_text() == "<svg/>" + + +@rx.memo +def _badge(text: str) -> Component: + return rx.el.span(text, class_name="badge") + + +def _page_with_user_memo() -> Component: + return rx.el.div(rx.el.h1("Memo page"), _badge(text="hello")) + + +def test_incremental_rebuild_rewrites_changed_user_memo( + tmp_path, monkeypatch, preserve_memo_registries +): + """Editing a user ``@rx.memo`` module must rewrite its mirrored memo file. + + The memo's module file is in the dep set of every page that imports it, so + those pages miss — but only auto-memo contributions were being written, + leaving the user memo's generated JS stale. + """ + from reflex.compiler import compiler + from reflex.compiler import utils as compiler_utils + + web = _use_tmp_web_dir(tmp_path, monkeypatch) + + app = rx.App() + app.add_page(_page_with_user_memo, route="/memo") + pages = list(app._unevaluated_pages.values()) + memo_route = pages[0].route + ctx = _compile(pages) + disk_cache.write_manifest(ctx, pages, ctx.all_imports, root=tmp_path) + _stub_externals(app, monkeypatch) + + # Simulate an edit to this module (which defines the user memo): record a + # stale hash for its file so the page importing the memo misses. + module_file = str(Path(__file__).resolve()) + manifest_path = web / disk_cache._MANIFEST_FILE + manifest = json.loads(manifest_path.read_text()) + manifest["pages"][memo_route]["dep_hashes"] = {module_file: "stale-hash"} + manifest_path.write_text(json.dumps(manifest)) + + assert ( + disk_cache.try_incremental_rebuild( + app, compiler_plugins=[], prerender_routes=False, root=tmp_path + ) + is True + ) + + # The user memo's mirrored file was re-emitted with its export. + from reflex_base.components.memo import MEMOS, MemoComponentDefinition + + badge_def = next(m for m in MEMOS.values() if m.source_module == __name__) + assert isinstance(badge_def, MemoComponentDefinition) + memo_files, _ = compiler.compile_memo_components([badge_def]) + assert memo_files + for mpath, _mcode in memo_files: + out_path = compiler_utils.resolve_path_of_web_dir(mpath) + assert out_path.exists() + assert badge_def.export_name in out_path.read_text(encoding="utf-8") + + +class _MemoCacheState(rx.State): + value: str = "x" + other: str = "y" + + +def _page_e() -> Component: + return rx.el.div(rx.el.p(_MemoCacheState.value, class_name="e"), rx.el.h1("E")) + + +def _page_f() -> Component: + return rx.el.div(rx.el.p(_MemoCacheState.other, class_name="f"), rx.el.h2("F")) + + +def test_incremental_miss_keeps_sibling_memo_exports( + tmp_path, monkeypatch, preserve_memo_registries +): + """A miss must not clobber memo exports owned by hit pages. + + Auto-memo output is grouped into one file per source module. Pages E and F + live in this module and each contributes a stateful auto memo, so both land + in the same mirrored file. When only E misses (e.g. a data-file edit), + rewriting that file from E's contributions alone drops F's export while + F's reused page module still imports it. + """ + from reflex.compiler import compiler + from reflex.compiler import utils as compiler_utils + + web = _use_tmp_web_dir(tmp_path, monkeypatch) + + app = rx.App() + app.add_page(_page_e, route="/e") + app.add_page(_page_f, route="/f") + pages = list(app._unevaluated_pages.values()) + route_e, route_f = pages[0].route, pages[1].route + ctx = _compile(pages) + + e_memos = list(ctx.compiled_pages[route_e].memo_contributions.values()) + f_memos = list(ctx.compiled_pages[route_f].memo_contributions.values()) + assert e_memos, "page E must contribute an auto memo" + assert f_memos, "page F must contribute an auto memo" + + # Simulate the full compile's on-disk memo state (shared grouped file). + memo_files, _ = compiler.compile_memo_components([*e_memos, *f_memos]) + assert memo_files + for mpath, mcode in memo_files: + compiler_utils.write_file(compiler_utils.resolve_path_of_web_dir(mpath), mcode) + + disk_cache.write_manifest(ctx, pages, ctx.all_imports, root=tmp_path) + _stub_externals(app, monkeypatch) + + # Make only page E miss (a dependency of E changed, e.g. a data file). + manifest_path = web / disk_cache._MANIFEST_FILE + manifest = json.loads(manifest_path.read_text()) + manifest["pages"][route_e]["dep_hashes"] = {str(tmp_path / "data.md"): "stale-hash"} + manifest_path.write_text(json.dumps(manifest)) + + assert ( + disk_cache.try_incremental_rebuild( + app, compiler_plugins=[], prerender_routes=False, root=tmp_path + ) + is True + ) + + # Every export (E's and F's) survives in the rewritten grouped file(s). + for mpath, _mcode in memo_files: + content = compiler_utils.resolve_path_of_web_dir(mpath).read_text( + encoding="utf-8" + ) + for memo_def in (*e_memos, *f_memos): + assert memo_def.export_name in content + + def test_load_manifest_rejects_wrong_schema(tmp_path, monkeypatch): - web = tmp_path / ".web" - web.mkdir() - monkeypatch.setattr(disk_cache.prerequisites, "get_web_dir", lambda: web) + web = _use_tmp_web_dir(tmp_path, monkeypatch) (web / disk_cache._MANIFEST_FILE).write_text(json.dumps({"schema": 999})) assert disk_cache.load_manifest() is None def test_update_manifest_for_misses_keeps_complete_imports(tmp_path, monkeypatch): - web = tmp_path / ".web" - web.mkdir() - monkeypatch.setattr(disk_cache.prerequisites, "get_web_dir", lambda: web) + web = _use_tmp_web_dir(tmp_path, monkeypatch) monkeypatch.setattr( page_cache, "state_dependency_index", lambda root=None: ({}, set()) ) monkeypatch.setattr(page_cache, "page_dependency_hashes", lambda *a, **k: {}) page = _FakePage(route="/a", component=_page_a) - page_ctx = SimpleNamespace(app_wrap_components={}, frontend_imports={}) + page_ctx = SimpleNamespace( + app_wrap_components={}, frontend_imports={}, memo_contributions={} + ) miss_ctx = SimpleNamespace(compiled_pages={"/a": page_ctx}, stateful_routes=set()) complete_imports = {"memo-lib": [ImportVar("MemoThing")]} manifest = _manifest({ diff --git a/tests/units/compiler/test_page_cache.py b/tests/units/compiler/test_page_cache.py index 45d84e4e7c1..aa4a4578093 100644 --- a/tests/units/compiler/test_page_cache.py +++ b/tests/units/compiler/test_page_cache.py @@ -352,6 +352,30 @@ def test_record_reads_tracks_executed_builtin_import(tmp_path, monkeypatch): assert str(child.resolve()) in reads +def test_record_reads_tracks_path_open(tmp_path): + """``Path.open`` calls ``io.open`` directly, bypassing the ``builtins.open`` + patch, so it must be patched itself for data reads to be recorded. + """ + page_cache.enable_read_tracking(root=tmp_path) + data = tmp_path / "content.md" + data.write_text("hello") + + with page_cache.record_reads() as reads, data.open() as f: + f.read() + + assert str(data.resolve()) in reads + + +def test_record_reads_ignores_path_open_writes(tmp_path): + page_cache.enable_read_tracking(root=tmp_path) + out = tmp_path / "out.md" + + with page_cache.record_reads() as reads, out.open("w") as f: + f.write("x") + + assert str(out.resolve()) not in reads + + def test_record_reads_ignores_unexecuted_import(tmp_path, monkeypatch): module_name = "uncalled_runtime_import_dep" module_file = _prepare_runtime_module(tmp_path, monkeypatch, module_name) From 14e5a3cd98acac2fc3721ce7ebe6605174579f3b Mon Sep 17 00:00:00 2001 From: Farhan Ali Raza <62690310+FarhanAliRaza@users.noreply.github.com> Date: Fri, 3 Jul 2026 01:28:56 +0000 Subject: [PATCH 15/56] fix(compiler): make compile output deterministic so dev HMR stays granular --- .../src/reflex_base/utils/imports.py | 10 ++--- .../src/reflex_base/utils/serializers.py | 9 ++++- reflex/app.py | 8 +++- reflex/compiler/compiler.py | 19 +++++---- tests/units/compiler/test_compiler.py | 40 +++++++++++++++++++ tests/units/utils/test_imports.py | 22 ++++++++++ tests/units/utils/test_serializers.py | 12 ++++++ 7 files changed, 104 insertions(+), 16 deletions(-) diff --git a/packages/reflex-base/src/reflex_base/utils/imports.py b/packages/reflex-base/src/reflex_base/utils/imports.py index 7e614f99524..0ec426f78fe 100644 --- a/packages/reflex-base/src/reflex_base/utils/imports.py +++ b/packages/reflex-base/src/reflex_base/utils/imports.py @@ -91,6 +91,10 @@ def collapse_imports( ) -> ParsedImportDict: """Remove all duplicate ImportVar within an ImportDict. + Deduplication preserves first-occurrence order: compiled import statements + follow this order, and a hash-seed-dependent order would rewrite every + page/memo module on each dev reload, defeating granular HMR. + Args: imports: The import dict to collapse. @@ -98,11 +102,7 @@ def collapse_imports( The collapsed import dict. """ return { - lib: ( - list(set(import_vars)) - if isinstance(import_vars, list) - else list(import_vars) - ) + lib: list(dict.fromkeys(import_vars)) for lib, import_vars in ( imports if isinstance(imports, tuple) else imports.items() ) diff --git a/packages/reflex-base/src/reflex_base/utils/serializers.py b/packages/reflex-base/src/reflex_base/utils/serializers.py index d20a29e3890..3a5be37ee53 100644 --- a/packages/reflex-base/src/reflex_base/utils/serializers.py +++ b/packages/reflex-base/src/reflex_base/utils/serializers.py @@ -292,13 +292,20 @@ def serialize_base_model(model: BaseModel) -> dict: def serialize_set(value: set) -> list: """Serialize a set to a JSON serializable list. + Sets have no meaningful order and their iteration order varies with the + per-process hash seed, so sort when possible to keep serialized output + (compiled JSX, config fingerprints) stable across processes. + Args: value: The set to serialize. Returns: The serialized list. """ - return list(value) + try: + return sorted(value) + except TypeError: + return list(value) @serializer diff --git a/reflex/app.py b/reflex/app.py index 31bb0341409..3a131c6728f 100644 --- a/reflex/app.py +++ b/reflex/app.py @@ -1346,8 +1346,12 @@ def _write_stateful_pages_marker(self): prerequisites.get_backend_dir() / constants.Dirs.STATEFUL_PAGES ) stateful_pages_marker.parent.mkdir(parents=True, exist_ok=True) - with stateful_pages_marker.open("w") as f: - json.dump(list(self._stateful_pages), f) + content = json.dumps(list(self._stateful_pages)) + if ( + not stateful_pages_marker.exists() + or stateful_pages_marker.read_text() != content + ): + stateful_pages_marker.write_text(content) def add_all_routes_endpoint(self): """Add an endpoint to the app that returns all the routes.""" diff --git a/reflex/compiler/compiler.py b/reflex/compiler/compiler.py index 2b368cd5c7b..98582b8df25 100644 --- a/reflex/compiler/compiler.py +++ b/reflex/compiler/compiler.py @@ -378,19 +378,22 @@ def _compile_root_stylesheet( target.parent.mkdir(parents=True, exist_ok=True) + # Skip rewriting an unchanged target: Vite watches .web/styles, and a + # rewrite-with-identical-content still fires an HMR update per reload. if stylesheet.suffix == ".css": - path_ops.cp(src=stylesheet, dest=target, overwrite=True) + data = stylesheet.read_bytes() + if not target.exists() or target.read_bytes() != data: + target.write_bytes(data) else: try: from sass import compile as sass_compile - target.write_text( - data=sass_compile( - filename=str(stylesheet), - output_style="compressed", - ), - encoding="utf8", - ) + compiled_css = sass_compile( + filename=str(stylesheet), + output_style="compressed", + ).encode("utf8") + if not target.exists() or target.read_bytes() != compiled_css: + target.write_bytes(compiled_css) except ImportError: failed_to_import_sass = True diff --git a/tests/units/compiler/test_compiler.py b/tests/units/compiler/test_compiler.py index 1dbb4ab27bc..47d6c636766 100644 --- a/tests/units/compiler/test_compiler.py +++ b/tests/units/compiler/test_compiler.py @@ -124,6 +124,46 @@ def test_compile_imports(import_dict: ParsedImportDict, test_dicts: list[dict]): ) +def test_compile_stylesheets_skips_unchanged_copy( + tmp_path: Path, mocker: MockerFixture +): + """An unchanged asset stylesheet is not rewritten into .web on recompile. + + Vite watches .web/styles, so a rewrite with identical content still fires + an HMR update on every dev reload. + + Args: + tmp_path: The test directory. + mocker: Pytest mocker object. + """ + project = tmp_path / "test_project" + project.mkdir() + assets_dir = project / "assets" + assets_dir.mkdir() + (assets_dir / "style.css").write_text("button { color: blue }") + + mocker.patch("reflex.compiler.compiler.Path.cwd", return_value=project) + mocker.patch( + "reflex.compiler.compiler.get_web_dir", + return_value=project / constants.Dirs.WEB, + ) + mocker.patch( + "reflex.compiler.utils.get_web_dir", return_value=project / constants.Dirs.WEB + ) + + compiler.compile_root_stylesheet(["/style.css"]) + target = project / constants.Dirs.WEB / "styles" / "style.css" + stat_before = target.stat() + + compiler.compile_root_stylesheet(["/style.css"]) + stat_after = target.stat() + assert stat_after.st_mtime_ns == stat_before.st_mtime_ns + + (assets_dir / "style.css").write_text("button { color: red }") + compiler.compile_root_stylesheet(["/style.css"]) + assert target.read_text() == "button { color: red }" + + def test_compile_stylesheets(tmp_path: Path, mocker: MockerFixture): """Test that stylesheets compile correctly. diff --git a/tests/units/utils/test_imports.py b/tests/units/utils/test_imports.py index 7b456d892c9..86720222b58 100644 --- a/tests/units/utils/test_imports.py +++ b/tests/units/utils/test_imports.py @@ -3,6 +3,7 @@ ImportDict, ImportVar, ParsedImportDict, + collapse_imports, merge_imports, parse_imports, ) @@ -118,3 +119,24 @@ def test_merge_imports(input_1, input_2, output): ) def test_parse_imports(input: ImportDict, output: ParsedImportDict): assert parse_imports(input) == output + + +def test_collapse_imports_preserves_order(): + """Deduplication must preserve first-occurrence order. + + Compiled JSX import order follows this ordering; a hash-seed-dependent + order rewrites every page/memo file on each dev reload and breaks + granular HMR. + """ + import_vars = [ + ImportVar(tag=f"Icon{i}", is_default=True, package_path=f"/Icon{i}") + for i in range(32) + ] + duplicated = [*import_vars, *import_vars[:5], import_vars[0]] + collapsed = collapse_imports({"@hugeicons/core-free-icons": duplicated}) + assert collapsed == {"@hugeicons/core-free-icons": import_vars} + # Tuple-valued entries (already-immutable parsed imports) keep order too. + collapsed_tuple = collapse_imports(( + ("@hugeicons/core-free-icons", tuple(duplicated)), + )) + assert collapsed_tuple == {"@hugeicons/core-free-icons": import_vars} diff --git a/tests/units/utils/test_serializers.py b/tests/units/utils/test_serializers.py index 01e77d64cc9..142966dbffc 100644 --- a/tests/units/utils/test_serializers.py +++ b/tests/units/utils/test_serializers.py @@ -207,6 +207,18 @@ def test_serialize(value: Any, expected: str): assert json.loads(json_dumps(value)) == json.loads(json_dumps(expected)) +def test_serialize_set_deterministic_order(): + """Sortable sets serialize in sorted order regardless of the hash seed. + + Compiled output and cache fingerprints embed serialized sets; a + hash-seed-dependent order churns files on every process restart. + """ + assert serializers.serialize_set({"b", "c", "a"}) == ["a", "b", "c"] + assert serializers.serialize_set({3, 1, 2}) == [1, 2, 3] + # Unsortable element mixes fall back to iteration order. + assert set(serializers.serialize_set({1, "a"})) == {1, "a"} + + @pytest.mark.parametrize( ("value", "expected", "exp_var_is_string"), [ From bf6a8bf919cd2a83dd9cd9a1ee1ef2528d06e5aa Mon Sep 17 00:00:00 2001 From: Farhan <www.mfarvirus@gmail.com> Date: Fri, 3 Jul 2026 17:20:01 +0500 Subject: [PATCH 16/56] fix(compiler): keep contexts file and HMR intact across hot reloads MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The incremental rebuild no longer re-emits the contexts file every time. It is rewritten only when a stateful page missed and that page's own state config actually changed (fingerprinted against the manifest); otherwise the on-disk file is reused untouched. When a rewrite is needed, the stateful hit pages are evaluated first so the state registry — and the frontend dispatch map compiled from it — stays complete. Supporting changes: - Reset the daemon's state registry surgically: states from modules that survive the purge (framework/installed/workspace packages) are re-registered in original order, while purged-module and reflex.istate.dynamic states are dropped so re-created local states get deterministic fresh-process names. - Patch react-router's served HMR runtime so an edit to a not-currently-open route no longer throws and poisons HMR until a full reload. - Store per-input epoch digests instead of one combined hash so a global-input mismatch can name the exact file that changed, and surface fallback reasons and detected file changes at info level. - Write the vite config atomically and regenerate it on the incremental path. --- .../src/reflex_base/compiler/templates.py | 27 ++ .../src/reflex_base/plugins/compiler.py | 11 +- reflex/compiler/compiler.py | 2 +- reflex/compiler/disk_cache.py | 319 +++++++++++++++--- reflex/compiler/page_cache.py | 83 ++++- reflex/utils/compile_daemon.py | 35 +- reflex/utils/frontend_skeleton.py | 6 +- tests/units/compiler/test_disk_cache.py | 309 ++++++++++++++++- tests/units/reflex_base/compiler/__init__.py | 0 .../reflex_base/compiler/test_templates.py | 53 +++ tests/units/utils/test_compile_daemon.py | 103 +++++- 11 files changed, 864 insertions(+), 84 deletions(-) create mode 100644 tests/units/reflex_base/compiler/__init__.py create mode 100644 tests/units/reflex_base/compiler/test_templates.py diff --git a/packages/reflex-base/src/reflex_base/compiler/templates.py b/packages/reflex-base/src/reflex_base/compiler/templates.py index 8e6e63a2bb3..5f3c06f021f 100644 --- a/packages/reflex-base/src/reflex_base/compiler/templates.py +++ b/packages/reflex-base/src/reflex_base/compiler/templates.py @@ -609,12 +609,39 @@ def vite_config_template( }}; }} +// react-router's HMR client (refresh-utils.mjs enqueueUpdate) throws when an +// update batch includes a route the browser hasn't loaded, and the throw skips +// the queue cleanup below it — one edit to any not-currently-open page then +// poisons HMR until a full page reload. Rewrite the served runtime so unloaded +// routes keep their manifest metadata update but stay lazy. +function patchReactRouterHmrRuntime() {{ + const unloadedRouteThrow = /if\s*\(!imported\)\s*\{{\s*throw\s+Error\([\s\S]*?\);\s*\}}/; + return {{ + name: "reflex-patch-react-router-hmr-runtime", + apply: "serve", + transform(code, id) {{ + if (id !== "\0virtual:react-router/hmr-runtime") return; + if (!unloadedRouteThrow.test(code)) {{ + this.warn( + "react-router hmr runtime changed; unloaded-route HMR patch skipped", + ); + return; + }} + return {{ + code: code.replace(unloadedRouteThrow, "if (!imported) continue;"), + map: null, + }}; + }}, + }}; +}} + export default defineConfig((config) => ({{ base: "{base}", plugins: [ alwaysUseReactDomServerNode(), reactRouter(), safariCacheBustPlugin(), + patchReactRouterHmrRuntime(), ].concat({"[fullReload()]" if force_full_reload else "[]"}), build: {{ sourcemap: {"true" if sourcemap is True else "false" if sourcemap is False else repr(sourcemap)}, diff --git a/packages/reflex-base/src/reflex_base/plugins/compiler.py b/packages/reflex-base/src/reflex_base/plugins/compiler.py index 97e4f0d9e36..813af943b1d 100644 --- a/packages/reflex-base/src/reflex_base/plugins/compiler.py +++ b/packages/reflex-base/src/reflex_base/plugins/compiler.py @@ -773,7 +773,9 @@ class CompileContext(BaseContext): app_wrap_components: dict[tuple[int, str], Component] = dataclasses.field( default_factory=dict ) - stateful_routes: dict[str, None] = dataclasses.field(default_factory=dict) + # Routes whose evaluation defined new state classes, mapped to the full + # names of the states each page defined. + stateful_routes: dict[str, list[str]] = dataclasses.field(default_factory=dict) # Auto-memoize wrapper tags seen during the tree walk (populated by # ``MemoizeStatefulPlugin``). memoize_wrappers: dict[str, None] = dataclasses.field(default_factory=dict) @@ -851,7 +853,12 @@ def compile( raise RuntimeError(msg) if len(all_base_state_classes) > n_states_before: - self.stateful_routes[page.route] = None + # Record which states this page defined (registration order is + # insertion order), so the compile cache can fingerprint the + # page's contribution to the contexts file. + self.stateful_routes[page.route] = list(all_base_state_classes)[ + n_states_before: + ] self.compiled_pages[page_ctx.route] = page_ctx diff --git a/reflex/compiler/compiler.py b/reflex/compiler/compiler.py index 98582b8df25..05908652f4f 100644 --- a/reflex/compiler/compiler.py +++ b/reflex/compiler/compiler.py @@ -1220,7 +1220,7 @@ def compile_app( raise TypeError(msg) app._pages[route] = page_ctx.root_component - app._stateful_pages.update(compile_ctx.stateful_routes) + app._stateful_pages.update(dict.fromkeys(compile_ctx.stateful_routes)) app._write_stateful_pages_marker() app._add_optional_endpoints() app._validate_var_dependencies() diff --git a/reflex/compiler/disk_cache.py b/reflex/compiler/disk_cache.py index 7e2cfc785c1..a1a966aba31 100644 --- a/reflex/compiler/disk_cache.py +++ b/reflex/compiler/disk_cache.py @@ -8,7 +8,9 @@ from __future__ import annotations +import contextlib import dataclasses +import hashlib import json import sys from pathlib import Path @@ -16,6 +18,7 @@ from reflex_base import constants from reflex_base.plugins import CompileContext, CompilerHooks +from reflex_base.utils.format import json_dumps from reflex_base.utils.imports import ImportVar, merge_imports from reflex.compiler import page_cache @@ -23,7 +26,7 @@ from reflex.utils import console, path_ops, prerequisites if TYPE_CHECKING: - from collections.abc import Callable, Sequence + from collections.abc import Callable, Iterable, Sequence from reflex_base.plugins import PageContext, PageDefinition from reflex_base.utils.imports import ParsedImportDict @@ -31,7 +34,7 @@ from reflex.app import App #: Bump when the manifest layout changes (old manifests are then ignored). -_SCHEMA = 4 +_SCHEMA = 6 #: Manifest filename under the web directory. _MANIFEST_FILE = "reflex_compile_cache.json" @@ -40,6 +43,40 @@ def _manifest_path() -> Path: return prerequisites.get_web_dir() / _MANIFEST_FILE +def format_path_list( + items: Iterable[str], root: Path | None = None, limit: int = 5 +) -> str: + """Render a bounded, root-relative summary of a path/label collection. + + Args: + items: The paths (or labels) to render. + root: When given, paths under it are shown relative to it. + limit: Maximum number of entries to show before truncating. + + Returns: + A comma-separated summary string, truncated with ``(+N more)``. + """ + + def rel(item: str) -> str: + if root is not None: + with contextlib.suppress(ValueError): + return str(Path(item).relative_to(root)) + return item + + shown = sorted(rel(item) for item in items) + extra = len(shown) - limit + return ", ".join(shown[:limit]) + (f" (+{extra} more)" if extra > 0 else "") + + +def _log_fallback(reason: str) -> None: + """Report why the incremental rebuild fell back to a full compile. + + Args: + reason: The human-readable fallback reason. + """ + console.info(f"Compile cache: falling back to a full compile — {reason}") + + def _serialize_imports(imports: ParsedImportDict) -> dict[str, list[dict[str, Any]]]: """Serialize a parsed import dict to JSON-able primitives. @@ -83,6 +120,7 @@ def _manifest_page_entry( hasher: Callable[[str], str | None], *, is_stateful: bool, + state_fingerprint: str | None = None, root: Path | None = None, ) -> dict[str, Any]: """Build the manifest entry for one compiled page. @@ -93,6 +131,8 @@ def _manifest_page_entry( state_index: The state-context identifier -> file index. hasher: A memoized path -> content-hash function. is_stateful: Whether the page registered state during compile. + state_fingerprint: Fingerprint of the page's contexts contribution + (see ``_contexts_fingerprint``), or None for stateless pages. root: Project root for dependency discovery. Defaults to cwd. Returns: @@ -104,6 +144,7 @@ def _manifest_page_entry( ), "app_wrap_keys": _wrap_key_strs(page_ctx.app_wrap_components.keys()), "is_stateful": is_stateful, + "state_fingerprint": state_fingerprint, # Whether the page contributed auto memos: pages sharing a source # module share one memo output file, so a memo-contributing hit page # must be recompiled alongside a same-module miss (see @@ -112,6 +153,92 @@ def _manifest_page_entry( } +def _contexts_fingerprint( + state_names: Sequence[str], + initial_state: dict[str, Any], + client_storage: dict[str, dict[str, Any]], +) -> str: + """Fingerprint some states' contribution to the compiled contexts file. + + A state's contribution is exactly its initial-state slice plus its + client-storage entries (see ``templates.context_template``), so equal + fingerprints mean re-emitting the contexts file would leave these states' + entries unchanged. + + Args: + state_names: Full names of the states to fingerprint. + initial_state: The complete initial-state mapping (full name -> vars). + client_storage: The compiled client-storage mapping per storage kind. + + Returns: + A stable hash of the states' contexts contribution. + """ + payload = [] + for name in sorted(state_names): + prefix = f"{name}." + payload.append(( + name, + initial_state.get(name), + { + kind: {k: v for k, v in entries.items() if k.startswith(prefix)} + for kind, entries in client_storage.items() + }, + )) + return hashlib.sha256(json_dumps(payload).encode()).hexdigest() + + +def _contexts_snapshot( + app: App | None, +) -> tuple[dict[str, Any], dict[str, Any]] | None: + """Capture the state-tree inputs of the contexts file. + + Args: + app: The app being compiled (absent in bare compile contexts). + + Returns: + The (initial state, client storage) mappings keyed by state full name, + or None when there is no state tree. + """ + if app is None or app._state is None: + return None + from reflex.compiler import utils as compiler_utils + + return ( + compiler_utils.compile_state(app._state), + compiler_utils.compile_client_storage(app._state), + ) + + +def _changed_state_config_route( + manifest: dict[str, Any], + miss_ctx: CompileContext, + snapshot: tuple[dict[str, Any], dict[str, Any]] | None, +) -> str | None: + """Find the first recompiled stateful route whose state config changed. + + Compares each stateful miss page's just-evaluated states against the + fingerprint recorded in the manifest. Only these pages' states can differ + from the on-disk contexts file: hit pages' definitions are unchanged by + construction (all their dependency files still match). + + Args: + manifest: The loaded compile manifest. + miss_ctx: The compile context of the recompiled pages. + snapshot: The ``_contexts_snapshot`` of the app, or None when there is + no state tree to fingerprint against. + + Returns: + The first route requiring a contexts rebuild, or None if none do. + """ + if snapshot is None: + return next(iter(miss_ctx.stateful_routes)) + for route, defined_states in miss_ctx.stateful_routes.items(): + stored = manifest["pages"].get(route, {}).get("state_fingerprint") + if stored != _contexts_fingerprint(defined_states, *snapshot): + return route + return None + + def load_manifest() -> dict[str, Any] | None: """Load the persisted compile manifest, or None if absent/unusable. @@ -154,7 +281,10 @@ def write_manifest( try: state_index, _ = page_cache.state_dependency_index(root) hasher = page_cache.make_hasher() - epoch = page_cache.global_epoch(root, pages=pages) + epoch_inputs = page_cache.global_epoch_inputs(root, pages=pages) + contexts_snapshot = ( + _contexts_snapshot(compile_ctx.app) if compile_ctx.stateful_routes else None + ) pages_data: dict[str, Any] = {} for page in pages: @@ -165,19 +295,27 @@ def write_manifest( or page_ctx.output_path is None ): return # incomplete compile -> do not write a partial manifest + defined_states = compile_ctx.stateful_routes.get(page.route) pages_data[page.route] = _manifest_page_entry( page_ctx, page.component, state_index, hasher, - is_stateful=page.route in compile_ctx.stateful_routes, + is_stateful=defined_states is not None, + state_fingerprint=( + _contexts_fingerprint(defined_states, *contexts_snapshot) + if defined_states and contexts_snapshot is not None + else None + ), root=root, ) manifest = { "schema": _SCHEMA, "reflex_version": page_cache._reflex_version(), - "epoch": epoch, + # Per-input digests (not one combined sha) so a later mismatch can + # name the exact global input that changed. + "epoch_inputs": epoch_inputs, "all_imports": _serialize_imports(install_imports), "pages": pages_data, } @@ -188,35 +326,54 @@ def write_manifest( console.debug(f"disk compile cache: manifest write skipped ({exc!r})") -def globals_match( +def globals_mismatch( manifest: dict[str, Any], *, routes: set[str], - epoch: str, -) -> bool: - """Whether the manifest's genuinely-global inputs match the current compile. + root: Path | None = None, +) -> str | None: + """Explain why the manifest's global inputs don't match, or None if they do. The fast rebuild needs the route set unchanged (adding/removing a route - changes the shared nav on every page) and the global epoch unchanged (Reflex + changes the shared nav on every page) and the global inputs unchanged (Reflex version + config/lockfiles + the app-level config files: the entrypoint and the theme/app-wrap/stylesheet modules it imports, which configure the app-wide files reused on disk). Everything else is decided per page via its dependency set, so a shared-component or markdown edit no longer blocks the fast path. Only the pages that depend on the changed file miss. + Global inputs are validated by re-hashing the manifest's *stored* input set + (see :func:`page_cache.changed_epoch_inputs`), never by recomputing the set + in this process — set membership is only decided when a full compile writes + the manifest. + Args: manifest: The loaded manifest. routes: The current set of page routes. - epoch: The current global epoch (see :func:`page_cache.global_epoch`). + root: Project root the stored inputs resolve against (also used to + shorten paths in the reason). Defaults to cwd. Returns: - True if the global inputs match. + A human-readable mismatch reason, or None when the global inputs match. """ - return ( - manifest.get("reflex_version") == page_cache._reflex_version() - and set(manifest.get("pages", {})) == routes - and manifest.get("epoch") == epoch - ) + old_version = manifest.get("reflex_version") + if old_version != page_cache._reflex_version(): + return ( + f"reflex version changed ({old_version} -> {page_cache._reflex_version()})" + ) + old_routes = set(manifest.get("pages", {})) + if old_routes != routes: + parts = [] + if added := routes - old_routes: + parts.append(f"added {format_path_list(added)}") + if removed := old_routes - routes: + parts.append(f"removed {format_path_list(removed)}") + return f"route set changed ({'; '.join(parts)})" + root = (root or Path.cwd()).resolve() + if stale := page_cache.changed_epoch_inputs(manifest.get("epoch_inputs", {}), root): + labels = {label.removeprefix("app:") for label in stale} + return f"global input(s) changed: {format_path_list(labels, root)}" + return None def partition_pages( @@ -378,11 +535,11 @@ def try_incremental_rebuild( unsafe to reuse: no/old manifest, a changed global input, a route change, or a miss page that altered its app-wrap set or stateful flag. - App-wide outputs that per-page dependency sets do not cover are always - re-emitted rather than tracked: the contexts file (state defaults and - client-storage config change with any state module) and the ``assets`` - copy (assets are excluded from dependency tracking). Both are cheap and - idempotent. + The ``assets`` copy is excluded from dependency tracking and always re-run + (cheap, idempotent). The contexts file is rewritten only when a stateful + page missed — and then only after evaluating the stateful hit pages, so the + state registry it is compiled from is complete (it must keep the states + that only a page's evaluation registers, e.g. exec'd docs demos). On success, reports (at info level) how many pages were recompiled vs reused and, while recompiling, shows a progress bar over the changed pages so a hot @@ -401,19 +558,33 @@ def try_incremental_rebuild( """ manifest = load_manifest() if manifest is None: + _log_fallback( + "no reusable manifest (first compile, unreadable, or schema changed)" + ) return False pages = list(app._unevaluated_pages.values()) routes = {p.route for p in pages} hasher = page_cache.make_hasher() - epoch = page_cache.global_epoch(root, pages=pages) - if not globals_match(manifest, routes=routes, epoch=epoch): + if (reason := globals_mismatch(manifest, routes=routes, root=root)) is not None: + _log_fallback(reason) return False + resolved_root = (root or Path.cwd()).resolve() miss_pages = partition_pages(pages, manifest, hasher) + changed_files: set[str] = set() if miss_pages: + # Nearly free: partition_pages already hashed every dependency file + # into the memoized hasher. + changed_files = _changed_dependency_files(manifest, hasher) miss_pages = _with_module_siblings(miss_pages, pages, manifest) + console.info( + f"Compile cache: recompiling {len(miss_pages)}/{len(pages)} pages; " + f"changed file(s): {format_path_list(changed_files, resolved_root)}" + ) + else: + console.info(f"Compile cache: reusing all {len(pages)} pages from disk") miss_routes = {p.route for p in miss_pages} # Recompile only the source-changed pages. @@ -464,14 +635,17 @@ def try_incremental_rebuild( or page_ctx.output_code is None or page_ctx.output_path is None ): + _log_fallback(f"page {page.route!r} produced no output") return False entry = manifest["pages"][page.route] if ( _wrap_key_strs(page_ctx.app_wrap_components.keys()) != entry["app_wrap_keys"] ): + _log_fallback(f"page {page.route!r} changed its app-wrap set") return False if (page.route in miss_ctx.stateful_routes) != entry["is_stateful"]: + _log_fallback(f"page {page.route!r} changed statefulness") return False from reflex.compiler import compiler @@ -486,6 +660,7 @@ def try_incremental_rebuild( output_path = page_ctx.output_path output_code = page_ctx.output_code if output_path is None or output_code is None: + _log_fallback(f"page {page.route!r} lost its output before write") return False compiler.utils.write_file( compiler.utils.resolve_path_of_web_dir(output_path), @@ -497,9 +672,7 @@ def try_incremental_rebuild( # with the complete definition set (all recompiled pages' contributions # plus the user memos sharing those files or whose module changed). memo_files, memo_imports = compiler.compile_memo_components( - _complete_memo_defs( - memo_contributions, _changed_dependency_files(manifest, hasher) - ) + _complete_memo_defs(memo_contributions, changed_files) ) for mpath, mcode in memo_files: compiler.utils.write_file( @@ -527,24 +700,58 @@ def try_incremental_rebuild( app._add_optional_endpoints() app._validate_var_dependencies() - # App-wide outputs that are cheap to re-emit and not gated by the epoch: - # contexts (state defaults/client-storage change with any state module, - # which per-page dependency sets do not force a regenerate of) and the - # assets copy (assets are excluded from dependency tracking entirely). - from reflex_components_radix.plugin import RadixThemesPlugin - - theme = next( - ( - plugin.get_theme() - for plugin in compiler_plugins - if isinstance(plugin, RadixThemesPlugin) - ), - None, - ) - context_path, context_code = compiler.compile_contexts(app._state, theme) - compiler.utils.write_file( - compiler.utils.resolve_path_of_web_dir(context_path), context_code - ) + # The contexts file holds EVERY state's defaults/dispatchers, including + # states only registered while their page evaluates (exec'd docs demos, + # dynamically imported modules). This process evaluated just the miss + # pages, so its registry is incomplete; rewriting contexts from it would + # drop the hit pages' states and break the frontend's dispatch map. Only a + # stateful miss can change state config — and only its OWN states can + # differ from the on-disk contexts file, so first fingerprint those + # against the manifest: a content-only edit leaves them identical and the + # contexts file is reused untouched. Otherwise evaluate the stateful hit + # pages (so the registry is complete) and rewrite contexts. + contexts_snapshot: tuple[dict[str, Any], dict[str, Any]] | None = None + if miss_ctx is not None and miss_ctx.stateful_routes: + contexts_snapshot = _contexts_snapshot(app) + changed_route = _changed_state_config_route( + manifest, miss_ctx, contexts_snapshot + ) + if changed_route is None: + console.info( + "Compile cache: recompiled pages define unchanged states; " + "reusing contexts file" + ) + else: + console.info( + f"Compile cache: page {changed_route!r} changed its state " + "config; rebuilding contexts" + ) + stateful_hits = [ + route + for route, entry in manifest["pages"].items() + if entry["is_stateful"] and route not in miss_routes + ] + with console.timing("Evaluate stateful hit pages (contexts)"): + for route in stateful_hits: + app._compile_page(route, save_page=False) + + from reflex_components_radix.plugin import RadixThemesPlugin + + theme = next( + ( + plugin.get_theme() + for plugin in compiler_plugins + if isinstance(plugin, RadixThemesPlugin) + ), + None, + ) + context_path, context_code = compiler.compile_contexts(app._state, theme) + compiler.utils.write_file( + compiler.utils.resolve_path_of_web_dir(context_path), context_code + ) + + # The assets copy is cheap, idempotent, and excluded from dependency + # tracking entirely, so it is always re-run. assets_src = (root or Path.cwd()) / constants.Dirs.APP_ASSETS if assets_src.is_dir(): path_ops.update_directory_tree( @@ -559,9 +766,17 @@ def try_incremental_rebuild( app._get_frontend_packages(install_imports) frontend_skeleton.update_react_router_config(prerender_routes=prerender_routes) frontend_skeleton.update_entry_client() + frontend_skeleton.initialize_vite_config() # Refresh the manifest for the next process. - _update_manifest_for_misses(manifest, miss_ctx, miss_pages, install_imports, root) + _update_manifest_for_misses( + manifest, + miss_ctx, + miss_pages, + install_imports, + root, + contexts_snapshot=contexts_snapshot, + ) return True @@ -572,6 +787,8 @@ def _update_manifest_for_misses( miss_pages: Sequence[PageDefinition], all_imports: ParsedImportDict, root: Path | None = None, + *, + contexts_snapshot: tuple[dict[str, Any], dict[str, Any]] | None = None, ) -> None: """Update the on-disk manifest entries for the recompiled pages. @@ -581,6 +798,8 @@ def _update_manifest_for_misses( miss_pages: The recompiled page definitions. all_imports: The complete frontend import set after recompiling misses. root: Project root for dependency discovery. Defaults to cwd. + contexts_snapshot: The app's ``_contexts_snapshot`` for fingerprinting + stateful pages, or None when no miss page was stateful. """ if miss_ctx is None or not miss_pages: return @@ -589,12 +808,18 @@ def _update_manifest_for_misses( hasher = page_cache.make_hasher() for page in miss_pages: page_ctx = miss_ctx.compiled_pages[page.route] + defined_states = miss_ctx.stateful_routes.get(page.route) manifest["pages"][page.route] = _manifest_page_entry( page_ctx, page.component, state_index, hasher, - is_stateful=page.route in miss_ctx.stateful_routes, + is_stateful=defined_states is not None, + state_fingerprint=( + _contexts_fingerprint(defined_states, *contexts_snapshot) + if defined_states and contexts_snapshot is not None + else None + ), root=root, ) manifest["all_imports"] = _serialize_imports(all_imports) diff --git a/reflex/compiler/page_cache.py b/reflex/compiler/page_cache.py index 38bcbb2b798..22cbde24da1 100644 --- a/reflex/compiler/page_cache.py +++ b/reflex/compiler/page_cache.py @@ -418,10 +418,10 @@ def _app_entrypoint_file(root: Path | None = None) -> Path | None: return rf if root in rf.parents else None -def global_epoch( +def global_epoch_inputs( root: Path | None = None, *, pages: Sequence[object] | None = None -) -> str: - """Fingerprint the genuinely-global inputs. +) -> dict[str, str]: + """Fingerprint each genuinely-global input individually. These can affect every page's output but belong to no single page, so they gate the whole cache rather than any one page's dependency set: the Reflex @@ -430,30 +430,91 @@ def global_epoch( stylesheets, head components; see :func:`app_dependency_files`). Kept small on purpose; per-file edits flow through per-page dependency sets instead. + Keeping each input's digest separate (rather than one combined sha) lets a + mismatch report *which* global input changed. + Args: root: Project root. Defaults to cwd. pages: The current page definitions, used as barriers so page modules (tracked per page) are excluded from the app-level config files. Returns: - A hex digest of the global inputs. + A mapping of input label (``reflex``, a global filename, or + ``app:<path>``) to its content digest (``<absent>`` if unreadable). """ root = (root or Path.cwd()).resolve() - parts: list[str] = [f"reflex={_reflex_version()}"] + inputs: dict[str, str] = {"reflex": _reflex_version()} for name in _GLOBAL_FILES: path = root / name try: - parts.append(f"{name}={hashlib.sha256(path.read_bytes()).hexdigest()}") + inputs[name] = hashlib.sha256(path.read_bytes()).hexdigest() except OSError: - parts.append(f"{name}=<absent>") - # Sorted for a deterministic digest regardless of set iteration order. - for path_str in sorted(app_dependency_files(pages, root)): + inputs[name] = "<absent>" + for path_str in app_dependency_files(pages, root): try: digest = hashlib.sha256(Path(path_str).read_bytes()).hexdigest() except OSError: digest = "<absent>" - parts.append(f"app:{path_str}={digest}") - return _sha(*parts) + inputs[f"app:{path_str}"] = digest + return inputs + + +def global_epoch( + root: Path | None = None, *, pages: Sequence[object] | None = None +) -> str: + """Fingerprint the genuinely-global inputs as one digest. + + Args: + root: Project root. Defaults to cwd. + pages: The current page definitions (see :func:`global_epoch_inputs`). + + Returns: + A hex digest of the global inputs. + """ + # Sorted for a deterministic digest regardless of dict insertion order. + return _sha( + *(f"{k}={v}" for k, v in sorted(global_epoch_inputs(root, pages=pages).items())) + ) + + +def changed_epoch_inputs(stored: dict[str, str], root: Path | None = None) -> set[str]: + """Return the labels of stored global inputs whose current content differs. + + Validates against the *stored* input set (like :func:`deps_unchanged` does + for page deps): membership is decided once, when the manifest is written by + a full compile; checking only re-hashes those inputs. Recomputing the set + via :func:`global_epoch_inputs` at check time is wrong — it depends on what + the current process happened to read/import during app import, which + differs between a cold compile and a warm forked reload (unpurged module + caches skip re-reads), spuriously invalidating every hot reload. + + Args: + stored: The manifest's ``{label: digest}`` global-input map + (see :func:`global_epoch_inputs` for the label forms). + root: Project root the plain-filename labels resolve against. + Defaults to cwd. + + Returns: + The labels whose content digest no longer matches. + """ + root = (root or Path.cwd()).resolve() + changed: set[str] = set() + for label, digest in stored.items(): + if label == "reflex": + current = _reflex_version() + else: + path = ( + Path(label.removeprefix("app:")) + if label.startswith("app:") + else root / label + ) + try: + current = hashlib.sha256(path.read_bytes()).hexdigest() + except OSError: + current = "<absent>" + if current != digest: + changed.add(label) + return changed def _module_file(component: object) -> Path | None: diff --git a/reflex/utils/compile_daemon.py b/reflex/utils/compile_daemon.py index 8d1d20525db..177d273d6a1 100644 --- a/reflex/utils/compile_daemon.py +++ b/reflex/utils/compile_daemon.py @@ -265,10 +265,18 @@ def _first_party_module_names(roots: list[Path]) -> set[str]: def _reset_first_party(roots: list[Path]) -> None: """Make this interpreter clean w.r.t. first-party code before re-importing. - Purges the user's first-party modules from ``sys.modules`` and clears the + Purges the user's first-party modules from ``sys.modules`` and resets the cross-module registries/caches that would otherwise pin old class objects. Third-party modules are left imported and warm. + The state registry is reset surgically, not blanket-cleared: a class body + in a module that survives the purge (framework internals, installed or + workspace packages) never re-executes in this process, so clearing its + registration would lose the state from the app's state tree — and from the + compiled contexts file — permanently. Those registrations are kept; states + from purged modules re-register on re-import, and runtime-created states in + ``reflex.istate.dynamic`` re-register when their page re-evaluates. + Args: roots: The resolved reload roots whose modules are first-party. """ @@ -281,11 +289,19 @@ def _reset_first_party(roots: list[Path]) -> None: from reflex_base.registry import RegistrationContext + import reflex.istate.dynamic as istate_dynamic from reflex.compiler import page_cache from reflex.page import DECORATED_PAGES from reflex.state import BaseState, all_base_state_classes ctx = RegistrationContext.ensure_context() + kept = [ + cls + for cls in ctx.base_states.values() + if (module_name := getattr(cls, "__module__", None)) is not None + and module_name != istate_dynamic.__name__ + and getattr(sys.modules.get(module_name), "__file__", None) is not None + ] ctx.base_states.clear() ctx.base_state_substates.clear() ctx.event_handlers.clear() @@ -298,6 +314,17 @@ def _reset_first_party(roots: list[Path]) -> None: BaseState.get_class_substate, ): cached.cache_clear() + # Locally-defined states are attached to ``reflex.istate.dynamic`` under + # collision-suffixed names; with the warm parent's attributes in place, + # every re-created state would drift to a new suffix and diverge from the + # names the (cold) backend computes. Reset the module so re-created states + # get their fresh-process names. + for attr in [name for name in vars(istate_dynamic) if not name.startswith("__")]: + delattr(istate_dynamic, attr) + # Original registration order, so parents always precede their children. + for cls in kept: + ctx._register_base_state(cls) + all_base_state_classes[cls.get_full_name()] = None DECORATED_PAGES.clear() # The import graph caches each module's parsed import edges; a changed file # may import differently now, so drop it to force a re-parse. Cross-compile @@ -496,6 +523,12 @@ def _serve() -> None: if not changed: continue + from reflex.compiler.disk_cache import format_path_list + + console.info( + f"Compile daemon: change detected in {format_path_list(map(str, changed), root)}" + ) + # A change to a genuinely-global input (rxconfig/lockfiles, or a reflex # upgrade) can't be applied to the warm parent (it imported the old # version); re-exec the daemon so the new world is actually loaded. diff --git a/reflex/utils/frontend_skeleton.py b/reflex/utils/frontend_skeleton.py index de9f29e7bbd..4f511703b9e 100644 --- a/reflex/utils/frontend_skeleton.py +++ b/reflex/utils/frontend_skeleton.py @@ -547,8 +547,10 @@ def _compile_vite_config(config: Config): def initialize_vite_config(): """Render and write in .web the vite.config.js file using Reflex config.""" - vite_config_file_path = get_web_dir() / constants.ReactRouter.VITE_CONFIG_FILE - vite_config_file_path.write_text(_compile_vite_config(get_config())) + write_file( + get_web_dir() / constants.ReactRouter.VITE_CONFIG_FILE, + _compile_vite_config(get_config()), + ) def initialize_bun_config(): diff --git a/tests/units/compiler/test_disk_cache.py b/tests/units/compiler/test_disk_cache.py index fb62ad25b2b..320d364e4b2 100644 --- a/tests/units/compiler/test_disk_cache.py +++ b/tests/units/compiler/test_disk_cache.py @@ -1,6 +1,8 @@ """Tests for the experimental disk-persisted incremental compile cache.""" import dataclasses +import hashlib +import itertools import json from collections.abc import Callable, Sequence from pathlib import Path @@ -64,8 +66,9 @@ def _page_c() -> Component: return rx.el.div(rx.el.h1("Page C"), _footer()) -def _compile(pages: Sequence[Any]) -> CompileContext: +def _compile(pages: Sequence[Any], app: Any = None) -> CompileContext: ctx = CompileContext( + app=app, pages=pages, hooks=CompilerHooks(plugins=default_page_plugins()), ) @@ -74,6 +77,28 @@ def _compile(pages: Sequence[Any]) -> CompileContext: return ctx +def _unregister_state(cls: type[rx.State]) -> None: + """Drop a state class from the registries, like the daemon registry reset. + + Lets a page evaluation re-define the same-named class in this process, + mirroring what happens in the fork child between hot reloads. + + Args: + cls: The state class to unregister. + """ + from reflex_base.registry import RegistrationContext + + from reflex.state import all_base_state_classes + + ctx = RegistrationContext.ensure_context() + full_name = cls.get_full_name() + ctx.base_states.pop(full_name, None) + parent = cls.get_parent_state() + if parent is not None: + ctx.base_state_substates.get(parent.get_full_name(), set()).discard(cls) + all_base_state_classes.pop(full_name, None) + + def test_imports_round_trip(): imports = { "react": [ImportVar("useEffect"), ImportVar("Fragment", is_default=False)], @@ -96,7 +121,7 @@ def _manifest(pages: dict[str, dict], **overrides) -> dict: base = { "schema": disk_cache._SCHEMA, "reflex_version": page_cache._reflex_version(), - "epoch": "EPOCH", + "epoch_inputs": {}, "all_imports": {}, "pages": pages, } @@ -104,18 +129,63 @@ def _manifest(pages: dict[str, dict], **overrides) -> dict: return base -def test_globals_match(): - m = _manifest({"/a": {}, "/b": {}}) +def test_globals_mismatch_names_the_changed_input(tmp_path): + m = _manifest({"/a": {}, "/b": {}}, epoch_inputs={"rxconfig.py": "<absent>"}) routes = {"/a", "/b"} - assert disk_cache.globals_match(m, routes=routes, epoch="EPOCH") - # a changed route set -> no match - assert not disk_cache.globals_match(m, routes={"/a"}, epoch="EPOCH") - # a changed global epoch -> no match - assert not disk_cache.globals_match(m, routes=routes, epoch="OTHER") - # a stale reflex version -> no match - assert not disk_cache.globals_match( - {**m, "reflex_version": "0.0.0-old"}, routes=routes, epoch="EPOCH" + assert disk_cache.globals_mismatch(m, routes=routes, root=tmp_path) is None + # a changed route set -> named added/removed routes + reason = disk_cache.globals_mismatch(m, routes={"/a", "/c"}, root=tmp_path) + assert reason is not None + assert "/c" in reason + assert "/b" in reason + # a stale reflex version -> named versions + reason = disk_cache.globals_mismatch( + {**m, "reflex_version": "0.0.0-old"}, routes=routes, root=tmp_path ) + assert reason is not None + assert "0.0.0-old" in reason + + +def test_globals_mismatch_validates_stored_inputs_only(tmp_path): + """Epoch validation re-hashes the *stored* input set, never a recomputed one. + + ``app_dependency_files`` depends on what the current process happened to + read/import during app import, which differs between a cold compile and a + warm forked reload (non-purged module caches skip re-reads). Comparing a + recomputed set against the stored one therefore mismatched on every hot + reload; only stored inputs whose *content* changed may invalidate. + """ + theme = tmp_path / "theme_config.py" + theme.write_text("PRIMARY = 'red'") + stored = { + "reflex": page_cache._reflex_version(), + # global files absent from this root at write time and still absent + "rxconfig.py": "<absent>", + f"app:{theme}": hashlib.sha256(theme.read_bytes()).hexdigest(), + } + m = _manifest({"/a": {}}, epoch_inputs=stored) + # nothing on disk changed -> match, regardless of what a re-recorded + # app-import read set would look like in this process + assert disk_cache.globals_mismatch(m, routes={"/a"}, root=tmp_path) is None + # a stored input's content changed -> mismatch naming that file + theme.write_text("PRIMARY = 'blue'") + reason = disk_cache.globals_mismatch(m, routes={"/a"}, root=tmp_path) + assert reason is not None + assert "theme_config.py" in reason + # a stored global file appearing counts as a change too + theme.write_text("PRIMARY = 'red'") + (tmp_path / "rxconfig.py").write_text("import reflex") + reason = disk_cache.globals_mismatch(m, routes={"/a"}, root=tmp_path) + assert reason is not None + assert "rxconfig.py" in reason + + +def test_format_path_list_relativizes_and_truncates(): + root = Path("/proj") + assert disk_cache.format_path_list({"/proj/a.py", "other"}, root) == "a.py, other" + many = {f"/proj/{i}.py" for i in range(8)} + out = disk_cache.format_path_list(many, root, limit=3) + assert out == "0.py, 1.py, 2.py (+5 more)" def test_partition_pages_detects_changed_source(): @@ -156,10 +226,12 @@ def test_write_and_load_manifest(tmp_path, monkeypatch): "dep_hashes", "app_wrap_keys", "is_stateful", + "state_fingerprint", "has_memos", } # these static pages register no new state and contribute no memos assert entry["is_stateful"] is False + assert entry["state_fingerprint"] is None assert entry["has_memos"] is False # rendered output is never persisted (it already lives in .web, and is # never read back from the manifest) -> keeps the manifest small @@ -227,6 +299,7 @@ def _stub_externals(app, monkeypatch): ) monkeypatch.setattr(fs, "update_react_router_config", lambda **k: None) monkeypatch.setattr(fs, "update_entry_client", lambda *a, **k: None) + monkeypatch.setattr(fs, "initialize_vite_config", lambda: None) def test_incremental_rebuild_all_hits(tmp_path, monkeypatch): @@ -334,10 +407,17 @@ def test_stateful_hit_is_marked_but_not_reevaluated(tmp_path, monkeypatch): assert route in app._stateful_pages -def test_incremental_rebuild_regenerates_contexts(tmp_path, monkeypatch): - """State defaults/client-storage are baked into the contexts file and a - state-module edit never bumps the epoch, so the incremental path must - always re-emit it. +def test_incremental_rebuild_preserves_contexts_without_stateful_miss( + tmp_path, monkeypatch +): + """The contexts file is reused unless a stateful page missed. + + The full compile's contexts file holds every state's defaults, including + states only registered while their page evaluates (exec'd demos, dynamic + imports). An incremental process that evaluated no stateful pages has an + incomplete registry; rewriting contexts from it drops those states and the + frontend's dispatch map with them (``dispatch is not a function``). With no + stateful miss, no state changed -> the on-disk file must stay untouched. """ from reflex.compiler import utils as compiler_utils @@ -350,18 +430,211 @@ def test_incremental_rebuild_regenerates_contexts(tmp_path, monkeypatch): disk_cache.write_manifest(ctx, pages, ctx.all_imports, root=tmp_path) _stub_externals(app, monkeypatch) + full_contexts = "// complete contexts from the full compile" + out_path = compiler_utils.resolve_path_of_web_dir(compiler_utils.get_context_path()) + out_path.parent.mkdir(parents=True, exist_ok=True) + out_path.write_text(full_contexts, encoding="utf-8") + assert ( disk_cache.try_incremental_rebuild( app, compiler_plugins=[], prerender_routes=False, root=tmp_path ) is True ) + assert out_path.read_text(encoding="utf-8") == full_contexts + + +_state_counter = itertools.count() + + +def _page_s() -> Component: + """A page that defines its state during evaluation (like exec'd docs demos). + + Each evaluation defines a fresh uniquely-named state class, so the page is + marked stateful and repeated evaluations in one test process don't collide. + + Returns: + The page component. + """ + name = f"_ContextsState{next(_state_counter)}" + state_cls: Any = type( + name, + (rx.State,), + {"__annotations__": {"value": str}, "value": "", "__module__": __name__}, + ) + return rx.el.div(rx.el.p(state_cls.value), _footer()) + +def test_stateful_miss_evaluates_stateful_hits_then_rewrites_contexts( + tmp_path, monkeypatch +): + """A stateful miss forces a contexts rewrite from a *complete* registry. + + The stateful hit pages must be evaluated first (registering their + evaluation-time states) so the rewritten contexts file keeps every state. + """ + from reflex.compiler import utils as compiler_utils + + web = _use_tmp_web_dir(tmp_path, monkeypatch) + + app = rx.App() + app.add_page(_page_s, route="/s") + app.add_page(_page_c, route="/c") + pages = list(app._unevaluated_pages.values()) + stateful_route, hit_route = pages[0].route, pages[1].route + ctx = _compile(pages) + assert stateful_route in ctx.stateful_routes + disk_cache.write_manifest(ctx, pages, ctx.all_imports, root=tmp_path) + _stub_externals(app, monkeypatch) + + # Make the stateful page a miss, and mark the hit page stateful so the + # rebuild must re-register its states before compiling contexts. + manifest_path = web / disk_cache._MANIFEST_FILE + manifest = json.loads(manifest_path.read_text()) + manifest["pages"][stateful_route]["dep_hashes"] = { + str(tmp_path / "view.py"): "stale-hash" + } + manifest["pages"][hit_route]["is_stateful"] = True + manifest_path.write_text(json.dumps(manifest)) + + reevaluated: list[str] = [] + monkeypatch.setattr( + app, "_compile_page", lambda route, **k: reevaluated.append(route) + ) + + assert ( + disk_cache.try_incremental_rebuild( + app, compiler_plugins=[], prerender_routes=False, root=tmp_path + ) + is True + ) + # Only the stateful HIT page is re-evaluated (the miss was compiled in the + # miss context); then contexts are rewritten from the complete registry. + assert reevaluated == [hit_route] out_path = compiler_utils.resolve_path_of_web_dir(compiler_utils.get_context_path()) - assert out_path.exists() assert out_path.read_text(encoding="utf-8") == _CONTEXTS_STUB +def test_contexts_fingerprint_sensitivity(): + initial = {"root.s_one": {"value": ""}, "root.s_two": {"count": 0}} + storage = { + "cookies": {"root.s_one.token_rx_field_": {"name": "t"}}, + "local_storage": {}, + "session_storage": {}, + } + base = disk_cache._contexts_fingerprint(["root.s_one"], initial, storage) + # Stable across equal inputs and insensitive to name order. + assert disk_cache._contexts_fingerprint(["root.s_one"], dict(initial), storage) == ( + base + ) + both = disk_cache._contexts_fingerprint( + ["root.s_one", "root.s_two"], initial, storage + ) + assert ( + disk_cache._contexts_fingerprint(["root.s_two", "root.s_one"], initial, storage) + == both + ) + # Sensitive to which states, their initial values, and their client storage. + assert disk_cache._contexts_fingerprint(["root.s_two"], initial, storage) != base + assert ( + disk_cache._contexts_fingerprint( + ["root.s_one"], {"root.s_one": {"value": "x"}}, storage + ) + != base + ) + assert ( + disk_cache._contexts_fingerprint( + ["root.s_one"], + initial, + {"cookies": {}, "local_storage": {}, "session_storage": {}}, + ) + != base + ) + + +_FP_HOLDER: dict[str, Any] = {} + + +def _page_fp() -> Component: + """A stateful page that defines the SAME state class on every evaluation. + + Mirrors a docs page whose exec'd demo code is unchanged between reloads. + The previous definition must be unregistered before re-evaluation (the + daemon's registry reset does this between hot reloads). + + Returns: + The page component. + """ + state_cls: Any = type( + "_FixedFpState", + (rx.State,), + {"__annotations__": {"value": str}, "value": "", "__module__": "fp_mod_x"}, + ) + _FP_HOLDER["cls"] = state_cls + return rx.el.div(rx.el.p(state_cls.value), _footer()) + + +def test_stateful_miss_with_unchanged_states_reuses_contexts(tmp_path, monkeypatch): + """A stateful miss whose states are unchanged must not rebuild contexts. + + Most content edits leave the page's evaluation-time states identical, so + re-evaluating every stateful hit page just to rewrite an identical contexts + file would waste nearly the whole hot reload. + """ + from reflex.compiler import utils as compiler_utils + + web = _use_tmp_web_dir(tmp_path, monkeypatch) + + app = rx.App() + app.add_page(_page_fp, route="/s") + app.add_page(_page_c, route="/c") + pages = list(app._unevaluated_pages.values()) + stateful_route, hit_route = pages[0].route, pages[1].route + ctx = _compile(pages, app=app) + assert stateful_route in ctx.stateful_routes + disk_cache.write_manifest(ctx, pages, ctx.all_imports, root=tmp_path) + _stub_externals(app, monkeypatch) + + manifest_path = web / disk_cache._MANIFEST_FILE + manifest = json.loads(manifest_path.read_text()) + stored_fp = manifest["pages"][stateful_route]["state_fingerprint"] + assert stored_fp # the compile recorded the page's state config + # Make the stateful page a miss; mark the hit page stateful so a contexts + # rebuild (if wrongly triggered) would have to re-evaluate it. + manifest["pages"][stateful_route]["dep_hashes"] = { + str(tmp_path / "view.py"): "stale-hash" + } + manifest["pages"][hit_route]["is_stateful"] = True + manifest_path.write_text(json.dumps(manifest)) + + # The daemon child resets the registry before recompiling; drop the class + # so the page's re-evaluation re-defines it identically. + _unregister_state(_FP_HOLDER.pop("cls")) + + reevaluated: list[str] = [] + monkeypatch.setattr( + app, "_compile_page", lambda route, **k: reevaluated.append(route) + ) + sentinel = "// pre-existing contexts" + out_path = compiler_utils.resolve_path_of_web_dir(compiler_utils.get_context_path()) + out_path.parent.mkdir(parents=True, exist_ok=True) + out_path.write_text(sentinel, encoding="utf-8") + + assert ( + disk_cache.try_incremental_rebuild( + app, compiler_plugins=[], prerender_routes=False, root=tmp_path + ) + is True + ) + # Unchanged state config: no hit re-evaluation, contexts file untouched. + assert reevaluated == [] + assert out_path.read_text(encoding="utf-8") == sentinel + # The refreshed manifest records the same fingerprint for the miss page. + refreshed = json.loads(manifest_path.read_text()) + assert refreshed["pages"][stateful_route]["state_fingerprint"] == stored_fp + _unregister_state(_FP_HOLDER.pop("cls")) + + def test_incremental_rebuild_copies_assets(tmp_path, monkeypatch): """An assets-only edit is an all-hit rebuild, so the incremental path must run the same assets -> public copy as the full compile. @@ -536,7 +809,7 @@ def test_update_manifest_for_misses_keeps_complete_imports(tmp_path, monkeypatch page_ctx = SimpleNamespace( app_wrap_components={}, frontend_imports={}, memo_contributions={} ) - miss_ctx = SimpleNamespace(compiled_pages={"/a": page_ctx}, stateful_routes=set()) + miss_ctx = SimpleNamespace(compiled_pages={"/a": page_ctx}, stateful_routes={}) complete_imports = {"memo-lib": [ImportVar("MemoThing")]} manifest = _manifest({ "/a": {"dep_hashes": {}, "app_wrap_keys": [], "is_stateful": False} diff --git a/tests/units/reflex_base/compiler/__init__.py b/tests/units/reflex_base/compiler/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/units/reflex_base/compiler/test_templates.py b/tests/units/reflex_base/compiler/test_templates.py new file mode 100644 index 00000000000..def7434a2d9 --- /dev/null +++ b/tests/units/reflex_base/compiler/test_templates.py @@ -0,0 +1,53 @@ +"""Tests for reflex_base.compiler.templates.""" + +import re + +from reflex_base.compiler import templates + +# The unloaded-route throw block exactly as shipped in the pinned +# @react-router/dev runtime (dist/static/refresh-utils.mjs, enqueueUpdate). +_REFRESH_UTILS_THROW = """\ + let imported = window.__reactRouterRouteModuleUpdates.get(route.id); + if (!imported) { + throw Error( + `[react-router:hmr] No module update found for route ${route.id}`, + ); + } + let routeModule = { +""" + + +def _render_vite_config() -> str: + return templates.vite_config_template( + base="/", + hmr=True, + force_full_reload=False, + experimental_hmr=False, + sourcemap=False, + ) + + +def test_vite_config_patches_react_router_hmr_runtime(): + """The generated vite config neutralizes react-router's unloaded-route throw. + + react-router's HMR client throws when an update batch includes a route the + browser hasn't loaded, and the throw aborts the batch before the update + queue is cleared — one edit to any not-currently-open page then poisons HMR + until a full page reload. The generated config must ship a plugin that + rewrites the served runtime to skip unloaded routes instead. + """ + config = _render_vite_config() + assert "patchReactRouterHmrRuntime()" in config + assert '"\\0virtual:react-router/hmr-runtime"' in config + + # The embedded regex must match the pinned runtime's throw block, and the + # replacement must drop the throw while keeping the update loop going. + regex_match = re.search(r"const unloadedRouteThrow = /(.+)/;", config) + assert regex_match is not None + js_regex = regex_match.group(1) + patched, n_subs = re.subn( + js_regex, "if (!imported) continue;", _REFRESH_UTILS_THROW + ) + assert n_subs == 1 + assert "throw" not in patched + assert "if (!imported) continue;" in patched diff --git a/tests/units/utils/test_compile_daemon.py b/tests/units/utils/test_compile_daemon.py index 93e3e75e38a..27de3b76db3 100644 --- a/tests/units/utils/test_compile_daemon.py +++ b/tests/units/utils/test_compile_daemon.py @@ -4,10 +4,17 @@ import pytest +import reflex as rx from reflex.compiler import disk_cache from reflex.utils import compile_daemon +class _SurvivingModuleState(rx.State): + """Module-level state, like one from a non-purged installed package.""" + + b: int = 0 + + def test_iter_source_files_picks_content_skips_build_dirs(tmp_path): (tmp_path / "page.py").write_text("x = 1\n") (tmp_path / "doc.md").write_text("# doc\n") @@ -168,6 +175,96 @@ class DaemonResetUser(rx.Model, table=True): assert out == b"1" +@pytest.mark.skipif(not hasattr(os, "fork"), reason="requires os.fork (POSIX)") +def test_reset_first_party_keeps_surviving_module_states(tmp_path): + """States from modules that survive the purge stay registered. + + A class body in a non-purged module (framework internals, installed or + workspace packages like reflex_site_shared) never re-executes in the child, + so dropping its registration loses the state from the app's state tree — + and from the compiled contexts file — breaking the frontend's dispatch map. + Purged-module states and runtime-created ``reflex.istate.dynamic`` states + must still be dropped (they re-register on re-import/re-evaluation), and + the dynamic module's attributes must be reset so re-created local states + get their deterministic fresh-process names, matching the cold backend. + """ + mod_file = tmp_path / "purged_state_mod.py" + mod_file.write_text( + "import reflex as rx\n\nclass PurgedState(rx.State):\n a: int = 0\n" + ) + + read_fd, write_fd = os.pipe() + pid = os.fork() + if pid == 0: # child + os.close(read_fd) + result = b"E" + try: + import importlib.util + import sys + + import reflex.istate.dynamic as istate_dynamic + from reflex.state import State, all_base_state_classes + + spec = importlib.util.spec_from_file_location("purged_state_mod", mod_file) + assert spec is not None + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) # type: ignore[union-attr] + sys.modules["purged_state_mod"] = module + + def _local_state() -> type[rx.State]: + class LocalState(rx.State): + c: int = 0 + + return LocalState + + local_cls = _local_state() + assert local_cls.__module__ == istate_dynamic.__name__ + local_name = local_cls.__name__ + + compile_daemon._reset_first_party([tmp_path.resolve()]) + + substate_names = {c.__name__ for c in State.get_substates()} + surviving_kept = ( + "_SurvivingModuleState" in substate_names + and _SurvivingModuleState.get_full_name() in all_base_state_classes + ) + purged_dropped = ( + "purged_state_mod" not in sys.modules + and "PurgedState" not in substate_names + ) + dynamic_dropped = local_name not in substate_names and not [ + n for n in vars(istate_dynamic) if not n.startswith("__") + ] + # A re-created local state gets its original (fresh-process) name, + # not a collision-suffixed drift. + redefined_deterministic = _local_state().__name__ == local_name + + result = ( + b"1" + if ( + surviving_kept + and purged_dropped + and dynamic_dropped + and redefined_deterministic + ) + else b"0" + ) + except Exception: + import traceback + + traceback.print_exc() + finally: + os.write(write_fd, result) + os.close(write_fd) + os._exit(0) + + os.close(write_fd) + out = os.read(read_fd, 1) + os.close(read_fd) + os.waitpid(pid, 0) + assert out == b"1" + + @pytest.mark.skipif(not hasattr(os, "fork"), reason="requires os.fork (POSIX)") def test_reset_first_party_purges_modules_and_registries(tmp_path): """``_reset_first_party`` purges first-party modules and clears registries. @@ -206,9 +303,11 @@ def test_reset_first_party_purges_modules_and_registries(tmp_path): compile_daemon._reset_first_party([tmp_path.resolve()]) purged = "fp_module_under_test" not in sys.modules + # The sentinel (a bare object, no surviving module) must be dropped; + # real states from surviving modules are kept (see the test above). cleared = ( - not RegistrationContext.ensure_context().base_states - and not all_base_state_classes + "sentinel" not in RegistrationContext.ensure_context().base_states + and "sentinel" not in all_base_state_classes and not DECORATED_PAGES ) result = b"1" if (purged and cleared) else b"0" From 5cb52c5cee5022279ed5b117776a9e6865b0cb55 Mon Sep 17 00:00:00 2001 From: Farhan <www.mfarvirus@gmail.com> Date: Fri, 3 Jul 2026 18:03:09 +0500 Subject: [PATCH 17/56] perf(compiler): cut redundant work in the incremental compile path The hot paths of a warm rebuild did far more work than needed: - Collapse duplicate ImportVars when serializing the manifest and merge the miss pages' import sets once, instead of re-merging the ~100k-entry app-wide set per page. A full docs-app compile accumulates ~107k import entries of which only ~6k are unique. - Key the module-file cache by the raw __file__ value and check it before any Path construction, so every import statement under an active read recorder skips the resolve. Cleared when the recorder root changes. - Compare path strings in _under_roots instead of building a Path per ancestor per module, which dominated the daemon's reset phase on every hot reload. --- reflex/compiler/disk_cache.py | 23 +++++++++++++++--- reflex/compiler/page_cache.py | 30 ++++++++++++++---------- reflex/utils/compile_daemon.py | 13 +++++++++- tests/units/compiler/test_disk_cache.py | 25 ++++++++++++++++++-- tests/units/compiler/test_page_cache.py | 28 ++++++++++++++++++++++ tests/units/utils/test_compile_daemon.py | 12 ++++++++++ 6 files changed, 113 insertions(+), 18 deletions(-) diff --git a/reflex/compiler/disk_cache.py b/reflex/compiler/disk_cache.py index a1a966aba31..bf7b99baca0 100644 --- a/reflex/compiler/disk_cache.py +++ b/reflex/compiler/disk_cache.py @@ -77,16 +77,30 @@ def _log_fallback(reason: str) -> None: console.info(f"Compile cache: falling back to a full compile — {reason}") +_IMPORT_VAR_FIELDS = tuple(f.name for f in dataclasses.fields(ImportVar)) + + def _serialize_imports(imports: ParsedImportDict) -> dict[str, list[dict[str, Any]]]: """Serialize a parsed import dict to JSON-able primitives. + Duplicates are collapsed in first-seen order (a full docs-app compile + accumulates ~107k entries, ~6k unique): the manifest's import set only + feeds package installation and later merges, where only the unique set + matters, and duplicates bloat the manifest and every pass over it. + Args: imports: The parsed import dict to serialize. Returns: A JSON-serializable representation. """ - return {lib: [dataclasses.asdict(iv) for iv in ivs] for lib, ivs in imports.items()} + return { + lib: [ + {name: getattr(iv, name) for name in _IMPORT_VAR_FIELDS} + for iv in dict.fromkeys(ivs) + ] + for lib, ivs in imports.items() + } def _deserialize_imports(data: dict[str, list[dict[str, Any]]]) -> ParsedImportDict: @@ -654,6 +668,7 @@ def try_incremental_rebuild( install_imports = _deserialize_imports(manifest["all_imports"]) if miss_ctx is not None: memo_contributions: dict[tuple[str, str | None], Any] = {} + miss_imports = [] for page in miss_pages: page_ctx = miss_ctx.compiled_pages[page.route] # Both are guaranteed non-None by the guard loop above. @@ -667,7 +682,7 @@ def try_incremental_rebuild( output_code, ) memo_contributions.update(page_ctx.memo_contributions) - install_imports = merge_imports(install_imports, page_ctx.frontend_imports) + miss_imports.append(page_ctx.frontend_imports) # Memo output files are grouped per source module, so compile them once # with the complete definition set (all recompiled pages' contributions # plus the user memos sharing those files or whose module changed). @@ -678,7 +693,9 @@ def try_incremental_rebuild( compiler.utils.write_file( compiler.utils.resolve_path_of_web_dir(mpath), mcode ) - install_imports = merge_imports(install_imports, memo_imports) + # Merge once: re-merging the app-wide set per page re-walks its ~100k + # entries each time. + install_imports = merge_imports(install_imports, *miss_imports, memo_imports) # Record which routes are stateful: miss pages from this compile, hit pages # from the manifest, so the stateful-pages marker is complete. We do NOT diff --git a/reflex/compiler/page_cache.py b/reflex/compiler/page_cache.py index 22cbde24da1..0ad2e8cdeb7 100644 --- a/reflex/compiler/page_cache.py +++ b/reflex/compiler/page_cache.py @@ -86,7 +86,9 @@ def _reflex_version() -> str: for prefix in {sys.base_exec_prefix, sys.base_prefix, sys.exec_prefix, sys.prefix} if prefix ) -_module_file_cache: dict[tuple[Path, str], str | None] = {} +#: Maps a module's raw ``__file__`` value to its recordable resolved path (or +#: None). Cleared by ``enable_read_tracking`` when the recorder root changes. +_module_file_cache: dict[object, str | None] = {} _app_import_reads: dict[Path, set[str]] = {} @@ -157,6 +159,20 @@ def _recordable_module_file(file: object) -> str | None: The resolved module file path to record, or None when it is outside the project root or otherwise not recordable. """ + root = _recorder_root + if root is None: + return None + # Hot path: every import statement under an active recorder lands here, so + # look up the raw ``__file__`` value before any Path construction. Keying + # by the raw value is safe because ``enable_read_tracking`` clears the + # cache whenever the recorder root changes. + try: + return _module_file_cache[file] + except KeyError: + pass + except TypeError: + return None + with _suspend_tracking(): try: path = Path(file).absolute() # type: ignore[arg-type] @@ -164,16 +180,6 @@ def _recordable_module_file(file: object) -> str | None: return None resolved_str = None - root = _recorder_root - if root is None: - return None - - cache_key = (root, str(path)) - try: - return _module_file_cache[cache_key] - except KeyError: - pass - if not any(part in _EXCLUDE_PARTS for part in path.parts) and ( _is_inside(path, root) or not _is_python_install_file(path) ): @@ -184,7 +190,7 @@ def _recordable_module_file(file: object) -> str | None: else: if not any(part in _EXCLUDE_PARTS for part in resolved.parts): resolved_str = str(resolved) if _is_inside(resolved, root) else None - _module_file_cache[cache_key] = resolved_str + _module_file_cache[file] = resolved_str return resolved_str diff --git a/reflex/utils/compile_daemon.py b/reflex/utils/compile_daemon.py index 177d273d6a1..69d1ffa07ac 100644 --- a/reflex/utils/compile_daemon.py +++ b/reflex/utils/compile_daemon.py @@ -96,6 +96,10 @@ def _reload_roots() -> list[Path]: def _under_roots(path: Path, roots: list[Path]) -> bool: """Whether ``path`` is one of, or lives under, the reload roots. + Compares path strings: this runs for every loaded module on every hot + reload, and ``root in path.parents`` constructs a Path object per ancestor + per check, which dominated the reset phase of a reload. + Args: path: The resolved path to test. roots: The resolved reload roots. @@ -103,7 +107,14 @@ def _under_roots(path: Path, roots: list[Path]) -> bool: Returns: True if the path is covered by a reload root. """ - return any(path == root or root in path.parents for root in roots) + path_str = str(path) + for root in roots: + root_str = str(root) + if path_str == root_str or path_str.startswith( + root_str if root_str.endswith(os.sep) else root_str + os.sep + ): + return True + return False def _iter_source_files(root: Path): diff --git a/tests/units/compiler/test_disk_cache.py b/tests/units/compiler/test_disk_cache.py index 320d364e4b2..5a1b3cdec92 100644 --- a/tests/units/compiler/test_disk_cache.py +++ b/tests/units/compiler/test_disk_cache.py @@ -108,6 +108,25 @@ def test_imports_round_trip(): assert restored == imports +def test_serialize_imports_collapses_duplicates(): + """The manifest only needs the unique import set, in first-seen order. + + A full docs-app compile accumulates ~107k entries of which ~6k are unique; + storing duplicates bloats the manifest and every later merge over it. + """ + use_effect = ImportVar("useEffect") + fragment = ImportVar("Fragment", is_default=False) + imports = { + "react": [use_effect, fragment, use_effect, use_effect, fragment], + "@emotion/react": [ImportVar("jsx"), ImportVar("jsx")], + } + restored = disk_cache._deserialize_imports(disk_cache._serialize_imports(imports)) + assert restored == { + "react": [use_effect, fragment], + "@emotion/react": [ImportVar("jsx")], + } + + def test_wrap_key_strs_is_sorted_and_stable(): keys = [(200, "StrictMode"), (0, "AppWrap"), (45, "ColorMode")] assert disk_cache._wrap_key_strs(keys) == [ @@ -237,9 +256,11 @@ def test_write_and_load_manifest(tmp_path, monkeypatch): # never read back from the manifest) -> keeps the manifest small assert "output_code" not in entry assert "frontend_imports" not in entry - # the app-wide merged imports round-trip cleanly + # the app-wide merged imports round-trip cleanly (duplicates collapsed) restored = disk_cache._deserialize_imports(manifest["all_imports"]) - assert restored == ctx.all_imports + assert restored == { + lib: list(dict.fromkeys(ivs)) for lib, ivs in ctx.all_imports.items() + } def test_unchanged_pages_compile_identically(tmp_path, monkeypatch): diff --git a/tests/units/compiler/test_page_cache.py b/tests/units/compiler/test_page_cache.py index aa4a4578093..a954167a6bb 100644 --- a/tests/units/compiler/test_page_cache.py +++ b/tests/units/compiler/test_page_cache.py @@ -352,6 +352,34 @@ def test_record_reads_tracks_executed_builtin_import(tmp_path, monkeypatch): assert str(child.resolve()) in reads +def test_read_tracking_root_change_invalidates_module_cache(tmp_path, monkeypatch): + """Recording a module under one root must not leak into another root. + + The module-file cache is keyed by the raw ``__file__`` value (the hot path + for every import statement), so switching recorder roots relies on + ``enable_read_tracking`` clearing it. + """ + root_a = tmp_path / "app_a" + root_a.mkdir() + root_b = tmp_path / "app_b" + root_b.mkdir() + module_name = "root_change_runtime_dep" + module_file = _prepare_runtime_module(root_a, monkeypatch, module_name) + page_cache.enable_read_tracking(root=root_a) + + try: + with page_cache.record_reads() as reads_a: + importlib.import_module(module_name) + assert str(module_file.resolve()) in reads_a + + page_cache.enable_read_tracking(root=root_b) + with page_cache.record_reads() as reads_b: + importlib.import_module(module_name) + assert str(module_file.resolve()) not in reads_b + finally: + _forget_modules(module_name) + + def test_record_reads_tracks_path_open(tmp_path): """``Path.open`` calls ``io.open`` directly, bypassing the ``builtins.open`` patch, so it must be patched itself for data reads to be recorded. diff --git a/tests/units/utils/test_compile_daemon.py b/tests/units/utils/test_compile_daemon.py index 27de3b76db3..ea9d858b552 100644 --- a/tests/units/utils/test_compile_daemon.py +++ b/tests/units/utils/test_compile_daemon.py @@ -31,6 +31,18 @@ def test_iter_source_files_picks_content_skips_build_dirs(tmp_path): assert found == {"page.py", "doc.md", "guide.mdx"} +def test_under_roots_matches_only_real_ancestors(tmp_path): + root = tmp_path / "app" + roots = [root] + assert compile_daemon._under_roots(root, roots) + assert compile_daemon._under_roots(root / "pages" / "index.py", roots) + # A sibling sharing the root's string prefix is NOT under it. + assert not compile_daemon._under_roots(tmp_path / "app_extra" / "m.py", roots) + assert not compile_daemon._under_roots(tmp_path / "other.py", roots) + # The root's own parent is not under it either. + assert not compile_daemon._under_roots(tmp_path, roots) + + def test_external_dependency_files_includes_sibling_markdown(tmp_path, monkeypatch): """A page's markdown read from a sibling dir (outside the app root) is watched. From 6dbfacec09ce3ae71a4a9d232042b9893914dcf2 Mon Sep 17 00:00:00 2001 From: Farhan <www.mfarvirus@gmail.com> Date: Fri, 3 Jul 2026 19:18:25 +0500 Subject: [PATCH 18/56] test(compiler): scope contexts snapshot to the test file's states MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The incremental-cache tests fingerprint state config via _contexts_snapshot, which walks the whole root state tree — picking up unrelated (and sometimes broken) state classes registered by other test modules. Stub it with a snapshot limited to the states this file defines, and stub externals before write_manifest so the fingerprint it records uses the same scoped snapshot. --- tests/units/compiler/test_disk_cache.py | 39 ++++++++++++++++++++++++- 1 file changed, 38 insertions(+), 1 deletion(-) diff --git a/tests/units/compiler/test_disk_cache.py b/tests/units/compiler/test_disk_cache.py index 5a1b3cdec92..a4d5f596d26 100644 --- a/tests/units/compiler/test_disk_cache.py +++ b/tests/units/compiler/test_disk_cache.py @@ -303,6 +303,41 @@ def test_unchanged_pages_compile_identically(tmp_path, monkeypatch): _CONTEXTS_STUB = "// contexts stub" +_TEST_STATE_MODULES = (__name__, "fp_mod_x") + + +def _scoped_contexts_snapshot(app) -> tuple[dict[str, Any], dict[str, Any]] | None: + """A ``_contexts_snapshot`` limited to the states this test file defines. + + Same shape and serialization as the real snapshot, but each test state is + compiled standalone instead of walking the whole root state tree, which + would pick up unrelated (and sometimes broken) state classes collected + from other test modules. + + Args: + app: The app being compiled. + + Returns: + The (initial state, client storage) mappings for this file's states, + or None when the app has no state tree. + """ + if app is None or app._state is None: + return None + from reflex_base.registry import RegistrationContext + + from reflex.compiler import utils as compiler_utils + + initial: dict[str, Any] = {} + storage: dict[str, dict[str, Any]] = {} + ctx = RegistrationContext.ensure_context() + for cls in list(ctx.base_states.values()): + if cls.__module__ in _TEST_STATE_MODULES: + initial.update(compiler_utils.compile_state(cls)) + for kind, entries in compiler_utils.compile_client_storage(cls).items(): + storage.setdefault(kind, {}).update(entries) + return initial, storage + + def _stub_externals(app, monkeypatch): """Stub the side-effecting steps the fast path runs on a real app.""" import reflex.utils.frontend_skeleton as fs @@ -318,6 +353,7 @@ def _stub_externals(app, monkeypatch): "reflex.compiler.compiler.compile_contexts", lambda state, theme: (compiler_utils.get_context_path(), _CONTEXTS_STUB), ) + monkeypatch.setattr(disk_cache, "_contexts_snapshot", _scoped_contexts_snapshot) monkeypatch.setattr(fs, "update_react_router_config", lambda **k: None) monkeypatch.setattr(fs, "update_entry_client", lambda *a, **k: None) monkeypatch.setattr(fs, "initialize_vite_config", lambda: None) @@ -613,8 +649,9 @@ def test_stateful_miss_with_unchanged_states_reuses_contexts(tmp_path, monkeypat stateful_route, hit_route = pages[0].route, pages[1].route ctx = _compile(pages, app=app) assert stateful_route in ctx.stateful_routes - disk_cache.write_manifest(ctx, pages, ctx.all_imports, root=tmp_path) + # Stub before write_manifest: it fingerprints via the contexts snapshot. _stub_externals(app, monkeypatch) + disk_cache.write_manifest(ctx, pages, ctx.all_imports, root=tmp_path) manifest_path = web / disk_cache._MANIFEST_FILE manifest = json.loads(manifest_path.read_text()) From 4cbe7b6c77ac7365bce303557f1490c356d60a1b Mon Sep 17 00:00:00 2001 From: Farhan <www.mfarvirus@gmail.com> Date: Sun, 9 Aug 2026 22:32:35 +0500 Subject: [PATCH 19/56] fix(compiler): emit user memo files for modules new to the compile cache MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit masenf's review repro: with REFLEX_COMPILE_CACHE=1, adding a new @rx.memo module and importing it from a page left the page importing $/app_components/<app>/<mod> while that file was never written (red Vite error). Root cause: memo emission derived dirtiness from _changed_dependency_files, which only reads the previous manifest's dep_hashes — a brand-new file can structurally never appear there. Rework the memo decision salsa-style. The manifest (schema 8) stores a per-output-file record: the definitions landing in it, content hashes of the source module's first-party import closure, and the owning route. A file whose record differs from the current registry's demand is rewritten — a new module has no record, so it is dirty by construction. Ownership (a registration_owner ContextVar set around each page evaluation, recorded on MemoDefinition.owner_route) extends this to modules imported inside page functions: route-owned entries are validated by their owner page's hit/miss status and recomputed post-evaluation for recompiled routes, while entries of hit owners carry forward untouched. Hit pages contributing auto memos to a rewritten file are re-evaluated so their exports survive. The closure-based record also fixes a latent staleness bug: editing a helper module used by a memo now rewrites the memo's generated JS. The import graph cache is dropped after evaluation when new first-party modules were loaded, so post-evaluation closures are complete; the old changed-files condition stays as a net for memos without a trackable module file (exec'd docs demos). --- .../src/reflex_base/components/memo.py | 42 ++- .../src/reflex_base/plugins/compiler.py | 26 +- pyi_hashes.json | 2 +- reflex/compiler/disk_cache.py | 335 ++++++++++++++---- reflex/compiler/page_cache.py | 15 + tests/units/compiler/test_disk_cache.py | 184 +++++++++- tests/units/components/test_memo.py | 30 ++ 7 files changed, 556 insertions(+), 78 deletions(-) diff --git a/packages/reflex-base/src/reflex_base/components/memo.py b/packages/reflex-base/src/reflex_base/components/memo.py index 727eed20ce0..be6ab36dff1 100644 --- a/packages/reflex-base/src/reflex_base/components/memo.py +++ b/packages/reflex-base/src/reflex_base/components/memo.py @@ -5,7 +5,9 @@ import dataclasses import inspect import sys -from collections.abc import Callable, Mapping, Sequence +from collections.abc import Callable, Iterator, Mapping, Sequence +from contextlib import contextmanager +from contextvars import ContextVar from copy import copy from enum import Enum from functools import cache, partial, update_wrapper @@ -283,6 +285,12 @@ class MemoDefinition: # used for memos that can't be mirrored. ``kw_only`` so subclasses can keep # their own required fields. source_module: str | None = dataclasses.field(default=None, kw_only=True) + # The page route whose evaluation triggered this memo's registration (see + # ``registration_owner``), or None for memos registered at module import + # outside any page evaluation. The compile cache attributes the memo's + # output file to this route: the file is reusable while the route's page + # is, and is recompiled by re-evaluating that page. + owner_route: str | None = dataclasses.field(default=None, kw_only=True) @dataclasses.dataclass(frozen=True, slots=True) @@ -444,6 +452,36 @@ def reset_memo_component_classes() -> None: MEMOS: dict[tuple[str, str | None], MemoDefinition] = {} +# The route of the page currently being evaluated, if any. Modules imported +# during a page's evaluation register their memos inside this scope, so the +# registered definition records that page as its owner. +_registration_owner: ContextVar[str | None] = ContextVar( + "_registration_owner", default=None +) + + +@contextmanager +def registration_owner(route: str) -> Iterator[None]: + """Attribute memo registrations to the page route that triggers them. + + Salsa-style entity ownership: a memo registered while a page evaluates + (e.g. from a module imported inside the page function) is an artifact of + that page's compilation, not of app import. Recording the owning route + lets the compile cache validate the memo's output file by the page's + hit/miss status and recompile it by re-evaluating that page. + + Args: + route: The route of the page being evaluated. + + Yields: + None. + """ + token = _registration_owner.set(route) + try: + yield + finally: + _registration_owner.reset(token) + def _memo_registry_key(definition: MemoDefinition) -> tuple[str, str | None]: """Get the registry key for a memo. @@ -505,7 +543,7 @@ def _register_memo_definition(definition: MemoDefinition) -> None: ) raise ValueError(msg) - MEMOS[key] = definition + MEMOS[key] = dataclasses.replace(definition, owner_route=_registration_owner.get()) def _annotation_inner_type(annotation: Any) -> Any: diff --git a/packages/reflex-base/src/reflex_base/plugins/compiler.py b/packages/reflex-base/src/reflex_base/plugins/compiler.py index 813af943b1d..8c9303c5d3e 100644 --- a/packages/reflex-base/src/reflex_base/plugins/compiler.py +++ b/packages/reflex-base/src/reflex_base/plugins/compiler.py @@ -807,6 +807,7 @@ def compile( """ from reflex.compiler import compiler from reflex.state import all_base_state_classes + from reflex_base.components.memo import registration_owner from reflex_base.vars.base import reset_unique_variable_names self.ensure_context_attached() @@ -824,23 +825,26 @@ def compile( for page in self.pages: page_fn = page.component n_states_before = len(all_base_state_classes) - if recorder is not None: - with recorder() as read_set: + # Memos registered while this page evaluates (e.g. from modules + # imported inside the page function) are owned by this route. + with registration_owner(page.route): + if recorder is not None: + with recorder() as read_set: + page_ctx = self.hooks.eval_page( + page_fn, + page=page, + compile_context=self, + **kwargs, + ) + if page_ctx is not None: + page_ctx.source_files = read_set + else: page_ctx = self.hooks.eval_page( page_fn, page=page, compile_context=self, **kwargs, ) - if page_ctx is not None: - page_ctx.source_files = read_set - else: - page_ctx = self.hooks.eval_page( - page_fn, - page=page, - compile_context=self, - **kwargs, - ) if page_ctx is None: page_name = getattr(page_fn, "__name__", repr(page_fn)) msg = ( diff --git a/pyi_hashes.json b/pyi_hashes.json index f83a2da704d..2eec6079aa8 100644 --- a/pyi_hashes.json +++ b/pyi_hashes.json @@ -120,5 +120,5 @@ "packages/reflex-components-sonner/src/reflex_components_sonner/toast.pyi": "f170ac685b6ba5892370166c80684db3", "reflex/__init__.pyi": "56385a4f0d9431eb0056dbc5553a58f9", "reflex/components/__init__.pyi": "9facd05a776d0641432696bbf8e34388", - "reflex/experimental/memo.pyi": "bc8b48357bef580e70a5881b65d3d3f7" + "reflex/experimental/memo.pyi": "de43361adffacbe0319a1c67a791bd2d" } diff --git a/reflex/compiler/disk_cache.py b/reflex/compiler/disk_cache.py index bf7b99baca0..f2ccd9cdd23 100644 --- a/reflex/compiler/disk_cache.py +++ b/reflex/compiler/disk_cache.py @@ -34,7 +34,7 @@ from reflex.app import App #: Bump when the manifest layout changes (old manifests are then ignored). -_SCHEMA = 6 +_SCHEMA = 8 #: Manifest filename under the web directory. _MANIFEST_FILE = "reflex_compile_cache.json" @@ -332,6 +332,9 @@ def write_manifest( "epoch_inputs": epoch_inputs, "all_imports": _serialize_imports(install_imports), "pages": pages_data, + # Post-evaluation and complete: the full compile evaluated every + # page, so both import-owned and route-owned memos are recorded. + "memo_files": _memo_state_entries(hasher, root, lambda _owner: True), } path = _manifest_path() path.parent.mkdir(parents=True, exist_ok=True) @@ -495,46 +498,171 @@ def _module_source_file(module_name: str | None) -> str | None: return None -def _complete_memo_defs( +def _memo_output_path(source_module: str | None) -> str | None: + """Resolve the mirrored output file path for a memo source module. + + Args: + source_module: The dotted module name that defined the memo. + + Returns: + The absolute output path string, or None when the module can't be + mirrored (unmirrored memos compile to one file per memo instead). + """ + from reflex_base.utils import memo_paths + + from reflex.compiler import utils as compiler_utils + + segments = memo_paths.module_to_mirrored_segments(source_module) + if segments is None: + return None + return compiler_utils.get_memo_module_path(segments) + + +def _memo_state_entries( + hasher: Callable[[str], str | None], + root: Path | None, + include_owner: Callable[[str | None], bool], +) -> dict[str, dict[str, Any]]: + """Describe the user-memo output files the current registry demands. + + For each mirrored memo output file: the definitions landing in it, the + content hashes of the source module's first-party import closure, and the + route owning the definitions (see ``registration_owner``) — None for memos + registered at module import. Import-owned entries are a pure function of + app import, so any process can compare them pre-evaluation; route-owned + entries only exist after their owner page evaluates, so they are compared + post-evaluation for recompiled routes and validated by their owner's + hit/miss status otherwise. Memos without a resolvable module file (exec'd + docs demos in synthetic modules) are excluded — they have no file to + track and are covered by the ``changed_files`` net in + :func:`_memo_defs_for_rewrite`. + + Args: + hasher: A memoized path -> content-hash function. + root: Project root for the import-graph walk. None defaults to cwd. + include_owner: Predicate over a definition's ``owner_route`` deciding + which registry slice to describe. + + Returns: + A JSON-able mapping of output path -> + ``def_keys``/``dep_hashes``/``owner_route``. + """ + from reflex_base.components.memo import MEMOS + + state: dict[str, dict[str, Any]] = {} + for (name, source_module), definition in MEMOS.items(): + if not include_owner(definition.owner_route): + continue + path = _memo_output_path(source_module) + if path is None: + continue + entry = state.get(path) + if entry is None: + module_file = _module_source_file(source_module) + if module_file is None: + continue + entry = state[path] = { + "def_keys": [], + "dep_hashes": { + dep: hasher(dep) + for dep in sorted( + page_cache.module_py_dependencies(module_file, root) + ) + }, + "owner_route": definition.owner_route, + } + entry["def_keys"].append([name, source_module]) + for entry in state.values(): + entry["def_keys"].sort() + return state + + +def _memo_defs_for_rewrite( contributions: dict[tuple[str, str | None], Any], + dirty_paths: set[str], changed_files: set[str], ) -> list[Any]: """Return the full definition set for the memo files being rewritten. Memo output is grouped one file per source module, so a rewrite must carry - every definition landing in that file: user ``@rx.memo`` definitions from - the global registry that share a module with a recompiled contribution, - plus user memos whose own module file changed (an edited memo body must be - re-emitted even though only its importer pages missed). + every definition landing in that file: the recompiled pages' auto-memo + contributions plus every user ``@rx.memo`` definition whose output file is + being rewritten — because its recorded memo state changed (``dirty_paths``, + see :func:`_current_memo_state`) or because a recompiled page contributes + auto memos to it. The ``changed_files`` condition runs post-evaluation, so + it additionally covers memos the pre-evaluation state can't see: modules + imported inside a page function whose file (a recorded page dependency) + changed. Args: contributions: The recompiled pages' auto-memo contributions. - changed_files: The dependency files whose content changed. + dirty_paths: Output paths whose stored memo state no longer matches. + changed_files: The recorded dependency files whose content changed. Returns: The memo definitions to compile, user memos first (matching the full compile's emit order). """ from reflex_base.components.memo import MEMOS - from reflex_base.utils import memo_paths - dirty_segments = { - segments + rewrite_paths = dirty_paths | { + path for definition in contributions.values() - if ( - segments := memo_paths.module_to_mirrored_segments(definition.source_module) - ) - is not None + if (path := _memo_output_path(definition.source_module)) is not None } user_memos = [ memo - for memo in MEMOS.values() - if memo_paths.module_to_mirrored_segments(memo.source_module) in dirty_segments - or _module_source_file(memo.source_module) in changed_files + for (_, source_module), memo in MEMOS.items() + if _memo_output_path(source_module) in rewrite_paths + or _module_source_file(source_module) in changed_files ] return [*user_memos, *contributions.values()] +def _with_memo_contributing_pages( + miss_pages: list[PageDefinition], + pages: Sequence[PageDefinition], + manifest: dict[str, Any], + dirty_paths: set[str], + stale_owner_routes: set[str], +) -> list[PageDefinition]: + """Add hit pages that must re-evaluate for a memo file rewrite. + + A dirty user-memo file is rewritten from scratch, so hit pages + contributing auto memos to the same file must be re-evaluated for their + contributions, or the rewrite would drop their exports. Likewise, a + route-owned memo entry whose dependency files changed needs its owner + page re-evaluated: only that page's evaluation re-registers the memo + definitions the rewrite compiles from. + + Args: + miss_pages: The dependency-changed pages. + pages: All current page definitions (in compile order). + manifest: The loaded manifest. + dirty_paths: Output paths whose stored memo state no longer matches. + stale_owner_routes: Routes owning stored memo entries with changed + dependency files. + + Returns: + The miss list extended with the affected hit pages. + """ + miss_routes = {page.route for page in miss_pages} + extra = [ + page + for page in pages + if page.route not in miss_routes + and ( + page.route in stale_owner_routes + or ( + manifest["pages"][page.route]["has_memos"] + and _memo_output_path(getattr(page, "_source_module", None)) + in dirty_paths + ) + ) + ] + return [*miss_pages, *extra] if extra else miss_pages + + def try_incremental_rebuild( app: App, *, @@ -549,6 +677,12 @@ def try_incremental_rebuild( unsafe to reuse: no/old manifest, a changed global input, a route change, or a miss page that altered its app-wrap set or stateful flag. + User-memo output files are reconciled against the manifest's stored memo + record (see :func:`_current_memo_state`): files whose record differs are + rewritten, with the affected memo-contributing hit pages re-evaluated so + their exports survive the rewrite. Files of deleted memo modules are left + orphaned (nothing imports them; the next full compile prunes them). + The ``assets`` copy is excluded from dependency tracking and always re-run (cheap, idempotent). The contexts file is rewritten only when a stateful page missed — and then only after evaluating the stateful hit pages, so the @@ -587,37 +721,74 @@ def try_incremental_rebuild( resolved_root = (root or Path.cwd()).resolve() miss_pages = partition_pages(pages, manifest, hasher) + # Memo files are decided salsa-style: derive the files the current memo + # registry demands and rewrite the ones whose stored record differs. This + # catches what changed-file diffing structurally cannot: a memo module the + # previous compile never saw (its file is in no recorded dependency set). + # Only import-owned memos are comparable here (pre-evaluation); + # route-owned entries are handled by owner status: a changed dependency + # pulls the owner page into the miss set, and its re-evaluation feeds the + # post-evaluation comparison below. + memo_state = _memo_state_entries(hasher, resolved_root, lambda owner: owner is None) + stored_memo_files = manifest["memo_files"] + dirty_memo_paths = { + path + for path, entry in memo_state.items() + if stored_memo_files.get(path) != entry + } + stale_owner_routes = { + owner + for entry in stored_memo_files.values() + if (owner := entry["owner_route"]) is not None + and any(hasher(dep) != digest for dep, digest in entry["dep_hashes"].items()) + } changed_files: set[str] = set() - if miss_pages: + if miss_pages or dirty_memo_paths or stale_owner_routes: # Nearly free: partition_pages already hashed every dependency file # into the memoized hasher. changed_files = _changed_dependency_files(manifest, hasher) + miss_pages = _with_memo_contributing_pages( + miss_pages, pages, manifest, dirty_memo_paths, stale_owner_routes + ) miss_pages = _with_module_siblings(miss_pages, pages, manifest) console.info( f"Compile cache: recompiling {len(miss_pages)}/{len(pages)} pages; " f"changed file(s): {format_path_list(changed_files, resolved_root)}" + + ( + f"; rewriting {len(dirty_memo_paths)} memo file(s)" + if dirty_memo_paths + else "" + ) ) else: console.info(f"Compile cache: reusing all {len(pages)} pages from disk") miss_routes = {p.route for p in miss_pages} - # Recompile only the source-changed pages. - miss_ctx = None - if miss_pages: + if miss_pages or dirty_memo_paths: from reflex_base.components.dynamic import ( bundle_library, reset_bundled_libraries, ) from reflex_base.components.memo import reset_memo_component_classes - from reflex.compiler.compiler import make_compile_progress - # Match the full compile's clean bundling/memo state before compiling. reset_bundled_libraries() reset_memo_component_classes() for plugin in compiler_plugins: for dependency in plugin.get_frontend_dependencies(): bundle_library(dependency) + + # Recompile only the source-changed pages. + miss_ctx = None + if miss_pages: + from reflex.compiler.compiler import make_compile_progress + + # Page evaluation can import modules the cached import graph (built + # pre-evaluation for the memo state) has never seen; drop the cache + # afterwards so post-evaluation closures (memo entries, page deps) + # include the newly loaded modules. + modules_before = set(page_cache._loaded_first_party_modules(resolved_root)) + miss_ctx = CompileContext( app=app, pages=miss_pages, @@ -640,6 +811,8 @@ def try_incremental_rebuild( ) finally: progress.stop() + if set(page_cache._loaded_first_party_modules(resolved_root)) != modules_before: + page_cache.clear_import_graph() # Guard: a miss must not change the app-wrap set or its stateful flag, or # the reused on-disk app root / state marker would be wrong. for page in miss_pages: @@ -663,35 +836,51 @@ def try_incremental_rebuild( return False from reflex.compiler import compiler + from reflex.compiler import utils as compiler_utils # Write changed pages + their memo files; reuse everything else on disk. install_imports = _deserialize_imports(manifest["all_imports"]) - if miss_ctx is not None: + eval_memo_state: dict[str, dict[str, Any]] = {} + if miss_ctx is not None or dirty_memo_paths: memo_contributions: dict[tuple[str, str | None], Any] = {} miss_imports = [] - for page in miss_pages: - page_ctx = miss_ctx.compiled_pages[page.route] - # Both are guaranteed non-None by the guard loop above. - output_path = page_ctx.output_path - output_code = page_ctx.output_code - if output_path is None or output_code is None: - _log_fallback(f"page {page.route!r} lost its output before write") - return False - compiler.utils.write_file( - compiler.utils.resolve_path_of_web_dir(output_path), - output_code, + if miss_ctx is not None: + for page in miss_pages: + page_ctx = miss_ctx.compiled_pages[page.route] + # Both are guaranteed non-None by the guard loop above. + output_path = page_ctx.output_path + output_code = page_ctx.output_code + if output_path is None or output_code is None: + _log_fallback(f"page {page.route!r} lost its output before write") + return False + compiler_utils.write_file( + compiler_utils.resolve_path_of_web_dir(output_path), + output_code, + ) + memo_contributions.update(page_ctx.memo_contributions) + miss_imports.append(page_ctx.frontend_imports) + # Post-evaluation pass: the miss pages just evaluated, so memos + # they own (registered during their evaluation, e.g. from modules + # imported inside the page function) are now in the registry. + # A brand-new such module has no stored record and must be + # written; an unchanged one matches its record and is skipped. + eval_memo_state = _memo_state_entries( + hasher, resolved_root, lambda owner: owner in miss_routes ) - memo_contributions.update(page_ctx.memo_contributions) - miss_imports.append(page_ctx.frontend_imports) + dirty_memo_paths = dirty_memo_paths | { + path + for path, entry in eval_memo_state.items() + if stored_memo_files.get(path) != entry + } # Memo output files are grouped per source module, so compile them once # with the complete definition set (all recompiled pages' contributions - # plus the user memos sharing those files or whose module changed). + # plus the user memos landing in a file being rewritten). memo_files, memo_imports = compiler.compile_memo_components( - _complete_memo_defs(memo_contributions, changed_files) + _memo_defs_for_rewrite(memo_contributions, dirty_memo_paths, changed_files) ) for mpath, mcode in memo_files: - compiler.utils.write_file( - compiler.utils.resolve_path_of_web_dir(mpath), mcode + compiler_utils.write_file( + compiler_utils.resolve_path_of_web_dir(mpath), mcode ) # Merge once: re-merging the app-wide set per page re-walks its ~100k # entries each time. @@ -763,8 +952,8 @@ def try_incremental_rebuild( None, ) context_path, context_code = compiler.compile_contexts(app._state, theme) - compiler.utils.write_file( - compiler.utils.resolve_path_of_web_dir(context_path), context_code + compiler_utils.write_file( + compiler_utils.resolve_path_of_web_dir(context_path), context_code ) # The assets copy is cheap, idempotent, and excluded from dependency @@ -785,6 +974,22 @@ def try_incremental_rebuild( frontend_skeleton.update_entry_client() frontend_skeleton.initialize_vite_config() + # The next manifest merges three memo-record sources, mirroring salsa's + # reuse of queries that did not re-run: route-owned entries whose owner + # page hit carry forward unchanged (their page never re-evaluated here), + # import-owned entries are recomputed from the pre-evaluation registry, + # and entries owned by the recompiled routes are recomputed post-eval. + final_memo_state = { + **{ + path: entry + for path, entry in stored_memo_files.items() + if entry["owner_route"] is not None + and entry["owner_route"] not in miss_routes + }, + **memo_state, + **eval_memo_state, + } + # Refresh the manifest for the next process. _update_manifest_for_misses( manifest, @@ -793,6 +998,7 @@ def try_incremental_rebuild( install_imports, root, contexts_snapshot=contexts_snapshot, + memo_state=final_memo_state, ) return True @@ -806,6 +1012,7 @@ def _update_manifest_for_misses( root: Path | None = None, *, contexts_snapshot: tuple[dict[str, Any], dict[str, Any]] | None = None, + memo_state: dict[str, dict[str, Any]], ) -> None: """Update the on-disk manifest entries for the recompiled pages. @@ -817,28 +1024,32 @@ def _update_manifest_for_misses( root: Project root for dependency discovery. Defaults to cwd. contexts_snapshot: The app's ``_contexts_snapshot`` for fingerprinting stateful pages, or None when no miss page was stateful. + memo_state: The current user-memo file record to persist (see + :func:`_current_memo_state`). """ - if miss_ctx is None or not miss_pages: + if (miss_ctx is None or not miss_pages) and manifest["memo_files"] == memo_state: return try: - state_index, _ = page_cache.state_dependency_index(root) - hasher = page_cache.make_hasher() - for page in miss_pages: - page_ctx = miss_ctx.compiled_pages[page.route] - defined_states = miss_ctx.stateful_routes.get(page.route) - manifest["pages"][page.route] = _manifest_page_entry( - page_ctx, - page.component, - state_index, - hasher, - is_stateful=defined_states is not None, - state_fingerprint=( - _contexts_fingerprint(defined_states, *contexts_snapshot) - if defined_states and contexts_snapshot is not None - else None - ), - root=root, - ) + manifest["memo_files"] = memo_state + if miss_ctx is not None: + state_index, _ = page_cache.state_dependency_index(root) + hasher = page_cache.make_hasher() + for page in miss_pages: + page_ctx = miss_ctx.compiled_pages[page.route] + defined_states = miss_ctx.stateful_routes.get(page.route) + manifest["pages"][page.route] = _manifest_page_entry( + page_ctx, + page.component, + state_index, + hasher, + is_stateful=defined_states is not None, + state_fingerprint=( + _contexts_fingerprint(defined_states, *contexts_snapshot) + if defined_states and contexts_snapshot is not None + else None + ), + root=root, + ) manifest["all_imports"] = _serialize_imports(all_imports) _manifest_path().write_text(json.dumps(manifest), encoding="utf-8") except Exception as exc: # best-effort diff --git a/reflex/compiler/page_cache.py b/reflex/compiler/page_cache.py index 0ad2e8cdeb7..c2a795061e3 100644 --- a/reflex/compiler/page_cache.py +++ b/reflex/compiler/page_cache.py @@ -766,6 +766,21 @@ def page_py_dependencies( return _walk_import_closure(graph, _component_source_files(component, root)) +def module_py_dependencies(module_file: str, root: Path | None = None) -> set[str]: + """Return the transitive first-party ``.py`` files a module depends on. + + Args: + module_file: The module's resolved source file path. + root: Project root. Defaults to cwd. + + Returns: + The reachable first-party dependency file paths, including + ``module_file`` itself. + """ + root = (root or Path.cwd()).resolve() + return _walk_import_closure(build_import_graph(root), {module_file}) + + def app_dependency_files( pages: Sequence[object] | None = None, root: Path | None = None ) -> set[str]: diff --git a/tests/units/compiler/test_disk_cache.py b/tests/units/compiler/test_disk_cache.py index a4d5f596d26..264b42799b7 100644 --- a/tests/units/compiler/test_disk_cache.py +++ b/tests/units/compiler/test_disk_cache.py @@ -4,6 +4,7 @@ import hashlib import itertools import json +import sys from collections.abc import Callable, Sequence from pathlib import Path from types import SimpleNamespace @@ -143,6 +144,7 @@ def _manifest(pages: dict[str, dict], **overrides) -> dict: "epoch_inputs": {}, "all_imports": {}, "pages": pages, + "memo_files": {}, } base.update(overrides) return base @@ -751,11 +753,15 @@ def test_incremental_rebuild_rewrites_changed_user_memo( _stub_externals(app, monkeypatch) # Simulate an edit to this module (which defines the user memo): record a - # stale hash for its file so the page importing the memo misses. + # stale hash for its file in the page's dep set (so the importing page + # misses) and in the stored memo record (so the memo file is dirty). module_file = str(Path(__file__).resolve()) manifest_path = web / disk_cache._MANIFEST_FILE manifest = json.loads(manifest_path.read_text()) manifest["pages"][memo_route]["dep_hashes"] = {module_file: "stale-hash"} + memo_path = disk_cache._memo_output_path(__name__) + assert memo_path is not None + manifest["memo_files"][memo_path]["dep_hashes"][module_file] = "stale-hash" manifest_path.write_text(json.dumps(manifest)) assert ( @@ -778,6 +784,175 @@ def test_incremental_rebuild_rewrites_changed_user_memo( assert badge_def.export_name in out_path.read_text(encoding="utf-8") +def test_new_memo_module_is_emitted_on_first_use( + tmp_path, monkeypatch, preserve_memo_registries +): + """A user memo module the previous compile never saw must be emitted. + + Mirrors the review repro: a page starts importing a ``@rx.memo`` from a + brand-new module. The page misses (its importer changed), but the memo's + module file is absent from every recorded dependency set — no + changed-file reasoning can select it — while the recompiled page imports + the memo's mirrored file, which must therefore be written. + """ + from reflex_base.components.memo import MEMOS, MemoComponentDefinition + + from reflex.compiler import compiler + from reflex.compiler import utils as compiler_utils + + web = _use_tmp_web_dir(tmp_path, monkeypatch) + + app = rx.App() + app.add_page(_page_with_user_memo, route="/memo") + pages = list(app._unevaluated_pages.values()) + memo_route = pages[0].route + ctx = _compile(pages) + disk_cache.write_manifest(ctx, pages, ctx.all_imports, root=tmp_path) + _stub_externals(app, monkeypatch) + + # Rewrite the manifest as if the previous compile never saw the memo's + # module: the page's recorded deps name only an unrelated changed file + # (so it misses), and no recorded memo state mentions the module. + manifest_path = web / disk_cache._MANIFEST_FILE + manifest = json.loads(manifest_path.read_text()) + manifest["pages"][memo_route]["dep_hashes"] = { + str(tmp_path / "app.py"): "stale-hash" + } + manifest["memo_files"] = {} + manifest_path.write_text(json.dumps(manifest)) + + assert ( + disk_cache.try_incremental_rebuild( + app, compiler_plugins=[], prerender_routes=False, root=tmp_path + ) + is True + ) + + badge_def = next(m for m in MEMOS.values() if m.source_module == __name__) + assert isinstance(badge_def, MemoComponentDefinition) + memo_files, _ = compiler.compile_memo_components([badge_def]) + assert memo_files + for mpath, _mcode in memo_files: + out_path = compiler_utils.resolve_path_of_web_dir(mpath) + assert out_path.exists(), "new memo module file was never written" + assert badge_def.export_name in out_path.read_text(encoding="utf-8") + + +def test_route_owned_memo_entry_carried_while_owner_hits( + tmp_path, monkeypatch, preserve_memo_registries +): + """A route-owned memo entry survives rebuilds while its owner page hits. + + The pre-evaluation registry can't see memos from modules imported inside + a page function; the entry's owner attribution keeps its on-disk file and + manifest record intact without re-evaluating anything. + """ + web = _use_tmp_web_dir(tmp_path, monkeypatch) + + app = rx.App() + app.add_page(_page_a, route="/a") + pages = list(app._unevaluated_pages.values()) + owner_route = pages[0].route + ctx = _compile(pages) + disk_cache.write_manifest(ctx, pages, ctx.all_imports, root=tmp_path) + _stub_externals(app, monkeypatch) + + lazy_file = web / "app_components" / "lazy_module.jsx" + lazy_file.parent.mkdir(parents=True, exist_ok=True) + lazy_file.write_text("export const Lazy = () => null;") + manifest_path = web / disk_cache._MANIFEST_FILE + manifest = json.loads(manifest_path.read_text()) + manifest["memo_files"][str(lazy_file)] = { + "def_keys": [["Lazy", "lazy_module"]], + "dep_hashes": {}, + "owner_route": owner_route, + } + manifest_path.write_text(json.dumps(manifest)) + + assert ( + disk_cache.try_incremental_rebuild( + app, compiler_plugins=[], prerender_routes=False, root=tmp_path + ) + is True + ) + + assert lazy_file.read_text() == "export const Lazy = () => null;" + # The all-hit rebuild left the manifest record (including the entry) intact. + written = json.loads(manifest_path.read_text()) + assert written["memo_files"][str(lazy_file)]["owner_route"] == owner_route + + +def test_new_lazy_memo_module_is_emitted_on_first_use( + tmp_path, monkeypatch, preserve_memo_registries +): + """A new memo module imported inside a page function must be emitted. + + The pre-evaluation registry cannot see this module, and the changed-file + net cannot see a new file. The post-evaluation pass over the recompiled + routes' owned memos must demand its output file. + """ + import importlib + + module_name = "lazy_memo_mod_t1" + (tmp_path / f"{module_name}.py").write_text( + "import reflex as rx\n" + "\n" + "@rx.memo\n" + "def lazy_badge(text: str) -> rx.Component:\n" + " return rx.el.span(text)\n" + ) + monkeypatch.syspath_prepend(str(tmp_path)) + + def _page_lazy() -> Component: + mod = importlib.import_module(module_name) + return rx.el.div(mod.lazy_badge(text="hi")) + + web = _use_tmp_web_dir(tmp_path, monkeypatch) + app = rx.App() + app.add_page(_page_lazy, route="/lazy") + pages = list(app._unevaluated_pages.values()) + lazy_route = pages[0].route + try: + ctx = _compile(pages) + disk_cache.write_manifest(ctx, pages, ctx.all_imports, root=tmp_path) + _stub_externals(app, monkeypatch) + + manifest_path = web / disk_cache._MANIFEST_FILE + manifest = json.loads(manifest_path.read_text()) + lazy_path = disk_cache._memo_output_path(module_name) + assert lazy_path is not None + # Sanity: the full compile recorded the entry as owned by the page. + assert manifest["memo_files"][lazy_path]["owner_route"] == lazy_route + + # Surgery: the previous compile never saw the lazy module, and the + # page misses via an unrelated changed file. + del manifest["memo_files"][lazy_path] + manifest["pages"][lazy_route]["dep_hashes"] = { + str(tmp_path / "app.py"): "stale-hash" + } + manifest_path.write_text(json.dumps(manifest)) + + assert ( + disk_cache.try_incremental_rebuild( + app, compiler_plugins=[], prerender_routes=False, root=tmp_path + ) + is True + ) + + from reflex_base.components.memo import MEMOS, MemoComponentDefinition + + lazy_def = next(m for m in MEMOS.values() if m.source_module == module_name) + assert isinstance(lazy_def, MemoComponentDefinition) + out_path = Path(lazy_path) + assert out_path.exists(), "lazy memo module file was never written" + assert lazy_def.export_name in out_path.read_text(encoding="utf-8") + # The refreshed manifest re-records the entry with its owner. + written = json.loads(manifest_path.read_text()) + assert written["memo_files"][lazy_path]["owner_route"] == lazy_route + finally: + sys.modules.pop(module_name, None) + + class _MemoCacheState(rx.State): value: str = "x" other: str = "y" @@ -874,7 +1049,12 @@ def test_update_manifest_for_misses_keeps_complete_imports(tmp_path, monkeypatch }) disk_cache._update_manifest_for_misses( - manifest, cast(Any, miss_ctx), [page], complete_imports, root=tmp_path + manifest, + cast(Any, miss_ctx), + [page], + complete_imports, + root=tmp_path, + memo_state={}, ) written = json.loads((web / disk_cache._MANIFEST_FILE).read_text()) diff --git a/tests/units/components/test_memo.py b/tests/units/components/test_memo.py index 3af6d694b47..cf24871b595 100644 --- a/tests/units/components/test_memo.py +++ b/tests/units/components/test_memo.py @@ -23,6 +23,7 @@ _LazyBody, _MemoCallBinding, _strip_optional, + registration_owner, ) from reflex_base.event import EventChain, EventHandler, no_args_event_spec from reflex_base.style import Style @@ -1874,3 +1875,32 @@ def recursive_count(n: rx.vars.NumberVar[int]) -> rx.Var[int]: invoked = recursive_count(n=Var(_js_expr="three", _var_type=int)) assert "recursive_count" in str(invoked) + + +def test_registration_owner_tags_definitions(): + """Registrations inside ``registration_owner`` record the owning route. + + A memo registered while a page evaluates belongs to that page; a + re-registration outside any owner scope re-tags it as import-owned. + """ + + def _definition(): + return next( + d + for (_, module), d in MEMOS.items() + if module == __name__ and d.python_name == "owned_badge" + ) + + with registration_owner("/owner-route"): + + @rx.memo + def owned_badge(text: str) -> Component: + return rx.el.span(text) + + assert _definition().owner_route == "/owner-route" + + @rx.memo + def owned_badge(text: str) -> Component: + return rx.el.span(text) + + assert _definition().owner_route is None From 510f1387565b3411271b7a3b875b13e085d6862a Mon Sep 17 00:00:00 2001 From: Claude <noreply@anthropic.com> Date: Sat, 5 Sep 2026 03:56:08 +0000 Subject: [PATCH 20/56] compile cache: make the warm fork real, watch with watchfiles, validate stat-first Fixes the performance and accuracy issues found reviewing #6688: - The daemon parent was never single-threaded after its initial compile (telemetry's ThreadPoolExecutor worker stays alive), so `_can_fork` always fell back to a cold subprocess. Add `telemetry.shutdown_executor()` and quiesce the parent before every fork; log the offending threads otherwise. - Replace the 50 ms stat-polling loop (plus 1 s tree walks and assets rglob) with watchfiles notifications over the reload roots, external dependency dirs, global files and assets; polling remains only as a fallback. Any recorded dependency counts as a compile input, not just .py/.md/.mdx. - Hand the watcher's changed set to the compile child so pages depending on none of the changed paths are hits without touching the filesystem. - Manifest schema 7: one shared file table `{path: [sha256, mtime_ns, size]}` and per-page dependency path lists. Validation is stat-first; a file is hashed only when its stat moved, and a touched-but-identical file has its entry refreshed instead of being re-hashed on every reload. Global inputs (now including pyproject.toml/requirements.txt) go through the same table. - Persist the import-name parse cache in the manifest keyed by stat, so an unchanged first-party module is never re-parsed by a child. - Cache the per-`__file__` first-party classification and per-class module file, warmed in the daemon parent so forked children inherit them. - Store each stateful page's contexts contribution (initial-state and client-storage slice) instead of a fingerprint; `compile_contexts` accepts extra entries, so a state-config change rebuilds contexts from the manifest without re-evaluating any stateful hit page. - Hold a compile-in-progress lock while the child runs; a backend reload worker that skips compilation waits on it before reading the stateful pages marker, closing the race on the same file save. - First-party roots include REFLEX_HOT_RELOAD_INCLUDE_PATHS; dynamically imported .py reads expand through the import closure; read/import hooks are not installed in backend workers; atomic writes use a dot-prefixed temp name so route discovery and watchers ignore it. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_015uV6WEQPemtsE9Nm6WFxyE --- reflex/compiler/compiler.py | 45 +- reflex/compiler/disk_cache.py | 502 ++++++++++++++++------- reflex/compiler/page_cache.py | 439 +++++++++++++++----- reflex/compiler/utils.py | 3 +- reflex/utils/compile_daemon.py | 467 ++++++++++++++++----- reflex/utils/telemetry.py | 22 + tests/units/compiler/test_disk_cache.py | 199 ++++++--- tests/units/compiler/test_page_cache.py | 128 ++++++ tests/units/utils/test_compile_daemon.py | 146 ++++++- 9 files changed, 1518 insertions(+), 433 deletions(-) diff --git a/reflex/compiler/compiler.py b/reflex/compiler/compiler.py index 3e1d6ccd16b..cfd78ee761b 100644 --- a/reflex/compiler/compiler.py +++ b/reflex/compiler/compiler.py @@ -218,31 +218,42 @@ def _resolve_default_color_mode(theme: Component | None) -> str: return get_config().default_color_mode -def _compile_contexts(state: type[BaseState] | None, theme: Component | None) -> str: +def _compile_contexts( + state: type[BaseState] | None, + theme: Component | None, + extra_state: tuple[dict[str, Any], dict[str, dict[str, Any]]] | None = None, +) -> str: """Compile the initial state and contexts. Args: state: The app state. theme: The top-level app theme. + extra_state: ``(initial_state, client_storage)`` entries for states that + are not in this process's state tree (states only registered while + a page evaluates, restored from the compile cache manifest). Returns: The compiled context file. """ default_color_mode = str(LiteralVar.create(_resolve_default_color_mode(theme))) - - return ( - templates.context_template( - initial_state=utils.compile_state(state), - state_name=state.get_name(), - client_storage=utils.compile_client_storage(state), - is_dev_mode=not is_prod_mode(), - default_color_mode=default_color_mode, - ) - if state - else templates.context_template( + if not state: + return templates.context_template( is_dev_mode=not is_prod_mode(), default_color_mode=default_color_mode, ) + initial_state = utils.compile_state(state) + client_storage = utils.compile_client_storage(state) + if extra_state is not None: + extra_initial, extra_storage = extra_state + initial_state = dict(sorted({**initial_state, **extra_initial}.items())) + for kind, entries in extra_storage.items(): + client_storage.setdefault(kind, {}).update(entries) + return templates.context_template( + initial_state=initial_state, + state_name=state.get_name(), + client_storage=client_storage, + is_dev_mode=not is_prod_mode(), + default_color_mode=default_color_mode, ) @@ -720,12 +731,14 @@ def compile_theme(style: ComponentStyle) -> tuple[str, str]: def compile_contexts( state: type[BaseState] | None, theme: Component | None, + extra_state: tuple[dict[str, Any], dict[str, dict[str, Any]]] | None = None, ) -> tuple[str, str]: """Compile the initial state / context. Args: state: The app state. theme: The top-level app theme. + extra_state: See :func:`_compile_contexts`. Returns: The path and code of the compiled context. @@ -733,7 +746,7 @@ def compile_contexts( # Get the path for the output file. output_path = utils.get_context_path() - return output_path, _compile_contexts(state, theme) + return output_path, _compile_contexts(state, theme, extra_state) def compile_page(path: str, component: BaseComponent) -> tuple[str, str]: @@ -1181,6 +1194,12 @@ def compile_app( should_compile = app._should_compile() backend_dir = prerequisites.get_backend_dir() if not dry_run and not should_compile and backend_dir.exists(): + if environment.REFLEX_COMPILE_CACHE.get(): + # The compile daemon reacts to the same save; read its marker only + # once it has finished rewriting .web for this change. + from reflex.utils import compile_daemon + + compile_daemon.wait_for_compile() stateful_pages_marker = backend_dir / constants.Dirs.STATEFUL_PAGES if stateful_pages_marker.exists(): with stateful_pages_marker.open("r") as file: diff --git a/reflex/compiler/disk_cache.py b/reflex/compiler/disk_cache.py index f2ccd9cdd23..605560df469 100644 --- a/reflex/compiler/disk_cache.py +++ b/reflex/compiler/disk_cache.py @@ -1,9 +1,16 @@ """Disk-persisted incremental compile cache for ``REFLEX_COMPILE_CACHE``. -The manifest stores only bookkeeping: per-page dependency hashes, app-wrap keys, -statefulness, and app-wide frontend imports. Rendered files stay in ``.web``. -When global inputs and routes still match, unchanged pages are reused from disk -and only dependency-changed pages are recompiled. +The manifest stores only bookkeeping: a file table (content hash plus stat key +per dependency file), each page's dependency paths, app-wrap keys, statefulness +and its contribution to the contexts file, the app-wide frontend imports, and +the import-name parse cache. Rendered files stay in ``.web``. When global +inputs and routes still match, unchanged pages are reused from disk and only +dependency-changed pages are recompiled. + +Validation is stat-first (see :class:`page_cache.FileValidator`): an unchanged +file is recognised from ``(mtime_ns, size)`` without being read, and when the +watcher reports which paths changed, pages depending on none of them are hits +without even a stat. """ from __future__ import annotations @@ -34,10 +41,25 @@ from reflex.app import App #: Bump when the manifest layout changes (old manifests are then ignored). -_SCHEMA = 8 +_SCHEMA = 9 #: Manifest filename under the web directory. _MANIFEST_FILE = "reflex_compile_cache.json" +#: Paths the watcher reported as changed for the current compile, resolved. +#: None when unknown (a cold start or a cache-less compile), in which case +#: every recorded dependency is validated. +_changed_hint: set[str] | None = None + + +def set_changed_hint(paths: Iterable[Path | str] | None) -> None: + """Tell the next incremental rebuild which paths the watcher saw change. + + Args: + paths: The changed paths, or None when unknown. + """ + global _changed_hint + _changed_hint = None if paths is None else {str(Path(p).resolve()) for p in paths} + def _manifest_path() -> Path: return prerequisites.get_web_dir() / _MANIFEST_FILE @@ -131,10 +153,11 @@ def _manifest_page_entry( page_ctx: PageContext, component: Any, state_index: dict[str, Path], - hasher: Callable[[str], str | None], + hasher: Callable[[str], page_cache.FileEntry | None], + files: dict[str, page_cache.FileEntry], *, is_stateful: bool, - state_fingerprint: str | None = None, + state_slice: dict[str, Any] | None = None, root: Path | None = None, ) -> dict[str, Any]: """Build the manifest entry for one compiled page. @@ -143,22 +166,23 @@ def _manifest_page_entry( page_ctx: The compiled page context. component: The page component/callable used for dependency discovery. state_index: The state-context identifier -> file index. - hasher: A memoized path -> content-hash function. + hasher: A memoized path -> file-entry function. + files: The manifest file table, extended with the page's dependencies. is_stateful: Whether the page registered state during compile. - state_fingerprint: Fingerprint of the page's contexts contribution - (see ``_contexts_fingerprint``), or None for stateless pages. + state_slice: The page's contribution to the contexts file + (see :func:`_state_slice`), or None for stateless pages. root: Project root for dependency discovery. Defaults to cwd. Returns: The JSON-able manifest entry for the page. """ return { - "dep_hashes": page_cache.page_dependency_hashes( - page_ctx, component, state_index, hasher, root + "deps": page_cache.page_dependency_entries( + page_ctx, component, state_index, hasher, files, root ), "app_wrap_keys": _wrap_key_strs(page_ctx.app_wrap_components.keys()), "is_stateful": is_stateful, - "state_fingerprint": state_fingerprint, + "state_slice": state_slice, # Whether the page contributed auto memos: pages sharing a source # module share one memo output file, so a memo-contributing hit page # must be recompiled alongside a same-module miss (see @@ -167,6 +191,40 @@ def _manifest_page_entry( } +def _state_slice( + state_names: Sequence[str], + initial_state: dict[str, Any], + client_storage: dict[str, dict[str, Any]], +) -> dict[str, Any]: + """Extract some states' contribution to the compiled contexts file. + + A state's contribution is exactly its initial-state entry plus its + client-storage entries (see ``templates.context_template``). Persisting the + slice, not just a fingerprint, lets an incremental rebuild re-emit the + contexts file for pages it did not evaluate (states only registered while + their page evaluates, e.g. exec'd docs demos) without evaluating them. + + Args: + state_names: Full names of the states to slice. + initial_state: The complete initial-state mapping (full name -> vars). + client_storage: The compiled client-storage mapping per storage kind. + + Returns: + ``{"initial_state": {...}, "client_storage": {kind: {...}}}``. + """ + names = sorted(state_names) + prefixes = tuple(f"{name}." for name in names) + return { + "initial_state": { + name: initial_state[name] for name in names if name in initial_state + }, + "client_storage": { + kind: {k: v for k, v in entries.items() if k.startswith(prefixes)} + for kind, entries in client_storage.items() + }, + } + + def _contexts_fingerprint( state_names: Sequence[str], initial_state: dict[str, Any], @@ -174,31 +232,27 @@ def _contexts_fingerprint( ) -> str: """Fingerprint some states' contribution to the compiled contexts file. - A state's contribution is exactly its initial-state slice plus its - client-storage entries (see ``templates.context_template``), so equal - fingerprints mean re-emitting the contexts file would leave these states' - entries unchanged. - Args: state_names: Full names of the states to fingerprint. initial_state: The complete initial-state mapping (full name -> vars). client_storage: The compiled client-storage mapping per storage kind. Returns: - A stable hash of the states' contexts contribution. + A stable hash of :func:`_state_slice`. """ - payload = [] - for name in sorted(state_names): - prefix = f"{name}." - payload.append(( - name, - initial_state.get(name), - { - kind: {k: v for k, v in entries.items() if k.startswith(prefix)} - for kind, entries in client_storage.items() - }, - )) - return hashlib.sha256(json_dumps(payload).encode()).hexdigest() + return _slice_digest(_state_slice(state_names, initial_state, client_storage)) + + +def _slice_digest(state_slice: dict[str, Any] | None) -> str: + """Hash a state slice for cheap equality. + + Args: + state_slice: A :func:`_state_slice` result (or None). + + Returns: + A stable digest. + """ + return hashlib.sha256(json_dumps(state_slice).encode()).hexdigest() def _contexts_snapshot( @@ -230,16 +284,16 @@ def _changed_state_config_route( ) -> str | None: """Find the first recompiled stateful route whose state config changed. - Compares each stateful miss page's just-evaluated states against the - fingerprint recorded in the manifest. Only these pages' states can differ - from the on-disk contexts file: hit pages' definitions are unchanged by + Compares each stateful miss page's just-evaluated states against the slice + recorded in the manifest. Only these pages' states can differ from the + on-disk contexts file: hit pages' definitions are unchanged by construction (all their dependency files still match). Args: manifest: The loaded compile manifest. miss_ctx: The compile context of the recompiled pages. snapshot: The ``_contexts_snapshot`` of the app, or None when there is - no state tree to fingerprint against. + no state tree to compare against. Returns: The first route requiring a contexts rebuild, or None if none do. @@ -247,15 +301,43 @@ def _changed_state_config_route( if snapshot is None: return next(iter(miss_ctx.stateful_routes)) for route, defined_states in miss_ctx.stateful_routes.items(): - stored = manifest["pages"].get(route, {}).get("state_fingerprint") - if stored != _contexts_fingerprint(defined_states, *snapshot): + stored = manifest["pages"].get(route, {}).get("state_slice") + if _slice_digest(stored) != _slice_digest( + _state_slice(defined_states, *snapshot) + ): return route return None +def _hit_state_extras( + manifest: dict[str, Any], miss_routes: set[str] +) -> tuple[dict[str, Any], dict[str, dict[str, Any]]]: + """Merge the stored contexts contributions of the stateful hit pages. + + Args: + manifest: The loaded manifest. + miss_routes: Routes recompiled in this rebuild (their states are live). + + Returns: + ``(initial_state, client_storage)`` extras for ``compile_contexts``. + """ + initial: dict[str, Any] = {} + storage: dict[str, dict[str, Any]] = {} + for route, entry in manifest["pages"].items(): + if route in miss_routes or not (state_slice := entry.get("state_slice")): + continue + initial.update(state_slice.get("initial_state", {})) + for kind, entries in state_slice.get("client_storage", {}).items(): + storage.setdefault(kind, {}).update(entries) + return initial, storage + + def load_manifest() -> dict[str, Any] | None: """Load the persisted compile manifest, or None if absent/unusable. + Also seeds the import-name parse cache from the manifest, so unchanged + first-party modules are not re-parsed in this process. + Returns: The parsed manifest dict, or None. """ @@ -268,9 +350,41 @@ def load_manifest() -> dict[str, Any] | None: return None if not isinstance(data, dict) or data.get("schema") != _SCHEMA: return None + page_cache.load_import_names_cache(data.get("import_names", {})) return data +def manifest_dependency_files(manifest: dict[str, Any]) -> set[str]: + """Every file the manifest records as a compile input. + + Args: + manifest: The loaded manifest. + + Returns: + Page dependency paths plus the file-backed global inputs. + """ + files: set[str] = set() + for entry in manifest.get("pages", {}).values(): + files.update(entry.get("deps", ())) + files.update( + path for label, path in manifest.get("globals", {}).items() if label != "reflex" + ) + files.update(manifest.get("globals_absent", ())) + return files + + +def _write(manifest: dict[str, Any]) -> None: + """Persist the manifest, including the current import-name parse cache. + + Args: + manifest: The manifest to write. + """ + manifest["import_names"] = page_cache.export_import_names_cache() + path = _manifest_path() + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(manifest), encoding="utf-8") + + def write_manifest( compile_ctx: CompileContext, pages: Sequence[PageDefinition], @@ -295,7 +409,7 @@ def write_manifest( try: state_index, _ = page_cache.state_dependency_index(root) hasher = page_cache.make_hasher() - epoch_inputs = page_cache.global_epoch_inputs(root, pages=pages) + files: dict[str, page_cache.FileEntry] = {} contexts_snapshot = ( _contexts_snapshot(compile_ctx.app) if compile_ctx.stateful_routes else None ) @@ -315,30 +429,43 @@ def write_manifest( page.component, state_index, hasher, + files, is_stateful=defined_states is not None, - state_fingerprint=( - _contexts_fingerprint(defined_states, *contexts_snapshot) + state_slice=( + _state_slice(defined_states, *contexts_snapshot) if defined_states and contexts_snapshot is not None else None ), root=root, ) + # Post-evaluation and complete: the full compile evaluated every page, + # so both import-owned and route-owned memos are recorded. + memo_files = _memo_state_entries(hasher, files, root, lambda _owner: True) + + globals_: dict[str, str] = {"reflex": page_cache._reflex_version()} + absent: list[str] = [] + for label, path in page_cache.global_input_paths(root, pages=pages).items(): + entry = hasher(path) + if entry is None: + absent.append(path) + else: + files[path] = entry + globals_[label] = path + manifest = { "schema": _SCHEMA, "reflex_version": page_cache._reflex_version(), - # Per-input digests (not one combined sha) so a later mismatch can - # name the exact global input that changed. - "epoch_inputs": epoch_inputs, + "files": files, + # Per-input labels (not one combined digest) so a later mismatch + # can name the exact global input that changed. + "globals": globals_, + "globals_absent": sorted(absent), "all_imports": _serialize_imports(install_imports), "pages": pages_data, - # Post-evaluation and complete: the full compile evaluated every - # page, so both import-owned and route-owned memos are recorded. - "memo_files": _memo_state_entries(hasher, root, lambda _owner: True), + "memo_files": memo_files, } - path = _manifest_path() - path.parent.mkdir(parents=True, exist_ok=True) - path.write_text(json.dumps(manifest), encoding="utf-8") + _write(manifest) except Exception as exc: # best-effort: never break the build console.debug(f"disk compile cache: manifest write skipped ({exc!r})") @@ -347,6 +474,7 @@ def globals_mismatch( manifest: dict[str, Any], *, routes: set[str], + validator: page_cache.FileValidator, root: Path | None = None, ) -> str | None: """Explain why the manifest's global inputs don't match, or None if they do. @@ -359,16 +487,15 @@ def globals_mismatch( set, so a shared-component or markdown edit no longer blocks the fast path. Only the pages that depend on the changed file miss. - Global inputs are validated by re-hashing the manifest's *stored* input set - (see :func:`page_cache.changed_epoch_inputs`), never by recomputing the set - in this process — set membership is only decided when a full compile writes - the manifest. + Global inputs are validated against the manifest's *stored* input set, + never by recomputing the set in this process — set membership is only + decided when a full compile writes the manifest. Args: manifest: The loaded manifest. routes: The current set of page routes. - root: Project root the stored inputs resolve against (also used to - shorten paths in the reason). Defaults to cwd. + validator: The file validator bound to the manifest's file table. + root: Project root used to shorten paths in the reason. Defaults to cwd. Returns: A human-readable mismatch reason, or None when the global inputs match. @@ -387,38 +514,50 @@ def globals_mismatch( parts.append(f"removed {format_path_list(removed)}") return f"route set changed ({'; '.join(parts)})" root = (root or Path.cwd()).resolve() - if stale := page_cache.changed_epoch_inputs(manifest.get("epoch_inputs", {}), root): - labels = {label.removeprefix("app:") for label in stale} - return f"global input(s) changed: {format_path_list(labels, root)}" + stale = { + label.removeprefix("app:") + for label, path in manifest.get("globals", {}).items() + if label != "reflex" and validator.changed(path) + } + stale.update( + path for path in manifest.get("globals_absent", ()) if Path(path).exists() + ) + if stale: + return f"global input(s) changed: {format_path_list(stale, root)}" return None def partition_pages( pages: Sequence[PageDefinition], manifest: dict[str, Any], - hasher: Callable[[str], str | None], + validator: page_cache.FileValidator, + changed_hint: set[str] | None = None, ) -> list[PageDefinition]: """Return the pages whose dependency set changed since the manifest. - Globals are assumed already matched (see :func:`globals_match`), so a page is - a hit if every file in its recorded dependency set is byte-unchanged. + Globals are assumed already matched (see :func:`globals_mismatch`), so a + page is a hit if every file in its recorded dependency set is unchanged. + When the watcher reported the changed paths, a page depending on none of + them is a hit without touching the filesystem at all. Args: pages: The current page definitions. manifest: The loaded manifest. - hasher: A memoized path -> content-hash function. + validator: The file validator bound to the manifest's file table. + changed_hint: Resolved paths the watcher saw change, or None if unknown. Returns: The list of miss pages (a dependency changed) to recompile. """ manifest_pages = manifest["pages"] - return [ - page - for page in pages - if not page_cache.deps_unchanged( - manifest_pages[page.route]["dep_hashes"], hasher - ) - ] + misses: list[PageDefinition] = [] + for page in pages: + deps = manifest_pages[page.route]["deps"] + if changed_hint is not None and changed_hint.isdisjoint(deps): + continue + if not validator.unchanged(deps): + misses.append(page) + return misses def _with_module_siblings( @@ -461,23 +600,26 @@ def _with_module_siblings( def _changed_dependency_files( - manifest: dict[str, Any], hasher: Callable[[str], str | None] + manifest: dict[str, Any], + validator: page_cache.FileValidator, + changed_hint: set[str] | None = None, ) -> set[str]: """Return every recorded dependency file whose content changed. Args: manifest: The loaded manifest. - hasher: A memoized path -> content-hash function. + validator: The file validator bound to the manifest's file table. + changed_hint: Resolved paths the watcher saw change, or None if unknown. Returns: The set of changed dependency file paths. """ - return { - path - for entry in manifest["pages"].values() - for path, digest in entry["dep_hashes"].items() - if hasher(path) != digest + candidates: Iterable[str] = { + path for entry in manifest["pages"].values() for path in entry["deps"] } + if changed_hint is not None: + candidates = changed_hint.intersection(candidates) + return {path for path in candidates if validator.changed(path)} def _module_source_file(module_name: str | None) -> str | None: @@ -519,14 +661,15 @@ def _memo_output_path(source_module: str | None) -> str | None: def _memo_state_entries( - hasher: Callable[[str], str | None], + hasher: Callable[[str], page_cache.FileEntry | None], + files: dict[str, page_cache.FileEntry], root: Path | None, include_owner: Callable[[str | None], bool], ) -> dict[str, dict[str, Any]]: """Describe the user-memo output files the current registry demands. For each mirrored memo output file: the definitions landing in it, the - content hashes of the source module's first-party import closure, and the + source module's first-party import closure (as file-table paths), and the route owning the definitions (see ``registration_owner``) — None for memos registered at module import. Import-owned entries are a pure function of app import, so any process can compare them pre-evaluation; route-owned @@ -538,14 +681,17 @@ def _memo_state_entries( :func:`_memo_defs_for_rewrite`. Args: - hasher: A memoized path -> content-hash function. + hasher: A memoized path -> :func:`page_cache.file_entry` function. + files: File table extended with the closures' entries. Kept separate + from the manifest's own table until the rebuild has finished + validating against it (see :func:`_update_manifest_for_misses`). root: Project root for the import-graph walk. None defaults to cwd. include_owner: Predicate over a definition's ``owner_route`` deciding which registry slice to describe. Returns: A JSON-able mapping of output path -> - ``def_keys``/``dep_hashes``/``owner_route``. + ``def_keys``/``deps``/``owner_route``. """ from reflex_base.components.memo import MEMOS @@ -561,14 +707,15 @@ def _memo_state_entries( module_file = _module_source_file(source_module) if module_file is None: continue + deps: list[str] = [] + for dep in sorted(page_cache.module_py_dependencies(module_file, root)): + dep_entry = hasher(dep) + if dep_entry is not None: + files[dep] = dep_entry + deps.append(dep) entry = state[path] = { "def_keys": [], - "dep_hashes": { - dep: hasher(dep) - for dep in sorted( - page_cache.module_py_dependencies(module_file, root) - ) - }, + "deps": deps, "owner_route": definition.owner_route, } entry["def_keys"].append([name, source_module]) @@ -577,6 +724,55 @@ def _memo_state_entries( return state +def _memo_deps_changed( + deps: Sequence[str], + validator: page_cache.FileValidator, + changed_hint: set[str] | None, +) -> bool: + """Whether any file in a memo record's import closure changed. + + Args: + deps: The record's dependency paths. + validator: The file validator bound to the manifest's file table. + changed_hint: Resolved paths the watcher saw change, or None if unknown. + + Returns: + True if a dependency's content changed since it was recorded. + """ + if changed_hint is not None and changed_hint.isdisjoint(deps): + return False + return not validator.unchanged(deps) + + +def _dirty_memo_paths( + state: dict[str, dict[str, Any]], + stored: dict[str, dict[str, Any]], + validator: page_cache.FileValidator, + changed_hint: set[str] | None, +) -> set[str]: + """Return the demanded memo output files whose stored record is stale. + + A record is stale when the registry's demand differs from what was stored + (a new or removed memo module, or a moved definition) or when a file in + its import closure changed (an edited memo body or helper). + + Args: + state: The memo records the current registry demands. + stored: The manifest's memo records. + validator: The file validator bound to the manifest's file table. + changed_hint: Resolved paths the watcher saw change, or None if unknown. + + Returns: + The output paths to rewrite. + """ + return { + path + for path, entry in state.items() + if stored.get(path) != entry + or _memo_deps_changed(entry["deps"], validator, changed_hint) + } + + def _memo_defs_for_rewrite( contributions: dict[tuple[str, str | None], Any], dirty_paths: set[str], @@ -663,6 +859,24 @@ def _with_memo_contributing_pages( return [*miss_pages, *extra] if extra else miss_pages +def _refresh_file_table( + manifest: dict[str, Any], validator: page_cache.FileValidator +) -> bool: + """Fold touched-but-identical files' fresh stat keys into the manifest. + + Args: + manifest: The loaded manifest (mutated). + validator: The validator whose ``refreshed`` entries to apply. + + Returns: + True if any entry was updated. + """ + if not validator.refreshed: + return False + manifest["files"].update(validator.refreshed) + return True + + def try_incremental_rebuild( app: App, *, @@ -685,9 +899,9 @@ def try_incremental_rebuild( The ``assets`` copy is excluded from dependency tracking and always re-run (cheap, idempotent). The contexts file is rewritten only when a stateful - page missed — and then only after evaluating the stateful hit pages, so the - state registry it is compiled from is complete (it must keep the states - that only a page's evaluation registers, e.g. exec'd docs demos). + miss changed its state config, and then from this process's state tree + plus the manifest's stored contributions of the stateful hit pages, so no + unchanged page is ever re-evaluated. On success, reports (at info level) how many pages were recompiled vs reused and, while recompiling, shows a progress bar over the changed pages so a hot @@ -713,14 +927,19 @@ def try_incremental_rebuild( pages = list(app._unevaluated_pages.values()) routes = {p.route for p in pages} - hasher = page_cache.make_hasher() + validator = page_cache.FileValidator(manifest["files"]) + changed_hint = _changed_hint - if (reason := globals_mismatch(manifest, routes=routes, root=root)) is not None: + if ( + reason := globals_mismatch( + manifest, routes=routes, validator=validator, root=root + ) + ) is not None: _log_fallback(reason) return False resolved_root = (root or Path.cwd()).resolve() - miss_pages = partition_pages(pages, manifest, hasher) + miss_pages = partition_pages(pages, manifest, validator, changed_hint) # Memo files are decided salsa-style: derive the files the current memo # registry demands and rewrite the ones whose stored record differs. This # catches what changed-file diffing structurally cannot: a memo module the @@ -729,24 +948,24 @@ def try_incremental_rebuild( # route-owned entries are handled by owner status: a changed dependency # pulls the owner page into the miss set, and its re-evaluation feeds the # post-evaluation comparison below. - memo_state = _memo_state_entries(hasher, resolved_root, lambda owner: owner is None) + memo_hasher = page_cache.make_hasher() + memo_files_table: dict[str, page_cache.FileEntry] = {} + memo_state = _memo_state_entries( + memo_hasher, memo_files_table, resolved_root, lambda owner: owner is None + ) stored_memo_files = manifest["memo_files"] - dirty_memo_paths = { - path - for path, entry in memo_state.items() - if stored_memo_files.get(path) != entry - } + dirty_memo_paths = _dirty_memo_paths( + memo_state, stored_memo_files, validator, changed_hint + ) stale_owner_routes = { owner for entry in stored_memo_files.values() if (owner := entry["owner_route"]) is not None - and any(hasher(dep) != digest for dep, digest in entry["dep_hashes"].items()) + and _memo_deps_changed(entry["deps"], validator, changed_hint) } changed_files: set[str] = set() if miss_pages or dirty_memo_paths or stale_owner_routes: - # Nearly free: partition_pages already hashed every dependency file - # into the memoized hasher. - changed_files = _changed_dependency_files(manifest, hasher) + changed_files = _changed_dependency_files(manifest, validator, changed_hint) miss_pages = _with_memo_contributing_pages( miss_pages, pages, manifest, dirty_memo_paths, stale_owner_routes ) @@ -865,13 +1084,14 @@ def try_incremental_rebuild( # A brand-new such module has no stored record and must be # written; an unchanged one matches its record and is skipped. eval_memo_state = _memo_state_entries( - hasher, resolved_root, lambda owner: owner in miss_routes + memo_hasher, + memo_files_table, + resolved_root, + lambda owner: owner in miss_routes, + ) + dirty_memo_paths = dirty_memo_paths | _dirty_memo_paths( + eval_memo_state, stored_memo_files, validator, changed_hint ) - dirty_memo_paths = dirty_memo_paths | { - path - for path, entry in eval_memo_state.items() - if stored_memo_files.get(path) != entry - } # Memo output files are grouped per source module, so compile them once # with the complete definition set (all recompiled pages' contributions # plus the user memos landing in a file being rewritten). @@ -891,8 +1111,7 @@ def try_incremental_rebuild( # re-evaluate hit pages to register their state in this process: it only # produces .web and exits (the daemon, the initial compile, and CLI compiles # never serve), and the serving backend re-evaluates the marked stateful - # pages itself. Re-evaluating them here re-ran the full render pipeline for - # every unchanged stateful page on every edit, for nothing. + # pages itself. stateful_routes: dict[str, None] = {} for page in pages: if page.route in miss_routes: @@ -909,13 +1128,12 @@ def try_incremental_rebuild( # The contexts file holds EVERY state's defaults/dispatchers, including # states only registered while their page evaluates (exec'd docs demos, # dynamically imported modules). This process evaluated just the miss - # pages, so its registry is incomplete; rewriting contexts from it would - # drop the hit pages' states and break the frontend's dispatch map. Only a - # stateful miss can change state config — and only its OWN states can - # differ from the on-disk contexts file, so first fingerprint those - # against the manifest: a content-only edit leaves them identical and the - # contexts file is reused untouched. Otherwise evaluate the stateful hit - # pages (so the registry is complete) and rewrite contexts. + # pages, so its state tree lacks the hit pages' page-registered states; + # their contributions come from the manifest instead (``state_slice``), + # so rebuilding contexts never re-evaluates an unchanged page. Only a + # stateful miss can change state config, and only its OWN states can differ + # from the on-disk contexts file, so compare those first: a content-only + # edit leaves them identical and the contexts file is reused untouched. contexts_snapshot: tuple[dict[str, Any], dict[str, Any]] | None = None if miss_ctx is not None and miss_ctx.stateful_routes: contexts_snapshot = _contexts_snapshot(app) @@ -932,15 +1150,6 @@ def try_incremental_rebuild( f"Compile cache: page {changed_route!r} changed its state " "config; rebuilding contexts" ) - stateful_hits = [ - route - for route, entry in manifest["pages"].items() - if entry["is_stateful"] and route not in miss_routes - ] - with console.timing("Evaluate stateful hit pages (contexts)"): - for route in stateful_hits: - app._compile_page(route, save_page=False) - from reflex_components_radix.plugin import RadixThemesPlugin theme = next( @@ -951,7 +1160,9 @@ def try_incremental_rebuild( ), None, ) - context_path, context_code = compiler.compile_contexts(app._state, theme) + context_path, context_code = compiler.compile_contexts( + app._state, theme, _hit_state_extras(manifest, miss_routes) + ) compiler_utils.write_file( compiler_utils.resolve_path_of_web_dir(context_path), context_code ) @@ -999,6 +1210,8 @@ def try_incremental_rebuild( root, contexts_snapshot=contexts_snapshot, memo_state=final_memo_state, + memo_file_entries=memo_files_table, + validator=validator, ) return True @@ -1013,25 +1226,37 @@ def _update_manifest_for_misses( *, contexts_snapshot: tuple[dict[str, Any], dict[str, Any]] | None = None, memo_state: dict[str, dict[str, Any]], + memo_file_entries: dict[str, page_cache.FileEntry], + validator: page_cache.FileValidator | None = None, ) -> None: """Update the on-disk manifest entries for the recompiled pages. + Also refreshes the stat keys of files that were touched without changing + (so they are not re-hashed on every later reload) and persists the + import-name parse cache. Rewritten only when something changed. + Args: manifest: The loaded manifest (mutated and rewritten). miss_ctx: The compile context of the recompiled pages, if any. miss_pages: The recompiled page definitions. all_imports: The complete frontend import set after recompiling misses. root: Project root for dependency discovery. Defaults to cwd. - contexts_snapshot: The app's ``_contexts_snapshot`` for fingerprinting + contexts_snapshot: The app's ``_contexts_snapshot`` for slicing stateful pages, or None when no miss page was stateful. memo_state: The current user-memo file record to persist (see - :func:`_current_memo_state`). + :func:`_memo_state_entries`). + memo_file_entries: File-table entries for the memo records' closures, + withheld until validation finished. + validator: The validator used for this rebuild, for its refreshed + entries. """ - if (miss_ctx is None or not miss_pages) and manifest["memo_files"] == memo_state: - return try: - manifest["memo_files"] = memo_state - if miss_ctx is not None: + dirty = validator is not None and _refresh_file_table(manifest, validator) + if manifest["memo_files"] != memo_state: + manifest["memo_files"] = memo_state + manifest["files"].update(memo_file_entries) + dirty = True + if miss_ctx is not None and miss_pages: state_index, _ = page_cache.state_dependency_index(root) hasher = page_cache.make_hasher() for page in miss_pages: @@ -1042,15 +1267,18 @@ def _update_manifest_for_misses( page.component, state_index, hasher, + manifest["files"], is_stateful=defined_states is not None, - state_fingerprint=( - _contexts_fingerprint(defined_states, *contexts_snapshot) + state_slice=( + _state_slice(defined_states, *contexts_snapshot) if defined_states and contexts_snapshot is not None else None ), root=root, ) - manifest["all_imports"] = _serialize_imports(all_imports) - _manifest_path().write_text(json.dumps(manifest), encoding="utf-8") + manifest["all_imports"] = _serialize_imports(all_imports) + dirty = True + if dirty: + _write(manifest) except Exception as exc: # best-effort console.debug(f"disk compile cache: manifest refresh skipped ({exc!r})") diff --git a/reflex/compiler/page_cache.py b/reflex/compiler/page_cache.py index c2a795061e3..84686a1803b 100644 --- a/reflex/compiler/page_cache.py +++ b/reflex/compiler/page_cache.py @@ -13,24 +13,37 @@ import hashlib import importlib import importlib.util +import os import re import sys -from collections.abc import Callable, Sequence +from collections.abc import Callable, Iterable, Mapping, Sequence from contextvars import ContextVar from importlib import metadata from pathlib import Path from typing import TYPE_CHECKING +from reflex_base.environment import environment + if TYPE_CHECKING: from reflex_base.components.component import BaseComponent from reflex_base.plugins import PageContext +#: A manifest file-table entry: ``[content sha256, st_mtime_ns, st_size]``. +FileEntry = list + #: Directories never worth hashing (build artifacts, deps, caches). _SKIP_DIRS = {".web", ".venv", "venv", "node_modules", "__pycache__", ".git", "assets"} #: Genuinely-global files: a change here can affect every page's output, so it #: bumps ``global_epoch`` rather than any single page's dependency set. -_GLOBAL_FILES = ("rxconfig.py", "reflex.lock", "uv.lock", "package.json") +_GLOBAL_FILES = ( + "rxconfig.py", + "reflex.lock", + "uv.lock", + "pyproject.toml", + "requirements.txt", + "package.json", +) #: Matches a JS state-context identifier in compiled page output. STATE_CONTEXT_RE = re.compile(r"reflex___state[A-Za-z0-9_]+") @@ -51,6 +64,40 @@ def _reflex_version() -> str: return "unknown" +def first_party_roots(root: Path | None = None) -> tuple[Path, ...]: + """The directories whose Python files count as the app's own source. + + The project root plus ``REFLEX_HOT_RELOAD_INCLUDE_PATHS`` (a sibling local + package the backend reloader also watches). Files under any of these are + hashed into page dependency sets; everything else is an installed + dependency whose change flows through the version/epoch instead. + + Args: + root: Project root. Defaults to cwd. + + Returns: + The resolved roots, project root first. + """ + root = (root or Path.cwd()).resolve() + include = tuple( + Path(p).resolve() for p in environment.REFLEX_HOT_RELOAD_INCLUDE_PATHS.get() + ) + return (root, *(p for p in include if p != root)) + + +def _under_any(path: Path, roots: Iterable[Path]) -> bool: + """Whether ``path`` is nested below (or is) one of ``roots``. + + Args: + path: Absolute path to test. + roots: Absolute root paths. + + Returns: + True when some root contains ``path``. + """ + return any(root == path or root in path.parents for root in roots) + + # A page's markdown/data dependencies are read lazily while the page is # evaluated. Track those reads only while ``record_reads`` is active, so the # idle overhead is one contextvar read. @@ -118,7 +165,7 @@ def _record_read(path: object) -> None: if any(part in _EXCLUDE_PARTS for part in resolved.parts): return root = _recorder_root - under_root = root is not None and root in resolved.parents + under_root = root is not None and _under_any(resolved, first_party_roots(root)) if not under_root and resolved.suffix.lower() not in _CONTENT_SUFFIXES: return target.add(str(resolved)) @@ -179,9 +226,10 @@ def _recordable_module_file(file: object) -> str | None: except (OSError, TypeError, ValueError): return None + roots = first_party_roots(root) resolved_str = None if not any(part in _EXCLUDE_PARTS for part in path.parts) and ( - _is_inside(path, root) or not _is_python_install_file(path) + _under_any(path, roots) or not _is_python_install_file(path) ): try: resolved = path.resolve() @@ -189,7 +237,9 @@ def _recordable_module_file(file: object) -> str | None: pass else: if not any(part in _EXCLUDE_PARTS for part in resolved.parts): - resolved_str = str(resolved) if _is_inside(resolved, root) else None + resolved_str = ( + str(resolved) if _under_any(resolved, roots) else None + ) _module_file_cache[file] = resolved_str return resolved_str @@ -465,6 +515,27 @@ def global_epoch_inputs( return inputs +def global_input_paths( + root: Path | None = None, *, pages: Sequence[object] | None = None +) -> dict[str, str]: + """Map each file-backed global input label to its path. + + The manifest stores these labels with their paths and keeps the content + entries in its file table, so validation is stat-first like page deps. + + Args: + root: Project root. Defaults to cwd. + pages: The current page definitions (see :func:`global_epoch_inputs`). + + Returns: + ``{label: path}`` for every global input except the Reflex version. + """ + root = (root or Path.cwd()).resolve() + paths = {name: str(root / name) for name in _GLOBAL_FILES} + paths.update({f"app:{p}": p for p in app_dependency_files(pages, root)}) + return paths + + def global_epoch( root: Path | None = None, *, pages: Sequence[object] | None = None ) -> str: @@ -483,50 +554,57 @@ def global_epoch( ) -def changed_epoch_inputs(stored: dict[str, str], root: Path | None = None) -> set[str]: - """Return the labels of stored global inputs whose current content differs. +def _module_file(component: object) -> Path | None: + mod = sys.modules.get(getattr(component, "__module__", "") or "") + file = getattr(mod, "__file__", None) + return Path(file) if file else None + + +#: Component/state class -> its resolved first-party module file (or None). +#: A rendered tree has tens of thousands of nodes but few distinct classes, +#: and a class's module never moves; resolving per node dominated the +#: dependency walk. Warmed in the daemon parent so forked children inherit it. +_class_file_cache: dict[type, Path | None] = {} - Validates against the *stored* input set (like :func:`deps_unchanged` does - for page deps): membership is decided once, when the manifest is written by - a full compile; checking only re-hashes those inputs. Recomputing the set - via :func:`global_epoch_inputs` at check time is wrong — it depends on what - the current process happened to read/import during app import, which - differs between a cold compile and a warm forked reload (unpurged module - caches skip re-reads), spuriously invalidating every hot reload. + +def _class_module_file(cls: type, roots: tuple[Path, ...]) -> Path | None: + """Resolve a class's module file when it is first-party. Args: - stored: The manifest's ``{label: digest}`` global-input map - (see :func:`global_epoch_inputs` for the label forms). - root: Project root the plain-filename labels resolve against. - Defaults to cwd. + cls: The component or state class. + roots: The first-party roots. Returns: - The labels whose content digest no longer matches. + The resolved module file under one of ``roots``, else None. """ - root = (root or Path.cwd()).resolve() - changed: set[str] = set() - for label, digest in stored.items(): - if label == "reflex": - current = _reflex_version() - else: - path = ( - Path(label.removeprefix("app:")) - if label.startswith("app:") - else root / label - ) - try: - current = hashlib.sha256(path.read_bytes()).hexdigest() - except OSError: - current = "<absent>" - if current != digest: - changed.add(label) - return changed + try: + return _class_file_cache[cls] + except KeyError: + pass + f = _module_file(cls) + resolved = None + if f is not None: + rf = f.resolve() + if _under_any(rf, roots): + resolved = rf + _class_file_cache[cls] = resolved + return resolved -def _module_file(component: object) -> Path | None: - mod = sys.modules.get(getattr(component, "__module__", "") or "") - file = getattr(mod, "__file__", None) - return Path(file) if file else None +def warm_module_file_cache(root: Path | None = None) -> None: + """Pre-resolve the module file of every loaded component and state class. + + Args: + root: Project root. Defaults to cwd. + """ + from reflex_base.components.component import Component + + from reflex.state import BaseState + + roots = first_party_roots(root) + for base in (Component, BaseState): + for cls in _subclasses(base): + _class_module_file(cls, roots) def component_module_files( @@ -545,7 +623,7 @@ def component_module_files( Returns: The set of resolved first-party module files the tree depends on. """ - root = (root or Path.cwd()).resolve() + roots = first_party_roots(root) files: set[Path] = set() seen: set[int] = set() stack: list[object] = [root_component] @@ -554,11 +632,8 @@ def component_module_files( if id(comp) in seen: continue seen.add(id(comp)) - f = _module_file(type(comp)) - if f is not None: - rf = f.resolve() - if root in rf.parents: - files.add(rf) + if (rf := _class_module_file(type(comp), roots)) is not None: + files.add(rf) children = getattr(comp, "children", None) if children: stack.extend(children) @@ -585,13 +660,14 @@ def _loaded_first_party_modules(root: Path) -> dict[str, str]: A mapping of resolved file path -> module name for loaded modules under ``root``. """ + roots = first_party_roots(root) file_to_mod: dict[str, str] = {} for name, mod in list(sys.modules.items()): file = getattr(mod, "__file__", None) if not file: continue rf = Path(file).resolve() - if root in rf.parents: + if _under_any(rf, roots): file_to_mod[str(rf)] = name return file_to_mod @@ -625,36 +701,111 @@ def _import_from_targets(node: object, modname: str) -> list[str]: return [base, *(f"{base}.{a.name}" for a in node.names)] -def _module_import_edges( - file: str, modname: str, file_to_mod: dict[str, str] -) -> set[str]: - """Return first-party files imported by one module. +#: file -> ((st_mtime_ns, st_size), imported dotted names). Parsing every +#: first-party module on every hot reload is the cost this avoids; the daemon +#: persists it in the manifest so a fresh process only parses changed files. +_import_names_cache: dict[str, tuple[tuple[int, int], list[str]]] = {} + + +def _stat_key(path: str) -> tuple[int, int] | None: + """Return ``(st_mtime_ns, st_size)`` for ``path``, or None if unreadable. + + Args: + path: The file path. + + Returns: + The stat key, or None. + """ + try: + st = Path(path).stat() + except OSError: + return None + return (st.st_mtime_ns, st.st_size) + + +def load_import_names_cache(data: Mapping[str, Sequence[object]]) -> None: + """Seed the import-name parse cache from its persisted form. + + Args: + data: ``{path: [[mtime_ns, size], [names...]]}`` as written by + :func:`export_import_names_cache`. + """ + _import_names_cache.clear() + for path, entry in data.items(): + if ( + isinstance(entry, Sequence) + and len(entry) == 2 + and isinstance(entry[0], Sequence) + and len(entry[0]) == 2 + and all(isinstance(n, int) for n in entry[0]) + and isinstance(entry[1], Sequence) + ): + key = (int(entry[0][0]), int(entry[0][1])) # type: ignore[index] + _import_names_cache[path] = (key, [str(n) for n in entry[1]]) + + +def export_import_names_cache() -> dict[str, list[object]]: + """Return the import-name parse cache in its persisted form. + + Returns: + ``{path: [[mtime_ns, size], [names...]]}``. + """ + return { + path: [list(key), names] for path, (key, names) in _import_names_cache.items() + } + + +def _module_import_names(file: str, modname: str) -> list[str]: + """Return the dotted names one module imports (statically). + + Cached by the file's stat key, so an unchanged file is never re-parsed. Args: file: The resolved module file path. modname: The module's dotted name. - file_to_mod: Loaded first-party module files. Returns: - The resolved first-party files imported by ``file``. + The imported dotted-name candidates (see :func:`_import_from_targets`). """ import ast - deps: set[str] = set() + key = _stat_key(file) + cached = _import_names_cache.get(file) + if cached is not None and key is not None and cached[0] == key: + return cached[1] + names: list[str] = [] try: tree = ast.parse(Path(file).read_bytes()) except (OSError, SyntaxError, ValueError): - return deps + return names for node in ast.walk(tree): - names: list[str] = [] if isinstance(node, ast.Import): - names = [a.name for a in node.names] + names.extend(a.name for a in node.names) elif isinstance(node, ast.ImportFrom): - names = _import_from_targets(node, modname) - for name in names: - target = _resolve_module_file(name) - if target is not None and target in file_to_mod: - deps.add(target) + names.extend(_import_from_targets(node, modname)) + if key is not None: + _import_names_cache[file] = (key, names) + return names + + +def _module_import_edges( + file: str, modname: str, file_to_mod: dict[str, str] +) -> set[str]: + """Return first-party files imported by one module. + + Args: + file: The resolved module file path. + modname: The module's dotted name. + file_to_mod: Loaded first-party module files. + + Returns: + The resolved first-party files imported by ``file``. + """ + deps: set[str] = set() + for name in _module_import_names(file, modname): + target = _resolve_module_file(name) + if target is not None and target in file_to_mod: + deps.add(target) return deps @@ -734,13 +885,14 @@ def _component_source_files(component: object, root: Path) -> set[str]: The set of resolved defining file path strings under ``root``. """ out: set[str] = set() + roots = first_party_roots(root) code = getattr(component, "__code__", None) filename = getattr(code, "co_filename", None) own = _module_file(component) for path in (filename, own): if path: rf = Path(path).resolve() - if root in rf.parents: + if _under_any(rf, roots): out.add(str(rf)) return out @@ -826,28 +978,100 @@ def app_dependency_files( return static_deps | dynamic_deps -def make_hasher() -> Callable[[str], str | None]: - """Return a content-hasher that memoizes each path within one compile. +def file_entry(path: str) -> FileEntry | None: + """Read a file once and describe it for the manifest file table. + + Args: + path: The file path. + + Returns: + ``[sha256, st_mtime_ns, st_size]``, or None if unreadable. + """ + try: + with Path(path).open("rb") as fh: + st = os.fstat(fh.fileno()) + digest = hashlib.sha256(fh.read()).hexdigest() + except OSError: + return None + return [digest, st.st_mtime_ns, st.st_size] + + +def make_hasher() -> Callable[[str], FileEntry | None]: + """Return a file-table entry builder that memoizes each path within one compile. - Shared component files appear in many pages' dependency sets; hashing each - file at most once keeps per-page validation cheap. + Shared component files appear in many pages' dependency sets; reading each + file at most once keeps manifest writes cheap. Returns: - A function mapping a path string to its content hash (None if unreadable). + A function mapping a path string to its :func:`file_entry` (None if + unreadable). """ - cache: dict[str, str | None] = {} + cache: dict[str, FileEntry | None] = {} - def hasher(path: str) -> str | None: + def hasher(path: str) -> FileEntry | None: if path not in cache: - try: - cache[path] = hashlib.sha256(Path(path).read_bytes()).hexdigest() - except OSError: - cache[path] = None + cache[path] = file_entry(path) return cache[path] return hasher +class FileValidator: + """Decide whether recorded files still match their manifest entries. + + Validation is stat-first: a file whose ``(mtime_ns, size)`` still matches + its entry is unchanged without being read. Only a file whose stat moved is + hashed, and a touch that left the content identical counts as unchanged + (its refreshed entry is exposed in ``refreshed`` so the manifest can stop + re-hashing it). Each path is examined at most once per compile. + """ + + def __init__(self, files: Mapping[str, Sequence[object]]) -> None: + """Bind the validator to a manifest file table. + + Args: + files: ``{path: [sha256, mtime_ns, size]}``. + """ + self._files = files + self._changed: dict[str, bool] = {} + self.refreshed: dict[str, FileEntry] = {} + + def changed(self, path: str) -> bool: + """Whether ``path`` differs from its recorded entry (or has none). + + Args: + path: The file path. + + Returns: + True if the file changed, is unreadable, or is not in the table. + """ + cached = self._changed.get(path) + if cached is not None: + return cached + entry = self._files.get(path) + result = True + if entry is not None and len(entry) == 3: + key = _stat_key(path) + if key is not None and (key[0], key[1]) == (entry[1], entry[2]): + result = False + elif (fresh := file_entry(path)) is not None and fresh[0] == entry[0]: + self.refreshed[path] = fresh + result = False + self._changed[path] = result + return result + + def unchanged(self, paths: Iterable[str]) -> bool: + """Whether every file in ``paths`` is unchanged. + + Args: + paths: The recorded dependency paths of one page. + + Returns: + True if no file changed. + """ + return not any(self.changed(path) for path in paths) + + def _subclasses(root_cls: type) -> list[type]: seen: set[type] = set() out: list[type] = [] @@ -880,24 +1104,19 @@ def state_dependency_index( Returns: ``(identifier_to_file, fine_state_files)``. """ - root = (root or Path.cwd()).resolve() + roots = first_party_roots(root) from reflex_base.components.component import Component from reflex_base.utils import format as fmt from reflex.state import BaseState - def under_root(comp: object) -> Path | None: - f = _module_file(comp) - if f is None: - return None - rf = f.resolve() - return rf if root in rf.parents else None - - component_files = {rf for cls in _subclasses(Component) if (rf := under_root(cls))} + component_files = { + rf for cls in _subclasses(Component) if (rf := _class_module_file(cls, roots)) + } id_to_file: dict[str, Path] = {} state_files: set[Path] = set() for cls in _subclasses(BaseState): - rf = under_root(cls) + rf = _class_module_file(cls, roots) if rf is None: continue with contextlib.suppress(Exception): @@ -979,48 +1198,42 @@ def page_dependency_files( # Transitive first-party .py imports (captures function-based views/helpers # that never appear as nodes in the rendered tree). deps |= page_py_dependencies(component, root) - # Files read while evaluating the page (markdown/data). + # Files read while evaluating the page (markdown/data), plus the import + # closure of any module the page imported at evaluation time: a helper + # already loaded by an earlier page does not re-execute its own imports + # under the recorder, so its dependencies come from the graph instead. deps |= set(page_ctx.source_files) + dynamic_py = {f for f in page_ctx.source_files if f.endswith(".py")} + if dynamic_py: + deps |= _walk_import_closure(build_import_graph(root), dynamic_py) return deps -def page_dependency_hashes( +def page_dependency_entries( page_ctx: PageContext, component: BaseComponent | object, state_index: dict[str, Path], - hasher: Callable[[str], str | None], + hasher: Callable[[str], FileEntry | None], + files: dict[str, FileEntry], root: Path | None = None, -) -> dict[str, str]: - """Hash a page's dependency set into a ``{path: hash}`` map. +) -> list[str]: + """Record a page's dependency set into the manifest file table. Args: page_ctx: The compiled page context. component: The page's component/callable. state_index: The state-context identifier -> file index. - hasher: A memoized path -> content-hash function (see :func:`make_hasher`). + hasher: A memoized path -> :func:`file_entry` function. + files: The manifest file table, extended in place. root: Project root. Defaults to cwd. Returns: - ``{path: content_hash}`` for every readable dependency file. + The sorted dependency paths that were readable (the page's ``deps``). """ - out: dict[str, str] = {} - for path in page_dependency_files(page_ctx, component, state_index, root): - digest = hasher(path) - if digest is not None: - out[path] = digest - return out - - -def deps_unchanged( - dep_hashes: dict[str, str], hasher: Callable[[str], str | None] -) -> bool: - """Whether every file in a stored dependency set still has the same content. - - Args: - dep_hashes: A page's stored ``{path: hash}`` dependency map. - hasher: A memoized path -> content-hash function. - - Returns: - True if every dependency file is byte-unchanged. - """ - return all(hasher(path) == digest for path, digest in dep_hashes.items()) + deps: list[str] = [] + for path in sorted(page_dependency_files(page_ctx, component, state_index, root)): + entry = hasher(path) + if entry is not None: + files[path] = entry + deps.append(path) + return deps diff --git a/reflex/compiler/utils.py b/reflex/compiler/utils.py index 6b01b3738a6..201f3013ab1 100644 --- a/reflex/compiler/utils.py +++ b/reflex/compiler/utils.py @@ -849,7 +849,8 @@ def write_file(path: str | Path, code: str): if path.exists() and path.read_text(encoding="utf-8") == code: return # Write atomically so readers never observe a half-written file. - tmp = path.with_name(f"{path.name}.{os.getpid()}.tmp") + # Dot-prefixed so route discovery and file watchers over .web ignore it. + tmp = path.with_name(f".{path.name}.{os.getpid()}.tmp") try: tmp.write_text(code, encoding="utf-8") tmp.replace(path) diff --git a/reflex/utils/compile_daemon.py b/reflex/utils/compile_daemon.py index 69d1ffa07ac..e9f98b4cea9 100644 --- a/reflex/utils/compile_daemon.py +++ b/reflex/utils/compile_daemon.py @@ -16,21 +16,23 @@ import sys import threading import time +from collections.abc import Callable, Iterator +from dataclasses import dataclass, field from pathlib import Path from reflex_base.environment import environment from reflex.utils import console -#: Seconds between poll passes. Each pass only re-``stat``s the known file set -#: (cheap), so this can be small for snappy detection of edits to existing files. -_POLL_INTERVAL = 0.05 -#: After a change is seen, wait this long and re-snapshot so a burst of saves -#: (e.g. format-on-save touching many files) collapses into a single compile. -_DEBOUNCE = 0.05 -#: How often to re-walk the tree to discover added/removed files. -#: Between rescans we only ``stat`` the known set; adds are rarer and tolerate a -#: little latency, so this keeps idle CPU low while polling fast for edits. +#: Fallback polling only (no ``watchfiles``): seconds between poll passes. +_POLL_INTERVAL = 0.5 +#: Milliseconds a burst of saves (e.g. format-on-save touching many files) is +#: allowed to settle before it is reported as one change set. +_DEBOUNCE_MS = 50 +#: How long the OS watcher blocks before yielding control back, so the daemon +#: can notice its parent died even while no file changes. +_WATCH_TIMEOUT_MS = 1000 +#: Fallback polling only: how often to re-walk the tree for added files. _RESCAN_INTERVAL = 1.0 #: Watchdog: kill a compile child that runs longer than this (a hung/deadlocked #: child must never wedge the daemon). Generous enough for a real full compile. @@ -143,13 +145,26 @@ def _iter_source_files(root: Path): yield path.resolve() +def _manifest_dependency_files() -> set[Path]: + """Every source dependency the last compile recorded, from the disk manifest. + + Returns: + Resolved dependency file paths (empty without a manifest). + """ + from reflex.compiler import disk_cache + + manifest = disk_cache.load_manifest() + if not manifest: + return set() + return {Path(dep) for dep in disk_cache.manifest_dependency_files(manifest)} + + def _external_dependency_files(roots: list[Path]) -> set[Path]: - """External content files the compiler read, taken from the disk manifest. + """Recorded dependencies that live *outside* the reload roots. - The manifest records each page's full dependency set (own module, markdown, - component/state modules). Any dependency that lives *outside* the reload - roots, such as a docs app's markdown in a sibling directory, is invisible to - ``get_reload_paths`` and must be watched explicitly so editing it rebuilds. + Such files (a docs app's markdown in a sibling directory, say) are invisible + to ``get_reload_paths`` and must be watched explicitly so editing them + rebuilds. Args: roots: The resolved reload roots. @@ -157,18 +172,7 @@ def _external_dependency_files(roots: list[Path]) -> set[Path]: Returns: Resolved external dependency file paths to watch. """ - from reflex.compiler import disk_cache - - manifest = disk_cache.load_manifest() - if not manifest: - return set() - out: set[Path] = set() - for page in manifest.get("pages", {}).values(): - for dep in page.get("dep_hashes", {}): - path = Path(dep) - if not _under_roots(path, roots): - out.add(path) - return out + return {p for p in _manifest_dependency_files() if not _under_roots(p, roots)} def _global_files(root: Path) -> set[Path]: @@ -204,29 +208,89 @@ def _mtime_ns(path: Path) -> int | None: return None -def _watch_paths(roots: list[Path], root: Path, external_files: set[Path]) -> set[Path]: - """Build the full set of files to watch this tick. +@dataclass +class _WatchState: + """What the daemon watches; refreshed after every compile.""" + + roots: list[Path] + root: Path + #: Every dependency the last compile recorded (any suffix), so an edit to a + #: JSON/CSV/etc. file a page reads triggers a rebuild too. + known: set[Path] = field(default_factory=set) + globals_: set[Path] = field(default_factory=set) + + @classmethod + def build(cls, roots: list[Path], root: Path) -> _WatchState: + """Snapshot the current watch inputs from the manifest and roots. + + Args: + roots: The resolved reload roots. + root: The project root. + + Returns: + The populated watch state. + """ + return cls( + roots=roots, + root=root, + known=_manifest_dependency_files(), + globals_=_global_files(root), + ) - ``external_files`` (the per-page external content deps from the manifest) is - passed in rather than re-read here, so the manifest is parsed once per compile - instead of on every poll tick. + @property + def assets(self) -> Path | None: + """The assets directory, when the project has one.""" + assets = self.root / "assets" + return assets if assets.is_dir() else None + + def targets(self) -> list[Path]: + """Directories/files handed to the OS watcher. + + Returns: + The reload roots, the parents of recorded dependencies outside + them, the project root's global files and the assets directory. + """ + targets: dict[Path, None] = dict.fromkeys(self.roots) + for dep in self.known: + if not _under_roots(dep, self.roots): + targets[dep.parent] = None + for path in self.globals_: + targets[path] = None + if (assets := self.assets) is not None: + targets[assets] = None + return [t for t in targets if t.exists()] + + def accepts(self, path: Path) -> bool: + """Whether a changed path is a compile input worth reacting to. + + Args: + path: The changed path, as reported by the watcher. + + Returns: + True for watched-suffix sources, recorded dependencies, global + inputs and assets; False for build output and unrelated files. + """ + if any(part in _SKIP_DIRS for part in path.parts): + return False + if path in self.globals_ or path in self.known: + return True + if (assets := self.assets) is not None and _under_roots(path, [assets]): + return True + return path.suffix in _WATCH_SUFFIXES and _under_roots(path, self.roots) - Args: - roots: The resolved reload roots. - root: The project root. - external_files: External content dependency files (from the manifest). + def watch_paths(self) -> set[Path]: + """Fallback polling only: the full file set to snapshot each tick. - Returns: - The set of paths to snapshot. - """ - paths: set[Path] = set(external_files) - for r in roots: - paths.update(_iter_source_files(r)) - paths.update(_global_files(root)) - assets = root / "assets" - if assets.is_dir(): - paths.update(p.resolve() for p in assets.rglob("*") if p.is_file()) - return paths + Returns: + The set of paths to stat. + """ + paths: set[Path] = set(self.known) + for r in self.roots: + paths.update(_iter_source_files(r)) + paths.update(self.globals_) + if (assets := self.assets) is not None: + paths.update(p.resolve() for p in assets.rglob("*") if p.is_file()) + return paths def _snapshot(paths: set[Path]) -> dict[Path, int]: @@ -241,6 +305,98 @@ def _snapshot(paths: set[Path]) -> dict[Path, int]: return {p: m for p in paths if (m := _mtime_ns(p)) is not None} +def _poll_changes(state: _WatchState, alive: Callable[[], bool]) -> Iterator[set[Path]]: + """Fallback change source when ``watchfiles`` is unavailable. + + Re-stats the known file set every ``_POLL_INTERVAL`` and re-walks the tree + every ``_RESCAN_INTERVAL`` to discover added files. + + Args: + state: The live watch state (read on every rescan, so a refresh after a + compile takes effect without restarting the poller). + alive: Returns False when the watcher should stop. + + Yields: + Each non-empty set of changed paths. + """ + paths = state.watch_paths() + snapshot = _snapshot(paths) + last_rescan = time.monotonic() + while alive(): + time.sleep(_POLL_INTERVAL) + if time.monotonic() - last_rescan >= _RESCAN_INTERVAL: + paths = state.watch_paths() + last_rescan = time.monotonic() + current = _snapshot(paths) + changed = { + p + for p in current.keys() | snapshot.keys() + if current.get(p) != snapshot.get(p) + } + if changed: + time.sleep(_DEBOUNCE_MS / 1000) # absorb the rest of a burst + yield changed + paths = state.watch_paths() + snapshot = _snapshot(paths) + last_rescan = time.monotonic() + else: + snapshot = current + + +def _iter_changes(state: _WatchState, alive: Callable[[], bool]) -> Iterator[set[Path]]: + """Yield sets of changed compile inputs as the OS reports them. + + Subscribes to filesystem notifications (``watchfiles``: inotify, FSEvents, + ReadDirectoryChangesW) over ``state.targets()`` and filters events with + ``state.accepts``. Polling is only a fallback when ``watchfiles`` is not + importable. The watcher is re-created when the target set changes (a + compile recorded a new external content directory). + + Args: + state: The live watch state; ``targets``/``accepts`` are re-read after + every yielded change set. + alive: Returns False when the watcher should stop. + + Yields: + Each non-empty, accepted set of changed paths. + """ + try: + import watchfiles + except ImportError: + console.warn( + "watchfiles is not installed; the compile daemon falls back to polling." + ) + yield from _poll_changes(state, alive) + return + + def watch_filter(_change: object, raw_path: str) -> bool: + return state.accepts(Path(raw_path)) + + while alive(): + targets = state.targets() + for events in watchfiles.watch( + *targets, + watch_filter=watch_filter, + debounce=_DEBOUNCE_MS, + step=25, + rust_timeout=_WATCH_TIMEOUT_MS, + yield_on_timeout=True, + raise_interrupt=False, + ): + if not alive(): + return + changed = {Path(raw_path) for _change, raw_path in events} + if changed: + yield changed + if state.targets() != targets: + break # re-create the watcher over the new target set + + +#: ``__file__`` -> whether it is under the reload roots; the "" key holds the +#: roots the cache was built for. +_first_party_file_cache: dict[str, object] = {} + + def _first_party_module_names(roots: list[Path]) -> set[str]: """Names of all loaded modules belonging to the user's first-party packages. @@ -257,16 +413,26 @@ def _first_party_module_names(roots: list[Path]) -> set[str]: Returns: The set of ``sys.modules`` keys to purge. """ + roots_key = tuple(str(r) for r in roots) + if _first_party_file_cache.get("") != roots_key: + _first_party_file_cache.clear() + _first_party_file_cache[""] = roots_key top_level: set[str] = set() for name, mod in list(sys.modules.items()): file = getattr(mod, "__file__", None) if not file: continue - try: - rf = Path(file).resolve() - except OSError: - continue - if _under_roots(rf, roots): + # Resolving every loaded module's file is thousands of realpath + # syscalls per hot reload; module files never move, so classify each + # ``__file__`` once. The parent warms this before forking. + is_first_party = _first_party_file_cache.get(file) + if is_first_party is None: + try: + is_first_party = _under_roots(Path(file).resolve(), roots) + except OSError: + is_first_party = False + _first_party_file_cache[file] = is_first_party + if is_first_party: top_level.add(name.partition(".")[0]) if not top_level: return set() @@ -370,7 +536,100 @@ def _reset_model_metadata() -> None: ModelRegistry._metadata = None -def _child_compile(roots: list[Path], prerender_routes: bool) -> None: +#: Name of the env var carrying the changed-path hint to a cold compile child. +_CHANGED_ENV = "REFLEX_COMPILE_CHANGED" +#: Lock file (under the backend dir) held while a compile is in progress, so a +#: backend reload worker can wait for the fresh stateful-pages marker. +_COMPILE_LOCK_FILE = ".compile_in_progress" +#: Give up waiting on the lock after this long (a wedged daemon must not hang +#: the backend forever; its own watchdog is ``_COMPILE_TIMEOUT``). +_LOCK_WAIT_TIMEOUT = _COMPILE_TIMEOUT + 10 + + +def _lock_path() -> Path: + from reflex.utils import prerequisites + + return prerequisites.get_backend_dir() / _COMPILE_LOCK_FILE + + +@contextlib.contextmanager +def _compile_lock(root: Path): + """Hold the compile-in-progress lock while a hot-reload compile runs. + + The backend's reload worker reacts to the same file save as the daemon. + Without this it can restart, read the previous ``stateful_pages`` marker + and register the old state set before the daemon has rewritten it (see + :func:`wait_for_compile`). + + Args: + root: The project root. + + Yields: + None. + """ + lock = _lock_path() + lock.parent.mkdir(parents=True, exist_ok=True) + lock.write_text(str(os.getpid())) + try: + yield + finally: + lock.unlink(missing_ok=True) + + +def _lock_holder_alive(lock: Path) -> bool: + try: + pid = int(lock.read_text().strip()) + os.kill(pid, 0) + except (OSError, ValueError): + return False + return True + + +def wait_for_compile(timeout: float = _LOCK_WAIT_TIMEOUT) -> None: + """Block while the compile daemon is rewriting ``.web`` for the same change. + + Called by a backend worker that skips frontend compilation + (``REFLEX_SKIP_COMPILE``) before it reads the stateful-pages marker. Returns + immediately when no compile is in progress, when the lock holder died, or + after ``timeout``. + + Args: + timeout: Maximum seconds to wait. + """ + lock = _lock_path() + deadline = time.monotonic() + timeout + waited = False + while lock.exists() and time.monotonic() < deadline: + if not _lock_holder_alive(lock): + break + if not waited: + console.debug( + "Waiting for the compile daemon to finish the current compile." + ) + waited = True + time.sleep(0.02) + + +def _prepare_fork_parent(roots: list[Path]) -> None: + """Make the warm parent a good fork base. + + Stops the framework's background threads (so ``_can_fork`` passes) and warms + the per-process caches every child would otherwise rebuild: the first-party + module classification and the compile cache's component-module lookups. + + Args: + roots: The resolved reload roots. + """ + from reflex.compiler import page_cache + + _quiesce_parent() + _first_party_module_names(roots) + page_cache.warm_module_file_cache() + + +def _child_compile( + roots: list[Path], prerender_routes: bool, changed: set[Path] | None = None +) -> None: """Reset first-party state, re-import the app fresh, and compile incrementally. Runs in a forked child (POSIX) or a one-shot subprocess (Windows). Must not @@ -379,9 +638,14 @@ def _child_compile(roots: list[Path], prerender_routes: bool) -> None: Args: roots: The resolved reload roots. prerender_routes: Whether to prerender routes during compile. + changed: The paths the watcher reported for this reload, when known. + Lets the incremental rebuild skip validating pages that depend on + none of them. """ + from reflex.compiler import disk_cache from reflex.utils import prerequisites + disk_cache.set_changed_hint(changed) # Timed in three steps so every hot reload reports where it spent its time # (resetting state vs re-importing first-party code vs compiling). t0 = time.perf_counter() @@ -422,6 +686,20 @@ def _await_child(pid: int) -> bool: time.sleep(0.02) +def _quiesce_parent() -> None: + """Stop the framework's own background threads before forking. + + The telemetry worker (a ``ThreadPoolExecutor`` thread) outlives the + initial compile's ``compile`` event and would otherwise make every later + ``_can_fork`` check fail, silently downgrading each hot reload to a cold + subprocess. Anything the user's app started at import time is out of our + hands and still forces the fallback. + """ + from reflex.utils import telemetry + + telemetry.shutdown_executor() + + def _can_fork() -> bool: """Whether forking is safe right now (POSIX and the process is single-threaded). @@ -433,10 +711,24 @@ def _can_fork() -> bool: Returns: True if a per-compile ``fork()`` is safe. """ - return hasattr(os, "fork") and threading.active_count() == 1 + if not hasattr(os, "fork"): + return False + _quiesce_parent() + if threading.active_count() == 1: + return True + others = [ + t.name for t in threading.enumerate() if t is not threading.current_thread() + ] + console.warn( + "Compile daemon: cannot fork a warm child while other threads are " + f"alive ({', '.join(others)}); falling back to a cold subprocess." + ) + return False -def _compile_once(roots: list[Path], prerender_routes: bool) -> bool: +def _compile_once( + roots: list[Path], prerender_routes: bool, changed: set[Path] | None = None +) -> bool: """Run one incremental compile in an isolated child; report success. Uses a copy-on-write ``fork()`` when safe (warm), else a fresh subprocess @@ -445,6 +737,7 @@ def _compile_once(roots: list[Path], prerender_routes: bool) -> bool: Args: roots: The resolved reload roots. prerender_routes: Whether to prerender routes during compile. + changed: The paths the watcher reported for this reload, when known. Returns: True if the child compiled successfully, else False. @@ -454,7 +747,7 @@ def _compile_once(roots: list[Path], prerender_routes: bool) -> bool: if pid == 0: # child code = 0 try: - _child_compile(roots, prerender_routes) + _child_compile(roots, prerender_routes, changed) except BaseException: # report any failure, never crash the daemon import traceback @@ -465,11 +758,15 @@ def _compile_once(roots: list[Path], prerender_routes: bool) -> bool: return _await_child(pid) # No fork (Windows) or unsafe to fork: a fresh (cold) subprocess compiles. + env = {**os.environ} + if changed is not None: + env[_CHANGED_ENV] = "\n".join(sorted(str(p) for p in changed)) try: proc = subprocess.run( [sys.executable, "-m", "reflex.utils.compile_daemon", "--once"], check=False, timeout=_COMPILE_TIMEOUT, + env=env, ) except subprocess.TimeoutExpired: console.error("Compile subprocess timed out; keeping the last good build.") @@ -503,37 +800,16 @@ def _serve() -> None: traceback.print_exc() console.error("Initial compile failed; watching for a fix.") + _prepare_fork_parent(roots) - # External content deps (e.g. a docs app's sibling-dir markdown) come from - # the manifest; recompute only after a compile, not on every poll tick. - external = _external_dependency_files(roots) - global_files = _global_files(root) - # `paths` is the watched set, refreshed by a tree rescan; each tick only - # re-stats it (cheap), so the poll can be fast without burning idle CPU. - paths = _watch_paths(roots, root, external) - snapshot = _snapshot(paths) - last_rescan = time.monotonic() + state = _WatchState.build(roots, root) console.info("Compile daemon ready (warm); watching for changes.") - while True: - time.sleep(_POLL_INTERVAL) + def alive() -> bool: # Never outlive reflex-run: if our parent died we were reparented. - if os.getppid() != parent_pid: - return - # Cheap stat of the known set every tick; re-walk the tree occasionally - # to discover added/removed files. - if time.monotonic() - last_rescan >= _RESCAN_INTERVAL: - paths = _watch_paths(roots, root, external) - last_rescan = time.monotonic() - current = _snapshot(paths) - changed = { - p - for p in current.keys() | snapshot.keys() - if current.get(p) != snapshot.get(p) - } - if not changed: - continue + return os.getppid() == parent_pid + for changed in _iter_changes(state, alive): from reflex.compiler.disk_cache import format_path_list console.info( @@ -543,25 +819,20 @@ def _serve() -> None: # A change to a genuinely-global input (rxconfig/lockfiles, or a reflex # upgrade) can't be applied to the warm parent (it imported the old # version); re-exec the daemon so the new world is actually loaded. - if changed & global_files: + if changed & state.globals_: console.info("Global config changed; restarting compile daemon.") os.execv( sys.executable, [sys.executable, "-m", "reflex.utils.compile_daemon"] ) - time.sleep(_DEBOUNCE) # absorb the rest of a burst of saves - roots = _reload_roots() - ok = _compile_once(roots, prerender_routes) + state.roots = roots = _reload_roots() + with _compile_lock(root): + ok = _compile_once(roots, prerender_routes, changed) if not ok: console.error("Compile failed; keeping the last good build.") - # Re-snapshot AFTER compiling so writes the compile itself made are the - # new baseline; refresh external deps + globals from the new manifest so - # a newly-referenced content dir becomes watched. - external = _external_dependency_files(roots) - global_files = _global_files(root) - paths = _watch_paths(roots, root, external) - snapshot = _snapshot(paths) - last_rescan = time.monotonic() + # Refresh what is watched from the new manifest so a newly-referenced + # content file becomes watched. + state = _WatchState.build(roots, root) def main(argv: list[str] | None = None) -> int: @@ -575,9 +846,17 @@ def main(argv: list[str] | None = None) -> int: """ argv = sys.argv[1:] if argv is None else argv if "--once" in argv: + changed_env = os.environ.get(_CHANGED_ENV) + changed = ( + {Path(line) for line in changed_env.splitlines() if line} + if changed_env is not None + else None + ) try: _child_compile( - _reload_roots(), bool(os.environ.get("REFLEX_PRERENDER_ROUTES")) + _reload_roots(), + bool(os.environ.get("REFLEX_PRERENDER_ROUTES")), + changed, ) except BaseException: # report any failure, never crash import traceback diff --git a/reflex/utils/telemetry.py b/reflex/utils/telemetry.py index ccad3bacd46..02d561b1c20 100644 --- a/reflex/utils/telemetry.py +++ b/reflex/utils/telemetry.py @@ -546,6 +546,28 @@ def _flush(timeout: float | None = None) -> bool: return True +def shutdown_executor(timeout: float | None = 5.0) -> None: + """Drain and stop the telemetry worker thread, if one was started. + + The worker outlives the events it delivered: ``ThreadPoolExecutor`` keeps + an idle thread alive until shutdown. A process that needs to be + single-threaded again (the compile daemon forks a child per hot reload, + which is only safe with no other threads) calls this after its last + telemetry send. A later send lazily creates a fresh executor. + + Args: + timeout: Maximum seconds to wait for queued events to drain first. + """ + global _executor + with _executor_lock: + executor, _executor = _executor, None + if executor is None: + return + with suppress(Exception): + executor.submit(lambda: None).result(timeout) + executor.shutdown(wait=True, cancel_futures=True) + + _legacy_alias_attempted = False diff --git a/tests/units/compiler/test_disk_cache.py b/tests/units/compiler/test_disk_cache.py index 264b42799b7..384d03a3c10 100644 --- a/tests/units/compiler/test_disk_cache.py +++ b/tests/units/compiler/test_disk_cache.py @@ -1,7 +1,6 @@ """Tests for the experimental disk-persisted incremental compile cache.""" import dataclasses -import hashlib import itertools import json import sys @@ -141,7 +140,9 @@ def _manifest(pages: dict[str, dict], **overrides) -> dict: base = { "schema": disk_cache._SCHEMA, "reflex_version": page_cache._reflex_version(), - "epoch_inputs": {}, + "files": {}, + "globals": {"reflex": page_cache._reflex_version()}, + "globals_absent": [], "all_imports": {}, "pages": pages, "memo_files": {}, @@ -150,18 +151,48 @@ def _manifest(pages: dict[str, dict], **overrides) -> dict: return base +def _stale_dep(manifest: dict, route: str, path: str) -> None: + """Make ``route`` depend on ``path`` with a recorded entry that cannot match. + + Simulates an edit: the page's dependency set is rewritten to a file whose + stored stat/hash no longer match the current content, so partitioning sees + the page as a miss. + + Args: + manifest: The manifest to mutate. + route: The page route. + path: The dependency path. + """ + manifest["pages"][route]["deps"] = [path] + manifest["files"][path] = ["stale-hash", 0, 0] + + +def _validator(manifest: dict) -> page_cache.FileValidator: + return page_cache.FileValidator(manifest["files"]) + + def test_globals_mismatch_names_the_changed_input(tmp_path): - m = _manifest({"/a": {}, "/b": {}}, epoch_inputs={"rxconfig.py": "<absent>"}) + m = _manifest({"/a": {}, "/b": {}}, globals_absent=[str(tmp_path / "rxconfig.py")]) routes = {"/a", "/b"} - assert disk_cache.globals_mismatch(m, routes=routes, root=tmp_path) is None + assert ( + disk_cache.globals_mismatch( + m, routes=routes, validator=_validator(m), root=tmp_path + ) + is None + ) # a changed route set -> named added/removed routes - reason = disk_cache.globals_mismatch(m, routes={"/a", "/c"}, root=tmp_path) + reason = disk_cache.globals_mismatch( + m, routes={"/a", "/c"}, validator=_validator(m), root=tmp_path + ) assert reason is not None assert "/c" in reason assert "/b" in reason # a stale reflex version -> named versions reason = disk_cache.globals_mismatch( - {**m, "reflex_version": "0.0.0-old"}, routes=routes, root=tmp_path + {**m, "reflex_version": "0.0.0-old"}, + routes=routes, + validator=_validator(m), + root=tmp_path, ) assert reason is not None assert "0.0.0-old" in reason @@ -178,25 +209,35 @@ def test_globals_mismatch_validates_stored_inputs_only(tmp_path): """ theme = tmp_path / "theme_config.py" theme.write_text("PRIMARY = 'red'") - stored = { - "reflex": page_cache._reflex_version(), + entry = page_cache.file_entry(str(theme)) + assert entry is not None + m = _manifest( + {"/a": {}}, + files={str(theme): entry}, + globals={"reflex": page_cache._reflex_version(), f"app:{theme}": str(theme)}, # global files absent from this root at write time and still absent - "rxconfig.py": "<absent>", - f"app:{theme}": hashlib.sha256(theme.read_bytes()).hexdigest(), - } - m = _manifest({"/a": {}}, epoch_inputs=stored) + globals_absent=[str(tmp_path / "rxconfig.py")], + ) + + def mismatch() -> str | None: + return disk_cache.globals_mismatch( + m, routes={"/a"}, validator=_validator(m), root=tmp_path + ) + # nothing on disk changed -> match, regardless of what a re-recorded # app-import read set would look like in this process - assert disk_cache.globals_mismatch(m, routes={"/a"}, root=tmp_path) is None + assert mismatch() is None # a stored input's content changed -> mismatch naming that file theme.write_text("PRIMARY = 'blue'") - reason = disk_cache.globals_mismatch(m, routes={"/a"}, root=tmp_path) + reason = mismatch() assert reason is not None assert "theme_config.py" in reason - # a stored global file appearing counts as a change too + # restored content with a fresh mtime: hashed once, still a match theme.write_text("PRIMARY = 'red'") + assert mismatch() is None + # a stored global file appearing counts as a change too (tmp_path / "rxconfig.py").write_text("import reflex") - reason = disk_cache.globals_mismatch(m, routes={"/a"}, root=tmp_path) + reason = mismatch() assert reason is not None assert "rxconfig.py" in reason @@ -209,19 +250,32 @@ def test_format_path_list_relativizes_and_truncates(): assert out == "0.py, 1.py, 2.py (+5 more)" -def test_partition_pages_detects_changed_source(): +def test_partition_pages_detects_changed_source(tmp_path): pages = [ _FakePage(route="/a", component=_page_a), _FakePage(route="/b", component=_page_b), ] - # /a depends on x.py, /b depends on y.py (each at a recorded content hash). - m = _manifest({ - "/a": {"dep_hashes": {"/proj/x.py": "h-x"}}, - "/b": {"dep_hashes": {"/proj/y.py": "h-y"}}, - }) - # The hasher reports /a's dep unchanged and /b's dep changed. - current = {"/proj/x.py": "h-x", "/proj/y.py": "h-y-new"} - miss = disk_cache.partition_pages(pages, m, lambda p: current.get(p)) + x, y = tmp_path / "x.py", tmp_path / "y.py" + x.write_text("X = 1\n") + y.write_text("Y = 1\n") + # /a depends on x.py, /b depends on y.py (each recorded in the file table). + m = _manifest( + {"/a": {"deps": [str(x)]}, "/b": {"deps": [str(y)]}}, + files={ + str(x): page_cache.file_entry(str(x)), + str(y): page_cache.file_entry(str(y)), + }, + ) + assert disk_cache.partition_pages(pages, m, _validator(m)) == [] + y.write_text("Y = 2\n") + miss = disk_cache.partition_pages(pages, m, _validator(m)) + assert {p.route for p in miss} == {"/b"} + # With the watcher's changed set, pages depending on none of the changed + # paths are hits without touching the filesystem. + assert ( + disk_cache.partition_pages(pages, m, _validator(m), changed_hint={str(x)}) == [] + ) + miss = disk_cache.partition_pages(pages, m, _validator(m), changed_hint={str(y)}) assert {p.route for p in miss} == {"/b"} @@ -244,16 +298,22 @@ def test_write_and_load_manifest(tmp_path, monkeypatch): entry = manifest["pages"][route] # the manifest is pure bookkeeping: dep set + app-wrap keys + flags assert set(entry) == { - "dep_hashes", + "deps", "app_wrap_keys", "is_stateful", - "state_fingerprint", + "state_slice", "has_memos", } # these static pages register no new state and contribute no memos assert entry["is_stateful"] is False - assert entry["state_fingerprint"] is None + assert entry["state_slice"] is None assert entry["has_memos"] is False + # every dependency has a file-table entry: [sha256, mtime_ns, size] + for dep in entry["deps"]: + digest, mtime_ns, size = manifest["files"][dep] + assert len(digest) == 64 + assert mtime_ns > 0 + assert size > 0 # rendered output is never persisted (it already lives in .web, and is # never read back from the manifest) -> keeps the manifest small assert "output_code" not in entry @@ -353,7 +413,10 @@ def _stub_externals(app, monkeypatch): # classes from other collected test modules. monkeypatch.setattr( "reflex.compiler.compiler.compile_contexts", - lambda state, theme: (compiler_utils.get_context_path(), _CONTEXTS_STUB), + lambda state, theme, extra_state=None: ( + compiler_utils.get_context_path(), + _CONTEXTS_STUB, + ), ) monkeypatch.setattr(disk_cache, "_contexts_snapshot", _scoped_contexts_snapshot) monkeypatch.setattr(fs, "update_react_router_config", lambda **k: None) @@ -400,9 +463,7 @@ def test_incremental_rebuild_one_miss_writes_only_that_page(tmp_path, monkeypatc edited_route = pages[0].route manifest_path = web / disk_cache._MANIFEST_FILE manifest = json.loads(manifest_path.read_text()) - manifest["pages"][edited_route]["dep_hashes"] = { - str(tmp_path / "view.py"): "stale-hash" - } + _stale_dep(manifest, edited_route, str(tmp_path / "view.py")) manifest_path.write_text(json.dumps(manifest)) assert ( @@ -524,13 +585,13 @@ def _page_s() -> Component: return rx.el.div(rx.el.p(state_cls.value), _footer()) -def test_stateful_miss_evaluates_stateful_hits_then_rewrites_contexts( - tmp_path, monkeypatch -): - """A stateful miss forces a contexts rewrite from a *complete* registry. +def test_stateful_miss_rewrites_contexts_from_manifest_slices(tmp_path, monkeypatch): + """A stateful miss rewrites contexts without re-evaluating any hit page. - The stateful hit pages must be evaluated first (registering their - evaluation-time states) so the rewritten contexts file keeps every state. + Hit pages' evaluation-time states are not in this process's state tree; + their recorded contribution (``state_slice``) is handed to + ``compile_contexts`` instead, so the rewritten contexts file keeps every + state and no unchanged page is evaluated. """ from reflex.compiler import utils as compiler_utils @@ -550,16 +611,32 @@ def test_stateful_miss_evaluates_stateful_hits_then_rewrites_contexts( # rebuild must re-register its states before compiling contexts. manifest_path = web / disk_cache._MANIFEST_FILE manifest = json.loads(manifest_path.read_text()) - manifest["pages"][stateful_route]["dep_hashes"] = { - str(tmp_path / "view.py"): "stale-hash" + _stale_dep(manifest, stateful_route, str(tmp_path / "view.py")) + hit_slice = { + "initial_state": {"root.hit_state": {"count": 0}}, + "client_storage": { + "cookies": {"root.hit_state.token_rx_field_": {"name": "t"}}, + "local_storage": {}, + "session_storage": {}, + }, } manifest["pages"][hit_route]["is_stateful"] = True + manifest["pages"][hit_route]["state_slice"] = hit_slice manifest_path.write_text(json.dumps(manifest)) reevaluated: list[str] = [] monkeypatch.setattr( app, "_compile_page", lambda route, **k: reevaluated.append(route) ) + captured: list[Any] = [] + + def fake_compile_contexts(state, theme, extra_state=None): + captured.append(extra_state) + return compiler_utils.get_context_path(), _CONTEXTS_STUB + + monkeypatch.setattr( + "reflex.compiler.compiler.compile_contexts", fake_compile_contexts + ) assert ( disk_cache.try_incremental_rebuild( @@ -567,9 +644,9 @@ def test_stateful_miss_evaluates_stateful_hits_then_rewrites_contexts( ) is True ) - # Only the stateful HIT page is re-evaluated (the miss was compiled in the - # miss context); then contexts are rewritten from the complete registry. - assert reevaluated == [hit_route] + # No page is re-evaluated; the hit page's stored slice is merged instead. + assert reevaluated == [] + assert captured == [(hit_slice["initial_state"], hit_slice["client_storage"])] out_path = compiler_utils.resolve_path_of_web_dir(compiler_utils.get_context_path()) assert out_path.read_text(encoding="utf-8") == _CONTEXTS_STUB @@ -657,13 +734,11 @@ def test_stateful_miss_with_unchanged_states_reuses_contexts(tmp_path, monkeypat manifest_path = web / disk_cache._MANIFEST_FILE manifest = json.loads(manifest_path.read_text()) - stored_fp = manifest["pages"][stateful_route]["state_fingerprint"] + stored_fp = manifest["pages"][stateful_route]["state_slice"] assert stored_fp # the compile recorded the page's state config # Make the stateful page a miss; mark the hit page stateful so a contexts # rebuild (if wrongly triggered) would have to re-evaluate it. - manifest["pages"][stateful_route]["dep_hashes"] = { - str(tmp_path / "view.py"): "stale-hash" - } + _stale_dep(manifest, stateful_route, str(tmp_path / "view.py")) manifest["pages"][hit_route]["is_stateful"] = True manifest_path.write_text(json.dumps(manifest)) @@ -691,7 +766,7 @@ def test_stateful_miss_with_unchanged_states_reuses_contexts(tmp_path, monkeypat assert out_path.read_text(encoding="utf-8") == sentinel # The refreshed manifest records the same fingerprint for the miss page. refreshed = json.loads(manifest_path.read_text()) - assert refreshed["pages"][stateful_route]["state_fingerprint"] == stored_fp + assert refreshed["pages"][stateful_route]["state_slice"] == stored_fp _unregister_state(_FP_HOLDER.pop("cls")) @@ -752,16 +827,17 @@ def test_incremental_rebuild_rewrites_changed_user_memo( disk_cache.write_manifest(ctx, pages, ctx.all_imports, root=tmp_path) _stub_externals(app, monkeypatch) - # Simulate an edit to this module (which defines the user memo): record a - # stale hash for its file in the page's dep set (so the importing page - # misses) and in the stored memo record (so the memo file is dirty). + # Simulate an edit to this module (which defines the user memo): record an + # unmatchable entry for its file, which both the page's dep set (so the + # importing page misses) and the stored memo record's closure (so the memo + # file is dirty) resolve through the shared file table. module_file = str(Path(__file__).resolve()) manifest_path = web / disk_cache._MANIFEST_FILE manifest = json.loads(manifest_path.read_text()) - manifest["pages"][memo_route]["dep_hashes"] = {module_file: "stale-hash"} memo_path = disk_cache._memo_output_path(__name__) assert memo_path is not None - manifest["memo_files"][memo_path]["dep_hashes"][module_file] = "stale-hash" + assert module_file in manifest["memo_files"][memo_path]["deps"] + _stale_dep(manifest, memo_route, module_file) manifest_path.write_text(json.dumps(manifest)) assert ( @@ -815,9 +891,7 @@ def test_new_memo_module_is_emitted_on_first_use( # (so it misses), and no recorded memo state mentions the module. manifest_path = web / disk_cache._MANIFEST_FILE manifest = json.loads(manifest_path.read_text()) - manifest["pages"][memo_route]["dep_hashes"] = { - str(tmp_path / "app.py"): "stale-hash" - } + _stale_dep(manifest, memo_route, str(tmp_path / "app.py")) manifest["memo_files"] = {} manifest_path.write_text(json.dumps(manifest)) @@ -864,7 +938,7 @@ def test_route_owned_memo_entry_carried_while_owner_hits( manifest = json.loads(manifest_path.read_text()) manifest["memo_files"][str(lazy_file)] = { "def_keys": [["Lazy", "lazy_module"]], - "dep_hashes": {}, + "deps": [], "owner_route": owner_route, } manifest_path.write_text(json.dumps(manifest)) @@ -927,9 +1001,7 @@ def _page_lazy() -> Component: # Surgery: the previous compile never saw the lazy module, and the # page misses via an unrelated changed file. del manifest["memo_files"][lazy_path] - manifest["pages"][lazy_route]["dep_hashes"] = { - str(tmp_path / "app.py"): "stale-hash" - } + _stale_dep(manifest, lazy_route, str(tmp_path / "app.py")) manifest_path.write_text(json.dumps(manifest)) assert ( @@ -1006,7 +1078,7 @@ def test_incremental_miss_keeps_sibling_memo_exports( # Make only page E miss (a dependency of E changed, e.g. a data file). manifest_path = web / disk_cache._MANIFEST_FILE manifest = json.loads(manifest_path.read_text()) - manifest["pages"][route_e]["dep_hashes"] = {str(tmp_path / "data.md"): "stale-hash"} + _stale_dep(manifest, route_e, str(tmp_path / "data.md")) manifest_path.write_text(json.dumps(manifest)) assert ( @@ -1036,7 +1108,7 @@ def test_update_manifest_for_misses_keeps_complete_imports(tmp_path, monkeypatch monkeypatch.setattr( page_cache, "state_dependency_index", lambda root=None: ({}, set()) ) - monkeypatch.setattr(page_cache, "page_dependency_hashes", lambda *a, **k: {}) + monkeypatch.setattr(page_cache, "page_dependency_entries", lambda *a, **k: []) page = _FakePage(route="/a", component=_page_a) page_ctx = SimpleNamespace( @@ -1045,7 +1117,7 @@ def test_update_manifest_for_misses_keeps_complete_imports(tmp_path, monkeypatch miss_ctx = SimpleNamespace(compiled_pages={"/a": page_ctx}, stateful_routes={}) complete_imports = {"memo-lib": [ImportVar("MemoThing")]} manifest = _manifest({ - "/a": {"dep_hashes": {}, "app_wrap_keys": [], "is_stateful": False} + "/a": {"deps": [], "app_wrap_keys": [], "is_stateful": False} }) disk_cache._update_manifest_for_misses( @@ -1055,6 +1127,7 @@ def test_update_manifest_for_misses_keeps_complete_imports(tmp_path, monkeypatch complete_imports, root=tmp_path, memo_state={}, + memo_file_entries={}, ) written = json.loads((web / disk_cache._MANIFEST_FILE).read_text()) diff --git a/tests/units/compiler/test_page_cache.py b/tests/units/compiler/test_page_cache.py index a954167a6bb..090ca2c9e6f 100644 --- a/tests/units/compiler/test_page_cache.py +++ b/tests/units/compiler/test_page_cache.py @@ -1,6 +1,7 @@ """Tests for the per-page dependency graph used by the incremental compile cache.""" import importlib +import os import sys from pathlib import Path @@ -500,3 +501,130 @@ def test_used_state_files_from_output_and_memos(tmp_path): # un-introspectable memo -> conservative (all fine files) boom = SimpleNamespace(render=lambda: (_ for _ in ()).throw(RuntimeError())) assert page_cache.used_state_files(out, [boom], id_to_file) == {sfile, mfile} + + +def test_file_validator_is_stat_first(tmp_path, monkeypatch): + """Unchanged files are recognised from their stat key without being read.""" + f = tmp_path / "dep.py" + f.write_text("x = 1\n") + entry = page_cache.file_entry(str(f)) + assert entry is not None + files = {str(f): entry} + + reads = 0 + original = page_cache.file_entry + + def counting(path): + nonlocal reads + reads += 1 + return original(path) + + monkeypatch.setattr(page_cache, "file_entry", counting) + v = page_cache.FileValidator(files) + assert v.changed(str(f)) is False + assert reads == 0 # stat matched, never read + assert v.unchanged([str(f)]) is False or True # memoized, still no read + assert reads == 0 + + # Touched with identical content: hashed once, unchanged, entry refreshed. + bumped = f.stat().st_mtime_ns + 1_000_000_000 + os.utime(f, ns=(bumped, bumped)) + v = page_cache.FileValidator(files) + assert v.changed(str(f)) is False + assert reads == 1 + assert v.refreshed[str(f)][1] == bumped + + # Same size, different content: the hash decides. + f.write_text("x = 2\n") + v = page_cache.FileValidator(files) + assert v.changed(str(f)) is True + # Unknown or missing files always count as changed. + assert v.changed(str(tmp_path / "missing.py")) is True + files[str(tmp_path / "gone.py")] = ["h", 1, 1] + assert page_cache.FileValidator(files).changed(str(tmp_path / "gone.py")) is True + + +def test_import_names_parse_cache_skips_unchanged_files(tmp_path, monkeypatch): + mod = tmp_path / "mod_a.py" + mod.write_text("import os\nfrom pathlib import Path\n") + page_cache._import_names_cache.clear() + + parses = 0 + import ast + + original = ast.parse + + def counting(*args, **kwargs): + nonlocal parses + parses += 1 + return original(*args, **kwargs) + + monkeypatch.setattr(ast, "parse", counting) + names = page_cache._module_import_names(str(mod), "mod_a") + assert names == ["os", "pathlib", "pathlib.Path"] + assert parses == 1 + assert page_cache._module_import_names(str(mod), "mod_a") == names + assert parses == 1 # cached by stat key + + # The cache round-trips through its persisted form. + exported = page_cache.export_import_names_cache() + page_cache._import_names_cache.clear() + page_cache.load_import_names_cache(exported) + assert page_cache._module_import_names(str(mod), "mod_a") == names + assert parses == 1 + # Malformed persisted entries are ignored, not fatal. + page_cache.load_import_names_cache({str(mod): "garbage", "x": [[1], []]}) + assert page_cache._import_names_cache == {} + + # An edit re-parses. + mod.write_text("import sys\n") + assert page_cache._module_import_names(str(mod), "mod_a") == ["sys"] + assert parses == 2 + + +def test_first_party_roots_include_hot_reload_include_paths(tmp_path, monkeypatch): + from reflex_base.environment import environment + + sibling = tmp_path / "sibling_pkg" + sibling.mkdir() + monkeypatch.setenv(environment.REFLEX_HOT_RELOAD_INCLUDE_PATHS.name, str(sibling)) + roots = page_cache.first_party_roots(tmp_path) + assert roots == (tmp_path.resolve(), sibling.resolve()) + + # A module under the include path is first-party for dependency tracking. + page_cache.enable_read_tracking(root=tmp_path) + module_file = sibling / "helper.py" + module_file.write_text("VALUE = 1\n") + assert page_cache._recordable_module_file(str(module_file)) == str( + module_file.resolve() + ) + assert page_cache._recordable_module_file(str(tmp_path / ".venv" / "x.py")) is None + + +def test_page_dependency_files_closes_over_dynamic_imports(tmp_path, monkeypatch): + """A module imported at evaluation time brings its own import closure. + + The helper may already be loaded (an earlier page imported it), so its own + imports never re-execute under the recorder; the graph supplies them. + """ + from types import SimpleNamespace + + helper = _prepare_runtime_module(tmp_path, monkeypatch, "dyn_helper_for_closure") + leaf = _prepare_runtime_module(tmp_path, monkeypatch, "dyn_leaf_for_closure") + helper.write_text("import dyn_leaf_for_closure\n") + try: + importlib.import_module("dyn_helper_for_closure") + page_cache.clear_import_graph() + page_ctx = SimpleNamespace( + root_component=SimpleNamespace(children=[]), + output_code="", + memo_contributions={}, + source_files={str(helper.resolve())}, + ) + deps = page_cache.page_dependency_files(page_ctx, lambda: None, {}, tmp_path) + finally: + _forget_modules("dyn_helper_for_closure", "dyn_leaf_for_closure") + page_cache.clear_import_graph() + + assert str(helper.resolve()) in deps + assert str(leaf.resolve()) in deps diff --git a/tests/units/utils/test_compile_daemon.py b/tests/units/utils/test_compile_daemon.py index ea9d858b552..523a7834322 100644 --- a/tests/units/utils/test_compile_daemon.py +++ b/tests/units/utils/test_compile_daemon.py @@ -1,6 +1,7 @@ """Tests for the warm fork-per-compile dev compile daemon.""" import os +from pathlib import Path import pytest @@ -61,10 +62,10 @@ def test_external_dependency_files_includes_sibling_markdown(tmp_path, monkeypat manifest = { "pages": { "/g": { - "dep_hashes": { - str(own_module): "h1", # under the app root -> not external - str(sibling_md): "h2", # sibling dir -> must be watched - } + "deps": [ + str(own_module), # under the app root -> not external + str(sibling_md), # sibling dir -> must be watched + ] } } } @@ -91,26 +92,124 @@ def test_snapshot_detects_external_markdown_change(tmp_path, monkeypatch): monkeypatch.setattr( disk_cache, "load_manifest", - lambda: {"pages": {"/g": {"dep_hashes": {str(sibling_md): "h"}}}}, + lambda: {"pages": {"/g": {"deps": [str(sibling_md)]}}}, ) roots = [app_root.resolve()] - external = compile_daemon._external_dependency_files(roots) - snap1 = compile_daemon._snapshot( - compile_daemon._watch_paths(roots, tmp_path, external) - ) + state = compile_daemon._WatchState.build(roots, tmp_path.resolve()) + snap1 = compile_daemon._snapshot(state.watch_paths()) assert sibling_md.resolve() in snap1 assert (app_root / "page.py").resolve() in snap1 + # The OS watcher is pointed at the sibling directory too. + assert sibling_md.resolve().parent in state.targets() # Force a distinct mtime (deterministic, independent of fs mtime resolution). bumped = snap1[sibling_md.resolve()] + 1_000_000_000 os.utime(sibling_md, ns=(bumped, bumped)) - snap2 = compile_daemon._snapshot( - compile_daemon._watch_paths(roots, tmp_path, external) - ) + snap2 = compile_daemon._snapshot(state.watch_paths()) assert snap2[sibling_md.resolve()] != snap1[sibling_md.resolve()] +def test_watch_state_accepts_only_compile_inputs(tmp_path, monkeypatch): + """Recorded data deps of any suffix count; build output never does.""" + app_root = tmp_path / "app" + app_root.mkdir() + data = app_root / "table.json" + data.write_text("{}") + monkeypatch.setattr( + disk_cache, "load_manifest", lambda: {"pages": {"/t": {"deps": [str(data)]}}} + ) + (tmp_path / "assets").mkdir() + state = compile_daemon._WatchState.build([app_root.resolve()], tmp_path.resolve()) + + assert state.accepts(app_root / "page.py") + assert state.accepts(app_root / "guide.md") + assert state.accepts(data) # not a watched suffix, but a recorded dependency + assert state.accepts(tmp_path / "assets" / "logo.svg") + assert not state.accepts(app_root / "other.json") # unrecorded data file + assert not state.accepts(tmp_path / ".web" / "app" / "routes" / "x.jsx") + assert not state.accepts(tmp_path / "elsewhere" / "page.py") # outside roots + + +def test_iter_changes_reports_an_edit(tmp_path): + """The OS watcher reports an edited source file (no polling involved).""" + import threading + import time + + pytest.importorskip("watchfiles") + app_root = tmp_path / "app" + app_root.mkdir() + page = app_root / "page.py" + page.write_text("x = 1\n") + state = compile_daemon._WatchState([app_root.resolve()], tmp_path.resolve()) + got: list[set] = [] + + def run() -> None: + for changed in compile_daemon._iter_changes(state, lambda: not got): + got.append(changed) + return + + thread = threading.Thread(target=run, daemon=True) + thread.start() + deadline = time.monotonic() + 10 + while not got and time.monotonic() < deadline: + page.write_text(f"x = {time.monotonic()}\n") + time.sleep(0.2) + thread.join(5) + assert got + assert page.resolve() in {p.resolve() for p in got[0]} + + +def test_wait_for_compile_respects_lock(tmp_path, monkeypatch): + import subprocess + import sys + import time + + monkeypatch.setattr("reflex.utils.prerequisites.get_backend_dir", lambda: tmp_path) + # No lock -> returns at once. + start = time.monotonic() + compile_daemon.wait_for_compile(timeout=1) + assert time.monotonic() - start < 0.2 + # A live holder -> waits until the lock clears (or the timeout). + with compile_daemon._compile_lock(tmp_path): + start = time.monotonic() + compile_daemon.wait_for_compile(timeout=0.3) + assert time.monotonic() - start >= 0.3 + assert not compile_daemon._lock_path().exists() + # A dead holder -> not waited on. + proc = subprocess.Popen([sys.executable, "-c", "pass"]) + proc.wait() + compile_daemon._lock_path().write_text(str(proc.pid)) + start = time.monotonic() + compile_daemon.wait_for_compile(timeout=5) + assert time.monotonic() - start < 0.5 + + +def test_first_party_module_names_caches_file_classification(tmp_path, monkeypatch): + """Module files are classified once; a repeat pass resolves no paths.""" + import sys + import types + + mod = types.ModuleType("fp_cached_mod") + mod.__file__ = str(tmp_path / "pkg" / "cached_mod.py") + monkeypatch.setitem(sys.modules, "fp_cached_mod", mod) + roots = [tmp_path.resolve()] + assert "fp_cached_mod" in compile_daemon._first_party_module_names(roots) + assert compile_daemon._first_party_file_cache[mod.__file__] is True + + calls = 0 + original = Path.resolve + + def counting_resolve(self, *args, **kwargs): + nonlocal calls + calls += 1 + return original(self, *args, **kwargs) + + monkeypatch.setattr(Path, "resolve", counting_resolve) + assert "fp_cached_mod" in compile_daemon._first_party_module_names(roots) + assert calls == 0 + + def test_first_party_module_names_includes_namespace_packages(tmp_path, monkeypatch): """Namespace packages (no ``__file__``) are captured for purge by name. @@ -337,3 +436,26 @@ def test_reset_first_party_purges_modules_and_registries(tmp_path): os.close(read_fd) os.waitpid(pid, 0) assert out == b"1" + + +def test_quiesce_parent_stops_telemetry_worker(): + """The telemetry worker must not keep the daemon parent multi-threaded. + + The initial compile sends a telemetry event, whose executor thread would + otherwise outlive it and make every later fork check fail silently. + """ + import threading + + from reflex.utils import telemetry + + telemetry._get_telemetry_executor().submit(lambda: None).result() + assert any(t.name.startswith("reflex-telemetry") for t in threading.enumerate()) + + compile_daemon._quiesce_parent() + + assert telemetry._executor is None + assert not any(t.name.startswith("reflex-telemetry") for t in threading.enumerate()) + # A later send lazily recreates a worker, so shutdown is not one-way. + telemetry._get_telemetry_executor().submit(lambda: None).result() + telemetry.shutdown_executor() + assert telemetry._executor is None From 7e38b65570420401544293368c45011271f1725a Mon Sep 17 00:00:00 2001 From: Claude <noreply@anthropic.com> Date: Sat, 5 Sep 2026 04:00:12 +0000 Subject: [PATCH 21/56] compile cache: seed generated names per page so partial recompiles stay unique Generated ref/loop names were drawn from one process-wide sequence that the compile reset at its start, so a page's names depended on which pages compiled before it: an incremental compile of page 3 alone reproduced the names page 1 already held (masenf's review on #6688). Seed the generator per page from its route, capture the state after evaluation and resume it for the page's render, and reset once for whatever compiles after the pages. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_015uV6WEQPemtsE9Nm6WFxyE --- news/6688.feature.md | 2 +- packages/reflex-base/news/6688.misc.md | 2 +- .../src/reflex_base/plugins/compiler.py | 22 +++++++++-- .../reflex-base/src/reflex_base/vars/base.py | 37 +++++++++++++++++++ tests/units/compiler/test_disk_cache.py | 36 ++++++++++++++++++ tests/units/compiler/test_page_cache.py | 5 ++- tests/units/vars/test_base.py | 32 ++++++++++++++++ 7 files changed, 129 insertions(+), 7 deletions(-) diff --git a/news/6688.feature.md b/news/6688.feature.md index 5ae88d4e82b..a42b4e87ab4 100644 --- a/news/6688.feature.md +++ b/news/6688.feature.md @@ -1 +1 @@ -Added an experimental disk-persisted incremental compile cache, enabled by the `REFLEX_COMPILE_CACHE` environment variable. When on, a fresh compile reuses the previous build already on disk in `.web` and recompiles only the pages whose source changed, tracked via a per-page dependency graph (Python import closure, files read during page evaluation, component modules, and referenced state). App-wide inputs (Reflex version, config/lockfiles, and the app entrypoint's config modules such as theme/app-wraps/stylesheets) gate the whole cache, falling back to a full compile when they change. `reflex run` dev additionally gains a warm fork-per-compile daemon so hot reloads skip the cold reimport and rebuild only what changed. Off by default — the compile path is unchanged when the flag is unset. +Added an experimental disk-persisted incremental compile cache, enabled by the `REFLEX_COMPILE_CACHE` environment variable. When on, a fresh compile reuses the previous build already on disk in `.web` and recompiles only the pages whose source changed, tracked via a per-page dependency graph (Python import closure, files read during page evaluation, component modules, and referenced state). Dependencies are validated stat-first, so unchanged files are never re-read, and the contexts file is rebuilt from the manifest without re-evaluating unchanged pages. App-wide inputs (Reflex version, config/lockfiles, and the app entrypoint's config modules such as theme/app-wraps/stylesheets) gate the whole cache, falling back to a full compile when they change. `reflex run` dev additionally gains a warm fork-per-compile daemon that watches the project with `watchfiles` (polling only as a fallback), so hot reloads skip the cold reimport and rebuild only what changed; the backend reload worker waits for that compile before registering state. Off by default — the compile path is unchanged when the flag is unset. diff --git a/packages/reflex-base/news/6688.misc.md b/packages/reflex-base/news/6688.misc.md index d0cfb39144c..c628017e8da 100644 --- a/packages/reflex-base/news/6688.misc.md +++ b/packages/reflex-base/news/6688.misc.md @@ -1 +1 @@ -Added an optional per-page source-read recorder hook (`page_source_recorder` in the compiler plugin) used by the incremental compile cache to track the exact files each page reads during evaluation, and made auto-generated unique ref names reproducible across in-process compiles so memo content hashes stay stable. No behavior change unless the compile cache is enabled. +Added an optional per-page source-read recorder hook (`page_source_recorder` in the compiler plugin) used by the incremental compile cache to track the exact files each page reads during evaluation. Auto-generated unique ref names are now seeded per page from its route (and the page's sequence is resumed for its render), so a page's generated names are reproducible and independent of which other pages compile alongside it. No behavior change unless the compile cache is enabled. diff --git a/packages/reflex-base/src/reflex_base/plugins/compiler.py b/packages/reflex-base/src/reflex_base/plugins/compiler.py index 8c9303c5d3e..3adbda82426 100644 --- a/packages/reflex-base/src/reflex_base/plugins/compiler.py +++ b/packages/reflex-base/src/reflex_base/plugins/compiler.py @@ -697,6 +697,9 @@ class PageContext(BaseContext): source_module: str | None = None # Source files read while evaluating this page, when a recorder is installed. source_files: set[str] = dataclasses.field(default_factory=set) + # The page's unique-name generator state after evaluation, resumed for its + # render so its generated names never depend on other pages. + unique_name_state: Any = None # Auto-memo components first registered while compiling this page. memo_contributions: dict[tuple[str, str | None], Any] = dataclasses.field( default_factory=dict @@ -808,7 +811,12 @@ def compile( from reflex.compiler import compiler from reflex.state import all_base_state_classes from reflex_base.components.memo import registration_owner - from reflex_base.vars.base import reset_unique_variable_names + from reflex_base.vars.base import ( + reset_unique_variable_names, + restore_unique_variable_name_state, + seed_unique_variable_names, + unique_variable_name_state, + ) self.ensure_context_attached() self.compiled_pages.clear() @@ -818,13 +826,14 @@ def compile( self.memoize_wrappers.clear() self.auto_memo_components.clear() - # Keep generated ref names stable across in-process compiles. - reset_unique_variable_names() - recorder = page_source_recorder for page in self.pages: page_fn = page.component n_states_before = len(all_base_state_classes) + # Names generated for a page are a function of the page alone, so + # recompiling any subset of pages reproduces them (see + # ``seed_unique_variable_names``). + seed_unique_variable_names(page.route) # Memos registered while this page evaluates (e.g. from modules # imported inside the page function) are owned by this route. with registration_owner(page.route): @@ -864,6 +873,7 @@ def compile( n_states_before: ] + page_ctx.unique_name_state = unique_variable_name_state() self.compiled_pages[page_ctx.route] = page_ctx if evaluate_progress is not None: @@ -875,6 +885,7 @@ def compile( strict=True, ): memo_before = set(self.auto_memo_components) + restore_unique_variable_name_state(page_ctx.unique_name_state) with page_ctx: page_ctx.root_component = self.hooks.compile_component( page_ctx.root_component, @@ -906,6 +917,9 @@ def compile( if render_progress is not None: render_progress() + # Whatever compiles after the pages (app root, memo modules) starts + # from the same state on every compile. + reset_unique_variable_names() return self.compiled_pages diff --git a/packages/reflex-base/src/reflex_base/vars/base.py b/packages/reflex-base/src/reflex_base/vars/base.py index dbe4f2da2e0..31ca1e22eac 100644 --- a/packages/reflex-base/src/reflex_base/vars/base.py +++ b/packages/reflex-base/src/reflex_base/vars/base.py @@ -3309,6 +3309,43 @@ def reset_unique_variable_names() -> None: _UNIQUE_NAME_RNG.seed(42) +def seed_unique_variable_names(seed: str) -> None: + """Start a fresh, seed-determined name sequence (one per compiled page). + + Generated names are order-dependent, so a process-wide sequence gives a + page different names depending on which pages compiled before it. An + incremental compile of a subset of pages would then reuse names that + other, cached pages already hold. Seeding per page from its route makes + each page's names a function of the page alone. + + Args: + seed: The page route (or another stable label). + """ + USED_VARIABLES.clear() + _UNIQUE_NAME_RNG.seed(f"reflex-unique-names:{seed}") + + +def unique_variable_name_state() -> tuple[Any, frozenset[str]]: + """Capture the generator state so a page's sequence can resume later. + + Returns: + An opaque state for :func:`restore_unique_variable_name_state`. + """ + return _UNIQUE_NAME_RNG.getstate(), frozenset(USED_VARIABLES) + + +def restore_unique_variable_name_state(state: tuple[Any, frozenset[str]]) -> None: + """Resume a page's name sequence captured by :func:`unique_variable_name_state`. + + Args: + state: The captured state. + """ + rng_state, used = state + _UNIQUE_NAME_RNG.setstate(rng_state) + USED_VARIABLES.clear() + USED_VARIABLES.update(used) + + def get_unique_variable_name() -> str: """Get a unique variable name. diff --git a/tests/units/compiler/test_disk_cache.py b/tests/units/compiler/test_disk_cache.py index 384d03a3c10..45ed243abd3 100644 --- a/tests/units/compiler/test_disk_cache.py +++ b/tests/units/compiler/test_disk_cache.py @@ -1132,3 +1132,39 @@ def test_update_manifest_for_misses_keeps_complete_imports(tmp_path, monkeypatch written = json.loads((web / disk_cache._MANIFEST_FILE).read_text()) assert disk_cache._deserialize_imports(written["all_imports"]) == complete_imports + + +def _page_loop_a() -> Component: + return rx.el.ul(rx.foreach(["a", "b"], lambda item: rx.el.li(item))) + + +def _page_loop_b() -> Component: + return rx.el.ul(rx.foreach(["c", "d"], lambda item: rx.el.li(item))) + + +def _page_loop_c() -> Component: + return rx.el.ul(rx.foreach(["e", "f"], lambda item: rx.el.li(item))) + + +def test_generated_names_do_not_depend_on_other_pages(): + """A page compiled alone gets the same generated names as in a full compile. + + ``rx.foreach`` draws its loop variable from the unique-name generator. If + that sequence were process-wide, an incremental compile of one page would + hand it the names the cached pages already use. + """ + full = _compile([ + _FakePage(route="/la", component=_page_loop_a), + _FakePage(route="/lb", component=_page_loop_b), + _FakePage(route="/lc", component=_page_loop_c), + ]) + alone = _compile([_FakePage(route="/lc", component=_page_loop_c)]) + assert ( + full.compiled_pages["/lc"].output_code + == alone.compiled_pages["/lc"].output_code + ) + # Different pages still draw different names. + la, lb = full.compiled_pages["/la"], full.compiled_pages["/lb"] + assert la.output_code is not None + assert lb.output_code is not None + assert la.output_code != lb.output_code diff --git a/tests/units/compiler/test_page_cache.py b/tests/units/compiler/test_page_cache.py index 090ca2c9e6f..264bc0f7f1c 100644 --- a/tests/units/compiler/test_page_cache.py +++ b/tests/units/compiler/test_page_cache.py @@ -4,6 +4,7 @@ import os import sys from pathlib import Path +from typing import Any, cast from reflex.compiler import page_cache @@ -621,7 +622,9 @@ def test_page_dependency_files_closes_over_dynamic_imports(tmp_path, monkeypatch memo_contributions={}, source_files={str(helper.resolve())}, ) - deps = page_cache.page_dependency_files(page_ctx, lambda: None, {}, tmp_path) + deps = page_cache.page_dependency_files( + cast(Any, page_ctx), lambda: None, {}, tmp_path + ) finally: _forget_modules("dyn_helper_for_closure", "dyn_leaf_for_closure") page_cache.clear_import_graph() diff --git a/tests/units/vars/test_base.py b/tests/units/vars/test_base.py index 6a332e0b116..439d66b20bc 100644 --- a/tests/units/vars/test_base.py +++ b/tests/units/vars/test_base.py @@ -99,3 +99,35 @@ def test_reset_unique_variable_names_is_deterministic(): # After a reset it reproduces the original sequence exactly. reset_unique_variable_names() assert [get_unique_variable_name() for _ in range(8)] == first + + +def test_seeded_unique_variable_names_are_per_page(): + """Each page seed yields its own reproducible sequence, resumable later. + + Incremental compiles regenerate a subset of pages; a page's names must not + depend on which other pages compiled before it (or the reused pages would + share "unique" names with the recompiled one). + """ + from reflex_base.vars.base import ( + get_unique_variable_name, + restore_unique_variable_name_state, + seed_unique_variable_names, + unique_variable_name_state, + ) + + seed_unique_variable_names("/a") + a_names = [get_unique_variable_name() for _ in range(4)] + a_state = unique_variable_name_state() + seed_unique_variable_names("/b") + b_names = [get_unique_variable_name() for _ in range(4)] + assert a_names != b_names + # Re-seeding a page reproduces its sequence regardless of what ran between. + seed_unique_variable_names("/a") + assert [get_unique_variable_name() for _ in range(4)] == a_names + a_continued = [get_unique_variable_name() for _ in range(2)] + # Restoring a captured state resumes the sequence from that point without + # repeating names already handed out. + restore_unique_variable_name_state(a_state) + resumed = [get_unique_variable_name() for _ in range(2)] + assert resumed == a_continued + assert not set(resumed) & set(a_names) From e7242f53186f58b1f3b2badaf4b88b48a5b377a0 Mon Sep 17 00:00:00 2001 From: Claude <noreply@anthropic.com> Date: Sat, 5 Sep 2026 04:12:51 +0000 Subject: [PATCH 22/56] compile cache: fix the daemon end to end on a real `reflex run` Found by driving the daemon with a browser rather than unit tests: - The forked child could not re-import the app: the warm parent's App stays bound to the RegistrationContext across the fork and the context refuses a second App. Release the slot in the first-party reset. - `reflex.lock/` is a directory of derived lockfiles rewritten on every install; treating it as a global input made every reload fall back to a full compile. Drop it from the global inputs; absent inputs are validated with is_file(). - Backend reload workers ran the cache path too. On Python 3.14 granian's workers come from a forkserver that the CLI's own initial compile started before `reflex run` set the skip flag, so environment set later never reaches them. Ownership is now decided from an on-disk marker written by `reflex run` (`compile_daemon.owns_compilation()`); the daemon and its children carry REFLEX_COMPILE_DAEMON and always own the build. - The file watcher runs a native thread; fork while it is alive trips Python's multi-threaded fork warning. Tear the watcher down before each compile, re-arm it after, and catch up on edits made in between from a stat snapshot of the recorded inputs. `_can_fork` also waits briefly for native threads to finish. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_015uV6WEQPemtsE9Nm6WFxyE --- .../src/reflex_base/environment.py | 5 + reflex/app.py | 7 + reflex/compiler/compiler.py | 6 +- reflex/compiler/disk_cache.py | 2 +- reflex/compiler/page_cache.py | 5 +- reflex/reflex.py | 3 + reflex/utils/compile_daemon.py | 196 ++++++++++++++---- reflex/utils/prerequisites.py | 8 +- tests/units/utils/test_compile_daemon.py | 52 ++++- 9 files changed, 235 insertions(+), 49 deletions(-) diff --git a/packages/reflex-base/src/reflex_base/environment.py b/packages/reflex-base/src/reflex_base/environment.py index 79fb493ec5a..5b3f8f81881 100644 --- a/packages/reflex-base/src/reflex_base/environment.py +++ b/packages/reflex-base/src/reflex_base/environment.py @@ -648,6 +648,11 @@ class EnvironmentVariables: # See reflex/compiler/disk_cache.py and reflex/compiler/page_cache.py. REFLEX_COMPILE_CACHE: EnvVar[bool] = env_var(False) + # Set only in the compile daemon's own process (and its forked children): + # marks the process that owns .web while a daemon runs, so backend reload + # workers of the same `reflex run` skip frontend compilation. + REFLEX_COMPILE_DAEMON: EnvVar[bool] = env_var(False, internal=True) + # Inherited by uvicorn/granian reload workers so the backend can distinguish # dev reload-capable worker boots from other backend starts. Never set in prod. REFLEX_DEV_BACKEND_RELOAD_ACTIVE: EnvVar[bool] = env_var(False, internal=True) diff --git a/reflex/app.py b/reflex/app.py index f5d269eb09e..4bdb73b8d5e 100644 --- a/reflex/app.py +++ b/reflex/app.py @@ -1565,6 +1565,13 @@ def _should_compile(self) -> bool: if environment.REFLEX_SKIP_COMPILE.get(): return False + # A running compile daemon owns .web; backend workers only evaluate + # pages to register state. + from reflex.utils import compile_daemon + + if not compile_daemon.owns_compilation(): + return False + nocompile = prerequisites.get_web_dir() / constants.NOCOMPILE_FILE # Check the nocompile file. diff --git a/reflex/compiler/compiler.py b/reflex/compiler/compiler.py index cfd78ee761b..ebca806c3db 100644 --- a/reflex/compiler/compiler.py +++ b/reflex/compiler/compiler.py @@ -1194,11 +1194,11 @@ def compile_app( should_compile = app._should_compile() backend_dir = prerequisites.get_backend_dir() if not dry_run and not should_compile and backend_dir.exists(): - if environment.REFLEX_COMPILE_CACHE.get(): + from reflex.utils import compile_daemon + + if compile_daemon.daemon_active(): # The compile daemon reacts to the same save; read its marker only # once it has finished rewriting .web for this change. - from reflex.utils import compile_daemon - compile_daemon.wait_for_compile() stateful_pages_marker = backend_dir / constants.Dirs.STATEFUL_PAGES if stateful_pages_marker.exists(): diff --git a/reflex/compiler/disk_cache.py b/reflex/compiler/disk_cache.py index 605560df469..f7ac8976c18 100644 --- a/reflex/compiler/disk_cache.py +++ b/reflex/compiler/disk_cache.py @@ -520,7 +520,7 @@ def globals_mismatch( if label != "reflex" and validator.changed(path) } stale.update( - path for path in manifest.get("globals_absent", ()) if Path(path).exists() + path for path in manifest.get("globals_absent", ()) if Path(path).is_file() ) if stale: return f"global input(s) changed: {format_path_list(stale, root)}" diff --git a/reflex/compiler/page_cache.py b/reflex/compiler/page_cache.py index 84686a1803b..dbf13be214c 100644 --- a/reflex/compiler/page_cache.py +++ b/reflex/compiler/page_cache.py @@ -36,9 +36,12 @@ #: Genuinely-global files: a change here can affect every page's output, so it #: bumps ``global_epoch`` rather than any single page's dependency set. +#: ``reflex.lock/`` is deliberately absent: its lockfiles are derived from +#: inputs tracked elsewhere (``rxconfig.py`` frontend packages, page imports, +#: the Reflex version) and are rewritten by every install, so treating them as +#: inputs made every reload fall back to a full compile. _GLOBAL_FILES = ( "rxconfig.py", - "reflex.lock", "uv.lock", "pyproject.toml", "requirements.txt", diff --git a/reflex/reflex.py b/reflex/reflex.py index 7fa0c2c4497..2dc733c8d1b 100644 --- a/reflex/reflex.py +++ b/reflex/reflex.py @@ -220,6 +220,9 @@ def _run_dev( compile_daemon.run_compile_daemon, exec.should_prerender_routes(), )) + # Backend workers are recognised through the daemon's on-disk marker + # (see ``compile_daemon.owns_compilation``): environment set here is + # not inherited by workers a forkserver started earlier. environment.REFLEX_SKIP_COMPILE.set(True) # Start the frontend and backend. diff --git a/reflex/utils/compile_daemon.py b/reflex/utils/compile_daemon.py index e9f98b4cea9..42d13b3d38e 100644 --- a/reflex/utils/compile_daemon.py +++ b/reflex/utils/compile_daemon.py @@ -16,7 +16,7 @@ import sys import threading import time -from collections.abc import Callable, Iterator +from collections.abc import Callable from dataclasses import dataclass, field from pathlib import Path @@ -58,13 +58,16 @@ def run_compile_daemon(prerender_routes: bool = False) -> None: # The daemon DOES compile; ensure the cache is on and the skip flag is off. env.pop(environment.REFLEX_SKIP_COMPILE.name, None) env[environment.REFLEX_COMPILE_CACHE.name] = "1" + env[environment.REFLEX_COMPILE_DAEMON.name] = "1" if prerender_routes: env["REFLEX_PRERENDER_ROUTES"] = "1" + mark_daemon_active() proc = subprocess.Popen( [sys.executable, "-m", "reflex.utils.compile_daemon"], env=env ) def _terminate() -> None: + clear_daemon_marker() if proc.poll() is None: proc.terminate() try: @@ -305,19 +308,19 @@ def _snapshot(paths: set[Path]) -> dict[Path, int]: return {p: m for p in paths if (m := _mtime_ns(p)) is not None} -def _poll_changes(state: _WatchState, alive: Callable[[], bool]) -> Iterator[set[Path]]: +def _poll_changes(state: _WatchState, alive: Callable[[], bool]) -> set[Path] | None: """Fallback change source when ``watchfiles`` is unavailable. Re-stats the known file set every ``_POLL_INTERVAL`` and re-walks the tree every ``_RESCAN_INTERVAL`` to discover added files. Args: - state: The live watch state (read on every rescan, so a refresh after a - compile takes effect without restarting the poller). + state: The live watch state. alive: Returns False when the watcher should stop. - Yields: - Each non-empty set of changed paths. + Returns: + The first non-empty set of changed paths, or None once ``alive`` is + False. """ paths = state.watch_paths() snapshot = _snapshot(paths) @@ -335,30 +338,30 @@ def _poll_changes(state: _WatchState, alive: Callable[[], bool]) -> Iterator[set } if changed: time.sleep(_DEBOUNCE_MS / 1000) # absorb the rest of a burst - yield changed - paths = state.watch_paths() - snapshot = _snapshot(paths) - last_rescan = time.monotonic() - else: - snapshot = current + return changed + snapshot = current + return None -def _iter_changes(state: _WatchState, alive: Callable[[], bool]) -> Iterator[set[Path]]: - """Yield sets of changed compile inputs as the OS reports them. +def _next_changes(state: _WatchState, alive: Callable[[], bool]) -> set[Path] | None: + """Block until compile inputs change; return the changed paths. Subscribes to filesystem notifications (``watchfiles``: inotify, FSEvents, - ReadDirectoryChangesW) over ``state.targets()`` and filters events with + ReadDirectoryChangesW) over ``state.targets()``, filtered by ``state.accepts``. Polling is only a fallback when ``watchfiles`` is not - importable. The watcher is re-created when the target set changes (a - compile recorded a new external content directory). + importable. + + The watcher is torn down before returning: it runs a native thread, and + the compile that follows forks the (otherwise single-threaded) daemon. + Edits made while the watcher is down are caught up by + :func:`_missed_changes` after the compile. Args: - state: The live watch state; ``targets``/``accepts`` are re-read after - every yielded change set. + state: The live watch state. alive: Returns False when the watcher should stop. - Yields: - Each non-empty, accepted set of changed paths. + Returns: + A non-empty set of changed paths, or None once ``alive`` is False. """ try: import watchfiles @@ -366,30 +369,55 @@ def _iter_changes(state: _WatchState, alive: Callable[[], bool]) -> Iterator[set console.warn( "watchfiles is not installed; the compile daemon falls back to polling." ) - yield from _poll_changes(state, alive) - return + return _poll_changes(state, alive) def watch_filter(_change: object, raw_path: str) -> bool: return state.accepts(Path(raw_path)) - while alive(): - targets = state.targets() - for events in watchfiles.watch( - *targets, - watch_filter=watch_filter, - debounce=_DEBOUNCE_MS, - step=25, - rust_timeout=_WATCH_TIMEOUT_MS, - yield_on_timeout=True, - raise_interrupt=False, - ): + events = watchfiles.watch( + *state.targets(), + watch_filter=watch_filter, + debounce=_DEBOUNCE_MS, + step=25, + rust_timeout=_WATCH_TIMEOUT_MS, + yield_on_timeout=True, + raise_interrupt=False, + ) + try: + for batch in events: if not alive(): - return - changed = {Path(raw_path) for _change, raw_path in events} + return None + changed = {Path(raw_path) for _change, raw_path in batch} if changed: - yield changed - if state.targets() != targets: - break # re-create the watcher over the new target set + return changed + finally: + events.close() # stops the native watcher thread before any fork + return None + + +def _dependency_snapshot(state: _WatchState) -> dict[Path, int]: + """Stat the recorded compile inputs (cheap: no tree walk). + + Args: + state: The watch state whose known dependencies to stat. + + Returns: + ``{path: mtime_ns}`` for every readable recorded input. + """ + return _snapshot(state.known | state.globals_) + + +def _missed_changes(before: dict[Path, int], after: dict[Path, int]) -> set[Path]: + """Inputs that changed between two dependency snapshots. + + Args: + before: Snapshot taken before the watcher was torn down. + after: Snapshot taken once it is about to be re-armed. + + Returns: + The paths whose mtime differs (or that appeared/disappeared). + """ + return {p for p in before.keys() | after.keys() if before.get(p) != after.get(p)} #: ``__file__`` -> whether it is under the reload roots; the "" key holds the @@ -546,12 +574,63 @@ def _reset_model_metadata() -> None: _LOCK_WAIT_TIMEOUT = _COMPILE_TIMEOUT + 10 +#: Marker (under the backend dir) holding the pid of the ``reflex run`` that +#: owns a compile daemon. Backend workers skip frontend compilation while it +#: names a live process. A file, not an environment variable: workers may be +#: forked from a ``forkserver`` whose environment predates ``reflex run`` +#: deciding to start the daemon. +_DAEMON_MARKER_FILE = ".compile_daemon" + + def _lock_path() -> Path: from reflex.utils import prerequisites return prerequisites.get_backend_dir() / _COMPILE_LOCK_FILE +def _daemon_marker_path() -> Path: + from reflex.utils import prerequisites + + return prerequisites.get_backend_dir() / _DAEMON_MARKER_FILE + + +def mark_daemon_active() -> None: + """Record that this ``reflex run`` process owns the frontend compile.""" + marker = _daemon_marker_path() + marker.parent.mkdir(parents=True, exist_ok=True) + marker.write_text(str(os.getpid())) + + +def clear_daemon_marker() -> None: + """Remove the daemon-ownership marker (on ``reflex run`` exit).""" + _daemon_marker_path().unlink(missing_ok=True) + + +def daemon_active() -> bool: + """Whether a live ``reflex run`` owns frontend compilation via the daemon. + + Returns: + True if the marker names a process that is still alive. + """ + marker = _daemon_marker_path() + return marker.exists() and _lock_holder_alive(marker) + + +def owns_compilation() -> bool: + """Whether this process should produce ``.web`` itself. + + False for a backend reload worker while a compile daemon owns the build; + the daemon (and its forked compile children) carry + ``REFLEX_COMPILE_DAEMON`` and always own it. Decided from the on-disk + marker rather than inherited environment, which a ``forkserver`` started + before ``reflex run`` chose the daemon would not carry. + + Returns: + True if this process compiles the frontend. + """ + return environment.REFLEX_COMPILE_DAEMON.get() or not daemon_active() + + @contextlib.contextmanager def _compile_lock(root: Path): """Hold the compile-in-progress lock while a hot-reload compile runs. @@ -700,6 +779,22 @@ def _quiesce_parent() -> None: telemetry.shutdown_executor() +def _os_thread_count() -> int | None: + """Count this process's OS threads, including native ones Python cannot see. + + Returns: + The thread count from ``/proc`` (Linux), else None. + """ + try: + with Path("/proc/self/status").open() as status: + for line in status: + if line.startswith("Threads:"): + return int(line.split()[1]) + except (OSError, ValueError): + return None + return None + + def _can_fork() -> bool: """Whether forking is safe right now (POSIX and the process is single-threaded). @@ -715,6 +810,17 @@ def _can_fork() -> bool: return False _quiesce_parent() if threading.active_count() == 1: + # A native thread that just finished (the file watcher's, the progress + # bar's) may still be winding down; give it a moment so the fork is + # really single-threaded. + deadline = time.monotonic() + 0.25 + while (count := _os_thread_count()) is not None and count > 1: + if time.monotonic() > deadline: + console.debug( + f"Compile daemon: forking with {count} OS threads still alive." + ) + break + time.sleep(0.005) return True others = [ t.name for t in threading.enumerate() if t is not threading.current_thread() @@ -809,7 +915,12 @@ def alive() -> bool: # Never outlive reflex-run: if our parent died we were reparented. return os.getppid() == parent_pid - for changed in _iter_changes(state, alive): + pending: set[Path] = set() + while alive(): + changed = pending or _next_changes(state, alive) + pending = set() + if not changed: + break from reflex.compiler.disk_cache import format_path_list console.info( @@ -825,14 +936,17 @@ def alive() -> bool: sys.executable, [sys.executable, "-m", "reflex.utils.compile_daemon"] ) + before = _dependency_snapshot(state) state.roots = roots = _reload_roots() with _compile_lock(root): ok = _compile_once(roots, prerender_routes, changed) if not ok: console.error("Compile failed; keeping the last good build.") # Refresh what is watched from the new manifest so a newly-referenced - # content file becomes watched. + # content file becomes watched, and catch up on edits made while the + # watcher was down for the compile. state = _WatchState.build(roots, root) + pending = _missed_changes(before, _dependency_snapshot(state)) def main(argv: list[str] | None = None) -> int: diff --git a/reflex/utils/prerequisites.py b/reflex/utils/prerequisites.py index df243182ec3..db2e0274940 100644 --- a/reflex/utils/prerequisites.py +++ b/reflex/utils/prerequisites.py @@ -194,7 +194,13 @@ def get_app(reload: bool = False) -> ModuleType: module = config.module sys.path.insert(0, getcwd()) # noqa: PTH109 - if environment.REFLEX_COMPILE_CACHE.get(): + from reflex.utils import compile_daemon + + if ( + environment.REFLEX_COMPILE_CACHE.get() + and not environment.REFLEX_SKIP_COMPILE.get() + and compile_daemon.owns_compilation() + ): from reflex.compiler import page_cache page_cache.enable_read_tracking() diff --git a/tests/units/utils/test_compile_daemon.py b/tests/units/utils/test_compile_daemon.py index 523a7834322..9f9d7bb25fa 100644 --- a/tests/units/utils/test_compile_daemon.py +++ b/tests/units/utils/test_compile_daemon.py @@ -145,9 +145,9 @@ def test_iter_changes_reports_an_edit(tmp_path): got: list[set] = [] def run() -> None: - for changed in compile_daemon._iter_changes(state, lambda: not got): + changed = compile_daemon._next_changes(state, lambda: not got) + if changed: got.append(changed) - return thread = threading.Thread(target=run, daemon=True) thread.start() @@ -158,6 +158,24 @@ def run() -> None: thread.join(5) assert got assert page.resolve() in {p.resolve() for p in got[0]} + # The native watcher thread is gone, so the daemon may fork again. + assert not any("watchfiles" in t.name.lower() for t in threading.enumerate()) + + +def test_missed_changes_catches_edits_during_compile(tmp_path): + a, b = tmp_path / "a.py", tmp_path / "b.py" + a.write_text("1") + b.write_text("1") + state = compile_daemon._WatchState([tmp_path], tmp_path, known={a, b}) + before = compile_daemon._dependency_snapshot(state) + bumped = before[b] + 1_000_000_000 + os.utime(b, ns=(bumped, bumped)) + c = tmp_path / "c.py" + c.write_text("1") + state.known.add(c) + after = compile_daemon._dependency_snapshot(state) + assert compile_daemon._missed_changes(before, after) == {b, c} + assert compile_daemon._missed_changes(after, after) == set() def test_wait_for_compile_respects_lock(tmp_path, monkeypatch): @@ -459,3 +477,33 @@ def test_quiesce_parent_stops_telemetry_worker(): telemetry._get_telemetry_executor().submit(lambda: None).result() telemetry.shutdown_executor() assert telemetry._executor is None + + +def test_owns_compilation_follows_the_daemon_marker(tmp_path, monkeypatch): + """Backend workers defer to a live daemon; the daemon itself always owns.""" + from reflex_base.environment import environment + + monkeypatch.setattr("reflex.utils.prerequisites.get_backend_dir", lambda: tmp_path) + monkeypatch.delenv(environment.REFLEX_COMPILE_DAEMON.name, raising=False) + # No marker: a plain process compiles as usual. + assert compile_daemon.daemon_active() is False + assert compile_daemon.owns_compilation() is True + # A live reflex-run marker: workers stand down. + compile_daemon.mark_daemon_active() + assert compile_daemon.daemon_active() is True + assert compile_daemon.owns_compilation() is False + # ...but the daemon's own processes still own the build. + monkeypatch.setenv(environment.REFLEX_COMPILE_DAEMON.name, "1") + assert compile_daemon.owns_compilation() is True + monkeypatch.delenv(environment.REFLEX_COMPILE_DAEMON.name) + # A marker left by a dead process is ignored. + import subprocess + import sys + + proc = subprocess.Popen([sys.executable, "-c", "pass"]) + proc.wait() + compile_daemon._daemon_marker_path().write_text(str(proc.pid)) + assert compile_daemon.daemon_active() is False + assert compile_daemon.owns_compilation() is True + compile_daemon.clear_daemon_marker() + assert not compile_daemon._daemon_marker_path().exists() From 612118fbb1167301a5b478ac72d7afbeb8f88edd Mon Sep 17 00:00:00 2001 From: Claude <noreply@anthropic.com> Date: Sat, 5 Sep 2026 04:15:49 +0000 Subject: [PATCH 23/56] compile cache: probe lock/marker holder liveness without os.kill on Windows os.kill(pid, 0) terminates the target on Windows; use OpenProcess there and map ProcessLookupError/PermissionError explicitly elsewhere. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_015uV6WEQPemtsE9Nm6WFxyE --- reflex/utils/compile_daemon.py | 32 ++++++++++++++++++++++-- tests/units/utils/test_compile_daemon.py | 10 ++++++++ 2 files changed, 40 insertions(+), 2 deletions(-) diff --git a/reflex/utils/compile_daemon.py b/reflex/utils/compile_daemon.py index 42d13b3d38e..c994efe8443 100644 --- a/reflex/utils/compile_daemon.py +++ b/reflex/utils/compile_daemon.py @@ -655,13 +655,41 @@ def _compile_lock(root: Path): lock.unlink(missing_ok=True) +def _pid_alive(pid: int) -> bool: + """Whether a process with ``pid`` exists. + + Args: + pid: The process id. + + Returns: + True if the process is alive (or exists but is not ours to signal). + """ + if os.name == "nt": + import ctypes + + kernel32 = ctypes.windll.kernel32 # type: ignore[attr-defined] + # PROCESS_QUERY_LIMITED_INFORMATION; os.kill(pid, 0) would *terminate* + # the process on Windows. + handle = kernel32.OpenProcess(0x1000, False, pid) + if not handle: + return False + kernel32.CloseHandle(handle) + return True + try: + os.kill(pid, 0) + except ProcessLookupError: + return False + except PermissionError: + return True + return True + + def _lock_holder_alive(lock: Path) -> bool: try: pid = int(lock.read_text().strip()) - os.kill(pid, 0) except (OSError, ValueError): return False - return True + return _pid_alive(pid) def wait_for_compile(timeout: float = _LOCK_WAIT_TIMEOUT) -> None: diff --git a/tests/units/utils/test_compile_daemon.py b/tests/units/utils/test_compile_daemon.py index 9f9d7bb25fa..2af0d27da75 100644 --- a/tests/units/utils/test_compile_daemon.py +++ b/tests/units/utils/test_compile_daemon.py @@ -507,3 +507,13 @@ def test_owns_compilation_follows_the_daemon_marker(tmp_path, monkeypatch): assert compile_daemon.owns_compilation() is True compile_daemon.clear_daemon_marker() assert not compile_daemon._daemon_marker_path().exists() + + +def test_pid_alive_distinguishes_live_and_exited_processes(): + import subprocess + import sys + + assert compile_daemon._pid_alive(os.getpid()) is True + proc = subprocess.Popen([sys.executable, "-c", "pass"]) + proc.wait() + assert compile_daemon._pid_alive(proc.pid) is False From 8c11a8e04e3f38e2809279c111a074f2528d304d Mon Sep 17 00:00:00 2001 From: Farhan <www.mfarvirus@gmail.com> Date: Sat, 5 Sep 2026 21:17:27 +0500 Subject: [PATCH 24/56] compile cache: gate on import-time state modules and surface cache failures States defined at import time have no per-page context slice, so a change to their definitions never invalidated anything and the shared frontend contexts went stale. Hash their source files (and their import closure) as global inputs, and serialize the manifest with reflex_base's json_dumps so state defaults needing Reflex serialization no longer abort the manifest write. Manifest write/refresh failures now warn instead of logging at debug level, and an incomplete compile says which page blocked the save. Key the module-file cache on (class, roots) so a different project root can't read another root's answer, and memoize first-party module resolution. In the daemon, snapshot every watchable input at the point the watcher stops and reconcile that checkpoint once it subscribes again, so edits made during a compile are not lost. When native threads are still alive past the deadline, fall back to a cold subprocess instead of forking anyway. Claude-Session: https://claude.ai/code/session_01VZ2gyiCc97WDZSNw8ozZxB --- news/6688.feature.md | 2 +- news/6688.fix.md | 1 + reflex/compiler/disk_cache.py | 49 ++++++++++-- reflex/compiler/page_cache.py | 31 ++++++-- reflex/utils/compile_daemon.py | 29 ++++++-- tests/units/compiler/test_disk_cache.py | 95 +++++++++++++++++++++++- tests/units/compiler/test_page_cache.py | 43 +++++++++++ tests/units/utils/test_compile_daemon.py | 45 +++++++++++ 8 files changed, 273 insertions(+), 22 deletions(-) create mode 100644 news/6688.fix.md diff --git a/news/6688.feature.md b/news/6688.feature.md index a42b4e87ab4..782789178b6 100644 --- a/news/6688.feature.md +++ b/news/6688.feature.md @@ -1 +1 @@ -Added an experimental disk-persisted incremental compile cache, enabled by the `REFLEX_COMPILE_CACHE` environment variable. When on, a fresh compile reuses the previous build already on disk in `.web` and recompiles only the pages whose source changed, tracked via a per-page dependency graph (Python import closure, files read during page evaluation, component modules, and referenced state). Dependencies are validated stat-first, so unchanged files are never re-read, and the contexts file is rebuilt from the manifest without re-evaluating unchanged pages. App-wide inputs (Reflex version, config/lockfiles, and the app entrypoint's config modules such as theme/app-wraps/stylesheets) gate the whole cache, falling back to a full compile when they change. `reflex run` dev additionally gains a warm fork-per-compile daemon that watches the project with `watchfiles` (polling only as a fallback), so hot reloads skip the cold reimport and rebuild only what changed; the backend reload worker waits for that compile before registering state. Off by default — the compile path is unchanged when the flag is unset. +Added an experimental disk-persisted incremental compile cache, enabled by the `REFLEX_COMPILE_CACHE` environment variable. When on, a fresh compile reuses the previous build already on disk in `.web` and recompiles only the pages whose source changed, tracked via a per-page dependency graph (Python import closure, files read during page evaluation, component modules, and referenced state). Dependencies are validated stat-first, so unchanged files are never re-read, and the contexts file is rebuilt from the manifest without re-evaluating unchanged pages. App-wide inputs (Reflex version, config/lockfiles, the app entrypoint's config modules such as theme/app-wraps/stylesheets, and module-level state definitions and their Python dependencies) gate the whole cache, falling back to a full compile when they change. `reflex run` dev additionally gains a warm fork-per-compile daemon that watches the project with `watchfiles` (polling only as a fallback), so hot reloads skip the cold reimport and rebuild only what changed; the backend reload worker waits for that compile before registering state. The daemon checks for source and asset changes made during compilation and falls back to a cold subprocess when native threads remain alive. Off by default — the compile path is unchanged when the flag is unset. diff --git a/news/6688.fix.md b/news/6688.fix.md new file mode 100644 index 00000000000..3bf4f37579d --- /dev/null +++ b/news/6688.fix.md @@ -0,0 +1 @@ +Fixed repeated full recompilation when the incremental compile cache encounters state defaults requiring Reflex serialization, such as Pydantic models or dates. Manifest write failures now produce visible warnings instead of silently disabling cache reuse. diff --git a/reflex/compiler/disk_cache.py b/reflex/compiler/disk_cache.py index f7ac8976c18..1286feddc8c 100644 --- a/reflex/compiler/disk_cache.py +++ b/reflex/compiler/disk_cache.py @@ -41,7 +41,7 @@ from reflex.app import App #: Bump when the manifest layout changes (old manifests are then ignored). -_SCHEMA = 9 +_SCHEMA = 10 #: Manifest filename under the web directory. _MANIFEST_FILE = "reflex_compile_cache.json" @@ -382,7 +382,7 @@ def _write(manifest: dict[str, Any]) -> None: manifest["import_names"] = page_cache.export_import_names_cache() path = _manifest_path() path.parent.mkdir(parents=True, exist_ok=True) - path.write_text(json.dumps(manifest), encoding="utf-8") + path.write_text(json_dumps(manifest), encoding="utf-8") def write_manifest( @@ -422,6 +422,10 @@ def write_manifest( or page_ctx.output_code is None or page_ctx.output_path is None ): + console.warn( + f"Compile cache: cannot save manifest because page {page.route!r} " + "has incomplete output. The next reload will need a full compile." + ) return # incomplete compile -> do not write a partial manifest defined_states = compile_ctx.stateful_routes.get(page.route) pages_data[page.route] = _manifest_page_entry( @@ -445,7 +449,11 @@ def write_manifest( globals_: dict[str, str] = {"reflex": page_cache._reflex_version()} absent: list[str] = [] - for label, path in page_cache.global_input_paths(root, pages=pages).items(): + global_paths = page_cache.global_input_paths(root, pages=pages) + global_paths.update({ + f"state:{path}": path for path in _imported_state_files(compile_ctx, root) + }) + for label, path in global_paths.items(): entry = hasher(path) if entry is None: absent.append(path) @@ -467,7 +475,38 @@ def write_manifest( } _write(manifest) except Exception as exc: # best-effort: never break the build - console.debug(f"disk compile cache: manifest write skipped ({exc!r})") + console.warn( + f"Compile cache: could not save manifest ({exc!r}). " + "The next reload may need a full compile." + ) + + +def _imported_state_files(compile_ctx: CompileContext, root: Path | None) -> set[str]: + """Find source dependencies of states not owned by page evaluation. + + These states have no per-page context slice. Changing their definitions + requires a full compile to refresh the shared frontend contexts. + + Args: + compile_ctx: The completed full compile. + root: Project root for dependency discovery. + + Returns: + First-party source files affecting import-time state definitions. + """ + from reflex_base.registry import RegistrationContext + + page_states = { + name for names in compile_ctx.stateful_routes.values() for name in names + } + roots = page_cache.first_party_roots(root) + sources = { + str(path) + for name, cls in RegistrationContext.ensure_context().base_states.items() + if name not in page_states + and (path := page_cache._class_module_file(cls, roots)) is not None + } + return page_cache._walk_import_closure(page_cache.build_import_graph(root), sources) def globals_mismatch( @@ -1281,4 +1320,4 @@ def _update_manifest_for_misses( if dirty: _write(manifest) except Exception as exc: # best-effort - console.debug(f"disk compile cache: manifest refresh skipped ({exc!r})") + console.warn(f"Compile cache: could not refresh manifest ({exc!r}).") diff --git a/reflex/compiler/page_cache.py b/reflex/compiler/page_cache.py index dbf13be214c..25d8074a5f7 100644 --- a/reflex/compiler/page_cache.py +++ b/reflex/compiler/page_cache.py @@ -18,6 +18,7 @@ import sys from collections.abc import Callable, Iterable, Mapping, Sequence from contextvars import ContextVar +from functools import lru_cache from importlib import metadata from pathlib import Path from typing import TYPE_CHECKING @@ -563,11 +564,11 @@ def _module_file(component: object) -> Path | None: return Path(file) if file else None -#: Component/state class -> its resolved first-party module file (or None). +#: Component/state class and roots -> its first-party module file (or None). #: A rendered tree has tens of thousands of nodes but few distinct classes, #: and a class's module never moves; resolving per node dominated the #: dependency walk. Warmed in the daemon parent so forked children inherit it. -_class_file_cache: dict[type, Path | None] = {} +_class_file_cache: dict[tuple[type, tuple[Path, ...]], Path | None] = {} def _class_module_file(cls: type, roots: tuple[Path, ...]) -> Path | None: @@ -580,8 +581,9 @@ def _class_module_file(cls: type, roots: tuple[Path, ...]) -> Path | None: Returns: The resolved module file under one of ``roots``, else None. """ + key = (cls, roots) try: - return _class_file_cache[cls] + return _class_file_cache[key] except KeyError: pass f = _module_file(cls) @@ -590,7 +592,7 @@ def _class_module_file(cls: type, roots: tuple[Path, ...]) -> Path | None: rf = f.resolve() if _under_any(rf, roots): resolved = rf - _class_file_cache[cls] = resolved + _class_file_cache[key] = resolved return resolved @@ -608,6 +610,7 @@ def warm_module_file_cache(root: Path | None = None) -> None: for base in (Component, BaseState): for cls in _subclasses(base): _class_module_file(cls, roots) + _loaded_first_party_modules(roots[0]) def component_module_files( @@ -669,12 +672,26 @@ def _loaded_first_party_modules(root: Path) -> dict[str, str]: file = getattr(mod, "__file__", None) if not file: continue - rf = Path(file).resolve() - if _under_any(rf, roots): - file_to_mod[str(rf)] = name + if (resolved := _first_party_module_file(file, roots)) is not None: + file_to_mod[resolved] = name return file_to_mod +@lru_cache(maxsize=8192) +def _first_party_module_file(file: str, roots: tuple[Path, ...]) -> str | None: + """Resolve a stable module filename once per set of reload roots. + + Args: + file: The module's source filename. + roots: Resolved first-party roots. + + Returns: + The resolved filename if it belongs to a first-party root, else None. + """ + resolved = Path(file).resolve() + return str(resolved) if _under_any(resolved, roots) else None + + def _import_from_targets(node: object, modname: str) -> list[str]: """Resolve a ``from ... import ...`` node to candidate module names. diff --git a/reflex/utils/compile_daemon.py b/reflex/utils/compile_daemon.py index c994efe8443..a97e1f82e1e 100644 --- a/reflex/utils/compile_daemon.py +++ b/reflex/utils/compile_daemon.py @@ -221,6 +221,8 @@ class _WatchState: #: JSON/CSV/etc. file a page reads triggers a rebuild too. known: set[Path] = field(default_factory=set) globals_: set[Path] = field(default_factory=set) + #: Last post-compile snapshot, reconciled once the watcher subscribes again. + checkpoint: dict[Path, int] | None = None @classmethod def build(cls, roots: list[Path], root: Path) -> _WatchState: @@ -282,7 +284,7 @@ def accepts(self, path: Path) -> bool: return path.suffix in _WATCH_SUFFIXES and _under_roots(path, self.roots) def watch_paths(self) -> set[Path]: - """Fallback polling only: the full file set to snapshot each tick. + """Collect inputs for polling and watch-subscription catch-up snapshots. Returns: The set of paths to stat. @@ -324,6 +326,10 @@ def _poll_changes(state: _WatchState, alive: Callable[[], bool]) -> set[Path] | """ paths = state.watch_paths() snapshot = _snapshot(paths) + if state.checkpoint is not None and ( + changed := _missed_changes(state.checkpoint, snapshot) + ): + return changed last_rescan = time.monotonic() while alive(): time.sleep(_POLL_INTERVAL) @@ -383,11 +389,15 @@ def watch_filter(_change: object, raw_path: str) -> bool: yield_on_timeout=True, raise_interrupt=False, ) + checkpoint = state.checkpoint try: for batch in events: if not alive(): return None changed = {Path(raw_path) for _change, raw_path in batch} + if checkpoint is not None: + changed.update(_missed_changes(checkpoint, _dependency_snapshot(state))) + checkpoint = None if changed: return changed finally: @@ -396,15 +406,15 @@ def watch_filter(_change: object, raw_path: str) -> bool: def _dependency_snapshot(state: _WatchState) -> dict[Path, int]: - """Stat the recorded compile inputs (cheap: no tree walk). + """Snapshot all watchable inputs while the OS watcher is stopped. Args: state: The watch state whose known dependencies to stat. Returns: - ``{path: mtime_ns}`` for every readable recorded input. + ``{path: mtime_ns}`` for every readable watched input. """ - return _snapshot(state.known | state.globals_) + return _snapshot(state.watch_paths()) def _missed_changes(before: dict[Path, int], after: dict[Path, int]) -> set[Path]: @@ -844,10 +854,11 @@ def _can_fork() -> bool: deadline = time.monotonic() + 0.25 while (count := _os_thread_count()) is not None and count > 1: if time.monotonic() > deadline: - console.debug( - f"Compile daemon: forking with {count} OS threads still alive." + console.warn( + f"Compile daemon: {count} OS threads still alive; " + "falling back to a cold subprocess." ) - break + return False time.sleep(0.005) return True others = [ @@ -937,6 +948,7 @@ def _serve() -> None: _prepare_fork_parent(roots) state = _WatchState.build(roots, root) + state.checkpoint = _dependency_snapshot(state) console.info("Compile daemon ready (warm); watching for changes.") def alive() -> bool: @@ -974,7 +986,8 @@ def alive() -> bool: # content file becomes watched, and catch up on edits made while the # watcher was down for the compile. state = _WatchState.build(roots, root) - pending = _missed_changes(before, _dependency_snapshot(state)) + state.checkpoint = _dependency_snapshot(state) + pending = _missed_changes(before, state.checkpoint) def main(argv: list[str] | None = None) -> int: diff --git a/tests/units/compiler/test_disk_cache.py b/tests/units/compiler/test_disk_cache.py index 45ed243abd3..9e3bb7d7a23 100644 --- a/tests/units/compiler/test_disk_cache.py +++ b/tests/units/compiler/test_disk_cache.py @@ -1,14 +1,16 @@ """Tests for the experimental disk-persisted incremental compile cache.""" import dataclasses +import datetime import itertools import json import sys from collections.abc import Callable, Sequence from pathlib import Path -from types import SimpleNamespace +from types import ModuleType, SimpleNamespace from typing import Any, cast +import pytest from reflex_base.components.component import Component from reflex_base.plugins import CompileContext, CompilerHooks from reflex_base.utils.imports import ImportVar @@ -99,6 +101,49 @@ def _unregister_state(cls: type[rx.State]) -> None: all_base_state_classes.pop(full_name, None) +@pytest.mark.parametrize("edited_module", ["state", "helper"]) +def test_imported_state_edit_invalidates_contexts(tmp_path, monkeypatch, edited_module): + """Module-level state edits need a full compile to refresh shared contexts.""" + _use_tmp_web_dir(tmp_path, monkeypatch) + source = tmp_path / "state_view.py" + helper_source = tmp_path / "state_helper.py" + helper_source.write_text("DEFAULT = 'old'\n") + helper = ModuleType("cache_review_state_helper") + helper.__file__ = str(helper_source) + helper.__dict__["DEFAULT"] = "old" + monkeypatch.setitem(sys.modules, helper.__name__, helper) + code = ( + "import reflex as rx\n" + "from cache_review_state_helper import DEFAULT\n" + "class ImportedState(rx.State):\n" + " value: str = DEFAULT\n" + "def index():\n" + " return rx.el.div(ImportedState.value)\n" + ) + source.write_text(code) + module = ModuleType("cache_review_state_view") + module.__file__ = str(source) + monkeypatch.setitem(sys.modules, module.__name__, module) + exec(compile(code, str(source), "exec"), module.__dict__) + try: + app = rx.App() + app.add_page(module.index, route="/state") + pages = list(app._unevaluated_pages.values()) + ctx = _compile(pages, app) + assert not ctx.stateful_routes + disk_cache.write_manifest(ctx, pages, ctx.all_imports, root=tmp_path) + if edited_module == "state": + source.write_text(code.replace("= DEFAULT", "= 'new default'")) + else: + helper_source.write_text("DEFAULT = 'new default'\n") + _stub_externals(app, monkeypatch) + assert not disk_cache.try_incremental_rebuild( + app, compiler_plugins=[], prerender_routes=False, root=tmp_path + ) + finally: + _unregister_state(module.ImportedState) + + def test_imports_round_trip(): imports = { "react": [ImportVar("useEffect"), ImportVar("Fragment", is_default=False)], @@ -108,6 +153,54 @@ def test_imports_round_trip(): assert restored == imports +@pytest.mark.parametrize("kind", ["date", "pydantic"]) +def test_manifest_serializes_state_defaults_like_contexts(tmp_path, monkeypatch, kind): + """Persist state defaults that require Reflex's frontend serializer.""" + from reflex_base.utils.format import json_dumps + + _use_tmp_web_dir(tmp_path, monkeypatch) + value = ( + datetime.date(2026, 9, 5) + if kind == "date" + else pytest.importorskip("pydantic").create_model( + "AppSettings", theme=(str, "light") + )() + ) + state_slice = { + "initial_state": {"state": {"value": value}}, + "client_storage": {}, + } + manifest = _manifest({"/date": {"state_slice": state_slice}}) + disk_cache._write(manifest) + loaded = disk_cache.load_manifest() + assert loaded is not None + assert loaded["pages"]["/date"]["state_slice"] == json.loads( + json_dumps(state_slice) + ) + + +def test_manifest_write_failure_is_visible(tmp_path, monkeypatch): + """A failed cache write must explain why subsequent reloads compile everything.""" + _use_tmp_web_dir(tmp_path, monkeypatch) + pages = [_FakePage(route="/a", component=_page_a)] + ctx = _compile(pages) + warnings = [] + + def fail_write(manifest): + """Simulate a failure writing the cache manifest. + + Raises: + OSError: The cache cannot be persisted. + """ + msg = "disk full" + raise OSError(msg) + + monkeypatch.setattr(disk_cache, "_write", fail_write) + monkeypatch.setattr(disk_cache.console, "warn", warnings.append) + disk_cache.write_manifest(ctx, pages, ctx.all_imports, root=tmp_path) + assert any("disk full" in warning for warning in warnings) + + def test_serialize_imports_collapses_duplicates(): """The manifest only needs the unique import set, in first-seen order. diff --git a/tests/units/compiler/test_page_cache.py b/tests/units/compiler/test_page_cache.py index 264bc0f7f1c..af277109b7c 100644 --- a/tests/units/compiler/test_page_cache.py +++ b/tests/units/compiler/test_page_cache.py @@ -4,11 +4,54 @@ import os import sys from pathlib import Path +from types import ModuleType from typing import Any, cast from reflex.compiler import page_cache +def test_class_module_file_respects_each_project_root(tmp_path, monkeypatch): + """A class excluded in one project can be a dependency in another.""" + module = ModuleType("review_class_roots") + module.__file__ = str(tmp_path / "component.py") + monkeypatch.setitem(sys.modules, module.__name__, module) + cls = type("RootScoped", (), {"__module__": module.__name__}) + assert page_cache._class_module_file(cls, (tmp_path / "other",)) is None + assert page_cache._class_module_file(cls, (tmp_path,)) == Path(module.__file__) + + +def test_loaded_modules_reuse_resolution_and_discover_new_modules( + tmp_path, monkeypatch +): + """Cache stable module paths without missing imports or changes to roots.""" + source = tmp_path / "cached_module.py" + source.write_text("VALUE = 1\n") + module = ModuleType("cached_review_module") + module.__file__ = str(source) + monkeypatch.setitem(sys.modules, module.__name__, module) + original = Path.resolve + resolutions = [] + + def resolve(path, *args, **kwargs): + """Count resolutions of the test module's source file. + + Returns: + The resolved path. + """ + if path == source: + resolutions.append(path) + return original(path, *args, **kwargs) + + monkeypatch.setattr(Path, "resolve", resolve) + assert str(source) in page_cache._loaded_first_party_modules(tmp_path) + added = ModuleType("new_review_module") + added.__file__ = str(tmp_path / "added.py") + monkeypatch.setitem(sys.modules, added.__name__, added) + assert added.__file__ in page_cache._loaded_first_party_modules(tmp_path) + assert len(resolutions) == 1 + assert str(source) not in page_cache._loaded_first_party_modules(tmp_path / "other") + + def _prepare_runtime_module(root, monkeypatch, module_name, *, import_root=None): """Create a temporary module and make it importable. diff --git a/tests/units/utils/test_compile_daemon.py b/tests/units/utils/test_compile_daemon.py index 2af0d27da75..2e4f332d330 100644 --- a/tests/units/utils/test_compile_daemon.py +++ b/tests/units/utils/test_compile_daemon.py @@ -16,6 +16,51 @@ class _SurvivingModuleState(rx.State): b: int = 0 +def test_can_fork_rejects_remaining_native_threads(monkeypatch): + """Native threads must force the subprocess fallback after the grace period.""" + monkeypatch.setattr(compile_daemon, "_quiesce_parent", lambda: None) + monkeypatch.setattr(compile_daemon.threading, "active_count", lambda: 1) + monkeypatch.setattr(compile_daemon, "_os_thread_count", lambda: 8) + ticks = iter([0.0, 1.0]) + monkeypatch.setattr(compile_daemon.time, "monotonic", lambda: next(ticks)) + assert not compile_daemon._can_fork() + + +def test_missed_changes_includes_new_sources_and_assets(tmp_path): + """Catch sources and assets created while the OS watcher is stopped.""" + state = compile_daemon._WatchState(roots=[tmp_path], root=tmp_path) + before = compile_daemon._dependency_snapshot(state) + source = tmp_path / "new_page.py" + source.write_text("VALUE = 1\n") + assets = tmp_path / "assets" + assets.mkdir() + asset = assets / "image.txt" + asset.write_text("new asset") + after = compile_daemon._dependency_snapshot(state) + assert {source, asset} <= compile_daemon._missed_changes(before, after) + + +def test_next_changes_catches_edit_before_watcher_starts(tmp_path, monkeypatch): + """Reconcile the previous snapshot after the watcher has subscribed.""" + import watchfiles + + state = compile_daemon._WatchState(roots=[tmp_path], root=tmp_path) + state.checkpoint = compile_daemon._dependency_snapshot(state) + source = tmp_path / "new_page.py" + source.write_text("VALUE = 1\n") + + def watch(*args, **kwargs): + """Simulate a watcher timeout with no event for the earlier edit. + + Yields: + An empty event batch. + """ + yield set() + + monkeypatch.setattr(watchfiles, "watch", watch) + assert compile_daemon._next_changes(state, lambda: True) == {source} + + def test_iter_source_files_picks_content_skips_build_dirs(tmp_path): (tmp_path / "page.py").write_text("x = 1\n") (tmp_path / "doc.md").write_text("# doc\n") From 717aae27615941d66b2265d7101f42f40b84f5b9 Mon Sep 17 00:00:00 2001 From: Farhan <www.mfarvirus@gmail.com> Date: Thu, 10 Sep 2026 23:57:55 +0500 Subject: [PATCH 25/56] compile cache: fix Windows unit-test failures On Windows, OpenProcess succeeds on an exited process while a handle to it is still open, so _pid_alive now checks GetExitCodeProcess for STILL_ACTIVE. Tests: give the same-size rewrite a distinct mtime (Windows file times are tick-granular) and pass the hot-reload include path relative to cwd, since the ":" delimiter cannot carry a drive letter. Claude-Session: https://claude.ai/code/session_01EXS4neepFi4cuR4TQoNhTi --- reflex/utils/compile_daemon.py | 8 ++++++-- tests/units/compiler/test_page_cache.py | 8 ++++++-- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/reflex/utils/compile_daemon.py b/reflex/utils/compile_daemon.py index d2c5a9a85b3..cd8a6e118be 100644 --- a/reflex/utils/compile_daemon.py +++ b/reflex/utils/compile_daemon.py @@ -702,12 +702,16 @@ def _pid_alive(pid: int) -> bool: kernel32 = ctypes.windll.kernel32 # type: ignore[attr-defined] # PROCESS_QUERY_LIMITED_INFORMATION; os.kill(pid, 0) would *terminate* - # the process on Windows. + # the process on Windows. OpenProcess still succeeds on an exited + # process while any handle to it stays open (e.g. a parent's Popen), so + # the exit code decides: STILL_ACTIVE (259) means running. handle = kernel32.OpenProcess(0x1000, False, pid) if not handle: return False + exit_code = ctypes.c_ulong() + queried = kernel32.GetExitCodeProcess(handle, ctypes.byref(exit_code)) kernel32.CloseHandle(handle) - return True + return bool(queried) and exit_code.value == 259 try: os.kill(pid, 0) except ProcessLookupError: diff --git a/tests/units/compiler/test_page_cache.py b/tests/units/compiler/test_page_cache.py index 68ff21cdffc..8d085f8c72b 100644 --- a/tests/units/compiler/test_page_cache.py +++ b/tests/units/compiler/test_page_cache.py @@ -578,8 +578,9 @@ def counting(path): assert reads == 1 assert v.refreshed[str(f)][1] == bumped - # Same size, different content: the hash decides. + # Same size, different content, distinct mtime: the hash decides. f.write_text("x = 2\n") + os.utime(f, ns=(bumped, bumped)) v = page_cache.FileValidator(files) assert v.changed(str(f)) is True # Unknown or missing files always count as changed. @@ -631,7 +632,10 @@ def test_first_party_roots_include_hot_reload_include_paths(tmp_path, monkeypatc sibling = tmp_path / "sibling_pkg" sibling.mkdir() - monkeypatch.setenv(environment.REFLEX_HOT_RELOAD_INCLUDE_PATHS.name, str(sibling)) + # Relative to cwd: the env var's ":" delimiter cannot carry a Windows + # drive-letter path. + monkeypatch.chdir(tmp_path) + monkeypatch.setenv(environment.REFLEX_HOT_RELOAD_INCLUDE_PATHS.name, sibling.name) roots = page_cache.first_party_roots(tmp_path) assert roots == (tmp_path.resolve(), sibling.resolve()) From 83dedd17ab0863d03b043321dbbba035bdafb3ac Mon Sep 17 00:00:00 2001 From: Farhan <www.mfarvirus@gmail.com> Date: Fri, 11 Sep 2026 00:12:46 +0500 Subject: [PATCH 26/56] compile cache: persist the import set after a memo-only rewrite A rewritten memo file can import a package no page recompile brought in. The manifest now stores the complete import set whenever it differs, so the next all-hit worker installs (and does not remove) that package. Claude-Session: https://claude.ai/code/session_01EXS4neepFi4cuR4TQoNhTi --- reflex/compiler/disk_cache.py | 7 +++++-- tests/units/compiler/test_disk_cache.py | 20 ++++++++++++++++++++ 2 files changed, 25 insertions(+), 2 deletions(-) diff --git a/reflex/compiler/disk_cache.py b/reflex/compiler/disk_cache.py index 0948edded78..a3f49d3ca60 100644 --- a/reflex/compiler/disk_cache.py +++ b/reflex/compiler/disk_cache.py @@ -1307,7 +1307,7 @@ def _update_manifest_for_misses( manifest: The loaded manifest (mutated and rewritten). miss_ctx: The compile context of the recompiled pages, if any. miss_pages: The recompiled page definitions. - all_imports: The complete frontend import set after recompiling misses. + all_imports: The complete frontend import set after this rebuild. root: Project root for dependency discovery. Defaults to cwd. contexts_snapshot: The app's ``_contexts_snapshot`` for slicing stateful pages, or None when no miss page was stateful. @@ -1344,7 +1344,10 @@ def _update_manifest_for_misses( ), root=root, ) - manifest["all_imports"] = _serialize_imports(all_imports) + dirty = True + # A memo-only rewrite can add packages with no page recompiled. + if (imports := _serialize_imports(all_imports)) != manifest["all_imports"]: + manifest["all_imports"] = imports dirty = True if dirty: _write(manifest) diff --git a/tests/units/compiler/test_disk_cache.py b/tests/units/compiler/test_disk_cache.py index c55974402a9..d86470bd91b 100644 --- a/tests/units/compiler/test_disk_cache.py +++ b/tests/units/compiler/test_disk_cache.py @@ -1240,6 +1240,26 @@ def test_update_manifest_for_misses_keeps_complete_imports(tmp_path, monkeypatch assert disk_cache._deserialize_imports(written["all_imports"]) == complete_imports +def test_update_manifest_refreshes_imports_without_misses(tmp_path, monkeypatch): + """A memo-only rewrite that adds a package must reach the manifest.""" + web = _use_tmp_web_dir(tmp_path, monkeypatch) + manifest = _manifest({}, all_imports={"old-lib": [{"tag": "Old"}]}) + memo_imports = {"old-lib": [ImportVar("Old")], "memo-lib": [ImportVar("Memo")]} + + disk_cache._update_manifest_for_misses( + manifest, + None, + [], + memo_imports, + root=tmp_path, + memo_state={}, + memo_file_entries={}, + ) + + written = json.loads((web / disk_cache._MANIFEST_FILE).read_text()) + assert disk_cache._deserialize_imports(written["all_imports"]) == memo_imports + + def _page_loop_a() -> Component: return rx.el.ul(rx.foreach(["a", "b"], lambda item: rx.el.li(item))) From f2bd03e8a19e482ea79d941900610d8cb86bcd14 Mon Sep 17 00:00:00 2001 From: Farhan <www.mfarvirus@gmail.com> Date: Fri, 11 Sep 2026 01:15:11 +0500 Subject: [PATCH 27/56] compile cache: key the manifest on compile mode and adopt evaluated pages on fallback The manifest records the compile mode, so a production build never reuses development output. A backend worker consumes its nocompile marker even while the compile daemon owns the build, edits from a failed hot reload stay pending until a compile succeeds, and a full compile adopts the pages an incremental attempt already evaluated instead of evaluating them twice. Also document the open state-default name defect: the stored state slice is drawn from a second compile_state call and never matches the contexts file, which makes the contexts-reuse check random on content-only edits and can give two states the same generated ref name (reproduced with rx.auto_scroll defaults). Fix direction is per-state scoping of default-name generation. Claude-Session: https://claude.ai/code/session_01EXS4neepFi4cuR4TQoNhTi --- news/6688.bugfix.md | 2 +- .../src/reflex_base/plugins/compiler.py | 20 ++++ reflex/app.py | 22 ++--- reflex/compiler/compiler.py | 43 ++++++--- reflex/compiler/disk_cache.py | 49 ++++++++-- reflex/utils/compile_daemon.py | 6 ++ tests/units/compiler/test_disk_cache.py | 89 +++++++++++++++++- tests/units/test_app.py | 93 +++++++++++++++++++ tests/units/utils/test_compile_daemon.py | 46 +++++++++ 9 files changed, 331 insertions(+), 39 deletions(-) diff --git a/news/6688.bugfix.md b/news/6688.bugfix.md index d22398160a0..d9df4cc3a54 100644 --- a/news/6688.bugfix.md +++ b/news/6688.bugfix.md @@ -1 +1 @@ -Fixed repeated full recompilation when the incremental compile cache encounters state defaults requiring Reflex serialization, such as Pydantic models or dates. Manifest write failures now produce visible warnings instead of silently disabling cache reuse. Cache invalidation now includes package initializers, missing file reads, and app entrypoints that also define pages. Incremental builds run plugin output hooks, preserve compile telemetry, and refresh stylesheets. The daemon records its own PID, releases backend compilation when it exits, preserves edits during watcher teardown, discovers new root-level sources, and reuses the parent's initial build. Explicit `app_module_import` configurations use full compilation because earlier import-time reads may be unavailable. Component-doc control IDs remain stable across partial builds, and Markdown-triggered Uvicorn reloads are limited to the experimental cache mode. +Fixed repeated full recompilation when the incremental compile cache encounters state defaults requiring Reflex serialization, such as Pydantic models or dates. Manifest write failures now produce visible warnings instead of silently disabling cache reuse. Cache invalidation now includes package initializers, missing file reads, and app entrypoints that also define pages. Incremental builds run plugin output hooks, preserve compile telemetry, and refresh stylesheets. The daemon records its own PID, releases backend compilation when it exits, preserves edits during watcher teardown, discovers new root-level sources, and reuses the parent's initial build. Explicit `app_module_import` configurations use full compilation because earlier import-time reads may be unavailable. Component-doc control IDs remain stable across partial builds, and Markdown-triggered Uvicorn reloads are limited to the experimental cache mode. The cache identity now includes the compile mode, so a production build never reuses development output, a backend worker consumes its `nocompile` marker even while the compile daemon owns the build, edits from a failed hot reload stay pending until a compile succeeds, and a full compile adopts the pages an incremental attempt already evaluated instead of evaluating them twice. diff --git a/packages/reflex-base/src/reflex_base/plugins/compiler.py b/packages/reflex-base/src/reflex_base/plugins/compiler.py index d73a6f0b5b4..24a7d77f3a2 100644 --- a/packages/reflex-base/src/reflex_base/plugins/compiler.py +++ b/packages/reflex-base/src/reflex_base/plugins/compiler.py @@ -922,6 +922,26 @@ def compile( reset_unique_variable_names() return self.compiled_pages + def absorb(self, other: CompileContext, pages: Sequence[PageDefinition]) -> None: + """Adopt the pages another context compiled, so none is compiled twice. + + Merges the other context's per-page results and app-wide accumulations + into this one and orders the compiled pages by ``pages``, which becomes + this context's page list. + + Args: + other: A context that compiled a disjoint set of pages. + pages: The complete page list, in compile order. + """ + compiled = {**self.compiled_pages, **other.compiled_pages} + self.pages = pages + self.compiled_pages = {page.route: compiled[page.route] for page in pages} + self.all_imports = merge_imports(self.all_imports, other.all_imports) + self.app_wrap_components.update(other.app_wrap_components) + self.stateful_routes.update(other.stateful_routes) + self.memoize_wrappers.update(other.memoize_wrappers) + self.auto_memo_components.update(other.auto_memo_components) + __all__ = [ "BaseContext", diff --git a/reflex/app.py b/reflex/app.py index 21b313aaef4..5981d00b0ed 100644 --- a/reflex/app.py +++ b/reflex/app.py @@ -1587,23 +1587,19 @@ def _should_compile(self) -> bool: if environment.REFLEX_SKIP_COMPILE.get(): return False + # The nocompile marker is the backend's: its first worker consumes it + # and stands down. The compile daemon and its children never read it. + if not environment.REFLEX_COMPILE_DAEMON.get(): + nocompile = prerequisites.get_web_dir() / constants.NOCOMPILE_FILE + if nocompile.exists(): + nocompile.unlink(missing_ok=True) + return False + # A running compile daemon owns .web; backend workers only evaluate # pages to register state. from reflex.utils import compile_daemon - if not compile_daemon.owns_compilation(): - return False - - nocompile = prerequisites.get_web_dir() / constants.NOCOMPILE_FILE - - # Check the nocompile file. - if nocompile.exists(): - # Delete the nocompile file - nocompile.unlink(missing_ok=True) - return False - - # By default, compile the app. - return True + return compile_daemon.owns_compilation() def _setup_sticky_badge(self): """Add the sticky badge to the app.""" diff --git a/reflex/compiler/compiler.py b/reflex/compiler/compiler.py index 54fd1344a37..024b4f2ea74 100644 --- a/reflex/compiler/compiler.py +++ b/reflex/compiler/compiler.py @@ -1367,19 +1367,24 @@ def compile_app( # Experimental incremental compile cache: in a fresh process, recompile only # the pages whose source changed and reuse the rest from the on-disk - # manifest. Falls back to a full compile on any unsafe condition. + # manifest. Falls back to a full compile on any unsafe condition; pages the + # attempt already compiled before falling back are adopted, not redone. + evaluated: CompileContext | None = None if cache_on: from reflex.compiler import disk_cache, page_cache page_cache.enable_read_tracking() outputs: dict[Path, str] = {} - if disk_cache.try_incremental_rebuild( + rebuilt = disk_cache.try_incremental_rebuild( app, compiler_plugins=compiler_plugins, prerender_routes=prerender_routes, use_rich=use_rich, outputs=outputs, - ): + ) + if isinstance(rebuilt, CompileContext): + evaluated = rebuilt + elif rebuilt: try: save_tasks, modify_tasks = _plugin_output_tasks( config.plugins, @@ -1419,22 +1424,30 @@ def compile_app( progress = make_compile_progress(use_rich) fixed_steps = 7 - reset_bundled_libraries() - # Drop cached memo wrapper classes so each compile recomputes a memo's - # ``library`` from the current module layout (handles a module flipping to - # a package across hot reloads). - reset_memo_component_classes() - for plugin in compiler_plugins: - for dependency in plugin.get_frontend_dependencies(): - bundle_library(dependency) - base_total = (len(app._unevaluated_pages) * 2) + fixed_steps + len(config.plugins) + all_pages = list(app._unevaluated_pages.values()) + if evaluated is None: + pages_to_compile = all_pages + reset_bundled_libraries() + # Drop cached memo wrapper classes so each compile recomputes a memo's + # ``library`` from the current module layout (handles a module flipping + # to a package across hot reloads). + reset_memo_component_classes() + for plugin in compiler_plugins: + for dependency in plugin.get_frontend_dependencies(): + bundle_library(dependency) + else: + # The incremental attempt already did the resets above and bundled what + # its pages registered; only the pages it never touched remain. + pages_to_compile = [ + page for page in all_pages if page.route not in evaluated.compiled_pages + ] + base_total = (len(pages_to_compile) * 2) + fixed_steps + len(config.plugins) progress.start() task = progress.add_task("Compiling:", total=base_total) - all_pages = list(app._unevaluated_pages.values()) compile_ctx = CompileContext( app=app, - pages=all_pages, + pages=pages_to_compile, hooks=CompilerHooks( plugins=default_page_plugins(style=app.style, plugins=compiler_plugins) ), @@ -1449,6 +1462,8 @@ def compile_app( evaluate_progress=lambda: progress.advance(task), render_progress=lambda: progress.advance(task), ) + if evaluated is not None: + compile_ctx.absorb(evaluated, all_pages) _register_compiled_pages(app, compile_ctx.compiled_pages) app._stateful_pages.update(dict.fromkeys(compile_ctx.stateful_routes)) diff --git a/reflex/compiler/disk_cache.py b/reflex/compiler/disk_cache.py index a3f49d3ca60..bf447197a06 100644 --- a/reflex/compiler/disk_cache.py +++ b/reflex/compiler/disk_cache.py @@ -11,6 +11,15 @@ file is recognised from ``(mtime_ns, size)`` without being read, and when the watcher reports which paths changed, pages depending on none of them are hits without even a stat. + +TODO(state-default names): a page's ``state_slice`` is taken from a second +``compile_state`` call, so its generated component-default names (``ref_*``) +never match the contexts file on disk. ``_changed_state_config_route`` then +compares unequal names on content-only edits and rebuilds contexts at random, +and a rebuild can give a live state the same name a hit page's stored state +holds (reproduced: two ``rx.auto_scroll`` defaults sharing one ref/id). Fix by +scoping default-name generation per state, then slicing the same +``compile_state`` result that is written to disk. """ from __future__ import annotations @@ -24,6 +33,7 @@ from typing import TYPE_CHECKING, Any from reflex_base import constants +from reflex_base.environment import environment from reflex_base.plugins import CompileContext, CompilerHooks from reflex_base.utils.format import json_dumps from reflex_base.utils.imports import ImportVar, merge_imports @@ -41,7 +51,7 @@ from reflex.app import App #: Bump when the manifest layout changes (old manifests are then ignored). -_SCHEMA = 11 +_SCHEMA = 12 #: Manifest filename under the web directory. _MANIFEST_FILE = "reflex_compile_cache.json" @@ -471,6 +481,7 @@ def write_manifest( "schema": _SCHEMA, "plugin_sources": plugin_sources or {}, "reflex_version": page_cache._reflex_version(), + "mode": _compile_mode(), "files": files, # Per-input labels (not one combined digest) so a later mismatch # can name the exact global input that changed. @@ -488,6 +499,15 @@ def write_manifest( ) +def _compile_mode() -> str: + """The compile mode the rendered output is specific to (``dev``/``prod``). + + Returns: + The current ``REFLEX_ENV_MODE`` value. + """ + return environment.REFLEX_ENV_MODE.get().value + + def _imported_state_files(compile_ctx: CompileContext, root: Path | None) -> set[str]: """Find source dependencies of states not owned by page evaluation. @@ -551,6 +571,9 @@ def globals_mismatch( return ( f"reflex version changed ({old_version} -> {page_cache._reflex_version()})" ) + old_mode = manifest.get("mode") + if old_mode != _compile_mode(): + return f"compile mode changed ({old_mode} -> {_compile_mode()})" old_routes = set(manifest.get("pages", {})) if old_routes != routes: parts = [] @@ -950,12 +973,17 @@ def try_incremental_rebuild( root: Path | None = None, use_rich: bool = True, outputs: dict[Path, str] | None = None, -) -> bool: +) -> bool | CompileContext: """Attempt a disk-cache-assisted partial rebuild; report whether it ran. Returns False (so the caller does a full compile) whenever anything is - unsafe to reuse: no/old manifest, a changed global input, a route change, or - a miss page that altered its app-wrap set or stateful flag. + unsafe to reuse: no/old manifest, a changed global input or a route change. + When a miss page turns out to have altered its app-wrap set or stateful + flag, the miss pages are already evaluated and rendered. A page is never + evaluated twice in one process (its locally defined states would register + under collision-suffixed names the backend does not use), so the context + holding those compiled pages is returned instead of False, for the full + compile to adopt (see ``CompileContext.absorb``). User-memo output files are reconciled against the manifest's stored memo record (see :func:`_current_memo_state`): files whose record differs are @@ -982,8 +1010,9 @@ def try_incremental_rebuild( outputs: Stage generated files here for plugin processing instead of writing. Returns: - True if the partial rebuild completed (the caller should return), else - False (the caller should run a full compile). + True if the partial rebuild completed (the caller should return), False + for a full compile, or the context of the pages already compiled here + for a full compile that adopts them. """ manifest = load_manifest() if manifest is None: @@ -1109,17 +1138,17 @@ def try_incremental_rebuild( or page_ctx.output_path is None ): _log_fallback(f"page {page.route!r} produced no output") - return False + return miss_ctx entry = manifest["pages"][page.route] if ( _wrap_key_strs(page_ctx.app_wrap_components.keys()) != entry["app_wrap_keys"] ): _log_fallback(f"page {page.route!r} changed its app-wrap set") - return False + return miss_ctx if (page.route in miss_ctx.stateful_routes) != entry["is_stateful"]: _log_fallback(f"page {page.route!r} changed statefulness") - return False + return miss_ctx from reflex.compiler import compiler @@ -1137,7 +1166,7 @@ def try_incremental_rebuild( output_code = page_ctx.output_code if output_path is None or output_code is None: _log_fallback(f"page {page.route!r} lost its output before write") - return False + return miss_ctx _save_incremental_output(output_path, output_code, outputs) memo_contributions.update(page_ctx.memo_contributions) miss_imports.append(page_ctx.frontend_imports) diff --git a/reflex/utils/compile_daemon.py b/reflex/utils/compile_daemon.py index cd8a6e118be..4055083ee59 100644 --- a/reflex/utils/compile_daemon.py +++ b/reflex/utils/compile_daemon.py @@ -986,11 +986,16 @@ def alive() -> bool: return os.getppid() == parent_pid pending: set[Path] = set() + # Changes a failed compile never applied. The manifest only moves on a + # successful compile, so they stay in the changed hint until one succeeds; + # otherwise the hint's stat-free hit shortcut would reuse their pages. + unapplied: set[Path] = set() while alive(): changed = pending or _next_changes(state, alive) pending = set() if not changed: break + changed = changed | unapplied from reflex.compiler.disk_cache import format_path_list console.info( @@ -1014,6 +1019,7 @@ def alive() -> bool: state.roots = roots = _reload_roots() with _compile_lock(root): ok = _compile_once(roots, prerender_routes, changed) + unapplied = set() if ok else changed if not ok: console.error("Compile failed; keeping the last good build.") # Refresh what is watched from the new manifest so a newly-referenced diff --git a/tests/units/compiler/test_disk_cache.py b/tests/units/compiler/test_disk_cache.py index d86470bd91b..ebc7e296477 100644 --- a/tests/units/compiler/test_disk_cache.py +++ b/tests/units/compiler/test_disk_cache.py @@ -12,12 +12,14 @@ import pytest from reflex_base.components.component import Component +from reflex_base.environment import environment from reflex_base.plugins import CompileContext, CompilerHooks from reflex_base.utils.imports import ImportVar import reflex as rx from reflex.compiler import disk_cache, page_cache from reflex.compiler.plugins import default_page_plugins +from reflex.state import BaseState @dataclasses.dataclass(slots=True) @@ -79,7 +81,7 @@ def _compile(pages: Sequence[Any], app: Any = None) -> CompileContext: return ctx -def _unregister_state(cls: type[rx.State]) -> None: +def _unregister_state(cls: type[BaseState]) -> None: """Drop a state class from the registries, like the daemon registry reset. Lets a page evaluation re-define the same-named class in this process, @@ -233,6 +235,7 @@ def _manifest(pages: dict[str, dict], **overrides) -> dict: base = { "schema": disk_cache._SCHEMA, "reflex_version": page_cache._reflex_version(), + "mode": environment.REFLEX_ENV_MODE.get().value, "files": {}, "globals": {"reflex": page_cache._reflex_version()}, "globals_absent": [], @@ -291,6 +294,27 @@ def test_globals_mismatch_names_the_changed_input(tmp_path): assert "0.0.0-old" in reason +def test_globals_mismatch_names_a_compile_mode_change(tmp_path, monkeypatch): + """A development build's output must never be reused by a production build. + + Rendered files differ by mode (e.g. ``isDevMode`` in the contexts file), so + the mode is part of the cache identity. + """ + m = _manifest({"/a": {}}) + assert ( + disk_cache.globals_mismatch( + m, routes={"/a"}, validator=_validator(m), root=tmp_path + ) + is None + ) + monkeypatch.setenv("REFLEX_ENV_MODE", "prod") + reason = disk_cache.globals_mismatch( + m, routes={"/a"}, validator=_validator(m), root=tmp_path + ) + assert reason is not None + assert "dev -> prod" in reason + + def test_globals_mismatch_validates_stored_inputs_only(tmp_path): """Epoch validation re-hashes the *stored* input set, never a recomputed one. @@ -386,6 +410,7 @@ def test_write_and_load_manifest(tmp_path, monkeypatch): manifest = disk_cache.load_manifest() assert manifest is not None assert manifest["schema"] == disk_cache._SCHEMA + assert manifest["mode"] == "dev" assert set(manifest["pages"]) == {"/a", "/b", "/c"} for route in ("/a", "/b", "/c"): entry = manifest["pages"][route] @@ -518,6 +543,68 @@ def _stub_externals(app, monkeypatch): monkeypatch.setattr(fs, "initialize_vite_config", lambda: None) +def _page_local_state() -> Component: + class CacheLocalState(rx.State): + value: int = 0 + + return rx.el.div(CacheLocalState.value) + + +def test_post_evaluation_fallback_hands_over_the_evaluated_pages(tmp_path, monkeypatch): + """A fallback decided after evaluating miss pages returns those pages. + + Evaluating a page registers the states it defines. Re-evaluating it in the + same process (as an in-process full compile would) registers a locally + defined state under a collision suffix (``CacheLocalState_0``) while the + backend, which evaluates the page once, knows it as ``CacheLocalState``. + The full compile therefore has to reuse this evaluation, not repeat it. + """ + from reflex_base.registry import RegistrationContext + + import reflex.istate.dynamic as istate_dynamic + + web = _use_tmp_web_dir(tmp_path, monkeypatch) + app = rx.App() + app.add_page(_page_a, route="/a") + app.add_page(_page_local_state, route="/s") + _stub_externals(app, monkeypatch) + pages = list(app._unevaluated_pages.values()) + route_s = pages[1].route + ctx = _compile(pages, app) + defined = ctx.stateful_routes[route_s] + assert defined[0].endswith("cache_local_state") + disk_cache.write_manifest(ctx, pages, ctx.all_imports, root=tmp_path) + + def forget_local_states() -> None: + """Drop the page-defined states, as the daemon's child reset does.""" + registry = RegistrationContext.ensure_context() + for name in defined: + cls = registry.base_states.get(name) + if cls is not None: + _unregister_state(cls) + if getattr(istate_dynamic, cls.__name__, None) is cls: + delattr(istate_dynamic, cls.__name__) + + forget_local_states() + try: + # The manifest recorded /s as stateless; its evaluation now flips + # statefulness, which only a full compile can resolve. + manifest_path = web / disk_cache._MANIFEST_FILE + manifest = json.loads(manifest_path.read_text()) + manifest["pages"][route_s]["is_stateful"] = False + _stale_dep(manifest, route_s, str(tmp_path / "demo.md")) + manifest_path.write_text(json.dumps(manifest)) + + result = disk_cache.try_incremental_rebuild( + app, compiler_plugins=[], prerender_routes=False, root=tmp_path + ) + assert isinstance(result, CompileContext) + assert set(result.compiled_pages) == {route_s} + assert result.stateful_routes[route_s] == defined + finally: + forget_local_states() + + def test_incremental_rebuild_all_hits(tmp_path, monkeypatch): _use_tmp_web_dir(tmp_path, monkeypatch) diff --git a/tests/units/test_app.py b/tests/units/test_app.py index 932456cd6ed..6f56e5b0a85 100644 --- a/tests/units/test_app.py +++ b/tests/units/test_app.py @@ -4612,3 +4612,96 @@ def test_compile_emits_stage_spans( parent = spans[name].parent assert parent is not None assert parent.span_id == root.get_span_context().span_id + + +def test_should_compile_consumes_the_nocompile_marker_under_a_daemon( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +): + """A backend worker consumes its ``nocompile`` marker even while a daemon owns .web. + + Otherwise the marker outlives the worker and the daemon's next compile + child finds it, consumes it and skips the compile, so the first edit never + reaches the browser. Daemon processes never touch the marker at all. + """ + from reflex.utils import compile_daemon + + web_dir = tmp_path / ".web" + web_dir.mkdir() + monkeypatch.setattr("reflex.utils.prerequisites.get_web_dir", lambda: web_dir) + monkeypatch.setattr( + compile_daemon, "_daemon_marker_path", lambda: tmp_path / "daemon.pid" + ) + monkeypatch.delenv(environment.REFLEX_SKIP_COMPILE.name, raising=False) + monkeypatch.delenv(environment.REFLEX_COMPILE_DAEMON.name, raising=False) + compile_daemon.mark_daemon_active() + marker = web_dir / constants.NOCOMPILE_FILE + marker.touch() + app = App() + + # The worker stands down and takes its marker with it. + assert app._should_compile() is False + assert not marker.exists() + + # The daemon (and its compile children) compile regardless of the marker. + marker.touch() + monkeypatch.setenv(environment.REFLEX_COMPILE_DAEMON.name, "1") + assert app._should_compile() is True + assert marker.exists() + + +def test_compile_reuses_pages_the_incremental_attempt_already_evaluated( + compilable_app: tuple[App, Path], + mocker: MockerFixture, + monkeypatch: pytest.MonkeyPatch, +): + """A post-evaluation cache fallback must not evaluate a page twice. + + Evaluating a page registers the states it defines; a second evaluation in + the same process renames a locally defined state (``LocalState_0``) while + the backend, which evaluates the page once, knows it as ``LocalState``. The + full compile therefore adopts the pages the incremental attempt compiled. + """ + from reflex.compiler import disk_cache + from reflex.compiler.plugins import default_page_plugins + + app, web_dir = compilable_app + mocker.patch("reflex.utils.prerequisites.get_web_dir", return_value=web_dir) + monkeypatch.setenv("REFLEX_COMPILE_CACHE", "1") + evaluations: list[str] = [] + + def local_state_page(): + evaluations.append("local") + + class AdoptedLocalState(rx.State): + value: int = 0 + + return rx.el.div(AdoptedLocalState.value) + + def other_page(): + evaluations.append("other") + return rx.el.div("other") + + app.add_page(local_state_page, route="local") + app.add_page(other_page, route="other") + + # What the incremental attempt did before deciding it must fall back. + evaluated = CompileContext( + app=app, + pages=[app._unevaluated_pages["local"]], + hooks=CompilerHooks(plugins=default_page_plugins(style=app.style)), + ) + with evaluated: + evaluated.compile() + registered = evaluated.stateful_routes["local"] + mocker.patch.object(disk_cache, "try_incremental_rebuild", return_value=evaluated) + + app._compile() + + assert evaluations == ["local", "other"] + assert app._stateful_pages.keys() >= {"local"} + assert set(app._pages) >= {"local", "other"} + routes_dir = web_dir / constants.Dirs.PAGES / constants.Dirs.ROUTES + assert (routes_dir / "[local]._index.jsx").exists() + assert (routes_dir / "[other]._index.jsx").exists() + assert registered[0].endswith("adopted_local_state") + assert all(not name.endswith("_0") for name in registered) diff --git a/tests/units/utils/test_compile_daemon.py b/tests/units/utils/test_compile_daemon.py index e162d6c21e8..6171b6c0f11 100644 --- a/tests/units/utils/test_compile_daemon.py +++ b/tests/units/utils/test_compile_daemon.py @@ -687,3 +687,49 @@ def watch(*args, **kwargs): monkeypatch.setattr(watchfiles, "watch", watch) assert compile_daemon._next_changes(state, lambda: True) == {first, second} + + +def test_serve_keeps_failed_changes_pending_until_a_compile_succeeds( + tmp_path, monkeypatch +): + """Paths from a failed compile are re-reported with the next change batch. + + The manifest is only refreshed by a successful compile, so a path that + missed a failed build is still a stale dependency: dropping it from the + changed hint would let the stat-free hit shortcut reuse its page. + """ + from reflex.utils import prerequisites + + monkeypatch.chdir(tmp_path) + monkeypatch.setattr(compile_daemon, "_lock_path", lambda: tmp_path / "lock") + monkeypatch.setattr(compile_daemon, "_reload_roots", lambda: [tmp_path]) + monkeypatch.setattr(compile_daemon, "_prepare_fork_parent", lambda roots: None) + monkeypatch.setattr(prerequisites, "get_app", lambda **kwargs: None) + monkeypatch.setenv("REFLEX_COMPILE_DAEMON_PRECOMPILED", "1") + monkeypatch.setattr( + compile_daemon._WatchState, + "build", + lambda *args: compile_daemon._WatchState([tmp_path], tmp_path), + ) + a, b = tmp_path / "a.py", tmp_path / "b.py" + batches = iter([{a, b}, {b}, {a}, None]) + monkeypatch.setattr(compile_daemon, "_next_changes", lambda *args: next(batches)) + compiles: list[set[Path]] = [] + + def compile_once(roots, prerender_routes, changed=None): + """Fail the first compile, succeed afterwards. + + Args: + roots: The reload roots. + prerender_routes: Whether to prerender routes. + changed: The changed-path hint handed to the compile. + + Returns: + Whether the simulated compile succeeded. + """ + compiles.append(set(changed or ())) + return len(compiles) > 1 + + monkeypatch.setattr(compile_daemon, "_compile_once", compile_once) + compile_daemon._serve() + assert compiles == [{a, b}, {a, b}, {a}] From 2b45a6aa02d248c00a8e3a60255c1b31811551f5 Mon Sep 17 00:00:00 2001 From: Farhan <www.mfarvirus@gmail.com> Date: Tue, 15 Sep 2026 01:43:47 +0500 Subject: [PATCH 28/56] Fix cached state slices --- reflex/compiler/compiler.py | 42 +++++++++++++++------ reflex/compiler/disk_cache.py | 21 +++++------ reflex/compiler/utils.py | 29 +++++++++++---- tests/units/compiler/test_disk_cache.py | 49 ++++++++++++++++++++++++- 4 files changed, 109 insertions(+), 32 deletions(-) diff --git a/reflex/compiler/compiler.py b/reflex/compiler/compiler.py index 12c7a77e06c..f882cc67928 100644 --- a/reflex/compiler/compiler.py +++ b/reflex/compiler/compiler.py @@ -33,7 +33,7 @@ from reflex_base.environment import environment from reflex_base.plugins import CompileContext, CompilerHooks, PageContext, Plugin from reflex_base.registry import RegistrationContext, _default_bundled_libraries -from reflex_base.utils import format, log, memo_paths +from reflex_base.utils import log, memo_paths from reflex_base.utils.exceptions import ReflexError from reflex_base.utils.format import to_title_case from reflex_base.utils.imports import ( @@ -265,6 +265,7 @@ def _compile_contexts( extra_state: tuple[dict[str, Any], dict[str, dict[str, Any]]] | None = None, *, component_imports: ParsedImportDict | None = None, + contexts_snapshot: tuple[dict[str, Any], dict[str, dict[str, Any]]] | None = None, ) -> str: """Compile the initial state and contexts. @@ -275,6 +276,8 @@ def _compile_contexts( are not in this process's state tree (states only registered while a page evaluates, restored from the compile cache manifest). component_imports: Optional accumulator for initial component dependencies. + contexts_snapshot: Initial state and client storage compiled for this + contexts output, when already available. Returns: The compiled context file. @@ -283,24 +286,28 @@ def _compile_contexts( disable_react_owner_stacks = ( not is_prod_mode() and not environment.REFLEX_REACT_OWNER_STACKS.get() ) - initial_state, initial_state_json = ( - utils._compile_initial_state(state, component_imports=component_imports) - if state - else (None, None) - ) if state is None: return templates.context_template( is_dev_mode=not is_prod_mode(), default_color_mode=default_color_mode, disable_react_owner_stacks=disable_react_owner_stacks, ) - assert initial_state is not None - assert initial_state_json is not None - client_storage = utils.compile_client_storage(state) + if contexts_snapshot is None: + initial_state, initial_state_json = utils._compile_initial_state( + state, component_imports=component_imports + ) + client_storage = utils.compile_client_storage(state) + else: + initial_state, client_storage = contexts_snapshot + initial_state_json = utils._serialize_initial_state( + initial_state, component_imports=component_imports + ) if extra_state is not None: extra_initial, extra_storage = extra_state initial_state = dict(sorted({**initial_state, **extra_initial}.items())) - initial_state_json = format.json_dumps(initial_state) + initial_state_json = utils._serialize_initial_state( + initial_state, component_imports=component_imports + ) for kind, entries in extra_storage.items(): client_storage.setdefault(kind, {}).update(entries) return templates.context_template( @@ -793,6 +800,7 @@ def compile_contexts( extra_state: tuple[dict[str, Any], dict[str, dict[str, Any]]] | None = None, *, component_imports: ParsedImportDict | None = None, + contexts_snapshot: tuple[dict[str, Any], dict[str, dict[str, Any]]] | None = None, ) -> tuple[str, str]: """Compile the initial state / context. @@ -801,6 +809,7 @@ def compile_contexts( theme: The top-level app theme. extra_state: See :func:`_compile_contexts`. component_imports: Optional accumulator for initial component dependencies. + contexts_snapshot: See :func:`_compile_contexts`. Returns: The path and code of the compiled context. @@ -813,6 +822,7 @@ def compile_contexts( theme, extra_state, component_imports=component_imports, + contexts_snapshot=contexts_snapshot, ) @@ -1659,11 +1669,17 @@ def compile_app( compile_results.append(result) progress.advance(task) + contexts_snapshot = ( + (utils.compile_state(app._state), utils.compile_client_storage(app._state)) + if app._state is not None + else None + ) compile_results.extend([ compile_contexts( app._state, radix_themes_plugin.get_theme(), component_imports=all_imports, + contexts_snapshot=contexts_snapshot, ), utils._compile_bundled_libraries(), ]) @@ -1727,7 +1743,11 @@ def compile_app( from reflex.compiler import disk_cache disk_cache.write_manifest( - compile_ctx, all_pages, all_imports, plugin_sources=plugin_sources + compile_ctx, + all_pages, + all_imports, + plugin_sources=plugin_sources, + contexts_snapshot=contexts_snapshot, ) return True diff --git a/reflex/compiler/disk_cache.py b/reflex/compiler/disk_cache.py index 6af1d67f9ba..43c41398236 100644 --- a/reflex/compiler/disk_cache.py +++ b/reflex/compiler/disk_cache.py @@ -12,14 +12,6 @@ watcher reports which paths changed, pages depending on none of them are hits without even a stat. -TODO(state-default names): a page's ``state_slice`` is taken from a second -``compile_state`` call, so its generated component-default names (``ref_*``) -never match the contexts file on disk. ``_changed_state_config_route`` then -compares unequal names on content-only edits and rebuilds contexts at random, -and a rebuild can give a live state the same name a hit page's stored state -holds (reproduced: two ``rx.auto_scroll`` defaults sharing one ref/id). Fix by -scoping default-name generation per state, then slicing the same -``compile_state`` result that is written to disk. """ from __future__ import annotations @@ -405,6 +397,7 @@ def write_manifest( root: Path | None = None, *, plugin_sources: dict[str, str] | None = None, + contexts_snapshot: tuple[dict[str, Any], dict[str, dict[str, Any]]] | None = None, ) -> None: """Persist a manifest of the just-completed full compile. @@ -421,14 +414,15 @@ def write_manifest( install from this complete set, not just the per-page union. root: Project root for fingerprinting. Defaults to cwd. plugin_sources: Unmodified source content for plugin file modifiers. + contexts_snapshot: Initial state and client storage used for the contexts + output. When absent, a snapshot is captured for direct callers. """ try: state_index, _ = page_cache.state_dependency_index(root) hasher = page_cache.make_hasher() files: dict[str, page_cache.FileEntry] = {} - contexts_snapshot = ( - _contexts_snapshot(compile_ctx.app) if compile_ctx.stateful_routes else None - ) + if contexts_snapshot is None and compile_ctx.stateful_routes: + contexts_snapshot = _contexts_snapshot(compile_ctx.app) pages_data: dict[str, Any] = {} for page in pages: @@ -1254,7 +1248,10 @@ def try_incremental_rebuild( None, ) context_path, context_code = compiler.compile_contexts( - app._state, theme, _hit_state_extras(manifest, miss_routes) + app._state, + theme, + _hit_state_extras(manifest, miss_routes), + contexts_snapshot=contexts_snapshot, ) _save_incremental_output(context_path, context_code, outputs) diff --git a/reflex/compiler/utils.py b/reflex/compiler/utils.py index 936bc0ed751..d9c5e4d92c2 100644 --- a/reflex/compiler/utils.py +++ b/reflex/compiler/utils.py @@ -237,17 +237,17 @@ def compile_state(state: type[BaseState]) -> dict: return _sorted_keys(asyncio.run(_resolve_delta(initial_state))) -def _compile_initial_state( - state: type[BaseState], *, component_imports: ParsedImportDict | None = None -) -> tuple[dict, str]: +def _serialize_initial_state( + initial_state: dict, *, component_imports: ParsedImportDict | None = None +) -> str: """Serialize initial state while discovering its dynamic component imports. Args: - state: The app state class. + initial_state: The compiled initial-state mapping. component_imports: Optional accumulator for frontend package installation. Returns: - The initial state dictionary and its serialized JSON. + The serialized initial-state JSON. """ def serialize_initial_value(value: Any) -> Any: @@ -267,9 +267,24 @@ def serialize_initial_value(value: Any) -> Any: component_imports.setdefault(library, []).extend(fields) return serializers.serialize(value) + return format.json_dumps(initial_state, default=serialize_initial_value) + + +def _compile_initial_state( + state: type[BaseState], *, component_imports: ParsedImportDict | None = None +) -> tuple[dict, str]: + """Compile and serialize initial state while discovering component imports. + + Args: + state: The app state class. + component_imports: Optional accumulator for frontend package installation. + + Returns: + The initial state dictionary and its serialized JSON. + """ initial_state = compile_state(state) - return initial_state, format.json_dumps( - initial_state, default=serialize_initial_value + return initial_state, _serialize_initial_state( + initial_state, component_imports=component_imports ) diff --git a/tests/units/compiler/test_disk_cache.py b/tests/units/compiler/test_disk_cache.py index 1e012ce26d3..1e37222c6bf 100644 --- a/tests/units/compiler/test_disk_cache.py +++ b/tests/units/compiler/test_disk_cache.py @@ -608,7 +608,7 @@ def _stub_externals(app, monkeypatch): # classes from other collected test modules. monkeypatch.setattr( "reflex.compiler.compiler.compile_contexts", - lambda state, theme, extra_state=None: ( + lambda state, theme, extra_state=None, **kwargs: ( compiler_utils.get_context_path(), _CONTEXTS_STUB, ), @@ -626,6 +626,51 @@ class CacheLocalState(rx.State): return rx.el.div(CacheLocalState.value) +def _page_local_component_default() -> Component: + class CacheComponentDefaultState(rx.State): + scroller: rx.Field[rx.Component] = rx.field(default_factory=rx.auto_scroll) + + return rx.el.div(CacheComponentDefaultState.scroller) + + +def test_manifest_state_slice_matches_the_compiled_contexts_snapshot( + tmp_path, monkeypatch +): + """State defaults in the manifest must use the contexts compilation values.""" + _use_tmp_web_dir(tmp_path, monkeypatch) + app = rx.App() + app.add_page(_page_local_component_default, route="/stateful") + pages = list(app._unevaluated_pages.values()) + ctx = _compile(pages, app) + route = next( + page.route for page in pages if page.component is _page_local_component_default + ) + defined_states = ctx.stateful_routes[route] + snapshot = disk_cache._contexts_snapshot(app) + assert snapshot is not None + expected = json.loads( + disk_cache.json_dumps(disk_cache._state_slice(defined_states, *snapshot)) + ) + + try: + disk_cache.write_manifest( + ctx, + pages, + ctx.all_imports, + root=tmp_path, + contexts_snapshot=snapshot, + ) + manifest = disk_cache.load_manifest() + assert manifest is not None + assert manifest["pages"][route]["state_slice"] == expected + finally: + from reflex_base.registry import RegistrationContext + + registry = RegistrationContext.ensure_context() + for state_name in defined_states: + _unregister_state(registry.base_states[state_name]) + + def test_post_evaluation_fallback_hands_over_the_evaluated_pages(tmp_path, monkeypatch): """A fallback decided after evaluating miss pages returns those pages. @@ -896,7 +941,7 @@ def test_stateful_miss_rewrites_contexts_from_manifest_slices(tmp_path, monkeypa ) captured: list[Any] = [] - def fake_compile_contexts(state, theme, extra_state=None): + def fake_compile_contexts(state, theme, extra_state=None, **kwargs): captured.append(extra_state) return compiler_utils.get_context_path(), _CONTEXTS_STUB From e4c64bf07be1337ce946209a69bc987730a0f5ee Mon Sep 17 00:00:00 2001 From: Farhan <www.mfarvirus@gmail.com> Date: Wed, 21 Oct 2026 00:26:42 +0500 Subject: [PATCH 29/56] fix(compiler): address compile cache review findings - Record each module's import-time reads, so every page that imports a cached helper depends on the files it read, not only the first page. - Re-evaluate every page that defines reflex.istate.dynamic states when any page misses, so collision-suffixed local state names match the backend (manifest flag has_dynamic_states, schema 14). - Fall back to a full compile when recompiled pages bundle libraries the last build's app root does not expose. - Compile inputs edited before the daemon's first snapshot. - Record the inputs each build reflects; backend workers wait while one changed, not only while the compile lock exists. - Test helper _unregister_state drops the reflex.istate.dynamic binding. --- reflex/compiler/disk_cache.py | 77 ++++++++++++-- reflex/compiler/page_cache.py | 63 ++++++++--- reflex/utils/compile_daemon.py | 107 +++++++++++++++++-- tests/units/compiler/test_disk_cache.py | 128 ++++++++++++++++++++++- tests/units/compiler/test_page_cache.py | 31 ++++++ tests/units/utils/test_compile_daemon.py | 72 +++++++++++++ 6 files changed, 446 insertions(+), 32 deletions(-) diff --git a/reflex/compiler/disk_cache.py b/reflex/compiler/disk_cache.py index 47e9587aa3f..97c4cac0b97 100644 --- a/reflex/compiler/disk_cache.py +++ b/reflex/compiler/disk_cache.py @@ -27,6 +27,7 @@ from reflex_base import constants from reflex_base.environment import environment from reflex_base.plugins import CompileContext, CompilerHooks +from reflex_base.registry import RegistrationContext from reflex_base.utils.format import json_dumps from reflex_base.utils.imports import ImportVar, merge_imports @@ -44,7 +45,7 @@ from reflex.compiler import utils as compiler_utils #: Bump when the manifest layout changes (old manifests are then ignored). -_SCHEMA = 13 +_SCHEMA = 14 #: Manifest filename under the web directory. _MANIFEST_FILE = "reflex_compile_cache.json" @@ -159,7 +160,7 @@ def _manifest_page_entry( hasher: Callable[[str], page_cache.FileEntry | None], files: dict[str, page_cache.FileEntry], *, - is_stateful: bool, + defined_states: Sequence[str] | None, state_slice: dict[str, Any] | None = None, root: Path | None = None, ) -> dict[str, Any]: @@ -171,7 +172,8 @@ def _manifest_page_entry( state_index: The state-context identifier -> file index. hasher: A memoized path -> file-entry function. files: The manifest file table, extended with the page's dependencies. - is_stateful: Whether the page registered state during compile. + defined_states: Full names of the states the page defined while it + evaluated, or None for a stateless page. state_slice: The page's contribution to the contexts file (see :func:`_state_slice`), or None for stateless pages. root: Project root for dependency discovery. Defaults to cwd. @@ -179,16 +181,24 @@ def _manifest_page_entry( Returns: The JSON-able manifest entry for the page. """ + import reflex.istate.dynamic as istate_dynamic from reflex.utils.telemetry_accounting import _count_components + base_states = RegistrationContext.ensure_context().base_states return { "component_counts": _count_components([page_ctx.root_component]), "deps": page_cache.page_dependency_entries( page_ctx, component, state_index, hasher, files, root ), "app_wrap_keys": _wrap_key_strs(page_ctx.app_wrap_components.keys()), - "is_stateful": is_stateful, + "is_stateful": defined_states is not None, "state_slice": state_slice, + # Whether the page defined states in ``reflex.istate.dynamic``, whose + # names depend on evaluation order (see ``_with_dynamic_state_pages``). + "has_dynamic_states": any( + base_states[name].__module__ == istate_dynamic.__name__ + for name in defined_states or () + ), # Whether the page contributed auto memos: pages sharing a source # module share one memo output file, so a memo-contributing hit page # must be recompiled alongside a same-module miss (see @@ -328,6 +338,19 @@ def _with_hit_state_slices( return dict(sorted(initial.items())), storage, schema_entries +def _built_bundled_libraries() -> set[str]: + """Read the bundled libraries the last full compile exposed. + + Returns: + The library names, or an empty set when the file is unreadable. + """ + path = prerequisites.get_web_dir() / constants.Dirs.BUNDLED_LIBRARIES + try: + return set(json.loads(path.read_text(encoding="utf-8"))) + except (OSError, ValueError, TypeError): + return set() + + def load_manifest() -> dict[str, Any] | None: """Load the persisted compile manifest, or None if absent/unusable. @@ -435,7 +458,7 @@ def write_manifest( state_index, hasher, files, - is_stateful=defined_states is not None, + defined_states=defined_states, state_slice=( _state_slice(defined_states, *contexts_snapshot) if defined_states and contexts_snapshot is not None @@ -506,8 +529,6 @@ def _imported_state_files(compile_ctx: CompileContext, root: Path | None) -> set Returns: First-party source files affecting import-time state definitions. """ - from reflex_base.registry import RegistrationContext - page_states = { name for names in compile_ctx.stateful_routes.values() for name in names } @@ -653,6 +674,37 @@ def _with_module_siblings( ] +def _with_dynamic_state_pages( + miss_pages: list[PageDefinition], + pages: Sequence[PageDefinition], + manifest: dict[str, Any], +) -> list[PageDefinition]: + """Expand a miss set with every page that defines dynamic-module states. + + States defined in ``reflex.istate.dynamic`` (function-local classes, + ``ComponentState`` instances) take collision-suffixed names in evaluation + order. Evaluating all of them in page order reproduces the names of a full + compile and of the backend, which evaluates every stateful page. + + Args: + miss_pages: The pages to recompile. + pages: All current page definitions (in compile order). + manifest: The loaded manifest. + + Returns: + The expanded miss list, in ``pages`` order. + """ + if not miss_pages: + return miss_pages + miss_routes = {page.route for page in miss_pages} + return [ + page + for page in pages + if page.route in miss_routes + or manifest["pages"][page.route]["has_dynamic_states"] + ] + + def _changed_dependency_files( manifest: dict[str, Any], validator: page_cache.FileValidator, @@ -1051,6 +1103,7 @@ def try_incremental_rebuild( miss_pages, pages, manifest, dirty_memo_paths, stale_owner_routes ) miss_pages = _with_module_siblings(miss_pages, pages, manifest) + miss_pages = _with_dynamic_state_pages(miss_pages, pages, manifest) console.info( f"Compile cache: recompiling {len(miss_pages)}/{len(pages)} pages; " f"changed file(s): {format_path_list(changed_files, resolved_root)}" @@ -1241,6 +1294,14 @@ def try_incremental_rebuild( ) _save_incremental_output(context_path, context_code, outputs) + # Evaluation and state serialization bundle libraries for dynamic + # components; only a full compile rewrites the app root that exposes them. + if miss_ctx is not None and not _built_bundled_libraries().issuperset( + RegistrationContext.ensure_context().bundled_libraries + ): + _log_fallback("recompiled pages bundle new libraries") + return miss_ctx + # The assets copy is cheap, idempotent, and excluded from dependency # tracking entirely, so it is always re-run. assets_src = (root or Path.cwd()) / constants.Dirs.APP_ASSETS @@ -1348,7 +1409,7 @@ def _update_manifest_for_misses( state_index, hasher, manifest["files"], - is_stateful=defined_states is not None, + defined_states=defined_states, state_slice=( _state_slice(defined_states, *contexts_snapshot) if defined_states and contexts_snapshot is not None diff --git a/reflex/compiler/page_cache.py b/reflex/compiler/page_cache.py index 11a9d78fdbc..004a4eae689 100644 --- a/reflex/compiler/page_cache.py +++ b/reflex/compiler/page_cache.py @@ -21,7 +21,7 @@ from functools import lru_cache from importlib import metadata from pathlib import Path -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, TypeVar from reflex_base.environment import environment @@ -141,6 +141,11 @@ def _under_any(path: Path, roots: Iterable[Path]) -> bool: #: None). Cleared by ``enable_read_tracking`` when the recorder root changes. _module_file_cache: dict[object, str | None] = {} _app_import_reads: dict[Path, set[str]] = {} +T = TypeVar("T") + +#: Maps an absolute import name to the files read while importing it under a +#: recorder, so every later importer of the cached module depends on them too. +_module_import_reads: dict[str, set[str]] = {} @contextlib.contextmanager @@ -313,20 +318,49 @@ def _record_imported_modules( result_id = id(result) if (module := sys.modules.get(name)) and id(module) != result_id: _record_module_file(module, target) + target.update(_module_import_reads.get(name, ())) parent = name while "." in parent: parent = parent.rsplit(".", 1)[0] if (module := sys.modules.get(parent)) is not None: _record_module_file(module, target) + target.update(_module_import_reads.get(parent, ())) if not fromlist: return for item in fromlist: if ( item != "*" - and (module := sys.modules.get(f"{name}.{item}")) + and (module := sys.modules.get(submodule := f"{name}.{item}")) and id(module) != result_id ): _record_module_file(module, target) + target.update(_module_import_reads.get(submodule, ())) + + +def _import_recording_reads(name: str, do_import: Callable[[], T]) -> T: + """Run an import under a fresh read set kept as the module's import reads. + + A module executing afresh replaces its recorded reads; importing a loaded + module only adds the reads of submodules it loads for the first time. + + Args: + name: The absolute requested module name. + do_import: Performs the import. + + Returns: + The result of ``do_import``. + """ + executes = name not in sys.modules + reads: set[str] = set() + token = _active_reads.set(reads) + try: + return do_import() + finally: + _active_reads.reset(token) + if executes: + _module_import_reads.pop(name, None) + if reads: + _module_import_reads.setdefault(name, set()).update(reads) def enable_read_tracking(root: Path | None = None) -> None: @@ -391,20 +425,23 @@ def import_( fromlist: Sequence[str] | None = (), level: int = 0, ): - result = orig_import(name, globals_, locals_, fromlist, level) - if (target := _active_reads.get()) is not None: - _record_imported_modules( - _absolute_import_name(name, globals_, level), - result, - target, - fromlist, - ) + if (target := _active_reads.get()) is None: + return orig_import(name, globals_, locals_, fromlist, level) + absolute = _absolute_import_name(name, globals_, level) + result = _import_recording_reads( + absolute, lambda: orig_import(name, globals_, locals_, fromlist, level) + ) + _record_imported_modules(absolute, result, target, fromlist) return result def import_module(name: str, package: str | None = None): - result = orig_import_module(name, package) - if (target := _active_reads.get()) is not None: - _record_imported_modules(result.__name__, result, target) + if (target := _active_reads.get()) is None: + return orig_import_module(name, package) + absolute = importlib.util.resolve_name(name, package) if package else name + result = _import_recording_reads( + absolute, lambda: orig_import_module(name, package) + ) + _record_imported_modules(result.__name__, result, target) return result Path.read_text = read_text # type: ignore[method-assign,assignment] diff --git a/reflex/utils/compile_daemon.py b/reflex/utils/compile_daemon.py index d76d42a49d4..86dcf54a654 100644 --- a/reflex/utils/compile_daemon.py +++ b/reflex/utils/compile_daemon.py @@ -10,6 +10,7 @@ import contextlib import importlib +import json import os import signal import subprocess @@ -436,6 +437,44 @@ def _dependency_snapshot(state: _WatchState) -> dict[Path, int]: return _snapshot(state.watch_paths()) +def _known_snapshot(state: _WatchState, snapshot: dict[Path, int]) -> dict[Path, int]: + """Restrict a snapshot to recorded compile inputs. + + Only their edits are sure to trigger a compile; a backend worker waiting on + any other file could wait for a build that never comes. + + Args: + state: The watch state the snapshot was taken for. + snapshot: ``{path: mtime_ns}`` of the watched inputs. + + Returns: + The entries of ``snapshot`` for ``state.known``. + """ + return {p: m for p, m in snapshot.items() if p in state.known} + + +def _unbuilt_changes(state: _WatchState) -> set[Path]: + """Recorded compile inputs that differ from the manifest on disk. + + Args: + state: The current watch state. + + Returns: + The inputs edited since the last build wrote its manifest. + """ + from reflex.compiler import disk_cache, page_cache + + if (manifest := disk_cache.load_manifest()) is None: + return set() + files = manifest["files"] + validator = page_cache.FileValidator(files) + return { + path + for path in state.known + if str(path) in files and validator.changed(str(path)) + } + + def _missed_changes(before: dict[Path, int], after: dict[Path, int]) -> set[Path]: """Inputs that changed between two dependency snapshots. @@ -596,6 +635,10 @@ def _reset_model_metadata() -> None: #: Give up waiting on the lock after this long (a wedged daemon must not hang #: the backend forever; its own watchdog is ``_COMPILE_TIMEOUT``). _LOCK_WAIT_TIMEOUT = _COMPILE_TIMEOUT + 10 +#: File (under the backend dir) holding ``{path: mtime_ns}`` of the compile +#: inputs the frontend build reflects, so a backend reload worker can tell an +#: edit the daemon has not compiled yet from one it has. +_BUILT_INPUTS_FILE = ".compiled_inputs.json" #: Marker (under the backend dir) holding the pid of the ``reflex run`` that @@ -724,13 +767,54 @@ def _lock_holder_alive(lock: Path) -> bool: return _pid_alive(pid) +def _built_inputs_path() -> Path: + """The built-inputs record under the backend dir. + + Returns: + The record's path. + """ + from reflex.utils import prerequisites + + return prerequisites.get_backend_dir() / _BUILT_INPUTS_FILE + + +def _record_built_inputs(snapshot: dict[Path, int]) -> None: + """Record the compile inputs the frontend build now reflects. + + Args: + snapshot: ``{path: mtime_ns}`` of the inputs the compile read. + """ + path = _built_inputs_path() + path.parent.mkdir(parents=True, exist_ok=True) + tmp = path.with_name(f"{path.name}.{os.getpid()}.tmp") + tmp.write_text(json.dumps({str(p): m for p, m in snapshot.items()})) + tmp.replace(path) + + +def _built_inputs_current() -> bool: + """Whether every recorded compile input still has its built mtime. + + Returns: + True when the build reflects the inputs (or nothing was recorded). + """ + try: + built = json.loads(_built_inputs_path().read_text()) + except FileNotFoundError: + return True + except (OSError, ValueError): + return False + return all(_mtime_ns(Path(p)) == mtime for p, mtime in built.items()) + + def wait_for_compile(timeout: float = _LOCK_WAIT_TIMEOUT) -> None: - """Block while the compile daemon is rewriting ``.web`` for the same change. + """Block until the compile daemon has rewritten ``.web`` for the current inputs. Called by a backend worker that skips frontend compilation - (``REFLEX_SKIP_COMPILE``) before it reads the stateful-pages marker. Returns - immediately when no compile is in progress, when the lock holder died, or - after ``timeout``. + (``REFLEX_SKIP_COMPILE``) before it reads the stateful-pages marker. Waits + while a compile is in progress, and otherwise while a built input changed + since the last compile (the daemon has not picked the edit up yet). + Returns when the build is current, when the lock holder died, or after + ``timeout``. Args: timeout: Maximum seconds to wait. @@ -738,8 +822,11 @@ def wait_for_compile(timeout: float = _LOCK_WAIT_TIMEOUT) -> None: lock = _lock_path() deadline = time.monotonic() + timeout waited = False - while lock.exists() and time.monotonic() < deadline: - if not _lock_holder_alive(lock): + while time.monotonic() < deadline: + if lock.exists(): + if not _lock_holder_alive(lock): + break + elif _built_inputs_current(): break if not waited: console.debug( @@ -975,12 +1062,15 @@ def _serve() -> None: state = _WatchState.build(roots, root) state.checkpoint = _dependency_snapshot(state) console.info("Compile daemon ready (warm); watching for changes.") + # The first snapshot is the watch baseline; edits made since the last build + # wrote the manifest raise no event, so compile them now. + if not (pending := _unbuilt_changes(state)): + _record_built_inputs(_known_snapshot(state, state.checkpoint)) def alive() -> bool: # Never outlive reflex-run: if our parent died we were reparented. return os.getppid() == parent_pid - pending: set[Path] = set() # Changes a failed compile never applied. The manifest only moves on a # successful compile, so they stay in the changed hint until one succeeds; # otherwise the hint's stat-free hit shortcut would reuse their pages. @@ -1014,6 +1104,9 @@ def alive() -> bool: state.roots = roots = _reload_roots() with _compile_lock(root): ok = _compile_once(roots, prerender_routes, changed) + # Also after a failure: workers must not wait on an edit that will + # not build; a fix is a newer edit. + _record_built_inputs(_known_snapshot(state, before)) unapplied = set() if ok else changed if not ok: console.error("Compile failed; keeping the last good build.") diff --git a/tests/units/compiler/test_disk_cache.py b/tests/units/compiler/test_disk_cache.py index 3fdf623476e..1ce3c18bd70 100644 --- a/tests/units/compiler/test_disk_cache.py +++ b/tests/units/compiler/test_disk_cache.py @@ -94,6 +94,7 @@ def _unregister_state(cls: type[BaseState]) -> None: """ from reflex_base.registry import RegistrationContext + import reflex.istate.dynamic as istate_dynamic from reflex.state import all_base_state_classes ctx = RegistrationContext.ensure_context() @@ -103,6 +104,8 @@ def _unregister_state(cls: type[BaseState]) -> None: if parent is not None: ctx.base_state_substates.get(parent.get_full_name(), set()).discard(cls) all_base_state_classes.pop(full_name, None) + if getattr(istate_dynamic, cls.__name__, None) is cls: + delattr(istate_dynamic, cls.__name__) @pytest.mark.parametrize("edited_module", ["state", "helper"]) @@ -500,11 +503,13 @@ def test_write_and_load_manifest(tmp_path, monkeypatch): "is_stateful", "state_slice", "has_memos", + "has_dynamic_states", } # these static pages register no new state and contribute no memos assert entry["is_stateful"] is False assert entry["state_slice"] is None assert entry["has_memos"] is False + assert entry["has_dynamic_states"] is False # every dependency has a file-table entry: [sha256, mtime_ns, size] for dep in entry["deps"]: digest, mtime_ns, size = manifest["files"][dep] @@ -637,6 +642,11 @@ def _stub_externals(app, monkeypatch): monkeypatch.setattr(fs, "update_react_router_config", lambda **k: None) monkeypatch.setattr(fs, "update_entry_client", lambda *a, **k: None) monkeypatch.setattr(fs, "initialize_vite_config", lambda: None) + # The full compile that preceded the rebuild wrote the bundled libraries. + libs_path, libs_code = compiler_utils._compile_bundled_libraries() + libs_file = compiler_utils.resolve_path_of_web_dir(libs_path) + libs_file.parent.mkdir(parents=True, exist_ok=True) + libs_file.write_text(libs_code) def _page_local_state() -> Component: @@ -699,8 +709,6 @@ def test_post_evaluation_fallback_hands_over_the_evaluated_pages(tmp_path, monke """ from reflex_base.registry import RegistrationContext - import reflex.istate.dynamic as istate_dynamic - web = _use_tmp_web_dir(tmp_path, monkeypatch) app = rx.App() app.add_page(_page_a, route="/a") @@ -720,8 +728,6 @@ def forget_local_states() -> None: cls = registry.base_states.get(name) if cls is not None: _unregister_state(cls) - if getattr(istate_dynamic, cls.__name__, None) is cls: - delattr(istate_dynamic, cls.__name__) forget_local_states() try: @@ -1249,6 +1255,120 @@ def test_handler_only_state_edit_rebuilds_contexts(tmp_path, monkeypatch): assert out_path.read_text(encoding="utf-8") == _CONTEXTS_STUB +_COMPONENT_DEFAULT = {"recharts": False} + + +def _page_component_default() -> Component: + def default() -> Component: + if _COMPONENT_DEFAULT["recharts"]: + return rx.recharts.line_chart(rx.recharts.line(data_key="v")) + return rx.el.div("plain") + + state_cls: Any = type( + "_ComponentDefaultState", + (rx.State,), + { + "__annotations__": {"comp": rx.Field[rx.Component]}, + "comp": rx.field(default_factory=default), + "__module__": "fp_mod_x", + }, + ) + _FP_HOLDER["_ComponentDefaultState"] = state_cls + return rx.el.div(rx.box(state_cls.comp)) + + +def test_state_default_from_new_library_needs_a_full_compile(tmp_path, monkeypatch): + """A component default from a library the app root lacks needs a full compile. + + The contexts reference the library through ``window.__reflex``, which only + a rewritten app root and bundled-library file provide. + """ + from reflex.compiler import compiler + + compile_contexts = compiler.compile_contexts + web = _use_tmp_web_dir(tmp_path, monkeypatch) + app = rx.App() + app.add_page(_page_component_default, route="/dyn") + pages = list(app._unevaluated_pages.values()) + ctx = _compile(pages, app=app) + _stub_externals(app, monkeypatch) + disk_cache.write_manifest(ctx, pages, ctx.all_imports, root=tmp_path) + manifest_path = web / disk_cache._MANIFEST_FILE + manifest = json.loads(manifest_path.read_text()) + _stale_dep(manifest, pages[0].route, str(tmp_path / "view.py")) + manifest_path.write_text(json.dumps(manifest)) + _unregister_state(_FP_HOLDER.pop("_ComponentDefaultState")) + monkeypatch.setitem(_COMPONENT_DEFAULT, "recharts", True) + monkeypatch.setattr(compiler, "compile_contexts", compile_contexts) + try: + result = disk_cache.try_incremental_rebuild( + app, compiler_plugins=[], prerender_routes=False, root=tmp_path + ) + finally: + _unregister_state(_FP_HOLDER.pop("_ComponentDefaultState")) + assert isinstance(result, CompileContext) + + +def _page_collide_one() -> Component: + class CollideState(rx.State): + one: int = 1 + + return rx.el.div(rx.el.p(CollideState.one), _footer()) + + +def _page_collide_two() -> Component: + class CollideState(rx.State): + two: int = 2 + + return rx.el.div(rx.el.p(CollideState.two), _footer()) + + +def _unregister_collide_states() -> None: + """Drop every registered ``CollideState``, like the daemon child's reset.""" + for name, cls in list(RegistrationContext.ensure_context().base_states.items()): + if "collide_state" in name: + _unregister_state(cls) + + +def test_incremental_rebuild_keeps_colliding_local_state_names(tmp_path, monkeypatch): + """A page whose local state took a collision suffix keeps it when it misses. + + Local states are named in evaluation order, and the backend evaluates every + stateful page, so the second page's state stays ``CollideState_0`` there. + """ + web = _use_tmp_web_dir(tmp_path, monkeypatch) + app = rx.App() + app.add_page(_page_collide_one, route="/one") + app.add_page(_page_collide_two, route="/two") + pages = list(app._unevaluated_pages.values()) + ctx = _compile(pages, app=app) + two_states = ctx.stateful_routes[pages[1].route] + assert two_states[0].endswith("collide_state_0") + _stub_externals(app, monkeypatch) + disk_cache.write_manifest(ctx, pages, ctx.all_imports, root=tmp_path) + manifest_path = web / disk_cache._MANIFEST_FILE + manifest = json.loads(manifest_path.read_text()) + _stale_dep(manifest, pages[1].route, str(tmp_path / "view.py")) + # Keep the first page out of the miss set, as if it lived in another module. + manifest["pages"][pages[0].route]["has_memos"] = False + manifest_path.write_text(json.dumps(manifest)) + _unregister_collide_states() + + try: + assert ( + disk_cache.try_incremental_rebuild( + app, compiler_plugins=[], prerender_routes=False, root=tmp_path + ) + is True + ) + finally: + _unregister_collide_states() + refreshed = json.loads(manifest_path.read_text()) + assert list(refreshed["pages"][pages[1].route]["state_slice"]["schema"]) == list( + two_states + ) + + def test_incremental_rebuild_copies_assets(tmp_path, monkeypatch): """An assets-only edit is an all-hit rebuild, so the incremental path must run the same assets -> public copy as the full compile. diff --git a/tests/units/compiler/test_page_cache.py b/tests/units/compiler/test_page_cache.py index 8d085f8c72b..64569fc54b9 100644 --- a/tests/units/compiler/test_page_cache.py +++ b/tests/units/compiler/test_page_cache.py @@ -7,6 +7,8 @@ from types import ModuleType from typing import Any, cast +import pytest + from reflex.compiler import page_cache @@ -740,3 +742,32 @@ def test_missing_read_remains_a_dependency(tmp_path, monkeypatch): assert deps == [missing] Path(missing).write_text("{}") assert not page_cache.FileValidator(files).unchanged(deps) + + +@pytest.mark.parametrize("statement", ["import", "import_module"]) +def test_import_time_reads_are_recorded_for_every_importer( + tmp_path, monkeypatch, statement +): + """A module's import-time reads are deps of each page that imports it. + + Only the first page's import executes the module; a later page gets the + cached module and must still depend on what its execution read. + """ + data = tmp_path / "shared_data.json" + data.write_text('{"title": "v1"}') + _prepare_runtime_module(tmp_path, monkeypatch, "review_shared_helper").write_text( + f"from pathlib import Path\nTITLE = Path({str(data)!r}).read_text()\n" + ) + page_cache.enable_read_tracking(tmp_path) + page_reads = [] + try: + for _ in range(2): + with page_cache.record_reads() as reads: + if statement == "import": + exec("import review_shared_helper", {}) + else: + importlib.import_module("review_shared_helper") + page_reads.append(reads) + finally: + _forget_modules("review_shared_helper") + assert [str(data) in reads for reads in page_reads] == [True, True] diff --git a/tests/units/utils/test_compile_daemon.py b/tests/units/utils/test_compile_daemon.py index 6171b6c0f11..c243e86d9b9 100644 --- a/tests/units/utils/test_compile_daemon.py +++ b/tests/units/utils/test_compile_daemon.py @@ -248,6 +248,39 @@ def test_wait_for_compile_respects_lock(tmp_path, monkeypatch): assert time.monotonic() - start < 0.5 +def test_wait_for_compile_waits_for_unbuilt_edits(tmp_path, monkeypatch): + """Without a lock, a worker still waits while a built input has changed. + + The daemon takes its lock only once its watcher reports the edit, so a + worker that restarts first would read the previous stateful-pages marker. + """ + import threading + import time + + monkeypatch.setattr("reflex.utils.prerequisites.get_backend_dir", lambda: tmp_path) + page = tmp_path / "page.py" + page.write_text("v1") + built = compile_daemon._snapshot({page}) + compile_daemon._record_built_inputs(built) + start = time.monotonic() + compile_daemon.wait_for_compile(timeout=1) + assert time.monotonic() - start < 0.2 + mtime = built[page] + 1_000_000_000 + os.utime(page, ns=(mtime, mtime)) + start = time.monotonic() + compile_daemon.wait_for_compile(timeout=0.3) + assert time.monotonic() - start >= 0.3 + # The daemon builds the edit: the worker proceeds. + timer = threading.Timer( + 0.1, compile_daemon._record_built_inputs, [compile_daemon._snapshot({page})] + ) + timer.start() + start = time.monotonic() + compile_daemon.wait_for_compile(timeout=5) + assert 0.1 <= time.monotonic() - start < 1 + timer.join() + + def test_first_party_module_names_caches_file_classification(tmp_path, monkeypatch): """Module files are classified once; a repeat pass resolves no paths.""" import sys @@ -733,3 +766,42 @@ def compile_once(roots, prerender_routes, changed=None): monkeypatch.setattr(compile_daemon, "_compile_once", compile_once) compile_daemon._serve() assert compiles == [{a, b}, {a, b}, {a}] + + +def test_daemon_compiles_an_edit_made_before_its_first_snapshot(tmp_path, monkeypatch): + """An input edited after the last build but before the watch starts compiles. + + The first snapshot is the watch baseline, so such an edit raises no event. + """ + from reflex.compiler import disk_cache, page_cache + from reflex.utils import prerequisites + + page = tmp_path / "page.py" + page.write_text("TITLE = 'v1'\n") + entry = page_cache.file_entry(str(page)) + assert entry is not None + files = {str(page): list(entry)} + page.write_text("TITLE = 'edited before the daemon watched'\n") + monkeypatch.chdir(tmp_path) + monkeypatch.setenv("REFLEX_COMPILE_DAEMON_PRECOMPILED", "1") + monkeypatch.setattr(compile_daemon, "_lock_path", lambda: tmp_path / "lock") + monkeypatch.setattr(compile_daemon, "_reload_roots", lambda: [tmp_path]) + monkeypatch.setattr(compile_daemon, "_prepare_fork_parent", lambda roots: None) + monkeypatch.setattr(compile_daemon, "_next_changes", lambda *args: None) + monkeypatch.setattr( + compile_daemon._WatchState, + "build", + lambda *args: compile_daemon._WatchState([tmp_path], tmp_path, known={page}), + ) + monkeypatch.setattr(prerequisites, "get_app", lambda **kwargs: None) + monkeypatch.setattr(disk_cache, "load_manifest", lambda: {"files": files}) + compiled = [] + monkeypatch.setattr( + compile_daemon, + "_compile_once", + lambda roots, prerender, changed: compiled.append(changed) or True, + ) + monkeypatch.setattr("reflex.utils.prerequisites.get_backend_dir", lambda: tmp_path) + compile_daemon._serve() + assert compiled == [{page}] + assert compile_daemon._built_inputs_current() From 52f16d8d056f16cd88aeca923dc3f793390bebad Mon Sep 17 00:00:00 2001 From: Farhan <www.mfarvirus@gmail.com> Date: Wed, 21 Oct 2026 01:08:46 +0500 Subject: [PATCH 30/56] fix(compile-daemon): own the daemon in the run scope and log through reflex - reflex run owns the daemon through compile_daemon.running(), a context manager around its run scope. A signal to reflex run alone (SIGTERM from an IDE or process manager, SIGINT to the process) used to hang: the daemon was supervised by a pool thread blocked in proc.wait(), which interpreter exit joins before the atexit backstop could stop the daemon. - python -m reflex.utils.compile_daemon runs the importable module, so the daemon's logger is in the reflex tree and no state is split between two module copies. - The daemon and the cache log through logging (log.timing, logger.exception) instead of the deprecated console helpers, so --json output and log levels apply to them. Progress bars use console.progress() again. - The manifest records the project root it describes (schema 15); a copied project falls back to a full compile instead of validating the original project's files. --- reflex/compiler/compiler.py | 22 +----- reflex/compiler/disk_cache.py | 48 ++++++++----- reflex/reflex.py | 19 +++-- reflex/utils/compile_daemon.py | 91 ++++++++++++------------ tests/units/compiler/test_compiler.py | 2 +- tests/units/compiler/test_disk_cache.py | 35 ++++++--- tests/units/test_reflex.py | 12 ++-- tests/units/utils/test_compile_daemon.py | 76 +++++++++++++++++++- 8 files changed, 194 insertions(+), 111 deletions(-) diff --git a/reflex/compiler/compiler.py b/reflex/compiler/compiler.py index ab2cb21d9f1..cc520b61e64 100644 --- a/reflex/compiler/compiler.py +++ b/reflex/compiler/compiler.py @@ -46,7 +46,7 @@ from reflex_components_core.base.app_wrap import AppWrap from reflex_components_core.base.fragment import Fragment from reflex_components_radix.plugin import RadixThemesPlugin -from rich.progress import MofNCompleteColumn, Progress, TimeElapsedColumn +from rich.progress import Progress from reflex.compiler import templates, utils from reflex.compiler.plugins import default_page_plugins @@ -75,24 +75,6 @@ def _set_progress_total( progress.update(task, total=total) -def make_compile_progress(use_rich: bool) -> Progress | console.PoorProgress: - """Build a compile progress bar. - - Args: - use_rich: Whether to use a rich progress bar (else a plain fallback). - - Returns: - A progress bar suitable for tracking a compile. - """ - if use_rich: - return Progress( - *Progress.get_default_columns()[:-1], - MofNCompleteColumn(), - TimeElapsedColumn(), - ) - return console.PoorProgress() - - def _apply_common_imports( imports: dict[str, list[ImportVar]], ): @@ -1533,7 +1515,7 @@ def compile_app( raise return True - progress = make_compile_progress(use_rich) + progress = console.progress() if use_rich else console.PoorProgress() fixed_steps = 7 all_pages = list(app._unevaluated_pages.values()) if evaluated is None: diff --git a/reflex/compiler/disk_cache.py b/reflex/compiler/disk_cache.py index 97c4cac0b97..cf6b0271015 100644 --- a/reflex/compiler/disk_cache.py +++ b/reflex/compiler/disk_cache.py @@ -20,6 +20,7 @@ import dataclasses import hashlib import json +import logging import sys from pathlib import Path from typing import TYPE_CHECKING, Any @@ -28,6 +29,7 @@ from reflex_base.environment import environment from reflex_base.plugins import CompileContext, CompilerHooks from reflex_base.registry import RegistrationContext +from reflex_base.utils import log from reflex_base.utils.format import json_dumps from reflex_base.utils.imports import ImportVar, merge_imports @@ -44,8 +46,10 @@ from reflex.app import App from reflex.compiler import utils as compiler_utils +logger = logging.getLogger(__name__) + #: Bump when the manifest layout changes (old manifests are then ignored). -_SCHEMA = 14 +_SCHEMA = 15 #: Manifest filename under the web directory. _MANIFEST_FILE = "reflex_compile_cache.json" @@ -100,7 +104,7 @@ def _log_fallback(reason: str) -> None: Args: reason: The human-readable fallback reason. """ - console.info(f"Compile cache: falling back to a full compile — {reason}") + logger.info(f"Compile cache: falling back to a full compile — {reason}") _IMPORT_VAR_FIELDS = tuple(f.name for f in dataclasses.fields(ImportVar)) @@ -446,7 +450,7 @@ def write_manifest( or page_ctx.output_code is None or page_ctx.output_path is None ): - console.warn( + logger.warning( f"Compile cache: cannot save manifest because page {page.route!r} " "has incomplete output. The next reload will need a full compile." ) @@ -487,6 +491,9 @@ def write_manifest( manifest = { "schema": _SCHEMA, + # File paths below are absolute: the manifest describes this + # project directory only. + "root": str((root or Path.cwd()).resolve()), "plugin_sources": plugin_sources or {}, "reflex_version": page_cache._reflex_version(), "mode": _compile_mode(), @@ -501,7 +508,7 @@ def write_manifest( } _write(manifest) except Exception as exc: # best-effort: never break the build - console.warn( + logger.warning( f"Compile cache: could not save manifest ({exc!r}). " "The next reload may need a full compile." ) @@ -567,11 +574,15 @@ def globals_mismatch( manifest: The loaded manifest. routes: The current set of page routes. validator: The file validator bound to the manifest's file table. - root: Project root used to shorten paths in the reason. Defaults to cwd. + root: Project root the manifest must describe; also shortens paths in + the reason. Defaults to cwd. Returns: A human-readable mismatch reason, or None when the global inputs match. """ + current_root = str((root or Path.cwd()).resolve()) + if manifest["root"] != current_root: + return f"project root changed ({manifest['root']} -> {current_root})" old_version = manifest.get("reflex_version") if old_version != page_cache._reflex_version(): return ( @@ -1104,17 +1115,18 @@ def try_incremental_rebuild( ) miss_pages = _with_module_siblings(miss_pages, pages, manifest) miss_pages = _with_dynamic_state_pages(miss_pages, pages, manifest) - console.info( + memo_note = ( + f"; rewriting {len(dirty_memo_paths)} memo file(s)" + if dirty_memo_paths + else "" + ) + logger.info( f"Compile cache: recompiling {len(miss_pages)}/{len(pages)} pages; " f"changed file(s): {format_path_list(changed_files, resolved_root)}" - + ( - f"; rewriting {len(dirty_memo_paths)} memo file(s)" - if dirty_memo_paths - else "" - ) + f"{memo_note}" ) else: - console.info(f"Compile cache: reusing all {len(pages)} pages from disk") + logger.info(f"Compile cache: reusing all {len(pages)} pages from disk") miss_routes = {p.route for p in miss_pages} from reflex.compiler import compiler @@ -1126,8 +1138,6 @@ def try_incremental_rebuild( # Recompile only the source-changed pages. miss_ctx = None if miss_pages: - from reflex.compiler.compiler import make_compile_progress - # Page evaluation can import modules the cached import graph (built # pre-evaluation for the memo state) has never seen; drop the cache # afterwards so post-evaluation closures (memo entries, page deps) @@ -1143,7 +1153,7 @@ def try_incremental_rebuild( ) # Progress over the changed pages (evaluate + render each), so a hot # reload shows how much is being recompiled. - progress = make_compile_progress(use_rich) + progress = console.progress() if use_rich else console.PoorProgress() progress.start() task = progress.add_task( "Recompiling changed pages:", total=len(miss_pages) * 2 @@ -1262,12 +1272,12 @@ def try_incremental_rebuild( manifest, miss_ctx, contexts_snapshot ) if changed_route is None: - console.info( + logger.info( "Compile cache: recompiled pages define unchanged states; " "reusing contexts file" ) else: - console.info( + logger.info( f"Compile cache: page {changed_route!r} changed its state " "config; rebuilding contexts" ) @@ -1314,7 +1324,7 @@ def try_incremental_rebuild( # Frontend packages + routing scaffolding (cheap, idempotent). from reflex.utils import frontend_skeleton - with console.timing("Install Frontend Packages"): + with log.timing(logger, "Install Frontend Packages"): app._get_frontend_packages(install_imports) frontend_skeleton.update_react_router_config(prerender_routes=prerender_routes) frontend_skeleton.update_entry_client() @@ -1425,4 +1435,4 @@ def _update_manifest_for_misses( if dirty: _write(manifest) except Exception as exc: # best-effort - console.warn(f"Compile cache: could not refresh manifest ({exc!r}).") + logger.warning(f"Compile cache: could not refresh manifest ({exc!r}).") diff --git a/reflex/reflex.py b/reflex/reflex.py index 0816ad3c21f..d0ed1ecffe1 100644 --- a/reflex/reflex.py +++ b/reflex/reflex.py @@ -6,7 +6,7 @@ import signal import sys from collections.abc import Callable -from contextlib import contextmanager, nullcontext +from contextlib import AbstractContextManager, contextmanager, nullcontext from importlib import import_module from importlib.util import find_spec from pathlib import Path @@ -464,20 +464,18 @@ def _run_dev( )) # The compile daemon owns .web regeneration; backend reload workers only - # evaluate pages to register state. + # evaluate pages to register state. They recognise it through its on-disk + # marker (see ``compile_daemon.owns_compilation``): environment set here is + # not inherited by workers a forkserver started earlier. from reflex_base.environment import environment + compile_daemon_scope: AbstractContextManager[None] = nullcontext() if running_mode.has_frontend() and environment.REFLEX_COMPILE_CACHE.get(): from reflex.utils import compile_daemon - commands.append(( - compile_daemon.run_compile_daemon, - exec.should_prerender_routes(), - True, - )) - # Backend workers are recognised through the daemon's on-disk marker - # (see ``compile_daemon.owns_compilation``): environment set here is - # not inherited by workers a forkserver started earlier. + compile_daemon_scope = compile_daemon.running( + exec.should_prerender_routes(), initial_compiled=True + ) with exec._frontend_process_lock: exec._frontend_shutting_down = False @@ -495,6 +493,7 @@ def stop_frontend() -> None: try: with ( + compile_daemon_scope, _frontend_sigterm_handler(frontend_only and sys.platform != "win32"), processes.run_concurrently_context( *commands, interrupt_on_failure=not frontend_only diff --git a/reflex/utils/compile_daemon.py b/reflex/utils/compile_daemon.py index 86dcf54a654..2f89581988d 100644 --- a/reflex/utils/compile_daemon.py +++ b/reflex/utils/compile_daemon.py @@ -11,19 +11,21 @@ import contextlib import importlib import json +import logging import os import signal import subprocess import sys import threading import time -from collections.abc import Callable +from collections.abc import Callable, Iterator from dataclasses import dataclass, field from pathlib import Path from reflex_base.environment import environment +from reflex_base.utils import log -from reflex.utils import console +logger = logging.getLogger(__name__) #: Fallback polling only (no ``watchfiles``): seconds between poll passes. _POLL_INTERVAL = 0.5 @@ -44,19 +46,26 @@ _SKIP_DIRS = {".web", ".venv", "venv", "node_modules", "__pycache__", ".git"} -def run_compile_daemon( +@contextlib.contextmanager +def running( prerender_routes: bool = False, initial_compiled: bool = False -) -> None: - """Supervise the compile daemon as its own (fork-safe) subprocess. +) -> Iterator[None]: + """Run the compile daemon as a subprocess for the duration of the block. - Runs on a ``reflex run`` worker thread alongside the frontend. Launching the - daemon as a separate process keeps it single-threaded so its per-edit - ``fork()`` is safe, and isolates its environment from the backend (which - checks the daemon ownership marker before frontend compilation). + ``reflex run`` owns the daemon's lifetime: however the block exits (Ctrl-C, + a signal to ``reflex run`` alone, an error), the daemon is stopped and its + ownership marker cleared, so backend workers compile again. The daemon is + its own process so that it stays single-threaded, which keeps its per-edit + ``fork()`` safe, and its environment stays separate from the backend's. If + ``reflex run`` is killed outright, the daemon exits on noticing its parent + is gone (see :func:`_serve`). Args: prerender_routes: Whether the daemon should prerender routes when compiling. initial_compiled: Whether the parent already produced the initial frontend. + + Yields: + None, while the daemon runs. """ env = {**os.environ} # The daemon DOES compile; ensure the cache is on and the skip flag is off. @@ -70,8 +79,10 @@ def run_compile_daemon( proc = subprocess.Popen( [sys.executable, "-m", "reflex.utils.compile_daemon"], env=env ) - - def _terminate() -> None: + try: + mark_daemon_active(proc.pid) + yield + finally: clear_daemon_marker() if proc.poll() is None: proc.terminate() @@ -79,19 +90,7 @@ def _terminate() -> None: proc.wait(timeout=3) except subprocess.TimeoutExpired: proc.kill() - - # Backstop: the daemon must never outlive reflex-run, even if this thread is - # interrupted before its finally runs. The daemon also self-exits if its - # parent dies (see _serve), so the two together prevent an orphan. - import atexit - - atexit.register(_terminate) - try: - mark_daemon_active(proc.pid) - proc.wait() - finally: - _terminate() - atexit.unregister(_terminate) + proc.wait() def _reload_roots() -> list[Path]: @@ -389,7 +388,7 @@ def _next_changes(state: _WatchState, alive: Callable[[], bool]) -> set[Path] | try: import watchfiles except ImportError: - console.warn( + logger.warning( "watchfiles is not installed; the compile daemon falls back to polling." ) return _poll_changes(state, alive) @@ -829,7 +828,7 @@ def wait_for_compile(timeout: float = _LOCK_WAIT_TIMEOUT) -> None: elif _built_inputs_current(): break if not waited: - console.debug( + logger.debug( "Waiting for the compile daemon to finish the current compile." ) waited = True @@ -881,7 +880,7 @@ def _child_compile( t2 = time.perf_counter() app._compile(prerender_routes=prerender_routes, use_rich=True, trigger="hot_reload") t3 = time.perf_counter() - console.info( + logger.info( f"Hot reload {t3 - t0:.2f}s (reset {t1 - t0:.2f}s, " f"reimport {t2 - t1:.2f}s, compile {t3 - t2:.2f}s)" ) @@ -907,7 +906,7 @@ def _await_child(pid: int) -> bool: os.kill(pid, signal.SIGKILL) with contextlib.suppress(OSError): os.waitpid(pid, 0) - console.error("Compile child timed out; killed it, keeping last build.") + logger.error("Compile child timed out; killed it, keeping last build.") return False time.sleep(0.02) @@ -963,7 +962,7 @@ def _can_fork() -> bool: deadline = time.monotonic() + 0.25 while (count := _os_thread_count()) is not None and count > 1: if time.monotonic() > deadline: - console.warn( + logger.warning( f"Compile daemon: {count} OS threads still alive; " "falling back to a cold subprocess." ) @@ -973,7 +972,7 @@ def _can_fork() -> bool: others = [ t.name for t in threading.enumerate() if t is not threading.current_thread() ] - console.warn( + logger.warning( "Compile daemon: cannot fork a warm child while other threads are " f"alive ({', '.join(others)}); falling back to a cold subprocess." ) @@ -1003,9 +1002,7 @@ def _compile_once( try: _child_compile(roots, prerender_routes, changed) except BaseException: # report any failure, never crash the daemon - import traceback - - traceback.print_exc() + logger.exception("Hot reload compile failed.") code = 1 finally: os._exit(code) @@ -1023,7 +1020,7 @@ def _compile_once( env=env, ) except subprocess.TimeoutExpired: - console.error("Compile subprocess timed out; keeping the last good build.") + logger.error("Compile subprocess timed out; keeping the last good build.") return False return proc.returncode == 0 @@ -1042,7 +1039,7 @@ def _serve() -> None: # here (e.g. the app is mid-edit and broken) must NOT kill the daemon; fall # through to the watch loop so the next edit that fixes it recompiles. try: - with _compile_lock(root), console.timing("Compile daemon: initial compile"): + with _compile_lock(root), log.timing(logger, "Compile daemon: initial compile"): if os.environ.pop("REFLEX_COMPILE_DAEMON_PRECOMPILED", None): prerequisites.get_app() else: @@ -1053,15 +1050,12 @@ def _serve() -> None: trigger="initial", ) except BaseException: # tolerate a broken initial state; keep watching - import traceback - - traceback.print_exc() - console.error("Initial compile failed; watching for a fix.") + logger.exception("Initial compile failed; watching for a fix.") _prepare_fork_parent(roots) state = _WatchState.build(roots, root) state.checkpoint = _dependency_snapshot(state) - console.info("Compile daemon ready (warm); watching for changes.") + logger.info("Compile daemon ready (warm); watching for changes.") # The first snapshot is the watch baseline; edits made since the last build # wrote the manifest raise no event, so compile them now. if not (pending := _unbuilt_changes(state)): @@ -1083,7 +1077,7 @@ def alive() -> bool: changed = changed | unapplied from reflex.compiler.disk_cache import format_path_list - console.info( + logger.info( f"Compile daemon: change detected in {format_path_list(map(str, changed), root)}" ) @@ -1091,7 +1085,7 @@ def alive() -> bool: # upgrade) can't be applied to the warm parent (it imported the old # version); re-exec the daemon so the new world is actually loaded. if changed & state.globals_: - console.info("Global config changed; restarting compile daemon.") + logger.info("Global config changed; restarting compile daemon.") os.execv( sys.executable, [sys.executable, "-m", "reflex.utils.compile_daemon"] ) @@ -1109,7 +1103,7 @@ def alive() -> bool: _record_built_inputs(_known_snapshot(state, before)) unapplied = set() if ok else changed if not ok: - console.error("Compile failed; keeping the last good build.") + logger.error("Compile failed; keeping the last good build.") # Refresh what is watched from the new manifest so a newly-referenced # content file becomes watched, and catch up on edits made while the # watcher was down for the compile. @@ -1142,9 +1136,7 @@ def main(argv: list[str] | None = None) -> int: changed, ) except BaseException: # report any failure, never crash - import traceback - - traceback.print_exc() + logger.exception("Hot reload compile failed.") return 1 return 0 try: @@ -1155,4 +1147,9 @@ def main(argv: list[str] | None = None) -> int: if __name__ == "__main__": - sys.exit(main()) + # ``python -m`` executes this file as ``__main__``, a second copy of the + # module. Run the importable module instead, so its logger sits in the + # ``reflex`` tree and no state is split between two copies. + from reflex.utils.compile_daemon import main as _main + + sys.exit(_main()) diff --git a/tests/units/compiler/test_compiler.py b/tests/units/compiler/test_compiler.py index c6a55cc3fd2..d85eb62dffb 100644 --- a/tests/units/compiler/test_compiler.py +++ b/tests/units/compiler/test_compiler.py @@ -1993,7 +1993,7 @@ class FullCompileReached(Exception): """Stop before full-compile side effects.""" mocker.patch.object( - compiler, "make_compile_progress", side_effect=FullCompileReached + compiler.console, "PoorProgress", side_effect=FullCompileReached ) with pytest.raises(FullCompileReached): compiler.compile_app(app, use_rich=False) diff --git a/tests/units/compiler/test_disk_cache.py b/tests/units/compiler/test_disk_cache.py index 1ce3c18bd70..bc2993817b0 100644 --- a/tests/units/compiler/test_disk_cache.py +++ b/tests/units/compiler/test_disk_cache.py @@ -186,12 +186,11 @@ def test_manifest_serializes_state_defaults_like_contexts(tmp_path, monkeypatch, ) -def test_manifest_write_failure_is_visible(tmp_path, monkeypatch): +def test_manifest_write_failure_is_visible(tmp_path, monkeypatch, caplog): """A failed cache write must explain why subsequent reloads compile everything.""" _use_tmp_web_dir(tmp_path, monkeypatch) pages = [_FakePage(route="/a", component=_page_a)] ctx = _compile(pages) - warnings = [] def fail_write(manifest): """Simulate a failure writing the cache manifest. @@ -203,9 +202,9 @@ def fail_write(manifest): raise OSError(msg) monkeypatch.setattr(disk_cache, "_write", fail_write) - monkeypatch.setattr(disk_cache.console, "warn", warnings.append) - disk_cache.write_manifest(ctx, pages, ctx.all_imports, root=tmp_path) - assert any("disk full" in warning for warning in warnings) + with caplog.at_level("WARNING", logger=disk_cache.__name__): + disk_cache.write_manifest(ctx, pages, ctx.all_imports, root=tmp_path) + assert any("disk full" in record.getMessage() for record in caplog.records) def test_serialize_imports_collapses_duplicates(): @@ -236,9 +235,10 @@ def test_wrap_key_strs_is_sorted_and_stable(): ] -def _manifest(pages: dict[str, dict], **overrides) -> dict: +def _manifest(pages: dict[str, dict], root: Path | None = None, **overrides) -> dict: base = { "schema": disk_cache._SCHEMA, + "root": str((root or Path.cwd()).resolve()), "reflex_version": page_cache._reflex_version(), "mode": environment.REFLEX_ENV_MODE.get().value, "files": {}, @@ -273,7 +273,11 @@ def _validator(manifest: dict) -> page_cache.FileValidator: def test_globals_mismatch_names_the_changed_input(tmp_path): - m = _manifest({"/a": {}, "/b": {}}, globals_absent=[str(tmp_path / "rxconfig.py")]) + m = _manifest( + {"/a": {}, "/b": {}}, + root=tmp_path, + globals_absent=[str(tmp_path / "rxconfig.py")], + ) routes = {"/a", "/b"} assert ( disk_cache.globals_mismatch( @@ -299,6 +303,20 @@ def test_globals_mismatch_names_the_changed_input(tmp_path): assert "0.0.0-old" in reason +def test_globals_mismatch_names_a_moved_project(tmp_path): + """A manifest only describes the project directory that wrote it. + + Its file table holds absolute paths, so a copy of the project made with + its ``.web`` would otherwise validate the original project's files. + """ + m = _manifest({"/a": {}}, root=tmp_path / "original") + reason = disk_cache.globals_mismatch( + m, routes={"/a"}, validator=_validator(m), root=tmp_path / "copy" + ) + assert reason is not None + assert "project root changed" in reason + + def test_globals_mismatch_names_a_compile_mode_change(tmp_path, monkeypatch): """A development build's output must never be reused by a production build. @@ -306,7 +324,7 @@ def test_globals_mismatch_names_a_compile_mode_change(tmp_path, monkeypatch): the mode is part of the cache identity. """ monkeypatch.setenv("REFLEX_ENV_MODE", "dev") - m = _manifest({"/a": {}}) + m = _manifest({"/a": {}}, root=tmp_path) assert ( disk_cache.globals_mismatch( m, routes={"/a"}, validator=_validator(m), root=tmp_path @@ -336,6 +354,7 @@ def test_globals_mismatch_validates_stored_inputs_only(tmp_path): assert entry is not None m = _manifest( {"/a": {}}, + root=tmp_path, files={str(theme): entry}, globals={"reflex": page_cache._reflex_version(), f"app:{theme}": str(theme)}, # global files absent from this root at write time and still absent diff --git a/tests/units/test_reflex.py b/tests/units/test_reflex.py index 1ffc66a07c0..e6d9f74c466 100644 --- a/tests/units/test_reflex.py +++ b/tests/units/test_reflex.py @@ -579,13 +579,17 @@ def test_dev_daemon_does_not_disable_backend_fallback(monkeypatch, mocker): concurrent = mocker.patch.object( processes, "run_concurrently_context", return_value=contextlib.nullcontext() ) + daemon = mocker.patch.object( + compile_daemon, "running", return_value=contextlib.nullcontext() + ) reflex._run_dev(constants.RunningMode.FRONTEND_ONLY, None, None, "localhost") compiled.assert_called_once() assert not environment.REFLEX_SKIP_COMPILE.get() - commands = concurrent.call_args.args - assert any( - command[0] is compile_daemon.run_compile_daemon and command[2] is True - for command in commands + # The run scope owns the daemon process; no concurrent task supervises it. + daemon.assert_called_once_with(mocker.ANY, initial_compiled=True) + assert all( + getattr(command[0], "__module__", None) != compile_daemon.__name__ + for command in concurrent.call_args.args ) diff --git a/tests/units/utils/test_compile_daemon.py b/tests/units/utils/test_compile_daemon.py index c243e86d9b9..a75fa6bede8 100644 --- a/tests/units/utils/test_compile_daemon.py +++ b/tests/units/utils/test_compile_daemon.py @@ -1,6 +1,8 @@ """Tests for the warm fork-per-compile dev compile daemon.""" +import contextlib import os +import sys from pathlib import Path import pytest @@ -610,8 +612,49 @@ def fail(*args, **kwargs): raise OSError(message) monkeypatch.setattr(compile_daemon.subprocess, "Popen", fail) - with pytest.raises(OSError, match="spawn failed"): - compile_daemon.run_compile_daemon() + with pytest.raises(OSError, match="spawn failed"), compile_daemon.running(): + pass + assert not marker.exists() + + +@pytest.mark.parametrize("exit_with", [None, KeyboardInterrupt]) +def test_daemon_stops_when_the_run_scope_exits(tmp_path, monkeypatch, exit_with): + """Leaving the ``reflex run`` scope stops the daemon, however the run ends. + + A signal to ``reflex run`` alone (an IDE stop, ``kill``) must not leave the + run waiting on a daemon that never got the signal. + """ + import subprocess + + marker = tmp_path / "daemon.pid" + monkeypatch.setattr(compile_daemon, "_daemon_marker_path", lambda: marker) + popen = subprocess.Popen + spawned: list[subprocess.Popen] = [] + + def spawn_sleeper(args, **kwargs): + """Stand in for the daemon with a process that never exits on its own. + + Args: + args: The daemon command (replaced). + kwargs: The Popen options. + + Returns: + The spawned process. + """ + proc = popen([sys.executable, "-c", "import time; time.sleep(60)"], **kwargs) + spawned.append(proc) + return proc + + monkeypatch.setattr(compile_daemon.subprocess, "Popen", spawn_sleeper) + with ( + contextlib.suppress(KeyboardInterrupt), + compile_daemon.running(), + ): + assert compile_daemon.daemon_active() + if exit_with is not None: + raise exit_with + [proc] = spawned + assert proc.poll() is not None assert not marker.exists() @@ -805,3 +848,32 @@ def test_daemon_compiles_an_edit_made_before_its_first_snapshot(tmp_path, monkey compile_daemon._serve() assert compiled == [{page}] assert compile_daemon._built_inputs_current() + + +def test_daemon_entry_point_runs_the_canonical_module(tmp_path): + """``python -m`` must run the importable module, not a ``__main__`` copy. + + Records go through the ``reflex`` logger tree only from the real module's + logger, and module state must not be split between two copies. + """ + import subprocess + + probe = ( + "import runpy, sys\n" + "import reflex.utils.compile_daemon as daemon\n" + "def main(argv=None):\n" + " print(daemon.logger.name, sys.modules['reflex.utils.compile_daemon'] is daemon)\n" + " return 0\n" + "daemon.main = main\n" + "runpy.run_module('reflex.utils.compile_daemon', run_name='__main__')\n" + ) + result = subprocess.run( + [sys.executable, "-c", probe], + capture_output=True, + text=True, + check=False, + timeout=15, + ) + assert result.stdout.split() == ["reflex.utils.compile_daemon", "True"], ( + result.stderr + ) From bd16ff1731964efb7d905521b165b01a66bdb9c6 Mon Sep 17 00:00:00 2001 From: Farhan <www.mfarvirus@gmail.com> Date: Wed, 21 Oct 2026 01:37:08 +0500 Subject: [PATCH 31/56] fix(compile-daemon): mark daemon ownership with OS locks, not pid files - The daemon holds an OS lock on .web/backend/.compile_daemon for its lifetime, and on .compile_in_progress while it compiles. Workers check whether the lock is held (path_ops.try_lock / path_ops.is_locked) instead of whether a pid written in the file is alive. The OS releases the lock when the holder exits, even when killed, so a stale file, a reused pid or a .web copied from a running project no longer makes workers defer to a foreign process. reflex run no longer writes or clears the marker, and the pid liveness code (including the Windows ctypes probe) is removed. - wait_for_compile stops waiting once the daemon is gone. - compile_app counts its eight fixed progress steps, so the bar ends at its total instead of one past it; a test pins the invariant. --- reflex/compiler/compiler.py | 2 +- reflex/utils/compile_daemon.py | 132 +++++++++-------------- reflex/utils/path_ops.py | 66 ++++++++++++ tests/units/compiler/test_compiler.py | 33 ++++++ tests/units/test_app.py | 13 +-- tests/units/utils/test_compile_daemon.py | 119 ++++++++++---------- tests/units/utils/test_path_ops.py | 43 ++++++++ 7 files changed, 263 insertions(+), 145 deletions(-) diff --git a/reflex/compiler/compiler.py b/reflex/compiler/compiler.py index cc520b61e64..0a01b309d3e 100644 --- a/reflex/compiler/compiler.py +++ b/reflex/compiler/compiler.py @@ -1516,7 +1516,7 @@ def compile_app( return True progress = console.progress() if use_rich else console.PoorProgress() - fixed_steps = 7 + fixed_steps = 8 all_pages = list(app._unevaluated_pages.values()) if evaluated is None: pages_to_compile = all_pages diff --git a/reflex/utils/compile_daemon.py b/reflex/utils/compile_daemon.py index 2f89581988d..aa7a4007304 100644 --- a/reflex/utils/compile_daemon.py +++ b/reflex/utils/compile_daemon.py @@ -25,6 +25,8 @@ from reflex_base.environment import environment from reflex_base.utils import log +from reflex.utils import path_ops + logger = logging.getLogger(__name__) #: Fallback polling only (no ``watchfiles``): seconds between poll passes. @@ -53,8 +55,8 @@ def running( """Run the compile daemon as a subprocess for the duration of the block. ``reflex run`` owns the daemon's lifetime: however the block exits (Ctrl-C, - a signal to ``reflex run`` alone, an error), the daemon is stopped and its - ownership marker cleared, so backend workers compile again. The daemon is + a signal to ``reflex run`` alone, an error), the daemon is stopped, which + releases its lock, so backend workers compile again. The daemon is its own process so that it stays single-threaded, which keeps its per-edit ``fork()`` safe, and its environment stays separate from the backend's. If ``reflex run`` is killed outright, the daemon exits on noticing its parent @@ -80,10 +82,8 @@ def running( [sys.executable, "-m", "reflex.utils.compile_daemon"], env=env ) try: - mark_daemon_active(proc.pid) yield finally: - clear_daemon_marker() if proc.poll() is None: proc.terminate() try: @@ -628,8 +628,9 @@ def _reset_model_metadata() -> None: #: Name of the env var carrying the changed-path hint to a cold compile child. _CHANGED_ENV = "REFLEX_COMPILE_CHANGED" -#: Lock file (under the backend dir) held while a compile is in progress, so a -#: backend reload worker can wait for the fresh stateful-pages marker. +#: Lock file (under the backend dir) whose OS lock the daemon holds while a +#: compile is in progress, so a backend reload worker can wait for the fresh +#: stateful-pages marker. _COMPILE_LOCK_FILE = ".compile_in_progress" #: Give up waiting on the lock after this long (a wedged daemon must not hang #: the backend forever; its own watchdog is ``_COMPILE_TIMEOUT``). @@ -638,14 +639,14 @@ def _reset_model_metadata() -> None: #: inputs the frontend build reflects, so a backend reload worker can tell an #: edit the daemon has not compiled yet from one it has. _BUILT_INPUTS_FILE = ".compiled_inputs.json" - - -#: Marker (under the backend dir) holding the pid of the ``reflex run`` that -#: owns a compile daemon. Backend workers skip frontend compilation while it -#: names a live process. A file, not an environment variable: workers may be -#: forked from a ``forkserver`` whose environment predates ``reflex run`` -#: deciding to start the daemon. -_DAEMON_MARKER_FILE = ".compile_daemon" +#: Lock file (under the backend dir) whose OS lock the daemon holds for its +#: lifetime. Backend workers skip frontend compilation while it is held. A +#: file, not an environment variable: workers may be forked from a +#: ``forkserver`` whose environment predates ``reflex run`` deciding to start +#: the daemon. +_DAEMON_LOCK_FILE = ".compile_daemon" +#: How long a lock holder waits out a worker's momentary probe of the lock. +_LOCK_ACQUIRE_TIMEOUT = 1.0 def _lock_path() -> Path: @@ -654,36 +655,36 @@ def _lock_path() -> Path: return prerequisites.get_backend_dir() / _COMPILE_LOCK_FILE -def _daemon_marker_path() -> Path: +def _daemon_lock_path() -> Path: from reflex.utils import prerequisites - return prerequisites.get_backend_dir() / _DAEMON_MARKER_FILE + return prerequisites.get_backend_dir() / _DAEMON_LOCK_FILE -def mark_daemon_active(pid: int | None = None) -> None: - """Record the process that owns frontend compilation. +def _acquire_lock(path: Path) -> int | None: + """Take the OS lock on ``path``, waiting out other processes' probes of it. Args: - pid: Daemon PID, defaulting to the current process. - """ - marker = _daemon_marker_path() - marker.parent.mkdir(parents=True, exist_ok=True) - marker.write_text(str(os.getpid() if pid is None else pid)) + path: The lock file. - -def clear_daemon_marker() -> None: - """Remove the daemon-ownership marker (on ``reflex run`` exit).""" - _daemon_marker_path().unlink(missing_ok=True) + Returns: + The descriptor holding the lock, or None when another process keeps it. + """ + deadline = time.monotonic() + _LOCK_ACQUIRE_TIMEOUT + while (descriptor := path_ops.try_lock(path)) is None: + if time.monotonic() > deadline: + return None + time.sleep(0.005) + return descriptor def daemon_active() -> bool: - """Whether a live ``reflex run`` owns frontend compilation via the daemon. + """Whether a live compile daemon owns frontend compilation. Returns: - True if the marker names a process that is still alive. + True while a daemon holds its lock. """ - marker = _daemon_marker_path() - return marker.exists() and _lock_holder_alive(marker) + return path_ops.is_locked(_daemon_lock_path()) def owns_compilation() -> bool: @@ -692,7 +693,7 @@ def owns_compilation() -> bool: False for a backend reload worker while a compile daemon owns the build; the daemon (and its forked compile children) carry ``REFLEX_COMPILE_DAEMON`` and always own it. Decided from the on-disk - marker rather than inherited environment, which a ``forkserver`` started + lock rather than inherited environment, which a ``forkserver`` started before ``reflex run`` chose the daemon would not carry. Returns: @@ -716,54 +717,14 @@ def _compile_lock(root: Path): Yields: None. """ - lock = _lock_path() - lock.parent.mkdir(parents=True, exist_ok=True) - lock.write_text(str(os.getpid())) + descriptor = _acquire_lock(_lock_path()) try: yield finally: - lock.unlink(missing_ok=True) - - -def _pid_alive(pid: int) -> bool: - """Whether a process with ``pid`` exists. - - Args: - pid: The process id. - - Returns: - True if the process is alive (or exists but is not ours to signal). - """ - if os.name == "nt": - import ctypes - - kernel32 = ctypes.windll.kernel32 # type: ignore[attr-defined] - # PROCESS_QUERY_LIMITED_INFORMATION; os.kill(pid, 0) would *terminate* - # the process on Windows. OpenProcess still succeeds on an exited - # process while any handle to it stays open (e.g. a parent's Popen), so - # the exit code decides: STILL_ACTIVE (259) means running. - handle = kernel32.OpenProcess(0x1000, False, pid) - if not handle: - return False - exit_code = ctypes.c_ulong() - queried = kernel32.GetExitCodeProcess(handle, ctypes.byref(exit_code)) - kernel32.CloseHandle(handle) - return bool(queried) and exit_code.value == 259 - try: - os.kill(pid, 0) - except ProcessLookupError: - return False - except PermissionError: - return True - return True - - -def _lock_holder_alive(lock: Path) -> bool: - try: - pid = int(lock.read_text().strip()) - except (OSError, ValueError): - return False - return _pid_alive(pid) + # Closing releases the lock; the file stays so every holder and + # waiter uses the same inode. + if descriptor is not None: + os.close(descriptor) def _built_inputs_path() -> Path: @@ -812,7 +773,7 @@ def wait_for_compile(timeout: float = _LOCK_WAIT_TIMEOUT) -> None: (``REFLEX_SKIP_COMPILE``) before it reads the stateful-pages marker. Waits while a compile is in progress, and otherwise while a built input changed since the last compile (the daemon has not picked the edit up yet). - Returns when the build is current, when the lock holder died, or after + Returns when the build is current, when the daemon is gone, or after ``timeout``. Args: @@ -821,11 +782,8 @@ def wait_for_compile(timeout: float = _LOCK_WAIT_TIMEOUT) -> None: lock = _lock_path() deadline = time.monotonic() + timeout waited = False - while time.monotonic() < deadline: - if lock.exists(): - if not _lock_holder_alive(lock): - break - elif _built_inputs_current(): + while time.monotonic() < deadline and daemon_active(): + if not path_ops.is_locked(lock) and _built_inputs_current(): break if not waited: logger.debug( @@ -1139,6 +1097,14 @@ def main(argv: list[str] | None = None) -> int: logger.exception("Hot reload compile failed.") return 1 return 0 + # Held for the process lifetime, so backend workers defer to this daemon + # exactly while it lives (see ``daemon_active``). Forked compile children + # share the lock; ``os.execv`` drops it and the new image retakes it. + if _acquire_lock(_daemon_lock_path()) is None: + logger.warning( + "Another compile daemon owns this project's .web; backend workers " + "defer to it." + ) try: _serve() except KeyboardInterrupt: diff --git a/reflex/utils/path_ops.py b/reflex/utils/path_ops.py index 0fa4a744316..ac4014de831 100644 --- a/reflex/utils/path_ops.py +++ b/reflex/utils/path_ops.py @@ -311,6 +311,72 @@ def _lock_descriptor(descriptor: int) -> None: fcntl.flock(descriptor, fcntl.LOCK_EX) +def _try_lock_descriptor(descriptor: int) -> bool: + """Take the OS's exclusive lock on a lock file without waiting. + + Args: + descriptor: The open lock file descriptor. + + Returns: + Whether the lock was taken. + """ + try: + if sys.platform == "win32": + msvcrt.locking(descriptor, msvcrt.LK_NBLCK, 1) + else: + fcntl.flock(descriptor, fcntl.LOCK_EX | fcntl.LOCK_NB) + except OSError: + return False + return True + + +def try_lock(path: Path) -> int | None: + """Take the OS's exclusive lock on ``path`` without waiting. + + The OS releases the lock when its last descriptor closes, including when + the holder is killed, so a held lock always names a live owner. A pid file + cannot: its process may be gone, or its pid reused by another process. + + Args: + path: The lock file, created if missing. + + Returns: + The descriptor holding the lock (closing it releases the lock), or None + when another holder has it. + """ + path.parent.mkdir(parents=True, exist_ok=True) + descriptor = os.open(path, os.O_CREAT | os.O_RDWR, 0o644) + if _try_lock_descriptor(descriptor): + return descriptor + os.close(descriptor) + return None + + +def is_locked(path: Path) -> bool: + """Whether another descriptor holds the OS lock on ``path``. + + Args: + path: The lock file. + + Returns: + True while a holder has the lock; False when it is free or the file + does not exist. + """ + try: + descriptor = os.open(path, os.O_RDWR) + except FileNotFoundError: + return False + try: + if not _try_lock_descriptor(descriptor): + return True + if sys.platform == "win32": + msvcrt.locking(descriptor, msvcrt.LK_UNLCK, 1) + return False + finally: + # Closing releases a POSIX probe's lock. + os.close(descriptor) + + @contextlib.contextmanager def file_lock(path: str | Path) -> Iterator[Path]: """Lock a file or directory across processes, or proceed unlocked if unavailable. diff --git a/tests/units/compiler/test_compiler.py b/tests/units/compiler/test_compiler.py index d85eb62dffb..f0766b04b7f 100644 --- a/tests/units/compiler/test_compiler.py +++ b/tests/units/compiler/test_compiler.py @@ -2085,3 +2085,36 @@ def test_compile_app_resolves_show_built_with_reflex( assert get_user_tier.called is ( compile_context == constants.CompileContext.DEPLOY and configured is None ) + + +@pytest.mark.parametrize("plugins", [[], [rx.plugins.SitemapPlugin()]]) +def test_compile_app_progress_ends_at_its_total( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, mocker: MockerFixture, plugins +): + """The progress bar finishes exactly at the total it announced. + + Args: + tmp_path: Directory for compiler output. + monkeypatch: Fixture for changing the app directory. + mocker: Fixture for configuring the test app. + plugins: The configured plugins. + """ + monkeypatch.chdir(tmp_path) + poor_progress = compiler.console.PoorProgress + bars: list[compiler.console.PoorProgress] = [] + + def recording_progress() -> compiler.console.PoorProgress: + bar = poor_progress() + bars.append(bar) + return bar + + mocker.patch.object(compiler.console, "PoorProgress", recording_progress) + with RegistrationContext(): + config = rx.Config(app_name="progress_test", plugins=plugins) + mocker.patch("reflex_base.config._get_config", return_value=config) + app = rx.App() + app.add_page(lambda: rx.el.div("hello"), route="/") + compiler.compile_app(app, dry_run=True, use_rich=False) + [bar] = bars + [task] = bar.tasks.values() + assert task["current"] == task["total"] diff --git a/tests/units/test_app.py b/tests/units/test_app.py index 5b8838e4102..9c2e60bbce4 100644 --- a/tests/units/test_app.py +++ b/tests/units/test_app.py @@ -5529,7 +5529,7 @@ def test_compile_emits_stage_spans( def test_should_compile_consumes_the_nocompile_marker_under_a_daemon( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, request: pytest.FixtureRequest ): """A backend worker consumes its ``nocompile`` marker even while a daemon owns .web. @@ -5537,17 +5537,18 @@ def test_should_compile_consumes_the_nocompile_marker_under_a_daemon( child finds it, consumes it and skips the compile, so the first edit never reaches the browser. Daemon processes never touch the marker at all. """ - from reflex.utils import compile_daemon + from reflex.utils import compile_daemon, path_ops web_dir = tmp_path / ".web" web_dir.mkdir() monkeypatch.setattr("reflex.utils.prerequisites.get_web_dir", lambda: web_dir) - monkeypatch.setattr( - compile_daemon, "_daemon_marker_path", lambda: tmp_path / "daemon.pid" - ) + daemon_lock = tmp_path / "daemon.lock" + monkeypatch.setattr(compile_daemon, "_daemon_lock_path", lambda: daemon_lock) monkeypatch.delenv(environment.REFLEX_SKIP_COMPILE.name, raising=False) monkeypatch.delenv(environment.REFLEX_COMPILE_DAEMON.name, raising=False) - compile_daemon.mark_daemon_active() + descriptor = path_ops.try_lock(daemon_lock) + assert descriptor is not None + request.addfinalizer(lambda: os.close(descriptor)) marker = web_dir / constants.NOCOMPILE_FILE marker.touch() app = App() diff --git a/tests/units/utils/test_compile_daemon.py b/tests/units/utils/test_compile_daemon.py index a75fa6bede8..32c33d07a19 100644 --- a/tests/units/utils/test_compile_daemon.py +++ b/tests/units/utils/test_compile_daemon.py @@ -226,28 +226,35 @@ def test_missed_changes_catches_edits_during_compile(tmp_path): def test_wait_for_compile_respects_lock(tmp_path, monkeypatch): - import subprocess - import sys import time + from reflex.utils import path_ops + monkeypatch.setattr("reflex.utils.prerequisites.get_backend_dir", lambda: tmp_path) - # No lock -> returns at once. + monkeypatch.setattr(compile_daemon, "daemon_active", lambda: True) + # No compile in progress -> returns at once. start = time.monotonic() compile_daemon.wait_for_compile(timeout=1) assert time.monotonic() - start < 0.2 - # A live holder -> waits until the lock clears (or the timeout). + # A compile in progress -> waits until it finishes (or the timeout). with compile_daemon._compile_lock(tmp_path): start = time.monotonic() compile_daemon.wait_for_compile(timeout=0.3) assert time.monotonic() - start >= 0.3 - assert not compile_daemon._lock_path().exists() - # A dead holder -> not waited on. - proc = subprocess.Popen([sys.executable, "-c", "pass"]) - proc.wait() - compile_daemon._lock_path().write_text(str(proc.pid)) + # A lock file nobody holds (a compile killed mid-way) is not waited on, + # even when it holds the pid of a live process. + lock = compile_daemon._lock_path() + lock.write_text(str(os.getpid())) + assert not path_ops.is_locked(lock) start = time.monotonic() compile_daemon.wait_for_compile(timeout=5) assert time.monotonic() - start < 0.5 + # A daemon that exits mid-wait releases the worker. + monkeypatch.setattr(compile_daemon, "daemon_active", lambda: False) + with compile_daemon._compile_lock(tmp_path): + start = time.monotonic() + compile_daemon.wait_for_compile(timeout=5) + assert time.monotonic() - start < 0.5 def test_wait_for_compile_waits_for_unbuilt_edits(tmp_path, monkeypatch): @@ -260,6 +267,7 @@ def test_wait_for_compile_waits_for_unbuilt_edits(tmp_path, monkeypatch): import time monkeypatch.setattr("reflex.utils.prerequisites.get_backend_dir", lambda: tmp_path) + monkeypatch.setattr(compile_daemon, "daemon_active", lambda: True) page = tmp_path / "page.py" page.write_text("v1") built = compile_daemon._snapshot({page}) @@ -561,50 +569,39 @@ def test_quiesce_parent_stops_telemetry_worker(): assert telemetry._executor is None -def test_owns_compilation_follows_the_daemon_marker(tmp_path, monkeypatch): - """Backend workers defer to a live daemon; the daemon itself always owns.""" +def test_owns_compilation_follows_the_daemon_lock(tmp_path, monkeypatch): + """Backend workers defer to a daemon holding its lock; the daemon always owns.""" from reflex_base.environment import environment + from reflex.utils import path_ops + monkeypatch.setattr("reflex.utils.prerequisites.get_backend_dir", lambda: tmp_path) monkeypatch.delenv(environment.REFLEX_COMPILE_DAEMON.name, raising=False) - # No marker: a plain process compiles as usual. + # No daemon: a plain process compiles as usual. assert compile_daemon.daemon_active() is False assert compile_daemon.owns_compilation() is True - # A live reflex-run marker: workers stand down. - compile_daemon.mark_daemon_active() - assert compile_daemon.daemon_active() is True - assert compile_daemon.owns_compilation() is False - # ...but the daemon's own processes still own the build. - monkeypatch.setenv(environment.REFLEX_COMPILE_DAEMON.name, "1") - assert compile_daemon.owns_compilation() is True - monkeypatch.delenv(environment.REFLEX_COMPILE_DAEMON.name) - # A marker left by a dead process is ignored. - import subprocess - import sys - - proc = subprocess.Popen([sys.executable, "-c", "pass"]) - proc.wait() - compile_daemon._daemon_marker_path().write_text(str(proc.pid)) + # A lock file nobody holds (a killed daemon, a copied .web) names no owner, + # even when it holds the pid of a live process. + lock = compile_daemon._daemon_lock_path() + lock.write_text(str(os.getpid())) assert compile_daemon.daemon_active() is False + # A daemon holding the lock: workers stand down. + descriptor = path_ops.try_lock(lock) + assert descriptor is not None + try: + assert compile_daemon.daemon_active() is True + assert compile_daemon.owns_compilation() is False + # ...but the daemon's own processes still own the build. + monkeypatch.setenv(environment.REFLEX_COMPILE_DAEMON.name, "1") + assert compile_daemon.owns_compilation() is True + finally: + os.close(descriptor) + monkeypatch.delenv(environment.REFLEX_COMPILE_DAEMON.name) assert compile_daemon.owns_compilation() is True - compile_daemon.clear_daemon_marker() - assert not compile_daemon._daemon_marker_path().exists() -def test_pid_alive_distinguishes_live_and_exited_processes(): - import subprocess - import sys - - assert compile_daemon._pid_alive(os.getpid()) is True - proc = subprocess.Popen([sys.executable, "-c", "pass"]) - proc.wait() - assert compile_daemon._pid_alive(proc.pid) is False - - -def test_failed_daemon_spawn_leaves_no_marker(tmp_path, monkeypatch): - """A failed spawn must leave backend workers owning compilation.""" - marker = tmp_path / "daemon.pid" - monkeypatch.setattr(compile_daemon, "_daemon_marker_path", lambda: marker) +def test_failed_daemon_spawn_propagates(monkeypatch): + """A failed spawn raises out of the run scope, leaving workers in charge.""" def fail(*args, **kwargs): """Simulate failure to launch the child.""" @@ -614,7 +611,6 @@ def fail(*args, **kwargs): monkeypatch.setattr(compile_daemon.subprocess, "Popen", fail) with pytest.raises(OSError, match="spawn failed"), compile_daemon.running(): pass - assert not marker.exists() @pytest.mark.parametrize("exit_with", [None, KeyboardInterrupt]) @@ -626,13 +622,13 @@ def test_daemon_stops_when_the_run_scope_exits(tmp_path, monkeypatch, exit_with) """ import subprocess - marker = tmp_path / "daemon.pid" - monkeypatch.setattr(compile_daemon, "_daemon_marker_path", lambda: marker) + monkeypatch.setattr("reflex.utils.prerequisites.get_backend_dir", lambda: tmp_path) + lock = compile_daemon._daemon_lock_path() popen = subprocess.Popen spawned: list[subprocess.Popen] = [] - def spawn_sleeper(args, **kwargs): - """Stand in for the daemon with a process that never exits on its own. + def spawn_holder(args, **kwargs): + """Stand in for the daemon: hold its lock and never exit on its own. Args: args: The daemon command (replaced). @@ -641,21 +637,34 @@ def spawn_sleeper(args, **kwargs): Returns: The spawned process. """ - proc = popen([sys.executable, "-c", "import time; time.sleep(60)"], **kwargs) + holder = ( + "import time\n" + "from pathlib import Path\n" + "from reflex.utils import path_ops\n" + f"assert path_ops.try_lock(Path({str(lock)!r})) is not None\n" + "print('held', flush=True)\n" + "time.sleep(60)\n" + ) + proc = popen( + [sys.executable, "-c", holder], stdout=subprocess.PIPE, text=True, **kwargs + ) spawned.append(proc) return proc - monkeypatch.setattr(compile_daemon.subprocess, "Popen", spawn_sleeper) + monkeypatch.setattr(compile_daemon.subprocess, "Popen", spawn_holder) with ( contextlib.suppress(KeyboardInterrupt), compile_daemon.running(), ): + [proc] = spawned + assert proc.stdout is not None + assert proc.stdout.readline().strip() == "held" assert compile_daemon.daemon_active() if exit_with is not None: raise exit_with [proc] = spawned assert proc.poll() is not None - assert not marker.exists() + assert compile_daemon.daemon_active() is False def test_watch_state_discovers_new_project_root_sources(tmp_path): @@ -704,7 +713,7 @@ def watch(*args, **kwargs): @pytest.mark.parametrize("warm_only", [False, True]) def test_daemon_initial_work_holds_lock(tmp_path, monkeypatch, warm_only): """Startup either warms or compiles, always under the backend wait lock.""" - from reflex.utils import prerequisites + from reflex.utils import path_ops, prerequisites lock = tmp_path / "compile.lock" monkeypatch.chdir(tmp_path) @@ -721,11 +730,11 @@ def test_daemon_initial_work_holds_lock(tmp_path, monkeypatch, warm_only): def compile_app(**kwargs): """Verify initial compilation blocks backend readers.""" - calls.append(("compile", lock.exists())) + calls.append(("compile", path_ops.is_locked(lock))) def import_app(**kwargs): """Verify warm startup only imports the application.""" - calls.append(("import", lock.exists())) + calls.append(("import", path_ops.is_locked(lock))) monkeypatch.setattr(prerequisites, "get_compiled_app", compile_app) monkeypatch.setattr(prerequisites, "get_app", import_app) @@ -735,7 +744,7 @@ def import_app(**kwargs): monkeypatch.delenv("REFLEX_COMPILE_DAEMON_PRECOMPILED", raising=False) compile_daemon._serve() assert calls == [("import" if warm_only else "compile", True)] - assert not lock.exists() + assert not path_ops.is_locked(lock) assert "REFLEX_COMPILE_DAEMON_PRECOMPILED" not in os.environ diff --git a/tests/units/utils/test_path_ops.py b/tests/units/utils/test_path_ops.py index a9b31d2d84f..cf86bd01498 100644 --- a/tests/units/utils/test_path_ops.py +++ b/tests/units/utils/test_path_ops.py @@ -934,3 +934,46 @@ def test_write_file_updates_changed_file(tmp_path: Path) -> None: write_file(path, "before") write_file(path, "after") assert path.read_text(encoding="utf-8") == "after" + + +def test_try_lock_marks_a_holder_until_it_closes(tmp_path: Path): + """A lock taken without waiting is visible to others until released.""" + lock = tmp_path / "owner.lock" + assert path_ops.is_locked(lock) is False + descriptor = path_ops.try_lock(lock) + assert descriptor is not None + try: + assert path_ops.is_locked(lock) is True + assert path_ops.try_lock(lock) is None + finally: + os.close(descriptor) + assert path_ops.is_locked(lock) is False + + +def test_lock_is_released_when_its_holder_is_killed(tmp_path: Path): + """A killed holder cannot leave a lock behind, unlike a pid file.""" + lock = tmp_path / "owner.lock" + holder = subprocess.Popen( + [ + sys.executable, + "-c", + ( + "import time\n" + "from pathlib import Path\n" + "from reflex.utils import path_ops\n" + f"assert path_ops.try_lock(Path({str(lock)!r})) is not None\n" + "print('held', flush=True)\n" + "time.sleep(60)\n" + ), + ], + stdout=subprocess.PIPE, + text=True, + ) + try: + assert holder.stdout is not None + assert holder.stdout.readline().strip() == "held" + assert path_ops.is_locked(lock) is True + finally: + holder.kill() + holder.wait() + assert path_ops.is_locked(lock) is False From 9f5238082af15ea10b67e9eb2a4278f8a2570f15 Mon Sep 17 00:00:00 2001 From: Farhan <www.mfarvirus@gmail.com> Date: Wed, 21 Oct 2026 01:48:13 +0500 Subject: [PATCH 32/56] perf(compiler): predict dynamic state names, verify, retry exactly on failure Re-evaluating every page that defines reflex.istate.dynamic states on any edit is exact but slow. The manifest now records each page's dynamic name claims (name and requested class, schema 16). A daemon compile child predicts: it re-evaluates only the changed pages, with the recorded names of the other pages reserved, then checks that every re-evaluated page claimed exactly its recorded names. A match proves every page keeps the names a full compile and the backend give it. On a mismatch, or on any fallback after evaluation, the child exits with a retry code and the daemon compiles again in a fresh child with the exact rule. Processes that cannot retry in a fresh process always use the exact rule. --- reflex/compiler/disk_cache.py | 143 ++++++++++++++++++++--- reflex/utils/compile_daemon.py | 86 ++++++++++++-- tests/units/compiler/test_disk_cache.py | 96 ++++++++++++--- tests/units/utils/test_compile_daemon.py | 33 ++++++ 4 files changed, 316 insertions(+), 42 deletions(-) diff --git a/reflex/compiler/disk_cache.py b/reflex/compiler/disk_cache.py index cf6b0271015..97e82fb0994 100644 --- a/reflex/compiler/disk_cache.py +++ b/reflex/compiler/disk_cache.py @@ -49,7 +49,7 @@ logger = logging.getLogger(__name__) #: Bump when the manifest layout changes (old manifests are then ignored). -_SCHEMA = 15 +_SCHEMA = 16 #: Manifest filename under the web directory. _MANIFEST_FILE = "reflex_compile_cache.json" @@ -69,6 +69,36 @@ def set_changed_hint(paths: Iterable[Path | str] | None) -> None: _changed_hint = None if paths is None else {str(Path(p).resolve()) for p in paths} +#: Whether this compile predicts the ``reflex.istate.dynamic`` state names of +#: the pages it does not re-evaluate, instead of re-evaluating every page that +#: defines such states (see :func:`_with_dynamic_state_pages`). Only a caller +#: that can discard this process and compile again sets it. +_predict_dynamic_names = False + + +def set_dynamic_name_prediction(enabled: bool) -> None: + """Predict dynamic state names in this process's compiles. + + A failed prediction raises :class:`DynamicStatePredictionError`; the + caller then compiles again in a fresh process without prediction. + + Args: + enabled: Whether to predict. + """ + global _predict_dynamic_names + _predict_dynamic_names = enabled + + +class DynamicStatePredictionError(Exception): + """A predicted rebuild cannot finish correctly in this process. + + Either a re-evaluated page claimed other ``reflex.istate.dynamic`` names + than recorded, which shifts the names of pages that were not + re-evaluated, or a fallback would evaluate further pages out of page + order. Compile again in a fresh process without prediction. + """ + + def _manifest_path() -> Path: return prerequisites.get_web_dir() / _MANIFEST_FILE @@ -185,10 +215,8 @@ def _manifest_page_entry( Returns: The JSON-able manifest entry for the page. """ - import reflex.istate.dynamic as istate_dynamic from reflex.utils.telemetry_accounting import _count_components - base_states = RegistrationContext.ensure_context().base_states return { "component_counts": _count_components([page_ctx.root_component]), "deps": page_cache.page_dependency_entries( @@ -197,12 +225,9 @@ def _manifest_page_entry( "app_wrap_keys": _wrap_key_strs(page_ctx.app_wrap_components.keys()), "is_stateful": defined_states is not None, "state_slice": state_slice, - # Whether the page defined states in ``reflex.istate.dynamic``, whose - # names depend on evaluation order (see ``_with_dynamic_state_pages``). - "has_dynamic_states": any( - base_states[name].__module__ == istate_dynamic.__name__ - for name in defined_states or () - ), + # The ``reflex.istate.dynamic`` names the page's states took, whose + # suffixes depend on evaluation order (see ``_with_dynamic_state_pages``). + "dynamic_claims": _dynamic_claims(defined_states or ()), # Whether the page contributed auto memos: pages sharing a source # module share one memo output file, so a memo-contributing hit page # must be recompiled alongside a same-module miss (see @@ -685,6 +710,73 @@ def _with_module_siblings( ] +def _dynamic_claims(defined_states: Iterable[str]) -> list[list[str]]: + """The ``reflex.istate.dynamic`` names a page's states took, in order. + + Each name is paired with what was requested: a local state's class name, + or the ``ComponentState`` a per-instance state was created from. A page + making the same claims as recorded took the names a full compile gives it. + + Args: + defined_states: Full names of the states the page defined. + + Returns: + ``[name, requested]`` pairs in definition order. + """ + import reflex.istate.dynamic as istate_dynamic + + base_states = RegistrationContext.ensure_context().base_states + claims = [] + for full_name in defined_states: + cls = base_states[full_name] + if cls.__module__ != istate_dynamic.__name__: + continue + requested = cls.__dict__.get("__original_name__") + if requested is None: + origin = cls.__bases__[0] + requested = f"{origin.__module__}.{origin.__qualname__}" + claims.append([cls.__name__, requested]) + return claims + + +def _claim_recorded_dynamic_names( + manifest: dict[str, Any], miss_routes: set[str] +) -> None: + """Reserve the dynamic state names of the pages that are not re-evaluated. + + A local state takes the first free name for its class name, so with these + reserved a re-evaluated page takes the suffix it has in a full compile + whenever its own claims are unchanged. + + Args: + manifest: The loaded manifest. + miss_routes: Routes re-evaluated in this rebuild. + """ + import reflex.istate.dynamic as istate_dynamic + + for route, entry in manifest["pages"].items(): + if route in miss_routes: + continue + for name, _ in entry["dynamic_claims"]: + if not hasattr(istate_dynamic, name): + setattr(istate_dynamic, name, None) + + +def _fall_back_after_evaluation(reason: str) -> None: + """Fall back after pages were evaluated, or fail a prediction. + + Args: + reason: Why the rebuild falls back. + + Raises: + DynamicStatePredictionError: While predicting dynamic state names, + where the fallback would evaluate pages out of page order. + """ + if _predict_dynamic_names: + raise DynamicStatePredictionError(reason) + _log_fallback(reason) + + def _with_dynamic_state_pages( miss_pages: list[PageDefinition], pages: Sequence[PageDefinition], @@ -695,7 +787,9 @@ def _with_dynamic_state_pages( States defined in ``reflex.istate.dynamic`` (function-local classes, ``ComponentState`` instances) take collision-suffixed names in evaluation order. Evaluating all of them in page order reproduces the names of a full - compile and of the backend, which evaluates every stateful page. + compile and of the backend, which evaluates every stateful page. This is + the exact rule; a compile that can retry in a fresh process predicts the + names instead (see :func:`set_dynamic_name_prediction`). Args: miss_pages: The pages to recompile. @@ -711,8 +805,7 @@ def _with_dynamic_state_pages( return [ page for page in pages - if page.route in miss_routes - or manifest["pages"][page.route]["has_dynamic_states"] + if page.route in miss_routes or manifest["pages"][page.route]["dynamic_claims"] ] @@ -1114,7 +1207,8 @@ def try_incremental_rebuild( miss_pages, pages, manifest, dirty_memo_paths, stale_owner_routes ) miss_pages = _with_module_siblings(miss_pages, pages, manifest) - miss_pages = _with_dynamic_state_pages(miss_pages, pages, manifest) + if not _predict_dynamic_names: + miss_pages = _with_dynamic_state_pages(miss_pages, pages, manifest) memo_note = ( f"; rewriting {len(dirty_memo_paths)} memo file(s)" if dirty_memo_paths @@ -1143,6 +1237,8 @@ def try_incremental_rebuild( # afterwards so post-evaluation closures (memo entries, page deps) # include the newly loaded modules. modules_before = set(page_cache._loaded_first_party_modules(resolved_root)) + if _predict_dynamic_names: + _claim_recorded_dynamic_names(manifest, miss_routes) miss_ctx = CompileContext( app=app, @@ -1177,18 +1273,27 @@ def try_incremental_rebuild( or page_ctx.output_code is None or page_ctx.output_path is None ): - _log_fallback(f"page {page.route!r} produced no output") + _fall_back_after_evaluation(f"page {page.route!r} produced no output") return False entry = manifest["pages"][page.route] if ( _wrap_key_strs(page_ctx.app_wrap_components.keys()) != entry["app_wrap_keys"] ): - _log_fallback(f"page {page.route!r} changed its app-wrap set") + _fall_back_after_evaluation( + f"page {page.route!r} changed its app-wrap set" + ) return miss_ctx if (page.route in miss_ctx.stateful_routes) != entry["is_stateful"]: - _log_fallback(f"page {page.route!r} changed statefulness") + _fall_back_after_evaluation(f"page {page.route!r} changed statefulness") return miss_ctx + if _predict_dynamic_names and ( + _dynamic_claims(miss_ctx.stateful_routes.get(page.route, ())) + != entry["dynamic_claims"] + ): + _fall_back_after_evaluation( + f"page {page.route!r} changed its dynamic state names" + ) # Write changed pages + their memo files; reuse everything else on disk. install_imports = _deserialize_imports(manifest["all_imports"]) @@ -1203,7 +1308,9 @@ def try_incremental_rebuild( output_path = page_ctx.output_path output_code = page_ctx.output_code if output_path is None or output_code is None: - _log_fallback(f"page {page.route!r} lost its output before write") + _fall_back_after_evaluation( + f"page {page.route!r} lost its output before write" + ) return False _save_incremental_output(output_path, output_code, outputs) memo_contributions.update(page_ctx.memo_contributions) @@ -1309,7 +1416,7 @@ def try_incremental_rebuild( if miss_ctx is not None and not _built_bundled_libraries().issuperset( RegistrationContext.ensure_context().bundled_libraries ): - _log_fallback("recompiled pages bundle new libraries") + _fall_back_after_evaluation("recompiled pages bundle new libraries") return miss_ctx # The assets copy is cheap, idempotent, and excluded from dependency diff --git a/reflex/utils/compile_daemon.py b/reflex/utils/compile_daemon.py index aa7a4007304..0595450bc15 100644 --- a/reflex/utils/compile_daemon.py +++ b/reflex/utils/compile_daemon.py @@ -628,6 +628,11 @@ def _reset_model_metadata() -> None: #: Name of the env var carrying the changed-path hint to a cold compile child. _CHANGED_ENV = "REFLEX_COMPILE_CHANGED" +#: Name of the env var telling a cold compile child to predict dynamic state names. +_PREDICT_ENV = "REFLEX_COMPILE_PREDICT_DYNAMIC_NAMES" +#: Exit code of a compile child whose dynamic state name prediction failed; the +#: daemon compiles again in a fresh child without prediction. +_RETRY_EXIT_CODE = 3 #: Lock file (under the backend dir) whose OS lock the daemon holds while a #: compile is in progress, so a backend reload worker can wait for the fresh #: stateful-pages marker. @@ -811,7 +816,10 @@ def _prepare_fork_parent(roots: list[Path]) -> None: def _child_compile( - roots: list[Path], prerender_routes: bool, changed: set[Path] | None = None + roots: list[Path], + prerender_routes: bool, + changed: set[Path] | None = None, + predict: bool = False, ) -> None: """Reset first-party state, re-import the app fresh, and compile incrementally. @@ -824,11 +832,14 @@ def _child_compile( changed: The paths the watcher reported for this reload, when known. Lets the incremental rebuild skip validating pages that depend on none of them. + predict: Whether to predict dynamic state names (see + ``disk_cache.set_dynamic_name_prediction``). """ from reflex.compiler import disk_cache from reflex.utils import prerequisites disk_cache.set_changed_hint(changed) + disk_cache.set_dynamic_name_prediction(predict) # Timed in three steps so every hot reload reports where it spent its time # (resetting state vs re-importing first-party code vs compiling). t0 = time.perf_counter() @@ -844,28 +855,29 @@ def _child_compile( ) -def _await_child(pid: int) -> bool: +def _await_child(pid: int) -> int: """Reap a forked compile child, killing it if it exceeds the watchdog timeout. Args: pid: The forked child's pid. Returns: - True if it exited 0; False on failure, timeout, or signal (a - signal-killed child, such as Ctrl-C during shutdown, is a quiet False). + The child's exit code; 1 on timeout or a signal (a signal-killed + child, such as Ctrl-C during shutdown, is a quiet failure). """ deadline = time.monotonic() + _COMPILE_TIMEOUT while True: done, status = os.waitpid(pid, os.WNOHANG) if done == pid: - return os.waitstatus_to_exitcode(status) == 0 + code = os.waitstatus_to_exitcode(status) + return code if code >= 0 else 1 if time.monotonic() > deadline: with contextlib.suppress(OSError): os.kill(pid, signal.SIGKILL) with contextlib.suppress(OSError): os.waitpid(pid, 0) logger.error("Compile child timed out; killed it, keeping last build.") - return False + return 1 time.sleep(0.02) @@ -942,6 +954,45 @@ def _compile_once( ) -> bool: """Run one incremental compile in an isolated child; report success. + The first child predicts dynamic state names, re-evaluating only the + changed pages. When the prediction fails, a fresh child compiles again + without it, re-evaluating every page that defines such states. + + Args: + roots: The resolved reload roots. + prerender_routes: Whether to prerender routes during compile. + changed: The paths the watcher reported for this reload, when known. + + Returns: + True if the child compiled successfully, else False. + """ + code = _run_compile_child(roots, prerender_routes, changed, predict=True) + if code == _RETRY_EXIT_CODE: + code = _run_compile_child(roots, prerender_routes, changed, predict=False) + return code == 0 + + +def _log_failed_prediction(exc: Exception) -> None: + """Report why a compile child gives up its dynamic state name prediction. + + Args: + exc: The failed prediction. + """ + logger.info( + f"Compile cache: {exc}; compiling again, re-evaluating every page that " + "defines dynamic states" + ) + + +def _run_compile_child( + roots: list[Path], + prerender_routes: bool, + changed: set[Path] | None, + *, + predict: bool, +) -> int: + """Compile in an isolated child process and return its exit code. + Uses a copy-on-write ``fork()`` when safe (warm), else a fresh subprocess (Windows, or when the warm parent is no longer single-threaded). @@ -949,16 +1000,23 @@ def _compile_once( roots: The resolved reload roots. prerender_routes: Whether to prerender routes during compile. changed: The paths the watcher reported for this reload, when known. + predict: Whether the child predicts dynamic state names. Returns: - True if the child compiled successfully, else False. + 0 on success, ``_RETRY_EXIT_CODE`` when the prediction failed, else + a failure code. """ + from reflex.compiler import disk_cache + if _can_fork(): pid = os.fork() if pid == 0: # child code = 0 try: - _child_compile(roots, prerender_routes, changed) + _child_compile(roots, prerender_routes, changed, predict) + except disk_cache.DynamicStatePredictionError as exc: + _log_failed_prediction(exc) + code = _RETRY_EXIT_CODE except BaseException: # report any failure, never crash the daemon logger.exception("Hot reload compile failed.") code = 1 @@ -970,6 +1028,8 @@ def _compile_once( env = {**os.environ} if changed is not None: env[_CHANGED_ENV] = "\n".join(sorted(str(p) for p in changed)) + if predict: + env[_PREDICT_ENV] = "1" try: proc = subprocess.run( [sys.executable, "-m", "reflex.utils.compile_daemon", "--once"], @@ -979,8 +1039,8 @@ def _compile_once( ) except subprocess.TimeoutExpired: logger.error("Compile subprocess timed out; keeping the last good build.") - return False - return proc.returncode == 0 + return 1 + return proc.returncode def _serve() -> None: @@ -1087,12 +1147,18 @@ def main(argv: list[str] | None = None) -> int: if changed_env is not None else None ) + from reflex.compiler import disk_cache + try: _child_compile( _reload_roots(), bool(os.environ.get("REFLEX_PRERENDER_ROUTES")), changed, + bool(os.environ.get(_PREDICT_ENV)), ) + except disk_cache.DynamicStatePredictionError as exc: + _log_failed_prediction(exc) + return _RETRY_EXIT_CODE except BaseException: # report any failure, never crash logger.exception("Hot reload compile failed.") return 1 diff --git a/tests/units/compiler/test_disk_cache.py b/tests/units/compiler/test_disk_cache.py index bc2993817b0..d3098072e71 100644 --- a/tests/units/compiler/test_disk_cache.py +++ b/tests/units/compiler/test_disk_cache.py @@ -522,13 +522,13 @@ def test_write_and_load_manifest(tmp_path, monkeypatch): "is_stateful", "state_slice", "has_memos", - "has_dynamic_states", + "dynamic_claims", } # these static pages register no new state and contribute no memos assert entry["is_stateful"] is False assert entry["state_slice"] is None assert entry["has_memos"] is False - assert entry["has_dynamic_states"] is False + assert entry["dynamic_claims"] == [] # every dependency has a file-table entry: [sha256, mtime_ns, size] for dep in entry["deps"]: digest, mtime_ns, size = manifest["files"][dep] @@ -1328,7 +1328,12 @@ def test_state_default_from_new_library_needs_a_full_compile(tmp_path, monkeypat assert isinstance(result, CompileContext) +_COLLIDE = {"one": 0, "two": 0, "two_extra": False} + + def _page_collide_one() -> Component: + _COLLIDE["one"] += 1 + class CollideState(rx.State): one: int = 1 @@ -1336,9 +1341,17 @@ class CollideState(rx.State): def _page_collide_two() -> Component: + _COLLIDE["two"] += 1 + class CollideState(rx.State): two: int = 2 + def define_extra() -> None: + class CollideState(rx.State): + extra: int = 3 + + if _COLLIDE["two_extra"]: + define_extra() return rx.el.div(rx.el.p(CollideState.two), _footer()) @@ -1349,11 +1362,20 @@ def _unregister_collide_states() -> None: _unregister_state(cls) -def test_incremental_rebuild_keeps_colliding_local_state_names(tmp_path, monkeypatch): - """A page whose local state took a collision suffix keeps it when it misses. +def _rebuild_second_collide_page( + tmp_path, monkeypatch, *, predict: bool, two_extra: bool = False +) -> tuple[Any, list[str], list[str]]: + """Compile two pages defining a same-named local state, then rebuild the second. - Local states are named in evaluation order, and the backend evaluates every - stateful page, so the second page's state stays ``CollideState_0`` there. + Args: + tmp_path: The project root. + monkeypatch: The pytest monkeypatch fixture. + predict: Whether the rebuild predicts dynamic state names. + two_extra: Whether the edited second page defines one more such state. + + Returns: + The rebuild result, the second page's states before the edit, and its + recorded states after the rebuild. """ web = _use_tmp_web_dir(tmp_path, monkeypatch) app = rx.App() @@ -1372,20 +1394,66 @@ def test_incremental_rebuild_keeps_colliding_local_state_names(tmp_path, monkeyp manifest["pages"][pages[0].route]["has_memos"] = False manifest_path.write_text(json.dumps(manifest)) _unregister_collide_states() - + monkeypatch.setitem(_COLLIDE, "one", 0) + monkeypatch.setitem(_COLLIDE, "two", 0) + monkeypatch.setitem(_COLLIDE, "two_extra", two_extra) + monkeypatch.setattr(disk_cache, "_predict_dynamic_names", predict) try: - assert ( - disk_cache.try_incremental_rebuild( - app, compiler_plugins=[], prerender_routes=False, root=tmp_path - ) - is True + result = disk_cache.try_incremental_rebuild( + app, compiler_plugins=[], prerender_routes=False, root=tmp_path ) finally: _unregister_collide_states() + import reflex.istate.dynamic as istate_dynamic + + for name in [n for n in vars(istate_dynamic) if "CollideState" in n]: + delattr(istate_dynamic, name) refreshed = json.loads(manifest_path.read_text()) - assert list(refreshed["pages"][pages[1].route]["state_slice"]["schema"]) == list( - two_states + return ( + result, + list(two_states), + list(refreshed["pages"][pages[1].route]["state_slice"]["schema"]), + ) + + +def test_incremental_rebuild_keeps_colliding_local_state_names(tmp_path, monkeypatch): + """A page whose local state took a collision suffix keeps it when it misses. + + Local states are named in evaluation order, and the backend evaluates every + stateful page, so the second page's state stays ``CollideState_0`` there. + Without prediction every page defining such states is re-evaluated. + """ + result, before, after = _rebuild_second_collide_page( + tmp_path, monkeypatch, predict=False + ) + assert result is True + assert after == before + assert (_COLLIDE["one"], _COLLIDE["two"]) == (1, 1) + + +def test_predicted_local_state_names_rebuild_only_the_changed_page( + tmp_path, monkeypatch +): + """With prediction, a verified rebuild evaluates only the changed page.""" + result, before, after = _rebuild_second_collide_page( + tmp_path, monkeypatch, predict=True ) + assert result is True + assert after == before + assert (_COLLIDE["one"], _COLLIDE["two"]) == (0, 1) + + +def test_unverified_local_state_prediction_asks_for_a_fresh_compile( + tmp_path, monkeypatch +): + """A page that claims different names than recorded fails the prediction. + + The caller then compiles again in a fresh process without prediction. + """ + with pytest.raises(disk_cache.DynamicStatePredictionError): + _rebuild_second_collide_page( + tmp_path, monkeypatch, predict=True, two_extra=True + ) def test_incremental_rebuild_copies_assets(tmp_path, monkeypatch): diff --git a/tests/units/utils/test_compile_daemon.py b/tests/units/utils/test_compile_daemon.py index 32c33d07a19..6832e0c391d 100644 --- a/tests/units/utils/test_compile_daemon.py +++ b/tests/units/utils/test_compile_daemon.py @@ -886,3 +886,36 @@ def test_daemon_entry_point_runs_the_canonical_module(tmp_path): assert result.stdout.split() == ["reflex.utils.compile_daemon", "True"], ( result.stderr ) + + +def test_compile_once_retries_a_failed_prediction_exactly(monkeypatch): + """A failed state-name prediction compiles again in a fresh child, exactly.""" + attempts = [] + + def run_child(roots, prerender_routes, changed, *, predict): + attempts.append(predict) + return compile_daemon._RETRY_EXIT_CODE if predict else 0 + + monkeypatch.setattr(compile_daemon, "_run_compile_child", run_child) + assert compile_daemon._compile_once([], False, None) is True + assert attempts == [True, False] + + +@pytest.mark.skipif(not hasattr(os, "fork"), reason="requires os.fork (POSIX)") +def test_failed_prediction_exits_the_child_with_the_retry_code(monkeypatch): + """The forked child reports a failed prediction distinctly from a failure.""" + from reflex.compiler import disk_cache + + def fail_prediction(roots, prerender_routes, changed, predict): + """Simulate a page claiming other state names than recorded. + + Raises: + DynamicStatePredictionError: Always. + """ + route = "/two" + raise disk_cache.DynamicStatePredictionError(route) + + monkeypatch.setattr(compile_daemon, "_can_fork", lambda: True) + monkeypatch.setattr(compile_daemon, "_child_compile", fail_prediction) + code = compile_daemon._run_compile_child([], False, None, predict=True) + assert code == compile_daemon._RETRY_EXIT_CODE From 6c5457e4b7866c6e20bdba8066d7df73f23a6d04 Mon Sep 17 00:00:00 2001 From: Masen Furer <m_github@0x26.net> Date: Tue, 20 Oct 2026 13:48:05 -0700 Subject: [PATCH 33/56] test: wait for OS lock cleanup after killed holders exit Windows may release byte-range locks asynchronously after process exit. Use the existing bounded polling helper for the two shutdown assertions, while retaining the checks that the child exited and the lock releases. Validated with a 100 ms delayed-release simulation (three failures before, three passes after), 72 affected-module tests, Ruff, and Pyright. --- tests/units/utils/test_compile_daemon.py | 4 +++- tests/units/utils/test_path_ops.py | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/tests/units/utils/test_compile_daemon.py b/tests/units/utils/test_compile_daemon.py index 6832e0c391d..12af310c39a 100644 --- a/tests/units/utils/test_compile_daemon.py +++ b/tests/units/utils/test_compile_daemon.py @@ -9,6 +9,7 @@ import reflex as rx from reflex.compiler import disk_cache +from reflex.testing import AppHarness from reflex.utils import compile_daemon @@ -664,7 +665,8 @@ def spawn_holder(args, **kwargs): raise exit_with [proc] = spawned assert proc.poll() is not None - assert compile_daemon.daemon_active() is False + # Windows can release byte-range locks asynchronously after process exit. + assert AppHarness._poll_for(lambda: not compile_daemon.daemon_active(), timeout=5) def test_watch_state_discovers_new_project_root_sources(tmp_path): diff --git a/tests/units/utils/test_path_ops.py b/tests/units/utils/test_path_ops.py index cf86bd01498..2e2987c4da7 100644 --- a/tests/units/utils/test_path_ops.py +++ b/tests/units/utils/test_path_ops.py @@ -18,6 +18,7 @@ import pytest from reflex import constants +from reflex.testing import AppHarness from reflex.utils import frontend_skeleton, path_ops from reflex.utils.path_ops import write_file @@ -976,4 +977,5 @@ def test_lock_is_released_when_its_holder_is_killed(tmp_path: Path): finally: holder.kill() holder.wait() - assert path_ops.is_locked(lock) is False + # Windows can release byte-range locks asynchronously after process exit. + assert AppHarness._poll_for(lambda: not path_ops.is_locked(lock), timeout=5) From 93b2ce78de9d5fefbb2d86c4b73ff64573816ec1 Mon Sep 17 00:00:00 2001 From: Farhan <www.mfarvirus@gmail.com> Date: Wed, 21 Oct 2026 02:07:53 +0500 Subject: [PATCH 34/56] fix(compiler): let a page gain or lose states without a full compile The app root depends on whether the app has state at all, which rx.App decides, not on which pages define states, and the stateful-pages marker is rewritten on every rebuild. A recompiled page that gained or lost states now only rebuilds the contexts file; dropping a page's states takes its recorded slice out of the contexts. --- reflex/compiler/disk_cache.py | 42 +++++---- tests/units/compiler/test_disk_cache.py | 109 +++++++++++++++++++++++- 2 files changed, 130 insertions(+), 21 deletions(-) diff --git a/reflex/compiler/disk_cache.py b/reflex/compiler/disk_cache.py index 97e82fb0994..3419122dd99 100644 --- a/reflex/compiler/disk_cache.py +++ b/reflex/compiler/disk_cache.py @@ -310,29 +310,35 @@ def _changed_state_config_route( manifest: dict[str, Any], miss_ctx: CompileContext, snapshot: compiler_utils.ContextsSnapshot | None, + miss_routes: Iterable[str], ) -> str | None: - """Find the first recompiled stateful route whose state config changed. + """Find the first recompiled route whose contribution to contexts changed. - Compares each stateful miss page's just-evaluated states against the slice - recorded in the manifest. Only these pages' states can differ from the - on-disk contexts file: hit pages' definitions are unchanged by - construction (all their dependency files still match). + Compares each recompiled page's just-evaluated states against the slice + recorded in the manifest; a page without states contributes no slice, so + gaining or losing states counts as a change. Only these pages' states can + differ from the on-disk contexts file: hit pages' definitions are + unchanged by construction (all their dependency files still match). Args: manifest: The loaded compile manifest. miss_ctx: The compile context of the recompiled pages. snapshot: The ``_contexts_snapshot`` of the app, or None when there is - no state tree to compare against. + no state tree. + miss_routes: The recompiled routes. Returns: The first route requiring a contexts rebuild, or None if none do. """ - if snapshot is None: - return next(iter(miss_ctx.stateful_routes)) - for route, defined_states in miss_ctx.stateful_routes.items(): - stored = manifest["pages"].get(route, {}).get("state_slice") - if _slice_digest(stored) != _slice_digest( + for route in miss_routes: + defined_states = miss_ctx.stateful_routes.get(route) + live = ( _state_slice(defined_states, *snapshot) + if defined_states and snapshot is not None + else None + ) + if _slice_digest(manifest["pages"][route]["state_slice"]) != _slice_digest( + live ): return route return None @@ -1264,8 +1270,8 @@ def try_incremental_rebuild( progress.stop() if set(page_cache._loaded_first_party_modules(resolved_root)) != modules_before: page_cache.clear_import_graph() - # Guard: a miss must not change the app-wrap set or its stateful flag, or - # the reused on-disk app root / state marker would be wrong. + # Guard: a miss must not change the app-wrap set, or the reused on-disk + # app root would be wrong. for page in miss_pages: page_ctx = miss_ctx.compiled_pages.get(page.route) if ( @@ -1284,9 +1290,6 @@ def try_incremental_rebuild( f"page {page.route!r} changed its app-wrap set" ) return miss_ctx - if (page.route in miss_ctx.stateful_routes) != entry["is_stateful"]: - _fall_back_after_evaluation(f"page {page.route!r} changed statefulness") - return miss_ctx if _predict_dynamic_names and ( _dynamic_claims(miss_ctx.stateful_routes.get(page.route, ())) != entry["dynamic_claims"] @@ -1373,10 +1376,13 @@ def try_incremental_rebuild( # from the on-disk contexts file, so compare those first: a content-only # edit leaves them identical and the contexts file is reused untouched. contexts_snapshot: compiler_utils.ContextsSnapshot | None = None - if miss_ctx is not None and miss_ctx.stateful_routes: + if miss_ctx is not None and any( + route in miss_ctx.stateful_routes or manifest["pages"][route]["is_stateful"] + for route in miss_routes + ): contexts_snapshot = _contexts_snapshot(app) changed_route = _changed_state_config_route( - manifest, miss_ctx, contexts_snapshot + manifest, miss_ctx, contexts_snapshot, miss_routes ) if changed_route is None: logger.info( diff --git a/tests/units/compiler/test_disk_cache.py b/tests/units/compiler/test_disk_cache.py index d3098072e71..389dbdcee23 100644 --- a/tests/units/compiler/test_disk_cache.py +++ b/tests/units/compiler/test_disk_cache.py @@ -750,11 +750,11 @@ def forget_local_states() -> None: forget_local_states() try: - # The manifest recorded /s as stateless; its evaluation now flips - # statefulness, which only a full compile can resolve. + # The manifest recorded another app-wrap set for /s; only a full + # compile can rebuild the app root. manifest_path = web / disk_cache._MANIFEST_FILE manifest = json.loads(manifest_path.read_text()) - manifest["pages"][route_s]["is_stateful"] = False + manifest["pages"][route_s]["app_wrap_keys"] = ["99:RecordedWrap"] _stale_dep(manifest, route_s, str(tmp_path / "demo.md")) manifest_path.write_text(json.dumps(manifest)) @@ -768,6 +768,109 @@ def forget_local_states() -> None: forget_local_states() +def _capture_contexts(monkeypatch) -> list[Any]: + """Record the snapshots handed to ``compile_contexts``. + + Args: + monkeypatch: The pytest monkeypatch fixture. + + Returns: + The list the snapshots are appended to. + """ + from reflex.compiler import utils as compiler_utils + + captured: list[Any] = [] + + def fake_compile_contexts(state, theme, *, contexts_snapshot, **kwargs): + captured.append(contexts_snapshot) + return compiler_utils.get_context_path(), _CONTEXTS_STUB + + monkeypatch.setattr( + "reflex.compiler.compiler.compile_contexts", fake_compile_contexts + ) + return captured + + +def _page_flip() -> Component: + return rx.el.div(rx.el.p(_fp_state("_FlipState").value), _footer()) + + +def test_page_becoming_stateful_rebuilds_incrementally(tmp_path, monkeypatch): + """A page that starts defining states needs only its contexts slice. + + The app root depends on whether the app has state at all, which ``rx.App`` + decides, not on which pages define states. + """ + web = _use_tmp_web_dir(tmp_path, monkeypatch) + app = rx.App() + app.add_page(_page_a, route="/a") + app.add_page(_page_flip, route="/s") + pages = list(app._unevaluated_pages.values()) + route_s = pages[1].route + ctx = _compile(pages, app) + defined = ctx.stateful_routes[route_s] + _stub_externals(app, monkeypatch) + disk_cache.write_manifest(ctx, pages, ctx.all_imports, root=tmp_path) + _unregister_state(_FP_HOLDER.pop("_FlipState")) + manifest_path = web / disk_cache._MANIFEST_FILE + manifest = json.loads(manifest_path.read_text()) + # Recorded before the edit that made /s define its state. + manifest["pages"][route_s].update(is_stateful=False, state_slice=None) + _stale_dep(manifest, route_s, str(tmp_path / "view.py")) + manifest_path.write_text(json.dumps(manifest)) + captured = _capture_contexts(monkeypatch) + try: + assert ( + disk_cache.try_incremental_rebuild( + app, compiler_plugins=[], prerender_routes=False, root=tmp_path + ) + is True + ) + finally: + _unregister_state(_FP_HOLDER.pop("_FlipState")) + [(initial, _, _)] = captured + assert set(defined) <= set(initial) + refreshed = json.loads(manifest_path.read_text()) + assert refreshed["pages"][route_s]["is_stateful"] is True + + +def test_page_dropping_its_states_rebuilds_contexts(tmp_path, monkeypatch): + """A page that stops defining states takes its slice out of the contexts.""" + web = _use_tmp_web_dir(tmp_path, monkeypatch) + app = rx.App() + app.add_page(_page_a, route="/a") + pages = list(app._unevaluated_pages.values()) + ctx = _compile(pages, app) + _stub_externals(app, monkeypatch) + disk_cache.write_manifest(ctx, pages, ctx.all_imports, root=tmp_path) + manifest_path = web / disk_cache._MANIFEST_FILE + manifest = json.loads(manifest_path.read_text()) + # Recorded before the edit that removed the page's state. + gone = "reflex___state____state.gone_state" + manifest["pages"]["a"].update( + is_stateful=True, + state_slice={ + "initial_state": {gone: {"value_rx_state_": 0}}, + "client_storage": {}, + "schema": {gone: [gone, [["value", "int"]], [], []]}, + }, + ) + _stale_dep(manifest, "a", str(tmp_path / "view.py")) + manifest_path.write_text(json.dumps(manifest)) + captured = _capture_contexts(monkeypatch) + assert ( + disk_cache.try_incremental_rebuild( + app, compiler_plugins=[], prerender_routes=False, root=tmp_path + ) + is True + ) + [(initial, _, schema_entries)] = captured + assert gone not in initial + assert gone not in schema_entries + refreshed = json.loads(manifest_path.read_text()) + assert refreshed["pages"]["a"]["is_stateful"] is False + + def test_incremental_rebuild_all_hits(tmp_path, monkeypatch): _use_tmp_web_dir(tmp_path, monkeypatch) From 7f62b86465d23bb9a52561f077938ffe6d16c9ca Mon Sep 17 00:00:00 2001 From: Farhan <www.mfarvirus@gmail.com> Date: Wed, 21 Oct 2026 02:22:15 +0500 Subject: [PATCH 35/56] perf(compiler): rebuild the app root from recorded inputs, not a full compile The app root depends on the pages' merged app wraps and on the bundled libraries. Both are now recorded per page in the manifest: each app wrap by a digest of what it contributes, stored once in an interned table after a round-trip check, and every library requested while a page evaluates and renders, plus the requests of each state's initial values and of the root build (record_bundle_requests). A rebuild merges recorded and live wraps the way the full compile's dict.update does and rebuilds the library order in full-compile order. When both are unchanged the root on disk stays; when they changed, the root, its memo files and the library file are rebuilt from the recorded wraps without evaluating unchanged pages. The full compile checks that its library order can be rebuilt from the records and falls back otherwise. compile_app and the rebuild share _build_app_root. --- .../src/reflex_base/components/dynamic.py | 34 ++ .../src/reflex_base/plugins/compiler.py | 24 +- reflex/compiler/compiler.py | 68 ++- reflex/compiler/disk_cache.py | 423 ++++++++++++++++-- tests/units/compiler/test_compiler.py | 31 ++ tests/units/compiler/test_disk_cache.py | 161 ++++++- .../reflex_base/components/test_dynamic.py | 16 + 7 files changed, 670 insertions(+), 87 deletions(-) diff --git a/packages/reflex-base/src/reflex_base/components/dynamic.py b/packages/reflex-base/src/reflex_base/components/dynamic.py index 159eb9277d1..c808579fb39 100644 --- a/packages/reflex-base/src/reflex_base/components/dynamic.py +++ b/packages/reflex-base/src/reflex_base/components/dynamic.py @@ -1,6 +1,9 @@ """Components that are dynamically generated on the backend.""" +import contextlib import dataclasses +from collections.abc import Iterator +from contextvars import ContextVar from typing import TYPE_CHECKING, Any, Union from reflex_base import constants @@ -141,6 +144,35 @@ def _bundle_imports( _bundle_library(library + subpath, explicit=explicit) +#: The list recording bundle requests in the current block, if any. +_bundle_requests: ContextVar[list[str] | None] = ContextVar( + "_bundle_requests", default=None +) + + +@contextlib.contextmanager +def record_bundle_requests() -> Iterator[list[str]]: + """Record every library requested for bundling within the block, in order. + + A request counts even when the library is already bundled: the registry + keeps first occurrences, so its order can be rebuilt from the requests of + each source in compile order. An enclosing recorder receives the requests + too. + + Yields: + The list the requests are appended to. + """ + requests: list[str] = [] + outer = _bundle_requests.get() + token = _bundle_requests.set(requests) + try: + yield requests + finally: + _bundle_requests.reset(token) + if outer is not None: + outer.extend(requests) + + def _bundle_library(library: str, *, explicit: bool = False) -> None: """Register a library, optionally retaining it across compiler resets. @@ -149,6 +181,8 @@ def _bundle_library(library: str, *, explicit: bool = False) -> None: explicit: Whether this registration belongs to the app rather than a compile. """ library = format_library_name(library) + if (requests := _bundle_requests.get()) is not None: + requests.append(library) context = RegistrationContext.ensure_context() if explicit: context._explicit_bundled_libraries[library] = None diff --git a/packages/reflex-base/src/reflex_base/plugins/compiler.py b/packages/reflex-base/src/reflex_base/plugins/compiler.py index ab25c9f706a..3b0a0a7e0eb 100644 --- a/packages/reflex-base/src/reflex_base/plugins/compiler.py +++ b/packages/reflex-base/src/reflex_base/plugins/compiler.py @@ -609,6 +609,9 @@ class PageContext(BaseContext): # The page's unique-name generator state after evaluation, resumed for its # render so its generated names never depend on other pages. unique_name_state: Any = None + # Libraries requested for bundling while the page evaluated and rendered. + evaluate_bundle_requests: list[str] = dataclasses.field(default_factory=list) + render_bundle_requests: list[str] = dataclasses.field(default_factory=list) # Auto-memo components first registered while compiling this page. memo_contributions: dict[tuple[str, str | None], Any] = dataclasses.field( default_factory=dict @@ -701,6 +704,10 @@ class CompileContext(BaseContext): auto_memo_components: dict[tuple[str, str | None], Any] = dataclasses.field( default_factory=dict ) + # The bundled libraries registered when this compile reset the registry, + # before any page evaluated; the compile cache rebuilds the registry's + # order from it and the libraries each page and state requested. + bundle_base: list[str] = dataclasses.field(default_factory=list) def compile( self, @@ -721,6 +728,7 @@ def compile( """ from reflex.compiler import compiler from reflex.state import all_base_state_classes + from reflex_base.components.dynamic import record_bundle_requests from reflex_base.components.memo import registration_owner from reflex_base.vars.base import ( reset_unique_variable_names, @@ -747,7 +755,10 @@ def compile( seed_unique_variable_names(page.route) # Memos registered while this page evaluates (e.g. from modules # imported inside the page function) are owned by this route. - with registration_owner(page.route): + with ( + registration_owner(page.route), + record_bundle_requests() as bundle_requests, + ): if recorder is not None: with recorder() as read_set: page_ctx = self.hooks.eval_page( @@ -785,6 +796,7 @@ def compile( ] page_ctx.unique_name_state = unique_variable_name_state() + page_ctx.evaluate_bundle_requests = bundle_requests self.compiled_pages[page_ctx.route] = page_ctx if evaluate_progress is not None: @@ -797,7 +809,7 @@ def compile( ): memo_before = set(self.auto_memo_components) restore_unique_variable_name_state(page_ctx.unique_name_state) - with page_ctx: + with page_ctx, record_bundle_requests() as render_requests: page_ctx.root_component = self.hooks.compile_component( page_ctx.root_component, page_context=page_ctx, @@ -821,9 +833,11 @@ def compile( self.all_imports, page_ctx.frontend_imports ) self.app_wrap_components.update(page_ctx.app_wrap_components) - page_ctx.output_path, page_ctx.output_code = ( - compiler.compile_page_from_context(page_ctx) - ) + with record_bundle_requests() as output_requests: + page_ctx.output_path, page_ctx.output_code = ( + compiler.compile_page_from_context(page_ctx) + ) + page_ctx.render_bundle_requests = render_requests + output_requests if render_progress is not None: render_progress() diff --git a/reflex/compiler/compiler.py b/reflex/compiler/compiler.py index 0a01b309d3e..c984e536b35 100644 --- a/reflex/compiler/compiler.py +++ b/reflex/compiler/compiler.py @@ -1146,6 +1146,42 @@ def memoized_toast_provider() -> Component: return app_wrappers +def _build_app_root( + app: App, + page_app_wrap_components: dict[tuple[int, str], Component], + compile_ctx: CompileContext, +) -> tuple[Component, str | None]: + """Assemble the app root from the pages' app wraps. + + Memo definitions the root needs (stateful wraps, the hydrate fallback) are + registered on ``compile_ctx.auto_memo_components``. + + Args: + app: The app being compiled. + page_app_wrap_components: The app wraps the pages require. + compile_ctx: The compile context receiving the root's memo definitions. + + Returns: + The app root and the hydrate fallback's export name, if any. + """ + app_wrappers = _resolve_app_wrap_components(app, page_app_wrap_components) + app_root = _memoize_stateful_app_wraps(app._app_root(app_wrappers), compile_ctx) + hydrate_fallback = app._resolve_hydrate_fallback() + hydrate_fallback_export = None + if hydrate_fallback is not None: + hydrate_fallback._add_style_recursive(app.style) + # Compile the fallback through the memo pipeline so it lands in its own + # JS module; root.jsx then re-exports it as HydrateFallback. + hydrate_fallback_definition = create_component_memo( + hydrate_fallback, "hydrate_fallback" + ) + compile_ctx.auto_memo_components[ + hydrate_fallback_definition.export_name, None + ] = hydrate_fallback_definition + hydrate_fallback_export = hydrate_fallback_definition.export_name + return app_root, hydrate_fallback_export + + def _memoize_stateful_app_wraps( app_root: Component, compile_context: CompileContext, @@ -1338,11 +1374,14 @@ def _apply_plugin_outputs( return sources -def _reset_compile_registries(compiler_plugins: Sequence[Plugin]) -> None: +def _reset_compile_registries(compiler_plugins: Sequence[Plugin]) -> list[str]: """Reset the registries a compile fills before any page is evaluated. Args: compiler_plugins: The resolved compiler plugins. + + Returns: + The bundled libraries registered after the reset. """ from reflex_base.components.dynamic import ( _bundle_library, @@ -1360,6 +1399,7 @@ def _reset_compile_registries(compiler_plugins: Sequence[Plugin]) -> None: for plugin in compiler_plugins: for dependency in plugin.get_frontend_dependencies(): _bundle_library(dependency) + return list(RegistrationContext.ensure_context().bundled_libraries) def _read_stateful_pages_marker() -> list[str] | None: @@ -1398,6 +1438,7 @@ def compile_app( ``True`` when a real frontend compile ran, ``False`` when the call short-circuited (backend-only paths that only re-evaluate pages). """ + from reflex_base.components.dynamic import record_bundle_requests from reflex_base.utils.exceptions import ReflexRuntimeError app._apply_decorated_pages() @@ -1520,8 +1561,9 @@ def compile_app( all_pages = list(app._unevaluated_pages.values()) if evaluated is None: pages_to_compile = all_pages - _reset_compile_registries(compiler_plugins) + bundle_base = _reset_compile_registries(compiler_plugins) else: + bundle_base = evaluated.bundle_base # The incremental attempt already did the resets above and bundled what # its pages registered; only the pages it never touched remain. pages_to_compile = [ @@ -1537,6 +1579,7 @@ def compile_app( hooks=CompilerHooks( plugins=default_page_plugins(style=app.style, plugins=compiler_plugins) ), + bundle_base=bundle_base, ) with ( @@ -1597,23 +1640,11 @@ def compile_app( raise ReflexRuntimeError(msg) progress.advance(task) - app_wrappers = _resolve_app_wrap_components(app, compile_ctx.app_wrap_components) - app_root = _memoize_stateful_app_wraps(app._app_root(app_wrappers), compile_ctx) - all_imports = utils.merge_imports(all_imports, app_root._get_all_imports()) - - hydrate_fallback = app._resolve_hydrate_fallback() - hydrate_fallback_export = None - if hydrate_fallback is not None: - hydrate_fallback._add_style_recursive(app.style) - # Compile the fallback through the memo pipeline so it lands in its own - # JS module; root.jsx then re-exports it as HydrateFallback. - hydrate_fallback_definition = create_component_memo( - hydrate_fallback, "hydrate_fallback" + with record_bundle_requests() as root_bundle_requests: + app_root, hydrate_fallback_export = _build_app_root( + app, compile_ctx.app_wrap_components, compile_ctx ) - compile_ctx.auto_memo_components[ - hydrate_fallback_definition.export_name, None - ] = hydrate_fallback_definition - hydrate_fallback_export = hydrate_fallback_definition.export_name + all_imports = utils.merge_imports(all_imports, app_root._get_all_imports()) memo_component_files, memo_components_imports = compile_memo_components( ( @@ -1765,6 +1796,7 @@ def compile_app( all_imports, plugin_sources=plugin_sources, contexts_snapshot=contexts_snapshot, + root_bundle_requests=root_bundle_requests, ) return True diff --git a/reflex/compiler/disk_cache.py b/reflex/compiler/disk_cache.py index 3419122dd99..e075a430d57 100644 --- a/reflex/compiler/disk_cache.py +++ b/reflex/compiler/disk_cache.py @@ -16,16 +16,19 @@ from __future__ import annotations +import base64 import contextlib import dataclasses import hashlib import json import logging +import pickle import sys from pathlib import Path from typing import TYPE_CHECKING, Any from reflex_base import constants +from reflex_base.components.dynamic import record_bundle_requests from reflex_base.environment import environment from reflex_base.plugins import CompileContext, CompilerHooks from reflex_base.registry import RegistrationContext @@ -49,7 +52,7 @@ logger = logging.getLogger(__name__) #: Bump when the manifest layout changes (old manifests are then ignored). -_SCHEMA = 16 +_SCHEMA = 17 #: Manifest filename under the web directory. _MANIFEST_FILE = "reflex_compile_cache.json" @@ -175,16 +178,144 @@ def _deserialize_imports(data: dict[str, list[dict[str, Any]]]) -> ParsedImportD return {lib: [ImportVar(**iv) for iv in ivs] for lib, ivs in data.items()} -def _wrap_key_strs(keys: Any) -> list[str]: - """Render app-wrap ``(priority, name)`` keys as sorted stable strings. +def _wrap_key_str(key: tuple[int, str]) -> str: + """Format an app-wrap key for the manifest. Args: - keys: An iterable of ``(priority, name)`` app-wrap keys. + key: The ``(priority, name)`` app-wrap key. Returns: - A sorted list of ``"priority:name"`` strings. + ``"<priority>:<name>"``. """ - return sorted(f"{p}:{n}" for p, n in keys) + return f"{key[0]}:{key[1]}" + + +def _wrap_key(key_str: str) -> tuple[int, str]: + """Parse an app-wrap key formatted by :func:`_wrap_key_str`. + + Args: + key_str: The formatted key. + + Returns: + The ``(priority, name)`` app-wrap key. + """ + priority, _, name = key_str.partition(":") + return int(priority), name + + +def _wrap_digest(component: Any) -> str: + """Digest what an app wrap contributes to the compiled app root. + + Args: + component: The app-wrap component. + + Returns: + A digest of its render, imports, hooks, custom code, dynamic imports + and the app wraps its own subtree declares. + """ + payload = [ + component.render(), + _serialize_imports(component._get_all_imports()), + list(component._get_all_hooks()), + list(component._get_all_custom_code()), + sorted(component._get_all_dynamic_imports()), + sorted(map(_wrap_key_str, component._get_all_app_wrap_components())), + ] + return hashlib.sha256(json_dumps(payload).encode()).hexdigest() + + +def _stored_wrap(component: Any, digest: str) -> str | None: + """Serialize an app wrap so the app root can be rebuilt without its page. + + Args: + component: The app-wrap component. + digest: Its :func:`_wrap_digest`. + + Returns: + The encoded component, or None when it does not round-trip to the same + contribution. + """ + try: + data = pickle.dumps(component) + if _wrap_digest(pickle.loads(data)) != digest: + return None + except Exception: + return None + return base64.b64encode(data).decode() + + +def _record_app_wraps( + page_ctx: PageContext, wrap_table: dict[str, str | None] +) -> dict[str, str]: + """Record a page's app wraps by digest, interning each in the wrap table. + + Args: + page_ctx: The compiled page context. + wrap_table: The manifest's digest -> stored component table (extended). + + Returns: + The page's app-wrap keys mapped to their digests, in page order. + """ + wraps = {} + for key, component in page_ctx.app_wrap_components.items(): + digest = _wrap_digest(component) + if digest not in wrap_table: + wrap_table[digest] = _stored_wrap(component, digest) + wraps[_wrap_key_str(key)] = digest + return wraps + + +def _state_libraries(initial_state: dict[str, Any]) -> dict[str, list[str]]: + """Record the libraries serializing each state's initial values requests. + + Args: + initial_state: Initial values keyed by state full name. + + Returns: + The requested libraries of each state that requests any. + """ + from reflex.compiler import utils as compiler_utils + + libraries = {} + for name, values in initial_state.items(): + with record_bundle_requests() as requests: + compiler_utils._serialize_initial_state({name: values}) + if requests: + libraries[name] = requests + return libraries + + +def _reconstruct_libraries( + base: Sequence[str], + page_requests: Sequence[Sequence[Sequence[str]]], + root_requests: Sequence[str], + state_libraries: dict[str, list[str]], +) -> list[str]: + """Rebuild the bundled-library registry in full-compile order. + + A full compile evaluates every page, then renders every page, builds the + app root and serializes the states in name order; the registry keeps the + first request of each library. + + Args: + base: The registry after the compile's reset. + page_requests: ``[evaluate, render]`` requests of each page, in page + order. + root_requests: The requests made while building the app root. + state_libraries: The requests of each state's initial values. + + Returns: + The bundled libraries, in registry order. + """ + ordered = [*base] + for evaluate, _ in page_requests: + ordered.extend(evaluate) + for _, render in page_requests: + ordered.extend(render) + ordered.extend(root_requests) + for name in sorted(state_libraries): + ordered.extend(state_libraries[name]) + return list(dict.fromkeys(ordered)) def _manifest_page_entry( @@ -195,6 +326,7 @@ def _manifest_page_entry( files: dict[str, page_cache.FileEntry], *, defined_states: Sequence[str] | None, + wrap_table: dict[str, str | None], state_slice: dict[str, Any] | None = None, root: Path | None = None, ) -> dict[str, Any]: @@ -208,6 +340,8 @@ def _manifest_page_entry( files: The manifest file table, extended with the page's dependencies. defined_states: Full names of the states the page defined while it evaluated, or None for a stateless page. + wrap_table: The manifest's app-wrap table, extended with the page's + wraps. state_slice: The page's contribution to the contexts file (see :func:`_state_slice`), or None for stateless pages. root: Project root for dependency discovery. Defaults to cwd. @@ -222,7 +356,13 @@ def _manifest_page_entry( "deps": page_cache.page_dependency_entries( page_ctx, component, state_index, hasher, files, root ), - "app_wrap_keys": _wrap_key_strs(page_ctx.app_wrap_components.keys()), + # The app wraps the page requires, by digest (see ``_refresh_app_root``). + "app_wraps": _record_app_wraps(page_ctx, wrap_table), + # Libraries requested while evaluating and rendering the page. + "bundle_requests": [ + page_ctx.evaluate_bundle_requests, + page_ctx.render_bundle_requests, + ], "is_stateful": defined_states is not None, "state_slice": state_slice, # The ``reflex.istate.dynamic`` names the page's states took, whose @@ -373,19 +513,6 @@ def _with_hit_state_slices( return dict(sorted(initial.items())), storage, schema_entries -def _built_bundled_libraries() -> set[str]: - """Read the bundled libraries the last full compile exposed. - - Returns: - The library names, or an empty set when the file is unreadable. - """ - path = prerequisites.get_web_dir() / constants.Dirs.BUNDLED_LIBRARIES - try: - return set(json.loads(path.read_text(encoding="utf-8"))) - except (OSError, ValueError, TypeError): - return set() - - def load_manifest() -> dict[str, Any] | None: """Load the persisted compile manifest, or None if absent/unusable. @@ -447,6 +574,7 @@ def write_manifest( *, plugin_sources: dict[str, str] | None = None, contexts_snapshot: compiler_utils.ContextsSnapshot | None = None, + root_bundle_requests: Sequence[str] = (), ) -> None: """Persist a manifest of the just-completed full compile. @@ -465,11 +593,14 @@ def write_manifest( plugin_sources: Unmodified source content for plugin file modifiers. contexts_snapshot: The state-tree inputs used for the contexts output. When absent, a snapshot is captured for direct callers. + root_bundle_requests: The libraries requested while building the app + root. """ try: state_index, _ = page_cache.state_dependency_index(root) hasher = page_cache.make_hasher() files: dict[str, page_cache.FileEntry] = {} + wrap_table: dict[str, str | None] = {} if contexts_snapshot is None and compile_ctx.stateful_routes: contexts_snapshot = _contexts_snapshot(compile_ctx.app) @@ -494,6 +625,7 @@ def write_manifest( hasher, files, defined_states=defined_states, + wrap_table=wrap_table, state_slice=( _state_slice(defined_states, *contexts_snapshot) if defined_states and contexts_snapshot is not None @@ -501,6 +633,23 @@ def write_manifest( ), root=root, ) + state_libraries = ( + _state_libraries(contexts_snapshot[0]) + if contexts_snapshot is not None + else {} + ) + libraries = list(RegistrationContext.ensure_context().bundled_libraries) + reconstructed = _reconstruct_libraries( + compile_ctx.bundle_base, + [pages_data[page.route]["bundle_requests"] for page in pages], + root_bundle_requests, + state_libraries, + ) + if reconstructed != libraries: + logger.debug( + "Compile cache: bundled libraries were requested outside the " + "recorded phases; changes to the app root need a full compile." + ) # Post-evaluation and complete: the full compile evaluated every page, # so both import-owned and route-owned memos are recorded. @@ -536,6 +685,19 @@ def write_manifest( "all_imports": _serialize_imports(install_imports), "pages": pages_data, "memo_files": memo_files, + "wrap_table": wrap_table, + "state_libraries": state_libraries, + # The app root's inputs: merged app wraps (by digest, in insertion + # order) and the bundled libraries (see ``_refresh_app_root``). + "app_root": { + "wraps": { + _wrap_key_str(key): _wrap_digest(component) + for key, component in compile_ctx.app_wrap_components.items() + }, + "libraries": libraries, + "root_requests": list(root_bundle_requests), + "reconstructible": reconstructed == libraries, + }, } _write(manifest) except Exception as exc: # best-effort: never break the build @@ -1231,9 +1393,10 @@ def try_incremental_rebuild( from reflex.compiler import compiler + bundle_base: list[str] = [] if miss_pages or dirty_memo_paths: # Match the full compile's clean bundling/memo state before compiling. - compiler._reset_compile_registries(compiler_plugins) + bundle_base = compiler._reset_compile_registries(compiler_plugins) # Recompile only the source-changed pages. miss_ctx = None @@ -1252,6 +1415,7 @@ def try_incremental_rebuild( hooks=CompilerHooks( plugins=default_page_plugins(style=app.style, plugins=compiler_plugins) ), + bundle_base=bundle_base, ) # Progress over the changed pages (evaluate + render each), so a hot # reload shows how much is being recompiled. @@ -1270,8 +1434,6 @@ def try_incremental_rebuild( progress.stop() if set(page_cache._loaded_first_party_modules(resolved_root)) != modules_before: page_cache.clear_import_graph() - # Guard: a miss must not change the app-wrap set, or the reused on-disk - # app root would be wrong. for page in miss_pages: page_ctx = miss_ctx.compiled_pages.get(page.route) if ( @@ -1282,14 +1444,6 @@ def try_incremental_rebuild( _fall_back_after_evaluation(f"page {page.route!r} produced no output") return False entry = manifest["pages"][page.route] - if ( - _wrap_key_strs(page_ctx.app_wrap_components.keys()) - != entry["app_wrap_keys"] - ): - _fall_back_after_evaluation( - f"page {page.route!r} changed its app-wrap set" - ) - return miss_ctx if _predict_dynamic_names and ( _dynamic_claims(miss_ctx.stateful_routes.get(page.route, ())) != entry["dynamic_claims"] @@ -1376,6 +1530,7 @@ def try_incremental_rebuild( # from the on-disk contexts file, so compare those first: a content-only # edit leaves them identical and the contexts file is reused untouched. contexts_snapshot: compiler_utils.ContextsSnapshot | None = None + state_libraries: dict[str, list[str]] = manifest["state_libraries"] if miss_ctx is not None and any( route in miss_ctx.stateful_routes or manifest["pages"][route]["is_stateful"] for route in miss_routes @@ -1404,26 +1559,39 @@ def try_incremental_rebuild( ), None, ) + merged_snapshot = ( + None + if contexts_snapshot is None + else _with_hit_state_slices(contexts_snapshot, manifest, miss_routes) + ) context_path, context_code = compiler.compile_contexts( - app._state, - theme, - contexts_snapshot=( - None - if contexts_snapshot is None - else _with_hit_state_slices( - contexts_snapshot, manifest, miss_routes - ) - ), + app._state, theme, contexts_snapshot=merged_snapshot ) _save_incremental_output(context_path, context_code, outputs) + if contexts_snapshot is not None and merged_snapshot is not None: + live_libraries = _state_libraries(contexts_snapshot[0]) + state_libraries = { + name: libraries + for name in merged_snapshot[0] + if ( + libraries := live_libraries.get(name) + if name in contexts_snapshot[0] + else state_libraries.get(name) + ) + } - # Evaluation and state serialization bundle libraries for dynamic - # components; only a full compile rewrites the app root that exposes them. - if miss_ctx is not None and not _built_bundled_libraries().issuperset( - RegistrationContext.ensure_context().bundled_libraries - ): - _fall_back_after_evaluation("recompiled pages bundle new libraries") - return miss_ctx + app_root_inputs = manifest["app_root"] + if miss_ctx is not None: + try: + refreshed = _refresh_app_root( + app, manifest, pages, miss_ctx, state_libraries, outputs + ) + except _AppRootUnavailableError as err: + _fall_back_after_evaluation(str(err)) + return miss_ctx + if refreshed is not None: + app_root_inputs, root_imports = refreshed + install_imports = merge_imports(install_imports, root_imports) # The assets copy is cheap, idempotent, and excluded from dependency # tracking entirely, so it is always re-run. @@ -1469,6 +1637,8 @@ def try_incremental_rebuild( contexts_snapshot=contexts_snapshot, memo_state=final_memo_state, memo_file_entries=memo_files_table, + app_root_inputs=app_root_inputs, + state_libraries=state_libraries, validator=validator, ) @@ -1481,6 +1651,143 @@ def try_incremental_rebuild( return True +class _AppRootUnavailableError(Exception): + """The app root cannot be rebuilt from the recorded inputs.""" + + +def _refresh_app_root( + app: App, + manifest: dict[str, Any], + pages: Sequence[PageDefinition], + miss_ctx: CompileContext, + state_libraries: dict[str, list[str]], + outputs: dict[Path, str] | None, +) -> tuple[dict[str, Any], ParsedImportDict] | None: + """Rebuild the app root and bundled-library file when their inputs changed. + + The app root is a function of the pages' merged app wraps (positioned by + the first page requiring each, valued by the last, like the full compile's + ``dict.update``) and of the bundled libraries, rebuilt in full-compile + order from each page's and state's recorded requests. When neither + changed, the root on disk stays. Otherwise it is rebuilt from the + recompiled pages' live wraps and the recorded wraps of the other pages, + without evaluating them. + + Args: + app: The app being compiled. + manifest: The loaded manifest; its wrap table gains the live wraps. + pages: All current page definitions (in compile order). + miss_ctx: The compile context of the recompiled pages. + state_libraries: The libraries each state's initial values request. + outputs: Output mapping supplied by the main compiler, if any. + + Returns: + The new recorded app-root inputs and the imports the root needs, or + None when the root is unchanged. + + Raises: + _AppRootUnavailableError: When the recorded inputs cannot rebuild it. + """ + from reflex_base.components.memo import MEMOS + + from reflex.compiler import compiler + from reflex.compiler import utils as compiler_utils + + stored = manifest["app_root"] + wrap_table = manifest["wrap_table"] + merged: dict[str, str] = {} + providers: dict[str, str] = {} + page_requests = [] + memo_paths: set[str] = set(manifest["memo_files"]) + for page in pages: + entry = manifest["pages"][page.route] + if (page_ctx := miss_ctx.compiled_pages.get(page.route)) is not None: + wraps = _record_app_wraps(page_ctx, wrap_table) + page_requests.append([ + page_ctx.evaluate_bundle_requests, + page_ctx.render_bundle_requests, + ]) + has_memos = bool(page_ctx.memo_contributions) + else: + wraps = entry["app_wraps"] + page_requests.append(entry["bundle_requests"]) + has_memos = entry["has_memos"] + if has_memos and ( + path := _memo_output_path(getattr(page, "_source_module", None)) + ): + memo_paths.add(path) + for key_str, digest in wraps.items(): + merged[key_str] = digest + providers[key_str] = page.route + libraries = _reconstruct_libraries( + miss_ctx.bundle_base, page_requests, stored["root_requests"], state_libraries + ) + if merged == stored["wraps"] and libraries == stored["libraries"]: + return None + if not stored["reconstructible"]: + msg = "the app root's bundled libraries cannot be rebuilt" + raise _AppRootUnavailableError(msg) + + components = {} + for key_str, digest in merged.items(): + key = _wrap_key(key_str) + if (page_ctx := miss_ctx.compiled_pages.get(providers[key_str])) is not None: + components[key] = page_ctx.app_wrap_components[key] + elif (data := wrap_table.get(digest)) is not None: + components[key] = pickle.loads(base64.b64decode(data)) + else: + msg = f"page {providers[key_str]!r} requires an app wrap that cannot be stored" + raise _AppRootUnavailableError(msg) + logger.info( + "Compile cache: the app wraps or bundled libraries changed; rebuilding the app root" + ) + memos_before = set(MEMOS) + root_ctx = CompileContext(app=app, pages=()) + with root_ctx, record_bundle_requests() as root_requests: + app_root, hydrate_fallback_export = compiler._build_app_root( + app, components, root_ctx + ) + root_memos = [ + *(memo for key, memo in MEMOS.items() if key not in memos_before), + *root_ctx.auto_memo_components.values(), + ] + root_memo_paths = { + path + for memo in root_memos + if (path := _memo_output_path(memo.source_module)) is not None + } + if root_memo_paths & memo_paths: + msg = "the app root's memos share a file with page memos" + raise _AppRootUnavailableError(msg) + memo_files, memo_imports = compiler.compile_memo_components([ + *( + memo + for key, memo in MEMOS.items() + if key in memos_before and _memo_output_path(key[1]) in root_memo_paths + ), + *root_memos, + ]) + libraries = _reconstruct_libraries( + miss_ctx.bundle_base, page_requests, root_requests, state_libraries + ) + RegistrationContext.ensure_context().bundled_libraries[:] = libraries + for path, code in ( + *memo_files, + compiler.compile_app_root(app_root, hydrate_fallback_export), + compiler_utils._compile_bundled_libraries(), + ): + _save_incremental_output(path, code, outputs) + return ( + { + "wraps": merged, + "libraries": libraries, + "root_requests": root_requests, + "reconstructible": True, + }, + merge_imports(app_root._get_all_imports(), memo_imports), + ) + + def _update_manifest_for_misses( manifest: dict[str, Any], miss_ctx: CompileContext | None, @@ -1491,6 +1798,8 @@ def _update_manifest_for_misses( contexts_snapshot: compiler_utils.ContextsSnapshot | None = None, memo_state: dict[str, dict[str, Any]], memo_file_entries: dict[str, page_cache.FileEntry], + app_root_inputs: dict[str, Any], + state_libraries: dict[str, list[str]], validator: page_cache.FileValidator | None = None, ) -> None: """Update the on-disk manifest entries for the recompiled pages. @@ -1511,6 +1820,8 @@ def _update_manifest_for_misses( :func:`_memo_state_entries`). memo_file_entries: File-table entries for the memo records' closures, withheld until validation finished. + app_root_inputs: The app root's recorded inputs after this rebuild. + state_libraries: The libraries each state's initial values request. validator: The validator used for this rebuild, for its refreshed entries. """ @@ -1533,6 +1844,7 @@ def _update_manifest_for_misses( hasher, manifest["files"], defined_states=defined_states, + wrap_table=manifest["wrap_table"], state_slice=( _state_slice(defined_states, *contexts_snapshot) if defined_states and contexts_snapshot is not None @@ -1541,6 +1853,23 @@ def _update_manifest_for_misses( root=root, ) dirty = True + if ( + manifest["app_root"] != app_root_inputs + or manifest["state_libraries"] != state_libraries + ): + manifest["app_root"] = app_root_inputs + manifest["state_libraries"] = state_libraries + dirty = True + if dirty: + # Keep only the stored wraps a page or the app root still uses. + used = set(app_root_inputs["wraps"].values()).union( + *(entry["app_wraps"].values() for entry in manifest["pages"].values()) + ) + manifest["wrap_table"] = { + digest: data + for digest, data in manifest["wrap_table"].items() + if digest in used + } # A memo-only rewrite can add packages with no page recompiled. if (imports := _serialize_imports(all_imports)) != manifest["all_imports"]: manifest["all_imports"] = imports diff --git a/tests/units/compiler/test_compiler.py b/tests/units/compiler/test_compiler.py index f0766b04b7f..59a14579055 100644 --- a/tests/units/compiler/test_compiler.py +++ b/tests/units/compiler/test_compiler.py @@ -2118,3 +2118,34 @@ def recording_progress() -> compiler.console.PoorProgress: [bar] = bars [task] = bar.tasks.values() assert task["current"] == task["total"] + + +def test_compile_records_bundle_requests_per_page_and_phase(): + """Each page records the libraries it requested while evaluating and rendering.""" + from reflex_base.plugins import CompileContext, CompilerHooks + + from reflex.compiler.plugins import default_page_plugins + + def page_one() -> rx.Component: + bundle_library("lib-one") + return rx.el.div("one") + + def page_two() -> rx.Component: + bundle_library("lib-one") + bundle_library("lib-two") + return rx.el.div("two") + + with RegistrationContext(): + app = rx.App() + app.add_page(page_one, route="/one") + app.add_page(page_two, route="/two") + ctx = CompileContext( + app=app, + pages=list(app._unevaluated_pages.values()), + hooks=CompilerHooks(plugins=default_page_plugins()), + ) + with ctx: + ctx.compile() + assert ctx.compiled_pages["one"].evaluate_bundle_requests == ["lib-one"] + assert ctx.compiled_pages["two"].evaluate_bundle_requests == ["lib-one", "lib-two"] + assert ctx.compiled_pages["two"].render_bundle_requests == [] diff --git a/tests/units/compiler/test_disk_cache.py b/tests/units/compiler/test_disk_cache.py index 389dbdcee23..490c8638205 100644 --- a/tests/units/compiler/test_disk_cache.py +++ b/tests/units/compiler/test_disk_cache.py @@ -11,6 +11,7 @@ from typing import Any, cast import pytest +from reflex_base import constants from reflex_base.components.component import Component from reflex_base.environment import environment from reflex_base.plugins import CompileContext, CompilerHooks, PageContext @@ -73,10 +74,15 @@ def _page_c() -> Component: def _compile(pages: Sequence[Any], app: Any = None) -> CompileContext: + from reflex.compiler import compiler + + # Reset like a full compile, so the manifest can rebuild the registry. + bundle_base = compiler._reset_compile_registries([]) ctx = CompileContext( app=app, pages=pages, hooks=CompilerHooks(plugins=default_page_plugins()), + bundle_base=bundle_base, ) with ctx: ctx.compile() @@ -226,13 +232,9 @@ def test_serialize_imports_collapses_duplicates(): } -def test_wrap_key_strs_is_sorted_and_stable(): - keys = [(200, "StrictMode"), (0, "AppWrap"), (45, "ColorMode")] - assert disk_cache._wrap_key_strs(keys) == [ - "0:AppWrap", - "200:StrictMode", - "45:ColorMode", - ] +def test_wrap_keys_round_trip_through_the_manifest_format(): + for key in [(200, "StrictMode"), (0, "AppWrap"), (45, "Name:With:Colons")]: + assert disk_cache._wrap_key(disk_cache._wrap_key_str(key)) == key def _manifest(pages: dict[str, dict], root: Path | None = None, **overrides) -> dict: @@ -247,6 +249,14 @@ def _manifest(pages: dict[str, dict], root: Path | None = None, **overrides) -> "all_imports": {}, "pages": pages, "memo_files": {}, + "wrap_table": {}, + "state_libraries": {}, + "app_root": { + "wraps": {}, + "libraries": [], + "root_requests": [], + "reconstructible": True, + }, } base.update(overrides) return base @@ -518,7 +528,8 @@ def test_write_and_load_manifest(tmp_path, monkeypatch): assert set(entry) == { "deps", "component_counts", - "app_wrap_keys", + "app_wraps", + "bundle_requests", "is_stateful", "state_slice", "has_memos", @@ -754,7 +765,10 @@ def forget_local_states() -> None: # compile can rebuild the app root. manifest_path = web / disk_cache._MANIFEST_FILE manifest = json.loads(manifest_path.read_text()) - manifest["pages"][route_s]["app_wrap_keys"] = ["99:RecordedWrap"] + # The app root can no longer be rebuilt from the records, while /s + # now brings different app wraps than recorded. + manifest["app_root"]["reconstructible"] = False + manifest["app_root"]["wraps"] = {} _stale_dep(manifest, route_s, str(tmp_path / "demo.md")) manifest_path.write_text(json.dumps(manifest)) @@ -1399,11 +1413,12 @@ def default() -> Component: return rx.el.div(rx.box(state_cls.comp)) -def test_state_default_from_new_library_needs_a_full_compile(tmp_path, monkeypatch): - """A component default from a library the app root lacks needs a full compile. +def test_state_default_from_new_library_rebuilds_the_app_root(tmp_path, monkeypatch): + """A component default from a new library rewrites the root and library file. - The contexts reference the library through ``window.__reflex``, which only - a rewritten app root and bundled-library file provide. + The contexts reference the library through ``window.__reflex``, which the + app root and bundled-library file provide; both are rebuilt from recorded + requests without evaluating the unchanged pages. """ from reflex.compiler import compiler @@ -1428,7 +1443,115 @@ def test_state_default_from_new_library_needs_a_full_compile(tmp_path, monkeypat ) finally: _unregister_state(_FP_HOLDER.pop("_ComponentDefaultState")) - assert isinstance(result, CompileContext) + assert result is True + libraries = json.loads((web / constants.Dirs.BUNDLED_LIBRARIES).read_text()) + assert "recharts" in libraries + assert "recharts" in _app_root_file(web).read_text() + + +def _app_root_file(web: Path) -> Path: + return web / constants.Dirs.PAGES / constants.PageNames.APP_ROOT + + +class _WrappedDiv(rx.el.Div): + """A div that requires an app-level wrap no other test page provides.""" + + @staticmethod + def _get_app_wrap_components() -> dict[tuple[int, str], Component]: + """Declare the extra app wrap. + + Returns: + The app wraps this component requires. + """ + return {(55, "TestWrap"): rx.el.section(class_name="test-wrap")} + + +_WRAP_EDIT = {"wrap": False, "state": False} + + +def _page_wrap_edit() -> Component: + if _WRAP_EDIT["wrap"]: + return rx.el.div(_WrappedDiv.create("wrapped"), _footer()) + if _WRAP_EDIT["state"]: + return rx.el.div(rx.el.p(_fp_state("_WrapEditState").value), _footer()) + return rx.el.div("plain", _footer()) + + +def _page_state_provider() -> Component: + return rx.el.div(rx.el.p(_fp_state("_ProviderState").value), _footer()) + + +def _root_rebuild(tmp_path, monkeypatch, **edit) -> tuple[Any, Path, str]: + """Fully compile a stateful page and a plain page, then edit the plain one. + + Args: + tmp_path: The project root. + monkeypatch: The pytest monkeypatch fixture. + edit: The ``_WRAP_EDIT`` flags the edit turns on. + + Returns: + The rebuild result, the app root file and its content before the edit. + """ + from reflex_base.components.dynamic import record_bundle_requests + + from reflex.compiler import compiler + + try: + web = _use_tmp_web_dir(tmp_path, monkeypatch) + app = rx.App() + app.add_page(_page_state_provider, route="/s") + app.add_page(_page_wrap_edit, route="/p") + pages = list(app._unevaluated_pages.values()) + ctx = _compile(pages, app=app) + _stub_externals(app, monkeypatch) + with record_bundle_requests() as root_requests: + root_path, root_code = compiler.compile_app_root( + *compiler._build_app_root(app, ctx.app_wrap_components, ctx) + ) + Path(root_path).parent.mkdir(parents=True, exist_ok=True) + Path(root_path).write_text(root_code) + disk_cache.write_manifest( + ctx, + pages, + ctx.all_imports, + root=tmp_path, + root_bundle_requests=root_requests, + ) + manifest_path = web / disk_cache._MANIFEST_FILE + manifest = json.loads(manifest_path.read_text()) + assert manifest["app_root"]["reconstructible"] + _stale_dep(manifest, pages[1].route, str(tmp_path / "view.py")) + manifest_path.write_text(json.dumps(manifest)) + # The daemon child starts without the states its pages define. + _unregister_state(_FP_HOLDER.pop("_ProviderState")) + for flag, value in edit.items(): + monkeypatch.setitem(_WRAP_EDIT, flag, value) + result = disk_cache.try_incremental_rebuild( + app, compiler_plugins=[], prerender_routes=False, root=tmp_path + ) + finally: + for name in ("_ProviderState", "_WrapEditState"): + if (cls := _FP_HOLDER.pop(name, None)) is not None: + _unregister_state(cls) + return result, _app_root_file(web), root_code + + +def test_page_gaining_a_provided_wrap_reuses_the_app_root(tmp_path, monkeypatch): + """The app root depends on the union of wraps, not on each page's own set.""" + result, root_file, before = _root_rebuild(tmp_path, monkeypatch, state=True) + assert result is True + assert root_file.read_text() == before + + +def test_page_bringing_a_new_wrap_rebuilds_only_the_app_root(tmp_path, monkeypatch): + """A new wrap rebuilds the root from recorded and recompiled wraps.""" + result, root_file, before = _root_rebuild(tmp_path, monkeypatch, wrap=True) + assert result is True + after = root_file.read_text() + assert after != before + assert "test-wrap" in after + # The unchanged stateful page still provides the state providers. + assert "StateProvider" in after _COLLIDE = {"one": 0, "two": 0, "two_extra": False} @@ -1905,12 +2028,12 @@ def test_update_manifest_for_misses_keeps_complete_imports(tmp_path, monkeypatch app_wrap_components={}, frontend_imports={}, memo_contributions={}, + evaluate_bundle_requests=[], + render_bundle_requests=[], ) miss_ctx = SimpleNamespace(compiled_pages={"/a": page_ctx}, stateful_routes={}) complete_imports = {"memo-lib": [ImportVar("MemoThing")]} - manifest = _manifest({ - "/a": {"deps": [], "app_wrap_keys": [], "is_stateful": False} - }) + manifest = _manifest({"/a": {"deps": [], "app_wraps": {}, "is_stateful": False}}) disk_cache._update_manifest_for_misses( manifest, @@ -1920,6 +2043,8 @@ def test_update_manifest_for_misses_keeps_complete_imports(tmp_path, monkeypatch root=tmp_path, memo_state={}, memo_file_entries={}, + app_root_inputs=manifest["app_root"], + state_libraries={}, ) written = json.loads((web / disk_cache._MANIFEST_FILE).read_text()) @@ -1940,6 +2065,8 @@ def test_update_manifest_refreshes_imports_without_misses(tmp_path, monkeypatch) root=tmp_path, memo_state={}, memo_file_entries={}, + app_root_inputs=manifest["app_root"], + state_libraries={}, ) written = json.loads((web / disk_cache._MANIFEST_FILE).read_text()) diff --git a/tests/units/reflex_base/components/test_dynamic.py b/tests/units/reflex_base/components/test_dynamic.py index 5f05159f213..48105b92409 100644 --- a/tests/units/reflex_base/components/test_dynamic.py +++ b/tests/units/reflex_base/components/test_dynamic.py @@ -165,3 +165,19 @@ def test_unknown_attribute_raises(module_name: str): module = importlib.import_module(f"{module_name}.components.dynamic") with pytest.raises(AttributeError, match=f"{module.__name__!r}"): _ = module.definitely_not_an_attribute + + +def test_bundle_requests_are_recorded_in_order(): + """Every request is recorded, bundled already or not, and reaches outer recorders. + + A compile reconstructs the bundled-library order from what each page and + state requested, so a request for an already bundled library still counts. + """ + with RegistrationContext(): + with dynamic.record_bundle_requests() as outer: + dynamic.bundle_library("lib-a") + with dynamic.record_bundle_requests() as inner: + dynamic.bundle_library("lib-a") + dynamic.bundle_library("lib-b") + assert inner == ["lib-a", "lib-b"] + assert outer == ["lib-a", "lib-a", "lib-b"] From 994ce698589548a60e1e14361344dd4bc8b98b93 Mon Sep 17 00:00:00 2001 From: Farhan <www.mfarvirus@gmail.com> Date: Wed, 21 Oct 2026 02:23:39 +0500 Subject: [PATCH 36/56] perf(compiler): fail a name prediction only on families other pages share Dynamic state names are handed out per family: a local state's stem and its _<n> suffixes, or a ComponentState class's counter. A re-evaluated page changing its claims in a family no unchanged page uses cannot move another page's names, so the prediction now fails only when a changed family is shared with a page that was not re-evaluated. --- reflex/compiler/disk_cache.py | 76 ++++++++++++++++++++++--- tests/units/compiler/test_disk_cache.py | 37 ++++++++++-- 2 files changed, 101 insertions(+), 12 deletions(-) diff --git a/reflex/compiler/disk_cache.py b/reflex/compiler/disk_cache.py index e075a430d57..fd3873841d1 100644 --- a/reflex/compiler/disk_cache.py +++ b/reflex/compiler/disk_cache.py @@ -23,6 +23,7 @@ import json import logging import pickle +import re import sys from pathlib import Path from typing import TYPE_CHECKING, Any @@ -907,6 +908,67 @@ def _dynamic_claims(defined_states: Iterable[str]) -> list[list[str]]: return claims +_NAME_SUFFIX = re.compile(r"_\d+$") + + +def _claims_by_family(claims: Iterable[Sequence[str]]) -> dict[str, list[list[str]]]: + """Group dynamic state name claims by the name family they draw from. + + A local state takes the first free name among its class name and its + ``_<n>`` suffixes, so it competes with every claim whose name or request + has the same stem. A ``ComponentState`` instance draws from its class's + counter; its request is the dotted class path, which no class name has. + + Args: + claims: ``[name, requested]`` pairs (see :func:`_dynamic_claims`). + + Returns: + The claims of each family, in order. + """ + families: dict[str, list[list[str]]] = {} + for name, requested in claims: + for family in {_NAME_SUFFIX.sub("", name), _NAME_SUFFIX.sub("", requested)}: + families.setdefault(family, []).append([name, requested]) + return families + + +def _shifting_dynamic_claims( + manifest: dict[str, Any], miss_ctx: CompileContext, miss_routes: set[str] +) -> str | None: + """Find a re-evaluated page whose new claims could shift other pages' names. + + Names in a family are handed out in evaluation order, so a re-evaluated + page changing its claims in a family moves the names of later pages in + that family. Pages that were not re-evaluated keep their recorded names, + which is exact only when no family they use changed. + + Args: + manifest: The loaded manifest. + miss_ctx: The compile context of the re-evaluated pages. + miss_routes: The re-evaluated routes. + + Returns: + The first re-evaluated route that changed a shared family, or None. + """ + shared = { + family + for route, entry in manifest["pages"].items() + if route not in miss_routes + for family in _claims_by_family(entry["dynamic_claims"]) + } + for route in miss_routes: + recorded = _claims_by_family(manifest["pages"][route]["dynamic_claims"]) + live = _claims_by_family( + _dynamic_claims(miss_ctx.stateful_routes.get(route, ())) + ) + if any( + live.get(family) != recorded.get(family) + for family in (live.keys() | recorded.keys()) & shared + ): + return route + return None + + def _claim_recorded_dynamic_names( manifest: dict[str, Any], miss_routes: set[str] ) -> None: @@ -1443,14 +1505,12 @@ def try_incremental_rebuild( ): _fall_back_after_evaluation(f"page {page.route!r} produced no output") return False - entry = manifest["pages"][page.route] - if _predict_dynamic_names and ( - _dynamic_claims(miss_ctx.stateful_routes.get(page.route, ())) - != entry["dynamic_claims"] - ): - _fall_back_after_evaluation( - f"page {page.route!r} changed its dynamic state names" - ) + if _predict_dynamic_names and ( + route := _shifting_dynamic_claims(manifest, miss_ctx, miss_routes) + ): + _fall_back_after_evaluation( + f"page {route!r} changed dynamic state names other pages share" + ) # Write changed pages + their memo files; reuse everything else on disk. install_imports = _deserialize_imports(manifest["all_imports"]) diff --git a/tests/units/compiler/test_disk_cache.py b/tests/units/compiler/test_disk_cache.py index 490c8638205..3e5ce432d6b 100644 --- a/tests/units/compiler/test_disk_cache.py +++ b/tests/units/compiler/test_disk_cache.py @@ -1554,7 +1554,7 @@ def test_page_bringing_a_new_wrap_rebuilds_only_the_app_root(tmp_path, monkeypat assert "StateProvider" in after -_COLLIDE = {"one": 0, "two": 0, "two_extra": False} +_COLLIDE = {"one": 0, "two": 0, "two_extra": False, "two_unique": False} def _page_collide_one() -> Component: @@ -1576,20 +1576,31 @@ def define_extra() -> None: class CollideState(rx.State): extra: int = 3 + def define_unique() -> None: + class UniqueToTwoState(rx.State): + unique: int = 4 + if _COLLIDE["two_extra"]: define_extra() + if _COLLIDE["two_unique"]: + define_unique() return rx.el.div(rx.el.p(CollideState.two), _footer()) def _unregister_collide_states() -> None: """Drop every registered ``CollideState``, like the daemon child's reset.""" for name, cls in list(RegistrationContext.ensure_context().base_states.items()): - if "collide_state" in name: + if "collide_state" in name or "unique_to_two_state" in name: _unregister_state(cls) def _rebuild_second_collide_page( - tmp_path, monkeypatch, *, predict: bool, two_extra: bool = False + tmp_path, + monkeypatch, + *, + predict: bool, + two_extra: bool = False, + two_unique: bool = False, ) -> tuple[Any, list[str], list[str]]: """Compile two pages defining a same-named local state, then rebuild the second. @@ -1598,6 +1609,8 @@ def _rebuild_second_collide_page( monkeypatch: The pytest monkeypatch fixture. predict: Whether the rebuild predicts dynamic state names. two_extra: Whether the edited second page defines one more such state. + two_unique: Whether the edited second page defines a state no other + page names the same way. Returns: The rebuild result, the second page's states before the edit, and its @@ -1623,6 +1636,7 @@ def _rebuild_second_collide_page( monkeypatch.setitem(_COLLIDE, "one", 0) monkeypatch.setitem(_COLLIDE, "two", 0) monkeypatch.setitem(_COLLIDE, "two_extra", two_extra) + monkeypatch.setitem(_COLLIDE, "two_unique", two_unique) monkeypatch.setattr(disk_cache, "_predict_dynamic_names", predict) try: result = disk_cache.try_incremental_rebuild( @@ -1632,7 +1646,11 @@ def _rebuild_second_collide_page( _unregister_collide_states() import reflex.istate.dynamic as istate_dynamic - for name in [n for n in vars(istate_dynamic) if "CollideState" in n]: + for name in [ + n + for n in vars(istate_dynamic) + if "CollideState" in n or "UniqueToTwoState" in n + ]: delattr(istate_dynamic, name) refreshed = json.loads(manifest_path.read_text()) return ( @@ -1669,6 +1687,17 @@ def test_predicted_local_state_names_rebuild_only_the_changed_page( assert (_COLLIDE["one"], _COLLIDE["two"]) == (0, 1) +def test_prediction_accepts_a_new_state_name_no_other_page_uses(tmp_path, monkeypatch): + """A new local state in a name family of its own cannot shift other pages.""" + result, before, after = _rebuild_second_collide_page( + tmp_path, monkeypatch, predict=True, two_unique=True + ) + assert result is True + assert after[: len(before)] == before + assert after[-1].endswith("unique_to_two_state") + assert (_COLLIDE["one"], _COLLIDE["two"]) == (0, 1) + + def test_unverified_local_state_prediction_asks_for_a_fresh_compile( tmp_path, monkeypatch ): From 966280e5d2c9005c69ff73e91d69dc6fc2d5bbdc Mon Sep 17 00:00:00 2001 From: Farhan <www.mfarvirus@gmail.com> Date: Wed, 21 Oct 2026 02:41:50 +0500 Subject: [PATCH 37/56] perf(compiler): add, remove and re-register pages without a full compile Route set changes no longer force a full compile. Each page records a digest of its registration (route, metadata, load events, component identity), taken before any page evaluates; a new route or a changed registration is a miss. A removed route loses its output file, manifest entry, route-owned memo records and its auto memos (its module's memo file is rewritten from the remaining pages, or deleted); a route change rebuilds the contexts and, through the recorded inputs, the app root, and rewrites the route table. Removing or adding a page that shares dynamic state name families with other pages fails the name prediction and takes the exact rule. The app entry module is still a global input, except for an edit that changes only its page registrations: page_cache.entrypoint_digest leaves out top-level add_page calls and the imports nothing else reads. Also: - The default 404 page has no source module; walking the call stack gave it the module of whatever process compiled the app. - State slices hold initial values serialized as the contexts file holds them; a component value's code depends on whether its library is bundled, so slices taken before and after bundling never matched. --- reflex/app.py | 6 +- reflex/compiler/compiler.py | 5 + reflex/compiler/disk_cache.py | 357 +++++++++++++++++++----- reflex/compiler/page_cache.py | 72 +++++ tests/units/compiler/test_disk_cache.py | 231 +++++++++++++-- tests/units/compiler/test_page_cache.py | 64 +++++ tests/units/test_app.py | 10 + 7 files changed, 639 insertions(+), 106 deletions(-) diff --git a/reflex/app.py b/reflex/app.py index cebfee0ea0e..ffcde3fbc11 100644 --- a/reflex/app.py +++ b/reflex/app.py @@ -1069,6 +1069,7 @@ def _prepare_page( msg = "Route must be set if component is not a callable." raise exceptions.RouteValueError(msg) + default_404 = route == constants.Page404.SLUG and component is None if route == constants.Page404.SLUG: if component is None: from reflex_components_core.el.elements import span @@ -1087,7 +1088,10 @@ def _prepare_page( # Check if the route given is valid verify_route_validity(route) - if isinstance(component, Callable): + if default_404: + # The framework's page, so no user module to mirror. + source_module = None + elif isinstance(component, Callable): source_module = memo_paths.capture_source_module(component) else: # The user passed a pre-built Component instance — fall back to diff --git a/reflex/compiler/compiler.py b/reflex/compiler/compiler.py index c984e536b35..fb4e3ce43bf 100644 --- a/reflex/compiler/compiler.py +++ b/reflex/compiler/compiler.py @@ -1504,10 +1504,13 @@ def compile_app( # manifest. Falls back to a full compile on any unsafe condition; pages the # attempt already compiled before falling back are adopted, not redone. evaluated: CompileContext | None = None + registrations: dict[str, str] | None = None if cache_on: from reflex.compiler import disk_cache, page_cache page_cache.enable_read_tracking() + # Taken before any page evaluates: evaluation mutates components. + registrations = disk_cache.registration_digests(app._unevaluated_pages.values()) outputs: dict[Path, str] = {} rebuilt = disk_cache.try_incremental_rebuild( app, @@ -1515,6 +1518,7 @@ def compile_app( prerender_routes=prerender_routes, use_rich=use_rich, outputs=outputs, + registrations=registrations, ) if isinstance(rebuilt, CompileContext): evaluated = rebuilt @@ -1797,6 +1801,7 @@ def compile_app( plugin_sources=plugin_sources, contexts_snapshot=contexts_snapshot, root_bundle_requests=root_bundle_requests, + registrations=registrations, ) return True diff --git a/reflex/compiler/disk_cache.py b/reflex/compiler/disk_cache.py index fd3873841d1..921af064d03 100644 --- a/reflex/compiler/disk_cache.py +++ b/reflex/compiler/disk_cache.py @@ -53,7 +53,7 @@ logger = logging.getLogger(__name__) #: Bump when the manifest layout changes (old manifests are then ignored). -_SCHEMA = 17 +_SCHEMA = 20 #: Manifest filename under the web directory. _MANIFEST_FILE = "reflex_compile_cache.json" @@ -319,14 +319,79 @@ def _reconstruct_libraries( return list(dict.fromkeys(ordered)) +def _stable_value(value: Any) -> Any: + """Turn a page registration value into a process-independent JSON value. + + Args: + value: A value ``add_page`` was given. + + Returns: + A JSON-able value that is equal across processes for equal inputs. + """ + from reflex_base.components.component import BaseComponent + from reflex_base.vars import Var + + if value is None or isinstance(value, (str, int, float, bool)): + return value + if isinstance(value, Var): + return str(value) + if isinstance(value, BaseComponent): + return value.render() + if isinstance(value, dict): + return {str(key): _stable_value(item) for key, item in value.items()} + if isinstance(value, (list, tuple)): + return [_stable_value(item) for item in value] + if dataclasses.is_dataclass(value) and not isinstance(value, type): + return { + field.name: _stable_value(getattr(value, field.name)) + for field in dataclasses.fields(value) + } + if callable(value): + # Which function, not its body: a page function's body is a recorded + # dependency of the page, and load events only run on the backend. + return [ + getattr(value, "__module__", None), + getattr(value, "__qualname__", None), + ] + return repr(value) + + +def _registration_digest(page: PageDefinition) -> str: + """Digest everything a page was registered with. + + Args: + page: The page definition. + + Returns: + A digest of its route, metadata, load events and component. + """ + return hashlib.sha256(json_dumps(_stable_value(page)).encode()).hexdigest() + + +def registration_digests(pages: Iterable[PageDefinition]) -> dict[str, str]: + """Digest each page's registration before any page is evaluated. + + Evaluation mutates component instances passed as pages (styles are + applied in place), so the digests are taken once, up front. + + Args: + pages: The page definitions. + + Returns: + Each route's registration digest. + """ + return {page.route: _registration_digest(page) for page in pages} + + def _manifest_page_entry( page_ctx: PageContext, - component: Any, + page: PageDefinition, state_index: dict[str, Path], hasher: Callable[[str], page_cache.FileEntry | None], files: dict[str, page_cache.FileEntry], *, defined_states: Sequence[str] | None, + registration: str, wrap_table: dict[str, str | None], state_slice: dict[str, Any] | None = None, root: Path | None = None, @@ -335,12 +400,14 @@ def _manifest_page_entry( Args: page_ctx: The compiled page context. - component: The page component/callable used for dependency discovery. + page: The page definition. state_index: The state-context identifier -> file index. hasher: A memoized path -> file-entry function. files: The manifest file table, extended with the page's dependencies. defined_states: Full names of the states the page defined while it evaluated, or None for a stateless page. + registration: The page's registration digest (see + :func:`registration_digests`). wrap_table: The manifest's app-wrap table, extended with the page's wraps. state_slice: The page's contribution to the contexts file @@ -355,8 +422,10 @@ def _manifest_page_entry( return { "component_counts": _count_components([page_ctx.root_component]), "deps": page_cache.page_dependency_entries( - page_ctx, component, state_index, hasher, files, root + page_ctx, page.component, state_index, hasher, files, root ), + # Everything the page was registered with (see ``partition_pages``). + "registration": registration, # The app wraps the page requires, by digest (see ``_refresh_app_root``). "app_wraps": _record_app_wraps(page_ctx, wrap_table), # Libraries requested while evaluating and rendering the page. @@ -374,6 +443,12 @@ def _manifest_page_entry( # must be recompiled alongside a same-module miss (see # ``_with_module_siblings``). "has_memos": bool(page_ctx.memo_contributions), + # Where those auto memos are written, so removing the page rewrites it. + "memo_path": ( + _memo_output_path(getattr(page, "_source_module", None)) + if page_ctx.memo_contributions + else None + ), } @@ -402,11 +477,19 @@ def _state_slice( ``{"initial_state": {...}, "client_storage": {kind: {...}}, "schema": {...}}``. """ + from reflex.compiler import utils as compiler_utils + names = sorted(state_names) prefixes = tuple(f"{name}." for name in names) return { + # Serialized as the contexts file holds them: a component value's code + # depends on whether its library is bundled, which serializing does. "initial_state": { - name: initial_state[name] for name in names if name in initial_state + name: json.loads( + compiler_utils._serialize_initial_state({name: initial_state[name]}) + )[name] + for name in names + if name in initial_state }, "client_storage": { kind: {k: v for k, v in entries.items() if k.startswith(prefixes)} @@ -576,6 +659,7 @@ def write_manifest( plugin_sources: dict[str, str] | None = None, contexts_snapshot: compiler_utils.ContextsSnapshot | None = None, root_bundle_requests: Sequence[str] = (), + registrations: dict[str, str] | None = None, ) -> None: """Persist a manifest of the just-completed full compile. @@ -596,12 +680,16 @@ def write_manifest( When absent, a snapshot is captured for direct callers. root_bundle_requests: The libraries requested while building the app root. + registrations: Each route's registration digest, taken before any page + evaluated; computed now when absent. """ try: state_index, _ = page_cache.state_dependency_index(root) hasher = page_cache.make_hasher() files: dict[str, page_cache.FileEntry] = {} wrap_table: dict[str, str | None] = {} + if registrations is None: + registrations = registration_digests(pages) if contexts_snapshot is None and compile_ctx.stateful_routes: contexts_snapshot = _contexts_snapshot(compile_ctx.app) @@ -621,11 +709,12 @@ def write_manifest( defined_states = compile_ctx.stateful_routes.get(page.route) pages_data[page.route] = _manifest_page_entry( page_ctx, - page.component, + page, state_index, hasher, files, defined_states=defined_states, + registration=registrations[page.route], wrap_table=wrap_table, state_slice=( _state_slice(defined_states, *contexts_snapshot) @@ -670,8 +759,19 @@ def write_manifest( files[path] = entry globals_[label] = path + entry_file = page_cache._app_entrypoint_file(root) manifest = { "schema": _SCHEMA, + # The app entry module apart from its page registrations (see + # ``globals_mismatch``). + "entry": { + "path": None if entry_file is None else str(entry_file), + "digest": ( + None + if entry_file is None + else page_cache.entrypoint_digest(str(entry_file)) + ), + }, # File paths below are absolute: the manifest describes this # project directory only. "root": str((root or Path.cwd()).resolve()), @@ -746,19 +846,18 @@ def _imported_state_files(compile_ctx: CompileContext, root: Path | None) -> set def globals_mismatch( manifest: dict[str, Any], *, - routes: set[str], validator: page_cache.FileValidator, root: Path | None = None, ) -> str | None: """Explain why the manifest's global inputs don't match, or None if they do. - The fast rebuild needs the route set unchanged (adding/removing a route - changes the shared nav on every page) and the global inputs unchanged (Reflex - version + config/lockfiles + the app-level config files: the entrypoint and - the theme/app-wrap/stylesheet modules it imports, which configure the app-wide - files reused on disk). Everything else is decided per page via its dependency - set, so a shared-component or markdown edit no longer blocks the fast path. - Only the pages that depend on the changed file miss. + The fast rebuild needs the global inputs unchanged: Reflex version, + config/lockfiles and the app-level config files (the entrypoint and the + theme/app-wrap/stylesheet modules it imports, which configure the app-wide + files). An entrypoint that only changed its page registrations still + matches; the routes and registrations are then compared page by page. + Everything else is decided per page via its dependency set, so a + shared-component or markdown edit no longer blocks the fast path. Global inputs are validated against the manifest's *stored* input set, never by recomputing the set in this process — set membership is only @@ -766,7 +865,6 @@ def globals_mismatch( Args: manifest: The loaded manifest. - routes: The current set of page routes. validator: The file validator bound to the manifest's file table. root: Project root the manifest must describe; also shortens paths in the reason. Defaults to cwd. @@ -785,19 +883,20 @@ def globals_mismatch( old_mode = manifest.get("mode") if old_mode != _compile_mode(): return f"compile mode changed ({old_mode} -> {_compile_mode()})" - old_routes = set(manifest.get("pages", {})) - if old_routes != routes: - parts = [] - if added := routes - old_routes: - parts.append(f"added {format_path_list(added)}") - if removed := old_routes - routes: - parts.append(f"removed {format_path_list(removed)}") - return f"route set changed ({'; '.join(parts)})" root = (root or Path.cwd()).resolve() + entry = manifest["entry"] stale = { label.removeprefix("app:") for label, path in manifest.get("globals", {}).items() - if label != "reflex" and validator.changed(path) + if label != "reflex" + and validator.changed(path) + # An entry module that only changed its page registrations is compared + # page by page (see ``partition_pages``). + and not ( + path == entry["path"] + and entry["digest"] is not None + and page_cache.entrypoint_digest(path) == entry["digest"] + ) } stale.update( path for path in manifest.get("globals_absent", ()) if Path(path).is_file() @@ -811,28 +910,34 @@ def partition_pages( pages: Sequence[PageDefinition], manifest: dict[str, Any], validator: page_cache.FileValidator, - changed_hint: set[str] | None = None, + changed_hint: set[str] | None, + registrations: dict[str, str], ) -> list[PageDefinition]: - """Return the pages whose dependency set changed since the manifest. + """Return the pages that are new or changed since the manifest. Globals are assumed already matched (see :func:`globals_mismatch`), so a - page is a hit if every file in its recorded dependency set is unchanged. - When the watcher reported the changed paths, a page depending on none of - them is a hit without touching the filesystem at all. + recorded page is a hit if it was registered the same way and every file + in its recorded dependency set is unchanged. When the watcher reported the + changed paths, a page depending on none of them skips the filesystem. Args: pages: The current page definitions. manifest: The loaded manifest. validator: The file validator bound to the manifest's file table. changed_hint: Resolved paths the watcher saw change, or None if unknown. + registrations: Each route's registration digest. Returns: - The list of miss pages (a dependency changed) to recompile. + The list of miss pages (new, re-registered or a dependency changed). """ manifest_pages = manifest["pages"] misses: list[PageDefinition] = [] for page in pages: - deps = manifest_pages[page.route]["deps"] + entry = manifest_pages.get(page.route) + if entry is None or entry["registration"] != registrations[page.route]: + misses.append(page) + continue + deps = entry["deps"] if changed_hint is not None and changed_hint.isdisjoint(deps): continue if not validator.unchanged(deps): @@ -933,22 +1038,26 @@ def _claims_by_family(claims: Iterable[Sequence[str]]) -> dict[str, list[list[st def _shifting_dynamic_claims( - manifest: dict[str, Any], miss_ctx: CompileContext, miss_routes: set[str] + manifest: dict[str, Any], + miss_ctx: CompileContext | None, + miss_routes: set[str], + removed_entries: dict[str, dict[str, Any]], ) -> str | None: - """Find a re-evaluated page whose new claims could shift other pages' names. + """Find a changed page whose claims could shift other pages' names. - Names in a family are handed out in evaluation order, so a re-evaluated - page changing its claims in a family moves the names of later pages in - that family. Pages that were not re-evaluated keep their recorded names, - which is exact only when no family they use changed. + Names in a family are handed out in evaluation order, so a re-evaluated, + added or removed page changing its claims in a family moves the names of + later pages in that family. Pages that were not re-evaluated keep their + recorded names, which is exact only when no family they use changed. Args: - manifest: The loaded manifest. - miss_ctx: The compile context of the re-evaluated pages. + manifest: The loaded manifest (without the removed pages). + miss_ctx: The compile context of the re-evaluated pages, if any. miss_routes: The re-evaluated routes. + removed_entries: The manifest entries of the removed routes. Returns: - The first re-evaluated route that changed a shared family, or None. + The first changed route that changed a shared family, or None. """ shared = { family @@ -956,11 +1065,22 @@ def _shifting_dynamic_claims( if route not in miss_routes for family in _claims_by_family(entry["dynamic_claims"]) } - for route in miss_routes: - recorded = _claims_by_family(manifest["pages"][route]["dynamic_claims"]) - live = _claims_by_family( + changes = [ + ( + route, + manifest["pages"].get(route, {}).get("dynamic_claims", []), _dynamic_claims(miss_ctx.stateful_routes.get(route, ())) + if miss_ctx is not None + else [], ) + for route in miss_routes + ] + changes.extend( + (route, entry["dynamic_claims"], []) for route, entry in removed_entries.items() + ) + for route, recorded_claims, live_claims in changes: + recorded = _claims_by_family(recorded_claims) + live = _claims_by_family(live_claims) if any( live.get(family) != recorded.get(family) for family in (live.keys() | recorded.keys()) & shared @@ -1011,6 +1131,8 @@ def _with_dynamic_state_pages( miss_pages: list[PageDefinition], pages: Sequence[PageDefinition], manifest: dict[str, Any], + *, + removed: bool = False, ) -> list[PageDefinition]: """Expand a miss set with every page that defines dynamic-module states. @@ -1025,11 +1147,12 @@ def _with_dynamic_state_pages( miss_pages: The pages to recompile. pages: All current page definitions (in compile order). manifest: The loaded manifest. + removed: Whether a removed page defined dynamic-module states. Returns: The expanded miss list, in ``pages`` order. """ - if not miss_pages: + if not miss_pages and not removed: return miss_pages miss_routes = {page.route for page in miss_pages} return [ @@ -1344,6 +1467,7 @@ def try_incremental_rebuild( root: Path | None = None, use_rich: bool = True, outputs: dict[Path, str] | None = None, + registrations: dict[str, str] | None = None, ) -> bool | CompileContext: """Attempt a disk-cache-assisted partial rebuild; report whether it ran. @@ -1379,6 +1503,8 @@ def try_incremental_rebuild( root: Project root for fingerprinting. Defaults to cwd. use_rich: Whether to use a rich progress bar (else a plain fallback). outputs: Stage generated files here for plugin processing instead of writing. + registrations: Each route's registration digest, taken before any page + evaluated; computed now when absent. Returns: True if the partial rebuild completed (the caller should return), False @@ -1394,19 +1520,27 @@ def try_incremental_rebuild( pages = list(app._unevaluated_pages.values()) routes = {p.route for p in pages} + if registrations is None: + registrations = registration_digests(pages) validator = page_cache.FileValidator(manifest["files"]) changed_hint = _changed_hint if ( - reason := globals_mismatch( - manifest, routes=routes, validator=validator, root=root - ) + reason := globals_mismatch(manifest, validator=validator, root=root) ) is not None: _log_fallback(reason) return False resolved_root = (root or Path.cwd()).resolve() - miss_pages = partition_pages(pages, manifest, validator, changed_hint) + # Removed routes leave the manifest; added ones have no entry, so they miss. + removed_entries = { + route: manifest["pages"].pop(route) + for route in [route for route in manifest["pages"] if route not in routes] + } + route_set_changed = bool(removed_entries) or not routes <= set(manifest["pages"]) + miss_pages = partition_pages( + pages, manifest, validator, changed_hint, registrations + ) # Memo files are decided salsa-style: derive the files the current memo # registry demands and rewrite the ones whose stored record differs. This # catches what changed-file diffing structurally cannot: a memo module the @@ -1424,21 +1558,32 @@ def try_incremental_rebuild( dirty_memo_paths = _dirty_memo_paths( memo_state, stored_memo_files, validator, changed_hint ) + # A removed page's auto memos must leave its module's memo file. + removed_memo_paths = { + path for entry in removed_entries.values() if (path := entry["memo_path"]) + } + dirty_memo_paths |= removed_memo_paths stale_owner_routes = { owner for entry in stored_memo_files.values() if (owner := entry["owner_route"]) is not None + and owner in routes and _memo_deps_changed(entry["deps"], validator, changed_hint) } changed_files: set[str] = set() - if miss_pages or dirty_memo_paths or stale_owner_routes: + if miss_pages or dirty_memo_paths or stale_owner_routes or removed_entries: changed_files = _changed_dependency_files(manifest, validator, changed_hint) miss_pages = _with_memo_contributing_pages( miss_pages, pages, manifest, dirty_memo_paths, stale_owner_routes ) miss_pages = _with_module_siblings(miss_pages, pages, manifest) if not _predict_dynamic_names: - miss_pages = _with_dynamic_state_pages(miss_pages, pages, manifest) + miss_pages = _with_dynamic_state_pages( + miss_pages, + pages, + manifest, + removed=any(e["dynamic_claims"] for e in removed_entries.values()), + ) memo_note = ( f"; rewriting {len(dirty_memo_paths)} memo file(s)" if dirty_memo_paths @@ -1454,9 +1599,10 @@ def try_incremental_rebuild( miss_routes = {p.route for p in miss_pages} from reflex.compiler import compiler + from reflex.compiler import utils as compiler_utils bundle_base: list[str] = [] - if miss_pages or dirty_memo_paths: + if miss_pages or dirty_memo_paths or route_set_changed: # Match the full compile's clean bundling/memo state before compiling. bundle_base = compiler._reset_compile_registries(compiler_plugins) @@ -1505,16 +1651,21 @@ def try_incremental_rebuild( ): _fall_back_after_evaluation(f"page {page.route!r} produced no output") return False - if _predict_dynamic_names and ( - route := _shifting_dynamic_claims(manifest, miss_ctx, miss_routes) - ): - _fall_back_after_evaluation( - f"page {route!r} changed dynamic state names other pages share" - ) + + if _predict_dynamic_names and ( + route := _shifting_dynamic_claims( + manifest, miss_ctx, miss_routes, removed_entries + ) + ): + _fall_back_after_evaluation( + f"page {route!r} changed dynamic state names other pages share" + ) # Write changed pages + their memo files; reuse everything else on disk. install_imports = _deserialize_imports(manifest["all_imports"]) eval_memo_state: dict[str, dict[str, Any]] = {} + # Outputs of removed pages, deleted once the rebuild succeeded. + stale_outputs = [compiler_utils.get_page_path(route) for route in removed_entries] if miss_ctx is not None or dirty_memo_paths: memo_contributions: dict[tuple[str, str | None], Any] = {} miss_imports = [] @@ -1554,6 +1705,12 @@ def try_incremental_rebuild( ) for mpath, mcode in memo_files: _save_incremental_output(mpath, mcode, outputs) + emitted = {mpath for mpath, _ in memo_files} + stale_outputs.extend( + path + for path in removed_memo_paths + if path not in emitted and path not in manifest["memo_files"] + ) # Merge once: re-merging the app-wide set per page re-walks its ~100k # entries each time. install_imports = merge_imports(install_imports, *miss_imports, memo_imports) @@ -1591,13 +1748,25 @@ def try_incremental_rebuild( # edit leaves them identical and the contexts file is reused untouched. contexts_snapshot: compiler_utils.ContextsSnapshot | None = None state_libraries: dict[str, list[str]] = manifest["state_libraries"] - if miss_ctx is not None and any( - route in miss_ctx.stateful_routes or manifest["pages"][route]["is_stateful"] - for route in miss_routes + if route_set_changed or ( + miss_ctx is not None + and any( + route in miss_ctx.stateful_routes + or manifest["pages"].get(route, {}).get("is_stateful") + for route in miss_routes + ) ): contexts_snapshot = _contexts_snapshot(app) - changed_route = _changed_state_config_route( - manifest, miss_ctx, contexts_snapshot, miss_routes + # A route change can drop slices or add dynamic route arguments to the + # root state, so it always rebuilds the contexts. + changed_route = ( + "the route set" + if route_set_changed + else _changed_state_config_route( + manifest, miss_ctx, contexts_snapshot, miss_routes + ) + if miss_ctx is not None + else None ) if changed_route is None: logger.info( @@ -1606,8 +1775,8 @@ def try_incremental_rebuild( ) else: logger.info( - f"Compile cache: page {changed_route!r} changed its state " - "config; rebuilding contexts" + f"Compile cache: {changed_route!r} changed the state config; " + "rebuilding contexts" ) from reflex_components_radix.plugin import RadixThemesPlugin @@ -1641,14 +1810,20 @@ def try_incremental_rebuild( } app_root_inputs = manifest["app_root"] - if miss_ctx is not None: + if miss_ctx is not None or route_set_changed: try: refreshed = _refresh_app_root( - app, manifest, pages, miss_ctx, state_libraries, outputs + app, + manifest, + pages, + miss_ctx.compiled_pages if miss_ctx is not None else {}, + bundle_base, + state_libraries, + outputs, ) except _AppRootUnavailableError as err: _fall_back_after_evaluation(str(err)) - return miss_ctx + return miss_ctx if miss_ctx is not None else False if refreshed is not None: app_root_inputs, root_imports = refreshed install_imports = merge_imports(install_imports, root_imports) @@ -1682,6 +1857,7 @@ def try_incremental_rebuild( for path, entry in stored_memo_files.items() if entry["owner_route"] is not None and entry["owner_route"] not in miss_routes + and entry["owner_route"] in routes }, **memo_state, **eval_memo_state, @@ -1699,9 +1875,18 @@ def try_incremental_rebuild( memo_file_entries=memo_files_table, app_root_inputs=app_root_inputs, state_libraries=state_libraries, + registrations=registrations, + route_set_changed=route_set_changed, validator=validator, ) + # The route table the production static server reads. + _save_incremental_output( + constants.Dirs.ROUTES_MANIFEST, json.dumps(app._page_routes), outputs + ) + for path in stale_outputs: + Path(path).unlink(missing_ok=True) + counts: dict[str, int] = {} for page in pages: page_counts = manifest["pages"][page.route].get("component_counts", {}) @@ -1719,7 +1904,8 @@ def _refresh_app_root( app: App, manifest: dict[str, Any], pages: Sequence[PageDefinition], - miss_ctx: CompileContext, + compiled_pages: dict[str, PageContext], + bundle_base: Sequence[str], state_libraries: dict[str, list[str]], outputs: dict[Path, str] | None, ) -> tuple[dict[str, Any], ParsedImportDict] | None: @@ -1737,7 +1923,9 @@ def _refresh_app_root( app: The app being compiled. manifest: The loaded manifest; its wrap table gains the live wraps. pages: All current page definitions (in compile order). - miss_ctx: The compile context of the recompiled pages. + compiled_pages: The recompiled pages' contexts, by route. + bundle_base: The bundled libraries registered after this rebuild's + registry reset. state_libraries: The libraries each state's initial values request. outputs: Output mapping supplied by the main compiler, if any. @@ -1760,8 +1948,7 @@ def _refresh_app_root( page_requests = [] memo_paths: set[str] = set(manifest["memo_files"]) for page in pages: - entry = manifest["pages"][page.route] - if (page_ctx := miss_ctx.compiled_pages.get(page.route)) is not None: + if (page_ctx := compiled_pages.get(page.route)) is not None: wraps = _record_app_wraps(page_ctx, wrap_table) page_requests.append([ page_ctx.evaluate_bundle_requests, @@ -1769,6 +1956,7 @@ def _refresh_app_root( ]) has_memos = bool(page_ctx.memo_contributions) else: + entry = manifest["pages"][page.route] wraps = entry["app_wraps"] page_requests.append(entry["bundle_requests"]) has_memos = entry["has_memos"] @@ -1780,7 +1968,7 @@ def _refresh_app_root( merged[key_str] = digest providers[key_str] = page.route libraries = _reconstruct_libraries( - miss_ctx.bundle_base, page_requests, stored["root_requests"], state_libraries + bundle_base, page_requests, stored["root_requests"], state_libraries ) if merged == stored["wraps"] and libraries == stored["libraries"]: return None @@ -1791,7 +1979,7 @@ def _refresh_app_root( components = {} for key_str, digest in merged.items(): key = _wrap_key(key_str) - if (page_ctx := miss_ctx.compiled_pages.get(providers[key_str])) is not None: + if (page_ctx := compiled_pages.get(providers[key_str])) is not None: components[key] = page_ctx.app_wrap_components[key] elif (data := wrap_table.get(digest)) is not None: components[key] = pickle.loads(base64.b64decode(data)) @@ -1828,7 +2016,7 @@ def _refresh_app_root( *root_memos, ]) libraries = _reconstruct_libraries( - miss_ctx.bundle_base, page_requests, root_requests, state_libraries + bundle_base, page_requests, root_requests, state_libraries ) RegistrationContext.ensure_context().bundled_libraries[:] = libraries for path, code in ( @@ -1860,6 +2048,8 @@ def _update_manifest_for_misses( memo_file_entries: dict[str, page_cache.FileEntry], app_root_inputs: dict[str, Any], state_libraries: dict[str, list[str]], + registrations: dict[str, str], + route_set_changed: bool = False, validator: page_cache.FileValidator | None = None, ) -> None: """Update the on-disk manifest entries for the recompiled pages. @@ -1882,11 +2072,25 @@ def _update_manifest_for_misses( withheld until validation finished. app_root_inputs: The app root's recorded inputs after this rebuild. state_libraries: The libraries each state's initial values request. + registrations: Each route's registration digest. + route_set_changed: Whether pages were added or removed. validator: The validator used for this rebuild, for its refreshed entries. """ try: - dirty = validator is not None and _refresh_file_table(manifest, validator) + dirty = route_set_changed or ( + validator is not None and _refresh_file_table(manifest, validator) + ) + # An entry module whose edit only changed page registrations stays the + # recorded one; record its new content so the next rebuild skips it. + if ( + validator is not None + and (entry_path := manifest["entry"]["path"]) is not None + and validator.changed(entry_path) + and (entry := page_cache.file_entry(entry_path)) is not None + ): + manifest["files"][entry_path] = list(entry) + dirty = True if manifest["memo_files"] != memo_state: manifest["memo_files"] = memo_state manifest["files"].update(memo_file_entries) @@ -1899,11 +2103,12 @@ def _update_manifest_for_misses( defined_states = miss_ctx.stateful_routes.get(page.route) manifest["pages"][page.route] = _manifest_page_entry( page_ctx, - page.component, + page, state_index, hasher, manifest["files"], defined_states=defined_states, + registration=registrations[page.route], wrap_table=manifest["wrap_table"], state_slice=( _state_slice(defined_states, *contexts_snapshot) diff --git a/reflex/compiler/page_cache.py b/reflex/compiler/page_cache.py index 004a4eae689..a7c66c92f14 100644 --- a/reflex/compiler/page_cache.py +++ b/reflex/compiler/page_cache.py @@ -8,6 +8,7 @@ from __future__ import annotations +import ast import builtins import contextlib import hashlib @@ -490,6 +491,77 @@ def record_app_import(root: Path | None = None): _app_import_reads[root] = recorded +def _is_page_registration(statement: ast.stmt) -> bool: + """Whether a top-level statement is an ``<app>.add_page(...)`` call. + + Args: + statement: A module-level statement. + + Returns: + True for an ``add_page`` call statement. + """ + return ( + isinstance(statement, ast.Expr) + and isinstance(statement.value, ast.Call) + and isinstance(statement.value.func, ast.Attribute) + and statement.value.func.attr == "add_page" + ) + + +def _loaded_names(nodes: Iterable[ast.AST]) -> set[str]: + """Collect the names the given syntax trees read. + + Args: + nodes: The syntax trees. + + Returns: + Every name loaded anywhere within them. + """ + return { + node.id + for tree in nodes + for node in ast.walk(tree) + if isinstance(node, ast.Name) and isinstance(node.ctx, ast.Load) + } + + +def entrypoint_digest(path: str) -> str | None: + """Digest the app entry module apart from its top-level page registrations. + + Left out are top-level ``<app>.add_page(...)`` statements and the imported + names no other statement reads (a page imported only to be registered), + plus what the syntax tree never holds (comments, formatting). + Two versions with the same digest differ only in the pages they register, + which the compile cache compares page by page; any other edit can change + every page. + + Args: + path: The entry module's file. + + Returns: + The digest, or None when the file cannot be read or parsed. + """ + try: + tree = ast.parse(Path(path).read_text(encoding="utf-8")) + except (OSError, SyntaxError, ValueError): + return None + others = [s for s in tree.body if not _is_page_registration(s)] + used = _loaded_names(others) + body: list[ast.stmt] = [] + for statement in others: + if isinstance(statement, (ast.Import, ast.ImportFrom)): + statement.names = [ + alias + for alias in statement.names + if (alias.asname or alias.name).partition(".")[0] in used + ] + if not statement.names: + continue + body.append(statement) + tree.body = body + return hashlib.sha256(ast.dump(tree).encode()).hexdigest() + + def _app_entrypoint_file(root: Path | None = None) -> Path | None: """Resolve the user's app entrypoint module file (where ``rx.App`` is built). diff --git a/tests/units/compiler/test_disk_cache.py b/tests/units/compiler/test_disk_cache.py index 3e5ce432d6b..6784123441b 100644 --- a/tests/units/compiler/test_disk_cache.py +++ b/tests/units/compiler/test_disk_cache.py @@ -249,6 +249,7 @@ def _manifest(pages: dict[str, dict], root: Path | None = None, **overrides) -> "all_imports": {}, "pages": pages, "memo_files": {}, + "entry": {"path": None, "digest": None}, "wrap_table": {}, "state_libraries": {}, "app_root": { @@ -288,24 +289,12 @@ def test_globals_mismatch_names_the_changed_input(tmp_path): root=tmp_path, globals_absent=[str(tmp_path / "rxconfig.py")], ) - routes = {"/a", "/b"} assert ( - disk_cache.globals_mismatch( - m, routes=routes, validator=_validator(m), root=tmp_path - ) - is None - ) - # a changed route set -> named added/removed routes - reason = disk_cache.globals_mismatch( - m, routes={"/a", "/c"}, validator=_validator(m), root=tmp_path + disk_cache.globals_mismatch(m, validator=_validator(m), root=tmp_path) is None ) - assert reason is not None - assert "/c" in reason - assert "/b" in reason # a stale reflex version -> named versions reason = disk_cache.globals_mismatch( {**m, "reflex_version": "0.0.0-old"}, - routes=routes, validator=_validator(m), root=tmp_path, ) @@ -313,6 +302,34 @@ def test_globals_mismatch_names_the_changed_input(tmp_path): assert "0.0.0-old" in reason +def test_globals_mismatch_accepts_entrypoint_registration_edits(tmp_path): + """An entry module that only changed its add_page calls is not global.""" + entry = tmp_path / "entry_app.py" + entry.write_text( + 'import reflex as rx\napp = rx.App()\napp.add_page(a, route="/a")\n' + ) + file_entry = page_cache.file_entry(str(entry)) + assert file_entry is not None + m = _manifest( + {}, + root=tmp_path, + files={str(entry): list(file_entry)}, + globals={"reflex": page_cache._reflex_version(), f"app:{entry}": str(entry)}, + entry={"path": str(entry), "digest": page_cache.entrypoint_digest(str(entry))}, + ) + entry.write_text( + 'import reflex as rx\napp = rx.App()\napp.add_page(a, route="/a")\n' + 'app.add_page(b, route="/b")\n' + ) + assert ( + disk_cache.globals_mismatch(m, validator=_validator(m), root=tmp_path) is None + ) + entry.write_text("import reflex as rx\napp = rx.App(enable_state=False)\n") + reason = disk_cache.globals_mismatch(m, validator=_validator(m), root=tmp_path) + assert reason is not None + assert "entry_app.py" in reason + + def test_globals_mismatch_names_a_moved_project(tmp_path): """A manifest only describes the project directory that wrote it. @@ -321,7 +338,7 @@ def test_globals_mismatch_names_a_moved_project(tmp_path): """ m = _manifest({"/a": {}}, root=tmp_path / "original") reason = disk_cache.globals_mismatch( - m, routes={"/a"}, validator=_validator(m), root=tmp_path / "copy" + m, validator=_validator(m), root=tmp_path / "copy" ) assert reason is not None assert "project root changed" in reason @@ -336,15 +353,10 @@ def test_globals_mismatch_names_a_compile_mode_change(tmp_path, monkeypatch): monkeypatch.setenv("REFLEX_ENV_MODE", "dev") m = _manifest({"/a": {}}, root=tmp_path) assert ( - disk_cache.globals_mismatch( - m, routes={"/a"}, validator=_validator(m), root=tmp_path - ) - is None + disk_cache.globals_mismatch(m, validator=_validator(m), root=tmp_path) is None ) monkeypatch.setenv("REFLEX_ENV_MODE", "prod") - reason = disk_cache.globals_mismatch( - m, routes={"/a"}, validator=_validator(m), root=tmp_path - ) + reason = disk_cache.globals_mismatch(m, validator=_validator(m), root=tmp_path) assert reason is not None assert "dev -> prod" in reason @@ -372,9 +384,7 @@ def test_globals_mismatch_validates_stored_inputs_only(tmp_path): ) def mismatch() -> str | None: - return disk_cache.globals_mismatch( - m, routes={"/a"}, validator=_validator(m), root=tmp_path - ) + return disk_cache.globals_mismatch(m, validator=_validator(m), root=tmp_path) # nothing on disk changed -> match, regardless of what a re-recorded # app-import read set would look like in this process @@ -407,27 +417,37 @@ def test_partition_pages_detects_changed_source(tmp_path): _FakePage(route="/a", component=_page_a), _FakePage(route="/b", component=_page_b), ] + registrations = {page.route: "recorded" for page in pages} x, y = tmp_path / "x.py", tmp_path / "y.py" x.write_text("X = 1\n") y.write_text("Y = 1\n") # /a depends on x.py, /b depends on y.py (each recorded in the file table). m = _manifest( - {"/a": {"deps": [str(x)]}, "/b": {"deps": [str(y)]}}, + { + page.route: { + "deps": [str(dep)], + "registration": "recorded", + } + for page, dep in zip(pages, (x, y), strict=True) + }, files={ str(x): page_cache.file_entry(str(x)), str(y): page_cache.file_entry(str(y)), }, ) - assert disk_cache.partition_pages(pages, m, _validator(m)) == [] + assert ( + disk_cache.partition_pages(pages, m, _validator(m), None, registrations) == [] + ) y.write_text("Y = 2\n") - miss = disk_cache.partition_pages(pages, m, _validator(m)) + miss = disk_cache.partition_pages(pages, m, _validator(m), None, registrations) assert {p.route for p in miss} == {"/b"} # With the watcher's changed set, pages depending on none of the changed # paths are hits without touching the filesystem. assert ( - disk_cache.partition_pages(pages, m, _validator(m), changed_hint={str(x)}) == [] + disk_cache.partition_pages(pages, m, _validator(m), {str(x)}, registrations) + == [] ) - miss = disk_cache.partition_pages(pages, m, _validator(m), changed_hint={str(y)}) + miss = disk_cache.partition_pages(pages, m, _validator(m), {str(y)}, registrations) assert {p.route for p in miss} == {"/b"} @@ -530,6 +550,8 @@ def test_write_and_load_manifest(tmp_path, monkeypatch): "component_counts", "app_wraps", "bundle_requests", + "registration", + "memo_path", "is_stateful", "state_slice", "has_memos", @@ -1449,6 +1471,45 @@ def test_state_default_from_new_library_rebuilds_the_app_root(tmp_path, monkeypa assert "recharts" in _app_root_file(web).read_text() +def test_unchanged_component_default_reuses_the_contexts(tmp_path, monkeypatch): + """A recompiled page with the same component default keeps the contexts. + + The recorded slice holds the default serialized the way the contexts file + holds it, whichever libraries were bundled when the slice was taken. + """ + from reflex.compiler import compiler + from reflex.compiler import utils as compiler_utils + + web = _use_tmp_web_dir(tmp_path, monkeypatch) + monkeypatch.setitem(_COMPONENT_DEFAULT, "recharts", True) + app = rx.App() + app.add_page(_page_component_default, route="/dyn") + pages = list(app._unevaluated_pages.values()) + ctx = _compile(pages, app=app) + _stub_externals(app, monkeypatch) + disk_cache.write_manifest(ctx, pages, ctx.all_imports, root=tmp_path) + manifest_path = web / disk_cache._MANIFEST_FILE + manifest = json.loads(manifest_path.read_text()) + _stale_dep(manifest, pages[0].route, str(tmp_path / "view.py")) + manifest_path.write_text(json.dumps(manifest)) + _unregister_state(_FP_HOLDER.pop("_ComponentDefaultState")) + # A fresh process has not bundled the default's library yet. + compiler._reset_compile_registries([]) + out_path = compiler_utils.resolve_path_of_web_dir(compiler_utils.get_context_path()) + out_path.parent.mkdir(parents=True, exist_ok=True) + out_path.write_text("// pre-existing contexts", encoding="utf-8") + try: + assert ( + disk_cache.try_incremental_rebuild( + app, compiler_plugins=[], prerender_routes=False, root=tmp_path + ) + is True + ) + finally: + _unregister_state(_FP_HOLDER.pop("_ComponentDefaultState")) + assert out_path.read_text(encoding="utf-8") == "// pre-existing contexts" + + def _app_root_file(web: Path) -> Path: return web / constants.Dirs.PAGES / constants.PageNames.APP_ROOT @@ -1711,6 +1772,116 @@ def test_unverified_local_state_prediction_asks_for_a_fresh_compile( ) +def _route_set_rebuild(tmp_path, monkeypatch, edit) -> tuple[Any, Any, Path]: + """Fully compile two pages, then rebuild after ``edit`` changes the app. + + Args: + tmp_path: The project root. + monkeypatch: The pytest monkeypatch fixture. + edit: Changes the app's page registrations. + + Returns: + The rebuild result, the app and the web directory. + """ + web = _use_tmp_web_dir(tmp_path, monkeypatch) + app = rx.App() + app.add_page(_page_a, route="/a") + app.add_page(_page_c, route="/c") + pages = list(app._unevaluated_pages.values()) + ctx = _compile(pages, app=app) + _stub_externals(app, monkeypatch) + for page_ctx in ctx.compiled_pages.values(): + assert page_ctx.output_path is not None + Path(page_ctx.output_path).parent.mkdir(parents=True, exist_ok=True) + Path(page_ctx.output_path).write_text(page_ctx.output_code or "") + disk_cache.write_manifest(ctx, pages, ctx.all_imports, root=tmp_path) + edit(app) + evaluated: list[str] = [] + original = disk_cache.CompileContext.compile + + def recording_compile(self, *args, **kwargs): + evaluated.extend(page.route for page in self.pages) + return original(self, *args, **kwargs) + + monkeypatch.setattr(disk_cache.CompileContext, "compile", recording_compile) + result = disk_cache.try_incremental_rebuild( + app, compiler_plugins=[], prerender_routes=False, root=tmp_path + ) + return result, evaluated, web + + +def test_added_page_rebuilds_incrementally(tmp_path, monkeypatch): + """A new route is a miss; the other pages keep their output.""" + from reflex.compiler import utils as compiler_utils + + result, evaluated, web = _route_set_rebuild( + tmp_path, monkeypatch, lambda app: app.add_page(_page_b, route="/b") + ) + assert result is True + assert evaluated == ["b"] + assert Path(compiler_utils.get_page_path("b")).exists() + manifest = json.loads((web / disk_cache._MANIFEST_FILE).read_text()) + assert set(manifest["pages"]) == {"a", "c", "b"} + assert "b" in json.loads((web / constants.Dirs.ROUTES_MANIFEST).read_text()) + + +def test_removed_page_rebuilds_incrementally(tmp_path, monkeypatch): + """A removed route loses its output and manifest entry, nothing else.""" + from reflex.compiler import utils as compiler_utils + + def remove_c(app): + del app._unevaluated_pages["c"] + + result, evaluated, web = _route_set_rebuild(tmp_path, monkeypatch, remove_c) + assert result is True + assert evaluated == [] + assert not Path(compiler_utils.get_page_path("c")).exists() + assert Path(compiler_utils.get_page_path("a")).exists() + manifest = json.loads((web / disk_cache._MANIFEST_FILE).read_text()) + assert set(manifest["pages"]) == {"a"} + + +def test_removed_stateful_page_leaves_the_contexts(tmp_path, monkeypatch): + """A removed page's states leave the contexts file.""" + web = _use_tmp_web_dir(tmp_path, monkeypatch) + app = rx.App() + app.add_page(_page_a, route="/a") + app.add_page(_page_flip, route="/s") + pages = list(app._unevaluated_pages.values()) + ctx = _compile(pages, app=app) + flip_states = ctx.stateful_routes[pages[1].route] + _stub_externals(app, monkeypatch) + disk_cache.write_manifest(ctx, pages, ctx.all_imports, root=tmp_path) + _unregister_state(_FP_HOLDER.pop("_FlipState")) + del app._unevaluated_pages[pages[1].route] + captured = _capture_contexts(monkeypatch) + assert ( + disk_cache.try_incremental_rebuild( + app, compiler_plugins=[], prerender_routes=False, root=tmp_path + ) + is True + ) + [(initial, _, schema_entries)] = captured + assert not set(flip_states) & set(initial) + assert not set(flip_states) & set(schema_entries) + marker = json.loads((web / disk_cache._MANIFEST_FILE).read_text()) + assert set(marker["pages"]) == {"a"} + + +def test_changed_registration_recompiles_only_that_page(tmp_path, monkeypatch): + """A page whose registration changed (here its title) is a miss.""" + import dataclasses + + def retitle_c(app): + app._unevaluated_pages["c"] = dataclasses.replace( + app._unevaluated_pages["c"], title="Renamed" + ) + + result, evaluated, _ = _route_set_rebuild(tmp_path, monkeypatch, retitle_c) + assert result is True + assert evaluated == ["c"] + + def test_incremental_rebuild_copies_assets(tmp_path, monkeypatch): """An assets-only edit is an all-hit rebuild, so the incremental path must run the same assets -> public copy as the full compile. @@ -2074,6 +2245,7 @@ def test_update_manifest_for_misses_keeps_complete_imports(tmp_path, monkeypatch memo_file_entries={}, app_root_inputs=manifest["app_root"], state_libraries={}, + registrations={"/a": "recorded"}, ) written = json.loads((web / disk_cache._MANIFEST_FILE).read_text()) @@ -2096,6 +2268,7 @@ def test_update_manifest_refreshes_imports_without_misses(tmp_path, monkeypatch) memo_file_entries={}, app_root_inputs=manifest["app_root"], state_libraries={}, + registrations={"/a": "recorded"}, ) written = json.loads((web / disk_cache._MANIFEST_FILE).read_text()) diff --git a/tests/units/compiler/test_page_cache.py b/tests/units/compiler/test_page_cache.py index 64569fc54b9..b6b43cd35bc 100644 --- a/tests/units/compiler/test_page_cache.py +++ b/tests/units/compiler/test_page_cache.py @@ -771,3 +771,67 @@ def test_import_time_reads_are_recorded_for_every_importer( finally: _forget_modules("review_shared_helper") assert [str(data) in reads for reads in page_reads] == [True, True] + + +def test_entrypoint_digest_ignores_page_registrations_only(tmp_path): + """Formatting, comments and top-level add_page calls do not change it.""" + entry = tmp_path / "entry_app.py" + + def digest(source: str) -> str | None: + entry.write_text(source) + return page_cache.entrypoint_digest(str(entry)) + + base = digest( + "import reflex as rx\nfrom .pages import a, b\n\napp = rx.App()\n" + 'app.add_page(a, route="/a")\n' + ) + assert base is not None + assert ( + digest( + "import reflex as rx\nfrom .pages import a, b\n\n# pages\napp = rx.App()\n" + 'app.add_page(a, route="/a", title="A")\napp.add_page(b, route="/b")\n' + ) + == base + ) + assert ( + digest( + "import reflex as rx\nfrom .pages import a, b\n\n" + 'app = rx.App(theme=rx.theme(accent_color="red"))\n' + 'app.add_page(a, route="/a")\n' + ) + != base + ) + assert digest("this is not python (") is None + + +def test_entrypoint_digest_ignores_imports_used_only_to_register_pages(tmp_path): + """Importing a page only to register it is a registration; other imports count.""" + entry = tmp_path / "entry_app.py" + + def digest(source: str) -> str | None: + entry.write_text(source) + return page_cache.entrypoint_digest(str(entry)) + + base = digest( + "import reflex as rx\nfrom .themes import dark as theme\n" + "from .pages.a import a\n\napp = rx.App(theme=theme)\n" + 'app.add_page(a, route="/a")\n' + ) + assert ( + digest( + "import reflex as rx\nfrom .themes import dark as theme\n" + "from .pages.a import a\nfrom .pages.b import b\n\n" + 'app = rx.App(theme=theme)\napp.add_page(a, route="/a")\n' + 'app.add_page(b, route="/b")\n' + ) + == base + ) + # The same App call with a theme imported from elsewhere is another app. + assert ( + digest( + "import reflex as rx\nfrom .themes import light as theme\n" + "from .pages.a import a\n\napp = rx.App(theme=theme)\n" + 'app.add_page(a, route="/a")\n' + ) + != base + ) diff --git a/tests/units/test_app.py b/tests/units/test_app.py index 9c2e60bbce4..c8897bb18f3 100644 --- a/tests/units/test_app.py +++ b/tests/units/test_app.py @@ -381,6 +381,16 @@ def test_prepare_404_page_empty_string_metadata_uses_defaults(): assert prepared.page.description == constants.Page404.DESCRIPTION +def test_default_404_page_has_no_source_module(): + """The framework's default 404 page belongs to no user module. + + Otherwise its module comes from the call stack of whatever process + compiles the app, which differs between processes. + """ + prepared = App()._prepare_page(route=constants.Page404.SLUG) + assert prepared.page._source_module is None + + def test_add_page_set_route(app: App, index_page: ComponentCallable): """Test adding a page to an app. From 53b16f94c48bcf05918b222b05b1d1a8d8422f5b Mon Sep 17 00:00:00 2001 From: Farhan <www.mfarvirus@gmail.com> Date: Wed, 21 Oct 2026 02:46:12 +0500 Subject: [PATCH 38/56] perf(compiler): rebuild the contexts for module-level state edits, not all pages Module-level states were global inputs, so editing one forced a full compile, while a state a recompiled page's module newly imported at import time never reached the contexts file: the recorded state files could not include a module the previous compile never saw. The manifest now records the states importing the app registers and the files they come from. A rebuild compares the live set, taken before any page evaluates, and those files; a change rebuilds the contexts, and the pages that read the states recompile through their own dependencies. A change to import-time dynamic-module states, which take names before every page, still takes a full compile. --- reflex/compiler/disk_cache.py | 147 ++++++++++++++++++++---- tests/units/compiler/test_disk_cache.py | 103 +++++++++++++++-- 2 files changed, 215 insertions(+), 35 deletions(-) diff --git a/reflex/compiler/disk_cache.py b/reflex/compiler/disk_cache.py index 921af064d03..483105420f2 100644 --- a/reflex/compiler/disk_cache.py +++ b/reflex/compiler/disk_cache.py @@ -53,7 +53,7 @@ logger = logging.getLogger(__name__) #: Bump when the manifest layout changes (old manifests are then ignored). -_SCHEMA = 20 +_SCHEMA = 21 #: Manifest filename under the web directory. _MANIFEST_FILE = "reflex_compile_cache.json" @@ -748,9 +748,19 @@ def write_manifest( globals_: dict[str, str] = {"reflex": page_cache._reflex_version()} absent: list[str] = [] global_paths = page_cache.global_input_paths(root, pages=pages) - global_paths.update({ - f"state:{path}": path for path in _imported_state_files(compile_ctx, root) - }) + page_states = { + name for names in compile_ctx.stateful_routes.values() for name in names + } + module_states = _record_module_states( + ( + name + for name in RegistrationContext.ensure_context().base_states + if name not in page_states + ), + hasher, + files, + root, + ) for label, path in global_paths.items(): entry = hasher(path) if entry is None: @@ -788,6 +798,9 @@ def write_manifest( "memo_files": memo_files, "wrap_table": wrap_table, "state_libraries": state_libraries, + # The states importing the app registers (see + # ``_record_module_states``). + "module_states": module_states, # The app root's inputs: merged app wraps (by digest, in insertion # order) and the bundled libraries (see ``_refresh_app_root``). "app_root": { @@ -817,32 +830,66 @@ def _compile_mode() -> str: return environment.REFLEX_ENV_MODE.get().value -def _imported_state_files(compile_ctx: CompileContext, root: Path | None) -> set[str]: - """Find source dependencies of states not owned by page evaluation. - - These states have no per-page context slice. Changing their definitions - requires a full compile to refresh the shared frontend contexts. +def _module_state_files(state_names: Iterable[str], root: Path | None) -> set[str]: + """Find the source files that define the given states, with their imports. Args: - compile_ctx: The completed full compile. + state_names: Full names of registered states. root: Project root for dependency discovery. Returns: - First-party source files affecting import-time state definitions. + First-party source files the states' definitions depend on. """ - page_states = { - name for names in compile_ctx.stateful_routes.values() for name in names - } roots = page_cache.first_party_roots(root) + base_states = RegistrationContext.ensure_context().base_states sources = { str(path) - for name, cls in RegistrationContext.ensure_context().base_states.items() - if name not in page_states - and (path := page_cache._class_module_file(cls, roots)) is not None + for name in state_names + if (path := page_cache._class_module_file(base_states[name], roots)) is not None } return page_cache._walk_import_closure(page_cache.build_import_graph(root), sources) +def _record_module_states( + state_names: Iterable[str], + hasher: Callable[[str], page_cache.FileEntry | None], + files: dict[str, page_cache.FileEntry], + root: Path | None, +) -> dict[str, list[str]]: + """Record the states importing the app registers, and their source files. + + They belong to no page, so the contexts file depends on them directly. + + Args: + state_names: Full names of the import-time states. + hasher: A memoized path -> file-entry function. + files: The manifest file table, extended with the source files. + root: Project root for dependency discovery. + + Returns: + ``{"names": [...], "files": [...], "dynamic": [...]}``. + """ + import reflex.istate.dynamic as istate_dynamic + + names = sorted(state_names) + recorded = [] + for path in sorted(_module_state_files(names, root)): + if (entry := hasher(path)) is not None: + files[path] = entry + recorded.append(path) + base_states = RegistrationContext.ensure_context().base_states + return { + "names": names, + "files": recorded, + # Import-time dynamic-module states, which take names before any page. + "dynamic": [ + name + for name in names + if base_states[name].__module__ == istate_dynamic.__name__ + ], + } + + def globals_mismatch( manifest: dict[str, Any], *, @@ -1538,6 +1585,40 @@ def try_incremental_rebuild( for route in [route for route in manifest["pages"] if route not in routes] } route_set_changed = bool(removed_entries) or not routes <= set(manifest["pages"]) + # The states importing the app registered (no page has evaluated yet). + page_states = { + name + for entry in (*manifest["pages"].values(), *removed_entries.values()) + if (state_slice := entry["state_slice"]) + for name in state_slice["schema"] + } + live_module_states = [ + name + for name in RegistrationContext.ensure_context().base_states + if name not in page_states + ] + recorded_module_states = manifest["module_states"] + module_states_changed = sorted(live_module_states) != recorded_module_states[ + "names" + ] or not ( + ( + changed_hint is not None + and changed_hint.isdisjoint(recorded_module_states["files"]) + ) + or validator.unchanged(recorded_module_states["files"]) + ) + if module_states_changed: + import reflex.istate.dynamic as istate_dynamic + + base_states = RegistrationContext.ensure_context().base_states + if sorted( + name + for name in live_module_states + if base_states[name].__module__ == istate_dynamic.__name__ + ) != sorted(recorded_module_states["dynamic"]): + # They take names before any page, so every page's could shift. + _log_fallback("the app's import-time dynamic states changed") + return False miss_pages = partition_pages( pages, manifest, validator, changed_hint, registrations ) @@ -1748,12 +1829,16 @@ def try_incremental_rebuild( # edit leaves them identical and the contexts file is reused untouched. contexts_snapshot: compiler_utils.ContextsSnapshot | None = None state_libraries: dict[str, list[str]] = manifest["state_libraries"] - if route_set_changed or ( - miss_ctx is not None - and any( - route in miss_ctx.stateful_routes - or manifest["pages"].get(route, {}).get("is_stateful") - for route in miss_routes + if ( + route_set_changed + or module_states_changed + or ( + miss_ctx is not None + and any( + route in miss_ctx.stateful_routes + or manifest["pages"].get(route, {}).get("is_stateful") + for route in miss_routes + ) ) ): contexts_snapshot = _contexts_snapshot(app) @@ -1762,6 +1847,8 @@ def try_incremental_rebuild( changed_route = ( "the route set" if route_set_changed + else "the app's states" + if module_states_changed else _changed_state_config_route( manifest, miss_ctx, contexts_snapshot, miss_routes ) @@ -1876,6 +1963,16 @@ def try_incremental_rebuild( app_root_inputs=app_root_inputs, state_libraries=state_libraries, registrations=registrations, + module_states=( + _record_module_states( + live_module_states, + page_cache.make_hasher(), + manifest["files"], + root, + ) + if module_states_changed + else recorded_module_states + ), route_set_changed=route_set_changed, validator=validator, ) @@ -2049,6 +2146,7 @@ def _update_manifest_for_misses( app_root_inputs: dict[str, Any], state_libraries: dict[str, list[str]], registrations: dict[str, str], + module_states: dict[str, list[str]], route_set_changed: bool = False, validator: page_cache.FileValidator | None = None, ) -> None: @@ -2073,6 +2171,7 @@ def _update_manifest_for_misses( app_root_inputs: The app root's recorded inputs after this rebuild. state_libraries: The libraries each state's initial values request. registrations: Each route's registration digest. + module_states: The import-time states and their source files. route_set_changed: Whether pages were added or removed. validator: The validator used for this rebuild, for its refreshed entries. @@ -2121,9 +2220,11 @@ def _update_manifest_for_misses( if ( manifest["app_root"] != app_root_inputs or manifest["state_libraries"] != state_libraries + or manifest["module_states"] != module_states ): manifest["app_root"] = app_root_inputs manifest["state_libraries"] = state_libraries + manifest["module_states"] = module_states dirty = True if dirty: # Keep only the stored wraps a page or the app root still uses. diff --git a/tests/units/compiler/test_disk_cache.py b/tests/units/compiler/test_disk_cache.py index 6784123441b..0b9f3a711a8 100644 --- a/tests/units/compiler/test_disk_cache.py +++ b/tests/units/compiler/test_disk_cache.py @@ -115,8 +115,15 @@ def _unregister_state(cls: type[BaseState]) -> None: @pytest.mark.parametrize("edited_module", ["state", "helper"]) -def test_imported_state_edit_invalidates_contexts(tmp_path, monkeypatch, edited_module): - """Module-level state edits need a full compile to refresh shared contexts.""" +def test_imported_state_edit_rebuilds_the_contexts( + tmp_path, monkeypatch, edited_module +): + """A module-level state edit rebuilds the contexts, not every page. + + The edited state's module (or a helper it imports) is an input of the + contexts file; the pages that read the state recompile through their own + recorded dependencies. + """ _use_tmp_web_dir(tmp_path, monkeypatch) source = tmp_path / "state_view.py" helper_source = tmp_path / "state_helper.py" @@ -133,30 +140,99 @@ def test_imported_state_edit_invalidates_contexts(tmp_path, monkeypatch, edited_ "def index():\n" " return rx.el.div(ImportedState.value)\n" ) - source.write_text(code) - module = ModuleType("cache_review_state_view") - module.__file__ = str(source) - monkeypatch.setitem(sys.modules, module.__name__, module) - exec(compile(code, str(source), "exec"), module.__dict__) + + def load(source_code: str) -> ModuleType: + """Execute the state module, as importing the app does. + + Args: + source_code: The module source. + + Returns: + The module. + """ + source.write_text(source_code) + module = ModuleType("cache_review_state_view") + module.__file__ = str(source) + monkeypatch.setitem(sys.modules, module.__name__, module) + exec(compile(source_code, str(source), "exec"), module.__dict__) + return module + + module = load(code) try: app = rx.App() app.add_page(module.index, route="/state") + app.add_page(_page_a, route="/a") pages = list(app._unevaluated_pages.values()) ctx = _compile(pages, app) assert not ctx.stateful_routes + _stub_externals(app, monkeypatch) disk_cache.write_manifest(ctx, pages, ctx.all_imports, root=tmp_path) + # The edit, then a fresh import of the app, as the daemon child does. + _unregister_state(module.ImportedState) if edited_module == "state": - source.write_text(code.replace("= DEFAULT", "= 'new default'")) + module = load(code.replace("= DEFAULT", "= 'new default'")) else: helper_source.write_text("DEFAULT = 'new default'\n") - _stub_externals(app, monkeypatch) - assert not disk_cache.try_incremental_rebuild( - app, compiler_plugins=[], prerender_routes=False, root=tmp_path + helper.__dict__["DEFAULT"] = "new default" + module = load(code) + app._unevaluated_pages["state"] = dataclasses.replace( + app._unevaluated_pages["state"], component=module.index ) + captured = _capture_contexts(monkeypatch) + assert ( + disk_cache.try_incremental_rebuild( + app, compiler_plugins=[], prerender_routes=False, root=tmp_path + ) + is True + ) + [(initial, _, _)] = captured + [values] = [ + values + for name, values in initial.items() + if name.endswith("imported_state") + ] + assert "new default" in json.dumps(values) finally: _unregister_state(module.ImportedState) +def test_new_import_time_state_rebuilds_the_contexts(tmp_path, monkeypatch): + """A state a recompiled page's module newly imports reaches the contexts. + + Importing the app registers it before any page evaluates, so no page + defines it; only comparing the import-time states sees it. + """ + web = _use_tmp_web_dir(tmp_path, monkeypatch) + app = rx.App() + app.add_page(_page_a, route="/a") + pages = list(app._unevaluated_pages.values()) + ctx = _compile(pages, app) + _stub_externals(app, monkeypatch) + disk_cache.write_manifest(ctx, pages, ctx.all_imports, root=tmp_path) + manifest_path = web / disk_cache._MANIFEST_FILE + manifest = json.loads(manifest_path.read_text()) + _stale_dep(manifest, "a", str(tmp_path / "view.py")) + manifest_path.write_text(json.dumps(manifest)) + # The edited page module now imports a module defining a new state. + new_state: Any = type( + "_ImportTimeState", + (rx.State,), + {"__annotations__": {"value": int}, "value": 3, "__module__": "fp_mod_x"}, + ) + captured = _capture_contexts(monkeypatch) + try: + assert ( + disk_cache.try_incremental_rebuild( + app, compiler_plugins=[], prerender_routes=False, root=tmp_path + ) + is True + ) + finally: + _unregister_state(new_state) + [(initial, _, _)] = captured + assert new_state.get_full_name() in initial + + def test_imports_round_trip(): imports = { "react": [ImportVar("useEffect"), ImportVar("Fragment", is_default=False)], @@ -252,6 +328,7 @@ def _manifest(pages: dict[str, dict], root: Path | None = None, **overrides) -> "entry": {"path": None, "digest": None}, "wrap_table": {}, "state_libraries": {}, + "module_states": {"names": [], "files": [], "dynamic": []}, "app_root": { "wraps": {}, "libraries": [], @@ -619,7 +696,7 @@ def test_unchanged_pages_compile_identically(tmp_path, monkeypatch): _CONTEXTS_STUB = "// contexts stub" -_TEST_STATE_MODULES = (__name__, "fp_mod_x") +_TEST_STATE_MODULES = (__name__, "fp_mod_x", "cache_review_state_view") def _scoped_contexts_snapshot(app) -> ContextsSnapshot | None: @@ -2246,6 +2323,7 @@ def test_update_manifest_for_misses_keeps_complete_imports(tmp_path, monkeypatch app_root_inputs=manifest["app_root"], state_libraries={}, registrations={"/a": "recorded"}, + module_states=manifest["module_states"], ) written = json.loads((web / disk_cache._MANIFEST_FILE).read_text()) @@ -2269,6 +2347,7 @@ def test_update_manifest_refreshes_imports_without_misses(tmp_path, monkeypatch) app_root_inputs=manifest["app_root"], state_libraries={}, registrations={"/a": "recorded"}, + module_states=manifest["module_states"], ) written = json.loads((web / disk_cache._MANIFEST_FILE).read_text()) From 5ba0f855abf6df8177df5948a5512a2272dc66fc Mon Sep 17 00:00:00 2001 From: Farhan <www.mfarvirus@gmail.com> Date: Wed, 21 Oct 2026 02:53:24 +0500 Subject: [PATCH 39/56] perf(compiler): predict ComponentState instance numbers per page A re-evaluated page drew ComponentState numbers from the counter of a fresh import, so a page sharing a ComponentState with an earlier kept page always failed the prediction and recompiled in a fresh process. Before each re-evaluated page, advance each counter past the numbers earlier kept pages recorded. --- reflex/compiler/disk_cache.py | 95 ++++++++++++- tests/units/compiler/test_disk_cache.py | 172 +++++++++++++++++++----- 2 files changed, 232 insertions(+), 35 deletions(-) diff --git a/reflex/compiler/disk_cache.py b/reflex/compiler/disk_cache.py index 483105420f2..73b98f72035 100644 --- a/reflex/compiler/disk_cache.py +++ b/reflex/compiler/disk_cache.py @@ -31,7 +31,7 @@ from reflex_base import constants from reflex_base.components.dynamic import record_bundle_requests from reflex_base.environment import environment -from reflex_base.plugins import CompileContext, CompilerHooks +from reflex_base.plugins import CompileContext, CompilerHooks, Plugin from reflex_base.registry import RegistrationContext from reflex_base.utils import log from reflex_base.utils.format import json_dumps @@ -1136,6 +1136,90 @@ def _shifting_dynamic_claims( return None +_INSTANCE_NUMBER = re.compile(r"_n(\d+)$") + + +def _numbers_before_miss_pages( + pages: Sequence[PageDefinition], manifest: dict[str, Any], miss_routes: set[str] +) -> dict[str, dict[str, int]]: + """The highest ``ComponentState`` numbers kept pages drew before each miss. + + Args: + pages: All current page definitions (in compile order). + manifest: The loaded manifest. + miss_routes: Routes re-evaluated in this rebuild. + + Returns: + For each re-evaluated route, the dotted ``ComponentState`` class path + mapped to the highest instance number drawn by an earlier kept page. + """ + numbers: dict[str, int] = {} + before: dict[str, dict[str, int]] = {} + for page in pages: + if page.route in miss_routes: + if numbers: + before[page.route] = dict(numbers) + continue + # A ComponentState claim requests its dotted class path; a local + # state's request is a bare class name. + for name, requested in manifest["pages"][page.route]["dynamic_claims"]: + if "." in requested and (match := _INSTANCE_NUMBER.search(name)): + numbers[requested] = max(numbers.get(requested, 0), int(match[1])) + return before + + +def _resolve_class_path(path: str) -> Any: + """Look up a loaded class by its dotted ``module.qualname`` path. + + Args: + path: The dotted path. + + Returns: + The object at the path, or None. + """ + parts = path.split(".") + for split in range(len(parts) - 1, 0, -1): + if (obj := sys.modules.get(".".join(parts[:split]))) is not None: + for part in parts[split:]: + obj = getattr(obj, part, None) + return obj + return None + + +@dataclasses.dataclass(frozen=True, slots=True) +class _ComponentStateNumbersPlugin(Plugin): + """Advance ``ComponentState`` counters before each re-evaluated page. + + Instance numbers are handed out in page order and only grow. Before a + re-evaluated page, a full compile's counter is past every number drawn + by the earlier pages: the kept ones (recorded) and the re-evaluated ones + (drawn live in this rebuild). + """ + + numbers_before: dict[str, dict[str, int]] + + def eval_page( + self, page_fn: Any, /, *, page: PageDefinition, **kwargs: Any + ) -> None: + """Advance the counters to the numbers kept pages drew before this page. + + Args: + page_fn: The page function (unused). + page: The page about to be evaluated. + kwargs: Other hook arguments (unused). + """ + from reflex.state import ComponentState + + for path, number in self.numbers_before.get(page.route, {}).items(): + cls = _resolve_class_path(path) + if ( + isinstance(cls, type) + and issubclass(cls, ComponentState) + and cls._per_component_state_instance_count < number + ): + cls._per_component_state_instance_count = number + + def _claim_recorded_dynamic_names( manifest: dict[str, Any], miss_routes: set[str] ) -> None: @@ -1695,14 +1779,21 @@ def try_incremental_rebuild( # afterwards so post-evaluation closures (memo entries, page deps) # include the newly loaded modules. modules_before = set(page_cache._loaded_first_party_modules(resolved_root)) + page_plugins: Sequence[Plugin] = compiler_plugins if _predict_dynamic_names: _claim_recorded_dynamic_names(manifest, miss_routes) + page_plugins = ( + _ComponentStateNumbersPlugin( + _numbers_before_miss_pages(pages, manifest, miss_routes) + ), + *compiler_plugins, + ) miss_ctx = CompileContext( app=app, pages=miss_pages, hooks=CompilerHooks( - plugins=default_page_plugins(style=app.style, plugins=compiler_plugins) + plugins=default_page_plugins(style=app.style, plugins=page_plugins) ), bundle_base=bundle_base, ) diff --git a/tests/units/compiler/test_disk_cache.py b/tests/units/compiler/test_disk_cache.py index 0b9f3a711a8..fe7abfeb63c 100644 --- a/tests/units/compiler/test_disk_cache.py +++ b/tests/units/compiler/test_disk_cache.py @@ -16,6 +16,7 @@ from reflex_base.environment import environment from reflex_base.plugins import CompileContext, CompilerHooks, PageContext from reflex_base.registry import RegistrationContext +from reflex_base.utils.format import to_snake_case from reflex_base.utils.imports import ImportVar import reflex as rx @@ -1725,30 +1726,27 @@ class UniqueToTwoState(rx.State): return rx.el.div(rx.el.p(CollideState.two), _footer()) -def _unregister_collide_states() -> None: - """Drop every registered ``CollideState``, like the daemon child's reset.""" - for name, cls in list(RegistrationContext.ensure_context().base_states.items()): - if "collide_state" in name or "unique_to_two_state" in name: - _unregister_state(cls) - - -def _rebuild_second_collide_page( +def _rebuild_second_page( tmp_path, monkeypatch, + first: Callable[[], Component], + second: Callable[[], Component], *, predict: bool, - two_extra: bool = False, - two_unique: bool = False, + markers: tuple[str, ...], + prepare: Callable[[], None], ) -> tuple[Any, list[str], list[str]]: - """Compile two pages defining a same-named local state, then rebuild the second. + """Compile two pages, then rebuild the second as if its module changed. Args: tmp_path: The project root. monkeypatch: The pytest monkeypatch fixture. + first: The first page. + second: The second page. predict: Whether the rebuild predicts dynamic state names. - two_extra: Whether the edited second page defines one more such state. - two_unique: Whether the edited second page defines a state no other - page names the same way. + markers: Name parts of the dynamic states the pages define, dropped + before the rebuild like the daemon child's fresh import does. + prepare: Resets page bookkeeping before the rebuild. Returns: The rebuild result, the second page's states before the edit, and its @@ -1756,12 +1754,11 @@ def _rebuild_second_collide_page( """ web = _use_tmp_web_dir(tmp_path, monkeypatch) app = rx.App() - app.add_page(_page_collide_one, route="/one") - app.add_page(_page_collide_two, route="/two") + app.add_page(first, route="/one") + app.add_page(second, route="/two") pages = list(app._unevaluated_pages.values()) ctx = _compile(pages, app=app) - two_states = ctx.stateful_routes[pages[1].route] - assert two_states[0].endswith("collide_state_0") + second_states = ctx.stateful_routes[pages[1].route] _stub_externals(app, monkeypatch) disk_cache.write_manifest(ctx, pages, ctx.all_imports, root=tmp_path) manifest_path = web / disk_cache._MANIFEST_FILE @@ -1770,34 +1767,80 @@ def _rebuild_second_collide_page( # Keep the first page out of the miss set, as if it lived in another module. manifest["pages"][pages[0].route]["has_memos"] = False manifest_path.write_text(json.dumps(manifest)) - _unregister_collide_states() - monkeypatch.setitem(_COLLIDE, "one", 0) - monkeypatch.setitem(_COLLIDE, "two", 0) - monkeypatch.setitem(_COLLIDE, "two_extra", two_extra) - monkeypatch.setitem(_COLLIDE, "two_unique", two_unique) + _drop_dynamic_states(*markers) + prepare() monkeypatch.setattr(disk_cache, "_predict_dynamic_names", predict) try: result = disk_cache.try_incremental_rebuild( app, compiler_plugins=[], prerender_routes=False, root=tmp_path ) finally: - _unregister_collide_states() - import reflex.istate.dynamic as istate_dynamic - - for name in [ - n - for n in vars(istate_dynamic) - if "CollideState" in n or "UniqueToTwoState" in n - ]: - delattr(istate_dynamic, name) + _drop_dynamic_states(*markers) refreshed = json.loads(manifest_path.read_text()) return ( result, - list(two_states), + list(second_states), list(refreshed["pages"][pages[1].route]["state_slice"]["schema"]), ) +def _drop_dynamic_states(*markers: str) -> None: + """Unregister dynamic-module states whose names contain a marker. + + Args: + markers: Class name parts, as in the ``reflex.istate.dynamic`` names. + """ + import reflex.istate.dynamic as istate_dynamic + + for name, cls in list(RegistrationContext.ensure_context().base_states.items()): + if any(to_snake_case(marker) in name for marker in markers): + _unregister_state(cls) + for name in [n for n in vars(istate_dynamic) if any(m in n for m in markers)]: + delattr(istate_dynamic, name) + + +def _rebuild_second_collide_page( + tmp_path, + monkeypatch, + *, + predict: bool, + two_extra: bool = False, + two_unique: bool = False, +) -> tuple[Any, list[str], list[str]]: + """Compile two pages defining a same-named local state, then rebuild the second. + + Args: + tmp_path: The project root. + monkeypatch: The pytest monkeypatch fixture. + predict: Whether the rebuild predicts dynamic state names. + two_extra: Whether the edited second page defines one more such state. + two_unique: Whether the edited second page defines a state no other + page names the same way. + + Returns: + The rebuild result, the second page's states before the edit, and its + recorded states after the rebuild. + """ + + def prepare() -> None: + monkeypatch.setitem(_COLLIDE, "one", 0) + monkeypatch.setitem(_COLLIDE, "two", 0) + monkeypatch.setitem(_COLLIDE, "two_extra", two_extra) + monkeypatch.setitem(_COLLIDE, "two_unique", two_unique) + + result, before, after = _rebuild_second_page( + tmp_path, + monkeypatch, + _page_collide_one, + _page_collide_two, + predict=predict, + markers=("CollideState", "UniqueToTwoState"), + prepare=prepare, + ) + assert before[0].endswith("collide_state_0") + return result, before, after + + def test_incremental_rebuild_keeps_colliding_local_state_names(tmp_path, monkeypatch): """A page whose local state took a collision suffix keeps it when it misses. @@ -1849,6 +1892,69 @@ def test_unverified_local_state_prediction_asks_for_a_fresh_compile( ) +class SharedCounter(rx.ComponentState): + """A ``ComponentState`` both counter pages create.""" + + value: int = 0 + + @classmethod + def get_component(cls, *children, **props) -> Component: + """Render the instance's value. + + Args: + children: Unused. + props: Unused. + + Returns: + The component. + """ + return rx.el.p(cls.value) + + +_COUNTER = {"one": 0, "two": 0} + + +def _page_counter_one() -> Component: + _COUNTER["one"] += 1 + return rx.el.div(SharedCounter.create(), _footer()) + + +def _page_counter_two() -> Component: + _COUNTER["two"] += 1 + return rx.el.div(SharedCounter.create(), _footer()) + + +def test_predicted_component_state_numbers_rebuild_only_the_changed_page( + tmp_path, monkeypatch +): + """A re-evaluated page draws the ``ComponentState`` number of a full compile. + + Instance numbers are handed out in page order. A fresh import starts the + counter at zero, so the rebuild advances it past the number the first page + drew instead of evaluating that page again. + """ + monkeypatch.setattr(SharedCounter, "_per_component_state_instance_count", 0) + + def prepare() -> None: + monkeypatch.setattr(SharedCounter, "_per_component_state_instance_count", 0) + monkeypatch.setitem(_COUNTER, "one", 0) + monkeypatch.setitem(_COUNTER, "two", 0) + + result, before, after = _rebuild_second_page( + tmp_path, + monkeypatch, + _page_counter_one, + _page_counter_two, + predict=True, + markers=("SharedCounter_n",), + prepare=prepare, + ) + assert result is True + assert before[0].endswith("shared_counter_n2") + assert after == before + assert (_COUNTER["one"], _COUNTER["two"]) == (0, 1) + + def _route_set_rebuild(tmp_path, monkeypatch, edit) -> tuple[Any, Any, Path]: """Fully compile two pages, then rebuild after ``edit`` changes the app. From 2ce8c71536d2735cfe4fcb99464e4218f2015846 Mon Sep 17 00:00:00 2001 From: Farhan <www.mfarvirus@gmail.com> Date: Wed, 21 Oct 2026 02:55:45 +0500 Subject: [PATCH 40/56] perf(compile-daemon): clear only the model registries the parent loaded Every hot reload child imported sqlmodel and reflex.model only to clear their registries, 0.3 s of a 0.4 s reload for an app without models. A model layer a page imports first starts empty, as in a fresh process. --- reflex/utils/compile_daemon.py | 25 +++++++++++------------- tests/units/utils/test_compile_daemon.py | 12 ++++++++++++ 2 files changed, 23 insertions(+), 14 deletions(-) diff --git a/reflex/utils/compile_daemon.py b/reflex/utils/compile_daemon.py index 0595450bc15..100477628fa 100644 --- a/reflex/utils/compile_daemon.py +++ b/reflex/utils/compile_daemon.py @@ -608,22 +608,19 @@ def _reset_model_metadata() -> None: forked child inherits warm and populated. Re-evaluating a page that defines such a model would then raise ``Table '...' is already defined``. A fresh respawn never hits this (empty registry); resetting here restores that - fresh-process contract. Best-effort: apps without a DB layer have nothing to - clear. + fresh-process contract. Only a loaded model layer holds tables; one a page + imports first starts empty, as in a fresh process. """ - with contextlib.suppress(Exception): - import sqlmodel - + if (sqlmodel := sys.modules.get("sqlmodel")) is not None: sqlmodel.SQLModel.metadata.clear() - with contextlib.suppress(Exception): - from reflex.model import Model - - Model.metadata.clear() - with contextlib.suppress(Exception): - from reflex.model import ModelRegistry - - ModelRegistry.models.clear() - ModelRegistry._metadata = None + if (model := sys.modules.get("reflex.model")) is None: + return + # Without the DB extras these are stubs that have no registries. + with contextlib.suppress(AttributeError): + model.Model.metadata.clear() + with contextlib.suppress(AttributeError): + model.ModelRegistry.models.clear() + model.ModelRegistry._metadata = None #: Name of the env var carrying the changed-path hint to a cold compile child. diff --git a/tests/units/utils/test_compile_daemon.py b/tests/units/utils/test_compile_daemon.py index 12af310c39a..867664211d1 100644 --- a/tests/units/utils/test_compile_daemon.py +++ b/tests/units/utils/test_compile_daemon.py @@ -345,6 +345,18 @@ def test_first_party_module_names_includes_namespace_packages(tmp_path, monkeypa sys.modules.pop("ns_under_test.leaf", None) +def test_reset_model_metadata_skips_unloaded_model_layers(monkeypatch): + """A child of an app that never loaded the model layer does not import it. + + Importing it only to clear its registries cost every hot reload ~0.3 s. + """ + for name in ("sqlmodel", "reflex.model"): + monkeypatch.delitem(sys.modules, name, raising=False) + compile_daemon._reset_model_metadata() + assert "sqlmodel" not in sys.modules + assert "reflex.model" not in sys.modules + + @pytest.mark.skipif(not hasattr(os, "fork"), reason="requires os.fork (POSIX)") def test_reset_model_metadata_allows_table_redefinition(): """After reset, an ``rx.Model`` table can be redefined without conflict. From 564c32c6d2202c56995f1aae0ab8c21912f4ebee Mon Sep 17 00:00:00 2001 From: Farhan <www.mfarvirus@gmail.com> Date: Wed, 21 Oct 2026 03:06:03 +0500 Subject: [PATCH 41/56] perf(compiler): track a page's own module by the code its function reads A page function now depends on its slice of its module instead of the whole file: the statements that run at import, the import-time parts of every definition, every class, and the module-level functions it reaches by name. Editing one page function recompiles only that page, also for pages defined in the app entry module, whose global digest leaves page functions out. A decorator that may register the function, or a computed global lookup, keeps the affected code in every slice. Imports stay tracked per file. --- reflex/compiler/disk_cache.py | 34 ++- reflex/compiler/page_cache.py | 340 ++++++++++++++++++++++-- tests/units/compiler/test_disk_cache.py | 58 ++++ tests/units/compiler/test_page_cache.py | 154 +++++++++++ 4 files changed, 553 insertions(+), 33 deletions(-) diff --git a/reflex/compiler/disk_cache.py b/reflex/compiler/disk_cache.py index 73b98f72035..b26c785598a 100644 --- a/reflex/compiler/disk_cache.py +++ b/reflex/compiler/disk_cache.py @@ -53,7 +53,7 @@ logger = logging.getLogger(__name__) #: Bump when the manifest layout changes (old manifests are then ignored). -_SCHEMA = 21 +_SCHEMA = 22 #: Manifest filename under the web directory. _MANIFEST_FILE = "reflex_compile_cache.json" @@ -630,7 +630,7 @@ def manifest_dependency_files(manifest: dict[str, Any]) -> set[str]: """ files: set[str] = set() for entry in manifest.get("pages", {}).values(): - files.update(entry.get("deps", ())) + files.update(page_cache.dep_file(dep) for dep in entry.get("deps", ())) files.update( path for label, path in manifest.get("globals", {}).items() if label != "reflex" ) @@ -953,6 +953,29 @@ def globals_mismatch( return None +def _with_hinted_slices( + changed_hint: set[str] | None, manifest: dict[str, Any] +) -> set[str] | None: + """Add the recorded function slices of the files the watcher reported. + + Args: + changed_hint: Resolved paths the watcher saw change, or None if unknown. + manifest: The loaded manifest. + + Returns: + The hint with every page dependency that reads a reported file. + """ + if not changed_hint: + return changed_hint + return changed_hint | { + dep + for entry in manifest["pages"].values() + for dep in entry["deps"] + if page_cache.SLICE_SEPARATOR in dep + and page_cache.dep_file(dep) in changed_hint + } + + def partition_pages( pages: Sequence[PageDefinition], manifest: dict[str, Any], @@ -1306,14 +1329,15 @@ def _changed_dependency_files( changed_hint: Resolved paths the watcher saw change, or None if unknown. Returns: - The set of changed dependency file paths. + The set of changed dependency file paths (a changed function slice + as its file). """ candidates: Iterable[str] = { path for entry in manifest["pages"].values() for path in entry["deps"] } if changed_hint is not None: candidates = changed_hint.intersection(candidates) - return {path for path in candidates if validator.changed(path)} + return {page_cache.dep_file(path) for path in candidates if validator.changed(path)} def _module_source_file(module_name: str | None) -> str | None: @@ -1654,7 +1678,7 @@ def try_incremental_rebuild( if registrations is None: registrations = registration_digests(pages) validator = page_cache.FileValidator(manifest["files"]) - changed_hint = _changed_hint + changed_hint = _with_hinted_slices(_changed_hint, manifest) if ( reason := globals_mismatch(manifest, validator=validator, root=root) diff --git a/reflex/compiler/page_cache.py b/reflex/compiler/page_cache.py index a7c66c92f14..e65197e77ee 100644 --- a/reflex/compiler/page_cache.py +++ b/reflex/compiler/page_cache.py @@ -11,6 +11,7 @@ import ast import builtins import contextlib +import dataclasses import hashlib import importlib import importlib.util @@ -525,41 +526,275 @@ def _loaded_names(nodes: Iterable[ast.AST]) -> set[str]: } -def entrypoint_digest(path: str) -> str | None: - """Digest the app entry module apart from its top-level page registrations. +#: Separates a module file from a module-level function name in a dependency +#: on the parts of the module that function can read (see +#: :func:`function_slice_digest`). +SLICE_SEPARATOR = "::" + +#: Builtins that reach module globals no syntax tree names. +_DYNAMIC_GLOBAL_NAMES = frozenset({ + "globals", + "vars", + "locals", + "eval", + "exec", + "__import__", +}) + +#: Decorators that keep the function they wrap to themselves at import time. +_INERT_DECORATORS = frozenset({"page", "memo", "cache", "lru_cache"}) - Left out are top-level ``<app>.add_page(...)`` statements and the imported - names no other statement reads (a page imported only to be registered), - plus what the syntax tree never holds (comments, formatting). - Two versions with the same digest differ only in the pages they register, - which the compile cache compares page by page; any other edit can change - every page. + +def dep_file(dep: str) -> str: + """Return the file a dependency reads. Args: - path: The entry module's file. + dep: A file path, or a function slice (``<file>::<function>``). Returns: - The digest, or None when the file cannot be read or parsed. + The file path. + """ + file, separator, _ = dep.rpartition(SLICE_SEPARATOR) + return file if separator else dep + + +@dataclasses.dataclass(frozen=True, slots=True) +class _ModulePart: + """One top-level statement of a module, as a slice sees it.""" + + # The statement's syntax tree and the names it reads. + dump: str + loads: frozenset[str] + # A function or class: its name, and the parts that run at import time + # (decorators, defaults, bases) with the names they read. + name: str | None = None + is_class: bool = False + header: str = "" + header_loads: frozenset[str] = frozenset() + # An import: each bound name (``*`` for a star import) and its syntax tree. + aliases: tuple[tuple[str, str], ...] = () + + +def _is_inert_decorator(decorator: ast.expr) -> bool: + """Whether a decorator keeps the function it wraps to itself. + + Args: + decorator: The decorator expression. + + Returns: + True for a known decorator such as ``rx.page`` or ``functools.cache``. + """ + if isinstance(decorator, ast.Call): + decorator = decorator.func + name = getattr(decorator, "attr", None) or getattr(decorator, "id", None) + return name in _INERT_DECORATORS + + +def _module_part(statement: ast.stmt) -> _ModulePart: + """Describe one top-level statement for slicing. + + Args: + statement: The statement. + + Returns: + Its slice description. + """ + if isinstance(statement, (ast.Import, ast.ImportFrom)): + origin = ( + f"{statement.module}:{statement.level}:" + if isinstance(statement, ast.ImportFrom) + else "" + ) + return _ModulePart( + dump="", + loads=frozenset(), + aliases=tuple( + ( + (alias.asname or alias.name).partition(".")[0], + origin + ast.dump(alias), + ) + for alias in statement.names + ), + ) + dump = ast.dump(statement) + loads = frozenset(_loaded_names([statement])) + # A decorator other than the inert ones may hand the function to code + # that runs at import (a registry), so it counts as import-time code. + if isinstance( + statement, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef) + ) and all(_is_inert_decorator(d) for d in statement.decorator_list): + header: list[ast.expr] = list(statement.decorator_list) + if isinstance(statement, ast.ClassDef): + header.extend(statement.bases) + header.extend(keyword.value for keyword in statement.keywords) + else: + header.extend(statement.args.defaults) + header.extend(d for d in statement.args.kw_defaults if d is not None) + return _ModulePart( + dump=dump, + loads=loads, + name=statement.name, + is_class=isinstance(statement, ast.ClassDef), + header="".join(ast.dump(node) for node in header), + header_loads=frozenset(_loaded_names(header)), + ) + return _ModulePart(dump=dump, loads=loads) + + +#: File -> (stat key, parts) for the modules sliced in this process. +_module_parts_cache: dict[str, tuple[tuple[int, int], list[_ModulePart]]] = {} + + +def _module_parts(path: str) -> list[_ModulePart] | None: + """Parse a module into slice parts, leaving out its page registrations. + + Args: + path: The module file. + + Returns: + The parts in source order, or None when the file cannot be read or + parsed. """ + key = _stat_key(path) + cached = _module_parts_cache.get(path) + if key is not None and cached is not None and cached[0] == key: + return cached[1] try: - tree = ast.parse(Path(path).read_text(encoding="utf-8")) + tree = ast.parse(Path(path).read_bytes()) except (OSError, SyntaxError, ValueError): return None - others = [s for s in tree.body if not _is_page_registration(s)] - used = _loaded_names(others) - body: list[ast.stmt] = [] - for statement in others: - if isinstance(statement, (ast.Import, ast.ImportFrom)): - statement.names = [ - alias - for alias in statement.names - if (alias.asname or alias.name).partition(".")[0] in used - ] - if not statement.names: - continue - body.append(statement) - tree.body = body - return hashlib.sha256(ast.dump(tree).encode()).hexdigest() + parts = [_module_part(s) for s in tree.body if not _is_page_registration(s)] + if key is not None: + _module_parts_cache[path] = (key, parts) + return parts + + +def _definition_closure( + definitions: Mapping[str, Sequence[_ModulePart]], starts: Iterable[str] +) -> set[str]: + """Name the definitions reachable from ``starts`` through the names they read. + + Args: + definitions: The module's definitions by name. + starts: The names to start from. + + Returns: + The reachable definition names. + """ + closure: set[str] = set() + stack = [name for name in starts if name in definitions] + while stack: + name = stack.pop() + if name in closure: + continue + closure.add(name) + stack.extend( + load + for part in definitions[name] + for load in part.loads + if load in definitions and load not in closure + ) + return closure + + +def _sliced_texts(part: _ModulePart, used: set[str], closure: set[str]) -> list[str]: + """Return what one module part adds to a slice. + + Args: + part: The part. + used: The names the slice's code reads. + closure: The definitions in the slice. + + Returns: + The texts to digest. + """ + if part.aliases: + return [ + dump + for bound, dump in part.aliases + if bound == "*" or bound in used or bound in closure + ] + if part.name is None or part.name in closure: + return [part.dump] + return [part.header] if part.header else [] + + +def _slice_digest(parts: Sequence[_ModulePart], seeds: Iterable[str]) -> str: + """Digest the parts of a module that code starting at ``seeds`` can read. + + That is every statement that runs at import, the import-time parts of + every definition, and the whole definitions named (transitively) by those + or by ``seeds``. Imports count only for the names that code reads. Code + reaching module globals by computed name makes every definition count. + Like a dependency across modules, a call-time mutation of a shared object + is not tracked. + + Args: + parts: The module's parts (see :func:`_module_parts`). + seeds: The definition names to start from. + + Returns: + The hex digest. + """ + definitions: dict[str, list[_ModulePart]] = {} + always: set[str] = set() + for part in parts: + if part.name is not None: + definitions.setdefault(part.name, []).append(part) + always |= part.header_loads + else: + always |= part.loads + closure = _definition_closure(definitions, (*seeds, *always)) + used = always.union(*(part.loads for name in closure for part in definitions[name])) + if not used.isdisjoint(_DYNAMIC_GLOBAL_NAMES): + closure = set(definitions) + digest = hashlib.sha256() + for part in parts: + for text in _sliced_texts(part, used, closure): + digest.update(text.encode()) + digest.update(b"\n") + return digest.hexdigest() + + +def function_slice_digest(path: str, name: str) -> str | None: + """Digest the parts of a module a module-level function can read. + + Every class counts: a page reaches the states and components of its own + module through its rendered tree, not only through names it reads. + + Args: + path: The module file. + name: The function's name. + + Returns: + The digest, or None when the file cannot be parsed or defines no + such module-level function or class. + """ + parts = _module_parts(path) + if parts is None or not any(part.name == name for part in parts): + return None + classes = [part.name for part in parts if part.name is not None and part.is_class] + return _slice_digest(parts, (name, *classes)) + + +def entrypoint_digest(path: str) -> str | None: + """Digest the app entry module apart from its pages. + + Left out are top-level ``<app>.add_page(...)`` statements and the code only + they read: page functions defined in the module (each page tracks its own + slice, see :func:`function_slice_digest`) and imported pages. So is what + the syntax tree never holds (comments, formatting). Two versions with the + same digest differ only in their pages, which the compile cache compares + page by page; any other edit can change every page. + + Args: + path: The entry module's file. + + Returns: + The digest, or None when the file cannot be read or parsed. + """ + parts = _module_parts(path) + return None if parts is None else _slice_digest(parts, ()) def _app_entrypoint_file(root: Path | None = None) -> Path | None: @@ -1132,8 +1367,17 @@ def file_entry(path: str) -> FileEntry | None: path: The file path. Returns: - ``[sha256, st_mtime_ns, st_size]``, or None if unreadable. + ``[sha256, st_mtime_ns, st_size]``, or None if unreadable. For a + function slice (see :data:`SLICE_SEPARATOR`), the slice digest and + the stat of its file. """ + file, separator, name = path.rpartition(SLICE_SEPARATOR) + if separator: + key = _stat_key(file) + digest = function_slice_digest(file, name) + if key is None or digest is None: + return None + return [digest, *key] try: with Path(path).open("rb") as fh: st = os.fstat(fh.fileno()) @@ -1198,7 +1442,7 @@ def changed(self, path: str) -> bool: entry = self._files.get(path) result = True if entry is not None and len(entry) == 3: - key = _stat_key(path) + key = _stat_key(dep_file(path)) if key is not None and (key[0], key[1]) == (entry[1], entry[2]): result = False elif (fresh := file_entry(path)) is not None and fresh[0] == entry[0]: @@ -1353,9 +1597,49 @@ def page_dependency_files( dynamic_py = {f for f in page_ctx.source_files if f.endswith(".py")} if dynamic_py: deps |= _walk_import_closure(build_import_graph(root), dynamic_py) + if (own := _own_function_slice(component, page_ctx.source_files, root)) is not None: + deps.discard(dep_file(own)) + deps.add(own) return deps +def _own_function_slice( + component: object, source_files: Iterable[str], root: Path +) -> str | None: + """The slice of its module a page function reads, when that is all it reads. + + A page reaching its module another way (an import cycle, an import or a + read during evaluation) depends on the whole file. + + Args: + component: The page component or callable. + source_files: The files read while the page was evaluated. + root: Resolved project root. + + Returns: + The slice dependency (``<file>::<function>``), or None. + """ + name = getattr(component, "__name__", None) + files = _component_source_files(component, root) + if ( + not isinstance(name, str) + or getattr(component, "__qualname__", None) != name + or len(files) != 1 + ): + return None + (file,) = files + source_files = set(source_files) + graph = build_import_graph(root) + starts = graph.get(file, set()) | {f for f in source_files if f.endswith(".py")} + if ( + file in source_files + or file in _walk_import_closure(graph, starts) + or function_slice_digest(file, name) is None + ): + return None + return f"{file}{SLICE_SEPARATOR}{name}" + + def page_dependency_entries( page_ctx: PageContext, component: BaseComponent | object, diff --git a/tests/units/compiler/test_disk_cache.py b/tests/units/compiler/test_disk_cache.py index fe7abfeb63c..acf942e9dab 100644 --- a/tests/units/compiler/test_disk_cache.py +++ b/tests/units/compiler/test_disk_cache.py @@ -2494,3 +2494,61 @@ def test_generated_names_do_not_depend_on_other_pages(): assert la.output_code is not None assert lb.output_code is not None assert la.output_code != lb.output_code + + +_TWO_PAGE_MODULE = """\ +import reflex as rx + +CALLS = {"a": 0, "b": 0} + + +def page_a(): + CALLS["a"] += 1 + return rx.el.p("a") + + +def page_b(): + CALLS["b"] += 1 + return rx.el.p("b") +""" + + +@pytest.mark.parametrize("hinted", [True, False]) +def test_incremental_rebuild_recompiles_only_the_edited_function( + tmp_path, monkeypatch, hinted +): + """Editing one page function recompiles only that page of its module. + + The watcher reports the module file; a page tracking its function's slice + of that file must still be validated. + """ + import importlib + + module_name = "two_page_module_t7" + module = tmp_path / f"{module_name}.py" + module.write_text(_TWO_PAGE_MODULE) + monkeypatch.syspath_prepend(str(tmp_path)) + web = _use_tmp_web_dir(tmp_path, monkeypatch) + try: + mod = importlib.import_module(module_name) + app = rx.App() + app.add_page(mod.page_a, route="/a") + app.add_page(mod.page_b, route="/b") + pages = list(app._unevaluated_pages.values()) + ctx = _compile(pages, app=app) + disk_cache.write_manifest(ctx, pages, ctx.all_imports, root=tmp_path) + _stub_externals(app, monkeypatch) + manifest = json.loads((web / disk_cache._MANIFEST_FILE).read_text()) + assert disk_cache.manifest_dependency_files(manifest) >= {str(module.resolve())} + + module.write_text(_TWO_PAGE_MODULE.replace('rx.el.p("a")', 'rx.el.p("a2")')) + mod.CALLS.update(a=0, b=0) + disk_cache.set_changed_hint({module} if hinted else None) + result = disk_cache.try_incremental_rebuild( + app, compiler_plugins=[], prerender_routes=False, root=tmp_path + ) + finally: + disk_cache.set_changed_hint(None) + sys.modules.pop(module_name, None) + assert result is True + assert mod.CALLS == {"a": 1, "b": 0} diff --git a/tests/units/compiler/test_page_cache.py b/tests/units/compiler/test_page_cache.py index b6b43cd35bc..a47155cb7e1 100644 --- a/tests/units/compiler/test_page_cache.py +++ b/tests/units/compiler/test_page_cache.py @@ -835,3 +835,157 @@ def digest(source: str) -> str | None: ) != base ) + + +_SLICED_MODULE = """\ +import reflex as rx +from .other import unused + +TITLE = "t" + + +class PageState(rx.State): + value: int = 0 + + +def helper(): + return rx.text(TITLE) + + +def page_a(): + return rx.box(helper(), PageState.value) + + +def page_b(): + return rx.text("b") +""" + + +@pytest.mark.parametrize( + ("old", "new", "changes"), + [ + ('rx.text("b")', 'rx.text("b2")', False), + ("\n\ndef page_b", "\n\n# a comment\ndef page_b", False), + ("def page_b", "def unrelated():\n return 1\n\n\ndef page_b", False), + ("from .other import unused\n", "", False), + ("rx.box(helper()", "rx.vstack(helper()", True), + ("rx.text(TITLE)", "rx.heading(TITLE)", True), + ('TITLE = "t"', 'TITLE = "u"', True), + ("value: int = 0", "value: int = 1", True), + ("from .other import unused\n", "from .other import *\n", True), + ], +) +def test_function_slice_digest_covers_what_the_function_reads( + tmp_path, old, new, changes +): + """A page function's slice changes only with the code it can read. + + That is its own body, the module-level functions it calls (transitively), + every class, and every statement that runs at import. + """ + module = tmp_path / "sliced.py" + module.write_text(_SLICED_MODULE) + before = page_cache.function_slice_digest(str(module), "page_a") + assert before is not None + module.write_text(_SLICED_MODULE.replace(old, new, 1)) + after = page_cache.function_slice_digest(str(module), "page_a") + assert (after != before) is changes + + +@pytest.mark.parametrize( + "source", + [ + # A decorator can hand the function to code the page reads. + ( + "REG = []\n\n\ndef register(f):\n REG.append(f)\n return f\n\n\n" + "@register\ndef widget():\n return 1\n\n\n" + "def page_a():\n return [f() for f in REG]\n" + ), + # Names reached through globals() are invisible to the syntax tree. + ( + "def widget():\n return 1\n\n\n" + 'def page_a():\n return globals()["widget"]()\n' + ), + ], +) +def test_function_slice_digest_keeps_code_reached_indirectly(tmp_path, source): + """Code the page reaches without naming it stays in the slice.""" + module = tmp_path / "indirect.py" + module.write_text(source) + before = page_cache.function_slice_digest(str(module), "page_a") + module.write_text(source.replace("return 1", "return 2")) + assert page_cache.function_slice_digest(str(module), "page_a") != before + + +def test_function_slice_digest_needs_a_top_level_function(tmp_path): + """Only a module-level function has a slice.""" + module = tmp_path / "nested.py" + module.write_text("def outer():\n def inner():\n return 1\n") + assert page_cache.function_slice_digest(str(module), "inner") is None + assert page_cache.function_slice_digest(str(tmp_path / "gone.py"), "x") is None + + +def test_entrypoint_digest_ignores_page_function_bodies(tmp_path): + """A page defined beside its App is compared page by page, not globally.""" + entry = tmp_path / "entry_app.py" + + def digest(source: str) -> str | None: + entry.write_text(source) + return page_cache.entrypoint_digest(str(entry)) + + source = ( + "import reflex as rx\n\n\ndef make_theme():\n return rx.theme()\n\n\n" + 'def index():\n return rx.text("a")\n\n\n' + "app = rx.App(theme=make_theme())\n" + 'app.add_page(index, route="/")\n' + ) + base = digest(source) + assert base is not None + assert digest(source.replace('rx.text("a")', 'rx.text("b")')) == base + assert digest(source.replace("rx.theme()", "rx.theme(accent_color='red')")) != base + + +def test_page_dependency_files_track_the_page_function_slice(tmp_path, monkeypatch): + """A page depends on its function's slice of its module, not the whole file.""" + from types import SimpleNamespace + + module = _prepare_runtime_module(tmp_path, monkeypatch, "sliced_pages_module") + module.write_text(_SLICED_MODULE.replace("from .other import unused\n", "")) + try: + page_a = importlib.import_module("sliced_pages_module").page_a + page_cache.clear_import_graph() + page_ctx = SimpleNamespace( + root_component=SimpleNamespace(children=[]), + output_code="", + memo_contributions={}, + source_files=set(), + ) + deps = page_cache.page_dependency_files( + cast(Any, page_ctx), page_a, {}, tmp_path + ) + finally: + _forget_modules("sliced_pages_module") + page_cache.clear_import_graph() + + file = str(module.resolve()) + assert f"{file}{page_cache.SLICE_SEPARATOR}page_a" in deps + assert file not in deps + + +def test_file_validator_compares_function_slices(tmp_path): + """A slice dependency is stat-first and changes only with its slice.""" + module = tmp_path / "sliced.py" + module.write_text(_SLICED_MODULE) + dep = f"{module}{page_cache.SLICE_SEPARATOR}page_a" + entry = page_cache.file_entry(dep) + assert entry is not None + files = {dep: entry} + assert page_cache.FileValidator(files).unchanged([dep]) + module.write_text(_SLICED_MODULE.replace('rx.text("b")', 'rx.text("b2")')) + validator = page_cache.FileValidator(files) + assert validator.unchanged([dep]) + assert dep in validator.refreshed + module.write_text(_SLICED_MODULE.replace("rx.box(", "rx.vstack(")) + assert not page_cache.FileValidator(files).unchanged([dep]) + assert page_cache.dep_file(dep) == str(module) + assert page_cache.dep_file(str(module)) == str(module) From 90fa2e88c42e1bb1aad5b428a474230d5bc8e142 Mon Sep 17 00:00:00 2001 From: Farhan <www.mfarvirus@gmail.com> Date: Wed, 21 Oct 2026 03:11:00 +0500 Subject: [PATCH 42/56] perf(compiler): predict dynamic state names in cold compiles too A cold compile with a reusable manifest re-evaluated every page that defines dynamic states. It now first tries the predicted rebuild in a forked child; only a child that compiled spares the exact in-process compile. Skipped on Windows, with other threads alive, and where the daemon chose the mode. --- reflex/compiler/compiler.py | 129 +++++++++++++++-------- reflex/compiler/disk_cache.py | 6 +- reflex/utils/compile_daemon.py | 48 +++++++++ tests/units/utils/test_compile_daemon.py | 60 +++++++++++ 4 files changed, 197 insertions(+), 46 deletions(-) diff --git a/reflex/compiler/compiler.py b/reflex/compiler/compiler.py index fb4e3ce43bf..4709a413bb4 100644 --- a/reflex/compiler/compiler.py +++ b/reflex/compiler/compiler.py @@ -1425,6 +1425,76 @@ def _read_stateful_pages_marker() -> list[str] | None: raise +def _compile_from_cache( + app: App, + compiler_plugins: Sequence[Plugin], + radix_themes_plugin: RadixThemesPlugin, + *, + prerender_routes: bool, + use_rich: bool, + registrations: dict[str, str], +) -> CompileContext | bool: + """Compile from the compile cache, writing the plugin outputs on success. + + Args: + app: The app. + compiler_plugins: The resolved compiler plugins. + radix_themes_plugin: The resolved Radix Themes plugin. + prerender_routes: Whether to prerender routes. + use_rich: Whether to show rich progress. + registrations: Each route's registration digest. + + Returns: + True when the cache compiled the app, the evaluated pages to adopt + when it fell back after evaluating, else False. + """ + from reflex.compiler import disk_cache + + config = get_config() + outputs: dict[Path, str] = {} + rebuilt = disk_cache.try_incremental_rebuild( + app, + compiler_plugins=compiler_plugins, + prerender_routes=prerender_routes, + use_rich=use_rich, + outputs=outputs, + registrations=registrations, + ) + if rebuilt is True: + try: + save_tasks, modify_tasks = _plugin_output_tasks( + config.plugins, + radix_themes_plugin, + list(app._unevaluated_pages.values()), + ) + manifest = disk_cache.load_manifest() + stylesheet_path, stylesheet_code = compile_root_stylesheet( + app.stylesheets, app.reset_style, plugins=compiler_plugins + ) + outputs[utils.resolve_path_of_web_dir(stylesheet_path)] = stylesheet_code + for task_fn, args, kwargs in save_tasks: + result = task_fn(*args, **kwargs) + if result is not None: + for path, code in result if isinstance(result, list) else [result]: + outputs[utils.resolve_path_of_web_dir(path)] = code + sources = _apply_plugin_outputs( + outputs, + config.plugins, + modify_tasks, + manifest.get("plugin_sources", {}) if manifest is not None else {}, + ) + for path, code in outputs.items(): + utils.write_file(path, code) + if manifest is not None and manifest.get("plugin_sources") != sources: + manifest["plugin_sources"] = sources + disk_cache._write(manifest) + except BaseException: + # The staged manifest must not validate outputs that failed to save. + disk_cache._manifest_path().unlink(missing_ok=True) + raise + return rebuilt + + def compile_app( app: App, *, @@ -1511,53 +1581,24 @@ def compile_app( page_cache.enable_read_tracking() # Taken before any page evaluates: evaluation mutates components. registrations = disk_cache.registration_digests(app._unevaluated_pages.values()) - outputs: dict[Path, str] = {} - rebuilt = disk_cache.try_incremental_rebuild( - app, - compiler_plugins=compiler_plugins, - prerender_routes=prerender_routes, - use_rich=use_rich, - outputs=outputs, - registrations=registrations, - ) + from reflex.utils import compile_daemon + + def attempt() -> CompileContext | bool: + return _compile_from_cache( + app, + compiler_plugins, + radix_themes_plugin, + prerender_routes=prerender_routes, + use_rich=use_rich, + registrations=registrations, + ) + + if compile_daemon.compiled_in_predicting_child(lambda: attempt() is True): + return True + rebuilt = attempt() if isinstance(rebuilt, CompileContext): evaluated = rebuilt elif rebuilt: - try: - save_tasks, modify_tasks = _plugin_output_tasks( - config.plugins, - radix_themes_plugin, - list(app._unevaluated_pages.values()), - ) - manifest = disk_cache.load_manifest() - stylesheet_path, stylesheet_code = compile_root_stylesheet( - app.stylesheets, app.reset_style, plugins=compiler_plugins - ) - outputs[utils.resolve_path_of_web_dir(stylesheet_path)] = ( - stylesheet_code - ) - for task_fn, args, kwargs in save_tasks: - result = task_fn(*args, **kwargs) - if result is not None: - for path, code in ( - result if isinstance(result, list) else [result] - ): - outputs[utils.resolve_path_of_web_dir(path)] = code - sources = _apply_plugin_outputs( - outputs, - config.plugins, - modify_tasks, - manifest.get("plugin_sources", {}) if manifest is not None else {}, - ) - for path, code in outputs.items(): - utils.write_file(path, code) - if manifest is not None and manifest.get("plugin_sources") != sources: - manifest["plugin_sources"] = sources - disk_cache._write(manifest) - except BaseException: - # The staged manifest must not validate outputs that failed to save. - disk_cache._manifest_path().unlink(missing_ok=True) - raise return True progress = console.progress() if use_rich else console.PoorProgress() diff --git a/reflex/compiler/disk_cache.py b/reflex/compiler/disk_cache.py index b26c785598a..6a201331fd1 100644 --- a/reflex/compiler/disk_cache.py +++ b/reflex/compiler/disk_cache.py @@ -76,8 +76,10 @@ def set_changed_hint(paths: Iterable[Path | str] | None) -> None: #: Whether this compile predicts the ``reflex.istate.dynamic`` state names of #: the pages it does not re-evaluate, instead of re-evaluating every page that #: defines such states (see :func:`_with_dynamic_state_pages`). Only a caller -#: that can discard this process and compile again sets it. -_predict_dynamic_names = False +#: that can discard this process and compile again sets it; None means no +#: caller chose, and a cold compile tries it in a forked child (see +#: ``compile_daemon.compiled_in_predicting_child``). +_predict_dynamic_names: bool | None = None def set_dynamic_name_prediction(enabled: bool) -> None: diff --git a/reflex/utils/compile_daemon.py b/reflex/utils/compile_daemon.py index 100477628fa..c80c7f369f7 100644 --- a/reflex/utils/compile_daemon.py +++ b/reflex/utils/compile_daemon.py @@ -946,6 +946,54 @@ def _can_fork() -> bool: return False +def _single_threaded() -> bool: + """Whether this process runs one thread, so a ``fork()`` is safe. + + Returns: + True if no other Python or OS thread is alive. + """ + return threading.active_count() == 1 and (_os_thread_count() or 1) == 1 + + +def compiled_in_predicting_child(attempt: Callable[[], bool]) -> bool: + """Run a cold incremental compile in a forked child that predicts state names. + + A failed prediction has already registered states under other names, so + it runs in a copy-on-write child. On any other outcome than a compile, + the caller compiles exactly in this process (which also reports errors). + + Skipped where a caller already chose the mode, such as the daemon's + exact retry. + + Args: + attempt: Runs the incremental compile; returns True when it compiled. + + Returns: + True when the child compiled. + """ + from reflex.compiler import disk_cache + + if ( + disk_cache._predict_dynamic_names is not None + or not hasattr(os, "fork") + or not _single_threaded() + ): + return False + pid = os.fork() + if pid == 0: # child + code = 1 + try: + disk_cache.set_dynamic_name_prediction(True) + code = 0 if attempt() else 2 + except disk_cache.DynamicStatePredictionError as exc: + _log_failed_prediction(exc) + except BaseException: # the caller's exact compile reports it + pass + finally: + os._exit(code) + return _await_child(pid) == 0 + + def _compile_once( roots: list[Path], prerender_routes: bool, changed: set[Path] | None = None ) -> bool: diff --git a/tests/units/utils/test_compile_daemon.py b/tests/units/utils/test_compile_daemon.py index 867664211d1..3479e1c6d07 100644 --- a/tests/units/utils/test_compile_daemon.py +++ b/tests/units/utils/test_compile_daemon.py @@ -933,3 +933,63 @@ def fail_prediction(roots, prerender_routes, changed, predict): monkeypatch.setattr(compile_daemon, "_child_compile", fail_prediction) code = compile_daemon._run_compile_child([], False, None, predict=True) assert code == compile_daemon._RETRY_EXIT_CODE + + +@pytest.mark.skipif(not hasattr(os, "fork"), reason="requires os.fork (POSIX)") +@pytest.mark.parametrize( + ("outcome", "compiled"), + [ + ("ok", True), + ("fell back", False), + ("failed prediction", False), + ("error", False), + ], +) +def test_cold_compile_predicts_names_in_a_child( + monkeypatch, tmp_path, outcome, compiled +): + """A cold compile predicts dynamic state names in a forked child. + + Only a child that compiled spares the caller its exact in-process compile. + """ + from reflex.compiler import disk_cache + + seen = tmp_path / "predicted" + + def attempt() -> bool: + """Record the prediction mode, then end as ``outcome`` says. + + Returns: + Whether the compile succeeded. + + Raises: + DynamicStatePredictionError: For a failed prediction. + RuntimeError: For an error. + """ + seen.write_text(str(disk_cache._predict_dynamic_names)) + if outcome == "failed prediction": + route = "/two" + raise disk_cache.DynamicStatePredictionError(route) + if outcome == "error": + msg = "boom" + raise RuntimeError(msg) + return outcome == "ok" + + monkeypatch.setattr(compile_daemon, "_single_threaded", lambda: True) + assert compile_daemon.compiled_in_predicting_child(attempt) is compiled + assert seen.read_text() == "True" + assert disk_cache._predict_dynamic_names is None + + +@pytest.mark.parametrize( + ("single_threaded", "chosen"), [(False, None), (True, False), (True, True)] +) +def test_cold_compile_without_a_safe_fork_compiles_in_process( + monkeypatch, single_threaded, chosen +): + """The caller compiles in-process with other threads alive or a mode chosen.""" + from reflex.compiler import disk_cache + + monkeypatch.setattr(compile_daemon, "_single_threaded", lambda: single_threaded) + monkeypatch.setattr(disk_cache, "_predict_dynamic_names", chosen) + assert compile_daemon.compiled_in_predicting_child(lambda: True) is False From a51e8b2e627ddf7aa2e40e7cfe9486c5e485d33f Mon Sep 17 00:00:00 2001 From: Farhan <www.mfarvirus@gmail.com> Date: Wed, 21 Oct 2026 03:34:19 +0500 Subject: [PATCH 43/56] fix(compiler): address compile cache review findings - Forget import-time reads when read tracking switches project roots, and keep the reads of an import that fails. - Expand dynamic-state pages before memo siblings, and re-evaluate them when their page order changes. - Fall back when a page stops registering a module state other pages may use. - Install the packages that component-valued state defaults import. - Rebuild the app root when the order of its wraps changes. - Compare functools.partial page arguments, and track the wrapped function. - Slices: recompute the read names for computed global lookups, trust only qualified framework/stdlib decorators, and count entry-module class bodies. - Record built inputs from the refreshed watch state; a newly tracked input is no edit. - Treat only lock contention as a held lock; a daemon without its lock does not serve. - Use directory constants, add missing docstrings, shorten the news entry. --- news/6688.bugfix.md | 1 - news/6688.feature.md | 2 +- reflex/compiler/disk_cache.py | 173 +++++++++++++++-- reflex/compiler/page_cache.py | 119 ++++++++++-- reflex/utils/compile_daemon.py | 56 ++++-- reflex/utils/path_ops.py | 41 +++- tests/units/compiler/test_disk_cache.py | 231 ++++++++++++++++++++--- tests/units/compiler/test_page_cache.py | 96 ++++++++++ tests/units/utils/test_compile_daemon.py | 80 ++++++++ tests/units/utils/test_path_ops.py | 13 ++ 10 files changed, 732 insertions(+), 80 deletions(-) delete mode 100644 news/6688.bugfix.md diff --git a/news/6688.bugfix.md b/news/6688.bugfix.md deleted file mode 100644 index d9df4cc3a54..00000000000 --- a/news/6688.bugfix.md +++ /dev/null @@ -1 +0,0 @@ -Fixed repeated full recompilation when the incremental compile cache encounters state defaults requiring Reflex serialization, such as Pydantic models or dates. Manifest write failures now produce visible warnings instead of silently disabling cache reuse. Cache invalidation now includes package initializers, missing file reads, and app entrypoints that also define pages. Incremental builds run plugin output hooks, preserve compile telemetry, and refresh stylesheets. The daemon records its own PID, releases backend compilation when it exits, preserves edits during watcher teardown, discovers new root-level sources, and reuses the parent's initial build. Explicit `app_module_import` configurations use full compilation because earlier import-time reads may be unavailable. Component-doc control IDs remain stable across partial builds, and Markdown-triggered Uvicorn reloads are limited to the experimental cache mode. The cache identity now includes the compile mode, so a production build never reuses development output, a backend worker consumes its `nocompile` marker even while the compile daemon owns the build, edits from a failed hot reload stay pending until a compile succeeds, and a full compile adopts the pages an incremental attempt already evaluated instead of evaluating them twice. diff --git a/news/6688.feature.md b/news/6688.feature.md index 782789178b6..9fd7a9e65c9 100644 --- a/news/6688.feature.md +++ b/news/6688.feature.md @@ -1 +1 @@ -Added an experimental disk-persisted incremental compile cache, enabled by the `REFLEX_COMPILE_CACHE` environment variable. When on, a fresh compile reuses the previous build already on disk in `.web` and recompiles only the pages whose source changed, tracked via a per-page dependency graph (Python import closure, files read during page evaluation, component modules, and referenced state). Dependencies are validated stat-first, so unchanged files are never re-read, and the contexts file is rebuilt from the manifest without re-evaluating unchanged pages. App-wide inputs (Reflex version, config/lockfiles, the app entrypoint's config modules such as theme/app-wraps/stylesheets, and module-level state definitions and their Python dependencies) gate the whole cache, falling back to a full compile when they change. `reflex run` dev additionally gains a warm fork-per-compile daemon that watches the project with `watchfiles` (polling only as a fallback), so hot reloads skip the cold reimport and rebuild only what changed; the backend reload worker waits for that compile before registering state. The daemon checks for source and asset changes made during compilation and falls back to a cold subprocess when native threads remain alive. Off by default — the compile path is unchanged when the flag is unset. +Added an experimental incremental compile cache, enabled with `REFLEX_COMPILE_CACHE=1`. A compile reuses the previous build in `.web` and recompiles only the pages whose inputs changed, and `reflex run` keeps a warm compile daemon so a hot reload rebuilds only what the edit touched. Off by default. diff --git a/reflex/compiler/disk_cache.py b/reflex/compiler/disk_cache.py index 6a201331fd1..123422b3279 100644 --- a/reflex/compiler/disk_cache.py +++ b/reflex/compiler/disk_cache.py @@ -19,6 +19,7 @@ import base64 import contextlib import dataclasses +import functools import hashlib import json import logging @@ -106,6 +107,11 @@ class DynamicStatePredictionError(Exception): def _manifest_path() -> Path: + """Return the manifest's path in the web directory. + + Returns: + The manifest path. + """ return prerequisites.get_web_dir() / _MANIFEST_FILE @@ -348,6 +354,12 @@ def _stable_value(value: Any) -> Any: field.name: _stable_value(getattr(value, field.name)) for field in dataclasses.fields(value) } + if isinstance(value, functools.partial): + return [ + _stable_value(value.func), + _stable_value(value.args), + _stable_value(value.keywords), + ] if callable(value): # Which function, not its body: a page function's body is a recorded # dependency of the page, and load events only run on the backend. @@ -1245,6 +1257,43 @@ def eval_page( cls._per_component_state_instance_count = number +def _route_dropping_a_shared_state( + manifest: dict[str, Any], + miss_routes: Iterable[str], + removed_entries: dict[str, dict[str, Any]], +) -> str | None: + """Find a recompiled or removed page whose recorded module state is gone. + + A page records the states its evaluation registered first. Another page + using the same module state recorded none, so when the first page stops + registering it, only a full compile keeps it. A local state belongs to + its page alone. + + Args: + manifest: The loaded manifest (without the removed pages). + miss_routes: The recompiled routes. + removed_entries: The manifest entries of the removed routes. + + Returns: + The first such route, or None. + """ + import reflex.istate.dynamic as istate_dynamic + + local_prefix = istate_dynamic.__name__.replace(".", "___") + "___" + live = RegistrationContext.ensure_context().base_states + entries = [(route, manifest["pages"].get(route)) for route in miss_routes] + entries.extend(removed_entries.items()) + for route, entry in entries: + if entry is None or not entry["state_slice"]: + continue + for name in entry["state_slice"]["schema"]: + if name not in live and not name.rpartition(".")[2].startswith( + local_prefix + ): + return route + return None + + def _claim_recorded_dynamic_names( manifest: dict[str, Any], miss_routes: set[str] ) -> None: @@ -1288,7 +1337,7 @@ def _with_dynamic_state_pages( pages: Sequence[PageDefinition], manifest: dict[str, Any], *, - removed: bool = False, + shifted: bool = False, ) -> list[PageDefinition]: """Expand a miss set with every page that defines dynamic-module states. @@ -1303,12 +1352,13 @@ def _with_dynamic_state_pages( miss_pages: The pages to recompile. pages: All current page definitions (in compile order). manifest: The loaded manifest. - removed: Whether a removed page defined dynamic-module states. + shifted: Whether names may have shifted without a miss: a removed + page defined dynamic-module states, or such pages were reordered. Returns: The expanded miss list, in ``pages`` order. """ - if not miss_pages and not removed: + if not miss_pages and not shifted: return miss_pages miss_routes = {page.route for page in miss_pages} return [ @@ -1318,6 +1368,65 @@ def _with_dynamic_state_pages( ] +def _dynamic_page_order_changed( + manifest: dict[str, Any], pages: Sequence[PageDefinition] +) -> bool: + """Whether the recorded pages defining dynamic states changed their order. + + Their states take names in page order. The manifest lists pages in the + order of the compile that wrote it. + + Args: + manifest: The loaded manifest (without the removed pages). + pages: All current page definitions (in compile order). + + Returns: + True when those pages are in another order now. + """ + recorded = manifest["pages"] + return [route for route, entry in recorded.items() if entry["dynamic_claims"]] != [ + page.route + for page in pages + if (entry := recorded.get(page.route)) is not None and entry["dynamic_claims"] + ] + + +def _expand_miss_pages( + miss_pages: list[PageDefinition], + pages: Sequence[PageDefinition], + manifest: dict[str, Any], + dirty_memo_paths: set[str], + stale_owner_routes: set[str], + *, + dynamic_names_shift: bool, +) -> list[PageDefinition]: + """Add the pages a rebuild of ``miss_pages`` must evaluate too. + + Module siblings come last: every page added before them may share a + memo file with hit pages of its module. + + Args: + miss_pages: The new or changed pages. + pages: All current page definitions (in compile order). + manifest: The loaded manifest. + dirty_memo_paths: Memo output paths the rebuild rewrites. + stale_owner_routes: Owners of route-owned memo records that changed. + dynamic_names_shift: Whether dynamic state names may have shifted + without a miss (see :func:`_with_dynamic_state_pages`). + + Returns: + The expanded miss list, in ``pages`` order. + """ + miss_pages = _with_memo_contributing_pages( + miss_pages, pages, manifest, dirty_memo_paths, stale_owner_routes + ) + if not _predict_dynamic_names: + miss_pages = _with_dynamic_state_pages( + miss_pages, pages, manifest, shifted=dynamic_names_shift + ) + return _with_module_siblings(miss_pages, pages, manifest) + + def _changed_dependency_files( manifest: dict[str, Any], validator: page_cache.FileValidator, @@ -1761,20 +1870,28 @@ def try_incremental_rebuild( and owner in routes and _memo_deps_changed(entry["deps"], validator, changed_hint) } + order_changed = _dynamic_page_order_changed(manifest, pages) + if order_changed and _predict_dynamic_names: + msg = "the order of the pages defining dynamic states changed" + raise DynamicStatePredictionError(msg) changed_files: set[str] = set() - if miss_pages or dirty_memo_paths or stale_owner_routes or removed_entries: + if ( + miss_pages + or dirty_memo_paths + or stale_owner_routes + or removed_entries + or order_changed + ): changed_files = _changed_dependency_files(manifest, validator, changed_hint) - miss_pages = _with_memo_contributing_pages( - miss_pages, pages, manifest, dirty_memo_paths, stale_owner_routes + miss_pages = _expand_miss_pages( + miss_pages, + pages, + manifest, + dirty_memo_paths, + stale_owner_routes, + dynamic_names_shift=order_changed + or any(e["dynamic_claims"] for e in removed_entries.values()), ) - miss_pages = _with_module_siblings(miss_pages, pages, manifest) - if not _predict_dynamic_names: - miss_pages = _with_dynamic_state_pages( - miss_pages, - pages, - manifest, - removed=any(e["dynamic_claims"] for e in removed_entries.values()), - ) memo_note = ( f"; rewriting {len(dirty_memo_paths)} memo file(s)" if dirty_memo_paths @@ -1858,6 +1975,11 @@ def try_incremental_rebuild( _fall_back_after_evaluation( f"page {route!r} changed dynamic state names other pages share" ) + if route := _route_dropping_a_shared_state(manifest, miss_routes, removed_entries): + _fall_back_after_evaluation( + f"page {route!r} no longer registers a state other pages may use" + ) + return miss_ctx if miss_ctx is not None else False # Write changed pages + their memo files; reuse everything else on disk. install_imports = _deserialize_imports(manifest["all_imports"]) @@ -1997,10 +2119,15 @@ def try_incremental_rebuild( if contexts_snapshot is None else _with_hit_state_slices(contexts_snapshot, manifest, miss_routes) ) + state_imports: ParsedImportDict = {} context_path, context_code = compiler.compile_contexts( - app._state, theme, contexts_snapshot=merged_snapshot + app._state, + theme, + contexts_snapshot=merged_snapshot, + component_imports=state_imports, ) _save_incremental_output(context_path, context_code, outputs) + install_imports = merge_imports(install_imports, state_imports) if contexts_snapshot is not None and merged_snapshot is not None: live_libraries = _state_libraries(contexts_snapshot[0]) state_libraries = { @@ -2080,6 +2207,7 @@ def try_incremental_rebuild( app_root_inputs=app_root_inputs, state_libraries=state_libraries, registrations=registrations, + page_order=[page.route for page in pages], module_states=( _record_module_states( live_module_states, @@ -2184,7 +2312,11 @@ def _refresh_app_root( libraries = _reconstruct_libraries( bundle_base, page_requests, stored["root_requests"], state_libraries ) - if merged == stored["wraps"] and libraries == stored["libraries"]: + # Wraps of one priority nest in insertion order, so the order counts. + if ( + list(merged.items()) == list(stored["wraps"].items()) + and libraries == stored["libraries"] + ): return None if not stored["reconstructible"]: msg = "the app root's bundled libraries cannot be rebuilt" @@ -2264,6 +2396,7 @@ def _update_manifest_for_misses( state_libraries: dict[str, list[str]], registrations: dict[str, str], module_states: dict[str, list[str]], + page_order: Sequence[str] = (), route_set_changed: bool = False, validator: page_cache.FileValidator | None = None, ) -> None: @@ -2289,6 +2422,7 @@ def _update_manifest_for_misses( state_libraries: The libraries each state's initial values request. registrations: Each route's registration digest. module_states: The import-time states and their source files. + page_order: The routes in compile order, which the manifest keeps. route_set_changed: Whether pages were added or removed. validator: The validator used for this rebuild, for its refreshed entries. @@ -2334,8 +2468,15 @@ def _update_manifest_for_misses( root=root, ) dirty = True + if page_order and list(manifest["pages"]) != list(page_order): + manifest["pages"] = { + route: manifest["pages"][route] for route in page_order + } + dirty = True if ( manifest["app_root"] != app_root_inputs + # Dict equality ignores the wrap order, which the root keeps. + or list(manifest["app_root"]["wraps"]) != list(app_root_inputs["wraps"]) or manifest["state_libraries"] != state_libraries or manifest["module_states"] != module_states ): diff --git a/reflex/compiler/page_cache.py b/reflex/compiler/page_cache.py index e65197e77ee..99fea19f8c3 100644 --- a/reflex/compiler/page_cache.py +++ b/reflex/compiler/page_cache.py @@ -12,6 +12,7 @@ import builtins import contextlib import dataclasses +import functools import hashlib import importlib import importlib.util @@ -20,11 +21,11 @@ import sys from collections.abc import Callable, Iterable, Mapping, Sequence from contextvars import ContextVar -from functools import lru_cache from importlib import metadata from pathlib import Path from typing import TYPE_CHECKING, TypeVar +from reflex_base import constants from reflex_base.environment import environment if TYPE_CHECKING: @@ -35,7 +36,15 @@ FileEntry = list #: Directories never worth hashing (build artifacts, deps, caches). -_SKIP_DIRS = {".web", ".venv", "venv", "node_modules", "__pycache__", ".git", "assets"} +_SKIP_DIRS = { + constants.Dirs.WEB, + ".venv", + "venv", + "node_modules", + "__pycache__", + ".git", + constants.Dirs.APP_ASSETS, +} #: Genuinely-global files: a change here can affect every page's output, so it #: bumps ``global_epoch`` rather than any single page's dependency set. @@ -56,6 +65,14 @@ def _sha(*parts: bytes | str) -> str: + """Digest parts with a separator, so their boundaries count. + + Args: + parts: The text or bytes to digest. + + Returns: + The hex digest. + """ h = hashlib.sha256() for p in parts: h.update(p.encode() if isinstance(p, str) else p) @@ -64,6 +81,11 @@ def _sha(*parts: bytes | str) -> str: def _reflex_version() -> str: + """Return the installed Reflex version. + + Returns: + The version, or ``unknown`` when the package metadata is missing. + """ try: return metadata.version("reflex") except Exception: @@ -165,6 +187,11 @@ def _suspend_tracking(): def _record_read(path: object) -> None: + """Add a first-party file read to the active read set, if any. + + Args: + path: The path the code read. + """ target = _active_reads.get() if target is None: return @@ -339,15 +366,19 @@ def _record_imported_modules( target.update(_module_import_reads.get(submodule, ())) -def _import_recording_reads(name: str, do_import: Callable[[], T]) -> T: +def _import_recording_reads( + name: str, do_import: Callable[[], T], target: set[str] +) -> T: """Run an import under a fresh read set kept as the module's import reads. A module executing afresh replaces its recorded reads; importing a loaded - module only adds the reads of submodules it loads for the first time. + module only adds the reads of submodules it loads for the first time. The + reads also go to ``target``, even when the import fails. Args: name: The absolute requested module name. do_import: Performs the import. + target: The active read set. Returns: The result of ``do_import``. @@ -363,6 +394,7 @@ def _import_recording_reads(name: str, do_import: Callable[[], T]) -> T: _module_import_reads.pop(name, None) if reads: _module_import_reads.setdefault(name, set()).update(reads) + target.update(reads) def enable_read_tracking(root: Path | None = None) -> None: @@ -380,6 +412,7 @@ def enable_read_tracking(root: Path | None = None) -> None: resolved_root = (root or Path.cwd()).resolve() if _recorder_root != resolved_root: _module_file_cache.clear() + _module_import_reads.clear() _recorder_root = resolved_root from reflex_base.plugins import compiler as _bc @@ -431,7 +464,9 @@ def import_( return orig_import(name, globals_, locals_, fromlist, level) absolute = _absolute_import_name(name, globals_, level) result = _import_recording_reads( - absolute, lambda: orig_import(name, globals_, locals_, fromlist, level) + absolute, + lambda: orig_import(name, globals_, locals_, fromlist, level), + target, ) _record_imported_modules(absolute, result, target, fromlist) return result @@ -441,7 +476,7 @@ def import_module(name: str, package: str | None = None): return orig_import_module(name, package) absolute = importlib.util.resolve_name(name, package) if package else name result = _import_recording_reads( - absolute, lambda: orig_import_module(name, package) + absolute, lambda: orig_import_module(name, package), target ) _record_imported_modules(result.__name__, result, target) return result @@ -541,8 +576,16 @@ def _loaded_names(nodes: Iterable[ast.AST]) -> set[str]: "__import__", }) -#: Decorators that keep the function they wrap to themselves at import time. -_INERT_DECORATORS = frozenset({"page", "memo", "cache", "lru_cache"}) +#: Decorators (``<module>.<name>``) that keep the function they wrap to +#: themselves at import time. +_INERT_DECORATORS = frozenset({ + ("rx", "page"), + ("rx", "memo"), + ("reflex", "page"), + ("reflex", "memo"), + ("functools", "cache"), + ("functools", "lru_cache"), +}) def dep_file(dep: str) -> str: @@ -586,8 +629,11 @@ def _is_inert_decorator(decorator: ast.expr) -> bool: """ if isinstance(decorator, ast.Call): decorator = decorator.func - name = getattr(decorator, "attr", None) or getattr(decorator, "id", None) - return name in _INERT_DECORATORS + return ( + isinstance(decorator, ast.Attribute) + and isinstance(decorator.value, ast.Name) + and (decorator.value.id, decorator.attr) in _INERT_DECORATORS + ) def _module_part(statement: ast.stmt) -> _ModulePart: @@ -748,6 +794,9 @@ def _slice_digest(parts: Sequence[_ModulePart], seeds: Iterable[str]) -> str: used = always.union(*(part.loads for name in closure for part in definitions[name])) if not used.isdisjoint(_DYNAMIC_GLOBAL_NAMES): closure = set(definitions) + used = always.union( + *(part.loads for group in definitions.values() for part in group) + ) digest = hashlib.sha256() for part in parts: for text in _sliced_texts(part, used, closure): @@ -756,6 +805,18 @@ def _slice_digest(parts: Sequence[_ModulePart], seeds: Iterable[str]) -> str: return digest.hexdigest() +def _class_names(parts: Sequence[_ModulePart]) -> list[str]: + """Name a module's top-level classes. + + Args: + parts: The module's parts. + + Returns: + The class names. + """ + return [part.name for part in parts if part.name is not None and part.is_class] + + def function_slice_digest(path: str, name: str) -> str | None: """Digest the parts of a module a module-level function can read. @@ -773,8 +834,7 @@ def function_slice_digest(path: str, name: str) -> str | None: parts = _module_parts(path) if parts is None or not any(part.name == name for part in parts): return None - classes = [part.name for part in parts if part.name is not None and part.is_class] - return _slice_digest(parts, (name, *classes)) + return _slice_digest(parts, (name, *_class_names(parts))) def entrypoint_digest(path: str) -> str | None: @@ -782,7 +842,8 @@ def entrypoint_digest(path: str) -> str | None: Left out are top-level ``<app>.add_page(...)`` statements and the code only they read: page functions defined in the module (each page tracks its own - slice, see :func:`function_slice_digest`) and imported pages. So is what + slice, see :func:`function_slice_digest`) and imported pages. Class bodies + run at import, so every class counts. Also left out is what the syntax tree never holds (comments, formatting). Two versions with the same digest differ only in their pages, which the compile cache compares page by page; any other edit can change every page. @@ -794,7 +855,9 @@ def entrypoint_digest(path: str) -> str | None: The digest, or None when the file cannot be read or parsed. """ parts = _module_parts(path) - return None if parts is None else _slice_digest(parts, ()) + if parts is None: + return None + return _slice_digest(parts, _class_names(parts)) def _app_entrypoint_file(root: Path | None = None) -> Path | None: @@ -908,6 +971,14 @@ def global_epoch( def _module_file(component: object) -> Path | None: + """Return the file of the module an object belongs to. + + Args: + component: The object. + + Returns: + The module file, or None. + """ mod = sys.modules.get(getattr(component, "__module__", "") or "") file = getattr(mod, "__file__", None) return Path(file) if file else None @@ -1000,6 +1071,14 @@ def component_module_files( def _resolve_module_file(name: str) -> str | None: + """Return the resolved file of a loaded module. + + Args: + name: The dotted module name. + + Returns: + The resolved file path, or None. + """ mod = sys.modules.get(name) file = getattr(mod, "__file__", None) return str(Path(file).resolve()) if file else None @@ -1026,7 +1105,7 @@ def _loaded_first_party_modules(root: Path) -> dict[str, str]: return file_to_mod -@lru_cache(maxsize=8192) +@functools.lru_cache(maxsize=8192) def _first_party_module_file(file: str, roots: tuple[Path, ...]) -> str | None: """Resolve a stable module filename once per set of reload roots. @@ -1265,6 +1344,8 @@ def _component_source_files(component: object, root: Path) -> set[str]: Returns: The set of resolved defining file path strings under ``root``. """ + while isinstance(component, functools.partial): + component = component.func out: set[str] = set() roots = first_party_roots(root) code = getattr(component, "__code__", None) @@ -1464,6 +1545,14 @@ def unchanged(self, paths: Iterable[str]) -> bool: def _subclasses(root_cls: type) -> list[type]: + """List a class and every subclass below it. + + Args: + root_cls: The class to start from. + + Returns: + The classes, each once. + """ seen: set[type] = set() out: list[type] = [] stack = [root_cls] diff --git a/reflex/utils/compile_daemon.py b/reflex/utils/compile_daemon.py index c80c7f369f7..22fb775c48a 100644 --- a/reflex/utils/compile_daemon.py +++ b/reflex/utils/compile_daemon.py @@ -22,6 +22,7 @@ from dataclasses import dataclass, field from pathlib import Path +from reflex_base import constants from reflex_base.environment import environment from reflex_base.utils import log @@ -45,7 +46,14 @@ #: Source suffixes edited under the app roots that should trigger a recompile. _WATCH_SUFFIXES = (".py", ".md", ".mdx") #: Directories never worth walking while building the watch snapshot. -_SKIP_DIRS = {".web", ".venv", "venv", "node_modules", "__pycache__", ".git"} +_SKIP_DIRS = { + constants.Dirs.WEB, + ".venv", + "venv", + "node_modules", + "__pycache__", + ".git", +} @contextlib.contextmanager @@ -251,7 +259,7 @@ def build(cls, roots: list[Path], root: Path) -> _WatchState: @property def assets(self) -> Path | None: """The assets directory, when the project has one.""" - assets = self.root / "assets" + assets = self.root / constants.Dirs.APP_ASSETS return assets if assets.is_dir() else None def targets(self) -> list[Path]: @@ -652,12 +660,22 @@ def _reset_model_metadata() -> None: def _lock_path() -> Path: + """Return the compile-in-progress lock file. + + Returns: + The lock file under the backend dir. + """ from reflex.utils import prerequisites return prerequisites.get_backend_dir() / _COMPILE_LOCK_FILE def _daemon_lock_path() -> Path: + """Return the daemon ownership lock file. + + Returns: + The lock file under the backend dir. + """ from reflex.utils import prerequisites return prerequisites.get_backend_dir() / _DAEMON_LOCK_FILE @@ -1161,18 +1179,23 @@ def alive() -> bool: state.roots = roots = _reload_roots() with _compile_lock(root): ok = _compile_once(roots, prerender_routes, changed) + # Refresh what is watched from the new manifest so a newly + # referenced content file becomes watched. + state = _WatchState.build(roots, root) + state.checkpoint = _dependency_snapshot(state) # Also after a failure: workers must not wait on an edit that will - # not build; a fix is a newer edit. - _record_built_inputs(_known_snapshot(state, before)) + # not build; a fix is a newer edit. An input edited during the + # compile keeps its earlier time, so workers wait for its compile. + _record_built_inputs(_known_snapshot(state, {**state.checkpoint, **before})) unapplied = set() if ok else changed if not ok: logger.error("Compile failed; keeping the last good build.") - # Refresh what is watched from the new manifest so a newly-referenced - # content file becomes watched, and catch up on edits made while the - # watcher was down for the compile. - state = _WatchState.build(roots, root) - state.checkpoint = _dependency_snapshot(state) + # Catch up on edits made while the watcher was down for the compile. An + # input the compile started to read was not watched before it; it + # changed only if it differs from what the compile recorded. pending = _missed_changes(before, state.checkpoint) + if appeared := (pending - before.keys()) & state.known: + pending -= appeared - _unbuilt_changes(state) def main(argv: list[str] | None = None) -> int: @@ -1211,11 +1234,20 @@ def main(argv: list[str] | None = None) -> int: # Held for the process lifetime, so backend workers defer to this daemon # exactly while it lives (see ``daemon_active``). Forked compile children # share the lock; ``os.execv`` drops it and the new image retakes it. - if _acquire_lock(_daemon_lock_path()) is None: + try: + descriptor = _acquire_lock(_daemon_lock_path()) + except OSError as exc: logger.warning( - "Another compile daemon owns this project's .web; backend workers " - "defer to it." + f"Compile daemon: this file system cannot lock files ({exc}); " + "backend workers compile instead." ) + return 0 + if descriptor is None: + logger.warning( + "Another compile daemon owns this project's .web; leaving the " + "compiles to it." + ) + return 0 try: _serve() except KeyboardInterrupt: diff --git a/reflex/utils/path_ops.py b/reflex/utils/path_ops.py index ac4014de831..39c89e7914d 100644 --- a/reflex/utils/path_ops.py +++ b/reflex/utils/path_ops.py @@ -311,6 +311,15 @@ def _lock_descriptor(descriptor: int) -> None: fcntl.flock(descriptor, fcntl.LOCK_EX) +#: The errors with which a non-blocking lock reports another holder. +_LOCK_CONTENTION_ERRNOS = frozenset({ + errno.EACCES, + errno.EAGAIN, + errno.EWOULDBLOCK, + errno.EDEADLK, +}) + + def _try_lock_descriptor(descriptor: int) -> bool: """Take the OS's exclusive lock on a lock file without waiting. @@ -319,14 +328,19 @@ def _try_lock_descriptor(descriptor: int) -> bool: Returns: Whether the lock was taken. + + Raises: + OSError: When the file system cannot lock files. """ try: if sys.platform == "win32": msvcrt.locking(descriptor, msvcrt.LK_NBLCK, 1) else: fcntl.flock(descriptor, fcntl.LOCK_EX | fcntl.LOCK_NB) - except OSError: - return False + except OSError as error: + if error.errno in _LOCK_CONTENTION_ERRNOS: + return False + raise return True @@ -343,11 +357,18 @@ def try_lock(path: Path) -> int | None: Returns: The descriptor holding the lock (closing it releases the lock), or None when another holder has it. + + Raises: + OSError: When the file system cannot lock files. """ path.parent.mkdir(parents=True, exist_ok=True) descriptor = os.open(path, os.O_CREAT | os.O_RDWR, 0o644) - if _try_lock_descriptor(descriptor): - return descriptor + try: + if _try_lock_descriptor(descriptor): + return descriptor + except OSError: + os.close(descriptor) + raise os.close(descriptor) return None @@ -359,16 +380,20 @@ def is_locked(path: Path) -> bool: path: The lock file. Returns: - True while a holder has the lock; False when it is free or the file - does not exist. + True while a holder has the lock; False when it is free, the file + does not exist, or the file system cannot lock files (so nothing can + hold it). """ try: descriptor = os.open(path, os.O_RDWR) except FileNotFoundError: return False try: - if not _try_lock_descriptor(descriptor): - return True + try: + if not _try_lock_descriptor(descriptor): + return True + except OSError: + return False if sys.platform == "win32": msvcrt.locking(descriptor, msvcrt.LK_UNLCK, 1) return False diff --git a/tests/units/compiler/test_disk_cache.py b/tests/units/compiler/test_disk_cache.py index acf942e9dab..abaa5afbd0b 100644 --- a/tests/units/compiler/test_disk_cache.py +++ b/tests/units/compiler/test_disk_cache.py @@ -948,8 +948,25 @@ def test_page_becoming_stateful_rebuilds_incrementally(tmp_path, monkeypatch): assert refreshed["pages"][route_s]["is_stateful"] is True -def test_page_dropping_its_states_rebuilds_contexts(tmp_path, monkeypatch): - """A page that stops defining states takes its slice out of the contexts.""" +def _drop_page_state( + tmp_path, + monkeypatch, + gone: str, + compile_contexts: Callable[..., Any] | None = None, + install: Callable[[Any], None] | None = None, +) -> tuple[Any, list[Any], dict[str, Any]]: + """Rebuild a page that defined ``gone`` before an edit and no state after it. + + Args: + tmp_path: The project root. + monkeypatch: The pytest monkeypatch fixture. + gone: The full name of the state the page stopped defining. + compile_contexts: Replaces ``compile_contexts`` after the capture. + install: Receives the imports whose packages the rebuild installs. + + Returns: + The rebuild result, the captured contexts snapshots, and the manifest. + """ web = _use_tmp_web_dir(tmp_path, monkeypatch) app = rx.App() app.add_page(_page_a, route="/a") @@ -959,8 +976,6 @@ def test_page_dropping_its_states_rebuilds_contexts(tmp_path, monkeypatch): disk_cache.write_manifest(ctx, pages, ctx.all_imports, root=tmp_path) manifest_path = web / disk_cache._MANIFEST_FILE manifest = json.loads(manifest_path.read_text()) - # Recorded before the edit that removed the page's state. - gone = "reflex___state____state.gone_state" manifest["pages"]["a"].update( is_stateful=True, state_slice={ @@ -972,19 +987,64 @@ def test_page_dropping_its_states_rebuilds_contexts(tmp_path, monkeypatch): _stale_dep(manifest, "a", str(tmp_path / "view.py")) manifest_path.write_text(json.dumps(manifest)) captured = _capture_contexts(monkeypatch) - assert ( - disk_cache.try_incremental_rebuild( - app, compiler_plugins=[], prerender_routes=False, root=tmp_path + if compile_contexts is not None: + monkeypatch.setattr( + "reflex.compiler.compiler.compile_contexts", compile_contexts ) - is True + if install is not None: + monkeypatch.setattr(app, "_get_frontend_packages", install) + result = disk_cache.try_incremental_rebuild( + app, compiler_plugins=[], prerender_routes=False, root=tmp_path ) + return result, captured, json.loads(manifest_path.read_text()) + + +#: A local state the page defined; no other page can use it. +_GONE_LOCAL = "reflex___state____state.reflex___istate___dynamic____gone_state" + + +def test_page_dropping_its_states_rebuilds_contexts(tmp_path, monkeypatch): + """A page that stops defining states takes its slice out of the contexts.""" + result, captured, refreshed = _drop_page_state(tmp_path, monkeypatch, _GONE_LOCAL) + assert result is True [(initial, _, schema_entries)] = captured - assert gone not in initial - assert gone not in schema_entries - refreshed = json.loads(manifest_path.read_text()) + assert _GONE_LOCAL not in initial + assert _GONE_LOCAL not in schema_entries assert refreshed["pages"]["a"]["is_stateful"] is False +def test_page_dropping_a_shared_state_falls_back(tmp_path, monkeypatch): + """A module state the page stops importing may still serve unchanged pages. + + Those pages recorded no slice for it, so only a full compile keeps it. + """ + gone = "reflex___state____state.shared_module___shared_state" + result, captured, _ = _drop_page_state(tmp_path, monkeypatch, gone) + assert isinstance(result, CompileContext) + assert captured == [] + + +def test_contexts_rebuild_installs_the_packages_state_defaults_import( + tmp_path, monkeypatch +): + """A component-valued state default can bring a package no page imports.""" + from reflex.compiler import utils as compiler_utils + + def compile_contexts(state, theme, *, contexts_snapshot, component_imports): + component_imports.setdefault("state-only-package", []).append( + ImportVar(tag="Widget") + ) + return compiler_utils.get_context_path(), _CONTEXTS_STUB + + installed: list[Any] = [] + result, _, refreshed = _drop_page_state( + tmp_path, monkeypatch, _GONE_LOCAL, compile_contexts, installed.append + ) + assert result is True + assert "state-only-package" in installed[-1] + assert "state-only-package" in refreshed["all_imports"] + + def test_incremental_rebuild_all_hits(tmp_path, monkeypatch): _use_tmp_web_dir(tmp_path, monkeypatch) @@ -1620,12 +1680,16 @@ def _page_state_provider() -> Component: return rx.el.div(rx.el.p(_fp_state("_ProviderState").value), _footer()) -def _root_rebuild(tmp_path, monkeypatch, **edit) -> tuple[Any, Path, str]: +def _root_rebuild( + tmp_path, monkeypatch, *, reversed_wraps: list[str] | None = None, **edit +) -> tuple[Any, Path, str]: """Fully compile a stateful page and a plain page, then edit the plain one. Args: tmp_path: The project root. monkeypatch: The pytest monkeypatch fixture. + reversed_wraps: When given, the app root's wraps are recorded in + reverse order, which is appended to it. edit: The ``_WRAP_EDIT`` flags the edit turns on. Returns: @@ -1659,6 +1723,11 @@ def _root_rebuild(tmp_path, monkeypatch, **edit) -> tuple[Any, Path, str]: manifest_path = web / disk_cache._MANIFEST_FILE manifest = json.loads(manifest_path.read_text()) assert manifest["app_root"]["reconstructible"] + if reversed_wraps is not None: + wraps = manifest["app_root"]["wraps"] + assert len(wraps) > 1 + manifest["app_root"]["wraps"] = dict(reversed(wraps.items())) + reversed_wraps.extend(manifest["app_root"]["wraps"]) _stale_dep(manifest, pages[1].route, str(tmp_path / "view.py")) manifest_path.write_text(json.dumps(manifest)) # The daemon child starts without the states its pages define. @@ -1682,6 +1751,15 @@ def test_page_gaining_a_provided_wrap_reuses_the_app_root(tmp_path, monkeypatch) assert root_file.read_text() == before +def test_app_root_follows_the_order_of_its_wraps(tmp_path, monkeypatch): + """Wraps of one priority nest in the order pages first require them.""" + recorded: list[str] = [] + result, _, _ = _root_rebuild(tmp_path, monkeypatch, reversed_wraps=recorded) + assert result is True + manifest = json.loads((tmp_path / ".web" / disk_cache._MANIFEST_FILE).read_text()) + assert list(manifest["app_root"]["wraps"]) == recorded[::-1] + + def test_page_bringing_a_new_wrap_rebuilds_only_the_app_root(tmp_path, monkeypatch): """A new wrap rebuilds the root from recorded and recompiled wraps.""" result, root_file, before = _root_rebuild(tmp_path, monkeypatch, wrap=True) @@ -2025,28 +2103,29 @@ def remove_c(app): def test_removed_stateful_page_leaves_the_contexts(tmp_path, monkeypatch): - """A removed page's states leave the contexts file.""" + """A removed page's local states leave the contexts file.""" web = _use_tmp_web_dir(tmp_path, monkeypatch) app = rx.App() app.add_page(_page_a, route="/a") - app.add_page(_page_flip, route="/s") + app.add_page(_page_collide_one, route="/s") pages = list(app._unevaluated_pages.values()) - ctx = _compile(pages, app=app) - flip_states = ctx.stateful_routes[pages[1].route] - _stub_externals(app, monkeypatch) - disk_cache.write_manifest(ctx, pages, ctx.all_imports, root=tmp_path) - _unregister_state(_FP_HOLDER.pop("_FlipState")) - del app._unevaluated_pages[pages[1].route] - captured = _capture_contexts(monkeypatch) - assert ( - disk_cache.try_incremental_rebuild( + try: + ctx = _compile(pages, app=app) + local_states = ctx.stateful_routes[pages[1].route] + _stub_externals(app, monkeypatch) + disk_cache.write_manifest(ctx, pages, ctx.all_imports, root=tmp_path) + _drop_dynamic_states("CollideState") + del app._unevaluated_pages[pages[1].route] + captured = _capture_contexts(monkeypatch) + result = disk_cache.try_incremental_rebuild( app, compiler_plugins=[], prerender_routes=False, root=tmp_path ) - is True - ) + finally: + _drop_dynamic_states("CollideState") + assert result is True [(initial, _, schema_entries)] = captured - assert not set(flip_states) & set(initial) - assert not set(flip_states) & set(schema_entries) + assert not set(local_states) & set(initial) + assert not set(local_states) & set(schema_entries) marker = json.loads((web / disk_cache._MANIFEST_FILE).read_text()) assert set(marker["pages"]) == {"a"} @@ -2552,3 +2631,101 @@ def test_incremental_rebuild_recompiles_only_the_edited_function( sys.modules.pop(module_name, None) assert result is True assert mod.CALLS == {"a": 1, "b": 0} + + +def _labelled(label: str) -> Component: + return rx.el.p(label) + + +def test_registration_digest_tracks_partial_arguments(): + """A page bound with ``functools.partial`` changes with its arguments.""" + import functools + + def digest(label: str) -> str: + page = _FakePage(route="/p", component=functools.partial(_labelled, label)) + return disk_cache.registration_digests([page])["/p"] + + assert digest("old") == digest("old") + assert digest("old") != digest("new") + + +def test_partial_page_depends_on_its_function_module(tmp_path, monkeypatch): + """A ``functools.partial`` page depends on its wrapped function's file.""" + import functools + import importlib + + module = tmp_path / "partial_view_module.py" + module.write_text("def view(label):\n return label\n") + monkeypatch.syspath_prepend(str(tmp_path)) + try: + view = importlib.import_module("partial_view_module").view + files = page_cache._component_source_files( + functools.partial(view, "x"), tmp_path.resolve() + ) + finally: + sys.modules.pop("partial_view_module", None) + assert files == {str(module.resolve())} + + +def test_dynamic_state_pages_bring_their_memo_siblings(): + """A dynamic-state page pulled into a rebuild brings its memo siblings. + + They share its module's memo file, which the rebuild rewrites. + """ + pages = [ + _FakePage("/a", _page_a, _source_module="first_module"), + _FakePage("/b", _page_b, _source_module="second_module"), + _FakePage("/c", _page_c, _source_module="second_module"), + ] + + def entry(claims: list[list[str]], has_memos: bool) -> dict[str, Any]: + return {"dynamic_claims": claims, "has_memos": has_memos} + + manifest = { + "pages": { + "/a": entry([], has_memos=False), + "/b": entry([["LocalState", "LocalState"]], has_memos=True), + "/c": entry([], has_memos=True), + } + } + misses = disk_cache._expand_miss_pages( + [pages[0]], pages, manifest, set(), set(), dynamic_names_shift=False + ) + assert [page.route for page in misses] == ["/a", "/b", "/c"] + + +@pytest.mark.parametrize("predict", [False, True]) +def test_reordering_dynamic_state_pages_reevaluates_them( + tmp_path, monkeypatch, predict +): + """Dynamic state names follow page order, so a reorder renames them.""" + web = _use_tmp_web_dir(tmp_path, monkeypatch) + app = rx.App() + app.add_page(_page_collide_one, route="/one") + app.add_page(_page_collide_two, route="/two") + pages = list(app._unevaluated_pages.values()) + ctx = _compile(pages, app=app) + _stub_externals(app, monkeypatch) + disk_cache.write_manifest(ctx, pages, ctx.all_imports, root=tmp_path) + assert (web / disk_cache._MANIFEST_FILE).exists() + monkeypatch.setattr( + app, "_unevaluated_pages", dict(reversed(app._unevaluated_pages.items())) + ) + _drop_dynamic_states("CollideState") + monkeypatch.setitem(_COLLIDE, "one", 0) + monkeypatch.setitem(_COLLIDE, "two", 0) + monkeypatch.setattr(disk_cache, "_predict_dynamic_names", predict) + try: + if predict: + with pytest.raises(disk_cache.DynamicStatePredictionError): + disk_cache.try_incremental_rebuild( + app, compiler_plugins=[], prerender_routes=False, root=tmp_path + ) + return + result = disk_cache.try_incremental_rebuild( + app, compiler_plugins=[], prerender_routes=False, root=tmp_path + ) + finally: + _drop_dynamic_states("CollideState") + assert result is True + assert (_COLLIDE["one"], _COLLIDE["two"]) == (1, 1) diff --git a/tests/units/compiler/test_page_cache.py b/tests/units/compiler/test_page_cache.py index a47155cb7e1..b77df4f3408 100644 --- a/tests/units/compiler/test_page_cache.py +++ b/tests/units/compiler/test_page_cache.py @@ -989,3 +989,99 @@ def test_file_validator_compares_function_slices(tmp_path): assert not page_cache.FileValidator(files).unchanged([dep]) assert page_cache.dep_file(dep) == str(module) assert page_cache.dep_file(str(module)) == str(module) + + +def test_read_tracking_root_change_forgets_import_reads(tmp_path, monkeypatch): + """A cached module's import-time reads from one root never reach another.""" + root_a = tmp_path / "app_a" + root_a.mkdir() + data = root_a / "data.json" + data.write_text("{}") + module_name = "root_change_import_reads" + _prepare_runtime_module(root_a, monkeypatch, module_name).write_text( + f"from pathlib import Path\nDATA = Path({str(data)!r}).read_text()\n" + ) + page_cache.enable_read_tracking(root=root_a) + try: + with page_cache.record_reads() as reads_a: + importlib.import_module(module_name) + assert str(data.resolve()) in reads_a + page_cache.enable_read_tracking(root=tmp_path / "app_b") + with page_cache.record_reads() as reads_b: + importlib.import_module(module_name) + finally: + _forget_modules(module_name) + assert str(data.resolve()) not in reads_b + + +def test_failed_import_keeps_its_reads(tmp_path, monkeypatch): + """A page that handles a failed import still depends on what it read.""" + missing = tmp_path / "optional.json" + module_name = "optional_data_helper" + _prepare_runtime_module(tmp_path, monkeypatch, module_name).write_text( + f"from pathlib import Path\nDATA = Path({str(missing)!r}).read_text()\n" + ) + page_cache.enable_read_tracking(root=tmp_path) + try: + with page_cache.record_reads() as reads, pytest.raises(FileNotFoundError): + importlib.import_module(module_name) + finally: + _forget_modules(module_name) + assert str(missing.resolve()) in reads + + +def test_function_slice_digest_keeps_imports_of_code_reached_by_computed_name( + tmp_path, +): + """With a computed global lookup, every definition's imports count too.""" + module = tmp_path / "computed.py" + source = ( + "from .widgets import card\n\n\n" + "def widget():\n return card()\n\n\n" + 'def page_a():\n return globals()["widget"]()\n' + ) + module.write_text(source) + before = page_cache.function_slice_digest(str(module), "page_a") + module.write_text( + source.replace("from .widgets import card", "from .other import card") + ) + assert page_cache.function_slice_digest(str(module), "page_a") != before + + +@pytest.mark.parametrize( + ("decorator", "inert"), + [ + ("@rx.page(route='/x')", True), + ("@rx.memo", True), + ("@functools.cache", True), + ("@functools.lru_cache(maxsize=4)", True), + ("@registry.page", False), + ("@cache", False), + ], +) +def test_only_known_decorators_keep_a_function_out_of_other_slices( + tmp_path, decorator, inert +): + """A decorator of unknown origin may register the function it wraps.""" + module = tmp_path / "decorated.py" + source = ( + f"{decorator}\ndef widget():\n return 1\n\n\ndef page_a():\n return 2\n" + ) + module.write_text(source) + before = page_cache.function_slice_digest(str(module), "page_a") + module.write_text(source.replace("return 1", "return 3")) + after = page_cache.function_slice_digest(str(module), "page_a") + assert (after == before) is inert + + +def test_entrypoint_digest_tracks_class_bodies(tmp_path): + """A class body runs at import, so it can configure the app.""" + entry = tmp_path / "entry_app.py" + source = ( + "import reflex as rx\n\n\nclass Settings(Base):\n color = 'red'\n\n\n" + "app = rx.App()\n" + ) + entry.write_text(source) + before = page_cache.entrypoint_digest(str(entry)) + entry.write_text(source.replace("'red'", "'blue'")) + assert page_cache.entrypoint_digest(str(entry)) != before diff --git a/tests/units/utils/test_compile_daemon.py b/tests/units/utils/test_compile_daemon.py index 3479e1c6d07..7615c945be8 100644 --- a/tests/units/utils/test_compile_daemon.py +++ b/tests/units/utils/test_compile_daemon.py @@ -1,6 +1,7 @@ """Tests for the warm fork-per-compile dev compile daemon.""" import contextlib +import errno import os import sys from pathlib import Path @@ -993,3 +994,82 @@ def test_cold_compile_without_a_safe_fork_compiles_in_process( monkeypatch.setattr(compile_daemon, "_single_threaded", lambda: single_threaded) monkeypatch.setattr(disk_cache, "_predict_dynamic_names", chosen) assert compile_daemon.compiled_in_predicting_child(lambda: True) is False + + +@pytest.mark.parametrize("lock", ["held", "unsupported"]) +def test_daemon_without_its_lock_does_not_serve(tmp_path, monkeypatch, lock): + """Only the daemon holding the ownership lock compiles the project.""" + from unittest.mock import Mock + + from reflex.utils import path_ops + + lock_path = tmp_path / "daemon.lock" + monkeypatch.setattr(compile_daemon, "_daemon_lock_path", lambda: lock_path) + monkeypatch.setattr(compile_daemon, "_LOCK_ACQUIRE_TIMEOUT", 0.05) + served: list[bool] = [] + monkeypatch.setattr(compile_daemon, "_serve", lambda: served.append(True)) + holder = path_ops.try_lock(lock_path) if lock == "held" else None + if lock == "unsupported": + monkeypatch.setattr( + path_ops, "try_lock", Mock(side_effect=OSError(errno.ENOLCK, "no locks")) + ) + try: + assert compile_daemon.main([]) == 0 + finally: + if holder is not None: + os.close(holder) + assert served == [] + + +def test_built_inputs_include_dependencies_the_compile_added(tmp_path, monkeypatch): + """Workers wait on an edit to a file the last compile started reading. + + The file was not watched before that compile, which is no edit. + """ + from reflex.utils import prerequisites + + page, data = tmp_path / "page.py", tmp_path / "data.json" + page.write_text("TITLE = 'v1'\n") + data.write_text("{}") + known = {page} + monkeypatch.chdir(tmp_path) + monkeypatch.setenv("REFLEX_COMPILE_DAEMON_PRECOMPILED", "1") + monkeypatch.setattr(compile_daemon, "_lock_path", lambda: tmp_path / "lock") + monkeypatch.setattr(compile_daemon, "_reload_roots", lambda: [tmp_path]) + monkeypatch.setattr(compile_daemon, "_prepare_fork_parent", lambda roots: None) + monkeypatch.setattr(compile_daemon, "_unbuilt_changes", lambda state: set()) + monkeypatch.setattr(prerequisites, "get_app", lambda **kwargs: None) + monkeypatch.setattr("reflex.utils.prerequisites.get_backend_dir", lambda: tmp_path) + monkeypatch.setattr( + compile_daemon._WatchState, + "build", + lambda *args: compile_daemon._WatchState( + [tmp_path], tmp_path, known=set(known) + ), + ) + batches = iter([{page}, None]) + monkeypatch.setattr(compile_daemon, "_next_changes", lambda *args: next(batches)) + + def compile_once(roots, prerender_routes, changed=None) -> bool: + """Compile a page that now reads ``data``. + + Args: + roots: The reload roots. + prerender_routes: Whether to prerender routes. + changed: The changed-path hint. + + Returns: + True. + """ + compiles.append(changed) + known.add(data) + return True + + compiles: list[set[Path] | None] = [] + monkeypatch.setattr(compile_daemon, "_compile_once", compile_once) + compile_daemon._serve() + assert compiles == [{page}] + assert compile_daemon._built_inputs_current() + data.write_text('{"edited": true}') + os.utime(data, ns=(1, 1)) + assert not compile_daemon._built_inputs_current() diff --git a/tests/units/utils/test_path_ops.py b/tests/units/utils/test_path_ops.py index 2e2987c4da7..ca1e9dc5ecf 100644 --- a/tests/units/utils/test_path_ops.py +++ b/tests/units/utils/test_path_ops.py @@ -979,3 +979,16 @@ def test_lock_is_released_when_its_holder_is_killed(tmp_path: Path): holder.wait() # Windows can release byte-range locks asynchronously after process exit. assert AppHarness._poll_for(lambda: not path_ops.is_locked(lock), timeout=5) + + +@pytest.mark.skipif(sys.platform == "win32", reason="POSIX flock") +def test_unsupported_file_locks_have_no_holder(tmp_path: Path, monkeypatch): + """A file system that cannot lock files reports no holder, not a held lock.""" + path = tmp_path / "owner.lock" + path.touch() + monkeypatch.setattr( + fcntl, "flock", Mock(side_effect=OSError(errno.ENOLCK, "No locks available")) + ) + assert path_ops.is_locked(path) is False + with pytest.raises(OSError, match="No locks available"): + path_ops.try_lock(path) From 1354854f61d1f1e02d9b87e0d69ffa6c12187044 Mon Sep 17 00:00:00 2001 From: Masen Furer <m_github@0x26.net> Date: Tue, 20 Oct 2026 15:53:18 -0700 Subject: [PATCH 44/56] test(compiler): advance timestamps for slice digest edits Immediate same-size writes can keep the previous modification timestamp on Windows. Give the edit fixtures distinct mtimes so they exercise the slice digest instead of reusing the stat cache. --- tests/units/compiler/test_page_cache.py | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/tests/units/compiler/test_page_cache.py b/tests/units/compiler/test_page_cache.py index b77df4f3408..c4dbca22b23 100644 --- a/tests/units/compiler/test_page_cache.py +++ b/tests/units/compiler/test_page_cache.py @@ -861,6 +861,20 @@ def page_b(): """ +def _rewrite_module(path: Path, source: str) -> None: + """Rewrite a module with a distinct modification time for the stat cache. + + Immediate same-size writes can retain their timestamp on Windows. + + Args: + path: The existing module file. + source: The replacement source. + """ + stat = path.stat() + path.write_text(source) + os.utime(path, ns=(stat.st_atime_ns, stat.st_mtime_ns + 1_000_000_000)) + + @pytest.mark.parametrize( ("old", "new", "changes"), [ @@ -887,7 +901,7 @@ def test_function_slice_digest_covers_what_the_function_reads( module.write_text(_SLICED_MODULE) before = page_cache.function_slice_digest(str(module), "page_a") assert before is not None - module.write_text(_SLICED_MODULE.replace(old, new, 1)) + _rewrite_module(module, _SLICED_MODULE.replace(old, new, 1)) after = page_cache.function_slice_digest(str(module), "page_a") assert (after != before) is changes @@ -913,7 +927,7 @@ def test_function_slice_digest_keeps_code_reached_indirectly(tmp_path, source): module = tmp_path / "indirect.py" module.write_text(source) before = page_cache.function_slice_digest(str(module), "page_a") - module.write_text(source.replace("return 1", "return 2")) + _rewrite_module(module, source.replace("return 1", "return 2")) assert page_cache.function_slice_digest(str(module), "page_a") != before @@ -1069,7 +1083,7 @@ def test_only_known_decorators_keep_a_function_out_of_other_slices( ) module.write_text(source) before = page_cache.function_slice_digest(str(module), "page_a") - module.write_text(source.replace("return 1", "return 3")) + _rewrite_module(module, source.replace("return 1", "return 3")) after = page_cache.function_slice_digest(str(module), "page_a") assert (after == before) is inert From 98e0259279f5063ca87f7475fdf8bfcac040d3d4 Mon Sep 17 00:00:00 2001 From: Masen Furer <m_github@0x26.net> Date: Tue, 20 Oct 2026 17:49:08 -0700 Subject: [PATCH 45/56] Retry compile-input record replacement blocked by Windows readers --- news/+compile-daemon-windows-record.bugfix.md | 1 + reflex/utils/compile_daemon.py | 21 ++++- tests/units/utils/test_compile_daemon.py | 88 ++++++++++++++++++- 3 files changed, 105 insertions(+), 5 deletions(-) create mode 100644 news/+compile-daemon-windows-record.bugfix.md diff --git a/news/+compile-daemon-windows-record.bugfix.md b/news/+compile-daemon-windows-record.bugfix.md new file mode 100644 index 00000000000..c08ca75707e --- /dev/null +++ b/news/+compile-daemon-windows-record.bugfix.md @@ -0,0 +1 @@ +Avoid Windows hot-reload failures when a backend worker briefly holds the compile-input record open. diff --git a/reflex/utils/compile_daemon.py b/reflex/utils/compile_daemon.py index 22fb775c48a..0545c3f8d2d 100644 --- a/reflex/utils/compile_daemon.py +++ b/reflex/utils/compile_daemon.py @@ -649,6 +649,9 @@ def _reset_model_metadata() -> None: #: inputs the frontend build reflects, so a backend reload worker can tell an #: edit the daemon has not compiled yet from one it has. _BUILT_INPUTS_FILE = ".compiled_inputs.json" +#: Briefly retry Windows readers that prevent atomically replacing the record. +_BUILT_INPUTS_WRITE_ATTEMPTS = 3 +_BUILT_INPUTS_RETRY_DELAY = 0.01 #: Lock file (under the backend dir) whose OS lock the daemon holds for its #: lifetime. Backend workers skip frontend compilation while it is held. A #: file, not an environment variable: workers may be forked from a @@ -767,8 +770,22 @@ def _record_built_inputs(snapshot: dict[Path, int]) -> None: path = _built_inputs_path() path.parent.mkdir(parents=True, exist_ok=True) tmp = path.with_name(f"{path.name}.{os.getpid()}.tmp") - tmp.write_text(json.dumps({str(p): m for p, m in snapshot.items()})) - tmp.replace(path) + try: + tmp.write_text(json.dumps({str(p): m for p, m in snapshot.items()})) + for attempt in range(_BUILT_INPUTS_WRITE_ATTEMPTS): + try: + tmp.replace(path) + except PermissionError: + if ( + not constants.IS_WINDOWS + or attempt == _BUILT_INPUTS_WRITE_ATTEMPTS - 1 + ): + raise + time.sleep(_BUILT_INPUTS_RETRY_DELAY * (attempt + 1)) + else: + return + finally: + tmp.unlink(missing_ok=True) def _built_inputs_current() -> bool: diff --git a/tests/units/utils/test_compile_daemon.py b/tests/units/utils/test_compile_daemon.py index 7615c945be8..f10ea0e485c 100644 --- a/tests/units/utils/test_compile_daemon.py +++ b/tests/units/utils/test_compile_daemon.py @@ -5,6 +5,7 @@ import os import sys from pathlib import Path +from unittest.mock import Mock import pytest @@ -259,6 +260,84 @@ def test_wait_for_compile_respects_lock(tmp_path, monkeypatch): assert time.monotonic() - start < 0.5 +@pytest.mark.parametrize("blocked_attempts", [1, 2]) +def test_record_built_inputs_retries_windows_readers( + tmp_path, monkeypatch, blocked_attempts +): + """Publish the new record after Windows readers release the old record.""" + monkeypatch.setattr("reflex.utils.prerequisites.get_backend_dir", lambda: tmp_path) + monkeypatch.setattr(compile_daemon.constants, "IS_WINDOWS", True) + page = tmp_path / "page.py" + page.write_text("v1") + built = compile_daemon._snapshot({page}) + compile_daemon._record_built_inputs(built) + record = compile_daemon._built_inputs_path() + previous_record = record.read_text() + mtime = built[page] + 1_000_000_000 + os.utime(page, ns=(mtime, mtime)) + replace = Path.replace + attempts = 0 + + def replace_after_reader_closes(source, target): + """Keep the previous record intact while a reader blocks replacement. + + Args: + source: Temporary record to publish. + target: Destination record. + + Returns: + The destination after the reader releases it. + + Raises: + PermissionError: While the reader still holds the destination open. + """ + nonlocal attempts + attempts += 1 + assert record.read_text() == previous_record + assert not compile_daemon._built_inputs_current() + if attempts <= blocked_attempts: + raise PermissionError(errno.EACCES, "Reader still has the record open") + return replace(source, target) + + monkeypatch.setattr(Path, "replace", replace_after_reader_closes) + sleep = Mock() + monkeypatch.setattr(compile_daemon.time, "sleep", sleep) + compile_daemon._record_built_inputs(compile_daemon._snapshot({page})) + assert compile_daemon._built_inputs_current() + assert sleep.call_count == blocked_attempts + assert not list(tmp_path.glob("*.tmp")) + + +@pytest.mark.parametrize( + ("is_windows", "error", "attempts"), + [ + (True, PermissionError(errno.EACCES, "Access denied"), 3), + (False, PermissionError(errno.EACCES, "Access denied"), 1), + (True, OSError(errno.ENOSPC, "No space left"), 1), + ], +) +def test_record_built_inputs_propagates_errors( + tmp_path, monkeypatch, is_windows, error, attempts +): + """Bound retries, retain the old record, and remove a failed temporary write.""" + monkeypatch.setattr("reflex.utils.prerequisites.get_backend_dir", lambda: tmp_path) + monkeypatch.setattr(compile_daemon.constants, "IS_WINDOWS", is_windows) + compile_daemon._record_built_inputs({}) + record = compile_daemon._built_inputs_path() + previous_record = record.read_text() + replace = Mock(side_effect=error) + monkeypatch.setattr(Path, "replace", replace) + sleep = Mock() + monkeypatch.setattr(compile_daemon.time, "sleep", sleep) + with pytest.raises(type(error)) as raised: + compile_daemon._record_built_inputs({tmp_path / "page.py": 1}) + assert raised.value is error + assert replace.call_count == attempts + assert sleep.call_count == attempts - 1 + assert record.read_text() == previous_record + assert not list(tmp_path.glob("*.tmp")) + + def test_wait_for_compile_waits_for_unbuilt_edits(tmp_path, monkeypatch): """Without a lock, a worker still waits while a built input has changed. @@ -288,9 +367,12 @@ def test_wait_for_compile_waits_for_unbuilt_edits(tmp_path, monkeypatch): ) timer.start() start = time.monotonic() - compile_daemon.wait_for_compile(timeout=5) - assert 0.1 <= time.monotonic() - start < 1 - timer.join() + try: + compile_daemon.wait_for_compile(timeout=5) + assert time.monotonic() - start < 1 + assert compile_daemon._built_inputs_current() + finally: + timer.join() def test_first_party_module_names_caches_file_classification(tmp_path, monkeypatch): From 0d40332c0fc642acb33eb8b2ecf0de10df17c23c Mon Sep 17 00:00:00 2001 From: Farhan <www.mfarvirus@gmail.com> Date: Wed, 21 Oct 2026 17:03:52 +0500 Subject: [PATCH 46/56] fix(compiler): address second round of compile cache review findings - A process that serves the app compiles without the incremental cache, so every page registers its states. - The daemon compares function-slice entries when it checks for edits made before its first snapshot, and keeps a newly tracked input pending unless the manifest records its current content. - A computed global lookup keeps every import in the function slice. --- reflex/app.py | 5 ++ reflex/compiler/compiler.py | 11 ++++ reflex/compiler/page_cache.py | 4 +- reflex/utils/compile_daemon.py | 64 +++++++++++++++++++----- tests/units/compiler/test_compiler.py | 26 ++++++++++ tests/units/compiler/test_page_cache.py | 14 ++++++ tests/units/test_app.py | 8 +++ tests/units/utils/test_compile_daemon.py | 53 ++++++++++++++++++-- 8 files changed, 167 insertions(+), 18 deletions(-) diff --git a/reflex/app.py b/reflex/app.py index ffcde3fbc11..102e19e4dd3 100644 --- a/reflex/app.py +++ b/reflex/app.py @@ -807,6 +807,7 @@ def __call__(self) -> ASGIApp: self._compile( prerender_routes=should_prerender_routes(), trigger=trigger, + serving=True, ) # In preview mode the frontend is served as a mounted static bundle rather @@ -1732,6 +1733,7 @@ def _compile( dry_run: bool = False, use_rich: bool = True, trigger: CompileTrigger | None = None, + serving: bool = False, ): """Compile the app and output it to the pages folder. @@ -1741,6 +1743,7 @@ def _compile( use_rich: Whether to use rich progress bars. trigger: Label identifying what initiated this compile. Recorded on the ``compile`` telemetry event. + serving: Whether this process serves the app after compiling. Raises: ReflexRuntimeError: When any page uses state, but no rx.State subclass is defined. @@ -1757,6 +1760,7 @@ def _compile( prerender_routes=prerender_routes, dry_run=dry_run, use_rich=use_rich, + serving=serving, ) return @@ -1768,6 +1772,7 @@ def _compile( prerender_routes=prerender_routes, dry_run=dry_run, use_rich=use_rich, + serving=serving, ) except Exception as exc: ctx.set_exception(exc) diff --git a/reflex/compiler/compiler.py b/reflex/compiler/compiler.py index 4709a413bb4..c1c57d4d0ad 100644 --- a/reflex/compiler/compiler.py +++ b/reflex/compiler/compiler.py @@ -1501,9 +1501,19 @@ def compile_app( prerender_routes: bool = False, dry_run: bool = False, use_rich: bool = True, + serving: bool = False, ) -> bool: """Compile an app using the compiler plugin pipeline. + Args: + app: The app to compile. + prerender_routes: Whether to prerender the routes. + dry_run: Whether to compile the app without saving it. + use_rich: Whether to use rich progress bars. + serving: Whether this process serves the app after compiling. It + needs every page evaluated to register their states, so the + incremental cache is not used. + Returns: ``True`` when a real frontend compile ran, ``False`` when the call short-circuited (backend-only paths that only re-evaluate pages). @@ -1561,6 +1571,7 @@ def compile_app( cache_on = ( not dry_run + and not serving and environment.REFLEX_COMPILE_CACHE.get() and config.app_module_import is None ) diff --git a/reflex/compiler/page_cache.py b/reflex/compiler/page_cache.py index 99fea19f8c3..642c4af5ad0 100644 --- a/reflex/compiler/page_cache.py +++ b/reflex/compiler/page_cache.py @@ -793,9 +793,11 @@ def _slice_digest(parts: Sequence[_ModulePart], seeds: Iterable[str]) -> str: closure = _definition_closure(definitions, (*seeds, *always)) used = always.union(*(part.loads for name in closure for part in definitions[name])) if not used.isdisjoint(_DYNAMIC_GLOBAL_NAMES): + # A computed name can reach every module global, imports included. closure = set(definitions) used = always.union( - *(part.loads for group in definitions.values() for part in group) + *(part.loads for group in definitions.values() for part in group), + (bound for part in parts for bound, _ in part.aliases), ) digest = hashlib.sha256() for part in parts: diff --git a/reflex/utils/compile_daemon.py b/reflex/utils/compile_daemon.py index 0545c3f8d2d..51ac02c1193 100644 --- a/reflex/utils/compile_daemon.py +++ b/reflex/utils/compile_daemon.py @@ -18,9 +18,10 @@ import sys import threading import time -from collections.abc import Callable, Iterator +from collections.abc import Callable, Iterable, Iterator from dataclasses import dataclass, field from pathlib import Path +from typing import TYPE_CHECKING from reflex_base import constants from reflex_base.environment import environment @@ -28,6 +29,9 @@ from reflex.utils import path_ops +if TYPE_CHECKING: + from reflex.compiler import page_cache + logger = logging.getLogger(__name__) #: Fallback polling only (no ``watchfiles``): seconds between poll passes. @@ -460,6 +464,34 @@ def _known_snapshot(state: _WatchState, snapshot: dict[Path, int]) -> dict[Path, return {p: m for p, m in snapshot.items() if p in state.known} +def _recorded_inputs( + paths: Iterable[Path], +) -> tuple[page_cache.FileValidator, dict[Path, list[str]]] | None: + """Find the manifest entries of input files. + + A file is recorded whole, as function slices (``<file>::<function>``), + or both. + + Args: + paths: The input files. + + Returns: + A validator for the manifest's file table and each recorded file's + entries, or None without a manifest. + """ + from reflex.compiler import disk_cache, page_cache + + if (manifest := disk_cache.load_manifest()) is None: + return None + files = manifest["files"] + wanted = {str(path): path for path in paths} + entries: dict[Path, list[str]] = {} + for key in files: + if (path := wanted.get(page_cache.dep_file(key))) is not None: + entries.setdefault(path, []).append(key) + return page_cache.FileValidator(files), entries + + def _unbuilt_changes(state: _WatchState) -> set[Path]: """Recorded compile inputs that differ from the manifest on disk. @@ -469,17 +501,25 @@ def _unbuilt_changes(state: _WatchState) -> set[Path]: Returns: The inputs edited since the last build wrote its manifest. """ - from reflex.compiler import disk_cache, page_cache + if (recorded := _recorded_inputs(state.known)) is None: + return set() + validator, entries = recorded + return {path for path, keys in entries.items() if not validator.unchanged(keys)} - if (manifest := disk_cache.load_manifest()) is None: + +def _built_inputs(paths: set[Path]) -> set[Path]: + """Inputs the manifest records with their current content. + + Args: + paths: The input files. + + Returns: + The paths that have entries, all unchanged. + """ + if (recorded := _recorded_inputs(paths)) is None: return set() - files = manifest["files"] - validator = page_cache.FileValidator(files) - return { - path - for path in state.known - if str(path) in files and validator.changed(str(path)) - } + validator, entries = recorded + return {path for path, keys in entries.items() if validator.unchanged(keys)} def _missed_changes(before: dict[Path, int], after: dict[Path, int]) -> set[Path]: @@ -1209,10 +1249,10 @@ def alive() -> bool: logger.error("Compile failed; keeping the last good build.") # Catch up on edits made while the watcher was down for the compile. An # input the compile started to read was not watched before it; it - # changed only if it differs from what the compile recorded. + # changed unless the compile recorded its current content. pending = _missed_changes(before, state.checkpoint) if appeared := (pending - before.keys()) & state.known: - pending -= appeared - _unbuilt_changes(state) + pending -= _built_inputs(appeared) def main(argv: list[str] | None = None) -> int: diff --git a/tests/units/compiler/test_compiler.py b/tests/units/compiler/test_compiler.py index 59a14579055..0b72173350a 100644 --- a/tests/units/compiler/test_compiler.py +++ b/tests/units/compiler/test_compiler.py @@ -2000,6 +2000,32 @@ class FullCompileReached(Exception): reuse.assert_not_called() +def test_serving_app_compiles_every_page(monkeypatch, mocker): + """A process that serves the app registers every page's states. + + The incremental cache evaluates only changed pages, so it is not used. + """ + from reflex.compiler import disk_cache + + mocker.patch("reflex.app.reload_config") + app = rx.App() + mocker.patch.object(app, "_should_compile", return_value=True) + reuse = mocker.patch.object( + disk_cache, "try_incremental_rebuild", return_value=True + ) + monkeypatch.setenv("REFLEX_COMPILE_CACHE", "1") + + class FullCompileReached(Exception): + """Stop before full-compile side effects.""" + + mocker.patch.object( + compiler.console, "PoorProgress", side_effect=FullCompileReached + ) + with pytest.raises(FullCompileReached): + compiler.compile_app(app, use_rich=False, serving=True) + reuse.assert_not_called() + + def test_compile_app_drops_event_caches_from_earlier_compiles( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, mocker: MockerFixture ): diff --git a/tests/units/compiler/test_page_cache.py b/tests/units/compiler/test_page_cache.py index c4dbca22b23..1da463b825d 100644 --- a/tests/units/compiler/test_page_cache.py +++ b/tests/units/compiler/test_page_cache.py @@ -1099,3 +1099,17 @@ def test_entrypoint_digest_tracks_class_bodies(tmp_path): before = page_cache.entrypoint_digest(str(entry)) entry.write_text(source.replace("'red'", "'blue'")) assert page_cache.entrypoint_digest(str(entry)) != before + + +def test_function_slice_digest_keeps_imports_reached_only_by_computed_name(tmp_path): + """A computed global lookup can reach an import no definition names.""" + module = tmp_path / "computed_import.py" + source = ( + "from reflex import text as widget\n\n\n" + 'def page_a():\n return globals()["widget"]()\n' + ) + module.write_text(source) + before = page_cache.function_slice_digest(str(module), "page_a") + module.write_text(source.replace("text as widget", "heading as widget")) + os.utime(module, ns=(1, 1)) + assert page_cache.function_slice_digest(str(module), "page_a") != before diff --git a/tests/units/test_app.py b/tests/units/test_app.py index c8897bb18f3..6211016fada 100644 --- a/tests/units/test_app.py +++ b/tests/units/test_app.py @@ -5799,3 +5799,11 @@ def reader(): for future in readers: future.result() assert json.loads(marker.read_text()) == routes + + +def test_backend_compile_is_marked_serving(mocker): + """The backend's own compile tells the compiler that it serves the app.""" + app = App() + compile_ = mocker.patch.object(app, "_compile") + app() + assert compile_.call_args.kwargs["serving"] is True diff --git a/tests/units/utils/test_compile_daemon.py b/tests/units/utils/test_compile_daemon.py index f10ea0e485c..e2657a5e91b 100644 --- a/tests/units/utils/test_compile_daemon.py +++ b/tests/units/utils/test_compile_daemon.py @@ -5,6 +5,7 @@ import os import sys from pathlib import Path +from typing import Any from unittest.mock import Mock import pytest @@ -1075,7 +1076,19 @@ def test_cold_compile_without_a_safe_fork_compiles_in_process( monkeypatch.setattr(compile_daemon, "_single_threaded", lambda: single_threaded) monkeypatch.setattr(disk_cache, "_predict_dynamic_names", chosen) - assert compile_daemon.compiled_in_predicting_child(lambda: True) is False + attempts: list[bool] = [] + + def attempt() -> bool: + """Record a compile the caller must not have handed over. + + Returns: + True. + """ + attempts.append(True) + return True + + assert compile_daemon.compiled_in_predicting_child(attempt) is False + assert attempts == [] @pytest.mark.parametrize("lock", ["held", "unsupported"]) @@ -1103,23 +1116,32 @@ def test_daemon_without_its_lock_does_not_serve(tmp_path, monkeypatch, lock): assert served == [] -def test_built_inputs_include_dependencies_the_compile_added(tmp_path, monkeypatch): +@pytest.mark.parametrize("recorded", [True, False]) +def test_built_inputs_include_dependencies_the_compile_added( + tmp_path, monkeypatch, recorded +): """Workers wait on an edit to a file the last compile started reading. - The file was not watched before that compile, which is no edit. + The file was not watched before that compile, which is no edit when the + manifest records it as read. A file created after the compile read it as + missing is an edit. """ + from reflex.compiler import disk_cache, page_cache from reflex.utils import prerequisites page, data = tmp_path / "page.py", tmp_path / "data.json" page.write_text("TITLE = 'v1'\n") - data.write_text("{}") + files: dict[str, Any] = {} + if recorded: + data.write_text("{}") + files[str(data)] = page_cache.file_entry(str(data)) known = {page} monkeypatch.chdir(tmp_path) monkeypatch.setenv("REFLEX_COMPILE_DAEMON_PRECOMPILED", "1") monkeypatch.setattr(compile_daemon, "_lock_path", lambda: tmp_path / "lock") monkeypatch.setattr(compile_daemon, "_reload_roots", lambda: [tmp_path]) monkeypatch.setattr(compile_daemon, "_prepare_fork_parent", lambda roots: None) - monkeypatch.setattr(compile_daemon, "_unbuilt_changes", lambda state: set()) + monkeypatch.setattr(disk_cache, "load_manifest", lambda: {"files": files}) monkeypatch.setattr(prerequisites, "get_app", lambda **kwargs: None) monkeypatch.setattr("reflex.utils.prerequisites.get_backend_dir", lambda: tmp_path) monkeypatch.setattr( @@ -1145,13 +1167,34 @@ def compile_once(roots, prerender_routes, changed=None) -> bool: """ compiles.append(changed) known.add(data) + if not data.exists(): + data.write_text("{}") return True compiles: list[set[Path] | None] = [] monkeypatch.setattr(compile_daemon, "_compile_once", compile_once) compile_daemon._serve() + if not recorded: + assert compiles == [{page}, {data}] + return assert compiles == [{page}] assert compile_daemon._built_inputs_current() data.write_text('{"edited": true}') os.utime(data, ns=(1, 1)) assert not compile_daemon._built_inputs_current() + + +def test_unbuilt_changes_compare_function_slices(tmp_path, monkeypatch): + """A module recorded only as function slices still reports its edits.""" + from reflex.compiler import disk_cache, page_cache + + page = tmp_path / "page.py" + page.write_text("def index():\n return 'v1'\n") + dep = f"{page}{page_cache.SLICE_SEPARATOR}index" + files = {dep: page_cache.file_entry(dep)} + monkeypatch.setattr(disk_cache, "load_manifest", lambda: {"files": files}) + state = compile_daemon._WatchState([tmp_path], tmp_path, known={page}) + assert compile_daemon._unbuilt_changes(state) == set() + page.write_text("def index():\n return 'edited'\n") + os.utime(page, ns=(1, 1)) + assert compile_daemon._unbuilt_changes(state) == {page} From a8f05fb8a3b72303f3408cf30462942c3bc0960c Mon Sep 17 00:00:00 2001 From: Farhan <www.mfarvirus@gmail.com> Date: Wed, 21 Oct 2026 13:38:11 +0000 Subject: [PATCH 47/56] fix(compiler): keep the compile manifest consistent with uncached output - A compile that bypasses the incremental cache (a serving process, a preloaded app, or the flag unset) discards the manifest before it writes frontend output, so a later cached compile cannot reuse files that no longer match what the manifest describes. - The daemon's check for edits made before its first snapshot compares only the file-table entries the manifest still references. An incremental rebuild extends the table without pruning it, so a renamed page function left a stale slice that reported the module as unbuilt on every start. Claude-Session: https://claude.ai/code/session_01Y27XUSyj8uGXAaLaYyzYsa --- reflex/compiler/compiler.py | 9 ++++-- reflex/compiler/disk_cache.py | 26 ++++++++++++++++ reflex/utils/compile_daemon.py | 7 +++-- tests/units/compiler/test_compiler.py | 30 ++++++++++++++++++ tests/units/utils/test_compile_daemon.py | 39 +++++++++++++++++++++--- 5 files changed, 102 insertions(+), 9 deletions(-) diff --git a/reflex/compiler/compiler.py b/reflex/compiler/compiler.py index c1c57d4d0ad..4302369fbe1 100644 --- a/reflex/compiler/compiler.py +++ b/reflex/compiler/compiler.py @@ -1839,13 +1839,18 @@ def attempt() -> CompileContext | bool: output_mapping, config.plugins, modify_files_tasks ) + from reflex.compiler import disk_cache + + if not cache_on: + # The output written below is not recorded: a manifest an earlier cached + # compile left must not validate files this compile overwrites. + disk_cache._manifest_path().unlink(missing_ok=True) + with log.timing(logger, "Write to Disk"), otel.span("reflex.compile.write"): for output_path, code in output_mapping.items(): utils.write_file(output_path, code) if cache_on: - from reflex.compiler import disk_cache - disk_cache.write_manifest( compile_ctx, all_pages, diff --git a/reflex/compiler/disk_cache.py b/reflex/compiler/disk_cache.py index 123422b3279..c5725a63072 100644 --- a/reflex/compiler/disk_cache.py +++ b/reflex/compiler/disk_cache.py @@ -633,6 +633,32 @@ def load_manifest() -> dict[str, Any] | None: return data +def manifest_dependency_entries(manifest: dict[str, Any]) -> set[str]: + """Every file-table key the manifest's current entries reference. + + An incremental rebuild extends the file table without pruning it, so a + dependency a page dropped (a renamed function's slice, say) keeps its + entry; comparing a file's recorded content must go through these keys. + + Args: + manifest: The loaded manifest. + + Returns: + Page dependencies, file-backed global inputs, import-time state files + and memo source closures (paths or function slices). + """ + entries: set[str] = set() + for entry in manifest.get("pages", {}).values(): + entries.update(entry.get("deps", ())) + entries.update( + path for label, path in manifest.get("globals", {}).items() if label != "reflex" + ) + entries.update(manifest.get("module_states", {}).get("files", ())) + for memo in manifest.get("memo_files", {}).values(): + entries.update(memo.get("deps", ())) + return entries + + def manifest_dependency_files(manifest: dict[str, Any]) -> set[str]: """Every file the manifest records as a compile input. diff --git a/reflex/utils/compile_daemon.py b/reflex/utils/compile_daemon.py index 51ac02c1193..b6e6f300ef9 100644 --- a/reflex/utils/compile_daemon.py +++ b/reflex/utils/compile_daemon.py @@ -470,7 +470,8 @@ def _recorded_inputs( """Find the manifest entries of input files. A file is recorded whole, as function slices (``<file>::<function>``), - or both. + or both. Only the entries the manifest still references count: the file + table keeps the ones a rebuild dropped. Args: paths: The input files. @@ -486,8 +487,8 @@ def _recorded_inputs( files = manifest["files"] wanted = {str(path): path for path in paths} entries: dict[Path, list[str]] = {} - for key in files: - if (path := wanted.get(page_cache.dep_file(key))) is not None: + for key in disk_cache.manifest_dependency_entries(manifest): + if key in files and (path := wanted.get(page_cache.dep_file(key))) is not None: entries.setdefault(path, []).append(key) return page_cache.FileValidator(files), entries diff --git a/tests/units/compiler/test_compiler.py b/tests/units/compiler/test_compiler.py index 0b72173350a..c1a5783a427 100644 --- a/tests/units/compiler/test_compiler.py +++ b/tests/units/compiler/test_compiler.py @@ -2175,3 +2175,33 @@ def page_two() -> rx.Component: assert ctx.compiled_pages["one"].evaluate_bundle_requests == ["lib-one"] assert ctx.compiled_pages["two"].evaluate_bundle_requests == ["lib-one", "lib-two"] assert ctx.compiled_pages["two"].render_bundle_requests == [] + + +def test_serving_compile_discards_the_manifest(tmp_path, monkeypatch, mocker): + """A compile that bypasses the cache leaves no manifest describing its output. + + The manifest an earlier cached compile wrote would otherwise validate + files this compile overwrote. + """ + from reflex.compiler import disk_cache + + monkeypatch.setenv("REFLEX_COMPILE_CACHE", "1") + monkeypatch.setenv("REFLEX_WEB_WORKDIR", str(tmp_path)) + mocker.patch("reflex.app.reload_config") + # A fresh registry: states other test modules left behind would otherwise + # join the compiled state tree. + with RegistrationContext(): + app = rx.App() + app.add_page(lambda: rx.el.div("page"), route="index") + mocker.patch.object( + compiler, + "get_config", + return_value=rx.Config(app_name="testing", plugins=[]), + ) + mocker.patch.object(app, "_should_compile", return_value=True) + mocker.patch.object(app, "_get_frontend_packages") + mocker.patch.object(app, "_add_optional_endpoints") + manifest = disk_cache._manifest_path() + manifest.write_text("{}") + assert compiler.compile_app(app, use_rich=False, serving=True) + assert not manifest.exists() diff --git a/tests/units/utils/test_compile_daemon.py b/tests/units/utils/test_compile_daemon.py index e2657a5e91b..245dd965a8c 100644 --- a/tests/units/utils/test_compile_daemon.py +++ b/tests/units/utils/test_compile_daemon.py @@ -944,7 +944,11 @@ def test_daemon_compiles_an_edit_made_before_its_first_snapshot(tmp_path, monkey lambda *args: compile_daemon._WatchState([tmp_path], tmp_path, known={page}), ) monkeypatch.setattr(prerequisites, "get_app", lambda **kwargs: None) - monkeypatch.setattr(disk_cache, "load_manifest", lambda: {"files": files}) + monkeypatch.setattr( + disk_cache, + "load_manifest", + lambda: {"files": files, "pages": {"index": {"deps": list(files)}}}, + ) compiled = [] monkeypatch.setattr( compile_daemon, @@ -1141,7 +1145,11 @@ def test_built_inputs_include_dependencies_the_compile_added( monkeypatch.setattr(compile_daemon, "_lock_path", lambda: tmp_path / "lock") monkeypatch.setattr(compile_daemon, "_reload_roots", lambda: [tmp_path]) monkeypatch.setattr(compile_daemon, "_prepare_fork_parent", lambda roots: None) - monkeypatch.setattr(disk_cache, "load_manifest", lambda: {"files": files}) + monkeypatch.setattr( + disk_cache, + "load_manifest", + lambda: {"files": files, "pages": {"index": {"deps": list(files)}}}, + ) monkeypatch.setattr(prerequisites, "get_app", lambda **kwargs: None) monkeypatch.setattr("reflex.utils.prerequisites.get_backend_dir", lambda: tmp_path) monkeypatch.setattr( @@ -1191,10 +1199,33 @@ def test_unbuilt_changes_compare_function_slices(tmp_path, monkeypatch): page = tmp_path / "page.py" page.write_text("def index():\n return 'v1'\n") dep = f"{page}{page_cache.SLICE_SEPARATOR}index" - files = {dep: page_cache.file_entry(dep)} - monkeypatch.setattr(disk_cache, "load_manifest", lambda: {"files": files}) + manifest = { + "files": {dep: page_cache.file_entry(dep)}, + "pages": {"index": {"deps": [dep]}}, + } + monkeypatch.setattr(disk_cache, "load_manifest", lambda: manifest) state = compile_daemon._WatchState([tmp_path], tmp_path, known={page}) assert compile_daemon._unbuilt_changes(state) == set() page.write_text("def index():\n return 'edited'\n") os.utime(page, ns=(1, 1)) assert compile_daemon._unbuilt_changes(state) == {page} + + +def test_unbuilt_changes_ignore_unreferenced_file_table_entries(tmp_path, monkeypatch): + """A slice no page depends on any more does not report its module as edited. + + An incremental rebuild extends the file table without pruning it, so a + renamed page function leaves its old slice behind. + """ + from reflex.compiler import disk_cache, page_cache + + page = tmp_path / "page.py" + page.write_text("def home():\n return 'v1'\n") + stale = f"{page}{page_cache.SLICE_SEPARATOR}index" + current = f"{page}{page_cache.SLICE_SEPARATOR}home" + files = {stale: ["0" * 64, 1, 1], current: page_cache.file_entry(current)} + manifest = {"files": files, "pages": {"home": {"deps": [current]}}} + monkeypatch.setattr(disk_cache, "load_manifest", lambda: manifest) + state = compile_daemon._WatchState([tmp_path], tmp_path, known={page}) + assert compile_daemon._unbuilt_changes(state) == set() + assert compile_daemon._built_inputs({page}) == {page} From 3565bb69db1692204efe8c4fe372bab5011de58b Mon Sep 17 00:00:00 2001 From: Farhan <www.mfarvirus@gmail.com> Date: Wed, 21 Oct 2026 14:54:10 +0000 Subject: [PATCH 48/56] fix(compiler): watch every recorded input; stop aborting a released barrier - The daemon's watch set and its startup check now cover import-time state files and memo source closures, which the manifest records outside any page's dependencies. - The stateful-pages marker test aborted its barrier right after the last round; a reader released from that round but not yet scheduled saw the aborted barrier and failed. The barrier is now aborted only when a writer fails. Claude-Session: https://claude.ai/code/session_01Y27XUSyj8uGXAaLaYyzYsa --- reflex/compiler/disk_cache.py | 10 +++------ tests/units/compiler/test_disk_cache.py | 30 +++++++++++++++++++++++++ tests/units/test_app.py | 8 +++++-- 3 files changed, 39 insertions(+), 9 deletions(-) diff --git a/reflex/compiler/disk_cache.py b/reflex/compiler/disk_cache.py index c5725a63072..7b09c84f3fe 100644 --- a/reflex/compiler/disk_cache.py +++ b/reflex/compiler/disk_cache.py @@ -666,14 +666,10 @@ def manifest_dependency_files(manifest: dict[str, Any]) -> set[str]: manifest: The loaded manifest. Returns: - Page dependency paths plus the file-backed global inputs. + The files of :func:`manifest_dependency_entries` plus the global + inputs recorded as absent. """ - files: set[str] = set() - for entry in manifest.get("pages", {}).values(): - files.update(page_cache.dep_file(dep) for dep in entry.get("deps", ())) - files.update( - path for label, path in manifest.get("globals", {}).items() if label != "reflex" - ) + files = {page_cache.dep_file(dep) for dep in manifest_dependency_entries(manifest)} files.update(manifest.get("globals_absent", ())) return files diff --git a/tests/units/compiler/test_disk_cache.py b/tests/units/compiler/test_disk_cache.py index abaa5afbd0b..0c2f0f375c1 100644 --- a/tests/units/compiler/test_disk_cache.py +++ b/tests/units/compiler/test_disk_cache.py @@ -2729,3 +2729,33 @@ def test_reordering_dynamic_state_pages_reevaluates_them( _drop_dynamic_states("CollideState") assert result is True assert (_COLLIDE["one"], _COLLIDE["two"]) == (1, 1) + + +def test_manifest_dependency_files_cover_every_recorded_input(): + """The daemon watches, and checks at startup, every input the manifest records. + + Import-time state files and memo source closures belong to no page, so + they are listed alongside the page dependencies and global inputs. + """ + manifest = { + "pages": {"index": {"deps": ["/app/pages.py::index", "/app/data.md"]}}, + "globals": {"reflex": "0.0.0", "rxconfig": "/app/rxconfig.py"}, + "globals_absent": ["/app/uv.lock"], + "module_states": {"files": ["/app/state.py"]}, + "memo_files": {"memo.js": {"deps": ["/lib/memo.py"]}}, + } + assert disk_cache.manifest_dependency_entries(manifest) == { + "/app/pages.py::index", + "/app/data.md", + "/app/rxconfig.py", + "/app/state.py", + "/lib/memo.py", + } + assert disk_cache.manifest_dependency_files(manifest) == { + "/app/pages.py", + "/app/data.md", + "/app/rxconfig.py", + "/app/uv.lock", + "/app/state.py", + "/lib/memo.py", + } diff --git a/tests/units/test_app.py b/tests/units/test_app.py index 6211016fada..8e5a28e6760 100644 --- a/tests/units/test_app.py +++ b/tests/units/test_app.py @@ -5790,12 +5790,16 @@ def reader(): with ThreadPoolExecutor(max_workers=8) as pool: readers = [pool.submit(reader) for _ in range(4)] + writers = [pool.submit(writer) for _ in range(4)] try: - writers = [pool.submit(writer) for _ in range(4)] for future in writers: future.result() - finally: + except BaseException: + # Free the readers so the pool can join. Never on success: a reader + # released from the last round but not yet scheduled would see the + # aborted barrier and raise. round_started.abort() + raise for future in readers: future.result() assert json.loads(marker.read_text()) == routes From 1a465cb894c820bc0abca2f809b81c9bb8af2d9a Mon Sep 17 00:00:00 2001 From: Claude <noreply@anthropic.com> Date: Wed, 21 Oct 2026 21:15:32 +0000 Subject: [PATCH 49/56] fix(compiler): attribute reads to modules, mark racy edits, track more read entry points Reads made while a page evaluates are now also attributed to the first-party module whose code performed them, and every page whose import closure holds that module depends on them. A read behind a shared cache (functools.cache, a module-level loader) happens once per process, for the first page only; before this the other pages never saw the file change. A dependency modified at or after the compile started imported the app is recorded as racy and always misses the next compile: the compile may have read earlier content than the manifest hashed. Also: exclusions apply to the parts below the project root, so a project in a directory named venv and reads under assets/ are tracked; io.open, tokenize.open (linecache, inspect.getsource) and io.open_code (pkgutil) are recorded; every import counts in a slice, and a slice reaching its module by a computed name (globals, sys, inspect) depends on the whole file; a dependency that was absent is recorded as such instead of missing on every compile; mark_volatile() lets a page opt out of reuse; dead code and the per-read root resolution are gone. Claude-Session: https://claude.ai/code/session_01RgKnnN8CCBqaR58qm23zQp --- reflex/compiler/disk_cache.py | 6 +- reflex/compiler/page_cache.py | 487 +++++++++++++----------- tests/units/compiler/test_page_cache.py | 356 +++++++++++++---- tests/units/conftest.py | 16 + 4 files changed, 567 insertions(+), 298 deletions(-) diff --git a/reflex/compiler/disk_cache.py b/reflex/compiler/disk_cache.py index 7b09c84f3fe..cd563dcc2b6 100644 --- a/reflex/compiler/disk_cache.py +++ b/reflex/compiler/disk_cache.py @@ -669,7 +669,11 @@ def manifest_dependency_files(manifest: dict[str, Any]) -> set[str]: The files of :func:`manifest_dependency_entries` plus the global inputs recorded as absent. """ - files = {page_cache.dep_file(dep) for dep in manifest_dependency_entries(manifest)} + files = { + page_cache.dep_file(dep) + for dep in manifest_dependency_entries(manifest) + if dep != page_cache.VOLATILE_DEP + } files.update(manifest.get("globals_absent", ())) return files diff --git a/reflex/compiler/page_cache.py b/reflex/compiler/page_cache.py index 642c4af5ad0..cb0632c904f 100644 --- a/reflex/compiler/page_cache.py +++ b/reflex/compiler/page_cache.py @@ -4,6 +4,14 @@ files read during page evaluation, component modules in its rendered tree, and fine-grained state files it references. ``disk_cache`` persists hashes of that set so only pages whose dependencies changed need to be recompiled. + +Reads are tracked through ``open``, ``io.open``, ``Path.open``, +``Path.read_text``/``read_bytes``, ``tokenize.open`` (``linecache`` and +``inspect.getsource``) and ``io.open_code`` (``pkgutil.get_data``), on the +thread that evaluates the page. A page whose output depends on anything else +(a read from a C extension or a worker thread, the environment, the clock, a +directory listing, a subprocess) calls :func:`mark_volatile` while it +evaluates, and is then recompiled by every compile. """ from __future__ import annotations @@ -16,14 +24,17 @@ import hashlib import importlib import importlib.util +import io import os import re import sys +import time +import tokenize from collections.abc import Callable, Iterable, Mapping, Sequence from contextvars import ContextVar from importlib import metadata from pathlib import Path -from typing import TYPE_CHECKING, TypeVar +from typing import TYPE_CHECKING from reflex_base import constants from reflex_base.environment import environment @@ -32,22 +43,26 @@ from reflex_base.components.component import BaseComponent from reflex_base.plugins import PageContext -#: A manifest file-table entry: ``[content sha256, st_mtime_ns, st_size]``. +#: A manifest file-table entry: ``[content sha256, st_mtime_ns, st_size]``, or +#: :data:`ABSENT_ENTRY` for a dependency that did not exist, or +#: :data:`RACY_ENTRY` for one that changed while the compile ran. FileEntry = list -#: Directories never worth hashing (build artifacts, deps, caches). -_SKIP_DIRS = { - constants.Dirs.WEB, - ".venv", - "venv", - "node_modules", - "__pycache__", - ".git", - constants.Dirs.APP_ASSETS, -} +#: The file-table entry of a dependency that was absent when recorded. It is +#: unchanged while the file stays absent. +ABSENT_ENTRY: FileEntry = ["<absent>"] + +#: The file-table entry of a dependency modified at or after the compile +#: started (the racy-git rule): the content the compile read is unknown, so +#: the next compile always counts the file as changed and records it afresh. +RACY_ENTRY: FileEntry = ["<racy>", -1, -1] + +#: The dependency a volatile page records (see :func:`mark_volatile`). It has +#: no file-table entry, so the page misses on every compile. +VOLATILE_DEP = "<volatile>" #: Genuinely-global files: a change here can affect every page's output, so it -#: bumps ``global_epoch`` rather than any single page's dependency set. +#: is a global input rather than part of any single page's dependency set. #: ``reflex.lock/`` is deliberately absent: its lockfiles are derived from #: inputs tracked elsewhere (``rxconfig.py`` frontend packages, page imports, #: the Reflex version) and are rewritten by every install, so treating them as @@ -64,22 +79,6 @@ STATE_CONTEXT_RE = re.compile(r"reflex___state[A-Za-z0-9_]+") -def _sha(*parts: bytes | str) -> str: - """Digest parts with a separator, so their boundaries count. - - Args: - parts: The text or bytes to digest. - - Returns: - The hex digest. - """ - h = hashlib.sha256() - for p in parts: - h.update(p.encode() if isinstance(p, str) else p) - h.update(b"\x1f") - return h.hexdigest() - - def _reflex_version() -> str: """Return the installed Reflex version. @@ -113,6 +112,29 @@ def first_party_roots(root: Path | None = None) -> tuple[Path, ...]: return (root, *(p for p in include if p != root)) +def _containing_root(path: Path, roots: Iterable[Path]) -> Path | None: + """Find the root that contains (or is) ``path``. + + Compares path strings: this runs for every import and read under a + recorder, and ``root in path.parents`` builds a Path per ancestor. + + Args: + path: Absolute path to test. + roots: Absolute root paths. + + Returns: + The first root containing ``path``, or None. + """ + path_str = str(path) + for root in roots: + root_str = str(root) + if path_str == root_str or path_str.startswith( + root_str if root_str.endswith(os.sep) else root_str + os.sep + ): + return root + return None + + def _under_any(path: Path, roots: Iterable[Path]) -> bool: """Whether ``path`` is nested below (or is) one of ``roots``. @@ -123,7 +145,7 @@ def _under_any(path: Path, roots: Iterable[Path]) -> bool: Returns: True when some root contains ``path``. """ - return any(root == path or root in path.parents for root in roots) + return _containing_root(path, roots) is not None # A page's markdown/data dependencies are read lazily while the page is @@ -133,10 +155,45 @@ def _under_any(path: Path, roots: Iterable[Path]) -> bool: _active_reads: ContextVar[set[str] | None] = ContextVar("_active_reads", default=None) _patched = False _recorder_root: Path | None = None +#: The first-party roots of ``_recorder_root``, resolved once per root. +_recorder_roots: tuple[Path, ...] = () +#: ``time.time_ns()`` when the current compile started importing the app. +#: A dependency modified at or after it is recorded as :data:`RACY_ENTRY`. +_compile_start_ns: int | None = None + +#: Path parts that mark a dependency/build location whose files are never a +#: page's own source dependency (a change there flows through the version or +#: the global inputs). Tested against the parts below the first-party root a +#: file is under, so a project living in a directory of such a name is not +#: excluded wholesale; a file outside every root is tested by its full path. +_EXCLUDE_PARTS = frozenset({ + constants.Dirs.WEB, + ".venv", + "venv", + "node_modules", + "__pycache__", + ".git", + "site-packages", + "dist-packages", +}) + + +def _excluded(path: Path, roots: tuple[Path, ...]) -> bool: + """Whether ``path`` lies in a dependency/build location. + + Args: + path: Absolute path to test. + roots: The first-party roots. + + Returns: + True when a part of the path (below its root, if any) names an + excluded directory. + """ + parts = path.parts + if (root := _containing_root(path, roots)) is not None: + parts = parts[len(root.parts) :] + return not _EXCLUDE_PARTS.isdisjoint(parts) -#: Path parts that mark a dependency/build location whose reads are never a -#: page's own source dependency (a change there flows through the version/epoch). -_EXCLUDE_PARTS = _SKIP_DIRS | {"site-packages", "dist-packages"} #: Suffixes of content files a page may read from *outside* the project root #: (e.g. the docs app reads its markdown from a sibling directory). Reads under @@ -165,11 +222,20 @@ def _under_any(path: Path, roots: Iterable[Path]) -> bool: #: None). Cleared by ``enable_read_tracking`` when the recorder root changes. _module_file_cache: dict[object, str | None] = {} _app_import_reads: dict[Path, set[str]] = {} -T = TypeVar("T") -#: Maps an absolute import name to the files read while importing it under a -#: recorder, so every later importer of the cached module depends on them too. -_module_import_reads: dict[str, set[str]] = {} +#: Maps a first-party module file to the files code in it read under a +#: recorder, whichever page (or the app import) was running. A read behind a +#: shared cache (``functools.cache``, a module-level loader) happens once per +#: process, for the first caller only, so a page depends on the reads of every +#: module in its import closure, not only on the reads its own evaluation +#: performed. The over-approximation is safe. +_module_reads: dict[str, set[str]] = {} +_THIS_FILE = __file__ + + +def forget_module_reads() -> None: + """Drop the reads attributed to modules (before first-party code reloads).""" + _module_reads.clear() @contextlib.contextmanager @@ -186,9 +252,32 @@ def _suspend_tracking(): _active_reads.reset(token) +def _reading_module_file() -> str | None: + """Find the first-party module whose code performed the current read. + + Walks the stack from the read hook's caller up to the first frame whose + code lives in a first-party module file, so a read made by a library on + a page's behalf is attributed to the page's code. + + Returns: + The resolved module file, or None when no first-party code is on the + stack. + """ + frame = sys._getframe(2) + while frame is not None: + file = frame.f_code.co_filename + if file != _THIS_FILE and (module_file := _recordable_module_file(file)): + return module_file + frame = frame.f_back + return None + + def _record_read(path: object) -> None: """Add a first-party file read to the active read set, if any. + The read is also attributed to the first-party module that performed it + (see :data:`_module_reads`). + Args: path: The path the code read. """ @@ -200,26 +289,29 @@ def _record_read(path: object) -> None: resolved = Path(path).resolve() # type: ignore[arg-type] except (OSError, TypeError, ValueError): return - if any(part in _EXCLUDE_PARTS for part in resolved.parts): + if _excluded(resolved, _recorder_roots): return - root = _recorder_root - under_root = root is not None and _under_any(resolved, first_party_roots(root)) - if not under_root and resolved.suffix.lower() not in _CONTENT_SUFFIXES: + if ( + not _under_any(resolved, _recorder_roots) + and resolved.suffix.lower() not in _CONTENT_SUFFIXES + ): return - target.add(str(resolved)) + read = str(resolved) + target.add(read) + if (module_file := _reading_module_file()) is not None: + _module_reads.setdefault(module_file, set()).add(read) -def _is_inside(path: Path, root: Path) -> bool: - """Return whether ``path`` is nested below ``root``. +def mark_volatile() -> None: + """Make the page being evaluated recompile on every compile. - Args: - path: Absolute path to test. - root: Absolute root path. - - Returns: - True when ``path`` is nested below ``root``. + For a page whose output depends on an input the cache cannot see: a read + made by a C extension or on another thread, an environment variable, the + clock, a directory listing, a subprocess. A no-op outside a compile that + records dependencies. """ - return root in path.parents + if (target := _active_reads.get()) is not None: + target.add(VOLATILE_DEP) def _is_python_install_file(path: Path) -> bool: @@ -264,9 +356,9 @@ def _recordable_module_file(file: object) -> str | None: except (OSError, TypeError, ValueError): return None - roots = first_party_roots(root) + roots = _recorder_roots resolved_str = None - if not any(part in _EXCLUDE_PARTS for part in path.parts) and ( + if not _excluded(path, roots) and ( _under_any(path, roots) or not _is_python_install_file(path) ): try: @@ -274,10 +366,8 @@ def _recordable_module_file(file: object) -> str | None: except (OSError, TypeError, ValueError): pass else: - if not any(part in _EXCLUDE_PARTS for part in resolved.parts): - resolved_str = ( - str(resolved) if _under_any(resolved, roots) else None - ) + if _under_any(resolved, roots) and not _excluded(resolved, roots): + resolved_str = str(resolved) _module_file_cache[file] = resolved_str return resolved_str @@ -347,72 +437,40 @@ def _record_imported_modules( result_id = id(result) if (module := sys.modules.get(name)) and id(module) != result_id: _record_module_file(module, target) - target.update(_module_import_reads.get(name, ())) parent = name while "." in parent: parent = parent.rsplit(".", 1)[0] if (module := sys.modules.get(parent)) is not None: _record_module_file(module, target) - target.update(_module_import_reads.get(parent, ())) if not fromlist: return for item in fromlist: if ( item != "*" - and (module := sys.modules.get(submodule := f"{name}.{item}")) + and (module := sys.modules.get(f"{name}.{item}")) and id(module) != result_id ): _record_module_file(module, target) - target.update(_module_import_reads.get(submodule, ())) - - -def _import_recording_reads( - name: str, do_import: Callable[[], T], target: set[str] -) -> T: - """Run an import under a fresh read set kept as the module's import reads. - - A module executing afresh replaces its recorded reads; importing a loaded - module only adds the reads of submodules it loads for the first time. The - reads also go to ``target``, even when the import fails. - - Args: - name: The absolute requested module name. - do_import: Performs the import. - target: The active read set. - - Returns: - The result of ``do_import``. - """ - executes = name not in sys.modules - reads: set[str] = set() - token = _active_reads.set(reads) - try: - return do_import() - finally: - _active_reads.reset(token) - if executes: - _module_import_reads.pop(name, None) - if reads: - _module_import_reads.setdefault(name, set()).update(reads) - target.update(reads) def enable_read_tracking(root: Path | None = None) -> None: """Install per-page source-read tracking and register the recorder hook. - Idempotent. Patches ``Path.read_text``/``read_bytes`` and ``open`` to record - reads (only while a recording set is active) and points the reflex-base - compile loop at :func:`record_reads`. Called when an incremental cache flag - is on; a no-op to the compile otherwise. + Idempotent. Patches the read entry points named in the module docstring + and the import functions to record reads and imported module files (only + while a recording set is active) and points the reflex-base compile loop + at :func:`record_reads`. Called when an incremental cache flag is on; a + no-op to the compile otherwise. Args: root: Project root; only reads under it are recorded. Defaults to cwd. """ - global _patched, _recorder_root + global _patched, _recorder_root, _recorder_roots resolved_root = (root or Path.cwd()).resolve() if _recorder_root != resolved_root: _module_file_cache.clear() - _module_import_reads.clear() + _module_reads.clear() + _recorder_roots = first_party_roots(resolved_root) _recorder_root = resolved_root from reflex_base.plugins import compiler as _bc @@ -427,6 +485,8 @@ def enable_read_tracking(root: Path | None = None) -> None: orig_read_bytes = Path.read_bytes orig_path_open = Path.open orig_open = builtins.open + orig_open_code = io.open_code + orig_tokenize_open = tokenize.open orig_import = builtins.__import__ orig_import_module = importlib.import_module @@ -453,6 +513,17 @@ def open_(file: object, mode: str = "r", *args: object, **kwargs: object): _record_read(file) return orig_open(file, mode, *args, **kwargs) # type: ignore[arg-type] + # ``importlib`` loaders (``pkgutil.get_data``) read through the C module. + def open_code(path: str): + _record_read(path) + return orig_open_code(path) + + # ``linecache`` (``inspect.getsource``) opens source through ``tokenize``, + # which bound ``builtins.open`` when it was imported. + def tokenize_open(filename: object): + _record_read(filename) + return orig_tokenize_open(filename) # type: ignore[arg-type] + def import_( name: str, globals_: dict[str, object] | None = None, @@ -460,31 +531,27 @@ def import_( fromlist: Sequence[str] | None = (), level: int = 0, ): - if (target := _active_reads.get()) is None: - return orig_import(name, globals_, locals_, fromlist, level) - absolute = _absolute_import_name(name, globals_, level) - result = _import_recording_reads( - absolute, - lambda: orig_import(name, globals_, locals_, fromlist, level), - target, - ) - _record_imported_modules(absolute, result, target, fromlist) + result = orig_import(name, globals_, locals_, fromlist, level) + if (target := _active_reads.get()) is not None: + absolute = _absolute_import_name(name, globals_, level) + _record_imported_modules(absolute, result, target, fromlist) return result def import_module(name: str, package: str | None = None): - if (target := _active_reads.get()) is None: - return orig_import_module(name, package) - absolute = importlib.util.resolve_name(name, package) if package else name - result = _import_recording_reads( - absolute, lambda: orig_import_module(name, package), target - ) - _record_imported_modules(result.__name__, result, target) + result = orig_import_module(name, package) + if (target := _active_reads.get()) is not None: + _record_imported_modules(result.__name__, result, target) return result Path.read_text = read_text # type: ignore[method-assign,assignment] Path.read_bytes = read_bytes # type: ignore[method-assign,assignment] Path.open = path_open # type: ignore[method-assign,assignment] builtins.open = open_ # type: ignore[assignment] + io.open = open_ # type: ignore[assignment] + io.open_code = open_code + # ``importlib`` loaders call ``open_code`` on the C module object. + setattr(sys.modules["_io"], "open_code", open_code) # noqa: B010 + tokenize.open = tokenize_open # type: ignore[assignment] builtins.__import__ = import_ # type: ignore[assignment] importlib.import_module = import_module @@ -508,13 +575,18 @@ def record_reads(): def record_app_import(root: Path | None = None): """Record files read/imported while importing the app module. + Also marks the start of the compile, for the racy-edit rule of + :func:`file_entry`: everything the compile reads is read after this. + Args: root: Project root. Defaults to cwd. Yields: None. """ + global _compile_start_ns root = (root or Path.cwd()).resolve() + _compile_start_ns = time.time_ns() recorded: set[str] = set() success = False try: @@ -527,6 +599,27 @@ def record_app_import(root: Path | None = None): _app_import_reads[root] = recorded +def _is_racy(mtime_ns: int) -> bool: + """Whether a file modified at ``mtime_ns`` may have changed during the compile. + + A file modified at or after the compile started may hold other content + than the compile read. A whole-second timestamp comes from a file system + with coarse timestamps, where a change in the compile's first second is + indistinguishable from one just before it. + + Args: + mtime_ns: The file's modification time. + + Returns: + True when the content the compile read cannot be vouched for. + """ + if (start := _compile_start_ns) is None: + return False + if mtime_ns % 1_000_000_000 == 0: + start -= start % 1_000_000_000 + return mtime_ns >= start + + def _is_page_registration(statement: ast.stmt) -> bool: """Whether a top-level statement is an ``<app>.add_page(...)`` call. @@ -566,7 +659,9 @@ def _loaded_names(nodes: Iterable[ast.AST]) -> set[str]: #: :func:`function_slice_digest`). SLICE_SEPARATOR = "::" -#: Builtins that reach module globals no syntax tree names. +#: Names through which code reaches module globals, or the module's own +#: source text or line numbers, that no syntax tree names: a slice reading +#: any of them depends on the whole module file. _DYNAMIC_GLOBAL_NAMES = frozenset({ "globals", "vars", @@ -574,6 +669,13 @@ def _loaded_names(nodes: Iterable[ast.AST]) -> set[str]: "eval", "exec", "__import__", + "sys", + "inspect", + "linecache", + "tokenize", + "getsource", + "getsourcelines", + "getsourcefile", }) #: Decorators (``<module>.<name>``) that keep the function they wrap to @@ -743,44 +845,41 @@ def _definition_closure( return closure -def _sliced_texts(part: _ModulePart, used: set[str], closure: set[str]) -> list[str]: +def _sliced_texts(part: _ModulePart, closure: set[str]) -> list[str]: """Return what one module part adds to a slice. Args: part: The part. - used: The names the slice's code reads. closure: The definitions in the slice. Returns: The texts to digest. """ if part.aliases: - return [ - dump - for bound, dump in part.aliases - if bound == "*" or bound in used or bound in closure - ] + # Every import counts: importing a module runs it, and that can + # change what any code in this module sees. + return [dump for _, dump in part.aliases] if part.name is None or part.name in closure: return [part.dump] return [part.header] if part.header else [] -def _slice_digest(parts: Sequence[_ModulePart], seeds: Iterable[str]) -> str: +def _slice_digest(parts: Sequence[_ModulePart], seeds: Iterable[str]) -> str | None: """Digest the parts of a module that code starting at ``seeds`` can read. - That is every statement that runs at import, the import-time parts of - every definition, and the whole definitions named (transitively) by those - or by ``seeds``. Imports count only for the names that code reads. Code - reaching module globals by computed name makes every definition count. - Like a dependency across modules, a call-time mutation of a shared object - is not tracked. + That is every statement that runs at import (imports included), the + import-time parts of every definition, and the whole definitions named + (transitively) by those or by ``seeds``. Like a dependency across modules, + a call-time mutation of a shared object is not tracked. Args: parts: The module's parts (see :func:`_module_parts`). seeds: The definition names to start from. Returns: - The hex digest. + The hex digest, or None when the code reaches module globals or the + module's own source by a computed name, so only the whole file is a + safe dependency. """ definitions: dict[str, list[_ModulePart]] = {} always: set[str] = set() @@ -793,15 +892,10 @@ def _slice_digest(parts: Sequence[_ModulePart], seeds: Iterable[str]) -> str: closure = _definition_closure(definitions, (*seeds, *always)) used = always.union(*(part.loads for name in closure for part in definitions[name])) if not used.isdisjoint(_DYNAMIC_GLOBAL_NAMES): - # A computed name can reach every module global, imports included. - closure = set(definitions) - used = always.union( - *(part.loads for group in definitions.values() for part in group), - (bound for part in parts for bound, _ in part.aliases), - ) + return None digest = hashlib.sha256() for part in parts: - for text in _sliced_texts(part, used, closure): + for text in _sliced_texts(part, closure): digest.update(text.encode()) digest.update(b"\n") return digest.hexdigest() @@ -830,8 +924,9 @@ def function_slice_digest(path: str, name: str) -> str | None: name: The function's name. Returns: - The digest, or None when the file cannot be parsed or defines no - such module-level function or class. + The digest, or None when the file cannot be parsed, defines no such + module-level function or class, or reaches its globals or source by + a computed name (see :func:`_slice_digest`). """ parts = _module_parts(path) if parts is None or not any(part.name == name for part in parts): @@ -842,19 +937,21 @@ def function_slice_digest(path: str, name: str) -> str | None: def entrypoint_digest(path: str) -> str | None: """Digest the app entry module apart from its pages. - Left out are top-level ``<app>.add_page(...)`` statements and the code only - they read: page functions defined in the module (each page tracks its own - slice, see :func:`function_slice_digest`) and imported pages. Class bodies - run at import, so every class counts. Also left out is what - the syntax tree never holds (comments, formatting). Two versions with the - same digest differ only in their pages, which the compile cache compares - page by page; any other edit can change every page. + Left out are top-level ``<app>.add_page(...)`` statements and the page + functions defined in the module (each page tracks its own slice, see + :func:`function_slice_digest`). Every import counts, a page's included: + importing a module runs it. Class bodies run at import, so every class + counts. Also left out is what the syntax tree never holds (comments, + formatting). Two versions with the same digest differ only in their + pages, which the compile cache compares page by page; any other edit can + change every page. Args: path: The entry module's file. Returns: - The digest, or None when the file cannot be read or parsed. + The digest, or None when the file cannot be read or parsed, or + reaches its globals or source by a computed name. """ parts = _module_parts(path) if parts is None: @@ -892,58 +989,24 @@ def _app_entrypoint_file(root: Path | None = None) -> Path | None: return rf if root in rf.parents else None -def global_epoch_inputs( +def global_input_paths( root: Path | None = None, *, pages: Sequence[object] | None = None ) -> dict[str, str]: - """Fingerprint each genuinely-global input individually. + """Map each file-backed global input label to its path. These can affect every page's output but belong to no single page, so they - gate the whole cache rather than any one page's dependency set: the Reflex - version, the config/lockfiles, and the app-level config files: the app - entrypoint module plus the config-only modules it imports (theme, app-wraps, + gate the whole cache rather than any one page's dependency set: the + config/lockfiles and the app-level config files, i.e. the app entrypoint + module plus the config-only modules it imports (theme, app-wraps, stylesheets, head components; see :func:`app_dependency_files`). Kept small on purpose; per-file edits flow through per-page dependency sets instead. - - Keeping each input's digest separate (rather than one combined sha) lets a - mismatch report *which* global input changed. - - Args: - root: Project root. Defaults to cwd. - pages: The current page definitions, used as barriers so page modules - (tracked per page) are excluded from the app-level config files. - - Returns: - A mapping of input label (``reflex``, a global filename, or - ``app:<path>``) to its content digest (``<absent>`` if unreadable). - """ - root = (root or Path.cwd()).resolve() - inputs: dict[str, str] = {"reflex": _reflex_version()} - for name in _GLOBAL_FILES: - path = root / name - try: - inputs[name] = hashlib.sha256(path.read_bytes()).hexdigest() - except OSError: - inputs[name] = "<absent>" - for path_str in app_dependency_files(pages, root): - try: - digest = hashlib.sha256(Path(path_str).read_bytes()).hexdigest() - except OSError: - digest = "<absent>" - inputs[f"app:{path_str}"] = digest - return inputs - - -def global_input_paths( - root: Path | None = None, *, pages: Sequence[object] | None = None -) -> dict[str, str]: - """Map each file-backed global input label to its path. - The manifest stores these labels with their paths and keeps the content entries in its file table, so validation is stat-first like page deps. Args: root: Project root. Defaults to cwd. - pages: The current page definitions (see :func:`global_epoch_inputs`). + pages: The current page definitions, used as barriers so page modules + (tracked per page) are excluded from the app-level config files. Returns: ``{label: path}`` for every global input except the Reflex version. @@ -954,24 +1017,6 @@ def global_input_paths( return paths -def global_epoch( - root: Path | None = None, *, pages: Sequence[object] | None = None -) -> str: - """Fingerprint the genuinely-global inputs as one digest. - - Args: - root: Project root. Defaults to cwd. - pages: The current page definitions (see :func:`global_epoch_inputs`). - - Returns: - A hex digest of the global inputs. - """ - # Sorted for a deterministic digest regardless of dict insertion order. - return _sha( - *(f"{k}={v}" for k, v in sorted(global_epoch_inputs(root, pages=pages).items())) - ) - - def _module_file(component: object) -> Path | None: """Return the file of the module an object belongs to. @@ -1140,8 +1185,6 @@ def _import_from_targets( Returns: Candidate dotted module names to resolve. """ - import ast - if not isinstance(node, ast.ImportFrom): return [] if node.level: # relative import: walk up from the importing package @@ -1225,8 +1268,6 @@ def _module_import_names(file: str, modname: str) -> list[str]: Returns: The imported dotted-name candidates (see :func:`_import_from_targets`). """ - import ast - key = _stat_key(file) cached = _import_names_cache.get(file) if cached is not None and key is not None and cached[0] == key: @@ -1412,7 +1453,7 @@ def app_dependency_files( These configure the app-wide files an incremental rebuild reuses on disk (app root, contexts, theme, stylesheet), so they are folded into - :func:`global_epoch`: editing one forces a full recompile instead of leaving + :func:`global_input_paths`: editing one forces a full recompile instead of leaving those files stale. Args: @@ -1452,7 +1493,8 @@ def file_entry(path: str) -> FileEntry | None: Returns: ``[sha256, st_mtime_ns, st_size]``, or None if unreadable. For a function slice (see :data:`SLICE_SEPARATOR`), the slice digest and - the stat of its file. + the stat of its file. :data:`RACY_ENTRY` for a file modified since + the compile started (see :func:`_is_racy`). """ file, separator, name = path.rpartition(SLICE_SEPARATOR) if separator: @@ -1460,13 +1502,15 @@ def file_entry(path: str) -> FileEntry | None: digest = function_slice_digest(file, name) if key is None or digest is None: return None - return [digest, *key] + return RACY_ENTRY if _is_racy(key[0]) else [digest, *key] try: with Path(path).open("rb") as fh: st = os.fstat(fh.fileno()) digest = hashlib.sha256(fh.read()).hexdigest() except OSError: return None + if _is_racy(st.st_mtime_ns): + return RACY_ENTRY return [digest, st.st_mtime_ns, st.st_size] @@ -1518,13 +1562,17 @@ def changed(self, path: str) -> bool: Returns: True if the file changed, is unreadable, or is not in the table. + A recorded-absent file is unchanged while it is still absent; a + racy entry (see :data:`RACY_ENTRY`) always counts as changed. """ cached = self._changed.get(path) if cached is not None: return cached entry = self._files.get(path) result = True - if entry is not None and len(entry) == 3: + if entry == ABSENT_ENTRY: + result = SLICE_SEPARATOR in path or Path(path).exists() + elif entry is not None and entry != RACY_ENTRY and len(entry) == 3: key = _stat_key(dep_file(path)) if key is not None and (key[0], key[1]) == (entry[1], entry[2]): result = False @@ -1656,7 +1704,7 @@ def page_dependency_files( evaluated (markdown/data, via the read recorder), the component modules in its rendered tree, and the fine-grained state files it references. A change to any file *outside* this set cannot change the page's output, except the - genuinely-global inputs tracked by :func:`global_epoch` and route additions + genuinely-global inputs tracked by :func:`global_input_paths` and route additions (shared nav). This is the dependency set that makes invalidation precise. Args: @@ -1688,6 +1736,9 @@ def page_dependency_files( dynamic_py = {f for f in page_ctx.source_files if f.endswith(".py")} if dynamic_py: deps |= _walk_import_closure(build_import_graph(root), dynamic_py) + # What the modules in the closure read on any page's behalf (a read + # behind a shared cache happens for the first caller only). + deps.update(*(_module_reads.get(dep, ()) for dep in list(deps))) if (own := _own_function_slice(component, page_ctx.source_files, root)) is not None: deps.discard(dep_file(own)) deps.add(own) @@ -1754,8 +1805,8 @@ def page_dependency_entries( """ deps: list[str] = [] for path in sorted(page_dependency_files(page_ctx, component, state_index, root)): - entry = hasher(path) - if entry is not None: - files[path] = entry + if path != VOLATILE_DEP: + entry = hasher(path) + files[path] = ABSENT_ENTRY if entry is None else entry deps.append(path) return deps diff --git a/tests/units/compiler/test_page_cache.py b/tests/units/compiler/test_page_cache.py index 1da463b825d..a153ba65e9b 100644 --- a/tests/units/compiler/test_page_cache.py +++ b/tests/units/compiler/test_page_cache.py @@ -80,19 +80,39 @@ def _forget_modules(*module_names): sys.modules.pop(module_name, None) -def test_global_epoch_tracks_global_files(tmp_path): +def _global_inputs(root: Path, pages: Any = None) -> dict[str, str | None]: + """Digest every global input as the manifest records it. + + Args: + root: The project root. + pages: The page definitions, as barriers. + + Returns: + Each global input label mapped to its content digest (None if absent). + """ + import hashlib + + return { + label: hashlib.sha256(Path(path).read_bytes()).hexdigest() + if Path(path).is_file() + else None + for label, path in page_cache.global_input_paths(root, pages=pages).items() + } + + +def test_global_inputs_tracks_global_files(tmp_path): (tmp_path / "rxconfig.py").write_text("config = 1\n") (tmp_path / "other.py").write_text("x = 1\n") - epoch = page_cache.global_epoch(root=tmp_path) + epoch = _global_inputs(tmp_path) # editing a non-global file does NOT change the epoch (tmp_path / "other.py").write_text("x = 2\n") - assert page_cache.global_epoch(root=tmp_path) == epoch + assert _global_inputs(tmp_path) == epoch # editing a genuinely-global file (rxconfig.py) DOES (tmp_path / "rxconfig.py").write_text("config = 2\n") - assert page_cache.global_epoch(root=tmp_path) != epoch + assert _global_inputs(tmp_path) != epoch -def test_global_epoch_tracks_app_entrypoint(tmp_path, monkeypatch): +def test_global_inputs_tracks_app_entrypoint(tmp_path, monkeypatch): """Editing the app entrypoint (theme/app_wraps live there) bumps the epoch. App-wide config is configured where ``rx.App`` is built, not in any page's @@ -105,10 +125,10 @@ def test_global_epoch_tracks_app_entrypoint(tmp_path, monkeypatch): monkeypatch.setattr( page_cache, "_app_entrypoint_file", lambda root=None: entrypoint ) - epoch = page_cache.global_epoch(root=tmp_path) + epoch = _global_inputs(tmp_path) # editing the app entrypoint DOES change the epoch entrypoint.write_text("app = rx.App(theme=dark)\n") - assert page_cache.global_epoch(root=tmp_path) != epoch + assert _global_inputs(tmp_path) != epoch def test_app_entrypoint_file_resolution(tmp_path, monkeypatch): @@ -183,7 +203,7 @@ def test_app_dependency_files_keeps_config_excludes_pages(tmp_path, monkeypatch) assert result == {str(entry), str(theme)} -def test_global_epoch_excludes_page_modules(tmp_path, monkeypatch): +def test_global_inputs_excludes_page_modules(tmp_path, monkeypatch): """A page-module edit keeps the epoch (incremental); an app-config edit bumps it.""" from types import SimpleNamespace @@ -215,16 +235,16 @@ def test_global_epoch_excludes_page_modules(tmp_path, monkeypatch): ) pages = [SimpleNamespace(component=object())] - epoch = page_cache.global_epoch(root=tmp_path, pages=pages) + epoch = _global_inputs(tmp_path, pages) # editing a page module does NOT change the epoch (tracked per page instead) page.write_text("p = 2\n") - assert page_cache.global_epoch(root=tmp_path, pages=pages) == epoch + assert _global_inputs(tmp_path, pages) == epoch # editing app-level config the entrypoint imports (theme) DOES theme.write_text("t = 2\n") - assert page_cache.global_epoch(root=tmp_path, pages=pages) != epoch + assert _global_inputs(tmp_path, pages) != epoch -def test_global_epoch_hashes_only_app_dependency_closure(tmp_path, monkeypatch): +def test_global_inputs_hashes_only_app_dependency_closure(tmp_path, monkeypatch): """Unrelated first-party modules do not make the app epoch coarse.""" from types import SimpleNamespace @@ -254,23 +274,11 @@ def test_global_epoch_hashes_only_app_dependency_closure(tmp_path, monkeypatch): lambda file, modname, file_to_mod: graph[file], ) - epoch = page_cache.global_epoch( - root=tmp_path, pages=[SimpleNamespace(component=object())] - ) + epoch = _global_inputs(tmp_path, [SimpleNamespace(component=object())]) unrelated.write_text("x = 2\n") - assert ( - page_cache.global_epoch( - root=tmp_path, pages=[SimpleNamespace(component=object())] - ) - == epoch - ) + assert _global_inputs(tmp_path, [SimpleNamespace(component=object())]) == epoch theme.write_text("t = 2\n") - assert ( - page_cache.global_epoch( - root=tmp_path, pages=[SimpleNamespace(component=object())] - ) - != epoch - ) + assert _global_inputs(tmp_path, [SimpleNamespace(component=object())]) != epoch def test_app_dependency_files_skips_graph_without_entrypoint(tmp_path, monkeypatch): @@ -317,7 +325,7 @@ def test_app_dependency_files_tracks_dynamic_import_config_read(tmp_path, monkey try: prerequisites.get_app() deps = page_cache.app_dependency_files(root=tmp_path) - epoch = page_cache.global_epoch(root=tmp_path) + epoch = _global_inputs(tmp_path) config_file.write_text('"dark"\n') finally: @@ -325,7 +333,7 @@ def test_app_dependency_files_tracks_dynamic_import_config_read(tmp_path, monkey assert str(dynamic.resolve()) in deps assert str(config_file.resolve()) in deps - assert page_cache.global_epoch(root=tmp_path) != epoch + assert _global_inputs(tmp_path) != epoch def test_app_dependency_files_subtracts_pages_from_app_import_reads( @@ -751,11 +759,13 @@ def test_import_time_reads_are_recorded_for_every_importer( """A module's import-time reads are deps of each page that imports it. Only the first page's import executes the module; a later page gets the - cached module and must still depend on what its execution read. + cached module and must still depend on what its execution read, which + the read's attribution to the module provides. """ data = tmp_path / "shared_data.json" data.write_text('{"title": "v1"}') - _prepare_runtime_module(tmp_path, monkeypatch, "review_shared_helper").write_text( + helper = _prepare_runtime_module(tmp_path, monkeypatch, "review_shared_helper") + helper.write_text( f"from pathlib import Path\nTITLE = Path({str(data)!r}).read_text()\n" ) page_cache.enable_read_tracking(tmp_path) @@ -770,7 +780,185 @@ def test_import_time_reads_are_recorded_for_every_importer( page_reads.append(reads) finally: _forget_modules("review_shared_helper") - assert [str(data) in reads for reads in page_reads] == [True, True] + helper_file = str(helper.resolve()) + assert [str(data) in reads for reads in page_reads] == [True, False] + assert [helper_file in reads for reads in page_reads] == [True, True] + assert page_cache._module_reads[helper_file] == {str(data)} + + +def test_reads_behind_a_shared_cache_reach_every_dependent_page(tmp_path, monkeypatch): + """A read a cached helper makes for the first page is a dep of every page. + + ``functools.cache`` (or a module-level loader) reads the file once per + process, while evaluating the first page that calls it. The read is + attributed to the helper's module, so a page whose import closure holds + that module depends on the file even when its own evaluation read nothing. + """ + from types import SimpleNamespace + + posts = tmp_path / "posts.md" + posts.write_text("v1") + helper = _prepare_runtime_module(tmp_path, monkeypatch, "cached_posts_loader") + helper.write_text( + "import functools\nfrom pathlib import Path\n\n\n" + "@functools.cache\ndef load_posts():\n" + f" return Path({str(posts)!r}).read_text()\n" + ) + page_cache.enable_read_tracking(tmp_path) + page_cache.clear_import_graph() + try: + module = importlib.import_module("cached_posts_loader") + page_reads = [] + for _ in range(2): + with page_cache.record_reads() as reads: + module.load_posts() + page_reads.append(reads) + assert [str(posts) in reads for reads in page_reads] == [True, False] + + def blog_b(): + return module.load_posts() + + page_ctx = SimpleNamespace( + root_component=SimpleNamespace(children=[]), + output_code="", + memo_contributions={}, + source_files={str(helper.resolve())}, + ) + deps = page_cache.page_dependency_files( + cast(Any, page_ctx), blog_b, {}, tmp_path + ) + finally: + _forget_modules("cached_posts_loader") + page_cache.clear_import_graph() + assert str(posts) in deps + + +def test_reads_are_attributed_to_the_first_party_caller(tmp_path, monkeypatch): + """A read a library performs on a page's behalf belongs to the page's module.""" + data = tmp_path / "note.txt" + data.write_text("hello") + reader = _prepare_runtime_module(tmp_path, monkeypatch, "first_party_reader") + reader.write_text( + "import json\n\n\ndef load(path):\n return json.load(open(path))\n" + ) + data.write_text('{"a": 1}') + page_cache.enable_read_tracking(tmp_path) + try: + module = importlib.import_module("first_party_reader") + with page_cache.record_reads() as reads: + module.load(str(data)) + finally: + _forget_modules("first_party_reader") + assert str(data) in reads + assert page_cache._module_reads[str(reader.resolve())] == {str(data)} + page_cache.forget_module_reads() + assert not page_cache._module_reads + + +@pytest.mark.parametrize("api", ["io.open", "pkgutil", "getsource"]) +def test_record_reads_tracks_stdlib_read_entry_points(tmp_path, monkeypatch, api): + """Reads through ``io.open``, loaders and ``linecache`` are recorded.""" + import inspect + import io + import pkgutil + + package = tmp_path / "read_api_pkg" + package.mkdir() + (package / "__init__.py").write_text("def f():\n return 1\n") + (package / "data.md").write_text("# data") + monkeypatch.syspath_prepend(str(tmp_path)) + _forget_modules("read_api_pkg") + importlib.invalidate_caches() + page_cache.enable_read_tracking(root=tmp_path) + try: + module = importlib.import_module("read_api_pkg") + with page_cache.record_reads() as reads: + if api == "io.open": + with io.open(package / "data.md") as fh: # noqa: UP020 + fh.read() + elif api == "pkgutil": + pkgutil.get_data("read_api_pkg", "data.md") + else: + import linecache + + linecache.clearcache() + inspect.getsource(module.f) + finally: + _forget_modules("read_api_pkg") + expected = package / ("__init__.py" if api == "getsource" else "data.md") + assert str(expected.resolve()) in reads + + +def test_mark_volatile_makes_the_page_miss_every_compile(tmp_path): + """A volatile page records a dependency no file table can satisfy.""" + page_cache.enable_read_tracking(root=tmp_path) + with page_cache.record_reads() as reads: + page_cache.mark_volatile() + assert reads == {page_cache.VOLATILE_DEP} + from types import SimpleNamespace + + page_ctx = SimpleNamespace( + root_component=SimpleNamespace(children=[]), + output_code="", + memo_contributions={}, + source_files=reads, + ) + files = {} + deps = page_cache.page_dependency_entries( + cast(Any, page_ctx), lambda: None, {}, page_cache.file_entry, files, tmp_path + ) + assert page_cache.VOLATILE_DEP in deps + assert page_cache.VOLATILE_DEP not in files + assert not page_cache.FileValidator(files).unchanged(deps) + page_cache.mark_volatile() # a no-op outside a recorder + + +@pytest.mark.parametrize("location", ["assets", "venv_project"]) +def test_reads_under_the_project_are_tracked_whatever_the_directory_names( + tmp_path, location +): + """Only parts below the project root decide the exclusion of a read.""" + root = tmp_path / ("venv" if location == "venv_project" else "app") + data_dir = root / ("assets" if location == "assets" else "data") + data_dir.mkdir(parents=True) + data = data_dir / "note.txt" + data.write_text("v1") + page_cache.enable_read_tracking(root=root) + with page_cache.record_reads() as reads: + data.read_text() + assert str(data.resolve()) in reads + excluded = root / "node_modules" / "pkg" / "readme.md" + excluded.parent.mkdir(parents=True) + excluded.write_text("x") + with page_cache.record_reads() as reads: + excluded.read_text() + assert reads == set() + + +def test_file_modified_during_the_compile_is_recorded_as_racy(tmp_path): + """A dependency edited after the compile started always misses next time. + + The compile may have read the earlier content, so the manifest cannot + vouch for the file (the racy-git rule). + """ + data = tmp_path / "slow.txt" + data.write_text("v2") + with page_cache.record_app_import(tmp_path): + pass + data.write_text("v3") + entry = page_cache.file_entry(str(data)) + assert entry == page_cache.RACY_ENTRY + files = {str(data): page_cache.RACY_ENTRY} + assert not page_cache.FileValidator(files).unchanged([str(data)]) + # Unchanged after the compile: a normal entry. + stat = data.stat() + start = page_cache._compile_start_ns + assert start is not None + os.utime(data, ns=(stat.st_atime_ns, start - 5_000_000_000)) + assert page_cache.file_entry(str(data)) != page_cache.RACY_ENTRY + # A whole-second timestamp in the compile's first second is racy too. + os.utime(data, ns=(stat.st_atime_ns, start - start % 1_000_000_000)) + assert page_cache.file_entry(str(data)) == page_cache.RACY_ENTRY def test_entrypoint_digest_ignores_page_registrations_only(tmp_path): @@ -804,8 +992,8 @@ def digest(source: str) -> str | None: assert digest("this is not python (") is None -def test_entrypoint_digest_ignores_imports_used_only_to_register_pages(tmp_path): - """Importing a page only to register it is a registration; other imports count.""" +def test_entrypoint_digest_counts_every_import(tmp_path): + """Every import counts, even one used only to register a page.""" entry = tmp_path / "entry_app.py" def digest(source: str) -> str | None: @@ -817,6 +1005,7 @@ def digest(source: str) -> str | None: "from .pages.a import a\n\napp = rx.App(theme=theme)\n" 'app.add_page(a, route="/a")\n' ) + # Importing a page runs its module, which can configure the app. assert ( digest( "import reflex as rx\nfrom .themes import dark as theme\n" @@ -824,7 +1013,7 @@ def digest(source: str) -> str | None: 'app = rx.App(theme=theme)\napp.add_page(a, route="/a")\n' 'app.add_page(b, route="/b")\n' ) - == base + != base ) # The same App call with a theme imported from elsewhere is another app. assert ( @@ -881,7 +1070,7 @@ def _rewrite_module(path: Path, source: str) -> None: ('rx.text("b")', 'rx.text("b2")', False), ("\n\ndef page_b", "\n\n# a comment\ndef page_b", False), ("def page_b", "def unrelated():\n return 1\n\n\ndef page_b", False), - ("from .other import unused\n", "", False), + ("from .other import unused\n", "", True), ("rx.box(helper()", "rx.vstack(helper()", True), ("rx.text(TITLE)", "rx.heading(TITLE)", True), ('TITLE = "t"', 'TITLE = "u"', True), @@ -906,31 +1095,61 @@ def test_function_slice_digest_covers_what_the_function_reads( assert (after != before) is changes -@pytest.mark.parametrize( - "source", - [ - # A decorator can hand the function to code the page reads. - ( - "REG = []\n\n\ndef register(f):\n REG.append(f)\n return f\n\n\n" - "@register\ndef widget():\n return 1\n\n\n" - "def page_a():\n return [f() for f in REG]\n" - ), - # Names reached through globals() are invisible to the syntax tree. - ( - "def widget():\n return 1\n\n\n" - 'def page_a():\n return globals()["widget"]()\n' - ), - ], -) -def test_function_slice_digest_keeps_code_reached_indirectly(tmp_path, source): - """Code the page reaches without naming it stays in the slice.""" +def test_function_slice_digest_keeps_code_reached_indirectly(tmp_path): + """Code a decorator hands to import-time code stays in the slice.""" module = tmp_path / "indirect.py" + source = ( + "REG = []\n\n\ndef register(f):\n REG.append(f)\n return f\n\n\n" + "@register\ndef widget():\n return 1\n\n\n" + "def page_a():\n return [f() for f in REG]\n" + ) module.write_text(source) before = page_cache.function_slice_digest(str(module), "page_a") _rewrite_module(module, source.replace("return 1", "return 2")) assert page_cache.function_slice_digest(str(module), "page_a") != before +@pytest.mark.parametrize( + "body", + [ + # Names reached through globals() are invisible to the syntax tree. + 'globals()["widget"]()', + 'getattr(sys.modules[__name__], "widget")()', + # The module's own source text and line numbers are not in the tree. + "inspect.getsource(widget)", + "sys._getframe().f_lineno", + ], +) +def test_function_slice_falls_back_to_the_whole_file_for_computed_access( + tmp_path, monkeypatch, body +): + """A page reaching its module by a computed name depends on the whole file.""" + from types import SimpleNamespace + + module = _prepare_runtime_module(tmp_path, monkeypatch, "computed_access_page") + module.write_text( + "import inspect\nimport sys\n\n\ndef widget():\n return 1\n\n\n" + f"def page_a():\n return {body}\n" + ) + assert page_cache.function_slice_digest(str(module), "page_a") is None + page_cache.clear_import_graph() + try: + page_a = importlib.import_module("computed_access_page").page_a + page_ctx = SimpleNamespace( + root_component=SimpleNamespace(children=[]), + output_code="", + memo_contributions={}, + source_files=set(), + ) + deps = page_cache.page_dependency_files( + cast(Any, page_ctx), page_a, {}, tmp_path + ) + finally: + _forget_modules("computed_access_page") + page_cache.clear_import_graph() + assert str(module.resolve()) in deps + + def test_function_slice_digest_needs_a_top_level_function(tmp_path): """Only a module-level function has a slice.""" module = tmp_path / "nested.py" @@ -1044,24 +1263,6 @@ def test_failed_import_keeps_its_reads(tmp_path, monkeypatch): assert str(missing.resolve()) in reads -def test_function_slice_digest_keeps_imports_of_code_reached_by_computed_name( - tmp_path, -): - """With a computed global lookup, every definition's imports count too.""" - module = tmp_path / "computed.py" - source = ( - "from .widgets import card\n\n\n" - "def widget():\n return card()\n\n\n" - 'def page_a():\n return globals()["widget"]()\n' - ) - module.write_text(source) - before = page_cache.function_slice_digest(str(module), "page_a") - module.write_text( - source.replace("from .widgets import card", "from .other import card") - ) - assert page_cache.function_slice_digest(str(module), "page_a") != before - - @pytest.mark.parametrize( ("decorator", "inert"), [ @@ -1101,13 +1302,10 @@ def test_entrypoint_digest_tracks_class_bodies(tmp_path): assert page_cache.entrypoint_digest(str(entry)) != before -def test_function_slice_digest_keeps_imports_reached_only_by_computed_name(tmp_path): - """A computed global lookup can reach an import no definition names.""" - module = tmp_path / "computed_import.py" - source = ( - "from reflex import text as widget\n\n\n" - 'def page_a():\n return globals()["widget"]()\n' - ) +def test_function_slice_digest_counts_imports_no_definition_names(tmp_path): + """An import counts for every slice, whether or not the code names it.""" + module = tmp_path / "unused_import.py" + source = "from reflex import text as widget\n\n\ndef page_a():\n return 1\n" module.write_text(source) before = page_cache.function_slice_digest(str(module), "page_a") module.write_text(source.replace("text as widget", "heading as widget")) diff --git a/tests/units/conftest.py b/tests/units/conftest.py index 87c64245b0b..b72b76eacde 100644 --- a/tests/units/conftest.py +++ b/tests/units/conftest.py @@ -40,6 +40,22 @@ from .states.upload import SubUploadState, UploadState +@pytest.fixture(autouse=True) +def _forget_compile_start() -> Generator[None, None, None]: + """Unset the compile cache's racy-edit threshold between tests. + + A compile sets it before importing the app; a test that writes files + after another test compiled would otherwise record them as racy. + + Yields: + None. + """ + from reflex.compiler import page_cache + + yield + page_cache._compile_start_ns = None + + @pytest.fixture(autouse=True) def _isolate_app_in_context() -> Generator[None, None, None]: """Reset the App slot on the active RegistrationContext between tests. From b88452e18b5a3f4d5954b5b12ec9e9e26a233317 Mon Sep 17 00:00:00 2001 From: Claude <noreply@anthropic.com> Date: Wed, 21 Oct 2026 21:15:32 +0000 Subject: [PATCH 50/56] refactor(compiler): content-addressed auto memos; app root and page set fall back instead of reconstructing Auto memos no longer mirror their page's module into one shared file. Each is content-addressed by its tag and compiles to its own file under utils/components, so identical subtrees anywhere in the app share one definition and a recompiled page's memo files never hold another page's memos. A page records the memo keys it uses (not the ones it registered first), the memo manifest is updated by incremental compiles, and auto memo files no page uses any more are dropped. The module-sibling expansion and the has_memos flag are gone with the bug they worked around: a shared memo owned only by the first page, whose edit dropped an export a reused page still imported. The app root is compared against its recorded inputs (the merged app wraps in page order, the bundled libraries) and a change falls back to a full compile that adopts the pages already evaluated. The pickled wrap table, the bundle request recording and the library-order reconstruction are deleted. The page registry is an input of every page (a navigation built from the registered pages is common): a page added, removed, reordered or registered differently now falls back to a full compile, which deletes the route-set, removed-page and page-order machinery. Pages added to a rebuild for any reason are evaluated in page order. Also: a reused page's Radix use is recorded and enables the implicit plugin so an all-hit rebuild keeps the Radix stylesheet and theme; a recompiled page no longer claims a state a reused page recorded; the daemon drops the memos of purged modules before re-importing them; the refreshed manifest is staged as the last output and the old one removed before any output is written; absorbed contexts keep page order for the stateful routes. Claude-Session: https://claude.ai/code/session_01RgKnnN8CCBqaR58qm23zQp --- .../src/reflex_base/components/dynamic.py | 34 - .../src/reflex_base/plugins/compiler.py | 53 +- reflex/compiler/compiler.py | 105 +- reflex/compiler/disk_cache.py | 1230 +++++++---------- reflex/compiler/page_cache.py | 55 +- reflex/compiler/plugins/memoize.py | 17 +- reflex/compiler/utils.py | 89 +- reflex/utils/compile_daemon.py | 13 +- tests/units/compiler/test_compiler.py | 108 +- tests/units/compiler/test_disk_cache.py | 509 ++++--- tests/units/compiler/test_memoize_plugin.py | 60 +- tests/units/compiler/test_page_cache.py | 10 +- tests/units/compiler/test_stale_cleanup.py | 35 + tests/units/components/test_memo.py | 8 +- .../reflex_base/components/test_dynamic.py | 16 - tests/units/utils/test_compile_daemon.py | 62 + 16 files changed, 1171 insertions(+), 1233 deletions(-) diff --git a/packages/reflex-base/src/reflex_base/components/dynamic.py b/packages/reflex-base/src/reflex_base/components/dynamic.py index c808579fb39..159eb9277d1 100644 --- a/packages/reflex-base/src/reflex_base/components/dynamic.py +++ b/packages/reflex-base/src/reflex_base/components/dynamic.py @@ -1,9 +1,6 @@ """Components that are dynamically generated on the backend.""" -import contextlib import dataclasses -from collections.abc import Iterator -from contextvars import ContextVar from typing import TYPE_CHECKING, Any, Union from reflex_base import constants @@ -144,35 +141,6 @@ def _bundle_imports( _bundle_library(library + subpath, explicit=explicit) -#: The list recording bundle requests in the current block, if any. -_bundle_requests: ContextVar[list[str] | None] = ContextVar( - "_bundle_requests", default=None -) - - -@contextlib.contextmanager -def record_bundle_requests() -> Iterator[list[str]]: - """Record every library requested for bundling within the block, in order. - - A request counts even when the library is already bundled: the registry - keeps first occurrences, so its order can be rebuilt from the requests of - each source in compile order. An enclosing recorder receives the requests - too. - - Yields: - The list the requests are appended to. - """ - requests: list[str] = [] - outer = _bundle_requests.get() - token = _bundle_requests.set(requests) - try: - yield requests - finally: - _bundle_requests.reset(token) - if outer is not None: - outer.extend(requests) - - def _bundle_library(library: str, *, explicit: bool = False) -> None: """Register a library, optionally retaining it across compiler resets. @@ -181,8 +149,6 @@ def _bundle_library(library: str, *, explicit: bool = False) -> None: explicit: Whether this registration belongs to the app rather than a compile. """ library = format_library_name(library) - if (requests := _bundle_requests.get()) is not None: - requests.append(library) context = RegistrationContext.ensure_context() if explicit: context._explicit_bundled_libraries[library] = None diff --git a/packages/reflex-base/src/reflex_base/plugins/compiler.py b/packages/reflex-base/src/reflex_base/plugins/compiler.py index 3b0a0a7e0eb..29f30691461 100644 --- a/packages/reflex-base/src/reflex_base/plugins/compiler.py +++ b/packages/reflex-base/src/reflex_base/plugins/compiler.py @@ -609,11 +609,9 @@ class PageContext(BaseContext): # The page's unique-name generator state after evaluation, resumed for its # render so its generated names never depend on other pages. unique_name_state: Any = None - # Libraries requested for bundling while the page evaluated and rendered. - evaluate_bundle_requests: list[str] = dataclasses.field(default_factory=list) - render_bundle_requests: list[str] = dataclasses.field(default_factory=list) - # Auto-memo components first registered while compiling this page. - memo_contributions: dict[tuple[str, str | None], Any] = dataclasses.field( + # The auto-memo components this page's output imports, by registry key + # (filled by ``MemoizeStatefulPlugin``). + memo_uses: dict[tuple[str, str | None], Any] = dataclasses.field( default_factory=dict ) # Stack of ``id(component)`` for components whose subtree is @@ -697,17 +695,13 @@ class CompileContext(BaseContext): # ``MemoizeStatefulPlugin``). memoize_wrappers: dict[str, None] = dataclasses.field(default_factory=dict) # Compiler-generated memo definitions for auto-memoized stateful wrappers. - # Keyed by ``(tag, source_module)`` so identical-rendering subtrees from - # different user modules each get their own entry and emit into the right - # mirrored memo file. Stored as ``Any`` to avoid an import cycle with - # ``reflex_base.components.memo``. + # Keyed by ``(tag, source_module)``; an auto memo is content-addressed by + # its tag and carries no source module, so identical subtrees anywhere in + # the app share one definition and one output file. Stored as ``Any`` to + # avoid an import cycle with ``reflex_base.components.memo``. auto_memo_components: dict[tuple[str, str | None], Any] = dataclasses.field( default_factory=dict ) - # The bundled libraries registered when this compile reset the registry, - # before any page evaluated; the compile cache rebuilds the registry's - # order from it and the libraries each page and state requested. - bundle_base: list[str] = dataclasses.field(default_factory=list) def compile( self, @@ -728,7 +722,6 @@ def compile( """ from reflex.compiler import compiler from reflex.state import all_base_state_classes - from reflex_base.components.dynamic import record_bundle_requests from reflex_base.components.memo import registration_owner from reflex_base.vars.base import ( reset_unique_variable_names, @@ -755,10 +748,7 @@ def compile( seed_unique_variable_names(page.route) # Memos registered while this page evaluates (e.g. from modules # imported inside the page function) are owned by this route. - with ( - registration_owner(page.route), - record_bundle_requests() as bundle_requests, - ): + with registration_owner(page.route): if recorder is not None: with recorder() as read_set: page_ctx = self.hooks.eval_page( @@ -796,7 +786,6 @@ def compile( ] page_ctx.unique_name_state = unique_variable_name_state() - page_ctx.evaluate_bundle_requests = bundle_requests self.compiled_pages[page_ctx.route] = page_ctx if evaluate_progress is not None: @@ -807,9 +796,8 @@ def compile( self.compiled_pages.values(), strict=True, ): - memo_before = set(self.auto_memo_components) restore_unique_variable_name_state(page_ctx.unique_name_state) - with page_ctx, record_bundle_requests() as render_requests: + with page_ctx: page_ctx.root_component = self.hooks.compile_component( page_ctx.root_component, page_context=page_ctx, @@ -821,23 +809,14 @@ def compile( compile_context=self, **kwargs, ) - # Attribute newly-registered auto-memo components to this page. - page_ctx.memo_contributions = { - key: value - for key, value in self.auto_memo_components.items() - if key not in memo_before - } - page_ctx.frontend_imports = page_ctx.merged_imports(collapse=True) self.all_imports = merge_imports( self.all_imports, page_ctx.frontend_imports ) self.app_wrap_components.update(page_ctx.app_wrap_components) - with record_bundle_requests() as output_requests: - page_ctx.output_path, page_ctx.output_code = ( - compiler.compile_page_from_context(page_ctx) - ) - page_ctx.render_bundle_requests = render_requests + output_requests + page_ctx.output_path, page_ctx.output_code = ( + compiler.compile_page_from_context(page_ctx) + ) if render_progress is not None: render_progress() @@ -867,7 +846,13 @@ def absorb(self, other: CompileContext, pages: Sequence[PageDefinition]) -> None self.app_wrap_components.update( self.compiled_pages[page.route].app_wrap_components ) - self.stateful_routes.update(other.stateful_routes) + # In page order, like a compile that evaluated every page. + stateful_routes = {**self.stateful_routes, **other.stateful_routes} + self.stateful_routes = { + page.route: stateful_routes[page.route] + for page in pages + if page.route in stateful_routes + } self.memoize_wrappers.update(other.memoize_wrappers) self.auto_memo_components.update(other.auto_memo_components) diff --git a/reflex/compiler/compiler.py b/reflex/compiler/compiler.py index 4302369fbe1..dfced7aca66 100644 --- a/reflex/compiler/compiler.py +++ b/reflex/compiler/compiler.py @@ -559,7 +559,6 @@ def _compile_memo_components( output_files: list[tuple[str, str]] = [] aggregate_imports: dict[str, list[ImportVar]] = {} unmirrored_files: list[tuple[str, str]] = [] - unmirrored_base_dir = utils.get_memo_components_dir() groups: collections.defaultdict[tuple[str, ...], _MemoGroup] = ( collections.defaultdict(_MemoGroup) ) @@ -570,10 +569,7 @@ def _emit_unmirrored( render_imports: dict, ) -> None: code, file_imports = compile_fn(render, render_imports) - unmirrored_files.append(( - _memo_component_file_path(unmirrored_base_dir, render["name"]), - code, - )) + unmirrored_files.append((utils.get_memo_component_path(render["name"]), code)) _extend_imports_in_place(aggregate_imports, file_imports) for memo in memos: @@ -683,19 +679,6 @@ def _compile_single_memo_function( return code, imports -def _memo_component_file_path(base_dir: str, name: str) -> str: - """Return the on-disk path for a per-memo module. - - Args: - base_dir: The directory that holds per-memo files. - name: The memo's export name. - - Returns: - The absolute path for the memo's ``.jsx`` file. - """ - return str(Path(base_dir) / f"{name}{constants.Ext.JSX}") - - def compile_document_root( head_components: list[Component], html_lang: str | None = None, @@ -1374,14 +1357,11 @@ def _apply_plugin_outputs( return sources -def _reset_compile_registries(compiler_plugins: Sequence[Plugin]) -> list[str]: +def _reset_compile_registries(compiler_plugins: Sequence[Plugin]) -> None: """Reset the registries a compile fills before any page is evaluated. Args: compiler_plugins: The resolved compiler plugins. - - Returns: - The bundled libraries registered after the reset. """ from reflex_base.components.dynamic import ( _bundle_library, @@ -1399,7 +1379,6 @@ def _reset_compile_registries(compiler_plugins: Sequence[Plugin]) -> list[str]: for plugin in compiler_plugins: for dependency in plugin.get_frontend_dependencies(): _bundle_library(dependency) - return list(RegistrationContext.ensure_context().bundled_libraries) def _read_stateful_pages_marker() -> list[str] | None: @@ -1461,37 +1440,42 @@ def _compile_from_cache( registrations=registrations, ) if rebuilt is True: - try: - save_tasks, modify_tasks = _plugin_output_tasks( - config.plugins, - radix_themes_plugin, - list(app._unevaluated_pages.values()), - ) - manifest = disk_cache.load_manifest() - stylesheet_path, stylesheet_code = compile_root_stylesheet( - app.stylesheets, app.reset_style, plugins=compiler_plugins - ) - outputs[utils.resolve_path_of_web_dir(stylesheet_path)] = stylesheet_code - for task_fn, args, kwargs in save_tasks: - result = task_fn(*args, **kwargs) - if result is not None: - for path, code in result if isinstance(result, list) else [result]: - outputs[utils.resolve_path_of_web_dir(path)] = code - sources = _apply_plugin_outputs( - outputs, - config.plugins, - modify_tasks, - manifest.get("plugin_sources", {}) if manifest is not None else {}, - ) - for path, code in outputs.items(): - utils.write_file(path, code) - if manifest is not None and manifest.get("plugin_sources") != sources: - manifest["plugin_sources"] = sources - disk_cache._write(manifest) - except BaseException: - # The staged manifest must not validate outputs that failed to save. - disk_cache._manifest_path().unlink(missing_ok=True) - raise + manifest_path = disk_cache._manifest_path().absolute() + # The refreshed manifest is written last: a manifest on disk must + # never vouch for outputs that were not written, so until every + # other file is on disk there is none. + staged_manifest = outputs.pop(manifest_path, None) + manifest = ( + json.loads(staged_manifest) + if staged_manifest is not None + else disk_cache.load_manifest() + ) + manifest_path.unlink(missing_ok=True) + save_tasks, modify_tasks = _plugin_output_tasks( + config.plugins, + radix_themes_plugin, + list(app._unevaluated_pages.values()), + ) + stylesheet_path, stylesheet_code = compile_root_stylesheet( + app.stylesheets, app.reset_style, plugins=compiler_plugins + ) + outputs[utils.resolve_path_of_web_dir(stylesheet_path)] = stylesheet_code + for task_fn, args, kwargs in save_tasks: + result = task_fn(*args, **kwargs) + if result is not None: + for path, code in result if isinstance(result, list) else [result]: + outputs[utils.resolve_path_of_web_dir(path)] = code + sources = _apply_plugin_outputs( + outputs, + config.plugins, + modify_tasks, + manifest.get("plugin_sources", {}) if manifest is not None else {}, + ) + for path, code in outputs.items(): + utils.write_file(path, code) + if manifest is not None: + manifest["plugin_sources"] = sources + disk_cache._write(manifest) return rebuilt @@ -1518,7 +1502,6 @@ def compile_app( ``True`` when a real frontend compile ran, ``False`` when the call short-circuited (backend-only paths that only re-evaluate pages). """ - from reflex_base.components.dynamic import record_bundle_requests from reflex_base.utils.exceptions import ReflexRuntimeError app._apply_decorated_pages() @@ -1617,9 +1600,8 @@ def attempt() -> CompileContext | bool: all_pages = list(app._unevaluated_pages.values()) if evaluated is None: pages_to_compile = all_pages - bundle_base = _reset_compile_registries(compiler_plugins) + _reset_compile_registries(compiler_plugins) else: - bundle_base = evaluated.bundle_base # The incremental attempt already did the resets above and bundled what # its pages registered; only the pages it never touched remain. pages_to_compile = [ @@ -1635,7 +1617,6 @@ def attempt() -> CompileContext | bool: hooks=CompilerHooks( plugins=default_page_plugins(style=app.style, plugins=compiler_plugins) ), - bundle_base=bundle_base, ) with ( @@ -1696,10 +1677,9 @@ def attempt() -> CompileContext | bool: raise ReflexRuntimeError(msg) progress.advance(task) - with record_bundle_requests() as root_bundle_requests: - app_root, hydrate_fallback_export = _build_app_root( - app, compile_ctx.app_wrap_components, compile_ctx - ) + app_root, hydrate_fallback_export = _build_app_root( + app, compile_ctx.app_wrap_components, compile_ctx + ) all_imports = utils.merge_imports(all_imports, app_root._get_all_imports()) memo_component_files, memo_components_imports = compile_memo_components( @@ -1857,7 +1837,6 @@ def attempt() -> CompileContext | bool: all_imports, plugin_sources=plugin_sources, contexts_snapshot=contexts_snapshot, - root_bundle_requests=root_bundle_requests, registrations=registrations, ) diff --git a/reflex/compiler/disk_cache.py b/reflex/compiler/disk_cache.py index cd563dcc2b6..a09b689cce7 100644 --- a/reflex/compiler/disk_cache.py +++ b/reflex/compiler/disk_cache.py @@ -1,36 +1,40 @@ """Disk-persisted incremental compile cache for ``REFLEX_COMPILE_CACHE``. The manifest stores only bookkeeping: a file table (content hash plus stat key -per dependency file), each page's dependency paths, app-wrap keys, statefulness -and its contribution to the contexts file, the app-wide frontend imports, and -the import-name parse cache. Rendered files stay in ``.web``. When global -inputs and routes still match, unchanged pages are reused from disk and only +per dependency file), each page's dependency paths, app-wrap digests, memo +uses, statefulness and its contribution to the contexts file, the app-wide +frontend imports, the app root's inputs, and the import-name parse cache. +Rendered files stay in ``.web``. When the global inputs, the page set and the +registrations still match, unchanged pages are reused from disk and only dependency-changed pages are recompiled. +Each page is a memoized query over the inputs it recorded. Outputs that +several pages feed are handled in one of two ways: the memo files are +content-addressed, one file per auto memo, so a page's memos never share a +file with another page's; and the app root and the bundled-library file are +compared against their recorded inputs and fall back to a full compile when a +recompiled page changed them. + Validation is stat-first (see :class:`page_cache.FileValidator`): an unchanged file is recognised from ``(mtime_ns, size)`` without being read, and when the watcher reports which paths changed, pages depending on none of them are hits without even a stat. - """ from __future__ import annotations -import base64 import contextlib import dataclasses import functools import hashlib import json import logging -import pickle import re import sys from pathlib import Path from typing import TYPE_CHECKING, Any from reflex_base import constants -from reflex_base.components.dynamic import record_bundle_requests from reflex_base.environment import environment from reflex_base.plugins import CompileContext, CompilerHooks, Plugin from reflex_base.registry import RegistrationContext @@ -54,7 +58,7 @@ logger = logging.getLogger(__name__) #: Bump when the manifest layout changes (old manifests are then ignored). -_SCHEMA = 22 +_SCHEMA = 23 #: Manifest filename under the web directory. _MANIFEST_FILE = "reflex_compile_cache.json" @@ -199,19 +203,6 @@ def _wrap_key_str(key: tuple[int, str]) -> str: return f"{key[0]}:{key[1]}" -def _wrap_key(key_str: str) -> tuple[int, str]: - """Parse an app-wrap key formatted by :func:`_wrap_key_str`. - - Args: - key_str: The formatted key. - - Returns: - The ``(priority, name)`` app-wrap key. - """ - priority, _, name = key_str.partition(":") - return int(priority), name - - def _wrap_digest(component: Any) -> str: """Digest what an app wrap contributes to the compiled app root. @@ -233,98 +224,19 @@ def _wrap_digest(component: Any) -> str: return hashlib.sha256(json_dumps(payload).encode()).hexdigest() -def _stored_wrap(component: Any, digest: str) -> str | None: - """Serialize an app wrap so the app root can be rebuilt without its page. - - Args: - component: The app-wrap component. - digest: Its :func:`_wrap_digest`. - - Returns: - The encoded component, or None when it does not round-trip to the same - contribution. - """ - try: - data = pickle.dumps(component) - if _wrap_digest(pickle.loads(data)) != digest: - return None - except Exception: - return None - return base64.b64encode(data).decode() - - -def _record_app_wraps( - page_ctx: PageContext, wrap_table: dict[str, str | None] -) -> dict[str, str]: - """Record a page's app wraps by digest, interning each in the wrap table. +def _record_app_wraps(page_ctx: PageContext) -> dict[str, str]: + """Record a page's app wraps by digest. Args: page_ctx: The compiled page context. - wrap_table: The manifest's digest -> stored component table (extended). Returns: The page's app-wrap keys mapped to their digests, in page order. """ - wraps = {} - for key, component in page_ctx.app_wrap_components.items(): - digest = _wrap_digest(component) - if digest not in wrap_table: - wrap_table[digest] = _stored_wrap(component, digest) - wraps[_wrap_key_str(key)] = digest - return wraps - - -def _state_libraries(initial_state: dict[str, Any]) -> dict[str, list[str]]: - """Record the libraries serializing each state's initial values requests. - - Args: - initial_state: Initial values keyed by state full name. - - Returns: - The requested libraries of each state that requests any. - """ - from reflex.compiler import utils as compiler_utils - - libraries = {} - for name, values in initial_state.items(): - with record_bundle_requests() as requests: - compiler_utils._serialize_initial_state({name: values}) - if requests: - libraries[name] = requests - return libraries - - -def _reconstruct_libraries( - base: Sequence[str], - page_requests: Sequence[Sequence[Sequence[str]]], - root_requests: Sequence[str], - state_libraries: dict[str, list[str]], -) -> list[str]: - """Rebuild the bundled-library registry in full-compile order. - - A full compile evaluates every page, then renders every page, builds the - app root and serializes the states in name order; the registry keeps the - first request of each library. - - Args: - base: The registry after the compile's reset. - page_requests: ``[evaluate, render]`` requests of each page, in page - order. - root_requests: The requests made while building the app root. - state_libraries: The requests of each state's initial values. - - Returns: - The bundled libraries, in registry order. - """ - ordered = [*base] - for evaluate, _ in page_requests: - ordered.extend(evaluate) - for _, render in page_requests: - ordered.extend(render) - ordered.extend(root_requests) - for name in sorted(state_libraries): - ordered.extend(state_libraries[name]) - return list(dict.fromkeys(ordered)) + return { + _wrap_key_str(key): _wrap_digest(component) + for key, component in page_ctx.app_wrap_components.items() + } def _stable_value(value: Any) -> Any: @@ -397,6 +309,38 @@ def registration_digests(pages: Iterable[PageDefinition]) -> dict[str, str]: return {page.route: _registration_digest(page) for page in pages} +def _memo_keys(page_ctx: PageContext) -> list[str]: + """The export names of the auto memos a page's output imports. + + Args: + page_ctx: The compiled page context. + + Returns: + The sorted memo tags. + """ + return sorted(tag for tag, _ in page_ctx.memo_uses) + + +def _uses_radix(root_component: object) -> bool: + """Whether a rendered tree holds a Radix Themes component. + + The implicit ``RadixThemesPlugin`` enables itself when it compiles one; + a page reused from disk is not compiled, so the manifest remembers. + + Args: + root_component: The page's root component. + + Returns: + True when the tree enables Radix Themes. + """ + from reflex_components_radix.themes.base import RadixThemesComponent + + return any( + issubclass(cls, RadixThemesComponent) + for cls in page_cache.component_tree_classes(root_component) + ) + + def _manifest_page_entry( page_ctx: PageContext, page: PageDefinition, @@ -406,7 +350,6 @@ def _manifest_page_entry( *, defined_states: Sequence[str] | None, registration: str, - wrap_table: dict[str, str | None], state_slice: dict[str, Any] | None = None, root: Path | None = None, ) -> dict[str, Any]: @@ -422,8 +365,6 @@ def _manifest_page_entry( evaluated, or None for a stateless page. registration: The page's registration digest (see :func:`registration_digests`). - wrap_table: The manifest's app-wrap table, extended with the page's - wraps. state_slice: The page's contribution to the contexts file (see :func:`_state_slice`), or None for stateless pages. root: Project root for dependency discovery. Defaults to cwd. @@ -438,31 +379,21 @@ def _manifest_page_entry( "deps": page_cache.page_dependency_entries( page_ctx, page.component, state_index, hasher, files, root ), - # Everything the page was registered with (see ``partition_pages``). + # Everything the page was registered with (see ``_pages_mismatch``). "registration": registration, - # The app wraps the page requires, by digest (see ``_refresh_app_root``). - "app_wraps": _record_app_wraps(page_ctx, wrap_table), - # Libraries requested while evaluating and rendering the page. - "bundle_requests": [ - page_ctx.evaluate_bundle_requests, - page_ctx.render_bundle_requests, - ], + # The app wraps the page requires, by digest (see ``_app_root_change``). + "app_wraps": _record_app_wraps(page_ctx), "is_stateful": defined_states is not None, "state_slice": state_slice, # The ``reflex.istate.dynamic`` names the page's states took, whose # suffixes depend on evaluation order (see ``_with_dynamic_state_pages``). "dynamic_claims": _dynamic_claims(defined_states or ()), - # Whether the page contributed auto memos: pages sharing a source - # module share one memo output file, so a memo-contributing hit page - # must be recompiled alongside a same-module miss (see - # ``_with_module_siblings``). - "has_memos": bool(page_ctx.memo_contributions), - # Where those auto memos are written, so removing the page rewrites it. - "memo_path": ( - _memo_output_path(getattr(page, "_source_module", None)) - if page_ctx.memo_contributions - else None - ), + # The auto memos the page imports, each in its own content-addressed + # file (see ``_memo_file``); the files a rebuild may drop are the ones + # no page uses any more. + "memo_keys": _memo_keys(page_ctx), + # Whether the page enables the implicit Radix Themes plugin. + "radix": _uses_radix(page_ctx.root_component), } @@ -611,6 +542,47 @@ def _with_hit_state_slices( return dict(sorted(initial.items())), storage, schema_entries +def _hit_page_states(manifest: dict[str, Any], miss_routes: set[str]) -> set[str]: + """The states the manifest records as defined by pages that hit. + + Args: + manifest: The loaded manifest. + miss_routes: The recompiled routes. + + Returns: + The full names of the hit pages' states. + """ + return { + name + for route, entry in manifest["pages"].items() + if route not in miss_routes and (state_slice := entry["state_slice"]) + for name in state_slice["schema"] + } + + +def _disown_hit_page_states( + miss_ctx: CompileContext, manifest: dict[str, Any], miss_routes: set[str] +) -> None: + """Leave a hit page's states to it when a recompiled page registers them. + + A module state is recorded by the page whose evaluation first imported + its module. In a process that evaluates only the recompiled pages, one + of them can import that module first and would otherwise claim the + state, marking itself stateful where a full compile would not. + + Args: + miss_ctx: The compile context of the recompiled pages (mutated). + manifest: The loaded manifest. + miss_routes: The recompiled routes. + """ + owned = _hit_page_states(manifest, miss_routes) + for route, names in list(miss_ctx.stateful_routes.items()): + if kept := [name for name in names if name not in owned]: + miss_ctx.stateful_routes[route] = kept + else: + del miss_ctx.stateful_routes[route] + + def load_manifest() -> dict[str, Any] | None: """Load the persisted compile manifest, or None if absent/unusable. @@ -678,16 +650,28 @@ def manifest_dependency_files(manifest: dict[str, Any]) -> set[str]: return files +def _manifest_text(manifest: dict[str, Any]) -> str: + """Serialize the manifest, including the current import-name parse cache. + + Args: + manifest: The manifest to serialize. + + Returns: + The manifest JSON. + """ + manifest["import_names"] = page_cache.export_import_names_cache() + return json_dumps(manifest) + + def _write(manifest: dict[str, Any]) -> None: - """Persist the manifest, including the current import-name parse cache. + """Persist the manifest. Args: manifest: The manifest to write. """ - manifest["import_names"] = page_cache.export_import_names_cache() path = _manifest_path() path.parent.mkdir(parents=True, exist_ok=True) - path.write_text(json_dumps(manifest), encoding="utf-8") + path.write_text(_manifest_text(manifest), encoding="utf-8") def write_manifest( @@ -698,7 +682,6 @@ def write_manifest( *, plugin_sources: dict[str, str] | None = None, contexts_snapshot: compiler_utils.ContextsSnapshot | None = None, - root_bundle_requests: Sequence[str] = (), registrations: dict[str, str] | None = None, ) -> None: """Persist a manifest of the just-completed full compile. @@ -707,7 +690,8 @@ def write_manifest( never breaks the build. Args: - compile_ctx: The completed compile context (all pages compiled). + compile_ctx: The completed compile context (all pages compiled, the + app root built). pages: The full list of page definitions that were compiled. install_imports: The **complete** frontend import set the full compile installed: page imports merged with the app-root (app-wrap, e.g. @@ -718,8 +702,6 @@ def write_manifest( plugin_sources: Unmodified source content for plugin file modifiers. contexts_snapshot: The state-tree inputs used for the contexts output. When absent, a snapshot is captured for direct callers. - root_bundle_requests: The libraries requested while building the app - root. registrations: Each route's registration digest, taken before any page evaluated; computed now when absent. """ @@ -727,7 +709,6 @@ def write_manifest( state_index, _ = page_cache.state_dependency_index(root) hasher = page_cache.make_hasher() files: dict[str, page_cache.FileEntry] = {} - wrap_table: dict[str, str | None] = {} if registrations is None: registrations = registration_digests(pages) if contexts_snapshot is None and compile_ctx.stateful_routes: @@ -755,7 +736,6 @@ def write_manifest( files, defined_states=defined_states, registration=registrations[page.route], - wrap_table=wrap_table, state_slice=( _state_slice(defined_states, *contexts_snapshot) if defined_states and contexts_snapshot is not None @@ -763,23 +743,6 @@ def write_manifest( ), root=root, ) - state_libraries = ( - _state_libraries(contexts_snapshot[0]) - if contexts_snapshot is not None - else {} - ) - libraries = list(RegistrationContext.ensure_context().bundled_libraries) - reconstructed = _reconstruct_libraries( - compile_ctx.bundle_base, - [pages_data[page.route]["bundle_requests"] for page in pages], - root_bundle_requests, - state_libraries, - ) - if reconstructed != libraries: - logger.debug( - "Compile cache: bundled libraries were requested outside the " - "recorded phases; changes to the app root need a full compile." - ) # Post-evaluation and complete: the full compile evaluated every page, # so both import-owned and route-owned memos are recorded. @@ -809,6 +772,9 @@ def write_manifest( files[path] = entry globals_[label] = path + page_memo_keys = { + tag for entry in pages_data.values() for tag in entry["memo_keys"] + } entry_file = page_cache._app_entrypoint_file(root) manifest = { "schema": _SCHEMA, @@ -836,21 +802,25 @@ def write_manifest( "all_imports": _serialize_imports(install_imports), "pages": pages_data, "memo_files": memo_files, - "wrap_table": wrap_table, - "state_libraries": state_libraries, # The states importing the app registers (see # ``_record_module_states``). "module_states": module_states, - # The app root's inputs: merged app wraps (by digest, in insertion - # order) and the bundled libraries (see ``_refresh_app_root``). + # The app root's inputs: the merged app wraps (by digest, in + # insertion order) and the bundled libraries (see + # ``_app_root_change``), plus the memo files only the root uses. "app_root": { "wraps": { _wrap_key_str(key): _wrap_digest(component) for key, component in compile_ctx.app_wrap_components.items() }, - "libraries": libraries, - "root_requests": list(root_bundle_requests), - "reconstructible": reconstructed == libraries, + "libraries": sorted( + RegistrationContext.ensure_context().bundled_libraries + ), + "memo_keys": sorted( + tag + for tag, _ in compile_ctx.auto_memo_components + if tag not in page_memo_keys + ), }, } _write(manifest) @@ -978,7 +948,7 @@ def globals_mismatch( if label != "reflex" and validator.changed(path) # An entry module that only changed its page registrations is compared - # page by page (see ``partition_pages``). + # page by page (see ``_pages_mismatch``). and not ( path == entry["path"] and entry["digest"] is not None @@ -993,6 +963,45 @@ def globals_mismatch( return None +def _pages_mismatch( + pages: Sequence[PageDefinition], + manifest: dict[str, Any], + registrations: dict[str, str], +) -> str | None: + """Explain why the manifest's page set no longer matches, or None if it does. + + The page registry is an input of every page (a navigation built from the + registered pages is common), so a page added, removed, reordered or + registered differently (its title, say) is a change to every page. + + Args: + pages: The current page definitions, in compile order. + manifest: The loaded manifest. + registrations: Each route's registration digest. + + Returns: + A human-readable mismatch reason, or None when the pages match. + """ + recorded = manifest["pages"] + routes = [page.route for page in pages] + if routes != list(recorded): + changes = [ + f"{label} {format_path_list(items)}" + for label, items in ( + ("added", set(routes) - set(recorded)), + ("removed", set(recorded) - set(routes)), + ) + if items + ] + if not changes: + return "the page order changed" + return f"the page set changed ({'; '.join(changes)})" + for route in routes: + if recorded[route]["registration"] != registrations[route]: + return f"page {route!r} was registered differently" + return None + + def _with_hinted_slices( changed_hint: set[str] | None, manifest: dict[str, Any] ) -> set[str] | None: @@ -1021,33 +1030,28 @@ def partition_pages( manifest: dict[str, Any], validator: page_cache.FileValidator, changed_hint: set[str] | None, - registrations: dict[str, str], ) -> list[PageDefinition]: - """Return the pages that are new or changed since the manifest. + """Return the pages whose recorded dependencies changed. - Globals are assumed already matched (see :func:`globals_mismatch`), so a - recorded page is a hit if it was registered the same way and every file - in its recorded dependency set is unchanged. When the watcher reported the - changed paths, a page depending on none of them skips the filesystem. + Globals and the page set are assumed already matched (see + :func:`globals_mismatch` and :func:`_pages_mismatch`), so a page is a + hit if every file in its recorded dependency set is unchanged. When the + watcher reported the changed paths, a page depending on none of them + skips the filesystem. Args: pages: The current page definitions. manifest: The loaded manifest. validator: The file validator bound to the manifest's file table. changed_hint: Resolved paths the watcher saw change, or None if unknown. - registrations: Each route's registration digest. Returns: - The list of miss pages (new, re-registered or a dependency changed). + The list of miss pages, in ``pages`` order. """ manifest_pages = manifest["pages"] misses: list[PageDefinition] = [] for page in pages: - entry = manifest_pages.get(page.route) - if entry is None or entry["registration"] != registrations[page.route]: - misses.append(page) - continue - deps = entry["deps"] + deps = manifest_pages[page.route]["deps"] if changed_hint is not None and changed_hint.isdisjoint(deps): continue if not validator.unchanged(deps): @@ -1055,45 +1059,6 @@ def partition_pages( return misses -def _with_module_siblings( - miss_pages: list[PageDefinition], - pages: Sequence[PageDefinition], - manifest: dict[str, Any], -) -> list[PageDefinition]: - """Expand the miss set with memo-contributing same-module hit pages. - - Auto-memo output is grouped into one file per source module, so rewriting - that file needs the contributions of *all* the module's pages that have - any. Hit pages that contributed no memos (per the manifest) have nothing - in that file and are left reused. - - Args: - miss_pages: The dependency-changed pages. - pages: All current page definitions (in compile order). - manifest: The loaded manifest. - - Returns: - The expanded miss list, in ``pages`` order. - """ - miss_modules = { - module - for page in miss_pages - if (module := getattr(page, "_source_module", None)) is not None - } - if not miss_modules: - return miss_pages - miss_routes = {page.route for page in miss_pages} - return [ - page - for page in pages - if page.route in miss_routes - or ( - getattr(page, "_source_module", None) in miss_modules - and manifest["pages"][page.route]["has_memos"] - ) - ] - - def _dynamic_claims(defined_states: Iterable[str]) -> list[list[str]]: """The ``reflex.istate.dynamic`` names a page's states took, in order. @@ -1148,26 +1113,22 @@ def _claims_by_family(claims: Iterable[Sequence[str]]) -> dict[str, list[list[st def _shifting_dynamic_claims( - manifest: dict[str, Any], - miss_ctx: CompileContext | None, - miss_routes: set[str], - removed_entries: dict[str, dict[str, Any]], + manifest: dict[str, Any], miss_ctx: CompileContext, miss_routes: set[str] ) -> str | None: - """Find a changed page whose claims could shift other pages' names. + """Find a re-evaluated page whose claims could shift other pages' names. - Names in a family are handed out in evaluation order, so a re-evaluated, - added or removed page changing its claims in a family moves the names of - later pages in that family. Pages that were not re-evaluated keep their - recorded names, which is exact only when no family they use changed. + Names in a family are handed out in evaluation order, so a re-evaluated + page changing its claims in a family moves the names of later pages in + that family. Pages that were not re-evaluated keep their recorded names, + which is exact only when no family they use changed. Args: - manifest: The loaded manifest (without the removed pages). - miss_ctx: The compile context of the re-evaluated pages, if any. + manifest: The loaded manifest. + miss_ctx: The compile context of the re-evaluated pages. miss_routes: The re-evaluated routes. - removed_entries: The manifest entries of the removed routes. Returns: - The first changed route that changed a shared family, or None. + The first re-evaluated route that changed a shared family, or None. """ shared = { family @@ -1175,22 +1136,11 @@ def _shifting_dynamic_claims( if route not in miss_routes for family in _claims_by_family(entry["dynamic_claims"]) } - changes = [ - ( - route, - manifest["pages"].get(route, {}).get("dynamic_claims", []), + for route in miss_routes: + recorded = _claims_by_family(manifest["pages"][route]["dynamic_claims"]) + live = _claims_by_family( _dynamic_claims(miss_ctx.stateful_routes.get(route, ())) - if miss_ctx is not None - else [], ) - for route in miss_routes - ] - changes.extend( - (route, entry["dynamic_claims"], []) for route, entry in removed_entries.items() - ) - for route, recorded_claims, live_claims in changes: - recorded = _claims_by_family(recorded_claims) - live = _claims_by_family(live_claims) if any( live.get(family) != recorded.get(family) for family in (live.keys() | recorded.keys()) & shared @@ -1284,11 +1234,9 @@ def eval_page( def _route_dropping_a_shared_state( - manifest: dict[str, Any], - miss_routes: Iterable[str], - removed_entries: dict[str, dict[str, Any]], + manifest: dict[str, Any], miss_routes: Iterable[str] ) -> str | None: - """Find a recompiled or removed page whose recorded module state is gone. + """Find a recompiled page whose recorded module state is gone. A page records the states its evaluation registered first. Another page using the same module state recorded none, so when the first page stops @@ -1296,9 +1244,8 @@ def _route_dropping_a_shared_state( its page alone. Args: - manifest: The loaded manifest (without the removed pages). + manifest: The loaded manifest. miss_routes: The recompiled routes. - removed_entries: The manifest entries of the removed routes. Returns: The first such route, or None. @@ -1307,10 +1254,9 @@ def _route_dropping_a_shared_state( local_prefix = istate_dynamic.__name__.replace(".", "___") + "___" live = RegistrationContext.ensure_context().base_states - entries = [(route, manifest["pages"].get(route)) for route in miss_routes] - entries.extend(removed_entries.items()) - for route, entry in entries: - if entry is None or not entry["state_slice"]: + for route in miss_routes: + entry = manifest["pages"][route] + if not entry["state_slice"]: continue for name in entry["state_slice"]["schema"]: if name not in live and not name.rpartition(".")[2].startswith( @@ -1362,8 +1308,6 @@ def _with_dynamic_state_pages( miss_pages: list[PageDefinition], pages: Sequence[PageDefinition], manifest: dict[str, Any], - *, - shifted: bool = False, ) -> list[PageDefinition]: """Expand a miss set with every page that defines dynamic-module states. @@ -1378,13 +1322,11 @@ def _with_dynamic_state_pages( miss_pages: The pages to recompile. pages: All current page definitions (in compile order). manifest: The loaded manifest. - shifted: Whether names may have shifted without a miss: a removed - page defined dynamic-module states, or such pages were reordered. Returns: The expanded miss list, in ``pages`` order. """ - if not miss_pages and not shifted: + if not miss_pages: return miss_pages miss_routes = {page.route for page in miss_pages} return [ @@ -1394,63 +1336,32 @@ def _with_dynamic_state_pages( ] -def _dynamic_page_order_changed( - manifest: dict[str, Any], pages: Sequence[PageDefinition] -) -> bool: - """Whether the recorded pages defining dynamic states changed their order. - - Their states take names in page order. The manifest lists pages in the - order of the compile that wrote it. - - Args: - manifest: The loaded manifest (without the removed pages). - pages: All current page definitions (in compile order). - - Returns: - True when those pages are in another order now. - """ - recorded = manifest["pages"] - return [route for route, entry in recorded.items() if entry["dynamic_claims"]] != [ - page.route - for page in pages - if (entry := recorded.get(page.route)) is not None and entry["dynamic_claims"] - ] - - def _expand_miss_pages( miss_pages: list[PageDefinition], pages: Sequence[PageDefinition], manifest: dict[str, Any], - dirty_memo_paths: set[str], stale_owner_routes: set[str], - *, - dynamic_names_shift: bool, ) -> list[PageDefinition]: """Add the pages a rebuild of ``miss_pages`` must evaluate too. - Module siblings come last: every page added before them may share a - memo file with hit pages of its module. + A route-owned memo entry whose dependency files changed needs its owner + page re-evaluated: only that page's evaluation re-registers the memo + definitions the rewrite compiles from. Args: - miss_pages: The new or changed pages. + miss_pages: The dependency-changed pages. pages: All current page definitions (in compile order). manifest: The loaded manifest. - dirty_memo_paths: Memo output paths the rebuild rewrites. stale_owner_routes: Owners of route-owned memo records that changed. - dynamic_names_shift: Whether dynamic state names may have shifted - without a miss (see :func:`_with_dynamic_state_pages`). Returns: The expanded miss list, in ``pages`` order. """ - miss_pages = _with_memo_contributing_pages( - miss_pages, pages, manifest, dirty_memo_paths, stale_owner_routes - ) - if not _predict_dynamic_names: - miss_pages = _with_dynamic_state_pages( - miss_pages, pages, manifest, shifted=dynamic_names_shift - ) - return _with_module_siblings(miss_pages, pages, manifest) + routes = {page.route for page in miss_pages} | stale_owner_routes + miss_pages = [page for page in pages if page.route in routes] + if _predict_dynamic_names: + return miss_pages + return _with_dynamic_state_pages(miss_pages, pages, manifest) def _changed_dependency_files( @@ -1496,7 +1407,7 @@ def _module_source_file(module_name: str | None) -> str | None: def _memo_output_path(source_module: str | None) -> str | None: - """Resolve the mirrored output file path for a memo source module. + """Resolve the mirrored output file path for a user memo's source module. Args: source_module: The dotted module name that defined the memo. @@ -1515,6 +1426,20 @@ def _memo_output_path(source_module: str | None) -> str | None: return compiler_utils.get_memo_module_path(segments) +def _memo_file(tag: str) -> str: + """The content-addressed file of an auto memo. + + Args: + tag: The memo's export name, derived from its rendered body. + + Returns: + The absolute output path string. + """ + from reflex.compiler import utils as compiler_utils + + return compiler_utils.get_memo_component_path(tag) + + def _memo_state_entries( hasher: Callable[[str], page_cache.FileEntry | None], files: dict[str, page_cache.FileEntry], @@ -1629,24 +1554,22 @@ def _dirty_memo_paths( def _memo_defs_for_rewrite( - contributions: dict[tuple[str, str | None], Any], + memo_uses: dict[tuple[str, str | None], Any], dirty_paths: set[str], changed_files: set[str], ) -> list[Any]: - """Return the full definition set for the memo files being rewritten. - - Memo output is grouped one file per source module, so a rewrite must carry - every definition landing in that file: the recompiled pages' auto-memo - contributions plus every user ``@rx.memo`` definition whose output file is - being rewritten — because its recorded memo state changed (``dirty_paths``, - see :func:`_current_memo_state`) or because a recompiled page contributes - auto memos to it. The ``changed_files`` condition runs post-evaluation, so - it additionally covers memos the pre-evaluation state can't see: modules - imported inside a page function whose file (a recorded page dependency) - changed. + """Return the definitions of the memo files a rebuild writes. + + User memo output is grouped one file per source module, so a rewrite + must carry every definition landing in that file: every user ``@rx.memo`` + whose output file is being rewritten because its recorded memo state + changed (``dirty_paths``, see :func:`_dirty_memo_paths`), or whose module + file (a recorded page dependency) changed, which the pre-evaluation state + can't see for modules imported inside a page function. Auto memos are one + file each, so the recompiled pages' uses are written as they are. Args: - contributions: The recompiled pages' auto-memo contributions. + memo_uses: The auto memos the recompiled pages use. dirty_paths: Output paths whose stored memo state no longer matches. changed_files: The recorded dependency files whose content changed. @@ -1656,62 +1579,13 @@ def _memo_defs_for_rewrite( """ from reflex_base.components.memo import MEMOS - rewrite_paths = dirty_paths | { - path - for definition in contributions.values() - if (path := _memo_output_path(definition.source_module)) is not None - } user_memos = [ memo for (_, source_module), memo in MEMOS.items() - if _memo_output_path(source_module) in rewrite_paths + if _memo_output_path(source_module) in dirty_paths or _module_source_file(source_module) in changed_files ] - return [*user_memos, *contributions.values()] - - -def _with_memo_contributing_pages( - miss_pages: list[PageDefinition], - pages: Sequence[PageDefinition], - manifest: dict[str, Any], - dirty_paths: set[str], - stale_owner_routes: set[str], -) -> list[PageDefinition]: - """Add hit pages that must re-evaluate for a memo file rewrite. - - A dirty user-memo file is rewritten from scratch, so hit pages - contributing auto memos to the same file must be re-evaluated for their - contributions, or the rewrite would drop their exports. Likewise, a - route-owned memo entry whose dependency files changed needs its owner - page re-evaluated: only that page's evaluation re-registers the memo - definitions the rewrite compiles from. - - Args: - miss_pages: The dependency-changed pages. - pages: All current page definitions (in compile order). - manifest: The loaded manifest. - dirty_paths: Output paths whose stored memo state no longer matches. - stale_owner_routes: Routes owning stored memo entries with changed - dependency files. - - Returns: - The miss list extended with the affected hit pages. - """ - miss_routes = {page.route for page in miss_pages} - extra = [ - page - for page in pages - if page.route not in miss_routes - and ( - page.route in stale_owner_routes - or ( - manifest["pages"][page.route]["has_memos"] - and _memo_output_path(getattr(page, "_source_module", None)) - in dirty_paths - ) - ) - ] - return [*miss_pages, *extra] if extra else miss_pages + return [*user_memos, *memo_uses.values()] def _refresh_file_table( @@ -1733,7 +1607,7 @@ def _refresh_file_table( def _save_incremental_output( - path: str, code: str, outputs: dict[Path, str] | None + path: str | Path, code: str, outputs: dict[Path, str] | None ) -> None: """Stage output for plugin processing or write it for a direct rebuild. @@ -1751,10 +1625,130 @@ def _save_incremental_output( outputs[resolved] = code +def _app_root_change( + manifest: dict[str, Any], + pages: Sequence[PageDefinition], + compiled_pages: dict[str, PageContext], +) -> str | None: + """Explain how the recompiled pages changed the app root, or None. + + The app root is a function of the pages' merged app wraps (positioned by + the first page requiring each, valued by the last, like the full + compile's ``dict.update``). Rebuilding it from stored wraps would need + every wrap component to survive a round trip through the manifest, so + a changed root is left to a full compile instead; a page first gaining a + provider is rare. + + Args: + manifest: The loaded manifest. + pages: All current page definitions (in compile order). + compiled_pages: The recompiled pages' contexts, by route. + + Returns: + A human-readable reason, or None when the root on disk still matches. + """ + merged: dict[str, str] = {} + providers: dict[str, str] = {} + for page in pages: + if (page_ctx := compiled_pages.get(page.route)) is not None: + wraps = _record_app_wraps(page_ctx) + else: + wraps = manifest["pages"][page.route]["app_wraps"] + for key, digest in wraps.items(): + merged[key] = digest + providers[key] = page.route + stored = manifest["app_root"]["wraps"] + # Wraps of one priority nest in insertion order, so the order counts. + if list(merged.items()) == list(stored.items()): + return None + for key in sorted(merged.keys() | stored.keys()): + if merged.get(key) != stored.get(key): + return f"page {providers.get(key)!r} changed the app wrap {key!r}" + return "the app wraps changed their order" + + +def _new_bundled_libraries(manifest: dict[str, Any]) -> set[str]: + """The libraries this compile bundled that the recorded app root lacks. + + The bundled-library file lists the registry of a full compile, which the + recompiled pages' requests are only a part of; a library none of them + needs can stay listed, a new one needs the file rewritten. + + Args: + manifest: The loaded manifest. + + Returns: + The libraries missing from the recorded registry. + """ + live = RegistrationContext.ensure_context().bundled_libraries + return set(live) - set(manifest["app_root"]["libraries"]) + + +def _reconcile_memo_files( + manifest: dict[str, Any], + miss_routes: set[str], + miss_memo_keys: dict[str, list[str]], + emitted: Iterable[str], +) -> None: + """Update the memo file manifest and drop the auto memo files no page uses. + + Args: + manifest: The loaded manifest, with the hit pages' entries. + miss_routes: The recompiled routes. + miss_memo_keys: The memo keys of each recompiled route, now. + emitted: The memo files this rebuild wrote. + """ + from reflex.compiler import utils as compiler_utils + + used = set(manifest["app_root"]["memo_keys"]) + for route, entry in manifest["pages"].items(): + used.update( + miss_memo_keys[route] if route in miss_routes else entry["memo_keys"] + ) + dropped = { + _memo_file(tag) + for route in miss_routes + for tag in manifest["pages"][route]["memo_keys"] + if tag not in used + } + compiler_utils.reconcile_memo_manifest(emitted, dropped) + + +def _enable_radix_for_hit_pages( + compiler_plugins: Sequence[Plugin], manifest: dict[str, Any], miss_routes: set[str] +) -> None: + """Enable the implicit Radix Themes plugin when a reused page needs it. + + The implicit plugin enables itself while compiling a Radix component; a + page reused from disk is not compiled, so without this the stylesheet, + the theme and the package would drop out of a rebuild that re-evaluates + no Radix page. + + Args: + compiler_plugins: The resolved compiler plugins for this compile. + manifest: The loaded manifest. + miss_routes: The recompiled routes (compiled, so they enable it + themselves). + """ + from reflex_components_radix.plugin import RadixThemesPlugin + + for plugin in compiler_plugins: + if ( + isinstance(plugin, RadixThemesPlugin) + and not plugin.enabled + and any( + entry["radix"] + for route, entry in manifest["pages"].items() + if route not in miss_routes + ) + ): + plugin.enabled = True + + def try_incremental_rebuild( app: App, *, - compiler_plugins: Any, + compiler_plugins: Sequence[Plugin], prerender_routes: bool, root: Path | None = None, use_rich: bool = True, @@ -1764,19 +1758,20 @@ def try_incremental_rebuild( """Attempt a disk-cache-assisted partial rebuild; report whether it ran. Returns False (so the caller does a full compile) whenever anything is - unsafe to reuse: no/old manifest, a changed global input or a route change. - When a miss page turns out to have altered its app-wrap set or stateful - flag, the miss pages are already evaluated and rendered. A page is never - evaluated twice in one process (its locally defined states would register - under collision-suffixed names the backend does not use), so the context - holding those compiled pages is returned instead of False, for the full - compile to adopt (see ``CompileContext.absorb``). + unsafe to reuse: no/old manifest, a changed global input, a changed page + set or registration. When a miss page turns out to have altered the app + root's inputs or its stateful flag, the miss pages are already evaluated + and rendered. A page is never evaluated twice in one process (its locally + defined states would register under collision-suffixed names the backend + does not use), so the context holding those compiled pages is returned + instead of False, for the full compile to adopt (see + ``CompileContext.absorb``). User-memo output files are reconciled against the manifest's stored memo - record (see :func:`_current_memo_state`): files whose record differs are - rewritten, with the affected memo-contributing hit pages re-evaluated so - their exports survive the rewrite. Files of deleted memo modules are left - orphaned (nothing imports them; the next full compile prunes them). + record (see :func:`_memo_state_entries`): files whose record differs are + rewritten. Auto memos are content-addressed, one file each, so the + recompiled pages write theirs without touching any other page's; files + no page uses any more are dropped. The ``assets`` copy is excluded from dependency tracking and always re-run (cheap, idempotent). The contexts file is rewritten only when a stateful @@ -1786,7 +1781,9 @@ def try_incremental_rebuild( On success, reports (at info level) how many pages were recompiled vs reused and, while recompiling, shows a progress bar over the changed pages so a hot - reload makes the incremental work visible. + reload makes the incremental work visible. The refreshed manifest is the + last output: with ``outputs`` it is staged after every other file, so a + manifest on disk never vouches for files that were not written. Args: app: The app being compiled. @@ -1811,32 +1808,20 @@ def try_incremental_rebuild( return False pages = list(app._unevaluated_pages.values()) - routes = {p.route for p in pages} if registrations is None: registrations = registration_digests(pages) validator = page_cache.FileValidator(manifest["files"]) changed_hint = _with_hinted_slices(_changed_hint, manifest) - if ( - reason := globals_mismatch(manifest, validator=validator, root=root) - ) is not None: + if (reason := globals_mismatch(manifest, validator=validator, root=root)) or ( + reason := _pages_mismatch(pages, manifest, registrations) + ): _log_fallback(reason) return False resolved_root = (root or Path.cwd()).resolve() - # Removed routes leave the manifest; added ones have no entry, so they miss. - removed_entries = { - route: manifest["pages"].pop(route) - for route in [route for route in manifest["pages"] if route not in routes] - } - route_set_changed = bool(removed_entries) or not routes <= set(manifest["pages"]) # The states importing the app registered (no page has evaluated yet). - page_states = { - name - for entry in (*manifest["pages"].values(), *removed_entries.values()) - if (state_slice := entry["state_slice"]) - for name in state_slice["schema"] - } + page_states = _hit_page_states(manifest, set()) live_module_states = [ name for name in RegistrationContext.ensure_context().base_states @@ -1864,17 +1849,15 @@ def try_incremental_rebuild( # They take names before any page, so every page's could shift. _log_fallback("the app's import-time dynamic states changed") return False - miss_pages = partition_pages( - pages, manifest, validator, changed_hint, registrations - ) - # Memo files are decided salsa-style: derive the files the current memo - # registry demands and rewrite the ones whose stored record differs. This - # catches what changed-file diffing structurally cannot: a memo module the - # previous compile never saw (its file is in no recorded dependency set). - # Only import-owned memos are comparable here (pre-evaluation); - # route-owned entries are handled by owner status: a changed dependency - # pulls the owner page into the miss set, and its re-evaluation feeds the - # post-evaluation comparison below. + miss_pages = partition_pages(pages, manifest, validator, changed_hint) + # User memo files are decided salsa-style: derive the files the current + # memo registry demands and rewrite the ones whose stored record differs. + # This catches what changed-file diffing structurally cannot: a memo + # module the previous compile never saw (its file is in no recorded + # dependency set). Only import-owned memos are comparable here + # (pre-evaluation); route-owned entries are handled by owner status: a + # changed dependency pulls the owner page into the miss set, and its + # re-evaluation feeds the post-evaluation comparison below. memo_hasher = page_cache.make_hasher() memo_files_table: dict[str, page_cache.FileEntry] = {} memo_state = _memo_state_entries( @@ -1884,40 +1867,17 @@ def try_incremental_rebuild( dirty_memo_paths = _dirty_memo_paths( memo_state, stored_memo_files, validator, changed_hint ) - # A removed page's auto memos must leave its module's memo file. - removed_memo_paths = { - path for entry in removed_entries.values() if (path := entry["memo_path"]) - } - dirty_memo_paths |= removed_memo_paths stale_owner_routes = { owner for entry in stored_memo_files.values() if (owner := entry["owner_route"]) is not None - and owner in routes + and owner in manifest["pages"] and _memo_deps_changed(entry["deps"], validator, changed_hint) } - order_changed = _dynamic_page_order_changed(manifest, pages) - if order_changed and _predict_dynamic_names: - msg = "the order of the pages defining dynamic states changed" - raise DynamicStatePredictionError(msg) changed_files: set[str] = set() - if ( - miss_pages - or dirty_memo_paths - or stale_owner_routes - or removed_entries - or order_changed - ): + if miss_pages or dirty_memo_paths or stale_owner_routes: changed_files = _changed_dependency_files(manifest, validator, changed_hint) - miss_pages = _expand_miss_pages( - miss_pages, - pages, - manifest, - dirty_memo_paths, - stale_owner_routes, - dynamic_names_shift=order_changed - or any(e["dynamic_claims"] for e in removed_entries.values()), - ) + miss_pages = _expand_miss_pages(miss_pages, pages, manifest, stale_owner_routes) memo_note = ( f"; rewriting {len(dirty_memo_paths)} memo file(s)" if dirty_memo_paths @@ -1931,14 +1891,13 @@ def try_incremental_rebuild( else: logger.info(f"Compile cache: reusing all {len(pages)} pages from disk") miss_routes = {p.route for p in miss_pages} + _enable_radix_for_hit_pages(compiler_plugins, manifest, miss_routes) from reflex.compiler import compiler - from reflex.compiler import utils as compiler_utils - bundle_base: list[str] = [] - if miss_pages or dirty_memo_paths or route_set_changed: + if miss_pages or dirty_memo_paths: # Match the full compile's clean bundling/memo state before compiling. - bundle_base = compiler._reset_compile_registries(compiler_plugins) + compiler._reset_compile_registries(compiler_plugins) # Recompile only the source-changed pages. miss_ctx = None @@ -1964,7 +1923,6 @@ def try_incremental_rebuild( hooks=CompilerHooks( plugins=default_page_plugins(style=app.style, plugins=page_plugins) ), - bundle_base=bundle_base, ) # Progress over the changed pages (evaluate + render each), so a hot # reload shows how much is being recompiled. @@ -1983,6 +1941,7 @@ def try_incremental_rebuild( progress.stop() if set(page_cache._loaded_first_party_modules(resolved_root)) != modules_before: page_cache.clear_import_graph() + _disown_hit_page_states(miss_ctx, manifest, miss_routes) for page in miss_pages: page_ctx = miss_ctx.compiled_pages.get(page.route) if ( @@ -1993,27 +1952,32 @@ def try_incremental_rebuild( _fall_back_after_evaluation(f"page {page.route!r} produced no output") return False - if _predict_dynamic_names and ( - route := _shifting_dynamic_claims( - manifest, miss_ctx, miss_routes, removed_entries - ) + if ( + _predict_dynamic_names + and miss_ctx is not None + and (route := _shifting_dynamic_claims(manifest, miss_ctx, miss_routes)) ): _fall_back_after_evaluation( f"page {route!r} changed dynamic state names other pages share" ) - if route := _route_dropping_a_shared_state(manifest, miss_routes, removed_entries): + if route := _route_dropping_a_shared_state(manifest, miss_routes): _fall_back_after_evaluation( f"page {route!r} no longer registers a state other pages may use" ) return miss_ctx if miss_ctx is not None else False + if miss_ctx is not None and ( + reason := _app_root_change(manifest, pages, miss_ctx.compiled_pages) + ): + _fall_back_after_evaluation(reason) + return miss_ctx # Write changed pages + their memo files; reuse everything else on disk. install_imports = _deserialize_imports(manifest["all_imports"]) eval_memo_state: dict[str, dict[str, Any]] = {} - # Outputs of removed pages, deleted once the rebuild succeeded. - stale_outputs = [compiler_utils.get_page_path(route) for route in removed_entries] + emitted_memo_files: list[str] = [] + miss_memo_keys: dict[str, list[str]] = {} if miss_ctx is not None or dirty_memo_paths: - memo_contributions: dict[tuple[str, str | None], Any] = {} + memo_uses: dict[tuple[str, str | None], Any] = {} miss_imports = [] if miss_ctx is not None: for page in miss_pages: @@ -2027,7 +1991,8 @@ def try_incremental_rebuild( ) return False _save_incremental_output(output_path, output_code, outputs) - memo_contributions.update(page_ctx.memo_contributions) + memo_uses.update(page_ctx.memo_uses) + miss_memo_keys[page.route] = _memo_keys(page_ctx) miss_imports.append(page_ctx.frontend_imports) # Post-evaluation pass: the miss pages just evaluated, so memos # they own (registered during their evaluation, e.g. from modules @@ -2043,20 +2008,12 @@ def try_incremental_rebuild( dirty_memo_paths = dirty_memo_paths | _dirty_memo_paths( eval_memo_state, stored_memo_files, validator, changed_hint ) - # Memo output files are grouped per source module, so compile them once - # with the complete definition set (all recompiled pages' contributions - # plus the user memos landing in a file being rewritten). memo_files, memo_imports = compiler.compile_memo_components( - _memo_defs_for_rewrite(memo_contributions, dirty_memo_paths, changed_files) + _memo_defs_for_rewrite(memo_uses, dirty_memo_paths, changed_files) ) for mpath, mcode in memo_files: _save_incremental_output(mpath, mcode, outputs) - emitted = {mpath for mpath, _ in memo_files} - stale_outputs.extend( - path - for path in removed_memo_paths - if path not in emitted and path not in manifest["memo_files"] - ) + emitted_memo_files.append(mpath) # Merge once: re-merging the app-wide set per page re-walks its ~100k # entries each time. install_imports = merge_imports(install_imports, *miss_imports, memo_imports) @@ -2093,26 +2050,16 @@ def try_incremental_rebuild( # from the on-disk contexts file, so compare those first: a content-only # edit leaves them identical and the contexts file is reused untouched. contexts_snapshot: compiler_utils.ContextsSnapshot | None = None - state_libraries: dict[str, list[str]] = manifest["state_libraries"] - if ( - route_set_changed - or module_states_changed - or ( - miss_ctx is not None - and any( - route in miss_ctx.stateful_routes - or manifest["pages"].get(route, {}).get("is_stateful") - for route in miss_routes - ) + if module_states_changed or ( + miss_ctx is not None + and any( + route in miss_ctx.stateful_routes or manifest["pages"][route]["is_stateful"] + for route in miss_routes ) ): contexts_snapshot = _contexts_snapshot(app) - # A route change can drop slices or add dynamic route arguments to the - # root state, so it always rebuilds the contexts. changed_route = ( - "the route set" - if route_set_changed - else "the app's states" + "the app's states" if module_states_changed else _changed_state_config_route( manifest, miss_ctx, contexts_snapshot, miss_routes @@ -2154,36 +2101,15 @@ def try_incremental_rebuild( ) _save_incremental_output(context_path, context_code, outputs) install_imports = merge_imports(install_imports, state_imports) - if contexts_snapshot is not None and merged_snapshot is not None: - live_libraries = _state_libraries(contexts_snapshot[0]) - state_libraries = { - name: libraries - for name in merged_snapshot[0] - if ( - libraries := live_libraries.get(name) - if name in contexts_snapshot[0] - else state_libraries.get(name) - ) - } - app_root_inputs = manifest["app_root"] - if miss_ctx is not None or route_set_changed: - try: - refreshed = _refresh_app_root( - app, - manifest, - pages, - miss_ctx.compiled_pages if miss_ctx is not None else {}, - bundle_base, - state_libraries, - outputs, - ) - except _AppRootUnavailableError as err: - _fall_back_after_evaluation(str(err)) - return miss_ctx if miss_ctx is not None else False - if refreshed is not None: - app_root_inputs, root_imports = refreshed - install_imports = merge_imports(install_imports, root_imports) + # The bundled-library file and the app root reference every bundled + # library; a library first requested by this rebuild (a recompiled page, + # a state default) needs both rewritten, which only a full compile does. + if new_libraries := _new_bundled_libraries(manifest): + _fall_back_after_evaluation( + f"new bundled libraries: {format_path_list(new_libraries)}" + ) + return miss_ctx if miss_ctx is not None else False # The assets copy is cheap, idempotent, and excluded from dependency # tracking entirely, so it is always re-run. @@ -2203,6 +2129,12 @@ def try_incremental_rebuild( frontend_skeleton.update_entry_client() frontend_skeleton.initialize_vite_config() + # The route table the production static server reads. + _save_incremental_output( + constants.Dirs.ROUTES_MANIFEST, json.dumps(app._page_routes), outputs + ) + _reconcile_memo_files(manifest, miss_routes, miss_memo_keys, emitted_memo_files) + # The next manifest merges three memo-record sources, mirroring salsa's # reuse of queries that did not re-run: route-owned entries whose owner # page hit carry forward unchanged (their page never re-evaluated here), @@ -2214,14 +2146,13 @@ def try_incremental_rebuild( for path, entry in stored_memo_files.items() if entry["owner_route"] is not None and entry["owner_route"] not in miss_routes - and entry["owner_route"] in routes }, **memo_state, **eval_memo_state, } - # Refresh the manifest for the next process. - _update_manifest_for_misses( + # Refresh the manifest for the next process; it is the last output. + if _update_manifest_for_misses( manifest, miss_ctx, miss_pages, @@ -2230,10 +2161,7 @@ def try_incremental_rebuild( contexts_snapshot=contexts_snapshot, memo_state=final_memo_state, memo_file_entries=memo_files_table, - app_root_inputs=app_root_inputs, - state_libraries=state_libraries, registrations=registrations, - page_order=[page.route for page in pages], module_states=( _record_module_states( live_module_states, @@ -2244,16 +2172,9 @@ def try_incremental_rebuild( if module_states_changed else recorded_module_states ), - route_set_changed=route_set_changed, validator=validator, - ) - - # The route table the production static server reads. - _save_incremental_output( - constants.Dirs.ROUTES_MANIFEST, json.dumps(app._page_routes), outputs - ) - for path in stale_outputs: - Path(path).unlink(missing_ok=True) + ): + _save_incremental_output(_manifest_path(), _manifest_text(manifest), outputs) counts: dict[str, int] = {} for page in pages: @@ -2264,150 +2185,6 @@ def try_incremental_rebuild( return True -class _AppRootUnavailableError(Exception): - """The app root cannot be rebuilt from the recorded inputs.""" - - -def _refresh_app_root( - app: App, - manifest: dict[str, Any], - pages: Sequence[PageDefinition], - compiled_pages: dict[str, PageContext], - bundle_base: Sequence[str], - state_libraries: dict[str, list[str]], - outputs: dict[Path, str] | None, -) -> tuple[dict[str, Any], ParsedImportDict] | None: - """Rebuild the app root and bundled-library file when their inputs changed. - - The app root is a function of the pages' merged app wraps (positioned by - the first page requiring each, valued by the last, like the full compile's - ``dict.update``) and of the bundled libraries, rebuilt in full-compile - order from each page's and state's recorded requests. When neither - changed, the root on disk stays. Otherwise it is rebuilt from the - recompiled pages' live wraps and the recorded wraps of the other pages, - without evaluating them. - - Args: - app: The app being compiled. - manifest: The loaded manifest; its wrap table gains the live wraps. - pages: All current page definitions (in compile order). - compiled_pages: The recompiled pages' contexts, by route. - bundle_base: The bundled libraries registered after this rebuild's - registry reset. - state_libraries: The libraries each state's initial values request. - outputs: Output mapping supplied by the main compiler, if any. - - Returns: - The new recorded app-root inputs and the imports the root needs, or - None when the root is unchanged. - - Raises: - _AppRootUnavailableError: When the recorded inputs cannot rebuild it. - """ - from reflex_base.components.memo import MEMOS - - from reflex.compiler import compiler - from reflex.compiler import utils as compiler_utils - - stored = manifest["app_root"] - wrap_table = manifest["wrap_table"] - merged: dict[str, str] = {} - providers: dict[str, str] = {} - page_requests = [] - memo_paths: set[str] = set(manifest["memo_files"]) - for page in pages: - if (page_ctx := compiled_pages.get(page.route)) is not None: - wraps = _record_app_wraps(page_ctx, wrap_table) - page_requests.append([ - page_ctx.evaluate_bundle_requests, - page_ctx.render_bundle_requests, - ]) - has_memos = bool(page_ctx.memo_contributions) - else: - entry = manifest["pages"][page.route] - wraps = entry["app_wraps"] - page_requests.append(entry["bundle_requests"]) - has_memos = entry["has_memos"] - if has_memos and ( - path := _memo_output_path(getattr(page, "_source_module", None)) - ): - memo_paths.add(path) - for key_str, digest in wraps.items(): - merged[key_str] = digest - providers[key_str] = page.route - libraries = _reconstruct_libraries( - bundle_base, page_requests, stored["root_requests"], state_libraries - ) - # Wraps of one priority nest in insertion order, so the order counts. - if ( - list(merged.items()) == list(stored["wraps"].items()) - and libraries == stored["libraries"] - ): - return None - if not stored["reconstructible"]: - msg = "the app root's bundled libraries cannot be rebuilt" - raise _AppRootUnavailableError(msg) - - components = {} - for key_str, digest in merged.items(): - key = _wrap_key(key_str) - if (page_ctx := compiled_pages.get(providers[key_str])) is not None: - components[key] = page_ctx.app_wrap_components[key] - elif (data := wrap_table.get(digest)) is not None: - components[key] = pickle.loads(base64.b64decode(data)) - else: - msg = f"page {providers[key_str]!r} requires an app wrap that cannot be stored" - raise _AppRootUnavailableError(msg) - logger.info( - "Compile cache: the app wraps or bundled libraries changed; rebuilding the app root" - ) - memos_before = set(MEMOS) - root_ctx = CompileContext(app=app, pages=()) - with root_ctx, record_bundle_requests() as root_requests: - app_root, hydrate_fallback_export = compiler._build_app_root( - app, components, root_ctx - ) - root_memos = [ - *(memo for key, memo in MEMOS.items() if key not in memos_before), - *root_ctx.auto_memo_components.values(), - ] - root_memo_paths = { - path - for memo in root_memos - if (path := _memo_output_path(memo.source_module)) is not None - } - if root_memo_paths & memo_paths: - msg = "the app root's memos share a file with page memos" - raise _AppRootUnavailableError(msg) - memo_files, memo_imports = compiler.compile_memo_components([ - *( - memo - for key, memo in MEMOS.items() - if key in memos_before and _memo_output_path(key[1]) in root_memo_paths - ), - *root_memos, - ]) - libraries = _reconstruct_libraries( - bundle_base, page_requests, root_requests, state_libraries - ) - RegistrationContext.ensure_context().bundled_libraries[:] = libraries - for path, code in ( - *memo_files, - compiler.compile_app_root(app_root, hydrate_fallback_export), - compiler_utils._compile_bundled_libraries(), - ): - _save_incremental_output(path, code, outputs) - return ( - { - "wraps": merged, - "libraries": libraries, - "root_requests": root_requests, - "reconstructible": True, - }, - merge_imports(app_root._get_all_imports(), memo_imports), - ) - - def _update_manifest_for_misses( manifest: dict[str, Any], miss_ctx: CompileContext | None, @@ -2418,22 +2195,18 @@ def _update_manifest_for_misses( contexts_snapshot: compiler_utils.ContextsSnapshot | None = None, memo_state: dict[str, dict[str, Any]], memo_file_entries: dict[str, page_cache.FileEntry], - app_root_inputs: dict[str, Any], - state_libraries: dict[str, list[str]], registrations: dict[str, str], module_states: dict[str, list[str]], - page_order: Sequence[str] = (), - route_set_changed: bool = False, validator: page_cache.FileValidator | None = None, -) -> None: - """Update the on-disk manifest entries for the recompiled pages. +) -> bool: + """Update the manifest entries for the recompiled pages. Also refreshes the stat keys of files that were touched without changing - (so they are not re-hashed on every later reload) and persists the - import-name parse cache. Rewritten only when something changed. + (so they are not re-hashed on every later reload). The caller persists + the manifest when something changed. Args: - manifest: The loaded manifest (mutated and rewritten). + manifest: The loaded manifest (mutated). miss_ctx: The compile context of the recompiled pages, if any. miss_pages: The recompiled page definitions. all_imports: The complete frontend import set after this rebuild. @@ -2444,87 +2217,56 @@ def _update_manifest_for_misses( :func:`_memo_state_entries`). memo_file_entries: File-table entries for the memo records' closures, withheld until validation finished. - app_root_inputs: The app root's recorded inputs after this rebuild. - state_libraries: The libraries each state's initial values request. registrations: Each route's registration digest. module_states: The import-time states and their source files. - page_order: The routes in compile order, which the manifest keeps. - route_set_changed: Whether pages were added or removed. validator: The validator used for this rebuild, for its refreshed entries. + + Returns: + Whether the manifest changed and must be written. """ - try: - dirty = route_set_changed or ( - validator is not None and _refresh_file_table(manifest, validator) - ) - # An entry module whose edit only changed page registrations stays the - # recorded one; record its new content so the next rebuild skips it. - if ( - validator is not None - and (entry_path := manifest["entry"]["path"]) is not None - and validator.changed(entry_path) - and (entry := page_cache.file_entry(entry_path)) is not None - ): - manifest["files"][entry_path] = list(entry) - dirty = True - if manifest["memo_files"] != memo_state: - manifest["memo_files"] = memo_state - manifest["files"].update(memo_file_entries) - dirty = True - if miss_ctx is not None and miss_pages: - state_index, _ = page_cache.state_dependency_index(root) - hasher = page_cache.make_hasher() - for page in miss_pages: - page_ctx = miss_ctx.compiled_pages[page.route] - defined_states = miss_ctx.stateful_routes.get(page.route) - manifest["pages"][page.route] = _manifest_page_entry( - page_ctx, - page, - state_index, - hasher, - manifest["files"], - defined_states=defined_states, - registration=registrations[page.route], - wrap_table=manifest["wrap_table"], - state_slice=( - _state_slice(defined_states, *contexts_snapshot) - if defined_states and contexts_snapshot is not None - else None - ), - root=root, - ) - dirty = True - if page_order and list(manifest["pages"]) != list(page_order): - manifest["pages"] = { - route: manifest["pages"][route] for route in page_order - } - dirty = True - if ( - manifest["app_root"] != app_root_inputs - # Dict equality ignores the wrap order, which the root keeps. - or list(manifest["app_root"]["wraps"]) != list(app_root_inputs["wraps"]) - or manifest["state_libraries"] != state_libraries - or manifest["module_states"] != module_states - ): - manifest["app_root"] = app_root_inputs - manifest["state_libraries"] = state_libraries - manifest["module_states"] = module_states - dirty = True - if dirty: - # Keep only the stored wraps a page or the app root still uses. - used = set(app_root_inputs["wraps"].values()).union( - *(entry["app_wraps"].values() for entry in manifest["pages"].values()) + dirty = validator is not None and _refresh_file_table(manifest, validator) + # An entry module whose edit only changed page registrations stays the + # recorded one; record its new content so the next rebuild skips it. + if ( + validator is not None + and (entry_path := manifest["entry"]["path"]) is not None + and validator.changed(entry_path) + and (entry := page_cache.file_entry(entry_path)) is not None + ): + manifest["files"][entry_path] = list(entry) + dirty = True + if manifest["memo_files"] != memo_state: + manifest["memo_files"] = memo_state + manifest["files"].update(memo_file_entries) + dirty = True + if miss_ctx is not None and miss_pages: + state_index, _ = page_cache.state_dependency_index(root) + hasher = page_cache.make_hasher() + for page in miss_pages: + page_ctx = miss_ctx.compiled_pages[page.route] + defined_states = miss_ctx.stateful_routes.get(page.route) + manifest["pages"][page.route] = _manifest_page_entry( + page_ctx, + page, + state_index, + hasher, + manifest["files"], + defined_states=defined_states, + registration=registrations[page.route], + state_slice=( + _state_slice(defined_states, *contexts_snapshot) + if defined_states and contexts_snapshot is not None + else None + ), + root=root, ) - manifest["wrap_table"] = { - digest: data - for digest, data in manifest["wrap_table"].items() - if digest in used - } - # A memo-only rewrite can add packages with no page recompiled. - if (imports := _serialize_imports(all_imports)) != manifest["all_imports"]: - manifest["all_imports"] = imports - dirty = True - if dirty: - _write(manifest) - except Exception as exc: # best-effort - logger.warning(f"Compile cache: could not refresh manifest ({exc!r}).") + dirty = True + if manifest["module_states"] != module_states: + manifest["module_states"] = module_states + dirty = True + # A memo-only rewrite can add packages with no page recompiled. + if (imports := _serialize_imports(all_imports)) != manifest["all_imports"]: + manifest["all_imports"] = imports + dirty = True + return dirty diff --git a/reflex/compiler/page_cache.py b/reflex/compiler/page_cache.py index cb0632c904f..9b8f4b5f9bc 100644 --- a/reflex/compiler/page_cache.py +++ b/reflex/compiler/page_cache.py @@ -1080,6 +1080,30 @@ def warm_module_file_cache(root: Path | None = None) -> None: _loaded_first_party_modules(roots[0]) +def component_tree_classes(root_component: object) -> set[type]: + """Collect the class of every component in a rendered tree. + + Args: + root_component: The page's root component (its rendered tree). + + Returns: + The distinct component classes. + """ + classes: set[type] = set() + seen: set[int] = set() + stack: list[object] = [root_component] + while stack: + comp = stack.pop() + if id(comp) in seen: + continue + seen.add(id(comp)) + classes.add(type(comp)) + children = getattr(comp, "children", None) + if children: + stack.extend(children) + return classes + + def component_module_files( root_component: object, root: Path | None = None ) -> set[Path]: @@ -1097,20 +1121,11 @@ def component_module_files( The set of resolved first-party module files the tree depends on. """ roots = first_party_roots(root) - files: set[Path] = set() - seen: set[int] = set() - stack: list[object] = [root_component] - while stack: - comp = stack.pop() - if id(comp) in seen: - continue - seen.add(id(comp)) - if (rf := _class_module_file(type(comp), roots)) is not None: - files.add(rf) - children = getattr(comp, "children", None) - if children: - stack.extend(children) - return files + return { + rf + for cls in component_tree_classes(root_component) + if (rf := _class_module_file(cls, roots)) is not None + } #: Cache of the first-party import graph, keyed by project root. @@ -1665,15 +1680,13 @@ def used_state_files( Stateful subtrees are auto-memoized into separate components, so a page's own ``output_code`` may not name the state it uses; the state lives in the - memo components it owns (its ``memo_contributions``). Each stateful memo - is owned by exactly one page, which regenerates it whenever it recompiles, - so scanning ``output_code`` plus the page's own memo components captures the - full dependency set. If a memo can't be introspected, depend on every fine - state file. + memo components it uses (its ``memo_uses``), so scanning ``output_code`` + plus those captures the full dependency set. If a memo can't be + introspected, depend on every fine state file. Args: output_code: The page's compiled JS. - memo_components: The page's own memo component subtrees (renderable). + memo_components: The memo component subtrees the page uses (renderable). id_to_file: The state-context identifier -> file index. Returns: @@ -1721,7 +1734,7 @@ def page_dependency_files( files |= component_module_files(page_ctx.root_component, root) files |= used_state_files( page_ctx.output_code or "", - [m.component for m in page_ctx.memo_contributions.values()], + [m.component for m in page_ctx.memo_uses.values()], state_index, ) deps = {str(f) for f in files} diff --git a/reflex/compiler/plugins/memoize.py b/reflex/compiler/plugins/memoize.py index 86b6b53a3e4..85243972621 100644 --- a/reflex/compiler/plugins/memoize.py +++ b/reflex/compiler/plugins/memoize.py @@ -385,16 +385,17 @@ def _build_wrapper( # tag is derived from the rendered memo body inside # ``create_passthrough_component_memo`` after the ``{children}`` hole # is substituted, so passthrough wrappers that differ only in their - # children collapse to a single definition. - wrapper_factory, definition = create_passthrough_component_memo( - comp, source_module=page_context.source_module - ) + # children collapse to a single definition. An auto memo carries no + # source module: it is content-addressed by its tag and compiles to + # its own file, so identical subtrees anywhere in the app share one + # definition and a page's memo files never hold another page's memos + # (the compile cache rewrites a page's memo files without touching + # the pages it reuses). + wrapper_factory, definition = create_passthrough_component_memo(comp) tag = definition.export_name compile_context.memoize_wrappers[tag] = None - # Key by (tag, source_module) so identical-rendering subtrees from - # different user modules each get their own entry and emit into the - # right mirrored memo file. - compile_context.auto_memo_components[tag, definition.source_module] = definition + compile_context.auto_memo_components[tag, None] = definition + page_context.memo_uses[tag, None] = definition wrapper = wrapper_factory() # The wrapper has no structural children at the page level, but parents diff --git a/reflex/compiler/utils.py b/reflex/compiler/utils.py index dcc349f9cbd..8e27998dd4f 100644 --- a/reflex/compiler/utils.py +++ b/reflex/compiler/utils.py @@ -1021,6 +1021,18 @@ def get_memo_components_dir() -> str: return str(get_web_dir() / constants.Dirs.COMPONENTS_PATH) +def get_memo_component_path(name: str) -> str: + """Get the on-disk path of an un-mirrored per-memo module. + + Args: + name: The memo's export name. + + Returns: + The absolute path of the memo's ``.jsx`` file. + """ + return str(Path(get_memo_components_dir()) / f"{name}{constants.Ext.JSX}") + + def get_memo_module_path(segments: tuple[str, ...]) -> str: """Get the on-disk path for a memo module mirrored from a Python module. @@ -1139,30 +1151,29 @@ def _write_memo_manifest(web_dir: Path, relative_paths: set[str]) -> None: raise -def prune_stale_memo_files(emitted_paths: Iterable[str | Path]) -> None: - """Delete memo files written previously that this compile no longer emits. - - Only paths that appear in the previous manifest are considered for - deletion — never a fresh filesystem walk — so files this code did not - emit are never touched. Empty parent directories created by mirrored - output are removed up to (but not including) the ``.web`` root. +def _memo_relative_paths(web_dir: Path, paths: Iterable[str | Path]) -> set[str]: + """Express memo file paths as the memo manifest records them. Args: - emitted_paths: Paths the current compile produced for the memo - pipeline, as built by joining :func:`get_web_dir` (so they share - its prefix — relative by default, absolute when overridden). + web_dir: The project's ``.web`` directory. + paths: Memo file paths, as built by joining :func:`get_web_dir` (so + they share its prefix — relative by default, absolute when + overridden). + + Returns: + The paths relative to ``web_dir``, with forward slashes. """ - web_dir = get_web_dir() + return {str(Path(path).relative_to(web_dir)).replace(os.sep, "/") for path in paths} - # Emitted paths are built by joining ``web_dir`` (see Args), so each is - # always under it — no need to guard ``relative_to``. - emitted_relative = { - str(Path(path).relative_to(web_dir)).replace(os.sep, "/") - for path in emitted_paths - } - previous = _read_memo_manifest(web_dir) - for relative in previous - emitted_relative: +def _delete_memo_files(web_dir: Path, relative_paths: Iterable[str]) -> None: + """Delete memo files, and the empty mirrored directories they leave. + + Args: + web_dir: The project's ``.web`` directory. + relative_paths: The files to delete, relative to ``web_dir``. + """ + for relative in relative_paths: target = web_dir / relative if target.is_file(): target.unlink() @@ -1174,10 +1185,50 @@ def prune_stale_memo_files(emitted_paths: Iterable[str | Path]) -> None: break parent = parent.parent + +def prune_stale_memo_files(emitted_paths: Iterable[str | Path]) -> None: + """Delete memo files written previously that this compile no longer emits. + + Only paths that appear in the previous manifest are considered for + deletion — never a fresh filesystem walk — so files this code did not + emit are never touched. Empty parent directories created by mirrored + output are removed up to (but not including) the ``.web`` root. + + Args: + emitted_paths: Paths the current compile produced for the memo + pipeline (see :func:`_memo_relative_paths`). + """ + web_dir = get_web_dir() + emitted_relative = _memo_relative_paths(web_dir, emitted_paths) + previous = _read_memo_manifest(web_dir) + _delete_memo_files(web_dir, previous - emitted_relative) if emitted_relative != previous: _write_memo_manifest(web_dir, emitted_relative) +def reconcile_memo_manifest( + emitted_paths: Iterable[str | Path], dropped_paths: Iterable[str | Path] +) -> None: + """Record the memo files an incremental compile wrote and dropped. + + The files the compile did not touch stay recorded, so a later full + compile still prunes them when it stops emitting them. Only files the + previous manifest recorded are deleted. + + Args: + emitted_paths: The memo files this compile wrote. + dropped_paths: The memo files no longer used by any page. + """ + web_dir = get_web_dir() + emitted_relative = _memo_relative_paths(web_dir, emitted_paths) + dropped_relative = _memo_relative_paths(web_dir, dropped_paths) - emitted_relative + previous = _read_memo_manifest(web_dir) + _delete_memo_files(web_dir, dropped_relative & previous) + current = (previous | emitted_relative) - dropped_relative + if current != previous: + _write_memo_manifest(web_dir, current) + + def empty_dir(path: str | Path, keep_files: list[str] | None = None): """Remove all files and folders in a directory except for the keep_files. diff --git a/reflex/utils/compile_daemon.py b/reflex/utils/compile_daemon.py index b6e6f300ef9..f4fd1db617f 100644 --- a/reflex/utils/compile_daemon.py +++ b/reflex/utils/compile_daemon.py @@ -601,19 +601,26 @@ def _reset_first_party(roots: list[Path]) -> None: Args: roots: The resolved reload roots whose modules are first-party. """ - for name in _first_party_module_names(roots): + purged = _first_party_module_names(roots) + for name in purged: sys.modules.pop(name, None) # The import-system finder caches were inherited from the warm parent via # fork and are now stale (they reference the purged modules); without this a # re-import can resolve a stale spec for a since-changed module. importlib.invalidate_caches() + from reflex_base.components.memo import MEMOS from reflex_base.registry import RegistrationContext import reflex.istate.dynamic as istate_dynamic from reflex.compiler import page_cache from reflex.state import BaseState, _frontend_schema, all_base_state_classes + # A purged module re-registers its memos when it re-imports; a renamed or + # deleted memo must not survive from the parent (a name collision with the + # old registration, or a memo file the app no longer defines). + for key in [key for key in MEMOS if key[1] in purged]: + del MEMOS[key] ctx = RegistrationContext.ensure_context() kept = [ cls @@ -644,8 +651,10 @@ def _reset_first_party(roots: list[Path]) -> None: object.__setattr__(ctx, "_app", None) # The import graph caches each module's parsed import edges; a changed file # may import differently now, so drop it to force a re-parse. Cross-compile - # page reuse comes from the on-disk manifest. + # page reuse comes from the on-disk manifest. The reads attributed to the + # purged modules are re-recorded as they re-execute. page_cache.clear_import_graph() + page_cache.forget_module_reads() _reset_model_metadata() diff --git a/tests/units/compiler/test_compiler.py b/tests/units/compiler/test_compiler.py index c1a5783a427..f5e571aff28 100644 --- a/tests/units/compiler/test_compiler.py +++ b/tests/units/compiler/test_compiler.py @@ -1975,6 +1975,83 @@ def rebuild(*args, outputs, **kwargs): assert (tmp_path / "page.txt").read_text() == ("new!" if fresh_page else "base!") +@pytest.mark.parametrize("write_fails", [False, True]) +def test_cached_compile_writes_the_manifest_last( + tmp_path, monkeypatch, mocker, write_fails +): + """The refreshed manifest reaches disk only after every other output. + + A manifest on disk vouches for the files it describes; it must never be + there while they are not (a kill between the two leaves pages stale). + """ + from reflex.compiler import disk_cache + + monkeypatch.setenv("REFLEX_COMPILE_CACHE", "1") + monkeypatch.setenv("REFLEX_WEB_WORKDIR", str(tmp_path)) + mocker.patch("reflex.app.reload_config") + app = rx.App() + app.add_page(lambda: rx.el.div("page"), route="index") + mocker.patch.object( + compiler, "get_config", return_value=rx.Config(app_name="testing", plugins=[]) + ) + mocker.patch.object(app, "_should_compile", return_value=True) + # The attempt runs in this process, where the writes can be observed. + from reflex.utils import compile_daemon + + mocker.patch.object( + compile_daemon, "compiled_in_predicting_child", return_value=False + ) + manifest_path = disk_cache._manifest_path().absolute() + manifest_path.write_text(json.dumps({"schema": 0, "plugin_sources": {}})) + page = tmp_path / "page.txt" + written: list[Path] = [] + original_write = compiler.utils.write_file + + def write_file(path, code): + """Write an output, failing the page when the test asks. + + Args: + path: The output path. + code: The output content. + + Raises: + OSError: When the page write is made to fail. + """ + if write_fails and Path(path) == page: + msg = "disk full" + raise OSError(msg) + written.append(Path(path)) + original_write(path, code) + + mocker.patch.object(compiler.utils, "write_file", side_effect=write_file) + + def rebuild(*args, outputs, **kwargs): + """Stage a page and the refreshed manifest, manifest first. + + Args: + args: Application arguments. + outputs: Compiler output mapping. + kwargs: Rebuild options. + + Returns: + Whether the incremental rebuild succeeded. + """ + outputs[manifest_path] = json.dumps({"schema": 1, "plugin_sources": {}}) + outputs[page] = "new" + return True + + mocker.patch.object(disk_cache, "try_incremental_rebuild", side_effect=rebuild) + if write_fails: + with pytest.raises(OSError, match="disk full"): + compiler.compile_app(app, use_rich=False) + assert not manifest_path.exists() + return + assert compiler.compile_app(app, use_rich=False) + assert json.loads(manifest_path.read_text())["schema"] == 1 + assert manifest_path not in written + assert written.index(page) < len(written) + + def test_preloaded_app_skips_incremental_reuse(monkeypatch, mocker): """Unobserved import-time reads require a full compile for preloaded apps.""" from reflex.compiler import disk_cache @@ -2146,37 +2223,6 @@ def recording_progress() -> compiler.console.PoorProgress: assert task["current"] == task["total"] -def test_compile_records_bundle_requests_per_page_and_phase(): - """Each page records the libraries it requested while evaluating and rendering.""" - from reflex_base.plugins import CompileContext, CompilerHooks - - from reflex.compiler.plugins import default_page_plugins - - def page_one() -> rx.Component: - bundle_library("lib-one") - return rx.el.div("one") - - def page_two() -> rx.Component: - bundle_library("lib-one") - bundle_library("lib-two") - return rx.el.div("two") - - with RegistrationContext(): - app = rx.App() - app.add_page(page_one, route="/one") - app.add_page(page_two, route="/two") - ctx = CompileContext( - app=app, - pages=list(app._unevaluated_pages.values()), - hooks=CompilerHooks(plugins=default_page_plugins()), - ) - with ctx: - ctx.compile() - assert ctx.compiled_pages["one"].evaluate_bundle_requests == ["lib-one"] - assert ctx.compiled_pages["two"].evaluate_bundle_requests == ["lib-one", "lib-two"] - assert ctx.compiled_pages["two"].render_bundle_requests == [] - - def test_serving_compile_discards_the_manifest(tmp_path, monkeypatch, mocker): """A compile that bypasses the cache leaves no manifest describing its output. diff --git a/tests/units/compiler/test_disk_cache.py b/tests/units/compiler/test_disk_cache.py index 0c2f0f375c1..4def39a8fb8 100644 --- a/tests/units/compiler/test_disk_cache.py +++ b/tests/units/compiler/test_disk_cache.py @@ -4,6 +4,7 @@ import datetime import itertools import json +import os import sys from collections.abc import Callable, Iterable, Sequence from pathlib import Path @@ -74,16 +75,17 @@ def _page_c() -> Component: return rx.el.div(rx.el.h1("Page C"), _footer()) -def _compile(pages: Sequence[Any], app: Any = None) -> CompileContext: +def _compile( + pages: Sequence[Any], app: Any = None, plugins: Sequence[Any] = () +) -> CompileContext: from reflex.compiler import compiler - # Reset like a full compile, so the manifest can rebuild the registry. - bundle_base = compiler._reset_compile_registries([]) + # Reset like a full compile does before evaluating any page. + compiler._reset_compile_registries(plugins) ctx = CompileContext( app=app, pages=pages, - hooks=CompilerHooks(plugins=default_page_plugins()), - bundle_base=bundle_base, + hooks=CompilerHooks(plugins=default_page_plugins(plugins=plugins)), ) with ctx: ctx.compile() @@ -309,11 +311,6 @@ def test_serialize_imports_collapses_duplicates(): } -def test_wrap_keys_round_trip_through_the_manifest_format(): - for key in [(200, "StrictMode"), (0, "AppWrap"), (45, "Name:With:Colons")]: - assert disk_cache._wrap_key(disk_cache._wrap_key_str(key)) == key - - def _manifest(pages: dict[str, dict], root: Path | None = None, **overrides) -> dict: base = { "schema": disk_cache._SCHEMA, @@ -327,15 +324,8 @@ def _manifest(pages: dict[str, dict], root: Path | None = None, **overrides) -> "pages": pages, "memo_files": {}, "entry": {"path": None, "digest": None}, - "wrap_table": {}, - "state_libraries": {}, "module_states": {"names": [], "files": [], "dynamic": []}, - "app_root": { - "wraps": {}, - "libraries": [], - "root_requests": [], - "reconstructible": True, - }, + "app_root": {"wraps": {}, "libraries": [], "memo_keys": []}, } base.update(overrides) return base @@ -495,7 +485,6 @@ def test_partition_pages_detects_changed_source(tmp_path): _FakePage(route="/a", component=_page_a), _FakePage(route="/b", component=_page_b), ] - registrations = {page.route: "recorded" for page in pages} x, y = tmp_path / "x.py", tmp_path / "y.py" x.write_text("X = 1\n") y.write_text("Y = 1\n") @@ -513,22 +502,42 @@ def test_partition_pages_detects_changed_source(tmp_path): str(y): page_cache.file_entry(str(y)), }, ) - assert ( - disk_cache.partition_pages(pages, m, _validator(m), None, registrations) == [] - ) + assert disk_cache.partition_pages(pages, m, _validator(m), None) == [] y.write_text("Y = 2\n") - miss = disk_cache.partition_pages(pages, m, _validator(m), None, registrations) + miss = disk_cache.partition_pages(pages, m, _validator(m), None) assert {p.route for p in miss} == {"/b"} # With the watcher's changed set, pages depending on none of the changed # paths are hits without touching the filesystem. - assert ( - disk_cache.partition_pages(pages, m, _validator(m), {str(x)}, registrations) - == [] - ) - miss = disk_cache.partition_pages(pages, m, _validator(m), {str(y)}, registrations) + assert disk_cache.partition_pages(pages, m, _validator(m), {str(x)}) == [] + miss = disk_cache.partition_pages(pages, m, _validator(m), {str(y)}) assert {p.route for p in miss} == {"/b"} +def test_pages_mismatch_names_the_change(tmp_path): + """The page registry is an input of every page, so any change is global.""" + pages = [ + _FakePage(route="/a", component=_page_a), + _FakePage(route="/b", component=_page_b), + ] + registrations = {page.route: "recorded" for page in pages} + m = _manifest({page.route: {"registration": "recorded"} for page in pages}) + assert disk_cache._pages_mismatch(pages, m, registrations) is None + reason = disk_cache._pages_mismatch(pages[::-1], m, registrations) + assert reason == "the page order changed" + reason = disk_cache._pages_mismatch( + [*pages, _FakePage(route="/c", component=_page_c)], + m, + {**registrations, "/c": "new"}, + ) + assert reason is not None + assert "added /c" in reason + reason = disk_cache._pages_mismatch(pages[:1], m, registrations) + assert reason is not None + assert "removed /b" in reason + reason = disk_cache._pages_mismatch(pages, m, {**registrations, "/b": "retitled"}) + assert reason == "page '/b' was registered differently" + + def test_incomplete_cache_miss_falls_back_without_adoption(tmp_path, monkeypatch): """A miss without page output must be compiled again by the full path.""" _use_tmp_web_dir(tmp_path, monkeypatch) @@ -627,18 +636,18 @@ def test_write_and_load_manifest(tmp_path, monkeypatch): "deps", "component_counts", "app_wraps", - "bundle_requests", "registration", - "memo_path", "is_stateful", "state_slice", - "has_memos", "dynamic_claims", + "memo_keys", + "radix", } - # these static pages register no new state and contribute no memos + # these static pages register no new state and use no memos assert entry["is_stateful"] is False assert entry["state_slice"] is None - assert entry["has_memos"] is False + assert entry["memo_keys"] == [] + assert entry["radix"] is False assert entry["dynamic_claims"] == [] # every dependency has a file-table entry: [sha256, mtime_ns, size] for dep in entry["deps"]: @@ -865,9 +874,6 @@ def forget_local_states() -> None: # compile can rebuild the app root. manifest_path = web / disk_cache._MANIFEST_FILE manifest = json.loads(manifest_path.read_text()) - # The app root can no longer be rebuilt from the records, while /s - # now brings different app wraps than recorded. - manifest["app_root"]["reconstructible"] = False manifest["app_root"]["wraps"] = {} _stale_dep(manifest, route_s, str(tmp_path / "demo.md")) manifest_path.write_text(json.dumps(manifest)) @@ -1488,8 +1494,6 @@ def _schema_pages(tmp_path, monkeypatch): manifest = json.loads(manifest_path.read_text()) miss_route = pages[0].route _stale_dep(manifest, miss_route, str(tmp_path / "view.py")) - # Keep the hit page out of the miss set, as if it lived in another module. - manifest["pages"][pages[1].route]["has_memos"] = False manifest_path.write_text(json.dumps(manifest)) # The daemon child starts without the states its pages define. for name in ("_SchemaMissState", "_SchemaHitState"): @@ -1573,12 +1577,12 @@ def default() -> Component: return rx.el.div(rx.box(state_cls.comp)) -def test_state_default_from_new_library_rebuilds_the_app_root(tmp_path, monkeypatch): - """A component default from a new library rewrites the root and library file. +def test_state_default_from_new_library_falls_back(tmp_path, monkeypatch): + """A component default from a new library needs a full compile. The contexts reference the library through ``window.__reflex``, which the - app root and bundled-library file provide; both are rebuilt from recorded - requests without evaluating the unchanged pages. + app root and bundled-library file provide; only a full compile rewrites + them, and it adopts the pages the attempt evaluated. """ from reflex.compiler import compiler @@ -1603,10 +1607,8 @@ def test_state_default_from_new_library_rebuilds_the_app_root(tmp_path, monkeypa ) finally: _unregister_state(_FP_HOLDER.pop("_ComponentDefaultState")) - assert result is True - libraries = json.loads((web / constants.Dirs.BUNDLED_LIBRARIES).read_text()) - assert "recharts" in libraries - assert "recharts" in _app_root_file(web).read_text() + assert isinstance(result, CompileContext) + assert set(result.compiled_pages) == {pages[0].route} def test_unchanged_component_default_reuses_the_contexts(tmp_path, monkeypatch): @@ -1695,8 +1697,6 @@ def _root_rebuild( Returns: The rebuild result, the app root file and its content before the edit. """ - from reflex_base.components.dynamic import record_bundle_requests - from reflex.compiler import compiler try: @@ -1707,22 +1707,14 @@ def _root_rebuild( pages = list(app._unevaluated_pages.values()) ctx = _compile(pages, app=app) _stub_externals(app, monkeypatch) - with record_bundle_requests() as root_requests: - root_path, root_code = compiler.compile_app_root( - *compiler._build_app_root(app, ctx.app_wrap_components, ctx) - ) + root_path, root_code = compiler.compile_app_root( + *compiler._build_app_root(app, ctx.app_wrap_components, ctx) + ) Path(root_path).parent.mkdir(parents=True, exist_ok=True) Path(root_path).write_text(root_code) - disk_cache.write_manifest( - ctx, - pages, - ctx.all_imports, - root=tmp_path, - root_bundle_requests=root_requests, - ) + disk_cache.write_manifest(ctx, pages, ctx.all_imports, root=tmp_path) manifest_path = web / disk_cache._MANIFEST_FILE manifest = json.loads(manifest_path.read_text()) - assert manifest["app_root"]["reconstructible"] if reversed_wraps is not None: wraps = manifest["app_root"]["wraps"] assert len(wraps) > 1 @@ -1751,24 +1743,26 @@ def test_page_gaining_a_provided_wrap_reuses_the_app_root(tmp_path, monkeypatch) assert root_file.read_text() == before -def test_app_root_follows_the_order_of_its_wraps(tmp_path, monkeypatch): - """Wraps of one priority nest in the order pages first require them.""" +def test_app_root_with_reordered_wraps_falls_back(tmp_path, monkeypatch): + """Wraps of one priority nest in page order, so another order is a change.""" recorded: list[str] = [] - result, _, _ = _root_rebuild(tmp_path, monkeypatch, reversed_wraps=recorded) - assert result is True - manifest = json.loads((tmp_path / ".web" / disk_cache._MANIFEST_FILE).read_text()) - assert list(manifest["app_root"]["wraps"]) == recorded[::-1] + result, root_file, before = _root_rebuild( + tmp_path, monkeypatch, reversed_wraps=recorded + ) + assert isinstance(result, CompileContext) + assert root_file.read_text() == before -def test_page_bringing_a_new_wrap_rebuilds_only_the_app_root(tmp_path, monkeypatch): - """A new wrap rebuilds the root from recorded and recompiled wraps.""" +def test_page_bringing_a_new_wrap_falls_back(tmp_path, monkeypatch): + """A wrap no recorded page provides needs the app root rebuilt in full. + + The attempt hands over the page it evaluated; the root on disk is left + for the full compile to replace. + """ result, root_file, before = _root_rebuild(tmp_path, monkeypatch, wrap=True) - assert result is True - after = root_file.read_text() - assert after != before - assert "test-wrap" in after - # The unchanged stateful page still provides the state providers. - assert "StateProvider" in after + assert isinstance(result, CompileContext) + assert set(result.compiled_pages) == {"p"} + assert root_file.read_text() == before _COLLIDE = {"one": 0, "two": 0, "two_extra": False, "two_unique": False} @@ -1842,8 +1836,6 @@ def _rebuild_second_page( manifest_path = web / disk_cache._MANIFEST_FILE manifest = json.loads(manifest_path.read_text()) _stale_dep(manifest, pages[1].route, str(tmp_path / "view.py")) - # Keep the first page out of the miss set, as if it lived in another module. - manifest["pages"][pages[0].route]["has_memos"] = False manifest_path.write_text(json.dumps(manifest)) _drop_dynamic_states(*markers) prepare() @@ -2071,77 +2063,53 @@ def recording_compile(self, *args, **kwargs): return result, evaluated, web -def test_added_page_rebuilds_incrementally(tmp_path, monkeypatch): - """A new route is a miss; the other pages keep their output.""" - from reflex.compiler import utils as compiler_utils +def _add_b(app) -> None: + app.add_page(_page_b, route="/b") - result, evaluated, web = _route_set_rebuild( - tmp_path, monkeypatch, lambda app: app.add_page(_page_b, route="/b") - ) - assert result is True - assert evaluated == ["b"] - assert Path(compiler_utils.get_page_path("b")).exists() - manifest = json.loads((web / disk_cache._MANIFEST_FILE).read_text()) - assert set(manifest["pages"]) == {"a", "c", "b"} - assert "b" in json.loads((web / constants.Dirs.ROUTES_MANIFEST).read_text()) +def _remove_c(app) -> None: + del app._unevaluated_pages["c"] -def test_removed_page_rebuilds_incrementally(tmp_path, monkeypatch): - """A removed route loses its output and manifest entry, nothing else.""" - from reflex.compiler import utils as compiler_utils - def remove_c(app): - del app._unevaluated_pages["c"] +def _reorder(app) -> None: + app._unevaluated_pages = dict(reversed(app._unevaluated_pages.items())) - result, evaluated, web = _route_set_rebuild(tmp_path, monkeypatch, remove_c) - assert result is True - assert evaluated == [] - assert not Path(compiler_utils.get_page_path("c")).exists() - assert Path(compiler_utils.get_page_path("a")).exists() - manifest = json.loads((web / disk_cache._MANIFEST_FILE).read_text()) - assert set(manifest["pages"]) == {"a"} +def _retitle_c(app) -> None: + import dataclasses -def test_removed_stateful_page_leaves_the_contexts(tmp_path, monkeypatch): - """A removed page's local states leave the contexts file.""" - web = _use_tmp_web_dir(tmp_path, monkeypatch) - app = rx.App() - app.add_page(_page_a, route="/a") - app.add_page(_page_collide_one, route="/s") - pages = list(app._unevaluated_pages.values()) - try: - ctx = _compile(pages, app=app) - local_states = ctx.stateful_routes[pages[1].route] - _stub_externals(app, monkeypatch) - disk_cache.write_manifest(ctx, pages, ctx.all_imports, root=tmp_path) - _drop_dynamic_states("CollideState") - del app._unevaluated_pages[pages[1].route] - captured = _capture_contexts(monkeypatch) - result = disk_cache.try_incremental_rebuild( - app, compiler_plugins=[], prerender_routes=False, root=tmp_path - ) - finally: - _drop_dynamic_states("CollideState") - assert result is True - [(initial, _, schema_entries)] = captured - assert not set(local_states) & set(initial) - assert not set(local_states) & set(schema_entries) - marker = json.loads((web / disk_cache._MANIFEST_FILE).read_text()) - assert set(marker["pages"]) == {"a"} + app._unevaluated_pages["c"] = dataclasses.replace( + app._unevaluated_pages["c"], title="Renamed" + ) -def test_changed_registration_recompiles_only_that_page(tmp_path, monkeypatch): - """A page whose registration changed (here its title) is a miss.""" - import dataclasses +@pytest.mark.parametrize( + ("edit", "reason"), + [ + (_add_b, "added b"), + (_remove_c, "removed c"), + (_reorder, "order changed"), + (_retitle_c, "registered differently"), + ], +) +def test_page_registry_change_falls_back(tmp_path, monkeypatch, caplog, edit, reason): + """The page registry is an input of every page (a sidebar lists them). - def retitle_c(app): - app._unevaluated_pages["c"] = dataclasses.replace( - app._unevaluated_pages["c"], title="Renamed" - ) + Adding, removing, reordering or re-registering a page (its title, say) + therefore re-evaluates every page, which a full compile does. + """ + from reflex.compiler import utils as compiler_utils - result, evaluated, _ = _route_set_rebuild(tmp_path, monkeypatch, retitle_c) - assert result is True - assert evaluated == ["c"] + with caplog.at_level("INFO", logger=disk_cache.logger.name): + result, evaluated, web = _route_set_rebuild(tmp_path, monkeypatch, edit) + assert result is False + assert evaluated == [] + assert reason in caplog.text + # Nothing was written or removed: the full compile does that. + assert Path(compiler_utils.get_page_path("a")).exists() + assert Path(compiler_utils.get_page_path("c")).exists() + manifest = json.loads((web / disk_cache._MANIFEST_FILE).read_text()) + assert list(manifest["pages"]) == ["a", "c"] def test_incremental_rebuild_copies_assets(tmp_path, monkeypatch): @@ -2404,71 +2372,200 @@ class _MemoCacheState(rx.State): other: str = "y" -def _page_e() -> Component: - return rx.el.div(rx.el.p(_MemoCacheState.value, class_name="e"), rx.el.h1("E")) +_SHARED_MEMO = {"edit_one": False} + +def _page_shared_one() -> Component: + if _SHARED_MEMO["edit_one"]: + return rx.el.div(rx.el.p(_MemoCacheState.other, class_name="one")) + return rx.el.div(rx.el.p(_MemoCacheState.value, class_name="shared")) -def _page_f() -> Component: - return rx.el.div(rx.el.p(_MemoCacheState.other, class_name="f"), rx.el.h2("F")) +def _page_shared_two() -> Component: + return rx.el.div(rx.el.p(_MemoCacheState.value, class_name="shared")) -def test_incremental_miss_keeps_sibling_memo_exports( + +def _memo_file_of(page_ctx: PageContext) -> Path: + """The single auto memo file a test page's output imports. + + Args: + page_ctx: The compiled page. + + Returns: + The memo file path. + """ + [(tag, _)] = page_ctx.memo_uses + return Path(disk_cache._memo_file(tag)) + + +def test_incremental_miss_leaves_the_memo_a_reused_page_shares( tmp_path, monkeypatch, preserve_memo_registries ): - """A miss must not clobber memo exports owned by hit pages. + """A miss never touches the memo files of the pages it reuses. - Auto-memo output is grouped into one file per source module. Pages E and F - live in this module and each contributes a stateful auto memo, so both land - in the same mirrored file. When only E misses (e.g. a data-file edit), - rewriting that file from E's contributions alone drops F's export while - F's reused page module still imports it. + Pages one and two render the same stateful subtree, so both import the + same content-addressed auto memo. Editing page one gives it another + memo; the shared memo must survive for page two, which still imports it. """ from reflex.compiler import compiler from reflex.compiler import utils as compiler_utils web = _use_tmp_web_dir(tmp_path, monkeypatch) - app = rx.App() - app.add_page(_page_e, route="/e") - app.add_page(_page_f, route="/f") + app.add_page(_page_shared_one, route="/one") + app.add_page(_page_shared_two, route="/two") pages = list(app._unevaluated_pages.values()) - route_e, route_f = pages[0].route, pages[1].route + route_one, route_two = pages[0].route, pages[1].route ctx = _compile(pages) + one_ctx, two_ctx = ctx.compiled_pages[route_one], ctx.compiled_pages[route_two] + shared = _memo_file_of(one_ctx) + assert shared == _memo_file_of(two_ctx), "both pages must share the memo" + # The full compile's on-disk memo state. + memo_files, _ = compiler.compile_memo_components(ctx.auto_memo_components.values()) + assert [Path(path) for path, _ in memo_files] == [shared] + for mpath, mcode in memo_files: + compiler_utils.write_file(compiler_utils.resolve_path_of_web_dir(mpath), mcode) + compiler_utils.prune_stale_memo_files(path for path, _ in memo_files) + disk_cache.write_manifest(ctx, pages, ctx.all_imports, root=tmp_path) + _stub_externals(app, monkeypatch) + manifest_path = web / disk_cache._MANIFEST_FILE + manifest = json.loads(manifest_path.read_text()) + [tag] = manifest["pages"][route_two]["memo_keys"] + assert manifest["pages"][route_one]["memo_keys"] == [tag] + assert manifest["app_root"]["memo_keys"] == [] - e_memos = list(ctx.compiled_pages[route_e].memo_contributions.values()) - f_memos = list(ctx.compiled_pages[route_f].memo_contributions.values()) - assert e_memos, "page E must contribute an auto memo" - assert f_memos, "page F must contribute an auto memo" + # Edit page one so it renders another subtree, and make only it miss. + monkeypatch.setitem(_SHARED_MEMO, "edit_one", True) + _stale_dep(manifest, route_one, str(tmp_path / "data.md")) + manifest_path.write_text(json.dumps(manifest)) + assert ( + disk_cache.try_incremental_rebuild( + app, compiler_plugins=[], prerender_routes=False, root=tmp_path + ) + is True + ) - # Simulate the full compile's on-disk memo state (shared grouped file). - memo_files, _ = compiler.compile_memo_components([*e_memos, *f_memos]) - assert memo_files + # The shared memo is still there for page two; page one got a new one. + assert shared.exists() + refreshed = json.loads(manifest_path.read_text()) + [new_tag] = refreshed["pages"][route_one]["memo_keys"] + assert new_tag != tag + assert refreshed["pages"][route_two]["memo_keys"] == [tag] + new_file = Path(disk_cache._memo_file(new_tag)) + assert new_file.exists() + page_one = Path(compiler_utils.get_page_path(route_one)).read_text() + assert new_tag in page_one + assert tag not in page_one + memo_manifest = compiler_utils._read_memo_manifest(web) + assert memo_manifest == { + str(path.relative_to(web)).replace(os.sep, "/") for path in (shared, new_file) + } + + +def test_incremental_miss_drops_the_memo_no_page_uses( + tmp_path, monkeypatch, preserve_memo_registries +): + """A memo only the recompiled page used is deleted once it stops using it.""" + from reflex.compiler import compiler + from reflex.compiler import utils as compiler_utils + + web = _use_tmp_web_dir(tmp_path, monkeypatch) + app = rx.App() + app.add_page(_page_shared_one, route="/one") + pages = list(app._unevaluated_pages.values()) + route_one = pages[0].route + ctx = _compile(pages) + old_file = _memo_file_of(ctx.compiled_pages[route_one]) + memo_files, _ = compiler.compile_memo_components(ctx.auto_memo_components.values()) for mpath, mcode in memo_files: compiler_utils.write_file(compiler_utils.resolve_path_of_web_dir(mpath), mcode) - + compiler_utils.prune_stale_memo_files(path for path, _ in memo_files) disk_cache.write_manifest(ctx, pages, ctx.all_imports, root=tmp_path) _stub_externals(app, monkeypatch) - - # Make only page E miss (a dependency of E changed, e.g. a data file). manifest_path = web / disk_cache._MANIFEST_FILE manifest = json.loads(manifest_path.read_text()) - _stale_dep(manifest, route_e, str(tmp_path / "data.md")) + monkeypatch.setitem(_SHARED_MEMO, "edit_one", True) + _stale_dep(manifest, route_one, str(tmp_path / "data.md")) manifest_path.write_text(json.dumps(manifest)) - assert ( disk_cache.try_incremental_rebuild( app, compiler_plugins=[], prerender_routes=False, root=tmp_path ) is True ) + assert not old_file.exists() + refreshed = json.loads(manifest_path.read_text()) + [new_tag] = refreshed["pages"][route_one]["memo_keys"] + new_relative = str(Path(disk_cache._memo_file(new_tag)).relative_to(web)).replace( + os.sep, "/" + ) + assert compiler_utils._read_memo_manifest(web) == {new_relative} - # Every export (E's and F's) survives in the rewritten grouped file(s). - for mpath, _mcode in memo_files: - content = compiler_utils.resolve_path_of_web_dir(mpath).read_text( - encoding="utf-8" + +def _page_radix() -> Component: + return rx.el.div(rx.button("Radix"), _footer()) + + +def test_reused_radix_page_keeps_the_implicit_plugin_enabled(tmp_path, monkeypatch): + """A rebuild that compiles no Radix page still ships the Radix stylesheet. + + The implicit plugin enables itself while compiling a Radix component; a + page reused from disk is not compiled, so the manifest records the page's + Radix use and the rebuild enables the plugin from it. + """ + from reflex_components_radix.plugin import RadixThemesPlugin + + web = _use_tmp_web_dir(tmp_path, monkeypatch) + app = rx.App() + app.add_page(_page_radix, route="/radix") + app.add_page(_page_a, route="/a") + pages = list(app._unevaluated_pages.values()) + ctx = _compile(pages, plugins=[RadixThemesPlugin.create_implicit()]) + _stub_externals(app, monkeypatch) + disk_cache.write_manifest(ctx, pages, ctx.all_imports, root=tmp_path) + manifest_path = web / disk_cache._MANIFEST_FILE + manifest = json.loads(manifest_path.read_text()) + assert manifest["pages"][pages[0].route]["radix"] is True + assert manifest["pages"][pages[1].route]["radix"] is False + + # An all-hit rebuild, and one that recompiles only the plain page. + for stale in (None, pages[1].route): + if stale is not None: + _stale_dep(manifest, stale, str(tmp_path / "view.py")) + manifest_path.write_text(json.dumps(manifest)) + plugin = RadixThemesPlugin.create_implicit() + assert not plugin.enabled + assert ( + disk_cache.try_incremental_rebuild( + app, compiler_plugins=[plugin], prerender_routes=False, root=tmp_path + ) + is True ) - for memo_def in (*e_memos, *f_memos): - assert memo_def.export_name in content + assert plugin.enabled + assert plugin.get_stylesheet_paths() + + +def test_recompiled_page_leaves_a_hit_pages_state_to_it(tmp_path, monkeypatch): + """A state a hit page recorded is not claimed by a recompiled page. + + In a process that evaluates only the recompiled pages, one of them can + be the first to import the module defining a state another page owns. + """ + miss_ctx = SimpleNamespace( + stateful_routes={ + "/miss": ["app.shared", "app.own"], + "/other": ["app.shared"], + } + ) + manifest = _manifest({ + "/hit": {"state_slice": {"schema": {"app.shared": []}}}, + "/miss": {"state_slice": None}, + "/other": {"state_slice": None}, + }) + disk_cache._disown_hit_page_states( + cast(Any, miss_ctx), manifest, {"/miss", "/other"} + ) + assert miss_ctx.stateful_routes == {"/miss": ["app.own"]} def test_load_manifest_rejects_wrong_schema(tmp_path, monkeypatch): @@ -2489,15 +2586,13 @@ def test_update_manifest_for_misses_keeps_complete_imports(tmp_path, monkeypatch root_component=rx.el.div(), app_wrap_components={}, frontend_imports={}, - memo_contributions={}, - evaluate_bundle_requests=[], - render_bundle_requests=[], + memo_uses={}, ) miss_ctx = SimpleNamespace(compiled_pages={"/a": page_ctx}, stateful_routes={}) complete_imports = {"memo-lib": [ImportVar("MemoThing")]} manifest = _manifest({"/a": {"deps": [], "app_wraps": {}, "is_stateful": False}}) - disk_cache._update_manifest_for_misses( + assert disk_cache._update_manifest_for_misses( manifest, cast(Any, miss_ctx), [page], @@ -2505,11 +2600,10 @@ def test_update_manifest_for_misses_keeps_complete_imports(tmp_path, monkeypatch root=tmp_path, memo_state={}, memo_file_entries={}, - app_root_inputs=manifest["app_root"], - state_libraries={}, registrations={"/a": "recorded"}, module_states=manifest["module_states"], ) + disk_cache._write(manifest) written = json.loads((web / disk_cache._MANIFEST_FILE).read_text()) assert disk_cache._deserialize_imports(written["all_imports"]) == complete_imports @@ -2521,7 +2615,7 @@ def test_update_manifest_refreshes_imports_without_misses(tmp_path, monkeypatch) manifest = _manifest({}, all_imports={"old-lib": [{"tag": "Old"}]}) memo_imports = {"old-lib": [ImportVar("Old")], "memo-lib": [ImportVar("Memo")]} - disk_cache._update_manifest_for_misses( + assert disk_cache._update_manifest_for_misses( manifest, None, [], @@ -2529,14 +2623,25 @@ def test_update_manifest_refreshes_imports_without_misses(tmp_path, monkeypatch) root=tmp_path, memo_state={}, memo_file_entries={}, - app_root_inputs=manifest["app_root"], - state_libraries={}, registrations={"/a": "recorded"}, module_states=manifest["module_states"], ) + disk_cache._write(manifest) written = json.loads((web / disk_cache._MANIFEST_FILE).read_text()) assert disk_cache._deserialize_imports(written["all_imports"]) == memo_imports + # Nothing changed: nothing to write. + assert not disk_cache._update_manifest_for_misses( + written, + None, + [], + memo_imports, + root=tmp_path, + memo_state={}, + memo_file_entries={}, + registrations={"/a": "recorded"}, + module_states=written["module_states"], + ) def _page_loop_a() -> Component: @@ -2667,38 +2772,34 @@ def test_partial_page_depends_on_its_function_module(tmp_path, monkeypatch): assert files == {str(module.resolve())} -def test_dynamic_state_pages_bring_their_memo_siblings(): - """A dynamic-state page pulled into a rebuild brings its memo siblings. +@pytest.mark.parametrize("predict", [False, True]) +def test_expand_miss_pages_keeps_page_order(monkeypatch, predict): + """Re-evaluated pages are evaluated in page order, whatever added them. - They share its module's memo file, which the rebuild rewrites. + ``ComponentState`` numbers and memo emit order follow evaluation order. """ - pages = [ - _FakePage("/a", _page_a, _source_module="first_module"), - _FakePage("/b", _page_b, _source_module="second_module"), - _FakePage("/c", _page_c, _source_module="second_module"), - ] + pages = [_FakePage(f"/{name}", _page_a) for name in "abcd"] - def entry(claims: list[list[str]], has_memos: bool) -> dict[str, Any]: - return {"dynamic_claims": claims, "has_memos": has_memos} + def entry(claims: list[list[str]]) -> dict[str, Any]: + return {"dynamic_claims": claims} manifest = { "pages": { - "/a": entry([], has_memos=False), - "/b": entry([["LocalState", "LocalState"]], has_memos=True), - "/c": entry([], has_memos=True), + "/a": entry([]), + "/b": entry([["LocalState", "LocalState"]]), + "/c": entry([]), + "/d": entry([]), } } - misses = disk_cache._expand_miss_pages( - [pages[0]], pages, manifest, set(), set(), dynamic_names_shift=False - ) - assert [page.route for page in misses] == ["/a", "/b", "/c"] + monkeypatch.setattr(disk_cache, "_predict_dynamic_names", predict) + misses = disk_cache._expand_miss_pages([pages[3]], pages, manifest, {"/a"}) + expected = ["/a", "/d"] if predict else ["/a", "/b", "/d"] + assert [page.route for page in misses] == expected @pytest.mark.parametrize("predict", [False, True]) -def test_reordering_dynamic_state_pages_reevaluates_them( - tmp_path, monkeypatch, predict -): - """Dynamic state names follow page order, so a reorder renames them.""" +def test_reordering_pages_falls_back_before_evaluating(tmp_path, monkeypatch, predict): + """A reorder changes the page registry, which every page may read.""" web = _use_tmp_web_dir(tmp_path, monkeypatch) app = rx.App() app.add_page(_page_collide_one, route="/one") @@ -2716,19 +2817,13 @@ def test_reordering_dynamic_state_pages_reevaluates_them( monkeypatch.setitem(_COLLIDE, "two", 0) monkeypatch.setattr(disk_cache, "_predict_dynamic_names", predict) try: - if predict: - with pytest.raises(disk_cache.DynamicStatePredictionError): - disk_cache.try_incremental_rebuild( - app, compiler_plugins=[], prerender_routes=False, root=tmp_path - ) - return result = disk_cache.try_incremental_rebuild( app, compiler_plugins=[], prerender_routes=False, root=tmp_path ) finally: _drop_dynamic_states("CollideState") - assert result is True - assert (_COLLIDE["one"], _COLLIDE["two"]) == (1, 1) + assert result is False + assert (_COLLIDE["one"], _COLLIDE["two"]) == (0, 0) def test_manifest_dependency_files_cover_every_recorded_input(): diff --git a/tests/units/compiler/test_memoize_plugin.py b/tests/units/compiler/test_memoize_plugin.py index ffea8c7ca2d..8da00c116a0 100644 --- a/tests/units/compiler/test_memoize_plugin.py +++ b/tests/units/compiler/test_memoize_plugin.py @@ -22,7 +22,6 @@ ) from reflex_base.constants.compiler import MemoizationDisposition, MemoizationMode from reflex_base.plugins import CompileContext, CompilerHooks, PageContext -from reflex_base.utils import memo_paths from reflex_base.utils.imports import ImportVar from reflex_base.vars import VarData from reflex_base.vars.base import Field, LiteralVar, Var, field @@ -857,7 +856,7 @@ def fake_create_passthrough_component_memo( first_compile = SimpleNamespace(memoize_wrappers={}, auto_memo_components={}) second_compile = SimpleNamespace(memoize_wrappers={}, auto_memo_components={}) - page_context = cast(PageContext, SimpleNamespace(source_module=None)) + page_context = cast(PageContext, SimpleNamespace(source_module=None, memo_uses={})) MemoizeStatefulPlugin._build_wrapper( first_component, @@ -898,15 +897,13 @@ def test_shared_subtree_across_pages_uses_same_tag() -> None: assert f"jsx({tag}," in output -def test_shared_subtree_in_distinct_source_modules_emits_per_module() -> None: - """Identical subtrees in different user modules emit one memo per module. +def test_shared_subtree_in_distinct_source_modules_shares_one_memo() -> None: + """Identical subtrees in different user modules share one auto memo. - Regression: the auto-memo registry was keyed by tag only, so when two - pages from different user modules produced the same memoizable subtree - (and therefore the same tag), the second registration overwrote the - first. Only the surviving definition's source module got a mirrored - memo file, leaving the other page importing a tag from a file that - never exported it. + Auto memos are content-addressed by their tag and compile to one file + each under ``utils/components/``, whatever module the page lives in, so + the two pages import the same file and neither page's memo files hold + another page's memos. """ from reflex.compiler.compiler import compile_memo_components @@ -930,43 +927,20 @@ def test_shared_subtree_in_distinct_source_modules_emits_per_module() -> None: assert len(ctx.memoize_wrappers) == 1 tag = next(iter(ctx.memoize_wrappers)) - # Both source modules survive registration as distinct entries. - assert set(ctx.auto_memo_components) == { - (tag, "memo_collision_test.module_a"), - (tag, "memo_collision_test.module_b"), - } - - # Each module emits and imports the tag under a per-module symbol so the - # two pages never collide on a shared JS identifier. - symbol_a = memo_paths.mirrored_symbol(tag, "memo_collision_test.module_a") - symbol_b = memo_paths.mirrored_symbol(tag, "memo_collision_test.module_b") - page_a_output = ctx.compiled_pages["/a"].output_code or "" - page_b_output = ctx.compiled_pages["/b"].output_code or "" - assert ( - f'import {{{symbol_a}}} from "$/app_components/memo_collision_test/module_a"' - in page_a_output - ) - assert ( - f'import {{{symbol_b}}} from "$/app_components/memo_collision_test/module_b"' - in page_b_output - ) + assert set(ctx.auto_memo_components) == {(tag, None)} + for route in ("/a", "/b"): + page_ctx = ctx.compiled_pages[route] + assert set(page_ctx.memo_uses) == {(tag, None)} + assert f'import {{{tag}}} from "$/utils/components/{tag}"' in ( + page_ctx.output_code or "" + ) output_files, _ = compile_memo_components( memos=tuple(ctx.auto_memo_components.values()), ) - emitted = {Path(path).as_posix(): code for path, code in output_files} - - def find_emitted(suffix: str) -> str | None: - return next( - (code for path, code in emitted.items() if path.endswith(suffix)), None - ) - - matched_a = find_emitted("memo_collision_test/module_a.jsx") - matched_b = find_emitted("memo_collision_test/module_b.jsx") - assert matched_a is not None, f"missing module_a memo file in {sorted(emitted)}" - assert matched_b is not None, f"missing module_b memo file in {sorted(emitted)}" - assert f"const {symbol_a} = memo" in matched_a - assert f"const {symbol_b} = memo" in matched_b + [(path, code)] = output_files + assert Path(path).as_posix().endswith(f"utils/components/{tag}.jsx") + assert f"const {tag} = memo" in code def test_shared_parent_instance_across_pages_preserves_original() -> None: diff --git a/tests/units/compiler/test_page_cache.py b/tests/units/compiler/test_page_cache.py index a153ba65e9b..17b2494e33f 100644 --- a/tests/units/compiler/test_page_cache.py +++ b/tests/units/compiler/test_page_cache.py @@ -676,7 +676,7 @@ def test_page_dependency_files_closes_over_dynamic_imports(tmp_path, monkeypatch page_ctx = SimpleNamespace( root_component=SimpleNamespace(children=[]), output_code="", - memo_contributions={}, + memo_uses={}, source_files={str(helper.resolve())}, ) deps = page_cache.page_dependency_files( @@ -821,7 +821,7 @@ def blog_b(): page_ctx = SimpleNamespace( root_component=SimpleNamespace(children=[]), output_code="", - memo_contributions={}, + memo_uses={}, source_files={str(helper.resolve())}, ) deps = page_cache.page_dependency_files( @@ -900,7 +900,7 @@ def test_mark_volatile_makes_the_page_miss_every_compile(tmp_path): page_ctx = SimpleNamespace( root_component=SimpleNamespace(children=[]), output_code="", - memo_contributions={}, + memo_uses={}, source_files=reads, ) files = {} @@ -1138,7 +1138,7 @@ def test_function_slice_falls_back_to_the_whole_file_for_computed_access( page_ctx = SimpleNamespace( root_component=SimpleNamespace(children=[]), output_code="", - memo_contributions={}, + memo_uses={}, source_files=set(), ) deps = page_cache.page_dependency_files( @@ -1190,7 +1190,7 @@ def test_page_dependency_files_track_the_page_function_slice(tmp_path, monkeypat page_ctx = SimpleNamespace( root_component=SimpleNamespace(children=[]), output_code="", - memo_contributions={}, + memo_uses={}, source_files=set(), ) deps = page_cache.page_dependency_files( diff --git a/tests/units/compiler/test_stale_cleanup.py b/tests/units/compiler/test_stale_cleanup.py index 500fc02861c..43deb5683d8 100644 --- a/tests/units/compiler/test_stale_cleanup.py +++ b/tests/units/compiler/test_stale_cleanup.py @@ -167,3 +167,38 @@ def test_prune_ignores_corrupt_manifest(fake_web_dir: Path): compiler_utils.prune_stale_memo_files([survivor]) assert survivor.exists() + + +def test_reconcile_keeps_untouched_files_and_drops_the_unused(fake_web_dir: Path): + """An incremental compile records what it wrote and deletes what it dropped. + + Files it did not touch stay recorded, so a later full compile still + prunes them; only files the previous manifest recorded are deleted. + """ + kept, dropped, foreign = _seed_files( + fake_web_dir, + [ + "utils/components/Kept.jsx", + "utils/components/Dropped.jsx", + "utils/components/Foreign.jsx", + ], + ) + _seed_manifest( + fake_web_dir, ["utils/components/Kept.jsx", "utils/components/Dropped.jsx"] + ) + written = fake_web_dir / "utils" / "components" / "Written.jsx" + compiler_utils.reconcile_memo_manifest([written], [dropped, foreign]) + assert kept.exists() + assert not dropped.exists() + assert foreign.exists(), "only files the manifest recorded are deleted" + recorded = json.loads( + (fake_web_dir / compiler_utils._MEMO_MANIFEST_FILENAME).read_text() + ) + assert recorded == ["utils/components/Kept.jsx", "utils/components/Written.jsx"] + # A file both written and dropped is written. + compiler_utils.reconcile_memo_manifest([kept], [kept]) + assert kept.exists() + recorded = json.loads( + (fake_web_dir / compiler_utils._MEMO_MANIFEST_FILENAME).read_text() + ) + assert recorded == ["utils/components/Kept.jsx", "utils/components/Written.jsx"] diff --git a/tests/units/components/test_memo.py b/tests/units/components/test_memo.py index 049cf6f6706..fbee162378b 100644 --- a/tests/units/components/test_memo.py +++ b/tests/units/components/test_memo.py @@ -1162,9 +1162,7 @@ def test_compile_memo_components_falls_back_when_no_source_module(): ) files, _ = compiler.compile_memo_components((legacy_definition,)) - exp_path = compiler._memo_component_file_path( - compiler_utils.get_memo_components_dir(), "LegacyMemo" - ) + exp_path = compiler_utils.get_memo_component_path("LegacyMemo") assert any(path == exp_path for path, _ in files) @@ -1339,9 +1337,7 @@ def test_component_memo_wrapper_none_in_unmirrored_module(): ) files, _ = compiler.compile_memo_components((definition,)) - exp_path = compiler._memo_component_file_path( - compiler_utils.get_memo_components_dir(), "BareSingle" - ) + exp_path = compiler_utils.get_memo_component_path("BareSingle") single_code = next(code for path, code in files if path == exp_path) assert "export const BareSingle = ((" in single_code assert " = memo(" not in single_code diff --git a/tests/units/reflex_base/components/test_dynamic.py b/tests/units/reflex_base/components/test_dynamic.py index 48105b92409..5f05159f213 100644 --- a/tests/units/reflex_base/components/test_dynamic.py +++ b/tests/units/reflex_base/components/test_dynamic.py @@ -165,19 +165,3 @@ def test_unknown_attribute_raises(module_name: str): module = importlib.import_module(f"{module_name}.components.dynamic") with pytest.raises(AttributeError, match=f"{module.__name__!r}"): _ = module.definitely_not_an_attribute - - -def test_bundle_requests_are_recorded_in_order(): - """Every request is recorded, bundled already or not, and reaches outer recorders. - - A compile reconstructs the bundled-library order from what each page and - state requested, so a request for an already bundled library still counts. - """ - with RegistrationContext(): - with dynamic.record_bundle_requests() as outer: - dynamic.bundle_library("lib-a") - with dynamic.record_bundle_requests() as inner: - dynamic.bundle_library("lib-a") - dynamic.bundle_library("lib-b") - assert inner == ["lib-a", "lib-b"] - assert outer == ["lib-a", "lib-a", "lib-b"] diff --git a/tests/units/utils/test_compile_daemon.py b/tests/units/utils/test_compile_daemon.py index 245dd965a8c..b72db895b11 100644 --- a/tests/units/utils/test_compile_daemon.py +++ b/tests/units/utils/test_compile_daemon.py @@ -580,6 +580,68 @@ class LocalState(rx.State): @pytest.mark.skipif(not hasattr(os, "fork"), reason="requires os.fork (POSIX)") +def test_reset_first_party_drops_memos_of_purged_modules(tmp_path): + """A purged module's memos leave the registry; the module re-registers them. + + Otherwise a renamed memo collides with the parent's registration of its + old name on every re-import, and a deleted memo keeps being emitted. + Memos of surviving modules stay. + """ + mod_file = tmp_path / "purged_memo_mod.py" + mod_file.write_text( + "import reflex as rx\n\n@rx.memo\ndef my_card() -> rx.Component:\n" + " return rx.text('card')\n" + ) + + read_fd, write_fd = os.pipe() + pid = os.fork() + if pid == 0: # child + os.close(read_fd) + result = b"E" + try: + import importlib.util + import sys + + from reflex_base.components.memo import MEMOS + + def load(source: str) -> None: + mod_file.write_text(source) + spec = importlib.util.spec_from_file_location( + "purged_memo_mod", mod_file + ) + assert spec is not None + module = importlib.util.module_from_spec(spec) + sys.modules["purged_memo_mod"] = module + spec.loader.exec_module(module) # type: ignore[union-attr] + + load(mod_file.read_text()) + surviving = {key for key in MEMOS if key[1] != "purged_memo_mod"} + assert ("MyCard", "purged_memo_mod") in MEMOS + + compile_daemon._reset_first_party([tmp_path.resolve()]) + dropped = ("MyCard", "purged_memo_mod") not in MEMOS + kept = surviving <= set(MEMOS) + # The renamed memo re-registers under its export name without a + # collision with the parent's registration. + load(mod_file.read_text().replace("def my_card", "def _my_card")) + renamed = ("MyCard", "purged_memo_mod") in MEMOS + result = b"1" if (dropped and kept and renamed) else b"0" + except Exception: + import traceback + + traceback.print_exc() + finally: + os.write(write_fd, result) + os.close(write_fd) + os._exit(0) + + os.close(write_fd) + out = os.read(read_fd, 1) + os.close(read_fd) + os.waitpid(pid, 0) + assert out == b"1" + + def test_reset_first_party_purges_modules_and_registries(tmp_path): """``_reset_first_party`` purges first-party modules and clears registries. From 8894621b0793c35803939ee76446113b9b4bf068 Mon Sep 17 00:00:00 2001 From: Claude <noreply@anthropic.com> Date: Wed, 21 Oct 2026 21:15:32 +0000 Subject: [PATCH 51/56] fix(compiler): deterministic generated names across import, pages, states and processes The post-page phase no longer reseeds the import-time sequence: it draws from its own sequence with the names drawn before the process's first compile reserved, so a component created after a compile cannot repeat a module-level component's ref name. A state's default values draw names from a sequence seeded by the state's full name, so the compile cache can merge values recorded by different processes without duplicate ids. The backend's state-registering page evaluation seeds per route like the compiling process. The compile daemon's child resets the generator before re-importing first-party code, so it draws the names a cold process draws. The per-page generator snapshot is dropped once the page rendered. Claude-Session: https://claude.ai/code/session_01RgKnnN8CCBqaR58qm23zQp --- .../src/reflex_base/plugins/compiler.py | 12 +++- .../reflex-base/src/reflex_base/vars/base.py | 63 ++++++++++++++++++- reflex/app.py | 16 +++-- reflex/state.py | 17 ++++- reflex/utils/compile_daemon.py | 3 + tests/units/compiler/test_compiler_utils.py | 44 +++++++++++++ tests/units/compiler/test_disk_cache.py | 22 +++++++ tests/units/utils/test_compile_daemon.py | 12 ++++ tests/units/vars/test_base.py | 34 ++++++++++ 9 files changed, 208 insertions(+), 15 deletions(-) diff --git a/packages/reflex-base/src/reflex_base/plugins/compiler.py b/packages/reflex-base/src/reflex_base/plugins/compiler.py index 29f30691461..574309edf93 100644 --- a/packages/reflex-base/src/reflex_base/plugins/compiler.py +++ b/packages/reflex-base/src/reflex_base/plugins/compiler.py @@ -724,12 +724,16 @@ def compile( from reflex.state import all_base_state_classes from reflex_base.components.memo import registration_owner from reflex_base.vars.base import ( - reset_unique_variable_names, + names_before_first_compile, restore_unique_variable_name_state, seed_unique_variable_names, unique_variable_name_state, ) + # The names drawn before the compile (at import, in the serving + # process) stay taken: whatever draws names after the pages must not + # repeat them. + names_before = names_before_first_compile() self.ensure_context_attached() self.compiled_pages.clear() self.all_imports.clear() @@ -797,6 +801,7 @@ def compile( strict=True, ): restore_unique_variable_name_state(page_ctx.unique_name_state) + page_ctx.unique_name_state = None with page_ctx: page_ctx.root_component = self.hooks.compile_component( page_ctx.root_component, @@ -822,8 +827,9 @@ def compile( render_progress() # Whatever compiles after the pages (app root, memo modules) starts - # from the same state on every compile. - reset_unique_variable_names() + # from the same state on every compile, and never repeats a name the + # process drew before the pages. + seed_unique_variable_names("app", reserved=names_before) return self.compiled_pages def absorb(self, other: CompileContext, pages: Sequence[PageDefinition]) -> None: diff --git a/packages/reflex-base/src/reflex_base/vars/base.py b/packages/reflex-base/src/reflex_base/vars/base.py index 4ada246817d..befe85d26c0 100644 --- a/packages/reflex-base/src/reflex_base/vars/base.py +++ b/packages/reflex-base/src/reflex_base/vars/base.py @@ -17,7 +17,7 @@ import string import warnings from abc import ABCMeta -from collections.abc import Callable, Coroutine, Iterable, Mapping, Sequence +from collections.abc import Callable, Coroutine, Iterable, Iterator, Mapping, Sequence from dataclasses import MISSING from decimal import Decimal from types import CodeType, FunctionType @@ -3721,17 +3721,44 @@ def get_uuid_string_var() -> Var: _UNIQUE_NAME_RNG = random.Random(42) +#: The names the process drew before its first compile (at import), which +#: every compile's post-page phase keeps reserved (see +#: :func:`names_before_first_compile`). +_names_before_first_compile: frozenset[str] | None = None + + def reset_unique_variable_names() -> None: """Reset the deterministic unique-name generator to its initial state. Names only need to be unique within one compile, so resetting before each - compile makes auto-generated ref names reproducible. + compile makes auto-generated ref names reproducible. A process that + re-imports its first-party code (the compile daemon's child) resets + first, so the re-imported modules draw the names a cold process draws. """ + global _names_before_first_compile USED_VARIABLES.clear() _UNIQUE_NAME_RNG.seed(42) + _names_before_first_compile = None + + +def names_before_first_compile() -> frozenset[str]: + """The names drawn before the first compile of this process. + + Module-level components draw their names at import, and the process + keeps using them; the names a compile draws after its pages (the app + root, runtime components) must not repeat them. Captured once so every + compile of the process draws the same post-page names. + + Returns: + The reserved names. + """ + global _names_before_first_compile + if _names_before_first_compile is None: + _names_before_first_compile = frozenset(USED_VARIABLES) + return _names_before_first_compile -def seed_unique_variable_names(seed: str) -> None: +def seed_unique_variable_names(seed: str, *, reserved: Iterable[str] = ()) -> None: """Start a fresh, seed-determined name sequence (one per compiled page). Generated names are order-dependent, so a process-wide sequence gives a @@ -3742,11 +3769,41 @@ def seed_unique_variable_names(seed: str) -> None: Args: seed: The page route (or another stable label). + reserved: Names the sequence must not hand out again (names drawn + before the compile, which the process keeps using). """ USED_VARIABLES.clear() + USED_VARIABLES.update(reserved) _UNIQUE_NAME_RNG.seed(f"reflex-unique-names:{seed}") +@contextlib.contextmanager +def seeded_unique_variable_names(seed: str) -> Iterator[None]: + """Draw names from a seed-determined sequence within the block. + + The names drawn in the block are a function of ``seed`` alone, so two + processes evaluating the same thing (a state's default values, say) + agree on them whatever they drew before. The outer sequence resumes + afterwards, with the names drawn in the block reserved. + + Args: + seed: A stable label for what the block evaluates. + + Yields: + None. + """ + rng_state = _UNIQUE_NAME_RNG.getstate() + used_before = set(USED_VARIABLES) + seed_unique_variable_names(seed) + try: + yield + finally: + drawn = set(USED_VARIABLES) + _UNIQUE_NAME_RNG.setstate(rng_state) + USED_VARIABLES.clear() + USED_VARIABLES.update(used_before | drawn) + + def unique_variable_name_state() -> tuple[Any, frozenset[str]]: """Capture the generator state so a page's sequence can resume later. diff --git a/reflex/app.py b/reflex/app.py index 102e19e4dd3..d64dec437da 100644 --- a/reflex/app.py +++ b/reflex/app.py @@ -60,6 +60,7 @@ Send, is_non_string_sequence, ) +from reflex_base.vars.base import seeded_unique_variable_names from reflex_base.vars.dep_tracking import is_dependency from reflex_components_core.base.error_boundary import ErrorBoundary from reflex_components_core.base.fragment import Fragment @@ -1361,12 +1362,15 @@ def _compile_page(self, route: str, save_page: bool = True): return n_states_before = len(all_base_state_classes) - component = compiler.compile_unevaluated_page( - route, - self._unevaluated_pages[route], - self.style, - self.theme, - ) + # The same names the compiling process drew for this page (a + # ``ClientStateVar`` created inside the page function is used by both). + with seeded_unique_variable_names(route): + component = compiler.compile_unevaluated_page( + route, + self._unevaluated_pages[route], + self.style, + self.theme, + ) # Indicate that evaluating this page creates one or more state classes. if len(all_base_state_classes) > n_states_before: diff --git a/reflex/state.py b/reflex/state.py index cccfc1023fd..79787374d11 100644 --- a/reflex/state.py +++ b/reflex/state.py @@ -70,6 +70,7 @@ computed_var, dispatch, is_computed_var, + seeded_unique_variable_names, ) from rich.markup import escape @@ -2007,9 +2008,19 @@ def dict( """ if include_computed: self._mark_dirty_computed_vars() - base_vars = { - prop_name: self.get_value(prop_name) for prop_name in self.base_vars - } + if initial: + # A compile serializes the initial values; a default component's + # generated names must not depend on which states came first (a + # set's order varies by process, and the compile cache merges + # values recorded by different processes). + with seeded_unique_variable_names(self.get_full_name()): + base_vars = { + prop_name: self.get_value(prop_name) for prop_name in self.base_vars + } + else: + base_vars = { + prop_name: self.get_value(prop_name) for prop_name in self.base_vars + } if initial and include_computed: computed_vars = { # Include initial computed vars. diff --git a/reflex/utils/compile_daemon.py b/reflex/utils/compile_daemon.py index f4fd1db617f..387f56c1f5c 100644 --- a/reflex/utils/compile_daemon.py +++ b/reflex/utils/compile_daemon.py @@ -611,6 +611,7 @@ def _reset_first_party(roots: list[Path]) -> None: from reflex_base.components.memo import MEMOS from reflex_base.registry import RegistrationContext + from reflex_base.vars.base import reset_unique_variable_names import reflex.istate.dynamic as istate_dynamic from reflex.compiler import page_cache @@ -621,6 +622,8 @@ def _reset_first_party(roots: list[Path]) -> None: # old registration, or a memo file the app no longer defines). for key in [key for key in MEMOS if key[1] in purged]: del MEMOS[key] + # The re-imported modules draw the names a cold process draws at import. + reset_unique_variable_names() ctx = RegistrationContext.ensure_context() kept = [ cls diff --git a/tests/units/compiler/test_compiler_utils.py b/tests/units/compiler/test_compiler_utils.py index e69c349c859..d092446ac95 100644 --- a/tests/units/compiler/test_compiler_utils.py +++ b/tests/units/compiler/test_compiler_utils.py @@ -329,3 +329,47 @@ def test_document_root_controls_preserve_no_id_and_page_refs(): assert not document_root._get_all_hooks() assert page_script._get_all_hooks() + + +def test_compile_state_default_component_names_do_not_depend_on_state_order(): + """A default component's generated names are a function of its state. + + Substates are a set, whose order varies by process; the compile cache + merges initial values recorded by different processes, which must agree. + """ + from reflex_base.vars.base import reset_unique_variable_names + + with RegistrationContext(): + + class _OrderRoot(rx.State): + pass + + class _OrderOne(_OrderRoot): + comp: rx.Field[rx.Component] = rx.field( + default_factory=lambda: rx.auto_scroll() + ) + + class _OrderTwo(_OrderRoot): + comp: rx.Field[rx.Component] = rx.field( + default_factory=lambda: rx.auto_scroll() + ) + + def serialized(substates: list[type[rx.State]]) -> str: + reset_unique_variable_names() + original = _OrderRoot.get_substates + _OrderRoot.get_substates = classmethod( # type: ignore[method-assign] + lambda cls: substates if cls is _OrderRoot else set() + ) + try: + return utils._serialize_initial_state(compile_state(_OrderRoot)) + finally: + _OrderRoot.get_substates = original # type: ignore[method-assign] + + first = serialized([_OrderOne, _OrderTwo]) + second = serialized([_OrderTwo, _OrderOne]) + assert first == second + one, two = ( + first.index(_OrderOne.get_full_name()), + first.index(_OrderTwo.get_full_name()), + ) + assert one != two diff --git a/tests/units/compiler/test_disk_cache.py b/tests/units/compiler/test_disk_cache.py index 4def39a8fb8..55e127965ab 100644 --- a/tests/units/compiler/test_disk_cache.py +++ b/tests/units/compiler/test_disk_cache.py @@ -703,6 +703,28 @@ def test_unchanged_pages_compile_identically(tmp_path, monkeypatch): ) +def test_names_drawn_after_the_pages_never_repeat_import_time_names(): + """The post-page phase keeps the names the process drew before the compile. + + A module-level component draws its ref name at import; after the pages, + the app root and runtime components draw more. Reseeding the import + sequence would hand the same names out again. + """ + from reflex_base.vars.base import ( + get_unique_variable_name, + reset_unique_variable_names, + ) + + reset_unique_variable_names() + import_names = {get_unique_variable_name() for _ in range(3)} + _compile([_FakePage(route="/a", component=_page_a)]) + after = {get_unique_variable_name() for _ in range(50)} + assert not after & import_names + # And the post-page sequence is the same on every compile. + _compile([_FakePage(route="/b", component=_page_b)]) + assert {get_unique_variable_name() for _ in range(50)} == after + + _CONTEXTS_STUB = "// contexts stub" diff --git a/tests/units/utils/test_compile_daemon.py b/tests/units/utils/test_compile_daemon.py index b72db895b11..bf651b94b0f 100644 --- a/tests/units/utils/test_compile_daemon.py +++ b/tests/units/utils/test_compile_daemon.py @@ -535,6 +535,12 @@ class LocalState(rx.State): assert local_cls.__module__ == istate_dynamic.__name__ local_name = local_cls.__name__ + from reflex_base.vars.base import ( + get_unique_variable_name, + reset_unique_variable_names, + ) + + get_unique_variable_name() # the parent drew names compile_daemon._reset_first_party([tmp_path.resolve()]) substate_names = {c.__name__ for c in State.get_substates()} @@ -552,6 +558,11 @@ class LocalState(rx.State): # A re-created local state gets its original (fresh-process) name, # not a collision-suffixed drift. redefined_deterministic = _local_state().__name__ == local_name + # Re-imported modules draw the names a cold process draws. + child_names = [get_unique_variable_name() for _ in range(3)] + reset_unique_variable_names() + cold_names = [get_unique_variable_name() for _ in range(3)] + names_reset = child_names == cold_names result = ( b"1" @@ -560,6 +571,7 @@ class LocalState(rx.State): and purged_dropped and dynamic_dropped and redefined_deterministic + and names_reset ) else b"0" ) diff --git a/tests/units/vars/test_base.py b/tests/units/vars/test_base.py index 439d66b20bc..c203ab96eae 100644 --- a/tests/units/vars/test_base.py +++ b/tests/units/vars/test_base.py @@ -101,6 +101,40 @@ def test_reset_unique_variable_names_is_deterministic(): assert [get_unique_variable_name() for _ in range(8)] == first +def test_seeded_unique_variable_names_block_resumes_the_outer_sequence(): + """A seeded block draws by its seed alone and keeps its names reserved.""" + from reflex_base.vars.base import ( + USED_VARIABLES, + get_unique_variable_name, + reset_unique_variable_names, + seed_unique_variable_names, + seeded_unique_variable_names, + ) + + reset_unique_variable_names() + outer_first = get_unique_variable_name() + with seeded_unique_variable_names("state.one"): + block_names = [get_unique_variable_name() for _ in range(3)] + outer_next = get_unique_variable_name() + # The block's names are a function of its seed, whatever ran before. + seed_unique_variable_names("unrelated") + get_unique_variable_name() + with seeded_unique_variable_names("state.one"): + assert [get_unique_variable_name() for _ in range(3)] == block_names + # The outer sequence resumed where it stopped, with the block's names taken. + reset_unique_variable_names() + assert get_unique_variable_name() == outer_first + with seeded_unique_variable_names("state.one"): + assert set(block_names) & USED_VARIABLES == set() + assert [get_unique_variable_name() for _ in range(3)] == block_names + assert set(block_names) <= USED_VARIABLES + assert get_unique_variable_name() == outer_next + # Reserved names are never handed out again by a seeded sequence. + seed_unique_variable_names("app", reserved=block_names) + assert set(block_names) <= USED_VARIABLES + assert get_unique_variable_name() not in block_names + + def test_seeded_unique_variable_names_are_per_page(): """Each page seed yields its own reproducible sequence, resumable later. From 0d144dcd01efa24a150d279ff01639c4376d038b Mon Sep 17 00:00:00 2001 From: Claude <noreply@anthropic.com> Date: Wed, 21 Oct 2026 21:15:32 +0000 Subject: [PATCH 52/56] fix(compile-daemon): keep library-created dynamic states, private bytecode cache, worker-only deferral, restart by exit code, locks outside .web, child telemetry via the daemon The child's registry reset now keeps a reflex.istate.dynamic state that a surviving (non-first-party) module created at import, and drops the ones the manifest records as defined by a page, which re-evaluates. The daemon and its children write bytecode to a private cache and drop the entries of the changed files before re-importing them, so a second same-size edit within a second no longer imports stale bytecode. Only backend reload workers defer to a running daemon: reflex compile and reflex export compile themselves. The watcher tests skip directories below the project root, so a project under a directory named venv is watched. A daemon stopped mid-compile kills its child on the way out, and a child dies with its parent on Linux. A global config change exits with a restart code that reflex run answers by starting a new daemon, on every OS, instead of an in-place execv. The lock files live in the per-user data directory, so rm -rf .web cannot unlink a held lock. A compile child spools its telemetry events for the daemon to send once it exited, which keeps the forked child free of the telemetry thread and macOS's post-fork proxy lookup; the duplicated executor shutdown is gone. Also: a cold compile no longer forks a predicting child when there is no manifest, and takes its component counts from the manifest when the child compiled; sets are sorted only in compiled literals, not on the state delta path. Claude-Session: https://claude.ai/code/session_01RgKnnN8CCBqaR58qm23zQp --- .../src/reflex_base/environment.py | 6 + .../src/reflex_base/utils/serializers.py | 21 +- .../src/reflex_base/vars/sequence.py | 13 +- reflex/compiler/compiler.py | 6 + reflex/compiler/disk_cache.py | 24 +- reflex/utils/compile_daemon.py | 428 ++++++++++++++---- reflex/utils/telemetry.py | 48 +- tests/units/test_telemetry.py | 20 + tests/units/utils/test_compile_daemon.py | 266 ++++++++++- tests/units/utils/test_serializers.py | 23 +- 10 files changed, 720 insertions(+), 135 deletions(-) diff --git a/packages/reflex-base/src/reflex_base/environment.py b/packages/reflex-base/src/reflex_base/environment.py index f8492a26b67..efc0e724af3 100644 --- a/packages/reflex-base/src/reflex_base/environment.py +++ b/packages/reflex-base/src/reflex_base/environment.py @@ -763,6 +763,12 @@ class EnvironmentVariables: # workers of the same `reflex run` skip frontend compilation. REFLEX_COMPILE_DAEMON: EnvVar[bool] = env_var(False, internal=True) + # Set by the compile daemon for its compile children: a file the child + # appends its telemetry events to instead of sending them, which the + # daemon sends once the child exited (a forked child must not start the + # telemetry thread, nor touch macOS's proxy lookup after fork). + REFLEX_TELEMETRY_SPOOL: EnvVar[Path | None] = env_var(None, internal=True) + # Inherited by uvicorn/granian reload workers so the backend can distinguish # dev reload-capable worker boots from other backend starts. Never set in prod. REFLEX_DEV_BACKEND_RELOAD_ACTIVE: EnvVar[bool] = env_var(False, internal=True) diff --git a/packages/reflex-base/src/reflex_base/utils/serializers.py b/packages/reflex-base/src/reflex_base/utils/serializers.py index 45e957ec4e1..ed2ef1816e8 100644 --- a/packages/reflex-base/src/reflex_base/utils/serializers.py +++ b/packages/reflex-base/src/reflex_base/utils/serializers.py @@ -402,16 +402,29 @@ def serialize_base_model(model: BaseModel) -> dict: def serialize_set(value: set) -> list: """Serialize a set to a JSON serializable list. - Sets have no meaningful order and their iteration order varies with the - per-process hash seed, so sort when possible to keep serialized output - (compiled JSX, config fingerprints) stable across processes. - Args: value: The set to serialize. Returns: The serialized list. """ + return list(value) + + +def sorted_set(value: set | frozenset) -> list: + """Order a set for output that must be the same in every process. + + A set's iteration order varies with the per-process hash seed; compiled + JSX embeds set literals, so a compile sorts them (a state delta does + not: its order carries no meaning and sorting costs on every update). + + Args: + value: The set. + + Returns: + The elements, sorted; elements of mixed types sort by their + serialized form. + """ try: return sorted(value) except TypeError: diff --git a/packages/reflex-base/src/reflex_base/vars/sequence.py b/packages/reflex-base/src/reflex_base/vars/sequence.py index 1ded6c57ab1..3e7e0a096c3 100644 --- a/packages/reflex-base/src/reflex_base/vars/sequence.py +++ b/packages/reflex-base/src/reflex_base/vars/sequence.py @@ -15,7 +15,7 @@ from reflex_base import constants from reflex_base.constants.base import REFLEX_VAR_OPENING_TAG, Dirs -from reflex_base.utils import console, types +from reflex_base.utils import console, serializers, types from reflex_base.utils.exceptions import VarTypeError from reflex_base.utils.imports import ImportDict, ImportVar from reflex_base.utils.types import GenericType, get_origin, unionize @@ -696,11 +696,18 @@ def create( Returns: The var. """ + if _var_type is None: + _var_type = figure_out_type(value) + # Compiled output embeds the literal; a set's order varies with the + # hash seed, so it is sorted once here, not on the delta path. + elements: Any = ( + serializers.sorted_set(value) if isinstance(value, set) else value + ) return LiteralArrayVar( _js_expr="", - _var_type=figure_out_type(value) if _var_type is None else _var_type, + _var_type=_var_type, _var_data=_var_data, - _var_value=value, + _var_value=elements, ) diff --git a/reflex/compiler/compiler.py b/reflex/compiler/compiler.py index dfced7aca66..99277f23da7 100644 --- a/reflex/compiler/compiler.py +++ b/reflex/compiler/compiler.py @@ -1588,6 +1588,12 @@ def attempt() -> CompileContext | bool: ) if compile_daemon.compiled_in_predicting_child(lambda: attempt() is True): + # The child's pages never existed here; the manifest it wrote has + # their counts for the compile telemetry. + if (manifest := disk_cache.load_manifest()) is not None: + app._cached_component_counts = disk_cache.manifest_component_counts( + manifest + ) return True rebuilt = attempt() if isinstance(rebuilt, CompileContext): diff --git a/reflex/compiler/disk_cache.py b/reflex/compiler/disk_cache.py index a09b689cce7..a7d1d665d1b 100644 --- a/reflex/compiler/disk_cache.py +++ b/reflex/compiler/disk_cache.py @@ -2176,13 +2176,27 @@ def try_incremental_rebuild( ): _save_incremental_output(_manifest_path(), _manifest_text(manifest), outputs) + app._cached_component_counts = manifest_component_counts(manifest) + return True + + +def manifest_component_counts(manifest: dict[str, Any]) -> dict[str, int]: + """Sum the component counts the manifest records per page. + + The compile telemetry of a process that reused every page from disk, or + compiled in a child, has no evaluated pages to count. + + Args: + manifest: The loaded manifest. + + Returns: + The count of each component class across the pages. + """ counts: dict[str, int] = {} - for page in pages: - page_counts = manifest["pages"][page.route].get("component_counts", {}) - for name, count in page_counts.items(): + for entry in manifest["pages"].values(): + for name, count in entry.get("component_counts", {}).items(): counts[name] = counts.get(name, 0) + count - app._cached_component_counts = counts - return True + return counts def _update_manifest_for_misses( diff --git a/reflex/utils/compile_daemon.py b/reflex/utils/compile_daemon.py index 387f56c1f5c..a3f2adba915 100644 --- a/reflex/utils/compile_daemon.py +++ b/reflex/utils/compile_daemon.py @@ -10,6 +10,7 @@ import contextlib import importlib +import importlib.util import json import logging import os @@ -49,15 +50,36 @@ _COMPILE_TIMEOUT = 300.0 #: Source suffixes edited under the app roots that should trigger a recompile. _WATCH_SUFFIXES = (".py", ".md", ".mdx") -#: Directories never worth walking while building the watch snapshot. -_SKIP_DIRS = { +#: Directories never worth walking while building the watch snapshot, or +#: reacting to changes in. Tested against the parts below the root a path is +#: under, so a project living in a directory of such a name is still watched. +_SKIP_DIRS = frozenset({ constants.Dirs.WEB, ".venv", "venv", "node_modules", "__pycache__", ".git", -} +}) +#: Exit code of the daemon asking ``reflex run`` to start it afresh: a change +#: to a global input (rxconfig, a lockfile, a Reflex upgrade) cannot be +#: applied to the warm parent, which imported the old version. +_RESTART_EXIT_CODE = 4 + + +def _stop(proc: subprocess.Popen) -> None: + """Stop a daemon process, killing it if it ignores the request. + + Args: + proc: The daemon process. + """ + if proc.poll() is None: + proc.terminate() + try: + proc.wait(timeout=3) + except subprocess.TimeoutExpired: + proc.kill() + proc.wait() @contextlib.contextmanager @@ -72,7 +94,8 @@ def running( its own process so that it stays single-threaded, which keeps its per-edit ``fork()`` safe, and its environment stays separate from the backend's. If ``reflex run`` is killed outright, the daemon exits on noticing its parent - is gone (see :func:`_serve`). + is gone (see :func:`_serve`). A daemon that exits asking for a restart + (see :data:`_RESTART_EXIT_CODE`) is started again, on every OS. Args: prerender_routes: Whether the daemon should prerender routes when compiling. @@ -90,19 +113,39 @@ def running( env["REFLEX_PRERENDER_ROUTES"] = "1" if initial_compiled: env["REFLEX_COMPILE_DAEMON_PRECOMPILED"] = "1" - proc = subprocess.Popen( - [sys.executable, "-m", "reflex.utils.compile_daemon"], env=env + + def spawn() -> subprocess.Popen: + return subprocess.Popen( + [sys.executable, "-m", "reflex.utils.compile_daemon"], env=env + ) + + lock = threading.Lock() + stopping = False + proc = spawn() + + def supervise() -> None: + nonlocal proc + while True: + current = proc + current.wait() + with lock: + if stopping or current.returncode != _RESTART_EXIT_CODE: + return + logger.info("Compile daemon: restarting after a global config change.") + proc = spawn() + + supervisor = threading.Thread( + target=supervise, name="reflex-compile-daemon", daemon=True ) + supervisor.start() try: yield finally: - if proc.poll() is None: - proc.terminate() - try: - proc.wait(timeout=3) - except subprocess.TimeoutExpired: - proc.kill() - proc.wait() + with lock: + stopping = True + current = proc + _stop(current) + supervisor.join(timeout=3) def _reload_roots() -> list[Path]: @@ -293,7 +336,7 @@ def accepts(self, path: Path) -> bool: True for watched-suffix sources, recorded dependencies, global inputs and assets; False for build output and unrelated files. """ - if any(part in _SKIP_DIRS for part in path.parts): + if _in_skipped_dir(path, [self.root, *self.roots]): return False if path in self.globals_ or path in self.known: return True @@ -436,9 +479,31 @@ def watch_filter(_change: object, raw_path: str) -> bool: return None +def _in_skipped_dir(path: Path, roots: Iterable[Path]) -> bool: + """Whether ``path`` lies in a build or dependency directory. + + Args: + path: The path to test. + roots: The roots the path may be under; only the parts below the + containing root count. + + Returns: + True when a directory on the way names a skipped directory. + """ + parts = path.parts + for root in roots: + if _under_roots(path, [root]): + parts = parts[len(root.parts) :] + break + return not _SKIP_DIRS.isdisjoint(parts) + + def _dependency_snapshot(state: _WatchState) -> dict[Path, int]: """Snapshot all watchable inputs while the OS watcher is stopped. + Taken once per compile (around the watcher's teardown), never per idle + watcher batch. + Args: state: The watch state whose known dependencies to stat. @@ -595,8 +660,10 @@ def _reset_first_party(roots: list[Path]) -> None: workspace packages) never re-executes in this process, so clearing its registration would lose the state from the app's state tree — and from the compiled contexts file — permanently. Those registrations are kept; states - from purged modules re-register on re-import, and runtime-created states in - ``reflex.istate.dynamic`` re-register when their page re-evaluates. + from purged modules re-register on re-import, and the runtime-created + states in ``reflex.istate.dynamic`` re-register when the code that + created them runs again: a page re-evaluating, or a purged module + re-importing. One a surviving module created at import is kept too. Args: roots: The resolved reload roots whose modules are first-party. @@ -625,12 +692,11 @@ def _reset_first_party(roots: list[Path]) -> None: # The re-imported modules draw the names a cold process draws at import. reset_unique_variable_names() ctx = RegistrationContext.ensure_context() + page_states = _page_defined_states() kept = [ cls - for cls in ctx.base_states.values() - if (module_name := getattr(cls, "__module__", None)) is not None - and module_name != istate_dynamic.__name__ - and getattr(sys.modules.get(module_name), "__file__", None) is not None + for name, cls in ctx.base_states.items() + if name not in page_states and _survives_purge(cls) ] ctx.base_states.clear() ctx.base_state_substates.clear() @@ -643,8 +709,15 @@ def _reset_first_party(roots: list[Path]) -> None: # collision-suffixed names; with the warm parent's attributes in place, # every re-created state would drift to a new suffix and diverge from the # names the (cold) backend computes. Reset the module so re-created states - # get their fresh-process names. - for attr in [name for name in vars(istate_dynamic) if not name.startswith("__")]: + # get their fresh-process names, keeping the names of the kept ones. + kept_dynamic = { + cls.__name__ for cls in kept if cls.__module__ == istate_dynamic.__name__ + } + for attr in [ + name + for name in vars(istate_dynamic) + if not name.startswith("__") and name not in kept_dynamic + ]: delattr(istate_dynamic, attr) # Original registration order, so parents always precede their children. for cls in kept: @@ -661,6 +734,62 @@ def _reset_first_party(roots: list[Path]) -> None: _reset_model_metadata() +def _page_defined_states() -> set[str]: + """The states the last compile recorded as defined by a page's evaluation. + + They are re-created when the page re-evaluates, so they never survive a + reset; without a manifest, every ``reflex.istate.dynamic`` state is + taken for one. + + Returns: + The full names of the page-defined states. + """ + from reflex.compiler import disk_cache + + if (manifest := disk_cache.load_manifest()) is None: + return set() + return disk_cache._hit_page_states(manifest, set()) + + +def _survives_purge(cls: type) -> bool: + """Whether a registered state stays registered across the first-party purge. + + A state's code re-registers it when it runs again, which only happens + for a purged module: a module-level state stays iff its module survives + the purge, and a ``reflex.istate.dynamic`` state (a function-local class, + a ``ComponentState`` instance) iff the module that created it at import + survives. A state a page defines is re-created when the page + re-evaluates (see :func:`_page_defined_states`); without a manifest to + tell them apart, every dynamic state is dropped. + + Args: + cls: The registered state class. + + Returns: + True when the state is kept. + """ + import reflex.istate.dynamic as istate_dynamic + + module_name = getattr(cls, "__module__", None) + if module_name == istate_dynamic.__name__: + from reflex.compiler import disk_cache + + if disk_cache.load_manifest() is None: + return False + # A local state remembers its module; a ComponentState instance is + # created from a class defined in its creating module. + module_name = cls.__dict__.get("__original_module__") + if module_name is None: + origin = cls.__bases__[0] + module_name = getattr(origin, "__module__", None) + if module_name == istate_dynamic.__name__: + return False + if module_name is None: + return False + module = sys.modules.get(module_name) + return module is not None and getattr(module, "__file__", None) is not None + + def _reset_model_metadata() -> None: """Clear the SQLAlchemy/SQLModel table + model registries. @@ -686,14 +815,23 @@ def _reset_model_metadata() -> None: #: Name of the env var carrying the changed-path hint to a cold compile child. _CHANGED_ENV = "REFLEX_COMPILE_CHANGED" +#: Where the daemon and its children write bytecode caches (under the +#: backend dir). Timestamp-based ``.pyc`` files are keyed by whole-second +#: mtime and size, so a second same-size edit within a second of the compile +#: that cached the first would import stale bytecode; a private cache lets +#: the daemon drop the entries of the files that changed, with no other +#: process writing them back. +_PYCACHE_DIR = ".pycache_daemon" #: Name of the env var telling a cold compile child to predict dynamic state names. _PREDICT_ENV = "REFLEX_COMPILE_PREDICT_DYNAMIC_NAMES" #: Exit code of a compile child whose dynamic state name prediction failed; the #: daemon compiles again in a fresh child without prediction. _RETRY_EXIT_CODE = 3 -#: Lock file (under the backend dir) whose OS lock the daemon holds while a -#: compile is in progress, so a backend reload worker can wait for the fresh -#: stateful-pages marker. +#: Lock whose OS lock the daemon holds while a compile is in progress, so a +#: backend reload worker can wait for the fresh stateful-pages marker. The +#: lock files live in Reflex's per-user data directory, keyed by the backend +#: dir they guard (see ``path_ops._file_lock_path``): a ``rm -rf .web`` while +#: the daemon runs must not unlink the lock it holds. _COMPILE_LOCK_FILE = ".compile_in_progress" #: Give up waiting on the lock after this long (a wedged daemon must not hang #: the backend forever; its own watchdog is ``_COMPILE_TIMEOUT``). @@ -705,11 +843,10 @@ def _reset_model_metadata() -> None: #: Briefly retry Windows readers that prevent atomically replacing the record. _BUILT_INPUTS_WRITE_ATTEMPTS = 3 _BUILT_INPUTS_RETRY_DELAY = 0.01 -#: Lock file (under the backend dir) whose OS lock the daemon holds for its -#: lifetime. Backend workers skip frontend compilation while it is held. A -#: file, not an environment variable: workers may be forked from a -#: ``forkserver`` whose environment predates ``reflex run`` deciding to start -#: the daemon. +#: Lock whose OS lock the daemon holds for its lifetime. Backend reload +#: workers skip frontend compilation while it is held. A file, not an +#: environment variable: workers may be forked from a ``forkserver`` whose +#: environment predates ``reflex run`` deciding to start the daemon. _DAEMON_LOCK_FILE = ".compile_daemon" #: How long a lock holder waits out a worker's momentary probe of the lock. _LOCK_ACQUIRE_TIMEOUT = 1.0 @@ -719,22 +856,26 @@ def _lock_path() -> Path: """Return the compile-in-progress lock file. Returns: - The lock file under the backend dir. + The lock file, in the per-user data directory. """ from reflex.utils import prerequisites - return prerequisites.get_backend_dir() / _COMPILE_LOCK_FILE + return path_ops._file_lock_path( + prerequisites.get_backend_dir().resolve() / _COMPILE_LOCK_FILE + ) def _daemon_lock_path() -> Path: """Return the daemon ownership lock file. Returns: - The lock file under the backend dir. + The lock file, in the per-user data directory. """ from reflex.utils import prerequisites - return prerequisites.get_backend_dir() / _DAEMON_LOCK_FILE + return path_ops._file_lock_path( + prerequisites.get_backend_dir().resolve() / _DAEMON_LOCK_FILE + ) def _acquire_lock(path: Path) -> int | None: @@ -768,14 +909,20 @@ def owns_compilation() -> bool: False for a backend reload worker while a compile daemon owns the build; the daemon (and its forked compile children) carry - ``REFLEX_COMPILE_DAEMON`` and always own it. Decided from the on-disk - lock rather than inherited environment, which a ``forkserver`` started - before ``reflex run`` chose the daemon would not carry. + ``REFLEX_COMPILE_DAEMON`` and always own it, and so does every other + process (``reflex compile``, ``reflex export``): they asked for a + compile. Whether a daemon runs is decided from the on-disk lock rather + than inherited environment, which a ``forkserver`` started before + ``reflex run`` chose the daemon would not carry. Returns: True if this process compiles the frontend. """ - return environment.REFLEX_COMPILE_DAEMON.get() or not daemon_active() + return ( + environment.REFLEX_COMPILE_DAEMON.get() + or not environment.REFLEX_DEV_BACKEND_RELOAD_ACTIVE.get() + or not daemon_active() + ) @contextlib.contextmanager @@ -900,6 +1047,27 @@ def _prepare_fork_parent(roots: list[Path]) -> None: page_cache.warm_module_file_cache() +def _use_private_pycache() -> None: + """Point this process's bytecode cache at the daemon's private directory.""" + from reflex.utils import prerequisites + + sys.pycache_prefix = str(prerequisites.get_backend_dir().resolve() / _PYCACHE_DIR) + + +def _forget_bytecode(changed: Iterable[Path]) -> None: + """Drop the cached bytecode of the changed modules before re-importing them. + + Args: + changed: The changed paths. + """ + for path in changed: + if path.suffix == ".py": + with contextlib.suppress(OSError): + Path(importlib.util.cache_from_source(str(path))).unlink( + missing_ok=True + ) + + def _child_compile( roots: list[Path], prerender_routes: bool, @@ -923,6 +1091,9 @@ def _child_compile( from reflex.compiler import disk_cache from reflex.utils import prerequisites + _use_private_pycache() + if changed: + _forget_bytecode(changed) disk_cache.set_changed_hint(changed) disk_cache.set_dynamic_name_prediction(predict) # Timed in three steps so every hot reload reports where it spent its time @@ -940,9 +1111,26 @@ def _child_compile( ) +def _kill_child(pid: int) -> None: + """Kill and reap a forked compile child. + + Args: + pid: The child's pid. + """ + with contextlib.suppress(OSError): + os.kill(pid, signal.SIGKILL) + with contextlib.suppress(OSError): + os.waitpid(pid, 0) + + def _await_child(pid: int) -> int: """Reap a forked compile child, killing it if it exceeds the watchdog timeout. + The child never outlives the daemon: a daemon stopped while a compile + runs (SIGTERM from ``reflex run``, Ctrl-C) kills the child on its way + out, so the child does not keep the locks and write ``.web`` after the + run ended. + Args: pid: The forked child's pid. @@ -951,19 +1139,31 @@ def _await_child(pid: int) -> int: child, such as Ctrl-C during shutdown, is a quiet failure). """ deadline = time.monotonic() + _COMPILE_TIMEOUT - while True: - done, status = os.waitpid(pid, os.WNOHANG) - if done == pid: - code = os.waitstatus_to_exitcode(status) - return code if code >= 0 else 1 - if time.monotonic() > deadline: - with contextlib.suppress(OSError): - os.kill(pid, signal.SIGKILL) - with contextlib.suppress(OSError): - os.waitpid(pid, 0) - logger.error("Compile child timed out; killed it, keeping last build.") - return 1 - time.sleep(0.02) + try: + while True: + done, status = os.waitpid(pid, os.WNOHANG) + if done == pid: + code = os.waitstatus_to_exitcode(status) + return code if code >= 0 else 1 + if time.monotonic() > deadline: + _kill_child(pid) + logger.error("Compile child timed out; killed it, keeping last build.") + return 1 + time.sleep(0.02) + except BaseException: + _kill_child(pid) + raise + + +def _die_with_parent() -> None: + """Ask the kernel to kill this forked child when its parent dies (Linux).""" + if sys.platform != "linux": + return + with contextlib.suppress(Exception): + import ctypes + + pr_set_pdeathsig = 1 + ctypes.CDLL("libc.so.6", use_errno=True).prctl(pr_set_pdeathsig, signal.SIGKILL) def _quiesce_parent() -> None: @@ -977,7 +1177,7 @@ def _quiesce_parent() -> None: """ from reflex.utils import telemetry - telemetry.shutdown_executor() + telemetry._shutdown_executor() def _os_thread_count() -> int | None: @@ -1051,7 +1251,7 @@ def compiled_in_predicting_child(attempt: Callable[[], bool]) -> bool: the caller compiles exactly in this process (which also reports errors). Skipped where a caller already chose the mode, such as the daemon's - exact retry. + exact retry, and where there is no manifest to compile from. Args: attempt: Runs the incremental compile; returns True when it compiled. @@ -1065,12 +1265,14 @@ def compiled_in_predicting_child(attempt: Callable[[], bool]) -> bool: disk_cache._predict_dynamic_names is not None or not hasattr(os, "fork") or not _single_threaded() + or disk_cache.load_manifest() is None ): return False pid = os.fork() if pid == 0: # child code = 1 try: + _die_with_parent() disk_cache.set_dynamic_name_prediction(True) code = 0 if attempt() else 2 except disk_cache.DynamicStatePredictionError as exc: @@ -1140,50 +1342,68 @@ def _run_compile_child( a failure code. """ from reflex.compiler import disk_cache - - if _can_fork(): - pid = os.fork() - if pid == 0: # child - code = 0 - try: - _child_compile(roots, prerender_routes, changed, predict) - except disk_cache.DynamicStatePredictionError as exc: - _log_failed_prediction(exc) - code = _RETRY_EXIT_CODE - except BaseException: # report any failure, never crash the daemon - logger.exception("Hot reload compile failed.") - code = 1 - finally: - os._exit(code) - return _await_child(pid) - - # No fork (Windows) or unsafe to fork: a fresh (cold) subprocess compiles. - env = {**os.environ} - if changed is not None: - env[_CHANGED_ENV] = "\n".join(sorted(str(p) for p in changed)) - if predict: - env[_PREDICT_ENV] = "1" + from reflex.utils import prerequisites, telemetry + + # The child spools its telemetry (the compile event) for the daemon to + # send once the child is gone: a forked child must not start the + # telemetry thread, nor touch macOS's proxy lookup after a fork, and + # ``os._exit`` would drop what it queued. + spool = prerequisites.get_backend_dir().resolve() / f".telemetry.{os.getpid()}" + environment.REFLEX_TELEMETRY_SPOOL.set(spool) try: - proc = subprocess.run( - [sys.executable, "-m", "reflex.utils.compile_daemon", "--once"], - check=False, - timeout=_COMPILE_TIMEOUT, - env=env, - ) - except subprocess.TimeoutExpired: - logger.error("Compile subprocess timed out; keeping the last good build.") - return 1 - return proc.returncode + if _can_fork(): + pid = os.fork() + if pid == 0: # child + code = 0 + try: + _die_with_parent() + _child_compile(roots, prerender_routes, changed, predict) + except disk_cache.DynamicStatePredictionError as exc: + _log_failed_prediction(exc) + code = _RETRY_EXIT_CODE + except BaseException: # report any failure, never crash the daemon + logger.exception("Hot reload compile failed.") + code = 1 + finally: + os._exit(code) + return _await_child(pid) + + # No fork (Windows) or unsafe to fork: a fresh (cold) subprocess compiles. + env = {**os.environ} + if changed is not None: + env[_CHANGED_ENV] = "\n".join(sorted(str(p) for p in changed)) + if predict: + env[_PREDICT_ENV] = "1" + try: + proc = subprocess.run( + [sys.executable, "-m", "reflex.utils.compile_daemon", "--once"], + check=False, + timeout=_COMPILE_TIMEOUT, + env=env, + ) + except subprocess.TimeoutExpired: + logger.error("Compile subprocess timed out; keeping the last good build.") + return 1 + return proc.returncode + finally: + environment.REFLEX_TELEMETRY_SPOOL.set(None) + telemetry.send_spooled(spool) + +def _serve() -> int: + """Run the warm compile daemon: initial compile, then watch-and-recompile. -def _serve() -> None: - """Run the warm compile daemon: initial compile, then watch-and-recompile.""" + Returns: + The process exit code: :data:`_RESTART_EXIT_CODE` to be started + afresh after a global config change, else 0. + """ from reflex.utils import prerequisites root = Path.cwd() prerender_routes = bool(os.environ.get("REFLEX_PRERENDER_ROUTES")) roots = _reload_roots() parent_pid = os.getppid() + _use_private_pycache() # Warm import + initial compile (writes .web + the manifest); keeps the app # and its third-party deps resident for copy-on-write children. A failure @@ -1234,12 +1454,12 @@ def alive() -> bool: # A change to a genuinely-global input (rxconfig/lockfiles, or a reflex # upgrade) can't be applied to the warm parent (it imported the old - # version); re-exec the daemon so the new world is actually loaded. + # version); exit so ``reflex run`` starts a daemon that loads the new + # world. (An in-place ``execv`` would leave ``reflex run`` holding a + # dead handle on Windows, where it starts a new process.) if changed & state.globals_: logger.info("Global config changed; restarting compile daemon.") - os.execv( - sys.executable, [sys.executable, "-m", "reflex.utils.compile_daemon"] - ) + return _RESTART_EXIT_CODE before = ( state.checkpoint @@ -1266,6 +1486,7 @@ def alive() -> bool: pending = _missed_changes(before, state.checkpoint) if appeared := (pending - before.keys()) & state.known: pending -= _built_inputs(appeared) + return 0 def main(argv: list[str] | None = None) -> int: @@ -1303,7 +1524,7 @@ def main(argv: list[str] | None = None) -> int: return 0 # Held for the process lifetime, so backend workers defer to this daemon # exactly while it lives (see ``daemon_active``). Forked compile children - # share the lock; ``os.execv`` drops it and the new image retakes it. + # share the lock; a restarted daemon takes it afresh. try: descriptor = _acquire_lock(_daemon_lock_path()) except OSError as exc: @@ -1318,11 +1539,28 @@ def main(argv: list[str] | None = None) -> int: "compiles to it." ) return 0 + # ``reflex run`` stops the daemon with SIGTERM; raising lets the compile + # in progress kill its child on the way out (see ``_await_child``). + signal.signal(signal.SIGTERM, _exit_on_sigterm) try: - _serve() + return _serve() except KeyboardInterrupt: return 0 # clean shutdown (Ctrl-C); no traceback - return 0 + except SystemExit: + return 0 + + +def _exit_on_sigterm(signum: int, frame: object) -> None: + """Turn SIGTERM into a ``SystemExit`` that unwinds the daemon. + + Args: + signum: The signal number. + frame: The interrupted frame. + + Raises: + SystemExit: Always. + """ + raise SystemExit(0) if __name__ == "__main__": diff --git a/reflex/utils/telemetry.py b/reflex/utils/telemetry.py index 5d29aee0bf7..0ad5dc1beec 100644 --- a/reflex/utils/telemetry.py +++ b/reflex/utils/telemetry.py @@ -642,26 +642,39 @@ def _flush(timeout: float | None = None) -> bool: return True -def shutdown_executor(timeout: float | None = 5.0) -> None: - """Drain and stop the telemetry worker thread, if one was started. +def _spool_event( + spool: Path, event: str, properties: dict[str, Any] | None, kwargs: dict[str, Any] +) -> None: + """Append an event to a spool file for another process to send. + + Args: + spool: The spool file. + event: The event name. + properties: The event's structured payload. + kwargs: Additional event data. + """ + with suppress(Exception), spool.open("a", encoding="utf-8") as fh: + fh.write( + json.dumps({"event": event, "properties": properties, "kwargs": kwargs}) + + "\n" + ) + - The worker outlives the events it delivered: ``ThreadPoolExecutor`` keeps - an idle thread alive until shutdown. A process that needs to be - single-threaded again (the compile daemon forks a child per hot reload, - which is only safe with no other threads) calls this after its last - telemetry send. A later send lazily creates a fresh executor. +def send_spooled(spool: Path) -> None: + """Send the events a compile child spooled, then remove the spool. Args: - timeout: Maximum seconds to wait for queued events to drain first. + spool: The spool file (see ``REFLEX_TELEMETRY_SPOOL``). """ - global _executor - with _executor_lock: - executor, _executor = _executor, None - if executor is None: + try: + lines = spool.read_text(encoding="utf-8").splitlines() + except OSError: return - with suppress(Exception): - executor.submit(lambda: None).result(timeout) - executor.shutdown(wait=True, cancel_futures=True) + spool.unlink(missing_ok=True) + for line in lines: + with suppress(Exception): + entry = json.loads(line) + send(entry["event"], properties=entry["properties"], **entry["kwargs"]) _legacy_alias_attempted = False @@ -745,6 +758,11 @@ def send( with suppress(Exception): if telemetry_enabled is None: telemetry_enabled = get_config().telemetry_enabled + if not telemetry_enabled: + return + if (spool := environment.REFLEX_TELEMETRY_SPOOL.get()) is not None: + _spool_event(spool, event, properties, kwargs) + return # Only spin up the worker pool when there is actually something to send. if telemetry_enabled: _submit( diff --git a/tests/units/test_telemetry.py b/tests/units/test_telemetry.py index db2cc600e3a..0fe812d4d60 100644 --- a/tests/units/test_telemetry.py +++ b/tests/units/test_telemetry.py @@ -963,3 +963,23 @@ def test_send_event_swallows_delivery_errors(mocker: MockerFixture): "reflex.utils.telemetry.urllib.request.urlopen", side_effect=OSError("down") ) assert not telemetry._send_event({"api_key": "k", "event": "e"}) # pyright: ignore[reportArgumentType] + + +def test_send_spools_events_for_another_process(tmp_path, monkeypatch, mocker): + """With a spool configured, events are appended to it instead of sent.""" + from reflex_base.environment import environment + + spool = tmp_path / "spool.jsonl" + monkeypatch.setenv(environment.REFLEX_TELEMETRY_SPOOL.name, str(spool)) + submit = mocker.patch.object(telemetry, "_submit") + telemetry.send("compile", telemetry_enabled=True, properties={"pages_count": 2}) + telemetry.send("export", telemetry_enabled=True, status="success") + submit.assert_not_called() + monkeypatch.delenv(environment.REFLEX_TELEMETRY_SPOOL.name) + sent = mocker.patch.object(telemetry, "send") + telemetry.send_spooled(spool) + assert [call.args for call in sent.call_args_list] == [("compile",), ("export",)] + assert sent.call_args_list[0].kwargs == {"properties": {"pages_count": 2}} + assert sent.call_args_list[1].kwargs == {"properties": None, "status": "success"} + assert not spool.exists() + telemetry.send_spooled(spool) # nothing spooled: nothing to do diff --git a/tests/units/utils/test_compile_daemon.py b/tests/units/utils/test_compile_daemon.py index bf651b94b0f..e2524577cdb 100644 --- a/tests/units/utils/test_compile_daemon.py +++ b/tests/units/utils/test_compile_daemon.py @@ -22,6 +22,18 @@ class _SurvivingModuleState(rx.State): b: int = 0 +@pytest.fixture(autouse=True) +def _isolated_daemon_process_state(tmp_path_factory, monkeypatch): + """Keep the daemon's lock files and bytecode cache out of the user's dirs. + + Args: + tmp_path_factory: The pytest temporary path factory. + monkeypatch: The pytest monkeypatch fixture. + """ + monkeypatch.setenv("REFLEX_DIR", str(tmp_path_factory.mktemp("reflex-data"))) + monkeypatch.setattr(sys, "pycache_prefix", sys.pycache_prefix) + + def test_can_fork_rejects_remaining_native_threads(monkeypatch): """Native threads must force the subprocess fallback after the grace period.""" monkeypatch.setattr(compile_daemon, "_quiesce_parent", lambda: None) @@ -248,6 +260,7 @@ def test_wait_for_compile_respects_lock(tmp_path, monkeypatch): # A lock file nobody holds (a compile killed mid-way) is not waited on, # even when it holds the pid of a live process. lock = compile_daemon._lock_path() + lock.parent.mkdir(parents=True, exist_ok=True) lock.write_text(str(os.getpid())) assert not path_ops.is_locked(lock) start = time.monotonic() @@ -592,6 +605,73 @@ class LocalState(rx.State): @pytest.mark.skipif(not hasattr(os, "fork"), reason="requires os.fork (POSIX)") +@pytest.mark.skipif(not hasattr(os, "fork"), reason="requires os.fork (POSIX)") +def test_reset_first_party_keeps_dynamic_states_surviving_modules_created(tmp_path): + """A dynamic state a surviving module created at import stays registered. + + An installed library's local state or ``ComponentState.create()`` runs + once, at import; purged modules re-import, surviving ones do not, so + dropping it would lose it from the state tree for good. A state a page + defined (per the manifest) is dropped: the page re-evaluates. + """ + read_fd, write_fd = os.pipe() + pid = os.fork() + if pid == 0: # child + os.close(read_fd) + result = b"E" + try: + import reflex.istate.dynamic as istate_dynamic + from reflex.state import State + + def _import_time_state() -> type[rx.State]: + class LibraryState(rx.State): + c: int = 0 + + return LibraryState + + def _page_state() -> type[rx.State]: + class PageLocalState(rx.State): + d: int = 0 + + return PageLocalState + + library_cls = _import_time_state() + page_cls = _page_state() + manifest = { + "pages": { + "/p": {"state_slice": {"schema": {page_cls.get_full_name(): []}}} + } + } + disk_cache.load_manifest = lambda: manifest + + compile_daemon._reset_first_party([tmp_path.resolve()]) + + names = {c.__name__ for c in State.get_substates()} + kept = library_cls.__name__ in names and hasattr( + istate_dynamic, library_cls.__name__ + ) + dropped = page_cls.__name__ not in names and not hasattr( + istate_dynamic, page_cls.__name__ + ) + # The page's local state re-registers under its original name. + redefined = _page_state().__name__ == page_cls.__name__ + result = b"1" if (kept and dropped and redefined) else b"0" + except Exception: + import traceback + + traceback.print_exc() + finally: + os.write(write_fd, result) + os.close(write_fd) + os._exit(0) + + os.close(write_fd) + out = os.read(read_fd, 1) + os.close(read_fd) + os.waitpid(pid, 0) + assert out == b"1" + + def test_reset_first_party_drops_memos_of_purged_modules(tmp_path): """A purged module's memos leave the registry; the module re-registers them. @@ -736,24 +816,31 @@ def test_quiesce_parent_stops_telemetry_worker(): assert not any(t.name.startswith("reflex-telemetry") for t in threading.enumerate()) # A later send lazily recreates a worker, so shutdown is not one-way. telemetry._get_telemetry_executor().submit(lambda: None).result() - telemetry.shutdown_executor() + telemetry._shutdown_executor() assert telemetry._executor is None def test_owns_compilation_follows_the_daemon_lock(tmp_path, monkeypatch): - """Backend workers defer to a daemon holding its lock; the daemon always owns.""" + """Backend reload workers defer to a daemon holding its lock; nobody else. + + The daemon always owns the build, and so does a process that asked for + a compile (``reflex compile``, ``reflex export``): deferring would leave + its output to the daemon's dev build. + """ from reflex_base.environment import environment from reflex.utils import path_ops monkeypatch.setattr("reflex.utils.prerequisites.get_backend_dir", lambda: tmp_path) monkeypatch.delenv(environment.REFLEX_COMPILE_DAEMON.name, raising=False) - # No daemon: a plain process compiles as usual. + monkeypatch.setenv(environment.REFLEX_DEV_BACKEND_RELOAD_ACTIVE.name, "1") + # No daemon: a worker compiles as usual. assert compile_daemon.daemon_active() is False assert compile_daemon.owns_compilation() is True # A lock file nobody holds (a killed daemon, a copied .web) names no owner, # even when it holds the pid of a live process. lock = compile_daemon._daemon_lock_path() + lock.parent.mkdir(parents=True, exist_ok=True) lock.write_text(str(os.getpid())) assert compile_daemon.daemon_active() is False # A daemon holding the lock: workers stand down. @@ -765,9 +852,169 @@ def test_owns_compilation_follows_the_daemon_lock(tmp_path, monkeypatch): # ...but the daemon's own processes still own the build. monkeypatch.setenv(environment.REFLEX_COMPILE_DAEMON.name, "1") assert compile_daemon.owns_compilation() is True + # ...and so does a CLI compile, which is no reload worker. + monkeypatch.delenv(environment.REFLEX_COMPILE_DAEMON.name) + monkeypatch.delenv(environment.REFLEX_DEV_BACKEND_RELOAD_ACTIVE.name) + assert compile_daemon.owns_compilation() is True finally: os.close(descriptor) - monkeypatch.delenv(environment.REFLEX_COMPILE_DAEMON.name) + + +def test_lock_files_live_outside_the_web_dir(tmp_path, monkeypatch): + """``rm -rf .web`` while the daemon runs must not unlink the lock it holds.""" + from reflex_base.environment import environment + + backend = tmp_path / ".web" / "backend" + monkeypatch.setattr("reflex.utils.prerequisites.get_backend_dir", lambda: backend) + for lock in (compile_daemon._lock_path(), compile_daemon._daemon_lock_path()): + assert not lock.is_relative_to(tmp_path / ".web") + assert lock.is_relative_to(environment.REFLEX_DIR.get().expanduser()) + assert compile_daemon._lock_path() != compile_daemon._daemon_lock_path() + # Keyed by the project: another project's daemon holds another lock. + this_project = compile_daemon._daemon_lock_path() + monkeypatch.setattr( + "reflex.utils.prerequisites.get_backend_dir", lambda: tmp_path / "other" + ) + assert compile_daemon._daemon_lock_path() != this_project + + +def test_running_restarts_the_daemon_on_its_restart_code(tmp_path, monkeypatch): + """A daemon exiting for a global config change is started again. + + The daemon cannot apply such a change to its warm parent; a new process + loads the new world. Re-executing in place would leave ``reflex run`` + with a dead handle on Windows, where that starts a new process. + """ + import subprocess + + popen = subprocess.Popen + spawned: list[subprocess.Popen] = [] + + def spawn(args, **kwargs): + """Stand in for the daemon: ask for a restart once, then wait. + + Args: + args: The daemon command (replaced). + kwargs: The Popen options. + + Returns: + The spawned process. + """ + code = compile_daemon._RESTART_EXIT_CODE if not spawned else 0 + body = f"import sys, time\nsys.exit({code}) if {code} else time.sleep(60)\n" + proc = popen([sys.executable, "-c", body], **kwargs) + spawned.append(proc) + return proc + + monkeypatch.setattr(compile_daemon.subprocess, "Popen", spawn) + with compile_daemon.running(): + assert AppHarness._poll_for(lambda: len(spawned) == 2, timeout=10) + assert spawned[0].poll() == compile_daemon._RESTART_EXIT_CODE + assert spawned[1].poll() is None + assert spawned[1].poll() is not None + assert len(spawned) == 2 + + +@pytest.mark.skipif(not hasattr(os, "fork"), reason="requires os.fork (POSIX)") +def test_await_child_kills_the_child_when_the_daemon_exits(monkeypatch): + """A daemon unwinding mid-compile (SIGTERM, Ctrl-C) takes its child with it. + + Otherwise the orphaned child keeps the locks and writes ``.web`` after + ``reflex run`` ended, and a new run finds a daemon that owns the project. + """ + import time + + pid = os.fork() + if pid == 0: # child: a compile that never ends + try: + time.sleep(60) + finally: + os._exit(0) + + def exit_now(_seconds: float) -> None: + raise SystemExit(0) + + monkeypatch.setattr(compile_daemon.time, "sleep", exit_now) + with pytest.raises(SystemExit): + compile_daemon._await_child(pid) + # Reaped: the pid is gone (or at least no longer ours to wait for). + with pytest.raises(ChildProcessError): + os.waitpid(pid, 0) + + +def test_watch_state_skips_build_dirs_below_the_project_only(tmp_path, monkeypatch): + """A project living under a directory named like a build dir is watched.""" + root = tmp_path / "venv" / "project" + app_root = root / "app" + app_root.mkdir(parents=True) + monkeypatch.setattr(disk_cache, "load_manifest", lambda: None) + state = compile_daemon._WatchState.build([app_root.resolve()], root.resolve()) + assert state.accepts(app_root / "page.py") + assert not state.accepts(app_root / "node_modules" / "pkg" / "index.py") + assert not state.accepts(root / ".web" / "x.py") + + +def test_child_compile_uses_a_private_bytecode_cache(tmp_path, monkeypatch): + """Changed modules re-import from source, not from same-second bytecode. + + Timestamp ``.pyc`` files are keyed by whole-second mtime and size, so a + second same-size edit within a second of the compile that cached the + first would import stale bytecode. + """ + import importlib.util + + monkeypatch.setattr("reflex.utils.prerequisites.get_backend_dir", lambda: tmp_path) + compile_daemon._use_private_pycache() + assert sys.pycache_prefix == str(tmp_path.resolve() / compile_daemon._PYCACHE_DIR) + source = tmp_path / "page.py" + source.write_text("x = 1\n") + cached = Path(importlib.util.cache_from_source(str(source))) + assert cached.is_relative_to(tmp_path / compile_daemon._PYCACHE_DIR) + cached.parent.mkdir(parents=True) + cached.write_bytes(b"stale") + compile_daemon._forget_bytecode({source, tmp_path / "data.md"}) + assert not cached.exists() + compile_daemon._forget_bytecode({source}) # already gone: no error + + +def test_compile_child_telemetry_is_sent_by_the_daemon(tmp_path, monkeypatch): + """A compile child spools its events; the daemon sends them afterwards. + + A forked child must not start the telemetry thread, nor touch macOS's + proxy lookup after a fork, and ``os._exit`` drops anything it queued. + """ + from reflex_base.environment import environment + + from reflex.utils import telemetry + + monkeypatch.setattr("reflex.utils.prerequisites.get_backend_dir", lambda: tmp_path) + sent: list[tuple[str, dict]] = [] + monkeypatch.setattr( + telemetry, "send", lambda event, **kwargs: sent.append((event, kwargs)) + ) + monkeypatch.setattr(compile_daemon, "_can_fork", lambda: False) + + def run(args, **kwargs): + """Stand in for the cold compile child: send an event, exit. + + Args: + args: The compile command (replaced). + kwargs: The run options. + + Returns: + A completed process. + """ + spool = Path(kwargs["env"][environment.REFLEX_TELEMETRY_SPOOL.name]) + telemetry._spool_event(spool, "compile", {"pages_count": 1}, {}) + return Mock(returncode=0) + + monkeypatch.setattr(compile_daemon.subprocess, "run", run) + assert ( + compile_daemon._run_compile_child([tmp_path], False, None, predict=False) == 0 + ) + assert sent == [("compile", {"properties": {"pages_count": 1}})] + assert not list(tmp_path.glob(".telemetry.*")) + assert environment.REFLEX_TELEMETRY_SPOOL.get() is None assert compile_daemon.owns_compilation() is True @@ -1138,11 +1385,22 @@ def attempt() -> bool: return outcome == "ok" monkeypatch.setattr(compile_daemon, "_single_threaded", lambda: True) + monkeypatch.setattr(disk_cache, "load_manifest", lambda: {"pages": {}}) assert compile_daemon.compiled_in_predicting_child(attempt) is compiled assert seen.read_text() == "True" assert disk_cache._predict_dynamic_names is None +def test_cold_compile_without_a_manifest_does_not_fork(monkeypatch): + """With nothing to reuse, the attempt would only fall back: no child.""" + from reflex.compiler import disk_cache + + monkeypatch.setattr(compile_daemon, "_single_threaded", lambda: True) + monkeypatch.setattr(disk_cache, "load_manifest", lambda: None) + monkeypatch.setattr(compile_daemon.os, "fork", Mock(side_effect=AssertionError)) + assert compile_daemon.compiled_in_predicting_child(lambda: True) is False + + @pytest.mark.parametrize( ("single_threaded", "chosen"), [(False, None), (True, False), (True, True)] ) diff --git a/tests/units/utils/test_serializers.py b/tests/units/utils/test_serializers.py index 0f65376fb50..66d4964a052 100644 --- a/tests/units/utils/test_serializers.py +++ b/tests/units/utils/test_serializers.py @@ -538,16 +538,23 @@ def test_serialize(value: Any, expected: str): assert json.loads(json_dumps(value)) == json.loads(json_dumps(expected)) -def test_serialize_set_deterministic_order(): - """Sortable sets serialize in sorted order regardless of the hash seed. +def test_sorted_set_is_deterministic_and_serialize_set_is_not_sorted(): + """Compiled output sorts sets; a state delta keeps the cheap list. Compiled output and cache fingerprints embed serialized sets; a - hash-seed-dependent order churns files on every process restart. + hash-seed-dependent order churns files on every process restart. A delta + is serialized on every update and its order carries no meaning. """ - assert serializers.serialize_set({"b", "c", "a"}) == ["a", "b", "c"] - assert serializers.serialize_set({3, 1, 2}) == [1, 2, 3] - # Unsortable element mixes fall back to iteration order. + from reflex_base.vars import LiteralVar + + assert serializers.sorted_set({"b", "c", "a"}) == ["a", "b", "c"] + assert serializers.sorted_set({3, 1, 2}) == [1, 2, 3] + # Unsortable element mixes sort by their serialized form. + assert serializers.sorted_set({1, "a"}) == serializers.sorted_set({"a", 1}) assert set(serializers.serialize_set({1, "a"})) == {1, "a"} + assert serializers.serialize_set({3, 1, 2}) == list({3, 1, 2}) + assert str(LiteralVar.create({"b", "c", "a"})) == '["a", "b", "c"]' + assert str(LiteralVar.create({3, 1, 2})) == "[1, 2, 3]" @pytest.mark.parametrize( @@ -612,6 +619,4 @@ def __iter__(self): return reversed(list(super().__iter__())) values = {Choice.A, Choice.B} - assert serializers.serialize_set(values) == serializers.serialize_set( - ReversedSet(values) - ) + assert serializers.sorted_set(values) == serializers.sorted_set(ReversedSet(values)) From 495a0ef899097dd660a003f02b72ac738989e802 Mon Sep 17 00:00:00 2001 From: Claude <noreply@anthropic.com> Date: Wed, 21 Oct 2026 21:15:32 +0000 Subject: [PATCH 53/56] fix(compile-daemon): load the manifest once per reset; a restarted daemon compiles Claude-Session: https://claude.ai/code/session_01RgKnnN8CCBqaR58qm23zQp --- news/+content-addressed-auto-memos.misc.md | 1 + packages/reflex-base/news/6688.misc.md | 2 +- reflex/utils/compile_daemon.py | 27 +++++++++++++--------- tests/units/utils/test_compile_daemon.py | 15 ++++++++++-- 4 files changed, 31 insertions(+), 14 deletions(-) create mode 100644 news/+content-addressed-auto-memos.misc.md diff --git a/news/+content-addressed-auto-memos.misc.md b/news/+content-addressed-auto-memos.misc.md new file mode 100644 index 00000000000..a220a183f26 --- /dev/null +++ b/news/+content-addressed-auto-memos.misc.md @@ -0,0 +1 @@ +Compiler-generated memo components (the auto-memoized stateful subtrees) now compile to one file each under `.web/utils/components/`, named by their content, instead of one file per source module under `.web/app_components/`; identical subtrees anywhere in the app share one file. `@rx.memo` components keep mirroring their source modules. diff --git a/packages/reflex-base/news/6688.misc.md b/packages/reflex-base/news/6688.misc.md index c628017e8da..f3abd7e694f 100644 --- a/packages/reflex-base/news/6688.misc.md +++ b/packages/reflex-base/news/6688.misc.md @@ -1 +1 @@ -Added an optional per-page source-read recorder hook (`page_source_recorder` in the compiler plugin) used by the incremental compile cache to track the exact files each page reads during evaluation. Auto-generated unique ref names are now seeded per page from its route (and the page's sequence is resumed for its render), so a page's generated names are reproducible and independent of which other pages compile alongside it. No behavior change unless the compile cache is enabled. +Added an optional per-page source-read recorder hook (`page_source_recorder` in the compiler plugin) used by the incremental compile cache to track the exact files each page reads during evaluation. Auto-generated unique ref names are now reproducible: each page draws them from a sequence seeded by its route, a state's default values from one seeded by the state's name, and whatever compiles after the pages from a sequence that never repeats a name drawn at import. Sets are sorted in compiled literals only; a state delta serializes a set as a plain list again. diff --git a/reflex/utils/compile_daemon.py b/reflex/utils/compile_daemon.py index a3f2adba915..91477f355ae 100644 --- a/reflex/utils/compile_daemon.py +++ b/reflex/utils/compile_daemon.py @@ -115,9 +115,13 @@ def running( env["REFLEX_COMPILE_DAEMON_PRECOMPILED"] = "1" def spawn() -> subprocess.Popen: - return subprocess.Popen( + proc = subprocess.Popen( [sys.executable, "-m", "reflex.utils.compile_daemon"], env=env ) + # A restarted daemon compiles: the config change it restarts for + # made the frontend the first daemon produced stale. + env.pop("REFLEX_COMPILE_DAEMON_PRECOMPILED", None) + return proc lock = threading.Lock() stopping = False @@ -696,7 +700,8 @@ def _reset_first_party(roots: list[Path]) -> None: kept = [ cls for name, cls in ctx.base_states.items() - if name not in page_states and _survives_purge(cls) + if (page_states is None or name not in page_states) + and _survives_purge(cls, page_states is not None) ] ctx.base_states.clear() ctx.base_state_substates.clear() @@ -734,24 +739,24 @@ def _reset_first_party(roots: list[Path]) -> None: _reset_model_metadata() -def _page_defined_states() -> set[str]: +def _page_defined_states() -> set[str] | None: """The states the last compile recorded as defined by a page's evaluation. They are re-created when the page re-evaluates, so they never survive a - reset; without a manifest, every ``reflex.istate.dynamic`` state is - taken for one. + reset. Returns: - The full names of the page-defined states. + The full names of the page-defined states, or None without a + manifest to tell them from the states created at import. """ from reflex.compiler import disk_cache if (manifest := disk_cache.load_manifest()) is None: - return set() + return None return disk_cache._hit_page_states(manifest, set()) -def _survives_purge(cls: type) -> bool: +def _survives_purge(cls: type, page_states_known: bool) -> bool: """Whether a registered state stays registered across the first-party purge. A state's code re-registers it when it runs again, which only happens @@ -764,6 +769,8 @@ def _survives_purge(cls: type) -> bool: Args: cls: The registered state class. + page_states_known: Whether a manifest told the page-defined states + apart from the ones created at import. Returns: True when the state is kept. @@ -772,9 +779,7 @@ def _survives_purge(cls: type) -> bool: module_name = getattr(cls, "__module__", None) if module_name == istate_dynamic.__name__: - from reflex.compiler import disk_cache - - if disk_cache.load_manifest() is None: + if not page_states_known: return False # A local state remembers its module; a ComponentState instance is # created from a class defined in its creating module. diff --git a/tests/units/utils/test_compile_daemon.py b/tests/units/utils/test_compile_daemon.py index e2524577cdb..91e30887fef 100644 --- a/tests/units/utils/test_compile_daemon.py +++ b/tests/units/utils/test_compile_daemon.py @@ -906,13 +906,24 @@ def spawn(args, **kwargs): spawned.append(proc) return proc - monkeypatch.setattr(compile_daemon.subprocess, "Popen", spawn) - with compile_daemon.running(): + environments: list[dict[str, str]] = [] + original_spawn = spawn + + def spawn_recording_env(args, **kwargs): + environments.append(dict(kwargs["env"])) + return original_spawn(args, **kwargs) + + monkeypatch.setattr(compile_daemon.subprocess, "Popen", spawn_recording_env) + with compile_daemon.running(initial_compiled=True): assert AppHarness._poll_for(lambda: len(spawned) == 2, timeout=10) assert spawned[0].poll() == compile_daemon._RESTART_EXIT_CODE assert spawned[1].poll() is None assert spawned[1].poll() is not None assert len(spawned) == 2 + # The first daemon found the frontend compiled; the restarted one must + # compile, since the config change made that frontend stale. + assert "REFLEX_COMPILE_DAEMON_PRECOMPILED" in environments[0] + assert "REFLEX_COMPILE_DAEMON_PRECOMPILED" not in environments[1] @pytest.mark.skipif(not hasattr(os, "fork"), reason="requires os.fork (POSIX)") From faed563c1726b428951422dc034dcc5acf2711ba Mon Sep 17 00:00:00 2001 From: Farhan <www.mfarvirus@gmail.com> Date: Thu, 22 Oct 2026 01:06:34 +0000 Subject: [PATCH 54/56] fix(compile-cache): address Greptile's four P1 findings - Skip every fork-based compile daemon test without os.fork, so the Windows unit suite does not raise AttributeError. - A volatile page misses even when the watcher's changed set misses its dependencies. - Record a page's Radix use from its auto memos too: memoization replaces a stateful Radix component in the page tree with its wrapper. - Record every memo the app root imports, including those a page shares, so a page dropping a shared memo no longer deletes the root's import. Claude-Session: https://claude.ai/code/session_01PXcFKAgnzhgdnSvhuCmxWE --- .../src/reflex_base/plugins/compiler.py | 4 + reflex/compiler/compiler.py | 2 + reflex/compiler/disk_cache.py | 37 +++++---- tests/units/compiler/test_disk_cache.py | 77 +++++++++++++++++++ tests/units/utils/test_compile_daemon.py | 3 +- 5 files changed, 105 insertions(+), 18 deletions(-) diff --git a/packages/reflex-base/src/reflex_base/plugins/compiler.py b/packages/reflex-base/src/reflex_base/plugins/compiler.py index 574309edf93..4b0ec3d668d 100644 --- a/packages/reflex-base/src/reflex_base/plugins/compiler.py +++ b/packages/reflex-base/src/reflex_base/plugins/compiler.py @@ -702,6 +702,9 @@ class CompileContext(BaseContext): auto_memo_components: dict[tuple[str, str | None], Any] = dataclasses.field( default_factory=dict ) + # The auto memo tags the app root imports, including those a page also + # uses (filled when the app root is built). + app_root_memo_keys: set[str] = dataclasses.field(default_factory=set) def compile( self, @@ -741,6 +744,7 @@ def compile( self.stateful_routes.clear() self.memoize_wrappers.clear() self.auto_memo_components.clear() + self.app_root_memo_keys.clear() recorder = page_source_recorder for page in self.pages: diff --git a/reflex/compiler/compiler.py b/reflex/compiler/compiler.py index 99277f23da7..51f01615bf3 100644 --- a/reflex/compiler/compiler.py +++ b/reflex/compiler/compiler.py @@ -1162,6 +1162,7 @@ def _build_app_root( hydrate_fallback_definition.export_name, None ] = hydrate_fallback_definition hydrate_fallback_export = hydrate_fallback_definition.export_name + compile_ctx.app_root_memo_keys.add(hydrate_fallback_export) return app_root, hydrate_fallback_export @@ -1198,6 +1199,7 @@ def _memoize_stateful_app_wraps( page_context=page_context, compile_context=compile_context, ) + compile_context.app_root_memo_keys.update(tag for tag, _ in page_context.memo_uses) if not isinstance(compiled_root, Component): msg = "Compiled app root must be a Component." raise TypeError(msg) diff --git a/reflex/compiler/disk_cache.py b/reflex/compiler/disk_cache.py index a7d1d665d1b..af2fceaadff 100644 --- a/reflex/compiler/disk_cache.py +++ b/reflex/compiler/disk_cache.py @@ -321,23 +321,29 @@ def _memo_keys(page_ctx: PageContext) -> list[str]: return sorted(tag for tag, _ in page_ctx.memo_uses) -def _uses_radix(root_component: object) -> bool: - """Whether a rendered tree holds a Radix Themes component. +def _uses_radix(page_ctx: PageContext) -> bool: + """Whether a compiled page holds a Radix Themes component. The implicit ``RadixThemesPlugin`` enables itself when it compiles one; - a page reused from disk is not compiled, so the manifest remembers. + a page reused from disk is not compiled, so the manifest remembers. The + page's auto memos are walked too: memoization replaces a stateful Radix + component in the page tree with its wrapper. Args: - root_component: The page's root component. + page_ctx: The compiled page context. Returns: - True when the tree enables Radix Themes. + True when the page enables Radix Themes. """ from reflex_components_radix.themes.base import RadixThemesComponent return any( issubclass(cls, RadixThemesComponent) - for cls in page_cache.component_tree_classes(root_component) + for root in ( + page_ctx.root_component, + *(definition.component for definition in page_ctx.memo_uses.values()), + ) + for cls in page_cache.component_tree_classes(root) ) @@ -393,7 +399,7 @@ def _manifest_page_entry( # no page uses any more. "memo_keys": _memo_keys(page_ctx), # Whether the page enables the implicit Radix Themes plugin. - "radix": _uses_radix(page_ctx.root_component), + "radix": _uses_radix(page_ctx), } @@ -772,9 +778,6 @@ def write_manifest( files[path] = entry globals_[label] = path - page_memo_keys = { - tag for entry in pages_data.values() for tag in entry["memo_keys"] - } entry_file = page_cache._app_entrypoint_file(root) manifest = { "schema": _SCHEMA, @@ -807,7 +810,7 @@ def write_manifest( "module_states": module_states, # The app root's inputs: the merged app wraps (by digest, in # insertion order) and the bundled libraries (see - # ``_app_root_change``), plus the memo files only the root uses. + # ``_app_root_change``), plus the memo files the root uses. "app_root": { "wraps": { _wrap_key_str(key): _wrap_digest(component) @@ -816,11 +819,7 @@ def write_manifest( "libraries": sorted( RegistrationContext.ensure_context().bundled_libraries ), - "memo_keys": sorted( - tag - for tag, _ in compile_ctx.auto_memo_components - if tag not in page_memo_keys - ), + "memo_keys": sorted(compile_ctx.app_root_memo_keys), }, } _write(manifest) @@ -1052,7 +1051,11 @@ def partition_pages( misses: list[PageDefinition] = [] for page in pages: deps = manifest_pages[page.route]["deps"] - if changed_hint is not None and changed_hint.isdisjoint(deps): + if ( + changed_hint is not None + and changed_hint.isdisjoint(deps) + and page_cache.VOLATILE_DEP not in deps + ): continue if not validator.unchanged(deps): misses.append(page) diff --git a/tests/units/compiler/test_disk_cache.py b/tests/units/compiler/test_disk_cache.py index 55e127965ab..4134e8bd398 100644 --- a/tests/units/compiler/test_disk_cache.py +++ b/tests/units/compiler/test_disk_cache.py @@ -513,6 +513,19 @@ def test_partition_pages_detects_changed_source(tmp_path): assert {p.route for p in miss} == {"/b"} +def test_partition_pages_rebuilds_a_volatile_page_whatever_the_hint(tmp_path): + """A volatile page misses even when the watcher reports only other files.""" + pages = [_FakePage(route="/a", component=_page_a)] + x = tmp_path / "x.py" + x.write_text("X = 1\n") + m = _manifest( + {"/a": {"deps": [str(x), page_cache.VOLATILE_DEP], "registration": "recorded"}}, + files={str(x): page_cache.file_entry(str(x))}, + ) + miss = disk_cache.partition_pages(pages, m, _validator(m), {str(tmp_path / "y.py")}) + assert [p.route for p in miss] == ["/a"] + + def test_pages_mismatch_names_the_change(tmp_path): """The page registry is an input of every page, so any change is global.""" pages = [ @@ -2524,10 +2537,74 @@ def test_incremental_miss_drops_the_memo_no_page_uses( assert compiler_utils._read_memo_manifest(web) == {new_relative} +def test_incremental_miss_keeps_the_memo_the_app_root_shares( + tmp_path, monkeypatch, preserve_memo_registries +): + """A memo the app root also imports survives the page dropping it.""" + from reflex.compiler import compiler + from reflex.compiler import utils as compiler_utils + + web = _use_tmp_web_dir(tmp_path, monkeypatch) + app = rx.App() + app.add_page(_page_shared_one, route="/one") + pages = list(app._unevaluated_pages.values()) + route_one = pages[0].route + ctx = _compile(pages) + shared = _memo_file_of(ctx.compiled_pages[route_one]) + # An app wrap rendering the same stateful subtree imports the same memo. + compiler._memoize_stateful_app_wraps( + rx.fragment(rx.el.p(_MemoCacheState.value, class_name="shared")), ctx + ) + memo_files, _ = compiler.compile_memo_components(ctx.auto_memo_components.values()) + assert [Path(path) for path, _ in memo_files] == [shared] + for mpath, mcode in memo_files: + compiler_utils.write_file(compiler_utils.resolve_path_of_web_dir(mpath), mcode) + compiler_utils.prune_stale_memo_files(path for path, _ in memo_files) + disk_cache.write_manifest(ctx, pages, ctx.all_imports, root=tmp_path) + _stub_externals(app, monkeypatch) + manifest_path = web / disk_cache._MANIFEST_FILE + manifest = json.loads(manifest_path.read_text()) + assert ( + manifest["app_root"]["memo_keys"] == manifest["pages"][route_one]["memo_keys"] + ) + monkeypatch.setitem(_SHARED_MEMO, "edit_one", True) + _stale_dep(manifest, route_one, str(tmp_path / "data.md")) + manifest_path.write_text(json.dumps(manifest)) + assert ( + disk_cache.try_incremental_rebuild( + app, compiler_plugins=[], prerender_routes=False, root=tmp_path + ) + is True + ) + assert shared.exists() + + def _page_radix() -> Component: return rx.el.div(rx.button("Radix"), _footer()) +def _page_memoized_radix() -> Component: + return rx.el.div( + rx.text("x", color=_MemoCacheState.value, on_click=rx.console_log("c")) + ) + + +def test_memoized_radix_component_is_recorded_as_radix(tmp_path, monkeypatch): + """A Radix component auto-memoization moved out of the page tree counts.""" + from reflex_components_radix.plugin import RadixThemesPlugin + + web = _use_tmp_web_dir(tmp_path, monkeypatch) + app = rx.App() + app.add_page(_page_memoized_radix, route="/m") + pages = list(app._unevaluated_pages.values()) + ctx = _compile(pages, plugins=[RadixThemesPlugin.create_implicit()]) + page_ctx = ctx.compiled_pages[pages[0].route] + assert page_ctx.memo_uses, "the Radix text must be auto-memoized" + disk_cache.write_manifest(ctx, pages, ctx.all_imports, root=tmp_path) + manifest = json.loads((web / disk_cache._MANIFEST_FILE).read_text()) + assert manifest["pages"][pages[0].route]["radix"] is True + + def test_reused_radix_page_keeps_the_implicit_plugin_enabled(tmp_path, monkeypatch): """A rebuild that compiles no Radix page still ships the Radix stylesheet. diff --git a/tests/units/utils/test_compile_daemon.py b/tests/units/utils/test_compile_daemon.py index 91e30887fef..be949e373e4 100644 --- a/tests/units/utils/test_compile_daemon.py +++ b/tests/units/utils/test_compile_daemon.py @@ -604,7 +604,6 @@ class LocalState(rx.State): assert out == b"1" -@pytest.mark.skipif(not hasattr(os, "fork"), reason="requires os.fork (POSIX)") @pytest.mark.skipif(not hasattr(os, "fork"), reason="requires os.fork (POSIX)") def test_reset_first_party_keeps_dynamic_states_surviving_modules_created(tmp_path): """A dynamic state a surviving module created at import stays registered. @@ -672,6 +671,7 @@ class PageLocalState(rx.State): assert out == b"1" +@pytest.mark.skipif(not hasattr(os, "fork"), reason="requires os.fork (POSIX)") def test_reset_first_party_drops_memos_of_purged_modules(tmp_path): """A purged module's memos leave the registry; the module re-registers them. @@ -734,6 +734,7 @@ def load(source: str) -> None: assert out == b"1" +@pytest.mark.skipif(not hasattr(os, "fork"), reason="requires os.fork (POSIX)") def test_reset_first_party_purges_modules_and_registries(tmp_path): """``_reset_first_party`` purges first-party modules and clears registries. From 396f418c2f8ff5ead9bf6560b45416d3a0ec59bf Mon Sep 17 00:00:00 2001 From: Farhan <www.mfarvirus@gmail.com> Date: Thu, 22 Oct 2026 01:16:35 +0000 Subject: [PATCH 55/56] fix(compile-cache): address cubic's review of the recent commits - SIGTERM during the daemon's initial compile stops the daemon instead of being logged as a failed compile (a dedicated `_Terminated` exit passes the failure handler). - A manifest refresh that raises no longer fails an incremental rebuild: it is logged and the manifest is removed, so none vouches for the new outputs, like `write_manifest`. - Removing the telemetry spool is best-effort, so it cannot fail a compile. - Tests: restore the unique-name state and the reserved names, delete the temporary `get_substates` instead of rebinding it, cover a set iterating out of sorted order, assert the manifest is the last write and fail a write after the page, patch `os.fork` on Windows too, and keep the plugin output test in process so it observes the outputs. Claude-Session: https://claude.ai/code/session_01PXcFKAgnzhgdnSvhuCmxWE --- reflex/compiler/disk_cache.py | 54 +++++++++++--------- reflex/utils/compile_daemon.py | 8 ++- reflex/utils/telemetry.py | 3 +- tests/units/compiler/test_compiler.py | 29 +++++++++-- tests/units/compiler/test_compiler_utils.py | 3 +- tests/units/compiler/test_disk_cache.py | 55 ++++++++++++++++++--- tests/units/utils/test_compile_daemon.py | 33 ++++++++++++- tests/units/utils/test_serializers.py | 15 +++++- 8 files changed, 159 insertions(+), 41 deletions(-) diff --git a/reflex/compiler/disk_cache.py b/reflex/compiler/disk_cache.py index af2fceaadff..638cdab3c8b 100644 --- a/reflex/compiler/disk_cache.py +++ b/reflex/compiler/disk_cache.py @@ -2155,28 +2155,38 @@ def try_incremental_rebuild( } # Refresh the manifest for the next process; it is the last output. - if _update_manifest_for_misses( - manifest, - miss_ctx, - miss_pages, - install_imports, - root, - contexts_snapshot=contexts_snapshot, - memo_state=final_memo_state, - memo_file_entries=memo_files_table, - registrations=registrations, - module_states=( - _record_module_states( - live_module_states, - page_cache.make_hasher(), - manifest["files"], - root, - ) - if module_states_changed - else recorded_module_states - ), - validator=validator, - ): + try: + refreshed = _update_manifest_for_misses( + manifest, + miss_ctx, + miss_pages, + install_imports, + root, + contexts_snapshot=contexts_snapshot, + memo_state=final_memo_state, + memo_file_entries=memo_files_table, + registrations=registrations, + module_states=( + _record_module_states( + live_module_states, + page_cache.make_hasher(), + manifest["files"], + root, + ) + if module_states_changed + else recorded_module_states + ), + validator=validator, + ) + except Exception as exc: # best-effort, like ``write_manifest`` + logger.warning( + f"Compile cache: could not refresh manifest ({exc!r}). " + "The next reload may need a full compile." + ) + # No manifest may vouch for the outputs about to be written. + _manifest_path().unlink(missing_ok=True) + refreshed = False + if refreshed: _save_incremental_output(_manifest_path(), _manifest_text(manifest), outputs) app._cached_component_counts = manifest_component_counts(manifest) diff --git a/reflex/utils/compile_daemon.py b/reflex/utils/compile_daemon.py index 91477f355ae..9c3c6a573a8 100644 --- a/reflex/utils/compile_daemon.py +++ b/reflex/utils/compile_daemon.py @@ -1425,6 +1425,8 @@ def _serve() -> int: use_rich=True, trigger="initial", ) + except _Terminated: + raise except BaseException: # tolerate a broken initial state; keep watching logger.exception("Initial compile failed; watching for a fix.") _prepare_fork_parent(roots) @@ -1555,6 +1557,10 @@ def main(argv: list[str] | None = None) -> int: return 0 +class _Terminated(SystemExit): + """The ``SystemExit`` SIGTERM raises, which no compile failure handler keeps.""" + + def _exit_on_sigterm(signum: int, frame: object) -> None: """Turn SIGTERM into a ``SystemExit`` that unwinds the daemon. @@ -1565,7 +1571,7 @@ def _exit_on_sigterm(signum: int, frame: object) -> None: Raises: SystemExit: Always. """ - raise SystemExit(0) + raise _Terminated(0) if __name__ == "__main__": diff --git a/reflex/utils/telemetry.py b/reflex/utils/telemetry.py index 0ad5dc1beec..ecd2ea42c18 100644 --- a/reflex/utils/telemetry.py +++ b/reflex/utils/telemetry.py @@ -670,7 +670,8 @@ def send_spooled(spool: Path) -> None: lines = spool.read_text(encoding="utf-8").splitlines() except OSError: return - spool.unlink(missing_ok=True) + with suppress(OSError): + spool.unlink(missing_ok=True) for line in lines: with suppress(Exception): entry = json.loads(line) diff --git a/tests/units/compiler/test_compiler.py b/tests/units/compiler/test_compiler.py index f5e571aff28..9be4fa14168 100644 --- a/tests/units/compiler/test_compiler.py +++ b/tests/units/compiler/test_compiler.py @@ -1968,6 +1968,12 @@ def rebuild(*args, outputs, **kwargs): "load_manifest", return_value={"plugin_sources": {str(tmp_path / "page.txt"): "base"}}, ) + # The attempt runs in this process, where the outputs can be observed. + from reflex.utils import compile_daemon + + mocker.patch.object( + compile_daemon, "compiled_in_predicting_child", return_value=False + ) (tmp_path / "page.txt").write_text("base!") assert compiler.compile_app(app, use_rich=False) assert (tmp_path / "asset.txt").read_text() == "asset" @@ -2008,22 +2014,34 @@ def test_cached_compile_writes_the_manifest_last( original_write = compiler.utils.write_file def write_file(path, code): - """Write an output, failing the page when the test asks. + """Write an output, failing every write after the page when asked. Args: path: The output path. code: The output content. Raises: - OSError: When the page write is made to fail. + OSError: When a write after the page is made to fail. """ - if write_fails and Path(path) == page: + if write_fails and page in written: msg = "disk full" raise OSError(msg) written.append(Path(path)) original_write(path, code) mocker.patch.object(compiler.utils, "write_file", side_effect=write_file) + original_manifest_write = disk_cache._write + + def write_manifest(manifest): + """Record when the manifest is written. + + Args: + manifest: The manifest to write. + """ + written.append(manifest_path) + original_manifest_write(manifest) + + mocker.patch.object(disk_cache, "_write", side_effect=write_manifest) def rebuild(*args, outputs, **kwargs): """Stage a page and the refreshed manifest, manifest first. @@ -2048,8 +2066,9 @@ def rebuild(*args, outputs, **kwargs): return assert compiler.compile_app(app, use_rich=False) assert json.loads(manifest_path.read_text())["schema"] == 1 - assert manifest_path not in written - assert written.index(page) < len(written) + assert written[0] == page + assert len(written) > 2 + assert written.index(manifest_path) == len(written) - 1 def test_preloaded_app_skips_incremental_reuse(monkeypatch, mocker): diff --git a/tests/units/compiler/test_compiler_utils.py b/tests/units/compiler/test_compiler_utils.py index d092446ac95..584e90856cf 100644 --- a/tests/units/compiler/test_compiler_utils.py +++ b/tests/units/compiler/test_compiler_utils.py @@ -356,14 +356,13 @@ class _OrderTwo(_OrderRoot): def serialized(substates: list[type[rx.State]]) -> str: reset_unique_variable_names() - original = _OrderRoot.get_substates _OrderRoot.get_substates = classmethod( # type: ignore[method-assign] lambda cls: substates if cls is _OrderRoot else set() ) try: return utils._serialize_initial_state(compile_state(_OrderRoot)) finally: - _OrderRoot.get_substates = original # type: ignore[method-assign] + del _OrderRoot.get_substates first = serialized([_OrderOne, _OrderTwo]) second = serialized([_OrderTwo, _OrderOne]) diff --git a/tests/units/compiler/test_disk_cache.py b/tests/units/compiler/test_disk_cache.py index 4134e8bd398..4a059608628 100644 --- a/tests/units/compiler/test_disk_cache.py +++ b/tests/units/compiler/test_disk_cache.py @@ -723,19 +723,28 @@ def test_names_drawn_after_the_pages_never_repeat_import_time_names(): the app root and runtime components draw more. Reseeding the import sequence would hand the same names out again. """ + from reflex_base.vars import base as vars_base from reflex_base.vars.base import ( get_unique_variable_name, reset_unique_variable_names, + restore_unique_variable_name_state, + unique_variable_name_state, ) - reset_unique_variable_names() - import_names = {get_unique_variable_name() for _ in range(3)} - _compile([_FakePage(route="/a", component=_page_a)]) - after = {get_unique_variable_name() for _ in range(50)} - assert not after & import_names - # And the post-page sequence is the same on every compile. - _compile([_FakePage(route="/b", component=_page_b)]) - assert {get_unique_variable_name() for _ in range(50)} == after + saved = unique_variable_name_state() + saved_reserved = vars_base._names_before_first_compile + try: + reset_unique_variable_names() + import_names = {get_unique_variable_name() for _ in range(3)} + _compile([_FakePage(route="/a", component=_page_a)]) + after = {get_unique_variable_name() for _ in range(50)} + assert not after & import_names + # And the post-page sequence is the same on every compile. + _compile([_FakePage(route="/b", component=_page_b)]) + assert {get_unique_variable_name() for _ in range(50)} == after + finally: + restore_unique_variable_name_state(saved) + vars_base._names_before_first_compile = saved_reserved _CONTEXTS_STUB = "// contexts stub" @@ -2537,6 +2546,36 @@ def test_incremental_miss_drops_the_memo_no_page_uses( assert compiler_utils._read_memo_manifest(web) == {new_relative} +def test_failed_manifest_refresh_leaves_no_manifest( + tmp_path, monkeypatch, preserve_memo_registries +): + """A refresh that raises still rebuilds, and leaves no stale manifest.""" + web = _use_tmp_web_dir(tmp_path, monkeypatch) + app = rx.App() + app.add_page(_page_shared_one, route="/one") + pages = list(app._unevaluated_pages.values()) + ctx = _compile(pages) + disk_cache.write_manifest(ctx, pages, ctx.all_imports, root=tmp_path) + _stub_externals(app, monkeypatch) + manifest_path = web / disk_cache._MANIFEST_FILE + manifest = json.loads(manifest_path.read_text()) + _stale_dep(manifest, pages[0].route, str(tmp_path / "data.md")) + manifest_path.write_text(json.dumps(manifest)) + + def fail(*args, **kwargs): + msg = "boom" + raise ValueError(msg) + + monkeypatch.setattr(disk_cache, "_update_manifest_for_misses", fail) + assert ( + disk_cache.try_incremental_rebuild( + app, compiler_plugins=[], prerender_routes=False, root=tmp_path + ) + is True + ) + assert not manifest_path.exists() + + def test_incremental_miss_keeps_the_memo_the_app_root_shares( tmp_path, monkeypatch, preserve_memo_registries ): diff --git a/tests/units/utils/test_compile_daemon.py b/tests/units/utils/test_compile_daemon.py index be949e373e4..7d49ad32459 100644 --- a/tests/units/utils/test_compile_daemon.py +++ b/tests/units/utils/test_compile_daemon.py @@ -1179,6 +1179,35 @@ def import_app(**kwargs): assert "REFLEX_COMPILE_DAEMON_PRECOMPILED" not in os.environ +def test_sigterm_during_the_initial_compile_stops_the_daemon(tmp_path, monkeypatch): + """SIGTERM is not a broken initial compile: the daemon exits, not watches.""" + from reflex.utils import prerequisites + + monkeypatch.chdir(tmp_path) + monkeypatch.setattr(compile_daemon, "_lock_path", lambda: tmp_path / "c.lock") + monkeypatch.setattr(compile_daemon, "_reload_roots", lambda: [tmp_path]) + monkeypatch.setattr(compile_daemon, "_prepare_fork_parent", lambda roots: None) + monkeypatch.setattr( + compile_daemon._WatchState, + "build", + lambda *args: compile_daemon._WatchState([tmp_path], tmp_path), + ) + watched = [] + monkeypatch.setattr( + compile_daemon, "_next_changes", lambda *args: watched.append(args) + ) + monkeypatch.delenv("REFLEX_COMPILE_DAEMON_PRECOMPILED", raising=False) + + def terminated(**kwargs): + """Receive SIGTERM mid-compile.""" + compile_daemon._exit_on_sigterm(0, None) + + monkeypatch.setattr(prerequisites, "get_compiled_app", terminated) + with pytest.raises(compile_daemon._Terminated): + compile_daemon._serve() + assert watched == [] + + def test_watcher_reconciles_edits_after_an_idle_batch(tmp_path, monkeypatch): """A late edit must be included even if its OS event is in a later batch.""" import watchfiles @@ -1409,7 +1438,9 @@ def test_cold_compile_without_a_manifest_does_not_fork(monkeypatch): monkeypatch.setattr(compile_daemon, "_single_threaded", lambda: True) monkeypatch.setattr(disk_cache, "load_manifest", lambda: None) - monkeypatch.setattr(compile_daemon.os, "fork", Mock(side_effect=AssertionError)) + monkeypatch.setattr( + compile_daemon.os, "fork", Mock(side_effect=AssertionError), raising=False + ) assert compile_daemon.compiled_in_predicting_child(lambda: True) is False diff --git a/tests/units/utils/test_serializers.py b/tests/units/utils/test_serializers.py index 66d4964a052..51279b8c396 100644 --- a/tests/units/utils/test_serializers.py +++ b/tests/units/utils/test_serializers.py @@ -538,6 +538,18 @@ def test_serialize(value: Any, expected: str): assert json.loads(json_dumps(value)) == json.loads(json_dumps(expected)) +class _ReversedSet(set): + """A set that iterates in descending order, unlike small-int sets.""" + + def __iter__(self): + """Iterate in descending order. + + Returns: + The descending iterator. + """ + return iter(sorted(set.__iter__(self), reverse=True)) + + def test_sorted_set_is_deterministic_and_serialize_set_is_not_sorted(): """Compiled output sorts sets; a state delta keeps the cheap list. @@ -552,7 +564,8 @@ def test_sorted_set_is_deterministic_and_serialize_set_is_not_sorted(): # Unsortable element mixes sort by their serialized form. assert serializers.sorted_set({1, "a"}) == serializers.sorted_set({"a", 1}) assert set(serializers.serialize_set({1, "a"})) == {1, "a"} - assert serializers.serialize_set({3, 1, 2}) == list({3, 1, 2}) + # The state-delta path keeps the set's own order: no sort per delta. + assert serializers.serialize_set(_ReversedSet({1, 2, 3})) == [3, 2, 1] assert str(LiteralVar.create({"b", "c", "a"})) == '["a", "b", "c"]' assert str(LiteralVar.create({3, 1, 2})) == "[1, 2, 3]" From 9752de1e60cd1553ef3f06d8880c55c854eb7bdc Mon Sep 17 00:00:00 2001 From: Farhan <www.mfarvirus@gmail.com> Date: Thu, 22 Oct 2026 01:56:46 +0000 Subject: [PATCH 56/56] test(memo): let React Compiler compile the derived-count memo Content-addressed auto memos moved the page's compilable memos out of the user memo module, leaving only DerivedCount, which React Compiler never compiled: its id renders a ref, written into the global refs during render. A class name keeps the lookup without the ref. Claude-Session: https://claude.ai/code/session_01PXcFKAgnzhgdnSvhuCmxWE --- tests/integration/tests_playwright/test_memo.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/integration/tests_playwright/test_memo.py b/tests/integration/tests_playwright/test_memo.py index 824eb2dda3b..34ce71d76ea 100644 --- a/tests/integration/tests_playwright/test_memo.py +++ b/tests/integration/tests_playwright/test_memo.py @@ -116,7 +116,8 @@ def derived_count(divisor: rx.Var[int]) -> rx.Component: """ return rx.text( MemoState.numbers.filter(lambda number: number % divisor == 0).length(), - id="derived-count", + # No ``id``: its ref write keeps React Compiler from compiling the memo. + class_name="derived-count", ) def index() -> rx.Component: @@ -364,7 +365,7 @@ def test_memo_derived_array_updates(memo_app: AppHarness, page: Page) -> None: page: Playwright page. """ _load_page(page, memo_app) - count = page.locator("#derived-count") + count = page.locator(".derived-count") expect(count).to_have_text("2") page.locator("#memo-input").fill("before replacement")