diff --git a/docs/app/reflex_docs/pages/docs/component.py b/docs/app/reflex_docs/pages/docs/component.py index 1c6bfd89137..119d2e39af8 100644 --- a/docs/app/reflex_docs/pages/docs/component.py +++ b/docs/app/reflex_docs/pages/docs/component.py @@ -57,22 +57,36 @@ def get_code_style(color: str): "Union": "gray", } -count = 0 +def get_id(identity: str) -> str: + """Derive a stable state variable name from a property-control identity. -def get_id(s): - global count - count += 1 - s = str(count) - hash_object = hashlib.sha256(s.encode()) - hex_dig = hash_object.hexdigest() - return "a_" + hex_dig[:8] + Args: + identity: Qualified component, property and control kind. + + Returns: + A valid state variable name independent of page evaluation order. + """ + return "a_" + hashlib.sha256(identity.encode()).hexdigest() class PropDocsState(rx.State): """Container for dynamic vars used by the prop docs.""" +def _ensure_prop_var(name: str, type_: type, default: object) -> None: + """Register a property-control variable once, including its setter. + + Args: + name: Stable state variable name. + type_: Control value type. + default: Initial control value. + """ + if name not in PropDocsState.__fields__: + PropDocsState.add_var(name, type_, default) + PropDocsState._create_setter(name, getattr(PropDocsState, name)) + + EXCLUDED_COMPONENTS = [ "Theme", "ThemePanel", @@ -163,11 +177,12 @@ def render_select(prop: PropDocumentation, component: type[Component], prop_dict "default_open", "default_checked", ]: - name = get_id(f"{component.__qualname__}_{prop.name}") + name = get_id( + f"{component.__module__}.{component.__qualname__}.{prop.name}" + ) default = prop.name == "loading" and component.__name__ == "Spinner" - PropDocsState.add_var(name, bool, default) + _ensure_prop_var(name, bool, default) var = getattr(PropDocsState, name) - PropDocsState._create_setter(name, var) setter = getattr(PropDocsState, f"set_{name}") prop_dict[prop.name] = var return _bool_pills(var, setter) @@ -196,20 +211,20 @@ def render_select(prop: PropDocumentation, component: type[Component], prop_dict if str(lit_arg) != "" ] option = literal_values[0] - name = get_id(f"{component.__qualname__}_{prop.name}") - PropDocsState.add_var(name, str, option) + name = get_id( + f"{component.__module__}.{component.__qualname__}.{prop.name}" + ) + _ensure_prop_var(name, str, option) var = getattr(PropDocsState, name) - PropDocsState._create_setter(name, var) setter = getattr(PropDocsState, f"set_{name}") prop_dict[prop.name] = var return _pill_row(literal_values, var, setter) # Get the first non-empty option. non_empty_args = [a for a in type_.__args__ if str(a) != ""] option = non_empty_args[0] if non_empty_args else type_.__args__[0] - name = get_id(f"{component.__qualname__}_{prop.name}") - PropDocsState.add_var(name, str, option) + name = get_id(f"{component.__module__}.{component.__qualname__}.{prop.name}") + _ensure_prop_var(name, str, option) var = getattr(PropDocsState, name) - PropDocsState._create_setter(name, var) setter = getattr(PropDocsState, f"set_{name}") prop_dict[prop.name] = var @@ -358,10 +373,11 @@ def prop_docs( expanded_name = None expanded = None if is_long_row: - expanded_name = get_id(f"{component.__qualname__}_{prop.name}_expanded") - PropDocsState.add_var(expanded_name, bool, False) + expanded_name = get_id( + f"{component.__module__}.{component.__qualname__}.{prop.name}.expanded" + ) + _ensure_prop_var(expanded_name, bool, False) expanded = getattr(PropDocsState, expanded_name) - PropDocsState._create_setter(expanded_name, expanded) cell_content_class = ( rx.cond(expanded, "cell-content", _PROPS_TABLE_COMPACT_CELL_CLASS) @@ -949,19 +965,9 @@ def multi_docs( ll_component_list: list | None = None, source: str | None = None, ): - shared = shared_html_props([item[0] for item in component_list[1:]]) - components = [ - component_docs(component_tuple, previews, shared) - for component_tuple in component_list[1:] - ] ll_actual_path = actual_path.replace(".md", "-ll.md") ll_doc_exists = os.path.exists(ll_actual_path) ll_list = ll_component_list if ll_component_list is not None else component_list - ll_shared = shared_html_props([item[0] for item in ll_list[1:]]) - ll_components = [ - component_docs(component_tuple, previews, ll_shared) - for component_tuple in ll_list[1:] - ] def shared_reference(props): """Render the common reference once, retaining it in prerendered HTML.""" @@ -1040,6 +1046,12 @@ def links(current_page, ll_doc_exists, path): source_path=actual_path, ) def out(): + # Build prop docs during page eval so imports stay cheap. + shared = shared_html_props([item[0] for item in component_list[1:]]) + components = [ + component_docs(component_tuple, previews, shared) + for component_tuple in component_list[1:] + ] toc = get_docgen_toc(actual_path) # Reuse the source already read by the caller to avoid a second read. doc_content = ( @@ -1092,6 +1104,11 @@ def out(): source_path=ll_actual_path, ) def ll(): + ll_shared = shared_html_props([item[0] for item in ll_list[1:]]) + ll_components = [ + component_docs(component_tuple, previews, ll_shared) + for component_tuple in ll_list[1:] + ] ll_virtual = virtual_path.replace(".md", "-ll.md") toc = get_docgen_toc(ll_actual_path) doc_content = Path(ll_actual_path).read_text(encoding="utf-8") diff --git a/docs/app/tests/test_prop_docs_ids.py b/docs/app/tests/test_prop_docs_ids.py new file mode 100644 index 00000000000..f5ef5877036 --- /dev/null +++ b/docs/app/tests/test_prop_docs_ids.py @@ -0,0 +1,36 @@ +"""Regression coverage for stable property-control identities.""" + + +def test_prop_ids_do_not_depend_on_evaluation_order(): + """Partial page compiles must reuse the same state variable names.""" + from reflex_docs.pages.docs.component import get_id + + first = get_id("module.Button.size") + other = get_id("module.Text.size") + expanded = get_id("module.Button.size.expanded") + assert get_id("module.Button.size") == first + assert len({first, other, expanded}) == 3 + + +def test_repeated_prop_controls_reuse_their_state_variable(): + """The same component documented twice can share its stable control ID.""" + from typing import Literal + + import reflex as rx + from reflex_components_radix.themes.components.button import Button + from reflex_docgen import PropDocumentation + + from reflex_docs.pages.docs.component import prop_docs, render_select + + prop = PropDocumentation( + name="size", + type=rx.Var[Literal["1", "2", "3"]], + description="Long description " * 20, + default_value="1", + ) + first = {} + second = {} + render_select(prop, Button, first) + render_select(prop, Button, second) + assert str(first["size"]) == str(second["size"]) + assert prop_docs(prop, Button)[2] == prop_docs(prop, Button)[2] diff --git a/news/+compile-daemon-windows-record.bugfix.md b/news/+compile-daemon-windows-record.bugfix.md new file mode 100644 index 00000000000..c08ca75707e --- /dev/null +++ b/news/+compile-daemon-windows-record.bugfix.md @@ -0,0 +1 @@ +Avoid Windows hot-reload failures when a backend worker briefly holds the compile-input record open. diff --git a/news/+content-addressed-auto-memos.misc.md b/news/+content-addressed-auto-memos.misc.md new file mode 100644 index 00000000000..a220a183f26 --- /dev/null +++ b/news/+content-addressed-auto-memos.misc.md @@ -0,0 +1 @@ +Compiler-generated memo components (the auto-memoized stateful subtrees) now compile to one file each under `.web/utils/components/`, named by their content, instead of one file per source module under `.web/app_components/`; identical subtrees anywhere in the app share one file. `@rx.memo` components keep mirroring their source modules. diff --git a/news/6688.feature.md b/news/6688.feature.md new file mode 100644 index 00000000000..9fd7a9e65c9 --- /dev/null +++ b/news/6688.feature.md @@ -0,0 +1 @@ +Added an experimental incremental compile cache, enabled with `REFLEX_COMPILE_CACHE=1`. A compile reuses the previous build in `.web` and recompiles only the pages whose inputs changed, and `reflex run` keeps a warm compile daemon so a hot reload rebuilds only what the edit touched. Off by default. diff --git a/packages/reflex-base/news/6688.misc.md b/packages/reflex-base/news/6688.misc.md new file mode 100644 index 00000000000..f3abd7e694f --- /dev/null +++ b/packages/reflex-base/news/6688.misc.md @@ -0,0 +1 @@ +Added an optional per-page source-read recorder hook (`page_source_recorder` in the compiler plugin) used by the incremental compile cache to track the exact files each page reads during evaluation. Auto-generated unique ref names are now reproducible: each page draws them from a sequence seeded by its route, a state's default values from one seeded by the state's name, and whatever compiles after the pages from a sequence that never repeats a name drawn at import. Sets are sorted in compiled literals only; a state delta serializes a set as a plain list again. diff --git a/packages/reflex-base/src/reflex_base/components/memo.py b/packages/reflex-base/src/reflex_base/components/memo.py index e447bb6e400..e5200782440 100644 --- a/packages/reflex-base/src/reflex_base/components/memo.py +++ b/packages/reflex-base/src/reflex_base/components/memo.py @@ -5,6 +5,8 @@ import dataclasses import inspect from collections.abc import Callable, Iterator, Mapping, Sequence +from contextlib import contextmanager +from contextvars import ContextVar from copy import copy from enum import Enum from functools import cache, partial, update_wrapper @@ -289,6 +291,12 @@ class MemoDefinition: # used for memos that can't be mirrored. ``kw_only`` so subclasses can keep # their own required fields. source_module: str | None = dataclasses.field(default=None, kw_only=True) + # The page route whose evaluation triggered this memo's registration (see + # ``registration_owner``), or None for memos registered at module import + # outside any page evaluation. The compile cache attributes the memo's + # output file to this route: the file is reusable while the route's page + # is, and is recompiled by re-evaluating that page. + owner_route: str | None = dataclasses.field(default=None, kw_only=True) @dataclasses.dataclass(frozen=True, slots=True) @@ -589,6 +597,36 @@ def reset_memo_component_classes() -> None: MEMOS: dict[tuple[str, str | None], MemoDefinition] = {} +# The route of the page currently being evaluated, if any. Modules imported +# during a page's evaluation register their memos inside this scope, so the +# registered definition records that page as its owner. +_registration_owner: ContextVar[str | None] = ContextVar( + "_registration_owner", default=None +) + + +@contextmanager +def registration_owner(route: str) -> Iterator[None]: + """Attribute memo registrations to the page route that triggers them. + + Salsa-style entity ownership: a memo registered while a page evaluates + (e.g. from a module imported inside the page function) is an artifact of + that page's compilation, not of app import. Recording the owning route + lets the compile cache validate the memo's output file by the page's + hit/miss status and recompile it by re-evaluating that page. + + Args: + route: The route of the page being evaluated. + + Yields: + None. + """ + token = _registration_owner.set(route) + try: + yield + finally: + _registration_owner.reset(token) + def _memo_registry_key(definition: MemoDefinition) -> tuple[str, str | None]: """Get the registry key for a memo. @@ -650,7 +688,7 @@ def _register_memo_definition(definition: MemoDefinition) -> None: ) raise ValueError(msg) - MEMOS[key] = definition + MEMOS[key] = dataclasses.replace(definition, owner_route=_registration_owner.get()) def _annotation_inner_type(annotation: Any) -> Any: diff --git a/packages/reflex-base/src/reflex_base/environment.py b/packages/reflex-base/src/reflex_base/environment.py index 00e38b9e882..efc0e724af3 100644 --- a/packages/reflex-base/src/reflex_base/environment.py +++ b/packages/reflex-base/src/reflex_base/environment.py @@ -752,6 +752,23 @@ class EnvironmentVariables: # If this env var is set to "yes", App.compile will be a no-op REFLEX_SKIP_COMPILE: EnvVar[bool] = env_var(False, internal=True) + # Experimental: incremental compile cache. A fresh compile process (e.g. a + # reflex-run hot-reload worker) reuses each page's compiled output from an + # on-disk manifest and recompiles only the pages whose source changed. + # See reflex/compiler/disk_cache.py and reflex/compiler/page_cache.py. + REFLEX_COMPILE_CACHE: EnvVar[bool] = env_var(False) + + # Set only in the compile daemon's own process (and its forked children): + # marks the process that owns .web while a daemon runs, so backend reload + # workers of the same `reflex run` skip frontend compilation. + REFLEX_COMPILE_DAEMON: EnvVar[bool] = env_var(False, internal=True) + + # Set by the compile daemon for its compile children: a file the child + # appends its telemetry events to instead of sending them, which the + # daemon sends once the child exited (a forked child must not start the + # telemetry thread, nor touch macOS's proxy lookup after fork). + REFLEX_TELEMETRY_SPOOL: EnvVar[Path | None] = env_var(None, internal=True) + # Inherited by uvicorn/granian reload workers so the backend can distinguish # dev reload-capable worker boots from other backend starts. Never set in prod. REFLEX_DEV_BACKEND_RELOAD_ACTIVE: EnvVar[bool] = env_var(False, internal=True) diff --git a/packages/reflex-base/src/reflex_base/plugins/compiler.py b/packages/reflex-base/src/reflex_base/plugins/compiler.py index aef1cb7dd66..4b0ec3d668d 100644 --- a/packages/reflex-base/src/reflex_base/plugins/compiler.py +++ b/packages/reflex-base/src/reflex_base/plugins/compiler.py @@ -6,6 +6,7 @@ import dataclasses import inspect from collections.abc import Callable, Sequence +from contextlib import AbstractContextManager from typing import TYPE_CHECKING, Any, ClassVar, Protocol, TypeAlias, TypeVar, cast from reflex_base.components.component import BaseComponent, Component @@ -32,6 +33,10 @@ _BaseComponentT = TypeVar("_BaseComponentT", bound=BaseComponent) +#: Optional recorder for source files read during each page evaluation. +page_source_recorder: Callable[[], AbstractContextManager[set[str]]] | None = None + + class PageDefinition(Protocol): """Protocol for page-like objects compiled by :class:`CompileContext`.""" @@ -599,6 +604,16 @@ class PageContext(BaseContext): output_path: str | None = None output_code: str | None = None source_module: str | None = None + # Source files read while evaluating this page, when a recorder is installed. + source_files: set[str] = dataclasses.field(default_factory=set) + # The page's unique-name generator state after evaluation, resumed for its + # render so its generated names never depend on other pages. + unique_name_state: Any = None + # The auto-memo components this page's output imports, by registry key + # (filled by ``MemoizeStatefulPlugin``). + memo_uses: dict[tuple[str, str | None], Any] = dataclasses.field( + default_factory=dict + ) # Stack of ``id(component)`` for components whose subtree is # memoize-suppressed. Populated by ``MemoizeStatefulPlugin`` when it # encounters a ``MemoizationLeaf``-style snapshot boundary and popped on @@ -673,18 +688,23 @@ class CompileContext(BaseContext): app_wrap_components: dict[tuple[int, str], Component] = dataclasses.field( default_factory=dict ) - stateful_routes: dict[str, None] = dataclasses.field(default_factory=dict) + # Routes whose evaluation defined new state classes, mapped to the full + # names of the states each page defined. + stateful_routes: dict[str, list[str]] = dataclasses.field(default_factory=dict) # Auto-memoize wrapper tags seen during the tree walk (populated by # ``MemoizeStatefulPlugin``). memoize_wrappers: dict[str, None] = dataclasses.field(default_factory=dict) # Compiler-generated memo definitions for auto-memoized stateful wrappers. - # Keyed by ``(tag, source_module)`` so identical-rendering subtrees from - # different user modules each get their own entry and emit into the right - # mirrored memo file. Stored as ``Any`` to avoid an import cycle with - # ``reflex_base.components.memo``. + # Keyed by ``(tag, source_module)``; an auto memo is content-addressed by + # its tag and carries no source module, so identical subtrees anywhere in + # the app share one definition and one output file. Stored as ``Any`` to + # avoid an import cycle with ``reflex_base.components.memo``. auto_memo_components: dict[tuple[str, str | None], Any] = dataclasses.field( default_factory=dict ) + # The auto memo tags the app root imports, including those a page also + # uses (filled when the app root is built). + app_root_memo_keys: set[str] = dataclasses.field(default_factory=set) def compile( self, @@ -705,7 +725,18 @@ def compile( """ from reflex.compiler import compiler from reflex.state import all_base_state_classes + from reflex_base.components.memo import registration_owner + from reflex_base.vars.base import ( + names_before_first_compile, + restore_unique_variable_name_state, + seed_unique_variable_names, + unique_variable_name_state, + ) + # The names drawn before the compile (at import, in the serving + # process) stay taken: whatever draws names after the pages must not + # repeat them. + names_before = names_before_first_compile() self.ensure_context_attached() self.compiled_pages.clear() self.all_imports.clear() @@ -713,16 +744,36 @@ def compile( self.stateful_routes.clear() self.memoize_wrappers.clear() self.auto_memo_components.clear() + self.app_root_memo_keys.clear() + recorder = page_source_recorder for page in self.pages: page_fn = page.component n_states_before = len(all_base_state_classes) - page_ctx = self.hooks.eval_page( - page_fn, - page=page, - compile_context=self, - **kwargs, - ) + # Names generated for a page are a function of the page alone, so + # recompiling any subset of pages reproduces them (see + # ``seed_unique_variable_names``). + seed_unique_variable_names(page.route) + # Memos registered while this page evaluates (e.g. from modules + # imported inside the page function) are owned by this route. + with registration_owner(page.route): + if recorder is not None: + with recorder() as read_set: + page_ctx = self.hooks.eval_page( + page_fn, + page=page, + compile_context=self, + **kwargs, + ) + if page_ctx is not None: + page_ctx.source_files = read_set + else: + page_ctx = self.hooks.eval_page( + page_fn, + page=page, + compile_context=self, + **kwargs, + ) if page_ctx is None: page_name = getattr(page_fn, "__name__", repr(page_fn)) msg = ( @@ -735,8 +786,14 @@ def compile( raise RuntimeError(msg) if len(all_base_state_classes) > n_states_before: - self.stateful_routes[page.route] = None - + # Record which states this page defined (registration order is + # insertion order), so the compile cache can fingerprint the + # page's contribution to the contexts file. + self.stateful_routes[page.route] = list(all_base_state_classes)[ + n_states_before: + ] + + page_ctx.unique_name_state = unique_variable_name_state() self.compiled_pages[page_ctx.route] = page_ctx if evaluate_progress is not None: @@ -747,6 +804,8 @@ def compile( self.compiled_pages.values(), strict=True, ): + restore_unique_variable_name_state(page_ctx.unique_name_state) + page_ctx.unique_name_state = None with page_ctx: page_ctx.root_component = self.hooks.compile_component( page_ctx.root_component, @@ -759,7 +818,6 @@ def compile( compile_context=self, **kwargs, ) - page_ctx.frontend_imports = page_ctx.merged_imports(collapse=True) self.all_imports = merge_imports( self.all_imports, page_ctx.frontend_imports @@ -772,8 +830,42 @@ def compile( if render_progress is not None: render_progress() + # Whatever compiles after the pages (app root, memo modules) starts + # from the same state on every compile, and never repeats a name the + # process drew before the pages. + seed_unique_variable_names("app", reserved=names_before) return self.compiled_pages + def absorb(self, other: CompileContext, pages: Sequence[PageDefinition]) -> None: + """Adopt the pages another context compiled, so none is compiled twice. + + Merges the other context's per-page results and app-wide accumulations + into this one and orders the compiled pages by ``pages``, which becomes + this context's page list. + + Args: + other: A context that compiled a disjoint set of pages. + pages: The complete page list, in compile order. + """ + compiled = {**self.compiled_pages, **other.compiled_pages} + self.pages = pages + self.compiled_pages = {page.route: compiled[page.route] for page in pages} + self.all_imports = merge_imports(self.all_imports, other.all_imports) + self.app_wrap_components = {} + for page in pages: + self.app_wrap_components.update( + self.compiled_pages[page.route].app_wrap_components + ) + # In page order, like a compile that evaluated every page. + stateful_routes = {**self.stateful_routes, **other.stateful_routes} + self.stateful_routes = { + page.route: stateful_routes[page.route] + for page in pages + if page.route in stateful_routes + } + self.memoize_wrappers.update(other.memoize_wrappers) + self.auto_memo_components.update(other.auto_memo_components) + __all__ = [ "BaseContext", diff --git a/packages/reflex-base/src/reflex_base/utils/imports.py b/packages/reflex-base/src/reflex_base/utils/imports.py index f82fd8d0090..6dc24b53e3a 100644 --- a/packages/reflex-base/src/reflex_base/utils/imports.py +++ b/packages/reflex-base/src/reflex_base/utils/imports.py @@ -184,6 +184,10 @@ def collapse_imports( ) -> ParsedImportDict: """Remove all duplicate ImportVar within an ImportDict. + Deduplication preserves first-occurrence order: compiled import statements + follow this order, and a hash-seed-dependent order would rewrite every + page/memo module on each dev reload, defeating granular HMR. + Args: imports: The import dict to collapse. @@ -191,13 +195,7 @@ def collapse_imports( The collapsed import dict. """ return { - lib: ( - # dict.fromkeys, not set: set order varies with PYTHONHASHSEED, so - # identical compiles would emit different memo names. - list(dict.fromkeys(import_vars)) - if isinstance(import_vars, list) - else list(import_vars) - ) + lib: list(dict.fromkeys(import_vars)) for lib, import_vars in ( imports if isinstance(imports, tuple) else imports.items() ) diff --git a/packages/reflex-base/src/reflex_base/utils/serializers.py b/packages/reflex-base/src/reflex_base/utils/serializers.py index 29ac48cf60f..ed2ef1816e8 100644 --- a/packages/reflex-base/src/reflex_base/utils/serializers.py +++ b/packages/reflex-base/src/reflex_base/utils/serializers.py @@ -411,6 +411,28 @@ def serialize_set(value: set) -> list: return list(value) +def sorted_set(value: set | frozenset) -> list: + """Order a set for output that must be the same in every process. + + A set's iteration order varies with the per-process hash seed; compiled + JSX embeds set literals, so a compile sorts them (a state delta does + not: its order carries no meaning and sorting costs on every update). + + Args: + value: The set. + + Returns: + The elements, sorted; elements of mixed types sort by their + serialized form. + """ + try: + return sorted(value) + except TypeError: + return sorted( + value, key=lambda item: json.dumps(item, default=serialize, sort_keys=True) + ) + + @serializer def serialize_sequence(value: Sequence) -> list: """Serialize a sequence to a JSON serializable list. diff --git a/packages/reflex-base/src/reflex_base/vars/base.py b/packages/reflex-base/src/reflex_base/vars/base.py index 0671d80c917..befe85d26c0 100644 --- a/packages/reflex-base/src/reflex_base/vars/base.py +++ b/packages/reflex-base/src/reflex_base/vars/base.py @@ -12,11 +12,12 @@ import inspect import json import logging +import random import re import string import warnings from abc import ABCMeta -from collections.abc import Callable, Coroutine, Iterable, Mapping, Sequence +from collections.abc import Callable, Coroutine, Iterable, Iterator, Mapping, Sequence from dataclasses import MISSING from decimal import Decimal from types import CodeType, FunctionType @@ -48,7 +49,6 @@ from reflex_base.environment import _on_env_var_set, environment from reflex_base.utils import exceptions, imports, serializers, types from reflex_base.utils.compat import MISSING_TYPE, annotations_from_namespace -from reflex_base.utils.decorator import once from reflex_base.utils.exceptions import ( BackendVarFormatError, ComputedVarSignatureError, @@ -3718,12 +3718,111 @@ def get_uuid_string_var() -> Var: # Set of unique variable names. USED_VARIABLES = set() +_UNIQUE_NAME_RNG = random.Random(42) -@once -def _rng(): - import random - return random.Random(42) +#: The names the process drew before its first compile (at import), which +#: every compile's post-page phase keeps reserved (see +#: :func:`names_before_first_compile`). +_names_before_first_compile: frozenset[str] | None = None + + +def reset_unique_variable_names() -> None: + """Reset the deterministic unique-name generator to its initial state. + + Names only need to be unique within one compile, so resetting before each + compile makes auto-generated ref names reproducible. A process that + re-imports its first-party code (the compile daemon's child) resets + first, so the re-imported modules draw the names a cold process draws. + """ + global _names_before_first_compile + USED_VARIABLES.clear() + _UNIQUE_NAME_RNG.seed(42) + _names_before_first_compile = None + + +def names_before_first_compile() -> frozenset[str]: + """The names drawn before the first compile of this process. + + Module-level components draw their names at import, and the process + keeps using them; the names a compile draws after its pages (the app + root, runtime components) must not repeat them. Captured once so every + compile of the process draws the same post-page names. + + Returns: + The reserved names. + """ + global _names_before_first_compile + if _names_before_first_compile is None: + _names_before_first_compile = frozenset(USED_VARIABLES) + return _names_before_first_compile + + +def seed_unique_variable_names(seed: str, *, reserved: Iterable[str] = ()) -> None: + """Start a fresh, seed-determined name sequence (one per compiled page). + + Generated names are order-dependent, so a process-wide sequence gives a + page different names depending on which pages compiled before it. An + incremental compile of a subset of pages would then reuse names that + other, cached pages already hold. Seeding per page from its route makes + each page's names a function of the page alone. + + Args: + seed: The page route (or another stable label). + reserved: Names the sequence must not hand out again (names drawn + before the compile, which the process keeps using). + """ + USED_VARIABLES.clear() + USED_VARIABLES.update(reserved) + _UNIQUE_NAME_RNG.seed(f"reflex-unique-names:{seed}") + + +@contextlib.contextmanager +def seeded_unique_variable_names(seed: str) -> Iterator[None]: + """Draw names from a seed-determined sequence within the block. + + The names drawn in the block are a function of ``seed`` alone, so two + processes evaluating the same thing (a state's default values, say) + agree on them whatever they drew before. The outer sequence resumes + afterwards, with the names drawn in the block reserved. + + Args: + seed: A stable label for what the block evaluates. + + Yields: + None. + """ + rng_state = _UNIQUE_NAME_RNG.getstate() + used_before = set(USED_VARIABLES) + seed_unique_variable_names(seed) + try: + yield + finally: + drawn = set(USED_VARIABLES) + _UNIQUE_NAME_RNG.setstate(rng_state) + USED_VARIABLES.clear() + USED_VARIABLES.update(used_before | drawn) + + +def unique_variable_name_state() -> tuple[Any, frozenset[str]]: + """Capture the generator state so a page's sequence can resume later. + + Returns: + An opaque state for :func:`restore_unique_variable_name_state`. + """ + return _UNIQUE_NAME_RNG.getstate(), frozenset(USED_VARIABLES) + + +def restore_unique_variable_name_state(state: tuple[Any, frozenset[str]]) -> None: + """Resume a page's name sequence captured by :func:`unique_variable_name_state`. + + Args: + state: The captured state. + """ + rng_state, used = state + _UNIQUE_NAME_RNG.setstate(rng_state) + USED_VARIABLES.clear() + USED_VARIABLES.update(used) def get_unique_variable_name() -> str: @@ -3732,7 +3831,7 @@ def get_unique_variable_name() -> str: Returns: The unique variable name. """ - name = "".join([_rng().choice(string.ascii_lowercase) for _ in range(8)]) + name = "".join([_UNIQUE_NAME_RNG.choice(string.ascii_lowercase) for _ in range(8)]) if name not in USED_VARIABLES: USED_VARIABLES.add(name) return name diff --git a/packages/reflex-base/src/reflex_base/vars/sequence.py b/packages/reflex-base/src/reflex_base/vars/sequence.py index 1ded6c57ab1..3e7e0a096c3 100644 --- a/packages/reflex-base/src/reflex_base/vars/sequence.py +++ b/packages/reflex-base/src/reflex_base/vars/sequence.py @@ -15,7 +15,7 @@ from reflex_base import constants from reflex_base.constants.base import REFLEX_VAR_OPENING_TAG, Dirs -from reflex_base.utils import console, types +from reflex_base.utils import console, serializers, types from reflex_base.utils.exceptions import VarTypeError from reflex_base.utils.imports import ImportDict, ImportVar from reflex_base.utils.types import GenericType, get_origin, unionize @@ -696,11 +696,18 @@ def create( Returns: The var. """ + if _var_type is None: + _var_type = figure_out_type(value) + # Compiled output embeds the literal; a set's order varies with the + # hash seed, so it is sorted once here, not on the delta path. + elements: Any = ( + serializers.sorted_set(value) if isinstance(value, set) else value + ) return LiteralArrayVar( _js_expr="", - _var_type=figure_out_type(value) if _var_type is None else _var_type, + _var_type=_var_type, _var_data=_var_data, - _var_value=value, + _var_value=elements, ) diff --git a/pyi_hashes.json b/pyi_hashes.json index 40fba0d5829..2898dd3def3 100644 --- a/pyi_hashes.json +++ b/pyi_hashes.json @@ -120,5 +120,5 @@ "packages/reflex-components-sonner/src/reflex_components_sonner/toast.pyi": "f170ac685b6ba5892370166c80684db3", "reflex/__init__.pyi": "1c841ad90d789507cd56d966b474d11c", "reflex/components/__init__.pyi": "9facd05a776d0641432696bbf8e34388", - "reflex/experimental/memo.pyi": "87e485038dc79f66b2ab15144845ef1e" + "reflex/experimental/memo.pyi": "c9d1c0edf60f1f98f30e3e3c98a63348" } diff --git a/reflex/app.py b/reflex/app.py index 12837ff60a2..d64dec437da 100644 --- a/reflex/app.py +++ b/reflex/app.py @@ -60,6 +60,7 @@ Send, is_non_string_sequence, ) +from reflex_base.vars.base import seeded_unique_variable_names from reflex_base.vars.dep_tracking import is_dependency from reflex_components_core.base.error_boundary import ErrorBoundary from reflex_components_core.base.fragment import Fragment @@ -474,6 +475,8 @@ class App(MiddlewareMixin, LifespanMixin): # A map from a page route to the component to render. Users should use `add_page`. _pages: dict[str, Component] = dataclasses.field(default_factory=dict) + _cached_component_counts: dict[str, int] | None = dataclasses.field(default=None) + # A mapping of pages which created states as they were being evaluated. _stateful_pages: dict[str, None] = dataclasses.field(default_factory=dict) @@ -805,6 +808,7 @@ def __call__(self) -> ASGIApp: self._compile( prerender_routes=should_prerender_routes(), trigger=trigger, + serving=True, ) # In preview mode the frontend is served as a mounted static bundle rather @@ -1067,6 +1071,7 @@ def _prepare_page( msg = "Route must be set if component is not a callable." raise exceptions.RouteValueError(msg) + default_404 = route == constants.Page404.SLUG and component is None if route == constants.Page404.SLUG: if component is None: from reflex_components_core.el.elements import span @@ -1085,7 +1090,10 @@ def _prepare_page( # Check if the route given is valid verify_route_validity(route) - if isinstance(component, Callable): + if default_404: + # The framework's page, so no user module to mirror. + source_module = None + elif isinstance(component, Callable): source_module = memo_paths.capture_source_module(component) else: # The user passed a pre-built Component instance — fall back to @@ -1354,12 +1362,15 @@ def _compile_page(self, route: str, save_page: bool = True): return n_states_before = len(all_base_state_classes) - component = compiler.compile_unevaluated_page( - route, - self._unevaluated_pages[route], - self.style, - self.theme, - ) + # The same names the compiling process drew for this page (a + # ``ClientStateVar`` created inside the page function is used by both). + with seeded_unique_variable_names(route): + component = compiler.compile_unevaluated_page( + route, + self._unevaluated_pages[route], + self.style, + self.theme, + ) # Indicate that evaluating this page creates one or more state classes. if len(all_base_state_classes) > n_states_before: @@ -1652,16 +1663,19 @@ def _should_compile(self) -> bool: if environment.REFLEX_SKIP_COMPILE.get(): return False - nocompile = prerequisites.get_web_dir() / constants.NOCOMPILE_FILE + # The nocompile marker is the backend's: its first worker consumes it + # and stands down. The compile daemon and its children never read it. + if not environment.REFLEX_COMPILE_DAEMON.get(): + nocompile = prerequisites.get_web_dir() / constants.NOCOMPILE_FILE + if nocompile.exists(): + nocompile.unlink(missing_ok=True) + return False - # Check the nocompile file. - if nocompile.exists(): - # Delete the nocompile file - nocompile.unlink(missing_ok=True) - return False + # A running compile daemon owns .web; backend workers only evaluate + # pages to register state. + from reflex.utils import compile_daemon - # By default, compile the app. - return True + return compile_daemon.owns_compilation() def _setup_sticky_badge(self): """Add the sticky badge to the app.""" @@ -1723,6 +1737,7 @@ def _compile( dry_run: bool = False, use_rich: bool = True, trigger: CompileTrigger | None = None, + serving: bool = False, ): """Compile the app and output it to the pages folder. @@ -1732,6 +1747,7 @@ def _compile( use_rich: Whether to use rich progress bars. trigger: Label identifying what initiated this compile. Recorded on the ``compile`` telemetry event. + serving: Whether this process serves the app after compiling. Raises: ReflexRuntimeError: When any page uses state, but no rx.State subclass is defined. @@ -1748,6 +1764,7 @@ def _compile( prerender_routes=prerender_routes, dry_run=dry_run, use_rich=use_rich, + serving=serving, ) return @@ -1759,6 +1776,7 @@ def _compile( prerender_routes=prerender_routes, dry_run=dry_run, use_rich=use_rich, + serving=serving, ) except Exception as exc: ctx.set_exception(exc) @@ -1786,6 +1804,13 @@ def _write_stateful_pages_marker(self): stateful_pages_marker = ( prerequisites.get_backend_dir() / constants.Dirs.STATEFUL_PAGES ) + content = json.dumps(list(self._stateful_pages)) + # Skip an identical rewrite so file watchers see no spurious change. + if ( + stateful_pages_marker.exists() + and stateful_pages_marker.read_text(encoding="utf-8") == content + ): + return stateful_pages_marker.parent.mkdir(parents=True, exist_ok=True) fd, tmp_path = tempfile.mkstemp( dir=stateful_pages_marker.parent, @@ -1795,8 +1820,7 @@ def _write_stateful_pages_marker(self): os.close(fd) tmp_marker = Path(tmp_path) try: - with tmp_marker.open("w", encoding="utf-8") as f: - json.dump(list(self._stateful_pages), f) + tmp_marker.write_text(content, encoding="utf-8") tmp_marker.chmod(0o644) for attempt in range(100): try: diff --git a/reflex/compiler/compiler.py b/reflex/compiler/compiler.py index dab3bfb578f..51f01615bf3 100644 --- a/reflex/compiler/compiler.py +++ b/reflex/compiler/compiler.py @@ -51,7 +51,12 @@ from reflex.compiler import templates, utils from reflex.compiler.plugins import default_page_plugins from reflex.compiler.plugins.memoize import MemoizeStatefulPlugin -from reflex.state import BaseState, code_uses_state_contexts, state_snapshot_hashes +from reflex.state import ( + BaseState, + _hash_frontend_schema, + code_uses_state_contexts, + state_snapshot_hashes, +) from reflex.utils import console, frontend_skeleton, path_ops, prerequisites from reflex.utils.exec import get_compile_context, is_prod_mode from reflex.utils.prerequisites import get_web_dir @@ -245,6 +250,7 @@ def _compile_contexts( theme: Component | None, *, component_imports: ParsedImportDict | None = None, + contexts_snapshot: utils.ContextsSnapshot | None = None, ) -> str: """Compile the initial state and contexts. @@ -252,6 +258,10 @@ def _compile_contexts( state: The app state. theme: The top-level app theme. component_imports: Optional accumulator for initial component dependencies. + contexts_snapshot: The state-tree inputs compiled for this contexts + output, when already available. They may include states that are + not in this process's state tree (states only registered while a + page evaluates, restored from the compile cache manifest). Returns: The compiled context file. @@ -260,30 +270,34 @@ def _compile_contexts( disable_react_owner_stacks = ( not is_prod_mode() and not environment.REFLEX_REACT_OWNER_STACKS.get() ) - initial_state, initial_state_json = ( - utils._compile_initial_state(state, component_imports=component_imports) - if state - else (None, None) - ) - - return ( - templates.context_template( - initial_state=initial_state, - initial_state_json=initial_state_json, - initial_state_hashes=state_snapshot_hashes(initial_state), - state_name=state.get_name(), - state_schema=state._to_frontend_schema(), - client_storage=utils.compile_client_storage(state), + if state is None: + return templates.context_template( is_dev_mode=not is_prod_mode(), default_color_mode=default_color_mode, disable_react_owner_stacks=disable_react_owner_stacks, ) - if state and initial_state is not None - else templates.context_template( - is_dev_mode=not is_prod_mode(), - default_color_mode=default_color_mode, - disable_react_owner_stacks=disable_react_owner_stacks, + if contexts_snapshot is None: + initial_state, initial_state_json = utils._compile_initial_state( + state, component_imports=component_imports + ) + client_storage = utils.compile_client_storage(state) + state_schema = state._to_frontend_schema() + else: + initial_state, client_storage, schema_entries = contexts_snapshot + initial_state_json = utils._serialize_initial_state( + initial_state, component_imports=component_imports ) + state_schema = _hash_frontend_schema(schema_entries) + return templates.context_template( + initial_state=initial_state, + initial_state_json=initial_state_json, + initial_state_hashes=state_snapshot_hashes(initial_state), + state_name=state.get_name(), + state_schema=state_schema, + client_storage=client_storage, + is_dev_mode=not is_prod_mode(), + default_color_mode=default_color_mode, + disable_react_owner_stacks=disable_react_owner_stacks, ) @@ -443,23 +457,22 @@ def _compile_root_stylesheet( target.parent.mkdir(parents=True, exist_ok=True) + # Skip rewriting an unchanged target: Vite watches .web/styles, and a + # rewrite-with-identical-content still fires an HMR update per reload. if stylesheet.suffix == ".css": - path_ops.cp(src=stylesheet, dest=target, overwrite=True) + data = stylesheet.read_bytes() + if not target.exists() or target.read_bytes() != data: + target.write_bytes(data) else: try: from sass import compile as sass_compile - target.write_text( - # libsass is untyped; compiling from a filename returns the CSS. - data=cast( - "str", - sass_compile( - filename=str(stylesheet), - output_style="compressed", - ), - ), - encoding="utf8", - ) + compiled_css = cast( + "str", + sass_compile(filename=str(stylesheet), output_style="compressed"), + ).encode("utf8") + if not target.exists() or target.read_bytes() != compiled_css: + target.write_bytes(compiled_css) except ImportError: failed_to_import_sass = True @@ -546,7 +559,6 @@ def _compile_memo_components( output_files: list[tuple[str, str]] = [] aggregate_imports: dict[str, list[ImportVar]] = {} unmirrored_files: list[tuple[str, str]] = [] - unmirrored_base_dir = utils.get_memo_components_dir() groups: collections.defaultdict[tuple[str, ...], _MemoGroup] = ( collections.defaultdict(_MemoGroup) ) @@ -557,10 +569,7 @@ def _emit_unmirrored( render_imports: dict, ) -> None: code, file_imports = compile_fn(render, render_imports) - unmirrored_files.append(( - _memo_component_file_path(unmirrored_base_dir, render["name"]), - code, - )) + unmirrored_files.append((utils.get_memo_component_path(render["name"]), code)) _extend_imports_in_place(aggregate_imports, file_imports) for memo in memos: @@ -670,19 +679,6 @@ def _compile_single_memo_function( return code, imports -def _memo_component_file_path(base_dir: str, name: str) -> str: - """Return the on-disk path for a per-memo module. - - Args: - base_dir: The directory that holds per-memo files. - name: The memo's export name. - - Returns: - The absolute path for the memo's ``.jsx`` file. - """ - return str(Path(base_dir) / f"{name}{constants.Ext.JSX}") - - def compile_document_root( head_components: list[Component], html_lang: str | None = None, @@ -766,6 +762,7 @@ def compile_contexts( theme: Component | None, *, component_imports: ParsedImportDict | None = None, + contexts_snapshot: utils.ContextsSnapshot | None = None, ) -> tuple[str, str]: """Compile the initial state / context. @@ -773,6 +770,7 @@ def compile_contexts( state: The app state. theme: The top-level app theme. component_imports: Optional accumulator for initial component dependencies. + contexts_snapshot: See :func:`_compile_contexts`. Returns: The path and code of the compiled context. @@ -781,7 +779,10 @@ def compile_contexts( output_path = utils.get_context_path() return output_path, _compile_contexts( - state, theme, component_imports=component_imports + state, + theme, + component_imports=component_imports, + contexts_snapshot=contexts_snapshot, ) @@ -1058,7 +1059,7 @@ def compile_unevaluated_page( meta_args["description"] = page.description # Add meta information to the component. - utils.add_meta( + component = utils.add_meta( component, **meta_args, ) @@ -1128,6 +1129,43 @@ def memoized_toast_provider() -> Component: return app_wrappers +def _build_app_root( + app: App, + page_app_wrap_components: dict[tuple[int, str], Component], + compile_ctx: CompileContext, +) -> tuple[Component, str | None]: + """Assemble the app root from the pages' app wraps. + + Memo definitions the root needs (stateful wraps, the hydrate fallback) are + registered on ``compile_ctx.auto_memo_components``. + + Args: + app: The app being compiled. + page_app_wrap_components: The app wraps the pages require. + compile_ctx: The compile context receiving the root's memo definitions. + + Returns: + The app root and the hydrate fallback's export name, if any. + """ + app_wrappers = _resolve_app_wrap_components(app, page_app_wrap_components) + app_root = _memoize_stateful_app_wraps(app._app_root(app_wrappers), compile_ctx) + hydrate_fallback = app._resolve_hydrate_fallback() + hydrate_fallback_export = None + if hydrate_fallback is not None: + hydrate_fallback._add_style_recursive(app.style) + # Compile the fallback through the memo pipeline so it lands in its own + # JS module; root.jsx then re-exports it as HydrateFallback. + hydrate_fallback_definition = create_component_memo( + hydrate_fallback, "hydrate_fallback" + ) + compile_ctx.auto_memo_components[ + hydrate_fallback_definition.export_name, None + ] = hydrate_fallback_definition + hydrate_fallback_export = hydrate_fallback_definition.export_name + compile_ctx.app_root_memo_keys.add(hydrate_fallback_export) + return app_root, hydrate_fallback_export + + def _memoize_stateful_app_wraps( app_root: Component, compile_context: CompileContext, @@ -1161,6 +1199,7 @@ def _memoize_stateful_app_wraps( page_context=page_context, compile_context=compile_context, ) + compile_context.app_root_memo_keys.update(tag for tag, _ in page_context.memo_uses) if not isinstance(compiled_root, Component): msg = "Compiled app root must be a Component." raise TypeError(msg) @@ -1206,6 +1245,144 @@ def _register_plugin_routes(app: App, plugins: Sequence[Plugin]) -> None: app._register_plugin_pages(plugins) +def _register_compiled_pages(app: App, compiled_pages: dict[str, PageContext]) -> None: + """Register evaluated page components and routes on the application. + + Args: + app: The application receiving compiled pages. + compiled_pages: Pages evaluated by this compilation. + + Raises: + TypeError: A compiled page root is not a Component. + """ + for route, page_ctx in compiled_pages.items(): + app._check_routes_conflict(route) + if not isinstance(page_ctx.root_component, Component): + msg = ( + f"Compiled page {route!r} root must be a Component before it can " + "be registered on the app." + ) + raise TypeError(msg) + app._pages[route] = page_ctx.root_component + + app._evaluated_pages.update(compiled_pages) + + +def _plugin_output_tasks( + plugins: Sequence[Plugin], + radix_themes_plugin: RadixThemesPlugin, + pages: Sequence[UnevaluatedPage], +): + """Collect output tasks from configured plugins. + + Args: + plugins: Configured plugins in execution order. + radix_themes_plugin: Resolved theme plugin. + pages: All registered page definitions. + + Returns: + Scheduled save and modify tasks. + """ + save_tasks: list[ + tuple[ + Callable[..., list[tuple[str, str]] | tuple[str, str] | None], + tuple[Any, ...], + dict[str, Any], + ] + ] = [] + modify_files_tasks: list[tuple[str, str, Callable[[str], str]]] = [] + + def add_save_task( + task_fn: Callable[..., list[tuple[str, str]] | tuple[str, str] | None], + /, + *args: Any, + **kwargs: Any, + ) -> None: + save_tasks.append((task_fn, args, kwargs)) + + for plugin in plugins: + plugin.pre_compile( + add_save_task=add_save_task, + add_modify_task=lambda *args, plugin=plugin: modify_files_tasks.append(( + plugin.__class__.__module__ + plugin.__class__.__name__, + *args, + )), + radix_themes_plugin=radix_themes_plugin, + unevaluated_pages=pages, + ) + + return save_tasks, modify_files_tasks + + +def _apply_plugin_outputs( + output_mapping: dict[Path, str], + plugins: Sequence[Plugin], + modify_files_tasks: Sequence[tuple[str, str, Callable[[str], str]]], + cached_sources: dict[str, str] | None = None, +) -> dict[str, str]: + """Apply static assets and modifiers, preserving unmodified source content. + + Args: + output_mapping: Fresh compiler and save-task output, updated in place. + plugins: Configured plugins. + modify_files_tasks: Ordered file modifications. + cached_sources: Original content of reused files before any modifiers. + + Returns: + Original contents needed to replay modifiers on the next cache hit. + """ + cached_sources = cached_sources or {} + sources = {} + for plugin in plugins: + for static_file_path, content in plugin.get_static_assets(): + path = utils.resolve_path_of_web_dir(static_file_path) + if path in output_mapping: + logger.warning( + f"Plugin {plugin.__class__.__name__} is overwriting existing files at {path}." + ) + output_mapping[path] = ( + content.decode("utf-8") if isinstance(content, bytes) else content + ) + + for plugin_name, file_path, modify_fn in modify_files_tasks: + path = utils.resolve_path_of_web_dir(file_path) + file_content = output_mapping.get(path, cached_sources.get(str(path))) + if file_content is None: + if path.exists(): + file_content = path.read_text() + else: + msg = f"Plugin {plugin_name} is trying to modify {path} but it does not exist." + raise FileNotFoundError(msg) + sources.setdefault(str(path), file_content) + output_mapping[path] = modify_fn(file_content) + + return sources + + +def _reset_compile_registries(compiler_plugins: Sequence[Plugin]) -> None: + """Reset the registries a compile fills before any page is evaluated. + + Args: + compiler_plugins: The resolved compiler plugins. + """ + from reflex_base.components.dynamic import ( + _bundle_library, + _reset_bundled_libraries_for_compile, + ) + + _reset_bundled_libraries_for_compile() + # Drop cached memo wrapper classes so each compile recomputes a memo's + # ``library`` from the current module layout (handles a module flipping to + # a package across hot reloads). + reset_memo_component_classes() + # Page evaluation rebuilds every chain that is not interned by handler, so + # entries from an earlier compile can only retain dead chains. + RegistrationContext.ensure_context()._reset_compile_caches() + for plugin in compiler_plugins: + for dependency in plugin.get_frontend_dependencies(): + _bundle_library(dependency) + + def _read_stateful_pages_marker() -> list[str] | None: """Read the routes that create state classes from a previous compile. @@ -1229,32 +1406,120 @@ def _read_stateful_pages_marker() -> list[str] | None: raise +def _compile_from_cache( + app: App, + compiler_plugins: Sequence[Plugin], + radix_themes_plugin: RadixThemesPlugin, + *, + prerender_routes: bool, + use_rich: bool, + registrations: dict[str, str], +) -> CompileContext | bool: + """Compile from the compile cache, writing the plugin outputs on success. + + Args: + app: The app. + compiler_plugins: The resolved compiler plugins. + radix_themes_plugin: The resolved Radix Themes plugin. + prerender_routes: Whether to prerender routes. + use_rich: Whether to show rich progress. + registrations: Each route's registration digest. + + Returns: + True when the cache compiled the app, the evaluated pages to adopt + when it fell back after evaluating, else False. + """ + from reflex.compiler import disk_cache + + config = get_config() + outputs: dict[Path, str] = {} + rebuilt = disk_cache.try_incremental_rebuild( + app, + compiler_plugins=compiler_plugins, + prerender_routes=prerender_routes, + use_rich=use_rich, + outputs=outputs, + registrations=registrations, + ) + if rebuilt is True: + manifest_path = disk_cache._manifest_path().absolute() + # The refreshed manifest is written last: a manifest on disk must + # never vouch for outputs that were not written, so until every + # other file is on disk there is none. + staged_manifest = outputs.pop(manifest_path, None) + manifest = ( + json.loads(staged_manifest) + if staged_manifest is not None + else disk_cache.load_manifest() + ) + manifest_path.unlink(missing_ok=True) + save_tasks, modify_tasks = _plugin_output_tasks( + config.plugins, + radix_themes_plugin, + list(app._unevaluated_pages.values()), + ) + stylesheet_path, stylesheet_code = compile_root_stylesheet( + app.stylesheets, app.reset_style, plugins=compiler_plugins + ) + outputs[utils.resolve_path_of_web_dir(stylesheet_path)] = stylesheet_code + for task_fn, args, kwargs in save_tasks: + result = task_fn(*args, **kwargs) + if result is not None: + for path, code in result if isinstance(result, list) else [result]: + outputs[utils.resolve_path_of_web_dir(path)] = code + sources = _apply_plugin_outputs( + outputs, + config.plugins, + modify_tasks, + manifest.get("plugin_sources", {}) if manifest is not None else {}, + ) + for path, code in outputs.items(): + utils.write_file(path, code) + if manifest is not None: + manifest["plugin_sources"] = sources + disk_cache._write(manifest) + return rebuilt + + def compile_app( app: App, *, prerender_routes: bool = False, dry_run: bool = False, use_rich: bool = True, + serving: bool = False, ) -> bool: """Compile an app using the compiler plugin pipeline. + Args: + app: The app to compile. + prerender_routes: Whether to prerender the routes. + dry_run: Whether to compile the app without saving it. + use_rich: Whether to use rich progress bars. + serving: Whether this process serves the app after compiling. It + needs every page evaluated to register their states, so the + incremental cache is not used. + Returns: ``True`` when a real frontend compile ran, ``False`` when the call short-circuited (backend-only paths that only re-evaluate pages). """ - from reflex_base.components.dynamic import ( - _bundle_library, - _reset_bundled_libraries_for_compile, - ) from reflex_base.utils.exceptions import ReflexRuntimeError app._apply_decorated_pages() config = get_config() _register_plugin_routes(app, config.plugins) app._pages = {} + app._cached_component_counts = None should_compile = app._should_compile() if not dry_run and not should_compile: + from reflex.utils import compile_daemon + + if compile_daemon.daemon_active(): + # The compile daemon reacts to the same save; read its marker only + # once it has finished rewriting .web for this change. + compile_daemon.wait_for_compile() stateful_pages = _read_stateful_pages_marker() else: stateful_pages = None @@ -1289,29 +1554,74 @@ def compile_app( app._add_optional_endpoints() return False - progress = console.progress() if use_rich else console.PoorProgress() - fixed_steps = 7 + cache_on = ( + not dry_run + and not serving + and environment.REFLEX_COMPILE_CACHE.get() + and config.app_module_import is None + ) + compiler_plugins, radix_themes_plugin = _resolve_radix_themes_plugin( app, config.plugins, ) - _reset_bundled_libraries_for_compile() - # Drop cached memo wrapper classes so each compile recomputes a memo's - # ``library`` from the current module layout (handles a module flipping to - # a package across hot reloads). - reset_memo_component_classes() - # Page evaluation rebuilds every chain that is not interned by handler, so - # entries from an earlier compile can only retain dead chains. - RegistrationContext.ensure_context()._reset_compile_caches() - for plugin in compiler_plugins: - for dependency in plugin.get_frontend_dependencies(): - _bundle_library(dependency) - base_total = (len(app._unevaluated_pages) * 2) + fixed_steps + len(config.plugins) + # Experimental incremental compile cache: in a fresh process, recompile only + # the pages whose source changed and reuse the rest from the on-disk + # manifest. Falls back to a full compile on any unsafe condition; pages the + # attempt already compiled before falling back are adopted, not redone. + evaluated: CompileContext | None = None + registrations: dict[str, str] | None = None + if cache_on: + from reflex.compiler import disk_cache, page_cache + + page_cache.enable_read_tracking() + # Taken before any page evaluates: evaluation mutates components. + registrations = disk_cache.registration_digests(app._unevaluated_pages.values()) + from reflex.utils import compile_daemon + + def attempt() -> CompileContext | bool: + return _compile_from_cache( + app, + compiler_plugins, + radix_themes_plugin, + prerender_routes=prerender_routes, + use_rich=use_rich, + registrations=registrations, + ) + + if compile_daemon.compiled_in_predicting_child(lambda: attempt() is True): + # The child's pages never existed here; the manifest it wrote has + # their counts for the compile telemetry. + if (manifest := disk_cache.load_manifest()) is not None: + app._cached_component_counts = disk_cache.manifest_component_counts( + manifest + ) + return True + rebuilt = attempt() + if isinstance(rebuilt, CompileContext): + evaluated = rebuilt + elif rebuilt: + return True + + progress = console.progress() if use_rich else console.PoorProgress() + fixed_steps = 8 + all_pages = list(app._unevaluated_pages.values()) + if evaluated is None: + pages_to_compile = all_pages + _reset_compile_registries(compiler_plugins) + else: + # The incremental attempt already did the resets above and bundled what + # its pages registered; only the pages it never touched remain. + pages_to_compile = [ + page for page in all_pages if page.route not in evaluated.compiled_pages + ] + base_total = (len(pages_to_compile) * 2) + fixed_steps + len(config.plugins) progress.start() task = progress.add_task("Compiling:", total=base_total) + compile_ctx = CompileContext( app=app, - pages=list(app._unevaluated_pages.values()), + pages=pages_to_compile, hooks=CompilerHooks( plugins=default_page_plugins(style=app.style, plugins=compiler_plugins) ), @@ -1326,19 +1636,11 @@ def compile_app( evaluate_progress=lambda: progress.advance(task), render_progress=lambda: progress.advance(task), ) + if evaluated is not None: + compile_ctx.absorb(evaluated, all_pages) - for route, page_ctx in compile_ctx.compiled_pages.items(): - app._check_routes_conflict(route) - if not isinstance(page_ctx.root_component, Component): - msg = ( - f"Compiled page {route!r} root must be a Component before it can " - "be registered on the app." - ) - raise TypeError(msg) - app._pages[route] = page_ctx.root_component - - app._evaluated_pages.update(compile_ctx.compiled_pages) - app._stateful_pages.update(compile_ctx.stateful_routes) + _register_compiled_pages(app, compile_ctx.compiled_pages) + app._stateful_pages.update(dict.fromkeys(compile_ctx.stateful_routes)) if not dry_run: app._write_stateful_pages_marker() app._add_optional_endpoints() @@ -1383,24 +1685,11 @@ def compile_app( raise ReflexRuntimeError(msg) progress.advance(task) - app_wrappers = _resolve_app_wrap_components(app, compile_ctx.app_wrap_components) - app_root = _memoize_stateful_app_wraps(app._app_root(app_wrappers), compile_ctx) + app_root, hydrate_fallback_export = _build_app_root( + app, compile_ctx.app_wrap_components, compile_ctx + ) all_imports = utils.merge_imports(all_imports, app_root._get_all_imports()) - hydrate_fallback = app._resolve_hydrate_fallback() - hydrate_fallback_export = None - if hydrate_fallback is not None: - hydrate_fallback._add_style_recursive(app.style) - # Compile the fallback through the memo pipeline so it lands in its own - # JS module; root.jsx then re-exports it as HydrateFallback. - hydrate_fallback_definition = create_component_memo( - hydrate_fallback, "hydrate_fallback" - ) - compile_ctx.auto_memo_components[ - hydrate_fallback_definition.export_name, None - ] = hydrate_fallback_definition - hydrate_fallback_export = hydrate_fallback_definition.export_name - memo_component_files, memo_components_imports = compile_memo_components( ( *MEMOS.values(), @@ -1435,33 +1724,9 @@ def compile_app( dest=Path.cwd() / prerequisites.get_web_dir() / constants.Dirs.PUBLIC, ) - save_tasks: list[ - tuple[ - Callable[..., list[tuple[str, str]] | tuple[str, str] | None], - tuple[Any, ...], - dict[str, Any], - ] - ] = [] - modify_files_tasks: list[tuple[str, str, Callable[[str], str]]] = [] - - def add_save_task( - task_fn: Callable[..., list[tuple[str, str]] | tuple[str, str] | None], - /, - *args: Any, - **kwargs: Any, - ) -> None: - save_tasks.append((task_fn, args, kwargs)) - - for plugin in config.plugins: - plugin.pre_compile( - add_save_task=add_save_task, - add_modify_task=lambda *args, plugin=plugin: modify_files_tasks.append(( - plugin.__class__.__module__ + plugin.__class__.__name__, - *args, - )), - radix_themes_plugin=radix_themes_plugin, - unevaluated_pages=list(app._unevaluated_pages.values()), - ) + save_tasks, modify_files_tasks = _plugin_output_tasks( + config.plugins, radix_themes_plugin, all_pages + ) if save_tasks: _set_progress_total(progress, task, base_total + len(save_tasks)) @@ -1491,11 +1756,15 @@ def add_save_task( compile_results.append(result) progress.advance(task) + contexts_snapshot = ( + utils.compile_contexts_snapshot(app._state) if app._state is not None else None + ) compile_results.extend([ compile_contexts( app._state, radix_themes_plugin.get_theme(), component_imports=all_imports, + contexts_snapshot=contexts_snapshot, ), utils._compile_bundled_libraries(), ]) @@ -1554,30 +1823,29 @@ def add_save_task( ) output_mapping[path] = code - for plugin in config.plugins: - for static_file_path, content in plugin.get_static_assets(): - path = utils.resolve_path_of_web_dir(static_file_path) - if path in output_mapping: - logger.warning( - f"Plugin {plugin.__class__.__name__} is overwriting existing files at {path}." - ) - output_mapping[path] = ( - content.decode("utf-8") if isinstance(content, bytes) else content - ) + plugin_sources = _apply_plugin_outputs( + output_mapping, config.plugins, modify_files_tasks + ) - for plugin_name, file_path, modify_fn in modify_files_tasks: - path = utils.resolve_path_of_web_dir(file_path) - file_content = output_mapping.get(path) - if file_content is None: - if path.exists(): - file_content = path.read_text() - else: - msg = f"Plugin {plugin_name} is trying to modify {path} but it does not exist." - raise FileNotFoundError(msg) - output_mapping[path] = modify_fn(file_content) + from reflex.compiler import disk_cache + + if not cache_on: + # The output written below is not recorded: a manifest an earlier cached + # compile left must not validate files this compile overwrites. + disk_cache._manifest_path().unlink(missing_ok=True) with log.timing(logger, "Write to Disk"), otel.span("reflex.compile.write"): for output_path, code in output_mapping.items(): utils.write_file(output_path, code) + if cache_on: + disk_cache.write_manifest( + compile_ctx, + all_pages, + all_imports, + plugin_sources=plugin_sources, + contexts_snapshot=contexts_snapshot, + registrations=registrations, + ) + return True diff --git a/reflex/compiler/disk_cache.py b/reflex/compiler/disk_cache.py new file mode 100644 index 00000000000..638cdab3c8b --- /dev/null +++ b/reflex/compiler/disk_cache.py @@ -0,0 +1,2299 @@ +"""Disk-persisted incremental compile cache for ``REFLEX_COMPILE_CACHE``. + +The manifest stores only bookkeeping: a file table (content hash plus stat key +per dependency file), each page's dependency paths, app-wrap digests, memo +uses, statefulness and its contribution to the contexts file, the app-wide +frontend imports, the app root's inputs, and the import-name parse cache. +Rendered files stay in ``.web``. When the global inputs, the page set and the +registrations still match, unchanged pages are reused from disk and only +dependency-changed pages are recompiled. + +Each page is a memoized query over the inputs it recorded. Outputs that +several pages feed are handled in one of two ways: the memo files are +content-addressed, one file per auto memo, so a page's memos never share a +file with another page's; and the app root and the bundled-library file are +compared against their recorded inputs and fall back to a full compile when a +recompiled page changed them. + +Validation is stat-first (see :class:`page_cache.FileValidator`): an unchanged +file is recognised from ``(mtime_ns, size)`` without being read, and when the +watcher reports which paths changed, pages depending on none of them are hits +without even a stat. +""" + +from __future__ import annotations + +import contextlib +import dataclasses +import functools +import hashlib +import json +import logging +import re +import sys +from pathlib import Path +from typing import TYPE_CHECKING, Any + +from reflex_base import constants +from reflex_base.environment import environment +from reflex_base.plugins import CompileContext, CompilerHooks, Plugin +from reflex_base.registry import RegistrationContext +from reflex_base.utils import log +from reflex_base.utils.format import json_dumps +from reflex_base.utils.imports import ImportVar, merge_imports + +from reflex.compiler import page_cache +from reflex.compiler.plugins import default_page_plugins +from reflex.utils import console, path_ops, prerequisites + +if TYPE_CHECKING: + from collections.abc import Callable, Iterable, Sequence + + from reflex_base.plugins import PageContext, PageDefinition + from reflex_base.utils.imports import ParsedImportDict + + from reflex.app import App + from reflex.compiler import utils as compiler_utils + +logger = logging.getLogger(__name__) + +#: Bump when the manifest layout changes (old manifests are then ignored). +_SCHEMA = 23 +#: Manifest filename under the web directory. +_MANIFEST_FILE = "reflex_compile_cache.json" + +#: Paths the watcher reported as changed for the current compile, resolved. +#: None when unknown (a cold start or a cache-less compile), in which case +#: every recorded dependency is validated. +_changed_hint: set[str] | None = None + + +def set_changed_hint(paths: Iterable[Path | str] | None) -> None: + """Tell the next incremental rebuild which paths the watcher saw change. + + Args: + paths: The changed paths, or None when unknown. + """ + global _changed_hint + _changed_hint = None if paths is None else {str(Path(p).resolve()) for p in paths} + + +#: Whether this compile predicts the ``reflex.istate.dynamic`` state names of +#: the pages it does not re-evaluate, instead of re-evaluating every page that +#: defines such states (see :func:`_with_dynamic_state_pages`). Only a caller +#: that can discard this process and compile again sets it; None means no +#: caller chose, and a cold compile tries it in a forked child (see +#: ``compile_daemon.compiled_in_predicting_child``). +_predict_dynamic_names: bool | None = None + + +def set_dynamic_name_prediction(enabled: bool) -> None: + """Predict dynamic state names in this process's compiles. + + A failed prediction raises :class:`DynamicStatePredictionError`; the + caller then compiles again in a fresh process without prediction. + + Args: + enabled: Whether to predict. + """ + global _predict_dynamic_names + _predict_dynamic_names = enabled + + +class DynamicStatePredictionError(Exception): + """A predicted rebuild cannot finish correctly in this process. + + Either a re-evaluated page claimed other ``reflex.istate.dynamic`` names + than recorded, which shifts the names of pages that were not + re-evaluated, or a fallback would evaluate further pages out of page + order. Compile again in a fresh process without prediction. + """ + + +def _manifest_path() -> Path: + """Return the manifest's path in the web directory. + + Returns: + The manifest path. + """ + return prerequisites.get_web_dir() / _MANIFEST_FILE + + +def format_path_list( + items: Iterable[str], root: Path | None = None, limit: int = 5 +) -> str: + """Render a bounded, root-relative summary of a path/label collection. + + Args: + items: The paths (or labels) to render. + root: When given, paths under it are shown relative to it. + limit: Maximum number of entries to show before truncating. + + Returns: + A comma-separated summary string, truncated with ``(+N more)``. + """ + + def rel(item: str) -> str: + if root is not None: + with contextlib.suppress(ValueError): + return str(Path(item).relative_to(root)) + return item + + shown = sorted(rel(item) for item in items) + extra = len(shown) - limit + return ", ".join(shown[:limit]) + (f" (+{extra} more)" if extra > 0 else "") + + +def _log_fallback(reason: str) -> None: + """Report why the incremental rebuild fell back to a full compile. + + Args: + reason: The human-readable fallback reason. + """ + logger.info(f"Compile cache: falling back to a full compile — {reason}") + + +_IMPORT_VAR_FIELDS = tuple(f.name for f in dataclasses.fields(ImportVar)) + + +def _serialize_imports(imports: ParsedImportDict) -> dict[str, list[dict[str, Any]]]: + """Serialize a parsed import dict to JSON-able primitives. + + Duplicates are collapsed in first-seen order (a full docs-app compile + accumulates ~107k entries, ~6k unique): the manifest's import set only + feeds package installation and later merges, where only the unique set + matters, and duplicates bloat the manifest and every pass over it. + + Args: + imports: The parsed import dict to serialize. + + Returns: + A JSON-serializable representation. + """ + return { + lib: [ + {name: getattr(iv, name) for name in _IMPORT_VAR_FIELDS} + for iv in dict.fromkeys(ivs) + ] + for lib, ivs in imports.items() + } + + +def _deserialize_imports(data: dict[str, list[dict[str, Any]]]) -> ParsedImportDict: + """Rebuild a parsed import dict from its serialized form. + + Args: + data: The serialized import dict. + + Returns: + The reconstructed parsed import dict. + """ + return {lib: [ImportVar(**iv) for iv in ivs] for lib, ivs in data.items()} + + +def _wrap_key_str(key: tuple[int, str]) -> str: + """Format an app-wrap key for the manifest. + + Args: + key: The ``(priority, name)`` app-wrap key. + + Returns: + ``":"``. + """ + return f"{key[0]}:{key[1]}" + + +def _wrap_digest(component: Any) -> str: + """Digest what an app wrap contributes to the compiled app root. + + Args: + component: The app-wrap component. + + Returns: + A digest of its render, imports, hooks, custom code, dynamic imports + and the app wraps its own subtree declares. + """ + payload = [ + component.render(), + _serialize_imports(component._get_all_imports()), + list(component._get_all_hooks()), + list(component._get_all_custom_code()), + sorted(component._get_all_dynamic_imports()), + sorted(map(_wrap_key_str, component._get_all_app_wrap_components())), + ] + return hashlib.sha256(json_dumps(payload).encode()).hexdigest() + + +def _record_app_wraps(page_ctx: PageContext) -> dict[str, str]: + """Record a page's app wraps by digest. + + Args: + page_ctx: The compiled page context. + + Returns: + The page's app-wrap keys mapped to their digests, in page order. + """ + return { + _wrap_key_str(key): _wrap_digest(component) + for key, component in page_ctx.app_wrap_components.items() + } + + +def _stable_value(value: Any) -> Any: + """Turn a page registration value into a process-independent JSON value. + + Args: + value: A value ``add_page`` was given. + + Returns: + A JSON-able value that is equal across processes for equal inputs. + """ + from reflex_base.components.component import BaseComponent + from reflex_base.vars import Var + + if value is None or isinstance(value, (str, int, float, bool)): + return value + if isinstance(value, Var): + return str(value) + if isinstance(value, BaseComponent): + return value.render() + if isinstance(value, dict): + return {str(key): _stable_value(item) for key, item in value.items()} + if isinstance(value, (list, tuple)): + return [_stable_value(item) for item in value] + if dataclasses.is_dataclass(value) and not isinstance(value, type): + return { + field.name: _stable_value(getattr(value, field.name)) + for field in dataclasses.fields(value) + } + if isinstance(value, functools.partial): + return [ + _stable_value(value.func), + _stable_value(value.args), + _stable_value(value.keywords), + ] + if callable(value): + # Which function, not its body: a page function's body is a recorded + # dependency of the page, and load events only run on the backend. + return [ + getattr(value, "__module__", None), + getattr(value, "__qualname__", None), + ] + return repr(value) + + +def _registration_digest(page: PageDefinition) -> str: + """Digest everything a page was registered with. + + Args: + page: The page definition. + + Returns: + A digest of its route, metadata, load events and component. + """ + return hashlib.sha256(json_dumps(_stable_value(page)).encode()).hexdigest() + + +def registration_digests(pages: Iterable[PageDefinition]) -> dict[str, str]: + """Digest each page's registration before any page is evaluated. + + Evaluation mutates component instances passed as pages (styles are + applied in place), so the digests are taken once, up front. + + Args: + pages: The page definitions. + + Returns: + Each route's registration digest. + """ + return {page.route: _registration_digest(page) for page in pages} + + +def _memo_keys(page_ctx: PageContext) -> list[str]: + """The export names of the auto memos a page's output imports. + + Args: + page_ctx: The compiled page context. + + Returns: + The sorted memo tags. + """ + return sorted(tag for tag, _ in page_ctx.memo_uses) + + +def _uses_radix(page_ctx: PageContext) -> bool: + """Whether a compiled page holds a Radix Themes component. + + The implicit ``RadixThemesPlugin`` enables itself when it compiles one; + a page reused from disk is not compiled, so the manifest remembers. The + page's auto memos are walked too: memoization replaces a stateful Radix + component in the page tree with its wrapper. + + Args: + page_ctx: The compiled page context. + + Returns: + True when the page enables Radix Themes. + """ + from reflex_components_radix.themes.base import RadixThemesComponent + + return any( + issubclass(cls, RadixThemesComponent) + for root in ( + page_ctx.root_component, + *(definition.component for definition in page_ctx.memo_uses.values()), + ) + for cls in page_cache.component_tree_classes(root) + ) + + +def _manifest_page_entry( + page_ctx: PageContext, + page: PageDefinition, + state_index: dict[str, Path], + hasher: Callable[[str], page_cache.FileEntry | None], + files: dict[str, page_cache.FileEntry], + *, + defined_states: Sequence[str] | None, + registration: str, + state_slice: dict[str, Any] | None = None, + root: Path | None = None, +) -> dict[str, Any]: + """Build the manifest entry for one compiled page. + + Args: + page_ctx: The compiled page context. + page: The page definition. + state_index: The state-context identifier -> file index. + hasher: A memoized path -> file-entry function. + files: The manifest file table, extended with the page's dependencies. + defined_states: Full names of the states the page defined while it + evaluated, or None for a stateless page. + registration: The page's registration digest (see + :func:`registration_digests`). + state_slice: The page's contribution to the contexts file + (see :func:`_state_slice`), or None for stateless pages. + root: Project root for dependency discovery. Defaults to cwd. + + Returns: + The JSON-able manifest entry for the page. + """ + from reflex.utils.telemetry_accounting import _count_components + + return { + "component_counts": _count_components([page_ctx.root_component]), + "deps": page_cache.page_dependency_entries( + page_ctx, page.component, state_index, hasher, files, root + ), + # Everything the page was registered with (see ``_pages_mismatch``). + "registration": registration, + # The app wraps the page requires, by digest (see ``_app_root_change``). + "app_wraps": _record_app_wraps(page_ctx), + "is_stateful": defined_states is not None, + "state_slice": state_slice, + # The ``reflex.istate.dynamic`` names the page's states took, whose + # suffixes depend on evaluation order (see ``_with_dynamic_state_pages``). + "dynamic_claims": _dynamic_claims(defined_states or ()), + # The auto memos the page imports, each in its own content-addressed + # file (see ``_memo_file``); the files a rebuild may drop are the ones + # no page uses any more. + "memo_keys": _memo_keys(page_ctx), + # Whether the page enables the implicit Radix Themes plugin. + "radix": _uses_radix(page_ctx), + } + + +def _state_slice( + state_names: Sequence[str], + initial_state: dict[str, Any], + client_storage: dict[str, dict[str, Any]], + schema_entries: dict[str, list[Any]], +) -> dict[str, Any]: + """Extract some states' contribution to the compiled contexts file. + + A state's contribution is exactly its initial-state entry, its + client-storage entries and its frontend schema entry (see + ``templates.context_template``). Persisting the slice, not just a + fingerprint, lets an incremental rebuild re-emit the contexts file for + pages it did not evaluate (states only registered while their page + evaluates, e.g. exec'd docs demos) without evaluating them. + + Args: + state_names: Full names of the states to slice. + initial_state: The complete initial-state mapping (full name -> vars). + client_storage: The compiled client-storage mapping per storage kind. + schema_entries: The frontend schema entries (full name -> entry). + + Returns: + ``{"initial_state": {...}, "client_storage": {kind: {...}}, + "schema": {...}}``. + """ + from reflex.compiler import utils as compiler_utils + + names = sorted(state_names) + prefixes = tuple(f"{name}." for name in names) + return { + # Serialized as the contexts file holds them: a component value's code + # depends on whether its library is bundled, which serializing does. + "initial_state": { + name: json.loads( + compiler_utils._serialize_initial_state({name: initial_state[name]}) + )[name] + for name in names + if name in initial_state + }, + "client_storage": { + kind: {k: v for k, v in entries.items() if k.startswith(prefixes)} + for kind, entries in client_storage.items() + }, + "schema": { + name: schema_entries[name] for name in names if name in schema_entries + }, + } + + +def _slice_digest(state_slice: dict[str, Any] | None) -> str: + """Hash a state slice for cheap equality. + + Args: + state_slice: A :func:`_state_slice` result (or None). + + Returns: + A stable digest. + """ + return hashlib.sha256(json_dumps(state_slice).encode()).hexdigest() + + +def _contexts_snapshot(app: App | None) -> compiler_utils.ContextsSnapshot | None: + """Capture the state-tree inputs of the contexts file. + + Args: + app: The app being compiled (absent in bare compile contexts). + + Returns: + The contexts snapshot of the app's state tree, or None when there is + no state tree. + """ + if app is None or app._state is None: + return None + from reflex.compiler import utils as compiler_utils + + return compiler_utils.compile_contexts_snapshot(app._state) + + +def _changed_state_config_route( + manifest: dict[str, Any], + miss_ctx: CompileContext, + snapshot: compiler_utils.ContextsSnapshot | None, + miss_routes: Iterable[str], +) -> str | None: + """Find the first recompiled route whose contribution to contexts changed. + + Compares each recompiled page's just-evaluated states against the slice + recorded in the manifest; a page without states contributes no slice, so + gaining or losing states counts as a change. Only these pages' states can + differ from the on-disk contexts file: hit pages' definitions are + unchanged by construction (all their dependency files still match). + + Args: + manifest: The loaded compile manifest. + miss_ctx: The compile context of the recompiled pages. + snapshot: The ``_contexts_snapshot`` of the app, or None when there is + no state tree. + miss_routes: The recompiled routes. + + Returns: + The first route requiring a contexts rebuild, or None if none do. + """ + for route in miss_routes: + defined_states = miss_ctx.stateful_routes.get(route) + live = ( + _state_slice(defined_states, *snapshot) + if defined_states and snapshot is not None + else None + ) + if _slice_digest(manifest["pages"][route]["state_slice"]) != _slice_digest( + live + ): + return route + return None + + +def _with_hit_state_slices( + snapshot: compiler_utils.ContextsSnapshot, + manifest: dict[str, Any], + miss_routes: set[str], +) -> compiler_utils.ContextsSnapshot: + """Merge the stored contexts contributions of the stateful hit pages. + + Args: + snapshot: The contexts snapshot of this process's state tree. + manifest: The loaded manifest. + miss_routes: Routes recompiled in this rebuild (their states are live). + + Returns: + The snapshot of the live states plus the hit pages' stored states. + """ + live_initial, live_storage, live_schema = snapshot + initial = dict(live_initial) + storage = {kind: dict(entries) for kind, entries in live_storage.items()} + schema_entries = dict(live_schema) + for route, entry in manifest["pages"].items(): + if route in miss_routes or not (state_slice := entry["state_slice"]): + continue + initial.update(state_slice["initial_state"]) + for kind, entries in state_slice["client_storage"].items(): + storage.setdefault(kind, {}).update(entries) + schema_entries.update(state_slice["schema"]) + return dict(sorted(initial.items())), storage, schema_entries + + +def _hit_page_states(manifest: dict[str, Any], miss_routes: set[str]) -> set[str]: + """The states the manifest records as defined by pages that hit. + + Args: + manifest: The loaded manifest. + miss_routes: The recompiled routes. + + Returns: + The full names of the hit pages' states. + """ + return { + name + for route, entry in manifest["pages"].items() + if route not in miss_routes and (state_slice := entry["state_slice"]) + for name in state_slice["schema"] + } + + +def _disown_hit_page_states( + miss_ctx: CompileContext, manifest: dict[str, Any], miss_routes: set[str] +) -> None: + """Leave a hit page's states to it when a recompiled page registers them. + + A module state is recorded by the page whose evaluation first imported + its module. In a process that evaluates only the recompiled pages, one + of them can import that module first and would otherwise claim the + state, marking itself stateful where a full compile would not. + + Args: + miss_ctx: The compile context of the recompiled pages (mutated). + manifest: The loaded manifest. + miss_routes: The recompiled routes. + """ + owned = _hit_page_states(manifest, miss_routes) + for route, names in list(miss_ctx.stateful_routes.items()): + if kept := [name for name in names if name not in owned]: + miss_ctx.stateful_routes[route] = kept + else: + del miss_ctx.stateful_routes[route] + + +def load_manifest() -> dict[str, Any] | None: + """Load the persisted compile manifest, or None if absent/unusable. + + Also seeds the import-name parse cache from the manifest, so unchanged + first-party modules are not re-parsed in this process. + + Returns: + The parsed manifest dict, or None. + """ + path = _manifest_path() + if not path.exists(): + return None + try: + data = json.loads(path.read_text(encoding="utf-8")) + except (OSError, ValueError): + return None + if not isinstance(data, dict) or data.get("schema") != _SCHEMA: + return None + page_cache.load_import_names_cache(data.get("import_names", {})) + return data + + +def manifest_dependency_entries(manifest: dict[str, Any]) -> set[str]: + """Every file-table key the manifest's current entries reference. + + An incremental rebuild extends the file table without pruning it, so a + dependency a page dropped (a renamed function's slice, say) keeps its + entry; comparing a file's recorded content must go through these keys. + + Args: + manifest: The loaded manifest. + + Returns: + Page dependencies, file-backed global inputs, import-time state files + and memo source closures (paths or function slices). + """ + entries: set[str] = set() + for entry in manifest.get("pages", {}).values(): + entries.update(entry.get("deps", ())) + entries.update( + path for label, path in manifest.get("globals", {}).items() if label != "reflex" + ) + entries.update(manifest.get("module_states", {}).get("files", ())) + for memo in manifest.get("memo_files", {}).values(): + entries.update(memo.get("deps", ())) + return entries + + +def manifest_dependency_files(manifest: dict[str, Any]) -> set[str]: + """Every file the manifest records as a compile input. + + Args: + manifest: The loaded manifest. + + Returns: + The files of :func:`manifest_dependency_entries` plus the global + inputs recorded as absent. + """ + files = { + page_cache.dep_file(dep) + for dep in manifest_dependency_entries(manifest) + if dep != page_cache.VOLATILE_DEP + } + files.update(manifest.get("globals_absent", ())) + return files + + +def _manifest_text(manifest: dict[str, Any]) -> str: + """Serialize the manifest, including the current import-name parse cache. + + Args: + manifest: The manifest to serialize. + + Returns: + The manifest JSON. + """ + manifest["import_names"] = page_cache.export_import_names_cache() + return json_dumps(manifest) + + +def _write(manifest: dict[str, Any]) -> None: + """Persist the manifest. + + Args: + manifest: The manifest to write. + """ + path = _manifest_path() + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(_manifest_text(manifest), encoding="utf-8") + + +def write_manifest( + compile_ctx: CompileContext, + pages: Sequence[PageDefinition], + install_imports: ParsedImportDict, + root: Path | None = None, + *, + plugin_sources: dict[str, str] | None = None, + contexts_snapshot: compiler_utils.ContextsSnapshot | None = None, + registrations: dict[str, str] | None = None, +) -> None: + """Persist a manifest of the just-completed full compile. + + Best-effort: any failure leaves no manifest (the next compile is full), it + never breaks the build. + + Args: + compile_ctx: The completed compile context (all pages compiled, the + app root built). + pages: The full list of page definitions that were compiled. + install_imports: The **complete** frontend import set the full compile + installed: page imports merged with the app-root (app-wrap, e.g. + the Toaster/``sonner`` provider) and memo-component imports. An + incremental rebuild reuses the on-disk app-wide files, so it must + install from this complete set, not just the per-page union. + root: Project root for fingerprinting. Defaults to cwd. + plugin_sources: Unmodified source content for plugin file modifiers. + contexts_snapshot: The state-tree inputs used for the contexts output. + When absent, a snapshot is captured for direct callers. + registrations: Each route's registration digest, taken before any page + evaluated; computed now when absent. + """ + try: + state_index, _ = page_cache.state_dependency_index(root) + hasher = page_cache.make_hasher() + files: dict[str, page_cache.FileEntry] = {} + if registrations is None: + registrations = registration_digests(pages) + if contexts_snapshot is None and compile_ctx.stateful_routes: + contexts_snapshot = _contexts_snapshot(compile_ctx.app) + + pages_data: dict[str, Any] = {} + for page in pages: + page_ctx = compile_ctx.compiled_pages.get(page.route) + if ( + page_ctx is None + or page_ctx.output_code is None + or page_ctx.output_path is None + ): + logger.warning( + f"Compile cache: cannot save manifest because page {page.route!r} " + "has incomplete output. The next reload will need a full compile." + ) + return # incomplete compile -> do not write a partial manifest + defined_states = compile_ctx.stateful_routes.get(page.route) + pages_data[page.route] = _manifest_page_entry( + page_ctx, + page, + state_index, + hasher, + files, + defined_states=defined_states, + registration=registrations[page.route], + state_slice=( + _state_slice(defined_states, *contexts_snapshot) + if defined_states and contexts_snapshot is not None + else None + ), + root=root, + ) + + # Post-evaluation and complete: the full compile evaluated every page, + # so both import-owned and route-owned memos are recorded. + memo_files = _memo_state_entries(hasher, files, root, lambda _owner: True) + + globals_: dict[str, str] = {"reflex": page_cache._reflex_version()} + absent: list[str] = [] + global_paths = page_cache.global_input_paths(root, pages=pages) + page_states = { + name for names in compile_ctx.stateful_routes.values() for name in names + } + module_states = _record_module_states( + ( + name + for name in RegistrationContext.ensure_context().base_states + if name not in page_states + ), + hasher, + files, + root, + ) + for label, path in global_paths.items(): + entry = hasher(path) + if entry is None: + absent.append(path) + else: + files[path] = entry + globals_[label] = path + + entry_file = page_cache._app_entrypoint_file(root) + manifest = { + "schema": _SCHEMA, + # The app entry module apart from its page registrations (see + # ``globals_mismatch``). + "entry": { + "path": None if entry_file is None else str(entry_file), + "digest": ( + None + if entry_file is None + else page_cache.entrypoint_digest(str(entry_file)) + ), + }, + # File paths below are absolute: the manifest describes this + # project directory only. + "root": str((root or Path.cwd()).resolve()), + "plugin_sources": plugin_sources or {}, + "reflex_version": page_cache._reflex_version(), + "mode": _compile_mode(), + "files": files, + # Per-input labels (not one combined digest) so a later mismatch + # can name the exact global input that changed. + "globals": globals_, + "globals_absent": sorted(absent), + "all_imports": _serialize_imports(install_imports), + "pages": pages_data, + "memo_files": memo_files, + # The states importing the app registers (see + # ``_record_module_states``). + "module_states": module_states, + # The app root's inputs: the merged app wraps (by digest, in + # insertion order) and the bundled libraries (see + # ``_app_root_change``), plus the memo files the root uses. + "app_root": { + "wraps": { + _wrap_key_str(key): _wrap_digest(component) + for key, component in compile_ctx.app_wrap_components.items() + }, + "libraries": sorted( + RegistrationContext.ensure_context().bundled_libraries + ), + "memo_keys": sorted(compile_ctx.app_root_memo_keys), + }, + } + _write(manifest) + except Exception as exc: # best-effort: never break the build + logger.warning( + f"Compile cache: could not save manifest ({exc!r}). " + "The next reload may need a full compile." + ) + + +def _compile_mode() -> str: + """The compile mode the rendered output is specific to (``dev``/``prod``). + + Returns: + The current ``REFLEX_ENV_MODE`` value. + """ + return environment.REFLEX_ENV_MODE.get().value + + +def _module_state_files(state_names: Iterable[str], root: Path | None) -> set[str]: + """Find the source files that define the given states, with their imports. + + Args: + state_names: Full names of registered states. + root: Project root for dependency discovery. + + Returns: + First-party source files the states' definitions depend on. + """ + roots = page_cache.first_party_roots(root) + base_states = RegistrationContext.ensure_context().base_states + sources = { + str(path) + for name in state_names + if (path := page_cache._class_module_file(base_states[name], roots)) is not None + } + return page_cache._walk_import_closure(page_cache.build_import_graph(root), sources) + + +def _record_module_states( + state_names: Iterable[str], + hasher: Callable[[str], page_cache.FileEntry | None], + files: dict[str, page_cache.FileEntry], + root: Path | None, +) -> dict[str, list[str]]: + """Record the states importing the app registers, and their source files. + + They belong to no page, so the contexts file depends on them directly. + + Args: + state_names: Full names of the import-time states. + hasher: A memoized path -> file-entry function. + files: The manifest file table, extended with the source files. + root: Project root for dependency discovery. + + Returns: + ``{"names": [...], "files": [...], "dynamic": [...]}``. + """ + import reflex.istate.dynamic as istate_dynamic + + names = sorted(state_names) + recorded = [] + for path in sorted(_module_state_files(names, root)): + if (entry := hasher(path)) is not None: + files[path] = entry + recorded.append(path) + base_states = RegistrationContext.ensure_context().base_states + return { + "names": names, + "files": recorded, + # Import-time dynamic-module states, which take names before any page. + "dynamic": [ + name + for name in names + if base_states[name].__module__ == istate_dynamic.__name__ + ], + } + + +def globals_mismatch( + manifest: dict[str, Any], + *, + validator: page_cache.FileValidator, + root: Path | None = None, +) -> str | None: + """Explain why the manifest's global inputs don't match, or None if they do. + + The fast rebuild needs the global inputs unchanged: Reflex version, + config/lockfiles and the app-level config files (the entrypoint and the + theme/app-wrap/stylesheet modules it imports, which configure the app-wide + files). An entrypoint that only changed its page registrations still + matches; the routes and registrations are then compared page by page. + Everything else is decided per page via its dependency set, so a + shared-component or markdown edit no longer blocks the fast path. + + Global inputs are validated against the manifest's *stored* input set, + never by recomputing the set in this process — set membership is only + decided when a full compile writes the manifest. + + Args: + manifest: The loaded manifest. + validator: The file validator bound to the manifest's file table. + root: Project root the manifest must describe; also shortens paths in + the reason. Defaults to cwd. + + Returns: + A human-readable mismatch reason, or None when the global inputs match. + """ + current_root = str((root or Path.cwd()).resolve()) + if manifest["root"] != current_root: + return f"project root changed ({manifest['root']} -> {current_root})" + old_version = manifest.get("reflex_version") + if old_version != page_cache._reflex_version(): + return ( + f"reflex version changed ({old_version} -> {page_cache._reflex_version()})" + ) + old_mode = manifest.get("mode") + if old_mode != _compile_mode(): + return f"compile mode changed ({old_mode} -> {_compile_mode()})" + root = (root or Path.cwd()).resolve() + entry = manifest["entry"] + stale = { + label.removeprefix("app:") + for label, path in manifest.get("globals", {}).items() + if label != "reflex" + and validator.changed(path) + # An entry module that only changed its page registrations is compared + # page by page (see ``_pages_mismatch``). + and not ( + path == entry["path"] + and entry["digest"] is not None + and page_cache.entrypoint_digest(path) == entry["digest"] + ) + } + stale.update( + path for path in manifest.get("globals_absent", ()) if Path(path).is_file() + ) + if stale: + return f"global input(s) changed: {format_path_list(stale, root)}" + return None + + +def _pages_mismatch( + pages: Sequence[PageDefinition], + manifest: dict[str, Any], + registrations: dict[str, str], +) -> str | None: + """Explain why the manifest's page set no longer matches, or None if it does. + + The page registry is an input of every page (a navigation built from the + registered pages is common), so a page added, removed, reordered or + registered differently (its title, say) is a change to every page. + + Args: + pages: The current page definitions, in compile order. + manifest: The loaded manifest. + registrations: Each route's registration digest. + + Returns: + A human-readable mismatch reason, or None when the pages match. + """ + recorded = manifest["pages"] + routes = [page.route for page in pages] + if routes != list(recorded): + changes = [ + f"{label} {format_path_list(items)}" + for label, items in ( + ("added", set(routes) - set(recorded)), + ("removed", set(recorded) - set(routes)), + ) + if items + ] + if not changes: + return "the page order changed" + return f"the page set changed ({'; '.join(changes)})" + for route in routes: + if recorded[route]["registration"] != registrations[route]: + return f"page {route!r} was registered differently" + return None + + +def _with_hinted_slices( + changed_hint: set[str] | None, manifest: dict[str, Any] +) -> set[str] | None: + """Add the recorded function slices of the files the watcher reported. + + Args: + changed_hint: Resolved paths the watcher saw change, or None if unknown. + manifest: The loaded manifest. + + Returns: + The hint with every page dependency that reads a reported file. + """ + if not changed_hint: + return changed_hint + return changed_hint | { + dep + for entry in manifest["pages"].values() + for dep in entry["deps"] + if page_cache.SLICE_SEPARATOR in dep + and page_cache.dep_file(dep) in changed_hint + } + + +def partition_pages( + pages: Sequence[PageDefinition], + manifest: dict[str, Any], + validator: page_cache.FileValidator, + changed_hint: set[str] | None, +) -> list[PageDefinition]: + """Return the pages whose recorded dependencies changed. + + Globals and the page set are assumed already matched (see + :func:`globals_mismatch` and :func:`_pages_mismatch`), so a page is a + hit if every file in its recorded dependency set is unchanged. When the + watcher reported the changed paths, a page depending on none of them + skips the filesystem. + + Args: + pages: The current page definitions. + manifest: The loaded manifest. + validator: The file validator bound to the manifest's file table. + changed_hint: Resolved paths the watcher saw change, or None if unknown. + + Returns: + The list of miss pages, in ``pages`` order. + """ + manifest_pages = manifest["pages"] + misses: list[PageDefinition] = [] + for page in pages: + deps = manifest_pages[page.route]["deps"] + if ( + changed_hint is not None + and changed_hint.isdisjoint(deps) + and page_cache.VOLATILE_DEP not in deps + ): + continue + if not validator.unchanged(deps): + misses.append(page) + return misses + + +def _dynamic_claims(defined_states: Iterable[str]) -> list[list[str]]: + """The ``reflex.istate.dynamic`` names a page's states took, in order. + + Each name is paired with what was requested: a local state's class name, + or the ``ComponentState`` a per-instance state was created from. A page + making the same claims as recorded took the names a full compile gives it. + + Args: + defined_states: Full names of the states the page defined. + + Returns: + ``[name, requested]`` pairs in definition order. + """ + import reflex.istate.dynamic as istate_dynamic + + base_states = RegistrationContext.ensure_context().base_states + claims = [] + for full_name in defined_states: + cls = base_states[full_name] + if cls.__module__ != istate_dynamic.__name__: + continue + requested = cls.__dict__.get("__original_name__") + if requested is None: + origin = cls.__bases__[0] + requested = f"{origin.__module__}.{origin.__qualname__}" + claims.append([cls.__name__, requested]) + return claims + + +_NAME_SUFFIX = re.compile(r"_\d+$") + + +def _claims_by_family(claims: Iterable[Sequence[str]]) -> dict[str, list[list[str]]]: + """Group dynamic state name claims by the name family they draw from. + + A local state takes the first free name among its class name and its + ``_`` suffixes, so it competes with every claim whose name or request + has the same stem. A ``ComponentState`` instance draws from its class's + counter; its request is the dotted class path, which no class name has. + + Args: + claims: ``[name, requested]`` pairs (see :func:`_dynamic_claims`). + + Returns: + The claims of each family, in order. + """ + families: dict[str, list[list[str]]] = {} + for name, requested in claims: + for family in {_NAME_SUFFIX.sub("", name), _NAME_SUFFIX.sub("", requested)}: + families.setdefault(family, []).append([name, requested]) + return families + + +def _shifting_dynamic_claims( + manifest: dict[str, Any], miss_ctx: CompileContext, miss_routes: set[str] +) -> str | None: + """Find a re-evaluated page whose claims could shift other pages' names. + + Names in a family are handed out in evaluation order, so a re-evaluated + page changing its claims in a family moves the names of later pages in + that family. Pages that were not re-evaluated keep their recorded names, + which is exact only when no family they use changed. + + Args: + manifest: The loaded manifest. + miss_ctx: The compile context of the re-evaluated pages. + miss_routes: The re-evaluated routes. + + Returns: + The first re-evaluated route that changed a shared family, or None. + """ + shared = { + family + for route, entry in manifest["pages"].items() + if route not in miss_routes + for family in _claims_by_family(entry["dynamic_claims"]) + } + for route in miss_routes: + recorded = _claims_by_family(manifest["pages"][route]["dynamic_claims"]) + live = _claims_by_family( + _dynamic_claims(miss_ctx.stateful_routes.get(route, ())) + ) + if any( + live.get(family) != recorded.get(family) + for family in (live.keys() | recorded.keys()) & shared + ): + return route + return None + + +_INSTANCE_NUMBER = re.compile(r"_n(\d+)$") + + +def _numbers_before_miss_pages( + pages: Sequence[PageDefinition], manifest: dict[str, Any], miss_routes: set[str] +) -> dict[str, dict[str, int]]: + """The highest ``ComponentState`` numbers kept pages drew before each miss. + + Args: + pages: All current page definitions (in compile order). + manifest: The loaded manifest. + miss_routes: Routes re-evaluated in this rebuild. + + Returns: + For each re-evaluated route, the dotted ``ComponentState`` class path + mapped to the highest instance number drawn by an earlier kept page. + """ + numbers: dict[str, int] = {} + before: dict[str, dict[str, int]] = {} + for page in pages: + if page.route in miss_routes: + if numbers: + before[page.route] = dict(numbers) + continue + # A ComponentState claim requests its dotted class path; a local + # state's request is a bare class name. + for name, requested in manifest["pages"][page.route]["dynamic_claims"]: + if "." in requested and (match := _INSTANCE_NUMBER.search(name)): + numbers[requested] = max(numbers.get(requested, 0), int(match[1])) + return before + + +def _resolve_class_path(path: str) -> Any: + """Look up a loaded class by its dotted ``module.qualname`` path. + + Args: + path: The dotted path. + + Returns: + The object at the path, or None. + """ + parts = path.split(".") + for split in range(len(parts) - 1, 0, -1): + if (obj := sys.modules.get(".".join(parts[:split]))) is not None: + for part in parts[split:]: + obj = getattr(obj, part, None) + return obj + return None + + +@dataclasses.dataclass(frozen=True, slots=True) +class _ComponentStateNumbersPlugin(Plugin): + """Advance ``ComponentState`` counters before each re-evaluated page. + + Instance numbers are handed out in page order and only grow. Before a + re-evaluated page, a full compile's counter is past every number drawn + by the earlier pages: the kept ones (recorded) and the re-evaluated ones + (drawn live in this rebuild). + """ + + numbers_before: dict[str, dict[str, int]] + + def eval_page( + self, page_fn: Any, /, *, page: PageDefinition, **kwargs: Any + ) -> None: + """Advance the counters to the numbers kept pages drew before this page. + + Args: + page_fn: The page function (unused). + page: The page about to be evaluated. + kwargs: Other hook arguments (unused). + """ + from reflex.state import ComponentState + + for path, number in self.numbers_before.get(page.route, {}).items(): + cls = _resolve_class_path(path) + if ( + isinstance(cls, type) + and issubclass(cls, ComponentState) + and cls._per_component_state_instance_count < number + ): + cls._per_component_state_instance_count = number + + +def _route_dropping_a_shared_state( + manifest: dict[str, Any], miss_routes: Iterable[str] +) -> str | None: + """Find a recompiled page whose recorded module state is gone. + + A page records the states its evaluation registered first. Another page + using the same module state recorded none, so when the first page stops + registering it, only a full compile keeps it. A local state belongs to + its page alone. + + Args: + manifest: The loaded manifest. + miss_routes: The recompiled routes. + + Returns: + The first such route, or None. + """ + import reflex.istate.dynamic as istate_dynamic + + local_prefix = istate_dynamic.__name__.replace(".", "___") + "___" + live = RegistrationContext.ensure_context().base_states + for route in miss_routes: + entry = manifest["pages"][route] + if not entry["state_slice"]: + continue + for name in entry["state_slice"]["schema"]: + if name not in live and not name.rpartition(".")[2].startswith( + local_prefix + ): + return route + return None + + +def _claim_recorded_dynamic_names( + manifest: dict[str, Any], miss_routes: set[str] +) -> None: + """Reserve the dynamic state names of the pages that are not re-evaluated. + + A local state takes the first free name for its class name, so with these + reserved a re-evaluated page takes the suffix it has in a full compile + whenever its own claims are unchanged. + + Args: + manifest: The loaded manifest. + miss_routes: Routes re-evaluated in this rebuild. + """ + import reflex.istate.dynamic as istate_dynamic + + for route, entry in manifest["pages"].items(): + if route in miss_routes: + continue + for name, _ in entry["dynamic_claims"]: + if not hasattr(istate_dynamic, name): + setattr(istate_dynamic, name, None) + + +def _fall_back_after_evaluation(reason: str) -> None: + """Fall back after pages were evaluated, or fail a prediction. + + Args: + reason: Why the rebuild falls back. + + Raises: + DynamicStatePredictionError: While predicting dynamic state names, + where the fallback would evaluate pages out of page order. + """ + if _predict_dynamic_names: + raise DynamicStatePredictionError(reason) + _log_fallback(reason) + + +def _with_dynamic_state_pages( + miss_pages: list[PageDefinition], + pages: Sequence[PageDefinition], + manifest: dict[str, Any], +) -> list[PageDefinition]: + """Expand a miss set with every page that defines dynamic-module states. + + States defined in ``reflex.istate.dynamic`` (function-local classes, + ``ComponentState`` instances) take collision-suffixed names in evaluation + order. Evaluating all of them in page order reproduces the names of a full + compile and of the backend, which evaluates every stateful page. This is + the exact rule; a compile that can retry in a fresh process predicts the + names instead (see :func:`set_dynamic_name_prediction`). + + Args: + miss_pages: The pages to recompile. + pages: All current page definitions (in compile order). + manifest: The loaded manifest. + + Returns: + The expanded miss list, in ``pages`` order. + """ + if not miss_pages: + return miss_pages + miss_routes = {page.route for page in miss_pages} + return [ + page + for page in pages + if page.route in miss_routes or manifest["pages"][page.route]["dynamic_claims"] + ] + + +def _expand_miss_pages( + miss_pages: list[PageDefinition], + pages: Sequence[PageDefinition], + manifest: dict[str, Any], + stale_owner_routes: set[str], +) -> list[PageDefinition]: + """Add the pages a rebuild of ``miss_pages`` must evaluate too. + + A route-owned memo entry whose dependency files changed needs its owner + page re-evaluated: only that page's evaluation re-registers the memo + definitions the rewrite compiles from. + + Args: + miss_pages: The dependency-changed pages. + pages: All current page definitions (in compile order). + manifest: The loaded manifest. + stale_owner_routes: Owners of route-owned memo records that changed. + + Returns: + The expanded miss list, in ``pages`` order. + """ + routes = {page.route for page in miss_pages} | stale_owner_routes + miss_pages = [page for page in pages if page.route in routes] + if _predict_dynamic_names: + return miss_pages + return _with_dynamic_state_pages(miss_pages, pages, manifest) + + +def _changed_dependency_files( + manifest: dict[str, Any], + validator: page_cache.FileValidator, + changed_hint: set[str] | None = None, +) -> set[str]: + """Return every recorded dependency file whose content changed. + + Args: + manifest: The loaded manifest. + validator: The file validator bound to the manifest's file table. + changed_hint: Resolved paths the watcher saw change, or None if unknown. + + Returns: + The set of changed dependency file paths (a changed function slice + as its file). + """ + candidates: Iterable[str] = { + path for entry in manifest["pages"].values() for path in entry["deps"] + } + if changed_hint is not None: + candidates = changed_hint.intersection(candidates) + return {page_cache.dep_file(path) for path in candidates if validator.changed(path)} + + +def _module_source_file(module_name: str | None) -> str | None: + """Resolve a loaded module's source file path. + + Args: + module_name: The dotted module name. + + Returns: + The resolved file path string, or None. + """ + file = getattr(sys.modules.get(module_name or ""), "__file__", None) + if not file: + return None + try: + return str(Path(file).resolve()) + except OSError: + return None + + +def _memo_output_path(source_module: str | None) -> str | None: + """Resolve the mirrored output file path for a user memo's source module. + + Args: + source_module: The dotted module name that defined the memo. + + Returns: + The absolute output path string, or None when the module can't be + mirrored (unmirrored memos compile to one file per memo instead). + """ + from reflex_base.utils import memo_paths + + from reflex.compiler import utils as compiler_utils + + segments = memo_paths.module_to_mirrored_segments(source_module) + if segments is None: + return None + return compiler_utils.get_memo_module_path(segments) + + +def _memo_file(tag: str) -> str: + """The content-addressed file of an auto memo. + + Args: + tag: The memo's export name, derived from its rendered body. + + Returns: + The absolute output path string. + """ + from reflex.compiler import utils as compiler_utils + + return compiler_utils.get_memo_component_path(tag) + + +def _memo_state_entries( + hasher: Callable[[str], page_cache.FileEntry | None], + files: dict[str, page_cache.FileEntry], + root: Path | None, + include_owner: Callable[[str | None], bool], +) -> dict[str, dict[str, Any]]: + """Describe the user-memo output files the current registry demands. + + For each mirrored memo output file: the definitions landing in it, the + source module's first-party import closure (as file-table paths), and the + route owning the definitions (see ``registration_owner``) — None for memos + registered at module import. Import-owned entries are a pure function of + app import, so any process can compare them pre-evaluation; route-owned + entries only exist after their owner page evaluates, so they are compared + post-evaluation for recompiled routes and validated by their owner's + hit/miss status otherwise. Memos without a resolvable module file (exec'd + docs demos in synthetic modules) are excluded — they have no file to + track and are covered by the ``changed_files`` net in + :func:`_memo_defs_for_rewrite`. + + Args: + hasher: A memoized path -> :func:`page_cache.file_entry` function. + files: File table extended with the closures' entries. Kept separate + from the manifest's own table until the rebuild has finished + validating against it (see :func:`_update_manifest_for_misses`). + root: Project root for the import-graph walk. None defaults to cwd. + include_owner: Predicate over a definition's ``owner_route`` deciding + which registry slice to describe. + + Returns: + A JSON-able mapping of output path -> + ``def_keys``/``deps``/``owner_route``. + """ + from reflex_base.components.memo import MEMOS + + state: dict[str, dict[str, Any]] = {} + for (name, source_module), definition in MEMOS.items(): + if not include_owner(definition.owner_route): + continue + path = _memo_output_path(source_module) + if path is None: + continue + entry = state.get(path) + if entry is None: + module_file = _module_source_file(source_module) + if module_file is None: + continue + deps: list[str] = [] + for dep in sorted(page_cache.module_py_dependencies(module_file, root)): + dep_entry = hasher(dep) + if dep_entry is not None: + files[dep] = dep_entry + deps.append(dep) + entry = state[path] = { + "def_keys": [], + "deps": deps, + "owner_route": definition.owner_route, + } + entry["def_keys"].append([name, source_module]) + for entry in state.values(): + entry["def_keys"].sort() + return state + + +def _memo_deps_changed( + deps: Sequence[str], + validator: page_cache.FileValidator, + changed_hint: set[str] | None, +) -> bool: + """Whether any file in a memo record's import closure changed. + + Args: + deps: The record's dependency paths. + validator: The file validator bound to the manifest's file table. + changed_hint: Resolved paths the watcher saw change, or None if unknown. + + Returns: + True if a dependency's content changed since it was recorded. + """ + if changed_hint is not None and changed_hint.isdisjoint(deps): + return False + return not validator.unchanged(deps) + + +def _dirty_memo_paths( + state: dict[str, dict[str, Any]], + stored: dict[str, dict[str, Any]], + validator: page_cache.FileValidator, + changed_hint: set[str] | None, +) -> set[str]: + """Return the demanded memo output files whose stored record is stale. + + A record is stale when the registry's demand differs from what was stored + (a new or removed memo module, or a moved definition) or when a file in + its import closure changed (an edited memo body or helper). + + Args: + state: The memo records the current registry demands. + stored: The manifest's memo records. + validator: The file validator bound to the manifest's file table. + changed_hint: Resolved paths the watcher saw change, or None if unknown. + + Returns: + The output paths to rewrite. + """ + return { + path + for path, entry in state.items() + if stored.get(path) != entry + or _memo_deps_changed(entry["deps"], validator, changed_hint) + } + + +def _memo_defs_for_rewrite( + memo_uses: dict[tuple[str, str | None], Any], + dirty_paths: set[str], + changed_files: set[str], +) -> list[Any]: + """Return the definitions of the memo files a rebuild writes. + + User memo output is grouped one file per source module, so a rewrite + must carry every definition landing in that file: every user ``@rx.memo`` + whose output file is being rewritten because its recorded memo state + changed (``dirty_paths``, see :func:`_dirty_memo_paths`), or whose module + file (a recorded page dependency) changed, which the pre-evaluation state + can't see for modules imported inside a page function. Auto memos are one + file each, so the recompiled pages' uses are written as they are. + + Args: + memo_uses: The auto memos the recompiled pages use. + dirty_paths: Output paths whose stored memo state no longer matches. + changed_files: The recorded dependency files whose content changed. + + Returns: + The memo definitions to compile, user memos first (matching the full + compile's emit order). + """ + from reflex_base.components.memo import MEMOS + + user_memos = [ + memo + for (_, source_module), memo in MEMOS.items() + if _memo_output_path(source_module) in dirty_paths + or _module_source_file(source_module) in changed_files + ] + return [*user_memos, *memo_uses.values()] + + +def _refresh_file_table( + manifest: dict[str, Any], validator: page_cache.FileValidator +) -> bool: + """Fold touched-but-identical files' fresh stat keys into the manifest. + + Args: + manifest: The loaded manifest (mutated). + validator: The validator whose ``refreshed`` entries to apply. + + Returns: + True if any entry was updated. + """ + if not validator.refreshed: + return False + manifest["files"].update(validator.refreshed) + return True + + +def _save_incremental_output( + path: str | Path, code: str, outputs: dict[Path, str] | None +) -> None: + """Stage output for plugin processing or write it for a direct rebuild. + + Args: + path: Generated path relative to the web directory. + code: Generated file content. + outputs: Output mapping supplied by the main compiler, if any. + """ + from reflex.compiler import utils as compiler_utils + + resolved = compiler_utils.resolve_path_of_web_dir(path) + if outputs is None: + compiler_utils.write_file(resolved, code) + else: + outputs[resolved] = code + + +def _app_root_change( + manifest: dict[str, Any], + pages: Sequence[PageDefinition], + compiled_pages: dict[str, PageContext], +) -> str | None: + """Explain how the recompiled pages changed the app root, or None. + + The app root is a function of the pages' merged app wraps (positioned by + the first page requiring each, valued by the last, like the full + compile's ``dict.update``). Rebuilding it from stored wraps would need + every wrap component to survive a round trip through the manifest, so + a changed root is left to a full compile instead; a page first gaining a + provider is rare. + + Args: + manifest: The loaded manifest. + pages: All current page definitions (in compile order). + compiled_pages: The recompiled pages' contexts, by route. + + Returns: + A human-readable reason, or None when the root on disk still matches. + """ + merged: dict[str, str] = {} + providers: dict[str, str] = {} + for page in pages: + if (page_ctx := compiled_pages.get(page.route)) is not None: + wraps = _record_app_wraps(page_ctx) + else: + wraps = manifest["pages"][page.route]["app_wraps"] + for key, digest in wraps.items(): + merged[key] = digest + providers[key] = page.route + stored = manifest["app_root"]["wraps"] + # Wraps of one priority nest in insertion order, so the order counts. + if list(merged.items()) == list(stored.items()): + return None + for key in sorted(merged.keys() | stored.keys()): + if merged.get(key) != stored.get(key): + return f"page {providers.get(key)!r} changed the app wrap {key!r}" + return "the app wraps changed their order" + + +def _new_bundled_libraries(manifest: dict[str, Any]) -> set[str]: + """The libraries this compile bundled that the recorded app root lacks. + + The bundled-library file lists the registry of a full compile, which the + recompiled pages' requests are only a part of; a library none of them + needs can stay listed, a new one needs the file rewritten. + + Args: + manifest: The loaded manifest. + + Returns: + The libraries missing from the recorded registry. + """ + live = RegistrationContext.ensure_context().bundled_libraries + return set(live) - set(manifest["app_root"]["libraries"]) + + +def _reconcile_memo_files( + manifest: dict[str, Any], + miss_routes: set[str], + miss_memo_keys: dict[str, list[str]], + emitted: Iterable[str], +) -> None: + """Update the memo file manifest and drop the auto memo files no page uses. + + Args: + manifest: The loaded manifest, with the hit pages' entries. + miss_routes: The recompiled routes. + miss_memo_keys: The memo keys of each recompiled route, now. + emitted: The memo files this rebuild wrote. + """ + from reflex.compiler import utils as compiler_utils + + used = set(manifest["app_root"]["memo_keys"]) + for route, entry in manifest["pages"].items(): + used.update( + miss_memo_keys[route] if route in miss_routes else entry["memo_keys"] + ) + dropped = { + _memo_file(tag) + for route in miss_routes + for tag in manifest["pages"][route]["memo_keys"] + if tag not in used + } + compiler_utils.reconcile_memo_manifest(emitted, dropped) + + +def _enable_radix_for_hit_pages( + compiler_plugins: Sequence[Plugin], manifest: dict[str, Any], miss_routes: set[str] +) -> None: + """Enable the implicit Radix Themes plugin when a reused page needs it. + + The implicit plugin enables itself while compiling a Radix component; a + page reused from disk is not compiled, so without this the stylesheet, + the theme and the package would drop out of a rebuild that re-evaluates + no Radix page. + + Args: + compiler_plugins: The resolved compiler plugins for this compile. + manifest: The loaded manifest. + miss_routes: The recompiled routes (compiled, so they enable it + themselves). + """ + from reflex_components_radix.plugin import RadixThemesPlugin + + for plugin in compiler_plugins: + if ( + isinstance(plugin, RadixThemesPlugin) + and not plugin.enabled + and any( + entry["radix"] + for route, entry in manifest["pages"].items() + if route not in miss_routes + ) + ): + plugin.enabled = True + + +def try_incremental_rebuild( + app: App, + *, + compiler_plugins: Sequence[Plugin], + prerender_routes: bool, + root: Path | None = None, + use_rich: bool = True, + outputs: dict[Path, str] | None = None, + registrations: dict[str, str] | None = None, +) -> bool | CompileContext: + """Attempt a disk-cache-assisted partial rebuild; report whether it ran. + + Returns False (so the caller does a full compile) whenever anything is + unsafe to reuse: no/old manifest, a changed global input, a changed page + set or registration. When a miss page turns out to have altered the app + root's inputs or its stateful flag, the miss pages are already evaluated + and rendered. A page is never evaluated twice in one process (its locally + defined states would register under collision-suffixed names the backend + does not use), so the context holding those compiled pages is returned + instead of False, for the full compile to adopt (see + ``CompileContext.absorb``). + + User-memo output files are reconciled against the manifest's stored memo + record (see :func:`_memo_state_entries`): files whose record differs are + rewritten. Auto memos are content-addressed, one file each, so the + recompiled pages write theirs without touching any other page's; files + no page uses any more are dropped. + + The ``assets`` copy is excluded from dependency tracking and always re-run + (cheap, idempotent). The contexts file is rewritten only when a stateful + miss changed its state config, and then from this process's state tree + plus the manifest's stored contributions of the stateful hit pages, so no + unchanged page is ever re-evaluated. + + On success, reports (at info level) how many pages were recompiled vs reused + and, while recompiling, shows a progress bar over the changed pages so a hot + reload makes the incremental work visible. The refreshed manifest is the + last output: with ``outputs`` it is staged after every other file, so a + manifest on disk never vouches for files that were not written. + + Args: + app: The app being compiled. + compiler_plugins: The resolved compiler plugins for this compile. + prerender_routes: Whether to prerender routes. + root: Project root for fingerprinting. Defaults to cwd. + use_rich: Whether to use a rich progress bar (else a plain fallback). + outputs: Stage generated files here for plugin processing instead of writing. + registrations: Each route's registration digest, taken before any page + evaluated; computed now when absent. + + Returns: + True if the partial rebuild completed (the caller should return), False + for a full compile, or the context of the pages already compiled here + for a full compile that adopts them. + """ + manifest = load_manifest() + if manifest is None: + _log_fallback( + "no reusable manifest (first compile, unreadable, or schema changed)" + ) + return False + + pages = list(app._unevaluated_pages.values()) + if registrations is None: + registrations = registration_digests(pages) + validator = page_cache.FileValidator(manifest["files"]) + changed_hint = _with_hinted_slices(_changed_hint, manifest) + + if (reason := globals_mismatch(manifest, validator=validator, root=root)) or ( + reason := _pages_mismatch(pages, manifest, registrations) + ): + _log_fallback(reason) + return False + + resolved_root = (root or Path.cwd()).resolve() + # The states importing the app registered (no page has evaluated yet). + page_states = _hit_page_states(manifest, set()) + live_module_states = [ + name + for name in RegistrationContext.ensure_context().base_states + if name not in page_states + ] + recorded_module_states = manifest["module_states"] + module_states_changed = sorted(live_module_states) != recorded_module_states[ + "names" + ] or not ( + ( + changed_hint is not None + and changed_hint.isdisjoint(recorded_module_states["files"]) + ) + or validator.unchanged(recorded_module_states["files"]) + ) + if module_states_changed: + import reflex.istate.dynamic as istate_dynamic + + base_states = RegistrationContext.ensure_context().base_states + if sorted( + name + for name in live_module_states + if base_states[name].__module__ == istate_dynamic.__name__ + ) != sorted(recorded_module_states["dynamic"]): + # They take names before any page, so every page's could shift. + _log_fallback("the app's import-time dynamic states changed") + return False + miss_pages = partition_pages(pages, manifest, validator, changed_hint) + # User memo files are decided salsa-style: derive the files the current + # memo registry demands and rewrite the ones whose stored record differs. + # This catches what changed-file diffing structurally cannot: a memo + # module the previous compile never saw (its file is in no recorded + # dependency set). Only import-owned memos are comparable here + # (pre-evaluation); route-owned entries are handled by owner status: a + # changed dependency pulls the owner page into the miss set, and its + # re-evaluation feeds the post-evaluation comparison below. + memo_hasher = page_cache.make_hasher() + memo_files_table: dict[str, page_cache.FileEntry] = {} + memo_state = _memo_state_entries( + memo_hasher, memo_files_table, resolved_root, lambda owner: owner is None + ) + stored_memo_files = manifest["memo_files"] + dirty_memo_paths = _dirty_memo_paths( + memo_state, stored_memo_files, validator, changed_hint + ) + stale_owner_routes = { + owner + for entry in stored_memo_files.values() + if (owner := entry["owner_route"]) is not None + and owner in manifest["pages"] + and _memo_deps_changed(entry["deps"], validator, changed_hint) + } + changed_files: set[str] = set() + if miss_pages or dirty_memo_paths or stale_owner_routes: + changed_files = _changed_dependency_files(manifest, validator, changed_hint) + miss_pages = _expand_miss_pages(miss_pages, pages, manifest, stale_owner_routes) + memo_note = ( + f"; rewriting {len(dirty_memo_paths)} memo file(s)" + if dirty_memo_paths + else "" + ) + logger.info( + f"Compile cache: recompiling {len(miss_pages)}/{len(pages)} pages; " + f"changed file(s): {format_path_list(changed_files, resolved_root)}" + f"{memo_note}" + ) + else: + logger.info(f"Compile cache: reusing all {len(pages)} pages from disk") + miss_routes = {p.route for p in miss_pages} + _enable_radix_for_hit_pages(compiler_plugins, manifest, miss_routes) + + from reflex.compiler import compiler + + if miss_pages or dirty_memo_paths: + # Match the full compile's clean bundling/memo state before compiling. + compiler._reset_compile_registries(compiler_plugins) + + # Recompile only the source-changed pages. + miss_ctx = None + if miss_pages: + # Page evaluation can import modules the cached import graph (built + # pre-evaluation for the memo state) has never seen; drop the cache + # afterwards so post-evaluation closures (memo entries, page deps) + # include the newly loaded modules. + modules_before = set(page_cache._loaded_first_party_modules(resolved_root)) + page_plugins: Sequence[Plugin] = compiler_plugins + if _predict_dynamic_names: + _claim_recorded_dynamic_names(manifest, miss_routes) + page_plugins = ( + _ComponentStateNumbersPlugin( + _numbers_before_miss_pages(pages, manifest, miss_routes) + ), + *compiler_plugins, + ) + + miss_ctx = CompileContext( + app=app, + pages=miss_pages, + hooks=CompilerHooks( + plugins=default_page_plugins(style=app.style, plugins=page_plugins) + ), + ) + # Progress over the changed pages (evaluate + render each), so a hot + # reload shows how much is being recompiled. + progress = console.progress() if use_rich else console.PoorProgress() + progress.start() + task = progress.add_task( + "Recompiling changed pages:", total=len(miss_pages) * 2 + ) + try: + with miss_ctx: + miss_ctx.compile( + evaluate_progress=lambda: progress.advance(task), + render_progress=lambda: progress.advance(task), + ) + finally: + progress.stop() + if set(page_cache._loaded_first_party_modules(resolved_root)) != modules_before: + page_cache.clear_import_graph() + _disown_hit_page_states(miss_ctx, manifest, miss_routes) + for page in miss_pages: + page_ctx = miss_ctx.compiled_pages.get(page.route) + if ( + page_ctx is None + or page_ctx.output_code is None + or page_ctx.output_path is None + ): + _fall_back_after_evaluation(f"page {page.route!r} produced no output") + return False + + if ( + _predict_dynamic_names + and miss_ctx is not None + and (route := _shifting_dynamic_claims(manifest, miss_ctx, miss_routes)) + ): + _fall_back_after_evaluation( + f"page {route!r} changed dynamic state names other pages share" + ) + if route := _route_dropping_a_shared_state(manifest, miss_routes): + _fall_back_after_evaluation( + f"page {route!r} no longer registers a state other pages may use" + ) + return miss_ctx if miss_ctx is not None else False + if miss_ctx is not None and ( + reason := _app_root_change(manifest, pages, miss_ctx.compiled_pages) + ): + _fall_back_after_evaluation(reason) + return miss_ctx + + # Write changed pages + their memo files; reuse everything else on disk. + install_imports = _deserialize_imports(manifest["all_imports"]) + eval_memo_state: dict[str, dict[str, Any]] = {} + emitted_memo_files: list[str] = [] + miss_memo_keys: dict[str, list[str]] = {} + if miss_ctx is not None or dirty_memo_paths: + memo_uses: dict[tuple[str, str | None], Any] = {} + miss_imports = [] + if miss_ctx is not None: + for page in miss_pages: + page_ctx = miss_ctx.compiled_pages[page.route] + # Both are guaranteed non-None by the guard loop above. + output_path = page_ctx.output_path + output_code = page_ctx.output_code + if output_path is None or output_code is None: + _fall_back_after_evaluation( + f"page {page.route!r} lost its output before write" + ) + return False + _save_incremental_output(output_path, output_code, outputs) + memo_uses.update(page_ctx.memo_uses) + miss_memo_keys[page.route] = _memo_keys(page_ctx) + miss_imports.append(page_ctx.frontend_imports) + # Post-evaluation pass: the miss pages just evaluated, so memos + # they own (registered during their evaluation, e.g. from modules + # imported inside the page function) are now in the registry. + # A brand-new such module has no stored record and must be + # written; an unchanged one matches its record and is skipped. + eval_memo_state = _memo_state_entries( + memo_hasher, + memo_files_table, + resolved_root, + lambda owner: owner in miss_routes, + ) + dirty_memo_paths = dirty_memo_paths | _dirty_memo_paths( + eval_memo_state, stored_memo_files, validator, changed_hint + ) + memo_files, memo_imports = compiler.compile_memo_components( + _memo_defs_for_rewrite(memo_uses, dirty_memo_paths, changed_files) + ) + for mpath, mcode in memo_files: + _save_incremental_output(mpath, mcode, outputs) + emitted_memo_files.append(mpath) + # Merge once: re-merging the app-wide set per page re-walks its ~100k + # entries each time. + install_imports = merge_imports(install_imports, *miss_imports, memo_imports) + + # Record which routes are stateful: miss pages from this compile, hit pages + # from the manifest, so the stateful-pages marker is complete. We do NOT + # re-evaluate hit pages to register their state in this process: it only + # produces .web and exits (the daemon, the initial compile, and CLI compiles + # never serve), and the serving backend re-evaluates the marked stateful + # pages itself. + stateful_routes: dict[str, None] = {} + for page in pages: + if page.route in miss_routes: + if miss_ctx is not None and page.route in miss_ctx.stateful_routes: + stateful_routes[page.route] = None + elif manifest["pages"][page.route]["is_stateful"]: + stateful_routes[page.route] = None + + if miss_ctx is not None: + compiler._register_compiled_pages(app, miss_ctx.compiled_pages) + + app._stateful_pages.update(stateful_routes) + app._write_stateful_pages_marker() + app._add_optional_endpoints() + app._validate_var_dependencies() + + # The contexts file holds EVERY state's defaults/dispatchers, including + # states only registered while their page evaluates (exec'd docs demos, + # dynamically imported modules). This process evaluated just the miss + # pages, so its state tree lacks the hit pages' page-registered states; + # their contributions come from the manifest instead (``state_slice``), + # so rebuilding contexts never re-evaluates an unchanged page. Only a + # stateful miss can change state config, and only its OWN states can differ + # from the on-disk contexts file, so compare those first: a content-only + # edit leaves them identical and the contexts file is reused untouched. + contexts_snapshot: compiler_utils.ContextsSnapshot | None = None + if module_states_changed or ( + miss_ctx is not None + and any( + route in miss_ctx.stateful_routes or manifest["pages"][route]["is_stateful"] + for route in miss_routes + ) + ): + contexts_snapshot = _contexts_snapshot(app) + changed_route = ( + "the app's states" + if module_states_changed + else _changed_state_config_route( + manifest, miss_ctx, contexts_snapshot, miss_routes + ) + if miss_ctx is not None + else None + ) + if changed_route is None: + logger.info( + "Compile cache: recompiled pages define unchanged states; " + "reusing contexts file" + ) + else: + logger.info( + f"Compile cache: {changed_route!r} changed the state config; " + "rebuilding contexts" + ) + from reflex_components_radix.plugin import RadixThemesPlugin + + theme = next( + ( + plugin.get_theme() + for plugin in compiler_plugins + if isinstance(plugin, RadixThemesPlugin) + ), + None, + ) + merged_snapshot = ( + None + if contexts_snapshot is None + else _with_hit_state_slices(contexts_snapshot, manifest, miss_routes) + ) + state_imports: ParsedImportDict = {} + context_path, context_code = compiler.compile_contexts( + app._state, + theme, + contexts_snapshot=merged_snapshot, + component_imports=state_imports, + ) + _save_incremental_output(context_path, context_code, outputs) + install_imports = merge_imports(install_imports, state_imports) + + # The bundled-library file and the app root reference every bundled + # library; a library first requested by this rebuild (a recompiled page, + # a state default) needs both rewritten, which only a full compile does. + if new_libraries := _new_bundled_libraries(manifest): + _fall_back_after_evaluation( + f"new bundled libraries: {format_path_list(new_libraries)}" + ) + return miss_ctx if miss_ctx is not None else False + + # The assets copy is cheap, idempotent, and excluded from dependency + # tracking entirely, so it is always re-run. + assets_src = (root or Path.cwd()) / constants.Dirs.APP_ASSETS + if assets_src.is_dir(): + path_ops.update_directory_tree( + src=assets_src, + dest=prerequisites.get_web_dir() / constants.Dirs.PUBLIC, + ) + + # Frontend packages + routing scaffolding (cheap, idempotent). + from reflex.utils import frontend_skeleton + + with log.timing(logger, "Install Frontend Packages"): + app._get_frontend_packages(install_imports) + frontend_skeleton.update_react_router_config(prerender_routes=prerender_routes) + frontend_skeleton.update_entry_client() + frontend_skeleton.initialize_vite_config() + + # The route table the production static server reads. + _save_incremental_output( + constants.Dirs.ROUTES_MANIFEST, json.dumps(app._page_routes), outputs + ) + _reconcile_memo_files(manifest, miss_routes, miss_memo_keys, emitted_memo_files) + + # The next manifest merges three memo-record sources, mirroring salsa's + # reuse of queries that did not re-run: route-owned entries whose owner + # page hit carry forward unchanged (their page never re-evaluated here), + # import-owned entries are recomputed from the pre-evaluation registry, + # and entries owned by the recompiled routes are recomputed post-eval. + final_memo_state = { + **{ + path: entry + for path, entry in stored_memo_files.items() + if entry["owner_route"] is not None + and entry["owner_route"] not in miss_routes + }, + **memo_state, + **eval_memo_state, + } + + # Refresh the manifest for the next process; it is the last output. + try: + refreshed = _update_manifest_for_misses( + manifest, + miss_ctx, + miss_pages, + install_imports, + root, + contexts_snapshot=contexts_snapshot, + memo_state=final_memo_state, + memo_file_entries=memo_files_table, + registrations=registrations, + module_states=( + _record_module_states( + live_module_states, + page_cache.make_hasher(), + manifest["files"], + root, + ) + if module_states_changed + else recorded_module_states + ), + validator=validator, + ) + except Exception as exc: # best-effort, like ``write_manifest`` + logger.warning( + f"Compile cache: could not refresh manifest ({exc!r}). " + "The next reload may need a full compile." + ) + # No manifest may vouch for the outputs about to be written. + _manifest_path().unlink(missing_ok=True) + refreshed = False + if refreshed: + _save_incremental_output(_manifest_path(), _manifest_text(manifest), outputs) + + app._cached_component_counts = manifest_component_counts(manifest) + return True + + +def manifest_component_counts(manifest: dict[str, Any]) -> dict[str, int]: + """Sum the component counts the manifest records per page. + + The compile telemetry of a process that reused every page from disk, or + compiled in a child, has no evaluated pages to count. + + Args: + manifest: The loaded manifest. + + Returns: + The count of each component class across the pages. + """ + counts: dict[str, int] = {} + for entry in manifest["pages"].values(): + for name, count in entry.get("component_counts", {}).items(): + counts[name] = counts.get(name, 0) + count + return counts + + +def _update_manifest_for_misses( + manifest: dict[str, Any], + miss_ctx: CompileContext | None, + miss_pages: Sequence[PageDefinition], + all_imports: ParsedImportDict, + root: Path | None = None, + *, + contexts_snapshot: compiler_utils.ContextsSnapshot | None = None, + memo_state: dict[str, dict[str, Any]], + memo_file_entries: dict[str, page_cache.FileEntry], + registrations: dict[str, str], + module_states: dict[str, list[str]], + validator: page_cache.FileValidator | None = None, +) -> bool: + """Update the manifest entries for the recompiled pages. + + Also refreshes the stat keys of files that were touched without changing + (so they are not re-hashed on every later reload). The caller persists + the manifest when something changed. + + Args: + manifest: The loaded manifest (mutated). + miss_ctx: The compile context of the recompiled pages, if any. + miss_pages: The recompiled page definitions. + all_imports: The complete frontend import set after this rebuild. + root: Project root for dependency discovery. Defaults to cwd. + contexts_snapshot: The app's ``_contexts_snapshot`` for slicing + stateful pages, or None when no miss page was stateful. + memo_state: The current user-memo file record to persist (see + :func:`_memo_state_entries`). + memo_file_entries: File-table entries for the memo records' closures, + withheld until validation finished. + registrations: Each route's registration digest. + module_states: The import-time states and their source files. + validator: The validator used for this rebuild, for its refreshed + entries. + + Returns: + Whether the manifest changed and must be written. + """ + dirty = validator is not None and _refresh_file_table(manifest, validator) + # An entry module whose edit only changed page registrations stays the + # recorded one; record its new content so the next rebuild skips it. + if ( + validator is not None + and (entry_path := manifest["entry"]["path"]) is not None + and validator.changed(entry_path) + and (entry := page_cache.file_entry(entry_path)) is not None + ): + manifest["files"][entry_path] = list(entry) + dirty = True + if manifest["memo_files"] != memo_state: + manifest["memo_files"] = memo_state + manifest["files"].update(memo_file_entries) + dirty = True + if miss_ctx is not None and miss_pages: + state_index, _ = page_cache.state_dependency_index(root) + hasher = page_cache.make_hasher() + for page in miss_pages: + page_ctx = miss_ctx.compiled_pages[page.route] + defined_states = miss_ctx.stateful_routes.get(page.route) + manifest["pages"][page.route] = _manifest_page_entry( + page_ctx, + page, + state_index, + hasher, + manifest["files"], + defined_states=defined_states, + registration=registrations[page.route], + state_slice=( + _state_slice(defined_states, *contexts_snapshot) + if defined_states and contexts_snapshot is not None + else None + ), + root=root, + ) + dirty = True + if manifest["module_states"] != module_states: + manifest["module_states"] = module_states + dirty = True + # A memo-only rewrite can add packages with no page recompiled. + if (imports := _serialize_imports(all_imports)) != manifest["all_imports"]: + manifest["all_imports"] = imports + dirty = True + return dirty diff --git a/reflex/compiler/page_cache.py b/reflex/compiler/page_cache.py new file mode 100644 index 00000000000..9b8f4b5f9bc --- /dev/null +++ b/reflex/compiler/page_cache.py @@ -0,0 +1,1825 @@ +"""Per-page dependency discovery for the incremental compile cache. + +Each page records the first-party Python import closure of its page callable, +files read during page evaluation, component modules in its rendered tree, and +fine-grained state files it references. ``disk_cache`` persists hashes of that +set so only pages whose dependencies changed need to be recompiled. + +Reads are tracked through ``open``, ``io.open``, ``Path.open``, +``Path.read_text``/``read_bytes``, ``tokenize.open`` (``linecache`` and +``inspect.getsource``) and ``io.open_code`` (``pkgutil.get_data``), on the +thread that evaluates the page. A page whose output depends on anything else +(a read from a C extension or a worker thread, the environment, the clock, a +directory listing, a subprocess) calls :func:`mark_volatile` while it +evaluates, and is then recompiled by every compile. +""" + +from __future__ import annotations + +import ast +import builtins +import contextlib +import dataclasses +import functools +import hashlib +import importlib +import importlib.util +import io +import os +import re +import sys +import time +import tokenize +from collections.abc import Callable, Iterable, Mapping, Sequence +from contextvars import ContextVar +from importlib import metadata +from pathlib import Path +from typing import TYPE_CHECKING + +from reflex_base import constants +from reflex_base.environment import environment + +if TYPE_CHECKING: + from reflex_base.components.component import BaseComponent + from reflex_base.plugins import PageContext + +#: A manifest file-table entry: ``[content sha256, st_mtime_ns, st_size]``, or +#: :data:`ABSENT_ENTRY` for a dependency that did not exist, or +#: :data:`RACY_ENTRY` for one that changed while the compile ran. +FileEntry = list + +#: The file-table entry of a dependency that was absent when recorded. It is +#: unchanged while the file stays absent. +ABSENT_ENTRY: FileEntry = [""] + +#: The file-table entry of a dependency modified at or after the compile +#: started (the racy-git rule): the content the compile read is unknown, so +#: the next compile always counts the file as changed and records it afresh. +RACY_ENTRY: FileEntry = ["", -1, -1] + +#: The dependency a volatile page records (see :func:`mark_volatile`). It has +#: no file-table entry, so the page misses on every compile. +VOLATILE_DEP = "" + +#: Genuinely-global files: a change here can affect every page's output, so it +#: is a global input rather than part of any single page's dependency set. +#: ``reflex.lock/`` is deliberately absent: its lockfiles are derived from +#: inputs tracked elsewhere (``rxconfig.py`` frontend packages, page imports, +#: the Reflex version) and are rewritten by every install, so treating them as +#: inputs made every reload fall back to a full compile. +_GLOBAL_FILES = ( + "rxconfig.py", + "uv.lock", + "pyproject.toml", + "requirements.txt", + "package.json", +) + +#: Matches a JS state-context identifier in compiled page output. +STATE_CONTEXT_RE = re.compile(r"reflex___state[A-Za-z0-9_]+") + + +def _reflex_version() -> str: + """Return the installed Reflex version. + + Returns: + The version, or ``unknown`` when the package metadata is missing. + """ + try: + return metadata.version("reflex") + except Exception: + return "unknown" + + +def first_party_roots(root: Path | None = None) -> tuple[Path, ...]: + """The directories whose Python files count as the app's own source. + + The project root plus ``REFLEX_HOT_RELOAD_INCLUDE_PATHS`` (a sibling local + package the backend reloader also watches). Files under any of these are + hashed into page dependency sets; everything else is an installed + dependency whose change flows through the version/epoch instead. + + Args: + root: Project root. Defaults to cwd. + + Returns: + The resolved roots, project root first. + """ + root = (root or Path.cwd()).resolve() + include = tuple( + Path(p).resolve() for p in environment.REFLEX_HOT_RELOAD_INCLUDE_PATHS.get() + ) + return (root, *(p for p in include if p != root)) + + +def _containing_root(path: Path, roots: Iterable[Path]) -> Path | None: + """Find the root that contains (or is) ``path``. + + Compares path strings: this runs for every import and read under a + recorder, and ``root in path.parents`` builds a Path per ancestor. + + Args: + path: Absolute path to test. + roots: Absolute root paths. + + Returns: + The first root containing ``path``, or None. + """ + path_str = str(path) + for root in roots: + root_str = str(root) + if path_str == root_str or path_str.startswith( + root_str if root_str.endswith(os.sep) else root_str + os.sep + ): + return root + return None + + +def _under_any(path: Path, roots: Iterable[Path]) -> bool: + """Whether ``path`` is nested below (or is) one of ``roots``. + + Args: + path: Absolute path to test. + roots: Absolute root paths. + + Returns: + True when some root contains ``path``. + """ + return _containing_root(path, roots) is not None + + +# A page's markdown/data dependencies are read lazily while the page is +# evaluated. Track those reads only while ``record_reads`` is active, so the +# idle overhead is one contextvar read. + +_active_reads: ContextVar[set[str] | None] = ContextVar("_active_reads", default=None) +_patched = False +_recorder_root: Path | None = None +#: The first-party roots of ``_recorder_root``, resolved once per root. +_recorder_roots: tuple[Path, ...] = () +#: ``time.time_ns()`` when the current compile started importing the app. +#: A dependency modified at or after it is recorded as :data:`RACY_ENTRY`. +_compile_start_ns: int | None = None + +#: Path parts that mark a dependency/build location whose files are never a +#: page's own source dependency (a change there flows through the version or +#: the global inputs). Tested against the parts below the first-party root a +#: file is under, so a project living in a directory of such a name is not +#: excluded wholesale; a file outside every root is tested by its full path. +_EXCLUDE_PARTS = frozenset({ + constants.Dirs.WEB, + ".venv", + "venv", + "node_modules", + "__pycache__", + ".git", + "site-packages", + "dist-packages", +}) + + +def _excluded(path: Path, roots: tuple[Path, ...]) -> bool: + """Whether ``path`` lies in a dependency/build location. + + Args: + path: Absolute path to test. + roots: The first-party roots. + + Returns: + True when a part of the path (below its root, if any) names an + excluded directory. + """ + parts = path.parts + if (root := _containing_root(path, roots)) is not None: + parts = parts[len(root.parts) :] + return not _EXCLUDE_PARTS.isdisjoint(parts) + + +#: Suffixes of content files a page may read from *outside* the project root +#: (e.g. the docs app reads its markdown from a sibling directory). Reads under +#: the project root are tracked regardless of suffix; reads elsewhere only if +#: they look like content, so stdlib/site reads of other types are ignored. +_CONTENT_SUFFIXES = { + ".md", + ".mdx", + ".py", + ".json", + ".yaml", + ".yml", + ".toml", + ".txt", + ".csv", + ".html", + ".rst", +} + +_PYTHON_PREFIXES = tuple( + Path(prefix).resolve() + for prefix in {sys.base_exec_prefix, sys.base_prefix, sys.exec_prefix, sys.prefix} + if prefix +) +#: Maps a module's raw ``__file__`` value to its recordable resolved path (or +#: None). Cleared by ``enable_read_tracking`` when the recorder root changes. +_module_file_cache: dict[object, str | None] = {} +_app_import_reads: dict[Path, set[str]] = {} + +#: Maps a first-party module file to the files code in it read under a +#: recorder, whichever page (or the app import) was running. A read behind a +#: shared cache (``functools.cache``, a module-level loader) happens once per +#: process, for the first caller only, so a page depends on the reads of every +#: module in its import closure, not only on the reads its own evaluation +#: performed. The over-approximation is safe. +_module_reads: dict[str, set[str]] = {} +_THIS_FILE = __file__ + + +def forget_module_reads() -> None: + """Drop the reads attributed to modules (before first-party code reloads).""" + _module_reads.clear() + + +@contextlib.contextmanager +def _suspend_tracking(): + """Ignore read/import hooks caused by dependency bookkeeping. + + Yields: + None. + """ + token = _active_reads.set(None) + try: + yield + finally: + _active_reads.reset(token) + + +def _reading_module_file() -> str | None: + """Find the first-party module whose code performed the current read. + + Walks the stack from the read hook's caller up to the first frame whose + code lives in a first-party module file, so a read made by a library on + a page's behalf is attributed to the page's code. + + Returns: + The resolved module file, or None when no first-party code is on the + stack. + """ + frame = sys._getframe(2) + while frame is not None: + file = frame.f_code.co_filename + if file != _THIS_FILE and (module_file := _recordable_module_file(file)): + return module_file + frame = frame.f_back + return None + + +def _record_read(path: object) -> None: + """Add a first-party file read to the active read set, if any. + + The read is also attributed to the first-party module that performed it + (see :data:`_module_reads`). + + Args: + path: The path the code read. + """ + target = _active_reads.get() + if target is None: + return + with _suspend_tracking(): + try: + resolved = Path(path).resolve() # type: ignore[arg-type] + except (OSError, TypeError, ValueError): + return + if _excluded(resolved, _recorder_roots): + return + if ( + not _under_any(resolved, _recorder_roots) + and resolved.suffix.lower() not in _CONTENT_SUFFIXES + ): + return + read = str(resolved) + target.add(read) + if (module_file := _reading_module_file()) is not None: + _module_reads.setdefault(module_file, set()).add(read) + + +def mark_volatile() -> None: + """Make the page being evaluated recompile on every compile. + + For a page whose output depends on an input the cache cannot see: a read + made by a C extension or on another thread, an environment variable, the + clock, a directory listing, a subprocess. A no-op outside a compile that + records dependencies. + """ + if (target := _active_reads.get()) is not None: + target.add(VOLATILE_DEP) + + +def _is_python_install_file(path: Path) -> bool: + """Return whether ``path`` is under this interpreter's install roots. + + Args: + path: Absolute path to test. + + Returns: + True when ``path`` is under the interpreter or virtualenv prefix. + """ + return any(prefix in path.parents for prefix in _PYTHON_PREFIXES) + + +def _recordable_module_file(file: object) -> str | None: + """Resolve a module file only when it can be a first-party dependency. + + Args: + file: The imported module's ``__file__`` value. + + Returns: + The resolved module file path to record, or None when it is outside the + project root or otherwise not recordable. + """ + root = _recorder_root + if root is None: + return None + # Hot path: every import statement under an active recorder lands here, so + # look up the raw ``__file__`` value before any Path construction. Keying + # by the raw value is safe because ``enable_read_tracking`` clears the + # cache whenever the recorder root changes. + try: + return _module_file_cache[file] + except KeyError: + pass + except TypeError: + return None + + with _suspend_tracking(): + try: + path = Path(file).absolute() # type: ignore[arg-type] + except (OSError, TypeError, ValueError): + return None + + roots = _recorder_roots + resolved_str = None + if not _excluded(path, roots) and ( + _under_any(path, roots) or not _is_python_install_file(path) + ): + try: + resolved = path.resolve() + except (OSError, TypeError, ValueError): + pass + else: + if _under_any(resolved, roots) and not _excluded(resolved, roots): + resolved_str = str(resolved) + _module_file_cache[file] = resolved_str + return resolved_str + + +def _record_module_file(module: object, target: set[str]) -> None: + """Record the source file for an imported module, if it has one. + + Args: + module: The imported module object. + target: The active read set. + """ + file = getattr(module, "__file__", None) + if not file: + return + if resolved_file := _recordable_module_file(file): + target.add(resolved_file) + + +def _absolute_import_name( + name: str, globals_: dict[str, object] | None, level: int +) -> str: + """Resolve an import name relative to the caller package. + + Args: + name: The import name passed to ``__import__``. + globals_: The caller globals passed to ``__import__``. + level: The relative import level. + + Returns: + The absolute module name when it can be resolved, else ``name``. + """ + if not level: + return name + package = None + if globals_ is not None: + package = globals_.get("__package__") + if not package and (module := globals_.get("__name__")): + package = ( + module + if isinstance(module, str) and globals_.get("__path__") is not None + else module.rpartition(".")[0] + if isinstance(module, str) + else None + ) + if not isinstance(package, str): + return name + with contextlib.suppress(Exception): + return importlib.util.resolve_name(f"{'.' * level}{name}", package) + return name + + +def _record_imported_modules( + name: str, + result: object, + target: set[str], + fromlist: Sequence[str] | None = None, +) -> None: + """Record source files for modules imported while a recorder is active. + + Args: + name: The absolute requested module name. + result: The object returned by ``__import__`` or ``import_module``. + target: The active read set. + fromlist: The ``fromlist`` passed to ``__import__``. + """ + _record_module_file(result, target) + result_id = id(result) + if (module := sys.modules.get(name)) and id(module) != result_id: + _record_module_file(module, target) + parent = name + while "." in parent: + parent = parent.rsplit(".", 1)[0] + if (module := sys.modules.get(parent)) is not None: + _record_module_file(module, target) + if not fromlist: + return + for item in fromlist: + if ( + item != "*" + and (module := sys.modules.get(f"{name}.{item}")) + and id(module) != result_id + ): + _record_module_file(module, target) + + +def enable_read_tracking(root: Path | None = None) -> None: + """Install per-page source-read tracking and register the recorder hook. + + Idempotent. Patches the read entry points named in the module docstring + and the import functions to record reads and imported module files (only + while a recording set is active) and points the reflex-base compile loop + at :func:`record_reads`. Called when an incremental cache flag is on; a + no-op to the compile otherwise. + + Args: + root: Project root; only reads under it are recorded. Defaults to cwd. + """ + global _patched, _recorder_root, _recorder_roots + resolved_root = (root or Path.cwd()).resolve() + if _recorder_root != resolved_root: + _module_file_cache.clear() + _module_reads.clear() + _recorder_roots = first_party_roots(resolved_root) + _recorder_root = resolved_root + + from reflex_base.plugins import compiler as _bc + + _bc.page_source_recorder = record_reads + + if _patched: + return + _patched = True + + orig_read_text = Path.read_text + orig_read_bytes = Path.read_bytes + orig_path_open = Path.open + orig_open = builtins.open + orig_open_code = io.open_code + orig_tokenize_open = tokenize.open + orig_import = builtins.__import__ + orig_import_module = importlib.import_module + + def _is_read_mode(mode: str) -> bool: + return "r" in mode and not ("w" in mode or "a" in mode or "x" in mode) + + def read_text(self: Path, *args: object, **kwargs: object): + _record_read(self) + return orig_read_text(self, *args, **kwargs) # type: ignore[arg-type] + + def read_bytes(self: Path): + _record_read(self) + return orig_read_bytes(self) + + # ``Path.open`` calls ``io.open`` directly (not ``builtins.open``), so it + # needs its own patch for reads through it to be recorded. + def path_open(self: Path, mode: str = "r", *args: object, **kwargs: object): + if _is_read_mode(mode): + _record_read(self) + return orig_path_open(self, mode, *args, **kwargs) # type: ignore[arg-type] + + def open_(file: object, mode: str = "r", *args: object, **kwargs: object): + if _is_read_mode(mode): + _record_read(file) + return orig_open(file, mode, *args, **kwargs) # type: ignore[arg-type] + + # ``importlib`` loaders (``pkgutil.get_data``) read through the C module. + def open_code(path: str): + _record_read(path) + return orig_open_code(path) + + # ``linecache`` (``inspect.getsource``) opens source through ``tokenize``, + # which bound ``builtins.open`` when it was imported. + def tokenize_open(filename: object): + _record_read(filename) + return orig_tokenize_open(filename) # type: ignore[arg-type] + + def import_( + name: str, + globals_: dict[str, object] | None = None, + locals_: dict[str, object] | None = None, + fromlist: Sequence[str] | None = (), + level: int = 0, + ): + result = orig_import(name, globals_, locals_, fromlist, level) + if (target := _active_reads.get()) is not None: + absolute = _absolute_import_name(name, globals_, level) + _record_imported_modules(absolute, result, target, fromlist) + return result + + def import_module(name: str, package: str | None = None): + result = orig_import_module(name, package) + if (target := _active_reads.get()) is not None: + _record_imported_modules(result.__name__, result, target) + return result + + Path.read_text = read_text # type: ignore[method-assign,assignment] + Path.read_bytes = read_bytes # type: ignore[method-assign,assignment] + Path.open = path_open # type: ignore[method-assign,assignment] + builtins.open = open_ # type: ignore[assignment] + io.open = open_ # type: ignore[assignment] + io.open_code = open_code + # ``importlib`` loaders call ``open_code`` on the C module object. + setattr(sys.modules["_io"], "open_code", open_code) # noqa: B010 + tokenize.open = tokenize_open # type: ignore[assignment] + builtins.__import__ = import_ # type: ignore[assignment] + importlib.import_module = import_module + + +@contextlib.contextmanager +def record_reads(): + """Record source-file reads on the current task into a fresh set. + + Yields: + The set of resolved source-file path strings read within the block. + """ + reads: set[str] = set() + token = _active_reads.set(reads) + try: + yield reads + finally: + _active_reads.reset(token) + + +@contextlib.contextmanager +def record_app_import(root: Path | None = None): + """Record files read/imported while importing the app module. + + Also marks the start of the compile, for the racy-edit rule of + :func:`file_entry`: everything the compile reads is read after this. + + Args: + root: Project root. Defaults to cwd. + + Yields: + None. + """ + global _compile_start_ns + root = (root or Path.cwd()).resolve() + _compile_start_ns = time.time_ns() + recorded: set[str] = set() + success = False + try: + with record_reads() as reads: + yield + recorded = set(reads) + success = True + finally: + if success: + _app_import_reads[root] = recorded + + +def _is_racy(mtime_ns: int) -> bool: + """Whether a file modified at ``mtime_ns`` may have changed during the compile. + + A file modified at or after the compile started may hold other content + than the compile read. A whole-second timestamp comes from a file system + with coarse timestamps, where a change in the compile's first second is + indistinguishable from one just before it. + + Args: + mtime_ns: The file's modification time. + + Returns: + True when the content the compile read cannot be vouched for. + """ + if (start := _compile_start_ns) is None: + return False + if mtime_ns % 1_000_000_000 == 0: + start -= start % 1_000_000_000 + return mtime_ns >= start + + +def _is_page_registration(statement: ast.stmt) -> bool: + """Whether a top-level statement is an ``.add_page(...)`` call. + + Args: + statement: A module-level statement. + + Returns: + True for an ``add_page`` call statement. + """ + return ( + isinstance(statement, ast.Expr) + and isinstance(statement.value, ast.Call) + and isinstance(statement.value.func, ast.Attribute) + and statement.value.func.attr == "add_page" + ) + + +def _loaded_names(nodes: Iterable[ast.AST]) -> set[str]: + """Collect the names the given syntax trees read. + + Args: + nodes: The syntax trees. + + Returns: + Every name loaded anywhere within them. + """ + return { + node.id + for tree in nodes + for node in ast.walk(tree) + if isinstance(node, ast.Name) and isinstance(node.ctx, ast.Load) + } + + +#: Separates a module file from a module-level function name in a dependency +#: on the parts of the module that function can read (see +#: :func:`function_slice_digest`). +SLICE_SEPARATOR = "::" + +#: Names through which code reaches module globals, or the module's own +#: source text or line numbers, that no syntax tree names: a slice reading +#: any of them depends on the whole module file. +_DYNAMIC_GLOBAL_NAMES = frozenset({ + "globals", + "vars", + "locals", + "eval", + "exec", + "__import__", + "sys", + "inspect", + "linecache", + "tokenize", + "getsource", + "getsourcelines", + "getsourcefile", +}) + +#: Decorators (``.``) that keep the function they wrap to +#: themselves at import time. +_INERT_DECORATORS = frozenset({ + ("rx", "page"), + ("rx", "memo"), + ("reflex", "page"), + ("reflex", "memo"), + ("functools", "cache"), + ("functools", "lru_cache"), +}) + + +def dep_file(dep: str) -> str: + """Return the file a dependency reads. + + Args: + dep: A file path, or a function slice (``::``). + + Returns: + The file path. + """ + file, separator, _ = dep.rpartition(SLICE_SEPARATOR) + return file if separator else dep + + +@dataclasses.dataclass(frozen=True, slots=True) +class _ModulePart: + """One top-level statement of a module, as a slice sees it.""" + + # The statement's syntax tree and the names it reads. + dump: str + loads: frozenset[str] + # A function or class: its name, and the parts that run at import time + # (decorators, defaults, bases) with the names they read. + name: str | None = None + is_class: bool = False + header: str = "" + header_loads: frozenset[str] = frozenset() + # An import: each bound name (``*`` for a star import) and its syntax tree. + aliases: tuple[tuple[str, str], ...] = () + + +def _is_inert_decorator(decorator: ast.expr) -> bool: + """Whether a decorator keeps the function it wraps to itself. + + Args: + decorator: The decorator expression. + + Returns: + True for a known decorator such as ``rx.page`` or ``functools.cache``. + """ + if isinstance(decorator, ast.Call): + decorator = decorator.func + return ( + isinstance(decorator, ast.Attribute) + and isinstance(decorator.value, ast.Name) + and (decorator.value.id, decorator.attr) in _INERT_DECORATORS + ) + + +def _module_part(statement: ast.stmt) -> _ModulePart: + """Describe one top-level statement for slicing. + + Args: + statement: The statement. + + Returns: + Its slice description. + """ + if isinstance(statement, (ast.Import, ast.ImportFrom)): + origin = ( + f"{statement.module}:{statement.level}:" + if isinstance(statement, ast.ImportFrom) + else "" + ) + return _ModulePart( + dump="", + loads=frozenset(), + aliases=tuple( + ( + (alias.asname or alias.name).partition(".")[0], + origin + ast.dump(alias), + ) + for alias in statement.names + ), + ) + dump = ast.dump(statement) + loads = frozenset(_loaded_names([statement])) + # A decorator other than the inert ones may hand the function to code + # that runs at import (a registry), so it counts as import-time code. + if isinstance( + statement, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef) + ) and all(_is_inert_decorator(d) for d in statement.decorator_list): + header: list[ast.expr] = list(statement.decorator_list) + if isinstance(statement, ast.ClassDef): + header.extend(statement.bases) + header.extend(keyword.value for keyword in statement.keywords) + else: + header.extend(statement.args.defaults) + header.extend(d for d in statement.args.kw_defaults if d is not None) + return _ModulePart( + dump=dump, + loads=loads, + name=statement.name, + is_class=isinstance(statement, ast.ClassDef), + header="".join(ast.dump(node) for node in header), + header_loads=frozenset(_loaded_names(header)), + ) + return _ModulePart(dump=dump, loads=loads) + + +#: File -> (stat key, parts) for the modules sliced in this process. +_module_parts_cache: dict[str, tuple[tuple[int, int], list[_ModulePart]]] = {} + + +def _module_parts(path: str) -> list[_ModulePart] | None: + """Parse a module into slice parts, leaving out its page registrations. + + Args: + path: The module file. + + Returns: + The parts in source order, or None when the file cannot be read or + parsed. + """ + key = _stat_key(path) + cached = _module_parts_cache.get(path) + if key is not None and cached is not None and cached[0] == key: + return cached[1] + try: + tree = ast.parse(Path(path).read_bytes()) + except (OSError, SyntaxError, ValueError): + return None + parts = [_module_part(s) for s in tree.body if not _is_page_registration(s)] + if key is not None: + _module_parts_cache[path] = (key, parts) + return parts + + +def _definition_closure( + definitions: Mapping[str, Sequence[_ModulePart]], starts: Iterable[str] +) -> set[str]: + """Name the definitions reachable from ``starts`` through the names they read. + + Args: + definitions: The module's definitions by name. + starts: The names to start from. + + Returns: + The reachable definition names. + """ + closure: set[str] = set() + stack = [name for name in starts if name in definitions] + while stack: + name = stack.pop() + if name in closure: + continue + closure.add(name) + stack.extend( + load + for part in definitions[name] + for load in part.loads + if load in definitions and load not in closure + ) + return closure + + +def _sliced_texts(part: _ModulePart, closure: set[str]) -> list[str]: + """Return what one module part adds to a slice. + + Args: + part: The part. + closure: The definitions in the slice. + + Returns: + The texts to digest. + """ + if part.aliases: + # Every import counts: importing a module runs it, and that can + # change what any code in this module sees. + return [dump for _, dump in part.aliases] + if part.name is None or part.name in closure: + return [part.dump] + return [part.header] if part.header else [] + + +def _slice_digest(parts: Sequence[_ModulePart], seeds: Iterable[str]) -> str | None: + """Digest the parts of a module that code starting at ``seeds`` can read. + + That is every statement that runs at import (imports included), the + import-time parts of every definition, and the whole definitions named + (transitively) by those or by ``seeds``. Like a dependency across modules, + a call-time mutation of a shared object is not tracked. + + Args: + parts: The module's parts (see :func:`_module_parts`). + seeds: The definition names to start from. + + Returns: + The hex digest, or None when the code reaches module globals or the + module's own source by a computed name, so only the whole file is a + safe dependency. + """ + definitions: dict[str, list[_ModulePart]] = {} + always: set[str] = set() + for part in parts: + if part.name is not None: + definitions.setdefault(part.name, []).append(part) + always |= part.header_loads + else: + always |= part.loads + closure = _definition_closure(definitions, (*seeds, *always)) + used = always.union(*(part.loads for name in closure for part in definitions[name])) + if not used.isdisjoint(_DYNAMIC_GLOBAL_NAMES): + return None + digest = hashlib.sha256() + for part in parts: + for text in _sliced_texts(part, closure): + digest.update(text.encode()) + digest.update(b"\n") + return digest.hexdigest() + + +def _class_names(parts: Sequence[_ModulePart]) -> list[str]: + """Name a module's top-level classes. + + Args: + parts: The module's parts. + + Returns: + The class names. + """ + return [part.name for part in parts if part.name is not None and part.is_class] + + +def function_slice_digest(path: str, name: str) -> str | None: + """Digest the parts of a module a module-level function can read. + + Every class counts: a page reaches the states and components of its own + module through its rendered tree, not only through names it reads. + + Args: + path: The module file. + name: The function's name. + + Returns: + The digest, or None when the file cannot be parsed, defines no such + module-level function or class, or reaches its globals or source by + a computed name (see :func:`_slice_digest`). + """ + parts = _module_parts(path) + if parts is None or not any(part.name == name for part in parts): + return None + return _slice_digest(parts, (name, *_class_names(parts))) + + +def entrypoint_digest(path: str) -> str | None: + """Digest the app entry module apart from its pages. + + Left out are top-level ``.add_page(...)`` statements and the page + functions defined in the module (each page tracks its own slice, see + :func:`function_slice_digest`). Every import counts, a page's included: + importing a module runs it. Class bodies run at import, so every class + counts. Also left out is what the syntax tree never holds (comments, + formatting). Two versions with the same digest differ only in their + pages, which the compile cache compares page by page; any other edit can + change every page. + + Args: + path: The entry module's file. + + Returns: + The digest, or None when the file cannot be read or parsed, or + reaches its globals or source by a computed name. + """ + parts = _module_parts(path) + if parts is None: + return None + return _slice_digest(parts, _class_names(parts)) + + +def _app_entrypoint_file(root: Path | None = None) -> Path | None: + """Resolve the user's app entrypoint module file (where ``rx.App`` is built). + + App-wide inputs like theme, app wraps, the toaster, stylesheets, and head + components are configured here, not in any single page's + dependency set. This is the root of :func:`app_dependency_files`, which walks + its imports (stopping at page modules) to find the config-only modules whose + change must invalidate the reused on-disk app-wide files. + + Args: + root: Project root; only a file under it is returned. Defaults to cwd. + + Returns: + The resolved entrypoint file path under ``root``, or None if it can't be + determined (no app module imported, or it lives outside ``root``). + """ + try: + from reflex.config import get_config + + module = get_config().module + except Exception: + return None + file = getattr(sys.modules.get(module), "__file__", None) + if not file: + return None + rf = Path(file).resolve() + root = (root or Path.cwd()).resolve() + return rf if root in rf.parents else None + + +def global_input_paths( + root: Path | None = None, *, pages: Sequence[object] | None = None +) -> dict[str, str]: + """Map each file-backed global input label to its path. + + These can affect every page's output but belong to no single page, so they + gate the whole cache rather than any one page's dependency set: the + config/lockfiles and the app-level config files, i.e. the app entrypoint + module plus the config-only modules it imports (theme, app-wraps, + stylesheets, head components; see :func:`app_dependency_files`). Kept small + on purpose; per-file edits flow through per-page dependency sets instead. + The manifest stores these labels with their paths and keeps the content + entries in its file table, so validation is stat-first like page deps. + + Args: + root: Project root. Defaults to cwd. + pages: The current page definitions, used as barriers so page modules + (tracked per page) are excluded from the app-level config files. + + Returns: + ``{label: path}`` for every global input except the Reflex version. + """ + root = (root or Path.cwd()).resolve() + paths = {name: str(root / name) for name in _GLOBAL_FILES} + paths.update({f"app:{p}": p for p in app_dependency_files(pages, root)}) + return paths + + +def _module_file(component: object) -> Path | None: + """Return the file of the module an object belongs to. + + Args: + component: The object. + + Returns: + The module file, or None. + """ + mod = sys.modules.get(getattr(component, "__module__", "") or "") + file = getattr(mod, "__file__", None) + return Path(file) if file else None + + +#: Component/state class and roots -> its first-party module file (or None). +#: A rendered tree has tens of thousands of nodes but few distinct classes, +#: and a class's module never moves; resolving per node dominated the +#: dependency walk. Warmed in the daemon parent so forked children inherit it. +_class_file_cache: dict[tuple[type, tuple[Path, ...]], Path | None] = {} + + +def _class_module_file(cls: type, roots: tuple[Path, ...]) -> Path | None: + """Resolve a class's module file when it is first-party. + + Args: + cls: The component or state class. + roots: The first-party roots. + + Returns: + The resolved module file under one of ``roots``, else None. + """ + key = (cls, roots) + try: + return _class_file_cache[key] + except KeyError: + pass + f = _module_file(cls) + resolved = None + if f is not None: + rf = f.resolve() + if _under_any(rf, roots): + resolved = rf + _class_file_cache[key] = resolved + return resolved + + +def warm_module_file_cache(root: Path | None = None) -> None: + """Pre-resolve the module file of every loaded component and state class. + + Args: + root: Project root. Defaults to cwd. + """ + from reflex_base.components.component import Component + + from reflex.state import BaseState + + roots = first_party_roots(root) + for base in (Component, BaseState): + for cls in _subclasses(base): + _class_module_file(cls, roots) + _loaded_first_party_modules(roots[0]) + + +def component_tree_classes(root_component: object) -> set[type]: + """Collect the class of every component in a rendered tree. + + Args: + root_component: The page's root component (its rendered tree). + + Returns: + The distinct component classes. + """ + classes: set[type] = set() + seen: set[int] = set() + stack: list[object] = [root_component] + while stack: + comp = stack.pop() + if id(comp) in seen: + continue + seen.add(id(comp)) + classes.add(type(comp)) + children = getattr(comp, "children", None) + if children: + stack.extend(children) + return classes + + +def component_module_files( + root_component: object, root: Path | None = None +) -> set[Path]: + """Resolve the first-party module files of every component in a tree. + + Walks the rendered component tree and collects each component class module + under ``root``. This catches component dependencies that static imports may + miss. + + Args: + root_component: The page's root component (its rendered tree). + root: Project root; only files under it are returned. Defaults to cwd. + + Returns: + The set of resolved first-party module files the tree depends on. + """ + roots = first_party_roots(root) + return { + rf + for cls in component_tree_classes(root_component) + if (rf := _class_module_file(cls, roots)) is not None + } + + +#: Cache of the first-party import graph, keyed by project root. +_import_graph_cache: dict[Path, dict[str, set[str]]] = {} + + +def _resolve_module_file(name: str) -> str | None: + """Return the resolved file of a loaded module. + + Args: + name: The dotted module name. + + Returns: + The resolved file path, or None. + """ + mod = sys.modules.get(name) + file = getattr(mod, "__file__", None) + return str(Path(file).resolve()) if file else None + + +def _loaded_first_party_modules(root: Path) -> dict[str, str]: + """Map first-party module files to module names. + + Args: + root: Resolved project root. + + Returns: + A mapping of resolved file path -> module name for loaded modules under + ``root``. + """ + roots = first_party_roots(root) + file_to_mod: dict[str, str] = {} + for name, mod in list(sys.modules.items()): + file = getattr(mod, "__file__", None) + if not file: + continue + if (resolved := _first_party_module_file(file, roots)) is not None: + file_to_mod[resolved] = name + return file_to_mod + + +@functools.lru_cache(maxsize=8192) +def _first_party_module_file(file: str, roots: tuple[Path, ...]) -> str | None: + """Resolve a stable module filename once per set of reload roots. + + Args: + file: The module's source filename. + roots: Resolved first-party roots. + + Returns: + The resolved filename if it belongs to a first-party root, else None. + """ + resolved = Path(file).resolve() + return str(resolved) if _under_any(resolved, roots) else None + + +def _import_from_targets( + node: object, modname: str, *, is_package: bool = False +) -> list[str]: + """Resolve a ``from ... import ...`` node to candidate module names. + + Handles relative imports via the importing module's package. Returns the + base module and each ``base.name``. A name may be a submodule or an + attribute; both candidates are resolved against ``sys.modules`` by the + caller. + + Args: + node: An ``ast.ImportFrom`` node. + modname: The dotted name of the module containing the import. + is_package: Whether the importing module is a package initializer. + + Returns: + Candidate dotted module names to resolve. + """ + if not isinstance(node, ast.ImportFrom): + return [] + if node.level: # relative import: walk up from the importing package + package = modname if is_package else modname.rpartition(".")[0] + try: + base = importlib.util.resolve_name( + "." * node.level + (node.module or ""), package + ) + except (ImportError, ValueError): + return [] + else: + base = node.module or "" + if not base: + return [] + return [base, *(f"{base}.{a.name}" for a in node.names)] + + +#: file -> ((st_mtime_ns, st_size), imported dotted names). Parsing every +#: first-party module on every hot reload is the cost this avoids; the daemon +#: persists it in the manifest so a fresh process only parses changed files. +_import_names_cache: dict[str, tuple[tuple[int, int], list[str]]] = {} + + +def _stat_key(path: str) -> tuple[int, int] | None: + """Return ``(st_mtime_ns, st_size)`` for ``path``, or None if unreadable. + + Args: + path: The file path. + + Returns: + The stat key, or None. + """ + try: + st = Path(path).stat() + except OSError: + return None + return (st.st_mtime_ns, st.st_size) + + +def load_import_names_cache(data: Mapping[str, Sequence[object]]) -> None: + """Seed the import-name parse cache from its persisted form. + + Args: + data: ``{path: [[mtime_ns, size], [names...]]}`` as written by + :func:`export_import_names_cache`. + """ + _import_names_cache.clear() + for path, entry in data.items(): + if ( + isinstance(entry, Sequence) + and len(entry) == 2 + and isinstance(entry[0], Sequence) + and len(entry[0]) == 2 + and all(isinstance(n, int) for n in entry[0]) + and isinstance(entry[1], Sequence) + ): + key = (int(entry[0][0]), int(entry[0][1])) # type: ignore[index] + _import_names_cache[path] = (key, [str(n) for n in entry[1]]) + + +def export_import_names_cache() -> dict[str, list[object]]: + """Return the import-name parse cache in its persisted form. + + Returns: + ``{path: [[mtime_ns, size], [names...]]}``. + """ + return { + path: [list(key), names] for path, (key, names) in _import_names_cache.items() + } + + +def _module_import_names(file: str, modname: str) -> list[str]: + """Return the dotted names one module imports (statically). + + Cached by the file's stat key, so an unchanged file is never re-parsed. + + Args: + file: The resolved module file path. + modname: The module's dotted name. + + Returns: + The imported dotted-name candidates (see :func:`_import_from_targets`). + """ + key = _stat_key(file) + cached = _import_names_cache.get(file) + if cached is not None and key is not None and cached[0] == key: + return cached[1] + names: list[str] = [] + try: + tree = ast.parse(Path(file).read_bytes()) + except (OSError, SyntaxError, ValueError): + return names + for node in ast.walk(tree): + if isinstance(node, ast.Import): + names.extend(a.name for a in node.names) + elif isinstance(node, ast.ImportFrom): + names.extend( + _import_from_targets( + node, modname, is_package=Path(file).name == "__init__.py" + ) + ) + if key is not None: + _import_names_cache[file] = (key, names) + return names + + +def _module_import_edges( + file: str, modname: str, file_to_mod: dict[str, str] +) -> set[str]: + """Return first-party files imported by one module. + + Args: + file: The resolved module file path. + modname: The module's dotted name. + file_to_mod: Loaded first-party module files. + + Returns: + The resolved first-party files imported by ``file``. + """ + deps: set[str] = set() + for name in _module_import_names(file, modname): + target = _resolve_module_file(name) + if target is not None and target in file_to_mod: + deps.add(target) + return deps + + +def build_import_graph(root: Path | None = None) -> dict[str, set[str]]: + """Build the first-party import graph (file -> files it imports). + + Parses already-imported first-party modules and resolves their imports to + files under ``root`` via ``sys.modules``. Cached per root for the duration of + the process. + + Args: + root: Project root. Defaults to cwd. + + Returns: + A mapping of resolved file path -> the set of first-party files it imports. + """ + root = (root or Path.cwd()).resolve() + cached = _import_graph_cache.get(root) + if cached is not None: + return cached + + file_to_mod = _loaded_first_party_modules(root) + graph = { + file: _module_import_edges(file, modname, file_to_mod) + for file, modname in file_to_mod.items() + } + _import_graph_cache[root] = graph + return graph + + +def clear_import_graph() -> None: + """Drop cached import graphs (e.g. after modules are reloaded).""" + _import_graph_cache.clear() + _app_import_reads.clear() + + +def _walk_import_closure( + graph: dict[str, set[str]], + starts: set[str], + barriers: set[str] | None = None, +) -> set[str]: + """Walk a first-party import graph from ``starts``. + + Args: + graph: Mapping of source file to imported source files. + starts: Source files where traversal begins. + barriers: Files included elsewhere and not traversed into. + + Returns: + The reachable source files, excluding barriers. + """ + barriers = barriers or set() + seen: set[str] = set() + stack = list(starts - barriers) + while stack: + cur = stack.pop() + if cur in seen: + continue + seen.add(cur) + stack.extend(dep for dep in graph.get(cur, ()) if dep not in barriers) + return seen + + +def _component_source_files(component: object, root: Path) -> set[str]: + """The component callable's own defining files under ``root``. + + The callable's *real* code filename (``__code__``, correct even when + ``__module__`` was reassigned, as the docs app does for generated pages) plus + its module file. These are the roots a page's import closure walks from, and + the barriers the app-config walk stops at. + + Args: + component: The page component or callable. + root: Resolved project root; only files under it are returned. + + Returns: + The set of resolved defining file path strings under ``root``. + """ + while isinstance(component, functools.partial): + component = component.func + out: set[str] = set() + roots = first_party_roots(root) + code = getattr(component, "__code__", None) + filename = getattr(code, "co_filename", None) + own = _module_file(component) + for path in (filename, own): + if path: + rf = Path(path).resolve() + if _under_any(rf, roots): + out.add(str(rf)) + return out + + +def page_py_dependencies( + component: BaseComponent | object, root: Path | None = None +) -> set[str]: + """Return the transitive first-party ``.py`` files a page's code depends on. + + Starts from the page callable's code filename plus its module file, then + walks the import graph. This captures function-based views and shared + helpers that the rendered-tree walk cannot see. + + Args: + component: The page component or callable. + root: Project root. Defaults to cwd. + + Returns: + The set of resolved first-party ``.py`` dependency file paths. + """ + root = (root or Path.cwd()).resolve() + graph = build_import_graph(root) + return _walk_import_closure(graph, _component_source_files(component, root)) + + +def module_py_dependencies(module_file: str, root: Path | None = None) -> set[str]: + """Return the transitive first-party ``.py`` files a module depends on. + + Args: + module_file: The module's resolved source file path. + root: Project root. Defaults to cwd. + + Returns: + The reachable first-party dependency file paths, including + ``module_file`` itself. + """ + root = (root or Path.cwd()).resolve() + return _walk_import_closure(build_import_graph(root), {module_file}) + + +def app_dependency_files( + pages: Sequence[object] | None = None, root: Path | None = None +) -> set[str]: + """First-party files whose change affects app-level config (not any page). + + Walks the first-party import graph from the app entrypoint + (:func:`_app_entrypoint_file`), treating each page-defining module as a + barrier (not entered), then folds in files read/imported while the app module + loaded. Static page dependency closures are removed from that dynamic set so + regular page edits still invalidate per-page rather than globally. A config + module shared with a page is still captured when it is reached from the + entrypoint directly. + + These configure the app-wide files an incremental rebuild reuses on disk + (app root, contexts, theme, stylesheet), so they are folded into + :func:`global_input_paths`: editing one forces a full recompile instead of leaving + those files stale. + + Args: + pages: The current page definitions, used as traversal barriers. When + None (no page set available), no barriers apply. + root: Project root. Defaults to cwd. + + Returns: + The set of resolved app-config dependency file path strings, or empty if + the entrypoint can't be resolved. + """ + root = (root or Path.cwd()).resolve() + entrypoint = _app_entrypoint_file(root) + if entrypoint is None: + return set() + graph = build_import_graph(root) + barriers: set[str] = set() + page_deps: set[str] = set() + for page in pages or (): + starts = _component_source_files(getattr(page, "component", None), root) + barriers |= starts + page_deps |= _walk_import_closure(graph, starts) + + start = str(entrypoint) + barriers.discard(start) + static_deps = _walk_import_closure(graph, {start}, barriers) + dynamic_deps = _app_import_reads.get(root, set()) - page_deps + return static_deps | dynamic_deps + + +def file_entry(path: str) -> FileEntry | None: + """Read a file once and describe it for the manifest file table. + + Args: + path: The file path. + + Returns: + ``[sha256, st_mtime_ns, st_size]``, or None if unreadable. For a + function slice (see :data:`SLICE_SEPARATOR`), the slice digest and + the stat of its file. :data:`RACY_ENTRY` for a file modified since + the compile started (see :func:`_is_racy`). + """ + file, separator, name = path.rpartition(SLICE_SEPARATOR) + if separator: + key = _stat_key(file) + digest = function_slice_digest(file, name) + if key is None or digest is None: + return None + return RACY_ENTRY if _is_racy(key[0]) else [digest, *key] + try: + with Path(path).open("rb") as fh: + st = os.fstat(fh.fileno()) + digest = hashlib.sha256(fh.read()).hexdigest() + except OSError: + return None + if _is_racy(st.st_mtime_ns): + return RACY_ENTRY + return [digest, st.st_mtime_ns, st.st_size] + + +def make_hasher() -> Callable[[str], FileEntry | None]: + """Return a file-table entry builder that memoizes each path within one compile. + + Shared component files appear in many pages' dependency sets; reading each + file at most once keeps manifest writes cheap. + + Returns: + A function mapping a path string to its :func:`file_entry` (None if + unreadable). + """ + cache: dict[str, FileEntry | None] = {} + + def hasher(path: str) -> FileEntry | None: + if path not in cache: + cache[path] = file_entry(path) + return cache[path] + + return hasher + + +class FileValidator: + """Decide whether recorded files still match their manifest entries. + + Validation is stat-first: a file whose ``(mtime_ns, size)`` still matches + its entry is unchanged without being read. Only a file whose stat moved is + hashed, and a touch that left the content identical counts as unchanged + (its refreshed entry is exposed in ``refreshed`` so the manifest can stop + re-hashing it). Each path is examined at most once per compile. + """ + + def __init__(self, files: Mapping[str, Sequence[object]]) -> None: + """Bind the validator to a manifest file table. + + Args: + files: ``{path: [sha256, mtime_ns, size]}``. + """ + self._files = files + self._changed: dict[str, bool] = {} + self.refreshed: dict[str, FileEntry] = {} + + def changed(self, path: str) -> bool: + """Whether ``path`` differs from its recorded entry (or has none). + + Args: + path: The file path. + + Returns: + True if the file changed, is unreadable, or is not in the table. + A recorded-absent file is unchanged while it is still absent; a + racy entry (see :data:`RACY_ENTRY`) always counts as changed. + """ + cached = self._changed.get(path) + if cached is not None: + return cached + entry = self._files.get(path) + result = True + if entry == ABSENT_ENTRY: + result = SLICE_SEPARATOR in path or Path(path).exists() + elif entry is not None and entry != RACY_ENTRY and len(entry) == 3: + key = _stat_key(dep_file(path)) + if key is not None and (key[0], key[1]) == (entry[1], entry[2]): + result = False + elif (fresh := file_entry(path)) is not None and fresh[0] == entry[0]: + self.refreshed[path] = fresh + result = False + self._changed[path] = result + return result + + def unchanged(self, paths: Iterable[str]) -> bool: + """Whether every file in ``paths`` is unchanged. + + Args: + paths: The recorded dependency paths of one page. + + Returns: + True if no file changed. + """ + return not any(self.changed(path) for path in paths) + + +def _subclasses(root_cls: type) -> list[type]: + """List a class and every subclass below it. + + Args: + root_cls: The class to start from. + + Returns: + The classes, each once. + """ + seen: set[type] = set() + out: list[type] = [] + stack = [root_cls] + while stack: + cls = stack.pop() + if cls in seen: + continue + seen.add(cls) + out.append(cls) + stack.extend(cls.__subclasses__()) + return out + + +def state_dependency_index( + root: Path | None = None, +) -> tuple[dict[str, Path], set[Path]]: + """Build the state-context -> file index for fine-grained invalidation. + + A page references the JS state-context identifier + ``format_state_name(state.get_full_name())`` in its output; mapping that to + the state's module file lets a state edit invalidate only its dependents. + Only *pure* state modules under ``root`` (no Component defined in them) are + fine-grained; mixed state/component modules stay coarse (in the shared + fingerprint) so a component edit there is never missed. + + Args: + root: Project root. Defaults to cwd. + + Returns: + ``(identifier_to_file, fine_state_files)``. + """ + roots = first_party_roots(root) + from reflex_base.components.component import Component + from reflex_base.utils import format as fmt + + from reflex.state import BaseState + + component_files = { + rf for cls in _subclasses(Component) if (rf := _class_module_file(cls, roots)) + } + id_to_file: dict[str, Path] = {} + state_files: set[Path] = set() + for cls in _subclasses(BaseState): + rf = _class_module_file(cls, roots) + if rf is None: + continue + with contextlib.suppress(Exception): + id_to_file[fmt.format_state_name(cls.get_full_name())] = rf + state_files.add(rf) + fine = state_files - component_files + return {i: f for i, f in id_to_file.items() if f in fine}, fine + + +def used_state_files( + output_code: str, + memo_components: object, + id_to_file: dict[str, Path], +) -> set[Path]: + """Return the fine-grained state files a compiled page depends on. + + Stateful subtrees are auto-memoized into separate components, so a page's + own ``output_code`` may not name the state it uses; the state lives in the + memo components it uses (its ``memo_uses``), so scanning ``output_code`` + plus those captures the full dependency set. If a memo can't be + introspected, depend on every fine state file. + + Args: + output_code: The page's compiled JS. + memo_components: The memo component subtrees the page uses (renderable). + id_to_file: The state-context identifier -> file index. + + Returns: + The set of fine-grained state files this page depends on. + """ + chunks = [output_code or ""] + try: + chunks.extend(str(comp.render()) for comp in memo_components) # type: ignore[attr-defined] + except Exception: + return set(id_to_file.values()) # conservative: depend on all + found: set[Path] = set() + for chunk in chunks: + for ident in STATE_CONTEXT_RE.findall(chunk): + if ident in id_to_file: + found.add(id_to_file[ident]) + return found + + +def page_dependency_files( + page_ctx: PageContext, + component: BaseComponent | object, + state_index: dict[str, Path], + root: Path | None = None, +) -> set[str]: + """Return the full set of source files a compiled page depends on. + + The union of: the page's own defining module, the files read while it was + evaluated (markdown/data, via the read recorder), the component modules in + its rendered tree, and the fine-grained state files it references. A change + to any file *outside* this set cannot change the page's output, except the + genuinely-global inputs tracked by :func:`global_input_paths` and route additions + (shared nav). This is the dependency set that makes invalidation precise. + + Args: + page_ctx: The compiled page context (tree, output, recorded reads). + component: The page's component/callable (for its defining module). + state_index: The state-context identifier -> file index. + root: Project root; only files under it are included. Defaults to cwd. + + Returns: + The set of resolved dependency file path strings. + """ + root = (root or Path.cwd()).resolve() + files: set[Path] = set() + files |= component_module_files(page_ctx.root_component, root) + files |= used_state_files( + page_ctx.output_code or "", + [m.component for m in page_ctx.memo_uses.values()], + state_index, + ) + deps = {str(f) for f in files} + # Transitive first-party .py imports (captures function-based views/helpers + # that never appear as nodes in the rendered tree). + deps |= page_py_dependencies(component, root) + # Files read while evaluating the page (markdown/data), plus the import + # closure of any module the page imported at evaluation time: a helper + # already loaded by an earlier page does not re-execute its own imports + # under the recorder, so its dependencies come from the graph instead. + deps |= set(page_ctx.source_files) + dynamic_py = {f for f in page_ctx.source_files if f.endswith(".py")} + if dynamic_py: + deps |= _walk_import_closure(build_import_graph(root), dynamic_py) + # What the modules in the closure read on any page's behalf (a read + # behind a shared cache happens for the first caller only). + deps.update(*(_module_reads.get(dep, ()) for dep in list(deps))) + if (own := _own_function_slice(component, page_ctx.source_files, root)) is not None: + deps.discard(dep_file(own)) + deps.add(own) + return deps + + +def _own_function_slice( + component: object, source_files: Iterable[str], root: Path +) -> str | None: + """The slice of its module a page function reads, when that is all it reads. + + A page reaching its module another way (an import cycle, an import or a + read during evaluation) depends on the whole file. + + Args: + component: The page component or callable. + source_files: The files read while the page was evaluated. + root: Resolved project root. + + Returns: + The slice dependency (``::``), or None. + """ + name = getattr(component, "__name__", None) + files = _component_source_files(component, root) + if ( + not isinstance(name, str) + or getattr(component, "__qualname__", None) != name + or len(files) != 1 + ): + return None + (file,) = files + source_files = set(source_files) + graph = build_import_graph(root) + starts = graph.get(file, set()) | {f for f in source_files if f.endswith(".py")} + if ( + file in source_files + or file in _walk_import_closure(graph, starts) + or function_slice_digest(file, name) is None + ): + return None + return f"{file}{SLICE_SEPARATOR}{name}" + + +def page_dependency_entries( + page_ctx: PageContext, + component: BaseComponent | object, + state_index: dict[str, Path], + hasher: Callable[[str], FileEntry | None], + files: dict[str, FileEntry], + root: Path | None = None, +) -> list[str]: + """Record a page's dependency set into the manifest file table. + + Args: + page_ctx: The compiled page context. + component: The page's component/callable. + state_index: The state-context identifier -> file index. + hasher: A memoized path -> :func:`file_entry` function. + files: The manifest file table, extended in place. + root: Project root. Defaults to cwd. + + Returns: + The sorted dependency paths, including missing files (the page's ``deps``). + """ + deps: list[str] = [] + for path in sorted(page_dependency_files(page_ctx, component, state_index, root)): + if path != VOLATILE_DEP: + entry = hasher(path) + files[path] = ABSENT_ENTRY if entry is None else entry + deps.append(path) + return deps diff --git a/reflex/compiler/plugins/builtin.py b/reflex/compiler/plugins/builtin.py index f8855b8a7b1..5d09dda758a 100644 --- a/reflex/compiler/plugins/builtin.py +++ b/reflex/compiler/plugins/builtin.py @@ -55,7 +55,7 @@ def eval_page( if (description := getattr(page, "description", None)) is not None: meta_args["description"] = description - utils.add_meta(component, **meta_args) + component = utils.add_meta(component, **meta_args) except Exception as err: if hasattr(err, "add_note"): err.add_note(f"Happened while evaluating page {page.route!r}") diff --git a/reflex/compiler/plugins/memoize.py b/reflex/compiler/plugins/memoize.py index 86b6b53a3e4..85243972621 100644 --- a/reflex/compiler/plugins/memoize.py +++ b/reflex/compiler/plugins/memoize.py @@ -385,16 +385,17 @@ def _build_wrapper( # tag is derived from the rendered memo body inside # ``create_passthrough_component_memo`` after the ``{children}`` hole # is substituted, so passthrough wrappers that differ only in their - # children collapse to a single definition. - wrapper_factory, definition = create_passthrough_component_memo( - comp, source_module=page_context.source_module - ) + # children collapse to a single definition. An auto memo carries no + # source module: it is content-addressed by its tag and compiles to + # its own file, so identical subtrees anywhere in the app share one + # definition and a page's memo files never hold another page's memos + # (the compile cache rewrites a page's memo files without touching + # the pages it reuses). + wrapper_factory, definition = create_passthrough_component_memo(comp) tag = definition.export_name compile_context.memoize_wrappers[tag] = None - # Key by (tag, source_module) so identical-rendering subtrees from - # different user modules each get their own entry and emit into the - # right mirrored memo file. - compile_context.auto_memo_components[tag, definition.source_module] = definition + compile_context.auto_memo_components[tag, None] = definition + page_context.memo_uses[tag, None] = definition wrapper = wrapper_factory() # The wrapper has no structural children at the page level, but parents diff --git a/reflex/compiler/utils.py b/reflex/compiler/utils.py index 8168fed9668..8e27998dd4f 100644 --- a/reflex/compiler/utils.py +++ b/reflex/compiler/utils.py @@ -41,7 +41,7 @@ from reflex.istate.delta import _resolve_delta from reflex.istate.storage import Cookie, LocalStorage, SessionStorage -from reflex.state import BaseState, cache_initial_snapshot +from reflex.state import BaseState, _frontend_schema_entries, cache_initial_snapshot from reflex.utils import path_ops from reflex.utils.exec import is_prod_mode from reflex.utils.prerequisites import get_web_dir @@ -242,17 +242,17 @@ def compile_state(state: type[BaseState]) -> dict: return _sorted_keys(resolved_initial_state) -def _compile_initial_state( - state: type[BaseState], *, component_imports: ParsedImportDict | None = None -) -> tuple[dict, str]: +def _serialize_initial_state( + initial_state: dict, *, component_imports: ParsedImportDict | None = None +) -> str: """Serialize initial state while discovering its dynamic component imports. Args: - state: The app state class. + initial_state: The compiled initial-state mapping. component_imports: Optional accumulator for frontend package installation. Returns: - The initial state dictionary and its serialized JSON. + The serialized initial-state JSON. """ def serialize_initial_value(value: Any) -> Any: @@ -272,9 +272,24 @@ def serialize_initial_value(value: Any) -> Any: component_imports.setdefault(library, []).extend(fields) return serializers.serialize(value) + return format.json_dumps(initial_state, default=serialize_initial_value) + + +def _compile_initial_state( + state: type[BaseState], *, component_imports: ParsedImportDict | None = None +) -> tuple[dict, str]: + """Compile and serialize initial state while discovering component imports. + + Args: + state: The app state class. + component_imports: Optional accumulator for frontend package installation. + + Returns: + The initial state dictionary and its serialized JSON. + """ initial_state = compile_state(state) - return initial_state, format.json_dumps( - initial_state, default=serialize_initial_value + return initial_state, _serialize_initial_state( + initial_state, component_imports=component_imports ) @@ -401,6 +416,28 @@ def compile_client_storage( } +ContextsSnapshot = tuple[ + dict[str, Any], dict[str, dict[str, Any]], dict[str, list[Any]] +] + + +def compile_contexts_snapshot(state: type[BaseState]) -> ContextsSnapshot: + """Compile the state-tree inputs of the contexts file. + + Args: + state: The app state. + + Returns: + The initial state, the client storage, and the frontend schema entries, + each keyed by state full name. + """ + return ( + compile_state(state), + compile_client_storage(state), + _frontend_schema_entries(state, RegistrationContext.get()), + ) + + def _apply_component_style_for_compile(component: Component) -> Component: """Apply the app style to a compiled component tree. @@ -984,6 +1021,18 @@ def get_memo_components_dir() -> str: return str(get_web_dir() / constants.Dirs.COMPONENTS_PATH) +def get_memo_component_path(name: str) -> str: + """Get the on-disk path of an un-mirrored per-memo module. + + Args: + name: The memo's export name. + + Returns: + The absolute path of the memo's ``.jsx`` file. + """ + return str(Path(get_memo_components_dir()) / f"{name}{constants.Ext.JSX}") + + def get_memo_module_path(segments: tuple[str, ...]) -> str: """Get the on-disk path for a memo module mirrored from a Python module. @@ -1027,10 +1076,12 @@ def add_meta( children.append(Description.create(content=description)) children.append(Image.create(content=image)) - page.children.extend(children) - page.children.extend(meta_tags) - - return page + # Page roots may be shared by the construction cache; copy before appending + # metadata so repeated reuse does not accumulate duplicate tags. + new_page = copy.copy(page) + new_page.children = [*page.children, *children, *meta_tags] + new_page._clear_compile_caches() + return new_page def resolve_path_of_web_dir(path: str | Path) -> Path: @@ -1100,30 +1151,29 @@ def _write_memo_manifest(web_dir: Path, relative_paths: set[str]) -> None: raise -def prune_stale_memo_files(emitted_paths: Iterable[str | Path]) -> None: - """Delete memo files written previously that this compile no longer emits. - - Only paths that appear in the previous manifest are considered for - deletion — never a fresh filesystem walk — so files this code did not - emit are never touched. Empty parent directories created by mirrored - output are removed up to (but not including) the ``.web`` root. +def _memo_relative_paths(web_dir: Path, paths: Iterable[str | Path]) -> set[str]: + """Express memo file paths as the memo manifest records them. Args: - emitted_paths: Paths the current compile produced for the memo - pipeline, as built by joining :func:`get_web_dir` (so they share - its prefix — relative by default, absolute when overridden). + web_dir: The project's ``.web`` directory. + paths: Memo file paths, as built by joining :func:`get_web_dir` (so + they share its prefix — relative by default, absolute when + overridden). + + Returns: + The paths relative to ``web_dir``, with forward slashes. """ - web_dir = get_web_dir() + return {str(Path(path).relative_to(web_dir)).replace(os.sep, "/") for path in paths} - # Emitted paths are built by joining ``web_dir`` (see Args), so each is - # always under it — no need to guard ``relative_to``. - emitted_relative = { - str(Path(path).relative_to(web_dir)).replace(os.sep, "/") - for path in emitted_paths - } - previous = _read_memo_manifest(web_dir) - for relative in previous - emitted_relative: +def _delete_memo_files(web_dir: Path, relative_paths: Iterable[str]) -> None: + """Delete memo files, and the empty mirrored directories they leave. + + Args: + web_dir: The project's ``.web`` directory. + relative_paths: The files to delete, relative to ``web_dir``. + """ + for relative in relative_paths: target = web_dir / relative if target.is_file(): target.unlink() @@ -1135,10 +1185,50 @@ def prune_stale_memo_files(emitted_paths: Iterable[str | Path]) -> None: break parent = parent.parent + +def prune_stale_memo_files(emitted_paths: Iterable[str | Path]) -> None: + """Delete memo files written previously that this compile no longer emits. + + Only paths that appear in the previous manifest are considered for + deletion — never a fresh filesystem walk — so files this code did not + emit are never touched. Empty parent directories created by mirrored + output are removed up to (but not including) the ``.web`` root. + + Args: + emitted_paths: Paths the current compile produced for the memo + pipeline (see :func:`_memo_relative_paths`). + """ + web_dir = get_web_dir() + emitted_relative = _memo_relative_paths(web_dir, emitted_paths) + previous = _read_memo_manifest(web_dir) + _delete_memo_files(web_dir, previous - emitted_relative) if emitted_relative != previous: _write_memo_manifest(web_dir, emitted_relative) +def reconcile_memo_manifest( + emitted_paths: Iterable[str | Path], dropped_paths: Iterable[str | Path] +) -> None: + """Record the memo files an incremental compile wrote and dropped. + + The files the compile did not touch stay recorded, so a later full + compile still prunes them when it stops emitting them. Only files the + previous manifest recorded are deleted. + + Args: + emitted_paths: The memo files this compile wrote. + dropped_paths: The memo files no longer used by any page. + """ + web_dir = get_web_dir() + emitted_relative = _memo_relative_paths(web_dir, emitted_paths) + dropped_relative = _memo_relative_paths(web_dir, dropped_paths) - emitted_relative + previous = _read_memo_manifest(web_dir) + _delete_memo_files(web_dir, dropped_relative & previous) + current = (previous | emitted_relative) - dropped_relative + if current != previous: + _write_memo_manifest(web_dir, current) + + def empty_dir(path: str | Path, keep_files: list[str] | None = None): """Remove all files and folders in a directory except for the keep_files. diff --git a/reflex/reflex.py b/reflex/reflex.py index 08384e1ff56..d0ed1ecffe1 100644 --- a/reflex/reflex.py +++ b/reflex/reflex.py @@ -6,7 +6,7 @@ import signal import sys from collections.abc import Callable -from contextlib import contextmanager, nullcontext +from contextlib import AbstractContextManager, contextmanager, nullcontext from importlib import import_module from importlib.util import find_spec from pathlib import Path @@ -463,6 +463,20 @@ def _run_dev( running_mode.has_backend(), )) + # The compile daemon owns .web regeneration; backend reload workers only + # evaluate pages to register state. They recognise it through its on-disk + # marker (see ``compile_daemon.owns_compilation``): environment set here is + # not inherited by workers a forkserver started earlier. + from reflex_base.environment import environment + + compile_daemon_scope: AbstractContextManager[None] = nullcontext() + if running_mode.has_frontend() and environment.REFLEX_COMPILE_CACHE.get(): + from reflex.utils import compile_daemon + + compile_daemon_scope = compile_daemon.running( + exec.should_prerender_routes(), initial_compiled=True + ) + with exec._frontend_process_lock: exec._frontend_shutting_down = False exec.frontend_process = None @@ -479,6 +493,7 @@ def stop_frontend() -> None: try: with ( + compile_daemon_scope, _frontend_sigterm_handler(frontend_only and sys.platform != "win32"), processes.run_concurrently_context( *commands, interrupt_on_failure=not frontend_only diff --git a/reflex/state.py b/reflex/state.py index bb7fbca3546..79787374d11 100644 --- a/reflex/state.py +++ b/reflex/state.py @@ -10,7 +10,6 @@ import inspect import json import logging -import operator import pickle import re import sys @@ -71,6 +70,7 @@ computed_var, dispatch, is_computed_var, + seeded_unique_variable_names, ) from rich.markup import escape @@ -2008,9 +2008,19 @@ def dict( """ if include_computed: self._mark_dirty_computed_vars() - base_vars = { - prop_name: self.get_value(prop_name) for prop_name in self.base_vars - } + if initial: + # A compile serializes the initial values; a default component's + # generated names must not depend on which states came first (a + # set's order varies by process, and the compile cache merges + # values recorded by different processes). + with seeded_unique_variable_names(self.get_full_name()): + base_vars = { + prop_name: self.get_value(prop_name) for prop_name in self.base_vars + } + else: + base_vars = { + prop_name: self.get_value(prop_name) for prop_name in self.base_vars + } if initial and include_computed: computed_vars = { # Include initial computed vars. @@ -2283,11 +2293,32 @@ def _frontend_schema(root: type[BaseState], context: RegistrationContext) -> str Returns: The hash of the frontend schema. """ + return _hash_frontend_schema(_frontend_schema_entries(root, context)) + + +def _frontend_schema_entries( + root: type[BaseState], context: RegistrationContext +) -> dict[str, list[Any]]: + """Collect the frontend schema entry of each state in the tree below ``root``. + + Each entry depends on its own state only, so entries collected from + different trees combine into the schema of their union. + + Args: + root: The state at the top of the tree. + context: The registration context holding the tree. - def _state_entry(state: type[BaseState]) -> list[Any]: + Returns: + The schema entry of each state, keyed by its full name. + """ + entries = {} + pending = [root] + while pending: + state = pending.pop() fields = state.get_fields() - return [ - state.get_full_name(), + full_name = state.get_full_name() + entries[full_name] = [ + full_name, sorted( (name, _serialize_type(fields[name].type_)) for name in state.base_vars ), @@ -2298,15 +2329,21 @@ def _state_entry(state: type[BaseState]) -> list[Any]: ), sorted(state.event_handlers), ] - - entries = [] - pending = [root] - while pending: - state = pending.pop() - entries.append(_state_entry(state)) pending.extend(context.get_substates(state)) - entries.sort(key=operator.itemgetter(0)) - return md5(json.dumps(entries, separators=(",", ":")).encode("utf-8")).hexdigest() + return entries + + +def _hash_frontend_schema(entries: dict[str, list[Any]]) -> str: + """Hash frontend schema entries keyed by state full name. + + Args: + entries: The entries, as from :func:`_frontend_schema_entries`. + + Returns: + The hash of the frontend schema. + """ + ordered = [entries[name] for name in sorted(entries)] + return md5(json.dumps(ordered, separators=(",", ":")).encode("utf-8")).hexdigest() def _legacy_state_schema(state_cls: type[BaseState]) -> str: diff --git a/reflex/utils/compile_daemon.py b/reflex/utils/compile_daemon.py new file mode 100644 index 00000000000..9c3c6a573a8 --- /dev/null +++ b/reflex/utils/compile_daemon.py @@ -0,0 +1,1583 @@ +"""Warm compile daemon for ``REFLEX_COMPILE_CACHE`` dev hot reloads. + +The daemon imports the app once, then compiles each change in an isolated child. +On POSIX it forks so third-party imports stay warm; otherwise it falls back to a +fresh subprocess. It also owns the watch loop so markdown and external content +dependencies recorded in the compile manifest trigger rebuilds. +""" + +from __future__ import annotations + +import contextlib +import importlib +import importlib.util +import json +import logging +import os +import signal +import subprocess +import sys +import threading +import time +from collections.abc import Callable, Iterable, Iterator +from dataclasses import dataclass, field +from pathlib import Path +from typing import TYPE_CHECKING + +from reflex_base import constants +from reflex_base.environment import environment +from reflex_base.utils import log + +from reflex.utils import path_ops + +if TYPE_CHECKING: + from reflex.compiler import page_cache + +logger = logging.getLogger(__name__) + +#: Fallback polling only (no ``watchfiles``): seconds between poll passes. +_POLL_INTERVAL = 0.5 +#: Milliseconds a burst of saves (e.g. format-on-save touching many files) is +#: allowed to settle before it is reported as one change set. +_DEBOUNCE_MS = 50 +#: How long the OS watcher blocks before yielding control back, so the daemon +#: can notice its parent died even while no file changes. +_WATCH_TIMEOUT_MS = 1000 +#: Fallback polling only: how often to re-walk the tree for added files. +_RESCAN_INTERVAL = 1.0 +#: Watchdog: kill a compile child that runs longer than this (a hung/deadlocked +#: child must never wedge the daemon). Generous enough for a real full compile. +_COMPILE_TIMEOUT = 300.0 +#: Source suffixes edited under the app roots that should trigger a recompile. +_WATCH_SUFFIXES = (".py", ".md", ".mdx") +#: Directories never worth walking while building the watch snapshot, or +#: reacting to changes in. Tested against the parts below the root a path is +#: under, so a project living in a directory of such a name is still watched. +_SKIP_DIRS = frozenset({ + constants.Dirs.WEB, + ".venv", + "venv", + "node_modules", + "__pycache__", + ".git", +}) +#: Exit code of the daemon asking ``reflex run`` to start it afresh: a change +#: to a global input (rxconfig, a lockfile, a Reflex upgrade) cannot be +#: applied to the warm parent, which imported the old version. +_RESTART_EXIT_CODE = 4 + + +def _stop(proc: subprocess.Popen) -> None: + """Stop a daemon process, killing it if it ignores the request. + + Args: + proc: The daemon process. + """ + if proc.poll() is None: + proc.terminate() + try: + proc.wait(timeout=3) + except subprocess.TimeoutExpired: + proc.kill() + proc.wait() + + +@contextlib.contextmanager +def running( + prerender_routes: bool = False, initial_compiled: bool = False +) -> Iterator[None]: + """Run the compile daemon as a subprocess for the duration of the block. + + ``reflex run`` owns the daemon's lifetime: however the block exits (Ctrl-C, + a signal to ``reflex run`` alone, an error), the daemon is stopped, which + releases its lock, so backend workers compile again. The daemon is + its own process so that it stays single-threaded, which keeps its per-edit + ``fork()`` safe, and its environment stays separate from the backend's. If + ``reflex run`` is killed outright, the daemon exits on noticing its parent + is gone (see :func:`_serve`). A daemon that exits asking for a restart + (see :data:`_RESTART_EXIT_CODE`) is started again, on every OS. + + Args: + prerender_routes: Whether the daemon should prerender routes when compiling. + initial_compiled: Whether the parent already produced the initial frontend. + + Yields: + None, while the daemon runs. + """ + env = {**os.environ} + # The daemon DOES compile; ensure the cache is on and the skip flag is off. + env.pop(environment.REFLEX_SKIP_COMPILE.name, None) + env[environment.REFLEX_COMPILE_CACHE.name] = "1" + env[environment.REFLEX_COMPILE_DAEMON.name] = "1" + if prerender_routes: + env["REFLEX_PRERENDER_ROUTES"] = "1" + if initial_compiled: + env["REFLEX_COMPILE_DAEMON_PRECOMPILED"] = "1" + + def spawn() -> subprocess.Popen: + proc = subprocess.Popen( + [sys.executable, "-m", "reflex.utils.compile_daemon"], env=env + ) + # A restarted daemon compiles: the config change it restarts for + # made the frontend the first daemon produced stale. + env.pop("REFLEX_COMPILE_DAEMON_PRECOMPILED", None) + return proc + + lock = threading.Lock() + stopping = False + proc = spawn() + + def supervise() -> None: + nonlocal proc + while True: + current = proc + current.wait() + with lock: + if stopping or current.returncode != _RESTART_EXIT_CODE: + return + logger.info("Compile daemon: restarting after a global config change.") + proc = spawn() + + supervisor = threading.Thread( + target=supervise, name="reflex-compile-daemon", daemon=True + ) + supervisor.start() + try: + yield + finally: + with lock: + stopping = True + current = proc + _stop(current) + supervisor.join(timeout=3) + + +def _reload_roots() -> list[Path]: + """Resolve the directories/files that hold the user's first-party source. + + Returns: + The resolved reload roots (the same set the backend reloader watches). + """ + from reflex.utils import exec as exec_utils + + return [Path(p).resolve() for p in exec_utils.get_reload_paths()] + + +def _under_roots(path: Path, roots: list[Path]) -> bool: + """Whether ``path`` is one of, or lives under, the reload roots. + + Compares path strings: this runs for every loaded module on every hot + reload, and ``root in path.parents`` constructs a Path object per ancestor + per check, which dominated the reset phase of a reload. + + Args: + path: The resolved path to test. + roots: The resolved reload roots. + + Returns: + True if the path is covered by a reload root. + """ + path_str = str(path) + for root in roots: + root_str = str(root) + if path_str == root_str or path_str.startswith( + root_str if root_str.endswith(os.sep) else root_str + os.sep + ): + return True + return False + + +def _iter_source_files(root: Path): + """Yield watchable source files under ``root`` (skipping build/dep dirs). + + Args: + root: A reload root directory (or file). + + Yields: + Resolved source file paths with a watched suffix. + """ + if root.is_file(): + if root.suffix in _WATCH_SUFFIXES: + yield root.resolve() + return + for dirpath, dirnames, filenames in os.walk(root): + dirnames[:] = [ + name + for name in dirnames + if name not in _SKIP_DIRS and not name.startswith(".") + ] + base = Path(dirpath) + for name in filenames: + path = base / name + if path.suffix in _WATCH_SUFFIXES: + yield path.resolve() + + +def _manifest_dependency_files() -> set[Path]: + """Every source dependency the last compile recorded, from the disk manifest. + + Returns: + Resolved dependency file paths (empty without a manifest). + """ + from reflex.compiler import disk_cache + + manifest = disk_cache.load_manifest() + if not manifest: + return set() + return {Path(dep) for dep in disk_cache.manifest_dependency_files(manifest)} + + +def _external_dependency_files(roots: list[Path]) -> set[Path]: + """Recorded dependencies that live *outside* the reload roots. + + Such files (a docs app's markdown in a sibling directory, say) are invisible + to ``get_reload_paths`` and must be watched explicitly so editing them + rebuilds. + + Args: + roots: The resolved reload roots. + + Returns: + Resolved external dependency file paths to watch. + """ + return {p for p in _manifest_dependency_files() if not _under_roots(p, roots)} + + +def _global_files(root: Path) -> set[Path]: + """Genuinely-global files whose change forces a full rebuild / daemon restart. + + Args: + root: The project root. + + Returns: + Resolved paths of rxconfig + lockfiles + package.json that exist. + """ + from reflex.compiler import page_cache + + return { + (root / name).resolve() + for name in page_cache._GLOBAL_FILES + if (root / name).exists() + } + + +def _mtime_ns(path: Path) -> int | None: + """Return the file's modification time in ns, or None if it can't be read. + + Args: + path: The file to stat. + + Returns: + The ``st_mtime_ns`` value, or None on error. + """ + try: + return path.stat().st_mtime_ns + except OSError: + return None + + +@dataclass +class _WatchState: + """What the daemon watches; refreshed after every compile.""" + + roots: list[Path] + root: Path + #: Every dependency the last compile recorded (any suffix), so an edit to a + #: JSON/CSV/etc. file a page reads triggers a rebuild too. + known: set[Path] = field(default_factory=set) + globals_: set[Path] = field(default_factory=set) + #: Last post-compile snapshot, reconciled once the watcher subscribes again. + checkpoint: dict[Path, int] | None = None + + @classmethod + def build(cls, roots: list[Path], root: Path) -> _WatchState: + """Snapshot the current watch inputs from the manifest and roots. + + Args: + roots: The resolved reload roots. + root: The project root. + + Returns: + The populated watch state. + """ + return cls( + roots=roots, + root=root, + known=_manifest_dependency_files(), + globals_=_global_files(root), + ) + + @property + def assets(self) -> Path | None: + """The assets directory, when the project has one.""" + assets = self.root / constants.Dirs.APP_ASSETS + return assets if assets.is_dir() else None + + def targets(self) -> list[Path]: + """Directories/files handed to the OS watcher. + + Returns: + The reload roots, the parents of recorded dependencies outside + them, the project root's global files and the assets directory. + """ + targets: dict[Path, None] = dict.fromkeys([self.root, *self.roots]) + for dep in self.known: + if not _under_roots(dep, self.roots): + targets[dep.parent] = None + for path in self.globals_: + targets[path] = None + if (assets := self.assets) is not None: + targets[assets] = None + return [t for t in targets if t.exists()] + + def accepts(self, path: Path) -> bool: + """Whether a changed path is a compile input worth reacting to. + + Args: + path: The changed path, as reported by the watcher. + + Returns: + True for watched-suffix sources, recorded dependencies, global + inputs and assets; False for build output and unrelated files. + """ + if _in_skipped_dir(path, [self.root, *self.roots]): + return False + if path in self.globals_ or path in self.known: + return True + if (assets := self.assets) is not None and _under_roots(path, [assets]): + return True + return path.suffix in _WATCH_SUFFIXES and ( + path.parent == self.root or _under_roots(path, self.roots) + ) + + def watch_paths(self) -> set[Path]: + """Collect inputs for polling and watch-subscription catch-up snapshots. + + Returns: + The set of paths to stat. + """ + paths: set[Path] = set(self.known) + for r in self.roots: + paths.update(_iter_source_files(r)) + paths.update( + p + for p in self.root.iterdir() + if p.is_file() and p.suffix in _WATCH_SUFFIXES + ) + paths.update(self.globals_) + if (assets := self.assets) is not None: + paths.update(p.resolve() for p in assets.rglob("*") if p.is_file()) + return paths + + +def _snapshot(paths: set[Path]) -> dict[Path, int]: + """Snapshot ``{path: mtime_ns}`` for the given files. + + Args: + paths: The files to stat. + + Returns: + A mapping of file path to its modification time (unreadable files omitted). + """ + return {p: m for p in paths if (m := _mtime_ns(p)) is not None} + + +def _poll_changes(state: _WatchState, alive: Callable[[], bool]) -> set[Path] | None: + """Fallback change source when ``watchfiles`` is unavailable. + + Re-stats the known file set every ``_POLL_INTERVAL`` and re-walks the tree + every ``_RESCAN_INTERVAL`` to discover added files. + + Args: + state: The live watch state. + alive: Returns False when the watcher should stop. + + Returns: + The first non-empty set of changed paths, or None once ``alive`` is + False. + """ + paths = state.watch_paths() + snapshot = _snapshot(paths) + if state.checkpoint is not None and ( + changed := _missed_changes(state.checkpoint, snapshot) + ): + state.checkpoint = snapshot + return changed + state.checkpoint = snapshot + last_rescan = time.monotonic() + while alive(): + time.sleep(_POLL_INTERVAL) + if time.monotonic() - last_rescan >= _RESCAN_INTERVAL: + paths = state.watch_paths() + last_rescan = time.monotonic() + current = _snapshot(paths) + changed = { + p + for p in current.keys() | snapshot.keys() + if current.get(p) != snapshot.get(p) + } + state.checkpoint = current + if changed: + time.sleep(_DEBOUNCE_MS / 1000) # absorb the rest of a burst + return changed + snapshot = current + return None + + +def _next_changes(state: _WatchState, alive: Callable[[], bool]) -> set[Path] | None: + """Block until compile inputs change; return the changed paths. + + Subscribes to filesystem notifications (``watchfiles``: inotify, FSEvents, + ReadDirectoryChangesW) over ``state.targets()``, filtered by + ``state.accepts``. Polling is only a fallback when ``watchfiles`` is not + importable. + + The watcher is torn down before returning: it runs a native thread, and + the compile that follows forks the (otherwise single-threaded) daemon. + Edits made while the watcher is down are caught up by + :func:`_missed_changes` after the compile. + + Args: + state: The live watch state. + alive: Returns False when the watcher should stop. + + Returns: + A non-empty set of changed paths, or None once ``alive`` is False. + """ + try: + import watchfiles + except ImportError: + logger.warning( + "watchfiles is not installed; the compile daemon falls back to polling." + ) + return _poll_changes(state, alive) + + def watch_filter(_change: object, raw_path: str) -> bool: + return state.accepts(Path(raw_path)) + + events = watchfiles.watch( + *state.targets(), + watch_filter=watch_filter, + debounce=_DEBOUNCE_MS, + step=25, + rust_timeout=_WATCH_TIMEOUT_MS, + yield_on_timeout=True, + raise_interrupt=False, + ) + checkpoint = state.checkpoint + try: + for batch in events: + if not alive(): + return None + changed = {Path(raw_path) for _change, raw_path in batch} + if changed or checkpoint is not None: + current = _dependency_snapshot(state) + if state.checkpoint is not None: + changed.update(_missed_changes(state.checkpoint, current)) + state.checkpoint = current + checkpoint = None + if changed: + return changed + finally: + events.close() # stops the native watcher thread before any fork + return None + + +def _in_skipped_dir(path: Path, roots: Iterable[Path]) -> bool: + """Whether ``path`` lies in a build or dependency directory. + + Args: + path: The path to test. + roots: The roots the path may be under; only the parts below the + containing root count. + + Returns: + True when a directory on the way names a skipped directory. + """ + parts = path.parts + for root in roots: + if _under_roots(path, [root]): + parts = parts[len(root.parts) :] + break + return not _SKIP_DIRS.isdisjoint(parts) + + +def _dependency_snapshot(state: _WatchState) -> dict[Path, int]: + """Snapshot all watchable inputs while the OS watcher is stopped. + + Taken once per compile (around the watcher's teardown), never per idle + watcher batch. + + Args: + state: The watch state whose known dependencies to stat. + + Returns: + ``{path: mtime_ns}`` for every readable watched input. + """ + return _snapshot(state.watch_paths()) + + +def _known_snapshot(state: _WatchState, snapshot: dict[Path, int]) -> dict[Path, int]: + """Restrict a snapshot to recorded compile inputs. + + Only their edits are sure to trigger a compile; a backend worker waiting on + any other file could wait for a build that never comes. + + Args: + state: The watch state the snapshot was taken for. + snapshot: ``{path: mtime_ns}`` of the watched inputs. + + Returns: + The entries of ``snapshot`` for ``state.known``. + """ + return {p: m for p, m in snapshot.items() if p in state.known} + + +def _recorded_inputs( + paths: Iterable[Path], +) -> tuple[page_cache.FileValidator, dict[Path, list[str]]] | None: + """Find the manifest entries of input files. + + A file is recorded whole, as function slices (``::``), + or both. Only the entries the manifest still references count: the file + table keeps the ones a rebuild dropped. + + Args: + paths: The input files. + + Returns: + A validator for the manifest's file table and each recorded file's + entries, or None without a manifest. + """ + from reflex.compiler import disk_cache, page_cache + + if (manifest := disk_cache.load_manifest()) is None: + return None + files = manifest["files"] + wanted = {str(path): path for path in paths} + entries: dict[Path, list[str]] = {} + for key in disk_cache.manifest_dependency_entries(manifest): + if key in files and (path := wanted.get(page_cache.dep_file(key))) is not None: + entries.setdefault(path, []).append(key) + return page_cache.FileValidator(files), entries + + +def _unbuilt_changes(state: _WatchState) -> set[Path]: + """Recorded compile inputs that differ from the manifest on disk. + + Args: + state: The current watch state. + + Returns: + The inputs edited since the last build wrote its manifest. + """ + if (recorded := _recorded_inputs(state.known)) is None: + return set() + validator, entries = recorded + return {path for path, keys in entries.items() if not validator.unchanged(keys)} + + +def _built_inputs(paths: set[Path]) -> set[Path]: + """Inputs the manifest records with their current content. + + Args: + paths: The input files. + + Returns: + The paths that have entries, all unchanged. + """ + if (recorded := _recorded_inputs(paths)) is None: + return set() + validator, entries = recorded + return {path for path, keys in entries.items() if validator.unchanged(keys)} + + +def _missed_changes(before: dict[Path, int], after: dict[Path, int]) -> set[Path]: + """Inputs that changed between two dependency snapshots. + + Args: + before: Snapshot taken before the watcher was torn down. + after: Snapshot taken once it is about to be re-armed. + + Returns: + The paths whose mtime differs (or that appeared/disappeared). + """ + return {p for p in before.keys() | after.keys() if before.get(p) != after.get(p)} + + +#: ``__file__`` -> whether it is under the reload roots; the "" key holds the +#: roots the cache was built for. +_first_party_file_cache: dict[str, object] = {} + + +def _first_party_module_names(roots: list[Path]) -> set[str]: + """Names of all loaded modules belonging to the user's first-party packages. + + First-party top-level package names are inferred from the *regular* modules + whose ``__file__`` resolves under a reload root (a plain attribute read, no + namespace-package ``__path__`` recalculation, which is lazy and would break + while ``sys.modules`` is being mutated). Every loaded module sharing one of + those top-level names is then first-party, which captures namespace packages + (they have no ``__file__``) purely by name string. + + Args: + roots: The resolved reload roots. + + Returns: + The set of ``sys.modules`` keys to purge. + """ + roots_key = tuple(str(r) for r in roots) + if _first_party_file_cache.get("") != roots_key: + _first_party_file_cache.clear() + _first_party_file_cache[""] = roots_key + top_level: set[str] = set() + for name, mod in list(sys.modules.items()): + file = getattr(mod, "__file__", None) + if not file: + continue + # Resolving every loaded module's file is thousands of realpath + # syscalls per hot reload; module files never move, so classify each + # ``__file__`` once. The parent warms this before forking. + is_first_party = _first_party_file_cache.get(file) + if is_first_party is None: + try: + is_first_party = _under_roots(Path(file).resolve(), roots) + except OSError: + is_first_party = False + _first_party_file_cache[file] = is_first_party + if is_first_party: + top_level.add(name.partition(".")[0]) + if not top_level: + return set() + return {name for name in sys.modules if name.partition(".")[0] in top_level} + + +def _reset_first_party(roots: list[Path]) -> None: + """Make this interpreter clean w.r.t. first-party code before re-importing. + + Purges the user's first-party modules from ``sys.modules`` and resets the + cross-module registries/caches that would otherwise pin old class objects. + Third-party modules are left imported and warm. + + The state registry is reset surgically, not blanket-cleared: a class body + in a module that survives the purge (framework internals, installed or + workspace packages) never re-executes in this process, so clearing its + registration would lose the state from the app's state tree — and from the + compiled contexts file — permanently. Those registrations are kept; states + from purged modules re-register on re-import, and the runtime-created + states in ``reflex.istate.dynamic`` re-register when the code that + created them runs again: a page re-evaluating, or a purged module + re-importing. One a surviving module created at import is kept too. + + Args: + roots: The resolved reload roots whose modules are first-party. + """ + purged = _first_party_module_names(roots) + for name in purged: + sys.modules.pop(name, None) + # The import-system finder caches were inherited from the warm parent via + # fork and are now stale (they reference the purged modules); without this a + # re-import can resolve a stale spec for a since-changed module. + importlib.invalidate_caches() + + from reflex_base.components.memo import MEMOS + from reflex_base.registry import RegistrationContext + from reflex_base.vars.base import reset_unique_variable_names + + import reflex.istate.dynamic as istate_dynamic + from reflex.compiler import page_cache + from reflex.state import BaseState, _frontend_schema, all_base_state_classes + + # A purged module re-registers its memos when it re-imports; a renamed or + # deleted memo must not survive from the parent (a name collision with the + # old registration, or a memo file the app no longer defines). + for key in [key for key in MEMOS if key[1] in purged]: + del MEMOS[key] + # The re-imported modules draw the names a cold process draws at import. + reset_unique_variable_names() + ctx = RegistrationContext.ensure_context() + page_states = _page_defined_states() + kept = [ + cls + for name, cls in ctx.base_states.items() + if (page_states is None or name not in page_states) + and _survives_purge(cls, page_states is not None) + ] + ctx.base_states.clear() + ctx.base_state_substates.clear() + ctx.event_handlers.clear() + all_base_state_classes.clear() + # Kept classes' per-class caches stay valid; these caches span the tree. + BaseState.get_class_substate.cache_clear() + _frontend_schema.cache_clear() + # Locally-defined states are attached to ``reflex.istate.dynamic`` under + # collision-suffixed names; with the warm parent's attributes in place, + # every re-created state would drift to a new suffix and diverge from the + # names the (cold) backend computes. Reset the module so re-created states + # get their fresh-process names, keeping the names of the kept ones. + kept_dynamic = { + cls.__name__ for cls in kept if cls.__module__ == istate_dynamic.__name__ + } + for attr in [ + name + for name in vars(istate_dynamic) + if not name.startswith("__") and name not in kept_dynamic + ]: + delattr(istate_dynamic, attr) + # Original registration order, so parents always precede their children. + for cls in kept: + ctx._register_base_state(cls) + all_base_state_classes[cls.get_full_name()] = None + ctx.decorated_pages.clear() + object.__setattr__(ctx, "_app", None) + # The import graph caches each module's parsed import edges; a changed file + # may import differently now, so drop it to force a re-parse. Cross-compile + # page reuse comes from the on-disk manifest. The reads attributed to the + # purged modules are re-recorded as they re-execute. + page_cache.clear_import_graph() + page_cache.forget_module_reads() + _reset_model_metadata() + + +def _page_defined_states() -> set[str] | None: + """The states the last compile recorded as defined by a page's evaluation. + + They are re-created when the page re-evaluates, so they never survive a + reset. + + Returns: + The full names of the page-defined states, or None without a + manifest to tell them from the states created at import. + """ + from reflex.compiler import disk_cache + + if (manifest := disk_cache.load_manifest()) is None: + return None + return disk_cache._hit_page_states(manifest, set()) + + +def _survives_purge(cls: type, page_states_known: bool) -> bool: + """Whether a registered state stays registered across the first-party purge. + + A state's code re-registers it when it runs again, which only happens + for a purged module: a module-level state stays iff its module survives + the purge, and a ``reflex.istate.dynamic`` state (a function-local class, + a ``ComponentState`` instance) iff the module that created it at import + survives. A state a page defines is re-created when the page + re-evaluates (see :func:`_page_defined_states`); without a manifest to + tell them apart, every dynamic state is dropped. + + Args: + cls: The registered state class. + page_states_known: Whether a manifest told the page-defined states + apart from the ones created at import. + + Returns: + True when the state is kept. + """ + import reflex.istate.dynamic as istate_dynamic + + module_name = getattr(cls, "__module__", None) + if module_name == istate_dynamic.__name__: + if not page_states_known: + return False + # A local state remembers its module; a ComponentState instance is + # created from a class defined in its creating module. + module_name = cls.__dict__.get("__original_module__") + if module_name is None: + origin = cls.__bases__[0] + module_name = getattr(origin, "__module__", None) + if module_name == istate_dynamic.__name__: + return False + if module_name is None: + return False + module = sys.modules.get(module_name) + return module is not None and getattr(module, "__file__", None) is not None + + +def _reset_model_metadata() -> None: + """Clear the SQLAlchemy/SQLModel table + model registries. + + ``rx.Model`` subclasses (including ones a docs demo ``exec``s) register their + table in a process-global ``MetaData`` that lives in the framework, which the + forked child inherits warm and populated. Re-evaluating a page that defines + such a model would then raise ``Table '...' is already defined``. A fresh + respawn never hits this (empty registry); resetting here restores that + fresh-process contract. Only a loaded model layer holds tables; one a page + imports first starts empty, as in a fresh process. + """ + if (sqlmodel := sys.modules.get("sqlmodel")) is not None: + sqlmodel.SQLModel.metadata.clear() + if (model := sys.modules.get("reflex.model")) is None: + return + # Without the DB extras these are stubs that have no registries. + with contextlib.suppress(AttributeError): + model.Model.metadata.clear() + with contextlib.suppress(AttributeError): + model.ModelRegistry.models.clear() + model.ModelRegistry._metadata = None + + +#: Name of the env var carrying the changed-path hint to a cold compile child. +_CHANGED_ENV = "REFLEX_COMPILE_CHANGED" +#: Where the daemon and its children write bytecode caches (under the +#: backend dir). Timestamp-based ``.pyc`` files are keyed by whole-second +#: mtime and size, so a second same-size edit within a second of the compile +#: that cached the first would import stale bytecode; a private cache lets +#: the daemon drop the entries of the files that changed, with no other +#: process writing them back. +_PYCACHE_DIR = ".pycache_daemon" +#: Name of the env var telling a cold compile child to predict dynamic state names. +_PREDICT_ENV = "REFLEX_COMPILE_PREDICT_DYNAMIC_NAMES" +#: Exit code of a compile child whose dynamic state name prediction failed; the +#: daemon compiles again in a fresh child without prediction. +_RETRY_EXIT_CODE = 3 +#: Lock whose OS lock the daemon holds while a compile is in progress, so a +#: backend reload worker can wait for the fresh stateful-pages marker. The +#: lock files live in Reflex's per-user data directory, keyed by the backend +#: dir they guard (see ``path_ops._file_lock_path``): a ``rm -rf .web`` while +#: the daemon runs must not unlink the lock it holds. +_COMPILE_LOCK_FILE = ".compile_in_progress" +#: Give up waiting on the lock after this long (a wedged daemon must not hang +#: the backend forever; its own watchdog is ``_COMPILE_TIMEOUT``). +_LOCK_WAIT_TIMEOUT = _COMPILE_TIMEOUT + 10 +#: File (under the backend dir) holding ``{path: mtime_ns}`` of the compile +#: inputs the frontend build reflects, so a backend reload worker can tell an +#: edit the daemon has not compiled yet from one it has. +_BUILT_INPUTS_FILE = ".compiled_inputs.json" +#: Briefly retry Windows readers that prevent atomically replacing the record. +_BUILT_INPUTS_WRITE_ATTEMPTS = 3 +_BUILT_INPUTS_RETRY_DELAY = 0.01 +#: Lock whose OS lock the daemon holds for its lifetime. Backend reload +#: workers skip frontend compilation while it is held. A file, not an +#: environment variable: workers may be forked from a ``forkserver`` whose +#: environment predates ``reflex run`` deciding to start the daemon. +_DAEMON_LOCK_FILE = ".compile_daemon" +#: How long a lock holder waits out a worker's momentary probe of the lock. +_LOCK_ACQUIRE_TIMEOUT = 1.0 + + +def _lock_path() -> Path: + """Return the compile-in-progress lock file. + + Returns: + The lock file, in the per-user data directory. + """ + from reflex.utils import prerequisites + + return path_ops._file_lock_path( + prerequisites.get_backend_dir().resolve() / _COMPILE_LOCK_FILE + ) + + +def _daemon_lock_path() -> Path: + """Return the daemon ownership lock file. + + Returns: + The lock file, in the per-user data directory. + """ + from reflex.utils import prerequisites + + return path_ops._file_lock_path( + prerequisites.get_backend_dir().resolve() / _DAEMON_LOCK_FILE + ) + + +def _acquire_lock(path: Path) -> int | None: + """Take the OS lock on ``path``, waiting out other processes' probes of it. + + Args: + path: The lock file. + + Returns: + The descriptor holding the lock, or None when another process keeps it. + """ + deadline = time.monotonic() + _LOCK_ACQUIRE_TIMEOUT + while (descriptor := path_ops.try_lock(path)) is None: + if time.monotonic() > deadline: + return None + time.sleep(0.005) + return descriptor + + +def daemon_active() -> bool: + """Whether a live compile daemon owns frontend compilation. + + Returns: + True while a daemon holds its lock. + """ + return path_ops.is_locked(_daemon_lock_path()) + + +def owns_compilation() -> bool: + """Whether this process should produce ``.web`` itself. + + False for a backend reload worker while a compile daemon owns the build; + the daemon (and its forked compile children) carry + ``REFLEX_COMPILE_DAEMON`` and always own it, and so does every other + process (``reflex compile``, ``reflex export``): they asked for a + compile. Whether a daemon runs is decided from the on-disk lock rather + than inherited environment, which a ``forkserver`` started before + ``reflex run`` chose the daemon would not carry. + + Returns: + True if this process compiles the frontend. + """ + return ( + environment.REFLEX_COMPILE_DAEMON.get() + or not environment.REFLEX_DEV_BACKEND_RELOAD_ACTIVE.get() + or not daemon_active() + ) + + +@contextlib.contextmanager +def _compile_lock(root: Path): + """Hold the compile-in-progress lock while a hot-reload compile runs. + + The backend's reload worker reacts to the same file save as the daemon. + Without this it can restart, read the previous ``stateful_pages`` marker + and register the old state set before the daemon has rewritten it (see + :func:`wait_for_compile`). + + Args: + root: The project root. + + Yields: + None. + """ + descriptor = _acquire_lock(_lock_path()) + try: + yield + finally: + # Closing releases the lock; the file stays so every holder and + # waiter uses the same inode. + if descriptor is not None: + os.close(descriptor) + + +def _built_inputs_path() -> Path: + """The built-inputs record under the backend dir. + + Returns: + The record's path. + """ + from reflex.utils import prerequisites + + return prerequisites.get_backend_dir() / _BUILT_INPUTS_FILE + + +def _record_built_inputs(snapshot: dict[Path, int]) -> None: + """Record the compile inputs the frontend build now reflects. + + Args: + snapshot: ``{path: mtime_ns}`` of the inputs the compile read. + """ + path = _built_inputs_path() + path.parent.mkdir(parents=True, exist_ok=True) + tmp = path.with_name(f"{path.name}.{os.getpid()}.tmp") + try: + tmp.write_text(json.dumps({str(p): m for p, m in snapshot.items()})) + for attempt in range(_BUILT_INPUTS_WRITE_ATTEMPTS): + try: + tmp.replace(path) + except PermissionError: + if ( + not constants.IS_WINDOWS + or attempt == _BUILT_INPUTS_WRITE_ATTEMPTS - 1 + ): + raise + time.sleep(_BUILT_INPUTS_RETRY_DELAY * (attempt + 1)) + else: + return + finally: + tmp.unlink(missing_ok=True) + + +def _built_inputs_current() -> bool: + """Whether every recorded compile input still has its built mtime. + + Returns: + True when the build reflects the inputs (or nothing was recorded). + """ + try: + built = json.loads(_built_inputs_path().read_text()) + except FileNotFoundError: + return True + except (OSError, ValueError): + return False + return all(_mtime_ns(Path(p)) == mtime for p, mtime in built.items()) + + +def wait_for_compile(timeout: float = _LOCK_WAIT_TIMEOUT) -> None: + """Block until the compile daemon has rewritten ``.web`` for the current inputs. + + Called by a backend worker that skips frontend compilation + (``REFLEX_SKIP_COMPILE``) before it reads the stateful-pages marker. Waits + while a compile is in progress, and otherwise while a built input changed + since the last compile (the daemon has not picked the edit up yet). + Returns when the build is current, when the daemon is gone, or after + ``timeout``. + + Args: + timeout: Maximum seconds to wait. + """ + lock = _lock_path() + deadline = time.monotonic() + timeout + waited = False + while time.monotonic() < deadline and daemon_active(): + if not path_ops.is_locked(lock) and _built_inputs_current(): + break + if not waited: + logger.debug( + "Waiting for the compile daemon to finish the current compile." + ) + waited = True + time.sleep(0.02) + + +def _prepare_fork_parent(roots: list[Path]) -> None: + """Make the warm parent a good fork base. + + Stops the framework's background threads (so ``_can_fork`` passes) and warms + the per-process caches every child would otherwise rebuild: the first-party + module classification and the compile cache's component-module lookups. + + Args: + roots: The resolved reload roots. + """ + from reflex.compiler import page_cache + + _quiesce_parent() + _first_party_module_names(roots) + page_cache.warm_module_file_cache() + + +def _use_private_pycache() -> None: + """Point this process's bytecode cache at the daemon's private directory.""" + from reflex.utils import prerequisites + + sys.pycache_prefix = str(prerequisites.get_backend_dir().resolve() / _PYCACHE_DIR) + + +def _forget_bytecode(changed: Iterable[Path]) -> None: + """Drop the cached bytecode of the changed modules before re-importing them. + + Args: + changed: The changed paths. + """ + for path in changed: + if path.suffix == ".py": + with contextlib.suppress(OSError): + Path(importlib.util.cache_from_source(str(path))).unlink( + missing_ok=True + ) + + +def _child_compile( + roots: list[Path], + prerender_routes: bool, + changed: set[Path] | None = None, + predict: bool = False, +) -> None: + """Reset first-party state, re-import the app fresh, and compile incrementally. + + Runs in a forked child (POSIX) or a one-shot subprocess (Windows). Must not + return normally on error; the caller maps the exit code to success/failure. + + Args: + roots: The resolved reload roots. + prerender_routes: Whether to prerender routes during compile. + changed: The paths the watcher reported for this reload, when known. + Lets the incremental rebuild skip validating pages that depend on + none of them. + predict: Whether to predict dynamic state names (see + ``disk_cache.set_dynamic_name_prediction``). + """ + from reflex.compiler import disk_cache + from reflex.utils import prerequisites + + _use_private_pycache() + if changed: + _forget_bytecode(changed) + disk_cache.set_changed_hint(changed) + disk_cache.set_dynamic_name_prediction(predict) + # Timed in three steps so every hot reload reports where it spent its time + # (resetting state vs re-importing first-party code vs compiling). + t0 = time.perf_counter() + _reset_first_party(roots) + t1 = time.perf_counter() + app, _ = prerequisites.get_and_validate_app(reload=False) + t2 = time.perf_counter() + app._compile(prerender_routes=prerender_routes, use_rich=True, trigger="hot_reload") + t3 = time.perf_counter() + logger.info( + f"Hot reload {t3 - t0:.2f}s (reset {t1 - t0:.2f}s, " + f"reimport {t2 - t1:.2f}s, compile {t3 - t2:.2f}s)" + ) + + +def _kill_child(pid: int) -> None: + """Kill and reap a forked compile child. + + Args: + pid: The child's pid. + """ + with contextlib.suppress(OSError): + os.kill(pid, signal.SIGKILL) + with contextlib.suppress(OSError): + os.waitpid(pid, 0) + + +def _await_child(pid: int) -> int: + """Reap a forked compile child, killing it if it exceeds the watchdog timeout. + + The child never outlives the daemon: a daemon stopped while a compile + runs (SIGTERM from ``reflex run``, Ctrl-C) kills the child on its way + out, so the child does not keep the locks and write ``.web`` after the + run ended. + + Args: + pid: The forked child's pid. + + Returns: + The child's exit code; 1 on timeout or a signal (a signal-killed + child, such as Ctrl-C during shutdown, is a quiet failure). + """ + deadline = time.monotonic() + _COMPILE_TIMEOUT + try: + while True: + done, status = os.waitpid(pid, os.WNOHANG) + if done == pid: + code = os.waitstatus_to_exitcode(status) + return code if code >= 0 else 1 + if time.monotonic() > deadline: + _kill_child(pid) + logger.error("Compile child timed out; killed it, keeping last build.") + return 1 + time.sleep(0.02) + except BaseException: + _kill_child(pid) + raise + + +def _die_with_parent() -> None: + """Ask the kernel to kill this forked child when its parent dies (Linux).""" + if sys.platform != "linux": + return + with contextlib.suppress(Exception): + import ctypes + + pr_set_pdeathsig = 1 + ctypes.CDLL("libc.so.6", use_errno=True).prctl(pr_set_pdeathsig, signal.SIGKILL) + + +def _quiesce_parent() -> None: + """Stop the framework's own background threads before forking. + + The telemetry worker (a ``ThreadPoolExecutor`` thread) outlives the + initial compile's ``compile`` event and would otherwise make every later + ``_can_fork`` check fail, silently downgrading each hot reload to a cold + subprocess. Anything the user's app started at import time is out of our + hands and still forces the fallback. + """ + from reflex.utils import telemetry + + telemetry._shutdown_executor() + + +def _os_thread_count() -> int | None: + """Count this process's OS threads, including native ones Python cannot see. + + Returns: + The thread count from ``/proc`` (Linux), else None. + """ + try: + with Path("/proc/self/status").open() as status: + for line in status: + if line.startswith("Threads:"): + return int(line.split()[1]) + except (OSError, ValueError): + return None + return None + + +def _can_fork() -> bool: + """Whether forking is safe right now (POSIX and the process is single-threaded). + + Forking a multi-threaded process and then running Python (not exec) inherits + locks held by threads that don't exist in the child. The + user app, imported warm in the parent, may have started a background thread + at import time, so this is checked per compile. + + Returns: + True if a per-compile ``fork()`` is safe. + """ + if not hasattr(os, "fork"): + return False + _quiesce_parent() + if threading.active_count() == 1: + # A native thread that just finished (the file watcher's, the progress + # bar's) may still be winding down; give it a moment so the fork is + # really single-threaded. + deadline = time.monotonic() + 0.25 + while (count := _os_thread_count()) is not None and count > 1: + if time.monotonic() > deadline: + logger.warning( + f"Compile daemon: {count} OS threads still alive; " + "falling back to a cold subprocess." + ) + return False + time.sleep(0.005) + return True + others = [ + t.name for t in threading.enumerate() if t is not threading.current_thread() + ] + logger.warning( + "Compile daemon: cannot fork a warm child while other threads are " + f"alive ({', '.join(others)}); falling back to a cold subprocess." + ) + return False + + +def _single_threaded() -> bool: + """Whether this process runs one thread, so a ``fork()`` is safe. + + Returns: + True if no other Python or OS thread is alive. + """ + return threading.active_count() == 1 and (_os_thread_count() or 1) == 1 + + +def compiled_in_predicting_child(attempt: Callable[[], bool]) -> bool: + """Run a cold incremental compile in a forked child that predicts state names. + + A failed prediction has already registered states under other names, so + it runs in a copy-on-write child. On any other outcome than a compile, + the caller compiles exactly in this process (which also reports errors). + + Skipped where a caller already chose the mode, such as the daemon's + exact retry, and where there is no manifest to compile from. + + Args: + attempt: Runs the incremental compile; returns True when it compiled. + + Returns: + True when the child compiled. + """ + from reflex.compiler import disk_cache + + if ( + disk_cache._predict_dynamic_names is not None + or not hasattr(os, "fork") + or not _single_threaded() + or disk_cache.load_manifest() is None + ): + return False + pid = os.fork() + if pid == 0: # child + code = 1 + try: + _die_with_parent() + disk_cache.set_dynamic_name_prediction(True) + code = 0 if attempt() else 2 + except disk_cache.DynamicStatePredictionError as exc: + _log_failed_prediction(exc) + except BaseException: # the caller's exact compile reports it + pass + finally: + os._exit(code) + return _await_child(pid) == 0 + + +def _compile_once( + roots: list[Path], prerender_routes: bool, changed: set[Path] | None = None +) -> bool: + """Run one incremental compile in an isolated child; report success. + + The first child predicts dynamic state names, re-evaluating only the + changed pages. When the prediction fails, a fresh child compiles again + without it, re-evaluating every page that defines such states. + + Args: + roots: The resolved reload roots. + prerender_routes: Whether to prerender routes during compile. + changed: The paths the watcher reported for this reload, when known. + + Returns: + True if the child compiled successfully, else False. + """ + code = _run_compile_child(roots, prerender_routes, changed, predict=True) + if code == _RETRY_EXIT_CODE: + code = _run_compile_child(roots, prerender_routes, changed, predict=False) + return code == 0 + + +def _log_failed_prediction(exc: Exception) -> None: + """Report why a compile child gives up its dynamic state name prediction. + + Args: + exc: The failed prediction. + """ + logger.info( + f"Compile cache: {exc}; compiling again, re-evaluating every page that " + "defines dynamic states" + ) + + +def _run_compile_child( + roots: list[Path], + prerender_routes: bool, + changed: set[Path] | None, + *, + predict: bool, +) -> int: + """Compile in an isolated child process and return its exit code. + + Uses a copy-on-write ``fork()`` when safe (warm), else a fresh subprocess + (Windows, or when the warm parent is no longer single-threaded). + + Args: + roots: The resolved reload roots. + prerender_routes: Whether to prerender routes during compile. + changed: The paths the watcher reported for this reload, when known. + predict: Whether the child predicts dynamic state names. + + Returns: + 0 on success, ``_RETRY_EXIT_CODE`` when the prediction failed, else + a failure code. + """ + from reflex.compiler import disk_cache + from reflex.utils import prerequisites, telemetry + + # The child spools its telemetry (the compile event) for the daemon to + # send once the child is gone: a forked child must not start the + # telemetry thread, nor touch macOS's proxy lookup after a fork, and + # ``os._exit`` would drop what it queued. + spool = prerequisites.get_backend_dir().resolve() / f".telemetry.{os.getpid()}" + environment.REFLEX_TELEMETRY_SPOOL.set(spool) + try: + if _can_fork(): + pid = os.fork() + if pid == 0: # child + code = 0 + try: + _die_with_parent() + _child_compile(roots, prerender_routes, changed, predict) + except disk_cache.DynamicStatePredictionError as exc: + _log_failed_prediction(exc) + code = _RETRY_EXIT_CODE + except BaseException: # report any failure, never crash the daemon + logger.exception("Hot reload compile failed.") + code = 1 + finally: + os._exit(code) + return _await_child(pid) + + # No fork (Windows) or unsafe to fork: a fresh (cold) subprocess compiles. + env = {**os.environ} + if changed is not None: + env[_CHANGED_ENV] = "\n".join(sorted(str(p) for p in changed)) + if predict: + env[_PREDICT_ENV] = "1" + try: + proc = subprocess.run( + [sys.executable, "-m", "reflex.utils.compile_daemon", "--once"], + check=False, + timeout=_COMPILE_TIMEOUT, + env=env, + ) + except subprocess.TimeoutExpired: + logger.error("Compile subprocess timed out; keeping the last good build.") + return 1 + return proc.returncode + finally: + environment.REFLEX_TELEMETRY_SPOOL.set(None) + telemetry.send_spooled(spool) + + +def _serve() -> int: + """Run the warm compile daemon: initial compile, then watch-and-recompile. + + Returns: + The process exit code: :data:`_RESTART_EXIT_CODE` to be started + afresh after a global config change, else 0. + """ + from reflex.utils import prerequisites + + root = Path.cwd() + prerender_routes = bool(os.environ.get("REFLEX_PRERENDER_ROUTES")) + roots = _reload_roots() + parent_pid = os.getppid() + _use_private_pycache() + + # Warm import + initial compile (writes .web + the manifest); keeps the app + # and its third-party deps resident for copy-on-write children. A failure + # here (e.g. the app is mid-edit and broken) must NOT kill the daemon; fall + # through to the watch loop so the next edit that fixes it recompiles. + try: + with _compile_lock(root), log.timing(logger, "Compile daemon: initial compile"): + if os.environ.pop("REFLEX_COMPILE_DAEMON_PRECOMPILED", None): + prerequisites.get_app() + else: + prerequisites.get_compiled_app( + reload=False, + prerender_routes=prerender_routes, + use_rich=True, + trigger="initial", + ) + except _Terminated: + raise + except BaseException: # tolerate a broken initial state; keep watching + logger.exception("Initial compile failed; watching for a fix.") + _prepare_fork_parent(roots) + + state = _WatchState.build(roots, root) + state.checkpoint = _dependency_snapshot(state) + logger.info("Compile daemon ready (warm); watching for changes.") + # The first snapshot is the watch baseline; edits made since the last build + # wrote the manifest raise no event, so compile them now. + if not (pending := _unbuilt_changes(state)): + _record_built_inputs(_known_snapshot(state, state.checkpoint)) + + def alive() -> bool: + # Never outlive reflex-run: if our parent died we were reparented. + return os.getppid() == parent_pid + + # Changes a failed compile never applied. The manifest only moves on a + # successful compile, so they stay in the changed hint until one succeeds; + # otherwise the hint's stat-free hit shortcut would reuse their pages. + unapplied: set[Path] = set() + while alive(): + changed = pending or _next_changes(state, alive) + pending = set() + if not changed: + break + changed = changed | unapplied + from reflex.compiler.disk_cache import format_path_list + + logger.info( + f"Compile daemon: change detected in {format_path_list(map(str, changed), root)}" + ) + + # A change to a genuinely-global input (rxconfig/lockfiles, or a reflex + # upgrade) can't be applied to the warm parent (it imported the old + # version); exit so ``reflex run`` starts a daemon that loads the new + # world. (An in-place ``execv`` would leave ``reflex run`` holding a + # dead handle on Windows, where it starts a new process.) + if changed & state.globals_: + logger.info("Global config changed; restarting compile daemon.") + return _RESTART_EXIT_CODE + + before = ( + state.checkpoint + if state.checkpoint is not None + else _dependency_snapshot(state) + ) + state.roots = roots = _reload_roots() + with _compile_lock(root): + ok = _compile_once(roots, prerender_routes, changed) + # Refresh what is watched from the new manifest so a newly + # referenced content file becomes watched. + state = _WatchState.build(roots, root) + state.checkpoint = _dependency_snapshot(state) + # Also after a failure: workers must not wait on an edit that will + # not build; a fix is a newer edit. An input edited during the + # compile keeps its earlier time, so workers wait for its compile. + _record_built_inputs(_known_snapshot(state, {**state.checkpoint, **before})) + unapplied = set() if ok else changed + if not ok: + logger.error("Compile failed; keeping the last good build.") + # Catch up on edits made while the watcher was down for the compile. An + # input the compile started to read was not watched before it; it + # changed unless the compile recorded its current content. + pending = _missed_changes(before, state.checkpoint) + if appeared := (pending - before.keys()) & state.known: + pending -= _built_inputs(appeared) + return 0 + + +def main(argv: list[str] | None = None) -> int: + """Entry point for ``python -m reflex.utils.compile_daemon``. + + Args: + argv: Command-line arguments (defaults to ``sys.argv[1:]``). + + Returns: + Process exit code. + """ + argv = sys.argv[1:] if argv is None else argv + if "--once" in argv: + changed_env = os.environ.get(_CHANGED_ENV) + changed = ( + {Path(line) for line in changed_env.splitlines() if line} + if changed_env is not None + else None + ) + from reflex.compiler import disk_cache + + try: + _child_compile( + _reload_roots(), + bool(os.environ.get("REFLEX_PRERENDER_ROUTES")), + changed, + bool(os.environ.get(_PREDICT_ENV)), + ) + except disk_cache.DynamicStatePredictionError as exc: + _log_failed_prediction(exc) + return _RETRY_EXIT_CODE + except BaseException: # report any failure, never crash + logger.exception("Hot reload compile failed.") + return 1 + return 0 + # Held for the process lifetime, so backend workers defer to this daemon + # exactly while it lives (see ``daemon_active``). Forked compile children + # share the lock; a restarted daemon takes it afresh. + try: + descriptor = _acquire_lock(_daemon_lock_path()) + except OSError as exc: + logger.warning( + f"Compile daemon: this file system cannot lock files ({exc}); " + "backend workers compile instead." + ) + return 0 + if descriptor is None: + logger.warning( + "Another compile daemon owns this project's .web; leaving the " + "compiles to it." + ) + return 0 + # ``reflex run`` stops the daemon with SIGTERM; raising lets the compile + # in progress kill its child on the way out (see ``_await_child``). + signal.signal(signal.SIGTERM, _exit_on_sigterm) + try: + return _serve() + except KeyboardInterrupt: + return 0 # clean shutdown (Ctrl-C); no traceback + except SystemExit: + return 0 + + +class _Terminated(SystemExit): + """The ``SystemExit`` SIGTERM raises, which no compile failure handler keeps.""" + + +def _exit_on_sigterm(signum: int, frame: object) -> None: + """Turn SIGTERM into a ``SystemExit`` that unwinds the daemon. + + Args: + signum: The signal number. + frame: The interrupted frame. + + Raises: + SystemExit: Always. + """ + raise _Terminated(0) + + +if __name__ == "__main__": + # ``python -m`` executes this file as ``__main__``, a second copy of the + # module. Run the importable module instead, so its logger sits in the + # ``reflex`` tree and no state is split between two copies. + from reflex.utils.compile_daemon import main as _main + + sys.exit(_main()) diff --git a/reflex/utils/exec.py b/reflex/utils/exec.py index 5c4a8fb9726..b1685666235 100644 --- a/reflex/utils/exec.py +++ b/reflex/utils/exec.py @@ -730,6 +730,13 @@ def run_uvicorn_backend(host: str, port: int, loglevel: LogLevel): log_level=loglevel.value, reload=True, reload_dirs=list(map(str, get_reload_paths())), + # uvicorn's reload filter defaults to *.py only, so markdown/data edits + # would never trigger a reload; include the content suffixes too. + reload_includes=( + ["*.py", "*.md", "*.mdx"] + if environment.REFLEX_COMPILE_CACHE.get() + else ["*.py"] + ), reload_delay=0.1, ) diff --git a/reflex/utils/frontend_skeleton.py b/reflex/utils/frontend_skeleton.py index 6d7565d7848..a4797517480 100644 --- a/reflex/utils/frontend_skeleton.py +++ b/reflex/utils/frontend_skeleton.py @@ -640,8 +640,10 @@ def _compile_vite_config(config: Config): def initialize_vite_config(): """Render and write in .web the vite.config.js file using Reflex config.""" - vite_config_file_path = get_web_dir() / constants.ReactRouter.VITE_CONFIG_FILE - vite_config_file_path.write_text(_compile_vite_config(get_config())) + write_file( + get_web_dir() / constants.ReactRouter.VITE_CONFIG_FILE, + _compile_vite_config(get_config()), + ) def initialize_bun_config(): diff --git a/reflex/utils/path_ops.py b/reflex/utils/path_ops.py index 0fa4a744316..39c89e7914d 100644 --- a/reflex/utils/path_ops.py +++ b/reflex/utils/path_ops.py @@ -311,6 +311,97 @@ def _lock_descriptor(descriptor: int) -> None: fcntl.flock(descriptor, fcntl.LOCK_EX) +#: The errors with which a non-blocking lock reports another holder. +_LOCK_CONTENTION_ERRNOS = frozenset({ + errno.EACCES, + errno.EAGAIN, + errno.EWOULDBLOCK, + errno.EDEADLK, +}) + + +def _try_lock_descriptor(descriptor: int) -> bool: + """Take the OS's exclusive lock on a lock file without waiting. + + Args: + descriptor: The open lock file descriptor. + + Returns: + Whether the lock was taken. + + Raises: + OSError: When the file system cannot lock files. + """ + try: + if sys.platform == "win32": + msvcrt.locking(descriptor, msvcrt.LK_NBLCK, 1) + else: + fcntl.flock(descriptor, fcntl.LOCK_EX | fcntl.LOCK_NB) + except OSError as error: + if error.errno in _LOCK_CONTENTION_ERRNOS: + return False + raise + return True + + +def try_lock(path: Path) -> int | None: + """Take the OS's exclusive lock on ``path`` without waiting. + + The OS releases the lock when its last descriptor closes, including when + the holder is killed, so a held lock always names a live owner. A pid file + cannot: its process may be gone, or its pid reused by another process. + + Args: + path: The lock file, created if missing. + + Returns: + The descriptor holding the lock (closing it releases the lock), or None + when another holder has it. + + Raises: + OSError: When the file system cannot lock files. + """ + path.parent.mkdir(parents=True, exist_ok=True) + descriptor = os.open(path, os.O_CREAT | os.O_RDWR, 0o644) + try: + if _try_lock_descriptor(descriptor): + return descriptor + except OSError: + os.close(descriptor) + raise + os.close(descriptor) + return None + + +def is_locked(path: Path) -> bool: + """Whether another descriptor holds the OS lock on ``path``. + + Args: + path: The lock file. + + Returns: + True while a holder has the lock; False when it is free, the file + does not exist, or the file system cannot lock files (so nothing can + hold it). + """ + try: + descriptor = os.open(path, os.O_RDWR) + except FileNotFoundError: + return False + try: + try: + if not _try_lock_descriptor(descriptor): + return True + except OSError: + return False + if sys.platform == "win32": + msvcrt.locking(descriptor, msvcrt.LK_UNLCK, 1) + return False + finally: + # Closing releases a POSIX probe's lock. + os.close(descriptor) + + @contextlib.contextmanager def file_lock(path: str | Path) -> Iterator[Path]: """Lock a file or directory across processes, or proceed unlocked if unavailable. diff --git a/reflex/utils/prerequisites.py b/reflex/utils/prerequisites.py index acfa8ac6062..d2215bcf791 100644 --- a/reflex/utils/prerequisites.py +++ b/reflex/utils/prerequisites.py @@ -259,23 +259,37 @@ def get_app(reload: bool = False) -> ModuleType: module = config.module sys.path.insert(0, getcwd()) # noqa: PTH109 - app = ( - __import__(module, fromlist=(constants.CompileVars.APP,)) - if not config.app_module - else config.app_module - ) - if reload: - from reflex.state import reload_state_module + from reflex.utils import compile_daemon + + if ( + environment.REFLEX_COMPILE_CACHE.get() + and not environment.REFLEX_SKIP_COMPILE.get() + and compile_daemon.owns_compilation() + ): + from reflex.compiler import page_cache + + page_cache.enable_read_tracking() + recorder = page_cache.record_app_import() + else: + recorder = contextlib.nullcontext() + with recorder: + app = ( + __import__(module, fromlist=(constants.CompileVars.APP,)) + if not config.app_module + else config.app_module + ) + if reload: + from reflex.state import reload_state_module - # Reset rx.State subclasses to avoid conflict when reloading. - reload_state_module(module=module) + # Reset rx.State subclasses to avoid conflict when reloading. + reload_state_module(module=module) - reg_ctx = RegistrationContext.ensure_context() - reg_ctx.decorated_pages.clear() - object.__setattr__(reg_ctx, "_app", None) + reg_ctx = RegistrationContext.ensure_context() + reg_ctx.decorated_pages.clear() + object.__setattr__(reg_ctx, "_app", None) - # Reload the app module. - importlib.reload(app) + # Reload the app module. + importlib.reload(app) except Exception as ex: telemetry.send_error(ex, context="frontend") raise diff --git a/reflex/utils/telemetry.py b/reflex/utils/telemetry.py index e669935681c..ecd2ea42c18 100644 --- a/reflex/utils/telemetry.py +++ b/reflex/utils/telemetry.py @@ -642,6 +642,42 @@ def _flush(timeout: float | None = None) -> bool: return True +def _spool_event( + spool: Path, event: str, properties: dict[str, Any] | None, kwargs: dict[str, Any] +) -> None: + """Append an event to a spool file for another process to send. + + Args: + spool: The spool file. + event: The event name. + properties: The event's structured payload. + kwargs: Additional event data. + """ + with suppress(Exception), spool.open("a", encoding="utf-8") as fh: + fh.write( + json.dumps({"event": event, "properties": properties, "kwargs": kwargs}) + + "\n" + ) + + +def send_spooled(spool: Path) -> None: + """Send the events a compile child spooled, then remove the spool. + + Args: + spool: The spool file (see ``REFLEX_TELEMETRY_SPOOL``). + """ + try: + lines = spool.read_text(encoding="utf-8").splitlines() + except OSError: + return + with suppress(OSError): + spool.unlink(missing_ok=True) + for line in lines: + with suppress(Exception): + entry = json.loads(line) + send(entry["event"], properties=entry["properties"], **entry["kwargs"]) + + _legacy_alias_attempted = False @@ -723,6 +759,11 @@ def send( with suppress(Exception): if telemetry_enabled is None: telemetry_enabled = get_config().telemetry_enabled + if not telemetry_enabled: + return + if (spool := environment.REFLEX_TELEMETRY_SPOOL.get()) is not None: + _spool_event(spool, event, properties, kwargs) + return # Only spin up the worker pool when there is actually something to send. if telemetry_enabled: _submit( diff --git a/reflex/utils/telemetry_accounting.py b/reflex/utils/telemetry_accounting.py index 45d067183c7..a291ac98177 100644 --- a/reflex/utils/telemetry_accounting.py +++ b/reflex/utils/telemetry_accounting.py @@ -98,8 +98,12 @@ def _collect_compile_event_payload( return { "plugins_enabled": [p.__class__.__name__ for p in config.plugins], "plugins_disabled": [p.__name__ for p in config.disable_plugins], - "pages_count": len(app._pages), - "component_counts": _count_components(app._pages.values()), + "pages_count": len(app._unevaluated_pages.keys() | app._pages.keys()), + "component_counts": ( + app._cached_component_counts + if app._cached_component_counts is not None + else _count_components(app._pages.values()) + ), "states": [_collect_state_stats(s) for s in user_states], "features_used": _collect_features_used(app, config, user_states), "duration_ms": ctx.elapsed_ms(), diff --git a/tests/integration/tests_playwright/test_memo.py b/tests/integration/tests_playwright/test_memo.py index 824eb2dda3b..34ce71d76ea 100644 --- a/tests/integration/tests_playwright/test_memo.py +++ b/tests/integration/tests_playwright/test_memo.py @@ -116,7 +116,8 @@ def derived_count(divisor: rx.Var[int]) -> rx.Component: """ return rx.text( MemoState.numbers.filter(lambda number: number % divisor == 0).length(), - id="derived-count", + # No ``id``: its ref write keeps React Compiler from compiling the memo. + class_name="derived-count", ) def index() -> rx.Component: @@ -364,7 +365,7 @@ def test_memo_derived_array_updates(memo_app: AppHarness, page: Page) -> None: page: Playwright page. """ _load_page(page, memo_app) - count = page.locator("#derived-count") + count = page.locator(".derived-count") expect(count).to_have_text("2") page.locator("#memo-input").fill("before replacement") diff --git a/tests/units/compiler/test_compiler.py b/tests/units/compiler/test_compiler.py index dc80f257cb8..9be4fa14168 100644 --- a/tests/units/compiler/test_compiler.py +++ b/tests/units/compiler/test_compiler.py @@ -167,6 +167,46 @@ def test_compile_imports(import_dict: ParsedImportDict, test_dicts: list[dict]): ) +def test_compile_stylesheets_skips_unchanged_copy( + tmp_path: Path, mocker: MockerFixture +): + """An unchanged asset stylesheet is not rewritten into .web on recompile. + + Vite watches .web/styles, so a rewrite with identical content still fires + an HMR update on every dev reload. + + Args: + tmp_path: The test directory. + mocker: Pytest mocker object. + """ + project = tmp_path / "test_project" + project.mkdir() + assets_dir = project / "assets" + assets_dir.mkdir() + (assets_dir / "style.css").write_text("button { color: blue }") + + mocker.patch("reflex.compiler.compiler.Path.cwd", return_value=project) + mocker.patch( + "reflex.compiler.compiler.get_web_dir", + return_value=project / constants.Dirs.WEB, + ) + mocker.patch( + "reflex.compiler.utils.get_web_dir", return_value=project / constants.Dirs.WEB + ) + + compiler.compile_root_stylesheet(["/style.css"]) + target = project / constants.Dirs.WEB / "styles" / "style.css" + stat_before = target.stat() + + compiler.compile_root_stylesheet(["/style.css"]) + stat_after = target.stat() + assert stat_after.st_mtime_ns == stat_before.st_mtime_ns + + (assets_dir / "style.css").write_text("button { color: red }") + compiler.compile_root_stylesheet(["/style.css"]) + assert target.read_text() == "button { color: red }" + + def test_compile_stylesheets(tmp_path: Path, mocker: MockerFixture): """Test that stylesheets compile correctly. @@ -842,9 +882,8 @@ def test_add_meta_accepts_dynamic_description(): class PageState(rx.State): description: str = "Dynamic description" - page = rx.box() - utils.add_meta( - page, + page = utils.add_meta( + rx.box(), title="title", image="", meta=(), @@ -858,9 +897,8 @@ class PageState(rx.State): def test_add_meta_drops_empty_description(): """An empty-string description keeps producing no description tag.""" - page = rx.box() - utils.add_meta( - page, + page = utils.add_meta( + rx.box(), title="title", image="", meta=(), @@ -1869,6 +1907,221 @@ def test_no_ssr_dynamic_import_names_the_client_side_wrapper(): assert Plotly.create()._get_dynamic_imports().endswith(', "Plot")') +@pytest.mark.parametrize("fresh_page", [False, True]) +def test_incremental_compile_runs_plugin_outputs( + tmp_path, monkeypatch, mocker, fresh_page +): + """Cache hits still refresh plugin assets and scheduled output tasks.""" + from reflex.compiler import disk_cache + + class OutputPlugin(rx.plugins.Plugin): + """Contribute a task, static asset and non-idempotent modifier.""" + + def pre_compile(self, **context): + """Schedule outputs through the normal plugin lifecycle. + + Args: + context: Compiler capabilities. + """ + context["add_save_task"](lambda: ("task.txt", "task")) + context["add_modify_task"]("page.txt", lambda text: text + "!") + + def get_static_assets(self, **context): + """Return a plugin asset. + + Args: + context: Compiler capabilities. + + Returns: + The static asset. + """ + return [(Path("asset.txt"), "asset")] + + monkeypatch.setenv("REFLEX_COMPILE_CACHE", "1") + monkeypatch.setenv("REFLEX_WEB_WORKDIR", str(tmp_path)) + mocker.patch("reflex.app.reload_config") + app = rx.App() + app.add_page(lambda: rx.el.div("page"), route="index") + plugin = OutputPlugin() + config = rx.Config(app_name="testing", plugins=[plugin]) + mocker.patch.object(compiler, "get_config", return_value=config) + mocker.patch.object(app, "_should_compile", return_value=True) + + def rebuild(*args, outputs, **kwargs): + """Stage a regenerated page when simulating a cache miss. + + Args: + args: Application arguments. + outputs: Compiler output mapping. + kwargs: Rebuild options. + + Returns: + Whether the incremental rebuild succeeded. + """ + if fresh_page: + outputs[tmp_path / "page.txt"] = "new" + return True + + mocker.patch.object(disk_cache, "try_incremental_rebuild", side_effect=rebuild) + mocker.patch.object( + disk_cache, + "load_manifest", + return_value={"plugin_sources": {str(tmp_path / "page.txt"): "base"}}, + ) + # The attempt runs in this process, where the outputs can be observed. + from reflex.utils import compile_daemon + + mocker.patch.object( + compile_daemon, "compiled_in_predicting_child", return_value=False + ) + (tmp_path / "page.txt").write_text("base!") + assert compiler.compile_app(app, use_rich=False) + assert (tmp_path / "asset.txt").read_text() == "asset" + assert (tmp_path / "task.txt").read_text() == "task" + assert (tmp_path / "page.txt").read_text() == ("new!" if fresh_page else "base!") + + +@pytest.mark.parametrize("write_fails", [False, True]) +def test_cached_compile_writes_the_manifest_last( + tmp_path, monkeypatch, mocker, write_fails +): + """The refreshed manifest reaches disk only after every other output. + + A manifest on disk vouches for the files it describes; it must never be + there while they are not (a kill between the two leaves pages stale). + """ + from reflex.compiler import disk_cache + + monkeypatch.setenv("REFLEX_COMPILE_CACHE", "1") + monkeypatch.setenv("REFLEX_WEB_WORKDIR", str(tmp_path)) + mocker.patch("reflex.app.reload_config") + app = rx.App() + app.add_page(lambda: rx.el.div("page"), route="index") + mocker.patch.object( + compiler, "get_config", return_value=rx.Config(app_name="testing", plugins=[]) + ) + mocker.patch.object(app, "_should_compile", return_value=True) + # The attempt runs in this process, where the writes can be observed. + from reflex.utils import compile_daemon + + mocker.patch.object( + compile_daemon, "compiled_in_predicting_child", return_value=False + ) + manifest_path = disk_cache._manifest_path().absolute() + manifest_path.write_text(json.dumps({"schema": 0, "plugin_sources": {}})) + page = tmp_path / "page.txt" + written: list[Path] = [] + original_write = compiler.utils.write_file + + def write_file(path, code): + """Write an output, failing every write after the page when asked. + + Args: + path: The output path. + code: The output content. + + Raises: + OSError: When a write after the page is made to fail. + """ + if write_fails and page in written: + msg = "disk full" + raise OSError(msg) + written.append(Path(path)) + original_write(path, code) + + mocker.patch.object(compiler.utils, "write_file", side_effect=write_file) + original_manifest_write = disk_cache._write + + def write_manifest(manifest): + """Record when the manifest is written. + + Args: + manifest: The manifest to write. + """ + written.append(manifest_path) + original_manifest_write(manifest) + + mocker.patch.object(disk_cache, "_write", side_effect=write_manifest) + + def rebuild(*args, outputs, **kwargs): + """Stage a page and the refreshed manifest, manifest first. + + Args: + args: Application arguments. + outputs: Compiler output mapping. + kwargs: Rebuild options. + + Returns: + Whether the incremental rebuild succeeded. + """ + outputs[manifest_path] = json.dumps({"schema": 1, "plugin_sources": {}}) + outputs[page] = "new" + return True + + mocker.patch.object(disk_cache, "try_incremental_rebuild", side_effect=rebuild) + if write_fails: + with pytest.raises(OSError, match="disk full"): + compiler.compile_app(app, use_rich=False) + assert not manifest_path.exists() + return + assert compiler.compile_app(app, use_rich=False) + assert json.loads(manifest_path.read_text())["schema"] == 1 + assert written[0] == page + assert len(written) > 2 + assert written.index(manifest_path) == len(written) - 1 + + +def test_preloaded_app_skips_incremental_reuse(monkeypatch, mocker): + """Unobserved import-time reads require a full compile for preloaded apps.""" + from reflex.compiler import disk_cache + + mocker.patch("reflex.app.reload_config") + app = rx.App() + config = rx.Config(app_name="testing", app_module_import="testing") + mocker.patch.object(compiler, "get_config", return_value=config) + mocker.patch.object(app, "_should_compile", return_value=True) + reuse = mocker.patch.object( + disk_cache, "try_incremental_rebuild", return_value=True + ) + monkeypatch.setenv("REFLEX_COMPILE_CACHE", "1") + + class FullCompileReached(Exception): + """Stop before full-compile side effects.""" + + mocker.patch.object( + compiler.console, "PoorProgress", side_effect=FullCompileReached + ) + with pytest.raises(FullCompileReached): + compiler.compile_app(app, use_rich=False) + reuse.assert_not_called() + + +def test_serving_app_compiles_every_page(monkeypatch, mocker): + """A process that serves the app registers every page's states. + + The incremental cache evaluates only changed pages, so it is not used. + """ + from reflex.compiler import disk_cache + + mocker.patch("reflex.app.reload_config") + app = rx.App() + mocker.patch.object(app, "_should_compile", return_value=True) + reuse = mocker.patch.object( + disk_cache, "try_incremental_rebuild", return_value=True + ) + monkeypatch.setenv("REFLEX_COMPILE_CACHE", "1") + + class FullCompileReached(Exception): + """Stop before full-compile side effects.""" + + mocker.patch.object( + compiler.console, "PoorProgress", side_effect=FullCompileReached + ) + with pytest.raises(FullCompileReached): + compiler.compile_app(app, use_rich=False, serving=True) + reuse.assert_not_called() + + def test_compile_app_drops_event_caches_from_earlier_compiles( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, mocker: MockerFixture ): @@ -1954,3 +2207,66 @@ def test_compile_app_resolves_show_built_with_reflex( assert get_user_tier.called is ( compile_context == constants.CompileContext.DEPLOY and configured is None ) + + +@pytest.mark.parametrize("plugins", [[], [rx.plugins.SitemapPlugin()]]) +def test_compile_app_progress_ends_at_its_total( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, mocker: MockerFixture, plugins +): + """The progress bar finishes exactly at the total it announced. + + Args: + tmp_path: Directory for compiler output. + monkeypatch: Fixture for changing the app directory. + mocker: Fixture for configuring the test app. + plugins: The configured plugins. + """ + monkeypatch.chdir(tmp_path) + poor_progress = compiler.console.PoorProgress + bars: list[compiler.console.PoorProgress] = [] + + def recording_progress() -> compiler.console.PoorProgress: + bar = poor_progress() + bars.append(bar) + return bar + + mocker.patch.object(compiler.console, "PoorProgress", recording_progress) + with RegistrationContext(): + config = rx.Config(app_name="progress_test", plugins=plugins) + mocker.patch("reflex_base.config._get_config", return_value=config) + app = rx.App() + app.add_page(lambda: rx.el.div("hello"), route="/") + compiler.compile_app(app, dry_run=True, use_rich=False) + [bar] = bars + [task] = bar.tasks.values() + assert task["current"] == task["total"] + + +def test_serving_compile_discards_the_manifest(tmp_path, monkeypatch, mocker): + """A compile that bypasses the cache leaves no manifest describing its output. + + The manifest an earlier cached compile wrote would otherwise validate + files this compile overwrote. + """ + from reflex.compiler import disk_cache + + monkeypatch.setenv("REFLEX_COMPILE_CACHE", "1") + monkeypatch.setenv("REFLEX_WEB_WORKDIR", str(tmp_path)) + mocker.patch("reflex.app.reload_config") + # A fresh registry: states other test modules left behind would otherwise + # join the compiled state tree. + with RegistrationContext(): + app = rx.App() + app.add_page(lambda: rx.el.div("page"), route="index") + mocker.patch.object( + compiler, + "get_config", + return_value=rx.Config(app_name="testing", plugins=[]), + ) + mocker.patch.object(app, "_should_compile", return_value=True) + mocker.patch.object(app, "_get_frontend_packages") + mocker.patch.object(app, "_add_optional_endpoints") + manifest = disk_cache._manifest_path() + manifest.write_text("{}") + assert compiler.compile_app(app, use_rich=False, serving=True) + assert not manifest.exists() diff --git a/tests/units/compiler/test_compiler_utils.py b/tests/units/compiler/test_compiler_utils.py index b92647a9677..584e90856cf 100644 --- a/tests/units/compiler/test_compiler_utils.py +++ b/tests/units/compiler/test_compiler_utils.py @@ -213,6 +213,34 @@ async def test_compile_state_resolves_async_computed_vars_with_running_event_loo assert values[f"async_value{FIELD_MARKER}"] == "resolved" +def test_write_file_creates_and_updates(tmp_path): + path = tmp_path / "sub" / "page.jsx" + write_file(path, "v1") + assert path.read_text() == "v1" + write_file(path, "v2") + assert path.read_text() == "v2" + + +def test_write_file_atomic_leaves_no_temp_files(tmp_path): + path = tmp_path / "page.jsx" + write_file(path, "content") + # The temp file used for the atomic replace must not linger. + assert [p.name for p in tmp_path.iterdir()] == ["page.jsx"] + + +def test_write_file_skips_byte_identical_write(tmp_path): + """An identical write must not touch the file (so vite isn't told to HMR).""" + path = tmp_path / "page.jsx" + write_file(path, "same") + before = path.stat().st_mtime_ns + import os + + os.utime(path, ns=(before + 1_000_000_000, before + 1_000_000_000)) + bumped = path.stat().st_mtime_ns + write_file(path, "same") # identical -> no rewrite + assert path.stat().st_mtime_ns == bumped + + def test_compile_state_hashes_dict_with_mixed_key_types( forked_registration_context: RegistrationContext, ): @@ -301,3 +329,46 @@ def test_document_root_controls_preserve_no_id_and_page_refs(): assert not document_root._get_all_hooks() assert page_script._get_all_hooks() + + +def test_compile_state_default_component_names_do_not_depend_on_state_order(): + """A default component's generated names are a function of its state. + + Substates are a set, whose order varies by process; the compile cache + merges initial values recorded by different processes, which must agree. + """ + from reflex_base.vars.base import reset_unique_variable_names + + with RegistrationContext(): + + class _OrderRoot(rx.State): + pass + + class _OrderOne(_OrderRoot): + comp: rx.Field[rx.Component] = rx.field( + default_factory=lambda: rx.auto_scroll() + ) + + class _OrderTwo(_OrderRoot): + comp: rx.Field[rx.Component] = rx.field( + default_factory=lambda: rx.auto_scroll() + ) + + def serialized(substates: list[type[rx.State]]) -> str: + reset_unique_variable_names() + _OrderRoot.get_substates = classmethod( # type: ignore[method-assign] + lambda cls: substates if cls is _OrderRoot else set() + ) + try: + return utils._serialize_initial_state(compile_state(_OrderRoot)) + finally: + del _OrderRoot.get_substates + + first = serialized([_OrderOne, _OrderTwo]) + second = serialized([_OrderTwo, _OrderOne]) + assert first == second + one, two = ( + first.index(_OrderOne.get_full_name()), + first.index(_OrderTwo.get_full_name()), + ) + assert one != two diff --git a/tests/units/compiler/test_disk_cache.py b/tests/units/compiler/test_disk_cache.py new file mode 100644 index 00000000000..4a059608628 --- /dev/null +++ b/tests/units/compiler/test_disk_cache.py @@ -0,0 +1,2994 @@ +"""Tests for the experimental disk-persisted incremental compile cache.""" + +import dataclasses +import datetime +import itertools +import json +import os +import sys +from collections.abc import Callable, Iterable, Sequence +from pathlib import Path +from types import ModuleType, SimpleNamespace +from typing import Any, cast + +import pytest +from reflex_base import constants +from reflex_base.components.component import Component +from reflex_base.environment import environment +from reflex_base.plugins import CompileContext, CompilerHooks, PageContext +from reflex_base.registry import RegistrationContext +from reflex_base.utils.format import to_snake_case +from reflex_base.utils.imports import ImportVar + +import reflex as rx +from reflex.compiler import disk_cache, page_cache +from reflex.compiler.plugins import default_page_plugins +from reflex.compiler.utils import ContextsSnapshot +from reflex.state import BaseState, _frontend_schema_entries + + +@dataclasses.dataclass(slots=True) +class _FakePage: + route: str + component: Callable[[], Component] + title: Any = None + description: Any = None + image: str = "" + meta: tuple[dict[str, Any], ...] = () + _source_module: str | None = None + + +def _use_tmp_web_dir(tmp_path, monkeypatch): + """Point every ``get_web_dir`` binding (module attr + env) at a tmp web dir. + + Args: + tmp_path: The test's tmp directory. + monkeypatch: The pytest monkeypatch fixture. + + Returns: + The created web directory path. + """ + web = tmp_path / ".web" + web.mkdir() + monkeypatch.setattr(disk_cache.prerequisites, "get_web_dir", lambda: web) + monkeypatch.setenv("REFLEX_WEB_WORKDIR", str(web)) + return web + + +def _footer() -> Component: + return rx.el.footer(rx.el.span("© Reflex"), class_name="footer") + + +def _page_a() -> Component: + return rx.el.div(rx.el.h1("Page A"), _footer()) + + +def _page_b() -> Component: + return rx.el.div(rx.el.h1("Page B"), _footer()) + + +def _page_b_edited() -> Component: + return rx.el.div(rx.el.h1("Page B (edited)"), rx.el.p("new body"), _footer()) + + +def _page_c() -> Component: + return rx.el.div(rx.el.h1("Page C"), _footer()) + + +def _compile( + pages: Sequence[Any], app: Any = None, plugins: Sequence[Any] = () +) -> CompileContext: + from reflex.compiler import compiler + + # Reset like a full compile does before evaluating any page. + compiler._reset_compile_registries(plugins) + ctx = CompileContext( + app=app, + pages=pages, + hooks=CompilerHooks(plugins=default_page_plugins(plugins=plugins)), + ) + with ctx: + ctx.compile() + return ctx + + +def _unregister_state(cls: type[BaseState]) -> None: + """Drop a state class from the registries, like the daemon registry reset. + + Lets a page evaluation re-define the same-named class in this process, + mirroring what happens in the fork child between hot reloads. + + Args: + cls: The state class to unregister. + """ + from reflex_base.registry import RegistrationContext + + import reflex.istate.dynamic as istate_dynamic + from reflex.state import all_base_state_classes + + ctx = RegistrationContext.ensure_context() + full_name = cls.get_full_name() + ctx.base_states.pop(full_name, None) + parent = cls.get_parent_state() + if parent is not None: + ctx.base_state_substates.get(parent.get_full_name(), set()).discard(cls) + all_base_state_classes.pop(full_name, None) + if getattr(istate_dynamic, cls.__name__, None) is cls: + delattr(istate_dynamic, cls.__name__) + + +@pytest.mark.parametrize("edited_module", ["state", "helper"]) +def test_imported_state_edit_rebuilds_the_contexts( + tmp_path, monkeypatch, edited_module +): + """A module-level state edit rebuilds the contexts, not every page. + + The edited state's module (or a helper it imports) is an input of the + contexts file; the pages that read the state recompile through their own + recorded dependencies. + """ + _use_tmp_web_dir(tmp_path, monkeypatch) + source = tmp_path / "state_view.py" + helper_source = tmp_path / "state_helper.py" + helper_source.write_text("DEFAULT = 'old'\n") + helper = ModuleType("cache_review_state_helper") + helper.__file__ = str(helper_source) + helper.__dict__["DEFAULT"] = "old" + monkeypatch.setitem(sys.modules, helper.__name__, helper) + code = ( + "import reflex as rx\n" + "from cache_review_state_helper import DEFAULT\n" + "class ImportedState(rx.State):\n" + " value: str = DEFAULT\n" + "def index():\n" + " return rx.el.div(ImportedState.value)\n" + ) + + def load(source_code: str) -> ModuleType: + """Execute the state module, as importing the app does. + + Args: + source_code: The module source. + + Returns: + The module. + """ + source.write_text(source_code) + module = ModuleType("cache_review_state_view") + module.__file__ = str(source) + monkeypatch.setitem(sys.modules, module.__name__, module) + exec(compile(source_code, str(source), "exec"), module.__dict__) + return module + + module = load(code) + try: + app = rx.App() + app.add_page(module.index, route="/state") + app.add_page(_page_a, route="/a") + pages = list(app._unevaluated_pages.values()) + ctx = _compile(pages, app) + assert not ctx.stateful_routes + _stub_externals(app, monkeypatch) + disk_cache.write_manifest(ctx, pages, ctx.all_imports, root=tmp_path) + # The edit, then a fresh import of the app, as the daemon child does. + _unregister_state(module.ImportedState) + if edited_module == "state": + module = load(code.replace("= DEFAULT", "= 'new default'")) + else: + helper_source.write_text("DEFAULT = 'new default'\n") + helper.__dict__["DEFAULT"] = "new default" + module = load(code) + app._unevaluated_pages["state"] = dataclasses.replace( + app._unevaluated_pages["state"], component=module.index + ) + captured = _capture_contexts(monkeypatch) + assert ( + disk_cache.try_incremental_rebuild( + app, compiler_plugins=[], prerender_routes=False, root=tmp_path + ) + is True + ) + [(initial, _, _)] = captured + [values] = [ + values + for name, values in initial.items() + if name.endswith("imported_state") + ] + assert "new default" in json.dumps(values) + finally: + _unregister_state(module.ImportedState) + + +def test_new_import_time_state_rebuilds_the_contexts(tmp_path, monkeypatch): + """A state a recompiled page's module newly imports reaches the contexts. + + Importing the app registers it before any page evaluates, so no page + defines it; only comparing the import-time states sees it. + """ + web = _use_tmp_web_dir(tmp_path, monkeypatch) + app = rx.App() + app.add_page(_page_a, route="/a") + pages = list(app._unevaluated_pages.values()) + ctx = _compile(pages, app) + _stub_externals(app, monkeypatch) + disk_cache.write_manifest(ctx, pages, ctx.all_imports, root=tmp_path) + manifest_path = web / disk_cache._MANIFEST_FILE + manifest = json.loads(manifest_path.read_text()) + _stale_dep(manifest, "a", str(tmp_path / "view.py")) + manifest_path.write_text(json.dumps(manifest)) + # The edited page module now imports a module defining a new state. + new_state: Any = type( + "_ImportTimeState", + (rx.State,), + {"__annotations__": {"value": int}, "value": 3, "__module__": "fp_mod_x"}, + ) + captured = _capture_contexts(monkeypatch) + try: + assert ( + disk_cache.try_incremental_rebuild( + app, compiler_plugins=[], prerender_routes=False, root=tmp_path + ) + is True + ) + finally: + _unregister_state(new_state) + [(initial, _, _)] = captured + assert new_state.get_full_name() in initial + + +def test_imports_round_trip(): + imports = { + "react": [ImportVar("useEffect"), ImportVar("Fragment", is_default=False)], + "@emotion/react": [ImportVar("jsx", alias="j", install=False)], + } + restored = disk_cache._deserialize_imports(disk_cache._serialize_imports(imports)) + assert restored == imports + + +@pytest.mark.parametrize("kind", ["date", "pydantic"]) +def test_manifest_serializes_state_defaults_like_contexts(tmp_path, monkeypatch, kind): + """Persist state defaults that require Reflex's frontend serializer.""" + from reflex_base.utils.format import json_dumps + + _use_tmp_web_dir(tmp_path, monkeypatch) + value = ( + datetime.date(2026, 9, 5) + if kind == "date" + else pytest.importorskip("pydantic").create_model( + "AppSettings", theme=(str, "light") + )() + ) + state_slice = { + "initial_state": {"state": {"value": value}}, + "client_storage": {}, + } + manifest = _manifest({"/date": {"state_slice": state_slice}}) + disk_cache._write(manifest) + loaded = disk_cache.load_manifest() + assert loaded is not None + assert loaded["pages"]["/date"]["state_slice"] == json.loads( + json_dumps(state_slice) + ) + + +def test_manifest_write_failure_is_visible(tmp_path, monkeypatch, caplog): + """A failed cache write must explain why subsequent reloads compile everything.""" + _use_tmp_web_dir(tmp_path, monkeypatch) + pages = [_FakePage(route="/a", component=_page_a)] + ctx = _compile(pages) + + def fail_write(manifest): + """Simulate a failure writing the cache manifest. + + Raises: + OSError: The cache cannot be persisted. + """ + msg = "disk full" + raise OSError(msg) + + monkeypatch.setattr(disk_cache, "_write", fail_write) + with caplog.at_level("WARNING", logger=disk_cache.__name__): + disk_cache.write_manifest(ctx, pages, ctx.all_imports, root=tmp_path) + assert any("disk full" in record.getMessage() for record in caplog.records) + + +def test_serialize_imports_collapses_duplicates(): + """The manifest only needs the unique import set, in first-seen order. + + A full docs-app compile accumulates ~107k entries of which ~6k are unique; + storing duplicates bloats the manifest and every later merge over it. + """ + use_effect = ImportVar("useEffect") + fragment = ImportVar("Fragment", is_default=False) + imports = { + "react": [use_effect, fragment, use_effect, use_effect, fragment], + "@emotion/react": [ImportVar("jsx"), ImportVar("jsx")], + } + restored = disk_cache._deserialize_imports(disk_cache._serialize_imports(imports)) + assert restored == { + "react": [use_effect, fragment], + "@emotion/react": [ImportVar("jsx")], + } + + +def _manifest(pages: dict[str, dict], root: Path | None = None, **overrides) -> dict: + base = { + "schema": disk_cache._SCHEMA, + "root": str((root or Path.cwd()).resolve()), + "reflex_version": page_cache._reflex_version(), + "mode": environment.REFLEX_ENV_MODE.get().value, + "files": {}, + "globals": {"reflex": page_cache._reflex_version()}, + "globals_absent": [], + "all_imports": {}, + "pages": pages, + "memo_files": {}, + "entry": {"path": None, "digest": None}, + "module_states": {"names": [], "files": [], "dynamic": []}, + "app_root": {"wraps": {}, "libraries": [], "memo_keys": []}, + } + base.update(overrides) + return base + + +def _stale_dep(manifest: dict, route: str, path: str) -> None: + """Make ``route`` depend on ``path`` with a recorded entry that cannot match. + + Simulates an edit: the page's dependency set is rewritten to a file whose + stored stat/hash no longer match the current content, so partitioning sees + the page as a miss. + + Args: + manifest: The manifest to mutate. + route: The page route. + path: The dependency path. + """ + manifest["pages"][route]["deps"] = [path] + manifest["files"][path] = ["stale-hash", 0, 0] + + +def _validator(manifest: dict) -> page_cache.FileValidator: + return page_cache.FileValidator(manifest["files"]) + + +def test_globals_mismatch_names_the_changed_input(tmp_path): + m = _manifest( + {"/a": {}, "/b": {}}, + root=tmp_path, + globals_absent=[str(tmp_path / "rxconfig.py")], + ) + assert ( + disk_cache.globals_mismatch(m, validator=_validator(m), root=tmp_path) is None + ) + # a stale reflex version -> named versions + reason = disk_cache.globals_mismatch( + {**m, "reflex_version": "0.0.0-old"}, + validator=_validator(m), + root=tmp_path, + ) + assert reason is not None + assert "0.0.0-old" in reason + + +def test_globals_mismatch_accepts_entrypoint_registration_edits(tmp_path): + """An entry module that only changed its add_page calls is not global.""" + entry = tmp_path / "entry_app.py" + entry.write_text( + 'import reflex as rx\napp = rx.App()\napp.add_page(a, route="/a")\n' + ) + file_entry = page_cache.file_entry(str(entry)) + assert file_entry is not None + m = _manifest( + {}, + root=tmp_path, + files={str(entry): list(file_entry)}, + globals={"reflex": page_cache._reflex_version(), f"app:{entry}": str(entry)}, + entry={"path": str(entry), "digest": page_cache.entrypoint_digest(str(entry))}, + ) + entry.write_text( + 'import reflex as rx\napp = rx.App()\napp.add_page(a, route="/a")\n' + 'app.add_page(b, route="/b")\n' + ) + assert ( + disk_cache.globals_mismatch(m, validator=_validator(m), root=tmp_path) is None + ) + entry.write_text("import reflex as rx\napp = rx.App(enable_state=False)\n") + reason = disk_cache.globals_mismatch(m, validator=_validator(m), root=tmp_path) + assert reason is not None + assert "entry_app.py" in reason + + +def test_globals_mismatch_names_a_moved_project(tmp_path): + """A manifest only describes the project directory that wrote it. + + Its file table holds absolute paths, so a copy of the project made with + its ``.web`` would otherwise validate the original project's files. + """ + m = _manifest({"/a": {}}, root=tmp_path / "original") + reason = disk_cache.globals_mismatch( + m, validator=_validator(m), root=tmp_path / "copy" + ) + assert reason is not None + assert "project root changed" in reason + + +def test_globals_mismatch_names_a_compile_mode_change(tmp_path, monkeypatch): + """A development build's output must never be reused by a production build. + + Rendered files differ by mode (e.g. ``isDevMode`` in the contexts file), so + the mode is part of the cache identity. + """ + monkeypatch.setenv("REFLEX_ENV_MODE", "dev") + m = _manifest({"/a": {}}, root=tmp_path) + assert ( + disk_cache.globals_mismatch(m, validator=_validator(m), root=tmp_path) is None + ) + monkeypatch.setenv("REFLEX_ENV_MODE", "prod") + reason = disk_cache.globals_mismatch(m, validator=_validator(m), root=tmp_path) + assert reason is not None + assert "dev -> prod" in reason + + +def test_globals_mismatch_validates_stored_inputs_only(tmp_path): + """Epoch validation re-hashes the *stored* input set, never a recomputed one. + + ``app_dependency_files`` depends on what the current process happened to + read/import during app import, which differs between a cold compile and a + warm forked reload (non-purged module caches skip re-reads). Comparing a + recomputed set against the stored one therefore mismatched on every hot + reload; only stored inputs whose *content* changed may invalidate. + """ + theme = tmp_path / "theme_config.py" + theme.write_text("PRIMARY = 'red'") + entry = page_cache.file_entry(str(theme)) + assert entry is not None + m = _manifest( + {"/a": {}}, + root=tmp_path, + files={str(theme): entry}, + globals={"reflex": page_cache._reflex_version(), f"app:{theme}": str(theme)}, + # global files absent from this root at write time and still absent + globals_absent=[str(tmp_path / "rxconfig.py")], + ) + + def mismatch() -> str | None: + return disk_cache.globals_mismatch(m, validator=_validator(m), root=tmp_path) + + # nothing on disk changed -> match, regardless of what a re-recorded + # app-import read set would look like in this process + assert mismatch() is None + # a stored input's content changed -> mismatch naming that file + theme.write_text("PRIMARY = 'blue'") + reason = mismatch() + assert reason is not None + assert "theme_config.py" in reason + # restored content with a fresh mtime: hashed once, still a match + theme.write_text("PRIMARY = 'red'") + assert mismatch() is None + # a stored global file appearing counts as a change too + (tmp_path / "rxconfig.py").write_text("import reflex") + reason = mismatch() + assert reason is not None + assert "rxconfig.py" in reason + + +def test_format_path_list_relativizes_and_truncates(): + root = Path("/proj") + assert disk_cache.format_path_list({"/proj/a.py", "other"}, root) == "a.py, other" + many = {f"/proj/{i}.py" for i in range(8)} + out = disk_cache.format_path_list(many, root, limit=3) + assert out == "0.py, 1.py, 2.py (+5 more)" + + +def test_partition_pages_detects_changed_source(tmp_path): + pages = [ + _FakePage(route="/a", component=_page_a), + _FakePage(route="/b", component=_page_b), + ] + x, y = tmp_path / "x.py", tmp_path / "y.py" + x.write_text("X = 1\n") + y.write_text("Y = 1\n") + # /a depends on x.py, /b depends on y.py (each recorded in the file table). + m = _manifest( + { + page.route: { + "deps": [str(dep)], + "registration": "recorded", + } + for page, dep in zip(pages, (x, y), strict=True) + }, + files={ + str(x): page_cache.file_entry(str(x)), + str(y): page_cache.file_entry(str(y)), + }, + ) + assert disk_cache.partition_pages(pages, m, _validator(m), None) == [] + y.write_text("Y = 2\n") + miss = disk_cache.partition_pages(pages, m, _validator(m), None) + assert {p.route for p in miss} == {"/b"} + # With the watcher's changed set, pages depending on none of the changed + # paths are hits without touching the filesystem. + assert disk_cache.partition_pages(pages, m, _validator(m), {str(x)}) == [] + miss = disk_cache.partition_pages(pages, m, _validator(m), {str(y)}) + assert {p.route for p in miss} == {"/b"} + + +def test_partition_pages_rebuilds_a_volatile_page_whatever_the_hint(tmp_path): + """A volatile page misses even when the watcher reports only other files.""" + pages = [_FakePage(route="/a", component=_page_a)] + x = tmp_path / "x.py" + x.write_text("X = 1\n") + m = _manifest( + {"/a": {"deps": [str(x), page_cache.VOLATILE_DEP], "registration": "recorded"}}, + files={str(x): page_cache.file_entry(str(x))}, + ) + miss = disk_cache.partition_pages(pages, m, _validator(m), {str(tmp_path / "y.py")}) + assert [p.route for p in miss] == ["/a"] + + +def test_pages_mismatch_names_the_change(tmp_path): + """The page registry is an input of every page, so any change is global.""" + pages = [ + _FakePage(route="/a", component=_page_a), + _FakePage(route="/b", component=_page_b), + ] + registrations = {page.route: "recorded" for page in pages} + m = _manifest({page.route: {"registration": "recorded"} for page in pages}) + assert disk_cache._pages_mismatch(pages, m, registrations) is None + reason = disk_cache._pages_mismatch(pages[::-1], m, registrations) + assert reason == "the page order changed" + reason = disk_cache._pages_mismatch( + [*pages, _FakePage(route="/c", component=_page_c)], + m, + {**registrations, "/c": "new"}, + ) + assert reason is not None + assert "added /c" in reason + reason = disk_cache._pages_mismatch(pages[:1], m, registrations) + assert reason is not None + assert "removed /b" in reason + reason = disk_cache._pages_mismatch(pages, m, {**registrations, "/b": "retitled"}) + assert reason == "page '/b' was registered differently" + + +def test_incomplete_cache_miss_falls_back_without_adoption(tmp_path, monkeypatch): + """A miss without page output must be compiled again by the full path.""" + _use_tmp_web_dir(tmp_path, monkeypatch) + app = rx.App() + app.add_page(_page_a, route="/a") + pages = list(app._unevaluated_pages.values()) + route = next(page.route for page in pages if page.component is _page_a) + ctx = _compile(pages, app) + disk_cache.write_manifest(ctx, pages, ctx.all_imports, root=tmp_path) + _stub_externals(app, monkeypatch) + + manifest_path = disk_cache._manifest_path() + manifest = json.loads(manifest_path.read_text()) + _stale_dep(manifest, route, str(tmp_path / "changed.py")) + manifest_path.write_text(json.dumps(manifest)) + + original_compile = CompileContext.compile + + def compile_without_output(self, **kwargs): + """Simulate a page that evaluates but cannot produce frontend output. + + Returns: + The contexts produced before their output is removed. + """ + result = original_compile(self, **kwargs) + for page_ctx in self.compiled_pages.values(): + page_ctx.output_code = None + return result + + monkeypatch.setattr(CompileContext, "compile", compile_without_output) + + assert ( + disk_cache.try_incremental_rebuild( + app, compiler_plugins=[], prerender_routes=False, root=tmp_path + ) + is False + ) + + +def test_absorb_preserves_page_order_for_app_wraps(): + """A later reused page must override an earlier recompiled page's wrap.""" + first = _FakePage(route="/first", component=_page_a) + second = _FakePage(route="/second", component=_page_b) + key = (1, "shared") + first_wrap = rx.box("first") + second_wrap = rx.box("second") + first_ctx = PageContext( + name="first", + route=first.route, + root_component=rx.box(), + app_wrap_components={key: first_wrap}, + ) + second_ctx = PageContext( + name="second", + route=second.route, + root_component=rx.box(), + app_wrap_components={key: second_wrap}, + ) + reused = CompileContext( + pages=[second], + compiled_pages={second.route: second_ctx}, + app_wrap_components={key: second_wrap}, + ) + recompiled = CompileContext( + pages=[first], + compiled_pages={first.route: first_ctx}, + app_wrap_components={key: first_wrap}, + ) + + reused.absorb(recompiled, [first, second]) + + assert reused.app_wrap_components[key] is second_wrap + + +def test_write_and_load_manifest(tmp_path, monkeypatch): + _use_tmp_web_dir(tmp_path, monkeypatch) + monkeypatch.setenv("REFLEX_ENV_MODE", "dev") + + pages = [ + _FakePage(route="/a", component=_page_a), + _FakePage(route="/b", component=_page_b), + _FakePage(route="/c", component=_page_c), + ] + ctx = _compile(pages) + disk_cache.write_manifest(ctx, pages, ctx.all_imports, root=tmp_path) + + manifest = disk_cache.load_manifest() + assert manifest is not None + assert manifest["schema"] == disk_cache._SCHEMA + assert manifest["mode"] == "dev" + assert set(manifest["pages"]) == {"/a", "/b", "/c"} + for route in ("/a", "/b", "/c"): + entry = manifest["pages"][route] + # the manifest is pure bookkeeping: dep set + app-wrap keys + flags + assert set(entry) == { + "deps", + "component_counts", + "app_wraps", + "registration", + "is_stateful", + "state_slice", + "dynamic_claims", + "memo_keys", + "radix", + } + # these static pages register no new state and use no memos + assert entry["is_stateful"] is False + assert entry["state_slice"] is None + assert entry["memo_keys"] == [] + assert entry["radix"] is False + assert entry["dynamic_claims"] == [] + # every dependency has a file-table entry: [sha256, mtime_ns, size] + for dep in entry["deps"]: + digest, mtime_ns, size = manifest["files"][dep] + assert len(digest) == 64 + assert mtime_ns > 0 + assert size > 0 + # rendered output is never persisted (it already lives in .web, and is + # never read back from the manifest) -> keeps the manifest small + assert "output_code" not in entry + assert "frontend_imports" not in entry + # the app-wide merged imports round-trip cleanly (duplicates collapsed) + restored = disk_cache._deserialize_imports(manifest["all_imports"]) + assert restored == { + lib: list(dict.fromkeys(ivs)) for lib, ivs in ctx.all_imports.items() + } + + +def test_unchanged_pages_compile_identically(tmp_path, monkeypatch): + """The reuse correctness property: an unchanged page recompiles byte-for-byte. + + The disk cache leaves a hit page's already-on-disk ``.web`` file untouched, so + reuse is correct if a fresh compile of that page yields identical output. + Compile A, B, C; then compile A, B(edited), C; A and C must be byte-identical. + """ + _use_tmp_web_dir(tmp_path, monkeypatch) + + pages = [ + _FakePage(route="/a", component=_page_a), + _FakePage(route="/b", component=_page_b), + _FakePage(route="/c", component=_page_c), + ] + ctx1 = _compile(pages) + + # "Edit" page B; recompile the whole app cleanly. + pages_edited = [ + _FakePage(route="/a", component=_page_a), + _FakePage(route="/b", component=_page_b_edited), + _FakePage(route="/c", component=_page_c), + ] + ctx2 = _compile(pages_edited) + + # Unchanged pages are byte-identical across compiles -> safe to reuse. + assert ( + ctx2.compiled_pages["/a"].output_code == ctx1.compiled_pages["/a"].output_code + ) + assert ( + ctx2.compiled_pages["/c"].output_code == ctx1.compiled_pages["/c"].output_code + ) + # B changed. + assert ( + ctx2.compiled_pages["/b"].output_code != ctx1.compiled_pages["/b"].output_code + ) + + +def test_names_drawn_after_the_pages_never_repeat_import_time_names(): + """The post-page phase keeps the names the process drew before the compile. + + A module-level component draws its ref name at import; after the pages, + the app root and runtime components draw more. Reseeding the import + sequence would hand the same names out again. + """ + from reflex_base.vars import base as vars_base + from reflex_base.vars.base import ( + get_unique_variable_name, + reset_unique_variable_names, + restore_unique_variable_name_state, + unique_variable_name_state, + ) + + saved = unique_variable_name_state() + saved_reserved = vars_base._names_before_first_compile + try: + reset_unique_variable_names() + import_names = {get_unique_variable_name() for _ in range(3)} + _compile([_FakePage(route="/a", component=_page_a)]) + after = {get_unique_variable_name() for _ in range(50)} + assert not after & import_names + # And the post-page sequence is the same on every compile. + _compile([_FakePage(route="/b", component=_page_b)]) + assert {get_unique_variable_name() for _ in range(50)} == after + finally: + restore_unique_variable_name_state(saved) + vars_base._names_before_first_compile = saved_reserved + + +_CONTEXTS_STUB = "// contexts stub" + + +_TEST_STATE_MODULES = (__name__, "fp_mod_x", "cache_review_state_view") + + +def _scoped_contexts_snapshot(app) -> ContextsSnapshot | None: + """A ``_contexts_snapshot`` limited to the states this test file defines. + + Same shape and serialization as the real snapshot, but each test state is + compiled standalone instead of walking the whole root state tree, which + would pick up unrelated (and sometimes broken) state classes collected + from other test modules. Schema entries only read class metadata, so they + still cover the whole tree. + + Args: + app: The app being compiled. + + Returns: + The contexts snapshot for this file's states, or None when the app has + no state tree. + """ + if app is None or app._state is None: + return None + context = RegistrationContext.ensure_context() + initial, storage, _ = _states_snapshot( + cls + for cls in list(context.base_states.values()) + if cls.__module__ in _TEST_STATE_MODULES + ) + return initial, storage, _frontend_schema_entries(app._state, context) + + +def _states_snapshot(classes: Iterable[type[BaseState]]) -> ContextsSnapshot: + """Compile a contexts snapshot from the given states, each standalone. + + Args: + classes: The states to compile. + + Returns: + The contexts snapshot of those states. + """ + from reflex.compiler import utils as compiler_utils + + context = RegistrationContext.ensure_context() + initial: dict[str, Any] = {} + storage: dict[str, dict[str, Any]] = {} + schema_entries: dict[str, list[Any]] = {} + for cls in classes: + initial.update(compiler_utils.compile_state(cls)) + for kind, entries in compiler_utils.compile_client_storage(cls).items(): + storage.setdefault(kind, {}).update(entries) + schema_entries.update(_frontend_schema_entries(cls, context)) + return initial, storage, schema_entries + + +def _stub_externals(app, monkeypatch): + """Stub the side-effecting steps the fast path runs on a real app.""" + import reflex.utils.frontend_skeleton as fs + from reflex.compiler import utils as compiler_utils + + monkeypatch.setattr(app, "_get_frontend_packages", lambda *a, **k: None) + monkeypatch.setattr(app, "_add_optional_endpoints", lambda *a, **k: None) + monkeypatch.setattr(app, "_validate_var_dependencies", lambda *a, **k: None) + monkeypatch.setattr(app, "_write_stateful_pages_marker", lambda *a, **k: None) + # Serializing the real root state tree would pick up unrelated state + # classes from other collected test modules. + monkeypatch.setattr( + "reflex.compiler.compiler.compile_contexts", + lambda state, theme, **kwargs: ( + compiler_utils.get_context_path(), + _CONTEXTS_STUB, + ), + ) + monkeypatch.setattr(disk_cache, "_contexts_snapshot", _scoped_contexts_snapshot) + monkeypatch.setattr(fs, "update_react_router_config", lambda **k: None) + monkeypatch.setattr(fs, "update_entry_client", lambda *a, **k: None) + monkeypatch.setattr(fs, "initialize_vite_config", lambda: None) + # The full compile that preceded the rebuild wrote the bundled libraries. + libs_path, libs_code = compiler_utils._compile_bundled_libraries() + libs_file = compiler_utils.resolve_path_of_web_dir(libs_path) + libs_file.parent.mkdir(parents=True, exist_ok=True) + libs_file.write_text(libs_code) + + +def _page_local_state() -> Component: + class CacheLocalState(rx.State): + value: int = 0 + + return rx.el.div(CacheLocalState.value) + + +def _page_local_component_default() -> Component: + class CacheComponentDefaultState(rx.State): + scroller: rx.Field[rx.Component] = rx.field(default_factory=rx.auto_scroll) + + return rx.el.div(CacheComponentDefaultState.scroller) + + +def test_manifest_state_slice_matches_the_compiled_contexts_snapshot( + tmp_path, monkeypatch +): + """State defaults in the manifest must use the contexts compilation values.""" + _use_tmp_web_dir(tmp_path, monkeypatch) + app = rx.App() + app.add_page(_page_local_component_default, route="/stateful") + pages = list(app._unevaluated_pages.values()) + ctx = _compile(pages, app) + route = next( + page.route for page in pages if page.component is _page_local_component_default + ) + defined_states = ctx.stateful_routes[route] + registry = RegistrationContext.ensure_context() + snapshot = _states_snapshot(registry.base_states[name] for name in defined_states) + expected = json.loads( + disk_cache.json_dumps(disk_cache._state_slice(defined_states, *snapshot)) + ) + + try: + disk_cache.write_manifest( + ctx, + pages, + ctx.all_imports, + root=tmp_path, + contexts_snapshot=snapshot, + ) + manifest = disk_cache.load_manifest() + assert manifest is not None + assert manifest["pages"][route]["state_slice"] == expected + finally: + for state_name in defined_states: + _unregister_state(registry.base_states[state_name]) + + +def test_post_evaluation_fallback_hands_over_the_evaluated_pages(tmp_path, monkeypatch): + """A fallback decided after evaluating miss pages returns those pages. + + Evaluating a page registers the states it defines. Re-evaluating it in the + same process (as an in-process full compile would) registers a locally + defined state under a collision suffix (``CacheLocalState_0``) while the + backend, which evaluates the page once, knows it as ``CacheLocalState``. + The full compile therefore has to reuse this evaluation, not repeat it. + """ + from reflex_base.registry import RegistrationContext + + web = _use_tmp_web_dir(tmp_path, monkeypatch) + app = rx.App() + app.add_page(_page_a, route="/a") + app.add_page(_page_local_state, route="/s") + _stub_externals(app, monkeypatch) + pages = list(app._unevaluated_pages.values()) + route_s = pages[1].route + ctx = _compile(pages, app) + defined = ctx.stateful_routes[route_s] + assert defined[0].endswith("cache_local_state") + disk_cache.write_manifest(ctx, pages, ctx.all_imports, root=tmp_path) + + def forget_local_states() -> None: + """Drop the page-defined states, as the daemon's child reset does.""" + registry = RegistrationContext.ensure_context() + for name in defined: + cls = registry.base_states.get(name) + if cls is not None: + _unregister_state(cls) + + forget_local_states() + try: + # The manifest recorded another app-wrap set for /s; only a full + # compile can rebuild the app root. + manifest_path = web / disk_cache._MANIFEST_FILE + manifest = json.loads(manifest_path.read_text()) + manifest["app_root"]["wraps"] = {} + _stale_dep(manifest, route_s, str(tmp_path / "demo.md")) + manifest_path.write_text(json.dumps(manifest)) + + result = disk_cache.try_incremental_rebuild( + app, compiler_plugins=[], prerender_routes=False, root=tmp_path + ) + assert isinstance(result, CompileContext) + assert set(result.compiled_pages) == {route_s} + assert result.stateful_routes[route_s] == defined + finally: + forget_local_states() + + +def _capture_contexts(monkeypatch) -> list[Any]: + """Record the snapshots handed to ``compile_contexts``. + + Args: + monkeypatch: The pytest monkeypatch fixture. + + Returns: + The list the snapshots are appended to. + """ + from reflex.compiler import utils as compiler_utils + + captured: list[Any] = [] + + def fake_compile_contexts(state, theme, *, contexts_snapshot, **kwargs): + captured.append(contexts_snapshot) + return compiler_utils.get_context_path(), _CONTEXTS_STUB + + monkeypatch.setattr( + "reflex.compiler.compiler.compile_contexts", fake_compile_contexts + ) + return captured + + +def _page_flip() -> Component: + return rx.el.div(rx.el.p(_fp_state("_FlipState").value), _footer()) + + +def test_page_becoming_stateful_rebuilds_incrementally(tmp_path, monkeypatch): + """A page that starts defining states needs only its contexts slice. + + The app root depends on whether the app has state at all, which ``rx.App`` + decides, not on which pages define states. + """ + web = _use_tmp_web_dir(tmp_path, monkeypatch) + app = rx.App() + app.add_page(_page_a, route="/a") + app.add_page(_page_flip, route="/s") + pages = list(app._unevaluated_pages.values()) + route_s = pages[1].route + ctx = _compile(pages, app) + defined = ctx.stateful_routes[route_s] + _stub_externals(app, monkeypatch) + disk_cache.write_manifest(ctx, pages, ctx.all_imports, root=tmp_path) + _unregister_state(_FP_HOLDER.pop("_FlipState")) + manifest_path = web / disk_cache._MANIFEST_FILE + manifest = json.loads(manifest_path.read_text()) + # Recorded before the edit that made /s define its state. + manifest["pages"][route_s].update(is_stateful=False, state_slice=None) + _stale_dep(manifest, route_s, str(tmp_path / "view.py")) + manifest_path.write_text(json.dumps(manifest)) + captured = _capture_contexts(monkeypatch) + try: + assert ( + disk_cache.try_incremental_rebuild( + app, compiler_plugins=[], prerender_routes=False, root=tmp_path + ) + is True + ) + finally: + _unregister_state(_FP_HOLDER.pop("_FlipState")) + [(initial, _, _)] = captured + assert set(defined) <= set(initial) + refreshed = json.loads(manifest_path.read_text()) + assert refreshed["pages"][route_s]["is_stateful"] is True + + +def _drop_page_state( + tmp_path, + monkeypatch, + gone: str, + compile_contexts: Callable[..., Any] | None = None, + install: Callable[[Any], None] | None = None, +) -> tuple[Any, list[Any], dict[str, Any]]: + """Rebuild a page that defined ``gone`` before an edit and no state after it. + + Args: + tmp_path: The project root. + monkeypatch: The pytest monkeypatch fixture. + gone: The full name of the state the page stopped defining. + compile_contexts: Replaces ``compile_contexts`` after the capture. + install: Receives the imports whose packages the rebuild installs. + + Returns: + The rebuild result, the captured contexts snapshots, and the manifest. + """ + web = _use_tmp_web_dir(tmp_path, monkeypatch) + app = rx.App() + app.add_page(_page_a, route="/a") + pages = list(app._unevaluated_pages.values()) + ctx = _compile(pages, app) + _stub_externals(app, monkeypatch) + disk_cache.write_manifest(ctx, pages, ctx.all_imports, root=tmp_path) + manifest_path = web / disk_cache._MANIFEST_FILE + manifest = json.loads(manifest_path.read_text()) + manifest["pages"]["a"].update( + is_stateful=True, + state_slice={ + "initial_state": {gone: {"value_rx_state_": 0}}, + "client_storage": {}, + "schema": {gone: [gone, [["value", "int"]], [], []]}, + }, + ) + _stale_dep(manifest, "a", str(tmp_path / "view.py")) + manifest_path.write_text(json.dumps(manifest)) + captured = _capture_contexts(monkeypatch) + if compile_contexts is not None: + monkeypatch.setattr( + "reflex.compiler.compiler.compile_contexts", compile_contexts + ) + if install is not None: + monkeypatch.setattr(app, "_get_frontend_packages", install) + result = disk_cache.try_incremental_rebuild( + app, compiler_plugins=[], prerender_routes=False, root=tmp_path + ) + return result, captured, json.loads(manifest_path.read_text()) + + +#: A local state the page defined; no other page can use it. +_GONE_LOCAL = "reflex___state____state.reflex___istate___dynamic____gone_state" + + +def test_page_dropping_its_states_rebuilds_contexts(tmp_path, monkeypatch): + """A page that stops defining states takes its slice out of the contexts.""" + result, captured, refreshed = _drop_page_state(tmp_path, monkeypatch, _GONE_LOCAL) + assert result is True + [(initial, _, schema_entries)] = captured + assert _GONE_LOCAL not in initial + assert _GONE_LOCAL not in schema_entries + assert refreshed["pages"]["a"]["is_stateful"] is False + + +def test_page_dropping_a_shared_state_falls_back(tmp_path, monkeypatch): + """A module state the page stops importing may still serve unchanged pages. + + Those pages recorded no slice for it, so only a full compile keeps it. + """ + gone = "reflex___state____state.shared_module___shared_state" + result, captured, _ = _drop_page_state(tmp_path, monkeypatch, gone) + assert isinstance(result, CompileContext) + assert captured == [] + + +def test_contexts_rebuild_installs_the_packages_state_defaults_import( + tmp_path, monkeypatch +): + """A component-valued state default can bring a package no page imports.""" + from reflex.compiler import utils as compiler_utils + + def compile_contexts(state, theme, *, contexts_snapshot, component_imports): + component_imports.setdefault("state-only-package", []).append( + ImportVar(tag="Widget") + ) + return compiler_utils.get_context_path(), _CONTEXTS_STUB + + installed: list[Any] = [] + result, _, refreshed = _drop_page_state( + tmp_path, monkeypatch, _GONE_LOCAL, compile_contexts, installed.append + ) + assert result is True + assert "state-only-package" in installed[-1] + assert "state-only-package" in refreshed["all_imports"] + + +def test_incremental_rebuild_all_hits(tmp_path, monkeypatch): + _use_tmp_web_dir(tmp_path, monkeypatch) + + app = rx.App() + app.add_page(_page_a, route="/a") + app.add_page(_page_c, route="/c") + pages = list(app._unevaluated_pages.values()) + ctx = _compile(pages) + disk_cache.write_manifest(ctx, pages, ctx.all_imports, root=tmp_path) + _stub_externals(app, monkeypatch) + + # Nothing changed -> every page is a hit -> fast path runs. + assert ( + disk_cache.try_incremental_rebuild( + app, compiler_plugins=[], prerender_routes=False, root=tmp_path + ) + is True + ) + + from reflex.utils.telemetry_accounting import _count_components + + assert app._cached_component_counts == _count_components( + page.root_component for page in ctx.compiled_pages.values() + ) + + +def test_incremental_rebuild_one_miss_writes_only_that_page(tmp_path, monkeypatch): + from reflex.compiler import utils as compiler_utils + + web = _use_tmp_web_dir(tmp_path, monkeypatch) + + app = rx.App() + app.add_page(_page_a, route="/a") + app.add_page(_page_c, route="/c") + pages = list(app._unevaluated_pages.values()) + ctx = _compile(pages) + disk_cache.write_manifest(ctx, pages, ctx.all_imports, root=tmp_path) + _stub_externals(app, monkeypatch) + + # Simulate an edit to the first page: rewrite its manifest dependency set to + # reference a file whose recorded hash no longer matches the current content, + # so partitioning sees its dependency set as changed -> a miss (only it). + edited_route = pages[0].route + manifest_path = web / disk_cache._MANIFEST_FILE + manifest = json.loads(manifest_path.read_text()) + _stale_dep(manifest, edited_route, str(tmp_path / "view.py")) + manifest_path.write_text(json.dumps(manifest)) + + assert ( + disk_cache.try_incremental_rebuild( + app, compiler_plugins=[], prerender_routes=False, root=tmp_path + ) + is True + ) + + assert edited_route in app._pages + assert edited_route in app._evaluated_pages + + # The edited page was recompiled and written; its content matches a clean + # compile of that page. + output_path = ctx.compiled_pages[edited_route].output_path + assert output_path is not None + out_path = compiler_utils.resolve_path_of_web_dir(output_path) + assert out_path.exists() + assert ( + out_path.read_text(encoding="utf-8") + == ctx.compiled_pages[edited_route].output_code + ) + # The same-module hit page contributed no memos, so it is reused, not + # recompiled (its output was never written to this fresh web dir). + hit_output_path = ctx.compiled_pages[pages[1].route].output_path + assert hit_output_path is not None + assert not compiler_utils.resolve_path_of_web_dir(hit_output_path).exists() + + +def test_stateful_hit_is_marked_but_not_reevaluated(tmp_path, monkeypatch): + """A stateful HIT page is recorded in the marker but never re-evaluated. + + The compile process only produces .web and exits; the serving backend + re-evaluates the marked stateful pages itself, so re-evaluating them during + the incremental rebuild was pure waste. + """ + web = _use_tmp_web_dir(tmp_path, monkeypatch) + + app = rx.App() + app.add_page(_page_a, route="/a") + pages = list(app._unevaluated_pages.values()) + route = pages[0].route + ctx = _compile(pages) + disk_cache.write_manifest(ctx, pages, ctx.all_imports, root=tmp_path) + # Mark the page as a stateful HIT page in the manifest. + manifest_path = web / disk_cache._MANIFEST_FILE + manifest = json.loads(manifest_path.read_text()) + manifest["pages"][route]["is_stateful"] = True + manifest_path.write_text(json.dumps(manifest)) + _stub_externals(app, monkeypatch) + + reevaluated: list[str] = [] + monkeypatch.setattr( + app, "_compile_page", lambda route, **k: reevaluated.append(route) + ) + + assert ( + disk_cache.try_incremental_rebuild( + app, compiler_plugins=[], prerender_routes=False, root=tmp_path + ) + is True + ) + assert reevaluated == [] + assert route in app._stateful_pages + + +def test_incremental_rebuild_preserves_contexts_without_stateful_miss( + tmp_path, monkeypatch +): + """The contexts file is reused unless a stateful page missed. + + The full compile's contexts file holds every state's defaults, including + states only registered while their page evaluates (exec'd demos, dynamic + imports). An incremental process that evaluated no stateful pages has an + incomplete registry; rewriting contexts from it drops those states and the + frontend's dispatch map with them (``dispatch is not a function``). With no + stateful miss, no state changed -> the on-disk file must stay untouched. + """ + from reflex.compiler import utils as compiler_utils + + _use_tmp_web_dir(tmp_path, monkeypatch) + + app = rx.App() + app.add_page(_page_a, route="/a") + pages = list(app._unevaluated_pages.values()) + ctx = _compile(pages) + disk_cache.write_manifest(ctx, pages, ctx.all_imports, root=tmp_path) + _stub_externals(app, monkeypatch) + + full_contexts = "// complete contexts from the full compile" + out_path = compiler_utils.resolve_path_of_web_dir(compiler_utils.get_context_path()) + out_path.parent.mkdir(parents=True, exist_ok=True) + out_path.write_text(full_contexts, encoding="utf-8") + + assert ( + disk_cache.try_incremental_rebuild( + app, compiler_plugins=[], prerender_routes=False, root=tmp_path + ) + is True + ) + assert out_path.read_text(encoding="utf-8") == full_contexts + + +_state_counter = itertools.count() + + +def _page_s() -> Component: + """A page that defines its state during evaluation (like exec'd docs demos). + + Each evaluation defines a fresh uniquely-named state class, so the page is + marked stateful and repeated evaluations in one test process don't collide. + + Returns: + The page component. + """ + name = f"_ContextsState{next(_state_counter)}" + state_cls: Any = type( + name, + (rx.State,), + {"__annotations__": {"value": str}, "value": "", "__module__": __name__}, + ) + return rx.el.div(rx.el.p(state_cls.value), _footer()) + + +def test_stateful_miss_rewrites_contexts_from_manifest_slices(tmp_path, monkeypatch): + """A stateful miss rewrites contexts without re-evaluating any hit page. + + Hit pages' evaluation-time states are not in this process's state tree; + their recorded contribution (``state_slice``) is handed to + ``compile_contexts`` instead, so the rewritten contexts file keeps every + state and no unchanged page is evaluated. + """ + from reflex.compiler import utils as compiler_utils + + web = _use_tmp_web_dir(tmp_path, monkeypatch) + + app = rx.App() + app.add_page(_page_s, route="/s") + app.add_page(_page_c, route="/c") + pages = list(app._unevaluated_pages.values()) + stateful_route, hit_route = pages[0].route, pages[1].route + ctx = _compile(pages) + assert stateful_route in ctx.stateful_routes + disk_cache.write_manifest(ctx, pages, ctx.all_imports, root=tmp_path) + _stub_externals(app, monkeypatch) + + # Make the stateful page a miss, and mark the hit page stateful so the + # rebuild must re-register its states before compiling contexts. + manifest_path = web / disk_cache._MANIFEST_FILE + manifest = json.loads(manifest_path.read_text()) + _stale_dep(manifest, stateful_route, str(tmp_path / "view.py")) + hit_slice = { + "initial_state": {"root.hit_state": {"count": 0}}, + "client_storage": { + "cookies": {"root.hit_state.token_rx_field_": {"name": "t"}}, + "local_storage": {}, + "session_storage": {}, + }, + "schema": {"root.hit_state": ["root.hit_state", [["count", "int"]], [], []]}, + } + manifest["pages"][hit_route]["is_stateful"] = True + manifest["pages"][hit_route]["state_slice"] = hit_slice + manifest_path.write_text(json.dumps(manifest)) + + reevaluated: list[str] = [] + monkeypatch.setattr( + app, "_compile_page", lambda route, **k: reevaluated.append(route) + ) + captured: list[Any] = [] + + def fake_compile_contexts(state, theme, *, contexts_snapshot, **kwargs): + captured.append(contexts_snapshot) + return compiler_utils.get_context_path(), _CONTEXTS_STUB + + monkeypatch.setattr( + "reflex.compiler.compiler.compile_contexts", fake_compile_contexts + ) + + assert ( + disk_cache.try_incremental_rebuild( + app, compiler_plugins=[], prerender_routes=False, root=tmp_path + ) + is True + ) + # No page is re-evaluated; the hit page's stored slice is merged instead. + assert reevaluated == [] + [(initial, storage, schema_entries)] = captured + assert initial["root.hit_state"] == {"count": 0} + assert storage["cookies"]["root.hit_state.token_rx_field_"] == {"name": "t"} + assert schema_entries["root.hit_state"] == hit_slice["schema"]["root.hit_state"] + out_path = compiler_utils.resolve_path_of_web_dir(compiler_utils.get_context_path()) + assert out_path.read_text(encoding="utf-8") == _CONTEXTS_STUB + + +def _slice_fingerprint( + names: list[str], + initial: dict[str, Any], + storage: dict[str, dict[str, Any]], + schema_entries: dict[str, list[Any]], +) -> str: + return disk_cache._slice_digest( + disk_cache._state_slice(names, initial, storage, schema_entries) + ) + + +def test_contexts_fingerprint_sensitivity(): + initial = {"root.s_one": {"value": ""}, "root.s_two": {"count": 0}} + storage = { + "cookies": {"root.s_one.token_rx_field_": {"name": "t"}}, + "local_storage": {}, + "session_storage": {}, + } + schema = { + "root.s_one": ["root.s_one", [["value", "str"]], [], []], + "root.s_two": ["root.s_two", [["count", "int"]], [], []], + } + base = _slice_fingerprint(["root.s_one"], initial, storage, schema) + # Stable across equal inputs and insensitive to name order. + assert _slice_fingerprint(["root.s_one"], dict(initial), storage, schema) == base + both = _slice_fingerprint(["root.s_one", "root.s_two"], initial, storage, schema) + assert ( + _slice_fingerprint(["root.s_two", "root.s_one"], initial, storage, schema) + == both + ) + # Sensitive to which states, their initial values, their client storage + # and their frontend schema. + assert _slice_fingerprint(["root.s_two"], initial, storage, schema) != base + assert ( + _slice_fingerprint( + ["root.s_one"], {"root.s_one": {"value": "x"}}, storage, schema + ) + != base + ) + assert ( + _slice_fingerprint( + ["root.s_one"], + initial, + {"cookies": {}, "local_storage": {}, "session_storage": {}}, + schema, + ) + != base + ) + assert ( + _slice_fingerprint( + ["root.s_one"], + initial, + storage, + {"root.s_one": ["root.s_one", [["value", "str"]], [], ["touch"]]}, + ) + != base + ) + + +_FP_HOLDER: dict[str, Any] = {} + + +def _page_fp() -> Component: + """A stateful page that defines the SAME state class on every evaluation. + + Mirrors a docs page whose exec'd demo code is unchanged between reloads. + The previous definition must be unregistered before re-evaluation (the + daemon's registry reset does this between hot reloads). + + Returns: + The page component. + """ + state_cls: Any = type( + "_FixedFpState", + (rx.State,), + {"__annotations__": {"value": str}, "value": "", "__module__": "fp_mod_x"}, + ) + _FP_HOLDER["cls"] = state_cls + return rx.el.div(rx.el.p(state_cls.value), _footer()) + + +def test_stateful_miss_with_unchanged_states_reuses_contexts(tmp_path, monkeypatch): + """A stateful miss whose states are unchanged must not rebuild contexts. + + Most content edits leave the page's evaluation-time states identical, so + re-evaluating every stateful hit page just to rewrite an identical contexts + file would waste nearly the whole hot reload. + """ + from reflex.compiler import utils as compiler_utils + + web = _use_tmp_web_dir(tmp_path, monkeypatch) + + app = rx.App() + app.add_page(_page_fp, route="/s") + app.add_page(_page_c, route="/c") + pages = list(app._unevaluated_pages.values()) + stateful_route, hit_route = pages[0].route, pages[1].route + ctx = _compile(pages, app=app) + assert stateful_route in ctx.stateful_routes + # Stub before write_manifest: it fingerprints via the contexts snapshot. + _stub_externals(app, monkeypatch) + disk_cache.write_manifest(ctx, pages, ctx.all_imports, root=tmp_path) + + manifest_path = web / disk_cache._MANIFEST_FILE + manifest = json.loads(manifest_path.read_text()) + stored_fp = manifest["pages"][stateful_route]["state_slice"] + assert stored_fp # the compile recorded the page's state config + # Make the stateful page a miss; mark the hit page stateful so a contexts + # rebuild (if wrongly triggered) would have to re-evaluate it. + _stale_dep(manifest, stateful_route, str(tmp_path / "view.py")) + manifest["pages"][hit_route]["is_stateful"] = True + manifest_path.write_text(json.dumps(manifest)) + + # The daemon child resets the registry before recompiling; drop the class + # so the page's re-evaluation re-defines it identically. + _unregister_state(_FP_HOLDER.pop("cls")) + + reevaluated: list[str] = [] + monkeypatch.setattr( + app, "_compile_page", lambda route, **k: reevaluated.append(route) + ) + sentinel = "// pre-existing contexts" + out_path = compiler_utils.resolve_path_of_web_dir(compiler_utils.get_context_path()) + out_path.parent.mkdir(parents=True, exist_ok=True) + out_path.write_text(sentinel, encoding="utf-8") + + assert ( + disk_cache.try_incremental_rebuild( + app, compiler_plugins=[], prerender_routes=False, root=tmp_path + ) + is True + ) + # Unchanged state config: no hit re-evaluation, contexts file untouched. + assert reevaluated == [] + assert out_path.read_text(encoding="utf-8") == sentinel + # The refreshed manifest records the same fingerprint for the miss page. + refreshed = json.loads(manifest_path.read_text()) + assert refreshed["pages"][stateful_route]["state_slice"] == stored_fp + _unregister_state(_FP_HOLDER.pop("cls")) + + +_SCHEMA_MISS_HANDLER = {"handler": False} + + +def _fp_state(name: str, *, handler: bool = False) -> Any: + """Define a test state in the ``fp_mod_x`` module, as a page evaluation does. + + Args: + name: The class name. + handler: Whether the state defines an event handler. + + Returns: + The state class, also kept in ``_FP_HOLDER`` under ``name``. + """ + namespace: dict[str, Any] = { + "__annotations__": {"value": str}, + "value": "", + "__module__": "fp_mod_x", + } + if handler: + + def touch(self): + """A handler that only exists in the edited state.""" + + namespace["touch"] = touch + state_cls = type(name, (rx.State,), namespace) + _FP_HOLDER[name] = state_cls + return state_cls + + +def _page_schema_miss() -> Component: + state_cls = _fp_state("_SchemaMissState", handler=_SCHEMA_MISS_HANDLER["handler"]) + return rx.el.div(rx.el.p(state_cls.value), _footer()) + + +def _page_schema_hit() -> Component: + return rx.el.div(rx.el.p(_fp_state("_SchemaHitState").value), _footer()) + + +def _schema_pages(tmp_path, monkeypatch): + """Compile a stateful miss page and a stateful hit page, then reset states. + + Args: + tmp_path: The project root. + monkeypatch: The pytest monkeypatch fixture. + + Returns: + The app, the manifest path, the miss route and the full-tree schema. + """ + web = _use_tmp_web_dir(tmp_path, monkeypatch) + app = rx.App() + app.add_page(_page_schema_miss, route="/miss") + app.add_page(_page_schema_hit, route="/hit") + pages = list(app._unevaluated_pages.values()) + ctx = _compile(pages, app=app) + assert app._state is not None + full_schema = app._state._to_frontend_schema() + _stub_externals(app, monkeypatch) + disk_cache.write_manifest(ctx, pages, ctx.all_imports, root=tmp_path) + manifest_path = web / disk_cache._MANIFEST_FILE + manifest = json.loads(manifest_path.read_text()) + miss_route = pages[0].route + _stale_dep(manifest, miss_route, str(tmp_path / "view.py")) + manifest_path.write_text(json.dumps(manifest)) + # The daemon child starts without the states its pages define. + for name in ("_SchemaMissState", "_SchemaHitState"): + _unregister_state(_FP_HOLDER.pop(name)) + return app, manifest_path, miss_route, full_schema + + +def test_contexts_rebuild_hashes_the_schema_of_cached_states(tmp_path, monkeypatch): + """A contexts rebuild hashes the hit pages' cached states into ``stateSchema``. + + The backend evaluates every stateful page, so its schema covers states + that this process never evaluated. A schema of the live tree alone would + make every frontend look stale to a matching backend. + """ + from reflex.compiler import compiler + from reflex.compiler import utils as compiler_utils + + compile_contexts = compiler.compile_contexts + app, manifest_path, miss_route, full_schema = _schema_pages(tmp_path, monkeypatch) + manifest = json.loads(manifest_path.read_text()) + manifest["pages"][miss_route]["state_slice"] = None + manifest_path.write_text(json.dumps(manifest)) + monkeypatch.setattr(compiler, "compile_contexts", compile_contexts) + + try: + assert ( + disk_cache.try_incremental_rebuild( + app, compiler_plugins=[], prerender_routes=False, root=tmp_path + ) + is True + ) + finally: + _unregister_state(_FP_HOLDER.pop("_SchemaMissState")) + contexts = compiler_utils.resolve_path_of_web_dir( + compiler_utils.get_context_path() + ).read_text(encoding="utf-8") + assert f"export const stateSchema = {json.dumps(full_schema)}" in contexts + + +def test_handler_only_state_edit_rebuilds_contexts(tmp_path, monkeypatch): + """An event handler added to a recompiled page's state rebuilds contexts.""" + from reflex.compiler import utils as compiler_utils + + app, _, _, _ = _schema_pages(tmp_path, monkeypatch) + out_path = compiler_utils.resolve_path_of_web_dir(compiler_utils.get_context_path()) + out_path.parent.mkdir(parents=True, exist_ok=True) + out_path.write_text("// pre-existing contexts", encoding="utf-8") + monkeypatch.setitem(_SCHEMA_MISS_HANDLER, "handler", True) + + try: + assert ( + disk_cache.try_incremental_rebuild( + app, compiler_plugins=[], prerender_routes=False, root=tmp_path + ) + is True + ) + finally: + _unregister_state(_FP_HOLDER.pop("_SchemaMissState")) + assert out_path.read_text(encoding="utf-8") == _CONTEXTS_STUB + + +_COMPONENT_DEFAULT = {"recharts": False} + + +def _page_component_default() -> Component: + def default() -> Component: + if _COMPONENT_DEFAULT["recharts"]: + return rx.recharts.line_chart(rx.recharts.line(data_key="v")) + return rx.el.div("plain") + + state_cls: Any = type( + "_ComponentDefaultState", + (rx.State,), + { + "__annotations__": {"comp": rx.Field[rx.Component]}, + "comp": rx.field(default_factory=default), + "__module__": "fp_mod_x", + }, + ) + _FP_HOLDER["_ComponentDefaultState"] = state_cls + return rx.el.div(rx.box(state_cls.comp)) + + +def test_state_default_from_new_library_falls_back(tmp_path, monkeypatch): + """A component default from a new library needs a full compile. + + The contexts reference the library through ``window.__reflex``, which the + app root and bundled-library file provide; only a full compile rewrites + them, and it adopts the pages the attempt evaluated. + """ + from reflex.compiler import compiler + + compile_contexts = compiler.compile_contexts + web = _use_tmp_web_dir(tmp_path, monkeypatch) + app = rx.App() + app.add_page(_page_component_default, route="/dyn") + pages = list(app._unevaluated_pages.values()) + ctx = _compile(pages, app=app) + _stub_externals(app, monkeypatch) + disk_cache.write_manifest(ctx, pages, ctx.all_imports, root=tmp_path) + manifest_path = web / disk_cache._MANIFEST_FILE + manifest = json.loads(manifest_path.read_text()) + _stale_dep(manifest, pages[0].route, str(tmp_path / "view.py")) + manifest_path.write_text(json.dumps(manifest)) + _unregister_state(_FP_HOLDER.pop("_ComponentDefaultState")) + monkeypatch.setitem(_COMPONENT_DEFAULT, "recharts", True) + monkeypatch.setattr(compiler, "compile_contexts", compile_contexts) + try: + result = disk_cache.try_incremental_rebuild( + app, compiler_plugins=[], prerender_routes=False, root=tmp_path + ) + finally: + _unregister_state(_FP_HOLDER.pop("_ComponentDefaultState")) + assert isinstance(result, CompileContext) + assert set(result.compiled_pages) == {pages[0].route} + + +def test_unchanged_component_default_reuses_the_contexts(tmp_path, monkeypatch): + """A recompiled page with the same component default keeps the contexts. + + The recorded slice holds the default serialized the way the contexts file + holds it, whichever libraries were bundled when the slice was taken. + """ + from reflex.compiler import compiler + from reflex.compiler import utils as compiler_utils + + web = _use_tmp_web_dir(tmp_path, monkeypatch) + monkeypatch.setitem(_COMPONENT_DEFAULT, "recharts", True) + app = rx.App() + app.add_page(_page_component_default, route="/dyn") + pages = list(app._unevaluated_pages.values()) + ctx = _compile(pages, app=app) + _stub_externals(app, monkeypatch) + disk_cache.write_manifest(ctx, pages, ctx.all_imports, root=tmp_path) + manifest_path = web / disk_cache._MANIFEST_FILE + manifest = json.loads(manifest_path.read_text()) + _stale_dep(manifest, pages[0].route, str(tmp_path / "view.py")) + manifest_path.write_text(json.dumps(manifest)) + _unregister_state(_FP_HOLDER.pop("_ComponentDefaultState")) + # A fresh process has not bundled the default's library yet. + compiler._reset_compile_registries([]) + out_path = compiler_utils.resolve_path_of_web_dir(compiler_utils.get_context_path()) + out_path.parent.mkdir(parents=True, exist_ok=True) + out_path.write_text("// pre-existing contexts", encoding="utf-8") + try: + assert ( + disk_cache.try_incremental_rebuild( + app, compiler_plugins=[], prerender_routes=False, root=tmp_path + ) + is True + ) + finally: + _unregister_state(_FP_HOLDER.pop("_ComponentDefaultState")) + assert out_path.read_text(encoding="utf-8") == "// pre-existing contexts" + + +def _app_root_file(web: Path) -> Path: + return web / constants.Dirs.PAGES / constants.PageNames.APP_ROOT + + +class _WrappedDiv(rx.el.Div): + """A div that requires an app-level wrap no other test page provides.""" + + @staticmethod + def _get_app_wrap_components() -> dict[tuple[int, str], Component]: + """Declare the extra app wrap. + + Returns: + The app wraps this component requires. + """ + return {(55, "TestWrap"): rx.el.section(class_name="test-wrap")} + + +_WRAP_EDIT = {"wrap": False, "state": False} + + +def _page_wrap_edit() -> Component: + if _WRAP_EDIT["wrap"]: + return rx.el.div(_WrappedDiv.create("wrapped"), _footer()) + if _WRAP_EDIT["state"]: + return rx.el.div(rx.el.p(_fp_state("_WrapEditState").value), _footer()) + return rx.el.div("plain", _footer()) + + +def _page_state_provider() -> Component: + return rx.el.div(rx.el.p(_fp_state("_ProviderState").value), _footer()) + + +def _root_rebuild( + tmp_path, monkeypatch, *, reversed_wraps: list[str] | None = None, **edit +) -> tuple[Any, Path, str]: + """Fully compile a stateful page and a plain page, then edit the plain one. + + Args: + tmp_path: The project root. + monkeypatch: The pytest monkeypatch fixture. + reversed_wraps: When given, the app root's wraps are recorded in + reverse order, which is appended to it. + edit: The ``_WRAP_EDIT`` flags the edit turns on. + + Returns: + The rebuild result, the app root file and its content before the edit. + """ + from reflex.compiler import compiler + + try: + web = _use_tmp_web_dir(tmp_path, monkeypatch) + app = rx.App() + app.add_page(_page_state_provider, route="/s") + app.add_page(_page_wrap_edit, route="/p") + pages = list(app._unevaluated_pages.values()) + ctx = _compile(pages, app=app) + _stub_externals(app, monkeypatch) + root_path, root_code = compiler.compile_app_root( + *compiler._build_app_root(app, ctx.app_wrap_components, ctx) + ) + Path(root_path).parent.mkdir(parents=True, exist_ok=True) + Path(root_path).write_text(root_code) + disk_cache.write_manifest(ctx, pages, ctx.all_imports, root=tmp_path) + manifest_path = web / disk_cache._MANIFEST_FILE + manifest = json.loads(manifest_path.read_text()) + if reversed_wraps is not None: + wraps = manifest["app_root"]["wraps"] + assert len(wraps) > 1 + manifest["app_root"]["wraps"] = dict(reversed(wraps.items())) + reversed_wraps.extend(manifest["app_root"]["wraps"]) + _stale_dep(manifest, pages[1].route, str(tmp_path / "view.py")) + manifest_path.write_text(json.dumps(manifest)) + # The daemon child starts without the states its pages define. + _unregister_state(_FP_HOLDER.pop("_ProviderState")) + for flag, value in edit.items(): + monkeypatch.setitem(_WRAP_EDIT, flag, value) + result = disk_cache.try_incremental_rebuild( + app, compiler_plugins=[], prerender_routes=False, root=tmp_path + ) + finally: + for name in ("_ProviderState", "_WrapEditState"): + if (cls := _FP_HOLDER.pop(name, None)) is not None: + _unregister_state(cls) + return result, _app_root_file(web), root_code + + +def test_page_gaining_a_provided_wrap_reuses_the_app_root(tmp_path, monkeypatch): + """The app root depends on the union of wraps, not on each page's own set.""" + result, root_file, before = _root_rebuild(tmp_path, monkeypatch, state=True) + assert result is True + assert root_file.read_text() == before + + +def test_app_root_with_reordered_wraps_falls_back(tmp_path, monkeypatch): + """Wraps of one priority nest in page order, so another order is a change.""" + recorded: list[str] = [] + result, root_file, before = _root_rebuild( + tmp_path, monkeypatch, reversed_wraps=recorded + ) + assert isinstance(result, CompileContext) + assert root_file.read_text() == before + + +def test_page_bringing_a_new_wrap_falls_back(tmp_path, monkeypatch): + """A wrap no recorded page provides needs the app root rebuilt in full. + + The attempt hands over the page it evaluated; the root on disk is left + for the full compile to replace. + """ + result, root_file, before = _root_rebuild(tmp_path, monkeypatch, wrap=True) + assert isinstance(result, CompileContext) + assert set(result.compiled_pages) == {"p"} + assert root_file.read_text() == before + + +_COLLIDE = {"one": 0, "two": 0, "two_extra": False, "two_unique": False} + + +def _page_collide_one() -> Component: + _COLLIDE["one"] += 1 + + class CollideState(rx.State): + one: int = 1 + + return rx.el.div(rx.el.p(CollideState.one), _footer()) + + +def _page_collide_two() -> Component: + _COLLIDE["two"] += 1 + + class CollideState(rx.State): + two: int = 2 + + def define_extra() -> None: + class CollideState(rx.State): + extra: int = 3 + + def define_unique() -> None: + class UniqueToTwoState(rx.State): + unique: int = 4 + + if _COLLIDE["two_extra"]: + define_extra() + if _COLLIDE["two_unique"]: + define_unique() + return rx.el.div(rx.el.p(CollideState.two), _footer()) + + +def _rebuild_second_page( + tmp_path, + monkeypatch, + first: Callable[[], Component], + second: Callable[[], Component], + *, + predict: bool, + markers: tuple[str, ...], + prepare: Callable[[], None], +) -> tuple[Any, list[str], list[str]]: + """Compile two pages, then rebuild the second as if its module changed. + + Args: + tmp_path: The project root. + monkeypatch: The pytest monkeypatch fixture. + first: The first page. + second: The second page. + predict: Whether the rebuild predicts dynamic state names. + markers: Name parts of the dynamic states the pages define, dropped + before the rebuild like the daemon child's fresh import does. + prepare: Resets page bookkeeping before the rebuild. + + Returns: + The rebuild result, the second page's states before the edit, and its + recorded states after the rebuild. + """ + web = _use_tmp_web_dir(tmp_path, monkeypatch) + app = rx.App() + app.add_page(first, route="/one") + app.add_page(second, route="/two") + pages = list(app._unevaluated_pages.values()) + ctx = _compile(pages, app=app) + second_states = ctx.stateful_routes[pages[1].route] + _stub_externals(app, monkeypatch) + disk_cache.write_manifest(ctx, pages, ctx.all_imports, root=tmp_path) + manifest_path = web / disk_cache._MANIFEST_FILE + manifest = json.loads(manifest_path.read_text()) + _stale_dep(manifest, pages[1].route, str(tmp_path / "view.py")) + manifest_path.write_text(json.dumps(manifest)) + _drop_dynamic_states(*markers) + prepare() + monkeypatch.setattr(disk_cache, "_predict_dynamic_names", predict) + try: + result = disk_cache.try_incremental_rebuild( + app, compiler_plugins=[], prerender_routes=False, root=tmp_path + ) + finally: + _drop_dynamic_states(*markers) + refreshed = json.loads(manifest_path.read_text()) + return ( + result, + list(second_states), + list(refreshed["pages"][pages[1].route]["state_slice"]["schema"]), + ) + + +def _drop_dynamic_states(*markers: str) -> None: + """Unregister dynamic-module states whose names contain a marker. + + Args: + markers: Class name parts, as in the ``reflex.istate.dynamic`` names. + """ + import reflex.istate.dynamic as istate_dynamic + + for name, cls in list(RegistrationContext.ensure_context().base_states.items()): + if any(to_snake_case(marker) in name for marker in markers): + _unregister_state(cls) + for name in [n for n in vars(istate_dynamic) if any(m in n for m in markers)]: + delattr(istate_dynamic, name) + + +def _rebuild_second_collide_page( + tmp_path, + monkeypatch, + *, + predict: bool, + two_extra: bool = False, + two_unique: bool = False, +) -> tuple[Any, list[str], list[str]]: + """Compile two pages defining a same-named local state, then rebuild the second. + + Args: + tmp_path: The project root. + monkeypatch: The pytest monkeypatch fixture. + predict: Whether the rebuild predicts dynamic state names. + two_extra: Whether the edited second page defines one more such state. + two_unique: Whether the edited second page defines a state no other + page names the same way. + + Returns: + The rebuild result, the second page's states before the edit, and its + recorded states after the rebuild. + """ + + def prepare() -> None: + monkeypatch.setitem(_COLLIDE, "one", 0) + monkeypatch.setitem(_COLLIDE, "two", 0) + monkeypatch.setitem(_COLLIDE, "two_extra", two_extra) + monkeypatch.setitem(_COLLIDE, "two_unique", two_unique) + + result, before, after = _rebuild_second_page( + tmp_path, + monkeypatch, + _page_collide_one, + _page_collide_two, + predict=predict, + markers=("CollideState", "UniqueToTwoState"), + prepare=prepare, + ) + assert before[0].endswith("collide_state_0") + return result, before, after + + +def test_incremental_rebuild_keeps_colliding_local_state_names(tmp_path, monkeypatch): + """A page whose local state took a collision suffix keeps it when it misses. + + Local states are named in evaluation order, and the backend evaluates every + stateful page, so the second page's state stays ``CollideState_0`` there. + Without prediction every page defining such states is re-evaluated. + """ + result, before, after = _rebuild_second_collide_page( + tmp_path, monkeypatch, predict=False + ) + assert result is True + assert after == before + assert (_COLLIDE["one"], _COLLIDE["two"]) == (1, 1) + + +def test_predicted_local_state_names_rebuild_only_the_changed_page( + tmp_path, monkeypatch +): + """With prediction, a verified rebuild evaluates only the changed page.""" + result, before, after = _rebuild_second_collide_page( + tmp_path, monkeypatch, predict=True + ) + assert result is True + assert after == before + assert (_COLLIDE["one"], _COLLIDE["two"]) == (0, 1) + + +def test_prediction_accepts_a_new_state_name_no_other_page_uses(tmp_path, monkeypatch): + """A new local state in a name family of its own cannot shift other pages.""" + result, before, after = _rebuild_second_collide_page( + tmp_path, monkeypatch, predict=True, two_unique=True + ) + assert result is True + assert after[: len(before)] == before + assert after[-1].endswith("unique_to_two_state") + assert (_COLLIDE["one"], _COLLIDE["two"]) == (0, 1) + + +def test_unverified_local_state_prediction_asks_for_a_fresh_compile( + tmp_path, monkeypatch +): + """A page that claims different names than recorded fails the prediction. + + The caller then compiles again in a fresh process without prediction. + """ + with pytest.raises(disk_cache.DynamicStatePredictionError): + _rebuild_second_collide_page( + tmp_path, monkeypatch, predict=True, two_extra=True + ) + + +class SharedCounter(rx.ComponentState): + """A ``ComponentState`` both counter pages create.""" + + value: int = 0 + + @classmethod + def get_component(cls, *children, **props) -> Component: + """Render the instance's value. + + Args: + children: Unused. + props: Unused. + + Returns: + The component. + """ + return rx.el.p(cls.value) + + +_COUNTER = {"one": 0, "two": 0} + + +def _page_counter_one() -> Component: + _COUNTER["one"] += 1 + return rx.el.div(SharedCounter.create(), _footer()) + + +def _page_counter_two() -> Component: + _COUNTER["two"] += 1 + return rx.el.div(SharedCounter.create(), _footer()) + + +def test_predicted_component_state_numbers_rebuild_only_the_changed_page( + tmp_path, monkeypatch +): + """A re-evaluated page draws the ``ComponentState`` number of a full compile. + + Instance numbers are handed out in page order. A fresh import starts the + counter at zero, so the rebuild advances it past the number the first page + drew instead of evaluating that page again. + """ + monkeypatch.setattr(SharedCounter, "_per_component_state_instance_count", 0) + + def prepare() -> None: + monkeypatch.setattr(SharedCounter, "_per_component_state_instance_count", 0) + monkeypatch.setitem(_COUNTER, "one", 0) + monkeypatch.setitem(_COUNTER, "two", 0) + + result, before, after = _rebuild_second_page( + tmp_path, + monkeypatch, + _page_counter_one, + _page_counter_two, + predict=True, + markers=("SharedCounter_n",), + prepare=prepare, + ) + assert result is True + assert before[0].endswith("shared_counter_n2") + assert after == before + assert (_COUNTER["one"], _COUNTER["two"]) == (0, 1) + + +def _route_set_rebuild(tmp_path, monkeypatch, edit) -> tuple[Any, Any, Path]: + """Fully compile two pages, then rebuild after ``edit`` changes the app. + + Args: + tmp_path: The project root. + monkeypatch: The pytest monkeypatch fixture. + edit: Changes the app's page registrations. + + Returns: + The rebuild result, the app and the web directory. + """ + web = _use_tmp_web_dir(tmp_path, monkeypatch) + app = rx.App() + app.add_page(_page_a, route="/a") + app.add_page(_page_c, route="/c") + pages = list(app._unevaluated_pages.values()) + ctx = _compile(pages, app=app) + _stub_externals(app, monkeypatch) + for page_ctx in ctx.compiled_pages.values(): + assert page_ctx.output_path is not None + Path(page_ctx.output_path).parent.mkdir(parents=True, exist_ok=True) + Path(page_ctx.output_path).write_text(page_ctx.output_code or "") + disk_cache.write_manifest(ctx, pages, ctx.all_imports, root=tmp_path) + edit(app) + evaluated: list[str] = [] + original = disk_cache.CompileContext.compile + + def recording_compile(self, *args, **kwargs): + evaluated.extend(page.route for page in self.pages) + return original(self, *args, **kwargs) + + monkeypatch.setattr(disk_cache.CompileContext, "compile", recording_compile) + result = disk_cache.try_incremental_rebuild( + app, compiler_plugins=[], prerender_routes=False, root=tmp_path + ) + return result, evaluated, web + + +def _add_b(app) -> None: + app.add_page(_page_b, route="/b") + + +def _remove_c(app) -> None: + del app._unevaluated_pages["c"] + + +def _reorder(app) -> None: + app._unevaluated_pages = dict(reversed(app._unevaluated_pages.items())) + + +def _retitle_c(app) -> None: + import dataclasses + + app._unevaluated_pages["c"] = dataclasses.replace( + app._unevaluated_pages["c"], title="Renamed" + ) + + +@pytest.mark.parametrize( + ("edit", "reason"), + [ + (_add_b, "added b"), + (_remove_c, "removed c"), + (_reorder, "order changed"), + (_retitle_c, "registered differently"), + ], +) +def test_page_registry_change_falls_back(tmp_path, monkeypatch, caplog, edit, reason): + """The page registry is an input of every page (a sidebar lists them). + + Adding, removing, reordering or re-registering a page (its title, say) + therefore re-evaluates every page, which a full compile does. + """ + from reflex.compiler import utils as compiler_utils + + with caplog.at_level("INFO", logger=disk_cache.logger.name): + result, evaluated, web = _route_set_rebuild(tmp_path, monkeypatch, edit) + assert result is False + assert evaluated == [] + assert reason in caplog.text + # Nothing was written or removed: the full compile does that. + assert Path(compiler_utils.get_page_path("a")).exists() + assert Path(compiler_utils.get_page_path("c")).exists() + manifest = json.loads((web / disk_cache._MANIFEST_FILE).read_text()) + assert list(manifest["pages"]) == ["a", "c"] + + +def test_incremental_rebuild_copies_assets(tmp_path, monkeypatch): + """An assets-only edit is an all-hit rebuild, so the incremental path must + run the same assets -> public copy as the full compile. + """ + web = _use_tmp_web_dir(tmp_path, monkeypatch) + assets = tmp_path / "assets" + assets.mkdir() + (assets / "logo.svg").write_text("") + + app = rx.App() + app.add_page(_page_a, route="/a") + pages = list(app._unevaluated_pages.values()) + ctx = _compile(pages) + disk_cache.write_manifest(ctx, pages, ctx.all_imports, root=tmp_path) + _stub_externals(app, monkeypatch) + + assert ( + disk_cache.try_incremental_rebuild( + app, compiler_plugins=[], prerender_routes=False, root=tmp_path + ) + is True + ) + + assert (web / "public" / "logo.svg").read_text() == "" + + +@rx.memo +def _badge(text: str) -> Component: + return rx.el.span(text, class_name="badge") + + +def _page_with_user_memo() -> Component: + return rx.el.div(rx.el.h1("Memo page"), _badge(text="hello")) + + +def test_incremental_rebuild_rewrites_changed_user_memo( + tmp_path, monkeypatch, preserve_memo_registries +): + """Editing a user ``@rx.memo`` module must rewrite its mirrored memo file. + + The memo's module file is in the dep set of every page that imports it, so + those pages miss — but only auto-memo contributions were being written, + leaving the user memo's generated JS stale. + """ + from reflex.compiler import compiler + from reflex.compiler import utils as compiler_utils + + web = _use_tmp_web_dir(tmp_path, monkeypatch) + + app = rx.App() + app.add_page(_page_with_user_memo, route="/memo") + pages = list(app._unevaluated_pages.values()) + memo_route = pages[0].route + ctx = _compile(pages) + disk_cache.write_manifest(ctx, pages, ctx.all_imports, root=tmp_path) + _stub_externals(app, monkeypatch) + + # Simulate an edit to this module (which defines the user memo): record an + # unmatchable entry for its file, which both the page's dep set (so the + # importing page misses) and the stored memo record's closure (so the memo + # file is dirty) resolve through the shared file table. + module_file = str(Path(__file__).resolve()) + manifest_path = web / disk_cache._MANIFEST_FILE + manifest = json.loads(manifest_path.read_text()) + memo_path = disk_cache._memo_output_path(__name__) + assert memo_path is not None + assert module_file in manifest["memo_files"][memo_path]["deps"] + _stale_dep(manifest, memo_route, module_file) + manifest_path.write_text(json.dumps(manifest)) + + assert ( + disk_cache.try_incremental_rebuild( + app, compiler_plugins=[], prerender_routes=False, root=tmp_path + ) + is True + ) + + # The user memo's mirrored file was re-emitted with its export. + from reflex_base.components.memo import MEMOS, MemoComponentDefinition + + badge_def = next(m for m in MEMOS.values() if m.source_module == __name__) + assert isinstance(badge_def, MemoComponentDefinition) + memo_files, _ = compiler.compile_memo_components([badge_def]) + assert memo_files + for mpath, _mcode in memo_files: + out_path = compiler_utils.resolve_path_of_web_dir(mpath) + assert out_path.exists() + assert badge_def.export_name in out_path.read_text(encoding="utf-8") + + +def test_new_memo_module_is_emitted_on_first_use( + tmp_path, monkeypatch, preserve_memo_registries +): + """A user memo module the previous compile never saw must be emitted. + + Mirrors the review repro: a page starts importing a ``@rx.memo`` from a + brand-new module. The page misses (its importer changed), but the memo's + module file is absent from every recorded dependency set — no + changed-file reasoning can select it — while the recompiled page imports + the memo's mirrored file, which must therefore be written. + """ + from reflex_base.components.memo import MEMOS, MemoComponentDefinition + + from reflex.compiler import compiler + from reflex.compiler import utils as compiler_utils + + web = _use_tmp_web_dir(tmp_path, monkeypatch) + + app = rx.App() + app.add_page(_page_with_user_memo, route="/memo") + pages = list(app._unevaluated_pages.values()) + memo_route = pages[0].route + ctx = _compile(pages) + disk_cache.write_manifest(ctx, pages, ctx.all_imports, root=tmp_path) + _stub_externals(app, monkeypatch) + + # Rewrite the manifest as if the previous compile never saw the memo's + # module: the page's recorded deps name only an unrelated changed file + # (so it misses), and no recorded memo state mentions the module. + manifest_path = web / disk_cache._MANIFEST_FILE + manifest = json.loads(manifest_path.read_text()) + _stale_dep(manifest, memo_route, str(tmp_path / "app.py")) + manifest["memo_files"] = {} + manifest_path.write_text(json.dumps(manifest)) + + assert ( + disk_cache.try_incremental_rebuild( + app, compiler_plugins=[], prerender_routes=False, root=tmp_path + ) + is True + ) + + badge_def = next(m for m in MEMOS.values() if m.source_module == __name__) + assert isinstance(badge_def, MemoComponentDefinition) + memo_files, _ = compiler.compile_memo_components([badge_def]) + assert memo_files + for mpath, _mcode in memo_files: + out_path = compiler_utils.resolve_path_of_web_dir(mpath) + assert out_path.exists(), "new memo module file was never written" + assert badge_def.export_name in out_path.read_text(encoding="utf-8") + + +def test_route_owned_memo_entry_carried_while_owner_hits( + tmp_path, monkeypatch, preserve_memo_registries +): + """A route-owned memo entry survives rebuilds while its owner page hits. + + The pre-evaluation registry can't see memos from modules imported inside + a page function; the entry's owner attribution keeps its on-disk file and + manifest record intact without re-evaluating anything. + """ + web = _use_tmp_web_dir(tmp_path, monkeypatch) + + app = rx.App() + app.add_page(_page_a, route="/a") + pages = list(app._unevaluated_pages.values()) + owner_route = pages[0].route + ctx = _compile(pages) + disk_cache.write_manifest(ctx, pages, ctx.all_imports, root=tmp_path) + _stub_externals(app, monkeypatch) + + lazy_file = web / "app_components" / "lazy_module.jsx" + lazy_file.parent.mkdir(parents=True, exist_ok=True) + lazy_file.write_text("export const Lazy = () => null;") + manifest_path = web / disk_cache._MANIFEST_FILE + manifest = json.loads(manifest_path.read_text()) + manifest["memo_files"][str(lazy_file)] = { + "def_keys": [["Lazy", "lazy_module"]], + "deps": [], + "owner_route": owner_route, + } + manifest_path.write_text(json.dumps(manifest)) + + assert ( + disk_cache.try_incremental_rebuild( + app, compiler_plugins=[], prerender_routes=False, root=tmp_path + ) + is True + ) + + assert lazy_file.read_text() == "export const Lazy = () => null;" + # The all-hit rebuild left the manifest record (including the entry) intact. + written = json.loads(manifest_path.read_text()) + assert written["memo_files"][str(lazy_file)]["owner_route"] == owner_route + + +def test_new_lazy_memo_module_is_emitted_on_first_use( + tmp_path, monkeypatch, preserve_memo_registries +): + """A new memo module imported inside a page function must be emitted. + + The pre-evaluation registry cannot see this module, and the changed-file + net cannot see a new file. The post-evaluation pass over the recompiled + routes' owned memos must demand its output file. + """ + import importlib + + module_name = "lazy_memo_mod_t1" + (tmp_path / f"{module_name}.py").write_text( + "import reflex as rx\n" + "\n" + "@rx.memo\n" + "def lazy_badge(text: str) -> rx.Component:\n" + " return rx.el.span(text)\n" + ) + monkeypatch.syspath_prepend(str(tmp_path)) + + def _page_lazy() -> Component: + mod = importlib.import_module(module_name) + return rx.el.div(mod.lazy_badge(text="hi")) + + web = _use_tmp_web_dir(tmp_path, monkeypatch) + app = rx.App() + app.add_page(_page_lazy, route="/lazy") + pages = list(app._unevaluated_pages.values()) + lazy_route = pages[0].route + try: + ctx = _compile(pages) + disk_cache.write_manifest(ctx, pages, ctx.all_imports, root=tmp_path) + _stub_externals(app, monkeypatch) + + manifest_path = web / disk_cache._MANIFEST_FILE + manifest = json.loads(manifest_path.read_text()) + lazy_path = disk_cache._memo_output_path(module_name) + assert lazy_path is not None + # Sanity: the full compile recorded the entry as owned by the page. + assert manifest["memo_files"][lazy_path]["owner_route"] == lazy_route + + # Surgery: the previous compile never saw the lazy module, and the + # page misses via an unrelated changed file. + del manifest["memo_files"][lazy_path] + _stale_dep(manifest, lazy_route, str(tmp_path / "app.py")) + manifest_path.write_text(json.dumps(manifest)) + + assert ( + disk_cache.try_incremental_rebuild( + app, compiler_plugins=[], prerender_routes=False, root=tmp_path + ) + is True + ) + + from reflex_base.components.memo import MEMOS, MemoComponentDefinition + + lazy_def = next(m for m in MEMOS.values() if m.source_module == module_name) + assert isinstance(lazy_def, MemoComponentDefinition) + out_path = Path(lazy_path) + assert out_path.exists(), "lazy memo module file was never written" + assert lazy_def.export_name in out_path.read_text(encoding="utf-8") + # The refreshed manifest re-records the entry with its owner. + written = json.loads(manifest_path.read_text()) + assert written["memo_files"][lazy_path]["owner_route"] == lazy_route + finally: + sys.modules.pop(module_name, None) + + +class _MemoCacheState(rx.State): + value: str = "x" + other: str = "y" + + +_SHARED_MEMO = {"edit_one": False} + + +def _page_shared_one() -> Component: + if _SHARED_MEMO["edit_one"]: + return rx.el.div(rx.el.p(_MemoCacheState.other, class_name="one")) + return rx.el.div(rx.el.p(_MemoCacheState.value, class_name="shared")) + + +def _page_shared_two() -> Component: + return rx.el.div(rx.el.p(_MemoCacheState.value, class_name="shared")) + + +def _memo_file_of(page_ctx: PageContext) -> Path: + """The single auto memo file a test page's output imports. + + Args: + page_ctx: The compiled page. + + Returns: + The memo file path. + """ + [(tag, _)] = page_ctx.memo_uses + return Path(disk_cache._memo_file(tag)) + + +def test_incremental_miss_leaves_the_memo_a_reused_page_shares( + tmp_path, monkeypatch, preserve_memo_registries +): + """A miss never touches the memo files of the pages it reuses. + + Pages one and two render the same stateful subtree, so both import the + same content-addressed auto memo. Editing page one gives it another + memo; the shared memo must survive for page two, which still imports it. + """ + from reflex.compiler import compiler + from reflex.compiler import utils as compiler_utils + + web = _use_tmp_web_dir(tmp_path, monkeypatch) + app = rx.App() + app.add_page(_page_shared_one, route="/one") + app.add_page(_page_shared_two, route="/two") + pages = list(app._unevaluated_pages.values()) + route_one, route_two = pages[0].route, pages[1].route + ctx = _compile(pages) + one_ctx, two_ctx = ctx.compiled_pages[route_one], ctx.compiled_pages[route_two] + shared = _memo_file_of(one_ctx) + assert shared == _memo_file_of(two_ctx), "both pages must share the memo" + # The full compile's on-disk memo state. + memo_files, _ = compiler.compile_memo_components(ctx.auto_memo_components.values()) + assert [Path(path) for path, _ in memo_files] == [shared] + for mpath, mcode in memo_files: + compiler_utils.write_file(compiler_utils.resolve_path_of_web_dir(mpath), mcode) + compiler_utils.prune_stale_memo_files(path for path, _ in memo_files) + disk_cache.write_manifest(ctx, pages, ctx.all_imports, root=tmp_path) + _stub_externals(app, monkeypatch) + manifest_path = web / disk_cache._MANIFEST_FILE + manifest = json.loads(manifest_path.read_text()) + [tag] = manifest["pages"][route_two]["memo_keys"] + assert manifest["pages"][route_one]["memo_keys"] == [tag] + assert manifest["app_root"]["memo_keys"] == [] + + # Edit page one so it renders another subtree, and make only it miss. + monkeypatch.setitem(_SHARED_MEMO, "edit_one", True) + _stale_dep(manifest, route_one, str(tmp_path / "data.md")) + manifest_path.write_text(json.dumps(manifest)) + assert ( + disk_cache.try_incremental_rebuild( + app, compiler_plugins=[], prerender_routes=False, root=tmp_path + ) + is True + ) + + # The shared memo is still there for page two; page one got a new one. + assert shared.exists() + refreshed = json.loads(manifest_path.read_text()) + [new_tag] = refreshed["pages"][route_one]["memo_keys"] + assert new_tag != tag + assert refreshed["pages"][route_two]["memo_keys"] == [tag] + new_file = Path(disk_cache._memo_file(new_tag)) + assert new_file.exists() + page_one = Path(compiler_utils.get_page_path(route_one)).read_text() + assert new_tag in page_one + assert tag not in page_one + memo_manifest = compiler_utils._read_memo_manifest(web) + assert memo_manifest == { + str(path.relative_to(web)).replace(os.sep, "/") for path in (shared, new_file) + } + + +def test_incremental_miss_drops_the_memo_no_page_uses( + tmp_path, monkeypatch, preserve_memo_registries +): + """A memo only the recompiled page used is deleted once it stops using it.""" + from reflex.compiler import compiler + from reflex.compiler import utils as compiler_utils + + web = _use_tmp_web_dir(tmp_path, monkeypatch) + app = rx.App() + app.add_page(_page_shared_one, route="/one") + pages = list(app._unevaluated_pages.values()) + route_one = pages[0].route + ctx = _compile(pages) + old_file = _memo_file_of(ctx.compiled_pages[route_one]) + memo_files, _ = compiler.compile_memo_components(ctx.auto_memo_components.values()) + for mpath, mcode in memo_files: + compiler_utils.write_file(compiler_utils.resolve_path_of_web_dir(mpath), mcode) + compiler_utils.prune_stale_memo_files(path for path, _ in memo_files) + disk_cache.write_manifest(ctx, pages, ctx.all_imports, root=tmp_path) + _stub_externals(app, monkeypatch) + manifest_path = web / disk_cache._MANIFEST_FILE + manifest = json.loads(manifest_path.read_text()) + monkeypatch.setitem(_SHARED_MEMO, "edit_one", True) + _stale_dep(manifest, route_one, str(tmp_path / "data.md")) + manifest_path.write_text(json.dumps(manifest)) + assert ( + disk_cache.try_incremental_rebuild( + app, compiler_plugins=[], prerender_routes=False, root=tmp_path + ) + is True + ) + assert not old_file.exists() + refreshed = json.loads(manifest_path.read_text()) + [new_tag] = refreshed["pages"][route_one]["memo_keys"] + new_relative = str(Path(disk_cache._memo_file(new_tag)).relative_to(web)).replace( + os.sep, "/" + ) + assert compiler_utils._read_memo_manifest(web) == {new_relative} + + +def test_failed_manifest_refresh_leaves_no_manifest( + tmp_path, monkeypatch, preserve_memo_registries +): + """A refresh that raises still rebuilds, and leaves no stale manifest.""" + web = _use_tmp_web_dir(tmp_path, monkeypatch) + app = rx.App() + app.add_page(_page_shared_one, route="/one") + pages = list(app._unevaluated_pages.values()) + ctx = _compile(pages) + disk_cache.write_manifest(ctx, pages, ctx.all_imports, root=tmp_path) + _stub_externals(app, monkeypatch) + manifest_path = web / disk_cache._MANIFEST_FILE + manifest = json.loads(manifest_path.read_text()) + _stale_dep(manifest, pages[0].route, str(tmp_path / "data.md")) + manifest_path.write_text(json.dumps(manifest)) + + def fail(*args, **kwargs): + msg = "boom" + raise ValueError(msg) + + monkeypatch.setattr(disk_cache, "_update_manifest_for_misses", fail) + assert ( + disk_cache.try_incremental_rebuild( + app, compiler_plugins=[], prerender_routes=False, root=tmp_path + ) + is True + ) + assert not manifest_path.exists() + + +def test_incremental_miss_keeps_the_memo_the_app_root_shares( + tmp_path, monkeypatch, preserve_memo_registries +): + """A memo the app root also imports survives the page dropping it.""" + from reflex.compiler import compiler + from reflex.compiler import utils as compiler_utils + + web = _use_tmp_web_dir(tmp_path, monkeypatch) + app = rx.App() + app.add_page(_page_shared_one, route="/one") + pages = list(app._unevaluated_pages.values()) + route_one = pages[0].route + ctx = _compile(pages) + shared = _memo_file_of(ctx.compiled_pages[route_one]) + # An app wrap rendering the same stateful subtree imports the same memo. + compiler._memoize_stateful_app_wraps( + rx.fragment(rx.el.p(_MemoCacheState.value, class_name="shared")), ctx + ) + memo_files, _ = compiler.compile_memo_components(ctx.auto_memo_components.values()) + assert [Path(path) for path, _ in memo_files] == [shared] + for mpath, mcode in memo_files: + compiler_utils.write_file(compiler_utils.resolve_path_of_web_dir(mpath), mcode) + compiler_utils.prune_stale_memo_files(path for path, _ in memo_files) + disk_cache.write_manifest(ctx, pages, ctx.all_imports, root=tmp_path) + _stub_externals(app, monkeypatch) + manifest_path = web / disk_cache._MANIFEST_FILE + manifest = json.loads(manifest_path.read_text()) + assert ( + manifest["app_root"]["memo_keys"] == manifest["pages"][route_one]["memo_keys"] + ) + monkeypatch.setitem(_SHARED_MEMO, "edit_one", True) + _stale_dep(manifest, route_one, str(tmp_path / "data.md")) + manifest_path.write_text(json.dumps(manifest)) + assert ( + disk_cache.try_incremental_rebuild( + app, compiler_plugins=[], prerender_routes=False, root=tmp_path + ) + is True + ) + assert shared.exists() + + +def _page_radix() -> Component: + return rx.el.div(rx.button("Radix"), _footer()) + + +def _page_memoized_radix() -> Component: + return rx.el.div( + rx.text("x", color=_MemoCacheState.value, on_click=rx.console_log("c")) + ) + + +def test_memoized_radix_component_is_recorded_as_radix(tmp_path, monkeypatch): + """A Radix component auto-memoization moved out of the page tree counts.""" + from reflex_components_radix.plugin import RadixThemesPlugin + + web = _use_tmp_web_dir(tmp_path, monkeypatch) + app = rx.App() + app.add_page(_page_memoized_radix, route="/m") + pages = list(app._unevaluated_pages.values()) + ctx = _compile(pages, plugins=[RadixThemesPlugin.create_implicit()]) + page_ctx = ctx.compiled_pages[pages[0].route] + assert page_ctx.memo_uses, "the Radix text must be auto-memoized" + disk_cache.write_manifest(ctx, pages, ctx.all_imports, root=tmp_path) + manifest = json.loads((web / disk_cache._MANIFEST_FILE).read_text()) + assert manifest["pages"][pages[0].route]["radix"] is True + + +def test_reused_radix_page_keeps_the_implicit_plugin_enabled(tmp_path, monkeypatch): + """A rebuild that compiles no Radix page still ships the Radix stylesheet. + + The implicit plugin enables itself while compiling a Radix component; a + page reused from disk is not compiled, so the manifest records the page's + Radix use and the rebuild enables the plugin from it. + """ + from reflex_components_radix.plugin import RadixThemesPlugin + + web = _use_tmp_web_dir(tmp_path, monkeypatch) + app = rx.App() + app.add_page(_page_radix, route="/radix") + app.add_page(_page_a, route="/a") + pages = list(app._unevaluated_pages.values()) + ctx = _compile(pages, plugins=[RadixThemesPlugin.create_implicit()]) + _stub_externals(app, monkeypatch) + disk_cache.write_manifest(ctx, pages, ctx.all_imports, root=tmp_path) + manifest_path = web / disk_cache._MANIFEST_FILE + manifest = json.loads(manifest_path.read_text()) + assert manifest["pages"][pages[0].route]["radix"] is True + assert manifest["pages"][pages[1].route]["radix"] is False + + # An all-hit rebuild, and one that recompiles only the plain page. + for stale in (None, pages[1].route): + if stale is not None: + _stale_dep(manifest, stale, str(tmp_path / "view.py")) + manifest_path.write_text(json.dumps(manifest)) + plugin = RadixThemesPlugin.create_implicit() + assert not plugin.enabled + assert ( + disk_cache.try_incremental_rebuild( + app, compiler_plugins=[plugin], prerender_routes=False, root=tmp_path + ) + is True + ) + assert plugin.enabled + assert plugin.get_stylesheet_paths() + + +def test_recompiled_page_leaves_a_hit_pages_state_to_it(tmp_path, monkeypatch): + """A state a hit page recorded is not claimed by a recompiled page. + + In a process that evaluates only the recompiled pages, one of them can + be the first to import the module defining a state another page owns. + """ + miss_ctx = SimpleNamespace( + stateful_routes={ + "/miss": ["app.shared", "app.own"], + "/other": ["app.shared"], + } + ) + manifest = _manifest({ + "/hit": {"state_slice": {"schema": {"app.shared": []}}}, + "/miss": {"state_slice": None}, + "/other": {"state_slice": None}, + }) + disk_cache._disown_hit_page_states( + cast(Any, miss_ctx), manifest, {"/miss", "/other"} + ) + assert miss_ctx.stateful_routes == {"/miss": ["app.own"]} + + +def test_load_manifest_rejects_wrong_schema(tmp_path, monkeypatch): + web = _use_tmp_web_dir(tmp_path, monkeypatch) + (web / disk_cache._MANIFEST_FILE).write_text(json.dumps({"schema": 999})) + assert disk_cache.load_manifest() is None + + +def test_update_manifest_for_misses_keeps_complete_imports(tmp_path, monkeypatch): + web = _use_tmp_web_dir(tmp_path, monkeypatch) + monkeypatch.setattr( + page_cache, "state_dependency_index", lambda root=None: ({}, set()) + ) + monkeypatch.setattr(page_cache, "page_dependency_entries", lambda *a, **k: []) + + page = _FakePage(route="/a", component=_page_a) + page_ctx = SimpleNamespace( + root_component=rx.el.div(), + app_wrap_components={}, + frontend_imports={}, + memo_uses={}, + ) + miss_ctx = SimpleNamespace(compiled_pages={"/a": page_ctx}, stateful_routes={}) + complete_imports = {"memo-lib": [ImportVar("MemoThing")]} + manifest = _manifest({"/a": {"deps": [], "app_wraps": {}, "is_stateful": False}}) + + assert disk_cache._update_manifest_for_misses( + manifest, + cast(Any, miss_ctx), + [page], + complete_imports, + root=tmp_path, + memo_state={}, + memo_file_entries={}, + registrations={"/a": "recorded"}, + module_states=manifest["module_states"], + ) + disk_cache._write(manifest) + + written = json.loads((web / disk_cache._MANIFEST_FILE).read_text()) + assert disk_cache._deserialize_imports(written["all_imports"]) == complete_imports + + +def test_update_manifest_refreshes_imports_without_misses(tmp_path, monkeypatch): + """A memo-only rewrite that adds a package must reach the manifest.""" + web = _use_tmp_web_dir(tmp_path, monkeypatch) + manifest = _manifest({}, all_imports={"old-lib": [{"tag": "Old"}]}) + memo_imports = {"old-lib": [ImportVar("Old")], "memo-lib": [ImportVar("Memo")]} + + assert disk_cache._update_manifest_for_misses( + manifest, + None, + [], + memo_imports, + root=tmp_path, + memo_state={}, + memo_file_entries={}, + registrations={"/a": "recorded"}, + module_states=manifest["module_states"], + ) + disk_cache._write(manifest) + + written = json.loads((web / disk_cache._MANIFEST_FILE).read_text()) + assert disk_cache._deserialize_imports(written["all_imports"]) == memo_imports + # Nothing changed: nothing to write. + assert not disk_cache._update_manifest_for_misses( + written, + None, + [], + memo_imports, + root=tmp_path, + memo_state={}, + memo_file_entries={}, + registrations={"/a": "recorded"}, + module_states=written["module_states"], + ) + + +def _page_loop_a() -> Component: + return rx.el.ul(rx.foreach(["a", "b"], lambda item: rx.el.li(item))) + + +def _page_loop_b() -> Component: + return rx.el.ul(rx.foreach(["c", "d"], lambda item: rx.el.li(item))) + + +def _page_loop_c() -> Component: + return rx.el.ul(rx.foreach(["e", "f"], lambda item: rx.el.li(item))) + + +def test_generated_names_do_not_depend_on_other_pages(): + """A page compiled alone gets the same generated names as in a full compile. + + ``rx.foreach`` draws its loop variable from the unique-name generator. If + that sequence were process-wide, an incremental compile of one page would + hand it the names the cached pages already use. + """ + full = _compile([ + _FakePage(route="/la", component=_page_loop_a), + _FakePage(route="/lb", component=_page_loop_b), + _FakePage(route="/lc", component=_page_loop_c), + ]) + alone = _compile([_FakePage(route="/lc", component=_page_loop_c)]) + assert ( + full.compiled_pages["/lc"].output_code + == alone.compiled_pages["/lc"].output_code + ) + # Different pages still draw different names. + la, lb = full.compiled_pages["/la"], full.compiled_pages["/lb"] + assert la.output_code is not None + assert lb.output_code is not None + assert la.output_code != lb.output_code + + +_TWO_PAGE_MODULE = """\ +import reflex as rx + +CALLS = {"a": 0, "b": 0} + + +def page_a(): + CALLS["a"] += 1 + return rx.el.p("a") + + +def page_b(): + CALLS["b"] += 1 + return rx.el.p("b") +""" + + +@pytest.mark.parametrize("hinted", [True, False]) +def test_incremental_rebuild_recompiles_only_the_edited_function( + tmp_path, monkeypatch, hinted +): + """Editing one page function recompiles only that page of its module. + + The watcher reports the module file; a page tracking its function's slice + of that file must still be validated. + """ + import importlib + + module_name = "two_page_module_t7" + module = tmp_path / f"{module_name}.py" + module.write_text(_TWO_PAGE_MODULE) + monkeypatch.syspath_prepend(str(tmp_path)) + web = _use_tmp_web_dir(tmp_path, monkeypatch) + try: + mod = importlib.import_module(module_name) + app = rx.App() + app.add_page(mod.page_a, route="/a") + app.add_page(mod.page_b, route="/b") + pages = list(app._unevaluated_pages.values()) + ctx = _compile(pages, app=app) + disk_cache.write_manifest(ctx, pages, ctx.all_imports, root=tmp_path) + _stub_externals(app, monkeypatch) + manifest = json.loads((web / disk_cache._MANIFEST_FILE).read_text()) + assert disk_cache.manifest_dependency_files(manifest) >= {str(module.resolve())} + + module.write_text(_TWO_PAGE_MODULE.replace('rx.el.p("a")', 'rx.el.p("a2")')) + mod.CALLS.update(a=0, b=0) + disk_cache.set_changed_hint({module} if hinted else None) + result = disk_cache.try_incremental_rebuild( + app, compiler_plugins=[], prerender_routes=False, root=tmp_path + ) + finally: + disk_cache.set_changed_hint(None) + sys.modules.pop(module_name, None) + assert result is True + assert mod.CALLS == {"a": 1, "b": 0} + + +def _labelled(label: str) -> Component: + return rx.el.p(label) + + +def test_registration_digest_tracks_partial_arguments(): + """A page bound with ``functools.partial`` changes with its arguments.""" + import functools + + def digest(label: str) -> str: + page = _FakePage(route="/p", component=functools.partial(_labelled, label)) + return disk_cache.registration_digests([page])["/p"] + + assert digest("old") == digest("old") + assert digest("old") != digest("new") + + +def test_partial_page_depends_on_its_function_module(tmp_path, monkeypatch): + """A ``functools.partial`` page depends on its wrapped function's file.""" + import functools + import importlib + + module = tmp_path / "partial_view_module.py" + module.write_text("def view(label):\n return label\n") + monkeypatch.syspath_prepend(str(tmp_path)) + try: + view = importlib.import_module("partial_view_module").view + files = page_cache._component_source_files( + functools.partial(view, "x"), tmp_path.resolve() + ) + finally: + sys.modules.pop("partial_view_module", None) + assert files == {str(module.resolve())} + + +@pytest.mark.parametrize("predict", [False, True]) +def test_expand_miss_pages_keeps_page_order(monkeypatch, predict): + """Re-evaluated pages are evaluated in page order, whatever added them. + + ``ComponentState`` numbers and memo emit order follow evaluation order. + """ + pages = [_FakePage(f"/{name}", _page_a) for name in "abcd"] + + def entry(claims: list[list[str]]) -> dict[str, Any]: + return {"dynamic_claims": claims} + + manifest = { + "pages": { + "/a": entry([]), + "/b": entry([["LocalState", "LocalState"]]), + "/c": entry([]), + "/d": entry([]), + } + } + monkeypatch.setattr(disk_cache, "_predict_dynamic_names", predict) + misses = disk_cache._expand_miss_pages([pages[3]], pages, manifest, {"/a"}) + expected = ["/a", "/d"] if predict else ["/a", "/b", "/d"] + assert [page.route for page in misses] == expected + + +@pytest.mark.parametrize("predict", [False, True]) +def test_reordering_pages_falls_back_before_evaluating(tmp_path, monkeypatch, predict): + """A reorder changes the page registry, which every page may read.""" + web = _use_tmp_web_dir(tmp_path, monkeypatch) + app = rx.App() + app.add_page(_page_collide_one, route="/one") + app.add_page(_page_collide_two, route="/two") + pages = list(app._unevaluated_pages.values()) + ctx = _compile(pages, app=app) + _stub_externals(app, monkeypatch) + disk_cache.write_manifest(ctx, pages, ctx.all_imports, root=tmp_path) + assert (web / disk_cache._MANIFEST_FILE).exists() + monkeypatch.setattr( + app, "_unevaluated_pages", dict(reversed(app._unevaluated_pages.items())) + ) + _drop_dynamic_states("CollideState") + monkeypatch.setitem(_COLLIDE, "one", 0) + monkeypatch.setitem(_COLLIDE, "two", 0) + monkeypatch.setattr(disk_cache, "_predict_dynamic_names", predict) + try: + result = disk_cache.try_incremental_rebuild( + app, compiler_plugins=[], prerender_routes=False, root=tmp_path + ) + finally: + _drop_dynamic_states("CollideState") + assert result is False + assert (_COLLIDE["one"], _COLLIDE["two"]) == (0, 0) + + +def test_manifest_dependency_files_cover_every_recorded_input(): + """The daemon watches, and checks at startup, every input the manifest records. + + Import-time state files and memo source closures belong to no page, so + they are listed alongside the page dependencies and global inputs. + """ + manifest = { + "pages": {"index": {"deps": ["/app/pages.py::index", "/app/data.md"]}}, + "globals": {"reflex": "0.0.0", "rxconfig": "/app/rxconfig.py"}, + "globals_absent": ["/app/uv.lock"], + "module_states": {"files": ["/app/state.py"]}, + "memo_files": {"memo.js": {"deps": ["/lib/memo.py"]}}, + } + assert disk_cache.manifest_dependency_entries(manifest) == { + "/app/pages.py::index", + "/app/data.md", + "/app/rxconfig.py", + "/app/state.py", + "/lib/memo.py", + } + assert disk_cache.manifest_dependency_files(manifest) == { + "/app/pages.py", + "/app/data.md", + "/app/rxconfig.py", + "/app/uv.lock", + "/app/state.py", + "/lib/memo.py", + } diff --git a/tests/units/compiler/test_memoize_plugin.py b/tests/units/compiler/test_memoize_plugin.py index ffea8c7ca2d..8da00c116a0 100644 --- a/tests/units/compiler/test_memoize_plugin.py +++ b/tests/units/compiler/test_memoize_plugin.py @@ -22,7 +22,6 @@ ) from reflex_base.constants.compiler import MemoizationDisposition, MemoizationMode from reflex_base.plugins import CompileContext, CompilerHooks, PageContext -from reflex_base.utils import memo_paths from reflex_base.utils.imports import ImportVar from reflex_base.vars import VarData from reflex_base.vars.base import Field, LiteralVar, Var, field @@ -857,7 +856,7 @@ def fake_create_passthrough_component_memo( first_compile = SimpleNamespace(memoize_wrappers={}, auto_memo_components={}) second_compile = SimpleNamespace(memoize_wrappers={}, auto_memo_components={}) - page_context = cast(PageContext, SimpleNamespace(source_module=None)) + page_context = cast(PageContext, SimpleNamespace(source_module=None, memo_uses={})) MemoizeStatefulPlugin._build_wrapper( first_component, @@ -898,15 +897,13 @@ def test_shared_subtree_across_pages_uses_same_tag() -> None: assert f"jsx({tag}," in output -def test_shared_subtree_in_distinct_source_modules_emits_per_module() -> None: - """Identical subtrees in different user modules emit one memo per module. +def test_shared_subtree_in_distinct_source_modules_shares_one_memo() -> None: + """Identical subtrees in different user modules share one auto memo. - Regression: the auto-memo registry was keyed by tag only, so when two - pages from different user modules produced the same memoizable subtree - (and therefore the same tag), the second registration overwrote the - first. Only the surviving definition's source module got a mirrored - memo file, leaving the other page importing a tag from a file that - never exported it. + Auto memos are content-addressed by their tag and compile to one file + each under ``utils/components/``, whatever module the page lives in, so + the two pages import the same file and neither page's memo files hold + another page's memos. """ from reflex.compiler.compiler import compile_memo_components @@ -930,43 +927,20 @@ def test_shared_subtree_in_distinct_source_modules_emits_per_module() -> None: assert len(ctx.memoize_wrappers) == 1 tag = next(iter(ctx.memoize_wrappers)) - # Both source modules survive registration as distinct entries. - assert set(ctx.auto_memo_components) == { - (tag, "memo_collision_test.module_a"), - (tag, "memo_collision_test.module_b"), - } - - # Each module emits and imports the tag under a per-module symbol so the - # two pages never collide on a shared JS identifier. - symbol_a = memo_paths.mirrored_symbol(tag, "memo_collision_test.module_a") - symbol_b = memo_paths.mirrored_symbol(tag, "memo_collision_test.module_b") - page_a_output = ctx.compiled_pages["/a"].output_code or "" - page_b_output = ctx.compiled_pages["/b"].output_code or "" - assert ( - f'import {{{symbol_a}}} from "$/app_components/memo_collision_test/module_a"' - in page_a_output - ) - assert ( - f'import {{{symbol_b}}} from "$/app_components/memo_collision_test/module_b"' - in page_b_output - ) + assert set(ctx.auto_memo_components) == {(tag, None)} + for route in ("/a", "/b"): + page_ctx = ctx.compiled_pages[route] + assert set(page_ctx.memo_uses) == {(tag, None)} + assert f'import {{{tag}}} from "$/utils/components/{tag}"' in ( + page_ctx.output_code or "" + ) output_files, _ = compile_memo_components( memos=tuple(ctx.auto_memo_components.values()), ) - emitted = {Path(path).as_posix(): code for path, code in output_files} - - def find_emitted(suffix: str) -> str | None: - return next( - (code for path, code in emitted.items() if path.endswith(suffix)), None - ) - - matched_a = find_emitted("memo_collision_test/module_a.jsx") - matched_b = find_emitted("memo_collision_test/module_b.jsx") - assert matched_a is not None, f"missing module_a memo file in {sorted(emitted)}" - assert matched_b is not None, f"missing module_b memo file in {sorted(emitted)}" - assert f"const {symbol_a} = memo" in matched_a - assert f"const {symbol_b} = memo" in matched_b + [(path, code)] = output_files + assert Path(path).as_posix().endswith(f"utils/components/{tag}.jsx") + assert f"const {tag} = memo" in code def test_shared_parent_instance_across_pages_preserves_original() -> None: diff --git a/tests/units/compiler/test_page_cache.py b/tests/units/compiler/test_page_cache.py new file mode 100644 index 00000000000..17b2494e33f --- /dev/null +++ b/tests/units/compiler/test_page_cache.py @@ -0,0 +1,1313 @@ +"""Tests for the per-page dependency graph used by the incremental compile cache.""" + +import importlib +import os +import sys +from pathlib import Path +from types import ModuleType +from typing import Any, cast + +import pytest + +from reflex.compiler import page_cache + + +def test_class_module_file_respects_each_project_root(tmp_path, monkeypatch): + """A class excluded in one project can be a dependency in another.""" + module = ModuleType("review_class_roots") + module.__file__ = str(tmp_path / "component.py") + monkeypatch.setitem(sys.modules, module.__name__, module) + cls = type("RootScoped", (), {"__module__": module.__name__}) + assert page_cache._class_module_file(cls, (tmp_path / "other",)) is None + assert page_cache._class_module_file(cls, (tmp_path,)) == Path(module.__file__) + + +def test_loaded_modules_reuse_resolution_and_discover_new_modules( + tmp_path, monkeypatch +): + """Cache stable module paths without missing imports or changes to roots.""" + source = tmp_path / "cached_module.py" + source.write_text("VALUE = 1\n") + module = ModuleType("cached_review_module") + module.__file__ = str(source) + monkeypatch.setitem(sys.modules, module.__name__, module) + original = Path.resolve + resolutions = [] + + def resolve(path, *args, **kwargs): + """Count resolutions of the test module's source file. + + Returns: + The resolved path. + """ + if path == source: + resolutions.append(path) + return original(path, *args, **kwargs) + + monkeypatch.setattr(Path, "resolve", resolve) + assert str(source) in page_cache._loaded_first_party_modules(tmp_path) + added = ModuleType("new_review_module") + added.__file__ = str(tmp_path / "added.py") + monkeypatch.setitem(sys.modules, added.__name__, added) + assert added.__file__ in page_cache._loaded_first_party_modules(tmp_path) + assert len(resolutions) == 1 + assert str(source) not in page_cache._loaded_first_party_modules(tmp_path / "other") + + +def _prepare_runtime_module(root, monkeypatch, module_name, *, import_root=None): + """Create a temporary module and make it importable. + + Args: + root: Directory where the module file is written. + monkeypatch: Pytest monkeypatch fixture. + module_name: Name of the module to create. + import_root: Directory added to ``sys.path``. Defaults to ``root``. + + Returns: + The created module file path. + """ + module_file = root / f"{module_name}.py" + module_file.write_text("VALUE = 1\n") + monkeypatch.syspath_prepend(str(import_root or root)) + sys.modules.pop(module_name, None) + importlib.invalidate_caches() + return module_file + + +def _forget_modules(*module_names): + """Remove temporary modules imported by a test.""" + for module_name in module_names: + sys.modules.pop(module_name, None) + + +def _global_inputs(root: Path, pages: Any = None) -> dict[str, str | None]: + """Digest every global input as the manifest records it. + + Args: + root: The project root. + pages: The page definitions, as barriers. + + Returns: + Each global input label mapped to its content digest (None if absent). + """ + import hashlib + + return { + label: hashlib.sha256(Path(path).read_bytes()).hexdigest() + if Path(path).is_file() + else None + for label, path in page_cache.global_input_paths(root, pages=pages).items() + } + + +def test_global_inputs_tracks_global_files(tmp_path): + (tmp_path / "rxconfig.py").write_text("config = 1\n") + (tmp_path / "other.py").write_text("x = 1\n") + epoch = _global_inputs(tmp_path) + # editing a non-global file does NOT change the epoch + (tmp_path / "other.py").write_text("x = 2\n") + assert _global_inputs(tmp_path) == epoch + # editing a genuinely-global file (rxconfig.py) DOES + (tmp_path / "rxconfig.py").write_text("config = 2\n") + assert _global_inputs(tmp_path) != epoch + + +def test_global_inputs_tracks_app_entrypoint(tmp_path, monkeypatch): + """Editing the app entrypoint (theme/app_wraps live there) bumps the epoch. + + App-wide config is configured where ``rx.App`` is built, not in any page's + dependency set, so without this an edit to it would leave every page a hit + and the reused on-disk app root / contexts / theme stale. + """ + (tmp_path / "rxconfig.py").write_text("config = 1\n") + entrypoint = (tmp_path / "myapp.py").resolve() + entrypoint.write_text("app = rx.App(theme=light)\n") + monkeypatch.setattr( + page_cache, "_app_entrypoint_file", lambda root=None: entrypoint + ) + epoch = _global_inputs(tmp_path) + # editing the app entrypoint DOES change the epoch + entrypoint.write_text("app = rx.App(theme=dark)\n") + assert _global_inputs(tmp_path) != epoch + + +def test_app_entrypoint_file_resolution(tmp_path, monkeypatch): + import sys + from types import ModuleType, SimpleNamespace + + monkeypatch.setattr( + "reflex.config.get_config", lambda: SimpleNamespace(module="fake_entry_mod") + ) + mod = ModuleType("fake_entry_mod") + monkeypatch.setitem(sys.modules, "fake_entry_mod", mod) + + # no __file__ on the module -> None + assert page_cache._app_entrypoint_file(root=tmp_path) is None + + # a file under the project root -> resolved + entry = tmp_path / "myapp" / "myapp.py" + entry.parent.mkdir() + entry.write_text("app = 1\n") + mod.__file__ = str(entry) + assert page_cache._app_entrypoint_file(root=tmp_path) == entry.resolve() + + # a file outside the project root -> None (not a project input) + outside = tmp_path.parent / "elsewhere.py" + mod.__file__ = str(outside) + assert page_cache._app_entrypoint_file(root=tmp_path) is None + + +def test_app_dependency_files_keeps_config_excludes_pages(tmp_path, monkeypatch): + """The entrypoint's config closure, with page modules as traversal barriers. + + entry imports a config-only ``theme.py`` and a page; the page imports a view + and (also) theme. The result must keep entry + theme (theme is reached from + the entrypoint directly, even though a page imports it too) and exclude the + page module and its view (tracked per page). + """ + from types import SimpleNamespace + + entry = tmp_path / "myapp.py" + theme = tmp_path / "theme.py" + page = tmp_path / "pages" / "index.py" + view = tmp_path / "components" / "hero.py" + graph = { + str(entry): {str(theme), str(page)}, + str(page): {str(view), str(theme)}, + str(theme): set(), + str(view): set(), + } + monkeypatch.setattr(page_cache, "_app_entrypoint_file", lambda root=None: entry) + monkeypatch.setattr( + page_cache, + "_loaded_first_party_modules", + lambda root: { + str(entry): "myapp", + str(theme): "theme", + str(page): "pages.index", + str(view): "components.hero", + }, + ) + monkeypatch.setattr( + page_cache, + "_module_import_edges", + lambda file, modname, file_to_mod: graph[file], + ) + monkeypatch.setattr( + page_cache, "_component_source_files", lambda comp, root: {str(page)} + ) + + result = page_cache.app_dependency_files( + [SimpleNamespace(component=object())], root=tmp_path + ) + assert result == {str(entry), str(theme)} + + +def test_global_inputs_excludes_page_modules(tmp_path, monkeypatch): + """A page-module edit keeps the epoch (incremental); an app-config edit bumps it.""" + from types import SimpleNamespace + + (tmp_path / "rxconfig.py").write_text("c = 1\n") + entry = tmp_path / "myapp.py" + entry.write_text("app = 1\n") + theme = tmp_path / "theme.py" + theme.write_text("t = 1\n") + page = tmp_path / "index.py" + page.write_text("p = 1\n") + graph = {str(entry): {str(theme), str(page)}, str(page): set(), str(theme): set()} + monkeypatch.setattr(page_cache, "_app_entrypoint_file", lambda root=None: entry) + monkeypatch.setattr( + page_cache, + "_loaded_first_party_modules", + lambda root: { + str(entry): "myapp", + str(theme): "theme", + str(page): "index", + }, + ) + monkeypatch.setattr( + page_cache, + "_module_import_edges", + lambda file, modname, file_to_mod: graph[file], + ) + monkeypatch.setattr( + page_cache, "_component_source_files", lambda comp, root: {str(page)} + ) + pages = [SimpleNamespace(component=object())] + + epoch = _global_inputs(tmp_path, pages) + # editing a page module does NOT change the epoch (tracked per page instead) + page.write_text("p = 2\n") + assert _global_inputs(tmp_path, pages) == epoch + # editing app-level config the entrypoint imports (theme) DOES + theme.write_text("t = 2\n") + assert _global_inputs(tmp_path, pages) != epoch + + +def test_global_inputs_hashes_only_app_dependency_closure(tmp_path, monkeypatch): + """Unrelated first-party modules do not make the app epoch coarse.""" + from types import SimpleNamespace + + entry = tmp_path / "myapp.py" + theme = tmp_path / "theme.py" + unrelated = tmp_path / "unrelated.py" + for path, code in ( + (entry, "import theme\n"), + (theme, "t = 1\n"), + (unrelated, "x = 1\n"), + ): + path.write_text(code) + graph = {str(entry): {str(theme)}, str(theme): set(), str(unrelated): set()} + monkeypatch.setattr(page_cache, "_app_entrypoint_file", lambda root=None: entry) + monkeypatch.setattr( + page_cache, + "_loaded_first_party_modules", + lambda root: { + str(entry): "myapp", + str(theme): "theme", + str(unrelated): "unrelated", + }, + ) + monkeypatch.setattr( + page_cache, + "_module_import_edges", + lambda file, modname, file_to_mod: graph[file], + ) + + epoch = _global_inputs(tmp_path, [SimpleNamespace(component=object())]) + unrelated.write_text("x = 2\n") + assert _global_inputs(tmp_path, [SimpleNamespace(component=object())]) == epoch + theme.write_text("t = 2\n") + assert _global_inputs(tmp_path, [SimpleNamespace(component=object())]) != epoch + + +def test_app_dependency_files_skips_graph_without_entrypoint(tmp_path, monkeypatch): + monkeypatch.setattr(page_cache, "_app_entrypoint_file", lambda root=None: None) + + def fail_loaded_first_party_modules(root): + msg = "module map should not be built without an entrypoint" + raise AssertionError(msg) + + monkeypatch.setattr( + page_cache, "_loaded_first_party_modules", fail_loaded_first_party_modules + ) + + assert page_cache.app_dependency_files(root=tmp_path) == set() + + +def test_app_dependency_files_tracks_dynamic_import_config_read(tmp_path, monkeypatch): + from types import SimpleNamespace + + from reflex.utils import prerequisites + + entry = tmp_path / "myapp.py" + dynamic = tmp_path / "dynamic_theme.py" + config_file = tmp_path / "theme.json" + entry.write_text( + "import importlib\ntheme = importlib.import_module('dynamic_theme').THEME\n" + ) + dynamic.write_text( + "from pathlib import Path\nTHEME = Path('theme.json').read_text()\n" + ) + config_file.write_text('"light"\n') + config = SimpleNamespace( + module="myapp", + app_module=None, + _app_name_is_valid=True, + ) + monkeypatch.chdir(tmp_path) + monkeypatch.syspath_prepend(str(tmp_path)) + monkeypatch.setenv("REFLEX_COMPILE_CACHE", "1") + monkeypatch.setattr(prerequisites, "get_config", lambda: config) + monkeypatch.setattr("reflex.config.get_config", lambda: config) + _forget_modules("myapp", "dynamic_theme") + + try: + prerequisites.get_app() + deps = page_cache.app_dependency_files(root=tmp_path) + epoch = _global_inputs(tmp_path) + + config_file.write_text('"dark"\n') + finally: + _forget_modules("myapp", "dynamic_theme") + + assert str(dynamic.resolve()) in deps + assert str(config_file.resolve()) in deps + assert _global_inputs(tmp_path) != epoch + + +def test_app_dependency_files_subtracts_pages_from_app_import_reads( + tmp_path, monkeypatch +): + from types import SimpleNamespace + + from reflex.utils import prerequisites + + entry = tmp_path / "myapp.py" + theme = tmp_path / "theme.py" + page = tmp_path / "page.py" + view = tmp_path / "view.py" + entry.write_text("import theme\nfrom page import index\napp_theme = theme.THEME\n") + theme.write_text("THEME = 'light'\n") + page.write_text("from view import render\n\ndef index():\n return render()\n") + view.write_text("def render():\n return 'view'\n") + config = SimpleNamespace( + module="myapp", + app_module=None, + _app_name_is_valid=True, + ) + monkeypatch.chdir(tmp_path) + monkeypatch.syspath_prepend(str(tmp_path)) + monkeypatch.setenv("REFLEX_COMPILE_CACHE", "1") + monkeypatch.setattr(prerequisites, "get_config", lambda: config) + monkeypatch.setattr("reflex.config.get_config", lambda: config) + _forget_modules("myapp", "theme", "page", "view") + + try: + prerequisites.get_app() + pages = [SimpleNamespace(component=sys.modules["page"].index)] + deps = page_cache.app_dependency_files(pages, root=tmp_path) + finally: + _forget_modules("myapp", "theme", "page", "view") + + assert deps == {str(entry.resolve()), str(theme.resolve())} + + +def test_record_reads_tracks_executed_importlib_import(tmp_path, monkeypatch): + module_name = "runtime_import_dep_for_page_cache" + module_file = _prepare_runtime_module(tmp_path, monkeypatch, module_name) + page_cache.enable_read_tracking(root=tmp_path) + + try: + with page_cache.record_reads() as reads: + importlib.import_module(module_name) + finally: + _forget_modules(module_name) + + assert str(module_file.resolve()) in reads + + +def test_record_reads_tracks_executed_builtin_import(tmp_path, monkeypatch): + package = tmp_path / "runtime_import_pkg" + package.mkdir() + (package / "__init__.py").write_text("") + child = package / "child.py" + child.write_text("VALUE = 1\n") + monkeypatch.syspath_prepend(str(tmp_path)) + _forget_modules("runtime_import_pkg", "runtime_import_pkg.child") + importlib.invalidate_caches() + page_cache.enable_read_tracking(root=tmp_path) + + try: + with page_cache.record_reads() as reads: + __import__("runtime_import_pkg.child") + finally: + _forget_modules("runtime_import_pkg", "runtime_import_pkg.child") + + assert str(child.resolve()) in reads + + +def test_read_tracking_root_change_invalidates_module_cache(tmp_path, monkeypatch): + """Recording a module under one root must not leak into another root. + + The module-file cache is keyed by the raw ``__file__`` value (the hot path + for every import statement), so switching recorder roots relies on + ``enable_read_tracking`` clearing it. + """ + root_a = tmp_path / "app_a" + root_a.mkdir() + root_b = tmp_path / "app_b" + root_b.mkdir() + module_name = "root_change_runtime_dep" + module_file = _prepare_runtime_module(root_a, monkeypatch, module_name) + page_cache.enable_read_tracking(root=root_a) + + try: + with page_cache.record_reads() as reads_a: + importlib.import_module(module_name) + assert str(module_file.resolve()) in reads_a + + page_cache.enable_read_tracking(root=root_b) + with page_cache.record_reads() as reads_b: + importlib.import_module(module_name) + assert str(module_file.resolve()) not in reads_b + finally: + _forget_modules(module_name) + + +def test_record_reads_tracks_path_open(tmp_path): + """``Path.open`` calls ``io.open`` directly, bypassing the ``builtins.open`` + patch, so it must be patched itself for data reads to be recorded. + """ + page_cache.enable_read_tracking(root=tmp_path) + data = tmp_path / "content.md" + data.write_text("hello") + + with page_cache.record_reads() as reads, data.open() as f: + f.read() + + assert str(data.resolve()) in reads + + +def test_record_reads_ignores_path_open_writes(tmp_path): + page_cache.enable_read_tracking(root=tmp_path) + out = tmp_path / "out.md" + + with page_cache.record_reads() as reads, out.open("w") as f: + f.write("x") + + assert str(out.resolve()) not in reads + + +def test_record_reads_ignores_unexecuted_import(tmp_path, monkeypatch): + module_name = "uncalled_runtime_import_dep" + module_file = _prepare_runtime_module(tmp_path, monkeypatch, module_name) + page_cache.enable_read_tracking(root=tmp_path) + + def import_if_called(): + return importlib.import_module(module_name) + + with page_cache.record_reads() as reads: + pass + + assert import_if_called + assert str(module_file.resolve()) not in reads + assert module_name not in sys.modules + + +def test_record_reads_no_recursion_when_recorder_import_triggers_import( + tmp_path, monkeypatch +): + """Recorder-internal imports must not re-enter dependency tracking.""" + module_name = "recorder_reentry_dep" + module_file = _prepare_runtime_module(tmp_path, monkeypatch, module_name) + page_cache.enable_read_tracking(root=tmp_path) + + real_absolute = Path.absolute + + def absolute_with_lazy_import(self: Path): + __import__("ntpath") + return real_absolute(self) + + monkeypatch.setattr(Path, "absolute", absolute_with_lazy_import) + + try: + with page_cache.record_reads() as reads: + importlib.import_module(module_name) + finally: + _forget_modules(module_name) + + assert str(module_file.resolve()) in reads + assert not any("ntpath" in read for read in reads) + + +def test_record_reads_imports_only_project_modules(tmp_path): + page_cache.enable_read_tracking(root=tmp_path) + + with page_cache.record_reads() as reads: + importlib.import_module("json") + + assert reads == set() + + +def test_record_reads_tracks_symlinked_project_import(tmp_path, monkeypatch): + import pytest + + root = tmp_path / "app" + root.mkdir() + module_name = "symlinked_runtime_dep" + linked_root = tmp_path / "linked_app" + try: + linked_root.symlink_to(root, target_is_directory=True) + except OSError as exc: + pytest.skip(f"symlinks unavailable: {exc}") + module_file = _prepare_runtime_module( + root, monkeypatch, module_name, import_root=linked_root + ) + page_cache.enable_read_tracking(root=root) + + try: + with page_cache.record_reads() as reads: + importlib.import_module(module_name) + finally: + _forget_modules(module_name) + + assert str(module_file.resolve()) in reads + + +def test_used_state_files_from_output_and_memos(tmp_path): + from types import SimpleNamespace + + sfile = (tmp_path / "state.py").resolve() + mfile = (tmp_path / "mstate.py").resolve() + id_to_file = { + "reflex___state____state____app_____s": sfile, + "reflex___state____state____app_____m": mfile, + } + out = 'jsx("div",{},reflex___state____state____app_____s.x_rx_state_)' + assert page_cache.used_state_files(out, [], id_to_file) == {sfile} + assert page_cache.used_state_files("no state", [], id_to_file) == set() + # state hidden inside an auto-memoized component is still captured + memo = SimpleNamespace( + render=lambda: {"contents": "reflex___state____state____app_____m.y_rx_state_"} + ) + assert page_cache.used_state_files(out, [memo], id_to_file) == {sfile, mfile} + # un-introspectable memo -> conservative (all fine files) + boom = SimpleNamespace(render=lambda: (_ for _ in ()).throw(RuntimeError())) + assert page_cache.used_state_files(out, [boom], id_to_file) == {sfile, mfile} + + +def test_file_validator_is_stat_first(tmp_path, monkeypatch): + """Unchanged files are recognised from their stat key without being read.""" + f = tmp_path / "dep.py" + f.write_text("x = 1\n") + entry = page_cache.file_entry(str(f)) + assert entry is not None + files = {str(f): entry} + + reads = 0 + original = page_cache.file_entry + + def counting(path): + nonlocal reads + reads += 1 + return original(path) + + monkeypatch.setattr(page_cache, "file_entry", counting) + v = page_cache.FileValidator(files) + assert v.changed(str(f)) is False + assert reads == 0 # stat matched, never read + assert v.unchanged([str(f)]) is True # memoized, still no read + assert reads == 0 + + # Touched with identical content: hashed once, unchanged, entry refreshed. + bumped = f.stat().st_mtime_ns + 1_000_000_000 + os.utime(f, ns=(bumped, bumped)) + v = page_cache.FileValidator(files) + assert v.changed(str(f)) is False + assert reads == 1 + assert v.refreshed[str(f)][1] == bumped + + # Same size, different content, distinct mtime: the hash decides. + f.write_text("x = 2\n") + os.utime(f, ns=(bumped, bumped)) + v = page_cache.FileValidator(files) + assert v.changed(str(f)) is True + # Unknown or missing files always count as changed. + assert v.changed(str(tmp_path / "missing.py")) is True + files[str(tmp_path / "gone.py")] = ["h", 1, 1] + assert page_cache.FileValidator(files).changed(str(tmp_path / "gone.py")) is True + + +def test_import_names_parse_cache_skips_unchanged_files(tmp_path, monkeypatch): + mod = tmp_path / "mod_a.py" + mod.write_text("import os\nfrom pathlib import Path\n") + page_cache._import_names_cache.clear() + + parses = 0 + import ast + + original = ast.parse + + def counting(*args, **kwargs): + nonlocal parses + parses += 1 + return original(*args, **kwargs) + + monkeypatch.setattr(ast, "parse", counting) + names = page_cache._module_import_names(str(mod), "mod_a") + assert names == ["os", "pathlib", "pathlib.Path"] + assert parses == 1 + assert page_cache._module_import_names(str(mod), "mod_a") == names + assert parses == 1 # cached by stat key + + # The cache round-trips through its persisted form. + exported = page_cache.export_import_names_cache() + page_cache._import_names_cache.clear() + page_cache.load_import_names_cache(exported) + assert page_cache._module_import_names(str(mod), "mod_a") == names + assert parses == 1 + # Malformed persisted entries are ignored, not fatal. + page_cache.load_import_names_cache({str(mod): "garbage", "x": [[1], []]}) + assert page_cache._import_names_cache == {} + + # An edit re-parses. + mod.write_text("import sys\n") + assert page_cache._module_import_names(str(mod), "mod_a") == ["sys"] + assert parses == 2 + + +def test_first_party_roots_include_hot_reload_include_paths(tmp_path, monkeypatch): + from reflex_base.environment import environment + + sibling = tmp_path / "sibling_pkg" + sibling.mkdir() + # Relative to cwd: the env var's ":" delimiter cannot carry a Windows + # drive-letter path. + monkeypatch.chdir(tmp_path) + monkeypatch.setenv(environment.REFLEX_HOT_RELOAD_INCLUDE_PATHS.name, sibling.name) + roots = page_cache.first_party_roots(tmp_path) + assert roots == (tmp_path.resolve(), sibling.resolve()) + + # A module under the include path is first-party for dependency tracking. + page_cache.enable_read_tracking(root=tmp_path) + module_file = sibling / "helper.py" + module_file.write_text("VALUE = 1\n") + assert page_cache._recordable_module_file(str(module_file)) == str( + module_file.resolve() + ) + assert page_cache._recordable_module_file(str(tmp_path / ".venv" / "x.py")) is None + + +def test_page_dependency_files_closes_over_dynamic_imports(tmp_path, monkeypatch): + """A module imported at evaluation time brings its own import closure. + + The helper may already be loaded (an earlier page imported it), so its own + imports never re-execute under the recorder; the graph supplies them. + """ + from types import SimpleNamespace + + helper = _prepare_runtime_module(tmp_path, monkeypatch, "dyn_helper_for_closure") + leaf = _prepare_runtime_module(tmp_path, monkeypatch, "dyn_leaf_for_closure") + helper.write_text("import dyn_leaf_for_closure\n") + try: + importlib.import_module("dyn_helper_for_closure") + page_cache.clear_import_graph() + page_ctx = SimpleNamespace( + root_component=SimpleNamespace(children=[]), + output_code="", + memo_uses={}, + source_files={str(helper.resolve())}, + ) + deps = page_cache.page_dependency_files( + cast(Any, page_ctx), lambda: None, {}, tmp_path + ) + finally: + _forget_modules("dyn_helper_for_closure", "dyn_leaf_for_closure") + page_cache.clear_import_graph() + + assert str(helper.resolve()) in deps + assert str(leaf.resolve()) in deps + + +def test_dynamic_import_records_parent_packages(tmp_path, monkeypatch): + """Cached dynamic imports still depend on package initialization.""" + package = tmp_path / "review_dynamic_parent" + package.mkdir() + parent = package / "__init__.py" + parent.write_text("VALUE = 1\n") + child = package / "child.py" + child.write_text("from . import VALUE\n") + monkeypatch.syspath_prepend(str(tmp_path)) + page_cache.enable_read_tracking(tmp_path) + try: + importlib.import_module("review_dynamic_parent.child") + with page_cache.record_reads() as reads: + importlib.import_module("review_dynamic_parent.child") + assert {str(parent), str(child)} <= reads + finally: + _forget_modules("review_dynamic_parent", "review_dynamic_parent.child") + + +def test_package_relative_import_names(tmp_path): + """Relative imports in package initializers resolve from that package.""" + source = tmp_path / "__init__.py" + source.write_text("from . import helper\nfrom ..common import value\n") + assert page_cache._module_import_names(str(source), "app.pages") == [ + "app.pages", + "app.pages.helper", + "app.common", + "app.common.value", + ] + + +def test_entrypoint_is_not_a_page_barrier(tmp_path, monkeypatch): + """A page defined beside its App cannot hide app-wide configuration.""" + from types import SimpleNamespace + + entry = tmp_path / "app.py" + entry.write_text("THEME = 'dark'\n") + monkeypatch.setattr(page_cache, "_app_entrypoint_file", lambda root: entry) + monkeypatch.setattr( + page_cache, "build_import_graph", lambda root: {str(entry): set()} + ) + monkeypatch.setattr( + page_cache, "_component_source_files", lambda component, root: {str(entry)} + ) + assert str(entry) in page_cache.app_dependency_files( + [SimpleNamespace(component=None)], tmp_path + ) + + +def test_missing_read_remains_a_dependency(tmp_path, monkeypatch): + """A handled missing read must invalidate when its file later appears.""" + missing = str(tmp_path / "optional.json") + monkeypatch.setattr(page_cache, "page_dependency_files", lambda *args: {missing}) + files = {} + deps = page_cache.page_dependency_entries( + cast(Any, None), None, {}, page_cache.file_entry, files, tmp_path + ) + assert deps == [missing] + Path(missing).write_text("{}") + assert not page_cache.FileValidator(files).unchanged(deps) + + +@pytest.mark.parametrize("statement", ["import", "import_module"]) +def test_import_time_reads_are_recorded_for_every_importer( + tmp_path, monkeypatch, statement +): + """A module's import-time reads are deps of each page that imports it. + + Only the first page's import executes the module; a later page gets the + cached module and must still depend on what its execution read, which + the read's attribution to the module provides. + """ + data = tmp_path / "shared_data.json" + data.write_text('{"title": "v1"}') + helper = _prepare_runtime_module(tmp_path, monkeypatch, "review_shared_helper") + helper.write_text( + f"from pathlib import Path\nTITLE = Path({str(data)!r}).read_text()\n" + ) + page_cache.enable_read_tracking(tmp_path) + page_reads = [] + try: + for _ in range(2): + with page_cache.record_reads() as reads: + if statement == "import": + exec("import review_shared_helper", {}) + else: + importlib.import_module("review_shared_helper") + page_reads.append(reads) + finally: + _forget_modules("review_shared_helper") + helper_file = str(helper.resolve()) + assert [str(data) in reads for reads in page_reads] == [True, False] + assert [helper_file in reads for reads in page_reads] == [True, True] + assert page_cache._module_reads[helper_file] == {str(data)} + + +def test_reads_behind_a_shared_cache_reach_every_dependent_page(tmp_path, monkeypatch): + """A read a cached helper makes for the first page is a dep of every page. + + ``functools.cache`` (or a module-level loader) reads the file once per + process, while evaluating the first page that calls it. The read is + attributed to the helper's module, so a page whose import closure holds + that module depends on the file even when its own evaluation read nothing. + """ + from types import SimpleNamespace + + posts = tmp_path / "posts.md" + posts.write_text("v1") + helper = _prepare_runtime_module(tmp_path, monkeypatch, "cached_posts_loader") + helper.write_text( + "import functools\nfrom pathlib import Path\n\n\n" + "@functools.cache\ndef load_posts():\n" + f" return Path({str(posts)!r}).read_text()\n" + ) + page_cache.enable_read_tracking(tmp_path) + page_cache.clear_import_graph() + try: + module = importlib.import_module("cached_posts_loader") + page_reads = [] + for _ in range(2): + with page_cache.record_reads() as reads: + module.load_posts() + page_reads.append(reads) + assert [str(posts) in reads for reads in page_reads] == [True, False] + + def blog_b(): + return module.load_posts() + + page_ctx = SimpleNamespace( + root_component=SimpleNamespace(children=[]), + output_code="", + memo_uses={}, + source_files={str(helper.resolve())}, + ) + deps = page_cache.page_dependency_files( + cast(Any, page_ctx), blog_b, {}, tmp_path + ) + finally: + _forget_modules("cached_posts_loader") + page_cache.clear_import_graph() + assert str(posts) in deps + + +def test_reads_are_attributed_to_the_first_party_caller(tmp_path, monkeypatch): + """A read a library performs on a page's behalf belongs to the page's module.""" + data = tmp_path / "note.txt" + data.write_text("hello") + reader = _prepare_runtime_module(tmp_path, monkeypatch, "first_party_reader") + reader.write_text( + "import json\n\n\ndef load(path):\n return json.load(open(path))\n" + ) + data.write_text('{"a": 1}') + page_cache.enable_read_tracking(tmp_path) + try: + module = importlib.import_module("first_party_reader") + with page_cache.record_reads() as reads: + module.load(str(data)) + finally: + _forget_modules("first_party_reader") + assert str(data) in reads + assert page_cache._module_reads[str(reader.resolve())] == {str(data)} + page_cache.forget_module_reads() + assert not page_cache._module_reads + + +@pytest.mark.parametrize("api", ["io.open", "pkgutil", "getsource"]) +def test_record_reads_tracks_stdlib_read_entry_points(tmp_path, monkeypatch, api): + """Reads through ``io.open``, loaders and ``linecache`` are recorded.""" + import inspect + import io + import pkgutil + + package = tmp_path / "read_api_pkg" + package.mkdir() + (package / "__init__.py").write_text("def f():\n return 1\n") + (package / "data.md").write_text("# data") + monkeypatch.syspath_prepend(str(tmp_path)) + _forget_modules("read_api_pkg") + importlib.invalidate_caches() + page_cache.enable_read_tracking(root=tmp_path) + try: + module = importlib.import_module("read_api_pkg") + with page_cache.record_reads() as reads: + if api == "io.open": + with io.open(package / "data.md") as fh: # noqa: UP020 + fh.read() + elif api == "pkgutil": + pkgutil.get_data("read_api_pkg", "data.md") + else: + import linecache + + linecache.clearcache() + inspect.getsource(module.f) + finally: + _forget_modules("read_api_pkg") + expected = package / ("__init__.py" if api == "getsource" else "data.md") + assert str(expected.resolve()) in reads + + +def test_mark_volatile_makes_the_page_miss_every_compile(tmp_path): + """A volatile page records a dependency no file table can satisfy.""" + page_cache.enable_read_tracking(root=tmp_path) + with page_cache.record_reads() as reads: + page_cache.mark_volatile() + assert reads == {page_cache.VOLATILE_DEP} + from types import SimpleNamespace + + page_ctx = SimpleNamespace( + root_component=SimpleNamespace(children=[]), + output_code="", + memo_uses={}, + source_files=reads, + ) + files = {} + deps = page_cache.page_dependency_entries( + cast(Any, page_ctx), lambda: None, {}, page_cache.file_entry, files, tmp_path + ) + assert page_cache.VOLATILE_DEP in deps + assert page_cache.VOLATILE_DEP not in files + assert not page_cache.FileValidator(files).unchanged(deps) + page_cache.mark_volatile() # a no-op outside a recorder + + +@pytest.mark.parametrize("location", ["assets", "venv_project"]) +def test_reads_under_the_project_are_tracked_whatever_the_directory_names( + tmp_path, location +): + """Only parts below the project root decide the exclusion of a read.""" + root = tmp_path / ("venv" if location == "venv_project" else "app") + data_dir = root / ("assets" if location == "assets" else "data") + data_dir.mkdir(parents=True) + data = data_dir / "note.txt" + data.write_text("v1") + page_cache.enable_read_tracking(root=root) + with page_cache.record_reads() as reads: + data.read_text() + assert str(data.resolve()) in reads + excluded = root / "node_modules" / "pkg" / "readme.md" + excluded.parent.mkdir(parents=True) + excluded.write_text("x") + with page_cache.record_reads() as reads: + excluded.read_text() + assert reads == set() + + +def test_file_modified_during_the_compile_is_recorded_as_racy(tmp_path): + """A dependency edited after the compile started always misses next time. + + The compile may have read the earlier content, so the manifest cannot + vouch for the file (the racy-git rule). + """ + data = tmp_path / "slow.txt" + data.write_text("v2") + with page_cache.record_app_import(tmp_path): + pass + data.write_text("v3") + entry = page_cache.file_entry(str(data)) + assert entry == page_cache.RACY_ENTRY + files = {str(data): page_cache.RACY_ENTRY} + assert not page_cache.FileValidator(files).unchanged([str(data)]) + # Unchanged after the compile: a normal entry. + stat = data.stat() + start = page_cache._compile_start_ns + assert start is not None + os.utime(data, ns=(stat.st_atime_ns, start - 5_000_000_000)) + assert page_cache.file_entry(str(data)) != page_cache.RACY_ENTRY + # A whole-second timestamp in the compile's first second is racy too. + os.utime(data, ns=(stat.st_atime_ns, start - start % 1_000_000_000)) + assert page_cache.file_entry(str(data)) == page_cache.RACY_ENTRY + + +def test_entrypoint_digest_ignores_page_registrations_only(tmp_path): + """Formatting, comments and top-level add_page calls do not change it.""" + entry = tmp_path / "entry_app.py" + + def digest(source: str) -> str | None: + entry.write_text(source) + return page_cache.entrypoint_digest(str(entry)) + + base = digest( + "import reflex as rx\nfrom .pages import a, b\n\napp = rx.App()\n" + 'app.add_page(a, route="/a")\n' + ) + assert base is not None + assert ( + digest( + "import reflex as rx\nfrom .pages import a, b\n\n# pages\napp = rx.App()\n" + 'app.add_page(a, route="/a", title="A")\napp.add_page(b, route="/b")\n' + ) + == base + ) + assert ( + digest( + "import reflex as rx\nfrom .pages import a, b\n\n" + 'app = rx.App(theme=rx.theme(accent_color="red"))\n' + 'app.add_page(a, route="/a")\n' + ) + != base + ) + assert digest("this is not python (") is None + + +def test_entrypoint_digest_counts_every_import(tmp_path): + """Every import counts, even one used only to register a page.""" + entry = tmp_path / "entry_app.py" + + def digest(source: str) -> str | None: + entry.write_text(source) + return page_cache.entrypoint_digest(str(entry)) + + base = digest( + "import reflex as rx\nfrom .themes import dark as theme\n" + "from .pages.a import a\n\napp = rx.App(theme=theme)\n" + 'app.add_page(a, route="/a")\n' + ) + # Importing a page runs its module, which can configure the app. + assert ( + digest( + "import reflex as rx\nfrom .themes import dark as theme\n" + "from .pages.a import a\nfrom .pages.b import b\n\n" + 'app = rx.App(theme=theme)\napp.add_page(a, route="/a")\n' + 'app.add_page(b, route="/b")\n' + ) + != base + ) + # The same App call with a theme imported from elsewhere is another app. + assert ( + digest( + "import reflex as rx\nfrom .themes import light as theme\n" + "from .pages.a import a\n\napp = rx.App(theme=theme)\n" + 'app.add_page(a, route="/a")\n' + ) + != base + ) + + +_SLICED_MODULE = """\ +import reflex as rx +from .other import unused + +TITLE = "t" + + +class PageState(rx.State): + value: int = 0 + + +def helper(): + return rx.text(TITLE) + + +def page_a(): + return rx.box(helper(), PageState.value) + + +def page_b(): + return rx.text("b") +""" + + +def _rewrite_module(path: Path, source: str) -> None: + """Rewrite a module with a distinct modification time for the stat cache. + + Immediate same-size writes can retain their timestamp on Windows. + + Args: + path: The existing module file. + source: The replacement source. + """ + stat = path.stat() + path.write_text(source) + os.utime(path, ns=(stat.st_atime_ns, stat.st_mtime_ns + 1_000_000_000)) + + +@pytest.mark.parametrize( + ("old", "new", "changes"), + [ + ('rx.text("b")', 'rx.text("b2")', False), + ("\n\ndef page_b", "\n\n# a comment\ndef page_b", False), + ("def page_b", "def unrelated():\n return 1\n\n\ndef page_b", False), + ("from .other import unused\n", "", True), + ("rx.box(helper()", "rx.vstack(helper()", True), + ("rx.text(TITLE)", "rx.heading(TITLE)", True), + ('TITLE = "t"', 'TITLE = "u"', True), + ("value: int = 0", "value: int = 1", True), + ("from .other import unused\n", "from .other import *\n", True), + ], +) +def test_function_slice_digest_covers_what_the_function_reads( + tmp_path, old, new, changes +): + """A page function's slice changes only with the code it can read. + + That is its own body, the module-level functions it calls (transitively), + every class, and every statement that runs at import. + """ + module = tmp_path / "sliced.py" + module.write_text(_SLICED_MODULE) + before = page_cache.function_slice_digest(str(module), "page_a") + assert before is not None + _rewrite_module(module, _SLICED_MODULE.replace(old, new, 1)) + after = page_cache.function_slice_digest(str(module), "page_a") + assert (after != before) is changes + + +def test_function_slice_digest_keeps_code_reached_indirectly(tmp_path): + """Code a decorator hands to import-time code stays in the slice.""" + module = tmp_path / "indirect.py" + source = ( + "REG = []\n\n\ndef register(f):\n REG.append(f)\n return f\n\n\n" + "@register\ndef widget():\n return 1\n\n\n" + "def page_a():\n return [f() for f in REG]\n" + ) + module.write_text(source) + before = page_cache.function_slice_digest(str(module), "page_a") + _rewrite_module(module, source.replace("return 1", "return 2")) + assert page_cache.function_slice_digest(str(module), "page_a") != before + + +@pytest.mark.parametrize( + "body", + [ + # Names reached through globals() are invisible to the syntax tree. + 'globals()["widget"]()', + 'getattr(sys.modules[__name__], "widget")()', + # The module's own source text and line numbers are not in the tree. + "inspect.getsource(widget)", + "sys._getframe().f_lineno", + ], +) +def test_function_slice_falls_back_to_the_whole_file_for_computed_access( + tmp_path, monkeypatch, body +): + """A page reaching its module by a computed name depends on the whole file.""" + from types import SimpleNamespace + + module = _prepare_runtime_module(tmp_path, monkeypatch, "computed_access_page") + module.write_text( + "import inspect\nimport sys\n\n\ndef widget():\n return 1\n\n\n" + f"def page_a():\n return {body}\n" + ) + assert page_cache.function_slice_digest(str(module), "page_a") is None + page_cache.clear_import_graph() + try: + page_a = importlib.import_module("computed_access_page").page_a + page_ctx = SimpleNamespace( + root_component=SimpleNamespace(children=[]), + output_code="", + memo_uses={}, + source_files=set(), + ) + deps = page_cache.page_dependency_files( + cast(Any, page_ctx), page_a, {}, tmp_path + ) + finally: + _forget_modules("computed_access_page") + page_cache.clear_import_graph() + assert str(module.resolve()) in deps + + +def test_function_slice_digest_needs_a_top_level_function(tmp_path): + """Only a module-level function has a slice.""" + module = tmp_path / "nested.py" + module.write_text("def outer():\n def inner():\n return 1\n") + assert page_cache.function_slice_digest(str(module), "inner") is None + assert page_cache.function_slice_digest(str(tmp_path / "gone.py"), "x") is None + + +def test_entrypoint_digest_ignores_page_function_bodies(tmp_path): + """A page defined beside its App is compared page by page, not globally.""" + entry = tmp_path / "entry_app.py" + + def digest(source: str) -> str | None: + entry.write_text(source) + return page_cache.entrypoint_digest(str(entry)) + + source = ( + "import reflex as rx\n\n\ndef make_theme():\n return rx.theme()\n\n\n" + 'def index():\n return rx.text("a")\n\n\n' + "app = rx.App(theme=make_theme())\n" + 'app.add_page(index, route="/")\n' + ) + base = digest(source) + assert base is not None + assert digest(source.replace('rx.text("a")', 'rx.text("b")')) == base + assert digest(source.replace("rx.theme()", "rx.theme(accent_color='red')")) != base + + +def test_page_dependency_files_track_the_page_function_slice(tmp_path, monkeypatch): + """A page depends on its function's slice of its module, not the whole file.""" + from types import SimpleNamespace + + module = _prepare_runtime_module(tmp_path, monkeypatch, "sliced_pages_module") + module.write_text(_SLICED_MODULE.replace("from .other import unused\n", "")) + try: + page_a = importlib.import_module("sliced_pages_module").page_a + page_cache.clear_import_graph() + page_ctx = SimpleNamespace( + root_component=SimpleNamespace(children=[]), + output_code="", + memo_uses={}, + source_files=set(), + ) + deps = page_cache.page_dependency_files( + cast(Any, page_ctx), page_a, {}, tmp_path + ) + finally: + _forget_modules("sliced_pages_module") + page_cache.clear_import_graph() + + file = str(module.resolve()) + assert f"{file}{page_cache.SLICE_SEPARATOR}page_a" in deps + assert file not in deps + + +def test_file_validator_compares_function_slices(tmp_path): + """A slice dependency is stat-first and changes only with its slice.""" + module = tmp_path / "sliced.py" + module.write_text(_SLICED_MODULE) + dep = f"{module}{page_cache.SLICE_SEPARATOR}page_a" + entry = page_cache.file_entry(dep) + assert entry is not None + files = {dep: entry} + assert page_cache.FileValidator(files).unchanged([dep]) + module.write_text(_SLICED_MODULE.replace('rx.text("b")', 'rx.text("b2")')) + validator = page_cache.FileValidator(files) + assert validator.unchanged([dep]) + assert dep in validator.refreshed + module.write_text(_SLICED_MODULE.replace("rx.box(", "rx.vstack(")) + assert not page_cache.FileValidator(files).unchanged([dep]) + assert page_cache.dep_file(dep) == str(module) + assert page_cache.dep_file(str(module)) == str(module) + + +def test_read_tracking_root_change_forgets_import_reads(tmp_path, monkeypatch): + """A cached module's import-time reads from one root never reach another.""" + root_a = tmp_path / "app_a" + root_a.mkdir() + data = root_a / "data.json" + data.write_text("{}") + module_name = "root_change_import_reads" + _prepare_runtime_module(root_a, monkeypatch, module_name).write_text( + f"from pathlib import Path\nDATA = Path({str(data)!r}).read_text()\n" + ) + page_cache.enable_read_tracking(root=root_a) + try: + with page_cache.record_reads() as reads_a: + importlib.import_module(module_name) + assert str(data.resolve()) in reads_a + page_cache.enable_read_tracking(root=tmp_path / "app_b") + with page_cache.record_reads() as reads_b: + importlib.import_module(module_name) + finally: + _forget_modules(module_name) + assert str(data.resolve()) not in reads_b + + +def test_failed_import_keeps_its_reads(tmp_path, monkeypatch): + """A page that handles a failed import still depends on what it read.""" + missing = tmp_path / "optional.json" + module_name = "optional_data_helper" + _prepare_runtime_module(tmp_path, monkeypatch, module_name).write_text( + f"from pathlib import Path\nDATA = Path({str(missing)!r}).read_text()\n" + ) + page_cache.enable_read_tracking(root=tmp_path) + try: + with page_cache.record_reads() as reads, pytest.raises(FileNotFoundError): + importlib.import_module(module_name) + finally: + _forget_modules(module_name) + assert str(missing.resolve()) in reads + + +@pytest.mark.parametrize( + ("decorator", "inert"), + [ + ("@rx.page(route='/x')", True), + ("@rx.memo", True), + ("@functools.cache", True), + ("@functools.lru_cache(maxsize=4)", True), + ("@registry.page", False), + ("@cache", False), + ], +) +def test_only_known_decorators_keep_a_function_out_of_other_slices( + tmp_path, decorator, inert +): + """A decorator of unknown origin may register the function it wraps.""" + module = tmp_path / "decorated.py" + source = ( + f"{decorator}\ndef widget():\n return 1\n\n\ndef page_a():\n return 2\n" + ) + module.write_text(source) + before = page_cache.function_slice_digest(str(module), "page_a") + _rewrite_module(module, source.replace("return 1", "return 3")) + after = page_cache.function_slice_digest(str(module), "page_a") + assert (after == before) is inert + + +def test_entrypoint_digest_tracks_class_bodies(tmp_path): + """A class body runs at import, so it can configure the app.""" + entry = tmp_path / "entry_app.py" + source = ( + "import reflex as rx\n\n\nclass Settings(Base):\n color = 'red'\n\n\n" + "app = rx.App()\n" + ) + entry.write_text(source) + before = page_cache.entrypoint_digest(str(entry)) + entry.write_text(source.replace("'red'", "'blue'")) + assert page_cache.entrypoint_digest(str(entry)) != before + + +def test_function_slice_digest_counts_imports_no_definition_names(tmp_path): + """An import counts for every slice, whether or not the code names it.""" + module = tmp_path / "unused_import.py" + source = "from reflex import text as widget\n\n\ndef page_a():\n return 1\n" + module.write_text(source) + before = page_cache.function_slice_digest(str(module), "page_a") + module.write_text(source.replace("text as widget", "heading as widget")) + os.utime(module, ns=(1, 1)) + assert page_cache.function_slice_digest(str(module), "page_a") != before diff --git a/tests/units/compiler/test_stale_cleanup.py b/tests/units/compiler/test_stale_cleanup.py index 500fc02861c..43deb5683d8 100644 --- a/tests/units/compiler/test_stale_cleanup.py +++ b/tests/units/compiler/test_stale_cleanup.py @@ -167,3 +167,38 @@ def test_prune_ignores_corrupt_manifest(fake_web_dir: Path): compiler_utils.prune_stale_memo_files([survivor]) assert survivor.exists() + + +def test_reconcile_keeps_untouched_files_and_drops_the_unused(fake_web_dir: Path): + """An incremental compile records what it wrote and deletes what it dropped. + + Files it did not touch stay recorded, so a later full compile still + prunes them; only files the previous manifest recorded are deleted. + """ + kept, dropped, foreign = _seed_files( + fake_web_dir, + [ + "utils/components/Kept.jsx", + "utils/components/Dropped.jsx", + "utils/components/Foreign.jsx", + ], + ) + _seed_manifest( + fake_web_dir, ["utils/components/Kept.jsx", "utils/components/Dropped.jsx"] + ) + written = fake_web_dir / "utils" / "components" / "Written.jsx" + compiler_utils.reconcile_memo_manifest([written], [dropped, foreign]) + assert kept.exists() + assert not dropped.exists() + assert foreign.exists(), "only files the manifest recorded are deleted" + recorded = json.loads( + (fake_web_dir / compiler_utils._MEMO_MANIFEST_FILENAME).read_text() + ) + assert recorded == ["utils/components/Kept.jsx", "utils/components/Written.jsx"] + # A file both written and dropped is written. + compiler_utils.reconcile_memo_manifest([kept], [kept]) + assert kept.exists() + recorded = json.loads( + (fake_web_dir / compiler_utils._MEMO_MANIFEST_FILENAME).read_text() + ) + assert recorded == ["utils/components/Kept.jsx", "utils/components/Written.jsx"] diff --git a/tests/units/components/test_memo.py b/tests/units/components/test_memo.py index 6f52239fbc1..fbee162378b 100644 --- a/tests/units/components/test_memo.py +++ b/tests/units/components/test_memo.py @@ -26,6 +26,7 @@ _MemoCallBinding, component_hash, memo_tag, + registration_owner, ) from reflex_base.event import EventChain, EventHandler, no_args_event_spec from reflex_base.registry import RegistrationContext @@ -1161,9 +1162,7 @@ def test_compile_memo_components_falls_back_when_no_source_module(): ) files, _ = compiler.compile_memo_components((legacy_definition,)) - exp_path = compiler._memo_component_file_path( - compiler_utils.get_memo_components_dir(), "LegacyMemo" - ) + exp_path = compiler_utils.get_memo_component_path("LegacyMemo") assert any(path == exp_path for path, _ in files) @@ -1338,9 +1337,7 @@ def test_component_memo_wrapper_none_in_unmirrored_module(): ) files, _ = compiler.compile_memo_components((definition,)) - exp_path = compiler._memo_component_file_path( - compiler_utils.get_memo_components_dir(), "BareSingle" - ) + exp_path = compiler_utils.get_memo_component_path("BareSingle") single_code = next(code for path, code in files if path == exp_path) assert "export const BareSingle = ((" in single_code assert " = memo(" not in single_code @@ -2010,6 +2007,35 @@ def recursive_count(n: rx.vars.NumberVar[int]) -> rx.Var[int]: assert "recursive_count" in str(invoked) +def test_registration_owner_tags_definitions(): + """Registrations inside ``registration_owner`` record the owning route. + + A memo registered while a page evaluates belongs to that page; a + re-registration outside any owner scope re-tags it as import-owned. + """ + + def _definition(): + return next( + d + for (_, module), d in MEMOS.items() + if module == __name__ and d.python_name == "owned_badge" + ) + + with registration_owner("/owner-route"): + + @rx.memo + def owned_badge(text: str) -> Component: + return rx.el.span(text) + + assert _definition().owner_route == "/owner-route" + + @rx.memo + def owned_badge(text: str) -> Component: + return rx.el.span(text) + + assert _definition().owner_route is None + + class _AppWrapProbe(Component): """A component whose only per-instance artifact is an app-wrap component.""" diff --git a/tests/units/conftest.py b/tests/units/conftest.py index 87c64245b0b..b72b76eacde 100644 --- a/tests/units/conftest.py +++ b/tests/units/conftest.py @@ -40,6 +40,22 @@ from .states.upload import SubUploadState, UploadState +@pytest.fixture(autouse=True) +def _forget_compile_start() -> Generator[None, None, None]: + """Unset the compile cache's racy-edit threshold between tests. + + A compile sets it before importing the app; a test that writes files + after another test compiled would otherwise record them as racy. + + Yields: + None. + """ + from reflex.compiler import page_cache + + yield + page_cache._compile_start_ns = None + + @pytest.fixture(autouse=True) def _isolate_app_in_context() -> Generator[None, None, None]: """Reset the App slot on the active RegistrationContext between tests. diff --git a/tests/units/reflex_base/compiler/__init__.py b/tests/units/reflex_base/compiler/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/units/reflex_base/compiler/test_templates.py b/tests/units/reflex_base/compiler/test_templates.py new file mode 100644 index 00000000000..def7434a2d9 --- /dev/null +++ b/tests/units/reflex_base/compiler/test_templates.py @@ -0,0 +1,53 @@ +"""Tests for reflex_base.compiler.templates.""" + +import re + +from reflex_base.compiler import templates + +# The unloaded-route throw block exactly as shipped in the pinned +# @react-router/dev runtime (dist/static/refresh-utils.mjs, enqueueUpdate). +_REFRESH_UTILS_THROW = """\ + let imported = window.__reactRouterRouteModuleUpdates.get(route.id); + if (!imported) { + throw Error( + `[react-router:hmr] No module update found for route ${route.id}`, + ); + } + let routeModule = { +""" + + +def _render_vite_config() -> str: + return templates.vite_config_template( + base="/", + hmr=True, + force_full_reload=False, + experimental_hmr=False, + sourcemap=False, + ) + + +def test_vite_config_patches_react_router_hmr_runtime(): + """The generated vite config neutralizes react-router's unloaded-route throw. + + react-router's HMR client throws when an update batch includes a route the + browser hasn't loaded, and the throw aborts the batch before the update + queue is cleared — one edit to any not-currently-open page then poisons HMR + until a full page reload. The generated config must ship a plugin that + rewrites the served runtime to skip unloaded routes instead. + """ + config = _render_vite_config() + assert "patchReactRouterHmrRuntime()" in config + assert '"\\0virtual:react-router/hmr-runtime"' in config + + # The embedded regex must match the pinned runtime's throw block, and the + # replacement must drop the throw while keeping the update loop going. + regex_match = re.search(r"const unloadedRouteThrow = /(.+)/;", config) + assert regex_match is not None + js_regex = regex_match.group(1) + patched, n_subs = re.subn( + js_regex, "if (!imported) continue;", _REFRESH_UTILS_THROW + ) + assert n_subs == 1 + assert "throw" not in patched + assert "if (!imported) continue;" in patched diff --git a/tests/units/test_app.py b/tests/units/test_app.py index feda7d3b7ef..8e5a28e6760 100644 --- a/tests/units/test_app.py +++ b/tests/units/test_app.py @@ -381,6 +381,16 @@ def test_prepare_404_page_empty_string_metadata_uses_defaults(): assert prepared.page.description == constants.Page404.DESCRIPTION +def test_default_404_page_has_no_source_module(): + """The framework's default 404 page belongs to no user module. + + Otherwise its module comes from the call stack of whatever process + compiles the app, which differs between processes. + """ + prepared = App()._prepare_page(route=constants.Page404.SLUG) + assert prepared.page._source_module is None + + def test_add_page_set_route(app: App, index_page: ComponentCallable): """Test adding a page to an app. @@ -5528,6 +5538,100 @@ def test_compile_emits_stage_spans( assert parent.span_id == root.get_span_context().span_id +def test_should_compile_consumes_the_nocompile_marker_under_a_daemon( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, request: pytest.FixtureRequest +): + """A backend worker consumes its ``nocompile`` marker even while a daemon owns .web. + + Otherwise the marker outlives the worker and the daemon's next compile + child finds it, consumes it and skips the compile, so the first edit never + reaches the browser. Daemon processes never touch the marker at all. + """ + from reflex.utils import compile_daemon, path_ops + + web_dir = tmp_path / ".web" + web_dir.mkdir() + monkeypatch.setattr("reflex.utils.prerequisites.get_web_dir", lambda: web_dir) + daemon_lock = tmp_path / "daemon.lock" + monkeypatch.setattr(compile_daemon, "_daemon_lock_path", lambda: daemon_lock) + monkeypatch.delenv(environment.REFLEX_SKIP_COMPILE.name, raising=False) + monkeypatch.delenv(environment.REFLEX_COMPILE_DAEMON.name, raising=False) + descriptor = path_ops.try_lock(daemon_lock) + assert descriptor is not None + request.addfinalizer(lambda: os.close(descriptor)) + marker = web_dir / constants.NOCOMPILE_FILE + marker.touch() + app = App() + + # The worker stands down and takes its marker with it. + assert app._should_compile() is False + assert not marker.exists() + + # The daemon (and its compile children) compile regardless of the marker. + marker.touch() + monkeypatch.setenv(environment.REFLEX_COMPILE_DAEMON.name, "1") + assert app._should_compile() is True + assert marker.exists() + + +def test_compile_reuses_pages_the_incremental_attempt_already_evaluated( + compilable_app: tuple[App, Path], + mocker: MockerFixture, + monkeypatch: pytest.MonkeyPatch, +): + """A post-evaluation cache fallback must not evaluate a page twice. + + Evaluating a page registers the states it defines; a second evaluation in + the same process renames a locally defined state (``LocalState_0``) while + the backend, which evaluates the page once, knows it as ``LocalState``. The + full compile therefore adopts the pages the incremental attempt compiled. + """ + from reflex.compiler import disk_cache + from reflex.compiler.plugins import default_page_plugins + + app, web_dir = compilable_app + mocker.patch("reflex.utils.prerequisites.get_web_dir", return_value=web_dir) + monkeypatch.setenv("REFLEX_COMPILE_CACHE", "1") + evaluations: list[str] = [] + + def local_state_page(): + evaluations.append("local") + + class AdoptedLocalState(rx.State): + value: int = 0 + + return rx.el.div(AdoptedLocalState.value) + + def other_page(): + evaluations.append("other") + return rx.el.div("other") + + app.add_page(local_state_page, route="local") + app.add_page(other_page, route="other") + + # What the incremental attempt did before deciding it must fall back. + evaluated = CompileContext( + app=app, + pages=[app._unevaluated_pages["local"]], + hooks=CompilerHooks(plugins=default_page_plugins(style=app.style)), + ) + with evaluated: + evaluated.compile() + registered = evaluated.stateful_routes["local"] + mocker.patch.object(disk_cache, "try_incremental_rebuild", return_value=evaluated) + + app._compile() + + assert evaluations == ["local", "other"] + assert app._stateful_pages.keys() >= {"local"} + assert set(app._pages) >= {"local", "other"} + routes_dir = web_dir / constants.Dirs.PAGES / constants.Dirs.ROUTES + assert (routes_dir / "[local]._index.jsx").exists() + assert (routes_dir / "[other]._index.jsx").exists() + assert registered[0].endswith("adopted_local_state") + assert all(not name.endswith("_0") for name in registered) + + def test_write_stateful_pages_marker_never_truncates_final_path( tmp_path: Path, mocker: MockerFixture, monkeypatch: pytest.MonkeyPatch ): @@ -5686,12 +5790,24 @@ def reader(): with ThreadPoolExecutor(max_workers=8) as pool: readers = [pool.submit(reader) for _ in range(4)] + writers = [pool.submit(writer) for _ in range(4)] try: - writers = [pool.submit(writer) for _ in range(4)] for future in writers: future.result() - finally: + except BaseException: + # Free the readers so the pool can join. Never on success: a reader + # released from the last round but not yet scheduled would see the + # aborted barrier and raise. round_started.abort() + raise for future in readers: future.result() assert json.loads(marker.read_text()) == routes + + +def test_backend_compile_is_marked_serving(mocker): + """The backend's own compile tells the compiler that it serves the app.""" + app = App() + compile_ = mocker.patch.object(app, "_compile") + app() + assert compile_.call_args.kwargs["serving"] is True diff --git a/tests/units/test_reflex.py b/tests/units/test_reflex.py index e2923104a86..e6d9f74c466 100644 --- a/tests/units/test_reflex.py +++ b/tests/units/test_reflex.py @@ -561,6 +561,38 @@ def test_missing_command_tolerates_flags(caplog: pytest.LogCaptureFixture): assert "No such option" not in result.output +def test_dev_daemon_does_not_disable_backend_fallback(monkeypatch, mocker): + """The daemon owns compilation only while its process remains alive.""" + import contextlib + + from reflex_base import constants + from reflex_base.environment import environment + + from reflex.utils import build, compile_daemon, processes, telemetry + + monkeypatch.setenv("REFLEX_COMPILE_CACHE", "1") + monkeypatch.delenv("REFLEX_SKIP_COMPILE", raising=False) + compiled = mocker.patch.object(reflex, "_compile_app") + mocker.patch.object(build, "setup_frontend") + mocker.patch.object(telemetry, "send") + mocker.patch("atexit.register") + concurrent = mocker.patch.object( + processes, "run_concurrently_context", return_value=contextlib.nullcontext() + ) + daemon = mocker.patch.object( + compile_daemon, "running", return_value=contextlib.nullcontext() + ) + reflex._run_dev(constants.RunningMode.FRONTEND_ONLY, None, None, "localhost") + compiled.assert_called_once() + assert not environment.REFLEX_SKIP_COMPILE.get() + # The run scope owns the daemon process; no concurrent task supervises it. + daemon.assert_called_once_with(mocker.ANY, initial_compiled=True) + assert all( + getattr(command[0], "__module__", None) != compile_daemon.__name__ + for command in concurrent.call_args.args + ) + + def test_init_records_version_check_after_frontend_setup( tmp_path, monkeypatch: pytest.MonkeyPatch ): diff --git a/tests/units/test_telemetry.py b/tests/units/test_telemetry.py index db2cc600e3a..0fe812d4d60 100644 --- a/tests/units/test_telemetry.py +++ b/tests/units/test_telemetry.py @@ -963,3 +963,23 @@ def test_send_event_swallows_delivery_errors(mocker: MockerFixture): "reflex.utils.telemetry.urllib.request.urlopen", side_effect=OSError("down") ) assert not telemetry._send_event({"api_key": "k", "event": "e"}) # pyright: ignore[reportArgumentType] + + +def test_send_spools_events_for_another_process(tmp_path, monkeypatch, mocker): + """With a spool configured, events are appended to it instead of sent.""" + from reflex_base.environment import environment + + spool = tmp_path / "spool.jsonl" + monkeypatch.setenv(environment.REFLEX_TELEMETRY_SPOOL.name, str(spool)) + submit = mocker.patch.object(telemetry, "_submit") + telemetry.send("compile", telemetry_enabled=True, properties={"pages_count": 2}) + telemetry.send("export", telemetry_enabled=True, status="success") + submit.assert_not_called() + monkeypatch.delenv(environment.REFLEX_TELEMETRY_SPOOL.name) + sent = mocker.patch.object(telemetry, "send") + telemetry.send_spooled(spool) + assert [call.args for call in sent.call_args_list] == [("compile",), ("export",)] + assert sent.call_args_list[0].kwargs == {"properties": {"pages_count": 2}} + assert sent.call_args_list[1].kwargs == {"properties": None, "status": "success"} + assert not spool.exists() + telemetry.send_spooled(spool) # nothing spooled: nothing to do diff --git a/tests/units/utils/test_compile_daemon.py b/tests/units/utils/test_compile_daemon.py new file mode 100644 index 00000000000..7d49ad32459 --- /dev/null +++ b/tests/units/utils/test_compile_daemon.py @@ -0,0 +1,1606 @@ +"""Tests for the warm fork-per-compile dev compile daemon.""" + +import contextlib +import errno +import os +import sys +from pathlib import Path +from typing import Any +from unittest.mock import Mock + +import pytest + +import reflex as rx +from reflex.compiler import disk_cache +from reflex.testing import AppHarness +from reflex.utils import compile_daemon + + +class _SurvivingModuleState(rx.State): + """Module-level state, like one from a non-purged installed package.""" + + b: int = 0 + + +@pytest.fixture(autouse=True) +def _isolated_daemon_process_state(tmp_path_factory, monkeypatch): + """Keep the daemon's lock files and bytecode cache out of the user's dirs. + + Args: + tmp_path_factory: The pytest temporary path factory. + monkeypatch: The pytest monkeypatch fixture. + """ + monkeypatch.setenv("REFLEX_DIR", str(tmp_path_factory.mktemp("reflex-data"))) + monkeypatch.setattr(sys, "pycache_prefix", sys.pycache_prefix) + + +def test_can_fork_rejects_remaining_native_threads(monkeypatch): + """Native threads must force the subprocess fallback after the grace period.""" + monkeypatch.setattr(compile_daemon, "_quiesce_parent", lambda: None) + monkeypatch.setattr(compile_daemon.threading, "active_count", lambda: 1) + monkeypatch.setattr(compile_daemon, "_os_thread_count", lambda: 8) + ticks = iter([0.0, 1.0]) + monkeypatch.setattr(compile_daemon.time, "monotonic", lambda: next(ticks)) + assert not compile_daemon._can_fork() + + +def test_missed_changes_includes_new_sources_and_assets(tmp_path): + """Catch sources and assets created while the OS watcher is stopped.""" + state = compile_daemon._WatchState(roots=[tmp_path], root=tmp_path) + before = compile_daemon._dependency_snapshot(state) + source = tmp_path / "new_page.py" + source.write_text("VALUE = 1\n") + assets = tmp_path / "assets" + assets.mkdir() + asset = assets / "image.txt" + asset.write_text("new asset") + after = compile_daemon._dependency_snapshot(state) + assert {source, asset} <= compile_daemon._missed_changes(before, after) + + +def test_next_changes_catches_edit_before_watcher_starts(tmp_path, monkeypatch): + """Reconcile the previous snapshot after the watcher has subscribed.""" + import watchfiles + + state = compile_daemon._WatchState(roots=[tmp_path], root=tmp_path) + state.checkpoint = compile_daemon._dependency_snapshot(state) + source = tmp_path / "new_page.py" + source.write_text("VALUE = 1\n") + + def watch(*args, **kwargs): + """Simulate a watcher timeout with no event for the earlier edit. + + Yields: + An empty event batch. + """ + yield set() + + monkeypatch.setattr(watchfiles, "watch", watch) + assert compile_daemon._next_changes(state, lambda: True) == {source} + + +def test_iter_source_files_picks_content_skips_build_dirs(tmp_path): + (tmp_path / "page.py").write_text("x = 1\n") + (tmp_path / "doc.md").write_text("# doc\n") + (tmp_path / "guide.mdx").write_text("mdx\n") + (tmp_path / "data.txt").write_text("ignored\n") # not a watched suffix + web = tmp_path / ".web" + web.mkdir() + (web / "build.js").write_text("// artifact\n") + pycache = tmp_path / "__pycache__" + pycache.mkdir() + (pycache / "stale.py").write_text("# compiled\n") + + found = {p.name for p in compile_daemon._iter_source_files(tmp_path)} + assert found == {"page.py", "doc.md", "guide.mdx"} + + +def test_under_roots_matches_only_real_ancestors(tmp_path): + root = tmp_path / "app" + roots = [root] + assert compile_daemon._under_roots(root, roots) + assert compile_daemon._under_roots(root / "pages" / "index.py", roots) + # A sibling sharing the root's string prefix is NOT under it. + assert not compile_daemon._under_roots(tmp_path / "app_extra" / "m.py", roots) + assert not compile_daemon._under_roots(tmp_path / "other.py", roots) + # The root's own parent is not under it either. + assert not compile_daemon._under_roots(tmp_path, roots) + + +def test_external_dependency_files_includes_sibling_markdown(tmp_path, monkeypatch): + """A page's markdown read from a sibling dir (outside the app root) is watched. + + This is the regression for markdown edits never triggering a reload: such + files live outside ``get_reload_paths`` and are only known via the compile + manifest's per-page dependency sets. + """ + app_root = tmp_path / "app" + app_root.mkdir() + own_module = app_root / "page.py" + own_module.write_text("x = 1\n") + sibling_md = tmp_path / "docs" / "guide.md" + sibling_md.parent.mkdir() + sibling_md.write_text("# guide\n") + + manifest = { + "pages": { + "/g": { + "deps": [ + str(own_module), # under the app root -> not external + str(sibling_md), # sibling dir -> must be watched + ] + } + } + } + monkeypatch.setattr(disk_cache, "load_manifest", lambda: manifest) + + external = compile_daemon._external_dependency_files([app_root.resolve()]) + assert sibling_md.resolve() in external + assert own_module.resolve() not in external + + +def test_external_dependency_files_empty_without_manifest(tmp_path, monkeypatch): + monkeypatch.setattr(disk_cache, "load_manifest", lambda: None) + assert compile_daemon._external_dependency_files([tmp_path]) == set() + + +def test_snapshot_detects_external_markdown_change(tmp_path, monkeypatch): + """The watch snapshot includes sibling markdown and reflects its mtime change.""" + app_root = tmp_path / "app" + app_root.mkdir() + (app_root / "page.py").write_text("x = 1\n") + sibling_md = tmp_path / "docs" / "guide.md" + sibling_md.parent.mkdir() + sibling_md.write_text("# v1\n") + monkeypatch.setattr( + disk_cache, + "load_manifest", + lambda: {"pages": {"/g": {"deps": [str(sibling_md)]}}}, + ) + + roots = [app_root.resolve()] + state = compile_daemon._WatchState.build(roots, tmp_path.resolve()) + snap1 = compile_daemon._snapshot(state.watch_paths()) + assert sibling_md.resolve() in snap1 + assert (app_root / "page.py").resolve() in snap1 + # The OS watcher is pointed at the sibling directory too. + assert sibling_md.resolve().parent in state.targets() + + # Force a distinct mtime (deterministic, independent of fs mtime resolution). + bumped = snap1[sibling_md.resolve()] + 1_000_000_000 + os.utime(sibling_md, ns=(bumped, bumped)) + snap2 = compile_daemon._snapshot(state.watch_paths()) + assert snap2[sibling_md.resolve()] != snap1[sibling_md.resolve()] + + +def test_watch_state_accepts_only_compile_inputs(tmp_path, monkeypatch): + """Recorded data deps of any suffix count; build output never does.""" + app_root = tmp_path / "app" + app_root.mkdir() + data = app_root / "table.json" + data.write_text("{}") + monkeypatch.setattr( + disk_cache, "load_manifest", lambda: {"pages": {"/t": {"deps": [str(data)]}}} + ) + (tmp_path / "assets").mkdir() + state = compile_daemon._WatchState.build([app_root.resolve()], tmp_path.resolve()) + + assert state.accepts(app_root / "page.py") + assert state.accepts(app_root / "guide.md") + assert state.accepts(data) # not a watched suffix, but a recorded dependency + assert state.accepts(tmp_path / "assets" / "logo.svg") + assert not state.accepts(app_root / "other.json") # unrecorded data file + assert not state.accepts(tmp_path / ".web" / "app" / "routes" / "x.jsx") + assert not state.accepts(tmp_path / "elsewhere" / "page.py") # outside roots + + +def test_iter_changes_reports_an_edit(tmp_path): + """The OS watcher reports an edited source file (no polling involved).""" + import threading + import time + + pytest.importorskip("watchfiles") + app_root = tmp_path / "app" + app_root.mkdir() + page = app_root / "page.py" + page.write_text("x = 1\n") + state = compile_daemon._WatchState([app_root.resolve()], tmp_path.resolve()) + got: list[set] = [] + + def run() -> None: + changed = compile_daemon._next_changes(state, lambda: not got) + if changed: + got.append(changed) + + thread = threading.Thread(target=run, daemon=True) + thread.start() + deadline = time.monotonic() + 10 + while not got and time.monotonic() < deadline: + page.write_text(f"x = {time.monotonic()}\n") + time.sleep(0.2) + thread.join(5) + assert got + assert page.resolve() in {p.resolve() for p in got[0]} + # The native watcher thread is gone, so the daemon may fork again. + assert not any("watchfiles" in t.name.lower() for t in threading.enumerate()) + + +def test_missed_changes_catches_edits_during_compile(tmp_path): + a, b = tmp_path / "a.py", tmp_path / "b.py" + a.write_text("1") + b.write_text("1") + state = compile_daemon._WatchState([tmp_path], tmp_path, known={a, b}) + before = compile_daemon._dependency_snapshot(state) + bumped = before[b] + 1_000_000_000 + os.utime(b, ns=(bumped, bumped)) + c = tmp_path / "c.py" + c.write_text("1") + state.known.add(c) + after = compile_daemon._dependency_snapshot(state) + assert compile_daemon._missed_changes(before, after) == {b, c} + assert compile_daemon._missed_changes(after, after) == set() + + +def test_wait_for_compile_respects_lock(tmp_path, monkeypatch): + import time + + from reflex.utils import path_ops + + monkeypatch.setattr("reflex.utils.prerequisites.get_backend_dir", lambda: tmp_path) + monkeypatch.setattr(compile_daemon, "daemon_active", lambda: True) + # No compile in progress -> returns at once. + start = time.monotonic() + compile_daemon.wait_for_compile(timeout=1) + assert time.monotonic() - start < 0.2 + # A compile in progress -> waits until it finishes (or the timeout). + with compile_daemon._compile_lock(tmp_path): + start = time.monotonic() + compile_daemon.wait_for_compile(timeout=0.3) + assert time.monotonic() - start >= 0.3 + # A lock file nobody holds (a compile killed mid-way) is not waited on, + # even when it holds the pid of a live process. + lock = compile_daemon._lock_path() + lock.parent.mkdir(parents=True, exist_ok=True) + lock.write_text(str(os.getpid())) + assert not path_ops.is_locked(lock) + start = time.monotonic() + compile_daemon.wait_for_compile(timeout=5) + assert time.monotonic() - start < 0.5 + # A daemon that exits mid-wait releases the worker. + monkeypatch.setattr(compile_daemon, "daemon_active", lambda: False) + with compile_daemon._compile_lock(tmp_path): + start = time.monotonic() + compile_daemon.wait_for_compile(timeout=5) + assert time.monotonic() - start < 0.5 + + +@pytest.mark.parametrize("blocked_attempts", [1, 2]) +def test_record_built_inputs_retries_windows_readers( + tmp_path, monkeypatch, blocked_attempts +): + """Publish the new record after Windows readers release the old record.""" + monkeypatch.setattr("reflex.utils.prerequisites.get_backend_dir", lambda: tmp_path) + monkeypatch.setattr(compile_daemon.constants, "IS_WINDOWS", True) + page = tmp_path / "page.py" + page.write_text("v1") + built = compile_daemon._snapshot({page}) + compile_daemon._record_built_inputs(built) + record = compile_daemon._built_inputs_path() + previous_record = record.read_text() + mtime = built[page] + 1_000_000_000 + os.utime(page, ns=(mtime, mtime)) + replace = Path.replace + attempts = 0 + + def replace_after_reader_closes(source, target): + """Keep the previous record intact while a reader blocks replacement. + + Args: + source: Temporary record to publish. + target: Destination record. + + Returns: + The destination after the reader releases it. + + Raises: + PermissionError: While the reader still holds the destination open. + """ + nonlocal attempts + attempts += 1 + assert record.read_text() == previous_record + assert not compile_daemon._built_inputs_current() + if attempts <= blocked_attempts: + raise PermissionError(errno.EACCES, "Reader still has the record open") + return replace(source, target) + + monkeypatch.setattr(Path, "replace", replace_after_reader_closes) + sleep = Mock() + monkeypatch.setattr(compile_daemon.time, "sleep", sleep) + compile_daemon._record_built_inputs(compile_daemon._snapshot({page})) + assert compile_daemon._built_inputs_current() + assert sleep.call_count == blocked_attempts + assert not list(tmp_path.glob("*.tmp")) + + +@pytest.mark.parametrize( + ("is_windows", "error", "attempts"), + [ + (True, PermissionError(errno.EACCES, "Access denied"), 3), + (False, PermissionError(errno.EACCES, "Access denied"), 1), + (True, OSError(errno.ENOSPC, "No space left"), 1), + ], +) +def test_record_built_inputs_propagates_errors( + tmp_path, monkeypatch, is_windows, error, attempts +): + """Bound retries, retain the old record, and remove a failed temporary write.""" + monkeypatch.setattr("reflex.utils.prerequisites.get_backend_dir", lambda: tmp_path) + monkeypatch.setattr(compile_daemon.constants, "IS_WINDOWS", is_windows) + compile_daemon._record_built_inputs({}) + record = compile_daemon._built_inputs_path() + previous_record = record.read_text() + replace = Mock(side_effect=error) + monkeypatch.setattr(Path, "replace", replace) + sleep = Mock() + monkeypatch.setattr(compile_daemon.time, "sleep", sleep) + with pytest.raises(type(error)) as raised: + compile_daemon._record_built_inputs({tmp_path / "page.py": 1}) + assert raised.value is error + assert replace.call_count == attempts + assert sleep.call_count == attempts - 1 + assert record.read_text() == previous_record + assert not list(tmp_path.glob("*.tmp")) + + +def test_wait_for_compile_waits_for_unbuilt_edits(tmp_path, monkeypatch): + """Without a lock, a worker still waits while a built input has changed. + + The daemon takes its lock only once its watcher reports the edit, so a + worker that restarts first would read the previous stateful-pages marker. + """ + import threading + import time + + monkeypatch.setattr("reflex.utils.prerequisites.get_backend_dir", lambda: tmp_path) + monkeypatch.setattr(compile_daemon, "daemon_active", lambda: True) + page = tmp_path / "page.py" + page.write_text("v1") + built = compile_daemon._snapshot({page}) + compile_daemon._record_built_inputs(built) + start = time.monotonic() + compile_daemon.wait_for_compile(timeout=1) + assert time.monotonic() - start < 0.2 + mtime = built[page] + 1_000_000_000 + os.utime(page, ns=(mtime, mtime)) + start = time.monotonic() + compile_daemon.wait_for_compile(timeout=0.3) + assert time.monotonic() - start >= 0.3 + # The daemon builds the edit: the worker proceeds. + timer = threading.Timer( + 0.1, compile_daemon._record_built_inputs, [compile_daemon._snapshot({page})] + ) + timer.start() + start = time.monotonic() + try: + compile_daemon.wait_for_compile(timeout=5) + assert time.monotonic() - start < 1 + assert compile_daemon._built_inputs_current() + finally: + timer.join() + + +def test_first_party_module_names_caches_file_classification(tmp_path, monkeypatch): + """Module files are classified once; a repeat pass resolves no paths.""" + import sys + import types + + mod = types.ModuleType("fp_cached_mod") + mod.__file__ = str(tmp_path / "pkg" / "cached_mod.py") + monkeypatch.setitem(sys.modules, "fp_cached_mod", mod) + roots = [tmp_path.resolve()] + assert "fp_cached_mod" in compile_daemon._first_party_module_names(roots) + assert compile_daemon._first_party_file_cache[mod.__file__] is True + + calls = 0 + original = Path.resolve + + def counting_resolve(self, *args, **kwargs): + nonlocal calls + calls += 1 + return original(self, *args, **kwargs) + + monkeypatch.setattr(Path, "resolve", counting_resolve) + assert "fp_cached_mod" in compile_daemon._first_party_module_names(roots) + assert calls == 0 + + +def test_first_party_module_names_includes_namespace_packages(tmp_path, monkeypatch): + """Namespace packages (no ``__file__``) are captured for purge by name. + + Regression: a namespace package left in ``sys.modules`` after its regular + siblings were purged broke re-import with a ``KeyError`` on the parent path. + They are now identified by sharing a first-party top-level name (derived from + a regular sibling's ``__file__``), never by their lazy ``__path__``. + """ + import importlib + import sys + + nspkg = tmp_path / "ns_under_test" + nspkg.mkdir() # no __init__.py -> namespace package + (nspkg / "leaf.py").write_text("Y = 1\n") # regular submodule with __file__ + monkeypatch.syspath_prepend(str(tmp_path)) + try: + pkg = importlib.import_module("ns_under_test") + importlib.import_module("ns_under_test.leaf") + assert getattr(pkg, "__file__", None) is None # confirm it's a namespace pkg + + names = compile_daemon._first_party_module_names([tmp_path.resolve()]) + assert "ns_under_test" in names # namespace pkg captured by name + assert "ns_under_test.leaf" in names # regular module captured via __file__ + finally: + sys.modules.pop("ns_under_test", None) + sys.modules.pop("ns_under_test.leaf", None) + + +def test_reset_model_metadata_skips_unloaded_model_layers(monkeypatch): + """A child of an app that never loaded the model layer does not import it. + + Importing it only to clear its registries cost every hot reload ~0.3 s. + """ + for name in ("sqlmodel", "reflex.model"): + monkeypatch.delitem(sys.modules, name, raising=False) + compile_daemon._reset_model_metadata() + assert "sqlmodel" not in sys.modules + assert "reflex.model" not in sys.modules + + +@pytest.mark.skipif(not hasattr(os, "fork"), reason="requires os.fork (POSIX)") +def test_reset_model_metadata_allows_table_redefinition(): + """After reset, an ``rx.Model`` table can be redefined without conflict. + + Regression: the forked child inherits the warm, populated SQLAlchemy + ``MetaData``; re-evaluating a page that defines a model raised + ``Table '...' is already defined``. Run in a fork so clearing the global + metadata can't affect the test process. + """ + # ``rx.Model(table=True)`` needs the SQLModel stack; the db-less unit-test + # job uninstalls it, so skip there rather than fail on the first definition. + pytest.importorskip("sqlmodel") + + read_fd, write_fd = os.pipe() + pid = os.fork() + if pid == 0: # child + os.close(read_fd) + result = b"E" + try: + import warnings + + warnings.simplefilter("ignore") + import reflex as rx + + def _define(): + class DaemonResetUser(rx.Model, table=True): + name: str + + _define() + compile_daemon._reset_model_metadata() + _define() # must NOT raise "Table 'daemonresetuser' is already defined" + result = b"1" + except Exception: + import traceback + + traceback.print_exc() + finally: + os.write(write_fd, result) + os.close(write_fd) + os._exit(0) + + os.close(write_fd) + out = os.read(read_fd, 1) + os.close(read_fd) + os.waitpid(pid, 0) + assert out == b"1" + + +@pytest.mark.skipif(not hasattr(os, "fork"), reason="requires os.fork (POSIX)") +def test_reset_first_party_keeps_surviving_module_states(tmp_path): + """States from modules that survive the purge stay registered. + + A class body in a non-purged module (framework internals, installed or + workspace packages like reflex_site_shared) never re-executes in the child, + so dropping its registration loses the state from the app's state tree — + and from the compiled contexts file — breaking the frontend's dispatch map. + Purged-module states and runtime-created ``reflex.istate.dynamic`` states + must still be dropped (they re-register on re-import/re-evaluation), and + the dynamic module's attributes must be reset so re-created local states + get their deterministic fresh-process names, matching the cold backend. + """ + mod_file = tmp_path / "purged_state_mod.py" + mod_file.write_text( + "import reflex as rx\n\nclass PurgedState(rx.State):\n a: int = 0\n" + ) + + read_fd, write_fd = os.pipe() + pid = os.fork() + if pid == 0: # child + os.close(read_fd) + result = b"E" + try: + import importlib.util + import sys + + import reflex.istate.dynamic as istate_dynamic + from reflex.state import State, all_base_state_classes + + spec = importlib.util.spec_from_file_location("purged_state_mod", mod_file) + assert spec is not None + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) # type: ignore[union-attr] + sys.modules["purged_state_mod"] = module + + def _local_state() -> type[rx.State]: + class LocalState(rx.State): + c: int = 0 + + return LocalState + + local_cls = _local_state() + assert local_cls.__module__ == istate_dynamic.__name__ + local_name = local_cls.__name__ + + from reflex_base.vars.base import ( + get_unique_variable_name, + reset_unique_variable_names, + ) + + get_unique_variable_name() # the parent drew names + compile_daemon._reset_first_party([tmp_path.resolve()]) + + substate_names = {c.__name__ for c in State.get_substates()} + surviving_kept = ( + "_SurvivingModuleState" in substate_names + and _SurvivingModuleState.get_full_name() in all_base_state_classes + ) + purged_dropped = ( + "purged_state_mod" not in sys.modules + and "PurgedState" not in substate_names + ) + dynamic_dropped = local_name not in substate_names and not [ + n for n in vars(istate_dynamic) if not n.startswith("__") + ] + # A re-created local state gets its original (fresh-process) name, + # not a collision-suffixed drift. + redefined_deterministic = _local_state().__name__ == local_name + # Re-imported modules draw the names a cold process draws. + child_names = [get_unique_variable_name() for _ in range(3)] + reset_unique_variable_names() + cold_names = [get_unique_variable_name() for _ in range(3)] + names_reset = child_names == cold_names + + result = ( + b"1" + if ( + surviving_kept + and purged_dropped + and dynamic_dropped + and redefined_deterministic + and names_reset + ) + else b"0" + ) + except Exception: + import traceback + + traceback.print_exc() + finally: + os.write(write_fd, result) + os.close(write_fd) + os._exit(0) + + os.close(write_fd) + out = os.read(read_fd, 1) + os.close(read_fd) + os.waitpid(pid, 0) + assert out == b"1" + + +@pytest.mark.skipif(not hasattr(os, "fork"), reason="requires os.fork (POSIX)") +def test_reset_first_party_keeps_dynamic_states_surviving_modules_created(tmp_path): + """A dynamic state a surviving module created at import stays registered. + + An installed library's local state or ``ComponentState.create()`` runs + once, at import; purged modules re-import, surviving ones do not, so + dropping it would lose it from the state tree for good. A state a page + defined (per the manifest) is dropped: the page re-evaluates. + """ + read_fd, write_fd = os.pipe() + pid = os.fork() + if pid == 0: # child + os.close(read_fd) + result = b"E" + try: + import reflex.istate.dynamic as istate_dynamic + from reflex.state import State + + def _import_time_state() -> type[rx.State]: + class LibraryState(rx.State): + c: int = 0 + + return LibraryState + + def _page_state() -> type[rx.State]: + class PageLocalState(rx.State): + d: int = 0 + + return PageLocalState + + library_cls = _import_time_state() + page_cls = _page_state() + manifest = { + "pages": { + "/p": {"state_slice": {"schema": {page_cls.get_full_name(): []}}} + } + } + disk_cache.load_manifest = lambda: manifest + + compile_daemon._reset_first_party([tmp_path.resolve()]) + + names = {c.__name__ for c in State.get_substates()} + kept = library_cls.__name__ in names and hasattr( + istate_dynamic, library_cls.__name__ + ) + dropped = page_cls.__name__ not in names and not hasattr( + istate_dynamic, page_cls.__name__ + ) + # The page's local state re-registers under its original name. + redefined = _page_state().__name__ == page_cls.__name__ + result = b"1" if (kept and dropped and redefined) else b"0" + except Exception: + import traceback + + traceback.print_exc() + finally: + os.write(write_fd, result) + os.close(write_fd) + os._exit(0) + + os.close(write_fd) + out = os.read(read_fd, 1) + os.close(read_fd) + os.waitpid(pid, 0) + assert out == b"1" + + +@pytest.mark.skipif(not hasattr(os, "fork"), reason="requires os.fork (POSIX)") +def test_reset_first_party_drops_memos_of_purged_modules(tmp_path): + """A purged module's memos leave the registry; the module re-registers them. + + Otherwise a renamed memo collides with the parent's registration of its + old name on every re-import, and a deleted memo keeps being emitted. + Memos of surviving modules stay. + """ + mod_file = tmp_path / "purged_memo_mod.py" + mod_file.write_text( + "import reflex as rx\n\n@rx.memo\ndef my_card() -> rx.Component:\n" + " return rx.text('card')\n" + ) + + read_fd, write_fd = os.pipe() + pid = os.fork() + if pid == 0: # child + os.close(read_fd) + result = b"E" + try: + import importlib.util + import sys + + from reflex_base.components.memo import MEMOS + + def load(source: str) -> None: + mod_file.write_text(source) + spec = importlib.util.spec_from_file_location( + "purged_memo_mod", mod_file + ) + assert spec is not None + module = importlib.util.module_from_spec(spec) + sys.modules["purged_memo_mod"] = module + spec.loader.exec_module(module) # type: ignore[union-attr] + + load(mod_file.read_text()) + surviving = {key for key in MEMOS if key[1] != "purged_memo_mod"} + assert ("MyCard", "purged_memo_mod") in MEMOS + + compile_daemon._reset_first_party([tmp_path.resolve()]) + dropped = ("MyCard", "purged_memo_mod") not in MEMOS + kept = surviving <= set(MEMOS) + # The renamed memo re-registers under its export name without a + # collision with the parent's registration. + load(mod_file.read_text().replace("def my_card", "def _my_card")) + renamed = ("MyCard", "purged_memo_mod") in MEMOS + result = b"1" if (dropped and kept and renamed) else b"0" + except Exception: + import traceback + + traceback.print_exc() + finally: + os.write(write_fd, result) + os.close(write_fd) + os._exit(0) + + os.close(write_fd) + out = os.read(read_fd, 1) + os.close(read_fd) + os.waitpid(pid, 0) + assert out == b"1" + + +@pytest.mark.skipif(not hasattr(os, "fork"), reason="requires os.fork (POSIX)") +def test_reset_first_party_purges_modules_and_registries(tmp_path): + """``_reset_first_party`` purges first-party modules and clears registries. + + Runs in a forked child so the registry reset cannot affect the test process. + """ + mod_file = tmp_path / "fp_module.py" + mod_file.write_text("VALUE = 1\n") + + read_fd, write_fd = os.pipe() + pid = os.fork() + if pid == 0: # child + os.close(read_fd) + result = b"E" + try: + import importlib.util + import sys + + spec = importlib.util.spec_from_file_location( + "fp_module_under_test", mod_file + ) + assert spec is not None + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) # type: ignore[union-attr] + sys.modules["fp_module_under_test"] = module + + from reflex_base.registry import RegistrationContext + + from reflex.state import all_base_state_classes + + RegistrationContext.ensure_context().base_states["sentinel"] = object() # type: ignore[assignment] + all_base_state_classes["sentinel"] = None + RegistrationContext.ensure_context().decorated_pages.append(( + lambda: None, + {}, + )) + + compile_daemon._reset_first_party([tmp_path.resolve()]) + + purged = "fp_module_under_test" not in sys.modules + # The sentinel (a bare object, no surviving module) must be dropped; + # real states from surviving modules are kept (see the test above). + cleared = ( + "sentinel" not in RegistrationContext.ensure_context().base_states + and "sentinel" not in all_base_state_classes + and not RegistrationContext.ensure_context().decorated_pages + ) + result = b"1" if (purged and cleared) else b"0" + except Exception: + import traceback + + traceback.print_exc() + finally: + os.write(write_fd, result) + os.close(write_fd) + os._exit(0) + + os.close(write_fd) + out = os.read(read_fd, 1) + os.close(read_fd) + os.waitpid(pid, 0) + assert out == b"1" + + +def test_quiesce_parent_stops_telemetry_worker(): + """The telemetry worker must not keep the daemon parent multi-threaded. + + The initial compile sends a telemetry event, whose executor thread would + otherwise outlive it and make every later fork check fail silently. + """ + import threading + + from reflex.utils import telemetry + + telemetry._get_telemetry_executor().submit(lambda: None).result() + assert any(t.name.startswith("reflex-telemetry") for t in threading.enumerate()) + + compile_daemon._quiesce_parent() + + assert telemetry._executor is None + assert not any(t.name.startswith("reflex-telemetry") for t in threading.enumerate()) + # A later send lazily recreates a worker, so shutdown is not one-way. + telemetry._get_telemetry_executor().submit(lambda: None).result() + telemetry._shutdown_executor() + assert telemetry._executor is None + + +def test_owns_compilation_follows_the_daemon_lock(tmp_path, monkeypatch): + """Backend reload workers defer to a daemon holding its lock; nobody else. + + The daemon always owns the build, and so does a process that asked for + a compile (``reflex compile``, ``reflex export``): deferring would leave + its output to the daemon's dev build. + """ + from reflex_base.environment import environment + + from reflex.utils import path_ops + + monkeypatch.setattr("reflex.utils.prerequisites.get_backend_dir", lambda: tmp_path) + monkeypatch.delenv(environment.REFLEX_COMPILE_DAEMON.name, raising=False) + monkeypatch.setenv(environment.REFLEX_DEV_BACKEND_RELOAD_ACTIVE.name, "1") + # No daemon: a worker compiles as usual. + assert compile_daemon.daemon_active() is False + assert compile_daemon.owns_compilation() is True + # A lock file nobody holds (a killed daemon, a copied .web) names no owner, + # even when it holds the pid of a live process. + lock = compile_daemon._daemon_lock_path() + lock.parent.mkdir(parents=True, exist_ok=True) + lock.write_text(str(os.getpid())) + assert compile_daemon.daemon_active() is False + # A daemon holding the lock: workers stand down. + descriptor = path_ops.try_lock(lock) + assert descriptor is not None + try: + assert compile_daemon.daemon_active() is True + assert compile_daemon.owns_compilation() is False + # ...but the daemon's own processes still own the build. + monkeypatch.setenv(environment.REFLEX_COMPILE_DAEMON.name, "1") + assert compile_daemon.owns_compilation() is True + # ...and so does a CLI compile, which is no reload worker. + monkeypatch.delenv(environment.REFLEX_COMPILE_DAEMON.name) + monkeypatch.delenv(environment.REFLEX_DEV_BACKEND_RELOAD_ACTIVE.name) + assert compile_daemon.owns_compilation() is True + finally: + os.close(descriptor) + + +def test_lock_files_live_outside_the_web_dir(tmp_path, monkeypatch): + """``rm -rf .web`` while the daemon runs must not unlink the lock it holds.""" + from reflex_base.environment import environment + + backend = tmp_path / ".web" / "backend" + monkeypatch.setattr("reflex.utils.prerequisites.get_backend_dir", lambda: backend) + for lock in (compile_daemon._lock_path(), compile_daemon._daemon_lock_path()): + assert not lock.is_relative_to(tmp_path / ".web") + assert lock.is_relative_to(environment.REFLEX_DIR.get().expanduser()) + assert compile_daemon._lock_path() != compile_daemon._daemon_lock_path() + # Keyed by the project: another project's daemon holds another lock. + this_project = compile_daemon._daemon_lock_path() + monkeypatch.setattr( + "reflex.utils.prerequisites.get_backend_dir", lambda: tmp_path / "other" + ) + assert compile_daemon._daemon_lock_path() != this_project + + +def test_running_restarts_the_daemon_on_its_restart_code(tmp_path, monkeypatch): + """A daemon exiting for a global config change is started again. + + The daemon cannot apply such a change to its warm parent; a new process + loads the new world. Re-executing in place would leave ``reflex run`` + with a dead handle on Windows, where that starts a new process. + """ + import subprocess + + popen = subprocess.Popen + spawned: list[subprocess.Popen] = [] + + def spawn(args, **kwargs): + """Stand in for the daemon: ask for a restart once, then wait. + + Args: + args: The daemon command (replaced). + kwargs: The Popen options. + + Returns: + The spawned process. + """ + code = compile_daemon._RESTART_EXIT_CODE if not spawned else 0 + body = f"import sys, time\nsys.exit({code}) if {code} else time.sleep(60)\n" + proc = popen([sys.executable, "-c", body], **kwargs) + spawned.append(proc) + return proc + + environments: list[dict[str, str]] = [] + original_spawn = spawn + + def spawn_recording_env(args, **kwargs): + environments.append(dict(kwargs["env"])) + return original_spawn(args, **kwargs) + + monkeypatch.setattr(compile_daemon.subprocess, "Popen", spawn_recording_env) + with compile_daemon.running(initial_compiled=True): + assert AppHarness._poll_for(lambda: len(spawned) == 2, timeout=10) + assert spawned[0].poll() == compile_daemon._RESTART_EXIT_CODE + assert spawned[1].poll() is None + assert spawned[1].poll() is not None + assert len(spawned) == 2 + # The first daemon found the frontend compiled; the restarted one must + # compile, since the config change made that frontend stale. + assert "REFLEX_COMPILE_DAEMON_PRECOMPILED" in environments[0] + assert "REFLEX_COMPILE_DAEMON_PRECOMPILED" not in environments[1] + + +@pytest.mark.skipif(not hasattr(os, "fork"), reason="requires os.fork (POSIX)") +def test_await_child_kills_the_child_when_the_daemon_exits(monkeypatch): + """A daemon unwinding mid-compile (SIGTERM, Ctrl-C) takes its child with it. + + Otherwise the orphaned child keeps the locks and writes ``.web`` after + ``reflex run`` ended, and a new run finds a daemon that owns the project. + """ + import time + + pid = os.fork() + if pid == 0: # child: a compile that never ends + try: + time.sleep(60) + finally: + os._exit(0) + + def exit_now(_seconds: float) -> None: + raise SystemExit(0) + + monkeypatch.setattr(compile_daemon.time, "sleep", exit_now) + with pytest.raises(SystemExit): + compile_daemon._await_child(pid) + # Reaped: the pid is gone (or at least no longer ours to wait for). + with pytest.raises(ChildProcessError): + os.waitpid(pid, 0) + + +def test_watch_state_skips_build_dirs_below_the_project_only(tmp_path, monkeypatch): + """A project living under a directory named like a build dir is watched.""" + root = tmp_path / "venv" / "project" + app_root = root / "app" + app_root.mkdir(parents=True) + monkeypatch.setattr(disk_cache, "load_manifest", lambda: None) + state = compile_daemon._WatchState.build([app_root.resolve()], root.resolve()) + assert state.accepts(app_root / "page.py") + assert not state.accepts(app_root / "node_modules" / "pkg" / "index.py") + assert not state.accepts(root / ".web" / "x.py") + + +def test_child_compile_uses_a_private_bytecode_cache(tmp_path, monkeypatch): + """Changed modules re-import from source, not from same-second bytecode. + + Timestamp ``.pyc`` files are keyed by whole-second mtime and size, so a + second same-size edit within a second of the compile that cached the + first would import stale bytecode. + """ + import importlib.util + + monkeypatch.setattr("reflex.utils.prerequisites.get_backend_dir", lambda: tmp_path) + compile_daemon._use_private_pycache() + assert sys.pycache_prefix == str(tmp_path.resolve() / compile_daemon._PYCACHE_DIR) + source = tmp_path / "page.py" + source.write_text("x = 1\n") + cached = Path(importlib.util.cache_from_source(str(source))) + assert cached.is_relative_to(tmp_path / compile_daemon._PYCACHE_DIR) + cached.parent.mkdir(parents=True) + cached.write_bytes(b"stale") + compile_daemon._forget_bytecode({source, tmp_path / "data.md"}) + assert not cached.exists() + compile_daemon._forget_bytecode({source}) # already gone: no error + + +def test_compile_child_telemetry_is_sent_by_the_daemon(tmp_path, monkeypatch): + """A compile child spools its events; the daemon sends them afterwards. + + A forked child must not start the telemetry thread, nor touch macOS's + proxy lookup after a fork, and ``os._exit`` drops anything it queued. + """ + from reflex_base.environment import environment + + from reflex.utils import telemetry + + monkeypatch.setattr("reflex.utils.prerequisites.get_backend_dir", lambda: tmp_path) + sent: list[tuple[str, dict]] = [] + monkeypatch.setattr( + telemetry, "send", lambda event, **kwargs: sent.append((event, kwargs)) + ) + monkeypatch.setattr(compile_daemon, "_can_fork", lambda: False) + + def run(args, **kwargs): + """Stand in for the cold compile child: send an event, exit. + + Args: + args: The compile command (replaced). + kwargs: The run options. + + Returns: + A completed process. + """ + spool = Path(kwargs["env"][environment.REFLEX_TELEMETRY_SPOOL.name]) + telemetry._spool_event(spool, "compile", {"pages_count": 1}, {}) + return Mock(returncode=0) + + monkeypatch.setattr(compile_daemon.subprocess, "run", run) + assert ( + compile_daemon._run_compile_child([tmp_path], False, None, predict=False) == 0 + ) + assert sent == [("compile", {"properties": {"pages_count": 1}})] + assert not list(tmp_path.glob(".telemetry.*")) + assert environment.REFLEX_TELEMETRY_SPOOL.get() is None + assert compile_daemon.owns_compilation() is True + + +def test_failed_daemon_spawn_propagates(monkeypatch): + """A failed spawn raises out of the run scope, leaving workers in charge.""" + + def fail(*args, **kwargs): + """Simulate failure to launch the child.""" + message = "spawn failed" + raise OSError(message) + + monkeypatch.setattr(compile_daemon.subprocess, "Popen", fail) + with pytest.raises(OSError, match="spawn failed"), compile_daemon.running(): + pass + + +@pytest.mark.parametrize("exit_with", [None, KeyboardInterrupt]) +def test_daemon_stops_when_the_run_scope_exits(tmp_path, monkeypatch, exit_with): + """Leaving the ``reflex run`` scope stops the daemon, however the run ends. + + A signal to ``reflex run`` alone (an IDE stop, ``kill``) must not leave the + run waiting on a daemon that never got the signal. + """ + import subprocess + + monkeypatch.setattr("reflex.utils.prerequisites.get_backend_dir", lambda: tmp_path) + lock = compile_daemon._daemon_lock_path() + popen = subprocess.Popen + spawned: list[subprocess.Popen] = [] + + def spawn_holder(args, **kwargs): + """Stand in for the daemon: hold its lock and never exit on its own. + + Args: + args: The daemon command (replaced). + kwargs: The Popen options. + + Returns: + The spawned process. + """ + holder = ( + "import time\n" + "from pathlib import Path\n" + "from reflex.utils import path_ops\n" + f"assert path_ops.try_lock(Path({str(lock)!r})) is not None\n" + "print('held', flush=True)\n" + "time.sleep(60)\n" + ) + proc = popen( + [sys.executable, "-c", holder], stdout=subprocess.PIPE, text=True, **kwargs + ) + spawned.append(proc) + return proc + + monkeypatch.setattr(compile_daemon.subprocess, "Popen", spawn_holder) + with ( + contextlib.suppress(KeyboardInterrupt), + compile_daemon.running(), + ): + [proc] = spawned + assert proc.stdout is not None + assert proc.stdout.readline().strip() == "held" + assert compile_daemon.daemon_active() + if exit_with is not None: + raise exit_with + [proc] = spawned + assert proc.poll() is not None + # Windows can release byte-range locks asynchronously after process exit. + assert AppHarness._poll_for(lambda: not compile_daemon.daemon_active(), timeout=5) + + +def test_watch_state_discovers_new_project_root_sources(tmp_path): + """Watching existing children must not hide newly created siblings.""" + existing = tmp_path / "app" + existing.mkdir() + state = compile_daemon._WatchState(roots=[existing], root=tmp_path) + source = tmp_path / "new_helper.py" + assert tmp_path in state.targets() + assert state.accepts(source) + source.write_text("VALUE = 1") + assert source in state.watch_paths() + + +def test_watcher_preserves_snapshot_before_teardown(tmp_path, monkeypatch): + """An edit during watcher teardown must be caught after compilation.""" + import watchfiles + + source = tmp_path / "page.py" + source.write_text("before") + state = compile_daemon._WatchState(roots=[tmp_path], root=tmp_path) + state.checkpoint = compile_daemon._dependency_snapshot(state) + + def watch(*args, **kwargs): + """Emit an edit, then mutate another input as the watcher closes. + + Yields: + A filesystem event batch. + """ + try: + source.write_text("first edit") + yield {(2, str(source))} + finally: + (tmp_path / "late.py").write_text("late edit") + + monkeypatch.setattr(watchfiles, "watch", watch) + assert compile_daemon._next_changes(state, lambda: True) == {source} + assert source not in compile_daemon._missed_changes( + state.checkpoint, compile_daemon._dependency_snapshot(state) + ) + assert tmp_path / "late.py" in compile_daemon._missed_changes( + state.checkpoint, compile_daemon._dependency_snapshot(state) + ) + + +@pytest.mark.parametrize("warm_only", [False, True]) +def test_daemon_initial_work_holds_lock(tmp_path, monkeypatch, warm_only): + """Startup either warms or compiles, always under the backend wait lock.""" + from reflex.utils import path_ops, prerequisites + + lock = tmp_path / "compile.lock" + monkeypatch.chdir(tmp_path) + monkeypatch.setattr(compile_daemon, "_lock_path", lambda: lock) + monkeypatch.setattr(compile_daemon, "_reload_roots", lambda: [tmp_path]) + monkeypatch.setattr(compile_daemon, "_prepare_fork_parent", lambda roots: None) + monkeypatch.setattr(compile_daemon, "_next_changes", lambda *args: None) + monkeypatch.setattr( + compile_daemon._WatchState, + "build", + lambda *args: compile_daemon._WatchState([tmp_path], tmp_path), + ) + calls = [] + + def compile_app(**kwargs): + """Verify initial compilation blocks backend readers.""" + calls.append(("compile", path_ops.is_locked(lock))) + + def import_app(**kwargs): + """Verify warm startup only imports the application.""" + calls.append(("import", path_ops.is_locked(lock))) + + monkeypatch.setattr(prerequisites, "get_compiled_app", compile_app) + monkeypatch.setattr(prerequisites, "get_app", import_app) + if warm_only: + monkeypatch.setenv("REFLEX_COMPILE_DAEMON_PRECOMPILED", "1") + else: + monkeypatch.delenv("REFLEX_COMPILE_DAEMON_PRECOMPILED", raising=False) + compile_daemon._serve() + assert calls == [("import" if warm_only else "compile", True)] + assert not path_ops.is_locked(lock) + assert "REFLEX_COMPILE_DAEMON_PRECOMPILED" not in os.environ + + +def test_sigterm_during_the_initial_compile_stops_the_daemon(tmp_path, monkeypatch): + """SIGTERM is not a broken initial compile: the daemon exits, not watches.""" + from reflex.utils import prerequisites + + monkeypatch.chdir(tmp_path) + monkeypatch.setattr(compile_daemon, "_lock_path", lambda: tmp_path / "c.lock") + monkeypatch.setattr(compile_daemon, "_reload_roots", lambda: [tmp_path]) + monkeypatch.setattr(compile_daemon, "_prepare_fork_parent", lambda roots: None) + monkeypatch.setattr( + compile_daemon._WatchState, + "build", + lambda *args: compile_daemon._WatchState([tmp_path], tmp_path), + ) + watched = [] + monkeypatch.setattr( + compile_daemon, "_next_changes", lambda *args: watched.append(args) + ) + monkeypatch.delenv("REFLEX_COMPILE_DAEMON_PRECOMPILED", raising=False) + + def terminated(**kwargs): + """Receive SIGTERM mid-compile.""" + compile_daemon._exit_on_sigterm(0, None) + + monkeypatch.setattr(prerequisites, "get_compiled_app", terminated) + with pytest.raises(compile_daemon._Terminated): + compile_daemon._serve() + assert watched == [] + + +def test_watcher_reconciles_edits_after_an_idle_batch(tmp_path, monkeypatch): + """A late edit must be included even if its OS event is in a later batch.""" + import watchfiles + + first = tmp_path / "first.py" + second = tmp_path / "second.py" + first.write_text("old") + second.write_text("old") + state = compile_daemon._WatchState([tmp_path], tmp_path) + state.checkpoint = compile_daemon._dependency_snapshot(state) + + def watch(*args, **kwargs): + """Return one event while another changed input awaits its event batch. + + Yields: + An idle batch, followed by the first file's event. + """ + yield set() + first.write_text("new first") + second.write_text("new second") + yield {(2, str(first))} + + monkeypatch.setattr(watchfiles, "watch", watch) + assert compile_daemon._next_changes(state, lambda: True) == {first, second} + + +def test_serve_keeps_failed_changes_pending_until_a_compile_succeeds( + tmp_path, monkeypatch +): + """Paths from a failed compile are re-reported with the next change batch. + + The manifest is only refreshed by a successful compile, so a path that + missed a failed build is still a stale dependency: dropping it from the + changed hint would let the stat-free hit shortcut reuse its page. + """ + from reflex.utils import prerequisites + + monkeypatch.chdir(tmp_path) + monkeypatch.setattr(compile_daemon, "_lock_path", lambda: tmp_path / "lock") + monkeypatch.setattr(compile_daemon, "_reload_roots", lambda: [tmp_path]) + monkeypatch.setattr(compile_daemon, "_prepare_fork_parent", lambda roots: None) + monkeypatch.setattr(prerequisites, "get_app", lambda **kwargs: None) + monkeypatch.setenv("REFLEX_COMPILE_DAEMON_PRECOMPILED", "1") + monkeypatch.setattr( + compile_daemon._WatchState, + "build", + lambda *args: compile_daemon._WatchState([tmp_path], tmp_path), + ) + a, b = tmp_path / "a.py", tmp_path / "b.py" + batches = iter([{a, b}, {b}, {a}, None]) + monkeypatch.setattr(compile_daemon, "_next_changes", lambda *args: next(batches)) + compiles: list[set[Path]] = [] + + def compile_once(roots, prerender_routes, changed=None): + """Fail the first compile, succeed afterwards. + + Args: + roots: The reload roots. + prerender_routes: Whether to prerender routes. + changed: The changed-path hint handed to the compile. + + Returns: + Whether the simulated compile succeeded. + """ + compiles.append(set(changed or ())) + return len(compiles) > 1 + + monkeypatch.setattr(compile_daemon, "_compile_once", compile_once) + compile_daemon._serve() + assert compiles == [{a, b}, {a, b}, {a}] + + +def test_daemon_compiles_an_edit_made_before_its_first_snapshot(tmp_path, monkeypatch): + """An input edited after the last build but before the watch starts compiles. + + The first snapshot is the watch baseline, so such an edit raises no event. + """ + from reflex.compiler import disk_cache, page_cache + from reflex.utils import prerequisites + + page = tmp_path / "page.py" + page.write_text("TITLE = 'v1'\n") + entry = page_cache.file_entry(str(page)) + assert entry is not None + files = {str(page): list(entry)} + page.write_text("TITLE = 'edited before the daemon watched'\n") + monkeypatch.chdir(tmp_path) + monkeypatch.setenv("REFLEX_COMPILE_DAEMON_PRECOMPILED", "1") + monkeypatch.setattr(compile_daemon, "_lock_path", lambda: tmp_path / "lock") + monkeypatch.setattr(compile_daemon, "_reload_roots", lambda: [tmp_path]) + monkeypatch.setattr(compile_daemon, "_prepare_fork_parent", lambda roots: None) + monkeypatch.setattr(compile_daemon, "_next_changes", lambda *args: None) + monkeypatch.setattr( + compile_daemon._WatchState, + "build", + lambda *args: compile_daemon._WatchState([tmp_path], tmp_path, known={page}), + ) + monkeypatch.setattr(prerequisites, "get_app", lambda **kwargs: None) + monkeypatch.setattr( + disk_cache, + "load_manifest", + lambda: {"files": files, "pages": {"index": {"deps": list(files)}}}, + ) + compiled = [] + monkeypatch.setattr( + compile_daemon, + "_compile_once", + lambda roots, prerender, changed: compiled.append(changed) or True, + ) + monkeypatch.setattr("reflex.utils.prerequisites.get_backend_dir", lambda: tmp_path) + compile_daemon._serve() + assert compiled == [{page}] + assert compile_daemon._built_inputs_current() + + +def test_daemon_entry_point_runs_the_canonical_module(tmp_path): + """``python -m`` must run the importable module, not a ``__main__`` copy. + + Records go through the ``reflex`` logger tree only from the real module's + logger, and module state must not be split between two copies. + """ + import subprocess + + probe = ( + "import runpy, sys\n" + "import reflex.utils.compile_daemon as daemon\n" + "def main(argv=None):\n" + " print(daemon.logger.name, sys.modules['reflex.utils.compile_daemon'] is daemon)\n" + " return 0\n" + "daemon.main = main\n" + "runpy.run_module('reflex.utils.compile_daemon', run_name='__main__')\n" + ) + result = subprocess.run( + [sys.executable, "-c", probe], + capture_output=True, + text=True, + check=False, + timeout=15, + ) + assert result.stdout.split() == ["reflex.utils.compile_daemon", "True"], ( + result.stderr + ) + + +def test_compile_once_retries_a_failed_prediction_exactly(monkeypatch): + """A failed state-name prediction compiles again in a fresh child, exactly.""" + attempts = [] + + def run_child(roots, prerender_routes, changed, *, predict): + attempts.append(predict) + return compile_daemon._RETRY_EXIT_CODE if predict else 0 + + monkeypatch.setattr(compile_daemon, "_run_compile_child", run_child) + assert compile_daemon._compile_once([], False, None) is True + assert attempts == [True, False] + + +@pytest.mark.skipif(not hasattr(os, "fork"), reason="requires os.fork (POSIX)") +def test_failed_prediction_exits_the_child_with_the_retry_code(monkeypatch): + """The forked child reports a failed prediction distinctly from a failure.""" + from reflex.compiler import disk_cache + + def fail_prediction(roots, prerender_routes, changed, predict): + """Simulate a page claiming other state names than recorded. + + Raises: + DynamicStatePredictionError: Always. + """ + route = "/two" + raise disk_cache.DynamicStatePredictionError(route) + + monkeypatch.setattr(compile_daemon, "_can_fork", lambda: True) + monkeypatch.setattr(compile_daemon, "_child_compile", fail_prediction) + code = compile_daemon._run_compile_child([], False, None, predict=True) + assert code == compile_daemon._RETRY_EXIT_CODE + + +@pytest.mark.skipif(not hasattr(os, "fork"), reason="requires os.fork (POSIX)") +@pytest.mark.parametrize( + ("outcome", "compiled"), + [ + ("ok", True), + ("fell back", False), + ("failed prediction", False), + ("error", False), + ], +) +def test_cold_compile_predicts_names_in_a_child( + monkeypatch, tmp_path, outcome, compiled +): + """A cold compile predicts dynamic state names in a forked child. + + Only a child that compiled spares the caller its exact in-process compile. + """ + from reflex.compiler import disk_cache + + seen = tmp_path / "predicted" + + def attempt() -> bool: + """Record the prediction mode, then end as ``outcome`` says. + + Returns: + Whether the compile succeeded. + + Raises: + DynamicStatePredictionError: For a failed prediction. + RuntimeError: For an error. + """ + seen.write_text(str(disk_cache._predict_dynamic_names)) + if outcome == "failed prediction": + route = "/two" + raise disk_cache.DynamicStatePredictionError(route) + if outcome == "error": + msg = "boom" + raise RuntimeError(msg) + return outcome == "ok" + + monkeypatch.setattr(compile_daemon, "_single_threaded", lambda: True) + monkeypatch.setattr(disk_cache, "load_manifest", lambda: {"pages": {}}) + assert compile_daemon.compiled_in_predicting_child(attempt) is compiled + assert seen.read_text() == "True" + assert disk_cache._predict_dynamic_names is None + + +def test_cold_compile_without_a_manifest_does_not_fork(monkeypatch): + """With nothing to reuse, the attempt would only fall back: no child.""" + from reflex.compiler import disk_cache + + monkeypatch.setattr(compile_daemon, "_single_threaded", lambda: True) + monkeypatch.setattr(disk_cache, "load_manifest", lambda: None) + monkeypatch.setattr( + compile_daemon.os, "fork", Mock(side_effect=AssertionError), raising=False + ) + assert compile_daemon.compiled_in_predicting_child(lambda: True) is False + + +@pytest.mark.parametrize( + ("single_threaded", "chosen"), [(False, None), (True, False), (True, True)] +) +def test_cold_compile_without_a_safe_fork_compiles_in_process( + monkeypatch, single_threaded, chosen +): + """The caller compiles in-process with other threads alive or a mode chosen.""" + from reflex.compiler import disk_cache + + monkeypatch.setattr(compile_daemon, "_single_threaded", lambda: single_threaded) + monkeypatch.setattr(disk_cache, "_predict_dynamic_names", chosen) + attempts: list[bool] = [] + + def attempt() -> bool: + """Record a compile the caller must not have handed over. + + Returns: + True. + """ + attempts.append(True) + return True + + assert compile_daemon.compiled_in_predicting_child(attempt) is False + assert attempts == [] + + +@pytest.mark.parametrize("lock", ["held", "unsupported"]) +def test_daemon_without_its_lock_does_not_serve(tmp_path, monkeypatch, lock): + """Only the daemon holding the ownership lock compiles the project.""" + from unittest.mock import Mock + + from reflex.utils import path_ops + + lock_path = tmp_path / "daemon.lock" + monkeypatch.setattr(compile_daemon, "_daemon_lock_path", lambda: lock_path) + monkeypatch.setattr(compile_daemon, "_LOCK_ACQUIRE_TIMEOUT", 0.05) + served: list[bool] = [] + monkeypatch.setattr(compile_daemon, "_serve", lambda: served.append(True)) + holder = path_ops.try_lock(lock_path) if lock == "held" else None + if lock == "unsupported": + monkeypatch.setattr( + path_ops, "try_lock", Mock(side_effect=OSError(errno.ENOLCK, "no locks")) + ) + try: + assert compile_daemon.main([]) == 0 + finally: + if holder is not None: + os.close(holder) + assert served == [] + + +@pytest.mark.parametrize("recorded", [True, False]) +def test_built_inputs_include_dependencies_the_compile_added( + tmp_path, monkeypatch, recorded +): + """Workers wait on an edit to a file the last compile started reading. + + The file was not watched before that compile, which is no edit when the + manifest records it as read. A file created after the compile read it as + missing is an edit. + """ + from reflex.compiler import disk_cache, page_cache + from reflex.utils import prerequisites + + page, data = tmp_path / "page.py", tmp_path / "data.json" + page.write_text("TITLE = 'v1'\n") + files: dict[str, Any] = {} + if recorded: + data.write_text("{}") + files[str(data)] = page_cache.file_entry(str(data)) + known = {page} + monkeypatch.chdir(tmp_path) + monkeypatch.setenv("REFLEX_COMPILE_DAEMON_PRECOMPILED", "1") + monkeypatch.setattr(compile_daemon, "_lock_path", lambda: tmp_path / "lock") + monkeypatch.setattr(compile_daemon, "_reload_roots", lambda: [tmp_path]) + monkeypatch.setattr(compile_daemon, "_prepare_fork_parent", lambda roots: None) + monkeypatch.setattr( + disk_cache, + "load_manifest", + lambda: {"files": files, "pages": {"index": {"deps": list(files)}}}, + ) + monkeypatch.setattr(prerequisites, "get_app", lambda **kwargs: None) + monkeypatch.setattr("reflex.utils.prerequisites.get_backend_dir", lambda: tmp_path) + monkeypatch.setattr( + compile_daemon._WatchState, + "build", + lambda *args: compile_daemon._WatchState( + [tmp_path], tmp_path, known=set(known) + ), + ) + batches = iter([{page}, None]) + monkeypatch.setattr(compile_daemon, "_next_changes", lambda *args: next(batches)) + + def compile_once(roots, prerender_routes, changed=None) -> bool: + """Compile a page that now reads ``data``. + + Args: + roots: The reload roots. + prerender_routes: Whether to prerender routes. + changed: The changed-path hint. + + Returns: + True. + """ + compiles.append(changed) + known.add(data) + if not data.exists(): + data.write_text("{}") + return True + + compiles: list[set[Path] | None] = [] + monkeypatch.setattr(compile_daemon, "_compile_once", compile_once) + compile_daemon._serve() + if not recorded: + assert compiles == [{page}, {data}] + return + assert compiles == [{page}] + assert compile_daemon._built_inputs_current() + data.write_text('{"edited": true}') + os.utime(data, ns=(1, 1)) + assert not compile_daemon._built_inputs_current() + + +def test_unbuilt_changes_compare_function_slices(tmp_path, monkeypatch): + """A module recorded only as function slices still reports its edits.""" + from reflex.compiler import disk_cache, page_cache + + page = tmp_path / "page.py" + page.write_text("def index():\n return 'v1'\n") + dep = f"{page}{page_cache.SLICE_SEPARATOR}index" + manifest = { + "files": {dep: page_cache.file_entry(dep)}, + "pages": {"index": {"deps": [dep]}}, + } + monkeypatch.setattr(disk_cache, "load_manifest", lambda: manifest) + state = compile_daemon._WatchState([tmp_path], tmp_path, known={page}) + assert compile_daemon._unbuilt_changes(state) == set() + page.write_text("def index():\n return 'edited'\n") + os.utime(page, ns=(1, 1)) + assert compile_daemon._unbuilt_changes(state) == {page} + + +def test_unbuilt_changes_ignore_unreferenced_file_table_entries(tmp_path, monkeypatch): + """A slice no page depends on any more does not report its module as edited. + + An incremental rebuild extends the file table without pruning it, so a + renamed page function leaves its old slice behind. + """ + from reflex.compiler import disk_cache, page_cache + + page = tmp_path / "page.py" + page.write_text("def home():\n return 'v1'\n") + stale = f"{page}{page_cache.SLICE_SEPARATOR}index" + current = f"{page}{page_cache.SLICE_SEPARATOR}home" + files = {stale: ["0" * 64, 1, 1], current: page_cache.file_entry(current)} + manifest = {"files": files, "pages": {"home": {"deps": [current]}}} + monkeypatch.setattr(disk_cache, "load_manifest", lambda: manifest) + state = compile_daemon._WatchState([tmp_path], tmp_path, known={page}) + assert compile_daemon._unbuilt_changes(state) == set() + assert compile_daemon._built_inputs({page}) == {page} diff --git a/tests/units/utils/test_exec.py b/tests/units/utils/test_exec.py index 23ab8c31f42..00a8ac6551c 100644 --- a/tests/units/utils/test_exec.py +++ b/tests/units/utils/test_exec.py @@ -538,6 +538,25 @@ def test_arbitrate_ssr_env_var_wins(monkeypatch: pytest.MonkeyPatch): assert exec_utils.arbitrate_ssr(True) is False +@pytest.mark.parametrize("cache_enabled", [False, True]) +def test_uvicorn_markdown_reload_requires_compile_cache( + cache_enabled, tmp_path, monkeypatch, mocker +): + """The default backend reloader keeps its Python-only behavior.""" + uvicorn = pytest.importorskip("uvicorn") + monkeypatch.setenv("REFLEX_COMPILE_CACHE", str(cache_enabled)) + mocker.patch.object( + exec_utils, "get_dev_backend_reload_marker", return_value=tmp_path / "marker" + ) + mocker.patch.object(exec_utils, "get_app_instance", return_value="app:app") + mocker.patch.object(exec_utils, "get_reload_paths", return_value=[]) + run = mocker.patch.object(uvicorn, "run") + exec_utils.run_uvicorn_backend("localhost", 8000, exec_utils.LogLevel.INFO) + assert run.call_args.kwargs["reload_includes"] == ( + ["*.py", "*.md", "*.mdx"] if cache_enabled else ["*.py"] + ) + + def _fake_granian_prod( mocker: MockerFixture, calls: list[str], telemetry_stopped: bool = True ): diff --git a/tests/units/utils/test_imports.py b/tests/units/utils/test_imports.py index c31881b15d6..358ebde574a 100644 --- a/tests/units/utils/test_imports.py +++ b/tests/units/utils/test_imports.py @@ -126,6 +126,27 @@ def test_parse_imports(input: ImportDict, output: ParsedImportDict): assert parse_imports(input) == output +def test_collapse_imports_preserves_order(): + """Deduplication must preserve first-occurrence order. + + Compiled JSX import order follows this ordering; a hash-seed-dependent + order rewrites every page/memo file on each dev reload and breaks + granular HMR. + """ + import_vars = [ + ImportVar(tag=f"Icon{i}", is_default=True, package_path=f"/Icon{i}") + for i in range(32) + ] + duplicated = [*import_vars, *import_vars[:5], import_vars[0]] + collapsed = collapse_imports({"@hugeicons/core-free-icons": duplicated}) + assert collapsed == {"@hugeicons/core-free-icons": import_vars} + # Tuple-valued entries (already-immutable parsed imports) keep order too. + collapsed_tuple = collapse_imports(( + ("@hugeicons/core-free-icons", tuple(duplicated)), + )) + assert collapsed_tuple == {"@hugeicons/core-free-icons": import_vars} + + def test_merge_imports_deduplicates(): """A tag merged from many subtrees is carried once, not once per subtree. diff --git a/tests/units/utils/test_path_ops.py b/tests/units/utils/test_path_ops.py index a9b31d2d84f..ca1e9dc5ecf 100644 --- a/tests/units/utils/test_path_ops.py +++ b/tests/units/utils/test_path_ops.py @@ -18,6 +18,7 @@ import pytest from reflex import constants +from reflex.testing import AppHarness from reflex.utils import frontend_skeleton, path_ops from reflex.utils.path_ops import write_file @@ -934,3 +935,60 @@ def test_write_file_updates_changed_file(tmp_path: Path) -> None: write_file(path, "before") write_file(path, "after") assert path.read_text(encoding="utf-8") == "after" + + +def test_try_lock_marks_a_holder_until_it_closes(tmp_path: Path): + """A lock taken without waiting is visible to others until released.""" + lock = tmp_path / "owner.lock" + assert path_ops.is_locked(lock) is False + descriptor = path_ops.try_lock(lock) + assert descriptor is not None + try: + assert path_ops.is_locked(lock) is True + assert path_ops.try_lock(lock) is None + finally: + os.close(descriptor) + assert path_ops.is_locked(lock) is False + + +def test_lock_is_released_when_its_holder_is_killed(tmp_path: Path): + """A killed holder cannot leave a lock behind, unlike a pid file.""" + lock = tmp_path / "owner.lock" + holder = subprocess.Popen( + [ + sys.executable, + "-c", + ( + "import time\n" + "from pathlib import Path\n" + "from reflex.utils import path_ops\n" + f"assert path_ops.try_lock(Path({str(lock)!r})) is not None\n" + "print('held', flush=True)\n" + "time.sleep(60)\n" + ), + ], + stdout=subprocess.PIPE, + text=True, + ) + try: + assert holder.stdout is not None + assert holder.stdout.readline().strip() == "held" + assert path_ops.is_locked(lock) is True + finally: + holder.kill() + holder.wait() + # Windows can release byte-range locks asynchronously after process exit. + assert AppHarness._poll_for(lambda: not path_ops.is_locked(lock), timeout=5) + + +@pytest.mark.skipif(sys.platform == "win32", reason="POSIX flock") +def test_unsupported_file_locks_have_no_holder(tmp_path: Path, monkeypatch): + """A file system that cannot lock files reports no holder, not a held lock.""" + path = tmp_path / "owner.lock" + path.touch() + monkeypatch.setattr( + fcntl, "flock", Mock(side_effect=OSError(errno.ENOLCK, "No locks available")) + ) + assert path_ops.is_locked(path) is False + with pytest.raises(OSError, match="No locks available"): + path_ops.try_lock(path) diff --git a/tests/units/utils/test_serializers.py b/tests/units/utils/test_serializers.py index 289b73b5525..51279b8c396 100644 --- a/tests/units/utils/test_serializers.py +++ b/tests/units/utils/test_serializers.py @@ -538,6 +538,38 @@ def test_serialize(value: Any, expected: str): assert json.loads(json_dumps(value)) == json.loads(json_dumps(expected)) +class _ReversedSet(set): + """A set that iterates in descending order, unlike small-int sets.""" + + def __iter__(self): + """Iterate in descending order. + + Returns: + The descending iterator. + """ + return iter(sorted(set.__iter__(self), reverse=True)) + + +def test_sorted_set_is_deterministic_and_serialize_set_is_not_sorted(): + """Compiled output sorts sets; a state delta keeps the cheap list. + + Compiled output and cache fingerprints embed serialized sets; a + hash-seed-dependent order churns files on every process restart. A delta + is serialized on every update and its order carries no meaning. + """ + from reflex_base.vars import LiteralVar + + assert serializers.sorted_set({"b", "c", "a"}) == ["a", "b", "c"] + assert serializers.sorted_set({3, 1, 2}) == [1, 2, 3] + # Unsortable element mixes sort by their serialized form. + assert serializers.sorted_set({1, "a"}) == serializers.sorted_set({"a", 1}) + assert set(serializers.serialize_set({1, "a"})) == {1, "a"} + # The state-delta path keeps the set's own order: no sort per delta. + assert serializers.serialize_set(_ReversedSet({1, 2, 3})) == [3, 2, 1] + assert str(LiteralVar.create({"b", "c", "a"})) == '["a", "b", "c"]' + assert str(LiteralVar.create({3, 1, 2})) == "[1, 2, 3]" + + @pytest.mark.parametrize( ("value", "expected", "exp_var_is_string"), [ @@ -578,3 +610,26 @@ def test_serialize_var_to_str(value: Any, expected: str, exp_var_is_string: bool """ v = LiteralVar.create(value) assert str(v) == expected + + +def test_serialize_unordered_enum_set_is_deterministic(): + """Non-orderable enum values serialize independently of set iteration.""" + from enum import Enum + + class Choice(Enum): + A = "a" + B = "b" + + class ReversedSet(set): + """Model a different process's iteration order.""" + + def __iter__(self): + """Iterate in the opposite order. + + Returns: + The reversed iterator. + """ + return reversed(list(super().__iter__())) + + values = {Choice.A, Choice.B} + assert serializers.sorted_set(values) == serializers.sorted_set(ReversedSet(values)) diff --git a/tests/units/utils/test_telemetry_accounting.py b/tests/units/utils/test_telemetry_accounting.py index 80ab4ba359e..f35059b3bcb 100644 --- a/tests/units/utils/test_telemetry_accounting.py +++ b/tests/units/utils/test_telemetry_accounting.py @@ -147,6 +147,7 @@ def _fake_app(**overrides): defaults = { "_state": None, "_pages": {}, + "_cached_component_counts": None, "_unevaluated_pages": {}, } lifespan_tasks = overrides.pop("_lifespan_tasks", {}) diff --git a/tests/units/vars/test_base.py b/tests/units/vars/test_base.py index e4d7e363e3c..c203ab96eae 100644 --- a/tests/units/vars/test_base.py +++ b/tests/units/vars/test_base.py @@ -77,3 +77,91 @@ def cv(self) -> int: replaced = cv._replace(_var_type=float) assert replaced._var_type is float + + +def test_reset_unique_variable_names_is_deterministic(): + """Resetting the unique-name generator reproduces the same name sequence. + + Auto-memo content hashes embed these names, so a second in-process compile + must regenerate identical ones (the RNG state and used-name set otherwise + persist process-wide and drift). + """ + from reflex_base.vars.base import ( + get_unique_variable_name, + reset_unique_variable_names, + ) + + reset_unique_variable_names() + first = [get_unique_variable_name() for _ in range(8)] + # Without a reset, the sequence keeps advancing (and dedups against the + # already-used set), so it differs. + assert [get_unique_variable_name() for _ in range(8)] != first + # After a reset it reproduces the original sequence exactly. + reset_unique_variable_names() + assert [get_unique_variable_name() for _ in range(8)] == first + + +def test_seeded_unique_variable_names_block_resumes_the_outer_sequence(): + """A seeded block draws by its seed alone and keeps its names reserved.""" + from reflex_base.vars.base import ( + USED_VARIABLES, + get_unique_variable_name, + reset_unique_variable_names, + seed_unique_variable_names, + seeded_unique_variable_names, + ) + + reset_unique_variable_names() + outer_first = get_unique_variable_name() + with seeded_unique_variable_names("state.one"): + block_names = [get_unique_variable_name() for _ in range(3)] + outer_next = get_unique_variable_name() + # The block's names are a function of its seed, whatever ran before. + seed_unique_variable_names("unrelated") + get_unique_variable_name() + with seeded_unique_variable_names("state.one"): + assert [get_unique_variable_name() for _ in range(3)] == block_names + # The outer sequence resumed where it stopped, with the block's names taken. + reset_unique_variable_names() + assert get_unique_variable_name() == outer_first + with seeded_unique_variable_names("state.one"): + assert set(block_names) & USED_VARIABLES == set() + assert [get_unique_variable_name() for _ in range(3)] == block_names + assert set(block_names) <= USED_VARIABLES + assert get_unique_variable_name() == outer_next + # Reserved names are never handed out again by a seeded sequence. + seed_unique_variable_names("app", reserved=block_names) + assert set(block_names) <= USED_VARIABLES + assert get_unique_variable_name() not in block_names + + +def test_seeded_unique_variable_names_are_per_page(): + """Each page seed yields its own reproducible sequence, resumable later. + + Incremental compiles regenerate a subset of pages; a page's names must not + depend on which other pages compiled before it (or the reused pages would + share "unique" names with the recompiled one). + """ + from reflex_base.vars.base import ( + get_unique_variable_name, + restore_unique_variable_name_state, + seed_unique_variable_names, + unique_variable_name_state, + ) + + seed_unique_variable_names("/a") + a_names = [get_unique_variable_name() for _ in range(4)] + a_state = unique_variable_name_state() + seed_unique_variable_names("/b") + b_names = [get_unique_variable_name() for _ in range(4)] + assert a_names != b_names + # Re-seeding a page reproduces its sequence regardless of what ran between. + seed_unique_variable_names("/a") + assert [get_unique_variable_name() for _ in range(4)] == a_names + a_continued = [get_unique_variable_name() for _ in range(2)] + # Restoring a captured state resumes the sequence from that point without + # repeating names already handed out. + restore_unique_variable_name_state(a_state) + resumed = [get_unique_variable_name() for _ in range(2)] + assert resumed == a_continued + assert not set(resumed) & set(a_names)