Release/ux test 2025 09 12

(pwd)/dev/test-runs/pw-browsers/.links/3810710a7adc295959bdc2790dead5e21821c0a7

@@ -0,0 +1 @@
+/home/patch/PycharmProjects/scidk/.venv/lib/python3.12/site-packages/playwright/driver/package

.commitmsg

@@ -0,0 +1,9 @@
+UX: Files and Snapshot parity; move scan controls; rescan visibility and progress
+
+- Files page: moved scan controls (Scan / Scan with selection / override toggle) into a toolbar above the file browser so they are never covered by the details panel; right panel now shows only item details.
+- Files page: kept select-all header checkbox for Gmail-like selection.
+- Snapshot browser: made table header include a select-all checkbox and row checkboxes for visual parity (informational for now), keeping the same crumb/detail panel behavior.
+- Rescans visibility: API /api/scans now includes rescan_of in summaries; detail includes rescan_of as well. UI Scans Summary and Scans panel display a "rescan" badge and the original scan id.
+- Rescan progress: Snapshot "Rescan" button now starts a background scan task using the original scan parameters and stored selection, so progress shows in the Tasks panel and persists across page switches.
+
+No backend schema change; minimal additions in /api/scans payload and frontend templates.

.github/workflows/tests.yml

@@ -0,0 +1,58 @@
+name: Tests
+
+on:
+  push:
+    branches: [ main, master, develop, release/** ]
+  pull_request:
+
+jobs:
+  tests:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: ["3.12"]
+        tier: [ unit, integration, e2e ]
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Install Python deps
+        run: |
+          python -m pip install --upgrade pip
+          pip install -e .[dev]
+      - name: Install Playwright browsers
+        if: matrix.tier == 'e2e'
+        uses: microsoft/playwright-github-action@v1
+      - name: Prepare repo-local temp directories
+        run: |
+          mkdir -p dev/test-runs/{tmp,pytest-tmp,artifacts,downloads,pw-browsers}
+      - name: Run tests by tier
+        env:
+          SCIDK_E2E: ${{ matrix.tier == 'e2e' && '1' || '0' }}
+          TMPDIR: ${{ github.workspace }}/dev/test-runs/tmp
+          PYTEST_ADDOPTS: --basetemp=${{ github.workspace }}/dev/test-runs/pytest-tmp
+          PLAYWRIGHT_BROWSERS_PATH: ${{ github.workspace }}/dev/test-runs/pw-browsers
+        run: |
+          case "${{ matrix.tier }}" in
+            unit)
+              pytest -m "not integration and not e2e" -q
+              ;;
+            integration)
+              pytest -m integration -q
+              ;;
+            e2e)
+              pytest -m e2e tests/e2e -v --maxfail=1 --suppress-no-test-exit-code
+              ;;
+          esac
+      - name: Upload Playwright report (on failure)
+        if: failure() && matrix.tier == 'e2e'
+        uses: actions/upload-artifact@v4
+        with:
+          name: playwright-report
+          path: |
+            playwright-report/
+            test-results/
+            dev/test-runs/artifacts/
+            dev/test-runs/pytest-tmp/
+          if-no-files-found: ignore

.gitignore

@@ -35,3 +35,14 @@ data/*
 *.db
 *.db-shm
 *.db-wal
+# Local test artifacts
+dev/test-runs/
+playwright-report/
+test-results/
+pytest-of-patch/
+.output.txt
+# Accidental directories from env expansion
+(pwd)/
+sqlite:/
+sqlite:/home
+sqlite:/tmp

Makefile

@@ -1,12 +1,70 @@
 # Convenience Makefile for docs/tools
 
-.PHONY: flags-index docs-check
+.PHONY: flags-index docs-check unit integration check e2e-install-browsers e2e e2e-headed e2e-parallel e2e-debug
 
 flags-index:
 	python -m dev.tools.feature_flags_index --write
 
 # docs-check: run generator and diff; non-zero exit if mismatched
 # Note: this target assumes Unix tools (diff)
 docs-check:
-	python -m dev.tools.feature_flags_index > /tmp/feature-flags.md
-	diff -q /tmp/feature-flags.md dev/features/feature-flags.md
+	@mkdir -p dev/test-runs/tmp
+	python -m dev.tools.feature_flags_index > dev/test-runs/tmp/feature-flags.md
+	diff -q dev/test-runs/tmp/feature-flags.md dev/features/feature-flags.md
+
+# Test tiers
+unit:
+	@mkdir -p dev/test-runs/{tmp,pytest-tmp}
+	TMPDIR=$$(pwd)/dev/test-runs/tmp \
+	PYTEST_ADDOPTS="--basetemp=$$(pwd)/dev/test-runs/pytest-tmp" \
+	pytest -m "not integration and not e2e" -q
+
+integration:
+	@mkdir -p dev/test-runs/{tmp,pytest-tmp}
+	TMPDIR=$$(pwd)/dev/test-runs/tmp \
+	PYTEST_ADDOPTS="--basetemp=$$(pwd)/dev/test-runs/pytest-tmp" \
+	pytest -m integration -q
+
+check:
+	$(MAKE) unit && $(MAKE) integration && $(MAKE) e2e
+
+# Install Playwright browsers locally (no root/apt deps); install into repo cache
+e2e-install-browsers:
+	@mkdir -p dev/test-runs/{tmp,pw-browsers}
+	PLAYWRIGHT_BROWSERS_PATH=$$(pwd)/dev/test-runs/pw-browsers \
+	TMPDIR=$$(pwd)/dev/test-runs/tmp \
+	.venv/bin/python -m playwright install chromium
+
+# Run headless E2E tests
+# Ensures port 5001 is used by tests; app is auto-started by tests/e2e/conftest.py
+e2e:
+	@mkdir -p dev/test-runs/{tmp,pytest-tmp,artifacts,downloads,pw-browsers}
+	 SCIDK_E2E=1 TMPDIR=$$(pwd)/dev/test-runs/tmp TMP=$$(pwd)/dev/test-runs/tmp TEMP=$$(pwd)/dev/test-runs/tmp PYTEST_ADDOPTS="--basetemp=$$(pwd)/dev/test-runs/pytest-tmp" PLAYWRIGHT_BROWSERS_PATH=$$(pwd)/dev/test-runs/pw-browsers pytest -m e2e tests/e2e -v --maxfail=1
+
+# Run E2E tests in headed mode with Playwright inspector
+e2e-headed:
+	@mkdir -p dev/test-runs/{tmp,pytest-tmp,artifacts,downloads,pw-browsers}
+	SCIDK_E2E=1 PLAYWRIGHT_HEADLESS=0 PWDEBUG=1 TMPDIR=$$(pwd)/dev/test-runs/tmp TMP=$$(pwd)/dev/test-runs/tmp TEMP=$$(pwd)/dev/test-runs/tmp PYTEST_ADDOPTS="--basetemp=$$(pwd)/dev/test-runs/pytest-tmp" PLAYWRIGHT_BROWSERS_PATH=$$(pwd)/dev/test-runs/pw-browsers pytest -m e2e tests/e2e -q
+
+# Run E2E in parallel (requires pytest-xdist if desired; Playwright supports built-in workers)
+e2e-parallel:
+	@mkdir -p dev/test-runs/{tmp,pytest-tmp,artifacts,downloads,pw-browsers}
+	SCIDK_E2E=1 TMPDIR=$$(pwd)/dev/test-runs/tmp PYTEST_ADDOPTS="--basetemp=$$(pwd)/dev/test-runs/pytest-tmp" PLAYWRIGHT_BROWSERS_PATH=$$(pwd)/dev/test-runs/pw-browsers pytest -m e2e tests/e2e -q -n auto || SCIDK_E2E=1 TMPDIR=$$(pwd)/dev/test-runs/tmp PYTEST_ADDOPTS="--basetemp=$$(pwd)/dev/test-runs/pytest-tmp" PLAYWRIGHT_BROWSERS_PATH=$$(pwd)/dev/test-runs/pw-browsers pytest -m e2e tests/e2e -q
+
+# Verbose debugging output for E2E runs
+e2e-debug:
+	@mkdir -p dev/test-runs/{tmp,pytest-tmp,artifacts,downloads,pw-browsers}
+	SCIDK_E2E=1 TMPDIR=$$(pwd)/dev/test-runs/tmp PYTEST_ADDOPTS="--basetemp=$$(pwd)/dev/test-runs/pytest-tmp -vv -s" PLAYWRIGHT_BROWSERS_PATH=$$(pwd)/dev/test-runs/pw-browsers pytest -m e2e tests/e2e
+
+# Demo recording: runs a single E2E that captures screenshots and API JSON
+# Artifacts go to dev/test-runs/last-demo by default (override with DEMO_ARTIFACTS_DIR)
+demo-record:
+	@mkdir -p dev/test-runs/{tmp,pytest-tmp,artifacts,downloads,pw-browsers}
+	SCIDK_E2E=1 DEMO_ARTIFACTS_DIR=$${DEMO_ARTIFACTS_DIR:-dev/test-runs/last-demo} TMPDIR=$$(pwd)/dev/test-runs/tmp PYTEST_ADDOPTS="--basetemp=$$(pwd)/dev/test-runs/pytest-tmp" PLAYWRIGHT_BROWSERS_PATH=$$(pwd)/dev/test-runs/pw-browsers pytest -m e2e tests/e2e/test_demo_recording.py -q ; \
+	echo "Artifacts saved under: $${DEMO_ARTIFACTS_DIR:-dev/test-runs/last-demo}"
+
+# Demo recording in headed mode with inspector
+demo-record-headed:
+	@mkdir -p dev/test-runs/{tmp,pytest-tmp,artifacts,downloads,pw-browsers}
+	SCIDK_E2E=1 PLAYWRIGHT_HEADLESS=0 PWDEBUG=1 DEMO_ARTIFACTS_DIR=$${DEMO_ARTIFACTS_DIR:-dev/test-runs/last-demo} TMPDIR=$$(pwd)/dev/test-runs/tmp PYTEST_ADDOPTS="--basetemp=$$(pwd)/dev/test-runs/pytest-tmp" PLAYWRIGHT_BROWSERS_PATH=$$(pwd)/dev/test-runs/pw-browsers pytest -m e2e tests/e2e/test_demo_recording.py -q ; \
+	echo "Artifacts saved under: $${DEMO_ARTIFACTS_DIR:-dev/test-runs/last-demo}"

README.md

@@ -48,6 +48,41 @@ Note: The scanner prefers NCDU for fast filesystem enumeration when available. I
 - Editable install error (Multiple top-level packages discovered): We ship setuptools config to include only the scidk package. If you previously had this error, pull latest and try again: `pip install -e .`.
 - Shell errors when initializing env: Use the script matching your shell (`init_env.sh` for bash/zsh, `init_env.fish` for fish). Avoid running `sh scripts/init_env.sh`; instead, source it.
 
+## End-to-End (E2E) tests
+
+These tests run in a real browser using Playwright and pytest. The test suite automatically starts the Flask app on port 5001 with safe defaults and no external Neo4j connection.
+
+Prereqs (once per machine):
+- Python virtual environment activated.
+- Install dev dependencies and Playwright browsers.
+
+Commands:
+```
+# Install dev deps (if not yet installed)
+pip install -e .[dev]
+
+# Install Playwright browsers (Chromium, Firefox, WebKit)
+make e2e-install-browsers  # or: python -m playwright install --with-deps
+
+# Run headless E2E tests
+make e2e                   # or: pytest -m e2e tests/e2e -q
+
+# Run headed with inspector (debug mode)
+make e2e-headed            # or: PLAYWRIGHT_HEADLESS=0 PWDEBUG=1 pytest -m e2e tests/e2e -q
+
+# Parallel execution (if pytest-xdist is installed; falls back to serial)
+make e2e-parallel
+```
+
+Notes:
+- The E2E test fixture sets:
+  - SCIDK_PORT=5001
+  - NEO4J_AUTH=none
+  - SCIDK_PROVIDERS=local_fs
+  - SCIDK_DB_PATH=sqlite:///:memory:
+- Ensure port 5001 is free before running, or adjust the fixture if needed.
+- For verbose logs during a failing test, use: `make e2e-debug` or `pytest -m e2e -vv -s`.
+
 ## Neo4j Password: How to Set/Change
 - Testing default: The testing Neo4j database uses password `neo4jiscool`. Set this in the app Settings or via environment.
 - Choose your password before first start by setting NEO4J_AUTH in .env or your shell (example uses testing default):
@@ -304,3 +339,145 @@ Notes for agents:
 - Prefer `menu --json` for a quick navigable overview of commands.
 - Use `introspect` to obtain full metadata about args/options, side-effects, and conventions.
 - Place `--json` before the subcommand to ensure the envelope applies to the whole invocation, e.g., `python -m dev.cli --json ready-queue`.
+
+
+## Neo4j in Docker with UI on port 7474
+
+We ship a docker-compose file that runs Neo4j 5 and exposes the built-in HTTP service on the classic port 7474, while Bolt remains on 7687. Follow Neo4j’s Docker volume guidelines for persistence (/data, /logs, /plugins, /import).
+
+Quick start:
+
+```
+# Optional: set password (default is neo4j/neo4jiscool)
+export NEO4J_AUTH=neo4j/neo4jiscool
+
+# Optional: override host directories (defaults are under ./data/neo4j)
+export NEO4J_HOST_DATA_DIR=${NEO4J_HOST_DATA_DIR:-./data/neo4j/data}
+export NEO4J_HOST_LOGS_DIR=${NEO4J_HOST_LOGS_DIR:-./data/neo4j/logs}
+export NEO4J_HOST_PLUGINS_DIR=${NEO4J_HOST_PLUGINS_DIR:-./data/neo4j/plugins}
+export NEO4J_HOST_IMPORT_DIR=${NEO4J_HOST_IMPORT_DIR:-./data/neo4j/import}
+
+# Start Neo4j in the background
+docker compose -f docker-compose.neo4j.yml up -d
+
+# Open the UI (Browser/Workspace availability depends on the server image/version)
+http://localhost:7474/
+```
+
+Notes:
+- We do NOT mount or write to system paths like /var/lib/neo4j on the host. By default we persist under the repository at ./data/neo4j, which works without root.
+- You can override the host directories per environment using NEO4J_HOST_* variables shown above (use absolute or relative paths you own).
+- Ports: 7474 (HTTP), 7687 (Bolt). Adjust in docker-compose.neo4j.yml if occupied.
+- Volumes per Neo4j Docker docs: bind host dirs to /data, /logs, /plugins, and /import inside the container.
+- Avoid mounting anything under /var/lib/neo4j inside the container. The entrypoint changes ownership in that path and can cause permission issues. Stick to /data, /logs, /plugins, and /import.
+
+Manage lifecycle:
+```
+# Stop containers
+docker compose -f docker-compose.neo4j.yml down
+
+# Stop and remove all data (DANGER: wipes the graph)
+docker compose -f docker-compose.neo4j.yml down -v
+```
+
+Connect SciDK to this Neo4j:
+```
+export NEO4J_URI=bolt://localhost:7687
+export NEO4J_AUTH=${NEO4J_AUTH:-neo4j/neo4jiscool}
+# Optional named database
+echo "SCIDK_NEO4J_DATABASE=neo4j" >> .env
+
+# Start SciDK
+scidk-serve
+# or
+python -m scidk.app
+```
+
+
+
+
+
+---
+
+## Test tiers (Python 3.12)
+
+Our CI runs all tests under Python 3.12 in three tiers using pytest markers:
+- unit: fast, pure unit tests that do not touch network/DB/browser
+- integration: tests that touch DB/files/HTTP without a browser
+- e2e: full-browser Playwright tests
+
+Local commands:
+- make unit → pytest -m "not integration and not e2e"
+- make integration → pytest -m integration
+- make e2e → pytest -m e2e tests/e2e -q
+- make check → runs unit, integration, and e2e sequentially
+
+See .github/workflows/tests.yml for the CI matrix that runs each tier.
+
+## Verify CI and Record Demo Artifacts
+
+Follow these steps to verify the full test suite and automatically capture screenshots/JSON for the demo.
+
+1) Verify CI on GitHub
+- Navigate to GitHub → Actions → "Tests" workflow (defined in `.github/workflows/tests.yml`).
+- Confirm that all three matrix jobs are green:
+  - tier=unit
+  - tier=integration
+  - tier=e2e (installs Playwright browsers automatically)
+- Click into the latest run to see logs if any job is red.
+
+2) Run all tests locally (mirrors CI)
+```
+make check
+```
+This runs unit → integration → e2e sequentially under Python 3.12.
+
+3) Capture demo screenshots and API snapshots (automated)
+- Headless (recommended for CI or quick local runs):
+```
+make demo-record
+```
+- Headed with Playwright inspector (debugging):
+```
+make demo-record-headed
+```
+Artifacts are saved under `dev/test-runs/last-demo` by default. Override the output directory with:
+```
+DEMO_ARTIFACTS_DIR=dev/test-runs/my-demo make demo-record
+```
+Generated artifacts include:
+- `01-home.png`, `02-datasets-before.png`, `03-datasets-after.png`, `04-map.png`
+- `api-api-health.json`, `api-api-scans.json`, `api-api-directories.json`, `api-api-tasks.json`
+- `SUMMARY.json` with the artifact path and timestamp
+
+4) Tag and record (optional)
+You can create a tag and attach the artifact folder to a GitHub Release:
+```
+git tag -a vX.Y.Z -m "Cycle demo: SQLite persistence + selective scan cache"
+git push origin vX.Y.Z
+# Then, on GitHub → Releases → Draft a new release → Attach the files from dev/test-runs/...
+```
+
+Troubleshooting:
+- First-time Playwright run locally: install browsers with `make e2e-install-browsers`.
+- Port conflicts: ensure 127.0.0.1:5001 is free; the E2E harness auto-starts the app on that port.
+- Backend toggle: default is SQLite. Override with `export SCIDK_STATE_BACKEND=memory` before `make e2e` if you need the legacy path.
+
+## State backend toggle and Health endpoint
+
+The app can read registry state (scans, directories, tasks, telemetry) through SQLite or in-memory structures.
+- Default: SCIDK_STATE_BACKEND=sqlite
+- Fallback: SCIDK_STATE_BACKEND=memory (restores legacy in-memory reads)
+
+Set the backend via environment before starting the app:
+```
+export SCIDK_STATE_BACKEND=sqlite   # or: memory
+scidk-serve
+```
+
+Health endpoint includes SQLite details useful during migrations and troubleshooting:
+- GET /api/health → { sqlite: { path, exists, journal_mode, wal_mode, schema_version, select1, error? } }
+
+Notes:
+- Auto-migrations run on boot and /api/health reports the final schema_version.
+- WAL mode is enabled by default; journal_mode and wal_mode are both reported for clarity.