From 5db5ce6e355863ae56b3c80b411ce62ec74d32aa Mon Sep 17 00:00:00 2001 From: Jonathan Flood Date: Wed, 10 Jun 2026 09:13:06 +0100 Subject: [PATCH 1/2] feat(collector): add external data collection framework Embed a plugin-style collector subsystem in the FastAPI API with Box, Google Drive, and Dropbox connectors, browse/select and per-project OAuth, ingestion pipeline, CI templates, and M9 UI scaffold. Co-authored-by: Cursor --- .github/CODEOWNERS | 11 + .github/ISSUE_TEMPLATE/bug.yml | 76 ++++ .github/ISSUE_TEMPLATE/feature.yml | 93 +++++ .github/ISSUE_TEMPLATE/task.yml | 66 ++++ .github/pull_request_template.md | 53 +++ .github/workflows/collectors-ci.yml | 109 ++++++ .gitignore | 5 + api/Dockerfile | 8 +- api/collectors/__init__.py | 3 + api/collectors/base/__init__.py | 21 ++ api/collectors/base/auth_context.py | 30 ++ api/collectors/base/base_collector.py | 116 ++++++ api/collectors/base/types.py | 101 +++++ api/collectors/box/__init__.py | 5 + api/collectors/box/box_auth.py | 67 ++++ api/collectors/box/box_collector.py | 155 ++++++++ api/collectors/core/__init__.py | 27 ++ api/collectors/core/browse_service.py | 152 ++++++++ api/collectors/core/logging_manager.py | 62 +++ api/collectors/core/oauth_manager.py | 191 ++++++++++ api/collectors/core/oauth_routes.py | 119 ++++++ api/collectors/core/orchestrator.py | 163 ++++++++ api/collectors/core/registry.py | 37 ++ api/collectors/core/security_manager.py | 163 ++++++++ api/collectors/core/selection_store.py | 133 +++++++ api/collectors/core/state_manager.py | 275 ++++++++++++++ api/collectors/core/token_store.py | 212 +++++++++++ api/collectors/core/trigger_manager.py | 119 ++++++ api/collectors/docs/architecture.md | 40 ++ api/collectors/docs/browse-and-select.md | 249 ++++++++++++ api/collectors/docs/connectors.md | 44 +++ api/collectors/docs/ingestion_flow.md | 58 +++ api/collectors/docs/m9-collection-ui.md | 124 ++++++ api/collectors/dropbox/__init__.py | 5 + api/collectors/dropbox/dropbox_auth.py | 61 +++ api/collectors/dropbox/dropbox_collector.py | 171 +++++++++ api/collectors/google_drive/__init__.py | 5 + api/collectors/google_drive/gdrive_auth.py | 68 ++++ .../google_drive/gdrive_collector.py | 179 +++++++++ api/collectors/ingestion/__init__.py | 16 + .../ingestion/ingestion_pipeline.py | 108 ++++++ .../ingestion/metadata_normalizer.py | 77 ++++ api/collectors/ingestion/streaming_engine.py | 89 +++++ api/collectors/processing/__init__.py | 5 + .../processing/processing_engine_client.py | 100 +++++ api/collectors/scripts/manual_trigger.py | 62 +++ api/collectors/scripts/run_collectors.py | 53 +++ api/collectors/storage/__init__.py | 7 + api/collectors/storage/filesystem_store.py | 31 ++ api/collectors/storage/sqlite_store.py | 97 +++++ api/collectors/storage/storage_interface.py | 26 ++ .../template/new_connector_template.py | 57 +++ api/collectors/tests/conftest.py | 8 + api/collectors/tests/integration/__init__.py | 1 + .../tests/integration/test_e2e_collection.py | 80 ++++ api/collectors/tests/unit/__init__.py | 1 + .../tests/unit/test_browse_service.py | 85 +++++ .../tests/unit/test_metadata_normalizer.py | 30 ++ .../tests/unit/test_mock_connector_http.py | 44 +++ .../tests/unit/test_oauth_manager.py | 134 +++++++ .../tests/unit/test_rate_limit_circuit.py | 50 +++ api/collectors/tests/unit/test_registry.py | 56 +++ .../tests/unit/test_security_manager.py | 76 ++++ .../tests/unit/test_selected_collection.py | 82 ++++ .../tests/unit/test_selection_store.py | 41 ++ .../tests/unit/test_state_manager.py | 51 +++ api/collectors/tests/unit/test_token_store.py | 46 +++ api/main.py | 356 +++++++++++++++++- api/requirements-collectors.txt | 11 + api/static/collectors/index.html | 76 ++++ docker-compose.yml | 4 + docs/collector-framework/WORKFLOW.md | 306 +++++++++++++++ docs/collector-framework/traceability.md | 263 +++++++++++++ secrets/.env.local.example | 19 + secrets/.gitignore | 5 + secrets/dropbox.json.example | 4 + 76 files changed, 6125 insertions(+), 8 deletions(-) create mode 100644 .github/CODEOWNERS create mode 100644 .github/ISSUE_TEMPLATE/bug.yml create mode 100644 .github/ISSUE_TEMPLATE/feature.yml create mode 100644 .github/ISSUE_TEMPLATE/task.yml create mode 100644 .github/pull_request_template.md create mode 100644 .github/workflows/collectors-ci.yml create mode 100644 api/collectors/__init__.py create mode 100644 api/collectors/base/__init__.py create mode 100644 api/collectors/base/auth_context.py create mode 100644 api/collectors/base/base_collector.py create mode 100644 api/collectors/base/types.py create mode 100644 api/collectors/box/__init__.py create mode 100644 api/collectors/box/box_auth.py create mode 100644 api/collectors/box/box_collector.py create mode 100644 api/collectors/core/__init__.py create mode 100644 api/collectors/core/browse_service.py create mode 100644 api/collectors/core/logging_manager.py create mode 100644 api/collectors/core/oauth_manager.py create mode 100644 api/collectors/core/oauth_routes.py create mode 100644 api/collectors/core/orchestrator.py create mode 100644 api/collectors/core/registry.py create mode 100644 api/collectors/core/security_manager.py create mode 100644 api/collectors/core/selection_store.py create mode 100644 api/collectors/core/state_manager.py create mode 100644 api/collectors/core/token_store.py create mode 100644 api/collectors/core/trigger_manager.py create mode 100644 api/collectors/docs/architecture.md create mode 100644 api/collectors/docs/browse-and-select.md create mode 100644 api/collectors/docs/connectors.md create mode 100644 api/collectors/docs/ingestion_flow.md create mode 100644 api/collectors/docs/m9-collection-ui.md create mode 100644 api/collectors/dropbox/__init__.py create mode 100644 api/collectors/dropbox/dropbox_auth.py create mode 100644 api/collectors/dropbox/dropbox_collector.py create mode 100644 api/collectors/google_drive/__init__.py create mode 100644 api/collectors/google_drive/gdrive_auth.py create mode 100644 api/collectors/google_drive/gdrive_collector.py create mode 100644 api/collectors/ingestion/__init__.py create mode 100644 api/collectors/ingestion/ingestion_pipeline.py create mode 100644 api/collectors/ingestion/metadata_normalizer.py create mode 100644 api/collectors/ingestion/streaming_engine.py create mode 100644 api/collectors/processing/__init__.py create mode 100644 api/collectors/processing/processing_engine_client.py create mode 100644 api/collectors/scripts/manual_trigger.py create mode 100644 api/collectors/scripts/run_collectors.py create mode 100644 api/collectors/storage/__init__.py create mode 100644 api/collectors/storage/filesystem_store.py create mode 100644 api/collectors/storage/sqlite_store.py create mode 100644 api/collectors/storage/storage_interface.py create mode 100644 api/collectors/template/new_connector_template.py create mode 100644 api/collectors/tests/conftest.py create mode 100644 api/collectors/tests/integration/__init__.py create mode 100644 api/collectors/tests/integration/test_e2e_collection.py create mode 100644 api/collectors/tests/unit/__init__.py create mode 100644 api/collectors/tests/unit/test_browse_service.py create mode 100644 api/collectors/tests/unit/test_metadata_normalizer.py create mode 100644 api/collectors/tests/unit/test_mock_connector_http.py create mode 100644 api/collectors/tests/unit/test_oauth_manager.py create mode 100644 api/collectors/tests/unit/test_rate_limit_circuit.py create mode 100644 api/collectors/tests/unit/test_registry.py create mode 100644 api/collectors/tests/unit/test_security_manager.py create mode 100644 api/collectors/tests/unit/test_selected_collection.py create mode 100644 api/collectors/tests/unit/test_selection_store.py create mode 100644 api/collectors/tests/unit/test_state_manager.py create mode 100644 api/collectors/tests/unit/test_token_store.py create mode 100644 api/requirements-collectors.txt create mode 100644 api/static/collectors/index.html create mode 100644 docs/collector-framework/WORKFLOW.md create mode 100644 docs/collector-framework/traceability.md create mode 100644 secrets/.env.local.example create mode 100644 secrets/.gitignore create mode 100644 secrets/dropbox.json.example diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS new file mode 100644 index 000000000..354fbeed1 --- /dev/null +++ b/.github/CODEOWNERS @@ -0,0 +1,11 @@ +# CODEOWNERS — External Data Collection Framework +# +# Add collector maintainers here when the private FreeEed-collectors repo is created. +# Example (uncomment and replace with GitHub usernames or teams): +# +# /api/collectors/ @shmsoft/collector-maintainers +# /api/collectors/docs/ @shmsoft/collector-maintainers +# /.github/workflows/collectors-ci.yml @shmsoft/collector-maintainers +# /docs/collector-framework/ @shmsoft/collector-maintainers +# +# Until maintainers are assigned, rely on manual review from core contributors. diff --git a/.github/ISSUE_TEMPLATE/bug.yml b/.github/ISSUE_TEMPLATE/bug.yml new file mode 100644 index 000000000..e1ea0ef73 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug.yml @@ -0,0 +1,76 @@ +name: Collector Bug +description: Report a defect in the External Data Collection Framework +title: "[COLLECTOR] " +labels: + - type:bug + - epic:collector-framework +body: + - type: markdown + attributes: + value: | + Use the **`[COLLECTOR]`** title prefix. Redact tokens, credentials, customer names, and PII before pasting logs. + + - type: dropdown + id: connector + attributes: + label: Connector + description: Which connector or layer is affected? + options: + - none (core/framework) + - box + - gdrive + - dropbox + - api + - ingestion + - storage + - security + validations: + required: true + + - type: textarea + id: repro_steps + attributes: + label: Steps to reproduce + description: Minimal steps to trigger the bug + placeholder: | + 1. Configure ... + 2. Run ... + 3. Observe ... + validations: + required: true + + - type: textarea + id: expected + attributes: + label: Expected behavior + placeholder: What should have happened? + validations: + required: true + + - type: textarea + id: actual + attributes: + label: Actual behavior + placeholder: What happened instead? + validations: + required: true + + - type: textarea + id: logs + attributes: + label: Logs (redacted) + description: Paste relevant log excerpts. Remove tokens, secrets, emails, and file paths containing PII. + placeholder: | + [INFO] job_id=abc123 connector=box ... + [ERROR] ... + validations: + required: false + + - type: input + id: environment + attributes: + label: Environment + description: e.g. local Docker, Azure container, air-gapped laptop + placeholder: local Docker / Python 3.11 + validations: + required: false diff --git a/.github/ISSUE_TEMPLATE/feature.yml b/.github/ISSUE_TEMPLATE/feature.yml new file mode 100644 index 000000000..7ec947439 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature.yml @@ -0,0 +1,93 @@ +name: Collector Feature +description: New connector capability or subsystem feature for the External Data Collection Framework +title: "[COLLECTOR] " +labels: + - type:feature + - epic:collector-framework +body: + - type: markdown + attributes: + value: | + Use the **`[COLLECTOR]`** title prefix for all collector-framework issues so they are easy to filter. + + Work is tracked in the private **`FreeEed-collectors`** companion repo (or locally until that repo exists). + Do not open in-progress collector issues on the public `shmsoft/FreeEed` repo. + + - type: dropdown + id: area + attributes: + label: Area + description: Primary subsystem or layer affected + options: + - core + - ingestion + - storage + - connectors + - api + - security + - ci + - docs + validations: + required: true + + - type: dropdown + id: connector + attributes: + label: Connector + description: External platform, if applicable + options: + - none + - box + - gdrive + - dropbox + validations: + required: true + + - type: dropdown + id: milestone + attributes: + label: Milestone + description: Target milestone (M0–M8) + options: + - M0 — GitHub & scaffolding + - M1 — Core framework + - M2 — Ingestion & storage + - M3 — Trigger & processing + - M4 — Box connector + - M5 — Drive & Dropbox + - M6 — API & deployment + - M7 — Tests & docs + - M8 — Public promotion + validations: + required: true + + - type: textarea + id: description + attributes: + label: Feature description + description: What should be built and why? + placeholder: | + As a [role], I need [capability] so that [outcome]. + validations: + required: true + + - type: textarea + id: acceptance_criteria + attributes: + label: Acceptance criteria + description: Testable conditions that define "done" + placeholder: | + - [ ] Criterion 1 + - [ ] Criterion 2 + - [ ] Unit tests added/updated + - [ ] No secrets or telemetry introduced + validations: + required: true + + - type: checkboxes + id: labels_hint + attributes: + label: Apply matching labels after creation + options: + - label: I will add the appropriate `area:*` and `connector:*` labels (if not none) + - label: I will link this issue to the **External Data Collection Framework** GitHub Project diff --git a/.github/ISSUE_TEMPLATE/task.yml b/.github/ISSUE_TEMPLATE/task.yml new file mode 100644 index 000000000..a2ae217f6 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/task.yml @@ -0,0 +1,66 @@ +name: Collector Task +description: Implementation task linked to a milestone +title: "[COLLECTOR] " +labels: + - epic:collector-framework +body: + - type: markdown + attributes: + value: | + Use the **`[COLLECTOR]`** title prefix. One task should map to one PR when possible. + + - type: dropdown + id: milestone + attributes: + label: Milestone + description: Target milestone + options: + - M0 + - M1 + - M2 + - M3 + - M4 + - M5 + - M6 + - M7 + - M8 + validations: + required: true + + - type: dropdown + id: area + attributes: + label: Area + options: + - core + - ingestion + - storage + - connectors + - api + - security + - ci + - docs + validations: + required: true + + - type: textarea + id: description + attributes: + label: Description + description: What needs to be done? + placeholder: Concrete implementation steps or deliverables for this task. + validations: + required: true + + - type: textarea + id: definition_of_done + attributes: + label: Definition of done + description: Checklist that confirms the task is complete + placeholder: | + - [ ] Code merged to private repo `main` + - [ ] Unit tests pass in CI + - [ ] Documentation updated (if applicable) + - [ ] Issue linked in PR with `Closes #NNN` + validations: + required: true diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md new file mode 100644 index 000000000..4c94049c8 --- /dev/null +++ b/.github/pull_request_template.md @@ -0,0 +1,53 @@ +## Summary + + + +## Linked issues + + +- Closes # + +## Milestone + + +- [ ] M0 — GitHub & scaffolding +- [ ] M1 — Core framework +- [ ] M2 — Ingestion & storage +- [ ] M3 — Trigger & processing +- [ ] M4 — Box connector +- [ ] M5 — Drive & Dropbox +- [ ] M6 — API & deployment +- [ ] M7 — Tests & docs +- [ ] M8 — Public promotion + +## Labels + + +- Area: `area:core` | `area:ingestion` | `area:storage` | `area:connectors` | `area:api` | `area:security` | `area:ci` | `area:docs` +- Connector (if applicable): `connector:box` | `connector:gdrive` | `connector:dropbox` | `connector:template` +- Type: `type:feature` | `type:bug` | `type:tech-debt` | `type:spike` | `type:docs` + +## Test plan + +- [ ] `pytest api/collectors/tests/unit` passes locally (or N/A — scaffold not yet present) +- [ ] Integration tests run / skipped appropriately (secrets not required for this PR) +- [ ] Manual smoke test described below (if applicable) + + + +## Security checklist + +- [ ] No secrets, tokens, or credentials committed +- [ ] No external telemetry (Sentry, Datadog, etc.) added +- [ ] Log output sanitized (tokens, emails, PII paths redacted) +- [ ] Outbound network limited to target platform APIs (no unexpected endpoints) + +## Public promotion ready + + +- [ ] **Yes** — ready for upstream promotion (sanitized description, no internal-only references) +- [ ] **No** — private-repo only until a later milestone + +**Notes:** + + diff --git a/.github/workflows/collectors-ci.yml b/.github/workflows/collectors-ci.yml new file mode 100644 index 000000000..8826134fd --- /dev/null +++ b/.github/workflows/collectors-ci.yml @@ -0,0 +1,109 @@ +# Collectors CI — intended for the private FreeEed-collectors repo. +# Copy this workflow (and sibling .github/ artifacts) when bootstrapping that repo, +# or keep here locally until the private companion repo is created. +name: Collectors CI + +on: + push: + branches: + - main + paths: + - 'api/collectors/**' + - 'api/main.py' + - '.github/workflows/collectors-ci.yml' + pull_request: + branches: + - main + paths: + - 'api/collectors/**' + - 'api/main.py' + - '.github/workflows/collectors-ci.yml' + +permissions: + contents: read + +jobs: + unit-tests: + name: Collector unit tests + runs-on: ubuntu-latest + defaults: + run: + working-directory: . + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Set up Python 3.11 + uses: actions/setup-python@v5 + with: + python-version: '3.11' + cache: pip + cache-dependency-path: | + api/requirements.txt + api/requirements-collectors.txt + + - name: Install base API dependencies + run: | + python -m pip install --upgrade pip + pip install -r api/requirements.txt + + - name: Install collector dependencies + if: hashFiles('api/requirements-collectors.txt') != '' + run: pip install -r api/requirements-collectors.txt + + - name: Install pytest + run: pip install pytest + + - name: Run collector unit tests + if: hashFiles('api/collectors/tests/unit/**') != '' + run: pytest api/collectors/tests/unit -v + + - name: Skip unit tests (scaffold not present) + if: hashFiles('api/collectors/tests/unit/**') == '' + run: | + echo "No unit tests found under api/collectors/tests/unit — skipping until M1+ scaffold exists." + + integration-tests: + name: Collector integration tests + runs-on: ubuntu-latest + # Gate on repo secrets; skip entirely when integration tests do not exist yet. + if: | + hashFiles('api/collectors/tests/integration/**') != '' && + (vars.RUN_COLLECTOR_INTEGRATION == 'true' || github.event_name == 'workflow_dispatch') + defaults: + run: + working-directory: . + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Set up Python 3.11 + uses: actions/setup-python@v5 + with: + python-version: '3.11' + cache: pip + cache-dependency-path: | + api/requirements.txt + api/requirements-collectors.txt + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install -r api/requirements.txt + if [ -f api/requirements-collectors.txt ]; then + pip install -r api/requirements-collectors.txt + fi + pip install pytest + + - name: Run collector integration tests + env: + # Sandbox credentials — configure as GitHub Actions secrets in the private repo. + BOX_CLIENT_ID: ${{ secrets.BOX_CLIENT_ID }} + BOX_CLIENT_SECRET: ${{ secrets.BOX_CLIENT_SECRET }} + GDRIVE_CLIENT_ID: ${{ secrets.GDRIVE_CLIENT_ID }} + GDRIVE_CLIENT_SECRET: ${{ secrets.GDRIVE_CLIENT_SECRET }} + DROPBOX_APP_KEY: ${{ secrets.DROPBOX_APP_KEY }} + DROPBOX_APP_SECRET: ${{ secrets.DROPBOX_APP_SECRET }} + run: pytest api/collectors/tests/integration -v diff --git a/.gitignore b/.gitignore index 2aa486088..081909bc5 100644 --- a/.gitignore +++ b/.gitignore @@ -41,7 +41,12 @@ freeeed-test/.classpath freeeed-processing/output # Python **/.env +.env.local myenv/ +# Collector dev secrets (commit only *.example files under secrets/) +secrets/*.json +!secrets/*.json.example +secrets/.env.local .ipynb_checkpoints __pycache__/ .vscode/ diff --git a/api/Dockerfile b/api/Dockerfile index 13092ad4f..fffedf643 100644 --- a/api/Dockerfile +++ b/api/Dockerfile @@ -34,14 +34,14 @@ RUN mkdir -p /app/db && \ COPY --from=builder /app/freeeed-processing/target/freeeed-processing-1.0-SNAPSHOT-jar-with-dependencies.jar /app/freeeed-processing.jar # Install Python dependencies -COPY api/requirements.txt . -RUN pip install --no-cache-dir -r requirements.txt +COPY api/requirements.txt api/requirements-collectors.txt ./ +RUN pip install --no-cache-dir -r requirements.txt -r requirements-collectors.txt # Copy FastAPI application COPY api/ . -# Create data directories -RUN mkdir -p /data/input /data/output +# Create data and secrets directories +RUN mkdir -p /data/input /data/output /data/input/collections /app/secrets /app/db # Set environment variables ENV PYTHONUNBUFFERED=1 diff --git a/api/collectors/__init__.py b/api/collectors/__init__.py new file mode 100644 index 000000000..efb9f3f58 --- /dev/null +++ b/api/collectors/__init__.py @@ -0,0 +1,3 @@ +"""FreeEed External Data Collection Framework.""" + +__version__ = "1.0.0" diff --git a/api/collectors/base/__init__.py b/api/collectors/base/__init__.py new file mode 100644 index 000000000..4d7cdc674 --- /dev/null +++ b/api/collectors/base/__init__.py @@ -0,0 +1,21 @@ +from .base_collector import BaseCollector +from .auth_context import AuthContext +from .types import ( + CollectionItem, + CollectionJob, + CollectionJobStatus, + ConnectorHealth, + ItemMetadata, + RateLimitInfo, +) + +__all__ = [ + "BaseCollector", + "AuthContext", + "CollectionItem", + "CollectionJob", + "CollectionJobStatus", + "ConnectorHealth", + "ItemMetadata", + "RateLimitInfo", +] diff --git a/api/collectors/base/auth_context.py b/api/collectors/base/auth_context.py new file mode 100644 index 000000000..8263ff33f --- /dev/null +++ b/api/collectors/base/auth_context.py @@ -0,0 +1,30 @@ +"""Authentication context for connector OAuth flows.""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from datetime import datetime +from typing import Any, Dict, Optional + + +@dataclass +class AuthContext: + """Holds OAuth tokens and connector-specific auth metadata.""" + + connector_id: str + access_token: Optional[str] = None + refresh_token: Optional[str] = None + token_type: str = "Bearer" + expires_at: Optional[datetime] = None + scopes: list[str] = field(default_factory=list) + extra: Dict[str, Any] = field(default_factory=dict) + + def is_expired(self, skew_seconds: int = 60) -> bool: + if self.expires_at is None: + return False + return datetime.utcnow().timestamp() >= (self.expires_at.timestamp() - skew_seconds) + + def authorization_header(self) -> Dict[str, str]: + if not self.access_token: + return {} + return {"Authorization": f"{self.token_type} {self.access_token}"} diff --git a/api/collectors/base/base_collector.py b/api/collectors/base/base_collector.py new file mode 100644 index 000000000..c4c2c85b4 --- /dev/null +++ b/api/collectors/base/base_collector.py @@ -0,0 +1,116 @@ +"""Abstract base class for external data connectors.""" + +from __future__ import annotations + +import abc +import time +from datetime import datetime +from typing import Any, Dict, Iterator, List, Optional + +from .auth_context import AuthContext +from .types import BrowseItem, CollectionItem, ConnectorHealth, ItemMetadata, RateLimitInfo, SelectedItem + + +class BaseCollector(abc.ABC): + """All connectors must implement this interface.""" + + def __init__(self, auth: AuthContext, config: Optional[Dict[str, Any]] = None): + self.auth = auth + self.config = config or {} + + @property + @abc.abstractmethod + def connector_id(self) -> str: + """Unique connector identifier.""" + + @abc.abstractmethod + def authenticate(self) -> AuthContext: + """Validate or refresh credentials.""" + + @abc.abstractmethod + def list_children(self, parent_id: Optional[str] = None) -> List[BrowseItem]: + """List immediate child folders and files at parent (root when parent_id is None).""" + + @abc.abstractmethod + def list_items(self, since: Optional[datetime] = None) -> List[ItemMetadata]: + """List available items, optionally filtered by modification time.""" + + @abc.abstractmethod + def fetch_item(self, item_id: str) -> CollectionItem: + """Download a single item by ID.""" + + @abc.abstractmethod + def stream_changes(self) -> Iterator[ItemMetadata]: + """Stream incremental changes from the remote source.""" + + @abc.abstractmethod + def handle_rate_limits(self, response: Any) -> Optional[RateLimitInfo]: + """Parse rate-limit response headers/body; return wait info if limited.""" + + @abc.abstractmethod + def serialize_metadata(self, metadata: ItemMetadata) -> Dict[str, Any]: + """Convert metadata to a JSON-serializable dict.""" + + def health_check(self) -> ConnectorHealth: + """Default health check: attempt authentication.""" + try: + self.authenticate() + return ConnectorHealth( + connector_id=self.connector_id, + healthy=True, + message="Authentication successful", + checked_at=datetime.utcnow(), + ) + except Exception as exc: + return ConnectorHealth( + connector_id=self.connector_id, + healthy=False, + message=str(exc), + checked_at=datetime.utcnow(), + ) + + def wait_for_rate_limit(self, info: RateLimitInfo) -> None: + """Block until rate limit window expires.""" + time.sleep(max(0.0, info.retry_after_seconds)) + + def expand_selection( + self, + selected: List[SelectedItem], + since: Optional[datetime] = None, + ) -> List[ItemMetadata]: + """Resolve selected folders/files into file metadata for collection.""" + self.authenticate() + seen: set[str] = set() + result: List[ItemMetadata] = [] + + def add_file(meta: ItemMetadata) -> None: + if meta.item_id in seen: + return + if since and meta.modified_at and meta.modified_at < since: + return + seen.add(meta.item_id) + result.append(meta) + + def walk_folder(folder_id: str) -> None: + for child in self.list_children(folder_id): + if child.type.value == "folder": + walk_folder(child.id) + else: + add_file(self._browse_item_to_metadata(child)) + + for item in selected: + if item.type.value == "folder": + walk_folder(item.id) + else: + add_file(self._browse_item_to_metadata(item)) + + return result + + def _browse_item_to_metadata(self, item: BrowseItem | SelectedItem) -> ItemMetadata: + return ItemMetadata( + item_id=item.id, + name=item.name or item.id, + size_bytes=getattr(item, "size", None), + modified_at=getattr(item, "modified_at", None), + source_path=f"{self.connector_id}://{item.path or item.id}", + ) diff --git a/api/collectors/base/types.py b/api/collectors/base/types.py new file mode 100644 index 000000000..bb81f6407 --- /dev/null +++ b/api/collectors/base/types.py @@ -0,0 +1,101 @@ +"""Shared types for the collector framework.""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from datetime import datetime +from enum import Enum +from typing import Any, Dict, List, Optional + + +class CollectionJobStatus(str, Enum): + PENDING = "pending" + RUNNING = "running" + COMPLETED = "completed" + FAILED = "failed" + CANCELLED = "cancelled" + + +class CircuitBreakerState(str, Enum): + CLOSED = "closed" + OPEN = "open" + HALF_OPEN = "half_open" + + +class BrowseItemType(str, Enum): + FOLDER = "folder" + FILE = "file" + + +@dataclass +class BrowseItem: + """Normalized remote folder/file entry for UI browsing.""" + + id: str + name: str + path: str + type: BrowseItemType + connector_id: str + size: Optional[int] = None + modified_at: Optional[datetime] = None + mime_type: Optional[str] = None + extra: Dict[str, Any] = field(default_factory=dict) + + +@dataclass +class SelectedItem: + """User selection persisted per project and connector.""" + + id: str + type: BrowseItemType + name: str = "" + path: str = "" + + +@dataclass +class ItemMetadata: + item_id: str + name: str + mime_type: Optional[str] = None + size_bytes: Optional[int] = None + modified_at: Optional[datetime] = None + source_path: Optional[str] = None + checksum: Optional[str] = None + extra: Dict[str, Any] = field(default_factory=dict) + + +@dataclass +class CollectionItem: + metadata: ItemMetadata + content: Optional[bytes] = None + local_path: Optional[str] = None + + +@dataclass +class RateLimitInfo: + retry_after_seconds: float + reason: str = "rate_limit" + + +@dataclass +class ConnectorHealth: + connector_id: str + healthy: bool + message: str = "" + checked_at: Optional[datetime] = None + + +@dataclass +class CollectionJob: + job_id: str + connector_id: str + status: CollectionJobStatus = CollectionJobStatus.PENDING + created_at: Optional[datetime] = None + started_at: Optional[datetime] = None + completed_at: Optional[datetime] = None + output_dir: Optional[str] = None + items_collected: int = 0 + items_failed: int = 0 + error_message: Optional[str] = None + auto_chain: bool = False + config: Dict[str, Any] = field(default_factory=dict) diff --git a/api/collectors/box/__init__.py b/api/collectors/box/__init__.py new file mode 100644 index 000000000..50375836b --- /dev/null +++ b/api/collectors/box/__init__.py @@ -0,0 +1,5 @@ +"""Box connector package.""" + +from .box_collector import BoxCollector + +__all__ = ["BoxCollector"] diff --git a/api/collectors/box/box_auth.py b/api/collectors/box/box_auth.py new file mode 100644 index 000000000..14bb3b847 --- /dev/null +++ b/api/collectors/box/box_auth.py @@ -0,0 +1,67 @@ +"""Box OAuth2 authentication.""" + +from __future__ import annotations + +import os +from datetime import datetime, timedelta +from typing import Any, Dict, Optional + +import requests + +from collectors.base.auth_context import AuthContext + + +class BoxAuth: + TOKEN_URL = "https://api.box.com/oauth2/token" + + def __init__(self, auth: AuthContext, config: Optional[Dict[str, Any]] = None): + self.auth = auth + self.config = config or {} + self.client_id = self.config.get("client_id") or os.getenv("BOX_CLIENT_ID") + self.client_secret = self.config.get("client_secret") or os.getenv("BOX_CLIENT_SECRET") + + def ensure_valid_token(self) -> AuthContext: + if not self.auth.is_expired(): + return self.auth + if not self.auth.refresh_token: + if self.auth.access_token: + return self.auth + raise ValueError("Box credentials missing: access_token or refresh_token required") + return self.refresh_token() + + def refresh_token(self) -> AuthContext: + if self.config.get("mock_refresh"): + self.auth.access_token = "mock-box-token" + self.auth.expires_at = datetime.utcnow() + timedelta(hours=1) + return self.auth + + response = requests.post( + self.TOKEN_URL, + data={ + "grant_type": "refresh_token", + "refresh_token": self.auth.refresh_token, + "client_id": self.client_id, + "client_secret": self.client_secret, + }, + timeout=30, + ) + if response.status_code >= 400: + raise RuntimeError(f"Box token refresh failed: {response.status_code}") + data = response.json() + self.auth.access_token = data["access_token"] + if "refresh_token" in data: + self.auth.refresh_token = data["refresh_token"] + expires_in = int(data.get("expires_in", 3600)) + self.auth.expires_at = datetime.utcnow() + timedelta(seconds=expires_in) + return self.auth + + def build_client(self, auth: AuthContext): + from boxsdk import Client, OAuth2 + + oauth = OAuth2( + client_id=self.client_id, + client_secret=self.client_secret, + access_token=auth.access_token, + refresh_token=auth.refresh_token, + ) + return Client(oauth) diff --git a/api/collectors/box/box_collector.py b/api/collectors/box/box_collector.py new file mode 100644 index 000000000..ec58c45b5 --- /dev/null +++ b/api/collectors/box/box_collector.py @@ -0,0 +1,155 @@ +"""Box.com connector.""" + +from __future__ import annotations + +import os +from datetime import datetime +from typing import Any, Dict, Iterator, List, Optional + +from collectors.base.auth_context import AuthContext +from collectors.base.base_collector import BaseCollector +from collectors.base.types import BrowseItem, BrowseItemType, CollectionItem, ItemMetadata, RateLimitInfo +from collectors.core.registry import register_connector + +from .box_auth import BoxAuth + + +@register_connector("box") +class BoxCollector(BaseCollector): + """Collect files from Box using OAuth2.""" + + def __init__(self, auth: AuthContext, config: Optional[Dict[str, Any]] = None): + super().__init__(auth, config) + self._auth_client = BoxAuth(auth, config) + self._client = None + self._mock_items = config.get("mock_items") if config else None + self._mock_browse = config.get("mock_browse") if config else None + + @property + def connector_id(self) -> str: + return "box" + + def authenticate(self) -> AuthContext: + self.auth = self._auth_client.ensure_valid_token() + if self._mock_items is None and self._mock_browse is None: + self._client = self._auth_client.build_client(self.auth) + return self.auth + + def list_children(self, parent_id: Optional[str] = None) -> List[BrowseItem]: + self.authenticate() + folder_id = parent_id or "0" + if self._mock_browse is not None: + return [self._dict_to_browse_item(entry) for entry in self._mock_browse.get(folder_id, [])] + + items = self._client.folder(folder_id).get_items(limit=1000) + result: List[BrowseItem] = [] + for entry in items: + item_type = BrowseItemType.FOLDER if entry.type == "folder" else BrowseItemType.FILE + modified = getattr(entry, "modified_at", None) + modified_at = ( + datetime.fromisoformat(str(modified).replace("Z", "+00:00")) if modified else None + ) + result.append( + BrowseItem( + id=str(entry.id), + name=str(entry.name), + path=f"/{entry.name}" if folder_id == "0" else f"{folder_id}/{entry.name}", + type=item_type, + connector_id=self.connector_id, + size=getattr(entry, "size", None), + modified_at=modified_at, + ) + ) + return result + + def list_items(self, since: Optional[datetime] = None) -> List[ItemMetadata]: + self.authenticate() + if self._mock_items is not None: + return [self._to_metadata(item) for item in self._mock_items] + folder_id = self.config.get("folder_id", "0") + items = self._client.folder(folder_id).get_items(limit=1000) + result = [] + for entry in items: + if entry.type != "file": + continue + meta = self._entry_to_metadata(entry) + if since and meta.modified_at and meta.modified_at < since: + continue + result.append(meta) + return result + + def fetch_item(self, item_id: str) -> CollectionItem: + self.authenticate() + if self._mock_items is not None: + for item in self._mock_items: + if item["id"] == item_id: + content = item.get("content", b"").encode() if isinstance(item.get("content"), str) else item.get("content", b"") + return CollectionItem(metadata=self._to_metadata(item), content=content) + raise FileNotFoundError(item_id) + + file_obj = self._client.file(item_id).get() + content = self._client.file(item_id).content() + return CollectionItem( + metadata=self._entry_to_metadata(file_obj), + content=content, + ) + + def stream_changes(self) -> Iterator[ItemMetadata]: + yield from self.list_items() + + def handle_rate_limits(self, response: Any) -> Optional[RateLimitInfo]: + headers = getattr(response, "headers", {}) or {} + retry_after = headers.get("Retry-After") or headers.get("retry-after") + status = getattr(response, "status_code", None) + if status == 429 or retry_after: + seconds = float(retry_after) if retry_after else 60.0 + return RateLimitInfo(retry_after_seconds=seconds, reason="box_rate_limit") + return None + + def serialize_metadata(self, metadata: ItemMetadata) -> Dict[str, Any]: + return { + "id": metadata.item_id, + "name": metadata.name, + "mime_type": metadata.mime_type, + "size": metadata.size_bytes, + "modified_at": metadata.modified_at.isoformat() if metadata.modified_at else None, + "source": "box", + } + + def _entry_to_metadata(self, entry: Any) -> ItemMetadata: + modified = getattr(entry, "modified_at", None) + modified_at = datetime.fromisoformat(str(modified).replace("Z", "+00:00")) if modified else None + return ItemMetadata( + item_id=str(entry.id), + name=str(entry.name), + size_bytes=getattr(entry, "size", None), + modified_at=modified_at, + source_path=f"box://{entry.id}", + ) + + def _dict_to_browse_item(self, item: Dict[str, Any]) -> BrowseItem: + item_type = BrowseItemType(item.get("type", "file")) + modified = item.get("modified_at") + modified_at = datetime.fromisoformat(modified) if modified else None + return BrowseItem( + id=str(item["id"]), + name=str(item.get("name", item["id"])), + path=str(item.get("path", item["id"])), + type=item_type, + connector_id=self.connector_id, + size=item.get("size"), + modified_at=modified_at, + mime_type=item.get("mime_type"), + ) + + def _to_metadata(self, item: Dict[str, Any]) -> ItemMetadata: + modified = item.get("modified_at") + modified_at = datetime.fromisoformat(modified) if modified else None + return ItemMetadata( + item_id=str(item["id"]), + name=str(item.get("name", item["id"])), + mime_type=item.get("mime_type"), + size_bytes=item.get("size"), + modified_at=modified_at, + source_path=f"box://{item['id']}", + ) diff --git a/api/collectors/core/__init__.py b/api/collectors/core/__init__.py new file mode 100644 index 000000000..b5c155e7f --- /dev/null +++ b/api/collectors/core/__init__.py @@ -0,0 +1,27 @@ +"""Core collector subsystems.""" + +from .registry import register_connector, get_connector_class, list_connectors +from .security_manager import SecurityManager +from .logging_manager import LoggingManager +from .state_manager import StateManager + +__all__ = [ + "register_connector", + "get_connector_class", + "list_connectors", + "Orchestrator", + "TriggerManager", + "SecurityManager", + "LoggingManager", + "StateManager", +] + + +def __getattr__(name: str): + if name == "Orchestrator": + from .orchestrator import Orchestrator + return Orchestrator + if name == "TriggerManager": + from .trigger_manager import TriggerManager + return TriggerManager + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") diff --git a/api/collectors/core/browse_service.py b/api/collectors/core/browse_service.py new file mode 100644 index 000000000..41a42bea2 --- /dev/null +++ b/api/collectors/core/browse_service.py @@ -0,0 +1,152 @@ +"""Project-scoped connect and browse operations.""" + +from __future__ import annotations + +from typing import Any, Dict, List, Optional + +from collectors.base.auth_context import AuthContext +from collectors.base.types import BrowseItem +from collectors.core.registry import get_connector_class +from collectors.core.security_manager import SecurityManager +from collectors.core.token_store import TokenStore + + +class CredentialsNotConfiguredError(Exception): + """Raised when connector secrets are missing.""" + + +class ConnectorAuthError(Exception): + """Raised when authentication fails.""" + + +class BrowseService: + """Build collectors and expose remote folder browsing.""" + + def __init__( + self, + security_manager: Optional[SecurityManager] = None, + token_store: Optional[TokenStore] = None, + ): + self.security = security_manager or SecurityManager() + self.token_store = token_store or TokenStore() + + def has_credentials(self, connector_id: str, project_id: Optional[str] = None) -> bool: + if project_id and self.token_store.has_tokens(project_id, connector_id): + return True + secrets = self.security.load_connector_secrets(connector_id) + return bool(secrets.get("access_token") or secrets.get("refresh_token")) + + @staticmethod + def _has_token_override(token_override: Optional[Dict[str, Any]]) -> bool: + if not token_override: + return False + return bool(token_override.get("access_token") or token_override.get("refresh_token")) + + def connect( + self, + connector_id: str, + project_id: Optional[str] = None, + token_override: Optional[Dict[str, Any]] = None, + config: Optional[Dict[str, Any]] = None, + ) -> Dict[str, Any]: + if not self._has_token_override(token_override) and not self.has_credentials( + connector_id, project_id + ): + raise CredentialsNotConfiguredError( + f"No credentials configured for connector '{connector_id}'. " + f"Complete OAuth at /collect/oauth/{connector_id}/authorize?project_id=YOUR_PROJECT " + f"or set COLLECTOR_{connector_id.upper()}_ACCESS_TOKEN / " + f"./secrets/{connector_id}.json" + ) + + auth = self._build_auth(connector_id, token_override, project_id) + collector = self._build_collector(connector_id, auth, config) + health = collector.health_check() + if not health.healthy: + raise ConnectorAuthError(health.message) + return { + "connector_id": connector_id, + "connected": True, + "message": health.message, + } + + def browse( + self, + connector_id: str, + parent_id: Optional[str] = None, + project_id: Optional[str] = None, + token_override: Optional[Dict[str, Any]] = None, + config: Optional[Dict[str, Any]] = None, + ) -> List[BrowseItem]: + if not self._has_token_override(token_override) and not self.has_credentials( + connector_id, project_id + ): + raise CredentialsNotConfiguredError( + f"No credentials configured for connector '{connector_id}'. " + f"Complete OAuth at /collect/oauth/{connector_id}/authorize?project_id=YOUR_PROJECT" + ) + + auth = self._build_auth(connector_id, token_override, project_id) + collector = self._build_collector(connector_id, auth, config) + return collector.list_children(parent_id) + + def _build_auth( + self, + connector_id: str, + token_override: Optional[Dict[str, Any]] = None, + project_id: Optional[str] = None, + ) -> AuthContext: + if token_override: + auth = AuthContext( + connector_id=connector_id, + access_token=token_override.get("access_token"), + refresh_token=token_override.get("refresh_token"), + extra={ + k: v + for k, v in token_override.items() + if k not in ("access_token", "refresh_token") + }, + ) + if project_id: + auth.extra["project_id"] = project_id + else: + auth = self.security.build_auth_context( + connector_id, + project_id=project_id, + token_store=self.token_store, + ) + prev_access = auth.access_token + refreshed = self.security.refresh_auth_if_needed(auth) + resolved_project = project_id or refreshed.extra.get("project_id") + if resolved_project and refreshed.access_token and refreshed.access_token != prev_access: + self.token_store.save_auth_context( + resolved_project, connector_id, refreshed, self.security + ) + return refreshed + + def _build_collector( + self, + connector_id: str, + auth: AuthContext, + config: Optional[Dict[str, Any]] = None, + ): + connector_cls = get_connector_class(connector_id) + merged_config = dict(config or {}) + secrets = self.security.load_connector_secrets(connector_id) + for key in ("client_id", "client_secret"): + if key in secrets and key not in merged_config: + merged_config[key] = secrets[key] + return connector_cls(auth=auth, config=merged_config) + + @staticmethod + def serialize_browse_item(item: BrowseItem) -> Dict[str, Any]: + return { + "id": item.id, + "name": item.name, + "path": item.path, + "type": item.type.value if hasattr(item.type, "value") else item.type, + "size": item.size, + "modified_at": item.modified_at.isoformat() if item.modified_at else None, + "connector_id": item.connector_id, + "mime_type": item.mime_type, + } diff --git a/api/collectors/core/logging_manager.py b/api/collectors/core/logging_manager.py new file mode 100644 index 000000000..450f3ef9b --- /dev/null +++ b/api/collectors/core/logging_manager.py @@ -0,0 +1,62 @@ +"""Structured JSON logging for collectors.""" + +from __future__ import annotations + +import json +import logging +import sys +from datetime import datetime +from typing import Any, Dict, Optional + + +class _JsonFormatter(logging.Formatter): + def format(self, record: logging.LogRecord) -> str: + payload: Dict[str, Any] = { + "timestamp": datetime.utcnow().isoformat() + "Z", + "level": record.levelname, + "logger": record.name, + "message": record.getMessage(), + } + if hasattr(record, "extra_fields") and isinstance(record.extra_fields, dict): + payload.update(record.extra_fields) + if record.exc_info: + payload["exception"] = self.formatException(record.exc_info) + return json.dumps(payload, default=str) + + +class LoggingManager: + """Centralized structured logging.""" + + _configured = False + + @classmethod + def configure(cls, level: int = logging.INFO, name: str = "collectors") -> logging.Logger: + logger = logging.getLogger(name) + if not cls._configured: + handler = logging.StreamHandler(sys.stdout) + handler.setFormatter(_JsonFormatter()) + logger.addHandler(handler) + logger.setLevel(level) + logger.propagate = False + cls._configured = True + return logger + + @classmethod + def log_event( + cls, + logger: logging.Logger, + event: str, + level: int = logging.INFO, + **fields: Any, + ) -> None: + record = logger.makeRecord( + logger.name, + level, + "(unknown)", + 0, + event, + (), + None, + ) + record.extra_fields = {"event": event, **fields} # type: ignore[attr-defined] + logger.handle(record) diff --git a/api/collectors/core/oauth_manager.py b/api/collectors/core/oauth_manager.py new file mode 100644 index 000000000..e7a544c41 --- /dev/null +++ b/api/collectors/core/oauth_manager.py @@ -0,0 +1,191 @@ +"""OAuth2 authorization code flow with PKCE for collector connectors.""" + +from __future__ import annotations + +import base64 +import hashlib +import os +import secrets +from dataclasses import dataclass +from datetime import datetime, timedelta +from typing import Any, Dict, Optional +from urllib.parse import urlencode + +import requests + +from collectors.base.auth_context import AuthContext +from collectors.core.security_manager import SecurityManager +from collectors.core.token_store import TokenStore + + +class OAuthError(Exception): + """OAuth flow failure.""" + + +@dataclass(frozen=True) +class OAuthProviderConfig: + authorize_url: str + token_url: str + scopes: str + client_id_env: str + client_secret_env: str + + +PROVIDER_CONFIGS: Dict[str, OAuthProviderConfig] = { + "dropbox": OAuthProviderConfig( + authorize_url="https://www.dropbox.com/oauth2/authorize", + token_url="https://api.dropboxapi.com/oauth2/token", + scopes="files.metadata.read files.content.read account_info.read", + client_id_env="DROPBOX_CLIENT_ID", + client_secret_env="DROPBOX_CLIENT_SECRET", + ), +} + + +class OAuthManager: + """Provider-agnostic OAuth2 manager (authorization code + PKCE).""" + + STATE_TTL_MINUTES = 10 + + def __init__( + self, + token_store: Optional[TokenStore] = None, + security_manager: Optional[SecurityManager] = None, + redirect_base: Optional[str] = None, + ): + self.token_store = token_store or TokenStore() + self.security = security_manager or SecurityManager() + self.redirect_base = ( + redirect_base or os.getenv("COLLECTOR_OAUTH_REDIRECT_BASE", "http://localhost:8000") + ).rstrip("/") + + def supports_connector(self, connector_id: str) -> bool: + return connector_id in PROVIDER_CONFIGS + + def callback_url(self, connector_id: str) -> str: + return f"{self.redirect_base}/collect/oauth/{connector_id}/callback" + + def build_authorize_url( + self, + connector_id: str, + project_id: str, + redirect_after: Optional[str] = None, + ) -> str: + provider = self._provider(connector_id) + client_id, _ = self._client_credentials(connector_id) + if not client_id: + raise OAuthError( + f"OAuth client_id not configured for '{connector_id}'. " + f"Set {provider.client_id_env} or secrets/{connector_id}.json" + ) + + code_verifier = _generate_code_verifier() + code_challenge = _code_challenge_s256(code_verifier) + state = secrets.token_urlsafe(32) + expires_at = datetime.utcnow() + timedelta(minutes=self.STATE_TTL_MINUTES) + self.token_store.save_oauth_state( + state=state, + project_id=project_id, + connector_id=connector_id, + code_verifier=code_verifier, + expires_at=expires_at, + redirect_after=redirect_after, + ) + + params = { + "client_id": client_id, + "response_type": "code", + "redirect_uri": self.callback_url(connector_id), + "state": state, + "code_challenge": code_challenge, + "code_challenge_method": "S256", + "token_access_type": "offline", + "scope": provider.scopes, + } + return f"{provider.authorize_url}?{urlencode(params)}" + + def handle_callback( + self, + connector_id: str, + code: str, + state: str, + ) -> tuple[AuthContext, str, Optional[str]]: + """Exchange code for tokens, persist, return auth + project_id + redirect_after.""" + if not code: + raise OAuthError("Missing authorization code") + if not state: + raise OAuthError("Missing OAuth state") + + oauth_state = self.token_store.consume_oauth_state(state, connector_id) + if not oauth_state: + raise OAuthError("Invalid or expired OAuth state") + + project_id = oauth_state["project_id"] + auth = self._exchange_code( + connector_id=connector_id, + code=code, + code_verifier=oauth_state["code_verifier"], + ) + self.token_store.save_auth_context(project_id, connector_id, auth, self.security) + return auth, project_id, oauth_state.get("redirect_after") + + def _exchange_code( + self, + connector_id: str, + code: str, + code_verifier: str, + ) -> AuthContext: + provider = self._provider(connector_id) + client_id, client_secret = self._client_credentials(connector_id) + if not client_id or not client_secret: + raise OAuthError(f"OAuth client credentials not configured for '{connector_id}'") + + response = requests.post( + provider.token_url, + data={ + "grant_type": "authorization_code", + "code": code, + "code_verifier": code_verifier, + "client_id": client_id, + "client_secret": client_secret, + "redirect_uri": self.callback_url(connector_id), + }, + timeout=30, + ) + if response.status_code >= 400: + detail = response.text[:500] + raise OAuthError(f"Token exchange failed ({response.status_code}): {detail}") + + data = response.json() + expires_in = int(data.get("expires_in", 14400)) + scopes_raw = data.get("scope", provider.scopes) + scopes = scopes_raw.split() if isinstance(scopes_raw, str) else list(scopes_raw or []) + return AuthContext( + connector_id=connector_id, + access_token=data.get("access_token"), + refresh_token=data.get("refresh_token"), + token_type=data.get("token_type", "Bearer"), + expires_at=datetime.utcnow() + timedelta(seconds=expires_in), + scopes=scopes, + ) + + def _provider(self, connector_id: str) -> OAuthProviderConfig: + if connector_id not in PROVIDER_CONFIGS: + raise OAuthError(f"OAuth not supported for connector '{connector_id}'") + return PROVIDER_CONFIGS[connector_id] + + def _client_credentials(self, connector_id: str) -> tuple[Optional[str], Optional[str]]: + provider = self._provider(connector_id) + secrets = self.security.load_connector_secrets(connector_id) + client_id = secrets.get("client_id") or os.getenv(provider.client_id_env) + client_secret = secrets.get("client_secret") or os.getenv(provider.client_secret_env) + return client_id, client_secret + + +def _generate_code_verifier() -> str: + return secrets.token_urlsafe(64)[:96] + + +def _code_challenge_s256(code_verifier: str) -> str: + digest = hashlib.sha256(code_verifier.encode("ascii")).digest() + return base64.urlsafe_b64encode(digest).rstrip(b"=").decode("ascii") diff --git a/api/collectors/core/oauth_routes.py b/api/collectors/core/oauth_routes.py new file mode 100644 index 000000000..35080dd2a --- /dev/null +++ b/api/collectors/core/oauth_routes.py @@ -0,0 +1,119 @@ +"""FastAPI routes for collector OAuth flows.""" + +from __future__ import annotations + +from typing import Optional + +from fastapi import APIRouter, HTTPException, Request +from fastapi.responses import HTMLResponse, JSONResponse, RedirectResponse + +from collectors.core.oauth_manager import OAuthError, OAuthManager +from collectors.core.registry import list_connectors + + +def _wants_json(request: Request, format_param: Optional[str]) -> bool: + if format_param == "json": + return True + accept = request.headers.get("accept", "") + return "application/json" in accept and "text/html" not in accept + + +def _success_html(connector_id: str, project_id: str) -> str: + label = connector_id.replace("_", " ").title() + return f""" + + + + Connected to {label} + + + +

Connected to {label}

+

OAuth completed for project {project_id}. You can close this window.

+

Next: POST /collect/projects/{project_id}/connect/{connector_id} or browse the remote tree.

+ +""" + + +def _error_html(message: str) -> str: + return f""" + +OAuth failed +

OAuth failed

{message}

+""" + + +def create_oauth_router(oauth_manager: OAuthManager) -> APIRouter: + router = APIRouter(tags=["collect-oauth"]) + + @router.get("/collect/oauth/{connector_id}/authorize") + async def oauth_authorize( + connector_id: str, + project_id: str, + redirect_after: Optional[str] = None, + ): + if connector_id not in list_connectors(): + raise HTTPException(status_code=404, detail=f"Unknown connector: {connector_id}") + if not oauth_manager.supports_connector(connector_id): + raise HTTPException( + status_code=400, + detail=f"OAuth not configured for connector: {connector_id}", + ) + try: + url = oauth_manager.build_authorize_url( + connector_id=connector_id, + project_id=project_id, + redirect_after=redirect_after, + ) + except OAuthError as exc: + raise HTTPException(status_code=503, detail=str(exc)) from exc + return RedirectResponse(url, status_code=302) + + @router.get("/collect/oauth/{connector_id}/callback") + async def oauth_callback( + connector_id: str, + request: Request, + code: Optional[str] = None, + state: Optional[str] = None, + error: Optional[str] = None, + error_description: Optional[str] = None, + format: Optional[str] = None, + ): + if connector_id not in list_connectors(): + raise HTTPException(status_code=404, detail=f"Unknown connector: {connector_id}") + + if error: + message = error_description or error + if _wants_json(request, format): + raise HTTPException(status_code=400, detail=message) + return HTMLResponse(_error_html(message), status_code=400) + + try: + auth, project_id, redirect_after = oauth_manager.handle_callback( + connector_id=connector_id, + code=code or "", + state=state or "", + ) + except OAuthError as exc: + if _wants_json(request, format): + raise HTTPException(status_code=400, detail=str(exc)) from exc + return HTMLResponse(_error_html(str(exc)), status_code=400) + + if redirect_after: + return RedirectResponse(redirect_after, status_code=302) + + payload = { + "status": "connected", + "project_id": project_id, + "connector_id": connector_id, + "has_refresh_token": bool(auth.refresh_token), + "scopes": auth.scopes, + } + if _wants_json(request, format): + return JSONResponse(payload) + return HTMLResponse(_success_html(connector_id, project_id)) + + return router diff --git a/api/collectors/core/orchestrator.py b/api/collectors/core/orchestrator.py new file mode 100644 index 000000000..81c2991ca --- /dev/null +++ b/api/collectors/core/orchestrator.py @@ -0,0 +1,163 @@ +"""Collection run orchestration.""" + +from __future__ import annotations + +import logging +import os +from datetime import datetime +from typing import Any, Dict, Optional + +from collectors.base.auth_context import AuthContext +from collectors.base.types import CollectionJob, CollectionJobStatus +from collectors.core.logging_manager import LoggingManager +from collectors.core.registry import get_connector_class +from collectors.core.security_manager import SecurityManager +from collectors.core.state_manager import StateManager +from collectors.ingestion.ingestion_pipeline import IngestionPipeline +from collectors.processing.processing_engine_client import ProcessingEngineClient +from collectors.storage.filesystem_store import FilesystemStore +from collectors.storage.sqlite_store import SQLiteMetadataStore + + +class Orchestrator: + """Coordinates connector auth, ingestion, and optional processing.""" + + def __init__( + self, + state_manager: Optional[StateManager] = None, + security_manager: Optional[SecurityManager] = None, + base_output_dir: Optional[str] = None, + ): + self.state = state_manager or StateManager() + self.security = security_manager or SecurityManager() + self.logger = LoggingManager.configure() + self.base_output_dir = base_output_dir or os.getenv( + "COLLECTOR_OUTPUT_BASE", "/data/input/collections" + ) + self.processing_client = ProcessingEngineClient() + + def run_collection(self, job: CollectionJob) -> CollectionJob: + connector_id = job.connector_id + if self.state.is_circuit_open(connector_id): + job.status = CollectionJobStatus.FAILED + job.error_message = "Circuit breaker open for connector" + job.completed_at = datetime.utcnow() + self.state.save_job(job) + return job + + job.status = CollectionJobStatus.RUNNING + job.started_at = datetime.utcnow() + output_dir = os.path.join(self.base_output_dir, job.job_id) + job.output_dir = output_dir + self.state.save_job(job) + + try: + auth = self.security.build_auth_context(connector_id) + auth = self.security.refresh_auth_if_needed(auth) + connector_cls = get_connector_class(connector_id) + collector = connector_cls(auth=auth, config=job.config) + + health = collector.health_check() + if not health.healthy: + raise RuntimeError(f"Connector unhealthy: {health.message}") + + filesystem = FilesystemStore(output_dir) + metadata_store = SQLiteMetadataStore( + db_path=os.path.join(output_dir, "metadata.db") + ) + pipeline = IngestionPipeline( + collector=collector, + filesystem=filesystem, + metadata_store=metadata_store, + state_manager=self.state, + logger=self.logger, + ) + + since = self._parse_since(job.config) + result = pipeline.run(since=since) + + job.items_collected = result["items_collected"] + job.items_failed = result["items_failed"] + job.status = CollectionJobStatus.COMPLETED + job.completed_at = datetime.utcnow() + self.state.record_success(connector_id, sync_token=result.get("sync_token")) + self.state.save_job(job) + + if job.auto_chain: + self._chain_processing(job, output_dir) + + LoggingManager.log_event( + self.logger, + "collection_completed", + job_id=job.job_id, + connector_id=connector_id, + items_collected=job.items_collected, + ) + except Exception as exc: + job.status = CollectionJobStatus.FAILED + job.error_message = str(exc) + job.completed_at = datetime.utcnow() + self.state.record_failure(connector_id) + self.state.save_job(job) + LoggingManager.log_event( + self.logger, + "collection_failed", + level=logging.ERROR, + job_id=job.job_id, + connector_id=connector_id, + error=str(exc), + ) + + return job + + def _chain_processing(self, job: CollectionJob, input_dir: str) -> None: + project_config = job.config.get("processing", {}) + self.processing_client.trigger_processing( + job_id=job.job_id, + input_dir=input_dir, + project_config=project_config, + ) + + def _parse_since(self, config: Dict[str, Any]) -> Optional[datetime]: + since_str = config.get("since") + if not since_str: + return None + return datetime.fromisoformat(since_str) + + def get_connector_health(self, connector_id: str) -> Dict[str, Any]: + try: + auth = self.security.build_auth_context(connector_id) + connector_cls = get_connector_class(connector_id) + collector = connector_cls(auth=auth) + health = collector.health_check() + state = self.state.get_connector_state(connector_id) + return { + "connector_id": connector_id, + "healthy": health.healthy, + "message": health.message, + "circuit_breaker": state.get("circuit_breaker_state"), + "last_successful_run_at": state.get("last_successful_run_at"), + } + except Exception as exc: + return { + "connector_id": connector_id, + "healthy": False, + "message": str(exc), + } + + def subsystem_status(self) -> Dict[str, Any]: + from collectors.core.registry import list_connectors + + connectors = list_connectors() + open_circuits = [ + cid + for cid in connectors + if self.state.is_circuit_open(cid, cooldown_seconds=0) + and self.state.get_connector_state(cid)["circuit_breaker_state"] == "open" + ] + return { + "status": "healthy" if not open_circuits else "degraded", + "registered_connectors": connectors, + "open_circuits": open_circuits, + "processing_busy": self.processing_client.is_busy(), + } diff --git a/api/collectors/core/registry.py b/api/collectors/core/registry.py new file mode 100644 index 000000000..69dc7eead --- /dev/null +++ b/api/collectors/core/registry.py @@ -0,0 +1,37 @@ +"""Connector registry with decorator-based registration.""" + +from __future__ import annotations + +from typing import Callable, Dict, List, Type + +from collectors.base.base_collector import BaseCollector + +_REGISTRY: Dict[str, Type[BaseCollector]] = {} + + +def register_connector(connector_id: str) -> Callable[[Type[BaseCollector]], Type[BaseCollector]]: + """Decorator to register a connector class by ID.""" + + def decorator(cls: Type[BaseCollector]) -> Type[BaseCollector]: + if connector_id in _REGISTRY: + raise ValueError(f"Connector already registered: {connector_id}") + _REGISTRY[connector_id] = cls + cls._registered_connector_id = connector_id # type: ignore[attr-defined] + return cls + + return decorator + + +def get_connector_class(connector_id: str) -> Type[BaseCollector]: + if connector_id not in _REGISTRY: + raise KeyError(f"Unknown connector: {connector_id}") + return _REGISTRY[connector_id] + + +def list_connectors() -> List[str]: + return sorted(_REGISTRY.keys()) + + +def clear_registry() -> None: + """Test helper to reset registry.""" + _REGISTRY.clear() diff --git a/api/collectors/core/security_manager.py b/api/collectors/core/security_manager.py new file mode 100644 index 000000000..cc929999e --- /dev/null +++ b/api/collectors/core/security_manager.py @@ -0,0 +1,163 @@ +"""Secrets management and log sanitization.""" + +from __future__ import annotations + +import json +import os +import re +from pathlib import Path +from typing import TYPE_CHECKING, Any, Callable, Dict, Optional + +from cryptography.fernet import Fernet, InvalidToken + +from collectors.base.auth_context import AuthContext + +if TYPE_CHECKING: + from collectors.core.token_store import TokenStore + + +class SecurityManager: + """Load secrets from env/files, encrypt tokens, sanitize logs.""" + + SENSITIVE_KEYS = frozenset( + { + "access_token", + "refresh_token", + "client_secret", + "password", + "api_key", + "authorization", + } + ) + + def __init__( + self, + encryption_key: Optional[str] = None, + secrets_dir: Optional[str] = None, + ): + key = encryption_key or os.getenv("COLLECTOR_ENCRYPTION_KEY") + self._fernet: Optional[Fernet] = None + if key: + self._fernet = Fernet(key.encode() if isinstance(key, str) else key) + self.secrets_dir = Path(secrets_dir or os.getenv("COLLECTOR_SECRETS_DIR", "/app/secrets")) + self._refresh_hooks: Dict[str, Callable[[AuthContext], AuthContext]] = {} + + def encrypt_token(self, plaintext: str) -> str: + if not self._fernet: + return plaintext + return self._fernet.encrypt(plaintext.encode()).decode() + + def decrypt_token(self, ciphertext: str) -> str: + if not self._fernet: + return ciphertext + try: + return self._fernet.decrypt(ciphertext.encode()).decode() + except InvalidToken as exc: + raise ValueError("Failed to decrypt token") from exc + + def get_env_secret(self, key: str, default: Optional[str] = None) -> Optional[str]: + value = os.getenv(key, default) + if value and value.startswith("enc:"): + return self.decrypt_token(value[4:]) + return value + + def load_connector_secrets(self, connector_id: str) -> Dict[str, Any]: + """Load secrets from env prefix or JSON file.""" + prefix = f"COLLECTOR_{connector_id.upper().replace('-', '_')}_" + secrets: Dict[str, Any] = {} + for env_key, env_val in os.environ.items(): + if env_key.startswith(prefix): + field = env_key[len(prefix) :].lower() + secrets[field] = self._maybe_decrypt(env_val) + + file_path = self.secrets_dir / f"{connector_id}.json" + if file_path.is_file(): + with open(file_path, encoding="utf-8") as fh: + file_secrets = json.load(fh) + for k, v in file_secrets.items(): + if isinstance(v, str): + secrets[k] = self._maybe_decrypt(v) + else: + secrets[k] = v + return secrets + + def _maybe_decrypt(self, value: str) -> str: + if value.startswith("enc:"): + return self.decrypt_token(value[4:]) + return value + + def get_connector_credentials( + self, + connector_id: str, + project_id: Optional[str] = None, + token_store: Optional["TokenStore"] = None, + ) -> Dict[str, Any]: + """Resolve credentials: project OAuth tokens first, then global secrets.""" + secrets = self.load_connector_secrets(connector_id) + if project_id and token_store: + stored = token_store.get_auth_context(project_id, connector_id, self) + if stored and (stored.access_token or stored.refresh_token): + merged = dict(secrets) + if stored.access_token: + merged["access_token"] = stored.access_token + if stored.refresh_token: + merged["refresh_token"] = stored.refresh_token + merged["_project_id"] = project_id + return merged + return secrets + + def build_auth_context( + self, + connector_id: str, + project_id: Optional[str] = None, + token_store: Optional["TokenStore"] = None, + ) -> AuthContext: + secrets = self.get_connector_credentials(connector_id, project_id, token_store) + project = secrets.pop("_project_id", None) or project_id + extra = {k: v for k, v in secrets.items() if k not in ("access_token", "refresh_token")} + if project: + extra["project_id"] = project + return AuthContext( + connector_id=connector_id, + access_token=secrets.get("access_token"), + refresh_token=secrets.get("refresh_token"), + extra=extra, + ) + + def register_refresh_hook( + self, + connector_id: str, + hook: Callable[[AuthContext], AuthContext], + ) -> None: + self._refresh_hooks[connector_id] = hook + + def refresh_auth_if_needed(self, auth: AuthContext) -> AuthContext: + if not auth.is_expired(): + return auth + hook = self._refresh_hooks.get(auth.connector_id) + if hook: + return hook(auth) + return auth + + def sanitize_for_log(self, data: Any) -> Any: + if isinstance(data, dict): + return { + k: ("***REDACTED***" if k.lower() in self.SENSITIVE_KEYS else self.sanitize_for_log(v)) + for k, v in data.items() + } + if isinstance(data, list): + return [self.sanitize_for_log(item) for item in data] + if isinstance(data, str): + return self._sanitize_string(data) + return data + + def _sanitize_string(self, text: str) -> str: + patterns = [ + (r"(Bearer\s+)[A-Za-z0-9\-._~+/]+=*", r"\1***REDACTED***"), + (r'("access_token"\s*:\s*")[^"]+(")', r"\1***REDACTED***\2"), + (r'("refresh_token"\s*:\s*")[^"]+(")', r"\1***REDACTED***\2"), + ] + result = text + for pattern, repl in patterns: + result = re.sub(pattern, repl, result) + return result diff --git a/api/collectors/core/selection_store.py b/api/collectors/core/selection_store.py new file mode 100644 index 000000000..af004cb41 --- /dev/null +++ b/api/collectors/core/selection_store.py @@ -0,0 +1,133 @@ +"""Persist per-project browse selections (not downloaded files).""" + +from __future__ import annotations + +import json +import os +import sqlite3 +import threading +from contextlib import contextmanager +from datetime import datetime +from typing import Any, Dict, List, Optional + +from collectors.base.types import BrowseItemType, SelectedItem + + +class SelectionStore: + """SQLite store keyed by (project_id, connector_id).""" + + def __init__(self, db_path: Optional[str] = None): + self.db_path = db_path or os.getenv("COLLECTOR_STATE_DB", "/app/db/collector_state.db") + self._lock = threading.Lock() + self._init_db() + + @contextmanager + def _connection(self): + conn = sqlite3.connect(self.db_path, check_same_thread=False) + conn.row_factory = sqlite3.Row + try: + yield conn + conn.commit() + finally: + conn.close() + + def _init_db(self) -> None: + os.makedirs(os.path.dirname(self.db_path), exist_ok=True) + with self._lock, self._connection() as conn: + conn.execute( + """ + CREATE TABLE IF NOT EXISTS project_selections ( + project_id TEXT NOT NULL, + connector_id TEXT NOT NULL, + items TEXT NOT NULL DEFAULT '[]', + updated_at TEXT, + PRIMARY KEY (project_id, connector_id) + ) + """ + ) + + def save_selections( + self, + project_id: str, + connector_id: str, + items: List[SelectedItem], + ) -> Dict[str, Any]: + payload = [self._item_to_dict(item) for item in items] + updated_at = datetime.utcnow().isoformat() + with self._lock, self._connection() as conn: + conn.execute( + """ + INSERT INTO project_selections (project_id, connector_id, items, updated_at) + VALUES (?, ?, ?, ?) + ON CONFLICT(project_id, connector_id) DO UPDATE SET + items = excluded.items, + updated_at = excluded.updated_at + """, + (project_id, connector_id, json.dumps(payload), updated_at), + ) + return { + "project_id": project_id, + "connector_id": connector_id, + "items": payload, + "updated_at": updated_at, + } + + def get_selections( + self, + project_id: str, + connector_id: Optional[str] = None, + ) -> List[Dict[str, Any]]: + with self._lock, self._connection() as conn: + if connector_id: + rows = conn.execute( + "SELECT * FROM project_selections WHERE project_id = ? AND connector_id = ?", + (project_id, connector_id), + ).fetchall() + else: + rows = conn.execute( + "SELECT * FROM project_selections WHERE project_id = ?", + (project_id,), + ).fetchall() + return [self._row_to_dict(row) for row in rows] + + def get_selected_items( + self, + project_id: str, + connector_id: str, + ) -> List[SelectedItem]: + rows = self.get_selections(project_id, connector_id) + if not rows: + return [] + return [self._dict_to_item(entry) for entry in rows[0]["items"]] + + @staticmethod + def _item_to_dict(item: SelectedItem) -> Dict[str, Any]: + return { + "id": item.id, + "type": item.type.value if isinstance(item.type, BrowseItemType) else item.type, + "name": item.name, + "path": item.path, + } + + @staticmethod + def _dict_to_item(data: Dict[str, Any]) -> SelectedItem: + item_type = data.get("type", "file") + if isinstance(item_type, BrowseItemType): + parsed_type = item_type + else: + parsed_type = BrowseItemType(item_type) + return SelectedItem( + id=str(data["id"]), + type=parsed_type, + name=str(data.get("name", "")), + path=str(data.get("path", "")), + ) + + @staticmethod + def _row_to_dict(row: sqlite3.Row) -> Dict[str, Any]: + return { + "project_id": row["project_id"], + "connector_id": row["connector_id"], + "items": json.loads(row["items"] or "[]"), + "updated_at": row["updated_at"], + } diff --git a/api/collectors/core/state_manager.py b/api/collectors/core/state_manager.py new file mode 100644 index 000000000..1daae2310 --- /dev/null +++ b/api/collectors/core/state_manager.py @@ -0,0 +1,275 @@ +"""SQLite-backed connector state persistence.""" + +from __future__ import annotations + +import json +import os +import sqlite3 +import threading +from contextlib import contextmanager +from datetime import datetime +from typing import Any, Dict, List, Optional + +from collectors.base.types import CircuitBreakerState, CollectionJob, CollectionJobStatus + + +class StateManager: + """Persists sync tokens, job state, and circuit breaker status.""" + + def __init__(self, db_path: Optional[str] = None): + self.db_path = db_path or os.getenv("COLLECTOR_STATE_DB", "/app/db/collector_state.db") + self._lock = threading.Lock() + self._init_db() + + @contextmanager + def _connection(self): + conn = sqlite3.connect(self.db_path, check_same_thread=False) + conn.row_factory = sqlite3.Row + try: + yield conn + conn.commit() + finally: + conn.close() + + def _init_db(self) -> None: + os.makedirs(os.path.dirname(self.db_path), exist_ok=True) + with self._lock, self._connection() as conn: + conn.executescript( + """ + CREATE TABLE IF NOT EXISTS connector_state ( + connector_id TEXT PRIMARY KEY, + last_sync_token TEXT, + last_successful_run_at TEXT, + failed_item_ids TEXT DEFAULT '[]', + circuit_breaker_state TEXT DEFAULT 'closed', + circuit_failure_count INTEGER DEFAULT 0, + circuit_opened_at TEXT + ); + CREATE TABLE IF NOT EXISTS collection_jobs ( + job_id TEXT PRIMARY KEY, + connector_id TEXT NOT NULL, + status TEXT NOT NULL, + created_at TEXT, + started_at TEXT, + completed_at TEXT, + output_dir TEXT, + items_collected INTEGER DEFAULT 0, + items_failed INTEGER DEFAULT 0, + error_message TEXT, + auto_chain INTEGER DEFAULT 0, + config TEXT DEFAULT '{}' + ); + CREATE TABLE IF NOT EXISTS schedules ( + schedule_id TEXT PRIMARY KEY, + connector_id TEXT NOT NULL, + cron_expression TEXT NOT NULL, + auto_chain INTEGER DEFAULT 0, + config TEXT DEFAULT '{}', + enabled INTEGER DEFAULT 1 + ); + """ + ) + + def get_connector_state(self, connector_id: str) -> Dict[str, Any]: + with self._lock, self._connection() as conn: + row = conn.execute( + "SELECT * FROM connector_state WHERE connector_id = ?", + (connector_id,), + ).fetchone() + if not row: + return { + "connector_id": connector_id, + "last_sync_token": None, + "last_successful_run_at": None, + "failed_item_ids": [], + "circuit_breaker_state": CircuitBreakerState.CLOSED.value, + "circuit_failure_count": 0, + } + return { + "connector_id": row["connector_id"], + "last_sync_token": row["last_sync_token"], + "last_successful_run_at": row["last_successful_run_at"], + "failed_item_ids": json.loads(row["failed_item_ids"] or "[]"), + "circuit_breaker_state": row["circuit_breaker_state"], + "circuit_failure_count": row["circuit_failure_count"], + "circuit_opened_at": row["circuit_opened_at"], + } + + def update_connector_state(self, connector_id: str, **fields: Any) -> None: + state = self.get_connector_state(connector_id) + state.update(fields) + if "failed_item_ids" in fields and isinstance(fields["failed_item_ids"], list): + state["failed_item_ids"] = fields["failed_item_ids"] + with self._lock, self._connection() as conn: + conn.execute( + """ + INSERT INTO connector_state ( + connector_id, last_sync_token, last_successful_run_at, + failed_item_ids, circuit_breaker_state, circuit_failure_count, + circuit_opened_at + ) VALUES (?, ?, ?, ?, ?, ?, ?) + ON CONFLICT(connector_id) DO UPDATE SET + last_sync_token = excluded.last_sync_token, + last_successful_run_at = excluded.last_successful_run_at, + failed_item_ids = excluded.failed_item_ids, + circuit_breaker_state = excluded.circuit_breaker_state, + circuit_failure_count = excluded.circuit_failure_count, + circuit_opened_at = excluded.circuit_opened_at + """, + ( + connector_id, + state.get("last_sync_token"), + state.get("last_successful_run_at"), + json.dumps(state.get("failed_item_ids", [])), + state.get("circuit_breaker_state", CircuitBreakerState.CLOSED.value), + state.get("circuit_failure_count", 0), + state.get("circuit_opened_at"), + ), + ) + + def record_success(self, connector_id: str, sync_token: Optional[str] = None) -> None: + self.update_connector_state( + connector_id, + last_sync_token=sync_token, + last_successful_run_at=datetime.utcnow().isoformat(), + circuit_breaker_state=CircuitBreakerState.CLOSED.value, + circuit_failure_count=0, + circuit_opened_at=None, + ) + + def record_failure(self, connector_id: str, threshold: int = 5) -> CircuitBreakerState: + state = self.get_connector_state(connector_id) + count = int(state.get("circuit_failure_count", 0)) + 1 + cb_state = CircuitBreakerState.CLOSED + opened_at = state.get("circuit_opened_at") + if count >= threshold: + cb_state = CircuitBreakerState.OPEN + opened_at = datetime.utcnow().isoformat() + self.update_connector_state( + connector_id, + circuit_failure_count=count, + circuit_breaker_state=cb_state.value, + circuit_opened_at=opened_at, + ) + return cb_state + + def is_circuit_open(self, connector_id: str, cooldown_seconds: int = 300) -> bool: + state = self.get_connector_state(connector_id) + if state["circuit_breaker_state"] != CircuitBreakerState.OPEN.value: + return False + opened_at = state.get("circuit_opened_at") + if not opened_at: + return True + elapsed = (datetime.utcnow() - datetime.fromisoformat(opened_at)).total_seconds() + if elapsed >= cooldown_seconds: + self.update_connector_state( + connector_id, + circuit_breaker_state=CircuitBreakerState.HALF_OPEN.value, + ) + return False + return True + + def add_failed_item(self, connector_id: str, item_id: str) -> None: + state = self.get_connector_state(connector_id) + failed: List[str] = list(state.get("failed_item_ids", [])) + if item_id not in failed: + failed.append(item_id) + self.update_connector_state(connector_id, failed_item_ids=failed) + + def save_job(self, job: CollectionJob) -> None: + with self._lock, self._connection() as conn: + conn.execute( + """ + INSERT INTO collection_jobs ( + job_id, connector_id, status, created_at, started_at, + completed_at, output_dir, items_collected, items_failed, + error_message, auto_chain, config + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + ON CONFLICT(job_id) DO UPDATE SET + status = excluded.status, + started_at = excluded.started_at, + completed_at = excluded.completed_at, + output_dir = excluded.output_dir, + items_collected = excluded.items_collected, + items_failed = excluded.items_failed, + error_message = excluded.error_message, + auto_chain = excluded.auto_chain, + config = excluded.config + """, + ( + job.job_id, + job.connector_id, + job.status.value, + job.created_at.isoformat() if job.created_at else None, + job.started_at.isoformat() if job.started_at else None, + job.completed_at.isoformat() if job.completed_at else None, + job.output_dir, + job.items_collected, + job.items_failed, + job.error_message, + 1 if job.auto_chain else 0, + json.dumps(job.config), + ), + ) + + def get_job(self, job_id: str) -> Optional[CollectionJob]: + with self._lock, self._connection() as conn: + row = conn.execute( + "SELECT * FROM collection_jobs WHERE job_id = ?", + (job_id,), + ).fetchone() + if not row: + return None + return CollectionJob( + job_id=row["job_id"], + connector_id=row["connector_id"], + status=CollectionJobStatus(row["status"]), + created_at=_parse_dt(row["created_at"]), + started_at=_parse_dt(row["started_at"]), + completed_at=_parse_dt(row["completed_at"]), + output_dir=row["output_dir"], + items_collected=row["items_collected"], + items_failed=row["items_failed"], + error_message=row["error_message"], + auto_chain=bool(row["auto_chain"]), + config=json.loads(row["config"] or "{}"), + ) + + def save_schedule( + self, + schedule_id: str, + connector_id: str, + cron_expression: str, + auto_chain: bool = False, + config: Optional[Dict[str, Any]] = None, + ) -> None: + with self._lock, self._connection() as conn: + conn.execute( + """ + INSERT INTO schedules (schedule_id, connector_id, cron_expression, auto_chain, config) + VALUES (?, ?, ?, ?, ?) + ON CONFLICT(schedule_id) DO UPDATE SET + connector_id = excluded.connector_id, + cron_expression = excluded.cron_expression, + auto_chain = excluded.auto_chain, + config = excluded.config + """, + ( + schedule_id, + connector_id, + cron_expression, + 1 if auto_chain else 0, + json.dumps(config or {}), + ), + ) + + def list_schedules(self) -> List[Dict[str, Any]]: + with self._lock, self._connection() as conn: + rows = conn.execute("SELECT * FROM schedules WHERE enabled = 1").fetchall() + return [dict(row) for row in rows] + + +def _parse_dt(value: Optional[str]) -> Optional[datetime]: + if not value: + return None + return datetime.fromisoformat(value) diff --git a/api/collectors/core/token_store.py b/api/collectors/core/token_store.py new file mode 100644 index 000000000..bbc3300ba --- /dev/null +++ b/api/collectors/core/token_store.py @@ -0,0 +1,212 @@ +"""Project-scoped OAuth token persistence.""" + +from __future__ import annotations + +import json +import os +import sqlite3 +import threading +from contextlib import contextmanager +from datetime import datetime +from typing import Any, Dict, Optional, TYPE_CHECKING + +from collectors.base.auth_context import AuthContext + +if TYPE_CHECKING: + from collectors.core.security_manager import SecurityManager + + +class TokenStore: + """SQLite store for OAuth tokens keyed by (project_id, connector_id).""" + + def __init__(self, db_path: Optional[str] = None): + self.db_path = db_path or os.getenv("COLLECTOR_STATE_DB", "/app/db/collector_state.db") + self._lock = threading.Lock() + self._init_db() + + @contextmanager + def _connection(self): + conn = sqlite3.connect(self.db_path, check_same_thread=False) + conn.row_factory = sqlite3.Row + try: + yield conn + conn.commit() + finally: + conn.close() + + def _init_db(self) -> None: + os.makedirs(os.path.dirname(self.db_path), exist_ok=True) + with self._lock, self._connection() as conn: + conn.executescript( + """ + CREATE TABLE IF NOT EXISTS oauth_tokens ( + project_id TEXT NOT NULL, + connector_id TEXT NOT NULL, + access_token TEXT, + refresh_token TEXT, + token_type TEXT DEFAULT 'Bearer', + expires_at TEXT, + scopes TEXT DEFAULT '[]', + updated_at TEXT, + PRIMARY KEY (project_id, connector_id) + ); + CREATE TABLE IF NOT EXISTS oauth_states ( + state TEXT PRIMARY KEY, + project_id TEXT NOT NULL, + connector_id TEXT NOT NULL, + code_verifier TEXT NOT NULL, + redirect_after TEXT, + expires_at TEXT NOT NULL, + created_at TEXT NOT NULL + ); + """ + ) + + def save_auth_context( + self, + project_id: str, + connector_id: str, + auth: AuthContext, + security: SecurityManager, + ) -> None: + access = self._encrypt_field(auth.access_token, security) + refresh = self._encrypt_field(auth.refresh_token, security) + updated_at = datetime.utcnow().isoformat() + with self._lock, self._connection() as conn: + conn.execute( + """ + INSERT INTO oauth_tokens ( + project_id, connector_id, access_token, refresh_token, + token_type, expires_at, scopes, updated_at + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?) + ON CONFLICT(project_id, connector_id) DO UPDATE SET + access_token = excluded.access_token, + refresh_token = excluded.refresh_token, + token_type = excluded.token_type, + expires_at = excluded.expires_at, + scopes = excluded.scopes, + updated_at = excluded.updated_at + """, + ( + project_id, + connector_id, + access, + refresh, + auth.token_type, + auth.expires_at.isoformat() if auth.expires_at else None, + json.dumps(auth.scopes), + updated_at, + ), + ) + + def get_auth_context( + self, + project_id: str, + connector_id: str, + security: SecurityManager, + ) -> Optional[AuthContext]: + with self._lock, self._connection() as conn: + row = conn.execute( + """ + SELECT * FROM oauth_tokens + WHERE project_id = ? AND connector_id = ? + """, + (project_id, connector_id), + ).fetchone() + if not row: + return None + access = self._decrypt_field(row["access_token"], security) + refresh = self._decrypt_field(row["refresh_token"], security) + if not access and not refresh: + return None + expires_at = None + if row["expires_at"]: + expires_at = datetime.fromisoformat(row["expires_at"]) + scopes = json.loads(row["scopes"] or "[]") + return AuthContext( + connector_id=connector_id, + access_token=access, + refresh_token=refresh, + token_type=row["token_type"] or "Bearer", + expires_at=expires_at, + scopes=scopes, + extra={"project_id": project_id}, + ) + + def has_tokens(self, project_id: str, connector_id: str) -> bool: + with self._lock, self._connection() as conn: + row = conn.execute( + """ + SELECT access_token, refresh_token FROM oauth_tokens + WHERE project_id = ? AND connector_id = ? + """, + (project_id, connector_id), + ).fetchone() + if not row: + return False + return bool(row["access_token"] or row["refresh_token"]) + + def save_oauth_state( + self, + state: str, + project_id: str, + connector_id: str, + code_verifier: str, + expires_at: datetime, + redirect_after: Optional[str] = None, + ) -> None: + now = datetime.utcnow().isoformat() + with self._lock, self._connection() as conn: + conn.execute( + """ + INSERT INTO oauth_states ( + state, project_id, connector_id, code_verifier, + redirect_after, expires_at, created_at + ) VALUES (?, ?, ?, ?, ?, ?, ?) + """, + ( + state, + project_id, + connector_id, + code_verifier, + redirect_after, + expires_at.isoformat(), + now, + ), + ) + + def consume_oauth_state(self, state: str, connector_id: str) -> Optional[Dict[str, Any]]: + with self._lock, self._connection() as conn: + row = conn.execute( + "SELECT * FROM oauth_states WHERE state = ? AND connector_id = ?", + (state, connector_id), + ).fetchone() + if not row: + return None + conn.execute("DELETE FROM oauth_states WHERE state = ?", (state,)) + expires_at = datetime.fromisoformat(row["expires_at"]) + if datetime.utcnow() > expires_at: + return None + return { + "project_id": row["project_id"], + "connector_id": row["connector_id"], + "code_verifier": row["code_verifier"], + "redirect_after": row["redirect_after"], + } + + @staticmethod + def _encrypt_field(value: Optional[str], security: SecurityManager) -> Optional[str]: + if not value: + return None + encrypted = security.encrypt_token(value) + if encrypted != value: + return f"enc:{encrypted}" + return value + + @staticmethod + def _decrypt_field(value: Optional[str], security: SecurityManager) -> Optional[str]: + if not value: + return None + if value.startswith("enc:"): + return security.decrypt_token(value[4:]) + return value diff --git a/api/collectors/core/trigger_manager.py b/api/collectors/core/trigger_manager.py new file mode 100644 index 000000000..e437d1d86 --- /dev/null +++ b/api/collectors/core/trigger_manager.py @@ -0,0 +1,119 @@ +"""Manual and scheduled collection triggers.""" + +from __future__ import annotations + +import logging +import threading +import uuid +from datetime import datetime +from typing import Any, Callable, Dict, Optional + +from apscheduler.schedulers.background import BackgroundScheduler +from apscheduler.triggers.cron import CronTrigger + +from collectors.base.types import CollectionJob, CollectionJobStatus +from collectors.core.logging_manager import LoggingManager +from collectors.core.state_manager import StateManager + + +class TriggerManager: + """Handles manual triggers and APScheduler cron jobs.""" + + def __init__( + self, + state_manager: StateManager, + run_callback: Callable[[CollectionJob], None], + ): + self.state = state_manager + self.run_callback = run_callback + self.logger = LoggingManager.configure() + self._scheduler = BackgroundScheduler() + self._scheduler.start() + self._shutdown = threading.Event() + + def trigger_manual( + self, + connector_id: str, + auto_chain: bool = False, + config: Optional[Dict[str, Any]] = None, + ) -> CollectionJob: + job = CollectionJob( + job_id=str(uuid.uuid4()), + connector_id=connector_id, + status=CollectionJobStatus.PENDING, + created_at=datetime.utcnow(), + auto_chain=auto_chain, + config=config or {}, + ) + self.state.save_job(job) + if not self._shutdown.is_set(): + thread = threading.Thread(target=self._execute_job, args=(job,), daemon=True) + thread.start() + return job + + def _execute_job(self, job: CollectionJob) -> None: + try: + self.run_callback(job) + except Exception as exc: + LoggingManager.log_event( + self.logger, + "trigger_job_failed", + level=logging.ERROR, + job_id=job.job_id, + error=str(exc), + ) + + def add_schedule( + self, + schedule_id: str, + connector_id: str, + cron_expression: str, + auto_chain: bool = False, + config: Optional[Dict[str, Any]] = None, + ) -> None: + parts = cron_expression.split() + if len(parts) != 5: + raise ValueError("Cron expression must have 5 fields: min hour dom month dow") + minute, hour, day, month, day_of_week = parts + trigger = CronTrigger( + minute=minute, + hour=hour, + day=day, + month=month, + day_of_week=day_of_week, + ) + self.state.save_schedule(schedule_id, connector_id, cron_expression, auto_chain, config) + + def _scheduled_run() -> None: + if self._shutdown.is_set(): + return + self.trigger_manual(connector_id, auto_chain=auto_chain, config=config) + + self._scheduler.add_job( + _scheduled_run, + trigger=trigger, + id=schedule_id, + replace_existing=True, + ) + LoggingManager.log_event( + self.logger, + "schedule_registered", + schedule_id=schedule_id, + connector_id=connector_id, + cron=cron_expression, + ) + + def load_schedules_from_db(self) -> None: + for row in self.state.list_schedules(): + self.add_schedule( + schedule_id=row["schedule_id"], + connector_id=row["connector_id"], + cron_expression=row["cron_expression"], + auto_chain=bool(row["auto_chain"]), + config=__import__("json").loads(row["config"] or "{}"), + ) + + def shutdown(self) -> None: + self._shutdown.set() + if self._scheduler.running: + self._scheduler.shutdown(wait=False) diff --git a/api/collectors/docs/architecture.md b/api/collectors/docs/architecture.md new file mode 100644 index 000000000..09b27a7fe --- /dev/null +++ b/api/collectors/docs/architecture.md @@ -0,0 +1,40 @@ +# External Data Collection Framework + +FreeEed collectors ingest documents from cloud storage providers into `/data/input/collections//` for optional Java processing. + +## Components + +| Layer | Module | Responsibility | +|-------|--------|----------------| +| Base | `base/base_collector.py` | Connector ABC | +| Core | `core/orchestrator.py` | Run coordination | +| Core | `core/trigger_manager.py` | Manual + cron triggers | +| Core | `core/security_manager.py` | Secrets + token encryption | +| Core | `core/state_manager.py` | SQLite sync state | +| Ingestion | `ingestion/ingestion_pipeline.py` | list → fetch → store | +| Storage | `storage/filesystem_store.py` | Item files | +| Processing | `processing/processing_engine_client.py` | `.project` + Java chain | + +## Data Flow + +1. Trigger (API, CLI, or cron) creates a `CollectionJob` +2. Orchestrator authenticates connector and runs ingestion pipeline +3. Files land under `/data/input/collections//` +4. Metadata stored in job-local SQLite (`metadata.db`) +5. If `auto_chain=true`, a Java-canonical `.project` file is written and `run_freeeed_process()` is invoked + +## State Database + +Global state: `COLLECTOR_STATE_DB` (default `/app/db/collector_state.db`) + +Tracks per-connector sync tokens, failed items, and circuit breaker status. + +## Security + +- Secrets via `COLLECTOR__*` env vars or `/app/secrets/.json` +- Optional Fernet encryption with `COLLECTOR_ENCRYPTION_KEY` +- Structured logs sanitize tokens automatically + +## Shutdown + +`TriggerManager.shutdown()` stops APScheduler gracefully on SIGTERM/SIGINT (see `scripts/run_collectors.py`). diff --git a/api/collectors/docs/browse-and-select.md b/api/collectors/docs/browse-and-select.md new file mode 100644 index 000000000..174093d03 --- /dev/null +++ b/api/collectors/docs/browse-and-select.md @@ -0,0 +1,249 @@ +# Browse & Select API (Project-Scoped) + +This document describes the API layer for remote browse/select workflows. A future Swing or web UI should call these endpoints; no Swing changes are included in this phase. + +## Project / Case Mapping + +FreeEed cases are defined by `.project` files (Java `Project` / `ParameterProcessing`): + +| `.project` key | Role | +|----------------|------| +| `project-code` | Stable case identifier (e.g. `1`, `0002`) — use as `project_id` in API paths | +| `project-name` | Human-readable case name | +| `custodian` | Comma-separated custodian list for collected evidence | + +When collection is triggered with `auto_chain=true`, the orchestrator writes a `.project` file via `ProcessingEngineClient`, passing `project_code`, `project_name`, and `custodian` from `config.processing`. + +**Selections are keyed by `(project_id, connector_id)`** in SQLite (`project_selections` table). They store remote item references only — files are downloaded at collection run time into `/data/input/collections/{job_id}/`. + +## Java UI Integration Hook + +Swing screens under `freeeed-processing/src/main/java/org/freeeed/ui/` do not yet call these endpoints. Recommended flow for a future UI: + +1. Read `project-code` from the active `.project` file → `project_id` +2. `GET /collect/oauth/{connector_id}/authorize?project_id={project_id}` — browser OAuth (Dropbox first) +3. `POST /collect/projects/{project_id}/connect/{connector_id}` — verify stored tokens +4. `GET /collect/projects/{project_id}/browse/{connector_id}?parent_id=...` — populate tree +5. `POST /collect/projects/{project_id}/selections` — persist user picks +6. `POST /collect/projects/{project_id}/trigger` — collect selected items only + +Use `http://localhost:8000` (or deployed API base URL) from Java via HTTP client. + +## API Endpoints + +### OAuth (Dropbox v1) + +Per-project tokens are stored in SQLite (`oauth_tokens` table) after a successful callback. Global `secrets/dropbox.json` tokens remain supported as a fallback when no project token exists. + +| Method | Path | Purpose | +|--------|------|---------| +| GET | `/collect/oauth/{connector_id}/authorize` | Query: `project_id`, optional `redirect_after`. Redirects to provider with PKCE. | +| GET | `/collect/oauth/{connector_id}/callback` | Provider callback; exchanges code, stores tokens. HTML success page or `?format=json`. | + +**Dropbox OAuth flow (dev):** + +1. Ensure `secrets/dropbox.json` has `client_id` and `client_secret` (no short-lived access token required). +2. Register redirect URI in [Dropbox App Console](https://www.dropbox.com/developers/apps): + `http://localhost:8000/collect/oauth/dropbox/callback` + (or `http://localhost:4001/collect/oauth/dropbox/callback` if you run uvicorn on port 4001 and set `COLLECTOR_OAUTH_REDIRECT_BASE=http://localhost:4001`). +3. Open in a browser: + `http://localhost:8000/collect/oauth/dropbox/authorize?project_id=1` +4. Approve access in Dropbox; callback stores refresh + access tokens for project `1`. +5. Connect and browse use stored tokens automatically: + +```bash +curl -s -X POST http://localhost:8000/collect/projects/1/connect/dropbox +curl -s "http://localhost:8000/collect/projects/1/browse/dropbox" +``` + +Environment: + +| Variable | Purpose | +|----------|---------| +| `COLLECTOR_OAUTH_REDIRECT_BASE` | Base URL for callback (default `http://localhost:8000`) | +| `DROPBOX_CLIENT_ID` / `DROPBOX_CLIENT_SECRET` | App credentials (or `secrets/dropbox.json`) | +| `COLLECTOR_ENCRYPTION_KEY` | Optional Fernet key to encrypt tokens at rest | + +### Connect + +``` +POST /collect/projects/{project_id}/connect/{connector_id} +``` + +Optional body: + +```json +{ + "access_token": "dev-token", + "refresh_token": "dev-refresh", + "config": { "mock_refresh": true } +} +``` + +Responses: + +- `200` — `{ "project_id", "connector_id", "connected": true, "message" }` +- `503` — credentials not configured +- `401` — authentication failed + +### Browse + +``` +GET /collect/projects/{project_id}/browse/{connector_id}?parent_id={optional} +``` + +Optional query params: `access_token`, `refresh_token` (dev overrides). + +Response: + +```json +{ + "project_id": "1", + "connector_id": "box", + "parent_id": "0", + "items": [ + { + "id": "123", + "name": "Contracts", + "path": "/Contracts", + "type": "folder", + "size": null, + "modified_at": "2024-01-15T10:00:00", + "connector_id": "box", + "mime_type": null + } + ] +} +``` + +Root parent IDs: `box` → `0` or omit; `google_drive` → `root` or omit; `dropbox` → `""` or omit (path-based). + +### Save Selections + +``` +POST /collect/projects/{project_id}/selections +``` + +```json +{ + "connector_id": "box", + "items": [ + { "id": "folder-1", "type": "folder", "name": "Contracts", "path": "/Contracts" }, + { "id": "file-9", "type": "file", "name": "memo.pdf", "path": "/memo.pdf" } + ] +} +``` + +### Get Selections + +``` +GET /collect/projects/{project_id}/selections?connector_id=box +``` + +### Trigger Selected Collection + +``` +POST /collect/projects/{project_id}/trigger +``` + +```json +{ + "connector_id": "box", + "auto_chain": true, + "config": { + "processing": { + "project_code": "1", + "project_name": "Enron review", + "custodian": "jsmith" + } + } +} +``` + +Loads saved selections for `(project_id, connector_id)`, sets `config.selected_items`, and starts a background job. Folders are expanded recursively at run time. + +## Dev Secrets + +Use OAuth for long-lived refresh tokens, or environment variables / mounted JSON for manual dev tokens. + +### Environment Variables + +| Connector | Token vars | App creds | +|-----------|------------|-----------| +| `box` | `COLLECTOR_BOX_ACCESS_TOKEN`, `COLLECTOR_BOX_REFRESH_TOKEN` | `BOX_CLIENT_ID`, `BOX_CLIENT_SECRET` | +| `google_drive` | `COLLECTOR_GOOGLE_DRIVE_ACCESS_TOKEN`, `COLLECTOR_GOOGLE_DRIVE_REFRESH_TOKEN` | `GOOGLE_DRIVE_CLIENT_ID`, `GOOGLE_DRIVE_CLIENT_SECRET` | +| `dropbox` | `COLLECTOR_DROPBOX_ACCESS_TOKEN`, `COLLECTOR_DROPBOX_REFRESH_TOKEN` | `DROPBOX_CLIENT_ID`, `DROPBOX_CLIENT_SECRET` | + +Set `COLLECTOR_SECRETS_DIR=./secrets` to load file-based secrets. + +### `./secrets/box.json` Example + +```json +{ + "access_token": "PLACEHOLDER_ACCESS_TOKEN", + "refresh_token": "PLACEHOLDER_REFRESH_TOKEN", + "client_id": "your-box-app-client-id", + "client_secret": "your-box-app-client-secret" +} +``` + +Equivalent files: `./secrets/google_drive.json`, `./secrets/dropbox.json`. + +Copy `secrets/dropbox.json.example` → `secrets/dropbox.json` and fill in values. Real JSON files are gitignored; only `*.example` templates are committed. + +### Dropbox Dev Setup (FreeEed.Collector) + +1. In the [Dropbox App Console](https://www.dropbox.com/developers/apps), open app **FreeEed.Collector**. +2. Enable scopes: **`files.metadata.read`**, **`files.content.read`**, **`account_info.read`**. +3. Add redirect URI matching your API base, e.g. + `http://localhost:8000/collect/oauth/dropbox/callback` + (existing `http://localhost:4001` entries must include the full path if using port 4001). +4. Create `secrets/dropbox.json` with app key/secret only: + +```json +{ + "client_id": "YOUR_APP_KEY", + "client_secret": "YOUR_APP_SECRET" +} +``` + +5. Start the API (`docker compose up freeeed-api` or local uvicorn on port 8000/4001). +6. Complete OAuth in the browser (step 3 in **OAuth** section above). + +For local Python runs outside Docker, export `COLLECTOR_SECRETS_DIR=./secrets` and `COLLECTOR_OAUTH_REDIRECT_BASE` (see `secrets/.env.local.example`). + +Browse root uses an empty `parent_id` for Dropbox (path-based). Subfolders: `?parent_id=/FolderName`. + +### Mock Mode (tests / no creds) + +Pass tokens in the connect/browse request body or query string, with `config.mock_refresh: true` and `config.mock_browse` / `config.mock_items` on the connector for unit tests. + +## Error Handling + +| Code | Meaning | +|------|---------| +| 503 | No secrets configured for connector | +| 401 | Token invalid or refresh failed | +| 400 | No selections saved before trigger | +| 404 | Unknown `connector_id` or job | + +## M9 UI (planned) + +A dedicated **Collection Selection UI** (milestone **M9**) will provide an interactive, case-scoped experience on top of the endpoints documented above. See [`m9-collection-ui.md`](m9-collection-ui.md) for the design spec. + +**Planned capabilities:** + +- Lazy folder tree with breadcrumbs and multi-select (folders and files) +- In-app OAuth via WebView or embedded browser (`GET /collect/oauth/{connector_id}/authorize`) +- Project/custodian binding from the active `.project` file +- Selection validation before `POST /collect/projects/{project_id}/trigger` +- Job status polling via `GET /collect/jobs/{job_id}` + +**UI options (both documented; web is the default recommendation):** + +| Option | Location | Notes | +|--------|----------|-------| +| **Web UI (recommended)** | `api/static/collectors/` served at `/collectors/` | Fastest path for tree rendering and OAuth redirects; ships with the FastAPI container | +| **Swing panel** | `freeeed-processing` Java UI | Reuses the same REST API; suitable when desktop-only workflow is required | + +Placeholder scaffold: `api/static/collectors/index.html` — minimal browse call against this API; full implementation tracked under M9. diff --git a/api/collectors/docs/connectors.md b/api/collectors/docs/connectors.md new file mode 100644 index 000000000..f8bc1e4b8 --- /dev/null +++ b/api/collectors/docs/connectors.md @@ -0,0 +1,44 @@ +# Connectors (v1) + +## Registered Connectors + +| ID | Provider | Auth | Env Prefix | +|----|----------|------|------------| +| `box` | Box.com | OAuth2 | `COLLECTOR_BOX_*` | +| `google_drive` | Google Drive | OAuth2 | `COLLECTOR_GOOGLE_DRIVE_*` | +| `dropbox` | Dropbox | OAuth2 | `COLLECTOR_DROPBOX_*` | +| `template` | Example stub | API token | — | + +## Required Secrets + +Each connector expects at minimum: + +- `access_token` — current OAuth access token +- `refresh_token` — for automatic refresh (recommended) + +Provider app credentials (shared across jobs): + +- Box: `BOX_CLIENT_ID`, `BOX_CLIENT_SECRET` +- Google Drive: `GOOGLE_DRIVE_CLIENT_ID`, `GOOGLE_DRIVE_CLIENT_SECRET` +- Dropbox: `DROPBOX_CLIENT_ID`, `DROPBOX_CLIENT_SECRET` + +Dropbox browse/collect requires app scopes **`files.metadata.read`** and **`files.content.read`**. See [browse-and-select.md](./browse-and-select.md#dropbox-dev-setup-freeeedcollector) for dev setup. + +Job `config` JSON may include: + +- `folder_id` / `path` — scope collection to a folder +- `since` — ISO datetime for incremental listing +- `selected_items` — list of `{id, type, name, path}` for browse/select collection +- `selected_item_ids` / `selected_paths` — shorthand file-only selection +- `mock_items` / `mock_browse` — test-only inline item trees +- `processing` — auto-chain project settings (when `auto_chain=true`) + +See [browse-and-select.md](./browse-and-select.md) for project-scoped browse/select API and dev secret setup. + +## Rate Limits + +Connectors honor `Retry-After` headers. Repeated failures open a per-connector circuit breaker (default threshold: 5). + +## Adding a Connector + +Copy `template/new_connector_template.py`, implement all ABC methods, decorate with `@register_connector("your_id")`, and import the module in `main.py`. diff --git a/api/collectors/docs/ingestion_flow.md b/api/collectors/docs/ingestion_flow.md new file mode 100644 index 000000000..6aa6c91bf --- /dev/null +++ b/api/collectors/docs/ingestion_flow.md @@ -0,0 +1,58 @@ +# Ingestion Flow + +## Pipeline Stages + +```mermaid +flowchart LR + A[Trigger] --> B[Orchestrator] + B --> C[Authenticate] + C --> D[Discover Items] + D --> E[Fetch Concurrent] + E --> F[Normalize Metadata] + F --> G[Filesystem Store] + G --> H[SQLite Metadata] + H --> I{auto_chain?} + I -->|yes| J[Write .project] + J --> K[run_freeeed_process] + I -->|no| L[Done] +``` + +## Idempotency + +Items are skipped when the same `item_id` + `checksum` already exists in the job metadata store. Writes use temp files + atomic rename. + +## Concurrency + +`StreamingEngine` fetches with a thread pool (default 4 workers). Rate-limit responses trigger backoff via `Retry-After`. + +## Metadata Normalization + +`MetadataNormalizer` produces canonical `ItemMetadata`: + +- Sanitized filenames +- SHA-256 content checksums +- Connector ID in `extra.connector_id` + +## Output Layout + +``` +/data/input/collections// + doc1.pdf + doc2.txt + metadata.db +``` + +## Processing Chain + +When enabled, `ProcessingEngineClient` writes keys matching Java expectations: + +``` +input=/data/input/collections/ +output-dir=/data/output +custodian=collector +stage=true +solr_endpoint=http://solr:8983 +processing_engine=Standard +``` + +Note: uses `input=` (not `input-dir`) per FreeEed canonical format. diff --git a/api/collectors/docs/m9-collection-ui.md b/api/collectors/docs/m9-collection-ui.md new file mode 100644 index 000000000..4be5801bf --- /dev/null +++ b/api/collectors/docs/m9-collection-ui.md @@ -0,0 +1,124 @@ +# M9 — Collection Selection UI (Design Spec) + +Interactive, case-scoped UI for browsing external sources (Box, Google Drive, Dropbox), selecting folders/files, and triggering collection jobs. Builds on the browse/select REST API documented in [`browse-and-select.md`](browse-and-select.md). + +## Goals + +- Let reviewers pick evidence from cloud connectors **in the context of a FreeEed case** (`project_id` = `.project` `project-code`). +- Support OAuth without leaving the workflow (embedded browser / WebView). +- Lazy-load large folder trees; multi-select folders and files with clear validation before collection. +- Surface collection job status after trigger. + +## UI options + +| Option | Path | Recommendation | +|--------|------|----------------| +| **Web UI** | `api/static/collectors/` → `/collectors/` via FastAPI `StaticFiles` | **Default** — faster tree UX, OAuth redirects, single Docker image | +| **Swing panel** | `freeeed-processing/src/main/java/org/freeeed/ui/` | Alternative for desktop-only installs; same REST API | + +This milestone implements the **web UI** first unless a desktop-only constraint is explicit. + +## Screens + +### 1. Case context bar + +- Displays `project_id`, `project_name`, `custodian` (from active `.project` or query params). +- Connector tabs: Box | Google Drive | Dropbox. +- Connection badge: connected / not connected / OAuth required. + +### 2. OAuth gate + +When `POST /collect/projects/{project_id}/connect/{connector_id}` returns `503` or `401`: + +- Show **Connect** button → opens `/collect/oauth/{connector_id}/authorize?project_id={project_id}&redirect_after=/collectors/?project_id={project_id}` in same window or popup. +- On callback success, return to browser with tokens stored server-side. + +### 3. Browse tree + +- **Breadcrumbs** from root to current `parent_id`. +- **Lazy children**: `GET /collect/projects/{project_id}/browse/{connector_id}?parent_id=...` +- Folder click → navigate; checkbox → add/remove from selection set. +- File rows: name, size, modified date, type icon. + +Root parent IDs: `box` → `0`; `google_drive` → `root`; `dropbox` → `""` (path-based). + +### 4. Selection panel + +- List of checked items (id, name, path, type). +- **Save selections** → `POST /collect/projects/{project_id}/selections` +- **Load saved** → `GET /collect/projects/{project_id}/selections?connector_id=...` +- Validation: at least one item; warn on empty folders-only if policy requires files. + +### 5. Collect & status + +- **Custodian** field (pre-filled from `.project`). +- **Auto-chain** toggle (default from case settings). +- **Start collection** → `POST /collect/projects/{project_id}/trigger` with `processing.project_code`, `project_name`, `custodian`. +- Poll `GET /collect/jobs/{job_id}` until terminal state; show manifest summary. + +## API mapping + +| UI action | Endpoint | +|-----------|----------| +| List connectors | `GET /collect/connectors` | +| Connect / verify auth | `POST /collect/projects/{project_id}/connect/{connector_id}` | +| Start OAuth | `GET /collect/oauth/{connector_id}/authorize?project_id=...` | +| List folder children | `GET /collect/projects/{project_id}/browse/{connector_id}?parent_id=...` | +| Save picks | `POST /collect/projects/{project_id}/selections` | +| Load picks | `GET /collect/projects/{project_id}/selections?connector_id=...` | +| Trigger job | `POST /collect/projects/{project_id}/trigger` | +| Job status | `GET /collect/jobs/{job_id}` | + +## OAuth flow (web) + +```mermaid +sequenceDiagram + participant UI as Collection Browser + participant API as FastAPI + participant IdP as Box/Drive/Dropbox + + UI->>API: GET /collect/oauth/{id}/authorize?project_id= + API->>IdP: Redirect with PKCE + IdP->>API: GET /collect/oauth/{id}/callback?code= + API->>API: Exchange code, store tokens (project scoped) + API->>UI: Redirect to /collectors/?project_id= + UI->>API: POST /connect/{id} + API-->>UI: connected: true +``` + +## Component breakdown (web) + +| Component | Responsibility | +|-----------|----------------| +| `CaseContext` | project_id, custodian, connector tab state | +| `ConnectorStatus` | connect/OAuth CTA, health badge | +| `BreadcrumbNav` | path stack, navigate to ancestor | +| `FolderTree` | lazy fetch, expand/collapse, loading/error states | +| `SelectionList` | checked items, save/load API | +| `CollectDialog` | custodian, auto_chain, trigger + poll job | +| `api.js` | thin fetch wrapper for `/collect/*` | + +Vanilla JS for v1 scaffold (`api/static/collectors/index.html`); consider a small framework only if complexity warrants it. + +## Static mount + +FastAPI serves assets from `api/static/collectors/` at `/collectors/` (see `api/main.py`). Placeholder HTML demonstrates a single browse call; full tree/OAuth UI is implemented incrementally under M9. + +## Swing alternative (documented, not M9 default) + +1. Read `project-code` from active `.project`. +2. Use `HttpClient` against the same endpoints (base URL from config, default `http://localhost:8000`). +3. OAuth: open system browser or `JEditorPane` / JavaFX `WebView` for authorize URL. +4. `JTree` + lazy `TreeExpansionListener` calling browse API per node. + +## Out of scope (M9 v1) + +- Real-time sync / delta badges on tree nodes +- Bulk rename or cloud-side operations +- Mobile-responsive polish beyond functional layout + +## Related docs + +- [`browse-and-select.md`](browse-and-select.md) — REST contract +- [`architecture.md`](architecture.md) — collector subsystem overview +- [`docs/collector-framework/traceability.md`](../../../docs/collector-framework/traceability.md) — M9 milestone tracking diff --git a/api/collectors/dropbox/__init__.py b/api/collectors/dropbox/__init__.py new file mode 100644 index 000000000..1dc9f23d9 --- /dev/null +++ b/api/collectors/dropbox/__init__.py @@ -0,0 +1,5 @@ +"""Dropbox connector package.""" + +from .dropbox_collector import DropboxCollector + +__all__ = ["DropboxCollector"] diff --git a/api/collectors/dropbox/dropbox_auth.py b/api/collectors/dropbox/dropbox_auth.py new file mode 100644 index 000000000..e5a729034 --- /dev/null +++ b/api/collectors/dropbox/dropbox_auth.py @@ -0,0 +1,61 @@ +"""Dropbox OAuth2 authentication.""" + +from __future__ import annotations + +import os +from datetime import datetime, timedelta +from typing import Any, Dict, Optional + +import requests + +from collectors.base.auth_context import AuthContext + + +class DropboxAuth: + TOKEN_URL = "https://api.dropboxapi.com/oauth2/token" + + def __init__(self, auth: AuthContext, config: Optional[Dict[str, Any]] = None): + self.auth = auth + self.config = config or {} + self.client_id = self.config.get("client_id") or os.getenv("DROPBOX_CLIENT_ID") + self.client_secret = self.config.get("client_secret") or os.getenv("DROPBOX_CLIENT_SECRET") + + def ensure_valid_token(self) -> AuthContext: + if not self.auth.is_expired(): + return self.auth + if not self.auth.refresh_token: + if self.auth.access_token: + return self.auth + raise ValueError("Dropbox credentials missing") + return self.refresh_token() + + def refresh_token(self) -> AuthContext: + if self.config.get("mock_refresh"): + self.auth.access_token = "mock-dropbox-token" + self.auth.expires_at = datetime.utcnow() + timedelta(hours=1) + return self.auth + + response = requests.post( + self.TOKEN_URL, + data={ + "grant_type": "refresh_token", + "refresh_token": self.auth.refresh_token, + "client_id": self.client_id, + "client_secret": self.client_secret, + }, + timeout=30, + ) + if response.status_code >= 400: + raise RuntimeError(f"Dropbox token refresh failed: {response.status_code}") + data = response.json() + self.auth.access_token = data["access_token"] + if "refresh_token" in data: + self.auth.refresh_token = data["refresh_token"] + expires_in = int(data.get("expires_in", 14400)) + self.auth.expires_at = datetime.utcnow() + timedelta(seconds=expires_in) + return self.auth + + def build_client(self, auth: AuthContext): + import dropbox + + return dropbox.Dropbox(oauth2_access_token=auth.access_token) diff --git a/api/collectors/dropbox/dropbox_collector.py b/api/collectors/dropbox/dropbox_collector.py new file mode 100644 index 000000000..ada78d8fc --- /dev/null +++ b/api/collectors/dropbox/dropbox_collector.py @@ -0,0 +1,171 @@ +"""Dropbox connector.""" + +from __future__ import annotations + +from datetime import datetime +from typing import Any, Dict, Iterator, List, Optional + +from collectors.base.auth_context import AuthContext +from collectors.base.base_collector import BaseCollector +from collectors.base.types import BrowseItem, BrowseItemType, CollectionItem, ItemMetadata, RateLimitInfo +from collectors.core.registry import register_connector + +from .dropbox_auth import DropboxAuth + + +@register_connector("dropbox") +class DropboxCollector(BaseCollector): + """Collect files from Dropbox using OAuth2.""" + + def __init__(self, auth: AuthContext, config: Optional[Dict[str, Any]] = None): + super().__init__(auth, config) + self._auth_client = DropboxAuth(auth, config) + self._client = None + self._mock_items = config.get("mock_items") if config else None + self._mock_browse = config.get("mock_browse") if config else None + + @property + def connector_id(self) -> str: + return "dropbox" + + def authenticate(self) -> AuthContext: + self.auth = self._auth_client.ensure_valid_token() + if self._mock_items is None and self._mock_browse is None: + self._client = self._auth_client.build_client(self.auth) + return self.auth + + def list_children(self, parent_id: Optional[str] = None) -> List[BrowseItem]: + self.authenticate() + path = parent_id if parent_id is not None else "" + if self._mock_browse is not None: + return [self._dict_to_browse_item(entry) for entry in self._mock_browse.get(path, [])] + + result = self._client.files_list_folder(path or "") + entries = list(result.entries) + while result.has_more: + result = self._client.files_list_folder_continue(result.cursor) + entries.extend(result.entries) + + items: List[BrowseItem] = [] + for entry in entries: + is_folder = entry.__class__.__name__ == "FolderMetadata" + item_type = BrowseItemType.FOLDER if is_folder else BrowseItemType.FILE + display_path = getattr(entry, "path_display", "") or "" + modified = getattr(entry, "server_modified", None) or getattr(entry, "client_modified", None) + modified_at = modified if isinstance(modified, datetime) else None + items.append( + BrowseItem( + id=display_path, + name=getattr(entry, "name", display_path), + path=display_path, + type=item_type, + connector_id=self.connector_id, + size=getattr(entry, "size", None), + modified_at=modified_at, + ) + ) + return items + + def list_items(self, since: Optional[datetime] = None) -> List[ItemMetadata]: + self.authenticate() + if self._mock_items is not None: + items = [self._to_metadata(item) for item in self._mock_items] + if since: + items = [m for m in items if not m.modified_at or m.modified_at >= since] + return items + + path = self.config.get("path", "") + result = self._client.files_list_folder(path, recursive=True) + entries = list(result.entries) + while result.has_more: + result = self._client.files_list_folder_continue(result.cursor) + entries.extend(result.entries) + + items = [] + for entry in entries: + if entry.__class__.__name__ != "FileMetadata": + continue + meta = self._entry_to_metadata(entry) + if since and meta.modified_at and meta.modified_at < since: + continue + items.append(meta) + return items + + def fetch_item(self, item_id: str) -> CollectionItem: + self.authenticate() + if self._mock_items is not None: + for item in self._mock_items: + if item["id"] == item_id: + content = item.get("content", b"") + if isinstance(content, str): + content = content.encode() + return CollectionItem(metadata=self._to_metadata(item), content=content) + raise FileNotFoundError(item_id) + + path = item_id if item_id.startswith("/") else f"/{item_id}" + _, response = self._client.files_download(path) + meta = self._client.files_get_metadata(path) + return CollectionItem(metadata=self._entry_to_metadata(meta), content=response.content) + + def stream_changes(self) -> Iterator[ItemMetadata]: + yield from self.list_items() + + def handle_rate_limits(self, response: Any) -> Optional[RateLimitInfo]: + headers = getattr(response, "headers", {}) or {} + retry_after = headers.get("Retry-After") or headers.get("retry-after") + status = getattr(response, "status_code", None) + if status == 429 or retry_after: + seconds = float(retry_after) if retry_after else 60.0 + return RateLimitInfo(retry_after_seconds=seconds, reason="dropbox_rate_limit") + return None + + def serialize_metadata(self, metadata: ItemMetadata) -> Dict[str, Any]: + return { + "id": metadata.item_id, + "name": metadata.name, + "mime_type": metadata.mime_type, + "size": metadata.size_bytes, + "modified_at": metadata.modified_at.isoformat() if metadata.modified_at else None, + "source": "dropbox", + } + + def _entry_to_metadata(self, entry: Any) -> ItemMetadata: + modified = getattr(entry, "server_modified", None) or getattr(entry, "client_modified", None) + modified_at = modified if isinstance(modified, datetime) else None + path = getattr(entry, "path_display", None) or getattr(entry, "id", "") + return ItemMetadata( + item_id=str(path), + name=getattr(entry, "name", str(path)), + size_bytes=getattr(entry, "size", None), + modified_at=modified_at, + source_path=f"dropbox://{path}", + ) + + def _dict_to_browse_item(self, item: Dict[str, Any]) -> BrowseItem: + item_type = BrowseItemType(item.get("type", "file")) + modified = item.get("modified_at") or item.get("server_modified") + modified_at = datetime.fromisoformat(modified) if isinstance(modified, str) else modified + item_id = str(item.get("id", item.get("path", ""))) + return BrowseItem( + id=item_id, + name=str(item.get("name", item_id)), + path=str(item.get("path", item_id)), + type=item_type, + connector_id=self.connector_id, + size=item.get("size"), + modified_at=modified_at, + mime_type=item.get("mime_type"), + ) + + def _to_metadata(self, item: Dict[str, Any]) -> ItemMetadata: + modified = item.get("modified_at") or item.get("server_modified") + modified_at = datetime.fromisoformat(modified) if isinstance(modified, str) else modified + item_id = str(item.get("id", item.get("path", ""))) + return ItemMetadata( + item_id=item_id, + name=str(item.get("name", item_id)), + mime_type=item.get("mime_type"), + size_bytes=item.get("size"), + modified_at=modified_at, + source_path=f"dropbox://{item_id}", + ) diff --git a/api/collectors/google_drive/__init__.py b/api/collectors/google_drive/__init__.py new file mode 100644 index 000000000..c79ad4d96 --- /dev/null +++ b/api/collectors/google_drive/__init__.py @@ -0,0 +1,5 @@ +"""Google Drive connector package.""" + +from .gdrive_collector import GoogleDriveCollector + +__all__ = ["GoogleDriveCollector"] diff --git a/api/collectors/google_drive/gdrive_auth.py b/api/collectors/google_drive/gdrive_auth.py new file mode 100644 index 000000000..08ed50cf4 --- /dev/null +++ b/api/collectors/google_drive/gdrive_auth.py @@ -0,0 +1,68 @@ +"""Google Drive OAuth2 authentication.""" + +from __future__ import annotations + +import os +from datetime import datetime, timedelta +from typing import Any, Dict, Optional + +import requests +from google.oauth2.credentials import Credentials +from googleapiclient.discovery import build + +from collectors.base.auth_context import AuthContext + + +class GoogleDriveAuth: + TOKEN_URL = "https://oauth2.googleapis.com/token" + SCOPES = ["https://www.googleapis.com/auth/drive.readonly"] + + def __init__(self, auth: AuthContext, config: Optional[Dict[str, Any]] = None): + self.auth = auth + self.config = config or {} + self.client_id = self.config.get("client_id") or os.getenv("GOOGLE_DRIVE_CLIENT_ID") + self.client_secret = self.config.get("client_secret") or os.getenv("GOOGLE_DRIVE_CLIENT_SECRET") + + def ensure_valid_token(self) -> AuthContext: + if not self.auth.is_expired(): + return self.auth + if not self.auth.refresh_token: + if self.auth.access_token: + return self.auth + raise ValueError("Google Drive credentials missing") + return self.refresh_token() + + def refresh_token(self) -> AuthContext: + if self.config.get("mock_refresh"): + self.auth.access_token = "mock-gdrive-token" + self.auth.expires_at = datetime.utcnow() + timedelta(hours=1) + return self.auth + + response = requests.post( + self.TOKEN_URL, + data={ + "grant_type": "refresh_token", + "refresh_token": self.auth.refresh_token, + "client_id": self.client_id, + "client_secret": self.client_secret, + }, + timeout=30, + ) + if response.status_code >= 400: + raise RuntimeError(f"Google token refresh failed: {response.status_code}") + data = response.json() + self.auth.access_token = data["access_token"] + expires_in = int(data.get("expires_in", 3600)) + self.auth.expires_at = datetime.utcnow() + timedelta(seconds=expires_in) + return self.auth + + def build_service(self, auth: AuthContext): + creds = Credentials( + token=auth.access_token, + refresh_token=auth.refresh_token, + token_uri=self.TOKEN_URL, + client_id=self.client_id, + client_secret=self.client_secret, + scopes=self.SCOPES, + ) + return build("drive", "v3", credentials=creds, cache_discovery=False) diff --git a/api/collectors/google_drive/gdrive_collector.py b/api/collectors/google_drive/gdrive_collector.py new file mode 100644 index 000000000..1488d5044 --- /dev/null +++ b/api/collectors/google_drive/gdrive_collector.py @@ -0,0 +1,179 @@ +"""Google Drive connector.""" + +from __future__ import annotations + +from datetime import datetime +from typing import Any, Dict, Iterator, List, Optional + +from collectors.base.auth_context import AuthContext +from collectors.base.base_collector import BaseCollector +from collectors.base.types import BrowseItem, BrowseItemType, CollectionItem, ItemMetadata, RateLimitInfo +from collectors.core.registry import register_connector + +from .gdrive_auth import GoogleDriveAuth + + +@register_connector("google_drive") +class GoogleDriveCollector(BaseCollector): + """Collect files from Google Drive using OAuth2.""" + + def __init__(self, auth: AuthContext, config: Optional[Dict[str, Any]] = None): + super().__init__(auth, config) + self._auth_client = GoogleDriveAuth(auth, config) + self._service = None + self._mock_items = config.get("mock_items") if config else None + self._mock_browse = config.get("mock_browse") if config else None + + @property + def connector_id(self) -> str: + return "google_drive" + + def authenticate(self) -> AuthContext: + self.auth = self._auth_client.ensure_valid_token() + if self._mock_items is None and self._mock_browse is None: + self._service = self._auth_client.build_service(self.auth) + return self.auth + + def list_children(self, parent_id: Optional[str] = None) -> List[BrowseItem]: + self.authenticate() + folder_id = parent_id or "root" + if self._mock_browse is not None: + return [self._dict_to_browse_item(entry) for entry in self._mock_browse.get(folder_id, [])] + + query = f"'{folder_id}' in parents and trashed = false" + results = ( + self._service.files() + .list( + q=query, + fields="files(id,name,mimeType,size,modifiedTime)", + orderBy="folder,name", + ) + .execute() + ) + items: List[BrowseItem] = [] + for file_obj in results.get("files", []): + mime = file_obj.get("mimeType", "") + is_folder = mime == "application/vnd.google-apps.folder" + item_type = BrowseItemType.FOLDER if is_folder else BrowseItemType.FILE + modified = file_obj.get("modifiedTime") + modified_at = ( + datetime.fromisoformat(modified.replace("Z", "+00:00")) if modified else None + ) + size = file_obj.get("size") + items.append( + BrowseItem( + id=file_obj["id"], + name=file_obj.get("name", file_obj["id"]), + path=f"/{file_obj.get('name', file_obj['id'])}", + type=item_type, + connector_id=self.connector_id, + size=int(size) if size else None, + modified_at=modified_at, + mime_type=mime, + ) + ) + return items + + def list_items(self, since: Optional[datetime] = None) -> List[ItemMetadata]: + self.authenticate() + if self._mock_items is not None: + return [self._to_metadata(item) for item in self._mock_items] + + query = "trashed = false" + if since: + query += f" and modifiedTime > '{since.isoformat()}Z'" + folder_id = self.config.get("folder_id") + if folder_id: + query += f" and '{folder_id}' in parents" + + results = ( + self._service.files() + .list(q=query, fields="files(id,name,mimeType,size,modifiedTime)") + .execute() + ) + return [self._file_to_metadata(f) for f in results.get("files", [])] + + def fetch_item(self, item_id: str) -> CollectionItem: + self.authenticate() + if self._mock_items is not None: + for item in self._mock_items: + if item["id"] == item_id: + content = item.get("content", b"") + if isinstance(content, str): + content = content.encode() + return CollectionItem(metadata=self._to_metadata(item), content=content) + raise FileNotFoundError(item_id) + + meta = self._service.files().get(fileId=item_id, fields="id,name,mimeType,size,modifiedTime").execute() + from googleapiclient.http import MediaIoBaseDownload + import io + + request = self._service.files().get_media(fileId=item_id) + buffer = io.BytesIO() + downloader = MediaIoBaseDownload(buffer, request) + done = False + while not done: + _, done = downloader.next_chunk() + return CollectionItem(metadata=self._file_to_metadata(meta), content=buffer.getvalue()) + + def stream_changes(self) -> Iterator[ItemMetadata]: + yield from self.list_items() + + def handle_rate_limits(self, response: Any) -> Optional[RateLimitInfo]: + headers = getattr(response, "headers", {}) or {} + retry_after = headers.get("Retry-After") or headers.get("retry-after") + status = getattr(response, "status_code", None) or getattr(response, "status", None) + if status in (429, "429") or retry_after: + seconds = float(retry_after) if retry_after else 60.0 + return RateLimitInfo(retry_after_seconds=seconds, reason="gdrive_rate_limit") + return None + + def serialize_metadata(self, metadata: ItemMetadata) -> Dict[str, Any]: + return { + "id": metadata.item_id, + "name": metadata.name, + "mime_type": metadata.mime_type, + "size": metadata.size_bytes, + "modified_at": metadata.modified_at.isoformat() if metadata.modified_at else None, + "source": "google_drive", + } + + def _file_to_metadata(self, file_obj: Dict[str, Any]) -> ItemMetadata: + modified = file_obj.get("modifiedTime") + modified_at = datetime.fromisoformat(modified.replace("Z", "+00:00")) if modified else None + size = file_obj.get("size") + return ItemMetadata( + item_id=file_obj["id"], + name=file_obj.get("name", file_obj["id"]), + mime_type=file_obj.get("mimeType"), + size_bytes=int(size) if size else None, + modified_at=modified_at, + source_path=f"gdrive://{file_obj['id']}", + ) + + def _dict_to_browse_item(self, item: Dict[str, Any]) -> BrowseItem: + item_type = BrowseItemType(item.get("type", "file")) + modified = item.get("modified_at") or item.get("modifiedTime") + modified_at = datetime.fromisoformat(modified.replace("Z", "+00:00")) if modified else None + return BrowseItem( + id=str(item["id"]), + name=str(item.get("name", item["id"])), + path=str(item.get("path", item["id"])), + type=item_type, + connector_id=self.connector_id, + size=int(item["size"]) if item.get("size") else None, + modified_at=modified_at, + mime_type=item.get("mime_type") or item.get("mimeType"), + ) + + def _to_metadata(self, item: Dict[str, Any]) -> ItemMetadata: + modified = item.get("modified_at") or item.get("modifiedTime") + modified_at = datetime.fromisoformat(modified.replace("Z", "+00:00")) if modified else None + return ItemMetadata( + item_id=str(item["id"]), + name=str(item.get("name", item["id"])), + mime_type=item.get("mime_type") or item.get("mimeType"), + size_bytes=int(item["size"]) if item.get("size") else None, + modified_at=modified_at, + source_path=f"gdrive://{item['id']}", + ) diff --git a/api/collectors/ingestion/__init__.py b/api/collectors/ingestion/__init__.py new file mode 100644 index 000000000..40a1af6f1 --- /dev/null +++ b/api/collectors/ingestion/__init__.py @@ -0,0 +1,16 @@ +"""Ingestion pipeline components.""" + +__all__ = ["IngestionPipeline", "MetadataNormalizer", "StreamingEngine"] + + +def __getattr__(name: str): + if name == "IngestionPipeline": + from .ingestion_pipeline import IngestionPipeline + return IngestionPipeline + if name == "MetadataNormalizer": + from .metadata_normalizer import MetadataNormalizer + return MetadataNormalizer + if name == "StreamingEngine": + from .streaming_engine import StreamingEngine + return StreamingEngine + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") diff --git a/api/collectors/ingestion/ingestion_pipeline.py b/api/collectors/ingestion/ingestion_pipeline.py new file mode 100644 index 000000000..749072bd4 --- /dev/null +++ b/api/collectors/ingestion/ingestion_pipeline.py @@ -0,0 +1,108 @@ +"""End-to-end ingestion: list/stream -> fetch -> normalize -> store.""" + +from __future__ import annotations + +import logging +from datetime import datetime +from typing import Any, Dict, Optional + +from collectors.base.base_collector import BaseCollector +from collectors.base.types import BrowseItemType, SelectedItem +from collectors.core.logging_manager import LoggingManager +from collectors.core.state_manager import StateManager +from collectors.ingestion.metadata_normalizer import MetadataNormalizer +from collectors.ingestion.streaming_engine import StreamingEngine +from collectors.storage.storage_interface import MetadataStore, StorageBackend + + +class IngestionPipeline: + """Orchestrates item discovery, download, and persistence.""" + + def __init__( + self, + collector: BaseCollector, + filesystem: StorageBackend, + metadata_store: MetadataStore, + state_manager: StateManager, + logger: Optional[logging.Logger] = None, + max_workers: int = 4, + ): + self.collector = collector + self.filesystem = filesystem + self.metadata_store = metadata_store + self.state = state_manager + self.logger = logger or LoggingManager.configure() + self.normalizer = MetadataNormalizer() + self.engine = StreamingEngine(collector, max_workers=max_workers, logger=self.logger) + + def run(self, since: Optional[datetime] = None) -> Dict[str, Any]: + connector_id = self.collector.connector_id + selected = self._resolve_selected_items() + if selected: + items = self.collector.expand_selection(selected, since=since) + else: + items = list(self.engine.iter_items(since=since)) + LoggingManager.log_event( + self.logger, + "items_discovered", + connector_id=connector_id, + count=len(items), + ) + + def on_item(collection_item) -> None: + meta = collection_item.metadata + if collection_item.content is None: + raise ValueError(f"No content for item {meta.item_id}") + + if not meta.checksum: + meta.checksum = self.normalizer.content_checksum(collection_item.content) + + if self.metadata_store.exists(meta.item_id, meta.checksum): + LoggingManager.log_event( + self.logger, + "item_skipped_idempotent", + item_id=meta.item_id, + ) + return + + local_path = self.filesystem.write_item(meta, collection_item.content) + record = self.normalizer.to_storage_record(meta, local_path) + self.metadata_store.save(record) + serialized = self.collector.serialize_metadata(meta) + self.metadata_store.save_raw(meta.item_id, serialized) + + def on_error(item_id: str, exc: Exception) -> None: + self.state.add_failed_item(connector_id, item_id) + + result = self.engine.fetch_all(items, on_item=on_item, on_error=on_error) + result["sync_token"] = datetime.utcnow().isoformat() + return result + + def _resolve_selected_items(self) -> List[SelectedItem]: + config = self.collector.config + raw_items = config.get("selected_items") + if raw_items: + resolved: List[SelectedItem] = [] + for entry in raw_items: + item_type = entry.get("type", "file") + resolved.append( + SelectedItem( + id=str(entry["id"]), + type=BrowseItemType(item_type), + name=str(entry.get("name", "")), + path=str(entry.get("path", "")), + ) + ) + return resolved + + selected_ids = config.get("selected_item_ids") or [] + selected_paths = config.get("selected_paths") or [] + resolved = [ + SelectedItem(id=str(item_id), type=BrowseItemType.FILE, path=str(item_id)) + for item_id in selected_ids + ] + resolved.extend( + SelectedItem(id=str(path), type=BrowseItemType.FILE, path=str(path)) + for path in selected_paths + ) + return resolved diff --git a/api/collectors/ingestion/metadata_normalizer.py b/api/collectors/ingestion/metadata_normalizer.py new file mode 100644 index 000000000..a74cbf8b5 --- /dev/null +++ b/api/collectors/ingestion/metadata_normalizer.py @@ -0,0 +1,77 @@ +"""Normalize connector metadata to canonical form.""" + +from __future__ import annotations + +import hashlib +import re +from datetime import datetime +from typing import Any, Dict + +from collectors.base.types import ItemMetadata + + +class MetadataNormalizer: + """Convert raw connector metadata into canonical ItemMetadata records.""" + + INVALID_FILENAME_CHARS = re.compile(r'[<>:"/\\|?*\x00-\x1f]') + + def normalize(self, raw: Dict[str, Any], connector_id: str) -> ItemMetadata: + item_id = str(raw.get("id") or raw.get("item_id") or "") + name = self._sanitize_filename(str(raw.get("name") or raw.get("title") or item_id)) + modified = raw.get("modified_at") or raw.get("modifiedTime") or raw.get("server_modified") + modified_at = self._parse_datetime(modified) + + return ItemMetadata( + item_id=item_id, + name=name, + mime_type=raw.get("mime_type") or raw.get("mimeType"), + size_bytes=self._parse_int(raw.get("size") or raw.get("size_bytes")), + modified_at=modified_at, + source_path=raw.get("path") or raw.get("source_path"), + checksum=raw.get("checksum") or raw.get("content_hash"), + extra={ + "connector_id": connector_id, + "raw": {k: v for k, v in raw.items() if k not in ("content",)}, + }, + ) + + def to_storage_record(self, metadata: ItemMetadata, local_path: str) -> Dict[str, Any]: + return { + "item_id": metadata.item_id, + "name": metadata.name, + "mime_type": metadata.mime_type, + "size_bytes": metadata.size_bytes, + "modified_at": metadata.modified_at.isoformat() if metadata.modified_at else None, + "source_path": metadata.source_path, + "checksum": metadata.checksum, + "local_path": local_path, + "connector_id": metadata.extra.get("connector_id"), + } + + def content_checksum(self, content: bytes) -> str: + return hashlib.sha256(content).hexdigest() + + def _sanitize_filename(self, name: str) -> str: + cleaned = self.INVALID_FILENAME_CHARS.sub("_", name.strip()) + return cleaned or "unnamed" + + def _parse_datetime(self, value: Any) -> datetime | None: + if value is None: + return None + if isinstance(value, datetime): + return value + if isinstance(value, (int, float)): + return datetime.utcfromtimestamp(value) + text = str(value).replace("Z", "+00:00") + try: + return datetime.fromisoformat(text) + except ValueError: + return None + + def _parse_int(self, value: Any) -> int | None: + if value is None: + return None + try: + return int(value) + except (TypeError, ValueError): + return None diff --git a/api/collectors/ingestion/streaming_engine.py b/api/collectors/ingestion/streaming_engine.py new file mode 100644 index 000000000..cc88bb79c --- /dev/null +++ b/api/collectors/ingestion/streaming_engine.py @@ -0,0 +1,89 @@ +"""Concurrent streaming fetch engine.""" + +from __future__ import annotations + +import logging +from concurrent.futures import ThreadPoolExecutor, as_completed +from datetime import datetime +from typing import Callable, Iterator, List, Optional + +from collectors.base.base_collector import BaseCollector +from collectors.base.types import CollectionItem, ItemMetadata, RateLimitInfo +from collectors.core.logging_manager import LoggingManager + + +class StreamingEngine: + """Fetch items with bounded concurrency and rate-limit handling.""" + + def __init__( + self, + collector: BaseCollector, + max_workers: int = 4, + logger: Optional[logging.Logger] = None, + ): + self.collector = collector + self.max_workers = max_workers + self.logger = logger or LoggingManager.configure() + + def iter_items(self, since: Optional[datetime] = None) -> Iterator[ItemMetadata]: + use_stream = self.collector.config.get("use_stream", True) + if use_stream: + yield from self.collector.stream_changes() + else: + yield from self.collector.list_items(since=since) + + def fetch_all( + self, + items: List[ItemMetadata], + on_item: Callable[[CollectionItem], None], + on_error: Optional[Callable[[str, Exception], None]] = None, + ) -> dict: + collected = 0 + failed = 0 + + with ThreadPoolExecutor(max_workers=self.max_workers) as executor: + futures = { + executor.submit(self._fetch_with_retry, meta.item_id): meta + for meta in items + } + for future in as_completed(futures): + meta = futures[future] + try: + item = future.result() + on_item(item) + collected += 1 + except Exception as exc: + failed += 1 + LoggingManager.log_event( + self.logger, + "fetch_item_failed", + level=logging.WARNING, + item_id=meta.item_id, + error=str(exc), + ) + if on_error: + on_error(meta.item_id, exc) + + return {"items_collected": collected, "items_failed": failed} + + def _fetch_with_retry(self, item_id: str, max_retries: int = 3) -> CollectionItem: + last_exc: Optional[Exception] = None + for attempt in range(max_retries): + try: + return self.collector.fetch_item(item_id) + except Exception as exc: + last_exc = exc + rate_info = self._extract_rate_limit(exc) + if rate_info: + self.collector.wait_for_rate_limit(rate_info) + continue + if attempt < max_retries - 1: + continue + raise + raise last_exc or RuntimeError(f"Failed to fetch {item_id}") + + def _extract_rate_limit(self, exc: Exception) -> Optional[RateLimitInfo]: + response = getattr(exc, "response", None) + if response is not None: + return self.collector.handle_rate_limits(response) + return None diff --git a/api/collectors/processing/__init__.py b/api/collectors/processing/__init__.py new file mode 100644 index 000000000..1fb602e01 --- /dev/null +++ b/api/collectors/processing/__init__.py @@ -0,0 +1,5 @@ +"""Processing engine integration.""" + +from .processing_engine_client import ProcessingEngineClient + +__all__ = ["ProcessingEngineClient"] diff --git a/api/collectors/processing/processing_engine_client.py b/api/collectors/processing/processing_engine_client.py new file mode 100644 index 000000000..73317ae8c --- /dev/null +++ b/api/collectors/processing/processing_engine_client.py @@ -0,0 +1,100 @@ +"""Client for chaining collection to FreeEed Java processing.""" + +from __future__ import annotations + +import os +import threading +from pathlib import Path +from typing import Any, Dict, Optional + + +class ProcessingEngineClient: + """Generate .project files and invoke run_freeeed_process.""" + + _lock = threading.Lock() + _busy = False + + DEFAULT_SOLR = os.getenv("SOLR_ENDPOINT", "http://solr:8983") + DEFAULT_OUTPUT = os.getenv("COLLECTOR_PROCESSING_OUTPUT", "/data/output") + + def is_busy(self) -> bool: + return self._busy + + def trigger_processing( + self, + job_id: str, + input_dir: str, + project_config: Optional[Dict[str, Any]] = None, + ) -> str: + with self._lock: + if self._busy: + raise RuntimeError("Processing engine is already running") + self._busy = True + + try: + config = project_config or {} + project_path = self.write_project_file(job_id, input_dir, config) + self._run_java(project_path) + return project_path + finally: + with self._lock: + self._busy = False + + def write_project_file( + self, + job_id: str, + input_dir: str, + config: Dict[str, Any], + ) -> str: + """Write a Java-canonical .project file.""" + projects_dir = Path(config.get("projects_dir", "/data/input/collections/projects")) + projects_dir.mkdir(parents=True, exist_ok=True) + project_path = projects_dir / f"{job_id}.project" + + project_name = config.get("project_name", f"collection-{job_id[:8]}") + output_dir = config.get("output_dir", self.DEFAULT_OUTPUT) + custodian = config.get("custodian", "collector") + if isinstance(custodian, list): + custodian = ",".join(custodian) + solr_endpoint = config.get("solr_endpoint", self.DEFAULT_SOLR) + processing_engine = config.get("processing_engine", "Standard") + stage = str(config.get("stage", True)).lower() + process_where = config.get("process_where", "local") + + lines = [ + f"project-name={project_name}", + f"input={input_dir}", + f"output-dir={output_dir}", + f"custodian={custodian}", + f"stage={stage}", + f"solr_endpoint={solr_endpoint}", + f"processing_engine={processing_engine}", + f"process-where={process_where}", + "metadata=standard", + "file-system=local", + "send_index_solr_enabled=true", + ] + + optional_keys = { + "project-code": "project_code", + "staging-dir": "staging_dir", + "preview": "preview", + "ocr_enabled": "ocr_enabled", + } + for java_key, cfg_key in optional_keys.items(): + if cfg_key in config: + val = config[cfg_key] + if isinstance(val, bool): + val = str(val).lower() + lines.append(f"{java_key}={val}") + + content = "\n".join(lines) + "\n" + temp_path = project_path.with_suffix(".project.tmp") + temp_path.write_text(content, encoding="utf-8") + temp_path.replace(project_path) + return str(project_path) + + def _run_java(self, project_path: str) -> None: + import main as api_main + + api_main.run_freeeed_process(project_path) diff --git a/api/collectors/scripts/manual_trigger.py b/api/collectors/scripts/manual_trigger.py new file mode 100644 index 000000000..cdc6a1e8c --- /dev/null +++ b/api/collectors/scripts/manual_trigger.py @@ -0,0 +1,62 @@ +#!/usr/bin/env python3 +"""Manually trigger a collection job from the CLI.""" + +from __future__ import annotations + +import argparse +import json +import sys +import time +from pathlib import Path + +API_ROOT = Path(__file__).resolve().parents[2] +if str(API_ROOT) not in sys.path: + sys.path.insert(0, str(API_ROOT)) + +from collectors.core.logging_manager import LoggingManager +from collectors.core.orchestrator import Orchestrator +from collectors.core.state_manager import StateManager +from collectors.core.trigger_manager import TriggerManager + +import collectors.box.box_collector # noqa: F401 +import collectors.google_drive.gdrive_collector # noqa: F401 +import collectors.dropbox.dropbox_collector # noqa: F401 + + +def main() -> int: + parser = argparse.ArgumentParser(description="Manually trigger a collection job") + parser.add_argument("connector_id", help="Connector ID (box, google_drive, dropbox)") + parser.add_argument("--auto-chain", action="store_true", help="Chain to Java processing") + parser.add_argument("--config", default="{}", help="JSON config override") + parser.add_argument("--wait", action="store_true", help="Wait for job completion") + args = parser.parse_args() + + LoggingManager.configure() + state = StateManager() + orchestrator = Orchestrator(state_manager=state) + trigger = TriggerManager(state, run_callback=orchestrator.run_collection) + + config = json.loads(args.config) + job = trigger.trigger_manual(args.connector_id, auto_chain=args.auto_chain, config=config) + print(json.dumps({"job_id": job.job_id, "status": job.status.value}, indent=2)) + + if args.wait: + while True: + current = state.get_job(job.job_id) + if current and current.status.value in ("completed", "failed", "cancelled"): + print(json.dumps({ + "job_id": current.job_id, + "status": current.status.value, + "items_collected": current.items_collected, + "items_failed": current.items_failed, + "error_message": current.error_message, + }, indent=2)) + break + time.sleep(1) + + trigger.shutdown() + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/api/collectors/scripts/run_collectors.py b/api/collectors/scripts/run_collectors.py new file mode 100644 index 000000000..7f6859d56 --- /dev/null +++ b/api/collectors/scripts/run_collectors.py @@ -0,0 +1,53 @@ +#!/usr/bin/env python3 +"""CLI entry point for running scheduled collectors.""" + +from __future__ import annotations + +import argparse +import signal +import sys +from pathlib import Path + +# Ensure api/ is on path when run from repo root +API_ROOT = Path(__file__).resolve().parents[2] +if str(API_ROOT) not in sys.path: + sys.path.insert(0, str(API_ROOT)) + +from collectors.core.logging_manager import LoggingManager +from collectors.core.orchestrator import Orchestrator +from collectors.core.state_manager import StateManager +from collectors.core.trigger_manager import TriggerManager + +# Import connectors to register them +import collectors.box.box_collector # noqa: F401 +import collectors.google_drive.gdrive_collector # noqa: F401 +import collectors.dropbox.dropbox_collector # noqa: F401 + + +def main() -> int: + parser = argparse.ArgumentParser(description="Run FreeEed collector scheduler") + parser.add_argument("--load-schedules", action="store_true", help="Load cron schedules from DB") + args = parser.parse_args() + + LoggingManager.configure() + state = StateManager() + orchestrator = Orchestrator(state_manager=state) + trigger = TriggerManager(state, run_callback=orchestrator.run_collection) + + if args.load_schedules: + trigger.load_schedules_from_db() + + def _shutdown(signum, frame): + trigger.shutdown() + sys.exit(0) + + signal.signal(signal.SIGINT, _shutdown) + signal.signal(signal.SIGTERM, _shutdown) + + print("Collector scheduler running. Press Ctrl+C to stop.") + signal.pause() + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/api/collectors/storage/__init__.py b/api/collectors/storage/__init__.py new file mode 100644 index 000000000..3de5f0856 --- /dev/null +++ b/api/collectors/storage/__init__.py @@ -0,0 +1,7 @@ +"""Storage backends for collected data.""" + +from .storage_interface import MetadataStore, StorageBackend +from .filesystem_store import FilesystemStore +from .sqlite_store import SQLiteMetadataStore + +__all__ = ["MetadataStore", "StorageBackend", "FilesystemStore", "SQLiteMetadataStore"] diff --git a/api/collectors/storage/filesystem_store.py b/api/collectors/storage/filesystem_store.py new file mode 100644 index 000000000..dfd3604a8 --- /dev/null +++ b/api/collectors/storage/filesystem_store.py @@ -0,0 +1,31 @@ +"""Filesystem storage for collected items.""" + +from __future__ import annotations + +import os +from pathlib import Path + +from collectors.base.types import ItemMetadata + + +class FilesystemStore: + """Write collected files to a job-specific directory.""" + + def __init__(self, base_dir: str): + self.base_dir = Path(base_dir) + self.base_dir.mkdir(parents=True, exist_ok=True) + + def write_item(self, metadata: ItemMetadata, content: bytes) -> str: + safe_name = metadata.name + target = self.base_dir / safe_name + if target.exists() and metadata.checksum: + stem = target.stem + suffix = target.suffix + target = self.base_dir / f"{stem}_{metadata.item_id[:8]}{suffix}" + + target.parent.mkdir(parents=True, exist_ok=True) + temp_path = target.with_suffix(target.suffix + ".tmp") + with open(temp_path, "wb") as fh: + fh.write(content) + os.replace(temp_path, target) + return str(target) diff --git a/api/collectors/storage/sqlite_store.py b/api/collectors/storage/sqlite_store.py new file mode 100644 index 000000000..94bcf92e1 --- /dev/null +++ b/api/collectors/storage/sqlite_store.py @@ -0,0 +1,97 @@ +"""SQLite metadata store.""" + +from __future__ import annotations + +import json +import os +import sqlite3 +import threading +from contextlib import contextmanager +from typing import Any, Dict + + +class SQLiteMetadataStore: + """Persist item metadata in SQLite for a collection job.""" + + def __init__(self, db_path: str): + self.db_path = db_path + self._lock = threading.Lock() + os.makedirs(os.path.dirname(db_path), exist_ok=True) + self._init_db() + + @contextmanager + def _connection(self): + conn = sqlite3.connect(self.db_path, check_same_thread=False) + conn.row_factory = sqlite3.Row + try: + yield conn + conn.commit() + finally: + conn.close() + + def _init_db(self) -> None: + with self._lock, self._connection() as conn: + conn.executescript( + """ + CREATE TABLE IF NOT EXISTS items ( + item_id TEXT PRIMARY KEY, + name TEXT, + mime_type TEXT, + size_bytes INTEGER, + modified_at TEXT, + source_path TEXT, + checksum TEXT, + local_path TEXT, + connector_id TEXT + ); + CREATE TABLE IF NOT EXISTS raw_metadata ( + item_id TEXT PRIMARY KEY, + data TEXT + ); + CREATE INDEX IF NOT EXISTS idx_items_checksum ON items(checksum); + """ + ) + + def save(self, record: Dict[str, Any]) -> None: + with self._lock, self._connection() as conn: + conn.execute( + """ + INSERT INTO items ( + item_id, name, mime_type, size_bytes, modified_at, + source_path, checksum, local_path, connector_id + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?) + ON CONFLICT(item_id) DO UPDATE SET + name = excluded.name, + checksum = excluded.checksum, + local_path = excluded.local_path + """, + ( + record["item_id"], + record.get("name"), + record.get("mime_type"), + record.get("size_bytes"), + record.get("modified_at"), + record.get("source_path"), + record.get("checksum"), + record.get("local_path"), + record.get("connector_id"), + ), + ) + + def save_raw(self, item_id: str, raw: Dict[str, Any]) -> None: + with self._lock, self._connection() as conn: + conn.execute( + """ + INSERT INTO raw_metadata (item_id, data) VALUES (?, ?) + ON CONFLICT(item_id) DO UPDATE SET data = excluded.data + """, + (item_id, json.dumps(raw)), + ) + + def exists(self, item_id: str, checksum: str) -> bool: + with self._lock, self._connection() as conn: + row = conn.execute( + "SELECT 1 FROM items WHERE item_id = ? AND checksum = ?", + (item_id, checksum), + ).fetchone() + return row is not None diff --git a/api/collectors/storage/storage_interface.py b/api/collectors/storage/storage_interface.py new file mode 100644 index 000000000..118cb6ead --- /dev/null +++ b/api/collectors/storage/storage_interface.py @@ -0,0 +1,26 @@ +"""Storage abstractions.""" + +from __future__ import annotations + +import abc +from typing import Any, Dict + + +class StorageBackend(abc.ABC): + @abc.abstractmethod + def write_item(self, metadata: Any, content: bytes) -> str: + """Persist item content; return local path.""" + + +class MetadataStore(abc.ABC): + @abc.abstractmethod + def save(self, record: Dict[str, Any]) -> None: + """Save normalized metadata record.""" + + @abc.abstractmethod + def save_raw(self, item_id: str, raw: Dict[str, Any]) -> None: + """Save connector-specific serialized metadata.""" + + @abc.abstractmethod + def exists(self, item_id: str, checksum: str) -> bool: + """Return True if item with same checksum already stored (idempotent).""" diff --git a/api/collectors/template/new_connector_template.py b/api/collectors/template/new_connector_template.py new file mode 100644 index 000000000..9d7b1a479 --- /dev/null +++ b/api/collectors/template/new_connector_template.py @@ -0,0 +1,57 @@ +"""Template for implementing a new connector.""" + +from __future__ import annotations + +from datetime import datetime +from typing import Any, Dict, Iterator, List, Optional + +from collectors.base.auth_context import AuthContext +from collectors.base.base_collector import BaseCollector +from collectors.base.types import BrowseItem, CollectionItem, ItemMetadata, RateLimitInfo +from collectors.core.registry import register_connector + + +@register_connector("template") +class TemplateCollector(BaseCollector): + """ + Copy this file to collectors// and replace template logic. + + 1. Rename connector_id and register_connector value + 2. Implement authenticate() with OAuth2 or API key flow + 3. Implement list_items, fetch_item, stream_changes + 4. Add auth module alongside this file + """ + + @property + def connector_id(self) -> str: + return "template" + + def authenticate(self) -> AuthContext: + if not self.auth.access_token: + raise ValueError("Template connector requires access_token") + return self.auth + + def list_children(self, parent_id: Optional[str] = None) -> List[BrowseItem]: + self.authenticate() + return [] + + def list_items(self, since: Optional[datetime] = None) -> List[ItemMetadata]: + self.authenticate() + return [] + + def fetch_item(self, item_id: str) -> CollectionItem: + self.authenticate() + raise NotImplementedError(f"fetch_item not implemented for {item_id}") + + def stream_changes(self) -> Iterator[ItemMetadata]: + yield from self.list_items() + + def handle_rate_limits(self, response: Any) -> Optional[RateLimitInfo]: + headers = getattr(response, "headers", {}) or {} + retry_after = headers.get("Retry-After") + if retry_after: + return RateLimitInfo(retry_after_seconds=float(retry_after)) + return None + + def serialize_metadata(self, metadata: ItemMetadata) -> Dict[str, Any]: + return {"id": metadata.item_id, "name": metadata.name, "source": "template"} diff --git a/api/collectors/tests/conftest.py b/api/collectors/tests/conftest.py new file mode 100644 index 000000000..1c2f7bc82 --- /dev/null +++ b/api/collectors/tests/conftest.py @@ -0,0 +1,8 @@ +"""Pytest configuration for collector tests.""" + +import sys +from pathlib import Path + +API_ROOT = Path(__file__).resolve().parents[2] +if str(API_ROOT) not in sys.path: + sys.path.insert(0, str(API_ROOT)) diff --git a/api/collectors/tests/integration/__init__.py b/api/collectors/tests/integration/__init__.py new file mode 100644 index 000000000..c210facce --- /dev/null +++ b/api/collectors/tests/integration/__init__.py @@ -0,0 +1 @@ +"""Integration tests.""" diff --git a/api/collectors/tests/integration/test_e2e_collection.py b/api/collectors/tests/integration/test_e2e_collection.py new file mode 100644 index 000000000..c2fa99a85 --- /dev/null +++ b/api/collectors/tests/integration/test_e2e_collection.py @@ -0,0 +1,80 @@ +"""End-to-end integration tests with mock connectors.""" + +import os +import tempfile +from unittest.mock import patch + +import pytest + +from collectors.base.auth_context import AuthContext +from collectors.core.orchestrator import Orchestrator +from collectors.core.state_manager import StateManager +from collectors.base.types import CollectionJob, CollectionJobStatus +from collectors.processing.processing_engine_client import ProcessingEngineClient + +import collectors.box.box_collector # noqa: F401 +import collectors.google_drive.gdrive_collector # noqa: F401 +import collectors.dropbox.dropbox_collector # noqa: F401 + + +MOCK_ITEMS = [ + {"id": "f1", "name": "doc1.txt", "content": "hello", "modified_at": "2024-06-01T12:00:00"}, + {"id": "f2", "name": "doc2.txt", "content": "world", "modified_at": "2024-06-01T12:00:00"}, +] + + +@pytest.fixture +def orchestrator(tmp_path, monkeypatch): + monkeypatch.setenv("COLLECTOR_BOX_ACCESS_TOKEN", "mock-token") + state = StateManager(db_path=str(tmp_path / "state.db")) + orch = Orchestrator( + state_manager=state, + base_output_dir=str(tmp_path / "collections"), + ) + return orch, state, tmp_path + + +def test_end_to_end_mock_collection(orchestrator): + orch, state, tmp_path = orchestrator + job = CollectionJob( + job_id="integration-job-1", + connector_id="box", + status=CollectionJobStatus.PENDING, + auto_chain=False, + config={"mock_items": MOCK_ITEMS, "use_stream": False}, + ) + result = orch.run_collection(job) + assert result.status == CollectionJobStatus.COMPLETED + assert result.items_collected == 2 + output_dir = tmp_path / "collections" / "integration-job-1" + assert (output_dir / "doc1.txt").exists() + assert (output_dir / "doc2.txt").exists() + + +def test_project_file_java_canonical_keys(tmp_path): + client = ProcessingEngineClient() + project_path = client.write_project_file( + job_id="test-job", + input_dir="/data/input/collections/test-job", + config={ + "project_name": "Test Collection", + "custodian": "c1,c2", + "output_dir": "/data/output", + "solr_endpoint": "http://solr:8983", + }, + ) + content = open(project_path, encoding="utf-8").read() + assert "input=/data/input/collections/test-job" in content + assert "input-dir=" not in content + assert "custodian=c1,c2" in content + assert "stage=true" in content + assert "processing_engine=Standard" in content + assert "solr_endpoint=http://solr:8983" in content + + +@pytest.mark.skipif( + not os.getenv("BOX_ACCESS_TOKEN") and not os.getenv("COLLECTOR_BOX_ACCESS_TOKEN"), + reason="No live Box credentials", +) +def test_live_box_skipped_without_credentials(): + pytest.skip("Live connector test requires credentials") diff --git a/api/collectors/tests/unit/__init__.py b/api/collectors/tests/unit/__init__.py new file mode 100644 index 000000000..e0310a01e --- /dev/null +++ b/api/collectors/tests/unit/__init__.py @@ -0,0 +1 @@ +"""Unit tests.""" diff --git a/api/collectors/tests/unit/test_browse_service.py b/api/collectors/tests/unit/test_browse_service.py new file mode 100644 index 000000000..2bcdb86e4 --- /dev/null +++ b/api/collectors/tests/unit/test_browse_service.py @@ -0,0 +1,85 @@ +"""Browse normalization and credential handling tests.""" + +from unittest.mock import MagicMock, patch + +import pytest + +from collectors.base.auth_context import AuthContext +from collectors.base.types import ConnectorHealth +from collectors.box.box_collector import BoxCollector +from collectors.core.browse_service import ( + BrowseService, + ConnectorAuthError, + CredentialsNotConfiguredError, +) + + +MOCK_BROWSE = { + "0": [ + {"id": "folder-1", "name": "Docs", "path": "/Docs", "type": "folder"}, + {"id": "file-1", "name": "readme.txt", "path": "/readme.txt", "type": "file", "size": 42}, + ], + "folder-1": [ + {"id": "file-2", "name": "nested.pdf", "path": "/Docs/nested.pdf", "type": "file", "size": 100}, + ], +} + + +def test_box_list_children_mock(): + auth = AuthContext(connector_id="box", access_token="dev") + collector = BoxCollector( + auth=auth, + config={"mock_browse": MOCK_BROWSE, "mock_refresh": True}, + ) + root = collector.list_children() + assert len(root) == 2 + assert root[0].type.value == "folder" + assert root[0].connector_id == "box" + + children = collector.list_children("folder-1") + assert len(children) == 1 + assert children[0].name == "nested.pdf" + + +def test_browse_service_serialize(): + auth = AuthContext(connector_id="box", access_token="dev") + collector = BoxCollector(auth=auth, config={"mock_browse": MOCK_BROWSE}) + items = collector.list_children() + serialized = BrowseService.serialize_browse_item(items[0]) + assert serialized["type"] == "folder" + assert serialized["connector_id"] == "box" + assert "modified_at" in serialized + + +def test_browse_service_missing_credentials(tmp_path, monkeypatch): + monkeypatch.setenv("COLLECTOR_SECRETS_DIR", str(tmp_path)) + service = BrowseService() + with pytest.raises(CredentialsNotConfiguredError): + service.browse("box") + + +def test_browse_service_connect_with_override(): + service = BrowseService() + result = service.connect( + connector_id="box", + token_override={"access_token": "dev"}, + config={"mock_refresh": True, "mock_browse": MOCK_BROWSE}, + ) + assert result["connected"] is True + + +@patch.object(BrowseService, "_build_collector") +def test_browse_service_auth_failure(mock_build): + mock_collector = MagicMock() + mock_collector.health_check.return_value = ConnectorHealth( + connector_id="box", + healthy=False, + message="Authentication failed", + ) + mock_build.return_value = mock_collector + service = BrowseService() + with pytest.raises(ConnectorAuthError): + service.connect( + connector_id="box", + token_override={"access_token": "invalid"}, + ) diff --git a/api/collectors/tests/unit/test_metadata_normalizer.py b/api/collectors/tests/unit/test_metadata_normalizer.py new file mode 100644 index 000000000..5f5276a5b --- /dev/null +++ b/api/collectors/tests/unit/test_metadata_normalizer.py @@ -0,0 +1,30 @@ +"""Tests for MetadataNormalizer.""" + +from datetime import datetime + +from collectors.ingestion.metadata_normalizer import MetadataNormalizer + + +def test_normalize_sanitizes_filename(): + normalizer = MetadataNormalizer() + meta = normalizer.normalize({"id": "1", "name": 'bad.txt'}, "box") + assert meta.name == "bad_file_.txt" + assert meta.item_id == "1" + assert meta.extra["connector_id"] == "box" + + +def test_content_checksum_stable(): + normalizer = MetadataNormalizer() + assert normalizer.content_checksum(b"hello") == normalizer.content_checksum(b"hello") + assert normalizer.content_checksum(b"hello") != normalizer.content_checksum(b"world") + + +def test_to_storage_record(): + normalizer = MetadataNormalizer() + meta = normalizer.normalize( + {"id": "1", "name": "doc.pdf", "modifiedTime": "2024-01-15T10:00:00Z"}, + "google_drive", + ) + record = normalizer.to_storage_record(meta, "/data/input/collections/job/doc.pdf") + assert record["local_path"].endswith("doc.pdf") + assert record["connector_id"] == "google_drive" diff --git a/api/collectors/tests/unit/test_mock_connector_http.py b/api/collectors/tests/unit/test_mock_connector_http.py new file mode 100644 index 000000000..a5829f796 --- /dev/null +++ b/api/collectors/tests/unit/test_mock_connector_http.py @@ -0,0 +1,44 @@ +"""Mock connector HTTP behavior tests.""" + +from datetime import datetime +from unittest.mock import MagicMock, patch + +import pytest + +from collectors.base.auth_context import AuthContext +from collectors.box.box_auth import BoxAuth +from collectors.google_drive.gdrive_auth import GoogleDriveAuth +from collectors.dropbox.dropbox_auth import DropboxAuth + + +def test_box_token_refresh_mock(): + auth = AuthContext(connector_id="box", refresh_token="r1") + client = BoxAuth(auth, config={"mock_refresh": True}) + refreshed = client.refresh_token() + assert refreshed.access_token == "mock-box-token" + + +@patch("collectors.box.box_auth.requests.post") +def test_box_token_refresh_http(mock_post): + mock_post.return_value = MagicMock( + status_code=200, + json=lambda: {"access_token": "new-token", "expires_in": 3600}, + ) + auth = AuthContext(connector_id="box", refresh_token="r1") + client = BoxAuth(auth, config={"client_id": "id", "client_secret": "secret"}) + refreshed = client.refresh_token() + assert refreshed.access_token == "new-token" + + +def test_gdrive_token_refresh_mock(): + auth = AuthContext(connector_id="google_drive", refresh_token="r1") + client = GoogleDriveAuth(auth, config={"mock_refresh": True}) + refreshed = client.refresh_token() + assert refreshed.access_token == "mock-gdrive-token" + + +def test_dropbox_token_refresh_mock(): + auth = AuthContext(connector_id="dropbox", refresh_token="r1") + client = DropboxAuth(auth, config={"mock_refresh": True}) + refreshed = client.refresh_token() + assert refreshed.access_token == "mock-dropbox-token" diff --git a/api/collectors/tests/unit/test_oauth_manager.py b/api/collectors/tests/unit/test_oauth_manager.py new file mode 100644 index 000000000..79cea4cdb --- /dev/null +++ b/api/collectors/tests/unit/test_oauth_manager.py @@ -0,0 +1,134 @@ +"""Unit tests for OAuth manager (no live provider calls).""" + +from datetime import datetime, timedelta +from unittest.mock import MagicMock, patch + +import pytest +from cryptography.fernet import Fernet + +from collectors.core.oauth_manager import ( + OAuthError, + OAuthManager, + _code_challenge_s256, + _generate_code_verifier, +) +from collectors.core.security_manager import SecurityManager +from collectors.core.token_store import TokenStore + + +@pytest.fixture +def oauth_stack(tmp_path): + key = Fernet.generate_key().decode() + secrets_dir = tmp_path / "secrets" + secrets_dir.mkdir() + (secrets_dir / "dropbox.json").write_text( + '{"client_id": "test-client", "client_secret": "test-secret"}', + encoding="utf-8", + ) + db_path = str(tmp_path / "state.db") + security = SecurityManager(encryption_key=key, secrets_dir=str(secrets_dir)) + token_store = TokenStore(db_path=db_path) + manager = OAuthManager( + token_store=token_store, + security_manager=security, + redirect_base="http://localhost:8000", + ) + return manager, token_store, security + + +def test_pkce_challenge_is_deterministic(): + verifier = "test-verifier-value" + challenge = _code_challenge_s256(verifier) + assert challenge == _code_challenge_s256(verifier) + assert "=" not in challenge + assert len(_generate_code_verifier()) >= 43 + + +def test_build_authorize_url_contains_pkce_and_state(oauth_stack): + manager, token_store, _ = oauth_stack + url = manager.build_authorize_url("dropbox", "project-1") + assert "dropbox.com/oauth2/authorize" in url + assert "code_challenge=" in url + assert "code_challenge_method=S256" in url + assert "state=" in url + assert "token_access_type=offline" in url + assert "redirect_uri=http%3A%2F%2Flocalhost%3A8000%2Fcollect%2Foauth%2Fdropbox%2Fcallback" in url + + +@patch("collectors.core.oauth_manager.requests.post") +def test_handle_callback_exchanges_and_persists(mock_post, oauth_stack): + manager, token_store, security = oauth_stack + url = manager.build_authorize_url("dropbox", "case-42") + state = url.split("state=")[1].split("&")[0] + + mock_response = MagicMock() + mock_response.status_code = 200 + mock_response.json.return_value = { + "access_token": "access-abc", + "refresh_token": "refresh-xyz", + "token_type": "bearer", + "expires_in": 14400, + "scope": "files.metadata.read files.content.read", + } + mock_post.return_value = mock_response + + auth, project_id, redirect_after = manager.handle_callback( + "dropbox", code="auth-code", state=state + ) + assert project_id == "case-42" + assert redirect_after is None + assert auth.access_token == "access-abc" + assert auth.refresh_token == "refresh-xyz" + + stored = token_store.get_auth_context("case-42", "dropbox", security) + assert stored is not None + assert stored.access_token == "access-abc" + assert stored.refresh_token == "refresh-xyz" + + mock_post.assert_called_once() + payload = mock_post.call_args.kwargs["data"] + assert payload["grant_type"] == "authorization_code" + assert payload["code"] == "auth-code" + assert payload["code_verifier"] + assert payload["client_id"] == "test-client" + assert payload["client_secret"] == "test-secret" + + +def test_handle_callback_rejects_reused_state(oauth_stack): + manager, _, _ = oauth_stack + url = manager.build_authorize_url("dropbox", "p1") + state = url.split("state=")[1].split("&")[0] + + with patch("collectors.core.oauth_manager.requests.post") as mock_post: + mock_post.return_value = MagicMock( + status_code=200, + json=lambda: { + "access_token": "a", + "refresh_token": "r", + "expires_in": 3600, + }, + ) + manager.handle_callback("dropbox", code="c", state=state) + + with pytest.raises(OAuthError, match="Invalid or expired"): + manager.handle_callback("dropbox", code="c", state=state) + + +def test_handle_callback_rejects_expired_state(oauth_stack): + manager, token_store, _ = oauth_stack + state = "expired-state-token" + token_store.save_oauth_state( + state=state, + project_id="p1", + connector_id="dropbox", + code_verifier="verifier", + expires_at=datetime.utcnow() - timedelta(minutes=1), + ) + with pytest.raises(OAuthError, match="Invalid or expired"): + manager.handle_callback("dropbox", code="code", state=state) + + +def test_unsupported_connector_raises(oauth_stack): + manager, _, _ = oauth_stack + with pytest.raises(OAuthError, match="not supported"): + manager.build_authorize_url("box", "1") diff --git a/api/collectors/tests/unit/test_rate_limit_circuit.py b/api/collectors/tests/unit/test_rate_limit_circuit.py new file mode 100644 index 000000000..293c6aeea --- /dev/null +++ b/api/collectors/tests/unit/test_rate_limit_circuit.py @@ -0,0 +1,50 @@ +"""Tests for rate limits and circuit breaker behavior.""" + +from datetime import datetime +from unittest.mock import MagicMock + +import pytest + +from collectors.base.auth_context import AuthContext +from collectors.base.types import RateLimitInfo +from collectors.core.registry import clear_registry, register_connector +from collectors.core.state_manager import StateManager +from collectors.box.box_collector import BoxCollector + + +@pytest.fixture(autouse=True) +def reset_registry(): + clear_registry() + yield + clear_registry() + + +def test_box_handle_rate_limits_retry_after(): + auth = AuthContext(connector_id="box", access_token="tok") + collector = BoxCollector(auth, config={"mock_items": []}) + response = MagicMock() + response.status_code = 429 + response.headers = {"Retry-After": "30"} + info = collector.handle_rate_limits(response) + assert info is not None + assert info.retry_after_seconds == 30.0 + + +def test_circuit_breaker_opens_after_threshold(tmp_path): + state = StateManager(db_path=str(tmp_path / "state.db")) + for _ in range(5): + state.record_failure("box", threshold=5) + assert state.is_circuit_open("box") is True + + +def test_wait_for_rate_limit(monkeypatch): + auth = AuthContext(connector_id="box", access_token="tok") + collector = BoxCollector(auth, config={"mock_items": []}) + slept = [] + + def fake_sleep(seconds): + slept.append(seconds) + + monkeypatch.setattr("collectors.base.base_collector.time.sleep", fake_sleep) + collector.wait_for_rate_limit(RateLimitInfo(retry_after_seconds=15)) + assert slept == [15.0] diff --git a/api/collectors/tests/unit/test_registry.py b/api/collectors/tests/unit/test_registry.py new file mode 100644 index 000000000..deb41fa2a --- /dev/null +++ b/api/collectors/tests/unit/test_registry.py @@ -0,0 +1,56 @@ +"""Tests for connector registry.""" + +import pytest + +from collectors.base.auth_context import AuthContext +from collectors.base.base_collector import BaseCollector +from collectors.base.types import CollectionItem, ItemMetadata, RateLimitInfo +from collectors.core.registry import clear_registry, get_connector_class, list_connectors, register_connector + + +@pytest.fixture(autouse=True) +def reset_registry(): + clear_registry() + yield + clear_registry() + + +class DummyCollector(BaseCollector): + @property + def connector_id(self) -> str: + return "dummy" + + def authenticate(self): + return self.auth + + def list_items(self, since=None): + return [] + + def fetch_item(self, item_id: str): + return CollectionItem(metadata=ItemMetadata(item_id=item_id, name="x"), content=b"") + + def stream_changes(self): + yield from [] + + def handle_rate_limits(self, response): + return None + + def serialize_metadata(self, metadata): + return {"id": metadata.item_id} + + +def test_register_and_lookup(): + registered = register_connector("dummy")(DummyCollector) + assert get_connector_class("dummy") is registered + assert "dummy" in list_connectors() + + +def test_duplicate_registration_raises(): + register_connector("dummy")(DummyCollector) + with pytest.raises(ValueError): + register_connector("dummy")(DummyCollector) + + +def test_unknown_connector_raises(): + with pytest.raises(KeyError): + get_connector_class("missing") diff --git a/api/collectors/tests/unit/test_security_manager.py b/api/collectors/tests/unit/test_security_manager.py new file mode 100644 index 000000000..c6960fc55 --- /dev/null +++ b/api/collectors/tests/unit/test_security_manager.py @@ -0,0 +1,76 @@ +"""Tests for SecurityManager.""" + +import os +import tempfile +from datetime import datetime, timedelta + +import pytest +from cryptography.fernet import Fernet + +from collectors.base.auth_context import AuthContext +from collectors.core.security_manager import SecurityManager + + +@pytest.fixture +def security(): + key = Fernet.generate_key().decode() + with tempfile.TemporaryDirectory() as tmp: + yield SecurityManager(encryption_key=key, secrets_dir=tmp) + + +def test_encrypt_decrypt_roundtrip(security): + encrypted = security.encrypt_token("secret-value") + assert security.decrypt_token(encrypted) == "secret-value" + + +def test_sanitize_redacts_tokens(security): + data = {"access_token": "abc123", "name": "file.txt"} + sanitized = security.sanitize_for_log(data) + assert sanitized["access_token"] == "***REDACTED***" + assert sanitized["name"] == "file.txt" + + +def test_refresh_hook(security): + auth = AuthContext( + connector_id="box", + access_token="old", + expires_at=datetime.utcnow() - timedelta(minutes=5), + ) + + def refresh(ctx): + ctx.access_token = "new" + ctx.expires_at = datetime.utcnow() + timedelta(hours=1) + return ctx + + security.register_refresh_hook("box", refresh) + refreshed = security.refresh_auth_if_needed(auth) + assert refreshed.access_token == "new" + + +def test_load_secrets_from_file(security): + secrets_file = security.secrets_dir / "box.json" + secrets_file.write_text('{"access_token": "file-token", "refresh_token": "refresh"}') + secrets = security.load_connector_secrets("box") + assert secrets["access_token"] == "file-token" + + +def test_get_connector_credentials_prefers_project_tokens(security, tmp_path): + from collectors.core.token_store import TokenStore + + token_store = TokenStore(db_path=str(tmp_path / "state.db")) + (security.secrets_dir / "dropbox.json").write_text( + '{"access_token": "global-token", "refresh_token": "global-refresh"}', + encoding="utf-8", + ) + token_store.save_auth_context( + "proj-1", + "dropbox", + AuthContext(connector_id="dropbox", access_token="oauth-access", refresh_token="oauth-refresh"), + security, + ) + creds = security.get_connector_credentials("dropbox", "proj-1", token_store) + assert creds["access_token"] == "oauth-access" + assert creds["refresh_token"] == "oauth-refresh" + + global_only = security.get_connector_credentials("dropbox", "other", token_store) + assert global_only["access_token"] == "global-token" diff --git a/api/collectors/tests/unit/test_selected_collection.py b/api/collectors/tests/unit/test_selected_collection.py new file mode 100644 index 000000000..bdda695ca --- /dev/null +++ b/api/collectors/tests/unit/test_selected_collection.py @@ -0,0 +1,82 @@ +"""Collect-only-selected ingestion tests.""" + +from datetime import datetime +from unittest.mock import MagicMock + +from collectors.base.auth_context import AuthContext +from collectors.base.types import BrowseItemType, SelectedItem +from collectors.box.box_collector import BoxCollector +from collectors.ingestion.ingestion_pipeline import IngestionPipeline + + +MOCK_BROWSE = { + "0": [ + {"id": "folder-1", "name": "Docs", "path": "/Docs", "type": "folder"}, + {"id": "file-1", "name": "root.txt", "path": "/root.txt", "type": "file", "content": "root"}, + ], + "folder-1": [ + {"id": "file-2", "name": "nested.pdf", "path": "/Docs/nested.pdf", "type": "file", "content": "nested"}, + ], +} + +MOCK_ITEMS = [ + {"id": "file-1", "name": "root.txt", "content": "root"}, + {"id": "file-2", "name": "nested.pdf", "content": "nested"}, +] + + +def test_expand_selection_folder_recursive(): + auth = AuthContext(connector_id="box", access_token="dev") + collector = BoxCollector( + auth=auth, + config={"mock_browse": MOCK_BROWSE, "mock_items": MOCK_ITEMS, "mock_refresh": True}, + ) + selected = [SelectedItem(id="folder-1", type=BrowseItemType.FOLDER, name="Docs", path="/Docs")] + expanded = collector.expand_selection(selected) + assert {item.item_id for item in expanded} == {"file-2"} + + +def test_expand_selection_mixed(): + auth = AuthContext(connector_id="box", access_token="dev") + collector = BoxCollector( + auth=auth, + config={"mock_browse": MOCK_BROWSE, "mock_items": MOCK_ITEMS, "mock_refresh": True}, + ) + selected = [ + SelectedItem(id="file-1", type=BrowseItemType.FILE, name="root.txt", path="/root.txt"), + SelectedItem(id="folder-1", type=BrowseItemType.FOLDER, name="Docs", path="/Docs"), + ] + expanded = collector.expand_selection(selected) + assert {item.item_id for item in expanded} == {"file-1", "file-2"} + + +def test_ingestion_pipeline_selected_only(tmp_path): + auth = AuthContext(connector_id="box", access_token="dev") + collector = BoxCollector( + auth=auth, + config={ + "mock_browse": MOCK_BROWSE, + "mock_items": MOCK_ITEMS, + "mock_refresh": True, + "selected_items": [ + {"id": "file-1", "type": "file", "name": "root.txt", "path": "/root.txt"}, + ], + }, + ) + + filesystem = MagicMock() + filesystem.write_item.return_value = str(tmp_path / "root.txt") + metadata_store = MagicMock() + metadata_store.exists.return_value = False + state_manager = MagicMock() + + pipeline = IngestionPipeline( + collector=collector, + filesystem=filesystem, + metadata_store=metadata_store, + state_manager=state_manager, + ) + result = pipeline.run() + assert result["items_collected"] == 1 + assert result["items_failed"] == 0 + filesystem.write_item.assert_called_once() diff --git a/api/collectors/tests/unit/test_selection_store.py b/api/collectors/tests/unit/test_selection_store.py new file mode 100644 index 000000000..e12812789 --- /dev/null +++ b/api/collectors/tests/unit/test_selection_store.py @@ -0,0 +1,41 @@ +"""Selection persistence tests.""" + +import os + +from collectors.base.types import BrowseItemType, SelectedItem +from collectors.core.selection_store import SelectionStore + + +def test_save_and_get_selections(tmp_path): + db_path = str(tmp_path / "state.db") + store = SelectionStore(db_path=db_path) + + items = [ + SelectedItem(id="f1", type=BrowseItemType.FOLDER, name="Docs", path="/Docs"), + SelectedItem(id="file-1", type=BrowseItemType.FILE, name="a.txt", path="/a.txt"), + ] + saved = store.save_selections("project-1", "box", items) + assert saved["project_id"] == "project-1" + assert len(saved["items"]) == 2 + + rows = store.get_selections("project-1", "box") + assert len(rows) == 1 + assert rows[0]["items"][0]["type"] == "folder" + + all_rows = store.get_selections("project-1") + assert len(all_rows) == 1 + + loaded = store.get_selected_items("project-1", "box") + assert loaded[1].name == "a.txt" + + +def test_selection_store_shares_db_with_state_manager(tmp_path): + db_path = str(tmp_path / "shared.db") + os.environ["COLLECTOR_STATE_DB"] = db_path + store = SelectionStore() + store.save_selections( + "1", + "dropbox", + [SelectedItem(id="/memo.pdf", type=BrowseItemType.FILE, path="/memo.pdf")], + ) + assert store.get_selections("1", "dropbox")[0]["connector_id"] == "dropbox" diff --git a/api/collectors/tests/unit/test_state_manager.py b/api/collectors/tests/unit/test_state_manager.py new file mode 100644 index 000000000..7ceac6c84 --- /dev/null +++ b/api/collectors/tests/unit/test_state_manager.py @@ -0,0 +1,51 @@ +"""Tests for StateManager.""" + +import os +import tempfile + +import pytest + +from collectors.base.types import CollectionJob, CollectionJobStatus, CircuitBreakerState +from collectors.core.state_manager import StateManager + + +@pytest.fixture +def state_db(): + with tempfile.TemporaryDirectory() as tmp: + yield StateManager(db_path=os.path.join(tmp, "state.db")) + + +def test_connector_state_defaults(state_db): + state = state_db.get_connector_state("box") + assert state["connector_id"] == "box" + assert state["circuit_breaker_state"] == CircuitBreakerState.CLOSED.value + + +def test_record_success_resets_circuit(state_db): + state_db.record_failure("box", threshold=1) + assert state_db.get_connector_state("box")["circuit_breaker_state"] == CircuitBreakerState.OPEN.value + state_db.record_success("box", sync_token="token-1") + updated = state_db.get_connector_state("box") + assert updated["last_sync_token"] == "token-1" + assert updated["circuit_breaker_state"] == CircuitBreakerState.CLOSED.value + + +def test_failed_items_tracking(state_db): + state_db.add_failed_item("box", "file-1") + state_db.add_failed_item("box", "file-1") + failed = state_db.get_connector_state("box")["failed_item_ids"] + assert failed == ["file-1"] + + +def test_job_persistence(state_db): + job = CollectionJob( + job_id="job-1", + connector_id="box", + status=CollectionJobStatus.COMPLETED, + items_collected=3, + ) + state_db.save_job(job) + loaded = state_db.get_job("job-1") + assert loaded is not None + assert loaded.items_collected == 3 + assert loaded.status == CollectionJobStatus.COMPLETED diff --git a/api/collectors/tests/unit/test_token_store.py b/api/collectors/tests/unit/test_token_store.py new file mode 100644 index 000000000..6445b02e2 --- /dev/null +++ b/api/collectors/tests/unit/test_token_store.py @@ -0,0 +1,46 @@ +"""Tests for project-scoped OAuth token store.""" + +import tempfile + +import pytest +from cryptography.fernet import Fernet + +from collectors.base.auth_context import AuthContext +from collectors.core.security_manager import SecurityManager +from collectors.core.token_store import TokenStore + + +@pytest.fixture +def store_and_security(): + key = Fernet.generate_key().decode() + with tempfile.TemporaryDirectory() as tmp: + security = SecurityManager(encryption_key=key, secrets_dir=tmp) + token_store = TokenStore(db_path=f"{tmp}/state.db") + yield token_store, security + + +def test_save_and_load_encrypted_tokens(store_and_security): + token_store, security = store_and_security + auth = AuthContext( + connector_id="dropbox", + access_token="plain-access", + refresh_token="plain-refresh", + ) + token_store.save_auth_context("1", "dropbox", auth, security) + loaded = token_store.get_auth_context("1", "dropbox", security) + assert loaded is not None + assert loaded.access_token == "plain-access" + assert loaded.refresh_token == "plain-refresh" + assert loaded.extra.get("project_id") == "1" + + +def test_has_tokens(store_and_security): + token_store, security = store_and_security + assert not token_store.has_tokens("1", "dropbox") + token_store.save_auth_context( + "1", + "dropbox", + AuthContext(connector_id="dropbox", refresh_token="r"), + security, + ) + assert token_store.has_tokens("1", "dropbox") diff --git a/api/main.py b/api/main.py index c70f6cc29..95b4c1589 100644 --- a/api/main.py +++ b/api/main.py @@ -1,17 +1,39 @@ +from contextlib import asynccontextmanager from fastapi import FastAPI, File, UploadFile, HTTPException, BackgroundTasks from fastapi.middleware.cors import CORSMiddleware from fastapi.responses import JSONResponse -from pydantic import BaseModel +from fastapi.staticfiles import StaticFiles +from pydantic import BaseModel, Field import jpype import jpype.imports import os -from typing import Optional, List +from typing import Any, Dict, Optional, List import tempfile import logging import sys import subprocess from pathlib import Path +# Register collectors on startup +from collectors.base.types import BrowseItemType, SelectedItem +from collectors.core.browse_service import ( + BrowseService, + ConnectorAuthError, + CredentialsNotConfiguredError, +) +from collectors.core.oauth_manager import OAuthManager +from collectors.core.oauth_routes import create_oauth_router +from collectors.core.orchestrator import Orchestrator +from collectors.core.registry import list_connectors +from collectors.core.selection_store import SelectionStore +from collectors.core.state_manager import StateManager +from collectors.core.token_store import TokenStore +from collectors.core.trigger_manager import TriggerManager + +import collectors.box.box_collector # noqa: F401 +import collectors.google_drive.gdrive_collector # noqa: F401 +import collectors.dropbox.dropbox_collector # noqa: F401 + # Configure logging logging.basicConfig( level=logging.INFO, @@ -22,7 +44,30 @@ ) logger = logging.getLogger(__name__) +_collector_state = StateManager() +_collector_token_store = TokenStore(db_path=_collector_state.db_path) +_collector_selections = SelectionStore(db_path=_collector_state.db_path) +_collector_browse = BrowseService(token_store=_collector_token_store) +_collector_oauth = OAuthManager(token_store=_collector_token_store) +_collector_orchestrator = Orchestrator(state_manager=_collector_state) +_collector_trigger = TriggerManager( + _collector_state, + run_callback=_collector_orchestrator.run_collection, +) + + +@asynccontextmanager +async def lifespan(app: FastAPI): + try: + _collector_trigger.load_schedules_from_db() + except Exception as exc: + logger.warning("Failed to load collector schedules: %s", exc) + yield + _collector_trigger.shutdown() + + app = FastAPI( + lifespan=lifespan, title="FreeEed Processing API", description=""" API for processing documents using FreeEed. This API provides endpoints to: @@ -45,6 +90,12 @@ redoc_url="/redoc" ) +app.include_router(create_oauth_router(_collector_oauth)) + +_collectors_static = Path(__file__).resolve().parent / "static" / "collectors" +if _collectors_static.is_dir(): + app.mount("/collectors", StaticFiles(directory=str(_collectors_static), html=True), name="collectors-ui") + class ProjectConfig(BaseModel): """ Configuration for document processing @@ -202,16 +253,313 @@ async def process_project_file( logger.error(f"Error processing project file: {str(e)}") raise HTTPException(status_code=500, detail=str(e)) +class CollectTriggerRequest(BaseModel): + connector_id: str + auto_chain: bool = False + config: Dict[str, Any] = Field(default_factory=dict) + + +class CollectScheduleRequest(BaseModel): + schedule_id: str + connector_id: str + cron_expression: str + auto_chain: bool = False + config: Dict[str, Any] = Field(default_factory=dict) + + +class ConnectRequest(BaseModel): + access_token: Optional[str] = None + refresh_token: Optional[str] = None + config: Dict[str, Any] = Field(default_factory=dict) + + +class SelectionItemRequest(BaseModel): + id: str + type: str = "file" + name: str = "" + path: str = "" + + +class SaveSelectionsRequest(BaseModel): + connector_id: str + items: List[SelectionItemRequest] + + +class ProjectCollectTriggerRequest(BaseModel): + connector_id: str + auto_chain: bool = False + config: Dict[str, Any] = Field(default_factory=dict) + + +@app.post("/collect/trigger", summary="Trigger a manual collection job") +async def collect_trigger(request: CollectTriggerRequest): + """Start a collection job for the given connector.""" + try: + job = _collector_trigger.trigger_manual( + connector_id=request.connector_id, + auto_chain=request.auto_chain, + config=request.config, + ) + return { + "job_id": job.job_id, + "connector_id": job.connector_id, + "status": job.status.value, + } + except KeyError: + raise HTTPException(status_code=404, detail=f"Unknown connector: {request.connector_id}") + except Exception as e: + logger.error("Collect trigger failed: %s", e) + raise HTTPException(status_code=500, detail=str(e)) + + +@app.get("/collect/jobs/{job_id}", summary="Get collection job status") +async def collect_job_status(job_id: str): + job = _collector_state.get_job(job_id) + if not job: + raise HTTPException(status_code=404, detail=f"Job not found: {job_id}") + return { + "job_id": job.job_id, + "connector_id": job.connector_id, + "status": job.status.value, + "created_at": job.created_at.isoformat() if job.created_at else None, + "started_at": job.started_at.isoformat() if job.started_at else None, + "completed_at": job.completed_at.isoformat() if job.completed_at else None, + "output_dir": job.output_dir, + "items_collected": job.items_collected, + "items_failed": job.items_failed, + "error_message": job.error_message, + "auto_chain": job.auto_chain, + } + + +@app.get("/collect/connectors", summary="List registered connectors") +async def collect_list_connectors(): + return {"connectors": list_connectors()} + + +def _connector_or_404(connector_id: str) -> None: + if connector_id not in list_connectors(): + raise HTTPException(status_code=404, detail=f"Unknown connector: {connector_id}") + + +def _token_override_from_request(request: Optional[ConnectRequest]) -> Optional[Dict[str, Any]]: + if not request: + return None + override: Dict[str, Any] = {} + if request.access_token: + override["access_token"] = request.access_token + if request.refresh_token: + override["refresh_token"] = request.refresh_token + return override or None + + +@app.post( + "/collect/projects/{project_id}/connect/{connector_id}", + summary="Initiate or verify connector connection for a project", +) +async def collect_project_connect( + project_id: str, + connector_id: str, + request: Optional[ConnectRequest] = None, +): + """Verify OAuth/token credentials for a project-scoped connector session.""" + _connector_or_404(connector_id) + try: + result = _collector_browse.connect( + connector_id=connector_id, + project_id=project_id, + token_override=_token_override_from_request(request), + config=request.config if request else None, + ) + return {"project_id": project_id, **result} + except CredentialsNotConfiguredError as exc: + raise HTTPException(status_code=503, detail=str(exc)) + except ConnectorAuthError as exc: + raise HTTPException(status_code=401, detail=str(exc)) + except KeyError: + raise HTTPException(status_code=404, detail=f"Unknown connector: {connector_id}") + except Exception as e: + logger.error("Project connect failed: %s", e) + raise HTTPException(status_code=500, detail=str(e)) + + +@app.get( + "/collect/projects/{project_id}/browse/{connector_id}", + summary="Browse remote folder children for a project", +) +async def collect_project_browse( + project_id: str, + connector_id: str, + parent_id: Optional[str] = None, + access_token: Optional[str] = None, + refresh_token: Optional[str] = None, +): + """List folders and files at the remote parent (root when parent_id omitted).""" + _connector_or_404(connector_id) + token_override = None + if access_token or refresh_token: + token_override = { + k: v + for k, v in {"access_token": access_token, "refresh_token": refresh_token}.items() + if v + } + try: + items = _collector_browse.browse( + connector_id=connector_id, + parent_id=parent_id, + project_id=project_id, + token_override=token_override, + ) + return { + "project_id": project_id, + "connector_id": connector_id, + "parent_id": parent_id, + "items": [_collector_browse.serialize_browse_item(item) for item in items], + } + except CredentialsNotConfiguredError as exc: + raise HTTPException(status_code=503, detail=str(exc)) + except ConnectorAuthError as exc: + raise HTTPException(status_code=401, detail=str(exc)) + except ValueError as exc: + raise HTTPException(status_code=400, detail=str(exc)) + except Exception as e: + logger.error("Project browse failed: %s", e) + raise HTTPException(status_code=500, detail=str(e)) + + +@app.post( + "/collect/projects/{project_id}/selections", + summary="Save selected remote items for a project", +) +async def collect_project_save_selections(project_id: str, request: SaveSelectionsRequest): + _connector_or_404(request.connector_id) + items = [ + SelectedItem( + id=item.id, + type=BrowseItemType(item.type), + name=item.name, + path=item.path, + ) + for item in request.items + ] + saved = _collector_selections.save_selections(project_id, request.connector_id, items) + return saved + + +@app.get( + "/collect/projects/{project_id}/selections", + summary="Retrieve saved remote selections for a project", +) +async def collect_project_get_selections( + project_id: str, + connector_id: Optional[str] = None, +): + if connector_id: + _connector_or_404(connector_id) + return { + "project_id": project_id, + "selections": _collector_selections.get_selections(project_id, connector_id), + } + + +@app.post( + "/collect/projects/{project_id}/trigger", + summary="Trigger collection for saved project selections only", +) +async def collect_project_trigger(project_id: str, request: ProjectCollectTriggerRequest): + """Run collection limited to items saved via the selections API.""" + _connector_or_404(request.connector_id) + selected = _collector_selections.get_selected_items(project_id, request.connector_id) + if not selected: + raise HTTPException( + status_code=400, + detail=( + f"No selections saved for project '{project_id}' " + f"and connector '{request.connector_id}'" + ), + ) + + config = dict(request.config) + config["project_id"] = project_id + config["selected_items"] = [ + { + "id": item.id, + "type": item.type.value, + "name": item.name, + "path": item.path, + } + for item in selected + ] + processing = config.setdefault("processing", {}) + processing.setdefault("project_code", project_id) + if "project_name" not in processing: + processing["project_name"] = f"collection-{project_id}" + + try: + job = _collector_trigger.trigger_manual( + connector_id=request.connector_id, + auto_chain=request.auto_chain, + config=config, + ) + return { + "project_id": project_id, + "job_id": job.job_id, + "connector_id": job.connector_id, + "status": job.status.value, + "selection_count": len(selected), + } + except KeyError: + raise HTTPException(status_code=404, detail=f"Unknown connector: {request.connector_id}") + except Exception as e: + logger.error("Project collect trigger failed: %s", e) + raise HTTPException(status_code=500, detail=str(e)) + + +@app.get("/collect/connectors/{connector_id}/health", summary="Connector health check") +async def collect_connector_health(connector_id: str): + if connector_id not in list_connectors(): + raise HTTPException(status_code=404, detail=f"Unknown connector: {connector_id}") + return _collector_orchestrator.get_connector_health(connector_id) + + +@app.post("/collect/schedules", summary="Register a cron collection schedule") +async def collect_create_schedule(request: CollectScheduleRequest): + try: + _collector_trigger.add_schedule( + schedule_id=request.schedule_id, + connector_id=request.connector_id, + cron_expression=request.cron_expression, + auto_chain=request.auto_chain, + config=request.config, + ) + return { + "schedule_id": request.schedule_id, + "connector_id": request.connector_id, + "cron_expression": request.cron_expression, + "status": "registered", + } + except ValueError as e: + raise HTTPException(status_code=400, detail=str(e)) + except KeyError: + raise HTTPException(status_code=404, detail=f"Unknown connector: {request.connector_id}") + + @app.get("/health", summary="Check API health status", description="Returns the health status of the API and its components.") async def health_check(): """Check if the service is running""" + collector_status = _collector_orchestrator.subsystem_status() + overall = "healthy" + if collector_status.get("status") != "healthy": + overall = "degraded" + return { - "status": "healthy", + "status": overall, "message": "Service is running", "components": { "api": "healthy", - "java": "running" if jpype.isJVMStarted() else "not running" + "java": "running" if jpype.isJVMStarted() else "not running", + "collectors": collector_status, } } diff --git a/api/requirements-collectors.txt b/api/requirements-collectors.txt new file mode 100644 index 000000000..8450269c3 --- /dev/null +++ b/api/requirements-collectors.txt @@ -0,0 +1,11 @@ +# Collector framework dependencies (install alongside api/requirements.txt) +APScheduler==3.10.4 +cryptography==42.0.8 +boxsdk==3.9.2 +google-api-python-client==2.131.0 +google-auth==2.29.0 +google-auth-httplib2==0.2.0 +dropbox>=12.0.2,<13 +requests==2.32.3 +pytest==8.2.2 +pytest-asyncio==0.23.7 diff --git a/api/static/collectors/index.html b/api/static/collectors/index.html new file mode 100644 index 000000000..92d2318be --- /dev/null +++ b/api/static/collectors/index.html @@ -0,0 +1,76 @@ + + + + + + Collection Browser — M9 + + + +

Collection Browser — M9

+

Placeholder scaffold. Full lazy tree, OAuth, and multi-select UI planned under M9.

+ + + + + + +
+ 
Click Browse to call GET /collect/projects/{project_id}/browse/{connector_id}
+ + + + diff --git a/docker-compose.yml b/docker-compose.yml index b651fa663..c258d2339 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -14,11 +14,15 @@ services: - ./freeeed-processing/test-data:/app/test-data - ./freeeed-processing/test-data/09-projects:/app/projects - sqlite_data:/app/db + - ./secrets:/app/secrets:ro environment: - JAVA_HOME=/usr/lib/jvm/java-11-openjdk-amd64 - PYTHONUNBUFFERED=1 - FREEEED_JAR=/app/freeeed-processing.jar - SQLITE_DB=/app/db/freeeed.db + - COLLECTOR_STATE_DB=/app/db/collector_state.db + - COLLECTOR_SECRETS_DIR=/app/secrets + - SOLR_ENDPOINT=http://solr:8983 networks: - freeeed-network restart: unless-stopped diff --git a/docs/collector-framework/WORKFLOW.md b/docs/collector-framework/WORKFLOW.md new file mode 100644 index 000000000..667ddeb57 --- /dev/null +++ b/docs/collector-framework/WORKFLOW.md @@ -0,0 +1,306 @@ +# External Data Collection Framework — GitHub Workflow + +This document describes how collector-framework work is tracked, branched, reviewed, and promoted between the **public** FreeEed platform repo and the **private** companion repo. + +## Repository strategy + +| Repository | Visibility | Role | +|------------|------------|------| +| [shmsoft/FreeEed](https://github.com/shmsoft/FreeEed) | **Public** | Upstream eDiscovery platform; receives merged collector code only when release-ready | +| `shmsoft/FreeEed-collectors` (planned) | **Private** | All development branches, PRs, issues, CI, project boards, and internal discussion | + +**Why a private companion repo?** The public FreeEed repo cannot host hidden issues, draft PRs, or internal milestone tracking. Collector work stays segregated until deliberately promoted at **M8**. + +### What lives where + +| Artifact | Private repo | Public repo (now) | Public repo (after M8) | +|----------|--------------|-------------------|------------------------| +| Issue templates (`.github/ISSUE_TEMPLATE/`) | Yes | Staged locally / copy on bootstrap | Optional subset | +| PR template | Yes | Staged locally | Optional | +| `collectors-ci.yml` | Yes | Staged locally | Yes (when `api/collectors/` lands) | +| `api/collectors/` source | Yes (during dev) | No until promotion | Yes | +| `docs/collector-framework/` | Yes | Yes (tracking docs only) | Yes | + +The `.github/` and `docs/collector-framework/` files in this tree can be **copied wholesale** when creating `FreeEed-collectors`, or kept here as a reference until the private repo exists. + +--- + +## Milestones (M0–M9) + +Create these milestones in the private repo (GitHub → Issues → Milestones): + +| Milestone | Scope | Primary deliverables | +|-----------|-------|----------------------| +| **M0 — GitHub & scaffolding** | Work tracking setup | Private repo, GitHub Project v2, labels, issue/PR templates, CI skeleton, `api/collectors/` package scaffold | +| **M1 — Core framework** | Foundation | `BaseCollector` ABC, types, registry, orchestrator skeleton, logging/state/security managers | +| **M2 — Ingestion & storage** | Data path | Ingestion pipeline, streaming engine, metadata normalizer, filesystem + SQLite stores | +| **M3 — Trigger & processing** | FreeEed integration | Trigger manager (manual + scheduled), `.project` writer, `processing_engine_client`, auto-chain | +| **M4 — Box connector** | Reference connector | Box auth, list/fetch/sync, rate limits, unit tests | +| **M5 — Drive & Dropbox** | Remaining v1 connectors | Google Drive + Dropbox collectors, `new_connector_template.py` | +| **M6 — API & deployment** | Surface + ops | FastAPI `/collect/*` routes, Dockerfile + docker-compose updates, env/secrets docs | +| **M7 — Tests & docs** | Quality + knowledge | Integration tests (VCR/sandbox), `architecture.md`, `connectors.md`, `ingestion_flow.md` | +| **M8 — Public promotion** | Upstream merge | Curated PR to public `shmsoft/FreeEed`, sanitized release notes, CHANGELOG | +| **M9 — Collection Selection UI** | Interactive browse/select | Case-scoped browser for Box/Drive/Dropbox; in-app OAuth; lazy tree, breadcrumbs, multi-select; project/custodian binding, validation, trigger + job status. **Default:** small web UI via FastAPI static mount; **alternative:** Swing panel | + +Align every issue and PR with exactly one milestone unless explicitly spanning tightly coupled work (document in both issue bodies). + +--- + +## Label taxonomy + +Apply labels consistently for filtering and project automation. + +### Area + +| Label | Use for | +|-------|---------| +| `area:core` | Registry, orchestrator, base types, trigger manager | +| `area:ingestion` | Pipeline, streaming, metadata normalization | +| `area:storage` | Filesystem/SQLite stores, state persistence | +| `area:connectors` | Box, Drive, Dropbox, template connector | +| `area:api` | FastAPI routes, request/response models | +| `area:security` | Secrets, token encryption, log sanitization | +| `area:ci` | GitHub Actions, test gates | +| `area:docs` | Markdown docs, README sections | + +### Connector + +| Label | Use for | +|-------|---------| +| `connector:box` | Box.com connector | +| `connector:gdrive` | Google Drive connector | +| `connector:dropbox` | Dropbox connector | +| `connector:template` | New-connector boilerplate | + +Use **no connector label** (or `connector: none` in issue forms) for cross-cutting framework work. + +### Type + +| Label | Use for | +|-------|---------| +| `type:feature` | New capability | +| `type:bug` | Defect fix | +| `type:tech-debt` | Refactor, cleanup | +| `type:spike` | Time-boxed research | +| `type:docs` | Documentation-only change | + +### Priority + +| Label | Use for | +|-------|---------| +| `priority:P0` | Blocker / production break | +| `priority:P1` | Current milestone must-have | +| `priority:P2` | Nice-to-have / next milestone | + +### Epic & status + +| Label | Use for | +|-------|---------| +| `epic:collector-framework` | **All** initiative issues | +| `status:blocked` | Waiting on dependency or decision | +| `status:needs-review` | PR ready for maintainer review | + +--- + +## GitHub Project (v2) + +**Board name:** External Data Collection Framework + +| Column / status | Meaning | +|-----------------|---------| +| Backlog | Planned, not started | +| In progress | Active implementation | +| In review | Open PR awaiting review | +| Blocked | External dependency | +| Done | Merged to private repo `main` | + +Link every issue and PR to this project. Update status when work moves forward. + +--- + +## Branch naming + +| Pattern | Use for | +|---------|---------| +| `feature/collector-` | New features, connectors, subsystems | +| `fix/collector-` | Bug fixes | +| `docs/collector-` | Documentation-only changes | + +Examples: + +- `feature/collector-box-auth` +- `feature/collector-ingestion-pipeline` +- `fix/collector-rate-limit-backoff` +- `docs/collector-oauth-setup` + +**Rules:** + +- Branch from private repo `main` (or `dev` if the private repo mirrors public conventions). +- Do **not** push in-progress collector feature branches to the public `FreeEed` remote. +- One logical change set per branch; prefer one issue per PR. + +--- + +## Traceability rules + +1. **One issue per PR** — use `Closes #NNN` in the PR description. For tightly coupled work, `Closes #a, Closes #b` is acceptable if documented in both issues. +2. **No drive-by commits** — every commit should relate to the linked issue. +3. **Commit message format:** + ``` + (collector): (#) + ``` + Types: `feat`, `fix`, `docs`, `test`, `refactor`, `ci`, `chore`. + + Example: `feat(collector): add Box OAuth refresh (#42)` +4. **Issue title prefix:** `[COLLECTOR]` on all framework issues. +5. **Project board** — move issues when status changes (manual or via GitHub Actions `actions/add-to-project`). +6. **CHANGELOG** — maintain `api/collectors/CHANGELOG.md` in the private repo; entries reference issue/PR numbers. +7. **Traceability matrix** — update [`traceability.md`](traceability.md) at milestone boundaries. + +--- + +## Creating the private repo with GitHub CLI + +Prerequisites: [GitHub CLI](https://cli.github.com/) authenticated (`gh auth login`). + +### 1. Create the private repository + +```bash +gh repo create shmsoft/FreeEed-collectors \ + --private \ + --description "Private development repo for FreeEed External Data Collection Framework" \ + --clone +``` + +If org policy requires a personal fork first, create under your account and transfer later. + +### 2. Bootstrap from FreeEed upstream + +```bash +cd FreeEed-collectors +git remote add upstream https://github.com/shmsoft/FreeEed.git +git fetch upstream +git checkout -b main +git pull upstream main # or merge a specific baseline commit +``` + +Copy staged artifacts from the public repo (or this local tree): + +```bash +# From your FreeEed clone +cp -r .github/ISSUE_TEMPLATE .github/pull_request_template.md \ + .github/workflows/collectors-ci.yml .github/CODEOWNERS \ + /path/to/FreeEed-collectors/.github/ +cp -r docs/collector-framework /path/to/FreeEed-collectors/docs/ +``` + +Commit and push: + +```bash +git add .github docs/collector-framework +git commit -m "chore(collector): bootstrap GitHub tracking artifacts (M0)" +git push -u origin main +``` + +### 3. Create milestones + +```bash +for title in \ + "M0 — GitHub & scaffolding" \ + "M1 — Core framework" \ + "M2 — Ingestion & storage" \ + "M3 — Trigger & processing" \ + "M4 — Box connector" \ + "M5 — Drive & Dropbox" \ + "M6 — API & deployment" \ + "M7 — Tests & docs" \ + "M8 — Public promotion" \ + "M9 — Collection Selection UI" +do + gh api repos/shmsoft/FreeEed-collectors/milestones -f title="$title" -f state=open +done +``` + +### 4. Create labels + +```bash +# Area +for label in core ingestion storage connectors api security ci docs; do + gh label create "area:$label" --repo shmsoft/FreeEed-collectors --color "1D76DB" --force +done + +# Connector +for label in box gdrive dropbox template; do + gh label create "connector:$label" --repo shmsoft/FreeEed-collectors --color "5319E7" --force +done + +# Type +for label in feature bug tech-debt spike docs; do + gh label create "type:$label" --repo shmsoft/FreeEed-collectors --color "0E8A16" --force +done + +# Priority +for label in P0 P1 P2; do + gh label create "priority:$label" --repo shmsoft/FreeEed-collectors --color "B60205" --force +done + +# Epic & status +gh label create "epic:collector-framework" --repo shmsoft/FreeEed-collectors --color "006B75" --force +gh label create "status:blocked" --repo shmsoft/FreeEed-collectors --color "D93F0B" --force +gh label create "status:needs-review" --repo shmsoft/FreeEed-collectors --color "FBCA04" --force +``` + +### 5. Create GitHub Project v2 + +Use the GitHub UI: **Projects → New project → Board** → name **External Data Collection Framework**. Add columns: Backlog, In progress, In review, Blocked, Done. + +Optionally link the project to the repo under project settings. + +### 6. Seed M0 issues + +Open tasks using the **Collector Task** template, for example: + +- `[COLLECTOR] Create milestones and labels` +- `[COLLECTOR] Configure collectors-ci workflow` +- `[COLLECTOR] Scaffold api/collectors/ package layout` + +--- + +## Promotion path to public repo (M8) + +When a milestone (typically **M8**) is release-ready: + +1. **Sanitize** — remove internal ticket references, customer names, and sandbox credentials from PR title/body. +2. **Verify security checklist** — no secrets, no telemetry, logs sanitized (see PR template). +3. **Run full CI** — unit tests required; integration tests with recorded cassettes or documented skip. +4. **Open upstream PR** to `shmsoft/FreeEed` `main` or `dev` (per team convention): + - Single squash/rebase PR preferred for the initial promotion. + - Include user-facing CHANGELOG entry and docs links. +5. **Close promoted issues** in the private repo; note the public PR URL in each issue. +6. **Decide ongoing model** — either continue private-first for v2 connectors, or shift new work to public repo with `area:connectors` labels. + +Per-milestone partial promotion is allowed (e.g. docs-only PRs) if the security and sanitization bars are met. + +--- + +## Agent / Cursor instructions + +When implementing collector work: + +1. Create or reference a GitHub issue in the **private** repo before coding. +2. Branch using `feature/collector-*`, `fix/collector-*`, or `docs/collector-*`. +3. Open PR against private repo `main`; link issue; set milestone and labels. +4. Do not push feature branches to the public FreeEed remote. +5. Record the PR URL in the issue for audit trail. +6. Update [`traceability.md`](traceability.md) at milestone completion. + +--- + +## Related files + +| File | Purpose | +|------|---------| +| [`.github/ISSUE_TEMPLATE/`](../.github/ISSUE_TEMPLATE/) | Issue forms (feature, bug, task) | +| [`.github/pull_request_template.md`](../.github/pull_request_template.md) | PR checklist | +| [`.github/workflows/collectors-ci.yml`](../.github/workflows/collectors-ci.yml) | CI for collector unit/integration tests | +| [`traceability.md`](traceability.md) | Milestone → issues → PRs → deliverables matrix | diff --git a/docs/collector-framework/traceability.md b/docs/collector-framework/traceability.md new file mode 100644 index 000000000..d6957f831 --- /dev/null +++ b/docs/collector-framework/traceability.md @@ -0,0 +1,263 @@ +# Collector Framework — Traceability Matrix + +Track milestone progress: **Milestone → Issues → PRs → Deliverables**. + +Update this file at each milestone boundary (or when major deliverables land). Issue and PR columns start as placeholders until work begins in the private `FreeEed-collectors` repo. + +**Legend:** `-` = not started | `#NNN` = GitHub issue/PR number | `✓` = complete + +--- + +## Summary matrix + +| Milestone | Issues | PRs | Status | Deliverables | +|-----------|--------|-----|--------|--------------| +| M0 — GitHub & scaffolding | — | — | Planned | See [M0](#m0--github--scaffolding) | +| M1 — Core framework | — | — | Planned | See [M1](#m1--core-framework) | +| M2 — Ingestion & storage | — | — | Planned | See [M2](#m2--ingestion--storage) | +| M3 — Trigger & processing | — | — | Planned | See [M3](#m3--trigger--processing) | +| M4 — Box connector | — | — | Planned | See [M4](#m4--box-connector) | +| M5 — Drive & Dropbox | — | — | Planned | See [M5](#m5--drive--dropbox) | +| M6 — API & deployment | — | — | Planned | See [M6](#m6--api--deployment) | +| M7 — Tests & docs | — | — | Planned | See [M7](#m7--tests--docs) | +| M8 — Public promotion | — | — | Planned | See [M8](#m8--public-promotion) | +| M9 — Collection Selection UI | — | — | Planned | See [M9](#m9--collection-selection-ui) | + +--- + +## M0 — GitHub & scaffolding + +| Field | Value | +|-------|-------| +| **Issues** | — | +| **PRs** | — | +| **Target** | Private repo bootstrap, tracking artifacts, CI skeleton | + +### Planned deliverables + +- [ ] Private repo `shmsoft/FreeEed-collectors` created +- [ ] GitHub Project v2: **External Data Collection Framework** (Backlog → Done columns) +- [ ] Milestones M0–M8 created +- [ ] Label taxonomy applied (`area:*`, `connector:*`, `type:*`, `priority:*`, `epic:collector-framework`) +- [ ] Issue templates: `feature.yml`, `bug.yml`, `task.yml` +- [ ] PR template with security + public-promotion checklist +- [ ] Workflow: `.github/workflows/collectors-ci.yml` (unit tests; integration gated) +- [ ] `docs/collector-framework/WORKFLOW.md` and this traceability matrix +- [ ] `api/collectors/` package scaffold (`__init__.py`, directory layout per architecture plan) +- [ ] Initial M0 seed issues opened and linked to project board + +--- + +## M1 — Core framework + +| Field | Value | +|-------|-------| +| **Issues** | — | +| **PRs** | — | +| **Target** | Base types, registry, orchestrator, cross-cutting managers | + +### Planned deliverables + +- [ ] `api/collectors/base/types.py` — `ItemMetadata`, `ItemContent`, `ChangeEvent`, `HealthStatus` +- [ ] `api/collectors/base/auth_context.py` — auth context model +- [ ] `api/collectors/base/base_collector.py` — `BaseCollector` ABC (mandatory interface) +- [ ] `api/collectors/core/registry.py` — `@register_connector` plugin discovery +- [ ] `api/collectors/core/orchestrator.py` — collector orchestration skeleton +- [ ] `api/collectors/core/logging_manager.py` — structured JSON logging +- [ ] `api/collectors/core/state_manager.py` — sync tokens, failed items, circuit breaker state +- [ ] `api/collectors/core/security_manager.py` — secrets loading, token encryption, log redaction skeleton +- [ ] Unit tests: registry, state manager, security encrypt/decrypt + redaction +- [ ] `api/requirements-collectors.txt` — pinned OSS deps (pytest, tenacity, etc.) + +--- + +## M2 — Ingestion & storage + +| Field | Value | +|-------|-------| +| **Issues** | — | +| **PRs** | — | +| **Target** | End-to-end collect → normalize → persist (no external connectors yet) | + +### Planned deliverables + +- [ ] `api/collectors/ingestion/streaming_engine.py` — concurrent fetch with limits +- [ ] `api/collectors/ingestion/metadata_normalizer.py` — unified metadata schema +- [ ] `api/collectors/ingestion/ingestion_pipeline.py` — orchestrated ingest flow +- [ ] `api/collectors/storage/storage_interface.py` — pluggable storage seam +- [ ] `api/collectors/storage/filesystem_store.py` — job-scoped dirs under `/data/input/collections//` +- [ ] `api/collectors/storage/sqlite_store.py` — metadata + job manifest in SQLite +- [ ] Unit tests: normalizer, filesystem store, SQLite store, pipeline (mock collector) +- [ ] Idempotent writes (content hash / item_id dedup) + +--- + +## M3 — Trigger & processing + +| Field | Value | +|-------|-------| +| **Issues** | — | +| **PRs** | — | +| **Target** | Manual/scheduled triggers + FreeEed `.project` generation + auto-chain | + +### Planned deliverables + +- [ ] `api/collectors/core/trigger_manager.py` — manual + APScheduler cron triggers +- [ ] Trigger config schema (YAML): `connector`, `trigger`, `schedule`, `auto_chain`, `custodian`, `since` +- [ ] `api/collectors/processing/processing_engine_client.py` — wraps `run_freeeed_process()` +- [ ] `.project` file writer using **Java-canonical keys** (`input`, `custodian`, `stage`, `output-dir`, `solr_endpoint`, …) +- [ ] `ProcessProgress` guard — no concurrent Java runs during auto-chain +- [ ] `api/collectors/scripts/run_collectors.py` — CLI entry point +- [ ] `api/collectors/scripts/manual_trigger.py` — manual job trigger +- [ ] Stub interfaces for future event/webhook triggers (no impl required v1) +- [ ] Unit tests: `.project` generation, auto-chain mock subprocess, trigger scheduling + +--- + +## M4 — Box connector + +| Field | Value | +|-------|-------| +| **Issues** | — | +| **PRs** | — | +| **Target** | Reference connector implementation + tests | + +### Planned deliverables + +- [ ] `api/collectors/box/box_auth.py` — OAuth2 via SecurityManager +- [ ] `api/collectors/box/box_collector.py` — list, fetch, incremental sync, rate limits +- [ ] Registration: `@register_connector("box")` +- [ ] `handle_rate_limits` using Box `Retry-After` headers +- [ ] Unit tests with mocked HTTP (list, fetch, sync token, rate limit backoff) +- [ ] Connector health check (auth + lightweight API ping) + +--- + +## M5 — Drive & Dropbox + +| Field | Value | +|-------|-------| +| **Issues** | — | +| **PRs** | — | +| **Target** | Remaining v1 connectors + extension template | + +### Planned deliverables + +- [ ] `api/collectors/google_drive/gdrive_auth.py` + `gdrive_collector.py` +- [ ] Google Drive: `files.list`, `changes.getStartPageToken`, `files.get` +- [ ] `api/collectors/dropbox/dropbox_auth.py` + `dropbox_collector.py` +- [ ] Dropbox: `files/list_folder`, `files/list_folder/continue`, `files/download` +- [ ] `api/collectors/template/new_connector_template.py` — boilerplate for all `BaseCollector` methods +- [ ] Unit tests per connector (mocked HTTP) +- [ ] Rate-limit handling per vendor + +--- + +## M6 — API & deployment + +| Field | Value | +|-------|-------| +| **Issues** | — | +| **PRs** | — | +| **Target** | FastAPI surface, Docker, compose, env documentation | + +### Planned deliverables + +- [ ] Extend `api/main.py` with `/collect/*` routes: + - `POST /collect/trigger` + - `GET /collect/jobs/{job_id}` + - `GET /collect/connectors` + - `GET /collect/connectors/{id}/health` + - `POST /collect/schedules` (optional v1) +- [ ] Extend `/health` with collector subsystem status +- [ ] Update `api/Dockerfile` — install `requirements-collectors.txt` +- [ ] Update `docker-compose.yml` — optional `./secrets:/run/secrets:ro` mount +- [ ] Env var documentation (OAuth client IDs, `COLLECTOR_ENCRYPTION_KEY`, etc.) +- [ ] Air-gapped install notes (offline wheelhouse, pre-seeded refresh tokens) + +--- + +## M7 — Tests & docs + +| Field | Value | +|-------|-------| +| **Issues** | — | +| **PRs** | — | +| **Target** | Integration test suite + architecture documentation | + +### Planned deliverables + +- [ ] Integration tests: `api/collectors/tests/integration/` (VCR cassettes or sandbox env; skip if unset) +- [ ] E2E: trigger → collect → filesystem write → `.project` generation +- [ ] Auto-chain integration test (mock Java subprocess) +- [ ] `api/collectors/docs/architecture.md` — diagram + component responsibilities +- [ ] `api/collectors/docs/connectors.md` — step-by-step new connector guide +- [ ] `api/collectors/docs/ingestion_flow.md` — data flow into FreeEed processing +- [ ] `api/collectors/CHANGELOG.md` — versioned change log with issue/PR refs +- [ ] README section: OAuth setup, example `curl` commands +- [ ] CI: integration job gated on repo secrets / `RUN_COLLECTOR_INTEGRATION` variable + +--- + +## M8 — Public promotion + +| Field | Value | +|-------|-------| +| **Issues** | — | +| **PRs** | — | +| **Target** | Curated merge to public `shmsoft/FreeEed` | + +### Planned deliverables + +- [ ] Full security review — no secrets, no telemetry, sanitized logs +- [ ] All M1–M7 unit tests passing in CI +- [ ] Integration tests passing or documented skip rationale +- [ ] Sanitized upstream PR to `shmsoft/FreeEed` (single squash/rebase preferred) +- [ ] Public-facing CHANGELOG / release notes entry +- [ ] Close private-repo issues with link to public PR +- [ ] Update this matrix: mark M0–M8 rows **Complete** with final issue/PR lists +- [ ] Decision recorded: ongoing private-first vs public-only for future connectors + +--- + +## M9 — Collection Selection UI + +| Field | Value | +|-------|-------| +| **Issues** | — | +| **PRs** | — | +| **Target** | Interactive case-scoped browser for external sources with OAuth, lazy tree, multi-select, and collection trigger | + +### Planned deliverables + +- [ ] Interactive case-scoped browser for Box, Google Drive, Dropbox +- [ ] In-app OAuth (WebView or embedded browser) wired to `/collect/oauth/*` +- [ ] Lazy folder tree, breadcrumbs, multi-select folders/files +- [ ] Project/custodian binding, selection validation, collection trigger + job status +- [ ] **UI approach (document both; default: web):** + - **Recommended:** small web UI at `/collectors/` via FastAPI static mount — faster tree/OAuth UX, shared with API deployment + - **Alternative:** Swing panel in `freeeed-processing` Java UI calling existing browse/select API +- [ ] Design spec: `api/collectors/docs/m9-collection-ui.md` +- [ ] Static scaffold: `api/static/collectors/index.html` (placeholder → full implementation) + +--- + +## Issue ↔ PR cross-reference (living section) + +Fill in as work progresses: + +| Issue | Title | PR(s) | Milestone | Merged | +|-------|-------|-------|-----------|--------| +| — | — | — | — | — | + +--- + +## Changelog pointers + +| Version | Date | Milestone | Notes | +|---------|------|-----------|-------| +| — | — | — | First entry after M1 code lands in `api/collectors/CHANGELOG.md` | + +--- + +See also: [WORKFLOW.md](WORKFLOW.md) for branch naming, label taxonomy, and private-repo setup. diff --git a/secrets/.env.local.example b/secrets/.env.local.example new file mode 100644 index 000000000..4816d2ca7 --- /dev/null +++ b/secrets/.env.local.example @@ -0,0 +1,19 @@ +# Copy to secrets/.env.local (gitignored) or export in your shell. +# File-based secrets in secrets/dropbox.json take precedence when both are set +# (env vars are merged first, then JSON file fields override). + +COLLECTOR_SECRETS_DIR=./secrets + +# OAuth redirect base (must match a redirect URI registered in Dropbox console) +COLLECTOR_OAUTH_REDIRECT_BASE=http://localhost:8000 + +# Optional: encrypt OAuth tokens at rest in SQLite +# COLLECTOR_ENCRYPTION_KEY= # Fernet key from: python -c "from cryptography.fernet import Fernet; print(Fernet.generate_key().decode())" + +# Dropbox OAuth app credentials (authorization + token refresh) +DROPBOX_CLIENT_ID=YOUR_DROPBOX_APP_KEY +DROPBOX_CLIENT_SECRET=YOUR_DROPBOX_APP_SECRET + +# Legacy: global connector tokens (optional if using per-project OAuth) +COLLECTOR_DROPBOX_ACCESS_TOKEN=YOUR_DROPBOX_ACCESS_TOKEN +COLLECTOR_DROPBOX_REFRESH_TOKEN=YOUR_DROPBOX_REFRESH_TOKEN_OPTIONAL diff --git a/secrets/.gitignore b/secrets/.gitignore new file mode 100644 index 000000000..2a2bf5f27 --- /dev/null +++ b/secrets/.gitignore @@ -0,0 +1,5 @@ +# Ignore real credentials; allow committed *.example templates +* +!.gitignore +!*.example +!*.json.example diff --git a/secrets/dropbox.json.example b/secrets/dropbox.json.example new file mode 100644 index 000000000..287467a90 --- /dev/null +++ b/secrets/dropbox.json.example @@ -0,0 +1,4 @@ +{ + "client_id": "YOUR_DROPBOX_APP_KEY", + "client_secret": "YOUR_DROPBOX_APP_SECRET" +} From e2ca52ad15e32bd54a4fa88b4227bfb66f8bd4e8 Mon Sep 17 00:00:00 2001 From: Jonathan Flood Date: Wed, 10 Jun 2026 09:19:19 +0100 Subject: [PATCH 2/2] feat(collector): add M9 interactive collection browser UI at /collectors/ Co-authored-by: Cursor --- api/collectors/docs/m9-collection-ui.md | 21 +- api/static/collectors/index.html | 914 ++++++++++++++++++++++-- 2 files changed, 883 insertions(+), 52 deletions(-) diff --git a/api/collectors/docs/m9-collection-ui.md b/api/collectors/docs/m9-collection-ui.md index 4be5801bf..7d9cdb33a 100644 --- a/api/collectors/docs/m9-collection-ui.md +++ b/api/collectors/docs/m9-collection-ui.md @@ -98,11 +98,28 @@ sequenceDiagram | `CollectDialog` | custodian, auto_chain, trigger + poll job | | `api.js` | thin fetch wrapper for `/collect/*` | -Vanilla JS for v1 scaffold (`api/static/collectors/index.html`); consider a small framework only if complexity warrants it. +Vanilla JS for v1 (`api/static/collectors/index.html`); consider a small framework only if complexity warrants it. + +## Implementation status (v1) + +| Feature | Status | Notes | +|---------|--------|-------| +| Project context (`project_id`, custodian) | **Done** | Default `1`; `?project_id=` query param; Apply updates URL | +| Connector tabs (dropbox, box, google_drive) | **Done** | Tab switch resets browse root and checks connection | +| OAuth gate (401/503 → Connect) | **Done** | Links to `/collect/oauth/{id}/authorize?project_id=&redirect_after=/collectors/` | +| Lazy folder browser + breadcrumbs | **Done** | `GET .../browse/...?parent_id=`; folder row click drills down | +| Multi-select (checkboxes, Select folder) | **Done** | In-memory selection panel with remove/clear | +| Save selections | **Done** | `POST .../selections` | +| Load saved selections | **Done** | `GET .../selections?connector_id=` | +| Trigger collection | **Done** | `POST .../trigger`; `auto_chain` off by default | +| Job status polling | **Done** | Polls `GET /collect/jobs/{job_id}` every 2s until terminal | +| Loading / error states | **Done** | Connection badge, browse spinner, message banner | + +**Open for later:** project name from `.project` file, empty-folder-only validation, mobile polish, real-time sync badges. ## Static mount -FastAPI serves assets from `api/static/collectors/` at `/collectors/` (see `api/main.py`). Placeholder HTML demonstrates a single browse call; full tree/OAuth UI is implemented incrementally under M9. +FastAPI serves assets from `api/static/collectors/` at `/collectors/` (see `api/main.py`). Open **`http://localhost:8000/collectors/`** (or your API base URL) when the API is running. ## Swing alternative (documented, not M9 default) diff --git a/api/static/collectors/index.html b/api/static/collectors/index.html index 92d2318be..e1d421ade 100644 --- a/api/static/collectors/index.html +++ b/api/static/collectors/index.html @@ -5,72 +5,886 @@ Collection Browser — M9 -

Collection Browser — M9

-

Placeholder scaffold. Full lazy tree, OAuth, and multi-select UI planned under M9.

- - - - - - -
- 
Click Browse to call GET /collect/projects/{project_id}/browse/{connector_id}
+
+

Collection Browser

+
+ + + +
+
+ + + +
+
+ Checking connection… + + +
+
+ +
+ +
+
+
Browse
+ +
+
Select a connector to browse.
+
+
+ +
+
+ Selected items (0) +
+
+
    +
    +
    + + + +
    +
    +
    + +
    +

    Collect

    +
    + + +
    +
    +