diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..e7c8ece --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,22 @@ +name: CI +on: + pull_request: + push: + branches: [main] +jobs: + test: + runs-on: ubuntu-latest + strategy: + matrix: + python-version: ["3.11", "3.12", "3.13"] + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + - name: Install (default deps, as users get them) + run: python -m pip install -e ".[test]" + - name: Default and alternate provider clients import + run: python -c "import anthropic, requests, engram.ingest" + - name: Tests + run: python -m pytest -q tests diff --git a/engram/cli.py b/engram/cli.py index 0f757e4..e73b9af 100644 --- a/engram/cli.py +++ b/engram/cli.py @@ -397,7 +397,9 @@ def ingest(ctx, sources): db_path = str(Path(cfg["index_db"]).expanduser()) source_dirs = sources.split(",") if sources else cfg.get("ingest_sources", ["Daily"]) - api_key = os.environ.get("ANTHROPIC_API_KEY") + provider = cfg.get("llm_provider") or "anthropic" + base_url = cfg.get("llm_base_url") or None + api_key = os.environ.get("ANTHROPIC_API_KEY") if provider == "anthropic" else None click.echo(f"Running ingest from: {source_dirs}") stats = run_ingest( @@ -405,6 +407,8 @@ def ingest(ctx, sources): source_dirs=source_dirs, model=cfg.get("llm_model", "claude-haiku-4-5"), api_key=api_key, + provider=provider, + base_url=base_url, ) click.echo( f"Ingest complete. files={stats['files_processed']} " @@ -497,7 +501,7 @@ def synthesize(ctx, days, json_output, api_key): def llm_call(prompt: str) -> str: if provider == "anthropic" and not base_url: - import anthropic + import anthropic # optional: pip install engram[anthropic] client = anthropic.Anthropic(api_key=resolved_key) msg = client.messages.create( model=model, diff --git a/engram/dialectical.py b/engram/dialectical.py index 8fd7813..e667038 100644 --- a/engram/dialectical.py +++ b/engram/dialectical.py @@ -123,9 +123,9 @@ def archive_raw_session( OUTPUT: Valid JSON only. No markdown fences. No explanation. -{ +{{ "new_facts": [ - { + {{ "title": "short descriptive title", "content": "the fact itself, one clear statement", "artifact_type": "durable_fact|preference|decision|lesson|constraint", @@ -133,26 +133,26 @@ def archive_raw_session( "importance": 0.0-1.0, "about": ["Entity Name"], "tags": ["tag1"] - } + }} ], "updated_facts": [ - { + {{ "title": "title of existing fact", "update": "what changed or the new information to append", "reason": "why this update is needed" - } + }} ], "new_entities": [ - { + {{ "name": "Entity Name", "entity_type": "person|project|tool|concept|place|org", "description": "brief description", "importance": 0.0-1.0, "tags": ["tag1"] - } + }} ], "synthesis": "1-2 sentence summary of what was learned from these sessions" -} +}} RULES: - Only extract facts worth keeping months from now diff --git a/engram/ingest.py b/engram/ingest.py index 8a7e794..70b630b 100644 --- a/engram/ingest.py +++ b/engram/ingest.py @@ -87,23 +87,95 @@ def _chunk_file(path: str, max_chars: int = 6000) -> list: return chunks or [content[:max_chars]] -def _call_llm(chunk: str, model: str, api_key: str, existing_titles: list = None) -> Optional[dict]: - """Call Anthropic API for extraction. Returns parsed JSON or None.""" +def _extract_json_text(text: str) -> str: + """Return the complete JSON object inside model output (NUBIS-001). + + Reasoning-only completions often wrap a formatted, multi-line JSON answer in + prose or a ```json fence. Try, in order: the whole text, the last fenced + block, then the widest balanced {...} span that parses. Non-JSON text is + returned unchanged so the caller's json.loads still fails safely. + """ + candidates = [text.strip()] + candidates += [m.strip() for m in reversed(re.findall(r"```(?:json)?\s*(.*?)```", text, re.S))] + starts = [i for i, ch in enumerate(text) if ch == "{"] + for start in starts: + depth, in_str, esc = 0, False, False + for j in range(start, len(text)): + ch = text[j] + if in_str: + if esc: + esc = False + elif ch == "\\": + esc = True + elif ch == '"': + in_str = False + continue + if ch == '"': + in_str = True + elif ch == "{": + depth += 1 + elif ch == "}": + depth -= 1 + if depth == 0: + candidates.append(text[start:j + 1]) + break + best = None + for cand in candidates: + try: + obj = json.loads(cand) + except (json.JSONDecodeError, ValueError): + continue + if isinstance(obj, dict) and (best is None or len(cand) > len(best)): + best = cand + return best if best is not None else text.strip() + + +def _call_llm(chunk: str, model: str, api_key: str, existing_titles: list = None, + base_url: str = None) -> Optional[dict]: + """Call LLM API for extraction. Supports Anthropic and OpenAI-compatible endpoints. + Returns parsed JSON or None.""" try: - import anthropic - client = anthropic.Anthropic(api_key=api_key) # Inject existing titles so LLM avoids extracting duplicates existing_block = "" if existing_titles: titles_list = "\n".join(f"- {t}" for t in existing_titles[:50]) existing_block = f"\nALREADY STORED (do NOT re-extract these or close variants):\n{titles_list}\n" prompt = EXTRACTION_PROMPT.replace("{document}", existing_block + "\n" + chunk) - msg = client.messages.create( - model=model, - max_tokens=2000, - messages=[{"role": "user", "content": prompt}], - ) - raw = msg.content[0].text.strip() + + if base_url: + # OpenAI-compatible endpoint + import requests + url = base_url.rstrip("/") + "/chat/completions" + resp = requests.post( + url, + headers={"Authorization": f"Bearer {api_key}", "Content-Type": "application/json"}, + json={ + "model": model, + "max_tokens": 2000, + "messages": [{"role": "user", "content": prompt}], + }, + timeout=60, + ) + resp.raise_for_status() + data = resp.json() + msg = data["choices"][0]["message"] + raw = msg.get("content", "") or "" + if not raw.strip(): + # Reasoning models may put output in reasoning_content + reasoning = msg.get("reasoning_content", "") or "" + if reasoning.strip(): + raw = _extract_json_text(reasoning) + else: + # Anthropic native + import anthropic + client = anthropic.Anthropic(api_key=api_key) + msg = client.messages.create( + model=model, + max_tokens=2000, + messages=[{"role": "user", "content": prompt}], + ) + raw = msg.content[0].text.strip() + # Strip markdown fences if present raw = re.sub(r"^```(?:json)?\s*", "", raw) raw = re.sub(r"\s*```$", "", raw) @@ -273,6 +345,8 @@ def run_ingest( source_dirs: list, model: str = "claude-haiku-4-5", api_key: Optional[str] = None, + provider: Optional[str] = None, + base_url: Optional[str] = None, max_workers: int = 5, ) -> dict: """ @@ -288,9 +362,27 @@ def run_ingest( "errors": 0, } - if not api_key: + # Fill provider/model/endpoint from the Engram config. Explicit arguments win (NUBIS-003). + cfg = {} + cfg_path = Path("~/.engram/config.json").expanduser() + if cfg_path.exists(): + try: + cfg = json.loads(cfg_path.read_text()) + except Exception: + cfg = {} + if not provider: + provider = cfg.get("llm_provider") or "anthropic" + if model == "claude-haiku-4-5" and cfg.get("llm_model"): + model = cfg["llm_model"] + if not base_url and provider != "anthropic": + base_url = cfg.get("llm_base_url") or None + explicit_endpoint = bool(base_url) + if not api_key and provider != "anthropic": + api_key = os.environ.get(cfg.get("llm_api_key_env") or "OPENAI_API_KEY") or None + + if not api_key and provider == "anthropic": api_key = os.environ.get("ANTHROPIC_API_KEY") - if not api_key: + if not api_key and provider == "anthropic": # Try loading from ~/.hermes/.env directly # Check multiple key names in priority order env_path = Path("~/.hermes/.env").expanduser() @@ -313,6 +405,46 @@ def run_ingest( if name in candidates: api_key = candidates[name] break + + # Resolve base_url for OpenAI-compatible providers (also provides api_key from credential pool) + # A configured/caller endpoint is never replaced by a credential-pool endpoint. + if provider != "anthropic" and not explicit_endpoint and not (api_key and base_url): + # Only check credential pool if credentials weren't fully provided + auth_path = Path("~/.hermes/auth.json").expanduser() + if auth_path.exists(): + try: + auth_data = json.loads(auth_path.read_text()) + pool = auth_data.get("credential_pool", {}) + hermes_cfg_path = Path("~/.hermes/config.yaml").expanduser() + hermes_provider = None + hermes_base_url = None + if hermes_cfg_path.exists(): + import yaml + hcfg = yaml.safe_load(hermes_cfg_path.read_text()) or {} + hermes_provider = hcfg.get("model", {}).get("provider", "") + hermes_base_url = hcfg.get("model", {}).get("base_url", "") + if hermes_provider and hermes_provider in pool: + for creds in pool[hermes_provider]: + token = creds.get("access_token", "") + if token and len(token) > 10: + base_url = creds.get("base_url") or hermes_base_url + api_key = token + break + if not base_url: + for prov_name, creds_list in pool.items(): + if prov_name == "anthropic": + continue + for creds in creds_list: + token = creds.get("access_token", "") + if token and len(token) > 10: + base_url = creds.get("base_url") or hermes_base_url + api_key = token + break + if base_url: + break + except Exception as e: + logger.warning(f"Failed to read Hermes auth.json: {e}") + if not api_key: raise ValueError("No API key -- set ANTHROPIC_API_KEY or pass api_key") @@ -387,7 +519,7 @@ def run_ingest( # Parallel LLM extraction def process_chunk(chunk): - return _call_llm(chunk, model, api_key, existing_titles=existing_titles_list) + return _call_llm(chunk, model, api_key, existing_titles=existing_titles_list, base_url=base_url) with ThreadPoolExecutor(max_workers=max_workers) as executor: futures = {executor.submit(process_chunk, c): c for c in chunks} diff --git a/pyproject.toml b/pyproject.toml index 7c36560..dc1d677 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,16 +4,25 @@ build-backend = "setuptools.build_meta" [project] name = "engram" -version = "1.0.0" +version = "1.3.0" description = "Jarvis persistent memory system — Obsidian-native, SQLite-indexed" requires-python = ">=3.11" dependencies = [ "watchdog>=4.0", "python-frontmatter>=1.1", - "anthropic>=0.40", "click>=8.1", + "anthropic>=0.40", + "requests>=2.31", ] +[project.optional-dependencies] +# anthropic and requests are core (NUBIS-002): the default ingest path imports +# anthropic and the OpenAI-compatible path imports requests. Extras stay as +# no-op aliases so existing `engram[anthropic]` installs keep working. +anthropic = [] +all = [] +test = ["pytest>=8"] + [project.scripts] engram = "engram.cli:cli" diff --git a/tests/test_ingest_endpoint.py b/tests/test_ingest_endpoint.py new file mode 100644 index 0000000..9aaa7c0 --- /dev/null +++ b/tests/test_ingest_endpoint.py @@ -0,0 +1,62 @@ +"""NUBIS-003: `engram ingest` sends completions to the configured endpoint, never a pool one.""" +import json +import sys +import types + +from click.testing import CliRunner + +from engram.cli import cli + +EXTRACT = {"entities": [], "facts": [{"fact": "Engram reads the configured endpoint", "confidence": 0.9}]} + + +def _setup(tmp_path, monkeypatch, with_pool=True): + home = tmp_path / "home" + vault = tmp_path / "vault" + (vault / "Daily").mkdir(parents=True) + (vault / "Daily" / "2026-10-23.md").write_text("# Today\nWe pointed Engram at a local endpoint.\n" * 5) + cfg = {"vault_path": str(vault), "index_db": str(home / ".engram" / "vault_index.db"), + "log_level": "warning", "log_path": str(home / ".engram" / "engram.log"), + "llm_provider": "openai", "llm_base_url": "http://engram-endpoint.local/v1", + "llm_model": "local-model", "ingest_sources": ["Daily"]} + (home / ".engram").mkdir(parents=True) + (home / ".engram" / "config.json").write_text(json.dumps(cfg)) + if with_pool: # an unrelated Hermes credential pool must NOT win over the configured endpoint + (home / ".hermes").mkdir() + (home / ".hermes" / "auth.json").write_text(json.dumps({"credential_pool": {"other": [ + {"access_token": "pool-token-1234567890", "base_url": "http://hermes-pool.local/v1"}]}})) + monkeypatch.setenv("HOME", str(home)) + monkeypatch.setenv("USERPROFILE", str(home)) + monkeypatch.setenv("OPENAI_API_KEY", "sk-test-engram") + monkeypatch.delenv("ANTHROPIC_API_KEY", raising=False) + calls = [] + + class Resp: + def raise_for_status(self): + pass + + def json(self): + return {"choices": [{"message": {"content": json.dumps(EXTRACT)}}]} + + mod = types.ModuleType("requests") + mod.post = lambda url, **kw: calls.append((url, kw)) or Resp() + monkeypatch.setitem(sys.modules, "requests", mod) + return home, calls + + +def test_cli_ingest_uses_configured_endpoint(tmp_path, monkeypatch): + home, calls = _setup(tmp_path, monkeypatch) + res = CliRunner().invoke(cli, ["--config", str(home / ".engram" / "config.json"), "ingest"]) + assert res.exit_code == 0, res.output + assert calls, "no completion request was sent" + assert {u for u, _ in calls} == {"http://engram-endpoint.local/v1/chat/completions"} + assert all(kw["headers"]["Authorization"] == "Bearer sk-test-engram" for _, kw in calls) + assert all(kw["json"]["model"] == "local-model" for _, kw in calls) + + +def test_run_ingest_reads_endpoint_from_config(tmp_path, monkeypatch): + home, calls = _setup(tmp_path, monkeypatch, with_pool=False) + from engram.ingest import run_ingest + cfg = json.loads((home / ".engram" / "config.json").read_text()) + run_ingest(vault_root=cfg["vault_path"], source_dirs=["Daily"]) + assert calls and {u for u, _ in calls} == {"http://engram-endpoint.local/v1/chat/completions"} diff --git a/tests/test_ingest_llm.py b/tests/test_ingest_llm.py new file mode 100644 index 0000000..faaefd7 --- /dev/null +++ b/tests/test_ingest_llm.py @@ -0,0 +1,50 @@ +"""NUBIS-001: reasoning-only completions must keep their full multi-line JSON.""" +import json +import sys +import types + +from engram import ingest + +EXPECTED = {"entities": [{"name": "Shana", "type": "person"}], "facts": [{"fact": "Shana likes Dr Pepper at night"}]} + + +def _fake_requests(message): + class Resp: + def raise_for_status(self): + pass + + def json(self): + return {"choices": [{"message": message}]} + + mod = types.ModuleType("requests") + mod.post = lambda *a, **k: Resp() + return mod + + +def _call(monkeypatch, message): + monkeypatch.setitem(sys.modules, "requests", _fake_requests(message)) + return ingest._call_llm("chunk", "m", "k", base_url="http://local/v1") + + +def test_reasoning_only_multiline_json_is_parsed(monkeypatch): + reasoning = "Let me extract the facts.\n\n" + json.dumps(EXPECTED, indent=2) + "\n\nThat covers it." + assert _call(monkeypatch, {"content": "", "reasoning_content": reasoning}) == EXPECTED + + +def test_reasoning_only_fenced_json_is_parsed(monkeypatch): + reasoning = "Thinking...\n```json\n" + json.dumps(EXPECTED, indent=2) + "\n```\n" + assert _call(monkeypatch, {"content": None, "reasoning_content": reasoning}) == EXPECTED + + +def test_reasoning_with_braces_in_strings(monkeypatch): + data = {"entities": [], "facts": [{"fact": "uses {curly} braces and a \"quote\""}]} + reasoning = "pre {not json} " + json.dumps(data, indent=2) + assert _call(monkeypatch, {"content": "", "reasoning_content": reasoning}) == data + + +def test_non_json_reasoning_fails_safely(monkeypatch): + assert _call(monkeypatch, {"content": "", "reasoning_content": "I could not find anything worth storing."}) is None + + +def test_plain_content_still_works(monkeypatch): + assert _call(monkeypatch, {"content": "```json\n" + json.dumps(EXPECTED) + "\n```"}) == EXPECTED