Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
22 changes: 22 additions & 0 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,22 @@
name: CI
on:
pull_request:
push:
branches: [main]
jobs:
test:
runs-on: ubuntu-latest
strategy:
matrix:
python-version: ["3.11", "3.12", "3.13"]
steps:
- uses: actions/checkout@v4
- uses: actions/setup-python@v5
with:
python-version: ${{ matrix.python-version }}
- name: Install (default deps, as users get them)
run: python -m pip install -e ".[test]"
- name: Default and alternate provider clients import
run: python -c "import anthropic, requests, engram.ingest"
- name: Tests
run: python -m pytest -q tests
8 changes: 6 additions & 2 deletions engram/cli.py
Original file line number Diff line number Diff line change
Expand Up @@ -397,14 +397,18 @@ def ingest(ctx, sources):
db_path = str(Path(cfg["index_db"]).expanduser())

source_dirs = sources.split(",") if sources else cfg.get("ingest_sources", ["Daily"])
api_key = os.environ.get("ANTHROPIC_API_KEY")
provider = cfg.get("llm_provider") or "anthropic"
base_url = cfg.get("llm_base_url") or None
api_key = os.environ.get("ANTHROPIC_API_KEY") if provider == "anthropic" else None

click.echo(f"Running ingest from: {source_dirs}")
stats = run_ingest(
vault_root=vault,
source_dirs=source_dirs,
model=cfg.get("llm_model", "claude-haiku-4-5"),
api_key=api_key,
provider=provider,
base_url=base_url,
)
click.echo(
f"Ingest complete. files={stats['files_processed']} "
Expand Down Expand Up @@ -497,7 +501,7 @@ def synthesize(ctx, days, json_output, api_key):

def llm_call(prompt: str) -> str:
if provider == "anthropic" and not base_url:
import anthropic
import anthropic # optional: pip install engram[anthropic]
client = anthropic.Anthropic(api_key=resolved_key)
msg = client.messages.create(
model=model,
Expand Down
16 changes: 8 additions & 8 deletions engram/dialectical.py
Original file line number Diff line number Diff line change
Expand Up @@ -123,36 +123,36 @@ def archive_raw_session(

OUTPUT: Valid JSON only. No markdown fences. No explanation.

{
{{
"new_facts": [
{
{{
"title": "short descriptive title",
"content": "the fact itself, one clear statement",
"artifact_type": "durable_fact|preference|decision|lesson|constraint",
"confidence": 0.0-1.0,
"importance": 0.0-1.0,
"about": ["Entity Name"],
"tags": ["tag1"]
}
}}
],
"updated_facts": [
{
{{
"title": "title of existing fact",
"update": "what changed or the new information to append",
"reason": "why this update is needed"
}
}}
],
"new_entities": [
{
{{
"name": "Entity Name",
"entity_type": "person|project|tool|concept|place|org",
"description": "brief description",
"importance": 0.0-1.0,
"tags": ["tag1"]
}
}}
],
"synthesis": "1-2 sentence summary of what was learned from these sessions"
}
}}

RULES:
- Only extract facts worth keeping months from now
Expand Down
158 changes: 145 additions & 13 deletions engram/ingest.py
Original file line number Diff line number Diff line change
Expand Up @@ -87,23 +87,95 @@ def _chunk_file(path: str, max_chars: int = 6000) -> list:
return chunks or [content[:max_chars]]


def _call_llm(chunk: str, model: str, api_key: str, existing_titles: list = None) -> Optional[dict]:
"""Call Anthropic API for extraction. Returns parsed JSON or None."""
def _extract_json_text(text: str) -> str:
"""Return the complete JSON object inside model output (NUBIS-001).

Reasoning-only completions often wrap a formatted, multi-line JSON answer in
prose or a ```json fence. Try, in order: the whole text, the last fenced
block, then the widest balanced {...} span that parses. Non-JSON text is
returned unchanged so the caller's json.loads still fails safely.
"""
candidates = [text.strip()]
candidates += [m.strip() for m in reversed(re.findall(r"```(?:json)?\s*(.*?)```", text, re.S))]
starts = [i for i, ch in enumerate(text) if ch == "{"]
for start in starts:
depth, in_str, esc = 0, False, False
for j in range(start, len(text)):
ch = text[j]
if in_str:
if esc:
esc = False
elif ch == "\\":
esc = True
elif ch == '"':
in_str = False
continue
if ch == '"':
in_str = True
elif ch == "{":
depth += 1
elif ch == "}":
depth -= 1
if depth == 0:
candidates.append(text[start:j + 1])
break
best = None
for cand in candidates:
try:
obj = json.loads(cand)
except (json.JSONDecodeError, ValueError):
continue
if isinstance(obj, dict) and (best is None or len(cand) > len(best)):
best = cand
return best if best is not None else text.strip()


def _call_llm(chunk: str, model: str, api_key: str, existing_titles: list = None,
base_url: str = None) -> Optional[dict]:
"""Call LLM API for extraction. Supports Anthropic and OpenAI-compatible endpoints.
Returns parsed JSON or None."""
try:
import anthropic
client = anthropic.Anthropic(api_key=api_key)
# Inject existing titles so LLM avoids extracting duplicates
existing_block = ""
if existing_titles:
titles_list = "\n".join(f"- {t}" for t in existing_titles[:50])
existing_block = f"\nALREADY STORED (do NOT re-extract these or close variants):\n{titles_list}\n"
prompt = EXTRACTION_PROMPT.replace("{document}", existing_block + "\n" + chunk)
msg = client.messages.create(
model=model,
max_tokens=2000,
messages=[{"role": "user", "content": prompt}],
)
raw = msg.content[0].text.strip()

if base_url:
# OpenAI-compatible endpoint
import requests
url = base_url.rstrip("/") + "/chat/completions"
resp = requests.post(
url,
headers={"Authorization": f"Bearer {api_key}", "Content-Type": "application/json"},
json={
"model": model,
"max_tokens": 2000,
"messages": [{"role": "user", "content": prompt}],
},
timeout=60,
)
resp.raise_for_status()
data = resp.json()
msg = data["choices"][0]["message"]
raw = msg.get("content", "") or ""
if not raw.strip():
# Reasoning models may put output in reasoning_content
reasoning = msg.get("reasoning_content", "") or ""
if reasoning.strip():
raw = _extract_json_text(reasoning)
else:
# Anthropic native
import anthropic
client = anthropic.Anthropic(api_key=api_key)
msg = client.messages.create(
model=model,
max_tokens=2000,
messages=[{"role": "user", "content": prompt}],
)
raw = msg.content[0].text.strip()

# Strip markdown fences if present
raw = re.sub(r"^```(?:json)?\s*", "", raw)
raw = re.sub(r"\s*```$", "", raw)
Expand Down Expand Up @@ -273,6 +345,8 @@ def run_ingest(
source_dirs: list,
model: str = "claude-haiku-4-5",
api_key: Optional[str] = None,
provider: Optional[str] = None,
base_url: Optional[str] = None,
max_workers: int = 5,
) -> dict:
"""
Expand All @@ -288,9 +362,27 @@ def run_ingest(
"errors": 0,
}

if not api_key:
# Fill provider/model/endpoint from the Engram config. Explicit arguments win (NUBIS-003).
cfg = {}
cfg_path = Path("~/.engram/config.json").expanduser()
if cfg_path.exists():
try:
cfg = json.loads(cfg_path.read_text())
except Exception:
cfg = {}
if not provider:
provider = cfg.get("llm_provider") or "anthropic"
if model == "claude-haiku-4-5" and cfg.get("llm_model"):
model = cfg["llm_model"]
if not base_url and provider != "anthropic":
base_url = cfg.get("llm_base_url") or None
explicit_endpoint = bool(base_url)
if not api_key and provider != "anthropic":
api_key = os.environ.get(cfg.get("llm_api_key_env") or "OPENAI_API_KEY") or None

if not api_key and provider == "anthropic":
api_key = os.environ.get("ANTHROPIC_API_KEY")
if not api_key:
if not api_key and provider == "anthropic":
# Try loading from ~/.hermes/.env directly
# Check multiple key names in priority order
env_path = Path("~/.hermes/.env").expanduser()
Expand All @@ -313,6 +405,46 @@ def run_ingest(
if name in candidates:
api_key = candidates[name]
break

# Resolve base_url for OpenAI-compatible providers (also provides api_key from credential pool)
# A configured/caller endpoint is never replaced by a credential-pool endpoint.
if provider != "anthropic" and not explicit_endpoint and not (api_key and base_url):
# Only check credential pool if credentials weren't fully provided
auth_path = Path("~/.hermes/auth.json").expanduser()
if auth_path.exists():
try:
auth_data = json.loads(auth_path.read_text())
pool = auth_data.get("credential_pool", {})
hermes_cfg_path = Path("~/.hermes/config.yaml").expanduser()
hermes_provider = None
hermes_base_url = None
if hermes_cfg_path.exists():
import yaml
hcfg = yaml.safe_load(hermes_cfg_path.read_text()) or {}
hermes_provider = hcfg.get("model", {}).get("provider", "")
hermes_base_url = hcfg.get("model", {}).get("base_url", "")
if hermes_provider and hermes_provider in pool:
for creds in pool[hermes_provider]:
token = creds.get("access_token", "")
if token and len(token) > 10:
base_url = creds.get("base_url") or hermes_base_url
api_key = token
break
if not base_url:
for prov_name, creds_list in pool.items():
if prov_name == "anthropic":
continue
for creds in creds_list:
token = creds.get("access_token", "")
if token and len(token) > 10:
base_url = creds.get("base_url") or hermes_base_url
api_key = token
break
if base_url:
break
except Exception as e:
logger.warning(f"Failed to read Hermes auth.json: {e}")

if not api_key:
raise ValueError("No API key -- set ANTHROPIC_API_KEY or pass api_key")

Expand Down Expand Up @@ -387,7 +519,7 @@ def run_ingest(

# Parallel LLM extraction
def process_chunk(chunk):
return _call_llm(chunk, model, api_key, existing_titles=existing_titles_list)
return _call_llm(chunk, model, api_key, existing_titles=existing_titles_list, base_url=base_url)

with ThreadPoolExecutor(max_workers=max_workers) as executor:
futures = {executor.submit(process_chunk, c): c for c in chunks}
Expand Down
13 changes: 11 additions & 2 deletions pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -4,16 +4,25 @@ build-backend = "setuptools.build_meta"

[project]
name = "engram"
version = "1.0.0"
version = "1.3.0"
description = "Jarvis persistent memory system — Obsidian-native, SQLite-indexed"
requires-python = ">=3.11"
dependencies = [
"watchdog>=4.0",
"python-frontmatter>=1.1",
"anthropic>=0.40",
"click>=8.1",
"anthropic>=0.40",
"requests>=2.31",
]

[project.optional-dependencies]
# anthropic and requests are core (NUBIS-002): the default ingest path imports
# anthropic and the OpenAI-compatible path imports requests. Extras stay as
# no-op aliases so existing `engram[anthropic]` installs keep working.
anthropic = []
all = []
test = ["pytest>=8"]

[project.scripts]
engram = "engram.cli:cli"

Expand Down
62 changes: 62 additions & 0 deletions tests/test_ingest_endpoint.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,62 @@
"""NUBIS-003: `engram ingest` sends completions to the configured endpoint, never a pool one."""
import json
import sys
import types

from click.testing import CliRunner

from engram.cli import cli

EXTRACT = {"entities": [], "facts": [{"fact": "Engram reads the configured endpoint", "confidence": 0.9}]}


def _setup(tmp_path, monkeypatch, with_pool=True):
home = tmp_path / "home"
vault = tmp_path / "vault"
(vault / "Daily").mkdir(parents=True)
(vault / "Daily" / "2026-10-23.md").write_text("# Today\nWe pointed Engram at a local endpoint.\n" * 5)
cfg = {"vault_path": str(vault), "index_db": str(home / ".engram" / "vault_index.db"),
"log_level": "warning", "log_path": str(home / ".engram" / "engram.log"),
"llm_provider": "openai", "llm_base_url": "http://engram-endpoint.local/v1",
"llm_model": "local-model", "ingest_sources": ["Daily"]}
(home / ".engram").mkdir(parents=True)
(home / ".engram" / "config.json").write_text(json.dumps(cfg))
if with_pool: # an unrelated Hermes credential pool must NOT win over the configured endpoint
(home / ".hermes").mkdir()
(home / ".hermes" / "auth.json").write_text(json.dumps({"credential_pool": {"other": [
{"access_token": "pool-token-1234567890", "base_url": "http://hermes-pool.local/v1"}]}}))
monkeypatch.setenv("HOME", str(home))
monkeypatch.setenv("USERPROFILE", str(home))
monkeypatch.setenv("OPENAI_API_KEY", "sk-test-engram")
monkeypatch.delenv("ANTHROPIC_API_KEY", raising=False)
calls = []

class Resp:
def raise_for_status(self):
pass

def json(self):
return {"choices": [{"message": {"content": json.dumps(EXTRACT)}}]}

mod = types.ModuleType("requests")
mod.post = lambda url, **kw: calls.append((url, kw)) or Resp()
monkeypatch.setitem(sys.modules, "requests", mod)
return home, calls


def test_cli_ingest_uses_configured_endpoint(tmp_path, monkeypatch):
home, calls = _setup(tmp_path, monkeypatch)
res = CliRunner().invoke(cli, ["--config", str(home / ".engram" / "config.json"), "ingest"])
assert res.exit_code == 0, res.output
assert calls, "no completion request was sent"
assert {u for u, _ in calls} == {"http://engram-endpoint.local/v1/chat/completions"}
assert all(kw["headers"]["Authorization"] == "Bearer sk-test-engram" for _, kw in calls)
assert all(kw["json"]["model"] == "local-model" for _, kw in calls)


def test_run_ingest_reads_endpoint_from_config(tmp_path, monkeypatch):
home, calls = _setup(tmp_path, monkeypatch, with_pool=False)
from engram.ingest import run_ingest
cfg = json.loads((home / ".engram" / "config.json").read_text())
run_ingest(vault_root=cfg["vault_path"], source_dirs=["Daily"])
assert calls and {u for u, _ in calls} == {"http://engram-endpoint.local/v1/chat/completions"}
Loading
Loading