hm

#!/bin/sh
# hm -- the documented steps, in the order that works.
#
# Not a framework and not a new way to run anything: every subcommand below shells the same
# scripts docs/INSTALL.md describes. It exists because the ORDER is load-bearing and getting it
# wrong is silent. Build the ANN index before the rows exist and it is far slower; skip it and
# search still answers, by sequential-scanning every vector for the life of the install. Re-run a
# full ingest over an existing scope and it deletes the documents first, taking every embedding
# with them. Neither raises an error.
#
#   ./hm init                  bring up the docker stack, load both schemas, print what to export
#   ./hm ingest <dir> <scope> [--walk]
#                              ingest -> embed -> index, incrementally if the scope already exists.
#                              --walk enumerates the tree instead of asking git, for a directory
#                              that is git-ignored or not a repository at all -- without it such a
#                              directory ingests ZERO documents and says why.
#                              A `.hmignore` in <dir> keeps paths OUT of the corpus (one
#                              pattern per line, trailing '/' = whole subtree). For the file of
#                              credentials and the superseded spec set that live next to the real
#                              documents: --walk cannot exclude them, it is all-or-nothing.
#   ./hm refresh               re-ingest every scope in $HM_SCOPES (default
#                              ~/.config/hypermnesia/scopes: <dir><TAB><scope><TAB>[--walk] per
#                              line), then assert no chunk was left without an embedding. This is
#                              the one to put on a timer -- freshness.py reports drift, this ends
#                              it.
#   ./hm doctor                is this install actually working, or only appearing to
#   ./hm latency               where the time goes: hook, embedder, database, reranker
#
# Env: DATABASE_URL (required by ingest/doctor; init prints the one it just set up).
set -eu

ROOT=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)
PY=${HM_PYTHON:-python3}

die()  { printf '%s\n' "hm: $*" >&2; exit 1; }
step() { printf '\n== %s\n' "$*"; }
need() { command -v "$1" >/dev/null 2>&1 || die "$1 is not on PATH"; }

# The header comment above IS the help text; printing it from the file keeps the two from
# drifting. Stop at the first line that is not a comment rather than a hardcoded line number,
# which goes stale the moment anyone edits the header.
usage() {
    awk 'NR > 1 && /^#/ { sub(/^# ?/, ""); print; next } NR > 1 { exit }' "$0"
    exit "${1:-1}"
}

cmd_init() {
    need docker
    env_file="$ROOT/deploy/docker/.env"
    if [ ! -f "$env_file" ]; then
        step "Writing $env_file"
        cp "$ROOT/deploy/docker/.env.example" "$env_file"
        # Generated, not chosen: a password typed in a hurry outlives the hurry.
        # Hex, not base64: a '/' or '+' in the password has to be percent-encoded in the
        # postgresql:// URI this prints, and a password that silently breaks the URI is worse
        # than a shorter one.
        pw=$(openssl rand -hex 16 2>/dev/null | tr -d '\n') \
            || die "could not generate a password -- set POSTGRES_PASSWORD in $env_file yourself"
        [ -n "$pw" ] || die "empty password generated -- set POSTGRES_PASSWORD in $env_file yourself"
        # BSD and GNU sed disagree about -i, so write a new file and move it.
        sed "s|^POSTGRES_PASSWORD=.*|POSTGRES_PASSWORD=$pw|" "$env_file" > "$env_file.tmp"
        mv "$env_file.tmp" "$env_file"
        chmod 600 "$env_file"
    else
        step "Keeping the existing $env_file"     # never clobber a password already in use
    fi

    # shellcheck disable=SC1090
    . "$env_file"
    : "${POSTGRES_USER:=hm}" "${POSTGRES_DB:=hypermnesia}" "${PG_PORT:=5432}"
    [ -n "${POSTGRES_PASSWORD:-}" ] || die "POSTGRES_PASSWORD is empty in $env_file"

    step "Starting Postgres and the embedder"
    (cd "$ROOT/deploy/docker" && docker compose up -d)

    step "Waiting for Postgres to accept connections"
    i=0
    until docker compose -f "$ROOT/deploy/docker/docker-compose.yml" \
              exec -T postgres pg_isready -U "$POSTGRES_USER" -d "$POSTGRES_DB" >/dev/null 2>&1; do
        i=$((i + 1))
        [ "$i" -lt 60 ] || die "Postgres did not come up in 60s -- 'docker compose logs postgres'"
        sleep 1
    done

    # mem_multiuser.sql belongs here even for one person: every memory write inserts `author`
    # and `scope`, which schema_mem.sql does not define, so without it the first write dies
    # with UndefinedColumn -- and dies QUIETLY, because the hooks are fail-open and `hm doctor`
    # only checks that mem.memories exists. All three are idempotent.
    step "Loading the schema (all three files are idempotent)"
    docker compose -f "$ROOT/deploy/docker/docker-compose.yml" exec -T postgres \
        psql -v ON_ERROR_STOP=1 -U "$POSTGRES_USER" -d "$POSTGRES_DB" \
        -f /sql/schema.sql -f /sql/schema_mem.sql -f /sql/mem_multiuser.sql >/dev/null

    url="postgresql://$POSTGRES_USER:$POSTGRES_PASSWORD@localhost:$PG_PORT/$POSTGRES_DB"
    cat <<EOF

Up. Export these, in this shell and in your MCP client's env:

    export DATABASE_URL='$url'
    export EMBED_BACKEND=tei TEI_URL=http://localhost:${TEI_PORT:-8080}

Then put a repo in:  ./hm ingest ~/code/myrepo myrepo
EOF
}

cmd_ingest() {
    # --walk anywhere among the arguments: a notes folder that is git-ignored is the common case
    # for it, and that is exactly the case which otherwise ingests nothing at all.
    #
    # Read with `case` rather than a `[ ]` test, and the positionals rebuilt by assignment rather
    # than by `set -- $(...)`: under `set -e` a failing test as the last statement of a loop ends
    # the script, and an unquoted expansion loses a directory whose name contains a space.
    walk=""; dir=""; scope=""; n=0
    for a in "$@"; do
        case "$a" in
            --walk) walk="--walk" ;;
            *) n=$((n + 1))
               case "$n" in
                   1) dir="$a" ;;
                   2) scope="$a" ;;
               esac ;;
        esac
    done
    [ "$n" -eq 2 ] || usage
    [ -d "$dir" ] || die "no such directory: $dir"
    # Before the first psql call, not inside the python step that runs after it: the scope is
    # interpolated into a single-quoted SQL literal below, so a quote in it produces a syntax
    # error at best and executes as written at worst. ingest_repo.py checks the same alphabet,
    # but by then two queries have already run.
    case "$scope" in
        *[!A-Za-z0-9._-]*|"") die "scope must match [A-Za-z0-9._-]+ (got: $scope)" ;;
    esac
    need psql
    [ -n "${DATABASE_URL:-}" ] || die "DATABASE_URL is not set (./hm init prints it)"

    tmp=$(mktemp -d); trap 'rm -rf "$tmp"' EXIT

    # Incremental when the scope already holds documents. The full path OPENS with
    # DELETE FROM documents WHERE repo=<scope>, which cascades to chunks and therefore to every
    # embedding -- a one-file edit costing a full re-embed, quietly. Taking the snapshot from the
    # database we are about to write to is the point: one from anywhere else would mark documents
    # "already stored" that were never there.
    # Do not collapse this into `|| die "cannot reach the store"`: a query that fails for any
    # other reason would then send the reader to debug Postgres connectivity while the store is
    # perfectly healthy.
    if ! n=$(psql "$DATABASE_URL" -tAX -v ON_ERROR_STOP=1 \
             -c "SELECT count(*) FROM documents WHERE repo='$scope'" 2>&1); then
        die "could not count documents for scope '$scope': $n"
    fi
    if [ "$n" -gt 0 ]; then
        step "Scope '$scope' already holds $n document(s) -- ingesting incrementally"
        psql "$DATABASE_URL" -tAX -v ON_ERROR_STOP=1 -F'	' \
            -c "SELECT path, content_hash FROM documents WHERE repo='$scope'" > "$tmp/known.tsv"
        "$PY" "$ROOT/ingest/ingest_repo.py" "$dir" "$scope" "$tmp/out.sql" \
            --known-hashes "$tmp/known.tsv" $walk
    else
        step "First ingest of scope '$scope'"
        "$PY" "$ROOT/ingest/ingest_repo.py" "$dir" "$scope" "$tmp/out.sql" $walk
    fi

    step "Loading the documents"
    psql "$DATABASE_URL" -v ON_ERROR_STOP=1 -q -f "$tmp/out.sql"

    step "Embedding what has no vector yet"
    EMBED_REPO="$scope" "$PY" "$ROOT/ingest/embed_chunks.py"

    # After the rows exist, never before: building HNSW on an empty table then filling it is far
    # slower than the other way round. IF NOT EXISTS makes the re-run a no-op.
    step "Building the ANN index if it is missing"
    psql "$DATABASE_URL" -v ON_ERROR_STOP=1 -q \
        -c "CREATE INDEX IF NOT EXISTS chunks_embedding_hnsw ON chunks USING hnsw (embedding vector_cosine_ops)"

    step "Checking the result"
    HM_REPO="$scope" "$PY" "$ROOT/ci/doctor.py" || true
    cat <<EOF

Documents are in. What is NOT in is the part that pays: the Tier 0/1 map -- which files belong to
which component, and which rules must hold for them. Nothing can generate that honestly from a
directory listing. See examples/seed_example.sql and skills/onboard-project.
EOF
}

cmd_refresh() {
    # Every scope, in one pass, from a list -- because the thing that rots a store is not a bad
    # ingest, it is a good one that nobody re-runs. `ci/freshness.py` already counts the drift and
    # writes it down; counting is not fixing, and a number in a log fixes nothing by itself.
    #
    # The list is tab-separated, one scope per line, `#` comments allowed:
    #
    #     /home/me/notes<TAB>notes<TAB>--walk
    #     /home/me/src/api<TAB>api
    #
    # Tabs, not spaces: a directory with a space in its name is ordinary, and splitting on
    # whitespace would quietly ingest the wrong path or nothing at all.
    file=${HM_SCOPES:-$HOME/.config/hypermnesia/scopes}
    [ -f "$file" ] || die "no scope list at $file -- one '<dir><TAB><scope><TAB>[--walk]' per line (or set HM_SCOPES)"
    [ -n "${DATABASE_URL:-}" ] || die "DATABASE_URL is not set (./hm init prints it)"
    need psql

    # One refresh at a time. `mkdir` is the atomic test-and-set every shell has. Two overlapping
    # runs would ingest the same scopes concurrently, and the second would delete and rebuild
    # chunks the first is embedding.
    lock="${TMPDIR:-/tmp}/hm-refresh.lock"
    if ! mkdir "$lock" 2>/dev/null; then
        die "another refresh holds $lock"
    fi
    trap 'rmdir "$lock" 2>/dev/null' EXIT INT TERM

    failed=""
    listed=""
    while IFS= read -r line; do
        case "$line" in ''|\#*) continue ;; esac
        dir=$(printf %s "$line" | cut -f1)
        scope=$(printf %s "$line" | cut -f2)
        extra=$(printf %s "$line" | cut -f3)
        [ -n "$scope" ] || die "malformed line in $file (need a TAB between the path and the scope): $line"
        # A scope on the list and not on disk is a misconfiguration, not a reason to skip
        # quietly: the list is hand-written, so a renamed or unmounted directory would drop a
        # whole scope out of every refresh from then on while the run still reported success.
        if [ ! -d "$dir" ]; then
            printf '!! %s: no such directory -- scope %s was NOT refreshed\n' "$dir" "$scope" >&2
            failed="$failed $scope:missing"
            continue
        fi
        step "Refreshing $scope from $dir"
        # In a subshell, because cmd_ingest calls die() on a bad scope or an unreachable store and
        # one bad entry must not take the rest of the list with it.
        #
        # `set +e` around it, and `set -e` INSIDE it, and neither is decoration. A subshell that
        # is the condition of `if` or the left side of `||` has errexit suppressed for everything
        # in it -- and re-setting it inside does not bring it back (checked in dash and in bash's
        # sh mode). Written as `if ! ( cmd_ingest ... )`, the embed step could die with a
        # traceback and the ingest would carry on to build the index and report success. It did:
        # the first run of this command against a live store crashed on a missing psycopg2, said
        # nothing, and was only caught by the unembedded-chunk assertion at the very end.
        set +e
        # shellcheck disable=SC2086  # $extra is one optional flag or nothing, by construction
        ( set -e; cmd_ingest "$dir" "$scope" $extra )
        rc=$?
        set -e
        [ "$rc" -eq 0 ] || failed="$failed $scope"
        listed="${listed:+$listed,}$scope"
    done < "$file"

    # Which scopes does the STORE hold that this list never names? Nothing ties the two together:
    # a directory ingested once by hand and never added to the list is never refreshed again, and
    # search keeps answering about it from whatever it held that day -- with no error, no empty
    # result and no sign that anything is wrong. That is this project's whole subject matter, and
    # the refresh command had the same blind spot it was written to close.
    #
    # A note and not a failure: a scope whose directory is gone for good is a legitimate resident
    # of the store, its documents still searchable history. Failing the run nightly over one would
    # teach whoever reads the output to stop reading it.
    step "Checking the list against the store"
    for r in $(psql "$DATABASE_URL" -tAX -c \
               "SELECT repo FROM documents GROUP BY repo ORDER BY repo" 2>/dev/null); do
        case ",$listed," in *",$r,"*) continue ;; esac
        printf -- '-- %s: in the store, not in %s -- nothing refreshes it\n' "$r" "$file"
    done

    # The check that makes the loop mean something. A chunk with no embedding raises no error
    # anywhere: it is simply absent from the vector half of every search, while the document it
    # belongs to still counts as indexed and the scope still counts as refreshed.
    step "Checking that nothing is left unembedded"
    if ! left=$(psql "$DATABASE_URL" -tAX -v ON_ERROR_STOP=1 \
                -c "SELECT count(*) FROM chunks WHERE embedding IS NULL" 2>&1); then
        printf '!! could not count unembedded chunks: %s\n' "$left" >&2
        failed="$failed count"
    else
        case "$left" in
            0) printf -- '-- all chunks embedded\n' ;;
            ''|*[!0-9]*) printf '!! unexpected count output: %s\n' "$left" >&2; failed="$failed count" ;;
            *) printf '!! %s chunk(s) left without an embedding\n' "$left" >&2
               failed="$failed embeddings:$left" ;;
        esac
    fi

    [ -z "$failed" ] || die "refresh finished with problems:$failed"
    printf '\nrefresh clean\n'
}

[ $# -ge 1 ] || usage 0
sub=$1; shift
case "$sub" in
    init)   cmd_init "$@" ;;
    ingest) cmd_ingest "$@" ;;
    refresh) cmd_refresh "$@" ;;
    doctor)  exec "$PY" "$ROOT/ci/doctor.py" "$@" ;;
    latency) exec "$PY" "$ROOT/ci/latency.py" "$@" ;;
    -h|--help|help) usage 0 ;;
    *)      die "unknown command: $sub (try ./hm help)" ;;
esac