Skip to content

ci: multi-transport (subprocess/http/unix) SQL E2E + externalize streaming cursors #2

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
rustyconover merged 1 commit into from
Jun 24, 2026
Merged

ci: multi-transport (subprocess/http/unix) SQL E2E + externalize streaming cursors #2

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
19 changes: 17 additions & 2 deletions .github/workflows/ci.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -53,10 +53,23 @@ jobs:
# SQL end-to-end: run the sqllogictest suite (test/sql/*.test) against the
# built Go worker through the real signed `vgi` community DuckDB extension via
# a prebuilt standalone `haybarn-unittest` — no C++ build. See ci/README.md.
#
# Transport matrix: the same suite runs over each transport the vgi extension
# supports, selected by ci/run-integration.sh's TRANSPORT env var (which
# changes what the .test files ATTACH as the worker LOCATION):
# subprocess worker spawned over stdio (the binary path)
# http worker started with --http, LOCATION = http://127.0.0.1:<port>
# unix worker started with --unix <sock>, LOCATION = unix://<sock>
# The mock OData server is started for ALL transports (the worker's table
# functions still call it). See ci/README.md for the per-transport notes.
integration:
name: SQL end-to-end (haybarn)
name: SQL E2E (${{ matrix.transport }})
needs: resolve-haybarn
runs-on: ubuntu-latest
strategy:
fail-fast: false
matrix:
transport: [subprocess, http, unix]
steps:
- uses: actions/checkout@v4

Expand Down Expand Up @@ -88,5 +101,7 @@ jobs:
echo "HAYBARN_UNITTEST=$UNITTEST" >> "$GITHUB_ENV"
echo "VGI_ODATA_WORKER=$PWD/vgi-odata-worker" >> "$GITHUB_ENV"

- name: Run extension integration suite
- name: Run extension integration suite (${{ matrix.transport }})
run: ci/run-integration.sh
env:
TRANSPORT: ${{ matrix.transport }}
182 changes: 164 additions & 18 deletions ci/run-integration.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,42 +5,82 @@
# VGI worker, using a prebuilt standalone `haybarn-unittest` and the signed
# community `vgi` extension — no C++ build from source. See ci/README.md.
#
# The odata worker queries OData services over HTTP, so the suite needs a
# server: this script builds the repo's `mockserver`, starts it on a free port,
# and points the tests at it via VGI_ODATA_TEST_URL (mirroring `make test-sql`).
# Multi-transport: the same suite runs over whichever transport the TRANSPORT
# env var selects, by changing what `VGI_ODATA_WORKER` resolves to (the vgi
# extension picks the transport from the ATTACH LOCATION string):
#
# subprocess (default) VGI_ODATA_WORKER = the stdio worker binary
# -> extension spawns it over stdin/stdout.
# http start `<worker> --http` (prints "PORT:<n>"), parse the
# port, VGI_ODATA_WORKER = http://127.0.0.1:<port>.
# (The extension POSTs each RPC method at <LOCATION>/<method>;
# the SDK mounts them at the root, so LOCATION has no path.)
# unix start `<worker> --unix /tmp/odata.sock` (prints
# "UNIX:<path>"), VGI_ODATA_WORKER = unix:///tmp/odata.sock.
#
# In every transport the odata worker queries OData over HTTP, so the suite
# ALWAYS needs the mock OData server: this script builds the repo's `mockserver`,
# starts it on a free port, and points the tests at it via VGI_ODATA_TEST_URL
# (mirroring `make test-sql`). All started processes are trap-killed on exit.
#
# Required environment:
# HAYBARN_UNITTEST path to the haybarn-unittest binary
# VGI_ODATA_WORKER worker LOCATION the .test files ATTACH (the built Go
# worker binary the vgi extension spawns over stdio)
# VGI_ODATA_WORKER for TRANSPORT=subprocess: the worker LOCATION the .test
# files ATTACH (the built Go worker binary, spawned over
# stdio). For http/unix this is OVERRIDDEN by this script,
# but the binary it points at is reused to launch the
# out-of-band server, so it must still be the worker path.
# Optional:
# TRANSPORT subprocess (default) | http | unix
# STAGE scratch dir for the preprocessed test tree (default: mktemp)
set -euo pipefail

: "${HAYBARN_UNITTEST:?path to the haybarn-unittest binary}"
: "${VGI_ODATA_WORKER:?worker LOCATION (the built Go worker binary)}"

TRANSPORT="${TRANSPORT:-subprocess}"
case "$TRANSPORT" in
subprocess|http|unix) ;;
*) echo "ERROR: unknown TRANSPORT='$TRANSPORT' (expected subprocess|http|unix)" >&2; exit 2 ;;
esac

HERE="$(cd "$(dirname "$0")" && pwd)"
REPO="$(cd "$HERE/.." && pwd)"
STAGE="${STAGE:-$(mktemp -d)}"

# --- Start the mock OData server (the .test files query it) -----------------
# The worker binary the subprocess transport ATTACHes to is also the binary we
# launch out-of-band for http/unix. Capture it before we possibly overwrite
# VGI_ODATA_WORKER with a URL.
WORKER_BIN="$VGI_ODATA_WORKER"

# Collected PIDs and paths to clean up on exit (mock + optional worker server).
MOCK_PID=""
WORKER_PID=""
UNIX_SOCK=""
cleanup() {
# Preserve the script's exit status: this runs on EXIT, so its own last
# command must not clobber the real exit code (a bare `[ -n "$x" ]` that is
# false returns 1 and would turn a green run red).
local rc=$?
if [ -n "$WORKER_PID" ]; then kill "$WORKER_PID" 2>/dev/null || true; wait "$WORKER_PID" 2>/dev/null || true; fi
if [ -n "$MOCK_PID" ]; then kill "$MOCK_PID" 2>/dev/null || true; wait "$MOCK_PID" 2>/dev/null || true; fi
if [ -n "$UNIX_SOCK" ]; then rm -f "$UNIX_SOCK"; fi
return "$rc"
}
trap cleanup EXIT

# --- Start the mock OData server (the .test files query it; all transports) ---
# Build + launch the repo's standalone mock server on a free port; it prints
# "PORT:<n>" on stdout (see cmd/mockserver/main.go). We capture that, export
# VGI_ODATA_TEST_URL, and kill the server on exit — exactly like `make test-sql`.
# "PORT:<n>" on stdout (see cmd/mockserver/main.go). We capture that and export
# VGI_ODATA_TEST_URL. The mock is required for every transport — the worker still
# makes the HTTP call.
MOCK_BIN="$STAGE/mockserver"
echo "Building mock OData server ..."
( cd "$REPO" && go build -o "$MOCK_BIN" ./cmd/mockserver )

MOCK_PORT_FILE="$(mktemp)"
"$MOCK_BIN" --addr 127.0.0.1:0 >"$MOCK_PORT_FILE" 2>/dev/null &
MOCK_PID=$!
cleanup() {
kill "$MOCK_PID" 2>/dev/null || true
wait "$MOCK_PID" 2>/dev/null || true
rm -f "$MOCK_PORT_FILE"
}
trap cleanup EXIT

PORT=""
for _ in $(seq 1 30); do
Expand All @@ -52,16 +92,99 @@ if [ -z "$PORT" ]; then
echo "ERROR: mock server did not report a port" >&2
exit 1
fi
rm -f "$MOCK_PORT_FILE"
export VGI_ODATA_TEST_URL="http://127.0.0.1:$PORT"
echo "Mock OData server listening on $VGI_ODATA_TEST_URL (pid $MOCK_PID)"

# --- Per-transport: resolve VGI_ODATA_WORKER (the ATTACH LOCATION) -----------
# subprocess keeps the binary path (extension spawns stdio). http/unix start the
# worker out-of-band and hand the extension a URL.
case "$TRANSPORT" in
subprocess)
echo "Transport: subprocess/stdio — VGI_ODATA_WORKER=$VGI_ODATA_WORKER"
;;

http)
# Start the worker in --http mode; it prints "PORT:<n>" once listening.
WORKER_PORT_FILE="$(mktemp)"
echo "Transport: http — starting '$WORKER_BIN --http' ..."
"$WORKER_BIN" --http >"$WORKER_PORT_FILE" 2>/dev/null &
WORKER_PID=$!
WPORT=""
for _ in $(seq 1 50); do
WPORT="$(sed -n 's/^PORT:\([0-9][0-9]*\)$/\1/p' "$WORKER_PORT_FILE" 2>/dev/null | head -1)"
[ -n "$WPORT" ] && break
kill -0 "$WORKER_PID" 2>/dev/null || { echo "ERROR: http worker exited before reporting a port" >&2; cat "$WORKER_PORT_FILE" >&2 || true; exit 1; }
sleep 0.2
done
rm -f "$WORKER_PORT_FILE"
if [ -z "$WPORT" ]; then
echo "ERROR: http worker did not report a port" >&2
exit 1
fi
# The LOCATION must be the bare scheme://host:port with NO path (the
# extension POSTs each RPC method at <LOCATION>/<method>, mounted at root).
export VGI_ODATA_WORKER="http://127.0.0.1:$WPORT"
echo "HTTP worker listening on $VGI_ODATA_WORKER (pid $WORKER_PID)"
;;

unix)
# Start the worker on an AF_UNIX socket; it prints "UNIX:<path>" once
# listening. idleTimeout is disabled (we own the process lifecycle).
UNIX_SOCK="${TMPDIR:-/tmp}/odata.$$.sock"
rm -f "$UNIX_SOCK"
WORKER_OUT_FILE="$(mktemp)"
echo "Transport: unix — starting '$WORKER_BIN --unix $UNIX_SOCK' ..."
"$WORKER_BIN" --unix "$UNIX_SOCK" >"$WORKER_OUT_FILE" 2>/dev/null &
WORKER_PID=$!
READY=""
for _ in $(seq 1 50); do
if grep -q '^UNIX:' "$WORKER_OUT_FILE" 2>/dev/null && [ -S "$UNIX_SOCK" ]; then
READY=1; break
fi
kill -0 "$WORKER_PID" 2>/dev/null || { echo "ERROR: unix worker exited before the socket was ready" >&2; cat "$WORKER_OUT_FILE" >&2 || true; exit 1; }
sleep 0.2
done
rm -f "$WORKER_OUT_FILE"
if [ -z "$READY" ]; then
echo "ERROR: unix worker did not report a ready socket at $UNIX_SOCK" >&2
exit 1
fi
export VGI_ODATA_WORKER="unix://$UNIX_SOCK"
echo "Unix worker listening on $VGI_ODATA_WORKER (pid $WORKER_PID)"
;;
esac

# --- Stage the preprocessed tests -------------------------------------------
echo "Staging preprocessed tests into $STAGE ..."
mkdir -p "$STAGE/test/sql"
for f in "$REPO"/test/sql/*.test; do
awk -f "$HERE/preprocess-require.awk" "$f" > "$STAGE/test/sql/$(basename "$f")"
done

# The HTTP transport drives the worker-RPC POSTs through DuckDB's HTTP client,
# which is only registered when the `httpfs` extension is loaded. The .test
# files only `LOAD vgi`, so over HTTP those POSTs fail with an "HTTP"-flavoured
# error (which the runner then silently SKIPS). Inject a signed
# `INSTALL httpfs FROM core; LOAD httpfs;` after each `LOAD vgi;` for the http
# transport only (subprocess/unix do not use the HTTP client).
if [ "$TRANSPORT" = "http" ]; then
echo "Transport http: injecting 'LOAD httpfs' (required for the worker HTTP RPC) ..."
for f in "$STAGE"/test/sql/*.test; do
awk '
{ print }
/^LOAD[ \t]+vgi;[ \t]*$/ {
print "";
print "statement ok";
print "INSTALL httpfs FROM core;";
print "";
print "statement ok";
print "LOAD httpfs;";
}
' "$f" > "$f.tmp" && mv "$f.tmp" "$f"
done
fi

cd "$STAGE"

# Warm the extension cache once: vgi from the signed community channel. A miss
Expand All @@ -79,7 +202,30 @@ EOF
"$HAYBARN_UNITTEST" "test/_warm.test" >/dev/null 2>&1 || echo "::warning::extension warm step did not fully succeed"
rm -f "$STAGE/test/_warm.test"

# Run the whole suite in one invocation, streaming the runner's native
# sqllogictest report. Any failed assertion exits non-zero and fails the job.
echo "Running suite (worker: $VGI_ODATA_WORKER) ..."
"$HAYBARN_UNITTEST" "test/sql/*"
# Run the whole suite in one invocation, capturing the runner's native
# sqllogictest report so we can both stream it AND guard against a silent skip.
#
# IMPORTANT: the DuckDB/Haybarn sqllogictest runner SKIPS (not fails, exit 0) a
# test whose error message matches a built-in network-error allowlist that
# includes the substring "HTTP". So a broken HTTP transport would otherwise show
# "All tests were skipped" and the job would go GREEN having run nothing — a
# fake pass. We detect that and fail explicitly.
echo "Running suite (transport: $TRANSPORT, worker: $VGI_ODATA_WORKER) ..."
RUN_LOG="$STAGE/run.log"
set +e
"$HAYBARN_UNITTEST" "test/sql/*" 2>&1 | tee "$RUN_LOG"
RUN_RC="${PIPESTATUS[0]}"
set -e

if [ "$RUN_RC" -ne 0 ]; then
echo "ERROR: suite failed (transport: $TRANSPORT, rc=$RUN_RC)" >&2
exit "$RUN_RC"
fi

if grep -q 'All tests were skipped' "$RUN_LOG"; then
echo "ERROR: every test was SKIPPED on transport '$TRANSPORT' (the runner's" >&2
echo " built-in network-error skip swallowed the real error). This is" >&2
echo " NOT a pass. Skip reason reported by the runner:" >&2
grep -A3 'Skipped tests for the following reasons' "$RUN_LOG" >&2 || true
exit 1
fi
18 changes: 15 additions & 3 deletions cmd/vgi-odata-worker/main.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,15 +16,18 @@ import (
)

func main() {
// Accept --http for HTTP transport; default is stdio. Unknown launcher flags
// are tolerated (the VGI extension varies argv to key its worker cache), so
// we filter to flags we actually define before parsing.
// Accept --http for HTTP transport and --unix for the AF_UNIX launcher
// transport; default is stdio. Unknown launcher flags are tolerated (the
// VGI extension varies argv to key its worker cache), so we filter to flags
// we actually define before parsing.
httpMode := flag.Bool("http", false, "Run as an HTTP server instead of stdio")
unixPath := flag.String("unix", "", "Serve the AF_UNIX launcher transport on this socket path instead of stdio")
logFlags := vgi.RegisterLoggingFlags(flag.CommandLine)
_ = flag.CommandLine.Parse(filterKnownFlags(os.Args[1:], map[string]bool{
"log-level": true,
"log-format": true,
"log-logger": true,
"unix": true,
}))
if err := logFlags.Apply(); err != nil {
log.Fatalf("logging flags: %v", err)
Expand All @@ -45,6 +48,15 @@ func main() {
}
return
}
if *unixPath != "" {
// AF_UNIX launcher transport: serve on the given socket path. The SDK
// prints "UNIX:<path>" once listening; idleTimeout=0 disables the
// self-shutdown timer (the launcher/CI owns the process lifecycle).
if err := w.RunUnix(*unixPath, 0); err != nil {
log.Fatal(err)
}
return
}
w.RunStdio()
}

Expand Down
Loading
Loading