Add coverage for operations on primitive types and introduce JSON schemas

.github/workflows/test.yml

@@ -22,3 +22,14 @@ jobs:
 
     - name: Run tests
       run: make test
+
+    - name: Validate test suites
+      run: make suite-validate
+
+    - name: Check generated methods spec doc is up-to-date
+      run: |
+        make specgen
+        if ! git diff --exit-code -- docs/methods-spec.md; then
+          echo "Generated methods spec doc is out of date. Run 'make specgen' and commit the updated docs/methods-spec.md."
+          exit 1
+        fi

Makefile

@@ -1,10 +1,10 @@
-.PHONY: all build test clean runner mock-handler
+.PHONY: all build test clean runner mock-handler suite-validate specgen
 
 BUILD_DIR := build
 RUNNER_BIN := $(BUILD_DIR)/runner
 MOCK_HANDLER_BIN := $(BUILD_DIR)/mock-handler
 
-all: build test
+all: build test suite-validate
 
 build: runner mock-handler
 
@@ -18,12 +18,20 @@ mock-handler:
 	@mkdir -p $(BUILD_DIR)
 	go build -o $(MOCK_HANDLER_BIN) ./cmd/mock-handler
 
-test:
+test: build
 	@echo "Running runner unit tests..."
 	go test -v ./runner/...
 	@echo "Running conformance tests with mock handler..."
 	$(RUNNER_BIN) --handler $(MOCK_HANDLER_BIN) -vv
 
+suite-validate:
+	@echo "Validating testdata against the suite schema..."
+	go run ./cmd/suite-validate
+
+specgen:
+	@echo "Generating method reference from schemas..."
+	go run ./cmd/specgen
+
 clean:
 	@echo "Cleaning build artifacts..."
 	rm -rf $(BUILD_DIR)

README.md

@@ -30,10 +30,11 @@ The framework ensures that all language bindings (Go, Python, Rust, etc.) behave
 ```
 
 **This repository contains:**
-1. [**Handler Specification**](./docs/handler-spec.md): Defines the protocol, message formats, and test suites that handlers must implement
-2. [**Test Runner**](./cmd/runner/main.go): Spawns handler binary, sends test requests via stdin, validates responses from stdout
-3. [**Test Cases**](./testdata): JSON files defining requests and expected responses
-4. [**Mock Handler**](./cmd/mock-handler/main.go): Validates the runner by echoing expected responses from test cases
+1. [**Specification**](./docs/handler-spec.md): Defines the handler protocol and links to the generated [**Method Reference**](./docs/methods-spec.md) for method parameters, results, and errors
+2. [**Test Suites**](./testdata): JSON files defining requests and expected responses
+3. [**Test Runner**](./cmd/runner/main.go): Runs suites against a handler by sending test requests via stdin, validating responses from stdout, and checking them against the expected results
+4. [**Schemas**](./docs/schemas): Define the suite format and per-method request/response shapes, with tools in [`cmd/suite-validate`](./cmd/suite-validate) and [`cmd/specgen`](./cmd/specgen) for validation and documentation generation
+5. [**Mock Handler**](./cmd/mock-handler/main.go): Validates the runner by echoing expected responses from test cases
 
 ** **Handler binaries** are not hosted in this repository. They must be implemented separately following the [**Handler Specification**](./docs/handler-spec.md) and should:
 - Implement the JSON protocol for communication with the test runner
@@ -44,7 +45,11 @@ The framework ensures that all language bindings (Go, Python, Rust, etc.) behave
 
 ### Testing Your Binding (Custom Handler)
 
-Test your handler implementation using the test runner:
+Test your handler implementation using the test runner.
+
+You can download a prebuilt runner binary from the latest GitHub release. Tagged releases are published automatically by GoReleaser and include archives for supported platforms.
+
+If you prefer to build the runner from source:
 
 ```bash
 # Build the test runner
@@ -54,11 +59,15 @@ make runner
 ./build/runner --handler <path-to-your-handler>
 
 # Configure timeouts (optional)
+# Max wait per test case (default: 10s)
+# Total execution limit (default: 30s)
 ./build/runner --handler <path-to-your-handler> \
-  --handler-timeout 30s \  # Max wait per test case (default: 10s)
-  --timeout 2m             # Total execution limit (default: 30s)
+  --handler-timeout 30s \
+  --timeout 2m
 ```
 
+The runner validates each handler response against the method's JSON schema before comparing it with the expected test outcome.
+
 #### Timeout Flags
 
 - **`--handler-timeout`** (default: 10s): Maximum time to wait for handler response to each test case. Prevents hangs on unresponsive handlers.
@@ -85,7 +94,7 @@ The request chains printed by verbose mode can be directly piped to the handler
 #
 #       Response:
 #       ────────────────────────────────────────
-#       {"result":"$chain_ref"}
+#       {"result":{"ref":"$chain_ref"}}
 
 # Copy the request chain and pipe it to your handler for debugging:
 echo '{"id":"chain#1","method":"btck_context_create","params":{"chain_parameters":{"chain_type":"btck_ChainType_REGTEST"}},"ref":"$context_ref"}
@@ -103,4 +112,17 @@ make build
 
 # Run runner unit tests and integration tests with mock handler
 make test
+
+# Validate all suite JSON files against the suite schema
+make suite-validate
+```
+
+### Updating Generated Documentation
+
+When schema definitions change, regenerate the method reference:
+
+```bash
+make specgen
 ```
+
+Do not edit [`docs/methods-spec.md`](./docs/methods-spec.md) manually. It is generated from the schema definitions in [`docs/schemas/`](./docs/schemas), so schema changes should be followed by `make specgen`, and CI checks that the generated spec is up to date.

cmd/runner/main.go

@@ -68,7 +68,7 @@ func main() {
 	totalTests := 0
 
 	for _, testFile := range testFiles {
-		fmt.Printf("\n=== Running test suite: %s ===\n", testFile)
+		fmt.Printf("\n=== Running test suite ===\n")
 
 		// Load test suite from embedded FS
 		suite, err := runner.LoadTestSuiteFromFS(testdata.FS, testFile)
@@ -106,7 +106,7 @@ func main() {
 }
 
 func printResults(suite *runner.TestSuite, result runner.TestResult) {
-	fmt.Printf("\nTest Suite: %s\n", result.SuiteName)
+	fmt.Printf("\nTest Suite: %s (%s)\n", result.SuiteTitle, result.SuiteFileName)
 	if suite.Description != "" {
 		fmt.Printf("Description: %s\n", suite.Description)
 	}

cmd/specgen/main.go

@@ -0,0 +1,13 @@
+package main
+
+import (
+	"fmt"
+	"os"
+)
+
+func main() {
+	if err := GenerateMethodReference("docs/schemas", "docs/methods-spec.md"); err != nil {
+		fmt.Fprintln(os.Stderr, err)
+		os.Exit(1)
+	}
+}

cmd/specgen/schema.go

@@ -0,0 +1,123 @@
+package main
+
+import (
+	"fmt"
+	"path/filepath"
+	"strings"
+
+	"github.com/santhosh-tekuri/jsonschema/v6"
+)
+
+const (
+	docOrderKeyword  = "x-doc-order"
+	docOrderVocabURL = "urn:kernel-bindings-tests:specgen:doc-order"
+)
+
+var docOrderVocabulary = mustDocOrderVocabulary()
+
+type docOrderExt struct {
+	Order []string
+}
+
+func (e *docOrderExt) Validate(*jsonschema.ValidatorContext, any) {}
+
+func compileSchema(path string) (*jsonschema.Schema, error) {
+	abs, err := filepath.Abs(path)
+	if err != nil {
+		return nil, err
+	}
+
+	c := jsonschema.NewCompiler()
+	c.AssertVocabs()
+	// Preserve x-doc-order so generated parameter docs follow the schema's
+	// reader-facing order instead of the map iteration order.
+	c.RegisterVocabulary(docOrderVocabulary)
+	return c.Compile(abs)
+}
+
+// docOrder reads the custom x-doc-order extension attached during compilation.
+// The schema library keeps object properties in a map, so this preserves the
+// reader-facing order declared in the source schema.
+func docOrder(schema *jsonschema.Schema) []string {
+	schema = resolveSchema(schema)
+	if schema == nil {
+		return nil
+	}
+	for _, ext := range schema.Extensions {
+		docExt, ok := ext.(*docOrderExt)
+		if ok {
+			return docExt.Order
+		}
+	}
+	return nil
+}
+
+// resolveSchema unwraps compiled $ref wrappers to the concrete target schema.
+// The jsonschema library resolves references during compilation; this helper
+// just follows Schema.Ref pointers so callers can inspect the target fields.
+func resolveSchema(schema *jsonschema.Schema) *jsonschema.Schema {
+	for schema != nil {
+		if schema.Ref == nil {
+			return schema
+		}
+		schema = schema.Ref
+	}
+	return nil
+}
+
+// mustDocOrderVocabulary registers a tiny custom vocabulary just for x-doc-order.
+// The compiled extension is later read back by docOrder during markdown rendering.
+func mustDocOrderVocabulary() *jsonschema.Vocabulary {
+	meta, err := jsonschema.UnmarshalJSON(strings.NewReader(`{
+		"properties": {
+			"x-doc-order": {
+				"type": "array",
+				"items": { "type": "string" },
+				"uniqueItems": true
+			}
+		}
+	}`))
+	if err != nil {
+		panic(err)
+	}
+
+	c := jsonschema.NewCompiler()
+	if err := c.AddResource(docOrderVocabURL, meta); err != nil {
+		panic(err)
+	}
+	schema, err := c.Compile(docOrderVocabURL)
+	if err != nil {
+		panic(err)
+	}
+
+	return &jsonschema.Vocabulary{
+		URL:    docOrderVocabURL,
+		Schema: schema,
+		// Compile stores x-doc-order on the compiled schema for later doc rendering.
+		Compile: compileDocOrder,
+	}
+}
+
+// compileDocOrder validates the raw extension payload and stores it on the
+// compiled schema as a strongly typed helper value.
+func compileDocOrder(_ *jsonschema.CompilerContext, obj map[string]any) (jsonschema.SchemaExt, error) {
+	raw, ok := obj[docOrderKeyword]
+	if !ok {
+		return nil, nil
+	}
+
+	values, ok := raw.([]any)
+	if !ok {
+		return nil, fmt.Errorf("%s must be an array of strings", docOrderKeyword)
+	}
+
+	order := make([]string, 0, len(values))
+	for _, value := range values {
+		name, ok := value.(string)
+		if !ok {
+			return nil, fmt.Errorf("%s entries must be strings", docOrderKeyword)
+		}
+		order = append(order, name)
+	}
+	return &docOrderExt{Order: order}, nil
+}