From c72f1b93b8b35055959a08d0c39da7c1fa63e697 Mon Sep 17 00:00:00 2001 From: Austin Richardson Date: Fri, 20 Feb 2026 20:51:22 -0800 Subject: [PATCH] Add headers to web_fetch run actions audy-2 v0.2.7-audytest1 --- cmd/clawlet/templates/TOOLS.md.tmpl | 2 +- tools/defs.go | 9 ++- tools/registry.go | 9 ++- tools/tool_web_fetch.go | 5 +- tools/tool_web_fetch_test.go | 113 ++++++++++++++++++++++++++++ 5 files changed, 131 insertions(+), 7 deletions(-) create mode 100644 tools/tool_web_fetch_test.go diff --git a/cmd/clawlet/templates/TOOLS.md.tmpl b/cmd/clawlet/templates/TOOLS.md.tmpl index 254e1cc..0061d32 100644 --- a/cmd/clawlet/templates/TOOLS.md.tmpl +++ b/cmd/clawlet/templates/TOOLS.md.tmpl @@ -54,7 +54,7 @@ web_search(query: string, count?: int) -> string ### web_fetch Fetch a URL and extract readable content. Returns a JSON object string with fields like `status` and `text`. ```text -web_fetch(url: string, extractMode?: "markdown"|"text", maxChars?: int) -> string +web_fetch(url: string, extractMode?: "markdown"|"text", maxChars?: int, headers?: {string: string}) -> string ``` Notes: diff --git a/tools/defs.go b/tools/defs.go index a319c10..0b6fd36 100644 --- a/tools/defs.go +++ b/tools/defs.go @@ -1,6 +1,10 @@ package tools -import "github.com/mosaxiv/clawlet/llm" +import ( + "encoding/json" + + "github.com/mosaxiv/clawlet/llm" +) func defReadFile() llm.ToolDefinition { return llm.ToolDefinition{ @@ -124,6 +128,9 @@ func defWebFetch() llm.ToolDefinition { Enum: []string{"markdown", "text"}, }, "maxChars": {Type: "integer", Description: "Max characters in extracted text (default 50000)."}, + "headers": { + Raw: json.RawMessage(`{"type":"object","description":"HTTP request headers to include (e.g. {\"Authorization\":\"Bearer token\"}).","additionalProperties":{"type":"string"}}`), + }, }, Required: []string{"url"}, }, diff --git a/tools/registry.go b/tools/registry.go index 47408dd..36d4bc2 100644 --- a/tools/registry.go +++ b/tools/registry.go @@ -157,14 +157,15 @@ func (r *Registry) Execute(ctx context.Context, tctx Context, name string, args return r.readSkill(a.Name) case "web_fetch": var a struct { - URL string `json:"url"` - ExtractMode string `json:"extractMode"` - MaxChars int `json:"maxChars"` + URL string `json:"url"` + ExtractMode string `json:"extractMode"` + MaxChars int `json:"maxChars"` + Headers map[string]string `json:"headers"` } if err := json.Unmarshal(args, &a); err != nil { return "", err } - return r.webFetch(ctx, a.URL, a.ExtractMode, a.MaxChars) + return r.webFetch(ctx, a.URL, a.ExtractMode, a.MaxChars, a.Headers) case "web_search": var a struct { Query string `json:"query"` diff --git a/tools/tool_web_fetch.go b/tools/tool_web_fetch.go index e53d272..ef0fdea 100644 --- a/tools/tool_web_fetch.go +++ b/tools/tool_web_fetch.go @@ -13,7 +13,7 @@ import ( "time" ) -func (r *Registry) webFetch(ctx context.Context, rawURL string, extractMode string, maxChars int) (string, error) { +func (r *Registry) webFetch(ctx context.Context, rawURL string, extractMode string, maxChars int, headers map[string]string) (string, error) { rawURL = strings.TrimSpace(rawURL) if rawURL == "" { return "", errors.New("url is empty") @@ -67,6 +67,9 @@ func (r *Registry) webFetch(ctx context.Context, rawURL string, extractMode stri return "", err } req.Header.Set("User-Agent", "clawlet/0.1") + for k, v := range headers { + req.Header.Set(k, v) + } resp, err := client.Do(req) if err != nil { b, _ := json.Marshal(outT{URL: rawURL, Status: 0, Extractor: "error", Truncated: false, Length: 0, Text: "", Error: err.Error()}) diff --git a/tools/tool_web_fetch_test.go b/tools/tool_web_fetch_test.go new file mode 100644 index 0000000..9d8ce66 --- /dev/null +++ b/tools/tool_web_fetch_test.go @@ -0,0 +1,113 @@ +package tools + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "testing" + "time" +) + +func newTestRegistry() *Registry { + return &Registry{ + WorkspaceDir: "/tmp", + ExecTimeout: 5 * time.Second, + } +} + +func TestWebFetch_BasicGet(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "text/plain") + w.Write([]byte("hello world")) + })) + defer srv.Close() + + r := newTestRegistry() + out, err := r.webFetch(context.Background(), srv.URL, "text", 0, nil) + if err != nil { + t.Fatal(err) + } + var result map[string]any + if err := json.Unmarshal([]byte(out), &result); err != nil { + t.Fatalf("invalid JSON: %v", err) + } + if status := result["status"].(float64); status != 200 { + t.Fatalf("expected status 200, got %v", status) + } + if text := result["text"].(string); text != "hello world" { + t.Fatalf("unexpected text: %q", text) + } +} + +func TestWebFetch_HeadersForwarded(t *testing.T) { + var gotAuth string + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + gotAuth = r.Header.Get("Authorization") + w.Header().Set("Content-Type", "text/plain") + w.Write([]byte("ok")) + })) + defer srv.Close() + + r := newTestRegistry() + headers := map[string]string{"Authorization": "Bearer secret"} + _, err := r.webFetch(context.Background(), srv.URL, "text", 0, headers) + if err != nil { + t.Fatal(err) + } + if gotAuth != "Bearer secret" { + t.Fatalf("expected Authorization header to be forwarded, got %q", gotAuth) + } +} + +func TestWebFetch_NilHeaders(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Write([]byte("ok")) + })) + defer srv.Close() + + r := newTestRegistry() + // nil headers must not panic + _, err := r.webFetch(context.Background(), srv.URL, "text", 0, nil) + if err != nil { + t.Fatal(err) + } +} + +func TestWebFetch_InvalidURL(t *testing.T) { + r := newTestRegistry() + _, err := r.webFetch(context.Background(), "", "text", 0, nil) + if err == nil { + t.Fatal("expected error for empty URL") + } + _, err = r.webFetch(context.Background(), "ftp://example.com", "text", 0, nil) + if err == nil { + t.Fatal("expected error for non-http scheme") + } +} + +func TestWebFetch_ExecuteDispatch(t *testing.T) { + var gotAccept string + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + gotAccept = r.Header.Get("Accept") + w.Header().Set("Content-Type", "application/json") + w.Write([]byte(`{"ok":true}`)) + })) + defer srv.Close() + + r := newTestRegistry() + args, _ := json.Marshal(map[string]any{ + "url": srv.URL, + "headers": map[string]string{"Accept": "application/json"}, + }) + out, err := r.Execute(context.Background(), Context{}, "web_fetch", args) + if err != nil { + t.Fatal(err) + } + if out == "" { + t.Fatal("expected non-empty output") + } + if gotAccept != "application/json" { + t.Fatalf("expected Accept header forwarded, got %q", gotAccept) + } +}