Add headers to web_fetch

cmd/clawlet/templates/TOOLS.md.tmpl

@@ -54,7 +54,7 @@ web_search(query: string, count?: int) -> string
 ### web_fetch
 Fetch a URL and extract readable content. Returns a JSON object string with fields like `status` and `text`.
 ```text
-web_fetch(url: string, extractMode?: "markdown"|"text", maxChars?: int) -> string
+web_fetch(url: string, extractMode?: "markdown"|"text", maxChars?: int, headers?: {string: string}) -> string
 ```
 
 Notes:

tools/defs.go

@@ -1,6 +1,10 @@
 package tools
 
-import "github.com/mosaxiv/clawlet/llm"
+import (
+	"encoding/json"
+
+	"github.com/mosaxiv/clawlet/llm"
+)
 
 func defReadFile() llm.ToolDefinition {
 	return llm.ToolDefinition{
@@ -124,6 +128,9 @@ func defWebFetch() llm.ToolDefinition {
 						Enum: []string{"markdown", "text"},
 					},
 					"maxChars": {Type: "integer", Description: "Max characters in extracted text (default 50000)."},
+					"headers": {
+						Raw: json.RawMessage(`{"type":"object","description":"HTTP request headers to include (e.g. {\"Authorization\":\"Bearer token\"}).","additionalProperties":{"type":"string"}}`),
+					},
 				},
 				Required: []string{"url"},
 			},

tools/registry.go

@@ -157,14 +157,15 @@ func (r *Registry) Execute(ctx context.Context, tctx Context, name string, args
 		return r.readSkill(a.Name)
 	case "web_fetch":
 		var a struct {
-			URL         string `json:"url"`
-			ExtractMode string `json:"extractMode"`
-			MaxChars    int    `json:"maxChars"`
+			URL         string            `json:"url"`
+			ExtractMode string            `json:"extractMode"`
+			MaxChars    int               `json:"maxChars"`
+			Headers     map[string]string `json:"headers"`
 		}
 		if err := json.Unmarshal(args, &a); err != nil {
 			return "", err
 		}
-		return r.webFetch(ctx, a.URL, a.ExtractMode, a.MaxChars)
+		return r.webFetch(ctx, a.URL, a.ExtractMode, a.MaxChars, a.Headers)
 	case "web_search":
 		var a struct {
 			Query string `json:"query"`

tools/tool_web_fetch.go

@@ -13,7 +13,7 @@ import (
 	"time"
 )
 
-func (r *Registry) webFetch(ctx context.Context, rawURL string, extractMode string, maxChars int) (string, error) {
+func (r *Registry) webFetch(ctx context.Context, rawURL string, extractMode string, maxChars int, headers map[string]string) (string, error) {
 	rawURL = strings.TrimSpace(rawURL)
 	if rawURL == "" {
 		return "", errors.New("url is empty")
@@ -67,6 +67,9 @@ func (r *Registry) webFetch(ctx context.Context, rawURL string, extractMode stri
 		return "", err
 	}
 	req.Header.Set("User-Agent", "clawlet/0.1")
+	for k, v := range headers {
+		req.Header.Set(k, v)
+	}
 	resp, err := client.Do(req)
 	if err != nil {
 		b, _ := json.Marshal(outT{URL: rawURL, Status: 0, Extractor: "error", Truncated: false, Length: 0, Text: "", Error: err.Error()})

tools/tool_web_fetch_test.go

@@ -0,0 +1,113 @@
+package tools
+
+import (
+	"context"
+	"encoding/json"
+	"net/http"
+	"net/http/httptest"
+	"testing"
+	"time"
+)
+
+func newTestRegistry() *Registry {
+	return &Registry{
+		WorkspaceDir: "/tmp",
+		ExecTimeout:  5 * time.Second,
+	}
+}
+
+func TestWebFetch_BasicGet(t *testing.T) {
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.Header().Set("Content-Type", "text/plain")
+		w.Write([]byte("hello world"))
+	}))
+	defer srv.Close()
+
+	r := newTestRegistry()
+	out, err := r.webFetch(context.Background(), srv.URL, "text", 0, nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	var result map[string]any
+	if err := json.Unmarshal([]byte(out), &result); err != nil {
+		t.Fatalf("invalid JSON: %v", err)
+	}
+	if status := result["status"].(float64); status != 200 {
+		t.Fatalf("expected status 200, got %v", status)
+	}
+	if text := result["text"].(string); text != "hello world" {
+		t.Fatalf("unexpected text: %q", text)
+	}
+}
+
+func TestWebFetch_HeadersForwarded(t *testing.T) {
+	var gotAuth string
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		gotAuth = r.Header.Get("Authorization")
+		w.Header().Set("Content-Type", "text/plain")
+		w.Write([]byte("ok"))
+	}))
+	defer srv.Close()
+
+	r := newTestRegistry()
+	headers := map[string]string{"Authorization": "Bearer secret"}
+	_, err := r.webFetch(context.Background(), srv.URL, "text", 0, headers)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if gotAuth != "Bearer secret" {
+		t.Fatalf("expected Authorization header to be forwarded, got %q", gotAuth)
+	}
+}
+
+func TestWebFetch_NilHeaders(t *testing.T) {
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.Write([]byte("ok"))
+	}))
+	defer srv.Close()
+
+	r := newTestRegistry()
+	// nil headers must not panic
+	_, err := r.webFetch(context.Background(), srv.URL, "text", 0, nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+}
+
+func TestWebFetch_InvalidURL(t *testing.T) {
+	r := newTestRegistry()
+	_, err := r.webFetch(context.Background(), "", "text", 0, nil)
+	if err == nil {
+		t.Fatal("expected error for empty URL")
+	}
+	_, err = r.webFetch(context.Background(), "ftp://example.com", "text", 0, nil)
+	if err == nil {
+		t.Fatal("expected error for non-http scheme")
+	}
+}
+
+func TestWebFetch_ExecuteDispatch(t *testing.T) {
+	var gotAccept string
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		gotAccept = r.Header.Get("Accept")
+		w.Header().Set("Content-Type", "application/json")
+		w.Write([]byte(`{"ok":true}`))
+	}))
+	defer srv.Close()
+
+	r := newTestRegistry()
+	args, _ := json.Marshal(map[string]any{
+		"url":     srv.URL,
+		"headers": map[string]string{"Accept": "application/json"},
+	})
+	out, err := r.Execute(context.Background(), Context{}, "web_fetch", args)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if out == "" {
+		t.Fatal("expected non-empty output")
+	}
+	if gotAccept != "application/json" {
+		t.Fatalf("expected Accept header forwarded, got %q", gotAccept)
+	}
+}