From 5861896d6d3dbf6b1843a86b3fa8556bf298f754 Mon Sep 17 00:00:00 2001 From: Vaayne Date: Tue, 12 May 2026 10:14:13 +0800 Subject: [PATCH 1/5] =?UTF-8?q?=E2=9C=A8=20feat:=20runtime,=20env,=20and?= =?UTF-8?q?=20headers=20for=20site=20scripts?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Add EnvDef/Env to script Meta with ValidateEnv and ResolveHeaders - Add runtime-based engine routing (http → QuickJS, browser → Browser) - Inject resolved headers into QuickJS fetch and Browser CDP - Update CLI info/list commands to display runtime/env/headers - Add full metadata to exa/search sample script --- .gitignore | 2 + cmd/tap/completion.go | 32 ++-------- cmd/tap/site.go | 51 +++++++++++++++- engine/browser.go | 4 +- engine/engine.go | 11 +++- engine/engine_test.go | 8 +-- engine/quickjs.go | 10 +++- script/parser.go | 86 +++++++++++++++++++++++---- script/parser_test.go | 128 ++++++++++++++++++++++++++++++++++++++++ script/registry.go | 94 +++++++++++++++++++---------- script/registry_test.go | 42 ++++++++++--- sites/embed.go | 6 ++ sites/exa/search.js | 112 +++++++++++++++++++++++++++++++++++ tap.go | 58 ++++++++++++++++-- tap_test.go | 44 ++++++++++++++ transport/transport.go | 22 +++++-- 16 files changed, 608 insertions(+), 102 deletions(-) create mode 100644 sites/embed.go create mode 100644 sites/exa/search.js diff --git a/.gitignore b/.gitignore index 3328fa0..28b5b26 100644 --- a/.gitignore +++ b/.gitignore @@ -28,3 +28,5 @@ Thumbs.db .env !.env.example /tap + +.agents/sessions/ diff --git a/cmd/tap/completion.go b/cmd/tap/completion.go index 07280b8..34e5fff 100644 --- a/cmd/tap/completion.go +++ b/cmd/tap/completion.go @@ -2,12 +2,12 @@ package main import ( "context" - "errors" "fmt" "os" "strings" "github.com/urfave/cli/v3" + "github.com/vaayne/tap" "github.com/vaayne/tap/script" ) @@ -86,36 +86,12 @@ func loadCompletionRegistry(cmd *cli.Command) (*script.Registry, error) { if dir == "" { dir = defaultSitesDir() } - overrideDir := defaultLocalOverrideDir() - if cmd.Bool("local-only") { - return loadRegistryDir(overrideDir) - } - reg, err := script.NewRegistryWithOverride(dir, overrideDir) - if err == nil { - return reg, nil - } - if !errors.Is(err, os.ErrNotExist) { - return nil, err - } - - return loadRegistryDir(overrideDir) -} - -func loadRegistryDir(dir string) (*script.Registry, error) { - reg, err := script.NewRegistry(dir) - if err != nil { - if errors.Is(err, os.ErrNotExist) { - return emptyRegistry(), nil - } - return nil, err + if cmd.Bool("local-only") { + return tap.DefaultRegistry(overrideDir, "") } - return reg, nil -} - -func emptyRegistry() *script.Registry { - return &script.Registry{} + return tap.DefaultRegistry(dir, overrideDir) } func printCompletion(cmd *cli.Command, value, description string) { diff --git a/cmd/tap/site.go b/cmd/tap/site.go index 8d55d89..fc69ff0 100644 --- a/cmd/tap/site.go +++ b/cmd/tap/site.go @@ -146,9 +146,14 @@ func siteListCmd() *cli.Command { if _, after, ok := strings.Cut(actionName, "/"); ok { actionName = after } - fmt.Printf(" %-24s %s%s\n", + runtimeBadge := "" + if s.Meta.Runtime != "" && s.Meta.Runtime != "auto" { + runtimeBadge = dim(color, " ["+s.Meta.Runtime+"]") + } + fmt.Printf(" %-24s %s%s%s\n", green(color, actionName), s.Meta.Description, + runtimeBadge, argHints, ) } @@ -191,10 +196,54 @@ func siteInfoCmd() *cli.Command { fmt.Printf(" %s %s\n", bold(color, "Domain:"), s.Meta.Domain) + if s.Meta.Runtime != "" && s.Meta.Runtime != "auto" { + fmt.Printf(" %s %s\n\n", bold(color, "Runtime:"), s.Meta.Runtime) + } + if s.Meta.Example != "" { fmt.Printf(" %s %s\n", bold(color, "Example:"), s.Meta.Example) } + if len(s.Meta.Env) > 0 { + fmt.Printf("\n %s\n", bold(color, "Env:")) + envNames := make([]string, 0, len(s.Meta.Env)) + for name := range s.Meta.Env { + envNames = append(envNames, name) + } + sort.Strings(envNames) + for _, envName := range envNames { + def := s.Meta.Env[envName] + req := dim(color, "optional") + if def.Required { + req = yellow(color, "required") + } + fmt.Printf(" %-16s %s %s\n", + green(color, envName), + dim(color, "("+req+")"), + def.Description, + ) + } + } + + if len(s.Meta.Headers) > 0 { + fmt.Printf("\n %s\n", bold(color, "Headers:")) + headerKeys := make([]string, 0, len(s.Meta.Headers)) + for k := range s.Meta.Headers { + headerKeys = append(headerKeys, k) + } + sort.Strings(headerKeys) + for _, k := range headerKeys { + v := s.Meta.Headers[k] + if strings.Contains(v, "${") { + v = "***" + } + fmt.Printf(" %-16s %s\n", + green(color, k), + v, + ) + } + } + if len(s.Meta.Args) > 0 { fmt.Printf("\n %s\n", bold(color, "Arguments:")) // Sort args for consistent output diff --git a/engine/browser.go b/engine/browser.go index 64fbf0f..45612fc 100644 --- a/engine/browser.go +++ b/engine/browser.go @@ -23,7 +23,7 @@ func NewBrowser(tp *transport.Transport, pauseFn transport.PauseFunc) *Browser { func (b *Browser) Name() string { return "Browser" } func (b *Browser) Close() error { return nil } -func (b *Browser) Run(ctx context.Context, s *script.Script, args map[string]string) (any, error) { +func (b *Browser) Run(ctx context.Context, s *script.Script, args map[string]string, opts RunOpts) (any, error) { argsJSON, err := json.Marshal(args) if err != nil { return nil, fmt.Errorf("marshal args: %w", err) @@ -36,5 +36,5 @@ func (b *Browser) Run(ctx context.Context, s *script.Script, args map[string]str navURL = "https://" + s.Meta.Domain } - return b.transport.BrowseEvalWithPause(ctx, navURL, js, b.pauseFn) + return b.transport.BrowseEvalWithPause(ctx, navURL, js, b.pauseFn, opts.Headers) } diff --git a/engine/engine.go b/engine/engine.go index f735d0d..7e84829 100644 --- a/engine/engine.go +++ b/engine/engine.go @@ -10,10 +10,15 @@ import ( "github.com/vaayne/tap/script" ) +// RunOpts holds per-run configuration for script execution. +type RunOpts struct { + Headers map[string]string // resolved meta headers (env vars interpolated) +} + // Engine can execute a site script with arguments and return a JSON-compatible result. type Engine interface { // Run executes a script with the given arguments. - Run(ctx context.Context, s *script.Script, args map[string]string) (any, error) + Run(ctx context.Context, s *script.Script, args map[string]string, opts RunOpts) (any, error) // Name returns the engine name for logging. Name() string @@ -26,10 +31,10 @@ type Engine interface { // If a result contains an "error" field (e.g. {"error":"HTTP 400"}), it is // treated as a failure and the next engine is tried. // If all engines fail, returns the last error. -func RunScript(ctx context.Context, engines []Engine, s *script.Script, args map[string]string) (any, error) { +func RunScript(ctx context.Context, engines []Engine, s *script.Script, args map[string]string, opts RunOpts) (any, error) { var lastErr error for _, e := range engines { - result, err := e.Run(ctx, s, args) + result, err := e.Run(ctx, s, args, opts) if err != nil { lastErr = err log.Printf("%s failed: %v", e.Name(), err) diff --git a/engine/engine_test.go b/engine/engine_test.go index 12db31f..342351e 100644 --- a/engine/engine_test.go +++ b/engine/engine_test.go @@ -17,7 +17,7 @@ type mockEngine struct { func (m *mockEngine) Name() string { return m.name } func (m *mockEngine) Close() error { return nil } -func (m *mockEngine) Run(_ context.Context, _ *script.Script, _ map[string]string) (any, error) { +func (m *mockEngine) Run(_ context.Context, _ *script.Script, _ map[string]string, _ RunOpts) (any, error) { return m.result, m.err } @@ -27,7 +27,7 @@ func TestRunScript_FirstEngineSucceeds(t *testing.T) { &mockEngine{name: "slow", result: map[string]any{"ok": true}}, } - result, err := RunScript(context.Background(), engines, &script.Script{}, nil) + result, err := RunScript(context.Background(), engines, &script.Script{}, nil, RunOpts{}) if err != nil { t.Fatalf("unexpected error: %v", err) } @@ -43,7 +43,7 @@ func TestRunScript_FallbackToSecond(t *testing.T) { &mockEngine{name: "slow", result: map[string]any{"fallback": true}}, } - result, err := RunScript(context.Background(), engines, &script.Script{}, nil) + result, err := RunScript(context.Background(), engines, &script.Script{}, nil, RunOpts{}) if err != nil { t.Fatalf("unexpected error: %v", err) } @@ -59,7 +59,7 @@ func TestRunScript_AllFail(t *testing.T) { &mockEngine{name: "e2", err: fmt.Errorf("fail2")}, } - _, err := RunScript(context.Background(), engines, &script.Script{}, nil) + _, err := RunScript(context.Background(), engines, &script.Script{}, nil, RunOpts{}) if err == nil { t.Fatal("expected error, got nil") } diff --git a/engine/quickjs.go b/engine/quickjs.go index b934214..5e0e6ab 100644 --- a/engine/quickjs.go +++ b/engine/quickjs.go @@ -27,7 +27,7 @@ func NewQuickJS(tp *transport.Transport) *QuickJS { func (q *QuickJS) Name() string { return "QuickJS" } func (q *QuickJS) Close() error { return nil } -func (q *QuickJS) Run(_ context.Context, s *script.Script, args map[string]string) (result any, err error) { +func (q *QuickJS) Run(_ context.Context, s *script.Script, args map[string]string, opts RunOpts) (result any, err error) { // The QJS WASM runtime can panic on certain async patterns (e.g. out of // bounds memory access). Recover so the engine fallback chain continues. defer func() { @@ -44,7 +44,7 @@ func (q *QuickJS) Run(_ context.Context, s *script.Script, args map[string]strin defer rt.Close() ctx := rt.Context() - injectFetch(ctx, q.transport) + injectFetch(ctx, q.transport, opts.Headers) argsJSON, qErr := json.Marshal(args) if qErr != nil { @@ -83,7 +83,8 @@ func stringify(ctx *qjs.Context, val *qjs.Value) string { } // injectFetch adds a fetch() function backed by the shared transport's HTTP client. -func injectFetch(ctx *qjs.Context, tp *transport.Transport) { +// metaHeaders are set first; JS-level headers override them. +func injectFetch(ctx *qjs.Context, tp *transport.Transport, metaHeaders map[string]string) { ctx.SetAsyncFunc("fetch", func(this *qjs.This) { c := this.Context() @@ -127,6 +128,9 @@ func injectFetch(ctx *qjs.Context, tp *transport.Transport) { req.Header.Set("User-Agent", transport.UserAgent) req.Header.Set("Accept", "application/json, text/plain, */*") + for k, v := range metaHeaders { + req.Header.Set(k, v) + } for k, v := range headers { req.Header.Set(k, v) } diff --git a/script/parser.go b/script/parser.go index c1af4d3..a43e1a5 100644 --- a/script/parser.go +++ b/script/parser.go @@ -4,6 +4,7 @@ package script import ( "encoding/json" "fmt" + "os" "strings" ) @@ -13,23 +14,43 @@ type ArgDef struct { Description string `json:"description"` } +// EnvDef describes a single environment variable dependency for a script. +type EnvDef struct { + Required bool `json:"required"` + Description string `json:"description"` +} + +// ScriptSource identifies where a script was loaded from. +type ScriptSource int + +const ( + ScriptSourceCache ScriptSource = iota // ~/.cache/tap/sites/ + ScriptSourceBuiltin // sites/ (embedded) + ScriptSourceOverride // ~/.config/tap/sites/ +) + // Meta holds the metadata extracted from a script's @meta block. type Meta struct { - Name string `json:"name"` - Description string `json:"description"` - Domain string `json:"domain"` - Args map[string]ArgDef `json:"args"` - ReadOnly bool `json:"readOnly"` - Example string `json:"example"` + Name string `json:"name"` + Description string `json:"description"` + Domain string `json:"domain"` + Args map[string]ArgDef `json:"args"` + ReadOnly bool `json:"readOnly"` + Example string `json:"example"` + Capabilities []string `json:"capabilities"` + Runtime string `json:"runtime"` + AuthRequired bool `json:"authRequired"` + Headers map[string]string `json:"headers"` + Env map[string]EnvDef `json:"env"` } // Script represents a parsed site script with metadata and function body. type Script struct { - Meta Meta - Body string // the async function body - Raw string // full file content - Path string // file path - LocalOverride bool // true when loaded from local override dir + Meta Meta + Body string // the async function body + Raw string // full file content + Path string // file path + Source ScriptSource // where the script was loaded from } // Parse parses a script file content, extracting @meta JSON and the function body. @@ -79,6 +100,49 @@ func parseMeta(content string) (*Meta, error) { return &meta, nil } +// ValidateEnv checks that all required environment variables are set. +func (m *Meta) ValidateEnv() error { + var missing []string + for name, def := range m.Env { + if def.Required { + if _, ok := os.LookupEnv(name); !ok { + missing = append(missing, fmt.Sprintf("%s (%s)", name, def.Description)) + } + } + } + if len(missing) > 0 { + return fmt.Errorf("missing required environment variables: %s", strings.Join(missing, ", ")) + } + return nil +} + +// ResolveHeaders copies Headers and interpolates ${ENV_VAR} values via os.Getenv. +// Headers referencing unset environment variables are skipped entirely. +func (m *Meta) ResolveHeaders() map[string]string { + result := make(map[string]string, len(m.Headers)) + for k, v := range m.Headers { + expanded, ok := expandEnv(v) + if !ok { + continue + } + result[k] = expanded + } + return result +} + +func expandEnv(s string) (string, bool) { + missing := false + expanded := os.Expand(s, func(key string) string { + val, ok := os.LookupEnv(key) + if !ok { + missing = true + return "" + } + return val + }) + return expanded, !missing +} + func parseBody(content string) (string, error) { end := strings.Index(content, "*/") if end == -1 { diff --git a/script/parser_test.go b/script/parser_test.go index ad055dd..9462e59 100644 --- a/script/parser_test.go +++ b/script/parser_test.go @@ -1,6 +1,7 @@ package script import ( + "strings" "testing" ) @@ -80,6 +81,133 @@ func TestParse_NoBody(t *testing.T) { } } +func TestParse_EnvDef(t *testing.T) { + content := `/* @meta +{ + "name": "test/env", + "description": "env test", + "domain": "example.com", + "env": { + "API_KEY": {"required": true, "description": "API key for service"}, + "OPTIONAL_VAR": {"required": false, "description": "Optional setting"} + } +} +*/ + +async function(args) { return {}; }` + + s, err := Parse(content) + if err != nil { + t.Fatalf("Parse failed: %v", err) + } + if len(s.Meta.Env) != 2 { + t.Fatalf("env count = %d, want 2", len(s.Meta.Env)) + } + if !s.Meta.Env["API_KEY"].Required { + t.Error("env[API_KEY].required = false, want true") + } + if s.Meta.Env["OPTIONAL_VAR"].Required { + t.Error("env[OPTIONAL_VAR].required = true, want false") + } + if s.Meta.Env["API_KEY"].Description != "API key for service" { + t.Errorf("env[API_KEY].description = %q, want %q", s.Meta.Env["API_KEY"].Description, "API key for service") + } +} + +func TestMeta_ValidateEnv_RequiredMissing(t *testing.T) { + m := Meta{ + Env: map[string]EnvDef{ + "REQ1": {Required: true, Description: "First required var"}, + "REQ2": {Required: true, Description: "Second required var"}, + }, + } + err := m.ValidateEnv() + if err == nil { + t.Fatal("expected error for missing required env vars") + } + msg := err.Error() + if !strings.Contains(msg, "REQ1") || !strings.Contains(msg, "REQ2") { + t.Errorf("error message should list both missing vars: %s", msg) + } +} + +func TestMeta_ValidateEnv_RequiredPresent(t *testing.T) { + t.Setenv("REQ_VAR", "value") + m := Meta{ + Env: map[string]EnvDef{ + "REQ_VAR": {Required: true, Description: "A required var"}, + }, + } + if err := m.ValidateEnv(); err != nil { + t.Fatalf("unexpected error: %v", err) + } +} + +func TestMeta_ValidateEnv_OptionalMissing(t *testing.T) { + m := Meta{ + Env: map[string]EnvDef{ + "OPT_VAR": {Required: false, Description: "Optional"}, + }, + } + if err := m.ValidateEnv(); err != nil { + t.Fatalf("optional missing var should not error: %v", err) + } +} + +func TestMeta_ResolveHeaders_AllSet(t *testing.T) { + t.Setenv("API_KEY", "secret123") + t.Setenv("USER_ID", "42") + m := Meta{ + Headers: map[string]string{ + "X-API-Key": "${API_KEY}", + "X-User-ID": "Bearer ${USER_ID}", + }, + } + resolved := m.ResolveHeaders() + if len(resolved) != 2 { + t.Fatalf("resolved count = %d, want 2", len(resolved)) + } + if resolved["X-API-Key"] != "secret123" { + t.Errorf("X-API-Key = %q, want %q", resolved["X-API-Key"], "secret123") + } + if resolved["X-User-ID"] != "Bearer 42" { + t.Errorf("X-User-ID = %q, want %q", resolved["X-User-ID"], "Bearer 42") + } +} + +func TestMeta_ResolveHeaders_SkipMissing(t *testing.T) { + // API_KEY is set, MISSING_VAR is not + t.Setenv("API_KEY", "secret123") + m := Meta{ + Headers: map[string]string{ + "X-API-Key": "${API_KEY}", + "X-Missing": "${MISSING_VAR}", + "X-Partial": "prefix-${MISSING_VAR}-suffix", + }, + } + resolved := m.ResolveHeaders() + if len(resolved) != 1 { + t.Fatalf("resolved count = %d, want 1", len(resolved)) + } + if _, ok := resolved["X-API-Key"]; !ok { + t.Error("X-API-Key should be present") + } + if _, ok := resolved["X-Missing"]; ok { + t.Error("X-Missing should be skipped") + } + if _, ok := resolved["X-Partial"]; ok { + t.Error("X-Partial should be skipped") + } +} + +func TestMeta_ResolveHeaders_EmptyHeaders(t *testing.T) { + m := Meta{Headers: map[string]string{}} + resolved := m.ResolveHeaders() + if len(resolved) != 0 { + t.Fatalf("resolved count = %d, want 0", len(resolved)) + } +} + func TestParse_EmptyArgs(t *testing.T) { content := `/* @meta { diff --git a/script/registry.go b/script/registry.go index 2b4f19a..177acdf 100644 --- a/script/registry.go +++ b/script/registry.go @@ -2,30 +2,32 @@ package script import ( "fmt" + "io/fs" "os" "path/filepath" "sort" ) -// Registry indexes scripts by their meta name. -type Registry struct { - scripts map[string]*Script - dir string - localOverrideDir string // checked first; empty = disabled +// Source describes one script source for the registry. +type Source struct { + FS fs.FS // virtual filesystem (nil → use Path) + Path string // filesystem path (ignored if FS is set) + Type ScriptSource // how to tag scripts from this source } -// NewRegistry scans a directory for .js script files and indexes them by meta name. -func NewRegistry(dir string) (*Registry, error) { - return NewRegistryWithOverride(dir, "") +// Registry indexes scripts by their meta name. +type Registry struct { + scripts map[string]*Script + sources []Source } -// NewRegistryWithOverride is like NewRegistry but also checks localOverrideDir -// before the main cache dir. Scripts found there shadow the cached versions. -func NewRegistryWithOverride(dir, localOverrideDir string) (*Registry, error) { +// NewRegistry creates a Registry from one or more sources. +// Sources are scanned in order; later sources overwrite earlier ones +// with the same script name. +func NewRegistry(sources ...Source) (*Registry, error) { r := &Registry{ - scripts: make(map[string]*Script), - dir: dir, - localOverrideDir: localOverrideDir, + scripts: make(map[string]*Script), + sources: sources, } if err := r.scan(); err != nil { return nil, err @@ -34,26 +36,26 @@ func NewRegistryWithOverride(dir, localOverrideDir string) (*Registry, error) { } func (r *Registry) scan() error { - // Load main cache dir first. - if err := r.scanDir(r.dir, false); err != nil { - return err - } - // Load local override dir second — overwrites any same-named cache entry. - if r.localOverrideDir != "" { - if err := r.scanDir(r.localOverrideDir, true); err != nil { - return err + for _, src := range r.sources { + if src.FS != nil { + if err := r.scanFS(src.FS, ".", src.Type); err != nil { + return err + } + } else if src.Path != "" { + if err := r.scanDir(src.Path, src.Type); err != nil { + return err + } } } return nil } -func (r *Registry) scanDir(dir string, isOverride bool) error { +func (r *Registry) scanDir(dir string, source ScriptSource) error { + if _, err := os.Stat(dir); os.IsNotExist(err) { + return nil // skip silently + } return filepath.Walk(dir, func(path string, info os.FileInfo, err error) error { if err != nil { - // Ignore missing override dir — it may not exist yet. - if os.IsNotExist(err) && isOverride { - return filepath.SkipDir - } return err } if info.IsDir() || filepath.Ext(path) != ".js" { @@ -72,7 +74,37 @@ func (r *Registry) scanDir(dir string, isOverride bool) error { } s.Path = path - s.LocalOverride = isOverride + s.Source = source + r.scripts[s.Meta.Name] = s + return nil + }) +} + +func (r *Registry) scanFS(fsys fs.FS, root string, source ScriptSource) error { + return fs.WalkDir(fsys, root, func(path string, d fs.DirEntry, err error) error { + if err != nil { + if path == "." { + return nil // missing root in FS — skip silently + } + return err + } + if d.IsDir() || filepath.Ext(path) != ".js" { + return nil + } + + content, err := fs.ReadFile(fsys, path) + if err != nil { + return fmt.Errorf("read %s: %w", path, err) + } + + s, err := Parse(string(content)) + if err != nil { + // Skip scripts that fail to parse + return nil + } + + s.Path = path + s.Source = source r.scripts[s.Meta.Name] = s return nil }) @@ -96,11 +128,11 @@ func (r *Registry) List() []*Script { return scripts } -// ListLocalOnly returns only scripts loaded from the local override directory. -func (r *Registry) ListLocalOnly() []*Script { +// ListOverrides returns only scripts loaded from the local override directory. +func (r *Registry) ListOverrides() []*Script { scripts := make([]*Script, 0) for _, s := range r.scripts { - if s.LocalOverride { + if s.Source == ScriptSourceOverride { scripts = append(scripts, s) } } diff --git a/script/registry_test.go b/script/registry_test.go index d10a88a..9731907 100644 --- a/script/registry_test.go +++ b/script/registry_test.go @@ -13,7 +13,7 @@ func TestNewRegistry(t *testing.T) { t.Skip("sites/ directory not found") } - reg, err := NewRegistry(dir) + reg, err := NewRegistry(Source{Path: dir, Type: ScriptSourceCache}) if err != nil { t.Fatalf("NewRegistry failed: %v", err) } @@ -23,13 +23,39 @@ func TestNewRegistry(t *testing.T) { t.Fatal("expected scripts, got none") } - // Verify v2ex/hot exists - s, ok := reg.Get("v2ex/hot") + // Verify exa/search exists + s, ok := reg.Get("exa/search") if !ok { - t.Fatal("v2ex/hot not found in registry") + t.Fatal("exa/search not found in registry") } - if s.Meta.Domain != "www.v2ex.com" { - t.Errorf("domain = %q, want %q", s.Meta.Domain, "www.v2ex.com") + if s.Meta.Domain != "mcp.exa.ai" { + t.Errorf("domain = %q, want %q", s.Meta.Domain, "mcp.exa.ai") + } + if s.Meta.Runtime != "http" { + t.Errorf("runtime = %q, want %q", s.Meta.Runtime, "http") + } + if len(s.Meta.Env) != 1 { + t.Errorf("env count = %d, want 1", len(s.Meta.Env)) + } else { + def, ok := s.Meta.Env["EXA_API_KEY"] + if !ok { + t.Error("env missing EXA_API_KEY") + } else { + if def.Required { + t.Error("env[EXA_API_KEY].required = true, want false") + } + if def.Description != "API key for Exa search (increases rate limit)" { + t.Errorf("env[EXA_API_KEY].description = %q, want %q", def.Description, "API key for Exa search (increases rate limit)") + } + } + } + if len(s.Meta.Headers) != 1 { + t.Errorf("headers count = %d, want 1", len(s.Meta.Headers)) + } else if s.Meta.Headers["X-API-Key"] != "${EXA_API_KEY}" { + t.Errorf("headers[X-API-Key] = %q, want %q", s.Meta.Headers["X-API-Key"], "${EXA_API_KEY}") + } + if s.Source != ScriptSourceCache { + t.Errorf("source = %d, want ScriptSourceCache (%d)", s.Source, ScriptSourceCache) } } @@ -39,7 +65,7 @@ func TestRegistry_Get_NotFound(t *testing.T) { t.Skip("sites/ directory not found") } - reg, err := NewRegistry(dir) + reg, err := NewRegistry(Source{Path: dir, Type: ScriptSourceCache}) if err != nil { t.Fatalf("NewRegistry failed: %v", err) } @@ -56,7 +82,7 @@ func TestRegistry_ListSorted(t *testing.T) { t.Skip("sites/ directory not found") } - reg, err := NewRegistry(dir) + reg, err := NewRegistry(Source{Path: dir, Type: ScriptSourceCache}) if err != nil { t.Fatalf("NewRegistry failed: %v", err) } diff --git a/sites/embed.go b/sites/embed.go new file mode 100644 index 0000000..275534c --- /dev/null +++ b/sites/embed.go @@ -0,0 +1,6 @@ +package sites + +import "embed" + +//go:embed all:* all:*/* +var FS embed.FS diff --git a/sites/exa/search.js b/sites/exa/search.js new file mode 100644 index 0000000..84d8fdf --- /dev/null +++ b/sites/exa/search.js @@ -0,0 +1,112 @@ +/* @meta +{ + "name": "exa/search", + "description": "Exa web search via MCP endpoint (title, url, text)", + "domain": "mcp.exa.ai", + "args": { + "query": {"required": true, "description": "Search query"}, + "count": {"required": false, "description": "Number of results (default 10)"} + }, + "runtime": "http", + "env": { + "EXA_API_KEY": {"required": false, "description": "API key for Exa search (increases rate limit)"} + }, + "headers": { + "X-API-Key": "${EXA_API_KEY}" + }, + "capabilities": ["network"], + "readOnly": true, + "example": "tap site exa/search \"vaayne\"" +} +*/ + +async function(args) { + if (!args.query) return {error: 'Missing argument: query', hint: 'Provide a search query string'}; + const numResults = args.count || 10; + + const resp = await fetch('https://mcp.exa.ai/mcp', { + method: 'POST', + headers: { + 'accept': 'application/json, text/event-stream', + 'content-type': 'application/json' + }, + body: JSON.stringify({ + jsonrpc: '2.0', + id: 1, + method: 'tools/call', + params: { + name: 'web_search_exa', + arguments: {query: args.query, type: 'auto', numResults, livecrawl: 'fallback'} + } + }) + }); + + if (!resp.ok) return {error: 'HTTP ' + resp.status}; + + const responseText = await resp.text(); + + function extractContentText(payload) { + let parsed; + try { parsed = JSON.parse(payload); } catch { return null; } + const content = parsed?.result?.content; + if (!Array.isArray(content)) return null; + const text = content.map(item => (item.text || '').trim()).filter(Boolean).join('\n\n'); + return text || null; + } + + function parseTextChunk(raw) { + const items = []; + // Results are separated by \n---\n + for (const chunk of raw.split(/\n---\n/)) { + const lines = chunk.split('\n'); + let title = '', url = '', fullText = '', contentStartIndex = -1; + lines.forEach((line, index) => { + if (line.startsWith('Title:')) { + title = line.replace(/^Title:\s*/, ''); + } else if (line.startsWith('URL:')) { + url = line.replace(/^URL:\s*/, ''); + } else if ((line.startsWith('Text:') || line.startsWith('Highlights:')) && contentStartIndex === -1) { + contentStartIndex = index; + fullText = line.replace(/^(?:Text|Highlights):\s*/, ''); + } + }); + if (contentStartIndex !== -1) { + const rest = lines.slice(contentStartIndex + 1).join('\n'); + if (rest.trim()) fullText = fullText ? `${fullText}\n${rest}` : rest; + } + if (title || url || fullText) items.push({title, url, text: fullText}); + } + return items; + } + + const payloadTexts = []; + + for (const line of responseText.split('\n')) { + if (!line.startsWith('data: ')) continue; + const payload = line.slice(6).trim(); + if (!payload || payload === '[DONE]') continue; + const text = extractContentText(payload); + if (text) payloadTexts.push(text); + } + + if (payloadTexts.length === 0) { + const text = extractContentText(responseText); + if (text) payloadTexts.push(text); + } + + if (payloadTexts.length === 0 && responseText.includes('Title:')) { + payloadTexts.push(responseText); + } + + if (payloadTexts.length === 0) return {error: 'No parseable content in response'}; + + const raw = payloadTexts.join('\n\n'); + const parsed = parseTextChunk(raw).filter(r => r.title || r.url || r.text); + const results = parsed.slice(0, numResults).map(r => ({ + title: r.title.trim(), + url: r.url.trim(), + content: r.text.trim() + })); + + return {query: args.query, count: results.length, results}; +} diff --git a/tap.go b/tap.go index 8a7509f..750285d 100644 --- a/tap.go +++ b/tap.go @@ -28,6 +28,7 @@ import ( "github.com/vaayne/tap/engine" "github.com/vaayne/tap/fetch" "github.com/vaayne/tap/script" + "github.com/vaayne/tap/sites" "github.com/vaayne/tap/transport" ) @@ -51,7 +52,7 @@ func New(ctx context.Context, optFns ...Option) (*Client, error) { var reg *script.Registry if opts.sitesDir != "" { var err error - reg, err = script.NewRegistryWithOverride(opts.sitesDir, opts.localOverrideDir) + reg, err = DefaultRegistry(opts.sitesDir, opts.localOverrideDir) if err != nil { return nil, fmt.Errorf("load scripts: %w", err) } @@ -94,6 +95,16 @@ func New(ctx context.Context, optFns ...Option) (*Client, error) { }, nil } +// DefaultRegistry creates the standard tap registry with cache, built-in, +// and override sources in the correct priority order. +func DefaultRegistry(cacheDir, overrideDir string) (*script.Registry, error) { + return script.NewRegistry( + script.Source{Path: cacheDir, Type: script.ScriptSourceCache}, + script.Source{FS: sites.FS, Type: script.ScriptSourceBuiltin}, + script.Source{Path: overrideDir, Type: script.ScriptSourceOverride}, + ) +} + // Close releases all resources. func (c *Client) Close() error { if c.fetcher != nil { @@ -120,7 +131,7 @@ func (c *Client) RunScript(ctx context.Context, name string, args map[string]str return nil, &ScriptNotFoundError{Name: name, Available: c.scriptNames()} } - if s.LocalOverride { + if s.Source == script.ScriptSourceOverride { fmt.Fprintf(os.Stderr, "Using local script: %s\n", name) } if args == nil { @@ -136,13 +147,48 @@ func (c *Client) RunScript(ctx context.Context, name string, args map[string]str } } + if err := s.Meta.ValidateEnv(); err != nil { + return nil, err + } + + engines := c.enginesByRuntime(s.Meta.Runtime) + if len(engines) == 0 { + return nil, fmt.Errorf("no engines available for runtime: %q", s.Meta.Runtime) + } + if c.opts.timeout > 0 { var cancel context.CancelFunc ctx, cancel = context.WithTimeout(ctx, c.opts.timeout) defer cancel() } - return engine.RunScript(ctx, c.engines, s, args) + return engine.RunScript(ctx, engines, s, args, engine.RunOpts{Headers: s.Meta.ResolveHeaders()}) +} + +func (c *Client) enginesByRuntime(runtime string) []engine.Engine { + if c.opts.forceBrowser { + return c.engines + } + switch runtime { + case "http": + var httpEngines []engine.Engine + for _, e := range c.engines { + if e.Name() == "QuickJS" { + httpEngines = append(httpEngines, e) + } + } + return httpEngines + case "browser", "lightpanda": + var browserEngines []engine.Engine + for _, e := range c.engines { + if e.Name() == "Browser" { + browserEngines = append(browserEngines, e) + } + } + return browserEngines + default: // "auto", "", or unknown + return c.engines + } } // Fetch retrieves a URL and extracts clean content using go-defuddle. @@ -184,12 +230,12 @@ func (c *Client) ListScripts() []*script.Script { return c.registry.List() } -// ListScriptsLocalOnly returns only scripts loaded from the local override directory. -func (c *Client) ListScriptsLocalOnly() []*script.Script { +// ListScriptsOverrides returns only scripts loaded from the local override directory. +func (c *Client) ListScriptsOverrides() []*script.Script { if c.registry == nil { return nil } - return c.registry.ListLocalOnly() + return c.registry.ListOverrides() } // GetScript returns a script by name. diff --git a/tap_test.go b/tap_test.go index 5f3d41c..90b9171 100644 --- a/tap_test.go +++ b/tap_test.go @@ -4,7 +4,10 @@ import ( "context" "os" "path/filepath" + "slices" "testing" + + "github.com/vaayne/tap/engine" ) func testSitesDir(t *testing.T) string { @@ -86,6 +89,47 @@ func TestRunScript_MissingRequiredArg(t *testing.T) { } } +func TestEnginesByRuntime(t *testing.T) { + quickjs := engine.NewQuickJS(nil) + browser := engine.NewBrowser(nil, nil) + + tests := []struct { + name string + engines []engine.Engine + forceBrowser bool + runtime string + want []string + }{ + {"normal auto", []engine.Engine{quickjs, browser}, false, "auto", []string{"QuickJS", "Browser"}}, + {"normal empty", []engine.Engine{quickjs, browser}, false, "", []string{"QuickJS", "Browser"}}, + {"normal http", []engine.Engine{quickjs, browser}, false, "http", []string{"QuickJS"}}, + {"normal browser", []engine.Engine{quickjs, browser}, false, "browser", []string{"Browser"}}, + {"normal lightpanda", []engine.Engine{quickjs, browser}, false, "lightpanda", []string{"Browser"}}, + {"normal unknown", []engine.Engine{quickjs, browser}, false, "unknown", []string{"QuickJS", "Browser"}}, + {"forceBrowser http", []engine.Engine{browser}, true, "http", []string{"Browser"}}, + {"forceBrowser browser", []engine.Engine{browser}, true, "browser", []string{"Browser"}}, + {"empty http", []engine.Engine{}, false, "http", []string{}}, + {"empty browser", []engine.Engine{}, false, "browser", []string{}}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + c := &Client{ + engines: tt.engines, + opts: options{forceBrowser: tt.forceBrowser}, + } + got := c.enginesByRuntime(tt.runtime) + gotNames := make([]string, len(got)) + for i, e := range got { + gotNames[i] = e.Name() + } + if !slices.Equal(gotNames, tt.want) { + t.Errorf("enginesByRuntime(%q) = %v, want %v", tt.runtime, gotNames, tt.want) + } + }) + } +} + func TestFetch(t *testing.T) { client, err := New(context.Background()) if err != nil { diff --git a/transport/transport.go b/transport/transport.go index a84ac9d..d9937f2 100644 --- a/transport/transport.go +++ b/transport/transport.go @@ -10,6 +10,7 @@ import ( "os" "path/filepath" + "github.com/chromedp/cdproto/network" "github.com/chromedp/cdproto/page" "github.com/chromedp/cdproto/runtime" "github.com/chromedp/chromedp" @@ -169,12 +170,12 @@ func (t *Transport) BrowseHTMLWithPause(ctx context.Context, url string, pauseFn } // BrowseEval navigates to a URL in a browser and evaluates JavaScript. -func (t *Transport) BrowseEval(ctx context.Context, url string, js string) (any, error) { - return t.BrowseEvalWithPause(ctx, url, js, nil) +func (t *Transport) BrowseEval(ctx context.Context, url string, js string, headers map[string]string) (any, error) { + return t.BrowseEvalWithPause(ctx, url, js, nil, headers) } // BrowseEvalWithPause is like BrowseEval but calls pauseFn after navigation. -func (t *Transport) BrowseEvalWithPause(ctx context.Context, url string, js string, pauseFn PauseFunc) (any, error) { +func (t *Transport) BrowseEvalWithPause(ctx context.Context, url string, js string, pauseFn PauseFunc, headers map[string]string) (any, error) { bctx, cancel := t.newBrowserContext(ctx) defer cancel() @@ -189,14 +190,25 @@ func (t *Transport) BrowseEvalWithPause(ctx context.Context, url string, js stri js, ) - if err := chromedp.Run(bctx, + actions := []chromedp.Action{ chromedp.ActionFunc(func(ctx context.Context) error { _, err := page.AddScriptToEvaluateOnNewDocument(preserveNativeFetch).Do(ctx) return err }), + } + if len(headers) > 0 { + nh := make(network.Headers, len(headers)) + for k, v := range headers { + nh[k] = v + } + actions = append(actions, network.SetExtraHTTPHeaders(nh)) + } + actions = append(actions, chromedp.Navigate(url), chromedp.WaitReady("body"), - ); err != nil { + ) + + if err := chromedp.Run(bctx, actions...); err != nil { return nil, fmt.Errorf("browse eval: %w", err) } From c023c4da1e3558ef830d54ca62284b923d80ad1e Mon Sep 17 00:00:00 2001 From: Vaayne Date: Tue, 12 May 2026 10:21:51 +0800 Subject: [PATCH 2/5] =?UTF-8?q?=F0=9F=90=9B=20fix:=20address=20review=20fe?= =?UTF-8?q?edback=20on=20runtime/env/headers=20PR?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Narrow embed pattern to */*.js (exclude markdown files from binary) - Remove lightpanda from runtime routing (needs --lightpanda CLI flag) - Sort ValidateEnv error output for deterministic messages - Fix siteInfoCmd formatting consistency (spacing, no header masking) --- cmd/tap/site.go | 5 +---- script/parser.go | 2 ++ sites/embed.go | 2 +- tap.go | 2 +- tap_test.go | 2 +- 5 files changed, 6 insertions(+), 7 deletions(-) diff --git a/cmd/tap/site.go b/cmd/tap/site.go index fc69ff0..379195b 100644 --- a/cmd/tap/site.go +++ b/cmd/tap/site.go @@ -197,7 +197,7 @@ func siteInfoCmd() *cli.Command { fmt.Printf(" %s %s\n", bold(color, "Domain:"), s.Meta.Domain) if s.Meta.Runtime != "" && s.Meta.Runtime != "auto" { - fmt.Printf(" %s %s\n\n", bold(color, "Runtime:"), s.Meta.Runtime) + fmt.Printf(" %s %s\n", bold(color, "Runtime:"), s.Meta.Runtime) } if s.Meta.Example != "" { @@ -234,9 +234,6 @@ func siteInfoCmd() *cli.Command { sort.Strings(headerKeys) for _, k := range headerKeys { v := s.Meta.Headers[k] - if strings.Contains(v, "${") { - v = "***" - } fmt.Printf(" %-16s %s\n", green(color, k), v, diff --git a/script/parser.go b/script/parser.go index a43e1a5..6b629b6 100644 --- a/script/parser.go +++ b/script/parser.go @@ -5,6 +5,7 @@ import ( "encoding/json" "fmt" "os" + "sort" "strings" ) @@ -110,6 +111,7 @@ func (m *Meta) ValidateEnv() error { } } } + sort.Strings(missing) if len(missing) > 0 { return fmt.Errorf("missing required environment variables: %s", strings.Join(missing, ", ")) } diff --git a/sites/embed.go b/sites/embed.go index 275534c..c4c36cf 100644 --- a/sites/embed.go +++ b/sites/embed.go @@ -2,5 +2,5 @@ package sites import "embed" -//go:embed all:* all:*/* +//go:embed */*.js var FS embed.FS diff --git a/tap.go b/tap.go index 750285d..cdb2173 100644 --- a/tap.go +++ b/tap.go @@ -178,7 +178,7 @@ func (c *Client) enginesByRuntime(runtime string) []engine.Engine { } } return httpEngines - case "browser", "lightpanda": + case "browser": var browserEngines []engine.Engine for _, e := range c.engines { if e.Name() == "Browser" { diff --git a/tap_test.go b/tap_test.go index 90b9171..4071429 100644 --- a/tap_test.go +++ b/tap_test.go @@ -104,7 +104,7 @@ func TestEnginesByRuntime(t *testing.T) { {"normal empty", []engine.Engine{quickjs, browser}, false, "", []string{"QuickJS", "Browser"}}, {"normal http", []engine.Engine{quickjs, browser}, false, "http", []string{"QuickJS"}}, {"normal browser", []engine.Engine{quickjs, browser}, false, "browser", []string{"Browser"}}, - {"normal lightpanda", []engine.Engine{quickjs, browser}, false, "lightpanda", []string{"Browser"}}, + {"normal lightpanda falls through", []engine.Engine{quickjs, browser}, false, "lightpanda", []string{"QuickJS", "Browser"}}, {"normal unknown", []engine.Engine{quickjs, browser}, false, "unknown", []string{"QuickJS", "Browser"}}, {"forceBrowser http", []engine.Engine{browser}, true, "http", []string{"Browser"}}, {"forceBrowser browser", []engine.Engine{browser}, true, "browser", []string{"Browser"}}, From a4946516c0101d44bb2b2b01345786e02446773d Mon Sep 17 00:00:00 2001 From: Vaayne Date: Tue, 12 May 2026 10:25:51 +0800 Subject: [PATCH 3/5] =?UTF-8?q?=E2=9C=A8=20feat:=20sync=20embedded=20sites?= =?UTF-8?q?=20scripts=20to=20D1=20alongside=20bb-sites?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Sync script now accepts multiple directories; later dirs override earlier ones by script name, matching the CLI registry priority. --- .github/scripts/sync-bb-sites.mjs | 41 ++++++++++++++++++++----------- .github/workflows/sync-sites.yml | 6 ++--- 2 files changed, 30 insertions(+), 17 deletions(-) diff --git a/.github/scripts/sync-bb-sites.mjs b/.github/scripts/sync-bb-sites.mjs index 839e89b..6dbc2c0 100644 --- a/.github/scripts/sync-bb-sites.mjs +++ b/.github/scripts/sync-bb-sites.mjs @@ -1,8 +1,10 @@ #!/usr/bin/env node /** - * Parse bb-sites scripts and POST them to the tap web API batch endpoint. + * Parse site scripts and POST them to the tap web API batch endpoint. + * Accepts one or more directories; later directories override earlier ones + * when script names collide (matching the CLI registry priority). * - * Usage: node sync-bb-sites.mjs + * Usage: node sync-bb-sites.mjs [ ...] * * Env: * TAP_SCRIPTS_SECRET - shared secret for X-Tap-Secret header @@ -10,12 +12,12 @@ */ import { readdir, readFile } from "node:fs/promises" -import { join, basename, dirname } from "node:path" +import { join } from "node:path" import { createHash } from "node:crypto" -const sitesDir = process.argv[2] -if (!sitesDir) { - console.error("Usage: node sync-bb-sites.mjs ") +const sitesDirs = process.argv.slice(2) +if (sitesDirs.length === 0) { + console.error("Usage: node sync-bb-sites.mjs [ ...]") process.exit(1) } @@ -42,19 +44,23 @@ function parseMeta(content) { } /** - * Discover all .js files under sitesDir organized as site/action.js. + * Discover all .js files under dir organized as site/action.js. */ -async function discoverScripts() { +async function discoverScripts(dir) { const scripts = [] - const entries = await readdir(sitesDir, { withFileTypes: true }) + let entries + try { + entries = await readdir(dir, { withFileTypes: true }) + } catch { + return scripts + } for (const entry of entries) { if (!entry.isDirectory()) continue const site = entry.name - // Skip hidden dirs and common non-script dirs if (site.startsWith(".") || site === "node_modules") continue - const siteDir = join(sitesDir, site) + const siteDir = join(dir, site) const files = await readdir(siteDir) for (const file of files) { @@ -68,7 +74,6 @@ async function discoverScripts() { } const hash = createHash("sha256").update(content).digest("hex") - const action = basename(file, ".js") scripts.push({ name: meta.name, @@ -89,13 +94,21 @@ async function discoverScripts() { } async function main() { - const scripts = await discoverScripts() + const byName = new Map() + for (const dir of sitesDirs) { + const found = await discoverScripts(dir) + for (const s of found) { + byName.set(s.name, s) + } + console.log(`${dir}: ${found.length} scripts`) + } + const scripts = [...byName.values()] if (scripts.length === 0) { console.error("No scripts found") process.exit(1) } - console.log(`Found ${scripts.length} scripts, posting to ${apiUrl}`) + console.log(`Total: ${scripts.length} scripts, posting to ${apiUrl}`) const resp = await fetch(apiUrl, { method: "POST", diff --git a/.github/workflows/sync-sites.yml b/.github/workflows/sync-sites.yml index 95d8422..68399cf 100644 --- a/.github/workflows/sync-sites.yml +++ b/.github/workflows/sync-sites.yml @@ -17,15 +17,15 @@ jobs: - uses: actions/checkout@v4 with: - repository: vaayne/bb-sites + repository: epiral/bb-sites path: bb-sites - uses: actions/setup-node@v4 with: - node-version: "22" + node-version: "24" - name: Build payload and sync env: TAP_SCRIPTS_SECRET: ${{ secrets.TAP_SCRIPTS_SECRET }} TAP_API_URL: https://tap.vaayne.com/api/batch - run: node .github/scripts/sync-bb-sites.mjs bb-sites + run: node .github/scripts/sync-bb-sites.mjs bb-sites sites From 254826584eda5baecf7540eecbda27f3bb165fee Mon Sep 17 00:00:00 2001 From: Vaayne Date: Tue, 12 May 2026 11:17:42 +0800 Subject: [PATCH 4/5] =?UTF-8?q?=F0=9F=90=9B=20fix:=20address=20PR=20review?= =?UTF-8?q?=20feedback=20on=20runtime/engine/header=20handling?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Add lightpanda as explicit browser runtime instead of falling through - forceBrowser now returns only browser engines, not the full list - Mask env-interpolated header values in site info output - Fix const alignment in ScriptSource block --- cmd/tap/site.go | 3 +++ script/parser.go | 4 ++-- tap.go | 28 ++++++++++++++++------------ tap_test.go | 7 ++++--- 4 files changed, 25 insertions(+), 17 deletions(-) diff --git a/cmd/tap/site.go b/cmd/tap/site.go index 379195b..fb78ac7 100644 --- a/cmd/tap/site.go +++ b/cmd/tap/site.go @@ -234,6 +234,9 @@ func siteInfoCmd() *cli.Command { sort.Strings(headerKeys) for _, k := range headerKeys { v := s.Meta.Headers[k] + if strings.Contains(v, "${") { + v = dim(color, "(from env)") + } fmt.Printf(" %-16s %s\n", green(color, k), v, diff --git a/script/parser.go b/script/parser.go index 6b629b6..9d81a09 100644 --- a/script/parser.go +++ b/script/parser.go @@ -26,8 +26,8 @@ type ScriptSource int const ( ScriptSourceCache ScriptSource = iota // ~/.cache/tap/sites/ - ScriptSourceBuiltin // sites/ (embedded) - ScriptSourceOverride // ~/.config/tap/sites/ + ScriptSourceBuiltin // sites/ (embedded) + ScriptSourceOverride // ~/.config/tap/sites/ ) // Meta holds the metadata extracted from a script's @meta block. diff --git a/tap.go b/tap.go index cdb2173..f739235 100644 --- a/tap.go +++ b/tap.go @@ -167,30 +167,34 @@ func (c *Client) RunScript(ctx context.Context, name string, args map[string]str func (c *Client) enginesByRuntime(runtime string) []engine.Engine { if c.opts.forceBrowser { - return c.engines + return c.browserEngines() } switch runtime { case "http": - var httpEngines []engine.Engine + var out []engine.Engine for _, e := range c.engines { if e.Name() == "QuickJS" { - httpEngines = append(httpEngines, e) + out = append(out, e) } } - return httpEngines - case "browser": - var browserEngines []engine.Engine - for _, e := range c.engines { - if e.Name() == "Browser" { - browserEngines = append(browserEngines, e) - } - } - return browserEngines + return out + case "browser", "lightpanda": + return c.browserEngines() default: // "auto", "", or unknown return c.engines } } +func (c *Client) browserEngines() []engine.Engine { + var out []engine.Engine + for _, e := range c.engines { + if e.Name() == "Browser" { + out = append(out, e) + } + } + return out +} + // Fetch retrieves a URL and extracts clean content using go-defuddle. func (c *Client) Fetch(ctx context.Context, url string, opts *fetch.Options) (*fetch.Result, error) { if opts == nil { diff --git a/tap_test.go b/tap_test.go index 4071429..55fbb33 100644 --- a/tap_test.go +++ b/tap_test.go @@ -104,10 +104,11 @@ func TestEnginesByRuntime(t *testing.T) { {"normal empty", []engine.Engine{quickjs, browser}, false, "", []string{"QuickJS", "Browser"}}, {"normal http", []engine.Engine{quickjs, browser}, false, "http", []string{"QuickJS"}}, {"normal browser", []engine.Engine{quickjs, browser}, false, "browser", []string{"Browser"}}, - {"normal lightpanda falls through", []engine.Engine{quickjs, browser}, false, "lightpanda", []string{"QuickJS", "Browser"}}, + {"normal lightpanda", []engine.Engine{quickjs, browser}, false, "lightpanda", []string{"Browser"}}, {"normal unknown", []engine.Engine{quickjs, browser}, false, "unknown", []string{"QuickJS", "Browser"}}, - {"forceBrowser http", []engine.Engine{browser}, true, "http", []string{"Browser"}}, - {"forceBrowser browser", []engine.Engine{browser}, true, "browser", []string{"Browser"}}, + {"forceBrowser http", []engine.Engine{quickjs, browser}, true, "http", []string{"Browser"}}, + {"forceBrowser browser", []engine.Engine{quickjs, browser}, true, "browser", []string{"Browser"}}, + {"forceBrowser auto", []engine.Engine{quickjs, browser}, true, "auto", []string{"Browser"}}, {"empty http", []engine.Engine{}, false, "http", []string{}}, {"empty browser", []engine.Engine{}, false, "browser", []string{}}, } From 832d2cc606c61ed1643031c5cff1cae7b844f6f1 Mon Sep 17 00:00:00 2001 From: Vaayne Date: Tue, 12 May 2026 12:07:52 +0800 Subject: [PATCH 5/5] =?UTF-8?q?=E2=9C=A8=20feat:=20add=20twitter=20site=20?= =?UTF-8?q?scripts=20and=20document=20meta=20field=20conventions?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Add twitter/{getxapi-tweet-detail,getxapi-article,post-tweet} scripts - Fix auth bug: remove redundant authorization header from fetch() calls; meta headers are injected automatically by the engine - Add field-level godoc to Meta struct in script/parser.go - Add sites/CLAUDE.md explaining script structure and key rules --- script/parser.go | 42 ++++++++++++++++++++------- sites/AGENTS.md | 26 +++++++++++++++++ sites/CLAUDE.md | 1 + sites/exa/search.js | 2 -- sites/twitter/getxapi-article.js | 33 +++++++++++++++++++++ sites/twitter/getxapi-tweet-detail.js | 34 ++++++++++++++++++++++ sites/twitter/post-tweet.js | 37 +++++++++++++++++++++++ 7 files changed, 162 insertions(+), 13 deletions(-) create mode 100644 sites/AGENTS.md create mode 120000 sites/CLAUDE.md create mode 100644 sites/twitter/getxapi-article.js create mode 100644 sites/twitter/getxapi-tweet-detail.js create mode 100644 sites/twitter/post-tweet.js diff --git a/script/parser.go b/script/parser.go index 9d81a09..8a80b63 100644 --- a/script/parser.go +++ b/script/parser.go @@ -32,17 +32,37 @@ const ( // Meta holds the metadata extracted from a script's @meta block. type Meta struct { - Name string `json:"name"` - Description string `json:"description"` - Domain string `json:"domain"` - Args map[string]ArgDef `json:"args"` - ReadOnly bool `json:"readOnly"` - Example string `json:"example"` - Capabilities []string `json:"capabilities"` - Runtime string `json:"runtime"` - AuthRequired bool `json:"authRequired"` - Headers map[string]string `json:"headers"` - Env map[string]EnvDef `json:"env"` + // Name is the script identifier in "site/action" form and must match the file path. + Name string `json:"name"` + // Description is a short human-readable summary shown in `tap site list`. + Description string `json:"description"` + // Domain is the primary API domain, used for display only. + Domain string `json:"domain"` + // Args declares the named arguments the script accepts. Each key maps to an + // ArgDef describing whether it is required and what it represents. + Args map[string]ArgDef `json:"args"` + // ReadOnly marks scripts that only read data and never mutate state. + ReadOnly bool `json:"readOnly"` + // Example is a sample CLI invocation shown by `tap site info`. + Example string `json:"example"` + // Capabilities is reserved for future capability declarations. + Capabilities []string `json:"capabilities"` + // Runtime controls which execution engine is used: + // "http" — QuickJS only (fast, no browser); use for plain API calls. + // "browser" — CDP browser only; use when cookies or DOM access is needed. + // "auto" — tries QuickJS first, falls back to browser (default). + // See Client.enginesByRuntime in tap.go. + Runtime string `json:"runtime"` + // AuthRequired indicates the script needs browser-based authentication. + AuthRequired bool `json:"authRequired"` + // Headers are HTTP headers injected into every fetch() call made by the script. + // Values may reference environment variables with ${VAR} syntax; headers whose + // variable is unset are omitted entirely. See ResolveHeaders. + Headers map[string]string `json:"headers"` + // Env declares environment variables the script depends on. Required variables + // are validated before execution. Values are surfaced to the script via Headers + // interpolation — not via args. See ValidateEnv. + Env map[string]EnvDef `json:"env"` } // Script represents a parsed site script with metadata and function body. diff --git a/sites/AGENTS.md b/sites/AGENTS.md new file mode 100644 index 0000000..522f25f --- /dev/null +++ b/sites/AGENTS.md @@ -0,0 +1,26 @@ +# Adding Site Scripts + +Scripts live in `sites//.js` and are embedded into the binary at build time (`embed.go`). + +## Structure + +Copy an existing script as a starting point: +- `exa/search.js` — API key auth, SSE response parsing +- `twitter/getxapi-tweet-detail.js` — simple authenticated GET +- `twitter/post-tweet.js` — authenticated POST, non-read-only + +## Meta fields + +See field-level docs on the `Meta` struct in `script/parser.go`. + +## Key rules + +- `env` + `headers` handle auth — **never re-read env vars inside the function body** (`args.MY_API_KEY` is always undefined). Meta headers are resolved via `tap.go:165` and injected into every `fetch()` at `engine/quickjs.go:131`, before any JS-level headers. +- Return `{error: 'message'}` on failure; any other JSON value is success + +## After adding + +```bash +mise run build # re-embeds sites/**/*.js +tap site / [key=value ...] +``` diff --git a/sites/CLAUDE.md b/sites/CLAUDE.md new file mode 120000 index 0000000..47dc3e3 --- /dev/null +++ b/sites/CLAUDE.md @@ -0,0 +1 @@ +AGENTS.md \ No newline at end of file diff --git a/sites/exa/search.js b/sites/exa/search.js index 84d8fdf..8fb0cf5 100644 --- a/sites/exa/search.js +++ b/sites/exa/search.js @@ -14,8 +14,6 @@ "headers": { "X-API-Key": "${EXA_API_KEY}" }, - "capabilities": ["network"], - "readOnly": true, "example": "tap site exa/search \"vaayne\"" } */ diff --git a/sites/twitter/getxapi-article.js b/sites/twitter/getxapi-article.js new file mode 100644 index 0000000..3078c7f --- /dev/null +++ b/sites/twitter/getxapi-article.js @@ -0,0 +1,33 @@ +/* @meta +{ + "name": "twitter/getxapi-article", + "description": "Fetch X / Twitter article by tweet ID via getxapi.com", + "domain": "api.getxapi.com", + "args": { + "id": {"required": true, "description": "Tweet / article ID"} + }, + "runtime": "http", + "env": { + "GET_X_API_KEY": {"required": true, "description": "API key for getxapi.com"} + }, + "headers": { + "Authorization": "Bearer ${GET_X_API_KEY}" + }, + "example": "tap site twitter/getxapi-article '1905545699552375179'" +} +*/ + +async function(args) { + if (!args.id) return {error: 'Missing argument: id', hint: 'Provide a tweet / article ID'}; + + const resp = await fetch(`https://api.getxapi.com/twitter/tweet/article?id=${encodeURIComponent(args.id)}`, { + headers: {'accept': 'application/json'} + }); + + if (!resp.ok) return {error: 'HTTP ' + resp.status}; + + const body = await resp.json(); + if (body.status !== 'success') return {error: body.msg || 'API error'}; + + return body.article; +} diff --git a/sites/twitter/getxapi-tweet-detail.js b/sites/twitter/getxapi-tweet-detail.js new file mode 100644 index 0000000..166ebbe --- /dev/null +++ b/sites/twitter/getxapi-tweet-detail.js @@ -0,0 +1,34 @@ +/* @meta +{ + "name": "twitter/getxapi-tweet-detail", + "description": "Fetch tweet detail by ID via getxapi.com", + "domain": "api.getxapi.com", + "args": { + "id": {"required": true, "description": "Tweet ID"} + }, + "runtime": "http", + "env": { + "GET_X_API_KEY": {"required": true, "description": "API key for getxapi.com"} + }, + "headers": { + "Authorization": "Bearer ${GET_X_API_KEY}" + }, + "readOnly": true, + "example": "tap site twitter/getxapi-tweet-detail '2019264360682778716'" +} +*/ + +async function(args) { + if (!args.id) return {error: 'Missing argument: id', hint: 'Provide a tweet ID'}; + + const resp = await fetch(`https://api.getxapi.com/twitter/tweet/detail?id=${encodeURIComponent(args.id)}`, { + headers: {'accept': 'application/json'} + }); + + if (!resp.ok) return {error: 'HTTP ' + resp.status}; + + const body = await resp.json(); + if (body.status !== 'success') return {error: body.msg || 'API error'}; + + return body.data; +} diff --git a/sites/twitter/post-tweet.js b/sites/twitter/post-tweet.js new file mode 100644 index 0000000..b13a339 --- /dev/null +++ b/sites/twitter/post-tweet.js @@ -0,0 +1,37 @@ +/* @meta +{ + "name": "twitter/post-tweet", + "description": "Post a tweet via the official X API", + "domain": "api.x.com", + "args": { + "text": {"required": true, "description": "Tweet text"} + }, + "runtime": "http", + "env": { + "X_ACCESS_TOKEN": {"required": true, "description": "User access token for the X API"} + }, + "headers": { + "Authorization": "Bearer ${X_ACCESS_TOKEN}" + }, + "readOnly": false, + "example": "tap site twitter/post-tweet 'Hello from the X API!'" +} +*/ + +async function(args) { + if (!args.text) return {error: 'Missing argument: text', hint: 'Provide tweet text'}; + + const resp = await fetch('https://api.x.com/2/tweets', { + method: 'POST', + headers: { + 'accept': 'application/json', + 'content-type': 'application/json' + }, + body: JSON.stringify({text: args.text}) + }); + + if (!resp.ok) return {error: 'HTTP ' + resp.status}; + + const body = await resp.json(); + return body.data || body; +}