From 75c830410dd454c2ce2b4eea5e944ab2b3149645 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Wed, 2 Sep 2026 20:07:36 +0000 Subject: [PATCH 1/2] Ask crawlers not to index the operator UI. /admin was already omitted from the sitemap and disallowed in robots.txt. Add X-Robots-Tag and Referrer-Policy on the operator HTML, a noindex meta on that path, and drop discovery Link headers so other sites and search engines are less likely to list it. Co-authored-by: shangeeth95 --- internal/server/adminhost.go | 26 ++++++++++++++++++ internal/server/adminhost_test.go | 44 +++++++++++++++++++++++++++++++ internal/server/agents.go | 3 +++ internal/server/agents_test.go | 2 +- internal/server/server.go | 8 +++--- internal/web/static/index.html | 36 +++++++++++++++++++++++++ 6 files changed, 115 insertions(+), 4 deletions(-) diff --git a/internal/server/adminhost.go b/internal/server/adminhost.go index 44f31f9..f88ca6e 100644 --- a/internal/server/adminhost.go +++ b/internal/server/adminhost.go @@ -30,6 +30,32 @@ func (s *Server) requireAdminHost(w http.ResponseWriter, r *http.Request) bool { return true } +const operatorRobotsTag = "noindex, nofollow, noarchive, nosnippet" + +func setOperatorNoIndex(w http.ResponseWriter) { + w.Header().Set("X-Robots-Tag", operatorRobotsTag) + w.Header().Set("Referrer-Policy", "no-referrer") +} + +func (s *Server) shouldNoIndex(r *http.Request) bool { + if r == nil { + return false + } + if s.isAdminHost(r) { + return true + } + return s.operatorPage(r.URL.Path) +} + +func (s *Server) noIndexOperator(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if s.shouldNoIndex(r) { + setOperatorNoIndex(w) + } + next.ServeHTTP(w, r) + }) +} + func (s *Server) operatorPage(path string) bool { p := strings.TrimRight(path, "/") if p == "" { diff --git a/internal/server/adminhost_test.go b/internal/server/adminhost_test.go index 5cbfefc..12a7a7b 100644 --- a/internal/server/adminhost_test.go +++ b/internal/server/adminhost_test.go @@ -62,6 +62,44 @@ func setupAdminOnHost(t *testing.T, srv *Server, host string) []*http.Cookie { return rec.Result().Cookies() } +func TestOperatorPageNoIndex(t *testing.T) { + dir := t.TempDir() + st, err := store.Open(dir) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = st.Close() }) + srv := New(config.Config{Addr: ":0", DataDir: dir, PublicURL: "http://localhost", AdminPath: "/admin"}, st, nil) + + rec := serveHost(srv, http.MethodGet, "/", "localhost", nil, nil) + if rec.Code != 200 { + t.Fatalf("landing %d", rec.Code) + } + if tag := rec.Header().Get("X-Robots-Tag"); tag != "" { + t.Fatalf("public landing must stay indexable, got X-Robots-Tag %q", tag) + } + if rec.Header().Get("Link") == "" { + t.Fatal("public landing should keep discovery Link") + } + + rec = serveHost(srv, http.MethodGet, "/admin", "localhost", nil, nil) + if rec.Code != 200 { + t.Fatalf("operator %d", rec.Code) + } + if got := rec.Header().Get("X-Robots-Tag"); got != operatorRobotsTag { + t.Fatalf("operator X-Robots-Tag %q", got) + } + if rec.Header().Get("Referrer-Policy") != "no-referrer" { + t.Fatalf("operator Referrer-Policy %q", rec.Header().Get("Referrer-Policy")) + } + if rec.Header().Get("Link") != "" { + t.Fatalf("operator HTML must not advertise discovery Link: %q", rec.Header().Get("Link")) + } + if !bytes.Contains(rec.Body.Bytes(), []byte(`noindex, nofollow, noarchive, nosnippet`)) { + t.Fatal("operator HTML should include a noindex script") + } +} + func TestAdminHostHidesOperatorOnPublic(t *testing.T) { srv, _ := newAdminHostServer(t) @@ -105,6 +143,12 @@ func TestAdminHostServesOperatorAndSetup(t *testing.T) { if rec.Code != 200 || !bytes.Contains(rec.Body.Bytes(), []byte(`isAdminPath`)) { t.Fatalf("admin host / should serve SPA: %d", rec.Code) } + if got := rec.Header().Get("X-Robots-Tag"); got != operatorRobotsTag { + t.Fatalf("admin host / X-Robots-Tag %q", got) + } + if rec.Header().Get("Link") != "" { + t.Fatalf("admin host / must not advertise discovery Link: %q", rec.Header().Get("Link")) + } rec = serveHost(srv, http.MethodGet, "/api/v1/setup", "admin.syncidian.com", nil, nil) if rec.Code != 200 { diff --git a/internal/server/agents.go b/internal/server/agents.go index f62a164..acab7cc 100644 --- a/internal/server/agents.go +++ b/internal/server/agents.go @@ -112,6 +112,9 @@ func (s *Server) handleRobots(w http.ResponseWriter, r *http.Request) { b.WriteString("Disallow: /admin\n") } } + b.WriteString("Disallow: /api/v1/setup\n") + b.WriteString("Disallow: /api/v1/auth/login\n") + b.WriteString("Disallow: /api/v1/users\n") b.WriteString("Content-Signal: search=yes, ai-train=yes, ai-input=yes\n\n") for _, bot := range strings.Split(aiBots, ", ") { fmt.Fprintf(&b, "User-agent: %s\nAllow: /\nContent-Signal: search=yes, ai-train=yes, ai-input=yes\n\n", bot) diff --git a/internal/server/agents_test.go b/internal/server/agents_test.go index 5a01bca..262fd94 100644 --- a/internal/server/agents_test.go +++ b/internal/server/agents_test.go @@ -42,7 +42,7 @@ func TestAgentDiscovery(t *testing.T) { t.Fatalf("robots.txt %d %s", res.StatusCode, body) } txt := string(body) - for _, needle := range []string{"User-agent: *", "User-agent: GPTBot", "Content-Signal:", "Sitemap:", "Agentmap:", "Disallow: /admin"} { + for _, needle := range []string{"User-agent: *", "User-agent: GPTBot", "Content-Signal:", "Sitemap:", "Agentmap:", "Disallow: /admin", "Disallow: /api/v1/setup", "Disallow: /api/v1/users"} { if !strings.Contains(txt, needle) { t.Fatalf("robots.txt missing %q:\n%s", needle, txt) } diff --git a/internal/server/server.go b/internal/server/server.go index e4f29ae..c18ded3 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -149,11 +149,13 @@ func (s *Server) Handler() http.Handler { mux.HandleFunc("GET /assets/obsidian.zip", s.handlePluginZip) mux.Handle("GET /assets/", fileServer) serveHTML := func(w http.ResponseWriter, r *http.Request) { - w.Header().Set("Link", s.discoveryLinkHeader(s.publicOrigin(r))) + // Operator HTML is unlisted: no discovery Link header, noindex via middleware. http.ServeFileFS(w, r, static, "index.html") } mux.HandleFunc("GET /{$}", func(w http.ResponseWriter, r *http.Request) { - w.Header().Set("Link", s.discoveryLinkHeader(s.publicOrigin(r))) + if !s.isAdminHost(r) { + w.Header().Set("Link", s.discoveryLinkHeader(s.publicOrigin(r))) + } w.Header().Set("Vary", "Accept") if wantsMarkdown(r.Header.Get("Accept")) { s.handleLandingMarkdown(w, r) @@ -188,7 +190,7 @@ func (s *Server) Handler() http.Handler { mux.HandleFunc("GET /.well-known/ai-catalog.json", s.handleAICatalog) } - return s.adminHostGate(s.cors(mux)) + return s.noIndexOperator(s.adminHostGate(s.cors(mux))) } func (s *Server) cors(next http.Handler) http.Handler { diff --git a/internal/web/static/index.html b/internal/web/static/index.html index ccfd5c5..36d9bb8 100644 --- a/internal/web/static/index.html +++ b/internal/web/static/index.html @@ -12,6 +12,25 @@ +