Do not index the operator UI

README.md

@@ -62,7 +62,7 @@ Users, GitHub App credentials, and vault files live in SQLite on disk. Railway
 4. Optional variables: `SYNCIDIAN_BOOTSTRAP_USER`, `SYNCIDIAN_BOOTSTRAP_PASSWORD`. To keep the GitHub App if the volume is missing, also set `SYNCIDIAN_GITHUB_APP_*` (see [Set up the GitHub App](docs/github-app.md)). Set `SYNCIDIAN_DATA_KEY` (32-byte hex) so GitHub App secrets stay encrypted even if `syncidian.db` is copied off the volume. Until public GitHub sign-in, set `SYNCIDIAN_GITHUB_ALLOWED_EMAIL` to the operator GitHub email (hosted: `Shangeeth95@gmail.com`) so only that account can complete GitHub OAuth.
 5. Open the public URL. Create the admin at `/admin`. People can then sign up with email on `/`. Register the GitHub App ([walkthrough](docs/github-app.md)) only if you want GitHub sign-in or backup. `/admin` warns if the data directory is still ephemeral. On the hosted landing, tap **Syncidian** six times to reveal GitHub sign-in. Point the plugin at that URL with that user's token.
 
-Health check: `GET /health`. Agents can also fetch `/robots.txt`, `/auth.md`, `/.well-known/mcp/server-card.json`, and markdown for `GET /` (`Accept: text/markdown`). If a CDN (Cloudflare AI Crawl Control) injects its own `robots.txt`, turn that off so the origin file is what crawlers see.
+Health check: `GET /health`. Agents can also fetch `/robots.txt`, `/auth.md`, `/.well-known/mcp/server-card.json`, and markdown for `GET /` (`Accept: text/markdown`). `/robots.txt` disallows the operator path (`/admin` by default) and operator APIs; that HTML also sends `X-Robots-Tag: noindex` so search engines should not list it. The operator page is not in `/sitemap.xml` and is not linked from `/`. If a CDN (Cloudflare AI Crawl Control) injects its own `robots.txt`, turn that off so the origin file is what crawlers see.
 
 ## 2. Install the Obsidian plugin
 

internal/server/adminhost.go

@@ -30,6 +30,32 @@ func (s *Server) requireAdminHost(w http.ResponseWriter, r *http.Request) bool {
 	return true
 }
 
+const operatorRobotsTag = "noindex, nofollow, noarchive, nosnippet"
+
+func setOperatorNoIndex(w http.ResponseWriter) {
+	w.Header().Set("X-Robots-Tag", operatorRobotsTag)
+	w.Header().Set("Referrer-Policy", "no-referrer")
+}
+
+func (s *Server) shouldNoIndex(r *http.Request) bool {
+	if r == nil {
+		return false
+	}
+	if s.isAdminHost(r) {
+		return true
+	}
+	return s.operatorPage(r.URL.Path)
+}
+
+func (s *Server) noIndexOperator(next http.Handler) http.Handler {
+	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if s.shouldNoIndex(r) {
+			setOperatorNoIndex(w)
+		}
+		next.ServeHTTP(w, r)
+	})
+}
+
 func (s *Server) operatorPage(path string) bool {
 	p := strings.TrimRight(path, "/")
 	if p == "" {

internal/server/adminhost_test.go

@@ -62,6 +62,44 @@ func setupAdminOnHost(t *testing.T, srv *Server, host string) []*http.Cookie {
 	return rec.Result().Cookies()
 }
 
+func TestOperatorPageNoIndex(t *testing.T) {
+	dir := t.TempDir()
+	st, err := store.Open(dir)
+	if err != nil {
+		t.Fatal(err)
+	}
+	t.Cleanup(func() { _ = st.Close() })
+	srv := New(config.Config{Addr: ":0", DataDir: dir, PublicURL: "http://localhost", AdminPath: "/admin"}, st, nil)
+
+	rec := serveHost(srv, http.MethodGet, "/", "localhost", nil, nil)
+	if rec.Code != 200 {
+		t.Fatalf("landing %d", rec.Code)
+	}
+	if tag := rec.Header().Get("X-Robots-Tag"); tag != "" {
+		t.Fatalf("public landing must stay indexable, got X-Robots-Tag %q", tag)
+	}
+	if rec.Header().Get("Link") == "" {
+		t.Fatal("public landing should keep discovery Link")
+	}
+
+	rec = serveHost(srv, http.MethodGet, "/admin", "localhost", nil, nil)
+	if rec.Code != 200 {
+		t.Fatalf("operator %d", rec.Code)
+	}
+	if got := rec.Header().Get("X-Robots-Tag"); got != operatorRobotsTag {
+		t.Fatalf("operator X-Robots-Tag %q", got)
+	}
+	if rec.Header().Get("Referrer-Policy") != "no-referrer" {
+		t.Fatalf("operator Referrer-Policy %q", rec.Header().Get("Referrer-Policy"))
+	}
+	if rec.Header().Get("Link") != "" {
+		t.Fatalf("operator HTML must not advertise discovery Link: %q", rec.Header().Get("Link"))
+	}
+	if !bytes.Contains(rec.Body.Bytes(), []byte(`noindex, nofollow, noarchive, nosnippet`)) {
+		t.Fatal("operator HTML should include a noindex script")
+	}
+}
+
 func TestAdminHostHidesOperatorOnPublic(t *testing.T) {
 	srv, _ := newAdminHostServer(t)
 
@@ -105,6 +143,12 @@ func TestAdminHostServesOperatorAndSetup(t *testing.T) {
 	if rec.Code != 200 || !bytes.Contains(rec.Body.Bytes(), []byte(`isAdminPath`)) {
 		t.Fatalf("admin host / should serve SPA: %d", rec.Code)
 	}
+	if got := rec.Header().Get("X-Robots-Tag"); got != operatorRobotsTag {
+		t.Fatalf("admin host / X-Robots-Tag %q", got)
+	}
+	if rec.Header().Get("Link") != "" {
+		t.Fatalf("admin host / must not advertise discovery Link: %q", rec.Header().Get("Link"))
+	}
 
 	rec = serveHost(srv, http.MethodGet, "/api/v1/setup", "admin.syncidian.com", nil, nil)
 	if rec.Code != 200 {

internal/server/agents.go

@@ -112,6 +112,9 @@ func (s *Server) handleRobots(w http.ResponseWriter, r *http.Request) {
 			b.WriteString("Disallow: /admin\n")
 		}
 	}
+	b.WriteString("Disallow: /api/v1/setup\n")
+	b.WriteString("Disallow: /api/v1/auth/login\n")
+	b.WriteString("Disallow: /api/v1/users\n")
 	b.WriteString("Content-Signal: search=yes, ai-train=yes, ai-input=yes\n\n")
 	for _, bot := range strings.Split(aiBots, ", ") {
 		fmt.Fprintf(&b, "User-agent: %s\nAllow: /\nContent-Signal: search=yes, ai-train=yes, ai-input=yes\n\n", bot)

internal/server/agents_test.go

@@ -42,7 +42,7 @@ func TestAgentDiscovery(t *testing.T) {
 		t.Fatalf("robots.txt %d %s", res.StatusCode, body)
 	}
 	txt := string(body)
-	for _, needle := range []string{"User-agent: *", "User-agent: GPTBot", "Content-Signal:", "Sitemap:", "Agentmap:", "Disallow: /admin"} {
+	for _, needle := range []string{"User-agent: *", "User-agent: GPTBot", "Content-Signal:", "Sitemap:", "Agentmap:", "Disallow: /admin", "Disallow: /api/v1/setup", "Disallow: /api/v1/users"} {
 		if !strings.Contains(txt, needle) {
 			t.Fatalf("robots.txt missing %q:\n%s", needle, txt)
 		}

internal/server/server.go

@@ -149,11 +149,13 @@ func (s *Server) Handler() http.Handler {
 		mux.HandleFunc("GET /assets/obsidian.zip", s.handlePluginZip)
 		mux.Handle("GET /assets/", fileServer)
 		serveHTML := func(w http.ResponseWriter, r *http.Request) {
-			w.Header().Set("Link", s.discoveryLinkHeader(s.publicOrigin(r)))
+			// Operator HTML is unlisted: no discovery Link header, noindex via middleware.
 			http.ServeFileFS(w, r, static, "index.html")
 		}
 		mux.HandleFunc("GET /{$}", func(w http.ResponseWriter, r *http.Request) {
-			w.Header().Set("Link", s.discoveryLinkHeader(s.publicOrigin(r)))
+			if !s.isAdminHost(r) {
+				w.Header().Set("Link", s.discoveryLinkHeader(s.publicOrigin(r)))
+			}
 			w.Header().Set("Vary", "Accept")
 			if wantsMarkdown(r.Header.Get("Accept")) {
 				s.handleLandingMarkdown(w, r)
@@ -188,7 +190,7 @@ func (s *Server) Handler() http.Handler {
 		mux.HandleFunc("GET /.well-known/ai-catalog.json", s.handleAICatalog)
 	}
 
-	return s.adminHostGate(s.cors(mux))
+	return s.noIndexOperator(s.adminHostGate(s.cors(mux)))
 }
 
 func (s *Server) cors(next http.Handler) http.Handler {

internal/web/static/index.html

@@ -12,6 +12,25 @@
   <link rel="preconnect" href="https://fonts.gstatic.com" crossorigin />
   <link href="https://fonts.googleapis.com/css2?family=Plus+Jakarta+Sans:wght@400;500;600;700;800;900&display=swap" rel="stylesheet" />
   <link rel="alternate" type="text/markdown" href="/" />
+  <script>
+    (function () {
+      var p = (location.pathname || "/").replace(/\/+$/, "") || "/";
+      if (p === "/") return;
+      var desc = document.querySelector('meta[name="description"]');
+      if (desc) desc.remove();
+      var robots = document.createElement("meta");
+      robots.name = "robots";
+      robots.content = "noindex, nofollow, noarchive, nosnippet";
+      document.head.appendChild(robots);
+      var ref = document.createElement("meta");
+      ref.name = "referrer";
+      ref.content = "no-referrer";
+      document.head.appendChild(ref);
+      document.querySelectorAll('link[rel="api-catalog"], link[rel="ai-catalog"], link[rel="describedby"], link[rel="alternate"]').forEach(function (el) {
+        el.remove();
+      });
+    })();
+  </script>
   <script>
     (function () {
       var t = localStorage.getItem("syncidian-theme");
@@ -735,6 +754,22 @@ <h1 id="help-title">Setup walkthrough</h1>
   const p = location.pathname.replace(/\/+$/, "") || "/";
   return p !== "/";
 }
+function applyOperatorNoIndex() {
+  if (!isAdminPath()) return;
+  if (document.querySelector('meta[name="robots"][content*="noindex"]')) return;
+  const desc = document.querySelector('meta[name="description"]');
+  if (desc) desc.remove();
+  const robots = document.createElement("meta");
+  robots.name = "robots";
+  robots.content = "noindex, nofollow, noarchive, nosnippet";
+  document.head.appendChild(robots);
+  if (!document.querySelector('meta[name="referrer"]')) {
+    const ref = document.createElement("meta");
+    ref.name = "referrer";
+    ref.content = "no-referrer";
+    document.head.appendChild(ref);
+  }
+}
 function nerdsOn() {
   return localStorage.getItem("syncidian-nerds") === "1";
 }
@@ -1152,6 +1187,7 @@ <h2>iOS / iPadOS</h2>
   }
   const st = await api("/api/v1/setup");
   publicInfo = st;
+  applyOperatorNoIndex();
   initAnalytics(st.ga_id);
   bindLandingAnalytics();
   needsSetup = st.needs_setup;