diff --git a/README.md b/README.md
index 19ef1b8..c69e3b5 100644
--- a/README.md
+++ b/README.md
@@ -62,7 +62,7 @@ Users, GitHub App credentials, and vault files live in SQLite on disk. Railway
4. Optional variables: `SYNCIDIAN_BOOTSTRAP_USER`, `SYNCIDIAN_BOOTSTRAP_PASSWORD`. To keep the GitHub App if the volume is missing, also set `SYNCIDIAN_GITHUB_APP_*` (see [Set up the GitHub App](docs/github-app.md)). Set `SYNCIDIAN_DATA_KEY` (32-byte hex) so GitHub App secrets stay encrypted even if `syncidian.db` is copied off the volume. Until public GitHub sign-in, set `SYNCIDIAN_GITHUB_ALLOWED_EMAIL` to the operator GitHub email (hosted: `Shangeeth95@gmail.com`) so only that account can complete GitHub OAuth.
5. Open the public URL. Create the admin at `/admin`. People can then sign up with email on `/`. Register the GitHub App ([walkthrough](docs/github-app.md)) only if you want GitHub sign-in or backup. `/admin` warns if the data directory is still ephemeral. On the hosted landing, tap **Syncidian** six times to reveal GitHub sign-in. Point the plugin at that URL with that user's token.
-Health check: `GET /health`. Agents can also fetch `/robots.txt`, `/auth.md`, `/.well-known/mcp/server-card.json`, and markdown for `GET /` (`Accept: text/markdown`). If a CDN (Cloudflare AI Crawl Control) injects its own `robots.txt`, turn that off so the origin file is what crawlers see.
+Health check: `GET /health`. Agents can also fetch `/robots.txt`, `/auth.md`, `/.well-known/mcp/server-card.json`, and markdown for `GET /` (`Accept: text/markdown`). `/robots.txt` disallows the operator path (`/admin` by default) and operator APIs; that HTML also sends `X-Robots-Tag: noindex` so search engines should not list it. The operator page is not in `/sitemap.xml` and is not linked from `/`. If a CDN (Cloudflare AI Crawl Control) injects its own `robots.txt`, turn that off so the origin file is what crawlers see.
## 2. Install the Obsidian plugin
diff --git a/internal/server/adminhost.go b/internal/server/adminhost.go
index 44f31f9..f88ca6e 100644
--- a/internal/server/adminhost.go
+++ b/internal/server/adminhost.go
@@ -30,6 +30,32 @@ func (s *Server) requireAdminHost(w http.ResponseWriter, r *http.Request) bool {
return true
}
+const operatorRobotsTag = "noindex, nofollow, noarchive, nosnippet"
+
+func setOperatorNoIndex(w http.ResponseWriter) {
+ w.Header().Set("X-Robots-Tag", operatorRobotsTag)
+ w.Header().Set("Referrer-Policy", "no-referrer")
+}
+
+func (s *Server) shouldNoIndex(r *http.Request) bool {
+ if r == nil {
+ return false
+ }
+ if s.isAdminHost(r) {
+ return true
+ }
+ return s.operatorPage(r.URL.Path)
+}
+
+func (s *Server) noIndexOperator(next http.Handler) http.Handler {
+ return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ if s.shouldNoIndex(r) {
+ setOperatorNoIndex(w)
+ }
+ next.ServeHTTP(w, r)
+ })
+}
+
func (s *Server) operatorPage(path string) bool {
p := strings.TrimRight(path, "/")
if p == "" {
diff --git a/internal/server/adminhost_test.go b/internal/server/adminhost_test.go
index 5cbfefc..12a7a7b 100644
--- a/internal/server/adminhost_test.go
+++ b/internal/server/adminhost_test.go
@@ -62,6 +62,44 @@ func setupAdminOnHost(t *testing.T, srv *Server, host string) []*http.Cookie {
return rec.Result().Cookies()
}
+func TestOperatorPageNoIndex(t *testing.T) {
+ dir := t.TempDir()
+ st, err := store.Open(dir)
+ if err != nil {
+ t.Fatal(err)
+ }
+ t.Cleanup(func() { _ = st.Close() })
+ srv := New(config.Config{Addr: ":0", DataDir: dir, PublicURL: "http://localhost", AdminPath: "/admin"}, st, nil)
+
+ rec := serveHost(srv, http.MethodGet, "/", "localhost", nil, nil)
+ if rec.Code != 200 {
+ t.Fatalf("landing %d", rec.Code)
+ }
+ if tag := rec.Header().Get("X-Robots-Tag"); tag != "" {
+ t.Fatalf("public landing must stay indexable, got X-Robots-Tag %q", tag)
+ }
+ if rec.Header().Get("Link") == "" {
+ t.Fatal("public landing should keep discovery Link")
+ }
+
+ rec = serveHost(srv, http.MethodGet, "/admin", "localhost", nil, nil)
+ if rec.Code != 200 {
+ t.Fatalf("operator %d", rec.Code)
+ }
+ if got := rec.Header().Get("X-Robots-Tag"); got != operatorRobotsTag {
+ t.Fatalf("operator X-Robots-Tag %q", got)
+ }
+ if rec.Header().Get("Referrer-Policy") != "no-referrer" {
+ t.Fatalf("operator Referrer-Policy %q", rec.Header().Get("Referrer-Policy"))
+ }
+ if rec.Header().Get("Link") != "" {
+ t.Fatalf("operator HTML must not advertise discovery Link: %q", rec.Header().Get("Link"))
+ }
+ if !bytes.Contains(rec.Body.Bytes(), []byte(`noindex, nofollow, noarchive, nosnippet`)) {
+ t.Fatal("operator HTML should include a noindex script")
+ }
+}
+
func TestAdminHostHidesOperatorOnPublic(t *testing.T) {
srv, _ := newAdminHostServer(t)
@@ -105,6 +143,12 @@ func TestAdminHostServesOperatorAndSetup(t *testing.T) {
if rec.Code != 200 || !bytes.Contains(rec.Body.Bytes(), []byte(`isAdminPath`)) {
t.Fatalf("admin host / should serve SPA: %d", rec.Code)
}
+ if got := rec.Header().Get("X-Robots-Tag"); got != operatorRobotsTag {
+ t.Fatalf("admin host / X-Robots-Tag %q", got)
+ }
+ if rec.Header().Get("Link") != "" {
+ t.Fatalf("admin host / must not advertise discovery Link: %q", rec.Header().Get("Link"))
+ }
rec = serveHost(srv, http.MethodGet, "/api/v1/setup", "admin.syncidian.com", nil, nil)
if rec.Code != 200 {
diff --git a/internal/server/agents.go b/internal/server/agents.go
index f62a164..acab7cc 100644
--- a/internal/server/agents.go
+++ b/internal/server/agents.go
@@ -112,6 +112,9 @@ func (s *Server) handleRobots(w http.ResponseWriter, r *http.Request) {
b.WriteString("Disallow: /admin\n")
}
}
+ b.WriteString("Disallow: /api/v1/setup\n")
+ b.WriteString("Disallow: /api/v1/auth/login\n")
+ b.WriteString("Disallow: /api/v1/users\n")
b.WriteString("Content-Signal: search=yes, ai-train=yes, ai-input=yes\n\n")
for _, bot := range strings.Split(aiBots, ", ") {
fmt.Fprintf(&b, "User-agent: %s\nAllow: /\nContent-Signal: search=yes, ai-train=yes, ai-input=yes\n\n", bot)
diff --git a/internal/server/agents_test.go b/internal/server/agents_test.go
index 5a01bca..262fd94 100644
--- a/internal/server/agents_test.go
+++ b/internal/server/agents_test.go
@@ -42,7 +42,7 @@ func TestAgentDiscovery(t *testing.T) {
t.Fatalf("robots.txt %d %s", res.StatusCode, body)
}
txt := string(body)
- for _, needle := range []string{"User-agent: *", "User-agent: GPTBot", "Content-Signal:", "Sitemap:", "Agentmap:", "Disallow: /admin"} {
+ for _, needle := range []string{"User-agent: *", "User-agent: GPTBot", "Content-Signal:", "Sitemap:", "Agentmap:", "Disallow: /admin", "Disallow: /api/v1/setup", "Disallow: /api/v1/users"} {
if !strings.Contains(txt, needle) {
t.Fatalf("robots.txt missing %q:\n%s", needle, txt)
}
diff --git a/internal/server/server.go b/internal/server/server.go
index e4f29ae..c18ded3 100644
--- a/internal/server/server.go
+++ b/internal/server/server.go
@@ -149,11 +149,13 @@ func (s *Server) Handler() http.Handler {
mux.HandleFunc("GET /assets/obsidian.zip", s.handlePluginZip)
mux.Handle("GET /assets/", fileServer)
serveHTML := func(w http.ResponseWriter, r *http.Request) {
- w.Header().Set("Link", s.discoveryLinkHeader(s.publicOrigin(r)))
+ // Operator HTML is unlisted: no discovery Link header, noindex via middleware.
http.ServeFileFS(w, r, static, "index.html")
}
mux.HandleFunc("GET /{$}", func(w http.ResponseWriter, r *http.Request) {
- w.Header().Set("Link", s.discoveryLinkHeader(s.publicOrigin(r)))
+ if !s.isAdminHost(r) {
+ w.Header().Set("Link", s.discoveryLinkHeader(s.publicOrigin(r)))
+ }
w.Header().Set("Vary", "Accept")
if wantsMarkdown(r.Header.Get("Accept")) {
s.handleLandingMarkdown(w, r)
@@ -188,7 +190,7 @@ func (s *Server) Handler() http.Handler {
mux.HandleFunc("GET /.well-known/ai-catalog.json", s.handleAICatalog)
}
- return s.adminHostGate(s.cors(mux))
+ return s.noIndexOperator(s.adminHostGate(s.cors(mux)))
}
func (s *Server) cors(next http.Handler) http.Handler {
diff --git a/internal/web/static/index.html b/internal/web/static/index.html
index ccfd5c5..36d9bb8 100644
--- a/internal/web/static/index.html
+++ b/internal/web/static/index.html
@@ -12,6 +12,25 @@
+