Expand the Yoast title templates in seo_title

server/internal/database/seo.go

@@ -3,6 +3,7 @@ package database
 import (
 	"context"
 	"database/sql"
+	"regexp"
 	"strings"
 	"unicode/utf8"
 
@@ -169,3 +170,155 @@ func hasFeaturedImage(photoURL string) bool {
 	trimmed := strings.TrimSpace(photoURL)
 	return trimmed != "" && trimmed != "-1"
 }
+
+// yoastVariablePattern matches a Yoast template variable: "%%title%%".
+var yoastVariablePattern = regexp.MustCompile(`%%[a-zA-Z0-9_]+%%`)
+
+// yoastGluedSeparatorPattern finds a separator an editor typed with no space
+// before the variable that follows it ("Pars pro Toto -%%sitename%%").
+var yoastGluedSeparatorPattern = regexp.MustCompile(`([-|\x{2013}\x{2014}])(%%)`)
+
+// yoastSeparatorRunPattern collapses separators a dropped variable has left
+// sitting next to each other.
+var yoastSeparatorRunPattern = regexp.MustCompile(`(?:\s*-\s*){2,}`)
+
+// HasYoastVariables reports whether a value still carries an unexpanded Yoast
+// template variable.
+func HasYoastVariables(value string) bool {
+	return yoastVariablePattern.MatchString(value)
+}
+
+// ExpandYoastTitle turns a Yoast SEO title template into finished text.
+//
+// WordPress stored an article's SEO title as a template rather than as a title
+// -- "%%title%% %%page%%", or a headline with "%%page%% %%sep%% %%sitename%%"
+// appended -- and Yoast substituted the variables when it rendered the page.
+// Nothing substitutes them here, so a template copied into seo_title verbatim
+// reaches the public site's <title>, og:title and twitter:title as the literal
+// tokens. Mirrors expandSeoVariables in Scalene's src/utils/seoTemplate.ts; the
+// two are meant to agree.
+//
+// %%page%% expands to nothing, as it did under Yoast: it numbers the pages of a
+// paginated archive, and an article is one page. A variable this corpus does
+// not carry is dropped rather than left visible -- an unrecognised token is
+// still a token, and printing it is the defect.
+//
+// Returns "" when nothing usable survives, which callers store as a blank
+// seo_title: the public site then renders the headline, which is what Yoast's
+// default template meant in the first place.
+func ExpandYoastTitle(template, title, siteTitle, primaryCategory string) string {
+	if template == "" {
+		return ""
+	}
+
+	spaced := yoastGluedSeparatorPattern.ReplaceAllString(template, "$1 $2")
+
+	const sep = "-"
+	expanded := yoastVariablePattern.ReplaceAllStringFunc(spaced, func(token string) string {
+		switch strings.ToLower(strings.Trim(token, "%")) {
+		case "title":
+			return strings.TrimSpace(title)
+		case "sitename":
+			return strings.TrimSpace(siteTitle)
+		case "primary_category":
+			return strings.TrimSpace(primaryCategory)
+		case "sep":
+			return sep
+		default:
+			return ""
+		}
+	})
+
+	return tidyYoastSeparators(expanded, sep)
+}
+
+// tidyYoastSeparators cleans up after a variable that expanded to nothing: the
+// punctuation framing it is still there, so a template ending in %%sep%% leaves
+// the title hanging on a dash and a dropped %%page%% leaves two separators in a
+// row. Returns "" when only punctuation survived.
+func tidyYoastSeparators(value, sep string) string {
+	collapsed := strings.Join(strings.Fields(value), " ")
+	collapsed = yoastSeparatorRunPattern.ReplaceAllString(collapsed, " "+sep+" ")
+	collapsed = strings.TrimSpace(strings.TrimPrefix(collapsed, sep))
+	collapsed = strings.TrimSpace(strings.TrimSuffix(collapsed, sep))
+	if collapsed == sep {
+		return ""
+	}
+	return collapsed
+}
+
+// isRedundantSEOTitle reports whether an expanded title says no more than the
+// public site would render on its own: the headline, or the headline with the
+// publication name appended, which is the fallback the article page already
+// builds (ArticleLayout.astro).
+func isRedundantSEOTitle(expanded, title, siteTitle string) bool {
+	normalize := func(value string) string {
+		return strings.ToLower(strings.Join(strings.Fields(value), " "))
+	}
+	candidate := normalize(expanded)
+	return candidate == "" ||
+		candidate == normalize(title) ||
+		candidate == normalize(title+" - "+siteTitle)
+}
+
+// ExpandYoastTitleTemplates rewrites every seo_title still holding a Yoast
+// template into the text Yoast would have rendered.
+//
+// Runs after the Yoast backfill rather than inside it: the backfill only fills
+// blanks and records a one-time flag, so on a database seeded before this
+// existed the templates are already in place and would never be revisited.
+// Idempotent -- expanded text carries no variables, so a second pass matches
+// nothing.
+func ExpandYoastTitleTemplates(ctx context.Context, conn *sql.DB) (int, error) {
+	siteTitle, err := GetSiteTitle(ctx, conn)
+	if err != nil {
+		return 0, err
+	}
+
+	rows, err := conn.QueryContext(ctx, `
+		SELECT id, COALESCE(title, ''), COALESCE(seo_title, ''),
+		       COALESCE(JSON_VALUE(categories, '$[0]'), '')
+		FROM articles
+		WHERE seo_title REGEXP '%%[a-zA-Z0-9_]+%%'
+	`)
+	if err != nil {
+		return 0, err
+	}
+	defer rows.Close()
+
+	type rewrite struct {
+		id       int64
+		seoTitle string
+	}
+	var pending []rewrite
+	for rows.Next() {
+		var id int64
+		var title, seoTitle, primaryCategory string
+		if err := rows.Scan(&id, &title, &seoTitle, &primaryCategory); err != nil {
+			return 0, err
+		}
+		expanded := ExpandYoastTitle(seoTitle, title, siteTitle, primaryCategory)
+		// Yoast's default template is the headline, so expanding it stores a
+		// custom SEO title that says what the article already says -- and then
+		// stops tracking the headline when an editor rewrites it. Blank means
+		// "use the headline", which is both what the template meant and what the
+		// SEO audit treats as fine (see auditArticle).
+		if isRedundantSEOTitle(expanded, title, siteTitle) {
+			expanded = ""
+		}
+		pending = append(pending, rewrite{id: id, seoTitle: expanded})
+	}
+	if err := rows.Err(); err != nil {
+		return 0, err
+	}
+
+	for _, row := range pending {
+		if _, err := conn.ExecContext(ctx,
+			"UPDATE articles SET seo_title = ? WHERE id = ?", row.seoTitle, row.id,
+		); err != nil {
+			return 0, err
+		}
+	}
+
+	return len(pending), nil
+}

server/internal/database/yoast_title_test.go

@@ -0,0 +1,131 @@
+package database
+
+import "testing"
+
+// Every template below is a real value from articles.seo_title in production,
+// paired with its article's headline.
+func TestExpandYoastTitleOnProductionTemplates(t *testing.T) {
+	const siteTitle = "The Triangle"
+
+	cases := []struct {
+		name            string
+		template        string
+		title           string
+		primaryCategory string
+		want            string
+	}{
+		{
+			name:     "the default template is just the headline",
+			template: "%%title%% %%page%%",
+			title:    "With You!",
+			want:     "With You!",
+		},
+		{
+			name:     "a dropped page variable does not stall the separator",
+			template: "Where to Eat on Drexel University's Campus %%page%% %%sep%% %%sitename%%",
+			title:    "Where to eat on Drexel's campus",
+			want:     "Where to Eat on Drexel University's Campus - The Triangle",
+		},
+		{
+			name:     "a trailing page variable leaves the headline alone",
+			template: "Yung Lean's \"Stardust\" tour omits new tracks in favor of old hits %%page%%",
+			title:    "Yung Lean's Stardust tour",
+			want:     "Yung Lean's \"Stardust\" tour omits new tracks in favor of old hits",
+		},
+		{
+			name:     "a template ending in a separator does not hang on the dash",
+			template: "%%title%% %%page%% %%sep%%",
+			title:    "Ready to dance? CRJ and Empress Of got you covered",
+			want:     "Ready to dance? CRJ and Empress Of got you covered",
+		},
+		{
+			name:     "an editor-typed separator keeps its space",
+			template: "Drexel Unveils Stone Installation \"Pars pro Toto\" -%%sitename%%",
+			title:    "Drexel unveils Pars pro Toto",
+			want:     "Drexel Unveils Stone Installation \"Pars pro Toto\" - The Triangle",
+		},
+		{
+			name:            "the primary category is substituted like any other variable",
+			template:        "%%title%% %%page%% %%sep%% %%primary_category%%",
+			title:           "Ten public safety tips and resources for Drexel students",
+			primaryCategory: "News",
+			want:            "Ten public safety tips and resources for Drexel students - News",
+		},
+		{
+			name:     "an unrecognised variable is dropped rather than printed",
+			template: "%%currentyear%% Real headline",
+			title:    "Real headline",
+			want:     "Real headline",
+		},
+		{
+			name:     "a template that is nothing but punctuation expands to nothing",
+			template: "%%page%% %%sep%%",
+			title:    "A headline",
+			want:     "",
+		},
+		{
+			name:     "text without variables is returned unchanged",
+			template: "A hand-written SEO title",
+			title:    "A headline",
+			want:     "A hand-written SEO title",
+		},
+	}
+
+	for _, testCase := range cases {
+		t.Run(testCase.name, func(t *testing.T) {
+			got := ExpandYoastTitle(testCase.template, testCase.title, siteTitle, testCase.primaryCategory)
+			if got != testCase.want {
+				t.Errorf("ExpandYoastTitle(%q) = %q, want %q", testCase.template, got, testCase.want)
+			}
+			if HasYoastVariables(got) {
+				t.Errorf("ExpandYoastTitle(%q) left a variable behind: %q", testCase.template, got)
+			}
+		})
+	}
+}
+
+// Expanding is not the same as rewriting: running the result back through has
+// to leave it alone, or a restart would keep rewriting the same rows.
+func TestExpandYoastTitleIsIdempotent(t *testing.T) {
+	once := ExpandYoastTitle("%%title%% %%page%% %%sep%% %%sitename%%", "A headline", "The Triangle", "")
+	twice := ExpandYoastTitle(once, "A headline", "The Triangle", "")
+	if once != twice {
+		t.Errorf("second pass changed the value: %q then %q", once, twice)
+	}
+}
+
+func TestIsRedundantSEOTitleMatchesWhatTheSiteRendersAnyway(t *testing.T) {
+	const title = "Meet the faces behind the Drexel Affirmations meme account"
+	const siteTitle = "The Triangle"
+
+	redundant := []string{
+		"",
+		title,
+		title + " - " + siteTitle,
+		"  " + title + "   ",
+	}
+	for _, value := range redundant {
+		if !isRedundantSEOTitle(value, title, siteTitle) {
+			t.Errorf("%q says no more than the headline, expected it to be redundant", value)
+		}
+	}
+
+	kept := []string{
+		"Drexel Affirmations: the meme account explained",
+		title + " - News",
+	}
+	for _, value := range kept {
+		if isRedundantSEOTitle(value, title, siteTitle) {
+			t.Errorf("%q is an editor's own title, expected it to be kept", value)
+		}
+	}
+}
+
+func TestHasYoastVariables(t *testing.T) {
+	if !HasYoastVariables("%%title%% %%page%%") {
+		t.Error("a template should be detected")
+	}
+	if HasYoastVariables("A 100%% real headline about 50% off") {
+		t.Error("percent signs that are not a variable should not match")
+	}
+}

server/main.go

@@ -203,6 +203,17 @@ func main() {
 		slog.Error("failed to backfill article SEO from Yoast export", "error", err)
 		os.Exit(1)
 	}
+	// What that backfill copies is a Yoast *template* ("%%title%% %%page%%"),
+	// which WordPress substituted at render time and nothing substitutes now, so
+	// the tokens reached the public site's <title> and og:title verbatim. Runs
+	// on every start rather than behind the backfill's one-time flag: a database
+	// seeded before this existed already holds the templates. Idempotent.
+	if expanded, err := database.ExpandYoastTitleTemplates(context.Background(), db); err != nil {
+		slog.Error("failed to expand Yoast SEO title templates", "error", err)
+		os.Exit(1)
+	} else if expanded > 0 {
+		slog.Info("expanded Yoast SEO title templates", "articles", expanded)
+	}
 	if err := database.EnsureTaxonomyTable(context.Background(), db); err != nil {
 		slog.Error("failed to create taxonomy table", "error", err)
 		os.Exit(1)