diff --git a/server/internal/database/seo.go b/server/internal/database/seo.go index 6b7882d..e6059c5 100644 --- a/server/internal/database/seo.go +++ b/server/internal/database/seo.go @@ -3,6 +3,7 @@ package database import ( "context" "database/sql" + "regexp" "strings" "unicode/utf8" @@ -169,3 +170,155 @@ func hasFeaturedImage(photoURL string) bool { trimmed := strings.TrimSpace(photoURL) return trimmed != "" && trimmed != "-1" } + +// yoastVariablePattern matches a Yoast template variable: "%%title%%". +var yoastVariablePattern = regexp.MustCompile(`%%[a-zA-Z0-9_]+%%`) + +// yoastGluedSeparatorPattern finds a separator an editor typed with no space +// before the variable that follows it ("Pars pro Toto -%%sitename%%"). +var yoastGluedSeparatorPattern = regexp.MustCompile(`([-|\x{2013}\x{2014}])(%%)`) + +// yoastSeparatorRunPattern collapses separators a dropped variable has left +// sitting next to each other. +var yoastSeparatorRunPattern = regexp.MustCompile(`(?:\s*-\s*){2,}`) + +// HasYoastVariables reports whether a value still carries an unexpanded Yoast +// template variable. +func HasYoastVariables(value string) bool { + return yoastVariablePattern.MatchString(value) +} + +// ExpandYoastTitle turns a Yoast SEO title template into finished text. +// +// WordPress stored an article's SEO title as a template rather than as a title +// -- "%%title%% %%page%%", or a headline with "%%page%% %%sep%% %%sitename%%" +// appended -- and Yoast substituted the variables when it rendered the page. +// Nothing substitutes them here, so a template copied into seo_title verbatim +// reaches the public site's , og:title and twitter:title as the literal +// tokens. Mirrors expandSeoVariables in Scalene's src/utils/seoTemplate.ts; the +// two are meant to agree. +// +// %%page%% expands to nothing, as it did under Yoast: it numbers the pages of a +// paginated archive, and an article is one page. A variable this corpus does +// not carry is dropped rather than left visible -- an unrecognised token is +// still a token, and printing it is the defect. +// +// Returns "" when nothing usable survives, which callers store as a blank +// seo_title: the public site then renders the headline, which is what Yoast's +// default template meant in the first place. +func ExpandYoastTitle(template, title, siteTitle, primaryCategory string) string { + if template == "" { + return "" + } + + spaced := yoastGluedSeparatorPattern.ReplaceAllString(template, "$1 $2") + + const sep = "-" + expanded := yoastVariablePattern.ReplaceAllStringFunc(spaced, func(token string) string { + switch strings.ToLower(strings.Trim(token, "%")) { + case "title": + return strings.TrimSpace(title) + case "sitename": + return strings.TrimSpace(siteTitle) + case "primary_category": + return strings.TrimSpace(primaryCategory) + case "sep": + return sep + default: + return "" + } + }) + + return tidyYoastSeparators(expanded, sep) +} + +// tidyYoastSeparators cleans up after a variable that expanded to nothing: the +// punctuation framing it is still there, so a template ending in %%sep%% leaves +// the title hanging on a dash and a dropped %%page%% leaves two separators in a +// row. Returns "" when only punctuation survived. +func tidyYoastSeparators(value, sep string) string { + collapsed := strings.Join(strings.Fields(value), " ") + collapsed = yoastSeparatorRunPattern.ReplaceAllString(collapsed, " "+sep+" ") + collapsed = strings.TrimSpace(strings.TrimPrefix(collapsed, sep)) + collapsed = strings.TrimSpace(strings.TrimSuffix(collapsed, sep)) + if collapsed == sep { + return "" + } + return collapsed +} + +// isRedundantSEOTitle reports whether an expanded title says no more than the +// public site would render on its own: the headline, or the headline with the +// publication name appended, which is the fallback the article page already +// builds (ArticleLayout.astro). +func isRedundantSEOTitle(expanded, title, siteTitle string) bool { + normalize := func(value string) string { + return strings.ToLower(strings.Join(strings.Fields(value), " ")) + } + candidate := normalize(expanded) + return candidate == "" || + candidate == normalize(title) || + candidate == normalize(title+" - "+siteTitle) +} + +// ExpandYoastTitleTemplates rewrites every seo_title still holding a Yoast +// template into the text Yoast would have rendered. +// +// Runs after the Yoast backfill rather than inside it: the backfill only fills +// blanks and records a one-time flag, so on a database seeded before this +// existed the templates are already in place and would never be revisited. +// Idempotent -- expanded text carries no variables, so a second pass matches +// nothing. +func ExpandYoastTitleTemplates(ctx context.Context, conn *sql.DB) (int, error) { + siteTitle, err := GetSiteTitle(ctx, conn) + if err != nil { + return 0, err + } + + rows, err := conn.QueryContext(ctx, ` + SELECT id, COALESCE(title, ''), COALESCE(seo_title, ''), + COALESCE(JSON_VALUE(categories, '$[0]'), '') + FROM articles + WHERE seo_title REGEXP '%%[a-zA-Z0-9_]+%%' + `) + if err != nil { + return 0, err + } + defer rows.Close() + + type rewrite struct { + id int64 + seoTitle string + } + var pending []rewrite + for rows.Next() { + var id int64 + var title, seoTitle, primaryCategory string + if err := rows.Scan(&id, &title, &seoTitle, &primaryCategory); err != nil { + return 0, err + } + expanded := ExpandYoastTitle(seoTitle, title, siteTitle, primaryCategory) + // Yoast's default template is the headline, so expanding it stores a + // custom SEO title that says what the article already says -- and then + // stops tracking the headline when an editor rewrites it. Blank means + // "use the headline", which is both what the template meant and what the + // SEO audit treats as fine (see auditArticle). + if isRedundantSEOTitle(expanded, title, siteTitle) { + expanded = "" + } + pending = append(pending, rewrite{id: id, seoTitle: expanded}) + } + if err := rows.Err(); err != nil { + return 0, err + } + + for _, row := range pending { + if _, err := conn.ExecContext(ctx, + "UPDATE articles SET seo_title = ? WHERE id = ?", row.seoTitle, row.id, + ); err != nil { + return 0, err + } + } + + return len(pending), nil +} diff --git a/server/internal/database/yoast_title_test.go b/server/internal/database/yoast_title_test.go new file mode 100644 index 0000000..0f677bb --- /dev/null +++ b/server/internal/database/yoast_title_test.go @@ -0,0 +1,131 @@ +package database + +import "testing" + +// Every template below is a real value from articles.seo_title in production, +// paired with its article's headline. +func TestExpandYoastTitleOnProductionTemplates(t *testing.T) { + const siteTitle = "The Triangle" + + cases := []struct { + name string + template string + title string + primaryCategory string + want string + }{ + { + name: "the default template is just the headline", + template: "%%title%% %%page%%", + title: "With You!", + want: "With You!", + }, + { + name: "a dropped page variable does not stall the separator", + template: "Where to Eat on Drexel University's Campus %%page%% %%sep%% %%sitename%%", + title: "Where to eat on Drexel's campus", + want: "Where to Eat on Drexel University's Campus - The Triangle", + }, + { + name: "a trailing page variable leaves the headline alone", + template: "Yung Lean's \"Stardust\" tour omits new tracks in favor of old hits %%page%%", + title: "Yung Lean's Stardust tour", + want: "Yung Lean's \"Stardust\" tour omits new tracks in favor of old hits", + }, + { + name: "a template ending in a separator does not hang on the dash", + template: "%%title%% %%page%% %%sep%%", + title: "Ready to dance? CRJ and Empress Of got you covered", + want: "Ready to dance? CRJ and Empress Of got you covered", + }, + { + name: "an editor-typed separator keeps its space", + template: "Drexel Unveils Stone Installation \"Pars pro Toto\" -%%sitename%%", + title: "Drexel unveils Pars pro Toto", + want: "Drexel Unveils Stone Installation \"Pars pro Toto\" - The Triangle", + }, + { + name: "the primary category is substituted like any other variable", + template: "%%title%% %%page%% %%sep%% %%primary_category%%", + title: "Ten public safety tips and resources for Drexel students", + primaryCategory: "News", + want: "Ten public safety tips and resources for Drexel students - News", + }, + { + name: "an unrecognised variable is dropped rather than printed", + template: "%%currentyear%% Real headline", + title: "Real headline", + want: "Real headline", + }, + { + name: "a template that is nothing but punctuation expands to nothing", + template: "%%page%% %%sep%%", + title: "A headline", + want: "", + }, + { + name: "text without variables is returned unchanged", + template: "A hand-written SEO title", + title: "A headline", + want: "A hand-written SEO title", + }, + } + + for _, testCase := range cases { + t.Run(testCase.name, func(t *testing.T) { + got := ExpandYoastTitle(testCase.template, testCase.title, siteTitle, testCase.primaryCategory) + if got != testCase.want { + t.Errorf("ExpandYoastTitle(%q) = %q, want %q", testCase.template, got, testCase.want) + } + if HasYoastVariables(got) { + t.Errorf("ExpandYoastTitle(%q) left a variable behind: %q", testCase.template, got) + } + }) + } +} + +// Expanding is not the same as rewriting: running the result back through has +// to leave it alone, or a restart would keep rewriting the same rows. +func TestExpandYoastTitleIsIdempotent(t *testing.T) { + once := ExpandYoastTitle("%%title%% %%page%% %%sep%% %%sitename%%", "A headline", "The Triangle", "") + twice := ExpandYoastTitle(once, "A headline", "The Triangle", "") + if once != twice { + t.Errorf("second pass changed the value: %q then %q", once, twice) + } +} + +func TestIsRedundantSEOTitleMatchesWhatTheSiteRendersAnyway(t *testing.T) { + const title = "Meet the faces behind the Drexel Affirmations meme account" + const siteTitle = "The Triangle" + + redundant := []string{ + "", + title, + title + " - " + siteTitle, + " " + title + " ", + } + for _, value := range redundant { + if !isRedundantSEOTitle(value, title, siteTitle) { + t.Errorf("%q says no more than the headline, expected it to be redundant", value) + } + } + + kept := []string{ + "Drexel Affirmations: the meme account explained", + title + " - News", + } + for _, value := range kept { + if isRedundantSEOTitle(value, title, siteTitle) { + t.Errorf("%q is an editor's own title, expected it to be kept", value) + } + } +} + +func TestHasYoastVariables(t *testing.T) { + if !HasYoastVariables("%%title%% %%page%%") { + t.Error("a template should be detected") + } + if HasYoastVariables("A 100%% real headline about 50% off") { + t.Error("percent signs that are not a variable should not match") + } +} diff --git a/server/main.go b/server/main.go index 8ebfbe1..b02f5de 100644 --- a/server/main.go +++ b/server/main.go @@ -203,6 +203,17 @@ func main() { slog.Error("failed to backfill article SEO from Yoast export", "error", err) os.Exit(1) } + // What that backfill copies is a Yoast *template* ("%%title%% %%page%%"), + // which WordPress substituted at render time and nothing substitutes now, so + // the tokens reached the public site's <title> and og:title verbatim. Runs + // on every start rather than behind the backfill's one-time flag: a database + // seeded before this existed already holds the templates. Idempotent. + if expanded, err := database.ExpandYoastTitleTemplates(context.Background(), db); err != nil { + slog.Error("failed to expand Yoast SEO title templates", "error", err) + os.Exit(1) + } else if expanded > 0 { + slog.Info("expanded Yoast SEO title templates", "articles", expanded) + } if err := database.EnsureTaxonomyTable(context.Background(), db); err != nil { slog.Error("failed to create taxonomy table", "error", err) os.Exit(1)