Auto-publish stories the AI moderator is confident about

backend/src/business/email/templates/StoryUserEmailTemplateData.ts

@@ -6,6 +6,7 @@ export interface StoryEmailTemplateData {
   photoThumbnailUrl: string;
   mapImageUrl: string | null;
   mapImageUrlRetina: string | null;
+  isPublished: boolean;
 }
 
 export interface StoryEmailMetadata {

backend/src/business/moderation/AiStoryModerationService.ts

@@ -6,14 +6,20 @@ import {
   buildState,
   combine,
   AiModerationFlag,
+  AiStoryModerationScore,
 } from './moderationRules';
 
 const MODEL = 'typesafe/jev-1.13';
 
-const client = new OpenRouter({ apiKey: process.env.OPENROUTER_SK });
+const client = new OpenRouter({
+  apiKey: process.env.OPENROUTER_SK,
+  timeoutMs: 5000,
+});
 const questions = buildNoulQuestions();
 
-export async function evaluateStory(story: Story): Promise<void> {
+export async function evaluateStory(
+  story: Story
+): Promise<AiStoryModerationScore> {
   const response = await client.alpha.decisions.create({
     decisionsRequest: {
       model: MODEL,
@@ -36,13 +42,15 @@ export async function evaluateStory(story: Story): Promise<void> {
 
   const { approveProbability } = combine(ruleProbabilities);
 
-  await StoryRepository().update(story.id, {
-    aiModerationScore: {
-      model: response.model ?? MODEL,
-      evaluatedAt: new Date().toISOString(),
-      costUsd: response.usage.cost ?? null,
-      approveProbability,
-      ruleProbabilities,
-    },
-  });
+  const score: AiStoryModerationScore = {
+    model: response.model ?? MODEL,
+    evaluatedAt: new Date().toISOString(),
+    costUsd: response.usage.cost ?? null,
+    approveProbability,
+    ruleProbabilities,
+  };
+
+  await StoryRepository().update(story.id, { aiModerationScore: score });
+
+  return score;
 }

backend/src/business/stories/StoriesService.ts

@@ -4,13 +4,18 @@ import User from '../../entities/User';
 import StoryState from '../../enum/StoryState';
 import StoryRepository from '../../repositories/StoryRepository';
 import { evaluateStory } from '../moderation/AiStoryModerationService';
+import { AiModerationFlag, combine } from '../moderation/moderationRules';
 import {
   sendPublishedEmail,
   sendSubmittedAgainEmail,
   sendSubmittedEmail,
   sendUserRemovedEmail,
 } from './StoryUserEmailService';
 
+// Matches the "% Human" flag shown in the admin review screen
+// (frontend/.../ReviewStories/index.tsx).
+const MIN_RECAPTCHA_SCORE_TO_AUTO_PUBLISH = 0.7;
+
 function getStoryOrThrow(
   storyId: Story['id'],
   state: StoryState
@@ -27,26 +32,23 @@ async function onStorySubmitted(storyId: Story['id']): Promise<void> {
 
   const hasSubmittedBefore = story.hasEverSubmitted;
 
+  let ruleProbabilities: Record<AiModerationFlag, number> | null = null;
   try {
-    if (hasSubmittedBefore) {
-      await sendSubmittedAgainEmail(story);
-      return;
-    }
-    await sendSubmittedEmail(story);
+    ({ ruleProbabilities } = await evaluateStory(story));
   } catch (e) {
-    console.error('Error sending story submitted email', e);
+    console.error('Error evaluating story with AI moderation', e);
   }
 
   await StoryRepository().update(story.id, {
     hasEverSubmitted: true,
   });
 
+  let isUserBanned = false;
   try {
-    // If user with this email is banned, reject the story
     const maybeUser = await userRepository.findOneBy({
       email: story.storytellerEmail ?? '',
     });
-    const isUserBanned = maybeUser?.isBanned ?? false;
+    isUserBanned = maybeUser?.isBanned ?? false;
     if (isUserBanned) {
       await StoryRepository().update(story.id, {
         state: StoryState.REJECTED,
@@ -57,10 +59,32 @@ async function onStorySubmitted(storyId: Story['id']): Promise<void> {
     console.error('Error auto-reviewing story', e);
   }
 
+  let isAutoPublished = false;
   try {
-    await evaluateStory(story);
+    if (
+      !isUserBanned &&
+      ruleProbabilities &&
+      story.recaptchaScore >= MIN_RECAPTCHA_SCORE_TO_AUTO_PUBLISH &&
+      combine(ruleProbabilities).approve
+    ) {
+      await StoryRepository().update(story.id, {
+        state: StoryState.PUBLISHED,
+        lastReviewer: 'system',
+      });
+      isAutoPublished = true;
+    }
   } catch (e) {
-    console.error('Error evaluating story with AI moderation', e);
+    console.error('Error auto-publishing story', e);
+  }
+
+  try {
+    if (hasSubmittedBefore) {
+      await sendSubmittedAgainEmail(story, isAutoPublished);
+    } else {
+      await sendSubmittedEmail(story, isAutoPublished);
+    }
+  } catch (e) {
+    console.error('Error sending story submitted email', e);
   }
 }
 
@@ -70,7 +94,7 @@ async function onStoryPublished(storyId: Story['id']): Promise<void> {
   try {
     // If we did not send submitted email before, do it now (will occur during rollout of email feature)
     if (!story.lastEmailMessageId) {
-      await sendSubmittedEmail(story);
+      await sendSubmittedEmail(story, false);
     }
     await sendPublishedEmail(story);
   } catch (e) {
@@ -120,7 +144,7 @@ export async function backfillUserStoryEmails(): Promise<void> {
     }
 
     try {
-      await sendSubmittedEmail(story);
+      await sendSubmittedEmail(story, false);
       await StoryRepository().update(story.id, {
         hasEverSubmitted: true,
       });

backend/src/business/stories/StoryUserEmailService.ts

@@ -70,7 +70,10 @@ function forgeImageThumbnailUrl(photo: Story['photo']): string {
   return `https://photos.1940s.nyc/420-jpg/${photo.identifier}.jpg`;
 }
 
-function forgeStoryTemplateContext(story: Story): StoryEmailTemplateData {
+function forgeStoryTemplateContext(
+  story: Story,
+  isPublished: boolean
+): StoryEmailTemplateData {
   return {
     storytellerName: required(story.storytellerName, 'storytellerName'),
     photoDescription: describePhoto(story.photo),
@@ -79,6 +82,7 @@ function forgeStoryTemplateContext(story: Story): StoryEmailTemplateData {
     photoThumbnailUrl: forgeImageThumbnailUrl(story.photo),
     mapImageUrl: null,
     mapImageUrlRetina: null,
+    isPublished,
   };
 }
 
@@ -90,15 +94,16 @@ function forgeStoryMetadata(story: Story): StoryEmailMetadata {
 
 async function sendStoryUserEmail(
   story: Story,
-  Template: EmailTemplate<StoryEmailTemplateData, StoryEmailMetadata>
+  Template: EmailTemplate<StoryEmailTemplateData, StoryEmailMetadata>,
+  isPublished: boolean
 ): Promise<void> {
   if (!story.photo) {
     throw new Error('Expected photo to be resolved');
   }
 
   const email: TemplatedEmailData = Template.createTemplatedEmail({
     to: required(story.storytellerEmail, 'storytellerEmail'),
-    templateContext: forgeStoryTemplateContext(story),
+    templateContext: forgeStoryTemplateContext(story, isPublished),
     metadata: forgeStoryMetadata(story),
     referenceMessageId: story.lastEmailMessageId ?? undefined,
   });
@@ -109,18 +114,24 @@ async function sendStoryUserEmail(
   await StoryRepository().update(story.id, { lastEmailMessageId: messageId });
 }
 
-export async function sendSubmittedEmail(story: Story): Promise<void> {
-  return sendStoryUserEmail(story, StorySubmittedTemplate);
+export async function sendSubmittedEmail(
+  story: Story,
+  isPublished: boolean
+): Promise<void> {
+  return sendStoryUserEmail(story, StorySubmittedTemplate, isPublished);
 }
 
-export async function sendSubmittedAgainEmail(story: Story): Promise<void> {
-  return sendStoryUserEmail(story, StorySubmittedAgainTemplate);
+export async function sendSubmittedAgainEmail(
+  story: Story,
+  isPublished: boolean
+): Promise<void> {
+  return sendStoryUserEmail(story, StorySubmittedAgainTemplate, isPublished);
 }
 
 export async function sendPublishedEmail(story: Story): Promise<void> {
-  return sendStoryUserEmail(story, StoryPublishedTemplate);
+  return sendStoryUserEmail(story, StoryPublishedTemplate, true);
 }
 
 export async function sendUserRemovedEmail(story: Story): Promise<void> {
-  return sendStoryUserEmail(story, StoryUserRemovedTemplate);
+  return sendStoryUserEmail(story, StoryUserRemovedTemplate, false);
 }