feat(tagged-pdf): retag an already-tagged PDF when the user confirms (#501)

docs/API.md

@@ -1007,6 +1007,9 @@ The answer is `{"filename", "pdf", "report"}`: the file name to save as, the PDF
 tagger's report. `report.warnings` lists what the tagger could not do cleanly, such as a form field
 it could not find in the HTML. The demo page turns each warning into a plain sentence.
 
+A PDF that is already tagged gets `422 already_tagged`. Send `"retag": true` to replace its tags
+with tags from the HTML. The report then has a `retagged` warning.
+
 The session must have finished (`ready_for_review` or `closed`), or you get `409 invalid_state`. The
 PDF is tagged from the first pass's HTML for each page, before review. A page that failed extraction is
 left out, and the report lists it as `page_not_in_html`. Errors from the tagger keep
@@ -4365,9 +4368,10 @@ pages reported blank, which an empty document could not.
 ### `tagged_pdf` / `tagged_pdf_failed`
 
 A [tagged PDF](#get-a-tagged-pdf-optional) was made, or the tagger refused. `ms` is how long it took.
-`tagged_pdf` lists `fields_given`, the names of the fields that were filled in. The values are
-never logged. `tagged_pdf_failed` has the tagger's `code` and `error`. An `internal_error`'s
-message is left out, because it may quote a value.
+`tagged_pdf` lists `fields_given`, the names of the fields that were filled in, and `retag`. The
+values are never logged. `tagged_pdf_failed` has the tagger's `code` and `error`. An `internal_error`'s
+message is left out, because it may quote a value. The demo asks before it retags, so a demo retag
+logs `tagged_pdf_failed` with `already_tagged` first, then `tagged_pdf` with `retag: true`.
 
 ### `form_fields` / `page_fields` / `page_fields_missing`
 

public/demo.html

@@ -538,6 +538,7 @@ <h2 class="visually-hidden">About</h2>
         xfa_removed: () => 'The PDF had two versions of its form. The one most readers cannot use was removed.',
         signature_invalidated: () => 'The PDF was signed. The signature no longer holds, because the file changed.',
         signature_field_skipped: () => 'Signature fields cannot be filled in here, so they were left empty.',
+        retagged: () => 'The PDF\'s old tags were replaced with tags from the HTML.',
       };
       return Object.keys(count).filter((c) => c !== 'no_title').map((c) => (say[c] ? say[c](count[c]) : 'Note from the tagger: ' + c + '.'));
     }
@@ -576,10 +577,24 @@ <h2 class="visually-hidden">About</h2>
       $('pdf-btn').disabled = true;
       live('Tagging your PDF. This can take a minute.');
       try {
-        const res = await api('/sessions/' + sessionId + '/pdf', {
+        const values = fieldValues($('pdf-field-list'));
+        const post = (retag) => api('/sessions/' + sessionId + '/pdf', {
           method: 'POST', headers: { 'content-type': 'application/json' },
-          body: JSON.stringify({ values: fieldValues($('pdf-field-list')) }),
+          body: JSON.stringify({ values, retag }),
         });
+        let res = await post(false);
+        // A PDF that is already tagged is only retagged if the reader says so.
+        if (res.status === 422) {
+          const code = await res.clone().json().then((j) => j.error && j.error.code, () => null);
+          if (code === 'already_tagged') {
+            if (!confirm('This PDF is already tagged. Retagging replaces its existing tags with tags from Iris\'s HTML. Retag it?')) {
+              live('The PDF was not retagged.');
+              return;
+            }
+            live('Retagging your PDF.');
+            res = await post(true);
+          }
+        }
         if (!res.ok) throw new Error(await errMessage(res));
         const out = await res.json();
         const bytes = Uint8Array.from(atob(out.pdf), (c) => c.charCodeAt(0));

src/routes/sessions.ts

@@ -634,6 +634,11 @@ export function sessionsRouter(cfg: IrisConfig, store: Store): Router {
       sendError(res, 400, "invalid_request", "`values` must be an object of field name to value.");
       return;
     }
+    const retag: unknown = (req.body ?? {}).retag ?? false;
+    if (typeof retag !== "boolean") {
+      sendError(res, 400, "invalid_request", "`retag` must be true or false.");
+      return;
+    }
     if (tooBusy(res)) return;
     const log = new RunLog(paths.sessionLog(s.session_id));
     const started = Date.now();
@@ -660,11 +665,12 @@ export function sessionsRouter(cfg: IrisConfig, store: Store): Router {
         pdfPath: src.pdf,
         input,
         values: values as Record<string, unknown>,
+        retag,
         scratchRoot: paths.pdfScratchRoot(),
         timeoutSeconds: tagTimeoutSeconds(cfg),
       });
       // Field names only. A value is never logged.
-      log.event("tagged_pdf", { ms: Date.now() - started, fields_given: Object.keys(values) });
+      log.event("tagged_pdf", { ms: Date.now() - started, fields_given: Object.keys(values), retag });
       res.json({ filename: `${base}_tagged.pdf`, pdf: pdf.toString("base64"), report });
     } catch (e) {
       // iris-pdf's own messages name fields and options, never values. Its

src/util/taggedPdf.ts

@@ -121,7 +121,7 @@ export type TagInput = { lang?: string; title?: string; pages: { sourcePage: num
 // neither them nor the tagged PDF.
 export async function tagPdf(
   command: string,
-  args: { pdfPath: string; input: TagInput; values: Record<string, unknown>; scratchRoot: string; timeoutSeconds: number },
+  args: { pdfPath: string; input: TagInput; values: Record<string, unknown>; retag?: boolean; scratchRoot: string; timeoutSeconds: number },
 ): Promise<{ pdf: Buffer; report: unknown }> {
   mkdirSync(args.scratchRoot, { recursive: true });
   const dir = mkdtempSync(join(args.scratchRoot, "pdf-"));
@@ -130,6 +130,8 @@ export async function tagPdf(
     writeFileSync(f("pages.json"), JSON.stringify(args.input));
     writeFileSync(f("values.json"), JSON.stringify(args.values), { mode: 0o600 });
     const argv = ["tag", "--pdf", args.pdfPath, "--pages", f("pages.json"), "--values", f("values.json"), "--out", f("out.pdf"), "--report", f("report.json")];
+    // An already-tagged PDF is refused (`already_tagged`) unless the user asked to replace its tags.
+    if (args.retag) argv.push("--retag");
     try {
       await execFileP(command, argv, { maxBuffer: MAX_BUFFER, timeout: args.timeoutSeconds * 1000 });
     } catch (e) {

test/demo-pdf-form.test.ts

@@ -128,3 +128,42 @@ test("a late /fields reply is dropped after a new document, a re-run, or a newer
   const check = body.indexOf("if (stale()) return;", last);
   assert.ok(check > last && check < body.indexOf("show('pdf-part')"), "no await between the last check and the change");
 });
+
+// The submit handler, run against stand-ins for the page and the server.
+function submitHandler(replies: { status: number; body: unknown }[], answer: boolean) {
+  const start = demoHtml.indexOf("async (ev) =>", demoHtml.indexOf("$('pdf-form').addEventListener('submit'"));
+  const end = demoHtml.lastIndexOf("});", demoHtml.indexOf("$('restart-btn')"));
+  const sent: unknown[] = [], said: string[] = [], asked: string[] = [], errors: string[] = [];
+  const api = async (_path: string, opts: { body: string }) => {
+    sent.push(JSON.parse(opts.body));
+    const r = replies.shift()!;
+    const res = { ok: r.status < 400, status: r.status, json: async () => r.body, clone: () => res };
+    return res;
+  };
+  const $ = () => ({ disabled: false });
+  const handler = new Function(
+    "$", "sessionId", "api", "confirm", "live", "setError", "fieldValues", "errMessage",
+    `return ${demoHtml.slice(start, end)}};`,
+  )($, "ses_1", api, (q: string) => (asked.push(q), answer), (m: string) => said.push(m), (m: string) => m && errors.push(m), () => ({}), async (res: { json: () => Promise<{ error: { message: string } }> }) => (await res.json()).error.message);
+  return { run: () => handler({ preventDefault() {} }), sent, said, asked, errors };
+}
+
+test("an already-tagged PDF is retagged only after the reader says yes", async () => {
+  const already = { status: 422, body: { error: { code: "already_tagged", message: "The PDF is already tagged." } } };
+  const no = submitHandler([already], false);
+  await no.run();
+  assert.deepEqual(no.sent, [{ values: {}, retag: false }]);
+  assert.equal(no.asked.length, 1);
+  assert.deepEqual(no.errors, []);
+  assert.equal(no.said.at(-1), "The PDF was not retagged.");
+
+  const yes = submitHandler([already, { status: 500, body: { error: { message: "stop here" } } }], true);
+  await yes.run();
+  assert.deepEqual(yes.sent, [{ values: {}, retag: false }, { values: {}, retag: true }]);
+  assert.deepEqual(yes.errors, ["Could not tag the PDF: stop here"]);
+
+  // Any other refusal is shown, not asked about.
+  const other = submitHandler([{ status: 422, body: { error: { code: "encrypted", message: "The PDF is encrypted." } } }], true);
+  await other.run();
+  assert.deepEqual([other.asked, other.sent.length, other.errors], [[], 1, ["Could not tag the PDF: The PDF is encrypted."]]);
+});

test/fixtures/fake-iris-pdf.mjs

@@ -8,7 +8,7 @@ const [command, ...rest] = process.argv.slice(2);
 const { values: a } = parseArgs({
   args: rest,
   strict: false,
-  options: { pdf: { type: "string" }, pages: { type: "string" }, values: { type: "string" }, out: { type: "string" }, report: { type: "string" }, json: { type: "boolean" }, help: { type: "boolean" } },
+  options: { pdf: { type: "string" }, pages: { type: "string" }, values: { type: "string" }, out: { type: "string" }, report: { type: "string" }, json: { type: "boolean" }, retag: { type: "boolean" }, help: { type: "boolean" } },
 });
 const fail = (code, message, exit) => {
   process.stderr.write(`iris-pdf: ${code}: ${message}\n`);
@@ -31,14 +31,15 @@ if (command === "fields") {
       : []),
   ]));
 } else if (command === "tag") {
+  if (pdf.includes("TAGGED") && !a.retag) fail("already_tagged", "The PDF is already tagged. Pass --retag to replace its tags.", 1);
   const values = JSON.parse(readFileSync(a.values, "utf8"));
   if ("unknown.field" in values) fail("bad_value", "No field is named unknown.field.", 3);
   if ("crash" in values) fail("internal_error", `Cannot set ${values.crash}.`, 3);
   if ("slow" in values) await new Promise((r) => setTimeout(r, 5000));
   const input = JSON.parse(readFileSync(a.pages, "utf8"));
   const mode = (statSync(a.values).mode & 0o777).toString(8);
   writeFileSync(a.out, "%PDF-fake " + JSON.stringify({ source: pdf, input, values, mode, scratch: a.values }));
-  writeFileSync(a.report, JSON.stringify({ tool: "iris-pdf 0.0.0-fake", warnings: [{ code: "duplicate_text_layer", page: 1 }] }));
+  writeFileSync(a.report, JSON.stringify({ tool: "iris-pdf 0.0.0-fake", warnings: [{ code: "duplicate_text_layer", page: 1 }, ...(a.retag ? [{ code: "retagged" }] : [])] }));
 } else {
   fail("bad_arguments", `Unknown command ${command}.`, 3);
 }

test/tagged-pdf.test.ts

@@ -248,6 +248,22 @@ test("POST /pdf passes the tagger's refusals on with its code", async () => {
   }
 });
 
+test("POST /pdf refuses an already-tagged PDF unless asked to retag it", async () => {
+  const s = await serve({ source: "%PDF TAGGED" });
+  try {
+    const refused = await s.tag({ values: {} });
+    assert.equal(refused.status, 422);
+    assert.equal((await refused.json()).error.code, "already_tagged");
+    assert.equal((await s.tag({ retag: "yes" })).status, 400);
+    const res = await s.tag({ values: {}, retag: true });
+    assert.equal(res.status, 200);
+    assert.deepEqual((await res.json()).report.warnings.map((w: { code: string }) => w.code), ["duplicate_text_layer", "retagged"]);
+    assert.match(readFileSync(s.paths.sessionLog(s.id), "utf8"), /"tagged_pdf".*"retag":true/);
+  } finally {
+    s.close();
+  }
+});
+
 test("POST /pdf waits for the finished document and stops a run that takes too long", async () => {
   const running = await serve({ status: "running" });
   try {