Always retag an already-tagged PDF

README.md

@@ -39,7 +39,6 @@ iris-pdf review --pdf out.pdf [--report review.json]   # optional AI review, bel
 | `--flatten` | Draw the field values into the page and remove the fields. |
 | `--password` | Open an encrypted PDF. The output keeps its encryption. |
 | `--allow-signed` | Tag a signed PDF. This breaks the signature, and the report says so. |
-| `--retag` | Tag a PDF that is already tagged, replacing its tags. Without it, such a PDF is refused with `already_tagged`. The output is an update of the input, so each retag adds to the file's size. |
 | `--partial` | Leave a page untagged, instead of failing, when it has no way to place text. |
 | `--strict` | Fail on any warning that means content went untagged or unmatched, or that the file was `repaired`. |
 
@@ -64,7 +63,7 @@ Text fields take strings, checkboxes `true`/`false`, radio groups and lists one
 
 ## How it works
 
-1. The page's original drawing is kept byte for byte and marked as an artifact.
+1. The page's original drawing is kept byte for byte and marked as an artifact. A PDF that is already tagged loses its old tags first (warning `retagged`); each retag adds to the file's size.
 2. Iris's words are matched to the words on the page (from the text layer, or from Tesseract on a scan).
 3. An invisible text layer is added with Iris's words at those positions, tagged with the structure from the HTML: headings, lists, tables with their headers, links, figures with alt text, form fields.
 4. The file is saved incrementally: the original bytes are the start of the output. A damaged file is instead rewritten from mupdf's repair of it, with warning `repaired`.
@@ -93,7 +92,7 @@ The output declares PDF/UA-1 only when it has a title, every page is tagged, eve
 | Exit | When |
 |---|---|
 | 0 | Done. |
-| 1 | Refused: `encrypted` (no or wrong password), `permissions_denied`, `too_many_pages` (over 25), `too_many_words` (over 4000 on a page), `already_tagged` (see `--retag`), `xfa` (dynamic form), `signed`, `no_acroform_field`, `no_text_positions`, `strict`. From `review`: `review_failed` (the model or its API failed on a page). |
+| 1 | Refused: `encrypted` (no or wrong password), `permissions_denied`, `too_many_pages` (over 25), `too_many_words` (over 4000 on a page), `xfa` (dynamic form), `signed`, `no_acroform_field`, `no_text_positions`, `strict`. From `review`: `review_failed` (the model or its API failed on a page). |
 | 2 | A check failed: `pixels_changed`, `text_lost`. |
 | 3 | Bad input: `unreadable`, `bad_pages`, `no_document_language`, `bad_value`, `field_not_settable`, `bad_arguments`. From `review`: `not_tagged`, `no_readable_structure` (not tagged by this tool), `bad_structure` (nested over 64 levels), `no_credentials`. |
 

src/cli.ts

@@ -12,7 +12,7 @@ const USAGE = `iris-pdf ${VERSION}
 iris-pdf tag --pdf <in.pdf> --pages <pages.json> --out <out.pdf>
              [--values <values.json>] [--report <report.json>] [--lang <bcp47>] [--title <text>]
              [--ocr auto|off|required] [--verify pixels,text|off] [--verify-dpi 150]
-             [--flatten] [--password <pw>] [--allow-signed] [--retag] [--partial] [--strict]
+             [--flatten] [--password <pw>] [--allow-signed] [--partial] [--strict]
 iris-pdf fields --pdf <in.pdf> [--json] [--password <pw>]
 iris-pdf check --pdf <in.pdf>
 iris-pdf review --pdf <tagged.pdf> [--report <review.json>] [--provider anthropic|bedrock] [--model <id>] [--password <pw>]`;
@@ -21,7 +21,8 @@ const OPTIONS = {
   pdf: { type: "string" }, pages: { type: "string" }, values: { type: "string" }, out: { type: "string" },
   report: { type: "string" }, lang: { type: "string" }, title: { type: "string" }, ocr: { type: "string" },
   verify: { type: "string" }, "verify-dpi": { type: "string" }, flatten: { type: "boolean" },
-  password: { type: "string" }, "allow-signed": { type: "boolean" }, retag: { type: "boolean" }, partial: { type: "boolean" },
+  password: { type: "string" }, "allow-signed": { type: "boolean" }, partial: { type: "boolean" },
+  retag: { type: "boolean" }, // ignored, for older callers: tagging always replaces old tags
   strict: { type: "boolean" }, provider: { type: "string" }, model: { type: "string" }, json: { type: "boolean" }, help: { type: "boolean", short: "h" },
 } as const;
 
@@ -98,7 +99,7 @@ async function main(argv: string[]): Promise<number> {
     values: args.values ? (readJson(args.values, "values") as TagOptions["values"]) : undefined,
     lang: args.lang, title: args.title, ocr: ocr as TagOptions["ocr"],
     verify: verify !== "off", verifyDpi: dpi, flatten: args.flatten, password: args.password,
-    allowSigned: args["allow-signed"], retag: args.retag, partial: args.partial, strict: args.strict,
+    allowSigned: args["allow-signed"], partial: args.partial, strict: args.strict,
   };
   const report = newReport();
   const pdf = readPdf(args.pdf);

src/pdf/document.ts

@@ -11,14 +11,13 @@ export type Source = {
   acroform: boolean;
   xfa: boolean;
   repaired: boolean; // damaged: saved as a full rewrite, not an update
-  restored: boolean; // retagging our own output: pages got their original content back
+  restored: boolean; // the PDF was our own output: pages got their original content back
   warnings: Warning[];
 };
 
 // readOnly: only reading (listing fields), so the refusals that protect the
 // file from changes do not apply.
-// retag: replace the tags of a PDF that has them.
-export type OpenOptions = { password?: string; allowSigned?: boolean; retag?: boolean; readOnly?: boolean };
+export type OpenOptions = { password?: string; allowSigned?: boolean; readOnly?: boolean };
 
 export function openPdf(bytes: Uint8Array, opts: OpenOptions = {}): Source {
   let doc: mupdf.PDFDocument;
@@ -56,7 +55,6 @@ export function openPdf(bytes: Uint8Array, opts: OpenOptions = {}): Source {
     throw new IrisPdfError("too_many_pages", `The PDF has ${pages} pages; the limit is ${MAX_PAGES}.`);
   }
   if (!root.get("StructTreeRoot").isNull()) {
-    if (!opts.retag) throw new IrisPdfError("already_tagged", "The PDF is already tagged. Pass --retag to replace its tags.");
     source.restored = untag(doc);
     warnings.push({ code: "retagged", detail: "The PDF's existing tags were removed and replaced." });
   }

src/tag.ts

@@ -38,7 +38,8 @@ const STRICT = ["no_title", "page_not_in_html", "unmatched_text", "missing_glyph
 export function tag(pdf: Uint8Array, input: PagesInput, opts: TagOptions = {}, report: Report = newReport()): Uint8Array {
   const src = openPdf(pdf, opts);
   const { doc } = src;
-  // The old overlay's text is gone on purpose, so the checks compare against the source without it.
+  // The old overlay's text is gone on purpose, so the text check compares against the source without it.
+  // Pixels compare against the input itself: the restore must not change a pixel.
   const baseline = src.restored ? save(doc, true) : pdf;
   const warn = (w: Warning) => report.warnings.push(w);
   src.warnings.forEach(warn);
@@ -119,7 +120,7 @@ export function tag(pdf: Uint8Array, input: PagesInput, opts: TagOptions = {}, r
 
   const out = save(doc, src.repaired);
   report.sizeIncreaseBytes = out.length - pdf.length;
-  if (opts.verify !== false) verify(baseline, out, opts, filled.changed, overlayText, report);
+  if (opts.verify !== false) verify(pdf, baseline, out, opts, filled.changed, overlayText, report);
   const strict = report.warnings.filter((w) => STRICT.includes(w.code));
   if (opts.strict && strict.length) {
     throw new IrisPdfError("strict", `--strict: ${[...new Set(strict.map((w) => w.code))].join(", ")}`);
@@ -431,15 +432,15 @@ function pagesByIndex(input: PagesInput, pageCount: number): Map<number, string>
   return out;
 }
 
-function verify(pdf: Uint8Array, out: Uint8Array, opts: TagOptions, changed: Map<number, Box[]>, added: Map<number, string>, report: Report) {
+function verify(pdf: Uint8Array, baseline: Uint8Array, out: Uint8Array, opts: TagOptions, changed: Map<number, Box[]>, added: Map<number, string>, report: Report) {
   const open = (b: Uint8Array) => {
     const d = new mupdf.PDFDocument(b);
     if (d.needsPassword()) d.authenticatePassword(opts.password ?? "");
     return d;
   };
-  const before = open(pdf), after = open(out);
-  const pixels = comparePixels(before, after, opts.verifyDpi ?? 150, changed);
-  const text = compareText(before, after, added);
+  const after = open(out);
+  const pixels = comparePixels(open(pdf), after, opts.verifyDpi ?? 150, changed);
+  const text = compareText(open(baseline), after, added);
   report.verification = {
     pixels: pixels.length ? "failed" : "identical-outside-fields",
     differingPixels: pixels.reduce((s, f) => s + f.differing, 0),

test/cli.test.ts

@@ -34,15 +34,14 @@ test("a refusal exits 1, writes no PDF, and still writes the report", () => {
   assert.equal(JSON.parse(readFileSync(report, "utf8")).error.code, "signed");
 });
 
-test("a tagged PDF is refused as already_tagged, and --retag tags it", () => {
+test("a tagged PDF is retagged, and --retag is still accepted", () => {
   const once = join(dir, "once.pdf"), twice = join(dir, "twice.pdf");
   assert.equal(run(...tagArgs("text-simple", once)).code, 0);
   const again = ["tag", "--pdf", once, "--pages", fixture("text-simple.pages.json"), "--out", twice];
-  const r = run(...again);
-  assert.equal(r.code, 1);
-  assert.match(r.err, /^iris-pdf: already_tagged: .*--retag/);
+  assert.equal(run(...again).code, 0);
+  const plain = readFileSync(twice);
   assert.equal(run(...again, "--retag").code, 0);
-  assert.ok(existsSync(twice));
+  assert.ok(readFileSync(twice).equals(plain), "--retag changes nothing");
 });
 
 test("a failed verification exits 2 and writes no PDF", () => {

test/document.test.ts

@@ -72,11 +72,10 @@ test("--allow-signed tags a signed PDF and says the signature is gone", () => {
   assert.equal(report.source.signed, true);
 });
 
-test("refuses a PDF that is already tagged, and retags it when asked", () => {
+test("retags a PDF that is already tagged", () => {
   const first = tagFixture("text-simple");
-  assert.deepEqual([refusal(first.out).code, refusal(first.out).message], ["already_tagged", "The PDF is already tagged. Pass --retag to replace its tags."]);
   const report = newReport();
-  const doc = new mupdf.PDFDocument(tag(first.out, simple, { retag: true }, report));
+  const doc = new mupdf.PDFDocument(tag(first.out, simple, {}, report));
   assert.ok(report.warnings.some((w) => w.code === "retagged"));
   // Our own output gets its original content back: one overlay, not two, and the same tags.
   assert.ok(!report.warnings.some((w) => w.code === "source_marked_content"));
@@ -86,39 +85,39 @@ test("refuses a PDF that is already tagged, and retags it when asked", () => {
   assert.equal(report.verification.differingPixels, 0);
 });
 
-test("--retag drops a foreign tag tree and what points into it", () => {
+test("retagging drops a foreign tag tree and what points into it", () => {
   const pdf = blankPdf(1, (doc) => {
     const page = doc.findPage(0);
     doc.getTrailer().get("Root").put("StructTreeRoot", doc.addObject({ Type: "StructTreeRoot", K: [] }));
     page.put("StructParents", 0);
     page.put("Annots", [doc.addObject({ Type: "Annot", Subtype: "Text", Rect: [0, 0, 10, 10], StructParent: 1 })]);
   });
-  const doc = new mupdf.PDFDocument(tag(pdf, { lang: "en", pages: [] }, { retag: true }));
+  const doc = new mupdf.PDFDocument(tag(pdf, { lang: "en", pages: [] }));
   const page = doc.findPage(0);
   assert.ok(page.get("StructParents").isNull());
   assert.ok(page.get("Annots").get(0).get("StructParent").isNull());
   assert.equal(doc.getTrailer().get("Root", "StructTreeRoot", "K", "S").asName(), "Document", "a new tree");
 });
 
-test("--retag rewrites link descriptions we wrote, and keeps the author's", () => {
+test("retagging rewrites link descriptions we wrote, and keeps the author's", () => {
   const html = (a: string) => ({ ...pagesOf("links"), pages: [{ sourcePage: 1, html: `<h1>Contact</h1><p>Apply online at ${a}.</p>` }] });
   const contents = (pdf: Uint8Array) => new mupdf.PDFDocument(pdf).findPage(0).get("Annots").get(0).get("Contents").asString();
   const once = tag(readFixture("links.pdf"), pagesOf("links"));
   assert.equal(contents(once), "the city website");
-  assert.equal(contents(tag(once, html(`the city <a href="https://example.org/permits">website</a>`), { retag: true })), "website");
+  assert.equal(contents(tag(once, html(`the city <a href="https://example.org/permits">website</a>`))), "website");
   const authored = new mupdf.PDFDocument(readFixture("links.pdf"));
   authored.findPage(0).get("Annots").get(0).put("Contents", authored.newString("Permits"));
-  const own = tag(tag(authored.saveToBuffer("").asUint8Array().slice(), pagesOf("links")), html(`the city <a href="https://example.org/permits">website</a>`), { retag: true });
+  const own = tag(tag(authored.saveToBuffer("").asUint8Array().slice(), pagesOf("links")), html(`the city <a href="https://example.org/permits">website</a>`));
   assert.equal(contents(own), "Permits");
 });
 
-test("--retag on a foreign tagged PDF keeps its marked content, so it claims no PDF/UA", () => {
+test("retagging a foreign tagged PDF keeps its marked content, so it claims no PDF/UA", () => {
   const doc = new mupdf.PDFDocument(readFixture("text-embedded.pdf"));
   const page = doc.findPage(0);
   page.put("Contents", doc.addStream("/P <</MCID 0>> BDC " + page.get("Contents").readStream().asString() + " EMC", {}));
   doc.getTrailer().get("Root").put("StructTreeRoot", doc.addObject({ Type: "StructTreeRoot" }));
   const report = newReport();
-  const out = new mupdf.PDFDocument(tag(doc.saveToBuffer("").asUint8Array().slice(), pagesOf("text-embedded"), { retag: true }, report));
+  const out = new mupdf.PDFDocument(tag(doc.saveToBuffer("").asUint8Array().slice(), pagesOf("text-embedded"), {}, report));
   assert.deepEqual(["retagged", "source_marked_content"].map((c) => report.warnings.some((w) => w.code === c)), [true, true]);
   assert.doesNotMatch(out.getTrailer().get("Root", "Metadata").readStream().asString(), /pdfuaid:part/);
 });