diff --git a/.github/workflows/code-review.yml b/.github/workflows/code-review.yml index 30c83b9..3d5236a 100644 --- a/.github/workflows/code-review.yml +++ b/.github/workflows/code-review.yml @@ -69,6 +69,7 @@ jobs: HEAD_SHA=$(echo "$META" | jq -r .head.sha) BASE_REF=$(echo "$META" | jq -r .base.ref) BASE_SHA=$(echo "$META" | jq -r .base.sha) + HEAD_REPO=$(echo "$META" | jq -r .head.repo.full_name) git fetch origin "pull/$PR_NUMBER/head:pr-$PR_NUMBER" git checkout "pr-$PR_NUMBER" else @@ -76,7 +77,10 @@ jobs: HEAD_SHA="${{ github.event.pull_request.head.sha }}" BASE_REF="${{ github.event.pull_request.base.ref }}" BASE_SHA="${{ github.event.pull_request.base.sha }}" + HEAD_REPO="${{ github.repository }}" # the job condition admits only same-repo PRs fi + # A fork's code must not run in this job: it holds id-token and the Bedrock role. + if [ "$HEAD_REPO" = "${{ github.repository }}" ]; then echo "fork=false" >> "$GITHUB_OUTPUT"; else echo "fork=true" >> "$GITHUB_OUTPUT"; fi if gh pr diff "$PR_NUMBER" --name-only | grep -qx '.github/workflows/code-review.yml'; then echo "changed_workflow=code-review.yml" >> "$GITHUB_OUTPUT" # Single-quoted: the escaped open-expression on the `github.event.*` line @@ -160,6 +164,26 @@ jobs: gh pr comment "$PR_NUMBER" --body-file "$BODY" fi + - uses: actions/setup-node@v7 + if: steps.guard.outputs.already_reviewed != 'true' + with: + # Must match .nvmrc: the sources are .ts, run directly by Node's type stripping. + node-version-file: .nvmrc + cache: npm + + # For the OCR tests. Failure-tolerant like every step before the model: a + # step that exits non-zero here would skip the fallback and verify steps. + # Without Tesseract the OCR tests skip themselves; the warning says so. + - name: Install Tesseract (OCR tests) + if: steps.guard.outputs.already_reviewed != 'true' + continue-on-error: true + run: | + if sudo apt-get update && sudo apt-get install -y tesseract-ocr; then + echo "tesseract installed" + else + echo "::warning::tesseract-ocr failed to install; the OCR tests will skip. The review still runs." + fi + - name: Build review context if: steps.guard.outputs.already_reviewed != 'true' # Every check records pass/fail/skip into the context rather than failing the @@ -172,6 +196,7 @@ jobs: HEAD_SHA: ${{ steps.ctx.outputs.head_sha }} BASE_REF: ${{ steps.ctx.outputs.base_ref }} BASE_SHA: ${{ steps.ctx.outputs.base_sha }} + FORK: ${{ steps.ctx.outputs.fork }} run: | set -euo pipefail OUT=/tmp/review-context.md @@ -257,12 +282,30 @@ jobs: fi { echo '```'; echo; echo "Result: **$SCRIPT_LINT**"; } >> "$OUT" + if [ "$FORK" = true ]; then + # Running a fork's install scripts or tests here would hand them this job's credentials. + { echo; echo "## Install, typecheck, unit tests"; echo; echo "Skipped: the head is a fork. See the Test workflow run on this PR."; } >> "$OUT" + INSTALL=skip TYPECHECK=skip UNIT=skip + else + { echo; echo "## Install — \`npm ci\`"; echo '```'; } >> "$OUT" + if npm ci 2>&1 | tail -40 >> "$OUT"; then INSTALL=pass; else INSTALL=fail; fi + { echo '```'; echo; echo "Result: **$INSTALL**"; } >> "$OUT" + + { echo; echo "## Typecheck — \`npm run typecheck\`"; echo '```'; } >> "$OUT" + if npm run typecheck 2>&1 | tail -80 >> "$OUT"; then TYPECHECK=pass; else TYPECHECK=fail; fi + { echo '```'; echo; echo "Result: **$TYPECHECK**"; } >> "$OUT" + + { echo; echo "## Unit tests — \`npm test\`"; echo '```'; } >> "$OUT" + if npm test 2>&1 | tail -120 >> "$OUT"; then UNIT=pass; else UNIT=fail; fi + { echo '```'; echo; echo "Result: **$UNIT**"; } >> "$OUT" + fi + # Also written to its own file so the fallback step can cat it without a pipe. - SUMMARY=$(printf '## Check summary\n- workflow lint (`actionlint`): %s\n- workflow scripts (`shellcheck`): %s\n' \ - "$WFLINT" "$SCRIPT_LINT") + SUMMARY=$(printf '## Check summary\n- workflow lint (`actionlint`): %s\n- workflow scripts (`shellcheck`): %s\n- npm ci: %s\n- typecheck: %s\n- unit (`npm test`): %s\n' \ + "$WFLINT" "$SCRIPT_LINT" "$INSTALL" "$TYPECHECK" "$UNIT") { echo; echo "$SUMMARY"; } >> "$OUT" echo "$SUMMARY" | tee -a "$GITHUB_STEP_SUMMARY" > /tmp/check-summary.md - if [ "$WFLINT" = fail ] || [ "$SCRIPT_LINT" = fail ]; then + if [ "$WFLINT" = fail ] || [ "$SCRIPT_LINT" = fail ] || { [ "$FORK" != true ] && [ "$INSTALL$TYPECHECK$UNIT" != passpasspass ]; }; then echo "::warning::One or more checks failed — the review continues so Claude reports it." fi diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 0000000..b01b398 --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,24 @@ +name: Test + +on: + push: + branches: [main] + pull_request: + +permissions: + contents: read + +jobs: + test: + runs-on: ubuntu-latest + timeout-minutes: 15 + steps: + - uses: actions/checkout@v7 + - uses: actions/setup-node@v7 + with: + node-version-file: .nvmrc + cache: npm + - run: sudo apt-get update && sudo apt-get install -y tesseract-ocr + - run: npm ci + - run: npm run typecheck + - run: npm test diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..9e52e77 --- /dev/null +++ b/.gitignore @@ -0,0 +1,3 @@ +node_modules/ +scratch/ +.claude/worktrees/ diff --git a/.nvmrc b/.nvmrc new file mode 100644 index 0000000..a45fd52 --- /dev/null +++ b/.nvmrc @@ -0,0 +1 @@ +24 diff --git a/CLAUDE.md b/CLAUDE.md index 17d2fd8..7f72b70 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -6,6 +6,17 @@ Uses accessible HTML to tag and update a PDF. It is a sibling of [equalify-iris] The **Iris PDF Maintainer** (a Claude agent) is this repo's primary maintainer. Other agents, human or automated, work alongside it. These include the PR reviewer in `.github/workflows/code-review.yml`. Their output is input to the maintainer, not a final decision. +## Working here + +Several agents work from this folder at once, so each piece of work gets its own git worktree: + +- Before you touch any file, start a new worktree (Claude Code: `EnterWorktree`). It lives under `.claude/worktrees/` on its own branch. +- Give subagents that edit files their own worktree too (`isolation: "worktree"`). The subagent commits on its branch and you merge that branch into yours; otherwise its work is lost. +- Never edit in the main checkout; keep it on `main`. (CI checks out there too; this rule is for local work.) +- Worktrees sit in an ignored folder inside the main checkout. Clean the main checkout with `git clean -fdx`, never `-ffdx`: the double `f` deletes other agents' worktrees. +- `scratch/` is ignored, for local experiments. +- Work lands through a branch and a pull request. + ## Standards - Everything here serves building and maintaining this library. diff --git a/README.md b/README.md index e852434..5af0364 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,105 @@ # equalify-iris-pdf + Use accessible HTML to tag and update a PDF. + +[Iris](https://github.com/EqualifyEverything/equalify-iris) turns page images into accessible HTML. This tool takes that HTML and the original PDF, and gives back **the same PDF, tagged**: a structure tree a screen reader can follow, with form fields filled in if you give it values. The page looks exactly as it did. + +It runs offline. It makes no network or model calls. + +## Install + +Node 24 or later. Tesseract 5 is optional; it is needed only for scanned pages. + +```sh +git clone https://github.com/EqualifyEverything/equalify-iris-pdf && cd equalify-iris-pdf +npm ci +npm test +npm link # puts iris-pdf on your PATH +``` + +The sources are TypeScript, run directly by Node. Node does not do that inside `node_modules`, so use a clone (as above), not `npm install` from another project. + +## Use + +```sh +iris-pdf tag --pdf in.pdf --pages pages.json --out out.pdf --report report.json +iris-pdf fields --pdf in.pdf [--json] +iris-pdf check --pdf out.pdf # runs veraPDF's PDF/UA-1 check, if installed +``` + +`tag` options: + +| Option | Meaning | +|---|---| +| `--values values.json` | Fill form fields (below). | +| `--lang`, `--title` | Used when `pages.json` has none. A language is required. | +| `--ocr auto\|off\|required` | Use Tesseract for pages with no text layer. Default `auto`. | +| `--verify pixels,text\|off` | The checks below. On by default. `--verify-dpi` sets the render resolution (36–600, default 150). | +| `--flatten` | Draw the field values into the page and remove the fields. | +| `--password` | Open an encrypted PDF. The output keeps its encryption. | +| `--allow-signed` | Tag a signed PDF. This breaks the signature, and the report says so. | +| `--partial` | Leave a page untagged, instead of failing, when it has no way to place text. | +| `--strict` | Fail on any warning that means content went untagged or unmatched. | + +### pages.json + +Iris's HTML, one entry per source page: + +```json +{ "lang": "en", "title": "Parking permit", + "pages": [{ "sourcePage": 1, "html": "

Parking Permit

…

" }] } +``` + +### values.json + +Field name to value. Names are the ones `iris-pdf fields` prints. + +```json +{ "applicant.name": "Ada Lovelace", "applicant.consent": true, "contact": "phone", "state": "WI" } +``` + +Text fields take strings, checkboxes `true`/`false`, radio groups and lists one of their options. A wrong type, an unknown option or a value over the field's length limit stops the run before anything is written. Read-only fields are skipped and counted. Unchecking always writes `/Off`, whatever the source used for "off". + +## How it works + +1. The page's original drawing is kept byte for byte and marked as an artifact. +2. Iris's words are matched to the words on the page (from the text layer, or from Tesseract on a scan). +3. An invisible text layer is added with Iris's words at those positions, tagged with the structure from the HTML: headings, lists, tables with their headers, links, figures with alt text, form fields. +4. The file is saved incrementally: the original bytes are the start of the output. + +Then two checks run, and if either fails nothing is written (exit 2): + +- **Pixels.** Every page renders the same as before, except inside fields whose values changed. +- **Text.** Every word the source had is still there, and every character of the added layer can be read back. + +## The report + +`--report` writes JSON: per page, where the text came from and how many words matched; the structure written; fields set and skipped; the check results; and every warning. Warnings name what could not be done, for example `unmatched_text` (page text missing from the HTML, kept as a paragraph), `missing_alt`, `field_not_in_html`, `unmatched_link`, `duplicate_text_layer`, `page_not_in_html` and `page_not_tagged` (the page is left as it was; a blank page needs no HTML and is not warned), `no_title`, `alignment_incomplete` (the page and the HTML differ too much to match every word in time; the rest is kept as unmatched text). + +The output declares PDF/UA-1 only when it has a title and every page is tagged. + +## Refusals and exit codes + +| Exit | When | +|---|---| +| 0 | Done. | +| 1 | Refused: `encrypted` (no or wrong password), `permissions_denied`, `damaged`, `too_many_pages` (over 25), `too_many_words` (over 4000 on a page), `already_tagged`, `xfa` (dynamic form), `signed`, `no_acroform_field`, `no_text_positions`, `strict`. | +| 2 | A check failed: `pixels_changed`, `text_lost`. | +| 3 | Bad input: `unreadable`, `bad_pages`, `no_document_language`, `bad_value`, `field_not_settable`, `bad_arguments`. | + +Errors print one line: `iris-pdf: : `. + +## Privacy + +Form values are personal data. They are never printed, logged, or put in the report or an error message; only field names are. + +## Known limits + +- **The text exists twice** on a page that already had a text layer: the original, now an artifact, and ours. Screen readers use ours. Plain copy-and-paste tools may show the text doubled. The report warns `duplicate_text_layer`. +- A table that continues onto the next page is tagged as two tables. +- `check` needs veraPDF installed. Checking the whole corpus in CI is not done yet. +- A form with no fields (a flat form) cannot be filled. + +## License + +[AGPL-3.0-or-later](LICENSE). diff --git a/package-lock.json b/package-lock.json new file mode 100644 index 0000000..be13072 --- /dev/null +++ b/package-lock.json @@ -0,0 +1,449 @@ +{ + "name": "equalify-iris-pdf", + "version": "0.1.0", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "equalify-iris-pdf", + "version": "0.1.0", + "license": "AGPL-3.0-or-later", + "dependencies": { + "mupdf": "^1.28.1", + "parse5": "^8.0.1" + }, + "bin": { + "iris-pdf": "src/cli.ts" + }, + "devDependencies": { + "@types/node": "^26.6.2", + "typescript": "^7.0.2" + }, + "engines": { + "node": ">=24" + } + }, + "node_modules/@types/node": { + "version": "26.6.2", + "resolved": "https://registry.npmjs.org/@types/node/-/node-26.6.2.tgz", + "integrity": "sha512-X1P21scMv4zGKLYqjdGjaKa7COa0RKVYYZZN/NfvLQ1JegxFhdhpZG/Lyn8AXx6CDUavKAd11v6BvfpkDByK8g==", + "dev": true, + "license": "MIT", + "dependencies": { + "undici-types": "~8.9.0" + } + }, + "node_modules/@typescript/typescript-aix-ppc64": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/@typescript/typescript-aix-ppc64/-/typescript-aix-ppc64-7.0.2.tgz", + "integrity": "sha512-MTKKkWB7p/0E9xi1d1tHtZ5PiLkGEMIq88pK2CubZjOsLtYTLqhgIgi6zepFa+9GHZ6h05NMCkQxGKiPXMxXtQ==", + "cpu": [ + "ppc64" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "aix" + ], + "engines": { + "node": ">=16.20.0" + } + }, + "node_modules/@typescript/typescript-darwin-arm64": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/@typescript/typescript-darwin-arm64/-/typescript-darwin-arm64-7.0.2.tgz", + "integrity": "sha512-gowzar9MwS/aRWp6f3a4KUqzRjAZjOsmGNCM6LcTgXum+dBfgsBVMN+AgvOCCbguXyick6LJhpBszxMebJ8syA==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=16.20.0" + } + }, + "node_modules/@typescript/typescript-darwin-x64": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/@typescript/typescript-darwin-x64/-/typescript-darwin-x64-7.0.2.tgz", + "integrity": "sha512-SZ9xZInqApNlNGc9s0W1VSsktYSOe9cFqNOIqmN1Gs8SmkjKZYFt017G4VwPxASInODuAdbTW7sXiFUf893RgA==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=16.20.0" + } + }, + "node_modules/@typescript/typescript-freebsd-arm64": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/@typescript/typescript-freebsd-arm64/-/typescript-freebsd-arm64-7.0.2.tgz", + "integrity": "sha512-W5NH4y/J0plIIS5b2xvTEkU7JFxyqdMAOgf+Ilhl0vHQXKO5dZoxd+C/jEtq56c4F3wk71RB4BMRQ2XdI+bwYQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">=16.20.0" + } + }, + "node_modules/@typescript/typescript-freebsd-x64": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/@typescript/typescript-freebsd-x64/-/typescript-freebsd-x64-7.0.2.tgz", + "integrity": "sha512-UMGDx5sTpzNw3WiPebH7l90IWfJggEd+egHt/q6p7/Cm3zqoV7VxkGXt+3DxPIw8CcmvAB0j3sVVfbhX+M4Tpw==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">=16.20.0" + } + }, + "node_modules/@typescript/typescript-linux-arm": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/@typescript/typescript-linux-arm/-/typescript-linux-arm-7.0.2.tgz", + "integrity": "sha512-gffT3xPz9sR7j/YJExkyPntrI0P2EP9XbOyWzth2/Gs0RstK+90RBcO0ncXoXy/beYll1SXw846Nf2zdnEz0QQ==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=16.20.0" + } + }, + "node_modules/@typescript/typescript-linux-arm64": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/@typescript/typescript-linux-arm64/-/typescript-linux-arm64-7.0.2.tgz", + "integrity": "sha512-Qh4eU4/y3yDjnfjjyPYihMj5/ODIlmt+Bzu17OI+fiSRDW57QmU5SiN63exPRNJPKUzcc1INa1NXdrJ+MqHjUQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=16.20.0" + } + }, + "node_modules/@typescript/typescript-linux-loong64": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/@typescript/typescript-linux-loong64/-/typescript-linux-loong64-7.0.2.tgz", + "integrity": "sha512-uEHck9i8hoAzXPiYRib1O7miOnz23SxIeVl6F4LXox+qov1K35jHcEW6VHKvZI+pyvl7fZEP4MCU5LYvIq1GuQ==", + "cpu": [ + "loong64" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=16.20.0" + } + }, + "node_modules/@typescript/typescript-linux-mips64el": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/@typescript/typescript-linux-mips64el/-/typescript-linux-mips64el-7.0.2.tgz", + "integrity": "sha512-R4KvAMnE43W5Qeqb0Ly56O3mWMWIAgsMyz36DCaycd5nbg/9kzm0liw3JocfRqyJY0KPmzFjbswozXyW0DnIYA==", + "cpu": [ + "mips64el" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=16.20.0" + } + }, + "node_modules/@typescript/typescript-linux-ppc64": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/@typescript/typescript-linux-ppc64/-/typescript-linux-ppc64-7.0.2.tgz", + "integrity": "sha512-DORx5b3sd/4S7eayxm4FQv+A7CrkUIGRaHiwI8oiHTAI1fAPWhF4J0vAlkC8biAlHSVVwxMQ3tjZ2/DVbnQiiA==", + "cpu": [ + "ppc64" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=16.20.0" + } + }, + "node_modules/@typescript/typescript-linux-riscv64": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/@typescript/typescript-linux-riscv64/-/typescript-linux-riscv64-7.0.2.tgz", + "integrity": "sha512-wf0jqEDOjrPRnKwYRyyJDRo11KMbvMFrU+q4zqKyChODBzvlkbhNQfKvLxQCcwTpdDaXSHZTVuh0JoCrKCUMHQ==", + "cpu": [ + "riscv64" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=16.20.0" + } + }, + "node_modules/@typescript/typescript-linux-s390x": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/@typescript/typescript-linux-s390x/-/typescript-linux-s390x-7.0.2.tgz", + "integrity": "sha512-IkwJc3L7yhytWd/ewjyxNDfOmswCm9GWMJT/ue/dU4aZNbwZeYAetq42VyLmsmSjvoX7z74X6ZaYCtzAr0EuGw==", + "cpu": [ + "s390x" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=16.20.0" + } + }, + "node_modules/@typescript/typescript-linux-x64": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/@typescript/typescript-linux-x64/-/typescript-linux-x64-7.0.2.tgz", + "integrity": "sha512-EYdf2cNg7rgCWJnxCdJ+F3V39O8ihb37eHAu1LK8oAFizgTQbPOK7zHHXbPt8rX24COqODXeI3sIf0fCXG7H/A==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=16.20.0" + } + }, + "node_modules/@typescript/typescript-netbsd-arm64": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/@typescript/typescript-netbsd-arm64/-/typescript-netbsd-arm64-7.0.2.tgz", + "integrity": "sha512-+polYF4MF04aPpO5FTkHran9yUQDSXqy5GiSDKpsll5jy3l3+g9QLhpf39T+ePtefhXLOGrLl0QIjkQP6VnelA==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "netbsd" + ], + "engines": { + "node": ">=16.20.0" + } + }, + "node_modules/@typescript/typescript-netbsd-x64": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/@typescript/typescript-netbsd-x64/-/typescript-netbsd-x64-7.0.2.tgz", + "integrity": "sha512-8YIT0EHM/3dq10ZOVF/A7pc/YSMtbcecct4rWtexrnSCHOPcpC2KTLXfTCR6vDpnSiY12heNb1GiN/wu+T/FyA==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "netbsd" + ], + "engines": { + "node": ">=16.20.0" + } + }, + "node_modules/@typescript/typescript-openbsd-arm64": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/@typescript/typescript-openbsd-arm64/-/typescript-openbsd-arm64-7.0.2.tgz", + "integrity": "sha512-APT8+ClYnuYm1u9+kgGXoMj2VzWzcymwh2gNSQVySHfkRDGOTVkoWLjCmOQSaO+PoqQ57B0flRp9SA+7GnnkzQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "openbsd" + ], + "engines": { + "node": ">=16.20.0" + } + }, + "node_modules/@typescript/typescript-openbsd-x64": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/@typescript/typescript-openbsd-x64/-/typescript-openbsd-x64-7.0.2.tgz", + "integrity": "sha512-yX7s+Q0Dln0Dt9tEzZsAjXXR/+ytBM7AlglaqyeMPxQszJ1JhlJdZ6jLA+IzldHtflX81em7lDao1xXu+aRRkg==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "openbsd" + ], + "engines": { + "node": ">=16.20.0" + } + }, + "node_modules/@typescript/typescript-sunos-x64": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/@typescript/typescript-sunos-x64/-/typescript-sunos-x64-7.0.2.tgz", + "integrity": "sha512-dLJDGaLZ1D4HPQn62u1n8mBDkJREwMsAkCdkwd4Ieqw+x3TUyTsqY0YiBCtE6H6OzzgGk3iuZ3vFWRS+E8/d1g==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "sunos" + ], + "engines": { + "node": ">=16.20.0" + } + }, + "node_modules/@typescript/typescript-win32-arm64": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/@typescript/typescript-win32-arm64/-/typescript-win32-arm64-7.0.2.tgz", + "integrity": "sha512-Gyl1Vy6OsWesLzmq+EP0Fb7b4Nid5232AvcA2SFcdYreldpNtYFFofPjnt62y9hQy7VTaZp65ICJjuAQRaVcIQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=16.20.0" + } + }, + "node_modules/@typescript/typescript-win32-x64": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/@typescript/typescript-win32-x64/-/typescript-win32-x64-7.0.2.tgz", + "integrity": "sha512-0BQ3HkAHHlKLSp1qRvf3SUhGpGsDuhB/jgFw75guyqbxJqEaS0Cw/VFO8i2nHglJUzQCRtMMR/IBAKE3ETMC4g==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=16.20.0" + } + }, + "node_modules/entities": { + "version": "8.1.0", + "resolved": "https://registry.npmjs.org/entities/-/entities-8.1.0.tgz", + "integrity": "sha512-kxL7msIffSuh9aaFAMD7rxAIuTRMAHMeBtgHW2yUdWw732ZNh4MehkF2gdjvtdmikkaIP9bFDDJOPlsvm7avrA==", + "license": "BSD-2-Clause", + "engines": { + "node": ">=20.19.0" + }, + "funding": { + "url": "https://github.com/fb55/entities?sponsor=1" + } + }, + "node_modules/mupdf": { + "version": "1.28.1", + "resolved": "https://registry.npmjs.org/mupdf/-/mupdf-1.28.1.tgz", + "integrity": "sha512-Gi11Ow2G1SlrXKJNZBL1eAIGFVih5+4ZKqjptamTVaj/5hnlrcVrVbyb7lHE2lfFKdxZPyv9ZtfOOq7XgjzEig==", + "license": "AGPL-3.0-or-later" + }, + "node_modules/parse5": { + "version": "8.0.1", + "resolved": "https://registry.npmjs.org/parse5/-/parse5-8.0.1.tgz", + "integrity": "sha512-z1e/HMG90obSGeidlli3hj7cbocou0/wa5HacvI3ASx34PecNjNQeaHNo5WIZpWofN9kgkqV1q5YvXe3F0FoPw==", + "license": "MIT", + "dependencies": { + "entities": "^8.0.0" + }, + "funding": { + "url": "https://github.com/inikulin/parse5?sponsor=1" + } + }, + "node_modules/typescript": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-7.0.2.tgz", + "integrity": "sha512-8FYau96o3NKOhbjKi/qNvG/W5jhzxkbdm5sj9AbZ/5T5sWqn3hJgLfGx27sRKZWTvyzCP8dLRBTf5tBTSRVUNA==", + "dev": true, + "license": "Apache-2.0", + "bin": { + "tsc": "bin/tsc" + }, + "engines": { + "node": ">=16.20.0" + }, + "optionalDependencies": { + "@typescript/typescript-aix-ppc64": "7.0.2", + "@typescript/typescript-darwin-arm64": "7.0.2", + "@typescript/typescript-darwin-x64": "7.0.2", + "@typescript/typescript-freebsd-arm64": "7.0.2", + "@typescript/typescript-freebsd-x64": "7.0.2", + "@typescript/typescript-linux-arm": "7.0.2", + "@typescript/typescript-linux-arm64": "7.0.2", + "@typescript/typescript-linux-loong64": "7.0.2", + "@typescript/typescript-linux-mips64el": "7.0.2", + "@typescript/typescript-linux-ppc64": "7.0.2", + "@typescript/typescript-linux-riscv64": "7.0.2", + "@typescript/typescript-linux-s390x": "7.0.2", + "@typescript/typescript-linux-x64": "7.0.2", + "@typescript/typescript-netbsd-arm64": "7.0.2", + "@typescript/typescript-netbsd-x64": "7.0.2", + "@typescript/typescript-openbsd-arm64": "7.0.2", + "@typescript/typescript-openbsd-x64": "7.0.2", + "@typescript/typescript-sunos-x64": "7.0.2", + "@typescript/typescript-win32-arm64": "7.0.2", + "@typescript/typescript-win32-x64": "7.0.2" + } + }, + "node_modules/undici-types": { + "version": "8.9.0", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-8.9.0.tgz", + "integrity": "sha512-KTDyRTYX8sWmKXAikPHHSyc63CRPETMctyjKFupcC6OBLXT3xsN0e9aF7m+mIXutFWpUXuedtowG7iLOzp0kQg==", + "dev": true, + "license": "MIT" + } + } +} diff --git a/package.json b/package.json new file mode 100644 index 0000000..cfcdca1 --- /dev/null +++ b/package.json @@ -0,0 +1,28 @@ +{ + "name": "equalify-iris-pdf", + "version": "0.1.0", + "description": "Tag a PDF for accessibility using Iris's HTML, and fill its form fields", + "type": "module", + "license": "AGPL-3.0-or-later", + "repository": "github:EqualifyEverything/equalify-iris-pdf", + "engines": { + "node": ">=24" + }, + "bin": { + "iris-pdf": "src/cli.ts" + }, + "exports": "./src/index.ts", + "scripts": { + "typecheck": "tsc --noEmit", + "test": "node --test \"test/*.test.ts\"", + "fixtures": "node test/fixtures/make.ts" + }, + "dependencies": { + "mupdf": "^1.28.1", + "parse5": "^8.0.1" + }, + "devDependencies": { + "@types/node": "^26.6.2", + "typescript": "^7.0.2" + } +} diff --git a/src/align/align.ts b/src/align/align.ts new file mode 100644 index 0000000..e2fec7c --- /dev/null +++ b/src/align/align.ts @@ -0,0 +1,118 @@ +// Word-level alignment of Iris's text to the page's words (spec §7.5 step 3). +// +// The two orders can differ (Iris fixes column order; the PDF's operators do +// not), so we align on content: find the best-matching stretch anywhere on the +// page (a local alignment), claim it, then repeat for the HTML words left over +// on either side. Each pass is O(n·m); a budget caps the total work, so a +// page whose orders disagree badly cannot run for minutes. + +const MATCH = 3, FUZZY = 1, MISMATCH = -3, GAP = -1; +export const MAX_WORDS = 4000; +const BUDGET = 50_000_000; // cells, over all passes on a page + +// For each HTML token, the index of the page token it matched, or -1. +// Empty tokens (pure punctuation) never match. complete is false if the +// budget ran out; the words not yet matched stay at -1. +export function align(html: string[], page: string[], budget = BUDGET): { match: number[]; complete: boolean } { + const scores = new Scores(html, page); + const match = html.map(() => -1); + const claimed = page.map((t) => !t); + const todo: [number, number][] = [[0, html.length]]; + while (todo.length) { + const [a, b] = todo.pop()!; + const hs: number[] = []; + for (let i = a; i < b; i++) if (html[i]) hs.push(i); + const ps = page.flatMap((_, j) => (claimed[j] ? [] : [j])); + if (!hs.length || !ps.length) continue; + if ((budget -= hs.length * ps.length) < 0) return { match, complete: false }; + const { best, pairs } = local(hs, ps, scores); + // A lone common word is not evidence; ask for two exact matches unless the + // stretch is shorter than that. + if (best < Math.min(2 * MATCH, MATCH * hs.length) || !pairs.length) continue; + for (const [x, y] of pairs) { + match[hs[x]] = ps[y]; + claimed[ps[y]] = true; + } + // Retry the gaps: before, between and after the matched words. + const hit = pairs.map(([x]) => hs[x]); + todo.push([a, hit[0]], [hit.at(-1)! + 1, b]); + for (let k = 1; k < hit.length; k++) if (hit[k] > hit[k - 1] + 1) todo.push([hit[k - 1] + 1, hit[k]]); + } + return { match, complete: true }; +} + +// Scores between distinct words, filled in as the alignment asks for them. +class Scores { + h: Int32Array; p: Int32Array; // word ids + private words: string[] = []; + private table: Int8Array; + private cols: number; + constructor(html: string[], page: string[]) { + const ids = new Map(); + const id = (t: string) => ids.get(t) ?? (ids.set(t, this.words.length), this.words.push(t) - 1); + this.h = Int32Array.from(html, id); + this.p = Int32Array.from(page, id); + this.cols = this.words.length; + this.table = new Int8Array(this.cols * this.cols); // 0: not yet known + } + get(a: number, b: number): number { + if (a === b) return MATCH; + const k = a * this.cols + b; + return this.table[k] || (this.table[k] = similar(this.words[a], this.words[b]) ? FUZZY : MISMATCH); + } +} + +const similar = (a: string, b: string) => { + const most = Math.floor(Math.max(a.length, b.length) / 4); + return Math.min(a.length, b.length) >= 4 && Math.abs(a.length - b.length) <= most && distance(a, b, most) <= most; +}; + +// Smith–Waterman over html[hs[x]] and page[ps[y]]. Returns the best local +// score and its matched pairs (x, y). +function local(hs: number[], ps: number[], s: Scores): { best: number; pairs: [number, number][] } { + const n = hs.length, m = ps.length, w = m + 1; + const H = new Int32Array((n + 1) * w); + const pw = Int32Array.from(ps, (j) => s.p[j]); + const score = (x: number, y: number) => s.get(s.h[hs[x]], pw[y]); + let best = 0, bi = 0, bj = 0; + for (let i = 1; i <= n; i++) { + const up = (i - 1) * w, row = i * w; + for (let j = 1; j <= m; j++) { + let v = H[up + j - 1] + score(i - 1, j - 1); + if (H[up + j] + GAP > v) v = H[up + j] + GAP; + if (H[row + j - 1] + GAP > v) v = H[row + j - 1] + GAP; + if (v < 0) v = 0; + H[row + j] = v; + if (v > best) best = v, bi = i, bj = j; + } + } + const pairs: [number, number][] = []; + for (let i = bi, j = bj; i > 0 && j > 0 && H[i * w + j] > 0; ) { + const v = H[i * w + j], sc = score(i - 1, j - 1); + if (v === H[(i - 1) * w + j - 1] + sc) { + if (sc > 0) pairs.push([i - 1, j - 1]); + i--, j--; + } else if (v === H[(i - 1) * w + j] + GAP) i--; + else j--; + } + return { best, pairs: pairs.reverse() }; +} + +// Levenshtein distance, or more than max as soon as that is certain. +const rows = [new Int32Array(64), new Int32Array(64)]; +function distance(a: string, b: string, max: number): number { + if (b.length >= rows[0].length) return a === b ? 0 : max + 1; + let [prev, cur] = rows; + for (let j = 0; j <= b.length; j++) prev[j] = j; + for (let i = 1; i <= a.length; i++) { + cur[0] = i; + let low = i; + for (let j = 1; j <= b.length; j++) { + cur[j] = Math.min(prev[j] + 1, cur[j - 1] + 1, prev[j - 1] + (a[i - 1] === b[j - 1] ? 0 : 1)); + if (cur[j] < low) low = cur[j]; + } + if (low > max) return low; + [prev, cur] = [cur, prev]; + } + return prev[b.length]; +} diff --git a/src/align/classify.ts b/src/align/classify.ts new file mode 100644 index 0000000..5caeee5 --- /dev/null +++ b/src/align/classify.ts @@ -0,0 +1,12 @@ +// What each unmatched page word means (spec §7.5, the outcomes table). +import { normalize, type PageWord } from "./words.ts"; + +// Page furniture: a running head, a footer, a page number. Iris strips these +// on purpose, and the original drawing of them is already an artifact. +// Anything else Iris left out is lost content, and is reported. +export function isFurniture(word: PageWord, pageHeight: number, pageNumber: number): boolean { + const norm = normalize(word.text); + if (!norm) return true; // bullets, rules and other punctuation + if (norm === String(pageNumber)) return true; + return word.box[3] <= pageHeight * 0.1 || word.box[1] >= pageHeight * 0.9; +} diff --git a/src/align/words.ts b/src/align/words.ts new file mode 100644 index 0000000..eaacde2 --- /dev/null +++ b/src/align/words.ts @@ -0,0 +1,60 @@ +// Normalising and tokenising words, the same way for the page and the HTML, +// so they can be compared (spec §7.5 step 2). + +// A box in mupdf's page space: x0, y0 top, x1, y1 bottom, in points. +export type Box = [number, number, number, number]; + +// A word as it sits on the page, from the text layer or from OCR. +export type PageWord = { text: string; box: Box; baseline: number; size: number }; + +// Chinese, Japanese and Korean are written without spaces, so each character +// is its own token. +const CJK = /[\u3040-\u30ff\u3400-\u4dbf\u4e00-\u9fff\uac00-\ud7af\uf900-\ufaff]/u; +export const isCJK = (c: string) => CJK.test(c); + +// Lower-case, accents stripped, punctuation trimmed from the ends. +export function normalize(word: string): string { + return word + .normalize("NFKD") + .replace(/\p{M}/gu, "") + .toLowerCase() + .replace(/^[^\p{L}\p{N}]+|[^\p{L}\p{N}]+$/gu, ""); +} + +// Split text into words: on whitespace, and between CJK characters. +// Soft hyphens are dropped. space: whether whitespace follows the word. +export function splitWords(text: string): { text: string; space: boolean }[] { + const out: { text: string; space: boolean }[] = []; + let cur = ""; + const push = () => { + if (cur) out.push({ text: cur, space: false }); + cur = ""; + }; + for (const c of text.replace(/\u00ad/g, "")) { + if (/\s/.test(c)) { + push(); + if (out.length) out[out.length - 1].space = true; + } else if (CJK.test(c)) { + push(); + cur = c; + push(); + } else cur += c; + } + push(); + return out; +} + +// A word the source broke at a line end ("per-" / "mit") is one token on the +// page side. Returns, for each token, the page words it covers. +export function joinHyphenated(words: PageWord[]): { norm: string; words: number[] }[] { + const out: { norm: string; words: number[] }[] = []; + for (let i = 0; i < words.length; i++) { + const w = words[i], next = words[i + 1]; + const lineEnd = next && next.box[1] > w.box[3] - 1; // the next word starts on a lower line + if (lineEnd && /\p{L}-$/u.test(w.text) && /^\p{Ll}/u.test(next.text)) { + out.push({ norm: normalize(w.text.slice(0, -1) + next.text), words: [i, i + 1] }); + i++; + } else out.push({ norm: normalize(w.text), words: [i] }); + } + return out; +} diff --git a/src/cli.ts b/src/cli.ts new file mode 100644 index 0000000..eac83f5 --- /dev/null +++ b/src/cli.ts @@ -0,0 +1,109 @@ +#!/usr/bin/env node +// The iris-pdf command. Exit codes: 0 done, 1 refused, 2 verification +// failed, 3 bad arguments or unreadable input. +import { readFileSync, writeFileSync } from "node:fs"; +import { parseArgs } from "node:util"; +import { tag, fields, newReport, IrisPdfError, EXIT, VERSION, type TagOptions } from "./index.ts"; +import { checkPdfUa } from "./verify/pdfua.ts"; + +const USAGE = `iris-pdf ${VERSION} + +iris-pdf tag --pdf --pages --out + [--values ] [--report ] [--lang ] [--title ] + [--ocr auto|off|required] [--verify pixels,text|off] [--verify-dpi 150] + [--flatten] [--password ] [--allow-signed] [--partial] [--strict] +iris-pdf fields --pdf [--json] [--password ] +iris-pdf check --pdf `; + +const OPTIONS = { + pdf: { type: "string" }, pages: { type: "string" }, values: { type: "string" }, out: { type: "string" }, + report: { type: "string" }, lang: { type: "string" }, title: { type: "string" }, ocr: { type: "string" }, + verify: { type: "string" }, "verify-dpi": { type: "string" }, flatten: { type: "boolean" }, + password: { type: "string" }, "allow-signed": { type: "boolean" }, partial: { type: "boolean" }, + strict: { type: "boolean" }, json: { type: "boolean" }, help: { type: "boolean", short: "h" }, +} as const; + +function badArgs(message: string): never { + throw new IrisPdfError("bad_arguments", message, EXIT.badInput); +} + +function readJson(path: string, what: string): unknown { + try { + return JSON.parse(readFileSync(path, "utf8")); + } catch (e) { + return badArgs(`Cannot read ${what} ${path}: ${(e as Error).message}`); + } +} + +function readPdf(path: string | undefined): Uint8Array { + if (!path) badArgs("--pdf is required."); + try { + return readFileSync(path); + } catch (e) { + return badArgs(`Cannot read ${path}: ${(e as Error).message}`); + } +} + +function main(argv: string[]): number { + const [command, ...rest] = argv; + let args; + try { + args = parseArgs({ args: rest, options: OPTIONS, strict: true }).values; + } catch (e) { + return badArgs((e as Error).message); + } + if (!command || args.help) { + console.log(USAGE); + return command ? 0 : EXIT.badInput; + } + + if (command === "fields") { + const list = fields(readPdf(args.pdf), { password: args.password }); + if (args.json) console.log(JSON.stringify(list, null, 2)); + else for (const f of list) console.log(`${f.name}\t${f.type}\tpage ${f.page}${f.options.length ? `\t[${f.options.join(", ")}]` : ""}${f.readonly ? "\treadonly" : ""}${f.required ? "\trequired" : ""}`); + return 0; + } + + if (command === "check") { + const result = checkPdfUa(args.pdf ?? badArgs("--pdf is required.")); + console.log(result.message); + return result.passed === false ? EXIT.verification : 0; + } + + if (command !== "tag") badArgs(`Unknown command "${command}".\n${USAGE}`); + if (!args.pages || !args.out) badArgs("tag needs --pdf, --pages and --out."); + const ocr = args.ocr ?? "auto"; + if (!["auto", "off", "required"].includes(ocr)) badArgs("--ocr is auto, off or required."); + const verify = args.verify ?? "pixels,text"; + if (verify !== "off" && verify !== "pixels,text") badArgs("--verify is pixels,text or off."); + const dpi = Number(args["verify-dpi"] ?? 150); + if (!(dpi >= 36 && dpi <= 600)) badArgs("--verify-dpi is between 36 and 600."); + + const opts: TagOptions = { + values: args.values ? (readJson(args.values, "values") as TagOptions["values"]) : undefined, + lang: args.lang, title: args.title, ocr: ocr as TagOptions["ocr"], + verify: verify !== "off", verifyDpi: dpi, flatten: args.flatten, password: args.password, + allowSigned: args["allow-signed"], partial: args.partial, strict: args.strict, + }; + const report = newReport(); + const pdf = readPdf(args.pdf); + const pages = readJson(args.pages, "pages"); + try { + writeFileSync(args.out, tag(pdf, pages as Parameters[1], opts, report)); + return 0; + } catch (e) { + const err = e instanceof IrisPdfError ? e : new IrisPdfError("internal_error", String((e as Error).message ?? e), EXIT.badInput); + report.error = { code: err.code, message: err.message }; + throw err; + } finally { + if (args.report) writeFileSync(args.report, JSON.stringify(report, null, 2) + "\n"); + } +} + +try { + process.exitCode = main(process.argv.slice(2)); +} catch (e) { + if (!(e instanceof IrisPdfError)) throw e; + console.error(`iris-pdf: ${e.code}: ${e.message}`); + process.exitCode = e.exit; +} diff --git a/src/html/build.ts b/src/html/build.ts new file mode 100644 index 0000000..59c71c9 --- /dev/null +++ b/src/html/build.ts @@ -0,0 +1,243 @@ +// Iris's HTML for one page -> the structure we will write: a tree of PDF +// structure types whose leaves are words. Positions are filled in later by +// the alignment. +import { parseHtml, textOf, type Elem } from "./parse.ts"; +import { STRUCT, TRANSPARENT, SKIP, listNumbering } from "./map.ts"; +import { normalize, splitWords, type Box } from "../align/words.ts"; +import type { Warning } from "../report.ts"; + +export type Placed = { box: Box; baseline: number; size: number }; +export type Word = { text: string; norm: string; space?: boolean; at?: Placed }; // space: one follows +export type Run = { words: Word[] }; +export type FormRef = { name: string; value?: string; label: string; group?: string }; + +export type Node = { + type: string; + kids: (Node | Run)[]; + alt?: string; + title?: string; // /T + lang?: string; + id?: string; + attrs?: Record; // the /A attribute object + headers?: string[]; // a TD's /Headers + href?: string; // Link + form?: FormRef; // Form + span?: [number, number]; // a table cell's colspan, rowspan +}; + +export const isRun = (k: Node | Run): k is Run => "words" in k; + +type Ctx = { + lang: string; + prefix: string; // keeps ids unique across pages + labels: Map; // input id -> its label's text + notes: Set; // ids that internal links point at + label?: string; // text of an enclosing