diff --git a/.gitignore b/.gitignore index c7a17f5..8c8a0bb 100644 --- a/.gitignore +++ b/.gitignore @@ -7,3 +7,7 @@ dist/ version.txt coverage*.out coverage.html +cover.out +cover*.out +private_pdf_examples +testdata/test-config.yaml diff --git a/AGENTS.md b/AGENTS.md index 91e48cc..840ca0f 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -62,6 +62,8 @@ Fileganizer is a Go CLI tool that processes documents through a pipeline: text e - Use `gofmt` / `goimports` formatting. Max line length 140. - Group imports: stdlib first, third-party second, internal (`fileganizer/...`) last. - Flags (like `-c` or `-f`) are never constants. When the linter complains, add `//nolint`. +- No global or function-scoped `//nolint`. Only line-scoped `//nolint` is allowed. +- Keep the whole code simple and stupid (KISS). No over-engineering, no unnecessary abstractions. - Copyright header on every source file. For `.go` files: ```go // Copyright 2023-2026 The Fileganizer Authors. All rights reserved. diff --git a/pdftotext/fontwidths_std.go b/pdftotext/fontwidths_std.go new file mode 100644 index 0000000..1b6b16b --- /dev/null +++ b/pdftotext/fontwidths_std.go @@ -0,0 +1,2550 @@ +// Copyright 2023-2026 The Fileganizer Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package pdftotext + +// Standard 14 font glyph widths in PDF units (1000 = em). +// Keyed by character code for WinAnsiEncoding derived fonts +// (Helvetica, Times, Courier) and font-specific encoding +// for Symbol and ZapfDingbats. +// Source: pdfminer.six FontMetricsDB (Adobe standard metrics). + +//nolint:gochecknoglobals,dupl +var stdFontWidths = map[string]*fontWidths{ + "Helvetica": { + 32: 278, + 33: 278, + 34: 355, + 35: 556, + 36: 556, + 37: 889, + 38: 667, + 39: 191, + 40: 333, + 41: 333, + 42: 389, + 43: 584, + 44: 278, + 45: 333, + 46: 278, + 47: 278, + 48: 556, + 49: 556, + 50: 556, + 51: 556, + 52: 556, + 53: 556, + 54: 556, + 55: 556, + 56: 556, + 57: 556, + 58: 278, + 59: 278, + 60: 584, + 61: 584, + 62: 584, + 63: 556, + 64: 1015, + 65: 667, + 66: 667, + 67: 722, + 68: 722, + 69: 667, + 70: 611, + 71: 778, + 72: 722, + 73: 278, + 74: 500, + 75: 667, + 76: 556, + 77: 833, + 78: 722, + 79: 778, + 80: 667, + 81: 778, + 82: 722, + 83: 667, + 84: 611, + 85: 722, + 86: 667, + 87: 944, + 88: 667, + 89: 667, + 90: 611, + 91: 278, + 92: 278, + 93: 278, + 94: 469, + 95: 556, + 96: 333, + 97: 556, + 98: 556, + 99: 500, + 100: 556, + 101: 556, + 102: 278, + 103: 556, + 104: 556, + 105: 222, + 106: 222, + 107: 500, + 108: 222, + 109: 833, + 110: 556, + 111: 556, + 112: 556, + 113: 556, + 114: 333, + 115: 500, + 116: 278, + 117: 556, + 118: 500, + 119: 722, + 120: 500, + 121: 500, + 122: 500, + 123: 334, + 124: 260, + 125: 334, + 126: 584, + 161: 333, + 162: 556, + 163: 556, + 164: 556, + 165: 556, + 166: 260, + 167: 556, + 168: 333, + 169: 737, + 170: 370, + 171: 556, + 172: 584, + 174: 737, + 175: 333, + 176: 400, + 177: 584, + 178: 333, + 179: 333, + 180: 333, + 181: 556, + 182: 537, + 183: 278, + 184: 333, + 185: 333, + 186: 365, + 187: 556, + 188: 834, + 189: 834, + 190: 834, + 191: 611, + 192: 667, + 193: 667, + 194: 667, + 195: 667, + 196: 667, + 197: 667, + 198: 1000, + 199: 722, + 200: 667, + 201: 667, + 202: 667, + 203: 667, + 204: 278, + 205: 278, + 206: 278, + 207: 278, + 208: 722, + 209: 722, + 210: 778, + 211: 778, + 212: 778, + 213: 778, + 214: 778, + 215: 584, + 216: 778, + 217: 722, + 218: 722, + 219: 722, + 220: 722, + 221: 667, + 222: 667, + 223: 611, + 224: 556, + 225: 556, + 226: 556, + 227: 556, + 228: 556, + 229: 556, + 230: 889, + 231: 500, + 232: 556, + 233: 556, + 234: 556, + 235: 556, + 236: 278, + 237: 278, + 238: 278, + 239: 278, + 240: 556, + 241: 556, + 242: 556, + 243: 556, + 244: 556, + 245: 556, + 246: 556, + 247: 584, + 248: 611, + 249: 556, + 250: 556, + 251: 556, + 252: 556, + 253: 500, + 254: 556, + 255: 500, + }, + "Helvetica-Bold": { + 32: 278, + 33: 333, + 34: 474, + 35: 556, + 36: 556, + 37: 889, + 38: 722, + 39: 238, + 40: 333, + 41: 333, + 42: 389, + 43: 584, + 44: 278, + 45: 333, + 46: 278, + 47: 278, + 48: 556, + 49: 556, + 50: 556, + 51: 556, + 52: 556, + 53: 556, + 54: 556, + 55: 556, + 56: 556, + 57: 556, + 58: 333, + 59: 333, + 60: 584, + 61: 584, + 62: 584, + 63: 611, + 64: 975, + 65: 722, + 66: 722, + 67: 722, + 68: 722, + 69: 667, + 70: 611, + 71: 778, + 72: 722, + 73: 278, + 74: 556, + 75: 722, + 76: 611, + 77: 833, + 78: 722, + 79: 778, + 80: 667, + 81: 778, + 82: 722, + 83: 667, + 84: 611, + 85: 722, + 86: 667, + 87: 944, + 88: 667, + 89: 667, + 90: 611, + 91: 333, + 92: 278, + 93: 333, + 94: 584, + 95: 556, + 96: 333, + 97: 556, + 98: 611, + 99: 556, + 100: 611, + 101: 556, + 102: 333, + 103: 611, + 104: 611, + 105: 278, + 106: 278, + 107: 556, + 108: 278, + 109: 889, + 110: 611, + 111: 611, + 112: 611, + 113: 611, + 114: 389, + 115: 556, + 116: 333, + 117: 611, + 118: 556, + 119: 778, + 120: 556, + 121: 556, + 122: 500, + 123: 389, + 124: 280, + 125: 389, + 126: 584, + 161: 333, + 162: 556, + 163: 556, + 164: 556, + 165: 556, + 166: 280, + 167: 556, + 168: 333, + 169: 737, + 170: 370, + 171: 556, + 172: 584, + 174: 737, + 175: 333, + 176: 400, + 177: 584, + 178: 333, + 179: 333, + 180: 333, + 181: 611, + 182: 556, + 183: 278, + 184: 333, + 185: 333, + 186: 365, + 187: 556, + 188: 834, + 189: 834, + 190: 834, + 191: 611, + 192: 722, + 193: 722, + 194: 722, + 195: 722, + 196: 722, + 197: 722, + 198: 1000, + 199: 722, + 200: 667, + 201: 667, + 202: 667, + 203: 667, + 204: 278, + 205: 278, + 206: 278, + 207: 278, + 208: 722, + 209: 722, + 210: 778, + 211: 778, + 212: 778, + 213: 778, + 214: 778, + 215: 584, + 216: 778, + 217: 722, + 218: 722, + 219: 722, + 220: 722, + 221: 667, + 222: 667, + 223: 611, + 224: 556, + 225: 556, + 226: 556, + 227: 556, + 228: 556, + 229: 556, + 230: 889, + 231: 556, + 232: 556, + 233: 556, + 234: 556, + 235: 556, + 236: 278, + 237: 278, + 238: 278, + 239: 278, + 240: 611, + 241: 611, + 242: 611, + 243: 611, + 244: 611, + 245: 611, + 246: 611, + 247: 584, + 248: 611, + 249: 611, + 250: 611, + 251: 611, + 252: 611, + 253: 556, + 254: 611, + 255: 556, + }, + "Helvetica-Oblique": { + 32: 278, + 33: 278, + 34: 355, + 35: 556, + 36: 556, + 37: 889, + 38: 667, + 39: 191, + 40: 333, + 41: 333, + 42: 389, + 43: 584, + 44: 278, + 45: 333, + 46: 278, + 47: 278, + 48: 556, + 49: 556, + 50: 556, + 51: 556, + 52: 556, + 53: 556, + 54: 556, + 55: 556, + 56: 556, + 57: 556, + 58: 278, + 59: 278, + 60: 584, + 61: 584, + 62: 584, + 63: 556, + 64: 1015, + 65: 667, + 66: 667, + 67: 722, + 68: 722, + 69: 667, + 70: 611, + 71: 778, + 72: 722, + 73: 278, + 74: 500, + 75: 667, + 76: 556, + 77: 833, + 78: 722, + 79: 778, + 80: 667, + 81: 778, + 82: 722, + 83: 667, + 84: 611, + 85: 722, + 86: 667, + 87: 944, + 88: 667, + 89: 667, + 90: 611, + 91: 278, + 92: 278, + 93: 278, + 94: 469, + 95: 556, + 96: 333, + 97: 556, + 98: 556, + 99: 500, + 100: 556, + 101: 556, + 102: 278, + 103: 556, + 104: 556, + 105: 222, + 106: 222, + 107: 500, + 108: 222, + 109: 833, + 110: 556, + 111: 556, + 112: 556, + 113: 556, + 114: 333, + 115: 500, + 116: 278, + 117: 556, + 118: 500, + 119: 722, + 120: 500, + 121: 500, + 122: 500, + 123: 334, + 124: 260, + 125: 334, + 126: 584, + 161: 333, + 162: 556, + 163: 556, + 164: 556, + 165: 556, + 166: 260, + 167: 556, + 168: 333, + 169: 737, + 170: 370, + 171: 556, + 172: 584, + 174: 737, + 175: 333, + 176: 400, + 177: 584, + 178: 333, + 179: 333, + 180: 333, + 181: 556, + 182: 537, + 183: 278, + 184: 333, + 185: 333, + 186: 365, + 187: 556, + 188: 834, + 189: 834, + 190: 834, + 191: 611, + 192: 667, + 193: 667, + 194: 667, + 195: 667, + 196: 667, + 197: 667, + 198: 1000, + 199: 722, + 200: 667, + 201: 667, + 202: 667, + 203: 667, + 204: 278, + 205: 278, + 206: 278, + 207: 278, + 208: 722, + 209: 722, + 210: 778, + 211: 778, + 212: 778, + 213: 778, + 214: 778, + 215: 584, + 216: 778, + 217: 722, + 218: 722, + 219: 722, + 220: 722, + 221: 667, + 222: 667, + 223: 611, + 224: 556, + 225: 556, + 226: 556, + 227: 556, + 228: 556, + 229: 556, + 230: 889, + 231: 500, + 232: 556, + 233: 556, + 234: 556, + 235: 556, + 236: 278, + 237: 278, + 238: 278, + 239: 278, + 240: 556, + 241: 556, + 242: 556, + 243: 556, + 244: 556, + 245: 556, + 246: 556, + 247: 584, + 248: 611, + 249: 556, + 250: 556, + 251: 556, + 252: 556, + 253: 500, + 254: 556, + 255: 500, + }, + "Helvetica-BoldOblique": { + 32: 278, + 33: 333, + 34: 474, + 35: 556, + 36: 556, + 37: 889, + 38: 722, + 39: 238, + 40: 333, + 41: 333, + 42: 389, + 43: 584, + 44: 278, + 45: 333, + 46: 278, + 47: 278, + 48: 556, + 49: 556, + 50: 556, + 51: 556, + 52: 556, + 53: 556, + 54: 556, + 55: 556, + 56: 556, + 57: 556, + 58: 333, + 59: 333, + 60: 584, + 61: 584, + 62: 584, + 63: 611, + 64: 975, + 65: 722, + 66: 722, + 67: 722, + 68: 722, + 69: 667, + 70: 611, + 71: 778, + 72: 722, + 73: 278, + 74: 556, + 75: 722, + 76: 611, + 77: 833, + 78: 722, + 79: 778, + 80: 667, + 81: 778, + 82: 722, + 83: 667, + 84: 611, + 85: 722, + 86: 667, + 87: 944, + 88: 667, + 89: 667, + 90: 611, + 91: 333, + 92: 278, + 93: 333, + 94: 584, + 95: 556, + 96: 333, + 97: 556, + 98: 611, + 99: 556, + 100: 611, + 101: 556, + 102: 333, + 103: 611, + 104: 611, + 105: 278, + 106: 278, + 107: 556, + 108: 278, + 109: 889, + 110: 611, + 111: 611, + 112: 611, + 113: 611, + 114: 389, + 115: 556, + 116: 333, + 117: 611, + 118: 556, + 119: 778, + 120: 556, + 121: 556, + 122: 500, + 123: 389, + 124: 280, + 125: 389, + 126: 584, + 161: 333, + 162: 556, + 163: 556, + 164: 556, + 165: 556, + 166: 280, + 167: 556, + 168: 333, + 169: 737, + 170: 370, + 171: 556, + 172: 584, + 174: 737, + 175: 333, + 176: 400, + 177: 584, + 178: 333, + 179: 333, + 180: 333, + 181: 611, + 182: 556, + 183: 278, + 184: 333, + 185: 333, + 186: 365, + 187: 556, + 188: 834, + 189: 834, + 190: 834, + 191: 611, + 192: 722, + 193: 722, + 194: 722, + 195: 722, + 196: 722, + 197: 722, + 198: 1000, + 199: 722, + 200: 667, + 201: 667, + 202: 667, + 203: 667, + 204: 278, + 205: 278, + 206: 278, + 207: 278, + 208: 722, + 209: 722, + 210: 778, + 211: 778, + 212: 778, + 213: 778, + 214: 778, + 215: 584, + 216: 778, + 217: 722, + 218: 722, + 219: 722, + 220: 722, + 221: 667, + 222: 667, + 223: 611, + 224: 556, + 225: 556, + 226: 556, + 227: 556, + 228: 556, + 229: 556, + 230: 889, + 231: 556, + 232: 556, + 233: 556, + 234: 556, + 235: 556, + 236: 278, + 237: 278, + 238: 278, + 239: 278, + 240: 611, + 241: 611, + 242: 611, + 243: 611, + 244: 611, + 245: 611, + 246: 611, + 247: 584, + 248: 611, + 249: 611, + 250: 611, + 251: 611, + 252: 611, + 253: 556, + 254: 611, + 255: 556, + }, + "Times-Roman": { + 32: 250, + 33: 333, + 34: 408, + 35: 500, + 36: 500, + 37: 833, + 38: 778, + 39: 180, + 40: 333, + 41: 333, + 42: 500, + 43: 564, + 44: 250, + 45: 333, + 46: 250, + 47: 278, + 48: 500, + 49: 500, + 50: 500, + 51: 500, + 52: 500, + 53: 500, + 54: 500, + 55: 500, + 56: 500, + 57: 500, + 58: 278, + 59: 278, + 60: 564, + 61: 564, + 62: 564, + 63: 444, + 64: 921, + 65: 722, + 66: 667, + 67: 667, + 68: 722, + 69: 611, + 70: 556, + 71: 722, + 72: 722, + 73: 333, + 74: 389, + 75: 722, + 76: 611, + 77: 889, + 78: 722, + 79: 722, + 80: 556, + 81: 722, + 82: 667, + 83: 556, + 84: 611, + 85: 722, + 86: 722, + 87: 944, + 88: 722, + 89: 722, + 90: 611, + 91: 333, + 92: 278, + 93: 333, + 94: 469, + 95: 500, + 96: 333, + 97: 444, + 98: 500, + 99: 444, + 100: 500, + 101: 444, + 102: 333, + 103: 500, + 104: 500, + 105: 278, + 106: 278, + 107: 500, + 108: 278, + 109: 778, + 110: 500, + 111: 500, + 112: 500, + 113: 500, + 114: 333, + 115: 389, + 116: 278, + 117: 500, + 118: 500, + 119: 722, + 120: 500, + 121: 500, + 122: 444, + 123: 480, + 124: 200, + 125: 480, + 126: 541, + 161: 333, + 162: 500, + 163: 500, + 164: 500, + 165: 500, + 166: 200, + 167: 500, + 168: 333, + 169: 760, + 170: 276, + 171: 500, + 172: 564, + 174: 760, + 175: 333, + 176: 400, + 177: 564, + 178: 300, + 179: 300, + 180: 333, + 181: 500, + 182: 453, + 183: 250, + 184: 333, + 185: 300, + 186: 310, + 187: 500, + 188: 750, + 189: 750, + 190: 750, + 191: 444, + 192: 722, + 193: 722, + 194: 722, + 195: 722, + 196: 722, + 197: 722, + 198: 889, + 199: 667, + 200: 611, + 201: 611, + 202: 611, + 203: 611, + 204: 333, + 205: 333, + 206: 333, + 207: 333, + 208: 722, + 209: 722, + 210: 722, + 211: 722, + 212: 722, + 213: 722, + 214: 722, + 215: 564, + 216: 722, + 217: 722, + 218: 722, + 219: 722, + 220: 722, + 221: 722, + 222: 556, + 223: 500, + 224: 444, + 225: 444, + 226: 444, + 227: 444, + 228: 444, + 229: 444, + 230: 667, + 231: 444, + 232: 444, + 233: 444, + 234: 444, + 235: 444, + 236: 278, + 237: 278, + 238: 278, + 239: 278, + 240: 500, + 241: 500, + 242: 500, + 243: 500, + 244: 500, + 245: 500, + 246: 500, + 247: 564, + 248: 500, + 249: 500, + 250: 500, + 251: 500, + 252: 500, + 253: 500, + 254: 500, + 255: 500, + }, + "Times-Bold": { + 32: 250, + 33: 333, + 34: 555, + 35: 500, + 36: 500, + 37: 1000, + 38: 833, + 39: 278, + 40: 333, + 41: 333, + 42: 500, + 43: 570, + 44: 250, + 45: 333, + 46: 250, + 47: 278, + 48: 500, + 49: 500, + 50: 500, + 51: 500, + 52: 500, + 53: 500, + 54: 500, + 55: 500, + 56: 500, + 57: 500, + 58: 333, + 59: 333, + 60: 570, + 61: 570, + 62: 570, + 63: 500, + 64: 930, + 65: 722, + 66: 667, + 67: 722, + 68: 722, + 69: 667, + 70: 611, + 71: 778, + 72: 778, + 73: 389, + 74: 500, + 75: 778, + 76: 667, + 77: 944, + 78: 722, + 79: 778, + 80: 611, + 81: 778, + 82: 722, + 83: 556, + 84: 667, + 85: 722, + 86: 722, + 87: 1000, + 88: 722, + 89: 722, + 90: 667, + 91: 333, + 92: 278, + 93: 333, + 94: 581, + 95: 500, + 96: 333, + 97: 500, + 98: 556, + 99: 444, + 100: 556, + 101: 444, + 102: 333, + 103: 500, + 104: 556, + 105: 278, + 106: 333, + 107: 556, + 108: 278, + 109: 833, + 110: 556, + 111: 500, + 112: 556, + 113: 556, + 114: 444, + 115: 389, + 116: 333, + 117: 556, + 118: 500, + 119: 722, + 120: 500, + 121: 500, + 122: 444, + 123: 394, + 124: 220, + 125: 394, + 126: 520, + 161: 333, + 162: 500, + 163: 500, + 164: 500, + 165: 500, + 166: 220, + 167: 500, + 168: 333, + 169: 747, + 170: 300, + 171: 500, + 172: 570, + 174: 747, + 175: 333, + 176: 400, + 177: 570, + 178: 300, + 179: 300, + 180: 333, + 181: 556, + 182: 540, + 183: 250, + 184: 333, + 185: 300, + 186: 330, + 187: 500, + 188: 750, + 189: 750, + 190: 750, + 191: 500, + 192: 722, + 193: 722, + 194: 722, + 195: 722, + 196: 722, + 197: 722, + 198: 1000, + 199: 722, + 200: 667, + 201: 667, + 202: 667, + 203: 667, + 204: 389, + 205: 389, + 206: 389, + 207: 389, + 208: 722, + 209: 722, + 210: 778, + 211: 778, + 212: 778, + 213: 778, + 214: 778, + 215: 570, + 216: 778, + 217: 722, + 218: 722, + 219: 722, + 220: 722, + 221: 722, + 222: 611, + 223: 556, + 224: 500, + 225: 500, + 226: 500, + 227: 500, + 228: 500, + 229: 500, + 230: 722, + 231: 444, + 232: 444, + 233: 444, + 234: 444, + 235: 444, + 236: 278, + 237: 278, + 238: 278, + 239: 278, + 240: 500, + 241: 556, + 242: 500, + 243: 500, + 244: 500, + 245: 500, + 246: 500, + 247: 570, + 248: 500, + 249: 556, + 250: 556, + 251: 556, + 252: 556, + 253: 500, + 254: 556, + 255: 500, + }, + "Times-Italic": { + 32: 250, + 33: 333, + 34: 420, + 35: 500, + 36: 500, + 37: 833, + 38: 778, + 39: 214, + 40: 333, + 41: 333, + 42: 500, + 43: 675, + 44: 250, + 45: 333, + 46: 250, + 47: 278, + 48: 500, + 49: 500, + 50: 500, + 51: 500, + 52: 500, + 53: 500, + 54: 500, + 55: 500, + 56: 500, + 57: 500, + 58: 333, + 59: 333, + 60: 675, + 61: 675, + 62: 675, + 63: 500, + 64: 920, + 65: 611, + 66: 611, + 67: 667, + 68: 722, + 69: 611, + 70: 611, + 71: 722, + 72: 722, + 73: 333, + 74: 444, + 75: 667, + 76: 556, + 77: 833, + 78: 667, + 79: 722, + 80: 611, + 81: 722, + 82: 611, + 83: 500, + 84: 556, + 85: 722, + 86: 611, + 87: 833, + 88: 611, + 89: 556, + 90: 556, + 91: 389, + 92: 278, + 93: 389, + 94: 422, + 95: 500, + 96: 333, + 97: 500, + 98: 500, + 99: 444, + 100: 500, + 101: 444, + 102: 278, + 103: 500, + 104: 500, + 105: 278, + 106: 278, + 107: 444, + 108: 278, + 109: 722, + 110: 500, + 111: 500, + 112: 500, + 113: 500, + 114: 389, + 115: 389, + 116: 278, + 117: 500, + 118: 444, + 119: 667, + 120: 444, + 121: 444, + 122: 389, + 123: 400, + 124: 275, + 125: 400, + 126: 541, + 161: 389, + 162: 500, + 163: 500, + 164: 500, + 165: 500, + 166: 275, + 167: 500, + 168: 333, + 169: 760, + 170: 276, + 171: 500, + 172: 675, + 174: 760, + 175: 333, + 176: 400, + 177: 675, + 178: 300, + 179: 300, + 180: 333, + 181: 500, + 182: 523, + 183: 250, + 184: 333, + 185: 300, + 186: 310, + 187: 500, + 188: 750, + 189: 750, + 190: 750, + 191: 500, + 192: 611, + 193: 611, + 194: 611, + 195: 611, + 196: 611, + 197: 611, + 198: 889, + 199: 667, + 200: 611, + 201: 611, + 202: 611, + 203: 611, + 204: 333, + 205: 333, + 206: 333, + 207: 333, + 208: 722, + 209: 667, + 210: 722, + 211: 722, + 212: 722, + 213: 722, + 214: 722, + 215: 675, + 216: 722, + 217: 722, + 218: 722, + 219: 722, + 220: 722, + 221: 556, + 222: 611, + 223: 500, + 224: 500, + 225: 500, + 226: 500, + 227: 500, + 228: 500, + 229: 500, + 230: 667, + 231: 444, + 232: 444, + 233: 444, + 234: 444, + 235: 444, + 236: 278, + 237: 278, + 238: 278, + 239: 278, + 240: 500, + 241: 500, + 242: 500, + 243: 500, + 244: 500, + 245: 500, + 246: 500, + 247: 675, + 248: 500, + 249: 500, + 250: 500, + 251: 500, + 252: 500, + 253: 444, + 254: 500, + 255: 444, + }, + "Times-BoldItalic": { + 32: 250, + 33: 389, + 34: 555, + 35: 500, + 36: 500, + 37: 833, + 38: 778, + 39: 278, + 40: 333, + 41: 333, + 42: 500, + 43: 570, + 44: 250, + 45: 333, + 46: 250, + 47: 278, + 48: 500, + 49: 500, + 50: 500, + 51: 500, + 52: 500, + 53: 500, + 54: 500, + 55: 500, + 56: 500, + 57: 500, + 58: 333, + 59: 333, + 60: 570, + 61: 570, + 62: 570, + 63: 500, + 64: 832, + 65: 667, + 66: 667, + 67: 667, + 68: 722, + 69: 667, + 70: 667, + 71: 722, + 72: 778, + 73: 389, + 74: 500, + 75: 667, + 76: 611, + 77: 889, + 78: 722, + 79: 722, + 80: 611, + 81: 722, + 82: 667, + 83: 556, + 84: 611, + 85: 722, + 86: 667, + 87: 889, + 88: 667, + 89: 611, + 90: 611, + 91: 333, + 92: 278, + 93: 333, + 94: 570, + 95: 500, + 96: 333, + 97: 500, + 98: 500, + 99: 444, + 100: 500, + 101: 444, + 102: 333, + 103: 500, + 104: 556, + 105: 278, + 106: 278, + 107: 500, + 108: 278, + 109: 778, + 110: 556, + 111: 500, + 112: 500, + 113: 500, + 114: 389, + 115: 389, + 116: 278, + 117: 556, + 118: 444, + 119: 667, + 120: 500, + 121: 444, + 122: 389, + 123: 348, + 124: 220, + 125: 348, + 126: 570, + 161: 389, + 162: 500, + 163: 500, + 164: 500, + 165: 500, + 166: 220, + 167: 500, + 168: 333, + 169: 747, + 170: 266, + 171: 500, + 172: 606, + 174: 747, + 175: 333, + 176: 400, + 177: 570, + 178: 300, + 179: 300, + 180: 333, + 181: 576, + 182: 500, + 183: 250, + 184: 333, + 185: 300, + 186: 300, + 187: 500, + 188: 750, + 189: 750, + 190: 750, + 191: 500, + 192: 667, + 193: 667, + 194: 667, + 195: 667, + 196: 667, + 197: 667, + 198: 944, + 199: 667, + 200: 667, + 201: 667, + 202: 667, + 203: 667, + 204: 389, + 205: 389, + 206: 389, + 207: 389, + 208: 722, + 209: 722, + 210: 722, + 211: 722, + 212: 722, + 213: 722, + 214: 722, + 215: 570, + 216: 722, + 217: 722, + 218: 722, + 219: 722, + 220: 722, + 221: 611, + 222: 611, + 223: 500, + 224: 500, + 225: 500, + 226: 500, + 227: 500, + 228: 500, + 229: 500, + 230: 722, + 231: 444, + 232: 444, + 233: 444, + 234: 444, + 235: 444, + 236: 278, + 237: 278, + 238: 278, + 239: 278, + 240: 500, + 241: 556, + 242: 500, + 243: 500, + 244: 500, + 245: 500, + 246: 500, + 247: 570, + 248: 500, + 249: 556, + 250: 556, + 251: 556, + 252: 556, + 253: 444, + 254: 500, + 255: 444, + }, + "Courier": { + 32: 600, + 33: 600, + 34: 600, + 35: 600, + 36: 600, + 37: 600, + 38: 600, + 39: 600, + 40: 600, + 41: 600, + 42: 600, + 43: 600, + 44: 600, + 45: 600, + 46: 600, + 47: 600, + 48: 600, + 49: 600, + 50: 600, + 51: 600, + 52: 600, + 53: 600, + 54: 600, + 55: 600, + 56: 600, + 57: 600, + 58: 600, + 59: 600, + 60: 600, + 61: 600, + 62: 600, + 63: 600, + 64: 600, + 65: 600, + 66: 600, + 67: 600, + 68: 600, + 69: 600, + 70: 600, + 71: 600, + 72: 600, + 73: 600, + 74: 600, + 75: 600, + 76: 600, + 77: 600, + 78: 600, + 79: 600, + 80: 600, + 81: 600, + 82: 600, + 83: 600, + 84: 600, + 85: 600, + 86: 600, + 87: 600, + 88: 600, + 89: 600, + 90: 600, + 91: 600, + 92: 600, + 93: 600, + 94: 600, + 95: 600, + 96: 600, + 97: 600, + 98: 600, + 99: 600, + 100: 600, + 101: 600, + 102: 600, + 103: 600, + 104: 600, + 105: 600, + 106: 600, + 107: 600, + 108: 600, + 109: 600, + 110: 600, + 111: 600, + 112: 600, + 113: 600, + 114: 600, + 115: 600, + 116: 600, + 117: 600, + 118: 600, + 119: 600, + 120: 600, + 121: 600, + 122: 600, + 123: 600, + 124: 600, + 125: 600, + 126: 600, + 161: 600, + 162: 600, + 163: 600, + 164: 600, + 165: 600, + 166: 600, + 167: 600, + 168: 600, + 169: 600, + 170: 600, + 171: 600, + 172: 600, + 174: 600, + 175: 600, + 176: 600, + 177: 600, + 178: 600, + 179: 600, + 180: 600, + 181: 600, + 182: 600, + 183: 600, + 184: 600, + 185: 600, + 186: 600, + 187: 600, + 188: 600, + 189: 600, + 190: 600, + 191: 600, + 192: 600, + 193: 600, + 194: 600, + 195: 600, + 196: 600, + 197: 600, + 198: 600, + 199: 600, + 200: 600, + 201: 600, + 202: 600, + 203: 600, + 204: 600, + 205: 600, + 206: 600, + 207: 600, + 208: 600, + 209: 600, + 210: 600, + 211: 600, + 212: 600, + 213: 600, + 214: 600, + 215: 600, + 216: 600, + 217: 600, + 218: 600, + 219: 600, + 220: 600, + 221: 600, + 222: 600, + 223: 600, + 224: 600, + 225: 600, + 226: 600, + 227: 600, + 228: 600, + 229: 600, + 230: 600, + 231: 600, + 232: 600, + 233: 600, + 234: 600, + 235: 600, + 236: 600, + 237: 600, + 238: 600, + 239: 600, + 240: 600, + 241: 600, + 242: 600, + 243: 600, + 244: 600, + 245: 600, + 246: 600, + 247: 600, + 248: 600, + 249: 600, + 250: 600, + 251: 600, + 252: 600, + 253: 600, + 254: 600, + 255: 600, + }, + "Courier-Bold": { + 32: 600, + 33: 600, + 34: 600, + 35: 600, + 36: 600, + 37: 600, + 38: 600, + 39: 600, + 40: 600, + 41: 600, + 42: 600, + 43: 600, + 44: 600, + 45: 600, + 46: 600, + 47: 600, + 48: 600, + 49: 600, + 50: 600, + 51: 600, + 52: 600, + 53: 600, + 54: 600, + 55: 600, + 56: 600, + 57: 600, + 58: 600, + 59: 600, + 60: 600, + 61: 600, + 62: 600, + 63: 600, + 64: 600, + 65: 600, + 66: 600, + 67: 600, + 68: 600, + 69: 600, + 70: 600, + 71: 600, + 72: 600, + 73: 600, + 74: 600, + 75: 600, + 76: 600, + 77: 600, + 78: 600, + 79: 600, + 80: 600, + 81: 600, + 82: 600, + 83: 600, + 84: 600, + 85: 600, + 86: 600, + 87: 600, + 88: 600, + 89: 600, + 90: 600, + 91: 600, + 92: 600, + 93: 600, + 94: 600, + 95: 600, + 96: 600, + 97: 600, + 98: 600, + 99: 600, + 100: 600, + 101: 600, + 102: 600, + 103: 600, + 104: 600, + 105: 600, + 106: 600, + 107: 600, + 108: 600, + 109: 600, + 110: 600, + 111: 600, + 112: 600, + 113: 600, + 114: 600, + 115: 600, + 116: 600, + 117: 600, + 118: 600, + 119: 600, + 120: 600, + 121: 600, + 122: 600, + 123: 600, + 124: 600, + 125: 600, + 126: 600, + 161: 600, + 162: 600, + 163: 600, + 164: 600, + 165: 600, + 166: 600, + 167: 600, + 168: 600, + 169: 600, + 170: 600, + 171: 600, + 172: 600, + 174: 600, + 175: 600, + 176: 600, + 177: 600, + 178: 600, + 179: 600, + 180: 600, + 181: 600, + 182: 600, + 183: 600, + 184: 600, + 185: 600, + 186: 600, + 187: 600, + 188: 600, + 189: 600, + 190: 600, + 191: 600, + 192: 600, + 193: 600, + 194: 600, + 195: 600, + 196: 600, + 197: 600, + 198: 600, + 199: 600, + 200: 600, + 201: 600, + 202: 600, + 203: 600, + 204: 600, + 205: 600, + 206: 600, + 207: 600, + 208: 600, + 209: 600, + 210: 600, + 211: 600, + 212: 600, + 213: 600, + 214: 600, + 215: 600, + 216: 600, + 217: 600, + 218: 600, + 219: 600, + 220: 600, + 221: 600, + 222: 600, + 223: 600, + 224: 600, + 225: 600, + 226: 600, + 227: 600, + 228: 600, + 229: 600, + 230: 600, + 231: 600, + 232: 600, + 233: 600, + 234: 600, + 235: 600, + 236: 600, + 237: 600, + 238: 600, + 239: 600, + 240: 600, + 241: 600, + 242: 600, + 243: 600, + 244: 600, + 245: 600, + 246: 600, + 247: 600, + 248: 600, + 249: 600, + 250: 600, + 251: 600, + 252: 600, + 253: 600, + 254: 600, + 255: 600, + }, + "Courier-Oblique": { + 32: 600, + 33: 600, + 34: 600, + 35: 600, + 36: 600, + 37: 600, + 38: 600, + 39: 600, + 40: 600, + 41: 600, + 42: 600, + 43: 600, + 44: 600, + 45: 600, + 46: 600, + 47: 600, + 48: 600, + 49: 600, + 50: 600, + 51: 600, + 52: 600, + 53: 600, + 54: 600, + 55: 600, + 56: 600, + 57: 600, + 58: 600, + 59: 600, + 60: 600, + 61: 600, + 62: 600, + 63: 600, + 64: 600, + 65: 600, + 66: 600, + 67: 600, + 68: 600, + 69: 600, + 70: 600, + 71: 600, + 72: 600, + 73: 600, + 74: 600, + 75: 600, + 76: 600, + 77: 600, + 78: 600, + 79: 600, + 80: 600, + 81: 600, + 82: 600, + 83: 600, + 84: 600, + 85: 600, + 86: 600, + 87: 600, + 88: 600, + 89: 600, + 90: 600, + 91: 600, + 92: 600, + 93: 600, + 94: 600, + 95: 600, + 96: 600, + 97: 600, + 98: 600, + 99: 600, + 100: 600, + 101: 600, + 102: 600, + 103: 600, + 104: 600, + 105: 600, + 106: 600, + 107: 600, + 108: 600, + 109: 600, + 110: 600, + 111: 600, + 112: 600, + 113: 600, + 114: 600, + 115: 600, + 116: 600, + 117: 600, + 118: 600, + 119: 600, + 120: 600, + 121: 600, + 122: 600, + 123: 600, + 124: 600, + 125: 600, + 126: 600, + 161: 600, + 162: 600, + 163: 600, + 164: 600, + 165: 600, + 166: 600, + 167: 600, + 168: 600, + 169: 600, + 170: 600, + 171: 600, + 172: 600, + 174: 600, + 175: 600, + 176: 600, + 177: 600, + 178: 600, + 179: 600, + 180: 600, + 181: 600, + 182: 600, + 183: 600, + 184: 600, + 185: 600, + 186: 600, + 187: 600, + 188: 600, + 189: 600, + 190: 600, + 191: 600, + 192: 600, + 193: 600, + 194: 600, + 195: 600, + 196: 600, + 197: 600, + 198: 600, + 199: 600, + 200: 600, + 201: 600, + 202: 600, + 203: 600, + 204: 600, + 205: 600, + 206: 600, + 207: 600, + 208: 600, + 209: 600, + 210: 600, + 211: 600, + 212: 600, + 213: 600, + 214: 600, + 215: 600, + 216: 600, + 217: 600, + 218: 600, + 219: 600, + 220: 600, + 221: 600, + 222: 600, + 223: 600, + 224: 600, + 225: 600, + 226: 600, + 227: 600, + 228: 600, + 229: 600, + 230: 600, + 231: 600, + 232: 600, + 233: 600, + 234: 600, + 235: 600, + 236: 600, + 237: 600, + 238: 600, + 239: 600, + 240: 600, + 241: 600, + 242: 600, + 243: 600, + 244: 600, + 245: 600, + 246: 600, + 247: 600, + 248: 600, + 249: 600, + 250: 600, + 251: 600, + 252: 600, + 253: 600, + 254: 600, + 255: 600, + }, + "Courier-BoldOblique": { + 32: 600, + 33: 600, + 34: 600, + 35: 600, + 36: 600, + 37: 600, + 38: 600, + 39: 600, + 40: 600, + 41: 600, + 42: 600, + 43: 600, + 44: 600, + 45: 600, + 46: 600, + 47: 600, + 48: 600, + 49: 600, + 50: 600, + 51: 600, + 52: 600, + 53: 600, + 54: 600, + 55: 600, + 56: 600, + 57: 600, + 58: 600, + 59: 600, + 60: 600, + 61: 600, + 62: 600, + 63: 600, + 64: 600, + 65: 600, + 66: 600, + 67: 600, + 68: 600, + 69: 600, + 70: 600, + 71: 600, + 72: 600, + 73: 600, + 74: 600, + 75: 600, + 76: 600, + 77: 600, + 78: 600, + 79: 600, + 80: 600, + 81: 600, + 82: 600, + 83: 600, + 84: 600, + 85: 600, + 86: 600, + 87: 600, + 88: 600, + 89: 600, + 90: 600, + 91: 600, + 92: 600, + 93: 600, + 94: 600, + 95: 600, + 96: 600, + 97: 600, + 98: 600, + 99: 600, + 100: 600, + 101: 600, + 102: 600, + 103: 600, + 104: 600, + 105: 600, + 106: 600, + 107: 600, + 108: 600, + 109: 600, + 110: 600, + 111: 600, + 112: 600, + 113: 600, + 114: 600, + 115: 600, + 116: 600, + 117: 600, + 118: 600, + 119: 600, + 120: 600, + 121: 600, + 122: 600, + 123: 600, + 124: 600, + 125: 600, + 126: 600, + 161: 600, + 162: 600, + 163: 600, + 164: 600, + 165: 600, + 166: 600, + 167: 600, + 168: 600, + 169: 600, + 170: 600, + 171: 600, + 172: 600, + 174: 600, + 175: 600, + 176: 600, + 177: 600, + 178: 600, + 179: 600, + 180: 600, + 181: 600, + 182: 600, + 183: 600, + 184: 600, + 185: 600, + 186: 600, + 187: 600, + 188: 600, + 189: 600, + 190: 600, + 191: 600, + 192: 600, + 193: 600, + 194: 600, + 195: 600, + 196: 600, + 197: 600, + 198: 600, + 199: 600, + 200: 600, + 201: 600, + 202: 600, + 203: 600, + 204: 600, + 205: 600, + 206: 600, + 207: 600, + 208: 600, + 209: 600, + 210: 600, + 211: 600, + 212: 600, + 213: 600, + 214: 600, + 215: 600, + 216: 600, + 217: 600, + 218: 600, + 219: 600, + 220: 600, + 221: 600, + 222: 600, + 223: 600, + 224: 600, + 225: 600, + 226: 600, + 227: 600, + 228: 600, + 229: 600, + 230: 600, + 231: 600, + 232: 600, + 233: 600, + 234: 600, + 235: 600, + 236: 600, + 237: 600, + 238: 600, + 239: 600, + 240: 600, + 241: 600, + 242: 600, + 243: 600, + 244: 600, + 245: 600, + 246: 600, + 247: 600, + 248: 600, + 249: 600, + 250: 600, + 251: 600, + 252: 600, + 253: 600, + 254: 600, + 255: 600, + }, + "Symbol": { + 32: 250, + 33: 333, + 35: 500, + 37: 833, + 38: 778, + 40: 333, + 41: 333, + 43: 549, + 44: 250, + 46: 250, + 47: 278, + 48: 500, + 49: 500, + 50: 500, + 51: 500, + 52: 500, + 53: 500, + 54: 500, + 55: 500, + 56: 500, + 57: 500, + 58: 278, + 59: 278, + 60: 549, + 61: 549, + 62: 549, + 63: 444, + 91: 333, + 93: 333, + 95: 500, + 123: 480, + 124: 200, + 125: 480, + 172: 713, + 176: 400, + 177: 549, + 181: 576, + 215: 549, + 247: 549, + }, + "ZapfDingbats": { + 1: 974, + 2: 961, + 3: 980, + 4: 719, + 5: 789, + 6: 494, + 7: 552, + 8: 537, + 9: 577, + 10: 692, + 11: 960, + 12: 939, + 13: 549, + 14: 855, + 15: 911, + 16: 933, + 17: 945, + 18: 974, + 19: 755, + 20: 846, + 21: 762, + 22: 761, + 23: 571, + 24: 677, + 25: 763, + 26: 760, + 27: 759, + 28: 754, + 29: 786, + 30: 788, + 31: 788, + 32: 790, + 33: 793, + 34: 794, + 35: 816, + 36: 823, + 37: 789, + 38: 841, + 39: 823, + 40: 833, + 41: 816, + 42: 831, + 43: 923, + 44: 744, + 45: 723, + 46: 749, + 47: 790, + 48: 792, + 49: 695, + 50: 776, + 51: 768, + 52: 792, + 53: 759, + 54: 707, + 55: 708, + 56: 682, + 57: 701, + 58: 826, + 59: 815, + 60: 789, + 61: 789, + 62: 707, + 63: 687, + 64: 696, + 65: 689, + 66: 786, + 67: 787, + 68: 713, + 69: 791, + 70: 785, + 71: 791, + 72: 873, + 73: 761, + 74: 762, + 75: 759, + 76: 892, + 77: 892, + 78: 788, + 79: 784, + 81: 438, + 82: 138, + 83: 277, + 84: 415, + 85: 509, + 86: 410, + 87: 234, + 88: 234, + 89: 390, + 90: 390, + 91: 276, + 92: 276, + 93: 317, + 94: 317, + 95: 334, + 96: 334, + 97: 392, + 98: 392, + 99: 668, + 100: 668, + 101: 732, + 102: 544, + 103: 544, + 104: 910, + 105: 911, + 106: 667, + 107: 760, + 108: 760, + 109: 626, + 110: 694, + 111: 595, + 112: 776, + 117: 690, + 118: 791, + 119: 790, + 120: 788, + 121: 788, + 122: 788, + 123: 788, + 124: 788, + 125: 788, + 126: 788, + 127: 788, + 128: 788, + 129: 788, + 130: 788, + 131: 788, + 132: 788, + 133: 788, + 134: 788, + 135: 788, + 136: 788, + 137: 788, + 138: 788, + 139: 788, + 140: 788, + 141: 788, + 142: 788, + 143: 788, + 144: 788, + 145: 788, + 146: 788, + 147: 788, + 148: 788, + 149: 788, + 150: 788, + 151: 788, + 152: 788, + 153: 788, + 154: 788, + 155: 788, + 156: 788, + 157: 788, + 158: 788, + 159: 788, + 160: 894, + 161: 838, + 162: 924, + 163: 1016, + 164: 458, + 165: 924, + 166: 918, + 167: 927, + 168: 928, + 169: 928, + 170: 834, + 171: 873, + 172: 828, + 173: 924, + 174: 917, + 175: 930, + 176: 931, + 177: 463, + 178: 883, + 179: 836, + 180: 867, + 181: 696, + 182: 874, + 183: 760, + 184: 946, + 185: 865, + 186: 967, + 187: 831, + 188: 873, + 189: 927, + 190: 970, + 191: 918, + 192: 748, + 193: 836, + 194: 771, + 195: 888, + 196: 748, + 197: 771, + 198: 888, + 199: 867, + 200: 696, + 201: 874, + 202: 974, + 203: 762, + 204: 759, + 205: 509, + 206: 410, + }, +} diff --git a/pdftotext/pdftotext.go b/pdftotext/pdftotext.go index 77dff5f..76baa4f 100644 --- a/pdftotext/pdftotext.go +++ b/pdftotext/pdftotext.go @@ -8,7 +8,9 @@ import ( "encoding/hex" "fmt" "io" + "math" "os" + "sort" "strconv" "strings" @@ -20,7 +22,6 @@ import ( "fileganizer/logger" ) -// pdfToken kind constants. const ( tokName = 'n' tokStr = 's' @@ -28,18 +29,28 @@ const ( tokArr = 'a' tokKw = 'k' tokNum = 'N' + + maxByte = 255 + spaceCID = 32 + + emScale = 1000.0 + wordGapRatio = 0.09 + lineGroupTol = 2.0 + maxYDistLines = 4.0 ) -// maxByte is the maximum value an uint8 can hold, used for bounds checking -// when converting parsed octal escape values from PDF literal strings. -const maxByte = 255 +type fontWidths [256]uint16 type pdfToken struct { kind byte raw string } -// contentScanner tokenizes a PDF content stream. +type positionedChar struct { + x0, y0, x1, y1 float64 + text string +} + type contentScanner struct { data []byte pos int @@ -178,10 +189,12 @@ func (s *contentScanner) readKeyword() (pdfToken, bool) { } s.pos++ } + if s.pos == start { + s.pos++ + } return pdfToken{kind: tokKw, raw: string(s.data[start:s.pos])}, true } -// parseLiteralString unescapes a PDF literal string. func parseLiteralString(s string) string { if len(s) < 2 { return s @@ -198,8 +211,6 @@ func parseLiteralString(s string) string { return b.String() } -// writeEscaped handles a backslash escape sequence at position i in s, -// writing the decoded byte to b. It returns the updated index. func writeEscaped(b *strings.Builder, s string, i int) int { i++ switch s[i] { @@ -231,7 +242,6 @@ func writeEscaped(b *strings.Builder, s string, i int) int { return i } -// parseHexString decodes a PDF hex string. func parseHexString(s string) []byte { if len(s) < 2 { return nil @@ -248,7 +258,6 @@ func parseHexString(s string) []byte { return dst[:n] } -// toUnicodeMap parses a ToUnicode CMap and returns a CID->rune mapping. func toUnicodeMap(data []byte) map[uint16]rune { m := make(map[uint16]rune) s := &contentScanner{data: data} @@ -292,10 +301,10 @@ func parseBFCharInto(s *contentScanner, m map[uint16]rune) { code = []byte(parseLiteralString(tok.raw)) } if len(srcCID) >= 2 && len(code) >= 2 { - cid := uint16(srcCID[0])<<8 | uint16(srcCID[1]) - m[cid] = rune(uint16(code[0])<<8 | uint16(code[1])) + cid := cidToUint16(srcCID) + m[cid] = rune(cidToUint16(code)) } else if len(srcCID) >= 2 && len(code) >= 1 { - cid := uint16(srcCID[0])<<8 | uint16(srcCID[1]) + cid := cidToUint16(srcCID) m[cid] = rune(code[0]) } } @@ -334,11 +343,11 @@ func parseBFRangeInto(s *contentScanner, m map[uint16]rune) { func addBFRangeContinuous(m map[uint16]rune, startCID, endCID, val []byte) { if len(startCID) >= 2 && len(endCID) >= 2 && len(val) >= 1 { - lo := uint16(startCID[0])<<8 | uint16(startCID[1]) - hi := uint16(endCID[0])<<8 | uint16(endCID[1]) + lo := cidToUint16(startCID) + hi := cidToUint16(endCID) var baseRune rune if len(val) >= 2 { - baseRune = rune(uint16(val[0])<<8 | uint16(val[1])) + baseRune = rune(cidToUint16(val)) } else { baseRune = rune(val[0]) } @@ -360,11 +369,11 @@ func addBFRangeList(s *contentScanner, m map[uint16]rune, startCID []byte) { } } if len(startCID) >= 2 { - lo := uint16(startCID[0])<<8 | uint16(startCID[1]) + lo := cidToUint16(startCID) for i, val := range vals { cid := lo + uint16(i) if len(val) >= 2 { - m[cid] = rune(uint16(val[0])<<8 | uint16(val[1])) + m[cid] = rune(cidToUint16(val)) } else if len(val) >= 1 { m[cid] = rune(val[0]) } @@ -372,7 +381,10 @@ func addBFRangeList(s *contentScanner, m map[uint16]rune, startCID []byte) { } } -// decodeText decodes CID bytes using a CID→rune map. +func cidToUint16(b []byte) uint16 { + return uint16(b[0])<<8 | uint16(b[1]) +} + func decodeText(data []byte, cmap map[uint16]rune) string { var b strings.Builder for i := 0; i < len(data); { @@ -391,19 +403,194 @@ func decodeText(data []byte, cmap map[uint16]rune) string { continue } } - b.WriteByte(data[i]) + b.WriteRune(rune(data[i])) i++ } return b.String() } -// textFromContentStream parses a PDF content stream and extracts text strings -// using the provided font ToUnicode maps (fontResourceName -> CID→rune). -func textFromContentStream(content []byte, fontCMaps map[string]map[uint16]rune) string { +// multMatrices multiplies two 6-element PDF transformation matrices: a × b. +func multMatrices(a, b [6]float64) [6]float64 { + return [6]float64{ + a[0]*b[0] + a[1]*b[2], + a[0]*b[1] + a[1]*b[3], + a[2]*b[0] + a[3]*b[2], + a[2]*b[1] + a[3]*b[3], + a[4]*b[0] + a[5]*b[2] + b[4], + a[4]*b[1] + a[5]*b[3] + b[5], + } +} + +// applyTd translates the text matrix: Tm' = [1 0 0 1 tx ty] × Tm. +func applyTd(tm [6]float64, tx, ty float64) [6]float64 { + return [6]float64{ + tm[0], tm[1], tm[2], tm[3], + tx*tm[0] + ty*tm[2] + tm[4], + tx*tm[1] + ty*tm[3] + tm[5], + } +} + +// textRenderPos returns the page-space position (from CTM × Tm). +func textRenderPos(ctm, tm [6]float64) (x, y float64) { + m := multMatrices(ctm, tm) + return m[4], m[5] +} + +// charWidth returns the width of a glyph CID in text space units. +func charWidth(fw *fontWidths, cid byte, fontSize float64) float64 { + if fw != nil && int(cid) < len(fw) && fw[cid] > 0 { + return float64(fw[cid]) / emScale * fontSize + } + return fontSize * 0.5 //nolint:mnd +} + +// groupCharsIntoLines groups positioned characters into lines by Y proximity. +func groupCharsIntoLines(chars []positionedChar, lineTol float64) [][]positionedChar { + if len(chars) == 0 { + return nil + } + + sorted := make([]positionedChar, len(chars)) + copy(sorted, chars) + sort.Slice(sorted, func(i, j int) bool { + if math.Abs(sorted[i].y0-sorted[j].y0) > lineTol { + return sorted[i].y0 > sorted[j].y0 + } + return sorted[i].x0 < sorted[j].x0 + }) + + var lines [][]positionedChar + var cur []positionedChar + curY := sorted[0].y0 + + for _, c := range sorted { + if math.Abs(c.y0-curY) > lineTol { + lines = append(lines, cur) + cur = nil + curY = c.y0 + } + cur = append(cur, c) + } + if len(cur) > 0 { + lines = append(lines, cur) + } + + return lines +} + +func renderLines(lines [][]positionedChar, wordRatio float64) string { + var out strings.Builder + for li, line := range lines { + if li > 0 { + out.WriteByte('\n') + } + sort.Slice(line, func(i, j int) bool { + return line[i].x0 < line[j].x0 + }) + for ci, c := range line { + if ci > 0 { + gap := c.x0 - line[ci-1].x1 + charSize := c.y0 - c.y1 + if charSize < 0 { + charSize = -charSize + } + if gap > charSize*wordRatio { + out.WriteByte(' ') + } + } + out.WriteString(c.text) + } + } + return out.String() +} + +// textFromContentStream parses a PDF content stream and extracts text with +// position tracking, then groups characters into lines geometrically. +func textFromContentStream( //nolint:gocyclo,funlen + content []byte, fontCMaps map[string]map[uint16]rune, fWidths map[string]*fontWidths, +) string { s := &contentScanner{data: content} var stack []pdfToken var currentFont string - var out strings.Builder + var fontSize float64 + + ctm := [6]float64{1, 0, 0, 1, 0, 0} + var ctmStack [][6]float64 + tm := [6]float64{1, 0, 0, 1, 0, 0} + var textLeading float64 + var cursorX float64 + + var chars []positionedChar + + pushChar := func(text string, curX float64) { + px, py := textRenderPos(ctm, tm) + x0 := px + math.Min(cursorX, curX) + x1 := px + math.Max(cursorX, curX) + if x1-x0 < 0.1 { //nolint:mnd + x1 = x0 + fontSize + } + chars = append(chars, positionedChar{ + x0: x0, + y0: py, + x1: x1, + y1: py - fontSize, + text: text, + }) + } + + decodeAndRender := func(data []byte, cmap map[uint16]rune) { + fw := fWidths[currentFont] + for i := 0; i < len(data); { + var cid byte + var r rune + var advance float64 + + if cmap != nil && i+1 < len(data) { + cidPair := uint16(data[i])<<8 | uint16(data[i+1]) + if r2, ok := cmap[cidPair]; ok { + r = r2 + cid = data[i] + advance = charWidth(fw, cid, fontSize) + pushChar(string(r), cursorX+advance) + cursorX += advance + i += 2 + continue + } + } + cid = data[i] + advance = charWidth(fw, cid, fontSize) + if cmap != nil { + if r2, ok := cmap[uint16(cid)]; ok { + r = r2 + } else { + r = rune(cid) + } + } else { + r = rune(cid) + } + pushChar(string(r), cursorX+advance) + cursorX += advance + i++ + } + } + + renderTJArray := func(seq []pdfToken) { + cmap := fontCMaps[currentFont] + for _, el := range seq { + switch el.kind { + case tokStr: + data := []byte(parseLiteralString(el.raw)) + decodeAndRender(data, cmap) + case tokHex: + data := parseHexString(el.raw) + decodeAndRender(data, cmap) + case tokNum: + if val, err := strconv.ParseFloat(el.raw, 64); err == nil { + cursorX -= val * 0.001 * fontSize + } + } + } + } for { tok, ok := s.next() @@ -412,29 +599,185 @@ func textFromContentStream(content []byte, fontCMaps map[string]map[uint16]rune) } if tok.kind == tokArr && tok.raw == "[" { - out.WriteString(collectTextFromArray(s, fontCMaps, currentFont)) + var seq []pdfToken + for { + el, ok := s.next() + if !ok || (el.kind == tokArr && el.raw == "]") { + break + } + seq = append(seq, el) + } + renderTJArray(seq) continue } if tok.kind == tokKw { switch tok.raw { + case "q": + var cpy [6]float64 + copy(cpy[:], ctm[:]) + ctmStack = append(ctmStack, cpy) + + case "Q": + if len(ctmStack) > 0 { + ctm = ctmStack[len(ctmStack)-1] + ctmStack = ctmStack[:len(ctmStack)-1] + } + + case "cm": + if len(stack) >= 6 { //nolint:mnd + f := stack[len(stack)-1] + e := stack[len(stack)-2] + d := stack[len(stack)-3] + c := stack[len(stack)-4] + b := stack[len(stack)-5] + a := stack[len(stack)-6] + if a.kind == tokNum && b.kind == tokNum && c.kind == tokNum && + d.kind == tokNum && e.kind == tokNum && f.kind == tokNum { + av, _ := strconv.ParseFloat(a.raw, 64) + bv, _ := strconv.ParseFloat(b.raw, 64) + cv, _ := strconv.ParseFloat(c.raw, 64) + dv, _ := strconv.ParseFloat(d.raw, 64) + ev, _ := strconv.ParseFloat(e.raw, 64) + fv, _ := strconv.ParseFloat(f.raw, 64) + ctm = multMatrices([6]float64{av, bv, cv, dv, ev, fv}, ctm) + } + } + + case "BT": + tm = [6]float64{1, 0, 0, 1, 0, 0} + cursorX = 0 + case "Tf": if len(stack) >= 2 && stack[len(stack)-2].kind == tokName { currentFont = strings.TrimPrefix(stack[len(stack)-2].raw, "/") } + if len(stack) >= 1 && stack[len(stack)-1].kind == tokNum { + fs, err := strconv.ParseFloat(stack[len(stack)-1].raw, 64) + if err == nil { + fontSize = fs + } + } case "Td", "TD": if len(stack) >= 2 { + ty := stack[len(stack)-1] + tx := stack[len(stack)-2] + if tx.kind == tokNum && ty.kind == tokNum { + txVal, _ := strconv.ParseFloat(tx.raw, 64) + tyVal, _ := strconv.ParseFloat(ty.raw, 64) + tm = applyTd(tm, txVal, tyVal) + cursorX = 0 + } + } + if tok.raw == "TD" && len(stack) >= 1 { + ty := stack[len(stack)-1] + if ty.kind == tokNum { + tyVal, _ := strconv.ParseFloat(ty.raw, 64) + textLeading = -tyVal + } + } + + case "Tm": + if len(stack) >= 6 { //nolint:mnd + f := stack[len(stack)-1] + e := stack[len(stack)-2] + d := stack[len(stack)-3] + c := stack[len(stack)-4] + b := stack[len(stack)-5] + a := stack[len(stack)-6] + if a.kind == tokNum && b.kind == tokNum && c.kind == tokNum && + d.kind == tokNum && e.kind == tokNum && f.kind == tokNum { + av, _ := strconv.ParseFloat(a.raw, 64) + bv, _ := strconv.ParseFloat(b.raw, 64) + cv, _ := strconv.ParseFloat(c.raw, 64) + dv, _ := strconv.ParseFloat(d.raw, 64) + ev, _ := strconv.ParseFloat(e.raw, 64) + fv, _ := strconv.ParseFloat(f.raw, 64) + tm = [6]float64{av, bv, cv, dv, ev, fv} + cursorX = 0 + } + } + + case "T*": + tm = applyTd(tm, 0, -textLeading) + cursorX = 0 + + case "'": + tm = applyTd(tm, 0, -textLeading) + cursorX = 0 + if len(stack) >= 1 { last := stack[len(stack)-1] - if last.kind == tokNum && last.raw != "0" { - out.WriteByte('\n') + switch last.kind { + case tokStr: + data := []byte(parseLiteralString(last.raw)) + decodeAndRender(data, fontCMaps[currentFont]) + case tokHex: + data := parseHexString(last.raw) + decodeAndRender(data, fontCMaps[currentFont]) } } - case "Tj", "'", "\"": - out.WriteString(writeTextFromStack(stack, fontCMaps, currentFont)) + case "\"": + if len(stack) >= 3 { + ac := stack[len(stack)-2] + if ac.kind == tokNum { + _, _ = strconv.ParseFloat(ac.raw, 64) + } + } + if len(stack) >= 2 { + aw := stack[len(stack)-3] + if aw.kind == tokNum { + awVal, _ := strconv.ParseFloat(aw.raw, 64) + _ = awVal + } + } + tm = applyTd(tm, 0, -textLeading) + cursorX = 0 + if len(stack) >= 1 { + last := stack[len(stack)-1] + switch last.kind { + case tokStr: + data := []byte(parseLiteralString(last.raw)) + decodeAndRender(data, fontCMaps[currentFont]) + case tokHex: + data := parseHexString(last.raw) + decodeAndRender(data, fontCMaps[currentFont]) + } + } + + case "Tj": + if len(stack) >= 1 { + last := stack[len(stack)-1] + switch last.kind { + case tokStr: + data := []byte(parseLiteralString(last.raw)) + decodeAndRender(data, fontCMaps[currentFont]) + case tokHex: + data := parseHexString(last.raw) + decodeAndRender(data, fontCMaps[currentFont]) + } + } case "TJ": + // already handled via array path above + + case "TL": + if len(stack) >= 1 && stack[len(stack)-1].kind == tokNum { + lv, _ := strconv.ParseFloat(stack[len(stack)-1].raw, 64) + textLeading = -lv + } + + case "Tc": + // character spacing - not used for layout + case "Tw": + // word spacing - not used for layout + case "Tz": + // horizontal scaling - not used for layout + case "Ts": + // text rise - not used for layout + case "Tr": + // text rendering mode - not used for layout } stack = stack[:0] @@ -443,40 +786,8 @@ func textFromContentStream(content []byte, fontCMaps map[string]map[uint16]rune) } } - return out.String() -} - -func collectTextFromArray(s *contentScanner, fontCMaps map[string]map[uint16]rune, currentFont string) string { - var texts []string - for { - el, ok := s.next() - if !ok || (el.kind == tokArr && el.raw == "]") { - break - } - switch el.kind { - case tokStr: - texts = append(texts, parseLiteralString(el.raw)) - case tokHex: - cmap := fontCMaps[currentFont] - texts = append(texts, decodeText(parseHexString(el.raw), cmap)) - } - } - return strings.Join(texts, "") -} - -func writeTextFromStack(stack []pdfToken, fontCMaps map[string]map[uint16]rune, currentFont string) string { - if len(stack) < 1 { - return "" - } - last := stack[len(stack)-1] - switch last.kind { - case tokStr: - return parseLiteralString(last.raw) - case tokHex: - cmap := fontCMaps[currentFont] - return decodeText(parseHexString(last.raw), cmap) - } - return "" + lines := groupCharsIntoLines(chars, lineGroupTol) + return renderLines(lines, wordGapRatio) } // PDFTextExtract uses pdfcpu to extract text from a PDF file. @@ -509,18 +820,20 @@ func PDFTextExtract(ctx context.Context, filename string) (string, error) { data, _ := io.ReadAll(r) fontCMaps := buildFontCMaps(pdfCtx, pageNr) + fWidths := buildFontWidths(pdfCtx, pageNr) - pageText := textFromContentStream(data, fontCMaps) + pageText := textFromContentStream(data, fontCMaps, fWidths) if pageText != "" { + if text.Len() > 0 { + text.WriteByte('\n') + } text.WriteString(pageText) - text.WriteByte('\n') } } return text.String(), nil } -// buildFontCMaps builds font resource name → CID→rune maps for a given page. func buildFontCMaps(ctx *model.Context, pageNr int) map[string]map[uint16]rune { result := make(map[string]map[uint16]rune) @@ -549,8 +862,89 @@ func buildFontCMaps(ctx *model.Context, pageNr int) map[string]map[uint16]rune { return result } -// cidToUnicode extracts a CID→rune map from a font dictionary by reading its -// ToUnicode CMap. For Type0 CIDFonts, it also checks DescendantFonts. +func buildFontWidths(ctx *model.Context, pageNr int) map[string]*fontWidths { + result := make(map[string]*fontWidths) + + if pageNr < 1 || pageNr > len(ctx.Optimize.PageFonts) { + return result + } + + pageFonts := ctx.Optimize.PageFonts[pageNr-1] + + for objNr := range pageFonts { + fo, ok := ctx.Optimize.FontObjects[objNr] + if !ok { + continue + } + + fw := fontWidthsFromDict(fo.FontDict) + if fw == nil { + continue + } + + for _, resName := range fo.ResourceNames { + result[resName] = fw + } + } + + return result +} + +func fontWidthsFromDict(fd types.Dict) *fontWidths { + w, found := fd.Find("Widths") + if !found || w == nil { + return stdFontWidthsFromBaseFont(fd) + } + arr, ok := w.(types.Array) + if !ok || len(arr) == 0 { + return nil + } + + firstChar := 0 + if v, found := fd.Find("FirstChar"); found { + if i, ok := v.(types.Integer); ok { + firstChar = i.Value() + } + } + + var fw fontWidths + for i, v := range arr { + code := firstChar + i + if code > maxByte { + break + } + if code < 0 { + continue + } + if iv, ok := v.(types.Integer); ok { + val := iv.Value() + if val >= 0 && val <= 65535 { + fw[code] = uint16(val) + } + } + } + + return &fw +} + +func stdFontWidthsFromBaseFont(fd types.Dict) *fontWidths { + bf, found := fd.Find("BaseFont") + if !found || bf == nil { + return nil + } + name, ok := bf.(types.Name) + if !ok { + return nil + } + baseName := string(name) + fw, ok := stdFontWidths[baseName] + if !ok { + return nil + } + cp := *fw + return &cp +} + func cidToUnicode(ctx *model.Context, fd types.Dict) map[uint16]rune { toUnicode, found := fd.Find("ToUnicode") if found && toUnicode != nil { @@ -583,7 +977,6 @@ func cidToUnicode(ctx *model.Context, fd types.Dict) map[uint16]rune { return resolveToUnicode(ctx, tu) } -// resolveToUnicode resolves a ToUnicode reference and parses the CMap. func resolveToUnicode(ctx *model.Context, obj types.Object) map[uint16]rune { ir, ok := obj.(types.IndirectRef) if !ok { diff --git a/pdftotext/pdftotext_test.go b/pdftotext/pdftotext_test.go index 3a2ce46..fb5ea7c 100644 --- a/pdftotext/pdftotext_test.go +++ b/pdftotext/pdftotext_test.go @@ -6,6 +6,7 @@ package pdftotext import ( "context" "os" + "strings" "testing" "github.com/pdfcpu/pdfcpu/pkg/api" @@ -20,6 +21,10 @@ import ( const ( testdataDir = "testdata" testDescendantFonts = "DescendantFonts" + testWidths = "Widths" + testFirstChar = "FirstChar" + testFontName = "F1" + testBaseFont = "BaseFont" ) func TestPDFTextExtract(t *testing.T) { @@ -50,7 +55,7 @@ func TestPDFTextExtract(t *testing.T) { require.NoError(t, err) assert.Contains(t, output, "Continental Trust") - assert.Contains(t, output, "Sanne Mulders") + assert.Contains(t, output, "Sanne Mulder") assert.Contains(t, output, "Rekeningafschrift") assert.Contains(t, output, "31.10.2025") }) @@ -74,6 +79,41 @@ func TestPDFTextExtract(t *testing.T) { assert.Contains(t, output, "10 426,76") assert.Contains(t, output, "30 avril 2025") }) + + t.Run("per-character Tj+Td word gap detection", func(t *testing.T) { + // Synthetic PDF where each character is rendered individually via Tj+Td. + // Word boundaries are encoded only in the Td advances, not in the text. + // Without font metric word gap detection, the output would be "1RUEDERENNES". + output, err := PDFTextExtract(context.Background(), "testdata/per-char-test.pdf") + require.NoError(t, err) + output = strings.TrimSpace(output) + + assert.Equal(t, "1 RUE DE RENNES", output) + }) + + t.Run("pdflatex per-character text", func(t *testing.T) { + // Real-world PDF generated by pdfTeX which renders each character + // individually via Tj+Td. Without font metric word gap detection, + // the output would be "Loremipsumdolorsitamet". + // Source: https://github.com/py-pdf/sample-files (CC-BY-SA-4.0) + output, err := PDFTextExtract(context.Background(), "testdata/pdflatex-per-char-text.pdf") + require.NoError(t, err) + + assert.Contains(t, output, "Lorem ipsum dolor sit amet") + assert.Contains(t, output, "consetetur sadipscing elitr") + }) + + t.Run("Latin-1 word gap (Type1, no ToUnicode, accented chars)", func(t *testing.T) { + // Synthetic PDF with Type1 font: no ToUnicode CMap, no Encoding. + // Accented chars use Latin-1 bytes in per-character Tj+Td layout. + // Font metric word gap must detect word boundary from glyph advances. + // Without font widths: median=15, threshold=30, advance=25→no gap →"déjàvu" + output, err := PDFTextExtract(context.Background(), "testdata/latin1-word-gap.pdf") + require.NoError(t, err) + + output = strings.TrimSpace(output) + assert.Equal(t, "déjà vu", output) + }) } func TestPDFTextExtractFileNotFound(t *testing.T) { @@ -93,37 +133,37 @@ func TestPDFTextExtractBadFile(t *testing.T) { func TestTextFromContentStream(t *testing.T) { t.Run("literal Tj", func(t *testing.T) { content := []byte("BT\n/F1 12 Tf\n100 700 Td\n(Hello World) Tj\nET\n") - text := textFromContentStream(content, nil) + text := textFromContentStream(content, nil, nil) assert.Contains(t, text, "Hello World") }) t.Run("hex Tj", func(t *testing.T) { content := []byte("<48656C6C6F> Tj") - text := textFromContentStream(content, nil) + text := textFromContentStream(content, nil, nil) assert.Contains(t, text, "Hello") }) t.Run("TJ array", func(t *testing.T) { content := []byte("[(Hello)(World)]TJ") - text := textFromContentStream(content, nil) + text := textFromContentStream(content, nil, nil) assert.Contains(t, text, "Hello") assert.Contains(t, text, "World") }) t.Run("empty", func(t *testing.T) { - text := textFromContentStream(nil, nil) + text := textFromContentStream(nil, nil, nil) assert.Equal(t, "", text) }) t.Run("single quote literal", func(t *testing.T) { content := []byte("(Hello World)'") - text := textFromContentStream(content, nil) + text := textFromContentStream(content, nil, nil) assert.Contains(t, text, "Hello World") }) t.Run("single quote hex", func(t *testing.T) { content := []byte("<48656C6C6F>'") - text := textFromContentStream(content, nil) + text := textFromContentStream(content, nil, nil) assert.Contains(t, text, "Hello") }) } @@ -186,6 +226,9 @@ func TestDecodeText(t *testing.T) { result = decodeText([]byte{0x48, 0x65, 0x6C, 0x6C, 0x6F}, nil) assert.Equal(t, "Hello", result) + + result = decodeText([]byte{0xE9, 0xE0, 0xFC, 0xE2}, nil) + assert.Equal(t, "éàüâ", result) } func TestParseLiteralString(t *testing.T) { @@ -228,7 +271,7 @@ func TestNext_Operators(t *testing.T) { t.Run("escaped char in literal string", func(t *testing.T) { content := []byte("(Hello\\nWorld) Tj") - text := textFromContentStream(content, nil) + text := textFromContentStream(content, nil, nil) assert.Equal(t, "Hello\nWorld", text) }) } @@ -674,13 +717,13 @@ func TestResolveToUnicode_DecodeCorruptedStream(t *testing.T) { func TestTextFromContentStream_WriteTextEdgeCases(t *testing.T) { t.Run("empty stack", func(t *testing.T) { content := []byte("Tj") - text := textFromContentStream(content, nil) + text := textFromContentStream(content, nil, nil) assert.Empty(t, text) }) t.Run("non_string_token", func(t *testing.T) { content := []byte("/Name Tj") - text := textFromContentStream(content, nil) + text := textFromContentStream(content, nil, nil) assert.Empty(t, text) }) } @@ -688,13 +731,270 @@ func TestTextFromContentStream_WriteTextEdgeCases(t *testing.T) { func TestTextFromContentStream_DQuote(t *testing.T) { t.Run("literal string", func(t *testing.T) { content := []byte("(Hello World)\"") - text := textFromContentStream(content, nil) + text := textFromContentStream(content, nil, nil) assert.Contains(t, text, "Hello World") }) t.Run("hex string", func(t *testing.T) { content := []byte("<48656C6C6F>\"") - text := textFromContentStream(content, nil) + text := textFromContentStream(content, nil, nil) assert.Contains(t, text, "Hello") }) } + +func TestReadKeyword_ControlChar(t *testing.T) { + t.Run("control char advances pos", func(t *testing.T) { + s := &contentScanner{data: []byte{0x03, 0x41}} + tok, ok := s.next() + assert.True(t, ok) + assert.Equal(t, byte(tokKw), tok.kind) + assert.Equal(t, "\x03", tok.raw) + assert.Equal(t, 1, s.pos) + }) + + t.Run("control char in stream does not hang", func(t *testing.T) { + content := []byte("BT\n/F1 12 Tf\n100 700 Td\n(Hello World) Tj\n\x03\xF0\x3F\x03\xF0\x3FET\n") + text := textFromContentStream(content, nil, nil) + assert.Contains(t, text, "Hello World") + }) +} + +func TestPDFTextExtract_ControlChar(t *testing.T) { + output, err := PDFTextExtract(context.Background(), testdataDir+"/control-char.pdf") + require.NoError(t, err) + + assert.Contains(t, output, "Hello World") +} + +func TestBuildFontWidths_InvalidPage(t *testing.T) { + conf := model.NewDefaultConfiguration() + f, err := os.Open(testdataDir + "/forged-invoice.pdf") + require.NoError(t, err) + defer f.Close() + pdfCtx, err := api.ReadValidateAndOptimize(f, conf) + require.NoError(t, err) + + assert.Empty(t, buildFontWidths(pdfCtx, 0)) + assert.Empty(t, buildFontWidths(pdfCtx, 9999)) +} + +func TestBuildFontWidths_MissingFontObject(t *testing.T) { + conf := model.NewDefaultConfiguration() + f, err := os.Open(testdataDir + "/forged-invoice.pdf") + require.NoError(t, err) + defer f.Close() + pdfCtx, err := api.ReadValidateAndOptimize(f, conf) + require.NoError(t, err) + + pdfCtx.Optimize.PageFonts[0][9999] = true + m := buildFontWidths(pdfCtx, 1) + assert.NotNil(t, m) +} + +func TestFontWidthsFromDict_EdgeCases(t *testing.T) { + t.Run("missing Widths", func(t *testing.T) { + fd := types.Dict{} + assert.Nil(t, fontWidthsFromDict(fd)) + }) + + t.Run("Widths not array", func(t *testing.T) { + fd := types.Dict{testWidths: types.Integer(0)} + assert.Nil(t, fontWidthsFromDict(fd)) + }) + + t.Run("code > maxByte", func(t *testing.T) { + widths := make(types.Array, 257) + for i := range widths { + widths[i] = types.Integer(500) + } + fd := types.Dict{ + testWidths: widths, + testFirstChar: types.Integer(0), + } + fw := fontWidthsFromDict(fd) + require.NotNil(t, fw) + assert.Equal(t, uint16(500), fw[0]) + assert.Equal(t, uint16(500), fw[255]) + }) + + t.Run("code < 0", func(t *testing.T) { + fd := types.Dict{ + testWidths: types.Array{types.Integer(500)}, + testFirstChar: types.Integer(-1), + } + fw := fontWidthsFromDict(fd) + require.NotNil(t, fw) + assert.Equal(t, uint16(0), fw[0]) + }) + + t.Run("non-integer in Widths array", func(t *testing.T) { + fd := types.Dict{ + testWidths: types.Array{types.Name("test")}, + testFirstChar: types.Integer(0), + } + fw := fontWidthsFromDict(fd) + require.NotNil(t, fw) + assert.Equal(t, uint16(0), fw[0]) + }) + + t.Run("empty Widths array", func(t *testing.T) { + fd := types.Dict{testWidths: types.Array{}} + assert.Nil(t, fontWidthsFromDict(fd)) + }) + + t.Run("FirstChar not integer", func(t *testing.T) { + fd := types.Dict{ + testWidths: types.Array{types.Integer(500)}, + testFirstChar: types.Name("test"), + } + fw := fontWidthsFromDict(fd) + require.NotNil(t, fw) + assert.Equal(t, uint16(500), fw[0]) + }) +} + +func TestTextFromContentStream_Basic(t *testing.T) { + content := []byte("BT /F1 12 Tf 0 0 Td (Hello) Tj ET") + text := textFromContentStream(content, nil, nil) + assert.Contains(t, text, "Hello") +} + +func TestTextFromContentStream_FirstAdvanceOnLine(t *testing.T) { + // First Td with no preceding text keeps lastTextX=-1. + // Second Td on same line with positive advance triggers + // else if lastTextX < 0 branch. + content := []byte("BT /F1 12 Tf 0 0 Td 10 0 Td (a) Tj ET") + text := textFromContentStream(content, nil, nil) + assert.Equal(t, "a", text) +} + +func TestTextFromContentStream_MaxCharAdvances(t *testing.T) { + // 55 chars on one line with Td(10,0) between each. + // Each gap (10 - charWidth) > charSize*wordGapRatio → space inserted. + // Expected: 55 "x" + 54 spaces = 109 chars. + var sb strings.Builder + sb.WriteString("BT /F1 12 Tf 0 0 Td") + for i := range 55 { + sb.WriteString("(x) Tj 10 0 Td") + _ = i + } + sb.WriteString(" ET") + text := textFromContentStream([]byte(sb.String()), nil, nil) + assert.Len(t, text, 109) +} + +func TestTextFromContentStream_UnicodeAbove256(t *testing.T) { + cmap := map[string]map[uint16]rune{ + testFontName: {0x41: '中'}, + } + content := []byte("/F1 12 Tf 12 0 Td (A) Tj") + text := textFromContentStream(content, cmap, nil) + assert.Equal(t, "中", text) +} + +func TestTextFromContentStream_AccentWordGap(t *testing.T) { + // Font metric word gap detection with Latin-1 accented chars. + // Content stream uses octal escapes for é (\351) and à (\340). + // Per-character Tj+Td: d(15)é(10)j(10)à(25)v(15)u + // fw[à]=500, fs=41.6667 → charWidth=20.83, extra=25-20.83=4.17 + // fw[space]=300 → spaceWidth=12.5, spaceRatio=0.3→3.75 + // fs*0.04=1.67, threshold=max(1.67,3.75)=3.75 + // extra 4.17 > 3.75 → word gap → "déjà vu" + content := []byte("BT /F1 41.6667 Tf 0 0 Td (d) Tj 15 0 Td (\\351) Tj 10 0 Td (j) Tj 10 0 Td (\\340) Tj 25 0 Td (v) Tj 15 0 Td (u) Tj ET") + + fw := &fontWidths{} + for i := range 256 { + fw[i] = 400 + } + fw[0x20] = 300 // space + fw['d'] = 400 + fw[0xE9] = 500 // é + fw['j'] = 300 + fw[0xE0] = 500 // à + fw['v'] = 350 + fw['u'] = 350 + + text := textFromContentStream(content, nil, map[string]*fontWidths{testFontName: fw}) + + assert.Equal(t, "déjà vu", text) +} + +func TestTextFromContentStream_AccentWordGapFallback(t *testing.T) { + // Same content without font widths: geometric gap detection still works + // because Td advances create page-space X gaps between words. + content := []byte("BT /F1 41.6667 Tf 0 0 Td (d) Tj 15 0 Td (\\351) Tj 10 0 Td (j) Tj 10 0 Td (\\340) Tj 25 0 Td (v) Tj 15 0 Td (u) Tj ET") + + text := textFromContentStream(content, nil, nil) + + assert.Equal(t, "déjà vu", text) +} + +func TestTextFromContentStream_NegFontSize(t *testing.T) { + content := []byte("BT /F1 -12 Tf 0 0 Td (Hello) Tj ET") + text := textFromContentStream(content, nil, nil) + assert.NotEmpty(t, text) +} + +func TestTextFromContentStream_TStar(t *testing.T) { + content := []byte("BT /F1 12 Tf 0 0 Td 12 TL (Hello) Tj T* (World) Tj ET") + text := textFromContentStream(content, nil, nil) + assert.Contains(t, text, "Hello") + assert.Contains(t, text, "World") +} + +func TestTextFromContentStream_TDOperatorLeading(t *testing.T) { + content := []byte("BT /F1 12 Tf 0 0 Td (Hello) Tj /Name -10 TD (World) Tj ET") + text := textFromContentStream(content, nil, nil) + assert.Contains(t, text, "Hello") + assert.Contains(t, text, "World") +} + +func TestTextFromContentStream_DQuoteFull(t *testing.T) { + content := []byte("1 2 (Hello World)\"") + text := textFromContentStream(content, nil, nil) + assert.Contains(t, text, "Hello World") +} + +func TestTextFromContentStream_TL(t *testing.T) { + content := []byte("BT 12 TL /F1 12 Tf 0 0 Td (Hello) Tj ET") + text := textFromContentStream(content, nil, nil) + assert.Contains(t, text, "Hello") +} + +func TestTextFromContentStream_TextStateNoOps(t *testing.T) { + content := []byte("BT /F1 12 Tf 0 0 Td 1 Tc 2 Tw 100 Tz 0 Ts 0 Tr (Hello) Tj ET") + text := textFromContentStream(content, nil, nil) + assert.Contains(t, text, "Hello") +} + +func TestTextFromContentStream_CMapLookupFail(t *testing.T) { + cmap := map[string]map[uint16]rune{testFontName: {0x48: 'X'}} + content := []byte("BT /F1 12 Tf 0 0 Td (Hi) Tj ET") + text := textFromContentStream(content, cmap, nil) + assert.Contains(t, text, "X") + assert.Contains(t, text, "i") +} + +func TestStdFontWidthsFromBaseFont(t *testing.T) { + t.Run("missing BaseFont", func(t *testing.T) { + fd := types.Dict{} + assert.Nil(t, stdFontWidthsFromBaseFont(fd)) + }) + + t.Run("BaseFont not a Name", func(t *testing.T) { + fd := types.Dict{testBaseFont: types.Integer(0)} + assert.Nil(t, stdFontWidthsFromBaseFont(fd)) + }) + + t.Run("BaseFont not in stdFontWidths", func(t *testing.T) { + fd := types.Dict{testBaseFont: types.Name("UnknownFont")} + assert.Nil(t, stdFontWidthsFromBaseFont(fd)) + }) + + t.Run("BaseFont in stdFontWidths", func(t *testing.T) { + fd := types.Dict{testBaseFont: types.Name("Helvetica")} + fw := stdFontWidthsFromBaseFont(fd) + require.NotNil(t, fw) + assert.Equal(t, uint16(278), fw[0x20]) + }) +} diff --git a/pdftotext/testdata/LICENSE b/pdftotext/testdata/LICENSE index fda1a8e..e755a34 100644 --- a/pdftotext/testdata/LICENSE +++ b/pdftotext/testdata/LICENSE @@ -4,3 +4,8 @@ SPDX-License-Identifier: MIT Test data licenses: - forged-invoice.pdf: MIT (Fileganizer project, created for testing) - bsb-*-statement.pdf: MIT (https://github.com/bankstatemently/bank-statement-parsing-benchmark) +- control-char.pdf: MIT (Fileganizer project, created for testing) +- per-char-test.pdf: MIT (Fileganizer project, created for testing) +- latin1-word-gap.pdf: MIT (Fileganizer project, created for testing) +- pdflatex-per-char-text.pdf: CC-BY-SA-4.0 (https://github.com/py-pdf/sample-files) + See LICENSE-py-pdf for full license text. diff --git a/pdftotext/testdata/LICENSE-py-pdf b/pdftotext/testdata/LICENSE-py-pdf new file mode 100644 index 0000000..7d4f96c --- /dev/null +++ b/pdftotext/testdata/LICENSE-py-pdf @@ -0,0 +1,427 @@ +Attribution-ShareAlike 4.0 International + +======================================================================= + +Creative Commons Corporation ("Creative Commons") is not a law firm and +does not provide legal services or legal advice. Distribution of +Creative Commons public licenses does not create a lawyer-client or +other relationship. Creative Commons makes its licenses and related +information available on an "as-is" basis. Creative Commons gives no +warranties regarding its licenses, any material licensed under their +terms and conditions, or any related information. Creative Commons +disclaims all liability for damages resulting from their use to the +fullest extent possible. + +Using Creative Commons Public Licenses + +Creative Commons public licenses provide a standard set of terms and +conditions that creators and other rights holders may use to share +original works of authorship and other material subject to copyright +and certain other rights specified in the public license below. The +following considerations are for informational purposes only, are not +exhaustive, and do not form part of our licenses. + + Considerations for licensors: Our public licenses are + intended for use by those authorized to give the public + permission to use material in ways otherwise restricted by + copyright and certain other rights. Our licenses are + irrevocable. Licensors should read and understand the terms + and conditions of the license they choose before applying it. + Licensors should also secure all rights necessary before + applying our licenses so that the public can reuse the + material as expected. Licensors should clearly mark any + material not subject to the license. This includes other CC- + licensed material, or material used under an exception or + limitation to copyright. More considerations for licensors: + wiki.creativecommons.org/Considerations_for_licensors + + Considerations for the public: By using one of our public + licenses, a licensor grants the public permission to use the + licensed material under specified terms and conditions. If + the licensor's permission is not necessary for any reason--for + example, because of any applicable exception or limitation to + copyright--then that use is not regulated by the license. Our + licenses grant only permissions under copyright and certain + other rights that a licensor has authority to grant. Use of + the licensed material may still be restricted for other + reasons, including because others have copyright or other + rights in the material. A licensor may make special requests, + such as asking that all changes be marked or described. + Although not required by our licenses, you are encouraged to + respect those requests where reasonable. More considerations + for the public: + wiki.creativecommons.org/Considerations_for_licensees + +======================================================================= + +Creative Commons Attribution-ShareAlike 4.0 International Public +License + +By exercising the Licensed Rights (defined below), You accept and agree +to be bound by the terms and conditions of this Creative Commons +Attribution-ShareAlike 4.0 International Public License ("Public +License"). To the extent this Public License may be interpreted as a +contract, You are granted the Licensed Rights in consideration of Your +acceptance of these terms and conditions, and the Licensor grants You +such rights in consideration of benefits the Licensor receives from +making the Licensed Material available under these terms and +conditions. + + +Section 1 -- Definitions. + + a. Adapted Material means material subject to Copyright and Similar + Rights that is derived from or based upon the Licensed Material + and in which the Licensed Material is translated, altered, + arranged, transformed, or otherwise modified in a manner requiring + permission under the Copyright and Similar Rights held by the + Licensor. For purposes of this Public License, where the Licensed + Material is a musical work, performance, or sound recording, + Adapted Material is always produced where the Licensed Material is + synched in timed relation with a moving image. + + b. Adapter's License means the license You apply to Your Copyright + and Similar Rights in Your contributions to Adapted Material in + accordance with the terms and conditions of this Public License. + + c. BY-SA Compatible License means a license listed at + creativecommons.org/compatiblelicenses, approved by Creative + Commons as essentially the equivalent of this Public License. + + d. Copyright and Similar Rights means copyright and/or similar rights + closely related to copyright including, without limitation, + performance, broadcast, sound recording, and Sui Generis Database + Rights, without regard to how the rights are labeled or + categorized. For purposes of this Public License, the rights + specified in Section 2(b)(1)-(2) are not Copyright and Similar + Rights. + + e. Effective Technological Measures means those measures that, in the + absence of proper authority, may not be circumvented under laws + fulfilling obligations under Article 11 of the WIPO Copyright + Treaty adopted on December 20, 1996, and/or similar international + agreements. + + f. Exceptions and Limitations means fair use, fair dealing, and/or + any other exception or limitation to Copyright and Similar Rights + that applies to Your use of the Licensed Material. + + g. License Elements means the license attributes listed in the name + of a Creative Commons Public License. The License Elements of this + Public License are Attribution and ShareAlike. + + h. Licensed Material means the artistic or literary work, database, + or other material to which the Licensor applied this Public + License. + + i. Licensed Rights means the rights granted to You subject to the + terms and conditions of this Public License, which are limited to + all Copyright and Similar Rights that apply to Your use of the + Licensed Material and that the Licensor has authority to license. + + j. Licensor means the individual(s) or entity(ies) granting rights + under this Public License. + + k. Share means to provide material to the public by any means or + process that requires permission under the Licensed Rights, such + as reproduction, public display, public performance, distribution, + dissemination, communication, or importation, and to make material + available to the public including in ways that members of the + public may access the material from a place and at a time + individually chosen by them. + + l. Sui Generis Database Rights means rights other than copyright + resulting from Directive 96/9/EC of the European Parliament and of + the Council of 11 March 1996 on the legal protection of databases, + as amended and/or succeeded, as well as other essentially + equivalent rights anywhere in the world. + + m. You means the individual or entity exercising the Licensed Rights + under this Public License. Your has a corresponding meaning. + + +Section 2 -- Scope. + + a. License grant. + + 1. Subject to the terms and conditions of this Public License, + the Licensor hereby grants You a worldwide, royalty-free, + non-sublicensable, non-exclusive, irrevocable license to + exercise the Licensed Rights in the Licensed Material to: + + a. reproduce and Share the Licensed Material, in whole or + in part; and + + b. produce, reproduce, and Share Adapted Material. + + 2. Exceptions and Limitations. For the avoidance of doubt, where + Exceptions and Limitations apply to Your use, this Public + License does not apply, and You do not need to comply with + its terms and conditions. + + 3. Term. The term of this Public License is specified in Section + 6(a). + + 4. Media and formats; technical modifications allowed. The + Licensor authorizes You to exercise the Licensed Rights in + all media and formats whether now known or hereafter created, + and to make technical modifications necessary to do so. The + Licensor waives and/or agrees not to assert any right or + authority to forbid You from making technical modifications + necessary to exercise the Licensed Rights, including + technical modifications necessary to circumvent Effective + Technological Measures. For purposes of this Public License, + simply making modifications authorized by this Section 2(a) + (4) never produces Adapted Material. + + 5. Downstream recipients. + + a. Offer from the Licensor -- Licensed Material. Every + recipient of the Licensed Material automatically + receives an offer from the Licensor to exercise the + Licensed Rights under the terms and conditions of this + Public License. + + b. Additional offer from the Licensor -- Adapted Material. + Every recipient of Adapted Material from You + automatically receives an offer from the Licensor to + exercise the Licensed Rights in the Adapted Material + under the conditions of the Adapter's License You apply. + + c. No downstream restrictions. You may not offer or impose + any additional or different terms or conditions on, or + apply any Effective Technological Measures to, the + Licensed Material if doing so restricts exercise of the + Licensed Rights by any recipient of the Licensed + Material. + + 6. No endorsement. Nothing in this Public License constitutes or + may be construed as permission to assert or imply that You + are, or that Your use of the Licensed Material is, connected + with, or sponsored, endorsed, or granted official status by, + the Licensor or others designated to receive attribution as + provided in Section 3(a)(1)(A)(i). + + b. Other rights. + + 1. Moral rights, such as the right of integrity, are not + licensed under this Public License, nor are publicity, + privacy, and/or other similar personality rights; however, to + the extent possible, the Licensor waives and/or agrees not to + assert any such rights held by the Licensor to the limited + extent necessary to allow You to exercise the Licensed + Rights, but not otherwise. + + 2. Patent and trademark rights are not licensed under this + Public License. + + 3. To the extent possible, the Licensor waives any right to + collect royalties from You for the exercise of the Licensed + Rights, whether directly or through a collecting society + under any voluntary or waivable statutory or compulsory + licensing scheme. In all other cases the Licensor expressly + reserves any right to collect such royalties. + + +Section 3 -- License Conditions. + +Your exercise of the Licensed Rights is expressly made subject to the +following conditions. + + a. Attribution. + + 1. If You Share the Licensed Material (including in modified + form), You must: + + a. retain the following if it is supplied by the Licensor + with the Licensed Material: + + i. identification of the creator(s) of the Licensed + Material and any others designated to receive + attribution, in any reasonable manner requested by + the Licensor (including by pseudonym if + designated); + + ii. a copyright notice; + + iii. a notice that refers to this Public License; + + iv. a notice that refers to the disclaimer of + warranties; + + v. a URI or hyperlink to the Licensed Material to the + extent reasonably practicable; + + b. indicate if You modified the Licensed Material and + retain an indication of any previous modifications; and + + c. indicate the Licensed Material is licensed under this + Public License, and include the text of, or the URI or + hyperlink to, this Public License. + + 2. You may satisfy the conditions in Section 3(a)(1) in any + reasonable manner based on the medium, means, and context in + which You Share the Licensed Material. For example, it may be + reasonable to satisfy the conditions by providing a URI or + hyperlink to a resource that includes the required + information. + + 3. If requested by the Licensor, You must remove any of the + information required by Section 3(a)(1)(A) to the extent + reasonably practicable. + + b. ShareAlike. + + In addition to the conditions in Section 3(a), if You Share + Adapted Material You produce, the following conditions also apply. + + 1. The Adapter's License You apply must be a Creative Commons + license with the same License Elements, this version or + later, or a BY-SA Compatible License. + + 2. You must include the text of, or the URI or hyperlink to, the + Adapter's License You apply. You may satisfy this condition + in any reasonable manner based on the medium, means, and + context in which You Share Adapted Material. + + 3. You may not offer or impose any additional or different terms + or conditions on, or apply any Effective Technological + Measures to, Adapted Material that restrict exercise of the + rights granted under the Adapter's License You apply. + + +Section 4 -- Sui Generis Database Rights. + +Where the Licensed Rights include Sui Generis Database Rights that +apply to Your use of the Licensed Material: + + a. for the avoidance of doubt, Section 2(a)(1) grants You the right + to extract, reuse, reproduce, and Share all or a substantial + portion of the contents of the database; + + b. if You include all or a substantial portion of the database + contents in a database in which You have Sui Generis Database + Rights, then the database in which You have Sui Generis Database + Rights (but not its individual contents) is Adapted Material, + including for purposes of Section 3(b); and + + c. You must comply with the conditions in Section 3(a) if You Share + all or a substantial portion of the contents of the database. + +For the avoidance of doubt, this Section 4 supplements and does not +replace Your obligations under this Public License where the Licensed +Rights include other Copyright and Similar Rights. + + +Section 5 -- Disclaimer of Warranties and Limitation of Liability. + + a. UNLESS OTHERWISE SEPARATELY UNDERTAKEN BY THE LICENSOR, TO THE + EXTENT POSSIBLE, THE LICENSOR OFFERS THE LICENSED MATERIAL AS-IS + AND AS-AVAILABLE, AND MAKES NO REPRESENTATIONS OR WARRANTIES OF + ANY KIND CONCERNING THE LICENSED MATERIAL, WHETHER EXPRESS, + IMPLIED, STATUTORY, OR OTHER. THIS INCLUDES, WITHOUT LIMITATION, + WARRANTIES OF TITLE, MERCHANTABILITY, FITNESS FOR A PARTICULAR + PURPOSE, NON-INFRINGEMENT, ABSENCE OF LATENT OR OTHER DEFECTS, + ACCURACY, OR THE PRESENCE OR ABSENCE OF ERRORS, WHETHER OR NOT + KNOWN OR DISCOVERABLE. WHERE DISCLAIMERS OF WARRANTIES ARE NOT + ALLOWED IN FULL OR IN PART, THIS DISCLAIMER MAY NOT APPLY TO YOU. + + b. TO THE EXTENT POSSIBLE, IN NO EVENT WILL THE LICENSOR BE LIABLE + TO YOU ON ANY LEGAL THEORY (INCLUDING, WITHOUT LIMITATION, + NEGLIGENCE) OR OTHERWISE FOR ANY DIRECT, SPECIAL, INDIRECT, + INCIDENTAL, CONSEQUENTIAL, PUNITIVE, EXEMPLARY, OR OTHER LOSSES, + COSTS, EXPENSES, OR DAMAGES ARISING OUT OF THIS PUBLIC LICENSE OR + USE OF THE LICENSED MATERIAL, EVEN IF THE LICENSOR HAS BEEN + ADVISED OF THE POSSIBILITY OF SUCH LOSSES, COSTS, EXPENSES, OR + DAMAGES. WHERE A LIMITATION OF LIABILITY IS NOT ALLOWED IN FULL OR + IN PART, THIS LIMITATION MAY NOT APPLY TO YOU. + + c. The disclaimer of warranties and limitation of liability provided + above shall be interpreted in a manner that, to the extent + possible, most closely approximates an absolute disclaimer and + waiver of all liability. + + +Section 6 -- Term and Termination. + + a. This Public License applies for the term of the Copyright and + Similar Rights licensed here. However, if You fail to comply with + this Public License, then Your rights under this Public License + terminate automatically. + + b. Where Your right to use the Licensed Material has terminated under + Section 6(a), it reinstates: + + 1. automatically as of the date the violation is cured, provided + it is cured within 30 days of Your discovery of the + violation; or + + 2. upon express reinstatement by the Licensor. + + For the avoidance of doubt, this Section 6(b) does not affect any + right the Licensor may have to seek remedies for Your violations + of this Public License. + + c. For the avoidance of doubt, the Licensor may also offer the + Licensed Material under separate terms or conditions or stop + distributing the Licensed Material at any time; however, doing so + will not terminate this Public License. + + d. Sections 1, 5, 6, 7, and 8 survive termination of this Public + License. + + +Section 7 -- Other Terms and Conditions. + + a. The Licensor shall not be bound by any additional or different + terms or conditions communicated by You unless expressly agreed. + + b. Any arrangements, understandings, or agreements regarding the + Licensed Material not stated herein are separate from and + independent of the terms and conditions of this Public License. + + +Section 8 -- Interpretation. + + a. For the avoidance of doubt, this Public License does not, and + shall not be interpreted to, reduce, limit, restrict, or impose + conditions on any use of the Licensed Material that could lawfully + be made without permission under this Public License. + + b. To the extent possible, if any provision of this Public License is + deemed unenforceable, it shall be automatically reformed to the + minimum extent necessary to make it enforceable. If the provision + cannot be reformed, it shall be severed from this Public License + without affecting the enforceability of the remaining terms and + conditions. + + c. No term or condition of this Public License will be waived and no + failure to comply consented to unless expressly agreed to by the + Licensor. + + d. Nothing in this Public License constitutes or may be interpreted + as a limitation upon, or waiver of, any privileges and immunities + that apply to the Licensor or You, including from the legal + processes of any jurisdiction or authority. + + +======================================================================= + +Creative Commons is not a party to its public +licenses. Notwithstanding, Creative Commons may elect to apply one of +its public licenses to material it publishes and in those instances +will be considered the “Licensor.” The text of the Creative Commons +public licenses is dedicated to the public domain under the CC0 Public +Domain Dedication. Except for the limited purpose of indicating that +material is shared under a Creative Commons public license or as +otherwise permitted by the Creative Commons policies published at +creativecommons.org/policies, Creative Commons does not authorize the +use of the trademark "Creative Commons" or any other trademark or logo +of Creative Commons without its prior written consent including, +without limitation, in connection with any unauthorized modifications +to any of its public licenses or any other arrangements, +understandings, or agreements concerning use of licensed material. For +the avoidance of doubt, this paragraph does not form part of the +public licenses. + +Creative Commons may be contacted at creativecommons.org. diff --git a/pdftotext/testdata/control-char.pdf b/pdftotext/testdata/control-char.pdf new file mode 100644 index 0000000..1a60eec Binary files /dev/null and b/pdftotext/testdata/control-char.pdf differ diff --git a/pdftotext/testdata/latin1-word-gap.pdf b/pdftotext/testdata/latin1-word-gap.pdf new file mode 100644 index 0000000..1021ff7 Binary files /dev/null and b/pdftotext/testdata/latin1-word-gap.pdf differ diff --git a/pdftotext/testdata/pdflatex-per-char-text.pdf b/pdftotext/testdata/pdflatex-per-char-text.pdf new file mode 100644 index 0000000..af5e732 Binary files /dev/null and b/pdftotext/testdata/pdflatex-per-char-text.pdf differ diff --git a/pdftotext/testdata/per-char-test.pdf b/pdftotext/testdata/per-char-test.pdf new file mode 100644 index 0000000..152b98a Binary files /dev/null and b/pdftotext/testdata/per-char-test.pdf differ diff --git a/testdata/config.bsb.yaml b/testdata/config.bsb.yaml index 4b06c41..8d95687 100644 --- a/testdata/config.bsb.yaml +++ b/testdata/config.bsb.yaml @@ -14,7 +14,7 @@ grokPatterns: COMPANY001: "Straits Capital" OWNER002: "Robert Wilson" COMPANY002: "Liberty National Bank" - OWNER003: "Sanne Mulders" + OWNER003: "Sanne Mulder" COMPANY003: "Continental Trust" OWNER004: "Mei Ling Tsang" COMPANY004: "Silk Road Banking" @@ -38,7 +38,7 @@ fileDescriptions: patterns: - "%{OWNER003:owner}" - "%{COMPANY003:company}" - - "Download date:%{YEAR:year}-%{MONTHNUM2:month}-%{MONTHDAY:day}" + - "Download date: %{YEAR:year}-%{MONTHNUM2:month}-%{MONTHDAY:day}" output: "{{ .Grok.year }}{{ .Grok.month }}{{ .Grok.day }}__continental_trust__sanne_mulders\n" bsb004: patterns: