diff --git a/lib/EpdFont/EpdFontFamily.h b/lib/EpdFont/EpdFontFamily.h index 4d81ab4e990..8943203b914 100644 --- a/lib/EpdFont/EpdFontFamily.h +++ b/lib/EpdFont/EpdFontFamily.h @@ -17,6 +17,11 @@ class EpdFontFamily { SUP = 16, // superscript: glyph scaled 50%, raised ~40% of ascender SUB = 32, // subscript: glyph scaled 50%, lowered ~25% of ascender RUBY_CONTINUE = 64, // Group ruby follower marker (used internally by Epub layout) + // Vertical layout only: the token is set the opposite way from what its text implies + // (an ASCII letter upright, a fullwidth ！ turned, a Latin run in one tate-chu-yoko + // cell). Carries CSS text-orientation / text-combine-upright through the page cache + // without a per-word behaviour array; TextBlock::renderVertical reads it, nothing else. + VERTICAL_FLIP = 128, }; static constexpr uint8_t TEXT_DECORATION_MASK = static_cast(UNDERLINE | STRIKETHROUGH); diff --git a/lib/EpdFont/SdCardFont.cpp b/lib/EpdFont/SdCardFont.cpp index 8760ba02d74..4fad84747eb 100644 --- a/lib/EpdFont/SdCardFont.cpp +++ b/lib/EpdFont/SdCardFont.cpp @@ -4,6 +4,7 @@ #include #include #include +#include #include #include @@ -11,6 +12,7 @@ #include #include +#include "../../src/util/InputDiag.h" #include "EpdFontFamily.h" // Resident SD-font buffers (glyph/kern arenas, interval + advance tables, the @@ -29,6 +31,65 @@ static_assert(sizeof(EpdLigaturePair) == 8, "EpdLigaturePair must be 8 bytes to namespace { +// One read cursor over the .cpfont for every parser and hot path. It serves a +// read from the flash copy (see SdCardFontCache) when the whole range lies in +// the copied prefix, and from the SD card otherwise -- the copy may hold only +// the regular style of a two-style file. The SD file is opened on first need, +// so a page whose glyphs all sit in the copy never touches the card. A flash +// read failure turns the copy off for the owning font and retries on SD. +class FontFile { + public: + FontFile(const char* path, bool* useFlash, size_t flashPayloadBytes) + : path_(path), useFlash_(useFlash), flashBytes_(flashPayloadBytes), flash_(useFlash && *useFlash) { + if (!flash_) openSd(); + } + + explicit operator bool() const { return flash_ || static_cast(sd_); } + + bool seekSet(size_t offset) { + position_ = offset; + return true; + } + + int read(void* data, size_t length) { + if (flash_ && position_ + length <= flashBytes_) { + if (SdCardFontCache::readAt(position_, data, length, flashBytes_)) { + position_ += length; + return static_cast(length); + } + LOG_ERR("SDCF", "Flash font copy read failed at %u; falling back to SD", static_cast(position_)); + flash_ = false; + *useFlash_ = false; + } + if (!openSd()) return -1; + if (sdPosition_ != position_ && !sd_.seekSet(position_)) return -1; + const int got = sd_.read(data, length); + if (got > 0) position_ += static_cast(got); + sdPosition_ = position_; + return got; + } + + bool close() { + flash_ = false; + return sd_ ? sd_.close() : true; + } + + private: + bool openSd() { + if (sd_) return true; + sdPosition_ = 0; + return Storage.openFileForRead("SDCF", path_, sd_); + } + + const char* path_; + bool* useFlash_; + size_t flashBytes_; + HalFile sd_; + size_t position_ = 0; + size_t sdPosition_ = 0; + bool flash_ = false; +}; + // FNV-1a hash for content-based font ID generation constexpr uint32_t FNV_OFFSET = 2166136261u; constexpr uint32_t FNV_PRIME = 16777619u; @@ -79,8 +140,6 @@ bool collectUniqueCodepoints(const char* text, uint32_t* codepoints, uint32_t& c // resetStyleMiniData retention bounds (see the PerStyle comment in the header). constexpr size_t MINI_RETAIN_MIN_FREE_HEAP = 40 * 1024; constexpr uint8_t MINI_UNDERUSE_RUNS_BEFORE_FREE = 3; -// Working headroom left outside the mini bitmap arena's single contiguous block. -constexpr uint32_t PREWARM_MAX_ALLOC_RESERVE = 4 * 1024; // Keep-if-fits buffer reuse: only reallocate when the needed size exceeds the // current capacity. Freeing + reallocating slightly different sizes every page @@ -104,13 +163,36 @@ SdCardFont::~SdCardFont() { freeAll(); } // --- Per-style free/cleanup --- +namespace { +SdCardFont::MiniFreeEvent miniFreeEventLog[SdCardFont::MINI_FREE_EVENTS]; +uint32_t miniFreeEventTotal = 0; +} // namespace + +const SdCardFont::MiniFreeEvent* SdCardFont::miniFreeEvents(uint32_t& total) { + total = miniFreeEventTotal; + return miniFreeEventLog; +} + void SdCardFont::freeStyleMiniData(PerStyle& s) { + if (s.miniGlyphCount > 0) { + auto& ev = miniFreeEventLog[miniFreeEventTotal % MINI_FREE_EVENTS]; + ev.caller = reinterpret_cast(__builtin_return_address(0)); + ev.ms = millis(); + ev.freeHeap = ESP.getFreeHeap(); + ev.glyphs = static_cast(s.miniGlyphCount); + ev.style = static_cast(&s - styles_); + ev.metadataOnly = s.miniMetadataOnly; + miniFreeEventTotal++; + } psramDeleteArray(s.miniIntervals); s.miniIntervals = nullptr; psramDeleteArray(s.miniGlyphs); s.miniGlyphs = nullptr; - psramDeleteArray(s.miniBitmap); - s.miniBitmap = nullptr; + for (auto& chunk : s.miniBitmapChunks) { + psramDeleteArray(chunk); + chunk = nullptr; + } + s.miniBitmapChunkCount = 0; s.miniIntervalCount = 0; s.miniGlyphCount = 0; s.miniIntervalCapacity = 0; @@ -253,8 +335,8 @@ bool SdCardFont::loadStyleKernLigatureData(PerStyle& s) { return true; } - HalFile file; - if (!Storage.openFileForRead("SDCF", filePath_, file)) { + FontFile file(filePath_, &useFlash_, flashPayloadBytes_); + if (!file) { LOG_ERR("SDCF", "Failed to open .cpfont for kern/lig: %s", filePath_); return false; } @@ -444,8 +526,8 @@ bool SdCardFont::buildMiniKernMatrix(PerStyle& s, const uint32_t* codepoints, ui // Step 6: read the full matrix's rows for each used left class, keep only // columns for used right classes. One SD seek + one read per used left class; // a row is kernRightClassCount bytes (~200 for Literata). - HalFile file; - if (!Storage.openFileForRead("SDCF", filePath_, file)) { + FontFile file(filePath_, &useFlash_, flashPayloadBytes_); + if (!file) { LOG_ERR("SDCF", "Failed to open .cpfont for mini kern: %s", filePath_); freeStyleMiniKern(s); return false; @@ -523,7 +605,96 @@ void SdCardFont::computeStyleFileOffsets(PerStyle& s, uint32_t baseOffset) { // --- Load --- -bool SdCardFont::load(const char* path) { +namespace { +// Floor and ceiling of a / b for b > 0 and any sign of a (C++ division truncates toward zero). +int32_t floorDiv(const int32_t a, const int32_t b) { return a >= 0 ? a / b : -((-a + b - 1) / b); } +int32_t ceilDiv(const int32_t a, const int32_t b) { return a >= 0 ? (a + b - 1) / b : -((-a) / b); } +} // namespace + +int16_t SdCardFont::scaledBearing(const int16_t v) const { + const int32_t mag = (static_cast(v < 0 ? -v : v) * scaleNum_ + scaleDen_ / 2) / scaleDen_; + return static_cast(v < 0 ? -mag : mag); +} + +uint16_t SdCardFont::scaleAdvance(const uint16_t advanceFP) const { + if (!isScaled()) return advanceFP; + return static_cast((static_cast(advanceFP) * scaleNum_ + scaleDen_ / 2) / scaleDen_); +} + +// Rounding the box's size and its origin separately (ceiling and nearest) left the ink up to +// a pixel off its scaled position, and the resample then stretched it to fill the rounded box +// at 1.13-1.17x instead of 9/8. A turned bracket, centred in its cell by that box, sat a pixel +// nearer the column axis than the real 18 pt one (2026-09-25). Scaling the edges and rounding +// outward keeps the ink where the exact scale puts it. +SdCardFont::ScaledBox SdCardFont::scaledBox(const EpdGlyph& g) const { + const int32_t num = scaleNum_; + const int32_t den = scaleDen_; + ScaledBox box{}; + if (g.width == 0 || g.height == 0) { + box.left = scaledBearing(g.left); + box.top = scaledBearing(g.top); + return box; + } + const int32_t x0 = floorDiv(static_cast(g.left) * num, den); + const int32_t x1 = ceilDiv((static_cast(g.left) + g.width) * num, den); + const int32_t y1 = ceilDiv(static_cast(g.top) * num, den); // top edge, up is + + const int32_t y0 = floorDiv((static_cast(g.top) - g.height) * num, den); // bottom edge + box.left = static_cast(x0); + box.top = static_cast(y1); + box.width = static_cast(std::min(x1 - x0, 255)); + box.height = static_cast(std::min(y1 - y0, 255)); + return box; +} + +void SdCardFont::scaleGlyphMetrics(EpdGlyph& g) const { + if (!isScaled()) return; + const ScaledBox box = scaledBox(g); + g.width = static_cast(box.width); + g.height = static_cast(box.height); + g.left = box.left; + g.top = box.top; + g.advanceX = scaleAdvance(g.advanceX); +} + +// Each destination pixel centre is carried back through the exact ratio into the source +// glyph's coordinates (source pixel centres at integers, in 8.8 fixed point) and the four +// neighbours are blended; neighbours outside the source box count as blank ink, which is +// what they are. Integer only: the ESP32-C3 has no FPU. +void SdCardFont::resampleBitmap2Bit(const uint8_t* src, const EpdGlyph& g, uint8_t* dst, const ScaledBox& box) const { + memset(dst, 0, bitmapBytes2Bit(box.width, box.height)); + const int32_t srcW = g.width; + const int32_t srcH = g.height; + if (srcW == 0 || srcH == 0 || box.width == 0 || box.height == 0) return; + const int32_t num = scaleNum_; + const int32_t den = scaleDen_; + const auto sample = [&](const int32_t x, const int32_t y) -> uint32_t { + if (x < 0 || y < 0 || x >= srcW || y >= srcH) return 0; + const uint32_t pos = static_cast(y * srcW + x); + return (src[pos >> 2] >> ((3 - (pos & 3)) * 2)) & 0x3; + }; + for (int32_t dy = 0; dy < box.height; dy++) { + // Row dy's centre sits (box.top - dy - 0.5) above the baseline in scaled pixels; source + // row j's centre sits (g.top - j - 0.5) above it in source pixels. + const int32_t fy = + static_cast(g.top) * 256 - floorDiv((2 * (box.top - dy) - 1) * den * 128, num) - 128; // 8.8 + const int32_t y0 = fy >> 8; + const uint32_t wy = static_cast(fy & 255); + for (int32_t dx = 0; dx < box.width; dx++) { + const int32_t fx = + floorDiv((2 * (box.left + dx) + 1) * den * 128, num) - static_cast(g.left) * 256 - 128; // 8.8 + const int32_t x0 = fx >> 8; + const uint32_t wx = static_cast(fx & 255); + const uint32_t v = sample(x0, y0) * (256 - wx) * (256 - wy) + sample(x0 + 1, y0) * wx * (256 - wy) + + sample(x0, y0 + 1) * (256 - wx) * wy + sample(x0 + 1, y0 + 1) * wx * wy; // level * 65536 + const uint32_t level = (v + 32768) >> 16; // 0..3 + const uint32_t pos = static_cast(dy * box.width + dx); + dst[pos >> 2] |= static_cast(level << ((3 - (pos & 3)) * 2)); + } + } +} + +bool SdCardFont::load(const char* path, const bool isReaderFont, const uint8_t scaleNum, const uint8_t scaleDen, + const bool preferFlash) { freeAll(); if (strlen(path) >= sizeof(filePath_)) { LOG_ERR("SDCF", "Path too long (%zu bytes, max %zu)", strlen(path), sizeof(filePath_) - 1); @@ -532,8 +703,30 @@ bool SdCardFont::load(const char* path) { strncpy(filePath_, path, sizeof(filePath_) - 1); filePath_[sizeof(filePath_) - 1] = '\0'; - HalFile file; - if (!Storage.openFileForRead("SDCF", path, file)) { + const unsigned long start = millis(); + flashPayloadBytes_ = 0; + useFlash_ = preferFlash && SdCardFontCache::isValidFor(path, &flashPayloadBytes_); + if (loadSelectedSource(isReaderFont, scaleNum, scaleDen)) { + LOG_INF("SDCF", "Loaded %s from %s in %lu ms", path, useFlash_ ? "flash" : "sd", millis() - start); + return true; + } + if (!useFlash_) return false; + + // The copy claimed to match the file but could not be read through: try the card itself. + LOG_ERR("SDCF", "Flash copy of %s is unreadable; retrying from SD", path); + freeAll(); + useFlash_ = false; + flashPayloadBytes_ = 0; + return loadSelectedSource(isReaderFont, scaleNum, scaleDen); +} + +bool SdCardFont::loadSelectedSource(const bool isReaderFont, const uint8_t scaleNum, const uint8_t scaleDen) { + const char* path = filePath_; + scaleNum_ = (scaleNum == 0 || scaleDen == 0) ? 1 : scaleNum; + scaleDen_ = (scaleNum == 0 || scaleDen == 0) ? 1 : scaleDen; + + FontFile file(filePath_, &useFlash_, flashPayloadBytes_); + if (!file) { LOG_ERR("SDCF", "Failed to open .cpfont: %s", path); return false; } @@ -600,6 +793,19 @@ bool SdCardFont::load(const char* path) { s.header.kernRightClassCount = tocBuf[22]; s.header.ligaturePairCount = tocBuf[23]; s.header.is2Bit = is2Bit; + if (isScaled() && !is2Bit) { + LOG_ERR("SDCF", "Scale %u/%u ignored: %s is not a 2-bit font", scaleNum_, scaleDen_, path); + scaleNum_ = scaleDen_ = 1; + } + if (isScaled()) { + // Line metrics scale with the glyphs, so the style reads as a font of the scaled size. + s.header.advanceY = static_cast( + std::min((static_cast(s.header.advanceY) * scaleNum_ + scaleDen_ / 2) / scaleDen_, 255)); + s.header.ascender = scaledBearing(s.header.ascender); + s.header.descender = scaledBearing(s.header.descender); + LOG_INF("SDCF", "Scaling %s by %u/%u: advY=%u asc=%d desc=%d", path, scaleNum_, scaleDen_, s.header.advanceY, + s.header.ascender, s.header.descender); + } // Sanity-check counts to reject malformed files before allocating. // Kern class counts are uint8 (bounded by type). Entry counts are uint16 @@ -776,13 +982,27 @@ bool SdCardFont::load(const char* path) { loaded_ = true; LOG_DBG("SDCF", "Loaded: %s (v%u, %u styles)", path, CPFONT_VERSION, styleCount_); + uint8_t firstAdvanceY = 0; + uint32_t firstGlyphCount = 0; + uint32_t residentBytes = 0; for (uint8_t i = 0; i < MAX_STYLES; i++) { if (!styles_[i].present) continue; const auto& h = styles_[i].header; + residentBytes += + h.intervalCount * static_cast(styles_[i].intervalsAreBmp16 ? sizeof(PerStyle::BmpInterval16) + : sizeof(EpdUnicodeInterval)); + if (firstAdvanceY == 0) { + firstAdvanceY = h.advanceY; + firstGlyphCount = h.glyphCount; + } LOG_DBG("SDCF", " style[%u]: %u intervals, %u glyphs, advY=%u, asc=%d, desc=%d, kernL=%u, kernR=%u, ligs=%u", i, h.intervalCount, h.glyphCount, h.advanceY, h.ascender, h.descender, h.kernLeftEntryCount, h.kernRightEntryCount, h.ligaturePairCount); } + if (isReaderFont) { + InputDiag::noteFontChoice(path, styleCount_, firstAdvanceY, firstGlyphCount, residentBytes, useFlash_, + flashPayloadBytes_, scaleNum_, scaleDen_); + } return true; } @@ -859,7 +1079,10 @@ int SdCardFont::prewarm(TextGetter getter, const void* ctx, uint32_t textCount, } } } - if (cpBudget == 0) return -1; + if (cpBudget == 0) { + prewarmEntryFails_++; + return -1; + } // Step 1: Extract unique codepoints from the UTF-8 texts (shared across all styles). // Dedup uses O(n^2) linear scan — worst case is MAX_PAGE_GLYPHS (512) unique codepoints @@ -870,6 +1093,7 @@ int SdCardFont::prewarm(TextGetter getter, const void* ctx, uint32_t textCount, std::unique_ptr codepoints(new (std::nothrow) uint32_t[MAX_PAGE_GLYPHS]); if (!codepoints) { LOG_ERR("SDCF", "Failed to allocate codepoint buffer (%u bytes)", MAX_PAGE_GLYPHS * 4); + prewarmEntryFails_++; return -1; } uint32_t cpCount = 0; @@ -957,21 +1181,24 @@ int SdCardFont::prewarm(TextGetter getter, const void* ctx, uint32_t textCount, if (!(styleMask & (1 << si)) || !styles_[si].present) continue; int missedForStyle = prewarmStyle(si, codepoints.get(), cpCount, metadataOnly, loadKernLig, accumulate); if (missedForStyle == PREWARM_ARENA_TOO_LARGE) { - // The arena is one contiguous block, so a fragmented heap can fail it with - // ample free bytes. Retry with the estimated largest prefix, backing off - // if variable-size glyphs made that estimate too large. - const uint32_t perGlyph = styles_[si].measuredBytesPerGlyph > 0 ? styles_[si].measuredBytesPerGlyph : 1; - const uint32_t maxAlloc = ESP.getMaxAllocHeap(); - const uint32_t arenaBytes = maxAlloc > PREWARM_MAX_ALLOC_RESERVE ? maxAlloc - PREWARM_MAX_ALLOC_RESERVE : 0; - uint32_t fit = arenaBytes / perGlyph; - if (fit > cpCount) fit = cpCount; - while (fit > 0) { - LOG_DBG("SDCF", "Arena retry: %u -> %u glyphs (%uB/glyph, maxAlloc=%u)", cpCount, fit, perGlyph, maxAlloc); + // The chunked arena ran out (a 4 KB chunk could not be allocated, or the chunk cap + // was reached) partway through the page. Rather than leaving every glyph to the + // 8-slot overflow ring, keep the prefix that fit: codepoints are sorted, so the + // prefix keeps kana and punctuation, and the dropped suffix falls to the ring. + // Sized from what was placed, then halved while it still does not fit. + constexpr uint32_t ARENA_RETRY_FLOOR = 16; + uint32_t fit = arenaValidGlyphs_ > 0 + ? static_cast(static_cast(cpCount) * arenaFitGlyphs_ / arenaValidGlyphs_) + : cpCount / 2; + if (fit >= cpCount) fit = cpCount / 2; + while (fit >= ARENA_RETRY_FLOOR) { + LOG_DBG("SDCF", "Arena retry: %u -> %u glyphs (placed %u/%u, maxAlloc=%u)", cpCount, fit, arenaFitGlyphs_, + arenaValidGlyphs_, static_cast(ESP.getMaxAllocHeap())); missedForStyle = prewarmStyle(si, codepoints.get(), fit, metadataOnly, loadKernLig, accumulate); if (missedForStyle != PREWARM_ARENA_TOO_LARGE) break; fit /= 2; } - if (missedForStyle == PREWARM_ARENA_TOO_LARGE) { + if (missedForStyle == PREWARM_ARENA_TOO_LARGE || fit < ARENA_RETRY_FLOOR) { missedForStyle = static_cast(cpCount); // nothing resident } else { missedForStyle += static_cast(cpCount - fit); // the dropped suffix is absent too @@ -1026,6 +1253,16 @@ int SdCardFont::prewarmStyle(uint8_t styleIdx, const uint32_t* codepoints, uint3 } } + // Past the resident-subset check, so everything below is a rebuild. Counted and timed on the + // way out because no other figure records one: the reads go straight into the arena rather + // than the overflow ring, and the buffers are reused rather than dropped. + struct RebuildTimer { + SdCardFont& self; + unsigned long startMs; + ~RebuildTimer() { self.miniRebuildMs_ += static_cast(millis() - startMs); } + } rebuildTimer{*this, millis()}; + miniRebuilds_++; + // Trim oversized buffers only on a cache miss. Trimming when a scope closes // would discard a freshly prefetched page before its actual draw. Count each // rebuild once, retaining the arena until several pages use less than 3/4. @@ -1183,8 +1420,8 @@ int SdCardFont::prewarmStyle(uint8_t styleIdx, const uint32_t* codepoints, uint3 std::sort(readOrder.get(), readOrder.get() + validCount, [&](uint32_t a, uint32_t b) { return mappings[a].globalIndex < mappings[b].globalIndex; }); - HalFile file; - if (!Storage.openFileForRead("SDCF", filePath_, file)) { + FontFile file(filePath_, &useFlash_, flashPayloadBytes_); + if (!file) { LOG_ERR("SDCF", "Failed to reopen .cpfont for prewarm (style %u)", styleIdx); freeStyleMiniData(s); return static_cast(cpCount); @@ -1224,56 +1461,30 @@ int SdCardFont::prewarmStyle(uint8_t styleIdx, const uint32_t* codepoints, uint3 // Mapping is no longer needed once glyph metadata has been read. mappings.reset(); - uint32_t totalBitmapSize = 0; + + if (metadataOnly && isScaled()) { + // No bitmap pass follows, so the records take their scaled metrics here; the bitmap pass + // below scales each record after it has resampled that record's bitmap instead. + for (uint32_t i = 0; i < validCount; i++) scaleGlyphMetrics(s.miniGlyphs[i]); + } + + // Scaled fonts read each source bitmap into this scratch before resampling it into the arena. + std::unique_ptr scaleScratch; + uint32_t scaleScratchBytes = 0; if (!metadataOnly) { - // Compute total bitmap size + // The scratch holds one source bitmap at a time, so it only needs the largest. for (uint32_t i = 0; i < validCount; i++) { - totalBitmapSize += s.miniGlyphs[i].dataLength; - } - - // Rounded up: the retry multiplies this back out to size a glyph set, and a - // floored figure can yield an arena that still does not fit. - if (validCount > 0) s.measuredBytesPerGlyph = (totalBitmapSize + validCount - 1) / validCount; - - // Release temporary allocations before replacing the large bitmap so their - // holes can coalesce with the old arena. Recreate only the small read order. - if (totalBitmapSize > s.miniBitmapCapacity) { - readOrder.reset(); - psramDeleteArray(s.miniBitmap); - s.miniBitmap = nullptr; - s.miniBitmapCapacity = 0; - if (ESP.getMaxAllocHeap() < totalBitmapSize) { - // Growing metadata can split the old bitmap's free region. Reserve the - // bitmap before rebuilding that metadata; the reserved capacity makes - // this a single extra pass, with no recursive growth on the next pass. + if (s.miniGlyphs[i].dataLength > scaleScratchBytes) scaleScratchBytes = s.miniGlyphs[i].dataLength; + } + if (isScaled() && scaleScratchBytes > 0) { + scaleScratch = makeUniqueNoThrow(scaleScratchBytes); + if (!scaleScratch) { + LOG_ERR("SDCF", "Prewarm: failed to allocate %u-byte scale scratch (style %u)", scaleScratchBytes, styleIdx); freeStyleMiniData(s); - file = HalFile{}; - bool bitmapReady = ensureArrayCapacity(s.miniBitmap, s.miniBitmapCapacity, totalBitmapSize); - if (!bitmapReady && hasAdvanceTable()) { - // Layout advances are rebuildable; release them before falling back to fewer glyphs. - const uint32_t largestBefore = ESP.getMaxAllocHeap(); - clearPersistentCache(); - LOG_DBG("SDCF", "Retrying bitmap after advance-cache release: maxAlloc=%u -> %u", largestBefore, - ESP.getMaxAllocHeap()); - bitmapReady = ensureArrayCapacity(s.miniBitmap, s.miniBitmapCapacity, totalBitmapSize); - } - if (!bitmapReady) { - LOG_ERR("SDCF", "Failed to reserve mini bitmap (%u bytes) for style %u", totalBitmapSize, styleIdx); - return PREWARM_ARENA_TOO_LARGE; - } - LOG_DBG("SDCF", "Reserved bitmap before metadata rebuild (%u bytes)", totalBitmapSize); - stats_.sdReadTimeMs += millis() - sdStart; - stats_.seekCount += seekCount; - return prewarmStyle(styleIdx, codepoints, cpCount, metadataOnly, loadKernLig, accumulate); + return static_cast(cpCount); } } - if (!ensureArrayCapacity(s.miniBitmap, s.miniBitmapCapacity, totalBitmapSize)) { - LOG_ERR("SDCF", "Failed to allocate mini bitmap (%u bytes) for style %u", totalBitmapSize, styleIdx); - freeStyleMiniData(s); - return PREWARM_ARENA_TOO_LARGE; - } - s.miniBitmapUsed = totalBitmapSize; // underuse-hysteresis signal for the next cache miss if (!readOrder) { readOrder = makeUniqueNoThrow(validCount); @@ -1289,16 +1500,59 @@ int SdCardFont::prewarmStyle(uint8_t styleIdx, const uint32_t* codepoints, uint3 std::sort(readOrder.get(), readOrder.get() + validCount, [&](uint32_t a, uint32_t b) { return s.miniGlyphs[a].dataOffset < s.miniGlyphs[b].dataOffset; }); - uint32_t miniBitmapOffset = 0; + // Assemble the bitmap arena from on-demand 4 KB chunks (see MINI_BM_CHUNK_*). + // `span` is the running virtual offset into the arena; each glyph is placed + // wholly within one chunk, skipping past a boundary rather than straddling it. + // glyph.dataOffset becomes the virtual offset, decoded at render time by + // miniGlyphBitmap(). Chunks are allocated only as span reaches them and kept + // for the next page, so a sparse page never allocates the full ceiling. + uint32_t span = 0; uint32_t lastBitmapEnd = UINT32_MAX; for (uint32_t i = 0; i < validCount; i++) { uint32_t mapIdx = readOrder[i]; EpdGlyph& glyph = s.miniGlyphs[mapIdx]; - - if (glyph.dataLength == 0) { - glyph.dataOffset = miniBitmapOffset; + // Source bytes on the card, and the bytes the arena holds (they differ when scaled). + const uint32_t srcLen = glyph.dataLength; + const ScaledBox dstBox = isScaled() ? scaledBox(glyph) : ScaledBox{}; + const uint32_t len = isScaled() ? bitmapBytes2Bit(dstBox.width, dstBox.height) : srcLen; + + if (srcLen == 0) { + glyph.dataOffset = span; + scaleGlyphMetrics(glyph); continue; } + if (len > MINI_BM_CHUNK_SIZE) { + // A glyph larger than one chunk cannot be placed; real fonts never get near it. + LOG_ERR("SDCF", "Prewarm: glyph %u B exceeds chunk %u B (style %u)", len, MINI_BM_CHUNK_SIZE, styleIdx); + freeStyleMiniData(s); + return static_cast(cpCount); + } + + // Skip to the next chunk boundary if this glyph would straddle one. + const uint32_t within = span & (MINI_BM_CHUNK_SIZE - 1); + if (within != 0 && within + len > MINI_BM_CHUNK_SIZE) { + span += MINI_BM_CHUNK_SIZE - within; + } + const uint32_t chunkIdx = span >> MINI_BM_CHUNK_SHIFT; + if (chunkIdx >= MINI_BM_MAX_CHUNKS) { + LOG_ERR("SDCF", "Prewarm: mini bitmap needs > %u chunks (style %u)", MINI_BM_MAX_CHUNKS, styleIdx); + arenaFitGlyphs_ = i; + arenaValidGlyphs_ = validCount; + freeStyleMiniData(s); + return PREWARM_ARENA_TOO_LARGE; + } + if (!s.miniBitmapChunks[chunkIdx]) { + s.miniBitmapChunks[chunkIdx] = psramNewArray(MINI_BM_CHUNK_SIZE); + if (!s.miniBitmapChunks[chunkIdx]) { + LOG_ERR("SDCF", "Failed to allocate mini bitmap chunk %u (style %u)", chunkIdx, styleIdx); + arenaFitGlyphs_ = i; + arenaValidGlyphs_ = validCount; + freeStyleMiniData(s); + return PREWARM_ARENA_TOO_LARGE; + } + } + if (chunkIdx + 1 > s.miniBitmapChunkCount) s.miniBitmapChunkCount = chunkIdx + 1; + uint8_t* dst = s.miniBitmapChunks[chunkIdx] + (span & (MINI_BM_CHUNK_SIZE - 1)); uint32_t fileOff = s.bitmapFileOffset + glyph.dataOffset; if (fileOff != lastBitmapEnd) { @@ -1310,16 +1564,28 @@ int SdCardFont::prewarmStyle(uint8_t styleIdx, const uint32_t* codepoints, uint3 } seekCount++; } - if (file.read(s.miniBitmap + miniBitmapOffset, glyph.dataLength) != static_cast(glyph.dataLength)) { + if (isScaled()) { + if (file.read(scaleScratch.get(), srcLen) != static_cast(srcLen)) { + LOG_ERR("SDCF", "Prewarm: short bitmap read (style %u)", styleIdx); + freeStyleMiniData(s); + return static_cast(cpCount); + } + resampleBitmap2Bit(scaleScratch.get(), glyph, dst, dstBox); + scaleGlyphMetrics(glyph); + glyph.dataLength = static_cast(len); + } else if (file.read(dst, len) != static_cast(len)) { LOG_ERR("SDCF", "Prewarm: short bitmap read (style %u)", styleIdx); freeStyleMiniData(s); return static_cast(cpCount); } - lastBitmapEnd = fileOff + glyph.dataLength; + lastBitmapEnd = fileOff + srcLen; - glyph.dataOffset = miniBitmapOffset; - miniBitmapOffset += glyph.dataLength; + glyph.dataOffset = span; + span += len; } + // Footprint including inter-chunk padding: the underuse-hysteresis signal. + s.miniBitmapUsed = span; + s.miniBitmapCapacity = s.miniBitmapChunkCount * MINI_BM_CHUNK_SIZE; } uint32_t sdTime = millis() - sdStart; @@ -1341,7 +1607,10 @@ int SdCardFont::prewarmStyle(uint8_t styleIdx, const uint32_t* codepoints, uint3 s.miniMetadataOnly = metadataOnly; s.miniHysteresisPending = !metadataOnly; // one hysteresis evaluation per rebuild memset(&s.miniData, 0, sizeof(s.miniData)); - s.miniData.bitmap = s.miniBitmap; + // The mini bitmap is chunked (non-contiguous), so there is no single base + // pointer. Prewarmed SD glyphs are resolved via SdCardFont::miniGlyphBitmap() + // in the renderer; leave bitmap null so any stray &bitmap[dataOffset] faults. + s.miniData.bitmap = nullptr; s.miniData.glyph = s.miniGlyphs; s.miniData.intervals = s.miniIntervals; s.miniData.intervalCount = s.miniIntervalCount; @@ -1362,7 +1631,7 @@ int SdCardFont::prewarmStyle(uint8_t styleIdx, const uint32_t* codepoints, uint3 stats_.sdReadTimeMs += sdTime; stats_.seekCount += seekCount; stats_.uniqueGlyphs += validCount; - stats_.bitmapBytes += totalBitmapSize; + if (!metadataOnly) stats_.bitmapBytes += s.miniBitmapUsed; return missed; } @@ -1451,6 +1720,91 @@ bool SdCardFont::hasAdvanceTable() const { return false; } +// A section build measures text and never draws it, yet once the advance table is full +// (ADVANCE_CACHE_LIMIT) every uncached codepoint went through the glyph-miss path and pulled its +// bitmap into the overflow ring just to read advanceX. Japanese chapters routinely use more +// distinct characters than the table holds: one 18 pt chapter build read 2,817 bitmaps in a +// single render (full_skips 416) and took 34.6 s. The glyph record is twelve bytes and sits +// before the bitmaps in the file, so read that and nothing else. +uint16_t SdCardFont::readAdvanceOnly(const uint32_t codepoint, uint8_t styleIdx) const { + styleIdx &= (MAX_STYLES - 1); + if (!loaded_ || !styles_[styleIdx].present) return 0; + const auto& s = styles_[styleIdx]; + advanceOnlyCalls_++; + + // Resident arena first: the same interval lookup EpdFont::getGlyph does before it asks for a + // load, so a measurement of a prewarmed glyph costs no SD access. + if (s.miniData.intervals && s.miniData.intervalCount > 0 && s.miniData.glyph) { + const auto* begin = s.miniData.intervals; + const auto* end = begin + s.miniData.intervalCount; + const auto it = std::upper_bound( + begin, end, codepoint, [](const uint32_t value, const EpdUnicodeInterval& iv) { return value < iv.first; }); + if (it != begin) { + const auto& iv = *(it - 1); + if (codepoint <= iv.last) return s.miniData.glyph[iv.offset + (codepoint - iv.first)].advanceX; + } + } + for (uint32_t i = 0; i < overflowCount_; i++) { + if (overflow_[i].codepoint == codepoint && overflow_[i].styleIdx == styleIdx) return overflow_[i].glyph.advanceX; + } + + const uint32_t memoKey = codepoint | (static_cast(styleIdx) << 22) | 0x80000000u; + const uint32_t memoSlot = (codepoint ^ (codepoint >> 9)) & (MEASURE_MEMO_SLOTS - 1); + if (measureMemoKeys_ && measureMemoKeys_[memoSlot] == memoKey) return measureMemoAdvances_[memoSlot]; + + const int32_t globalIdx = findGlobalGlyphIndex(s, codepoint); + if (globalIdx < 0) return 0; + EpdGlyph glyph = {}; + const uint32_t off = s.glyphsFileOffset + static_cast(globalIdx) * sizeof(EpdGlyph); + bool fromFlash = false; + if (useFlash_ && off + sizeof(EpdGlyph) <= flashPayloadBytes_) { + fromFlash = SdCardFontCache::readAt(off, &glyph, sizeof(EpdGlyph), flashPayloadBytes_); + if (!fromFlash) { + LOG_ERR("SDCF", "Flash font copy read failed at %u; falling back to SD", static_cast(off)); + useFlash_ = false; + } + } + if (!fromFlash) { + struct SdTimer { + const SdCardFont& self; + unsigned long startMs; + ~SdTimer() { self.advanceOnlyMs_ += static_cast(millis() - startMs); } + } sdTimer{*this, millis()}; + advanceOnlySdReads_++; + if (!measureFile_) { + measureFile_ = makeUniqueNoThrow(); + if (!measureFile_) return 0; + } + if (!*measureFile_ && !Storage.openFileForRead("SDCF", filePath_, *measureFile_)) return 0; + if (!measureFile_->seekSet(off) || + measureFile_->read(reinterpret_cast(&glyph), sizeof(EpdGlyph)) != sizeof(EpdGlyph)) { + return 0; + } + } + glyph.advanceX = scaleAdvance(glyph.advanceX); + if (!measureMemoKeys_) { + measureMemoKeys_ = makeUniqueNoThrow(MEASURE_MEMO_SLOTS); + measureMemoAdvances_ = makeUniqueNoThrow(MEASURE_MEMO_SLOTS); + if (!measureMemoKeys_ || !measureMemoAdvances_) { + measureMemoKeys_.reset(); + measureMemoAdvances_.reset(); + } else { + memset(measureMemoKeys_.get(), 0, MEASURE_MEMO_SLOTS * sizeof(uint32_t)); + } + } + if (measureMemoKeys_) { + measureMemoKeys_[memoSlot] = memoKey; + measureMemoAdvances_[memoSlot] = glyph.advanceX; + } + return glyph.advanceX; +} + +void SdCardFont::closeMeasureFile() const { + measureFile_.reset(); + measureMemoKeys_.reset(); + measureMemoAdvances_.reset(); +} + uint16_t SdCardFont::getAdvance(uint32_t codepoint, uint8_t style) const { style &= (MAX_STYLES - 1); if (!advanceTable_[style]) return 0; @@ -1477,6 +1831,8 @@ uint16_t SdCardFont::getAdvance(uint32_t codepoint, uint8_t style) const { // Caller owns the codepoints buffer. int SdCardFont::fetchAdvancesForCodepoints(uint32_t* codepoints, uint32_t cpCount, uint8_t styleMask) { int totalMissed = 0; + const unsigned long fetchStartMs = millis(); + advanceFetchCalls_++; for (uint8_t si = 0; si < MAX_STYLES; si++) { if (!(styleMask & (1 << si)) || !styles_[si].present) continue; const auto& s = styles_[si]; @@ -1484,7 +1840,13 @@ int SdCardFont::fetchAdvancesForCodepoints(uint32_t* codepoints, uint32_t cpCoun // Stop fetching once the cache is full — further inserts would be dropped // by the merge anyway. The renderer fast path tolerates missing entries // (returns 0); the slow path is still correct for those codepoints. - if (advanceTableSize_[si] >= ADVANCE_CACHE_LIMIT) continue; + if (advanceTableSize_[si] >= ADVANCE_CACHE_LIMIT) { + // Counted, not just skipped: past the cap every measurement of an uncached codepoint + // falls through to a per-glyph load, so this is the point where a build's font cost + // changes character. + advanceFullSkips_++; + continue; + } // For each codepoint in `codepoints`, skip those already cached, then // resolve to a glyph index. Build a parallel array sorted by glyph index @@ -1527,8 +1889,8 @@ int SdCardFont::fetchAdvancesForCodepoints(uint32_t* codepoints, uint32_t cpCoun [](const CpIdx& a, const CpIdx& b) { return a.glyphIndex < b.glyphIndex; }); // Open file once and read advanceX for each needed glyph. - HalFile file; - if (!Storage.openFileForRead("SDCF", filePath_, file)) { + FontFile file(filePath_, &useFlash_, flashPayloadBytes_); + if (!file) { LOG_ERR("SDCF", "buildAdvanceTable: failed to open .cpfont for style %u", si); continue; } @@ -1558,7 +1920,7 @@ int SdCardFont::fetchAdvancesForCodepoints(uint32_t* codepoints, uint32_t cpCoun } lastReadIndex = gIdx; staged[fetched].codepoint = mappings[i].codepoint; - staged[fetched].advanceX = tempGlyph.advanceX; + staged[fetched].advanceX = scaleAdvance(tempGlyph.advanceX); fetched++; } file.close(); @@ -1574,6 +1936,7 @@ int SdCardFont::fetchAdvancesForCodepoints(uint32_t* codepoints, uint32_t cpCoun ADVANCE_CACHE_LIMIT); } + advanceFetchMs_ += static_cast(millis() - fetchStartMs); return totalMissed; } @@ -1633,8 +1996,9 @@ int SdCardFont::buildAdvanceTable(const char* utf8Text, uint8_t styleMask, const // --- Stats --- void SdCardFont::logStats(const char* label) { - LOG_DBG("SDCF", "[%s] total=%ums sd_read=%ums seeks=%u glyphs=%u bitmap=%u bytes", label, stats_.prewarmTotalMs, - stats_.sdReadTimeMs, stats_.seekCount, stats_.uniqueGlyphs, stats_.bitmapBytes); + LOG_DBG("SDCF", "[%s] source=%s total=%ums read=%ums seeks=%u glyphs=%u bitmap=%u bytes", label, + useFlash_ ? "flash" : "sd", stats_.prewarmTotalMs, stats_.sdReadTimeMs, stats_.seekCount, stats_.uniqueGlyphs, + stats_.bitmapBytes); } void SdCardFont::resetStats() { stats_ = Stats{}; } @@ -1707,8 +2071,8 @@ const EpdGlyph* SdCardFont::onGlyphMiss(void* ctx, uint32_t codepoint) { bool wasAtCapacity = (self->overflowCount_ == OVERFLOW_CAPACITY); // Read glyph metadata into temporary - HalFile file; - if (!Storage.openFileForRead("SDCF", self->filePath_, file)) { + FontFile file(self->filePath_, &self->useFlash_, self->flashPayloadBytes_); + if (!file) { LOG_ERR("SDCF", "Overflow: failed to open .cpfont"); return nullptr; } @@ -1746,6 +2110,24 @@ const EpdGlyph* SdCardFont::onGlyphMiss(void* ctx, uint32_t codepoint) { } } + if (self->isScaled()) { + if (tempBitmap) { + const ScaledBox dstBox = self->scaledBox(tempGlyph); + const uint32_t dstLen = bitmapBytes2Bit(dstBox.width, dstBox.height); + uint8_t* scaledBitmap = psramNewArray(dstLen); + if (!scaledBitmap) { + LOG_ERR("SDCF", "Overflow: failed to allocate %u bytes for scaled U+%04X", dstLen, codepoint); + psramDeleteArray(tempBitmap); + return nullptr; + } + self->resampleBitmap2Bit(tempBitmap, tempGlyph, scaledBitmap, dstBox); + psramDeleteArray(tempBitmap); + tempBitmap = scaledBitmap; + tempGlyph.dataLength = static_cast(dstLen); + } + self->scaleGlyphMetrics(tempGlyph); + } + // All reads succeeded — commit to slot and advance ring buffer if (wasAtCapacity) { psramDeleteArray(self->overflow_[slot].bitmap); @@ -1758,6 +2140,8 @@ const EpdGlyph* SdCardFont::onGlyphMiss(void* ctx, uint32_t codepoint) { self->overflow_[slot].codepoint = codepoint; self->overflow_[slot].styleIdx = styleIdx; + self->overflowLoads_++; + InputDiag::noteGlyphMiss(codepoint, styleIdx); LOG_DBG("SDCF", "Overflow: loaded U+%04X style %u on demand (slot %u/%u)", codepoint, styleIdx, slot, OVERFLOW_CAPACITY); @@ -1780,4 +2164,14 @@ const uint8_t* SdCardFont::getOverflowBitmap(const EpdGlyph* glyph) const { return nullptr; } +const uint8_t* SdCardFont::miniGlyphBitmap(const void* ctx, uint32_t dataOffset) const { + const auto* octx = static_cast(ctx); + const PerStyle& s = styles_[octx->styleIdx]; + const uint32_t chunkIdx = dataOffset >> MINI_BM_CHUNK_SHIFT; + if (chunkIdx >= s.miniBitmapChunkCount) return nullptr; + const uint8_t* chunk = s.miniBitmapChunks[chunkIdx]; + if (!chunk) return nullptr; + return chunk + (dataOffset & (MINI_BM_CHUNK_SIZE - 1)); +} + SdCardFont* SdCardFont::fromMissCtx(void* ctx) { return static_cast(ctx)->self; } diff --git a/lib/EpdFont/SdCardFont.h b/lib/EpdFont/SdCardFont.h index 3411c993094..edd23f6aafd 100644 --- a/lib/EpdFont/SdCardFont.h +++ b/lib/EpdFont/SdCardFont.h @@ -1,12 +1,15 @@ #pragma once #include +#include #include #include #include "EpdFont.h" #include "EpdFontData.h" +class HalFile; + // On-disk binary format version for .cpfont files. Defined as a preprocessor // macro (rather than a constexpr) so it can be stringified into the SD-fonts // release URL — see FONT_MANIFEST_URL in FontDownloadActivity.h. No integer @@ -21,11 +24,20 @@ class SdCardFont { public: static constexpr uint16_t MAX_PAGE_GLYPHS = 512; - // prewarmStyle: the bitmap arena did not fit the largest free block. - // Distinct from a missed-glyph count so the caller can retry smaller. - static constexpr int PREWARM_ARENA_TOO_LARGE = -2; static constexpr uint8_t MAX_STYLES = 4; + // Recent frees of a style's mini arena, for INPUT_DIAG (who dropped the page's glyphs, and when). + struct MiniFreeEvent { + uint32_t caller = 0; + uint32_t ms = 0; + uint32_t freeHeap = 0; + uint16_t glyphs = 0; + uint8_t style = 0; + bool metadataOnly = false; + }; + static constexpr uint8_t MINI_FREE_EVENTS = 4; + static const MiniFreeEvent* miniFreeEvents(uint32_t& total); + SdCardFont() = default; ~SdCardFont(); // Owns raw buffers freed in dtor — no shallow-copy semantics. Make any @@ -38,7 +50,19 @@ class SdCardFont { // Load .cpfont file: reads header + intervals into RAM, records file layout offsets. // Supports v4 (multi-style) format. // Returns true on success. - bool load(const char* path); + // isReaderFont: the family's body-text size (its first load), reported by INPUT_DIAG. + // scaleNum/scaleDen: every glyph record (width, height, bearings, advance) and bitmap is + // scaled by this ratio as it enters the resident caches, and the style's line metrics at + // load, so the renderer sees a font of the scaled size. 1/1 = the file as it is. Only 2-bit + // fonts are scaled; a 1-bit font ignores the ratio. Kerning values are not scaled (the CJK + // fonts this is for carry none). + // preferFlash: read through the copy in the inactive OTA slot (SdCardFontCache) when it holds + // this very file; otherwise, and for any range the copy does not hold, the SD card is read. + bool load(const char* path, bool isReaderFont = false, uint8_t scaleNum = 1, uint8_t scaleDen = 1, + bool preferFlash = false); + // True while reads are served from the flash copy (a read failure turns it off for good). + bool usingFlash() const { return useFlash_; } + size_t flashPayloadBytes() const { return flashPayloadBytes_; } // Pre-read glyphs needed for the given UTF-8 text from SD card. // styleMask: bitmask of styles to prewarm (bit 0=regular, 1=bold, 2=italic, 3=bolditalic). @@ -81,6 +105,12 @@ class SdCardFont { // Look up advanceX for a codepoint from the advance table. // Returns the 12.4 fixed-point advance, or 0 if not found. uint16_t getAdvance(uint32_t codepoint, uint8_t style) const; + // Advance of one codepoint the table does not hold, without loading its bitmap: the resident + // arena, then the overflow ring, then the 12-byte glyph record straight from the card. 0 when + // the font has no such glyph (or the read failed), so callers can fall back to a full load. + uint16_t readAdvanceOnly(uint32_t codepoint, uint8_t styleIdx) const; + // Close the file readAdvanceOnly keeps open for the duration of a measuring pass. + void closeMeasureFile() const; // Returns true if advance table is populated for at least one style. bool hasAdvanceTable() const; @@ -126,6 +156,12 @@ class SdCardFont { // Returns the bitmap for an on-demand-loaded (overflow) glyph. const uint8_t* getOverflowBitmap(const EpdGlyph* glyph) const; + // Resolves a prewarmed mini glyph's chunked bitmap. `ctx` is the glyphMissCtx + // (an OverflowContext identifying the style); `dataOffset` is the glyph's + // virtual offset into the style's chunked arena. Returns nullptr if the chunk + // is absent or out of range. Called by GfxRenderer::getGlyphBitmap(). + const uint8_t* miniGlyphBitmap(const void* ctx, uint32_t dataOffset) const; + // Extract SdCardFont* from an opaque glyphMissCtx pointer. // Used by GfxRenderer::getGlyphBitmap() to recover the SdCardFont from EpdFontData::glyphMissCtx. static SdCardFont* fromMissCtx(void* ctx); @@ -141,6 +177,30 @@ class SdCardFont { void resetStats(); const Stats& getStats() const { return stats_; } + // Counters for INPUT_DIAG. Glyphs read one at a time through the overflow ring because no + // prewarm covered them; mini-arena rebuilds (the reads go straight into the arena, so no other + // figure records them) and their time; prewarms that bailed before any style was touched; and + // the advance-table work a section build spends its font time on. + uint32_t overflowLoads() const { return overflowLoads_; } + uint32_t miniRebuilds() const { return miniRebuilds_; } + uint32_t miniRebuildMs() const { return miniRebuildMs_; } + uint32_t prewarmEntryFails() const { return prewarmEntryFails_; } + uint32_t advanceFetchCalls() const { return advanceFetchCalls_; } + uint32_t advanceFetchMs() const { return advanceFetchMs_; } + uint32_t advanceFullSkips() const { return advanceFullSkips_; } + // readAdvanceOnly: calls, the ones that had to read the card, and the time those reads took. + uint32_t advanceOnlyCalls() const { return advanceOnlyCalls_; } + uint32_t advanceOnlySdReads() const { return advanceOnlySdReads_; } + uint32_t advanceOnlyMs() const { return advanceOnlyMs_; } + uint32_t advanceTableMax() const { + uint32_t largest = 0; + for (uint8_t i = 0; i < MAX_STYLES; i++) { + if (advanceTableSize_[i] > largest) largest = advanceTableSize_[i]; + } + return largest; + } + static constexpr uint32_t advanceTableLimit() { return ADVANCE_CACHE_LIMIT; } + // Content hash of the file header + style TOC entries (computed during load). // Used to generate deterministic font IDs for section cache invalidation. uint32_t contentHash() const { return contentHash_; } @@ -162,6 +222,16 @@ class SdCardFont { }; // All per-style data: file offsets, intervals, kern/lig, prewarm cache, EpdFont + // The per-style mini bitmap arena is stored as a list of fixed-size chunks + // instead of one contiguous block: a fragmented heap routinely has the bytes + // but not one block large enough for a page's glyphs, and a failed arena sent + // every glyph through the 8-slot overflow ring. Each glyph's bitmap is placed + // wholly within one chunk (never straddling); its EpdGlyph::dataOffset is a + // virtual offset that maps to (chunk index, offset-in-chunk) via miniGlyphBitmap(). + static constexpr uint32_t MINI_BM_CHUNK_SHIFT = 12; // 4 KB chunks + static constexpr uint32_t MINI_BM_CHUNK_SIZE = 1u << MINI_BM_CHUNK_SHIFT; + static constexpr uint32_t MINI_BM_MAX_CHUNKS = 24; // 96 KB ceiling per style/page + struct PerStyle { CpFontHeader header{}; @@ -225,18 +295,20 @@ class SdCardFont { EpdFontData miniData{}; EpdUnicodeInterval* miniIntervals = nullptr; EpdGlyph* miniGlyphs = nullptr; - uint8_t* miniBitmap = nullptr; + // Chunked mini bitmap arena (see MINI_BM_CHUNK_* above). Chunks are allocated + // on demand during prewarm and kept for reuse; miniBitmapChunkCount is how many are live. + uint8_t* miniBitmapChunks[MINI_BM_MAX_CHUNKS] = {}; + uint32_t miniBitmapChunkCount = 0; uint32_t miniIntervalCount = 0; uint32_t miniGlyphCount = 0; uint32_t miniIntervalCapacity = 0; uint32_t miniGlyphCapacity = 0; + // Bytes held by the live chunks (miniBitmapChunkCount * MINI_BM_CHUNK_SIZE). uint32_t miniBitmapCapacity = 0; // Bitmap bytes the current page actually used (set by prewarmStyle), the // underuse-hysteresis signal; 0 = no bitmap built this scope (metadata-only // prewarm), which leaves the hysteresis counter untouched. uint32_t miniBitmapUsed = 0; - // Exact bitmap bytes per glyph of the last requested set, for the arena retry. - uint32_t measuredBytesPerGlyph = 0; uint8_t miniUnderuseRuns = 0; // True when the resident mini was built metadata-only (no bitmaps): it can // serve metadata requests but a full render request must rebuild. @@ -274,6 +346,34 @@ class SdCardFont { uint8_t styleCount_ = 0; char filePath_[128] = {}; + // Flash copy state (see load()). Mutable: the const read paths flip it off on a read failure. + mutable bool useFlash_ = false; + size_t flashPayloadBytes_ = 0; + bool loadSelectedSource(bool isReaderFont, uint8_t scaleNum, uint8_t scaleDen); + + // Glyph scale applied on the way into the resident caches (see load()). + uint8_t scaleNum_ = 1; + uint8_t scaleDen_ = 1; + bool isScaled() const { return scaleNum_ != scaleDen_; } + // Nearest for bearings, ascender/descender and advances. + int16_t scaledBearing(int16_t v) const; + uint16_t scaleAdvance(uint16_t advanceFP) const; + // The scaled glyph box: the source box's edges multiplied by the exact ratio and rounded + // outward, so the ink keeps its scaled position inside it (never clipped, never shifted). + struct ScaledBox { + int16_t left; + int16_t top; + uint16_t width; + uint16_t height; + }; + ScaledBox scaledBox(const EpdGlyph& g) const; + // width/height/left/top/advanceX only; dataLength/dataOffset are the caller's. + void scaleGlyphMetrics(EpdGlyph& g) const; + static uint32_t bitmapBytes2Bit(uint32_t w, uint32_t h) { return (w * h + 3) / 4; } + // Bilinear resample of a packed 2-bit (4 levels) glyph bitmap `src` (box `g`, unscaled) into + // `dst` (box `box` = scaledBox(g)) at exactly scaleNum_/scaleDen_; dst must hold + // bitmapBytes2Bit(box.width, box.height) bytes and is fully written. + void resampleBitmap2Bit(const uint8_t* src, const EpdGlyph& g, uint8_t* dst, const ScaledBox& box) const; // Overflow context: glyphMissHandler needs to know which style it's serving struct OverflowContext { @@ -313,6 +413,26 @@ class SdCardFont { void mergeIntoAdvanceTable(uint8_t styleIdx, const AdvanceEntry* sortedNew, uint32_t newCount); Stats stats_; + uint32_t overflowLoads_ = 0; + uint32_t miniRebuilds_ = 0; + uint32_t miniRebuildMs_ = 0; + uint32_t prewarmEntryFails_ = 0; + uint32_t advanceFetchCalls_ = 0; + uint32_t advanceFetchMs_ = 0; + uint32_t advanceFullSkips_ = 0; + mutable uint32_t advanceOnlyCalls_ = 0; + mutable uint32_t advanceOnlySdReads_ = 0; + mutable uint32_t advanceOnlyMs_ = 0; + // Kept open across readAdvanceOnly calls while a build measures (closed by closeMeasureFile): + // opening the file by its long name was nearly all of each 3.9 ms read. + mutable std::unique_ptr measureFile_; + // Widths readAdvanceOnly already read from the card during this measuring pass, direct-mapped + // by codepoint (key = codepoint | style << 22 | valid bit). A chapter repeats its characters, so + // most calls past the full advance table are for a character read moments before. 3 KB, freed + // with the file. + static constexpr uint32_t MEASURE_MEMO_SLOTS = 512; + mutable std::unique_ptr measureMemoKeys_; + mutable std::unique_ptr measureMemoAdvances_; uint32_t contentHash_ = 0; bool loaded_ = false; @@ -331,6 +451,13 @@ class SdCardFont { void applyGlyphMissCallback(uint8_t styleIdx); int32_t findGlobalGlyphIndex(const PerStyle& s, uint32_t codepoint) const; int fetchAdvancesForCodepoints(uint32_t* codepoints, uint32_t cpCount, uint8_t styleMask); + // prewarmStyle's return when the chunked arena could not hold the whole set (a chunk + // allocation failed or MINI_BM_MAX_CHUNKS was reached). The style is left freed and + // arenaFitGlyphs_ holds how many of its validGlyphs_ glyphs were placed first, which + // prewarm() uses to size a smaller retry. + static constexpr int PREWARM_ARENA_TOO_LARGE = -2; + uint32_t arenaFitGlyphs_ = 0; + uint32_t arenaValidGlyphs_ = 0; int prewarmStyle(uint8_t styleIdx, const uint32_t* codepoints, uint32_t cpCount, bool metadataOnly, bool loadKernLig, bool accumulate); diff --git a/lib/EpdFont/SdCardFontCache.cpp b/lib/EpdFont/SdCardFontCache.cpp new file mode 100644 index 00000000000..0cdcdd1bb71 --- /dev/null +++ b/lib/EpdFont/SdCardFontCache.cpp @@ -0,0 +1,255 @@ +#include "SdCardFontCache.h" + +#include +#include +#include +#include +#include + +#include +#include + +#include "SdCardFontCacheFormat.h" + +namespace SdCardFontCache { +namespace { + +using sd_card_font_cache_format::Header; + +constexpr size_t CHUNK_SIZE = 4096; +constexpr size_t ERASE_BLOCK_SIZE = 64 * 1024; +// The 0x640000-byte app slot minus the header sector. Fixed here as well as +// derived from the partition so a larger slot on another board cannot silently +// change the format's assumptions. +constexpr size_t MAX_PAYLOAD_SIZE = 6549504; +constexpr size_t CPFONT_HEADER_SIZE = 32; +constexpr size_t CPFONT_TOC_ENTRY_SIZE = 32; +constexpr uint8_t CPFONT_MAGIC[8] = {'C', 'P', 'F', 'O', 'N', 'T', '\0', '\0'}; +constexpr uint32_t FNV_OFFSET = 2166136261u; +constexpr uint32_t FNV_PRIME = 16777619u; + +struct SourceIdentity { + size_t size = 0; + uint32_t contentHash = 0; + // Byte after the regular style's last section: the least a copy must hold. + size_t regularEnd = 0; +}; + +uint16_t readU16(const uint8_t* data) { return static_cast(data[0]) | static_cast(data[1] << 8); } +uint32_t readU32(const uint8_t* data) { + return static_cast(data[0]) | (static_cast(data[1]) << 8) | + (static_cast(data[2]) << 16) | (static_cast(data[3]) << 24); +} + +uint32_t fnv1a(const uint8_t* data, size_t length, uint32_t hash = FNV_OFFSET) { + for (size_t i = 0; i < length; ++i) { + hash ^= data[i]; + hash *= FNV_PRIME; + } + return hash; +} + +// Reads the .cpfont header and style TOC the same way SdCardFont::load() does, +// so contentHash matches SdCardFont::contentHash() for the same file. +bool identifySource(const char* sourcePath, SourceIdentity& identity) { + if (!sourcePath || strlen(sourcePath) >= sizeof(Header{}.sourcePath)) return false; + + HalFile file; + if (!Storage.openFileForRead("SDFCACHE", sourcePath, file)) return false; + + uint8_t data[CPFONT_HEADER_SIZE]; + if (file.read(data, sizeof(data)) != static_cast(sizeof(data)) || + memcmp(data, CPFONT_MAGIC, sizeof(CPFONT_MAGIC)) != 0 || readU16(data + 8) != CPFONT_VERSION || data[12] == 0 || + data[12] > SdCardFont::MAX_STYLES) { + return false; + } + + const uint8_t styleCount = data[12]; + uint32_t hash = fnv1a(data, sizeof(data)); + uint32_t dataOffsets[SdCardFont::MAX_STYLES] = {}; + uint8_t styleIds[SdCardFont::MAX_STYLES] = {}; + for (uint8_t i = 0; i < styleCount; ++i) { + if (file.read(data, CPFONT_TOC_ENTRY_SIZE) != static_cast(CPFONT_TOC_ENTRY_SIZE)) return false; + hash = fnv1a(data, CPFONT_TOC_ENTRY_SIZE, hash); + styleIds[i] = data[0]; + dataOffsets[i] = readU32(data + 24); + } + + identity.size = file.fileSize(); + identity.contentHash = hash; + if (identity.size < CPFONT_HEADER_SIZE + static_cast(styleCount) * CPFONT_TOC_ENTRY_SIZE) return false; + + // The regular style (id 0) ends where the next style's data begins, or at + // the end of the file when it is last (or the only style). + uint32_t regularStart = 0; + bool haveRegular = false; + for (uint8_t i = 0; i < styleCount; ++i) { + if (styleIds[i] == 0) { + regularStart = dataOffsets[i]; + haveRegular = true; + } + } + if (!haveRegular) return false; + size_t regularEnd = identity.size; + for (uint8_t i = 0; i < styleCount; ++i) { + if (styleIds[i] != 0 && dataOffsets[i] > regularStart && dataOffsets[i] < regularEnd) regularEnd = dataOffsets[i]; + } + identity.regularEnd = regularEnd; + return true; +} + +size_t payloadCapacity(const HalOtaSlot& slot) { + return slot.size() > sd_card_font_cache_format::HEADER_AREA_SIZE + ? std::min(slot.size() - sd_card_font_cache_format::HEADER_AREA_SIZE, MAX_PAYLOAD_SIZE) + : 0; +} + +bool readHeader(const HalOtaSlot& slot, Header& header) { + return slot.valid() && slot.size() > sd_card_font_cache_format::HEADER_AREA_SIZE && + slot.read(0, &header, sizeof(header)) && + sd_card_font_cache_format::isHeaderValid(header, payloadCapacity(slot)); +} + +void report(ProgressCallback progress, size_t completed, size_t total, void* context) { + if (progress) progress(completed, total, context); +} + +size_t roundUp(size_t value, size_t alignment) { return (value + alignment - 1) / alignment * alignment; } + +} // namespace + +size_t capacity() { return payloadCapacity(HalOtaSlot::inactive()); } + +bool sourceFits(const char* sourcePath, size_t* payloadBytes) { + if (payloadBytes) *payloadBytes = 0; + SourceIdentity source{}; + if (!identifySource(sourcePath, source)) return false; + const size_t cap = capacity(); + if (source.regularEnd > cap) return false; + if (payloadBytes) *payloadBytes = std::min(source.size, cap); + return true; +} + +bool isValidFor(const char* sourcePath, size_t* payloadBytes) { + if (payloadBytes) *payloadBytes = 0; + + const HalOtaSlot slot = HalOtaSlot::inactive(); + Header header{}; + SourceIdentity source{}; + const bool valid = readHeader(slot, header) && identifySource(sourcePath, source) && + strcmp(header.sourcePath, sourcePath) == 0 && header.sourceSize == source.size && + header.contentHash == source.contentHash && header.payloadSize >= source.regularEnd; + if (valid && payloadBytes) *payloadBytes = header.payloadSize; + return valid; +} + +bool readAt(size_t offset, void* data, size_t length, size_t payloadBytes) { + static const HalOtaSlot slot = HalOtaSlot::inactive(); + if (payloadBytes > payloadCapacity(slot) || + !sd_card_font_cache_format::containsPayloadRange(payloadBytes, offset, length)) { + return false; + } + return length == 0 || slot.read(sd_card_font_cache_format::HEADER_AREA_SIZE + offset, data, length); +} + +Result preload(const char* sourcePath, ProgressCallback progress, void* context) { + SourceIdentity source{}; + if (!identifySource(sourcePath, source)) return Result::InvalidFont; + if (isValidFor(sourcePath)) return Result::AlreadyCached; + + const HalOtaSlot slot = HalOtaSlot::inactive(); + if (!slot.valid() || !slot.safeForScratchWrite()) return Result::NotSafe; + const size_t cap = payloadCapacity(slot); + if (source.regularEnd > cap) return Result::TooLarge; + const size_t payloadSize = std::min(source.size, cap); + + auto buffer = makeUniqueNoThrow(CHUNK_SIZE); + if (!buffer) return Result::Oom; + + HalFile file; + if (!Storage.openFileForRead("SDFCACHE", sourcePath, file)) return Result::OpenFailed; + // Erasing the header sector first invalidates whatever copy was there, so a + // power loss mid-way leaves an invalid cache rather than a half-new one. + if (!slot.erase(0, HalOtaSlot::ERASE_SIZE)) return Result::EraseFailed; + + const size_t total = payloadSize * 2; + uint32_t payloadCrc = UINT32_MAX; + size_t offset = 0; + while (offset < payloadSize) { + const size_t eraseLength = std::min(roundUp(payloadSize - offset, HalOtaSlot::ERASE_SIZE), ERASE_BLOCK_SIZE); + if (!slot.erase(sd_card_font_cache_format::HEADER_AREA_SIZE + offset, eraseLength)) return Result::EraseFailed; + + const size_t blockEnd = std::min(offset + eraseLength, payloadSize); + while (offset < blockEnd) { + const size_t length = std::min(CHUNK_SIZE, blockEnd - offset); + if (file.read(buffer.get(), length) != static_cast(length)) return Result::ReadFailed; + payloadCrc = sd_card_font_cache_format::crc32Update(payloadCrc, buffer.get(), length); + if (!slot.write(sd_card_font_cache_format::HEADER_AREA_SIZE + offset, buffer.get(), length)) { + return Result::WriteFailed; + } + offset += length; + report(progress, offset, total, context); + } + } + payloadCrc ^= UINT32_MAX; + + uint32_t flashCrc = UINT32_MAX; + offset = 0; + while (offset < payloadSize) { + const size_t length = std::min(CHUNK_SIZE, payloadSize - offset); + if (!slot.read(sd_card_font_cache_format::HEADER_AREA_SIZE + offset, buffer.get(), length)) { + return Result::VerifyFailed; + } + flashCrc = sd_card_font_cache_format::crc32Update(flashCrc, buffer.get(), length); + offset += length; + report(progress, payloadSize + offset, total, context); + } + flashCrc ^= UINT32_MAX; + if (flashCrc != payloadCrc) return Result::VerifyFailed; + + Header header{}; + memcpy(header.magic, sd_card_font_cache_format::MAGIC, sizeof(header.magic)); + header.version = sd_card_font_cache_format::VERSION; + header.headerSize = sizeof(header); + header.payloadSize = payloadSize; + header.sourceSize = source.size; + header.contentHash = source.contentHash; + header.payloadCrc = payloadCrc; + strncpy(header.sourcePath, sourcePath, sizeof(header.sourcePath) - 1); + header.headerCrc = sd_card_font_cache_format::headerCrc(header); + if (!slot.write(0, &header, sizeof(header))) return Result::WriteFailed; + + LOG_INF("SDFCACHE", "Cached %s: %u of %u bytes, crc=%08x", sourcePath, static_cast(payloadSize), + static_cast(source.size), static_cast(payloadCrc)); + return Result::Ok; +} + +const char* resultName(Result result) { + switch (result) { + case Result::Ok: + return "ok"; + case Result::AlreadyCached: + return "already_cached"; + case Result::OpenFailed: + return "open_failed"; + case Result::InvalidFont: + return "invalid_font"; + case Result::TooLarge: + return "too_large"; + case Result::NotSafe: + return "not_safe"; + case Result::Oom: + return "oom"; + case Result::EraseFailed: + return "erase_failed"; + case Result::ReadFailed: + return "read_failed"; + case Result::WriteFailed: + return "write_failed"; + case Result::VerifyFailed: + return "verify_failed"; + } + return "unknown"; +} + +} // namespace SdCardFontCache diff --git a/lib/EpdFont/SdCardFontCache.h b/lib/EpdFont/SdCardFontCache.h new file mode 100644 index 00000000000..d716599bc2c --- /dev/null +++ b/lib/EpdFont/SdCardFontCache.h @@ -0,0 +1,47 @@ +#pragma once + +#include +#include + +// A copy of one SD .cpfont in the inactive OTA application slot, read with +// bounded esp_partition_read() calls instead of SD-SPI transactions (about +// 20x less latency per read on the X3). The copy is the leading bytes of the +// file: the whole file when it fits the slot, otherwise as much as fits, which +// must still include the complete regular style. Reads past the copied range +// go to the SD card. Ported and adapted from crossmux (0x1abin/crossmux PR #57). +namespace SdCardFontCache { + +enum class Result { + Ok, + AlreadyCached, + OpenFailed, + InvalidFont, + TooLarge, + NotSafe, + Oom, + EraseFailed, + ReadFailed, + WriteFailed, + VerifyFailed, +}; + +using ProgressCallback = void (*)(size_t completed, size_t total, void* context); + +// Payload bytes the slot can hold (0 when there is no usable inactive slot). +size_t capacity(); +// True when the file's regular style lies within capacity(); `payloadBytes` +// receives the number of leading bytes a copy would hold. +bool sourceFits(const char* sourcePath, size_t* payloadBytes = nullptr); +// True when the slot holds a valid copy of exactly this file (path, size and +// header/TOC hash match); `payloadBytes` receives the copied length. +bool isValidFor(const char* sourcePath, size_t* payloadBytes = nullptr); +// Read `length` bytes at `offset` of the source from the copy. False when the +// range is not within the copied `payloadBytes` or the flash read fails. +bool readAt(size_t offset, void* data, size_t length, size_t payloadBytes); +// Copy the file (or its fitting prefix) into the slot, verify it, then commit +// the header. `progress` runs on the calling task; total = 2 x payload bytes +// (copy, then read-back verification). +Result preload(const char* sourcePath, ProgressCallback progress = nullptr, void* context = nullptr); +const char* resultName(Result result); + +} // namespace SdCardFontCache diff --git a/lib/EpdFont/SdCardFontCacheFormat.h b/lib/EpdFont/SdCardFontCacheFormat.h new file mode 100644 index 00000000000..41782cbf1b9 --- /dev/null +++ b/lib/EpdFont/SdCardFontCacheFormat.h @@ -0,0 +1,60 @@ +#pragma once + +#include +#include +#include + +// On-flash layout of the SD font copy held in the inactive OTA slot. The first +// 4 KiB erase sector carries the header; the payload starts at HEADER_AREA_SIZE +// and is the leading `payloadSize` bytes of the source .cpfont (the whole file +// when it fits, otherwise a prefix that still holds the complete regular style). +// Derived from crossmux's CPSDFC1 layout; the prefix semantics and the source +// size field are this fork's, hence a magic of its own. +namespace sd_card_font_cache_format { + +constexpr size_t HEADER_AREA_SIZE = 4096; +constexpr uint16_t VERSION = 1; +constexpr uint8_t MAGIC[8] = {'O', 'S', 'T', 'S', 'D', 'F', 'C', '1'}; + +struct Header { + uint8_t magic[8]; + uint16_t version; + uint16_t headerSize; + uint32_t payloadSize; // bytes of the source copied to flash (<= sourceSize) + uint32_t sourceSize; // byte size of the source file when copied + uint32_t contentHash; // FNV-1a of the .cpfont header + style TOC + uint32_t payloadCrc; // CRC-32 of the payload as written + uint32_t headerCrc; // CRC-32 of this header with headerCrc = 0 + char sourcePath[128]; +}; +static_assert(sizeof(Header) == 160, "SD-card font cache header layout changed"); + +inline uint32_t crc32Update(uint32_t crc, const void* data, size_t length) { + const auto* bytes = static_cast(data); + for (size_t i = 0; i < length; ++i) { + crc ^= bytes[i]; + for (uint8_t bit = 0; bit < 8; ++bit) { + crc = (crc >> 1) ^ (0xEDB88320u & (0u - (crc & 1u))); + } + } + return crc; +} + +inline uint32_t headerCrc(const Header& header) { + Header copy = header; + copy.headerCrc = 0; + return crc32Update(UINT32_MAX, ©, sizeof(copy)) ^ UINT32_MAX; +} + +inline bool isHeaderValid(const Header& header, size_t payloadCapacity) { + return memcmp(header.magic, MAGIC, sizeof(MAGIC)) == 0 && header.version == VERSION && + header.headerSize == sizeof(Header) && header.payloadSize > 0 && header.payloadSize <= payloadCapacity && + header.payloadSize <= header.sourceSize && + memchr(header.sourcePath, '\0', sizeof(header.sourcePath)) != nullptr && header.headerCrc == headerCrc(header); +} + +inline bool containsPayloadRange(size_t payloadSize, size_t offset, size_t length) { + return offset <= payloadSize && length <= payloadSize - offset; +} + +} // namespace sd_card_font_cache_format diff --git a/lib/EpdFont/SdCardFontManager.cpp b/lib/EpdFont/SdCardFontManager.cpp index 45aa8244c35..828b246e303 100644 --- a/lib/EpdFont/SdCardFontManager.cpp +++ b/lib/EpdFont/SdCardFontManager.cpp @@ -4,6 +4,7 @@ #include #include #include +#include #include SdCardFontManager::~SdCardFontManager() { @@ -28,20 +29,25 @@ int SdCardFontManager::computeFontId(uint32_t contentHash, const char* familyNam return id != 0 ? id : 1; // 0 is reserved as "not found" sentinel } -int SdCardFontManager::loadFile(const SdCardFontFileInfo& file, const char* familyName, GfxRenderer& renderer) { +int SdCardFontManager::loadFile(const SdCardFontFileInfo& file, const char* familyName, GfxRenderer& renderer, + const uint8_t idPointSize, const uint8_t scaleNum, const uint8_t scaleDen, + const bool preferFlash) { auto* font = new (std::nothrow) SdCardFont(); if (!font) { LOG_ERR("SDMGR", "Failed to allocate SdCardFont for %s", file.path.c_str()); return 0; } - if (!font->load(file.path.c_str())) { + // The family's first load is the body-text size; the UI fallback sizes follow it. + if (!font->load(file.path.c_str(), loaded_.empty(), scaleNum, scaleDen, preferFlash)) { LOG_ERR("SDMGR", "Failed to load %s", file.path.c_str()); delete font; return 0; } - int fontId = computeFontId(font->contentHash(), familyName, file.pointSize); + // A scaled load carries its own size code, so its section caches and font id never + // collide with the unscaled file's. + int fontId = computeFontId(font->contentHash(), familyName, idPointSize); // Guard against collision with built-in font IDs (astronomically unlikely // with FNV-1a hashes, but provides a safety net) if (renderer.getFontMap().count(fontId) != 0) { @@ -50,16 +56,43 @@ int SdCardFontManager::loadFile(const SdCardFontFileInfo& file, const char* fami return 0; } renderer.registerSdCardFont(fontId, font); - loaded_.push_back({font, fontId, file.pointSize}); + loaded_.push_back({font, fontId, idPointSize}); - LOG_DBG("SDMGR", "Loaded %s size=%u id=%d styles=%u", file.path.c_str(), file.pointSize, fontId, font->styleCount()); + LOG_DBG("SDMGR", "Loaded %s size=%u id=%d styles=%u scale=%u/%u source=%s", file.path.c_str(), idPointSize, fontId, + font->styleCount(), scaleNum, scaleDen, font->usingFlash() ? "flash" : "sd"); EpdFontFamily fontFamily(font->getEpdFont(0), font->getEpdFont(1), font->getEpdFont(2), font->getEpdFont(3)); renderer.insertFont(fontId, fontFamily); return fontId; } -bool SdCardFontManager::loadFamily(const SdCardFontFamilyInfo& family, GfxRenderer& renderer, uint8_t pointSize) { +SdCardFontManager::CacheCandidate SdCardFontManager::cacheCandidate(const SdCardFontFamilyInfo& family, + const uint8_t pointSize) { + CacheCandidate out; + const SdCardFontFileInfo* own = family.findFile(pointSize); + if (own && SdCardFontCache::sourceFits(own->path.c_str())) { + out.file = own; + return out; + } + // Largest smaller size within the upscale limit (base * 9 >= pointSize * 8, i.e. at most 9/8). + const SdCardFontFileInfo* base = nullptr; + for (const auto& f : family.files) { + if (f.style != 0 || f.pointSize >= pointSize) continue; + if (static_cast(f.pointSize) * 9 < static_cast(pointSize) * 8) continue; + if (!base || f.pointSize > base->pointSize) base = &f; + } + if (base && SdCardFontCache::sourceFits(base->path.c_str())) { + out.file = base; + out.scaleNum = pointSize; + out.scaleDen = base->pointSize; + } + return out; +} + +bool SdCardFontManager::readerFontFromFlash() const { return !loaded_.empty() && loaded_.front().font->usingFlash(); } + +bool SdCardFontManager::loadFamily(const SdCardFontFamilyInfo& family, GfxRenderer& renderer, uint8_t pointSize, + const bool preferFlash) { // Unload any previously loaded family first if (!loadedFamilyName_.empty()) { unloadAll(renderer); @@ -70,13 +103,28 @@ bool SdCardFontManager::loadFamily(const SdCardFontFamilyInfo& family, GfxRender LOG_ERR("SDMGR", "Family %s has no files to load", family.name.c_str()); return false; } + // The size is the selected file's own; the file read may be a smaller one drawn scaled + // when that is what the flash copy holds for this size. + const uint8_t idPointSize = selected->pointSize; + uint8_t scaleNum = 1; + uint8_t scaleDen = 1; + bool fromFlash = false; + if (preferFlash) { + const CacheCandidate cand = cacheCandidate(family, idPointSize); + if (cand.file && SdCardFontCache::isValidFor(cand.file->path.c_str())) { + selected = cand.file; + scaleNum = cand.scaleNum; + scaleDen = cand.scaleDen; + fromFlash = true; + } + } - if (loadFile(*selected, family.name.c_str(), renderer) == 0) { + if (loadFile(*selected, family.name.c_str(), renderer, idPointSize, scaleNum, scaleDen, fromFlash) == 0) { return false; } loadedFamilyName_ = family.name; - loadedPointSize_ = selected->pointSize; + loadedPointSize_ = idPointSize; return true; } @@ -91,7 +139,9 @@ int SdCardFontManager::loadFamilyExtraSize(const SdCardFontFamilyInfo& family, G if (lf.size == pointSize) return lf.fontId; } - return loadFile(*file, family.name.c_str(), renderer); + // The copy holds the reader size; a UI size only ever reads the card. Passing preferFlash + // would open the file once per size just to learn that. + return loadFile(*file, family.name.c_str(), renderer, file->pointSize); } void SdCardFontManager::unloadAll(GfxRenderer& renderer) { diff --git a/lib/EpdFont/SdCardFontManager.h b/lib/EpdFont/SdCardFontManager.h index 282110ca402..6f4000e75c3 100644 --- a/lib/EpdFont/SdCardFontManager.h +++ b/lib/EpdFont/SdCardFontManager.h @@ -20,7 +20,27 @@ class SdCardFontManager { // that exact size is not installed. Only one .cpfont file is loaded; other // sizes remain on disk. This keeps resident interval + kern/ligature tables to // one size's worth of memory. Returns true on success. - bool loadFamily(const SdCardFontFamilyInfo& family, GfxRenderer& renderer, uint8_t pointSize); + // preferFlash: use the copy in the inactive OTA slot when it holds the file + // cacheCandidate() names for this size (possibly a smaller file drawn scaled); + // otherwise the size's own file is read from the card as usual. + bool loadFamily(const SdCardFontFamilyInfo& family, GfxRenderer& renderer, uint8_t pointSize, + bool preferFlash = false); + + // The file the flash copy should hold to serve `pointSize`, and the scale to + // draw it at. The size's own file when its regular style fits the slot; + // otherwise the next smaller size that fits and is within 8/9 of it (only + // 16 pt for 18 pt among the shipped sizes: a larger upscale visibly thickens + // the strokes), scaled up by pointSize/its size. file == nullptr when nothing + // fits. Opens each candidate's header on the card, so not for hot paths. + struct CacheCandidate { + const SdCardFontFileInfo* file = nullptr; + uint8_t scaleNum = 1; + uint8_t scaleDen = 1; + }; + static CacheCandidate cacheCandidate(const SdCardFontFamilyInfo& family, uint8_t pointSize); + + // True when the reader-size font reads from the flash copy. + bool readerFontFromFlash() const; // Additively load the .cpfont of `family` at the exact physical `pointSize` // (used for size-matched CJK UI fallback alongside the reader-size font). @@ -53,7 +73,11 @@ class SdCardFontManager { // Load+register a single .cpfont file and append it to loaded_. // Returns the font id, or 0 on failure (allocation, read, or id collision). - int loadFile(const SdCardFontFileInfo& file, const char* familyName, GfxRenderer& renderer); + // idPointSize is the size the font is registered and identified as (it is the file's own + // size unless the file is loaded scaled, see cacheCandidate()); scaleNum/scaleDen and + // preferFlash go to SdCardFont::load(). + int loadFile(const SdCardFontFileInfo& file, const char* familyName, GfxRenderer& renderer, uint8_t idPointSize, + uint8_t scaleNum = 1, uint8_t scaleDen = 1, bool preferFlash = false); std::string loadedFamilyName_; uint8_t loadedPointSize_ = 0; diff --git a/lib/Epub/Epub.cpp b/lib/Epub/Epub.cpp index 867af47f73a..defe1b170f1 100644 --- a/lib/Epub/Epub.cpp +++ b/lib/Epub/Epub.cpp @@ -14,6 +14,18 @@ #include "Epub/parsers/TocNavParser.h" #include "Epub/parsers/TocNcxParser.h" +#if INPUT_DIAG +#include "../../../src/util/InputDiag.h" +#define COVER_DIAG(fmt, ...) \ + do { \ + char coverDiagBuf[72]; \ + snprintf(coverDiagBuf, sizeof(coverDiagBuf), fmt, ##__VA_ARGS__); \ + InputDiag::noteImageEvent(coverDiagBuf); \ + } while (0) +#else +#define COVER_DIAG(fmt, ...) +#endif + bool Epub::findContentOpfFile(std::string* contentOpfFile, ZipFile* sharedZip) const { const auto containerPath = "META-INF/container.xml"; size_t containerSize; @@ -88,6 +100,8 @@ bool Epub::parseContentOpf(BookMetadataCache::BookMetadata& bookMetadata, const // mark) text renders correctly — the device fonts have no mark positioning. bookMetadata.title = utf8ComposeNfc(opfParser.title); bookMetadata.author = utf8ComposeNfc(opfParser.author); + bookMetadata.titleFileAs = utf8ComposeNfc(opfParser.titleFileAs); + bookMetadata.authorFileAs = utf8ComposeNfc(opfParser.authorFileAs); bookMetadata.language = opfParser.language; if (metadataOnly) { @@ -101,48 +115,30 @@ bool Epub::parseContentOpf(BookMetadataCache::BookMetadata& bookMetadata, const // try extracting the image reference from the guide's cover page XHTML if (bookMetadata.coverItemHref.empty() && !opfParser.guideCoverPageHref.empty()) { LOG_DBG("EBP", "No cover from metadata, trying guide cover page: %s", opfParser.guideCoverPageHref.c_str()); - size_t coverPageSize; - uint8_t* coverPageData = readItemContentsToBytes(opfParser.guideCoverPageHref, &coverPageSize, true); - if (coverPageData) { - const std::string coverPageHtml(reinterpret_cast(coverPageData), coverPageSize); - free(coverPageData); - - // Determine base path of the cover page for resolving relative image references - std::string coverPageBase; - const auto lastSlash = opfParser.guideCoverPageHref.rfind('/'); - if (lastSlash != std::string::npos) { - coverPageBase = opfParser.guideCoverPageHref.substr(0, lastSlash + 1); - } - - // Search for image references: xlink:href="..." (SVG) and src="..." (img) - std::string imageRef; - for (const char* pattern : {"xlink:href=\"", "src=\""}) { - auto pos = coverPageHtml.find(pattern); - while (pos != std::string::npos) { - pos += strlen(pattern); - const auto endPos = coverPageHtml.find('"', pos); - if (endPos != std::string::npos) { - const auto ref = std::string_view{coverPageHtml}.substr(pos, endPos - pos); - // Cover BMP generation supports JPG/PNG only; skip GIF so an unsupported wrapper image - // does not block a later supported cover reference. - if (FsHelpers::hasPngExtension(ref) || FsHelpers::hasJpgExtension(ref)) { - imageRef = ref; - break; - } - } - pos = coverPageHtml.find(pattern, pos); - } - if (!imageRef.empty()) break; - } + const auto fromGuide = findCoverImageInDocument(opfParser.guideCoverPageHref); + if (!fromGuide.empty()) { + bookMetadata.coverItemHref = fromGuide; + LOG_DBG("EBP", "Found cover image from guide: %s", bookMetadata.coverItemHref.c_str()); + } + } - if (!imageRef.empty()) { - bookMetadata.coverItemHref = FsHelpers::normalisePath(FsHelpers::decodeUriEscapes(coverPageBase + imageRef)); - LOG_DBG("EBP", "Found cover image from guide: %s", bookMetadata.coverItemHref.c_str()); - } + // The named cover is an SVG wrapper rather than a picture. Cover BMP generation only + // understands JPEG and PNG, so follow the one inside it. + if (FsHelpers::checkFileExtension(bookMetadata.coverItemHref, ".svg")) { + const auto inWrapper = findCoverImageInDocument(bookMetadata.coverItemHref); + if (inWrapper.empty()) { + LOG_DBG("EBP", "Cover SVG holds no JPEG/PNG reference: %s", bookMetadata.coverItemHref.c_str()); + bookMetadata.coverItemHref.clear(); + } else { + LOG_DBG("EBP", "Cover SVG wraps %s", inWrapper.c_str()); + bookMetadata.coverItemHref = inWrapper; } } + COVER_DIAG("opf cover=%.50s", bookMetadata.coverItemHref.empty() ? "(none)" : bookMetadata.coverItemHref.c_str()); + bookMetadata.textReferenceHref = opfParser.textReferenceHref; + bookMetadata.pageProgressionRtl = opfParser.pageProgressionRtl; if (!opfParser.tocNcxPath.empty()) { tocNcxItem = opfParser.tocNcxPath; @@ -593,14 +589,12 @@ bool Epub::load(const bool buildIfMissing, const bool skipLoadingCss) { return true; } -bool Epub::loadMetadata(std::string& title, std::string& author) { - title.clear(); - author.clear(); +bool Epub::loadMetadata(BookMetadataCache::BookMetadata& out) { + out = BookMetadataCache::BookMetadata{}; auto metadataCache = makeUniqueNoThrow(cachePath); if (metadataCache && metadataCache->load()) { - title = metadataCache->coreMetadata.title; - author = metadataCache->coreMetadata.author; + out = metadataCache->coreMetadata; return true; } if (!metadataCache) { @@ -614,14 +608,9 @@ bool Epub::loadMetadata(std::string& title, std::string& author) { return false; } - BookMetadataCache::BookMetadata metadata; - const bool loaded = parseContentOpf(metadata, /*writeSpineEntries=*/false, /*metadataOnly=*/true, &zip); + const bool loaded = parseContentOpf(out, /*writeSpineEntries=*/false, /*metadataOnly=*/true, &zip); zip.close(); - if (!loaded) return false; - - title = std::move(metadata.title); - author = std::move(metadata.author); - return true; + return loaded; } bool Epub::clearCache() const { @@ -701,7 +690,13 @@ bool Epub::generateCoverBmp(bool cropped, bool originalThresholds) const { if (!Storage.openFileForWrite("EBP", coverJpgTempPath, coverJpg)) { return false; } - readItemContentsToStream(coverImageHref, coverJpg, 1024); + // An extraction that fails (deflate's 32 KB window on a split heap) must not hand an empty + // file to the decoder, which would report it as a broken image. + if (!readItemContentsToStream(coverImageHref, coverJpg, 1024)) { + coverJpg.close(); + Storage.remove(coverJpgTempPath.c_str()); + return false; + } // Explicitly close() file before reopening for reading coverJpg.close(); @@ -736,7 +731,13 @@ bool Epub::generateCoverBmp(bool cropped, bool originalThresholds) const { if (!Storage.openFileForWrite("EBP", coverPngTempPath, coverPng)) { return false; } - readItemContentsToStream(coverImageHref, coverPng, 1024); + // An extraction that fails (deflate's 32 KB window on a split heap) must not hand an empty + // file to the decoder, which would report it as a broken image. + if (!readItemContentsToStream(coverImageHref, coverPng, 1024)) { + coverPng.close(); + Storage.remove(coverPngTempPath.c_str()); + return false; + } // Explicitly close() file before reopening for reading coverPng.close(); @@ -766,6 +767,50 @@ bool Epub::generateCoverBmp(bool cropped, bool originalThresholds) const { return false; } +bool Epub::hasCoverImage() const { + return bookMetadataCache && bookMetadataCache->isLoaded() && !bookMetadataCache->coreMetadata.coverItemHref.empty(); +} + +std::string Epub::findCoverImageInDocument(const std::string& docHref) const { + if (docHref.empty()) return {}; + + size_t docSize; + uint8_t* docData = readItemContentsToBytes(docHref, &docSize, true); + if (!docData) return {}; + const std::string doc(reinterpret_cast(docData), docSize); + free(docData); + + // Relative references resolve against the document's own directory. + std::string base; + const auto lastSlash = docHref.rfind('/'); + if (lastSlash != std::string::npos) { + base = docHref.substr(0, lastSlash + 1); + } + + // xlink:href="..." (SVG ) and src="..." (). Only JPEG and PNG are taken, so a GIF + // ornament earlier in the document does not shadow the picture that follows it. + std::string imageRef; + for (const char* pattern : {"xlink:href=\"", "src=\""}) { + auto pos = doc.find(pattern); + while (pos != std::string::npos) { + pos += strlen(pattern); + const auto endPos = doc.find('"', pos); + if (endPos != std::string::npos) { + const auto ref = std::string_view{doc}.substr(pos, endPos - pos); + if (FsHelpers::hasPngExtension(ref) || FsHelpers::hasJpgExtension(ref)) { + imageRef = ref; + break; + } + } + pos = doc.find(pattern, pos); + } + if (!imageRef.empty()) break; + } + if (imageRef.empty()) return {}; + + return FsHelpers::normalisePath(FsHelpers::decodeUriEscapes(base + imageRef)); +} + std::string Epub::getThumbBmpPath() const { return cachePath + "/thumb_[HEIGHT].bmp"; } std::string Epub::getThumbBmpPath(int height) const { return cachePath + "/thumb_" + std::to_string(height) + ".bmp"; } @@ -803,6 +848,7 @@ bool Epub::generateThumbBmpFromSource(int height) { } bool Epub::generateThumbBmpForCover(int height, const std::string& coverImageHref) const { + COVER_DIAG("thumb h=%d href=%.44s", height, coverImageHref.empty() ? "(none)" : coverImageHref.c_str()); if (coverImageHref.empty()) { LOG_DBG("EBP", "No known cover image for thumbnail"); } else if (FsHelpers::hasJpgExtension(coverImageHref)) { @@ -813,7 +859,14 @@ bool Epub::generateThumbBmpForCover(int height, const std::string& coverImageHre if (!Storage.openFileForWrite("EBP", coverJpgTempPath, coverJpg)) { return false; } - readItemContentsToStream(coverImageHref, coverJpg, 1024); + // An extraction that fails (deflate's 32 KB window on a split heap) must not hand an empty + // file to the decoder, which would report it as a broken image. + if (!readItemContentsToStream(coverImageHref, coverJpg, 1024)) { + COVER_DIAG("thumb extract FAIL jpg"); + coverJpg.close(); + Storage.remove(coverJpgTempPath.c_str()); + return false; + } // Explicitly close() file before reopening for reading coverJpg.close(); @@ -841,6 +894,7 @@ bool Epub::generateThumbBmpForCover(int height, const std::string& coverImageHre Storage.remove(getThumbBmpPath(height).c_str()); } LOG_DBG("EBP", "Generated thumb BMP from JPG cover image, success: %s", success ? "yes" : "no"); + COVER_DIAG("thumb jpg decode %s", success ? "ok" : "FAIL"); return success; } else if (FsHelpers::hasPngExtension(coverImageHref)) { LOG_DBG("EBP", "Generating thumb BMP from PNG cover image"); @@ -850,7 +904,14 @@ bool Epub::generateThumbBmpForCover(int height, const std::string& coverImageHre if (!Storage.openFileForWrite("EBP", coverPngTempPath, coverPng)) { return false; } - readItemContentsToStream(coverImageHref, coverPng, 1024); + // An extraction that fails (deflate's 32 KB window on a split heap) must not hand an empty + // file to the decoder, which would report it as a broken image. + if (!readItemContentsToStream(coverImageHref, coverPng, 1024)) { + COVER_DIAG("thumb extract FAIL png"); + coverPng.close(); + Storage.remove(coverPngTempPath.c_str()); + return false; + } // Explicitly close() file before reopening for reading coverPng.close(); @@ -876,6 +937,7 @@ bool Epub::generateThumbBmpForCover(int height, const std::string& coverImageHre Storage.remove(getThumbBmpPath(height).c_str()); } LOG_DBG("EBP", "Generated thumb BMP from PNG cover image, success: %s", success ? "yes" : "no"); + COVER_DIAG("thumb png decode %s", success ? "ok" : "FAIL"); return success; } else { LOG_ERR("EBP", "Cover image is not a supported format, skipping thumbnail"); @@ -1007,7 +1069,11 @@ int Epub::getSpineIndexForTocIndex(const int tocIndex) const { return spineIndex; } -int Epub::getTocIndexForSpineIndex(const int spineIndex) const { return getSpineItem(spineIndex).tocIndex; } +int Epub::getTocIndexForSpineIndex(const int spineIndex) const { + // The reader uses spineCount as the end-of-book position, not a chapter. + if (spineIndex < 0 || spineIndex >= getSpineItemsCount()) return -1; + return getSpineItem(spineIndex).tocIndex; +} size_t Epub::getBookSize() const { if (!bookMetadataCache || !bookMetadataCache->isLoaded() || bookMetadataCache->getSpineCount() == 0) { @@ -1050,6 +1116,9 @@ float Epub::calculateProgress(const int currentSpineIndex, const float currentSp if (bookSize == 0) { return 0.0f; } + // Handle the reader's end-of-book position before looking up chapter sizes. + if (currentSpineIndex >= getSpineItemsCount()) return 1.0f; + if (currentSpineIndex < 0) return 0.0f; const size_t prevChapterSize = (currentSpineIndex >= 1) ? getCumulativeSpineItemSize(currentSpineIndex - 1) : 0; const size_t curChapterSize = getCumulativeSpineItemSize(currentSpineIndex) - prevChapterSize; const float sectionProgSize = currentSpineRead * static_cast(curChapterSize); @@ -1083,3 +1152,10 @@ int Epub::resolveHrefToSpineIndex(const std::string& href) const { } return -1; } + +bool Epub::isPageProgressionRtl() const { + if (!bookMetadataCache || !bookMetadataCache->isLoaded()) { + return false; + } + return bookMetadataCache->coreMetadata.pageProgressionRtl; +} diff --git a/lib/Epub/Epub.h b/lib/Epub/Epub.h index ecab22548da..abb1cbd0b74 100644 --- a/lib/Epub/Epub.h +++ b/lib/Epub/Epub.h @@ -47,7 +47,9 @@ class Epub { ~Epub() = default; std::string& getBasePath() { return contentBasePath; } bool load(bool buildIfMissing = true, bool skipLoadingCss = false); - bool loadMetadata(std::string& title, std::string& author); + // Package metadata only (title, author, their file-as sort forms, language): + // the existing cache when there is one, else the OPF parsed up to . + bool loadMetadata(BookMetadataCache::BookMetadata& out); bool clearCache() const; void setupCacheDir() const; const std::string& getCachePath() const; @@ -55,6 +57,15 @@ class Epub { const std::string& getTitle() const; const std::string& getAuthor() const; const std::string& getLanguage() const; + // True if the spine declares page-progression-direction="rtl" (RTL / vertical book). + // Used to auto-detect tategaki (vertical writing). + bool isPageProgressionRtl() const; + // First JPEG/PNG reference inside a cover document (the guide's cover XHTML, or an SVG wrapper + // named as the cover image), resolved against that document's directory; empty when none. + std::string findCoverImageInDocument(const std::string& docHref) const; + // True when the book names a cover image: tells "no cover" apart from "could not build it now", + // which generateThumbBmp() reports the same way. + bool hasCoverImage() const; std::string getCoverBmpPath(bool cropped = false, bool originalThresholds = false) const; bool generateCoverBmp(bool cropped = false, bool originalThresholds = false) const; std::string getThumbBmpPath() const; diff --git a/lib/Epub/Epub/BookMetadataCache.cpp b/lib/Epub/Epub/BookMetadataCache.cpp index c919db461dd..e4dad4f1ee1 100644 --- a/lib/Epub/Epub/BookMetadataCache.cpp +++ b/lib/Epub/Epub/BookMetadataCache.cpp @@ -11,7 +11,12 @@ #include "FsHelpers.h" namespace { -constexpr uint8_t BOOK_CACHE_VERSION = 10; // v10: ignore ambiguous guide text references +// v10: ignore ambiguous guide text references +// v11 (this tree): BookMetadata gains pageProgressionRtl (spine page-progression-direction), for +// tategaki auto-detect. Upstream 1.6.5 is still v10. +// v12: coverItemHref follows an SVG wrapper to the picture inside it +// v13: title and author file-as sort forms (Library reading order) +constexpr uint8_t BOOK_CACHE_VERSION = 13; constexpr char bookBinFile[] = "/book.bin"; constexpr char tmpSpineBinFile[] = "/spine.bin.tmp"; constexpr char tmpTocBinFile[] = "/toc.bin.tmp"; @@ -42,10 +47,17 @@ uint32_t writeTocEntryTo(F& file, const BookMetadataCache::TocEntry& entry) { return pos; } +// Does not check readString()'s bool return: SpineEntry/TocEntry are returned +// by value through the public getSpineEntry()/getTocEntry() API, so signaling +// failure here would mean changing that return type for every caller across +// the reader. A corrupt length still can't crash (readString() itself refuses +// to grow past MAX_SERIALIZED_STRING_LEN), but sequential callers of this +// template during buildBookBin (no reseek between iterations) will desync for +// the rest of that pass, not just this one entry. template BookMetadataCache::SpineEntry readSpineEntryFrom(F& file) { BookMetadataCache::SpineEntry entry; - serialization::readString(file, entry.href); + (void)serialization::readString(file, entry.href); // see comment above serialization::readPod(file, entry.cumulativeSize); serialization::readPod(file, entry.tocIndex); return entry; @@ -54,9 +66,9 @@ BookMetadataCache::SpineEntry readSpineEntryFrom(F& file) { template BookMetadataCache::TocEntry readTocEntryFrom(F& file) { BookMetadataCache::TocEntry entry; - serialization::readString(file, entry.title); - serialization::readString(file, entry.href); - serialization::readString(file, entry.anchor); + (void)serialization::readString(file, entry.title); // see comment above + (void)serialization::readString(file, entry.href); + (void)serialization::readString(file, entry.anchor); serialization::readPod(file, entry.level); serialization::readPod(file, entry.spineIndex); return entry; @@ -193,9 +205,10 @@ bool BookMetadataCache::buildBookBin(const std::string& epubPath, const BookMeta constexpr uint32_t headerASize = sizeof(BOOK_CACHE_VERSION) + /* LUT Offset */ sizeof(uint32_t) + sizeof(spineCount) + sizeof(tocCount); - const uint32_t metadataSize = metadata.title.size() + metadata.author.size() + metadata.language.size() + + const uint32_t metadataSize = metadata.title.size() + metadata.author.size() + metadata.titleFileAs.size() + + metadata.authorFileAs.size() + metadata.language.size() + metadata.coverItemHref.size() + metadata.textReferenceHref.size() + - sizeof(uint32_t) * 5; + sizeof(uint32_t) * 7 + sizeof(metadata.pageProgressionRtl); const uint32_t lutSize = sizeof(uint32_t) * spineCount + sizeof(uint32_t) * tocCount; const uint32_t lutOffset = headerASize + metadataSize; @@ -207,9 +220,12 @@ bool BookMetadataCache::buildBookBin(const std::string& epubPath, const BookMeta // Metadata serialization::writeString(bookOut, metadata.title); serialization::writeString(bookOut, metadata.author); + serialization::writeString(bookOut, metadata.titleFileAs); + serialization::writeString(bookOut, metadata.authorFileAs); serialization::writeString(bookOut, metadata.language); serialization::writeString(bookOut, metadata.coverItemHref); serialization::writeString(bookOut, metadata.textReferenceHref); + serialization::writePod(bookOut, metadata.pageProgressionRtl); // Loop through spine entries, writing LUT positions spineIn.seek(0); @@ -475,11 +491,18 @@ bool BookMetadataCache::load() { serialization::readPod(bookFile, spineCount); serialization::readPod(bookFile, tocCount); - serialization::readString(bookFile, coreMetadata.title); - serialization::readString(bookFile, coreMetadata.author); - serialization::readString(bookFile, coreMetadata.language); - serialization::readString(bookFile, coreMetadata.coverItemHref); - serialization::readString(bookFile, coreMetadata.textReferenceHref); + if (!serialization::readString(bookFile, coreMetadata.title) || + !serialization::readString(bookFile, coreMetadata.author) || + !serialization::readString(bookFile, coreMetadata.titleFileAs) || + !serialization::readString(bookFile, coreMetadata.authorFileAs) || + !serialization::readString(bookFile, coreMetadata.language) || + !serialization::readString(bookFile, coreMetadata.coverItemHref) || + !serialization::readString(bookFile, coreMetadata.textReferenceHref)) { + LOG_ERR("BMC", "Corrupt metadata string length -- treating cache as invalid"); + bookFile.close(); + return false; + } + serialization::readPod(bookFile, coreMetadata.pageProgressionRtl); // Cache cumulative spine sizes in RAM. The progress bar (every render) and percent // jumps otherwise pay 2 seeks + a heap-allocating SpineEntry read per access. Spine diff --git a/lib/Epub/Epub/BookMetadataCache.h b/lib/Epub/Epub/BookMetadataCache.h index c9d545b7063..a6c1e0f2b3e 100644 --- a/lib/Epub/Epub/BookMetadataCache.h +++ b/lib/Epub/Epub/BookMetadataCache.h @@ -14,9 +14,16 @@ class BookMetadataCache { struct BookMetadata { std::string title; std::string author; + // Sort forms from the package's file-as metadata (the kana reading of a + // Japanese title or name); empty when the book carries none. + std::string titleFileAs; + std::string authorFileAs; std::string language; std::string coverItemHref; std::string textReferenceHref; + // : marks a right-to-left / vertical book. + // Used to auto-detect tategaki (vertical writing) when the writing-mode is AUTO. + bool pageProgressionRtl = false; }; struct SpineEntry { diff --git a/lib/Epub/Epub/CjkLineBreak.h b/lib/Epub/Epub/CjkLineBreak.h new file mode 100644 index 00000000000..80036c681da --- /dev/null +++ b/lib/Epub/Epub/CjkLineBreak.h @@ -0,0 +1,231 @@ +#pragma once + +#include + +#include + +// Line-break prohibitions between adjacent CJK characters. ParsedText tokenizes CJK text one +// character at a time and consults hasCjkBreakOpportunityBetween() for every adjacent pair; +// a pair with no opportunity stays in one token, which is how a prohibited character is kept +// off the head or foot of a line. The classes follow JLREQ (W3C, Requirements for Japanese Text +// Layout): 3.1.7 lists what cannot start a line (cl-02 to cl-11, cl-13), 3.1.8 what cannot end +// one (cl-01, and cl-12 by the same reasoning), and cl-08 runs are not split. Members are taken +// from Appendix A. https://www.w3.org/TR/jlreq/ +namespace CjkLineBreak { + +inline bool isNoBreakBeforeCjkPunctuation(const uint32_t cp) { + switch (cp) { + case '.': + case ',': + case ':': + case ';': + case '!': + case '?': + case ')': + case ']': + case '}': + case 0x00BB: // » + case 0x2019: // ’ + case 0x201D: // ” + case 0x3001: // 、 + case 0x3002: // 。 + case 0x3009: // 〉 + case 0x300B: // 》 + case 0x300D: // 」 + case 0x300F: // 』 + case 0x3011: // 】 + case 0x3015: // 〕 + case 0x3017: // 〗 + case 0x3019: // 〙 + case 0x301B: // 〛 + case 0xFF01: // ！ + case 0xFF09: // ） + case 0xFF0C: // ， + case 0xFF0E: // ． + case 0xFF1A: // ： + case 0xFF1B: // ； + case 0xFF1F: // ？ + case 0xFF3D: // ］ + case 0xFF5D: // ｝ + // JLREQ 3.1.7 (Characters Not Starting a Line) lists these classes in the rule itself; the + // original set carried only brackets and sentence punctuation. Members follow Appendix A. + case 0x30FB: // ・ (cl-05 middle dots) + case 0xFF65: // ･ + case 0x2010: // ‐ (cl-03 hyphens) + case 0x2013: // – + case 0x301C: // 〜 + case 0x30A0: // ゠ + case 0xFF5E: // ～ + case 0x2014: // — (cl-08 inseparable characters) + case 0x2015: // ― + case 0x2025: // ‥ + case 0x2026: // … + case 0x3005: // 々 (cl-09 iteration marks) + case 0x303B: // 〻 + case 0x309D: // ゝ + case 0x309E: // ゞ + case 0x30FD: // ヽ + case 0x30FE: // ヾ + case 0x30FC: // ー (cl-10 prolonged sound mark) + case 0xFF70: // ｰ + case 0xFF9E: // ﾞ (halfwidth voiced marks attach to the preceding kana) + case 0xFF9F: // ﾟ + case 0x00B0: // ° (cl-13 postfixed abbreviations) + case 0x2030: // ‰ + case 0x2032: // ′ + case 0x2033: // ″ + case 0x2103: // ℃ + case 0xFF05: // ％ + case 0x301F: // 〟 (cl-02 closing quotation marks missing above) + case 0xFF60: // ｠ + case 0xFF61: // ｡ (halfwidth full stop and comma) + case 0xFF64: // ､ + // cl-11 small kana: JLREQ 3.1.7 lists them in the rule itself; the Note records that some + // books relax this together with 々 and ー, but the principle keeps them off the line head. + case 0x3041: // ぁ + case 0x3043: // ぃ + case 0x3045: // ぅ + case 0x3047: // ぇ + case 0x3049: // ぉ + case 0x3063: // っ + case 0x3083: // ゃ + case 0x3085: // ゅ + case 0x3087: // ょ + case 0x308E: // ゎ + case 0x3095: // ゕ + case 0x3096: // ゖ + case 0x30A1: // ァ + case 0x30A3: // ィ + case 0x30A5: // ゥ + case 0x30A7: // ェ + case 0x30A9: // ォ + case 0x30C3: // ッ + case 0x30E3: // ャ + case 0x30E5: // ュ + case 0x30E7: // ョ + case 0x30EE: // ヮ + case 0x30F5: // ヵ + case 0x30F6: // ヶ + case 0x31F0: // ㇰ .. ㇿ (Ainu small katakana) + case 0x31F1: + case 0x31F2: + case 0x31F3: + case 0x31F4: + case 0x31F5: + case 0x31F6: + case 0x31F7: + case 0x31F8: + case 0x31F9: + case 0x31FA: + case 0x31FB: + case 0x31FC: + case 0x31FD: + case 0x31FE: + case 0x31FF: + // cl-04 dividing punctuation beyond ! and ? + case 0x203C: // ‼ + case 0x2047: // ⁇ + case 0x2048: // ⁈ + case 0x2049: // ⁉ + // cl-13 postfixed abbreviations, the rest of Appendix A.13 + case 0x0025: // % + case 0x00A2: // ¢ + case 0xFFE0: // ￠ + case 0x2113: // ℓ + case 0x33CB: // ㏋ + case 0x3303: // ㌃ .. ㍄ squared unit symbols + case 0x330D: + case 0x3314: + case 0x3318: + case 0x3322: + case 0x3323: + case 0x3326: + case 0x3327: + case 0x332B: + case 0x3336: + case 0x333B: + case 0x3349: + case 0x334A: + case 0x334D: + case 0x3351: + case 0x3357: + case 0x338E: + case 0x338F: + case 0x339C: + case 0x339D: + case 0x339E: + case 0x33A1: + case 0x33C4: + return true; + default: + return false; + } +} + +inline bool isNoBreakAfterCjkPunctuation(const uint32_t cp) { + switch (cp) { + case '(': + case '[': + case '{': + case 0x00AB: // « + case 0x2018: // ‘ + case 0x201C: // “ + case 0x3008: // 〈 + case 0x300A: // 《 + case 0x300C: // 「 + case 0x300E: // 『 + case 0x3010: // 【 + case 0x3014: // 〔 + case 0x3016: // 〖 + case 0x3018: // 〘 + case 0x301A: // 〚 + case 0xFF08: // （ + case 0xFF3B: // ［ + case 0xFF5B: // ｛ + case 0x301D: // 〝 (cl-01 opening quotation marks missing above) + case 0xFF5F: // ｟ + case 0x0024: // $ (cl-12 prefixed abbreviations, JLREQ 3.1.8 / Appendix A.12: they belong to the number that + // follows) + case 0x00A3: // £ + case 0x00A5: // ¥ + case 0x20AC: // € + case 0x2116: // № + case 0x0023: // # + case 0xFF03: // ＃ + case 0xFF04: // ＄ + case 0xFFE1: // ￡ + case 0xFFE5: // ￥ + return true; + default: + return false; + } +} + +// cl-08 inseparable characters: a run of the same leader or dash (……, ――) is one mark +// and must not be split. Only same-character pairs qualify; a leader followed by text +// may break after it. +inline bool isCjkInseparablePair(const uint32_t leftCp, const uint32_t rightCp) { + if (leftCp != rightCp) return false; + switch (leftCp) { + case 0x2014: // — + case 0x2015: // ― + case 0x2025: // ‥ + case 0x2026: // … + case 0x3033: // 〳 + case 0x3034: // 〴 + case 0x3035: // 〵 + return true; + default: + return false; + } +} + +inline bool hasCjkBreakOpportunityBetween(const uint32_t leftCp, const uint32_t rightCp) { + if (!utf8IsCjkBreakable(leftCp) && !utf8IsCjkBreakable(rightCp)) return false; + if (isNoBreakAfterCjkPunctuation(leftCp) || isNoBreakBeforeCjkPunctuation(rightCp)) return false; + if (isCjkInseparablePair(leftCp, rightCp)) return false; + if (utf8IsCombiningMark(rightCp)) return false; + return true; +} + +} // namespace CjkLineBreak diff --git a/lib/Epub/Epub/InlineImageToken.h b/lib/Epub/Epub/InlineImageToken.h new file mode 100644 index 00000000000..e5dd1b04fb8 --- /dev/null +++ b/lib/Epub/Epub/InlineImageToken.h @@ -0,0 +1,71 @@ +#pragma once + +#include +#include +#include +#include + +// A character-sized image (EBPAJ "gaiji": at 1em, the odd 2-3em glyph +// image) set inline in vertical text. It rides the column as one word token so it takes the +// cell its author put it in, instead of a column of its own centred on the page. +// +// Encoding: U+F8F0 (private use) then ",,\t". The section cache +// stores word text verbatim, so nothing changes in the block format; TextBlock re-derives the +// image from the text at draw time the way it re-derives tate-chu-yoko and sideways tokens. +// imagePath is the extracted file in the book's cache dir (its .pxc sits beside it), srcPath +// the zip entry for a lazy re-extract if the cache dir was cleaned. +namespace InlineImageToken { +constexpr char MARKER[] = "\xEF\xA3\xB0"; // U+F8F0 +constexpr size_t MARKER_LEN = 3; + +struct Spec { + int16_t width = 0; + int16_t height = 0; + std::string imagePath; + std::string srcPath; +}; + +inline bool is(const char* word) { return word != nullptr && std::memcmp(word, MARKER, MARKER_LEN) == 0; } + +inline std::string encode(const int width, const int height, const std::string& imagePath, const std::string& srcPath) { + std::string s(MARKER, MARKER_LEN); + s += std::to_string(width); + s += ','; + s += std::to_string(height); + s += ','; + s += imagePath; + s += '\t'; + s += srcPath; + return s; +} + +inline bool decode(const char* word, Spec& out) { + if (!is(word)) return false; + const char* p = word + MARKER_LEN; + char* end = nullptr; + const long w = std::strtol(p, &end, 10); + if (end == p || *end != ',') return false; + p = end + 1; + const long h = std::strtol(p, &end, 10); + if (end == p || *end != ',') return false; + p = end + 1; + const char* tab = std::strchr(p, '\t'); + if (tab == nullptr) return false; + out.width = static_cast(w); + out.height = static_cast(h); + out.imagePath.assign(p, static_cast(tab - p)); + out.srcPath.assign(tab + 1); + return out.width > 0 && out.height > 0 && !out.imagePath.empty(); +} + +// Advance along the column (the image's height), or -1 when the word is not an image token. +inline int advance(const char* word) { + if (!is(word)) return -1; + const char* p = word + MARKER_LEN; + char* end = nullptr; + std::strtol(p, &end, 10); + if (end == p || *end != ',') return -1; + const long h = std::strtol(end + 1, nullptr, 10); + return h > 0 ? static_cast(h) : -1; +} +} // namespace InlineImageToken diff --git a/lib/Epub/Epub/ParsedText.cpp b/lib/Epub/Epub/ParsedText.cpp index fd0a49d10eb..eaa2ae50a83 100644 --- a/lib/Epub/Epub/ParsedText.cpp +++ b/lib/Epub/Epub/ParsedText.cpp @@ -1,5 +1,6 @@ #include "ParsedText.h" +#include #include #include #include @@ -14,6 +15,8 @@ #include #include +#include "CjkLineBreak.h" +#include "InlineImageToken.h" #include "TokenBoundary.h" #include "hyphenation/HyphenationCommon.h" #include "hyphenation/Hyphenator.h" @@ -66,72 +69,6 @@ uint32_t lastCodepoint(const std::string_view word) { bool containsSoftHyphen(const std::string_view word) { return word.find(SOFT_HYPHEN_UTF8) != std::string_view::npos; } -bool isNoBreakBeforeCjkPunctuation(const uint32_t cp) { - switch (cp) { - case '.': - case ',': - case ':': - case ';': - case '!': - case '?': - case ')': - case ']': - case '}': - case 0x00BB: // » - case 0x2019: // ’ - case 0x201D: // ” - case 0x3001: // 、 - case 0x3002: // 。 - case 0x3009: // 〉 - case 0x300B: // 》 - case 0x300D: // 」 - case 0x300F: // 』 - case 0x3011: // 】 - case 0x3015: // 〕 - case 0x3017: // 〗 - case 0x3019: // 〙 - case 0x301B: // 〛 - case 0xFF01: // ！ - case 0xFF09: // ） - case 0xFF0C: // ， - case 0xFF0E: // ． - case 0xFF1A: // ： - case 0xFF1B: // ； - case 0xFF1F: // ？ - case 0xFF3D: // ］ - case 0xFF5D: // ｝ - return true; - default: - return false; - } -} - -bool isNoBreakAfterCjkPunctuation(const uint32_t cp) { - switch (cp) { - case '(': - case '[': - case '{': - case 0x00AB: // « - case 0x2018: // ‘ - case 0x201C: // “ - case 0x3008: // 〈 - case 0x300A: // 《 - case 0x300C: // 「 - case 0x300E: // 『 - case 0x3010: // 【 - case 0x3014: // 〔 - case 0x3016: // 〖 - case 0x3018: // 〘 - case 0x301A: // 〚 - case 0xFF08: // （ - case 0xFF3B: // ［ - case 0xFF5B: // ｛ - return true; - default: - return false; - } -} - bool containsCjkBreakableCodepoint(const std::string& text) { const auto* ptr = reinterpret_cast(text.c_str()); while (*ptr) { @@ -154,18 +91,11 @@ uint32_t countCodepoints(const std::string_view text) { return count; } -bool cjkBoundaryAllowsBreak(const uint32_t leftCp, const uint32_t rightCp) { - if (!utf8IsCjkBreakable(leftCp) && !utf8IsCjkBreakable(rightCp)) return false; - if (isNoBreakAfterCjkPunctuation(leftCp) || isNoBreakBeforeCjkPunctuation(rightCp)) return false; - if (utf8IsCombiningMark(rightCp)) return false; - return true; -} - // Korean separates words with spaces, so a boundary touching Hangul is not a gap-less break inside // a line. hangulLineEndBreaks() still lets a Hangul word split there at a line end. bool hasCjkBreakOpportunityBetween(const uint32_t leftCp, const uint32_t rightCp) { if (utf8IsHangul(leftCp) || utf8IsHangul(rightCp)) return false; - return cjkBoundaryAllowsBreak(leftCp, rightCp); + return CjkLineBreak::hasCjkBreakOpportunityBetween(leftCp, rightCp); } // Line-end split points inside a Hangul word, using the CJK boundary rules (no hyphen is drawn). @@ -178,7 +108,7 @@ std::vector hangulLineEndBreaks(const std::string& word) while (*ptr) { const size_t offset = static_cast(ptr - start); const uint32_t cur = utf8NextCodepoint(&ptr); - if ((utf8IsHangul(prev) || utf8IsHangul(cur)) && cjkBoundaryAllowsBreak(prev, cur)) { + if ((utf8IsHangul(prev) || utf8IsHangul(cur)) && CjkLineBreak::hasCjkBreakOpportunityBetween(prev, cur)) { breaks.push_back({offset, false}); } prev = cur; @@ -187,36 +117,44 @@ std::vector hangulLineEndBreaks(const std::string& word) } std::vector cjkCharacterBreakByteOffsets(const std::string& text) { - struct CodepointBoundary { - uint32_t cp; - size_t endOffset; - }; - - std::vector codepoints; - codepoints.reserve(text.size()); - bool hasCjkBreakable = false; - + // Single pass over the codepoints, keeping only the previous one. The break rule + // (hasCjkBreakOpportunityBetween) looks at adjacent pairs, so no per-codepoint table + // is needed. The earlier form materialised every codepoint as an 8-byte record first, + // which for a Japanese paragraph -- one unspaced "word" -- cost 8x the paragraph's + // bytes in one block: 20 KB for a 2.5 KB paragraph, requested mid-build where the + // largest free block was 5 KB. With -fno-exceptions the failed reserve was a + // terminate() (crash 2026-09-17, horizontal-regression-test ch.14). The result vector + // is bounded by one entry per codepoint; pure CJK is 3 bytes each, so reserve a third + // and let a mixed run grow. + std::vector allowedOffsets; const auto* ptr = reinterpret_cast(text.c_str()); const auto* const start = ptr; + uint32_t prevCp = utf8NextCodepoint(&ptr); + if (prevCp == 0) return {}; + size_t prevEnd = static_cast(ptr - start); + bool hasCjkBreakable = utf8IsCjkBreakable(prevCp); + bool reserved = false; while (*ptr) { const uint32_t cp = utf8NextCodepoint(&ptr); if (cp == 0) break; - if (utf8IsCjkBreakable(cp)) { - hasCjkBreakable = true; + if (utf8IsCjkBreakable(cp)) hasCjkBreakable = true; + if (hasCjkBreakOpportunityBetween(prevCp, cp)) { + if (!reserved) { + // First opportunity found: size the result once. Skip the whole per-character + // split when even that block cannot be placed -- the word then stays one token, + // and the line pre-check reports the page instead of the allocator aborting. + const size_t want = text.size() / 3 + 1; + constexpr size_t RESERVE_HEADROOM = 4 * 1024; + if (ESP.getMaxAllocHeap() < want * sizeof(size_t) + RESERVE_HEADROOM) return {}; + allowedOffsets.reserve(want); + reserved = true; + } + allowedOffsets.push_back(prevEnd); } - codepoints.push_back({cp, static_cast(ptr - start)}); - } - - if (!hasCjkBreakable || codepoints.size() < 2) return {}; - - std::vector allowedOffsets; - allowedOffsets.reserve(codepoints.size() - 1); - for (size_t i = 0; i + 1 < codepoints.size(); ++i) { - const uint32_t current = codepoints[i].cp; - const uint32_t next = codepoints[i + 1].cp; - if (!hasCjkBreakOpportunityBetween(current, next)) continue; - allowedOffsets.push_back(codepoints[i].endOffset); + prevCp = cp; + prevEnd = static_cast(ptr - start); } + if (!hasCjkBreakable) return {}; return allowedOffsets; } @@ -440,6 +378,56 @@ bool ParsedText::storeWord(const std::string_view text, WordStore::StoredWord& o return false; } +// Bulk-reserve the per-token parallel arrays before a burst of pushes so they don't repeatedly +// double. Only the std::vector arrays are reserved: words and rubyTexts are std::deque (chunked +// growth, no reserve()/capacity() and no large contiguous reallocation to avoid). wordStyles' +// capacity gauges them all since every push path keeps the arrays in lockstep. +void ParsedText::ensureTokenCapacity(const size_t additionalTokens) { + if (additionalTokens == 0) return; + const size_t requiredSize = words.size() + additionalTokens; + if (wordStyles.capacity() >= requiredSize) return; + + size_t newCapacity = wordStyles.capacity() < 16 ? 16 : wordStyles.capacity(); + while (newCapacity < requiredSize) { + newCapacity *= 2; + } + + wordStyles.reserve(newCapacity); + wordContinues.reserve(newCapacity); + wordNoSpaceBefore.reserve(newCapacity); + wordFocusBoundary.reserve(newCapacity); + wordLinkIds.reserve(newCapacity); + wordVisibleOffsetDeltas.reserve(newCapacity); + if (verticalMode) { + wordVerticalBehaviors.reserve(newCapacity); + } +} + +void ParsedText::addVerticalToken(const std::string_view token, const EpdFontFamily::Style fontStyle, + const VerticalTextUtils::VerticalBehavior vb, const uint32_t visibleTextOffset) { + if (token.empty()) return; + const std::string composed = utf8ComposeNfc(std::string(token)); + WordStore::StoredWord stored; + if (!storeWord(composed, stored)) return; + words.push_back(stored); + wordStyles.push_back(fontStyle); + // Kept in lockstep with words[] so the parallel-array invariant holds; the horizontal-only + // fields are unused by layoutVerticalColumns but must stay the same length. + wordContinues.push_back(false); + wordNoSpaceBefore.push_back(false); + wordFocusBoundary.push_back(0); + wordLinkIds.push_back(0); + wordVerticalBehaviors.push_back(vb); + pushVisibleOffset(visibleTextOffset); + // Ruby is parsed independently of writing mode, so a vertical block can carry annotations even + // though the vertical renderer draws them in its own pass. Keep the array in lockstep + // regardless: once it is non-empty, a missing entry would shift every later word's ruby onto + // its neighbour. + if (!rubyTexts.empty()) { + rubyTexts.push_back(""); + } +} + void ParsedText::addWord(std::string word, const EpdFontFamily::Style fontStyle, const bool underline, const bool attachToPrevious, const uint32_t visibleTextOffset, const uint8_t linkId) { if (word.empty()) return; @@ -473,6 +461,12 @@ void ParsedText::addWord(std::string word, const EpdFontFamily::Style fontStyle, wordNoSpaceBefore.push_back(noSpaceBefore); wordFocusBoundary.push_back(focusBoundary); wordLinkIds.push_back(linkId); + // Vertical layout needs one orientation entry per word. Callers in vertical mode use + // addVerticalToken, but shared markup paths reach addWord too (the
  • bullet); topping + // the array up keeps behaviors[i] describing words[i]. No-op in horizontal mode. + if (verticalMode) { + wordVerticalBehaviors.resize(words.size(), VerticalTextUtils::VerticalBehavior::Upright); + } pushVisibleOffset(tokenOffset); if (padRuby && !rubyTexts.empty()) { rubyTexts.push_back(""); @@ -496,29 +490,6 @@ void ParsedText::addWord(std::string word, const EpdFontFamily::Style fontStyle, effectiveNoSpaceBefore = true; } - // Bulk-reserve the per-token parallel arrays before a burst of pushes so they - // don't repeatedly double. Only the std::vector arrays are reserved: words and - // rubyTexts are std::deque (chunked growth, no reserve()/capacity() and no large - // contiguous reallocation to avoid). wordStyles' capacity gauges them all since - // pushToken() keeps every array in lockstep. - const auto ensureTokenCapacity = [&](const size_t additionalTokens) { - if (additionalTokens == 0) return; - const size_t requiredSize = words.size() + additionalTokens; - if (wordStyles.capacity() >= requiredSize) return; - - size_t newCapacity = wordStyles.capacity() < 16 ? 16 : wordStyles.capacity(); - while (newCapacity < requiredSize) { - newCapacity *= 2; - } - - wordStyles.reserve(newCapacity); - wordContinues.reserve(newCapacity); - wordNoSpaceBefore.reserve(newCapacity); - wordFocusBoundary.reserve(newCapacity); - wordLinkIds.reserve(newCapacity); - wordVisibleOffsetDeltas.reserve(newCapacity); - }; - if (auto breakOffsets = cjkCharacterBreakByteOffsets(word); !breakOffsets.empty()) { // CJK-heavy paragraphs can push hundreds of tiny tokens quickly when CSS toggles // inline styles. Reserve once up front to avoid repeated vector growth reallocations. @@ -709,8 +680,19 @@ void ParsedText::ensureRubyCapacity() { // and no large contiguous reallocation to avoid). Kept for call-site stability. } +bool ParsedText::beginsWithIdeographicSpace() const { return !words.empty() && firstCodepoint(wordAt(0)) == 0x3000; } + int ParsedText::resolveFirstLineIndent(const bool isFirstLine, const GfxRenderer& renderer, const int fontId) const { - if (!isFirstLine || !isNaturalAlign) { + // lineEmitted: a paragraph past the parser's soft-flush threshold reaches this in several + // passes, and every pass would otherwise call its own first line the paragraph's first line + // and indent it again mid-paragraph. + if (!isFirstLine || lineEmitted || !isNaturalAlign) { + return 0; + } + // A leading U+3000 is the indent. Adding ours would set the line in twice -- the shape most + // paperback-derived EPUBs take. An explicit text-indent below still wins: that is the author + // asking for a specific measure rather than relying on the default. + if (!blockStyle.textIndentDefined && beginsWithIdeographicSpace()) { return 0; } if (blockStyle.textIndentDefined) { @@ -724,6 +706,261 @@ int ParsedText::resolveFirstLineIndent(const bool isFirstLine, const GfxRenderer } return 0; } +void ParsedText::layoutVerticalColumns(const GfxRenderer& renderer, const int fontId, const uint16_t columnHeight, + const std::function, uint32_t)>& processColumn, + int* cjkCellWidthMemo, const bool includeLastColumn, + const int8_t characterSpacing) { + if (words.empty()) return; + // Stamped here rather than at construction, as layoutAndExtractLines does: the parser replaces + // blockStyle as CSS resolves, and every column's TextBlock carries it on to the page cache. + blockStyle.characterSpacing = characterSpacing; + + // Load SD-card font advance metrics (no bitmaps) so getTextAdvanceX needs no per-glyph SD I/O. + // Same packed-arena handoff as layoutAndExtractLines. + if (renderer.isSdCardFont(fontId)) { + uint8_t styleMask = 0; + for (auto st : wordStyles) styleMask |= static_cast(1u << (static_cast(st) & 0x03)); + if (styleMask == 0) styleMask = 0x01; + std::vector segments; + std::vector segmentLens; + segments.reserve(wordStore.chunkCount()); + segmentLens.reserve(wordStore.chunkCount()); + for (size_t i = 0; i < wordStore.chunkCount(); ++i) { + const char* data = wordStore.chunkData(i); + if (!data) continue; // retired chunk + segments.push_back(data); + segmentLens.push_back(wordStore.chunkUsed(i)); + } + renderer.ensureSdCardFontReady(fontId, segments.data(), segmentLens.data(), segments.size(), words.size() > 1, + hyphenationEnabled, styleMask); + } + + const auto behaviorAt = [&](const size_t i) { + return i < wordVerticalBehaviors.size() ? wordVerticalBehaviors[i] : VerticalTextUtils::VerticalBehavior::Upright; + }; + + // Reference CJK cell advance from the first Upright word (cannot hardcode "一": it may be + // absent from the advance table). Used as the TateChuYoko cell height and spacing base. + int cjkCharAdvance = 0; + for (size_t i = 0; i < words.size() && cjkCharAdvance == 0; i++) { + if (behaviorAt(i) == VerticalTextUtils::VerticalBehavior::Upright) { + cjkCharAdvance = renderer.getTextAdvanceX(fontId, wordStore.cstr(words[i]), wordStyles[i]); + } + } + if (cjkCharAdvance != 0) { + if (cjkCellWidthMemo != nullptr) *cjkCellWidthMemo = cjkCharAdvance; + } else if (cjkCellWidthMemo != nullptr && *cjkCellWidthMemo > 0) { + cjkCharAdvance = *cjkCellWidthMemo; + } + // Last resort for a paragraph with no upright token at all (pure Latin): the font's own + // full-width cell, not its line height -- the columns around this one are placed on the cell. + if (cjkCharAdvance == 0) cjkCharAdvance = renderer.getCjkCellWidth(fontId); + + // Per-word stacked height including inter-cell spacing. A plain nothrow array: this is the + // largest single allocation the pass makes, and a vector's reserve would end the firmware + // (-fno-exceptions) instead of reporting. On failure the paragraph's text is marked dropped, + // which the parser turns into a failed build like any other layout OOM. + const auto wordHeights = makeUniqueNoThrow(words.size()); + if (!wordHeights) { + LOG_ERR("PTX", "Column layout: no room for %u token heights", static_cast(words.size())); + droppedWords = true; + return; + } + // Character spacing (upstream #3528, -2..+2 px) runs along the column in vertical text: the + // same pixels the horizontal path adds after every glyph go after every cell, whatever sits in + // it. A sideways Latin run is one cell here and takes the spacing once, not per letter -- + // drawTextSideways sets the run itself without tracking, so per-letter spacing would only + // push the next cell away from ink that did not move. + const int track = blockStyle.characterSpacing; + const auto spaced = [track](const int h) { return static_cast(std::max(1, h + track)); }; + for (size_t i = 0; i < words.size(); i++) { + const auto vb = behaviorAt(i); + const char* text = wordStore.cstr(words[i]); + uint16_t baseHeight; + if (vb == VerticalTextUtils::VerticalBehavior::InlineImage || InlineImageToken::is(text)) { + const int adv = InlineImageToken::advance(text); + baseHeight = static_cast(adv > 0 ? adv : cjkCharAdvance); + wordHeights[i] = spaced(baseHeight); + continue; + } + if (vb == VerticalTextUtils::VerticalBehavior::TateChuYoko) { + baseHeight = static_cast(cjkCharAdvance); + } else if (vb == VerticalTextUtils::VerticalBehavior::Upright && + VerticalTextUtils::verticalHalfWidthKind(firstCodepoint(wordAt(i))) != + VerticalTextUtils::HalfWidthKind::None) { + // 約物の二分アキ: half the em, not the glyph's own advance -- the face gives every + // one of these a full-width advance with one half blank. TextBlock::renderVertical + // shortens the same cells by the same rule, so the drawn ink stays where the layout + // put it. Kept off the spacing below: a squeezed mark takes the Upright spacing that + // its neighbours do, so a column's cell count is unchanged by where the marks fall. + baseHeight = static_cast(cjkCharAdvance / 2); + } else { // Upright and Sideways both advance by the glyph's own width + baseHeight = static_cast(renderer.getTextAdvanceX(fontId, text, wordStyles[i])); + } + wordHeights[i] = spaced(baseHeight); + } + + // First-column indent. An explicit text-indent is honoured in both signs: the EBPAJ + // template's hanging indents pair text-indent:-Nem with padding-top:Nem, so the column's + // own top inset (applied by the page when it places the column) leaves exactly the room + // a negative indent takes back, and the clamp only matters for a stylesheet that set one + // without the other. Unlike resolveFirstLineIndent, a positive explicit indent is kept + // when paragraph spacing is on: in the surveyed vertical books the class sits on the + // paragraphs an author singled out, not on every one, so it carries meaning of its own. + // Without an explicit value, one CJK cell, mirroring the spirit of resolveFirstLineIndent: + // only for natural-aligned paragraphs and when paragraph spacing is not used instead. + // isNaturalAlign is set inside the horizontal layout path (not reached here), so + // recompute the condition locally. + const bool naturalAlign = + blockStyle.alignment == CssTextAlign::Justify || + (blockStyle.isRtl ? blockStyle.alignment == CssTextAlign::Right : blockStyle.alignment == CssTextAlign::Left); + // Only for the paragraph's real first column: a soft-flushed paragraph is laid out in several + // passes, each numbering its own columns from zero (see lineEmitted). + int verticalIndent = 0; + if (!lineEmitted) { + if (blockStyle.textIndentDefined) { + verticalIndent = std::max(blockStyle.textIndent, -static_cast(blockStyle.topInset())); + } else if (naturalAlign && !extraParagraphSpacing && !beginsWithIdeographicSpace()) { + verticalIndent = cjkCharAdvance; + } + } + + // First pass: column boundaries. columnEnds[i] is the exclusive end index of column i. + std::vector columnEnds; + { + size_t columnStart = 0; + int currentY = verticalIndent; + for (size_t i = 0; i < words.size(); i++) { + // A token taller than the whole column cannot be made to fit by breaking + // around it, and the test below only fires when something precedes it in + // the column -- so left alone it would be stacked from the column head and + // run off the foot of the page. Give it a column to itself and carry on; + // it still overflows, but by as little as its own excess. The parser keeps + // sideways runs under this size, which is where such a token would + // otherwise come from, so this is the guard rather than the mechanism. + if (wordHeights[i] > columnHeight) { + if (i > columnStart) columnEnds.push_back(i); + columnEnds.push_back(i + 1); + columnStart = i + 1; + currentY = 0; + continue; + } + if (currentY + wordHeights[i] > columnHeight && i > columnStart) { + size_t breakAt = i; + // Kinsoku-head pullback: closing brackets / small kana cannot start a column. An inter-word + // separator is pulled back for the same reason: at the head of a column it reads as an + // indent, while at the foot of the previous one it is invisible. + while (breakAt > columnStart + 1 && + (wordAt(breakAt) == " " || VerticalTextUtils::isKinsokuHead(firstCodepoint(wordAt(breakAt))))) { + breakAt--; + } + // Kinsoku-tail pullback: opening brackets cannot end a column. Bounded loop rather + // than a single step: nested quotes (『「, 「『) are common in fiction, and one step + // left the outer bracket at the foot of the column. + for (int pulled = 0; pulled < 4 && breakAt > columnStart + 1 && + VerticalTextUtils::isKinsokuTail(firstCodepoint(wordAt(breakAt - 1))); + ++pulled) { + breakAt--; + } + columnEnds.push_back(breakAt); + columnStart = breakAt; + currentY = 0; + for (size_t j = columnStart; j <= i; j++) currentY += wordHeights[j]; + continue; + } + currentY += wordHeights[i]; + } + if (columnStart < words.size()) columnEnds.push_back(words.size()); + } + + // Mid-block flushes (includeLastColumn=false) keep the trailing partial column for the next + // call so columns don't come out short at flush boundaries; makePages passes true to flush all. + const size_t totalCols = columnEnds.size(); + const size_t emitCols = (includeLastColumn || totalCols <= 1) ? totalCols : totalCols - 1; + + // Second pass: emit columns. Each column's words share xpos 0; the page positions the column + // (right-to-left) when it composes the block. ypos is the stacking offset within the column. + bool isFirstColumn = true; + size_t emitStart = 0; + for (size_t i = 0; i < emitCols; i++) { + const size_t start = emitStart; + const size_t end = columnEnds[i]; + const size_t count = end - start; + std::vector colWords; + colWords.reserve(count); + for (size_t j = start; j < end; j++) colWords.emplace_back(wordAt(j)); + std::vector colStyles(wordStyles.begin() + start, wordStyles.begin() + end); + std::vector colXpos(count, 0); + std::vector colYpos; + colYpos.reserve(count); + int y = isFirstColumn ? verticalIndent : 0; + for (size_t j = start; j < end; j++) { + colYpos.push_back(static_cast(y)); + y += wordHeights[j]; + } + // Ruby rides along with the words it annotates. rubyTexts is kept in lockstep with + // words[] by addVerticalToken, so the same [start, end) slice lines up; an empty + // rubyTexts means the paragraph has no annotations and the block gets none. + std::vector colRuby; + if (!rubyTexts.empty()) { + colRuby.assign(std::make_move_iterator(rubyTexts.begin() + start), + std::make_move_iterator(rubyTexts.begin() + end)); + } + // Link spans along the column: x is where the linked run starts down the column and width + // how far it runs; addColumnToPage turns them into page rectangles once the column has a place. + std::vector colLinks; + for (size_t j = start; j < end; j++) { + const uint8_t linkId = j < wordLinkIds.size() ? wordLinkIds[j] : 0; + if (linkId == 0 || linkId > linkTargets.size()) continue; + const int top = colYpos[j - start]; + const int bottom = top + wordHeights[j]; + if (!colLinks.empty() && j > start && wordLinkIds[j - 1] == linkId) { + auto& span = colLinks.back(); + span.width = static_cast(bottom - span.x); + continue; + } + colLinks.emplace_back(); + auto& span = colLinks.back(); + strncpy(span.href, linkTargets[linkId - 1].c_str(), sizeof(span.href) - 1); + span.href[sizeof(span.href) - 1] = '\0'; + span.x = static_cast(top); + span.width = static_cast(bottom - top); + span.topLift = 0; + } + auto column = makeUniqueNoThrow(colWords, colXpos, colYpos, colStyles, blockStyle, std::move(colRuby), + static_cast(cjkCharAdvance), std::move(colLinks)); + if (!column || !column->valid()) { + LOG_ERR("PTX", "Column layout: no room for a column of %u tokens", static_cast(count)); + droppedWords = true; + return; + } + const uint32_t columnOffset = visibleOffsetAt(start); + lineEmitted = true; + processColumn(std::move(column), columnOffset); + isFirstColumn = false; + emitStart = end; + } + + // Consume emitted words from every parallel array (keep them the same length). + if (emitStart > 0) { + for (size_t i = 0; i < emitStart; ++i) { + wordStore.release(words[i]); // retires arena chunks as columns are consumed + } + const auto eraseFront = [emitStart](auto& vec) { + if (!vec.empty()) vec.erase(vec.begin(), vec.begin() + std::min(emitStart, vec.size())); + }; + eraseFront(words); + eraseFront(wordStyles); + eraseFront(wordContinues); + eraseFront(wordNoSpaceBefore); + eraseFront(wordFocusBoundary); + eraseFront(wordLinkIds); + eraseFront(wordVerticalBehaviors); + eraseFront(rubyTexts); + eraseVisibleOffsetPrefix(emitStart); + } +} + // Consumes data to minimize memory usage void ParsedText::layoutAndExtractLines(const GfxRenderer& renderer, const int fontId, const uint16_t viewportWidth, const std::function, uint32_t)>& processLine, @@ -1735,6 +1972,7 @@ void ParsedText::extractLine(const size_t breakIndex, const int pageWidth, const droppedWords = true; return; } + lineEmitted = true; processLine(std::move(block), lineVisibleOffset); return; } @@ -1760,5 +1998,6 @@ void ParsedText::extractLine(const size_t breakIndex, const int pageWidth, const droppedWords = true; // see the non-focus branch above return; } + lineEmitted = true; processLine(std::move(block), lineVisibleOffset); } diff --git a/lib/Epub/Epub/ParsedText.h b/lib/Epub/Epub/ParsedText.h index a277aa01a0c..33ed5e908fc 100644 --- a/lib/Epub/Epub/ParsedText.h +++ b/lib/Epub/Epub/ParsedText.h @@ -1,6 +1,7 @@ #pragma once #include +#include #include #include @@ -46,6 +47,9 @@ class ParsedText { // token and discarded after layout, never added to the page-cache TextBlock. std::vector wordLinkIds; std::vector linkTargets; + // Per-word vertical orientation (tategaki). Populated only in vertical mode, in lockstep + // with words[]; empty in horizontal mode. Consumed by layoutVerticalColumns. + std::vector wordVerticalBehaviors; // Zero-based visible Unicode-codepoint offsets in the spine body, stored as // uint16_t deltas from a shared base to keep this layout-only metadata small. // Pathological spans wider than uint16_t use sparse rebases; rendered @@ -63,7 +67,18 @@ class ParsedText { bool extraParagraphSpacing; bool hyphenationEnabled; bool focusReadingEnabled; + // True when this block is laid out as vertical (tategaki) columns. Kept here so addWord can + // maintain the wordVerticalBehaviors invariant even on shared markup paths that don't know + // about vertical mode. + bool verticalMode; bool isNaturalAlign; + // True once this paragraph has handed a line to the page. The parser soft-flushes a paragraph + // over the word threshold, so layout runs several times over the same ParsedText; each run + // numbers its own lines from zero and would otherwise re-apply the first-line indent at every + // flush boundary. Never reset: one ParsedText is one paragraph (startNewTextBlock builds a + // fresh one). + bool lineEmitted = false; + bool topSpacingClaimed = false; bool hasRtlWord; bool droppedWords = false; std::vector reorderedWordsScratch; @@ -85,6 +100,10 @@ class ParsedText { int calculateRubyExtraEndOffset(size_t lineStartIdx, size_t lineBreakIdx, const GfxRenderer& renderer, int fontId) const; int resolveFirstLineIndent(bool isFirstLine, const GfxRenderer& renderer, int fontId) const; + // True when the paragraph already opens with an ideographic space (U+3000), which + // paperback-derived EPUBs use as the first-line indent itself. Adding the reader's own + // indent on top of it sets the line in two characters. + bool beginsWithIdeographicSpace() const; std::vector computeLineBreaks(const GfxRenderer& renderer, int fontId, int pageWidth, std::vector& wordWidths, std::vector& continuesVec, std::vector& noSpaceBeforeVec); @@ -102,11 +121,13 @@ class ParsedText { public: explicit ParsedText(const bool extraParagraphSpacing, const bool hyphenationEnabled = false, - const bool focusReadingEnabled = false, const BlockStyle& blockStyle = BlockStyle()) + const bool focusReadingEnabled = false, const BlockStyle& blockStyle = BlockStyle(), + const bool verticalMode = false) : blockStyle(blockStyle), extraParagraphSpacing(extraParagraphSpacing), hyphenationEnabled(hyphenationEnabled), focusReadingEnabled(focusReadingEnabled), + verticalMode(verticalMode), isNaturalAlign(false), hasRtlWord(false) {} ~ParsedText() = default; @@ -115,6 +136,22 @@ class ParsedText { uint32_t visibleTextOffset = 0, uint8_t linkId = 0); uint8_t addLinkTarget(const char* href); bool linkTargetMatches(uint8_t linkId, const char* href) const; + // Marks tokens [first, size()) as belonging to link target linkId. The vertical tokenizer + // pushes a run of cells at once and tags them afterwards, as the emphasis path does. + void setLinkIdFrom(size_t first, uint8_t linkId) { + for (size_t i = first; i < wordLinkIds.size(); ++i) wordLinkIds[i] = linkId; + } + // Vertical (tategaki) token: one already-tokenized unit (typically a single codepoint) + // plus its orientation class. Bypasses the horizontal focus/bidi machinery -- vertical + // layout stacks tokens down a column and composes columns right-to-left. + void addVerticalToken(std::string_view token, EpdFontFamily::Style fontStyle, VerticalTextUtils::VerticalBehavior vb, + uint32_t visibleTextOffset = 0); + // Reserve the per-token parallel arrays for `additionalTokens` more pushes. Callers that + // append a burst of tokens (a CJK-split word, a buffer of vertical cells) should call this + // first so the arrays grow once instead of doubling repeatedly mid-burst. + void ensureTokenCapacity(size_t additionalTokens); + // Read access to a token's text (the parser's bouten path needs each CJK token's length). + std::string_view tokenAt(size_t index) const { return wordAt(index); } void setRubyForWordAt(size_t index, const std::string& ruby); void setRubyGroupAt(size_t startIndex, size_t count, const std::string& ruby); EpdFontFamily::Style getWordStyleAt(size_t index) const { @@ -126,6 +163,14 @@ class ParsedText { BlockStyle& getBlockStyle() { return blockStyle; } size_t size() const { return words.size(); } bool isEmpty() const { return words.empty(); } + // True the first time only. The block's top margin belongs before its first line, and a + // paragraph past the soft-flush threshold is laid out in several passes -- whichever pass + // comes first claims it, so it is neither lost at the head nor inserted mid-paragraph. + bool claimTopSpacing() { + if (topSpacingClaimed) return false; + topSpacingClaimed = true; + return true; + } // True once any word was dropped because the text arena could not allocate. // Callers must treat the block as incomplete and fail the section build. bool hadDroppedWords() const { return droppedWords; } @@ -133,4 +178,17 @@ class ParsedText { const std::function, uint32_t)>& processLine, bool includeLastLine = true, int8_t characterSpacing = 0, uint8_t wordSpacingPercent = 100); + // Vertical (tategaki) analogue of layoutAndExtractLines: stacks tokens down columns of + // height columnHeight, applying kinsoku at column boundaries, and emits one TextBlock per + // column (words positioned by ypos; the page composes columns right-to-left) with the + // visible offset of its first token. Consumes emitted words like the horizontal path; + // includeLastColumn=false preserves a trailing partial column across mid-block flushes. + // cjkCellWidthMemo carries the full-width cell advance across paragraphs: a block with + // an upright word writes its advance there, and a block without one (a pure-Latin + // paragraph) reads it back instead of falling back to the line height, which is ~40% + // wider and used to push such columns' ruby into the neighbouring column. + void layoutVerticalColumns(const GfxRenderer& renderer, int fontId, uint16_t columnHeight, + const std::function, uint32_t)>& processColumn, + int* cjkCellWidthMemo = nullptr, bool includeLastColumn = true, + int8_t characterSpacing = 0); }; diff --git a/lib/Epub/Epub/ReaderRenderSpec.h b/lib/Epub/Epub/ReaderRenderSpec.h index 70dd4d09d63..6cbbaaf104d 100644 --- a/lib/Epub/Epub/ReaderRenderSpec.h +++ b/lib/Epub/Epub/ReaderRenderSpec.h @@ -23,4 +23,14 @@ struct ReaderRenderSpec { bool embeddedStyle = true; uint8_t imageRendering = 0; bool focusReadingEnabled = false; + // Vertical writing (tategaki / vertical-rl). Resolved per book by the reader (the EPUB's + // page-progression-direction and language) and written into the spec before layout, so + // switching a book between vertical and horizontal invalidates its section cache. + bool isVertical = false; + // White space the reader keeps outside the viewport on the right, in pixels (the user's screen + // margin, not the bezel inset, which is under the frame and cannot be drawn into usefully). + // Vertical ruby is set beside its column and the rightmost column's lands here, so the page + // only has to reserve what this does not already cover. Not a cache key: it only matters to + // vertical layout through the viewport, which already is one. + uint8_t rightMargin = 0; }; diff --git a/lib/Epub/Epub/Section.cpp b/lib/Epub/Epub/Section.cpp index be0240de452..a3d678cfd72 100644 --- a/lib/Epub/Epub/Section.cpp +++ b/lib/Epub/Epub/Section.cpp @@ -9,6 +9,7 @@ #include "../../../src/util/InputDiag.h" #include "Epub/css/CssParser.h" +#include "Epub/css/CssSelectorUsage.h" #include "Page.h" #include "hyphenation/Hyphenator.h" #include "parsers/ChapterHtmlSlimParser.h" @@ -55,7 +56,18 @@ namespace { // v47: Word and character spacing in the header (cache validation); cached BlockStyle stores only character spacing. // v48: Hangul words wrap at spaces; with hyphenation on they may also split at a line end. // Justification no longer stretches between syllables. -constexpr uint8_t SECTION_FILE_VERSION = 48; +// v49: Variation selectors (VS1-VS16, IVS VS17-VS256) are dropped instead of drawn as tofu. +// v50: CJK line-break prohibitions cover the JLREQ classes (hyphens, middle dots, iteration marks, +// prolonged sound mark, small kana, prefixed/postfixed abbreviations), so cached layouts change. +// v52: Vertical writing (tategaki): the header carries isVertical as a cache key; vertical +// TextBlocks serialize an isVertical flag, a per-word ypos array and the full-width cell. +// v51: In-chapter anchors name the page their first line lands on; a soft-flushed paragraph +// is indented once, and not at all on top of a leading U+3000; its top margin goes before +// its first line instead of before the final pass. +// v53: Vertical columns carry link rectangles, so the footnote selector can reach them. +// v54: Images honour max-width/max-height; vertical books set images in the column flow. +// v55: A vertical page holding a single image centres it horizontally. +constexpr uint8_t SECTION_FILE_VERSION = 55; // Written into the version field while a build is in progress; patched to // SECTION_FILE_VERSION only when the build is finalized. An abandoned / // crash-interrupted .bin therefore carries version 0, which loadSectionFile rejects @@ -77,7 +89,7 @@ constexpr uint32_t HEADER_SIZE = sizeof(uint8_t) + sizeof(int) + sizeof(float) + sizeof(uint16_t) + sizeof(uint16_t) + sizeof(uint16_t) + sizeof(bool) + sizeof(bool) + sizeof(uint8_t) + sizeof(bool) + sizeof(uint32_t) + sizeof(uint32_t) + sizeof(uint32_t) + sizeof(uint32_t) + sizeof(uint32_t) + sizeof(int8_t) + - sizeof(uint8_t); + sizeof(uint8_t) + sizeof(bool); } // namespace // Out-of-line so the unique_ptr in BuildContext can be @@ -130,8 +142,9 @@ void Section::writeSectionFileHeader(const ReaderRenderSpec& spec) { sizeof(spec.viewportWidth) + sizeof(spec.viewportHeight) + sizeof(pageCount) + sizeof(spec.hyphenationEnabled) + sizeof(spec.embeddedStyle) + sizeof(spec.imageRendering) + sizeof(spec.focusReadingEnabled) + - sizeof(spec.characterSpacing) + sizeof(spec.wordSpacingPercent) + sizeof(uint32_t) + - sizeof(uint32_t) + sizeof(uint32_t) + sizeof(uint32_t) + sizeof(uint32_t), + sizeof(spec.characterSpacing) + sizeof(spec.wordSpacingPercent) + + sizeof(spec.isVertical) + sizeof(uint32_t) + sizeof(uint32_t) + sizeof(uint32_t) + + sizeof(uint32_t) + sizeof(uint32_t), "Header size mismatch"); // Written as the incomplete sentinel; finalizeBuild() patches it to // SECTION_FILE_VERSION as the last step, committing the file. @@ -148,6 +161,7 @@ void Section::writeSectionFileHeader(const ReaderRenderSpec& spec) { serialization::writePod(file, spec.focusReadingEnabled); serialization::writePod(file, spec.characterSpacing); serialization::writePod(file, spec.wordSpacingPercent); + serialization::writePod(file, spec.isVertical); serialization::writePod(file, pageCount); // Placeholder for page count (will be initially 0, patched later) serialization::writePod(file, static_cast(0)); // Placeholder for LUT offset (patched later) serialization::writePod(file, static_cast(0)); // Placeholder for anchor map offset (patched later) @@ -186,6 +200,7 @@ bool Section::loadSectionFile(const ReaderRenderSpec& spec) { bool fileFocusReadingEnabled; int8_t fileCharacterSpacing; uint8_t fileWordSpacingPercent; + bool fileIsVertical; serialization::readPod(file, fileFontId); serialization::readPod(file, fileLineCompression); serialization::readPod(file, fileExtraParagraphSpacing); @@ -198,13 +213,15 @@ bool Section::loadSectionFile(const ReaderRenderSpec& spec) { serialization::readPod(file, fileFocusReadingEnabled); serialization::readPod(file, fileCharacterSpacing); serialization::readPod(file, fileWordSpacingPercent); + serialization::readPod(file, fileIsVertical); if (spec.fontId != fileFontId || spec.lineCompression != fileLineCompression || spec.extraParagraphSpacing != fileExtraParagraphSpacing || spec.paragraphAlignment != fileParagraphAlignment || spec.viewportWidth != fileViewportWidth || spec.viewportHeight != fileViewportHeight || spec.hyphenationEnabled != fileHyphenationEnabled || spec.embeddedStyle != fileEmbeddedStyle || spec.imageRendering != fileImageRendering || spec.focusReadingEnabled != fileFocusReadingEnabled || - spec.characterSpacing != fileCharacterSpacing || spec.wordSpacingPercent != fileWordSpacingPercent) { + spec.characterSpacing != fileCharacterSpacing || spec.wordSpacingPercent != fileWordSpacingPercent || + spec.isVertical != fileIsVertical) { file.close(); LOG_ERR("SCT", "Deserialization failed: Parameters do not match"); clearCache(); @@ -278,6 +295,11 @@ bool Section::createSectionFile(const ReaderRenderSpec& spec, const std::functio } bool Section::startBuild(const ReaderRenderSpec& spec, const std::function& popupFn) { + // A build measures and never draws, so every uncached codepoint's width comes from the glyph + // record on the card rather than a bitmap load into the overflow ring (see readAdvanceOnly). + // Scoped over the whole build, not just ParsedText's layout: the parser measures too (list + // markers, table cells, inline-image decisions). + const GfxRenderer::MeasureOnlyScope measureOnly(renderer); if (build_) { LOG_ERR("SCT", "startBuild called while a build is already active"); return false; @@ -404,7 +426,13 @@ bool Section::startBuild(const ReaderRenderSpec& spec, const std::functioncssParser = epub->getCssParser(); if (ctx->cssParser) { - const CssParser::CacheLoadResult cacheResult = ctx->cssParser->loadFromCache(); + // Load only the cached rules this chapter can reference: publisher templates (the EBPAJ / + // KADOKAWA ones most Japanese EPUBs carry) register hundreds of rules of which a chapter uses + // a handful, and the full set costs tens of KB right while the build needs the heap. If the + // scan fails, load everything. + CssSelectorUsage usage; + const bool scanned = usage.scanHtmlFile(ctx->parsePath); + const CssParser::CacheLoadResult cacheResult = ctx->cssParser->loadFromCache(scanned ? &usage : nullptr); if (cacheResult == CssParser::CacheLoadResult::LowMemory) { LOG_ERR("SCT", "Insufficient heap to hydrate CSS; section build deferred"); ctx->cssParser->clear(); @@ -440,14 +468,21 @@ bool Section::startBuild(const ReaderRenderSpec& spec, const std::functionparser = makeUniqueNoThrow( epub, ctxPtr->parsePath, renderer, spec.fontId, spec.lineCompression, spec.extraParagraphSpacing, spec.paragraphAlignment, spec.viewportWidth, spec.viewportHeight, spec.hyphenationEnabled, - spec.focusReadingEnabled, - [this, ctxPtr](std::unique_ptr page, const uint16_t paragraphIndex, const uint16_t listItemIndex, - const uint32_t visibleTextOffset) { + spec.focusReadingEnabled, spec.isVertical, + [this, ctxPtr, isVertical = spec.isVertical, viewportWidth = spec.viewportWidth]( + std::unique_ptr page, const uint16_t paragraphIndex, const uint16_t listItemIndex, + const uint32_t visibleTextOffset) { + // Vertical flow places an image at the column cursor, i.e. against the right edge; a + // page holding nothing but that image is an illustration page and centres instead. + if (isVertical && page->elements.size() == 1 && page->elements[0]->getTag() == TAG_PageImage) { + const int width = static_cast(*page->elements[0]).getImageBlock().getWidth(); + if (width < viewportWidth) page->elements[0]->xPos = static_cast((viewportWidth - width) / 2); + } ctxPtr->lut.push_back( {this->onPageComplete(std::move(page)), paragraphIndex, listItemIndex, visibleTextOffset}); }, spec.embeddedStyle, ctxPtr->contentBase, ctxPtr->imageBasePath, spec.imageRendering, std::move(tocAnchors), - popupFn, ctxPtr->cssParser); + popupFn, ctxPtr->cssParser, spec.rightMargin); if (!ctx->parser) { LOG_ERR("SCT", "OOM: ChapterHtmlSlimParser"); if (ctx->cssParser) ctx->cssParser->clear(); @@ -471,6 +506,7 @@ bool Section::startBuild(const ReaderRenderSpec& spec, const std::functionparser) { LOG_ERR("SCT", "buildSomeMore with no active build"); return false; @@ -892,7 +928,10 @@ std::optional Section::getPageForAnchor(const std::string& anchor) con for (uint16_t i = 0; i < count; i++) { std::string key; uint16_t page; - serialization::readString(f, key); + if (!serialization::readString(f, key)) { + LOG_ERR("SCT", "getPageForAnchor: corrupt anchor-map entry %u, aborting scan", i); + return std::nullopt; + } serialization::readPod(f, page); if (key == anchor) { return page; diff --git a/lib/Epub/Epub/blocks/ImageBlock.cpp b/lib/Epub/Epub/blocks/ImageBlock.cpp index 4de55a7885e..5b2e3e16115 100644 --- a/lib/Epub/Epub/blocks/ImageBlock.cpp +++ b/lib/Epub/Epub/blocks/ImageBlock.cpp @@ -1,6 +1,7 @@ #include "ImageBlock.h" #include +#include #include #include #include @@ -12,6 +13,20 @@ #include "Epub/converters/DirectPixelWriter.h" #include "Epub/converters/ImageDecoderFactory.h" +#include "Epub/converters/PngStreamDecoder.h" + +#if INPUT_DIAG +#include "../../../../src/util/InputDiag.h" +// Page-time image outcomes go to /image-diag.txt next to the heap at that moment. +#define IMG_DIAG(fmt, ...) \ + do { \ + char imgDiagBuf[72]; \ + snprintf(imgDiagBuf, sizeof(imgDiagBuf), fmt, ##__VA_ARGS__); \ + InputDiag::noteImageEvent(imgDiagBuf); \ + } while (0) +#else +#define IMG_DIAG(fmt, ...) +#endif // Cache file format: // - uint16_t width @@ -291,6 +306,18 @@ bool renderFromCache(GfxRenderer& renderer, const std::string& cachePath, int x, } // namespace +std::string ImageBlock::cachePathFor(const std::string& imagePath) { return getCachePath(imagePath); } + +bool ImageBlock::hasValidCacheFor(const std::string& imagePath, const int width, const int height) { + const auto cachePath = getCachePath(imagePath); + HalFile cacheFile; + if (!Storage.openFileForRead("IMG", cachePath, cacheFile)) { + return false; + } + uint16_t cachedWidth, cachedHeight; + return readValidCacheHeader(cacheFile, width, height, cachedWidth, cachedHeight); +} + bool ImageBlock::hasValidCache() const { const auto cachePath = getCachePath(imagePath); HalFile cacheFile; @@ -364,6 +391,7 @@ void ImageBlock::render(GfxRenderer& renderer, const int x, const int y) { LOG_DBG("IMG", "Lazy-extracting %s -> %s", srcPath.c_str(), imagePath.c_str()); if (!extractFn(extractCtx, srcPath.c_str(), imagePath.c_str())) { LOG_ERR("IMG", "Lazy extraction failed: %s", srcPath.c_str()); + IMG_DIAG("page img extract FAIL %.36s", srcPath.c_str()); } } @@ -372,6 +400,7 @@ void ImageBlock::render(GfxRenderer& renderer, const int x, const int y) { HalFile file; if (!Storage.openFileForRead("IMG", imagePath, file)) { LOG_ERR("IMG", "Image file not found: %s", imagePath.c_str()); + IMG_DIAG("page img missing %.40s", srcPath.c_str()); rememberImageFailure(imagePath); renderPlaceholder(renderer, x, y); return; @@ -381,11 +410,22 @@ void ImageBlock::render(GfxRenderer& renderer, const int x, const int y) { if (fileSize == 0) { LOG_ERR("IMG", "Image file is empty: %s", imagePath.c_str()); + IMG_DIAG("page img empty %.40s", srcPath.c_str()); rememberImageFailure(imagePath); renderPlaceholder(renderer, x, y); return; } + // Streamed PNG path first: its inflate state is two separate heap blocks of 11KB and 32KB, + // both available mid-read where PNGdec's single ~62KB object (and its 60KB free-heap gate) + // is not -- measured 43-55KB free while a page draws, so every PNG came out as its frame. + // On success the .pxc exists and the cache render takes over, for this pass and every future one. + if (FsHelpers::hasPngExtension(imagePath) && PngStreamDecoder::decodeToCache(imagePath, cachePath, width, height) && + renderFromCache(renderer, cachePath, x, y, width, height)) { + renderer.preserveImagePolarity(x, y, width, height); + return; + } + LOG_DBG("IMG", "Decoding and caching: %s", imagePath.c_str()); RenderConfig config; @@ -412,6 +452,7 @@ void ImageBlock::render(GfxRenderer& renderer, const int x, const int y) { bool success = decoder->decodeToFramebuffer(imagePath, renderer, config); if (!success) { LOG_ERR("IMG", "Failed to decode image: %s", imagePath.c_str()); + IMG_DIAG("page img decode FAIL %dx%d %.30s", width, height, srcPath.c_str()); rememberImageFailure(imagePath); renderPlaceholder(renderer, x, y); return; @@ -419,6 +460,7 @@ void ImageBlock::render(GfxRenderer& renderer, const int x, const int y) { renderer.preserveImagePolarity(x, y, width, height); LOG_DBG("IMG", "Decode successful"); + IMG_DIAG("page img decode ok %dx%d", width, height); } bool ImageBlock::serialize(HalFile& file) { @@ -432,8 +474,10 @@ bool ImageBlock::serialize(HalFile& file) { std::unique_ptr ImageBlock::deserialize(HalFile& file) { std::string path; std::string src; - serialization::readString(file, path); - serialization::readString(file, src); + if (!serialization::readString(file, path) || !serialization::readString(file, src)) { + LOG_ERR("IMG", "Deserialization failed: corrupt path/src length"); + return nullptr; + } int16_t w, h; serialization::readPod(file, w); serialization::readPod(file, h); diff --git a/lib/Epub/Epub/blocks/ImageBlock.h b/lib/Epub/Epub/blocks/ImageBlock.h index 78c2770cc30..30ec248378c 100644 --- a/lib/Epub/Epub/blocks/ImageBlock.h +++ b/lib/Epub/Epub/blocks/ImageBlock.h @@ -19,6 +19,10 @@ class ImageBlock final : public Block { bool hasValidCache() const; bool needsDecode() const; void renderPlaceholder(GfxRenderer& renderer, int x, int y) const; + // The .pxc path for an extracted image, and whether a finished cache of exactly that size exists. + // Used by the chapter build to pregenerate the cache while it owns the heap. + static std::string cachePathFor(const std::string& imagePath); + static bool hasValidCacheFor(const std::string& imagePath, int width, int height); static void clearRenderFailures(); // A page render draws its image up to ~13 times (BW double-refresh plus every diff --git a/lib/Epub/Epub/blocks/TextBlock.cpp b/lib/Epub/Epub/blocks/TextBlock.cpp index 3219b85dc92..2d62a39d789 100644 --- a/lib/Epub/Epub/blocks/TextBlock.cpp +++ b/lib/Epub/Epub/blocks/TextBlock.cpp @@ -6,14 +6,27 @@ #include #include #include +#include +#include +#include #include +#include "../InlineImageToken.h" +#include "ImageBlock.h" +#ifdef INPUT_DIAG +#include +#endif + #include "../../../../src/fontIds.h" -size_t TextBlock::arenaSize(const uint16_t wordCount, const bool hasFocus, const uint16_t textBytes) { +size_t TextBlock::arenaSize(const uint16_t wordCount, const bool hasFocus, const bool hasVertical, + const uint16_t textBytes) { // Layout documented in TextBlock.h: 16-bit arrays first, then 8-bit arrays, then text. size_t size = static_cast(wordCount) * (sizeof(uint16_t) + sizeof(int16_t) + sizeof(uint8_t)); + if (hasVertical) { + size += static_cast(wordCount) * sizeof(int16_t); + } if (hasFocus) { size += static_cast(wordCount) * (sizeof(uint16_t) + sizeof(uint8_t)); } @@ -26,6 +39,11 @@ void TextBlock::bindArenaPointers() { textOffArr = reinterpret_cast(base); xposArr = reinterpret_cast(base + wc * 2); size_t off = wc * 4; + // ypos stays among the 16-bit arrays (right after xpos) so 2-byte alignment holds. + if (isVertical) { + yposArr = reinterpret_cast(base + off); + off += wc * 2; + } if (focusPresent) { focusSuffixXArr = reinterpret_cast(base + off); off += wc * 2; @@ -84,7 +102,7 @@ TextBlock::TextBlock(const std::vector& words, const std::vector(totalText); - const size_t size = arenaSize(numWords, focusPresent, textBytes); + const size_t size = arenaSize(numWords, focusPresent, /*hasVertical=*/false, textBytes); arena = makeUniqueNoThrow(size); if (!arena) { // Evict rebuildable caches (SD-font mini data, render glyph cache) and @@ -126,6 +144,585 @@ TextBlock::TextBlock(const std::vector& words, const std::vector& words, const std::vector& wordXpos, + const std::vector& wordYpos, const std::vector& wordStyles, + const BlockStyle& blockStyle, std::vector rubyTexts, const uint16_t cellWidth, + std::vector linkSpans) + : blockStyle(blockStyle), + vertCellWidth(cellWidth), + rubyTexts(std::move(rubyTexts)), + linkSpans(std::move(linkSpans)) { + // Same invariant the horizontal constructor keeps: never hold an all-empty rubyTexts. + // A column split out of a ruby-bearing paragraph often lands entirely on unannotated + // words, and that column should not pay for a vector of empty strings. + if (!hasRuby()) { + this->rubyTexts = std::vector{}; + } + + if (words.size() != wordXpos.size() || words.size() != wordYpos.size() || words.size() != wordStyles.size() || + words.size() > 10000) { + LOG_ERR("TXB", "Vertical construction failed: size mismatch (words=%u, xpos=%u, ypos=%u, styles=%u)", + static_cast(words.size()), static_cast(wordXpos.size()), + static_cast(wordYpos.size()), static_cast(wordStyles.size())); + isValid = false; + return; + } + + numWords = static_cast(words.size()); + isVertical = true; + focusPresent = false; + if (numWords == 0) { + return; // valid empty block, no arena + } + + size_t totalText = 0; + for (const auto& w : words) totalText += w.size() + 1; + if (totalText > UINT16_MAX) { + LOG_ERR("TXB", "Vertical construction failed: text size %u exceeds arena limit", static_cast(totalText)); + numWords = 0; + isVertical = false; + isValid = false; + return; + } + textBytes = static_cast(totalText); + + const size_t size = arenaSize(numWords, focusPresent, isVertical, textBytes); + arena = makeUniqueNoThrow(size); + if (!arena) { + LOG_ERR("TXB", "OOM: vertical arena %u bytes", static_cast(size)); + numWords = 0; + textBytes = 0; + isVertical = false; + isValid = false; + return; + } + bindArenaPointers(); + + auto* textOff = const_cast(textOffArr); + auto* xpos = const_cast(xposArr); + auto* ypos = const_cast(yposArr); + auto* styles = const_cast(stylesArr); + auto* text = const_cast(textArr); + uint16_t off = 0; + for (uint16_t i = 0; i < numWords; i++) { + textOff[i] = off; + xpos[i] = wordXpos[i]; + ypos[i] = wordYpos[i]; + styles[i] = static_cast(wordStyles[i]); + memcpy(text + off, words[i].data(), words[i].size()); + off += static_cast(words[i].size()); + text[off++] = '\0'; + } +} + +namespace { +// Re-derive the token's vertical behaviour at draw time. layoutVerticalColumns +// classified it the same way from the same text, so this stays in step without +// widening the cached block format with a per-word behaviour array. +// +// Upright: CJK, kana and the punctuation table. Tate-chu-yoko: runs of 1-2 digits and +// an exclamation/question pair, set upright inside one cell. Everything else (Latin +// words, 3+ digit numbers) is sideways. +bool isTateChuYokoToken(const char* word) { + if (VerticalTextUtils::isTateChuYokoPunctuationPair(word)) return true; + if (word[0] < '0' || word[0] > '9') return false; + int digits = 0; + for (const char* p = word; *p != '\0'; ++p) { + if (*p < '0' || *p > '9') return false; + digits++; + } + return digits <= 2; +} + +bool isSidewaysToken(const char* word) { + const auto* p = reinterpret_cast(word); + const uint32_t first = utf8NextCodepoint(&p); + if (first == 0) return false; + if (VerticalTextUtils::isUprightInVertical(first) || + VerticalTextUtils::getVerticalPunctuationOffset(first) != nullptr) { + return false; + } + return !isTateChuYokoToken(word); +} + +// Ruby written in Latin script is turned clockwise and run down the column, the same way +// a Latin word in the body is. Stacking it a letter at a time upright is what a reader +// sees as a smear rather than a word: the stack advances by each letter's own width, and +// a proportional 'i' or 'r' is a fraction as wide as it is tall, so the letters print on +// top of each other. Anything with a CJK character in it keeps the upright stack, where +// mixing the two would be worse than either. +// +// A reading that is only digits turns with the rest, where the body would set one or two +// of them upright as tate-chu-yoko. That divergence is deliberate, not an oversight: ruby +// is already half size, so a two-digit reading set upright across the column would leave +// each digit a quarter of the body's width -- consistent with the body, and unreadable. +// Checked on the device against the chapter of test/epubs-ja that carries 3, 12 and 2024 +// as readings for exactly this comparison. +bool isSidewaysRuby(const char* ruby) { + const auto* p = reinterpret_cast(ruby); + bool sawGlyph = false; + uint32_t cp; + while ((cp = utf8NextCodepoint(&p)) != 0) { + if (VerticalTextUtils::isUprightInVertical(cp) || VerticalTextUtils::getVerticalPunctuationOffset(cp) != nullptr) { + return false; + } + sawGlyph = true; + } + return sawGlyph; +} + +// How far a reading runs down the column. It depends on the reading alone and not on what +// it annotates, which is what lets the column's total be counted in one cheap walk before +// any of it is placed. +// +// The upright stack advances on a fixed half-em, not on each glyph's own width. For kana +// and kanji the two are the same number, since a full-width glyph halved by SUP is exactly +// the half cell; for anything proportional they are not, and the glyph width is the wrong +// one -- it measures across the line, and the stack runs down it. JIS X 4051 sets ruby on +// that fixed pitch for the same reason. Turned, a run's extent down the column is instead +// its width across the line, which is what getTextAdvanceX reports. +int rubyRunSpan(const GfxRenderer& renderer, const int fontId, const char* ruby, const int rubyCellWidth) { + if (isSidewaysRuby(ruby)) { + return renderer.getTextAdvanceX(fontId, ruby, EpdFontFamily::SUP); + } + int cells = 0; + const auto* cp = reinterpret_cast(ruby); + while (utf8NextCodepoint(&cp) != 0) cells++; + return cells * rubyCellWidth; +} + +// Separation kept between the ruby of one group and the next down the column. One pixel +// is the least that reads as two runs rather than one on a 1-bit panel; more than that +// costs column space, which the ruby that needs separating is already short of. +constexpr int RUBY_GROUP_GAP = 1; +} // namespace + +#ifdef INPUT_DIAG +namespace { +// Accumulated across the blocks of one page; drained by takeVerticalRenderStats(). +uint32_t vBodyMs = 0; +uint32_t vBodyCells = 0; +uint32_t vRubyMeasureMs = 0; +uint32_t vRubyDrawMs = 0; +uint32_t vRubyGroups = 0; +} // namespace + +TextBlock::VerticalRenderStats TextBlock::takeVerticalRenderStats() { + const VerticalRenderStats stats{vBodyMs, vBodyCells, vRubyMeasureMs, vRubyDrawMs, vRubyGroups}; + vBodyMs = 0; + vBodyCells = 0; + vRubyMeasureMs = 0; + vRubyDrawMs = 0; + vRubyGroups = 0; + return stats; +} +#endif + +void TextBlock::renderVertical(const GfxRenderer& renderer, const int fontId, const int x, const int y) const { + // Each token is stacked at its precomputed (xpos, ypos): CJK and kana upright, + // Latin runs and the rotating punctuation turned clockwise (see VERTICAL_PUNCTUATION). + // + // Both adjustments below are measured against the full-width character cell -- the + // em advance layoutVerticalColumns stacked by -- and NOT the column pitch. The two + // differ: the pitch is the font's line height, which for a CJK face runs well wider + // than the em (NotoSansJP 16pt: 48 px pitch, 33 px em). Upright glyphs land against + // the cell's left edge, so centring anything on the pitch instead pushes it right, + // out of line with the column of kanji above and below it. + // + // Every upright token is full-width, so the first one measures the cell for the whole + // block. A block of nothing but Latin has none to measure; fall back to the line height, + // matching layoutVerticalColumns' own fallback for cjkCharAdvance. + // The layout's cell width travels with the block; the scan below only serves + // blocks serialized before it did, and misses on columns holding no upright word. + int cellWidth = vertCellWidth; + for (uint16_t i = 0; i < numWords && cellWidth == 0; i++) { + if (InlineImageToken::is(wordText(i))) continue; + if ((wordStyle(i) & EpdFontFamily::VERTICAL_FLIP) == 0 && !isSidewaysToken(wordText(i))) { + cellWidth = renderer.getTextAdvanceX(fontId, wordText(i), wordStyle(i)); + } + } + if (cellWidth == 0) { + // Not the line height: that is the source font's own hhea metric and differs by nearly half + // between faces at the same point size, which would draw this block's cells at a different + // width from the one the parser laid the columns out with (see getCjkCellWidth). + cellWidth = renderer.getCjkCellWidth(fontId); + } + + // Cell top -> the y drawText expects. drawText adds the ascender itself, and for a CJK + // face that reaches past the em box, dropping every glyph low in its cell until the last + // one of a column overlaps the status bar. Centring the line box in the cell pulls it back. + const int ascender = renderer.getFontAscenderSize(fontId); + const int lineBox = ascender - renderer.getFontDescenderSize(fontId); + const int uprightYAdjust = (cellWidth - lineBox) / 2; + + // Extent of word i down the column: an inline image's height, a squeezed bracket's half + // cell (see 約物の二分アキ below), else the glyph advance. + const auto wordAdvance = [&](const uint16_t i) { + const int img = InlineImageToken::advance(wordText(i)); + if (img > 0) return img; + const auto* p = reinterpret_cast(wordText(i)); + if (VerticalTextUtils::verticalHalfWidthKind(utf8NextCodepoint(&p)) != VerticalTextUtils::HalfWidthKind::None) { + return cellWidth / 2; + } + return renderer.getTextAdvanceX(fontId, wordText(i), wordStyle(i)); + }; + + // A vertical block is one column, so its own stacking positions give the column's + // extent: ruby is clamped to it below. + int columnTop = 0; + int columnEnd = 0; + if (blockHasRubyExtent()) { + columnTop = yposArr[0] + y; + const uint16_t lastWord = numWords - 1; + columnEnd = yposArr[lastWord] + y + wordAdvance(lastWord); + } + + // Ruby sits to the right of the words it annotates, the vertical counterpart of + // sitting above them. The grouping is the horizontal path's, unchanged: the leader + // word carries the text and each continuation word is flagged RUBY_CONTINUE, so the + // group is however many flagged tokens follow. Per-codepoint tokenisation makes those + // groups longer than in horizontal mode, not different in kind. + const bool blockHasRuby = hasRuby(); + + // Decoration lines run along the column here, not across it: a side line beside the + // characters and a strikethrough down through them. Japanese vertical setting puts the + // side line to the right of the characters, which is also where ruby goes, so a block + // carrying ruby takes the left instead of overprinting it: ruby owns the right half-cell, + // and JLREQ has no rule on the side a bousen takes, so the one that does not collide wins. Accumulated across tokens + // rather than drawn per token so a decorated run comes out as one line: consecutive cells share an x, and the run + // ends where the style stops. + const bool scanning = renderer.isFontCacheScanning(); + struct VerticalDecorationTracker { + EpdFontFamily::Style style; + int startY = -1; + int endY = -1; + int xPos = 0; + + bool active() const { return startY != -1; } + void reset() { + startY = -1; + endY = -1; + xPos = 0; + } + }; + + VerticalDecorationTracker verticalDecorations[] = { + {EpdFontFamily::UNDERLINE}, + {EpdFontFamily::STRIKETHROUGH}, + }; + + // Upright glyphs sit against the cell's left edge and the column pitch is wider than the + // cell, so the gap the side line wants is just past cellWidth; the strikethrough splits + // the cell instead. + const auto verticalDecorationX = [&](const EpdFontFamily::Style style, const int cellX) { + if ((style & EpdFontFamily::STRIKETHROUGH) != 0) return cellX + cellWidth / 2; + return blockHasRuby ? cellX - 3 : cellX + cellWidth + 2; + }; + const auto flushVerticalDecoration = [&](VerticalDecorationTracker& line) { + if (line.active()) { + renderer.drawLine(line.xPos, line.startY, line.xPos, line.endY, 2, true); + line.reset(); + } + }; + const auto flushVerticalDecorations = [&]() { + for (auto& line : verticalDecorations) { + flushVerticalDecoration(line); + } + }; + +#ifdef INPUT_DIAG + const uint32_t vBodyStart = millis(); + vBodyCells += numWords; +#endif + for (uint16_t i = 0; i < numWords; i++) { + const char* word = wordText(i); + const int cellX = xposArr[i] + x; + const int cellY = yposArr[i] + y; + + if (InlineImageToken::is(word)) { + // A character-sized image in the flow: centred on the cell like an upright glyph, drawn + // through ImageBlock so it comes from the same .pxc cache the page images use (and is + // skipped during the font-cache scan like they are). The renderer is only const here + // because the text path never mutates it; the image cache state does. + flushVerticalDecorations(); + InlineImageToken::Spec spec; + if (!scanning && InlineImageToken::decode(word, spec)) { + ImageBlock image(spec.imagePath, spec.srcPath, spec.width, spec.height); + int drawX = cellX + (cellWidth - spec.width) / 2; + if (drawX < 0) drawX = 0; + image.render(const_cast(renderer), drawX, cellY); + } + continue; + } + + // Which way the token is set. Derived from the text (Latin runs and the rotating + // punctuation turn, everything else stands), then reversed by the VERTICAL_FLIP bit + // the parser sets for CSS text-orientation / text-combine-upright. + const auto* p = reinterpret_cast(word); + const uint32_t cp = utf8NextCodepoint(&p); + const VerticalTextUtils::PunctuationOffset* punct = VerticalTextUtils::getVerticalPunctuationOffset(cp); + const bool flipped = (wordStyle(i) & EpdFontFamily::VERTICAL_FLIP) != 0; + bool turned = isSidewaysToken(word) || (punct != nullptr && punct->rotate); + if (flipped) turned = !turned; + + // 約物の二分アキ: layoutVerticalColumns gave this mark half a cell. The ink already sits + // in one half of the em -- an opening bracket in the far half, a closing one in the near + // half -- so a closing mark needs nothing and an opening mark is drawn from half a cell + // back, which lands its ink exactly in the shortened cell. + const auto halfKind = VerticalTextUtils::verticalHalfWidthKind(cp); + const int halfCell = cellWidth / 2; + const int halfShift = halfKind == VerticalTextUtils::HalfWidthKind::Opening ? -halfCell : 0; + const int cellExtent = halfKind == VerticalTextUtils::HalfWidthKind::None ? cellWidth : halfCell; + + // Before the draw branches below, all of which continue: every token contributes its + // own extent down the column whichever way it is set. A sideways run reserved its + // width, which is what the layout stacked; everything else occupies one cell. + if (!scanning) { + const EpdFontFamily::Style style = wordStyle(i); + if (EpdFontFamily::hasTextDecoration(style)) { + const int extent = turned && halfKind == VerticalTextUtils::HalfWidthKind::None + ? renderer.getTextAdvanceX(fontId, word, style) + : cellExtent; + for (auto& line : verticalDecorations) { + if ((style & line.style) == 0) { + flushVerticalDecoration(line); + continue; + } + const int lineX = verticalDecorationX(line.style, cellX); + if (line.active() && line.xPos != lineX) { + flushVerticalDecoration(line); + } + if (!line.active()) { + line.startY = cellY; + line.xPos = lineX; + } + line.endY = cellY + extent; + } + } else { + flushVerticalDecorations(); + } + } + + // Sideways runs: the column reserved the run's *width* as its vertical extent, + // so drawing it upright would spill across the columns to the left. Brackets and + // long marks turn the same way: rotating the horizontal glyph also carries its ink + // to the right corner of the cell on its own, so 「 ends up opening downward at the + // cell top, 」 closing at the bottom, ー running along the column. The advance is + // unchanged, so the cell still measures one em. + if (turned) { + // centreInk for the punctuation table's marks only: a Latin run keeps the line-box + // centring that holds successive runs on one axis (see drawTextSideways). + renderer.drawTextSideways(fontId, cellX, cellY + halfShift, word, cellWidth, true, wordStyle(i), + punct != nullptr); + continue; + } + + // Punctuation drawn from a horizontal-layout font is in the wrong place for a + // vertical column (see VERTICAL_PUNCTUATION). + int drawX = cellX; + int drawY = cellY + uprightYAdjust + halfShift; + if (flipped) { + // Set upright against its text's nature: a Latin letter or a digit given a full + // cell, or a run combined into one. Centre it on the cell whatever its width, so + // a three-digit tate-chu-yoko overhangs both sides evenly instead of one. + const int advance = renderer.getTextAdvanceX(fontId, word, wordStyle(i)); + drawX += (cellWidth - advance) / 2; + } else if (punct != nullptr) { + drawX += cellWidth * punct->dxEighths / 8; + drawY += cellWidth * punct->dyEighths / 8; + } else if (VerticalTextUtils::isSmallKana(cp)) { + // The face of a small kana belongs against the right of its cell in + // vertical writing and against the bottom in horizontal (JLREQ 2.1.2 + // note 1). Fonts ship the horizontal placement, so a column of them + // drawn as-is reads as a row of characters leaning off the column axis. + // Only the x moves: the spec keeps them centred top to bottom, and the + // advance is untouched, so column breaks, ruby and line spacing hold. + drawX += cellWidth * VerticalTextUtils::SMALL_KANA_DX_EIGHTHS / 8; + } else if (isTateChuYokoToken(word)) { + // Tate-chu-yoko sets its run upright inside one cell, which the layout reserved + // at a full em. Two half-width digits fill that exactly, but a lone digit covers + // half of it, and a proportional pair like "!?" is narrower still (26.6 px against + // a 33.3 px cell in NotoSansJP) -- all of which would hug the cell's left edge, + // off the column's axis. Centre whatever comes out narrower than the cell. + const int advance = renderer.getTextAdvanceX(fontId, word, wordStyle(i)); + if (advance < cellWidth) { + drawX += (cellWidth - advance) / 2; + } + } else if (cp < 0x3000) { + // Upright symbols that are not CJK are drawn from proportional glyphs: § © ® ± ¼ × + // are a fraction of the em cell the column reserved, and left at the cell's left + // edge they read as leaning off the column axis -- ® showed up left of centre on the + // device. Same centring tate-chu-yoko gets above, for the same reason. Gated on the + // codepoint rather than measured unconditionally so a page of kanji, where the + // advance always equals the cell, does not pay a metric lookup per character on + // every one of the ~20 passes an antialiased page makes. + const int advance = renderer.getTextAdvanceX(fontId, word, wordStyle(i)); + if (advance > 0 && advance < cellWidth) { + drawX += (cellWidth - advance) / 2; + } + } + + renderer.drawText(fontId, drawX, drawY, word, true, wordStyle(i)); + } + flushVerticalDecorations(); + + // Ruby is placed in a pass of its own, and needs two things the body's pass cannot give + // it. A group has to clear the one above -- a reading longer than what it annotates + // overhangs its group at both ends, so neighbours collide -- and it has to leave the + // groups below room to reach the foot of the column, or the last of them is drawn under + // the status bar, where ruby is not misplaced but gone. + // + // The two pull opposite ways and neither can be answered on its own. Pushing only + // downward piles the overflow at the foot; pushing only upward piles it at the head, + // which is where the device showed it -- two eleven-character readings on two-character + // bases, overprinted at the top of a column. + // + // Both fit in two running totals, no table of positions needed, because how far a + // reading runs does not depend on what it annotates. Count the column's readings once, + // then walk them from the top: what is still to be placed below is room this group may + // not take, and the foot of the column less that room is the lowest it may end. +#ifdef INPUT_DIAG + vBodyMs += static_cast(millis() - vBodyStart); +#endif + + if (blockHasRuby) { + // A ruby glyph is half-width, so it sits in the half cell just right of the body one. + const int rubyCellWidth = cellWidth / 2; + +#ifdef INPUT_DIAG + const uint32_t vRubyMeasureStart = millis(); +#endif + int rubyTotalExtent = 0; + for (uint16_t i = 0; i < numWords; i++) { + if (i >= rubyTexts.size() || rubyTexts[i].empty() || (wordStyle(i) & EpdFontFamily::RUBY_CONTINUE) != 0) { + continue; + } + rubyTotalExtent += rubyRunSpan(renderer, fontId, rubyTexts[i].c_str(), rubyCellWidth) + RUBY_GROUP_GAP; + } + +#ifdef INPUT_DIAG + vRubyMeasureMs += static_cast(millis() - vRubyMeasureStart); + const uint32_t vRubyDrawStart = millis(); +#endif + int rubyPlacedExtent = 0; + int prevRubyFoot = INT_MIN; + for (uint16_t i = 0; i < numWords; i++) { + if (i >= rubyTexts.size() || rubyTexts[i].empty() || (wordStyle(i) & EpdFontFamily::RUBY_CONTINUE) != 0) { + continue; + } + const int cellX = xposArr[i] + x; +#ifdef INPUT_DIAG + vRubyGroups++; +#endif + // Ruby runs down the column beside the body. A CJK ruby is stacked a glyph at a + // time like the body is -- one drawText for the whole string would lay it across + // the column instead -- while a Latin one is turned clockwise and drawn as a single + // run, which is both how Japanese typesetting sets it and what keeps it legible. + uint16_t groupWords = 1; + while (i + groupWords < numWords && (wordStyle(i + groupWords) & EpdFontFamily::RUBY_CONTINUE) != 0) { + groupWords++; + } + // Span the group along the column from the first cell's top to the last cell's + // bottom, taken from the stacking positions the layout already computed rather + // than re-summing advances -- ypos carries the inter-cell spacing too. + const int groupTop = yposArr[i] + y; + const uint16_t lastWord = i + groupWords - 1; + const int lastAdvance = wordAdvance(lastWord); + const int groupSpan = (yposArr[lastWord] + y + lastAdvance) - groupTop; + + const bool rubySideways = isSidewaysRuby(rubyTexts[i].c_str()); + const int rubyX = cellX + cellWidth; + const int rubySpan = rubyRunSpan(renderer, fontId, rubyTexts[i].c_str(), rubyCellWidth); + int rubyCursorY = groupTop + (groupSpan - rubySpan) / 2; + + // The earliest this group may start: clear of the one above, and not above the head + // of the column. Japanese typography answers the overhang at a column's ends the + // same way (JIS X 4051) -- ruby that would overhang the line start is set flush with + // it instead of centred. + int rubyEarliest = columnTop; + if (prevRubyFoot != INT_MIN && prevRubyFoot + RUBY_GROUP_GAP > rubyEarliest) { + rubyEarliest = prevRubyFoot + RUBY_GROUP_GAP; + } + if (rubyCursorY < rubyEarliest) rubyCursorY = rubyEarliest; + + if (columnEnd > columnTop) { + // ...and the latest it may end: far enough above the foot for everything still to + // be placed below it, packed tight. + const int roomBelow = rubyTotalExtent - rubyPlacedExtent - (rubySpan + RUBY_GROUP_GAP); + const int latestEnd = columnEnd - roomBelow; + if (rubyCursorY + rubySpan > latestEnd) { + // Never past the earliest, though. A column can carry more ruby than it has room + // for -- two eleven-character readings and three more in fourteen cells is over + // its length before anything is placed -- and then no arrangement satisfies both + // bounds. Keeping clear of the group above is the one to hold: readings that + // overprint are readable as neither, while the overflow at the foot is at worst + // a tail that runs under the status bar. + rubyCursorY = (latestEnd - rubySpan > rubyEarliest) ? latestEnd - rubySpan : rubyEarliest; + } + } + prevRubyFoot = rubyCursorY + rubySpan; + rubyPlacedExtent += rubySpan + RUBY_GROUP_GAP; + + if (rubySideways) { + renderer.drawTextSideways(fontId, rubyX, rubyCursorY, rubyTexts[i].c_str(), rubyCellWidth, true, + EpdFontFamily::SUP); + } else { + const auto* rp = reinterpret_cast(rubyTexts[i].c_str()); + while (*rp != '\0') { + const unsigned char* cpStart = rp; + if (utf8NextCodepoint(&rp) == 0) break; + const size_t cpLen = static_cast(rp - cpStart); + // A UTF-8 sequence is at most 4 bytes; a longer step means a malformed string. + char glyph[5] = {}; + if (cpLen == 0 || cpLen >= sizeof(glyph)) break; + memcpy(glyph, cpStart, cpLen); + + // Ruby is set in the column, so its punctuation needs turning and shifting for + // the column exactly as the body's does -- an ー left upright in a katakana + // reading is as wrong there as it is in the text it annotates. The body's own + // table and the same two cases answer it, on the ruby cell instead of the full + // one. Sideways drawing scales for SUP, so the rotating case is the same call. + const auto* gp = reinterpret_cast(glyph); + const uint32_t gcp = utf8NextCodepoint(&gp); + const VerticalTextUtils::PunctuationOffset* rubyPunct = VerticalTextUtils::getVerticalPunctuationOffset(gcp); + + if (rubyPunct != nullptr && rubyPunct->rotate) { + renderer.drawTextSideways(fontId, rubyX, rubyCursorY, glyph, rubyCellWidth, true, EpdFontFamily::SUP); + rubyCursorY += rubyCellWidth; + continue; + } + + // drawText always offsets by the full ascender, but a SUP glyph is half-scale, so + // centre the halved line box in the ruby cell the same way the body centres the + // full one -- otherwise the glyph lands an ascender's worth below its cell. + int rubyDrawX = rubyX; + int rubyDrawY = rubyCursorY + (rubyCellWidth - lineBox / 2) / 2 - ascender / 2; + if (rubyPunct != nullptr) { + rubyDrawX += rubyCellWidth * rubyPunct->dxEighths / 8; + rubyDrawY += rubyCellWidth * rubyPunct->dyEighths / 8; + } else { + // The stack steps down the column on the half-em cell, but each glyph is drawn + // from that cell's left edge, and a proportional one does not fill it. A thin + // letter then sits off the axis the rest of the ruby runs along -- an 'I' among + // 'A' and 'S' reads as pushed left, because it is. Centre whatever comes out + // narrower than the cell, the same way the body centres a tate-chu-yoko run in + // its own. Kana and kanji are unaffected: halved by SUP they are the cell. + const int glyphAdvance = renderer.getTextAdvanceX(fontId, glyph, EpdFontFamily::SUP); + if (glyphAdvance < rubyCellWidth) { + rubyDrawX += (rubyCellWidth - glyphAdvance) / 2; + } + } + renderer.drawText(fontId, rubyDrawX, rubyDrawY, glyph, true, EpdFontFamily::SUP); + rubyCursorY += rubyCellWidth; + } + } + } +#ifdef INPUT_DIAG + vRubyDrawMs += static_cast(millis() - vRubyDrawStart); +#endif + } +} + bool TextBlock::hasRuby() const { for (const auto& rt : rubyTexts) { if (!rt.empty()) return true; @@ -140,6 +737,11 @@ void TextBlock::render(const GfxRenderer& renderer, const int fontId, const int } const int8_t tracking = blockStyle.characterSpacing; + if (isVertical) { + renderVertical(renderer, fontId, x, y); + return; + } + const bool scanning = renderer.isFontCacheScanning(); const int ascender = renderer.getFontAscenderSize(fontId); @@ -313,9 +915,13 @@ bool TextBlock::serialize(HalFile& file) const { // per-word arrays and the text blob. serialization::writePod(file, numWords); serialization::writePod(file, static_cast(focusPresent ? 1 : 0)); + serialization::writePod(file, static_cast(isVertical ? 1 : 0)); serialization::writePod(file, textBytes); + if (isVertical) { + serialization::writePod(file, vertCellWidth); + } if (numWords > 0) { - const size_t size = arenaSize(numWords, focusPresent, textBytes); + const size_t size = arenaSize(numWords, focusPresent, isVertical, textBytes); if (file.write(arena.get(), size) != size) { LOG_ERR("TXB", "Serialization failed: arena write (%u bytes)", static_cast(size)); return false; @@ -350,9 +956,11 @@ bool TextBlock::serialize(HalFile& file) const { std::unique_ptr TextBlock::deserialize(HalFile& file) { uint16_t wc; uint8_t hasFocus; + uint8_t hasVertical; uint16_t textBytes; serialization::readPod(file, wc); serialization::readPod(file, hasFocus); + serialization::readPod(file, hasVertical); serialization::readPod(file, textBytes); // Sanity checks: cap the arena allocation and reject impossible geometry @@ -374,9 +982,13 @@ std::unique_ptr TextBlock::deserialize(HalFile& file) { block->numWords = wc; block->textBytes = textBytes; block->focusPresent = hasFocus != 0; + block->isVertical = hasVertical != 0; + if (block->isVertical) { + serialization::readPod(file, block->vertCellWidth); + } if (wc > 0) { - const size_t size = arenaSize(wc, block->focusPresent, textBytes); + const size_t size = arenaSize(wc, block->focusPresent, block->isVertical, textBytes); block->arena = makeUniqueNoThrow(size); if (!block->arena) { LOG_ERR("TXB", "OOM: arena %u bytes", static_cast(size)); @@ -417,7 +1029,10 @@ std::unique_ptr TextBlock::deserialize(HalFile& file) { // overwrites every byte, so a moved-from value carries nothing into the next iteration. std::string scratch; for (uint16_t i = 0; i < wc; i++) { - serialization::readString(file, scratch); + if (!serialization::readString(file, scratch)) { + LOG_ERR("TXB", "Deserialization failed: corrupt ruby-text length for word %u", i); + return nullptr; + } if (scratch.empty()) continue; if (block->rubyTexts.empty()) { block->rubyTexts.resize(wc); diff --git a/lib/Epub/Epub/blocks/TextBlock.h b/lib/Epub/Epub/blocks/TextBlock.h index 1ffd0769385..0d975ab8d80 100644 --- a/lib/Epub/Epub/blocks/TextBlock.h +++ b/lib/Epub/Epub/blocks/TextBlock.h @@ -23,11 +23,18 @@ // unaligned multi-byte access): // uint16_t textOff[wordCount] byte offset of word i's text in text[] // int16_t xpos[wordCount] +// int16_t ypos[wordCount] present only when isVertical (tategaki) // uint16_t focusSuffixX[wordCount] present only when focusPresent // uint8_t styles[wordCount] // uint8_t focusBoundary[wordCount] present only when focusPresent // char text[textBytes] all words back to back, NUL-terminated // +// Vertical (tategaki) blocks carry a per-word ypos array and lay each word out +// down a column (xpos is the shared column offset, ypos the stacking position); +// the page composes columns right-to-left. Vertical blocks never carry focus +// splits (bionic reading is a horizontal-only feature), so ypos and focus arrays +// are mutually exclusive in practice, though the layout permits either. +// // Each word is stored NUL-terminated so render() can hand `text + textOff[i]` // straight to C APIs (drawText) with no std::string materialization. // @@ -52,6 +59,12 @@ class TextBlock final : public Block { uint16_t numWords = 0; uint16_t textBytes = 0; // total size of the text region, including NULs bool focusPresent = false; + bool isVertical = false; // tategaki: arena carries a per-word ypos array + // Full-width (CJK em) cell advance the layout used, carried because a column + // holding only sideways runs has no upright word to re-derive it from at draw + // time -- the line-height stand-in it fell back to overshot the cell by ~40%, + // pushing the column's ruby into its right-hand neighbour. 0 = re-derive. + uint16_t vertCellWidth = 0; bool isValid = true; // The ONLY allocation: makeUniqueNoThrow, so OOM yields an invalid block // instead of abort() (bare new is not nothrow with -fno-exceptions). @@ -60,6 +73,7 @@ class TextBlock final : public Block { // 16-bit bases sit at even offsets, so direct dereference is alignment-safe. const uint16_t* textOffArr = nullptr; const int16_t* xposArr = nullptr; + const int16_t* yposArr = nullptr; // null when !isVertical const uint16_t* focusSuffixXArr = nullptr; // null when !focusPresent const uint8_t* stylesArr = nullptr; const uint8_t* focusBoundaryArr = nullptr; // null when !focusPresent @@ -70,8 +84,9 @@ class TextBlock final : public Block { std::vector linkSpans; TextBlock() = default; // deserialize() fills the fields directly - static size_t arenaSize(uint16_t wordCount, bool hasFocus, uint16_t textBytes); + static size_t arenaSize(uint16_t wordCount, bool hasFocus, bool hasVertical, uint16_t textBytes); void bindArenaPointers(); + void renderVertical(const GfxRenderer& renderer, int fontId, int x, int y) const; public: // Flatten-on-construct: copies the layout-time vectors into the arena; the @@ -81,6 +96,12 @@ class TextBlock final : public Block { const std::vector& wordStyles, const std::vector& focusBoundary, const std::vector& focusSuffixX, const BlockStyle& blockStyle = BlockStyle(), std::vector rubyTexts = {}, std::vector linkSpans = {}); + // Vertical (tategaki) block: per-word Y stacking positions instead of a shared + // baseline. No focus-split support (bionic reading is horizontal-only). + explicit TextBlock(const std::vector& words, const std::vector& wordXpos, + const std::vector& wordYpos, const std::vector& wordStyles, + const BlockStyle& blockStyle = BlockStyle(), std::vector rubyTexts = {}, + uint16_t cellWidth = 0, std::vector linkSpans = {}); ~TextBlock() override = default; TextBlock(const TextBlock&) = delete; TextBlock& operator=(const TextBlock&) = delete; @@ -97,14 +118,32 @@ class TextBlock final : public Block { return end - textOffArr[i] - 1; // exclude the NUL } int16_t wordXpos(const uint16_t i) const { return xposArr[i]; } + int16_t wordYpos(const uint16_t i) const { return isVertical ? yposArr[i] : 0; } + bool vertical() const { return isVertical; } EpdFontFamily::Style wordStyle(const uint16_t i) const { return static_cast(stylesArr[i]); } uint8_t focusBoundary(const uint16_t i) const { return focusPresent ? focusBoundaryArr[i] : 0; } uint16_t focusSuffixX(const uint16_t i) const { return focusPresent ? focusSuffixXArr[i] : 0; } bool hasRuby() const; int getRubyShift(int ascender) const { return hasRuby() ? (ascender / 2) : 0; } + // Vertical only: whether it is worth measuring the column's extent for ruby clamping. + bool blockHasRubyExtent() const { return isVertical && numWords > 0 && hasRuby(); } const std::vector& getRubyTexts() const { return rubyTexts; } std::vector takeLinkSpans() { return std::move(linkSpans); } +#ifdef INPUT_DIAG + // Diagnostic only: where renderVertical spends its time, accumulated across the blocks of one + // page. Reading the stats zeroes them, so a caller brackets the pass it cares about: the scan + // pass runs these same loops and would otherwise be counted alongside the real draw. + struct VerticalRenderStats { + uint32_t bodyMs; // stacking and drawing the body cells + uint32_t bodyCells; // how many cells that was + uint32_t rubyMeasureMs; // measuring each ruby run before any of it is placed + uint32_t rubyDrawMs; // drawing the ruby, a glyph at a time + uint32_t rubyGroups; // how many annotations that was + }; + static VerticalRenderStats takeVerticalRenderStats(); +#endif + void render(const GfxRenderer& renderer, int fontId, int x, int y) const; BlockType getType() override { return TEXT_BLOCK; } bool serialize(HalFile& file) const; diff --git a/lib/Epub/Epub/converters/ImageToFramebufferDecoder.h b/lib/Epub/Epub/converters/ImageToFramebufferDecoder.h index e7d9de023d5..993358fff01 100644 --- a/lib/Epub/Epub/converters/ImageToFramebufferDecoder.h +++ b/lib/Epub/Epub/converters/ImageToFramebufferDecoder.h @@ -23,6 +23,10 @@ struct RenderConfig { float sourceCropY = 0.0f; // Fraction cropped equally from the top and bottom edges bool preserveAlpha = false; // Skip transparent pixels instead of compositing them against white std::string cachePath; // If non-empty, decoder will write pixel cache to this path + // Decode straight into the pixel cache without touching the framebuffer (build-time + // pregeneration). Requires a non-empty cachePath; x/y are cache metadata only and should be 0. + // Only JpegToFramebufferConverter honours it. + bool cacheOnly = false; }; class ImageToFramebufferDecoder { diff --git a/lib/Epub/Epub/converters/JpegToFramebufferConverter.cpp b/lib/Epub/Epub/converters/JpegToFramebufferConverter.cpp index a7102fffb2d..5cf59ff8623 100644 --- a/lib/Epub/Epub/converters/JpegToFramebufferConverter.cpp +++ b/lib/Epub/Epub/converters/JpegToFramebufferConverter.cpp @@ -1,5 +1,6 @@ #include "JpegToFramebufferConverter.h" +#include #include #include #include @@ -10,6 +11,7 @@ #include #include #include +#include #include "DirectPixelWriter.h" #include "DitherUtils.h" @@ -46,6 +48,7 @@ struct JpegContext { PixelCache cache; bool caching{false}; + bool drawToFb{true}; // false in cacheOnly mode: pixels go to the cache stream alone uint32_t lastYieldMs{0}; // throttle state for yieldDuringDecode() }; @@ -207,7 +210,7 @@ int jpegDrawCallback(JPEGDRAW* pDraw) { dithered = gray / 85; if (dithered > 3) dithered = 3; } - pw.writePixel(outX, dithered); + if (ctx->drawToFb) pw.writePixel(outX, dithered); if (caching) cw.writePixel(outX, dithered); } } @@ -266,7 +269,7 @@ int jpegDrawCallback(JPEGDRAW* pDraw) { dithered = gray / 85; if (dithered > 3) dithered = 3; } - pw.writePixel(outX, dithered); + if (ctx->drawToFb) pw.writePixel(outX, dithered); if (caching) cw.writePixel(outX, dithered); } @@ -289,7 +292,7 @@ int jpegDrawCallback(JPEGDRAW* pDraw) { dithered = gray / 85; if (dithered > 3) dithered = 3; } - pw.writePixel(outX, dithered); + if (ctx->drawToFb) pw.writePixel(outX, dithered); if (caching) cw.writePixel(outX, dithered); } @@ -315,7 +318,7 @@ int jpegDrawCallback(JPEGDRAW* pDraw) { dithered = gray / 85; if (dithered > 3) dithered = 3; } - pw.writePixel(outX, dithered); + if (ctx->drawToFb) pw.writePixel(outX, dithered); if (caching) cw.writePixel(outX, dithered); } } @@ -348,7 +351,7 @@ int jpegDrawCallback(JPEGDRAW* pDraw) { dithered = gray / 85; if (dithered > 3) dithered = 3; } - pw.writePixel(outX, dithered); + if (ctx->drawToFb) pw.writePixel(outX, dithered); if (caching) cw.writePixel(outX, dithered); } } @@ -390,17 +393,40 @@ bool JpegToFramebufferConverter::decodeToFramebuffer(const std::string& imagePat const RenderConfig& config) { LOG_DBG("JPG", "Decoding JPEG: %s", imagePath.c_str()); + // The decoder object (~18 KB on the C3) must be one block, and a reader session fragments the + // heap below that. A cache-only decode draws nothing, so when the largest block or the total is + // short it borrows the framebuffer bytes for the object instead. Declared before the decoder so + // the loan outlives it. + std::optional decoderLoan; + uint8_t* decoderScratch = nullptr; + if (config.cacheOnly && + (ESP.getMaxAllocHeap() < sizeof(JPEGDEC) + 1024 || ESP.getFreeHeap() < MIN_FREE_HEAP_FOR_JPEG)) { + decoderLoan.emplace(renderer); + decoderScratch = buildscratch::claim(sizeof(JPEGDEC)); + } + size_t freeHeap = ESP.getFreeHeap(); - if (freeHeap < MIN_FREE_HEAP_FOR_JPEG) { - LOG_ERR("JPG", "Not enough heap for JPEG decoder (%u free, need %u)", freeHeap, MIN_FREE_HEAP_FOR_JPEG); + const size_t minFreeHeap = + decoderScratch ? MIN_FREE_HEAP_FOR_JPEG - JPEG_DECODER_APPROX_SIZE : MIN_FREE_HEAP_FOR_JPEG; + if (freeHeap < minFreeHeap) { + LOG_ERR("JPG", "Not enough heap for JPEG decoder (%u free, need %u)", freeHeap, minFreeHeap); + if (decoderScratch) buildscratch::release(decoderScratch); return false; } - std::unique_ptr jpeg(new (std::nothrow) JPEGDEC()); + JPEGDEC* jpeg = decoderScratch ? new (decoderScratch) JPEGDEC() : new (std::nothrow) JPEGDEC(); if (!jpeg) { LOG_ERR("JPG", "Failed to allocate JPEG decoder"); return false; } + const ScopedCleanup freeDecoder{[jpeg, decoderScratch]() { + if (decoderScratch) { + jpeg->~JPEGDEC(); + buildscratch::release(decoderScratch); + } else { + delete jpeg; + } + }}; JpegContext ctx; ctx.renderer = &renderer; @@ -409,7 +435,7 @@ bool JpegToFramebufferConverter::decodeToFramebuffer(const std::string& imagePat ctx.screenHeight = renderer.getScreenHeight(); int rc = jpeg->open(imagePath.c_str(), jpegOpen, jpegClose, jpegRead, jpegSeek, jpegDrawCallback); - const ScopedCleanup cleanup{[&jpeg]() { jpeg->close(); }}; + const ScopedCleanup cleanup{[jpeg]() { jpeg->close(); }}; if (rc != 1) { LOG_ERR("JPG", "Failed to open JPEG (err=%d): %s", jpeg->getLastError(), imagePath.c_str()); return false; @@ -483,6 +509,7 @@ bool JpegToFramebufferConverter::decodeToFramebuffer(const std::string& imagePat // tallest single decode block: a JPEGDEC MCU cell is at most 16 scaled-source // rows tall, which our fine scale maps to this many output rows. ctx.caching = !config.cachePath.empty(); + ctx.drawToFb = !config.cacheOnly; if (ctx.caching) { const int maxBlockDstRows = (int)(((int64_t)16 * ctx.fineScaleFPY) >> FP_SHIFT) + 2; if (!ctx.cache.begin(config.cachePath, destWidth, destHeight, config.x, config.y, maxBlockDstRows)) { @@ -491,6 +518,12 @@ bool JpegToFramebufferConverter::decodeToFramebuffer(const std::string& imagePat } } + if (config.cacheOnly && !ctx.caching) { + // The cache is the sole output in this mode; decoding without it would only spend seconds. + LOG_ERR("JPG", "Cache-only decode with no cache stream, aborting"); + return false; + } + unsigned long decodeStart = millis(); ctx.lastYieldMs = decodeStart; rc = jpeg->decode(0, 0, jpegScaleOption); diff --git a/lib/Epub/Epub/converters/PngStreamDecoder.cpp b/lib/Epub/Epub/converters/PngStreamDecoder.cpp new file mode 100644 index 00000000000..a524ddbab62 --- /dev/null +++ b/lib/Epub/Epub/converters/PngStreamDecoder.cpp @@ -0,0 +1,346 @@ +#include "PngStreamDecoder.h" + +#include +#include +#include +#include + +#include + +#include "DirectPixelWriter.h" +#include "DitherUtils.h" +#include "ImageToFramebufferDecoder.h" +#include "PixelCache.h" + +namespace { + +constexpr uint8_t PNG_SIGNATURE[8] = {0x89, 'P', 'N', 'G', '\r', '\n', 0x1A, '\n'}; + +uint32_t readBE32(const uint8_t* p) { + return (uint32_t(p[0]) << 24) | (uint32_t(p[1]) << 16) | (uint32_t(p[2]) << 8) | uint32_t(p[3]); +} + +// Pulls IDAT payload bytes for InflateStream, walking chunk boundaries as it +// goes: after one IDAT's data it skips the CRC, reads the next chunk header, +// and continues only into another IDAT (the spec requires them consecutive). +struct IdatSource { + HalFile* file = nullptr; + uint32_t remainingInChunk = 0; + bool exhausted = false; + uint8_t buf[2048]; + + static size_t fill(void* ctx, const uint8_t** data) { + auto* self = static_cast(ctx); + if (self->exhausted) return 0; + + while (self->remainingInChunk == 0) { + // End of this IDAT: skip its CRC, then look at the next chunk. + uint8_t header[8]; + if (!self->file->seek(self->file->position() + 4) || self->file->read(header, 8) != 8) { + self->exhausted = true; + return 0; + } + if (memcmp(header + 4, "IDAT", 4) != 0) { + self->exhausted = true; // first non-IDAT chunk ends the stream + return 0; + } + self->remainingInChunk = readBE32(header); + } + + const size_t want = self->remainingInChunk < sizeof(self->buf) ? self->remainingInChunk : sizeof(self->buf); + const int got = self->file->read(self->buf, want); + if (got <= 0) { + self->exhausted = true; + return 0; + } + self->remainingInChunk -= static_cast(got); + *data = self->buf; + return static_cast(got); + } +}; + +int paethPredictor(const int a, const int b, const int c) { + const int p = a + b - c; + const int pa = p > a ? p - a : a - p; + const int pb = p > b ? p - b : b - p; + const int pc = p > c ? p - c : c - p; + if (pa <= pb && pa <= pc) return a; + if (pb <= pc) return b; + return c; +} + +// In-place reversal of the per-row filter. prev is the reconstructed previous +// row (all zeros for the first row, as the spec defines). +bool unfilterRow(const uint8_t filter, uint8_t* row, const uint8_t* prev, const size_t rowBytes, const size_t bpp) { + switch (filter) { + case 0: // None + return true; + case 1: // Sub + for (size_t i = bpp; i < rowBytes; i++) row[i] = uint8_t(row[i] + row[i - bpp]); + return true; + case 2: // Up + for (size_t i = 0; i < rowBytes; i++) row[i] = uint8_t(row[i] + prev[i]); + return true; + case 3: // Average + for (size_t i = 0; i < bpp; i++) row[i] = uint8_t(row[i] + prev[i] / 2); + for (size_t i = bpp; i < rowBytes; i++) row[i] = uint8_t(row[i] + (row[i - bpp] + prev[i]) / 2); + return true; + case 4: // Paeth + for (size_t i = 0; i < bpp; i++) row[i] = uint8_t(row[i] + prev[i]); + for (size_t i = bpp; i < rowBytes; i++) + row[i] = uint8_t(row[i] + paethPredictor(row[i - bpp], prev[i], prev[i - bpp])); + return true; + default: + return false; + } +} + +uint8_t luma(const uint8_t r, const uint8_t g, const uint8_t b) { return uint8_t((r * 77 + g * 150 + b * 29) >> 8); } + +uint8_t compositeWhite(const uint8_t gray, const uint8_t alpha) { + return uint8_t((gray * alpha + 255 * (255 - alpha)) / 255); +} + +// Scale factors for expanding sub-byte grayscale samples to 8 bits. +uint8_t expandGray(const uint32_t v, const int bitDepth) { + switch (bitDepth) { + case 1: + return v ? 255 : 0; + case 2: + return uint8_t(v * 85); + case 4: + return uint8_t(v * 17); + default: + return uint8_t(v); + } +} + +} // namespace + +bool PngStreamDecoder::decodeToCache(const std::string& pngPath, const std::string& cachePath, const int dstWidth, + const int dstHeight) { + if (dstWidth <= 0 || dstHeight <= 0) return false; + + HalFile file; + if (!Storage.openFileForRead("PNS", pngPath, file)) return false; + + uint8_t sig[8]; + if (file.read(sig, 8) != 8 || memcmp(sig, PNG_SIGNATURE, 8) != 0) return false; + + // --- Chunk walk up to the first IDAT --------------------------------------- + int srcWidth = 0, srcHeight = 0, bitDepth = 0, colorType = 0; + uint8_t palette[256 * 3]; + uint8_t paletteAlpha[256]; + int paletteEntries = 0; + bool haveTrns = false; + memset(paletteAlpha, 0xFF, sizeof(paletteAlpha)); + // tRNS for gray/RGB names one fully transparent sample value. + uint32_t transparentGray = 0xFFFFFFFF; + uint32_t transparentRgb = 0xFFFFFFFF; + + bool haveIhdr = false; + uint32_t idatLength = 0; + for (;;) { + uint8_t header[8]; + if (file.read(header, 8) != 8) return false; + const uint32_t length = readBE32(header); + const char* type = reinterpret_cast(header + 4); + + if (memcmp(type, "IHDR", 4) == 0) { + uint8_t ihdr[13]; + if (length != 13 || file.read(ihdr, 13) != 13) return false; + srcWidth = static_cast(readBE32(ihdr)); + srcHeight = static_cast(readBE32(ihdr + 4)); + bitDepth = ihdr[8]; + colorType = ihdr[9]; + const uint8_t interlace = ihdr[12]; + ImageDimensions dims; + if (!ImageToFramebufferDecoder::validateAndStoreDimensions(srcWidth, srcHeight, dims, "PNG(stream)")) + return false; + if (interlace != 0 || ihdr[10] != 0 || ihdr[11] != 0) { + LOG_DBG("PNS", "Declining interlaced/nonstandard PNG (interlace=%d)", interlace); + return false; // PNGdec fallback handles Adam7 when the heap allows + } + const bool depthOk = (colorType == 0 || colorType == 3) + ? (bitDepth == 1 || bitDepth == 2 || bitDepth == 4 || bitDepth == 8) + : (bitDepth == 8); + const bool typeOk = colorType == 0 || colorType == 2 || colorType == 3 || colorType == 4 || colorType == 6; + if (!depthOk || !typeOk) { + LOG_DBG("PNS", "Declining PNG type=%d depth=%d", colorType, bitDepth); + return false; + } + if (!file.seek(file.position() + 4)) return false; // CRC + haveIhdr = true; + } else if (memcmp(type, "PLTE", 4) == 0) { + paletteEntries = static_cast(length / 3); + if (paletteEntries > 256) paletteEntries = 256; + if (file.read(palette, static_cast(paletteEntries) * 3) != paletteEntries * 3) return false; + if (!file.seek(file.position() + (length - static_cast(paletteEntries) * 3) + 4)) return false; + } else if (memcmp(type, "tRNS", 4) == 0) { + haveTrns = true; + if (colorType == 3) { + const uint32_t n = length < 256 ? length : 256; + if (file.read(paletteAlpha, n) != static_cast(n)) return false; + if (!file.seek(file.position() + (length - n) + 4)) return false; + } else if (colorType == 0 && length >= 2) { + uint8_t v[2]; + if (file.read(v, 2) != 2) return false; + transparentGray = (uint32_t(v[0]) << 8) | v[1]; + if (!file.seek(file.position() + (length - 2) + 4)) return false; + } else if (colorType == 2 && length >= 6) { + uint8_t v[6]; + if (file.read(v, 6) != 6) return false; + // 8-bit images store the sample in the low byte of each 16-bit field. + transparentRgb = (uint32_t(v[1]) << 16) | (uint32_t(v[3]) << 8) | v[5]; + if (!file.seek(file.position() + (length - 6) + 4)) return false; + } else { + if (!file.seek(file.position() + length + 4)) return false; + } + } else if (memcmp(type, "IDAT", 4) == 0) { + idatLength = length; + break; // file is positioned at the IDAT payload + } else if (memcmp(type, "IEND", 4) == 0) { + return false; // no pixel data + } else { + if (!file.seek(file.position() + length + 4)) return false; + } + } + if (!haveIhdr) return false; + if (colorType == 3 && paletteEntries == 0) return false; + + // --- Working buffers ------------------------------------------------------- + const int channels = (colorType == 2) ? 3 : (colorType == 4) ? 2 : (colorType == 6) ? 4 : 1; + const size_t rowBytes = (static_cast(srcWidth) * channels * bitDepth + 7) / 8; + const size_t bpp = (static_cast(channels) * bitDepth + 7) / 8; + + auto curRow = makeUniqueNoThrow(rowBytes); + auto prevRow = makeUniqueNoThrow(rowBytes); + auto grayLine = makeUniqueNoThrow(static_cast(srcWidth)); + if (!curRow || !prevRow || !grayLine) { + LOG_ERR("PNS", "OOM for row buffers (%u bytes/row)", static_cast(rowBytes)); + return false; + } + memset(prevRow.get(), 0, rowBytes); + + IdatSource source; + source.file = &file; + source.remainingInChunk = idatLength; + + InflateStream inflate; + if (!inflate.init(/*streaming=*/true)) { + LOG_ERR("PNS", "OOM for inflate state"); + return false; + } + inflate.setZlibWrapped(); + inflate.setFill(&IdatSource::fill, &source); + + PixelCache cache; + if (!cache.begin(cachePath, dstWidth, dstHeight, 0, 0, 1)) { + LOG_ERR("PNS", "Failed to start cache stream"); + return false; + } + + // --- Row loop -------------------------------------------------------------- + uint32_t lastYieldMs = 0; + int lastDstY = -1; + bool ok = true; + for (int srcY = 0; srcY < srcHeight && ok; srcY++) { + uint8_t filter; + if (!inflate.read(&filter, 1) || !inflate.read(curRow.get(), rowBytes) || + !unfilterRow(filter, curRow.get(), prevRow.get(), rowBytes, bpp)) { + LOG_ERR("PNS", "Stream error at row %d", srcY); + ok = false; + break; + } + + // Same output-row selection as the PNGdec callback: downscaling picks one + // destination row for several source rows, upscaling repeats one source + // row across its whole destination range. + int firstDstY = (srcY * dstHeight) / srcHeight; + int endDstY = dstHeight > srcHeight ? ((srcY + 1) * dstHeight) / srcHeight : firstDstY + 1; + if (firstDstY <= lastDstY) firstDstY = lastDstY + 1; + if (endDstY > dstHeight) endDstY = dstHeight; + + if (firstDstY < endDstY) { + // Expand this row to 8-bit gray, alpha composited against white. + const uint8_t* src = curRow.get(); + for (int x = 0; x < srcWidth; x++) { + uint8_t g; + switch (colorType) { + case 0: { + uint32_t v; + if (bitDepth == 8) { + v = src[x]; + } else { + const size_t bitPos = static_cast(x) * bitDepth; + v = (src[bitPos >> 3] >> (8 - bitDepth - (bitPos & 7))) & ((1u << bitDepth) - 1); + } + g = expandGray(v, bitDepth); + if (haveTrns && v == transparentGray) g = 255; + break; + } + case 2: { + const uint8_t* p = src + static_cast(x) * 3; + g = luma(p[0], p[1], p[2]); + if (haveTrns && ((uint32_t(p[0]) << 16) | (uint32_t(p[1]) << 8) | p[2]) == transparentRgb) g = 255; + break; + } + case 3: { + uint32_t idx; + if (bitDepth == 8) { + idx = src[x]; + } else { + const size_t bitPos = static_cast(x) * bitDepth; + idx = (src[bitPos >> 3] >> (8 - bitDepth - (bitPos & 7))) & ((1u << bitDepth) - 1); + } + if (idx >= static_cast(paletteEntries)) idx = 0; + const uint8_t* p = palette + idx * 3; + g = compositeWhite(luma(p[0], p[1], p[2]), paletteAlpha[idx]); + break; + } + case 4: { + const uint8_t* p = src + static_cast(x) * 2; + g = compositeWhite(p[0], p[1]); + break; + } + default: { // 6 + const uint8_t* p = src + static_cast(x) * 4; + g = compositeWhite(luma(p[0], p[1], p[2]), p[3]); + break; + } + } + grayLine[x] = g; + } + + for (int dstY = firstDstY; dstY < endDstY && ok; dstY++) { + lastDstY = dstY; + if (!cache.advanceTo(dstY)) { + ok = false; + break; + } + DirectCacheWriter cw; + cw.init(cache.buffer, cache.bytesPerRow, cache.bandRows, cache.originX); + cw.beginRow(dstY, cache.bandStart); + int srcX = 0; + int error = 0; + for (int dstX = 0; dstX < dstWidth; dstX++) { + cw.writePixel(dstX, applyBayerDither4Level(grayLine[srcX], dstX, dstY)); + error += srcWidth; + while (error >= dstWidth) { + error -= dstWidth; + if (srcX < srcWidth - 1) srcX++; + } + } + } + } + + std::swap(curRow, prevRow); + ImageToFramebufferDecoder::yieldDuringDecode(lastYieldMs); + } + + if (!ok) { + cache.abort(); + return false; + } + return cache.finalize(); +} diff --git a/lib/Epub/Epub/converters/PngStreamDecoder.h b/lib/Epub/Epub/converters/PngStreamDecoder.h new file mode 100644 index 00000000000..90669698d71 --- /dev/null +++ b/lib/Epub/Epub/converters/PngStreamDecoder.h @@ -0,0 +1,36 @@ +#pragma once + +#include + +// PNG -> .pxc pixel-cache decoder that streams IDAT through InflateStream +// instead of PNGdec. +// +// PNGdec's working set is one ~62KB object (32KB zlib window + inflate state + +// row buffers, allocated as a single new PNG()), and the heap frequently +// cannot produce that hole: a session was measured pinned at a 45,044-byte +// largest block from boot onward, where releasing every font cache moved it by +// zero bytes. Under that ceiling every decode -- build-time and render-time -- +// fails, and which chapters of a book get their pictures becomes a lottery +// over heap layout. +// +// This decoder's inflate state comes from InflateStream: ~43KB claimed from +// the lent framebuffer when a FrameBufferLoan is active (build-time path, +// costing the heap nothing), or two separate heap blocks of 11KB + 32KB +// otherwise (render-time path -- 32KB contiguous is available even under the +// measured ceiling). What remains on the heap here is two filter rows, a gray +// line and the cache's single-row band: a few KB, bounded by image width. +// +// Coverage: non-interlaced PNGs, bit depth 1/2/4/8 for grayscale and palette, +// 8-bit for RGB / gray+alpha / RGBA. Alpha (channel or tRNS) is composited +// against white -- the page background -- rather than skipped, so transparent +// regions cache as white instead of the black that skipped pixels leave. +// Anything else (interlace, 16-bit) is declined so the caller can fall back +// to PNGdec, which handles them when the heap allows. +class PngStreamDecoder { + public: + // Decode the PNG file at pngPath into a finalized .pxc at cachePath, scaled + // to exactly dstWidth x dstHeight (the layout's display size). Returns false + // on unsupported form, OOM, or a corrupt stream; a partial cache file is + // never left behind. + static bool decodeToCache(const std::string& pngPath, const std::string& cachePath, int dstWidth, int dstHeight); +}; diff --git a/lib/Epub/Epub/css/CssParser.cpp b/lib/Epub/Epub/css/CssParser.cpp index da079fe8973..4d39120f452 100644 --- a/lib/Epub/Epub/css/CssParser.cpp +++ b/lib/Epub/Epub/css/CssParser.cpp @@ -12,6 +12,8 @@ #include #include +#include "CssSelectorUsage.h" + namespace { // Stack-allocated string buffer to avoid heap reallocations during parsing @@ -45,10 +47,6 @@ constexpr size_t MAX_RULES = 1500; constexpr size_t SELECTOR_POOL_CAP = 32 * 1024; constexpr size_t MAX_UNIQUE_STYLES = 256; -// Minimum free heap required to apply CSS during rendering -// If below this threshold, we skip CSS to avoid display artifacts. -constexpr size_t MIN_FREE_HEAP_FOR_CSS = 48 * 1024; - // Maximum length for a single selector string // Prevents parsing of extremely long or malformed selectors constexpr size_t MAX_SELECTOR_LENGTH = 256; @@ -138,14 +136,15 @@ std::string_view stripTrailingImportant(std::string_view value) { } constexpr std::array STYLE_LENGTH_FIELDS = { - &CssStyle::textIndent, &CssStyle::marginTop, &CssStyle::marginBottom, &CssStyle::marginLeft, - &CssStyle::marginRight, &CssStyle::paddingTop, &CssStyle::paddingBottom, &CssStyle::paddingLeft, - &CssStyle::paddingRight, &CssStyle::imageHeight, &CssStyle::imageWidth, + &CssStyle::textIndent, &CssStyle::marginTop, &CssStyle::marginBottom, &CssStyle::marginLeft, + &CssStyle::marginRight, &CssStyle::paddingTop, &CssStyle::paddingBottom, &CssStyle::paddingLeft, + &CssStyle::paddingRight, &CssStyle::imageHeight, &CssStyle::imageWidth, &CssStyle::imageMaxHeight, + &CssStyle::imageMaxWidth, }; constexpr size_t STYLE_LENGTH_FIELD_COUNT = STYLE_LENGTH_FIELDS.size(); constexpr size_t STYLE_WIRE_BYTES = - 5 + STYLE_LENGTH_FIELD_COUNT * (sizeof(decltype(CssLength::value)) + 1) + 3 + sizeof(uint32_t); -constexpr uint32_t CSS_DEFINED_BITS_MASK = (1u << 19) - 1; + 5 + STYLE_LENGTH_FIELD_COUNT * (sizeof(decltype(CssLength::value)) + 1) + 5 + sizeof(uint32_t); +constexpr uint32_t CSS_DEFINED_BITS_MASK = (1u << 23) - 1; void encodeStyleWire(const CssStyle& style, uint8_t (&out)[STYLE_WIRE_BYTES]) { size_t offset = 0; @@ -166,6 +165,8 @@ void encodeStyleWire(const CssStyle& style, uint8_t (&out)[STYLE_WIRE_BYTES]) { out[offset++] = static_cast(style.display); out[offset++] = static_cast(style.verticalAlign); out[offset++] = static_cast(style.listStyleType); + out[offset++] = static_cast(style.textEmphasis); + out[offset++] = static_cast(style.textOrientation); uint32_t definedBits = 0; if (style.defined.textAlign) definedBits |= 1 << 0; @@ -187,6 +188,10 @@ void encodeStyleWire(const CssStyle& style, uint8_t (&out)[STYLE_WIRE_BYTES]) { if (style.defined.direction) definedBits |= 1 << 16; if (style.defined.verticalAlign) definedBits |= 1 << 17; if (style.defined.listStyleType) definedBits |= 1 << 18; + if (style.defined.textEmphasis) definedBits |= 1 << 19; + if (style.defined.textOrientation) definedBits |= 1 << 20; + if (style.defined.imageMaxHeight) definedBits |= 1 << 21; + if (style.defined.imageMaxWidth) definedBits |= 1 << 22; memcpy(out + offset, &definedBits, sizeof(definedBits)); } @@ -225,13 +230,19 @@ bool decodeStyleWire(const uint8_t (&in)[STYLE_WIRE_BYTES], CssStyle& style) { const uint8_t display = in[offset++]; const uint8_t verticalAlign = in[offset++]; const uint8_t listStyleType = in[offset++]; + const uint8_t textEmphasis = in[offset++]; + const uint8_t textOrientation = in[offset++]; if (display > static_cast(CssDisplay::None) || verticalAlign > static_cast(CssVerticalAlign::Sub) || - listStyleType > static_cast(CssListStyleType::None)) { + listStyleType > static_cast(CssListStyleType::None) || + textEmphasis > static_cast(CssTextEmphasis::OpenDoubleCircle) || + textOrientation > static_cast(CssTextOrientation::Combine)) { return false; } style.display = static_cast(display); style.verticalAlign = static_cast(verticalAlign); style.listStyleType = static_cast(listStyleType); + style.textEmphasis = static_cast(textEmphasis); + style.textOrientation = static_cast(textOrientation); uint32_t definedBits = 0; memcpy(&definedBits, in + offset, sizeof(definedBits)); @@ -255,18 +266,21 @@ bool decodeStyleWire(const uint8_t (&in)[STYLE_WIRE_BYTES], CssStyle& style) { style.defined.direction = (definedBits & 1 << 16) != 0; style.defined.verticalAlign = (definedBits & 1 << 17) != 0; style.defined.listStyleType = (definedBits & 1 << 18) != 0; + style.defined.textEmphasis = (definedBits & 1 << 19) != 0; + style.defined.textOrientation = (definedBits & 1 << 20) != 0; + style.defined.imageMaxHeight = (definedBits & 1 << 21) != 0; + style.defined.imageMaxWidth = (definedBits & 1 << 22) != 0; return true; } } // anonymous namespace -int CssParser::compareEntryToPieces(const SelectorEntry& entry, const std::string_view p0, const std::string_view p1, - const std::string_view p2) const { +int CssParser::compareEntryToPieces(const SelectorEntry& entry, const std::string_view* pieces, + const size_t pieceCount) const { const char* stored = selectorPool_.get() + entry.offset; - const std::string_view pieces[] = {p0, p1, p2}; size_t index = 0; - for (const std::string_view piece : pieces) { - for (const char c : piece) { + for (size_t p = 0; p < pieceCount; ++p) { + for (const char c : pieces[p]) { if (index == entry.length) return -1; const auto storedByte = static_cast(stored[index]); const auto probeByte = static_cast(asciiToLower(c)); @@ -277,22 +291,27 @@ int CssParser::compareEntryToPieces(const SelectorEntry& entry, const std::strin return index == entry.length ? 0 : 1; } -size_t CssParser::lowerBound(const std::string_view p0, const std::string_view p1, const std::string_view p2, - bool& exact) const { +size_t CssParser::lowerBound(const std::string_view* pieces, const size_t pieceCount, bool& exact) const { size_t low = 0; size_t high = entryCount_; while (low < high) { const size_t middle = low + (high - low) / 2; - if (compareEntryToPieces(entries_[middle], p0, p1, p2) < 0) { + if (compareEntryToPieces(entries_[middle], pieces, pieceCount) < 0) { low = middle + 1; } else { high = middle; } } - exact = low < entryCount_ && compareEntryToPieces(entries_[low], p0, p1, p2) == 0; + exact = low < entryCount_ && compareEntryToPieces(entries_[low], pieces, pieceCount) == 0; return low; } +size_t CssParser::lowerBound(const std::string_view p0, const std::string_view p1, const std::string_view p2, + bool& exact) const { + const std::string_view pieces[] = {p0, p1, p2}; + return lowerBound(pieces, 3, exact); +} + const CssStyle* CssParser::findStyle(const std::string_view p0, const std::string_view p1, const std::string_view p2) const { bool exact = false; @@ -300,6 +319,14 @@ const CssStyle* CssParser::findStyle(const std::string_view p0, const std::strin return exact ? &stylePool_[entries_[index].styleIndex] : nullptr; } +const CssStyle* CssParser::findDescendantStyle(const std::string_view ancestorPart, const std::string_view p0, + const std::string_view p1, const std::string_view p2) const { + const std::string_view pieces[] = {ancestorPart, " ", p0, p1, p2}; + bool exact = false; + const size_t index = lowerBound(pieces, 5, exact); + return exact ? &stylePool_[entries_[index].styleIndex] : nullptr; +} + std::string_view CssParser::selectorAt(const size_t index) const { const SelectorEntry& entry = entries_[index]; return {selectorPool_.get() + entry.offset, entry.length}; @@ -424,6 +451,7 @@ CssParser::RuleInsertResult CssParser::insertOrMerge(const std::string_view sele char* destination = selectorPool_.get() + selectorOffset; for (const char c : selector) *destination++ = asciiToLower(c); selectorPoolSize_ = static_cast(requiredSelectorBytes); + if (selector.find(' ') != std::string_view::npos) hasDescendantRules_ = true; SelectorEntry* entries = entries_.get(); memmove(entries + position + 1, entries + position, (entryCount_ - position) * sizeof(SelectorEntry)); @@ -486,6 +514,56 @@ CssTextDecoration CssParser::interpretDecoration(std::string_view val) { return explicitNone ? CssTextDecoration::None : result; } +CssTextEmphasis CssParser::interpretTextEmphasis(std::string_view val) { + // Shorthand grammar is " || " in either order, with the colour of the + // text-emphasis shorthand ignored here. An omitted fill is "filled" and an omitted + // shape is "sesame", matching what a bare "text-emphasis: dot" or ": open" means. + bool none = false; + bool open = false; + bool haveShape = false; + CssTextEmphasis shape = CssTextEmphasis::FilledSesame; + forEachDelimitedToken(stripTrailingImportant(val), isCssWhitespace, [&](const std::string_view token) { + if (iequalsAscii(token, "none")) { + none = true; + } else if (iequalsAscii(token, "open")) { + open = true; + } else if (iequalsAscii(token, "filled")) { + open = false; + } else if (iequalsAscii(token, "sesame")) { + shape = CssTextEmphasis::FilledSesame; + haveShape = true; + } else if (iequalsAscii(token, "double-circle")) { + shape = CssTextEmphasis::FilledDoubleCircle; + haveShape = true; + } else if (iequalsAscii(token, "circle")) { + shape = CssTextEmphasis::FilledCircle; + haveShape = true; + } else if (iequalsAscii(token, "triangle")) { + shape = CssTextEmphasis::FilledTriangle; + haveShape = true; + } else if (iequalsAscii(token, "dot")) { + shape = CssTextEmphasis::FilledDot; + haveShape = true; + } + }); + + if (none) return CssTextEmphasis::None; + if (!haveShape) shape = CssTextEmphasis::FilledSesame; + if (!open) return shape; + switch (shape) { + case CssTextEmphasis::FilledDoubleCircle: + return CssTextEmphasis::OpenDoubleCircle; + case CssTextEmphasis::FilledCircle: + return CssTextEmphasis::OpenCircle; + case CssTextEmphasis::FilledTriangle: + return CssTextEmphasis::OpenTriangle; + case CssTextEmphasis::FilledDot: + return CssTextEmphasis::OpenDot; + default: + return CssTextEmphasis::OpenSesame; + } +} + CssLength CssParser::interpretLength(std::string_view val) { CssLength result; tryInterpretLength(val, result); @@ -638,6 +716,47 @@ void CssParser::parseDeclarationIntoStyle(std::string_view decl, CssStyle& style const std::string_view listStyleValue = stripTrailingImportant(value); style.listStyleType = iequalsAscii(listStyleValue, "none") ? CssListStyleType::None : CssListStyleType::Disc; style.defined.listStyleType = 1; + } else if (iequalsAscii(name, "max-height")) { + CssLength len; + if (tryInterpretLength(value, len)) { + style.imageMaxHeight = len; + style.defined.imageMaxHeight = 1; + } + } else if (iequalsAscii(name, "max-width")) { + CssLength len; + if (tryInterpretLength(value, len)) { + style.imageMaxWidth = len; + style.defined.imageMaxWidth = 1; + } + } else if (iequalsAscii(name, "text-emphasis-style") || iequalsAscii(name, "text-emphasis") || + iequalsAscii(name, "-epub-text-emphasis-style") || iequalsAscii(name, "-epub-text-emphasis") || + iequalsAscii(name, "-webkit-text-emphasis-style") || iequalsAscii(name, "-webkit-text-emphasis")) { + style.textEmphasis = interpretTextEmphasis(value); + style.defined.textEmphasis = 1; + } else if (iequalsAscii(name, "text-orientation") || iequalsAscii(name, "-epub-text-orientation") || + iequalsAscii(name, "-webkit-text-orientation")) { + // sideways-right is the pre-standard spelling of sideways; use-glyph-orientation and + // anything else fall back to mixed, which is what a reader without the property does. + if (iequalsAscii(value, "upright")) { + style.textOrientation = CssTextOrientation::Upright; + style.defined.textOrientation = 1; + } else if (iequalsAscii(value, "sideways") || iequalsAscii(value, "sideways-right")) { + style.textOrientation = CssTextOrientation::Sideways; + style.defined.textOrientation = 1; + } else if (iequalsAscii(value, "mixed")) { + style.textOrientation = CssTextOrientation::Mixed; + style.defined.textOrientation = 1; + } + } else if (iequalsAscii(name, "text-combine-upright") || iequalsAscii(name, "-epub-text-combine") || + iequalsAscii(name, "-webkit-text-combine") || iequalsAscii(name, "-epub-text-combine-upright") || + iequalsAscii(name, "-webkit-text-combine-upright")) { + // "all" is the standard value; "horizontal" the old -webkit- one. "none" is left alone + // rather than recorded as Mixed so it cannot cancel an upright text-orientation on the + // same element (the two properties share one slot here). + if (iequalsAscii(value, "all") || iequalsAscii(value, "horizontal")) { + style.textOrientation = CssTextOrientation::Combine; + style.defined.textOrientation = 1; + } } } @@ -676,8 +795,9 @@ void CssParser::processRuleBlockWithStyle(std::string_view selectorGroup, const } // TODO: Support richer CSS selector syntax in the future. For now we only - // handle `tag`, `.class`, or `tag.class`. Reject anything containing a - // character that introduces unsupported syntax: + // handle `tag`, `.class`, or `tag.class`, alone or as the two parts of one + // descendant selector (`.a .b`). Reject anything containing a character that + // introduces unsupported syntax: // '+' adjacent sibling combinator // '>' child combinator // '[' attribute selector @@ -685,11 +805,25 @@ void CssParser::processRuleBlockWithStyle(std::string_view selectorGroup, const // '#' ID selector // '~' general sibling combinator // '*' wildcard - // ' ' descendant combinator - // Single-pass scan via find_first_of instead of eight sequential find() calls. - constexpr std::string_view kUnsupportedSelectorChars = "+>[:#~* "; + // Single-pass scan via find_first_of instead of seven sequential find() calls. + constexpr std::string_view kUnsupportedSelectorChars = "+>[:#~*"; if (sel.find_first_of(kUnsupportedSelectorChars) != std::string_view::npos) return; + // Descendant combinator: keep exactly two simple parts, stored as " " with one + // space. Deeper chains are still dropped -- resolveStyle matches one ancestor level + // against the first part, which is the shape the EBPAJ writing-mode classes take + // (".vrtl .h-indent-1em"; 956 of that template's 1,905 rules). + std::string descendantKey; + if (sel.find_first_of(" \t\r\n\f") != std::string_view::npos) { + size_t parts = 0; + forEachDelimitedToken(sel, isCssWhitespace, [&](std::string_view part) { + if (parts++ > 0) descendantKey += ' '; + descendantKey.append(part.data(), part.size()); + }); + if (parts != 2) return; + sel = descendantKey; + } + if (ruleGrowthStopped_) { // Continue the cascade for stored selectors without retrying failed // allocations for new rules. @@ -870,15 +1004,10 @@ CssParser::ParseResult CssParser::loadFromStream(HalFile& source) { // Style resolution CssStyle CssParser::resolveStyle(std::string_view tagName, std::string_view classAttr) const { - static bool lowHeapWarningLogged = false; - if (ESP.getFreeHeap() < MIN_FREE_HEAP_FOR_CSS) { - if (!lowHeapWarningLogged) { - lowHeapWarningLogged = true; - LOG_DBG("CSS", "Warning: low heap (%u bytes) below MIN_FREE_HEAP_FOR_CSS (%u), returning empty style", - ESP.getFreeHeap(), static_cast(MIN_FREE_HEAP_FOR_CSS)); - } - return CssStyle{}; - } + // No heap guard here: nothing below allocates. result is a stack struct of enums, lengths and + // bitfields and findStyle() only reads the bounded store. Upstream returns an empty style below + // 48KB free; on the X3 that sits inside the 44-62KB reading range, so a chapter's vertical, + // ruby and emphasis classes applied to some paragraphs and silently not to others. CssStyle result; @@ -908,6 +1037,39 @@ CssStyle CssParser::resolveStyle(std::string_view tagName, std::string_view clas return result; } +CssStyle CssParser::resolveStyle(const std::string_view tagName, const std::string_view classAttr, + const AncestorRef* ancestors, const size_t ancestorCount) const { + CssStyle result = resolveStyle(tagName, classAttr); + if (!hasDescendantRules_ || ancestors == nullptr || ancestorCount == 0) return result; + + // Descendant rules, in CSS specificity order. The single-level lookups above already hold + // (0,0,1), (0,1,0) and (0,1,1); a tag-first descendant adds one element, a class-first one + // adds one class, so the passes run: "tag x" (0,0,2)/(0,1,1)/(0,1,2), then ".a x" + // (0,1,1)/(0,2,0)/(0,2,1). Later passes win, as do nearer ancestors within a pass, which is + // how a browser breaks the remaining ties (source order is not tracked here). + const auto applyForAncestorPart = [&](const std::string_view part) { + if (const CssStyle* style = findDescendantStyle(part, tagName)) result.applyOver(*style); + forEachDelimitedToken(classAttr, isCssWhitespace, [&](std::string_view cls) { + if (const CssStyle* style = findDescendantStyle(part, ".", cls)) result.applyOver(*style); + }); + forEachDelimitedToken(classAttr, isCssWhitespace, [&](std::string_view cls) { + if (const CssStyle* style = findDescendantStyle(part, tagName, ".", cls)) result.applyOver(*style); + }); + }; + for (size_t a = 0; a < ancestorCount; ++a) { + if (!ancestors[a].tag.empty()) applyForAncestorPart(ancestors[a].tag); + } + std::string part; + for (size_t a = 0; a < ancestorCount; ++a) { + forEachDelimitedToken(ancestors[a].classAttr, isCssWhitespace, [&](std::string_view cls) { + part.assign("."); + part.append(cls.data(), cls.size()); + applyForAncestorPart(part); + }); + } + return result; +} + // Inline style parsing (static - doesn't need rule database) CssStyle CssParser::parseInlineStyle(std::string_view styleValue) { return parseDeclarations(styleValue); } @@ -1082,7 +1244,7 @@ bool CssParser::saveToCache(const bool complete) const { return true; } -CssParser::CacheLoadResult CssParser::loadFromCache() { +CssParser::CacheLoadResult CssParser::loadFromCache(const CssSelectorUsage* usage) { if (cachePath.empty()) { return CacheLoadResult::Invalid; } @@ -1174,6 +1336,12 @@ CssParser::CacheLoadResult CssParser::loadFromCache() { return CacheLoadResult::Invalid; } + // Skip rules that can never match the scanned chapter; generic publisher templates register + // hundreds of rules of which a chapter uses a handful, and the bytes have already been consumed. + if (usage != nullptr && !usage->matches(std::string(selectorBuffer.get(), selectorLen))) { + continue; + } + const RuleInsertResult insertResult = insertOrMerge(std::string_view(selectorBuffer.get(), selectorLen), style); if (insertResult == RuleInsertResult::OutOfMemory) { clear(); @@ -1196,6 +1364,7 @@ CssParser::CacheLoadResult CssParser::loadFromCache() { } const bool partial = (flags & CSS_CACHE_FLAG_PARTIAL) != 0; - LOG_DBG("CSS", "Loaded %u rules from %s cache", ruleCount, partial ? "partial" : "complete"); + LOG_DBG("CSS", "Loaded %u of %u rules from %s cache%s", entryCount_, ruleCount, partial ? "partial" : "complete", + usage != nullptr ? " (usage-filtered)" : ""); return CacheLoadResult::Complete; } diff --git a/lib/Epub/Epub/css/CssParser.h b/lib/Epub/Epub/css/CssParser.h index e200877d671..16bd5ce238f 100644 --- a/lib/Epub/Epub/css/CssParser.h +++ b/lib/Epub/Epub/css/CssParser.h @@ -9,6 +9,8 @@ #include "CssStyle.h" +class CssSelectorUsage; + /** * Lightweight CSS parser for EPUB stylesheets * @@ -50,7 +52,12 @@ class CssParser { }; // Bump when CSS cache format or rules change; section caches are invalidated when this changes - static constexpr uint8_t CSS_CACHE_VERSION = 12; + // v13 (this tree): the style record also carries text-emphasis and text-orientation / + // text-combine-upright (two enum bytes, defined bits 19-20), and two-part descendant + // selectors (".vrtl .start-1em") are stored under a key with a single space. Upstream + // 1.6.5 is v12 with the shorter record, so the number has to differ. + // v14 (this tree): max-width / max-height (two more lengths, defined bits 21-22). + static constexpr uint8_t CSS_CACHE_VERSION = 14; explicit CssParser(std::string cachePath) : cachePath(std::move(cachePath)) {} ~CssParser() = default; @@ -77,6 +84,20 @@ class CssParser { */ [[nodiscard]] CssStyle resolveStyle(std::string_view tagName, std::string_view classAttr) const; + /** + * Same, with the open ancestors of the element (outermost first) so two-part descendant + * selectors can match. Each ancestor is its tag name and class attribute. Only ".a .b", + * ".a tag", ".a tag.b" and the same three with a tag as the first part are stored; the + * EBPAJ template every surveyed Japanese book carries writes its writing-mode-specific + * rules that way (".vrtl .h-indent-1em", ".hltr .start-1em"), 956 of its 1,905 rules. + */ + struct AncestorRef { + std::string tag; + std::string classAttr; + }; + [[nodiscard]] CssStyle resolveStyle(std::string_view tagName, std::string_view classAttr, + const AncestorRef* ancestors, size_t ancestorCount) const; + /** * Parse an inline style attribute string. * @param styleValue The value of a style="" attribute @@ -105,6 +126,7 @@ class CssParser { selectorPoolSize_ = selectorPoolCapacity_ = 0; styleCount_ = styleCapacity_ = 0; ruleGrowthStopped_ = false; + hasDescendantRules_ = false; } /** @@ -131,7 +153,8 @@ class CssParser { * Clears any existing rules before loading. * @return Complete when loaded, LowMemory when it should be retried, otherwise Invalid */ - CacheLoadResult loadFromCache(); + // usage: keep only rules whose selector can match the scanned chapter (nullptr = all). + CacheLoadResult loadFromCache(const CssSelectorUsage* usage = nullptr); private: enum class RuleInsertResult : uint8_t { @@ -166,17 +189,24 @@ class CssParser { uint16_t styleCount_ = 0; uint16_t styleCapacity_ = 0; bool ruleGrowthStopped_ = false; + // Set once any stored key carries a descendant combinator, so resolveStyle can skip the + // ancestor lookups entirely for the many stylesheets that have none. + bool hasDescendantRules_ = false; std::string cachePath; // Internal parsing helpers bool restoreCacheBackupIfNeeded() const; void processRuleBlockWithStyle(std::string_view selectorGroup, const CssStyle& style); - [[nodiscard]] int compareEntryToPieces(const SelectorEntry& entry, std::string_view p0, std::string_view p1, - std::string_view p2) const; + [[nodiscard]] int compareEntryToPieces(const SelectorEntry& entry, const std::string_view* pieces, + size_t pieceCount) const; + [[nodiscard]] size_t lowerBound(const std::string_view* pieces, size_t pieceCount, bool& exact) const; [[nodiscard]] size_t lowerBound(std::string_view p0, std::string_view p1, std::string_view p2, bool& exact) const; [[nodiscard]] const CssStyle* findStyle(std::string_view p0, std::string_view p1 = {}, std::string_view p2 = {}) const; + // Descendant key: " ", e.g. (".vrtl", "p", ".", "x") -> ".vrtl p.x". + [[nodiscard]] const CssStyle* findDescendantStyle(std::string_view ancestorPart, std::string_view p0, + std::string_view p1 = {}, std::string_view p2 = {}) const; [[nodiscard]] std::string_view selectorAt(size_t index) const; RuleInsertResult insertOrMerge(std::string_view selector, const CssStyle& style); PoolResult ensureEntryCapacity(size_t needed); @@ -191,6 +221,7 @@ class CssParser { static CssFontStyle interpretFontStyle(std::string_view val); static CssFontWeight interpretFontWeight(std::string_view val); static CssTextDecoration interpretDecoration(std::string_view val); + static CssTextEmphasis interpretTextEmphasis(std::string_view val); static CssLength interpretLength(std::string_view val); /** Returns true only when a numeric length was parsed (e.g. 2em, 50%). False for auto/inherit/initial. */ static bool tryInterpretLength(std::string_view val, CssLength& out); diff --git a/lib/Epub/Epub/css/CssSelectorUsage.cpp b/lib/Epub/Epub/css/CssSelectorUsage.cpp new file mode 100644 index 00000000000..6811196b1a0 --- /dev/null +++ b/lib/Epub/Epub/css/CssSelectorUsage.cpp @@ -0,0 +1,234 @@ +#include "CssSelectorUsage.h" + +#include +#include + +#include + +namespace { + +// Buffer size for streaming the HTML file (matches CssParser::loadFromStream) +constexpr size_t READ_BUFFER_SIZE = 512; + +bool isNameChar(const char c) { + return std::isalnum(static_cast(c)) || c == '-' || c == '_' || c == ':'; +} + +bool isHtmlWhitespace(const char c) { return c == ' ' || c == '\t' || c == '\n' || c == '\r' || c == '\f'; } + +} // namespace + +bool CssSelectorUsage::contains(const std::vector& list, const std::string_view name) { + for (const auto& entry : list) { + if (entry == name) { + return true; + } + } + return false; +} + +void CssSelectorUsage::addTag(const std::string_view name) { + if (name.empty() || overflowed_ || contains(tags_, name)) { + return; + } + if (tags_.size() >= MAX_TAGS) { + overflowed_ = true; + return; + } + tags_.emplace_back(name); +} + +void CssSelectorUsage::addClass(const std::string_view name) { + if (name.empty() || overflowed_ || contains(classes_, name)) { + return; + } + if (classes_.size() >= MAX_CLASSES) { + overflowed_ = true; + return; + } + classes_.emplace_back(name); +} + +bool CssSelectorUsage::matches(const std::string& selectorKey) const { + if (overflowed_) { + return true; + } + const std::string_view key(selectorKey); + // A two-part descendant key (" ") can match only if each part can on its own. + const size_t space = key.find(' '); + if (space != std::string_view::npos) { + return matches(std::string(key.substr(0, space))) && matches(std::string(key.substr(space + 1))); + } + const size_t dot = key.find('.'); + if (dot == std::string_view::npos) { + return containsTag(key); + } + if (dot == 0) { + return containsClass(key.substr(1)); + } + // "tag.class" — both parts must be present in the document. + // Keys with multiple dots (e.g. "p.a.b") are unreachable by resolveStyle() + // and correctly fail the class lookup here. + return containsTag(key.substr(0, dot)) && containsClass(key.substr(dot + 1)); +} + +bool CssSelectorUsage::scanHtmlFile(const std::string& path) { + HalFile file; + if (!Storage.openFileForRead("CSU", path, file)) { + return false; + } + + // Minimal streaming markup scanner. EPUB chapters are well-formed XHTML, + // so attribute values are always quoted; everything else (comments, + // processing instructions, closing tags) is skipped. Over-collection from + // odd markup is harmless — it only means loading a few extra CSS rules. + enum class State : uint8_t { + Text, // outside any markup + TagOpen, // just consumed '<' + SkipMarkup, // "' + TagName, // reading the element name + InTag, // inside a tag, between attributes + AttrName, // reading an attribute name + AfterAttrName, // after attribute name, before '=' or the next attribute + BeforeValue, // after '=', before the opening quote + AttrValue, // inside a quoted attribute value + }; + + State state = State::Text; + char token[MAX_TOKEN_LENGTH]; + size_t tokenLen = 0; + bool tokenOverflow = false; + bool inClassAttr = false; + char quoteChar = '"'; + + const auto tokenView = [&]() { return std::string_view(token, tokenLen); }; + const auto resetToken = [&]() { + tokenLen = 0; + tokenOverflow = false; + }; + const auto appendToken = [&](const char c) { + if (tokenLen < sizeof(token)) { + token[tokenLen++] = static_cast(std::tolower(static_cast(c))); + } else { + // Token longer than any realistic tag/class name; drop it rather than + // record a truncated name that could spuriously match a shorter rule. + tokenOverflow = true; + } + }; + + char buffer[READ_BUFFER_SIZE]; + while (file.available()) { + const int bytesRead = file.read(buffer, sizeof(buffer)); + if (bytesRead <= 0) { + break; + } + for (int i = 0; i < bytesRead; ++i) { + const char c = buffer[i]; + switch (state) { + case State::Text: + if (c == '<') { + state = State::TagOpen; + } + break; + case State::TagOpen: + if (c == '/' || c == '!' || c == '?') { + state = State::SkipMarkup; + } else if (std::isalpha(static_cast(c))) { + resetToken(); + appendToken(c); + state = State::TagName; + } else { + state = State::Text; + } + break; + case State::SkipMarkup: + if (c == '>') { + state = State::Text; + } + break; + case State::TagName: + if (isNameChar(c)) { + appendToken(c); + } else { + if (!tokenOverflow) { + addTag(tokenView()); + } + state = (c == '>') ? State::Text : State::InTag; + } + break; + case State::InTag: + if (c == '>') { + state = State::Text; + } else if (std::isalpha(static_cast(c))) { + resetToken(); + appendToken(c); + state = State::AttrName; + } + break; + case State::AttrName: + if (isNameChar(c)) { + appendToken(c); + } else { + inClassAttr = !tokenOverflow && tokenView() == "class"; + if (c == '=') { + state = State::BeforeValue; + } else if (c == '>') { + state = State::Text; + } else { + state = State::AfterAttrName; + } + } + break; + case State::AfterAttrName: + if (c == '=') { + state = State::BeforeValue; + } else if (c == '>') { + state = State::Text; + } else if (std::isalpha(static_cast(c))) { + resetToken(); + appendToken(c); + state = State::AttrName; + } + break; + case State::BeforeValue: + if (c == '"' || c == '\'') { + quoteChar = c; + resetToken(); + state = State::AttrValue; + } else if (c == '>') { + state = State::Text; + } else if (!isHtmlWhitespace(c)) { + // Unquoted value — invalid in XHTML; skip it as generic tag content + state = State::InTag; + } + break; + case State::AttrValue: + if (c == quoteChar) { + if (inClassAttr && !tokenOverflow) { + addClass(tokenView()); + } + state = State::InTag; + } else if (inClassAttr) { + if (isHtmlWhitespace(c)) { + if (!tokenOverflow) { + addClass(tokenView()); + } + resetToken(); + } else { + appendToken(c); + } + } + break; + } + } + } + file.close(); + + // ChapterHtmlSlimParser resolves "img" styles for image elements regardless + // of the source tag name (e.g. SVG ), so always keep img rules. + addTag("img"); + + LOG_DBG("CSU", "Scanned %s: %zu tags, %zu classes%s", path.c_str(), tags_.size(), classes_.size(), + overflowed_ ? " (overflow, fail-open)" : ""); + return true; +} diff --git a/lib/Epub/Epub/css/CssSelectorUsage.h b/lib/Epub/Epub/css/CssSelectorUsage.h new file mode 100644 index 00000000000..a82093a4323 --- /dev/null +++ b/lib/Epub/Epub/css/CssSelectorUsage.h @@ -0,0 +1,54 @@ +#pragma once + +#include +#include +#include + +/** + * Collects the set of tag names and class names actually used by a chapter's + * (X)HTML document, so CSS cache loading can skip rules that can never match. + * + * CssParser::resolveStyle() only ever looks up "tag", ".class" and + * "tag.class" keys, so a cached rule is worth loading into RAM iff its tag + * and class parts appear somewhere in the document. Generic publisher + * stylesheets (e.g. the EBPAJ template used by most Japanese EPUBs) register + * hundreds of rules of which a typical chapter references a handful; skipping + * the rest keeps tens of KB of heap free during section building (issue #105). + */ +class CssSelectorUsage { + public: + /** + * Stream the (X)HTML file at path and record used tag and class names. + * @return false if the file could not be opened; callers should then fall + * back to loading the full CSS cache. + */ + bool scanHtmlFile(const std::string& path); + + /** + * Whether a CSS cache key ("tag", ".class" or "tag.class") can match this + * document. Fails open (returns true) when the collection overflowed. + */ + [[nodiscard]] bool matches(const std::string& selectorKey) const; + + [[nodiscard]] size_t tagCount() const { return tags_.size(); } + [[nodiscard]] size_t classCount() const { return classes_.size(); } + + private: + // Caps keep pathological documents from growing the sets unbounded; + // on overflow we fail open and matches() accepts every rule. + static constexpr size_t MAX_TAGS = 64; + static constexpr size_t MAX_CLASSES = 256; + static constexpr size_t MAX_TOKEN_LENGTH = 128; + + void addTag(std::string_view name); + void addClass(std::string_view name); + [[nodiscard]] bool containsTag(std::string_view name) const { return contains(tags_, name); } + [[nodiscard]] bool containsClass(std::string_view name) const { return contains(classes_, name); } + static bool contains(const std::vector& list, std::string_view name); + + // Small sets (tens of entries) held as flat vectors with linear lookup; + // an unordered_set would cost one heap node per entry for no gain here. + std::vector tags_; + std::vector classes_; + bool overflowed_ = false; +}; diff --git a/lib/Epub/Epub/css/CssStyle.h b/lib/Epub/Epub/css/CssStyle.h index 319787e6f9b..eaf167014fe 100644 --- a/lib/Epub/Epub/css/CssStyle.h +++ b/lib/Epub/Epub/css/CssStyle.h @@ -75,6 +75,36 @@ enum class CssVerticalAlign : uint8_t { Baseline = 0, Super = 1, Sub = 2 }; // list-style-type — only None and Disc (bullet) are relevant for rendering enum class CssListStyleType : uint8_t { Disc = 0, None = 1 }; +// text-emphasis-style (bouten / 圏点). The fill keyword and the shape keyword are +// collapsed into one enum because only the resulting mark glyph matters here; +// "none" resets a mark inherited from an ancestor. Parsed from text-emphasis and +// text-emphasis-style, including the -epub-/-webkit- prefixed spellings that +// Japanese EPUB templates still ship. +enum class CssTextEmphasis : uint8_t { + None = 0, + FilledDot = 1, + OpenDot = 2, + FilledCircle = 3, + OpenCircle = 4, + FilledSesame = 5, + OpenSesame = 6, + FilledTriangle = 7, + OpenTriangle = 8, + FilledDoubleCircle = 9, + OpenDoubleCircle = 10, +}; + +// text-orientation and text-combine-upright, folded into one value: both decide how a run +// sits in a vertical column, and a span carries at most one of them in practice (the EBPAJ +// template's upright / sideways / tcy classes). Only the -webkit-/-epub- prefixed spellings +// appear in the surveyed commercial books, so those are read as well as the bare name. +enum class CssTextOrientation : uint8_t { + Mixed = 0, // the default: CJK upright, Latin turned (also text-combine-upright: none) + Upright = 1, // every character upright, one cell each + Sideways = 2, // every character turned with the column + Combine = 3, // text-combine-upright: all -- the whole run upright inside one cell +}; + // Bitmask for tracking which properties have been explicitly set struct CssPropertyFlags { uint16_t textAlign : 1; @@ -96,6 +126,10 @@ struct CssPropertyFlags { uint16_t direction : 1; uint16_t verticalAlign : 1; uint16_t listStyleType : 1; + uint16_t textEmphasis : 1; + uint16_t textOrientation : 1; + uint16_t imageMaxHeight : 1; + uint16_t imageMaxWidth : 1; CssPropertyFlags() : textAlign(0), @@ -116,12 +150,17 @@ struct CssPropertyFlags { display(0), direction(0), verticalAlign(0), - listStyleType(0) {} + listStyleType(0), + textEmphasis(0), + textOrientation(0), + imageMaxHeight(0), + imageMaxWidth(0) {} [[nodiscard]] bool anySet() const { return textAlign || fontStyle || fontWeight || textDecoration || textIndent || marginTop || marginBottom || marginLeft || marginRight || paddingTop || paddingBottom || paddingLeft || paddingRight || imageHeight || - imageWidth || display || direction || verticalAlign || listStyleType; + imageWidth || display || direction || verticalAlign || listStyleType || textEmphasis || textOrientation || + imageMaxHeight || imageMaxWidth; } void clearAll() { @@ -129,13 +168,14 @@ struct CssPropertyFlags { marginTop = marginBottom = marginLeft = marginRight = 0; paddingTop = paddingBottom = paddingLeft = paddingRight = 0; imageHeight = imageWidth = display = direction = verticalAlign = listStyleType = 0; + textEmphasis = textOrientation = imageMaxHeight = imageMaxWidth = 0; } }; -// Cache serializes defined flags as uint32_t with bit indices 0..18. +// Cache serializes defined flags as uint32_t with bit indices 0..22. static_assert(sizeof(CssPropertyFlags) <= sizeof(uint32_t), "CssPropertyFlags exceeds 32 bits; update cache read/write in CssParser.cpp"); -static_assert(sizeof(CssPropertyFlags) * 8 >= 19, +static_assert(sizeof(CssPropertyFlags) * 8 >= 23, "CssPropertyFlags has fewer bits than properties; update bitfield widths"); // Represents a collection of CSS style properties @@ -159,9 +199,15 @@ struct CssStyle { CssLength paddingRight; // Padding right CssLength imageHeight; // Height for img (e.g. 2em) – width derived from aspect ratio when only height set CssLength imageWidth; // Width for img when both or only width set - CssDisplay display = CssDisplay::Block; // display property (Block or None) - CssVerticalAlign verticalAlign = CssVerticalAlign::Baseline; // vertical-align (super/sub positioning) - CssListStyleType listStyleType = CssListStyleType::Disc; // list-style-type (Disc or None) + // Upper bounds for img. Unlike width/height these only ever shrink a picture: a source smaller + // than the bound keeps its own size. This is how commercial EPUBs size their illustrations. + CssLength imageMaxHeight; + CssLength imageMaxWidth; + CssDisplay display = CssDisplay::Block; // display property (Block or None) + CssVerticalAlign verticalAlign = CssVerticalAlign::Baseline; // vertical-align (super/sub positioning) + CssListStyleType listStyleType = CssListStyleType::Disc; // list-style-type (Disc or None) + CssTextEmphasis textEmphasis = CssTextEmphasis::None; // text-emphasis (bouten marks beside the text) + CssTextOrientation textOrientation = CssTextOrientation::Mixed; // text-orientation / text-combine-upright CssPropertyFlags defined; // Tracks which properties were explicitly set @@ -244,6 +290,22 @@ struct CssStyle { listStyleType = base.listStyleType; defined.listStyleType = 1; } + if (base.hasImageMaxHeight()) { + imageMaxHeight = base.imageMaxHeight; + defined.imageMaxHeight = 1; + } + if (base.hasImageMaxWidth()) { + imageMaxWidth = base.imageMaxWidth; + defined.imageMaxWidth = 1; + } + if (base.hasTextEmphasis()) { + textEmphasis = base.textEmphasis; + defined.textEmphasis = 1; + } + if (base.hasTextOrientation()) { + textOrientation = base.textOrientation; + defined.textOrientation = 1; + } } [[nodiscard]] bool hasTextAlign() const { return defined.textAlign; } @@ -265,6 +327,10 @@ struct CssStyle { [[nodiscard]] bool hasDirection() const { return defined.direction; } [[nodiscard]] bool hasVerticalAlign() const { return defined.verticalAlign; } [[nodiscard]] bool hasListStyleType() const { return defined.listStyleType; } + [[nodiscard]] bool hasTextEmphasis() const { return defined.textEmphasis; } + [[nodiscard]] bool hasImageMaxHeight() const { return defined.imageMaxHeight; } + [[nodiscard]] bool hasImageMaxWidth() const { return defined.imageMaxWidth; } + [[nodiscard]] bool hasTextOrientation() const { return defined.textOrientation; } void reset() { textAlign = CssTextAlign::Left; @@ -275,10 +341,12 @@ struct CssStyle { textIndent = CssLength{}; marginTop = marginBottom = marginLeft = marginRight = CssLength{}; paddingTop = paddingBottom = paddingLeft = paddingRight = CssLength{}; - imageHeight = imageWidth = CssLength{}; + imageHeight = imageWidth = imageMaxHeight = imageMaxWidth = CssLength{}; display = CssDisplay::Block; verticalAlign = CssVerticalAlign::Baseline; listStyleType = CssListStyleType::Disc; + textEmphasis = CssTextEmphasis::None; + textOrientation = CssTextOrientation::Mixed; defined.clearAll(); } }; diff --git a/lib/Epub/Epub/parsers/ChapterHtmlSlimParser.cpp b/lib/Epub/Epub/parsers/ChapterHtmlSlimParser.cpp index e379e57e977..c0aa66dd4de 100644 --- a/lib/Epub/Epub/parsers/ChapterHtmlSlimParser.cpp +++ b/lib/Epub/Epub/parsers/ChapterHtmlSlimParser.cpp @@ -1,5 +1,6 @@ #include "ChapterHtmlSlimParser.h" +#include #include #include #include @@ -16,12 +17,15 @@ #include #include "../../../../src/fontIds.h" +#include "../../../../src/util/InputDiag.h" +#include "../InlineImageToken.h" #include "Epub.h" #include "Epub/Page.h" #include "Epub/VisibleTextUtils.h" #include "Epub/converters/ImageDecoderFactory.h" #include "Epub/converters/ImageDimsProbe.h" #include "Epub/converters/ImageToFramebufferDecoder.h" +#include "Epub/converters/PngStreamDecoder.h" #include "Epub/htmlEntities.h" // Minimum file size (in bytes) to show indexing popup - smaller chapters don't benefit from it @@ -45,6 +49,9 @@ constexpr size_t TEXT_BLOCK_SOFT_FLUSH_WORDS_WITH_CSS = 320; // every text fragment (e.g. Kobo KePub spans). The cap prevents unbounded heap growth // on resource-constrained devices (~380KB heap). TOC anchors bypass this cap. constexpr size_t MAX_ANCHORS_PER_CHAPTER = 1024; +// Anchors held back until their block places something. Consecutive empty anchored elements are +// rare; past this many, the rest fall back to naming the page in progress. +constexpr size_t MAX_ANCHORS_AWAITING_PLACEMENT = 16; // Reuse serializable PageLine/PageHorizontalRule elements for a small grid. constexpr int16_t TABLE_CELL_HORIZONTAL_PADDING = 4; @@ -52,6 +59,19 @@ constexpr int16_t TABLE_ROW_SEPARATOR_GAP = 4; constexpr uint8_t TABLE_ROW_SEPARATOR_THICKNESS = 1; constexpr int16_t TABLE_MIN_CELL_WIDTH_LINE_HEIGHTS = 3; +// Column pitch in vertical writing, as a multiple of the full-width cell, before the reader's +// line-spacing setting scales it. JLREQ 2.3 sets the line pitch as a ratio of the character +// size (body text runs about 1.5 to 1.75 em); the pitch used to be the font's own line height +// plus a quarter, which is not a fixed ratio at all -- BIZUDGothic_12 came out at 1.25 em and +// NotoSansJP_12 at 1.8 em from the same setting, so changing font silently changed how much +// text a page held. With the four line-spacing steps (0.95/1.0/1.1/1.2) this spans 1.43 to +// 1.8 em, which covers the range JLREQ describes. +constexpr float VERTICAL_COLUMN_PITCH_EM = 1.5f; +// Cap on the marks a single horizontal word gets. Japanese bouten runs are short; a longer +// word is almost always Latin, where the per-character annotation buys nothing and the +// spacer string would outweigh the word. +constexpr size_t MAX_EMPHASIS_CODEPOINTS_PER_WORD = 24; + constexpr const char* HEADER_TAGS[] = {"h1", "h2", "h3", "h4", "h5", "h6"}; constexpr const char* BLOCK_TAGS[] = {"p", "li", "div", "br", "blockquote", "ul", "ol"}; constexpr const char* BOLD_TAGS[] = {"b", "strong"}; @@ -61,6 +81,119 @@ constexpr const char* LINETHROUGH_TAGS[] = {"del", "s", "strike"}; constexpr const char* IMAGE_TAGS[] = {"img", "image"}; bool isWhitespace(const char c) { return c == ' ' || c == '\r' || c == '\n' || c == '\t'; } +// CJK Compatibility Ideographs (U+F900-FAFF) -> the unified ideograph each one +// decomposes to, 0 where there is none or it lives outside the BMP (no reading face +// carries those). Publishers' typesetting systems reach for a compatibility codepoint +// to pin a particular glyph shape, but a face that never drew that shape has nothing +// there: BIZ UD has 蓮 U+F999 and neither 溺 U+F9EC nor 煉 U+F993, so a book using them +// drew the replacement box. The character is the same either way, so falling back to +// the unified form shows the word instead of a hole. +// +// 512 uint16 = 1 KB of flash, no RAM: the table is only read on the rare codepoint +// that lands in the block. +constexpr uint16_t CJK_COMPAT_UNIFIED[512] = { + 0x8C48, 0x66F4, 0x8ECA, 0x8CC8, 0x6ED1, 0x4E32, 0x53E5, 0x9F9C, // U+F900 + 0x9F9C, 0x5951, 0x91D1, 0x5587, 0x5948, 0x61F6, 0x7669, 0x7F85, // U+F908 + 0x863F, 0x87BA, 0x88F8, 0x908F, 0x6A02, 0x6D1B, 0x70D9, 0x73DE, // U+F910 + 0x843D, 0x916A, 0x99F1, 0x4E82, 0x5375, 0x6B04, 0x721B, 0x862D, // U+F918 + 0x9E1E, 0x5D50, 0x6FEB, 0x85CD, 0x8964, 0x62C9, 0x81D8, 0x881F, // U+F920 + 0x5ECA, 0x6717, 0x6D6A, 0x72FC, 0x90CE, 0x4F86, 0x51B7, 0x52DE, // U+F928 + 0x64C4, 0x6AD3, 0x7210, 0x76E7, 0x8001, 0x8606, 0x865C, 0x8DEF, // U+F930 + 0x9732, 0x9B6F, 0x9DFA, 0x788C, 0x797F, 0x7DA0, 0x83C9, 0x9304, // U+F938 + 0x9E7F, 0x8AD6, 0x58DF, 0x5F04, 0x7C60, 0x807E, 0x7262, 0x78CA, // U+F940 + 0x8CC2, 0x96F7, 0x58D8, 0x5C62, 0x6A13, 0x6DDA, 0x6F0F, 0x7D2F, // U+F948 + 0x7E37, 0x964B, 0x52D2, 0x808B, 0x51DC, 0x51CC, 0x7A1C, 0x7DBE, // U+F950 + 0x83F1, 0x9675, 0x8B80, 0x62CF, 0x6A02, 0x8AFE, 0x4E39, 0x5BE7, // U+F958 + 0x6012, 0x7387, 0x7570, 0x5317, 0x78FB, 0x4FBF, 0x5FA9, 0x4E0D, // U+F960 + 0x6CCC, 0x6578, 0x7D22, 0x53C3, 0x585E, 0x7701, 0x8449, 0x8AAA, // U+F968 + 0x6BBA, 0x8FB0, 0x6C88, 0x62FE, 0x82E5, 0x63A0, 0x7565, 0x4EAE, // U+F970 + 0x5169, 0x51C9, 0x6881, 0x7CE7, 0x826F, 0x8AD2, 0x91CF, 0x52F5, // U+F978 + 0x5442, 0x5973, 0x5EEC, 0x65C5, 0x6FFE, 0x792A, 0x95AD, 0x9A6A, // U+F980 + 0x9E97, 0x9ECE, 0x529B, 0x66C6, 0x6B77, 0x8F62, 0x5E74, 0x6190, // U+F988 + 0x6200, 0x649A, 0x6F23, 0x7149, 0x7489, 0x79CA, 0x7DF4, 0x806F, // U+F990 + 0x8F26, 0x84EE, 0x9023, 0x934A, 0x5217, 0x52A3, 0x54BD, 0x70C8, // U+F998 + 0x88C2, 0x8AAA, 0x5EC9, 0x5FF5, 0x637B, 0x6BAE, 0x7C3E, 0x7375, // U+F9A0 + 0x4EE4, 0x56F9, 0x5BE7, 0x5DBA, 0x601C, 0x73B2, 0x7469, 0x7F9A, // U+F9A8 + 0x8046, 0x9234, 0x96F6, 0x9748, 0x9818, 0x4F8B, 0x79AE, 0x91B4, // U+F9B0 + 0x96B8, 0x60E1, 0x4E86, 0x50DA, 0x5BEE, 0x5C3F, 0x6599, 0x6A02, // U+F9B8 + 0x71CE, 0x7642, 0x84FC, 0x907C, 0x9F8D, 0x6688, 0x962E, 0x5289, // U+F9C0 + 0x677B, 0x67F3, 0x6D41, 0x6E9C, 0x7409, 0x7559, 0x786B, 0x7D10, // U+F9C8 + 0x985E, 0x516D, 0x622E, 0x9678, 0x502B, 0x5D19, 0x6DEA, 0x8F2A, // U+F9D0 + 0x5F8B, 0x6144, 0x6817, 0x7387, 0x9686, 0x5229, 0x540F, 0x5C65, // U+F9D8 + 0x6613, 0x674E, 0x68A8, 0x6CE5, 0x7406, 0x75E2, 0x7F79, 0x88CF, // U+F9E0 + 0x88E1, 0x91CC, 0x96E2, 0x533F, 0x6EBA, 0x541D, 0x71D0, 0x7498, // U+F9E8 + 0x85FA, 0x96A3, 0x9C57, 0x9E9F, 0x6797, 0x6DCB, 0x81E8, 0x7ACB, // U+F9F0 + 0x7B20, 0x7C92, 0x72C0, 0x7099, 0x8B58, 0x4EC0, 0x8336, 0x523A, // U+F9F8 + 0x5207, 0x5EA6, 0x62D3, 0x7CD6, 0x5B85, 0x6D1E, 0x66B4, 0x8F3B, // U+FA00 + 0x884C, 0x964D, 0x898B, 0x5ED3, 0x5140, 0x55C0, 0x0000, 0x0000, // U+FA08 + 0x585A, 0x0000, 0x6674, 0x0000, 0x0000, 0x51DE, 0x732A, 0x76CA, // U+FA10 + 0x793C, 0x795E, 0x7965, 0x798F, 0x9756, 0x7CBE, 0x7FBD, 0x0000, // U+FA18 + 0x8612, 0x0000, 0x8AF8, 0x0000, 0x0000, 0x9038, 0x90FD, 0x0000, // U+FA20 + 0x0000, 0x0000, 0x98EF, 0x98FC, 0x9928, 0x9DB4, 0x90DE, 0x96B7, // U+FA28 + 0x4FAE, 0x50E7, 0x514D, 0x52C9, 0x52E4, 0x5351, 0x559D, 0x5606, // U+FA30 + 0x5668, 0x5840, 0x58A8, 0x5C64, 0x5C6E, 0x6094, 0x6168, 0x618E, // U+FA38 + 0x61F2, 0x654F, 0x65E2, 0x6691, 0x6885, 0x6D77, 0x6E1A, 0x6F22, // U+FA40 + 0x716E, 0x722B, 0x7422, 0x7891, 0x793E, 0x7949, 0x7948, 0x7950, // U+FA48 + 0x7956, 0x795D, 0x798D, 0x798E, 0x7A40, 0x7A81, 0x7BC0, 0x7DF4, // U+FA50 + 0x7E09, 0x7E41, 0x7F72, 0x8005, 0x81ED, 0x8279, 0x8279, 0x8457, // U+FA58 + 0x8910, 0x8996, 0x8B01, 0x8B39, 0x8CD3, 0x8D08, 0x8FB6, 0x9038, // U+FA60 + 0x96E3, 0x97FF, 0x983B, 0x6075, 0x0000, 0x8218, 0x0000, 0x0000, // U+FA68 + 0x4E26, 0x51B5, 0x5168, 0x4F80, 0x5145, 0x5180, 0x52C7, 0x52FA, // U+FA70 + 0x559D, 0x5555, 0x5599, 0x55E2, 0x585A, 0x58B3, 0x5944, 0x5954, // U+FA78 + 0x5A62, 0x5B28, 0x5ED2, 0x5ED9, 0x5F69, 0x5FAD, 0x60D8, 0x614E, // U+FA80 + 0x6108, 0x618E, 0x6160, 0x61F2, 0x6234, 0x63C4, 0x641C, 0x6452, // U+FA88 + 0x6556, 0x6674, 0x6717, 0x671B, 0x6756, 0x6B79, 0x6BBA, 0x6D41, // U+FA90 + 0x6EDB, 0x6ECB, 0x6F22, 0x701E, 0x716E, 0x77A7, 0x7235, 0x72AF, // U+FA98 + 0x732A, 0x7471, 0x7506, 0x753B, 0x761D, 0x761F, 0x76CA, 0x76DB, // U+FAA0 + 0x76F4, 0x774A, 0x7740, 0x78CC, 0x7AB1, 0x7BC0, 0x7C7B, 0x7D5B, // U+FAA8 + 0x7DF4, 0x7F3E, 0x8005, 0x8352, 0x83EF, 0x8779, 0x8941, 0x8986, // U+FAB0 + 0x8996, 0x8ABF, 0x8AF8, 0x8ACB, 0x8B01, 0x8AFE, 0x8AED, 0x8B39, // U+FAB8 + 0x8B8A, 0x8D08, 0x8F38, 0x9072, 0x9199, 0x9276, 0x967C, 0x96E3, // U+FAC0 + 0x9756, 0x97DB, 0x97FF, 0x980B, 0x983B, 0x9B12, 0x9F9C, 0x0000, // U+FAC8 + 0x0000, 0x0000, 0x3B9D, 0x4018, 0x4039, 0x0000, 0x0000, 0x0000, // U+FAD0 + 0x9F43, 0x9F8E, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, // U+FAD8 + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, // U+FAE0 + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, // U+FAE8 + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, // U+FAF0 + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, // U+FAF8 +}; + +// Unified form for a compatibility ideograph, 0 when there is no BMP one. +uint32_t unifiedIdeographFor(const uint32_t cp) { + if (cp < 0xF900 || cp > 0xFAFF) return 0; + return CJK_COMPAT_UNIFIED[cp - 0xF900]; +} + +// UTF-8 mark glyph for a text-emphasis style, nullptr for none. The sesame forms are the +// Vertical Forms codepoints; every mark here is inside the coverage the reading faces +// already carry for kutouten and enclosed CJK. +const char* emphasisMarkUtf8(const CssTextEmphasis e) { + switch (e) { + case CssTextEmphasis::FilledDot: + return "\xE2\x80\xA2"; // • + case CssTextEmphasis::OpenDot: + return "\xE2\x97\xA6"; // ◦ + case CssTextEmphasis::FilledCircle: + return "\xE2\x97\x8F"; // ● + case CssTextEmphasis::OpenCircle: + return "\xE2\x97\x8B"; // ○ + case CssTextEmphasis::FilledSesame: + return "\xEF\xB9\x85"; // ﹅ + case CssTextEmphasis::OpenSesame: + return "\xEF\xB9\x86"; // ﹆ + case CssTextEmphasis::FilledTriangle: + return "\xE2\x96\xB2"; // ▲ + case CssTextEmphasis::OpenTriangle: + return "\xE2\x96\xB3"; // △ + case CssTextEmphasis::FilledDoubleCircle: + return "\xE2\x97\x89"; // ◉ + case CssTextEmphasis::OpenDoubleCircle: + return "\xE2\x97\x8E"; // ◎ + default: + return nullptr; + } +} + std::string trimAndNormalize(const std::string& str) { if (str.empty()) return ""; size_t start = 0; @@ -165,6 +298,20 @@ EpdFontFamily::Style ChapterHtmlSlimParser::fontStyleForTextDecoration(const Css return style; } +void ChapterHtmlSlimParser::applyTextEmphasisToEntry(StyleStackEntry& entry, const CssStyle& css) { + if (css.hasTextEmphasis()) { + entry.hasEmphasis = true; + entry.emphasis = css.textEmphasis; + } +} + +void ChapterHtmlSlimParser::applyTextOrientationToEntry(StyleStackEntry& entry, const CssStyle& css) { + if (css.hasTextOrientation()) { + entry.hasOrientation = true; + entry.orientation = css.textOrientation; + } +} + void ChapterHtmlSlimParser::applyTextDecorationToEntry(StyleStackEntry& entry, const CssStyle& css) { if (css.hasTextDecoration()) { entry.hasTextDecoration = true; @@ -225,6 +372,8 @@ void ChapterHtmlSlimParser::pushDecorationStyleEntry(const CssTextDecoration def entry.italic = cssStyle.fontStyle == CssFontStyle::Italic; } applyDirectionToEntry(entry, cssStyle); + applyTextEmphasisToEntry(entry, cssStyle); + applyTextOrientationToEntry(entry, cssStyle); inlineStyleStack.push_back(entry); updateEffectiveInlineStyle(); } @@ -249,6 +398,9 @@ void ChapterHtmlSlimParser::updateEffectiveInlineStyle() { effectiveTextAlign = currentCssStyle.textAlign; effectiveSup = false; effectiveSub = false; + effectiveEmphasis = currentCssStyle.hasTextEmphasis() ? currentCssStyle.textEmphasis : CssTextEmphasis::None; + effectiveOrientation = + currentCssStyle.hasTextOrientation() ? currentCssStyle.textOrientation : CssTextOrientation::Mixed; // Apply inline style stack in order for (const auto& entry : inlineStyleStack) { @@ -283,6 +435,13 @@ void ChapterHtmlSlimParser::updateEffectiveInlineStyle() { effectiveSub = entry.sub; if (entry.sub) effectiveSup = false; } + // Unlike line decorations, a descendant's "none" cancels an ancestor's mark. + if (entry.hasEmphasis) { + effectiveEmphasis = entry.emphasis; + } + if (entry.hasOrientation) { + effectiveOrientation = entry.orientation; + } } // Keep flow direction in the active empty text block. Inline direction remains @@ -309,11 +468,26 @@ void ChapterHtmlSlimParser::flushPendingAnchor() { } } - // Record deferred anchor after previous block is flushed (and any TOC page break) - anchorData.push_back({std::move(pendingAnchorId), static_cast(completedPageCount)}); + // Hold the anchor until a line of its block has actually been placed. completedPageCount here + // is the page in progress, and the block's first line may not fit on it -- addLineToPage then + // emits that page and puts the line on the next one. A TOC anchor came out right before this + // change only because it forces the break above first; nothing else did. + if (anchorsAwaitingPlacement.size() < MAX_ANCHORS_AWAITING_PLACEMENT) { + anchorsAwaitingPlacement.push_back(std::move(pendingAnchorId)); + } else { + anchorData.push_back({std::move(pendingAnchorId), static_cast(completedPageCount)}); + } pendingAnchorId.clear(); } +void ChapterHtmlSlimParser::commitAnchorsAwaitingPlacement() { + if (anchorsAwaitingPlacement.empty()) return; + for (auto& anchor : anchorsAwaitingPlacement) { + anchorData.push_back({std::move(anchor), static_cast(completedPageCount)}); + } + anchorsAwaitingPlacement.clear(); +} + void ChapterHtmlSlimParser::setCurrentPageVisibleOffset(const uint32_t offset) { if (currentPageVisibleOffsetSet) return; // The first page always begins at the start of the body, even when the XHTML @@ -353,23 +527,64 @@ void ChapterHtmlSlimParser::flushPartWordBuffer() { // flush the buffer partWordBuffer[partWordBufferIndex] = '\0'; + substituteMissingCompatibilityIdeographs(); const size_t wordBytes = static_cast(partWordBufferIndex); - if (insideTableCell && !tableRowStacked && tableCellTextBytes + wordBytes > MAX_GRID_TABLE_CELL_BYTES) { + // Upstream's grid tables lay cells out in horizontal-model coordinates; a vertical page has + // no place for that, so vertical books keep the stacked (flattened) table form. + if (insideTableCell && !tableRowStacked && + (isVertical || tableCellTextBytes + wordBytes > MAX_GRID_TABLE_CELL_BYTES)) { fallbackTableRowToStacked(); } - - uint8_t linkId = 0; - if (insideFootnoteLink) { - if (!currentTextBlock->linkTargetMatches(currentFootnoteLinkId, currentFootnote.href)) { - currentFootnoteLinkId = currentTextBlock->addLinkTarget(currentFootnote.href); + if (isVertical) { + // Spend the pending whitespace run as a separator token. + // + // characterData drops HTML whitespace as a bare word boundary, which is right for horizontal + // layout: that path re-inserts the gap at layout time via getSpaceAdvance() between words that + // do not continue. Vertical layout has no such step — layoutVerticalColumns stacks each token by + // its own advance and nothing else — so with no token to carry it the space simply vanished and + // Latin phrases came out run together ("character length calculator" as one string). + // + // Emitting a token here rather than adding a rule to the vertical layout keeps the two writing + // modes agreeing on where a space belongs, instead of giving vertical its own notion of it. + // + // Guarded on a non-empty buffer so an empty flush (an inline tag boundary, say) neither spends + // the run nor emits a trailing separator, and on a non-empty block so a run that opens a + // paragraph is discarded the way CSS collapsing discards it. + if (partWordBufferIndex > 0) { + if (pendingVerticalWhitespace && currentTextBlock && !currentTextBlock->isEmpty()) { + currentTextBlock->addVerticalToken(" ", fontStyle, VerticalTextUtils::VerticalBehavior::Sideways); + } + pendingVerticalWhitespace = false; + } + // Vertical layout tokenizes per codepoint: each glyph is its own cell, classified + // (upright CJK / sideways Latin / tate-chu-yoko digits) so layoutVerticalColumns can + // stack and orient it. Latin runs and 1-2 digit numbers are grouped into one token. + // Per-token visible-offset tracking is not implemented for this path (see + // addColumnToPage for the page-granularity fallback used instead). + const size_t firstToken = currentTextBlock->size(); + flushPartWordBufferVertical(fontStyle); + if (insideFootnoteLink) { + if (!currentTextBlock->linkTargetMatches(currentFootnoteLinkId, currentFootnote.href)) { + currentFootnoteLinkId = currentTextBlock->addLinkTarget(currentFootnote.href); + } + currentTextBlock->setLinkIdFrom(firstToken, currentFootnoteLinkId); } - linkId = currentFootnoteLinkId; - } - currentTextBlock->addWord(partWordBuffer, fontStyle, false, nextWordContinues, partWordVisibleOffset, linkId); - if (insideTableCell && !tableRowStacked) { - tableCellTextBytes += wordBytes; - if (currentTextBlock->size() > MAX_GRID_TABLE_CELL_WORDS) { - fallbackTableRowToStacked(); + } else { + uint8_t linkId = 0; + if (insideFootnoteLink) { + if (!currentTextBlock->linkTargetMatches(currentFootnoteLinkId, currentFootnote.href)) { + currentFootnoteLinkId = currentTextBlock->addLinkTarget(currentFootnote.href); + } + linkId = currentFootnoteLinkId; + } + const size_t wordIndex = currentTextBlock->size(); + currentTextBlock->addWord(partWordBuffer, fontStyle, false, nextWordContinues, partWordVisibleOffset, linkId); + applyHorizontalEmphasis(wordIndex); + if (insideTableCell && !tableRowStacked) { + tableCellTextBytes += wordBytes; + if (currentTextBlock->size() > MAX_GRID_TABLE_CELL_WORDS) { + fallbackTableRowToStacked(); + } } } partWordBufferIndex = 0; @@ -377,9 +592,384 @@ void ChapterHtmlSlimParser::flushPartWordBuffer() { listItemBulletOnly = false; } +// Attach bouten to a word the horizontal path just added, as a synthetic ruby annotation. +// +// The spacer is what makes the marks line up. Ruby draws in SUP style, which the renderer +// puts out at 50% scale, and one annotation is centred over the whole word, so bare marks +// advance only half a cell each and bunch into the middle of the run they mark. U+3000 is +// full-width, so at SUP it is the other half: mark + space is exactly one character cell, +// making the annotation as wide as the word and landing one mark per character. +// +// U+3000 shares its block with the brackets and kutouten on every page, so it is present +// in any face that can render the text being marked. +void ChapterHtmlSlimParser::applyHorizontalEmphasis(const size_t wordIndex) { + const char* mark = resolveEmphasisMark(effectiveEmphasis); + if (!mark) return; + + size_t codepoints = 0; + for (int i = 0; i < partWordBufferIndex; i++) { + if ((static_cast(partWordBuffer[i]) & 0xC0) != 0x80) codepoints++; + } + // Long runs would build a string bigger than the word itself for no legibility gain. + if (codepoints == 0 || codepoints > MAX_EMPHASIS_CODEPOINTS_PER_WORD) return; + + static constexpr char IDEOGRAPHIC_SPACE[] = "\xE3\x80\x80"; // U+3000 + constexpr size_t SPACE_LEN = sizeof(IDEOGRAPHIC_SPACE) - 1; + const size_t markLen = strlen(mark); + + // addWord splits CJK text into one token per character, so the run this flush produced is + // words[wordIndex .. size()). Each token gets as many marks as it has codepoints: one per kanji + // or kana, several for a Latin or digit token that stayed whole. Hanging the whole run's marks + // on the first token (the previous form) made that one character carry a ruby wider than the + // line, and the horizontal ruby layout then spread its neighbours across the page to fit it + // (seen 2026-09-06 on test_horizontal_ja.epub: 「は 一 文」 with thirteen marks above). + const size_t tokenEnd = currentTextBlock->size(); + for (size_t t = wordIndex; t < tokenEnd; t++) { + const std::string_view token = currentTextBlock->tokenAt(t); + size_t tokenCodepoints = 0; + for (const char c : token) { + if ((static_cast(c) & 0xC0) != 0x80) tokenCodepoints++; + } + if (tokenCodepoints == 0) continue; + std::string marks; + marks.reserve((markLen + SPACE_LEN) * tokenCodepoints); + for (size_t i = 0; i < tokenCodepoints; i++) { + marks.append(mark, markLen); + if (i + 1 < tokenCodepoints) marks.append(IDEOGRAPHIC_SPACE, SPACE_LEN); + } + currentTextBlock->setRubyForWordAt(t, marks); + } +} + +// Tokenize the pending buffer into vertical cells. Emits one token per CJK/upright +// codepoint; consecutive ASCII letters coalesce into a Sideways run and 1-2 digit +// numbers into a TateChuYoko token (3+ digits fall back to Sideways). A number keeps +// any separator standing between two of its digits, so 3.14 and 12:34 are one cell each +// and cannot be broken across a column. +void ChapterHtmlSlimParser::flushPartWordBufferVertical(const EpdFontFamily::Style fontStyle) { + // Vertical layout emits roughly one token per codepoint, so a full buffer becomes a burst of + // pushes. Reserve up front (worst case one token per byte) so the parallel arrays grow once. + currentTextBlock->ensureTokenCapacity(static_cast(partWordBufferIndex)); + + // Bouten ride the ruby path, so they need the token range this flush produces. + const char* emphasisMark = resolveEmphasisMark(effectiveEmphasis); + const size_t emphasisFirstToken = emphasisMark ? currentTextBlock->size() : 0; + // One mark per token. Vertical layout already gives every upright codepoint its own + // cell, so a per-token annotation lands one mark beside one character with no spacing + // trick needed; a coalesced Latin or tate-chu-yoko run takes a single mark over the + // cell it occupies. The mark is 3 UTF-8 bytes, inside the small-string buffer, so this + // costs no allocation per token. + const auto applyEmphasis = [&]() { + if (!emphasisMark) return; + const size_t tokenEnd = currentTextBlock->size(); + for (size_t i = emphasisFirstToken; i < tokenEnd; i++) { + currentTextBlock->setRubyForWordAt(i, emphasisMark); + } + }; + + // CSS text-orientation / text-combine-upright on this run (the EBPAJ upright / sideways / + // tcy classes). The classifier below decides per character; these decide for the span. + if (effectiveOrientation != CssTextOrientation::Mixed && flushForcedOrientation(fontStyle, effectiveOrientation)) { + applyEmphasis(); + return; + } + + const auto* p = reinterpret_cast(partWordBuffer); + const auto* end = p + partWordBufferIndex; + while (p < end) { + const unsigned char* cpStart = p; + const uint32_t cp = utf8NextCodepoint(&p); + if (cp == 0) break; + + if (VerticalTextUtils::isUprightInVertical(cp) || VerticalTextUtils::getVerticalPunctuationOffset(cp) != nullptr) { + // Upright CJK/kana/punctuation: one cell each. 、。， become their Vertical + // Forms counterparts when the reading face carries those glyphs, so the + // draw path gets a real vertical glyph instead of the shifted horizontal one. + const uint32_t formCp = VerticalTextUtils::verticalPresentationForm(cp); + if (formCp != 0 && fontHasVerticalForm(formCp)) { + std::string token; + utf8AppendCodepoint(formCp, token); + currentTextBlock->addVerticalToken(std::move(token), fontStyle, VerticalTextUtils::VerticalBehavior::Upright); + continue; + } + currentTextBlock->addVerticalToken(std::string(reinterpret_cast(cpStart), p - cpStart), fontStyle, + VerticalTextUtils::VerticalBehavior::Upright); + continue; + } + + // Coalesce a run of ASCII digits or letters into a single sideways/tate-chu-yoko token. + const bool isDigit = (cp >= '0' && cp <= '9'); + const unsigned char* runStart = cpStart; + const unsigned char* runEnd = p; + int runChars = 1; + while (runEnd < end) { + const unsigned char* peek = runEnd; + const uint32_t next = utf8NextCodepoint(&peek); + const bool nextDigit = (next >= '0' && next <= '9'); + const bool nextAscii = (next >= '!' && next <= '~'); + if (isDigit) { + if (nextDigit) { + runEnd = peek; + runChars++; + continue; + } + // A separator standing between two digits is part of the number, not a break in + // it: 3.14, 12:34, 1,000, 3/4. Left out of the run, each of those became three + // cells with a column break free to fall between them, and 3.14 duly came back + // from the device split across two columns. The digit on the far side is what + // makes it safe -- the full stop ending a sentence has no digit after it, so it + // is not swallowed, and neither is the colon introducing a quotation. + if (next == '.' || next == ',' || next == ':' || next == '/') { + const unsigned char* after = peek; + if (after < end) { + const uint32_t following = utf8NextCodepoint(&after); + if (following >= '0' && following <= '9') { + runEnd = after; + runChars += 2; + continue; + } + } + } + break; + } + if (nextAscii && !nextDigit) { + runEnd = peek; + runChars++; + continue; + } + break; + } + p = runEnd; + std::string token(reinterpret_cast(runStart), runEnd - runStart); + // 1-2 digits and an exclamation/question pair share one upright cell; every other + // ASCII run turns with the column. + const bool tateChuYoko = + (isDigit && runChars <= 2) || VerticalTextUtils::isTateChuYokoPunctuationPair(token.c_str()); + const auto behavior = + tateChuYoko ? VerticalTextUtils::VerticalBehavior::TateChuYoko : VerticalTextUtils::VerticalBehavior::Sideways; + + if (behavior == VerticalTextUtils::VerticalBehavior::Sideways) { + addSidewaysToken(std::move(token), fontStyle); + continue; + } + + currentTextBlock->addVerticalToken(std::move(token), fontStyle, behavior); + } + + applyEmphasis(); +} + +// A sideways run occupies its own WIDTH as the column's vertical extent, and the tokenizer +// coalesces every unbroken ASCII stretch into one token -- so a path or an identifier with +// no space in it ("package/metadata/manifest/spine/item/itemref") becomes a single token +// taller than the column. Column breaking works between tokens, so it cannot split that +// one, and the run is drawn straight through the status bar and off the panel. Break the +// run into column-sized pieces here, where the character boundaries are still known; +// layout then treats them as ordinary adjacent tokens. Only over-long runs are touched, so +// ordinary words are unchanged. The pieces are cut at byte boundaries, which for a Latin +// run is the same thing as character boundaries; a forced-sideways run of CJK text (see +// flushForcedOrientation) is cut at the character instead. +void ChapterHtmlSlimParser::addSidewaysToken(std::string token, const EpdFontFamily::Style fontStyle) { + constexpr auto kSideways = VerticalTextUtils::VerticalBehavior::Sideways; + if (viewportHeight <= 0 || renderer.getTextAdvanceX(fontId, token.c_str(), fontStyle) <= viewportHeight) { + currentTextBlock->addVerticalToken(std::move(token), fontStyle, kSideways); + return; + } + const auto nextCharEnd = [&token](const size_t from) { + size_t next = from + 1; + while (next < token.size() && (static_cast(token[next]) & 0xC0) == 0x80) next++; + return next; + }; + size_t pieceStart = 0; + while (pieceStart < token.size()) { + // Grow a piece one character at a time until the next one would overflow. + size_t pieceEnd = pieceStart; + size_t lastFitting = pieceStart; + while (pieceEnd < token.size()) { + const size_t next = nextCharEnd(pieceEnd); + if (renderer.getTextAdvanceX(fontId, token.substr(pieceStart, next - pieceStart).c_str(), fontStyle) > + viewportHeight) { + break; + } + lastFitting = next; + pieceEnd = next; + } + // A single character wider than the column would loop forever; emit it anyway. + if (lastFitting == pieceStart) lastFitting = nextCharEnd(pieceStart); + currentTextBlock->addVerticalToken(token.substr(pieceStart, lastFitting - pieceStart), fontStyle, kSideways); + pieceStart = lastFitting; + } +} + +// The three non-default orientations, all of which the draw side reads back from the +// VERTICAL_FLIP style bit (TextBlock::renderVertical derives a token's setting from its +// text and the bit reverses that verdict), so nothing new travels through the page cache: +// +// upright every character in its own em cell, set upright. Characters that are upright +// anyway go the usual way (vertical forms for 、。， included); a Latin letter, +// a digit or an arrow gets a full cell and the flag, and is centred in it. +// sideways every character turned with the column. Characters that turn anyway are +// coalesced into one run like ordinary Latin; an upright one (a fullwidth ！, +// kana in a "横倒し" span) becomes a one-character sideways token with the flag. +// combine the whole run in one cell (tate-chu-yoko). Only for runs the tokenizer would +// have turned: 1-2 digits and !? are set that way already, and anything +// containing an upright character is prose the class was put on by mistake. +// Wider than one and a half cells cannot be squeezed in and turns as usual. +bool ChapterHtmlSlimParser::flushForcedOrientation(const EpdFontFamily::Style fontStyle, + const CssTextOrientation orientation) { + using VerticalTextUtils::VerticalBehavior; + const auto flipped = static_cast(fontStyle | EpdFontFamily::VERTICAL_FLIP); + const auto* p = reinterpret_cast(partWordBuffer); + const auto* end = p + partWordBufferIndex; + + const auto isUprightByDefault = [](const uint32_t cp) { + return VerticalTextUtils::isUprightInVertical(cp) || VerticalTextUtils::getVerticalPunctuationOffset(cp) != nullptr; + }; + // What the draw-time classifier would set upright inside one cell on its own. + const auto isNaturalTateChuYoko = [](const std::string& run) { + if (VerticalTextUtils::isTateChuYokoPunctuationPair(run.c_str())) return true; + if (run.empty() || run.size() > 2) return false; + for (const char c : run) { + if (c < '0' || c > '9') return false; + } + return true; + }; + + if (orientation == CssTextOrientation::Combine) { + for (const unsigned char* q = p; q < end;) { + const uint32_t cp = utf8NextCodepoint(&q); + if (cp == 0) break; + if (isUprightByDefault(cp)) return false; + } + std::string run(partWordBuffer, static_cast(partWordBufferIndex)); + if (isNaturalTateChuYoko(run)) return false; + const int cell = verticalCellWidthMemo > 0 ? verticalCellWidthMemo : renderer.getCjkCellWidth(fontId); + if (renderer.getTextAdvanceX(fontId, run.c_str(), fontStyle) > cell * 3 / 2) return false; + currentTextBlock->addVerticalToken(std::move(run), flipped, VerticalBehavior::TateChuYoko); + return true; + } + + const bool upright = orientation == CssTextOrientation::Upright; + std::string run; // sideways: consecutive characters that turn anyway, kept as one token + const auto flushRun = [&]() { + if (run.empty()) return; + // A lone digit pair or !? would be set upright by the draw-time classifier; flag it so + // it turns with the rest of the span. + const EpdFontFamily::Style style = isNaturalTateChuYoko(run) ? flipped : fontStyle; + addSidewaysToken(std::move(run), style); + run.clear(); + }; + while (p < end) { + const unsigned char* cpStart = p; + const uint32_t cp = utf8NextCodepoint(&p); + if (cp == 0) break; + const VerticalTextUtils::PunctuationOffset* punct = VerticalTextUtils::getVerticalPunctuationOffset(cp); + const bool uprightByDefault = isUprightByDefault(cp); + // Brackets and long marks are upright tokens that the draw side turns anyway. + const bool turnsByDefault = punct != nullptr ? punct->rotate : !uprightByDefault; + std::string token(reinterpret_cast(cpStart), p - cpStart); + if (upright) { + if (uprightByDefault) { + const uint32_t formCp = VerticalTextUtils::verticalPresentationForm(cp); + if (formCp != 0 && fontHasVerticalForm(formCp)) { + token.clear(); + utf8AppendCodepoint(formCp, token); + } + currentTextBlock->addVerticalToken(std::move(token), fontStyle, VerticalBehavior::Upright); + } else if (cp == ' ') { + currentTextBlock->addVerticalToken(std::move(token), fontStyle, VerticalBehavior::Sideways); + } else { + currentTextBlock->addVerticalToken(std::move(token), flipped, VerticalBehavior::TateChuYoko); + } + continue; + } + if (turnsByDefault) { + run += token; + continue; + } + flushRun(); + currentTextBlock->addVerticalToken(std::move(token), flipped, VerticalBehavior::Sideways); + } + flushRun(); + return true; +} + +// Coverage-interval probe of the reading face, cached per form because the answer is +// needed once per 、。， in the chapter. Faces predating the Vertical Forms block +// answer false and the shifted-horizontal-glyph fallback stays in effect. +// Mark glyph for a text-emphasis style, substituting a dot when the reading face has no +// sesame. U+FE45/FE46 are the default shape for CSS text-emphasis and the usual choice in +// Japanese books, but they sit in Vertical Forms and most faces stop short of it -- the +// generated OST reading faces included. Drawing the replacement box for every marked +// character is worse than the dot every reader recognises as bouten, so the face is probed +// once and the shape downgraded if it comes back empty. +const char* ChapterHtmlSlimParser::resolveEmphasisMark(const CssTextEmphasis e) { + const bool filledSesame = e == CssTextEmphasis::FilledSesame; + if (!filledSesame && e != CssTextEmphasis::OpenSesame) { + return emphasisMarkUtf8(e); + } + + const uint8_t bit = filledSesame ? 1u : 2u; + if ((sesameProbe & bit) == 0) { + sesameProbe |= bit; + const uint32_t cp = filledSesame ? 0xFE45 : 0xFE46; + const auto& fonts = renderer.getFontMap(); + const auto it = fonts.find(fontId); + if (it != fonts.end() && it->second.hasCodepoint(cp)) { + sesameProbe |= static_cast(bit << 2); + } + } + if ((sesameProbe & static_cast(bit << 2)) != 0) { + return emphasisMarkUtf8(e); + } + return emphasisMarkUtf8(filledSesame ? CssTextEmphasis::FilledDot : CssTextEmphasis::OpenDot); +} + +bool ChapterHtmlSlimParser::fontHasCodepoint(const uint32_t cp) const { + const auto& fonts = renderer.getFontMap(); + const auto it = fonts.find(fontId); + return it != fonts.end() && it->second.hasCodepoint(cp); +} + +// Rewrite compatibility ideographs the reading face has no glyph for into their unified +// form. Both sit in the BMP and so encode to three UTF-8 bytes, which is what lets this +// patch the buffer in place instead of rebuilding it. Runs on the flush path so the +// vertical and horizontal tokenizers both see the substituted text. +void ChapterHtmlSlimParser::substituteMissingCompatibilityIdeographs() { + auto* write = reinterpret_cast(partWordBuffer); + const auto* p = write; + const auto* end = p + partWordBufferIndex; + while (p < end) { + const unsigned char* cpStart = p; + const uint32_t cp = utf8NextCodepoint(&p); + if (cp == 0) break; + if (cp < 0xF900 || cp > 0xFAFF || (p - cpStart) != 3) continue; + const uint32_t unified = unifiedIdeographFor(cp); + if (unified == 0 || fontHasCodepoint(cp)) continue; + auto* out = write + (cpStart - write); + out[0] = static_cast(0xE0 | (unified >> 12)); + out[1] = static_cast(0x80 | ((unified >> 6) & 0x3F)); + out[2] = static_cast(0x80 | (unified & 0x3F)); + } +} + +bool ChapterHtmlSlimParser::fontHasVerticalForm(const uint32_t formCp) { + const uint8_t bit = 1u << (formCp - 0xFE10); + if ((vertFormProbe & bit) == 0) { + vertFormProbe |= bit; + const auto& fonts = renderer.getFontMap(); + const auto it = fonts.find(fontId); + if (it != fonts.end() && it->second.hasCodepoint(formCp)) { + vertFormProbe |= bit << 4; + } + } + return (vertFormProbe & (bit << 4)) != 0; +} + // start a new text block if needed void ChapterHtmlSlimParser::startNewTextBlock(const BlockStyle& blockStyle) { - nextWordContinues = false; // New block = new paragraph, no continuation + nextWordContinues = false; // New block = new paragraph, no continuation + pendingVerticalWhitespace = false; // and no separator carried across the paragraph boundary if (currentTextBlock) { // already have a text block running and it is empty - just reuse it if (currentTextBlock->isEmpty()) { @@ -398,7 +988,16 @@ void ChapterHtmlSlimParser::startNewTextBlock(const BlockStyle& blockStyle) { incoming.marginTop = static_cast(incoming.marginTop + lineHeight); } - currentTextBlock->setBlockStyle(style.getCombinedBlockStyle(incoming, BlockStyle::CombineAxis::Vertical)); + auto merged = style.getCombinedBlockStyle(incoming, BlockStyle::CombineAxis::Vertical); + if (isVertical) { + // In vertical text the stack already hands every descendant the container's padding + // (inheritColumnInsets), so the deposit on the empty block is the same padding again. + // The vertical merge sums padding, which put the first child of a padding-top:2em box + // 4em down while its siblings sat at 2em (test 25, 2026-09-26). Take the larger instead, + // as the merge already does for margins. + merged.paddingTop = std::max(style.paddingTop, incoming.paddingTop); + } + currentTextBlock->setBlockStyle(merged); flushPendingAnchor(); return; @@ -420,13 +1019,13 @@ void ChapterHtmlSlimParser::startNewTextBlock(const BlockStyle& blockStyle) { // If the pending anchor is a TOC chapter boundary, force a page break after the previous // block is flushed so the chapter starts on a fresh page. flushPendingAnchor(); - currentTextBlock = - makeUniqueNoThrow(extraParagraphSpacing, hyphenationEnabled, focusReadingEnabled, blockStyle); + currentTextBlock = makeUniqueNoThrow(extraParagraphSpacing, hyphenationEnabled, focusReadingEnabled, + blockStyle, isVertical); if (!currentTextBlock) { // Evict rebuildable caches and retry once before failing the build. freeink::MemoryManager::instance().ensureFree(4 * 1024); - currentTextBlock = - makeUniqueNoThrow(extraParagraphSpacing, hyphenationEnabled, focusReadingEnabled, blockStyle); + currentTextBlock = makeUniqueNoThrow(extraParagraphSpacing, hyphenationEnabled, focusReadingEnabled, + blockStyle, isVertical); } if (!currentTextBlock) { LOG_ERR("EHP", "OOM: ParsedText"); @@ -493,6 +1092,8 @@ void ChapterHtmlSlimParser::emitHorizontalRule(const BlockStyle& blockStyle) { currentPage->elements.push_back(std::move(pageRule)); setCurrentPageVisibleOffset(visibleTextOffset); currentPageNextY = static_cast(currentPageNextY + ruleThickness + bottomSpacing); + // The rule is on this page, so anything still waiting is on it too. + commitAnchorsAwaitingPlacement(); if (!pendingAnchorId.empty()) { anchorData.push_back({std::move(pendingAnchorId), static_cast(completedPageCount)}); @@ -704,8 +1305,36 @@ void ChapterHtmlSlimParser::finishTableRow() { clearLayoutLines(); } +// Vertical writing turns the physical top and bottom into the line's start and end: every +// column of every paragraph inside
    (margin-top: 5em under .vrtl) starts +// 5em down. The style stack only carried left/right insets to descendants, as horizontal flow +// needs, so the top inset reached the first child alone (through the deposit on the empty +// block) and the rest started at the column head (2026-09-26). Here it accumulates like the +// horizontal insets do; the first child's deposit merge takes the max, so it is not doubled. +static BlockStyle inheritColumnInsets(const BlockStyle& parent, BlockStyle child) { + child.marginTop = static_cast(child.marginTop + parent.marginTop); + child.paddingTop = static_cast(child.paddingTop + parent.paddingTop); + child.marginBottom = static_cast(child.marginBottom + parent.marginBottom); + child.paddingBottom = static_cast(child.paddingBottom + parent.paddingBottom); + return child; +} + void XMLCALL ChapterHtmlSlimParser::startElement(void* userData, const XML_Char* name, const XML_Char** atts) { auto* self = static_cast(userData); + // Ancestor record for descendant selectors (".vrtl .start-1em"): every element goes on + // here and comes off at the top of endElement, whatever else the two handlers do with it. + { + const char* classValue = ""; + if (atts != nullptr) { + for (int i = 0; atts[i]; i += 2) { + if (strcmp(atts[i], "class") == 0) { + classValue = atts[i + 1]; + break; + } + } + } + self->ancestorStack.push_back({name, classValue}); + } if (strcasecmp(name, "body") == 0) { // Case-insensitive to match ParagraphStreamer's tag matching (ProgressMapper). A case // mismatch here would leave visibleTextOffset at 0 for the whole section, so every page @@ -751,7 +1380,11 @@ void XMLCALL ChapterHtmlSlimParser::startElement(void* userData, const XML_Char* const char* idValue = atts[i + 1]; const bool isTocAnchor = std::find(self->tocAnchors.begin(), self->tocAnchors.end(), idValue) != self->tocAnchors.end(); - if (isTocAnchor || (!isNonNavigableInlineElement(name) && self->anchorData.size() < MAX_ANCHORS_PER_CHAPTER)) { + // Both lists count toward the cap: an anchor waiting for placement is one that will be + // recorded, just not yet. + if (isTocAnchor || + (!isNonNavigableInlineElement(name) && + self->anchorData.size() + self->anchorsAwaitingPlacement.size() < MAX_ANCHORS_PER_CHAPTER)) { // Flush a displaced anchor before overwriting. Consecutive non-block elements // (e.g.