diff --git a/backend/core/image_processing.py b/backend/core/image_processing.py new file mode 100644 index 0000000..1bc50b4 --- /dev/null +++ b/backend/core/image_processing.py @@ -0,0 +1,90 @@ +"""Reusable image fitting and e-ink quantization helpers.""" +from __future__ import annotations + +from PIL import Image, ImageEnhance, ImageFilter, ImageOps + +from . import native_dither + + +def _aligned_offset(container: int, content: int, align: str) -> int: + if align in ("left", "top", "start"): + return 0 + if align in ("right", "bottom", "end"): + return container - content + return (container - content) // 2 + + +def fit_image_to_box( + src: Image.Image, + width: int, + height: int, + *, + fit: str = "fill", + align_x: str = "center", + align_y: str = "center", +) -> Image.Image: + """Fit an image into an RGB box using JSON image-block semantics.""" + src_rgba = ImageOps.exif_transpose(src).convert("RGBA") + fit_mode = str(fit or "fill").lower() + if fit_mode in ("fill", "stretch"): + base = Image.new("RGBA", (width, height), (255, 255, 255, 255)) + base.alpha_composite(src_rgba.resize((width, height), Image.LANCZOS)) + return base.convert("RGB") + + src_w = max(1, src_rgba.size[0]) + src_h = max(1, src_rgba.size[1]) + scale_x = width / src_w + scale_y = height / src_h + scale = min(scale_x, scale_y) if fit_mode == "contain" else max(scale_x, scale_y) + resized_w = max(1, int(round(src_w * scale))) + resized_h = max(1, int(round(src_h * scale))) + resized = src_rgba.resize((resized_w, resized_h), Image.LANCZOS) + base = Image.new("RGBA", (width, height), (255, 255, 255, 255)) + paste_x = _aligned_offset(width, resized_w, align_x) + paste_y = _aligned_offset(height, resized_h, align_y) + base.alpha_composite(resized, (paste_x, paste_y)) + return base.convert("RGB") + + +def enhance_photo_for_eink(rgb: Image.Image) -> Image.Image: + """Conservative photo preparation before e-ink quantization.""" + img = ImageOps.autocontrast(rgb.convert("RGB"), cutoff=1) + img = ImageEnhance.Contrast(img).enhance(1.12) + img = ImageEnhance.Sharpness(img).enhance(1.25) + return img.filter(ImageFilter.UnsharpMask(radius=0.8, percent=80, threshold=3)) + + +def quantize_image_for_eink( + rgb: Image.Image, + *, + colors: int, + photo_enhance: bool = False, +) -> Image.Image: + """Quantize RGB image data for 2-, 3-, or 4-color e-ink output with Atkinson dithering.""" + prepared = enhance_photo_for_eink(rgb) if photo_enhance else rgb.convert("RGB") + + if colors < 3: + gray = ImageOps.autocontrast(prepared.convert("L"), cutoff=1) + return native_dither.atkinson_bw(gray) + + return native_dither.atkinson_palette(prepared, 3 if colors == 3 else 4) + + +def convert_image_block( + src: Image.Image, + width: int, + height: int, + colors: int, + *, + fit: str = "fill", + align_x: str = "center", + align_y: str = "center", + photo_enhance: bool = False, +) -> Image.Image: + """Fit and quantize an image for a JSON image block.""" + fitted = fit_image_to_box(src, width, height, fit=fit, align_x=align_x, align_y=align_y) + return quantize_image_for_eink( + fitted, + colors=colors, + photo_enhance=photo_enhance, + ) diff --git a/backend/core/json_renderer.py b/backend/core/json_renderer.py index ecc4e2b..3dc35ef 100644 --- a/backend/core/json_renderer.py +++ b/backend/core/json_renderer.py @@ -37,6 +37,7 @@ ) from .layout_presets import expand_layout_presets from .mode_catalog import builtin_catalog_map +from .image_processing import convert_image_block logger = logging.getLogger(__name__) @@ -53,12 +54,6 @@ _BACKEND_ROOT = Path(__file__).resolve().parent.parent _UPLOAD_DIR = _BACKEND_ROOT / "runtime_uploads" -_PALETTE_RGB = { - 0: (0, 0, 0), - 1: (255, 255, 255), - 2: (232, 176, 0), - 3: (200, 0, 0), -} STATUS_BAR_BOTTOM_DEFAULT = 36 # Used when screen_h unknown (e.g. dataclass default) @@ -156,73 +151,12 @@ def _replace(m: re.Match) -> str: return re.sub(r"\{(\w+)\}", _replace, template) -def _aligned_offset(container: int, content: int, align: str) -> int: - if align in ("left", "top", "start"): - return 0 - if align in ("right", "bottom", "end"): - return container - content - return (container - content) // 2 - - def _resolve_named_color(ctx: RenderContext, color_name: Any, default: int = EINK_FG) -> int: if not isinstance(color_name, str) or not color_name: return default return ctx.color_index(color_name, default) -def _convert_image_block( - src: Image.Image, - width: int, - height: int, - colors: int, - fit: str = "fill", - align_x: str = "center", - align_y: str = "center", -) -> Image.Image: - src_rgba = src.convert("RGBA") - fit_mode = str(fit or "fill").lower() - if fit_mode in ("fill", "stretch"): - base = Image.new("RGBA", (width, height), (255, 255, 255, 255)) - base.alpha_composite(src_rgba.resize((width, height), Image.LANCZOS)) - else: - src_w = max(1, src_rgba.size[0]) - src_h = max(1, src_rgba.size[1]) - scale_x = width / src_w - scale_y = height / src_h - scale = min(scale_x, scale_y) if fit_mode == "contain" else max(scale_x, scale_y) - resized_w = max(1, int(round(src_w * scale))) - resized_h = max(1, int(round(src_h * scale))) - resized = src_rgba.resize((resized_w, resized_h), Image.LANCZOS) - base = Image.new("RGBA", (width, height), (255, 255, 255, 255)) - paste_x = _aligned_offset(width, resized_w, align_x) - paste_y = _aligned_offset(height, resized_h, align_y) - base.alpha_composite(resized, (paste_x, paste_y)) - rgb = base.convert("RGB") - if colors < 3: - return rgb.convert("L").convert("1") - out = Image.new("P", rgb.size, EINK_BG) - pal = EINK_4COLOR_PALETTE + [0] * (768 - len(EINK_4COLOR_PALETTE)) - out.putpalette(pal) - allowed = (0, 1, 3) if colors == 3 else (0, 1, 2, 3) - cache: dict[tuple[int, int, int], int] = {} - mapped: list[int] = [] - for pixel in rgb.getdata(): - idx = cache.get(pixel) - if idx is None: - idx = min( - allowed, - key=lambda candidate: ( - (pixel[0] - _PALETTE_RGB[candidate][0]) ** 2 - + (pixel[1] - _PALETTE_RGB[candidate][1]) ** 2 - + (pixel[2] - _PALETTE_RGB[candidate][2]) ** 2 - ), - ) - cache[pixel] = idx - mapped.append(idx) - out.putdata(mapped) - return out - - @dataclass class RenderContext: """Mutable state threaded through block renderers.""" @@ -2521,12 +2455,22 @@ def _render_image(ctx: RenderContext, block: dict) -> None: fit = str(block.get("fit", "fill") or "fill") align_x = str(block.get("align_x", "center") or "center") align_y = str(block.get("align_y", "center") or "center") + photo_enhance = bool(block.get("photo_enhance", False)) margin_bottom = int(block.get("margin_bottom", 6) * ctx.scale) # Try pre-fetched data first (async download from json_content.py) prefetched = ctx.content.get(f"_prefetched_{field_name}") if prefetched: from io import BytesIO - img = _convert_image_block(Image.open(BytesIO(prefetched)), width, height, ctx.colors, fit=fit, align_x=align_x, align_y=align_y) + img = convert_image_block( + Image.open(BytesIO(prefetched)), + width, + height, + ctx.colors, + fit=fit, + align_x=align_x, + align_y=align_y, + photo_enhance=photo_enhance, + ) if ctx.colors >= 3: ctx.img.paste(img, (x, y)) else: @@ -2536,7 +2480,16 @@ def _render_image(ctx: RenderContext, block: dict) -> None: local_path = _resolve_local_asset(image_url) if local_path: try: - img = _convert_image_block(Image.open(local_path), width, height, ctx.colors, fit=fit, align_x=align_x, align_y=align_y) + img = convert_image_block( + Image.open(local_path), + width, + height, + ctx.colors, + fit=fit, + align_x=align_x, + align_y=align_y, + photo_enhance=photo_enhance, + ) if ctx.colors >= 3: ctx.img.paste(img, (x, y)) else: @@ -2569,7 +2522,16 @@ def _render_image(ctx: RenderContext, block: dict) -> None: if resp is None: raise last_error if last_error else ValueError("image fetch failed") from io import BytesIO - img = _convert_image_block(Image.open(BytesIO(resp.content)), width, height, ctx.colors, fit=fit, align_x=align_x, align_y=align_y) + img = convert_image_block( + Image.open(BytesIO(resp.content)), + width, + height, + ctx.colors, + fit=fit, + align_x=align_x, + align_y=align_y, + photo_enhance=photo_enhance, + ) if ctx.colors >= 3: ctx.img.paste(img, (x, y)) else: diff --git a/backend/core/modes/builtin/en/my_adaptive.json b/backend/core/modes/builtin/en/my_adaptive.json index 09f36f9..4115a0c 100644 --- a/backend/core/modes/builtin/en/my_adaptive.json +++ b/backend/core/modes/builtin/en/my_adaptive.json @@ -20,6 +20,7 @@ "x": 0, "y": 36, "fit": "cover", + "photo_enhance": true, "align_x": "center", "align_y": "center", "margin_bottom": 0 @@ -37,6 +38,7 @@ "width": 322, "height": 99, "fit": "contain", + "photo_enhance": true, "align_x": "center", "align_y": "center", "margin_bottom": 0 @@ -53,6 +55,7 @@ "x": 0, "y": 57, "fit": "cover", + "photo_enhance": true, "align_x": "center", "align_y": "center", "margin_bottom": 0 diff --git a/backend/core/modes/builtin/my_adaptive.json b/backend/core/modes/builtin/my_adaptive.json index 94526b6..ca5e3a6 100644 --- a/backend/core/modes/builtin/my_adaptive.json +++ b/backend/core/modes/builtin/my_adaptive.json @@ -26,6 +26,7 @@ "x": 0, "y": 36, "fit": "cover", + "photo_enhance": true, "align_x": "center", "align_y": "center", "margin_bottom": 0 @@ -45,6 +46,7 @@ "width": 322, "height": 99, "fit": "contain", + "photo_enhance": true, "align_x": "center", "align_y": "center", "margin_bottom": 0 @@ -61,6 +63,7 @@ "x": 0, "y": 57, "fit": "cover", + "photo_enhance": true, "align_x": "center", "align_y": "center", "margin_bottom": 0 diff --git a/backend/core/modes/schema/mode_schema.json b/backend/core/modes/schema/mode_schema.json index 56904b7..f2a1dd5 100644 --- a/backend/core/modes/schema/mode_schema.json +++ b/backend/core/modes/schema/mode_schema.json @@ -422,7 +422,11 @@ "x": { "type": "integer", "default": 0 }, "y": { "type": "integer", "default": 0 }, "width": { "type": "integer" }, - "height": { "type": "integer" } + "height": { "type": "integer" }, + "fit": { "type": "string", "enum": ["fill", "stretch", "contain", "cover"], "default": "fill" }, + "align_x": { "type": "string", "enum": ["left", "center", "right", "start", "end"], "default": "center" }, + "align_y": { "type": "string", "enum": ["top", "center", "bottom", "start", "end"], "default": "center" }, + "photo_enhance": { "type": "boolean", "default": false, "description": "Apply conservative photo contrast/sharpness enhancement before quantization." } } }, "progress_bar_block": { diff --git a/backend/core/native/README.md b/backend/core/native/README.md new file mode 100644 index 0000000..c00c15a --- /dev/null +++ b/backend/core/native/README.md @@ -0,0 +1,12 @@ +# Native e-ink dithering + +This directory contains the optional C++ implementation for Atkinson dithering. + +Build from the repository root: + +```bash +python3 backend/scripts/build_native_dither.py +``` + +The generated `libeink_dither.so` is ignored by git. Runtime code automatically +uses it when present and falls back to the Python implementation when it is not. diff --git a/backend/core/native/eink_dither.cpp b/backend/core/native/eink_dither.cpp new file mode 100644 index 0000000..70cd030 --- /dev/null +++ b/backend/core/native/eink_dither.cpp @@ -0,0 +1,165 @@ +#include +#include +#include +#include + +namespace { + +constexpr int kPalette[4][3] = { + {0, 0, 0}, + {255, 255, 255}, + {232, 176, 0}, + {200, 0, 0}, +}; + +constexpr int kKernel[6][3] = { + {1, 0, 1}, + {2, 0, 1}, + {-1, 1, 1}, + {0, 1, 1}, + {1, 1, 1}, + {0, 2, 1}, +}; + +void set_error(char* error, int error_len, const char* message) { + if (!error || error_len <= 0) { + return; + } + std::strncpy(error, message, static_cast(error_len - 1)); + error[error_len - 1] = '\0'; +} + +double clamp_byte(double value) { + return std::max(0.0, std::min(255.0, value)); +} + +int nearest_index(const double* pixel, const int* allowed, int allowed_count) { + int best = allowed[0]; + double best_dist = 1e30; + for (int i = 0; i < allowed_count; ++i) { + int idx = allowed[i]; + double dr = pixel[0] - kPalette[idx][0]; + double dg = pixel[1] - kPalette[idx][1]; + double db = pixel[2] - kPalette[idx][2]; + double dist = dr * dr + dg * dg + db * db; + if (dist < best_dist) { + best_dist = dist; + best = idx; + } + } + return best; +} + +} // namespace + +extern "C" int inksight_atkinson_palette( + const std::uint8_t* rgb, + int width, + int height, + int colors, + std::uint8_t* out, + char* error, + int error_len +) { + if (!rgb || !out) { + set_error(error, error_len, "null input/output pointer"); + return 1; + } + if (width <= 0 || height <= 0) { + set_error(error, error_len, "invalid dimensions"); + return 2; + } + if (colors != 3 && colors != 4) { + set_error(error, error_len, "colors must be 3 or 4"); + return 3; + } + + const int allowed_3[] = {0, 1, 3}; + const int allowed_4[] = {0, 1, 2, 3}; + const int* allowed = colors == 3 ? allowed_3 : allowed_4; + const int allowed_count = colors == 3 ? 3 : 4; + + const int pixel_count = width * height; + std::vector data(static_cast(pixel_count) * 3); + for (int i = 0; i < pixel_count * 3; ++i) { + data[static_cast(i)] = static_cast(rgb[i]); + } + + for (int y = 0; y < height; ++y) { + for (int x = 0; x < width; ++x) { + const int pos = y * width + x; + const int base = pos * 3; + double old_pixel[3] = {data[base], data[base + 1], data[base + 2]}; + int idx = nearest_index(old_pixel, allowed, allowed_count); + out[pos] = static_cast(idx); + + double err[3] = { + old_pixel[0] - kPalette[idx][0], + old_pixel[1] - kPalette[idx][1], + old_pixel[2] - kPalette[idx][2], + }; + + for (const auto& step : kKernel) { + const int nx = x + step[0]; + const int ny = y + step[1]; + if (nx < 0 || nx >= width || ny < 0 || ny >= height) { + continue; + } + const int nbase = (ny * width + nx) * 3; + for (int channel = 0; channel < 3; ++channel) { + data[nbase + channel] = clamp_byte( + data[nbase + channel] + err[channel] * (static_cast(step[2]) / 8.0) + ); + } + } + } + } + + return 0; +} + +extern "C" int inksight_atkinson_bw( + const std::uint8_t* gray, + int width, + int height, + std::uint8_t* out, + char* error, + int error_len +) { + if (!gray || !out) { + set_error(error, error_len, "null input/output pointer"); + return 1; + } + if (width <= 0 || height <= 0) { + set_error(error, error_len, "invalid dimensions"); + return 2; + } + + const int pixel_count = width * height; + std::vector data(static_cast(pixel_count)); + for (int i = 0; i < pixel_count; ++i) { + data[static_cast(i)] = static_cast(gray[i]); + } + + for (int y = 0; y < height; ++y) { + for (int x = 0; x < width; ++x) { + const int pos = y * width + x; + const double old_value = data[pos]; + const double new_value = old_value >= 128.0 ? 255.0 : 0.0; + out[pos] = static_cast(new_value); + const double err = old_value - new_value; + + for (const auto& step : kKernel) { + const int nx = x + step[0]; + const int ny = y + step[1]; + if (nx < 0 || nx >= width || ny < 0 || ny >= height) { + continue; + } + const int npos = ny * width + nx; + data[npos] = clamp_byte(data[npos] + err * (static_cast(step[2]) / 8.0)); + } + } + } + + return 0; +} diff --git a/backend/core/native_dither.py b/backend/core/native_dither.py new file mode 100644 index 0000000..3e6c94b --- /dev/null +++ b/backend/core/native_dither.py @@ -0,0 +1,90 @@ +"""ctypes bridge for native e-ink dithering.""" +from __future__ import annotations + +import ctypes +from pathlib import Path + +from PIL import Image + +from .config import EINK_4COLOR_PALETTE + +_LIB_PATH = Path(__file__).resolve().parent / "native" / "libeink_dither.so" +_LIB: ctypes.CDLL | None = None +_BUILD_HINT = "run `python3 backend/scripts/build_native_dither.py` from the repository root" + + +def _load_lib() -> ctypes.CDLL: + global _LIB + if _LIB is not None: + return _LIB + if not _LIB_PATH.exists(): + raise RuntimeError(f"native dithering library not found at {_LIB_PATH}; {_BUILD_HINT}") + try: + lib = ctypes.CDLL(str(_LIB_PATH)) + lib.inksight_atkinson_bw.argtypes = [ + ctypes.c_void_p, + ctypes.c_int, + ctypes.c_int, + ctypes.c_void_p, + ctypes.c_char_p, + ctypes.c_int, + ] + lib.inksight_atkinson_bw.restype = ctypes.c_int + lib.inksight_atkinson_palette.argtypes = [ + ctypes.c_void_p, + ctypes.c_int, + ctypes.c_int, + ctypes.c_int, + ctypes.c_void_p, + ctypes.c_char_p, + ctypes.c_int, + ] + lib.inksight_atkinson_palette.restype = ctypes.c_int + _LIB = lib + return lib + except OSError as exc: + raise RuntimeError(f"failed to load native dithering library at {_LIB_PATH}: {exc}") from exc + +def atkinson_bw(gray: Image.Image) -> Image.Image: + lib = _load_lib() + src = gray.convert("L") + w, h = src.size + in_buf = src.tobytes() + out_buf = ctypes.create_string_buffer(w * h) + err_buf = ctypes.create_string_buffer(256) + status = lib.inksight_atkinson_bw( + in_buf, + w, + h, + out_buf, + err_buf, + len(err_buf), + ) + if status != 0: + raise RuntimeError(f"native black/white Atkinson dithering failed: {err_buf.value.decode('utf-8', errors='replace')}") + return Image.frombytes("L", (w, h), out_buf.raw).convert("1", dither=Image.Dither.NONE) + + +def atkinson_palette(rgb: Image.Image, colors: int) -> Image.Image: + lib = _load_lib() + if colors not in (3, 4): + raise ValueError("native palette Atkinson dithering supports only 3 or 4 colors") + src = rgb.convert("RGB") + w, h = src.size + in_buf = src.tobytes() + out_buf = ctypes.create_string_buffer(w * h) + err_buf = ctypes.create_string_buffer(256) + status = lib.inksight_atkinson_palette( + in_buf, + w, + h, + int(colors), + out_buf, + err_buf, + len(err_buf), + ) + if status != 0: + raise RuntimeError(f"native palette Atkinson dithering failed: {err_buf.value.decode('utf-8', errors='replace')}") + out = Image.frombytes("P", (w, h), out_buf.raw) + out.putpalette(EINK_4COLOR_PALETTE + [0] * (768 - len(EINK_4COLOR_PALETTE))) + return out diff --git a/backend/scripts/build_native_dither.py b/backend/scripts/build_native_dither.py new file mode 100644 index 0000000..4d56d1e --- /dev/null +++ b/backend/scripts/build_native_dither.py @@ -0,0 +1,33 @@ +from __future__ import annotations + +import shutil +import subprocess +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[1] +SRC = ROOT / "core" / "native" / "eink_dither.cpp" +OUT = ROOT / "core" / "native" / "libeink_dither.so" + + +def main() -> None: + compiler = shutil.which("g++") + if not compiler: + raise SystemExit("g++ not found") + OUT.parent.mkdir(parents=True, exist_ok=True) + cmd = [ + compiler, + "-O3", + "-std=c++17", + "-fPIC", + "-shared", + str(SRC), + "-o", + str(OUT), + ] + subprocess.run(cmd, check=True) + print(OUT) + + +if __name__ == "__main__": + main() diff --git a/backend/tests/conftest.py b/backend/tests/conftest.py index 4d11f24..0c6791e 100644 --- a/backend/tests/conftest.py +++ b/backend/tests/conftest.py @@ -2,11 +2,15 @@ Shared pytest fixtures for InkSight unit tests. """ import os +import subprocess import sys +from pathlib import Path + import pytest # Ensure backend root is on sys.path so `core.*` imports work -sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..")) +BACKEND_ROOT = Path(__file__).resolve().parent.parent +sys.path.insert(0, str(BACKEND_ROOT)) # Set dummy env vars so modules can import without real keys os.environ.setdefault("DEEPSEEK_API_KEY", "sk-test-dummy-key-000") @@ -14,6 +18,15 @@ os.environ.setdefault("MOONSHOT_API_KEY", "sk-test-dummy-key-002") +def pytest_sessionstart(session): + """Build the native dithering library before tests import renderers.""" + native_lib = BACKEND_ROOT / "core" / "native" / "libeink_dither.so" + if native_lib.exists(): + return + script = BACKEND_ROOT / "scripts" / "build_native_dither.py" + subprocess.run([sys.executable, str(script)], cwd=BACKEND_ROOT, check=True) + + @pytest.fixture def sample_config(): """A typical device configuration dict.""" diff --git a/backend/tests/test_image_processing.py b/backend/tests/test_image_processing.py new file mode 100644 index 0000000..96d36ee --- /dev/null +++ b/backend/tests/test_image_processing.py @@ -0,0 +1,65 @@ +from PIL import Image + +from core import native_dither +from core.image_processing import convert_image_block, quantize_image_for_eink + + +def test_native_dither_library_is_available(): + assert native_dither._load_lib() is not None + + +def test_quantizes_black_white_image_to_1bit_with_atkinson(): + src = Image.new("RGB", (8, 1), "white") + for x in range(4): + src.putpixel((x, 0), (40, 40, 40)) + + out = quantize_image_for_eink(src, colors=2) + + assert out.mode == "1" + assert set(out.getdata()).issubset({0, 255}) + + +def test_three_color_uses_black_white_red_only_with_atkinson(): + src = Image.new("RGB", (4, 1), "white") + src.putpixel((0, 0), (0, 0, 0)) + src.putpixel((1, 0), (255, 255, 255)) + src.putpixel((2, 0), (200, 0, 0)) + src.putpixel((3, 0), (232, 176, 0)) + + out = quantize_image_for_eink(src, colors=3) + + assert out.mode == "P" + assert set(out.getdata()).issubset({0, 1, 3}) + assert 2 not in set(out.getdata()) + + +def test_four_color_can_use_yellow_and_red_with_atkinson(): + src = Image.new("RGB", (4, 1), "white") + src.putpixel((0, 0), (0, 0, 0)) + src.putpixel((1, 0), (255, 255, 255)) + src.putpixel((2, 0), (200, 0, 0)) + src.putpixel((3, 0), (232, 176, 0)) + + out = quantize_image_for_eink(src, colors=4) + values = set(out.getdata()) + + assert out.mode == "P" + assert values.issubset({0, 1, 2, 3}) + assert 2 in values + assert 3 in values + + +def test_convert_image_block_supports_photo_enhance(): + src = Image.new("RGB", (8, 4), (120, 120, 120)) + + out = convert_image_block( + src, + 20, + 10, + 2, + fit="cover", + photo_enhance=True, + ) + + assert out.size == (20, 10) + assert out.mode == "1"