png_set_quantize: validate palette count parameters

contrib/libtests/pnggetset.c

@@ -773,6 +773,115 @@ test_plte_palette_sync(void)
    return 0;
 }
 
+/* Test: png_set_quantize must accept a palette that is larger than
+ * maximum_colors (it is reduced to fit), must reject invalid counts with an
+ * application error, and must not access its internally sized buffers out of
+ * bounds.
+ */
+#ifdef PNG_READ_QUANTIZE_SUPPORTED
+static int
+test_quantize_palette_length_case(const char *what, int num_palette,
+    int maximum_colors, const png_uint_16 *histogram, int expect_error)
+{
+   png_structp png_ptr;
+   png_infop info_ptr;
+   png_color palette[PNG_MAX_PALETTE_LENGTH + 1];
+   int i;
+   volatile int error_seen = 0;
+
+   /* Initialize one entry beyond the PNG maximum so that a larger-than-max
+    * palette never reads past the test's own buffer.
+    */
+   for (i = 0; i < PNG_MAX_PALETTE_LENGTH + 1; i++)
+   {
+      palette[i].red   = (png_byte)i;
+      palette[i].green = (png_byte)(i >> 1);
+      palette[i].blue  = (png_byte)(i >> 2);
+   }
+
+   png_ptr = png_create_read_struct(PNG_LIBPNG_VER_STRING, NULL, NULL, NULL);
+   if (png_ptr == NULL)
+   {
+      fprintf(stderr, "pnggetset: png_create_read_struct failed\n");
+      return 1;
+   }
+   info_ptr = png_create_info_struct(png_ptr);
+   if (info_ptr == NULL)
+   {
+      fprintf(stderr, "pnggetset: png_create_info_struct failed\n");
+      png_destroy_read_struct(&png_ptr, NULL, NULL);
+      return 1;
+   }
+   if (setjmp(png_jmpbuf(png_ptr)))
+   {
+      error_seen = 1;
+      png_destroy_read_struct(&png_ptr, &info_ptr, NULL);
+   }
+   else
+   {
+      png_set_quantize(png_ptr, palette, num_palette, maximum_colors,
+          histogram, 0);
+      png_destroy_read_struct(&png_ptr, &info_ptr, NULL);
+   }
+
+   if (error_seen != expect_error)
+   {
+      fprintf(stderr, "pnggetset: %s: expected %s, got %s\n", what,
+          expect_error ? "an error" : "no error",
+          error_seen ? "an error" : "no error");
+      return 1;
+   }
+
+   return 0;
+}
+
+static int
+test_quantize_palette_length(void)
+{
+   png_uint_16 histogram[PNG_MAX_PALETTE_LENGTH + 1];
+   int i;
+
+   for (i = 0; i < PNG_MAX_PALETTE_LENGTH + 1; i++)
+      histogram[i] = (png_uint_16)(PNG_MAX_PALETTE_LENGTH + 1 - i);
+
+   /* Valid calls, including a palette larger than maximum_colors (reduced to fit),
+    * provided both counts are within PNG_MAX_PALETTE_LENGTH. */
+   if (test_quantize_palette_length_case("valid palette", 4, 4, NULL, 0) != 0)
+      return 1;
+   if (test_quantize_palette_length_case("boundary palette",
+       PNG_MAX_PALETTE_LENGTH, PNG_MAX_PALETTE_LENGTH, NULL, 0) != 0)
+      return 1;
+   if (test_quantize_palette_length_case("valid reduction palette",
+       PNG_MAX_PALETTE_LENGTH, 128, NULL, 0) != 0)
+      return 1;
+   if (test_quantize_palette_length_case("valid reduction palette, histogram",
+       PNG_MAX_PALETTE_LENGTH, 128, histogram, 0) != 0)
+      return 1;
+
+   /* Invalid calls must raise an application error. */
+   if (test_quantize_palette_length_case("zero palette",
+       0, PNG_MAX_PALETTE_LENGTH, NULL, 1) != 0)
+      return 1;
+   if (test_quantize_palette_length_case("negative palette",
+       -1, PNG_MAX_PALETTE_LENGTH, NULL, 1) != 0)
+      return 1;
+   if (test_quantize_palette_length_case("oversized input palette",
+       PNG_MAX_PALETTE_LENGTH + 1, PNG_MAX_PALETTE_LENGTH, NULL, 1) != 0)
+      return 1;
+   if (test_quantize_palette_length_case("negative maximum colors",
+       PNG_MAX_PALETTE_LENGTH, -1, NULL, 1) != 0)
+      return 1;
+   if (test_quantize_palette_length_case("zero maximum colors",
+       4, 0, NULL, 1) != 0)
+      return 1;
+   if (test_quantize_palette_length_case("oversized maximum colors",
+       4, PNG_MAX_PALETTE_LENGTH + 1, NULL, 1) != 0)
+      return 1;
+
+   return 0;
+}
+#endif
+
 int
 main(void)
 {
@@ -858,5 +967,17 @@ main(void)
    else
       printf("PASS\n");
 
+#ifdef PNG_READ_QUANTIZE_SUPPORTED
+   printf("Testing quantize palette length validation... ");
+   fflush(stdout);
+   if (test_quantize_palette_length() != 0)
+   {
+      printf("FAIL\n");
+      result = 1;
+   }
+   else
+      printf("PASS\n");
+#endif
+
    return result;
 }

guard_malloc.c

@@ -0,0 +1,71 @@
+/* LD_PRELOAD malloc interposer: every allocation is placed at the end of a
+ * page whose following page is PROT_NONE, so ANY write past the requested
+ * size segfaults. Detects heap OOB writes deterministically without ASan.
+ */
+#define _GNU_SOURCE
+#include <stddef.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <unistd.h>
+#include <sys/mman.h>
+
+typedef struct { void *user; void *base; size_t size; int guard; } slot;
+static slot slots[65536];
+static int nslots = 0;
+
+static void *guarded_malloc(size_t size)
+{
+   long page = sysconf(_SC_PAGESIZE);
+   if (size == 0) size = 1;
+   size_t chunk = (size + page - 1) / page * page;
+   unsigned char *base = mmap(NULL, chunk + page, PROT_READ | PROT_WRITE,
+       MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
+   if (base == MAP_FAILED) return NULL;
+   if (mprotect(base + chunk, page, PROT_NONE) != 0) { munmap(base, chunk + page); return NULL; }
+   void *user = base + chunk - size;
+   if (nslots < 65536) { slots[nslots].user = user; slots[nslots].base = base; slots[nslots].size = size; nslots++; }
+   return user;
+}
+
+void *malloc(size_t size) { return guarded_malloc(size); }
+
+void *calloc(size_t n, size_t size)
+{
+   size_t total = n * size;
+   void *p = guarded_malloc(total);
+   if (p) memset(p, 0, total);
+   return p;
+}
+
+void *realloc(void *ptr, size_t size)
+{
+   void *np = guarded_malloc(size);
+   if (np && ptr)
+   {
+      int i;
+      size_t old = 0;
+      for (i = 0; i < nslots; i++) if (slots[i].user == ptr) { old = slots[i].size; break; }
+      if (old > size) old = size;
+      memcpy(np, ptr, old);
+   }
+   free(ptr);
+   return np;
+}
+
+void free(void *ptr)
+{
+   if (ptr == NULL) return;
+   int i;
+   for (i = 0; i < nslots; i++)
+   {
+      if (slots[i].user == ptr)
+      {
+         munmap(slots[i].base, ((slots[i].size + sysconf(_SC_PAGESIZE) - 1) / sysconf(_SC_PAGESIZE)) * sysconf(_SC_PAGESIZE) + sysconf(_SC_PAGESIZE));
+         slots[i].user = NULL;
+         return;
+      }
+   }
+   /* not tracked; fall back to a plain free */
+   /* nothing to do - unmapped unknowns are leaks in this interposer */
+}

pngrtran.c

@@ -479,6 +479,21 @@ png_set_quantize(png_struct *png_ptr, png_color *palette,
    if (palette == NULL)
       return;
 
+   /* Validate the palette counts.  A palette larger than maximum_colors is
+    * valid and is reduced below, but num_palette and maximum_colors must be
+    * positive and cannot exceed PNG_MAX_PALETTE_LENGTH entries, because PNG
+    * palettes and 8-bit quantized output indices are bounded at 256.
+    * Reject invalid values as an application error instead of silently
+    * skipping the transform, so that the application does not assume the
+    * palette mapping has been set up.
+    */
+   if (num_palette <= 0 || num_palette > PNG_MAX_PALETTE_LENGTH ||
+       maximum_colors <= 0 || maximum_colors > PNG_MAX_PALETTE_LENGTH)
+   {
+      png_app_error(png_ptr, "Invalid palette length in png_set_quantize");
+      return;
+   }
+
    png_ptr->transformations |= PNG_QUANTIZE;
 
    if (full_quantize == 0)
@@ -487,9 +502,8 @@ png_set_quantize(png_struct *png_ptr, png_color *palette,
 
       /* Initialize the array to index colors.
        *
-       * Ensure quantize_index can fit 256 elements (PNG_MAX_PALETTE_LENGTH)
-       * rather than num_palette elements. This is to prevent buffer overflows
-       * caused by malformed PNG files with out-of-range palette indices.
+       * quantize_index holds PNG_MAX_PALETTE_LENGTH (256) entries to cover
+       * all 8-bit palette indices (0..255).
        *
        * Be careful to avoid leaking memory. Applications are allowed to call
        * this function more than once per png_struct.

repro_quantize.c

@@ -0,0 +1,78 @@
+/*
+ * repro_quantize.c — deterministic reproduction of the png_set_quantize()
+ * out-of-bounds access on unpatched libpng.
+ *
+ * Usage:
+ *   ./repro_quantize <num_palette> <maximum_colors> [histogram 0|1]
+ *
+ * Cases that crash on unpatched libpng (SIGSEGV):
+ *   num_palette > 256   (e.g. 257 257)   -> owned-palette memcpy overflows
+ *                                           its 768-byte buffer
+ *   num_palette < 0     (e.g. -1 256)    -> memcpy of ~12.9 GB
+ *   maximum_colors <= 0 (e.g. 256 -1)    -> num_palette becomes negative,
+ *                                           then the same oversized memcpy
+ *
+ * Boundary cases that are SAFE on both patched and unpatched libpng:
+ *   1 256, 256 256, 256 255
+ *
+ * On unpatched libpng the -1 and >256 cases fault immediately (the huge
+ * memcpy reads past a small buffer).  With the guard-page malloc interposer
+ * (guard_malloc.c) every out-of-bounds case faults deterministically.
+ *
+ * Note: no PNG file is required; png_set_quantize() is called on a fresh
+ * read struct.
+ */
+#include "png.h"
+#include <stdio.h>
+#include <stdlib.h>
+
+int main(int argc, char **argv)
+{
+   int num_palette = (argc > 1) ? atoi(argv[1]) : 257;
+   int maximum_colors = (argc > 2) ? atoi(argv[2]) : 257;
+   int use_hist = (argc > 3) ? atoi(argv[3]) : 0;
+   png_structp png_ptr;
+   png_infop info_ptr;
+   png_color *palette;
+   png_uint_16 *hist = NULL;
+   int i, n;
+
+   n = (num_palette < 0) ? 4 : (num_palette > 0 ? num_palette : 1);
+   palette = (png_color *)malloc((size_t)n * sizeof(png_color));
+   if (palette == NULL) return 2;
+   for (i = 0; i < n; i++)
+   {
+      palette[i].red   = (png_byte)i;
+      palette[i].green = (png_byte)(i >> 1);
+      palette[i].blue  = (png_byte)(i >> 2);
+   }
+   if (use_hist)
+   {
+      hist = (png_uint_16 *)malloc((size_t)n * sizeof(png_uint_16));
+      if (hist == NULL) return 2;
+      for (i = 0; i < n; i++)
+         hist[i] = (png_uint_16)(n - i);
+   }
+
+   png_ptr = png_create_read_struct(PNG_LIBPNG_VER_STRING, NULL, NULL, NULL);
+   if (png_ptr == NULL) return 2;
+   info_ptr = png_create_info_struct(png_ptr);
+   if (info_ptr == NULL) { png_destroy_read_struct(&png_ptr, NULL, NULL); return 2; }
+
+   if (setjmp(png_jmpbuf(png_ptr)))
+   {
+      png_destroy_read_struct(&png_ptr, &info_ptr, NULL);
+      fprintf(stderr, "libpng raised an error (png_error/longjmp)\n");
+      return 1;
+   }
+
+   fprintf(stderr, "png_set_quantize(num_palette=%d, maximum_colors=%d, hist=%d)\n",
+       num_palette, maximum_colors, use_hist);
+   png_set_quantize(png_ptr, palette, num_palette, maximum_colors, hist, 0);
+
+   png_destroy_read_struct(&png_ptr, &info_ptr, NULL);
+   free(palette);
+   free(hist);
+   fprintf(stderr, "OK: call completed without fault\n");
+   return 0;
+}