n64texconv: Improve automatic palette generation (#2731)

* n64texconv: Improve automatic palette generation

* Reword documentation on automatic palettes

docs/assets/images.md

@@ -63,6 +63,10 @@ Otherwise the images are automatically co-quantized and the resulting images and
 Note the N64 supports CI images with IA16 palettes instead of RGBA16 palettes, but OoT doesn't have such textures.
 For simplicity, CI images with IA16 palettes are not supported in the build system, and all CI images are assumed to use RGBA16 palettes.
 
+### Automatic Palette Generation
+
+`n64texconv` supports automatically generating palettes when converting a png without a stored palette to a CI formatted image. However note that this process makes a number of assumptions about alpha channel handling for RGBA16 palettes. If using a CI image with an alpha channel it is strongly recommended to pre-generate a palette externally to ensure the results are optimal for the particular use. `n64texconv` will convert the alpha channel to single-bit with a fixed threshold of 128 and set the rgb content of all transparent pixels to transparent black, prioritizing filling the generated palette with visible pixels. This comes at the cost of black texture samples bleeding into visible pixels when filtered. Especially for CI4, since there are only 16 channels it is inadvisable to rely on correct bilinear sampling adjacent to alpha pixels, as storing multiple transparent pixels with different rgb contents will starve the palette of visible colors.
+
 # Skybox textures
 
 Skybox textures, located in (`extracted/VERSION/`)`assets/textures/skyboxes`, are ci8 images with the additional specificity of being limited to a palette of 128 colors that can be loaded as either the first or last 128 colors of a 256-colors palette, depending on the skybox.

tools/assets/build_from_png/build_from_png.c

@@ -539,7 +539,7 @@ static bool handle_ci_shared_tlut(const char* png_p, const struct fmt_info* fmt,
             const float dither_level = 0.5f;
 
             success = n64texconv_quantize_shared(out_indices, out_pal, &out_pal_count, texels, widths, heights,
-                                                 num_images, max_colors, dither_level) == 0;
+                                                 num_images, max_colors, dither_level, G_IM_FMT_RGBA) == 0;
             if (!success) {
                 fprintf(stderr, "Could not co-palettize images\n");
             }

tools/assets/n64texconv/src/libn64texconv/n64texconv.c

@@ -80,21 +80,49 @@ palette_dim(size_t count, size_t *width, size_t *height)
     *height = best_height;
 }
 
+static bool
+ci_alpha_fixup(struct color *texels, size_t width, size_t height, int pal_fmt)
+{
+    bool has_alpha = false;
+    if (pal_fmt == G_IM_FMT_RGBA) {
+        // For RGBA16 palette targets, preprocess the alpha channel prior to quantization
+        // to enforce single-bit alpha resolution, and set transparent pixels to transparent
+        // black to better guide quantization.
+        for (size_t i = 0; i < width * height; i++) {
+            has_alpha |= texels[i].a < 128;
+            if (texels[i].a < 128)
+                texels[i].w = 0; // Set to transparent black
+            else
+                texels[i].a = 255; // Set to opaque
+        }
+    } else {
+        has_alpha = true;
+    }
+    return has_alpha;
+}
+
 static void
 n64texconv_quantize(uint8_t *out_indices, struct color *out_pal, size_t *out_pal_count, struct color *texels,
-                    size_t width, size_t height, unsigned int max_colors, float dither_level)
+                    size_t width, size_t height, unsigned int max_colors, float dither_level, int pal_fmt)
 {
     assert(out_indices != NULL);
     assert(out_pal != NULL);
     assert(out_pal_count != NULL);
     assert(texels != NULL);
 
+    bool has_alpha = ci_alpha_fixup(texels, width, height, pal_fmt);
+
     // Set options
     liq_attr *attr = liq_attr_create();
     liq_set_max_colors(attr, max_colors);
+    // For RGBA16, ignore 3 lsbits in each color channel since the target format will be 5 bits per channel in the end
+    if (pal_fmt == G_IM_FMT_RGBA)
+        liq_set_min_posterization(attr, 3);
 
     // Quantize
     liq_image *img = liq_image_create_rgba(attr, (void *)texels, width, height, 0.0);
+    if (has_alpha)
+        liq_image_add_fixed_color(img, (liq_color){ 0, 0, 0, 0 });
     liq_result *result;
     liq_error err = liq_image_quantize(img, attr, &result);
     assert(err == LIQ_OK);
@@ -122,7 +150,7 @@ n64texconv_quantize(uint8_t *out_indices, struct color *out_pal, size_t *out_pal
 int
 n64texconv_quantize_shared(uint8_t **out_indices, struct color *out_pal, size_t *out_pal_count, struct color **texels,
                            size_t *widths, size_t *heights, size_t num_images, unsigned int max_colors,
-                           float dither_level)
+                           float dither_level, int pal_fmt)
 {
     assert(out_indices != NULL);
     assert(out_pal != NULL);
@@ -131,11 +159,18 @@ n64texconv_quantize_shared(uint8_t **out_indices, struct color *out_pal, size_t
     assert(widths != NULL);
     assert(heights != NULL);
 
+    bool has_alpha = false;
+    for (size_t i = 0; i < num_images; i++)
+        has_alpha |= ci_alpha_fixup(texels[i], widths[i], heights[i], pal_fmt);
+
     int rv = 0;
 
     // Set options
     liq_attr *attr = liq_attr_create();
     liq_set_max_colors(attr, max_colors);
+    // For RGBA16, ignore 3 lsbits in each color channel since the target format will be 5 bits per channel in the end
+    if (pal_fmt == G_IM_FMT_RGBA)
+        liq_set_min_posterization(attr, 3);
 
     // Create histogram
     liq_histogram *hist = liq_histogram_create(attr);
@@ -144,6 +179,8 @@ n64texconv_quantize_shared(uint8_t **out_indices, struct color *out_pal, size_t
     liq_image *images[num_images];
     for (size_t i = 0; i < num_images; i++) {
         images[i] = liq_image_create_rgba(attr, (void *)texels[i], widths[i], heights[i], 0);
+        if (has_alpha)
+            liq_image_add_fixed_color(images[i], (liq_color){ 0, 0, 0, 0 });
         liq_histogram_add_image(hist, attr, images[i]);
     }
 
@@ -948,7 +985,7 @@ n64texconv_image_from_png(const char *path, int fmt, int siz, int pal_fmt)
                 goto error_post_create_img;
 
             n64texconv_quantize(img->color_indices, pal->texels, &pal->count, img->texels, width, height, max_colors,
-                                0.5f);
+                                0.5f, pal_fmt);
         }
 
         // Populate texels from color indices and palette
@@ -1077,7 +1114,7 @@ n64texconv_image_reformat(struct n64_image *img, int fmt, int siz, struct n64_pa
 
             // Quantize and fill color indices
             n64texconv_quantize(new_img->color_indices, pal->texels, &pal->count, new_img->texels, width, height,
-                                max_colors, 0.5f);
+                                max_colors, 0.5f, pal->fmt);
             // Replace old texels
             for (size_t i = 0; i < width * height; i++) {
                 new_img->texels[i] = pal->texels[new_img->color_indices[i]];

tools/assets/n64texconv/src/libn64texconv/n64texconv.h

@@ -125,6 +125,6 @@ n64texconv_png_extension(struct n64_image *img);
 int
 n64texconv_quantize_shared(uint8_t **out_indices, struct color *out_pal, size_t *out_pal_count, struct color **texels,
                            size_t *widths, size_t *heights, size_t num_images, unsigned int max_colors,
-                           float dither_level);
+                           float dither_level, int pal_fmt);
 
 #endif