omap: draw nothing of a glyph clipped off the page

A glyph wholly off the page's top or left edge is clipped to no rows
or columns and is not blended, and the room left on the page is taken
after the clip. The font_lifecycle omap lane blends FreeType's glyphs,
whose boxes are their ink, from the shipping driver into a page exactly
its pixels in size, with the message at the top, where '-' lies wholly
above the page.

.github/workflows/Linux-samples-gfx.yml

@@ -670,7 +670,7 @@ jobs:
     # installation on a slow runner it took the job past 30 min.
     timeout-minutes: 30
     env:
-      APT_PKGS: build-essential libx11-dev libxrandr-dev libdrm-dev xserver-xorg-core xserver-xorg-video-dummy x11-xserver-utils libsdl2-dev libxinerama-dev libwayland-dev libegl-dev libgles-dev libgbm-dev gcc-mingw-w64-x86-64 wine64 openbox cmake ninja-build ccache libgl-dev xvfb libdbus-1-dev dbus python3-dbus python3-gi weston xwayland mutter libvkd3d1 mesa-vulkan-drivers fonts-dejavu-core fonts-dejavu-mono
+      APT_PKGS: build-essential libx11-dev libxrandr-dev libdrm-dev xserver-xorg-core xserver-xorg-video-dummy x11-xserver-utils libsdl2-dev libxinerama-dev libwayland-dev libegl-dev libgles-dev libgbm-dev gcc-mingw-w64-x86-64 wine64 openbox cmake ninja-build ccache libgl-dev xvfb libdbus-1-dev dbus python3-dbus python3-gi weston xwayland mutter libvkd3d1 mesa-vulkan-drivers fonts-dejavu-core fonts-dejavu-mono libfreetype-dev
 
     steps:
       - name: Checkout
@@ -2505,7 +2505,10 @@ jobs:
           # vg and sdl2 draw on-screen messages through the shipping
           # OpenVG and SDL2 drivers - OpenVG onto a software OpenVG in
           # the test, SDL2 through its own software renderer - and check
-          # every glyph texel lands where it should.
+          # every glyph texel lands where it should; omap blends
+          # FreeType's glyphs into a page exactly its pixels in size,
+          # so a glyph clipped off its edge that is blended anyway is
+          # an overflow the sanitizer reports.
           make clean SANITIZER=address,undefined && make -j$(nproc) all SANITIZER=address,undefined
           ASAN_OPTIONS=detect_leaks=1:allocator_may_return_null=1 \
           UBSAN_OPTIONS=print_stacktrace=1 timeout 120 make check SANITIZER=address,undefined

.gitignore

@@ -319,6 +319,7 @@ samples/gfx/font_lifecycle/chain
 samples/gfx/font_lifecycle/frame
 samples/gfx/font_lifecycle/vg
 samples/gfx/font_lifecycle/sdl2
+samples/gfx/font_lifecycle/omap
 samples/gfx/slang_cache/mapped
 samples/gfx/slang_cache/read
 samples/tasks/database/database_task

gfx/drivers/omap_gfx.c

@@ -930,8 +930,6 @@ static void omap_render_msg(omap_video_t *vid, const char *msg)
          const uint8_t *src = NULL; \
          base_x               = (line_x + (pen_x)) + glyph->draw_offset_x; \
          base_y               = (line_y + (pen_y)) + glyph->draw_offset_y; \
-         max_width            = vid_width  - base_x; \
-         max_height           = vid_height - base_y; \
          glyph_width          = VIDEO_SCALE_W(glyph->dims); \
          glyph_height         = VIDEO_SCALE_H(glyph->dims); \
          src                  = atlas->buffer + VIDEO_SCALE_W(glyph->atlas_pos) + \
@@ -948,7 +946,12 @@ static void omap_render_msg(omap_video_t *vid, const char *msg)
             glyph_height += base_y; \
             base_y        = 0; \
          } \
-         if (max_width <= 0 || max_height <= 0) \
+         /* Clipped to nothing: wholly off the page's left or top, or \
+          * past its right or bottom */ \
+         max_width            = vid_width  - base_x; \
+         max_height           = vid_height - base_y; \
+         if (     glyph_width <= 0 || glyph_height <= 0 \
+               || max_width   <= 0 || max_height   <= 0) \
             break; \
          if (glyph_width > max_width) \
             glyph_width = max_width; \

samples/gfx/font_lifecycle/Makefile

@@ -11,7 +11,7 @@ SRC     := $(RA)/gfx/font_driver.c \
            $(RA)/libretro-common/encodings/encoding_utf.c \
            $(RA)/libretro-common/lists/string_list.c \
            $(RA)/libretro-common/time/rtime.c
-all: lifecycle lifecycle-threaded owner create fallback chain frame vg sdl2
+all: lifecycle lifecycle-threaded owner create fallback chain frame vg sdl2 omap
 lifecycle: font_lifecycle_test.c stubs_retroarch.c $(SRC)
 	$(CC) $(CFLAGS) -o $@ $^ -lm
 # Same tests with the marshalling branch compiled in, so a rebuild
@@ -71,6 +71,27 @@ else
 	printf '#!/bin/sh\necho "  skip: no SDL2 development files"\n' > $@
 	chmod +x $@
 endif
+# The OMAP framebuffer driver's on-screen message path, from the
+# shipping gfx/drivers/omap_gfx.c, blended into a page in memory, with
+# FreeType's glyphs, whose boxes are their ink rather than a whole
+# cell. The test includes the driver and calls only that path, so the
+# frontend the rest of it names is left unresolved (GNU ld, non-PIE).
+# Linux only: the driver includes the kernel's omapfb header. Skipped
+# without FreeType's development files.
+FT_CFLAGS := $(shell pkg-config --cflags freetype2 2>/dev/null)
+FT_LIBS   := $(shell pkg-config --libs freetype2 2>/dev/null)
+omap: font_omap_test.c stubs_retroarch.c $(SRC) \
+        $(RA)/gfx/drivers_font_renderer/stb.c $(RA)/gfx/bitmapfont.c \
+        $(RA)/gfx/drivers_font_renderer/freetype.c \
+        $(RA)/gfx/drivers/omap_gfx.c $(RA)/gfx/font_layout.h
+ifneq ($(FT_LIBS),)
+	$(CC) $(CFLAGS) -DFONT_TEST_REAL_STB -DHAVE_FREETYPE $(FT_CFLAGS) -no-pie \
+	   -o $@ $(filter %.c,$(filter-out $(RA)/gfx/drivers/omap_gfx.c,$^)) \
+	   -Wl,--unresolved-symbols=ignore-all $(FT_LIBS) -lm
+else
+	printf '#!/bin/sh\necho "  skip: no FreeType development files"\n' > $@
+	chmod +x $@
+endif
 check: all
 	ASAN_OPTIONS=detect_leaks=1 ./lifecycle
 	ASAN_OPTIONS=detect_leaks=1 ./lifecycle-threaded
@@ -81,6 +102,7 @@ check: all
 	ASAN_OPTIONS=detect_leaks=1 ./frame
 	ASAN_OPTIONS=detect_leaks=1 ./vg
 	ASAN_OPTIONS=detect_leaks=1 ./sdl2
+	ASAN_OPTIONS=detect_leaks=1 ./omap
 clean:
-	rm -f lifecycle lifecycle-threaded owner create fallback chain frame vg sdl2
+	rm -f lifecycle lifecycle-threaded owner create fallback chain frame vg sdl2 omap
 .PHONY: all check clean

samples/gfx/font_lifecycle/font_omap_test.c

@@ -0,0 +1,104 @@
+/* The OMAP framebuffer driver's on-screen message path, omap_render_msg()
+ * in the shipping gfx/drivers/omap_gfx.c, blended into a page in memory.
+ *
+ * A glyph is clipped to the page before it is blended: one partly off
+ * the top or left edge loses the rows or columns there, and one wholly
+ * off it draws nothing. FreeType's glyph boxes are their ink, so with
+ * the message at the top of the page a glyph sitting above the
+ * baseline, as '-' does, lies wholly above the page, and one with a
+ * descender, as 'g', reaches into it. The page is exactly its pixels
+ * in size, so a blend running past it is a heap overflow the sanitizer
+ * reports.
+ *
+ * omap_gfx.c is included here; the rest of the driver is never called,
+ * and its frontend references are left unresolved at link time. */
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+#include "../../../gfx/drivers/omap_gfx.c"
+
+int read_should_fail = 0;
+extern int read_real_files;
+
+static int fails = 0;
+#define CHECK(c, m) do { if (!(c)) { printf("  FAIL: %s\n", m); fails++; } } while (0)
+
+#define PAGE_W 320
+#define PAGE_H 120
+
+static unsigned page_ink(const uint32_t *px)
+{
+   unsigned i, n = 0;
+   for (i = 0; i < PAGE_W * PAGE_H; i++)
+      n += (px[i] & 0xFFFFFFu) != 0;
+   return n;
+}
+
+int main(void)
+{
+   static const char *dejavu =
+      "/usr/share/fonts/truetype/dejavu/DejaVuSans.ttf";
+   omap_video_t vid;
+   omapfb_data_t data;
+   omapfb_state_t state;
+   omapfb_page_t page;
+   uint32_t *px;
+   FILE *f;
+
+   if (!(f = fopen(dejavu, "rb")))
+   {
+      printf("  skip: no %s on this runner\n", dejavu);
+      return 0;
+   }
+   fclose(f);
+   read_real_files = 1;
+
+   memset(&vid, 0, sizeof(vid));
+   memset(&data, 0, sizeof(data));
+   memset(&state, 0, sizeof(state));
+   memset(&page, 0, sizeof(page));
+   /* Exactly the page's pixels: a blend past it is an overflow */
+   px                  = (uint32_t*)calloc(PAGE_W * PAGE_H, sizeof(*px));
+   page.buf            = px;
+   state.si.xres       = PAGE_W;
+   data.current_state  = &state;
+   data.cur_page       = &page;
+   data.bpp            = 4;
+   vid.omap            = &data;
+   vid.bytes_per_pixel = 4;
+   vid.dims            = VIDEO_SCALE_PACK(PAGE_W, PAGE_H);
+   vid.font_rgb[0]     = vid.font_rgb[1] = vid.font_rgb[2] = 255;
+
+   if (!font_renderer_create_default(&vid.font_driver, &vid.font,
+            dejavu, 24, FONT_ATLAS_FORMAT_A8))
+   {
+      printf("  FAIL: the font\n");
+      return 1;
+   }
+
+   /* At the top of the page: '-' lies above it, wholly clipped */
+   config_get_ptr()->floats.video_msg_pos_x = 0.1f;
+   config_get_ptr()->floats.video_msg_pos_y = 1.0f;
+   font_driver_frame_begin();
+   omap_render_msg(&vid, "--");
+   CHECK(page_ink(px) == 0, "a glyph wholly above the page draws nothing");
+
+   /* A descender there reaches down into the page and is clipped */
+   omap_render_msg(&vid, "g");
+   CHECK(page_ink(px) > 0, "a glyph partly above the page draws its rest");
+
+   /* Mid-page, whole */
+   memset(px, 0, PAGE_W * PAGE_H * sizeof(*px));
+   config_get_ptr()->floats.video_msg_pos_y = 0.5f;
+   font_driver_frame_begin();
+   omap_render_msg(&vid, "-T-");
+   CHECK(page_ink(px) > 0, "a message mid-page draws");
+
+   vid.font_driver->free(vid.font);
+   free(px);
+   if (fails)
+      return 1;
+   printf("  omap: glyphs clipped to the page, nothing blended past it\n");
+   return 0;
+}