d3d10: a core may push a window inside the software framebuffer it was lent

The menu job's cropped/d3d10 case lent nothing: the harness core asks
for the whole frame and pushes a window inside it, as a core cropping
its overscan does. d3d10_get_current_software_framebuffer() only lent
at history[0]'s size, which after the first window pushed was the
window's, so every later request was declined; and
d3d10_sw_lend_find() only knew a frame starting at a slot's first
texel and as large as the loan. The loan is now the size asked for,
whatever history[0]'s, and a frame anywhere inside a lent slot at its
row pitch is found with its offset, which the GPU copy into
history[0] starts from.

sw_lend10_gpu_test checks a window pushed from inside a larger loan
reaching history[0] in place of the size it used to decline. Under
Wine on 26.04 with vkd3d's compiler, swfb.py passes all six d3d10
cases, cropped/d3d10 lending nothing without the change, and the unit
test passes.

gfx/drivers/d3d10.c

@@ -718,10 +718,10 @@ static bool d3d10_get_current_software_framebuffer(void *data,
    hist = &d3d10->chain.history[0];
    bpp  = (d3d10->format == DXGI_FORMAT_B5G6R5_UNORM) ? 2 : 4;
    dims = VIDEO_SCALE_PACK(fb->width, fb->height);
+   /* The loan is the size asked for, which history[0] need not be: a
+    * core may push a window inside it, history[0] at the window's size */
    if (     !hist->handle
-         || hist->desc.Format != d3d10->format
-         || hist->desc.Width  != fb->width
-         || hist->desc.Height != fb->height)
+         || hist->desc.Format != d3d10->format)
       return false;
 
    if (     d3d10->sw_lend.dims   != dims
@@ -729,6 +729,8 @@ static bool d3d10_get_current_software_framebuffer(void *data,
    {
       D3D10_TEXTURE2D_DESC desc = hist->desc;
       d3d10_sw_lend_release(d3d10);
+      desc.Width              = fb->width;
+      desc.Height             = fb->height;
       desc.MipLevels          = 1;
       desc.ArraySize          = 1;
       desc.SampleDesc.Count   = 1;
@@ -778,22 +780,41 @@ static bool d3d10_get_current_software_framebuffer(void *data,
 }
 
 /* The slot @frame was rendered into, or -1 for a frame from anywhere
- * else; remembered as the one the cached frame may now point into. */
+ * else; remembered as the one the cached frame may now point into.
+ * The frame may be a window inside the slot, as a core cropping its
+ * overscan pushes one: *x and *y say where it starts. */
 static int d3d10_sw_lend_find(d3d10_video_t *d3d10, const void *frame,
-      unsigned width, unsigned height, unsigned pitch)
+      unsigned width, unsigned height, unsigned pitch,
+      unsigned *x, unsigned *y)
 {
    unsigned k;
-   if (     !d3d10->sw_lend.mapped_bits
-         || d3d10->sw_lend.dims != VIDEO_SCALE_PACK(width, height))
+   unsigned lend_w = VIDEO_SCALE_W(d3d10->sw_lend.dims);
+   unsigned lend_h = VIDEO_SCALE_H(d3d10->sw_lend.dims);
+   unsigned bpp    = (d3d10->sw_lend.format == DXGI_FORMAT_B5G6R5_UNORM)
+      ? 2 : 4;
+   if (!d3d10->sw_lend.mapped_bits || !width || !height)
       return -1;
    for (k = 0; k < D3D10_SW_LEND_SLOTS; k++)
-      if (     (d3d10->sw_lend.mapped_bits & (1u << k))
-            && d3d10->sw_lend.mapped[k].pData    == frame
-            && d3d10->sw_lend.mapped[k].RowPitch == pitch)
-      {
-         d3d10->sw_lend.last = k;
-         return (int)k;
-      }
+   {
+      const uint8_t *base;
+      size_t off;
+      if (     !(d3d10->sw_lend.mapped_bits & (1u << k))
+            || d3d10->sw_lend.mapped[k].RowPitch != pitch)
+         continue;
+      base = (const uint8_t*)d3d10->sw_lend.mapped[k].pData;
+      if ((const uint8_t*)frame < base)
+         continue;
+      off = (size_t)((const uint8_t*)frame - base);
+      if (     off >= (size_t)pitch * lend_h
+            || (off % pitch) % bpp
+            || (off % pitch) / bpp + width  > lend_w
+            || off / pitch         + height > lend_h)
+         continue;
+      *x = (unsigned)((off % pitch) / bpp);
+      *y = (unsigned)(off / pitch);
+      d3d10->sw_lend.last = k;
+      return (int)k;
+   }
    return -1;
 }
 
@@ -802,7 +823,7 @@ static int d3d10_sw_lend_find(d3d10_video_t *d3d10, const void *frame,
  * points into the slot, is retired: a redraw draws history[0] as it
  * stands. False when history[0] no longer takes it. */
 static bool d3d10_sw_lend_push(d3d10_video_t *d3d10, int slot,
-      unsigned width, unsigned height)
+      unsigned x, unsigned y, unsigned width, unsigned height)
 {
    D3D10_BOX box;
    d3d10_texture_t *hist = &d3d10->chain.history[0];
@@ -812,11 +833,11 @@ static bool d3d10_sw_lend_push(d3d10_video_t *d3d10, int slot,
          || hist->desc.Width  != width
          || hist->desc.Height != height)
       return false;
-   box.left   = 0;
-   box.top    = 0;
+   box.left   = x;
+   box.top    = y;
    box.front  = 0;
-   box.right  = width;
-   box.bottom = height;
+   box.right  = x + width;
+   box.bottom = y + height;
    box.back   = 1;
    d3d10->sw_lend.staging[slot]->lpVtbl->Unmap(
          d3d10->sw_lend.staging[slot], 0);
@@ -4518,9 +4539,13 @@ static bool d3d10_gfx_frame(
 
       /* Rendered into a lent slot: copied on the GPU, drawn from
        * history[0]. Views may read the frame, so it stays mapped. */
-      lent = !views && frame != RETRO_HW_FRAME_BUFFER_VALID
-         && d3d10_sw_lend_push(d3d10, d3d10_sw_lend_find(d3d10, frame,
-                  width, height, pitch), width, height);
+      if (!views && frame != RETRO_HW_FRAME_BUFFER_VALID)
+      {
+         unsigned lx = 0, ly = 0;
+         int slot = d3d10_sw_lend_find(d3d10, frame, width, height,
+               pitch, &lx, &ly);
+         lent = d3d10_sw_lend_push(d3d10, slot, lx, ly, width, height);
+      }
       /* Without a preset a software frame is drawn from where it was
        * written; a preset's passes read frame.texture[0]. */
       d3d10->sw_direct.active = !lent && !d3d10->chain.shader_preset

samples/gfx/d3d_failure_paths/sw_lend10_gpu_test.c

@@ -2,8 +2,9 @@
  * device: a core renders frame after frame into the lent staging slots,
  * each pushed frame reaches chain.history[0] through the GPU copy and
  * nothing else, the slot the cached frame may point into is never lent
- * straight back, every push retires the cached frame, a frame at a size
- * history[0] does not have is declined, and the slots go unmapped. */
+ * straight back, every push retires the cached frame, a window pushed
+ * from inside a larger loan reaches history[0] at its own size, and the
+ * slots go unmapped. */
 #include <stdio.h>
 #include <stdlib.h>
 #ifdef _WIN32
@@ -92,7 +93,7 @@ int main(void)
    for (frame = 0; frame < 120; frame++)
    {
       int slot;
-      unsigned x, y;
+      unsigned x, y, lx = 1, ly = 1;
       memset(&fb, 0, sizeof(fb));
       fb.width = W; fb.height = H;
       if (!d3d10_get_current_software_framebuffer(&v, &fb))
@@ -107,12 +108,13 @@ int main(void)
          for (x = 0; x < W; x++)
             ((uint32_t*)((uint8_t*)fb.data + (size_t)y * fb.pitch))[x] =
                (frame * 2654435761u + y * W + x) & 0x00ffffffu;
-      slot = d3d10_sw_lend_find(&v, fb.data, W, H, (unsigned)fb.pitch);
-      CHECK(slot >= 0);
+      slot = d3d10_sw_lend_find(&v, fb.data, W, H, (unsigned)fb.pitch,
+            &lx, &ly);
+      CHECK(slot >= 0 && !lx && !ly);
       /* the slot the cached frame may point into is not lent straight back */
       CHECK((unsigned)slot != last);
       last = (unsigned)slot;
-      CHECK(d3d10_sw_lend_push(&v, slot, W, H));
+      CHECK(d3d10_sw_lend_push(&v, slot, 0, 0, W, H));
       CHECK(!(v.sw_lend.mapped_bits & (1u << slot)));
       CHECK(retired == ++lends);
       if (!(frame % 10))
@@ -123,12 +125,39 @@ int main(void)
    CHECK(lends >= 100);
 
    /* A frame pushed from anywhere else is not the lend's */
-   CHECK(d3d10_sw_lend_find(&v, readback, W, H, W * 4) < 0);
-   /* A size history[0] does not have yet is declined */
+   {
+      unsigned lx, ly;
+      CHECK(d3d10_sw_lend_find(&v, readback, W, H, W * 4, &lx, &ly) < 0);
+   }
+   /* A loan larger than history[0], the frame pushed as a window inside
+    * it, as a core cropping its overscan does: found at its offset and
+    * copied into history[0] at the window's size */
+   {
+      unsigned tries, x, y, lx = 0, ly = 0;
+      int slot = -1;
+      bool got = false;
+      for (tries = 0; tries < 50 && !got; tries++)
+      {
+         memset(&fb, 0, sizeof(fb));
+         fb.width = W + 64; fb.height = H + 8;
+         got = d3d10_get_current_software_framebuffer(&v, &fb);
+      }
+      CHECK(got && fb.pitch == (W + 64) * 4);
+      for (y = 0; y < H; y++)
+         for (x = 0; x < W; x++)
+            ((uint32_t*)((uint8_t*)fb.data + (size_t)(y + 4) * fb.pitch))[x + 32] =
+               (frame * 2654435761u + y * W + x) & 0x00ffffffu;
+      slot = d3d10_sw_lend_find(&v, (uint8_t*)fb.data + 4 * fb.pitch + 32 * 4,
+            W, H, (unsigned)fb.pitch, &lx, &ly);
+      CHECK(slot >= 0 && lx == 32 && ly == 4);
+      /* A window running past the loan is not the lend's */
+      CHECK(d3d10_sw_lend_find(&v, (uint8_t*)fb.data + 4 * fb.pitch + 96 * 4,
+            W, H, (unsigned)fb.pitch, &lx, &ly) < 0);
+      CHECK(d3d10_sw_lend_push(&v, slot, lx, ly, W, H));
+      CHECK(verify(&v, readback, frame));
+   }
    memset(&fb, 0, sizeof(fb));
-   fb.width = W + 16; fb.height = H;
-   CHECK(!d3d10_get_current_software_framebuffer(&v, &fb));
-   fb.width = W;
+   fb.width = W; fb.height = H;
 
    /* A lent slot never pushed still goes unmapped, the cached frame
     * retired first */