Bug 1637796: Scale down mask render tasks if they exceed the maximum size r=gw

Differential Revision: https://phabricator.services.mozilla.com/D75590

gfx/wr/webrender/src/prim_store/mod.rs

@@ -85,6 +85,8 @@ pub fn register_prim_chase_id(_: PrimitiveDebugId) {
 const MIN_BRUSH_SPLIT_AREA: f32 = 128.0 * 128.0;
 pub const VECS_PER_SEGMENT: usize = 2;
 
+const MAX_MASK_SIZE: f32 = 4096.0;
+
 #[cfg_attr(feature = "capture", derive(Serialize))]
 #[cfg_attr(feature = "replay", derive(Deserialize))]
 #[derive(Debug, Copy, Clone, MallocSizeOf)]
@@ -1035,6 +1037,8 @@ impl BrushSegment {
                     }
                 };
 
+                let (device_rect, device_pixel_scale) = adjust_mask_scale_for_max_size(device_rect, device_pixel_scale);
+
                 let clip_task_id = RenderTask::new_mask(
                     device_rect.to_i32(),
                     clip_chain.clips_range,
@@ -4249,6 +4253,8 @@ impl PrimitiveInstance {
                 prim_info.clipped_world_rect,
                 device_pixel_scale,
             ) {
+                let (device_rect, device_pixel_scale) = adjust_mask_scale_for_max_size(device_rect, device_pixel_scale);
+
                 let clip_task_id = RenderTask::new_mask(
                     device_rect,
                     prim_info.clip_chain.clips_range,
@@ -4278,6 +4284,23 @@ impl PrimitiveInstance {
     }
 }
 
+// Ensures that the size of mask render tasks are within MAX_MASK_SIZE.
+fn adjust_mask_scale_for_max_size(device_rect: DeviceIntRect, device_pixel_scale: DevicePixelScale) -> (DeviceIntRect, DevicePixelScale) {
+    if device_rect.width() > MAX_MASK_SIZE as i32 || device_rect.height() > MAX_MASK_SIZE as i32 {
+        // round_out will grow by 1 integer pixel if origin is on a
+        // fractional position, so keep that margin for error with -1:
+        let scale = (MAX_MASK_SIZE - 1.0) /
+            (i32::max(device_rect.width(), device_rect.height()) as f32);
+        let new_device_pixel_scale = device_pixel_scale * Scale::new(scale);
+        let new_device_rect = (device_rect.to_f32() * Scale::new(scale))
+            .round_out()
+            .to_i32();
+        (new_device_rect, new_device_pixel_scale)
+    } else {
+        (device_rect, device_pixel_scale)
+    }
+}
+
 /// Retrieve the exact unsnapped device space rectangle for a primitive.
 fn get_unclipped_device_rect(
     prim_rect: PictureRect,

gfx/wr/wrench/reftests/transforms/raster-root-large-mask.yaml

@@ -0,0 +1,29 @@
+# Make sure we don't panic by trying to create an excessively large mask render task
+# See bug 1637796
+---
+root:
+  items:
+    - type: stacking-context
+      bounds: 0 0 0 0
+      transform: perspective(300)
+      items:
+        - type: stacking-context
+          bounds: 0 0 0 0
+          transform-style: preserve-3d
+          transform: ["scale(0.001667, 0.001667)"]
+          items:
+            - type: clip
+              bounds: [10, 10, 60000, 60000]
+              clip-rect: [10, 10, 60000, 60000]
+              id: 2
+              complex:
+                - rect: [10, 10, 60000, 60000]
+                  radius: [10000, 10000]
+                  "clip-mode": clip
+            - type: stacking-context
+              bounds: 0 0 0 0
+              clip-and-scroll: 2
+              items:
+                - type: rect
+                  bounds: 0 0 60000 60000
+                  color: blue

gfx/wr/wrench/reftests/transforms/reftest.list

@@ -46,4 +46,5 @@ platform(linux,mac) fuzzy(1,74) == border-scale-4.yaml border-scale-4.png
 skip_on(android) fuzzy-range(<=3,*3077,<=10,*133,<=93,*490) == raster_root_A_8192.yaml raster_root_A_ref.yaml
 # Same as large-raster-root.yaml but resulting in a 10302×100 raster root (= >4096) vs 4000x100 in ref:
 skip_on(android) fuzzy(60,917) == raster_root_B_8192.yaml raster_root_B_ref.yaml
-
+# Make sure we don't panic
+!= raster-root-large-mask.yaml blank.yaml