Bug 1579235 - Part 12 - Support native compositor surfaces. r=Bert,sotaro

This patch adds support for external compositor surfaces when
using native compositor mode.

Differential Revision: https://phabricator.services.mozilla.com/D65436

--HG--
extra : moz-landing-system : lando

gfx/wr/webrender/src/composite.rs

@@ -100,6 +100,12 @@ pub struct ExternalSurfaceDescriptor {
     pub yuv_format: YuvFormat,
     pub yuv_rescale: f32,
     pub z_id: ZBufferId,
+    /// If native compositing is enabled, the native compositor surface handle.
+    /// Otherwise, this will be None
+    pub native_surface_id: Option<NativeSurfaceId>,
+    /// If the native surface needs to be updated, this will contain the size
+    /// of the native surface as Some(size). If not dirty, this is None.
+    pub update_params: Option<DeviceIntSize>,
 }
 
 /// Information about a plane in a YUV surface.
@@ -140,6 +146,9 @@ pub struct ResolvedExternalSurface {
     pub yuv_format: YuvFormat,
     pub yuv_rescale: f32,
     pub image_buffer_kind: ImageBufferKind,
+
+    // Update information for a native surface if it's dirty
+    pub update_params: Option<(NativeSurfaceId, DeviceIntSize)>,
 }
 
 /// Public interface specified in `RendererOptions` that configures
@@ -501,6 +510,16 @@ impl CompositeState {
                 external_surface_index: ResolvedExternalSurfaceIndex(self.external_surfaces.len()),
             };
 
+            // If the external surface descriptor reports that the native surface
+            // needs to be updated, create an update params tuple for the renderer
+            // to use.
+            let update_params = external_surface.update_params.map(|surface_size| {
+                (
+                    external_surface.native_surface_id.expect("bug: no native surface!"),
+                    surface_size
+                )
+            });
+
             self.external_surfaces.push(ResolvedExternalSurface {
                 yuv_color_space: external_surface.yuv_color_space,
                 yuv_format: external_surface.yuv_format,
@@ -508,6 +527,7 @@ impl CompositeState {
                 image_buffer_kind: get_buffer_kind(yuv_planes[0].texture),
                 image_dependencies: external_surface.image_dependencies,
                 yuv_planes,
+                update_params,
             });
 
             let tile = CompositeTile {
@@ -524,10 +544,7 @@ impl CompositeState {
             // a dependency on the compositor surface external image keys / generations.
             self.descriptor.surfaces.push(
                 CompositeSurfaceDescriptor {
-                    // TODO(gw): When we add native compositor surfaces, this be
-                    //           need to be set, as that's how native compositor
-                    //           surfaces are added to the visual tree.
-                    surface_id: None,
+                    surface_id: external_surface.native_surface_id,
                     offset: tile.rect.origin,
                     clip_rect: tile.clip_rect,
                     image_dependencies: external_surface.image_dependencies,

gfx/wr/webrender/src/frame_builder.rs

@@ -406,6 +406,10 @@ impl FrameBuilder {
                     visibility_state.resource_cache.destroy_compositor_surface(native_surface.opaque);
                     visibility_state.resource_cache.destroy_compositor_surface(native_surface.alpha);
                 }
+
+                for (_, external_surface) in cache_state.external_native_surface_cache.drain() {
+                    visibility_state.resource_cache.destroy_compositor_surface(external_surface.native_surface_id)
+                }
             }
         }
 

gfx/wr/webrender/src/picture.rs

@@ -273,6 +273,8 @@ pub struct PictureCacheState {
     allocations: PictureCacheRecycledAllocations,
     /// Currently allocated native compositor surface for this picture cache.
     pub native_surface: Option<NativeSurface>,
+    /// A cache of compositor surfaces that are retained between display lists
+    pub external_native_surface_cache: FastHashMap<ExternalNativeSurfaceKey, ExternalNativeSurface>,
 }
 
 pub struct PictureCacheRecycledAllocations {
@@ -2128,6 +2130,28 @@ pub struct NativeSurface {
     pub alpha: NativeSurfaceId,
 }
 
+/// Hash key for an external native compositor surface
+#[derive(PartialEq, Eq, Hash)]
+pub struct ExternalNativeSurfaceKey {
+    /// The YUV image keys that are used to draw this surface.
+    pub image_keys: [ImageKey; 3],
+    /// The current device size of the surface.
+    pub size: DeviceIntSize,
+}
+
+/// Information about a native compositor surface cached between frames.
+pub struct ExternalNativeSurface {
+    /// If true, the surface was used this frame. Used for a simple form
+    /// of GC to remove old surfaces.
+    pub used_this_frame: bool,
+    /// The native compositor surface handle
+    pub native_surface_id: NativeSurfaceId,
+    /// List of image keys, and current image generations, that are drawn in this surface.
+    /// The image generations are used to check if the compositor surface is dirty and
+    /// needs to be updated.
+    pub image_dependencies: [ImageDependency; 3],
+}
+
 /// Represents a cache of tiles that make up a picture primitives.
 pub struct TileCacheInstance {
     /// Index of the tile cache / slice for this frame builder. It's determined
@@ -2228,6 +2252,8 @@ pub struct TileCacheInstance {
     pub external_surfaces: Vec<ExternalSurfaceDescriptor>,
     /// z-buffer ID assigned to opaque tiles in this slice
     pub z_id_opaque: ZBufferId,
+    /// A cache of compositor surfaces that are retained between frames
+    pub external_native_surface_cache: FastHashMap<ExternalNativeSurfaceKey, ExternalNativeSurface>,
 }
 
 impl TileCacheInstance {
@@ -2282,6 +2308,7 @@ impl TileCacheInstance {
             tile_size_override: None,
             external_surfaces: Vec::new(),
             z_id_opaque: ZBufferId::invalid(),
+            external_native_surface_cache: FastHashMap::default(),
         }
     }
 
@@ -2410,6 +2437,7 @@ impl TileCacheInstance {
             self.color_bindings = prev_state.color_bindings;
             self.current_tile_size = prev_state.current_tile_size;
             self.native_surface = prev_state.native_surface;
+            self.external_native_surface_cache = prev_state.external_native_surface_cache;
 
             fn recycle_map<K: std::cmp::Eq + std::hash::Hash, V>(
                 ideal_len: usize,
@@ -2442,6 +2470,15 @@ impl TileCacheInstance {
             );
         }
 
+        // At the start of the frame, step through each current compositor surface
+        // and mark it as unused. Later, this is used to free old compositor surfaces.
+        // TODO(gw): In future, we might make this more sophisticated - for example,
+        //           retaining them for >1 frame if unused, or retaining them in some
+        //           kind of pool to reduce future allocations.
+        for external_native_surface in self.external_native_surface_cache.values_mut() {
+            external_native_surface.used_this_frame = false;
+        }
+
         // Only evaluate what tile size to use fairly infrequently, so that we don't end
         // up constantly invalidating and reallocating tiles if the picture rect size is
         // changing near a threshold value.
@@ -2684,6 +2721,10 @@ impl TileCacheInstance {
                     frame_state.resource_cache.destroy_compositor_surface(native_surface.opaque);
                     frame_state.resource_cache.destroy_compositor_surface(native_surface.alpha);
                 }
+
+                for (_, external_surface) in self.external_native_surface_cache.drain() {
+                    frame_state.resource_cache.destroy_compositor_surface(external_surface.native_surface_id)
+                }
             }
             CompositorKind::Native { .. } => {
                 // This could hit even when compositor mode is not changed,
@@ -2712,7 +2753,7 @@ impl TileCacheInstance {
         data_stores: &DataStores,
         clip_store: &ClipStore,
         pictures: &[PicturePrimitive],
-        resource_cache: &ResourceCache,
+        resource_cache: &mut ResourceCache,
         opacity_binding_store: &OpacityBindingStorage,
         color_bindings: &ColorBindingStorage,
         image_instances: &ImageInstanceStorage,
@@ -2909,10 +2950,9 @@ impl TileCacheInstance {
                 //           extract the logic below and support RGBA compositor surfaces too.
                 let mut promote_to_surface = false;
 
-                // For initial implementation, only the simple (draw) compositor mode
-                // supports primitives as compositor surfaces. We can remove this restriction
-                // as a follow up, when we support this for native compositor modes.
-                if let CompositorKind::Draw { .. } = composite_state.compositor_kind {
+
+                // If picture caching is disabled, we can't support any compositor surfaces.
+                if composite_state.picture_caching_is_enabled {
                     // Check if this primitive _wants_ to be promoted to a compositor surface.
                     if prim_data.common.flags.contains(PrimitiveFlags::PREFER_COMPOSITOR_SURFACE) {
                         promote_to_surface = true;
@@ -3001,6 +3041,64 @@ impl TileCacheInstance {
                             }
                         }
 
+                        // When using native compositing, we need to find an existing native surface
+                        // handle to use, or allocate a new one. For existing native surfaces, we can
+                        // also determine whether this needs to be updated, depending on whether the
+                        // image generation(s) of the YUV planes have changed since last composite.
+                        let (native_surface_id, update_params) = match composite_state.compositor_kind {
+                            CompositorKind::Draw { .. } => {
+                                (None, None)
+                            }
+                            CompositorKind::Native { .. } => {
+                                let native_surface_size = device_rect.size.round().to_i32();
+
+                                let key = ExternalNativeSurfaceKey {
+                                    image_keys: prim_data.kind.yuv_key,
+                                    size: native_surface_size,
+                                };
+
+                                let native_surface = self.external_native_surface_cache
+                                    .entry(key)
+                                    .or_insert_with(|| {
+                                        // No existing surface, so allocate a new compositor surface and
+                                        // a single compositor tile that covers the entire compositor surface.
+
+                                        let native_surface_id = resource_cache.create_compositor_surface(
+                                            native_surface_size,
+                                            true,
+                                        );
+
+                                        let tile_id = NativeTileId {
+                                            surface_id: native_surface_id,
+                                            x: 0,
+                                            y: 0,
+                                        };
+
+                                        resource_cache.create_compositor_tile(tile_id);
+
+                                        ExternalNativeSurface {
+                                            used_this_frame: true,
+                                            native_surface_id,
+                                            image_dependencies: [ImageDependency::INVALID; 3],
+                                        }
+                                    });
+
+                                // Mark that the surface is referenced this frame so that the
+                                // backing native surface handle isn't freed.
+                                native_surface.used_this_frame = true;
+
+                                // If the image dependencies match, there is no need to update
+                                // the backing native surface.
+                                let update_params = if image_dependencies == native_surface.image_dependencies {
+                                    None
+                                } else {
+                                    Some(native_surface_size)
+                                };
+
+                                (Some(native_surface.native_surface_id), update_params)
+                            }
+                        };
+
                         // Each compositor surface allocates a unique z-id
                         self.external_surfaces.push(ExternalSurfaceDescriptor {
                             local_rect: prim_info.prim_clip_rect,
@@ -3013,6 +3111,8 @@ impl TileCacheInstance {
                             yuv_format: prim_data.kind.format,
                             yuv_rescale: prim_data.kind.color_depth.rescaling_factor(),
                             z_id: composite_state.z_generator.next(),
+                            native_surface_id,
+                            update_params,
                         });
                     }
                 } else {
@@ -3239,6 +3339,16 @@ impl TileCacheInstance {
             );
         }
 
+        // A simple GC of the native external surface cache, to remove and free any
+        // surfaces that were not referenced during the update_prim_dependencies pass.
+        self.external_native_surface_cache.retain(|_, surface| {
+            if !surface.used_this_frame {
+                frame_state.resource_cache.destroy_compositor_surface(surface.native_surface_id);
+            }
+
+            surface.used_this_frame
+        });
+
         // Detect if the picture cache was scrolled or scaled. In this case,
         // the device space dirty rects aren't applicable (until we properly
         // integrate with OS compositors that can handle scrolling slices).
@@ -4174,6 +4284,7 @@ impl PicturePrimitive {
                         root_transform: tile_cache.root_transform,
                         current_tile_size: tile_cache.current_tile_size,
                         native_surface: tile_cache.native_surface,
+                        external_native_surface_cache: tile_cache.external_native_surface_cache,
                         allocations: PictureCacheRecycledAllocations {
                             old_opacity_bindings: tile_cache.old_opacity_bindings,
                             old_color_bindings: tile_cache.old_color_bindings,

gfx/wr/webrender/src/renderer.rs

@@ -68,7 +68,7 @@ use crate::glyph_rasterizer::{GlyphFormat, GlyphRasterizer};
 use crate::gpu_cache::{GpuBlockData, GpuCacheUpdate, GpuCacheUpdateList};
 use crate::gpu_cache::{GpuCacheDebugChunk, GpuCacheDebugCmd};
 use crate::gpu_types::{PrimitiveHeaderI, PrimitiveHeaderF, ScalingInstance, SvgFilterInstance, TransformData};
-use crate::gpu_types::{CompositeInstance, ResolveInstanceData};
+use crate::gpu_types::{CompositeInstance, ResolveInstanceData, ZBufferId};
 use crate::internal_types::{TextureSource, ORTHO_FAR_PLANE, ORTHO_NEAR_PLANE, ResourceCacheError};
 use crate::internal_types::{CacheTextureId, DebugOutput, FastHashMap, FastHashSet, LayerIndex, RenderedDocument, ResultMsg};
 use crate::internal_types::{TextureCacheAllocationKind, TextureCacheUpdate, TextureUpdateList, TextureUpdateSource};
@@ -4251,6 +4251,127 @@ impl Renderer {
         }
     }
 
+    /// Rasterize any external compositor surfaces that require updating
+    fn update_external_native_surfaces(
+        &mut self,
+        external_surfaces: &[ResolvedExternalSurface],
+        results: &mut RenderResults,
+    ) {
+        if external_surfaces.is_empty() {
+            return;
+        }
+
+        let opaque_sampler = self.gpu_profile.start_sampler(GPU_SAMPLER_TAG_OPAQUE);
+
+        self.device.disable_depth();
+        self.set_blend(false, FramebufferKind::Main);
+
+        for surface in external_surfaces {
+            // See if this surface needs to be updated
+            let (native_surface_id, surface_size) = match surface.update_params {
+                Some(params) => params,
+                None => continue,
+            };
+
+            // When updating an external surface, the entire surface rect is used
+            // for all of the draw, dirty, valid and clip rect parameters.
+            let surface_rect = surface_size.into();
+
+            // Bind the native compositor surface to update
+            let surface_info = self.compositor_config
+                .compositor()
+                .unwrap()
+                .bind(
+                    NativeTileId {
+                        surface_id: native_surface_id,
+                        x: 0,
+                        y: 0,
+                    },
+                    surface_rect,
+                    surface_rect,
+                );
+
+            // Bind the native surface to current FBO target
+            let draw_target = DrawTarget::NativeSurface {
+                offset: surface_info.origin,
+                external_fbo_id: surface_info.fbo_id,
+                dimensions: surface_size,
+            };
+            self.device.bind_draw_target(draw_target);
+
+            let projection = Transform3D::ortho(
+                0.0,
+                surface_size.width as f32,
+                0.0,
+                surface_size.height as f32,
+                ORTHO_NEAR_PLANE,
+                ORTHO_FAR_PLANE,
+            );
+
+            // Bind an appropriate YUV shader for the texture format kind
+            self.shaders
+                .borrow_mut()
+                .get_composite_shader(
+                    CompositeSurfaceFormat::Yuv,
+                    surface.image_buffer_kind,
+                ).bind(
+                    &mut self.device,
+                    &projection,
+                    &mut self.renderer_errors
+                );
+
+            let textures = BatchTextures {
+                colors: [
+                    surface.yuv_planes[0].texture,
+                    surface.yuv_planes[1].texture,
+                    surface.yuv_planes[2].texture,
+                ],
+            };
+
+            // When the texture is an external texture, the UV rect is not known when
+            // the external surface descriptor is created, because external textures
+            // are not resolved until the lock() callback is invoked at the start of
+            // the frame render. To handle this, query the texture resolver for the
+            // UV rect if it's an external texture, otherwise use the default UV rect.
+            let uv_rects = [
+                self.texture_resolver.get_uv_rect(&textures.colors[0], surface.yuv_planes[0].uv_rect),
+                self.texture_resolver.get_uv_rect(&textures.colors[1], surface.yuv_planes[1].uv_rect),
+                self.texture_resolver.get_uv_rect(&textures.colors[2], surface.yuv_planes[2].uv_rect),
+            ];
+
+            let instance = CompositeInstance::new_yuv(
+                surface_rect.to_f32(),
+                surface_rect.to_f32(),
+                // z-id is not relevant when updating a native compositor surface.
+                // TODO(gw): Support compositor surfaces without z-buffer, for memory / perf win here.
+                ZBufferId(0),
+                surface.yuv_color_space,
+                surface.yuv_format,
+                surface.yuv_rescale,
+                [
+                    surface.yuv_planes[0].texture_layer as f32,
+                    surface.yuv_planes[1].texture_layer as f32,
+                    surface.yuv_planes[2].texture_layer as f32,
+                ],
+                uv_rects,
+            );
+
+            self.draw_instanced_batch(
+                &[instance],
+                VertexArrayKind::Composite,
+                &textures,
+                &mut results.stats,
+            );
+
+            self.compositor_config
+                .compositor()
+                .unwrap()
+                .unbind();
+        }
+
+        self.gpu_profile.finish_sampler(opaque_sampler);
+    }
+
     /// Draw a list of tiles to the framebuffer
     fn draw_tile_list<'a, I: Iterator<Item = &'a CompositeTile>>(
         &mut self,
@@ -5448,6 +5569,10 @@ impl Renderer {
                             // to specify how to composite each of the picture cache surfaces.
                             match self.current_compositor_kind {
                                 CompositorKind::Native { .. } => {
+                                    self.update_external_native_surfaces(
+                                        &frame.composite_state.external_surfaces,
+                                        results,
+                                    );
                                     let compositor = self.compositor_config.compositor().unwrap();
                                     frame.composite_state.composite_native(&mut **compositor);
                                 }