Bug 1517616 - Update webrender to commit a5b036eced81dcfc012f1335277af595f931fb11 (WR PRs #3463 and #3464). r=kats

https://github.com/servo/webrender/pull/3463
https://github.com/servo/webrender/pull/3464

Differential Revision: https://phabricator.services.mozilla.com/D15699

--HG--
extra : moz-landing-system : lando

gfx/webrender_bindings/revision.txt

@@ -1 +1 @@
-4ff07fe75b0c28ee1987092b3eefed3017f5a25e
+a5b036eced81dcfc012f1335277af595f931fb11

gfx/wr/Cargo.lock

@@ -1571,6 +1571,7 @@ dependencies = [
  "thread_profiler 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)",
  "time 0.1.40 (registry+https://github.com/rust-lang/crates.io-index)",
  "webrender_api 0.58.0",
+ "webrender_build 0.0.1",
  "ws 0.7.3 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
@@ -1607,6 +1608,14 @@ dependencies = [
  "time 0.1.40 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
+[[package]]
+name = "webrender_build"
+version = "0.0.1"
+dependencies = [
+ "serde 1.0.80 (registry+https://github.com/rust-lang/crates.io-index)",
+ "sha2 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
 [[package]]
 name = "winapi"
 version = "0.2.8"

gfx/wr/webrender/Cargo.toml

@@ -16,7 +16,10 @@ capture = ["webrender_api/serialize", "ron", "serde", "debug_renderer"]
 replay = ["webrender_api/deserialize", "ron", "serde"]
 debug_renderer = []
 pathfinder = ["pathfinder_font_renderer", "pathfinder_gfx_utils", "pathfinder_partitioner", "pathfinder_path_utils"]
-serialize_program = ["serde"]
+serialize_program = ["serde", "webrender_build/serialize_program"]
+
+[build-dependencies]
+webrender_build = { version = "0.0.1", path = "../webrender_build" }
 
 [dependencies]
 app_units = "0.7"
@@ -42,6 +45,7 @@ smallvec = "0.6"
 thread_profiler = "0.1.1"
 time = "0.1"
 webrender_api = { version = "0.58.0", path = "../webrender_api" }
+webrender_build = { version = "0.0.1", path = "../webrender_build" }
 ws = { optional = true, version = "0.7.3" }
 
 [dependencies.pathfinder_font_renderer]

gfx/wr/webrender/build.rs

@@ -2,36 +2,63 @@
  * License, v. 2.0. If a copy of the MPL was not distributed with this
  * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
 
+extern crate webrender_build;
+
+use std::borrow::Cow;
 use std::env;
 use std::fs::{canonicalize, read_dir, File};
 use std::io::prelude::*;
 use std::path::{Path, PathBuf};
+use webrender_build::shader::*;
 
 fn write_shaders(glsl_files: Vec<PathBuf>, shader_file_path: &Path) {
     let mut shader_file = File::create(shader_file_path).unwrap();
 
     write!(shader_file, "/// AUTO GENERATED BY build.rs\n\n").unwrap();
-    write!(shader_file, "use std::collections::HashMap;\n").unwrap();
+    write!(shader_file, "use std::collections::HashMap;\n\n").unwrap();
+    write!(shader_file, "pub struct SourceWithDigest {{ pub source: &'static str, pub digest: &'static str }}\n\n")
+        .unwrap();
     write!(shader_file, "lazy_static! {{\n").unwrap();
     write!(
         shader_file,
-        "  pub static ref SHADERS: HashMap<&'static str, &'static str> = {{\n"
+        "  pub static ref SHADERS: HashMap<&'static str, SourceWithDigest> = {{\n"
     ).unwrap();
     write!(shader_file, "    let mut h = HashMap::new();\n").unwrap();
     for glsl in glsl_files {
+        // Compute the shader name.
+        assert!(glsl.is_file());
         let shader_name = glsl.file_name().unwrap().to_str().unwrap();
-        // strip .glsl
         let shader_name = shader_name.replace(".glsl", "");
+
+        // Compute a digest of the #include-expanded shader source. We store
+        // this as a literal alongside the source string so that we don't need
+        // to hash large strings at runtime.
+        let mut hasher = Sha256::new();
+        let base = glsl.parent().unwrap();
+        assert!(base.is_dir());
+        parse_shader_source(
+            Cow::Owned(shader_source_from_file(&glsl)),
+            &|f| Cow::Owned(shader_source_from_file(&base.join(&format!("{}.glsl", f)))),
+            &mut |s| hasher.input(s.as_bytes()),
+        );
+        let digest: ProgramSourceDigest = hasher.into();
+
+        // Compute the shader path for insertion into the include_str!() macro.
+        // This makes for more compact generated code than inserting the literal
+        // shader source into the generated file.
+        //
+        // If someone is building on a network share, I'm sorry.
         let full_path = canonicalize(&glsl).unwrap();
         let full_name = full_path.as_os_str().to_str().unwrap();
-        // if someone is building on a network share, I'm sorry.
         let full_name = full_name.replace("\\\\?\\", "");
         let full_name = full_name.replace("\\", "/");
+
         write!(
             shader_file,
-            "    h.insert(\"{}\", include_str!(\"{}\"));\n",
+            "    h.insert(\"{}\", SourceWithDigest {{ source: include_str!(\"{}\"), digest: \"{}\"}});\n",
             shader_name,
-            full_name
+            full_name,
+            digest,
         ).unwrap();
     }
     write!(shader_file, "    h\n").unwrap();

gfx/wr/webrender/src/device/gl.rs

@@ -2,7 +2,7 @@
  * License, v. 2.0. If a copy of the MPL was not distributed with this
  * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
 
-use super::super::shader_source;
+use super::super::shader_source::SHADERS;
 use api::{ColorF, ImageFormat, MemoryReport};
 use api::{DeviceIntPoint, DeviceIntRect, DeviceIntSize};
 use api::TextureTarget;
@@ -19,8 +19,6 @@ use std::borrow::Cow;
 use std::cell::{Cell, RefCell};
 use std::cmp;
 use std::collections::hash_map::Entry;
-use std::fs::File;
-use std::io::Read;
 use std::marker::PhantomData;
 use std::mem;
 use std::os::raw::c_void;
@@ -32,6 +30,8 @@ use std::slice;
 use std::sync::Arc;
 use std::sync::atomic::{AtomicUsize, ATOMIC_USIZE_INIT, Ordering};
 use std::thread;
+use webrender_build::shader::ProgramSourceDigest;
+use webrender_build::shader::{parse_shader_source, shader_source_from_file};
 
 /// Sequence number for frames, as tracked by the device layer.
 #[derive(Debug, Copy, Clone, PartialEq, Ord, Eq, PartialOrd)]
@@ -82,7 +82,6 @@ const SHADER_VERSION_GLES: &str = "#version 300 es\n";
 
 const SHADER_KIND_VERTEX: &str = "#define WR_VERTEX_SHADER\n";
 const SHADER_KIND_FRAGMENT: &str = "#define WR_FRAGMENT_SHADER\n";
-const SHADER_IMPORT: &str = "#include ";
 
 pub struct TextureSlot(pub usize);
 
@@ -184,41 +183,17 @@ fn get_shader_version(gl: &gl::Gl) -> &'static str {
 
 // Get a shader string by name, from the built in resources or
 // an override path, if supplied.
-fn get_shader_source(shader_name: &str, base_path: Option<&PathBuf>) -> Option<Cow<'static, str>> {
+fn get_shader_source(shader_name: &str, base_path: Option<&PathBuf>) -> Cow<'static, str> {
     if let Some(ref base) = base_path {
         let shader_path = base.join(&format!("{}.glsl", shader_name));
-        if shader_path.exists() {
-            let mut source = String::new();
-            File::open(&shader_path)
-                .unwrap()
-                .read_to_string(&mut source)
-                .unwrap();
-            return Some(Cow::Owned(source));
-        }
-    }
-
-    shader_source::SHADERS
-        .get(shader_name)
-        .map(|s| Cow::Borrowed(*s))
-}
-
-// Parse a shader string for imports. Imports are recursively processed, and
-// prepended to the output stream.
-fn parse_shader_source<F: FnMut(&str)>(source: Cow<'static, str>, base_path: Option<&PathBuf>, output: &mut F) {
-    for line in source.lines() {
-        if line.starts_with(SHADER_IMPORT) {
-            let imports = line[SHADER_IMPORT.len() ..].split(',');
-
-            // For each import, get the source, and recurse.
-            for import in imports {
-                if let Some(include) = get_shader_source(import, base_path) {
-                    parse_shader_source(include, base_path, output);
-                }
-            }
-        } else {
-            output(line);
-            output("\n");
-        }
+        Cow::Owned(shader_source_from_file(&shader_path))
+    } else {
+        Cow::Borrowed(
+            SHADERS
+            .get(shader_name)
+            .expect("Shader not found")
+            .source
+        )
     }
 }
 
@@ -263,6 +238,19 @@ fn do_build_shader_string<F: FnMut(&str)>(
     base_filename: &str,
     override_path: Option<&PathBuf>,
     mut output: F,
+) {
+    build_shader_prefix_string(gl_version_string, features, kind, base_filename, &mut output);
+    build_shader_main_string(base_filename, override_path, &mut output);
+}
+
+/// Walks the prefix section of the shader string, which manages the various
+/// defines for features etc.
+fn build_shader_prefix_string<F: FnMut(&str)>(
+    gl_version_string: &str,
+    features: &str,
+    kind: &str,
+    base_filename: &str,
+    output: &mut F,
 ) {
     // GLSL requires that the version number comes first.
     output(gl_version_string);
@@ -276,12 +264,16 @@ fn do_build_shader_string<F: FnMut(&str)>(
 
     // Add any defines that were passed by the caller.
     output(features);
+}
 
-    // Parse the main .glsl file, including any imports
-    // and append them to the list of sources.
-    if let Some(shared_source) = get_shader_source(base_filename, override_path) {
-        parse_shader_source(shared_source, override_path, &mut output);
-    }
+/// Walks the main .glsl file, including any imports.
+fn build_shader_main_string<F: FnMut(&str)>(
+    base_filename: &str,
+    override_path: Option<&PathBuf>,
+    output: &mut F,
+) {
+    let shared_source = get_shader_source(base_filename, override_path);
+    parse_shader_source(shared_source, &|f| get_shader_source(f, override_path), output);
 }
 
 pub trait FileWatcherHandler: Send {
@@ -691,19 +683,6 @@ pub struct VBOId(gl::GLuint);
 #[derive(PartialEq, Eq, Hash, Debug, Copy, Clone)]
 struct IBOId(gl::GLuint);
 
-#[derive(PartialEq, Eq, Hash, Debug, Clone, Default)]
-#[cfg_attr(feature = "serialize_program", derive(Deserialize, Serialize))]
-pub struct ProgramSourceDigest([u8; 32]);
-
-impl ::std::fmt::Display for ProgramSourceDigest {
-    fn fmt(&self, f: &mut ::std::fmt::Formatter) -> ::std::fmt::Result {
-        for byte in self.0.iter() {
-            f.write_fmt(format_args!("{:02x}", byte))?;
-        }
-        Ok(())
-    }
-}
-
 #[derive(Clone, PartialEq, Eq, Hash, Debug)]
 pub struct ProgramSourceInfo {
     base_filename: &'static str,
@@ -714,7 +693,7 @@ pub struct ProgramSourceInfo {
 impl ProgramSourceInfo {
     fn new(
         device: &Device,
-        base_filename: &'static str,
+        name: &'static str,
         features: String,
     ) -> Self {
         // Compute the digest. Assuming the device has a `ProgramCache`, this
@@ -722,37 +701,49 @@ impl ProgramSourceInfo {
         // we compute the hash by walking the static strings in the same order
         // as we would when concatenating the source, to avoid heap-allocating
         // in the common case.
+        //
+        // Note that we cheat a bit to make the hashing more efficient. First,
+        // the only difference between the vertex and fragment shader is a
+        // single deterministic define, so we don't need to hash both. Second,
+        // we precompute the digest of the expanded source file at build time,
+        // and then just hash that digest here.
 
-        // Construct the hasher.
+        // Setup.
         let mut hasher = Sha256::new();
+        let version_str = get_shader_version(&*device.gl());
+        let override_path = device.resource_override_path.as_ref();
+        let source_and_digest = SHADERS.get(&name).expect("Shader not found");
 
         // Hash the renderer name.
         hasher.input(device.renderer_name.as_bytes());
 
-        // Hash the vertex shader.
-        device.build_shader_string(
+        // Hash the prefix string.
+        build_shader_prefix_string(
+            version_str,
             &features,
-            SHADER_KIND_VERTEX,
-            &base_filename,
-            |s| hasher.input(s.as_bytes()),
+            &"DUMMY",
+            &name,
+            &mut |s| hasher.input(s.as_bytes()),
         );
 
-        // Hash the fragment shader.
-        device.build_shader_string(
-            &features,
-            SHADER_KIND_FRAGMENT,
-            base_filename,
-            |s| hasher.input(s.as_bytes()),
-        );
+        // Hash the shader file contents. We use a precomputed digest, and
+        // verify it in debug builds.
+        if override_path.is_some() || cfg!(debug_assertions) {
+            let mut h = Sha256::new();
+            build_shader_main_string(&name, override_path, &mut |s| h.input(s.as_bytes()));
+            let d: ProgramSourceDigest = h.into();
+            let digest = format!("{}", d);
+            debug_assert!(override_path.is_some() || digest == source_and_digest.digest);
+            hasher.input(digest.as_bytes());
+        } else {
+            hasher.input(source_and_digest.digest.as_bytes());
+        };
 
         // Finish.
-        let mut digest = ProgramSourceDigest::default();
-        digest.0.copy_from_slice(hasher.result().as_slice());
-
         ProgramSourceInfo {
-            base_filename,
+            base_filename: name,
             features,
-            digest,
+            digest: hasher.into(),
         }
     }
 

gfx/wr/webrender/src/lib.rs

@@ -194,6 +194,7 @@ extern crate png;
 extern crate rand;
 
 pub extern crate webrender_api;
+extern crate webrender_build;
 
 #[doc(hidden)]
 pub use device::{build_shader_strings, ReadPixelsFormat, UploadMethod, VertexUsageHint};

gfx/wr/webrender_build/Cargo.toml

@@ -0,0 +1,13 @@
+[package]
+name = "webrender_build"
+version = "0.0.1"
+license = "MPL-2.0"
+repository = "https://github.com/servo/webrender"
+description = "Code shared between precompilation (build.rs) and the rest of WebRender"
+
+[features]
+serialize_program = ["serde"]
+
+[dependencies]
+serde = { optional = true, version = "1.0", features = ["serde_derive"] }
+sha2 = "0.8"

gfx/wr/webrender_build/src/lib.rs

@@ -0,0 +1,11 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#[cfg(any(feature = "serde"))]
+#[macro_use]
+extern crate serde;
+
+extern crate sha2;
+
+pub mod shader;

gfx/wr/webrender_build/src/shader.rs

@@ -0,0 +1,71 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+//! Functionality for managing source code for shaders.
+//!
+//! This module is used during precompilation (build.rs) and regular compilation,
+//! so it has minimal dependencies.
+
+pub use sha2::{Digest, Sha256};
+use std::borrow::Cow;
+use std::fs::File;
+use std::io::Read;
+use std::path::Path;
+
+#[derive(PartialEq, Eq, Hash, Debug, Clone, Default)]
+#[cfg_attr(feature = "serialize_program", derive(Deserialize, Serialize))]
+pub struct ProgramSourceDigest([u8; 32]);
+
+impl ::std::fmt::Display for ProgramSourceDigest {
+    fn fmt(&self, f: &mut ::std::fmt::Formatter) -> ::std::fmt::Result {
+        for byte in self.0.iter() {
+            f.write_fmt(format_args!("{:02x}", byte))?;
+        }
+        Ok(())
+    }
+}
+
+impl From<Sha256> for ProgramSourceDigest {
+    fn from(hasher: Sha256) -> Self {
+        let mut digest = Self::default();
+        digest.0.copy_from_slice(hasher.result().as_slice());
+        digest
+    }
+}
+
+const SHADER_IMPORT: &str = "#include ";
+
+/// Parses a shader string for imports. Imports are recursively processed, and
+/// prepended to the output stream.
+pub fn parse_shader_source<F: FnMut(&str), G: Fn(&str) -> Cow<'static, str>>(
+    source: Cow<'static, str>,
+    get_source: &G,
+    output: &mut F,
+) {
+    for line in source.lines() {
+        if line.starts_with(SHADER_IMPORT) {
+            let imports = line[SHADER_IMPORT.len() ..].split(',');
+
+            // For each import, get the source, and recurse.
+            for import in imports {
+                let include = get_source(import);
+                parse_shader_source(include, get_source, output);
+            }
+        } else {
+            output(line);
+            output("\n");
+        }
+    }
+}
+
+/// Reads a shader source file from disk into a String.
+pub fn shader_source_from_file(shader_path: &Path) -> String {
+    assert!(shader_path.exists(), "Shader not found");
+    let mut source = String::new();
+    File::open(&shader_path)
+        .expect("Shader not found")
+        .read_to_string(&mut source)
+        .unwrap();
+    source
+}