Bug 1418202 - Serialize ProgramBinary to/from blob/disk r=nical

This commit is contained in:
sotaro 2018-05-31 15:07:34 +09:00
parent cd08fcc02b
commit 624fefcfd6
13 changed files with 456 additions and 15 deletions

4
Cargo.lock generated
View File

@ -2361,15 +2361,19 @@ name = "webrender_bindings"
version = "0.1.0"
dependencies = [
"app_units 0.6.0 (registry+https://github.com/rust-lang/crates.io-index)",
"bincode 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)",
"core-foundation 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)",
"core-graphics 0.13.0 (registry+https://github.com/rust-lang/crates.io-index)",
"dwrote 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)",
"euclid 0.17.3 (registry+https://github.com/rust-lang/crates.io-index)",
"foreign-types 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)",
"fxhash 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)",
"gleam 0.5.0 (registry+https://github.com/rust-lang/crates.io-index)",
"log 0.4.1 (registry+https://github.com/rust-lang/crates.io-index)",
"nsstring 0.1.0",
"rayon 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)",
"thread_profiler 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)",
"uuid 0.1.18 (registry+https://github.com/rust-lang/crates.io-index)",
"webrender 0.57.2",
]

View File

@ -39,9 +39,11 @@ class gfxVarReceiver;
_(UseWebRenderANGLE, bool, false) \
_(UseWebRenderDCompWin, bool, false) \
_(UseWebRenderProgramBinary, bool, false) \
_(UseWebRenderProgramBinaryDisk, bool, false) \
_(WebRenderDebugFlags, int32_t, 0) \
_(ScreenDepth, int32_t, 0) \
_(GREDirectory, nsString, nsString()) \
_(ProfDirectory, nsString, nsString()) \
_(UseOMTP, bool, false) \
_(AllowD3D11KeyedMutex, bool, false) \

View File

@ -22,6 +22,7 @@
#include "mozilla/Logging.h"
#include "mozilla/Services.h"
#include "nsAppDirectoryServiceDefs.h"
#include "gfxCrashReporterUtils.h"
#include "gfxPlatform.h"
@ -866,6 +867,18 @@ gfxPlatform::Init()
Preferences::SetBool(FONT_VARIATIONS_PREF, false);
Preferences::Lock(FONT_VARIATIONS_PREF);
}
nsCOMPtr<nsIFile> profDir;
rv = NS_GetSpecialDirectory(NS_APP_PROFILE_DIR_STARTUP, getter_AddRefs(profDir));
if (NS_FAILED(rv)) {
gfxVars::SetProfDirectory(nsString());
} else {
nsAutoString path;
profDir->GetPath(path);
gfxVars::SetProfDirectory(nsString(path));
}
gfxUtils::RemoveShaderCacheFromDiskIfNecessary();
}
if (obs) {
@ -2626,7 +2639,10 @@ gfxPlatform::InitWebRenderConfig()
#endif
if (Preferences::GetBool("gfx.webrender.program-binary", false)) {
gfx::gfxVars::SetUseWebRenderProgramBinary(gfxConfig::IsEnabled(Feature::WEBRENDER));
gfxVars::SetUseWebRenderProgramBinary(gfxConfig::IsEnabled(Feature::WEBRENDER));
if (Preferences::GetBool("gfx.webrender.program-binary-disk", false)) {
gfxVars::SetUseWebRenderProgramBinaryDisk(gfxConfig::IsEnabled(Feature::WEBRENDER));
}
}
#ifdef MOZ_WIDGET_ANDROID

View File

@ -22,11 +22,14 @@
#include "mozilla/gfx/Logging.h"
#include "mozilla/gfx/PathHelpers.h"
#include "mozilla/gfx/Swizzle.h"
#include "mozilla/gfx/gfxVars.h"
#include "mozilla/Maybe.h"
#include "mozilla/RefPtr.h"
#include "mozilla/UniquePtrExtensions.h"
#include "mozilla/Unused.h"
#include "mozilla/Vector.h"
#include "mozilla/webrender/webrender_ffi.h"
#include "nsAppRunner.h"
#include "nsComponentManagerUtils.h"
#include "nsIClipboardHelper.h"
#include "nsIFile.h"
@ -1468,6 +1471,54 @@ gfxUtils::ThreadSafeGetFeatureStatus(const nsCOMPtr<nsIGfxInfo>& gfxInfo,
return gfxInfo->GetFeatureStatus(feature, failureId, status);
}
#define GFX_SHADER_CHECK_BUILD_VERSION_PREF "gfx-shader-check.build-version"
#define GFX_SHADER_CHECK_DEVICE_ID_PREF "gfx-shader-check.device-id"
#define GFX_SHADER_CHECK_DRIVER_VERSION_PREF "gfx-shader-check.driver-version"
/* static */ void
gfxUtils::RemoveShaderCacheFromDiskIfNecessary()
{
if (!gfxVars::UseWebRenderProgramBinaryDisk()) {
return;
}
nsCOMPtr<nsIGfxInfo> gfxInfo = services::GetGfxInfo();
// Get current values
nsCString buildID(mozilla::PlatformBuildID());
nsString deviceID, driverVersion;
gfxInfo->GetAdapterDeviceID(deviceID);
gfxInfo->GetAdapterDriverVersion(driverVersion);
// Get pref stored values
nsAutoCString buildIDChecked;
Preferences::GetCString(GFX_SHADER_CHECK_BUILD_VERSION_PREF, buildIDChecked);
nsAutoString deviceIDChecked, driverVersionChecked;
Preferences::GetString(GFX_SHADER_CHECK_DEVICE_ID_PREF, deviceIDChecked);
Preferences::GetString(GFX_SHADER_CHECK_DRIVER_VERSION_PREF, driverVersionChecked);
if (buildID == buildIDChecked &&
deviceID == deviceIDChecked &&
driverVersion == driverVersionChecked) {
return;
}
nsAutoString path(gfx::gfxVars::ProfDirectory());
if (!wr::remove_program_binary_disk_cache(&path)) {
// Failed to remove program binary disk cache. The disk cache might have
// invalid data. Disable program binary disk cache usage.
gfxVars::SetUseWebRenderProgramBinaryDisk(false);
return;
}
Preferences::SetCString(GFX_SHADER_CHECK_BUILD_VERSION_PREF, buildID);
Preferences::SetString(GFX_SHADER_CHECK_DEVICE_ID_PREF, deviceID);
Preferences::SetString(GFX_SHADER_CHECK_DRIVER_VERSION_PREF, driverVersion);
return;
}
/* static */ bool
gfxUtils::DumpDisplayList() {
return gfxPrefs::LayoutDumpDisplayList() ||

View File

@ -295,6 +295,8 @@ public:
nsACString& failureId,
int32_t* status);
static void RemoveShaderCacheFromDiskIfNecessary();
/**
* Copy to the clipboard as a PNG encoded Data URL.
*/

View File

@ -11,12 +11,16 @@ euclid = { version = "0.17", features = ["serde"] }
app_units = "0.6"
gleam = "0.5"
log = "0.4"
nsstring = { path = "../../servo/support/gecko/nsstring" }
bincode = "1.0"
uuid = {version = "0.1.18"}
fxhash = "0.2.1"
[dependencies.webrender]
path = "../webrender"
version = "0.57.2"
default-features = false
features = ["capture"]
features = ["capture", "serialize_program"]
[target.'cfg(target_os = "windows")'.dependencies]
dwrote = "0.4.1"

View File

@ -70,6 +70,16 @@ RenderThread::Start()
widget::WinCompositorWindowThread::Start();
#endif
layers::SharedSurfacesParent::Initialize();
if (XRE_IsGPUProcess() &&
gfx::gfxVars::UseWebRenderProgramBinary()) {
MOZ_ASSERT(gfx::gfxVars::UseWebRender());
// Initialize program cache if necessary
RefPtr<Runnable> runnable = WrapRunnable(
RefPtr<RenderThread>(sRenderThread.get()),
&RenderThread::ProgramCacheTask);
sRenderThread->Loop()->PostTask(runnable.forget());
}
}
// static
@ -511,13 +521,19 @@ RenderThread::GetRenderTexture(wr::WrExternalImageId aExternalImageId)
return mRenderTextures.GetWeak(aExternalImageId.mHandle);
}
void
RenderThread::ProgramCacheTask()
{
ProgramCache();
}
WebRenderProgramCache*
RenderThread::ProgramCache()
{
MOZ_ASSERT(IsInRenderThread());
if (!mProgramCache) {
mProgramCache = MakeUnique<WebRenderProgramCache>();
mProgramCache = MakeUnique<WebRenderProgramCache>(ThreadPool().Raw());
}
return mProgramCache.get();
}
@ -532,9 +548,16 @@ WebRenderThreadPool::~WebRenderThreadPool()
wr_thread_pool_delete(mThreadPool);
}
WebRenderProgramCache::WebRenderProgramCache()
WebRenderProgramCache::WebRenderProgramCache(wr::WrThreadPool* aThreadPool)
{
mProgramCache = wr_program_cache_new();
MOZ_ASSERT(aThreadPool);
nsAutoString path;
if (gfxVars::UseWebRenderProgramBinaryDisk()) {
path.Append(gfx::gfxVars::ProfDirectory());
}
mProgramCache = wr_program_cache_new(&path, aThreadPool);
wr_try_load_shader_from_disk(mProgramCache);
}
WebRenderProgramCache::~WebRenderProgramCache()

View File

@ -42,7 +42,7 @@ protected:
class WebRenderProgramCache {
public:
WebRenderProgramCache();
explicit WebRenderProgramCache(wr::WrThreadPool* aThreadPool);
~WebRenderProgramCache();
@ -170,6 +170,7 @@ private:
void DeferredRenderTextureHostDestroy(RefPtr<RenderTextureHost> aTexture);
void ShutDownTask(layers::SynchronousTask* aTask);
void ProgramCacheTask();
~RenderThread();

View File

@ -13,12 +13,14 @@ use webrender::{ExternalImage, ExternalImageHandler, ExternalImageSource};
use webrender::DebugFlags;
use webrender::{ApiRecordingReceiver, BinaryRecorder};
use webrender::{AsyncPropertySampler, PipelineInfo, SceneBuilderHooks};
use webrender::{ProgramCache, UploadMethod, VertexUsageHint};
use webrender::{UploadMethod, VertexUsageHint};
use thread_profiler::register_thread_with_profiler;
use moz2d_renderer::Moz2dImageRenderer;
use program_cache::{WrProgramCache, remove_disk_cache};
use app_units::Au;
use rayon;
use euclid::SideOffsets2D;
use nsstring::nsAString;
#[cfg(target_os = "windows")]
use dwrote::{FontDescriptor, FontWeight, FontStretch, FontStyle};
@ -827,23 +829,40 @@ pub unsafe extern "C" fn wr_thread_pool_delete(thread_pool: *mut WrThreadPool) {
Box::from_raw(thread_pool);
}
pub struct WrProgramCache(Rc<ProgramCache>);
#[no_mangle]
pub unsafe extern "C" fn wr_program_cache_new() -> *mut WrProgramCache {
let program_cache = ProgramCache::new(None);
Box::into_raw(Box::new(WrProgramCache(program_cache)))
pub unsafe extern "C" fn wr_program_cache_new(prof_path: &nsAString, thread_pool: *mut WrThreadPool) -> *mut WrProgramCache {
let workers = &(*thread_pool).0;
let program_cache = WrProgramCache::new(prof_path, workers);
Box::into_raw(Box::new(program_cache))
}
/// cbindgen:postfix=WR_DESTRUCTOR_SAFE_FUNC
#[no_mangle]
pub unsafe extern "C" fn wr_program_cache_delete(program_cache: *mut WrProgramCache) {
Rc::from_raw(program_cache);
Box::from_raw(program_cache);
}
#[no_mangle]
pub unsafe extern "C" fn wr_try_load_shader_from_disk(program_cache: *mut WrProgramCache) {
if !program_cache.is_null() {
(*program_cache).try_load_from_disk();
}
}
#[no_mangle]
pub unsafe extern "C" fn remove_program_binary_disk_cache(prof_path: &nsAString) -> bool {
match remove_disk_cache(prof_path) {
Ok(_) => true,
Err(_) => {
error!("Failed to remove program binary disk cache");
false
}
}
}
#[no_mangle]
pub extern "C" fn wr_renderer_update_program_cache(renderer: &mut Renderer, program_cache: &mut WrProgramCache) {
let program_cache = Rc::clone(&program_cache.0);
let program_cache = Rc::clone(&program_cache.rc_get());
renderer.update_program_cache(program_cache);
}

View File

@ -8,8 +8,12 @@ extern crate webrender;
extern crate euclid;
extern crate app_units;
extern crate gleam;
extern crate nsstring;
extern crate rayon;
extern crate thread_profiler;
extern crate bincode;
extern crate uuid;
extern crate fxhash;
#[macro_use]
extern crate log;
@ -17,6 +21,7 @@ extern crate log;
#[cfg(target_os = "windows")]
extern crate dwrote;
#[cfg(target_os = "macos")]
extern crate core_foundation;
#[cfg(target_os = "macos")]
@ -24,6 +29,8 @@ extern crate core_graphics;
#[cfg(target_os = "macos")]
extern crate foreign_types;
mod program_cache;
#[allow(non_snake_case)]
pub mod bindings;
pub mod moz2d_renderer;

View File

@ -0,0 +1,298 @@
use std::cell::RefCell;
use std::io::{Error, ErrorKind};
use std::fs::{File, create_dir_all, read_dir};
use std::io::{Read, Write};
use std::path::{PathBuf};
use std::rc::Rc;
use std::sync::Arc;
use webrender::{ProgramBinary, ProgramCache, ProgramCacheObserver};
use bincode;
use fxhash;
use nsstring::nsAString;
use rayon::ThreadPool;
use uuid::Uuid;
const MAX_LOAD_TIME_MS: u64 = 400;
const MAX_CACHED_PROGRAM_COUNT: u32 = 15;
fn deserialize_program_binary(path: &PathBuf) -> Result<Arc<ProgramBinary>, Error> {
let mut buf = vec![];
let mut file = File::open(path)?;
file.read_to_end(&mut buf)?;
if buf.len() <= 8 {
return Err(Error::new(ErrorKind::InvalidData, "File size is too small"));
}
let hash = &buf[0 .. 8];
let data = &buf[8 ..];
// Check if hash is correct
let hash:u64 = bincode::deserialize(&hash).unwrap();
let hash_data = fxhash::hash64(&data);
if hash != hash_data {
return Err(Error::new(ErrorKind::InvalidData, "File data is invalid"));
}
// Deserialize ProgramBinary
let binary = match bincode::deserialize(&data) {
Ok(binary) => binary,
Err(_) => return Err(Error::new(ErrorKind::InvalidData, "Failed to deserialize ProgramBinary")),
};
Ok(Arc::new(binary))
}
#[cfg(target_os = "windows")]
fn get_cache_path_from_prof_path(prof_path: &nsAString) -> Option<PathBuf> {
if prof_path.is_empty() {
// Empty means that we do not use disk cache.
return None;
}
use std::ffi::OsString;
use std::os::windows::prelude::*;
let prof_path = OsString::from_wide(prof_path.as_ref());
let mut cache_path = PathBuf::from(&prof_path);
cache_path.push("shader-cache");
Some(cache_path)
}
#[cfg(not(target_os="windows"))]
fn get_cache_path_from_prof_path(_prof_path: &nsAString) -> Option<PathBuf> {
// Not supported yet.
None
}
struct WrProgramBinaryDiskCache {
cache_path: Option<PathBuf>,
program_count: u32,
is_enabled: bool,
workers: Arc<ThreadPool>,
}
impl WrProgramBinaryDiskCache {
#[allow(dead_code)]
fn new(prof_path: &nsAString, workers: &Arc<ThreadPool>) -> Self {
let cache_path = get_cache_path_from_prof_path(prof_path);
let is_enabled = cache_path.is_some();
let workers = Arc::clone(workers);
WrProgramBinaryDiskCache{
cache_path,
program_count: 0,
is_enabled,
workers,
}
}
fn notify_binary_added(&mut self, program_binary: &Arc<ProgramBinary>) {
if !self.is_enabled {
return;
}
if let Some(ref cache_path) = self.cache_path {
if let Err(_) = create_dir_all(&cache_path) {
error!("failed to create dir for shader disk cache");
return;
}
self.program_count += 1;
if self.program_count > MAX_CACHED_PROGRAM_COUNT {
// Disable disk cache to avoid storing more shader programs to disk
self.is_enabled = false;
return;
}
// Use uuid for file name
let uuid1 = Uuid::new_v4();
let file_name = uuid1.to_hyphenated_string();
let program_binary = Arc::clone(program_binary);
let file_path = cache_path.join(&file_name);
let program_count = self.program_count;
// Save to disk on worker thread
self.workers.spawn(move || {
use std::time::{Instant};
let start = Instant::now();
let data: Vec<u8> = match bincode::serialize(&*program_binary) {
Ok(data) => data,
Err(err) => {
error!("Failed to serialize program binary error: {}", err);
return;
}
};
let mut file = match File::create(&file_path) {
Ok(file) => file,
Err(err) => {
error!("Unable to create file for program binary error: {}", err);
return;
}
};
// Write hash
let hash = fxhash::hash64(&data);
let hash = bincode::serialize(&hash).unwrap();
assert!(hash.len() == 8);
match file.write_all(&hash) {
Err(err) => {
error!("Failed to write hash to file error: {}", err);
}
_ => {},
};
// Write serialized data
match file.write_all(&data) {
Err(err) => {
error!("Failed to write program binary to file error: {}", err);
}
_ => {},
};
let elapsed = start.elapsed();
info!("notify_binary_added: {} ms program_count {}",
(elapsed.as_secs() * 1_000) + (elapsed.subsec_nanos() / 1_000_000) as u64, program_count);
});
}
}
pub fn try_load_from_disk(&mut self, program_cache: &Rc<ProgramCache>) {
if !self.is_enabled {
return;
}
if let Some(ref cache_path) = self.cache_path {
use std::time::{Instant};
let start = Instant::now();
// Load program binaries if exist
if cache_path.exists() && cache_path.is_dir() {
for entry in read_dir(cache_path).unwrap() {
let entry = entry.unwrap();
let path = entry.path();
info!("loading shader file");
match deserialize_program_binary(&path) {
Ok(program) => {
program_cache.load_program_binary(program);
}
Err(err) => {
error!("Failed to desriralize program binary error: {}", err);
}
};
self.program_count += 1;
let elapsed = start.elapsed();
let elapsed_ms = (elapsed.as_secs() * 1_000) + (elapsed.subsec_nanos() / 1_000_000) as u64;
info!("deserialize_program_binary: {} ms program_count {}", elapsed_ms, self.program_count);
if self.program_count > MAX_CACHED_PROGRAM_COUNT || elapsed_ms > MAX_LOAD_TIME_MS {
// Disable disk cache to avoid storing more shader programs to disk
self.is_enabled = false;
break;
}
}
}
}
}
}
pub struct WrProgramCacheObserver {
disk_cache: Rc<RefCell<WrProgramBinaryDiskCache>>,
}
impl WrProgramCacheObserver {
#[allow(dead_code)]
fn new(disk_cache: Rc<RefCell<WrProgramBinaryDiskCache>>) -> Self {
WrProgramCacheObserver{
disk_cache,
}
}
}
impl ProgramCacheObserver for WrProgramCacheObserver {
fn notify_binary_added(&self, program_binary: &Arc<ProgramBinary>) {
self.disk_cache.borrow_mut().notify_binary_added(program_binary);
}
fn notify_program_binary_failed(&self, _program_binary: &Arc<ProgramBinary>) {
error!("Failed program_binary");
}
}
pub struct WrProgramCache {
program_cache: Rc<ProgramCache>,
disk_cache: Option<Rc<RefCell<WrProgramBinaryDiskCache>>>,
}
impl WrProgramCache {
#[cfg(target_os = "windows")]
pub fn new(prof_path: &nsAString, workers: &Arc<ThreadPool>) -> Self {
let disk_cache = Rc::new(RefCell::new(WrProgramBinaryDiskCache::new(prof_path, workers)));
let program_cache_observer = Box::new(WrProgramCacheObserver::new(Rc::clone(&disk_cache)));
let program_cache = ProgramCache::new(Some(program_cache_observer));
WrProgramCache {
program_cache,
disk_cache: Some(disk_cache),
}
}
#[cfg(not(target_os="windows"))]
pub fn new(_prof_path: &nsAString, _: &Arc<ThreadPool>) -> Self {
let program_cache = ProgramCache::new(None);
WrProgramCache {
program_cache,
disk_cache: None,
}
}
pub fn rc_get(&self) -> &Rc<ProgramCache> {
&self.program_cache
}
pub fn try_load_from_disk(&self) {
if let Some(ref disk_cache) = self.disk_cache {
disk_cache.borrow_mut().try_load_from_disk(&self.program_cache);
} else {
error!("Shader disk cache is not supported");
}
}
}
#[cfg(target_os = "windows")]
pub fn remove_disk_cache(prof_path: &nsAString) -> Result<(), Error> {
use std::fs::remove_dir_all;
use std::time::{Instant};
if let Some(cache_path) = get_cache_path_from_prof_path(prof_path) {
if cache_path.exists() {
let start = Instant::now();
remove_dir_all(&cache_path)?;
let elapsed = start.elapsed();
let elapsed_ms = (elapsed.as_secs() * 1_000) + (elapsed.subsec_nanos() / 1_000_000) as u64;
info!("remove_disk_cache: {} ms", elapsed_ms);
}
}
Ok(())
}
#[cfg(not(target_os="windows"))]
pub fn remove_disk_cache(_prof_path: &nsAString) -> Result<(), Error> {
error!("Shader disk cache is not supported");
return Err(Error::new(ErrorKind::Other, "Not supported"))
}

View File

@ -17,6 +17,10 @@
namespace mozilla {
namespace wr {
static const uint32_t MAX_CACHED_PROGRAM_COUNT = 15;
static const uint64_t MAX_LOAD_TIME_MS = 400;
enum class BorderStyle : uint32_t {
None = 0,
Solid = 1,
@ -1000,6 +1004,10 @@ extern bool is_in_main_thread();
extern bool is_in_render_thread();
WR_INLINE
bool remove_program_binary_disk_cache(const nsAString *aProfPath)
WR_FUNC;
WR_INLINE
const VecU8 *wr_add_ref_arc(const ArcVecU8 *aArc)
WR_FUNC;
@ -1411,7 +1419,8 @@ void wr_program_cache_delete(WrProgramCache *aProgramCache)
WR_DESTRUCTOR_SAFE_FUNC;
WR_INLINE
WrProgramCache *wr_program_cache_new()
WrProgramCache *wr_program_cache_new(const nsAString *aProfPath,
WrThreadPool *aThreadPool)
WR_FUNC;
WR_INLINE
@ -1662,6 +1671,10 @@ void wr_transaction_update_epoch(Transaction *aTxn,
WrEpoch aEpoch)
WR_FUNC;
WR_INLINE
void wr_try_load_shader_from_disk(WrProgramCache *aProgramCache)
WR_FUNC;
WR_INLINE
void wr_vec_u8_free(WrVecU8 aV)
WR_FUNC;

View File

@ -866,6 +866,7 @@ pref("gfx.webrender.enabled", false);
pref("gfx.webrender.force-angle", true);
pref("gfx.webrender.dcomp-win.enabled", true);
pref("gfx.webrender.program-binary", true);
pref("gfx.webrender.program-binary-disk", true);
#endif
#ifdef XP_MACOSX