mirror of
https://github.com/mozilla/gecko-dev.git
synced 2024-10-28 12:45:27 +00:00
43b02049e1
We've discussed this some and I think there's consensus to do it as a pragmatic decision for now. CPU painting is more stable, especially with buggy drivers, and faster (because we aren't caching the necessary OpenGL objects yet and possibly for other reasons), so it provides a better "out of the box" experience for newcomers to Servo who don't know to pass the `-c` option. This patch continues to reftest both Skia and Skia-GL out of a desire to keep options open. Skia-GL remains a first-class citizen. r? @metajack Source-Repo: https://github.com/servo/servo Source-Revision: 1fd7650de504611016d1ce10a5af2c1a4e0f6b9c
306 lines
10 KiB
Rust
306 lines
10 KiB
Rust
// Copyright 2013 The Servo Project Developers. See the COPYRIGHT
|
|
// file at the top-level directory of this distribution.
|
|
//
|
|
// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
|
|
// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
|
|
// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
|
|
// option. This file may not be copied, modified, or distributed
|
|
// except according to those terms.
|
|
|
|
#![deny(unused_imports, unused_variable)]
|
|
|
|
extern crate png;
|
|
extern crate test;
|
|
extern crate regex;
|
|
extern crate url;
|
|
|
|
use std::ascii::StrAsciiExt;
|
|
use std::io;
|
|
use std::io::{File, Reader, Command};
|
|
use std::io::process::ExitStatus;
|
|
use std::io::fs::PathExtensions;
|
|
use std::os;
|
|
use std::path::Path;
|
|
use test::{AutoColor, DynTestName, DynTestFn, TestDesc, TestOpts, TestDescAndFn};
|
|
use test::run_tests_console;
|
|
use regex::Regex;
|
|
use url::Url;
|
|
|
|
|
|
bitflags!(
|
|
flags RenderMode: u32 {
|
|
static CpuRendering = 0x00000001,
|
|
static GpuRendering = 0x00000010,
|
|
static LinuxTarget = 0x00000100,
|
|
static MacOsTarget = 0x00001000,
|
|
static AndroidTarget = 0x00010000
|
|
}
|
|
)
|
|
|
|
|
|
fn main() {
|
|
let args = os::args();
|
|
let mut parts = args.tail().split(|e| "--" == e.as_slice());
|
|
|
|
let harness_args = parts.next().unwrap(); // .split() is never empty
|
|
let servo_args = parts.next().unwrap_or(&[]);
|
|
|
|
let (render_mode_string, base_path, testname) = match harness_args {
|
|
[] | [_] => fail!("USAGE: cpu|gpu base_path [testname regex]"),
|
|
[ref render_mode_string, ref base_path] => (render_mode_string, base_path, None),
|
|
[ref render_mode_string, ref base_path, ref testname, ..] => (render_mode_string, base_path, Some(Regex::new(testname.as_slice()).unwrap())),
|
|
};
|
|
|
|
let mut render_mode = match render_mode_string.as_slice() {
|
|
"cpu" => CpuRendering,
|
|
"gpu" => GpuRendering,
|
|
_ => fail!("First argument must specify cpu or gpu as rendering mode")
|
|
};
|
|
if cfg!(target_os = "linux") {
|
|
render_mode.insert(LinuxTarget);
|
|
}
|
|
if cfg!(target_os = "macos") {
|
|
render_mode.insert(MacOsTarget);
|
|
}
|
|
if cfg!(target_os = "android") {
|
|
render_mode.insert(AndroidTarget);
|
|
}
|
|
|
|
let mut all_tests = vec!();
|
|
println!("Scanning {} for manifests\n", base_path);
|
|
|
|
for file in io::fs::walk_dir(&Path::new(base_path.as_slice())).unwrap() {
|
|
let maybe_extension = file.extension_str();
|
|
match maybe_extension {
|
|
Some(extension) => {
|
|
if extension.to_ascii_lower().as_slice() == "list" && file.is_file() {
|
|
let tests = parse_lists(&file, servo_args, render_mode, all_tests.len());
|
|
println!("\t{} [{} tests]", file.display(), tests.len());
|
|
all_tests.extend(tests.into_iter());
|
|
}
|
|
}
|
|
_ => {}
|
|
}
|
|
}
|
|
|
|
let test_opts = TestOpts {
|
|
filter: testname,
|
|
run_ignored: false,
|
|
logfile: None,
|
|
run_tests: true,
|
|
run_benchmarks: false,
|
|
ratchet_noise_percent: None,
|
|
ratchet_metrics: None,
|
|
save_metrics: None,
|
|
test_shard: None,
|
|
nocapture: false,
|
|
color: AutoColor
|
|
};
|
|
|
|
match run_tests_console(&test_opts, all_tests) {
|
|
Ok(false) => os::set_exit_status(1), // tests failed
|
|
Err(_) => os::set_exit_status(2), // I/O-related failure
|
|
_ => (),
|
|
}
|
|
}
|
|
|
|
#[deriving(PartialEq)]
|
|
enum ReftestKind {
|
|
Same,
|
|
Different,
|
|
}
|
|
|
|
struct Reftest {
|
|
name: String,
|
|
kind: ReftestKind,
|
|
files: [Path, ..2],
|
|
id: uint,
|
|
servo_args: Vec<String>,
|
|
render_mode: RenderMode,
|
|
is_flaky: bool,
|
|
experimental: bool,
|
|
fragment_identifier: Option<String>,
|
|
}
|
|
|
|
struct TestLine<'a> {
|
|
conditions: &'a str,
|
|
kind: &'a str,
|
|
file_left: &'a str,
|
|
file_right: &'a str,
|
|
}
|
|
|
|
fn parse_lists(file: &Path, servo_args: &[String], render_mode: RenderMode, id_offset: uint) -> Vec<TestDescAndFn> {
|
|
let mut tests = Vec::new();
|
|
let contents = File::open_mode(file, io::Open, io::Read)
|
|
.and_then(|mut f| f.read_to_string())
|
|
.ok().expect("Could not read file");
|
|
|
|
for line in contents.as_slice().lines() {
|
|
// ignore comments or empty lines
|
|
if line.starts_with("#") || line.is_empty() {
|
|
continue;
|
|
}
|
|
|
|
let parts: Vec<&str> = line.split(' ').filter(|p| !p.is_empty()).collect();
|
|
|
|
let test_line = match parts.len() {
|
|
3 => TestLine {
|
|
conditions: "",
|
|
kind: parts[0],
|
|
file_left: parts[1],
|
|
file_right: parts[2],
|
|
},
|
|
4 => TestLine {
|
|
conditions: parts[0],
|
|
kind: parts[1],
|
|
file_left: parts[2],
|
|
file_right: parts[3],
|
|
},
|
|
_ => fail!("reftest line: '{:s}' doesn't match '[CONDITIONS] KIND LEFT RIGHT'", line),
|
|
};
|
|
|
|
let kind = match test_line.kind {
|
|
"==" => Same,
|
|
"!=" => Different,
|
|
part => fail!("reftest line: '{:s}' has invalid kind '{:s}'", line, part)
|
|
};
|
|
let base = file.dir_path();
|
|
let file_left = base.join(test_line.file_left);
|
|
let file_right = base.join(test_line.file_right);
|
|
|
|
let mut conditions_list = test_line.conditions.split(',');
|
|
let mut flakiness = RenderMode::empty();
|
|
let mut experimental = false;
|
|
let mut fragment_identifier = None;
|
|
for condition in conditions_list {
|
|
match condition {
|
|
"flaky_cpu" => flakiness.insert(CpuRendering),
|
|
"flaky_gpu" => flakiness.insert(GpuRendering),
|
|
"flaky_linux" => flakiness.insert(LinuxTarget),
|
|
"flaky_macos" => flakiness.insert(MacOsTarget),
|
|
"experimental" => experimental = true,
|
|
_ => (),
|
|
}
|
|
if condition.starts_with("fragment=") {
|
|
fragment_identifier = Some(condition.slice_from("fragment=".len()).to_string());
|
|
}
|
|
}
|
|
|
|
let reftest = Reftest {
|
|
name: format!("{} {} {}", test_line.file_left, test_line.kind, test_line.file_right),
|
|
kind: kind,
|
|
files: [file_left, file_right],
|
|
id: id_offset + tests.len(),
|
|
render_mode: render_mode,
|
|
servo_args: servo_args.iter().map(|x| x.clone()).collect(),
|
|
is_flaky: render_mode.intersects(flakiness),
|
|
experimental: experimental,
|
|
fragment_identifier: fragment_identifier,
|
|
};
|
|
|
|
tests.push(make_test(reftest));
|
|
}
|
|
tests
|
|
}
|
|
|
|
fn make_test(reftest: Reftest) -> TestDescAndFn {
|
|
let name = reftest.name.clone();
|
|
TestDescAndFn {
|
|
desc: TestDesc {
|
|
name: DynTestName(name),
|
|
ignore: false,
|
|
should_fail: false,
|
|
},
|
|
testfn: DynTestFn(proc() {
|
|
check_reftest(reftest);
|
|
}),
|
|
}
|
|
}
|
|
|
|
fn capture(reftest: &Reftest, side: uint) -> (u32, u32, Vec<u8>) {
|
|
let png_filename = format!("/tmp/servo-reftest-{:06u}-{:u}.png", reftest.id, side);
|
|
let mut command = Command::new("target/servo");
|
|
command
|
|
.args(reftest.servo_args.as_slice())
|
|
// Allows pixel perfect rendering of Ahem font for reftests.
|
|
.arg("--disable-text-aa")
|
|
.args(["-f", "-o"])
|
|
.arg(png_filename.as_slice())
|
|
.arg({
|
|
let mut url = Url::from_file_path(&reftest.files[side]).unwrap();
|
|
url.fragment = reftest.fragment_identifier.clone();
|
|
url.to_string()
|
|
});
|
|
// CPU rendering is the default
|
|
if reftest.render_mode.contains(CpuRendering) {
|
|
command.arg("-c");
|
|
}
|
|
if reftest.render_mode.contains(GpuRendering) {
|
|
command.arg("-g");
|
|
}
|
|
if reftest.experimental {
|
|
command.arg("--experimental");
|
|
}
|
|
let retval = match command.status() {
|
|
Ok(status) => status,
|
|
Err(e) => fail!("failed to execute process: {}", e),
|
|
};
|
|
assert_eq!(retval, ExitStatus(0));
|
|
|
|
let image = png::load_png(&from_str::<Path>(png_filename.as_slice()).unwrap()).unwrap();
|
|
let rgba8_bytes = match image.pixels {
|
|
png::RGBA8(pixels) => pixels,
|
|
_ => fail!(),
|
|
};
|
|
(image.width, image.height, rgba8_bytes)
|
|
}
|
|
|
|
fn check_reftest(reftest: Reftest) {
|
|
let (left_width, left_height, left_bytes) = capture(&reftest, 0);
|
|
let (right_width, right_height, right_bytes) = capture(&reftest, 1);
|
|
|
|
assert_eq!(left_width, right_width);
|
|
assert_eq!(left_height, right_height);
|
|
|
|
let left_all_white = left_bytes.iter().all(|&p| p == 255);
|
|
let right_all_white = right_bytes.iter().all(|&p| p == 255);
|
|
|
|
if left_all_white && right_all_white {
|
|
fail!("Both renderings are empty")
|
|
}
|
|
|
|
let pixels = left_bytes.iter().zip(right_bytes.iter()).map(|(&a, &b)| {
|
|
if a as i8 - b as i8 == 0 {
|
|
// White for correct
|
|
0xFF
|
|
} else {
|
|
// "1100" in the RGBA channel with an error for an incorrect value
|
|
// This results in some number of C0 and FFs, which is much more
|
|
// readable (and distinguishable) than the previous difference-wise
|
|
// scaling but does not require reconstructing the actual RGBA pixel.
|
|
0xC0
|
|
}
|
|
}).collect::<Vec<u8>>();
|
|
|
|
if pixels.iter().any(|&a| a < 255) {
|
|
let output_str = format!("/tmp/servo-reftest-{:06u}-diff.png", reftest.id);
|
|
let output = from_str::<Path>(output_str.as_slice()).unwrap();
|
|
|
|
let mut img = png::Image {
|
|
width: left_width,
|
|
height: left_height,
|
|
pixels: png::RGBA8(pixels),
|
|
};
|
|
let res = png::store_png(&mut img, &output);
|
|
assert!(res.is_ok());
|
|
|
|
match (reftest.kind, reftest.is_flaky) {
|
|
(Same, true) => println!("flaky test - rendering difference: {}", output_str),
|
|
(Same, false) => fail!("rendering difference: {}", output_str),
|
|
(Different, _) => {} // Result was different and that's what was expected
|
|
}
|
|
} else {
|
|
assert!(reftest.is_flaky || reftest.kind == Same);
|
|
}
|
|
}
|