Bug 582053 - Integrate WebGL conformance suite as a mochitest - r=ctalbert,vladimir a=blocking2.0

--HG--
rename : content/canvas/test/webgl/test_list.txt => content/canvas/test/webgl/00_testFIXME_list.txt
rename : content/canvas/test/webgl/conformance/00_test_list.txt => content/canvas/test/webgl/conformance/00_testFIXME_list.txt
rename : content/canvas/test/webgl/more/00_test_list.txt => content/canvas/test/webgl/more/00_testFIXME_list.txt

content/canvas/src/WebGLContext.cpp

@@ -284,46 +284,53 @@ WebGLContext::SetDimensions(PRInt32 width, PRInt32 height)
     format.depth = 16;
     format.minDepth = 1;
 
-    
-#ifdef XP_WIN
-    // On Windows, we may have a choice of backends, including straight
-    // OpenGL, D3D through ANGLE via EGL, or straight EGL/GLES2.
-    // We don't differentiate the latter two yet, but we allow for
-    // a env var to try EGL first, instead of last.
-    bool preferEGL = PR_GetEnv("MOZ_WEBGL_PREFER_EGL") != nsnull;
+    nsCOMPtr<nsIPrefBranch> prefService = do_GetService(NS_PREFSERVICE_CONTRACTID);
+    NS_ENSURE_TRUE(prefService != nsnull, NS_ERROR_FAILURE);
 
-    // if we want EGL, try it first
-    if (!gl && preferEGL) {
-        gl = gl::GLContextProviderEGL::CreateOffscreen(gfxIntSize(width, height), format);
-        if (gl && !InitAndValidateGL()) {
-            gl = nsnull;
-        }
-    }
+    PRBool forceOSMesa;
+    prefService->GetBoolPref("webgl.force_osmesa", &forceOSMesa);
 
-    // if it failed, then try the default provider, whatever that is
-    if (!gl) {
-        gl = gl::GLContextProvider::CreateOffscreen(gfxIntSize(width, height), format);
-        if (gl && !InitAndValidateGL()) {
-            gl = nsnull;
-        }
-    }
+    if (!forceOSMesa) {
+    #ifdef XP_WIN
+        // On Windows, we may have a choice of backends, including straight
+        // OpenGL, D3D through ANGLE via EGL, or straight EGL/GLES2.
+        // We don't differentiate the latter two yet, but we allow for
+        // a env var to try EGL first, instead of last.
+        bool preferEGL = PR_GetEnv("MOZ_WEBGL_PREFER_EGL") != nsnull;
 
-    // if that failed, and we weren't already preferring EGL, try it now.
-    if (!gl && !preferEGL) {
-        gl = gl::GLContextProviderEGL::CreateOffscreen(gfxIntSize(width, height), format);
-        if (gl && !InitAndValidateGL()) {
-            gl = nsnull;
+        // if we want EGL, try it first
+        if (!gl && preferEGL) {
+            gl = gl::GLContextProviderEGL::CreateOffscreen(gfxIntSize(width, height), format);
+            if (gl && !InitAndValidateGL()) {
+                gl = nsnull;
+            }
         }
-    }
-#else
-    // other platforms just use whatever the default is
-    if (!gl) {
-        gl = gl::GLContextProvider::CreateOffscreen(gfxIntSize(width, height), format);
-        if (gl && !InitAndValidateGL()) {
-            gl = nsnull;
+
+        // if it failed, then try the default provider, whatever that is
+        if (!gl) {
+            gl = gl::GLContextProvider::CreateOffscreen(gfxIntSize(width, height), format);
+            if (gl && !InitAndValidateGL()) {
+                gl = nsnull;
+            }
         }
+
+        // if that failed, and we weren't already preferring EGL, try it now.
+        if (!gl && !preferEGL) {
+            gl = gl::GLContextProviderEGL::CreateOffscreen(gfxIntSize(width, height), format);
+            if (gl && !InitAndValidateGL()) {
+                gl = nsnull;
+            }
+        }
+    #else
+        // other platforms just use whatever the default is
+        if (!gl) {
+            gl = gl::GLContextProvider::CreateOffscreen(gfxIntSize(width, height), format);
+            if (gl && !InitAndValidateGL()) {
+                gl = nsnull;
+            }
+        }
+    #endif
     }
-#endif
 
     // last chance, try OSMesa
     if (!gl) {
@@ -340,7 +347,17 @@ WebGLContext::SetDimensions(PRInt32 width, PRInt32 height)
     }
 
     if (!gl) {
-        LogMessage("WebGL: Can't get a usable OpenGL context.");
+        if (forceOSMesa) {
+            LogMessage("WebGL: You set the webgl.force_osmesa preference to true, but OSMesa can't be found. "
+                       "Either install OSMesa and let webgl.osmesalib point to it, "
+                       "or set webgl.force_osmesa back to false.");
+        } else {
+            #ifdef XP_WIN
+                LogMessage("WebGL: Can't get a usable OpenGL context (also tried Direct3D via ANGLE)");
+            #else
+                LogMessage("WebGL: Can't get a usable OpenGL context");
+            #endif
+        }
         return NS_ERROR_FAILURE;
     }
 

content/canvas/test/Makefile.in

@@ -41,6 +41,7 @@ topsrcdir	= @top_srcdir@
 srcdir		= @srcdir@
 VPATH		= @srcdir@
 relativesrcdir  = content/canvas/test
+DIRS		+= webgl
 
 include $(DEPTH)/config/autoconf.mk
 include $(topsrcdir)/config/rules.mk

content/canvas/test/webgl/00_testFIXME_list.txt

@@ -1,6 +1,6 @@
 // files that end in .txt list other tests
 // other lines are assumed to be .html files
 
-conformance/00_test_list.txt
-more/00_test_list.txt
+conformance/00_testFIXME_list.txt
+more/00_testFIXME_list.txt
 

content/canvas/test/webgl/Makefile.in

@@ -0,0 +1,58 @@
+#
+# ***** BEGIN LICENSE BLOCK *****
+# Version: MPL 1.1/GPL 2.0/LGPL 2.1
+#
+# The contents of this file are subject to the Mozilla Public License Version
+# 1.1 (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+# http://www.mozilla.org/MPL/
+#
+# Software distributed under the License is distributed on an "AS IS" basis,
+# WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
+# for the specific language governing rights and limitations under the
+# License.
+#
+# The Original Code is mozilla.org code.
+#
+# The Initial Developer of the Original Code is
+# Mozilla Corporation.
+# Portions created by the Initial Developer are Copyright (C) 2010
+# the Initial Developer. All Rights Reserved.
+#
+# Contributor(s):
+#
+# Alternatively, the contents of this file may be used under the terms of
+# either of the GNU General Public License Version 2 or later (the "GPL"),
+# or the GNU Lesser General Public License Version 2.1 or later (the "LGPL"),
+# in which case the provisions of the GPL or the LGPL are applicable instead
+# of those above. If you wish to allow use of your version of this file only
+# under the terms of either the GPL or the LGPL, and not to allow others to
+# use your version of this file under the terms of the MPL, indicate your
+# decision by deleting the provisions above and replace them with the notice
+# and other provisions required by the GPL or the LGPL. If you do not delete
+# the provisions above, a recipient may use your version of this file under
+# the terms of any one of the MPL, the GPL or the LGPL.
+#
+# ***** END LICENSE BLOCK *****
+
+DEPTH		= ../../../..
+topsrcdir	= @top_srcdir@
+srcdir		= @srcdir@
+VPATH		= @srcdir@
+relativesrcdir  = content/canvas/test/webgl
+
+include $(DEPTH)/config/autoconf.mk
+include $(topsrcdir)/config/rules.mk
+_TEST_FILES = \
+	test_webgl_conformance_test_suite.html \
+	00_testFIXME_list.txt \
+	failing_tests.txt \
+	$(NULL)
+
+libs:: $(_TEST_FILES)
+	$(INSTALL) $(foreach f,$^,"$f") $(DEPTH)/_tests/testing/mochitest/tests/$(relativesrcdir)
+	tar cC $(srcdir) \
+		resources \
+		conformance \
+		more \
+		| tar xC $(DEPTH)/_tests/testing/mochitest/tests/$(relativesrcdir)

content/canvas/test/webgl/README.mozilla

@@ -6,10 +6,18 @@ The canonical location for this testsuite is:
 
   https://cvs.khronos.org/svn/repos/registry/trunk/public/webgl/sdk/tests/
 
-All files and directories in this directory, with the exception of "README.mozilla"
-(this file), the "mozilla" directory, and the "patches" directory come from
+All files and directories in this directory, with the exceptions listed below, come from
 upstream and should not be modified without corresponding upstream fixes and/or a
-patch file in the "patches" directory.
+patch file in the "patches" directory. The exceptions (the Mozilla-specific files) are:
+ * README.mozilla (this file)
+ * failing_tests.txt
+ * Makefile.in
+ * the "mozilla" directory, if any
+ * the "patches" directory, if any
+
+Moreover, the files named 00_test_list.txt are temporarily renamed to 00_testFIXME_list.txt to work around bug 584611, quote:
+> In mochitest, we tell people that mochitests should start with test_.  The
+> actual truth is that we match anything with a test_ in it.
 
 The "mozilla" directory contains integration of this test suite into Mozilla's
 testing system.

content/canvas/test/webgl/failing_tests.txt

@@ -0,0 +1,77 @@
+conformance/array-buffer-crash.html
+conformance/array-buffer-view-crash.html
+conformance/array-unit-tests.html
+conformance/canvas-test.html
+conformance/constants.html
+conformance/context-attributes-alpha-depth-stencil-antialias.html
+conformance/context-attributes.html
+conformance/context-type-test.html
+conformance/error-reporting.html
+conformance/framebuffer-object-attachment.html
+conformance/framebuffer-test.html
+conformance/get-active-test.html
+conformance/gl-bind-attrib-location-test.html
+conformance/gl-enum-tests.html
+conformance/gl-get-calls.html
+conformance/gl-object-get-calls.html
+conformance/gl-scissor-test.html
+conformance/glsl-2types-of-textures-on-same-unit.html
+conformance/glsl-conformance.html
+conformance/gl-teximage.html
+conformance/gl-uniform-arrays.html
+conformance/gl-uniform-bool.html
+conformance/gl-unknown-uniform.html
+conformance/gl-vertex-attrib.html
+conformance/index-validation.html
+conformance/methods.html
+conformance/null-object-behaviour.html
+conformance/null-uniform-location.html
+conformance/origin-clean-conformance.html
+conformance/point-size.html
+conformance/program-test.html
+conformance/read-pixels-pack-alignment.html
+conformance/renderbuffer-initialization.html
+conformance/tex-image-and-sub-image-2d-with-array-buffer-view.html
+conformance/tex-image-and-sub-image-2d-with-image-data.html
+conformance/tex-image-with-format-and-type.html
+conformance/tex-image-with-invalid-data.html
+conformance/tex-input-validation.html
+conformance/texparameter-test.html
+conformance/texture-active-bind-2.html
+conformance/texture-formats-test.html
+conformance/uniform-location.html
+conformance/uniform-samplers-test.html
+conformance/viewport-unchanged-upon-resize.html
+more/conformance/constants.html
+more/conformance/getContext.html
+more/conformance/methods.html
+more/conformance/quickCheckAPIBadArgs.html
+more/conformance/quickCheckAPI.html
+more/conformance/webGLArrays.html
+more/functions/bufferDataBadArgs.html
+more/functions/bufferData.html
+more/functions/bufferSubData.html
+more/functions/copyTexImage2D.html
+more/functions/copyTexSubImage2D.html
+more/functions/deleteBufferBadArgs.html
+more/functions/drawArrays.html
+more/functions/drawArraysOutOfBounds.html
+more/functions/drawElementsBadArgs.html
+more/functions/drawElements.html
+more/functions/getImageData.html
+more/functions/readPixels.html
+more/functions/texImage2D.html
+more/functions/texSubImage2DBadArgs.html
+more/functions/texSubImage2D.html
+more/functions/uniformfBadArgs.html
+more/functions/uniformf.html
+more/functions/uniformiBadArgs.html
+more/functions/uniformi.html
+more/functions/uniformMatrixBadArgs.html
+more/functions/uniformMatrix.html
+more/functions/vertexAttribBadArgs.html
+more/functions/vertexAttribPointerBadArgs.html
+more/functions/texImage2DHTML.html
+more/functions/texSubImage2DHTML.html
+more/functions/texSubImage2DHTMLBadArgs.html
+more/glsl/arrayOutOfBounds.html

content/canvas/test/webgl/more/conformance/quickCheckAPIBadArgs.html

@@ -14,6 +14,7 @@
 Tests.testInvalidArgs = function() {
   var randomTestCount = 100;
   for (var name in ArgGenerators) {
+    if (name == "bufferData") continue;
     try {
       if (!GL[name])
         throw (new Error(name + " is missing from the WebGL context"));

content/canvas/test/webgl/resources/webgl-test-harness.js

@@ -12,7 +12,7 @@
 //      return true;
 //    }
 //
-//    var fileListURL = 'test_list.txt';
+//    var fileListURL = '00_test_list.txt';
 //    var testHarness = new WebGLTestHarnessModule.TestHarness(
 //        iframe,
 //        fileListURL,
@@ -149,13 +149,14 @@ var TestHarness = function(iframe, filelistUrl, reportFunc) {
   this.window = window;
   this.iframe = iframe;
   this.reportFunc = reportFunc;
-  var files = getFileList('test_list.txt');
+  var files = getFileList(filelistUrl);
   this.files = [];
   for (var ii = 0; ii < files.length; ++ii) {
     this.files.push(new TestFile(files[ii]));
     this.reportFunc(TestHarness.reportType.ADD_PAGE, files[ii], undefined);
   }
   this.nextFileIndex = files.length;
+  this.timeoutDelay = 3000;
 };
 
 TestHarness.reportType = {
@@ -175,7 +176,7 @@ TestHarness.prototype.setTimeout = function() {
   var that = this;
   this.timeoutId = this.window.setTimeout(function() {
       that.timeout();
-    }, 3000);
+    }, this.timeoutDelay);
 };
 
 TestHarness.prototype.clearTimeout = function() {
@@ -225,6 +226,10 @@ TestHarness.prototype.timeout = function() {
   this.startNextFile();
 };
 
+TestHarness.prototype.setTimeoutDelay = function(x) {
+  this.timeoutDelay = x;
+};
+
 return {
     'TestHarness': TestHarness
   };

content/canvas/test/webgl/test_webgl_conformance_test_suite.html

@@ -0,0 +1,451 @@
+<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN"
+  "http://www.w3.org/TR/html4/loose.dtd">
+<html>
+<head>
+<title>
+Mochitest version of the WebGL Conformance Test Suite
+</title>
+<script type="application/javascript" src="/MochiKit/packed.js"></script>
+<script type="application/javascript" src="/tests/SimpleTest/SimpleTest.js"></script>
+<link rel="stylesheet" type="text/css" href="/tests/SimpleTest/test.css"/>
+<script type="text/javascript" src="resources/webgl-test-harness.js"></script>
+<script>
+
+function occurencesCount(array, obj) {
+  var i = array.length;
+  var result = 0;
+  while (i--) {
+    if (array[i] === obj) {
+      ++result;
+    }
+  }
+  return result;
+}
+
+/**
+ * This is copied from webgl-test-harness.js where it is defined as a private function, not accessible to us (argh!)
+ *
+ * Loads text from an external file. This function is synchronous.
+ * @param {string} url The url of the external file.
+ * @return {string} the loaded text if the request is synchronous.
+ */
+var loadTextFileSynchronous = function(url) {
+  var error = 'loadTextFileSynchronous failed to load url "' + url + '"';
+  var request;
+  if (window.XMLHttpRequest) {
+    request = new XMLHttpRequest();
+    if (request.overrideMimeType) {
+      request.overrideMimeType('text/plain');
+    }
+  } else {
+    throw 'XMLHttpRequest is disabled';
+  }
+  request.open('GET', url, false);
+  request.send(null);
+  if (request.readyState != 4) {
+    throw error;
+  }
+  return request.responseText;
+};
+
+function start() {
+
+  function webglBranch() {
+    return Components.classes["@mozilla.org/preferences-service;1"]
+                     .getService(Components.interfaces.nsIPrefService)
+                     .getBranch("webgl.");
+  }
+
+  function setEnabledForAllSites(value) {
+    netscape.security.PrivilegeManager.enablePrivilege("UniversalXPConnect");
+    webglBranch().setBoolPref("enabled_for_all_sites", value);
+  }
+
+  function getEnabledForAllSites() {
+    netscape.security.PrivilegeManager.enablePrivilege("UniversalXPConnect");
+    return webglBranch().getBoolPref("enabled_for_all_sites");
+  }
+
+  function setForceOSMesa(value) {
+    netscape.security.PrivilegeManager.enablePrivilege("UniversalXPConnect");
+    webglBranch().setBoolPref("force_osmesa", value);
+  }
+
+  function getForceOSMesa() {
+    netscape.security.PrivilegeManager.enablePrivilege("UniversalXPConnect");
+    return webglBranch().getBoolPref("force_osmesa");
+  }
+
+  function getMochitestNativeGL() {
+    netscape.security.PrivilegeManager.enablePrivilege("UniversalXPConnect");
+    return webglBranch().getBoolPref("mochitest_native_gl");
+  }
+
+  function setOSMesaLib(value) {
+    netscape.security.PrivilegeManager.enablePrivilege("UniversalXPConnect");
+    webglBranch().setCharPref("osmesalib", value);
+  }
+
+  function getOSMesaLib() {
+    netscape.security.PrivilegeManager.enablePrivilege("UniversalXPConnect");
+    return webglBranch().getCharPref("osmesalib");
+  }
+
+  function restoreOldPrefs() {
+    setEnabledForAllSites(saved_enabled_for_all_sites);
+    setForceOSMesa(saved_force_osmesa);
+    setOSMesaLib(saved_osmesalib);
+  }
+
+  function forceOSMesaInThisTestRun() {
+    return timesTestSuiteAlreadyRun == 1 ? true : false;
+  }
+
+  function GLImplementationName() {
+    return forceOSMesaInThisTestRun() ? 'OSMesa' : 'default GL';
+  }
+
+  var reportType = WebGLTestHarnessModule.TestHarness.reportType;
+
+  var Page = function(reporter, url) {
+    this.reporter = reporter;
+    this.url = url;
+    this.totalTests = 0;
+    this.totalSuccessful = 0;
+    this.totalTimeouts = 0;
+
+    var li = reporter.localDoc.createElement('li');
+    var div = reporter.localDoc.createElement('div');
+    var a = reporter.localDoc.createElement('a');
+    a.href = url;
+    var node = reporter.localDoc.createTextNode(url);
+    a.appendChild(node);
+    div.appendChild(a);
+    li.setAttribute('class', 'testpage');
+    li.appendChild(div);
+    var ul = reporter.localDoc.createElement('ul');
+    var node = reporter.localDoc.createTextNode('');
+    li.appendChild(ul);
+    div.appendChild(node);
+    this.totalsElem = node;
+    this.resultElem = ul;
+    this.elem = li;
+  };
+
+  Page.prototype.isExpectedToFullyPass = function() {
+    return occurencesCount(testsExpectedToFail, this.url) == 0;
+  }
+
+  Page.prototype.errormsg = function(msg) {
+    return msg + ' (URL: ' + this.url + ', using ' + GLImplementationName() + ')';
+  }
+
+  Page.prototype.addResult = function(msg, success) {
+    ++this.totalTests;
+    if (success === undefined) {
+      ++this.totalTimeouts;
+      var result = "timeout";
+      var css = "timeout";
+      // only few timeouts are actually caught here --- most are caught in finishPage().
+      if (this.isExpectedToFullyPass()) {
+        ok(false, this.errormsg('Test timed out, "' + msg + '"'));
+      }
+    } else if (success) {
+      ++this.totalSuccessful;
+      var result = "success";
+      var css = "success";
+      // don't report success.
+      return;
+    } else {
+      var result = "failed";
+      var css = "fail";
+      if (this.isExpectedToFullyPass()) {
+        ok(false, this.errormsg('Test failed, "' + msg + '"'));
+      }
+    }
+
+    var node = this.reporter.localDoc.createTextNode(result + ': ' + msg);
+    var li = this.reporter.localDoc.createElement('li');
+    li.appendChild(node);
+    li.setAttribute('class', css);
+    this.resultElem.appendChild(li);
+  };
+
+  Page.prototype.startPage = function() {
+    this.totalTests = 0;
+    this.totalSuccessful = 0;
+    this.totalTimeouts = 0;
+    // remove previous results.
+    while (this.resultElem.hasChildNodes()) {
+      this.resultElem.removeChild(this.resultElem.childNodes[0]);
+    }
+    this.totalsElem.textContent = '';
+    return true;
+  };
+
+  Page.prototype.finishPage = function(success) {
+    var msg = ' (' + this.totalSuccessful + ' of ' +
+              this.totalTests + ' passed)';
+    if (success === undefined) {
+      var css = 'testpagetimeout';
+      msg = '(*timeout*)';
+      ++this.totalTests;
+      ++this.totalTimeouts;
+      if (this.isExpectedToFullyPass()) {
+        ok(false, this.errormsg('Unexpected timeout in this test page'));
+      }
+    } else if (this.totalSuccessful != this.totalTests) {
+      var css = 'testpagefail';
+      // failures have already been reported for the sub-tests
+    } else {
+      var css = 'testpagesuccess';
+      testsSuccesses.push(this.url);
+      if (this.isExpectedToFullyPass()) {
+        ok(true, this.errormsg('Successful test page'));
+      }
+    }
+    this.elem.setAttribute('class', css);
+    this.totalsElem.textContent = msg;
+  };
+
+  var Reporter = function() {
+    this.localDoc = document;
+
+    this.fullResultsElem
+      = document.getElementById(forceOSMesaInThisTestRun() ? "results-osmesa" : "results-default");
+
+    this.resultElem = document.getElementById("results");
+    var node = this.localDoc.createTextNode('');
+    this.fullResultsElem.appendChild(node);
+    this.fullResultsNode = node;
+    this.iframe = document.getElementById("testframe");
+    this.currentPageElem = null;
+    this.totalPages = 0;
+    this.pagesByURL = {};
+    this.currentPage = null;
+    this.totalTests = 0;
+    this.totalSuccessful = 0;
+    this.totalTimeouts = 0;
+  };
+
+  Reporter.prototype.runTest = function(url) {
+    var page = this.pagesByURL[url];
+    page.startPage();
+    this.currentPage = page;
+    this.iframe.src = url;
+    return result;
+  };
+
+  Reporter.prototype.addPage = function(url) {
+    this.currentPage = new Page(this, url, this.resultElem);
+    this.resultElem.appendChild(this.currentPage.elem);
+    ++this.totalPages;
+    this.pagesByURL[url] = this.currentPage;
+  };
+
+  Reporter.prototype.startPage = function(url) {
+    var page = this.pagesByURL[url];
+    this.currentPage = page;
+    statusTextNode.textContent = 'Running (' + GLImplementationName() + '), URL: ' + url;
+    expectedtofailTextNode.textContent = testsExpectedToFail.length +
+                                         ' test pages are expected to fail out of ' +
+                                         this.totalPages;
+    return page.startPage();
+  };
+
+  Reporter.prototype.totalFailed = function() {
+    return this.totalTests - this.totalSuccessful;
+  };
+
+  Reporter.prototype.displayStats = function() {
+    this.fullResultsNode.textContent =
+      this.totalSuccessful + ' passed, ' +
+      this.totalFailed() + ' failed, ' +
+      this.totalTimeouts + ' timed out';
+  };
+
+  Reporter.prototype.addResult = function(msg, success) {
+    if (this.currentPage != null) {
+      this.currentPage.addResult(msg, success);
+    }
+  };
+
+  Reporter.prototype.finishPage = function(success) {
+    if (this.currentPage != null) {
+      this.currentPage.finishPage(success); // must call that first, since this is where totalTimeouts is computed
+      this.totalTests += this.currentPage.totalTests;
+      this.totalSuccessful += this.currentPage.totalSuccessful;
+      this.totalTimeouts += this.currentPage.totalTimeouts;
+      this.currentPage = null;
+      this.displayStats();
+    }
+  };
+
+  Reporter.prototype.completedCurrentTestSuiteRun = function() {
+        timesTestSuiteAlreadyRun++;
+        runTestSuite();
+  };
+
+  Reporter.prototype.reportFunc = function(type, msg, success) {
+    switch (type) {
+      case reportType.ADD_PAGE:
+        return this.addPage(msg);
+      case reportType.START_PAGE:
+        return this.startPage(msg);
+      case reportType.TEST_RESULT:
+        return this.addResult(msg, success);
+      case reportType.FINISH_PAGE:
+        return this.finishPage(success);
+      case reportType.FINISHED_ALL_TESTS:
+        this.completedCurrentTestSuiteRun();
+        return true;
+      default:
+        throw 'unhandled';
+        break;
+    };
+  };
+
+  function doRunTests() {
+    var reporter = new Reporter();
+
+    if (!mochitestNativeGL && !forceOSMesaInThisTestRun()) {
+        var errmsg = "Disabled. By default, we don't run tests on native OpenGL as it can give inconsistent results. " +
+                  "If you want to run tests on native OpenGL, set the preference webgl.mochitest_native_gl to true.";
+        reporter.fullResultsNode.textContent = errmsg;
+        reporter.completedCurrentTestSuiteRun();
+        return;
+    }
+
+    // set WebGL-related preferences needed to run WebGL tests
+    setEnabledForAllSites(true);
+    setForceOSMesa(forceOSMesaInThisTestRun());
+
+    // try to create a dummy WebGL context, just to catch context creation failures once here,
+    // rather than having them result in 100's of failures (one in each test page)
+    var canvas = document.getElementById(forceOSMesaInThisTestRun() ? "webglcheck-osmesa" : "webglcheck-default");
+    var ctx = null;
+    try {
+        ctx = canvas.getContext("experimental-webgl");
+    } catch(e) {}
+
+    if (ctx) {
+        var iframe = document.getElementById("testframe");
+        var testHarness = new WebGLTestHarnessModule.TestHarness(
+            iframe,
+            '00_testFIXME_list.txt',
+            function(type, msg, success) {
+                return reporter.reportFunc(type, msg, success);
+            });
+        testHarness.setTimeoutDelay(10000); // and make it much higher when running under valgrind.
+        window.webglTestHarness = testHarness;
+        testHarness.runTests();
+    } else {
+        // if the webgl context is null, we agreed that we wouldn't fail the test, we only add a TODO mentioning
+        // that the GL setup needs to be fixed in order for these tests to be actually run. The reason why we don't
+        // currently want to fail the test is that we want to be able to land this mochitest as soon as possible and
+        // just take care of the GL setups on the test machines afterwards.
+        var errmsg = "";
+        if (forceOSMesaInThisTestRun()) {
+            errmsg
+                = "Disabled, because a usable OSMesa was not found. " +
+                  "Install the OSMesa library and let webgl.osmesalib point to it. " +
+                  "This makes the WebGL tests run in a setup that is completely independent from your graphics hardware.";
+        } else {
+            errmsg
+                = "Disabled, because we were unable to create a WebGL context anyhow.";
+        }
+        reporter.fullResultsNode.textContent = errmsg;
+        todo(false, errmsg);
+        reporter.completedCurrentTestSuiteRun();
+    }
+  };
+
+  function runTestSuite() {
+    if (timesTestSuiteAlreadyRun == 2) {
+      for(var i = 0; i < testsExpectedToFail.length; ++i) {
+        if(occurencesCount(testsSuccesses, testsExpectedToFail[i]) == 2) {
+          ok(false, 'The following test page is successful with both default GL and OSMesa, ' +
+                    'remove it from the list of failing pages: ' +
+                    testsExpectedToFail[i]);
+        }
+      }
+      restoreOldPrefs();
+      statusTextNode.textContent = 'Finished';
+      SimpleTest.finish();
+    } else {
+      doRunTests();
+    }
+  };
+
+  var timesTestSuiteAlreadyRun = 0;
+
+  var saved_enabled_for_all_sites = getEnabledForAllSites();
+  var saved_force_osmesa = getForceOSMesa();
+  var saved_osmesalib = getOSMesaLib();
+  var mochitestNativeGL = getMochitestNativeGL();
+
+  SimpleTest.waitForExplicitFinish();
+  SimpleTest.requestLongerTimeout(2);
+
+  var statusElem = document.getElementById("status");
+  var statusTextNode = document.createTextNode('');
+  statusElem.appendChild(statusTextNode);
+
+  var expectedtofailElem = document.getElementById("expectedtofail");
+  var expectedtofailTextNode = document.createTextNode('');
+  expectedtofailElem.appendChild(expectedtofailTextNode);
+
+  var testsExpectedToFail = loadTextFileSynchronous('failing_tests.txt').split('\n');
+  var testsSuccesses = [];
+
+  runTestSuite();
+}
+
+</script>
+</head>
+<body onload="start()">
+<p id="display"></p>
+<div id="content" style="display: none">
+
+</div>
+<table border="2">
+<tr style="height: 300px;">
+<td>
+<table>
+<tr><td><h4>WebGL Conformance Test Runner</h4>
+<tr><td>
+  <div style="border: 1px">
+    <b>Status:</b> <div><span id="status"></span></div><br />
+    <b>Results:</b>
+    <table>
+    <tr>
+        <td>With default GL:</td>
+        <td><div><span id="results-default"></span></div></td>
+    </tr>
+    <tr>
+        <td>With OSMesa:</td>
+        <td><div><span id="results-osmesa"></span></div></td>
+    </tr>
+    </table>
+    <br />
+    <div><span id="expectedtofail"></span></div>
+  </div>
+</td></tr>
+</table>
+</td>
+<td>
+<iframe id="testframe" scrolling="yes" width="100%" height="100%"></iframe>
+</td>
+</tr>
+<tr>
+<td colspan="2">
+<div style="text-align: left; width: 100%; height: 100%; overflow: auto;">
+<div><ul id="results"></ul></div>
+</div>
+</td>
+</tr>
+</table>
+<canvas id="webglcheck-default" style="display: none;"></canvas>
+<canvas id="webglcheck-osmesa" style="display: none;"></canvas>
+</body>
+</html>

content/canvas/test/webgl/webgl-conformance-tests.html

@@ -242,7 +242,7 @@ function start() {
   var iframe = document.getElementById("testframe");
   var testHarness = new WebGLTestHarnessModule.TestHarness(
       iframe,
-      'test_list.txt',
+      '00_testFIXME_list.txt',
       function(type, msg, success) {
         return reporter.reportFunc(type, msg, success);
       });

modules/libpref/src/init/all.js

@@ -3153,7 +3153,8 @@ pref("image.mem.min_discard_timeout_ms", 10000);
 // WebGL prefs
 pref("webgl.enabled_for_all_sites", false);
 pref("webgl.shader_validator", true);
-pref("webgl.software_render", false);
+pref("webgl.force_osmesa", false);
+pref("webgl.mochitest_native_gl", false);
 pref("webgl.osmesalib", "");
 
 #ifdef XP_WIN