mirror of
https://github.com/mozilla/gecko-dev.git
synced 2024-11-24 21:31:04 +00:00
Bug 1091284 - Remove systemMemory, environment from automationutils. r=jgriffin
--HG-- extra : commitid : BHwyu2fUyFY extra : rebase_source : 152496df202df4355268cb54987a587eddbe6ddc
This commit is contained in:
Julien Pagès
parent
f24dd15190
commit
3af2d87e17
@ -18,8 +18,6 @@ import mozinfo
|
||||
__all__ = [
|
||||
"dumpLeakLog",
|
||||
"processLeakLog",
|
||||
'systemMemory',
|
||||
'environment',
|
||||
'dumpScreen',
|
||||
"setAutomationLog",
|
||||
]
|
||||
@ -282,136 +280,6 @@ def processLeakLog(leakLogFile, options):
|
||||
processSingleLeakFile(thisFile, processType, leakThreshold,
|
||||
processType in ignoreMissingLeaks)
|
||||
|
||||
def systemMemory():
|
||||
"""
|
||||
Returns total system memory in kilobytes.
|
||||
Works only on unix-like platforms where `free` is in the path.
|
||||
"""
|
||||
return int(os.popen("free").readlines()[1].split()[1])
|
||||
|
||||
def environment(xrePath, env=None, crashreporter=True, debugger=False, dmdPath=None, lsanPath=None):
|
||||
"""populate OS environment variables for mochitest"""
|
||||
|
||||
env = os.environ.copy() if env is None else env
|
||||
|
||||
assert os.path.isabs(xrePath)
|
||||
|
||||
if mozinfo.isMac:
|
||||
ldLibraryPath = os.path.join(os.path.dirname(xrePath), "MacOS")
|
||||
else:
|
||||
ldLibraryPath = xrePath
|
||||
|
||||
envVar = None
|
||||
dmdLibrary = None
|
||||
preloadEnvVar = None
|
||||
if 'toolkit' in mozinfo.info and mozinfo.info['toolkit'] == "gonk":
|
||||
# Skip all of this, it's only valid for the host.
|
||||
pass
|
||||
elif mozinfo.isUnix:
|
||||
envVar = "LD_LIBRARY_PATH"
|
||||
env['MOZILLA_FIVE_HOME'] = xrePath
|
||||
dmdLibrary = "libdmd.so"
|
||||
preloadEnvVar = "LD_PRELOAD"
|
||||
elif mozinfo.isMac:
|
||||
envVar = "DYLD_LIBRARY_PATH"
|
||||
dmdLibrary = "libdmd.dylib"
|
||||
preloadEnvVar = "DYLD_INSERT_LIBRARIES"
|
||||
elif mozinfo.isWin:
|
||||
envVar = "PATH"
|
||||
dmdLibrary = "dmd.dll"
|
||||
preloadEnvVar = "MOZ_REPLACE_MALLOC_LIB"
|
||||
if envVar:
|
||||
envValue = ((env.get(envVar), str(ldLibraryPath))
|
||||
if mozinfo.isWin
|
||||
else (ldLibraryPath, dmdPath, env.get(envVar)))
|
||||
env[envVar] = os.path.pathsep.join([path for path in envValue if path])
|
||||
|
||||
if dmdPath and dmdLibrary and preloadEnvVar:
|
||||
env[preloadEnvVar] = os.path.join(dmdPath, dmdLibrary)
|
||||
|
||||
# crashreporter
|
||||
env['GNOME_DISABLE_CRASH_DIALOG'] = '1'
|
||||
env['XRE_NO_WINDOWS_CRASH_DIALOG'] = '1'
|
||||
|
||||
if crashreporter and not debugger:
|
||||
env['MOZ_CRASHREPORTER_NO_REPORT'] = '1'
|
||||
env['MOZ_CRASHREPORTER'] = '1'
|
||||
else:
|
||||
env['MOZ_CRASHREPORTER_DISABLE'] = '1'
|
||||
|
||||
# Crash on non-local network connections by default.
|
||||
# MOZ_DISABLE_NONLOCAL_CONNECTIONS can be set to "0" to temporarily
|
||||
# enable non-local connections for the purposes of local testing. Don't
|
||||
# override the user's choice here. See bug 1049688.
|
||||
env.setdefault('MOZ_DISABLE_NONLOCAL_CONNECTIONS', '1')
|
||||
|
||||
# Set WebRTC logging in case it is not set yet
|
||||
env.setdefault('NSPR_LOG_MODULES', 'signaling:3,mtransport:4,datachannel:4,jsep:4,MediaPipelineFactory:4')
|
||||
env.setdefault('R_LOG_LEVEL', '6')
|
||||
env.setdefault('R_LOG_DESTINATION', 'stderr')
|
||||
env.setdefault('R_LOG_VERBOSE', '1')
|
||||
|
||||
# ASan specific environment stuff
|
||||
asan = bool(mozinfo.info.get("asan"))
|
||||
if asan and (mozinfo.isLinux or mozinfo.isMac):
|
||||
try:
|
||||
# Symbolizer support
|
||||
llvmsym = os.path.join(xrePath, "llvm-symbolizer")
|
||||
if os.path.isfile(llvmsym):
|
||||
env["ASAN_SYMBOLIZER_PATH"] = llvmsym
|
||||
log.info("INFO | runtests.py | ASan using symbolizer at %s" % llvmsym)
|
||||
else:
|
||||
log.info("TEST-UNEXPECTED-FAIL | runtests.py | Failed to find ASan symbolizer at %s" % llvmsym)
|
||||
|
||||
totalMemory = systemMemory()
|
||||
|
||||
# Only 4 GB RAM or less available? Use custom ASan options to reduce
|
||||
# the amount of resources required to do the tests. Standard options
|
||||
# will otherwise lead to OOM conditions on the current test slaves.
|
||||
message = "INFO | runtests.py | ASan running in %s configuration"
|
||||
asanOptions = []
|
||||
if totalMemory <= 1024 * 1024 * 4:
|
||||
message = message % 'low-memory'
|
||||
asanOptions = ['quarantine_size=50331648', 'malloc_context_size=5']
|
||||
else:
|
||||
message = message % 'default memory'
|
||||
|
||||
if lsanPath:
|
||||
log.info("LSan enabled.")
|
||||
asanOptions.append('detect_leaks=1')
|
||||
lsanOptions = ["exitcode=0"]
|
||||
suppressionsFile = os.path.join(lsanPath, 'lsan_suppressions.txt')
|
||||
if os.path.exists(suppressionsFile):
|
||||
log.info("LSan using suppression file " + suppressionsFile)
|
||||
lsanOptions.append("suppressions=" + suppressionsFile)
|
||||
else:
|
||||
log.info("WARNING | runtests.py | LSan suppressions file does not exist! " + suppressionsFile)
|
||||
env["LSAN_OPTIONS"] = ':'.join(lsanOptions)
|
||||
# Run shutdown GCs and CCs to avoid spurious leaks.
|
||||
env['MOZ_CC_RUN_DURING_SHUTDOWN'] = '1'
|
||||
|
||||
if len(asanOptions):
|
||||
env['ASAN_OPTIONS'] = ':'.join(asanOptions)
|
||||
|
||||
except OSError,err:
|
||||
log.info("Failed determine available memory, disabling ASan low-memory configuration: %s" % err.strerror)
|
||||
except:
|
||||
log.info("Failed determine available memory, disabling ASan low-memory configuration")
|
||||
else:
|
||||
log.info(message)
|
||||
|
||||
tsan = bool(mozinfo.info.get("tsan"))
|
||||
if tsan and mozinfo.isLinux:
|
||||
# Symbolizer support.
|
||||
llvmsym = os.path.join(xrePath, "llvm-symbolizer")
|
||||
if os.path.isfile(llvmsym):
|
||||
env["TSAN_OPTIONS"] = "external_symbolizer_path=%s" % llvmsym
|
||||
log.info("INFO | runtests.py | TSan using symbolizer at %s" % llvmsym)
|
||||
else:
|
||||
log.info("TEST-UNEXPECTED-FAIL | runtests.py | Failed to find TSan symbolizer at %s" % llvmsym)
|
||||
|
||||
return env
|
||||
|
||||
def dumpScreen(utilityPath):
|
||||
"""dumps a screenshot of the entire screen to a directory specified by
|
||||
the MOZ_UPLOAD_DIR environment variable"""
|
||||
|
||||
@ -23,7 +23,6 @@ sys.path.insert(0, SCRIPT_DIRECTORY)
|
||||
|
||||
from automationutils import (
|
||||
dumpScreen,
|
||||
environment,
|
||||
printstatus,
|
||||
processLeakLog
|
||||
)
|
||||
@ -33,6 +32,7 @@ import mozinfo
|
||||
import mozprocess
|
||||
import mozprofile
|
||||
import mozrunner
|
||||
from mozrunner.utils import test_environment
|
||||
|
||||
here = os.path.abspath(os.path.dirname(__file__))
|
||||
|
||||
@ -273,7 +273,8 @@ class RefTest(object):
|
||||
return profile
|
||||
|
||||
def environment(self, **kwargs):
|
||||
return environment(**kwargs)
|
||||
kwargs['log'] = log
|
||||
return test_environment(**kwargs)
|
||||
|
||||
def buildBrowserEnv(self, options, profileDir):
|
||||
browserEnv = self.environment(xrePath = options.xrePath, debugger=options.debugger)
|
||||
|
||||
@ -35,7 +35,6 @@ import zipfile
|
||||
import bisection
|
||||
|
||||
from automationutils import (
|
||||
environment,
|
||||
processLeakLog,
|
||||
dumpScreen,
|
||||
printstatus,
|
||||
@ -59,6 +58,7 @@ from mozprofile.permissions import ServerLocations
|
||||
from urllib import quote_plus as encodeURIComponent
|
||||
from mozlog.structured.formatters import TbplFormatter
|
||||
from mozlog.structured import commandline
|
||||
from mozrunner.utils import test_environment
|
||||
|
||||
here = os.path.abspath(os.path.dirname(__file__))
|
||||
|
||||
@ -359,7 +359,7 @@ class MochitestServer(object):
|
||||
"Run the Mochitest server, returning the process ID of the server."
|
||||
|
||||
# get testing environment
|
||||
env = environment(xrePath=self._xrePath)
|
||||
env = test_environment(xrePath=self._xrePath, log=self._log)
|
||||
env["XPCOM_DEBUG_BREAK"] = "warn"
|
||||
env["LD_LIBRARY_PATH"] = self._xrePath
|
||||
|
||||
@ -1064,7 +1064,7 @@ class SSLTunnel:
|
||||
ssltunnel)
|
||||
exit(1)
|
||||
|
||||
env = environment(xrePath=self.xrePath)
|
||||
env = test_environment(xrePath=self.xrePath, log=self.log)
|
||||
env["LD_LIBRARY_PATH"] = self.xrePath
|
||||
self.process = mozprocess.ProcessHandler([ssltunnel, self.configFile],
|
||||
env=env)
|
||||
@ -1234,9 +1234,6 @@ class Mochitest(MochitestUtilsMixin):
|
||||
def __init__(self, logger_options):
|
||||
super(Mochitest, self).__init__(logger_options)
|
||||
|
||||
# environment function for browserEnv
|
||||
self.environment = environment
|
||||
|
||||
# Max time in seconds to wait for server startup before tests will fail -- if
|
||||
# this seems big, it's mostly for debug machines where cold startup
|
||||
# (particularly after a build) takes forever.
|
||||
@ -1254,6 +1251,10 @@ class Mochitest(MochitestUtilsMixin):
|
||||
self.expectedError = {}
|
||||
self.result = {}
|
||||
|
||||
def environment(self, **kwargs):
|
||||
kwargs['log'] = self.log
|
||||
return test_environment(**kwargs)
|
||||
|
||||
def extraPrefs(self, extraPrefs):
|
||||
"""interpolate extra preferences from option strings"""
|
||||
|
||||
|
||||
@ -94,3 +94,152 @@ def uses_marionette(func):
|
||||
|
||||
return ret
|
||||
return _
|
||||
|
||||
|
||||
def _raw_log():
|
||||
import logging
|
||||
return logging.getLogger(__name__)
|
||||
|
||||
|
||||
def test_environment(xrePath, env=None, crashreporter=True, debugger=False,
|
||||
dmdPath=None, lsanPath=None, log=None):
|
||||
"""
|
||||
populate OS environment variables for mochitest and reftests.
|
||||
|
||||
Originally comes from automationutils.py. Don't use that for new code.
|
||||
"""
|
||||
|
||||
env = os.environ.copy() if env is None else env
|
||||
log = log or _raw_log()
|
||||
|
||||
assert os.path.isabs(xrePath)
|
||||
|
||||
if mozinfo.isMac:
|
||||
ldLibraryPath = os.path.join(os.path.dirname(xrePath), "MacOS")
|
||||
else:
|
||||
ldLibraryPath = xrePath
|
||||
|
||||
envVar = None
|
||||
dmdLibrary = None
|
||||
preloadEnvVar = None
|
||||
if 'toolkit' in mozinfo.info and mozinfo.info['toolkit'] == "gonk":
|
||||
# Skip all of this, it's only valid for the host.
|
||||
pass
|
||||
elif mozinfo.isUnix:
|
||||
envVar = "LD_LIBRARY_PATH"
|
||||
env['MOZILLA_FIVE_HOME'] = xrePath
|
||||
dmdLibrary = "libdmd.so"
|
||||
preloadEnvVar = "LD_PRELOAD"
|
||||
elif mozinfo.isMac:
|
||||
envVar = "DYLD_LIBRARY_PATH"
|
||||
dmdLibrary = "libdmd.dylib"
|
||||
preloadEnvVar = "DYLD_INSERT_LIBRARIES"
|
||||
elif mozinfo.isWin:
|
||||
envVar = "PATH"
|
||||
dmdLibrary = "dmd.dll"
|
||||
preloadEnvVar = "MOZ_REPLACE_MALLOC_LIB"
|
||||
if envVar:
|
||||
envValue = ((env.get(envVar), str(ldLibraryPath))
|
||||
if mozinfo.isWin
|
||||
else (ldLibraryPath, dmdPath, env.get(envVar)))
|
||||
env[envVar] = os.path.pathsep.join([path for path in envValue if path])
|
||||
|
||||
if dmdPath and dmdLibrary and preloadEnvVar:
|
||||
env[preloadEnvVar] = os.path.join(dmdPath, dmdLibrary)
|
||||
|
||||
# crashreporter
|
||||
env['GNOME_DISABLE_CRASH_DIALOG'] = '1'
|
||||
env['XRE_NO_WINDOWS_CRASH_DIALOG'] = '1'
|
||||
|
||||
if crashreporter and not debugger:
|
||||
env['MOZ_CRASHREPORTER_NO_REPORT'] = '1'
|
||||
env['MOZ_CRASHREPORTER'] = '1'
|
||||
else:
|
||||
env['MOZ_CRASHREPORTER_DISABLE'] = '1'
|
||||
|
||||
# Crash on non-local network connections by default.
|
||||
# MOZ_DISABLE_NONLOCAL_CONNECTIONS can be set to "0" to temporarily
|
||||
# enable non-local connections for the purposes of local testing. Don't
|
||||
# override the user's choice here. See bug 1049688.
|
||||
env.setdefault('MOZ_DISABLE_NONLOCAL_CONNECTIONS', '1')
|
||||
|
||||
# Set WebRTC logging in case it is not set yet
|
||||
env.setdefault(
|
||||
'NSPR_LOG_MODULES',
|
||||
'signaling:3,mtransport:4,datachannel:4,jsep:4,MediaPipelineFactory:4'
|
||||
)
|
||||
env.setdefault('R_LOG_LEVEL', '6')
|
||||
env.setdefault('R_LOG_DESTINATION', 'stderr')
|
||||
env.setdefault('R_LOG_VERBOSE', '1')
|
||||
|
||||
# ASan specific environment stuff
|
||||
asan = bool(mozinfo.info.get("asan"))
|
||||
if asan and (mozinfo.isLinux or mozinfo.isMac):
|
||||
try:
|
||||
# Symbolizer support
|
||||
llvmsym = os.path.join(xrePath, "llvm-symbolizer")
|
||||
if os.path.isfile(llvmsym):
|
||||
env["ASAN_SYMBOLIZER_PATH"] = llvmsym
|
||||
log.info("INFO | runtests.py | ASan using symbolizer at %s"
|
||||
% llvmsym)
|
||||
else:
|
||||
log.info("TEST-UNEXPECTED-FAIL | runtests.py | Failed to find"
|
||||
" ASan symbolizer at %s" % llvmsym)
|
||||
|
||||
# Returns total system memory in kilobytes.
|
||||
# Works only on unix-like platforms where `free` is in the path.
|
||||
totalMemory = int(os.popen("free").readlines()[1].split()[1])
|
||||
|
||||
# Only 4 GB RAM or less available? Use custom ASan options to reduce
|
||||
# the amount of resources required to do the tests. Standard options
|
||||
# will otherwise lead to OOM conditions on the current test slaves.
|
||||
message = "INFO | runtests.py | ASan running in %s configuration"
|
||||
asanOptions = []
|
||||
if totalMemory <= 1024 * 1024 * 4:
|
||||
message = message % 'low-memory'
|
||||
asanOptions = [
|
||||
'quarantine_size=50331648', 'malloc_context_size=5']
|
||||
else:
|
||||
message = message % 'default memory'
|
||||
|
||||
if lsanPath:
|
||||
log.info("LSan enabled.")
|
||||
asanOptions.append('detect_leaks=1')
|
||||
lsanOptions = ["exitcode=0"]
|
||||
suppressionsFile = os.path.join(
|
||||
lsanPath, 'lsan_suppressions.txt')
|
||||
if os.path.exists(suppressionsFile):
|
||||
log.info("LSan using suppression file " + suppressionsFile)
|
||||
lsanOptions.append("suppressions=" + suppressionsFile)
|
||||
else:
|
||||
log.info("WARNING | runtests.py | LSan suppressions file"
|
||||
" does not exist! " + suppressionsFile)
|
||||
env["LSAN_OPTIONS"] = ':'.join(lsanOptions)
|
||||
# Run shutdown GCs and CCs to avoid spurious leaks.
|
||||
env['MOZ_CC_RUN_DURING_SHUTDOWN'] = '1'
|
||||
|
||||
if len(asanOptions):
|
||||
env['ASAN_OPTIONS'] = ':'.join(asanOptions)
|
||||
|
||||
except OSError, err:
|
||||
log.info("Failed determine available memory, disabling ASan"
|
||||
" low-memory configuration: %s" % err.strerror)
|
||||
except:
|
||||
log.info("Failed determine available memory, disabling ASan"
|
||||
" low-memory configuration")
|
||||
else:
|
||||
log.info(message)
|
||||
|
||||
tsan = bool(mozinfo.info.get("tsan"))
|
||||
if tsan and mozinfo.isLinux:
|
||||
# Symbolizer support.
|
||||
llvmsym = os.path.join(xrePath, "llvm-symbolizer")
|
||||
if os.path.isfile(llvmsym):
|
||||
env["TSAN_OPTIONS"] = "external_symbolizer_path=%s" % llvmsym
|
||||
log.info("INFO | runtests.py | TSan using symbolizer at %s"
|
||||
% llvmsym)
|
||||
else:
|
||||
log.info("TEST-UNEXPECTED-FAIL | runtests.py | Failed to find TSan"
|
||||
" symbolizer at %s" % llvmsym)
|
||||
|
||||
return env
|
||||
|
||||
Loading…
Reference in New Issue
Block a user