Magic-Mirror/gecko-dev
1
0
Fork 0
mirror of https://github.com/mozilla/gecko-dev.git synced 2025-01-27 15:55:16 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Bug 1120983 - [manifestparser] Split manifestparser.py into several smaller files, r=wlach

Browse Source 
Simple refactor that moves logic out of manifestparser.py and into cli.py, expression.py and ini.py.

--HG--
extra : rebase_source : dd454973cdb3bcb7ec29dd2e1c0c594e3b3fb817
This commit is contained in:
Andrew Halberstadt 2015-01-15 09:37:51 -05:00
parent f22aabad26
commit eb68fdf7f2
7 changed files with 625 additions and 602 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
testing/mochitest/moz.build
View File

@ -33,7 +33,6 @@ TEST_HARNESS_FILES.testing.mochitest += [
'/build/pgo/server-locations.txt',
'/build/sanitizers/lsan_suppressions.txt',
'/netwerk/test/httpserver/httpd.js',
'/testing/mozbase/manifestparser/manifestparser/manifestparser.py',
'/testing/mozbase/mozdevice/mozdevice/devicemanager.py',
'/testing/mozbase/mozdevice/mozdevice/devicemanagerADB.py',
'/testing/mozbase/mozdevice/mozdevice/devicemanagerSUT.py',

4
testing/mozbase/manifestparser/manifestparser/__init__.py
View File

@ -2,4 +2,6 @@
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
# You can obtain one at http://mozilla.org/MPL/2.0/.
from manifestparser import *
from .manifestparser import *
from .expression import *
from .ini import *

237
testing/mozbase/manifestparser/manifestparser/cli.py Normal file
View File

@ -0,0 +1,237 @@
#!/usr/bin/env python
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
# You can obtain one at http://mozilla.org/MPL/2.0/.
"""
Mozilla universal manifest parser
"""
from optparse import OptionParser
import os
import sys
from .manifestparser import (
convert,
ManifestParser,
)
class ParserError(Exception):
"""error for exceptions while parsing the command line"""
def parse_args(_args):
"""
parse and return:
--keys=value (or --key value)
-tags
args
"""
# return values
_dict = {}
tags = []
args = []
# parse the arguments
key = None
for arg in _args:
if arg.startswith('---'):
raise ParserError("arguments should start with '-' or '--' only")
elif arg.startswith('--'):
if key:
raise ParserError("Key %s still open" % key)
key = arg[2:]
if '=' in key:
key, value = key.split('=', 1)
_dict[key] = value
key = None
continue
elif arg.startswith('-'):
if key:
raise ParserError("Key %s still open" % key)
tags.append(arg[1:])
continue
else:
if key:
_dict[key] = arg
continue
args.append(arg)
# return values
return (_dict, tags, args)
class CLICommand(object):
usage = '%prog [options] command'
def __init__(self, parser):
self._parser = parser # master parser
def parser(self):
return OptionParser(usage=self.usage, description=self.__doc__,
add_help_option=False)
class Copy(CLICommand):
usage = '%prog [options] copy manifest directory -tag1 -tag2 --key1=value1 --key2=value2 ...'
def __call__(self, options, args):
# parse the arguments
try:
kwargs, tags, args = parse_args(args)
except ParserError, e:
self._parser.error(e.message)
# make sure we have some manifests, otherwise it will
# be quite boring
if not len(args) == 2:
HelpCLI(self._parser)(options, ['copy'])
return
# read the manifests
# TODO: should probably ensure these exist here
manifests = ManifestParser()
manifests.read(args[0])
# print the resultant query
manifests.copy(args[1], None, *tags, **kwargs)
class CreateCLI(CLICommand):
"""
create a manifest from a list of directories
"""
usage = '%prog [options] create directory <directory> <...>'
def parser(self):
parser = CLICommand.parser(self)
parser.add_option('-p', '--pattern', dest='pattern',
help="glob pattern for files")
parser.add_option('-i', '--ignore', dest='ignore',
default=[], action='append',
help='directories to ignore')
parser.add_option('-w', '--in-place', dest='in_place',
help='Write .ini files in place; filename to write to')
return parser
def __call__(self, _options, args):
parser = self.parser()
options, args = parser.parse_args(args)
# need some directories
if not len(args):
parser.print_usage()
return
# add the directories to the manifest
for arg in args:
assert os.path.exists(arg)
assert os.path.isdir(arg)
manifest = convert(args, pattern=options.pattern, ignore=options.ignore,
write=options.in_place)
if manifest:
print manifest
class WriteCLI(CLICommand):
"""
write a manifest based on a query
"""
usage = '%prog [options] write manifest <manifest> -tag1 -tag2 --key1=value1 --key2=value2 ...'
def __call__(self, options, args):
# parse the arguments
try:
kwargs, tags, args = parse_args(args)
except ParserError, e:
self._parser.error(e.message)
# make sure we have some manifests, otherwise it will
# be quite boring
if not args:
HelpCLI(self._parser)(options, ['write'])
return
# read the manifests
# TODO: should probably ensure these exist here
manifests = ManifestParser()
manifests.read(*args)
# print the resultant query
manifests.write(global_tags=tags, global_kwargs=kwargs)
class HelpCLI(CLICommand):
"""
get help on a command
"""
usage = '%prog [options] help [command]'
def __call__(self, options, args):
if len(args) == 1 and args[0] in commands:
commands[args[0]](self._parser).parser().print_help()
else:
self._parser.print_help()
print '\nCommands:'
for command in sorted(commands):
print ' %s : %s' % (command, commands[command].__doc__.strip())
class UpdateCLI(CLICommand):
"""
update the tests as listed in a manifest from a directory
"""
usage = '%prog [options] update manifest directory -tag1 -tag2 --key1=value1 --key2=value2 ...'
def __call__(self, options, args):
# parse the arguments
try:
kwargs, tags, args = parse_args(args)
except ParserError, e:
self._parser.error(e.message)
# make sure we have some manifests, otherwise it will
# be quite boring
if not len(args) == 2:
HelpCLI(self._parser)(options, ['update'])
return
# read the manifests
# TODO: should probably ensure these exist here
manifests = ManifestParser()
manifests.read(args[0])
# print the resultant query
manifests.update(args[1], None, *tags, **kwargs)
# command -> class mapping
commands = { 'create': CreateCLI,
'help': HelpCLI,
'update': UpdateCLI,
'write': WriteCLI }
def main(args=sys.argv[1:]):
"""console_script entry point"""
# set up an option parser
usage = '%prog [options] [command] ...'
description = "%s. Use `help` to display commands" % __doc__.strip()
parser = OptionParser(usage=usage, description=description)
parser.add_option('-s', '--strict', dest='strict',
action='store_true', default=False,
help='adhere strictly to errors')
parser.disable_interspersed_args()
options, args = parser.parse_args(args)
if not args:
HelpCLI(parser)(options, args)
parser.exit()
# get the command
command = args[0]
if command not in commands:
parser.error("Command must be one of %s (you gave '%s')" % (', '.join(sorted(commands.keys())), command))
handler = commands[command](parser)
handler(options, args[1:])
if __name__ == '__main__':
main()

252
testing/mozbase/manifestparser/manifestparser/expression.py Normal file
View File

@ -0,0 +1,252 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
# You can obtain one at http://mozilla.org/MPL/2.0/.
import re
__all__ = ['parse', 'ParseError', 'ExpressionParser']
# expr.py
# from:
# http://k0s.org/mozilla/hg/expressionparser
# http://hg.mozilla.org/users/tmielczarek_mozilla.com/expressionparser
# Implements a top-down parser/evaluator for simple boolean expressions.
# ideas taken from http://effbot.org/zone/simple-top-down-parsing.htm
#
# Rough grammar:
# expr := literal
# | '(' expr ')'
# | expr '&&' expr
# | expr '||' expr
# | expr '==' expr
# | expr '!=' expr
# literal := BOOL
# | INT
# | STRING
# | IDENT
# BOOL := true|false
# INT := [0-9]+
# STRING := "[^"]*"
# IDENT := [A-Za-z_]\w*
# Identifiers take their values from a mapping dictionary passed as the second
# argument.
# Glossary (see above URL for details):
# - nud: null denotation
# - led: left detonation
# - lbp: left binding power
# - rbp: right binding power
class ident_token(object):
def __init__(self, scanner, value):
self.value = value
def nud(self, parser):
# identifiers take their value from the value mappings passed
# to the parser
return parser.value(self.value)
class literal_token(object):
def __init__(self, scanner, value):
self.value = value
def nud(self, parser):
return self.value
class eq_op_token(object):
"=="
def led(self, parser, left):
return left == parser.expression(self.lbp)
class neq_op_token(object):
"!="
def led(self, parser, left):
return left != parser.expression(self.lbp)
class not_op_token(object):
"!"
def nud(self, parser):
return not parser.expression(100)
class and_op_token(object):
"&&"
def led(self, parser, left):
right = parser.expression(self.lbp)
return left and right
class or_op_token(object):
"||"
def led(self, parser, left):
right = parser.expression(self.lbp)
return left or right
class lparen_token(object):
"("
def nud(self, parser):
expr = parser.expression()
parser.advance(rparen_token)
return expr
class rparen_token(object):
")"
class end_token(object):
"""always ends parsing"""
### derived literal tokens
class bool_token(literal_token):
def __init__(self, scanner, value):
value = {'true':True, 'false':False}[value]
literal_token.__init__(self, scanner, value)
class int_token(literal_token):
def __init__(self, scanner, value):
literal_token.__init__(self, scanner, int(value))
class string_token(literal_token):
def __init__(self, scanner, value):
literal_token.__init__(self, scanner, value[1:-1])
precedence = [(end_token, rparen_token),
(or_op_token,),
(and_op_token,),
(eq_op_token, neq_op_token),
(lparen_token,),
]
for index, rank in enumerate(precedence):
for token in rank:
token.lbp = index # lbp = lowest left binding power
class ParseError(Exception):
"""error parsing conditional expression"""
class ExpressionParser(object):
"""
A parser for a simple expression language.
The expression language can be described as follows::
EXPRESSION ::= LITERAL | '(' EXPRESSION ')' | '!' EXPRESSION | EXPRESSION OP EXPRESSION
OP ::= '==' | '!=' | '&&' | '||'
LITERAL ::= BOOL | INT | IDENT | STRING
BOOL ::= 'true' | 'false'
INT ::= [0-9]+
IDENT ::= [a-zA-Z_]\w*
STRING ::= '"' [^\"] '"' | ''' [^\'] '''
At its core, expressions consist of booleans, integers, identifiers and.
strings. Booleans are one of *true* or *false*. Integers are a series
of digits. Identifiers are a series of English letters and underscores.
Strings are a pair of matching quote characters (single or double) with
zero or more characters inside.
Expressions can be combined with operators: the equals (==) and not
equals (!=) operators compare two expressions and produce a boolean. The
and (&&) and or (||) operators take two expressions and produce the logical
AND or OR value of them, respectively. An expression can also be prefixed
with the not (!) operator, which produces its logical negation.
Finally, any expression may be contained within parentheses for grouping.
Identifiers take their values from the mapping provided.
"""
scanner = None
def __init__(self, text, valuemapping, strict=False):
"""
Initialize the parser
:param text: The expression to parse as a string.
:param valuemapping: A dict mapping identifier names to values.
:param strict: If true, referencing an identifier that was not
provided in :valuemapping: will raise an error.
"""
self.text = text
self.valuemapping = valuemapping
self.strict = strict
def _tokenize(self):
"""
Lex the input text into tokens and yield them in sequence.
"""
if not ExpressionParser.scanner:
ExpressionParser.scanner = re.Scanner([
# Note: keep these in sync with the class docstring above.
(r"true|false", bool_token),
(r"[a-zA-Z_]\w*", ident_token),
(r"[0-9]+", int_token),
(r'("[^"]*")|(\'[^\']*\')', string_token),
(r"==", eq_op_token()),
(r"!=", neq_op_token()),
(r"\|\|", or_op_token()),
(r"!", not_op_token()),
(r"&&", and_op_token()),
(r"\(", lparen_token()),
(r"\)", rparen_token()),
(r"\s+", None), # skip whitespace
])
tokens, remainder = ExpressionParser.scanner.scan(self.text)
for t in tokens:
yield t
yield end_token()
def value(self, ident):
"""
Look up the value of |ident| in the value mapping passed in the
constructor.
"""
if self.strict:
return self.valuemapping[ident]
else:
return self.valuemapping.get(ident, None)
def advance(self, expected):
"""
Assert that the next token is an instance of |expected|, and advance
to the next token.
"""
if not isinstance(self.token, expected):
raise Exception, "Unexpected token!"
self.token = self.iter.next()
def expression(self, rbp=0):
"""
Parse and return the value of an expression until a token with
right binding power greater than rbp is encountered.
"""
t = self.token
self.token = self.iter.next()
left = t.nud(self)
while rbp < self.token.lbp:
t = self.token
self.token = self.iter.next()
left = t.led(self, left)
return left
def parse(self):
"""
Parse and return the value of the expression in the text
passed to the constructor. Raises a ParseError if the expression
could not be parsed.
"""
try:
self.iter = self._tokenize()
self.token = self.iter.next()
return self.expression()
except:
raise ParseError("could not parse: %s; variables: %s" % (self.text, self.valuemapping))
__call__ = parse
def parse(text, **values):
"""
Parse and evaluate a boolean expression.
:param text: The expression to parse, as a string.
:param values: A dict containing a name to value mapping for identifiers
referenced in *text*.
:rtype: the final value of the expression.
:raises: :py:exc::ParseError: will be raised if parsing fails.
"""
return ExpressionParser(text, values).parse()

125
testing/mozbase/manifestparser/manifestparser/ini.py Normal file
View File

@ -0,0 +1,125 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
# You can obtain one at http://mozilla.org/MPL/2.0/.
__all__ = ['read_ini']
import os
def read_ini(fp, variables=None, default='DEFAULT', defaults_only=False,
comments=';#', separators=('=', ':'),
strict=True):
"""
read an .ini file and return a list of [(section, values)]
- fp : file pointer or path to read
- variables : default set of variables
- default : name of the section for the default section
- defaults_only : if True, return the default section only
- comments : characters that if they start a line denote a comment
- separators : strings that denote key, value separation in order
- strict : whether to be strict about parsing
"""
# variables
variables = variables or {}
sections = []
key = value = None
section_names = set()
if isinstance(fp, basestring):
fp = file(fp)
# read the lines
for (linenum, line) in enumerate(fp.readlines(), start=1):
stripped = line.strip()
# ignore blank lines
if not stripped:
# reset key and value to avoid continuation lines
key = value = None
continue
# ignore comment lines
if stripped[0] in comments:
continue
# check for a new section
if len(stripped) > 2 and stripped[0] == '[' and stripped[-1] == ']':
section = stripped[1:-1].strip()
key = value = None
# deal with DEFAULT section
if section.lower() == default.lower():
if strict:
assert default not in section_names
section_names.add(default)
current_section = variables
continue
if strict:
# make sure this section doesn't already exist
assert section not in section_names, "Section '%s' already found in '%s'" % (section, section_names)
section_names.add(section)
current_section = {}
sections.append((section, current_section))
continue
# if there aren't any sections yet, something bad happen
if not section_names:
raise Exception('No sections found')
# (key, value) pair
for separator in separators:
if separator in stripped:
key, value = stripped.split(separator, 1)
key = key.strip()
value = value.strip()
if strict:
# make sure this key isn't already in the section or empty
assert key
if current_section is not variables:
assert key not in current_section
current_section[key] = value
break
else:
# continuation line ?
if line[0].isspace() and key:
value = '%s%s%s' % (value, os.linesep, stripped)
current_section[key] = value
else:
# something bad happened!
if hasattr(fp, 'name'):
filename = fp.name
else:
filename = 'unknown'
raise Exception("Error parsing manifest file '%s', line %s" %
(filename, linenum))
# server-root is a special os path declared relative to the manifest file.
# inheritance demands we expand it as absolute
if 'server-root' in variables:
root = os.path.join(os.path.dirname(fp.name),
variables['server-root'])
variables['server-root'] = os.path.abspath(root)
# return the default section only if requested
if defaults_only:
return [(default, variables)]
# interpret the variables
def interpret_variables(global_dict, local_dict):
variables = global_dict.copy()
if 'skip-if' in local_dict and 'skip-if' in variables:
local_dict['skip-if'] = "(%s) || (%s)" % (variables['skip-if'].split('#')[0], local_dict['skip-if'].split('#')[0])
variables.update(local_dict)
return variables
sections = [(i, interpret_variables(variables, j)) for i, j in sections]
return sections

606
testing/mozbase/manifestparser/manifestparser/manifestparser.py
View File

@ -1,276 +1,26 @@
#!/usr/bin/env python
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
# You can obtain one at http://mozilla.org/MPL/2.0/.
"""
Mozilla universal manifest parser
"""
__all__ = ['read_ini', # .ini reader
'ManifestParser', 'TestManifest', 'convert', # manifest handling
'parse', 'ParseError', 'ExpressionParser'] # conditional expression parser
__all__ = ['ManifestParser', 'TestManifest', 'convert']
from StringIO import StringIO
import json
import fnmatch
import os
import re
import shutil
import sys
from optparse import OptionParser
from StringIO import StringIO
from .ini import read_ini
from .expression import (
parse,
ParseError,
)
relpath = os.path.relpath
string = (basestring,)
# expr.py
# from:
# http://k0s.org/mozilla/hg/expressionparser
# http://hg.mozilla.org/users/tmielczarek_mozilla.com/expressionparser
# Implements a top-down parser/evaluator for simple boolean expressions.
# ideas taken from http://effbot.org/zone/simple-top-down-parsing.htm
#
# Rough grammar:
# expr := literal
# | '(' expr ')'
# | expr '&&' expr
# | expr '||' expr
# | expr '==' expr
# | expr '!=' expr
# literal := BOOL
# | INT
# | STRING
# | IDENT
# BOOL := true|false
# INT := [0-9]+
# STRING := "[^"]*"
# IDENT := [A-Za-z_]\w*
# Identifiers take their values from a mapping dictionary passed as the second
# argument.
# Glossary (see above URL for details):
# - nud: null denotation
# - led: left detonation
# - lbp: left binding power
# - rbp: right binding power
class ident_token(object):
def __init__(self, scanner, value):
self.value = value
def nud(self, parser):
# identifiers take their value from the value mappings passed
# to the parser
return parser.value(self.value)
class literal_token(object):
def __init__(self, scanner, value):
self.value = value
def nud(self, parser):
return self.value
class eq_op_token(object):
"=="
def led(self, parser, left):
return left == parser.expression(self.lbp)
class neq_op_token(object):
"!="
def led(self, parser, left):
return left != parser.expression(self.lbp)
class not_op_token(object):
"!"
def nud(self, parser):
return not parser.expression(100)
class and_op_token(object):
"&&"
def led(self, parser, left):
right = parser.expression(self.lbp)
return left and right
class or_op_token(object):
"||"
def led(self, parser, left):
right = parser.expression(self.lbp)
return left or right
class lparen_token(object):
"("
def nud(self, parser):
expr = parser.expression()
parser.advance(rparen_token)
return expr
class rparen_token(object):
")"
class end_token(object):
"""always ends parsing"""
### derived literal tokens
class bool_token(literal_token):
def __init__(self, scanner, value):
value = {'true':True, 'false':False}[value]
literal_token.__init__(self, scanner, value)
class int_token(literal_token):
def __init__(self, scanner, value):
literal_token.__init__(self, scanner, int(value))
class string_token(literal_token):
def __init__(self, scanner, value):
literal_token.__init__(self, scanner, value[1:-1])
precedence = [(end_token, rparen_token),
(or_op_token,),
(and_op_token,),
(eq_op_token, neq_op_token),
(lparen_token,),
]
for index, rank in enumerate(precedence):
for token in rank:
token.lbp = index # lbp = lowest left binding power
class ParseError(Exception):
"""error parsing conditional expression"""
class ExpressionParser(object):
"""
A parser for a simple expression language.
The expression language can be described as follows::
EXPRESSION ::= LITERAL | '(' EXPRESSION ')' | '!' EXPRESSION | EXPRESSION OP EXPRESSION
OP ::= '==' | '!=' | '&&' | '||'
LITERAL ::= BOOL | INT | IDENT | STRING
BOOL ::= 'true' | 'false'
INT ::= [0-9]+
IDENT ::= [a-zA-Z_]\w*
STRING ::= '"' [^\"] '"' | ''' [^\'] '''
At its core, expressions consist of booleans, integers, identifiers and.
strings. Booleans are one of *true* or *false*. Integers are a series
of digits. Identifiers are a series of English letters and underscores.
Strings are a pair of matching quote characters (single or double) with
zero or more characters inside.
Expressions can be combined with operators: the equals (==) and not
equals (!=) operators compare two expressions and produce a boolean. The
and (&&) and or (||) operators take two expressions and produce the logical
AND or OR value of them, respectively. An expression can also be prefixed
with the not (!) operator, which produces its logical negation.
Finally, any expression may be contained within parentheses for grouping.
Identifiers take their values from the mapping provided.
"""
scanner = None
def __init__(self, text, valuemapping, strict=False):
"""
Initialize the parser
:param text: The expression to parse as a string.
:param valuemapping: A dict mapping identifier names to values.
:param strict: If true, referencing an identifier that was not
provided in :valuemapping: will raise an error.
"""
self.text = text
self.valuemapping = valuemapping
self.strict = strict
def _tokenize(self):
"""
Lex the input text into tokens and yield them in sequence.
"""
if not ExpressionParser.scanner:
ExpressionParser.scanner = re.Scanner([
# Note: keep these in sync with the class docstring above.
(r"true|false", bool_token),
(r"[a-zA-Z_]\w*", ident_token),
(r"[0-9]+", int_token),
(r'("[^"]*")|(\'[^\']*\')', string_token),
(r"==", eq_op_token()),
(r"!=", neq_op_token()),
(r"\|\|", or_op_token()),
(r"!", not_op_token()),
(r"&&", and_op_token()),
(r"\(", lparen_token()),
(r"\)", rparen_token()),
(r"\s+", None), # skip whitespace
])
tokens, remainder = ExpressionParser.scanner.scan(self.text)
for t in tokens:
yield t
yield end_token()
def value(self, ident):
"""
Look up the value of |ident| in the value mapping passed in the
constructor.
"""
if self.strict:
return self.valuemapping[ident]
else:
return self.valuemapping.get(ident, None)
def advance(self, expected):
"""
Assert that the next token is an instance of |expected|, and advance
to the next token.
"""
if not isinstance(self.token, expected):
raise Exception, "Unexpected token!"
self.token = self.iter.next()
def expression(self, rbp=0):
"""
Parse and return the value of an expression until a token with
right binding power greater than rbp is encountered.
"""
t = self.token
self.token = self.iter.next()
left = t.nud(self)
while rbp < self.token.lbp:
t = self.token
self.token = self.iter.next()
left = t.led(self, left)
return left
def parse(self):
"""
Parse and return the value of the expression in the text
passed to the constructor. Raises a ParseError if the expression
could not be parsed.
"""
try:
self.iter = self._tokenize()
self.token = self.iter.next()
return self.expression()
except:
raise ParseError("could not parse: %s; variables: %s" % (self.text, self.valuemapping))
__call__ = parse
def parse(text, **values):
"""
Parse and evaluate a boolean expression.
:param text: The expression to parse, as a string.
:param values: A dict containing a name to value mapping for identifiers
referenced in *text*.
:rtype: the final value of the expression.
:raises: :py:exc::ParseError: will be raised if parsing fails.
"""
return ExpressionParser(text, values).parse()
### path normalization
def normalize_path(path):
@ -286,124 +36,6 @@ def denormalize_path(path):
return path
### .ini reader
def read_ini(fp, variables=None, default='DEFAULT', defaults_only=False,
comments=';#', separators=('=', ':'),
strict=True):
"""
read an .ini file and return a list of [(section, values)]
- fp : file pointer or path to read
- variables : default set of variables
- default : name of the section for the default section
- defaults_only : if True, return the default section only
- comments : characters that if they start a line denote a comment
- separators : strings that denote key, value separation in order
- strict : whether to be strict about parsing
"""
# variables
variables = variables or {}
sections = []
key = value = None
section_names = set()
if isinstance(fp, basestring):
fp = file(fp)
# read the lines
for (linenum, line) in enumerate(fp.readlines(), start=1):
stripped = line.strip()
# ignore blank lines
if not stripped:
# reset key and value to avoid continuation lines
key = value = None
continue
# ignore comment lines
if stripped[0] in comments:
continue
# check for a new section
if len(stripped) > 2 and stripped[0] == '[' and stripped[-1] == ']':
section = stripped[1:-1].strip()
key = value = None
# deal with DEFAULT section
if section.lower() == default.lower():
if strict:
assert default not in section_names
section_names.add(default)
current_section = variables
continue
if strict:
# make sure this section doesn't already exist
assert section not in section_names, "Section '%s' already found in '%s'" % (section, section_names)
section_names.add(section)
current_section = {}
sections.append((section, current_section))
continue
# if there aren't any sections yet, something bad happen
if not section_names:
raise Exception('No sections found')
# (key, value) pair
for separator in separators:
if separator in stripped:
key, value = stripped.split(separator, 1)
key = key.strip()
value = value.strip()
if strict:
# make sure this key isn't already in the section or empty
assert key
if current_section is not variables:
assert key not in current_section
current_section[key] = value
break
else:
# continuation line ?
if line[0].isspace() and key:
value = '%s%s%s' % (value, os.linesep, stripped)
current_section[key] = value
else:
# something bad happened!
if hasattr(fp, 'name'):
filename = fp.name
else:
filename = 'unknown'
raise Exception("Error parsing manifest file '%s', line %s" %
(filename, linenum))
# server-root is a special os path declared relative to the manifest file.
# inheritance demands we expand it as absolute
if 'server-root' in variables:
root = os.path.join(os.path.dirname(fp.name),
variables['server-root'])
variables['server-root'] = os.path.abspath(root)
# return the default section only if requested
if defaults_only:
return [(default, variables)]
# interpret the variables
def interpret_variables(global_dict, local_dict):
variables = global_dict.copy()
if 'skip-if' in local_dict and 'skip-if' in variables:
local_dict['skip-if'] = "(%s) || (%s)" % (variables['skip-if'].split('#')[0], local_dict['skip-if'].split('#')[0])
variables.update(local_dict)
return variables
sections = [(i, interpret_variables(variables, j)) for i, j in sections]
return sections
### objects for parsing manifests
class ManifestParser(object):
@ -1183,227 +815,3 @@ class TestManifest(ManifestParser):
def test_paths(self):
return [test['path'] for test in self.active_tests()]
### command line attributes
class ParserError(Exception):
"""error for exceptions while parsing the command line"""
def parse_args(_args):
"""
parse and return:
--keys=value (or --key value)
-tags
args
"""
# return values
_dict = {}
tags = []
args = []
# parse the arguments
key = None
for arg in _args:
if arg.startswith('---'):
raise ParserError("arguments should start with '-' or '--' only")
elif arg.startswith('--'):
if key:
raise ParserError("Key %s still open" % key)
key = arg[2:]
if '=' in key:
key, value = key.split('=', 1)
_dict[key] = value
key = None
continue
elif arg.startswith('-'):
if key:
raise ParserError("Key %s still open" % key)
tags.append(arg[1:])
continue
else:
if key:
_dict[key] = arg
continue
args.append(arg)
# return values
return (_dict, tags, args)
### classes for subcommands
class CLICommand(object):
usage = '%prog [options] command'
def __init__(self, parser):
self._parser = parser # master parser
def parser(self):
return OptionParser(usage=self.usage, description=self.__doc__,
add_help_option=False)
class Copy(CLICommand):
usage = '%prog [options] copy manifest directory -tag1 -tag2 --key1=value1 --key2=value2 ...'
def __call__(self, options, args):
# parse the arguments
try:
kwargs, tags, args = parse_args(args)
except ParserError, e:
self._parser.error(e.message)
# make sure we have some manifests, otherwise it will
# be quite boring
if not len(args) == 2:
HelpCLI(self._parser)(options, ['copy'])
return
# read the manifests
# TODO: should probably ensure these exist here
manifests = ManifestParser()
manifests.read(args[0])
# print the resultant query
manifests.copy(args[1], None, *tags, **kwargs)
class CreateCLI(CLICommand):
"""
create a manifest from a list of directories
"""
usage = '%prog [options] create directory <directory> <...>'
def parser(self):
parser = CLICommand.parser(self)
parser.add_option('-p', '--pattern', dest='pattern',
help="glob pattern for files")
parser.add_option('-i', '--ignore', dest='ignore',
default=[], action='append',
help='directories to ignore')
parser.add_option('-w', '--in-place', dest='in_place',
help='Write .ini files in place; filename to write to')
return parser
def __call__(self, _options, args):
parser = self.parser()
options, args = parser.parse_args(args)
# need some directories
if not len(args):
parser.print_usage()
return
# add the directories to the manifest
for arg in args:
assert os.path.exists(arg)
assert os.path.isdir(arg)
manifest = convert(args, pattern=options.pattern, ignore=options.ignore,
write=options.in_place)
if manifest:
print manifest
class WriteCLI(CLICommand):
"""
write a manifest based on a query
"""
usage = '%prog [options] write manifest <manifest> -tag1 -tag2 --key1=value1 --key2=value2 ...'
def __call__(self, options, args):
# parse the arguments
try:
kwargs, tags, args = parse_args(args)
except ParserError, e:
self._parser.error(e.message)
# make sure we have some manifests, otherwise it will
# be quite boring
if not args:
HelpCLI(self._parser)(options, ['write'])
return
# read the manifests
# TODO: should probably ensure these exist here
manifests = ManifestParser()
manifests.read(*args)
# print the resultant query
manifests.write(global_tags=tags, global_kwargs=kwargs)
class HelpCLI(CLICommand):
"""
get help on a command
"""
usage = '%prog [options] help [command]'
def __call__(self, options, args):
if len(args) == 1 and args[0] in commands:
commands[args[0]](self._parser).parser().print_help()
else:
self._parser.print_help()
print '\nCommands:'
for command in sorted(commands):
print ' %s : %s' % (command, commands[command].__doc__.strip())
class UpdateCLI(CLICommand):
"""
update the tests as listed in a manifest from a directory
"""
usage = '%prog [options] update manifest directory -tag1 -tag2 --key1=value1 --key2=value2 ...'
def __call__(self, options, args):
# parse the arguments
try:
kwargs, tags, args = parse_args(args)
except ParserError, e:
self._parser.error(e.message)
# make sure we have some manifests, otherwise it will
# be quite boring
if not len(args) == 2:
HelpCLI(self._parser)(options, ['update'])
return
# read the manifests
# TODO: should probably ensure these exist here
manifests = ManifestParser()
manifests.read(args[0])
# print the resultant query
manifests.update(args[1], None, *tags, **kwargs)
# command -> class mapping
commands = { 'create': CreateCLI,
'help': HelpCLI,
'update': UpdateCLI,
'write': WriteCLI }
def main(args=sys.argv[1:]):
"""console_script entry point"""
# set up an option parser
usage = '%prog [options] [command] ...'
description = "%s. Use `help` to display commands" % __doc__.strip()
parser = OptionParser(usage=usage, description=description)
parser.add_option('-s', '--strict', dest='strict',
action='store_true', default=False,
help='adhere strictly to errors')
parser.disable_interspersed_args()
options, args = parser.parse_args(args)
if not args:
HelpCLI(parser)(options, args)
parser.exit()
# get the command
command = args[0]
if command not in commands:
parser.error("Command must be one of %s (you gave '%s')" % (', '.join(sorted(commands.keys())), command))
handler = commands[command](parser)
handler(options, args[1:])
if __name__ == '__main__':
main()

2
testing/mozbase/manifestparser/setup.py
View File

@ -22,6 +22,6 @@ setup(name=PACKAGE_NAME,
install_requires=[],
entry_points="""
[console_scripts]
manifestparser = manifestparser.manifestparser:main
manifestparser = manifestparser.cli:main
""",
)