Merge pull request #1785 from benhowes/chore/black

Black format codebase
This commit is contained in:
Liam Newman 2020-07-07 19:42:02 -07:00 committed by GitHub
commit 71b6545d85
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
42 changed files with 1746 additions and 1270 deletions

1
.pylintrc Normal file
View File

@ -0,0 +1 @@
max-line-length=88

View File

@ -30,6 +30,7 @@ js: generate-tests js/lib/*.js
py: generate-tests $(BUILD_DIR)/python
@echo Testing python beautify functionality...
$(SCRIPT_DIR)/python-dev3 black --config=python/pyproject.toml python
$(SCRIPT_DIR)/python-dev python python/js-beautify-test.py || exit 1
jstest: depends js build/*.tgz
@ -129,6 +130,7 @@ $(BUILD_DIR)/virtualenv: | $(BUILD_DIR)
virtualenv --version || pip install virtualenv
virtualenv build/python-dev
virtualenv build/python-rel
$(SCRIPT_DIR)/python-dev3 pip install black
@touch $(BUILD_DIR)/virtualenv

View File

@ -42,8 +42,8 @@ jobs:
Python27_Node8:
python.version: '2.7'
node_version: 8.x
Python35_Node10:
python.version: '3.5'
Python36_Node10:
python.version: '3.6'
node_version: 10.x
Python37_Node12:
python.version: '3.7'

View File

@ -46,7 +46,7 @@ def beautify(string, opts=default_options()):
def beautify_file(file_name, opts=default_options()):
if file_name == '-': # stdin
if file_name == "-": # stdin
try:
if sys.stdin.isatty():
raise Exception()
@ -59,14 +59,17 @@ def beautify_file(file_name, opts=default_options()):
else:
stream = open(file_name)
content = ''.join(stream.readlines())
content = "".join(stream.readlines())
b = Beautifier(content, opts)
return b.beautify()
def usage(stream=sys.stdout):
print("cssbeautifier.py@" + __version__ + """
print(
"cssbeautifier.py@"
+ __version__
+ """
CSS beautifier (https://beautifier.io/)
@ -102,7 +105,9 @@ Rarely needed options:
-h, --help, --usage Prints this help statement.
-v, --version Show the version
""", file=stream)
""",
file=stream,
)
if stream == sys.stderr:
return 1
else:
@ -114,13 +119,29 @@ def main():
argv = sys.argv[1:]
try:
opts, args = getopt.getopt(argv, "hvio:rs:c:e:tnb:",
['help', 'usage', 'version', 'stdin', 'outfile=', 'replace',
'indent-size=', 'indent-char=', 'eol=', 'indent-with-tabs',
'preserve-newlines', 'brace-style=',
'disable-selector-separator-newline', 'end-with-newline',
'disable-newline-between-rules',
'space-around-combinator', 'indent-empty-lines'])
opts, args = getopt.getopt(
argv,
"hvio:rs:c:e:tnb:",
[
"help",
"usage",
"version",
"stdin",
"outfile=",
"replace",
"indent-size=",
"indent-char=",
"eol=",
"indent-with-tabs",
"preserve-newlines",
"brace-style=",
"disable-selector-separator-newline",
"end-with-newline",
"disable-newline-between-rules",
"space-around-combinator",
"indent-empty-lines",
],
)
except getopt.GetoptError as ex:
print(ex, file=sys.stderr)
return usage(sys.stderr)
@ -128,60 +149,61 @@ def main():
css_options = default_options()
file = None
outfile = 'stdout'
outfile = "stdout"
replace = False
if len(args) == 1:
file = args[0]
for opt, arg in opts:
if opt in ('--stdin', '-i'):
file = '-'
elif opt in ('--outfile', '-o'):
if opt in ("--stdin", "-i"):
file = "-"
elif opt in ("--outfile", "-o"):
outfile = arg
elif opt in ('--replace', '-r'):
elif opt in ("--replace", "-r"):
replace = True
elif opt in ('--version', '-v'):
elif opt in ("--version", "-v"):
return print(__version__)
elif opt in ('--help', '--usage', '-h'):
elif opt in ("--help", "--usage", "-h"):
return usage()
elif opt in ('--indent-size', '-s'):
elif opt in ("--indent-size", "-s"):
css_options.indent_size = int(arg)
elif opt in ('--indent-char', '-c'):
elif opt in ("--indent-char", "-c"):
css_options.indent_char = arg
elif opt in ('--eol', '-e'):
elif opt in ("--eol", "-e"):
css_options.eol = arg
elif opt in ('--indent-with-tabs', '-t'):
elif opt in ("--indent-with-tabs", "-t"):
css_options.indent_with_tabs = True
elif opt in ('--preserve-newlines'):
elif opt in ("--preserve-newlines"):
css_options.preserve_newlines = True
elif opt in ('--disable-selector-separator-newline'):
elif opt in ("--disable-selector-separator-newline"):
css_options.selector_separator_newline = False
elif opt in ('--brace-style', '-b'):
elif opt in ("--brace-style", "-b"):
css_options.brace_style = arg
elif opt in ('--end-with-newline', '-n'):
elif opt in ("--end-with-newline", "-n"):
css_options.end_with_newline = True
elif opt in ('--disable-newline-between-rules'):
elif opt in ("--disable-newline-between-rules"):
css_options.newline_between_rules = False
elif opt in ('--space-around-combinator'):
elif opt in ("--space-around-combinator"):
css_options.space_around_combinator = True
elif opt in ('--indent-empty-lines'):
elif opt in ("--indent-empty-lines"):
css_options.indent_empty_lines = True
if not file:
file = '-'
file = "-"
try:
if outfile == 'stdout' and replace and not file == '-':
if outfile == "stdout" and replace and not file == "-":
outfile = file
pretty = beautify_file(file, css_options)
if outfile == 'stdout':
if outfile == "stdout":
# python automatically converts newlines in text to "\r\n" when on windows
# switch to binary to prevent this
if sys.platform == "win32":
import msvcrt
msvcrt.setmode(sys.stdout.fileno(), os.O_BINARY)
sys.stdout.write(pretty)
@ -191,8 +213,8 @@ def main():
# python automatically converts newlines in text to "\r\n" when on windows
# set newline to empty to prevent this
with io.open(outfile, 'wt', newline='') as f:
print('writing ' + outfile, file=sys.stderr)
with io.open(outfile, "wt", newline="") as f:
print("writing " + outfile, file=sys.stderr)
try:
f.write(pretty)
except TypeError:

View File

@ -39,7 +39,7 @@ six = __import__("six")
# SOFTWARE.
directives_core = Directives(r'/\*', r'\*/')
directives_core = Directives(r"/\*", r"\*/")
whitespaceChar = re.compile(r"\s")
whitespacePattern = re.compile(r"(?:\s|\n)+")
@ -57,31 +57,34 @@ def beautify(string, opts=default_options()):
def beautify_file(file_name, opts=default_options()):
if file_name == '-': # stdin
if file_name == "-": # stdin
stream = sys.stdin
else:
stream = open(file_name)
content = ''.join(stream.readlines())
content = "".join(stream.readlines())
b = Beautifier(content, opts)
return b.beautify()
def usage(stream=sys.stdout):
print("cssbeautifier.py@" + __version__ + """
print(
"cssbeautifier.py@"
+ __version__
+ """
CSS beautifier (https://beautifier.io/)
""", file=stream)
""",
file=stream,
)
if stream == sys.stderr:
return 1
else:
return 0
class Beautifier:
def __init__(self, source_text, opts=default_options()):
# in javascript, these two differ
# in python they are the same, different methods are called on them
@ -89,13 +92,11 @@ class Beautifier:
self.lineBreak = re.compile(six.u(r"\r\n|[\n\r]"))
self.allLineBreaks = self.lineBreak
self.comment_pattern = re.compile(
six.u(r"\/\/(?:[^\n\r\u2028\u2029]*)"))
self.block_comment_pattern = re.compile(
r"\/\*(?:[\s\S]*?)((?:\*\/)|$)")
self.comment_pattern = re.compile(six.u(r"\/\/(?:[^\n\r\u2028\u2029]*)"))
self.block_comment_pattern = re.compile(r"\/\*(?:[\s\S]*?)((?:\*\/)|$)")
if not source_text:
source_text = ''
source_text = ""
self.__source_text = source_text
@ -115,14 +116,12 @@ class Beautifier:
"@keyframes",
"@media",
"@supports",
"@document"}
self.CONDITIONAL_GROUP_RULE = {
"@media",
"@supports",
"@document"}
"@document",
}
self.CONDITIONAL_GROUP_RULE = {"@media", "@supports", "@document"}
def eatString(self, endChars):
result = ''
result = ""
self._ch = self._input.next()
while self._ch:
result += self._ch
@ -138,10 +137,10 @@ class Beautifier:
# newline character found; if the user has preserve_newlines off, only
# the first newline will be output
def eatWhitespace(self, allowAtLeastOneNewLine=False):
result = whitespaceChar.search(self._input.peek() or '') is not None
result = whitespaceChar.search(self._input.peek() or "") is not None
isFirstNewLine = True
while whitespaceChar.search(self._input.peek() or '') is not None:
while whitespaceChar.search(self._input.peek() or "") is not None:
self._ch = self._input.next()
if allowAtLeastOneNewLine and self._ch == "\n":
if self._options.preserve_newlines or isFirstNewLine:
@ -173,7 +172,6 @@ class Beautifier:
return False
def indent(self):
self._indentLevel += 1
@ -190,22 +188,20 @@ class Beautifier:
self._output.non_breaking_space = True
self._output.add_token(output_string)
def beautify(self):
if self._options.disabled:
return self.__source_text
source_text = self.__source_text
if self._options.eol == 'auto':
self._options.eol = '\n'
if self.lineBreak.search(source_text or ''):
if self._options.eol == "auto":
self._options.eol = "\n"
if self.lineBreak.search(source_text or ""):
self._options.eol = self.lineBreak.search(source_text).group()
# HACK: newline parsing inconsistent. This brute force normalizes the
# input newlines.
source_text = re.sub(self.allLineBreaks, '\n', source_text)
source_text = re.sub(self.allLineBreaks, "\n", source_text)
baseIndentString = re.search("^[\t ]*", source_text).group(0)
self._output = Output(self._options, baseIndentString)
@ -227,16 +223,16 @@ class Beautifier:
while True:
whitespace = self._input.read(whitespacePattern)
isAfterSpace = whitespace != ''
isAfterSpace = whitespace != ""
previous_ch = topCharacter
self._ch = self._input.next()
if self._ch == '\\' and self._input.hasNext():
if self._ch == "\\" and self._input.hasNext():
self._ch += self._input.next()
topCharacter = self._ch
if not self._ch:
break
elif self._ch == '/' and self._input.peek() == '*':
elif self._ch == "/" and self._input.peek() == "*":
# /* css comment */
# Always start block comments on a new line.
# This handles scenarios where a block comment immediately
@ -248,7 +244,7 @@ class Beautifier:
# handle ignore directive
directives = directives_core.get_directives(comment)
if directives and directives.get('ignore') == 'start':
if directives and directives.get("ignore") == "start":
comment += directives_core.readIgnored(self._input)
self.print_string(comment)
@ -259,7 +255,7 @@ class Beautifier:
# Block comments are followed by a new line so they don't
# share a line with other properties
self._output.add_new_line()
elif self._ch == '/' and self._input.peek() == '/':
elif self._ch == "/" and self._input.peek() == "/":
# // single line comment
# Preserves the space before a comment
# on the same line as a rule
@ -269,17 +265,18 @@ class Beautifier:
# Ensures any new lines following the comment are preserved
self.eatWhitespace(True)
elif self._ch == '@':
elif self._ch == "@":
self.preserveSingleSpace(isAfterSpace)
# deal with less propery mixins @{...}
if self._input.peek() == '{':
self.print_string(self._ch + self.eatString('}'))
if self._input.peek() == "{":
self.print_string(self._ch + self.eatString("}"))
else:
self.print_string(self._ch)
# strip trailing space, for hash property check
variableOrRule = self._input.peekUntilAfter(
re.compile(r"[: ,;{}()[\]\/='\"]"))
re.compile(r"[: ,;{}()[\]\/='\"]")
)
if variableOrRule[-1] in ": ":
# wwe have a variable or pseudo-class, add it and
@ -303,14 +300,15 @@ class Beautifier:
self._nestedLevel += 1
if variableOrRule in self.CONDITIONAL_GROUP_RULE:
enteringConditionalGroup = True
elif not insideRule and parenLevel == 0 and \
variableOrRule[-1] == ":":
elif (
not insideRule and parenLevel == 0 and variableOrRule[-1] == ":"
):
insidePropertyValue = True
self.indent()
elif self._ch == '#' and self._input.peek() == '{':
elif self._ch == "#" and self._input.peek() == "{":
self.preserveSingleSpace(isAfterSpace)
self.print_string(self._ch + self.eatString('}'))
elif self._ch == '{':
self.print_string(self._ch + self.eatString("}"))
elif self._ch == "{":
if insidePropertyValue:
insidePropertyValue = False
self.outdent()
@ -325,16 +323,18 @@ class Beautifier:
insideRule = self._indentLevel >= self._nestedLevel - 1
if self._options.newline_between_rules and insideRule:
if self._output.previous_line and \
not self._output.previous_line.is_empty() and \
self._output.previous_line.item(-1) != '{':
self._output.ensure_empty_line_above('/', ',')
if (
self._output.previous_line
and not self._output.previous_line.is_empty()
and self._output.previous_line.item(-1) != "{"
):
self._output.ensure_empty_line_above("/", ",")
self._output.space_before_token = True
# The difference in print_string and indent order
# is necessary to indent the '{' correctly
if self._options.brace_style == 'expand':
if self._options.brace_style == "expand":
self._output.add_new_line()
self.print_string(self._ch)
self.indent()
@ -345,10 +345,10 @@ class Beautifier:
self.eatWhitespace(True)
self._output.add_new_line()
elif self._ch == '}':
elif self._ch == "}":
self.outdent()
self._output.add_new_line()
if previous_ch == '{':
if previous_ch == "{":
self._output.trim(True)
insideAtExtend = False
insideAtImport = False
@ -363,16 +363,20 @@ class Beautifier:
self.eatWhitespace(True)
self._output.add_new_line()
if self._options.newline_between_rules and \
not self._output.just_added_blankline():
if self._input.peek() != '}':
if (
self._options.newline_between_rules
and not self._output.just_added_blankline()
):
if self._input.peek() != "}":
self._output.add_new_line(True)
elif self._ch == ":":
if (insideRule or enteringConditionalGroup) and \
not (self._input.lookBack('&') or
self.foundNestedPseudoClass()) and \
not self._input.lookBack('(') and not insideAtExtend and \
parenLevel == 0:
if (
(insideRule or enteringConditionalGroup)
and not (self._input.lookBack("&") or self.foundNestedPseudoClass())
and not self._input.lookBack("(")
and not insideAtExtend
and parenLevel == 0
):
# 'property: value' delimiter
# which could be in a conditional group query
self.print_string(":")
@ -388,7 +392,7 @@ class Beautifier:
# preserve space before pseudoclasses/pseudoelements,
# as it means "in any child"
if self._input.lookBack(' '):
if self._input.lookBack(" "):
self._output.space_before_token = True
if self._input.peek() == ":":
# pseudo-element
@ -397,11 +401,11 @@ class Beautifier:
else:
# pseudo-element
self.print_string(":")
elif self._ch == '"' or self._ch == '\'':
elif self._ch == '"' or self._ch == "'":
self.preserveSingleSpace(isAfterSpace)
self.print_string(self._ch + self.eatString(self._ch))
self.eatWhitespace(True)
elif self._ch == ';':
elif self._ch == ";":
if parenLevel == 0:
if insidePropertyValue:
self.outdent()
@ -415,13 +419,13 @@ class Beautifier:
# line. Block comments are also affected, but
# a new line is always output before one inside
# that section
if self._input.peek() is not '/':
if self._input.peek() is not "/":
self._output.add_new_line()
else:
self.print_string(self._ch)
self.eatWhitespace(True)
self._output.space_before_token = True
elif self._ch == '(':
elif self._ch == "(":
# may be a url
if self._input.lookBack("url"):
self.print_string(self._ch)
@ -429,10 +433,10 @@ class Beautifier:
parenLevel += 1
self.indent()
self._ch = self._input.next()
if self._ch in {')', '"', '\''}:
if self._ch in {")", '"', "'"}:
self._input.back()
elif self._ch is not None:
self.print_string(self._ch + self.eatString(')'))
self.print_string(self._ch + self.eatString(")"))
if parenLevel:
parenLevel -= 1
self.outdent()
@ -442,22 +446,28 @@ class Beautifier:
self.eatWhitespace()
parenLevel += 1
self.indent()
elif self._ch == ')':
elif self._ch == ")":
if parenLevel:
parenLevel -= 1
self.outdent()
self.print_string(self._ch)
elif self._ch == ',':
elif self._ch == ",":
self.print_string(self._ch)
self.eatWhitespace(True)
if self._options.selector_separator_newline and \
not insidePropertyValue and parenLevel == 0 and \
not insideAtImport:
if (
self._options.selector_separator_newline
and not insidePropertyValue
and parenLevel == 0
and not insideAtImport
):
self._output.add_new_line()
else:
self._output.space_before_token = True
elif (self._ch == '>' or self._ch == '+' or self._ch == '~') and \
not insidePropertyValue and parenLevel == 0:
elif (
(self._ch == ">" or self._ch == "+" or self._ch == "~")
and not insidePropertyValue
and parenLevel == 0
):
# handle combinator spacing
if self._options.space_around_combinator:
self._output.space_before_token = True
@ -468,21 +478,21 @@ class Beautifier:
self.eatWhitespace()
# squash extra whitespace
if self._ch and bool(whitespaceChar.search(self._ch)):
self._ch = ''
elif self._ch == ']':
self._ch = ""
elif self._ch == "]":
self.print_string(self._ch)
elif self._ch == '[':
elif self._ch == "[":
self.preserveSingleSpace(isAfterSpace)
self.print_string(self._ch)
elif self._ch == '=':
elif self._ch == "=":
# no whitespace before or after
self.eatWhitespace()
self.print_string('=')
self.print_string("=")
if bool(whitespaceChar.search(self._ch)):
self._ch = ''
elif self._ch == '!' and not (self._input.lookBack('\\')):
self._ch = ""
elif self._ch == "!" and not (self._input.lookBack("\\")):
# !important
self.print_string(' ')
self.print_string(" ")
self.print_string(self._ch)
else:
self.preserveSingleSpace(isAfterSpace)

View File

@ -25,25 +25,35 @@
from jsbeautifier.core.options import Options as BaseOptions
class BeautifierOptions(BaseOptions):
def __init__(self, options=None):
BaseOptions.__init__(self, options, 'css')
BaseOptions.__init__(self, options, "css")
self.selector_separator_newline = self._get_boolean('selector_separator_newline', True)
self.newline_between_rules = self._get_boolean('newline_between_rules', True)
self.selector_separator_newline = self._get_boolean(
"selector_separator_newline", True
)
self.newline_between_rules = self._get_boolean("newline_between_rules", True)
brace_style_split = self._get_selection_list('brace_style', ['collapse', 'expand', 'end-expand', 'none', 'preserve-inline'])
self.brace_style = 'collapse'
brace_style_split = self._get_selection_list(
"brace_style",
["collapse", "expand", "end-expand", "none", "preserve-inline"],
)
self.brace_style = "collapse"
for bs in brace_style_split:
if bs != 'expand':
if bs != "expand":
# default to collapse, as only collapse|expand is implemented for now
self.brace_style = 'collapse'
self.brace_style = "collapse"
else:
self.brace_style = bs
# deprecated
space_around_selector_separator = self._get_boolean('space_around_selector_separator')
space_around_selector_separator = self._get_boolean(
"space_around_selector_separator"
)
# Continue to accept deprecated option
self.space_around_combinator = self._get_boolean('space_around_combinator') or \
space_around_selector_separator
self.space_around_combinator = (
self._get_boolean("space_around_combinator")
or space_around_selector_separator
)

View File

@ -5,10 +5,8 @@ import unittest
def run_tests():
suite = unittest.TestLoader().discover(
'jsbeautifier', pattern="test*.py")
suite.addTests(unittest.TestLoader().discover(
'cssbeautifier', pattern="test*.py"))
suite = unittest.TestLoader().discover("jsbeautifier", pattern="test*.py")
suite.addTests(unittest.TestLoader().discover("cssbeautifier", pattern="test*.py"))
return unittest.TextTestRunner(verbosity=2).run(suite)

View File

@ -67,6 +67,7 @@ from jsbeautifier.javascript.beautifier import Beautifier
class MissingInputStreamError(Exception):
pass
def default_options():
return BeautifierOptions()
@ -78,6 +79,7 @@ def beautify(string, opts=default_options()):
def set_file_editorconfig_opts(filename, js_options):
from editorconfig import get_properties, EditorConfigError
try:
_ecoptions = get_properties(os.path.abspath(filename))
@ -93,51 +95,54 @@ def set_file_editorconfig_opts(filename, js_options):
if _ecoptions.get("max_line_length") == "off":
js_options.wrap_line_length = 0
else:
js_options.wrap_line_length = int(
_ecoptions["max_line_length"])
js_options.wrap_line_length = int(_ecoptions["max_line_length"])
if _ecoptions.get("insert_final_newline") == 'true':
if _ecoptions.get("insert_final_newline") == "true":
js_options.end_with_newline = True
elif _ecoptions.get("insert_final_newline") == 'false':
elif _ecoptions.get("insert_final_newline") == "false":
js_options.end_with_newline = False
if _ecoptions.get("end_of_line"):
if _ecoptions["end_of_line"] == "cr":
js_options.eol = '\r'
js_options.eol = "\r"
elif _ecoptions["end_of_line"] == "lf":
js_options.eol = '\n'
js_options.eol = "\n"
elif _ecoptions["end_of_line"] == "crlf":
js_options.eol = '\r\n'
js_options.eol = "\r\n"
except EditorConfigError:
# do not error on bad editor config
print("Error loading EditorConfig. Ignoring.", file=sys.stderr)
def beautify_file(file_name, opts=default_options()):
input_string = ''
if file_name == '-': # stdin
input_string = ""
if file_name == "-": # stdin
if sys.stdin.isatty():
raise MissingInputStreamError()
stream = sys.stdin
if platform.platform().lower().startswith('windows'):
if platform.platform().lower().startswith("windows"):
if sys.version_info.major >= 3:
# for python 3 on windows this prevents conversion
stream = io.TextIOWrapper(sys.stdin.buffer, newline='')
elif platform.architecture()[0] == '32bit':
stream = io.TextIOWrapper(sys.stdin.buffer, newline="")
elif platform.architecture()[0] == "32bit":
# for python 2 x86 on windows this prevents conversion
import msvcrt
msvcrt.setmode(sys.stdin.fileno(), os.O_BINARY)
else:
raise Exception('Pipe to stdin not supported on Windows with Python 2.x 64-bit.')
raise Exception(
"Pipe to stdin not supported on Windows with Python 2.x 64-bit."
)
input_string = stream.read()
# if you pipe an empty string, that is a failure
if input_string == '':
if input_string == "":
raise MissingInputStreamError()
else:
stream = io.open(file_name, 'rt', newline='', encoding='UTF-8')
stream = io.open(file_name, "rt", newline="", encoding="UTF-8")
input_string = stream.read()
return beautify(input_string, opts)
@ -145,7 +150,10 @@ def beautify_file(file_name, opts=default_options()):
def usage(stream=sys.stdout):
print("jsbeautifier.py@" + __version__ + """
print(
"jsbeautifier.py@"
+ __version__
+ """
Javascript beautifier (https://beautifier.io/)
@ -200,7 +208,9 @@ Rarely needed options:
-h, --help, --usage Prints this help statement.
-v, --version Show the version
""", file=stream)
""",
file=stream,
)
if stream == sys.stderr:
return 1
else:
@ -220,12 +230,7 @@ def mkdir_p(path):
def isFileDifferent(filepath, expected):
try:
return (
''.join(
io.open(
filepath,
'rt',
newline='').readlines()) != expected)
return "".join(io.open(filepath, "rt", newline="").readlines()) != expected
except BaseException:
return True
@ -235,13 +240,45 @@ def main():
argv = sys.argv[1:]
try:
opts, args = getopt.getopt(argv, "f:s:c:e:o:rdEPjab:kil:xhtvXnCO:w:m:",
[ 'brace-style=', 'comma-first', 'disable-preserve-newlines', 'e4x', 'editorconfig', 'end-with-newline',
'eol=', 'eval-code', 'file=', 'help', 'indent-char=', 'indent-empty-lines',
'indent-level=', 'indent-size=', 'indent-with-tabs', 'jslint-happy', 'keep-array-indentation', 'keep-function-indentation',
'max-preserve-newlines=', 'operator-position=', 'outfile=', 'quiet', 'replace', 'space-after-anon-function',
'space-after-named-function', 'space-in-empty-paren', 'space-in-paren', 'stdin', 'templating', 'unescape-strings',
'usage', 'version', 'wrap-line-length'])
opts, args = getopt.getopt(
argv,
"f:s:c:e:o:rdEPjab:kil:xhtvXnCO:w:m:",
[
"brace-style=",
"comma-first",
"disable-preserve-newlines",
"e4x",
"editorconfig",
"end-with-newline",
"eol=",
"eval-code",
"file=",
"help",
"indent-char=",
"indent-empty-lines",
"indent-level=",
"indent-size=",
"indent-with-tabs",
"jslint-happy",
"keep-array-indentation",
"keep-function-indentation",
"max-preserve-newlines=",
"operator-position=",
"outfile=",
"quiet",
"replace",
"space-after-anon-function",
"space-after-named-function",
"space-in-empty-paren",
"space-in-paren",
"stdin",
"templating",
"unescape-strings",
"usage",
"version",
"wrap-line-length",
],
)
except getopt.GetoptError as ex:
print(ex, file=sys.stderr)
return usage(sys.stderr)
@ -251,110 +288,111 @@ def main():
filepath_params = []
filepath_params.extend(args)
outfile_param = 'stdout'
outfile_param = "stdout"
replace = False
for opt, arg in opts:
if opt in ('--file', '-f'):
if opt in ("--file", "-f"):
filepath_params.append(arg)
elif opt in ('--keep-array-indentation', '-k'):
elif opt in ("--keep-array-indentation", "-k"):
js_options.keep_array_indentation = True
elif opt in ('--keep-function-indentation'):
elif opt in ("--keep-function-indentation"):
js_options.keep_function_indentation = True
elif opt in ('--outfile', '-o'):
elif opt in ("--outfile", "-o"):
outfile_param = arg
elif opt in ('--replace', '-r'):
elif opt in ("--replace", "-r"):
replace = True
elif opt in ('--indent-size', '-s'):
elif opt in ("--indent-size", "-s"):
js_options.indent_size = int(arg)
elif opt in ('--indent-char', '-c'):
elif opt in ("--indent-char", "-c"):
js_options.indent_char = arg
elif opt in ('--eol', '-e'):
elif opt in ("--eol", "-e"):
js_options.eol = arg
elif opt in ('--indent-with-tabs', '-t'):
elif opt in ("--indent-with-tabs", "-t"):
js_options.indent_with_tabs = True
elif opt in ('--disable-preserve-newlines', '-d'):
elif opt in ("--disable-preserve-newlines", "-d"):
js_options.preserve_newlines = False
elif opt in ('--max-preserve-newlines', '-m'):
elif opt in ("--max-preserve-newlines", "-m"):
js_options.max_preserve_newlines = int(arg)
elif opt in ('--space-in-paren', '-P'):
elif opt in ("--space-in-paren", "-P"):
js_options.space_in_paren = True
elif opt in ('--space-in-empty-paren', '-E'):
elif opt in ("--space-in-empty-paren", "-E"):
js_options.space_in_empty_paren = True
elif opt in ('--jslint-happy', '-j'):
elif opt in ("--jslint-happy", "-j"):
js_options.jslint_happy = True
elif opt in ('--space-after-anon-function', '-a'):
elif opt in ("--space-after-anon-function", "-a"):
js_options.space_after_anon_function = True
elif opt in ('--space-after-named-function'):
elif opt in ("--space-after-named-function"):
js_options.space_after_named_function = True
elif opt in ('--eval-code'):
elif opt in ("--eval-code"):
js_options.eval_code = True
elif opt in ('--quiet'):
elif opt in ("--quiet"):
js_options.keep_quiet = True
elif opt in ('--brace-style', '-b'):
elif opt in ("--brace-style", "-b"):
js_options.brace_style = arg
elif opt in ('--unescape-strings', '-x'):
elif opt in ("--unescape-strings", "-x"):
js_options.unescape_strings = True
elif opt in ('--e4x', '-X'):
elif opt in ("--e4x", "-X"):
js_options.e4x = True
elif opt in ('--end-with-newline', '-n'):
elif opt in ("--end-with-newline", "-n"):
js_options.end_with_newline = True
elif opt in ('--comma-first', '-C'):
elif opt in ("--comma-first", "-C"):
js_options.comma_first = True
elif opt in ('--operator-position', '-O'):
elif opt in ("--operator-position", "-O"):
js_options.operator_position = arg
elif opt in ('--wrap-line-length ', '-w'):
elif opt in ("--wrap-line-length ", "-w"):
js_options.wrap_line_length = int(arg)
elif opt in ('--indent-empty-lines'):
elif opt in ("--indent-empty-lines"):
js_options.indent_empty_lines = True
elif opt in ('--templating'):
js_options.templating = arg.split(',')
elif opt in ('--stdin', '-i'):
elif opt in ("--templating"):
js_options.templating = arg.split(",")
elif opt in ("--stdin", "-i"):
# stdin is the default if no files are passed
filepath_params = []
elif opt in ('--editorconfig'):
elif opt in ("--editorconfig"):
js_options.editorconfig = True
elif opt in ('--version', '-v'):
elif opt in ("--version", "-v"):
return print(__version__)
elif opt in ('--help', '--usage', '-h'):
elif opt in ("--help", "--usage", "-h"):
return usage()
try:
filepaths = []
if not filepath_params or (
len(filepath_params) == 1 and filepath_params[0] == '-'):
len(filepath_params) == 1 and filepath_params[0] == "-"
):
# default to stdin
filepath_params = []
filepaths.append('-')
filepaths.append("-")
for filepath_param in filepath_params:
# ignore stdin setting if files are specified
if '-' == filepath_param:
if "-" == filepath_param:
continue
# Check if each literal filepath exists
if os.path.isfile(filepath_param):
filepaths.append(filepath_param)
elif '*' in filepath_param or '?' in filepath_param:
elif "*" in filepath_param or "?" in filepath_param:
# handle globs
# empty result is okay
if sys.version_info.major == 2 or (
sys.version_info.major == 3 and
sys.version_info.minor <= 4):
if '**' in filepath_param:
raise Exception('Recursive globs not supported on Python <= 3.4.')
sys.version_info.major == 3 and sys.version_info.minor <= 4
):
if "**" in filepath_param:
raise Exception(
"Recursive globs not supported on Python <= 3.4."
)
filepaths.extend(glob.glob(filepath_param))
else:
filepaths.extend(glob.glob(filepath_param, recursive=True))
else:
# not a glob and not a file
raise OSError(errno.ENOENT, os.strerror(errno.ENOENT),
filepath_param)
raise OSError(errno.ENOENT, os.strerror(errno.ENOENT), filepath_param)
if len(filepaths) > 1:
replace = True
elif filepaths and filepaths[0] == '-':
elif filepaths and filepaths[0] == "-":
replace = False
# remove duplicates
@ -367,15 +405,15 @@ def main():
outfile = filepath
# Editorconfig used only on files, not stdin
if getattr(js_options, 'editorconfig'):
if getattr(js_options, "editorconfig"):
editorconfig_filepath = filepath
if editorconfig_filepath == '-':
if outfile != 'stdout':
if editorconfig_filepath == "-":
if outfile != "stdout":
editorconfig_filepath = outfile
else:
fileType = 'js'
editorconfig_filepath = 'stdin.' + fileType
fileType = "js"
editorconfig_filepath = "stdin." + fileType
# debug("EditorConfig is enabled for ", editorconfig_filepath);
js_options = copy.copy(js_options)
@ -383,21 +421,24 @@ def main():
pretty = beautify_file(filepath, js_options)
if outfile == 'stdout':
if outfile == "stdout":
stream = sys.stdout
# python automatically converts newlines in text to "\r\n" when on windows
# switch to binary to prevent this
if platform.platform().lower().startswith('windows'):
if platform.platform().lower().startswith("windows"):
if sys.version_info.major >= 3:
# for python 3 on windows this prevents conversion
stream = io.TextIOWrapper(sys.stdout.buffer, newline='')
elif platform.architecture()[0] == '32bit':
stream = io.TextIOWrapper(sys.stdout.buffer, newline="")
elif platform.architecture()[0] == "32bit":
# for python 2 x86 on windows this prevents conversion
import msvcrt
msvcrt.setmode(sys.stdout.fileno(), os.O_BINARY)
else:
raise Exception('Pipe to stdout not supported on Windows with Python 2.x 64-bit.')
raise Exception(
"Pipe to stdout not supported on Windows with Python 2.x 64-bit."
)
stream.write(pretty)
else:
@ -406,8 +447,8 @@ def main():
# python automatically converts newlines in text to "\r\n" when on windows
# set newline to empty to prevent this
with io.open(outfile, 'wt', newline='', encoding='UTF-8') as f:
print('beautified ' + outfile, file=sys.stdout)
with io.open(outfile, "wt", newline="", encoding="UTF-8") as f:
print("beautified " + outfile, file=sys.stdout)
try:
f.write(pretty)
except TypeError:
@ -417,19 +458,15 @@ def main():
six = __import__("six")
f.write(six.u(pretty))
elif not js_options.keep_quiet:
print('beautified ' + outfile + ' - unchanged', file=sys.stdout)
print("beautified " + outfile + " - unchanged", file=sys.stdout)
except MissingInputStreamError:
print(
"Must pipe input or define at least one file.\n",
file=sys.stderr)
print("Must pipe input or define at least one file.\n", file=sys.stderr)
usage(sys.stderr)
return 1
except UnicodeError as ex:
print("Error while decoding input or encoding output:",
file=sys.stderr)
print("Error while decoding input or encoding output:", file=sys.stderr)
print(ex, file=sys.stderr)
return 1

View File

@ -1 +1 @@
__version__ = '1.11.0'
__version__ = "1.11.0"

View File

@ -26,13 +26,16 @@ import re
class Directives:
def __init__(self, start_block_pattern, end_block_pattern):
self.__directives_block_pattern = re.compile(start_block_pattern + r' beautify( \w+[:]\w+)+ ' + end_block_pattern)
self.__directive_pattern = re.compile(r' (\w+)[:](\w+)')
self.__directives_block_pattern = re.compile(
start_block_pattern + r" beautify( \w+[:]\w+)+ " + end_block_pattern
)
self.__directive_pattern = re.compile(r" (\w+)[:](\w+)")
self.__directives_end_ignore_pattern = re.compile(start_block_pattern + r'\sbeautify\signore:end\s' + end_block_pattern)
self.__directives_end_ignore_pattern = re.compile(
start_block_pattern + r"\sbeautify\signore:end\s" + end_block_pattern
)
def get_directives(self, text):
if not self.__directives_block_pattern.match(text):
@ -44,8 +47,8 @@ class Directives:
while directive_match:
directives[directive_match.group(1)] = directive_match.group(2)
directive_match = self.__directive_pattern.search(
text, directive_match.end())
text, directive_match.end()
)
return directives

View File

@ -29,7 +29,7 @@ class InputScanner:
def __init__(self, input_string):
self.__six = __import__("six")
if input_string is None:
input_string = ''
input_string = ""
self.__input = input_string
self.__input_length = len(self.__input)
self.__position = 0
@ -62,8 +62,11 @@ class InputScanner:
def test(self, pattern, index=0):
index += self.__position
return index >= 0 and index < self.__input_length and bool(
pattern.match(self.__input, index))
return (
index >= 0
and index < self.__input_length
and bool(pattern.match(self.__input, index))
)
def testChar(self, pattern, index=0):
# test one character regex match
@ -79,21 +82,20 @@ class InputScanner:
return pattern_match
def read(self, starting_pattern, until_pattern=None, until_after=False):
val = ''
val = ""
pattern_match = None
if bool(starting_pattern):
pattern_match = self.match(starting_pattern)
if bool(pattern_match):
val = pattern_match.group(0)
if bool(until_pattern) and \
(bool(pattern_match) or not bool(starting_pattern)):
if bool(until_pattern) and (bool(pattern_match) or not bool(starting_pattern)):
val += self.readUntil(until_pattern, until_after)
return val
def readUntil(self, pattern, include_match=False):
val = ''
val = ""
pattern_match = None
match_index = self.__position
if self.hasNext():
@ -106,7 +108,7 @@ class InputScanner:
else:
match_index = self.__input_length
val = self.__input[self.__position:match_index]
val = self.__input[self.__position : match_index]
self.__position = match_index
return val
@ -117,7 +119,7 @@ class InputScanner:
def get_regexp(self, pattern, match_from=False):
result = None
# strings are converted to regexp
if isinstance(pattern, self.__six.string_types) and pattern != '':
if isinstance(pattern, self.__six.string_types) and pattern != "":
result = re.compile(pattern)
elif pattern is not None:
result = re.compile(pattern.pattern)
@ -132,5 +134,7 @@ class InputScanner:
def lookBack(self, testVal):
start = self.__position - 1
return start >= len(testVal) and \
self.__input[start - len(testVal):start].lower() == testVal
return (
start >= len(testVal)
and self.__input[start - len(testVal) : start].lower() == testVal
)

View File

@ -36,25 +36,25 @@ class Options:
self.raw_options = _mergeOpts(options, merge_child_field)
# Support passing the source text back with no change
self.disabled = self._get_boolean('disabled')
self.disabled = self._get_boolean("disabled")
self.eol = self._get_characters('eol', 'auto')
self.end_with_newline = self._get_boolean('end_with_newline')
self.indent_size = self._get_number('indent_size', 4)
self.indent_char = self._get_characters('indent_char', ' ')
self.indent_level = self._get_number('indent_level')
self.eol = self._get_characters("eol", "auto")
self.end_with_newline = self._get_boolean("end_with_newline")
self.indent_size = self._get_number("indent_size", 4)
self.indent_char = self._get_characters("indent_char", " ")
self.indent_level = self._get_number("indent_level")
self.preserve_newlines = self._get_boolean('preserve_newlines', True)
self.max_preserve_newlines = self._get_number(
'max_preserve_newlines', 32786)
self.preserve_newlines = self._get_boolean("preserve_newlines", True)
self.max_preserve_newlines = self._get_number("max_preserve_newlines", 32786)
if not self.preserve_newlines:
self.max_preserve_newlines = 0
self.indent_with_tabs = self._get_boolean(
'indent_with_tabs', self.indent_char == '\t')
"indent_with_tabs", self.indent_char == "\t"
)
if self.indent_with_tabs:
self.indent_char = '\t'
self.indent_char = "\t"
# indent_size behavior changed after 1.8.6
# It used to be that indent_size would be
@ -68,17 +68,19 @@ class Options:
# Backwards compat with 1.3.x
self.wrap_line_length = self._get_number(
'wrap_line_length', self._get_number('max_char'))
self.indent_empty_lines = self._get_boolean('indent_empty_lines')
"wrap_line_length", self._get_number("max_char")
)
self.indent_empty_lines = self._get_boolean("indent_empty_lines")
# valid templating languages ['django', 'erb', 'handlebars', 'php']
# For now, 'auto' = all off for javascript, all on for html (and inline javascript).
# other values ignored
self.templating = self._get_selection_list('templating',
['auto', 'none', 'django', 'erb', 'handlebars', 'php'], ['auto'])
self.templating = self._get_selection_list(
"templating",
["auto", "none", "django", "erb", "handlebars", "php"],
["auto"],
)
def _get_array(self, name, default_value=[]):
option_value = getattr(self.raw_options, name, default_value)
@ -100,12 +102,15 @@ class Options:
return result
def _get_characters(self, name, default_value=''):
def _get_characters(self, name, default_value=""):
option_value = getattr(self.raw_options, name, default_value)
result = ''
result = ""
if isinstance(option_value, str):
result = option_value.replace('\\r', '\r').replace(
'\\n', '\n').replace('\\t', '\t')
result = (
option_value.replace("\\r", "\r")
.replace("\\n", "\n")
.replace("\\t", "\t")
)
return result
@ -123,11 +128,14 @@ class Options:
result = self._get_selection_list(name, selection_list, default_value)
if len(result) != 1:
raise ValueError(
"Invalid Option Value: The option '" + name + "' can only be one of the following values:\n" +
str(selection_list) +
"\nYou passed in: '" +
str(getattr(self.raw_options, name, None)) +
"'")
"Invalid Option Value: The option '"
+ name
+ "' can only be one of the following values:\n"
+ str(selection_list)
+ "\nYou passed in: '"
+ str(getattr(self.raw_options, name, None))
+ "'"
)
return result[0]
@ -143,11 +151,14 @@ class Options:
result = self._get_array(name, default_value)
if not self._is_valid_selection(result, selection_list):
raise ValueError(
"Invalid Option Value: The option '" + name + "' can contain only the following values:\n" +
str(selection_list) +
"\nYou passed in: '" +
str(getattr(self.raw_options, name, None)) +
"'")
"Invalid Option Value: The option '"
+ name
+ "' can contain only the following values:\n"
+ str(selection_list)
+ "\nYou passed in: '"
+ str(getattr(self.raw_options, name, None))
+ "'"
)
return result
@ -181,11 +192,10 @@ def _mergeOpts(options, childFieldName):
if isinstance(options, dict):
local = finalOpts.get(childFieldName, None)
if local:
del(finalOpts[childFieldName])
del finalOpts[childFieldName]
for key in local:
finalOpts[key] = local[key]
finalOpts = namedtuple("CustomOptions", finalOpts.keys())(
*finalOpts.values())
finalOpts = namedtuple("CustomOptions", finalOpts.keys())(*finalOpts.values())
if isinstance(options, Options):
local = getattr(finalOpts, childFieldName, None)
@ -202,15 +212,16 @@ def _normalizeOpts(options):
if isinstance(convertedOpts, dict):
option_keys = list(convertedOpts.keys())
for key in option_keys:
if '-' in key:
if "-" in key:
del convertedOpts[key]
convertedOpts[key.replace('-', '_')] = options[key]
convertedOpts[key.replace("-", "_")] = options[key]
else:
option_keys = list(getattr(convertedOpts, '__dict__', {}))
option_keys = list(getattr(convertedOpts, "__dict__", {}))
for key in option_keys:
if '-' in key:
if "-" in key:
delattr(convertedOpts, key)
setattr(convertedOpts, key.replace(
'-', '_'), getattr(options, key, None))
setattr(
convertedOpts, key.replace("-", "_"), getattr(options, key, None)
)
return convertedOpts

View File

@ -60,36 +60,39 @@ class OutputLine:
self.__indent_count = indent
self.__alignment_count = alignment
self.__character_count = self.__parent.get_indent_size(
self.__indent_count, self.__alignment_count)
self.__indent_count, self.__alignment_count
)
def _set_wrap_point(self):
if self.__parent.wrap_line_length:
self.__wrap_point_index = len(self.__items)
self.__wrap_point_character_count = self.__character_count
self.__wrap_point_indent_count = \
self.__parent.next_line.__indent_count
self.__wrap_point_alignment_count = \
self.__wrap_point_indent_count = self.__parent.next_line.__indent_count
self.__wrap_point_alignment_count = (
self.__parent.next_line.__alignment_count
)
def _should_wrap(self):
return self.__wrap_point_index and \
self.__character_count > \
self.__parent.wrap_line_length and \
self.__wrap_point_character_count > \
self.__parent.next_line.__character_count
return (
self.__wrap_point_index
and self.__character_count > self.__parent.wrap_line_length
and self.__wrap_point_character_count
> self.__parent.next_line.__character_count
)
def _allow_wrap(self):
if self._should_wrap():
self.__parent.add_new_line()
next = self.__parent.current_line
next.set_indent(self.__wrap_point_indent_count,
self.__wrap_point_alignment_count)
next.__items = self.__items[self.__wrap_point_index:]
self.__items = self.__items[:self.__wrap_point_index]
next.set_indent(
self.__wrap_point_indent_count, self.__wrap_point_alignment_count
)
next.__items = self.__items[self.__wrap_point_index :]
self.__items = self.__items[: self.__wrap_point_index]
next.__character_count += self.__character_count - \
self.__wrap_point_character_count
next.__character_count += (
self.__character_count - self.__wrap_point_character_count
)
self.__character_count = self.__wrap_point_character_count
if next.__items[0] == " ":
@ -108,7 +111,7 @@ class OutputLine:
def push(self, item):
self.__items.append(item)
last_newline_index = item.rfind('\n')
last_newline_index = item.rfind("\n")
if last_newline_index != -1:
self.__character_count = len(item) - last_newline_index
else:
@ -131,32 +134,33 @@ class OutputLine:
self.__wrap_point_indent_count -= 1
def trim(self):
while self.last() == ' ':
while self.last() == " ":
self.__items.pop()
self.__character_count -= 1
def toString(self):
result = ''
result = ""
if self.is_empty():
if self.__parent.indent_empty_lines:
result = self.__parent.get_indent_string(self.__indent_count)
else:
result = self.__parent.get_indent_string(
self.__indent_count, self.__alignment_count)
result += ''.join(self.__items)
self.__indent_count, self.__alignment_count
)
result += "".join(self.__items)
return result
class IndentStringCache:
def __init__(self, options, base_string):
self.__cache = ['']
self.__cache = [""]
self.__indent_size = options.indent_size
self.__indent_string = options.indent_char
if not options.indent_with_tabs:
self.__indent_string = options.indent_char * options.indent_size
# Set to null to continue support of auto detection of base indent
base_string = base_string or ''
base_string = base_string or ""
if options.indent_level > 0:
base_string = options.indent_level * self.__indent_string
@ -175,7 +179,7 @@ class IndentStringCache:
result = self.__base_string
if indent_level < 0:
indent_level = 0
result = ''
result = ""
column += indent_level * self.__indent_size
self.__ensure_cache(column)
result += self.__cache[column]
@ -188,18 +192,18 @@ class IndentStringCache:
def __add_column(self):
column = len(self.__cache)
indent = 0
result = ''
result = ""
if self.__indent_size and column >= self.__indent_size:
indent = int(math.floor(column / self.__indent_size))
column -= indent * self.__indent_size
result = indent * self.__indent_string
if column:
result += column * ' '
result += column * " "
self.__cache.append(result)
class Output:
def __init__(self, options, baseIndentString=''):
def __init__(self, options, baseIndentString=""):
self.__indent_cache = IndentStringCache(options, baseIndentString)
self.raw = False
@ -237,8 +241,7 @@ class Output:
def add_new_line(self, force_newline=False):
# never newline at the start of file
# otherwise, newline only if we didn't just add one or we're forced
if self.is_empty() or \
(not force_newline and self.just_added_newline()):
if self.is_empty() or (not force_newline and self.just_added_newline()):
return False
# if raw output is enabled, don't print additional newlines,
@ -254,8 +257,8 @@ class Output:
# has text that ends with newline(s)
last_item = self.current_line.pop()
if last_item:
if last_item[-1] == '\n':
last_item = re.sub(r'[\n]+$', '', last_item)
if last_item[-1] == "\n":
last_item = re.sub(r"[\n]+$", "", last_item)
self.current_line.push(last_item)
if self._end_with_newline:
@ -263,8 +266,8 @@ class Output:
sweet_code = "\n".join(line.toString() for line in self.__lines)
if not eol == '\n':
sweet_code = sweet_code.replace('\n', eol)
if not eol == "\n":
sweet_code = sweet_code.replace("\n", eol)
return sweet_code
@ -304,7 +307,7 @@ class Output:
if self.space_before_token and not self.just_added_newline():
if not self.non_breaking_space:
self.set_wrap_point()
self.current_line.push(' ')
self.current_line.push(" ")
self.space_before_token = False
def remove_indent(self, index):
@ -316,8 +319,7 @@ class Output:
def trim(self, eat_newlines=False):
self.current_line.trim()
while eat_newlines and len(
self.__lines) > 1 and self.current_line.is_empty():
while eat_newlines and len(self.__lines) > 1 and self.current_line.is_empty():
self.__lines.pop()
self.current_line = self.__lines[-1]
self.current_line.trim()
@ -331,8 +333,9 @@ class Output:
return self.current_line.is_empty()
def just_added_blankline(self):
return self.is_empty() or \
(self.current_line.is_empty() and self.previous_line.is_empty())
return self.is_empty() or (
self.current_line.is_empty() and self.previous_line.is_empty()
)
def ensure_empty_line_above(self, starts_with, ends_with):
index = len(self.__lines) - 2
@ -340,8 +343,10 @@ class Output:
potentialEmptyLine = self.__lines[index]
if potentialEmptyLine.is_empty():
break
elif not potentialEmptyLine.item(0).startswith(starts_with) and \
potentialEmptyLine.item(-1) != ends_with:
elif (
not potentialEmptyLine.item(0).startswith(starts_with)
and potentialEmptyLine.item(-1) != ends_with
):
self.__lines.insert(index + 1, OutputLine(self))
self.previous_line = self.__lines[-2]
break

View File

@ -24,6 +24,7 @@
__all__ = ["Pattern"]
class Pattern:
def __init__(self, input_scanner, parent=None):
self._input = input_scanner
@ -41,8 +42,9 @@ class Pattern:
def read(self):
result = self._input.read(self._starting_pattern)
if (self._starting_pattern is None) or result:
result += self._input.read(self._match_pattern,
self._until_pattern, self._until_after)
result += self._input.read(
self._match_pattern, self._until_pattern, self._until_after
)
return result
def read_match(self):
@ -79,4 +81,3 @@ class Pattern:
def _update(self):
pass

View File

@ -27,6 +27,7 @@ from ..core.pattern import Pattern
__all__ = ["TemplatablePattern"]
class TemplateNames:
def __init__(self):
self.django = False
@ -34,21 +35,22 @@ class TemplateNames:
self.handlebars = False
self.php = False
class TemplatePatterns:
def __init__(self, input_scanner):
pattern = Pattern(input_scanner)
self.handlebars_comment = pattern.starting_with(r'{{!--').until_after(r'--}}')
self.handlebars_unescaped = pattern.starting_with(r'{{{').until_after(r'}}}')
self.handlebars = pattern.starting_with(r'{{').until_after(r'}}')
self.php = pattern.starting_with(r'<\?(?:[=]|php)').until_after(r'\?>')
self.erb = pattern.starting_with(r'<%[^%]').until_after(r'[^%]%>')
self.handlebars_comment = pattern.starting_with(r"{{!--").until_after(r"--}}")
self.handlebars_unescaped = pattern.starting_with(r"{{{").until_after(r"}}}")
self.handlebars = pattern.starting_with(r"{{").until_after(r"}}")
self.php = pattern.starting_with(r"<\?(?:[=]|php)").until_after(r"\?>")
self.erb = pattern.starting_with(r"<%[^%]").until_after(r"[^%]%>")
# django coflicts with handlebars a bit.
self.django = pattern.starting_with(r'{%').until_after(r'%}')
self.django_value = pattern.starting_with(r'{{').until_after(r'}}')
self.django_comment = pattern.starting_with(r'{#').until_after(r'#}')
self.django = pattern.starting_with(r"{%").until_after(r"%}")
self.django_value = pattern.starting_with(r"{{").until_after(r"}}")
self.django_comment = pattern.starting_with(r"{#").until_after(r"#}")
class TemplatablePattern(Pattern):
def __init__(self, input_scanner, parent=None):
Pattern.__init__(self, input_scanner, parent)
self.__template_pattern = None
@ -56,8 +58,7 @@ class TemplatablePattern(Pattern):
self._excluded = TemplateNames()
if parent is not None:
self.__template_pattern = \
self._input.get_regexp(parent.__template_pattern)
self.__template_pattern = self._input.get_regexp(parent.__template_pattern)
self._disabled = copy.copy(parent._disabled)
self._excluded = copy.copy(parent._excluded)
@ -71,9 +72,8 @@ class TemplatablePattern(Pattern):
def read_options(self, options):
result = self._create()
for language in ['django', 'erb', 'handlebars', 'php']:
setattr(result._disabled, language,
not (language in options.templating))
for language in ["django", "erb", "handlebars", "php"]:
setattr(result._disabled, language, not (language in options.templating))
result._update()
return result
@ -90,16 +90,15 @@ class TemplatablePattern(Pattern):
return result
def read(self):
result = ''
result = ""
if bool(self._match_pattern):
result = self._input.read(self._starting_pattern)
else:
result = self._input.read(self._starting_pattern,
self.__template_pattern)
result = self._input.read(self._starting_pattern, self.__template_pattern)
next = self._read_template()
while (bool(next)):
while bool(next):
if self._match_pattern is not None:
next += self._input.read(self._match_pattern)
else:
@ -133,45 +132,38 @@ class TemplatablePattern(Pattern):
if self._until_pattern:
items.append(self._until_pattern.pattern)
self.__template_pattern = self._input.get_regexp(
r'(?:' + '|'.join(items) + ')')
self.__template_pattern = self._input.get_regexp(r"(?:" + "|".join(items) + ")")
def _read_template(self):
resulting_string = ''
resulting_string = ""
c = self._input.peek()
if c == '<':
if c == "<":
peek1 = self._input.peek(1)
if not self._disabled.php and \
not self._excluded.php and \
peek1 == '?':
resulting_string = resulting_string or \
self.__patterns.php.read()
if not self._disabled.php and not self._excluded.php and peek1 == "?":
resulting_string = resulting_string or self.__patterns.php.read()
if not self._disabled.erb and \
not self._excluded.erb and \
peek1 == '%':
resulting_string = resulting_string or \
self.__patterns.erb.read()
elif c == '{':
if not self._disabled.handlebars and \
not self._excluded.handlebars:
resulting_string = resulting_string or \
self.__patterns.handlebars_comment.read()
resulting_string = resulting_string or \
self.__patterns.handlebars_unescaped.read()
resulting_string = resulting_string or \
self.__patterns.handlebars.read()
if not self._disabled.erb and not self._excluded.erb and peek1 == "%":
resulting_string = resulting_string or self.__patterns.erb.read()
elif c == "{":
if not self._disabled.handlebars and not self._excluded.handlebars:
resulting_string = (
resulting_string or self.__patterns.handlebars_comment.read()
)
resulting_string = (
resulting_string or self.__patterns.handlebars_unescaped.read()
)
resulting_string = resulting_string or self.__patterns.handlebars.read()
if not self._disabled.django:
# django coflicts with handlebars a bit.
if not self._excluded.django and \
not self._excluded.handlebars:
resulting_string = resulting_string or \
self.__patterns.django_value.read()
if not self._excluded.django and not self._excluded.handlebars:
resulting_string = (
resulting_string or self.__patterns.django_value.read()
)
if not self._excluded.django:
resulting_string = resulting_string or \
self.__patterns.django_comment.read()
resulting_string = resulting_string or \
self.__patterns.django.read()
resulting_string = (
resulting_string or self.__patterns.django_comment.read()
)
resulting_string = resulting_string or self.__patterns.django.read()
return resulting_string

View File

@ -24,12 +24,7 @@
class Token:
def __init__(
self,
type,
text,
newlines=0,
whitespace_before=''):
def __init__(self, type, text, newlines=0, whitespace_before=""):
self.type = type
self.text = text
self.comments_before = None

View File

@ -31,24 +31,25 @@ from ..core.whitespacepattern import WhitespacePattern
__all__ = ["TOKEN", "Tokenizer", "TokenizerPatterns", "TokenTypes"]
class TokenTypes:
START = 'TK_START'
RAW = 'TK_RAW'
EOF = 'TK_EOF'
START = "TK_START"
RAW = "TK_RAW"
EOF = "TK_EOF"
def __init__(self):
pass
TOKEN = TokenTypes()
class TokenizerPatterns:
def __init__(self, input_scanner):
self.whitespace = WhitespacePattern(input_scanner)
class Tokenizer:
def __init__(self, input_string, options):
self._input = InputScanner(input_string)
self._options = options
@ -61,7 +62,7 @@ class Tokenizer:
self.__tokens = TokenStream()
current = None
previous = Token(TOKEN.START,'')
previous = Token(TOKEN.START, "")
open_token = None
open_stack = []
comments = TokenStream()
@ -72,8 +73,7 @@ class Tokenizer:
if self._is_opening(current):
open_stack.append(open_token)
open_token = current
elif open_token is not None and \
self._is_closing(current, open_token):
elif open_token is not None and self._is_closing(current, open_token):
current.opened = open_token
open_token.closed = current
open_token = open_stack.pop()
@ -110,11 +110,11 @@ class Tokenizer:
def _get_next_token(self, previous_token, open_token):
self._readWhitespace()
resulting_string = self._input.read(re.compile(r'.+'))
resulting_string = self._input.read(re.compile(r".+"))
if resulting_string:
return self._create_token(TOKEN.RAW, resulting_string)
else:
return self._create_token(TOKEN.EOF, '')
return self._create_token(TOKEN.EOF, "")
def _is_comment(self, current_token):
return False
@ -126,9 +126,12 @@ class Tokenizer:
return False
def _create_token(self, token_type, text):
token = Token(token_type, text,
self._patterns.whitespace.newline_count,
self._patterns.whitespace.whitespace_before_token)
token = Token(
token_type,
text,
self._patterns.whitespace.newline_count,
self._patterns.whitespace.whitespace_before_token,
)
return token
def _readWhitespace(self):

View File

@ -26,8 +26,8 @@ import re
from ..core.inputscanner import InputScanner
from ..core.token import Token
class TokenStream:
class TokenStream:
def __init__(self, parent_token=None):
self.__tokens = []
self.__tokens_length = len(self.__tokens)

View File

@ -33,31 +33,29 @@ class WhitespacePattern(Pattern):
Pattern.__init__(self, input_scanner, parent)
if parent is not None:
self._newline_regexp = \
self._input.get_regexp(parent._newline_regexp)
self._newline_regexp = self._input.get_regexp(parent._newline_regexp)
else:
self.__set_whitespace_patterns('', '')
self.__set_whitespace_patterns("", "")
self.newline_count = 0
self.whitespace_before_token = ''
self.whitespace_before_token = ""
def __set_whitespace_patterns(self, whitespace_chars, newline_chars):
whitespace_chars += '\\t '
newline_chars += '\\n\\r'
whitespace_chars += "\\t "
newline_chars += "\\n\\r"
self._match_pattern = self._input.get_regexp(
'[' + whitespace_chars + newline_chars + ']+')
self._newline_regexp = self._input.get_regexp(
'\\r\\n|[' + newline_chars + ']')
"[" + whitespace_chars + newline_chars + "]+"
)
self._newline_regexp = self._input.get_regexp("\\r\\n|[" + newline_chars + "]")
def read(self):
self.newline_count = 0
self.whitespace_before_token = ''
self.whitespace_before_token = ""
resulting_string = self._input.read(self._match_pattern)
if resulting_string == ' ':
self.whitespace_before_token = ' '
if resulting_string == " ":
self.whitespace_before_token = " "
elif bool(resulting_string):
lines = self._newline_regexp.split(resulting_string)
self.newline_count = len(lines) - 1
@ -65,7 +63,6 @@ class WhitespacePattern(Pattern):
return resulting_string
def matching(self, whitespace_chars, newline_chars):
result = self._create()
result.__set_whitespace_patterns(whitespace_chars, newline_chars)
@ -74,5 +71,3 @@ class WhitespacePattern(Pattern):
def _create(self):
return WhitespacePattern(self._input, self)

View File

@ -34,32 +34,43 @@ _baseASCIIidentifierChars = six.u(r"\x24\x30-\x39\x41-\x5a\x5f\x61-\x7a")
# are only applied when a character is found to actually have a
# code point above 128.
# IMPORTANT: These strings must be run through six to handle \u chars
_nonASCIIidentifierStartChars = six.u(r"\xaa\xb5\xba\xc0-\xd6\xd8-\xf6\xf8-\u02c1\u02c6-\u02d1\u02e0-\u02e4\u02ec\u02ee\u0370-\u0374\u0376\u0377\u037a-\u037d\u0386\u0388-\u038a\u038c\u038e-\u03a1\u03a3-\u03f5\u03f7-\u0481\u048a-\u0527\u0531-\u0556\u0559\u0561-\u0587\u05d0-\u05ea\u05f0-\u05f2\u0620-\u064a\u066e\u066f\u0671-\u06d3\u06d5\u06e5\u06e6\u06ee\u06ef\u06fa-\u06fc\u06ff\u0710\u0712-\u072f\u074d-\u07a5\u07b1\u07ca-\u07ea\u07f4\u07f5\u07fa\u0800-\u0815\u081a\u0824\u0828\u0840-\u0858\u08a0\u08a2-\u08ac\u0904-\u0939\u093d\u0950\u0958-\u0961\u0971-\u0977\u0979-\u097f\u0985-\u098c\u098f\u0990\u0993-\u09a8\u09aa-\u09b0\u09b2\u09b6-\u09b9\u09bd\u09ce\u09dc\u09dd\u09df-\u09e1\u09f0\u09f1\u0a05-\u0a0a\u0a0f\u0a10\u0a13-\u0a28\u0a2a-\u0a30\u0a32\u0a33\u0a35\u0a36\u0a38\u0a39\u0a59-\u0a5c\u0a5e\u0a72-\u0a74\u0a85-\u0a8d\u0a8f-\u0a91\u0a93-\u0aa8\u0aaa-\u0ab0\u0ab2\u0ab3\u0ab5-\u0ab9\u0abd\u0ad0\u0ae0\u0ae1\u0b05-\u0b0c\u0b0f\u0b10\u0b13-\u0b28\u0b2a-\u0b30\u0b32\u0b33\u0b35-\u0b39\u0b3d\u0b5c\u0b5d\u0b5f-\u0b61\u0b71\u0b83\u0b85-\u0b8a\u0b8e-\u0b90\u0b92-\u0b95\u0b99\u0b9a\u0b9c\u0b9e\u0b9f\u0ba3\u0ba4\u0ba8-\u0baa\u0bae-\u0bb9\u0bd0\u0c05-\u0c0c\u0c0e-\u0c10\u0c12-\u0c28\u0c2a-\u0c33\u0c35-\u0c39\u0c3d\u0c58\u0c59\u0c60\u0c61\u0c85-\u0c8c\u0c8e-\u0c90\u0c92-\u0ca8\u0caa-\u0cb3\u0cb5-\u0cb9\u0cbd\u0cde\u0ce0\u0ce1\u0cf1\u0cf2\u0d05-\u0d0c\u0d0e-\u0d10\u0d12-\u0d3a\u0d3d\u0d4e\u0d60\u0d61\u0d7a-\u0d7f\u0d85-\u0d96\u0d9a-\u0db1\u0db3-\u0dbb\u0dbd\u0dc0-\u0dc6\u0e01-\u0e30\u0e32\u0e33\u0e40-\u0e46\u0e81\u0e82\u0e84\u0e87\u0e88\u0e8a\u0e8d\u0e94-\u0e97\u0e99-\u0e9f\u0ea1-\u0ea3\u0ea5\u0ea7\u0eaa\u0eab\u0ead-\u0eb0\u0eb2\u0eb3\u0ebd\u0ec0-\u0ec4\u0ec6\u0edc-\u0edf\u0f00\u0f40-\u0f47\u0f49-\u0f6c\u0f88-\u0f8c\u1000-\u102a\u103f\u1050-\u1055\u105a-\u105d\u1061\u1065\u1066\u106e-\u1070\u1075-\u1081\u108e\u10a0-\u10c5\u10c7\u10cd\u10d0-\u10fa\u10fc-\u1248\u124a-\u124d\u1250-\u1256\u1258\u125a-\u125d\u1260-\u1288\u128a-\u128d\u1290-\u12b0\u12b2-\u12b5\u12b8-\u12be\u12c0\u12c2-\u12c5\u12c8-\u12d6\u12d8-\u1310\u1312-\u1315\u1318-\u135a\u1380-\u138f\u13a0-\u13f4\u1401-\u166c\u166f-\u167f\u1681-\u169a\u16a0-\u16ea\u16ee-\u16f0\u1700-\u170c\u170e-\u1711\u1720-\u1731\u1740-\u1751\u1760-\u176c\u176e-\u1770\u1780-\u17b3\u17d7\u17dc\u1820-\u1877\u1880-\u18a8\u18aa\u18b0-\u18f5\u1900-\u191c\u1950-\u196d\u1970-\u1974\u1980-\u19ab\u19c1-\u19c7\u1a00-\u1a16\u1a20-\u1a54\u1aa7\u1b05-\u1b33\u1b45-\u1b4b\u1b83-\u1ba0\u1bae\u1baf\u1bba-\u1be5\u1c00-\u1c23\u1c4d-\u1c4f\u1c5a-\u1c7d\u1ce9-\u1cec\u1cee-\u1cf1\u1cf5\u1cf6\u1d00-\u1dbf\u1e00-\u1f15\u1f18-\u1f1d\u1f20-\u1f45\u1f48-\u1f4d\u1f50-\u1f57\u1f59\u1f5b\u1f5d\u1f5f-\u1f7d\u1f80-\u1fb4\u1fb6-\u1fbc\u1fbe\u1fc2-\u1fc4\u1fc6-\u1fcc\u1fd0-\u1fd3\u1fd6-\u1fdb\u1fe0-\u1fec\u1ff2-\u1ff4\u1ff6-\u1ffc\u2071\u207f\u2090-\u209c\u2102\u2107\u210a-\u2113\u2115\u2119-\u211d\u2124\u2126\u2128\u212a-\u212d\u212f-\u2139\u213c-\u213f\u2145-\u2149\u214e\u2160-\u2188\u2c00-\u2c2e\u2c30-\u2c5e\u2c60-\u2ce4\u2ceb-\u2cee\u2cf2\u2cf3\u2d00-\u2d25\u2d27\u2d2d\u2d30-\u2d67\u2d6f\u2d80-\u2d96\u2da0-\u2da6\u2da8-\u2dae\u2db0-\u2db6\u2db8-\u2dbe\u2dc0-\u2dc6\u2dc8-\u2dce\u2dd0-\u2dd6\u2dd8-\u2dde\u2e2f\u3005-\u3007\u3021-\u3029\u3031-\u3035\u3038-\u303c\u3041-\u3096\u309d-\u309f\u30a1-\u30fa\u30fc-\u30ff\u3105-\u312d\u3131-\u318e\u31a0-\u31ba\u31f0-\u31ff\u3400-\u4db5\u4e00-\u9fcc\ua000-\ua48c\ua4d0-\ua4fd\ua500-\ua60c\ua610-\ua61f\ua62a\ua62b\ua640-\ua66e\ua67f-\ua697\ua6a0-\ua6ef\ua717-\ua71f\ua722-\ua788\ua78b-\ua78e\ua790-\ua793\ua7a0-\ua7aa\ua7f8-\ua801\ua803-\ua805\ua807-\ua80a\ua80c-\ua822\ua840-\ua873\ua882-\ua8b3\ua8f2-\ua8f7\ua8fb\ua90a-\ua925\ua930-\ua946\ua960-\ua97c\ua984-\ua9b2\ua9cf\uaa00-\uaa28\uaa40-\uaa42\uaa44-\uaa4b\uaa60-\uaa76\uaa7a\uaa80-\uaaaf\uaab1\uaab5\uaab6\uaab9-\uaabd\uaac0\uaac2\uaadb-\uaadd\uaae0-\uaaea\uaaf2-\uaaf4\uab01-\uab06\uab09-\uab0e\uab11-\uab16\uab20-\uab26\uab28-\uab2e\uabc0-\uabe2\uac00-\ud7a3\ud7b0-\ud7c6\ud7cb-\ud7fb\uf900-\ufa6d\ufa70-\ufad9\ufb00-\ufb06\ufb13-\ufb17\ufb1d\ufb1f-\ufb28\ufb2a-\ufb36\ufb38-\ufb3c\ufb3e\ufb40\ufb41\ufb43\ufb44\ufb46-\ufbb1\ufbd3-\ufd3d\ufd50-\ufd8f\ufd92-\ufdc7\ufdf0-\ufdfb\ufe70-\ufe74\ufe76-\ufefc\uff21-\uff3a\uff41-\uff5a\uff66-\uffbe\uffc2-\uffc7\uffca-\uffcf\uffd2-\uffd7\uffda-\uffdc")
_nonASCIIidentifierChars = six.u(r"\u0300-\u036f\u0483-\u0487\u0591-\u05bd\u05bf\u05c1\u05c2\u05c4\u05c5\u05c7\u0610-\u061a\u0620-\u0649\u0672-\u06d3\u06e7-\u06e8\u06fb-\u06fc\u0730-\u074a\u0800-\u0814\u081b-\u0823\u0825-\u0827\u0829-\u082d\u0840-\u0857\u08e4-\u08fe\u0900-\u0903\u093a-\u093c\u093e-\u094f\u0951-\u0957\u0962-\u0963\u0966-\u096f\u0981-\u0983\u09bc\u09be-\u09c4\u09c7\u09c8\u09d7\u09df-\u09e0\u0a01-\u0a03\u0a3c\u0a3e-\u0a42\u0a47\u0a48\u0a4b-\u0a4d\u0a51\u0a66-\u0a71\u0a75\u0a81-\u0a83\u0abc\u0abe-\u0ac5\u0ac7-\u0ac9\u0acb-\u0acd\u0ae2-\u0ae3\u0ae6-\u0aef\u0b01-\u0b03\u0b3c\u0b3e-\u0b44\u0b47\u0b48\u0b4b-\u0b4d\u0b56\u0b57\u0b5f-\u0b60\u0b66-\u0b6f\u0b82\u0bbe-\u0bc2\u0bc6-\u0bc8\u0bca-\u0bcd\u0bd7\u0be6-\u0bef\u0c01-\u0c03\u0c46-\u0c48\u0c4a-\u0c4d\u0c55\u0c56\u0c62-\u0c63\u0c66-\u0c6f\u0c82\u0c83\u0cbc\u0cbe-\u0cc4\u0cc6-\u0cc8\u0cca-\u0ccd\u0cd5\u0cd6\u0ce2-\u0ce3\u0ce6-\u0cef\u0d02\u0d03\u0d46-\u0d48\u0d57\u0d62-\u0d63\u0d66-\u0d6f\u0d82\u0d83\u0dca\u0dcf-\u0dd4\u0dd6\u0dd8-\u0ddf\u0df2\u0df3\u0e34-\u0e3a\u0e40-\u0e45\u0e50-\u0e59\u0eb4-\u0eb9\u0ec8-\u0ecd\u0ed0-\u0ed9\u0f18\u0f19\u0f20-\u0f29\u0f35\u0f37\u0f39\u0f41-\u0f47\u0f71-\u0f84\u0f86-\u0f87\u0f8d-\u0f97\u0f99-\u0fbc\u0fc6\u1000-\u1029\u1040-\u1049\u1067-\u106d\u1071-\u1074\u1082-\u108d\u108f-\u109d\u135d-\u135f\u170e-\u1710\u1720-\u1730\u1740-\u1750\u1772\u1773\u1780-\u17b2\u17dd\u17e0-\u17e9\u180b-\u180d\u1810-\u1819\u1920-\u192b\u1930-\u193b\u1951-\u196d\u19b0-\u19c0\u19c8-\u19c9\u19d0-\u19d9\u1a00-\u1a15\u1a20-\u1a53\u1a60-\u1a7c\u1a7f-\u1a89\u1a90-\u1a99\u1b46-\u1b4b\u1b50-\u1b59\u1b6b-\u1b73\u1bb0-\u1bb9\u1be6-\u1bf3\u1c00-\u1c22\u1c40-\u1c49\u1c5b-\u1c7d\u1cd0-\u1cd2\u1d00-\u1dbe\u1e01-\u1f15\u200c\u200d\u203f\u2040\u2054\u20d0-\u20dc\u20e1\u20e5-\u20f0\u2d81-\u2d96\u2de0-\u2dff\u3021-\u3028\u3099\u309a\ua640-\ua66d\ua674-\ua67d\ua69f\ua6f0-\ua6f1\ua7f8-\ua800\ua806\ua80b\ua823-\ua827\ua880-\ua881\ua8b4-\ua8c4\ua8d0-\ua8d9\ua8f3-\ua8f7\ua900-\ua909\ua926-\ua92d\ua930-\ua945\ua980-\ua983\ua9b3-\ua9c0\uaa00-\uaa27\uaa40-\uaa41\uaa4c-\uaa4d\uaa50-\uaa59\uaa7b\uaae0-\uaae9\uaaf2-\uaaf3\uabc0-\uabe1\uabec\uabed\uabf0-\uabf9\ufb20-\ufb28\ufe00-\ufe0f\ufe20-\ufe26\ufe33\ufe34\ufe4d-\ufe4f\uff10-\uff19\uff3f")
#_nonASCIIidentifierStart = re.compile("[" + _nonASCIIidentifierStartChars + "]")
#_nonASCIIidentifier = re.compile("[" + _nonASCIIidentifierStartChars + _nonASCIIidentifierChars + "]")
_nonASCIIidentifierStartChars = six.u(
r"\xaa\xb5\xba\xc0-\xd6\xd8-\xf6\xf8-\u02c1\u02c6-\u02d1\u02e0-\u02e4\u02ec\u02ee\u0370-\u0374\u0376\u0377\u037a-\u037d\u0386\u0388-\u038a\u038c\u038e-\u03a1\u03a3-\u03f5\u03f7-\u0481\u048a-\u0527\u0531-\u0556\u0559\u0561-\u0587\u05d0-\u05ea\u05f0-\u05f2\u0620-\u064a\u066e\u066f\u0671-\u06d3\u06d5\u06e5\u06e6\u06ee\u06ef\u06fa-\u06fc\u06ff\u0710\u0712-\u072f\u074d-\u07a5\u07b1\u07ca-\u07ea\u07f4\u07f5\u07fa\u0800-\u0815\u081a\u0824\u0828\u0840-\u0858\u08a0\u08a2-\u08ac\u0904-\u0939\u093d\u0950\u0958-\u0961\u0971-\u0977\u0979-\u097f\u0985-\u098c\u098f\u0990\u0993-\u09a8\u09aa-\u09b0\u09b2\u09b6-\u09b9\u09bd\u09ce\u09dc\u09dd\u09df-\u09e1\u09f0\u09f1\u0a05-\u0a0a\u0a0f\u0a10\u0a13-\u0a28\u0a2a-\u0a30\u0a32\u0a33\u0a35\u0a36\u0a38\u0a39\u0a59-\u0a5c\u0a5e\u0a72-\u0a74\u0a85-\u0a8d\u0a8f-\u0a91\u0a93-\u0aa8\u0aaa-\u0ab0\u0ab2\u0ab3\u0ab5-\u0ab9\u0abd\u0ad0\u0ae0\u0ae1\u0b05-\u0b0c\u0b0f\u0b10\u0b13-\u0b28\u0b2a-\u0b30\u0b32\u0b33\u0b35-\u0b39\u0b3d\u0b5c\u0b5d\u0b5f-\u0b61\u0b71\u0b83\u0b85-\u0b8a\u0b8e-\u0b90\u0b92-\u0b95\u0b99\u0b9a\u0b9c\u0b9e\u0b9f\u0ba3\u0ba4\u0ba8-\u0baa\u0bae-\u0bb9\u0bd0\u0c05-\u0c0c\u0c0e-\u0c10\u0c12-\u0c28\u0c2a-\u0c33\u0c35-\u0c39\u0c3d\u0c58\u0c59\u0c60\u0c61\u0c85-\u0c8c\u0c8e-\u0c90\u0c92-\u0ca8\u0caa-\u0cb3\u0cb5-\u0cb9\u0cbd\u0cde\u0ce0\u0ce1\u0cf1\u0cf2\u0d05-\u0d0c\u0d0e-\u0d10\u0d12-\u0d3a\u0d3d\u0d4e\u0d60\u0d61\u0d7a-\u0d7f\u0d85-\u0d96\u0d9a-\u0db1\u0db3-\u0dbb\u0dbd\u0dc0-\u0dc6\u0e01-\u0e30\u0e32\u0e33\u0e40-\u0e46\u0e81\u0e82\u0e84\u0e87\u0e88\u0e8a\u0e8d\u0e94-\u0e97\u0e99-\u0e9f\u0ea1-\u0ea3\u0ea5\u0ea7\u0eaa\u0eab\u0ead-\u0eb0\u0eb2\u0eb3\u0ebd\u0ec0-\u0ec4\u0ec6\u0edc-\u0edf\u0f00\u0f40-\u0f47\u0f49-\u0f6c\u0f88-\u0f8c\u1000-\u102a\u103f\u1050-\u1055\u105a-\u105d\u1061\u1065\u1066\u106e-\u1070\u1075-\u1081\u108e\u10a0-\u10c5\u10c7\u10cd\u10d0-\u10fa\u10fc-\u1248\u124a-\u124d\u1250-\u1256\u1258\u125a-\u125d\u1260-\u1288\u128a-\u128d\u1290-\u12b0\u12b2-\u12b5\u12b8-\u12be\u12c0\u12c2-\u12c5\u12c8-\u12d6\u12d8-\u1310\u1312-\u1315\u1318-\u135a\u1380-\u138f\u13a0-\u13f4\u1401-\u166c\u166f-\u167f\u1681-\u169a\u16a0-\u16ea\u16ee-\u16f0\u1700-\u170c\u170e-\u1711\u1720-\u1731\u1740-\u1751\u1760-\u176c\u176e-\u1770\u1780-\u17b3\u17d7\u17dc\u1820-\u1877\u1880-\u18a8\u18aa\u18b0-\u18f5\u1900-\u191c\u1950-\u196d\u1970-\u1974\u1980-\u19ab\u19c1-\u19c7\u1a00-\u1a16\u1a20-\u1a54\u1aa7\u1b05-\u1b33\u1b45-\u1b4b\u1b83-\u1ba0\u1bae\u1baf\u1bba-\u1be5\u1c00-\u1c23\u1c4d-\u1c4f\u1c5a-\u1c7d\u1ce9-\u1cec\u1cee-\u1cf1\u1cf5\u1cf6\u1d00-\u1dbf\u1e00-\u1f15\u1f18-\u1f1d\u1f20-\u1f45\u1f48-\u1f4d\u1f50-\u1f57\u1f59\u1f5b\u1f5d\u1f5f-\u1f7d\u1f80-\u1fb4\u1fb6-\u1fbc\u1fbe\u1fc2-\u1fc4\u1fc6-\u1fcc\u1fd0-\u1fd3\u1fd6-\u1fdb\u1fe0-\u1fec\u1ff2-\u1ff4\u1ff6-\u1ffc\u2071\u207f\u2090-\u209c\u2102\u2107\u210a-\u2113\u2115\u2119-\u211d\u2124\u2126\u2128\u212a-\u212d\u212f-\u2139\u213c-\u213f\u2145-\u2149\u214e\u2160-\u2188\u2c00-\u2c2e\u2c30-\u2c5e\u2c60-\u2ce4\u2ceb-\u2cee\u2cf2\u2cf3\u2d00-\u2d25\u2d27\u2d2d\u2d30-\u2d67\u2d6f\u2d80-\u2d96\u2da0-\u2da6\u2da8-\u2dae\u2db0-\u2db6\u2db8-\u2dbe\u2dc0-\u2dc6\u2dc8-\u2dce\u2dd0-\u2dd6\u2dd8-\u2dde\u2e2f\u3005-\u3007\u3021-\u3029\u3031-\u3035\u3038-\u303c\u3041-\u3096\u309d-\u309f\u30a1-\u30fa\u30fc-\u30ff\u3105-\u312d\u3131-\u318e\u31a0-\u31ba\u31f0-\u31ff\u3400-\u4db5\u4e00-\u9fcc\ua000-\ua48c\ua4d0-\ua4fd\ua500-\ua60c\ua610-\ua61f\ua62a\ua62b\ua640-\ua66e\ua67f-\ua697\ua6a0-\ua6ef\ua717-\ua71f\ua722-\ua788\ua78b-\ua78e\ua790-\ua793\ua7a0-\ua7aa\ua7f8-\ua801\ua803-\ua805\ua807-\ua80a\ua80c-\ua822\ua840-\ua873\ua882-\ua8b3\ua8f2-\ua8f7\ua8fb\ua90a-\ua925\ua930-\ua946\ua960-\ua97c\ua984-\ua9b2\ua9cf\uaa00-\uaa28\uaa40-\uaa42\uaa44-\uaa4b\uaa60-\uaa76\uaa7a\uaa80-\uaaaf\uaab1\uaab5\uaab6\uaab9-\uaabd\uaac0\uaac2\uaadb-\uaadd\uaae0-\uaaea\uaaf2-\uaaf4\uab01-\uab06\uab09-\uab0e\uab11-\uab16\uab20-\uab26\uab28-\uab2e\uabc0-\uabe2\uac00-\ud7a3\ud7b0-\ud7c6\ud7cb-\ud7fb\uf900-\ufa6d\ufa70-\ufad9\ufb00-\ufb06\ufb13-\ufb17\ufb1d\ufb1f-\ufb28\ufb2a-\ufb36\ufb38-\ufb3c\ufb3e\ufb40\ufb41\ufb43\ufb44\ufb46-\ufbb1\ufbd3-\ufd3d\ufd50-\ufd8f\ufd92-\ufdc7\ufdf0-\ufdfb\ufe70-\ufe74\ufe76-\ufefc\uff21-\uff3a\uff41-\uff5a\uff66-\uffbe\uffc2-\uffc7\uffca-\uffcf\uffd2-\uffd7\uffda-\uffdc"
)
_nonASCIIidentifierChars = six.u(
r"\u0300-\u036f\u0483-\u0487\u0591-\u05bd\u05bf\u05c1\u05c2\u05c4\u05c5\u05c7\u0610-\u061a\u0620-\u0649\u0672-\u06d3\u06e7-\u06e8\u06fb-\u06fc\u0730-\u074a\u0800-\u0814\u081b-\u0823\u0825-\u0827\u0829-\u082d\u0840-\u0857\u08e4-\u08fe\u0900-\u0903\u093a-\u093c\u093e-\u094f\u0951-\u0957\u0962-\u0963\u0966-\u096f\u0981-\u0983\u09bc\u09be-\u09c4\u09c7\u09c8\u09d7\u09df-\u09e0\u0a01-\u0a03\u0a3c\u0a3e-\u0a42\u0a47\u0a48\u0a4b-\u0a4d\u0a51\u0a66-\u0a71\u0a75\u0a81-\u0a83\u0abc\u0abe-\u0ac5\u0ac7-\u0ac9\u0acb-\u0acd\u0ae2-\u0ae3\u0ae6-\u0aef\u0b01-\u0b03\u0b3c\u0b3e-\u0b44\u0b47\u0b48\u0b4b-\u0b4d\u0b56\u0b57\u0b5f-\u0b60\u0b66-\u0b6f\u0b82\u0bbe-\u0bc2\u0bc6-\u0bc8\u0bca-\u0bcd\u0bd7\u0be6-\u0bef\u0c01-\u0c03\u0c46-\u0c48\u0c4a-\u0c4d\u0c55\u0c56\u0c62-\u0c63\u0c66-\u0c6f\u0c82\u0c83\u0cbc\u0cbe-\u0cc4\u0cc6-\u0cc8\u0cca-\u0ccd\u0cd5\u0cd6\u0ce2-\u0ce3\u0ce6-\u0cef\u0d02\u0d03\u0d46-\u0d48\u0d57\u0d62-\u0d63\u0d66-\u0d6f\u0d82\u0d83\u0dca\u0dcf-\u0dd4\u0dd6\u0dd8-\u0ddf\u0df2\u0df3\u0e34-\u0e3a\u0e40-\u0e45\u0e50-\u0e59\u0eb4-\u0eb9\u0ec8-\u0ecd\u0ed0-\u0ed9\u0f18\u0f19\u0f20-\u0f29\u0f35\u0f37\u0f39\u0f41-\u0f47\u0f71-\u0f84\u0f86-\u0f87\u0f8d-\u0f97\u0f99-\u0fbc\u0fc6\u1000-\u1029\u1040-\u1049\u1067-\u106d\u1071-\u1074\u1082-\u108d\u108f-\u109d\u135d-\u135f\u170e-\u1710\u1720-\u1730\u1740-\u1750\u1772\u1773\u1780-\u17b2\u17dd\u17e0-\u17e9\u180b-\u180d\u1810-\u1819\u1920-\u192b\u1930-\u193b\u1951-\u196d\u19b0-\u19c0\u19c8-\u19c9\u19d0-\u19d9\u1a00-\u1a15\u1a20-\u1a53\u1a60-\u1a7c\u1a7f-\u1a89\u1a90-\u1a99\u1b46-\u1b4b\u1b50-\u1b59\u1b6b-\u1b73\u1bb0-\u1bb9\u1be6-\u1bf3\u1c00-\u1c22\u1c40-\u1c49\u1c5b-\u1c7d\u1cd0-\u1cd2\u1d00-\u1dbe\u1e01-\u1f15\u200c\u200d\u203f\u2040\u2054\u20d0-\u20dc\u20e1\u20e5-\u20f0\u2d81-\u2d96\u2de0-\u2dff\u3021-\u3028\u3099\u309a\ua640-\ua66d\ua674-\ua67d\ua69f\ua6f0-\ua6f1\ua7f8-\ua800\ua806\ua80b\ua823-\ua827\ua880-\ua881\ua8b4-\ua8c4\ua8d0-\ua8d9\ua8f3-\ua8f7\ua900-\ua909\ua926-\ua92d\ua930-\ua945\ua980-\ua983\ua9b3-\ua9c0\uaa00-\uaa27\uaa40-\uaa41\uaa4c-\uaa4d\uaa50-\uaa59\uaa7b\uaae0-\uaae9\uaaf2-\uaaf3\uabc0-\uabe1\uabec\uabed\uabf0-\uabf9\ufb20-\ufb28\ufe00-\ufe0f\ufe20-\ufe26\ufe33\ufe34\ufe4d-\ufe4f\uff10-\uff19\uff3f"
)
# _nonASCIIidentifierStart = re.compile("[" + _nonASCIIidentifierStartChars + "]")
# _nonASCIIidentifier = re.compile("[" + _nonASCIIidentifierStartChars + _nonASCIIidentifierChars + "]")
_identifierStart = six.u(r"(?:\\u[0-9a-fA-F]{4}|[") + \
_baseASCIIidentifierStartChars + \
_nonASCIIidentifierStartChars + \
six.u("])")
_identifierChars = six.u(r"(?:\\u[0-9a-fA-F]{4}|[") + \
_baseASCIIidentifierChars + \
_nonASCIIidentifierStartChars + \
_nonASCIIidentifierChars + \
six.u("])*")
_identifierStart = (
six.u(r"(?:\\u[0-9a-fA-F]{4}|[")
+ _baseASCIIidentifierStartChars
+ _nonASCIIidentifierStartChars
+ six.u("])")
)
_identifierChars = (
six.u(r"(?:\\u[0-9a-fA-F]{4}|[")
+ _baseASCIIidentifierChars
+ _nonASCIIidentifierStartChars
+ _nonASCIIidentifierChars
+ six.u("])*")
)
identifier = re.compile(_identifierStart + _identifierChars)
identifierStart = re.compile(_identifierStart)
identifierMatch = re.compile(six.u(r"(?:\\u[0-9a-fA-F]{4}|[") + \
_baseASCIIidentifierChars + \
_nonASCIIidentifierStartChars + \
_nonASCIIidentifierChars + \
six.u("])+"))
identifierMatch = re.compile(
six.u(r"(?:\\u[0-9a-fA-F]{4}|[")
+ _baseASCIIidentifierChars
+ _nonASCIIidentifierStartChars
+ _nonASCIIidentifierChars
+ six.u("])+")
)
_nonASCIIwhitespace = re.compile(
six.u(r"[\u1680\u180e\u2000-\u200a\u202f\u205f\u3000\ufeff]"))
six.u(r"[\u1680\u180e\u2000-\u200a\u202f\u205f\u3000\ufeff]")
)
# Whether a single character denotes a newline.
# IMPORTANT: This string must be run through six to handle \u chars
@ -72,4 +83,4 @@ newline = re.compile(six.u(r"[\n\r\u2028\u2029]"))
# in python they are the same, different methods are called on them
# IMPORTANT: This string must be run through six to handle \u chars
lineBreak = re.compile(six.u(r"\r\n|[\n\r\u2028\u2029]"))
allLineBreaks = lineBreak
allLineBreaks = lineBreak

File diff suppressed because it is too large Load Diff

View File

@ -25,15 +25,12 @@
from ..core.options import Options as BaseOptions
OPERATOR_POSITION = [
'before-newline',
'after-newline',
'preserve-newline'
]
OPERATOR_POSITION = ["before-newline", "after-newline", "preserve-newline"]
class BeautifierOptions(BaseOptions):
def __init__(self, options=None):
BaseOptions.__init__(self, options, 'js')
BaseOptions.__init__(self, options, "js")
self.css = None
self.js = None
@ -41,11 +38,13 @@ class BeautifierOptions(BaseOptions):
# compatibility, re
raw_brace_style = getattr(self.raw_options, 'brace_style', None)
if raw_brace_style == "expand-strict": # graceful handling of deprecated option
setattr(self.raw_options, 'brace_style', "expand")
elif raw_brace_style == "collapse-preserve-inline": # graceful handling of deprecated option
setattr(self.raw_options, 'brace_style', "collapse,preserve-inline")
raw_brace_style = getattr(self.raw_options, "brace_style", None)
if raw_brace_style == "expand-strict": # graceful handling of deprecated option
setattr(self.raw_options, "brace_style", "expand")
elif (
raw_brace_style == "collapse-preserve-inline"
): # graceful handling of deprecated option
setattr(self.raw_options, "brace_style", "collapse,preserve-inline")
# elif bool(self.raw_options.braces_on_own_line): # graceful handling of deprecated option
# raw_brace_style = "expand": "collapse"
# elif raw_brace_style is None: # Nothing exists to set it
@ -54,7 +53,10 @@ class BeautifierOptions(BaseOptions):
# preserve-inline in delimited string will trigger brace_preserve_inline, everything
# else is considered a brace_style and the last one only will have an effect
brace_style_split = self._get_selection_list('brace_style', ['collapse', 'expand', 'end-expand', 'none', 'preserve-inline'])
brace_style_split = self._get_selection_list(
"brace_style",
["collapse", "expand", "end-expand", "none", "preserve-inline"],
)
# preserve-inline in delimited string will trigger brace_preserve_inline
# Everything else is considered a brace_style and the last one only will
@ -69,19 +71,25 @@ class BeautifierOptions(BaseOptions):
else:
self.brace_style = bs
self.unindent_chained_methods = self._get_boolean('unindent_chained_methods')
self.break_chained_methods = self._get_boolean('break_chained_methods')
self.space_in_paren = self._get_boolean('space_in_paren')
self.space_in_empty_paren = self._get_boolean('space_in_empty_paren')
self.jslint_happy = self._get_boolean('jslint_happy')
self.space_after_anon_function = self._get_boolean('space_after_anon_function')
self.space_after_named_function = self._get_boolean('space_after_named_function')
self.keep_array_indentation = self._get_boolean('keep_array_indentation')
self.space_before_conditional = self._get_boolean('space_before_conditional', True)
self.unescape_strings = self._get_boolean('unescape_strings')
self.e4x = self._get_boolean('e4x')
self.comma_first = self._get_boolean('comma_first')
self.operator_position = self._get_selection('operator_position', OPERATOR_POSITION)
self.unindent_chained_methods = self._get_boolean("unindent_chained_methods")
self.break_chained_methods = self._get_boolean("break_chained_methods")
self.space_in_paren = self._get_boolean("space_in_paren")
self.space_in_empty_paren = self._get_boolean("space_in_empty_paren")
self.jslint_happy = self._get_boolean("jslint_happy")
self.space_after_anon_function = self._get_boolean("space_after_anon_function")
self.space_after_named_function = self._get_boolean(
"space_after_named_function"
)
self.keep_array_indentation = self._get_boolean("keep_array_indentation")
self.space_before_conditional = self._get_boolean(
"space_before_conditional", True
)
self.unescape_strings = self._get_boolean("unescape_strings")
self.e4x = self._get_boolean("e4x")
self.comma_first = self._get_boolean("comma_first")
self.operator_position = self._get_selection(
"operator_position", OPERATOR_POSITION
)
# For testing of beautify preserve:start directive
self.test_output_raw = False

View File

@ -35,22 +35,23 @@ from ..core.templatablepattern import TemplatablePattern
__all__ = ["TOKEN", "Tokenizer", "TokenTypes"]
class TokenTypes(BaseTokenTypes):
START_EXPR = 'TK_START_EXPR'
END_EXPR = 'TK_END_EXPR'
START_BLOCK = 'TK_START_BLOCK'
END_BLOCK = 'TK_END_BLOCK'
WORD = 'TK_WORD'
RESERVED = 'TK_RESERVED'
SEMICOLON = 'TK_SEMICOLON'
STRING = 'TK_STRING'
EQUALS = 'TK_EQUALS'
OPERATOR = 'TK_OPERATOR'
COMMA = 'TK_COMMA'
BLOCK_COMMENT = 'TK_BLOCK_COMMENT'
COMMENT = 'TK_COMMENT'
DOT = 'TK_DOT'
UNKNOWN = 'TK_UNKNOWN'
START_EXPR = "TK_START_EXPR"
END_EXPR = "TK_END_EXPR"
START_BLOCK = "TK_START_BLOCK"
END_BLOCK = "TK_END_BLOCK"
WORD = "TK_WORD"
RESERVED = "TK_RESERVED"
SEMICOLON = "TK_SEMICOLON"
STRING = "TK_STRING"
EQUALS = "TK_EQUALS"
OPERATOR = "TK_OPERATOR"
COMMA = "TK_COMMA"
BLOCK_COMMENT = "TK_BLOCK_COMMENT"
COMMENT = "TK_COMMENT"
DOT = "TK_DOT"
UNKNOWN = "TK_UNKNOWN"
def __init__(self):
pass
@ -58,56 +59,69 @@ class TokenTypes(BaseTokenTypes):
TOKEN = TokenTypes()
dot_pattern = re.compile(r'[^\d\.]')
dot_pattern = re.compile(r"[^\d\.]")
number_pattern = re.compile(
r'0[xX][0123456789abcdefABCDEF]*|0[oO][01234567]*|0[bB][01]*|\d+n|(?:\.\d+|\d+\.?\d*)(?:[eE][+-]?\d+)?')
digit = re.compile(r'[0-9]')
r"0[xX][0123456789abcdefABCDEF]*|0[oO][01234567]*|0[bB][01]*|\d+n|(?:\.\d+|\d+\.?\d*)(?:[eE][+-]?\d+)?"
)
digit = re.compile(r"[0-9]")
positionable_operators = frozenset(
(">>> === !== " +
"<< && >= ** != == <= >> || |> " +
"< / - + > : & % ? ^ | *").split(' '))
(
">>> === !== " + "<< && >= ** != == <= >> || |> " + "< / - + > : & % ? ^ | *"
).split(" ")
)
punct = (">>>= " +
"... >>= <<= === >>> !== **= " +
"=> ^= :: /= << <= == && -= >= >> != -- += ** || ++ %= &= *= |= |> " +
"= ! ? > < : / ^ - + * & % ~ |")
punct = (
">>>= "
+ "... >>= <<= === >>> !== **= "
+ "=> ^= :: /= << <= == && -= >= >> != -- += ** || ++ %= &= *= |= |> "
+ "= ! ? > < : / ^ - + * & % ~ |"
)
punct = re.compile(r'([-[\]{}()*+?.,\\^$|#])').sub(r'\\\1', punct)
punct = re.compile(r"([-[\]{}()*+?.,\\^$|#])").sub(r"\\\1", punct)
# ?. but not if followed by a number
punct = '\\?\\.(?!\\d) ' + punct
punct = punct.replace(' ', '|')
punct = "\\?\\.(?!\\d) " + punct
punct = punct.replace(" ", "|")
punct_pattern = re.compile(punct)
# Words which always should start on a new line
line_starters = frozenset(
('continue,try,throw,return,var,let,const,if,switch,case,default,for,' +
'while,break,function,import,export').split(','))
reserved_words = line_starters | frozenset(['do',
'in',
'of',
'else',
'get',
'set',
'new',
'catch',
'finally',
'typeof',
'yield',
'async',
'await',
'from',
'as'])
(
"continue,try,throw,return,var,let,const,if,switch,case,default,for,"
+ "while,break,function,import,export"
).split(",")
)
reserved_words = line_starters | frozenset(
[
"do",
"in",
"of",
"else",
"get",
"set",
"new",
"catch",
"finally",
"typeof",
"yield",
"async",
"await",
"from",
"as",
]
)
reserved_word_pattern = re.compile(r'^(?:' + '|'.join(reserved_words) + r')$')
reserved_word_pattern = re.compile(r"^(?:" + "|".join(reserved_words) + r")$")
directives_core = Directives(r'/\*', r'\*/')
directives_core = Directives(r"/\*", r"\*/")
xmlRegExp = re.compile(
r'[\s\S]*?<(\/?)([-a-zA-Z:0-9_.]+|{[\s\S]+?}|!\[CDATA\[[\s\S]*?\]\])(\s+{[\s\S]+?}|\s+[-a-zA-Z:0-9_.]+|\s+[-a-zA-Z:0-9_.]+\s*=\s*(\'[^\']*\'|"[^"]*"|{[\s\S]+?}))*\s*(/?)\s*>')
r'[\s\S]*?<(\/?)([-a-zA-Z:0-9_.]+|{[\s\S]+?}|!\[CDATA\[[\s\S]*?\]\])(\s+{[\s\S]+?}|\s+[-a-zA-Z:0-9_.]+|\s+[-a-zA-Z:0-9_.]+\s*=\s*(\'[^\']*\'|"[^"]*"|{[\s\S]+?}))*\s*(/?)\s*>'
)
class TokenizerPatterns(BaseTokenizerPatterns):
def __init__(self, input_scanner, acorn, options):
@ -120,34 +134,31 @@ class TokenizerPatterns(BaseTokenizerPatterns):
# IMPORTANT: This string must be run through six to handle \u chars
self.whitespace = self.whitespace.matching(
six.u(r'\u00A0\u1680\u180e\u2000-\u200a\u202f\u205f\u3000\ufeff'),
six.u(r'\u2028\u2029'))
six.u(r"\u00A0\u1680\u180e\u2000-\u200a\u202f\u205f\u3000\ufeff"),
six.u(r"\u2028\u2029"),
)
pattern = Pattern(input_scanner)
templatable = TemplatablePattern(input_scanner) \
.read_options(options)
templatable = TemplatablePattern(input_scanner).read_options(options)
self.identifier = templatable.starting_with(acorn.identifier \
).matching(acorn.identifierMatch)
self.identifier = templatable.starting_with(acorn.identifier).matching(
acorn.identifierMatch
)
self.number = pattern.matching(number_pattern)
self.punct = pattern.matching(punct_pattern)
self.comment = pattern.starting_with(r'//').until(
six.u(r'[\n\r\u2028\u2029]'))
self.block_comment = pattern.starting_with(r'/\*').until_after(r'\*/')
self.html_comment_start = pattern.matching(r'<!--')
self.html_comment_end = pattern.matching(r'-->')
self.include = pattern.starting_with(r'#include' \
).until_after(acorn.lineBreak)
self.shebang = pattern.starting_with(r'#!' \
).until_after(acorn.lineBreak)
self.comment = pattern.starting_with(r"//").until(six.u(r"[\n\r\u2028\u2029]"))
self.block_comment = pattern.starting_with(r"/\*").until_after(r"\*/")
self.html_comment_start = pattern.matching(r"<!--")
self.html_comment_end = pattern.matching(r"-->")
self.include = pattern.starting_with(r"#include").until_after(acorn.lineBreak)
self.shebang = pattern.starting_with(r"#!").until_after(acorn.lineBreak)
self.xml = pattern.matching(xmlRegExp)
self.single_quote = templatable.until(six.u(r"['\\\n\r\u2028\u2029]"))
self.double_quote = templatable.until(six.u(r'["\\\n\r\u2028\u2029]'))
self.template_text = templatable.until(r'[`\\$]')
self.template_expression = templatable.until(r'[`}\\]')
self.template_text = templatable.until(r"[`\\$]")
self.template_expression = templatable.until(r"[`}\\]")
class Tokenizer(BaseTokenizer):
@ -158,6 +169,7 @@ class Tokenizer(BaseTokenizer):
BaseTokenizer.__init__(self, input_string, opts)
import jsbeautifier.javascript.acorn as acorn
self.acorn = acorn
self.in_html_comment = False
@ -165,25 +177,34 @@ class Tokenizer(BaseTokenizer):
self._patterns = TokenizerPatterns(self._input, self.acorn, opts)
def _reset(self):
self.in_html_comment = False
def _is_comment(self, current_token):
return current_token.type == TOKEN.COMMENT or \
current_token.type == TOKEN.BLOCK_COMMENT or \
current_token.type == TOKEN.UNKNOWN
return (
current_token.type == TOKEN.COMMENT
or current_token.type == TOKEN.BLOCK_COMMENT
or current_token.type == TOKEN.UNKNOWN
)
def _is_opening(self, current_token):
return current_token.type == TOKEN.START_BLOCK or current_token.type == TOKEN.START_EXPR
return (
current_token.type == TOKEN.START_BLOCK
or current_token.type == TOKEN.START_EXPR
)
def _is_closing(self, current_token, open_token):
return (current_token.type == TOKEN.END_BLOCK or current_token.type == TOKEN.END_EXPR) and \
(open_token is not None and (
(current_token.text == ']' and open_token.text == '[') or
(current_token.text == ')' and open_token.text == '(') or
(current_token.text == '}' and open_token.text == '{')))
return (
current_token.type == TOKEN.END_BLOCK
or current_token.type == TOKEN.END_EXPR
) and (
open_token is not None
and (
(current_token.text == "]" and open_token.text == "[")
or (current_token.text == ")" and open_token.text == "(")
or (current_token.text == "}" and open_token.text == "{")
)
)
def _get_next_token(self, previous_token, open_token):
token = None
@ -191,7 +212,7 @@ class Tokenizer(BaseTokenizer):
c = self._input.peek()
if c is None:
token = self._create_token(TOKEN.EOF, '')
token = self._create_token(TOKEN.EOF, "")
token = token or self._read_non_javascript(c)
token = token or self._read_string(c)
@ -208,20 +229,23 @@ class Tokenizer(BaseTokenizer):
def _read_singles(self, c):
token = None
if c == '(' or c == '[':
if c == "(" or c == "[":
token = self._create_token(TOKEN.START_EXPR, c)
elif c == ')' or c == ']':
elif c == ")" or c == "]":
token = self._create_token(TOKEN.END_EXPR, c)
elif c == '{':
elif c == "{":
token = self._create_token(TOKEN.START_BLOCK, c)
elif c == '}':
elif c == "}":
token = self._create_token(TOKEN.END_BLOCK, c)
elif c == ';':
elif c == ";":
token = self._create_token(TOKEN.SEMICOLON, c)
elif c == '.' and self._input.peek(1) is not None and \
bool(dot_pattern.match(self._input.peek(1))):
elif (
c == "."
and self._input.peek(1) is not None
and bool(dot_pattern.match(self._input.peek(1)))
):
token = self._create_token(TOKEN.DOT, c)
elif c == ',':
elif c == ",":
token = self._create_token(TOKEN.COMMA, c)
if token is not None:
@ -233,12 +257,15 @@ class Tokenizer(BaseTokenizer):
resulting_string = self._patterns.identifier.read()
if bool(resulting_string):
resulting_string = re.sub(self.acorn.allLineBreaks, '\n', resulting_string)
if not (previous_token.type == TOKEN.DOT or (
previous_token.type == TOKEN.RESERVED and (
previous_token.text == 'set' or previous_token.text == 'get')
)) and reserved_word_pattern.match(resulting_string):
if resulting_string == 'in' or resulting_string == 'of':
resulting_string = re.sub(self.acorn.allLineBreaks, "\n", resulting_string)
if not (
previous_token.type == TOKEN.DOT
or (
previous_token.type == TOKEN.RESERVED
and (previous_token.text == "set" or previous_token.text == "get")
)
) and reserved_word_pattern.match(resulting_string):
if resulting_string == "in" or resulting_string == "of":
# in and of are operators, need to hack
return self._create_token(TOKEN.OPERATOR, resulting_string)
@ -247,37 +274,36 @@ class Tokenizer(BaseTokenizer):
return self._create_token(TOKEN.WORD, resulting_string)
resulting_string = self._patterns.number.read()
if resulting_string != '':
if resulting_string != "":
return self._create_token(TOKEN.WORD, resulting_string)
def _read_comment(self, c):
token = None
if c == '/':
comment = ''
if self._input.peek(1) == '*': # peek /* .. */ comment
if c == "/":
comment = ""
if self._input.peek(1) == "*": # peek /* .. */ comment
comment = self._patterns.block_comment.read()
directives = directives_core.get_directives(comment)
if directives and directives.get('ignore') == 'start':
if directives and directives.get("ignore") == "start":
comment += directives_core.readIgnored(self._input)
comment = re.sub(self.acorn.allLineBreaks, '\n', comment)
comment = re.sub(self.acorn.allLineBreaks, "\n", comment)
token = self._create_token(TOKEN.BLOCK_COMMENT, comment)
token.directives = directives
elif self._input.peek(1) == '/': # peek // comment
elif self._input.peek(1) == "/": # peek // comment
comment = self._patterns.comment.read()
token = self._create_token(TOKEN.COMMENT, comment)
return token
def _read_string(self, c):
if c == '`' or c == "'" or c == '"':
if c == "`" or c == "'" or c == '"':
resulting_string = self._input.next()
self.has_char_escapes = False
if c == '`':
resulting_string += self.parse_string('`', True, '${')
if c == "`":
resulting_string += self.parse_string("`", True, "${")
else:
resulting_string += self.parse_string(c)
@ -287,8 +313,7 @@ class Tokenizer(BaseTokenizer):
if self._input.peek() == c:
resulting_string += self._input.next()
resulting_string = re.sub(
self.acorn.allLineBreaks, '\n', resulting_string)
resulting_string = re.sub(self.acorn.allLineBreaks, "\n", resulting_string)
return self._create_token(TOKEN.STRING, resulting_string)
@ -296,21 +321,23 @@ class Tokenizer(BaseTokenizer):
def _read_regexp(self, c, previous_token):
if c == '/' and self.allowRegExOrXML(previous_token):
if c == "/" and self.allowRegExOrXML(previous_token):
# handle regexp
resulting_string = self._input.next()
esc = False
in_char_class = False
while self._input.hasNext() and \
(esc or in_char_class or self._input.peek() != c) and \
not self._input.testChar(self.acorn.newline):
while (
self._input.hasNext()
and (esc or in_char_class or self._input.peek() != c)
and not self._input.testChar(self.acorn.newline)
):
resulting_string += self._input.peek()
if not esc:
esc = self._input.peek() == '\\'
if self._input.peek() == '[':
esc = self._input.peek() == "\\"
if self._input.peek() == "[":
in_char_class = True
elif self._input.peek() == ']':
elif self._input.peek() == "]":
in_char_class = False
else:
esc = False
@ -319,18 +346,16 @@ class Tokenizer(BaseTokenizer):
if self._input.peek() == c:
resulting_string += self._input.next()
if c == '/':
if c == "/":
# regexps may have modifiers /regexp/MOD, so fetch those too
# Only [gim] are valid, but if the user puts in garbage, do
# what we can to take it.
resulting_string += self._input.read(
self.acorn.identifier)
resulting_string += self._input.read(self.acorn.identifier)
return self._create_token(TOKEN.STRING, resulting_string)
return None
def _read_xml(self, c, previous_token):
if self._options.e4x and c == "<" and self.allowRegExOrXML(previous_token):
# handle e4x xml literals
@ -338,16 +363,22 @@ class Tokenizer(BaseTokenizer):
match = self._patterns.xml.read_match()
if match and not match.group(1):
rootTag = match.group(2)
rootTag = re.sub(r'^{\s+', '{', re.sub(r'\s+}$', '}', rootTag))
isCurlyRoot = rootTag.startswith('{')
rootTag = re.sub(r"^{\s+", "{", re.sub(r"\s+}$", "}", rootTag))
isCurlyRoot = rootTag.startswith("{")
depth = 0
while bool(match):
isEndTag = match.group(1)
tagName = match.group(2)
isSingletonTag = (
match.groups()[-1] != "") or (match.group(2)[0:8] == "![CDATA[")
if not isSingletonTag and (tagName == rootTag or (
isCurlyRoot and re.sub(r'^{\s+', '{', re.sub(r'\s+}$', '}', tagName)))):
isSingletonTag = (match.groups()[-1] != "") or (
match.group(2)[0:8] == "![CDATA["
)
if not isSingletonTag and (
tagName == rootTag
or (
isCurlyRoot
and re.sub(r"^{\s+", "{", re.sub(r"\s+}$", "}", tagName))
)
):
if isEndTag:
depth -= 1
else:
@ -361,29 +392,33 @@ class Tokenizer(BaseTokenizer):
# if we didn't close correctly, keep unformatted.
if not match:
xmlStr += self._input.match(re.compile(r'[\s\S]*')).group(0)
xmlStr += self._input.match(re.compile(r"[\s\S]*")).group(0)
xmlStr = re.sub(self.acorn.allLineBreaks, '\n', xmlStr)
xmlStr = re.sub(self.acorn.allLineBreaks, "\n", xmlStr)
return self._create_token(TOKEN.STRING, xmlStr)
return None
def _read_non_javascript(self, c):
resulting_string = ''
resulting_string = ""
if c == '#':
if c == "#":
# she-bang
if self._is_first_token():
resulting_string = self._patterns.shebang.read()
if resulting_string:
return self._create_token(TOKEN.UNKNOWN, resulting_string.strip() + '\n')
return self._create_token(
TOKEN.UNKNOWN, resulting_string.strip() + "\n"
)
# handles extendscript #includes
resulting_string = self._patterns.include.read()
if resulting_string:
return self._create_token(TOKEN.UNKNOWN, resulting_string.strip() + '\n')
return self._create_token(
TOKEN.UNKNOWN, resulting_string.strip() + "\n"
)
c = self._input.next()
@ -391,21 +426,21 @@ class Tokenizer(BaseTokenizer):
# https://developer.mozilla.org/En/Sharp_variables_in_JavaScript
# http://mxr.mozilla.org/mozilla-central/source/js/src/jsscan.cpp
# around line 1935
sharp = '#'
sharp = "#"
if self._input.hasNext() and self._input.testChar(digit):
while True:
c = self._input.next()
sharp += c
if (not self._input.hasNext()) or c == '#' or c == '=':
if (not self._input.hasNext()) or c == "#" or c == "=":
break
if c == '#':
if c == "#":
pass
elif self._input.peek() == '[' and self._input.peek(1) == ']':
sharp += '[]'
elif self._input.peek() == "[" and self._input.peek(1) == "]":
sharp += "[]"
self._input.next()
self._input.next()
elif self._input.peek() == '{' and self._input.peek(1) == '}':
sharp += '{}'
elif self._input.peek() == "{" and self._input.peek(1) == "}":
sharp += "{}"
self._input.next()
self._input.next()
@ -413,87 +448,103 @@ class Tokenizer(BaseTokenizer):
self._input.back()
elif c == '<' and self._is_first_token():
elif c == "<" and self._is_first_token():
if self._patterns.html_comment_start.read():
c = '<!--'
while self._input.hasNext() and not self._input.testChar(self.acorn.newline):
c = "<!--"
while self._input.hasNext() and not self._input.testChar(
self.acorn.newline
):
c += self._input.next()
self.in_html_comment = True
return self._create_token(TOKEN.COMMENT, c)
elif c == '-' and self.in_html_comment and \
self._patterns.html_comment_end.read():
elif (
c == "-" and self.in_html_comment and self._patterns.html_comment_end.read()
):
self.in_html_comment = False
return self._create_token(TOKEN.COMMENT, '-->')
return self._create_token(TOKEN.COMMENT, "-->")
return None
def _read_punctuation(self):
token = None
resulting_string = self._patterns.punct.read()
if resulting_string != '':
if resulting_string == '=':
if resulting_string != "":
if resulting_string == "=":
token = self._create_token(TOKEN.EQUALS, resulting_string)
elif resulting_string == '?.':
elif resulting_string == "?.":
token = self._create_token(TOKEN.DOT, resulting_string)
else:
token = self._create_token(TOKEN.OPERATOR, resulting_string)
return token
__regexTokens = { TOKEN.COMMENT, TOKEN.START_EXPR, TOKEN.START_BLOCK,
TOKEN.START, TOKEN.END_BLOCK, TOKEN.OPERATOR,
TOKEN.EQUALS, TOKEN.EOF, TOKEN.SEMICOLON, TOKEN.COMMA }
def allowRegExOrXML(self, previous_token):
return (previous_token.type == TOKEN.RESERVED and previous_token.text in {'return', 'case', 'throw', 'else', 'do', 'typeof', 'yield'}) or \
(previous_token.type == TOKEN.END_EXPR and previous_token.text == ')' and
previous_token.opened.previous.type == TOKEN.RESERVED and previous_token.opened.previous.text in {'if', 'while', 'for'}) or \
(previous_token.type in self.__regexTokens )
__regexTokens = {
TOKEN.COMMENT,
TOKEN.START_EXPR,
TOKEN.START_BLOCK,
TOKEN.START,
TOKEN.END_BLOCK,
TOKEN.OPERATOR,
TOKEN.EQUALS,
TOKEN.EOF,
TOKEN.SEMICOLON,
TOKEN.COMMA,
}
def parse_string(
self,
delimiter,
allow_unescaped_newlines=False,
start_sub=None):
if delimiter == '\'':
def allowRegExOrXML(self, previous_token):
return (
(
previous_token.type == TOKEN.RESERVED
and previous_token.text
in {"return", "case", "throw", "else", "do", "typeof", "yield"}
)
or (
previous_token.type == TOKEN.END_EXPR
and previous_token.text == ")"
and previous_token.opened.previous.type == TOKEN.RESERVED
and previous_token.opened.previous.text in {"if", "while", "for"}
)
or (previous_token.type in self.__regexTokens)
)
def parse_string(self, delimiter, allow_unescaped_newlines=False, start_sub=None):
if delimiter == "'":
pattern = self._patterns.single_quote
elif delimiter == '"':
pattern = self._patterns.double_quote
elif delimiter == '`':
elif delimiter == "`":
pattern = self._patterns.template_text
elif delimiter == '}':
elif delimiter == "}":
pattern = self._patterns.template_expression
resulting_string = pattern.read()
next = ''
next = ""
while self._input.hasNext():
next = self._input.next()
if next == delimiter or \
(not allow_unescaped_newlines and
self.acorn.newline.match(next)):
if next == delimiter or (
not allow_unescaped_newlines and self.acorn.newline.match(next)
):
self._input.back()
break
elif next == '\\' and self._input.hasNext():
elif next == "\\" and self._input.hasNext():
current_char = self._input.peek()
if current_char == 'x' or current_char == 'u':
if current_char == "x" or current_char == "u":
self.has_char_escapes = True
elif current_char == '\r' and self._input.peek(1) == '\n':
elif current_char == "\r" and self._input.peek(1) == "\n":
self._input.next()
next += self._input.next()
elif start_sub is not None:
if start_sub == '${' and next == '$' and \
self._input.peek() == '{':
if start_sub == "${" and next == "$" and self._input.peek() == "{":
next += self._input.next()
if start_sub == next:
if delimiter == '`':
next += self.parse_string(
'}', allow_unescaped_newlines, '`')
if delimiter == "`":
next += self.parse_string("}", allow_unescaped_newlines, "`")
else:
next += self.parse_string(
'`', allow_unescaped_newlines, '${')
next += self.parse_string("`", allow_unescaped_newlines, "${")
if self._input.hasNext():
next += self._input.next()
@ -502,14 +553,13 @@ class Tokenizer(BaseTokenizer):
resulting_string += next
return resulting_string
def unescape_string(self, s):
# You think that a regex would work for this
# return s.replace(/\\x([0-9a-f]{2})/gi, function(match, val) {
# return String.fromCharCode(parseInt(val, 16));
# })
# However, dealing with '\xff', '\\xff', '\\\xff' makes this more fun.
out = self.acorn.six.u('')
out = self.acorn.six.u("")
escaped = 0
input_scan = InputScanner(s)
@ -518,21 +568,21 @@ class Tokenizer(BaseTokenizer):
while input_scan.hasNext():
# Keep any whitespace, non-slash characters
# also keep slash pairs.
matched = input_scan.match(re.compile(r'([\s]|[^\\]|\\\\)+'))
matched = input_scan.match(re.compile(r"([\s]|[^\\]|\\\\)+"))
if matched:
out += matched.group(0)
if input_scan.peek() != '\\':
if input_scan.peek() != "\\":
continue
input_scan.next()
if input_scan.peek() == 'x':
matched = input_scan.match(re.compile(r'x([0-9A-Fa-f]{2})'))
elif input_scan.peek() == 'u':
matched = input_scan.match(re.compile(r'u([0-9A-Fa-f]{4})'))
if input_scan.peek() == "x":
matched = input_scan.match(re.compile(r"x([0-9A-Fa-f]{2})"))
elif input_scan.peek() == "u":
matched = input_scan.match(re.compile(r"u([0-9A-Fa-f]{4})"))
else:
out += '\\'
out += "\\"
if input_scan.hasNext():
out += input_scan.next()
continue
@ -543,19 +593,18 @@ class Tokenizer(BaseTokenizer):
escaped = int(matched.group(1), 16)
if escaped > 0x7e and escaped <= 0xff and matched.group(
0).startswith('x'):
if escaped > 0x7E and escaped <= 0xFF and matched.group(0).startswith("x"):
# we bail out on \x7f..\xff,
# leaving whole string escaped,
# as it's probably completely binary
return s
elif escaped >= 0x00 and escaped < 0x20:
# leave 0x00...0x1f escaped
out += '\\' + matched.group(0)
out += "\\" + matched.group(0)
continue
elif escaped == 0x22 or escaped == 0x27 or escaped == 0x5c:
elif escaped == 0x22 or escaped == 0x27 or escaped == 0x5C:
# single-quote, apostrophe, backslash - escape these
out += ('\\' + chr(escaped))
out += "\\" + chr(escaped)
else:
out += self.acorn.six.unichr(escaped)

View File

@ -4,13 +4,12 @@ from ...core.inputscanner import InputScanner
class TestInputScanner(unittest.TestCase):
@classmethod
def setUpClass(cls):
pass
def setUp(self):
self.value = 'howdy'
self.value = "howdy"
self.inputscanner = InputScanner(self.value)
def test_new(self):
@ -32,17 +31,17 @@ class TestInputScanner(unittest.TestCase):
self.assertEqual(self.inputscanner.peek(), self.value[1])
def test_pattern(self):
pattern = re.compile(r'how')
pattern = re.compile(r"how")
index = 0
self.assertEqual(self.inputscanner.test(pattern, index), True)
self.inputscanner.next()
self.assertEqual(self.inputscanner.test(pattern, index), False)
def test_Char(self):
pattern = re.compile(r'o')
pattern = re.compile(r"o")
index = 1
self.assertEqual(self.inputscanner.testChar(pattern, index), True)
if __name__ == '__main__':
if __name__ == "__main__":
unittest.main()

View File

@ -4,137 +4,130 @@ from ...core.options import _mergeOpts, _normalizeOpts, Options
class TestOptions(unittest.TestCase):
@classmethod
def setUpClass(cls):
pass
def test_mergeOpts(self):
# should convert tuple to dict and merge child with parent
result = _mergeOpts((('a', 1), ('b', {'a': 2})), 'b')
result = _mergeOpts((("a", 1), ("b", {"a": 2})), "b")
self.assertEqual(result.a, 2)
self.assertNotIn('b', result)
self.assertNotIn("b", result)
# should merge child option a with the parent options (dict)
result = _mergeOpts({'a': 1, 'b': {'a': 2}}, 'b')
result = _mergeOpts({"a": 1, "b": {"a": 2}}, "b")
self.assertEqual(result.a, 2)
self.assertNotIn('b', result)
self.assertNotIn("b", result)
# should include child option c and d with the parent options (dict)
result = _mergeOpts({'a': 1, 'b': {'c': 2, 'd': 3}}, 'b')
result = _mergeOpts({"a": 1, "b": {"c": 2, "d": 3}}, "b")
self.assertEqual(result.a, 1)
self.assertEqual(result.c, 2)
self.assertEqual(result.d, 3)
self.assertNotIn('b', result)
self.assertNotIn("b", result)
# should merge child option a and include c as parent options (dict)
result = _mergeOpts({'a': 1, 'b': {'a': 2, 'c': 3}}, 'b')
result = _mergeOpts({"a": 1, "b": {"a": 2, "c": 3}}, "b")
self.assertEqual(result.a, 2)
self.assertEqual(result.c, 3)
self.assertNotIn('b', result)
self.assertNotIn("b", result)
# should merge Options instance with child dict key
instance = Options()
instance.a = {'disabled': True}
result = _mergeOpts(instance, 'a')
instance.a = {"disabled": True}
result = _mergeOpts(instance, "a")
self.assertEqual(result.disabled, True)
self.assertNotIn('a', list(getattr(result, '__dict__', {})))
self.assertNotIn("a", list(getattr(result, "__dict__", {})))
def test_normalizeOpts(self):
# should replace key with - to _ in dict
result = _normalizeOpts({
'a-b': 1
})
self.assertEqual(result['a_b'], 1)
self.assertNotIn('a-b', result)
result = _normalizeOpts({"a-b": 1})
self.assertEqual(result["a_b"], 1)
self.assertNotIn("a-b", result)
# should replace key with - to _ in Options instance
instance = Options()
setattr(instance, 'a-b', 1)
setattr(instance, "a-b", 1)
result = _normalizeOpts(instance)
self.assertEqual(result.a_b, 1)
self.assertNotIn('a-b', list(getattr(result, '__dict__', {})))
self.assertNotIn("a-b", list(getattr(result, "__dict__", {})))
# should do nothing
result = _normalizeOpts({
'a_b': 1
})
self.assertEqual(result['a_b'], 1)
result = _normalizeOpts({"a_b": 1})
self.assertEqual(result["a_b"], 1)
def test__get_boolean(self):
# should return default value since no option
self.assertEqual(Options()._get_boolean('a'), False)
self.assertEqual(Options()._get_boolean("a"), False)
# should return true as default since no option
self.assertEqual(Options()._get_boolean('a', True), True)
self.assertEqual(Options()._get_boolean("a", True), True)
# should return false as in option
self.assertEqual(Options({'a': False})._get_boolean('a', True), False)
self.assertEqual(Options({"a": False})._get_boolean("a", True), False)
def test__get_characters(self):
# should return default value since no option
self.assertEqual(Options()._get_characters('a'), '')
self.assertEqual(Options()._get_characters("a"), "")
# should return \'character\' as default since no option
self.assertEqual(Options()._get_characters(
'a', 'character'), 'character')
self.assertEqual(Options()._get_characters("a", "character"), "character")
# should return \'char\' as in option
self.assertEqual(Options({'a': 'char'})._get_characters(
'a', 'character'), 'char')
self.assertEqual(
Options({"a": "char"})._get_characters("a", "character"), "char"
)
def test__get_number(self):
# should return default value since no option
self.assertEqual(Options()._get_number('a'), 0)
self.assertEqual(Options()._get_number("a"), 0)
# should return 1 as default since no option
self.assertEqual(Options()._get_number('a', 1), 1)
self.assertEqual(Options()._get_number("a", 1), 1)
# should return 10 as in option
self.assertEqual(Options({'a': 10})._get_number('a', 1), 10)
self.assertEqual(Options({"a": 10})._get_number("a", 1), 10)
# should return 0 for NaN as in option
self.assertEqual(Options({'a': 'abc'})._get_number('a'), 0)
self.assertEqual(Options({"a": "abc"})._get_number("a"), 0)
# should return 0 for NaN as in default
self.assertEqual(Options()._get_number('a', 'abc'), 0)
self.assertEqual(Options()._get_number("a", "abc"), 0)
def test__get_array(self):
# should return [] with no option
self.assertEqual(Options()._get_array('a'), [])
self.assertEqual(Options()._get_array("a"), [])
# should return [\'a\',\'b\'] as default since no option
self.assertEqual(Options()._get_array('a', ['a', 'b']), ['a', 'b'])
self.assertEqual(Options()._get_array("a", ["a", "b"]), ["a", "b"])
# should return [\'c\',\'d\'] as in option
self.assertEqual(Options({'a': ['c', 'd']})._get_array(
'a', ['a', 'b']), ['c', 'd'])
self.assertEqual(
Options({"a": ["c", "d"]})._get_array("a", ["a", "b"]), ["c", "d"]
)
# should return [\'c\',\'d\'] as in option comma separated
self.assertEqual(Options({'a': 'c,d'})._get_array(
'a', ['a', 'b']), ['c', 'd'])
self.assertEqual(Options({"a": "c,d"})._get_array("a", ["a", "b"]), ["c", "d"])
def test__is_valid_selection(self):
# should return false with empty selection
self.assertEqual(Options()._is_valid_selection(['a', 'b'], []), False)
self.assertEqual(Options()._is_valid_selection(["a", "b"], []), False)
# should return false with selection inexistent
self.assertEqual(Options()._is_valid_selection(
['a', 'b'], ['c']), False)
self.assertEqual(Options()._is_valid_selection(["a", "b"], ["c"]), False)
# should return true with selection existent
self.assertEqual(Options()._is_valid_selection(
['a', 'b'], ['a', 'b']), True)
self.assertEqual(Options()._is_valid_selection(["a", "b"], ["a", "b"]), True)
def test__get_selection_list(self):
# should raise error with empty selection
with self.assertRaisesRegexp(ValueError, 'Selection list cannot'
+ ' be empty.'):
Options()._get_selection_list('a', [])
with self.assertRaisesRegexp(
ValueError, "Selection list cannot" + " be empty."
):
Options()._get_selection_list("a", [])
# should raise error with invalid default
with self.assertRaisesRegexp(ValueError, 'Invalid Default Value!'):
Options()._get_selection_list('a', ['a', 'b'], ['c'])
with self.assertRaisesRegexp(ValueError, "Invalid Default Value!"):
Options()._get_selection_list("a", ["a", "b"], ["c"])
# should raise error with invalid option
with self.assertRaisesRegexp(ValueError, '^Invalid Option Value:'
+ ' The option'):
Options({'a': ['c', 'd']})._get_selection_list(
'a', ['a', 'b'], ['a', 'b'])
with self.assertRaisesRegexp(
ValueError, "^Invalid Option Value:" + " The option"
):
Options({"a": ["c", "d"]})._get_selection_list("a", ["a", "b"], ["a", "b"])
# should return [\'c\'] as in option
opts = Options({'c': ['c']})
self.assertEqual(opts._get_selection_list(
'c', ['c', 'd'], ['c']), ['c'])
opts = Options({"c": ["c"]})
self.assertEqual(opts._get_selection_list("c", ["c", "d"], ["c"]), ["c"])
def test__get_selection(self):
# should raise error with multiple selection
with self.assertRaisesRegexp(ValueError, '^Invalid Option'
+ ' Value: The option'):
Options({'a': ['a', 'b']})._get_selection('a', ['a', 'b'], ['a'])
with self.assertRaisesRegexp(
ValueError, "^Invalid Option" + " Value: The option"
):
Options({"a": ["a", "b"]})._get_selection("a", ["a", "b"], ["a"])
# should return [\'a\'] as in option
options = Options({'a': ['a']})
self.assertEqual(options._get_selection(
'a', ['a', 'b'], ['a']), 'a')
options = Options({"a": ["a"]})
self.assertEqual(options._get_selection("a", ["a", "b"], ["a"]), "a")
if __name__ == '__main__':
if __name__ == "__main__":
unittest.main()

View File

@ -8,7 +8,7 @@ class TestJSBeautifierIndentation(unittest.TestCase):
test_fragment = self.decodesto
self.options.indent_with_tabs = 1
test_fragment('{tabs()}', "{\n\ttabs()\n}")
test_fragment("{tabs()}", "{\n\ttabs()\n}")
def test_function_indent(self):
test_fragment = self.decodesto
@ -16,33 +16,34 @@ class TestJSBeautifierIndentation(unittest.TestCase):
self.options.indent_with_tabs = 1
self.options.keep_function_indentation = 1
test_fragment(
'var foo = function(){ bar() }();',
"var foo = function() {\n\tbar()\n}();")
"var foo = function(){ bar() }();", "var foo = function() {\n\tbar()\n}();"
)
self.options.tabs = 1
self.options.keep_function_indentation = 0
test_fragment(
'var foo = function(){ baz() }();',
"var foo = function() {\n\tbaz()\n}();")
"var foo = function(){ baz() }();", "var foo = function() {\n\tbaz()\n}();"
)
def decodesto(self, input, expectation=None):
self.assertEqual(
jsbeautifier.beautify(input, self.options), expectation or input)
jsbeautifier.beautify(input, self.options), expectation or input
)
@classmethod
def setUpClass(cls):
options = jsbeautifier.default_options()
options.indent_size = 4
options.indent_char = ' '
options.indent_char = " "
options.preserve_newlines = True
options.jslint_happy = False
options.keep_array_indentation = False
options.brace_style = 'collapse'
options.brace_style = "collapse"
options.indent_level = 0
cls.options = options
cls.wrapregex = re.compile('^(.+)$', re.MULTILINE)
cls.wrapregex = re.compile("^(.+)$", re.MULTILINE)
if __name__ == '__main__':
if __name__ == "__main__":
unittest.main()

View File

@ -10,12 +10,13 @@ import re
from jsbeautifier.unpackers import evalbased
# NOTE: AT THE MOMENT, IT IS DEACTIVATED FOR YOUR SECURITY: it runs js!
BLACKLIST = ['jsbeautifier.unpackers.evalbased']
BLACKLIST = ["jsbeautifier.unpackers.evalbased"]
class UnpackingError(Exception):
"""Badly packed source or general error. Argument is a
meaningful description."""
pass
@ -25,15 +26,15 @@ def getunpackers():
adhere to naming conventions) and it is not blacklisted (i.e. inserted
into BLACKLIST."""
path = __path__
prefix = __name__ + '.'
prefix = __name__ + "."
unpackers = []
interface = ['unpack', 'detect', 'PRIORITY']
interface = ["unpack", "detect", "PRIORITY"]
for _importer, modname, _ispkg in pkgutil.iter_modules(path, prefix):
if 'tests' not in modname and modname not in BLACKLIST:
if "tests" not in modname and modname not in BLACKLIST:
try:
module = __import__(modname, fromlist=interface)
except ImportError:
raise UnpackingError('Bad unpacker: %s' % modname)
raise UnpackingError("Bad unpacker: %s" % modname)
else:
unpackers.append(module)
@ -58,15 +59,15 @@ def filtercomments(source):
comment = True
while comment:
if re.search(r'^\s*\/\*', source):
comment = source[0, source.index('*/') + 2]
elif re.search(r'^\s*\/\/', source):
comment = re.search(r'^\s*\/\/', source).group(0)
if re.search(r"^\s*\/\*", source):
comment = source[0, source.index("*/") + 2]
elif re.search(r"^\s*\/\/", source):
comment = re.search(r"^\s*\/\/", source).group(0)
else:
comment = None
if comment:
source = re.sub(r'^\s+', '', source[len(comment):])
source = re.sub(r"^\s+", "", source[len(comment) :])
trailing_comments.append(comment)
return '\n'.join(trailing_comments) + source
return "\n".join(trailing_comments) + source

View File

@ -21,12 +21,13 @@ PRIORITY = 3
def detect(source):
"""Detects if source is likely to be eval() packed."""
return source.strip().lower().startswith('eval(function(')
return source.strip().lower().startswith("eval(function(")
def unpack(source):
"""Runs source and return resulting code."""
return jseval('print %s;' % source[4:]) if detect(source) else source
return jseval("print %s;" % source[4:]) if detect(source) else source
# In case of failure, we'll just return the original, without crashing on user.
@ -34,7 +35,7 @@ def unpack(source):
def jseval(script):
"""Run code in the JS interpreter and return output."""
try:
interpreter = Popen(['js'], stdin=PIPE, stdout=PIPE)
interpreter = Popen(["js"], stdin=PIPE, stdout=PIPE)
except OSError:
return script
result, errors = interpreter.communicate(script)

View File

@ -27,13 +27,13 @@ def smartsplit(code):
pos = 0
while pos < len(code):
if code[pos] == '"':
word = '' # new word
word = "" # new word
pos += 1
while pos < len(code):
if code[pos] == '"':
break
if code[pos] == '\\':
word += '\\'
if code[pos] == "\\":
word += "\\"
pos += 1
word += code[pos]
pos += 1
@ -45,17 +45,17 @@ def smartsplit(code):
def detect(code):
"""Detects if `code` is JavascriptObfuscator.com packed."""
# prefer `is not` idiom, so that a true boolean is returned
return (re.search(r'^var _0x[a-f0-9]+ ?\= ?\[', code) is not None)
return re.search(r"^var _0x[a-f0-9]+ ?\= ?\[", code) is not None
def unpack(code):
"""Unpacks JavascriptObfuscator.com packed code."""
if detect(code):
matches = re.search(r'var (_0x[a-f\d]+) ?\= ?\[(.*?)\];', code)
matches = re.search(r"var (_0x[a-f\d]+) ?\= ?\[(.*?)\];", code)
if matches:
variable = matches.group(1)
dictionary = smartsplit(matches.group(2))
code = code[len(matches.group(0)):]
code = code[len(matches.group(0)) :]
for key, value in enumerate(dictionary):
code = code.replace(r'%s[%s]' % (variable, key), value)
code = code.replace(r"%s[%s]" % (variable, key), value)
return code

View File

@ -53,12 +53,13 @@ CAVEAT = """//
SIGNATURE = (
r'["\x41\x42\x43\x44\x45\x46\x47\x48\x49\x4A\x4B\x4C\x4D\x4E\x4F'
r'\x50\x51\x52\x53\x54\x55\x56\x57\x58\x59\x5A\x61\x62\x63\x64\x65'
r'\x66\x67\x68\x69\x6A\x6B\x6C\x6D\x6E\x6F\x70\x71\x72\x73\x74\x75'
r'\x76\x77\x78\x79\x7A\x30\x31\x32\x33\x34\x35\x36\x37\x38\x39\x2B'
r"\x50\x51\x52\x53\x54\x55\x56\x57\x58\x59\x5A\x61\x62\x63\x64\x65"
r"\x66\x67\x68\x69\x6A\x6B\x6C\x6D\x6E\x6F\x70\x71\x72\x73\x74\x75"
r"\x76\x77\x78\x79\x7A\x30\x31\x32\x33\x34\x35\x36\x37\x38\x39\x2B"
r'\x2F\x3D","","\x63\x68\x61\x72\x41\x74","\x69\x6E\x64\x65\x78'
r'\x4F\x66","\x66\x72\x6F\x6D\x43\x68\x61\x72\x43\x6F\x64\x65","'
r'\x6C\x65\x6E\x67\x74\x68"]')
r'\x6C\x65\x6E\x67\x74\x68"]'
)
def detect(source):
@ -71,8 +72,7 @@ def unpack(source):
if not detect(source):
return source
payload = unquote(_filter(source))
match = re.search(r"^var _escape\='<script>(.*)<\/script>'",
payload, re.DOTALL)
match = re.search(r"^var _escape\='<script>(.*)<\/script>'", payload, re.DOTALL)
polished = match.group(1) if match else source
return CAVEAT + polished
@ -80,11 +80,11 @@ def unpack(source):
def _filter(source):
"""Extracts and decode payload (original file) from `source`"""
try:
varname = re.search(r'eval\(\w+\(\w+\((\w+)\)\)\);', source).group(1)
varname = re.search(r"eval\(\w+\(\w+\((\w+)\)\)\);", source).group(1)
reverse = re.search(r"var +%s *\= *'(.*)';" % varname, source).group(1)
except AttributeError:
raise UnpackingError('Malformed MyObfuscate data.')
raise UnpackingError("Malformed MyObfuscate data.")
try:
return base64.b64decode(reverse[::-1].encode('utf8')).decode('utf8')
return base64.b64decode(reverse[::-1].encode("utf8")).decode("utf8")
except TypeError:
raise UnpackingError('MyObfuscate payload is not base64-encoded.')
raise UnpackingError("MyObfuscate payload is not base64-encoded.")

View File

@ -23,26 +23,29 @@ PRIORITY = 1
def detect(source):
global beginstr
global endstr
beginstr = ''
endstr = ''
beginstr = ""
endstr = ""
begin_offset = -1
"""Detects whether `source` is P.A.C.K.E.R. coded."""
mystr = re.search('eval[ ]*\([ ]*function[ ]*\([ ]*p[ ]*,[ ]*a[ ]*,[ ]*c['
' ]*,[ ]*k[ ]*,[ ]*e[ ]*,[ ]*', source)
if(mystr):
mystr = re.search(
"eval[ ]*\([ ]*function[ ]*\([ ]*p[ ]*,[ ]*a[ ]*,[ ]*c["
" ]*,[ ]*k[ ]*,[ ]*e[ ]*,[ ]*",
source,
)
if mystr:
begin_offset = mystr.start()
beginstr = source[:begin_offset]
if(begin_offset != -1):
if begin_offset != -1:
""" Find endstr"""
source_end = source[begin_offset:]
if(source_end.split("')))", 1)[0] == source_end):
if source_end.split("')))", 1)[0] == source_end:
try:
endstr = source_end.split("}))", 1)[1]
except IndexError:
endstr = ''
endstr = ""
else:
endstr = source_end.split("')))", 1)[1]
return (mystr is not None)
return mystr is not None
def unpack(source):
@ -50,12 +53,12 @@ def unpack(source):
payload, symtab, radix, count = _filterargs(source)
if count != len(symtab):
raise UnpackingError('Malformed p.a.c.k.e.r. symtab.')
raise UnpackingError("Malformed p.a.c.k.e.r. symtab.")
try:
unbase = Unbaser(radix)
except TypeError:
raise UnpackingError('Unknown p.a.c.k.e.r. encoding.')
raise UnpackingError("Unknown p.a.c.k.e.r. encoding.")
def lookup(match):
"""Look up symbols in the synthetic symtab."""
@ -64,9 +67,9 @@ def unpack(source):
payload = payload.replace("\\\\", "\\").replace("\\'", "'")
if sys.version_info.major == 2:
source = re.sub(r'\b\w+\b', lookup, payload)
source = re.sub(r"\b\w+\b", lookup, payload)
else:
source = re.sub(r'\b\w+\b', lookup, payload, flags=re.ASCII)
source = re.sub(r"\b\w+\b", lookup, payload, flags=re.ASCII)
return _replacestrings(source)
@ -85,13 +88,14 @@ def _filterargs(source):
a[1] = 62
a = tuple(a)
try:
return a[0], a[3].split('|'), int(a[1]), int(a[2])
return a[0], a[3].split("|"), int(a[1]), int(a[2])
except ValueError:
raise UnpackingError('Corrupted p.a.c.k.e.r. data.')
raise UnpackingError("Corrupted p.a.c.k.e.r. data.")
# could not find a satisfying regex
raise UnpackingError(
'Could not make sense of p.a.c.k.e.r data (unexpected code structure)')
"Could not make sense of p.a.c.k.e.r data (unexpected code structure)"
)
def _replacestrings(source):
@ -104,7 +108,7 @@ def _replacestrings(source):
varname, strings = match.groups()
startpoint = len(match.group(0))
lookup = strings.split('","')
variable = '%s[%%d]' % varname
variable = "%s[%%d]" % varname
for index, value in enumerate(lookup):
source = source.replace(variable % index, '"%s"' % value)
return source[startpoint:]
@ -114,10 +118,13 @@ def _replacestrings(source):
class Unbaser(object):
"""Functor for a given base. Will efficiently convert
strings to natural numbers."""
ALPHABET = {
62: '0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ',
95: (' !"#$%&\'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ'
'[\\]^_`abcdefghijklmnopqrstuvwxyz{|}~')
62: "0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ",
95: (
" !\"#$%&'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ"
"[\\]^_`abcdefghijklmnopqrstuvwxyz{|}~"
),
}
def __init__(self, base):
@ -136,10 +143,10 @@ class Unbaser(object):
# Build conversion dictionary cache
try:
self.dictionary = dict(
(cipher, index) for index, cipher in enumerate(
self.ALPHABET[base]))
(cipher, index) for index, cipher in enumerate(self.ALPHABET[base])
)
except KeyError:
raise TypeError('Unsupported base encoding.')
raise TypeError("Unsupported base encoding.")
self.unbase = self._dictunbaser

View File

@ -5,8 +5,7 @@
"""Tests for JavaScriptObfuscator unpacker."""
import unittest
from jsbeautifier.unpackers.javascriptobfuscator import (
unpack, detect, smartsplit)
from jsbeautifier.unpackers.javascriptobfuscator import unpack, detect, smartsplit
# pylint: disable=R0904
@ -18,37 +17,43 @@ class TestJavascriptObfuscator(unittest.TestCase):
"""Test smartsplit() function."""
split = smartsplit
def equals(data, result): return self.assertEqual(split(data), result)
def equals(data, result):
return self.assertEqual(split(data), result)
equals('', [])
equals("", [])
equals('"a", "b"', ['"a"', '"b"'])
equals('"aaa","bbbb"', ['"aaa"', '"bbbb"'])
equals('"a", "b\\\""', ['"a"', '"b\\\""'])
equals('"a", "b\\""', ['"a"', '"b\\""'])
def test_detect(self):
"""Test detect() function."""
def positive(source): return self.assertTrue(detect(source))
def negative(source): return self.assertFalse(detect(source))
def positive(source):
return self.assertTrue(detect(source))
negative('')
negative('abcd')
negative('var _0xaaaa')
def negative(source):
return self.assertFalse(detect(source))
negative("")
negative("abcd")
negative("var _0xaaaa")
positive('var _0xaaaa = ["a", "b"]')
positive('var _0xaaaa=["a", "b"]')
positive('var _0x1234=["a","b"]')
def test_unpack(self):
"""Test unpack() function."""
def decodeto(
ob, original): return self.assertEqual(
unpack(ob), original)
decodeto('var _0x8df3=[];var a=10;', 'var a=10;')
decodeto('var _0xb2a7=["\x74\x27\x65\x73\x74"];var i;for(i=0;i<10;++i)'
'{alert(_0xb2a7[0]);} ;', 'var i;for(i=0;i<10;++i){alert'
'("t\'est");} ;')
def decodeto(ob, original):
return self.assertEqual(unpack(ob), original)
decodeto("var _0x8df3=[];var a=10;", "var a=10;")
decodeto(
'var _0xb2a7=["\x74\x27\x65\x73\x74"];var i;for(i=0;i<10;++i)'
"{alert(_0xb2a7[0]);} ;",
"var i;for(i=0;i<10;++i){alert" '("t\'est");} ;',
)
if __name__ == '__main__':
if __name__ == "__main__":
unittest.main()

View File

@ -9,8 +9,8 @@ import os
from jsbeautifier.unpackers.myobfuscate import detect, unpack
from jsbeautifier.unpackers.tests import __path__ as path
INPUT = os.path.join(path[0], 'test-myobfuscate-input.js')
OUTPUT = os.path.join(path[0], 'test-myobfuscate-output.js')
INPUT = os.path.join(path[0], "test-myobfuscate-input.js")
OUTPUT = os.path.join(path[0], "test-myobfuscate-output.js")
# pylint: disable=R0904
@ -18,26 +18,31 @@ OUTPUT = os.path.join(path[0], 'test-myobfuscate-output.js')
class TestMyObfuscate(unittest.TestCase):
# pylint: disable=C0103
"""MyObfuscate obfuscator testcase."""
@classmethod
def setUpClass(cls):
"""Load source files (encoded and decoded version) for tests."""
with open(INPUT, 'r') as data:
with open(INPUT, "r") as data:
cls.input = data.read()
with open(OUTPUT, 'r') as data:
with open(OUTPUT, "r") as data:
cls.output = data.read()
def test_detect(self):
"""Test detect() function."""
def detected(source): return self.assertTrue(detect(source))
def detected(source):
return self.assertTrue(detect(source))
detected(self.input)
def test_unpack(self):
"""Test unpack() function."""
def check(inp, out): return self.assertEqual(unpack(inp), out)
def check(inp, out):
return self.assertEqual(unpack(inp), out)
check(self.input, self.output)
if __name__ == '__main__':
if __name__ == "__main__":
unittest.main()

View File

@ -16,42 +16,58 @@ class TestPacker(unittest.TestCase):
def test_detect(self):
"""Test detect() function."""
def positive(source): return self.assertTrue(detect(source))
def negative(source): return self.assertFalse(detect(source))
def positive(source):
return self.assertTrue(detect(source))
negative('')
negative('var a = b')
positive('eval(function(p,a,c,k,e,r')
positive('eval ( function(p, a, c, k, e, r')
def negative(source):
return self.assertFalse(detect(source))
negative("")
negative("var a = b")
positive("eval(function(p,a,c,k,e,r")
positive("eval ( function(p, a, c, k, e, r")
def test_unpack(self):
"""Test unpack() function."""
def check(inp, out):
return detect(inp) and self.assertEqual(unpack(inp), out)
check("eval(function(p,a,c,k,e,r){e=String;if(!''.replace(/^/,String)"
"){while(c--)r[c]=k[c]||c;k=[function(e){return r[e]}];e="
"function(){return'\\\\w+'};c=1};while(c--)if(k[c])p=p.replace("
"new RegExp('\\\\b'+e(c)+'\\\\b','g'),k[c]);return p}('0 2=1',"
"62,3,'var||a'.split('|'),0,{}))", 'var a=1')
check(
"eval(function(p,a,c,k,e,r){e=String;if(!''.replace(/^/,String)"
"){while(c--)r[c]=k[c]||c;k=[function(e){return r[e]}];e="
"function(){return'\\\\w+'};c=1};while(c--)if(k[c])p=p.replace("
"new RegExp('\\\\b'+e(c)+'\\\\b','g'),k[c]);return p}('0 2=1',"
"62,3,'var||a'.split('|'),0,{}))",
"var a=1",
)
check("function test (){alert ('This is a test!')}; "
"eval(function(p,a,c,k,e,r){e=String;if(!''.replace(/^/,String))"
"{while(c--)r[c]=k[c]||c;k=[function(e){return r[e]}];e=function"
"(){return'\\w+'};c=1};while(c--)if(k[c])p=p.replace(new RegExp("
"'\\b'+e(c)+'\\b','g'),k[c]);return p}('0 2=\\\'{Íâ–+›ï;ã†Ù¥#\\\'',3,3,"
"'var||a'.split('|'),0,{}))",
"function test (){alert ('This is a test!')}; var a='{Íâ–+›ï;ã†Ù¥#'")
check(
"function test (){alert ('This is a test!')}; "
"eval(function(p,a,c,k,e,r){e=String;if(!''.replace(/^/,String))"
"{while(c--)r[c]=k[c]||c;k=[function(e){return r[e]}];e=function"
"(){return'\\w+'};c=1};while(c--)if(k[c])p=p.replace(new RegExp("
"'\\b'+e(c)+'\\b','g'),k[c]);return p}('0 2=\\'{Íâ–+›ï;ã†Ù¥#\\'',3,3,"
"'var||a'.split('|'),0,{}))",
"function test (){alert ('This is a test!')}; var a='{Íâ–+›ï;ã†Ù¥#'",
)
check("eval(function(p,a,c,k,e,d){e=function(c){return c.toString(36)};if(!''.replace(/^/,String)){while(c--){d[c.toString(a)]=k[c]||c.toString(a)}k=[function(e){return d[e]}];e=function(){return'\\w+'};c=1};while(c--){if(k[c]){p=p.replace(new RegExp('\\b'+e(c)+'\\b','g'),k[c])}}return p}('2 0=\"4 3!\";2 1=0.5(/b/6);a.9(\"8\").7=1;',12,12,'str|n|var|W3Schools|Visit|search|i|innerHTML|demo|getElementById|document|w3Schools'.split('|'),0,{}))",
"var str=\"Visit W3Schools!\";var n=str.search(/w3Schools/i);document.getElementById(\"demo\").innerHTML=n;")
check(
"eval(function(p,a,c,k,e,d){e=function(c){return c.toString(36)};if(!''.replace(/^/,String)){while(c--){d[c.toString(a)]=k[c]||c.toString(a)}k=[function(e){return d[e]}];e=function(){return'\\w+'};c=1};while(c--){if(k[c]){p=p.replace(new RegExp('\\b'+e(c)+'\\b','g'),k[c])}}return p}('2 0=\"4 3!\";2 1=0.5(/b/6);a.9(\"8\").7=1;',12,12,'str|n|var|W3Schools|Visit|search|i|innerHTML|demo|getElementById|document|w3Schools'.split('|'),0,{}))",
'var str="Visit W3Schools!";var n=str.search(/w3Schools/i);document.getElementById("demo").innerHTML=n;',
)
check("a=b;\r\nwhile(1){\ng=h;{return'\\w+'};break;eval(function(p,a,c,k,e,d){e=function(c){return c.toString(36)};if(!''.replace(/^/,String)){while(c--){d[c.toString(a)]=k[c]||c.toString(a)}k=[function(e){return d[e]}];e=function(){return'\\w+'};c=1};while(c--){if(k[c]){p=p.replace(new RegExp('\\b'+e(c)+'\\b','g'),k[c])}}return p}('$(5).4(3(){$(\'.1\').0(2);$(\'.6\').0(d);$(\'.7\').0(b);$(\'.a\').0(8);$(\'.9\').0(c)});',14,14,'html|r5e57|8080|function|ready|document|r1655|rc15b|8888|r39b0|r6ae9|3128|65309|80'.split('|'),0,{}))c=abx;",
"a=b;\r\nwhile(1){\ng=h;{return'\\w+'};break;$(document).ready(function(){$('.r5e57').html(8080);$('.r1655').html(80);$('.rc15b').html(3128);$('.r6ae9').html(8888);$('.r39b0').html(65309)});c=abx;")
check(
"a=b;\r\nwhile(1){\ng=h;{return'\\w+'};break;eval(function(p,a,c,k,e,d){e=function(c){return c.toString(36)};if(!''.replace(/^/,String)){while(c--){d[c.toString(a)]=k[c]||c.toString(a)}k=[function(e){return d[e]}];e=function(){return'\\w+'};c=1};while(c--){if(k[c]){p=p.replace(new RegExp('\\b'+e(c)+'\\b','g'),k[c])}}return p}('$(5).4(3(){$('.1').0(2);$('.6').0(d);$('.7').0(b);$('.a').0(8);$('.9').0(c)});',14,14,'html|r5e57|8080|function|ready|document|r1655|rc15b|8888|r39b0|r6ae9|3128|65309|80'.split('|'),0,{}))c=abx;",
"a=b;\r\nwhile(1){\ng=h;{return'\\w+'};break;$(document).ready(function(){$('.r5e57').html(8080);$('.r1655').html(80);$('.rc15b').html(3128);$('.r6ae9').html(8888);$('.r39b0').html(65309)});c=abx;",
)
check("eval(function(p,a,c,k,e,r){e=function(c){return c.toString(36)};if('0'.replace(0,e)==0){while(c--)r[e(c)]=k[c];k=[function(e){return r[e]||e}];e=function(){return'[0-9ab]'};c=1};while(c--)if(k[c])p=p.replace(new RegExp('\\b'+e(c)+'\\b','g'),k[c]);return p}('$(5).a(6(){ $(\'.8\').0(1); $(\'.b\').0(4); $(\'.9\').0(2); $(\'.7\').0(3)})',[],12,'html|52136|555|65103|8088|document|function|r542c|r8ce6|rb0de|ready|rfab0'.split('|'),0,{}))",
"$(document).ready(function(){ $('.r8ce6').html(52136); $('.rfab0').html(8088); $('.rb0de').html(555); $('.r542c').html(65103)})")
check(
"eval(function(p,a,c,k,e,r){e=function(c){return c.toString(36)};if('0'.replace(0,e)==0){while(c--)r[e(c)]=k[c];k=[function(e){return r[e]||e}];e=function(){return'[0-9ab]'};c=1};while(c--)if(k[c])p=p.replace(new RegExp('\\b'+e(c)+'\\b','g'),k[c]);return p}('$(5).a(6(){ $('.8').0(1); $('.b').0(4); $('.9').0(2); $('.7').0(3)})',[],12,'html|52136|555|65103|8088|document|function|r542c|r8ce6|rb0de|ready|rfab0'.split('|'),0,{}))",
"$(document).ready(function(){ $('.r8ce6').html(52136); $('.rfab0').html(8088); $('.rb0de').html(555); $('.r542c').html(65103)})",
)
if __name__ == '__main__':
if __name__ == "__main__":
unittest.main()

View File

@ -16,30 +16,31 @@ class TestUrlencode(unittest.TestCase):
def test_detect(self):
"""Test detect() function."""
def encoded(source): return self.assertTrue(detect(source))
def unencoded(source): return self.assertFalse(detect(source))
def encoded(source):
return self.assertTrue(detect(source))
unencoded('')
unencoded('var a = b')
encoded('var%20a+=+b')
encoded('var%20a=b')
encoded('var%20%21%22')
def unencoded(source):
return self.assertFalse(detect(source))
unencoded("")
unencoded("var a = b")
encoded("var%20a+=+b")
encoded("var%20a=b")
encoded("var%20%21%22")
def test_unpack(self):
"""Test unpack function."""
def equals(
source,
result): return self.assertEqual(
unpack(source),
result)
equals('', '')
equals('abcd', 'abcd')
equals('var a = b', 'var a = b')
equals('var%20a=b', 'var a=b')
equals('var%20a+=+b', 'var a = b')
def equals(source, result):
return self.assertEqual(unpack(source), result)
equals("", "")
equals("abcd", "abcd")
equals("var a = b", "var a = b")
equals("var%20a=b", "var a=b")
equals("var%20a+=+b", "var a = b")
if __name__ == '__main__':
if __name__ == "__main__":
unittest.main()

View File

@ -28,7 +28,7 @@ def detect(code):
"""Detects if a scriptlet is urlencoded."""
# the fact that script doesn't contain any space, but has %20 instead
# should be sufficient check for now.
return ' ' not in code and ('%20' in code or code.count('%') > 3)
return " " not in code and ("%20" in code or code.count("%") > 3)
def unpack(code):

17
python/pyproject.toml Normal file
View File

@ -0,0 +1,17 @@
[tool.black]
target-version = ['py27', 'py36', 'py37', 'py38']
exclude = '''
/(
\.eggs
| \.git
| \.hg
| \.mypy_cache
| \.tox
| \.venv
| _build
| buck-out
| build
| dist
| tests/generated
)/
'''

View File

@ -10,37 +10,36 @@ from jsbeautifier.__version__ import __version__
from setuptools.command.test import test as TestCommand
DIR_CSS = 'cssbeautifier/tests/'
DIR_CSS = "cssbeautifier/tests/"
class PyTestCSS(PyTest):
def initialize_options(self):
TestCommand.initialize_options(self)
self.pytest_args = ['--assert=plain'] + \
[DIR_CSS + x for x in os.listdir(DIR_CSS)
if x.endswith('.py') and x[0] not in '._']
self.pytest_args = ["--assert=plain"] + [
DIR_CSS + x
for x in os.listdir(DIR_CSS)
if x.endswith(".py") and x[0] not in "._"
]
setup(name='cssbeautifier',
version=__version__,
description='CSS unobfuscator and beautifier.',
long_description=('Beautify, unpack or deobfuscate CSS'),
author='Liam Newman, Einar Lielmanis, et al.',
author_email='team@beautifier.io',
url='https://beautifier.io',
entry_points={
'console_scripts': [
'css-beautify = cssbeautifier:main'
]
},
packages=['cssbeautifier',
'cssbeautifier.tests', 'cssbeautifier.tests.generated',
'cssbeautifier.css'],
install_requires=["jsbeautifier",
"six>=1.13.0",
"editorconfig>=0.12.2"],
license='MIT',
test_suite='pytest.collector',
cmdclass={'test': PyTestCSS},
)
setup(
name="cssbeautifier",
version=__version__,
description="CSS unobfuscator and beautifier.",
long_description=("Beautify, unpack or deobfuscate CSS"),
author="Liam Newman, Einar Lielmanis, et al.",
author_email="team@beautifier.io",
url="https://beautifier.io",
entry_points={"console_scripts": ["css-beautify = cssbeautifier:main"]},
packages=[
"cssbeautifier",
"cssbeautifier.tests",
"cssbeautifier.tests.generated",
"cssbeautifier.css",
],
install_requires=["jsbeautifier", "six>=1.13.0", "editorconfig>=0.12.2"],
license="MIT",
test_suite="pytest.collector",
cmdclass={"test": PyTestCSS},
)

View File

@ -8,46 +8,49 @@ from jsbeautifier.__version__ import __version__
from setuptools.command.test import test as TestCommand
DIR = 'jsbeautifier/tests/'
DIR = "jsbeautifier/tests/"
class PyTest(TestCommand):
user_options = [('pytest-args=', 'a', "Arguments to pass to py.test")]
user_options = [("pytest-args=", "a", "Arguments to pass to py.test")]
def initialize_options(self):
TestCommand.initialize_options(self)
self.pytest_args = ['--assert=plain'] + \
[DIR + x for x in os.listdir(DIR)
if x.endswith('.py') and x[0] not in '._']
self.pytest_args = ["--assert=plain"] + [
DIR + x for x in os.listdir(DIR) if x.endswith(".py") and x[0] not in "._"
]
def run_tests(self):
# import here, cause outside the eggs aren't loaded
import pytest
errno = pytest.main(self.pytest_args)
sys.exit(errno)
setup(name='jsbeautifier',
version=__version__,
description='JavaScript unobfuscator and beautifier.',
long_description=('Beautify, unpack or deobfuscate JavaScript. '
'Handles popular online obfuscators.'),
author='Liam Newman, Einar Lielmanis, et al.',
author_email='team@beautifier.io',
url='https://beautifier.io',
entry_points={
'console_scripts': [
'js-beautify = jsbeautifier:main'
]
},
packages=['jsbeautifier',
'jsbeautifier.tests', 'jsbeautifier.tests.generated',
'jsbeautifier.core',
'jsbeautifier.javascript',
'jsbeautifier.unpackers', 'jsbeautifier.unpackers.tests'],
install_requires=["six>=1.13.0", "editorconfig>=0.12.2"],
license='MIT',
test_suite='pytest.collector',
cmdclass={'test': PyTest},
)
setup(
name="jsbeautifier",
version=__version__,
description="JavaScript unobfuscator and beautifier.",
long_description=(
"Beautify, unpack or deobfuscate JavaScript. "
"Handles popular online obfuscators."
),
author="Liam Newman, Einar Lielmanis, et al.",
author_email="team@beautifier.io",
url="https://beautifier.io",
entry_points={"console_scripts": ["js-beautify = jsbeautifier:main"]},
packages=[
"jsbeautifier",
"jsbeautifier.tests",
"jsbeautifier.tests.generated",
"jsbeautifier.core",
"jsbeautifier.javascript",
"jsbeautifier.unpackers",
"jsbeautifier.unpackers.tests",
],
install_requires=["six>=1.13.0", "editorconfig>=0.12.2"],
license="MIT",
test_suite="pytest.collector",
cmdclass={"test": PyTest},
)

View File

@ -5,9 +5,10 @@ import io
import os
import copy
import cssbeautifier
options = cssbeautifier.default_options()
options.wrap_line_length = 80
data = ''
data = ""
def beautifier_test_github_css():
@ -16,22 +17,18 @@ def beautifier_test_github_css():
def report_perf(fn):
import timeit
iter = 5
time = timeit.timeit(
fn +
"()",
setup="from __main__ import " +
fn +
"; gc.enable()",
number=iter)
fn + "()", setup="from __main__ import " + fn + "; gc.enable()", number=iter
)
print(fn + ": " + str(iter / time) + " cycles/sec")
if __name__ == '__main__':
if __name__ == "__main__":
dirname = os.path.dirname(os.path.abspath(__file__))
github_file = os.path.join(
dirname, "../", "test/resources/github.css")
data = copy.copy(''.join(io.open(github_file).readlines()))
github_file = os.path.join(dirname, "../", "test/resources/github.css")
data = copy.copy("".join(io.open(github_file).readlines()))
# warm up
beautifier_test_github_css()

View File

@ -5,10 +5,11 @@ import io
import os
import copy
import jsbeautifier
options = jsbeautifier.default_options()
options.wrap_line_length = 80
data = ''
data_min = ''
data = ""
data_min = ""
def beautifier_test_underscore():
@ -18,34 +19,35 @@ def beautifier_test_underscore():
def beautifier_test_underscore_min():
jsbeautifier.beautify(data_min, options)
def beautifier_test_github_min():
jsbeautifier.beautify(github_min, options)
def report_perf(fn):
import timeit
iter = 5
time = timeit.timeit(
fn +
"()",
setup="from __main__ import " +
fn +
"; gc.enable()",
number=iter)
fn + "()", setup="from __main__ import " + fn + "; gc.enable()", number=iter
)
print(fn + ": " + str(iter / time) + " cycles/sec")
if __name__ == '__main__':
if __name__ == "__main__":
dirname = os.path.dirname(os.path.abspath(__file__))
underscore_file = os.path.join(
dirname, "../", "test/resources/underscore.js")
underscore_file = os.path.join(dirname, "../", "test/resources/underscore.js")
underscore_min_file = os.path.join(
dirname, "../", "test/resources/underscore-min.js")
github_min_file = os.path.join(
dirname, "../", "test/resources/github-min.js")
data = copy.copy(''.join(io.open(underscore_file, encoding="UTF-8").readlines()))
data_min = copy.copy(''.join(io.open(underscore_min_file, encoding="UTF-8").readlines()))
github_min = copy.copy(''.join(io.open(github_min_file, encoding="UTF-8").readlines()))
dirname, "../", "test/resources/underscore-min.js"
)
github_min_file = os.path.join(dirname, "../", "test/resources/github-min.js")
data = copy.copy("".join(io.open(underscore_file, encoding="UTF-8").readlines()))
data_min = copy.copy(
"".join(io.open(underscore_min_file, encoding="UTF-8").readlines())
)
github_min = copy.copy(
"".join(io.open(github_min_file, encoding="UTF-8").readlines())
)
# warm up
beautifier_test_underscore()

13
tools/python-dev3 Executable file
View File

@ -0,0 +1,13 @@
#!/usr/bin/env bash
REL_SCRIPT_DIR="`dirname \"$0\"`"
SCRIPT_DIR="`( cd \"$REL_SCRIPT_DIR\" && pwd )`"
PROJECT_DIR="`( cd \"$SCRIPT_DIR/..\" && pwd )`"
PYTHON_ENV=python-dev
PYTHON_VERSION="`$PROJECT_DIR/build/$PYTHON_ENV/bin/python --version 2>&1`"
# only run the command on python3
if [ -z "${PYTHON_VERSION##Python 3.*}" ]; then
$SCRIPT_DIR/python-dev $@
fi