Fix private fields by adding new start token to identifiers

This commit is contained in:
Michael Dombrowski 2019-12-03 08:02:39 -08:00
parent c26968ab88
commit e19cf93ae4
4 changed files with 6 additions and 17 deletions

View File

@ -17,11 +17,11 @@
// acorn used char codes to squeeze the last bit of performance out
// Beautifier is okay without that, so we're using regex
// permit $ (36) and @ (64). @ is used in ES7 decorators.
// permit # (23), $ (36), and @ (64). @ is used in ES7 decorators.
// 65 through 91 are uppercase letters.
// permit _ (95).
// 97 through 123 are lowercase letters.
var baseASCIIidentifierStartChars = "\\x24\\x40\\x41-\\x5a\\x5f\\x61-\\x7a";
var baseASCIIidentifierStartChars = "\\x23\\x24\\x40\\x41-\\x5a\\x5f\\x61-\\x7a";
// inside an identifier @ is not allowed but 0-9 are.
var baseASCIIidentifierChars = "\\x24\\x30-\\x39\\x41-\\x5a\\x5f\\x61-\\x7a";

View File

@ -163,13 +163,13 @@ Tokenizer.prototype._get_next_token = function(previous_token, open_token) { //
return this._create_token(TOKEN.EOF, '');
}
token = token || this._read_non_javascript(c);
token = token || this._read_string(c);
token = token || this._read_word(previous_token);
token = token || this._read_singles(c);
token = token || this._read_comment(c);
token = token || this._read_regexp(c, previous_token);
token = token || this._read_xml(c, previous_token);
token = token || this._read_non_javascript(c);
token = token || this._read_punctuation();
token = token || this._create_token(TOKEN.UNKNOWN, this._input.next());
@ -274,12 +274,6 @@ Tokenizer.prototype._read_non_javascript = function(c) {
this._input.next();
}
return this._create_token(TOKEN.WORD, sharp);
} else if (this._input.hasNext()) {
// Handle private field (ie. #privateExample)
resulting_string = this.__patterns.identifier.read();
if (resulting_string) {
return this._create_token(TOKEN.WORD, sharp + resulting_string);
}
}
this._input.back();

View File

@ -20,11 +20,11 @@ six = __import__("six")
# acorn used char codes to squeeze the last bit of performance out
# Beautifier is okay without that, so we're using regex
# permit $ (36) and @ (64). @ is used in ES7 decorators.
# permit #(23), $ (36), and @ (64). @ is used in ES7 decorators.
# 65 through 91 are uppercase letters.
# permit _ (95).
# 97 through 123 are lowercase letters.
_baseASCIIidentifierStartChars = six.u(r"\x24\x40\x41-\x5a\x5f\x61-\x7a")
_baseASCIIidentifierStartChars = six.u(r"\x23\x24\x40\x41-\x5a\x5f\x61-\x7a")
# inside an identifier @ is not allowed but 0-9 are.
_baseASCIIidentifierChars = six.u(r"\x24\x30-\x39\x41-\x5a\x5f\x61-\x7a")

View File

@ -191,13 +191,13 @@ class Tokenizer(BaseTokenizer):
if c is None:
token = self._create_token(TOKEN.EOF, '')
token = token or self._read_non_javascript(c)
token = token or self._read_string(c)
token = token or self._read_word(previous_token)
token = token or self._read_singles(c)
token = token or self._read_comment(c)
token = token or self._read_regexp(c, previous_token)
token = token or self._read_xml(c, previous_token)
token = token or self._read_non_javascript(c)
token = token or self._read_punctuation()
token = token or self._create_token(TOKEN.UNKNOWN, self._input.next())
@ -408,11 +408,6 @@ class Tokenizer(BaseTokenizer):
self._input.next()
return self._create_token(TOKEN.WORD, sharp)
elif self._input.hasNext():
# Handle private field (ie. #privateExample)
resulting_string = self._patterns.identifier.read()
if resulting_string:
return self._create_token(TOKEN.WORD, sharp + resulting_string)
self._input.back()