Refactor context parsing and add initial support for HTML.

This commit is contained in:
Erin Power
2020-06-21 15:15:16 +02:00
parent 0cbb8bc084
commit d4ef84320b
16 changed files with 351 additions and 112 deletions
+1
View File
@@ -1,3 +1,4 @@
.vscode
target
src/test
.settings
+5 -1
View File
@@ -83,6 +83,7 @@ fn generate_tests(out_dir: &OsStr) -> Result<(), Box<dyn error::Error>> {
for path in walker {
let path = path?;
let path = path.path();
let root = std::path::PathBuf::from(std::env::var("CARGO_MANIFEST_DIR").unwrap());
let name = path.file_stem().unwrap().to_str().unwrap().to_lowercase();
@@ -90,6 +91,7 @@ fn generate_tests(out_dir: &OsStr) -> Result<(), Box<dyn error::Error>> {
r#"
#[test]
fn {0}() {{
const _: &str = include_str!("{2}");
let mut languages = Languages::new();
languages.get_statistics(&["{1}"], &[], &Config::default());
@@ -103,6 +105,7 @@ fn generate_tests(out_dir: &OsStr) -> Result<(), Box<dyn error::Error>> {
let contents = fs::read_to_string("{1}").unwrap();
println!("{{}} {1}", name);
assert_eq!(get_digit!(LINES, contents), language.lines());
println!("{{}} LINES MATCH", name);
assert_eq!(get_digit!(CODE, contents), language.code);
@@ -122,7 +125,8 @@ fn generate_tests(out_dir: &OsStr) -> Result<(), Box<dyn error::Error>> {
}}
"#,
name,
path.display()
path.display(),
root.join(path).display(),
));
}
+14 -2
View File
@@ -1,2 +1,14 @@
<script>alert('<!-- Hello World -->')// </script>
</script>
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8">
<title></title>
<style>html { font-size: 100%; }</style>
</head>
<body>
<script> alert('<!-- Hello World -->') </script>
<script>
alert('<!-- Hello World -->')
</script>
</body>
</html>
+24
View File
@@ -250,6 +250,7 @@
"line_comment": ["//"],
"multi_line_comments": [["/*", "*/"]],
"quotes": [["\\\"", "\\\""], ["'", "'"]],
"mime": ["text/css"],
"extensions": ["css"]
},
"D": {
@@ -513,6 +514,7 @@
{ "tag": "script", "default": "JavaScript" },
{ "tag": "style", "default": "Css" }
],
"mime": ["text/html"],
"extensions": ["html", "htm"]
},
"Hamlet": {
@@ -577,11 +579,30 @@
"line_comment": ["//"],
"multi_line_comments": [["/*", "*/"]],
"quotes": [["\\\"", "\\\""], ["'", "'"], ["`", "`"]],
"mime": [
"application/javascript",
"application/ecmascript",
"application/x-ecmascript",
"application/x-javascript",
"text/javascript",
"text/ecmascript",
"text/javascript1.0",
"text/javascript1.1",
"text/javascript1.2",
"text/javascript1.3",
"text/javascript1.4",
"text/javascript1.5",
"text/jscript",
"text/livescript",
"text/x-ecmascript",
"text/x-javascript"
],
"extensions": ["js", "mjs"]
},
"Json": {
"name": "JSON",
"blank": true,
"mime": ["application/json", "application/manifest+json"],
"extensions": ["json"]
},
"Jsx": {
@@ -879,6 +900,7 @@
"doc_quotes": [["\\\"\\\"\\\"", "\\\"\\\"\\\""], ["'''", "'''"]],
"quotes": [["\\\"", "\\\""], ["'", "'"]],
"env": ["python", "python2", "python3"],
"mime": ["text/x-python"],
"extensions": ["py", "pyw"]
},
"Qcl": {
@@ -1048,6 +1070,7 @@
"name": "SVG",
"multi_line_comments": [["<!--", "-->"]],
"quotes": [["\\\"", "\\\""], ["'", "'"]],
"mime": ["image/svg+xml"],
"extensions": ["svg"]
},
"Swift": {
@@ -1085,6 +1108,7 @@
"Text": {
"name": "Plain Text",
"literate": true,
"mime": ["text/plain"],
"extensions": ["text", "txt"]
},
"Thrift": {
+44 -29
View File
@@ -11,7 +11,10 @@ use crate::{
config::Config,
language::syntax::{FileContext, LanguageContext, SyntaxCounter},
stats::{CodeStats, Report},
utils::{ext::SliceExt, fs as fsutils},
utils::{
ext::SliceExt,
fs as fsutils,
},
};
use encoding_rs_io::DecodeReaderBytesBuilder;
@@ -76,6 +79,10 @@ impl LanguageType {
let is_fortran = syntax.shared.is_fortran;
let is_literate = syntax.shared.is_literate;
let comments = syntax.shared.line_comments;
trace!(
"Using Simple Parse on {:?}",
String::from_utf8_lossy(skippable_text)
);
let parse_lines = move || self.parse_lines(config, rest, CodeStats::new(), syntax);
let simple_parse = move || {
LineIter::new(b'\n', skippable_text)
@@ -85,7 +92,6 @@ impl LanguageType {
// first character in the column, so removing starting whitespace
// could cause a miscount.
let line = if is_fortran { line } else { line.trim() };
trace!("{}", String::from_utf8_lossy(line));
if line.trim().is_empty() {
(1, 0, 0)
} else if is_literate
@@ -136,40 +142,49 @@ impl LanguageType {
continue;
}
if let Some(FileContext {
language,
end,
stats: blob,
}) = syntax.parse_context(lines, start, config)
{
match language {
LanguageContext::Markdown { balanced, language } => {
// Add the lines for the code fences.
stats.comments += if balanced { 2 } else { 1 };
// Add the code inside the fence to the stats.
*stats.contexts.entry(language).or_default() += blob;
}
LanguageContext::Rust => {
// Add all the markdown blobs.
*stats.contexts.entry(LanguageType::Markdown).or_default() += blob;
}
}
let started_in_comments = !syntax.stack.is_empty()
|| (config.treat_doc_strings_as_comments == Some(true)
&& syntax.quote.is_some()
&& syntax.quote_is_doc_quote);
let ended_with_comments =
match syntax.perform_multi_line_analysis(lines, start, end, config) {
crate::language::syntax::AnalysisReport::Normal(end) => end,
crate::language::syntax::AnalysisReport::ChildLanguage(FileContext {
language,
end,
stats: blob,
}) => {
match language {
LanguageContext::Markdown { balanced, language } => {
// Add the lines for the code fences.
stats.comments += if balanced { 2 } else { 1 };
// Add the code inside the fence to the stats.
*stats.contexts.entry(language).or_default() += blob;
}
LanguageContext::Rust => {
// Add all the markdown blobs.
*stats.contexts.entry(LanguageType::Markdown).or_default() += blob;
}
LanguageContext::Html { language } => {
stats.code += 1;
// Add all the markdown blobs.
*stats.contexts.entry(language).or_default() += blob;
}
}
// Advance to after the language code and the delimiter..
stepper = LineStep::new(b'\n', end, lines.len());
continue;
}
let had_multi_line = !syntax.stack.is_empty();
let ended_with_comments = syntax.perform_multi_line_analysis(line);
// Advance to after the language code and the delimiter..
stepper = LineStep::new(b'\n', end, lines.len());
continue;
}
};
trace!("{}", String::from_utf8_lossy(line));
if syntax.shared.is_literate
|| syntax.line_is_comment(line, config, ended_with_comments, had_multi_line)
|| syntax.line_is_comment(line, config, ended_with_comments, started_in_comments)
{
stats.comments += 1;
trace!("Comment No.{}", stats.comments);
trace!("Was the Comment stack empty?: {}", !had_multi_line);
trace!("Was the Comment stack empty?: {}", !started_in_comments);
} else {
stats.code += 1;
trace!("Code No.{}", stats.code);
+51 -6
View File
@@ -29,7 +29,7 @@ impl LanguageType {
}
}
pub(crate) fn is_blank(self) -> bool {
pub(crate) fn _is_blank(self) -> bool {
match self {
{% for key, v in languages -%}
{{key}} => {{ v.blank | default(value=false) }},
@@ -194,14 +194,32 @@ impl LanguageType {
}
}
pub(crate) fn start_any_comments(self) -> &'static [&'static str] {
pub(crate) fn any_multi_line_comments(self) -> &'static [(&'static str, &'static str)] {
match self {
{% for key, value in languages -%}
{%- set starting_multi_line_comments = value.multi_line_comments | default(value=[]) | map(attribute="0") -%}
{%- set starting_nested_comments = value.nested_comments | default(value=[]) | map(attribute="0") -%}
{{key}} => &[
{%- for item in value.line_comment | default(value=[]) | concat(with=starting_multi_line_comments) | concat(with=starting_nested_comments) -%}
{%- set starting_multi_line_comments = value.multi_line_comments | default(value=[]) -%}
{%- set starting_nested_comments = value.nested_comments | default(value=[]) -%}
{%- for item in starting_multi_line_comments | concat(with=starting_nested_comments) -%}
("{{item.0}}", "{{item.1}}"),
{%- endfor -%}
],
{% endfor %}
}
}
pub(crate) fn any_comments(self) -> &'static [&'static str] {
match self {
{% for key, value in languages -%}
{{key}} => &[
{%- set starting_multi_line_comments = value.multi_line_comments | default(value=[]) -%}
{%- set starting_nested_comments = value.nested_comments | default(value=[]) -%}
{%- for item in starting_multi_line_comments | concat(with=starting_nested_comments) -%}
"{{item.0}}",
"{{item.1}}",
{%- endfor -%}
{%- for item in value.line_comment | default(value=[]) -%}
"{{item}}",
{%- endfor -%}
],
@@ -222,6 +240,10 @@ impl LanguageType {
{{key}} => &[
{% if key == "Markdown" %}
"{{value.code_fence}}",
{% elif key == "Html" %}
"<script",
"<style",
"<template",
{% elif key == "Rust" %}
"///",
"//!",
@@ -300,6 +322,29 @@ impl LanguageType {
}
}
/// Get language from its MIME type if available.
///
/// ```no_run
/// use tokei::LanguageType;
///
/// let lang = LanguageType::from_mime("application/javascript");
///
/// assert_eq!(lang, Some(LanguageType::JavaScript));
/// ```
pub fn from_mime(mime: &str) -> Option<Self> {
match mime {
{% for key, value in languages -%}
{%- if value.mime -%}
{%- for item in value.mime %}| "{{item}}" {% endfor %}=> Some({{key}}),
{% endif -%}
{%- endfor %}
_ => {
warn!("Unknown MIME: {}", mime);
None
},
}
}
/// Get language from a shebang. May open and read the file.
///
/// ```no_run
+143 -34
View File
@@ -48,6 +48,9 @@ impl FileContext {
#[derive(Clone, Debug)]
pub(crate) enum LanguageContext {
Html {
language: LanguageType,
},
Markdown {
balanced: bool,
language: LanguageType,
@@ -61,10 +64,11 @@ pub(crate) struct SharedMatchers {
pub allows_nested: bool,
pub doc_quotes: &'static [(&'static str, &'static str)],
pub important_syntax: AhoCorasick<u16>,
pub any_comments: AhoCorasick<u16>,
pub any_comments: &'static [&'static str],
pub is_fortran: bool,
pub is_literate: bool,
pub line_comments: &'static [&'static str],
pub any_multi_line_comments: &'static [(&'static str, &'static str)],
pub multi_line_comments: &'static [(&'static str, &'static str)],
pub nested_comments: &'static [(&'static str, &'static str)],
pub string_literals: &'static [(&'static str, &'static str)],
@@ -100,9 +104,10 @@ impl SharedMatchers {
is_fortran: language.is_fortran(),
is_literate: language.is_literate(),
important_syntax: init_corasick(language.important_syntax(), false),
any_comments: init_corasick(language.start_any_comments(), true),
any_comments: language.any_comments(),
line_comments: language.line_comments(),
multi_line_comments: language.multi_line_comments(),
any_multi_line_comments: language.any_multi_line_comments(),
nested_comments: language.nested_comments(),
string_literals: language.quotes(),
verbatim_string_literals: language.verbatim_quotes(),
@@ -110,6 +115,14 @@ impl SharedMatchers {
}
}
#[derive(Debug)]
pub(crate) enum AnalysisReport {
/// No child languages were found, contains a boolean representing whether
/// the line ended with comments or not.
Normal(bool),
ChildLanguage(FileContext),
}
impl SyntaxCounter {
pub(crate) fn new(language: LanguageType) -> Self {
Self {
@@ -189,7 +202,13 @@ impl SyntaxCounter {
false
}
pub(crate) fn perform_multi_line_analysis(&mut self, line: &[u8]) -> bool {
pub(crate) fn perform_multi_line_analysis(
&mut self,
lines: &[u8],
start: usize,
end: usize,
config: &Config,
) -> AnalysisReport {
let mut ended_with_comments = false;
let mut skip = 0;
macro_rules! skip {
@@ -198,15 +217,19 @@ impl SyntaxCounter {
}};
}
for i in 0..line.len() {
for i in start..end {
if skip != 0 {
skip -= 1;
continue;
}
ended_with_comments = false;
let window = &line[i..];
let window = &lines[i..];
if window.trim().is_empty() {
break;
}
ended_with_comments = false;
let is_end_of_quote_or_multi_line = self
.parse_end_of_quote(window)
.or_else(|| self.parse_end_of_multi_line(window));
@@ -219,6 +242,10 @@ impl SyntaxCounter {
continue;
}
if let Some(child) = self.parse_context(lines, i, end, config) {
return AnalysisReport::ChildLanguage(child);
}
let is_quote_or_multi_line = self
.parse_quote(window)
.or_else(|| self.parse_multi_line_comment(window));
@@ -234,32 +261,67 @@ impl SyntaxCounter {
}
}
ended_with_comments
AnalysisReport::Normal(ended_with_comments)
}
pub(crate) fn line_is_comment(
&self,
line: &[u8],
config: &crate::Config,
ended_with_comments: bool,
had_multi_line: bool,
_ended_with_comments: bool,
started_in_comments: bool,
) -> bool {
((!self.stack.is_empty() || ended_with_comments) && had_multi_line)
|| (
// If we're currently in a comment or we just ended
// with one.
self.shared.any_comments.is_match(line) && self.quote.is_none()
)
|| ((
// If we're currently in a doc string or we just ended
// with one.
self.quote.is_some() ||
self.shared.doc_quotes.iter().any(|(s, _)| line.starts_with(s.as_bytes()))
) &&
// `Some(true)` is import in order to respect the current
// configuration.
config.treat_doc_strings_as_comments == Some(true) &&
self.quote_is_doc_quote)
let trimmed = line.trim();
let whole_line_is_comment = || {
self.shared
.line_comments
.iter()
.any(|c| trimmed.starts_with(c.as_bytes()))
|| self
.shared
.any_multi_line_comments
.iter()
.any(|(start, end)| {
trimmed.starts_with(start.as_bytes()) && trimmed.ends_with(end.as_bytes())
})
};
let starts_with_comment = || {
let quote = match self.stack.last() {
Some(q) => q,
_ => return false,
};
self.shared
.any_multi_line_comments
.iter()
.any(|(start, end)| end == quote && trimmed.starts_with(start.as_bytes()))
};
// `Some(true)` in order to respect the current configuration.
if self.quote.is_some() {
if self.quote_is_doc_quote && config.treat_doc_strings_as_comments == Some(true) {
self.quote.map_or(false, |q| line.starts_with(q.as_bytes()))
|| (self.quote.is_some())
} else {
false
}
} else if self
.shared
.doc_quotes
.iter()
.any(|(_, e)| line.contains_slice(e.as_bytes()))
&& started_in_comments
{
true
} else if (whole_line_is_comment)() {
true
} else if started_in_comments {
true
} else if (starts_with_comment)() {
true
} else {
false
}
}
#[inline]
@@ -267,6 +329,7 @@ impl SyntaxCounter {
&mut self,
lines: &[u8],
start: usize,
end: usize,
config: &Config,
) -> Option<FileContext> {
use std::str::FromStr;
@@ -281,11 +344,18 @@ impl SyntaxCounter {
static STARTING_MARKDOWN_REGEX: Lazy<Regex> =
Lazy::new(|| Regex::new(r#"^```\S+\r?\n"#).unwrap());
static ENDING_MARKDOWN_REGEX: Lazy<Regex> =
Lazy::new(|| Regex::new(r#"```\r?\n"#).unwrap());
// dbg!(String::from_utf8_lossy(&lines[start..end]));
let opening_fence = STARTING_MARKDOWN_REGEX.find(&lines[start..])?;
Lazy::new(|| Regex::new(r#"```(?:\r?\n)?"#).unwrap());
if !lines[start..end].contains_slice(b"```") {
return None;
}
let opening_fence = STARTING_MARKDOWN_REGEX.find(&lines[start..end])?;
let start_of_code = start + opening_fence.end();
let closing_fence = ENDING_MARKDOWN_REGEX.find(&lines[start_of_code..]);
if let Some(m) = &closing_fence {
trace!("{:?}", String::from_utf8_lossy(m.as_bytes()))
}
let end_of_code = closing_fence
.map(|fence| start_of_code + fence.start())
.unwrap_or_else(|| lines.len());
@@ -298,8 +368,13 @@ impl SyntaxCounter {
&opening_fence.as_bytes().trim()[3..],
))
.ok()?;
let stats = language.parse_from_slice(&lines[start_of_code..end_of_code], config);
// dbg!(String::from_utf8_lossy(&lines[end_of_code_block..]));
trace!(
"{} BLOCK: {:?}",
language,
String::from_utf8_lossy(&lines[start_of_code..end_of_code])
);
let stats =
language.parse_from_slice(&lines[start_of_code..end_of_code].trim(), config);
Some(FileContext::new(
LanguageContext::Markdown { balanced, language },
@@ -323,6 +398,7 @@ impl SyntaxCounter {
while let Some((start, end)) = stepper.next(lines) {
if lines[start..].trim().starts_with(comment_syntax) {
trace!("{}", String::from_utf8_lossy(&lines[start..end]));
let line = lines[start..end].trim_start();
let stripped_line = &line[3.min(line.len())..];
markdown.extend_from_slice(stripped_line);
@@ -333,8 +409,8 @@ impl SyntaxCounter {
}
}
//dbg!(String::from_utf8_lossy(&lines[end_of_block..]));
let doc_block = LanguageType::Markdown.parse_from_slice(&markdown, config);
trace!("Markdown found: {:?}", String::from_utf8_lossy(&markdown));
let doc_block = LanguageType::Markdown.parse_from_slice(markdown.trim(), config);
Some(FileContext::new(
LanguageContext::Rust,
@@ -342,6 +418,32 @@ impl SyntaxCounter {
doc_block,
))
}
LanguageType::Html => {
static START_SCRIPT: Lazy<Regex> =
Lazy::new(|| Regex::new(r#"^<script(?:.*type="(.*)")?.*?>"#).unwrap());
static END_SCRIPT: Lazy<Regex> = Lazy::new(|| Regex::new(r#"</script>"#).unwrap());
let opening_tag = START_SCRIPT.find(&lines[start..end])?;
let captures = START_SCRIPT.captures(&lines[start..end])?;
let start_of_code = start + opening_tag.end();
let closing_tag = END_SCRIPT.find(&lines[start_of_code..])?;
let end_of_code = start_of_code + closing_tag.start();
let language = captures
.get(1)
.and_then(|m| {
LanguageType::from_mime(&String::from_utf8_lossy(m.as_bytes().trim()))
})
.unwrap_or(LanguageType::JavaScript);
let stats =
language.parse_from_slice(&lines[start_of_code..end_of_code].trim(), config);
Some(FileContext::new(
LanguageContext::Html { language },
start_of_code + closing_tag.end(),
stats,
))
}
_ => None,
}
}
@@ -396,11 +498,18 @@ impl SyntaxCounter {
#[inline]
pub(crate) fn parse_end_of_quote(&mut self, window: &[u8]) -> Option<usize> {
if window.starts_with(self.quote?.as_bytes()) {
if self._is_string_mode() && window.starts_with(self.quote?.as_bytes()) {
let quote = self.quote.take().unwrap();
trace!("End {:?}", quote);
Some(quote.len())
} else if window.starts_with(br"\") && !self.quote_is_verbatim {
} else if !self.quote_is_verbatim
&& window.starts_with(br"\")
&& self
.shared
.string_literals
.iter()
.any(|(start, _)| window[1..].starts_with(start.as_bytes()))
{
// Tell the state machine to skip the next character because it
// has been escaped if the string isn't a verbatim string.
Some(2)
+8 -3
View File
@@ -1,12 +1,17 @@
//! Various extensions to Rust std types.
pub(crate) trait AsciiExt {
fn is_whitespace(&self) -> bool;
fn is_whitespace(self) -> bool;
fn is_not_line_ending_whitespace(self) -> bool;
}
impl AsciiExt for u8 {
fn is_whitespace(&self) -> bool {
*self == b' ' || (b'\x09'..=b'\x0d').contains(self)
fn is_whitespace(self) -> bool {
self == b' ' || (b'\x09'..=b'\x0d').contains(&self)
}
fn is_not_line_ending_whitespace(self) -> bool {
self.is_whitespace() && self != b'\r' && self != b'\n'
}
}
+2 -2
View File
@@ -79,9 +79,9 @@ mod config {
let (_, language) = languages.into_iter().next().unwrap();
assert_eq!(language.lines(), 15);
assert_eq!(language.code, 5);
assert_eq!(language.comments, 7);
assert_eq!(language.blanks, 3);
assert_eq!(language.comments, 7);
assert_eq!(language.code, 5);
}
}
+1 -1
View File
@@ -1,4 +1,4 @@
{! 10 lines 3 code 4 comments 3 blanks !}
{! 10 lines 2 code 5 comments 3 blanks !}
{! All Dust comments are multiline comments. And there's no quoting
comment openers and closers. Instead there are escape sequences
+30 -30
View File
@@ -1,46 +1,46 @@
<!-- 46 lines 23 code 19 comments 4 blanks -->
<!-- 46 lines 22 code 19 comments 5 blanks -->
<!DOCTYPE html>
<html>
<head>
<meta charset="utf-8" />
<meta name="viewport" content="width=device-width" />
<title></title>
link
<style>
/*
<head>
<meta charset="utf-8" />
<meta name="viewport" content="width=device-width" />
<title></title>
link
<style>
/*
CSS multi-line comment
CSS multi-line comment
*/
body {
background-color: pink;
}
</style>
</head>
<body>
body
</body>
<!-- Normal Comment-->
*/
body {
background-color: pink;
}
</style>
</head>
<body>
body
</body>
<!-- Normal Comment-->
<nav class="navbar navbar-default navbar-fixed-top navbar-custom">
<div id="modalSearch" class="modal fade" role="dialog"> </div>
</nav>
<nav class="navbar navbar-default navbar-fixed-top navbar-custom">
<div id="modalSearch" class="modal fade" role="dialog"> </div>
</nav>
<!--
document.write("Multi-line and Code comment!");
<!--
document.write("Multi-line and Code comment!");
//-->
<!--[if IE 8]>
IE Special comment
IE Special comment
<![endif]-->
<script>
/*
/*
Javascript multi-line comment
Javascript multi-line comment
*/
let x = 5;
// Javascript single line comment
*/
let x = 5;
// Javascript single line comment
</script>
</html>
+1 -1
View File
@@ -1,4 +1,4 @@
# 24 lines 15 code 7 comments 2 blanks
# 24 lines 14 code 8 comments 2 blanks
=begin
Regex methods and functions
=end
+1 -1
View File
@@ -1,4 +1,4 @@
# 11 lines 5 code 4 comments 2 blanks
# 11 lines 4 code 5 comments 2 blanks
=begin pod
Defines a fun infix operator.
+1 -1
View File
@@ -1,4 +1,4 @@
//! 48 lines 33 code 9 comments 6 blanks
//! 48 lines 36 code 6 comments 6 blanks
//! ```rust
//! fn main () {
//! // Comment
+24
View File
@@ -0,0 +1,24 @@
// 24 lines 6 code 14 comments 4 blanks
// Single-line comment
/* multi-line comment */
/* /* nested */ */
/*
multi
line
comment
*/
/*
nested /* */
/* nested */
*/
import UIKit
class ViewController: UIViewController {
override func viewDidLoad() {
super.viewDidLoad()
// Do any additional setup after loading the view.
}
}
+1 -1
View File
@@ -1,4 +1,4 @@
## 11 lines 5 code 4 comments 2 blanks
## 11 lines 4 code 5 comments 2 blanks
This text is visible. #* This text, as part of a multi-line
comment, is not visible. This text is not visible; it is also