mirror of
https://github.com/XAMPPRocky/tokei.git
synced 2026-10-11 22:29:59 +00:00
Refactor context parsing and add initial support for HTML.
This commit is contained in:
@@ -1,3 +1,4 @@
|
||||
.vscode
|
||||
target
|
||||
src/test
|
||||
.settings
|
||||
|
||||
@@ -83,6 +83,7 @@ fn generate_tests(out_dir: &OsStr) -> Result<(), Box<dyn error::Error>> {
|
||||
for path in walker {
|
||||
let path = path?;
|
||||
let path = path.path();
|
||||
let root = std::path::PathBuf::from(std::env::var("CARGO_MANIFEST_DIR").unwrap());
|
||||
|
||||
let name = path.file_stem().unwrap().to_str().unwrap().to_lowercase();
|
||||
|
||||
@@ -90,6 +91,7 @@ fn generate_tests(out_dir: &OsStr) -> Result<(), Box<dyn error::Error>> {
|
||||
r#"
|
||||
#[test]
|
||||
fn {0}() {{
|
||||
const _: &str = include_str!("{2}");
|
||||
let mut languages = Languages::new();
|
||||
languages.get_statistics(&["{1}"], &[], &Config::default());
|
||||
|
||||
@@ -103,6 +105,7 @@ fn generate_tests(out_dir: &OsStr) -> Result<(), Box<dyn error::Error>> {
|
||||
|
||||
let contents = fs::read_to_string("{1}").unwrap();
|
||||
|
||||
println!("{{}} {1}", name);
|
||||
assert_eq!(get_digit!(LINES, contents), language.lines());
|
||||
println!("{{}} LINES MATCH", name);
|
||||
assert_eq!(get_digit!(CODE, contents), language.code);
|
||||
@@ -122,7 +125,8 @@ fn generate_tests(out_dir: &OsStr) -> Result<(), Box<dyn error::Error>> {
|
||||
}}
|
||||
"#,
|
||||
name,
|
||||
path.display()
|
||||
path.display(),
|
||||
root.join(path).display(),
|
||||
));
|
||||
}
|
||||
|
||||
|
||||
+14
-2
@@ -1,2 +1,14 @@
|
||||
<script>alert('<!-- Hello World -->')// </script>
|
||||
</script>
|
||||
<!DOCTYPE html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="UTF-8">
|
||||
<title></title>
|
||||
<style>html { font-size: 100%; }</style>
|
||||
</head>
|
||||
<body>
|
||||
<script> alert('<!-- Hello World -->') </script>
|
||||
<script>
|
||||
alert('<!-- Hello World -->')
|
||||
</script>
|
||||
</body>
|
||||
</html>
|
||||
|
||||
@@ -250,6 +250,7 @@
|
||||
"line_comment": ["//"],
|
||||
"multi_line_comments": [["/*", "*/"]],
|
||||
"quotes": [["\\\"", "\\\""], ["'", "'"]],
|
||||
"mime": ["text/css"],
|
||||
"extensions": ["css"]
|
||||
},
|
||||
"D": {
|
||||
@@ -513,6 +514,7 @@
|
||||
{ "tag": "script", "default": "JavaScript" },
|
||||
{ "tag": "style", "default": "Css" }
|
||||
],
|
||||
"mime": ["text/html"],
|
||||
"extensions": ["html", "htm"]
|
||||
},
|
||||
"Hamlet": {
|
||||
@@ -577,11 +579,30 @@
|
||||
"line_comment": ["//"],
|
||||
"multi_line_comments": [["/*", "*/"]],
|
||||
"quotes": [["\\\"", "\\\""], ["'", "'"], ["`", "`"]],
|
||||
"mime": [
|
||||
"application/javascript",
|
||||
"application/ecmascript",
|
||||
"application/x-ecmascript",
|
||||
"application/x-javascript",
|
||||
"text/javascript",
|
||||
"text/ecmascript",
|
||||
"text/javascript1.0",
|
||||
"text/javascript1.1",
|
||||
"text/javascript1.2",
|
||||
"text/javascript1.3",
|
||||
"text/javascript1.4",
|
||||
"text/javascript1.5",
|
||||
"text/jscript",
|
||||
"text/livescript",
|
||||
"text/x-ecmascript",
|
||||
"text/x-javascript"
|
||||
],
|
||||
"extensions": ["js", "mjs"]
|
||||
},
|
||||
"Json": {
|
||||
"name": "JSON",
|
||||
"blank": true,
|
||||
"mime": ["application/json", "application/manifest+json"],
|
||||
"extensions": ["json"]
|
||||
},
|
||||
"Jsx": {
|
||||
@@ -879,6 +900,7 @@
|
||||
"doc_quotes": [["\\\"\\\"\\\"", "\\\"\\\"\\\""], ["'''", "'''"]],
|
||||
"quotes": [["\\\"", "\\\""], ["'", "'"]],
|
||||
"env": ["python", "python2", "python3"],
|
||||
"mime": ["text/x-python"],
|
||||
"extensions": ["py", "pyw"]
|
||||
},
|
||||
"Qcl": {
|
||||
@@ -1048,6 +1070,7 @@
|
||||
"name": "SVG",
|
||||
"multi_line_comments": [["<!--", "-->"]],
|
||||
"quotes": [["\\\"", "\\\""], ["'", "'"]],
|
||||
"mime": ["image/svg+xml"],
|
||||
"extensions": ["svg"]
|
||||
},
|
||||
"Swift": {
|
||||
@@ -1085,6 +1108,7 @@
|
||||
"Text": {
|
||||
"name": "Plain Text",
|
||||
"literate": true,
|
||||
"mime": ["text/plain"],
|
||||
"extensions": ["text", "txt"]
|
||||
},
|
||||
"Thrift": {
|
||||
|
||||
@@ -11,7 +11,10 @@ use crate::{
|
||||
config::Config,
|
||||
language::syntax::{FileContext, LanguageContext, SyntaxCounter},
|
||||
stats::{CodeStats, Report},
|
||||
utils::{ext::SliceExt, fs as fsutils},
|
||||
utils::{
|
||||
ext::SliceExt,
|
||||
fs as fsutils,
|
||||
},
|
||||
};
|
||||
|
||||
use encoding_rs_io::DecodeReaderBytesBuilder;
|
||||
@@ -76,6 +79,10 @@ impl LanguageType {
|
||||
let is_fortran = syntax.shared.is_fortran;
|
||||
let is_literate = syntax.shared.is_literate;
|
||||
let comments = syntax.shared.line_comments;
|
||||
trace!(
|
||||
"Using Simple Parse on {:?}",
|
||||
String::from_utf8_lossy(skippable_text)
|
||||
);
|
||||
let parse_lines = move || self.parse_lines(config, rest, CodeStats::new(), syntax);
|
||||
let simple_parse = move || {
|
||||
LineIter::new(b'\n', skippable_text)
|
||||
@@ -85,7 +92,6 @@ impl LanguageType {
|
||||
// first character in the column, so removing starting whitespace
|
||||
// could cause a miscount.
|
||||
let line = if is_fortran { line } else { line.trim() };
|
||||
trace!("{}", String::from_utf8_lossy(line));
|
||||
if line.trim().is_empty() {
|
||||
(1, 0, 0)
|
||||
} else if is_literate
|
||||
@@ -136,40 +142,49 @@ impl LanguageType {
|
||||
continue;
|
||||
}
|
||||
|
||||
if let Some(FileContext {
|
||||
language,
|
||||
end,
|
||||
stats: blob,
|
||||
}) = syntax.parse_context(lines, start, config)
|
||||
{
|
||||
match language {
|
||||
LanguageContext::Markdown { balanced, language } => {
|
||||
// Add the lines for the code fences.
|
||||
stats.comments += if balanced { 2 } else { 1 };
|
||||
// Add the code inside the fence to the stats.
|
||||
*stats.contexts.entry(language).or_default() += blob;
|
||||
}
|
||||
LanguageContext::Rust => {
|
||||
// Add all the markdown blobs.
|
||||
*stats.contexts.entry(LanguageType::Markdown).or_default() += blob;
|
||||
}
|
||||
}
|
||||
let started_in_comments = !syntax.stack.is_empty()
|
||||
|| (config.treat_doc_strings_as_comments == Some(true)
|
||||
&& syntax.quote.is_some()
|
||||
&& syntax.quote_is_doc_quote);
|
||||
let ended_with_comments =
|
||||
match syntax.perform_multi_line_analysis(lines, start, end, config) {
|
||||
crate::language::syntax::AnalysisReport::Normal(end) => end,
|
||||
crate::language::syntax::AnalysisReport::ChildLanguage(FileContext {
|
||||
language,
|
||||
end,
|
||||
stats: blob,
|
||||
}) => {
|
||||
match language {
|
||||
LanguageContext::Markdown { balanced, language } => {
|
||||
// Add the lines for the code fences.
|
||||
stats.comments += if balanced { 2 } else { 1 };
|
||||
// Add the code inside the fence to the stats.
|
||||
*stats.contexts.entry(language).or_default() += blob;
|
||||
}
|
||||
LanguageContext::Rust => {
|
||||
// Add all the markdown blobs.
|
||||
*stats.contexts.entry(LanguageType::Markdown).or_default() += blob;
|
||||
}
|
||||
LanguageContext::Html { language } => {
|
||||
stats.code += 1;
|
||||
// Add all the markdown blobs.
|
||||
*stats.contexts.entry(language).or_default() += blob;
|
||||
}
|
||||
}
|
||||
|
||||
// Advance to after the language code and the delimiter..
|
||||
stepper = LineStep::new(b'\n', end, lines.len());
|
||||
continue;
|
||||
}
|
||||
|
||||
let had_multi_line = !syntax.stack.is_empty();
|
||||
let ended_with_comments = syntax.perform_multi_line_analysis(line);
|
||||
// Advance to after the language code and the delimiter..
|
||||
stepper = LineStep::new(b'\n', end, lines.len());
|
||||
continue;
|
||||
}
|
||||
};
|
||||
trace!("{}", String::from_utf8_lossy(line));
|
||||
|
||||
if syntax.shared.is_literate
|
||||
|| syntax.line_is_comment(line, config, ended_with_comments, had_multi_line)
|
||||
|| syntax.line_is_comment(line, config, ended_with_comments, started_in_comments)
|
||||
{
|
||||
stats.comments += 1;
|
||||
trace!("Comment No.{}", stats.comments);
|
||||
trace!("Was the Comment stack empty?: {}", !had_multi_line);
|
||||
trace!("Was the Comment stack empty?: {}", !started_in_comments);
|
||||
} else {
|
||||
stats.code += 1;
|
||||
trace!("Code No.{}", stats.code);
|
||||
|
||||
@@ -29,7 +29,7 @@ impl LanguageType {
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn is_blank(self) -> bool {
|
||||
pub(crate) fn _is_blank(self) -> bool {
|
||||
match self {
|
||||
{% for key, v in languages -%}
|
||||
{{key}} => {{ v.blank | default(value=false) }},
|
||||
@@ -194,14 +194,32 @@ impl LanguageType {
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn start_any_comments(self) -> &'static [&'static str] {
|
||||
pub(crate) fn any_multi_line_comments(self) -> &'static [(&'static str, &'static str)] {
|
||||
match self {
|
||||
{% for key, value in languages -%}
|
||||
{%- set starting_multi_line_comments = value.multi_line_comments | default(value=[]) | map(attribute="0") -%}
|
||||
{%- set starting_nested_comments = value.nested_comments | default(value=[]) | map(attribute="0") -%}
|
||||
|
||||
{{key}} => &[
|
||||
{%- for item in value.line_comment | default(value=[]) | concat(with=starting_multi_line_comments) | concat(with=starting_nested_comments) -%}
|
||||
{%- set starting_multi_line_comments = value.multi_line_comments | default(value=[]) -%}
|
||||
{%- set starting_nested_comments = value.nested_comments | default(value=[]) -%}
|
||||
{%- for item in starting_multi_line_comments | concat(with=starting_nested_comments) -%}
|
||||
("{{item.0}}", "{{item.1}}"),
|
||||
{%- endfor -%}
|
||||
],
|
||||
{% endfor %}
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn any_comments(self) -> &'static [&'static str] {
|
||||
match self {
|
||||
{% for key, value in languages -%}
|
||||
{{key}} => &[
|
||||
{%- set starting_multi_line_comments = value.multi_line_comments | default(value=[]) -%}
|
||||
{%- set starting_nested_comments = value.nested_comments | default(value=[]) -%}
|
||||
|
||||
{%- for item in starting_multi_line_comments | concat(with=starting_nested_comments) -%}
|
||||
"{{item.0}}",
|
||||
"{{item.1}}",
|
||||
{%- endfor -%}
|
||||
{%- for item in value.line_comment | default(value=[]) -%}
|
||||
"{{item}}",
|
||||
{%- endfor -%}
|
||||
],
|
||||
@@ -222,6 +240,10 @@ impl LanguageType {
|
||||
{{key}} => &[
|
||||
{% if key == "Markdown" %}
|
||||
"{{value.code_fence}}",
|
||||
{% elif key == "Html" %}
|
||||
"<script",
|
||||
"<style",
|
||||
"<template",
|
||||
{% elif key == "Rust" %}
|
||||
"///",
|
||||
"//!",
|
||||
@@ -300,6 +322,29 @@ impl LanguageType {
|
||||
}
|
||||
}
|
||||
|
||||
/// Get language from its MIME type if available.
|
||||
///
|
||||
/// ```no_run
|
||||
/// use tokei::LanguageType;
|
||||
///
|
||||
/// let lang = LanguageType::from_mime("application/javascript");
|
||||
///
|
||||
/// assert_eq!(lang, Some(LanguageType::JavaScript));
|
||||
/// ```
|
||||
pub fn from_mime(mime: &str) -> Option<Self> {
|
||||
match mime {
|
||||
{% for key, value in languages -%}
|
||||
{%- if value.mime -%}
|
||||
{%- for item in value.mime %}| "{{item}}" {% endfor %}=> Some({{key}}),
|
||||
{% endif -%}
|
||||
{%- endfor %}
|
||||
_ => {
|
||||
warn!("Unknown MIME: {}", mime);
|
||||
None
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
/// Get language from a shebang. May open and read the file.
|
||||
///
|
||||
/// ```no_run
|
||||
|
||||
+143
-34
@@ -48,6 +48,9 @@ impl FileContext {
|
||||
|
||||
#[derive(Clone, Debug)]
|
||||
pub(crate) enum LanguageContext {
|
||||
Html {
|
||||
language: LanguageType,
|
||||
},
|
||||
Markdown {
|
||||
balanced: bool,
|
||||
language: LanguageType,
|
||||
@@ -61,10 +64,11 @@ pub(crate) struct SharedMatchers {
|
||||
pub allows_nested: bool,
|
||||
pub doc_quotes: &'static [(&'static str, &'static str)],
|
||||
pub important_syntax: AhoCorasick<u16>,
|
||||
pub any_comments: AhoCorasick<u16>,
|
||||
pub any_comments: &'static [&'static str],
|
||||
pub is_fortran: bool,
|
||||
pub is_literate: bool,
|
||||
pub line_comments: &'static [&'static str],
|
||||
pub any_multi_line_comments: &'static [(&'static str, &'static str)],
|
||||
pub multi_line_comments: &'static [(&'static str, &'static str)],
|
||||
pub nested_comments: &'static [(&'static str, &'static str)],
|
||||
pub string_literals: &'static [(&'static str, &'static str)],
|
||||
@@ -100,9 +104,10 @@ impl SharedMatchers {
|
||||
is_fortran: language.is_fortran(),
|
||||
is_literate: language.is_literate(),
|
||||
important_syntax: init_corasick(language.important_syntax(), false),
|
||||
any_comments: init_corasick(language.start_any_comments(), true),
|
||||
any_comments: language.any_comments(),
|
||||
line_comments: language.line_comments(),
|
||||
multi_line_comments: language.multi_line_comments(),
|
||||
any_multi_line_comments: language.any_multi_line_comments(),
|
||||
nested_comments: language.nested_comments(),
|
||||
string_literals: language.quotes(),
|
||||
verbatim_string_literals: language.verbatim_quotes(),
|
||||
@@ -110,6 +115,14 @@ impl SharedMatchers {
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
pub(crate) enum AnalysisReport {
|
||||
/// No child languages were found, contains a boolean representing whether
|
||||
/// the line ended with comments or not.
|
||||
Normal(bool),
|
||||
ChildLanguage(FileContext),
|
||||
}
|
||||
|
||||
impl SyntaxCounter {
|
||||
pub(crate) fn new(language: LanguageType) -> Self {
|
||||
Self {
|
||||
@@ -189,7 +202,13 @@ impl SyntaxCounter {
|
||||
false
|
||||
}
|
||||
|
||||
pub(crate) fn perform_multi_line_analysis(&mut self, line: &[u8]) -> bool {
|
||||
pub(crate) fn perform_multi_line_analysis(
|
||||
&mut self,
|
||||
lines: &[u8],
|
||||
start: usize,
|
||||
end: usize,
|
||||
config: &Config,
|
||||
) -> AnalysisReport {
|
||||
let mut ended_with_comments = false;
|
||||
let mut skip = 0;
|
||||
macro_rules! skip {
|
||||
@@ -198,15 +217,19 @@ impl SyntaxCounter {
|
||||
}};
|
||||
}
|
||||
|
||||
for i in 0..line.len() {
|
||||
for i in start..end {
|
||||
if skip != 0 {
|
||||
skip -= 1;
|
||||
continue;
|
||||
}
|
||||
|
||||
ended_with_comments = false;
|
||||
let window = &line[i..];
|
||||
let window = &lines[i..];
|
||||
|
||||
if window.trim().is_empty() {
|
||||
break;
|
||||
}
|
||||
|
||||
ended_with_comments = false;
|
||||
let is_end_of_quote_or_multi_line = self
|
||||
.parse_end_of_quote(window)
|
||||
.or_else(|| self.parse_end_of_multi_line(window));
|
||||
@@ -219,6 +242,10 @@ impl SyntaxCounter {
|
||||
continue;
|
||||
}
|
||||
|
||||
if let Some(child) = self.parse_context(lines, i, end, config) {
|
||||
return AnalysisReport::ChildLanguage(child);
|
||||
}
|
||||
|
||||
let is_quote_or_multi_line = self
|
||||
.parse_quote(window)
|
||||
.or_else(|| self.parse_multi_line_comment(window));
|
||||
@@ -234,32 +261,67 @@ impl SyntaxCounter {
|
||||
}
|
||||
}
|
||||
|
||||
ended_with_comments
|
||||
AnalysisReport::Normal(ended_with_comments)
|
||||
}
|
||||
|
||||
pub(crate) fn line_is_comment(
|
||||
&self,
|
||||
line: &[u8],
|
||||
config: &crate::Config,
|
||||
ended_with_comments: bool,
|
||||
had_multi_line: bool,
|
||||
_ended_with_comments: bool,
|
||||
started_in_comments: bool,
|
||||
) -> bool {
|
||||
((!self.stack.is_empty() || ended_with_comments) && had_multi_line)
|
||||
|| (
|
||||
// If we're currently in a comment or we just ended
|
||||
// with one.
|
||||
self.shared.any_comments.is_match(line) && self.quote.is_none()
|
||||
)
|
||||
|| ((
|
||||
// If we're currently in a doc string or we just ended
|
||||
// with one.
|
||||
self.quote.is_some() ||
|
||||
self.shared.doc_quotes.iter().any(|(s, _)| line.starts_with(s.as_bytes()))
|
||||
) &&
|
||||
// `Some(true)` is import in order to respect the current
|
||||
// configuration.
|
||||
config.treat_doc_strings_as_comments == Some(true) &&
|
||||
self.quote_is_doc_quote)
|
||||
let trimmed = line.trim();
|
||||
let whole_line_is_comment = || {
|
||||
self.shared
|
||||
.line_comments
|
||||
.iter()
|
||||
.any(|c| trimmed.starts_with(c.as_bytes()))
|
||||
|| self
|
||||
.shared
|
||||
.any_multi_line_comments
|
||||
.iter()
|
||||
.any(|(start, end)| {
|
||||
trimmed.starts_with(start.as_bytes()) && trimmed.ends_with(end.as_bytes())
|
||||
})
|
||||
};
|
||||
let starts_with_comment = || {
|
||||
let quote = match self.stack.last() {
|
||||
Some(q) => q,
|
||||
_ => return false,
|
||||
};
|
||||
|
||||
self.shared
|
||||
.any_multi_line_comments
|
||||
.iter()
|
||||
.any(|(start, end)| end == quote && trimmed.starts_with(start.as_bytes()))
|
||||
};
|
||||
|
||||
// `Some(true)` in order to respect the current configuration.
|
||||
if self.quote.is_some() {
|
||||
if self.quote_is_doc_quote && config.treat_doc_strings_as_comments == Some(true) {
|
||||
self.quote.map_or(false, |q| line.starts_with(q.as_bytes()))
|
||||
|| (self.quote.is_some())
|
||||
} else {
|
||||
false
|
||||
}
|
||||
} else if self
|
||||
.shared
|
||||
.doc_quotes
|
||||
.iter()
|
||||
.any(|(_, e)| line.contains_slice(e.as_bytes()))
|
||||
&& started_in_comments
|
||||
{
|
||||
true
|
||||
} else if (whole_line_is_comment)() {
|
||||
true
|
||||
} else if started_in_comments {
|
||||
true
|
||||
} else if (starts_with_comment)() {
|
||||
true
|
||||
} else {
|
||||
false
|
||||
}
|
||||
}
|
||||
|
||||
#[inline]
|
||||
@@ -267,6 +329,7 @@ impl SyntaxCounter {
|
||||
&mut self,
|
||||
lines: &[u8],
|
||||
start: usize,
|
||||
end: usize,
|
||||
config: &Config,
|
||||
) -> Option<FileContext> {
|
||||
use std::str::FromStr;
|
||||
@@ -281,11 +344,18 @@ impl SyntaxCounter {
|
||||
static STARTING_MARKDOWN_REGEX: Lazy<Regex> =
|
||||
Lazy::new(|| Regex::new(r#"^```\S+\r?\n"#).unwrap());
|
||||
static ENDING_MARKDOWN_REGEX: Lazy<Regex> =
|
||||
Lazy::new(|| Regex::new(r#"```\r?\n"#).unwrap());
|
||||
// dbg!(String::from_utf8_lossy(&lines[start..end]));
|
||||
let opening_fence = STARTING_MARKDOWN_REGEX.find(&lines[start..])?;
|
||||
Lazy::new(|| Regex::new(r#"```(?:\r?\n)?"#).unwrap());
|
||||
|
||||
if !lines[start..end].contains_slice(b"```") {
|
||||
return None;
|
||||
}
|
||||
|
||||
let opening_fence = STARTING_MARKDOWN_REGEX.find(&lines[start..end])?;
|
||||
let start_of_code = start + opening_fence.end();
|
||||
let closing_fence = ENDING_MARKDOWN_REGEX.find(&lines[start_of_code..]);
|
||||
if let Some(m) = &closing_fence {
|
||||
trace!("{:?}", String::from_utf8_lossy(m.as_bytes()))
|
||||
}
|
||||
let end_of_code = closing_fence
|
||||
.map(|fence| start_of_code + fence.start())
|
||||
.unwrap_or_else(|| lines.len());
|
||||
@@ -298,8 +368,13 @@ impl SyntaxCounter {
|
||||
&opening_fence.as_bytes().trim()[3..],
|
||||
))
|
||||
.ok()?;
|
||||
let stats = language.parse_from_slice(&lines[start_of_code..end_of_code], config);
|
||||
// dbg!(String::from_utf8_lossy(&lines[end_of_code_block..]));
|
||||
trace!(
|
||||
"{} BLOCK: {:?}",
|
||||
language,
|
||||
String::from_utf8_lossy(&lines[start_of_code..end_of_code])
|
||||
);
|
||||
let stats =
|
||||
language.parse_from_slice(&lines[start_of_code..end_of_code].trim(), config);
|
||||
|
||||
Some(FileContext::new(
|
||||
LanguageContext::Markdown { balanced, language },
|
||||
@@ -323,6 +398,7 @@ impl SyntaxCounter {
|
||||
|
||||
while let Some((start, end)) = stepper.next(lines) {
|
||||
if lines[start..].trim().starts_with(comment_syntax) {
|
||||
trace!("{}", String::from_utf8_lossy(&lines[start..end]));
|
||||
let line = lines[start..end].trim_start();
|
||||
let stripped_line = &line[3.min(line.len())..];
|
||||
markdown.extend_from_slice(stripped_line);
|
||||
@@ -333,8 +409,8 @@ impl SyntaxCounter {
|
||||
}
|
||||
}
|
||||
|
||||
//dbg!(String::from_utf8_lossy(&lines[end_of_block..]));
|
||||
let doc_block = LanguageType::Markdown.parse_from_slice(&markdown, config);
|
||||
trace!("Markdown found: {:?}", String::from_utf8_lossy(&markdown));
|
||||
let doc_block = LanguageType::Markdown.parse_from_slice(markdown.trim(), config);
|
||||
|
||||
Some(FileContext::new(
|
||||
LanguageContext::Rust,
|
||||
@@ -342,6 +418,32 @@ impl SyntaxCounter {
|
||||
doc_block,
|
||||
))
|
||||
}
|
||||
LanguageType::Html => {
|
||||
static START_SCRIPT: Lazy<Regex> =
|
||||
Lazy::new(|| Regex::new(r#"^<script(?:.*type="(.*)")?.*?>"#).unwrap());
|
||||
static END_SCRIPT: Lazy<Regex> = Lazy::new(|| Regex::new(r#"</script>"#).unwrap());
|
||||
|
||||
let opening_tag = START_SCRIPT.find(&lines[start..end])?;
|
||||
let captures = START_SCRIPT.captures(&lines[start..end])?;
|
||||
let start_of_code = start + opening_tag.end();
|
||||
let closing_tag = END_SCRIPT.find(&lines[start_of_code..])?;
|
||||
let end_of_code = start_of_code + closing_tag.start();
|
||||
|
||||
let language = captures
|
||||
.get(1)
|
||||
.and_then(|m| {
|
||||
LanguageType::from_mime(&String::from_utf8_lossy(m.as_bytes().trim()))
|
||||
})
|
||||
.unwrap_or(LanguageType::JavaScript);
|
||||
let stats =
|
||||
language.parse_from_slice(&lines[start_of_code..end_of_code].trim(), config);
|
||||
|
||||
Some(FileContext::new(
|
||||
LanguageContext::Html { language },
|
||||
start_of_code + closing_tag.end(),
|
||||
stats,
|
||||
))
|
||||
}
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
@@ -396,11 +498,18 @@ impl SyntaxCounter {
|
||||
|
||||
#[inline]
|
||||
pub(crate) fn parse_end_of_quote(&mut self, window: &[u8]) -> Option<usize> {
|
||||
if window.starts_with(self.quote?.as_bytes()) {
|
||||
if self._is_string_mode() && window.starts_with(self.quote?.as_bytes()) {
|
||||
let quote = self.quote.take().unwrap();
|
||||
trace!("End {:?}", quote);
|
||||
Some(quote.len())
|
||||
} else if window.starts_with(br"\") && !self.quote_is_verbatim {
|
||||
} else if !self.quote_is_verbatim
|
||||
&& window.starts_with(br"\")
|
||||
&& self
|
||||
.shared
|
||||
.string_literals
|
||||
.iter()
|
||||
.any(|(start, _)| window[1..].starts_with(start.as_bytes()))
|
||||
{
|
||||
// Tell the state machine to skip the next character because it
|
||||
// has been escaped if the string isn't a verbatim string.
|
||||
Some(2)
|
||||
|
||||
+8
-3
@@ -1,12 +1,17 @@
|
||||
//! Various extensions to Rust std types.
|
||||
|
||||
pub(crate) trait AsciiExt {
|
||||
fn is_whitespace(&self) -> bool;
|
||||
fn is_whitespace(self) -> bool;
|
||||
fn is_not_line_ending_whitespace(self) -> bool;
|
||||
}
|
||||
|
||||
impl AsciiExt for u8 {
|
||||
fn is_whitespace(&self) -> bool {
|
||||
*self == b' ' || (b'\x09'..=b'\x0d').contains(self)
|
||||
fn is_whitespace(self) -> bool {
|
||||
self == b' ' || (b'\x09'..=b'\x0d').contains(&self)
|
||||
}
|
||||
|
||||
fn is_not_line_ending_whitespace(self) -> bool {
|
||||
self.is_whitespace() && self != b'\r' && self != b'\n'
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
+2
-2
@@ -79,9 +79,9 @@ mod config {
|
||||
let (_, language) = languages.into_iter().next().unwrap();
|
||||
|
||||
assert_eq!(language.lines(), 15);
|
||||
assert_eq!(language.code, 5);
|
||||
assert_eq!(language.comments, 7);
|
||||
assert_eq!(language.blanks, 3);
|
||||
assert_eq!(language.comments, 7);
|
||||
assert_eq!(language.code, 5);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
{! 10 lines 3 code 4 comments 3 blanks !}
|
||||
{! 10 lines 2 code 5 comments 3 blanks !}
|
||||
|
||||
{! All Dust comments are multiline comments. And there's no quoting
|
||||
comment openers and closers. Instead there are escape sequences
|
||||
|
||||
+30
-30
@@ -1,46 +1,46 @@
|
||||
<!-- 46 lines 23 code 19 comments 4 blanks -->
|
||||
<!-- 46 lines 22 code 19 comments 5 blanks -->
|
||||
<!DOCTYPE html>
|
||||
<html>
|
||||
<head>
|
||||
<meta charset="utf-8" />
|
||||
<meta name="viewport" content="width=device-width" />
|
||||
<title></title>
|
||||
link
|
||||
<style>
|
||||
/*
|
||||
<head>
|
||||
<meta charset="utf-8" />
|
||||
<meta name="viewport" content="width=device-width" />
|
||||
<title></title>
|
||||
link
|
||||
<style>
|
||||
/*
|
||||
|
||||
CSS multi-line comment
|
||||
CSS multi-line comment
|
||||
|
||||
*/
|
||||
body {
|
||||
background-color: pink;
|
||||
}
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
body
|
||||
</body>
|
||||
<!-- Normal Comment-->
|
||||
*/
|
||||
body {
|
||||
background-color: pink;
|
||||
}
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
body
|
||||
</body>
|
||||
<!-- Normal Comment-->
|
||||
|
||||
<nav class="navbar navbar-default navbar-fixed-top navbar-custom">
|
||||
<div id="modalSearch" class="modal fade" role="dialog"> </div>
|
||||
</nav>
|
||||
<nav class="navbar navbar-default navbar-fixed-top navbar-custom">
|
||||
<div id="modalSearch" class="modal fade" role="dialog"> </div>
|
||||
</nav>
|
||||
|
||||
<!--
|
||||
document.write("Multi-line and Code comment!");
|
||||
<!--
|
||||
document.write("Multi-line and Code comment!");
|
||||
//-->
|
||||
|
||||
<!--[if IE 8]>
|
||||
IE Special comment
|
||||
IE Special comment
|
||||
<![endif]-->
|
||||
<script>
|
||||
/*
|
||||
/*
|
||||
|
||||
Javascript multi-line comment
|
||||
Javascript multi-line comment
|
||||
|
||||
*/
|
||||
let x = 5;
|
||||
// Javascript single line comment
|
||||
*/
|
||||
let x = 5;
|
||||
// Javascript single line comment
|
||||
</script>
|
||||
|
||||
</html>
|
||||
|
||||
+1
-1
@@ -1,4 +1,4 @@
|
||||
# 24 lines 15 code 7 comments 2 blanks
|
||||
# 24 lines 14 code 8 comments 2 blanks
|
||||
=begin
|
||||
Regex methods and functions
|
||||
=end
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
# 11 lines 5 code 4 comments 2 blanks
|
||||
# 11 lines 4 code 5 comments 2 blanks
|
||||
|
||||
=begin pod
|
||||
Defines a fun infix operator.
|
||||
|
||||
+1
-1
@@ -1,4 +1,4 @@
|
||||
//! 48 lines 33 code 9 comments 6 blanks
|
||||
//! 48 lines 36 code 6 comments 6 blanks
|
||||
//! ```rust
|
||||
//! fn main () {
|
||||
//! // Comment
|
||||
|
||||
@@ -0,0 +1,24 @@
|
||||
// 24 lines 6 code 14 comments 4 blanks
|
||||
|
||||
// Single-line comment
|
||||
/* multi-line comment */
|
||||
/* /* nested */ */
|
||||
/*
|
||||
multi
|
||||
line
|
||||
comment
|
||||
*/
|
||||
/*
|
||||
nested /* */
|
||||
/* nested */
|
||||
*/
|
||||
|
||||
import UIKit
|
||||
|
||||
class ViewController: UIViewController {
|
||||
override func viewDidLoad() {
|
||||
super.viewDidLoad()
|
||||
|
||||
// Do any additional setup after loading the view.
|
||||
}
|
||||
}
|
||||
@@ -1,4 +1,4 @@
|
||||
## 11 lines 5 code 4 comments 2 blanks
|
||||
## 11 lines 4 code 5 comments 2 blanks
|
||||
|
||||
This text is visible. #* This text, as part of a multi-line
|
||||
comment, is not visible. This text is not visible; it is also
|
||||
|
||||
Reference in New Issue
Block a user