openharmony/third_party_rust_nom
1
0
Fork 0
mirror of https://gitee.com/openharmony/third_party_rust_nom synced 2024-11-23 07:29:54 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

many0 should not return until its child parser fails

Browse Source 
if we reached the end of input, test with at_eof, to see if we should
return the whole input slice or not.
This will potentially break some existing parsers that rely on many0
trying to consume everything, but this behaviour is more correct and
aligned with the rest of nom
This commit is contained in:
Geoffroy Couprie 2018-01-11 19:29:46 +01:00
parent 1b4ec7e756
commit 47deaa1d03
5 changed files with 121 additions and 112 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
src/multi.rs
View File

@ -238,18 +238,13 @@ macro_rules! many0(
($i:expr, $submac:ident!( $($args:tt)* )) => (
{
use ::std::result::Result::*;
use $crate::{Err,InputLength};
use $crate::{Err,InputLength,Needed,AtEof};
let ret;
let mut res = ::std::vec::Vec::new();
let mut input = $i.clone();
loop {
if input.input_len() == 0 {
ret = Ok((input, res));
break;
}
let input_ = input.clone();
match $submac!(input_, $($args)*) {
Err(Err::Error(_)) => {
@ -271,6 +266,15 @@ macro_rules! many0(
input = i;
}
}
if input.input_len() == 0 {
if input.at_eof() {
ret = Ok((input, res));
} else {
ret = Err(Err::Incomplete(Needed::Unknown));
}
break;
}
}
ret
@ -1219,8 +1223,8 @@ mod tests {
assert_eq!(multi(&b"abcdabcdefgh"[..]),Ok((&b"efgh"[..], vec![&b"abcd"[..], &b"abcd"[..]])));
assert_eq!(multi(&b"azerty"[..]),Ok((&b"azerty"[..], Vec::new())));
assert_eq!(multi(&b"abcdab"[..]), Err(Err::Incomplete(Needed::Size(4))));
assert_eq!(multi(&b"abcd"[..]),Ok((&b""[..], vec![&b"abcd"[..]])));
assert_eq!(multi(&b""[..]),Ok((&b""[..], Vec::new())));
assert_eq!(multi(&b"abcd"[..]), Err(Err::Incomplete(Needed::Unknown)));
assert_eq!(multi(&b""[..]), Err(Err::Incomplete(Needed::Size(4))));
assert_eq!(multi_empty(&b"abcdef"[..]), Err(Err::Error(error_position!(&b"abcdef"[..], ErrorKind::Many0))));
}

45
tests/arithmetic_ast.rs
View File

@ -8,6 +8,7 @@ use std::str;
use std::str::FromStr;
use nom::{digit, multispace};
use nom::types::CompleteStr;
pub enum Expr {
Value(i64),
@ -53,23 +54,21 @@ impl Debug for Expr {
}
}
named!(parens< Expr >, delimited!(
named!(parens< CompleteStr, Expr >, delimited!(
delimited!(opt!(multispace), tag!("("), opt!(multispace)),
map!(map!(expr, Box::new), Expr::Paren),
delimited!(opt!(multispace), tag!(")"), opt!(multispace))
)
);
named!(factor< Expr >, alt_complete!(
named!(factor< CompleteStr, Expr >, alt_complete!(
map!(
map_res!(
map_res!(
delimited!(opt!(multispace), digit, opt!(multispace)),
str::from_utf8
),
FromStr::from_str
),
Expr::Value)
delimited!(opt!(multispace), digit, opt!(multispace)),
|s: CompleteStr| { FromStr::from_str(s.0) }
),
Expr::Value
)
| parens
)
);
@ -86,7 +85,7 @@ fn fold_exprs(initial: Expr, remainder: Vec<(Oper, Expr)>) -> Expr {
})
}
named!(term< Expr >, do_parse!(
named!(term< CompleteStr, Expr >, do_parse!(
initial: factor >>
remainder: many0!(
alt!(
@ -97,7 +96,7 @@ named!(term< Expr >, do_parse!(
(fold_exprs(initial, remainder))
));
named!(expr< Expr >, do_parse!(
named!(expr< CompleteStr, Expr >, do_parse!(
initial: term >>
remainder: many0!(
alt!(
@ -111,30 +110,30 @@ named!(expr< Expr >, do_parse!(
#[test]
fn factor_test() {
assert_eq!(factor(&b" 3 "[..]).map(|(i,x)| (i,format!("{:?}", x))),
Ok((&b""[..], String::from("3"))));
assert_eq!(factor(CompleteStr(" 3 ")).map(|(i,x)| (i,format!("{:?}", x))),
Ok((CompleteStr(""), String::from("3"))));
}
#[test]
fn term_test() {
assert_eq!(term(&b" 3 * 5 "[..]).map(|(i,x)| (i,format!("{:?}", x))),
Ok( (&b""[..], String::from("(3 * 5)")) ));
assert_eq!(term(CompleteStr(" 3 * 5 ")).map(|(i,x)| (i,format!("{:?}", x))),
Ok( (CompleteStr(""), String::from("(3 * 5)")) ));
}
#[test]
fn expr_test() {
assert_eq!(expr(&b" 1 + 2 * 3 "[..]).map(|(i,x)| (i,format!("{:?}", x))),
Ok( (&b""[..], String::from("(1 + (2 * 3))")) ));
assert_eq!(expr(&b" 1 + 2 * 3 / 4 - 5 "[..]).map(|(i,x)| (i,format!("{:?}", x))),
Ok( (&b""[..], String::from("((1 + ((2 * 3) / 4)) - 5)")) ));
assert_eq!(expr(&b" 72 / 2 / 3 "[..]).map(|(i,x)| (i,format!("{:?}", x))),
Ok( (&b""[..], String::from("((72 / 2) / 3)")) ));
assert_eq!(expr(CompleteStr(" 1 + 2 * 3 ")).map(|(i,x)| (i,format!("{:?}", x))),
Ok( (CompleteStr(""), String::from("(1 + (2 * 3))")) ));
assert_eq!(expr(CompleteStr(" 1 + 2 * 3 / 4 - 5 ")).map(|(i,x)| (i,format!("{:?}", x))),
Ok( (CompleteStr(""), String::from("((1 + ((2 * 3) / 4)) - 5)")) ));
assert_eq!(expr(CompleteStr(" 72 / 2 / 3 ")).map(|(i,x)| (i,format!("{:?}", x))),
Ok( (CompleteStr(""), String::from("((72 / 2) / 3)")) ));
}
#[test]
fn parens_test() {
assert_eq!(
expr(&b" ( 1 + 2 ) * 3 "[..]).map(|(i,x)| (i,format!("{:?}", x))),
Ok( (&b""[..], String::from("([(1 + 2)] * 3)")) )
expr(CompleteStr(" ( 1 + 2 ) * 3 ")).map(|(i,x)| (i,format!("{:?}", x))),
Ok( (CompleteStr(""), String::from("([(1 + 2)] * 3)")) )
);
}

79
tests/ini.rs
View File

@ -2,28 +2,33 @@
extern crate nom;
use nom::{space, alphanumeric, multispace};
use nom::types::CompleteByteSlice;
use std::str;
use std::collections::HashMap;
named!(category<&str>, map_res!(
named!(category<CompleteByteSlice, &str>, map_res!(
delimited!(
char!('['),
take_while!(call!(|c| c != b']')),
char!(']')
),
str::from_utf8
complete_byte_slice_to_str
));
named!(key_value <&[u8],(&str,&str)>,
fn complete_byte_slice_to_str<'a>(s: CompleteByteSlice<'a>) -> Result<&'a str, str::Utf8Error> {
str::from_utf8(s.0)
}
named!(key_value <CompleteByteSlice,(&str,&str)>,
do_parse!(
key: map_res!(alphanumeric, str::from_utf8)
key: map_res!(alphanumeric, complete_byte_slice_to_str)
>> opt!(space)
>> char!('=')
>> opt!(space)
>> val: map_res!(
take_while!(call!(|c| c != b'\n' && c != b';')),
str::from_utf8
complete_byte_slice_to_str
)
>> opt!(pair!(char!(';'), take_while!(call!(|c| c != b'\n'))))
>> (key, val)
@ -31,7 +36,7 @@ named!(key_value <&[u8],(&str,&str)>,
);
named!(keys_and_values<&[u8], HashMap<&str, &str> >,
named!(keys_and_values<CompleteByteSlice, HashMap<&str, &str> >,
map!(
many0!(terminated!(key_value, opt!(multispace))),
|vec: Vec<_>| vec.into_iter().collect()
@ -39,7 +44,7 @@ named!(keys_and_values<&[u8], HashMap<&str, &str> >,
);
named!(category_and_keys<&[u8],(&str,HashMap<&str,&str>)>,
named!(category_and_keys<CompleteByteSlice,(&str,HashMap<&str,&str>)>,
do_parse!(
category: category >>
opt!(multispace) >>
@ -48,7 +53,7 @@ named!(category_and_keys<&[u8],(&str,HashMap<&str,&str>)>,
)
);
named!(categories<&[u8], HashMap<&str, HashMap<&str,&str> > >,
named!(categories<CompleteByteSlice, HashMap<&str, HashMap<&str,&str> > >,
map!(
many0!(
separated_pair!(
@ -66,18 +71,18 @@ named!(categories<&[u8], HashMap<&str, HashMap<&str,&str> > >,
#[test]
fn parse_category_test() {
let ini_file = &b"[category]
let ini_file = CompleteByteSlice(b"[category]
parameter=value
key = value2"[..];
key = value2");
let ini_without_category = &b"\n\nparameter=value
key = value2"[..];
let ini_without_category = CompleteByteSlice(b"\n\nparameter=value
key = value2");
let res = category(ini_file);
println!("{:?}", res);
match res {
Ok((i, o)) => println!("i: {:?} | o: {:?}", str::from_utf8(i), o),
Ok((i, o)) => println!("i: {:?} | o: {:?}", str::from_utf8(i.0), o),
_ => println!("error"),
}
@ -86,15 +91,15 @@ key = value2"[..];
#[test]
fn parse_key_value_test() {
let ini_file = &b"parameter=value
key = value2"[..];
let ini_file = CompleteByteSlice(b"parameter=value
key = value2");
let ini_without_key_value = &b"\nkey = value2"[..];
let ini_without_key_value = CompleteByteSlice(b"\nkey = value2");
let res = key_value(ini_file);
println!("{:?}", res);
match res {
Ok((i, (o1, o2))) => println!("i: {:?} | o: ({:?},{:?})", str::from_utf8(i), o1, o2),
Ok((i, (o1, o2))) => println!("i: {:?} | o: ({:?},{:?})", str::from_utf8(i.0), o1, o2),
_ => println!("error"),
}
@ -104,15 +109,15 @@ key = value2"[..];
#[test]
fn parse_key_value_with_space_test() {
let ini_file = &b"parameter = value
key = value2"[..];
let ini_file = CompleteByteSlice(b"parameter = value
key = value2");
let ini_without_key_value = &b"\nkey = value2"[..];
let ini_without_key_value = CompleteByteSlice(b"\nkey = value2");
let res = key_value(ini_file);
println!("{:?}", res);
match res {
Ok((i, (o1, o2))) => println!("i: {:?} | o: ({:?},{:?})", str::from_utf8(i), o1, o2),
Ok((i, (o1, o2))) => println!("i: {:?} | o: ({:?},{:?})", str::from_utf8(i.0), o1, o2),
_ => println!("error"),
}
@ -121,15 +126,15 @@ key = value2"[..];
#[test]
fn parse_key_value_with_comment_test() {
let ini_file = &b"parameter=value;abc
key = value2"[..];
let ini_file = CompleteByteSlice(b"parameter=value;abc
key = value2");
let ini_without_key_value = &b"\nkey = value2"[..];
let ini_without_key_value = CompleteByteSlice(b"\nkey = value2");
let res = key_value(ini_file);
println!("{:?}", res);
match res {
Ok((i, (o1, o2))) => println!("i: {:?} | o: ({:?},{:?})", str::from_utf8(i), o1, o2),
Ok((i, (o1, o2))) => println!("i: {:?} | o: ({:?},{:?})", str::from_utf8(i.0), o1, o2),
_ => println!("error"),
}
@ -138,18 +143,18 @@ key = value2"[..];
#[test]
fn parse_multiple_keys_and_values_test() {
let ini_file = &b"parameter=value;abc
let ini_file = CompleteByteSlice(b"parameter=value;abc
key = value2
[category]"[..];
[category]");
let ini_without_key_value = &b"[category]"[..];
let ini_without_key_value = CompleteByteSlice(b"[category]");
let res = keys_and_values(ini_file);
println!("{:?}", res);
match res {
Ok((i, ref o)) => println!("i: {:?} | o: {:?}", str::from_utf8(i), o),
Ok((i, ref o)) => println!("i: {:?} | o: {:?}", str::from_utf8(i.0), o),
_ => println!("error"),
}
@ -162,19 +167,19 @@ key = value2
#[test]
fn parse_category_then_multiple_keys_and_values_test() {
//FIXME: there can be an empty line or a comment line after a category
let ini_file = &b"[abcd]
let ini_file = CompleteByteSlice(b"[abcd]
parameter=value;abc
key = value2
[category]"[..];
[category]");
let ini_after_parser = &b"[category]"[..];
let ini_after_parser = CompleteByteSlice(b"[category]");
let res = category_and_keys(ini_file);
println!("{:?}", res);
match res {
Ok((i, ref o)) => println!("i: {:?} | o: {:?}", str::from_utf8(i), o),
Ok((i, ref o)) => println!("i: {:?} | o: {:?}", str::from_utf8(i.0), o),
_ => println!("error"),
}
@ -186,7 +191,7 @@ key = value2
#[test]
fn parse_multiple_categories_test() {
let ini_file = &b"[abcd]
let ini_file = CompleteByteSlice(b"[abcd]
parameter=value;abc
@ -195,14 +200,14 @@ key = value2
[category]
parameter3=value3
key4 = value4
"[..];
");
let ini_after_parser = &b""[..];
let ini_after_parser = CompleteByteSlice(b"");
let res = categories(ini_file);
//println!("{:?}", res);
match res {
Ok((i, ref o)) => println!("i: {:?} | o: {:?}", str::from_utf8(i), o),
Ok((i, ref o)) => println!("i: {:?} | o: {:?}", str::from_utf8(i.0), o),
_ => println!("error"),
}

85
tests/ini_str.rs
View File

@ -2,6 +2,7 @@
extern crate nom;
use nom::IResult;
use nom::types::CompleteStr;
use std::collections::HashMap;
@ -25,26 +26,26 @@ fn is_line_ending_or_comment(chr: char) -> bool {
chr == ';' || chr == '\n'
}
named!(alphanumeric<&str,&str>, take_while_s!(is_alphanumeric));
named!(not_line_ending<&str,&str>, is_not_s!("\r\n"));
named!(space<&str,&str>, take_while_s!(is_space));
named!(space_or_line_ending<&str,&str>, is_a_s!(" \r\n"));
named!(alphanumeric<CompleteStr,CompleteStr>, take_while_s!(is_alphanumeric));
named!(not_line_ending<CompleteStr,CompleteStr>, is_not_s!("\r\n"));
named!(space<CompleteStr,CompleteStr>, take_while_s!(is_space));
named!(space_or_line_ending<CompleteStr,CompleteStr>, is_a_s!(" \r\n"));
fn right_bracket(c: char) -> bool {
c == ']'
}
named!(category <&str, &str>,
named!(category <CompleteStr, &str>,
do_parse!(
tag_s!("[") >>
name: take_till_s!(right_bracket) >>
tag_s!("]") >>
opt!(space_or_line_ending) >>
(name)
(name.0)
)
);
named!(key_value <&str,(&str,&str)>,
named!(key_value <CompleteStr,(&str,&str)>,
do_parse!(
key: alphanumeric >>
opt!(space) >>
@ -54,13 +55,13 @@ named!(key_value <&str,(&str,&str)>,
opt!(space) >>
opt!(pair!(tag_s!(";"), not_line_ending)) >>
opt!(space_or_line_ending) >>
(key, val)
(key.0, val.0)
)
);
named!(keys_and_values_aggregator<&str, Vec<(&str,&str)> >, many0!(key_value));
named!(keys_and_values_aggregator<CompleteStr, Vec<(&str, &str)> >, many0!(key_value));
fn keys_and_values(input: &str) -> IResult<&str, HashMap<&str, &str>> {
fn keys_and_values(input: CompleteStr) -> IResult<CompleteStr, HashMap<&str, &str>> {
match keys_and_values_aggregator(input) {
Ok((i, tuple_vec)) => Ok((i, tuple_vec.into_iter().collect())),
Err(e) => Err(e),
@ -68,13 +69,13 @@ fn keys_and_values(input: &str) -> IResult<&str, HashMap<&str, &str>> {
}
named!(category_and_keys<&str,(&str,HashMap<&str,&str>)>,
named!(category_and_keys<CompleteStr,(&str,HashMap<&str,&str>)>,
pair!(category, keys_and_values)
);
named!(categories_aggregator<&str, Vec<(&str, HashMap<&str,&str>)> >, many0!(category_and_keys));
named!(categories_aggregator<CompleteStr, Vec<(&str, HashMap<&str,&str>)> >, many0!(category_and_keys));
fn categories(input: &str) -> IResult<&str, HashMap<&str, HashMap<&str, &str>>> {
fn categories(input: CompleteStr) -> IResult<CompleteStr, HashMap<&str, HashMap<&str, &str>>> {
match categories_aggregator(input) {
Ok((i, tuple_vec)) => Ok((i, tuple_vec.into_iter().collect())),
Err(e) => Err(e),
@ -84,18 +85,18 @@ fn categories(input: &str) -> IResult<&str, HashMap<&str, HashMap<&str, &str>>>
#[test]
fn parse_category_test() {
let ini_file = "[category]
let ini_file = CompleteStr("[category]
parameter=value
key = value2";
key = value2");
let ini_without_category = "parameter=value
key = value2";
let ini_without_category = CompleteStr("parameter=value
key = value2");
let res = category(ini_file);
println!("{:?}", res);
match res {
Ok((i, o)) => println!("i: {} | o: {:?}", i, o),
Ok((i, o)) => println!("i: {} | o: {:?}", i.0, o),
_ => println!("error"),
}
@ -104,15 +105,15 @@ key = value2";
#[test]
fn parse_key_value_test() {
let ini_file = "parameter=value
key = value2";
let ini_file = CompleteStr("parameter=value
key = value2");
let ini_without_key_value = "key = value2";
let ini_without_key_value = CompleteStr("key = value2");
let res = key_value(ini_file);
println!("{:?}", res);
match res {
Ok((i, (o1, o2))) => println!("i: {} | o: ({:?},{:?})", i, o1, o2),
Ok((i, (o1, o2))) => println!("i: {} | o: ({:?},{:?})", i.0, o1, o2),
_ => println!("error"),
}
@ -121,15 +122,15 @@ key = value2";
#[test]
fn parse_key_value_with_space_test() {
let ini_file = "parameter = value
key = value2";
let ini_file = CompleteStr("parameter = value
key = value2");
let ini_without_key_value = "key = value2";
let ini_without_key_value = CompleteStr("key = value2");
let res = key_value(ini_file);
println!("{:?}", res);
match res {
Ok((i, (o1, o2))) => println!("i: {} | o: ({:?},{:?})", i, o1, o2),
Ok((i, (o1, o2))) => println!("i: {} | o: ({:?},{:?})", i.0, o1, o2),
_ => println!("error"),
}
@ -138,15 +139,15 @@ key = value2";
#[test]
fn parse_key_value_with_comment_test() {
let ini_file = "parameter=value;abc
key = value2";
let ini_file = CompleteStr("parameter=value;abc
key = value2");
let ini_without_key_value = "key = value2";
let ini_without_key_value = CompleteStr("key = value2");
let res = key_value(ini_file);
println!("{:?}", res);
match res {
Ok((i, (o1, o2))) => println!("i: {} | o: ({:?},{:?})", i, o1, o2),
Ok((i, (o1, o2))) => println!("i: {} | o: ({:?},{:?})", i.0, o1, o2),
_ => println!("error"),
}
@ -155,18 +156,18 @@ key = value2";
#[test]
fn parse_multiple_keys_and_values_test() {
let ini_file = "parameter=value;abc
let ini_file = CompleteStr("parameter=value;abc
key = value2
[category]";
[category]");
let ini_without_key_value = "[category]";
let ini_without_key_value = CompleteStr("[category]");
let res = keys_and_values(ini_file);
println!("{:?}", res);
match res {
Ok((i, ref o)) => println!("i: {} | o: {:?}", i, o),
Ok((i, ref o)) => println!("i: {} | o: {:?}", i.0, o),
_ => println!("error"),
}
@ -179,19 +180,19 @@ key = value2
#[test]
fn parse_category_then_multiple_keys_and_values_test() {
//FIXME: there can be an empty line or a comment line after a category
let ini_file = "[abcd]
let ini_file = CompleteStr("[abcd]
parameter=value;abc
key = value2
[category]";
[category]");
let ini_after_parser = "[category]";
let ini_after_parser = CompleteStr("[category]");
let res = category_and_keys(ini_file);
println!("{:?}", res);
match res {
Ok((i, ref o)) => println!("i: {} | o: {:?}", i, o),
Ok((i, ref o)) => println!("i: {} | o: {:?}", i.0, o),
_ => println!("error"),
}
@ -203,7 +204,7 @@ key = value2
#[test]
fn parse_multiple_categories_test() {
let ini_file = "[abcd]
let ini_file = CompleteStr("[abcd]
parameter=value;abc
@ -212,12 +213,12 @@ key = value2
[category]
parameter3=value3
key4 = value4
";
");
let res = categories(ini_file);
//println!("{:?}", res);
match res {
Ok((i, ref o)) => println!("i: {} | o: {:?}", i, o),
Ok((i, ref o)) => println!("i: {} | o: {:?}", i.0, o),
_ => println!("error"),
}
@ -230,5 +231,5 @@ key4 = value4
let mut expected_h: HashMap<&str, HashMap<&str, &str>> = HashMap::new();
expected_h.insert("abcd", expected_1);
expected_h.insert("category", expected_2);
assert_eq!(res, Ok(("", expected_h)));
assert_eq!(res, Ok((CompleteStr(""), expected_h)));
}

4
tests/issues.rs
View File

@ -95,8 +95,8 @@ mod parse_int {
#[test]
fn issue_142() {
let subject = parse_ints(&b"12 34 5689"[..]);
let expected = Ok((&b""[..], vec![12, 34, 5689]));
let subject = parse_ints(&b"12 34 5689a"[..]);
let expected = Ok((&b"a"[..], vec![12, 34, 5689]));
assert_eq!(subject, expected);
let subject = parse_ints(&b"12 34 5689 "[..]);