Add simple fuzzing

Add infrastructure to automatically run fuzzers in CI,
and implement a simple fuzzing test based on triggering all (most)
public APIs in a randimized way.

As far as I was able to try it catches the previous unsoundness issues
in a matter of seconds. This can be tried by changing the `path = "../"` dependency to
`version = "=0.6.3"` etc. and running the fuzzer manually. (Note: You'll need
to tweak the `Cargo.lock` to allow downloading the yanked versions).

Related to #124
This commit is contained in:
Dawid Ciężarkiewicz
2019-10-13 14:29:37 -07:00
parent 782dd470bd
commit 47028a7e02
7 changed files with 521 additions and 7 deletions
+1 -1
View File
@@ -1,2 +1,2 @@
target
Cargo.lock
./Cargo.lock
+28 -6
View File
@@ -1,9 +1,24 @@
language: rust
rust:
- 1.36.0
- nightly
- beta
- stable
addons:
apt:
update: true
packages:
- binutils-dev
- libunwind8-dev
- libcurl4-openssl-dev
- libelf-dev
- libdw-dev
- cmake
- gcc
- libiberty-dev
matrix:
include:
- rust: 1.36.0
- rust: nightly
- rust: beta
env: DO_FUZZ=true
- rust: stable
env: DO_FUZZ=true
script: |
cargo build --verbose &&
cargo test --verbose &&
@@ -12,4 +27,11 @@ script: |
([ $TRAVIS_RUST_VERSION != nightly ] || cargo test --verbose --features union) &&
([ $TRAVIS_RUST_VERSION != nightly ] || cargo test --verbose --all-features) &&
([ $TRAVIS_RUST_VERSION != nightly ] || cargo bench --verbose bench) &&
([ $TRAVIS_RUST_VERSION != nightly ] || bash ./scripts/run_miri.sh)
([ $TRAVIS_RUST_VERSION != nightly ] || bash ./scripts/run_miri.sh) &&
if [ "$DO_FUZZ" = true ]
then
(
cd fuzz
./travis-fuzz.sh
)
fi
+192
View File
@@ -0,0 +1,192 @@
# This file is automatically @generated by Cargo.
# It is not intended for manual editing.
[[package]]
name = "afl"
version = "0.4.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"cc 1.0.45 (registry+https://github.com/rust-lang/crates.io-index)",
"clap 2.33.0 (registry+https://github.com/rust-lang/crates.io-index)",
"rustc_version 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)",
"xdg 2.2.0 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "ansi_term"
version = "0.11.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"winapi 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "arbitrary"
version = "0.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
[[package]]
name = "atty"
version = "0.2.13"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"libc 0.2.62 (registry+https://github.com/rust-lang/crates.io-index)",
"winapi 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "bitflags"
version = "1.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
[[package]]
name = "cc"
version = "1.0.45"
source = "registry+https://github.com/rust-lang/crates.io-index"
[[package]]
name = "clap"
version = "2.33.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"ansi_term 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)",
"atty 0.2.13 (registry+https://github.com/rust-lang/crates.io-index)",
"bitflags 1.2.0 (registry+https://github.com/rust-lang/crates.io-index)",
"strsim 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)",
"textwrap 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)",
"unicode-width 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)",
"vec_map 0.8.1 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "honggfuzz"
version = "0.5.45"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"arbitrary 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)",
"lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)",
"memmap 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "lazy_static"
version = "1.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
[[package]]
name = "libc"
version = "0.2.62"
source = "registry+https://github.com/rust-lang/crates.io-index"
[[package]]
name = "memmap"
version = "0.7.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"libc 0.2.62 (registry+https://github.com/rust-lang/crates.io-index)",
"winapi 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "rustc_version"
version = "0.2.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"semver 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "semver"
version = "0.9.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"semver-parser 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "semver-parser"
version = "0.7.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
[[package]]
name = "smallvec"
version = "0.6.10"
[[package]]
name = "smallvec-fuzz"
version = "0.1.0"
dependencies = [
"afl 0.4.4 (registry+https://github.com/rust-lang/crates.io-index)",
"honggfuzz 0.5.45 (registry+https://github.com/rust-lang/crates.io-index)",
"smallvec 0.6.10",
]
[[package]]
name = "strsim"
version = "0.8.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
[[package]]
name = "textwrap"
version = "0.11.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"unicode-width 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "unicode-width"
version = "0.1.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
[[package]]
name = "vec_map"
version = "0.8.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
[[package]]
name = "winapi"
version = "0.3.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"winapi-i686-pc-windows-gnu 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)",
"winapi-x86_64-pc-windows-gnu 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "winapi-i686-pc-windows-gnu"
version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
[[package]]
name = "winapi-x86_64-pc-windows-gnu"
version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
[[package]]
name = "xdg"
version = "2.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
[metadata]
"checksum afl 0.4.4 (registry+https://github.com/rust-lang/crates.io-index)" = "2d2b15f7ed343b7446090d0de7789a640c22b163fb200d81cef958fcb9856692"
"checksum ansi_term 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ee49baf6cb617b853aa8d93bf420db2383fab46d314482ca2803b40d5fde979b"
"checksum arbitrary 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "64cf76cb6e2222ed0ea86b2b0ee2f71c96ec6edd5af42e84d59160e91b836ec4"
"checksum atty 0.2.13 (registry+https://github.com/rust-lang/crates.io-index)" = "1803c647a3ec87095e7ae7acfca019e98de5ec9a7d01343f611cf3152ed71a90"
"checksum bitflags 1.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "8a606a02debe2813760609f57a64a2ffd27d9fdf5b2f133eaca0b248dd92cdd2"
"checksum cc 1.0.45 (registry+https://github.com/rust-lang/crates.io-index)" = "4fc9a35e1f4290eb9e5fc54ba6cf40671ed2a2514c3eeb2b2a908dda2ea5a1be"
"checksum clap 2.33.0 (registry+https://github.com/rust-lang/crates.io-index)" = "5067f5bb2d80ef5d68b4c87db81601f0b75bca627bc2ef76b141d7b846a3c6d9"
"checksum honggfuzz 0.5.45 (registry+https://github.com/rust-lang/crates.io-index)" = "24c27b4aa3049d6d10d8e33d52c9d03ca9aec18f8a449b246f8c4a5b0c10fb34"
"checksum lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646"
"checksum libc 0.2.62 (registry+https://github.com/rust-lang/crates.io-index)" = "34fcd2c08d2f832f376f4173a231990fa5aef4e99fb569867318a227ef4c06ba"
"checksum memmap 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)" = "6585fd95e7bb50d6cc31e20d4cf9afb4e2ba16c5846fc76793f11218da9c475b"
"checksum rustc_version 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)" = "138e3e0acb6c9fb258b19b67cb8abd63c00679d2851805ea151465464fe9030a"
"checksum semver 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)" = "1d7eb9ef2c18661902cc47e535f9bc51b78acd254da71d375c2f6720d9a40403"
"checksum semver-parser 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)" = "388a1df253eca08550bef6c72392cfe7c30914bf41df5269b68cbd6ff8f570a3"
"checksum strsim 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)" = "8ea5119cdb4c55b55d432abb513a0429384878c15dde60cc77b1c99de1a95a6a"
"checksum textwrap 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)" = "d326610f408c7a4eb6f51c37c330e496b08506c9457c9d34287ecc38809fb060"
"checksum unicode-width 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)" = "7007dbd421b92cc6e28410fe7362e2e0a2503394908f417b68ec8d1c364c4e20"
"checksum vec_map 0.8.1 (registry+https://github.com/rust-lang/crates.io-index)" = "05c78687fb1a80548ae3250346c3db86a80a7cdd77bda190189f2d0a0987c81a"
"checksum winapi 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)" = "8093091eeb260906a183e6ae1abdba2ef5ef2257a21801128899c3fc699229c6"
"checksum winapi-i686-pc-windows-gnu 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6"
"checksum winapi-x86_64-pc-windows-gnu 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f"
"checksum xdg 2.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "d089681aa106a86fade1b0128fb5daf07d5867a509ab036d99988dec80429a57"
+26
View File
@@ -0,0 +1,26 @@
[package]
name = "smallvec-fuzz"
version = "0.1.0"
authors = ["Dawid Ciężarkiewicz <dpc@dpc.pw>"]
edition = "2018"
publish = false
[package.metadata]
cargo-fuzz = true
[features]
afl_fuzz = ["afl"]
honggfuzz_fuzz = ["honggfuzz"]
[dependencies]
honggfuzz = { version = "0.5.45", optional = true }
afl = { version = "0.4", optional = true }
smallvec = { path = ".." }
[workspace]
members = ["."]
[[bin]]
name = "smallvec_ops"
path = "fuzz_targets/smallvec_ops.rs"
+12
View File
@@ -0,0 +1,12 @@
# Fuzzer for smallvec
Based on fuzzing in [rust-bitcoin](https://github.com/rust-bitcoin/rust-bitcoin/tree/c8ac25219a09bf9d017f1b05abe3e746e2136f73/fuzz)
## Running manually with afl
```
cargo afl build --release --bin smallvec_ops --features afl && cargo afl fuzz -i in -o out target/release/smallvec_ops
```
# Useful links:
* https://rust-fuzz.github.io/book/afl.html
+243
View File
@@ -0,0 +1,243 @@
//! Simple fuzzer testing all available `SmallVec` operations
use smallvec::SmallVec;
// There's no point growing too much, so try not to grow
// over this size.
const CAP_GROWTH: usize = 256;
macro_rules! next_usize {
($b:ident) => {
$b.next().unwrap_or(0) as usize
};
}
macro_rules! next_u8 {
($b:ident) => {
$b.next().unwrap_or(0)
};
}
fn black_box_iter(i: impl Iterator<Item = u8>) {
// print to work as a black_box
print!("{}", i.fold(0u8, |acc, e| acc.wrapping_add(e)));
}
fn black_box_slice(s: &[u8]) {
black_box_iter(s.iter().copied())
}
fn black_box_mut_slice(s: &mut [u8]) {
s.iter_mut().map(|e| *e = e.wrapping_add(1)).count();
black_box_iter((s as &[u8]).iter().copied())
}
fn do_test<A: smallvec::Array<Item = u8>>(data: &[u8]) -> SmallVec<A> {
let mut v = SmallVec::<A>::new();
let mut bytes = data.iter().copied();
while let Some(op) = bytes.next() {
match op % 27 {
0 => {
v = SmallVec::new();
}
1 => {
v = SmallVec::with_capacity(next_usize!(bytes));
}
2 => {
v = SmallVec::from_vec(v.to_vec());
}
3 => {
black_box_iter(v.drain(..));
}
4 => {
if v.len() < CAP_GROWTH {
v.push(next_u8!(bytes))
}
}
5 => {
v.pop();
}
6 => v.grow(next_usize!(bytes) + v.len()),
7 => {
if v.len() < CAP_GROWTH {
v.reserve(next_usize!(bytes))
}
}
8 => {
if v.len() < CAP_GROWTH {
v.reserve_exact(next_usize!(bytes))
}
}
9 => v.shrink_to_fit(),
10 => v.truncate(next_usize!(bytes)),
11 => black_box_slice(v.as_slice()),
12 => black_box_mut_slice(v.as_mut_slice()),
13 => {
if !v.is_empty() {
v.swap_remove(next_usize!(bytes) % v.len());
}
}
14 => {
v.clear();
}
15 => {
if !v.is_empty() {
v.remove(next_usize!(bytes) % v.len());
}
}
16 => {
let insert_pos = next_usize!(bytes) % (v.len() + 1);
v.insert(insert_pos, next_u8!(bytes));
}
17 => {
let insert_pos = next_usize!(bytes) % (v.len() + 1);
let how_many = next_usize!(bytes);
let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
v.insert_many(insert_pos, (0..how_many).map(|_| bytes.next().unwrap()));
}));
if result.is_err() {
assert!(bytes.next().is_none());
}
}
18 => {
v = SmallVec::from_vec(v.into_vec());
}
19 => {
let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
v.retain(|e| {
let alt_e = bytes.next().unwrap();
let retain = *e >= alt_e;
*e = e.wrapping_add(alt_e);
retain
});
}));
if result.is_err() {
assert!(bytes.next().is_none());
}
}
20 => {
v.dedup();
}
21 => {
let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
v.dedup_by(|a, b| {
let substitute = bytes.next().unwrap();
let dedup = a == b;
*a = a.wrapping_add(substitute);
*b = b.wrapping_add(substitute);
dedup
});
}));
if result.is_err() {
assert!(bytes.next().is_none());
}
}
22 => {
v = SmallVec::from_slice(data);
}
23 => {
if v.len() < CAP_GROWTH {
v.extend_from_slice(data)
}
}
24 => {
if v.len() < CAP_GROWTH {
let insert_pos = next_usize!(bytes) % (v.len() + 1);
v.insert_from_slice(insert_pos, data);
}
}
25 => {
if v.len() < CAP_GROWTH {
v.resize(next_usize!(bytes), next_u8!(bytes));
}
}
26 => {
v = SmallVec::from_elem(next_u8!(bytes), next_usize!(bytes));
}
_ => panic!("booo"),
}
}
v
}
fn do_test_all(data: &[u8]) {
do_test::<[u8; 0]>(data);
do_test::<[u8; 1]>(data);
do_test::<[u8; 2]>(data);
do_test::<[u8; 7]>(data);
do_test::<[u8; 8]>(data);
}
#[cfg(feature = "afl")]
#[macro_use]
extern crate afl;
#[cfg(feature = "afl")]
fn main() {
fuzz!(|data| {
// Remove the panic hook so we can actually catch panic
// See https://github.com/rust-fuzz/afl.rs/issues/150
std::panic::set_hook(Box::new(|_| {}));
do_test_all(data);
});
}
#[cfg(feature = "honggfuzz")]
#[macro_use]
extern crate honggfuzz;
#[cfg(feature = "honggfuzz")]
fn main() {
loop {
fuzz!(|data| {
// Remove the panic hook so we can actually catch panic
// See https://github.com/rust-fuzz/afl.rs/issues/150
std::panic::set_hook(Box::new(|_| {}));
do_test_all(data);
});
}
}
#[cfg(test)]
mod tests {
fn extend_vec_from_hex(hex: &str, out: &mut Vec<u8>) {
let mut b = 0;
for (idx, c) in hex.as_bytes().iter().enumerate() {
b <<= 4;
match *c {
b'A'..=b'F' => b |= c - b'A' + 10,
b'a'..=b'f' => b |= c - b'a' + 10,
b'0'..=b'9' => b |= c - b'0',
b'\n' => {}
b' ' => {}
_ => panic!("Bad hex"),
}
if (idx & 1) == 1 {
out.push(b);
b = 0;
}
}
}
#[test]
fn duplicate_crash() {
let mut a = Vec::new();
// paste the output of `xxd -p <crash_dump>` here and run `cargo test`
extend_vec_from_hex(
r#"
646e21f9f910f90200f9d9f9c7030000def9000010646e2af9f910f90264
6e21f9f910f90200f9d9f9c7030000def90000106400f9f9d9f9c7030000
def90000106400f9d9f9e7f1000000d9f9e7f1000000f9
"#,
&mut a,
);
super::do_test_all(&a);
}
}
+19
View File
@@ -0,0 +1,19 @@
#!/bin/bash
set -e
cargo install --force honggfuzz
for TARGET in fuzz_targets/*; do
FILENAME=$(basename $TARGET)
FILE="${FILENAME%.*}"
if [ -d hfuzz_input/$FILE ]; then
HFUZZ_INPUT_ARGS="-f hfuzz_input/$FILE/input"
fi
HFUZZ_BUILD_ARGS="--features honggfuzz_fuzz" HFUZZ_RUN_ARGS="--run_time 30 --exit_upon_crash -v $HFUZZ_INPUT_ARGS" cargo hfuzz run $FILE
if [ -f hfuzz_workspace/$FILE/HONGGFUZZ.REPORT.TXT ]; then
cat hfuzz_workspace/$FILE/HONGGFUZZ.REPORT.TXT
for CASE in hfuzz_workspace/$FILE/SIG*; do
cat $CASE | xxd -p
done
exit 1
fi
done