diff --git a/.cargo-ok b/.cargo-ok deleted file mode 100644 index b5754e2..0000000 --- a/.cargo-ok +++ /dev/null @@ -1 +0,0 @@ -ok \ No newline at end of file diff --git a/.cargo_vcs_info.json b/.cargo_vcs_info.json deleted file mode 100644 index 0448b81..0000000 --- a/.cargo_vcs_info.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "git": { - "sha1": "ea778eb80729a01094b033c45633122052d96fd4" - }, - "path_in_vcs": "" -} \ No newline at end of file diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c51ca9c..5723f2b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -24,7 +24,7 @@ jobs: strategy: fail-fast: false matrix: - rust: [1.56.0, stable, beta] + rust: [1.63.0, stable, beta] timeout-minutes: 45 steps: - uses: actions/checkout@v4 @@ -56,7 +56,7 @@ jobs: with: components: rust-src - name: Enable type layout randomization - run: echo RUSTFLAGS=${RUSTFLAGS}\ -Zrandomize-layout >> $GITHUB_ENV + run: echo RUSTFLAGS=${RUSTFLAGS}\ -Zrandomize-layout\ --cfg=randomize_layout >> $GITHUB_ENV - run: cargo check env: RUSTFLAGS: --cfg procmacro2_nightly_testing ${{env.RUSTFLAGS}} @@ -77,6 +77,51 @@ jobs: run: cargo test env: RUSTFLAGS: -Z allow-features= --cfg procmacro2_backtrace ${{env.RUSTFLAGS}} + - uses: actions/upload-artifact@v4 + if: always() + with: + name: Cargo.lock + path: Cargo.lock + continue-on-error: true + + layout: + name: Layout + needs: pre_ci + if: needs.pre_ci.outputs.continue + runs-on: ubuntu-latest + timeout-minutes: 45 + steps: + - uses: actions/checkout@v4 + - uses: dtolnay/rust-toolchain@nightly + with: + components: rust-src + - run: cargo test --test test_size + - run: cargo test --test test_size --features span-locations + - run: cargo test --test test_size --no-default-features + - run: cargo test --test test_size --no-default-features --features span-locations + + msrv: + name: Rust 1.56.0 + needs: pre_ci + if: needs.pre_ci.outputs.continue + runs-on: ubuntu-latest + timeout-minutes: 45 + steps: + - uses: actions/checkout@v4 + - uses: dtolnay/rust-toolchain@1.56.0 + with: + components: rust-src + - run: cargo check + - run: cargo check --no-default-features + - run: cargo check --features span-locations + - name: RUSTFLAGS='--cfg procmacro2_semver_exempt' cargo check + run: cargo check + env: + RUSTFLAGS: --cfg procmacro2_semver_exempt ${{env.RUSTFLAGS}} + - name: RUSTFLAGS='--cfg procmacro2_semver_exempt' cargo check --no-default-features + run: cargo check --no-default-features + env: + RUSTFLAGS: --cfg procmacro2_semver_exempt ${{env.RUSTFLAGS}} minimal: name: Minimal versions @@ -175,6 +220,7 @@ jobs: timeout-minutes: 45 steps: - uses: actions/checkout@v4 + - uses: dtolnay/rust-toolchain@stable - uses: dtolnay/install@cargo-outdated - run: cargo outdated --workspace --exit-code 1 - run: cargo outdated --manifest-path fuzz/Cargo.toml --exit-code 1 diff --git a/BUILD.gn b/BUILD.gn index fa6a97a..2b848c6 100644 --- a/BUILD.gn +++ b/BUILD.gn @@ -20,7 +20,7 @@ ohos_cargo_crate("lib") { sources = [ "src/lib.rs" ] edition = "2021" - cargo_pkg_version = "1.0.76" + cargo_pkg_version = "1.0.92" cargo_pkg_authors = "David Tolnay , Alex Crichton " cargo_pkg_name = "proc-macro2" diff --git a/Cargo.toml b/Cargo.toml index 280bbdf..f9d775e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,67 +1,62 @@ -# THIS FILE IS AUTOMATICALLY GENERATED BY CARGO -# -# When uploading crates to the registry Cargo will automatically -# "normalize" Cargo.toml files for maximal compatibility -# with all versions of Cargo and also rewrite `path` dependencies -# to registry (e.g., crates.io) dependencies. -# -# If you are reading this file be aware that the original Cargo.toml -# will likely look very different (and much more reasonable). -# See Cargo.toml.orig for the original contents. - [package] -edition = "2021" -rust-version = "1.56" name = "proc-macro2" -version = "1.0.76" -authors = [ - "David Tolnay ", - "Alex Crichton ", -] +version = "1.0.92" +authors = ["David Tolnay ", "Alex Crichton "] autobenches = false +categories = ["development-tools::procedural-macro-helpers"] description = "A substitute implementation of the compiler's `proc_macro` API to decouple token-based libraries from the procedural macro use case." documentation = "https://docs.rs/proc-macro2" -readme = "README.md" -keywords = [ - "macros", - "syn", -] -categories = ["development-tools::procedural-macro-helpers"] +edition = "2021" +keywords = ["macros", "syn"] license = "MIT OR Apache-2.0" repository = "https://github.com/dtolnay/proc-macro2" +rust-version = "1.56" [package.metadata.docs.rs] -rustc-args = [ - "--cfg", - "procmacro2_semver_exempt", -] -rustdoc-args = [ - "--cfg", - "procmacro2_semver_exempt", - "--cfg", - "doc_cfg", - "--generate-link-to-definition", -] +rustc-args = ["--cfg", "procmacro2_semver_exempt"] +rustdoc-args = ["--cfg", "procmacro2_semver_exempt", "--generate-link-to-definition"] targets = ["x86_64-unknown-linux-gnu"] [package.metadata.playground] features = ["span-locations"] +[dependencies] +unicode-ident = "1.0" + +[dev-dependencies] +flate2 = "1.0" +quote = { version = "1.0", default-features = false } +rayon = "1.0" +rustversion = "1" +tar = "0.4" + +[features] +proc-macro = [] +default = ["proc-macro"] + +# Expose methods Span::start and Span::end which give the line/column location +# of a token. +span-locations = [] + +# This feature no longer means anything. +nightly = [] + [lib] doc-scrape-examples = false -[dependencies.unicode-ident] -version = "1.0" +[workspace] +members = ["benches/bench-libproc-macro", "tests/ui"] -[dev-dependencies.quote] -version = "1.0" -default_features = false - -[dev-dependencies.rustversion] -version = "1" - -[features] -default = ["proc-macro"] -nightly = [] -proc-macro = [] -span-locations = [] +[patch.crates-io] +# Our doc tests depend on quote which depends on proc-macro2. Without this line, +# the proc-macro2 dependency of quote would be the released version of +# proc-macro2. Quote would implement its traits for types from that proc-macro2, +# meaning impls would be missing when tested against types from the local +# proc-macro2. +# +# GitHub Actions builds that are in progress at the time that you publish may +# spuriously fail. This is because they'll be building a local proc-macro2 which +# carries the second-most-recent version number, pulling in quote which resolves +# to a dependency on the just-published most recent version number. Thus the +# patch will fail to apply because the version numbers are different. +proc-macro2 = { path = "." } diff --git a/Cargo.toml.orig b/Cargo.toml.orig deleted file mode 100644 index 6f7c786..0000000 --- a/Cargo.toml.orig +++ /dev/null @@ -1,59 +0,0 @@ -[package] -name = "proc-macro2" -version = "1.0.76" -authors = ["David Tolnay ", "Alex Crichton "] -autobenches = false -categories = ["development-tools::procedural-macro-helpers"] -description = "A substitute implementation of the compiler's `proc_macro` API to decouple token-based libraries from the procedural macro use case." -documentation = "https://docs.rs/proc-macro2" -edition = "2021" -keywords = ["macros", "syn"] -license = "MIT OR Apache-2.0" -repository = "https://github.com/dtolnay/proc-macro2" -rust-version = "1.56" - -[package.metadata.docs.rs] -rustc-args = ["--cfg", "procmacro2_semver_exempt"] -rustdoc-args = ["--cfg", "procmacro2_semver_exempt", "--cfg", "doc_cfg", "--generate-link-to-definition"] -targets = ["x86_64-unknown-linux-gnu"] - -[package.metadata.playground] -features = ["span-locations"] - -[dependencies] -unicode-ident = "1.0" - -[dev-dependencies] -quote = { version = "1.0", default_features = false } -rustversion = "1" - -[features] -proc-macro = [] -default = ["proc-macro"] - -# Expose methods Span::start and Span::end which give the line/column location -# of a token. -span-locations = [] - -# This feature no longer means anything. -nightly = [] - -[lib] -doc-scrape-examples = false - -[workspace] -members = ["benches/bench-libproc-macro", "tests/ui"] - -[patch.crates-io] -# Our doc tests depend on quote which depends on proc-macro2. Without this line, -# the proc-macro2 dependency of quote would be the released version of -# proc-macro2. Quote would implement its traits for types from that proc-macro2, -# meaning impls would be missing when tested against types from the local -# proc-macro2. -# -# GitHub Actions builds that are in progress at the time that you publish may -# spuriously fail. This is because they'll be building a local proc-macro2 which -# carries the second-most-recent version number, pulling in quote which resolves -# to a dependency on the just-published most recent version number. Thus the -# patch will fail to apply because the version numbers are different. -proc-macro2 = { path = "." } diff --git a/README.OpenSource b/README.OpenSource index 4100b6f..311adc9 100644 --- a/README.OpenSource +++ b/README.OpenSource @@ -3,7 +3,7 @@ "Name": "proc-macro2", "License": "Apache License V2.0, MIT", "License File": "LICENSE-APACHE, LICENSE-MIT", - "Version Number": "1.0.76", + "Version Number": "1.0.92", "Owner": "fangting12@huawei.com", "Upstream URL": "https://github.com/dtolnay/proc-macro2", "Description": "A Rust library that provides support for error handling in procedural macros." diff --git a/benches/bench-libproc-macro/Cargo.toml b/benches/bench-libproc-macro/Cargo.toml new file mode 100644 index 0000000..5f8e56b --- /dev/null +++ b/benches/bench-libproc-macro/Cargo.toml @@ -0,0 +1,14 @@ +[package] +name = "bench-libproc-macro" +version = "0.0.0" +authors = ["David Tolnay "] +edition = "2018" +publish = false + +[lib] +path = "lib.rs" +proc-macro = true + +[[bin]] +name = "bench-libproc-macro" +path = "main.rs" diff --git a/benches/bench-libproc-macro/README.md b/benches/bench-libproc-macro/README.md new file mode 100644 index 0000000..e286ddd --- /dev/null +++ b/benches/bench-libproc-macro/README.md @@ -0,0 +1,10 @@ +Example output: + +```console +$ cargo check --release + + Compiling bench-libproc-macro v0.0.0 +STRING: 37 millis +TOKENSTREAM: 276 millis + Finished release [optimized] target(s) in 1.16s +``` diff --git a/benches/bench-libproc-macro/lib.rs b/benches/bench-libproc-macro/lib.rs new file mode 100644 index 0000000..44c2e68 --- /dev/null +++ b/benches/bench-libproc-macro/lib.rs @@ -0,0 +1,49 @@ +extern crate proc_macro; + +use proc_macro::{Ident, Punct, Spacing, Span, TokenStream, TokenTree}; +use std::iter::once; +use std::time::Instant; + +const N: u32 = 20000; + +#[proc_macro] +pub fn bench(_input: TokenStream) -> TokenStream { + let start = Instant::now(); + let mut string = String::new(); + for _ in 0..N { + string += "core"; + string += ":"; + string += ":"; + string += "option"; + string += ":"; + string += ":"; + string += "Option"; + string += ":"; + string += ":"; + string += "None"; + string += ","; + } + string.parse::().unwrap(); + eprintln!("STRING: {} millis", start.elapsed().as_millis()); + + let start = Instant::now(); + let span = Span::call_site(); + let mut tokens = TokenStream::new(); + for _ in 0..N { + // Similar to what is emitted by quote. + tokens.extend(once(TokenTree::Ident(Ident::new("core", span)))); + tokens.extend(once(TokenTree::Punct(Punct::new(':', Spacing::Joint)))); + tokens.extend(once(TokenTree::Punct(Punct::new(':', Spacing::Alone)))); + tokens.extend(once(TokenTree::Ident(Ident::new("option", span)))); + tokens.extend(once(TokenTree::Punct(Punct::new(':', Spacing::Joint)))); + tokens.extend(once(TokenTree::Punct(Punct::new(':', Spacing::Alone)))); + tokens.extend(once(TokenTree::Ident(Ident::new("Option", span)))); + tokens.extend(once(TokenTree::Punct(Punct::new(':', Spacing::Joint)))); + tokens.extend(once(TokenTree::Punct(Punct::new(':', Spacing::Alone)))); + tokens.extend(once(TokenTree::Ident(Ident::new("None", span)))); + tokens.extend(once(TokenTree::Punct(Punct::new(',', Spacing::Joint)))); + } + eprintln!("TOKENSTREAM: {} millis", start.elapsed().as_millis()); + + TokenStream::new() +} diff --git a/benches/bench-libproc-macro/main.rs b/benches/bench-libproc-macro/main.rs new file mode 100644 index 0000000..34eedf6 --- /dev/null +++ b/benches/bench-libproc-macro/main.rs @@ -0,0 +1,3 @@ +bench_libproc_macro::bench!(); + +fn main() {} diff --git a/build.rs b/build.rs index 3347f87..bced1e0 100644 --- a/build.rs +++ b/build.rs @@ -1,49 +1,34 @@ -// rustc-cfg emitted by the build script: -// -// "wrap_proc_macro" -// Wrap types from libproc_macro rather than polyfilling the whole API. -// Enabled on rustc 1.29+ as long as procmacro2_semver_exempt is not set, -// because we can't emulate the unstable API without emulating everything -// else. Also enabled unconditionally on nightly, in which case the -// procmacro2_semver_exempt surface area is implemented by using the -// nightly-only proc_macro API. -// -// "hygiene" -// Enable Span::mixed_site() and non-dummy behavior of Span::resolved_at -// and Span::located_at. Enabled on Rust 1.45+. -// -// "proc_macro_span" -// Enable non-dummy behavior of Span::start and Span::end methods which -// requires an unstable compiler feature. Enabled when building with -// nightly, unless `-Z allow-feature` in RUSTFLAGS disallows unstable -// features. -// -// "super_unstable" -// Implement the semver exempt API in terms of the nightly-only proc_macro -// API. Enabled when using procmacro2_semver_exempt on a nightly compiler. -// -// "span_locations" -// Provide methods Span::start and Span::end which give the line/column -// location of a token. Enabled by procmacro2_semver_exempt or the -// "span-locations" Cargo cfg. This is behind a cfg because tracking -// location inside spans is a performance hit. -// -// "is_available" -// Use proc_macro::is_available() to detect if the proc macro API is -// available or needs to be polyfilled instead of trying to use the proc -// macro API and catching a panic if it isn't available. Enabled on Rust -// 1.57+. +#![allow(unknown_lints)] +#![allow(unexpected_cfgs)] use std::env; use std::ffi::OsString; +use std::fs; +use std::io::ErrorKind; +use std::iter; use std::path::Path; use std::process::{self, Command, Stdio}; use std::str; -use std::u32; fn main() { let rustc = rustc_minor_version().unwrap_or(u32::MAX); + if rustc >= 80 { + println!("cargo:rustc-check-cfg=cfg(fuzzing)"); + println!("cargo:rustc-check-cfg=cfg(no_is_available)"); + println!("cargo:rustc-check-cfg=cfg(no_literal_byte_character)"); + println!("cargo:rustc-check-cfg=cfg(no_literal_c_string)"); + println!("cargo:rustc-check-cfg=cfg(no_source_text)"); + println!("cargo:rustc-check-cfg=cfg(proc_macro_span)"); + println!("cargo:rustc-check-cfg=cfg(procmacro2_backtrace)"); + println!("cargo:rustc-check-cfg=cfg(procmacro2_nightly_testing)"); + println!("cargo:rustc-check-cfg=cfg(procmacro2_semver_exempt)"); + println!("cargo:rustc-check-cfg=cfg(randomize_layout)"); + println!("cargo:rustc-check-cfg=cfg(span_locations)"); + println!("cargo:rustc-check-cfg=cfg(super_unstable)"); + println!("cargo:rustc-check-cfg=cfg(wrap_proc_macro)"); + } + let docs_rs = env::var_os("DOCS_RS").is_some(); let semver_exempt = cfg!(procmacro2_semver_exempt) || docs_rs; if semver_exempt { @@ -52,17 +37,32 @@ fn main() { } if semver_exempt || cfg!(feature = "span-locations") { + // Provide methods Span::start and Span::end which give the line/column + // location of a token. This is behind a cfg because tracking location + // inside spans is a performance hit. println!("cargo:rustc-cfg=span_locations"); } if rustc < 57 { + // Do not use proc_macro::is_available() to detect whether the proc + // macro API is available vs needs to be polyfilled. Instead, use the + // proc macro API unconditionally and catch the panic that occurs if it + // isn't available. println!("cargo:rustc-cfg=no_is_available"); } if rustc < 66 { + // Do not call libproc_macro's Span::source_text. Always return None. println!("cargo:rustc-cfg=no_source_text"); } + if rustc < 79 { + // Do not call Literal::byte_character nor Literal::c_string. They can + // be emulated by way of Literal::from_str. + println!("cargo:rustc-cfg=no_literal_byte_character"); + println!("cargo:rustc-cfg=no_literal_c_string"); + } + if !cfg!(feature = "proc-macro") { println!("cargo:rerun-if-changed=build.rs"); return; @@ -106,14 +106,26 @@ fn main() { } if proc_macro_span || !semver_exempt { + // Wrap types from libproc_macro rather than polyfilling the whole API. + // Enabled as long as procmacro2_semver_exempt is not set, because we + // can't emulate the unstable API without emulating everything else. + // Also enabled unconditionally on nightly, in which case the + // procmacro2_semver_exempt surface area is implemented by using the + // nightly-only proc_macro API. println!("cargo:rustc-cfg=wrap_proc_macro"); } if proc_macro_span { + // Enable non-dummy behavior of Span::start and Span::end methods which + // requires an unstable compiler feature. Enabled when building with + // nightly, unless `-Z allow-feature` in RUSTFLAGS disallows unstable + // features. println!("cargo:rustc-cfg=proc_macro_span"); } if semver_exempt && proc_macro_span { + // Implement the semver exempt API in terms of the nightly-only + // proc_macro API. println!("cargo:rustc-cfg=super_unstable"); } @@ -136,17 +148,25 @@ fn compile_probe(rustc_bootstrap: bool) -> bool { let rustc = cargo_env_var("RUSTC"); let out_dir = cargo_env_var("OUT_DIR"); + let out_subdir = Path::new(&out_dir).join("probe"); let probefile = Path::new("build").join("probe.rs"); - // Make sure to pick up Cargo rustc configuration. - let mut cmd = if let Some(wrapper) = env::var_os("RUSTC_WRAPPER") { - let mut cmd = Command::new(wrapper); - // The wrapper's first argument is supposed to be the path to rustc. - cmd.arg(rustc); - cmd - } else { - Command::new(rustc) - }; + if let Err(err) = fs::create_dir(&out_subdir) { + if err.kind() != ErrorKind::AlreadyExists { + eprintln!("Failed to create {}: {}", out_subdir.display(), err); + process::exit(1); + } + } + + let rustc_wrapper = env::var_os("RUSTC_WRAPPER").filter(|wrapper| !wrapper.is_empty()); + let rustc_workspace_wrapper = + env::var_os("RUSTC_WORKSPACE_WRAPPER").filter(|wrapper| !wrapper.is_empty()); + let mut rustc = rustc_wrapper + .into_iter() + .chain(rustc_workspace_wrapper) + .chain(iter::once(rustc)); + let mut cmd = Command::new(rustc.next().unwrap()); + cmd.args(rustc); if !rustc_bootstrap { cmd.env_remove("RUSTC_BOOTSTRAP"); @@ -156,9 +176,10 @@ fn compile_probe(rustc_bootstrap: bool) -> bool { .arg("--edition=2021") .arg("--crate-name=proc_macro2") .arg("--crate-type=lib") + .arg("--cap-lints=allow") .arg("--emit=dep-info,metadata") .arg("--out-dir") - .arg(out_dir) + .arg(&out_subdir) .arg(probefile); if let Some(target) = env::var_os("TARGET") { @@ -174,10 +195,22 @@ fn compile_probe(rustc_bootstrap: bool) -> bool { } } - match cmd.status() { + let success = match cmd.status() { Ok(status) => status.success(), Err(_) => false, + }; + + // Clean up to avoid leaving nondeterministic absolute paths in the dep-info + // file in OUT_DIR, which causes nonreproducible builds in build systems + // that treat the entire OUT_DIR as an artifact. + if let Err(err) = fs::remove_dir_all(&out_subdir) { + if err.kind() != ErrorKind::NotFound { + eprintln!("Failed to clean up {}: {}", out_subdir.display(), err); + process::exit(1); + } } + + success } fn rustc_minor_version() -> Option { diff --git a/build/probe.rs b/build/probe.rs index 5afa13a..79c8ae2 100644 --- a/build/probe.rs +++ b/build/probe.rs @@ -6,9 +6,29 @@ extern crate proc_macro; -use core::ops::RangeBounds; +use core::ops::{Range, RangeBounds}; use proc_macro::{Literal, Span}; +pub fn byte_range(this: &Span) -> Range { + this.byte_range() +} + +pub fn start(this: &Span) -> Span { + this.start() +} + +pub fn end(this: &Span) -> Span { + this.end() +} + +pub fn line(this: &Span) -> usize { + this.line() +} + +pub fn column(this: &Span) -> usize { + this.column() +} + pub fn join(this: &Span, other: Span) -> Option { this.join(other) } diff --git a/fuzz/.gitignore b/fuzz/.gitignore new file mode 100644 index 0000000..2cd8199 --- /dev/null +++ b/fuzz/.gitignore @@ -0,0 +1,8 @@ +artifacts/ +corpus/ +coverage/ +hfuzz_target/ +hfuzz_workspace/ +in/ +out/ +target/ diff --git a/fuzz/Cargo.toml b/fuzz/Cargo.toml new file mode 100644 index 0000000..9efaa0e --- /dev/null +++ b/fuzz/Cargo.toml @@ -0,0 +1,30 @@ +[package] +name = "proc-macro2-fuzz" +version = "0.0.0" +authors = ["David Tolnay "] +edition = "2021" +publish = false + +[package.metadata] +cargo-fuzz = true + +[dependencies] +afl = { version = "0.15", optional = true } +honggfuzz = { version = "0.5", optional = true } +libfuzzer-sys = { version = "0.4.7", optional = true } +proc-macro2 = { path = "..", default-features = false } + +[features] +default = ["libfuzzer"] +afl = ["dep:afl"] +honggfuzz = ["dep:honggfuzz"] +libfuzzer = ["dep:libfuzzer-sys"] +span-locations = ["proc-macro2/span-locations"] + +[[bin]] +name = "parse_token_stream" +path = "fuzz_targets/parse_token_stream.rs" +test = false +doc = false + +[workspace] diff --git a/fuzz/fuzz_targets/parse_token_stream.rs b/fuzz/fuzz_targets/parse_token_stream.rs new file mode 100644 index 0000000..c8d7966 --- /dev/null +++ b/fuzz/fuzz_targets/parse_token_stream.rs @@ -0,0 +1,67 @@ +// libfuzzer: +// +// cargo install cargo-fuzz +// cargo fuzz run parse_token_stream -j $(nproc) -- -max_len=200 -timeout=1 +// +// afl++: +// +// cargo install cargo-afl +// cargo afl build --no-default-features --features afl --release +// cargo afl fuzz -i in -o out target/release/parse_token_stream +// +// honggfuzz: +// +// cargo install honggfuzz +// cargo hfuzz build --no-default-features --features honggfuzz +// HFUZZ_RUN_ARGS="--threads $(nproc) --max_file_size 200 --timeout 1" cargo hfuzz run parse_token_stream + +#![cfg_attr(feature = "libfuzzer", no_main)] + +use std::str; + +#[cfg(not(any( + all( + feature = "libfuzzer", + not(feature = "afl"), + not(feature = "honggfuzz") + ), + all( + not(feature = "libfuzzer"), + feature = "afl", + not(feature = "honggfuzz") + ), + all( + not(feature = "libfuzzer"), + not(feature = "afl"), + feature = "honggfuzz" + ), +)))] +fn main() { + compile_error! { + r#"exactly one of feature="libfuzzer" or feature="afl" or feature="honggfuzz" must be enabled"# + } +} + +#[cfg(feature = "libfuzzer")] +libfuzzer_sys::fuzz_target!(|bytes: &[u8]| do_fuzz(bytes)); + +#[cfg(feature = "afl")] +fn main() { + let hook = true; // turn panic into crashes + afl::fuzz(hook, do_fuzz); +} + +#[cfg(feature = "honggfuzz")] +fn main() { + loop { + honggfuzz::fuzz(do_fuzz); + } +} + +fn do_fuzz(bytes: &[u8]) { + let ..=199 = bytes.len() else { return }; + let Ok(string) = str::from_utf8(bytes) else { + return; + }; + let _ = string.parse::(); +} diff --git a/src/extra.rs b/src/extra.rs index 4a69d46..522a90e 100644 --- a/src/extra.rs +++ b/src/extra.rs @@ -3,18 +3,85 @@ use crate::fallback; use crate::imp; -use crate::marker::Marker; +use crate::marker::{ProcMacroAutoTraits, MARKER}; use crate::Span; use core::fmt::{self, Debug}; +/// Invalidate any `proc_macro2::Span` that exist on the current thread. +/// +/// The implementation of `Span` uses thread-local data structures and this +/// function clears them. Calling any method on a `Span` on the current thread +/// created prior to the invalidation will return incorrect values or crash. +/// +/// This function is useful for programs that process more than 232 +/// bytes of Rust source code on the same thread. Just like rustc, proc-macro2 +/// uses 32-bit source locations, and these wrap around when the total source +/// code processed by the same thread exceeds 232 bytes (4 +/// gigabytes). After a wraparound, `Span` methods such as `source_text()` can +/// return wrong data. +/// +/// # Example +/// +/// As of late 2023, there is 200 GB of Rust code published on crates.io. +/// Looking at just the newest version of every crate, it is 16 GB of code. So a +/// workload that involves parsing it all would overflow a 32-bit source +/// location unless spans are being invalidated. +/// +/// ``` +/// use flate2::read::GzDecoder; +/// use std::ffi::OsStr; +/// use std::io::{BufReader, Read}; +/// use std::str::FromStr; +/// use tar::Archive; +/// +/// rayon::scope(|s| { +/// for krate in every_version_of_every_crate() { +/// s.spawn(move |_| { +/// proc_macro2::extra::invalidate_current_thread_spans(); +/// +/// let reader = BufReader::new(krate); +/// let tar = GzDecoder::new(reader); +/// let mut archive = Archive::new(tar); +/// for entry in archive.entries().unwrap() { +/// let mut entry = entry.unwrap(); +/// let path = entry.path().unwrap(); +/// if path.extension() != Some(OsStr::new("rs")) { +/// continue; +/// } +/// let mut content = String::new(); +/// entry.read_to_string(&mut content).unwrap(); +/// match proc_macro2::TokenStream::from_str(&content) { +/// Ok(tokens) => {/* ... */}, +/// Err(_) => continue, +/// } +/// } +/// }); +/// } +/// }); +/// # +/// # fn every_version_of_every_crate() -> Vec { +/// # Vec::new() +/// # } +/// ``` +/// +/// # Panics +/// +/// This function is not applicable to and will panic if called from a +/// procedural macro. +#[cfg(span_locations)] +#[cfg_attr(docsrs, doc(cfg(feature = "span-locations")))] +pub fn invalidate_current_thread_spans() { + crate::imp::invalidate_current_thread_spans(); +} + /// An object that holds a [`Group`]'s `span_open()` and `span_close()` together -/// (in a more compact representation than holding those 2 spans individually. +/// in a more compact representation than holding those 2 spans individually. /// /// [`Group`]: crate::Group #[derive(Copy, Clone)] pub struct DelimSpan { inner: DelimSpanEnum, - _marker: Marker, + _marker: ProcMacroAutoTraits, } #[derive(Copy, Clone)] @@ -45,7 +112,7 @@ impl DelimSpan { DelimSpan { inner, - _marker: Marker, + _marker: MARKER, } } diff --git a/src/fallback.rs b/src/fallback.rs index 7b40427..92b342f 100644 --- a/src/fallback.rs +++ b/src/fallback.rs @@ -1,3 +1,5 @@ +#[cfg(wrap_proc_macro)] +use crate::imp; #[cfg(span_locations)] use crate::location::LineColumn; use crate::parse::{self, Cursor}; @@ -11,9 +13,17 @@ use core::cell::RefCell; use core::cmp; use core::fmt::{self, Debug, Display, Write}; use core::mem::ManuallyDrop; +#[cfg(span_locations)] +use core::ops::Range; use core::ops::RangeBounds; use core::ptr; +use core::str; +#[cfg(feature = "proc-macro")] use core::str::FromStr; +use std::ffi::CStr; +#[cfg(wrap_proc_macro)] +use std::panic; +#[cfg(procmacro2_semver_exempt)] use std::path::PathBuf; /// Force use of proc-macro2's fallback implementation of the API for now, even @@ -53,13 +63,31 @@ impl LexError { } impl TokenStream { - pub fn new() -> Self { + pub(crate) fn new() -> Self { TokenStream { inner: RcVecBuilder::new().build(), } } - pub fn is_empty(&self) -> bool { + pub(crate) fn from_str_checked(src: &str) -> Result { + // Create a dummy file & add it to the source map + let mut cursor = get_cursor(src); + + // Strip a byte order mark if present + const BYTE_ORDER_MARK: &str = "\u{feff}"; + if cursor.starts_with(BYTE_ORDER_MARK) { + cursor = cursor.advance(BYTE_ORDER_MARK.len()); + } + + parse::token_stream(cursor) + } + + #[cfg(feature = "proc-macro")] + pub(crate) fn from_str_unchecked(src: &str) -> Self { + Self::from_str_checked(src).unwrap() + } + + pub(crate) fn is_empty(&self) -> bool { self.inner.len() == 0 } @@ -121,23 +149,23 @@ pub(crate) struct TokenStreamBuilder { } impl TokenStreamBuilder { - pub fn new() -> Self { + pub(crate) fn new() -> Self { TokenStreamBuilder { inner: RcVecBuilder::new(), } } - pub fn with_capacity(cap: usize) -> Self { + pub(crate) fn with_capacity(cap: usize) -> Self { TokenStreamBuilder { inner: RcVecBuilder::with_capacity(cap), } } - pub fn push_token_from_parser(&mut self, tt: TokenTree) { + pub(crate) fn push_token_from_parser(&mut self, tt: TokenTree) { self.inner.push(tt); } - pub fn build(self) -> TokenStream { + pub(crate) fn build(self) -> TokenStream { TokenStream { inner: self.inner.build(), } @@ -151,9 +179,9 @@ fn get_cursor(src: &str) -> Cursor { // Create a dummy file & add it to the source map #[cfg(not(fuzzing))] - SOURCE_MAP.with(|cm| { - let mut cm = cm.borrow_mut(); - let span = cm.add_file(src); + SOURCE_MAP.with(|sm| { + let mut sm = sm.borrow_mut(); + let span = sm.add_file(src); Cursor { rest: src, off: span.lo, @@ -166,23 +194,6 @@ fn get_cursor(src: &str) -> Cursor { Cursor { rest: src } } -impl FromStr for TokenStream { - type Err = LexError; - - fn from_str(src: &str) -> Result { - // Create a dummy file & add it to the source map - let mut cursor = get_cursor(src); - - // Strip a byte order mark if present - const BYTE_ORDER_MARK: &str = "\u{feff}"; - if cursor.starts_with(BYTE_ORDER_MARK) { - cursor = cursor.advance(BYTE_ORDER_MARK.len()); - } - - parse::token_stream(cursor) - } -} - impl Display for LexError { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { f.write_str("cannot parse string into token stream") @@ -222,20 +233,14 @@ impl Debug for TokenStream { #[cfg(feature = "proc-macro")] impl From for TokenStream { fn from(inner: proc_macro::TokenStream) -> Self { - inner - .to_string() - .parse() - .expect("compiler token stream parse failed") + TokenStream::from_str_unchecked(&inner.to_string()) } } #[cfg(feature = "proc-macro")] impl From for proc_macro::TokenStream { fn from(inner: TokenStream) -> Self { - inner - .to_string() - .parse() - .expect("failed to parse to compiler tokens") + proc_macro::TokenStream::from_str_unchecked(&inner.to_string()) } } @@ -295,22 +300,25 @@ impl IntoIterator for TokenStream { } } +#[cfg(procmacro2_semver_exempt)] #[derive(Clone, PartialEq, Eq)] pub(crate) struct SourceFile { path: PathBuf, } +#[cfg(procmacro2_semver_exempt)] impl SourceFile { /// Get the path to this source file as a string. - pub fn path(&self) -> PathBuf { + pub(crate) fn path(&self) -> PathBuf { self.path.clone() } - pub fn is_real(&self) -> bool { + pub(crate) fn is_real(&self) -> bool { false } } +#[cfg(procmacro2_semver_exempt)] impl Debug for SourceFile { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { f.debug_struct("SourceFile") @@ -334,6 +342,12 @@ thread_local! { }); } +#[cfg(span_locations)] +pub(crate) fn invalidate_current_thread_spans() { + #[cfg(not(fuzzing))] + SOURCE_MAP.with(|sm| sm.borrow_mut().files.truncate(1)); +} + #[cfg(all(span_locations, not(fuzzing)))] struct FileInfo { source_text: String, @@ -366,7 +380,7 @@ impl FileInfo { span.lo >= self.span.lo && span.hi <= self.span.hi } - fn source_text(&mut self, span: Span) -> String { + fn byte_range(&mut self, span: Span) -> Range { let lo_char = (span.lo - self.span.lo) as usize; // Look up offset of the largest already-computed char index that is @@ -395,11 +409,15 @@ impl FileInfo { let trunc_lo = &self.source_text[lo_byte..]; let char_len = (span.hi - span.lo) as usize; - let source_text = match trunc_lo.char_indices().nth(char_len) { - Some((offset, _ch)) => &trunc_lo[..offset], - None => trunc_lo, - }; - source_text.to_owned() + lo_byte..match trunc_lo.char_indices().nth(char_len) { + Some((offset, _ch)) => lo_byte + offset, + None => self.source_text.len(), + } + } + + fn source_text(&mut self, span: Span) -> String { + let byte_range = self.byte_range(span); + self.source_text[byte_range].to_owned() } } @@ -497,83 +515,98 @@ pub(crate) struct Span { impl Span { #[cfg(not(span_locations))] - pub fn call_site() -> Self { + pub(crate) fn call_site() -> Self { Span {} } #[cfg(span_locations)] - pub fn call_site() -> Self { + pub(crate) fn call_site() -> Self { Span { lo: 0, hi: 0 } } - pub fn mixed_site() -> Self { + pub(crate) fn mixed_site() -> Self { Span::call_site() } #[cfg(procmacro2_semver_exempt)] - pub fn def_site() -> Self { + pub(crate) fn def_site() -> Self { Span::call_site() } - pub fn resolved_at(&self, _other: Span) -> Span { + pub(crate) fn resolved_at(&self, _other: Span) -> Span { // Stable spans consist only of line/column information, so // `resolved_at` and `located_at` only select which span the // caller wants line/column information from. *self } - pub fn located_at(&self, other: Span) -> Span { + pub(crate) fn located_at(&self, other: Span) -> Span { other } #[cfg(procmacro2_semver_exempt)] - pub fn source_file(&self) -> SourceFile { + pub(crate) fn source_file(&self) -> SourceFile { #[cfg(fuzzing)] return SourceFile { path: PathBuf::from(""), }; #[cfg(not(fuzzing))] - SOURCE_MAP.with(|cm| { - let cm = cm.borrow(); - let path = cm.filepath(*self); + SOURCE_MAP.with(|sm| { + let sm = sm.borrow(); + let path = sm.filepath(*self); SourceFile { path } }) } #[cfg(span_locations)] - pub fn start(&self) -> LineColumn { + pub(crate) fn byte_range(&self) -> Range { + #[cfg(fuzzing)] + return 0..0; + + #[cfg(not(fuzzing))] + { + if self.is_call_site() { + 0..0 + } else { + SOURCE_MAP.with(|sm| sm.borrow_mut().fileinfo_mut(*self).byte_range(*self)) + } + } + } + + #[cfg(span_locations)] + pub(crate) fn start(&self) -> LineColumn { #[cfg(fuzzing)] return LineColumn { line: 0, column: 0 }; #[cfg(not(fuzzing))] - SOURCE_MAP.with(|cm| { - let cm = cm.borrow(); - let fi = cm.fileinfo(*self); + SOURCE_MAP.with(|sm| { + let sm = sm.borrow(); + let fi = sm.fileinfo(*self); fi.offset_line_column(self.lo as usize) }) } #[cfg(span_locations)] - pub fn end(&self) -> LineColumn { + pub(crate) fn end(&self) -> LineColumn { #[cfg(fuzzing)] return LineColumn { line: 0, column: 0 }; #[cfg(not(fuzzing))] - SOURCE_MAP.with(|cm| { - let cm = cm.borrow(); - let fi = cm.fileinfo(*self); + SOURCE_MAP.with(|sm| { + let sm = sm.borrow(); + let fi = sm.fileinfo(*self); fi.offset_line_column(self.hi as usize) }) } #[cfg(not(span_locations))] - pub fn join(&self, _other: Span) -> Option { + pub(crate) fn join(&self, _other: Span) -> Option { Some(Span {}) } #[cfg(span_locations)] - pub fn join(&self, other: Span) -> Option { + pub(crate) fn join(&self, other: Span) -> Option { #[cfg(fuzzing)] return { let _ = other; @@ -581,10 +614,10 @@ impl Span { }; #[cfg(not(fuzzing))] - SOURCE_MAP.with(|cm| { - let cm = cm.borrow(); + SOURCE_MAP.with(|sm| { + let sm = sm.borrow(); // If `other` is not within the same FileInfo as us, return None. - if !cm.fileinfo(*self).span_within(other) { + if !sm.fileinfo(*self).span_within(other) { return None; } Some(Span { @@ -595,12 +628,12 @@ impl Span { } #[cfg(not(span_locations))] - pub fn source_text(&self) -> Option { + pub(crate) fn source_text(&self) -> Option { None } #[cfg(span_locations)] - pub fn source_text(&self) -> Option { + pub(crate) fn source_text(&self) -> Option { #[cfg(fuzzing)] return None; @@ -609,7 +642,7 @@ impl Span { if self.is_call_site() { None } else { - Some(SOURCE_MAP.with(|cm| cm.borrow_mut().fileinfo_mut(*self).source_text(*self))) + Some(SOURCE_MAP.with(|sm| sm.borrow_mut().fileinfo_mut(*self).source_text(*self))) } } } @@ -677,7 +710,7 @@ pub(crate) struct Group { } impl Group { - pub fn new(delimiter: Delimiter, stream: TokenStream) -> Self { + pub(crate) fn new(delimiter: Delimiter, stream: TokenStream) -> Self { Group { delimiter, stream, @@ -685,27 +718,27 @@ impl Group { } } - pub fn delimiter(&self) -> Delimiter { + pub(crate) fn delimiter(&self) -> Delimiter { self.delimiter } - pub fn stream(&self) -> TokenStream { + pub(crate) fn stream(&self) -> TokenStream { self.stream.clone() } - pub fn span(&self) -> Span { + pub(crate) fn span(&self) -> Span { self.span } - pub fn span_open(&self) -> Span { + pub(crate) fn span_open(&self) -> Span { self.span.first_byte() } - pub fn span_close(&self) -> Span { + pub(crate) fn span_close(&self) -> Span { self.span.last_byte() } - pub fn set_span(&mut self, span: Span) { + pub(crate) fn set_span(&mut self, span: Span) { self.span = span; } } @@ -749,45 +782,45 @@ impl Debug for Group { #[derive(Clone)] pub(crate) struct Ident { - sym: String, + sym: Box, span: Span, raw: bool, } impl Ident { #[track_caller] - pub fn new_checked(string: &str, span: Span) -> Self { + pub(crate) fn new_checked(string: &str, span: Span) -> Self { validate_ident(string); Ident::new_unchecked(string, span) } - pub fn new_unchecked(string: &str, span: Span) -> Self { + pub(crate) fn new_unchecked(string: &str, span: Span) -> Self { Ident { - sym: string.to_owned(), + sym: Box::from(string), span, raw: false, } } #[track_caller] - pub fn new_raw_checked(string: &str, span: Span) -> Self { + pub(crate) fn new_raw_checked(string: &str, span: Span) -> Self { validate_ident_raw(string); Ident::new_raw_unchecked(string, span) } - pub fn new_raw_unchecked(string: &str, span: Span) -> Self { + pub(crate) fn new_raw_unchecked(string: &str, span: Span) -> Self { Ident { - sym: string.to_owned(), + sym: Box::from(string), span, raw: true, } } - pub fn span(&self) -> Span { + pub(crate) fn span(&self) -> Span { self.span } - pub fn set_span(&mut self, span: Span) { + pub(crate) fn set_span(&mut self, span: Span) { self.span = span; } } @@ -854,9 +887,9 @@ where fn eq(&self, other: &T) -> bool { let other = other.as_ref(); if self.raw { - other.starts_with("r#") && self.sym == other[2..] + other.starts_with("r#") && *self.sym == other[2..] } else { - self.sym == other + *self.sym == *other } } } @@ -895,13 +928,13 @@ impl Debug for Ident { #[derive(Clone)] pub(crate) struct Literal { - repr: String, + pub(crate) repr: String, span: Span, } macro_rules! suffixed_numbers { ($($name:ident => $kind:ident,)*) => ($( - pub fn $name(n: $kind) -> Literal { + pub(crate) fn $name(n: $kind) -> Literal { Literal::_new(format!(concat!("{}", stringify!($kind)), n)) } )*) @@ -909,7 +942,7 @@ macro_rules! suffixed_numbers { macro_rules! unsuffixed_numbers { ($($name:ident => $kind:ident,)*) => ($( - pub fn $name(n: $kind) -> Literal { + pub(crate) fn $name(n: $kind) -> Literal { Literal::_new(n.to_string()) } )*) @@ -923,6 +956,36 @@ impl Literal { } } + pub(crate) fn from_str_checked(repr: &str) -> Result { + let mut cursor = get_cursor(repr); + #[cfg(span_locations)] + let lo = cursor.off; + + let negative = cursor.starts_with_char('-'); + if negative { + cursor = cursor.advance(1); + if !cursor.starts_with_fn(|ch| ch.is_ascii_digit()) { + return Err(LexError::call_site()); + } + } + + if let Ok((rest, mut literal)) = parse::literal(cursor) { + if rest.is_empty() { + if negative { + literal.repr.insert(0, '-'); + } + literal.span = Span { + #[cfg(span_locations)] + lo, + #[cfg(span_locations)] + hi: rest.off, + }; + return Ok(literal); + } + } + Err(LexError::call_site()) + } + pub(crate) unsafe fn from_str_unchecked(repr: &str) -> Self { Literal::_new(repr.to_owned()) } @@ -960,7 +1023,7 @@ impl Literal { isize_unsuffixed => isize, } - pub fn f32_unsuffixed(f: f32) -> Literal { + pub(crate) fn f32_unsuffixed(f: f32) -> Literal { let mut s = f.to_string(); if !s.contains('.') { s.push_str(".0"); @@ -968,7 +1031,7 @@ impl Literal { Literal::_new(s) } - pub fn f64_unsuffixed(f: f64) -> Literal { + pub(crate) fn f64_unsuffixed(f: f64) -> Literal { let mut s = f.to_string(); if !s.contains('.') { s.push_str(".0"); @@ -976,82 +1039,109 @@ impl Literal { Literal::_new(s) } - pub fn string(t: &str) -> Literal { - let mut repr = String::with_capacity(t.len() + 2); + pub(crate) fn string(string: &str) -> Literal { + let mut repr = String::with_capacity(string.len() + 2); repr.push('"'); - let mut chars = t.chars(); - while let Some(ch) = chars.next() { - if ch == '\0' { - repr.push_str( - if chars - .as_str() - .starts_with(|next| '0' <= next && next <= '7') - { - // circumvent clippy::octal_escapes lint - "\\x00" - } else { - "\\0" - }, - ); - } else if ch == '\'' { - // escape_debug turns this into "\'" which is unnecessary. - repr.push(ch); - } else { - repr.extend(ch.escape_debug()); - } - } + escape_utf8(string, &mut repr); repr.push('"'); Literal::_new(repr) } - pub fn character(t: char) -> Literal { + pub(crate) fn character(ch: char) -> Literal { let mut repr = String::new(); repr.push('\''); - if t == '"' { + if ch == '"' { // escape_debug turns this into '\"' which is unnecessary. - repr.push(t); + repr.push(ch); } else { - repr.extend(t.escape_debug()); + repr.extend(ch.escape_debug()); } repr.push('\''); Literal::_new(repr) } - pub fn byte_string(bytes: &[u8]) -> Literal { - let mut escaped = "b\"".to_string(); + pub(crate) fn byte_character(byte: u8) -> Literal { + let mut repr = "b'".to_string(); + #[allow(clippy::match_overlapping_arm)] + match byte { + b'\0' => repr.push_str(r"\0"), + b'\t' => repr.push_str(r"\t"), + b'\n' => repr.push_str(r"\n"), + b'\r' => repr.push_str(r"\r"), + b'\'' => repr.push_str(r"\'"), + b'\\' => repr.push_str(r"\\"), + b'\x20'..=b'\x7E' => repr.push(byte as char), + _ => { + let _ = write!(repr, r"\x{:02X}", byte); + } + } + repr.push('\''); + Literal::_new(repr) + } + + pub(crate) fn byte_string(bytes: &[u8]) -> Literal { + let mut repr = "b\"".to_string(); let mut bytes = bytes.iter(); while let Some(&b) = bytes.next() { #[allow(clippy::match_overlapping_arm)] match b { - b'\0' => escaped.push_str(match bytes.as_slice().first() { + b'\0' => repr.push_str(match bytes.as_slice().first() { // circumvent clippy::octal_escapes lint Some(b'0'..=b'7') => r"\x00", _ => r"\0", }), - b'\t' => escaped.push_str(r"\t"), - b'\n' => escaped.push_str(r"\n"), - b'\r' => escaped.push_str(r"\r"), - b'"' => escaped.push_str("\\\""), - b'\\' => escaped.push_str("\\\\"), - b'\x20'..=b'\x7E' => escaped.push(b as char), + b'\t' => repr.push_str(r"\t"), + b'\n' => repr.push_str(r"\n"), + b'\r' => repr.push_str(r"\r"), + b'"' => repr.push_str("\\\""), + b'\\' => repr.push_str(r"\\"), + b'\x20'..=b'\x7E' => repr.push(b as char), _ => { - let _ = write!(escaped, "\\x{:02X}", b); + let _ = write!(repr, r"\x{:02X}", b); } } } - escaped.push('"'); - Literal::_new(escaped) + repr.push('"'); + Literal::_new(repr) } - pub fn span(&self) -> Span { + pub(crate) fn c_string(string: &CStr) -> Literal { + let mut repr = "c\"".to_string(); + let mut bytes = string.to_bytes(); + while !bytes.is_empty() { + let (valid, invalid) = match str::from_utf8(bytes) { + Ok(all_valid) => { + bytes = b""; + (all_valid, bytes) + } + Err(utf8_error) => { + let (valid, rest) = bytes.split_at(utf8_error.valid_up_to()); + let valid = str::from_utf8(valid).unwrap(); + let invalid = utf8_error + .error_len() + .map_or(rest, |error_len| &rest[..error_len]); + bytes = &bytes[valid.len() + invalid.len()..]; + (valid, invalid) + } + }; + escape_utf8(valid, &mut repr); + for &byte in invalid { + let _ = write!(repr, r"\x{:02X}", byte); + } + } + repr.push('"'); + Literal::_new(repr) + } + + pub(crate) fn span(&self) -> Span { self.span } - pub fn set_span(&mut self, span: Span) { + pub(crate) fn set_span(&mut self, span: Span) { self.span = span; } - pub fn subspan>(&self, range: R) -> Option { + pub(crate) fn subspan>(&self, range: R) -> Option { #[cfg(not(span_locations))] { let _ = range; @@ -1093,40 +1183,6 @@ impl Literal { } } -impl FromStr for Literal { - type Err = LexError; - - fn from_str(repr: &str) -> Result { - let mut cursor = get_cursor(repr); - #[cfg(span_locations)] - let lo = cursor.off; - - let negative = cursor.starts_with_char('-'); - if negative { - cursor = cursor.advance(1); - if !cursor.starts_with_fn(|ch| ch.is_ascii_digit()) { - return Err(LexError::call_site()); - } - } - - if let Ok((rest, mut literal)) = parse::literal(cursor) { - if rest.is_empty() { - if negative { - literal.repr.insert(0, '-'); - } - literal.span = Span { - #[cfg(span_locations)] - lo, - #[cfg(span_locations)] - hi: rest.off, - }; - return Ok(literal); - } - } - Err(LexError::call_site()) - } -} - impl Display for Literal { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { Display::fmt(&self.repr, f) @@ -1141,3 +1197,70 @@ impl Debug for Literal { debug.finish() } } + +fn escape_utf8(string: &str, repr: &mut String) { + let mut chars = string.chars(); + while let Some(ch) = chars.next() { + if ch == '\0' { + repr.push_str( + if chars + .as_str() + .starts_with(|next| '0' <= next && next <= '7') + { + // circumvent clippy::octal_escapes lint + r"\x00" + } else { + r"\0" + }, + ); + } else if ch == '\'' { + // escape_debug turns this into "\'" which is unnecessary. + repr.push(ch); + } else { + repr.extend(ch.escape_debug()); + } + } +} + +#[cfg(feature = "proc-macro")] +pub(crate) trait FromStr2: FromStr { + #[cfg(wrap_proc_macro)] + fn valid(src: &str) -> bool; + + #[cfg(wrap_proc_macro)] + fn from_str_checked(src: &str) -> Result { + // Validate using fallback parser, because rustc is incapable of + // returning a recoverable Err for certain invalid token streams, and + // will instead permanently poison the compilation. + if !Self::valid(src) { + return Err(imp::LexError::CompilerPanic); + } + + // Catch panic to work around https://github.com/rust-lang/rust/issues/58736. + match panic::catch_unwind(|| Self::from_str(src)) { + Ok(Ok(ok)) => Ok(ok), + Ok(Err(lex)) => Err(imp::LexError::Compiler(lex)), + Err(_panic) => Err(imp::LexError::CompilerPanic), + } + } + + fn from_str_unchecked(src: &str) -> Self { + Self::from_str(src).unwrap() + } +} + +#[cfg(feature = "proc-macro")] +impl FromStr2 for proc_macro::TokenStream { + #[cfg(wrap_proc_macro)] + fn valid(src: &str) -> bool { + TokenStream::from_str_checked(src).is_ok() + } +} + +#[cfg(feature = "proc-macro")] +impl FromStr2 for proc_macro::Literal { + #[cfg(wrap_proc_macro)] + fn valid(src: &str) -> bool { + Literal::from_str_checked(src).is_ok() + } +} diff --git a/src/lib.rs b/src/lib.rs index 7e8f543..c73a271 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -86,24 +86,27 @@ //! a different thread. // Proc-macro2 types in rustdoc of other crates get linked to here. -#![doc(html_root_url = "https://docs.rs/proc-macro2/1.0.76")] +#![doc(html_root_url = "https://docs.rs/proc-macro2/1.0.92")] #![cfg_attr(any(proc_macro_span, super_unstable), feature(proc_macro_span))] #![cfg_attr(super_unstable, feature(proc_macro_def_site))] -#![cfg_attr(doc_cfg, feature(doc_cfg))] +#![cfg_attr(docsrs, feature(doc_cfg))] #![deny(unsafe_op_in_unsafe_fn)] #![allow( clippy::cast_lossless, clippy::cast_possible_truncation, clippy::checked_conversions, clippy::doc_markdown, + clippy::incompatible_msrv, clippy::items_after_statements, clippy::iter_without_into_iter, clippy::let_underscore_untyped, clippy::manual_assert, clippy::manual_range_contains, + clippy::missing_panics_doc, clippy::missing_safety_doc, clippy::must_use_candidate, clippy::needless_doctest_main, + clippy::needless_lifetimes, clippy::new_without_default, clippy::return_self_not_must_use, clippy::shadow_unrelated, @@ -160,18 +163,21 @@ mod imp; mod location; use crate::extra::DelimSpan; -use crate::marker::Marker; +use crate::marker::{ProcMacroAutoTraits, MARKER}; use core::cmp::Ordering; use core::fmt::{self, Debug, Display}; use core::hash::{Hash, Hasher}; +#[cfg(span_locations)] +use core::ops::Range; use core::ops::RangeBounds; use core::str::FromStr; use std::error::Error; +use std::ffi::CStr; #[cfg(procmacro2_semver_exempt)] use std::path::PathBuf; #[cfg(span_locations)] -#[cfg_attr(doc_cfg, doc(cfg(feature = "span-locations")))] +#[cfg_attr(docsrs, doc(cfg(feature = "span-locations")))] pub use crate::location::LineColumn; /// An abstract stream of tokens, or more concretely a sequence of token trees. @@ -184,27 +190,27 @@ pub use crate::location::LineColumn; #[derive(Clone)] pub struct TokenStream { inner: imp::TokenStream, - _marker: Marker, + _marker: ProcMacroAutoTraits, } /// Error returned from `TokenStream::from_str`. pub struct LexError { inner: imp::LexError, - _marker: Marker, + _marker: ProcMacroAutoTraits, } impl TokenStream { fn _new(inner: imp::TokenStream) -> Self { TokenStream { inner, - _marker: Marker, + _marker: MARKER, } } fn _new_fallback(inner: fallback::TokenStream) -> Self { TokenStream { - inner: inner.into(), - _marker: Marker, + inner: imp::TokenStream::from(inner), + _marker: MARKER, } } @@ -239,27 +245,29 @@ impl FromStr for TokenStream { type Err = LexError; fn from_str(src: &str) -> Result { - let e = src.parse().map_err(|e| LexError { - inner: e, - _marker: Marker, - })?; - Ok(TokenStream::_new(e)) + match imp::TokenStream::from_str_checked(src) { + Ok(tokens) => Ok(TokenStream::_new(tokens)), + Err(lex) => Err(LexError { + inner: lex, + _marker: MARKER, + }), + } } } #[cfg(feature = "proc-macro")] -#[cfg_attr(doc_cfg, doc(cfg(feature = "proc-macro")))] +#[cfg_attr(docsrs, doc(cfg(feature = "proc-macro")))] impl From for TokenStream { fn from(inner: proc_macro::TokenStream) -> Self { - TokenStream::_new(inner.into()) + TokenStream::_new(imp::TokenStream::from(inner)) } } #[cfg(feature = "proc-macro")] -#[cfg_attr(doc_cfg, doc(cfg(feature = "proc-macro")))] +#[cfg_attr(docsrs, doc(cfg(feature = "proc-macro")))] impl From for proc_macro::TokenStream { fn from(inner: TokenStream) -> Self { - inner.inner.into() + proc_macro::TokenStream::from(inner.inner) } } @@ -335,11 +343,11 @@ impl Error for LexError {} /// /// This type is semver exempt and not exposed by default. #[cfg(all(procmacro2_semver_exempt, any(not(wrap_proc_macro), super_unstable)))] -#[cfg_attr(doc_cfg, doc(cfg(procmacro2_semver_exempt)))] +#[cfg_attr(docsrs, doc(cfg(procmacro2_semver_exempt)))] #[derive(Clone, PartialEq, Eq)] pub struct SourceFile { inner: imp::SourceFile, - _marker: Marker, + _marker: ProcMacroAutoTraits, } #[cfg(all(procmacro2_semver_exempt, any(not(wrap_proc_macro), super_unstable)))] @@ -347,7 +355,7 @@ impl SourceFile { fn _new(inner: imp::SourceFile) -> Self { SourceFile { inner, - _marker: Marker, + _marker: MARKER, } } @@ -386,21 +394,21 @@ impl Debug for SourceFile { #[derive(Copy, Clone)] pub struct Span { inner: imp::Span, - _marker: Marker, + _marker: ProcMacroAutoTraits, } impl Span { fn _new(inner: imp::Span) -> Self { Span { inner, - _marker: Marker, + _marker: MARKER, } } fn _new_fallback(inner: fallback::Span) -> Self { Span { - inner: inner.into(), - _marker: Marker, + inner: imp::Span::from(inner), + _marker: MARKER, } } @@ -424,7 +432,7 @@ impl Span { /// /// This method is semver exempt and not exposed by default. #[cfg(procmacro2_semver_exempt)] - #[cfg_attr(doc_cfg, doc(cfg(procmacro2_semver_exempt)))] + #[cfg_attr(docsrs, doc(cfg(procmacro2_semver_exempt)))] pub fn def_site() -> Self { Span::_new(imp::Span::def_site()) } @@ -467,11 +475,26 @@ impl Span { /// /// This method is semver exempt and not exposed by default. #[cfg(all(procmacro2_semver_exempt, any(not(wrap_proc_macro), super_unstable)))] - #[cfg_attr(doc_cfg, doc(cfg(procmacro2_semver_exempt)))] + #[cfg_attr(docsrs, doc(cfg(procmacro2_semver_exempt)))] pub fn source_file(&self) -> SourceFile { SourceFile::_new(self.inner.source_file()) } + /// Returns the span's byte position range in the source file. + /// + /// This method requires the `"span-locations"` feature to be enabled. + /// + /// When executing in a procedural macro context, the returned range is only + /// accurate if compiled with a nightly toolchain. The stable toolchain does + /// not have this information available. When executing outside of a + /// procedural macro, such as main.rs or build.rs, the byte range is always + /// accurate regardless of toolchain. + #[cfg(span_locations)] + #[cfg_attr(docsrs, doc(cfg(feature = "span-locations")))] + pub fn byte_range(&self) -> Range { + self.inner.byte_range() + } + /// Get the starting line/column in the source file for this span. /// /// This method requires the `"span-locations"` feature to be enabled. @@ -482,7 +505,7 @@ impl Span { /// outside of a procedural macro, such as main.rs or build.rs, the /// line/column are always meaningful regardless of toolchain. #[cfg(span_locations)] - #[cfg_attr(doc_cfg, doc(cfg(feature = "span-locations")))] + #[cfg_attr(docsrs, doc(cfg(feature = "span-locations")))] pub fn start(&self) -> LineColumn { self.inner.start() } @@ -497,7 +520,7 @@ impl Span { /// outside of a procedural macro, such as main.rs or build.rs, the /// line/column are always meaningful regardless of toolchain. #[cfg(span_locations)] - #[cfg_attr(doc_cfg, doc(cfg(feature = "span-locations")))] + #[cfg_attr(docsrs, doc(cfg(feature = "span-locations")))] pub fn end(&self) -> LineColumn { self.inner.end() } @@ -519,7 +542,7 @@ impl Span { /// /// This method is semver exempt and not exposed by default. #[cfg(procmacro2_semver_exempt)] - #[cfg_attr(doc_cfg, doc(cfg(procmacro2_semver_exempt)))] + #[cfg_attr(docsrs, doc(cfg(procmacro2_semver_exempt)))] pub fn eq(&self, other: &Span) -> bool { self.inner.eq(&other.inner) } @@ -659,13 +682,25 @@ pub enum Delimiter { Brace, /// `[ ... ]` Bracket, - /// `Ø ... Ø` + /// `∅ ... ∅` /// - /// An implicit delimiter, that may, for example, appear around tokens + /// An invisible delimiter, that may, for example, appear around tokens /// coming from a "macro variable" `$var`. It is important to preserve /// operator priorities in cases like `$var * 3` where `$var` is `1 + 2`. - /// Implicit delimiters may not survive roundtrip of a token stream through + /// Invisible delimiters may not survive roundtrip of a token stream through /// a string. + /// + ///
+ /// + /// Note: rustc currently can ignore the grouping of tokens delimited by `None` in the output + /// of a proc_macro. Only `None`-delimited groups created by a macro_rules macro in the input + /// of a proc_macro macro are preserved, and only in very specific circumstances. + /// Any `None`-delimited groups (re)created by a proc_macro will therefore not preserve + /// operator priorities as indicated above. The other `Delimiter` variants should be used + /// instead in this context. This is a rustc bug. For details, see + /// [rust-lang/rust#67062](https://github.com/rust-lang/rust/issues/67062). + /// + ///
None, } @@ -676,7 +711,7 @@ impl Group { fn _new_fallback(inner: fallback::Group) -> Self { Group { - inner: inner.into(), + inner: imp::Group::from(inner), } } @@ -802,10 +837,16 @@ impl Punct { /// The returned `Punct` will have the default span of `Span::call_site()` /// which can be further configured with the `set_span` method below. pub fn new(ch: char, spacing: Spacing) -> Self { - Punct { - ch, - spacing, - span: Span::call_site(), + if let '!' | '#' | '$' | '%' | '&' | '\'' | '*' | '+' | ',' | '-' | '.' | '/' | ':' | ';' + | '<' | '=' | '>' | '?' | '@' | '^' | '|' | '~' = ch + { + Punct { + ch, + spacing, + span: Span::call_site(), + } + } else { + panic!("unsupported proc macro punctuation character {:?}", ch); } } @@ -919,14 +960,21 @@ impl Debug for Punct { #[derive(Clone)] pub struct Ident { inner: imp::Ident, - _marker: Marker, + _marker: ProcMacroAutoTraits, } impl Ident { fn _new(inner: imp::Ident) -> Self { Ident { inner, - _marker: Marker, + _marker: MARKER, + } + } + + fn _new_fallback(inner: fallback::Ident) -> Self { + Ident { + inner: imp::Ident::from(inner), + _marker: MARKER, } } @@ -1046,7 +1094,7 @@ impl Debug for Ident { #[derive(Clone)] pub struct Literal { inner: imp::Literal, - _marker: Marker, + _marker: ProcMacroAutoTraits, } macro_rules! suffixed_int_literals { @@ -1093,14 +1141,14 @@ impl Literal { fn _new(inner: imp::Literal) -> Self { Literal { inner, - _marker: Marker, + _marker: MARKER, } } fn _new_fallback(inner: fallback::Literal) -> Self { Literal { - inner: inner.into(), - _marker: Marker, + inner: imp::Literal::from(inner), + _marker: MARKER, } } @@ -1216,9 +1264,19 @@ impl Literal { Literal::_new(imp::Literal::character(ch)) } + /// Byte character literal. + pub fn byte_character(byte: u8) -> Literal { + Literal::_new(imp::Literal::byte_character(byte)) + } + /// Byte string literal. - pub fn byte_string(s: &[u8]) -> Literal { - Literal::_new(imp::Literal::byte_string(s)) + pub fn byte_string(bytes: &[u8]) -> Literal { + Literal::_new(imp::Literal::byte_string(bytes)) + } + + /// C string literal. + pub fn c_string(string: &CStr) -> Literal { + Literal::_new(imp::Literal::c_string(string)) } /// Returns the span encompassing this literal. @@ -1258,10 +1316,13 @@ impl FromStr for Literal { type Err = LexError; fn from_str(repr: &str) -> Result { - repr.parse().map(Literal::_new).map_err(|inner| LexError { - inner, - _marker: Marker, - }) + match imp::Literal::from_str_checked(repr) { + Ok(lit) => Ok(Literal::_new(lit)), + Err(lex) => Err(LexError { + inner: lex, + _marker: MARKER, + }), + } } } @@ -1279,7 +1340,7 @@ impl Display for Literal { /// Public implementation details for the `TokenStream` type, such as iterators. pub mod token_stream { - use crate::marker::Marker; + use crate::marker::{ProcMacroAutoTraits, MARKER}; use crate::{imp, TokenTree}; use core::fmt::{self, Debug}; @@ -1292,7 +1353,7 @@ pub mod token_stream { #[derive(Clone)] pub struct IntoIter { inner: imp::TokenTreeIter, - _marker: Marker, + _marker: ProcMacroAutoTraits, } impl Iterator for IntoIter { @@ -1321,7 +1382,7 @@ pub mod token_stream { fn into_iter(self) -> IntoIter { IntoIter { inner: self.inner.into_iter(), - _marker: Marker, + _marker: MARKER, } } } diff --git a/src/location.rs b/src/location.rs index 463026c..7190e2d 100644 --- a/src/location.rs +++ b/src/location.rs @@ -3,7 +3,7 @@ use core::cmp::Ordering; /// A line-column pair representing the start or end of a `Span`. /// /// This type is semver exempt and not exposed by default. -#[cfg_attr(doc_cfg, doc(cfg(feature = "span-locations")))] +#[cfg_attr(docsrs, doc(cfg(feature = "span-locations")))] #[derive(Copy, Clone, Debug, PartialEq, Eq, Hash)] pub struct LineColumn { /// The 1-indexed line in the source file on which the span starts or ends diff --git a/src/marker.rs b/src/marker.rs index e8874bd..23b94ce 100644 --- a/src/marker.rs +++ b/src/marker.rs @@ -4,18 +4,14 @@ use core::panic::{RefUnwindSafe, UnwindSafe}; // Zero sized marker with the correct set of autotrait impls we want all proc // macro types to have. -pub(crate) type Marker = PhantomData; +#[derive(Copy, Clone)] +#[cfg_attr( + all(procmacro2_semver_exempt, any(not(wrap_proc_macro), super_unstable)), + derive(PartialEq, Eq) +)] +pub(crate) struct ProcMacroAutoTraits(PhantomData>); -pub(crate) use self::value::*; - -mod value { - pub(crate) use core::marker::PhantomData as Marker; -} - -pub(crate) struct ProcMacroAutoTraits( - #[allow(dead_code)] // https://github.com/rust-lang/rust/issues/119645 - Rc<()>, -); +pub(crate) const MARKER: ProcMacroAutoTraits = ProcMacroAutoTraits(PhantomData); impl UnwindSafe for ProcMacroAutoTraits {} impl RefUnwindSafe for ProcMacroAutoTraits {} diff --git a/src/parse.rs b/src/parse.rs index 07239bc..497b68d 100644 --- a/src/parse.rs +++ b/src/parse.rs @@ -1,5 +1,5 @@ use crate::fallback::{ - self, is_ident_continue, is_ident_start, Group, LexError, Literal, Span, TokenStream, + self, is_ident_continue, is_ident_start, Group, Ident, LexError, Literal, Span, TokenStream, TokenStreamBuilder, }; use crate::{Delimiter, Punct, Spacing, TokenTree}; @@ -8,13 +8,13 @@ use core::str::{Bytes, CharIndices, Chars}; #[derive(Copy, Clone, Eq, PartialEq)] pub(crate) struct Cursor<'a> { - pub rest: &'a str, + pub(crate) rest: &'a str, #[cfg(span_locations)] - pub off: u32, + pub(crate) off: u32, } impl<'a> Cursor<'a> { - pub fn advance(&self, bytes: usize) -> Cursor<'a> { + pub(crate) fn advance(&self, bytes: usize) -> Cursor<'a> { let (_front, rest) = self.rest.split_at(bytes); Cursor { rest, @@ -23,22 +23,22 @@ impl<'a> Cursor<'a> { } } - pub fn starts_with(&self, s: &str) -> bool { + pub(crate) fn starts_with(&self, s: &str) -> bool { self.rest.starts_with(s) } - pub fn starts_with_char(&self, ch: char) -> bool { + pub(crate) fn starts_with_char(&self, ch: char) -> bool { self.rest.starts_with(ch) } - pub fn starts_with_fn(&self, f: Pattern) -> bool + pub(crate) fn starts_with_fn(&self, f: Pattern) -> bool where Pattern: FnMut(char) -> bool, { self.rest.starts_with(f) } - pub fn is_empty(&self) -> bool { + pub(crate) fn is_empty(&self) -> bool { self.rest.is_empty() } @@ -300,10 +300,8 @@ fn ident_any(input: Cursor) -> PResult { let (rest, sym) = ident_not_raw(rest)?; if !raw { - let ident = crate::Ident::_new(crate::imp::Ident::new_unchecked( - sym, - fallback::Span::call_site(), - )); + let ident = + crate::Ident::_new_fallback(Ident::new_unchecked(sym, fallback::Span::call_site())); return Ok((rest, ident)); } @@ -312,10 +310,8 @@ fn ident_any(input: Cursor) -> PResult { _ => {} } - let ident = crate::Ident::_new(crate::imp::Ident::new_raw_unchecked( - sym, - fallback::Span::call_site(), - )); + let ident = + crate::Ident::_new_fallback(Ident::new_raw_unchecked(sym, fallback::Span::call_site())); Ok((rest, ident)) } @@ -941,10 +937,10 @@ fn doc_comment<'a>(input: Cursor<'a>, trees: &mut TokenStreamBuilder) -> PResult trees.push_token_from_parser(TokenTree::Punct(bang)); } - let doc_ident = crate::Ident::_new(crate::imp::Ident::new_unchecked("doc", fallback_span)); + let doc_ident = crate::Ident::_new_fallback(Ident::new_unchecked("doc", fallback_span)); let mut equal = Punct::new('=', Spacing::Alone); equal.set_span(span); - let mut literal = crate::Literal::string(comment); + let mut literal = crate::Literal::_new_fallback(Literal::string(comment)); literal.set_span(span); let mut bracketed = TokenStreamBuilder::with_capacity(3); bracketed.push_token_from_parser(TokenTree::Ident(doc_ident)); diff --git a/src/rcvec.rs b/src/rcvec.rs index 37955af..cde9f25 100644 --- a/src/rcvec.rs +++ b/src/rcvec.rs @@ -22,19 +22,19 @@ pub(crate) struct RcVecIntoIter { } impl RcVec { - pub fn is_empty(&self) -> bool { + pub(crate) fn is_empty(&self) -> bool { self.inner.is_empty() } - pub fn len(&self) -> usize { + pub(crate) fn len(&self) -> usize { self.inner.len() } - pub fn iter(&self) -> slice::Iter { + pub(crate) fn iter(&self) -> slice::Iter { self.inner.iter() } - pub fn make_mut(&mut self) -> RcVecMut + pub(crate) fn make_mut(&mut self) -> RcVecMut where T: Clone, { @@ -43,12 +43,12 @@ impl RcVec { } } - pub fn get_mut(&mut self) -> Option> { + pub(crate) fn get_mut(&mut self) -> Option> { let inner = Rc::get_mut(&mut self.inner)?; Some(RcVecMut { inner }) } - pub fn make_owned(mut self) -> RcVecBuilder + pub(crate) fn make_owned(mut self) -> RcVecBuilder where T: Clone, { @@ -62,31 +62,31 @@ impl RcVec { } impl RcVecBuilder { - pub fn new() -> Self { + pub(crate) fn new() -> Self { RcVecBuilder { inner: Vec::new() } } - pub fn with_capacity(cap: usize) -> Self { + pub(crate) fn with_capacity(cap: usize) -> Self { RcVecBuilder { inner: Vec::with_capacity(cap), } } - pub fn push(&mut self, element: T) { + pub(crate) fn push(&mut self, element: T) { self.inner.push(element); } - pub fn extend(&mut self, iter: impl IntoIterator) { + pub(crate) fn extend(&mut self, iter: impl IntoIterator) { self.inner.extend(iter); } - pub fn as_mut(&mut self) -> RcVecMut { + pub(crate) fn as_mut(&mut self) -> RcVecMut { RcVecMut { inner: &mut self.inner, } } - pub fn build(self) -> RcVec { + pub(crate) fn build(self) -> RcVec { RcVec { inner: Rc::new(self.inner), } @@ -94,19 +94,19 @@ impl RcVecBuilder { } impl<'a, T> RcVecMut<'a, T> { - pub fn push(&mut self, element: T) { + pub(crate) fn push(&mut self, element: T) { self.inner.push(element); } - pub fn extend(&mut self, iter: impl IntoIterator) { + pub(crate) fn extend(&mut self, iter: impl IntoIterator) { self.inner.extend(iter); } - pub fn pop(&mut self) -> Option { + pub(crate) fn pop(&mut self) -> Option { self.inner.pop() } - pub fn as_mut(&mut self) -> RcVecMut { + pub(crate) fn as_mut(&mut self) -> RcVecMut { RcVecMut { inner: self.inner } } } diff --git a/src/wrapper.rs b/src/wrapper.rs index f5eb826..17d7b41 100644 --- a/src/wrapper.rs +++ b/src/wrapper.rs @@ -1,11 +1,13 @@ use crate::detection::inside_proc_macro; +use crate::fallback::{self, FromStr2 as _}; #[cfg(span_locations)] use crate::location::LineColumn; -use crate::{fallback, Delimiter, Punct, Spacing, TokenTree}; +use crate::{Delimiter, Punct, Spacing, TokenTree}; use core::fmt::{self, Debug, Display}; +#[cfg(span_locations)] +use core::ops::Range; use core::ops::RangeBounds; -use core::str::FromStr; -use std::panic; +use std::ffi::CStr; #[cfg(super_unstable)] use std::path::PathBuf; @@ -39,11 +41,11 @@ fn mismatch(line: u32) -> ! { #[cfg(procmacro2_backtrace)] { let backtrace = std::backtrace::Backtrace::force_capture(); - panic!("compiler/fallback mismatch #{}\n\n{}", line, backtrace) + panic!("compiler/fallback mismatch L{}\n\n{}", line, backtrace) } #[cfg(not(procmacro2_backtrace))] { - panic!("compiler/fallback mismatch #{}", line) + panic!("compiler/fallback mismatch L{}", line) } } @@ -75,7 +77,7 @@ impl DeferredTokenStream { } impl TokenStream { - pub fn new() -> Self { + pub(crate) fn new() -> Self { if inside_proc_macro() { TokenStream::Compiler(DeferredTokenStream::new(proc_macro::TokenStream::new())) } else { @@ -83,7 +85,19 @@ impl TokenStream { } } - pub fn is_empty(&self) -> bool { + pub(crate) fn from_str_checked(src: &str) -> Result { + if inside_proc_macro() { + Ok(TokenStream::Compiler(DeferredTokenStream::new( + proc_macro::TokenStream::from_str_checked(src)?, + ))) + } else { + Ok(TokenStream::Fallback( + fallback::TokenStream::from_str_checked(src)?, + )) + } + } + + pub(crate) fn is_empty(&self) -> bool { match self { TokenStream::Compiler(tts) => tts.is_empty(), TokenStream::Fallback(tts) => tts.is_empty(), @@ -105,26 +119,6 @@ impl TokenStream { } } -impl FromStr for TokenStream { - type Err = LexError; - - fn from_str(src: &str) -> Result { - if inside_proc_macro() { - Ok(TokenStream::Compiler(DeferredTokenStream::new( - proc_macro_parse(src)?, - ))) - } else { - Ok(TokenStream::Fallback(src.parse()?)) - } - } -} - -// Work around https://github.com/rust-lang/rust/issues/58736. -fn proc_macro_parse(src: &str) -> Result { - let result = panic::catch_unwind(|| src.parse().map_err(LexError::Compiler)); - result.unwrap_or_else(|_| Err(LexError::CompilerPanic)) -} - impl Display for TokenStream { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { match self { @@ -144,7 +138,9 @@ impl From for proc_macro::TokenStream { fn from(inner: TokenStream) -> Self { match inner { TokenStream::Compiler(inner) => inner.into_token_stream(), - TokenStream::Fallback(inner) => inner.to_string().parse().unwrap(), + TokenStream::Fallback(inner) => { + proc_macro::TokenStream::from_str_unchecked(&inner.to_string()) + } } } } @@ -158,7 +154,7 @@ impl From for TokenStream { // Assumes inside_proc_macro(). fn into_compiler_token(token: TokenTree) -> proc_macro::TokenTree { match token { - TokenTree::Group(tt) => tt.inner.unwrap_nightly().into(), + TokenTree::Group(tt) => proc_macro::TokenTree::Group(tt.inner.unwrap_nightly()), TokenTree::Punct(tt) => { let spacing = match tt.spacing() { Spacing::Joint => proc_macro::Spacing::Joint, @@ -166,19 +162,21 @@ fn into_compiler_token(token: TokenTree) -> proc_macro::TokenTree { }; let mut punct = proc_macro::Punct::new(tt.as_char(), spacing); punct.set_span(tt.span().inner.unwrap_nightly()); - punct.into() + proc_macro::TokenTree::Punct(punct) } - TokenTree::Ident(tt) => tt.inner.unwrap_nightly().into(), - TokenTree::Literal(tt) => tt.inner.unwrap_nightly().into(), + TokenTree::Ident(tt) => proc_macro::TokenTree::Ident(tt.inner.unwrap_nightly()), + TokenTree::Literal(tt) => proc_macro::TokenTree::Literal(tt.inner.unwrap_nightly()), } } impl From for TokenStream { fn from(token: TokenTree) -> Self { if inside_proc_macro() { - TokenStream::Compiler(DeferredTokenStream::new(into_compiler_token(token).into())) + TokenStream::Compiler(DeferredTokenStream::new(proc_macro::TokenStream::from( + into_compiler_token(token), + ))) } else { - TokenStream::Fallback(token.into()) + TokenStream::Fallback(fallback::TokenStream::from(token)) } } } @@ -333,7 +331,9 @@ impl Iterator for TokenTreeIter { TokenTreeIter::Fallback(iter) => return iter.next(), }; Some(match token { - proc_macro::TokenTree::Group(tt) => crate::Group::_new(Group::Compiler(tt)).into(), + proc_macro::TokenTree::Group(tt) => { + TokenTree::Group(crate::Group::_new(Group::Compiler(tt))) + } proc_macro::TokenTree::Punct(tt) => { let spacing = match tt.spacing() { proc_macro::Spacing::Joint => Spacing::Joint, @@ -341,10 +341,14 @@ impl Iterator for TokenTreeIter { }; let mut o = Punct::new(tt.as_char(), spacing); o.set_span(crate::Span::_new(Span::Compiler(tt.span()))); - o.into() + TokenTree::Punct(o) + } + proc_macro::TokenTree::Ident(s) => { + TokenTree::Ident(crate::Ident::_new(Ident::Compiler(s))) + } + proc_macro::TokenTree::Literal(l) => { + TokenTree::Literal(crate::Literal::_new(Literal::Compiler(l))) } - proc_macro::TokenTree::Ident(s) => crate::Ident::_new(Ident::Compiler(s)).into(), - proc_macro::TokenTree::Literal(l) => crate::Literal::_new(Literal::Compiler(l)).into(), }) } @@ -370,14 +374,14 @@ impl SourceFile { } /// Get the path to this source file as a string. - pub fn path(&self) -> PathBuf { + pub(crate) fn path(&self) -> PathBuf { match self { SourceFile::Compiler(a) => a.path(), SourceFile::Fallback(a) => a.path(), } } - pub fn is_real(&self) -> bool { + pub(crate) fn is_real(&self) -> bool { match self { SourceFile::Compiler(a) => a.is_real(), SourceFile::Fallback(a) => a.is_real(), @@ -402,7 +406,7 @@ pub(crate) enum Span { } impl Span { - pub fn call_site() -> Self { + pub(crate) fn call_site() -> Self { if inside_proc_macro() { Span::Compiler(proc_macro::Span::call_site()) } else { @@ -410,7 +414,7 @@ impl Span { } } - pub fn mixed_site() -> Self { + pub(crate) fn mixed_site() -> Self { if inside_proc_macro() { Span::Compiler(proc_macro::Span::mixed_site()) } else { @@ -419,7 +423,7 @@ impl Span { } #[cfg(super_unstable)] - pub fn def_site() -> Self { + pub(crate) fn def_site() -> Self { if inside_proc_macro() { Span::Compiler(proc_macro::Span::def_site()) } else { @@ -427,7 +431,7 @@ impl Span { } } - pub fn resolved_at(&self, other: Span) -> Span { + pub(crate) fn resolved_at(&self, other: Span) -> Span { match (self, other) { (Span::Compiler(a), Span::Compiler(b)) => Span::Compiler(a.resolved_at(b)), (Span::Fallback(a), Span::Fallback(b)) => Span::Fallback(a.resolved_at(b)), @@ -436,7 +440,7 @@ impl Span { } } - pub fn located_at(&self, other: Span) -> Span { + pub(crate) fn located_at(&self, other: Span) -> Span { match (self, other) { (Span::Compiler(a), Span::Compiler(b)) => Span::Compiler(a.located_at(b)), (Span::Fallback(a), Span::Fallback(b)) => Span::Fallback(a.located_at(b)), @@ -445,7 +449,7 @@ impl Span { } } - pub fn unwrap(self) -> proc_macro::Span { + pub(crate) fn unwrap(self) -> proc_macro::Span { match self { Span::Compiler(s) => s, Span::Fallback(_) => panic!("proc_macro::Span is only available in procedural macros"), @@ -453,7 +457,7 @@ impl Span { } #[cfg(super_unstable)] - pub fn source_file(&self) -> SourceFile { + pub(crate) fn source_file(&self) -> SourceFile { match self { Span::Compiler(s) => SourceFile::nightly(s.source_file()), Span::Fallback(s) => SourceFile::Fallback(s.source_file()), @@ -461,22 +465,48 @@ impl Span { } #[cfg(span_locations)] - pub fn start(&self) -> LineColumn { + pub(crate) fn byte_range(&self) -> Range { match self { + #[cfg(proc_macro_span)] + Span::Compiler(s) => s.byte_range(), + #[cfg(not(proc_macro_span))] + Span::Compiler(_) => 0..0, + Span::Fallback(s) => s.byte_range(), + } + } + + #[cfg(span_locations)] + pub(crate) fn start(&self) -> LineColumn { + match self { + #[cfg(proc_macro_span)] + Span::Compiler(s) => LineColumn { + line: s.line(), + column: s.column().saturating_sub(1), + }, + #[cfg(not(proc_macro_span))] Span::Compiler(_) => LineColumn { line: 0, column: 0 }, Span::Fallback(s) => s.start(), } } #[cfg(span_locations)] - pub fn end(&self) -> LineColumn { + pub(crate) fn end(&self) -> LineColumn { match self { + #[cfg(proc_macro_span)] + Span::Compiler(s) => { + let end = s.end(); + LineColumn { + line: end.line(), + column: end.column().saturating_sub(1), + } + } + #[cfg(not(proc_macro_span))] Span::Compiler(_) => LineColumn { line: 0, column: 0 }, Span::Fallback(s) => s.end(), } } - pub fn join(&self, other: Span) -> Option { + pub(crate) fn join(&self, other: Span) -> Option { let ret = match (self, other) { #[cfg(proc_macro_span)] (Span::Compiler(a), Span::Compiler(b)) => Span::Compiler(a.join(b)?), @@ -487,7 +517,7 @@ impl Span { } #[cfg(super_unstable)] - pub fn eq(&self, other: &Span) -> bool { + pub(crate) fn eq(&self, other: &Span) -> bool { match (self, other) { (Span::Compiler(a), Span::Compiler(b)) => a.eq(b), (Span::Fallback(a), Span::Fallback(b)) => a.eq(b), @@ -495,7 +525,7 @@ impl Span { } } - pub fn source_text(&self) -> Option { + pub(crate) fn source_text(&self) -> Option { match self { #[cfg(not(no_source_text))] Span::Compiler(s) => s.source_text(), @@ -550,7 +580,7 @@ pub(crate) enum Group { } impl Group { - pub fn new(delimiter: Delimiter, stream: TokenStream) -> Self { + pub(crate) fn new(delimiter: Delimiter, stream: TokenStream) -> Self { match stream { TokenStream::Compiler(tts) => { let delimiter = match delimiter { @@ -567,7 +597,7 @@ impl Group { } } - pub fn delimiter(&self) -> Delimiter { + pub(crate) fn delimiter(&self) -> Delimiter { match self { Group::Compiler(g) => match g.delimiter() { proc_macro::Delimiter::Parenthesis => Delimiter::Parenthesis, @@ -579,35 +609,35 @@ impl Group { } } - pub fn stream(&self) -> TokenStream { + pub(crate) fn stream(&self) -> TokenStream { match self { Group::Compiler(g) => TokenStream::Compiler(DeferredTokenStream::new(g.stream())), Group::Fallback(g) => TokenStream::Fallback(g.stream()), } } - pub fn span(&self) -> Span { + pub(crate) fn span(&self) -> Span { match self { Group::Compiler(g) => Span::Compiler(g.span()), Group::Fallback(g) => Span::Fallback(g.span()), } } - pub fn span_open(&self) -> Span { + pub(crate) fn span_open(&self) -> Span { match self { Group::Compiler(g) => Span::Compiler(g.span_open()), Group::Fallback(g) => Span::Fallback(g.span_open()), } } - pub fn span_close(&self) -> Span { + pub(crate) fn span_close(&self) -> Span { match self { Group::Compiler(g) => Span::Compiler(g.span_close()), Group::Fallback(g) => Span::Fallback(g.span_close()), } } - pub fn set_span(&mut self, span: Span) { + pub(crate) fn set_span(&mut self, span: Span) { match (self, span) { (Group::Compiler(g), Span::Compiler(s)) => g.set_span(s), (Group::Fallback(g), Span::Fallback(s)) => g.set_span(s), @@ -656,37 +686,29 @@ pub(crate) enum Ident { impl Ident { #[track_caller] - pub fn new_checked(string: &str, span: Span) -> Self { + pub(crate) fn new_checked(string: &str, span: Span) -> Self { match span { Span::Compiler(s) => Ident::Compiler(proc_macro::Ident::new(string, s)), Span::Fallback(s) => Ident::Fallback(fallback::Ident::new_checked(string, s)), } } - pub fn new_unchecked(string: &str, span: fallback::Span) -> Self { - Ident::Fallback(fallback::Ident::new_unchecked(string, span)) - } - #[track_caller] - pub fn new_raw_checked(string: &str, span: Span) -> Self { + pub(crate) fn new_raw_checked(string: &str, span: Span) -> Self { match span { Span::Compiler(s) => Ident::Compiler(proc_macro::Ident::new_raw(string, s)), Span::Fallback(s) => Ident::Fallback(fallback::Ident::new_raw_checked(string, s)), } } - pub fn new_raw_unchecked(string: &str, span: fallback::Span) -> Self { - Ident::Fallback(fallback::Ident::new_raw_unchecked(string, span)) - } - - pub fn span(&self) -> Span { + pub(crate) fn span(&self) -> Span { match self { Ident::Compiler(t) => Span::Compiler(t.span()), Ident::Fallback(t) => Span::Fallback(t.span()), } } - pub fn set_span(&mut self, span: Span) { + pub(crate) fn set_span(&mut self, span: Span) { match (self, span) { (Ident::Compiler(t), Span::Compiler(s)) => t.set_span(s), (Ident::Fallback(t), Span::Fallback(s)) => t.set_span(s), @@ -703,6 +725,12 @@ impl Ident { } } +impl From for Ident { + fn from(inner: fallback::Ident) -> Self { + Ident::Fallback(inner) + } +} + impl PartialEq for Ident { fn eq(&self, other: &Ident) -> bool { match (self, other) { @@ -753,7 +781,7 @@ pub(crate) enum Literal { macro_rules! suffixed_numbers { ($($name:ident => $kind:ident,)*) => ($( - pub fn $name(n: $kind) -> Literal { + pub(crate) fn $name(n: $kind) -> Literal { if inside_proc_macro() { Literal::Compiler(proc_macro::Literal::$name(n)) } else { @@ -765,7 +793,7 @@ macro_rules! suffixed_numbers { macro_rules! unsuffixed_integers { ($($name:ident => $kind:ident,)*) => ($( - pub fn $name(n: $kind) -> Literal { + pub(crate) fn $name(n: $kind) -> Literal { if inside_proc_macro() { Literal::Compiler(proc_macro::Literal::$name(n)) } else { @@ -776,9 +804,19 @@ macro_rules! unsuffixed_integers { } impl Literal { - pub unsafe fn from_str_unchecked(repr: &str) -> Self { + pub(crate) fn from_str_checked(repr: &str) -> Result { if inside_proc_macro() { - Literal::Compiler(proc_macro::Literal::from_str(repr).expect("invalid literal")) + let literal = proc_macro::Literal::from_str_checked(repr)?; + Ok(Literal::Compiler(literal)) + } else { + let literal = fallback::Literal::from_str_checked(repr)?; + Ok(Literal::Fallback(literal)) + } + } + + pub(crate) unsafe fn from_str_unchecked(repr: &str) -> Self { + if inside_proc_macro() { + Literal::Compiler(proc_macro::Literal::from_str_unchecked(repr)) } else { Literal::Fallback(unsafe { fallback::Literal::from_str_unchecked(repr) }) } @@ -817,7 +855,7 @@ impl Literal { isize_unsuffixed => isize, } - pub fn f32_unsuffixed(f: f32) -> Literal { + pub(crate) fn f32_unsuffixed(f: f32) -> Literal { if inside_proc_macro() { Literal::Compiler(proc_macro::Literal::f32_unsuffixed(f)) } else { @@ -825,7 +863,7 @@ impl Literal { } } - pub fn f64_unsuffixed(f: f64) -> Literal { + pub(crate) fn f64_unsuffixed(f: f64) -> Literal { if inside_proc_macro() { Literal::Compiler(proc_macro::Literal::f64_unsuffixed(f)) } else { @@ -833,23 +871,42 @@ impl Literal { } } - pub fn string(t: &str) -> Literal { + pub(crate) fn string(string: &str) -> Literal { if inside_proc_macro() { - Literal::Compiler(proc_macro::Literal::string(t)) + Literal::Compiler(proc_macro::Literal::string(string)) } else { - Literal::Fallback(fallback::Literal::string(t)) + Literal::Fallback(fallback::Literal::string(string)) } } - pub fn character(t: char) -> Literal { + pub(crate) fn character(ch: char) -> Literal { if inside_proc_macro() { - Literal::Compiler(proc_macro::Literal::character(t)) + Literal::Compiler(proc_macro::Literal::character(ch)) } else { - Literal::Fallback(fallback::Literal::character(t)) + Literal::Fallback(fallback::Literal::character(ch)) } } - pub fn byte_string(bytes: &[u8]) -> Literal { + pub(crate) fn byte_character(byte: u8) -> Literal { + if inside_proc_macro() { + Literal::Compiler({ + #[cfg(not(no_literal_byte_character))] + { + proc_macro::Literal::byte_character(byte) + } + + #[cfg(no_literal_byte_character)] + { + let fallback = fallback::Literal::byte_character(byte); + proc_macro::Literal::from_str_unchecked(&fallback.repr) + } + }) + } else { + Literal::Fallback(fallback::Literal::byte_character(byte)) + } + } + + pub(crate) fn byte_string(bytes: &[u8]) -> Literal { if inside_proc_macro() { Literal::Compiler(proc_macro::Literal::byte_string(bytes)) } else { @@ -857,14 +914,33 @@ impl Literal { } } - pub fn span(&self) -> Span { + pub(crate) fn c_string(string: &CStr) -> Literal { + if inside_proc_macro() { + Literal::Compiler({ + #[cfg(not(no_literal_c_string))] + { + proc_macro::Literal::c_string(string) + } + + #[cfg(no_literal_c_string)] + { + let fallback = fallback::Literal::c_string(string); + proc_macro::Literal::from_str_unchecked(&fallback.repr) + } + }) + } else { + Literal::Fallback(fallback::Literal::c_string(string)) + } + } + + pub(crate) fn span(&self) -> Span { match self { Literal::Compiler(lit) => Span::Compiler(lit.span()), Literal::Fallback(lit) => Span::Fallback(lit.span()), } } - pub fn set_span(&mut self, span: Span) { + pub(crate) fn set_span(&mut self, span: Span) { match (self, span) { (Literal::Compiler(lit), Span::Compiler(s)) => lit.set_span(s), (Literal::Fallback(lit), Span::Fallback(s)) => lit.set_span(s), @@ -873,7 +949,7 @@ impl Literal { } } - pub fn subspan>(&self, range: R) -> Option { + pub(crate) fn subspan>(&self, range: R) -> Option { match self { #[cfg(proc_macro_span)] Literal::Compiler(lit) => lit.subspan(range).map(Span::Compiler), @@ -897,20 +973,6 @@ impl From for Literal { } } -impl FromStr for Literal { - type Err = LexError; - - fn from_str(repr: &str) -> Result { - if inside_proc_macro() { - let literal = proc_macro::Literal::from_str(repr)?; - Ok(Literal::Compiler(literal)) - } else { - let literal = fallback::Literal::from_str(repr)?; - Ok(Literal::Fallback(literal)) - } - } -} - impl Display for Literal { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { match self { @@ -928,3 +990,14 @@ impl Debug for Literal { } } } + +#[cfg(span_locations)] +pub(crate) fn invalidate_current_thread_spans() { + if inside_proc_macro() { + panic!( + "proc_macro2::extra::invalidate_current_thread_spans is not available in procedural macros" + ); + } else { + crate::fallback::invalidate_current_thread_spans(); + } +} diff --git a/tests/marker.rs b/tests/marker.rs index d08fbfc..99f64c0 100644 --- a/tests/marker.rs +++ b/tests/marker.rs @@ -21,6 +21,7 @@ macro_rules! assert_impl { $( { // Implemented for types that implement $marker. + #[allow(dead_code)] trait IsNotImplemented { fn assert_not_implemented() {} } diff --git a/tests/test.rs b/tests/test.rs index b75cd55..0d7c88d 100644 --- a/tests/test.rs +++ b/tests/test.rs @@ -1,11 +1,14 @@ #![allow( clippy::assertions_on_result_states, clippy::items_after_statements, + clippy::needless_pass_by_value, + clippy::needless_raw_string_hashes, clippy::non_ascii_literal, clippy::octal_escapes )] use proc_macro2::{Ident, Literal, Punct, Spacing, Span, TokenStream, TokenTree}; +use std::ffi::CStr; use std::iter; use std::str::{self, FromStr}; @@ -96,12 +99,22 @@ fn lifetime_invalid() { #[test] fn literal_string() { - assert_eq!(Literal::string("foo").to_string(), "\"foo\""); - assert_eq!(Literal::string("\"").to_string(), "\"\\\"\""); - assert_eq!(Literal::string("didn't").to_string(), "\"didn't\""); - assert_eq!( - Literal::string("a\00b\07c\08d\0e\0").to_string(), - "\"a\\x000b\\x007c\\08d\\0e\\0\"", + #[track_caller] + fn assert(literal: Literal, expected: &str) { + assert_eq!(literal.to_string(), expected.trim()); + } + + assert(Literal::string(""), r#" "" "#); + assert(Literal::string("aA"), r#" "aA" "#); + assert(Literal::string("\t"), r#" "\t" "#); + assert(Literal::string("❤"), r#" "❤" "#); + assert(Literal::string("'"), r#" "'" "#); + assert(Literal::string("\""), r#" "\"" "#); + assert(Literal::string("\0"), r#" "\0" "#); + assert(Literal::string("\u{1}"), r#" "\u{1}" "#); + assert( + Literal::string("a\00b\07c\08d\0e\0"), + r#" "a\x000b\x007c\08d\0e\0" "#, ); "\"\\\r\n x\"".parse::().unwrap(); @@ -132,16 +145,43 @@ fn literal_raw_string() { .unwrap_err(); } +#[test] +fn literal_byte_character() { + #[track_caller] + fn assert(literal: Literal, expected: &str) { + assert_eq!(literal.to_string(), expected.trim()); + } + + assert(Literal::byte_character(b'a'), r#" b'a' "#); + assert(Literal::byte_character(b'\0'), r#" b'\0' "#); + assert(Literal::byte_character(b'\t'), r#" b'\t' "#); + assert(Literal::byte_character(b'\n'), r#" b'\n' "#); + assert(Literal::byte_character(b'\r'), r#" b'\r' "#); + assert(Literal::byte_character(b'\''), r#" b'\'' "#); + assert(Literal::byte_character(b'\\'), r#" b'\\' "#); + assert(Literal::byte_character(b'\x1f'), r#" b'\x1F' "#); + assert(Literal::byte_character(b'"'), r#" b'"' "#); +} + #[test] fn literal_byte_string() { - assert_eq!(Literal::byte_string(b"").to_string(), "b\"\""); - assert_eq!( - Literal::byte_string(b"\0\t\n\r\"\\2\x10").to_string(), - "b\"\\0\\t\\n\\r\\\"\\\\2\\x10\"", - ); - assert_eq!( - Literal::byte_string(b"a\00b\07c\08d\0e\0").to_string(), - "b\"a\\x000b\\x007c\\08d\\0e\\0\"", + #[track_caller] + fn assert(literal: Literal, expected: &str) { + assert_eq!(literal.to_string(), expected.trim()); + } + + assert(Literal::byte_string(b""), r#" b"" "#); + assert(Literal::byte_string(b"\0"), r#" b"\0" "#); + assert(Literal::byte_string(b"\t"), r#" b"\t" "#); + assert(Literal::byte_string(b"\n"), r#" b"\n" "#); + assert(Literal::byte_string(b"\r"), r#" b"\r" "#); + assert(Literal::byte_string(b"\""), r#" b"\"" "#); + assert(Literal::byte_string(b"\\"), r#" b"\\" "#); + assert(Literal::byte_string(b"\x1f"), r#" b"\x1F" "#); + assert(Literal::byte_string(b"'"), r#" b"'" "#); + assert( + Literal::byte_string(b"a\00b\07c\08d\0e\0"), + r#" b"a\x000b\x007c\08d\0e\0" "#, ); "b\"\\\r\n x\"".parse::().unwrap(); @@ -152,6 +192,41 @@ fn literal_byte_string() { #[test] fn literal_c_string() { + #[track_caller] + fn assert(literal: Literal, expected: &str) { + assert_eq!(literal.to_string(), expected.trim()); + } + + assert(Literal::c_string(<&CStr>::default()), r#" c"" "#); + assert( + Literal::c_string(CStr::from_bytes_with_nul(b"aA\0").unwrap()), + r#" c"aA" "#, + ); + assert( + Literal::c_string(CStr::from_bytes_with_nul(b"aA\0").unwrap()), + r#" c"aA" "#, + ); + assert( + Literal::c_string(CStr::from_bytes_with_nul(b"\t\0").unwrap()), + r#" c"\t" "#, + ); + assert( + Literal::c_string(CStr::from_bytes_with_nul(b"\xE2\x9D\xA4\0").unwrap()), + r#" c"❤" "#, + ); + assert( + Literal::c_string(CStr::from_bytes_with_nul(b"'\0").unwrap()), + r#" c"'" "#, + ); + assert( + Literal::c_string(CStr::from_bytes_with_nul(b"\"\0").unwrap()), + r#" c"\"" "#, + ); + assert( + Literal::c_string(CStr::from_bytes_with_nul(b"\x7F\xFF\xFE\xCC\xB3\0").unwrap()), + r#" c"\u{7f}\xFF\xFE\u{333}" "#, + ); + let strings = r###" c"hello\x80我叫\u{1F980}" // from the RFC cr"\" @@ -188,49 +263,80 @@ fn literal_c_string() { #[test] fn literal_character() { - assert_eq!(Literal::character('x').to_string(), "'x'"); - assert_eq!(Literal::character('\'').to_string(), "'\\''"); - assert_eq!(Literal::character('"').to_string(), "'\"'"); + #[track_caller] + fn assert(literal: Literal, expected: &str) { + assert_eq!(literal.to_string(), expected.trim()); + } + + assert(Literal::character('a'), r#" 'a' "#); + assert(Literal::character('\t'), r#" '\t' "#); + assert(Literal::character('❤'), r#" '❤' "#); + assert(Literal::character('\''), r#" '\'' "#); + assert(Literal::character('"'), r#" '"' "#); + assert(Literal::character('\0'), r#" '\0' "#); + assert(Literal::character('\u{1}'), r#" '\u{1}' "#); } #[test] fn literal_integer() { - assert_eq!(Literal::u8_suffixed(10).to_string(), "10u8"); - assert_eq!(Literal::u16_suffixed(10).to_string(), "10u16"); - assert_eq!(Literal::u32_suffixed(10).to_string(), "10u32"); - assert_eq!(Literal::u64_suffixed(10).to_string(), "10u64"); - assert_eq!(Literal::u128_suffixed(10).to_string(), "10u128"); - assert_eq!(Literal::usize_suffixed(10).to_string(), "10usize"); + #[track_caller] + fn assert(literal: Literal, expected: &str) { + assert_eq!(literal.to_string(), expected); + } - assert_eq!(Literal::i8_suffixed(10).to_string(), "10i8"); - assert_eq!(Literal::i16_suffixed(10).to_string(), "10i16"); - assert_eq!(Literal::i32_suffixed(10).to_string(), "10i32"); - assert_eq!(Literal::i64_suffixed(10).to_string(), "10i64"); - assert_eq!(Literal::i128_suffixed(10).to_string(), "10i128"); - assert_eq!(Literal::isize_suffixed(10).to_string(), "10isize"); + assert(Literal::u8_suffixed(10), "10u8"); + assert(Literal::u16_suffixed(10), "10u16"); + assert(Literal::u32_suffixed(10), "10u32"); + assert(Literal::u64_suffixed(10), "10u64"); + assert(Literal::u128_suffixed(10), "10u128"); + assert(Literal::usize_suffixed(10), "10usize"); - assert_eq!(Literal::u8_unsuffixed(10).to_string(), "10"); - assert_eq!(Literal::u16_unsuffixed(10).to_string(), "10"); - assert_eq!(Literal::u32_unsuffixed(10).to_string(), "10"); - assert_eq!(Literal::u64_unsuffixed(10).to_string(), "10"); - assert_eq!(Literal::u128_unsuffixed(10).to_string(), "10"); - assert_eq!(Literal::usize_unsuffixed(10).to_string(), "10"); + assert(Literal::i8_suffixed(10), "10i8"); + assert(Literal::i16_suffixed(10), "10i16"); + assert(Literal::i32_suffixed(10), "10i32"); + assert(Literal::i64_suffixed(10), "10i64"); + assert(Literal::i128_suffixed(10), "10i128"); + assert(Literal::isize_suffixed(10), "10isize"); - assert_eq!(Literal::i8_unsuffixed(10).to_string(), "10"); - assert_eq!(Literal::i16_unsuffixed(10).to_string(), "10"); - assert_eq!(Literal::i32_unsuffixed(10).to_string(), "10"); - assert_eq!(Literal::i64_unsuffixed(10).to_string(), "10"); - assert_eq!(Literal::i128_unsuffixed(10).to_string(), "10"); - assert_eq!(Literal::isize_unsuffixed(10).to_string(), "10"); + assert(Literal::u8_unsuffixed(10), "10"); + assert(Literal::u16_unsuffixed(10), "10"); + assert(Literal::u32_unsuffixed(10), "10"); + assert(Literal::u64_unsuffixed(10), "10"); + assert(Literal::u128_unsuffixed(10), "10"); + assert(Literal::usize_unsuffixed(10), "10"); + + assert(Literal::i8_unsuffixed(10), "10"); + assert(Literal::i16_unsuffixed(10), "10"); + assert(Literal::i32_unsuffixed(10), "10"); + assert(Literal::i64_unsuffixed(10), "10"); + assert(Literal::i128_unsuffixed(10), "10"); + assert(Literal::isize_unsuffixed(10), "10"); + + assert(Literal::i32_suffixed(-10), "-10i32"); + assert(Literal::i32_unsuffixed(-10), "-10"); } #[test] fn literal_float() { - assert_eq!(Literal::f32_suffixed(10.0).to_string(), "10f32"); - assert_eq!(Literal::f64_suffixed(10.0).to_string(), "10f64"); + #[track_caller] + fn assert(literal: Literal, expected: &str) { + assert_eq!(literal.to_string(), expected); + } - assert_eq!(Literal::f32_unsuffixed(10.0).to_string(), "10.0"); - assert_eq!(Literal::f64_unsuffixed(10.0).to_string(), "10.0"); + assert(Literal::f32_suffixed(10.0), "10f32"); + assert(Literal::f32_suffixed(-10.0), "-10f32"); + assert(Literal::f64_suffixed(10.0), "10f64"); + assert(Literal::f64_suffixed(-10.0), "-10f64"); + + assert(Literal::f32_unsuffixed(10.0), "10.0"); + assert(Literal::f32_unsuffixed(-10.0), "-10.0"); + assert(Literal::f64_unsuffixed(10.0), "10.0"); + assert(Literal::f64_unsuffixed(-10.0), "-10.0"); + + assert( + Literal::f64_unsuffixed(1e100), + "10000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000.0", + ); } #[test] @@ -248,9 +354,13 @@ fn literal_suffix() { assert_eq!(token_count("1._m"), 3); assert_eq!(token_count("\"\"s"), 1); assert_eq!(token_count("r\"\"r"), 1); + assert_eq!(token_count("r#\"\"#r"), 1); assert_eq!(token_count("b\"\"b"), 1); assert_eq!(token_count("br\"\"br"), 1); - assert_eq!(token_count("r#\"\"#r"), 1); + assert_eq!(token_count("br#\"\"#br"), 1); + assert_eq!(token_count("c\"\"c"), 1); + assert_eq!(token_count("cr\"\"cr"), 1); + assert_eq!(token_count("cr#\"\"#cr"), 1); assert_eq!(token_count("'c'c"), 1); assert_eq!(token_count("b'b'b"), 1); assert_eq!(token_count("0E"), 1); @@ -378,7 +488,7 @@ fn roundtrip() { roundtrip("'a"); roundtrip("'_"); roundtrip("'static"); - roundtrip("'\\u{10__FFFF}'"); + roundtrip(r"'\u{10__FFFF}'"); roundtrip("\"\\u{10_F0FF__}foo\\u{1_0_0_0__}\""); } @@ -401,6 +511,7 @@ fn fail() { fail("\"\\\r \""); // backslash carriage return fail("'aa'aa"); fail("br##\"\"#"); + fail("cr##\"\"#"); fail("\"\\\n\u{85}\r\""); } @@ -430,7 +541,6 @@ testing 123 } #[cfg(procmacro2_semver_exempt)] -#[cfg(not(nightly))] #[test] fn default_span() { let start = Span::call_site().start(); @@ -757,3 +867,39 @@ fn byte_order_mark() { let string = "foo\u{feff}"; string.parse::().unwrap_err(); } + +#[cfg(span_locations)] +fn create_span() -> proc_macro2::Span { + let tts: TokenStream = "1".parse().unwrap(); + match tts.into_iter().next().unwrap() { + TokenTree::Literal(literal) => literal.span(), + _ => unreachable!(), + } +} + +#[cfg(span_locations)] +#[test] +fn test_invalidate_current_thread_spans() { + let actual = format!("{:#?}", create_span()); + assert_eq!(actual, "bytes(1..2)"); + let actual = format!("{:#?}", create_span()); + assert_eq!(actual, "bytes(3..4)"); + + proc_macro2::extra::invalidate_current_thread_spans(); + + let actual = format!("{:#?}", create_span()); + // Test that span offsets have been reset after the call + // to invalidate_current_thread_spans() + assert_eq!(actual, "bytes(1..2)"); +} + +#[cfg(span_locations)] +#[test] +#[should_panic(expected = "Invalid span with no related FileInfo!")] +fn test_use_span_after_invalidation() { + let span = create_span(); + + proc_macro2::extra::invalidate_current_thread_spans(); + + span.source_text(); +} diff --git a/tests/test_size.rs b/tests/test_size.rs index 46e58db..8b67915 100644 --- a/tests/test_size.rs +++ b/tests/test_size.rs @@ -1,42 +1,81 @@ +#![allow(unused_attributes)] + extern crate proc_macro; use std::mem; -#[rustversion::attr(before(1.32), ignore)] +#[rustversion::attr(before(1.64), ignore = "requires Rust 1.64+")] +#[cfg_attr(not(target_pointer_width = "64"), ignore = "only applicable to 64-bit")] +#[cfg_attr(randomize_layout, ignore = "disabled due to randomized layout")] #[test] -fn test_proc_macro_span_size() { +fn test_proc_macro_size() { assert_eq!(mem::size_of::(), 4); assert_eq!(mem::size_of::>(), 4); + assert_eq!(mem::size_of::(), 20); + assert_eq!(mem::size_of::(), 12); + assert_eq!(mem::size_of::(), 8); + assert_eq!(mem::size_of::(), 16); + assert_eq!(mem::size_of::(), 4); } -#[cfg_attr(not(all(not(wrap_proc_macro), not(span_locations))), ignore)] +#[cfg_attr(not(target_pointer_width = "64"), ignore = "only applicable to 64-bit")] +#[cfg_attr(randomize_layout, ignore = "disabled due to randomized layout")] +#[cfg_attr(wrap_proc_macro, ignore = "wrapper mode")] +#[cfg_attr(span_locations, ignore = "span locations are on")] #[test] -fn test_proc_macro2_fallback_span_size_without_locations() { +fn test_proc_macro2_fallback_size_without_locations() { assert_eq!(mem::size_of::(), 0); assert_eq!(mem::size_of::>(), 1); + assert_eq!(mem::size_of::(), 16); + assert_eq!(mem::size_of::(), 24); + assert_eq!(mem::size_of::(), 8); + assert_eq!(mem::size_of::(), 24); + assert_eq!(mem::size_of::(), 8); } -#[cfg_attr(not(all(not(wrap_proc_macro), span_locations)), ignore)] +#[cfg_attr(not(target_pointer_width = "64"), ignore = "only applicable to 64-bit")] +#[cfg_attr(randomize_layout, ignore = "disabled due to randomized layout")] +#[cfg_attr(wrap_proc_macro, ignore = "wrapper mode")] +#[cfg_attr(not(span_locations), ignore = "span locations are off")] #[test] -fn test_proc_macro2_fallback_span_size_with_locations() { +fn test_proc_macro2_fallback_size_with_locations() { assert_eq!(mem::size_of::(), 8); assert_eq!(mem::size_of::>(), 12); + assert_eq!(mem::size_of::(), 24); + assert_eq!(mem::size_of::(), 32); + assert_eq!(mem::size_of::(), 16); + assert_eq!(mem::size_of::(), 32); + assert_eq!(mem::size_of::(), 8); } -#[rustversion::attr(before(1.32), ignore)] -#[rustversion::attr( - since(1.32), - cfg_attr(not(all(wrap_proc_macro, not(span_locations))), ignore) -)] +#[rustversion::attr(before(1.71), ignore = "requires Rust 1.71+")] +#[cfg_attr(not(target_pointer_width = "64"), ignore = "only applicable to 64-bit")] +#[cfg_attr(randomize_layout, ignore = "disabled due to randomized layout")] +#[cfg_attr(not(wrap_proc_macro), ignore = "fallback mode")] +#[cfg_attr(span_locations, ignore = "span locations are on")] #[test] -fn test_proc_macro2_wrapper_span_size_without_locations() { +fn test_proc_macro2_wrapper_size_without_locations() { assert_eq!(mem::size_of::(), 4); assert_eq!(mem::size_of::>(), 8); + assert_eq!(mem::size_of::(), 24); + assert_eq!(mem::size_of::(), 24); + assert_eq!(mem::size_of::(), 12); + assert_eq!(mem::size_of::(), 24); + assert_eq!(mem::size_of::(), 32); } -#[cfg_attr(not(all(wrap_proc_macro, span_locations)), ignore)] +#[rustversion::attr(before(1.65), ignore = "requires Rust 1.65+")] +#[cfg_attr(not(target_pointer_width = "64"), ignore = "only applicable to 64-bit")] +#[cfg_attr(randomize_layout, ignore = "disabled due to randomized layout")] +#[cfg_attr(not(wrap_proc_macro), ignore = "fallback mode")] +#[cfg_attr(not(span_locations), ignore = "span locations are off")] #[test] -fn test_proc_macro2_wrapper_span_size_with_locations() { +fn test_proc_macro2_wrapper_size_with_locations() { assert_eq!(mem::size_of::(), 12); assert_eq!(mem::size_of::>(), 12); + assert_eq!(mem::size_of::(), 32); + assert_eq!(mem::size_of::(), 32); + assert_eq!(mem::size_of::(), 20); + assert_eq!(mem::size_of::(), 32); + assert_eq!(mem::size_of::(), 32); } diff --git a/tests/ui/Cargo.toml b/tests/ui/Cargo.toml new file mode 100644 index 0000000..1e65169 --- /dev/null +++ b/tests/ui/Cargo.toml @@ -0,0 +1,15 @@ +[package] +name = "proc-macro2-ui-test" +version = "0.0.0" +authors = ["David Tolnay "] +edition = "2018" +publish = false + +[[test]] +name = "compiletest" +path = "compiletest.rs" + +[dev-dependencies] +proc-macro2 = { path = "../.." } +rustversion = "1.0" +trybuild = { version = "1.0.49", features = ["diff"] } diff --git a/tests/ui/compiletest.rs b/tests/ui/compiletest.rs new file mode 100644 index 0000000..00c6d2c --- /dev/null +++ b/tests/ui/compiletest.rs @@ -0,0 +1,7 @@ +#[rustversion::attr(not(nightly), ignore = "requires nightly")] +#[rustversion::attr(nightly, cfg_attr(miri, ignore = "incompatible with miri"))] +#[test] +fn ui() { + let t = trybuild::TestCases::new(); + t.compile_fail("test-*.rs"); +} diff --git a/tests/ui/test-not-send.rs b/tests/ui/test-not-send.rs new file mode 100644 index 0000000..171be97 --- /dev/null +++ b/tests/ui/test-not-send.rs @@ -0,0 +1,6 @@ +use proc_macro2::Span; + +fn main() { + fn requires_send() {} + requires_send::(); +} diff --git a/tests/ui/test-not-send.stderr b/tests/ui/test-not-send.stderr new file mode 100644 index 0000000..ac8db47 --- /dev/null +++ b/tests/ui/test-not-send.stderr @@ -0,0 +1,50 @@ +error[E0277]: `proc_macro::Span` cannot be sent between threads safely + --> test-not-send.rs:5:21 + | +5 | requires_send::(); + | ^^^^ `proc_macro::Span` cannot be sent between threads safely + | + = help: within `Span`, the trait `Send` is not implemented for `proc_macro::Span` +note: required because it appears within the type `proc_macro2::imp::Span` + --> $WORKSPACE/src/wrapper.rs + | + | pub(crate) enum Span { + | ^^^^ +note: required because it appears within the type `Span` + --> $WORKSPACE/src/lib.rs + | + | pub struct Span { + | ^^^^ +note: required by a bound in `requires_send` + --> test-not-send.rs:4:25 + | +4 | fn requires_send() {} + | ^^^^ required by this bound in `requires_send` + +error[E0277]: `Rc<()>` cannot be sent between threads safely + --> test-not-send.rs:5:21 + | +5 | requires_send::(); + | ^^^^ `Rc<()>` cannot be sent between threads safely + | + = help: within `Span`, the trait `Send` is not implemented for `Rc<()>` +note: required because it appears within the type `PhantomData>` + --> $RUST/core/src/marker.rs + | + | pub struct PhantomData; + | ^^^^^^^^^^^ +note: required because it appears within the type `proc_macro2::marker::ProcMacroAutoTraits` + --> $WORKSPACE/src/marker.rs + | + | pub(crate) struct ProcMacroAutoTraits(PhantomData>); + | ^^^^^^^^^^^^^^^^^^^ +note: required because it appears within the type `Span` + --> $WORKSPACE/src/lib.rs + | + | pub struct Span { + | ^^^^ +note: required by a bound in `requires_send` + --> test-not-send.rs:4:25 + | +4 | fn requires_send() {} + | ^^^^ required by this bound in `requires_send`