openharmony/third_party_rust_proc-macro2
1
0
Fork 0
mirror of https://github.com/openharmony/third_party_rust_proc-macro2.git synced 2026-07-01 20:24:03 -04:00
Code Issues Packages Projects Releases Wiki Activity

升级到 1.0.92

Browse Source 
Signed-off-by: ljy9810 <longjianyin@h-partners.com>
This commit is contained in:
ljy9810
2025-08-29 12:20:25 +08:00
parent bdd95c0a17
commit 4377c8e58b
31 changed files with 1389 additions and 600 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.cargo-ok
-1
View File
@@ -1 +0,0 @@
ok
.cargo_vcs_info.json
-6
View File
@@ -1,6 +0,0 @@
{
"git": {
"sha1": "ea778eb80729a01094b033c45633122052d96fd4"
},
"path_in_vcs": ""
}
.github/workflows/ci.yml
+48 -2
View File
@@ -24,7 +24,7 @@ jobs:
strategy:
fail-fast: false
matrix:
rust: [1.56.0, stable, beta]
rust: [1.63.0, stable, beta]
timeout-minutes: 45
steps:
- uses: actions/checkout@v4
@@ -56,7 +56,7 @@ jobs:
with:
components: rust-src
- name: Enable type layout randomization
run: echo RUSTFLAGS=${RUSTFLAGS}\ -Zrandomize-layout >> $GITHUB_ENV
run: echo RUSTFLAGS=${RUSTFLAGS}\ -Zrandomize-layout\ --cfg=randomize_layout >> $GITHUB_ENV
- run: cargo check
env:
RUSTFLAGS: --cfg procmacro2_nightly_testing ${{env.RUSTFLAGS}}
@@ -77,6 +77,51 @@ jobs:
run: cargo test
env:
RUSTFLAGS: -Z allow-features= --cfg procmacro2_backtrace ${{env.RUSTFLAGS}}
- uses: actions/upload-artifact@v4
if: always()
with:
name: Cargo.lock
path: Cargo.lock
continue-on-error: true
layout:
name: Layout
needs: pre_ci
if: needs.pre_ci.outputs.continue
runs-on: ubuntu-latest
timeout-minutes: 45
steps:
- uses: actions/checkout@v4
- uses: dtolnay/rust-toolchain@nightly
with:
components: rust-src
- run: cargo test --test test_size
- run: cargo test --test test_size --features span-locations
- run: cargo test --test test_size --no-default-features
- run: cargo test --test test_size --no-default-features --features span-locations
msrv:
name: Rust 1.56.0
needs: pre_ci
if: needs.pre_ci.outputs.continue
runs-on: ubuntu-latest
timeout-minutes: 45
steps:
- uses: actions/checkout@v4
- uses: dtolnay/rust-toolchain@1.56.0
with:
components: rust-src
- run: cargo check
- run: cargo check --no-default-features
- run: cargo check --features span-locations
- name: RUSTFLAGS='--cfg procmacro2_semver_exempt' cargo check
run: cargo check
env:
RUSTFLAGS: --cfg procmacro2_semver_exempt ${{env.RUSTFLAGS}}
- name: RUSTFLAGS='--cfg procmacro2_semver_exempt' cargo check --no-default-features
run: cargo check --no-default-features
env:
RUSTFLAGS: --cfg procmacro2_semver_exempt ${{env.RUSTFLAGS}}
minimal:
name: Minimal versions
@@ -175,6 +220,7 @@ jobs:
timeout-minutes: 45
steps:
- uses: actions/checkout@v4
- uses: dtolnay/rust-toolchain@stable
- uses: dtolnay/install@cargo-outdated
- run: cargo outdated --workspace --exit-code 1
- run: cargo outdated --manifest-path fuzz/Cargo.toml --exit-code 1
BUILD.gn
+1 -1
View File
@@ -20,7 +20,7 @@ ohos_cargo_crate("lib") {
sources = [ "src/lib.rs" ]
edition = "2021"
cargo_pkg_version = "1.0.76"
cargo_pkg_version = "1.0.92"
cargo_pkg_authors =
"David Tolnay <dtolnay@gmail.com>, Alex Crichton <alex@alexcrichton.com>"
cargo_pkg_name = "proc-macro2"
Cargo.toml
+44 -49
View File
@@ -1,67 +1,62 @@
# THIS FILE IS AUTOMATICALLY GENERATED BY CARGO
#
# When uploading crates to the registry Cargo will automatically
# "normalize" Cargo.toml files for maximal compatibility
# with all versions of Cargo and also rewrite `path` dependencies
# to registry (e.g., crates.io) dependencies.
#
# If you are reading this file be aware that the original Cargo.toml
# will likely look very different (and much more reasonable).
# See Cargo.toml.orig for the original contents.
[package]
edition = "2021"
rust-version = "1.56"
name = "proc-macro2"
version = "1.0.76"
authors = [
"David Tolnay <dtolnay@gmail.com>",
"Alex Crichton <alex@alexcrichton.com>",
]
version = "1.0.92"
authors = ["David Tolnay <dtolnay@gmail.com>", "Alex Crichton <alex@alexcrichton.com>"]
autobenches = false
categories = ["development-tools::procedural-macro-helpers"]
description = "A substitute implementation of the compiler's `proc_macro` API to decouple token-based libraries from the procedural macro use case."
documentation = "https://docs.rs/proc-macro2"
readme = "README.md"
keywords = [
"macros",
"syn",
]
categories = ["development-tools::procedural-macro-helpers"]
edition = "2021"
keywords = ["macros", "syn"]
license = "MIT OR Apache-2.0"
repository = "https://github.com/dtolnay/proc-macro2"
rust-version = "1.56"
[package.metadata.docs.rs]
rustc-args = [
"--cfg",
"procmacro2_semver_exempt",
]
rustdoc-args = [
"--cfg",
"procmacro2_semver_exempt",
"--cfg",
"doc_cfg",
"--generate-link-to-definition",
]
rustc-args = ["--cfg", "procmacro2_semver_exempt"]
rustdoc-args = ["--cfg", "procmacro2_semver_exempt", "--generate-link-to-definition"]
targets = ["x86_64-unknown-linux-gnu"]
[package.metadata.playground]
features = ["span-locations"]
[dependencies]
unicode-ident = "1.0"
[dev-dependencies]
flate2 = "1.0"
quote = { version = "1.0", default-features = false }
rayon = "1.0"
rustversion = "1"
tar = "0.4"
[features]
proc-macro = []
default = ["proc-macro"]
# Expose methods Span::start and Span::end which give the line/column location
# of a token.
span-locations = []
# This feature no longer means anything.
nightly = []
[lib]
doc-scrape-examples = false
[dependencies.unicode-ident]
version = "1.0"
[workspace]
members = ["benches/bench-libproc-macro", "tests/ui"]
[dev-dependencies.quote]
version = "1.0"
default_features = false
[dev-dependencies.rustversion]
version = "1"
[features]
default = ["proc-macro"]
nightly = []
proc-macro = []
span-locations = []
[patch.crates-io]
# Our doc tests depend on quote which depends on proc-macro2. Without this line,
# the proc-macro2 dependency of quote would be the released version of
# proc-macro2. Quote would implement its traits for types from that proc-macro2,
# meaning impls would be missing when tested against types from the local
# proc-macro2.
#
# GitHub Actions builds that are in progress at the time that you publish may
# spuriously fail. This is because they'll be building a local proc-macro2 which
# carries the second-most-recent version number, pulling in quote which resolves
# to a dependency on the just-published most recent version number. Thus the
# patch will fail to apply because the version numbers are different.
proc-macro2 = { path = "." }
Cargo.toml.orig
Generated
-59
View File
@@ -1,59 +0,0 @@
[package]
name = "proc-macro2"
version = "1.0.76"
authors = ["David Tolnay <dtolnay@gmail.com>", "Alex Crichton <alex@alexcrichton.com>"]
autobenches = false
categories = ["development-tools::procedural-macro-helpers"]
description = "A substitute implementation of the compiler's `proc_macro` API to decouple token-based libraries from the procedural macro use case."
documentation = "https://docs.rs/proc-macro2"
edition = "2021"
keywords = ["macros", "syn"]
license = "MIT OR Apache-2.0"
repository = "https://github.com/dtolnay/proc-macro2"
rust-version = "1.56"
[package.metadata.docs.rs]
rustc-args = ["--cfg", "procmacro2_semver_exempt"]
rustdoc-args = ["--cfg", "procmacro2_semver_exempt", "--cfg", "doc_cfg", "--generate-link-to-definition"]
targets = ["x86_64-unknown-linux-gnu"]
[package.metadata.playground]
features = ["span-locations"]
[dependencies]
unicode-ident = "1.0"
[dev-dependencies]
quote = { version = "1.0", default_features = false }
rustversion = "1"
[features]
proc-macro = []
default = ["proc-macro"]
# Expose methods Span::start and Span::end which give the line/column location
# of a token.
span-locations = []
# This feature no longer means anything.
nightly = []
[lib]
doc-scrape-examples = false
[workspace]
members = ["benches/bench-libproc-macro", "tests/ui"]
[patch.crates-io]
# Our doc tests depend on quote which depends on proc-macro2. Without this line,
# the proc-macro2 dependency of quote would be the released version of
# proc-macro2. Quote would implement its traits for types from that proc-macro2,
# meaning impls would be missing when tested against types from the local
# proc-macro2.
#
# GitHub Actions builds that are in progress at the time that you publish may
# spuriously fail. This is because they'll be building a local proc-macro2 which
# carries the second-most-recent version number, pulling in quote which resolves
# to a dependency on the just-published most recent version number. Thus the
# patch will fail to apply because the version numbers are different.
proc-macro2 = { path = "." }
README.OpenSource
+1 -1
View File
@@ -3,7 +3,7 @@
"Name": "proc-macro2",
"License": "Apache License V2.0, MIT",
"License File": "LICENSE-APACHE, LICENSE-MIT",
"Version Number": "1.0.76",
"Version Number": "1.0.92",
"Owner": "fangting12@huawei.com",
"Upstream URL": "https://github.com/dtolnay/proc-macro2",
"Description": "A Rust library that provides support for error handling in procedural macros."
benches/bench-libproc-macro/Cargo.toml
+14
View File
@@ -0,0 +1,14 @@
[package]
name = "bench-libproc-macro"
version = "0.0.0"
authors = ["David Tolnay <dtolnay@gmail.com>"]
edition = "2018"
publish = false
[lib]
path = "lib.rs"
proc-macro = true
[[bin]]
name = "bench-libproc-macro"
path = "main.rs"
benches/bench-libproc-macro/README.md
+10
View File
@@ -0,0 +1,10 @@
Example output:
```console
$ cargo check --release
Compiling bench-libproc-macro v0.0.0
STRING: 37 millis
TOKENSTREAM: 276 millis
Finished release [optimized] target(s) in 1.16s
```
benches/bench-libproc-macro/lib.rs
+49
View File
@@ -0,0 +1,49 @@
extern crate proc_macro;
use proc_macro::{Ident, Punct, Spacing, Span, TokenStream, TokenTree};
use std::iter::once;
use std::time::Instant;
const N: u32 = 20000;
#[proc_macro]
pub fn bench(_input: TokenStream) -> TokenStream {
let start = Instant::now();
let mut string = String::new();
for _ in 0..N {
string += "core";
string += ":";
string += ":";
string += "option";
string += ":";
string += ":";
string += "Option";
string += ":";
string += ":";
string += "None";
string += ",";
}
string.parse::<TokenStream>().unwrap();
eprintln!("STRING: {} millis", start.elapsed().as_millis());
let start = Instant::now();
let span = Span::call_site();
let mut tokens = TokenStream::new();
for _ in 0..N {
// Similar to what is emitted by quote.
tokens.extend(once(TokenTree::Ident(Ident::new("core", span))));
tokens.extend(once(TokenTree::Punct(Punct::new(':', Spacing::Joint))));
tokens.extend(once(TokenTree::Punct(Punct::new(':', Spacing::Alone))));
tokens.extend(once(TokenTree::Ident(Ident::new("option", span))));
tokens.extend(once(TokenTree::Punct(Punct::new(':', Spacing::Joint))));
tokens.extend(once(TokenTree::Punct(Punct::new(':', Spacing::Alone))));
tokens.extend(once(TokenTree::Ident(Ident::new("Option", span))));
tokens.extend(once(TokenTree::Punct(Punct::new(':', Spacing::Joint))));
tokens.extend(once(TokenTree::Punct(Punct::new(':', Spacing::Alone))));
tokens.extend(once(TokenTree::Ident(Ident::new("None", span))));
tokens.extend(once(TokenTree::Punct(Punct::new(',', Spacing::Joint))));
}
eprintln!("TOKENSTREAM: {} millis", start.elapsed().as_millis());
TokenStream::new()
}
benches/bench-libproc-macro/main.rs
+3
View File
@@ -0,0 +1,3 @@
bench_libproc_macro::bench!();
fn main() {}
build.rs
+80 -47
View File
@@ -1,49 +1,34 @@
// rustc-cfg emitted by the build script:
//
// "wrap_proc_macro"
// Wrap types from libproc_macro rather than polyfilling the whole API.
// Enabled on rustc 1.29+ as long as procmacro2_semver_exempt is not set,
// because we can't emulate the unstable API without emulating everything
// else. Also enabled unconditionally on nightly, in which case the
// procmacro2_semver_exempt surface area is implemented by using the
// nightly-only proc_macro API.
//
// "hygiene"
// Enable Span::mixed_site() and non-dummy behavior of Span::resolved_at
// and Span::located_at. Enabled on Rust 1.45+.
//
// "proc_macro_span"
// Enable non-dummy behavior of Span::start and Span::end methods which
// requires an unstable compiler feature. Enabled when building with
// nightly, unless `-Z allow-feature` in RUSTFLAGS disallows unstable
// features.
//
// "super_unstable"
// Implement the semver exempt API in terms of the nightly-only proc_macro
// API. Enabled when using procmacro2_semver_exempt on a nightly compiler.
//
// "span_locations"
// Provide methods Span::start and Span::end which give the line/column
// location of a token. Enabled by procmacro2_semver_exempt or the
// "span-locations" Cargo cfg. This is behind a cfg because tracking
// location inside spans is a performance hit.
//
// "is_available"
// Use proc_macro::is_available() to detect if the proc macro API is
// available or needs to be polyfilled instead of trying to use the proc
// macro API and catching a panic if it isn't available. Enabled on Rust
// 1.57+.
#![allow(unknown_lints)]
#![allow(unexpected_cfgs)]
use std::env;
use std::ffi::OsString;
use std::fs;
use std::io::ErrorKind;
use std::iter;
use std::path::Path;
use std::process::{self, Command, Stdio};
use std::str;
use std::u32;
fn main() {
let rustc = rustc_minor_version().unwrap_or(u32::MAX);
if rustc >= 80 {
println!("cargo:rustc-check-cfg=cfg(fuzzing)");
println!("cargo:rustc-check-cfg=cfg(no_is_available)");
println!("cargo:rustc-check-cfg=cfg(no_literal_byte_character)");
println!("cargo:rustc-check-cfg=cfg(no_literal_c_string)");
println!("cargo:rustc-check-cfg=cfg(no_source_text)");
println!("cargo:rustc-check-cfg=cfg(proc_macro_span)");
println!("cargo:rustc-check-cfg=cfg(procmacro2_backtrace)");
println!("cargo:rustc-check-cfg=cfg(procmacro2_nightly_testing)");
println!("cargo:rustc-check-cfg=cfg(procmacro2_semver_exempt)");
println!("cargo:rustc-check-cfg=cfg(randomize_layout)");
println!("cargo:rustc-check-cfg=cfg(span_locations)");
println!("cargo:rustc-check-cfg=cfg(super_unstable)");
println!("cargo:rustc-check-cfg=cfg(wrap_proc_macro)");
}
let docs_rs = env::var_os("DOCS_RS").is_some();
let semver_exempt = cfg!(procmacro2_semver_exempt) || docs_rs;
if semver_exempt {
@@ -52,17 +37,32 @@ fn main() {
}
if semver_exempt || cfg!(feature = "span-locations") {
// Provide methods Span::start and Span::end which give the line/column
// location of a token. This is behind a cfg because tracking location
// inside spans is a performance hit.
println!("cargo:rustc-cfg=span_locations");
}
if rustc < 57 {
// Do not use proc_macro::is_available() to detect whether the proc
// macro API is available vs needs to be polyfilled. Instead, use the
// proc macro API unconditionally and catch the panic that occurs if it
// isn't available.
println!("cargo:rustc-cfg=no_is_available");
}
if rustc < 66 {
// Do not call libproc_macro's Span::source_text. Always return None.
println!("cargo:rustc-cfg=no_source_text");
}
if rustc < 79 {
// Do not call Literal::byte_character nor Literal::c_string. They can
// be emulated by way of Literal::from_str.
println!("cargo:rustc-cfg=no_literal_byte_character");
println!("cargo:rustc-cfg=no_literal_c_string");
}
if !cfg!(feature = "proc-macro") {
println!("cargo:rerun-if-changed=build.rs");
return;
@@ -106,14 +106,26 @@ fn main() {
}
if proc_macro_span || !semver_exempt {
// Wrap types from libproc_macro rather than polyfilling the whole API.
// Enabled as long as procmacro2_semver_exempt is not set, because we
// can't emulate the unstable API without emulating everything else.
// Also enabled unconditionally on nightly, in which case the
// procmacro2_semver_exempt surface area is implemented by using the
// nightly-only proc_macro API.
println!("cargo:rustc-cfg=wrap_proc_macro");
}
if proc_macro_span {
// Enable non-dummy behavior of Span::start and Span::end methods which
// requires an unstable compiler feature. Enabled when building with
// nightly, unless `-Z allow-feature` in RUSTFLAGS disallows unstable
// features.
println!("cargo:rustc-cfg=proc_macro_span");
}
if semver_exempt && proc_macro_span {
// Implement the semver exempt API in terms of the nightly-only
// proc_macro API.
println!("cargo:rustc-cfg=super_unstable");
}
@@ -136,17 +148,25 @@ fn compile_probe(rustc_bootstrap: bool) -> bool {
let rustc = cargo_env_var("RUSTC");
let out_dir = cargo_env_var("OUT_DIR");
let out_subdir = Path::new(&out_dir).join("probe");
let probefile = Path::new("build").join("probe.rs");
// Make sure to pick up Cargo rustc configuration.
let mut cmd = if let Some(wrapper) = env::var_os("RUSTC_WRAPPER") {
let mut cmd = Command::new(wrapper);
// The wrapper's first argument is supposed to be the path to rustc.
cmd.arg(rustc);
cmd
} else {
Command::new(rustc)
};
if let Err(err) = fs::create_dir(&out_subdir) {
if err.kind() != ErrorKind::AlreadyExists {
eprintln!("Failed to create {}: {}", out_subdir.display(), err);
process::exit(1);
}
}
let rustc_wrapper = env::var_os("RUSTC_WRAPPER").filter(|wrapper| !wrapper.is_empty());
let rustc_workspace_wrapper =
env::var_os("RUSTC_WORKSPACE_WRAPPER").filter(|wrapper| !wrapper.is_empty());
let mut rustc = rustc_wrapper
.into_iter()
.chain(rustc_workspace_wrapper)
.chain(iter::once(rustc));
let mut cmd = Command::new(rustc.next().unwrap());
cmd.args(rustc);
if !rustc_bootstrap {
cmd.env_remove("RUSTC_BOOTSTRAP");
@@ -156,9 +176,10 @@ fn compile_probe(rustc_bootstrap: bool) -> bool {
.arg("--edition=2021")
.arg("--crate-name=proc_macro2")
.arg("--crate-type=lib")
.arg("--cap-lints=allow")
.arg("--emit=dep-info,metadata")
.arg("--out-dir")
.arg(out_dir)
.arg(&out_subdir)
.arg(probefile);
if let Some(target) = env::var_os("TARGET") {
@@ -174,10 +195,22 @@ fn compile_probe(rustc_bootstrap: bool) -> bool {
}
}
match cmd.status() {
let success = match cmd.status() {
Ok(status) => status.success(),
Err(_) => false,
};
// Clean up to avoid leaving nondeterministic absolute paths in the dep-info
// file in OUT_DIR, which causes nonreproducible builds in build systems
// that treat the entire OUT_DIR as an artifact.
if let Err(err) = fs::remove_dir_all(&out_subdir) {
if err.kind() != ErrorKind::NotFound {
eprintln!("Failed to clean up {}: {}", out_subdir.display(), err);
process::exit(1);
}
}
success
}
fn rustc_minor_version() -> Option<u32> {
build/probe.rs
+21 -1
View File
@@ -6,9 +6,29 @@
extern crate proc_macro;
use core::ops::RangeBounds;
use core::ops::{Range, RangeBounds};
use proc_macro::{Literal, Span};
pub fn byte_range(this: &Span) -> Range<usize> {
this.byte_range()
}
pub fn start(this: &Span) -> Span {
this.start()
}
pub fn end(this: &Span) -> Span {
this.end()
}
pub fn line(this: &Span) -> usize {
this.line()
}
pub fn column(this: &Span) -> usize {
this.column()
}
pub fn join(this: &Span, other: Span) -> Option<Span> {
this.join(other)
}
fuzz/.gitignore
+8
View File
@@ -0,0 +1,8 @@
artifacts/
corpus/
coverage/
hfuzz_target/
hfuzz_workspace/
in/
out/
target/
fuzz/Cargo.toml
+30
View File
@@ -0,0 +1,30 @@
[package]
name = "proc-macro2-fuzz"
version = "0.0.0"
authors = ["David Tolnay <dtolnay@gmail.com>"]
edition = "2021"
publish = false
[package.metadata]
cargo-fuzz = true
[dependencies]
afl = { version = "0.15", optional = true }
honggfuzz = { version = "0.5", optional = true }
libfuzzer-sys = { version = "0.4.7", optional = true }
proc-macro2 = { path = "..", default-features = false }
[features]
default = ["libfuzzer"]
afl = ["dep:afl"]
honggfuzz = ["dep:honggfuzz"]
libfuzzer = ["dep:libfuzzer-sys"]
span-locations = ["proc-macro2/span-locations"]
[[bin]]
name = "parse_token_stream"
path = "fuzz_targets/parse_token_stream.rs"
test = false
doc = false
[workspace]
fuzz/fuzz_targets/parse_token_stream.rs
+67
View File
@@ -0,0 +1,67 @@
// libfuzzer:
//
// cargo install cargo-fuzz
// cargo fuzz run parse_token_stream -j $(nproc) -- -max_len=200 -timeout=1
//
// afl++:
//
// cargo install cargo-afl
// cargo afl build --no-default-features --features afl --release
// cargo afl fuzz -i in -o out target/release/parse_token_stream
//
// honggfuzz:
//
// cargo install honggfuzz
// cargo hfuzz build --no-default-features --features honggfuzz
// HFUZZ_RUN_ARGS="--threads $(nproc) --max_file_size 200 --timeout 1" cargo hfuzz run parse_token_stream
#![cfg_attr(feature = "libfuzzer", no_main)]
use std::str;
#[cfg(not(any(
all(
feature = "libfuzzer",
not(feature = "afl"),
not(feature = "honggfuzz")
),
all(
not(feature = "libfuzzer"),
feature = "afl",
not(feature = "honggfuzz")
),
all(
not(feature = "libfuzzer"),
not(feature = "afl"),
feature = "honggfuzz"
),
)))]
fn main() {
compile_error! {
r#"exactly one of feature="libfuzzer" or feature="afl" or feature="honggfuzz" must be enabled"#
}
}
#[cfg(feature = "libfuzzer")]
libfuzzer_sys::fuzz_target!(|bytes: &[u8]| do_fuzz(bytes));
#[cfg(feature = "afl")]
fn main() {
let hook = true; // turn panic into crashes
afl::fuzz(hook, do_fuzz);
}
#[cfg(feature = "honggfuzz")]
fn main() {
loop {
honggfuzz::fuzz(do_fuzz);
}
}
fn do_fuzz(bytes: &[u8]) {
let ..=199 = bytes.len() else { return };
let Ok(string) = str::from_utf8(bytes) else {
return;
};
let _ = string.parse::<proc_macro2::TokenStream>();
}
src/extra.rs
+71 -4
View File
@@ -3,18 +3,85 @@
use crate::fallback;
use crate::imp;
use crate::marker::Marker;
use crate::marker::{ProcMacroAutoTraits, MARKER};
use crate::Span;
use core::fmt::{self, Debug};
/// Invalidate any `proc_macro2::Span` that exist on the current thread.
///
/// The implementation of `Span` uses thread-local data structures and this
/// function clears them. Calling any method on a `Span` on the current thread
/// created prior to the invalidation will return incorrect values or crash.
///
/// This function is useful for programs that process more than 2<sup>32</sup>
/// bytes of Rust source code on the same thread. Just like rustc, proc-macro2
/// uses 32-bit source locations, and these wrap around when the total source
/// code processed by the same thread exceeds 2<sup>32</sup> bytes (4
/// gigabytes). After a wraparound, `Span` methods such as `source_text()` can
/// return wrong data.
///
/// # Example
///
/// As of late 2023, there is 200 GB of Rust code published on crates.io.
/// Looking at just the newest version of every crate, it is 16 GB of code. So a
/// workload that involves parsing it all would overflow a 32-bit source
/// location unless spans are being invalidated.
///
/// ```
/// use flate2::read::GzDecoder;
/// use std::ffi::OsStr;
/// use std::io::{BufReader, Read};
/// use std::str::FromStr;
/// use tar::Archive;
///
/// rayon::scope(|s| {
/// for krate in every_version_of_every_crate() {
/// s.spawn(move |_| {
/// proc_macro2::extra::invalidate_current_thread_spans();
///
/// let reader = BufReader::new(krate);
/// let tar = GzDecoder::new(reader);
/// let mut archive = Archive::new(tar);
/// for entry in archive.entries().unwrap() {
/// let mut entry = entry.unwrap();
/// let path = entry.path().unwrap();
/// if path.extension() != Some(OsStr::new("rs")) {
/// continue;
/// }
/// let mut content = String::new();
/// entry.read_to_string(&mut content).unwrap();
/// match proc_macro2::TokenStream::from_str(&content) {
/// Ok(tokens) => {/* ... */},
/// Err(_) => continue,
/// }
/// }
/// });
/// }
/// });
/// #
/// # fn every_version_of_every_crate() -> Vec<std::fs::File> {
/// # Vec::new()
/// # }
/// ```
///
/// # Panics
///
/// This function is not applicable to and will panic if called from a
/// procedural macro.
#[cfg(span_locations)]
#[cfg_attr(docsrs, doc(cfg(feature = "span-locations")))]
pub fn invalidate_current_thread_spans() {
crate::imp::invalidate_current_thread_spans();
}
/// An object that holds a [`Group`]'s `span_open()` and `span_close()` together
/// (in a more compact representation than holding those 2 spans individually.
/// in a more compact representation than holding those 2 spans individually.
///
/// [`Group`]: crate::Group
#[derive(Copy, Clone)]
pub struct DelimSpan {
inner: DelimSpanEnum,
_marker: Marker,
_marker: ProcMacroAutoTraits,
}
#[derive(Copy, Clone)]
@@ -45,7 +112,7 @@ impl DelimSpan {
DelimSpan {
inner,
_marker: Marker,
_marker: MARKER,
}
}
src/fallback.rs
+290 -167
View File
@@ -1,3 +1,5 @@
#[cfg(wrap_proc_macro)]
use crate::imp;
#[cfg(span_locations)]
use crate::location::LineColumn;
use crate::parse::{self, Cursor};
@@ -11,9 +13,17 @@ use core::cell::RefCell;
use core::cmp;
use core::fmt::{self, Debug, Display, Write};
use core::mem::ManuallyDrop;
#[cfg(span_locations)]
use core::ops::Range;
use core::ops::RangeBounds;
use core::ptr;
use core::str;
#[cfg(feature = "proc-macro")]
use core::str::FromStr;
use std::ffi::CStr;
#[cfg(wrap_proc_macro)]
use std::panic;
#[cfg(procmacro2_semver_exempt)]
use std::path::PathBuf;
/// Force use of proc-macro2's fallback implementation of the API for now, even
@@ -53,13 +63,31 @@ impl LexError {
}
impl TokenStream {
pub fn new() -> Self {
pub(crate) fn new() -> Self {
TokenStream {
inner: RcVecBuilder::new().build(),
}
}
pub fn is_empty(&self) -> bool {
pub(crate) fn from_str_checked(src: &str) -> Result<Self, LexError> {
// Create a dummy file & add it to the source map
let mut cursor = get_cursor(src);
// Strip a byte order mark if present
const BYTE_ORDER_MARK: &str = "\u{feff}";
if cursor.starts_with(BYTE_ORDER_MARK) {
cursor = cursor.advance(BYTE_ORDER_MARK.len());
}
parse::token_stream(cursor)
}
#[cfg(feature = "proc-macro")]
pub(crate) fn from_str_unchecked(src: &str) -> Self {
Self::from_str_checked(src).unwrap()
}
pub(crate) fn is_empty(&self) -> bool {
self.inner.len() == 0
}
@@ -121,23 +149,23 @@ pub(crate) struct TokenStreamBuilder {
}
impl TokenStreamBuilder {
pub fn new() -> Self {
pub(crate) fn new() -> Self {
TokenStreamBuilder {
inner: RcVecBuilder::new(),
}
}
pub fn with_capacity(cap: usize) -> Self {
pub(crate) fn with_capacity(cap: usize) -> Self {
TokenStreamBuilder {
inner: RcVecBuilder::with_capacity(cap),
}
}
pub fn push_token_from_parser(&mut self, tt: TokenTree) {
pub(crate) fn push_token_from_parser(&mut self, tt: TokenTree) {
self.inner.push(tt);
}
pub fn build(self) -> TokenStream {
pub(crate) fn build(self) -> TokenStream {
TokenStream {
inner: self.inner.build(),
}
@@ -151,9 +179,9 @@ fn get_cursor(src: &str) -> Cursor {
// Create a dummy file & add it to the source map
#[cfg(not(fuzzing))]
SOURCE_MAP.with(|cm| {
let mut cm = cm.borrow_mut();
let span = cm.add_file(src);
SOURCE_MAP.with(|sm| {
let mut sm = sm.borrow_mut();
let span = sm.add_file(src);
Cursor {
rest: src,
off: span.lo,
@@ -166,23 +194,6 @@ fn get_cursor(src: &str) -> Cursor {
Cursor { rest: src }
}
impl FromStr for TokenStream {
type Err = LexError;
fn from_str(src: &str) -> Result<TokenStream, LexError> {
// Create a dummy file & add it to the source map
let mut cursor = get_cursor(src);
// Strip a byte order mark if present
const BYTE_ORDER_MARK: &str = "\u{feff}";
if cursor.starts_with(BYTE_ORDER_MARK) {
cursor = cursor.advance(BYTE_ORDER_MARK.len());
}
parse::token_stream(cursor)
}
}
impl Display for LexError {
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
f.write_str("cannot parse string into token stream")
@@ -222,20 +233,14 @@ impl Debug for TokenStream {
#[cfg(feature = "proc-macro")]
impl From<proc_macro::TokenStream> for TokenStream {
fn from(inner: proc_macro::TokenStream) -> Self {
inner
.to_string()
.parse()
.expect("compiler token stream parse failed")
TokenStream::from_str_unchecked(&inner.to_string())
}
}
#[cfg(feature = "proc-macro")]
impl From<TokenStream> for proc_macro::TokenStream {
fn from(inner: TokenStream) -> Self {
inner
.to_string()
.parse()
.expect("failed to parse to compiler tokens")
proc_macro::TokenStream::from_str_unchecked(&inner.to_string())
}
}
@@ -295,22 +300,25 @@ impl IntoIterator for TokenStream {
}
}
#[cfg(procmacro2_semver_exempt)]
#[derive(Clone, PartialEq, Eq)]
pub(crate) struct SourceFile {
path: PathBuf,
}
#[cfg(procmacro2_semver_exempt)]
impl SourceFile {
/// Get the path to this source file as a string.
pub fn path(&self) -> PathBuf {
pub(crate) fn path(&self) -> PathBuf {
self.path.clone()
}
pub fn is_real(&self) -> bool {
pub(crate) fn is_real(&self) -> bool {
false
}
}
#[cfg(procmacro2_semver_exempt)]
impl Debug for SourceFile {
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
f.debug_struct("SourceFile")
@@ -334,6 +342,12 @@ thread_local! {
});
}
#[cfg(span_locations)]
pub(crate) fn invalidate_current_thread_spans() {
#[cfg(not(fuzzing))]
SOURCE_MAP.with(|sm| sm.borrow_mut().files.truncate(1));
}
#[cfg(all(span_locations, not(fuzzing)))]
struct FileInfo {
source_text: String,
@@ -366,7 +380,7 @@ impl FileInfo {
span.lo >= self.span.lo && span.hi <= self.span.hi
}
fn source_text(&mut self, span: Span) -> String {
fn byte_range(&mut self, span: Span) -> Range<usize> {
let lo_char = (span.lo - self.span.lo) as usize;
// Look up offset of the largest already-computed char index that is
@@ -395,11 +409,15 @@ impl FileInfo {
let trunc_lo = &self.source_text[lo_byte..];
let char_len = (span.hi - span.lo) as usize;
let source_text = match trunc_lo.char_indices().nth(char_len) {
Some((offset, _ch)) => &trunc_lo[..offset],
None => trunc_lo,
};
source_text.to_owned()
lo_byte..match trunc_lo.char_indices().nth(char_len) {
Some((offset, _ch)) => lo_byte + offset,
None => self.source_text.len(),
}
}
fn source_text(&mut self, span: Span) -> String {
let byte_range = self.byte_range(span);
self.source_text[byte_range].to_owned()
}
}
@@ -497,83 +515,98 @@ pub(crate) struct Span {
impl Span {
#[cfg(not(span_locations))]
pub fn call_site() -> Self {
pub(crate) fn call_site() -> Self {
Span {}
}
#[cfg(span_locations)]
pub fn call_site() -> Self {
pub(crate) fn call_site() -> Self {
Span { lo: 0, hi: 0 }
}
pub fn mixed_site() -> Self {
pub(crate) fn mixed_site() -> Self {
Span::call_site()
}
#[cfg(procmacro2_semver_exempt)]
pub fn def_site() -> Self {
pub(crate) fn def_site() -> Self {
Span::call_site()
}
pub fn resolved_at(&self, _other: Span) -> Span {
pub(crate) fn resolved_at(&self, _other: Span) -> Span {
// Stable spans consist only of line/column information, so
// `resolved_at` and `located_at` only select which span the
// caller wants line/column information from.
*self
}
pub fn located_at(&self, other: Span) -> Span {
pub(crate) fn located_at(&self, other: Span) -> Span {
other
}
#[cfg(procmacro2_semver_exempt)]
pub fn source_file(&self) -> SourceFile {
pub(crate) fn source_file(&self) -> SourceFile {
#[cfg(fuzzing)]
return SourceFile {
path: PathBuf::from("<unspecified>"),
};
#[cfg(not(fuzzing))]
SOURCE_MAP.with(|cm| {
let cm = cm.borrow();
let path = cm.filepath(*self);
SOURCE_MAP.with(|sm| {
let sm = sm.borrow();
let path = sm.filepath(*self);
SourceFile { path }
})
}
#[cfg(span_locations)]
pub fn start(&self) -> LineColumn {
pub(crate) fn byte_range(&self) -> Range<usize> {
#[cfg(fuzzing)]
return 0..0;
#[cfg(not(fuzzing))]
{
if self.is_call_site() {
0..0
} else {
SOURCE_MAP.with(|sm| sm.borrow_mut().fileinfo_mut(*self).byte_range(*self))
}
}
}
#[cfg(span_locations)]
pub(crate) fn start(&self) -> LineColumn {
#[cfg(fuzzing)]
return LineColumn { line: 0, column: 0 };
#[cfg(not(fuzzing))]
SOURCE_MAP.with(|cm| {
let cm = cm.borrow();
let fi = cm.fileinfo(*self);
SOURCE_MAP.with(|sm| {
let sm = sm.borrow();
let fi = sm.fileinfo(*self);
fi.offset_line_column(self.lo as usize)
})
}
#[cfg(span_locations)]
pub fn end(&self) -> LineColumn {
pub(crate) fn end(&self) -> LineColumn {
#[cfg(fuzzing)]
return LineColumn { line: 0, column: 0 };
#[cfg(not(fuzzing))]
SOURCE_MAP.with(|cm| {
let cm = cm.borrow();
let fi = cm.fileinfo(*self);
SOURCE_MAP.with(|sm| {
let sm = sm.borrow();
let fi = sm.fileinfo(*self);
fi.offset_line_column(self.hi as usize)
})
}
#[cfg(not(span_locations))]
pub fn join(&self, _other: Span) -> Option<Span> {
pub(crate) fn join(&self, _other: Span) -> Option<Span> {
Some(Span {})
}
#[cfg(span_locations)]
pub fn join(&self, other: Span) -> Option<Span> {
pub(crate) fn join(&self, other: Span) -> Option<Span> {
#[cfg(fuzzing)]
return {
let _ = other;
@@ -581,10 +614,10 @@ impl Span {
};
#[cfg(not(fuzzing))]
SOURCE_MAP.with(|cm| {
let cm = cm.borrow();
SOURCE_MAP.with(|sm| {
let sm = sm.borrow();
// If `other` is not within the same FileInfo as us, return None.
if !cm.fileinfo(*self).span_within(other) {
if !sm.fileinfo(*self).span_within(other) {
return None;
}
Some(Span {
@@ -595,12 +628,12 @@ impl Span {
}
#[cfg(not(span_locations))]
pub fn source_text(&self) -> Option<String> {
pub(crate) fn source_text(&self) -> Option<String> {
None
}
#[cfg(span_locations)]
pub fn source_text(&self) -> Option<String> {
pub(crate) fn source_text(&self) -> Option<String> {
#[cfg(fuzzing)]
return None;
@@ -609,7 +642,7 @@ impl Span {
if self.is_call_site() {
None
} else {
Some(SOURCE_MAP.with(|cm| cm.borrow_mut().fileinfo_mut(*self).source_text(*self)))
Some(SOURCE_MAP.with(|sm| sm.borrow_mut().fileinfo_mut(*self).source_text(*self)))
}
}
}
@@ -677,7 +710,7 @@ pub(crate) struct Group {
}
impl Group {
pub fn new(delimiter: Delimiter, stream: TokenStream) -> Self {
pub(crate) fn new(delimiter: Delimiter, stream: TokenStream) -> Self {
Group {
delimiter,
stream,
@@ -685,27 +718,27 @@ impl Group {
}
}
pub fn delimiter(&self) -> Delimiter {
pub(crate) fn delimiter(&self) -> Delimiter {
self.delimiter
}
pub fn stream(&self) -> TokenStream {
pub(crate) fn stream(&self) -> TokenStream {
self.stream.clone()
}
pub fn span(&self) -> Span {
pub(crate) fn span(&self) -> Span {
self.span
}
pub fn span_open(&self) -> Span {
pub(crate) fn span_open(&self) -> Span {
self.span.first_byte()
}
pub fn span_close(&self) -> Span {
pub(crate) fn span_close(&self) -> Span {
self.span.last_byte()
}
pub fn set_span(&mut self, span: Span) {
pub(crate) fn set_span(&mut self, span: Span) {
self.span = span;
}
}
@@ -749,45 +782,45 @@ impl Debug for Group {
#[derive(Clone)]
pub(crate) struct Ident {
sym: String,
sym: Box<str>,
span: Span,
raw: bool,
}
impl Ident {
#[track_caller]
pub fn new_checked(string: &str, span: Span) -> Self {
pub(crate) fn new_checked(string: &str, span: Span) -> Self {
validate_ident(string);
Ident::new_unchecked(string, span)
}
pub fn new_unchecked(string: &str, span: Span) -> Self {
pub(crate) fn new_unchecked(string: &str, span: Span) -> Self {
Ident {
sym: string.to_owned(),
sym: Box::from(string),
span,
raw: false,
}
}
#[track_caller]
pub fn new_raw_checked(string: &str, span: Span) -> Self {
pub(crate) fn new_raw_checked(string: &str, span: Span) -> Self {
validate_ident_raw(string);
Ident::new_raw_unchecked(string, span)
}
pub fn new_raw_unchecked(string: &str, span: Span) -> Self {
pub(crate) fn new_raw_unchecked(string: &str, span: Span) -> Self {
Ident {
sym: string.to_owned(),
sym: Box::from(string),
span,
raw: true,
}
}
pub fn span(&self) -> Span {
pub(crate) fn span(&self) -> Span {
self.span
}
pub fn set_span(&mut self, span: Span) {
pub(crate) fn set_span(&mut self, span: Span) {
self.span = span;
}
}
@@ -854,9 +887,9 @@ where
fn eq(&self, other: &T) -> bool {
let other = other.as_ref();
if self.raw {
other.starts_with("r#") && self.sym == other[2..]
other.starts_with("r#") && *self.sym == other[2..]
} else {
self.sym == other
*self.sym == *other
}
}
}
@@ -895,13 +928,13 @@ impl Debug for Ident {
#[derive(Clone)]
pub(crate) struct Literal {
repr: String,
pub(crate) repr: String,
span: Span,
}
macro_rules! suffixed_numbers {
($($name:ident => $kind:ident,)*) => ($(
pub fn $name(n: $kind) -> Literal {
pub(crate) fn $name(n: $kind) -> Literal {
Literal::_new(format!(concat!("{}", stringify!($kind)), n))
}
)*)
@@ -909,7 +942,7 @@ macro_rules! suffixed_numbers {
macro_rules! unsuffixed_numbers {
($($name:ident => $kind:ident,)*) => ($(
pub fn $name(n: $kind) -> Literal {
pub(crate) fn $name(n: $kind) -> Literal {
Literal::_new(n.to_string())
}
)*)
@@ -923,6 +956,36 @@ impl Literal {
}
}
pub(crate) fn from_str_checked(repr: &str) -> Result<Self, LexError> {
let mut cursor = get_cursor(repr);
#[cfg(span_locations)]
let lo = cursor.off;
let negative = cursor.starts_with_char('-');
if negative {
cursor = cursor.advance(1);
if !cursor.starts_with_fn(|ch| ch.is_ascii_digit()) {
return Err(LexError::call_site());
}
}
if let Ok((rest, mut literal)) = parse::literal(cursor) {
if rest.is_empty() {
if negative {
literal.repr.insert(0, '-');
}
literal.span = Span {
#[cfg(span_locations)]
lo,
#[cfg(span_locations)]
hi: rest.off,
};
return Ok(literal);
}
}
Err(LexError::call_site())
}
pub(crate) unsafe fn from_str_unchecked(repr: &str) -> Self {
Literal::_new(repr.to_owned())
}
@@ -960,7 +1023,7 @@ impl Literal {
isize_unsuffixed => isize,
}
pub fn f32_unsuffixed(f: f32) -> Literal {
pub(crate) fn f32_unsuffixed(f: f32) -> Literal {
let mut s = f.to_string();
if !s.contains('.') {
s.push_str(".0");
@@ -968,7 +1031,7 @@ impl Literal {
Literal::_new(s)
}
pub fn f64_unsuffixed(f: f64) -> Literal {
pub(crate) fn f64_unsuffixed(f: f64) -> Literal {
let mut s = f.to_string();
if !s.contains('.') {
s.push_str(".0");
@@ -976,82 +1039,109 @@ impl Literal {
Literal::_new(s)
}
pub fn string(t: &str) -> Literal {
let mut repr = String::with_capacity(t.len() + 2);
pub(crate) fn string(string: &str) -> Literal {
let mut repr = String::with_capacity(string.len() + 2);
repr.push('"');
let mut chars = t.chars();
while let Some(ch) = chars.next() {
if ch == '\0' {
repr.push_str(
if chars
.as_str()
.starts_with(|next| '0' <= next && next <= '7')
{
// circumvent clippy::octal_escapes lint
"\\x00"
} else {
"\\0"
},
);
} else if ch == '\'' {
// escape_debug turns this into "\'" which is unnecessary.
repr.push(ch);
} else {
repr.extend(ch.escape_debug());
}
}
escape_utf8(string, &mut repr);
repr.push('"');
Literal::_new(repr)
}
pub fn character(t: char) -> Literal {
pub(crate) fn character(ch: char) -> Literal {
let mut repr = String::new();
repr.push('\'');
if t == '"' {
if ch == '"' {
// escape_debug turns this into '\"' which is unnecessary.
repr.push(t);
repr.push(ch);
} else {
repr.extend(t.escape_debug());
repr.extend(ch.escape_debug());
}
repr.push('\'');
Literal::_new(repr)
}
pub fn byte_string(bytes: &[u8]) -> Literal {
let mut escaped = "b\"".to_string();
pub(crate) fn byte_character(byte: u8) -> Literal {
let mut repr = "b'".to_string();
#[allow(clippy::match_overlapping_arm)]
match byte {
b'\0' => repr.push_str(r"\0"),
b'\t' => repr.push_str(r"\t"),
b'\n' => repr.push_str(r"\n"),
b'\r' => repr.push_str(r"\r"),
b'\'' => repr.push_str(r"\'"),
b'\\' => repr.push_str(r"\\"),
b'\x20'..=b'\x7E' => repr.push(byte as char),
_ => {
let _ = write!(repr, r"\x{:02X}", byte);
}
}
repr.push('\'');
Literal::_new(repr)
}
pub(crate) fn byte_string(bytes: &[u8]) -> Literal {
let mut repr = "b\"".to_string();
let mut bytes = bytes.iter();
while let Some(&b) = bytes.next() {
#[allow(clippy::match_overlapping_arm)]
match b {
b'\0' => escaped.push_str(match bytes.as_slice().first() {
b'\0' => repr.push_str(match bytes.as_slice().first() {
// circumvent clippy::octal_escapes lint
Some(b'0'..=b'7') => r"\x00",
_ => r"\0",
}),
b'\t' => escaped.push_str(r"\t"),
b'\n' => escaped.push_str(r"\n"),
b'\r' => escaped.push_str(r"\r"),
b'"' => escaped.push_str("\\\""),
b'\\' => escaped.push_str("\\\\"),
b'\x20'..=b'\x7E' => escaped.push(b as char),
b'\t' => repr.push_str(r"\t"),
b'\n' => repr.push_str(r"\n"),
b'\r' => repr.push_str(r"\r"),
b'"' => repr.push_str("\\\""),
b'\\' => repr.push_str(r"\\"),
b'\x20'..=b'\x7E' => repr.push(b as char),
_ => {
let _ = write!(escaped, "\\x{:02X}", b);
let _ = write!(repr, r"\x{:02X}", b);
}
}
}
escaped.push('"');
Literal::_new(escaped)
repr.push('"');
Literal::_new(repr)
}
pub fn span(&self) -> Span {
pub(crate) fn c_string(string: &CStr) -> Literal {
let mut repr = "c\"".to_string();
let mut bytes = string.to_bytes();
while !bytes.is_empty() {
let (valid, invalid) = match str::from_utf8(bytes) {
Ok(all_valid) => {
bytes = b"";
(all_valid, bytes)
}
Err(utf8_error) => {
let (valid, rest) = bytes.split_at(utf8_error.valid_up_to());
let valid = str::from_utf8(valid).unwrap();
let invalid = utf8_error
.error_len()
.map_or(rest, |error_len| &rest[..error_len]);
bytes = &bytes[valid.len() + invalid.len()..];
(valid, invalid)
}
};
escape_utf8(valid, &mut repr);
for &byte in invalid {
let _ = write!(repr, r"\x{:02X}", byte);
}
}
repr.push('"');
Literal::_new(repr)
}
pub(crate) fn span(&self) -> Span {
self.span
}
pub fn set_span(&mut self, span: Span) {
pub(crate) fn set_span(&mut self, span: Span) {
self.span = span;
}
pub fn subspan<R: RangeBounds<usize>>(&self, range: R) -> Option<Span> {
pub(crate) fn subspan<R: RangeBounds<usize>>(&self, range: R) -> Option<Span> {
#[cfg(not(span_locations))]
{
let _ = range;
@@ -1093,40 +1183,6 @@ impl Literal {
}
}
impl FromStr for Literal {
type Err = LexError;
fn from_str(repr: &str) -> Result<Self, Self::Err> {
let mut cursor = get_cursor(repr);
#[cfg(span_locations)]
let lo = cursor.off;
let negative = cursor.starts_with_char('-');
if negative {
cursor = cursor.advance(1);
if !cursor.starts_with_fn(|ch| ch.is_ascii_digit()) {
return Err(LexError::call_site());
}
}
if let Ok((rest, mut literal)) = parse::literal(cursor) {
if rest.is_empty() {
if negative {
literal.repr.insert(0, '-');
}
literal.span = Span {
#[cfg(span_locations)]
lo,
#[cfg(span_locations)]
hi: rest.off,
};
return Ok(literal);
}
}
Err(LexError::call_site())
}
}
impl Display for Literal {
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
Display::fmt(&self.repr, f)
@@ -1141,3 +1197,70 @@ impl Debug for Literal {
debug.finish()
}
}
fn escape_utf8(string: &str, repr: &mut String) {
let mut chars = string.chars();
while let Some(ch) = chars.next() {
if ch == '\0' {
repr.push_str(
if chars
.as_str()
.starts_with(|next| '0' <= next && next <= '7')
{
// circumvent clippy::octal_escapes lint
r"\x00"
} else {
r"\0"
},
);
} else if ch == '\'' {
// escape_debug turns this into "\'" which is unnecessary.
repr.push(ch);
} else {
repr.extend(ch.escape_debug());
}
}
}
#[cfg(feature = "proc-macro")]
pub(crate) trait FromStr2: FromStr<Err = proc_macro::LexError> {
#[cfg(wrap_proc_macro)]
fn valid(src: &str) -> bool;
#[cfg(wrap_proc_macro)]
fn from_str_checked(src: &str) -> Result<Self, imp::LexError> {
// Validate using fallback parser, because rustc is incapable of
// returning a recoverable Err for certain invalid token streams, and
// will instead permanently poison the compilation.
if !Self::valid(src) {
return Err(imp::LexError::CompilerPanic);
}
// Catch panic to work around https://github.com/rust-lang/rust/issues/58736.
match panic::catch_unwind(|| Self::from_str(src)) {
Ok(Ok(ok)) => Ok(ok),
Ok(Err(lex)) => Err(imp::LexError::Compiler(lex)),
Err(_panic) => Err(imp::LexError::CompilerPanic),
}
}
fn from_str_unchecked(src: &str) -> Self {
Self::from_str(src).unwrap()
}
}
#[cfg(feature = "proc-macro")]
impl FromStr2 for proc_macro::TokenStream {
#[cfg(wrap_proc_macro)]
fn valid(src: &str) -> bool {
TokenStream::from_str_checked(src).is_ok()
}
}
#[cfg(feature = "proc-macro")]
impl FromStr2 for proc_macro::Literal {
#[cfg(wrap_proc_macro)]
fn valid(src: &str) -> bool {
Literal::from_str_checked(src).is_ok()
}
}
src/lib.rs
+114 -53
View File
@@ -86,24 +86,27 @@
//! a different thread.
// Proc-macro2 types in rustdoc of other crates get linked to here.
#![doc(html_root_url = "https://docs.rs/proc-macro2/1.0.76")]
#![doc(html_root_url = "https://docs.rs/proc-macro2/1.0.92")]
#![cfg_attr(any(proc_macro_span, super_unstable), feature(proc_macro_span))]
#![cfg_attr(super_unstable, feature(proc_macro_def_site))]
#![cfg_attr(doc_cfg, feature(doc_cfg))]
#![cfg_attr(docsrs, feature(doc_cfg))]
#![deny(unsafe_op_in_unsafe_fn)]
#![allow(
clippy::cast_lossless,
clippy::cast_possible_truncation,
clippy::checked_conversions,
clippy::doc_markdown,
clippy::incompatible_msrv,
clippy::items_after_statements,
clippy::iter_without_into_iter,
clippy::let_underscore_untyped,
clippy::manual_assert,
clippy::manual_range_contains,
clippy::missing_panics_doc,
clippy::missing_safety_doc,
clippy::must_use_candidate,
clippy::needless_doctest_main,
clippy::needless_lifetimes,
clippy::new_without_default,
clippy::return_self_not_must_use,
clippy::shadow_unrelated,
@@ -160,18 +163,21 @@ mod imp;
mod location;
use crate::extra::DelimSpan;
use crate::marker::Marker;
use crate::marker::{ProcMacroAutoTraits, MARKER};
use core::cmp::Ordering;
use core::fmt::{self, Debug, Display};
use core::hash::{Hash, Hasher};
#[cfg(span_locations)]
use core::ops::Range;
use core::ops::RangeBounds;
use core::str::FromStr;
use std::error::Error;
use std::ffi::CStr;
#[cfg(procmacro2_semver_exempt)]
use std::path::PathBuf;
#[cfg(span_locations)]
#[cfg_attr(doc_cfg, doc(cfg(feature = "span-locations")))]
#[cfg_attr(docsrs, doc(cfg(feature = "span-locations")))]
pub use crate::location::LineColumn;
/// An abstract stream of tokens, or more concretely a sequence of token trees.
@@ -184,27 +190,27 @@ pub use crate::location::LineColumn;
#[derive(Clone)]
pub struct TokenStream {
inner: imp::TokenStream,
_marker: Marker,
_marker: ProcMacroAutoTraits,
}
/// Error returned from `TokenStream::from_str`.
pub struct LexError {
inner: imp::LexError,
_marker: Marker,
_marker: ProcMacroAutoTraits,
}
impl TokenStream {
fn _new(inner: imp::TokenStream) -> Self {
TokenStream {
inner,
_marker: Marker,
_marker: MARKER,
}
}
fn _new_fallback(inner: fallback::TokenStream) -> Self {
TokenStream {
inner: inner.into(),
_marker: Marker,
inner: imp::TokenStream::from(inner),
_marker: MARKER,
}
}
@@ -239,27 +245,29 @@ impl FromStr for TokenStream {
type Err = LexError;
fn from_str(src: &str) -> Result<TokenStream, LexError> {
let e = src.parse().map_err(|e| LexError {
inner: e,
_marker: Marker,
})?;
Ok(TokenStream::_new(e))
match imp::TokenStream::from_str_checked(src) {
Ok(tokens) => Ok(TokenStream::_new(tokens)),
Err(lex) => Err(LexError {
inner: lex,
_marker: MARKER,
}),
}
}
}
#[cfg(feature = "proc-macro")]
#[cfg_attr(doc_cfg, doc(cfg(feature = "proc-macro")))]
#[cfg_attr(docsrs, doc(cfg(feature = "proc-macro")))]
impl From<proc_macro::TokenStream> for TokenStream {
fn from(inner: proc_macro::TokenStream) -> Self {
TokenStream::_new(inner.into())
TokenStream::_new(imp::TokenStream::from(inner))
}
}
#[cfg(feature = "proc-macro")]
#[cfg_attr(doc_cfg, doc(cfg(feature = "proc-macro")))]
#[cfg_attr(docsrs, doc(cfg(feature = "proc-macro")))]
impl From<TokenStream> for proc_macro::TokenStream {
fn from(inner: TokenStream) -> Self {
inner.inner.into()
proc_macro::TokenStream::from(inner.inner)
}
}
@@ -335,11 +343,11 @@ impl Error for LexError {}
///
/// This type is semver exempt and not exposed by default.
#[cfg(all(procmacro2_semver_exempt, any(not(wrap_proc_macro), super_unstable)))]
#[cfg_attr(doc_cfg, doc(cfg(procmacro2_semver_exempt)))]
#[cfg_attr(docsrs, doc(cfg(procmacro2_semver_exempt)))]
#[derive(Clone, PartialEq, Eq)]
pub struct SourceFile {
inner: imp::SourceFile,
_marker: Marker,
_marker: ProcMacroAutoTraits,
}
#[cfg(all(procmacro2_semver_exempt, any(not(wrap_proc_macro), super_unstable)))]
@@ -347,7 +355,7 @@ impl SourceFile {
fn _new(inner: imp::SourceFile) -> Self {
SourceFile {
inner,
_marker: Marker,
_marker: MARKER,
}
}
@@ -386,21 +394,21 @@ impl Debug for SourceFile {
#[derive(Copy, Clone)]
pub struct Span {
inner: imp::Span,
_marker: Marker,
_marker: ProcMacroAutoTraits,
}
impl Span {
fn _new(inner: imp::Span) -> Self {
Span {
inner,
_marker: Marker,
_marker: MARKER,
}
}
fn _new_fallback(inner: fallback::Span) -> Self {
Span {
inner: inner.into(),
_marker: Marker,
inner: imp::Span::from(inner),
_marker: MARKER,
}
}
@@ -424,7 +432,7 @@ impl Span {
///
/// This method is semver exempt and not exposed by default.
#[cfg(procmacro2_semver_exempt)]
#[cfg_attr(doc_cfg, doc(cfg(procmacro2_semver_exempt)))]
#[cfg_attr(docsrs, doc(cfg(procmacro2_semver_exempt)))]
pub fn def_site() -> Self {
Span::_new(imp::Span::def_site())
}
@@ -467,11 +475,26 @@ impl Span {
///
/// This method is semver exempt and not exposed by default.
#[cfg(all(procmacro2_semver_exempt, any(not(wrap_proc_macro), super_unstable)))]
#[cfg_attr(doc_cfg, doc(cfg(procmacro2_semver_exempt)))]
#[cfg_attr(docsrs, doc(cfg(procmacro2_semver_exempt)))]
pub fn source_file(&self) -> SourceFile {
SourceFile::_new(self.inner.source_file())
}
/// Returns the span's byte position range in the source file.
///
/// This method requires the `"span-locations"` feature to be enabled.
///
/// When executing in a procedural macro context, the returned range is only
/// accurate if compiled with a nightly toolchain. The stable toolchain does
/// not have this information available. When executing outside of a
/// procedural macro, such as main.rs or build.rs, the byte range is always
/// accurate regardless of toolchain.
#[cfg(span_locations)]
#[cfg_attr(docsrs, doc(cfg(feature = "span-locations")))]
pub fn byte_range(&self) -> Range<usize> {
self.inner.byte_range()
}
/// Get the starting line/column in the source file for this span.
///
/// This method requires the `"span-locations"` feature to be enabled.
@@ -482,7 +505,7 @@ impl Span {
/// outside of a procedural macro, such as main.rs or build.rs, the
/// line/column are always meaningful regardless of toolchain.
#[cfg(span_locations)]
#[cfg_attr(doc_cfg, doc(cfg(feature = "span-locations")))]
#[cfg_attr(docsrs, doc(cfg(feature = "span-locations")))]
pub fn start(&self) -> LineColumn {
self.inner.start()
}
@@ -497,7 +520,7 @@ impl Span {
/// outside of a procedural macro, such as main.rs or build.rs, the
/// line/column are always meaningful regardless of toolchain.
#[cfg(span_locations)]
#[cfg_attr(doc_cfg, doc(cfg(feature = "span-locations")))]
#[cfg_attr(docsrs, doc(cfg(feature = "span-locations")))]
pub fn end(&self) -> LineColumn {
self.inner.end()
}
@@ -519,7 +542,7 @@ impl Span {
///
/// This method is semver exempt and not exposed by default.
#[cfg(procmacro2_semver_exempt)]
#[cfg_attr(doc_cfg, doc(cfg(procmacro2_semver_exempt)))]
#[cfg_attr(docsrs, doc(cfg(procmacro2_semver_exempt)))]
pub fn eq(&self, other: &Span) -> bool {
self.inner.eq(&other.inner)
}
@@ -659,13 +682,25 @@ pub enum Delimiter {
Brace,
/// `[ ... ]`
Bracket,
/// `Ø ... Ø`
/// `∅ ... ∅`
///
/// An implicit delimiter, that may, for example, appear around tokens
/// An invisible delimiter, that may, for example, appear around tokens
/// coming from a "macro variable" `$var`. It is important to preserve
/// operator priorities in cases like `$var * 3` where `$var` is `1 + 2`.
/// Implicit delimiters may not survive roundtrip of a token stream through
/// Invisible delimiters may not survive roundtrip of a token stream through
/// a string.
///
/// <div class="warning">
///
/// Note: rustc currently can ignore the grouping of tokens delimited by `None` in the output
/// of a proc_macro. Only `None`-delimited groups created by a macro_rules macro in the input
/// of a proc_macro macro are preserved, and only in very specific circumstances.
/// Any `None`-delimited groups (re)created by a proc_macro will therefore not preserve
/// operator priorities as indicated above. The other `Delimiter` variants should be used
/// instead in this context. This is a rustc bug. For details, see
/// [rust-lang/rust#67062](https://github.com/rust-lang/rust/issues/67062).
///
/// </div>
None,
}
@@ -676,7 +711,7 @@ impl Group {
fn _new_fallback(inner: fallback::Group) -> Self {
Group {
inner: inner.into(),
inner: imp::Group::from(inner),
}
}
@@ -802,10 +837,16 @@ impl Punct {
/// The returned `Punct` will have the default span of `Span::call_site()`
/// which can be further configured with the `set_span` method below.
pub fn new(ch: char, spacing: Spacing) -> Self {
Punct {
ch,
spacing,
span: Span::call_site(),
if let '!' | '#' | '$' | '%' | '&' | '\'' | '*' | '+' | ',' | '-' | '.' | '/' | ':' | ';'
| '<' | '=' | '>' | '?' | '@' | '^' | '|' | '~' = ch
{
Punct {
ch,
spacing,
span: Span::call_site(),
}
} else {
panic!("unsupported proc macro punctuation character {:?}", ch);
}
}
@@ -919,14 +960,21 @@ impl Debug for Punct {
#[derive(Clone)]
pub struct Ident {
inner: imp::Ident,
_marker: Marker,
_marker: ProcMacroAutoTraits,
}
impl Ident {
fn _new(inner: imp::Ident) -> Self {
Ident {
inner,
_marker: Marker,
_marker: MARKER,
}
}
fn _new_fallback(inner: fallback::Ident) -> Self {
Ident {
inner: imp::Ident::from(inner),
_marker: MARKER,
}
}
@@ -1046,7 +1094,7 @@ impl Debug for Ident {
#[derive(Clone)]
pub struct Literal {
inner: imp::Literal,
_marker: Marker,
_marker: ProcMacroAutoTraits,
}
macro_rules! suffixed_int_literals {
@@ -1093,14 +1141,14 @@ impl Literal {
fn _new(inner: imp::Literal) -> Self {
Literal {
inner,
_marker: Marker,
_marker: MARKER,
}
}
fn _new_fallback(inner: fallback::Literal) -> Self {
Literal {
inner: inner.into(),
_marker: Marker,
inner: imp::Literal::from(inner),
_marker: MARKER,
}
}
@@ -1216,9 +1264,19 @@ impl Literal {
Literal::_new(imp::Literal::character(ch))
}
/// Byte character literal.
pub fn byte_character(byte: u8) -> Literal {
Literal::_new(imp::Literal::byte_character(byte))
}
/// Byte string literal.
pub fn byte_string(s: &[u8]) -> Literal {
Literal::_new(imp::Literal::byte_string(s))
pub fn byte_string(bytes: &[u8]) -> Literal {
Literal::_new(imp::Literal::byte_string(bytes))
}
/// C string literal.
pub fn c_string(string: &CStr) -> Literal {
Literal::_new(imp::Literal::c_string(string))
}
/// Returns the span encompassing this literal.
@@ -1258,10 +1316,13 @@ impl FromStr for Literal {
type Err = LexError;
fn from_str(repr: &str) -> Result<Self, LexError> {
repr.parse().map(Literal::_new).map_err(|inner| LexError {
inner,
_marker: Marker,
})
match imp::Literal::from_str_checked(repr) {
Ok(lit) => Ok(Literal::_new(lit)),
Err(lex) => Err(LexError {
inner: lex,
_marker: MARKER,
}),
}
}
}
@@ -1279,7 +1340,7 @@ impl Display for Literal {
/// Public implementation details for the `TokenStream` type, such as iterators.
pub mod token_stream {
use crate::marker::Marker;
use crate::marker::{ProcMacroAutoTraits, MARKER};
use crate::{imp, TokenTree};
use core::fmt::{self, Debug};
@@ -1292,7 +1353,7 @@ pub mod token_stream {
#[derive(Clone)]
pub struct IntoIter {
inner: imp::TokenTreeIter,
_marker: Marker,
_marker: ProcMacroAutoTraits,
}
impl Iterator for IntoIter {
@@ -1321,7 +1382,7 @@ pub mod token_stream {
fn into_iter(self) -> IntoIter {
IntoIter {
inner: self.inner.into_iter(),
_marker: Marker,
_marker: MARKER,
}
}
}
src/location.rs
+1 -1
View File
@@ -3,7 +3,7 @@ use core::cmp::Ordering;
/// A line-column pair representing the start or end of a `Span`.
///
/// This type is semver exempt and not exposed by default.
#[cfg_attr(doc_cfg, doc(cfg(feature = "span-locations")))]
#[cfg_attr(docsrs, doc(cfg(feature = "span-locations")))]
#[derive(Copy, Clone, Debug, PartialEq, Eq, Hash)]
pub struct LineColumn {
/// The 1-indexed line in the source file on which the span starts or ends
src/marker.rs
+7 -11
View File
@@ -4,18 +4,14 @@ use core::panic::{RefUnwindSafe, UnwindSafe};
// Zero sized marker with the correct set of autotrait impls we want all proc
// macro types to have.
pub(crate) type Marker = PhantomData<ProcMacroAutoTraits>;
#[derive(Copy, Clone)]
#[cfg_attr(
all(procmacro2_semver_exempt, any(not(wrap_proc_macro), super_unstable)),
derive(PartialEq, Eq)
)]
pub(crate) struct ProcMacroAutoTraits(PhantomData<Rc<()>>);
pub(crate) use self::value::*;
mod value {
pub(crate) use core::marker::PhantomData as Marker;
}
pub(crate) struct ProcMacroAutoTraits(
#[allow(dead_code)] // https://github.com/rust-lang/rust/issues/119645
Rc<()>,
);
pub(crate) const MARKER: ProcMacroAutoTraits = ProcMacroAutoTraits(PhantomData);
impl UnwindSafe for ProcMacroAutoTraits {}
impl RefUnwindSafe for ProcMacroAutoTraits {}
src/parse.rs
+14 -18
View File
@@ -1,5 +1,5 @@
use crate::fallback::{
self, is_ident_continue, is_ident_start, Group, LexError, Literal, Span, TokenStream,
self, is_ident_continue, is_ident_start, Group, Ident, LexError, Literal, Span, TokenStream,
TokenStreamBuilder,
};
use crate::{Delimiter, Punct, Spacing, TokenTree};
@@ -8,13 +8,13 @@ use core::str::{Bytes, CharIndices, Chars};
#[derive(Copy, Clone, Eq, PartialEq)]
pub(crate) struct Cursor<'a> {
pub rest: &'a str,
pub(crate) rest: &'a str,
#[cfg(span_locations)]
pub off: u32,
pub(crate) off: u32,
}
impl<'a> Cursor<'a> {
pub fn advance(&self, bytes: usize) -> Cursor<'a> {
pub(crate) fn advance(&self, bytes: usize) -> Cursor<'a> {
let (_front, rest) = self.rest.split_at(bytes);
Cursor {
rest,
@@ -23,22 +23,22 @@ impl<'a> Cursor<'a> {
}
}
pub fn starts_with(&self, s: &str) -> bool {
pub(crate) fn starts_with(&self, s: &str) -> bool {
self.rest.starts_with(s)
}
pub fn starts_with_char(&self, ch: char) -> bool {
pub(crate) fn starts_with_char(&self, ch: char) -> bool {
self.rest.starts_with(ch)
}
pub fn starts_with_fn<Pattern>(&self, f: Pattern) -> bool
pub(crate) fn starts_with_fn<Pattern>(&self, f: Pattern) -> bool
where
Pattern: FnMut(char) -> bool,
{
self.rest.starts_with(f)
}
pub fn is_empty(&self) -> bool {
pub(crate) fn is_empty(&self) -> bool {
self.rest.is_empty()
}
@@ -300,10 +300,8 @@ fn ident_any(input: Cursor) -> PResult<crate::Ident> {
let (rest, sym) = ident_not_raw(rest)?;
if !raw {
let ident = crate::Ident::_new(crate::imp::Ident::new_unchecked(
sym,
fallback::Span::call_site(),
));
let ident =
crate::Ident::_new_fallback(Ident::new_unchecked(sym, fallback::Span::call_site()));
return Ok((rest, ident));
}
@@ -312,10 +310,8 @@ fn ident_any(input: Cursor) -> PResult<crate::Ident> {
_ => {}
}
let ident = crate::Ident::_new(crate::imp::Ident::new_raw_unchecked(
sym,
fallback::Span::call_site(),
));
let ident =
crate::Ident::_new_fallback(Ident::new_raw_unchecked(sym, fallback::Span::call_site()));
Ok((rest, ident))
}
@@ -941,10 +937,10 @@ fn doc_comment<'a>(input: Cursor<'a>, trees: &mut TokenStreamBuilder) -> PResult
trees.push_token_from_parser(TokenTree::Punct(bang));
}
let doc_ident = crate::Ident::_new(crate::imp::Ident::new_unchecked("doc", fallback_span));
let doc_ident = crate::Ident::_new_fallback(Ident::new_unchecked("doc", fallback_span));
let mut equal = Punct::new('=', Spacing::Alone);
equal.set_span(span);
let mut literal = crate::Literal::string(comment);
let mut literal = crate::Literal::_new_fallback(Literal::string(comment));
literal.set_span(span);
let mut bracketed = TokenStreamBuilder::with_capacity(3);
bracketed.push_token_from_parser(TokenTree::Ident(doc_ident));
src/rcvec.rs
+16 -16
View File
@@ -22,19 +22,19 @@ pub(crate) struct RcVecIntoIter<T> {
}
impl<T> RcVec<T> {
pub fn is_empty(&self) -> bool {
pub(crate) fn is_empty(&self) -> bool {
self.inner.is_empty()
}
pub fn len(&self) -> usize {
pub(crate) fn len(&self) -> usize {
self.inner.len()
}
pub fn iter(&self) -> slice::Iter<T> {
pub(crate) fn iter(&self) -> slice::Iter<T> {
self.inner.iter()
}
pub fn make_mut(&mut self) -> RcVecMut<T>
pub(crate) fn make_mut(&mut self) -> RcVecMut<T>
where
T: Clone,
{
@@ -43,12 +43,12 @@ impl<T> RcVec<T> {
}
}
pub fn get_mut(&mut self) -> Option<RcVecMut<T>> {
pub(crate) fn get_mut(&mut self) -> Option<RcVecMut<T>> {
let inner = Rc::get_mut(&mut self.inner)?;
Some(RcVecMut { inner })
}
pub fn make_owned(mut self) -> RcVecBuilder<T>
pub(crate) fn make_owned(mut self) -> RcVecBuilder<T>
where
T: Clone,
{
@@ -62,31 +62,31 @@ impl<T> RcVec<T> {
}
impl<T> RcVecBuilder<T> {
pub fn new() -> Self {
pub(crate) fn new() -> Self {
RcVecBuilder { inner: Vec::new() }
}
pub fn with_capacity(cap: usize) -> Self {
pub(crate) fn with_capacity(cap: usize) -> Self {
RcVecBuilder {
inner: Vec::with_capacity(cap),
}
}
pub fn push(&mut self, element: T) {
pub(crate) fn push(&mut self, element: T) {
self.inner.push(element);
}
pub fn extend(&mut self, iter: impl IntoIterator<Item = T>) {
pub(crate) fn extend(&mut self, iter: impl IntoIterator<Item = T>) {
self.inner.extend(iter);
}
pub fn as_mut(&mut self) -> RcVecMut<T> {
pub(crate) fn as_mut(&mut self) -> RcVecMut<T> {
RcVecMut {
inner: &mut self.inner,
}
}
pub fn build(self) -> RcVec<T> {
pub(crate) fn build(self) -> RcVec<T> {
RcVec {
inner: Rc::new(self.inner),
}
@@ -94,19 +94,19 @@ impl<T> RcVecBuilder<T> {
}
impl<'a, T> RcVecMut<'a, T> {
pub fn push(&mut self, element: T) {
pub(crate) fn push(&mut self, element: T) {
self.inner.push(element);
}
pub fn extend(&mut self, iter: impl IntoIterator<Item = T>) {
pub(crate) fn extend(&mut self, iter: impl IntoIterator<Item = T>) {
self.inner.extend(iter);
}
pub fn pop(&mut self) -> Option<T> {
pub(crate) fn pop(&mut self) -> Option<T> {
self.inner.pop()
}
pub fn as_mut(&mut self) -> RcVecMut<T> {
pub(crate) fn as_mut(&mut self) -> RcVecMut<T> {
RcVecMut { inner: self.inner }
}
}
src/wrapper.rs
+174 -101
View File
@@ -1,11 +1,13 @@
use crate::detection::inside_proc_macro;
use crate::fallback::{self, FromStr2 as _};
#[cfg(span_locations)]
use crate::location::LineColumn;
use crate::{fallback, Delimiter, Punct, Spacing, TokenTree};
use crate::{Delimiter, Punct, Spacing, TokenTree};
use core::fmt::{self, Debug, Display};
#[cfg(span_locations)]
use core::ops::Range;
use core::ops::RangeBounds;
use core::str::FromStr;
use std::panic;
use std::ffi::CStr;
#[cfg(super_unstable)]
use std::path::PathBuf;
@@ -39,11 +41,11 @@ fn mismatch(line: u32) -> ! {
#[cfg(procmacro2_backtrace)]
{
let backtrace = std::backtrace::Backtrace::force_capture();
panic!("compiler/fallback mismatch #{}\n\n{}", line, backtrace)
panic!("compiler/fallback mismatch L{}\n\n{}", line, backtrace)
}
#[cfg(not(procmacro2_backtrace))]
{
panic!("compiler/fallback mismatch #{}", line)
panic!("compiler/fallback mismatch L{}", line)
}
}
@@ -75,7 +77,7 @@ impl DeferredTokenStream {
}
impl TokenStream {
pub fn new() -> Self {
pub(crate) fn new() -> Self {
if inside_proc_macro() {
TokenStream::Compiler(DeferredTokenStream::new(proc_macro::TokenStream::new()))
} else {
@@ -83,7 +85,19 @@ impl TokenStream {
}
}
pub fn is_empty(&self) -> bool {
pub(crate) fn from_str_checked(src: &str) -> Result<Self, LexError> {
if inside_proc_macro() {
Ok(TokenStream::Compiler(DeferredTokenStream::new(
proc_macro::TokenStream::from_str_checked(src)?,
)))
} else {
Ok(TokenStream::Fallback(
fallback::TokenStream::from_str_checked(src)?,
))
}
}
pub(crate) fn is_empty(&self) -> bool {
match self {
TokenStream::Compiler(tts) => tts.is_empty(),
TokenStream::Fallback(tts) => tts.is_empty(),
@@ -105,26 +119,6 @@ impl TokenStream {
}
}
impl FromStr for TokenStream {
type Err = LexError;
fn from_str(src: &str) -> Result<TokenStream, LexError> {
if inside_proc_macro() {
Ok(TokenStream::Compiler(DeferredTokenStream::new(
proc_macro_parse(src)?,
)))
} else {
Ok(TokenStream::Fallback(src.parse()?))
}
}
}
// Work around https://github.com/rust-lang/rust/issues/58736.
fn proc_macro_parse(src: &str) -> Result<proc_macro::TokenStream, LexError> {
let result = panic::catch_unwind(|| src.parse().map_err(LexError::Compiler));
result.unwrap_or_else(|_| Err(LexError::CompilerPanic))
}
impl Display for TokenStream {
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
match self {
@@ -144,7 +138,9 @@ impl From<TokenStream> for proc_macro::TokenStream {
fn from(inner: TokenStream) -> Self {
match inner {
TokenStream::Compiler(inner) => inner.into_token_stream(),
TokenStream::Fallback(inner) => inner.to_string().parse().unwrap(),
TokenStream::Fallback(inner) => {
proc_macro::TokenStream::from_str_unchecked(&inner.to_string())
}
}
}
}
@@ -158,7 +154,7 @@ impl From<fallback::TokenStream> for TokenStream {
// Assumes inside_proc_macro().
fn into_compiler_token(token: TokenTree) -> proc_macro::TokenTree {
match token {
TokenTree::Group(tt) => tt.inner.unwrap_nightly().into(),
TokenTree::Group(tt) => proc_macro::TokenTree::Group(tt.inner.unwrap_nightly()),
TokenTree::Punct(tt) => {
let spacing = match tt.spacing() {
Spacing::Joint => proc_macro::Spacing::Joint,
@@ -166,19 +162,21 @@ fn into_compiler_token(token: TokenTree) -> proc_macro::TokenTree {
};
let mut punct = proc_macro::Punct::new(tt.as_char(), spacing);
punct.set_span(tt.span().inner.unwrap_nightly());
punct.into()
proc_macro::TokenTree::Punct(punct)
}
TokenTree::Ident(tt) => tt.inner.unwrap_nightly().into(),
TokenTree::Literal(tt) => tt.inner.unwrap_nightly().into(),
TokenTree::Ident(tt) => proc_macro::TokenTree::Ident(tt.inner.unwrap_nightly()),
TokenTree::Literal(tt) => proc_macro::TokenTree::Literal(tt.inner.unwrap_nightly()),
}
}
impl From<TokenTree> for TokenStream {
fn from(token: TokenTree) -> Self {
if inside_proc_macro() {
TokenStream::Compiler(DeferredTokenStream::new(into_compiler_token(token).into()))
TokenStream::Compiler(DeferredTokenStream::new(proc_macro::TokenStream::from(
into_compiler_token(token),
)))
} else {
TokenStream::Fallback(token.into())
TokenStream::Fallback(fallback::TokenStream::from(token))
}
}
}
@@ -333,7 +331,9 @@ impl Iterator for TokenTreeIter {
TokenTreeIter::Fallback(iter) => return iter.next(),
};
Some(match token {
proc_macro::TokenTree::Group(tt) => crate::Group::_new(Group::Compiler(tt)).into(),
proc_macro::TokenTree::Group(tt) => {
TokenTree::Group(crate::Group::_new(Group::Compiler(tt)))
}
proc_macro::TokenTree::Punct(tt) => {
let spacing = match tt.spacing() {
proc_macro::Spacing::Joint => Spacing::Joint,
@@ -341,10 +341,14 @@ impl Iterator for TokenTreeIter {
};
let mut o = Punct::new(tt.as_char(), spacing);
o.set_span(crate::Span::_new(Span::Compiler(tt.span())));
o.into()
TokenTree::Punct(o)
}
proc_macro::TokenTree::Ident(s) => {
TokenTree::Ident(crate::Ident::_new(Ident::Compiler(s)))
}
proc_macro::TokenTree::Literal(l) => {
TokenTree::Literal(crate::Literal::_new(Literal::Compiler(l)))
}
proc_macro::TokenTree::Ident(s) => crate::Ident::_new(Ident::Compiler(s)).into(),
proc_macro::TokenTree::Literal(l) => crate::Literal::_new(Literal::Compiler(l)).into(),
})
}
@@ -370,14 +374,14 @@ impl SourceFile {
}
/// Get the path to this source file as a string.
pub fn path(&self) -> PathBuf {
pub(crate) fn path(&self) -> PathBuf {
match self {
SourceFile::Compiler(a) => a.path(),
SourceFile::Fallback(a) => a.path(),
}
}
pub fn is_real(&self) -> bool {
pub(crate) fn is_real(&self) -> bool {
match self {
SourceFile::Compiler(a) => a.is_real(),
SourceFile::Fallback(a) => a.is_real(),
@@ -402,7 +406,7 @@ pub(crate) enum Span {
}
impl Span {
pub fn call_site() -> Self {
pub(crate) fn call_site() -> Self {
if inside_proc_macro() {
Span::Compiler(proc_macro::Span::call_site())
} else {
@@ -410,7 +414,7 @@ impl Span {
}
}
pub fn mixed_site() -> Self {
pub(crate) fn mixed_site() -> Self {
if inside_proc_macro() {
Span::Compiler(proc_macro::Span::mixed_site())
} else {
@@ -419,7 +423,7 @@ impl Span {
}
#[cfg(super_unstable)]
pub fn def_site() -> Self {
pub(crate) fn def_site() -> Self {
if inside_proc_macro() {
Span::Compiler(proc_macro::Span::def_site())
} else {
@@ -427,7 +431,7 @@ impl Span {
}
}
pub fn resolved_at(&self, other: Span) -> Span {
pub(crate) fn resolved_at(&self, other: Span) -> Span {
match (self, other) {
(Span::Compiler(a), Span::Compiler(b)) => Span::Compiler(a.resolved_at(b)),
(Span::Fallback(a), Span::Fallback(b)) => Span::Fallback(a.resolved_at(b)),
@@ -436,7 +440,7 @@ impl Span {
}
}
pub fn located_at(&self, other: Span) -> Span {
pub(crate) fn located_at(&self, other: Span) -> Span {
match (self, other) {
(Span::Compiler(a), Span::Compiler(b)) => Span::Compiler(a.located_at(b)),
(Span::Fallback(a), Span::Fallback(b)) => Span::Fallback(a.located_at(b)),
@@ -445,7 +449,7 @@ impl Span {
}
}
pub fn unwrap(self) -> proc_macro::Span {
pub(crate) fn unwrap(self) -> proc_macro::Span {
match self {
Span::Compiler(s) => s,
Span::Fallback(_) => panic!("proc_macro::Span is only available in procedural macros"),
@@ -453,7 +457,7 @@ impl Span {
}
#[cfg(super_unstable)]
pub fn source_file(&self) -> SourceFile {
pub(crate) fn source_file(&self) -> SourceFile {
match self {
Span::Compiler(s) => SourceFile::nightly(s.source_file()),
Span::Fallback(s) => SourceFile::Fallback(s.source_file()),
@@ -461,22 +465,48 @@ impl Span {
}
#[cfg(span_locations)]
pub fn start(&self) -> LineColumn {
pub(crate) fn byte_range(&self) -> Range<usize> {
match self {
#[cfg(proc_macro_span)]
Span::Compiler(s) => s.byte_range(),
#[cfg(not(proc_macro_span))]
Span::Compiler(_) => 0..0,
Span::Fallback(s) => s.byte_range(),
}
}
#[cfg(span_locations)]
pub(crate) fn start(&self) -> LineColumn {
match self {
#[cfg(proc_macro_span)]
Span::Compiler(s) => LineColumn {
line: s.line(),
column: s.column().saturating_sub(1),
},
#[cfg(not(proc_macro_span))]
Span::Compiler(_) => LineColumn { line: 0, column: 0 },
Span::Fallback(s) => s.start(),
}
}
#[cfg(span_locations)]
pub fn end(&self) -> LineColumn {
pub(crate) fn end(&self) -> LineColumn {
match self {
#[cfg(proc_macro_span)]
Span::Compiler(s) => {
let end = s.end();
LineColumn {
line: end.line(),
column: end.column().saturating_sub(1),
}
}
#[cfg(not(proc_macro_span))]
Span::Compiler(_) => LineColumn { line: 0, column: 0 },
Span::Fallback(s) => s.end(),
}
}
pub fn join(&self, other: Span) -> Option<Span> {
pub(crate) fn join(&self, other: Span) -> Option<Span> {
let ret = match (self, other) {
#[cfg(proc_macro_span)]
(Span::Compiler(a), Span::Compiler(b)) => Span::Compiler(a.join(b)?),
@@ -487,7 +517,7 @@ impl Span {
}
#[cfg(super_unstable)]
pub fn eq(&self, other: &Span) -> bool {
pub(crate) fn eq(&self, other: &Span) -> bool {
match (self, other) {
(Span::Compiler(a), Span::Compiler(b)) => a.eq(b),
(Span::Fallback(a), Span::Fallback(b)) => a.eq(b),
@@ -495,7 +525,7 @@ impl Span {
}
}
pub fn source_text(&self) -> Option<String> {
pub(crate) fn source_text(&self) -> Option<String> {
match self {
#[cfg(not(no_source_text))]
Span::Compiler(s) => s.source_text(),
@@ -550,7 +580,7 @@ pub(crate) enum Group {
}
impl Group {
pub fn new(delimiter: Delimiter, stream: TokenStream) -> Self {
pub(crate) fn new(delimiter: Delimiter, stream: TokenStream) -> Self {
match stream {
TokenStream::Compiler(tts) => {
let delimiter = match delimiter {
@@ -567,7 +597,7 @@ impl Group {
}
}
pub fn delimiter(&self) -> Delimiter {
pub(crate) fn delimiter(&self) -> Delimiter {
match self {
Group::Compiler(g) => match g.delimiter() {
proc_macro::Delimiter::Parenthesis => Delimiter::Parenthesis,
@@ -579,35 +609,35 @@ impl Group {
}
}
pub fn stream(&self) -> TokenStream {
pub(crate) fn stream(&self) -> TokenStream {
match self {
Group::Compiler(g) => TokenStream::Compiler(DeferredTokenStream::new(g.stream())),
Group::Fallback(g) => TokenStream::Fallback(g.stream()),
}
}
pub fn span(&self) -> Span {
pub(crate) fn span(&self) -> Span {
match self {
Group::Compiler(g) => Span::Compiler(g.span()),
Group::Fallback(g) => Span::Fallback(g.span()),
}
}
pub fn span_open(&self) -> Span {
pub(crate) fn span_open(&self) -> Span {
match self {
Group::Compiler(g) => Span::Compiler(g.span_open()),
Group::Fallback(g) => Span::Fallback(g.span_open()),
}
}
pub fn span_close(&self) -> Span {
pub(crate) fn span_close(&self) -> Span {
match self {
Group::Compiler(g) => Span::Compiler(g.span_close()),
Group::Fallback(g) => Span::Fallback(g.span_close()),
}
}
pub fn set_span(&mut self, span: Span) {
pub(crate) fn set_span(&mut self, span: Span) {
match (self, span) {
(Group::Compiler(g), Span::Compiler(s)) => g.set_span(s),
(Group::Fallback(g), Span::Fallback(s)) => g.set_span(s),
@@ -656,37 +686,29 @@ pub(crate) enum Ident {
impl Ident {
#[track_caller]
pub fn new_checked(string: &str, span: Span) -> Self {
pub(crate) fn new_checked(string: &str, span: Span) -> Self {
match span {
Span::Compiler(s) => Ident::Compiler(proc_macro::Ident::new(string, s)),
Span::Fallback(s) => Ident::Fallback(fallback::Ident::new_checked(string, s)),
}
}
pub fn new_unchecked(string: &str, span: fallback::Span) -> Self {
Ident::Fallback(fallback::Ident::new_unchecked(string, span))
}
#[track_caller]
pub fn new_raw_checked(string: &str, span: Span) -> Self {
pub(crate) fn new_raw_checked(string: &str, span: Span) -> Self {
match span {
Span::Compiler(s) => Ident::Compiler(proc_macro::Ident::new_raw(string, s)),
Span::Fallback(s) => Ident::Fallback(fallback::Ident::new_raw_checked(string, s)),
}
}
pub fn new_raw_unchecked(string: &str, span: fallback::Span) -> Self {
Ident::Fallback(fallback::Ident::new_raw_unchecked(string, span))
}
pub fn span(&self) -> Span {
pub(crate) fn span(&self) -> Span {
match self {
Ident::Compiler(t) => Span::Compiler(t.span()),
Ident::Fallback(t) => Span::Fallback(t.span()),
}
}
pub fn set_span(&mut self, span: Span) {
pub(crate) fn set_span(&mut self, span: Span) {
match (self, span) {
(Ident::Compiler(t), Span::Compiler(s)) => t.set_span(s),
(Ident::Fallback(t), Span::Fallback(s)) => t.set_span(s),
@@ -703,6 +725,12 @@ impl Ident {
}
}
impl From<fallback::Ident> for Ident {
fn from(inner: fallback::Ident) -> Self {
Ident::Fallback(inner)
}
}
impl PartialEq for Ident {
fn eq(&self, other: &Ident) -> bool {
match (self, other) {
@@ -753,7 +781,7 @@ pub(crate) enum Literal {
macro_rules! suffixed_numbers {
($($name:ident => $kind:ident,)*) => ($(
pub fn $name(n: $kind) -> Literal {
pub(crate) fn $name(n: $kind) -> Literal {
if inside_proc_macro() {
Literal::Compiler(proc_macro::Literal::$name(n))
} else {
@@ -765,7 +793,7 @@ macro_rules! suffixed_numbers {
macro_rules! unsuffixed_integers {
($($name:ident => $kind:ident,)*) => ($(
pub fn $name(n: $kind) -> Literal {
pub(crate) fn $name(n: $kind) -> Literal {
if inside_proc_macro() {
Literal::Compiler(proc_macro::Literal::$name(n))
} else {
@@ -776,9 +804,19 @@ macro_rules! unsuffixed_integers {
}
impl Literal {
pub unsafe fn from_str_unchecked(repr: &str) -> Self {
pub(crate) fn from_str_checked(repr: &str) -> Result<Self, LexError> {
if inside_proc_macro() {
Literal::Compiler(proc_macro::Literal::from_str(repr).expect("invalid literal"))
let literal = proc_macro::Literal::from_str_checked(repr)?;
Ok(Literal::Compiler(literal))
} else {
let literal = fallback::Literal::from_str_checked(repr)?;
Ok(Literal::Fallback(literal))
}
}
pub(crate) unsafe fn from_str_unchecked(repr: &str) -> Self {
if inside_proc_macro() {
Literal::Compiler(proc_macro::Literal::from_str_unchecked(repr))
} else {
Literal::Fallback(unsafe { fallback::Literal::from_str_unchecked(repr) })
}
@@ -817,7 +855,7 @@ impl Literal {
isize_unsuffixed => isize,
}
pub fn f32_unsuffixed(f: f32) -> Literal {
pub(crate) fn f32_unsuffixed(f: f32) -> Literal {
if inside_proc_macro() {
Literal::Compiler(proc_macro::Literal::f32_unsuffixed(f))
} else {
@@ -825,7 +863,7 @@ impl Literal {
}
}
pub fn f64_unsuffixed(f: f64) -> Literal {
pub(crate) fn f64_unsuffixed(f: f64) -> Literal {
if inside_proc_macro() {
Literal::Compiler(proc_macro::Literal::f64_unsuffixed(f))
} else {
@@ -833,23 +871,42 @@ impl Literal {
}
}
pub fn string(t: &str) -> Literal {
pub(crate) fn string(string: &str) -> Literal {
if inside_proc_macro() {
Literal::Compiler(proc_macro::Literal::string(t))
Literal::Compiler(proc_macro::Literal::string(string))
} else {
Literal::Fallback(fallback::Literal::string(t))
Literal::Fallback(fallback::Literal::string(string))
}
}
pub fn character(t: char) -> Literal {
pub(crate) fn character(ch: char) -> Literal {
if inside_proc_macro() {
Literal::Compiler(proc_macro::Literal::character(t))
Literal::Compiler(proc_macro::Literal::character(ch))
} else {
Literal::Fallback(fallback::Literal::character(t))
Literal::Fallback(fallback::Literal::character(ch))
}
}
pub fn byte_string(bytes: &[u8]) -> Literal {
pub(crate) fn byte_character(byte: u8) -> Literal {
if inside_proc_macro() {
Literal::Compiler({
#[cfg(not(no_literal_byte_character))]
{
proc_macro::Literal::byte_character(byte)
}
#[cfg(no_literal_byte_character)]
{
let fallback = fallback::Literal::byte_character(byte);
proc_macro::Literal::from_str_unchecked(&fallback.repr)
}
})
} else {
Literal::Fallback(fallback::Literal::byte_character(byte))
}
}
pub(crate) fn byte_string(bytes: &[u8]) -> Literal {
if inside_proc_macro() {
Literal::Compiler(proc_macro::Literal::byte_string(bytes))
} else {
@@ -857,14 +914,33 @@ impl Literal {
}
}
pub fn span(&self) -> Span {
pub(crate) fn c_string(string: &CStr) -> Literal {
if inside_proc_macro() {
Literal::Compiler({
#[cfg(not(no_literal_c_string))]
{
proc_macro::Literal::c_string(string)
}
#[cfg(no_literal_c_string)]
{
let fallback = fallback::Literal::c_string(string);
proc_macro::Literal::from_str_unchecked(&fallback.repr)
}
})
} else {
Literal::Fallback(fallback::Literal::c_string(string))
}
}
pub(crate) fn span(&self) -> Span {
match self {
Literal::Compiler(lit) => Span::Compiler(lit.span()),
Literal::Fallback(lit) => Span::Fallback(lit.span()),
}
}
pub fn set_span(&mut self, span: Span) {
pub(crate) fn set_span(&mut self, span: Span) {
match (self, span) {
(Literal::Compiler(lit), Span::Compiler(s)) => lit.set_span(s),
(Literal::Fallback(lit), Span::Fallback(s)) => lit.set_span(s),
@@ -873,7 +949,7 @@ impl Literal {
}
}
pub fn subspan<R: RangeBounds<usize>>(&self, range: R) -> Option<Span> {
pub(crate) fn subspan<R: RangeBounds<usize>>(&self, range: R) -> Option<Span> {
match self {
#[cfg(proc_macro_span)]
Literal::Compiler(lit) => lit.subspan(range).map(Span::Compiler),
@@ -897,20 +973,6 @@ impl From<fallback::Literal> for Literal {
}
}
impl FromStr for Literal {
type Err = LexError;
fn from_str(repr: &str) -> Result<Self, Self::Err> {
if inside_proc_macro() {
let literal = proc_macro::Literal::from_str(repr)?;
Ok(Literal::Compiler(literal))
} else {
let literal = fallback::Literal::from_str(repr)?;
Ok(Literal::Fallback(literal))
}
}
}
impl Display for Literal {
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
match self {
@@ -928,3 +990,14 @@ impl Debug for Literal {
}
}
}
#[cfg(span_locations)]
pub(crate) fn invalidate_current_thread_spans() {
if inside_proc_macro() {
panic!(
"proc_macro2::extra::invalidate_current_thread_spans is not available in procedural macros"
);
} else {
crate::fallback::invalidate_current_thread_spans();
}
}
tests/marker.rs
+1
View File
@@ -21,6 +21,7 @@ macro_rules! assert_impl {
$(
{
// Implemented for types that implement $marker.
#[allow(dead_code)]
trait IsNotImplemented {
fn assert_not_implemented() {}
}
tests/test.rs
+194 -48
View File
@@ -1,11 +1,14 @@
#![allow(
clippy::assertions_on_result_states,
clippy::items_after_statements,
clippy::needless_pass_by_value,
clippy::needless_raw_string_hashes,
clippy::non_ascii_literal,
clippy::octal_escapes
)]
use proc_macro2::{Ident, Literal, Punct, Spacing, Span, TokenStream, TokenTree};
use std::ffi::CStr;
use std::iter;
use std::str::{self, FromStr};
@@ -96,12 +99,22 @@ fn lifetime_invalid() {
#[test]
fn literal_string() {
assert_eq!(Literal::string("foo").to_string(), "\"foo\"");
assert_eq!(Literal::string("\"").to_string(), "\"\\\"\"");
assert_eq!(Literal::string("didn't").to_string(), "\"didn't\"");
assert_eq!(
Literal::string("a\00b\07c\08d\0e\0").to_string(),
"\"a\\x000b\\x007c\\08d\\0e\\0\"",
#[track_caller]
fn assert(literal: Literal, expected: &str) {
assert_eq!(literal.to_string(), expected.trim());
}
assert(Literal::string(""), r#" "" "#);
assert(Literal::string("aA"), r#" "aA" "#);
assert(Literal::string("\t"), r#" "\t" "#);
assert(Literal::string("❤"), r#" "❤" "#);
assert(Literal::string("'"), r#" "'" "#);
assert(Literal::string("\""), r#" "\"" "#);
assert(Literal::string("\0"), r#" "\0" "#);
assert(Literal::string("\u{1}"), r#" "\u{1}" "#);
assert(
Literal::string("a\00b\07c\08d\0e\0"),
r#" "a\x000b\x007c\08d\0e\0" "#,
);
"\"\\\r\n x\"".parse::<TokenStream>().unwrap();
@@ -132,16 +145,43 @@ fn literal_raw_string() {
.unwrap_err();
}
#[test]
fn literal_byte_character() {
#[track_caller]
fn assert(literal: Literal, expected: &str) {
assert_eq!(literal.to_string(), expected.trim());
}
assert(Literal::byte_character(b'a'), r#" b'a' "#);
assert(Literal::byte_character(b'\0'), r#" b'\0' "#);
assert(Literal::byte_character(b'\t'), r#" b'\t' "#);
assert(Literal::byte_character(b'\n'), r#" b'\n' "#);
assert(Literal::byte_character(b'\r'), r#" b'\r' "#);
assert(Literal::byte_character(b'\''), r#" b'\'' "#);
assert(Literal::byte_character(b'\\'), r#" b'\\' "#);
assert(Literal::byte_character(b'\x1f'), r#" b'\x1F' "#);
assert(Literal::byte_character(b'"'), r#" b'"' "#);
}
#[test]
fn literal_byte_string() {
assert_eq!(Literal::byte_string(b"").to_string(), "b\"\"");
assert_eq!(
Literal::byte_string(b"\0\t\n\r\"\\2\x10").to_string(),
"b\"\\0\\t\\n\\r\\\"\\\\2\\x10\"",
);
assert_eq!(
Literal::byte_string(b"a\00b\07c\08d\0e\0").to_string(),
"b\"a\\x000b\\x007c\\08d\\0e\\0\"",
#[track_caller]
fn assert(literal: Literal, expected: &str) {
assert_eq!(literal.to_string(), expected.trim());
}
assert(Literal::byte_string(b""), r#" b"" "#);
assert(Literal::byte_string(b"\0"), r#" b"\0" "#);
assert(Literal::byte_string(b"\t"), r#" b"\t" "#);
assert(Literal::byte_string(b"\n"), r#" b"\n" "#);
assert(Literal::byte_string(b"\r"), r#" b"\r" "#);
assert(Literal::byte_string(b"\""), r#" b"\"" "#);
assert(Literal::byte_string(b"\\"), r#" b"\\" "#);
assert(Literal::byte_string(b"\x1f"), r#" b"\x1F" "#);
assert(Literal::byte_string(b"'"), r#" b"'" "#);
assert(
Literal::byte_string(b"a\00b\07c\08d\0e\0"),
r#" b"a\x000b\x007c\08d\0e\0" "#,
);
"b\"\\\r\n x\"".parse::<TokenStream>().unwrap();
@@ -152,6 +192,41 @@ fn literal_byte_string() {
#[test]
fn literal_c_string() {
#[track_caller]
fn assert(literal: Literal, expected: &str) {
assert_eq!(literal.to_string(), expected.trim());
}
assert(Literal::c_string(<&CStr>::default()), r#" c"" "#);
assert(
Literal::c_string(CStr::from_bytes_with_nul(b"aA\0").unwrap()),
r#" c"aA" "#,
);
assert(
Literal::c_string(CStr::from_bytes_with_nul(b"aA\0").unwrap()),
r#" c"aA" "#,
);
assert(
Literal::c_string(CStr::from_bytes_with_nul(b"\t\0").unwrap()),
r#" c"\t" "#,
);
assert(
Literal::c_string(CStr::from_bytes_with_nul(b"\xE2\x9D\xA4\0").unwrap()),
r#" c"❤" "#,
);
assert(
Literal::c_string(CStr::from_bytes_with_nul(b"'\0").unwrap()),
r#" c"'" "#,
);
assert(
Literal::c_string(CStr::from_bytes_with_nul(b"\"\0").unwrap()),
r#" c"\"" "#,
);
assert(
Literal::c_string(CStr::from_bytes_with_nul(b"\x7F\xFF\xFE\xCC\xB3\0").unwrap()),
r#" c"\u{7f}\xFF\xFE\u{333}" "#,
);
let strings = r###"
c"hello\x80我叫\u{1F980}" // from the RFC
cr"\"
@@ -188,49 +263,80 @@ fn literal_c_string() {
#[test]
fn literal_character() {
assert_eq!(Literal::character('x').to_string(), "'x'");
assert_eq!(Literal::character('\'').to_string(), "'\\''");
assert_eq!(Literal::character('"').to_string(), "'\"'");
#[track_caller]
fn assert(literal: Literal, expected: &str) {
assert_eq!(literal.to_string(), expected.trim());
}
assert(Literal::character('a'), r#" 'a' "#);
assert(Literal::character('\t'), r#" '\t' "#);
assert(Literal::character('❤'), r#" '❤' "#);
assert(Literal::character('\''), r#" '\'' "#);
assert(Literal::character('"'), r#" '"' "#);
assert(Literal::character('\0'), r#" '\0' "#);
assert(Literal::character('\u{1}'), r#" '\u{1}' "#);
}
#[test]
fn literal_integer() {
assert_eq!(Literal::u8_suffixed(10).to_string(), "10u8");
assert_eq!(Literal::u16_suffixed(10).to_string(), "10u16");
assert_eq!(Literal::u32_suffixed(10).to_string(), "10u32");
assert_eq!(Literal::u64_suffixed(10).to_string(), "10u64");
assert_eq!(Literal::u128_suffixed(10).to_string(), "10u128");
assert_eq!(Literal::usize_suffixed(10).to_string(), "10usize");
#[track_caller]
fn assert(literal: Literal, expected: &str) {
assert_eq!(literal.to_string(), expected);
}
assert_eq!(Literal::i8_suffixed(10).to_string(), "10i8");
assert_eq!(Literal::i16_suffixed(10).to_string(), "10i16");
assert_eq!(Literal::i32_suffixed(10).to_string(), "10i32");
assert_eq!(Literal::i64_suffixed(10).to_string(), "10i64");
assert_eq!(Literal::i128_suffixed(10).to_string(), "10i128");
assert_eq!(Literal::isize_suffixed(10).to_string(), "10isize");
assert(Literal::u8_suffixed(10), "10u8");
assert(Literal::u16_suffixed(10), "10u16");
assert(Literal::u32_suffixed(10), "10u32");
assert(Literal::u64_suffixed(10), "10u64");
assert(Literal::u128_suffixed(10), "10u128");
assert(Literal::usize_suffixed(10), "10usize");
assert_eq!(Literal::u8_unsuffixed(10).to_string(), "10");
assert_eq!(Literal::u16_unsuffixed(10).to_string(), "10");
assert_eq!(Literal::u32_unsuffixed(10).to_string(), "10");
assert_eq!(Literal::u64_unsuffixed(10).to_string(), "10");
assert_eq!(Literal::u128_unsuffixed(10).to_string(), "10");
assert_eq!(Literal::usize_unsuffixed(10).to_string(), "10");
assert(Literal::i8_suffixed(10), "10i8");
assert(Literal::i16_suffixed(10), "10i16");
assert(Literal::i32_suffixed(10), "10i32");
assert(Literal::i64_suffixed(10), "10i64");
assert(Literal::i128_suffixed(10), "10i128");
assert(Literal::isize_suffixed(10), "10isize");
assert_eq!(Literal::i8_unsuffixed(10).to_string(), "10");
assert_eq!(Literal::i16_unsuffixed(10).to_string(), "10");
assert_eq!(Literal::i32_unsuffixed(10).to_string(), "10");
assert_eq!(Literal::i64_unsuffixed(10).to_string(), "10");
assert_eq!(Literal::i128_unsuffixed(10).to_string(), "10");
assert_eq!(Literal::isize_unsuffixed(10).to_string(), "10");
assert(Literal::u8_unsuffixed(10), "10");
assert(Literal::u16_unsuffixed(10), "10");
assert(Literal::u32_unsuffixed(10), "10");
assert(Literal::u64_unsuffixed(10), "10");
assert(Literal::u128_unsuffixed(10), "10");
assert(Literal::usize_unsuffixed(10), "10");
assert(Literal::i8_unsuffixed(10), "10");
assert(Literal::i16_unsuffixed(10), "10");
assert(Literal::i32_unsuffixed(10), "10");
assert(Literal::i64_unsuffixed(10), "10");
assert(Literal::i128_unsuffixed(10), "10");
assert(Literal::isize_unsuffixed(10), "10");
assert(Literal::i32_suffixed(-10), "-10i32");
assert(Literal::i32_unsuffixed(-10), "-10");
}
#[test]
fn literal_float() {
assert_eq!(Literal::f32_suffixed(10.0).to_string(), "10f32");
assert_eq!(Literal::f64_suffixed(10.0).to_string(), "10f64");
#[track_caller]
fn assert(literal: Literal, expected: &str) {
assert_eq!(literal.to_string(), expected);
}
assert_eq!(Literal::f32_unsuffixed(10.0).to_string(), "10.0");
assert_eq!(Literal::f64_unsuffixed(10.0).to_string(), "10.0");
assert(Literal::f32_suffixed(10.0), "10f32");
assert(Literal::f32_suffixed(-10.0), "-10f32");
assert(Literal::f64_suffixed(10.0), "10f64");
assert(Literal::f64_suffixed(-10.0), "-10f64");
assert(Literal::f32_unsuffixed(10.0), "10.0");
assert(Literal::f32_unsuffixed(-10.0), "-10.0");
assert(Literal::f64_unsuffixed(10.0), "10.0");
assert(Literal::f64_unsuffixed(-10.0), "-10.0");
assert(
Literal::f64_unsuffixed(1e100),
"10000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000.0",
);
}
#[test]
@@ -248,9 +354,13 @@ fn literal_suffix() {
assert_eq!(token_count("1._m"), 3);
assert_eq!(token_count("\"\"s"), 1);
assert_eq!(token_count("r\"\"r"), 1);
assert_eq!(token_count("r#\"\"#r"), 1);
assert_eq!(token_count("b\"\"b"), 1);
assert_eq!(token_count("br\"\"br"), 1);
assert_eq!(token_count("r#\"\"#r"), 1);
assert_eq!(token_count("br#\"\"#br"), 1);
assert_eq!(token_count("c\"\"c"), 1);
assert_eq!(token_count("cr\"\"cr"), 1);
assert_eq!(token_count("cr#\"\"#cr"), 1);
assert_eq!(token_count("'c'c"), 1);
assert_eq!(token_count("b'b'b"), 1);
assert_eq!(token_count("0E"), 1);
@@ -378,7 +488,7 @@ fn roundtrip() {
roundtrip("'a");
roundtrip("'_");
roundtrip("'static");
roundtrip("'\\u{10__FFFF}'");
roundtrip(r"'\u{10__FFFF}'");
roundtrip("\"\\u{10_F0FF__}foo\\u{1_0_0_0__}\"");
}
@@ -401,6 +511,7 @@ fn fail() {
fail("\"\\\r \""); // backslash carriage return
fail("'aa'aa");
fail("br##\"\"#");
fail("cr##\"\"#");
fail("\"\\\n\u{85}\r\"");
}
@@ -430,7 +541,6 @@ testing 123
}
#[cfg(procmacro2_semver_exempt)]
#[cfg(not(nightly))]
#[test]
fn default_span() {
let start = Span::call_site().start();
@@ -757,3 +867,39 @@ fn byte_order_mark() {
let string = "foo\u{feff}";
string.parse::<TokenStream>().unwrap_err();
}
#[cfg(span_locations)]
fn create_span() -> proc_macro2::Span {
let tts: TokenStream = "1".parse().unwrap();
match tts.into_iter().next().unwrap() {
TokenTree::Literal(literal) => literal.span(),
_ => unreachable!(),
}
}
#[cfg(span_locations)]
#[test]
fn test_invalidate_current_thread_spans() {
let actual = format!("{:#?}", create_span());
assert_eq!(actual, "bytes(1..2)");
let actual = format!("{:#?}", create_span());
assert_eq!(actual, "bytes(3..4)");
proc_macro2::extra::invalidate_current_thread_spans();
let actual = format!("{:#?}", create_span());
// Test that span offsets have been reset after the call
// to invalidate_current_thread_spans()
assert_eq!(actual, "bytes(1..2)");
}
#[cfg(span_locations)]
#[test]
#[should_panic(expected = "Invalid span with no related FileInfo!")]
fn test_use_span_after_invalidation() {
let span = create_span();
proc_macro2::extra::invalidate_current_thread_spans();
span.source_text();
}
tests/test_size.rs
+53 -14
View File
@@ -1,42 +1,81 @@
#![allow(unused_attributes)]
extern crate proc_macro;
use std::mem;
#[rustversion::attr(before(1.32), ignore)]
#[rustversion::attr(before(1.64), ignore = "requires Rust 1.64+")]
#[cfg_attr(not(target_pointer_width = "64"), ignore = "only applicable to 64-bit")]
#[cfg_attr(randomize_layout, ignore = "disabled due to randomized layout")]
#[test]
fn test_proc_macro_span_size() {
fn test_proc_macro_size() {
assert_eq!(mem::size_of::<proc_macro::Span>(), 4);
assert_eq!(mem::size_of::<Option<proc_macro::Span>>(), 4);
assert_eq!(mem::size_of::<proc_macro::Group>(), 20);
assert_eq!(mem::size_of::<proc_macro::Ident>(), 12);
assert_eq!(mem::size_of::<proc_macro::Punct>(), 8);
assert_eq!(mem::size_of::<proc_macro::Literal>(), 16);
assert_eq!(mem::size_of::<proc_macro::TokenStream>(), 4);
}
#[cfg_attr(not(all(not(wrap_proc_macro), not(span_locations))), ignore)]
#[cfg_attr(not(target_pointer_width = "64"), ignore = "only applicable to 64-bit")]
#[cfg_attr(randomize_layout, ignore = "disabled due to randomized layout")]
#[cfg_attr(wrap_proc_macro, ignore = "wrapper mode")]
#[cfg_attr(span_locations, ignore = "span locations are on")]
#[test]
fn test_proc_macro2_fallback_span_size_without_locations() {
fn test_proc_macro2_fallback_size_without_locations() {
assert_eq!(mem::size_of::<proc_macro2::Span>(), 0);
assert_eq!(mem::size_of::<Option<proc_macro2::Span>>(), 1);
assert_eq!(mem::size_of::<proc_macro2::Group>(), 16);
assert_eq!(mem::size_of::<proc_macro2::Ident>(), 24);
assert_eq!(mem::size_of::<proc_macro2::Punct>(), 8);
assert_eq!(mem::size_of::<proc_macro2::Literal>(), 24);
assert_eq!(mem::size_of::<proc_macro2::TokenStream>(), 8);
}
#[cfg_attr(not(all(not(wrap_proc_macro), span_locations)), ignore)]
#[cfg_attr(not(target_pointer_width = "64"), ignore = "only applicable to 64-bit")]
#[cfg_attr(randomize_layout, ignore = "disabled due to randomized layout")]
#[cfg_attr(wrap_proc_macro, ignore = "wrapper mode")]
#[cfg_attr(not(span_locations), ignore = "span locations are off")]
#[test]
fn test_proc_macro2_fallback_span_size_with_locations() {
fn test_proc_macro2_fallback_size_with_locations() {
assert_eq!(mem::size_of::<proc_macro2::Span>(), 8);
assert_eq!(mem::size_of::<Option<proc_macro2::Span>>(), 12);
assert_eq!(mem::size_of::<proc_macro2::Group>(), 24);
assert_eq!(mem::size_of::<proc_macro2::Ident>(), 32);
assert_eq!(mem::size_of::<proc_macro2::Punct>(), 16);
assert_eq!(mem::size_of::<proc_macro2::Literal>(), 32);
assert_eq!(mem::size_of::<proc_macro2::TokenStream>(), 8);
}
#[rustversion::attr(before(1.32), ignore)]
#[rustversion::attr(
since(1.32),
cfg_attr(not(all(wrap_proc_macro, not(span_locations))), ignore)
)]
#[rustversion::attr(before(1.71), ignore = "requires Rust 1.71+")]
#[cfg_attr(not(target_pointer_width = "64"), ignore = "only applicable to 64-bit")]
#[cfg_attr(randomize_layout, ignore = "disabled due to randomized layout")]
#[cfg_attr(not(wrap_proc_macro), ignore = "fallback mode")]
#[cfg_attr(span_locations, ignore = "span locations are on")]
#[test]
fn test_proc_macro2_wrapper_span_size_without_locations() {
fn test_proc_macro2_wrapper_size_without_locations() {
assert_eq!(mem::size_of::<proc_macro2::Span>(), 4);
assert_eq!(mem::size_of::<Option<proc_macro2::Span>>(), 8);
assert_eq!(mem::size_of::<proc_macro2::Group>(), 24);
assert_eq!(mem::size_of::<proc_macro2::Ident>(), 24);
assert_eq!(mem::size_of::<proc_macro2::Punct>(), 12);
assert_eq!(mem::size_of::<proc_macro2::Literal>(), 24);
assert_eq!(mem::size_of::<proc_macro2::TokenStream>(), 32);
}
#[cfg_attr(not(all(wrap_proc_macro, span_locations)), ignore)]
#[rustversion::attr(before(1.65), ignore = "requires Rust 1.65+")]
#[cfg_attr(not(target_pointer_width = "64"), ignore = "only applicable to 64-bit")]
#[cfg_attr(randomize_layout, ignore = "disabled due to randomized layout")]
#[cfg_attr(not(wrap_proc_macro), ignore = "fallback mode")]
#[cfg_attr(not(span_locations), ignore = "span locations are off")]
#[test]
fn test_proc_macro2_wrapper_span_size_with_locations() {
fn test_proc_macro2_wrapper_size_with_locations() {
assert_eq!(mem::size_of::<proc_macro2::Span>(), 12);
assert_eq!(mem::size_of::<Option<proc_macro2::Span>>(), 12);
assert_eq!(mem::size_of::<proc_macro2::Group>(), 32);
assert_eq!(mem::size_of::<proc_macro2::Ident>(), 32);
assert_eq!(mem::size_of::<proc_macro2::Punct>(), 20);
assert_eq!(mem::size_of::<proc_macro2::Literal>(), 32);
assert_eq!(mem::size_of::<proc_macro2::TokenStream>(), 32);
}
tests/ui/Cargo.toml
+15
View File
@@ -0,0 +1,15 @@
[package]
name = "proc-macro2-ui-test"
version = "0.0.0"
authors = ["David Tolnay <dtolnay@gmail.com>"]
edition = "2018"
publish = false
[[test]]
name = "compiletest"
path = "compiletest.rs"
[dev-dependencies]
proc-macro2 = { path = "../.." }
rustversion = "1.0"
trybuild = { version = "1.0.49", features = ["diff"] }
tests/ui/compiletest.rs
+7
View File
@@ -0,0 +1,7 @@
#[rustversion::attr(not(nightly), ignore = "requires nightly")]
#[rustversion::attr(nightly, cfg_attr(miri, ignore = "incompatible with miri"))]
#[test]
fn ui() {
let t = trybuild::TestCases::new();
t.compile_fail("test-*.rs");
}
tests/ui/test-not-send.rs
+6
View File
@@ -0,0 +1,6 @@
use proc_macro2::Span;
fn main() {
fn requires_send<T: Send>() {}
requires_send::<Span>();
}
tests/ui/test-not-send.stderr
+50
View File
@@ -0,0 +1,50 @@
error[E0277]: `proc_macro::Span` cannot be sent between threads safely
--> test-not-send.rs:5:21
|
5 | requires_send::<Span>();
| ^^^^ `proc_macro::Span` cannot be sent between threads safely
|
= help: within `Span`, the trait `Send` is not implemented for `proc_macro::Span`
note: required because it appears within the type `proc_macro2::imp::Span`
--> $WORKSPACE/src/wrapper.rs
|
| pub(crate) enum Span {
| ^^^^
note: required because it appears within the type `Span`
--> $WORKSPACE/src/lib.rs
|
| pub struct Span {
| ^^^^
note: required by a bound in `requires_send`
--> test-not-send.rs:4:25
|
4 | fn requires_send<T: Send>() {}
| ^^^^ required by this bound in `requires_send`
error[E0277]: `Rc<()>` cannot be sent between threads safely
--> test-not-send.rs:5:21
|
5 | requires_send::<Span>();
| ^^^^ `Rc<()>` cannot be sent between threads safely
|
= help: within `Span`, the trait `Send` is not implemented for `Rc<()>`
note: required because it appears within the type `PhantomData<Rc<()>>`
--> $RUST/core/src/marker.rs
|
| pub struct PhantomData<T: ?Sized>;
| ^^^^^^^^^^^
note: required because it appears within the type `proc_macro2::marker::ProcMacroAutoTraits`
--> $WORKSPACE/src/marker.rs
|
| pub(crate) struct ProcMacroAutoTraits(PhantomData<Rc<()>>);
| ^^^^^^^^^^^^^^^^^^^
note: required because it appears within the type `Span`
--> $WORKSPACE/src/lib.rs
|
| pub struct Span {
| ^^^^
note: required by a bound in `requires_send`
--> test-not-send.rs:4:25
|
4 | fn requires_send<T: Send>() {}
| ^^^^ required by this bound in `requires_send`