openharmony/third_party_rust_proc-macro2
1
0
Fork 0
mirror of https://github.com/openharmony/third_party_rust_proc-macro2.git synced 2026-07-01 20:24:03 -04:00
Code Issues Packages Projects Releases Wiki Activity

!14 merge master into master

Browse Source 
proc-macro2升级到1.0.105版本

Created-by: dragonswordy
Commit-by: ljy9810
Merged-by: openharmony_ci
Description: ### 一、内容说明（相关的Issue）
https://gitcode.com/openharmony/third_party_rust_unicode-ident/issues/2


### 二、建议测试周期和提测地址  
  建议测试完成时间：xxxx.xx.xx  
  投产上线时间：xxxx.xx.xx  
  提测地址：CI环境/压测环境  
  测试账号：  

### 三、变更内容
  * 3.1 关联PR列表

  * 3.2 数据库和部署说明  
    1. 常规更新 
    2. 重启unicorn
    3. 重启sidekiq
    4. 迁移任务：是否有迁移任务，没有写 "无"
    5. rake脚本：`bundle exec xxx RAILS_ENV = production`；没有写 "无"

  * 3.4 其他技术优化内容（做了什么，变更了什么）
    - 重构了 xxxx 代码
    - xxxx 算法优化


  * 3.5 废弃通知（什么字段、方法弃用？）



  * 3.6  后向不兼容变更（是否有无法向后兼容的变更？）


  
### 四、研发自测点（自测哪些？冒烟用例全部自测？）
  自测测试结论：


### 五、测试关注点（需要提醒QA重点关注的、可能会忽略的地方）
  检查点：

| 需求名称 | 是否影响xx公共模块 | 是否需要xx功能 | 需求升级是否依赖其他子产品 |
|------|------------|----------|---------------|
| xxx  | 否          | 需要       | 不需要           |
|      |            |          |               |

  接口测试：

  性能测试：

  并发测试：

  其他：



See merge request: openharmony/third_party_rust_proc-macro2!14
This commit is contained in:
openharmony_ci
2026-03-18 18:59:07 +08:00
parent 9d11d263f8 f77d53ac54
commit 80c4acdf1a
28 changed files with 1506 additions and 367 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/workflows/ci.yml
+23 -17
View File
@@ -24,10 +24,10 @@ jobs:
strategy:
fail-fast: false
matrix:
rust: [1.63.0, stable, beta]
rust: [1.80.0, stable, beta]
timeout-minutes: 45
steps:
- uses: actions/checkout@v4
- uses: actions/checkout@v6
- uses: dtolnay/rust-toolchain@master
with:
toolchain: ${{matrix.rust}}
@@ -51,7 +51,7 @@ jobs:
runs-on: ubuntu-latest
timeout-minutes: 45
steps:
- uses: actions/checkout@v4
- uses: actions/checkout@v6
- uses: dtolnay/rust-toolchain@nightly
with:
components: rust-src
@@ -77,7 +77,7 @@ jobs:
run: cargo test
env:
RUSTFLAGS: -Z allow-features= --cfg procmacro2_backtrace ${{env.RUSTFLAGS}}
- uses: actions/upload-artifact@v4
- uses: actions/upload-artifact@v6
if: always()
with:
name: Cargo.lock
@@ -91,7 +91,7 @@ jobs:
runs-on: ubuntu-latest
timeout-minutes: 45
steps:
- uses: actions/checkout@v4
- uses: actions/checkout@v6
- uses: dtolnay/rust-toolchain@nightly
with:
components: rust-src
@@ -101,14 +101,14 @@ jobs:
- run: cargo test --test test_size --no-default-features --features span-locations
msrv:
name: Rust 1.56.0
name: Rust 1.68.0
needs: pre_ci
if: needs.pre_ci.outputs.continue
runs-on: ubuntu-latest
timeout-minutes: 45
steps:
- uses: actions/checkout@v4
- uses: dtolnay/rust-toolchain@1.56.0
- uses: actions/checkout@v6
- uses: dtolnay/rust-toolchain@1.68.0
with:
components: rust-src
- run: cargo check
@@ -130,7 +130,7 @@ jobs:
runs-on: ubuntu-latest
timeout-minutes: 45
steps:
- uses: actions/checkout@v4
- uses: actions/checkout@v6
- uses: dtolnay/rust-toolchain@nightly
- run: cargo generate-lockfile -Z minimal-versions
- run: cargo check --locked
@@ -142,11 +142,13 @@ jobs:
runs-on: ubuntu-latest
timeout-minutes: 45
steps:
- uses: actions/checkout@v4
- uses: actions/checkout@v6
- uses: dtolnay/rust-toolchain@nightly
with:
target: wasm32-unknown-unknown
components: rust-src
- name: Ignore WebAssembly linker warning
run: echo RUSTFLAGS=${RUSTFLAGS}\ -Alinker_messages >> $GITHUB_ENV
- run: cargo test --target wasm32-unknown-unknown --no-run
fuzz:
@@ -156,7 +158,7 @@ jobs:
runs-on: ubuntu-latest
timeout-minutes: 45
steps:
- uses: actions/checkout@v4
- uses: actions/checkout@v6
- uses: dtolnay/rust-toolchain@nightly
with:
components: rust-src
@@ -165,8 +167,12 @@ jobs:
- run: cargo check --no-default-features --features afl
working-directory: fuzz
- uses: dtolnay/install@honggfuzz
- run: sudo apt-get update # https://github.com/actions/runner-images/issues/8953
- run: sudo apt-get install binutils-dev libunwind-dev
- name: Run apt install binutils-dev libunwind-dev
run: |
sudo sed -i 's/^update_initramfs=yes$/update_initramfs=no/' /etc/initramfs-tools/update-initramfs.conf
sudo rm -f /var/lib/man-db/auto-update
sudo apt-get update
sudo apt-get install binutils-dev libunwind-dev
- run: cargo hfuzz build --no-default-features --features honggfuzz
working-directory: fuzz
@@ -179,7 +185,7 @@ jobs:
env:
RUSTDOCFLAGS: -Dwarnings
steps:
- uses: actions/checkout@v4
- uses: actions/checkout@v6
- uses: dtolnay/rust-toolchain@nightly
with:
components: rust-src
@@ -192,7 +198,7 @@ jobs:
if: github.event_name != 'pull_request'
timeout-minutes: 45
steps:
- uses: actions/checkout@v4
- uses: actions/checkout@v6
- uses: dtolnay/rust-toolchain@nightly
with:
components: clippy, rust-src
@@ -206,7 +212,7 @@ jobs:
runs-on: ubuntu-latest
timeout-minutes: 45
steps:
- uses: actions/checkout@v4
- uses: actions/checkout@v6
- uses: dtolnay/rust-toolchain@miri
- run: cargo miri setup
- run: cargo miri test
@@ -219,7 +225,7 @@ jobs:
if: github.event_name != 'pull_request'
timeout-minutes: 45
steps:
- uses: actions/checkout@v4
- uses: actions/checkout@v6
- uses: dtolnay/rust-toolchain@stable
- uses: dtolnay/install@cargo-outdated
- run: cargo outdated --workspace --exit-code 1
.gitignore
+2 -3
View File
@@ -1,3 +1,2 @@
/target
**/*.rs.bk
Cargo.lock
/target/
/Cargo.lock
BUILD.gn
+1 -1
View File
@@ -20,7 +20,7 @@ ohos_cargo_crate("lib") {
sources = [ "src/lib.rs" ]
edition = "2021"
cargo_pkg_version = "1.0.92"
cargo_pkg_version = "1.0.105"
cargo_pkg_authors =
"David Tolnay <dtolnay@gmail.com>, Alex Crichton <alex@alexcrichton.com>"
cargo_pkg_name = "proc-macro2"
Cargo.toml
+12 -7
View File
@@ -1,6 +1,6 @@
[package]
name = "proc-macro2"
version = "1.0.92"
version = "1.0.105"
authors = ["David Tolnay <dtolnay@gmail.com>", "Alex Crichton <alex@alexcrichton.com>"]
autobenches = false
categories = ["development-tools::procedural-macro-helpers"]
@@ -10,12 +10,20 @@ edition = "2021"
keywords = ["macros", "syn"]
license = "MIT OR Apache-2.0"
repository = "https://github.com/dtolnay/proc-macro2"
rust-version = "1.56"
rust-version = "1.68"
[package.metadata.docs.rs]
rustc-args = ["--cfg", "procmacro2_semver_exempt"]
rustdoc-args = ["--cfg", "procmacro2_semver_exempt", "--generate-link-to-definition"]
rustc-args = ["--cfg=procmacro2_semver_exempt"]
targets = ["x86_64-unknown-linux-gnu"]
rustdoc-args = [
"--cfg=procmacro2_semver_exempt",
"--generate-link-to-definition",
"--generate-macro-expansion",
"--extern-html-root-url=core=https://doc.rust-lang.org",
"--extern-html-root-url=alloc=https://doc.rust-lang.org",
"--extern-html-root-url=std=https://doc.rust-lang.org",
"--extern-html-root-url=proc_macro=https://doc.rust-lang.org",
]
[package.metadata.playground]
features = ["span-locations"]
@@ -41,9 +49,6 @@ span-locations = []
# This feature no longer means anything.
nightly = []
[lib]
doc-scrape-examples = false
[workspace]
members = ["benches/bench-libproc-macro", "tests/ui"]
README.OpenSource
+5 -2
View File
@@ -3,9 +3,12 @@
"Name": "proc-macro2",
"License": "Apache License V2.0, MIT",
"License File": "LICENSE-APACHE, LICENSE-MIT",
"Version Number": "1.0.92",
"Version Number": "1.0.105",
"Owner": "fangting12@huawei.com",
"Upstream URL": "https://github.com/dtolnay/proc-macro2",
"Description": "A Rust library that provides support for error handling in procedural macros."
"Description": "A Rust library that provides support for error handling in procedural macros.",
"Dependencies": [
"unicode-ident"
]
}
]
README.md
+1 -1
View File
@@ -62,7 +62,7 @@ proc-macro2 by default.
To opt into the additional APIs available in the most recent nightly compiler,
the `procmacro2_semver_exempt` config flag must be passed to rustc. We will
polyfill those nightly-only APIs back to Rust 1.56.0. As these are unstable APIs
polyfill those nightly-only APIs back to Rust 1.68.0. As these are unstable APIs
that track the nightly compiler, minor versions of proc-macro2 may make breaking
changes to them at any time.
benches/bench-libproc-macro/Cargo.toml
+1 -1
View File
@@ -2,7 +2,7 @@
name = "bench-libproc-macro"
version = "0.0.0"
authors = ["David Tolnay <dtolnay@gmail.com>"]
edition = "2018"
edition = "2021"
publish = false
[lib]
build.rs
+55 -23
View File
@@ -1,5 +1,6 @@
#![allow(unknown_lints)]
#![allow(unexpected_cfgs)]
#![allow(clippy::uninlined_format_args)]
use std::env;
use std::ffi::OsString;
@@ -20,7 +21,10 @@ fn main() {
println!("cargo:rustc-check-cfg=cfg(no_literal_c_string)");
println!("cargo:rustc-check-cfg=cfg(no_source_text)");
println!("cargo:rustc-check-cfg=cfg(proc_macro_span)");
println!("cargo:rustc-check-cfg=cfg(proc_macro_span_file)");
println!("cargo:rustc-check-cfg=cfg(proc_macro_span_location)");
println!("cargo:rustc-check-cfg=cfg(procmacro2_backtrace)");
println!("cargo:rustc-check-cfg=cfg(procmacro2_build_probe)");
println!("cargo:rustc-check-cfg=cfg(procmacro2_nightly_testing)");
println!("cargo:rustc-check-cfg=cfg(procmacro2_semver_exempt)");
println!("cargo:rustc-check-cfg=cfg(randomize_layout)");
@@ -29,8 +33,7 @@ fn main() {
println!("cargo:rustc-check-cfg=cfg(wrap_proc_macro)");
}
let docs_rs = env::var_os("DOCS_RS").is_some();
let semver_exempt = cfg!(procmacro2_semver_exempt) || docs_rs;
let semver_exempt = cfg!(procmacro2_semver_exempt);
if semver_exempt {
// https://github.com/dtolnay/proc-macro2/issues/147
println!("cargo:rustc-cfg=procmacro2_semver_exempt");
@@ -68,18 +71,16 @@ fn main() {
return;
}
println!("cargo:rerun-if-changed=build/probe.rs");
let proc_macro_span;
let consider_rustc_bootstrap;
if compile_probe(false) {
if compile_probe_unstable("proc_macro_span", false) {
// This is a nightly or dev compiler, so it supports unstable features
// regardless of RUSTC_BOOTSTRAP. No need to rerun build script if
// RUSTC_BOOTSTRAP is changed.
proc_macro_span = true;
consider_rustc_bootstrap = false;
} else if let Some(rustc_bootstrap) = env::var_os("RUSTC_BOOTSTRAP") {
if compile_probe(true) {
if compile_probe_unstable("proc_macro_span", true) {
// This is a stable or beta compiler for which the user has set
// RUSTC_BOOTSTRAP to turn on unstable features. Rerun build script
// if they change it.
@@ -116,13 +117,25 @@ fn main() {
}
if proc_macro_span {
// Enable non-dummy behavior of Span::start and Span::end methods which
// requires an unstable compiler feature. Enabled when building with
// nightly, unless `-Z allow-feature` in RUSTFLAGS disallows unstable
// features.
// Enable non-dummy behavior of Span::byte_range and Span::join methods
// which requires an unstable compiler feature. Enabled when building
// with nightly, unless `-Z allow-feature` in RUSTFLAGS disallows
// unstable features.
println!("cargo:rustc-cfg=proc_macro_span");
}
if proc_macro_span || (rustc >= 88 && compile_probe_stable("proc_macro_span_location")) {
// Enable non-dummy behavior of Span::start and Span::end methods on
// Rust 1.88+.
println!("cargo:rustc-cfg=proc_macro_span_location");
}
if proc_macro_span || (rustc >= 88 && compile_probe_stable("proc_macro_span_file")) {
// Enable non-dummy behavior of Span::file and Span::local_file methods
// on Rust 1.88+.
println!("cargo:rustc-cfg=proc_macro_span_file");
}
if semver_exempt && proc_macro_span {
// Implement the semver exempt API in terms of the nightly-only
// proc_macro API.
@@ -134,22 +147,31 @@ fn main() {
}
}
fn compile_probe(rustc_bootstrap: bool) -> bool {
if env::var_os("RUSTC_STAGE").is_some() {
// We are running inside rustc bootstrap. This is a highly non-standard
// environment with issues such as:
//
// https://github.com/rust-lang/cargo/issues/11138
// https://github.com/rust-lang/rust/issues/114839
//
// Let's just not use nightly features here.
return false;
}
fn compile_probe_unstable(feature: &str, rustc_bootstrap: bool) -> bool {
// RUSTC_STAGE indicates that this crate is being compiled as a dependency
// of a multistage rustc bootstrap. This environment uses Cargo in a highly
// non-standard way with issues such as:
//
// https://github.com/rust-lang/cargo/issues/11138
// https://github.com/rust-lang/rust/issues/114839
//
env::var_os("RUSTC_STAGE").is_none() && do_compile_probe(feature, rustc_bootstrap)
}
fn compile_probe_stable(feature: &str) -> bool {
env::var_os("RUSTC_STAGE").is_some() || do_compile_probe(feature, true)
}
fn do_compile_probe(feature: &str, rustc_bootstrap: bool) -> bool {
println!("cargo:rerun-if-changed=src/probe/{}.rs", feature);
let rustc = cargo_env_var("RUSTC");
let out_dir = cargo_env_var("OUT_DIR");
let out_subdir = Path::new(&out_dir).join("probe");
let probefile = Path::new("build").join("probe.rs");
let probefile = Path::new("src")
.join("probe")
.join(feature)
.with_extension("rs");
if let Err(err) = fs::create_dir(&out_subdir) {
if err.kind() != ErrorKind::AlreadyExists {
@@ -173,6 +195,7 @@ fn compile_probe(rustc_bootstrap: bool) -> bool {
}
cmd.stderr(Stdio::null())
.arg("--cfg=procmacro2_build_probe")
.arg("--edition=2021")
.arg("--crate-name=proc_macro2")
.arg("--crate-type=lib")
@@ -204,7 +227,16 @@ fn compile_probe(rustc_bootstrap: bool) -> bool {
// file in OUT_DIR, which causes nonreproducible builds in build systems
// that treat the entire OUT_DIR as an artifact.
if let Err(err) = fs::remove_dir_all(&out_subdir) {
if err.kind() != ErrorKind::NotFound {
// libc::ENOTEMPTY
// Some filesystems (NFSv3) have timing issues under load where '.nfs*'
// dummy files can continue to get created for a short period after the
// probe command completes, breaking remove_dir_all.
// To be replaced with ErrorKind::DirectoryNotEmpty (Rust 1.83+).
const ENOTEMPTY: i32 = 39;
if !(err.kind() == ErrorKind::NotFound
|| (cfg!(target_os = "linux") && err.raw_os_error() == Some(ENOTEMPTY)))
{
eprintln!("Failed to clean up {}: {}", out_subdir.display(), err);
process::exit(1);
}
fuzz/.gitignore
+9 -8
View File
@@ -1,8 +1,9 @@
artifacts/
corpus/
coverage/
hfuzz_target/
hfuzz_workspace/
in/
out/
target/
/artifacts/
/corpus/
/coverage/
/hfuzz_target/
/hfuzz_workspace/
/in/
/out/
/target/
/Cargo.lock
fuzz/Cargo.toml
+1 -1
View File
@@ -9,7 +9,7 @@ publish = false
cargo-fuzz = true
[dependencies]
afl = { version = "0.15", optional = true }
afl = { version = "0.17", optional = true }
honggfuzz = { version = "0.5", optional = true }
libfuzzer-sys = { version = "0.4.7", optional = true }
proc-macro2 = { path = "..", default-features = false }
src/fallback.rs
+84 -85
View File
@@ -5,12 +5,22 @@ use crate::location::LineColumn;
use crate::parse::{self, Cursor};
use crate::rcvec::{RcVec, RcVecBuilder, RcVecIntoIter, RcVecMut};
use crate::{Delimiter, Spacing, TokenTree};
use alloc::borrow::ToOwned as _;
use alloc::boxed::Box;
#[cfg(all(span_locations, not(fuzzing)))]
use alloc::collections::BTreeMap;
use alloc::format;
use alloc::string::{String, ToString as _};
#[cfg(all(span_locations, not(fuzzing)))]
use alloc::vec;
use alloc::vec::Vec;
#[cfg(all(span_locations, not(fuzzing)))]
use core::cell::RefCell;
#[cfg(span_locations)]
use core::cmp;
#[cfg(all(span_locations, not(fuzzing)))]
use core::cmp::Ordering;
use core::ffi::CStr;
use core::fmt::{self, Debug, Display, Write};
use core::mem::ManuallyDrop;
#[cfg(span_locations)]
@@ -20,11 +30,12 @@ use core::ptr;
use core::str;
#[cfg(feature = "proc-macro")]
use core::str::FromStr;
use std::ffi::CStr;
#[cfg(wrap_proc_macro)]
use std::panic;
#[cfg(procmacro2_semver_exempt)]
#[cfg(span_locations)]
use std::path::PathBuf;
#[cfg(all(span_locations, not(fuzzing)))]
use std::thread_local;
/// Force use of proc-macro2's fallback implementation of the API for now, even
/// if the compiler's implementation is available.
@@ -125,21 +136,32 @@ fn push_token_from_proc_macro(mut vec: RcVecMut<TokenTree>, token: TokenTree) {
// Nonrecursive to prevent stack overflow.
impl Drop for TokenStream {
fn drop(&mut self) {
let mut inner = match self.inner.get_mut() {
Some(inner) => inner,
let mut stack = Vec::new();
let mut current = match self.inner.get_mut() {
Some(inner) => inner.take().into_iter(),
None => return,
};
while let Some(token) = inner.pop() {
let group = match token {
TokenTree::Group(group) => group.inner,
_ => continue,
};
#[cfg(wrap_proc_macro)]
let group = match group {
crate::imp::Group::Fallback(group) => group,
crate::imp::Group::Compiler(_) => continue,
};
inner.extend(group.stream.take_inner());
loop {
while let Some(token) = current.next() {
let group = match token {
TokenTree::Group(group) => group.inner,
_ => continue,
};
#[cfg(wrap_proc_macro)]
let group = match group {
crate::imp::Group::Fallback(group) => group,
crate::imp::Group::Compiler(_) => continue,
};
let mut group = group;
if let Some(inner) = group.stream.inner.get_mut() {
stack.push(current);
current = inner.take().into_iter();
}
}
match stack.pop() {
Some(next) => current = next,
None => return,
}
}
}
}
@@ -209,13 +231,13 @@ impl Display for TokenStream {
}
joint = false;
match tt {
TokenTree::Group(tt) => Display::fmt(tt, f),
TokenTree::Ident(tt) => Display::fmt(tt, f),
TokenTree::Group(tt) => write!(f, "{}", tt),
TokenTree::Ident(tt) => write!(f, "{}", tt),
TokenTree::Punct(tt) => {
joint = tt.spacing() == Spacing::Joint;
Display::fmt(tt, f)
write!(f, "{}", tt)
}
TokenTree::Literal(tt) => Display::fmt(tt, f),
TokenTree::Literal(tt) => write!(f, "{}", tt),
}?;
}
@@ -300,34 +322,6 @@ impl IntoIterator for TokenStream {
}
}
#[cfg(procmacro2_semver_exempt)]
#[derive(Clone, PartialEq, Eq)]
pub(crate) struct SourceFile {
path: PathBuf,
}
#[cfg(procmacro2_semver_exempt)]
impl SourceFile {
/// Get the path to this source file as a string.
pub(crate) fn path(&self) -> PathBuf {
self.path.clone()
}
pub(crate) fn is_real(&self) -> bool {
false
}
}
#[cfg(procmacro2_semver_exempt)]
impl Debug for SourceFile {
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
f.debug_struct("SourceFile")
.field("path", &self.path())
.field("is_real", &self.is_real())
.finish()
}
}
#[cfg(all(span_locations, not(fuzzing)))]
thread_local! {
static SOURCE_MAP: RefCell<SourceMap> = RefCell::new(SourceMap {
@@ -472,36 +466,39 @@ impl SourceMap {
span
}
#[cfg(procmacro2_semver_exempt)]
fn filepath(&self, span: Span) -> PathBuf {
for (i, file) in self.files.iter().enumerate() {
if file.span_within(span) {
return PathBuf::from(if i == 0 {
"<unspecified>".to_owned()
} else {
format!("<parsed string {}>", i)
});
fn find(&self, span: Span) -> usize {
match self.files.binary_search_by(|file| {
if file.span.hi < span.lo {
Ordering::Less
} else if file.span.lo > span.hi {
Ordering::Greater
} else {
assert!(file.span_within(span));
Ordering::Equal
}
}) {
Ok(i) => i,
Err(_) => unreachable!("Invalid span with no related FileInfo!"),
}
}
fn filepath(&self, span: Span) -> String {
let i = self.find(span);
if i == 0 {
"<unspecified>".to_owned()
} else {
format!("<parsed string {}>", i)
}
unreachable!("Invalid span with no related FileInfo!");
}
fn fileinfo(&self, span: Span) -> &FileInfo {
for file in &self.files {
if file.span_within(span) {
return file;
}
}
unreachable!("Invalid span with no related FileInfo!");
let i = self.find(span);
&self.files[i]
}
fn fileinfo_mut(&mut self, span: Span) -> &mut FileInfo {
for file in &mut self.files {
if file.span_within(span) {
return file;
}
}
unreachable!("Invalid span with no related FileInfo!");
let i = self.find(span);
&mut self.files[i]
}
}
@@ -544,21 +541,6 @@ impl Span {
other
}
#[cfg(procmacro2_semver_exempt)]
pub(crate) fn source_file(&self) -> SourceFile {
#[cfg(fuzzing)]
return SourceFile {
path: PathBuf::from("<unspecified>"),
};
#[cfg(not(fuzzing))]
SOURCE_MAP.with(|sm| {
let sm = sm.borrow();
let path = sm.filepath(*self);
SourceFile { path }
})
}
#[cfg(span_locations)]
pub(crate) fn byte_range(&self) -> Range<usize> {
#[cfg(fuzzing)]
@@ -600,6 +582,23 @@ impl Span {
})
}
#[cfg(span_locations)]
pub(crate) fn file(&self) -> String {
#[cfg(fuzzing)]
return "<unspecified>".to_owned();
#[cfg(not(fuzzing))]
SOURCE_MAP.with(|sm| {
let sm = sm.borrow();
sm.filepath(*self)
})
}
#[cfg(span_locations)]
pub(crate) fn local_file(&self) -> Option<PathBuf> {
None
}
#[cfg(not(span_locations))]
pub(crate) fn join(&self, _other: Span) -> Option<Span> {
Some(Span {})
@@ -899,7 +898,7 @@ impl Display for Ident {
if self.raw {
f.write_str("r#")?;
}
Display::fmt(&self.sym, f)
f.write_str(&self.sym)
}
}
src/lib.rs
+215 -77
View File
@@ -9,8 +9,6 @@
//! A wrapper around the procedural macro API of the compiler's [`proc_macro`]
//! crate. This library serves two purposes:
//!
//! [`proc_macro`]: https://doc.rust-lang.org/proc_macro/
//!
//! - **Bring proc-macro-like functionality to other contexts like build.rs and
//! main.rs.** Types from `proc_macro` are entirely specific to procedural
//! macros and cannot ever exist in code outside of a procedural macro.
@@ -65,7 +63,7 @@
//!
//! To opt into the additional APIs available in the most recent nightly
//! compiler, the `procmacro2_semver_exempt` config flag must be passed to
//! rustc. We will polyfill those nightly-only APIs back to Rust 1.56.0. As
//! rustc. We will polyfill those nightly-only APIs back to Rust 1.68.0. As
//! these are unstable APIs that track the nightly compiler, minor versions of
//! proc-macro2 may make breaking changes to them at any time.
//!
@@ -85,8 +83,8 @@
//! types make use of thread-local memory, meaning they cannot be accessed from
//! a different thread.
// Proc-macro2 types in rustdoc of other crates get linked to here.
#![doc(html_root_url = "https://docs.rs/proc-macro2/1.0.92")]
#![no_std]
#![doc(html_root_url = "https://docs.rs/proc-macro2/1.0.105")]
#![cfg_attr(any(proc_macro_span, super_unstable), feature(proc_macro_span))]
#![cfg_attr(super_unstable, feature(proc_macro_def_site))]
#![cfg_attr(docsrs, feature(doc_cfg))]
@@ -96,6 +94,7 @@
clippy::cast_possible_truncation,
clippy::checked_conversions,
clippy::doc_markdown,
clippy::elidable_lifetime_names,
clippy::incompatible_msrv,
clippy::items_after_statements,
clippy::iter_without_into_iter,
@@ -111,11 +110,13 @@
clippy::return_self_not_must_use,
clippy::shadow_unrelated,
clippy::trivially_copy_pass_by_ref,
clippy::uninlined_format_args,
clippy::unnecessary_wraps,
clippy::unused_self,
clippy::used_underscore_binding,
clippy::vec_init_then_push
)]
#![allow(unknown_lints, mismatched_lifetime_syntaxes)]
#[cfg(all(procmacro2_semver_exempt, wrap_proc_macro, not(super_unstable)))]
compile_error! {"\
@@ -135,12 +136,14 @@ compile_error! {"\
"}
extern crate alloc;
extern crate std;
#[cfg(feature = "proc-macro")]
extern crate proc_macro;
mod marker;
mod parse;
mod probe;
mod rcvec;
#[cfg(wrap_proc_macro)]
@@ -162,9 +165,23 @@ mod imp;
#[cfg(span_locations)]
mod location;
#[cfg(procmacro2_semver_exempt)]
mod num;
#[cfg(procmacro2_semver_exempt)]
#[allow(dead_code)]
mod rustc_literal_escaper;
use crate::extra::DelimSpan;
use crate::marker::{ProcMacroAutoTraits, MARKER};
#[cfg(procmacro2_semver_exempt)]
use crate::rustc_literal_escaper::MixedUnit;
#[cfg(procmacro2_semver_exempt)]
use alloc::borrow::ToOwned as _;
use alloc::string::{String, ToString as _};
#[cfg(procmacro2_semver_exempt)]
use alloc::vec::Vec;
use core::cmp::Ordering;
use core::ffi::CStr;
use core::fmt::{self, Debug, Display};
use core::hash::{Hash, Hasher};
#[cfg(span_locations)]
@@ -172,14 +189,17 @@ use core::ops::Range;
use core::ops::RangeBounds;
use core::str::FromStr;
use std::error::Error;
use std::ffi::CStr;
#[cfg(procmacro2_semver_exempt)]
#[cfg(span_locations)]
use std::path::PathBuf;
#[cfg(span_locations)]
#[cfg_attr(docsrs, doc(cfg(feature = "span-locations")))]
pub use crate::location::LineColumn;
#[cfg(procmacro2_semver_exempt)]
#[cfg_attr(docsrs, doc(cfg(procmacro2_semver_exempt)))]
pub use crate::rustc_literal_escaper::EscapeError;
/// An abstract stream of tokens, or more concretely a sequence of token trees.
///
/// This type provides interfaces for iterating over token trees and for
@@ -278,8 +298,8 @@ impl From<TokenTree> for TokenStream {
}
impl Extend<TokenTree> for TokenStream {
fn extend<I: IntoIterator<Item = TokenTree>>(&mut self, streams: I) {
self.inner.extend(streams);
fn extend<I: IntoIterator<Item = TokenTree>>(&mut self, tokens: I) {
self.inner.extend(tokens);
}
}
@@ -290,12 +310,38 @@ impl Extend<TokenStream> for TokenStream {
}
}
/// Collects a number of token trees into a single stream.
impl FromIterator<TokenTree> for TokenStream {
fn from_iter<I: IntoIterator<Item = TokenTree>>(streams: I) -> Self {
TokenStream::_new(streams.into_iter().collect())
impl Extend<Group> for TokenStream {
fn extend<I: IntoIterator<Item = Group>>(&mut self, tokens: I) {
self.inner.extend(tokens.into_iter().map(TokenTree::Group));
}
}
impl Extend<Ident> for TokenStream {
fn extend<I: IntoIterator<Item = Ident>>(&mut self, tokens: I) {
self.inner.extend(tokens.into_iter().map(TokenTree::Ident));
}
}
impl Extend<Punct> for TokenStream {
fn extend<I: IntoIterator<Item = Punct>>(&mut self, tokens: I) {
self.inner.extend(tokens.into_iter().map(TokenTree::Punct));
}
}
impl Extend<Literal> for TokenStream {
fn extend<I: IntoIterator<Item = Literal>>(&mut self, tokens: I) {
self.inner
.extend(tokens.into_iter().map(TokenTree::Literal));
}
}
/// Collects a number of token trees into a single stream.
impl FromIterator<TokenTree> for TokenStream {
fn from_iter<I: IntoIterator<Item = TokenTree>>(tokens: I) -> Self {
TokenStream::_new(tokens.into_iter().collect())
}
}
impl FromIterator<TokenStream> for TokenStream {
fn from_iter<I: IntoIterator<Item = TokenStream>>(streams: I) -> Self {
TokenStream::_new(streams.into_iter().map(|i| i.inner).collect())
@@ -339,57 +385,6 @@ impl Display for LexError {
impl Error for LexError {}
/// The source file of a given `Span`.
///
/// This type is semver exempt and not exposed by default.
#[cfg(all(procmacro2_semver_exempt, any(not(wrap_proc_macro), super_unstable)))]
#[cfg_attr(docsrs, doc(cfg(procmacro2_semver_exempt)))]
#[derive(Clone, PartialEq, Eq)]
pub struct SourceFile {
inner: imp::SourceFile,
_marker: ProcMacroAutoTraits,
}
#[cfg(all(procmacro2_semver_exempt, any(not(wrap_proc_macro), super_unstable)))]
impl SourceFile {
fn _new(inner: imp::SourceFile) -> Self {
SourceFile {
inner,
_marker: MARKER,
}
}
/// Get the path to this source file.
///
/// ### Note
///
/// If the code span associated with this `SourceFile` was generated by an
/// external macro, this may not be an actual path on the filesystem. Use
/// [`is_real`] to check.
///
/// Also note that even if `is_real` returns `true`, if
/// `--remap-path-prefix` was passed on the command line, the path as given
/// may not actually be valid.
///
/// [`is_real`]: #method.is_real
pub fn path(&self) -> PathBuf {
self.inner.path()
}
/// Returns `true` if this source file is a real source file, and not
/// generated by an external macro's expansion.
pub fn is_real(&self) -> bool {
self.inner.is_real()
}
}
#[cfg(all(procmacro2_semver_exempt, any(not(wrap_proc_macro), super_unstable)))]
impl Debug for SourceFile {
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
Debug::fmt(&self.inner, f)
}
}
/// A region of source code, along with macro expansion information.
#[derive(Copy, Clone)]
pub struct Span {
@@ -471,15 +466,6 @@ impl Span {
self.unwrap()
}
/// The original source file into which this span points.
///
/// This method is semver exempt and not exposed by default.
#[cfg(all(procmacro2_semver_exempt, any(not(wrap_proc_macro), super_unstable)))]
#[cfg_attr(docsrs, doc(cfg(procmacro2_semver_exempt)))]
pub fn source_file(&self) -> SourceFile {
SourceFile::_new(self.inner.source_file())
}
/// Returns the span's byte position range in the source file.
///
/// This method requires the `"span-locations"` feature to be enabled.
@@ -525,6 +511,29 @@ impl Span {
self.inner.end()
}
/// The path to the source file in which this span occurs, for display
/// purposes.
///
/// This might not correspond to a valid file system path. It might be
/// remapped, or might be an artificial path such as `"<macro expansion>"`.
#[cfg(span_locations)]
#[cfg_attr(docsrs, doc(cfg(feature = "span-locations")))]
pub fn file(&self) -> String {
self.inner.file()
}
/// The path to the source file in which this span occurs on disk.
///
/// This is the actual path on disk. It is unaffected by path remapping.
///
/// This path should not be embedded in the output of the macro; prefer
/// `file()` instead.
#[cfg(span_locations)]
#[cfg_attr(docsrs, doc(cfg(feature = "span-locations")))]
pub fn local_file(&self) -> Option<PathBuf> {
self.inner.local_file()
}
/// Create a new span encompassing `self` and `other`.
///
/// Returns `None` if `self` and `other` are from different files.
@@ -532,8 +541,6 @@ impl Span {
/// Warning: the underlying [`proc_macro::Span::join`] method is
/// nightly-only. When called from within a procedural macro not using a
/// nightly compiler, this method will always return `None`.
///
/// [`proc_macro::Span::join`]: https://doc.rust-lang.org/proc_macro/struct.Span.html#method.join
pub fn join(&self, other: Span) -> Option<Span> {
self.inner.join(other.inner).map(Span::_new)
}
@@ -1296,12 +1303,116 @@ impl Literal {
/// Warning: the underlying [`proc_macro::Literal::subspan`] method is
/// nightly-only. When called from within a procedural macro not using a
/// nightly compiler, this method will always return `None`.
///
/// [`proc_macro::Literal::subspan`]: https://doc.rust-lang.org/proc_macro/struct.Literal.html#method.subspan
pub fn subspan<R: RangeBounds<usize>>(&self, range: R) -> Option<Span> {
self.inner.subspan(range).map(Span::_new)
}
/// Returns the unescaped string value if this is a string literal.
#[cfg(procmacro2_semver_exempt)]
pub fn str_value(&self) -> Result<String, ConversionErrorKind> {
let repr = self.to_string();
if repr.starts_with('"') && repr[1..].ends_with('"') {
let quoted = &repr[1..repr.len() - 1];
let mut value = String::with_capacity(quoted.len());
let mut error = None;
rustc_literal_escaper::unescape_str(quoted, |_range, res| match res {
Ok(ch) => value.push(ch),
Err(err) => {
if err.is_fatal() {
error = Some(ConversionErrorKind::FailedToUnescape(err));
}
}
});
return match error {
Some(error) => Err(error),
None => Ok(value),
};
}
if repr.starts_with('r') {
if let Some(raw) = get_raw(&repr[1..]) {
return Ok(raw.to_owned());
}
}
Err(ConversionErrorKind::InvalidLiteralKind)
}
/// Returns the unescaped string value (including nul terminator) if this is
/// a c-string literal.
#[cfg(procmacro2_semver_exempt)]
pub fn cstr_value(&self) -> Result<Vec<u8>, ConversionErrorKind> {
let repr = self.to_string();
if repr.starts_with("c\"") && repr[2..].ends_with('"') {
let quoted = &repr[2..repr.len() - 1];
let mut value = Vec::with_capacity(quoted.len());
let mut error = None;
rustc_literal_escaper::unescape_c_str(quoted, |_range, res| match res {
Ok(MixedUnit::Char(ch)) => {
value.extend_from_slice(ch.get().encode_utf8(&mut [0; 4]).as_bytes());
}
Ok(MixedUnit::HighByte(byte)) => value.push(byte.get()),
Err(err) => {
if err.is_fatal() {
error = Some(ConversionErrorKind::FailedToUnescape(err));
}
}
});
return match error {
Some(error) => Err(error),
None => {
value.push(b'\0');
Ok(value)
}
};
}
if repr.starts_with("cr") {
if let Some(raw) = get_raw(&repr[2..]) {
let mut value = Vec::with_capacity(raw.len() + 1);
value.extend_from_slice(raw.as_bytes());
value.push(b'\0');
return Ok(value);
}
}
Err(ConversionErrorKind::InvalidLiteralKind)
}
/// Returns the unescaped string value if this is a byte string literal.
#[cfg(procmacro2_semver_exempt)]
pub fn byte_str_value(&self) -> Result<Vec<u8>, ConversionErrorKind> {
let repr = self.to_string();
if repr.starts_with("b\"") && repr[2..].ends_with('"') {
let quoted = &repr[2..repr.len() - 1];
let mut value = Vec::with_capacity(quoted.len());
let mut error = None;
rustc_literal_escaper::unescape_byte_str(quoted, |_range, res| match res {
Ok(byte) => value.push(byte),
Err(err) => {
if err.is_fatal() {
error = Some(ConversionErrorKind::FailedToUnescape(err));
}
}
});
return match error {
Some(error) => Err(error),
None => Ok(value),
};
}
if repr.starts_with("br") {
if let Some(raw) = get_raw(&repr[2..]) {
return Ok(raw.as_bytes().to_owned());
}
}
Err(ConversionErrorKind::InvalidLiteralKind)
}
// Intended for the `quote!` macro to use when constructing a proc-macro2
// token out of a macro_rules $:literal token, which is already known to be
// a valid literal. This avoids reparsing/validating the literal's string
@@ -1338,6 +1449,33 @@ impl Display for Literal {
}
}
/// Error when retrieving a string literal's unescaped value.
#[cfg(procmacro2_semver_exempt)]
#[derive(Debug, PartialEq, Eq)]
pub enum ConversionErrorKind {
/// The literal is of the right string kind, but its contents are malformed
/// in a way that cannot be unescaped to a value.
FailedToUnescape(EscapeError),
/// The literal is not of the string kind whose value was requested, for
/// example byte string vs UTF-8 string.
InvalidLiteralKind,
}
// ###"..."### -> ...
#[cfg(procmacro2_semver_exempt)]
fn get_raw(repr: &str) -> Option<&str> {
let pounds = repr.len() - repr.trim_start_matches('#').len();
if repr.len() >= pounds + 1 + 1 + pounds
&& repr[pounds..].starts_with('"')
&& repr.trim_end_matches('#').len() + pounds == repr.len()
&& repr[..repr.len() - pounds].ends_with('"')
{
Some(&repr[pounds + 1..repr.len() - pounds - 1])
} else {
None
}
}
/// Public implementation details for the `TokenStream` type, such as iterators.
pub mod token_stream {
use crate::marker::{ProcMacroAutoTraits, MARKER};
src/num.rs
+17
View File
@@ -0,0 +1,17 @@
// TODO: use NonZero<char> in Rust 1.89+
#[derive(Copy, Clone, Debug, PartialEq, Eq)]
pub struct NonZeroChar(char);
impl NonZeroChar {
pub fn new(ch: char) -> Option<Self> {
if ch == '\0' {
None
} else {
Some(NonZeroChar(ch))
}
}
pub fn get(self) -> char {
self.0
}
}
src/parse.rs
+32 -33
View File
@@ -3,6 +3,9 @@ use crate::fallback::{
TokenStreamBuilder,
};
use crate::{Delimiter, Punct, Spacing, TokenTree};
use alloc::borrow::ToOwned as _;
use alloc::string::ToString as _;
use alloc::vec::Vec;
use core::char;
use core::str::{Bytes, CharIndices, Chars};
@@ -166,13 +169,13 @@ fn word_break(input: Cursor) -> Result<Cursor, Reject> {
const ERROR: &str = "(/*ERROR*/)";
pub(crate) fn token_stream(mut input: Cursor) -> Result<TokenStream, LexError> {
let mut trees = TokenStreamBuilder::new();
let mut tokens = TokenStreamBuilder::new();
let mut stack = Vec::new();
loop {
input = skip_whitespace(input);
if let Ok((rest, ())) = doc_comment(input, &mut trees) {
if let Ok((rest, ())) = doc_comment(input, &mut tokens) {
input = rest;
continue;
}
@@ -180,19 +183,16 @@ pub(crate) fn token_stream(mut input: Cursor) -> Result<TokenStream, LexError> {
#[cfg(span_locations)]
let lo = input.off;
let first = match input.bytes().next() {
Some(first) => first,
None => match stack.last() {
None => return Ok(trees.build()),
let Some(first) = input.bytes().next() else {
return match stack.last() {
None => Ok(tokens.build()),
#[cfg(span_locations)]
Some((lo, _frame)) => {
return Err(LexError {
span: Span { lo: *lo, hi: *lo },
})
}
Some((lo, _frame)) => Err(LexError {
span: Span { lo: *lo, hi: *lo },
}),
#[cfg(not(span_locations))]
Some(_frame) => return Err(LexError { span: Span {} }),
},
Some(_frame) => Err(LexError { span: Span {} }),
};
};
if let Some(open_delimiter) = match first {
@@ -202,20 +202,19 @@ pub(crate) fn token_stream(mut input: Cursor) -> Result<TokenStream, LexError> {
_ => None,
} {
input = input.advance(1);
let frame = (open_delimiter, trees);
let frame = (open_delimiter, tokens);
#[cfg(span_locations)]
let frame = (lo, frame);
stack.push(frame);
trees = TokenStreamBuilder::new();
tokens = TokenStreamBuilder::new();
} else if let Some(close_delimiter) = match first {
b')' => Some(Delimiter::Parenthesis),
b']' => Some(Delimiter::Bracket),
b'}' => Some(Delimiter::Brace),
_ => None,
} {
let frame = match stack.pop() {
Some(frame) => frame,
None => return Err(lex_error(input)),
let Some(frame) = stack.pop() else {
return Err(lex_error(input));
};
#[cfg(span_locations)]
let (lo, frame) = frame;
@@ -224,15 +223,15 @@ pub(crate) fn token_stream(mut input: Cursor) -> Result<TokenStream, LexError> {
return Err(lex_error(input));
}
input = input.advance(1);
let mut g = Group::new(open_delimiter, trees.build());
let mut g = Group::new(open_delimiter, tokens.build());
g.set_span(Span {
#[cfg(span_locations)]
lo,
#[cfg(span_locations)]
hi: input.off,
});
trees = outer;
trees.push_token_from_parser(TokenTree::Group(crate::Group::_new_fallback(g)));
tokens = outer;
tokens.push_token_from_parser(TokenTree::Group(crate::Group::_new_fallback(g)));
} else {
let (rest, mut tt) = match leaf_token(input) {
Ok((rest, tt)) => (rest, tt),
@@ -244,7 +243,7 @@ pub(crate) fn token_stream(mut input: Cursor) -> Result<TokenStream, LexError> {
#[cfg(span_locations)]
hi: rest.off,
}));
trees.push_token_from_parser(tt);
tokens.push_token_from_parser(tt);
input = rest;
}
}
@@ -857,7 +856,7 @@ fn digits(mut input: Cursor) -> Result<Cursor, Reject> {
continue;
}
_ => break,
};
}
len += 1;
empty = false;
}
@@ -871,7 +870,10 @@ fn digits(mut input: Cursor) -> Result<Cursor, Reject> {
fn punct(input: Cursor) -> PResult<Punct> {
let (rest, ch) = punct_char(input)?;
if ch == '\'' {
if ident_any(rest)?.0.starts_with_char('\'') {
let (after_lifetime, _ident) = ident_any(rest)?;
if after_lifetime.starts_with_char('\'')
|| (after_lifetime.starts_with_char('#') && !rest.starts_with("r#"))
{
Err(Reject)
} else {
Ok((rest, Punct::new('\'', Spacing::Joint)))
@@ -892,11 +894,8 @@ fn punct_char(input: Cursor) -> PResult<char> {
}
let mut chars = input.chars();
let first = match chars.next() {
Some(ch) => ch,
None => {
return Err(Reject);
}
let Some(first) = chars.next() else {
return Err(Reject);
};
let recognized = "~!@#$%^&*-=+|;:,<.>/?'";
if recognized.contains(first) {
@@ -906,7 +905,7 @@ fn punct_char(input: Cursor) -> PResult<char> {
}
}
fn doc_comment<'a>(input: Cursor<'a>, trees: &mut TokenStreamBuilder) -> PResult<'a, ()> {
fn doc_comment<'a>(input: Cursor<'a>, tokens: &mut TokenStreamBuilder) -> PResult<'a, ()> {
#[cfg(span_locations)]
let lo = input.off;
let (rest, (comment, inner)) = doc_comment_contents(input)?;
@@ -929,12 +928,12 @@ fn doc_comment<'a>(input: Cursor<'a>, trees: &mut TokenStreamBuilder) -> PResult
let mut pound = Punct::new('#', Spacing::Alone);
pound.set_span(span);
trees.push_token_from_parser(TokenTree::Punct(pound));
tokens.push_token_from_parser(TokenTree::Punct(pound));
if inner {
let mut bang = Punct::new('!', Spacing::Alone);
bang.set_span(span);
trees.push_token_from_parser(TokenTree::Punct(bang));
tokens.push_token_from_parser(TokenTree::Punct(bang));
}
let doc_ident = crate::Ident::_new_fallback(Ident::new_unchecked("doc", fallback_span));
@@ -949,7 +948,7 @@ fn doc_comment<'a>(input: Cursor<'a>, trees: &mut TokenStreamBuilder) -> PResult
let group = Group::new(Delimiter::Bracket, bracketed.build());
let mut group = crate::Group::_new_fallback(group);
group.set_span(span);
trees.push_token_from_parser(TokenTree::Group(group));
tokens.push_token_from_parser(TokenTree::Group(group));
Ok((rest, ()))
}
src/probe.rs
+10
View File
@@ -0,0 +1,10 @@
#![allow(dead_code)]
#[cfg(proc_macro_span)]
pub(crate) mod proc_macro_span;
#[cfg(proc_macro_span_file)]
pub(crate) mod proc_macro_span_file;
#[cfg(proc_macro_span_location)]
pub(crate) mod proc_macro_span_location;
build/probe.rs → src/probe/proc_macro_span.rs
+15 -1
View File
@@ -2,12 +2,17 @@
// If the current toolchain is able to compile it, then proc-macro2 is able to
// offer these APIs too.
#![feature(proc_macro_span)]
#![cfg_attr(procmacro2_build_probe, no_std)]
#![cfg_attr(procmacro2_build_probe, feature(proc_macro_span))]
extern crate alloc;
extern crate proc_macro;
extern crate std;
use alloc::string::String;
use core::ops::{Range, RangeBounds};
use proc_macro::{Literal, Span};
use std::path::PathBuf;
pub fn byte_range(this: &Span) -> Range<usize> {
this.byte_range()
@@ -29,6 +34,14 @@ pub fn column(this: &Span) -> usize {
this.column()
}
pub fn file(this: &Span) -> String {
this.file()
}
pub fn local_file(this: &Span) -> Option<PathBuf> {
this.local_file()
}
pub fn join(this: &Span, other: Span) -> Option<Span> {
this.join(other)
}
@@ -38,4 +51,5 @@ pub fn subspan<R: RangeBounds<usize>>(this: &Literal, range: R) -> Option<Span>
}
// Include in sccache cache key.
#[cfg(procmacro2_build_probe)]
const _: Option<&str> = option_env!("RUSTC_BOOTSTRAP");
src/probe/proc_macro_span_file.rs
+19
View File
@@ -0,0 +1,19 @@
// The subset of Span's API stabilized in Rust 1.88.
#![cfg_attr(procmacro2_build_probe, no_std)]
extern crate alloc;
extern crate proc_macro;
extern crate std;
use alloc::string::String;
use proc_macro::Span;
use std::path::PathBuf;
pub fn file(this: &Span) -> String {
this.file()
}
pub fn local_file(this: &Span) -> Option<PathBuf> {
this.local_file()
}
src/probe/proc_macro_span_location.rs
+25
View File
@@ -0,0 +1,25 @@
// The subset of Span's API stabilized in Rust 1.88.
#![cfg_attr(procmacro2_build_probe, no_std)]
extern crate alloc;
extern crate proc_macro;
extern crate std;
use proc_macro::Span;
pub fn start(this: &Span) -> Span {
this.start()
}
pub fn end(this: &Span) -> Span {
this.end()
}
pub fn line(this: &Span) -> usize {
this.line()
}
pub fn column(this: &Span) -> usize {
this.column()
}
src/rcvec.rs
+6 -5
View File
@@ -1,5 +1,5 @@
use alloc::rc::Rc;
use alloc::vec;
use alloc::vec::{self, Vec};
use core::mem;
use core::panic::RefUnwindSafe;
use core::slice;
@@ -102,13 +102,14 @@ impl<'a, T> RcVecMut<'a, T> {
self.inner.extend(iter);
}
pub(crate) fn pop(&mut self) -> Option<T> {
self.inner.pop()
}
pub(crate) fn as_mut(&mut self) -> RcVecMut<T> {
RcVecMut { inner: self.inner }
}
pub(crate) fn take(self) -> RcVecBuilder<T> {
let vec = mem::take(self.inner);
RcVecBuilder { inner: vec }
}
}
impl<T> Clone for RcVec<T> {
src/rustc_literal_escaper.rs
+701
View File
@@ -0,0 +1,701 @@
// Vendored from rustc-literal-escaper v0.0.5
// https://github.com/rust-lang/literal-escaper/tree/v0.0.5
//! Utilities for validating (raw) string, char, and byte literals and
//! turning escape sequences into the values they represent.
use crate::num::NonZeroChar;
use core::ffi::CStr;
use core::num::NonZeroU8;
use core::ops::Range;
use core::str::Chars;
/// Errors and warnings that can occur during string, char, and byte unescaping.
///
/// Mostly relating to malformed escape sequences, but also a few other problems.
#[derive(Debug, PartialEq, Eq)]
pub enum EscapeError {
/// Expected 1 char, but 0 were found.
ZeroChars,
/// Expected 1 char, but more than 1 were found.
MoreThanOneChar,
/// Escaped '\' character without continuation.
LoneSlash,
/// Invalid escape character (e.g. '\z').
InvalidEscape,
/// Raw '\r' encountered.
BareCarriageReturn,
/// Raw '\r' encountered in raw string.
BareCarriageReturnInRawString,
/// Unescaped character that was expected to be escaped (e.g. raw '\t').
EscapeOnlyChar,
/// Numeric character escape is too short (e.g. '\x1').
TooShortHexEscape,
/// Invalid character in numeric escape (e.g. '\xz')
InvalidCharInHexEscape,
/// Character code in numeric escape is non-ascii (e.g. '\xFF').
OutOfRangeHexEscape,
/// '\u' not followed by '{'.
NoBraceInUnicodeEscape,
/// Non-hexadecimal value in '\u{..}'.
InvalidCharInUnicodeEscape,
/// '\u{}'
EmptyUnicodeEscape,
/// No closing brace in '\u{..}', e.g. '\u{12'.
UnclosedUnicodeEscape,
/// '\u{_12}'
LeadingUnderscoreUnicodeEscape,
/// More than 6 characters in '\u{..}', e.g. '\u{10FFFF_FF}'
OverlongUnicodeEscape,
/// Invalid in-bound unicode character code, e.g. '\u{DFFF}'.
LoneSurrogateUnicodeEscape,
/// Out of bounds unicode character code, e.g. '\u{FFFFFF}'.
OutOfRangeUnicodeEscape,
/// Unicode escape code in byte literal.
UnicodeEscapeInByte,
/// Non-ascii character in byte literal, byte string literal, or raw byte string literal.
NonAsciiCharInByte,
/// `\0` in a C string literal.
NulInCStr,
/// After a line ending with '\', the next line contains whitespace
/// characters that are not skipped.
UnskippedWhitespaceWarning,
/// After a line ending with '\', multiple lines are skipped.
MultipleSkippedLinesWarning,
}
impl EscapeError {
/// Returns true for actual errors, as opposed to warnings.
pub fn is_fatal(&self) -> bool {
!matches!(
self,
EscapeError::UnskippedWhitespaceWarning | EscapeError::MultipleSkippedLinesWarning
)
}
}
/// Check a raw string literal for validity
///
/// Takes the contents of a raw string literal (without quotes)
/// and produces a sequence of characters or errors,
/// which are returned by invoking `callback`.
/// NOTE: Does no escaping, but produces errors for bare carriage return ('\r').
pub fn check_raw_str(src: &str, callback: impl FnMut(Range<usize>, Result<char, EscapeError>)) {
str::check_raw(src, callback);
}
/// Check a raw byte string literal for validity
///
/// Takes the contents of a raw byte string literal (without quotes)
/// and produces a sequence of bytes or errors,
/// which are returned by invoking `callback`.
/// NOTE: Does no escaping, but produces errors for bare carriage return ('\r').
pub fn check_raw_byte_str(src: &str, callback: impl FnMut(Range<usize>, Result<u8, EscapeError>)) {
<[u8]>::check_raw(src, callback);
}
/// Check a raw C string literal for validity
///
/// Takes the contents of a raw C string literal (without quotes)
/// and produces a sequence of characters or errors,
/// which are returned by invoking `callback`.
/// NOTE: Does no escaping, but produces errors for bare carriage return ('\r').
pub fn check_raw_c_str(
src: &str,
callback: impl FnMut(Range<usize>, Result<NonZeroChar, EscapeError>),
) {
CStr::check_raw(src, callback);
}
/// Trait for checking raw string literals for validity
trait CheckRaw {
/// Unit type of the implementing string type (`char` for string, `u8` for byte string)
type RawUnit;
/// Converts chars to the unit type of the literal type
fn char2raw_unit(c: char) -> Result<Self::RawUnit, EscapeError>;
/// Takes the contents of a raw literal (without quotes)
/// and produces a sequence of `Result<Self::RawUnit, EscapeError>`
/// which are returned via `callback`.
///
/// NOTE: Does no escaping, but produces errors for bare carriage return ('\r').
fn check_raw(
src: &str,
mut callback: impl FnMut(Range<usize>, Result<Self::RawUnit, EscapeError>),
) {
let mut chars = src.chars();
while let Some(c) = chars.next() {
let start = src.len() - chars.as_str().len() - c.len_utf8();
let res = match c {
'\r' => Err(EscapeError::BareCarriageReturnInRawString),
_ => Self::char2raw_unit(c),
};
let end = src.len() - chars.as_str().len();
callback(start..end, res);
}
// Unfortunately, it is a bit unclear whether the following equivalent code is slower or faster: bug 141855
// src.char_indices().for_each(|(pos, c)| {
// callback(
// pos..pos + c.len_utf8(),
// if c == '\r' {
// Err(EscapeError::BareCarriageReturnInRawString)
// } else {
// Self::char2raw_unit(c)
// },
// );
// });
}
}
impl CheckRaw for str {
type RawUnit = char;
#[inline]
fn char2raw_unit(c: char) -> Result<Self::RawUnit, EscapeError> {
Ok(c)
}
}
impl CheckRaw for [u8] {
type RawUnit = u8;
#[inline]
fn char2raw_unit(c: char) -> Result<Self::RawUnit, EscapeError> {
char2byte(c)
}
}
/// Turn an ascii char into a byte
#[inline]
fn char2byte(c: char) -> Result<u8, EscapeError> {
// do NOT do: c.try_into().ok_or(EscapeError::NonAsciiCharInByte)
if c.is_ascii() {
Ok(c as u8)
} else {
Err(EscapeError::NonAsciiCharInByte)
}
}
impl CheckRaw for CStr {
type RawUnit = NonZeroChar;
#[inline]
fn char2raw_unit(c: char) -> Result<Self::RawUnit, EscapeError> {
NonZeroChar::new(c).ok_or(EscapeError::NulInCStr)
}
}
/// Unescape a char literal
///
/// Takes the contents of a char literal (without quotes),
/// and returns an unescaped char or an error.
#[inline]
pub fn unescape_char(src: &str) -> Result<char, EscapeError> {
str::unescape_single(&mut src.chars())
}
/// Unescape a byte literal
///
/// Takes the contents of a byte literal (without quotes),
/// and returns an unescaped byte or an error.
#[inline]
pub fn unescape_byte(src: &str) -> Result<u8, EscapeError> {
<[u8]>::unescape_single(&mut src.chars())
}
/// Unescape a string literal
///
/// Takes the contents of a string literal (without quotes)
/// and produces a sequence of escaped characters or errors,
/// which are returned by invoking `callback`.
pub fn unescape_str(src: &str, callback: impl FnMut(Range<usize>, Result<char, EscapeError>)) {
str::unescape(src, callback)
}
/// Unescape a byte string literal
///
/// Takes the contents of a byte string literal (without quotes)
/// and produces a sequence of escaped bytes or errors,
/// which are returned by invoking `callback`.
pub fn unescape_byte_str(src: &str, callback: impl FnMut(Range<usize>, Result<u8, EscapeError>)) {
<[u8]>::unescape(src, callback)
}
/// Unescape a C string literal
///
/// Takes the contents of a C string literal (without quotes)
/// and produces a sequence of escaped MixedUnits or errors,
/// which are returned by invoking `callback`.
pub fn unescape_c_str(
src: &str,
callback: impl FnMut(Range<usize>, Result<MixedUnit, EscapeError>),
) {
CStr::unescape(src, callback)
}
/// Enum representing either a char or a byte
///
/// Used for mixed utf8 string literals, i.e. those that allow both unicode
/// chars and high bytes.
#[derive(Copy, Clone, Debug, PartialEq, Eq)]
pub enum MixedUnit {
/// Used for ASCII chars (written directly or via `\x00`..`\x7f` escapes)
/// and Unicode chars (written directly or via `\u` escapes).
///
/// For example, if '¥' appears in a string it is represented here as
/// `MixedUnit::Char('¥')`, and it will be appended to the relevant byte
/// string as the two-byte UTF-8 sequence `[0xc2, 0xa5]`
Char(NonZeroChar),
/// Used for high bytes (`\x80`..`\xff`).
///
/// For example, if `\xa5` appears in a string it is represented here as
/// `MixedUnit::HighByte(0xa5)`, and it will be appended to the relevant
/// byte string as the single byte `0xa5`.
HighByte(NonZeroU8),
}
impl From<NonZeroChar> for MixedUnit {
#[inline]
fn from(c: NonZeroChar) -> Self {
MixedUnit::Char(c)
}
}
impl From<NonZeroU8> for MixedUnit {
#[inline]
fn from(byte: NonZeroU8) -> Self {
if byte.get().is_ascii() {
MixedUnit::Char(NonZeroChar::new(byte.get() as char).unwrap())
} else {
MixedUnit::HighByte(byte)
}
}
}
impl TryFrom<char> for MixedUnit {
type Error = EscapeError;
#[inline]
fn try_from(c: char) -> Result<Self, EscapeError> {
NonZeroChar::new(c)
.map(MixedUnit::Char)
.ok_or(EscapeError::NulInCStr)
}
}
impl TryFrom<u8> for MixedUnit {
type Error = EscapeError;
#[inline]
fn try_from(byte: u8) -> Result<Self, EscapeError> {
NonZeroU8::new(byte)
.map(From::from)
.ok_or(EscapeError::NulInCStr)
}
}
/// Trait for unescaping escape sequences in strings
trait Unescape {
/// Unit type of the implementing string type (`char` for string, `u8` for byte string)
type Unit;
/// Result of unescaping the zero char ('\0')
const ZERO_RESULT: Result<Self::Unit, EscapeError>;
/// Converts non-zero bytes to the unit type
fn nonzero_byte2unit(b: NonZeroU8) -> Self::Unit;
/// Converts chars to the unit type
fn char2unit(c: char) -> Result<Self::Unit, EscapeError>;
/// Converts the byte of a hex escape to the unit type
fn hex2unit(b: u8) -> Result<Self::Unit, EscapeError>;
/// Converts the result of a unicode escape to the unit type
fn unicode2unit(r: Result<char, EscapeError>) -> Result<Self::Unit, EscapeError>;
/// Unescape a single unit (single quote syntax)
fn unescape_single(chars: &mut Chars<'_>) -> Result<Self::Unit, EscapeError> {
let res = match chars.next().ok_or(EscapeError::ZeroChars)? {
'\\' => Self::unescape_1(chars),
'\n' | '\t' | '\'' => Err(EscapeError::EscapeOnlyChar),
'\r' => Err(EscapeError::BareCarriageReturn),
c => Self::char2unit(c),
}?;
if chars.next().is_some() {
return Err(EscapeError::MoreThanOneChar);
}
Ok(res)
}
/// Unescape the first unit of a string (double quoted syntax)
fn unescape_1(chars: &mut Chars<'_>) -> Result<Self::Unit, EscapeError> {
// Previous character was '\\', unescape what follows.
let c = chars.next().ok_or(EscapeError::LoneSlash)?;
if c == '0' {
Self::ZERO_RESULT
} else {
simple_escape(c)
.map(|b| Self::nonzero_byte2unit(b))
.or_else(|c| match c {
'x' => Self::hex2unit(hex_escape(chars)?),
'u' => Self::unicode2unit({
let value = unicode_escape(chars)?;
if value > char::MAX as u32 {
Err(EscapeError::OutOfRangeUnicodeEscape)
} else {
char::from_u32(value).ok_or(EscapeError::LoneSurrogateUnicodeEscape)
}
}),
_ => Err(EscapeError::InvalidEscape),
})
}
}
/// Unescape a string literal
///
/// Takes the contents of a raw string literal (without quotes)
/// and produces a sequence of `Result<Self::Unit, EscapeError>`
/// which are returned via `callback`.
fn unescape(
src: &str,
mut callback: impl FnMut(Range<usize>, Result<Self::Unit, EscapeError>),
) {
let mut chars = src.chars();
while let Some(c) = chars.next() {
let start = src.len() - chars.as_str().len() - c.len_utf8();
let res = match c {
'\\' => {
if let Some(b'\n') = chars.as_str().as_bytes().first() {
let _ = chars.next();
// skip whitespace for backslash newline, see [Rust language reference]
// (https://doc.rust-lang.org/reference/tokens.html#string-literals).
let callback_err = |range, err| callback(range, Err(err));
skip_ascii_whitespace(&mut chars, start, callback_err);
continue;
} else {
Self::unescape_1(&mut chars)
}
}
'"' => Err(EscapeError::EscapeOnlyChar),
'\r' => Err(EscapeError::BareCarriageReturn),
c => Self::char2unit(c),
};
let end = src.len() - chars.as_str().len();
callback(start..end, res);
}
}
}
/// Interpret a non-nul ASCII escape
///
/// Parses the character of an ASCII escape (except nul) without the leading backslash.
#[inline] // single use in Unescape::unescape_1
fn simple_escape(c: char) -> Result<NonZeroU8, char> {
// Previous character was '\\', unescape what follows.
Ok(NonZeroU8::new(match c {
'"' => b'"',
'n' => b'\n',
'r' => b'\r',
't' => b'\t',
'\\' => b'\\',
'\'' => b'\'',
_ => Err(c)?,
})
.unwrap())
}
/// Interpret a hexadecimal escape
///
/// Parses the two hexadecimal characters of a hexadecimal escape without the leading r"\x".
#[inline] // single use in Unescape::unescape_1
fn hex_escape(chars: &mut impl Iterator<Item = char>) -> Result<u8, EscapeError> {
let hi = chars.next().ok_or(EscapeError::TooShortHexEscape)?;
let hi = hi.to_digit(16).ok_or(EscapeError::InvalidCharInHexEscape)?;
let lo = chars.next().ok_or(EscapeError::TooShortHexEscape)?;
let lo = lo.to_digit(16).ok_or(EscapeError::InvalidCharInHexEscape)?;
Ok((hi * 16 + lo) as u8)
}
/// Interpret a unicode escape
///
/// Parse the braces with hexadecimal characters (and underscores) part of a unicode escape.
/// This r"{...}" normally comes after r"\u" and cannot start with an underscore.
#[inline] // single use in Unescape::unescape_1
fn unicode_escape(chars: &mut impl Iterator<Item = char>) -> Result<u32, EscapeError> {
if chars.next() != Some('{') {
return Err(EscapeError::NoBraceInUnicodeEscape);
}
// First character must be a hexadecimal digit.
let mut value: u32 = match chars.next().ok_or(EscapeError::UnclosedUnicodeEscape)? {
'_' => return Err(EscapeError::LeadingUnderscoreUnicodeEscape),
'}' => return Err(EscapeError::EmptyUnicodeEscape),
c => c
.to_digit(16)
.ok_or(EscapeError::InvalidCharInUnicodeEscape)?,
};
// First character is valid, now parse the rest of the number
// and closing brace.
let mut n_digits = 1;
loop {
match chars.next() {
None => return Err(EscapeError::UnclosedUnicodeEscape),
Some('_') => continue,
Some('}') => {
// Incorrect syntax has higher priority for error reporting
// than unallowed value for a literal.
return if n_digits > 6 {
Err(EscapeError::OverlongUnicodeEscape)
} else {
Ok(value)
};
}
Some(c) => {
let digit: u32 = c
.to_digit(16)
.ok_or(EscapeError::InvalidCharInUnicodeEscape)?;
n_digits += 1;
if n_digits > 6 {
// Stop updating value since we're sure that it's incorrect already.
continue;
}
value = value * 16 + digit;
}
};
}
}
/// Interpret a string continuation escape (https://doc.rust-lang.org/reference/expressions/literal-expr.html#string-continuation-escapes)
///
/// Skip ASCII whitespace, except for the formfeed character
/// (see [this issue](https://github.com/rust-lang/rust/issues/136600)).
/// Warns on unescaped newline and following non-ASCII whitespace.
#[inline] // single use in Unescape::unescape
fn skip_ascii_whitespace(
chars: &mut Chars<'_>,
start: usize,
mut callback: impl FnMut(Range<usize>, EscapeError),
) {
let rest = chars.as_str();
let first_non_space = rest
.bytes()
.position(|b| b != b' ' && b != b'\t' && b != b'\n' && b != b'\r')
.unwrap_or(rest.len());
let (space, rest) = rest.split_at(first_non_space);
// backslash newline adds 2 bytes
let end = start + 2 + first_non_space;
if space.contains('\n') {
callback(start..end, EscapeError::MultipleSkippedLinesWarning);
}
*chars = rest.chars();
if let Some(c) = chars.clone().next() {
if c.is_whitespace() {
// for error reporting, include the character that was not skipped in the span
callback(
start..end + c.len_utf8(),
EscapeError::UnskippedWhitespaceWarning,
);
}
}
}
impl Unescape for str {
type Unit = char;
const ZERO_RESULT: Result<Self::Unit, EscapeError> = Ok('\0');
#[inline]
fn nonzero_byte2unit(b: NonZeroU8) -> Self::Unit {
b.get().into()
}
#[inline]
fn char2unit(c: char) -> Result<Self::Unit, EscapeError> {
Ok(c)
}
#[inline]
fn hex2unit(b: u8) -> Result<Self::Unit, EscapeError> {
if b.is_ascii() {
Ok(b as char)
} else {
Err(EscapeError::OutOfRangeHexEscape)
}
}
#[inline]
fn unicode2unit(r: Result<char, EscapeError>) -> Result<Self::Unit, EscapeError> {
r
}
}
impl Unescape for [u8] {
type Unit = u8;
const ZERO_RESULT: Result<Self::Unit, EscapeError> = Ok(b'\0');
#[inline]
fn nonzero_byte2unit(b: NonZeroU8) -> Self::Unit {
b.get()
}
#[inline]
fn char2unit(c: char) -> Result<Self::Unit, EscapeError> {
char2byte(c)
}
#[inline]
fn hex2unit(b: u8) -> Result<Self::Unit, EscapeError> {
Ok(b)
}
#[inline]
fn unicode2unit(_r: Result<char, EscapeError>) -> Result<Self::Unit, EscapeError> {
Err(EscapeError::UnicodeEscapeInByte)
}
}
impl Unescape for CStr {
type Unit = MixedUnit;
const ZERO_RESULT: Result<Self::Unit, EscapeError> = Err(EscapeError::NulInCStr);
#[inline]
fn nonzero_byte2unit(b: NonZeroU8) -> Self::Unit {
b.into()
}
#[inline]
fn char2unit(c: char) -> Result<Self::Unit, EscapeError> {
c.try_into()
}
#[inline]
fn hex2unit(byte: u8) -> Result<Self::Unit, EscapeError> {
byte.try_into()
}
#[inline]
fn unicode2unit(r: Result<char, EscapeError>) -> Result<Self::Unit, EscapeError> {
Self::char2unit(r?)
}
}
/// Enum of the different kinds of literal
#[derive(Debug, Clone, Copy, PartialEq)]
pub enum Mode {
/// `'a'`
Char,
/// `b'a'`
Byte,
/// `"hello"`
Str,
/// `r"hello"`
RawStr,
/// `b"hello"`
ByteStr,
/// `br"hello"`
RawByteStr,
/// `c"hello"`
CStr,
/// `cr"hello"`
RawCStr,
}
impl Mode {
pub fn in_double_quotes(self) -> bool {
match self {
Mode::Str
| Mode::RawStr
| Mode::ByteStr
| Mode::RawByteStr
| Mode::CStr
| Mode::RawCStr => true,
Mode::Char | Mode::Byte => false,
}
}
pub fn prefix_noraw(self) -> &'static str {
match self {
Mode::Char | Mode::Str | Mode::RawStr => "",
Mode::Byte | Mode::ByteStr | Mode::RawByteStr => "b",
Mode::CStr | Mode::RawCStr => "c",
}
}
}
/// Check a literal only for errors
///
/// Takes the contents of a literal (without quotes)
/// and produces a sequence of only errors,
/// which are returned by invoking `error_callback`.
///
/// NB Does not produce any output other than errors
pub fn check_for_errors(
src: &str,
mode: Mode,
mut error_callback: impl FnMut(Range<usize>, EscapeError),
) {
match mode {
Mode::Char => {
let mut chars = src.chars();
if let Err(e) = str::unescape_single(&mut chars) {
error_callback(0..(src.len() - chars.as_str().len()), e);
}
}
Mode::Byte => {
let mut chars = src.chars();
if let Err(e) = <[u8]>::unescape_single(&mut chars) {
error_callback(0..(src.len() - chars.as_str().len()), e);
}
}
Mode::Str => unescape_str(src, |range, res| {
if let Err(e) = res {
error_callback(range, e);
}
}),
Mode::ByteStr => unescape_byte_str(src, |range, res| {
if let Err(e) = res {
error_callback(range, e);
}
}),
Mode::CStr => unescape_c_str(src, |range, res| {
if let Err(e) = res {
error_callback(range, e);
}
}),
Mode::RawStr => check_raw_str(src, |range, res| {
if let Err(e) = res {
error_callback(range, e);
}
}),
Mode::RawByteStr => check_raw_byte_str(src, |range, res| {
if let Err(e) = res {
error_callback(range, e);
}
}),
Mode::RawCStr => check_raw_c_str(src, |range, res| {
if let Err(e) = res {
error_callback(range, e);
}
}),
}
}
src/wrapper.rs
+52 -67
View File
@@ -2,13 +2,23 @@ use crate::detection::inside_proc_macro;
use crate::fallback::{self, FromStr2 as _};
#[cfg(span_locations)]
use crate::location::LineColumn;
#[cfg(proc_macro_span)]
use crate::probe::proc_macro_span;
#[cfg(all(span_locations, proc_macro_span_file))]
use crate::probe::proc_macro_span_file;
#[cfg(all(span_locations, proc_macro_span_location))]
use crate::probe::proc_macro_span_location;
use crate::{Delimiter, Punct, Spacing, TokenTree};
#[cfg(all(span_locations, not(proc_macro_span_file)))]
use alloc::borrow::ToOwned as _;
use alloc::string::{String, ToString as _};
use alloc::vec::Vec;
use core::ffi::CStr;
use core::fmt::{self, Debug, Display};
#[cfg(span_locations)]
use core::ops::Range;
use core::ops::RangeBounds;
use std::ffi::CStr;
#[cfg(super_unstable)]
#[cfg(span_locations)]
use std::path::PathBuf;
#[derive(Clone)]
@@ -182,13 +192,13 @@ impl From<TokenTree> for TokenStream {
}
impl FromIterator<TokenTree> for TokenStream {
fn from_iter<I: IntoIterator<Item = TokenTree>>(trees: I) -> Self {
fn from_iter<I: IntoIterator<Item = TokenTree>>(tokens: I) -> Self {
if inside_proc_macro() {
TokenStream::Compiler(DeferredTokenStream::new(
trees.into_iter().map(into_compiler_token).collect(),
tokens.into_iter().map(into_compiler_token).collect(),
))
} else {
TokenStream::Fallback(trees.into_iter().collect())
TokenStream::Fallback(tokens.into_iter().collect())
}
}
}
@@ -218,15 +228,15 @@ impl FromIterator<TokenStream> for TokenStream {
}
impl Extend<TokenTree> for TokenStream {
fn extend<I: IntoIterator<Item = TokenTree>>(&mut self, stream: I) {
fn extend<I: IntoIterator<Item = TokenTree>>(&mut self, tokens: I) {
match self {
TokenStream::Compiler(tts) => {
// Here is the reason for DeferredTokenStream.
for token in stream {
for token in tokens {
tts.extra.push(into_compiler_token(token));
}
}
TokenStream::Fallback(tts) => tts.extend(stream),
TokenStream::Fallback(tts) => tts.extend(tokens),
}
}
}
@@ -360,45 +370,6 @@ impl Iterator for TokenTreeIter {
}
}
#[derive(Clone, PartialEq, Eq)]
#[cfg(super_unstable)]
pub(crate) enum SourceFile {
Compiler(proc_macro::SourceFile),
Fallback(fallback::SourceFile),
}
#[cfg(super_unstable)]
impl SourceFile {
fn nightly(sf: proc_macro::SourceFile) -> Self {
SourceFile::Compiler(sf)
}
/// Get the path to this source file as a string.
pub(crate) fn path(&self) -> PathBuf {
match self {
SourceFile::Compiler(a) => a.path(),
SourceFile::Fallback(a) => a.path(),
}
}
pub(crate) fn is_real(&self) -> bool {
match self {
SourceFile::Compiler(a) => a.is_real(),
SourceFile::Fallback(a) => a.is_real(),
}
}
}
#[cfg(super_unstable)]
impl Debug for SourceFile {
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
match self {
SourceFile::Compiler(a) => Debug::fmt(a, f),
SourceFile::Fallback(a) => Debug::fmt(a, f),
}
}
}
#[derive(Copy, Clone)]
pub(crate) enum Span {
Compiler(proc_macro::Span),
@@ -456,19 +427,11 @@ impl Span {
}
}
#[cfg(super_unstable)]
pub(crate) fn source_file(&self) -> SourceFile {
match self {
Span::Compiler(s) => SourceFile::nightly(s.source_file()),
Span::Fallback(s) => SourceFile::Fallback(s.source_file()),
}
}
#[cfg(span_locations)]
pub(crate) fn byte_range(&self) -> Range<usize> {
match self {
#[cfg(proc_macro_span)]
Span::Compiler(s) => s.byte_range(),
Span::Compiler(s) => proc_macro_span::byte_range(s),
#[cfg(not(proc_macro_span))]
Span::Compiler(_) => 0..0,
Span::Fallback(s) => s.byte_range(),
@@ -478,12 +441,12 @@ impl Span {
#[cfg(span_locations)]
pub(crate) fn start(&self) -> LineColumn {
match self {
#[cfg(proc_macro_span)]
#[cfg(proc_macro_span_location)]
Span::Compiler(s) => LineColumn {
line: s.line(),
column: s.column().saturating_sub(1),
line: proc_macro_span_location::line(s),
column: proc_macro_span_location::column(s).saturating_sub(1),
},
#[cfg(not(proc_macro_span))]
#[cfg(not(proc_macro_span_location))]
Span::Compiler(_) => LineColumn { line: 0, column: 0 },
Span::Fallback(s) => s.start(),
}
@@ -492,24 +455,46 @@ impl Span {
#[cfg(span_locations)]
pub(crate) fn end(&self) -> LineColumn {
match self {
#[cfg(proc_macro_span)]
#[cfg(proc_macro_span_location)]
Span::Compiler(s) => {
let end = s.end();
let end = proc_macro_span_location::end(s);
LineColumn {
line: end.line(),
column: end.column().saturating_sub(1),
line: proc_macro_span_location::line(&end),
column: proc_macro_span_location::column(&end).saturating_sub(1),
}
}
#[cfg(not(proc_macro_span))]
#[cfg(not(proc_macro_span_location))]
Span::Compiler(_) => LineColumn { line: 0, column: 0 },
Span::Fallback(s) => s.end(),
}
}
#[cfg(span_locations)]
pub(crate) fn file(&self) -> String {
match self {
#[cfg(proc_macro_span_file)]
Span::Compiler(s) => proc_macro_span_file::file(s),
#[cfg(not(proc_macro_span_file))]
Span::Compiler(_) => "<token stream>".to_owned(),
Span::Fallback(s) => s.file(),
}
}
#[cfg(span_locations)]
pub(crate) fn local_file(&self) -> Option<PathBuf> {
match self {
#[cfg(proc_macro_span_file)]
Span::Compiler(s) => proc_macro_span_file::local_file(s),
#[cfg(not(proc_macro_span_file))]
Span::Compiler(_) => None,
Span::Fallback(s) => s.local_file(),
}
}
pub(crate) fn join(&self, other: Span) -> Option<Span> {
let ret = match (self, other) {
#[cfg(proc_macro_span)]
(Span::Compiler(a), Span::Compiler(b)) => Span::Compiler(a.join(b)?),
(Span::Compiler(a), Span::Compiler(b)) => Span::Compiler(proc_macro_span::join(a, b)?),
(Span::Fallback(a), Span::Fallback(b)) => Span::Fallback(a.join(b)?),
_ => return None,
};
@@ -952,7 +937,7 @@ impl Literal {
pub(crate) fn subspan<R: RangeBounds<usize>>(&self, range: R) -> Option<Span> {
match self {
#[cfg(proc_macro_span)]
Literal::Compiler(lit) => lit.subspan(range).map(Span::Compiler),
Literal::Compiler(lit) => proc_macro_span::subspan(lit, range).map(Span::Compiler),
#[cfg(not(proc_macro_span))]
Literal::Compiler(_lit) => None,
Literal::Fallback(lit) => lit.subspan(range).map(Span::Fallback),
tests/comments.rs
+1 -1
View File
@@ -1,4 +1,4 @@
#![allow(clippy::assertions_on_result_states)]
#![allow(clippy::assertions_on_result_states, clippy::uninlined_format_args)]
use proc_macro2::{Delimiter, Literal, Spacing, TokenStream, TokenTree};
tests/features.rs
+2
View File
@@ -1,3 +1,5 @@
#![allow(clippy::assertions_on_constants, clippy::ignore_without_reason)]
#[test]
#[ignore]
fn make_sure_no_proc_macro() {
tests/marker.rs
+3 -6
View File
@@ -56,19 +56,17 @@ assert_impl!(TokenTree is not Send or Sync);
#[cfg(procmacro2_semver_exempt)]
mod semver_exempt {
use proc_macro2::{LineColumn, SourceFile};
use proc_macro2::LineColumn;
assert_impl!(LineColumn is Send and Sync);
assert_impl!(SourceFile is not Send or Sync);
}
mod unwind_safe {
#[cfg(procmacro2_semver_exempt)]
use proc_macro2::LineColumn;
use proc_macro2::{
Delimiter, Group, Ident, LexError, Literal, Punct, Spacing, Span, TokenStream, TokenTree,
};
#[cfg(procmacro2_semver_exempt)]
use proc_macro2::{LineColumn, SourceFile};
use std::panic::{RefUnwindSafe, UnwindSafe};
macro_rules! assert_unwind_safe {
@@ -95,6 +93,5 @@ mod unwind_safe {
#[cfg(procmacro2_semver_exempt)]
assert_unwind_safe! {
LineColumn
SourceFile
}
}
tests/test.rs
+211 -23
View File
@@ -4,7 +4,8 @@
clippy::needless_pass_by_value,
clippy::needless_raw_string_hashes,
clippy::non_ascii_literal,
clippy::octal_escapes
clippy::octal_escapes,
clippy::uninlined_format_args
)]
use proc_macro2::{Ident, Literal, Punct, Spacing, Span, TokenStream, TokenTree};
@@ -145,6 +146,30 @@ fn literal_raw_string() {
.unwrap_err();
}
#[cfg(procmacro2_semver_exempt)]
#[test]
fn literal_string_value() {
for string in ["", "...", "...\t...", "...\\...", "...\0...", "...\u{1}..."] {
assert_eq!(string, Literal::string(string).str_value().unwrap());
assert_eq!(
string,
format!("r\"{string}\"")
.parse::<Literal>()
.unwrap()
.str_value()
.unwrap(),
);
assert_eq!(
string,
format!("r##\"{string}\"##")
.parse::<Literal>()
.unwrap()
.str_value()
.unwrap(),
);
}
}
#[test]
fn literal_byte_character() {
#[track_caller]
@@ -190,6 +215,42 @@ fn literal_byte_string() {
"br\"\u{a0}\"".parse::<TokenStream>().unwrap_err();
}
#[cfg(procmacro2_semver_exempt)]
#[test]
fn literal_byte_string_value() {
for bytestr in [
&b""[..],
b"...",
b"...\t...",
b"...\\...",
b"...\0...",
b"...\xF0...",
] {
assert_eq!(
bytestr,
Literal::byte_string(bytestr).byte_str_value().unwrap(),
);
if let Ok(string) = str::from_utf8(bytestr) {
assert_eq!(
bytestr,
format!("br\"{string}\"")
.parse::<Literal>()
.unwrap()
.byte_str_value()
.unwrap(),
);
assert_eq!(
bytestr,
format!("br##\"{string}\"##")
.parse::<Literal>()
.unwrap()
.byte_str_value()
.unwrap(),
);
}
}
}
#[test]
fn literal_c_string() {
#[track_caller]
@@ -261,6 +322,42 @@ fn literal_c_string() {
}
}
#[cfg(procmacro2_semver_exempt)]
#[test]
fn literal_c_string_value() {
for cstr in [
c"",
c"...",
c"...\t...",
c"...\\...",
c"...\u{1}...",
c"...\xF0...",
] {
assert_eq!(
cstr.to_bytes_with_nul(),
Literal::c_string(cstr).cstr_value().unwrap(),
);
if let Ok(string) = cstr.to_str() {
assert_eq!(
cstr.to_bytes_with_nul(),
format!("cr\"{string}\"")
.parse::<Literal>()
.unwrap()
.cstr_value()
.unwrap(),
);
assert_eq!(
cstr.to_bytes_with_nul(),
format!("cr##\"{string}\"##")
.parse::<Literal>()
.unwrap()
.cstr_value()
.unwrap(),
);
}
}
}
#[test]
fn literal_character() {
#[track_caller]
@@ -453,6 +550,81 @@ fn source_text() {
assert_eq!("a", second.span().source_text().unwrap());
}
#[test]
fn lifetimes() {
let mut tokens = "'a 'static 'struct 'r#gen 'r#prefix#lifetime"
.parse::<TokenStream>()
.unwrap()
.into_iter();
assert!(match tokens.next() {
Some(TokenTree::Punct(punct)) => {
punct.as_char() == '\'' && punct.spacing() == Spacing::Joint
}
_ => false,
});
assert!(match tokens.next() {
Some(TokenTree::Ident(ident)) => ident == "a",
_ => false,
});
assert!(match tokens.next() {
Some(TokenTree::Punct(punct)) => {
punct.as_char() == '\'' && punct.spacing() == Spacing::Joint
}
_ => false,
});
assert!(match tokens.next() {
Some(TokenTree::Ident(ident)) => ident == "static",
_ => false,
});
assert!(match tokens.next() {
Some(TokenTree::Punct(punct)) => {
punct.as_char() == '\'' && punct.spacing() == Spacing::Joint
}
_ => false,
});
assert!(match tokens.next() {
Some(TokenTree::Ident(ident)) => ident == "struct",
_ => false,
});
assert!(match tokens.next() {
Some(TokenTree::Punct(punct)) => {
punct.as_char() == '\'' && punct.spacing() == Spacing::Joint
}
_ => false,
});
assert!(match tokens.next() {
Some(TokenTree::Ident(ident)) => ident == "r#gen",
_ => false,
});
assert!(match tokens.next() {
Some(TokenTree::Punct(punct)) => {
punct.as_char() == '\'' && punct.spacing() == Spacing::Joint
}
_ => false,
});
assert!(match tokens.next() {
Some(TokenTree::Ident(ident)) => ident == "r#prefix",
_ => false,
});
assert!(match tokens.next() {
Some(TokenTree::Punct(punct)) => {
punct.as_char() == '#' && punct.spacing() == Spacing::Alone
}
_ => false,
});
assert!(match tokens.next() {
Some(TokenTree::Ident(ident)) => ident == "lifetime",
_ => false,
});
"' a".parse::<TokenStream>().unwrap_err();
"' r#gen".parse::<TokenStream>().unwrap_err();
"' prefix#lifetime".parse::<TokenStream>().unwrap_err();
"'prefix#lifetime".parse::<TokenStream>().unwrap_err();
"'aa'bb".parse::<TokenStream>().unwrap_err();
"'r#gen'a".parse::<TokenStream>().unwrap_err();
}
#[test]
fn roundtrip() {
fn roundtrip(p: &str) {
@@ -549,9 +721,8 @@ fn default_span() {
let end = Span::call_site().end();
assert_eq!(end.line, 1);
assert_eq!(end.column, 0);
let source_file = Span::call_site().source_file();
assert_eq!(source_file.path().to_string_lossy(), "<unspecified>");
assert!(!source_file.is_real());
assert_eq!(Span::call_site().file(), "<unspecified>");
assert!(Span::call_site().local_file().is_none());
}
#[cfg(procmacro2_semver_exempt)]
@@ -568,11 +739,8 @@ fn span_join() {
.into_iter()
.collect::<Vec<_>>();
assert!(source1[0].span().source_file() != source2[0].span().source_file());
assert_eq!(
source1[0].span().source_file(),
source1[1].span().source_file()
);
assert!(source1[0].span().file() != source2[0].span().file());
assert_eq!(source1[0].span().file(), source1[1].span().file());
let joined1 = source1[0].span().join(source1[1].span());
let joined2 = source1[0].span().join(source2[0].span());
@@ -586,10 +754,7 @@ fn span_join() {
assert_eq!(end.line, 2);
assert_eq!(end.column, 3);
assert_eq!(
joined1.unwrap().source_file(),
source1[0].span().source_file()
);
assert_eq!(joined1.unwrap().file(), source1[0].span().file());
}
#[test]
@@ -619,9 +784,8 @@ fn joint_last_token() {
let joint_punct = Punct::new(':', Spacing::Joint);
let stream = TokenStream::from(TokenTree::Punct(joint_punct));
let punct = match stream.into_iter().next().unwrap() {
TokenTree::Punct(punct) => punct,
_ => unreachable!(),
let TokenTree::Punct(punct) = stream.into_iter().next().unwrap() else {
unreachable!();
};
assert_eq!(punct.spacing(), Spacing::Joint);
}
@@ -636,17 +800,41 @@ fn raw_identifier() {
assert!(tts.next().is_none());
}
#[test]
fn test_display_ident() {
let ident = Ident::new("proc_macro", Span::call_site());
assert_eq!(format!("{ident}"), "proc_macro");
assert_eq!(format!("{ident:-^14}"), "proc_macro");
let ident = Ident::new_raw("proc_macro", Span::call_site());
assert_eq!(format!("{ident}"), "r#proc_macro");
assert_eq!(format!("{ident:-^14}"), "r#proc_macro");
}
#[test]
fn test_debug_ident() {
let ident = Ident::new("proc_macro", Span::call_site());
#[cfg(not(span_locations))]
let expected = "Ident(proc_macro)";
#[cfg(span_locations)]
let expected = "Ident { sym: proc_macro }";
let expected = if cfg!(span_locations) {
"Ident { sym: proc_macro }"
} else {
"Ident(proc_macro)"
};
assert_eq!(expected, format!("{:?}", ident));
let ident = Ident::new_raw("proc_macro", Span::call_site());
let expected = if cfg!(span_locations) {
"Ident { sym: r#proc_macro }"
} else {
"Ident(r#proc_macro)"
};
assert_eq!(expected, format!("{:?}", ident));
}
#[test]
fn test_display_tokenstream() {
let tts = TokenStream::from_str("[a + 1]").unwrap();
assert_eq!(format!("{tts}"), "[a + 1]");
assert_eq!(format!("{tts:-^5}"), "[a + 1]");
}
#[test]
tests/test_size.rs
-2
View File
@@ -4,7 +4,6 @@ extern crate proc_macro;
use std::mem;
#[rustversion::attr(before(1.64), ignore = "requires Rust 1.64+")]
#[cfg_attr(not(target_pointer_width = "64"), ignore = "only applicable to 64-bit")]
#[cfg_attr(randomize_layout, ignore = "disabled due to randomized layout")]
#[test]
@@ -64,7 +63,6 @@ fn test_proc_macro2_wrapper_size_without_locations() {
assert_eq!(mem::size_of::<proc_macro2::TokenStream>(), 32);
}
#[rustversion::attr(before(1.65), ignore = "requires Rust 1.65+")]
#[cfg_attr(not(target_pointer_width = "64"), ignore = "only applicable to 64-bit")]
#[cfg_attr(randomize_layout, ignore = "disabled due to randomized layout")]
#[cfg_attr(not(wrap_proc_macro), ignore = "fallback mode")]
tests/ui/Cargo.toml
+2 -2
View File
@@ -2,7 +2,7 @@
name = "proc-macro2-ui-test"
version = "0.0.0"
authors = ["David Tolnay <dtolnay@gmail.com>"]
edition = "2018"
edition = "2021"
publish = false
[[test]]
@@ -12,4 +12,4 @@ path = "compiletest.rs"
[dev-dependencies]
proc-macro2 = { path = "../.." }
rustversion = "1.0"
trybuild = { version = "1.0.49", features = ["diff"] }
trybuild = { version = "1.0.108", features = ["diff"] }
tests/ui/test-not-send.stderr
+1 -1
View File
@@ -31,7 +31,7 @@ error[E0277]: `Rc<()>` cannot be sent between threads safely
note: required because it appears within the type `PhantomData<Rc<()>>`
--> $RUST/core/src/marker.rs
|
| pub struct PhantomData<T: ?Sized>;
| pub struct PhantomData<T: PointeeSized>;
| ^^^^^^^^^^^
note: required because it appears within the type `proc_macro2::marker::ProcMacroAutoTraits`
--> $WORKSPACE/src/marker.rs