From 76b407eb79bd1fa25db860925989d8afbd087b52 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BE=90=E6=9C=AA=E6=9D=A5?= Date: Tue, 2 Apr 2024 16:50:13 +0800 Subject: [PATCH] =?UTF-8?q?syn=201.0.107=E5=8D=87=E7=BA=A7=E8=87=B32.0.48?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: 徐未来 --- .clippy.toml | 1 - .github/workflows/ci.yml | 68 +- BUILD.gn | 4 +- Cargo.toml | 35 +- LICENSE-APACHE | 25 - README.OpenSource | 2 +- README.md | 21 +- benches/file.rs | 10 +- benches/rust.rs | 34 +- build.rs | 102 +- codegen/Cargo.toml | 19 +- codegen/src/cfg.rs | 27 +- codegen/src/clone.rs | 21 +- codegen/src/debug.rs | 130 +- codegen/src/eq.rs | 25 +- codegen/src/file.rs | 6 +- codegen/src/fold.rs | 74 +- codegen/src/gen.rs | 9 +- codegen/src/hash.rs | 33 +- codegen/src/json.rs | 5 +- codegen/src/main.rs | 10 +- codegen/src/parse.rs | 374 +- codegen/src/snapshot.rs | 221 +- codegen/src/version.rs | 7 +- codegen/src/visit.rs | 62 +- codegen/src/visit_mut.rs | 64 +- codegen/src/workspace_path.rs | 8 + dev/Cargo.toml | 8 +- examples/dump-syntax/Cargo.toml | 8 +- examples/dump-syntax/src/main.rs | 17 +- examples/heapsize/example/Cargo.toml | 2 +- examples/heapsize/heapsize/Cargo.toml | 2 +- examples/heapsize/heapsize_derive/Cargo.toml | 6 +- examples/lazy-static/example/Cargo.toml | 2 +- examples/lazy-static/lazy-static/Cargo.toml | 6 +- examples/trace-var/README.md | 2 +- examples/trace-var/example/Cargo.toml | 2 +- examples/trace-var/trace-var/Cargo.toml | 8 +- examples/trace-var/trace-var/src/lib.rs | 24 +- fuzz/.gitignore | 1 + fuzz/Cargo.toml | 5 +- fuzz/fuzz_targets/create_token_buffer.rs | 2 +- fuzz/fuzz_targets/parse_file.rs | 2 +- json/Cargo.toml | 19 +- json/src/lib.rs | 13 +- src/attr.rs | 654 +-- src/await.rs | 2 - src/bigint.rs | 6 +- src/buffer.rs | 100 +- src/custom_keyword.rs | 48 +- src/custom_punctuation.rs | 36 +- src/data.rs | 339 +- src/derive.rs | 41 +- src/discouraged.rs | 25 + src/error.rs | 113 +- src/export.rs | 70 +- src/expr.rs | 1972 ++++---- src/ext.rs | 8 +- src/file.rs | 4 +- src/gen/clone.rs | 432 +- src/gen/debug.rs | 2954 ++++++------ src/gen/eq.rs | 481 +- src/gen/fold.rs | 1598 ++++--- src/gen/hash.rs | 601 ++- src/gen/visit.rs | 1876 ++++---- src/gen/visit_mut.rs | 1939 ++++---- src/gen_helper.rs | 124 +- src/generics.rs | 429 +- src/group.rs | 19 +- src/ident.rs | 122 +- src/item.rs | 1773 +++---- src/lib.rs | 542 ++- src/lifetime.rs | 12 +- src/lit.rs | 325 +- src/lookahead.rs | 8 +- src/mac.rs | 64 +- src/macros.rs | 87 +- src/meta.rs | 426 ++ src/op.rs | 156 +- src/parse.rs | 168 +- src/parse_macro_input.rs | 57 +- src/parse_quote.rs | 58 +- src/pat.rs | 756 ++- src/path.rs | 523 ++- src/print.rs | 2 +- src/punctuated.rs | 129 +- src/reserved.rs | 44 - src/restriction.rs | 171 + src/sealed.rs | 2 +- src/span.rs | 38 +- src/spanned.rs | 18 +- src/stmt.rs | 253 +- src/thread.rs | 25 +- src/token.rs | 691 +-- src/tt.rs | 4 +- src/ty.rs | 409 +- src/verbatim.rs | 4 +- src/whitespace.rs | 2 +- syn.json | 1062 +++-- tests/common/eq.rs | 173 +- tests/common/parse.rs | 5 +- tests/crates/Cargo.toml | 17 - tests/crates/main.rs | 82 - tests/debug/gen.rs | 4397 ++++++++---------- tests/debug/mod.rs | 30 +- tests/features/Cargo.toml | 6 +- tests/features/lib.rs | 17 +- tests/features/macro/Cargo.toml | 16 - tests/features/macro/lib.rs | 55 - tests/macros/mod.rs | 41 +- tests/regression.rs | 2 + tests/regression/issue1108.rs | 2 +- tests/repo/mod.rs | 254 +- tests/test_asyncness.rs | 16 +- tests/test_attribute.rs | 171 +- tests/test_derive_input.rs | 291 +- tests/test_expr.rs | 366 +- tests/test_generics.rs | 77 +- tests/test_grouping.rs | 9 +- tests/test_item.rs | 64 +- tests/test_iterators.rs | 2 + tests/test_lit.rs | 27 +- tests/test_meta.rs | 272 +- tests/test_parse_buffer.rs | 4 +- tests/test_parse_quote.rs | 164 + tests/test_parse_stream.rs | 12 +- tests/test_pat.rs | 111 +- tests/test_path.rs | 40 +- tests/test_precedence.rs | 448 +- tests/test_receiver.rs | 328 +- tests/test_round_trip.rs | 48 +- tests/test_shebang.rs | 38 +- tests/test_size.rs | 17 +- tests/test_stmt.rs | 281 +- tests/test_token_trees.rs | 2 + tests/test_ty.rs | 117 +- tests/test_visibility.rs | 12 +- 137 files changed, 16255 insertions(+), 15087 deletions(-) delete mode 100644 .clippy.toml create mode 100644 codegen/src/workspace_path.rs delete mode 100644 src/await.rs create mode 100644 src/meta.rs delete mode 100644 src/reserved.rs create mode 100644 src/restriction.rs delete mode 100644 tests/crates/Cargo.toml delete mode 100644 tests/crates/main.rs delete mode 100644 tests/features/macro/Cargo.toml delete mode 100644 tests/features/macro/lib.rs create mode 100644 tests/test_parse_quote.rs diff --git a/.clippy.toml b/.clippy.toml deleted file mode 100644 index 3d30690f..00000000 --- a/.clippy.toml +++ /dev/null @@ -1 +0,0 @@ -msrv = "1.31.0" diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e277c969..5d9f630a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -3,6 +3,7 @@ name: CI on: push: pull_request: + workflow_dispatch: schedule: [cron: "40 1 * * *"] permissions: @@ -12,24 +13,31 @@ env: RUSTFLAGS: -Dwarnings jobs: + pre_ci: + uses: dtolnay/.github/.github/workflows/pre_ci.yml@master + test: name: Tests + needs: pre_ci + if: needs.pre_ci.outputs.continue runs-on: ubuntu-latest timeout-minutes: 45 steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - uses: dtolnay/rust-toolchain@nightly with: - components: rustc-dev + components: llvm-tools, rustc-dev - run: cargo test --all-features --release --tests build: name: ${{matrix.name || format('Rust {0}', matrix.rust)}} + needs: pre_ci + if: needs.pre_ci.outputs.continue runs-on: ${{matrix.os || 'ubuntu'}}-latest strategy: fail-fast: false matrix: - rust: [stable, beta, 1.31.0] + rust: [stable, beta, 1.56.0] include: - rust: nightly components: rustc-dev @@ -46,7 +54,7 @@ jobs: target: ${{matrix.target && format('--target={0}', matrix.target)}} timeout-minutes: 45 steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - uses: dtolnay/rust-toolchain@master with: toolchain: ${{matrix.rust}} @@ -72,65 +80,80 @@ jobs: examples: name: Examples + needs: pre_ci + if: needs.pre_ci.outputs.continue runs-on: ubuntu-latest timeout-minutes: 45 steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - uses: dtolnay/rust-toolchain@nightly - run: cargo check --manifest-path examples/dump-syntax/Cargo.toml - run: cargo check --manifest-path examples/heapsize/example/Cargo.toml - run: cargo check --manifest-path examples/lazy-static/example/Cargo.toml - run: cargo check --manifest-path examples/trace-var/example/Cargo.toml - docs: - name: Docs + doc: + name: Documentation + needs: pre_ci + if: needs.pre_ci.outputs.continue runs-on: ubuntu-latest - env: - RUSTDOCFLAGS: --cfg=doc_cfg -Dbroken_intra_doc_links timeout-minutes: 45 + env: + RUSTDOCFLAGS: -Dwarnings steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - uses: dtolnay/rust-toolchain@nightly + - uses: dtolnay/install@cargo-docs-rs + - run: cargo docs-rs + - run: cargo docs-rs --manifest-path json/Cargo.toml - run: cargo test --all-features --doc - - run: cargo doc --all-features codegen: name: Codegen + needs: pre_ci + if: needs.pre_ci.outputs.continue runs-on: ubuntu-latest timeout-minutes: 45 steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - uses: dtolnay/rust-toolchain@stable - - run: cd codegen && cargo run + - run: cargo run --manifest-path codegen/Cargo.toml - run: git diff --exit-code - msrv: + minimal: name: Minimal versions + needs: pre_ci + if: needs.pre_ci.outputs.continue runs-on: ubuntu-latest timeout-minutes: 45 steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - uses: dtolnay/rust-toolchain@nightly - - run: cargo update -Z minimal-versions - - run: cargo check --all-features + - run: cargo generate-lockfile -Z minimal-versions + - run: cargo check --all-features --locked fuzz: name: Fuzz + needs: pre_ci + if: needs.pre_ci.outputs.continue runs-on: ubuntu-latest timeout-minutes: 45 steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - uses: dtolnay/rust-toolchain@nightly - uses: dtolnay/install@cargo-fuzz - - run: cargo fuzz build -O + - run: cargo fuzz check miri: name: Miri + needs: pre_ci + if: needs.pre_ci.outputs.continue runs-on: ubuntu-latest timeout-minutes: 45 steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - uses: dtolnay/rust-toolchain@miri + - run: cargo miri setup - run: cargo miri test --all-features env: MIRIFLAGS: -Zmiri-strict-provenance @@ -141,11 +164,12 @@ jobs: if: github.event_name != 'pull_request' timeout-minutes: 45 steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - uses: dtolnay/rust-toolchain@nightly with: components: clippy,rustc-dev - run: cargo clippy --all-features --tests --benches -- -Dclippy::all -Dclippy::pedantic + - run: cargo clippy --manifest-path codegen/Cargo.toml -- -Dclippy::all -Dclippy::pedantic outdated: name: Outdated @@ -153,7 +177,7 @@ jobs: if: github.event_name != 'pull_request' timeout-minutes: 45 steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - uses: dtolnay/install@cargo-outdated - run: cargo outdated --workspace --exit-code 1 - run: cargo outdated --manifest-path fuzz/Cargo.toml --exit-code 1 diff --git a/BUILD.gn b/BUILD.gn index e2796827..21ce2e47 100644 --- a/BUILD.gn +++ b/BUILD.gn @@ -19,8 +19,8 @@ ohos_cargo_crate("lib") { crate_root = "src/lib.rs" sources = [ "src/lib.rs" ] - edition = "2018" - cargo_pkg_version = "1.0.107" + edition = "2021" + cargo_pkg_version = "2.0.48" cargo_pkg_authors = "David Tolnay " cargo_pkg_name = "syn" cargo_pkg_description = "Parser for Rust source code" diff --git a/Cargo.toml b/Cargo.toml index dd32f92d..c17ef658 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,14 +1,13 @@ [package] name = "syn" -version = "1.0.107" # don't forget to update html_root_url and syn.json +version = "2.0.48" # don't forget to update html_root_url and syn.json authors = ["David Tolnay "] categories = ["development-tools::procedural-macro-helpers", "parser-implementations"] description = "Parser for Rust source code" documentation = "https://docs.rs/syn" -edition = "2018" +edition = "2021" include = [ "/benches/**", - "/build.rs", "/Cargo.toml", "/LICENSE-APACHE", "/LICENSE-MIT", @@ -19,7 +18,7 @@ include = [ keywords = ["macros", "syn"] license = "MIT OR Apache-2.0" repository = "https://github.com/dtolnay/syn" -rust-version = "1.31" +rust-version = "1.56" [features] default = ["derive", "parsing", "printing", "clone-impls", "proc-macro"] @@ -36,23 +35,23 @@ proc-macro = ["proc-macro2/proc-macro", "quote/proc-macro"] test = ["syn-test-suite/all-features"] [dependencies] -proc-macro2 = { version = "1.0.46", default-features = false } -quote = { version = "1.0", optional = true, default-features = false } -unicode-ident = "1.0" +proc-macro2 = { version = "1.0.75", default-features = false } +quote = { version = "1.0.35", optional = true, default-features = false } +unicode-ident = "1" [dev-dependencies] -anyhow = "1.0" -automod = "1.0" -flate2 = "1.0" -insta = "1.0" -rayon = "1.0" -ref-cast = "1.0" -regex = "1.0" +anyhow = "1" +automod = "1" +flate2 = "1" +insta = "1" +rayon = "1" +ref-cast = "1" reqwest = { version = "0.11", features = ["blocking"] } +rustversion = "1" syn-test-suite = { version = "0", path = "tests/features" } tar = "0.4.16" -termcolor = "1.0" -walkdir = "2.1" +termcolor = "1" +walkdir = "2.3.2" [lib] doc-scrape-examples = false @@ -69,7 +68,7 @@ required-features = ["full", "parsing"] [package.metadata.docs.rs] all-features = true targets = ["x86_64-unknown-linux-gnu"] -rustdoc-args = ["--cfg", "doc_cfg"] +rustdoc-args = ["--cfg", "doc_cfg", "--generate-link-to-definition"] [package.metadata.playground] features = ["full", "visit", "visit-mut", "fold", "extra-traits"] @@ -85,7 +84,5 @@ members = [ "examples/lazy-static/lazy-static", "examples/trace-var/example", "examples/trace-var/trace-var", - "json", - "tests/crates", "tests/features", ] diff --git a/LICENSE-APACHE b/LICENSE-APACHE index 16fe87b0..1b5ec8b7 100644 --- a/LICENSE-APACHE +++ b/LICENSE-APACHE @@ -174,28 +174,3 @@ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION of your accepting any such warranty or additional liability. END OF TERMS AND CONDITIONS - -APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - -Copyright [yyyy] [name of copyright owner] - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. diff --git a/README.OpenSource b/README.OpenSource index f51c5bd1..aaf91898 100644 --- a/README.OpenSource +++ b/README.OpenSource @@ -3,7 +3,7 @@ "Name": "syn", "License": "Apache License V2.0, MIT", "License File": "LICENSE-APACHE, LICENSE-MIT", - "Version Number": "1.0.107", + "Version Number": "2.0.48", "Owner": "fangting12@huawei.com", "Upstream URL": "https://github.com/dtolnay/syn", "Description": "A Rust library that provides support for parsing Rust code." diff --git a/README.md b/README.md index eeef83dd..e8d99abc 100644 --- a/README.md +++ b/README.md @@ -39,14 +39,14 @@ contains some APIs that may be useful more generally. procedural macros enable only what they need, and do not pay in compile time for all the rest. -[`syn::File`]: https://docs.rs/syn/1.0/syn/struct.File.html -[`syn::Item`]: https://docs.rs/syn/1.0/syn/enum.Item.html -[`syn::Expr`]: https://docs.rs/syn/1.0/syn/enum.Expr.html -[`syn::Type`]: https://docs.rs/syn/1.0/syn/enum.Type.html -[`syn::DeriveInput`]: https://docs.rs/syn/1.0/syn/struct.DeriveInput.html -[parser functions]: https://docs.rs/syn/1.0/syn/parse/index.html +[`syn::File`]: https://docs.rs/syn/2.0/syn/struct.File.html +[`syn::Item`]: https://docs.rs/syn/2.0/syn/enum.Item.html +[`syn::Expr`]: https://docs.rs/syn/2.0/syn/enum.Expr.html +[`syn::Type`]: https://docs.rs/syn/2.0/syn/enum.Type.html +[`syn::DeriveInput`]: https://docs.rs/syn/2.0/syn/struct.DeriveInput.html +[parser functions]: https://docs.rs/syn/2.0/syn/parse/index.html -*Version requirement: Syn supports rustc 1.31 and up.* +*Version requirement: Syn supports rustc 1.56 and up.* [*Release notes*](https://github.com/dtolnay/syn/releases) @@ -76,7 +76,7 @@ tokens back to the compiler to compile into the user's crate. ```toml [dependencies] -syn = "1.0" +syn = "2.0" quote = "1.0" [lib] @@ -104,9 +104,8 @@ pub fn my_macro(input: TokenStream) -> TokenStream { ``` The [`heapsize`] example directory shows a complete working implementation of a -derive macro. It works on any Rust compiler 1.31+. The example derives a -`HeapSize` trait which computes an estimate of the amount of heap memory owned -by a value. +derive macro. The example derives a `HeapSize` trait which computes an estimate +of the amount of heap memory owned by a value. [`heapsize`]: examples/heapsize diff --git a/benches/file.rs b/benches/file.rs index bd4a247d..b4247239 100644 --- a/benches/file.rs +++ b/benches/file.rs @@ -4,8 +4,11 @@ #![recursion_limit = "1024"] #![allow( clippy::items_after_statements, + clippy::manual_let_else, + clippy::match_like_matches_macro, clippy::missing_panics_doc, - clippy::must_use_candidate + clippy::must_use_candidate, + clippy::uninlined_format_args )] extern crate test; @@ -14,10 +17,9 @@ extern crate test; #[path = "../tests/macros/mod.rs"] mod macros; -#[path = "../tests/common/mod.rs"] -mod common; +#[allow(dead_code)] #[path = "../tests/repo/mod.rs"] -pub mod repo; +mod repo; use proc_macro2::{Span, TokenStream}; use std::fs; diff --git a/benches/rust.rs b/benches/rust.rs index e3f8f550..15536db4 100644 --- a/benches/rust.rs +++ b/benches/rust.rs @@ -5,14 +5,21 @@ #![cfg_attr(not(syn_only), feature(rustc_private))] #![recursion_limit = "1024"] -#![allow(clippy::cast_lossless, clippy::unnecessary_wraps)] +#![allow( + clippy::arc_with_non_send_sync, + clippy::cast_lossless, + clippy::let_underscore_untyped, + clippy::manual_let_else, + clippy::match_like_matches_macro, + clippy::uninlined_format_args, + clippy::unnecessary_wraps +)] #[macro_use] #[path = "../tests/macros/mod.rs"] mod macros; -#[path = "../tests/common/mod.rs"] -mod common; +#[allow(dead_code)] #[path = "../tests/repo/mod.rs"] mod repo; @@ -38,6 +45,7 @@ mod syn_parse { #[cfg(not(syn_only))] mod librustc_parse { extern crate rustc_data_structures; + extern crate rustc_driver; extern crate rustc_error_messages; extern crate rustc_errors; extern crate rustc_parse; @@ -46,7 +54,7 @@ mod librustc_parse { use rustc_data_structures::sync::Lrc; use rustc_error_messages::FluentBundle; - use rustc_errors::{emitter::Emitter, translation::Translate, Diagnostic, Handler}; + use rustc_errors::{emitter::Emitter, translation::Translate, DiagCtxt, Diagnostic}; use rustc_session::parse::ParseSess; use rustc_span::source_map::{FilePathMapping, SourceMap}; use rustc_span::{edition::Edition, FileName}; @@ -71,10 +79,10 @@ mod librustc_parse { } rustc_span::create_session_if_not_set_then(Edition::Edition2018, |_| { - let cm = Lrc::new(SourceMap::new(FilePathMapping::empty())); + let source_map = Lrc::new(SourceMap::new(FilePathMapping::empty())); let emitter = Box::new(SilentEmitter); - let handler = Handler::with_emitter(false, None, emitter); - let sess = ParseSess::with_span_handler(handler, cm); + let handler = DiagCtxt::with_emitter(emitter); + let sess = ParseSess::with_dcx(handler, source_map); if let Err(diagnostic) = rustc_parse::parse_crate_from_source_str( FileName::Custom("bench".to_owned()), content.to_owned(), @@ -91,7 +99,7 @@ mod librustc_parse { #[cfg(not(syn_only))] mod read_from_disk { pub fn bench(content: &str) -> Result<(), ()> { - _ = content; + let _ = content; Ok(()) } } @@ -101,9 +109,13 @@ fn exec(mut codepath: impl FnMut(&str) -> Result<(), ()>) -> Duration { let mut success = 0; let mut total = 0; - walkdir::WalkDir::new("tests/rust/src") - .into_iter() - .filter_entry(repo::base_dir_filter) + ["tests/rust/compiler", "tests/rust/library"] + .iter() + .flat_map(|dir| { + walkdir::WalkDir::new(dir) + .into_iter() + .filter_entry(repo::base_dir_filter) + }) .for_each(|entry| { let entry = entry.unwrap(); let path = entry.path(); diff --git a/build.rs b/build.rs index 1a2c077b..f14ce219 100644 --- a/build.rs +++ b/build.rs @@ -1,51 +1,81 @@ use std::env; -use std::process::Command; -use std::str; +use std::ffi::OsString; +use std::process::{self, Command, Stdio}; // The rustc-cfg strings below are *not* public API. Please let us know by // opening a GitHub issue if your build environment requires some way to enable // these cfgs other than by executing our build script. fn main() { - let compiler = match rustc_version() { - Some(compiler) => compiler, - None => return, - }; + println!("cargo:rerun-if-changed=build.rs"); - if compiler.minor < 36 { - println!("cargo:rustc-cfg=syn_omit_await_from_token_macro"); - } + // Note: add "/build.rs" to package.include in Cargo.toml if adding any + // conditional compilation within the library. - if compiler.minor < 39 { - println!("cargo:rustc-cfg=syn_no_const_vec_new"); - } - - if compiler.minor < 40 { - println!("cargo:rustc-cfg=syn_no_non_exhaustive"); - } - - if compiler.minor < 56 { - println!("cargo:rustc-cfg=syn_no_negative_literal_parse"); - } - - if !compiler.nightly { + if !unstable() { println!("cargo:rustc-cfg=syn_disable_nightly_tests"); } } -struct Compiler { - minor: u32, - nightly: bool, +fn unstable() -> bool { + let rustc = cargo_env_var("RUSTC"); + + // Pick up Cargo rustc configuration. + let mut cmd = if let Some(wrapper) = env::var_os("RUSTC_WRAPPER") { + let mut cmd = Command::new(wrapper); + // The wrapper's first argument is supposed to be the path to rustc. + cmd.arg(rustc); + cmd + } else { + Command::new(rustc) + }; + + cmd.stdin(Stdio::null()); + cmd.stdout(Stdio::null()); + cmd.stderr(Stdio::null()); + cmd.arg("-"); + + // Find out whether this is a nightly or dev build. + cmd.env_remove("RUSTC_BOOTSTRAP"); + cmd.arg("-Zcrate-attr=feature(rustc_private)"); + + // Pass `-Zunpretty` to terminate earlier without writing out any "emit" + // files. Use `expanded` to proceed far enough to actually apply crate + // attrs. With `unpretty=normal` or `--print`, not enough compilation + // happens to recognize that the feature attribute is unstable. + cmd.arg("-Zunpretty=expanded"); + + // Set #![no_std] to bypass loading libstd.rlib. This is a 7.5% speedup. + cmd.arg("-Zcrate-attr=no_std"); + + cmd.arg("--crate-type=lib"); + cmd.arg("--edition=2021"); + + if let Some(target) = env::var_os("TARGET") { + cmd.arg("--target").arg(target); + } + + // If Cargo wants to set RUSTFLAGS, use that. + if let Ok(rustflags) = env::var("CARGO_ENCODED_RUSTFLAGS") { + if !rustflags.is_empty() { + for arg in rustflags.split('\x1f') { + cmd.arg(arg); + } + } + } + + // This rustc invocation should take around 0.03 seconds. + match cmd.status() { + Ok(status) => status.success(), + Err(_) => false, + } } -fn rustc_version() -> Option { - let rustc = env::var_os("RUSTC")?; - let output = Command::new(rustc).arg("--version").output().ok()?; - let version = str::from_utf8(&output.stdout).ok()?; - let mut pieces = version.split('.'); - if pieces.next() != Some("rustc 1") { - return None; - } - let minor = pieces.next()?.parse().ok()?; - let nightly = version.contains("nightly") || version.ends_with("-dev"); - Some(Compiler { minor, nightly }) +fn cargo_env_var(key: &str) -> OsString { + env::var_os(key).unwrap_or_else(|| { + eprintln!( + "Environment variable ${} is not set during execution of build script", + key, + ); + process::exit(1); + }) } diff --git a/codegen/Cargo.toml b/codegen/Cargo.toml index 41daf832..0cfb10af 100644 --- a/codegen/Cargo.toml +++ b/codegen/Cargo.toml @@ -2,24 +2,25 @@ name = "syn-internal-codegen" version = "0.0.0" authors = ["David Tolnay ", "Nika Layzell "] -edition = "2018" +edition = "2021" publish = false # this is an internal crate which should never be published [dependencies] -anyhow = "1.0" +anyhow = "1" color-backtrace = "0.4" -indexmap = { version = "1.0", features = ["serde-1"] } +indexmap = { version = "2", features = ["serde"] } inflections = "1.1" -prettyplease = "0.1" +prettyplease = "0.2.3" proc-macro2 = { version = "1.0.20", features = ["span-locations"] } -quote = "1.0" -semver = { version = "1.0", features = ["serde"] } -serde = { version = "1.0.88", features = ["derive"] } +quote = "1" +semver = { version = "1", features = ["serde"] } +serde = "1.0.88" +serde_derive = "1.0.88" serde_json = "1.0.38" +syn = { version = "2", features = ["derive", "full", "parsing", "printing"], default-features = false } syn-codegen = { path = "../json" } -syn = { version = "1.0", features = ["derive", "parsing", "printing", "full"], default-features = false } -thiserror = "1.0" +thiserror = "1" toml = "0.5" [workspace] diff --git a/codegen/src/cfg.rs b/codegen/src/cfg.rs index 5932860f..f11dc869 100644 --- a/codegen/src/cfg.rs +++ b/codegen/src/cfg.rs @@ -2,11 +2,28 @@ use proc_macro2::TokenStream; use quote::quote; use syn_codegen::Features; -pub fn features(features: &Features) -> TokenStream { +pub fn features<'a>( + features: &Features, + overriding_cfg: impl Into>, +) -> TokenStream { let features = &features.any; - match features.len() { - 0 => quote!(), - 1 => quote!(#[cfg(feature = #(#features)*)]), - _ => quote!(#[cfg(any(#(feature = #features),*))]), + let cfg = match features.len() { + 0 => None, + 1 => Some(quote! { cfg(feature = #(#features)*) }), + _ => Some(quote! { cfg(any(#(feature = #features),*)) }), + }; + match (cfg, overriding_cfg.into()) { + (Some(cfg), Some(overriding_cfg)) => quote! { + #[#cfg] + #[cfg_attr(doc_cfg, doc(cfg(feature = #overriding_cfg)))] + }, + (Some(cfg), None) => quote! { + #[#cfg] + #[cfg_attr(doc_cfg, doc(#cfg))] + }, + (None, Some(overriding_cfg)) => quote! { + #[cfg_attr(doc_cfg, doc(cfg(feature = #overriding_cfg)))] + }, + (None, None) => TokenStream::new(), } } diff --git a/codegen/src/clone.rs b/codegen/src/clone.rs index 7f6718ad..9db771e7 100644 --- a/codegen/src/clone.rs +++ b/codegen/src/clone.rs @@ -4,13 +4,14 @@ use proc_macro2::{Ident, Span, TokenStream}; use quote::{format_ident, quote}; use syn_codegen::{Data, Definitions, Node, Type}; -const DEBUG_SRC: &str = "../src/gen/clone.rs"; +const CLONE_SRC: &str = "src/gen/clone.rs"; fn expand_impl_body(defs: &Definitions, node: &Node) -> TokenStream { let type_name = &node.ident; let ident = Ident::new(type_name, Span::call_site()); match &node.data { + Data::Enum(variants) if variants.is_empty() => quote!(match *self {}), Data::Enum(variants) => { let arms = variants.iter().map(|(variant_name, fields)| { let variant = Ident::new(variant_name, Span::call_site()); @@ -40,18 +41,13 @@ fn expand_impl_body(defs: &Definitions, node: &Node) -> TokenStream { } } }); - let nonexhaustive = if node.exhaustive { - None - } else if node.ident == "Expr" { + let nonexhaustive = if node.ident == "Expr" { Some(quote! { - #[cfg(any(syn_no_non_exhaustive, not(feature = "full")))] + #[cfg(not(feature = "full"))] _ => unreachable!(), }) } else { - Some(quote! { - #[cfg(syn_no_non_exhaustive)] - _ => unreachable!(), - }) + None }; quote! { match self { @@ -80,7 +76,7 @@ fn expand_impl(defs: &Definitions, node: &Node) -> TokenStream { } let ident = Ident::new(&node.ident, Span::call_site()); - let cfg_features = cfg::features(&node.features); + let cfg_features = cfg::features(&node.features, "clone-impls"); let copy = node.ident == "AttrStyle" || node.ident == "BinOp" @@ -90,10 +86,8 @@ fn expand_impl(defs: &Definitions, node: &Node) -> TokenStream { if copy { return quote! { #cfg_features - #[cfg_attr(doc_cfg, doc(cfg(feature = "clone-impls")))] impl Copy for #ident {} #cfg_features - #[cfg_attr(doc_cfg, doc(cfg(feature = "clone-impls")))] impl Clone for #ident { fn clone(&self) -> Self { *self @@ -106,7 +100,6 @@ fn expand_impl(defs: &Definitions, node: &Node) -> TokenStream { quote! { #cfg_features - #[cfg_attr(doc_cfg, doc(cfg(feature = "clone-impls")))] impl Clone for #ident { fn clone(&self) -> Self { #body @@ -122,7 +115,7 @@ pub fn generate(defs: &Definitions) -> Result<()> { } file::write( - DEBUG_SRC, + CLONE_SRC, quote! { #![allow(clippy::clone_on_copy, clippy::expl_impl_clone_on_copy)] diff --git a/codegen/src/debug.rs b/codegen/src/debug.rs index 331421e6..b0f51752 100644 --- a/codegen/src/debug.rs +++ b/codegen/src/debug.rs @@ -2,16 +2,57 @@ use crate::{cfg, file, lookup}; use anyhow::Result; use proc_macro2::{Ident, Span, TokenStream}; use quote::{format_ident, quote}; +use std::collections::BTreeSet as Set; use syn_codegen::{Data, Definitions, Node, Type}; -const DEBUG_SRC: &str = "../src/gen/debug.rs"; +const DEBUG_SRC: &str = "src/gen/debug.rs"; -fn expand_impl_body(defs: &Definitions, node: &Node) -> TokenStream { +fn syntax_tree_enum<'a>( + enum_name: &str, + variant_name: &str, + fields: &'a [Type], +) -> Option<&'a str> { + if fields.len() != 1 { + return None; + } + const WHITELIST: &[(&str, &str)] = &[ + ("Meta", "Path"), + ("Pat", "Const"), + ("Pat", "Lit"), + ("Pat", "Macro"), + ("Pat", "Path"), + ("Pat", "Range"), + ("PathArguments", "AngleBracketed"), + ("PathArguments", "Parenthesized"), + ("Stmt", "Local"), + ("TypeParamBound", "Lifetime"), + ("Visibility", "Public"), + ("Visibility", "Restricted"), + ]; + match &fields[0] { + Type::Syn(ty) + if WHITELIST.contains(&(enum_name, variant_name)) + || enum_name.to_owned() + variant_name == *ty => + { + Some(ty) + } + _ => None, + } +} + +fn expand_impl_body( + defs: &Definitions, + node: &Node, + syntax_tree_variants: &Set<&str>, +) -> TokenStream { let type_name = &node.ident; let ident = Ident::new(type_name, Span::call_site()); + let is_syntax_tree_variant = syntax_tree_variants.contains(type_name.as_str()); - match &node.data { + let body = match &node.data { + Data::Enum(variants) if variants.is_empty() => quote!(match *self {}), Data::Enum(variants) => { + assert!(!is_syntax_tree_variant); let arms = variants.iter().map(|(variant_name, fields)| { let variant = Ident::new(variant_name, Span::call_site()); if fields.is_empty() { @@ -19,9 +60,6 @@ fn expand_impl_body(defs: &Definitions, node: &Node) -> TokenStream { #ident::#variant => formatter.write_str(#variant_name), } } else { - let pats = (0..fields.len()) - .map(|i| format_ident!("v{}", i)) - .collect::>(); let mut cfg = None; if node.ident == "Expr" { if let Type::Syn(ty) = &fields[0] { @@ -30,30 +68,37 @@ fn expand_impl_body(defs: &Definitions, node: &Node) -> TokenStream { } } } - quote! { - #cfg - #ident::#variant(#(#pats),*) => { - let mut formatter = formatter.debug_tuple(#variant_name); - #(formatter.field(#pats);)* - formatter.finish() + if syntax_tree_enum(type_name, variant_name, fields).is_some() { + quote! { + #cfg + #ident::#variant(v0) => v0.debug(formatter, #variant_name), + } + } else { + let pats = (0..fields.len()) + .map(|i| format_ident!("v{}", i)) + .collect::>(); + quote! { + #cfg + #ident::#variant(#(#pats),*) => { + let mut formatter = formatter.debug_tuple(#variant_name); + #(formatter.field(#pats);)* + formatter.finish() + } } } } }); - let nonexhaustive = if node.exhaustive { - None - } else if node.ident == "Expr" { + let nonexhaustive = if node.ident == "Expr" { Some(quote! { - #[cfg(any(syn_no_non_exhaustive, not(feature = "full")))] + #[cfg(not(feature = "full"))] _ => unreachable!(), }) } else { - Some(quote! { - #[cfg(syn_no_non_exhaustive)] - _ => unreachable!(), - }) + None }; + let prefix = format!("{}::", type_name); quote! { + formatter.write_str(#prefix)?; match self { #(#arms)* #nonexhaustive @@ -61,6 +106,11 @@ fn expand_impl_body(defs: &Definitions, node: &Node) -> TokenStream { } } Data::Struct(fields) => { + let type_name = if is_syntax_tree_variant { + quote!(name) + } else { + quote!(#type_name) + }; let fields = fields.keys().map(|f| { let ident = Ident::new(f, Span::call_site()); quote! { @@ -74,24 +124,40 @@ fn expand_impl_body(defs: &Definitions, node: &Node) -> TokenStream { } } Data::Private => unreachable!(), + }; + + if is_syntax_tree_variant { + quote! { + impl #ident { + fn debug(&self, formatter: &mut fmt::Formatter, name: &str) -> fmt::Result { + #body + } + } + self.debug(formatter, #type_name) + } + } else { + body } } -fn expand_impl(defs: &Definitions, node: &Node) -> TokenStream { +fn expand_impl(defs: &Definitions, node: &Node, syntax_tree_variants: &Set<&str>) -> TokenStream { let manual_debug = node.data == Data::Private || node.ident == "LitBool"; if manual_debug { return TokenStream::new(); } let ident = Ident::new(&node.ident, Span::call_site()); - let cfg_features = cfg::features(&node.features); - let body = expand_impl_body(defs, node); + let cfg_features = cfg::features(&node.features, "extra-traits"); + let body = expand_impl_body(defs, node, syntax_tree_variants); + let formatter = match &node.data { + Data::Enum(variants) if variants.is_empty() => quote!(_formatter), + _ => quote!(formatter), + }; quote! { #cfg_features - #[cfg_attr(doc_cfg, doc(cfg(feature = "extra-traits")))] impl Debug for #ident { - fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + fn fmt(&self, #formatter: &mut fmt::Formatter) -> fmt::Result { #body } } @@ -99,9 +165,21 @@ fn expand_impl(defs: &Definitions, node: &Node) -> TokenStream { } pub fn generate(defs: &Definitions) -> Result<()> { + let mut syntax_tree_variants = Set::new(); + for node in &defs.types { + if let Data::Enum(variants) = &node.data { + let enum_name = &node.ident; + for (variant_name, fields) in variants { + if let Some(inner) = syntax_tree_enum(enum_name, variant_name, fields) { + syntax_tree_variants.insert(inner); + } + } + } + } + let mut impls = TokenStream::new(); for node in &defs.types { - impls.extend(expand_impl(defs, node)); + impls.extend(expand_impl(defs, node, &syntax_tree_variants)); } file::write( diff --git a/codegen/src/eq.rs b/codegen/src/eq.rs index b9f652c0..d709a90a 100644 --- a/codegen/src/eq.rs +++ b/codegen/src/eq.rs @@ -4,11 +4,10 @@ use proc_macro2::{Ident, Span, TokenStream}; use quote::{format_ident, quote}; use syn_codegen::{Data, Definitions, Node, Type}; -const DEBUG_SRC: &str = "../src/gen/eq.rs"; +const EQ_SRC: &str = "src/gen/eq.rs"; fn always_eq(field_type: &Type) -> bool { match field_type { - Type::Syn(node) => node == "Reserved", Type::Ext(ty) => ty == "Span", Type::Token(_) | Type::Group(_) => true, Type::Box(inner) => always_eq(inner), @@ -22,6 +21,7 @@ fn expand_impl_body(defs: &Definitions, node: &Node) -> TokenStream { let ident = Ident::new(type_name, Span::call_site()); match &node.data { + Data::Enum(variants) if variants.is_empty() => quote!(match *self {}), Data::Enum(variants) => { let arms = variants.iter().map(|(variant_name, fields)| { let variant = Ident::new(variant_name, Span::call_site()); @@ -72,10 +72,15 @@ fn expand_impl_body(defs: &Definitions, node: &Node) -> TokenStream { } } }); + let fallthrough = if variants.len() == 1 { + None + } else { + Some(quote!(_ => false,)) + }; quote! { match (self, other) { #(#arms)* - _ => false, + #fallthrough } } } @@ -109,11 +114,10 @@ fn expand_impl(defs: &Definitions, node: &Node) -> TokenStream { } let ident = Ident::new(&node.ident, Span::call_site()); - let cfg_features = cfg::features(&node.features); + let cfg_features = cfg::features(&node.features, "extra-traits"); let eq = quote! { #cfg_features - #[cfg_attr(doc_cfg, doc(cfg(feature = "extra-traits")))] impl Eq for #ident {} }; @@ -123,17 +127,16 @@ fn expand_impl(defs: &Definitions, node: &Node) -> TokenStream { } let body = expand_impl_body(defs, node); - let other = if body.to_string() == "true" { - quote!(_other) - } else { - quote!(other) + let other = match &node.data { + Data::Enum(variants) if variants.is_empty() => quote!(_other), + Data::Struct(fields) if fields.values().all(always_eq) => quote!(_other), + _ => quote!(other), }; quote! { #eq #cfg_features - #[cfg_attr(doc_cfg, doc(cfg(feature = "extra-traits")))] impl PartialEq for #ident { fn eq(&self, #other: &Self) -> bool { #body @@ -149,7 +152,7 @@ pub fn generate(defs: &Definitions) -> Result<()> { } file::write( - DEBUG_SRC, + EQ_SRC, quote! { #[cfg(any(feature = "derive", feature = "full"))] use crate::tt::TokenStreamHelper; diff --git a/codegen/src/file.rs b/codegen/src/file.rs index 002e303b..beeb6601 100644 --- a/codegen/src/file.rs +++ b/codegen/src/file.rs @@ -1,10 +1,11 @@ +use crate::workspace_path; use anyhow::Result; use proc_macro2::TokenStream; use std::fs; use std::io::Write; use std::path::Path; -pub fn write>(path: P, content: TokenStream) -> Result<()> { +pub fn write(relative_to_workspace_root: impl AsRef, content: TokenStream) -> Result<()> { let mut formatted = Vec::new(); writeln!( formatted, @@ -17,7 +18,8 @@ pub fn write>(path: P, content: TokenStream) -> Result<()> { let pretty = prettyplease::unparse(&syntax_tree); write!(formatted, "{}", pretty)?; - if path.as_ref().is_file() && fs::read(&path)? == formatted { + let path = workspace_path::get(relative_to_workspace_root); + if path.is_file() && fs::read(&path)? == formatted { return Ok(()); } diff --git a/codegen/src/fold.rs b/codegen/src/fold.rs index 281c70df..13a5390a 100644 --- a/codegen/src/fold.rs +++ b/codegen/src/fold.rs @@ -5,7 +5,7 @@ use quote::{format_ident, quote}; use syn::Index; use syn_codegen::{Data, Definitions, Features, Node, Type}; -const FOLD_SRC: &str = "../src/gen/fold.rs"; +const FOLD_SRC: &str = "src/gen/fold.rs"; fn simple_visit(item: &str, name: &TokenStream) -> TokenStream { let ident = gen::under_name(item); @@ -62,29 +62,6 @@ fn visit( (#code) }) } - Type::Token(t) => { - let repr = &defs.tokens[t]; - let is_keyword = repr.chars().next().unwrap().is_alphabetic(); - let spans = if is_keyword { - quote!(span) - } else { - quote!(spans) - }; - let ty = if repr == "await" { - quote!(crate::token::Await) - } else { - syn::parse_str(&format!("Token![{}]", repr)).unwrap() - }; - Some(quote! { - #ty(tokens_helper(f, &#name.#spans)) - }) - } - Type::Group(t) => { - let ty = Ident::new(t, Span::call_site()); - Some(quote! { - #ty(tokens_helper(f, &#name.span)) - }) - } Type::Syn(t) => { fn requires_full(features: &Features) -> bool { features.any.contains("full") && features.any.len() == 1 @@ -97,7 +74,7 @@ fn visit( Some(res) } Type::Ext(t) if gen::TERMINAL_TYPES.contains(&&t[..]) => Some(simple_visit(t, name)), - Type::Ext(_) | Type::Std(_) => None, + Type::Ext(_) | Type::Std(_) | Type::Token(_) | Type::Group(_) => None, } } @@ -151,19 +128,9 @@ fn node(traits: &mut TokenStream, impls: &mut TokenStream, s: &Node, defs: &Defi } } - let nonexhaustive = if s.exhaustive { - None - } else { - Some(quote! { - #[cfg(syn_no_non_exhaustive)] - _ => unreachable!(), - }) - }; - fold_impl.extend(quote! { match node { #fold_variants - #nonexhaustive } }); } @@ -171,32 +138,17 @@ fn node(traits: &mut TokenStream, impls: &mut TokenStream, s: &Node, defs: &Defi let mut fold_fields = TokenStream::new(); for (field, ty) in fields { - let id = Ident::new(&field, Span::call_site()); + let id = Ident::new(field, Span::call_site()); let ref_toks = quote!(node.#id); - if let Type::Syn(ty) = ty { - if ty == "Reserved" { - fold_fields.extend(quote! { - #id: #ref_toks, - }); - continue; - } - } - - let fold = visit(&ty, &s.features, defs, &ref_toks).unwrap_or(ref_toks); + let fold = visit(ty, &s.features, defs, &ref_toks).unwrap_or(ref_toks); fold_fields.extend(quote! { #id: #fold, }); } - if !fields.is_empty() { - fold_impl.extend(quote! { - #ty { - #fold_fields - } - }) - } else { + if fields.is_empty() { if ty == "Ident" { fold_impl.extend(quote! { let mut node = node; @@ -207,6 +159,12 @@ fn node(traits: &mut TokenStream, impls: &mut TokenStream, s: &Node, defs: &Defi fold_impl.extend(quote! { node }); + } else { + fold_impl.extend(quote! { + #ty { + #fold_fields + } + }); } } Data::Private => { @@ -258,12 +216,14 @@ pub fn generate(defs: &Definitions) -> Result<()> { quote! { // Unreachable code is generated sometimes without the full feature. #![allow(unreachable_code, unused_variables)] - #![allow(clippy::match_wildcard_for_single_variants)] + #![allow( + clippy::match_wildcard_for_single_variants, + clippy::needless_match, + clippy::needless_pass_by_ref_mut, + )] #[cfg(any(feature = "full", feature = "derive"))] use crate::gen::helper::fold::*; - #[cfg(any(feature = "full", feature = "derive"))] - use crate::token::{Brace, Bracket, Group, Paren}; use crate::*; use proc_macro2::Span; @@ -274,8 +234,6 @@ pub fn generate(defs: &Definitions) -> Result<()> { /// See the [module documentation] for details. /// /// [module documentation]: self - /// - /// *This trait is available only if Syn is built with the `"fold"` feature.* pub trait Fold { #traits } diff --git a/codegen/src/gen.rs b/codegen/src/gen.rs index 3b2bb70f..57a9b68f 100644 --- a/codegen/src/gen.rs +++ b/codegen/src/gen.rs @@ -14,9 +14,9 @@ pub fn traverse( node: fn(&mut TokenStream, &mut TokenStream, &Node, &Definitions), ) -> (TokenStream, TokenStream) { let mut types = defs.types.clone(); - for terminal in TERMINAL_TYPES { + for &terminal in TERMINAL_TYPES { types.push(Node { - ident: terminal.to_string(), + ident: terminal.to_owned(), features: Features::default(), data: Data::Private, exhaustive: true, @@ -27,10 +27,7 @@ pub fn traverse( let mut traits = TokenStream::new(); let mut impls = TokenStream::new(); for s in types { - if s.ident == "Reserved" { - continue; - } - let features = cfg::features(&s.features); + let features = cfg::features(&s.features, None); traits.extend(features.clone()); impls.extend(features); node(&mut traits, &mut impls, &s, defs); diff --git a/codegen/src/hash.rs b/codegen/src/hash.rs index c378e9de..c0868e2f 100644 --- a/codegen/src/hash.rs +++ b/codegen/src/hash.rs @@ -4,11 +4,10 @@ use proc_macro2::{Ident, Span, TokenStream}; use quote::{format_ident, quote}; use syn_codegen::{Data, Definitions, Node, Type}; -const DEBUG_SRC: &str = "../src/gen/hash.rs"; +const HASH_SRC: &str = "src/gen/hash.rs"; fn skip(field_type: &Type) -> bool { match field_type { - Type::Syn(node) => node == "Reserved", Type::Ext(ty) => ty == "Span", Type::Token(_) | Type::Group(_) => true, Type::Box(inner) => skip(inner), @@ -22,12 +21,13 @@ fn expand_impl_body(defs: &Definitions, node: &Node) -> TokenStream { let ident = Ident::new(type_name, Span::call_site()); match &node.data { + Data::Enum(variants) if variants.is_empty() => quote!(match *self {}), Data::Enum(variants) => { let arms = variants .iter() .enumerate() .map(|(i, (variant_name, fields))| { - let i = i as u8; + let i = u8::try_from(i).unwrap(); let variant = Ident::new(variant_name, Span::call_site()); if fields.is_empty() { quote! { @@ -76,18 +76,13 @@ fn expand_impl_body(defs: &Definitions, node: &Node) -> TokenStream { } } }); - let nonexhaustive = if node.exhaustive { - None - } else if node.ident == "Expr" { + let nonexhaustive = if node.ident == "Expr" { Some(quote! { - #[cfg(any(syn_no_non_exhaustive, not(feature = "full")))] + #[cfg(not(feature = "full"))] _ => unreachable!(), }) } else { - Some(quote! { - #[cfg(syn_no_non_exhaustive)] - _ => unreachable!(), - }) + None }; quote! { match self { @@ -128,20 +123,20 @@ fn expand_impl(defs: &Definitions, node: &Node) -> TokenStream { } let ident = Ident::new(&node.ident, Span::call_site()); - let cfg_features = cfg::features(&node.features); + let cfg_features = cfg::features(&node.features, "extra-traits"); let body = expand_impl_body(defs, node); - let state = if body.is_empty() { - quote!(_state) - } else { - quote!(state) + + let hasher = match &node.data { + Data::Struct(_) if body.is_empty() => quote!(_state), + Data::Enum(variants) if variants.is_empty() => quote!(_state), + _ => quote!(state), }; quote! { #cfg_features - #[cfg_attr(doc_cfg, doc(cfg(feature = "extra-traits")))] impl Hash for #ident { - fn hash(&self, #state: &mut H) + fn hash(&self, #hasher: &mut H) where H: Hasher, { @@ -158,7 +153,7 @@ pub fn generate(defs: &Definitions) -> Result<()> { } file::write( - DEBUG_SRC, + HASH_SRC, quote! { #[cfg(any(feature = "derive", feature = "full"))] use crate::tt::TokenStreamHelper; diff --git a/codegen/src/json.rs b/codegen/src/json.rs index 53904bfb..28ceee8b 100644 --- a/codegen/src/json.rs +++ b/codegen/src/json.rs @@ -1,6 +1,6 @@ +use crate::workspace_path; use anyhow::Result; use std::fs; -use std::path::Path; use syn_codegen::Definitions; pub fn generate(defs: &Definitions) -> Result<()> { @@ -10,8 +10,7 @@ pub fn generate(defs: &Definitions) -> Result<()> { let check: Definitions = serde_json::from_str(&j)?; assert_eq!(*defs, check); - let codegen_root = Path::new(env!("CARGO_MANIFEST_DIR")); - let json_path = codegen_root.join("../syn.json"); + let json_path = workspace_path::get("syn.json"); fs::write(json_path, j)?; Ok(()) diff --git a/codegen/src/main.rs b/codegen/src/main.rs index 022494ef..8b8670c5 100644 --- a/codegen/src/main.rs +++ b/codegen/src/main.rs @@ -9,7 +9,14 @@ // Finally this crate generates the Visit, VisitMut, and Fold traits in Syn // programmatically from the syntax tree description. -#![allow(clippy::needless_pass_by_value)] +#![allow( + clippy::items_after_statements, + clippy::manual_let_else, + clippy::match_like_matches_macro, + clippy::similar_names, + clippy::too_many_lines, + clippy::uninlined_format_args +)] mod cfg; mod clone; @@ -28,6 +35,7 @@ mod snapshot; mod version; mod visit; mod visit_mut; +mod workspace_path; fn main() -> anyhow::Result<()> { color_backtrace::install(); diff --git a/codegen/src/parse.rs b/codegen/src/parse.rs index 716ba0dc..f232fedf 100644 --- a/codegen/src/parse.rs +++ b/codegen/src/parse.rs @@ -1,4 +1,4 @@ -use crate::version; +use crate::{version, workspace_path}; use anyhow::{bail, Result}; use indexmap::IndexMap; use quote::quote; @@ -8,35 +8,46 @@ use std::path::{Path, PathBuf}; use syn::parse::{Error, Parser}; use syn::{ parse_quote, Attribute, Data, DataEnum, DataStruct, DeriveInput, Fields, GenericArgument, - Ident, Item, PathArguments, TypeMacro, TypePath, TypeTuple, Visibility, + Ident, Item, PathArguments, TypeMacro, TypePath, TypeTuple, UseTree, Visibility, }; use syn_codegen as types; use thiserror::Error; -const SYN_CRATE_ROOT: &str = "../src/lib.rs"; -const TOKEN_SRC: &str = "../src/token.rs"; +const SYN_CRATE_ROOT: &str = "src/lib.rs"; +const TOKEN_SRC: &str = "src/token.rs"; const IGNORED_MODS: &[&str] = &["fold", "visit", "visit_mut"]; const EXTRA_TYPES: &[&str] = &["Lifetime"]; -// NOTE: BTreeMap is used here instead of HashMap to have deterministic output. -type ItemLookup = BTreeMap; -type TokenLookup = BTreeMap; +struct Lookup { + items: BTreeMap, + // "+" => "Add" + tokens: BTreeMap, + // "PatLit" => "ExprLit" + aliases: BTreeMap, +} /// Parse the contents of `src` and return a list of AST types. pub fn parse() -> Result { - let mut item_lookup = BTreeMap::new(); - load_file(SYN_CRATE_ROOT, &[], &mut item_lookup)?; + let tokens = load_token_file(TOKEN_SRC)?; - let token_lookup = load_token_file(TOKEN_SRC)?; + let mut lookup = Lookup { + items: BTreeMap::new(), + tokens, + aliases: BTreeMap::new(), + }; + + load_file(SYN_CRATE_ROOT, &[], &mut lookup)?; let version = version::get()?; - let types = item_lookup + let types = lookup + .items .values() - .map(|item| introspect_item(item, &item_lookup, &token_lookup)) + .map(|item| introspect_item(item, &lookup)) .collect(); - let tokens = token_lookup + let tokens = lookup + .tokens .into_iter() .map(|(name, ty)| (ty, name)) .collect(); @@ -54,22 +65,23 @@ pub struct AstItem { features: Vec, } -fn introspect_item(item: &AstItem, items: &ItemLookup, tokens: &TokenLookup) -> types::Node { +fn introspect_item(item: &AstItem, lookup: &Lookup) -> types::Node { let features = introspect_features(&item.features); match &item.ast.data { - Data::Enum(ref data) => types::Node { + Data::Enum(data) => types::Node { ident: item.ast.ident.to_string(), features, - data: types::Data::Enum(introspect_enum(data, items, tokens)), - exhaustive: !data.variants.iter().any(|v| is_doc_hidden(&v.attrs)), + data: types::Data::Enum(introspect_enum(data, lookup)), + exhaustive: !(is_non_exhaustive(&item.ast.attrs) + || data.variants.iter().any(|v| is_doc_hidden(&v.attrs))), }, - Data::Struct(ref data) => types::Node { + Data::Struct(data) => types::Node { ident: item.ast.ident.to_string(), features, data: { if data.fields.iter().all(|f| is_pub(&f.vis)) { - types::Data::Struct(introspect_struct(data, items, tokens)) + types::Data::Struct(introspect_struct(data, lookup)) } else { types::Data::Private } @@ -80,7 +92,7 @@ fn introspect_item(item: &AstItem, items: &ItemLookup, tokens: &TokenLookup) -> } } -fn introspect_enum(item: &DataEnum, items: &ItemLookup, tokens: &TokenLookup) -> types::Variants { +fn introspect_enum(item: &DataEnum, lookup: &Lookup) -> types::Variants { item.variants .iter() .filter_map(|variant| { @@ -91,17 +103,17 @@ fn introspect_enum(item: &DataEnum, items: &ItemLookup, tokens: &TokenLookup) -> Fields::Unnamed(fields) => fields .unnamed .iter() - .map(|field| introspect_type(&field.ty, items, tokens)) + .map(|field| introspect_type(&field.ty, lookup)) .collect(), Fields::Unit => vec![], - _ => panic!("Enum representation not supported"), + Fields::Named(_) => panic!("Enum representation not supported"), }; Some((variant.ident.to_string(), fields)) }) .collect() } -fn introspect_struct(item: &DataStruct, items: &ItemLookup, tokens: &TokenLookup) -> types::Fields { +fn introspect_struct(item: &DataStruct, lookup: &Lookup) -> types::Fields { match &item.fields { Fields::Named(fields) => fields .named @@ -109,74 +121,71 @@ fn introspect_struct(item: &DataStruct, items: &ItemLookup, tokens: &TokenLookup .map(|field| { ( field.ident.as_ref().unwrap().to_string(), - introspect_type(&field.ty, items, tokens), + introspect_type(&field.ty, lookup), ) }) .collect(), Fields::Unit => IndexMap::new(), - _ => panic!("Struct representation not supported"), + Fields::Unnamed(_) => panic!("Struct representation not supported"), } } -fn introspect_type(item: &syn::Type, items: &ItemLookup, tokens: &TokenLookup) -> types::Type { +fn introspect_type(item: &syn::Type, lookup: &Lookup) -> types::Type { match item { - syn::Type::Path(TypePath { - qself: None, - ref path, - }) => { + syn::Type::Path(TypePath { qself: None, path }) => { let last = path.segments.last().unwrap(); let string = last.ident.to_string(); match string.as_str() { "Option" => { - let nested = introspect_type(first_arg(&last.arguments), items, tokens); + let nested = introspect_type(first_arg(&last.arguments), lookup); types::Type::Option(Box::new(nested)) } "Punctuated" => { - let nested = introspect_type(first_arg(&last.arguments), items, tokens); - let punct = match introspect_type(last_arg(&last.arguments), items, tokens) { - types::Type::Token(s) => s, - _ => panic!(), + let nested = introspect_type(first_arg(&last.arguments), lookup); + let types::Type::Token(punct) = + introspect_type(last_arg(&last.arguments), lookup) + else { + panic!() }; - types::Type::Punctuated(types::Punctuated { element: Box::new(nested), punct, }) } "Vec" => { - let nested = introspect_type(first_arg(&last.arguments), items, tokens); + let nested = introspect_type(first_arg(&last.arguments), lookup); types::Type::Vec(Box::new(nested)) } "Box" => { - let nested = introspect_type(first_arg(&last.arguments), items, tokens); + let nested = introspect_type(first_arg(&last.arguments), lookup); types::Type::Box(Box::new(nested)) } "Brace" | "Bracket" | "Paren" | "Group" => types::Type::Group(string), "TokenStream" | "Literal" | "Ident" | "Span" => types::Type::Ext(string), "String" | "u32" | "usize" | "bool" => types::Type::Std(string), - "Await" => types::Type::Token("Await".to_string()), _ => { - if items.get(&last.ident).is_some() || last.ident == "Reserved" { - types::Type::Syn(string) + let mut resolved = &last.ident; + while let Some(alias) = lookup.aliases.get(resolved) { + resolved = alias; + } + if lookup.items.get(resolved).is_some() { + types::Type::Syn(resolved.to_string()) } else { - unimplemented!("{}", string); + unimplemented!("{}", resolved); } } } } - syn::Type::Tuple(TypeTuple { ref elems, .. }) => { - let tys = elems - .iter() - .map(|ty| introspect_type(&ty, items, tokens)) - .collect(); + syn::Type::Tuple(TypeTuple { elems, .. }) => { + let tys = elems.iter().map(|ty| introspect_type(ty, lookup)).collect(); types::Type::Tuple(tys) } - syn::Type::Macro(TypeMacro { ref mac }) + syn::Type::Macro(TypeMacro { mac }) if mac.path.segments.last().unwrap().ident == "Token" => { let content = mac.tokens.to_string(); - let ty = tokens.get(&content).unwrap().to_string(); + let ty = lookup.tokens.get(&content).unwrap().to_string(); types::Type::Token(ty) } @@ -188,11 +197,11 @@ fn introspect_features(attrs: &[Attribute]) -> types::Features { let mut ret = types::Features::default(); for attr in attrs { - if !attr.path.is_ident("cfg") { + if !attr.path().is_ident("cfg") { continue; } - let features = parsing::parse_features.parse2(attr.tokens.clone()).unwrap(); + let features = attr.parse_args_with(parsing::parse_features).unwrap(); if ret.any.is_empty() { ret = features; @@ -214,60 +223,65 @@ fn is_pub(vis: &Visibility) -> bool { } } +fn is_non_exhaustive(attrs: &[Attribute]) -> bool { + for attr in attrs { + if attr.path().is_ident("non_exhaustive") { + return true; + } + } + false +} + fn is_doc_hidden(attrs: &[Attribute]) -> bool { for attr in attrs { - if attr.path.is_ident("doc") { - if parsing::parse_doc_hidden_attr - .parse2(attr.tokens.clone()) - .is_ok() - { - return true; - } + if attr.path().is_ident("doc") && attr.parse_args::().is_ok() { + return true; } } false } fn first_arg(params: &PathArguments) -> &syn::Type { - let data = match *params { - PathArguments::AngleBracketed(ref data) => data, + let data = match params { + PathArguments::AngleBracketed(data) => data, _ => panic!("Expected at least 1 type argument here"), }; - match *data + match data .args .first() .expect("Expected at least 1 type argument here") { - GenericArgument::Type(ref ty) => ty, + GenericArgument::Type(ty) => ty, _ => panic!("Expected at least 1 type argument here"), } } fn last_arg(params: &PathArguments) -> &syn::Type { - let data = match *params { - PathArguments::AngleBracketed(ref data) => data, + let data = match params { + PathArguments::AngleBracketed(data) => data, _ => panic!("Expected at least 1 type argument here"), }; - match *data + match data .args .last() .expect("Expected at least 1 type argument here") { - GenericArgument::Type(ref ty) => ty, + GenericArgument::Type(ty) => ty, _ => panic!("Expected at least 1 type argument here"), } } mod parsing { - use super::{AstItem, TokenLookup}; - use proc_macro2::{TokenStream, TokenTree}; + use super::AstItem; + use proc_macro2::TokenStream; use quote::quote; use std::collections::{BTreeMap, BTreeSet}; - use syn::parse::{ParseStream, Parser, Result}; + use syn::parse::{ParseStream, Result}; use syn::{ - braced, bracketed, parenthesized, parse_quote, token, Attribute, Ident, LitStr, Path, Token, + braced, bracketed, parenthesized, parse_quote, token, Attribute, Expr, Ident, Lit, LitStr, + Path, Token, }; use syn_codegen as types; @@ -313,32 +327,18 @@ mod parsing { Ok(res) } - fn no_visit(input: ParseStream) -> bool { - if peek_tag(input, "no_visit") { - input.parse::().unwrap(); - input.parse::().unwrap(); - true - } else { - false - } - } - - pub fn ast_enum(input: ParseStream) -> Result> { - input.call(Attribute::parse_outer)?; + pub fn ast_enum(input: ParseStream) -> Result { + let attrs = input.call(Attribute::parse_outer)?; input.parse::()?; input.parse::()?; let ident: Ident = input.parse()?; - let no_visit = no_visit(input); let rest: TokenStream = input.parse()?; - Ok(if no_visit { - None - } else { - Some(AstItem { - ast: syn::parse2(quote! { - pub enum #ident #rest - })?, - features: vec![], - }) + Ok(AstItem { + ast: syn::parse2(quote! { + #(#attrs)* + pub enum #ident #rest + })?, + features: vec![], }) } @@ -368,7 +368,7 @@ mod parsing { } pub fn ast_enum_of_structs(input: ParseStream) -> Result { - input.call(Attribute::parse_outer)?; + let attrs = input.call(Attribute::parse_outer)?; input.parse::()?; input.parse::()?; let ident: Ident = input.parse()?; @@ -380,20 +380,18 @@ mod parsing { variants.push(content.call(eos_variant)?); } - if let Some(ident) = input.parse::>()? { - assert_eq!(ident, "do_not_generate_to_tokens"); - } - let enum_item = { let variants = variants.iter().map(|v| { let attrs = &v.attrs; let name = &v.name; - match v.member { - Some(ref member) => quote!(#(#attrs)* #name(#member)), - None => quote!(#(#attrs)* #name), + if let Some(member) = &v.member { + quote!(#(#attrs)* #name(#member)) + } else { + quote!(#(#attrs)* #name) } }); parse_quote! { + #(#attrs)* pub enum #ident { #(#variants),* } @@ -405,47 +403,26 @@ mod parsing { }) } - mod kw { + pub mod kw { syn::custom_keyword!(hidden); syn::custom_keyword!(macro_rules); syn::custom_keyword!(Token); } - pub fn parse_token_macro(input: ParseStream) -> Result { - input.parse::()?; - input.parse::]>()?; - - let definition; - braced!(definition in input); - definition.call(Attribute::parse_outer)?; - definition.parse::()?; - definition.parse::()?; - definition.parse::()?; - - let rules; - braced!(rules in definition); - input.parse::()?; - + pub fn parse_token_macro(input: ParseStream) -> Result> { let mut tokens = BTreeMap::new(); - while !rules.is_empty() { - if rules.peek(Token![$]) { - rules.parse::()?; - rules.parse::()?; - rules.parse::()?; - tokens.insert("await".to_owned(), "Await".to_owned()); - } else { - let pattern; - bracketed!(pattern in rules); - let token = pattern.parse::()?.to_string(); - rules.parse::]>()?; - let expansion; - braced!(expansion in rules); - rules.parse::()?; - expansion.parse::()?; - let path: Path = expansion.parse()?; - let ty = path.segments.last().unwrap().ident.to_string(); - tokens.insert(token, ty.to_string()); - } + while !input.is_empty() { + let pattern; + bracketed!(pattern in input); + let token = pattern.parse::()?.to_string(); + input.parse::]>()?; + let expansion; + braced!(expansion in input); + input.parse::()?; + expansion.parse::()?; + let path: Path = expansion.parse()?; + let ty = path.segments.last().unwrap().ident.to_string(); + tokens.insert(token, ty.to_string()); } Ok(tokens) } @@ -463,56 +440,43 @@ mod parsing { pub fn parse_features(input: ParseStream) -> Result { let mut features = BTreeSet::new(); - let level_1; - parenthesized!(level_1 in input); - - let i: Ident = level_1.fork().parse()?; + let i: Ident = input.fork().parse()?; if i == "any" { - level_1.parse::()?; + input.parse::()?; - let level_2; - parenthesized!(level_2 in level_1); + let nested; + parenthesized!(nested in input); - while !level_2.is_empty() { - features.insert(parse_feature(&level_2)?); + while !nested.is_empty() { + features.insert(parse_feature(&nested)?); - if !level_2.is_empty() { - level_2.parse::()?; + if !nested.is_empty() { + nested.parse::()?; } } } else if i == "feature" { - features.insert(parse_feature(&level_1)?); - assert!(level_1.is_empty()); + features.insert(parse_feature(input)?); + assert!(input.is_empty()); } else { panic!("{:?}", i); } - assert!(input.is_empty()); - Ok(types::Features { any: features }) } - pub fn path_attr(attrs: &[Attribute]) -> Result> { + pub fn path_attr(attrs: &[Attribute]) -> Result> { for attr in attrs { - if attr.path.is_ident("path") { - fn parser(input: ParseStream) -> Result { - input.parse::()?; - input.parse() + if attr.path().is_ident("path") { + if let Expr::Lit(expr) = &attr.meta.require_name_value()?.value { + if let Lit::Str(lit) = &expr.lit { + return Ok(Some(lit)); + } } - let filename = parser.parse2(attr.tokens.clone())?; - return Ok(Some(filename)); } } Ok(None) } - - pub fn parse_doc_hidden_attr(input: ParseStream) -> Result<()> { - let content; - parenthesized!(content in input); - content.parse::()?; - Ok(()) - } } fn clone_features(features: &[Attribute]) -> Vec { @@ -523,7 +487,7 @@ fn get_features(attrs: &[Attribute], base: &[Attribute]) -> Vec { let mut ret = clone_features(base); for attr in attrs { - if attr.path.is_ident("cfg") { + if attr.path().is_ident("cfg") { ret.push(parse_quote!(#attr)); } } @@ -540,12 +504,12 @@ struct LoadFileError { error: Error, } -fn load_file>( - name: P, +fn load_file( + relative_to_workspace_root: impl AsRef, features: &[Attribute], - lookup: &mut ItemLookup, + lookup: &mut Lookup, ) -> Result<()> { - let error = match do_load_file(&name, features, lookup).err() { + let error = match do_load_file(&relative_to_workspace_root, features, lookup).err() { None => return Ok(()), Some(error) => error, }; @@ -554,23 +518,23 @@ fn load_file>( let span = error.span().start(); bail!(LoadFileError { - path: name.as_ref().to_owned(), + path: relative_to_workspace_root.as_ref().to_owned(), line: span.line, column: span.column + 1, error, }) } -fn do_load_file>( - name: P, +fn do_load_file( + relative_to_workspace_root: impl AsRef, features: &[Attribute], - lookup: &mut ItemLookup, + lookup: &mut Lookup, ) -> Result<()> { - let name = name.as_ref(); - let parent = name.parent().expect("no parent path"); + let relative_to_workspace_root = relative_to_workspace_root.as_ref(); + let parent = relative_to_workspace_root.parent().expect("no parent path"); // Parse the file - let src = fs::read_to_string(name)?; + let src = fs::read_to_string(workspace_path::get(relative_to_workspace_root))?; let file = syn::parse_file(&src)?; // Collect all of the interesting AstItems declared in this file or submodules. @@ -603,9 +567,10 @@ fn do_load_file>( // Look up the submodule file, and recursively parse it. // Only handles same-directory .rs file submodules for now. - let filename = match parsing::path_attr(&item.attrs)? { - Some(filename) => filename.value(), - None => format!("{}.rs", item.ident), + let filename = if let Some(filename) = parsing::path_attr(&item.attrs)? { + filename.value() + } else { + format!("{}.rs", item.ident) }; let path = parent.join(filename); load_file(path, &features, lookup)?; @@ -617,28 +582,24 @@ fn do_load_file>( // Try to parse the AstItem declaration out of the item. let tts = item.mac.tokens.clone(); - let found = if item.mac.path.is_ident("ast_struct") { - Some(parsing::ast_struct.parse2(tts)?) + let mut found = if item.mac.path.is_ident("ast_struct") { + parsing::ast_struct.parse2(tts) } else if item.mac.path.is_ident("ast_enum") { - parsing::ast_enum.parse2(tts)? + parsing::ast_enum.parse2(tts) } else if item.mac.path.is_ident("ast_enum_of_structs") { - Some(parsing::ast_enum_of_structs.parse2(tts)?) + parsing::ast_enum_of_structs.parse2(tts) } else { continue; - }; + }?; // Record our features on the parsed AstItems. - if let Some(mut item) = found { - if item.ast.ident != "Reserved" { - item.features.extend(clone_features(&features)); - lookup.insert(item.ast.ident.clone(), item); - } - } + found.features.extend(clone_features(&features)); + lookup.items.insert(found.ast.ident.clone(), found); } Item::Struct(item) => { let ident = item.ident; if EXTRA_TYPES.contains(&&ident.to_string()[..]) { - lookup.insert( + lookup.items.insert( ident.clone(), AstItem { ast: DeriveInput { @@ -657,20 +618,43 @@ fn do_load_file>( ); } } + Item::Use(item) + if relative_to_workspace_root == Path::new(SYN_CRATE_ROOT) + && matches!(item.vis, Visibility::Public(_)) => + { + load_aliases(item.tree, lookup); + } _ => {} } } Ok(()) } -fn load_token_file>(name: P) -> Result { - let name = name.as_ref(); - let src = fs::read_to_string(name)?; +fn load_aliases(use_tree: UseTree, lookup: &mut Lookup) { + match use_tree { + UseTree::Path(use_tree) => load_aliases(*use_tree.tree, lookup), + UseTree::Rename(use_tree) => { + lookup.aliases.insert(use_tree.rename, use_tree.ident); + } + UseTree::Group(use_tree) => { + for use_tree in use_tree.items { + load_aliases(use_tree, lookup); + } + } + UseTree::Name(_) | UseTree::Glob(_) => {} + } +} + +fn load_token_file( + relative_to_workspace_root: impl AsRef, +) -> Result> { + let path = workspace_path::get(relative_to_workspace_root); + let src = fs::read_to_string(path)?; let file = syn::parse_file(&src)?; for item in file.items { if let Item::Macro(item) = item { match item.ident { - Some(ref i) if i == "export_token_macro" => {} + Some(i) if i == "Token" => {} _ => continue, } let tokens = item.mac.parse_body_with(parsing::parse_token_macro)?; diff --git a/codegen/src/snapshot.rs b/codegen/src/snapshot.rs index 46c6a8f1..17588dbc 100644 --- a/codegen/src/snapshot.rs +++ b/codegen/src/snapshot.rs @@ -1,3 +1,4 @@ +use crate::operand::{Borrowed, Operand, Owned}; use crate::{file, lookup}; use anyhow::Result; use proc_macro2::{Ident, Span, TokenStream}; @@ -5,7 +6,7 @@ use quote::{format_ident, quote}; use syn::Index; use syn_codegen::{Data, Definitions, Node, Type}; -const DEBUG_SRC: &str = "../tests/debug/gen.rs"; +const TESTS_DEBUG_SRC: &str = "tests/debug/gen.rs"; fn rust_type(ty: &Type) -> TokenStream { match ty { @@ -55,44 +56,44 @@ fn is_printable(ty: &Type) -> bool { Type::Box(ty) => is_printable(ty), Type::Tuple(ty) => ty.iter().any(is_printable), Type::Token(_) | Type::Group(_) => false, - Type::Syn(name) => name != "Reserved", - Type::Std(_) | Type::Punctuated(_) | Type::Option(_) | Type::Vec(_) => true, + Type::Syn(_) | Type::Std(_) | Type::Punctuated(_) | Type::Option(_) | Type::Vec(_) => true, } } -fn format_field(val: &TokenStream, ty: &Type) -> Option { +fn format_field(val: &Operand, ty: &Type) -> Option { if !is_printable(ty) { return None; } let format = match ty { Type::Option(ty) => { - let inner = quote!(_val); - let format = format_field(&inner, ty).map(|format| { - quote! { - formatter.write_str("(")?; - Debug::fmt(#format, formatter)?; - formatter.write_str(")")?; - } - }); - let ty = rust_type(ty); - quote!({ - #[derive(RefCast)] - #[repr(transparent)] - struct Print(Option<#ty>); - impl Debug for Print { - fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { - match &self.0 { - Some(#inner) => { - formatter.write_str("Some")?; - #format - Ok(()) + if let Some(format) = format_field(&Borrowed(quote!(_val)), ty) { + let ty = rust_type(ty); + let val = val.ref_tokens(); + quote!({ + #[derive(RefCast)] + #[repr(transparent)] + struct Print(Option<#ty>); + impl Debug for Print { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + match &self.0 { + Some(_val) => { + formatter.write_str("Some(")?; + Debug::fmt(#format, formatter)?; + formatter.write_str(")")?; + Ok(()) + } + None => formatter.write_str("None"), } - None => formatter.write_str("None"), } } + Print::ref_cast(#val) + }) + } else { + let val = val.tokens(); + quote! { + &super::Option { present: #val.is_some() } } - Print::ref_cast(#val) - }) + } } Type::Tuple(ty) => { let printable: Vec = ty @@ -100,8 +101,9 @@ fn format_field(val: &TokenStream, ty: &Type) -> Option { .enumerate() .filter_map(|(i, ty)| { let index = Index::from(i); - let val = quote!(&#val.#index); - format_field(&val, ty) + let val = val.tokens(); + let inner = Owned(quote!(#val.#index)); + format_field(&inner, ty) }) .collect(); if printable.len() == 1 { @@ -112,7 +114,10 @@ fn format_field(val: &TokenStream, ty: &Type) -> Option { } } } - _ => quote! { Lite(#val) }, + _ => { + let val = val.ref_tokens(); + quote! { Lite(#val) } + } }; Some(format) } @@ -121,27 +126,43 @@ fn syntax_tree_enum<'a>(outer: &str, inner: &str, fields: &'a [Type]) -> Option< if fields.len() != 1 { return None; } - const WHITELIST: &[&str] = &["PathArguments", "Visibility"]; + const WHITELIST: &[(&str, &str)] = &[ + ("Meta", "Path"), + ("PathArguments", "AngleBracketed"), + ("PathArguments", "Parenthesized"), + ("Stmt", "Local"), + ("TypeParamBound", "Lifetime"), + ("Visibility", "Public"), + ("Visibility", "Restricted"), + ]; match &fields[0] { - Type::Syn(ty) if WHITELIST.contains(&outer) || outer.to_owned() + inner == *ty => Some(ty), + Type::Syn(ty) if WHITELIST.contains(&(outer, inner)) || outer.to_owned() + inner == *ty => { + Some(ty) + } _ => None, } } -fn expand_impl_body(defs: &Definitions, node: &Node, name: &str) -> TokenStream { +fn expand_impl_body(defs: &Definitions, node: &Node, name: &str, val: &Operand) -> TokenStream { let ident = Ident::new(&node.ident, Span::call_site()); match &node.data { + Data::Enum(variants) if variants.is_empty() => quote!(unreachable!()), Data::Enum(variants) => { let arms = variants.iter().map(|(v, fields)| { + let path = format!("{}::{}", name, v); let variant = Ident::new(v, Span::call_site()); if fields.is_empty() { quote! { - syn::#ident::#variant => formatter.write_str(#v), + syn::#ident::#variant => formatter.write_str(#path), } } else if let Some(inner) = syntax_tree_enum(name, v, fields) { - let path = format!("{}::{}", name, v); - let format = expand_impl_body(defs, lookup::node(defs, inner), &path); + let format = expand_impl_body( + defs, + lookup::node(defs, inner), + &path, + &Borrowed(quote!(_val)), + ); quote! { syn::#ident::#variant(_val) => { #format @@ -157,7 +178,7 @@ fn expand_impl_body(defs: &Definitions, node: &Node, name: &str) -> TokenStream }) } else { let ty = &fields[0]; - format_field(&val, ty).map(|format| { + format_field(&Borrowed(val), ty).map(|format| { quote! { formatter.write_str("(")?; Debug::fmt(#format, formatter)?; @@ -167,7 +188,7 @@ fn expand_impl_body(defs: &Definitions, node: &Node, name: &str) -> TokenStream }; quote! { syn::#ident::#variant(_val) => { - formatter.write_str(#v)?; + formatter.write_str(#path)?; #format Ok(()) } @@ -177,14 +198,14 @@ fn expand_impl_body(defs: &Definitions, node: &Node, name: &str) -> TokenStream let fields = fields.iter().enumerate().filter_map(|(i, ty)| { let index = format_ident!("_v{}", i); let val = quote!(#index); - let format = format_field(&val, ty)?; + let format = format_field(&Borrowed(val), ty)?; Some(quote! { formatter.field(#format); }) }); quote! { syn::#ident::#variant(#(#pats),*) => { - let mut formatter = formatter.debug_tuple(#v); + let mut formatter = formatter.debug_tuple(#path); #(#fields)* formatter.finish() } @@ -196,8 +217,9 @@ fn expand_impl_body(defs: &Definitions, node: &Node, name: &str) -> TokenStream } else { Some(quote!(_ => unreachable!())) }; + let val = val.ref_tokens(); quote! { - match _val { + match #val { #(#arms)* #nonexhaustive } @@ -207,43 +229,65 @@ fn expand_impl_body(defs: &Definitions, node: &Node, name: &str) -> TokenStream let fields = fields.iter().filter_map(|(f, ty)| { let ident = Ident::new(f, Span::call_site()); if let Type::Option(ty) = ty { - let inner = quote!(_val); - let format = format_field(&inner, ty).map(|format| { + Some(if let Some(format) = format_field(&Owned(quote!(self.0)), ty) { + let val = val.tokens(); + let ty = rust_type(ty); quote! { - let #inner = &self.0; - formatter.write_str("(")?; - Debug::fmt(#format, formatter)?; - formatter.write_str(")")?; - } - }); - let ty = rust_type(ty); - Some(quote! { - if let Some(val) = &_val.#ident { - #[derive(RefCast)] - #[repr(transparent)] - struct Print(#ty); - impl Debug for Print { - fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { - formatter.write_str("Some")?; - #format - Ok(()) + if let Some(val) = &#val.#ident { + #[derive(RefCast)] + #[repr(transparent)] + struct Print(#ty); + impl Debug for Print { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str("Some(")?; + Debug::fmt(#format, formatter)?; + formatter.write_str(")")?; + Ok(()) + } } + formatter.field(#f, Print::ref_cast(val)); + } + } + } else { + let val = val.tokens(); + quote! { + if #val.#ident.is_some() { + formatter.field(#f, &Present); } - formatter.field(#f, Print::ref_cast(val)); } }) } else { - let val = quote!(&_val.#ident); - let format = format_field(&val, ty)?; + let val = val.tokens(); + let inner = Owned(quote!(#val.#ident)); + let format = format_field(&inner, ty)?; let mut call = quote! { formatter.field(#f, #format); }; - if let Type::Vec(_) | Type::Punctuated(_) = ty { + if node.ident == "Block" && f == "stmts" { + // Format regardless of whether is_empty(). + } else if let Type::Vec(_) | Type::Punctuated(_) = ty { call = quote! { - if !_val.#ident.is_empty() { + if !#val.#ident.is_empty() { #call } }; + } else if let Type::Syn(inner) = ty { + for node in &defs.types { + if node.ident == *inner { + if let Data::Enum(variants) = &node.data { + if variants.get("None").map_or(false, Vec::is_empty) { + let ty = rust_type(ty); + call = quote! { + match #val.#ident { + #ty::None => {} + _ => { #call } + } + }; + } + } + break; + } + } } Some(call) } @@ -256,12 +300,14 @@ fn expand_impl_body(defs: &Definitions, node: &Node, name: &str) -> TokenStream } Data::Private => { if node.ident == "LitInt" || node.ident == "LitFloat" { + let val = val.ref_tokens(); quote! { - write!(formatter, "{}", _val) + write!(formatter, "{}", #val) } } else { + let val = val.tokens(); quote! { - write!(formatter, "{:?}", _val.value()) + write!(formatter, "{:?}", #val.value()) } } } @@ -269,33 +315,54 @@ fn expand_impl_body(defs: &Definitions, node: &Node, name: &str) -> TokenStream } fn expand_impl(defs: &Definitions, node: &Node) -> TokenStream { - if node.ident == "Reserved" { - return TokenStream::new(); - } - let ident = Ident::new(&node.ident, Span::call_site()); - let body = expand_impl_body(defs, node, &node.ident); + let body = expand_impl_body(defs, node, &node.ident, &Owned(quote!(self.value))); + let formatter = match &node.data { + Data::Enum(variants) if variants.is_empty() => quote!(_formatter), + _ => quote!(formatter), + }; quote! { impl Debug for Lite { - fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { - let _val = &self.value; + fn fmt(&self, #formatter: &mut fmt::Formatter) -> fmt::Result { #body } } } } +fn expand_token_impl(name: &str, symbol: &str) -> TokenStream { + let ident = Ident::new(name, Span::call_site()); + let repr = format!("Token![{}]", symbol); + + quote! { + impl Debug for Lite { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str(#repr) + } + } + } +} + pub fn generate(defs: &Definitions) -> Result<()> { let mut impls = TokenStream::new(); for node in &defs.types { - impls.extend(expand_impl(&defs, node)); + impls.extend(expand_impl(defs, node)); + } + for (name, symbol) in &defs.tokens { + impls.extend(expand_token_impl(name, symbol)); } file::write( - DEBUG_SRC, + TESTS_DEBUG_SRC, quote! { - use super::{Lite, RefCast}; + // False positive: https://github.com/rust-lang/rust/issues/78586#issuecomment-1722680482 + #![allow(repr_transparent_external_private_fields)] + + #![allow(clippy::match_wildcard_for_single_variants)] + + use super::{Lite, Present}; + use ref_cast::RefCast; use std::fmt::{self, Debug, Display}; #impls diff --git a/codegen/src/version.rs b/codegen/src/version.rs index 3a1f967c..db6a8f8c 100644 --- a/codegen/src/version.rs +++ b/codegen/src/version.rs @@ -1,12 +1,11 @@ +use crate::workspace_path; use anyhow::Result; use semver::Version; -use serde::Deserialize; +use serde_derive::Deserialize; use std::fs; -use std::path::Path; pub fn get() -> Result { - let codegen_root = Path::new(env!("CARGO_MANIFEST_DIR")); - let syn_cargo_toml = codegen_root.join("../Cargo.toml"); + let syn_cargo_toml = workspace_path::get("Cargo.toml"); let manifest = fs::read_to_string(syn_cargo_toml)?; let parsed: Manifest = toml::from_str(&manifest)?; Ok(parsed.package.version) diff --git a/codegen/src/visit.rs b/codegen/src/visit.rs index 9576d098..573798a1 100644 --- a/codegen/src/visit.rs +++ b/codegen/src/visit.rs @@ -6,7 +6,7 @@ use quote::{format_ident, quote}; use syn::Index; use syn_codegen::{Data, Definitions, Features, Node, Type}; -const VISIT_SRC: &str = "../src/gen/visit.rs"; +const VISIT_SRC: &str = "src/gen/visit.rs"; fn simple_visit(item: &str, name: &Operand) -> TokenStream { let ident = gen::under_name(item); @@ -51,20 +51,17 @@ fn visit( let name = name.ref_tokens(); Some(quote! { for el in Punctuated::pairs(#name) { - let (it, p) = el.into_tuple(); + let it = el.value(); #val; - if let Some(p) = p { - tokens_helper(v, &p.spans); - } } }) } Type::Option(t) => { let it = Borrowed(quote!(it)); let val = visit(t, features, defs, &it)?; - let name = name.owned_tokens(); + let name = name.ref_tokens(); Some(quote! { - if let Some(it) = &#name { + if let Some(it) = #name { #val; } }) @@ -81,25 +78,6 @@ fn visit( } Some(code) } - Type::Token(t) => { - let name = name.tokens(); - let repr = &defs.tokens[t]; - let is_keyword = repr.chars().next().unwrap().is_alphabetic(); - let spans = if is_keyword { - quote!(span) - } else { - quote!(spans) - }; - Some(quote! { - tokens_helper(v, &#name.#spans); - }) - } - Type::Group(_) => { - let name = name.tokens(); - Some(quote! { - tokens_helper(v, &#name.span); - }) - } Type::Syn(t) => { fn requires_full(features: &Features) -> bool { features.any.contains("full") && features.any.len() == 1 @@ -112,7 +90,7 @@ fn visit( Some(res) } Type::Ext(t) if gen::TERMINAL_TYPES.contains(&&t[..]) => Some(simple_visit(t, name)), - Type::Ext(_) | Type::Std(_) => None, + Type::Ext(_) | Type::Std(_) | Type::Token(_) | Type::Group(_) => None, } } @@ -124,6 +102,11 @@ fn node(traits: &mut TokenStream, impls: &mut TokenStream, s: &Node, defs: &Defi let mut visit_impl = TokenStream::new(); match &s.data { + Data::Enum(variants) if variants.is_empty() => { + visit_impl.extend(quote! { + match *node {} + }); + } Data::Enum(variants) => { let mut visit_variants = TokenStream::new(); @@ -163,33 +146,17 @@ fn node(traits: &mut TokenStream, impls: &mut TokenStream, s: &Node, defs: &Defi } } - let nonexhaustive = if s.exhaustive { - None - } else { - Some(quote! { - #[cfg(syn_no_non_exhaustive)] - _ => unreachable!(), - }) - }; - visit_impl.extend(quote! { match node { #visit_variants - #nonexhaustive } }); } Data::Struct(fields) => { for (field, ty) in fields { - if let Type::Syn(ty) = ty { - if ty == "Reserved" { - continue; - } - } - - let id = Ident::new(&field, Span::call_site()); + let id = Ident::new(field, Span::call_site()); let ref_toks = Owned(quote!(node.#id)); - let visit_field = visit(&ty, &s.features, defs, &ref_toks) + let visit_field = visit(ty, &s.features, defs, &ref_toks) .unwrap_or_else(|| noop_visit(&ref_toks)); visit_impl.extend(quote! { #visit_field; @@ -234,9 +201,8 @@ pub fn generate(defs: &Definitions) -> Result<()> { VISIT_SRC, quote! { #![allow(unused_variables)] + #![allow(clippy::needless_pass_by_ref_mut)] - #[cfg(any(feature = "full", feature = "derive"))] - use crate::gen::helper::visit::*; #[cfg(any(feature = "full", feature = "derive"))] use crate::punctuated::Punctuated; use crate::*; @@ -253,8 +219,6 @@ pub fn generate(defs: &Definitions) -> Result<()> { /// See the [module documentation] for details. /// /// [module documentation]: self - /// - /// *This trait is available only if Syn is built with the `"visit"` feature.* pub trait Visit<'ast> { #traits } diff --git a/codegen/src/visit_mut.rs b/codegen/src/visit_mut.rs index 294d247f..5b96c871 100644 --- a/codegen/src/visit_mut.rs +++ b/codegen/src/visit_mut.rs @@ -6,7 +6,7 @@ use quote::{format_ident, quote}; use syn::Index; use syn_codegen::{Data, Definitions, Features, Node, Type}; -const VISIT_MUT_SRC: &str = "../src/gen/visit_mut.rs"; +const VISIT_MUT_SRC: &str = "src/gen/visit_mut.rs"; fn simple_visit(item: &str, name: &Operand) -> TokenStream { let ident = gen::under_name(item); @@ -50,21 +50,18 @@ fn visit( let val = visit(&p.element, features, defs, &operand)?; let name = name.ref_mut_tokens(); Some(quote! { - for el in Punctuated::pairs_mut(#name) { - let (it, p) = el.into_tuple(); + for mut el in Punctuated::pairs_mut(#name) { + let it = el.value_mut(); #val; - if let Some(p) = p { - tokens_helper(v, &mut p.spans); - } } }) } Type::Option(t) => { let it = Borrowed(quote!(it)); let val = visit(t, features, defs, &it)?; - let name = name.owned_tokens(); + let name = name.ref_mut_tokens(); Some(quote! { - if let Some(it) = &mut #name { + if let Some(it) = #name { #val; } }) @@ -81,25 +78,6 @@ fn visit( } Some(code) } - Type::Token(t) => { - let name = name.tokens(); - let repr = &defs.tokens[t]; - let is_keyword = repr.chars().next().unwrap().is_alphabetic(); - let spans = if is_keyword { - quote!(span) - } else { - quote!(spans) - }; - Some(quote! { - tokens_helper(v, &mut #name.#spans); - }) - } - Type::Group(_) => { - let name = name.tokens(); - Some(quote! { - tokens_helper(v, &mut #name.span); - }) - } Type::Syn(t) => { fn requires_full(features: &Features) -> bool { features.any.contains("full") && features.any.len() == 1 @@ -112,7 +90,7 @@ fn visit( Some(res) } Type::Ext(t) if gen::TERMINAL_TYPES.contains(&&t[..]) => Some(simple_visit(t, name)), - Type::Ext(_) | Type::Std(_) => None, + Type::Ext(_) | Type::Std(_) | Type::Token(_) | Type::Group(_) => None, } } @@ -124,6 +102,11 @@ fn node(traits: &mut TokenStream, impls: &mut TokenStream, s: &Node, defs: &Defi let mut visit_mut_impl = TokenStream::new(); match &s.data { + Data::Enum(variants) if variants.is_empty() => { + visit_mut_impl.extend(quote! { + match *node {} + }); + } Data::Enum(variants) => { let mut visit_mut_variants = TokenStream::new(); @@ -163,33 +146,17 @@ fn node(traits: &mut TokenStream, impls: &mut TokenStream, s: &Node, defs: &Defi } } - let nonexhaustive = if s.exhaustive { - None - } else { - Some(quote! { - #[cfg(syn_no_non_exhaustive)] - _ => unreachable!(), - }) - }; - visit_mut_impl.extend(quote! { match node { #visit_mut_variants - #nonexhaustive } }); } Data::Struct(fields) => { for (field, ty) in fields { - if let Type::Syn(ty) = ty { - if ty == "Reserved" { - continue; - } - } - - let id = Ident::new(&field, Span::call_site()); + let id = Ident::new(field, Span::call_site()); let ref_toks = Owned(quote!(node.#id)); - let visit_mut_field = visit(&ty, &s.features, defs, &ref_toks) + let visit_mut_field = visit(ty, &s.features, defs, &ref_toks) .unwrap_or_else(|| noop_visit(&ref_toks)); visit_mut_impl.extend(quote! { #visit_mut_field; @@ -230,9 +197,8 @@ pub fn generate(defs: &Definitions) -> Result<()> { VISIT_MUT_SRC, quote! { #![allow(unused_variables)] + #![allow(clippy::needless_pass_by_ref_mut)] - #[cfg(any(feature = "full", feature = "derive"))] - use crate::gen::helper::visit_mut::*; #[cfg(any(feature = "full", feature = "derive"))] use crate::punctuated::Punctuated; use crate::*; @@ -250,8 +216,6 @@ pub fn generate(defs: &Definitions) -> Result<()> { /// See the [module documentation] for details. /// /// [module documentation]: self - /// - /// *This trait is available only if Syn is built with the `"visit-mut"` feature.* pub trait VisitMut { #traits } diff --git a/codegen/src/workspace_path.rs b/codegen/src/workspace_path.rs new file mode 100644 index 00000000..eb29bce2 --- /dev/null +++ b/codegen/src/workspace_path.rs @@ -0,0 +1,8 @@ +use std::path::{Path, PathBuf}; + +pub fn get(relative_to_workspace_root: impl AsRef) -> PathBuf { + let mut path = PathBuf::from(env!("CARGO_MANIFEST_DIR")); + assert!(path.pop()); + path.push(relative_to_workspace_root); + path +} diff --git a/dev/Cargo.toml b/dev/Cargo.toml index 79486c12..b61b5618 100644 --- a/dev/Cargo.toml +++ b/dev/Cargo.toml @@ -2,7 +2,7 @@ name = "syn-dev" version = "0.0.0" authors = ["David Tolnay "] -edition = "2018" +edition = "2021" publish = false [lib] @@ -14,9 +14,9 @@ path = "main.rs" name = "syn-dev" [dependencies] -quote = "1.0" +quote = "1" [dependencies.syn] -path = ".." default-features = false -features = ["parsing", "full", "extra-traits", "proc-macro"] +features = ["extra-traits", "full", "parsing", "proc-macro"] +path = ".." diff --git a/examples/dump-syntax/Cargo.toml b/examples/dump-syntax/Cargo.toml index 2363f829..6239141b 100644 --- a/examples/dump-syntax/Cargo.toml +++ b/examples/dump-syntax/Cargo.toml @@ -2,14 +2,14 @@ name = "dump-syntax" version = "0.0.0" authors = ["David Tolnay "] -edition = "2018" +edition = "2021" publish = false [dependencies] colored = "2" -proc-macro2 = { version = "1.0", features = ["span-locations"] } +proc-macro2 = { version = "1", features = ["span-locations"] } [dependencies.syn] -path = "../.." default-features = false -features = ["parsing", "full", "extra-traits"] +features = ["extra-traits", "full", "parsing"] +path = "../.." diff --git a/examples/dump-syntax/src/main.rs b/examples/dump-syntax/src/main.rs index d2b31025..855047a0 100644 --- a/examples/dump-syntax/src/main.rs +++ b/examples/dump-syntax/src/main.rs @@ -13,7 +13,7 @@ //! attrs: [ //! Attribute { //! pound_token: Pound, -//! style: Inner( +//! style: AttrStyle::Inner( //! ... //! } @@ -55,14 +55,14 @@ impl Display for Error { fn main() { if let Err(error) = try_main() { - _ = writeln!(io::stderr(), "{}", error); + let _ = writeln!(io::stderr(), "{}", error); process::exit(1); } } fn try_main() -> Result<(), Error> { let mut args = env::args_os(); - _ = args.next(); // executable name + let _ = args.next(); // executable name let filepath = match (args.next(), args.next()) { (Some(arg), None) => PathBuf::from(arg), @@ -99,11 +99,7 @@ fn render_location( let start = err.span().start(); let mut end = err.span().end(); - if start.line == end.line && start.column == end.column { - return render_fallback(formatter, err); - } - - let code_line = match code.lines().nth(start.line - 1) { + let code_line = match start.line.checked_sub(1).and_then(|n| code.lines().nth(n)) { Some(line) => line, None => return render_fallback(formatter, err), }; @@ -138,7 +134,10 @@ fn render_location( label = start.line.to_string().blue().bold(), code = code_line.trim_end(), offset = " ".repeat(start.column), - underline = "^".repeat(end.column - start.column).red().bold(), + underline = "^" + .repeat(end.column.saturating_sub(start.column).max(1)) + .red() + .bold(), message = err.to_string().red(), ) } diff --git a/examples/heapsize/example/Cargo.toml b/examples/heapsize/example/Cargo.toml index 85c7699c..ab2c4266 100644 --- a/examples/heapsize/example/Cargo.toml +++ b/examples/heapsize/example/Cargo.toml @@ -2,7 +2,7 @@ name = "heapsize_example" version = "0.0.0" authors = ["David Tolnay "] -edition = "2018" +edition = "2021" publish = false [dependencies] diff --git a/examples/heapsize/heapsize/Cargo.toml b/examples/heapsize/heapsize/Cargo.toml index 27bb9541..2b2c31e9 100644 --- a/examples/heapsize/heapsize/Cargo.toml +++ b/examples/heapsize/heapsize/Cargo.toml @@ -2,7 +2,7 @@ name = "heapsize" version = "0.0.0" authors = ["David Tolnay "] -edition = "2018" +edition = "2021" publish = false [dependencies] diff --git a/examples/heapsize/heapsize_derive/Cargo.toml b/examples/heapsize/heapsize_derive/Cargo.toml index f4357b98..06066cca 100644 --- a/examples/heapsize/heapsize_derive/Cargo.toml +++ b/examples/heapsize/heapsize_derive/Cargo.toml @@ -2,13 +2,13 @@ name = "heapsize_derive" version = "0.0.0" authors = ["David Tolnay "] -edition = "2018" +edition = "2021" publish = false [lib] proc-macro = true [dependencies] -proc-macro2 = "1.0" -quote = "1.0" +proc-macro2 = "1" +quote = "1" syn = { path = "../../.." } diff --git a/examples/lazy-static/example/Cargo.toml b/examples/lazy-static/example/Cargo.toml index 44d155eb..b6e95a2b 100644 --- a/examples/lazy-static/example/Cargo.toml +++ b/examples/lazy-static/example/Cargo.toml @@ -2,7 +2,7 @@ name = "lazy-static-example" version = "0.0.0" authors = ["David Tolnay "] -edition = "2018" +edition = "2021" publish = false [dependencies] diff --git a/examples/lazy-static/lazy-static/Cargo.toml b/examples/lazy-static/lazy-static/Cargo.toml index bf65787c..be966caa 100644 --- a/examples/lazy-static/lazy-static/Cargo.toml +++ b/examples/lazy-static/lazy-static/Cargo.toml @@ -2,13 +2,13 @@ name = "lazy_static" version = "0.0.0" authors = ["David Tolnay "] -edition = "2018" +edition = "2021" publish = false [lib] proc-macro = true [dependencies] -proc-macro2 = { version = "1.0", features = ["nightly"] } -quote = "1.0" +proc-macro2 = { version = "1", features = ["nightly"] } +quote = "1" syn = { path = "../../../", features = ["full"] } diff --git a/examples/trace-var/README.md b/examples/trace-var/README.md index b93fae2b..294ed7e4 100644 --- a/examples/trace-var/README.md +++ b/examples/trace-var/README.md @@ -42,7 +42,7 @@ n = 1 The procedural macro uses a syntax tree [`Fold`] to rewrite every `let` statement and assignment expression in the following way: -[`Fold`]: https://docs.rs/syn/1.0/syn/fold/trait.Fold.html +[`Fold`]: https://docs.rs/syn/2.0/syn/fold/trait.Fold.html ```rust // Before diff --git a/examples/trace-var/example/Cargo.toml b/examples/trace-var/example/Cargo.toml index 45ab3fd2..b95e4c35 100644 --- a/examples/trace-var/example/Cargo.toml +++ b/examples/trace-var/example/Cargo.toml @@ -2,7 +2,7 @@ name = "trace-var-example" version = "0.0.0" authors = ["David Tolnay "] -edition = "2018" +edition = "2021" publish = false [dependencies] diff --git a/examples/trace-var/trace-var/Cargo.toml b/examples/trace-var/trace-var/Cargo.toml index 72f56e92..ffa1da08 100644 --- a/examples/trace-var/trace-var/Cargo.toml +++ b/examples/trace-var/trace-var/Cargo.toml @@ -2,13 +2,13 @@ name = "trace-var" version = "0.0.0" authors = ["David Tolnay "] -edition = "2018" +edition = "2021" publish = false [lib] proc-macro = true [dependencies] -proc-macro2 = { version = "1.0", features = ["nightly"] } -quote = "1.0" -syn = { path = "../../../", features = ["full", "fold"] } +proc-macro2 = { version = "1", features = ["nightly"] } +quote = "1" +syn = { path = "../../../", features = ["fold", "full"] } diff --git a/examples/trace-var/trace-var/src/lib.rs b/examples/trace-var/trace-var/src/lib.rs index 67059416..0dcaa94b 100644 --- a/examples/trace-var/trace-var/src/lib.rs +++ b/examples/trace-var/trace-var/src/lib.rs @@ -4,7 +4,7 @@ use std::collections::HashSet as Set; use syn::fold::{self, Fold}; use syn::parse::{Parse, ParseStream, Result}; use syn::punctuated::Punctuated; -use syn::{parse_macro_input, parse_quote, Expr, Ident, ItemFn, Local, Pat, Stmt, Token}; +use syn::{parse_macro_input, parse_quote, BinOp, Expr, Ident, ItemFn, Local, Pat, Stmt, Token}; /// Parses a list of variable names separated by commas. /// @@ -84,7 +84,7 @@ impl Args { /// let VAR = { let VAR = INIT; println!("VAR = {:?}", VAR); VAR }; fn let_and_print(&mut self, local: Local) -> Stmt { let Local { pat, init, .. } = local; - let init = self.fold_expr(*init.unwrap().1); + let init = self.fold_expr(*init.unwrap().expr); let ident = match pat { Pat::Ident(ref p) => &p.ident, _ => unreachable!(), @@ -122,11 +122,11 @@ impl Fold for Args { Expr::Assign(fold::fold_expr_assign(self, e)) } } - Expr::AssignOp(e) => { + Expr::Binary(e) if is_assign_op(e.op) => { if self.should_print_expr(&e.left) { self.assign_and_print(*e.left, &e.op, *e.right) } else { - Expr::AssignOp(fold::fold_expr_assign_op(self, e)) + Expr::Binary(fold::fold_expr_binary(self, e)) } } _ => fold::fold_expr(self, e), @@ -147,6 +147,22 @@ impl Fold for Args { } } +fn is_assign_op(op: BinOp) -> bool { + match op { + BinOp::AddAssign(_) + | BinOp::SubAssign(_) + | BinOp::MulAssign(_) + | BinOp::DivAssign(_) + | BinOp::RemAssign(_) + | BinOp::BitXorAssign(_) + | BinOp::BitAndAssign(_) + | BinOp::BitOrAssign(_) + | BinOp::ShlAssign(_) + | BinOp::ShrAssign(_) => true, + _ => false, + } +} + /// Attribute to print the value of the given variables each time they are /// reassigned. /// diff --git a/fuzz/.gitignore b/fuzz/.gitignore index 188f1960..f83457ae 100644 --- a/fuzz/.gitignore +++ b/fuzz/.gitignore @@ -1,3 +1,4 @@ artifacts/ corpus/ +coverage/ target/ diff --git a/fuzz/Cargo.toml b/fuzz/Cargo.toml index ed4b8897..94578e62 100644 --- a/fuzz/Cargo.toml +++ b/fuzz/Cargo.toml @@ -10,9 +10,12 @@ cargo-fuzz = true [dependencies] libfuzzer-sys = "0.4" -proc-macro2 = "1" +proc-macro2 = "1.0.52" syn = { path = "..", default-features = false, features = ["full", "parsing"] } +[features] +span-locations = ["proc-macro2/span-locations"] + [[bin]] name = "create_token_buffer" path = "fuzz_targets/create_token_buffer.rs" diff --git a/fuzz/fuzz_targets/create_token_buffer.rs b/fuzz/fuzz_targets/create_token_buffer.rs index 7b6622a1..48540e1b 100644 --- a/fuzz/fuzz_targets/create_token_buffer.rs +++ b/fuzz/fuzz_targets/create_token_buffer.rs @@ -12,7 +12,7 @@ fn immediate_fail(_input: ParseStream) -> syn::Result<()> { fuzz_target!(|data: &[u8]| { if data.len() < 300 { if let Ok(string) = str::from_utf8(data) { - _ = immediate_fail.parse_str(string); + let _ = immediate_fail.parse_str(string); } } }); diff --git a/fuzz/fuzz_targets/parse_file.rs b/fuzz/fuzz_targets/parse_file.rs index 4fa11dd9..b30d694e 100644 --- a/fuzz/fuzz_targets/parse_file.rs +++ b/fuzz/fuzz_targets/parse_file.rs @@ -6,7 +6,7 @@ use std::str; fuzz_target!(|data: &[u8]| { if data.len() < 300 { if let Ok(string) = str::from_utf8(data) { - _ = syn::parse_file(string); + let _ = syn::parse_file(string); } } }); diff --git a/json/Cargo.toml b/json/Cargo.toml index afa160f5..e6ff2d54 100644 --- a/json/Cargo.toml +++ b/json/Cargo.toml @@ -1,22 +1,29 @@ [package] name = "syn-codegen" -version = "0.3.0" # also update html_root_url +version = "0.4.1" # also update html_root_url authors = ["David Tolnay "] categories = ["development-tools::procedural-macro-helpers"] description = "Syntax tree describing Syn's syntax tree" documentation = "https://docs.rs/syn-codegen" -edition = "2018" +edition = "2021" keywords = ["syn"] license = "MIT OR Apache-2.0" repository = "https://github.com/dtolnay/syn" [dependencies] -indexmap = { version = "1.0", features = ["serde-1"] } -semver = { version = "1.0", features = ["serde"] } -serde = { version = "1.0.88", features = ["derive"] } +indexmap = { version = "2", features = ["serde"] } +semver = { version = "1", features = ["serde"] } +serde = "1.0.88" +serde_derive = "1.0.88" [dev-dependencies] -serde_json = "1.0" +serde_json = "1" + +[lib] +doc-scrape-examples = false [package.metadata.docs.rs] targets = ["x86_64-unknown-linux-gnu"] +rustdoc-args = ["--generate-link-to-definition"] + +[workspace] diff --git a/json/src/lib.rs b/json/src/lib.rs index be5a02e0..77f47d4c 100644 --- a/json/src/lib.rs +++ b/json/src/lib.rs @@ -13,9 +13,9 @@ //! of the [`visit`], [`visit_mut`], and [`fold`] modules can be generated //! programmatically from a description of the syntax tree. //! -//! [`visit`]: https://docs.rs/syn/1.0/syn/visit/index.html -//! [`visit_mut`]: https://docs.rs/syn/1.0/syn/visit_mut/index.html -//! [`fold`]: https://docs.rs/syn/1.0/syn/fold/index.html +//! [`visit`]: https://docs.rs/syn/2.0/syn/visit/index.html +//! [`visit_mut`]: https://docs.rs/syn/2.0/syn/visit_mut/index.html +//! [`fold`]: https://docs.rs/syn/2.0/syn/fold/index.html //! //! To make this type of code as easy as possible to implement in any language, //! every Syn release comes with a machine-readable description of that version @@ -44,11 +44,12 @@ //! } //! ``` -#![doc(html_root_url = "https://docs.rs/syn-codegen/0.2.0")] +#![doc(html_root_url = "https://docs.rs/syn-codegen/0.4.1")] use indexmap::IndexMap; use semver::Version; -use serde::{Deserialize, Deserializer, Serialize}; +use serde::de::{Deserialize, Deserializer}; +use serde_derive::{Deserialize, Serialize}; use std::collections::{BTreeMap, BTreeSet}; /// Top-level content of the syntax tree description. @@ -96,7 +97,7 @@ pub struct Node { /// Content of a syntax tree data structure. #[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] pub enum Data { - /// This is an opaque type with no publicy accessible structure. + /// This is an opaque type with no publicly accessible structure. Private, /// This type is a braced struct with named fields. diff --git a/src/attr.rs b/src/attr.rs index bace94f4..b6c4675b 100644 --- a/src/attr.rs +++ b/src/attr.rs @@ -1,19 +1,15 @@ use super::*; -use crate::punctuated::Punctuated; use proc_macro2::TokenStream; use std::iter; use std::slice; #[cfg(feature = "parsing")] -use crate::parse::{Parse, ParseBuffer, ParseStream, Parser, Result}; +use crate::meta::{self, ParseNestedMeta}; #[cfg(feature = "parsing")] -use crate::punctuated::Pair; +use crate::parse::{Parse, ParseStream, Parser, Result}; ast_struct! { - /// An attribute like `#[repr(transparent)]`. - /// - /// *This type is available only if Syn is built with the `"derive"` or `"full"` - /// feature.* + /// An attribute, like `#[repr(transparent)]`. /// ///
/// @@ -23,27 +19,52 @@ ast_struct! { /// /// - Outer attributes like `#[repr(transparent)]`. These appear outside or /// in front of the item they describe. + /// /// - Inner attributes like `#![feature(proc_macro)]`. These appear inside /// of the item they describe, usually a module. - /// - Outer doc comments like `/// # Example`. - /// - Inner doc comments like `//! Please file an issue`. - /// - Outer block comments `/** # Example */`. - /// - Inner block comments `/*! Please file an issue */`. + /// + /// - Outer one-line doc comments like `/// Example`. + /// + /// - Inner one-line doc comments like `//! Please file an issue`. + /// + /// - Outer documentation blocks `/** Example */`. + /// + /// - Inner documentation blocks `/*! Please file an issue */`. /// /// The `style` field of type `AttrStyle` distinguishes whether an attribute - /// is outer or inner. Doc comments and block comments are promoted to - /// attributes, as this is how they are processed by the compiler and by + /// is outer or inner. + /// + /// Every attribute has a `path` that indicates the intended interpretation + /// of the rest of the attribute's contents. The path and the optional + /// additional contents are represented together in the `meta` field of the + /// attribute in three possible varieties: + /// + /// - Meta::Path — attributes whose information content conveys just a + /// path, for example the `#[test]` attribute. + /// + /// - Meta::List — attributes that carry arbitrary tokens after the + /// path, surrounded by a delimiter (parenthesis, bracket, or brace). For + /// example `#[derive(Copy)]` or `#[precondition(x < 5)]`. + /// + /// - Meta::NameValue — attributes with an `=` sign after the path, + /// followed by a Rust expression. For example `#[path = + /// "sys/windows.rs"]`. + /// + /// All doc comments are represented in the NameValue style with a path of + /// "doc", as this is how they are processed by the compiler and by /// `macro_rules!` macros. /// - /// The `path` field gives the possibly colon-delimited path against which - /// the attribute is resolved. It is equal to `"doc"` for desugared doc - /// comments. The `tokens` field contains the rest of the attribute body as - /// tokens. - /// /// ```text - /// #[derive(Copy)] #[crate::precondition x < 5] - /// ^^^^^^~~~~~~ ^^^^^^^^^^^^^^^^^^^ ~~~~~ - /// path tokens path tokens + /// #[derive(Copy, Clone)] + /// ~~~~~~Path + /// ^^^^^^^^^^^^^^^^^^^Meta::List + /// + /// #[path = "sys/windows.rs"] + /// ~~~~Path + /// ^^^^^^^^^^^^^^^^^^^^^^^Meta::NameValue + /// + /// #[test] + /// ^^^^Meta::Path /// ``` /// ///
@@ -93,18 +114,13 @@ ast_struct! { /// /// The grammar of attributes in Rust is very flexible, which makes the /// syntax tree not that useful on its own. In particular, arguments of the - /// attribute are held in an arbitrary `tokens: TokenStream`. Macros are - /// expected to check the `path` of the attribute, decide whether they - /// recognize it, and then parse the remaining tokens according to whatever - /// grammar they wish to require for that kind of attribute. + /// `Meta::List` variety of attribute are held in an arbitrary `tokens: + /// TokenStream`. Macros are expected to check the `path` of the attribute, + /// decide whether they recognize it, and then parse the remaining tokens + /// according to whatever grammar they wish to require for that kind of + /// attribute. Use [`parse_args()`] to parse those tokens into the expected + /// data structure. /// - /// If the attribute you are parsing is expected to conform to the - /// conventional structured form of attribute, use [`parse_meta()`] to - /// obtain that structured representation. If the attribute follows some - /// other grammar of its own, use [`parse_args()`] to parse that into the - /// expected data structure. - /// - /// [`parse_meta()`]: Attribute::parse_meta /// [`parse_args()`]: Attribute::parse_args /// ///


@@ -150,65 +166,49 @@ ast_struct! { pub pound_token: Token![#], pub style: AttrStyle, pub bracket_token: token::Bracket, - pub path: Path, - pub tokens: TokenStream, + pub meta: Meta, } } impl Attribute { - /// Parses the content of the attribute, consisting of the path and tokens, - /// as a [`Meta`] if possible. + /// Returns the path that identifies the interpretation of this attribute. /// - /// *This function is available only if Syn is built with the `"parsing"` - /// feature.* - #[cfg(feature = "parsing")] - #[cfg_attr(doc_cfg, doc(cfg(feature = "parsing")))] - pub fn parse_meta(&self) -> Result { - fn clone_ident_segment(segment: &PathSegment) -> PathSegment { - PathSegment { - ident: segment.ident.clone(), - arguments: PathArguments::None, - } - } - - let path = Path { - leading_colon: self - .path - .leading_colon - .as_ref() - .map(|colon| Token![::](colon.spans)), - segments: self - .path - .segments - .pairs() - .map(|pair| match pair { - Pair::Punctuated(seg, punct) => { - Pair::Punctuated(clone_ident_segment(seg), Token![::](punct.spans)) - } - Pair::End(seg) => Pair::End(clone_ident_segment(seg)), - }) - .collect(), - }; - - let parser = |input: ParseStream| parsing::parse_meta_after_path(path, input); - parse::Parser::parse2(parser, self.tokens.clone()) + /// For example this would return the `test` in `#[test]`, the `derive` in + /// `#[derive(Copy)]`, and the `path` in `#[path = "sys/windows.rs"]`. + pub fn path(&self) -> &Path { + self.meta.path() } /// Parse the arguments to the attribute as a syntax tree. /// - /// This is similar to `syn::parse2::(attr.tokens)` except that: + /// This is similar to pulling out the `TokenStream` from `Meta::List` and + /// doing `syn::parse2::(meta_list.tokens)`, except that using + /// `parse_args` the error message has a more useful span when `tokens` is + /// empty. /// - /// - the surrounding delimiters are *not* included in the input to the - /// parser; and - /// - the error message has a more useful span when `tokens` is empty. + /// The surrounding delimiters are *not* included in the input to the + /// parser. /// /// ```text /// #[my_attr(value < 5)] /// ^^^^^^^^^ what gets parsed /// ``` /// - /// *This function is available only if Syn is built with the `"parsing"` - /// feature.* + /// # Example + /// + /// ``` + /// use syn::{parse_quote, Attribute, Expr}; + /// + /// let attr: Attribute = parse_quote! { + /// #[precondition(value < 5)] + /// }; + /// + /// if attr.path().is_ident("precondition") { + /// let precondition: Expr = attr.parse_args()?; + /// // ... + /// } + /// # anyhow::Ok(()) + /// ``` #[cfg(feature = "parsing")] #[cfg_attr(doc_cfg, doc(cfg(feature = "parsing")))] pub fn parse_args(&self) -> Result { @@ -217,22 +217,182 @@ impl Attribute { /// Parse the arguments to the attribute using the given parser. /// - /// *This function is available only if Syn is built with the `"parsing"` - /// feature.* + /// # Example + /// + /// ``` + /// use syn::{parse_quote, Attribute}; + /// + /// let attr: Attribute = parse_quote! { + /// #[inception { #[brrrrrrraaaaawwwwrwrrrmrmrmmrmrmmmmm] }] + /// }; + /// + /// let bwom = attr.parse_args_with(Attribute::parse_outer)?; + /// + /// // Attribute does not have a Parse impl, so we couldn't directly do: + /// // let bwom: Attribute = attr.parse_args()?; + /// # anyhow::Ok(()) + /// ``` #[cfg(feature = "parsing")] #[cfg_attr(doc_cfg, doc(cfg(feature = "parsing")))] pub fn parse_args_with(&self, parser: F) -> Result { - let parser = |input: ParseStream| { - let args = enter_args(self, input)?; - parse::parse_stream(parser, &args) - }; - parser.parse2(self.tokens.clone()) + match &self.meta { + Meta::Path(path) => Err(crate::error::new2( + path.segments.first().unwrap().ident.span(), + path.segments.last().unwrap().ident.span(), + format!( + "expected attribute arguments in parentheses: {}[{}(...)]", + parsing::DisplayAttrStyle(&self.style), + parsing::DisplayPath(path), + ), + )), + Meta::NameValue(meta) => Err(Error::new( + meta.eq_token.span, + format_args!( + "expected parentheses: {}[{}(...)]", + parsing::DisplayAttrStyle(&self.style), + parsing::DisplayPath(&meta.path), + ), + )), + Meta::List(meta) => meta.parse_args_with(parser), + } + } + + /// Parse the arguments to the attribute, expecting it to follow the + /// conventional structure used by most of Rust's built-in attributes. + /// + /// The [*Meta Item Attribute Syntax*][syntax] section in the Rust reference + /// explains the convention in more detail. Not all attributes follow this + /// convention, so [`parse_args()`][Self::parse_args] is available if you + /// need to parse arbitrarily goofy attribute syntax. + /// + /// [syntax]: https://doc.rust-lang.org/reference/attributes.html#meta-item-attribute-syntax + /// + /// # Example + /// + /// We'll parse a struct, and then parse some of Rust's `#[repr]` attribute + /// syntax. + /// + /// ``` + /// use syn::{parenthesized, parse_quote, token, ItemStruct, LitInt}; + /// + /// let input: ItemStruct = parse_quote! { + /// #[repr(C, align(4))] + /// pub struct MyStruct(u16, u32); + /// }; + /// + /// let mut repr_c = false; + /// let mut repr_transparent = false; + /// let mut repr_align = None::; + /// let mut repr_packed = None::; + /// for attr in &input.attrs { + /// if attr.path().is_ident("repr") { + /// attr.parse_nested_meta(|meta| { + /// // #[repr(C)] + /// if meta.path.is_ident("C") { + /// repr_c = true; + /// return Ok(()); + /// } + /// + /// // #[repr(transparent)] + /// if meta.path.is_ident("transparent") { + /// repr_transparent = true; + /// return Ok(()); + /// } + /// + /// // #[repr(align(N))] + /// if meta.path.is_ident("align") { + /// let content; + /// parenthesized!(content in meta.input); + /// let lit: LitInt = content.parse()?; + /// let n: usize = lit.base10_parse()?; + /// repr_align = Some(n); + /// return Ok(()); + /// } + /// + /// // #[repr(packed)] or #[repr(packed(N))], omitted N means 1 + /// if meta.path.is_ident("packed") { + /// if meta.input.peek(token::Paren) { + /// let content; + /// parenthesized!(content in meta.input); + /// let lit: LitInt = content.parse()?; + /// let n: usize = lit.base10_parse()?; + /// repr_packed = Some(n); + /// } else { + /// repr_packed = Some(1); + /// } + /// return Ok(()); + /// } + /// + /// Err(meta.error("unrecognized repr")) + /// })?; + /// } + /// } + /// # anyhow::Ok(()) + /// ``` + /// + /// # Alternatives + /// + /// In some cases, for attributes which have nested layers of structured + /// content, the following less flexible approach might be more convenient: + /// + /// ``` + /// # use syn::{parse_quote, ItemStruct}; + /// # + /// # let input: ItemStruct = parse_quote! { + /// # #[repr(C, align(4))] + /// # pub struct MyStruct(u16, u32); + /// # }; + /// # + /// use syn::punctuated::Punctuated; + /// use syn::{parenthesized, token, Error, LitInt, Meta, Token}; + /// + /// let mut repr_c = false; + /// let mut repr_transparent = false; + /// let mut repr_align = None::; + /// let mut repr_packed = None::; + /// for attr in &input.attrs { + /// if attr.path().is_ident("repr") { + /// let nested = attr.parse_args_with(Punctuated::::parse_terminated)?; + /// for meta in nested { + /// match meta { + /// // #[repr(C)] + /// Meta::Path(path) if path.is_ident("C") => { + /// repr_c = true; + /// } + /// + /// // #[repr(align(N))] + /// Meta::List(meta) if meta.path.is_ident("align") => { + /// let lit: LitInt = meta.parse_args()?; + /// let n: usize = lit.base10_parse()?; + /// repr_align = Some(n); + /// } + /// + /// /* ... */ + /// + /// _ => { + /// return Err(Error::new_spanned(meta, "unrecognized repr")); + /// } + /// } + /// } + /// } + /// } + /// # Ok(()) + /// ``` + #[cfg(feature = "parsing")] + #[cfg_attr(doc_cfg, doc(cfg(feature = "parsing")))] + pub fn parse_nested_meta( + &self, + logic: impl FnMut(ParseNestedMeta) -> Result<()>, + ) -> Result<()> { + self.parse_args_with(meta::parser(logic)) } /// Parses zero or more outer attributes from the stream. /// - /// *This function is available only if Syn is built with the `"parsing"` - /// feature.* + /// # Example + /// + /// See + /// [*Parsing from tokens to Attribute*](#parsing-from-tokens-to-attribute). #[cfg(feature = "parsing")] #[cfg_attr(doc_cfg, doc(cfg(feature = "parsing")))] pub fn parse_outer(input: ParseStream) -> Result> { @@ -245,8 +405,10 @@ impl Attribute { /// Parses zero or more inner attributes from the stream. /// - /// *This function is available only if Syn is built with the `"parsing"` - /// feature.* + /// # Example + /// + /// See + /// [*Parsing from tokens to Attribute*](#parsing-from-tokens-to-attribute). #[cfg(feature = "parsing")] #[cfg_attr(doc_cfg, doc(cfg(feature = "parsing")))] pub fn parse_inner(input: ParseStream) -> Result> { @@ -256,65 +418,10 @@ impl Attribute { } } -#[cfg(feature = "parsing")] -fn expected_parentheses(attr: &Attribute) -> String { - let style = match attr.style { - AttrStyle::Outer => "#", - AttrStyle::Inner(_) => "#!", - }; - - let mut path = String::new(); - for segment in &attr.path.segments { - if !path.is_empty() || attr.path.leading_colon.is_some() { - path += "::"; - } - path += &segment.ident.to_string(); - } - - format!("{}[{}(...)]", style, path) -} - -#[cfg(feature = "parsing")] -fn enter_args<'a>(attr: &Attribute, input: ParseStream<'a>) -> Result> { - if input.is_empty() { - let expected = expected_parentheses(attr); - let msg = format!("expected attribute arguments in parentheses: {}", expected); - return Err(crate::error::new2( - attr.pound_token.span, - attr.bracket_token.span, - msg, - )); - } else if input.peek(Token![=]) { - let expected = expected_parentheses(attr); - let msg = format!("expected parentheses: {}", expected); - return Err(input.error(msg)); - }; - - let content; - if input.peek(token::Paren) { - parenthesized!(content in input); - } else if input.peek(token::Bracket) { - bracketed!(content in input); - } else if input.peek(token::Brace) { - braced!(content in input); - } else { - return Err(input.error("unexpected token in attribute arguments")); - } - - if input.is_empty() { - Ok(content) - } else { - Err(input.error("unexpected token in attribute arguments")) - } -} - ast_enum! { /// Distinguishes between attributes that decorate an item and attributes /// that are contained within an item. /// - /// *This type is available only if Syn is built with the `"derive"` or `"full"` - /// feature.* - /// /// # Outer attributes /// /// - `#[repr(transparent)]` @@ -336,9 +443,6 @@ ast_enum! { ast_enum_of_structs! { /// Content of a compile-time structured attribute. /// - /// *This type is available only if Syn is built with the `"derive"` or `"full"` - /// feature.* - /// /// ## Path /// /// A meta path is like the `test` in `#[test]`. @@ -371,32 +475,26 @@ ast_enum_of_structs! { ast_struct! { /// A structured list within an attribute, like `derive(Copy, Clone)`. - /// - /// *This type is available only if Syn is built with the `"derive"` or - /// `"full"` feature.* #[cfg_attr(doc_cfg, doc(cfg(any(feature = "full", feature = "derive"))))] pub struct MetaList { pub path: Path, - pub paren_token: token::Paren, - pub nested: Punctuated, + pub delimiter: MacroDelimiter, + pub tokens: TokenStream, } } ast_struct! { /// A name-value pair within an attribute, like `feature = "nightly"`. - /// - /// *This type is available only if Syn is built with the `"derive"` or - /// `"full"` feature.* #[cfg_attr(doc_cfg, doc(cfg(any(feature = "full", feature = "derive"))))] pub struct MetaNameValue { pub path: Path, pub eq_token: Token![=], - pub lit: Lit, + pub value: Expr, } } impl Meta { - /// Returns the identifier that begins this structured meta item. + /// Returns the path that begins this structured meta item. /// /// For example this would return the `test` in `#[test]`, the `derive` in /// `#[derive(Copy)]`, and the `path` in `#[path = "sys/windows.rs"]`. @@ -407,63 +505,84 @@ impl Meta { Meta::NameValue(meta) => &meta.path, } } -} -ast_enum_of_structs! { - /// Element of a compile-time attribute list. - /// - /// *This type is available only if Syn is built with the `"derive"` or `"full"` - /// feature.* - #[cfg_attr(doc_cfg, doc(cfg(any(feature = "full", feature = "derive"))))] - pub enum NestedMeta { - /// A structured meta item, like the `Copy` in `#[derive(Copy)]` which - /// would be a nested `Meta::Path`. - Meta(Meta), + /// Error if this is a `Meta::List` or `Meta::NameValue`. + #[cfg(feature = "parsing")] + #[cfg_attr(doc_cfg, doc(cfg(feature = "parsing")))] + pub fn require_path_only(&self) -> Result<&Path> { + let error_span = match self { + Meta::Path(path) => return Ok(path), + Meta::List(meta) => meta.delimiter.span().open(), + Meta::NameValue(meta) => meta.eq_token.span, + }; + Err(Error::new(error_span, "unexpected token in attribute")) + } - /// A Rust literal, like the `"new_name"` in `#[rename("new_name")]`. - Lit(Lit), + /// Error if this is a `Meta::Path` or `Meta::NameValue`. + #[cfg(feature = "parsing")] + #[cfg_attr(doc_cfg, doc(cfg(feature = "parsing")))] + pub fn require_list(&self) -> Result<&MetaList> { + match self { + Meta::List(meta) => Ok(meta), + Meta::Path(path) => Err(crate::error::new2( + path.segments.first().unwrap().ident.span(), + path.segments.last().unwrap().ident.span(), + format!( + "expected attribute arguments in parentheses: `{}(...)`", + parsing::DisplayPath(path), + ), + )), + Meta::NameValue(meta) => Err(Error::new(meta.eq_token.span, "expected `(`")), + } + } + + /// Error if this is a `Meta::Path` or `Meta::List`. + #[cfg(feature = "parsing")] + #[cfg_attr(doc_cfg, doc(cfg(feature = "parsing")))] + pub fn require_name_value(&self) -> Result<&MetaNameValue> { + match self { + Meta::NameValue(meta) => Ok(meta), + Meta::Path(path) => Err(crate::error::new2( + path.segments.first().unwrap().ident.span(), + path.segments.last().unwrap().ident.span(), + format!( + "expected a value for this attribute: `{} = ...`", + parsing::DisplayPath(path), + ), + )), + Meta::List(meta) => Err(Error::new(meta.delimiter.span().open(), "expected `=`")), + } } } -/// Conventional argument type associated with an invocation of an attribute -/// macro. -/// -/// For example if we are developing an attribute macro that is intended to be -/// invoked on function items as follows: -/// -/// ``` -/// # const IGNORE: &str = stringify! { -/// #[my_attribute(path = "/v1/refresh")] -/// # }; -/// pub fn refresh() { -/// /* ... */ -/// } -/// ``` -/// -/// The implementation of this macro would want to parse its attribute arguments -/// as type `AttributeArgs`. -/// -/// ``` -/// # extern crate proc_macro; -/// # -/// use proc_macro::TokenStream; -/// use syn::{parse_macro_input, AttributeArgs, ItemFn}; -/// -/// # const IGNORE: &str = stringify! { -/// #[proc_macro_attribute] -/// # }; -/// pub fn my_attribute(args: TokenStream, input: TokenStream) -> TokenStream { -/// let args = parse_macro_input!(args as AttributeArgs); -/// let input = parse_macro_input!(input as ItemFn); -/// -/// /* ... */ -/// # "".parse().unwrap() -/// } -/// ``` -#[cfg_attr(doc_cfg, doc(cfg(any(feature = "full", feature = "derive"))))] -pub type AttributeArgs = Vec; +impl MetaList { + /// See [`Attribute::parse_args`]. + #[cfg(feature = "parsing")] + #[cfg_attr(doc_cfg, doc(cfg(feature = "parsing")))] + pub fn parse_args(&self) -> Result { + self.parse_args_with(T::parse) + } -pub trait FilterAttrs<'a> { + /// See [`Attribute::parse_args_with`]. + #[cfg(feature = "parsing")] + #[cfg_attr(doc_cfg, doc(cfg(feature = "parsing")))] + pub fn parse_args_with(&self, parser: F) -> Result { + let scope = self.delimiter.span().close(); + crate::parse::parse_scoped(parser, scope, self.tokens.clone()) + } + + /// See [`Attribute::parse_nested_meta`]. + #[cfg(feature = "parsing")] + #[cfg_attr(doc_cfg, doc(cfg(feature = "parsing")))] + pub fn parse_nested_meta( + &self, + logic: impl FnMut(ParseNestedMeta) -> Result<()>, + ) -> Result<()> { + self.parse_args_with(meta::parser(logic)) + } +} + +pub(crate) trait FilterAttrs<'a> { type Ret: Iterator; fn outer(self) -> Self::Ret; @@ -495,69 +614,43 @@ impl<'a> FilterAttrs<'a> for &'a [Attribute] { } #[cfg(feature = "parsing")] -pub mod parsing { +pub(crate) mod parsing { use super::*; - use crate::ext::IdentExt; + use crate::parse::discouraged::Speculative as _; use crate::parse::{Parse, ParseStream, Result}; + use std::fmt::{self, Display}; - pub fn parse_inner(input: ParseStream, attrs: &mut Vec) -> Result<()> { + pub(crate) fn parse_inner(input: ParseStream, attrs: &mut Vec) -> Result<()> { while input.peek(Token![#]) && input.peek2(Token![!]) { attrs.push(input.call(parsing::single_parse_inner)?); } Ok(()) } - pub fn single_parse_inner(input: ParseStream) -> Result { + pub(crate) fn single_parse_inner(input: ParseStream) -> Result { let content; Ok(Attribute { pound_token: input.parse()?, style: AttrStyle::Inner(input.parse()?), bracket_token: bracketed!(content in input), - path: content.call(Path::parse_mod_style)?, - tokens: content.parse()?, + meta: content.parse()?, }) } - pub fn single_parse_outer(input: ParseStream) -> Result { + pub(crate) fn single_parse_outer(input: ParseStream) -> Result { let content; Ok(Attribute { pound_token: input.parse()?, style: AttrStyle::Outer, bracket_token: bracketed!(content in input), - path: content.call(Path::parse_mod_style)?, - tokens: content.parse()?, - }) - } - - // Like Path::parse_mod_style but accepts keywords in the path. - fn parse_meta_path(input: ParseStream) -> Result { - Ok(Path { - leading_colon: input.parse()?, - segments: { - let mut segments = Punctuated::new(); - while input.peek(Ident::peek_any) { - let ident = Ident::parse_any(input)?; - segments.push_value(PathSegment::from(ident)); - if !input.peek(Token![::]) { - break; - } - let punct = input.parse()?; - segments.push_punct(punct); - } - if segments.is_empty() { - return Err(input.error("expected path")); - } else if segments.trailing_punct() { - return Err(input.error("expected path segment")); - } - segments - }, + meta: content.parse()?, }) } #[cfg_attr(doc_cfg, doc(cfg(feature = "parsing")))] impl Parse for Meta { fn parse(input: ParseStream) -> Result { - let path = input.call(parse_meta_path)?; + let path = input.call(Path::parse_mod_style)?; parse_meta_after_path(path, input) } } @@ -565,7 +658,7 @@ pub mod parsing { #[cfg_attr(doc_cfg, doc(cfg(feature = "parsing")))] impl Parse for MetaList { fn parse(input: ParseStream) -> Result { - let path = input.call(parse_meta_path)?; + let path = input.call(Path::parse_mod_style)?; parse_meta_list_after_path(path, input) } } @@ -573,28 +666,13 @@ pub mod parsing { #[cfg_attr(doc_cfg, doc(cfg(feature = "parsing")))] impl Parse for MetaNameValue { fn parse(input: ParseStream) -> Result { - let path = input.call(parse_meta_path)?; + let path = input.call(Path::parse_mod_style)?; parse_meta_name_value_after_path(path, input) } } - #[cfg_attr(doc_cfg, doc(cfg(feature = "parsing")))] - impl Parse for NestedMeta { - fn parse(input: ParseStream) -> Result { - if input.peek(Lit) && !(input.peek(LitBool) && input.peek2(Token![=])) { - input.parse().map(NestedMeta::Lit) - } else if input.peek(Ident::peek_any) - || input.peek(Token![::]) && input.peek3(Ident::peek_any) - { - input.parse().map(NestedMeta::Meta) - } else { - Err(input.error("expected identifier or literal")) - } - } - } - - pub fn parse_meta_after_path(path: Path, input: ParseStream) -> Result { - if input.peek(token::Paren) { + pub(crate) fn parse_meta_after_path(path: Path, input: ParseStream) -> Result { + if input.peek(token::Paren) || input.peek(token::Bracket) || input.peek(token::Brace) { parse_meta_list_after_path(path, input).map(Meta::List) } else if input.peek(Token![=]) { parse_meta_name_value_after_path(path, input).map(Meta::NameValue) @@ -604,21 +682,60 @@ pub mod parsing { } fn parse_meta_list_after_path(path: Path, input: ParseStream) -> Result { - let content; + let (delimiter, tokens) = mac::parse_delimiter(input)?; Ok(MetaList { path, - paren_token: parenthesized!(content in input), - nested: content.parse_terminated(NestedMeta::parse)?, + delimiter, + tokens, }) } fn parse_meta_name_value_after_path(path: Path, input: ParseStream) -> Result { + let eq_token: Token![=] = input.parse()?; + let ahead = input.fork(); + let lit: Option = ahead.parse()?; + let value = if let (Some(lit), true) = (lit, ahead.is_empty()) { + input.advance_to(&ahead); + Expr::Lit(ExprLit { + attrs: Vec::new(), + lit, + }) + } else if input.peek(Token![#]) && input.peek2(token::Bracket) { + return Err(input.error("unexpected attribute inside of attribute")); + } else { + input.parse()? + }; Ok(MetaNameValue { path, - eq_token: input.parse()?, - lit: input.parse()?, + eq_token, + value, }) } + + pub(super) struct DisplayAttrStyle<'a>(pub &'a AttrStyle); + + impl<'a> Display for DisplayAttrStyle<'a> { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str(match self.0 { + AttrStyle::Outer => "#", + AttrStyle::Inner(_) => "#!", + }) + } + } + + pub(super) struct DisplayPath<'a>(pub &'a Path); + + impl<'a> Display for DisplayPath<'a> { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + for (i, segment) in self.0.segments.iter().enumerate() { + if i > 0 || self.0.leading_colon.is_some() { + formatter.write_str("::")?; + } + write!(formatter, "{}", segment.ident)?; + } + Ok(()) + } + } } #[cfg(feature = "printing")] @@ -635,8 +752,7 @@ mod printing { b.to_tokens(tokens); } self.bracket_token.surround(tokens, |tokens| { - self.path.to_tokens(tokens); - self.tokens.to_tokens(tokens); + self.meta.to_tokens(tokens); }); } } @@ -645,9 +761,7 @@ mod printing { impl ToTokens for MetaList { fn to_tokens(&self, tokens: &mut TokenStream) { self.path.to_tokens(tokens); - self.paren_token.surround(tokens, |tokens| { - self.nested.to_tokens(tokens); - }); + self.delimiter.surround(tokens, self.tokens.clone()); } } @@ -656,7 +770,7 @@ mod printing { fn to_tokens(&self, tokens: &mut TokenStream) { self.path.to_tokens(tokens); self.eq_token.to_tokens(tokens); - self.lit.to_tokens(tokens); + self.value.to_tokens(tokens); } } } diff --git a/src/await.rs b/src/await.rs deleted file mode 100644 index 038c6a5d..00000000 --- a/src/await.rs +++ /dev/null @@ -1,2 +0,0 @@ -// See include!("await.rs") in token.rs. -export_token_macro! {[await]} diff --git a/src/bigint.rs b/src/bigint.rs index 5397d6be..66aaa937 100644 --- a/src/bigint.rs +++ b/src/bigint.rs @@ -1,16 +1,16 @@ use std::ops::{AddAssign, MulAssign}; // For implementing base10_digits() accessor on LitInt. -pub struct BigInt { +pub(crate) struct BigInt { digits: Vec, } impl BigInt { - pub fn new() -> Self { + pub(crate) fn new() -> Self { BigInt { digits: Vec::new() } } - pub fn to_string(&self) -> String { + pub(crate) fn to_string(&self) -> String { let mut repr = String::with_capacity(self.digits.len()); let mut has_nonzero = false; diff --git a/src/buffer.rs b/src/buffer.rs index 0d5cf30d..86dec46a 100644 --- a/src/buffer.rs +++ b/src/buffer.rs @@ -1,18 +1,12 @@ //! A stably addressed token buffer supporting efficient traversal based on a //! cheaply copyable cursor. -//! -//! *This module is available only if Syn is built with the `"parsing"` feature.* // This module is heavily commented as it contains most of the unsafe code in // Syn, and caution should be used when editing it. The public-facing interface // is 100% safe but the implementation is fragile internally. -#[cfg(all( - not(all(target_arch = "wasm32", any(target_os = "unknown", target_os = "wasi"))), - feature = "proc-macro" -))] -use crate::proc_macro as pm; use crate::Lifetime; +use proc_macro2::extra::DelimSpan; use proc_macro2::{Delimiter, Group, Ident, Literal, Punct, Spacing, Span, TokenStream, TokenTree}; use std::cmp::Ordering; use std::marker::PhantomData; @@ -33,8 +27,6 @@ enum Entry { /// A buffer that can be efficiently traversed multiple times, unlike /// `TokenStream` which requires a deep copy in order to traverse more than /// once. -/// -/// *This type is available only if Syn is built with the `"parsing"` feature.* pub struct TokenBuffer { // NOTE: Do not implement clone on this - while the current design could be // cloned, other designs which could be desirable may not be cloneable. @@ -63,14 +55,9 @@ impl TokenBuffer { /// Creates a `TokenBuffer` containing all the tokens from the input /// `proc_macro::TokenStream`. - /// - /// *This method is available only if Syn is built with both the `"parsing"` and - /// `"proc-macro"` features.* - #[cfg(all( - not(all(target_arch = "wasm32", any(target_os = "unknown", target_os = "wasi"))), - feature = "proc-macro" - ))] - pub fn new(stream: pm::TokenStream) -> Self { + #[cfg(feature = "proc-macro")] + #[cfg_attr(doc_cfg, doc(cfg(feature = "proc-macro")))] + pub fn new(stream: proc_macro::TokenStream) -> Self { Self::new2(stream.into()) } @@ -101,11 +88,6 @@ impl TokenBuffer { /// /// An empty `Cursor` can be created directly, or one may create a `TokenBuffer` /// object and get a cursor to its first token with `begin()`. -/// -/// Two cursors are equal if they have the same location in the same input -/// stream, and have the same scope. -/// -/// *This type is available only if Syn is built with the `"parsing"` feature.* pub struct Cursor<'a> { // The current entry which the `Cursor` is pointing at. ptr: *const Entry, @@ -146,11 +128,11 @@ impl<'a> Cursor<'a> { // past it, unless `ptr == scope`, which means that we're at the edge of // our cursor's scope. We should only have `ptr != scope` at the exit // from None-delimited groups entered with `ignore_none`. - while let Entry::End(_) = *ptr { + while let Entry::End(_) = unsafe { &*ptr } { if ptr == scope { break; } - ptr = ptr.add(1); + ptr = unsafe { ptr.add(1) }; } Cursor { @@ -172,7 +154,7 @@ impl<'a> Cursor<'a> { /// If the cursor is looking at an `Entry::Group`, the bumped cursor will /// point at the first token in the group (with the same scope end). unsafe fn bump_ignore_group(self) -> Cursor<'a> { - Cursor::create(self.ptr.offset(1), self.scope) + unsafe { Cursor::create(self.ptr.offset(1), self.scope) } } /// While the cursor is looking at a `None`-delimited group, move it to look @@ -199,7 +181,7 @@ impl<'a> Cursor<'a> { /// If the cursor is pointing at a `Group` with the given delimiter, returns /// a cursor into that group and one pointing to the next `TokenTree`. - pub fn group(mut self, delim: Delimiter) -> Option<(Cursor<'a>, Span, Cursor<'a>)> { + pub fn group(mut self, delim: Delimiter) -> Option<(Cursor<'a>, DelimSpan, Cursor<'a>)> { // If we're not trying to enter a none-delimited group, we want to // ignore them. We have to make sure to _not_ ignore them when we want // to enter them, of course. For obvious reasons. @@ -209,16 +191,40 @@ impl<'a> Cursor<'a> { if let Entry::Group(group, end_offset) = self.entry() { if group.delimiter() == delim { + let span = group.delim_span(); let end_of_group = unsafe { self.ptr.add(*end_offset) }; let inside_of_group = unsafe { Cursor::create(self.ptr.add(1), end_of_group) }; let after_group = unsafe { Cursor::create(end_of_group, self.scope) }; - return Some((inside_of_group, group.span(), after_group)); + return Some((inside_of_group, span, after_group)); } } None } + pub(crate) fn any_group(self) -> Option<(Cursor<'a>, Delimiter, DelimSpan, Cursor<'a>)> { + if let Entry::Group(group, end_offset) = self.entry() { + let delimiter = group.delimiter(); + let span = group.delim_span(); + let end_of_group = unsafe { self.ptr.add(*end_offset) }; + let inside_of_group = unsafe { Cursor::create(self.ptr.add(1), end_of_group) }; + let after_group = unsafe { Cursor::create(end_of_group, self.scope) }; + return Some((inside_of_group, delimiter, span, after_group)); + } + + None + } + + pub(crate) fn any_group_token(self) -> Option<(Group, Cursor<'a>)> { + if let Entry::Group(group, end_offset) = self.entry() { + let end_of_group = unsafe { self.ptr.add(*end_offset) }; + let after_group = unsafe { Cursor::create(end_of_group, self.scope) }; + return Some((group.clone(), after_group)); + } + + None + } + /// If the cursor is pointing at a `Ident`, returns it along with a cursor /// pointing at the next `TokenTree`. pub fn ident(mut self) -> Option<(Ident, Cursor<'a>)> { @@ -313,6 +319,33 @@ impl<'a> Cursor<'a> { } } + /// Returns the `Span` of the token immediately prior to the position of + /// this cursor, or of the current token if there is no previous one. + #[cfg(any(feature = "full", feature = "derive"))] + pub(crate) fn prev_span(mut self) -> Span { + if start_of_buffer(self) < self.ptr { + self.ptr = unsafe { self.ptr.offset(-1) }; + if let Entry::End(_) = self.entry() { + // Locate the matching Group begin token. + let mut depth = 1; + loop { + self.ptr = unsafe { self.ptr.offset(-1) }; + match self.entry() { + Entry::Group(group, _) => { + depth -= 1; + if depth == 0 { + return group.span(); + } + } + Entry::End(_) => depth += 1, + Entry::Literal(_) | Entry::Ident(_) | Entry::Punct(_) => {} + } + } + } + } + self.span() + } + /// Skip over the next token without cloning it. Returns `None` if this /// cursor points to eof. /// @@ -356,7 +389,7 @@ impl<'a> PartialEq for Cursor<'a> { impl<'a> PartialOrd for Cursor<'a> { fn partial_cmp(&self, other: &Self) -> Option { if same_buffer(*self, *other) { - Some(self.ptr.cmp(&other.ptr)) + Some(cmp_assuming_same_buffer(*self, *other)) } else { None } @@ -368,17 +401,18 @@ pub(crate) fn same_scope(a: Cursor, b: Cursor) -> bool { } pub(crate) fn same_buffer(a: Cursor, b: Cursor) -> bool { + start_of_buffer(a) == start_of_buffer(b) +} + +fn start_of_buffer(cursor: Cursor) -> *const Entry { unsafe { - match (&*a.scope, &*b.scope) { - (Entry::End(a_offset), Entry::End(b_offset)) => { - a.scope.offset(*a_offset) == b.scope.offset(*b_offset) - } + match &*cursor.scope { + Entry::End(offset) => cursor.scope.offset(*offset), _ => unreachable!(), } } } -#[cfg(any(feature = "full", feature = "derive"))] pub(crate) fn cmp_assuming_same_buffer(a: Cursor, b: Cursor) -> Ordering { a.ptr.cmp(&b.ptr) } diff --git a/src/custom_keyword.rs b/src/custom_keyword.rs index a3ec9d4c..6ce23db4 100644 --- a/src/custom_keyword.rs +++ b/src/custom_keyword.rs @@ -96,26 +96,28 @@ macro_rules! custom_keyword { #[doc(hidden)] #[allow(dead_code, non_snake_case)] - pub fn $ident<__S: $crate::__private::IntoSpans<[$crate::__private::Span; 1]>>( + pub fn $ident<__S: $crate::__private::IntoSpans<$crate::__private::Span>>( span: __S, ) -> $ident { $ident { - span: $crate::__private::IntoSpans::into_spans(span)[0], + span: $crate::__private::IntoSpans::into_spans(span), } } - impl $crate::__private::Default for $ident { - fn default() -> Self { - $ident { - span: $crate::__private::Span::call_site(), + const _: () = { + impl $crate::__private::Default for $ident { + fn default() -> Self { + $ident { + span: $crate::__private::Span::call_site(), + } } } - } - $crate::impl_parse_for_custom_keyword!($ident); - $crate::impl_to_tokens_for_custom_keyword!($ident); - $crate::impl_clone_for_custom_keyword!($ident); - $crate::impl_extra_traits_for_custom_keyword!($ident); + $crate::impl_parse_for_custom_keyword!($ident); + $crate::impl_to_tokens_for_custom_keyword!($ident); + $crate::impl_clone_for_custom_keyword!($ident); + $crate::impl_extra_traits_for_custom_keyword!($ident); + }; }; } @@ -126,17 +128,17 @@ macro_rules! custom_keyword { macro_rules! impl_parse_for_custom_keyword { ($ident:ident) => { // For peek. - impl $crate::token::CustomToken for $ident { + impl $crate::__private::CustomToken for $ident { fn peek(cursor: $crate::buffer::Cursor) -> $crate::__private::bool { if let $crate::__private::Some((ident, _rest)) = cursor.ident() { - ident == stringify!($ident) + ident == $crate::__private::stringify!($ident) } else { false } } fn display() -> &'static $crate::__private::str { - concat!("`", stringify!($ident), "`") + $crate::__private::concat!("`", $crate::__private::stringify!($ident), "`") } } @@ -144,14 +146,14 @@ macro_rules! impl_parse_for_custom_keyword { fn parse(input: $crate::parse::ParseStream) -> $crate::parse::Result<$ident> { input.step(|cursor| { if let $crate::__private::Some((ident, rest)) = cursor.ident() { - if ident == stringify!($ident) { + if ident == $crate::__private::stringify!($ident) { return $crate::__private::Ok(($ident { span: ident.span() }, rest)); } } - $crate::__private::Err(cursor.error(concat!( + $crate::__private::Err(cursor.error($crate::__private::concat!( "expected `", - stringify!($ident), - "`" + $crate::__private::stringify!($ident), + "`", ))) }) } @@ -175,7 +177,7 @@ macro_rules! impl_to_tokens_for_custom_keyword { ($ident:ident) => { impl $crate::__private::ToTokens for $ident { fn to_tokens(&self, tokens: &mut $crate::__private::TokenStream2) { - let ident = $crate::Ident::new(stringify!($ident), self.span); + let ident = $crate::Ident::new($crate::__private::stringify!($ident), self.span); $crate::__private::TokenStreamExt::append(tokens, ident); } } @@ -222,10 +224,14 @@ macro_rules! impl_clone_for_custom_keyword { macro_rules! impl_extra_traits_for_custom_keyword { ($ident:ident) => { impl $crate::__private::Debug for $ident { - fn fmt(&self, f: &mut $crate::__private::Formatter) -> $crate::__private::fmt::Result { + fn fmt(&self, f: &mut $crate::__private::Formatter) -> $crate::__private::FmtResult { $crate::__private::Formatter::write_str( f, - concat!("Keyword [", stringify!($ident), "]"), + $crate::__private::concat!( + "Keyword [", + $crate::__private::stringify!($ident), + "]", + ), ) } } diff --git a/src/custom_punctuation.rs b/src/custom_punctuation.rs index 118a8453..1b2c768f 100644 --- a/src/custom_punctuation.rs +++ b/src/custom_punctuation.rs @@ -92,16 +92,18 @@ macro_rules! custom_punctuation { } } - impl $crate::__private::Default for $ident { - fn default() -> Self { - $ident($crate::__private::Span::call_site()) + const _: () = { + impl $crate::__private::Default for $ident { + fn default() -> Self { + $ident($crate::__private::Span::call_site()) + } } - } - $crate::impl_parse_for_custom_punctuation!($ident, $($tt)+); - $crate::impl_to_tokens_for_custom_punctuation!($ident, $($tt)+); - $crate::impl_clone_for_custom_punctuation!($ident, $($tt)+); - $crate::impl_extra_traits_for_custom_punctuation!($ident, $($tt)+); + $crate::impl_parse_for_custom_punctuation!($ident, $($tt)+); + $crate::impl_to_tokens_for_custom_punctuation!($ident, $($tt)+); + $crate::impl_clone_for_custom_punctuation!($ident, $($tt)+); + $crate::impl_extra_traits_for_custom_punctuation!($ident, $($tt)+); + }; }; } @@ -111,20 +113,20 @@ macro_rules! custom_punctuation { #[macro_export] macro_rules! impl_parse_for_custom_punctuation { ($ident:ident, $($tt:tt)+) => { - impl $crate::token::CustomToken for $ident { - fn peek(cursor: $crate::buffer::Cursor) -> bool { - $crate::token::parsing::peek_punct(cursor, $crate::stringify_punct!($($tt)+)) + impl $crate::__private::CustomToken for $ident { + fn peek(cursor: $crate::buffer::Cursor) -> $crate::__private::bool { + $crate::__private::peek_punct(cursor, $crate::stringify_punct!($($tt)+)) } fn display() -> &'static $crate::__private::str { - concat!("`", $crate::stringify_punct!($($tt)+), "`") + $crate::__private::concat!("`", $crate::stringify_punct!($($tt)+), "`") } } impl $crate::parse::Parse for $ident { fn parse(input: $crate::parse::ParseStream) -> $crate::parse::Result<$ident> { let spans: $crate::custom_punctuation_repr!($($tt)+) = - $crate::token::parsing::punct(input, $crate::stringify_punct!($($tt)+))?; + $crate::__private::parse_punct(input, $crate::stringify_punct!($($tt)+))?; Ok($ident(spans)) } } @@ -147,7 +149,7 @@ macro_rules! impl_to_tokens_for_custom_punctuation { ($ident:ident, $($tt:tt)+) => { impl $crate::__private::ToTokens for $ident { fn to_tokens(&self, tokens: &mut $crate::__private::TokenStream2) { - $crate::token::printing::punct($crate::stringify_punct!($($tt)+), &self.spans, tokens) + $crate::__private::print_punct($crate::stringify_punct!($($tt)+), &self.spans, tokens) } } }; @@ -193,8 +195,8 @@ macro_rules! impl_clone_for_custom_punctuation { macro_rules! impl_extra_traits_for_custom_punctuation { ($ident:ident, $($tt:tt)+) => { impl $crate::__private::Debug for $ident { - fn fmt(&self, f: &mut $crate::__private::Formatter) -> $crate::__private::fmt::Result { - $crate::__private::Formatter::write_str(f, stringify!($ident)) + fn fmt(&self, f: &mut $crate::__private::Formatter) -> $crate::__private::FmtResult { + $crate::__private::Formatter::write_str(f, $crate::__private::stringify!($ident)) } } @@ -295,6 +297,6 @@ macro_rules! custom_punctuation_unexpected { #[macro_export] macro_rules! stringify_punct { ($($tt:tt)+) => { - concat!($(stringify!($tt)),+) + $crate::__private::concat!($($crate::__private::stringify!($tt)),+) }; } diff --git a/src/data.rs b/src/data.rs index 3b466618..134b76bb 100644 --- a/src/data.rs +++ b/src/data.rs @@ -3,12 +3,8 @@ use crate::punctuated::Punctuated; ast_struct! { /// An enum variant. - /// - /// *This type is available only if Syn is built with the `"derive"` or `"full"` - /// feature.* #[cfg_attr(doc_cfg, doc(cfg(any(feature = "full", feature = "derive"))))] pub struct Variant { - /// Attributes tagged on the variant. pub attrs: Vec, /// Name of the variant. @@ -25,9 +21,6 @@ ast_struct! { ast_enum_of_structs! { /// Data stored within an enum variant or struct. /// - /// *This type is available only if Syn is built with the `"derive"` or `"full"` - /// feature.* - /// /// # Syntax tree enum /// /// This type is a [syntax tree enum]. @@ -50,9 +43,6 @@ ast_enum_of_structs! { ast_struct! { /// Named fields of a struct or struct variant such as `Point { x: f64, /// y: f64 }`. - /// - /// *This type is available only if Syn is built with the `"derive"` or - /// `"full"` feature.* #[cfg_attr(doc_cfg, doc(cfg(any(feature = "full", feature = "derive"))))] pub struct FieldsNamed { pub brace_token: token::Brace, @@ -62,9 +52,6 @@ ast_struct! { ast_struct! { /// Unnamed fields of a tuple struct or tuple variant such as `Some(T)`. - /// - /// *This type is available only if Syn is built with the `"derive"` or - /// `"full"` feature.* #[cfg_attr(doc_cfg, doc(cfg(any(feature = "full", feature = "derive"))))] pub struct FieldsUnnamed { pub paren_token: token::Paren, @@ -147,17 +134,14 @@ impl<'a> IntoIterator for &'a mut Fields { ast_struct! { /// A field of a struct or enum variant. - /// - /// *This type is available only if Syn is built with the `"derive"` or `"full"` - /// feature.* #[cfg_attr(doc_cfg, doc(cfg(any(feature = "full", feature = "derive"))))] pub struct Field { - /// Attributes tagged on the field. pub attrs: Vec, - /// Visibility of the field. pub vis: Visibility, + pub mutability: FieldMutability, + /// Name of the field, if any. /// /// Fields of tuple structs have no names. @@ -165,82 +149,16 @@ ast_struct! { pub colon_token: Option, - /// Type of the field. pub ty: Type, } } -ast_enum_of_structs! { - /// The visibility level of an item: inherited or `pub` or - /// `pub(restricted)`. - /// - /// *This type is available only if Syn is built with the `"derive"` or `"full"` - /// feature.* - /// - /// # Syntax tree enum - /// - /// This type is a [syntax tree enum]. - /// - /// [syntax tree enum]: Expr#syntax-tree-enums - #[cfg_attr(doc_cfg, doc(cfg(any(feature = "full", feature = "derive"))))] - pub enum Visibility { - /// A public visibility level: `pub`. - Public(VisPublic), - - /// A crate-level visibility: `crate`. - Crate(VisCrate), - - /// A visibility level restricted to some path: `pub(self)` or - /// `pub(super)` or `pub(crate)` or `pub(in some::module)`. - Restricted(VisRestricted), - - /// An inherited visibility, which usually means private. - Inherited, - } -} - -ast_struct! { - /// A public visibility level: `pub`. - /// - /// *This type is available only if Syn is built with the `"derive"` or - /// `"full"` feature.* - #[cfg_attr(doc_cfg, doc(cfg(any(feature = "full", feature = "derive"))))] - pub struct VisPublic { - pub pub_token: Token![pub], - } -} - -ast_struct! { - /// A crate-level visibility: `crate`. - /// - /// *This type is available only if Syn is built with the `"derive"` or - /// `"full"` feature.* - #[cfg_attr(doc_cfg, doc(cfg(any(feature = "full", feature = "derive"))))] - pub struct VisCrate { - pub crate_token: Token![crate], - } -} - -ast_struct! { - /// A visibility level restricted to some path: `pub(self)` or - /// `pub(super)` or `pub(crate)` or `pub(in some::module)`. - /// - /// *This type is available only if Syn is built with the `"derive"` or - /// `"full"` feature.* - #[cfg_attr(doc_cfg, doc(cfg(any(feature = "full", feature = "derive"))))] - pub struct VisRestricted { - pub pub_token: Token![pub], - pub paren_token: token::Paren, - pub in_token: Option, - pub path: Box, - } -} - #[cfg(feature = "parsing")] -pub mod parsing { +pub(crate) mod parsing { use super::*; - use crate::ext::IdentExt; - use crate::parse::discouraged::Speculative; + use crate::ext::IdentExt as _; + #[cfg(not(feature = "full"))] + use crate::parse::discouraged::Speculative as _; use crate::parse::{Parse, ParseStream, Result}; #[cfg_attr(doc_cfg, doc(cfg(feature = "parsing")))] @@ -258,7 +176,20 @@ pub mod parsing { }; let discriminant = if input.peek(Token![=]) { let eq_token: Token![=] = input.parse()?; + #[cfg(feature = "full")] let discriminant: Expr = input.parse()?; + #[cfg(not(feature = "full"))] + let discriminant = { + let begin = input.fork(); + let ahead = input.fork(); + let mut discriminant: Result = ahead.parse(); + if discriminant.is_ok() { + input.advance_to(&ahead); + } else if scan_lenient_discriminant(input).is_ok() { + discriminant = Ok(Expr::Verbatim(verbatim::between(&begin, input))); + } + discriminant? + }; Some((eq_token, discriminant)) } else { None @@ -272,13 +203,86 @@ pub mod parsing { } } + #[cfg(not(feature = "full"))] + pub(crate) fn scan_lenient_discriminant(input: ParseStream) -> Result<()> { + use proc_macro2::Delimiter::{self, Brace, Bracket, Parenthesis}; + + let consume = |delimiter: Delimiter| { + Result::unwrap(input.step(|cursor| match cursor.group(delimiter) { + Some((_inside, _span, rest)) => Ok((true, rest)), + None => Ok((false, *cursor)), + })) + }; + + macro_rules! consume { + [$token:tt] => { + input.parse::>().unwrap().is_some() + }; + } + + let mut initial = true; + let mut depth = 0usize; + loop { + if initial { + if consume![&] { + input.parse::>()?; + } else if consume![if] || consume![match] || consume![while] { + depth += 1; + } else if input.parse::>()?.is_some() + || (consume(Brace) || consume(Bracket) || consume(Parenthesis)) + || (consume![async] || consume![const] || consume![loop] || consume![unsafe]) + && (consume(Brace) || break) + { + initial = false; + } else if consume![let] { + while !consume![=] { + if !((consume![|] || consume![ref] || consume![mut] || consume![@]) + || (consume![!] || input.parse::>()?.is_some()) + || (consume![..=] || consume![..] || consume![&] || consume![_]) + || (consume(Brace) || consume(Bracket) || consume(Parenthesis))) + { + path::parsing::qpath(input, true)?; + } + } + } else if input.parse::>()?.is_some() && !consume![:] { + break; + } else if input.parse::().is_err() { + path::parsing::qpath(input, true)?; + initial = consume![!] || depth == 0 && input.peek(token::Brace); + } + } else if input.is_empty() || input.peek(Token![,]) { + return Ok(()); + } else if depth > 0 && consume(Brace) { + if consume![else] && !consume(Brace) { + initial = consume![if] || break; + } else { + depth -= 1; + } + } else if input.parse::().is_ok() || (consume![..] | consume![=]) { + initial = true; + } else if consume![.] { + if input.parse::>()?.is_none() + && (input.parse::()?.is_named() && consume![::]) + { + AngleBracketedGenericArguments::do_parse(None, input)?; + } + } else if consume![as] { + input.parse::()?; + } else if !(consume(Brace) || consume(Bracket) || consume(Parenthesis)) { + break; + } + } + + Err(input.error("unsupported expression")) + } + #[cfg_attr(doc_cfg, doc(cfg(feature = "parsing")))] impl Parse for FieldsNamed { fn parse(input: ParseStream) -> Result { let content; Ok(FieldsNamed { brace_token: braced!(content in input), - named: content.parse_terminated(Field::parse_named)?, + named: content.parse_terminated(Field::parse_named, Token![,])?, }) } } @@ -289,7 +293,7 @@ pub mod parsing { let content; Ok(FieldsUnnamed { paren_token: parenthesized!(content in input), - unnamed: content.parse_terminated(Field::parse_unnamed)?, + unnamed: content.parse_terminated(Field::parse_unnamed, Token![,])?, }) } } @@ -298,16 +302,37 @@ pub mod parsing { /// Parses a named (braced struct) field. #[cfg_attr(doc_cfg, doc(cfg(feature = "parsing")))] pub fn parse_named(input: ParseStream) -> Result { + let attrs = input.call(Attribute::parse_outer)?; + let vis: Visibility = input.parse()?; + + let unnamed_field = cfg!(feature = "full") && input.peek(Token![_]); + let ident = if unnamed_field { + input.call(Ident::parse_any) + } else { + input.parse() + }?; + + let colon_token: Token![:] = input.parse()?; + + let ty: Type = if unnamed_field + && (input.peek(Token![struct]) + || input.peek(Token![union]) && input.peek2(token::Brace)) + { + let begin = input.fork(); + input.call(Ident::parse_any)?; + input.parse::()?; + Type::Verbatim(verbatim::between(&begin, input)) + } else { + input.parse()? + }; + Ok(Field { - attrs: input.call(Attribute::parse_outer)?, - vis: input.parse()?, - ident: Some(if input.peek(Token![_]) { - input.call(Ident::parse_any) - } else { - input.parse() - }?), - colon_token: Some(input.parse()?), - ty: input.parse()?, + attrs, + vis, + mutability: FieldMutability::None, + ident: Some(ident), + colon_token: Some(colon_token), + ty, }) } @@ -317,100 +342,13 @@ pub mod parsing { Ok(Field { attrs: input.call(Attribute::parse_outer)?, vis: input.parse()?, + mutability: FieldMutability::None, ident: None, colon_token: None, ty: input.parse()?, }) } } - - #[cfg_attr(doc_cfg, doc(cfg(feature = "parsing")))] - impl Parse for Visibility { - fn parse(input: ParseStream) -> Result { - // Recognize an empty None-delimited group, as produced by a $:vis - // matcher that matched no tokens. - if input.peek(token::Group) { - let ahead = input.fork(); - let group = crate::group::parse_group(&ahead)?; - if group.content.is_empty() { - input.advance_to(&ahead); - return Ok(Visibility::Inherited); - } - } - - if input.peek(Token![pub]) { - Self::parse_pub(input) - } else if input.peek(Token![crate]) { - Self::parse_crate(input) - } else { - Ok(Visibility::Inherited) - } - } - } - - impl Visibility { - fn parse_pub(input: ParseStream) -> Result { - let pub_token = input.parse::()?; - - if input.peek(token::Paren) { - let ahead = input.fork(); - - let content; - let paren_token = parenthesized!(content in ahead); - if content.peek(Token![crate]) - || content.peek(Token![self]) - || content.peek(Token![super]) - { - let path = content.call(Ident::parse_any)?; - - // Ensure there are no additional tokens within `content`. - // Without explicitly checking, we may misinterpret a tuple - // field as a restricted visibility, causing a parse error. - // e.g. `pub (crate::A, crate::B)` (Issue #720). - if content.is_empty() { - input.advance_to(&ahead); - return Ok(Visibility::Restricted(VisRestricted { - pub_token, - paren_token, - in_token: None, - path: Box::new(Path::from(path)), - })); - } - } else if content.peek(Token![in]) { - let in_token: Token![in] = content.parse()?; - let path = content.call(Path::parse_mod_style)?; - - input.advance_to(&ahead); - return Ok(Visibility::Restricted(VisRestricted { - pub_token, - paren_token, - in_token: Some(in_token), - path: Box::new(path), - })); - } - } - - Ok(Visibility::Public(VisPublic { pub_token })) - } - - fn parse_crate(input: ParseStream) -> Result { - if input.peek2(Token![::]) { - Ok(Visibility::Inherited) - } else { - Ok(Visibility::Crate(VisCrate { - crate_token: input.parse()?, - })) - } - } - - #[cfg(feature = "full")] - pub(crate) fn is_some(&self) -> bool { - match self { - Visibility::Inherited => false, - _ => true, - } - } - } } #[cfg(feature = "printing")] @@ -463,31 +401,4 @@ mod printing { self.ty.to_tokens(tokens); } } - - #[cfg_attr(doc_cfg, doc(cfg(feature = "printing")))] - impl ToTokens for VisPublic { - fn to_tokens(&self, tokens: &mut TokenStream) { - self.pub_token.to_tokens(tokens); - } - } - - #[cfg_attr(doc_cfg, doc(cfg(feature = "printing")))] - impl ToTokens for VisCrate { - fn to_tokens(&self, tokens: &mut TokenStream) { - self.crate_token.to_tokens(tokens); - } - } - - #[cfg_attr(doc_cfg, doc(cfg(feature = "printing")))] - impl ToTokens for VisRestricted { - fn to_tokens(&self, tokens: &mut TokenStream) { - self.pub_token.to_tokens(tokens); - self.paren_token.surround(tokens, |tokens| { - // TODO: If we have a path which is not "self" or "super" or - // "crate", automatically add the "in" token. - self.in_token.to_tokens(tokens); - self.path.to_tokens(tokens); - }); - } - } } diff --git a/src/derive.rs b/src/derive.rs index af9bb91b..25fa4c91 100644 --- a/src/derive.rs +++ b/src/derive.rs @@ -3,32 +3,19 @@ use crate::punctuated::Punctuated; ast_struct! { /// Data structure sent to a `proc_macro_derive` macro. - /// - /// *This type is available only if Syn is built with the `"derive"` feature.* #[cfg_attr(doc_cfg, doc(cfg(feature = "derive")))] pub struct DeriveInput { - /// Attributes tagged on the whole struct or enum. pub attrs: Vec, - - /// Visibility of the struct or enum. pub vis: Visibility, - - /// Name of the struct or enum. pub ident: Ident, - - /// Generics required to complete the definition. pub generics: Generics, - - /// Data within the struct or enum. pub data: Data, } } -ast_enum_of_structs! { +ast_enum! { /// The storage of a struct, enum or union data structure. /// - /// *This type is available only if Syn is built with the `"derive"` feature.* - /// /// # Syntax tree enum /// /// This type is a [syntax tree enum]. @@ -36,24 +23,14 @@ ast_enum_of_structs! { /// [syntax tree enum]: Expr#syntax-tree-enums #[cfg_attr(doc_cfg, doc(cfg(feature = "derive")))] pub enum Data { - /// A struct input to a `proc_macro_derive` macro. Struct(DataStruct), - - /// An enum input to a `proc_macro_derive` macro. Enum(DataEnum), - - /// An untagged union input to a `proc_macro_derive` macro. Union(DataUnion), } - - do_not_generate_to_tokens } ast_struct! { /// A struct input to a `proc_macro_derive` macro. - /// - /// *This type is available only if Syn is built with the `"derive"` - /// feature.* #[cfg_attr(doc_cfg, doc(cfg(feature = "derive")))] pub struct DataStruct { pub struct_token: Token![struct], @@ -64,9 +41,6 @@ ast_struct! { ast_struct! { /// An enum input to a `proc_macro_derive` macro. - /// - /// *This type is available only if Syn is built with the `"derive"` - /// feature.* #[cfg_attr(doc_cfg, doc(cfg(feature = "derive")))] pub struct DataEnum { pub enum_token: Token![enum], @@ -77,9 +51,6 @@ ast_struct! { ast_struct! { /// An untagged union input to a `proc_macro_derive` macro. - /// - /// *This type is available only if Syn is built with the `"derive"` - /// feature.* #[cfg_attr(doc_cfg, doc(cfg(feature = "derive")))] pub struct DataUnion { pub union_token: Token![union], @@ -88,7 +59,7 @@ ast_struct! { } #[cfg(feature = "parsing")] -pub mod parsing { +pub(crate) mod parsing { use super::*; use crate::parse::{Parse, ParseStream, Result}; @@ -161,7 +132,7 @@ pub mod parsing { } } - pub fn data_struct( + pub(crate) fn data_struct( input: ParseStream, ) -> Result<(Option, Fields, Option)> { let mut lookahead = input.lookahead1(); @@ -197,7 +168,7 @@ pub mod parsing { } } - pub fn data_enum( + pub(crate) fn data_enum( input: ParseStream, ) -> Result<( Option, @@ -208,12 +179,12 @@ pub mod parsing { let content; let brace = braced!(content in input); - let variants = content.parse_terminated(Variant::parse)?; + let variants = content.parse_terminated(Variant::parse, Token![,])?; Ok((where_clause, brace, variants)) } - pub fn data_union(input: ParseStream) -> Result<(Option, FieldsNamed)> { + pub(crate) fn data_union(input: ParseStream) -> Result<(Option, FieldsNamed)> { let where_clause = input.parse()?; let fields = input.parse()?; Ok((where_clause, fields)) diff --git a/src/discouraged.rs b/src/discouraged.rs index a46129b6..fb98d633 100644 --- a/src/discouraged.rs +++ b/src/discouraged.rs @@ -1,6 +1,7 @@ //! Extensions to the parsing API with niche applicability. use super::*; +use proc_macro2::extra::DelimSpan; /// Extensions to the `ParseStream` API to support speculative parsing. pub trait Speculative { @@ -192,3 +193,27 @@ impl<'a> Speculative for ParseBuffer<'a> { .set(unsafe { mem::transmute::>(fork.cursor()) }); } } + +/// Extensions to the `ParseStream` API to support manipulating invisible +/// delimiters the same as if they were visible. +pub trait AnyDelimiter { + /// Returns the delimiter, the span of the delimiter token, and the nested + /// contents for further parsing. + fn parse_any_delimiter(&self) -> Result<(Delimiter, DelimSpan, ParseBuffer)>; +} + +impl<'a> AnyDelimiter for ParseBuffer<'a> { + fn parse_any_delimiter(&self) -> Result<(Delimiter, DelimSpan, ParseBuffer)> { + self.step(|cursor| { + if let Some((content, delimiter, span, rest)) = cursor.any_group() { + let scope = crate::buffer::close_span_of_group(*cursor); + let nested = crate::parse::advance_step_cursor(cursor, content); + let unexpected = crate::parse::get_unexpected(self); + let content = crate::parse::new_parse_buffer(scope, nested, unexpected); + Ok(((delimiter, span, content), rest)) + } else { + Err(cursor.error("expected any delimiter")) + } + }) + } +} diff --git a/src/error.rs b/src/error.rs index e301367d..71247cde 100644 --- a/src/error.rs +++ b/src/error.rs @@ -7,7 +7,6 @@ use proc_macro2::{ #[cfg(feature = "printing")] use quote::ToTokens; use std::fmt::{self, Debug, Display}; -use std::iter::FromIterator; use std::slice; use std::vec; @@ -34,18 +33,34 @@ pub type Result = std::result::Result; /// # extern crate proc_macro; /// # /// use proc_macro::TokenStream; -/// use syn::{parse_macro_input, AttributeArgs, ItemFn}; +/// use syn::parse::{Parse, ParseStream, Result}; +/// use syn::{parse_macro_input, ItemFn}; /// /// # const IGNORE: &str = stringify! { /// #[proc_macro_attribute] /// # }; /// pub fn my_attr(args: TokenStream, input: TokenStream) -> TokenStream { -/// let args = parse_macro_input!(args as AttributeArgs); +/// let args = parse_macro_input!(args as MyAttrArgs); /// let input = parse_macro_input!(input as ItemFn); /// /// /* ... */ /// # TokenStream::new() /// } +/// +/// struct MyAttrArgs { +/// # _k: [(); { stringify! { +/// ... +/// # }; 0 }] +/// } +/// +/// impl Parse for MyAttrArgs { +/// fn parse(input: ParseStream) -> Result { +/// # stringify! { +/// ... +/// # }; +/// # unimplemented!() +/// } +/// } /// ``` /// /// For errors that arise later than the initial parsing stage, the @@ -89,14 +104,21 @@ pub struct Error { struct ErrorMessage { // Span is implemented as an index into a thread-local interner to keep the // size small. It is not safe to access from a different thread. We want - // errors to be Send and Sync to play nicely with the Failure crate, so pin - // the span we're given to its original thread and assume it is - // Span::call_site if accessed from any other thread. - start_span: ThreadBound, - end_span: ThreadBound, + // errors to be Send and Sync to play nicely with ecosystem crates for error + // handling, so pin the span we're given to its original thread and assume + // it is Span::call_site if accessed from any other thread. + span: ThreadBound, message: String, } +// Cannot use std::ops::Range because that does not implement Copy, +// whereas ThreadBound requires a Copy impl as a way to ensure no Drop impls +// are involved. +struct SpanRange { + start: Span, + end: Span, +} + #[cfg(test)] struct _Test where @@ -139,8 +161,10 @@ impl Error { fn new(span: Span, message: String) -> Error { Error { messages: vec![ErrorMessage { - start_span: ThreadBound::new(span), - end_span: ThreadBound::new(span), + span: ThreadBound::new(SpanRange { + start: span, + end: span, + }), message, }], } @@ -161,6 +185,7 @@ impl Error { /// When in doubt it's recommended to stick to `Error::new` (or /// `ParseStream::error`)! #[cfg(feature = "printing")] + #[cfg_attr(doc_cfg, doc(cfg(feature = "printing")))] pub fn new_spanned(tokens: T, message: U) -> Self { return new_spanned(tokens.into_token_stream(), message.to_string()); @@ -170,8 +195,7 @@ impl Error { let end = iter.last().map_or(start, |t| t.span()); Error { messages: vec![ErrorMessage { - start_span: ThreadBound::new(start), - end_span: ThreadBound::new(end), + span: ThreadBound::new(SpanRange { start, end }), message, }], } @@ -184,11 +208,7 @@ impl Error { /// if called from a different thread than the one on which the `Error` was /// originally created. pub fn span(&self) -> Span { - let start = match self.messages[0].start_span.get() { - Some(span) => *span, - None => return Span::call_site(), - }; - let end = match self.messages[0].end_span.get() { + let SpanRange { start, end } = match self.messages[0].span.get() { Some(span) => *span, None => return Span::call_site(), }; @@ -254,15 +274,34 @@ impl Error { impl ErrorMessage { fn to_compile_error(&self) -> TokenStream { - let start = self - .start_span - .get() - .cloned() - .unwrap_or_else(Span::call_site); - let end = self.end_span.get().cloned().unwrap_or_else(Span::call_site); + let (start, end) = match self.span.get() { + Some(range) => (range.start, range.end), + None => (Span::call_site(), Span::call_site()), + }; - // compile_error!($message) + // ::core::compile_error!($message) TokenStream::from_iter(vec![ + TokenTree::Punct({ + let mut punct = Punct::new(':', Spacing::Joint); + punct.set_span(start); + punct + }), + TokenTree::Punct({ + let mut punct = Punct::new(':', Spacing::Alone); + punct.set_span(start); + punct + }), + TokenTree::Ident(Ident::new("core", start)), + TokenTree::Punct({ + let mut punct = Punct::new(':', Spacing::Joint); + punct.set_span(start); + punct + }), + TokenTree::Punct({ + let mut punct = Punct::new(':', Spacing::Alone); + punct.set_span(start); + punct + }), TokenTree::Ident(Ident::new("compile_error", start)), TokenTree::Punct({ let mut punct = Punct::new('!', Spacing::Alone); @@ -285,7 +324,7 @@ impl ErrorMessage { } #[cfg(feature = "parsing")] -pub fn new_at(scope: Span, cursor: Cursor, message: T) -> Error { +pub(crate) fn new_at(scope: Span, cursor: Cursor, message: T) -> Error { if cursor.eof() { Error::new(scope, format!("unexpected end of input, {}", message)) } else { @@ -295,14 +334,13 @@ pub fn new_at(scope: Span, cursor: Cursor, message: T) -> Error { } #[cfg(all(feature = "parsing", any(feature = "full", feature = "derive")))] -pub fn new2(start: Span, end: Span, message: T) -> Error { +pub(crate) fn new2(start: Span, end: Span, message: T) -> Error { return new2(start, end, message.to_string()); fn new2(start: Span, end: Span, message: String) -> Error { Error { messages: vec![ErrorMessage { - start_span: ThreadBound::new(start), - end_span: ThreadBound::new(end), + span: ThreadBound::new(SpanRange { start, end }), message, }], } @@ -347,25 +385,26 @@ impl Clone for Error { impl Clone for ErrorMessage { fn clone(&self) -> Self { - let start = self - .start_span - .get() - .cloned() - .unwrap_or_else(Span::call_site); - let end = self.end_span.get().cloned().unwrap_or_else(Span::call_site); ErrorMessage { - start_span: ThreadBound::new(start), - end_span: ThreadBound::new(end), + span: self.span, message: self.message.clone(), } } } +impl Clone for SpanRange { + fn clone(&self) -> Self { + *self + } +} + +impl Copy for SpanRange {} + impl std::error::Error for Error {} impl From for Error { fn from(err: LexError) -> Self { - Error::new(err.span(), "lex error") + Error::new(err.span(), err) } } diff --git a/src/export.rs b/src/export.rs index f478d091..b9ea5c74 100644 --- a/src/export.rs +++ b/src/export.rs @@ -1,39 +1,73 @@ +#[doc(hidden)] pub use std::clone::Clone; +#[doc(hidden)] pub use std::cmp::{Eq, PartialEq}; +#[doc(hidden)] +pub use std::concat; +#[doc(hidden)] pub use std::default::Default; -pub use std::fmt::{self, Debug, Formatter}; +#[doc(hidden)] +pub use std::fmt::Debug; +#[doc(hidden)] pub use std::hash::{Hash, Hasher}; +#[doc(hidden)] pub use std::marker::Copy; +#[doc(hidden)] pub use std::option::Option::{None, Some}; +#[doc(hidden)] pub use std::result::Result::{Err, Ok}; +#[doc(hidden)] +pub use std::stringify; + +#[doc(hidden)] +pub type Formatter<'a> = std::fmt::Formatter<'a>; +#[doc(hidden)] +pub type FmtResult = std::fmt::Result; + +#[doc(hidden)] +pub type bool = std::primitive::bool; +#[doc(hidden)] +pub type str = std::primitive::str; #[cfg(feature = "printing")] -pub extern crate quote; +#[doc(hidden)] +pub use quote; -pub use proc_macro2::{Span, TokenStream as TokenStream2}; +#[doc(hidden)] +pub type Span = proc_macro2::Span; +#[doc(hidden)] +pub type TokenStream2 = proc_macro2::TokenStream; #[cfg(feature = "parsing")] +#[doc(hidden)] pub use crate::group::{parse_braces, parse_brackets, parse_parens}; +#[doc(hidden)] pub use crate::span::IntoSpans; -#[cfg(all( - not(all(target_arch = "wasm32", any(target_os = "unknown", target_os = "wasi"))), - feature = "proc-macro" -))] -pub use proc_macro::TokenStream; +#[cfg(all(feature = "parsing", feature = "printing"))] +#[doc(hidden)] +pub use crate::parse_quote::parse as parse_quote; + +#[cfg(feature = "parsing")] +#[doc(hidden)] +pub use crate::token::parsing::{peek_punct, punct as parse_punct}; #[cfg(feature = "printing")] +#[doc(hidden)] +pub use crate::token::printing::punct as print_punct; + +#[cfg(feature = "parsing")] +#[doc(hidden)] +pub use crate::token::private::CustomToken; + +#[cfg(feature = "proc-macro")] +#[doc(hidden)] +pub type TokenStream = proc_macro::TokenStream; + +#[cfg(feature = "printing")] +#[doc(hidden)] pub use quote::{ToTokens, TokenStreamExt}; -#[allow(non_camel_case_types)] -pub type bool = help::Bool; -#[allow(non_camel_case_types)] -pub type str = help::Str; - -mod help { - pub type Bool = bool; - pub type Str = str; -} - +#[doc(hidden)] pub struct private(pub(crate) ()); diff --git a/src/expr.rs b/src/expr.rs index 93a59b0e..7fb0f7b4 100644 --- a/src/expr.rs +++ b/src/expr.rs @@ -1,7 +1,5 @@ use super::*; use crate::punctuated::Punctuated; -#[cfg(feature = "full")] -use crate::reserved::Reserved; use proc_macro2::{Span, TokenStream}; #[cfg(feature = "printing")] use quote::IdentFragment; @@ -87,7 +85,7 @@ ast_enum_of_structs! { /// see names getting repeated in your code, like accessing /// `receiver.receiver` or `pat.pat` or `cond.cond`. #[cfg_attr(doc_cfg, doc(cfg(any(feature = "full", feature = "derive"))))] - #[cfg_attr(not(syn_no_non_exhaustive), non_exhaustive)] + #[non_exhaustive] pub enum Expr { /// A slice literal expression: `[a, b, c, d]`. Array(ExprArray), @@ -95,24 +93,18 @@ ast_enum_of_structs! { /// An assignment expression: `a = compute()`. Assign(ExprAssign), - /// A compound assignment expression: `counter += 1`. - AssignOp(ExprAssignOp), - /// An async block: `async { ... }`. Async(ExprAsync), /// An await expression: `fut.await`. Await(ExprAwait), - /// A binary operation: `a + b`, `a * b`. + /// A binary operation: `a + b`, `a += b`. Binary(ExprBinary), /// A blocked scope: `{ ... }`. Block(ExprBlock), - /// A box expression: `box f`. - Box(ExprBox), - /// A `break`, with an optional label to break and an optional /// expression. Break(ExprBreak), @@ -126,6 +118,9 @@ ast_enum_of_structs! { /// A closure expression: `|a, b| a + b`. Closure(ExprClosure), + /// A const block: `const { ... }`. + Const(ExprConst), + /// A `continue`, with an optional label. Continue(ExprContinue), @@ -153,6 +148,9 @@ ast_enum_of_structs! { /// A square bracketed indexing expression: `vector[2]`. Index(ExprIndex), + /// The inferred value of a const generic argument, denoted `_`. + Infer(ExprInfer), + /// A `let` guard: `let Some(x) = opt`. Let(ExprLet), @@ -207,9 +205,6 @@ ast_enum_of_structs! { /// A tuple expression: `(a, b, c, d)`. Tuple(ExprTuple), - /// A type ascription expression: `foo: f64`. - Type(ExprType), - /// A unary operation: `!x`, `*x`. Unary(ExprUnary), @@ -225,17 +220,16 @@ ast_enum_of_structs! { /// A yield expression: `yield expr`. Yield(ExprYield), - // Not public API. - // // For testing exhaustiveness in downstream code, use the following idiom: // // match expr { + // #![cfg_attr(test, deny(non_exhaustive_omitted_patterns))] + // // Expr::Array(expr) => {...} // Expr::Assign(expr) => {...} // ... // Expr::Yield(expr) => {...} // - // #[cfg_attr(test, deny(non_exhaustive_omitted_patterns))] // _ => { /* some sane fallback */ } // } // @@ -243,16 +237,11 @@ ast_enum_of_structs! { // a variant. You will be notified by a test failure when a variant is // added, so that you can add code to handle it, but your library will // continue to compile and work for downstream users in the interim. - #[cfg(syn_no_non_exhaustive)] - #[doc(hidden)] - __NonExhaustive, } } ast_struct! { /// A slice literal expression: `[a, b, c, d]`. - /// - /// *This type is available only if Syn is built with the `"full"` feature.* #[cfg_attr(doc_cfg, doc(cfg(feature = "full")))] pub struct ExprArray #full { pub attrs: Vec, @@ -263,8 +252,6 @@ ast_struct! { ast_struct! { /// An assignment expression: `a = compute()`. - /// - /// *This type is available only if Syn is built with the `"full"` feature.* #[cfg_attr(doc_cfg, doc(cfg(feature = "full")))] pub struct ExprAssign #full { pub attrs: Vec, @@ -274,23 +261,8 @@ ast_struct! { } } -ast_struct! { - /// A compound assignment expression: `counter += 1`. - /// - /// *This type is available only if Syn is built with the `"full"` feature.* - #[cfg_attr(doc_cfg, doc(cfg(feature = "full")))] - pub struct ExprAssignOp #full { - pub attrs: Vec, - pub left: Box, - pub op: BinOp, - pub right: Box, - } -} - ast_struct! { /// An async block: `async { ... }`. - /// - /// *This type is available only if Syn is built with the `"full"` feature.* #[cfg_attr(doc_cfg, doc(cfg(feature = "full")))] pub struct ExprAsync #full { pub attrs: Vec, @@ -302,22 +274,17 @@ ast_struct! { ast_struct! { /// An await expression: `fut.await`. - /// - /// *This type is available only if Syn is built with the `"full"` feature.* #[cfg_attr(doc_cfg, doc(cfg(feature = "full")))] pub struct ExprAwait #full { pub attrs: Vec, pub base: Box, pub dot_token: Token![.], - pub await_token: token::Await, + pub await_token: Token![await], } } ast_struct! { - /// A binary operation: `a + b`, `a * b`. - /// - /// *This type is available only if Syn is built with the `"derive"` or - /// `"full"` feature.* + /// A binary operation: `a + b`, `a += b`. #[cfg_attr(doc_cfg, doc(cfg(any(feature = "full", feature = "derive"))))] pub struct ExprBinary { pub attrs: Vec, @@ -329,8 +296,6 @@ ast_struct! { ast_struct! { /// A blocked scope: `{ ... }`. - /// - /// *This type is available only if Syn is built with the `"full"` feature.* #[cfg_attr(doc_cfg, doc(cfg(feature = "full")))] pub struct ExprBlock #full { pub attrs: Vec, @@ -339,23 +304,9 @@ ast_struct! { } } -ast_struct! { - /// A box expression: `box f`. - /// - /// *This type is available only if Syn is built with the `"full"` feature.* - #[cfg_attr(doc_cfg, doc(cfg(feature = "full")))] - pub struct ExprBox #full { - pub attrs: Vec, - pub box_token: Token![box], - pub expr: Box, - } -} - ast_struct! { /// A `break`, with an optional label to break and an optional /// expression. - /// - /// *This type is available only if Syn is built with the `"full"` feature.* #[cfg_attr(doc_cfg, doc(cfg(feature = "full")))] pub struct ExprBreak #full { pub attrs: Vec, @@ -367,9 +318,6 @@ ast_struct! { ast_struct! { /// A function call expression: `invoke(a, b)`. - /// - /// *This type is available only if Syn is built with the `"derive"` or - /// `"full"` feature.* #[cfg_attr(doc_cfg, doc(cfg(any(feature = "full", feature = "derive"))))] pub struct ExprCall { pub attrs: Vec, @@ -381,9 +329,6 @@ ast_struct! { ast_struct! { /// A cast expression: `foo as f64`. - /// - /// *This type is available only if Syn is built with the `"derive"` or - /// `"full"` feature.* #[cfg_attr(doc_cfg, doc(cfg(any(feature = "full", feature = "derive"))))] pub struct ExprCast { pub attrs: Vec, @@ -395,11 +340,11 @@ ast_struct! { ast_struct! { /// A closure expression: `|a, b| a + b`. - /// - /// *This type is available only if Syn is built with the `"full"` feature.* #[cfg_attr(doc_cfg, doc(cfg(feature = "full")))] pub struct ExprClosure #full { pub attrs: Vec, + pub lifetimes: Option, + pub constness: Option, pub movability: Option, pub asyncness: Option, pub capture: Option, @@ -411,10 +356,18 @@ ast_struct! { } } +ast_struct! { + /// A const block: `const { ... }`. + #[cfg_attr(doc_cfg, doc(cfg(feature = "full")))] + pub struct ExprConst #full { + pub attrs: Vec, + pub const_token: Token![const], + pub block: Block, + } +} + ast_struct! { /// A `continue`, with an optional label. - /// - /// *This type is available only if Syn is built with the `"full"` feature.* #[cfg_attr(doc_cfg, doc(cfg(feature = "full")))] pub struct ExprContinue #full { pub attrs: Vec, @@ -426,8 +379,6 @@ ast_struct! { ast_struct! { /// Access of a named struct field (`obj.k`) or unnamed tuple struct /// field (`obj.0`). - /// - /// *This type is available only if Syn is built with the `"full"` feature.* #[cfg_attr(doc_cfg, doc(cfg(any(feature = "full", feature = "derive"))))] pub struct ExprField { pub attrs: Vec, @@ -439,14 +390,12 @@ ast_struct! { ast_struct! { /// A for loop: `for pat in expr { ... }`. - /// - /// *This type is available only if Syn is built with the `"full"` feature.* #[cfg_attr(doc_cfg, doc(cfg(feature = "full")))] pub struct ExprForLoop #full { pub attrs: Vec, pub label: Option