Bug 1693832 - Use a patched version of rustc/cargo for tsan/asan builds. r=glandium

Differential Revision: https://phabricator.services.mozilla.com/D105812
This commit is contained in:
Alexis Beingessner 2021-06-15 05:30:05 +00:00
parent caa8f413f4
commit d647cb3bfa
6 changed files with 521 additions and 29 deletions

View File

@ -0,0 +1,417 @@
Teaches Cargo to source all std dependencies from vendored sources in the rust-src
component, making -Zbuild-std compatible with vendored builds.
This was originally landed in https://github.com/rust-lang/cargo/pull/8834
but was backed out for causing breakage in other situations. It works fine
for Firefox's usecase, though.
Most of these changes just add/edit tests for the functionality. Only the
change to src/cargo/core/compiler/standard_lib.rs is important.
diff --git a/src/cargo/core/compiler/standard_lib.rs b/src/cargo/core/compiler/standard_lib.rs
index 0b14df805..7bc5151a4 100644
--- a/src/cargo/core/compiler/standard_lib.rs
+++ b/src/cargo/core/compiler/standard_lib.rs
@@ -10,6 +10,7 @@ use crate::ops::{self, Packages};
use crate::util::errors::CargoResult;
use std::collections::{HashMap, HashSet};
use std::env;
+use std::fs;
use std::path::PathBuf;
/// Parse the `-Zbuild-std` flag.
@@ -38,28 +39,46 @@ pub fn resolve_std<'cfg>(
crates: &[String],
) -> CargoResult<(PackageSet<'cfg>, Resolve, ResolvedFeatures)> {
let src_path = detect_sysroot_src_path(target_data)?;
- let to_patch = [
- "rustc-std-workspace-core",
- "rustc-std-workspace-alloc",
- "rustc-std-workspace-std",
- ];
- let patches = to_patch
- .iter()
- .map(|&name| {
- let source_path = SourceId::for_path(&src_path.join("library").join(name))?;
- let dep = Dependency::parse_no_deprecated(name, None, source_path)?;
+
+ // Special std packages should be pulled from `library/` and should be
+ // prefixed with `rustc-std-workspace-` in certain places.
+ let libs_prefix = "library/";
+ let special_std_prefix = "rustc-std-workspace-";
+ let libs_path = src_path.join(libs_prefix);
+
+ // Crates in rust-src to build. libtest is in some sense the "root" package
+ // of std, as nothing else depends on it, so it must be explicitly added.
+ let mut members = vec![format!("{}test", libs_prefix)];
+
+ // If rust-src contains a "vendor" directory, then patch in all the crates it contains.
+ let vendor_path = src_path.join("vendor");
+ let vendor_dir = fs::read_dir(vendor_path)?;
+ let patches = vendor_dir
+ .into_iter()
+ .map(|entry| {
+ let entry = entry?;
+ let name = entry
+ .file_name()
+ .into_string()
+ .map_err(|_| anyhow::anyhow!("package name wasn't utf8"))?;
+
+ // Remap the rustc-std-workspace crates to the actual rust-src libraries
+ let path = if let Some(real_name) = name.strip_prefix(special_std_prefix) {
+ // Record this crate as something to build in the workspace
+ members.push(format!("{}{}", libs_prefix, real_name));
+ libs_path.join(&name)
+ } else {
+ entry.path()
+ };
+ let source_path = SourceId::for_path(&path)?;
+ let dep = Dependency::parse_no_deprecated(&name, None, source_path)?;
Ok(dep)
})
.collect::<CargoResult<Vec<_>>>()?;
+
let crates_io_url = crate::sources::CRATES_IO_INDEX.parse().unwrap();
let mut patch = HashMap::new();
patch.insert(crates_io_url, patches);
- let members = vec![
- String::from("library/std"),
- String::from("library/core"),
- String::from("library/alloc"),
- String::from("library/test"),
- ];
let ws_config = crate::core::WorkspaceConfig::Root(crate::core::WorkspaceRootConfig::new(
&src_path,
&Some(members),
diff --git a/tests/testsuite/mock-std/library/test/Cargo.toml b/tests/testsuite/mock-std/library/test/Cargo.toml
index 299db7bfd..eb81bbf5e 100644
--- a/tests/testsuite/mock-std/library/test/Cargo.toml
+++ b/tests/testsuite/mock-std/library/test/Cargo.toml
@@ -10,6 +10,7 @@ std = { path = "../std" }
panic_unwind = { path = "../panic_unwind" }
compiler_builtins = { path = "../compiler_builtins" }
registry-dep-using-std = { version = "*", features = ['mockbuild'] }
+registry-dep-only-used-by-test = { version = "*" }
[features]
panic-unwind = []
diff --git a/tests/testsuite/mock-std/library/test/src/lib.rs b/tests/testsuite/mock-std/library/test/src/lib.rs
index a112855f5..224b89bb2 100644
--- a/tests/testsuite/mock-std/library/test/src/lib.rs
+++ b/tests/testsuite/mock-std/library/test/src/lib.rs
@@ -7,4 +7,5 @@ extern crate test;
pub use test::*;
pub fn custom_api() {
+ registry_dep_only_used_by_test::wow_testing_is_so_easy();
}
diff --git a/tests/testsuite/mock-std/vendor/registry-dep-only-used-by-test/Cargo.toml b/tests/testsuite/mock-std/vendor/registry-dep-only-used-by-test/Cargo.toml
new file mode 100644
index 000000000..31ba65a98
--- /dev/null
+++ b/tests/testsuite/mock-std/vendor/registry-dep-only-used-by-test/Cargo.toml
@@ -0,0 +1,9 @@
+[package]
+name = "registry-dep-only-used-by-test"
+version = "1.0.0"
+authors = ["Alex Crichton <alex@alexcrichton.com>"]
+edition = "2018"
+
+[dependencies]
+
+[features]
diff --git a/tests/testsuite/mock-std/vendor/registry-dep-only-used-by-test/src/lib.rs b/tests/testsuite/mock-std/vendor/registry-dep-only-used-by-test/src/lib.rs
new file mode 100644
index 000000000..a68d2aeef
--- /dev/null
+++ b/tests/testsuite/mock-std/vendor/registry-dep-only-used-by-test/src/lib.rs
@@ -0,0 +1,2 @@
+pub fn wow_testing_is_so_easy() {
+}
\ No newline at end of file
diff --git a/tests/testsuite/mock-std/vendor/registry-dep-using-alloc/Cargo.toml b/tests/testsuite/mock-std/vendor/registry-dep-using-alloc/Cargo.toml
new file mode 100644
index 000000000..f7e4ab232
--- /dev/null
+++ b/tests/testsuite/mock-std/vendor/registry-dep-using-alloc/Cargo.toml
@@ -0,0 +1,12 @@
+[package]
+name = "registry-dep-using-alloc"
+version = "1.0.0"
+authors = ["Alex Crichton <alex@alexcrichton.com>"]
+edition = "2018"
+
+[dependencies]
+rustc-std-workspace-alloc = { version = "*", optional = true }
+rustc-std-workspace-core = { version = "*", optional = true }
+
+[features]
+mockbuild = ["rustc-std-workspace-alloc", "rustc-std-workspace-core"]
\ No newline at end of file
diff --git a/tests/testsuite/mock-std/vendor/registry-dep-using-alloc/src/lib.rs b/tests/testsuite/mock-std/vendor/registry-dep-using-alloc/src/lib.rs
new file mode 100644
index 000000000..b9ab30339
--- /dev/null
+++ b/tests/testsuite/mock-std/vendor/registry-dep-using-alloc/src/lib.rs
@@ -0,0 +1,9 @@
+#[cfg(feature = "mockbuild")]
+pub fn custom_api() {
+}
+
+#[cfg(not(feature = "mockbuild"))]
+pub fn non_sysroot_api() {
+ core::custom_api();
+ alloc::custom_api();
+}
\ No newline at end of file
diff --git a/tests/testsuite/mock-std/vendor/registry-dep-using-core/Cargo.toml b/tests/testsuite/mock-std/vendor/registry-dep-using-core/Cargo.toml
new file mode 100644
index 000000000..befb83a63
--- /dev/null
+++ b/tests/testsuite/mock-std/vendor/registry-dep-using-core/Cargo.toml
@@ -0,0 +1,11 @@
+[package]
+name = "registry-dep-using-core"
+version = "1.0.0"
+authors = ["Alex Crichton <alex@alexcrichton.com>"]
+edition = "2018"
+
+[dependencies]
+rustc-std-workspace-core = { version = "*", optional = true }
+
+[features]
+mockbuild = ["rustc-std-workspace-core"]
\ No newline at end of file
diff --git a/tests/testsuite/mock-std/vendor/registry-dep-using-core/src/lib.rs b/tests/testsuite/mock-std/vendor/registry-dep-using-core/src/lib.rs
new file mode 100644
index 000000000..f9dbac0f4
--- /dev/null
+++ b/tests/testsuite/mock-std/vendor/registry-dep-using-core/src/lib.rs
@@ -0,0 +1,8 @@
+#[cfg(feature = "mockbuild")]
+pub fn custom_api() {
+}
+
+#[cfg(not(feature = "mockbuild"))]
+pub fn non_sysroot_api() {
+ core::custom_api();
+}
\ No newline at end of file
diff --git a/tests/testsuite/mock-std/vendor/registry-dep-using-std/Cargo.toml b/tests/testsuite/mock-std/vendor/registry-dep-using-std/Cargo.toml
new file mode 100644
index 000000000..71ef0a42f
--- /dev/null
+++ b/tests/testsuite/mock-std/vendor/registry-dep-using-std/Cargo.toml
@@ -0,0 +1,11 @@
+[package]
+name = "registry-dep-using-std"
+version = "1.0.0"
+authors = ["Alex Crichton <alex@alexcrichton.com>"]
+edition = "2018"
+
+[dependencies]
+rustc-std-workspace-std = { version = "*", optional = true }
+
+[features]
+mockbuild = ["rustc-std-workspace-std"]
\ No newline at end of file
diff --git a/tests/testsuite/mock-std/vendor/registry-dep-using-std/src/lib.rs b/tests/testsuite/mock-std/vendor/registry-dep-using-std/src/lib.rs
new file mode 100644
index 000000000..f3af39178
--- /dev/null
+++ b/tests/testsuite/mock-std/vendor/registry-dep-using-std/src/lib.rs
@@ -0,0 +1,8 @@
+#[cfg(feature = "mockbuild")]
+pub fn custom_api() {
+}
+
+#[cfg(not(feature = "mockbuild"))]
+pub fn non_sysroot_api() {
+ std::custom_api();
+}
\ No newline at end of file
diff --git a/tests/testsuite/mock-std/vendor/rustc-std-workspace-alloc/Cargo.toml b/tests/testsuite/mock-std/vendor/rustc-std-workspace-alloc/Cargo.toml
new file mode 100644
index 000000000..4465a08a8
--- /dev/null
+++ b/tests/testsuite/mock-std/vendor/rustc-std-workspace-alloc/Cargo.toml
@@ -0,0 +1 @@
+this file shouldn't be read
\ No newline at end of file
diff --git a/tests/testsuite/mock-std/vendor/rustc-std-workspace-core/Cargo.toml b/tests/testsuite/mock-std/vendor/rustc-std-workspace-core/Cargo.toml
new file mode 100644
index 000000000..4465a08a8
--- /dev/null
+++ b/tests/testsuite/mock-std/vendor/rustc-std-workspace-core/Cargo.toml
@@ -0,0 +1 @@
+this file shouldn't be read
\ No newline at end of file
diff --git a/tests/testsuite/mock-std/vendor/rustc-std-workspace-std/Cargo.toml b/tests/testsuite/mock-std/vendor/rustc-std-workspace-std/Cargo.toml
new file mode 100644
index 000000000..4465a08a8
--- /dev/null
+++ b/tests/testsuite/mock-std/vendor/rustc-std-workspace-std/Cargo.toml
@@ -0,0 +1 @@
+this file shouldn't be read
\ No newline at end of file
diff --git a/tests/testsuite/standard_lib.rs b/tests/testsuite/standard_lib.rs
index b0d42b38d..3bbc49db7 100644
--- a/tests/testsuite/standard_lib.rs
+++ b/tests/testsuite/standard_lib.rs
@@ -27,71 +27,18 @@ fn setup() -> Option<Setup> {
return None;
}
- // Our mock sysroot requires a few packages from crates.io, so make sure
- // they're "published" to crates.io. Also edit their code a bit to make sure
- // that they have access to our custom crates with custom apis.
+ // Register a version of one of the std dependencies that doesn't compile.
+ // This ensures that the mock-std's vendor is actually being used.
Package::new("registry-dep-using-core", "1.0.0")
.file(
"src/lib.rs",
"
- #![no_std]
-
- #[cfg(feature = \"mockbuild\")]
- pub fn custom_api() {
- }
-
- #[cfg(not(feature = \"mockbuild\"))]
- pub fn non_sysroot_api() {
- core::custom_api();
- }
+ don't compile me bro!!
",
)
.add_dep(Dependency::new("rustc-std-workspace-core", "*").optional(true))
.feature("mockbuild", &["rustc-std-workspace-core"])
.publish();
- Package::new("registry-dep-using-alloc", "1.0.0")
- .file(
- "src/lib.rs",
- "
- #![no_std]
-
- extern crate alloc;
-
- #[cfg(feature = \"mockbuild\")]
- pub fn custom_api() {
- }
-
- #[cfg(not(feature = \"mockbuild\"))]
- pub fn non_sysroot_api() {
- core::custom_api();
- alloc::custom_api();
- }
- ",
- )
- .add_dep(Dependency::new("rustc-std-workspace-core", "*").optional(true))
- .add_dep(Dependency::new("rustc-std-workspace-alloc", "*").optional(true))
- .feature(
- "mockbuild",
- &["rustc-std-workspace-core", "rustc-std-workspace-alloc"],
- )
- .publish();
- Package::new("registry-dep-using-std", "1.0.0")
- .file(
- "src/lib.rs",
- "
- #[cfg(feature = \"mockbuild\")]
- pub fn custom_api() {
- }
-
- #[cfg(not(feature = \"mockbuild\"))]
- pub fn non_sysroot_api() {
- std::custom_api();
- }
- ",
- )
- .add_dep(Dependency::new("rustc-std-workspace-std", "*").optional(true))
- .feature("mockbuild", &["rustc-std-workspace-std"])
- .publish();
let p = ProjectBuilder::new(paths::root().join("rustc-wrapper"))
.file(
@@ -327,6 +274,81 @@ fn depend_same_as_std() {
None => return,
};
+ // Our mock sysroot requires a few packages from crates.io, so make sure
+ // they're "published" to crates.io. Also edit their code a bit to make sure
+ // that they have access to our custom crates with custom apis.
+ Package::new("registry-dep-using-core", "1.0.0")
+ .file(
+ "src/lib.rs",
+ "
+ #![no_std]
+
+ #[cfg(feature = \"mockbuild\")]
+ pub fn custom_api() {
+ }
+
+ #[cfg(not(feature = \"mockbuild\"))]
+ pub fn non_sysroot_api() {
+ core::custom_api();
+ }
+ ",
+ )
+ .add_dep(Dependency::new("rustc-std-workspace-core", "*").optional(true))
+ .feature("mockbuild", &["rustc-std-workspace-core"])
+ .publish();
+ Package::new("registry-dep-using-alloc", "1.0.0")
+ .file(
+ "src/lib.rs",
+ "
+ #![no_std]
+
+ extern crate alloc;
+
+ #[cfg(feature = \"mockbuild\")]
+ pub fn custom_api() {
+ }
+
+ #[cfg(not(feature = \"mockbuild\"))]
+ pub fn non_sysroot_api() {
+ core::custom_api();
+ alloc::custom_api();
+ }
+ ",
+ )
+ .add_dep(Dependency::new("rustc-std-workspace-core", "*").optional(true))
+ .add_dep(Dependency::new("rustc-std-workspace-alloc", "*").optional(true))
+ .feature(
+ "mockbuild",
+ &["rustc-std-workspace-core", "rustc-std-workspace-alloc"],
+ )
+ .publish();
+ Package::new("registry-dep-using-std", "1.0.0")
+ .file(
+ "src/lib.rs",
+ "
+ #[cfg(feature = \"mockbuild\")]
+ pub fn custom_api() {
+ }
+
+ #[cfg(not(feature = \"mockbuild\"))]
+ pub fn non_sysroot_api() {
+ std::custom_api();
+ }
+ ",
+ )
+ .add_dep(Dependency::new("rustc-std-workspace-std", "*").optional(true))
+ .feature("mockbuild", &["rustc-std-workspace-std"])
+ .publish();
+ Package::new("registry-dep-only-used-by-test", "1.0.0")
+ .file(
+ "src/lib.rs",
+ "
+ pub fn wow_testing_is_so_easy() {
+ }
+ ",
+ )
+ .publish();
+
let p = project()
.file(
"src/lib.rs",

View File

@ -0,0 +1,80 @@
Teaches Rust's build system to vendor std's dependencies into the
rust-src component.
This was originally landed in https://github.com/rust-lang/rust/pull/78790
but was backed out for causing some breakage for distro maintainers who
need to build Rust itself in a vendored/offline context. It doesn't actually
fetch anything interesting from crates.io, just the magic fake std/core crates
that exist to make the build work right. Those crates *are* vendored but
their contents are ignored in favour of the actual stdlib.
For firefox's purposes, these patches still work fine, and are necessary
to make -Zbuild-std work in a vendored environment.
diff --git a/src/bootstrap/dist.rs b/src/bootstrap/dist.rs
index 802b5c99500..438b76a472b 100644
--- a/src/bootstrap/dist.rs
+++ b/src/bootstrap/dist.rs
@@ -847,6 +847,30 @@ fn run(self, builder: &Builder<'_>) -> GeneratedTarball {
builder.copy(&builder.src.join(file), &dst_src.join(file));
}
+ // libtest includes std and everything else, so vendoring it
+ // creates exactly what's needed for `cargo -Zbuild-std` or any
+ // other analysis of the stdlib's source. Cargo also needs help
+ // finding the lock, so we copy it to libtest temporarily.
+ //
+ // Note that this requires std to only have one version of each
+ // crate. e.g. two versions of getopts won't be patchable.
+ let dst_libtest = dst_src.join("library/test");
+ let dst_vendor = dst_src.join("vendor");
+ let root_lock = dst_src.join("Cargo.lock");
+ let temp_lock = dst_libtest.join("Cargo.lock");
+
+ // `cargo vendor` will delete everything from the lockfile that
+ // isn't used by libtest, so we need to not use any links!
+ builder.really_copy(&root_lock, &temp_lock);
+
+ let mut cmd = Command::new(&builder.initial_cargo);
+ cmd.arg("vendor").arg(dst_vendor).current_dir(&dst_libtest);
+ builder.info("Dist src");
+ let _time = timeit(builder);
+ builder.run(&mut cmd);
+
+ builder.remove(&temp_lock);
+
tarball.generate()
}
}
diff --git a/src/bootstrap/lib.rs b/src/bootstrap/lib.rs
index 88fdcfa2d43..293324c17af 100644
--- a/src/bootstrap/lib.rs
+++ b/src/bootstrap/lib.rs
@@ -1179,6 +1179,27 @@ fn read_stamp_file(&self, stamp: &Path) -> Vec<(PathBuf, DependencyType)> {
paths
}
+ /// Copies a file from `src` to `dst` and doesn't use links, so
+ /// that the copy can be modified without affecting the original.
+ pub fn really_copy(&self, src: &Path, dst: &Path) {
+ if self.config.dry_run {
+ return;
+ }
+ self.verbose_than(1, &format!("Copy {:?} to {:?}", src, dst));
+ if src == dst {
+ return;
+ }
+ let _ = fs::remove_file(&dst);
+ let metadata = t!(src.symlink_metadata());
+ if let Err(e) = fs::copy(src, dst) {
+ panic!("failed to copy `{}` to `{}`: {}", src.display(), dst.display(), e)
+ }
+ t!(fs::set_permissions(dst, metadata.permissions()));
+ let atime = FileTime::from_last_access_time(&metadata);
+ let mtime = FileTime::from_last_modification_time(&metadata);
+ t!(filetime::set_file_times(dst, atime, mtime));
+ }
+
/// Copies a file from `src` to `dst`
pub fn copy(&self, src: &Path, dst: &Path) {
if self.config.dry_run {

View File

@ -861,7 +861,7 @@ linux64-asan/opt:
toolchain:
- linux64-binutils
- linux64-clang
- linux64-rust-nightly
- linux64-rust-dev
- linux64-rust-size
- linux64-cbindgen
- linux64-dump-syms
@ -905,7 +905,7 @@ linux64-tsan/opt:
toolchain:
- linux64-binutils
- linux64-clang
- linux64-rust-nightly
- linux64-rust-dev
- linux64-rust-size
- linux64-cbindgen
- linux64-dump-syms
@ -951,7 +951,7 @@ linux64-tsan-fuzzing/opt:
toolchain:
- linux64-binutils
- linux64-clang
- linux64-rust-nightly
- linux64-rust-dev
- linux64-rust-size
- linux64-cbindgen
- linux64-dump-syms
@ -1000,7 +1000,7 @@ linux64-asan-fuzzing/noopt:
toolchain:
- linux64-binutils
- linux64-clang
- linux64-rust-nightly
- linux64-rust-dev
- linux64-rust-size
- linux64-cbindgen
- linux64-dump-syms
@ -1049,7 +1049,7 @@ linux64-asan-fuzzing/opt:
toolchain:
- linux64-binutils
- linux64-clang
- linux64-rust-nightly
- linux64-rust-dev
- linux64-rust-size
- linux64-cbindgen
- linux64-dump-syms
@ -1247,7 +1247,7 @@ linux64-asan/debug:
toolchain:
- linux64-binutils
- linux64-clang
- linux64-rust-nightly
- linux64-rust-dev
- linux64-rust-size
- linux64-cbindgen
- linux64-dump-syms

View File

@ -460,12 +460,21 @@ clang-trunk:
repo: https://github.com/llvm/llvm-project
branch: main
rust-1.47.0-dev:
description: Rust 1.47.0-dev source code
# This revision currently corresponds to rust nightly-2021-03-24,
# which is the first nightly with the version 1.53.0,
# so it should be as close as possible to the behaviour of
# stable-1.52.0.
#
# We use this to build from source, so the resulting non-official
# artifacts will report their version as "1.53.0-dev", but will otherwise
# behave like a nightly build. This means they will allow you to use
# unstable features like -Zbuild-std and sanitizers.
rust-1.53.0-dev:
description: Rust 1.53.0-dev source code
fetch:
type: git
repo: https://github.com/rust-lang/rust/
revision: 18bf6b4f01a6feaf7259ba7cdae58031af1b7b39
revision: 07e0e2ec268c140e607e1ac7f49f145612d0f597
wasi-sdk:
description: wasi-sdk source code

View File

@ -317,7 +317,7 @@ sm-tsan-linux64/opt:
- linux64-breakpad-injector
- linux64-minidump-stackwalk
- linux64-llvm-symbolizer
- linux64-rust-nightly
- linux64-rust-dev
- linux64-sysroot
sm-rootanalysis-linux64/debug:

View File

@ -44,23 +44,8 @@ linux64-rust-1.52:
toolchain:
- linux64-clang
linux64-rust-nightly:
description: "rust nightly repack"
treeherder:
symbol: TL(rust-nightly)
run:
arguments: [
# First nightly with the version after 1.49.0, so that it's the
# closest to 1.49 release that has a bigger version number.
# This can't be upgraded to the first with the version after 1.51.0
# because of https://github.com/rust-lang/rust/pull/80082, which
# reverted the changes needed to use -Zbuild-std in TSAN builds.
'--channel', 'nightly-2020-11-14',
'--host', 'x86_64-unknown-linux-gnu',
'--target', 'x86_64-unknown-linux-gnu',
'--target', 'i686-unknown-linux-gnu',
]
# A patched rust toolchain that allows us to use sanitizers in our vendored
# build environment. See the rust fetch's comments for more details.
linux64-rust-dev:
description: "build rust from source"
worker-type: b-linux-large
@ -68,7 +53,8 @@ linux64-rust-dev:
symbol: TL(rust-dev)
run:
arguments: [
'--patch', 'example.patch',
'--patch', 'rust-vendor-std.patch',
'--patch', 'src/tools/cargo:cargo-vendor-std.patch',
'--channel', 'dev',
'--host', 'x86_64-unknown-linux-gnu',
'--target', 'x86_64-unknown-linux-gnu',
@ -76,7 +62,7 @@ linux64-rust-dev:
]
fetches:
fetch:
- rust-1.47.0-dev
- rust-1.53.0-dev
toolchain:
- linux64-clang
- linux64-binutils