diff --git a/Cargo.lock b/Cargo.lock index 118730418142..ed3acaf2c0e6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3871,9 +3871,9 @@ dependencies = [ [[package]] name = "once_cell" -version = "1.10.0" +version = "1.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87f3e037eac156d1775da914196f0f37741a274155e34a0b7e427c35d2a2ecb9" +checksum = "7709cef83f0c1f58f666e746a08b21e0085f7440fa6a29cc194d68aac97a4225" [[package]] name = "ordered-float" diff --git a/supply-chain/config.toml b/supply-chain/config.toml index 87f26da8474f..805744396fff 100644 --- a/supply-chain/config.toml +++ b/supply-chain/config.toml @@ -1018,7 +1018,7 @@ version = "0.28.4" criteria = "safe-to-deploy" [[unaudited.once_cell]] -version = "1.10.0" +version = "1.12.0" criteria = "safe-to-deploy" [[unaudited.ordered-float]] diff --git a/third_party/rust/once_cell/.cargo-checksum.json b/third_party/rust/once_cell/.cargo-checksum.json index 4e443dbbe3be..79ab95a08932 100644 --- a/third_party/rust/once_cell/.cargo-checksum.json +++ b/third_party/rust/once_cell/.cargo-checksum.json @@ -1 +1 @@ -{"files":{"CHANGELOG.md":"ec168d0d5ebc881969130c0932cf2f818f6cf38561541c2f3c579274de380282","Cargo.lock":"4c3897ebbcc358b31c6d8bad72f824df36b2fee6adb191ff0c599a9b425135e6","Cargo.toml":"4338838068c7e96f0ec8c7416ed3bb1c6a8dd1b09a280a0ec2944624812012a7","LICENSE-APACHE":"a60eea817514531668d7e00765731449fe14d059d3249e0bc93b36de45f759f2","LICENSE-MIT":"23f18e03dc49df91622fe2a76176497404e46ced8a715d9d2b67a7446571cca3","README.md":"813d262a320611ba874c4b2488256bdb2b4073649616a1471b389d464a704301","bors.toml":"ebd69f714a49dceb8fd10ebadfea6e2767be4732fdef49eddf6239151b4bc78c","examples/bench.rs":"1597a52529f75d6c5ad0b86759a775b1d723dfa810e2016317283b13594219da","examples/bench_acquire.rs":"9f4912ca262194cb55e893c33739c85c2f4868d07905b9dd3238552b6ce8a6e4","examples/bench_vs_lazy_static.rs":"d527294a2e73b53ac5faed8b316dfd1ae2a06adb31384134af21f10ce76333a5","examples/lazy_static.rs":"90541b093ed1d1cbb73f4097ff02cf80657e28264d281d6a31d96a708fdfea90","examples/reentrant_init_deadlocks.rs":"ff84929de27a848e5b155549caa96db5db5f030afca975f8ba3f3da640083001","examples/regex.rs":"4a2e0fb093c7f5bbe0fff8689fc0c670c5334344a1bfda376f5faa98a05d459f","examples/test_synchronization.rs":"88abd5c16275bb2f2d77eaecf369d97681404a77b8edd0021f24bfd377c46be3","src/imp_pl.rs":"cac67286119fcfa2b69ffb9ada51d5cb57c89e95a533d95d5d40b0ae4d38a6c2","src/imp_std.rs":"7a9b58444f71ca3025655f060dabc0f33a775d3b27916e9c22fe4182b286265d","src/lib.rs":"a12ab5c81725b96793555e01eda23642fc19fb4fb9a6e4b8fd1394981a18bb4c","src/race.rs":"2a6613a50df41c1433e692160cba427f3c7a5624960b82bc6f84ea7b383286c2","tests/it.rs":"ed9ff44665f29fa138055800d7199405740bce912f9c568c11aa2ff2249c17f8"},"package":"87f3e037eac156d1775da914196f0f37741a274155e34a0b7e427c35d2a2ecb9"} \ No newline at end of file +{"files":{"CHANGELOG.md":"b2fd6a1cdee0ac05cb44f0ce93c15cf21ab8efd18b36734b40e25a59b4497a6a","Cargo.lock":"73e0ab98cd9c6528a833eab9ebab7da1a04993fccb7550786de1672a5ebd6aa4","Cargo.toml":"57fcde8845829a18ea66855873946c612a97045aa2b1b15316e6f17fd7f5cc17","LICENSE-APACHE":"a60eea817514531668d7e00765731449fe14d059d3249e0bc93b36de45f759f2","LICENSE-MIT":"23f18e03dc49df91622fe2a76176497404e46ced8a715d9d2b67a7446571cca3","README.md":"813d262a320611ba874c4b2488256bdb2b4073649616a1471b389d464a704301","bors.toml":"ebd69f714a49dceb8fd10ebadfea6e2767be4732fdef49eddf6239151b4bc78c","examples/bench.rs":"1597a52529f75d6c5ad0b86759a775b1d723dfa810e2016317283b13594219da","examples/bench_acquire.rs":"9f4912ca262194cb55e893c33739c85c2f4868d07905b9dd3238552b6ce8a6e4","examples/bench_vs_lazy_static.rs":"d527294a2e73b53ac5faed8b316dfd1ae2a06adb31384134af21f10ce76333a5","examples/lazy_static.rs":"90541b093ed1d1cbb73f4097ff02cf80657e28264d281d6a31d96a708fdfea90","examples/reentrant_init_deadlocks.rs":"ff84929de27a848e5b155549caa96db5db5f030afca975f8ba3f3da640083001","examples/regex.rs":"4a2e0fb093c7f5bbe0fff8689fc0c670c5334344a1bfda376f5faa98a05d459f","examples/test_synchronization.rs":"88abd5c16275bb2f2d77eaecf369d97681404a77b8edd0021f24bfd377c46be3","src/imp_pl.rs":"a5d84a2f707b21455ee02592888fb46a9744e1b5d7eb0c01f960f242572cd414","src/imp_std.rs":"3491f91cd853193fe59abf35efc6c29f8d9016781390bd62699eea9d9440320d","src/lib.rs":"1872bc0514519909d1171f81f24d07a767e0ee19d4925b4bf94248c471367ffa","src/race.rs":"5a19afca4b5510d09ca7317b96f5642725c58b0969b2bdeb7275ed674d061e5d","tests/it.rs":"2e84ed653b7a38de417e31c37c74a0ea52b913c06b4abd1334a4e0c9e2538619"},"package":"7709cef83f0c1f58f666e746a08b21e0085f7440fa6a29cc194d68aac97a4225"} \ No newline at end of file diff --git a/third_party/rust/once_cell/CHANGELOG.md b/third_party/rust/once_cell/CHANGELOG.md index 5d1a02d15b2c..e5c0ae73d04a 100644 --- a/third_party/rust/once_cell/CHANGELOG.md +++ b/third_party/rust/once_cell/CHANGELOG.md @@ -1,10 +1,24 @@ # Changelog -## 1.10 +## Unreleased + +- + +## 1.12.0 + +- Add `OnceCell::wait`, a blocking variant of `get`. + +## 1.11.0 + +- Add `OnceCell::with_value` to create initialized `OnceCell` in `const` context. +- Improve `Clone` implementation for `OnceCell`. +- Rewrite `parking_lot` version on top of `parking_lot_core`, for even smaller cells! + +## 1.10.0 - upgrade `parking_lot` to `0.12.0` (note that this bumps MSRV with `parking_lot` feature enabled to `1.49.0`). -## 1.9 +## 1.9.0 - Added an `atomic-polyfill` optional dependency to compile `race` on platforms without atomics diff --git a/third_party/rust/once_cell/Cargo.lock b/third_party/rust/once_cell/Cargo.lock index 2482e9b8a135..08123eb2bf42 100644 --- a/third_party/rust/once_cell/Cargo.lock +++ b/third_party/rust/once_cell/Cargo.lock @@ -13,20 +13,13 @@ dependencies = [ [[package]] name = "atomic-polyfill" -version = "0.1.6" +version = "0.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ee6adc1648f03fbc1bc1b5cf0f2fdfb5edbc96215b711edcfe6ce2641ef9b347" +checksum = "e14bf7b4f565e5e717d7a7a65b2a05c0b8c96e4db636d6f780f03b15108cdd1b" dependencies = [ "critical-section", - "riscv-target", ] -[[package]] -name = "autocfg" -version = "1.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" - [[package]] name = "bare-metal" version = "0.2.5" @@ -60,12 +53,6 @@ version = "1.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" -[[package]] -name = "cfg-if" -version = "0.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c4e7bb64a8ebb0d856483e1e682ea3422f883c5f5615a90d51a2c82fe87fdd3" - [[package]] name = "cfg-if" version = "1.0.0" @@ -86,24 +73,23 @@ dependencies = [ [[package]] name = "critical-section" -version = "0.2.5" +version = "0.2.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "01e191a5a6f6edad9b679777ef6b6c0f2bdd4a333f2ecb8f61c3e28109a03d70" +checksum = "95da181745b56d4bd339530ec393508910c909c784e8962d15d722bacf0bcbcd" dependencies = [ "bare-metal 1.0.0", - "cfg-if 1.0.0", + "cfg-if", "cortex-m", "riscv", ] [[package]] name = "crossbeam-utils" -version = "0.7.2" +version = "0.8.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c3c7c73a2d1e9fc0886a08b93e98eb643461230d5f1925e4036204d5f2e261a8" +checksum = "0bf124c720b7686e3c2663cf54062ab0f68a88af2fb6a030e87e30bf721fcb38" dependencies = [ - "autocfg", - "cfg-if 0.1.5", + "cfg-if", "lazy_static", ] @@ -125,18 +111,9 @@ checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" [[package]] name = "libc" -version = "0.2.119" +version = "0.2.126" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1bf2e165bb3457c8e098ea76f3e3bc9db55f87aa90d52d0e6be741470916aaa4" - -[[package]] -name = "lock_api" -version = "0.4.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "88943dd7ef4a2e5a4bfa2753aaab3013e34ce2533d1996fb18ef591e315e2b3b" -dependencies = [ - "scopeguard", -] +checksum = "349d5a591cd28b49e1d1037471617a32ddcda5731b99419008085f72d5a53836" [[package]] name = "memchr" @@ -161,32 +138,22 @@ checksum = "546c37ac5d9e56f55e73b677106873d9d9f5190605e41a856503623648488cae" [[package]] name = "once_cell" -version = "1.10.0" +version = "1.12.0" dependencies = [ "atomic-polyfill", "crossbeam-utils", "lazy_static", - "parking_lot", + "parking_lot_core", "regex", ] -[[package]] -name = "parking_lot" -version = "0.12.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87f5ec2493a61ac0506c0f4199f99070cbe83857b0337006a30f3e6719b8ef58" -dependencies = [ - "lock_api", - "parking_lot_core", -] - [[package]] name = "parking_lot_core" -version = "0.9.1" +version = "0.9.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "28141e0cc4143da2443301914478dc976a61ffdb3f043058310c70df2fed8954" +checksum = "09a279cbf25cb0757810394fbc1e359949b59e348145c643a939a525692e6929" dependencies = [ - "cfg-if 1.0.0", + "cfg-if", "libc", "redox_syscall", "smallvec", @@ -195,9 +162,9 @@ dependencies = [ [[package]] name = "redox_syscall" -version = "0.2.11" +version = "0.2.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8380fe0152551244f0747b1bf41737e0f8a74f97a14ccefd1148187271634f3c" +checksum = "62f25bc4c7e55e0b0b7a1d43fb893f4fa1361d0abe38b9ce4f323c2adfe6ef42" dependencies = [ "bitflags", ] @@ -254,12 +221,6 @@ dependencies = [ "semver", ] -[[package]] -name = "scopeguard" -version = "1.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd" - [[package]] name = "semver" version = "0.9.0" @@ -325,9 +286,9 @@ dependencies = [ [[package]] name = "windows-sys" -version = "0.32.0" +version = "0.36.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3df6e476185f92a12c072be4a189a0210dcdcf512a1891d6dff9edb874deadc6" +checksum = "ea04155a16a59f9eab786fe12a4a450e75cdb175f9e0d80da1e17db09f55b8d2" dependencies = [ "windows_aarch64_msvc", "windows_i686_gnu", @@ -338,30 +299,30 @@ dependencies = [ [[package]] name = "windows_aarch64_msvc" -version = "0.32.0" +version = "0.36.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d8e92753b1c443191654ec532f14c199742964a061be25d77d7a96f09db20bf5" +checksum = "9bb8c3fd39ade2d67e9874ac4f3db21f0d710bee00fe7cab16949ec184eeaa47" [[package]] name = "windows_i686_gnu" -version = "0.32.0" +version = "0.36.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a711c68811799e017b6038e0922cb27a5e2f43a2ddb609fe0b6f3eeda9de615" +checksum = "180e6ccf01daf4c426b846dfc66db1fc518f074baa793aa7d9b9aaeffad6a3b6" [[package]] name = "windows_i686_msvc" -version = "0.32.0" +version = "0.36.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "146c11bb1a02615db74680b32a68e2d61f553cc24c4eb5b4ca10311740e44172" +checksum = "e2e7917148b2812d1eeafaeb22a97e4813dfa60a3f8f78ebe204bcc88f12f024" [[package]] name = "windows_x86_64_gnu" -version = "0.32.0" +version = "0.36.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c912b12f7454c6620635bbff3450962753834be2a594819bd5e945af18ec64bc" +checksum = "4dcd171b8776c41b97521e5da127a2d86ad280114807d0b2ab1e462bc764d9e1" [[package]] name = "windows_x86_64_msvc" -version = "0.32.0" +version = "0.36.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "504a2476202769977a040c6364301a3f65d0cc9e3fb08600b2bda150a0488316" +checksum = "c811ca4a8c853ef420abd8592ba53ddbbac90410fab6903b3e79972a631f7680" diff --git a/third_party/rust/once_cell/Cargo.toml b/third_party/rust/once_cell/Cargo.toml index 1f13a166895f..4e5636133794 100644 --- a/third_party/rust/once_cell/Cargo.toml +++ b/third_party/rust/once_cell/Cargo.toml @@ -12,16 +12,28 @@ [package] edition = "2018" name = "once_cell" -version = "1.10.0" +version = "1.12.0" authors = ["Aleksey Kladov "] -exclude = ["*.png", "*.svg", "/Cargo.lock.msrv", "rustfmt.toml"] +exclude = [ + "*.png", + "*.svg", + "/Cargo.lock.msrv", + "rustfmt.toml", +] description = "Single assignment cells and lazy values." documentation = "https://docs.rs/once_cell" readme = "README.md" -keywords = ["lazy", "static"] -categories = ["rust-patterns", "memory-management"] +keywords = [ + "lazy", + "static", +] +categories = [ + "rust-patterns", + "memory-management", +] license = "MIT OR Apache-2.0" repository = "https://github.com/matklad/once_cell" + [package.metadata.docs.rs] all-features = true @@ -52,16 +64,18 @@ required-features = ["std"] [[example]] name = "test_synchronization" required-features = ["std"] + [dependencies.atomic-polyfill] version = "0.1" optional = true -[dependencies.parking_lot] -version = "0.12" +[dependencies.parking_lot_core] +version = "0.9.3" optional = true default_features = false + [dev-dependencies.crossbeam-utils] -version = "0.7.2" +version = "0.8.7" [dev-dependencies.lazy_static] version = "1.0.0" @@ -72,6 +86,7 @@ version = "1.2.0" [features] alloc = ["race"] default = ["std"] +parking_lot = ["parking_lot_core"] race = [] std = ["alloc"] unstable = [] diff --git a/third_party/rust/once_cell/src/imp_pl.rs b/third_party/rust/once_cell/src/imp_pl.rs index 6c9b0fe84cf0..2bd80fa315df 100644 --- a/third_party/rust/once_cell/src/imp_pl.rs +++ b/third_party/rust/once_cell/src/imp_pl.rs @@ -2,19 +2,18 @@ use std::{ cell::UnsafeCell, hint, panic::{RefUnwindSafe, UnwindSafe}, - sync::atomic::{AtomicBool, Ordering}, + sync::atomic::{AtomicU8, Ordering}, }; -use parking_lot::Mutex; - -use crate::take_unchecked; - pub(crate) struct OnceCell { - mutex: Mutex<()>, - is_initialized: AtomicBool, + state: AtomicU8, value: UnsafeCell>, } +const INCOMPLETE: u8 = 0x0; +const RUNNING: u8 = 0x1; +const COMPLETE: u8 = 0x2; + // Why do we need `T: Send`? // Thread A creates a `OnceCell` and shares it with // scoped thread B, which fills the cell, which is @@ -28,17 +27,17 @@ impl UnwindSafe for OnceCell {} impl OnceCell { pub(crate) const fn new() -> OnceCell { - OnceCell { - mutex: parking_lot::const_mutex(()), - is_initialized: AtomicBool::new(false), - value: UnsafeCell::new(None), - } + OnceCell { state: AtomicU8::new(INCOMPLETE), value: UnsafeCell::new(None) } + } + + pub(crate) const fn with_value(value: T) -> OnceCell { + OnceCell { state: AtomicU8::new(COMPLETE), value: UnsafeCell::new(Some(value)) } } /// Safety: synchronizes with store to value via Release/Acquire. #[inline] pub(crate) fn is_initialized(&self) -> bool { - self.is_initialized.load(Ordering::Acquire) + self.state.load(Ordering::Acquire) == COMPLETE } /// Safety: synchronizes with store to value via `is_initialized` or mutex @@ -51,7 +50,7 @@ impl OnceCell { let mut f = Some(f); let mut res: Result<(), E> = Ok(()); let slot: *mut Option = self.value.get(); - initialize_inner(&self.mutex, &self.is_initialized, &mut || { + initialize_inner(&self.state, &mut || { // We are calling user-supplied function and need to be careful. // - if it returns Err, we unlock mutex and return without touching anything // - if it panics, we unlock mutex and propagate panic without touching anything @@ -60,7 +59,7 @@ impl OnceCell { // but that is more complicated // - finally, if it returns Ok, we store the value and store the flag with // `Release`, which synchronizes with `Acquire`s. - let f = unsafe { take_unchecked(&mut f) }; + let f = unsafe { crate::take_unchecked(&mut f) }; match f() { Ok(value) => unsafe { // Safe b/c we have a unique access and no panic may happen @@ -78,6 +77,21 @@ impl OnceCell { res } + #[cold] + pub(crate) fn wait(&self) { + let key = &self.state as *const _ as usize; + unsafe { + parking_lot_core::park( + key, + || self.state.load(Ordering::Acquire) != COMPLETE, + || (), + |_, _| (), + parking_lot_core::DEFAULT_PARK_TOKEN, + None, + ); + } + } + /// Get the reference to the underlying value, without checking if the cell /// is initialized. /// @@ -113,14 +127,48 @@ impl OnceCell { } } +struct Guard<'a> { + state: &'a AtomicU8, + new_state: u8, +} + +impl<'a> Drop for Guard<'a> { + fn drop(&mut self) { + self.state.store(self.new_state, Ordering::Release); + unsafe { + let key = self.state as *const AtomicU8 as usize; + parking_lot_core::unpark_all(key, parking_lot_core::DEFAULT_UNPARK_TOKEN); + } + } +} + // Note: this is intentionally monomorphic #[inline(never)] -fn initialize_inner(mutex: &Mutex<()>, is_initialized: &AtomicBool, init: &mut dyn FnMut() -> bool) { - let _guard = mutex.lock(); - - if !is_initialized.load(Ordering::Acquire) { - if init() { - is_initialized.store(true, Ordering::Release); +fn initialize_inner(state: &AtomicU8, init: &mut dyn FnMut() -> bool) { + loop { + let exchange = + state.compare_exchange_weak(INCOMPLETE, RUNNING, Ordering::Acquire, Ordering::Acquire); + match exchange { + Ok(_) => { + let mut guard = Guard { state, new_state: INCOMPLETE }; + if init() { + guard.new_state = COMPLETE; + } + return; + } + Err(COMPLETE) => return, + Err(RUNNING) => unsafe { + let key = state as *const AtomicU8 as usize; + parking_lot_core::park( + key, + || state.load(Ordering::Relaxed) == RUNNING, + || (), + |_, _| (), + parking_lot_core::DEFAULT_PARK_TOKEN, + None, + ); + }, + Err(_) => debug_assert!(false), } } } @@ -129,5 +177,5 @@ fn initialize_inner(mutex: &Mutex<()>, is_initialized: &AtomicBool, init: &mut d fn test_size() { use std::mem::size_of; - assert_eq!(size_of::>(), 2 * size_of::() + size_of::()); + assert_eq!(size_of::>(), 1 * size_of::() + size_of::()); } diff --git a/third_party/rust/once_cell/src/imp_std.rs b/third_party/rust/once_cell/src/imp_std.rs index d7dda9672863..c01078cabd5e 100644 --- a/third_party/rust/once_cell/src/imp_std.rs +++ b/third_party/rust/once_cell/src/imp_std.rs @@ -16,9 +16,15 @@ use crate::take_unchecked; #[derive(Debug)] pub(crate) struct OnceCell { - // This `state` word is actually an encoded version of just a pointer to a - // `Waiter`, so we add the `PhantomData` appropriately. - state_and_queue: AtomicUsize, + // This `queue` field is the core of the implementation. It encodes two + // pieces of information: + // + // * The current state of the cell (`INCOMPLETE`, `RUNNING`, `COMPLETE`) + // * Linked list of threads waiting for the current cell. + // + // State is encoded in two low bits. Only `INCOMPLETE` and `RUNNING` states + // allow waiters. + queue: AtomicUsize, _marker: PhantomData<*mut Waiter>, value: UnsafeCell>, } @@ -34,41 +40,23 @@ unsafe impl Send for OnceCell {} impl RefUnwindSafe for OnceCell {} impl UnwindSafe for OnceCell {} -// Three states that a OnceCell can be in, encoded into the lower bits of `state` in -// the OnceCell structure. -const INCOMPLETE: usize = 0x0; -const RUNNING: usize = 0x1; -const COMPLETE: usize = 0x2; - -// Mask to learn about the state. All other bits are the queue of waiters if -// this is in the RUNNING state. -const STATE_MASK: usize = 0x3; - -// Representation of a node in the linked list of waiters in the RUNNING state. -#[repr(align(4))] // Ensure the two lower bits are free to use as state bits. -struct Waiter { - thread: Cell>, - signaled: AtomicBool, - next: *const Waiter, -} - -// Head of a linked list of waiters. -// Every node is a struct on the stack of a waiting thread. -// Will wake up the waiters when it gets dropped, i.e. also on panic. -struct WaiterQueue<'a> { - state_and_queue: &'a AtomicUsize, - set_state_on_drop_to: usize, -} - impl OnceCell { pub(crate) const fn new() -> OnceCell { OnceCell { - state_and_queue: AtomicUsize::new(INCOMPLETE), + queue: AtomicUsize::new(INCOMPLETE), _marker: PhantomData, value: UnsafeCell::new(None), } } + pub(crate) const fn with_value(value: T) -> OnceCell { + OnceCell { + queue: AtomicUsize::new(COMPLETE), + _marker: PhantomData, + value: UnsafeCell::new(Some(value)), + } + } + /// Safety: synchronizes with store to value via Release/(Acquire|SeqCst). #[inline] pub(crate) fn is_initialized(&self) -> bool { @@ -76,7 +64,7 @@ impl OnceCell { // operations visible to us, and, this being a fast path, weaker // ordering helps with performance. This `Acquire` synchronizes with // `SeqCst` operations on the slow path. - self.state_and_queue.load(Ordering::Acquire) == COMPLETE + self.queue.load(Ordering::Acquire) == COMPLETE } /// Safety: synchronizes with store to value via SeqCst read from state, @@ -90,22 +78,30 @@ impl OnceCell { let mut f = Some(f); let mut res: Result<(), E> = Ok(()); let slot: *mut Option = self.value.get(); - initialize_inner(&self.state_and_queue, &mut || { - let f = unsafe { take_unchecked(&mut f) }; - match f() { - Ok(value) => { - unsafe { *slot = Some(value) }; - true + initialize_or_wait( + &self.queue, + Some(&mut || { + let f = unsafe { take_unchecked(&mut f) }; + match f() { + Ok(value) => { + unsafe { *slot = Some(value) }; + true + } + Err(err) => { + res = Err(err); + false + } } - Err(err) => { - res = Err(err); - false - } - } - }); + }), + ); res } + #[cold] + pub(crate) fn wait(&self) { + initialize_or_wait(&self.queue, None); + } + /// Get the reference to the underlying value, without checking if the cell /// is initialized. /// @@ -144,67 +140,111 @@ impl OnceCell { } } -// Corresponds to `std::sync::Once::call_inner` -// Note: this is intentionally monomorphic -#[inline(never)] -fn initialize_inner(my_state_and_queue: &AtomicUsize, init: &mut dyn FnMut() -> bool) -> bool { - let mut state_and_queue = my_state_and_queue.load(Ordering::Acquire); +// Three states that a OnceCell can be in, encoded into the lower bits of `queue` in +// the OnceCell structure. +const INCOMPLETE: usize = 0x0; +const RUNNING: usize = 0x1; +const COMPLETE: usize = 0x2; - loop { - match state_and_queue { - COMPLETE => return true, - INCOMPLETE => { - let exchange = my_state_and_queue.compare_exchange( - state_and_queue, - RUNNING, - Ordering::Acquire, - Ordering::Acquire, - ); - if let Err(old) = exchange { - state_and_queue = old; - continue; - } - let mut waiter_queue = WaiterQueue { - state_and_queue: my_state_and_queue, - set_state_on_drop_to: INCOMPLETE, // Difference, std uses `POISONED` - }; - let success = init(); +// Mask to learn about the state. All other bits are the queue of waiters if +// this is in the RUNNING state. +const STATE_MASK: usize = 0x3; - // Difference, std always uses `COMPLETE` - waiter_queue.set_state_on_drop_to = if success { COMPLETE } else { INCOMPLETE }; - return success; - } - _ => { - assert!(state_and_queue & STATE_MASK == RUNNING); - wait(&my_state_and_queue, state_and_queue); - state_and_queue = my_state_and_queue.load(Ordering::Acquire); +/// Representation of a node in the linked list of waiters in the RUNNING state. +/// A waiters is stored on the stack of the waiting threads. +#[repr(align(4))] // Ensure the two lower bits are free to use as state bits. +struct Waiter { + thread: Cell>, + signaled: AtomicBool, + next: *const Waiter, +} + +/// Drains and notifies the queue of waiters on drop. +struct Guard<'a> { + queue: &'a AtomicUsize, + new_queue: usize, +} + +impl Drop for Guard<'_> { + fn drop(&mut self) { + let queue = self.queue.swap(self.new_queue, Ordering::AcqRel); + + assert_eq!(queue & STATE_MASK, RUNNING); + + unsafe { + let mut waiter = (queue & !STATE_MASK) as *const Waiter; + while !waiter.is_null() { + let next = (*waiter).next; + let thread = (*waiter).thread.take().unwrap(); + (*waiter).signaled.store(true, Ordering::Release); + waiter = next; + thread.unpark(); } } } } -// Copy-pasted from std exactly. -fn wait(state_and_queue: &AtomicUsize, mut current_state: usize) { - loop { - if current_state & STATE_MASK != RUNNING { - return; - } +// Corresponds to `std::sync::Once::call_inner`. +// +// Originally copied from std, but since modified to remove poisoning and to +// support wait. +// +// Note: this is intentionally monomorphic +#[inline(never)] +fn initialize_or_wait(queue: &AtomicUsize, mut init: Option<&mut dyn FnMut() -> bool>) { + let mut curr_queue = queue.load(Ordering::Acquire); + loop { + let curr_state = curr_queue & STATE_MASK; + match (curr_state, &mut init) { + (COMPLETE, _) => return, + (INCOMPLETE, Some(init)) => { + let exchange = queue.compare_exchange( + curr_queue, + (curr_queue & !STATE_MASK) | RUNNING, + Ordering::Acquire, + Ordering::Acquire, + ); + if let Err(new_queue) = exchange { + curr_queue = new_queue; + continue; + } + let mut guard = Guard { queue, new_queue: INCOMPLETE }; + if init() { + guard.new_queue = COMPLETE; + } + return; + } + (INCOMPLETE, None) | (RUNNING, _) => { + wait(&queue, curr_queue); + curr_queue = queue.load(Ordering::Acquire); + } + _ => debug_assert!(false), + } + } +} + +fn wait(queue: &AtomicUsize, mut curr_queue: usize) { + let curr_state = curr_queue & STATE_MASK; + loop { let node = Waiter { thread: Cell::new(Some(thread::current())), signaled: AtomicBool::new(false), - next: (current_state & !STATE_MASK) as *const Waiter, + next: (curr_queue & !STATE_MASK) as *const Waiter, }; let me = &node as *const Waiter as usize; - let exchange = state_and_queue.compare_exchange( - current_state, - me | RUNNING, + let exchange = queue.compare_exchange( + curr_queue, + me | curr_state, Ordering::Release, Ordering::Relaxed, ); - if let Err(old) = exchange { - current_state = old; + if let Err(new_queue) = exchange { + if new_queue & STATE_MASK != curr_state { + return; + } + curr_queue = new_queue; continue; } @@ -215,27 +255,6 @@ fn wait(state_and_queue: &AtomicUsize, mut current_state: usize) { } } -// Copy-pasted from std exactly. -impl Drop for WaiterQueue<'_> { - fn drop(&mut self) { - let state_and_queue = - self.state_and_queue.swap(self.set_state_on_drop_to, Ordering::AcqRel); - - assert_eq!(state_and_queue & STATE_MASK, RUNNING); - - unsafe { - let mut queue = (state_and_queue & !STATE_MASK) as *const Waiter; - while !queue.is_null() { - let next = (*queue).next; - let thread = (*queue).thread.replace(None).unwrap(); - (*queue).signaled.store(true, Ordering::Release); - queue = next; - thread.unpark(); - } - } - } -} - // These test are snatched from std as well. #[cfg(test)] mod tests { diff --git a/third_party/rust/once_cell/src/lib.rs b/third_party/rust/once_cell/src/lib.rs index 3c7dfcb75894..0b23a36ec817 100644 --- a/third_party/rust/once_cell/src/lib.rs +++ b/third_party/rust/once_cell/src/lib.rs @@ -311,6 +311,10 @@ //! At the moment, `unsync` has an additional benefit that reentrant initialization causes a panic, which //! might be easier to debug than a deadlock. //! +//! **Does this crate support async?** +//! +//! No, but you can use [`async_once_cell`](https://crates.io/crates/async_once_cell) instead. +//! //! # Related crates //! //! * [double-checked-cell](https://github.com/niklasf/double-checked-cell) @@ -318,6 +322,7 @@ //! * [lazycell](https://crates.io/crates/lazycell) //! * [mitochondria](https://crates.io/crates/mitochondria) //! * [lazy_static](https://crates.io/crates/lazy_static) +//! * [async_once_cell](https://crates.io/crates/async_once_cell) //! //! Most of this crate's functionality is available in `std` in nightly Rust. //! See the [tracking issue](https://github.com/rust-lang/rust/issues/74465). @@ -399,14 +404,17 @@ pub mod unsync { impl Clone for OnceCell { fn clone(&self) -> OnceCell { - let res = OnceCell::new(); - if let Some(value) = self.get() { - match res.set(value.clone()) { - Ok(()) => (), - Err(_) => unreachable!(), - } + match self.get() { + Some(value) => OnceCell::with_value(value.clone()), + None => OnceCell::new(), + } + } + + fn clone_from(&mut self, source: &Self) { + match (self.get_mut(), source.get()) { + (Some(this), Some(source)) => this.clone_from(source), + _ => *self = source.clone(), } - res } } @@ -420,7 +428,7 @@ pub mod unsync { impl From for OnceCell { fn from(value: T) -> Self { - OnceCell { inner: UnsafeCell::new(Some(value)) } + OnceCell::with_value(value) } } @@ -430,6 +438,11 @@ pub mod unsync { OnceCell { inner: UnsafeCell::new(None) } } + /// Creates a new initialized cell. + pub const fn with_value(value: T) -> OnceCell { + OnceCell { inner: UnsafeCell::new(Some(value)) } + } + /// Gets a reference to the underlying value. /// /// Returns `None` if the cell is empty. @@ -810,22 +823,23 @@ pub mod sync { impl Clone for OnceCell { fn clone(&self) -> OnceCell { - let res = OnceCell::new(); - if let Some(value) = self.get() { - match res.set(value.clone()) { - Ok(()) => (), - Err(_) => unreachable!(), - } + match self.get() { + Some(value) => Self::with_value(value.clone()), + None => Self::new(), + } + } + + fn clone_from(&mut self, source: &Self) { + match (self.get_mut(), source.get()) { + (Some(this), Some(source)) => this.clone_from(source), + _ => *self = source.clone(), } - res } } impl From for OnceCell { fn from(value: T) -> Self { - let cell = Self::new(); - cell.get_or_init(|| value); - cell + Self::with_value(value) } } @@ -843,6 +857,11 @@ pub mod sync { OnceCell(Imp::new()) } + /// Creates a new initialized cell. + pub const fn with_value(value: T) -> OnceCell { + OnceCell(Imp::with_value(value)) + } + /// Gets the reference to the underlying value. /// /// Returns `None` if the cell is empty, or being initialized. This @@ -856,6 +875,36 @@ pub mod sync { } } + /// Gets the reference to the underlying value, blocking the current + /// thread until it is set. + /// + /// ``` + /// use once_cell::sync::OnceCell; + /// + /// let mut cell = std::sync::Arc::new(OnceCell::new()); + /// let t = std::thread::spawn({ + /// let cell = std::sync::Arc::clone(&cell); + /// move || cell.set(92).unwrap() + /// }); + /// + /// // Returns immediately, but might return None. + /// let _value_or_none = cell.get(); + /// + /// // Will return 92, but might block until the other thread does `.set`. + /// let value: &u32 = cell.wait(); + /// assert_eq!(*value, 92); + /// t.join().unwrap();; + /// ``` + pub fn wait(&self) -> &T { + if !self.0.is_initialized() { + self.0.wait() + } + debug_assert!(self.0.is_initialized()); + // Safe b/c of the wait call above and the fact that we didn't + // relinquish our borrow. + unsafe { self.get_unchecked() } + } + /// Gets the mutable reference to the underlying value. /// /// Returns `None` if the cell is empty. diff --git a/third_party/rust/once_cell/src/race.rs b/third_party/rust/once_cell/src/race.rs index 3576420d36b1..e83e0b921675 100644 --- a/third_party/rust/once_cell/src/race.rs +++ b/third_party/rust/once_cell/src/race.rs @@ -5,6 +5,19 @@ //! them stores the result. //! //! This module does not require `std` feature. +//! +//! # Atomic orderings +//! +//! All types in this module use `Acquire` and `Release` +//! [atomic orderings](Ordering) for all their operations. While this is not +//! strictly necessary for types other than `OnceBox`, it is useful for users as +//! it allows them to be certain that after `get` or `get_or_init` returns on +//! one thread, any side-effects caused by the setter thread prior to them +//! calling `set` or `get_or_init` will be made visible to that thread; without +//! it, it's possible for it to appear as if they haven't happened yet from the +//! getter thread's perspective. This is an acceptable tradeoff to make since +//! `Acquire` and `Release` have very little performance overhead on most +//! architectures versus `Relaxed`. #[cfg(feature = "atomic-polyfill")] use atomic_polyfill as atomic; diff --git a/third_party/rust/once_cell/tests/it.rs b/third_party/rust/once_cell/tests/it.rs index 81faaffe46bb..95483bd9ed07 100644 --- a/third_party/rust/once_cell/tests/it.rs +++ b/third_party/rust/once_cell/tests/it.rs @@ -17,6 +17,13 @@ mod unsync { assert_eq!(c.get(), Some(&92)); } + #[test] + fn once_cell_with_value() { + const CELL: OnceCell = OnceCell::with_value(12); + let cell = CELL; + assert_eq!(cell.get(), Some(&12)); + } + #[test] fn once_cell_get_mut() { let mut c = OnceCell::new(); @@ -230,6 +237,12 @@ mod sync { assert_eq!(c.get(), Some(&92)); } + #[test] + fn once_cell_with_value() { + static CELL: OnceCell = OnceCell::with_value(12); + assert_eq!(CELL.get(), Some(&12)); + } + #[test] fn once_cell_get_mut() { let mut c = OnceCell::new(); @@ -306,6 +319,41 @@ mod sync { assert_eq!(cell.get(), Some(&"hello".to_string())); } + #[test] + fn wait() { + let cell: OnceCell = OnceCell::new(); + scope(|s| { + s.spawn(|_| cell.set("hello".to_string())); + let greeting = cell.wait(); + assert_eq!(greeting, "hello") + }) + .unwrap(); + } + + #[test] + #[cfg_attr(miri, ignore)] // miri doesn't support Barrier + fn get_or_init_stress() { + use std::sync::Barrier; + let n_threads = 1_000; + let n_cells = 1_000; + let cells: Vec<_> = std::iter::repeat_with(|| (Barrier::new(n_threads), OnceCell::new())) + .take(n_cells) + .collect(); + scope(|s| { + for t in 0..n_threads { + let cells = &cells; + s.spawn(move |_| { + for (i, (b, s)) in cells.iter().enumerate() { + b.wait(); + let j = if t % 2 == 0 { s.wait() } else { s.get_or_init(|| i) }; + assert_eq!(*j, i); + } + }); + } + }) + .unwrap(); + } + #[test] fn from_impl() { assert_eq!(OnceCell::from("value").get(), Some(&"value"));