rui314/mold
1
0 Fork 0
mirror of https://github.com/rui314/mold.git synced 2026-11-09 17:52:02 -10:00
Code Issues Packages Projects Releases 1 Wiki Activity

Replace pointer-and-assertion patterns with bounds-checked views

Browse Source 
The port kept C++ idioms in which a raw pointer or an unchecked index
stood in for a slice, guarded by an assertion. Add SyncUnsafeCell, a
slice-of-cells view for parallel passes, and use it for the file
mapping, the symbol-table allocators, the symbol editor and decoded CREL
tables. Make the fields that parallel passes write atomic, which turns
allocate_aux, the dynsym pass, the gdb_index prefix scan and the COMDAT
signature slots into safe code. Write merged sections and the gdb_index
tables through split slices, and look up files and sections with plain
bounds checks.

This removes 58 unsafe blocks and 16 assertions.
This commit is contained in:
Rui Ueyama committed 2026-09-21 15:20:19 +09:00
1 parent 2d4a214d6e
commit a5511c1a7b
12 files changed
+374 -620
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

src/chunks/dynsym.rs
+1 -1
View File
@@ -44,7 +44,7 @@ impl<E: Target> DynsymSection<E> {
}
if symbols[sym].dynsym_idx(symbols).is_none() {
// Mark the symbol as queued before sort_dynsyms assigns its real index.
symbols.aux_mut(sym).dynsym_idx = u32::MAX - 1;
*symbols.aux_mut(sym).dynsym_idx.get_mut() = u32::MAX - 1;
self.symbols.push(Some(sym));
}
}
src/chunks/gnu_hash.rs
+6 -2
View File
@@ -1,5 +1,7 @@
//! `.gnu.hash`, the GNU hash table for dynamic symbol lookup.
use std::sync::atomic::Ordering;
use crate::chunks::ChunkHeader;
use crate::context::Context;
use crate::elf::*;
@@ -88,7 +90,8 @@ pub fn copy_buf<E: Target>(ctx: &Context<E>, buf: &mut [u8]) {
let word_bits = word * 8;
let mut indices = Vec::with_capacity(syms.len());
for id in syms {
let h = ctx.symbols[id.unwrap()].aux(&ctx.symbols).unwrap().djb_hash;
let h =
ctx.symbols[id.unwrap()].aux(&ctx.symbols).unwrap().djb_hash.load(Ordering::Relaxed);
indices.push(h % gh.num_buckets);
let idx = (h as usize / word_bits) % gh.num_bloom as usize;
let bits = (1u64 << (h as usize % word_bits))
@@ -112,7 +115,8 @@ pub fn copy_buf<E: Target>(ctx: &Context<E>, buf: &mut [u8]) {
for (i, &id) in syms.iter().enumerate() {
// The last entry in a chain must be terminated with an entry with
// least-significant bit 1.
let h = ctx.symbols[id.unwrap()].aux(&ctx.symbols).unwrap().djb_hash;
let h =
ctx.symbols[id.unwrap()].aux(&ctx.symbols).unwrap().djb_hash.load(Ordering::Relaxed);
let last = i + 1 == syms.len() || indices[i] != indices[i + 1];
E::write_u32(&mut buf[table_off + i * 4..], if last { h | 1 } else { h & !1 });
}
src/chunks/merged.rs
+17 -21
View File
@@ -623,32 +623,28 @@ pub fn write_to<E: Target>(ctx: &Context<E>, id: MergedSectionId, buf: &mut [u8]
// If that's the case, we need to zero-clear them.
let has_gaps = msec.hdr.shdr.sh_addralign.get() > 1
&& msec.hdr.shdr.sh_addralign.get() != msec.hdr.shdr.sh_entsize.get();
if has_gaps {
split_at_offsets(buf, &msec.shard_offsets[..NUM_SHARDS * 2])
.into_par_iter()
.for_each(|region| region.fill(0));
}
// Copy strings
let output = buf.as_mut_ptr() as usize;
let output_len = buf.len();
msec.shards.par_iter().for_each(|shard| {
// Layout gave each shard one region for its 32-bit fragments and one
// for the rest, so each shard writes only its two regions.
let offsets = &msec.shard_offsets;
let mut regions = split_at_offsets(buf, &offsets[..NUM_SHARDS * 2]);
let (near, far) = regions.split_at_mut(NUM_SHARDS);
msec.shards.par_iter().enumerate().zip(near).zip(far).for_each(|(((i, shard), near), far)| {
if has_gaps {
near.fill(0);
far.fill(0);
}
for &entry in &shard.fragments {
let frag = frags.get(entry);
if frag.is_alive() {
let key = frags.key(entry);
let offset = frag.offset() as usize;
debug_assert!(offset + key.len() <= output_len);
// SAFETY: layout assigns every live fragment a distinct range
// within `buf`; each entry occurs in exactly one shard, so the
// parallel copies do not overlap.
unsafe {
std::ptr::copy_nonoverlapping(
key.as_ptr(),
(output as *mut u8).add(offset),
key.len(),
);
}
let (region, base) = if frag.is_32bit() {
(&mut **near, offsets[i])
} else {
(&mut **far, offsets[i + NUM_SHARDS])
};
let start = (frag.offset() - base) as usize;
region[start..start + key.len()].copy_from_slice(key);
}
}
});
src/context.rs
+1 -4
View File
@@ -395,10 +395,7 @@ impl<E: Target> Context<E> {
#[inline]
pub fn input_section(&self, id: InputSectionId) -> &InputSection<E> {
// SAFETY: InputSectionIds are created only when a section is inserted
// into the stable file pool and that file's dense section vector.
let file = unsafe { self.objs.get_unchecked(id.file().index()) };
unsafe { file.sections.input_unchecked(id.index()) }
self.objs[id.file().index()].sections.input(id.index())
}
/// Formats an input section for diagnostics.
src/gdb_index.rs
+99 -105
View File
@@ -76,7 +76,8 @@ use std::borrow::Cow;
use std::path::Path;
use std::sync::atomic::{AtomicU32, Ordering};
use crate::util::concurrent_map::{ConcurrentMap, EntryId, FrozenMap, MapEntryRef};
use crate::util::SyncUnsafeCell;
use crate::util::concurrent_map::{ConcurrentMap, EntryId, FrozenMap};
use crate::util::hyperloglog::HyperLogLog;
use crate::util::perf::Timer;
use crate::util::read_uleb;
@@ -189,9 +190,9 @@ struct NameEntry {
gdb_hash: u32,
/// The number of units the name occurs in.
count: AtomicU32,
/// Offsets in the constant pool.
type_vector_offset: u32,
name_offset: u32,
/// Offsets in the constant pool, assigned by a parallel prefix scan.
type_vector_offset: AtomicU32,
name_offset: AtomicU32,
}
#[derive(Clone, Copy, Default)]
@@ -207,19 +208,16 @@ pub struct GdbIndexData {
tus: Vec<Typeunit>,
names: Option<FrozenMap<NameEntry>>,
/// The names in output order.
entries: Vec<MapEntryRef<NameEntry>>,
entries: Vec<EntryId>,
type_pool_size: u32,
name_pool_size: u32,
ht_size: u32,
/// The hash table followed by the constant pool, once built.
tables: Box<[u32]>,
}
/// Views word-aligned table storage as its serialized bytes.
fn words_as_bytes(words: &[u32]) -> &[u8] {
// SAFETY: u8 has alignment one and the byte length covers the same
// allocation without outliving it.
unsafe { std::slice::from_raw_parts(words.as_ptr().cast::<u8>(), std::mem::size_of_val(words)) }
/// The name hash table, once built.
ht: Vec<u32>,
/// The type vectors of the constant pool, once built.
type_pool: Vec<u32>,
/// The NUL-terminated names of the constant pool, once built.
name_pool: Vec<u8>,
}
/// Normalized view of a DWARF unit header. DWARF32 and DWARF64 describe the
@@ -858,8 +856,8 @@ fn initialize_gdb_name(name: *const u8) -> (u32, NameEntry) {
NameEntry {
gdb_hash: hash,
count: AtomicU32::new(0),
type_vector_offset: 0,
name_offset: 0,
type_vector_offset: AtomicU32::new(0),
name_offset: AtomicU32::new(0),
},
)
}
@@ -973,7 +971,7 @@ pub fn read_inputs<E: Target>(timer: Timer, files: Vec<GdbInputFile>) -> GdbInde
// Lay out the constant pool: all type vectors, then all names, in a
// deterministic order.
let names = map.freeze();
let entries = names.sorted_entry_refs_all();
let entries = names.sorted_entries_all();
// The map may contain millions of names. Assign their type and string
// ranges with a parallel prefix sum.
@@ -982,10 +980,9 @@ pub fn read_inputs<E: Target>(timer: Timer, files: Vec<GdbInputFile>) -> GdbInde
.par_chunks(chunk_size)
.map(|chunk| {
let mut size = PoolSize::default();
for &entry_ref in chunk {
let entry = entry_ref.value(&names);
size.type_bytes += entry.count.load(Ordering::Relaxed) * 4 + 4;
size.name_bytes += entry_ref.key_len(&names) as u32 + 1;
for &id in chunk {
size.type_bytes += names.get(id).count.load(Ordering::Relaxed) * 4 + 4;
size.name_bytes += names.key(id).len() as u32 + 1;
}
size
})
@@ -1001,14 +998,12 @@ pub fn read_inputs<E: Target>(timer: Timer, files: Vec<GdbInputFile>) -> GdbInde
})
.collect();
entries.par_chunks(chunk_size).zip(chunk_offsets).for_each(|(chunk, mut size)| {
for &entry_ref in chunk {
// SAFETY: entries contains each map entry exactly once, so
// parallel chunks update disjoint values.
let entry = unsafe { &mut *entry_ref.value_mut_ptr(&names) };
entry.type_vector_offset = size.type_bytes;
entry.name_offset = pool_size.type_bytes + size.name_bytes;
for &id in chunk {
let entry = names.get(id);
entry.type_vector_offset.store(size.type_bytes, Ordering::Relaxed);
entry.name_offset.store(pool_size.type_bytes + size.name_bytes, Ordering::Relaxed);
size.type_bytes += entry.count.load(Ordering::Relaxed) * 4 + 4;
size.name_bytes += entry_ref.key_len(&names) as u32 + 1;
size.name_bytes += names.key(id).len() as u32 + 1;
}
});
let ht_size = (entries.len() as u32 * 5 / 4 + 1).next_power_of_two();
@@ -1020,7 +1015,9 @@ pub fn read_inputs<E: Target>(timer: Timer, files: Vec<GdbInputFile>) -> GdbInde
type_pool_size: pool_size.type_bytes,
name_pool_size: pool_size.name_bytes,
ht_size,
tables: Box::default(),
ht: Vec::new(),
type_pool: Vec::new(),
name_pool: Vec::new(),
}
}
@@ -1084,54 +1081,40 @@ pub fn build_tables(timer: Timer, mut data: GdbIndexData, workers: usize) -> Gdb
data.tus.sort_by_key(|tu| tu.offset);
let names = data.names.as_ref().unwrap();
let symtab_size = data.ht_size as usize * 8;
let pool_size = (data.type_pool_size + data.name_pool_size) as usize;
let table_size = symtab_size + pool_size;
let table_words = table_size.div_ceil(4);
let mut tables = Box::<[u32]>::new_uninit_slice(table_words);
let table_addr = tables.as_mut_ptr() as usize;
// `tables` contains the name hash table followed by the constant pool. The
// constant pool contains all type vectors followed by all name strings.
// Each occupied hash-table slot contains the constant-pool offsets of a
// name and its type vector.
// SAFETY: table storage has at least symtab_size bytes. Initializing them
// before making the u32 slice avoids forming references to uninitialized
// integers.
unsafe { std::ptr::write_bytes(table_addr as *mut u8, 0, symtab_size) };
let ht = unsafe { std::slice::from_raw_parts_mut(table_addr as *mut u32, symtab_size / 4) };
// This probing sequence is part of the .gdb_index format. The table size
// is a power of two, so an odd step visits every slot.
// name and its type vector. This probing sequence is part of the
// .gdb_index format. The table size is a power of two, so an odd step
// visits every slot.
let mut ht = vec![0u32; data.ht_size as usize * 2];
let mask = data.ht_size - 1;
for &entry_ref in &data.entries {
let entry = entry_ref.value(names);
for &id in &data.entries {
let entry = names.get(id);
let step = ((entry.gdb_hash.wrapping_mul(17)) & mask) | 1;
let mut i = entry.gdb_hash & mask;
while ht[i as usize * 2] != 0 || ht[i as usize * 2 + 1] != 0 {
i = (i + step) & mask;
}
ht[i as usize * 2] = entry.name_offset.to_le();
ht[i as usize * 2 + 1] = entry.type_vector_offset.to_le();
ht[i as usize * 2] = entry.name_offset.load(Ordering::Relaxed);
ht[i as usize * 2 + 1] = entry.type_vector_offset.load(Ordering::Relaxed);
}
let pool_addr = table_addr + symtab_size;
// Each occurrence of a name contributes one value to its type vector. Each
// occurrence was assigned a distinct slot while the names were interned, so
// the vectors can be filled in parallel. The high byte is the name's type
// and the low 24 bits are the unit number. TU numbers follow CU numbers.
let mut type_pool = vec![0u32; data.type_pool_size as usize / 4];
let pool = SyncUnsafeCell::from_mut(&mut type_pool);
let write_names = |records: &[NameRecord], unit: usize| {
for record in records {
let name = record.indexed();
let entry = names.get(name.entry);
let offset = entry.type_vector_offset as usize + name.type_vector_idx as usize * 4;
debug_assert!(offset + 4 <= data.type_pool_size as usize);
let offset = entry.type_vector_offset.load(Ordering::Relaxed) as usize;
let index = offset / 4 + name.type_vector_idx as usize;
let value = (name.kind as u32) << 24 | unit as u32;
// SAFETY: the pool starts on a u32 boundary, all offsets are
// multiples of four, and every occurrence reserved a distinct
// slot before this parallel phase.
unsafe { (pool_addr as *mut u32).add(offset / 4).write(value) };
// SAFETY: every occurrence reserved a distinct slot before this
// parallel phase.
unsafe { *pool[index].get() = value };
}
};
let num_cus = data.cus.len();
@@ -1143,48 +1126,48 @@ pub fn build_tables(timer: Timer, mut data: GdbIndexData, workers: usize) -> Gdb
});
// Prefix each type vector with its length and sort it for deterministic
// output. Store the NUL-terminated name at its assigned string-pool offset.
limited_parallel_for_mut_init(&mut data.entries, workers, Vec::new, |scratch, _, entry_ref| {
let entry_ref = *entry_ref;
let entry = entry_ref.value(names);
let count = entry.count.load(Ordering::Relaxed);
let words = (pool_addr as *mut u32).wrapping_add(entry.type_vector_offset as usize / 4);
// SAFETY: every occurrence filled its distinct value slot above;
// writing the prefix completes this vector before a slice is made.
unsafe { words.write(count) };
let values = unsafe { std::slice::from_raw_parts_mut(words.add(1), count as usize) };
if values.len() < 256 {
values.sort_unstable();
} else {
radix_sort(values, scratch);
}
// SAFETY: the prefix and all count values are now initialized.
let words = unsafe { std::slice::from_raw_parts_mut(words, count as usize + 1) };
for word in &mut *words {
*word = word.to_le();
}
let key = entry_ref.key(names);
let name = (pool_addr as *mut u8).wrapping_add(entry.name_offset as usize);
// SAFETY: prefix-scan offsets assign this entry a distinct
// key.len()+1 byte range in the name pool.
unsafe {
name.copy_from_nonoverlapping(key.as_ptr(), key.len());
name.add(key.len()).write(0);
// output, and store each name at its string-pool offset; the pool's zero
// fill provides the NUL terminators. The entries are in pool order, so a
// chunk of entries owns one range of each pool.
let type_pool_size = data.type_pool_size as usize;
let mut name_pool = vec![0u8; data.name_pool_size as usize];
let chunk_size = data.entries.len().div_ceil(workers.max(1)).max(1);
let chunks: Vec<&[EntryId]> = data.entries.chunks(chunk_size).collect();
let chunk_base = |chunk: &[EntryId]| {
let first = names.get(chunk[0]);
let type_offset = first.type_vector_offset.load(Ordering::Relaxed) as usize;
let name_offset = first.name_offset.load(Ordering::Relaxed) as usize;
(type_offset / 4, name_offset - type_pool_size)
};
let type_offsets: Vec<u64> = chunks.iter().map(|chunk| chunk_base(chunk).0 as u64).collect();
let name_offsets: Vec<u64> = chunks.iter().map(|chunk| chunk_base(chunk).1 as u64).collect();
let mut regions: Vec<_> = chunks
.into_iter()
.zip(split_at_offsets(&mut type_pool, &type_offsets))
.zip(split_at_offsets(&mut name_pool, &name_offsets))
.collect();
limited_parallel_for_mut_init(&mut regions, workers, Vec::new, |scratch, _, region| {
let ((entries, words), bytes) = region;
let (type_base, name_base) = chunk_base(entries);
for &id in *entries {
let entry = names.get(id);
let count = entry.count.load(Ordering::Relaxed) as usize;
let start = entry.type_vector_offset.load(Ordering::Relaxed) as usize / 4 - type_base;
let words = &mut words[start..start + count + 1];
words[0] = count as u32;
let values = &mut words[1..];
if values.len() < 256 {
values.sort_unstable();
} else {
radix_sort(values, scratch);
}
let key = names.key(id);
let start =
entry.name_offset.load(Ordering::Relaxed) as usize - type_pool_size - name_base;
bytes[start..start + key.len()].copy_from_slice(key);
}
});
// Box<[u32]> rounds the byte allocation up to a whole word; initialize only
// those padding bytes, which are not part of the serialized tables.
let padded_size = table_words * 4;
for i in table_size..padded_size {
// SAFETY: these are the allocation's final padding bytes and no worker
// remains active.
unsafe { (table_addr as *mut u8).add(i).write(0) };
}
// SAFETY: the hash table, every type vector, every name including its NUL,
// and the final allocation padding have all been initialized.
let tables = unsafe { tables.assume_init() };
// The serialized tables contain everything needed from names and the map.
// Release their storage here so reclamation remains part of this background
// phase rather than delaying the final output path.
@@ -1196,7 +1179,9 @@ pub fn build_tables(timer: Timer, mut data: GdbIndexData, workers: usize) -> Gdb
});
data.entries = Vec::new();
data.names = None;
data.tables = tables;
data.ht = ht;
data.type_pool = type_pool;
data.name_pool = name_pool;
data
}
@@ -1239,12 +1224,23 @@ fn section_contents<'a, E: Target>(ctx: &'a Context<E>, buf: &'a [u8], name: &[u
fn parallel_copy(dst: &mut [u8], src: &[u8]) {
const BLOCK_SIZE: usize = 2 * 1024 * 1024;
debug_assert_eq!(dst.len(), src.len());
dst.par_chunks_mut(BLOCK_SIZE)
.zip(src.par_chunks(BLOCK_SIZE))
.for_each(|(dst, src)| dst.copy_from_slice(src));
}
/// Writes words as little-endian bytes, in parallel.
fn copy_words(dst: &mut [u8], src: &[u32]) {
const BLOCK_SIZE: usize = 512 * 1024;
let dst = dst.as_chunks_mut::<4>().0;
dst.par_chunks_mut(BLOCK_SIZE).zip(src.par_chunks(BLOCK_SIZE)).for_each(|(dst, src)| {
for (dst, &word) in dst.iter_mut().zip(src) {
*dst = word.to_le_bytes();
}
});
}
/// Read relocated address ranges and serialize the index prepared above.
pub fn write<E: Target>(ctx: &mut Context<E>, output: &mut OutputFile) {
let _t = ctx.timer("write_gdb_index");
@@ -1344,12 +1340,10 @@ pub fn write<E: Target>(ctx: &mut Context<E>, output: &mut OutputFile) {
});
let symtab_size = data.ht_size as usize * 8;
let tables = words_as_bytes(&data.tables);
parallel_copy(&mut buf[symtab_offset..symtab_offset + symtab_size], &tables[..symtab_size]);
parallel_copy(
&mut buf[const_pool_offset..size],
&tables[symtab_size..symtab_size + size - const_pool_offset],
);
let type_pool_size = data.type_pool_size as usize;
copy_words(&mut buf[symtab_offset..symtab_offset + symtab_size], &data.ht);
copy_words(&mut buf[const_pool_offset..const_pool_offset + type_pool_size], &data.type_pool);
parallel_copy(&mut buf[const_pool_offset + type_pool_size..size], &data.name_pool);
// Update the section size and rewrite the section header
if let Some(gdb_index) = &mut ctx.gdb_index {
src/input_files.rs
+64 -132
View File
@@ -4,14 +4,13 @@
#![allow(non_upper_case_globals)]
use std::borrow::Cow;
use std::cell::UnsafeCell;
use std::collections::BTreeMap;
use std::fmt;
use std::marker::PhantomData;
use std::mem::MaybeUninit;
use std::ops::{Index, IndexMut};
use std::ops::{Index, IndexMut, Range};
use std::path::Path;
use std::sync::atomic::{AtomicBool, Ordering};
use std::sync::atomic::{AtomicBool, AtomicU32, Ordering};
use std::sync::{OnceLock, RwLock};
use rayon::prelude::*;
@@ -31,7 +30,9 @@ use crate::symbol::{
};
use crate::target::{Family, Target};
use crate::util::perf::Counter;
use crate::util::{self, align_to, bits, cstr_at, leak_bytes, path_clean, read_uleb};
use crate::util::{
self, SyncUnsafeCell, align_to, bits, cstr_at, leak_bytes, path_clean, read_uleb,
};
use crate::{error, fatal, out, warn};
use bstr::BStr;
@@ -161,18 +162,6 @@ impl<T: FileInPool> FileList<T> {
self.pool.len()
}
/// Returns a file by its stable pool index without a bounds check.
///
/// # Safety
///
/// `index` must have been returned by this list's `push` method.
#[inline]
pub(crate) unsafe fn get_unchecked(&self, index: usize) -> &T {
debug_assert!(index < self.pool.len());
// SAFETY: guaranteed by the caller.
unsafe { self.pool.get_unchecked(index) }
}
pub fn is_empty(&self) -> bool {
self.live.is_empty()
}
@@ -519,34 +508,29 @@ impl<E: Target> InputFile<E> {
if idx >= self.shdrs.len() {
fatal!("{display}: invalid section index: {idx}");
}
let shdr = &self.shdrs[idx];
let (sh_offset, sh_size) = (shdr.sh_offset.get(), shdr.sh_size.get());
self.section_contents_range_checked(sh_offset, sh_size, display)
&self.data()[self.section_range_checked(&self.shdrs[idx], display)]
}
/// The contents described by a section header.
#[inline]
pub(crate) fn section_contents_from_shdr(&self, shdr: &ElfShdr<E>) -> &'static [u8] {
self.section_contents_range_checked(
shdr.sh_offset.get(),
shdr.sh_size.get(),
&self.filename,
)
&self.data()[self.section_range(shdr)]
}
fn section_contents_range_checked(
&self,
sh_offset: u64,
sh_size: u64,
display: &dyn fmt::Display,
) -> &'static [u8] {
let data = self.data();
/// The file range described by a section header, checked against the
/// file size.
pub(crate) fn section_range(&self, shdr: &ElfShdr<E>) -> Range<usize> {
self.section_range_checked(shdr, &self.filename)
}
fn section_range_checked(&self, shdr: &ElfShdr<E>, display: &dyn fmt::Display) -> Range<usize> {
let (sh_offset, sh_size) = (shdr.sh_offset.get(), shdr.sh_size.get());
let start = sh_offset as usize;
let end = start.saturating_add(sh_size as usize);
if end > data.len() {
if end > self.data().len() {
fatal!("{display}: section header is out of range: {sh_offset}");
}
&data[start..end]
start..end
}
pub fn find_section(&self, sh_type: u32) -> Option<usize> {
@@ -597,7 +581,9 @@ pub struct ComdatGroupRef {
pub sect_idx: u32,
// The high bit records ownership; symbol IDs occupy the remaining bits.
signature_and_owner: u32,
// An exceptional signature is interned after the group is created and
// stored by the symbol map's shard, so the word is atomic.
signature_and_owner: AtomicU32,
}
impl ComdatGroupRef {
@@ -605,34 +591,35 @@ impl ComdatGroupRef {
fn new(sect_idx: u32, signature: SymbolId) -> Self {
assert!(signature.0 < Self::IS_OWNER);
Self { sect_idx, signature_and_owner: signature.0 }
Self { sect_idx, signature_and_owner: AtomicU32::new(signature.0) }
}
#[inline]
pub fn signature(&self) -> SymbolId {
SymbolId(self.signature_and_owner & !Self::IS_OWNER)
SymbolId(self.signature_and_owner.load(Ordering::Relaxed) & !Self::IS_OWNER)
}
#[inline]
pub fn is_owner(&self) -> bool {
self.signature_and_owner & Self::IS_OWNER != 0
self.signature_and_owner.load(Ordering::Relaxed) & Self::IS_OWNER != 0
}
#[inline]
pub fn set_owner(&mut self, is_owner: bool) {
let word = self.signature_and_owner.get_mut();
if is_owner {
self.signature_and_owner |= Self::IS_OWNER;
*word |= Self::IS_OWNER;
} else {
self.signature_and_owner &= !Self::IS_OWNER;
*word &= !Self::IS_OWNER;
}
}
/// Returns the word that receives an exceptional signature while COMDAT
/// metadata is gathered. Ownership has not been selected at that point.
/// Stores an exceptional signature while COMDAT metadata is gathered.
/// Ownership has not been selected at that point.
#[inline]
pub(crate) fn signature_word_mut(&mut self) -> &mut u32 {
debug_assert!(!self.is_owner());
&mut self.signature_and_owner
pub(crate) fn set_signature(&self, signature: SymbolId) {
assert!(signature.0 < Self::IS_OWNER);
self.signature_and_owner.store(signature.0, Ordering::Relaxed);
}
}
@@ -665,22 +652,22 @@ pub struct RiscvAttributes {
/// so those writes are disjoint even though the files themselves are shared
/// by the parallel copy tasks.
struct DecodedRelocations<R> {
records: UnsafeCell<Box<[R]>>,
records: SyncUnsafeCell<Box<[R]>>,
}
impl<R> DecodedRelocations<R> {
fn new(records: Box<[R]>) -> Self {
Self { records: UnsafeCell::new(records) }
Self { records: SyncUnsafeCell::new(records) }
}
fn as_slice(&self) -> &[R] {
// SAFETY: mutable access is restricted to exclusive linker phases and
// disjoint relocation sections.
unsafe { (&*self.records.get()).as_ref() }
unsafe { &*self.records.get() }
}
fn as_mut_slice(&mut self) -> &mut [R] {
self.records.get_mut().as_mut()
self.records.get_mut()
}
/// Gives `f` mutable access through a shared file reference.
@@ -691,7 +678,7 @@ impl<R> DecodedRelocations<R> {
/// returns.
unsafe fn with_mut_slice(&self, f: impl FnOnce(&mut [R])) {
// SAFETY: the caller upholds the exclusive-access requirement.
f(unsafe { (&mut *self.records.get()).as_mut() });
f(unsafe { &mut *self.records.get() });
}
fn is_empty(&self) -> bool {
@@ -705,10 +692,6 @@ impl<R: fmt::Debug> fmt::Debug for DecodedRelocations<R> {
}
}
// SAFETY: mutable access is permitted only for disjoint relocation tables
// owned by separate copy tasks, as documented by with_mut_slice().
unsafe impl<R: Send + Sync> Sync for DecodedRelocations<R> {}
/// Whether an object is native code, plugin input, or plugin output.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum ObjectOrigin {
@@ -842,10 +825,7 @@ impl<E: Target> ObjectFile<E> {
#[inline(always)]
fn input_relocation_data(&self, relsec_idx: u32) -> &'static [u8] {
// Relocation sections are range-checked when sections are parsed.
let shdr = &self.base.shdrs[relsec_idx as usize];
let (offset, size) = (shdr.sh_offset.get(), shdr.sh_size.get());
&self.base.data()[offset as usize..(offset + size) as usize]
self.base.section_contents_from_shdr(&self.base.shdrs[relsec_idx as usize])
}
}
@@ -1028,15 +1008,7 @@ impl<E: Target> ExactSizeIterator for RelocationIter<'_, E> {}
//
// This function converts a CREL relocation table to a regular one.
fn decode_crel<E: Target>(file: &dyn fmt::Display, data: &[u8]) -> Box<[ElfRel<E>]> {
let reader = CrelReader::<E>::new(file, data);
// Own a fixed-size array without value-initializing trivial elements
// that the caller is about to overwrite.
let mut rels = Box::<[ElfRel<E>]>::new_uninit_slice(reader.len());
for (i, rel) in reader.enumerate() {
rels[i].write(rel);
}
// SAFETY: CrelReader visits every index from zero to len once.
unsafe { rels.assume_init() }
CrelReader::<E>::new(file, data).collect()
}
impl<E: Target> ObjectFile<E> {
@@ -1177,11 +1149,7 @@ impl<E: Target> ObjectFile<E> {
#[inline]
fn section_with_id(&self, shndx: usize) -> Option<(InputSectionId, &InputSection<E>)> {
let index = self.sections.input_index(shndx)?;
Some((
InputSectionId::new(self.id(), index),
// SAFETY: `input_index` only returns indices assigned by insertion.
unsafe { self.sections.input_unchecked(index as usize) },
))
Some((InputSectionId::new(self.id(), index), self.sections.input(index as usize)))
}
#[inline]
@@ -1240,15 +1208,11 @@ impl<E: Target> ObjectFile<E> {
}
let base = &self.base;
let shdr = &base.shdrs[index];
let (offset, size) = (shdr.sh_offset.get(), shdr.sh_size.get());
let range = base.section_range(&base.shdrs[index]);
let mf = base.mf.expect("input relocations without a mapped file");
// SAFETY: a mutable ObjectFile owns this relocation section for the
// duration of the pass, and section parsing checked its range.
let data = unsafe { mf.data_mut_ptr(offset as usize..(offset + size) as usize) };
// SAFETY: the same exclusive ownership applies while the returned
// relocation view is alive.
rels_from_bytes_mut::<E>(unsafe { &mut *data })
// SAFETY: a mutable ObjectFile owns this relocation section while
// the returned relocation view is alive.
rels_from_bytes_mut::<E>(unsafe { mf.data_mut(range) })
}
/// Gives `f` mutable access to a relocation table during the copy phase.
@@ -1276,13 +1240,11 @@ impl<E: Target> ObjectFile<E> {
}
let base = &self.base;
let shdr = &base.shdrs[index];
let (offset, size) = (shdr.sh_offset.get(), shdr.sh_size.get());
let range = base.section_range(&base.shdrs[index]);
let mf = base.mf.expect("input relocations without a mapped file");
// SAFETY: the caller exclusively owns this checked relocation range.
let data = unsafe { mf.data_mut_ptr(offset as usize..(offset + size) as usize) };
// SAFETY: the exclusive access lasts until f returns.
f(rels_from_bytes_mut::<E>(unsafe { &mut *data }));
// SAFETY: the caller exclusively owns this relocation section until
// f returns.
f(rels_from_bytes_mut::<E>(unsafe { mf.data_mut(range) }));
}
fn set_decoded_crel(&mut self, index: usize, rels: Box<[ElfRel<E>]>) {
@@ -1340,11 +1302,7 @@ impl<E: Target> ObjectFile<E> {
/// group section as they are needed.
#[inline]
pub fn comdat_members(&self, group: &ComdatGroupRef) -> impl Iterator<Item = u32> + '_ {
let data = self.base.data();
let shdr = &self.base.shdrs[group.sect_idx as usize];
let (sh_offset, sh_size) = (shdr.sh_offset.get(), shdr.sh_size.get());
let start = sh_offset as usize;
let bytes = &data[start..start + sh_size as usize];
let bytes = self.base.section_contents_from_shdr(&self.base.shdrs[group.sect_idx as usize]);
let is_little_endian = self.base.is_little_endian;
bytes.as_chunks::<4>().0.iter().skip(1).map(move |b| {
let b = [b[0], b[1], b[2], b[3]];
@@ -1460,9 +1418,7 @@ impl<E: Target> ObjectFile<E> {
pub fn read_section_metadata(&mut self) {
debug_assert!(!self.sections_parsed);
for i in 0..self.num_elf_sections {
// SAFETY: `i` comes from the section-header table's range.
let shdr = unsafe { self.base.shdrs.get_unchecked(i) };
for (i, shdr) in self.base.shdrs.iter().enumerate() {
let (sh_type, sh_flags) = (shdr.sh_type.get(), shdr.sh_flags.get());
if sh_flags & SHF_EXCLUDE as u64 != 0
@@ -1619,20 +1575,14 @@ impl<E: Target> ObjectFile<E> {
fn initialize_sections(&mut self, args: &Args, id: ObjId) {
// Read sections
let nsections = self.num_elf_sections;
let shdrs = self.base.shdrs;
let expected_reloc_type = if E::IS_RELA { SHT_RELA } else { SHT_REL };
debug_assert!(self.comdat_discarded.is_empty() || self.comdat_discarded.len() == nsections);
for i in 0..nsections {
if !self.comdat_discarded.is_empty()
// SAFETY: a nonempty COMDAT bitmap has one entry per input
// section, and `i` is in that range.
&& unsafe { *self.comdat_discarded.get_unchecked(i) }
{
for (i, shdr) in shdrs.iter().enumerate() {
// The COMDAT bitmap is empty if no group has been discarded.
if self.comdat_discarded.get(i).is_some_and(|&discarded| discarded) {
continue;
}
// SAFETY: `i` comes from the file's section-header range.
let shdr = unsafe { self.base.shdrs.get_unchecked(i) };
let (sh_type, flags) = (shdr.sh_type.get(), shdr.sh_flags.get());
if flags & SHF_EXCLUDE as u64 != 0
&& flags & SHF_ALLOC as u64 == 0
@@ -1818,9 +1768,7 @@ impl<E: Target> ObjectFile<E> {
}
// Attach relocation sections to their target sections.
for i in 0..nsections {
// SAFETY: `i` comes from the file's section-header range.
let shdr = unsafe { self.base.shdrs.get_unchecked(i) };
for (i, shdr) in shdrs.iter().enumerate() {
let sh_type = shdr.sh_type.get();
if sh_type != expected_reloc_type && sh_type != SHT_CREL {
continue;
@@ -2410,17 +2358,12 @@ impl<E: Target> ObjectFile<E> {
Some(data) => data.as_mut_slice(),
None => {
let base = &self.base;
let shdr = &base.shdrs[relsec_idx];
let (offset, size) = (shdr.sh_offset.get(), shdr.sh_size.get());
let range = base.section_range(&base.shdrs[relsec_idx]);
let mf = base.mf.expect("input relocations without a mapped file");
// SAFETY: this file-parallel pass exclusively owns the
// relocation section, whose range was checked at parse
// time. No overlapping shared slice is used here.
let data =
unsafe { mf.data_mut_ptr(offset as usize..(offset + size) as usize) };
// SAFETY: the exclusive access described above lasts
// until this relocation view is dropped.
rels_from_bytes_mut::<E>(unsafe { &mut *data })
// relocation section while this view is alive, and uses
// no overlapping shared slice.
rels_from_bytes_mut::<E>(unsafe { mf.data_mut(range) })
}
};
@@ -3407,34 +3350,26 @@ pub fn resolved_symbol_rank(sym: &Symbol, is_dso: bool, is_in_archive: bool, pri
/// An exclusive view of the symbol table shared by file-parallel passes.
/// Individual symbols are serialized by their byte-sized spin locks.
pub struct SymbolEditor<'a> {
symbols: *mut Symbol,
len: usize,
_symbols: PhantomData<&'a mut [Symbol]>,
symbols: &'a [SyncUnsafeCell<Symbol>],
}
// The exclusive slice represented by `symbols` cannot be accessed except
// through `with_symbol`, which serializes accesses to each element.
unsafe impl Sync for SymbolEditor<'_> {}
impl<'a> SymbolEditor<'a> {
pub(crate) fn new(symbols: &'a mut [Symbol]) -> Self {
SymbolEditor { symbols: symbols.as_mut_ptr(), len: symbols.len(), _symbols: PhantomData }
SymbolEditor { symbols: SyncUnsafeCell::from_mut(symbols) }
}
#[inline]
pub(crate) fn with_symbol<R>(&self, id: SymbolId, f: impl FnOnce(&mut Symbol) -> R) -> R {
debug_assert!(id.index() < self.len);
// SAFETY: the editor owns an exclusive borrow of the whole table,
// and Symbol::with_resolution_lock serializes accesses to this slot.
unsafe { Symbol::with_resolution_lock(self.symbols.add(id.index()), f) }
unsafe { Symbol::with_resolution_lock(self.symbols[id.index()].get(), f) }
}
#[inline]
pub(crate) fn skip_dso(&self, id: SymbolId) -> bool {
debug_assert!(id.index() < self.len);
// SAFETY: the editor owns a live symbol array; skip_dso_at reads only
// the atomic state byte shared with the resolution lock.
unsafe { Symbol::skip_dso_at(self.symbols.add(id.index())) }
// SAFETY: skip_dso_at reads only the atomic state byte shared with
// the resolution lock.
unsafe { Symbol::skip_dso_at(self.symbols[id.index()].get()) }
}
}
@@ -3446,9 +3381,6 @@ pub struct SymbolResolver<'a, E: Target> {
default_version: u16,
}
// SymbolResolver's mutable symbol-table access is serialized by its editor.
unsafe impl<E: Target> Sync for SymbolResolver<'_, E> {}
impl<'a, E: Target> SymbolResolver<'a, E> {
pub fn new(
symbols: &'a mut [Symbol],
src/input_sections.rs
+2 -16
View File
@@ -1675,18 +1675,6 @@ impl<E: Target> SectionList<E> {
&self.inputs[index]
}
/// Returns an input section without checking its dense index.
///
/// # Safety
///
/// `index` must have been assigned by this list's `insert` method.
#[inline]
pub(crate) unsafe fn input_unchecked(&self, index: usize) -> &InputSection<E> {
debug_assert!(index < self.inputs.len());
// SAFETY: guaranteed by the caller.
unsafe { self.inputs.get_unchecked(index) }
}
/// The number of section indices.
#[inline]
pub(crate) fn len(&self) -> usize {
@@ -1728,15 +1716,13 @@ impl<E: Target> SectionList<E> {
#[inline]
pub fn section(&self, shndx: usize) -> Option<&InputSection<E>> {
let index = self.input_index(shndx)?;
// SAFETY: table entries contain only indices assigned by `insert`.
Some(unsafe { self.input_unchecked(index as usize) })
Some(&self.inputs[index as usize])
}
#[inline]
pub fn section_mut(&mut self, shndx: usize) -> Option<&mut InputSection<E>> {
let index = self.input_index(shndx)?;
// SAFETY: table entries contain only indices assigned by `insert`.
Some(unsafe { self.inputs.get_unchecked_mut(index as usize) })
Some(&mut self.inputs[index as usize])
}
#[inline]
src/mapped_file.rs
+21 -36
View File
@@ -12,7 +12,6 @@ use std::fs::File;
use std::io::{self, Read};
use std::ops::Range;
use std::path::{Path, PathBuf};
use std::ptr::NonNull;
use std::sync::Mutex;
use std::sync::atomic::{AtomicBool, Ordering};
@@ -21,6 +20,7 @@ use rayon::prelude::*;
use crate::fatal;
use crate::util;
use crate::util::SyncUnsafeCell;
/// All files opened during this link.
static FILE_POOL: Mutex<Vec<&'static MappedFile>> = Mutex::new(Vec::new());
@@ -73,10 +73,10 @@ const READ_THRESHOLD: u64 = 32 * 1024;
#[derive(Debug)]
pub struct MappedFile {
pub name: PathBuf,
/// A byte range whose allocation is deliberately leaked with the input file.
/// Keeping a raw slice pointer lets the linker modify private relocation
/// records in place, as C++ mold does.
pub(crate) data: NonNull<[u8]>,
/// The contents, in an allocation that is deliberately leaked with the
/// input file. The bytes are cells because the linker modifies private
/// relocation records in place, as C++ mold does.
pub(crate) data: &'static [SyncUnsafeCell<u8>],
/// False if the file was found by searching library paths (`-l`),
/// which affects how a shared library's soname defaults.
@@ -92,11 +92,6 @@ pub struct MappedFile {
pub is_dependency: AtomicBool,
}
// The bytes are normally read concurrently. Mutable access is restricted to
// disjoint relocation ranges owned by file-parallel passes.
unsafe impl Send for MappedFile {}
unsafe impl Sync for MappedFile {}
impl MappedFile {
fn open_impl(path: &Path) -> io::Result<&'static Self> {
let file = File::open(path)?;
@@ -108,8 +103,8 @@ impl MappedFile {
// True if `data` is a memory mapping of the file rather than a copy of
// its contents in anonymous memory. See open_file_impl().
let mut is_mmapped = false;
let data = if size == 0 {
NonNull::slice_from_raw_parts(NonNull::dangling(), 0)
let data: &'static [SyncUnsafeCell<u8>] = if size == 0 {
&[]
} else if size <= READ_THRESHOLD && metadata.is_file() {
let mut buf = Vec::with_capacity(size as usize);
(&file)
@@ -119,7 +114,7 @@ impl MappedFile {
if buf.len() as u64 != size {
fatal!("{display}: file is shorter than its reported size");
}
NonNull::from(Vec::leak(buf))
SyncUnsafeCell::from_mut(Vec::leak(buf))
} else {
// C++ mold maps inputs MAP_PRIVATE with write permission so it
// can redirect ordinary relocation records in place without
@@ -133,7 +128,7 @@ impl MappedFile {
let map = unsafe { memmap2::MmapOptions::new().len(map_len).map_copy(&file) }
.unwrap_or_else(|e| fatal!("{display}: mmap failed: {e}"));
is_mmapped = true;
NonNull::from(Box::leak(Box::new(map)).as_mut())
SyncUnsafeCell::from_mut(Box::leak(Box::new(map)).as_mut())
};
let mf = util::leak(Self {
@@ -169,12 +164,9 @@ impl MappedFile {
/// Returns a view of a member of this archive.
pub fn slice(&'static self, name: PathBuf, start: usize, size: usize) -> &'static Self {
assert!(start <= self.size() && size <= self.size() - start);
// SAFETY: the checked range lies in the same allocation.
let data = unsafe { self.data.cast::<u8>().add(start) };
let mf = util::leak(Self {
name,
data: NonNull::slice_from_raw_parts(data, size),
data: &self.data[start..start + size],
given_fullpath: true,
parent: Some(self),
thin_parent: None,
@@ -203,35 +195,28 @@ impl MappedFile {
#[inline]
pub fn data(&self) -> &[u8] {
// SAFETY: the allocation is leaked and the range was checked when
// this file or archive member was created. Mutable accesses are to
// relocation ranges during exclusive linker phases.
unsafe { self.data.as_ref() }
// SAFETY: the only writes are to relocation ranges during exclusive
// linker phases, which use no shared view of those ranges.
unsafe { SyncUnsafeCell::as_slice(self.data) }
}
/// Returns a mutable raw pointer to a relocation range.
/// Returns a relocation range for modification.
///
/// # Safety
///
/// The caller must have exclusive access to `range` while dereferencing
/// the pointer, and no shared reference to an overlapping range may be
/// used during that time.
pub(crate) unsafe fn data_mut_ptr(&self, range: Range<usize>) -> *mut [u8] {
assert!(range.start <= range.end && range.end <= self.size());
// SAFETY: the range is in bounds. Dereferencing remains the caller's
// responsibility under the contract above.
std::ptr::slice_from_raw_parts_mut(
unsafe { self.data.cast::<u8>().as_ptr().add(range.start) },
range.end - range.start,
)
/// The caller must have exclusive access to `range` while the slice is
/// alive, and no shared reference to an overlapping range may be used
/// during that time.
pub(crate) unsafe fn data_mut(&self, range: Range<usize>) -> &mut [u8] {
// SAFETY: guaranteed by the caller.
unsafe { SyncUnsafeCell::as_mut_slice(&self.data[range]) }
}
/// The offset of this slice within its top-level archive file.
pub fn offset(&self) -> usize {
match self.parent {
Some(parent) => {
let base = parent.data.cast::<u8>().as_ptr() as usize;
self.data.cast::<u8>().as_ptr() as usize - base + parent.offset()
self.data.as_ptr() as usize - parent.data.as_ptr() as usize + parent.offset()
}
None => 0,
}
src/passes.rs
+25 -80
View File
@@ -4,7 +4,6 @@ use crate::util::worker_local::WorkerLocal;
use std::borrow::Cow;
use std::collections::{HashMap, HashSet};
use std::fmt::Write as _;
use std::ptr::NonNull;
use std::sync::atomic::Ordering;
use std::sync::{Mutex, RwLock};
@@ -28,8 +27,8 @@ use crate::cmdline::{
use crate::context::Context;
use crate::elf::*;
use crate::input_files::{
ComdatGroupRef, FileId, FileList, ObjId, ObjectFile, ObjectOrigin, SymbolEditor,
SymbolResolver, resolved_symbol_rank, symbol_resolution_rank,
FileId, FileList, ObjId, ObjectFile, ObjectOrigin, SymbolEditor, SymbolResolver,
resolved_symbol_rank, symbol_resolution_rank,
};
use crate::input_sections::{InputSection, InputSectionId, SectionRef};
use crate::linker_script::VersionPattern;
@@ -459,42 +458,6 @@ fn resolve_skip_dso_symbols_pass<E: Target>(ctx: &mut Context<E>) {
});
}
/// An exceptional COMDAT signature whose Symbol slot is filled by gathering.
/// The owning ObjectFile and its group vector stay in place through the
/// gather, just as they do for the files' ordinary SymbolSlots.
struct PendingComdatOwner {
group: NonNull<ComdatGroupRef>,
priority: u32,
is_lto_output: bool,
}
// SAFETY: every pending record names a distinct group, and it is consumed only
// after gathering has stopped writing the group's signature slot.
unsafe impl Send for PendingComdatOwner {}
/// A packed COMDAT signature word that receives the interned SymbolId before
/// ownership is selected.
#[derive(Clone, Copy)]
struct ComdatSymbolSlot(NonNull<u32>);
// SAFETY: each exceptional COMDAT signature has one pending slot, and one
// symbol-map shard writes it before the groups are examined again.
unsafe impl Send for ComdatSymbolSlot {}
unsafe impl Sync for ComdatSymbolSlot {}
impl ComdatSymbolSlot {
fn new(group: &mut ComdatGroupRef) -> Self {
Self(NonNull::from(group.signature_word_mut()))
}
fn assign(self, id: SymbolId) {
assert!(id.0 < 1 << 31);
// SAFETY: guaranteed by the construction and synchronization rules
// documented on ComdatSymbolSlot.
unsafe { self.0.write(id.0) };
}
}
// Select COMDAT groups and construct input sections. If LTO will run,
// the first invocation also constructs the losing copies of COMDAT
// members because this function runs again after LTO and may then
@@ -508,7 +471,7 @@ fn parse_input_sections<E: Target>(ctx: &mut Context<E>) {
// Ordinary global signatures already refer to the files' symbols; record
// the other signatures for interning while each file's metadata is hot.
let t = ctx.timer("read_section_metadata");
let (bins, pending): (Vec<Bins<ComdatSymbolSlot>>, Vec<Vec<PendingComdatOwner>>) = {
let (bins, pending): (Vec<Bins<(ObjId, u32)>>, Vec<Vec<(ObjId, u32)>>) = {
let Context { objs, symbols, .. } = ctx;
// Reuse each worker's buffers across jobs, locking once per file.
let work = WorkerLocal::new(|| (Bins::new(), Vec::new()));
@@ -532,20 +495,11 @@ fn parse_input_sections<E: Target>(ctx: &mut Context<E>) {
}
}
}
let signatures = &file.pending_comdat_signatures;
let groups = &mut file.comdat_groups;
for signature in signatures {
let group = &mut groups[signature.group_idx as usize];
bins.record(
signature.key,
signature.name_len as usize,
ComdatSymbolSlot::new(group),
);
pending.push(PendingComdatOwner {
group: NonNull::from(group),
priority,
is_lto_output,
});
// Exceptional signatures name their group by file and index.
for signature in &file.pending_comdat_signatures {
let group = (file.id(), signature.group_idx);
bins.record(signature.key, signature.name_len as usize, group);
pending.push(group);
}
}
});
@@ -558,19 +512,18 @@ fn parse_input_sections<E: Target>(ctx: &mut Context<E>) {
let t = ctx.timer("comdat_signatures");
{
let Context { objs, symbols, .. } = ctx;
symbols.gather(bins, 0, ComdatSymbolSlot::assign);
symbols.gather(bins, 0, |(file, group_idx): (ObjId, u32), id| {
objs[file.index()].comdat_groups[group_idx as usize].set_signature(id);
});
// Signatures just interned could not participate in the metadata
// traversal above. Record them now.
let symbols: &crate::symbol::SymbolTable = symbols;
pending.into_par_iter().flatten().for_each(|pending| {
// SAFETY: each pending pointer came from a distinct group that
// stays in place, and symbol gathering has joined before this
// parallel traversal begins.
let group = unsafe { &*pending.group.as_ptr() };
let sym = &symbols[group.signature()];
if !sym.comdat_claimed_by_ir() || pending.is_lto_output {
sym.record_comdat_owner(pending.priority);
pending.into_par_iter().flatten().for_each(|(file, group_idx)| {
let file = &objs[file.index()];
let sym = &symbols[file.comdat_groups[group_idx as usize].signature()];
if !sym.comdat_claimed_by_ir() || file.origin == ObjectOrigin::LtoOutput {
sym.record_comdat_owner(file.base.priority);
}
});
objs.par_iter_mut().for_each(|file| file.pending_comdat_signatures = Vec::new());
@@ -2504,9 +2457,7 @@ pub fn scan_relocations<E: Target>(ctx: &mut Context<E>) {
// Every dynamic symbol gets its auxiliary record. The loop below
// assigns table entries in order and so runs on one thread; the
// records are allocated beforehand in the side vector.
// SAFETY: every group was filtered to symbols owned by its input file,
// and each input file has a unique id.
unsafe { ctx.symbols.allocate_aux(&groups) };
ctx.symbols.allocate_aux(&groups);
let mut syms = Vec::with_capacity(groups.iter().map(Vec::len).sum());
syms.extend(groups.into_iter().flatten());
@@ -2662,12 +2613,9 @@ pub fn sort_dynsyms<E: Target>(ctx: &mut Context<E>) {
Entry { bucket: hash % num_buckets, name, id, hash }
})
.collect();
// SAFETY: .dynsym contains each symbol once and every symbol has aux.
unsafe {
ctx.symbols.par_for_each_aux_mut(exported, |i, _, aux| {
aux.djb_hash = entries[i].hash;
});
}
ctx.symbols.par_for_each_aux(exported, |i, _, aux| {
aux.djb_hash.store(entries[i].hash, Ordering::Relaxed);
});
entries.par_sort_unstable_by(|a, b| (a.bucket, a.name).cmp(&(b.bucket, b.name)));
exported
.par_iter_mut()
@@ -2682,14 +2630,11 @@ pub fn sort_dynsyms<E: Target>(ctx: &mut Context<E>) {
}
let offset = ctx.dynstr.hdr.shdr.sh_size.get();
// SAFETY: .dynsym contains each symbol once and every symbol has aux.
unsafe {
ctx.symbols.par_for_each_aux_mut(&syms, |i, _, aux| {
let idx = (i as u32).checked_add(1).unwrap();
assert_ne!(idx, u32::MAX);
aux.dynsym_idx = idx;
});
}
ctx.symbols.par_for_each_aux(&syms, |i, _, aux| {
let idx = (i as u32).checked_add(1).unwrap();
assert_ne!(idx, u32::MAX);
aux.dynsym_idx.store(idx, Ordering::Relaxed);
});
dynstr_entries[1..first_exported + 1]
.par_iter_mut()
.zip(&syms[..first_exported])
src/symbol.rs
+71 -161
View File
@@ -6,11 +6,10 @@
use std::fmt;
use std::hash::{BuildHasherDefault, Hash, Hasher};
use std::marker::PhantomData;
use std::mem::MaybeUninit;
use std::ops::{Index, IndexMut, Range};
use std::ptr::NonNull;
use std::sync::atomic::{AtomicPtr, AtomicU8, AtomicU32, AtomicUsize, Ordering};
use std::sync::atomic::{AtomicU8, AtomicU32, AtomicUsize, Ordering};
// Atomic accesses use relaxed ordering unless stronger synchronization is
// required, matching C++ mold's default atomic wrapper.
@@ -25,6 +24,7 @@ use crate::error::demangle_enabled;
use crate::input_files::FileId;
use crate::input_sections::{FragmentRef, InputSection, InputSectionId};
use crate::target::Target;
use crate::util::SyncUnsafeCell;
use crate::util::demangle::{demangle_cpp, demangle_rust};
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Hash, PartialOrd, Ord)]
@@ -164,9 +164,11 @@ pub struct SymbolAux {
pub tlsdesc_idx: u32,
pub plt_idx: u32,
pub pltgot_idx: u32,
pub dynsym_idx: u32,
pub opd_idx: u32,
pub djb_hash: u32,
// The .dynsym index and the GNU hash of the name are assigned to all
// dynamic symbols in parallel through shared records.
pub dynsym_idx: AtomicU32,
pub djb_hash: AtomicU32,
// For range extension thunks
pub thunk_addrs: Vec<u64>,
}
@@ -183,9 +185,9 @@ impl Default for SymbolAux {
tlsdesc_idx: u32::MAX,
plt_idx: u32::MAX,
pltgot_idx: u32::MAX,
dynsym_idx: u32::MAX,
opd_idx: u32::MAX,
djb_hash: 0,
dynsym_idx: AtomicU32::new(u32::MAX),
djb_hash: AtomicU32::new(0),
thunk_addrs: Vec::new(),
}
}
@@ -275,8 +277,9 @@ pub struct Symbol {
pub flags: AtomicU8,
// Index into SymbolTable's side array of auxiliary data, allocated on
// demand for dynamic symbols.
aux_idx: u32,
// demand for dynamic symbols. Records are claimed in parallel, one file
// at a time, through shared Symbol references.
aux_idx: AtomicU32,
/// The symbol's boolean attributes, packed; the accessors below name
/// them.
@@ -490,7 +493,7 @@ impl Symbol {
ver_idx: VER_NDX_UNSPECIFIED as u16,
visibility: AtomicU8::new(STV_DEFAULT as u8),
flags: AtomicU8::new(0),
aux_idx: NO_AUX,
aux_idx: AtomicU32::new(NO_AUX),
bits: 0,
}
}
@@ -733,7 +736,8 @@ impl Symbol {
#[inline]
pub fn aux<'a>(&self, symbols: &'a SymbolTable) -> Option<&'a SymbolAux> {
(self.aux_idx != NO_AUX).then(|| &symbols.aux[self.aux_idx as usize])
let index = self.aux_idx.load(Ordering::Relaxed);
(index != NO_AUX).then(|| &symbols.aux[index as usize])
}
pub fn got_idx(&self, symbols: &SymbolTable) -> Option<u32> {
@@ -769,7 +773,7 @@ impl Symbol {
}
pub fn dynsym_idx(&self, symbols: &SymbolTable) -> Option<u32> {
let idx = self.aux(symbols)?.dynsym_idx;
let idx = self.aux(symbols)?.dynsym_idx.load(Ordering::Relaxed);
(idx != u32::MAX).then_some(idx)
}
@@ -1401,11 +1405,9 @@ impl SymbolSlot {
/// Each file allocates its local symbols while it is being parsed. This gives
/// every worker a disjoint range in the central vector.
pub struct ParallelSymbolAllocator<'a> {
slots: AtomicPtr<MaybeUninit<Symbol>>,
slots: &'a [SyncUnsafeCell<MaybeUninit<Symbol>>],
first: usize,
next: AtomicUsize,
maximum: usize,
marker: PhantomData<&'a mut [MaybeUninit<Symbol>]>,
}
impl ParallelSymbolAllocator<'_> {
@@ -1421,70 +1423,14 @@ impl ParallelSymbolAllocator<'_> {
init: impl FnOnce(SymbolId, &mut [MaybeUninit<Symbol>]),
) -> SymbolId {
let offset = self.next.fetch_add(n, Ordering::Relaxed);
let end = offset.checked_add(n).expect("too many symbols");
assert!(end <= self.maximum);
let slots = &self.slots[offset..offset + n];
let base_id = SymbolId((self.first + offset) as u32);
let ptr = self.slots.load(Ordering::Relaxed);
// SAFETY: the table reserved `maximum` slots before publishing the
// pointer, and the atomic bump pointer gives this call an exclusive
// range within them.
let slots = unsafe { std::slice::from_raw_parts_mut(ptr.add(offset), n) };
init(base_id, slots);
// SAFETY: the atomic bump pointer gives this call an exclusive range.
init(base_id, unsafe { SyncUnsafeCell::as_mut_slice(slots) });
base_id
}
}
/// Mutable access to stable symbol storage during a parallel pass.
struct SymbolBlockPtr(*mut Symbol);
// SAFETY: the table holds the vector exclusively and gives each task either
// non-overlapping shard blocks or symbol ids owned by one input file.
unsafe impl Sync for SymbolBlockPtr {}
/// A pointer to the auxiliary vector during a parallel scatter to unique symbols.
struct SymbolAuxPtr(*mut SymbolAux);
// SAFETY: the two methods using this wrapper require distinct symbol ids, so
// their parallel tasks access non-overlapping auxiliary records.
unsafe impl Sync for SymbolAuxPtr {}
impl SymbolAuxPtr {
/// Applies `f` to the record at `index`.
///
/// # Safety
/// The caller must have exclusive access to this record.
unsafe fn with_mut<R>(&self, index: usize, f: impl FnOnce(&mut SymbolAux) -> R) -> R {
// SAFETY: the caller supplies a valid, exclusively owned index.
f(unsafe { &mut *self.0.add(index) })
}
}
impl SymbolBlockPtr {
/// Mutates only a symbol's auxiliary index.
///
/// # Safety
///
/// `id` must be valid, and no other task may access that symbol's index.
unsafe fn with_aux_index<R>(&self, id: SymbolId, f: impl FnOnce(&mut u32) -> R) -> R {
// SAFETY: the caller provides an exclusive, valid symbol id.
unsafe { f(&mut (*self.0.add(id.index())).aux_idx) }
}
/// Applies `f` to the symbols in non-overlapping vector ranges.
///
/// # Safety
/// The ranges must be initialized and exclusively owned by this task.
unsafe fn for_each(&self, ranges: &[Range<u32>], f: &(impl Fn(&mut Symbol) + Sync)) {
for range in ranges {
let len = (range.end - range.start) as usize;
// SAFETY: guaranteed by the caller for every shard block.
let symbols =
unsafe { std::slice::from_raw_parts_mut(self.0.add(range.start as usize), len) };
symbols.iter_mut().for_each(&f);
}
}
}
/// Asks the operating system to back the interior pages of an allocation with
/// transparent huge pages when that advice is available.
fn madvise_hugepage<T>(values: &Vec<T>) {
@@ -1585,44 +1531,39 @@ impl SymbolTable {
/// vector if this symbol has none. C++ mold likewise keeps this rarely
/// used state outside `Symbol` and refers to it with an index.
pub fn aux_mut(&mut self, id: SymbolId) -> &mut SymbolAux {
let mut index = self.symbols[id.index()].aux_idx;
if index == NO_AUX {
index = self.aux.len() as u32;
assert_ne!(index, NO_AUX, "too many symbol auxiliary records");
let aux_idx = self.symbols[id.index()].aux_idx.get_mut();
if *aux_idx == NO_AUX {
*aux_idx = self.aux.len() as u32;
assert_ne!(*aux_idx, NO_AUX, "too many symbol auxiliary records");
self.aux.push(SymbolAux::default());
self.symbols[id.index()].aux_idx = index;
}
&mut self.aux[index as usize]
&mut self.aux[*aux_idx as usize]
}
/// Allocates records in each file's first-use order, without a global
/// symbol sort. Each group contains symbols owned by that file, so new
/// symbols cannot occur in two groups. Repeated ids within a group share
/// one record, and existing auxiliary data is preserved.
///
/// # Safety
///
/// All ids must be valid and no symbol may occur in more than one group.
pub(crate) unsafe fn allocate_aux(&mut self, groups: &[Vec<SymbolId>]) {
/// symbol sort. Repeated ids share one record, and existing auxiliary
/// data is preserved.
pub(crate) fn allocate_aux(&mut self, groups: &[Vec<SymbolId>]) {
let first = self.aux.len();
let count: usize = groups.iter().map(Vec::len).sum();
assert!(first + count < NO_AUX as usize);
let symbols = SymbolBlockPtr(self.symbols.as_mut_ptr());
let symbols = &self.symbols;
let fresh: Vec<Vec<SymbolId>> = groups
.par_iter()
.map(|ids| {
let mut fresh = Vec::new();
for &id in ids {
// SAFETY: this group is the only task accessing these ids.
unsafe {
symbols.with_aux_index(id, |index| {
if *index == NO_AUX {
// Suppress repeats within the file. Records are
// read only after final indices are assigned.
*index = 0;
fresh.push(id);
}
});
// Claim the record for the group that sees the symbol
// first. Records are read only after final indices are
// assigned.
let claimed = symbols[id.index()].aux_idx.compare_exchange(
NO_AUX,
0,
Ordering::Relaxed,
Ordering::Relaxed,
);
if claimed.is_ok() {
fresh.push(id);
}
}
fresh
@@ -1642,11 +1583,7 @@ impl SymbolTable {
.collect();
parts.into_par_iter().for_each(|(first, ids)| {
for (i, id) in ids.into_iter().enumerate() {
// SAFETY: groups own disjoint ids, and each fresh id was
// recorded only once within its group.
unsafe {
symbols.with_aux_index(id, |index| *index = (first + i) as u32);
}
symbols[id.index()].aux_idx.store((first + i) as u32, Ordering::Relaxed);
}
});
}
@@ -1685,9 +1622,8 @@ impl SymbolTable {
if self.symbols.capacity() != old_capacity {
madvise_hugepage(&self.symbols);
}
let capacity = self.symbols.capacity();
let storage = AtomicPtr::new(self.symbols.as_mut_ptr());
let next = AtomicUsize::new(first);
let slots = SyncUnsafeCell::from_mut(self.symbols.spare_capacity_mut());
let next = AtomicUsize::new(0);
let ranges: Vec<Vec<Range<u32>>> = self
.shards
@@ -1696,7 +1632,6 @@ impl SymbolTable {
.map(|(i, shard)| {
let count: usize = bins.iter().map(|bin| bin.0[i].len()).sum();
shard.reserve(count);
let symbols = storage.load(Ordering::Relaxed);
let mut blocks: Vec<(usize, usize)> = Vec::new();
for p in bins.iter().flat_map(|bin| &bin.0[i]) {
let id = match shard.entry(p.key) {
@@ -1704,22 +1639,17 @@ impl SymbolTable {
hashbrown::hash_map::Entry::Vacant(entry) => {
if blocks.last().is_none_or(|&(_, used)| used == BLOCK_SIZE) {
let start = next.fetch_add(BLOCK_SIZE, Ordering::Relaxed);
let end = start.checked_add(BLOCK_SIZE).expect("too many symbols");
assert!(end <= capacity && end < u32::MAX as usize);
assert!(first + start + BLOCK_SIZE < u32::MAX as usize);
blocks.push((start, 0));
}
let (start, used) = blocks.last_mut().unwrap();
let index = *start + *used;
*used += 1;
let id = SymbolId(index as u32);
// SAFETY: reserve keeps `symbols` stable, and the
// atomic bump pointer gives this shard exclusive
// ownership of the slot.
unsafe {
symbols.add(index).write(Symbol::new(BStr::new(
&p.key.key[..p.name_len as usize],
)));
}
let id = SymbolId((first + index) as u32);
let name = BStr::new(&p.key.key[..p.name_len as usize]);
// SAFETY: the atomic bump pointer gives this shard
// exclusive ownership of the slot.
unsafe { (*slots[index].get()).write(Symbol::new(name)) };
entry.insert(id);
id
}
@@ -1732,19 +1662,17 @@ impl SymbolTable {
// SAFETY: these are the unused slots in this shard's
// exclusive block. Initializing them makes the whole
// vector prefix valid while global scans skip them.
unsafe {
symbols.add(index).write(Symbol::new(BStr::new(b"")));
}
unsafe { (*slots[index].get()).write(Symbol::new(BStr::new(b""))) };
}
}
blocks
.into_iter()
.map(|(start, used)| start as u32..(start + used) as u32)
.map(|(start, used)| (first + start) as u32..(first + start + used) as u32)
.collect()
})
.collect();
let len = next.load(Ordering::Relaxed);
let len = first + next.load(Ordering::Relaxed);
// SAFETY: every allocated block, including its unused tail, was
// initialized by its owning shard.
unsafe { self.symbols.set_len(len) };
@@ -1819,20 +1747,13 @@ impl SymbolTable {
}
let allocator = ParallelSymbolAllocator {
// SAFETY: reserve made the entire tail available, even though it
// is outside the vector's initialized length.
slots: AtomicPtr::new(unsafe {
self.symbols.as_mut_ptr().add(first).cast::<MaybeUninit<Symbol>>()
}),
slots: SyncUnsafeCell::from_mut(&mut self.symbols.spare_capacity_mut()[..maximum]),
first,
next: AtomicUsize::new(0),
maximum,
marker: PhantomData,
};
init(&allocator);
let added = allocator.next.load(Ordering::Relaxed);
debug_assert!(added <= maximum);
// SAFETY: every allocated range was initialized before `init`
// returned, and the bump pointer leaves no gaps between ranges.
unsafe { self.symbols.set_len(first + added) };
@@ -1853,36 +1774,26 @@ impl SymbolTable {
/// Applies `f` to all named symbols in parallel, one task per map shard.
pub fn par_for_each_global_mut(&mut self, f: impl Fn(&mut Symbol) + Send + Sync) {
let symbols = SymbolBlockPtr(self.symbols.as_mut_ptr());
let symbols = SyncUnsafeCell::from_mut(&mut self.symbols);
self.globals.par_iter().for_each(|ranges| {
// SAFETY: The vector is exclusively borrowed, every global range
// belongs to exactly one shard, and shard blocks never overlap.
unsafe { symbols.for_each(ranges, &f) };
for range in ranges {
let block = &symbols[range.start as usize..range.end as usize];
// SAFETY: every global range belongs to exactly one shard,
// and shard blocks never overlap.
unsafe { SyncUnsafeCell::as_mut_slice(block) }.iter_mut().for_each(&f);
}
});
}
/// Applies `f` in parallel to symbols and their auxiliary records.
///
/// # Safety
///
/// `ids` must contain no duplicates, since each invocation receives mutable
/// access to the corresponding auxiliary record.
pub(crate) unsafe fn par_for_each_aux_mut(
&mut self,
pub(crate) fn par_for_each_aux(
&self,
ids: &[SymbolId],
f: impl Fn(usize, &Symbol, &mut SymbolAux) + Send + Sync,
f: impl Fn(usize, &Symbol, &SymbolAux) + Send + Sync,
) {
let symbols = &self.symbols;
let aux = SymbolAuxPtr(self.aux.as_mut_ptr());
let aux_len = self.aux.len();
ids.par_iter().enumerate().for_each(|(i, &id)| {
let sym = &symbols[id.index()];
let index = sym.aux_idx;
debug_assert_ne!(index, NO_AUX);
debug_assert!((index as usize) < aux_len);
// SAFETY: the caller guarantees that ids, and hence aux_idx values,
// are distinct, and the exclusive table borrow keeps the vector fixed.
unsafe { aux.with_mut(index as usize, |record| f(i, sym, record)) };
let sym = &self.symbols[id.index()];
f(i, sym, &self.aux[sym.aux_idx.load(Ordering::Relaxed) as usize]);
});
}
@@ -1963,17 +1874,16 @@ mod tests {
let old = table.intern(b"old");
table.aux_mut(old).got_idx = 17;
// SAFETY: the groups contain disjoint, valid symbol ids.
unsafe { table.allocate_aux(&[vec![b, a, b], vec![old, c, c]]) };
assert_eq!(table[old].aux_idx, 0);
assert_eq!(table[b].aux_idx, 1);
assert_eq!(table[a].aux_idx, 2);
assert_eq!(table[c].aux_idx, 3);
table.allocate_aux(&[vec![b, a, b], vec![old, c, c]]);
let aux_idx = |id: SymbolId| table[id].aux_idx.load(Ordering::Relaxed);
assert_eq!(aux_idx(old), 0);
assert_eq!(aux_idx(b), 1);
assert_eq!(aux_idx(a), 2);
assert_eq!(aux_idx(c), 3);
assert_eq!(table.aux.len(), 4);
assert_eq!(table[old].got_idx(&table), Some(17));
// SAFETY: the groups contain disjoint, valid symbol ids.
unsafe { table.allocate_aux(&[vec![a, b], vec![old, c]]) };
table.allocate_aux(&[vec![a, b], vec![old, c]]);
assert_eq!(table.aux.len(), 4);
assert_eq!(table[old].got_idx(&table), Some(17));
}
src/util/concurrent_map.rs
+4 -62
View File
@@ -25,7 +25,7 @@ use rayon::prelude::*;
use std::alloc::{Layout, alloc_zeroed, dealloc};
use std::cell::UnsafeCell;
use std::mem::MaybeUninit;
use std::ptr::{self, NonNull};
use std::ptr;
use std::sync::atomic::{AtomicPtr, Ordering};
pub const NUM_SHARDS: usize = 64;
@@ -96,22 +96,6 @@ unsafe fn deallocate_entries<T>(entries: *mut Entry<T>, bufsize: usize) {
unsafe { dealloc(entries.cast(), layout) };
}
/// A stable reference to an occupied map entry.
pub struct MapEntryRef<T>(NonNull<Entry<T>>);
impl<T> Clone for MapEntryRef<T> {
fn clone(&self) -> Self {
*self
}
}
impl<T> Copy for MapEntryRef<T> {}
// SAFETY: map entries are shared between threads only through atomics and
// values whose synchronization is provided by T.
unsafe impl<T: Send + Sync> Send for MapEntryRef<T> {}
unsafe impl<T: Send + Sync> Sync for MapEntryRef<T> {}
/// The index of a bucket, which identifies an entry for the life of
/// the map.
#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord)]
@@ -357,10 +341,6 @@ impl<T> Default for FrozenMap<T> {
}
impl<T> FrozenMap<T> {
fn entry_ref(&self, id: EntryId) -> MapEntryRef<T> {
MapEntryRef(NonNull::from(self.0.entry(id.0 as usize)))
}
/// The published key of a bucket, if any.
fn key_at(&self, idx: usize) -> Option<&'static [u8]> {
let ent = self.0.entry(idx);
@@ -440,17 +420,12 @@ impl<T> FrozenMap<T> {
vec
}
/// Returns all map entries as stable references in deterministic order.
pub(crate) fn sorted_entry_refs_all(&self) -> Vec<MapEntryRef<T>>
/// Returns all map entries in deterministic order.
pub(crate) fn sorted_entries_all(&self) -> Vec<EntryId>
where
T: Send + Sync,
{
(0..NUM_SHARDS)
.into_par_iter()
.flat_map_iter(|shard| {
self.sorted_entries(shard).into_iter().map(|id| self.entry_ref(id))
})
.collect()
(0..NUM_SHARDS).into_par_iter().flat_map_iter(|shard| self.sorted_entries(shard)).collect()
}
pub(crate) fn len(&self) -> usize {
@@ -458,39 +433,6 @@ impl<T> FrozenMap<T> {
}
}
impl<T> MapEntryRef<T> {
fn entry(self, _owner: &FrozenMap<T>) -> &Entry<T> {
// SAFETY: the reference was made from an occupied bucket in this map,
// and the owner keeps the map's mmap allocation live.
unsafe { self.0.as_ref() }
}
pub(crate) fn value(self, owner: &FrozenMap<T>) -> &T {
// SAFETY: sorted entry references name published entries, whose values
// were initialized before publication.
unsafe { (*self.entry(owner).value.get()).assume_init_ref() }
}
pub(crate) fn value_mut_ptr(self, owner: &FrozenMap<T>) -> *mut T {
self.entry(owner).value.get().cast()
}
pub(crate) fn key(self, owner: &FrozenMap<T>) -> &'static [u8] {
let ent = self.entry(owner);
let key = ent.key.load(Ordering::Acquire);
debug_assert!(!key.is_null() && key != CLAIMED);
// SAFETY: this is a published key whose length was written before the
// key pointer, and map keys remain live for the complete link.
unsafe { std::slice::from_raw_parts(key, *ent.keylen.get() as usize) }
}
pub(crate) fn key_len(self, owner: &FrozenMap<T>) -> usize {
// SAFETY: the entry is published, so its key length is initialized and
// immutable.
unsafe { *self.entry(owner).keylen.get() as usize }
}
}
#[cfg(test)]
mod tests {
use super::*;
src/util/mod.rs
+63
View File
@@ -17,6 +17,69 @@ pub(crate) mod worker_local;
pub(crate) use prefetch::prefetch;
/// An `UnsafeCell` that may be shared between threads, as std's unstable
/// type of the same name. A parallel pass views a slice as cells so that
/// each task can write the elements it owns through a shared reference,
/// with the slice's bounds checks still applied to every access.
#[repr(transparent)]
pub(crate) struct SyncUnsafeCell<T>(std::cell::UnsafeCell<T>);
// SAFETY: the unsafe methods below require callers to keep concurrent
// accesses to one cell disjoint, which is what `Sync` promises.
unsafe impl<T: Send + Sync> Sync for SyncUnsafeCell<T> {}
impl<T> SyncUnsafeCell<T> {
pub(crate) fn new(value: T) -> Self {
Self(std::cell::UnsafeCell::new(value))
}
/// Views an exclusively borrowed slice as shared cells.
pub(crate) fn from_mut(slice: &mut [T]) -> &[Self] {
// SAFETY: a cell has the same layout as its content, and the
// exclusive borrow of the slice is given up for the shared view.
unsafe { &*(std::ptr::from_mut(slice) as *const [Self]) }
}
/// A raw pointer to the content.
pub(crate) fn get(&self) -> *mut T {
self.0.get()
}
pub(crate) fn get_mut(&mut self) -> &mut T {
self.0.get_mut()
}
/// Views cells as their contents.
///
/// # Safety
///
/// No cell in `cells` may be written while the returned slice is alive.
pub(crate) unsafe fn as_slice(cells: &[Self]) -> &[T] {
// SAFETY: a cell has the same layout as its content; the caller
// guarantees that the contents do not change.
unsafe { &*(std::ptr::from_ref(cells) as *const [T]) }
}
/// Views cells as their exclusively borrowed contents.
///
/// # Safety
///
/// No cell in `cells` may be accessed otherwise while the returned slice
/// is alive.
pub(crate) unsafe fn as_mut_slice(cells: &[Self]) -> &mut [T] {
let ptr = std::cell::UnsafeCell::raw_get(cells.as_ptr().cast());
// SAFETY: a cell has the same layout as its content; the caller
// guarantees exclusive access.
unsafe { std::slice::from_raw_parts_mut(ptr, cells.len()) }
}
}
impl<T> std::fmt::Debug for SyncUnsafeCell<T> {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("SyncUnsafeCell").finish_non_exhaustive()
}
}
/// Sets flag bits without taking exclusive ownership of a cache line when
/// all requested bits are already set. Callers needing the previous value
/// must use `fetch_or` directly.