Merge and lay out non-allocated strings in the background

Resolve non-allocated merged sections and assign their offsets in a
background Rayon task while symbol processing, GC and relocation
scanning continue. Give the task owned metadata and immutable input
bytes, create .comment before starting it, and publish the result before
section sorting and sizing. Keep allocated-fragment layout after GC, and
reassign offsets when debug sections are reordered.

Upstream-Commit: f585fee6ca

src/chunks/merged.rs

@@ -18,7 +18,7 @@ use crate::chunks::ChunkHeader;
 use crate::cmdline::Args;
 use crate::context::Context;
 use crate::elf::*;
-use crate::input_sections::{InputSection, MergeInfo, SectionFragment, SectionRef};
+use crate::input_sections::{MergeInfo, SectionFragment, SectionRef};
 use crate::out;
 use crate::output_file::split_at_offsets;
 use crate::util::align_to;
@@ -74,9 +74,9 @@ pub struct MergedSection<E: Layout> {
 /// are resolved in parallel. C++ mold keeps the same pointers in
 /// `MergedSection::members`; Rust keeps the durable section references there
 /// and borrows the stable objects directly for this phase.
-pub struct ResolveMember<'a, E: Arch> {
+pub struct ResolveMember<'a> {
     pub merge_info: &'a mut MergeInfo,
-    pub section: &'a InputSection<E>,
+    pub data: &'static [u8],
     pub filename: &'a str,
     pub archive_name: &'a str,
     pub name: &'static BStr,
@@ -90,6 +90,112 @@ pub struct ResolveOptions<'a> {
     pub timers: &'a Timers,
 }
 
+struct BackgroundMember {
+    reference: SectionRef,
+    info: MergeInfo,
+    data: &'static [u8],
+    filename: std::sync::Arc<str>,
+    archive_name: std::sync::Arc<str>,
+    name: &'static BStr,
+}
+
+/// Owns non-allocated merging state while foreground passes use the context.
+/// Input bytes are immutable and remain mapped for the entire link.
+pub struct BackgroundMerge<E: Layout> {
+    sections: Vec<MergedSection<E>>,
+    members: Vec<Vec<BackgroundMember>>,
+}
+
+impl<E: Arch> BackgroundMerge<E> {
+    pub fn prepare(ctx: &Context<E>) -> Self {
+        let sections: Vec<_> = ctx
+            .merged_sections
+            .iter()
+            .map(|section| {
+                let hdr = &section.hdr;
+                let mut copy = MergedSection::new(
+                    hdr.name,
+                    hdr.shdr.sh_flags.get(),
+                    hdr.shdr.sh_type.get(),
+                    hdr.shdr.sh_entsize.get(),
+                );
+                copy.members = section.members.clone();
+                // Allocated sections have already been resolved in the foreground.
+                copy.resolved = section.resolved;
+                copy
+            })
+            .collect();
+        let mut members: Vec<Vec<BackgroundMember>> =
+            (0..sections.len()).map(|_| Vec::new()).collect();
+        for file in &ctx.objs {
+            let filename: std::sync::Arc<str> = file.base.filename.as_str().into();
+            let archive_name: std::sync::Arc<str> = file.archive_name.as_str().into();
+            for info in file.merge_infos() {
+                if sections[info.parent.index()].resolved {
+                    continue;
+                }
+                let input = file.section_at(info.shndx);
+                members[info.parent.index()].push(BackgroundMember {
+                    reference: SectionRef {
+                        file: file.id(),
+                        shndx: info.shndx,
+                    },
+                    info: info.clone(),
+                    data: input.contents(),
+                    filename: filename.clone(),
+                    archive_name: archive_name.clone(),
+                    name: input.name(file),
+                });
+            }
+        }
+        Self { sections, members }
+    }
+
+    pub fn run(mut self, options: ResolveOptions<'_>) -> Self {
+        let mut members: Vec<Vec<_>> = self
+            .members
+            .iter_mut()
+            .map(|members| {
+                members
+                    .iter_mut()
+                    .map(|member| ResolveMember {
+                        merge_info: &mut member.info,
+                        data: member.data,
+                        filename: &member.filename,
+                        archive_name: &member.archive_name,
+                        name: member.name,
+                    })
+                    .collect()
+            })
+            .collect();
+        resolve_sections(&mut self.sections, &mut members, options);
+        self
+    }
+
+    pub fn finish(self, ctx: &mut Context<E>) {
+        for (i, (mut section, members)) in self.sections.into_iter().zip(self.members).enumerate() {
+            if section.is_alloc() {
+                continue;
+            }
+            let size = section.hdr.shdr.sh_size.get();
+            let align = section.hdr.shdr.sh_addralign.get();
+            // Preserve header bookkeeping done by foreground passes.
+            std::mem::swap(&mut section.hdr, &mut ctx.merged_sections[i].hdr);
+            section.hdr.shdr.sh_size.set(size);
+            section.hdr.shdr.sh_addralign.set(align);
+            ctx.merged_sections[i] = section;
+            for member in members {
+                let file = &mut ctx.objs[member.reference.file.index()];
+                let (info, _) = file
+                    .sections
+                    .merge_info_with_section_mut(member.reference.shndx as usize)
+                    .unwrap();
+                *info = member.info;
+            }
+        }
+    }
+}
+
 struct FileName<'a> {
     filename: &'a str,
     archive_name: &'a str,
@@ -273,7 +379,7 @@ pub fn resolve<E: Arch>(ctx: &mut Context<E>, id: MergedSectionId) {
                     .merge_info_with_section_mut(i as usize)
                     .expect("a mergeable section");
                 let name = isec.name(file);
-                m.split_contents::<E>(file, isec, name, msec, &mut sketch);
+                m.split_contents::<E>(file, isec.contents(), name, msec, &mut sketch);
             }
             file.sections = slots;
             sketch
@@ -301,7 +407,7 @@ pub fn resolve<E: Arch>(ctx: &mut Context<E>, id: MergedSectionId) {
                     .sections
                     .merge_info_with_section_mut(i as usize)
                     .expect("a mergeable section");
-                m.resolve_contents(isec, msec, gc_sections);
+                m.resolve_contents(isec.contents(), msec, gc_sections);
             }
         });
     drop(t);
@@ -325,6 +431,9 @@ pub fn resolve<E: Arch>(ctx: &mut Context<E>, id: MergedSectionId) {
     msec.hdr.shdr.sh_addralign.set(1 << p2align);
     msec.fragments = std::mem::take(&mut msec.map).freeze();
     msec.resolved = true;
+    if !msec.is_alloc() {
+        layout(msec);
+    }
 }
 
 /// Resolves selected merged sections concurrently, as C++ mold does.
@@ -332,7 +441,7 @@ pub fn resolve<E: Arch>(ctx: &mut Context<E>, id: MergedSectionId) {
 /// `MergeInfo`s even when they belong to the same object file.
 pub fn resolve_sections<E: Arch>(
     sections: &mut [MergedSection<E>],
-    members: &mut [Vec<ResolveMember<'_, E>>],
+    members: &mut [Vec<ResolveMember<'_>>],
     options: ResolveOptions<'_>,
 ) {
     let ResolveOptions {
@@ -356,7 +465,7 @@ pub fn resolve_sections<E: Arch>(
                                 filename: member.filename,
                                 archive_name: member.archive_name,
                             },
-                            member.section,
+                            member.data,
                             member.name,
                             section,
                             &mut sketch,
@@ -391,7 +500,7 @@ pub fn resolve_sections<E: Arch>(
             members.par_iter_mut().for_each(|member| {
                 member
                     .merge_info
-                    .resolve_contents(member.section, section, gc_sections);
+                    .resolve_contents(member.data, section, gc_sections);
             });
         });
     drop(t);
@@ -415,6 +524,9 @@ pub fn resolve_sections<E: Arch>(
             section.hdr.shdr.sh_addralign.set(1 << p2align);
             section.fragments = std::mem::take(&mut section.map).freeze();
             section.resolved = true;
+            if !section.is_alloc() {
+                layout(section);
+            }
         });
 }
 
@@ -447,7 +559,9 @@ pub fn compute_section_size<E: Arch>(ctx: &mut Context<E>, id: MergedSectionId)
     if !ctx.merged_sections[id.index()].resolved {
         resolve(ctx, id);
     }
-    layout(&mut ctx.merged_sections[id.index()]);
+    if ctx.merged_sections[id.index()].is_alloc() {
+        layout(&mut ctx.merged_sections[id.index()]);
+    }
 }
 
 /// Lays out one resolved merged section. Different merged sections have no

src/driver.rs

@@ -272,6 +272,24 @@ pub fn link<E: Arch>(cmdline: &[String]) -> Result<i32, String> {
         return Ok(0);
     }
 
+    // Non-allocated strings are independent of symbol and relocation passes.
+    // Give the background task owned metadata and publish it before layout.
+    let merge_input = chunks::merged::BackgroundMerge::prepare(&ctx);
+    let merge_timers = ctx.timers.clone();
+    let merge_comment = ctx.comment;
+    let merge_cmdline = ctx.cmdline_args.clone();
+    let (merge_sender, merge_receiver) = mpsc::sync_channel(1);
+    rayon::spawn(move || {
+        let result = merge_input.run(chunks::merged::ResolveOptions {
+            allocated_only: false,
+            gc_sections: false,
+            comment: merge_comment,
+            cmdline_args: &merge_cmdline,
+            timers: &merge_timers,
+        });
+        let _ = merge_sender.send(result);
+    });
+
     // Create .bss sections for common symbols.
     passes::convert_common_symbols(&mut ctx);
 
@@ -411,6 +429,8 @@ pub fn link<E: Arch>(cmdline: &[String]) -> Result<i32, String> {
     // Compute the is_weak bit for each imported symbol.
     passes::compute_imported_symbol_weakness(&mut ctx);
 
+    wait_for_background(merge_receiver, "non-allocated string merging").finish(&mut ctx);
+
     // Sort sections by section attributes so that we'll have to
     // create as few segments as possible.
     let t = ctx.timer("sort_output_sections");

src/input_sections.rs

@@ -1473,7 +1473,7 @@ impl SectionFragment {
 }
 
 /// Fragment metadata for an input section with the `SHF_MERGE` flag.
-#[derive(Debug)]
+#[derive(Clone, Debug)]
 pub struct MergeInfo {
     pub parent: MergedSectionId,
     pub p2align: u8,
@@ -1537,12 +1537,11 @@ impl MergeInfo {
     pub fn split_contents<E: Arch>(
         &mut self,
         file: &dyn fmt::Display,
-        section: &InputSection<E>,
+        data: &'static [u8],
         name: &BStr,
         parent: &MergedSection<E>,
         sketch: &mut HyperLogLog,
     ) {
-        let data = section.contents();
         if data.len() > u32::MAX as usize {
             fatal!(
                 "{}: mergeable section too large",
@@ -1593,7 +1592,7 @@ impl MergeInfo {
     /// Inserts the pieces into the parent section's fragment map.
     pub fn resolve_contents<E: Arch>(
         &mut self,
-        section: &InputSection<E>,
+        data: &'static [u8],
         parent: &crate::chunks::merged::MergedSection<E>,
         gc_sections: bool,
     ) {
@@ -1610,7 +1609,7 @@ impl MergeInfo {
                 parent.map.prefetch(hash);
             }
             let id = parent.insert(
-                self.contents(section, i),
+                self.contents(data, i),
                 self.hashes[i],
                 self.p2align,
                 gc_sections,
@@ -1658,8 +1657,7 @@ impl MergeInfo {
     }
 
     #[inline]
-    fn contents<E: Arch>(&self, section: &InputSection<E>, i: usize) -> &'static [u8] {
-        let contents = section.contents();
+    fn contents(&self, contents: &'static [u8], i: usize) -> &'static [u8] {
         let start = self.frag_offsets[i] as usize;
         match self.frag_offsets.get(i + 1) {
             Some(&end) => &contents[start..end as usize],

src/passes.rs

@@ -183,20 +183,6 @@ pub fn create_synthetic_sections<E: Arch>(ctx: &mut Context<E>) {
     chunks.push(ChunkId::Verneed);
     chunks.push(ChunkId::NotePackage);
 
-    if !ctx.args.oformat_binary {
-        let mut shdr = ElfShdr::<E>::default();
-        shdr.sh_type.set(SHT_PROGBITS);
-        shdr.sh_flags.set((SHF_MERGE | SHF_STRINGS) as u64);
-        let merged = RwLock::new(std::mem::take(&mut ctx.merged_sections));
-        ctx.comment = crate::chunks::merged::MergedSection::get_instance(
-            &ctx.args,
-            &merged,
-            BStr::new(b".comment"),
-            &shdr,
-        );
-        ctx.merged_sections = merged.into_inner().unwrap();
-    }
-
     if E::IS_X86 {
         ctx.note_property = Some(NotePropertySection::<E>::new());
         chunks.push(ChunkId::NoteProperty);
@@ -983,8 +969,8 @@ pub fn parse_sframe_sections<E: Arch>(ctx: &mut Context<E>) {
 fn merged_resolve_members<E: Arch>(
     objs: &mut FileList<ObjectFile<E>>,
     count: usize,
-) -> Vec<Vec<crate::chunks::merged::ResolveMember<'_, E>>> {
-    let mut members: Vec<Vec<crate::chunks::merged::ResolveMember<'_, E>>> =
+) -> Vec<Vec<crate::chunks::merged::ResolveMember<'_>>> {
+    let mut members: Vec<Vec<crate::chunks::merged::ResolveMember<'_>>> =
         (0..count).map(|_| Vec::new()).collect();
     for file in objs {
         let filename = file.base.filename.as_str();
@@ -995,7 +981,7 @@ fn merged_resolve_members<E: Arch>(
             let parent = merge_info.parent.index();
             members[parent].push(crate::chunks::merged::ResolveMember {
                 merge_info,
-                section: input,
+                data: input.contents(),
                 filename,
                 archive_name,
                 name: input.name_in(shstrtab, num_elf_sections),
@@ -1008,6 +994,20 @@ fn merged_resolve_members<E: Arch>(
 pub fn create_merged_sections<E: Arch>(ctx: &mut Context<E>) {
     let _t = ctx.timer("create_merged_sections");
 
+    if !ctx.args.oformat_binary && !ctx.args.relocatable {
+        let mut shdr = ElfShdr::<E>::default();
+        shdr.sh_type.set(SHT_PROGBITS);
+        shdr.sh_flags.set((SHF_MERGE | SHF_STRINGS) as u64);
+        let merged = RwLock::new(std::mem::take(&mut ctx.merged_sections));
+        ctx.comment = crate::chunks::merged::MergedSection::get_instance(
+            &ctx.args,
+            &merged,
+            BStr::new(b".comment"),
+            &shdr,
+        );
+        ctx.merged_sections = merged.into_inner().unwrap();
+    }
+
     // Create fragment metadata for eligible SHF_MERGE input sections. The
     // original InputSections remain stored but are marked dead.
     let t = ctx.timer("convert_mergeable_sections");
@@ -2356,7 +2356,7 @@ pub fn sort_debug_info_sections<E: Arch>(ctx: &mut Context<E>) {
         }
     }
     for id in vec2 {
-        chunks::compute_section_size(ctx, ChunkId::Merged(id));
+        crate::chunks::merged::layout(&mut ctx.merged_sections[id.index()]);
     }
 }
 
@@ -2524,10 +2524,9 @@ pub fn compute_section_sizes<E: Arch>(ctx: &mut Context<E>) {
         }
     }
 
-    // Merged sections without SHF_ALLOC are resolved lazily here. C++ mold
-    // processes all chunks in one parallel loop, so resolve their independent
-    // parent sections concurrently too.
-    {
+    // Relocatable links resolve non-allocated sections here; ordinary links
+    // already completed that work in the background.
+    if ctx.merged_sections.iter().any(|section| !section.resolved) {
         let mut members = merged_resolve_members(&mut ctx.objs, ctx.merged_sections.len());
         crate::chunks::merged::resolve_sections::<E>(
             &mut ctx.merged_sections,
@@ -2564,6 +2563,7 @@ pub fn compute_section_sizes<E: Arch>(ctx: &mut Context<E>) {
     // layout, just as each C++ Chunk does in the parallel chunk loop.
     ctx.merged_sections
         .par_iter_mut()
+        .filter(|section| section.is_alloc())
         .for_each(crate::chunks::merged::layout);
 
     for id in ctx.chunks.clone() {