Kernel: Give all declarations hidden visibility

This causes GCC and Clang to generate relative loads when accessing
global variables instead of using the global offset table (GOT).
This saves one load for every global variable access.
We unfortunately have to do this by injecting a header into every
source file, since "-fvisibility=hidden" doesn't affect `extern`
declarations.

This additionally means that accessing global variables doesn't
depend on relocations anymore, so pre_init code no longer needs to
access variables through `adjust_by_mapping_base()`.
Next to this, some other small tweaks are required in pre_init because
global variables now have the "correct" addresses while we are still
identity mapped.

Being able to simplify pre_init like this was also the original
motivation for this patch. Clang trunk started optimizing
`adjust_by_mapping_base()` on AArch64 such that it avoids using the GOT
when we attempt to get the address of the global variable in high
memory. As a consequence, we now directly get the address in low memory
on Clang trunk, causing us to redundantly subtract the physical to link
time address offset and end up with an incorrect address.
With this patch, we no longer need to rely on the compiler not doing
such optimizations anymore.

The .got section still exists on non-x86 but only contains 2-3 entries:
- the start of the .dynamic section (I'm not sure why),
- the __stack_chk_guard (only actually used on RISC-V GCC),
- __clz_tab (only on RISC-V GCC).
Additionally, there are some other GOT entries caused by the RISC-V `la`
pseudo-instruction, which will be removed in the next commit.
It's a bit unfortunate that RISC-V GCC still uses the GOT to access the
stack protector (__stack_chk_guard), but that's not under our control.

RISC-V GCC and AArch64 GCC also still contain a .got.plt section, but
none of the other toolchain+architecture combinations.

Kernel/Arch/aarch64/MMU.cpp

@@ -93,18 +93,6 @@ static UNMAP_AFTER_INIT FlatPtr calculate_physical_to_link_time_address_offset()
     return KERNEL_MAPPING_BASE - physical_address;
 }
 
-// NOTE: To access global variables while the MMU is not yet enabled, we need
-//       to convert the address of a global variable to a physical address by
-//       subtracting calculate_physical_to_link_time_address_offset(). This is because the kernel is linked
-//       for virtual memory at KERNEL_MAPPING_BASE, so a regular access to global variables
-//       will use the high virtual memory address. This does not work when the MMU is not yet
-//       enabled, so this function must be used for accessing global variables.
-template<typename T>
-inline T* adjust_by_mapping_base(T* ptr)
-{
-    return (T*)((FlatPtr)ptr - calculate_physical_to_link_time_address_offset());
-}
-
 static u64* insert_page_table(PageBumpAllocator& allocator, u64* page_table, VirtualAddress virtual_addr)
 {
     // Each level has 9 bits (512 entries)
@@ -156,20 +144,20 @@ static void setup_quickmap_page_table(PageBumpAllocator& allocator, u64* root_ta
 {
     // FIXME: Rename boot_pd_kernel_pt1023 to quickmap_page_table
     // FIXME: Rename KERNEL_PT1024_BASE to quickmap_page_table_address
-    auto kernel_pt1024_base = VirtualAddress(*adjust_by_mapping_base(&g_boot_info.kernel_mapping_base) + KERNEL_PT1024_OFFSET);
+    auto kernel_pt1024_base = VirtualAddress(g_boot_info.kernel_mapping_base + KERNEL_PT1024_OFFSET);
 
     auto quickmap_page_table = PhysicalAddress((PhysicalPtr)insert_page_table(allocator, root_table, kernel_pt1024_base));
-    *adjust_by_mapping_base(&g_boot_info.boot_pd_kernel_pt1023) = (PageTableEntry*)quickmap_page_table.offset(calculate_physical_to_link_time_address_offset()).get();
+    g_boot_info.boot_pd_kernel_pt1023 = (PageTableEntry*)quickmap_page_table.offset(calculate_physical_to_link_time_address_offset()).get();
 }
 
 static void build_mappings(PageBumpAllocator& allocator, u64* root_table)
 {
     u64 normal_memory_flags = ACCESS_FLAG | PAGE_DESCRIPTOR | INNER_SHAREABLE | NORMAL_MEMORY;
 
-    auto start_of_kernel_range = VirtualAddress((FlatPtr)start_of_kernel_image);
-    auto end_of_kernel_range = VirtualAddress((FlatPtr)end_of_kernel_image);
+    auto start_of_kernel_range = VirtualAddress { KERNEL_MAPPING_BASE };
+    auto end_of_kernel_range = start_of_kernel_range.offset((+end_of_kernel_image) - (+start_of_kernel_image));
 
-    auto start_of_physical_kernel_range = PhysicalAddress(start_of_kernel_range.get()).offset(-calculate_physical_to_link_time_address_offset());
+    auto start_of_physical_kernel_range = PhysicalAddress { bit_cast<PhysicalPtr>(+start_of_kernel_image) };
 
     // Insert identity mapping
     insert_entries_for_memory_range(allocator, root_table, start_of_kernel_range.offset(-calculate_physical_to_link_time_address_offset()), end_of_kernel_range.offset(-calculate_physical_to_link_time_address_offset()), start_of_physical_kernel_range, normal_memory_flags);
@@ -255,17 +243,17 @@ static u64* get_page_directory_table(u64* root_table, VirtualAddress virtual_add
 
 static void setup_kernel_page_directory(u64* root_table)
 {
-    auto kernel_page_directory = (PhysicalPtr)get_page_directory(root_table, VirtualAddress { *adjust_by_mapping_base(&g_boot_info.kernel_mapping_base) });
+    auto kernel_page_directory = (PhysicalPtr)get_page_directory(root_table, VirtualAddress { g_boot_info.kernel_mapping_base });
     if (!kernel_page_directory)
         panic_without_mmu("Could not find kernel page directory!"sv);
 
-    *adjust_by_mapping_base(&g_boot_info.boot_pd_kernel) = PhysicalAddress(kernel_page_directory);
+    g_boot_info.boot_pd_kernel = PhysicalAddress(kernel_page_directory);
 
     // FIXME: Rename boot_pml4t to something architecture agnostic.
-    *adjust_by_mapping_base(&g_boot_info.boot_pml4t) = PhysicalAddress((PhysicalPtr)root_table);
+    g_boot_info.boot_pml4t = PhysicalAddress((PhysicalPtr)root_table);
 
     // FIXME: Rename to directory_table or similar
-    *adjust_by_mapping_base(&g_boot_info.boot_pdpt) = PhysicalAddress((PhysicalPtr)get_page_directory_table(root_table, VirtualAddress { *adjust_by_mapping_base(&g_boot_info.kernel_mapping_base) }));
+    g_boot_info.boot_pdpt = PhysicalAddress((PhysicalPtr)get_page_directory_table(root_table, VirtualAddress { g_boot_info.kernel_mapping_base }));
 }
 
 void init_page_tables(PhysicalPtr flattened_devicetree_paddr)
@@ -280,24 +268,24 @@ void init_page_tables(PhysicalPtr flattened_devicetree_paddr)
         panic_without_mmu("Passed FDT is bigger than the internal storage"sv);
     for (size_t o = 0; o < fdt_header->totalsize; o += 1) {
         // FIXME: Maybe increase the IO size here
-        adjust_by_mapping_base(DeviceTree::s_fdt_storage)[o] = fdt_storage[o];
+        DeviceTree::s_fdt_storage[o] = fdt_storage[o];
     }
 
-    *adjust_by_mapping_base(&g_boot_info.boot_method) = BootMethod::PreInit;
+    g_boot_info.boot_method = BootMethod::PreInit;
 
-    *adjust_by_mapping_base(&g_boot_info.flattened_devicetree_paddr) = PhysicalAddress { flattened_devicetree_paddr };
-    *adjust_by_mapping_base(&g_boot_info.flattened_devicetree_size) = fdt_header->totalsize;
-    *adjust_by_mapping_base(&g_boot_info.physical_to_virtual_offset) = calculate_physical_to_link_time_address_offset();
-    *adjust_by_mapping_base(&g_boot_info.kernel_mapping_base) = KERNEL_MAPPING_BASE;
-    *adjust_by_mapping_base(&g_boot_info.kernel_load_base) = KERNEL_MAPPING_BASE;
+    g_boot_info.flattened_devicetree_paddr = PhysicalAddress { flattened_devicetree_paddr };
+    g_boot_info.flattened_devicetree_size = fdt_header->totalsize;
+    g_boot_info.physical_to_virtual_offset = calculate_physical_to_link_time_address_offset();
+    g_boot_info.kernel_mapping_base = KERNEL_MAPPING_BASE;
+    g_boot_info.kernel_load_base = KERNEL_MAPPING_BASE;
 
-    PageBumpAllocator allocator(adjust_by_mapping_base((u64*)page_tables_phys_start), adjust_by_mapping_base((u64*)page_tables_phys_end));
+    PageBumpAllocator allocator((u64*)page_tables_phys_start, (u64*)page_tables_phys_end);
     auto root_table = allocator.take_page();
     build_mappings(allocator, root_table);
     setup_quickmap_page_table(allocator, root_table);
     setup_kernel_page_directory(root_table);
 
-    switch_to_page_table(adjust_by_mapping_base(page_tables_phys_start));
+    switch_to_page_table(page_tables_phys_start);
     activate_mmu();
 }
 

Kernel/Arch/aarch64/pre_init.cpp

@@ -49,7 +49,7 @@ extern "C" [[noreturn]] void pre_init(PhysicalPtr flattened_devicetree_paddr);
 extern "C" [[noreturn]] void pre_init(PhysicalPtr flattened_devicetree_paddr)
 {
     // Apply relative relocations as if we were running at KERNEL_MAPPING_BASE.
-    // This means that all global variables must be accessed with adjust_by_mapping_base, since we are still running identity mapped.
+    // This means that we shouldn't access anything during pre_init that relies on relocations (e.g. vtables).
     // Otherwise, we would have to relocate twice: once while running identity mapped, and again when we enable the MMU.
     if (!ELF::perform_relative_relocations(physical_load_base(), KERNEL_MAPPING_BASE, dynamic_section_addr()))
         panic_without_mmu("Failed to perform relative relocations"sv);

Kernel/Arch/riscv64/MMU.cpp

@@ -76,12 +76,6 @@ static UNMAP_AFTER_INIT FlatPtr calculate_physical_to_link_time_address_offset()
     return link_time_address - physical_address;
 }
 
-template<typename T>
-inline T* adjust_by_mapping_base(T* ptr)
-{
-    return bit_cast<T*>(bit_cast<FlatPtr>(ptr) - calculate_physical_to_link_time_address_offset());
-}
-
 static UNMAP_AFTER_INIT bool page_table_entry_valid(u64 entry)
 {
     return (entry & to_underlying(PageTableEntryBits::Valid)) != 0;
@@ -142,18 +136,18 @@ static UNMAP_AFTER_INIT void insert_entries_for_memory_range(PageBumpAllocator&
 
 static UNMAP_AFTER_INIT void setup_quickmap_page_table(PageBumpAllocator& allocator, u64* root_table)
 {
-    auto kernel_pt1024_base = VirtualAddress { *adjust_by_mapping_base(&g_boot_info.kernel_mapping_base) + KERNEL_PT1024_OFFSET };
+    auto kernel_pt1024_base = VirtualAddress { g_boot_info.kernel_mapping_base + KERNEL_PT1024_OFFSET };
 
     auto quickmap_page_table = PhysicalAddress { bit_cast<PhysicalPtr>(insert_page_table(allocator, root_table, kernel_pt1024_base)) };
-    *adjust_by_mapping_base(&g_boot_info.boot_pd_kernel_pt1023) = bit_cast<PageTableEntry*>(quickmap_page_table.offset(calculate_physical_to_link_time_address_offset()).get());
+    g_boot_info.boot_pd_kernel_pt1023 = bit_cast<PageTableEntry*>(quickmap_page_table.offset(calculate_physical_to_link_time_address_offset()).get());
 }
 
 static UNMAP_AFTER_INIT void build_mappings(PageBumpAllocator& allocator, u64* root_table)
 {
-    auto start_of_kernel_range = VirtualAddress { bit_cast<FlatPtr>(+start_of_kernel_image) };
-    auto end_of_kernel_range = VirtualAddress { bit_cast<FlatPtr>(+end_of_kernel_image) };
+    auto start_of_kernel_range = VirtualAddress { KERNEL_MAPPING_BASE };
+    auto end_of_kernel_range = start_of_kernel_range.offset((+end_of_kernel_image) - (+start_of_kernel_image));
 
-    auto start_of_physical_kernel_range = PhysicalAddress { start_of_kernel_range.get() }.offset(-calculate_physical_to_link_time_address_offset());
+    auto start_of_physical_kernel_range = PhysicalAddress { bit_cast<PhysicalPtr>(+start_of_kernel_image) };
 
     // FIXME: dont map everything RWX
 
@@ -163,16 +157,16 @@ static UNMAP_AFTER_INIT void build_mappings(PageBumpAllocator& allocator, u64* r
 
 static UNMAP_AFTER_INIT void setup_kernel_page_directory(u64* root_table)
 {
-    auto kernel_page_directory = bit_cast<PhysicalPtr>(get_page_directory(root_table, VirtualAddress { *adjust_by_mapping_base(&g_boot_info.kernel_mapping_base) }));
+    auto kernel_page_directory = bit_cast<PhysicalPtr>(get_page_directory(root_table, VirtualAddress { g_boot_info.kernel_mapping_base }));
     if (kernel_page_directory == 0)
         panic_without_mmu("Could not find kernel page directory!"sv);
 
-    *adjust_by_mapping_base(&g_boot_info.boot_pd_kernel) = PhysicalAddress { kernel_page_directory };
+    g_boot_info.boot_pd_kernel = PhysicalAddress { kernel_page_directory };
 
     // There is no level 4 table in Sv39
-    *adjust_by_mapping_base(&g_boot_info.boot_pml4t) = PhysicalAddress { 0 };
+    g_boot_info.boot_pml4t = PhysicalAddress { 0 };
 
-    *adjust_by_mapping_base(&g_boot_info.boot_pdpt) = PhysicalAddress { bit_cast<PhysicalPtr>(root_table) };
+    g_boot_info.boot_pdpt = PhysicalAddress { bit_cast<PhysicalPtr>(root_table) };
 }
 
 // This function has to fit into one page as it will be identity mapped.
@@ -181,7 +175,7 @@ static UNMAP_AFTER_INIT void setup_kernel_page_directory(u64* root_table)
     // Switch current root page table to argument 1. This will immediately take effect, but we won't not crash as this function is identity mapped.
     // Also, set up a temporary trap handler to catch any traps while switching page tables.
     auto offset = calculate_physical_to_link_time_address_offset();
-    register FlatPtr a0 asm("a0") = bit_cast<FlatPtr>(&info);
+    register FlatPtr a0 asm("a0") = bit_cast<FlatPtr>(&info) + offset;
     asm volatile(
         "   lla t0, 1f \n"
         "   csrw stvec, t0 \n"
@@ -233,20 +227,20 @@ static UNMAP_AFTER_INIT void setup_kernel_page_directory(u64* root_table)
         panic_without_mmu("Passed FDT is bigger than the internal storage"sv);
     for (size_t o = 0; o < fdt_header->totalsize; o += 1) {
         // FIXME: Maybe increase the IO size here
-        adjust_by_mapping_base(DeviceTree::s_fdt_storage)[o] = fdt_storage[o];
+        DeviceTree::s_fdt_storage[o] = fdt_storage[o];
     }
 
-    *adjust_by_mapping_base(&g_boot_info.boot_method) = BootMethod::PreInit;
+    g_boot_info.boot_method = BootMethod::PreInit;
 
-    *adjust_by_mapping_base(&g_boot_info.flattened_devicetree_paddr) = PhysicalAddress { flattened_devicetree_paddr };
-    *adjust_by_mapping_base(&g_boot_info.flattened_devicetree_size) = fdt_header->totalsize;
-    *adjust_by_mapping_base(&g_boot_info.physical_to_virtual_offset) = calculate_physical_to_link_time_address_offset();
-    *adjust_by_mapping_base(&g_boot_info.kernel_mapping_base) = KERNEL_MAPPING_BASE;
-    *adjust_by_mapping_base(&g_boot_info.kernel_load_base) = KERNEL_MAPPING_BASE;
+    g_boot_info.flattened_devicetree_paddr = PhysicalAddress { flattened_devicetree_paddr };
+    g_boot_info.flattened_devicetree_size = fdt_header->totalsize;
+    g_boot_info.physical_to_virtual_offset = calculate_physical_to_link_time_address_offset();
+    g_boot_info.kernel_mapping_base = KERNEL_MAPPING_BASE;
+    g_boot_info.kernel_load_base = KERNEL_MAPPING_BASE;
 
-    *adjust_by_mapping_base(&g_boot_info.arch_specific.boot_hart_id) = boot_hart_id;
+    g_boot_info.arch_specific.boot_hart_id = boot_hart_id;
 
-    PageBumpAllocator allocator(adjust_by_mapping_base(reinterpret_cast<u64*>(page_tables_phys_start)), adjust_by_mapping_base(reinterpret_cast<u64*>(page_tables_phys_end)));
+    PageBumpAllocator allocator(reinterpret_cast<u64*>(page_tables_phys_start), reinterpret_cast<u64*>(page_tables_phys_end));
     auto* root_table = allocator.take_page();
     build_mappings(allocator, root_table);
     setup_quickmap_page_table(allocator, root_table);

Kernel/Arch/riscv64/pre_init.cpp

@@ -71,7 +71,7 @@ static UNMAP_AFTER_INIT PhysicalPtr dynamic_section_addr()
 extern "C" [[noreturn]] UNMAP_AFTER_INIT void pre_init(FlatPtr boot_hart_id, PhysicalPtr flattened_devicetree_paddr)
 {
     // Apply relative relocations as if we were running at KERNEL_MAPPING_BASE.
-    // This means that all global variables must be accessed with adjust_by_mapping_base, since we are still running identity mapped.
+    // This means that we shouldn't access anything during pre_init that relies on relocations (e.g. vtables).
     // Otherwise, we would have to relocate twice: once while running identity mapped, and again when we enable the MMU.
     if (!ELF::perform_relative_relocations(physical_load_base(), KERNEL_MAPPING_BASE, dynamic_section_addr()))
         panic_without_mmu("Failed to perform relative relocations"sv);

Kernel/CMakeLists.txt

@@ -702,6 +702,7 @@ elseif("${SERENITY_ARCH}" STREQUAL "aarch64")
     add_compile_options(-mgeneral-regs-only)
 endif()
 
+add_compile_options(-include "${CMAKE_CURRENT_SOURCE_DIR}/HiddenVisibility.h")
 add_compile_options(-fno-asynchronous-unwind-tables)
 add_compile_options(-fstack-protector-strong)
 add_compile_options(-fno-exceptions)

Kernel/HiddenVisibility.h

@@ -0,0 +1,14 @@
+/*
+ * Copyright (c) 2026, Sönke Holz <[email protected]>
+ *
+ * SPDX-License-Identifier: BSD-2-Clause
+ */
+
+#pragma once
+
+// This header is injected into every source file via -include.
+// It gives all declarations hidden visibility, preventing global variable accesses from going through the global offset table (GOT).
+// GOT indirections are unnecessary in the kernel since it isn't dynamically linked.
+// "-fvisibility=hidden" isn't sufficient because it doesn't make `extern` declarations hidden.
+
+#pragma GCC visibility push(hidden)