/* * Copyright (c) 2024, Sönke Holz * * SPDX-License-Identifier: BSD-2-Clause */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include // FIXME: Initialize the __stack_chk_guard with a random value via the EFI_RNG_PROTOCOL or other arch-specific methods. uintptr_t __stack_chk_guard __attribute__((used)); extern "C" u8 pe_image_base[]; extern "C" u8 start_of_kernel_image[]; extern "C" u8 end_of_kernel_image[]; namespace Kernel { static_assert(EFI::EFI_PAGE_SIZE == PAGE_SIZE, "The EFIPrekernel assumes that EFI_PAGE_SIZE == PAGE_SIZE"); EFI::Handle g_efi_image_handle = 0; EFI::SystemTable* g_efi_system_table = nullptr; static size_t pages_needed(size_t bytes) { return ceil_div(bytes, static_cast(PAGE_SIZE)); } extern "C" [[noreturn]] void __stack_chk_fail(); extern "C" [[noreturn]] void __stack_chk_fail() { PANIC("Stack protector failure, stack smashing detected!"); } static char* allocate_and_map_cmdline_buffer(void* root_page_table, size_t length) { // Allocate pages for the cmdline buffer and map it to KERNEL_CMDLINE_VADDR. // TODO: KASLR EFI::PhysicalAddress buffer_paddr = 0; if (auto status = g_efi_system_table->boot_services->allocate_pages(EFI::AllocateType::AnyPages, EFI::MemoryType::LoaderData, pages_needed(length), &buffer_paddr); status != EFI::Status::Success) PANIC("Failed to allocate pages for the cmdline buffer: {}", status); if (auto result = map_pages(root_page_table, KERNEL_CMDLINE_VADDR, buffer_paddr, pages_needed(length), Access::Read); result.is_error()) PANIC("Failed to map the cmdline buffer: {}", result.release_error()); return bit_cast(buffer_paddr); } static void set_cmdline_from_load_options(EFI::LoadedImageProtocol* loaded_image_protocol, void* root_page_table, BootInfo& boot_info) { // Get the cmdline from loaded_image_protocol->load_options. // FIXME: Support non-ASCII characters. VERIFY(loaded_image_protocol->load_options_size != 0); VERIFY(loaded_image_protocol->load_options != nullptr); char16_t* load_options_ucs_2 = reinterpret_cast(loaded_image_protocol->load_options); size_t cmdline_length = loaded_image_protocol->load_options_size / sizeof(char16_t); char* cmdline_buffer = allocate_and_map_cmdline_buffer(root_page_table, cmdline_length); size_t actual_length = 0; for (size_t i = 0; i < cmdline_length && load_options_ucs_2[i] != u'\0'; i++) { cmdline_buffer[i] = static_cast(load_options_ucs_2[i]); actual_length++; } dbgln("Kernel cmdline (from load options): \"{}\"", StringView { cmdline_buffer, actual_length }); boot_info.cmdline = StringView { bit_cast(KERNEL_CMDLINE_VADDR), actual_length }; } static void set_cmdline_from_cmdline_file(EFI::FileProtocol* root_directory, void* root_page_table, BootInfo& boot_info) { auto cmdline_file_or_error = open_file(root_directory, u"cmdline.txt", EFI::FileOpenMode::Read); if (cmdline_file_or_error.is_error()) { dbgln("Failed to open cmdline.txt: {}", cmdline_file_or_error.release_error()); return; } auto* cmdline_file = cmdline_file_or_error.release_value(); ScopeGuard close_cmdline_file = [cmdline_file] { (void)close_file(cmdline_file); }; auto cmdline_or_error = read_entire_file(cmdline_file); if (cmdline_or_error.is_error()) { dbgln("Failed to read cmdline.txt: {}", cmdline_or_error.release_error()); return; } auto cmdline = cmdline_or_error.release_value(); // The cmdline.txt file might contain a trailing newline or other whitespace, so remove it. auto trimmed_cmdline = StringView { cmdline }.trim_whitespace(); dbgln("Kernel cmdline (from cmdline.txt): \"{}\"", trimmed_cmdline); char* cmdline_buffer = allocate_and_map_cmdline_buffer(root_page_table, trimmed_cmdline.length()); __builtin_memcpy(cmdline_buffer, trimmed_cmdline.characters_without_null_termination(), trimmed_cmdline.length()); boot_info.cmdline = StringView { bit_cast(KERNEL_CMDLINE_VADDR), trimmed_cmdline.length() }; } static void set_cmdline(EFI::FileProtocol* root_directory, EFI::LoadedImageProtocol* loaded_image_protocol, void* root_page_table, BootInfo& boot_info) { // Prefer setting the cmdline from our load options and fall back to reading it from cmdline.txt. if (loaded_image_protocol->load_options_size != 0 && loaded_image_protocol->load_options != nullptr) set_cmdline_from_load_options(loaded_image_protocol, root_page_table, boot_info); else if (root_directory != nullptr) set_cmdline_from_cmdline_file(root_directory, root_page_table, boot_info); } static void map_kernel_image(void* root_page_table, ELF::Image const& kernel_elf_image, ReadonlyBytes kernel_elf_image_data, FlatPtr kernel_load_base) { kernel_elf_image.for_each_program_header([root_page_table, kernel_elf_image_data, kernel_load_base](ELF::Image::ProgramHeader const& program_header) { if (program_header.type() != PT_LOAD) return IterationDecision::Continue; auto page_count = pages_needed(program_header.size_in_memory()); auto start_vaddr = kernel_load_base + program_header.vaddr().get(); auto start_paddr = bit_cast(kernel_elf_image_data.data()) + program_header.offset(); if (program_header.size_in_memory() != program_header.size_in_image()) { if (program_header.size_in_image() != 0) PANIC("Program headers with p_memsz != p_filesz && p_filesz != 0 are not supported"); // Allocate a zeroed memory region for the program header. EFI::PhysicalAddress segment_data_paddr = 0; if (auto status = g_efi_system_table->boot_services->allocate_pages(EFI::AllocateType::AnyPages, EFI::MemoryType::LoaderData, page_count, &segment_data_paddr); status != EFI::Status::Success) PANIC("Failed to allocate memory for program header {}: {}", program_header.index(), status); __builtin_memset(bit_cast(segment_data_paddr), 0, page_count * PAGE_SIZE); start_paddr = segment_data_paddr; } VERIFY(program_header.alignment() % PAGE_SIZE == 0); VERIFY(start_vaddr % PAGE_SIZE == 0); VERIFY(start_paddr % PAGE_SIZE == 0); auto access = Access::None; if (program_header.is_readable()) access |= Access::Read; if (program_header.is_writable()) access |= Access::Write; if (program_header.is_executable()) access |= Access::Execute; if (auto result = map_pages(root_page_table, start_vaddr, start_paddr, page_count, access); result.is_error()) PANIC("Failed to map program header {}: {}", program_header.index(), result.release_error()); return IterationDecision::Continue; }); } static void get_memory_map_and_exit_boot_services(void* root_page_table, BootInfo& boot_info) { auto* boot_services = g_efi_system_table->boot_services; auto& memory_map = boot_info.boot_method_specific.efi.memory_map; // Print this message before the first call to GetMemoryMap(), as calling OutputString() could change the memory map. dbgln("Exiting EFI Boot Services..."); // Get the required size for the memory map. if (auto status = boot_services->get_memory_map(&memory_map.descriptor_array_size, nullptr, &memory_map.map_key, &memory_map.descriptor_size, &memory_map.descriptor_version); status != EFI::Status::BufferTooSmall) PANIC("Failed to acquire the required size for memory map: {}", status); // Reserve space for 10 extra descriptors in the memory map, as the memory map could change between the first GetMemoryMap() and ExitBootServices(). // This also allows us to reuse the memory map even if the first call to ExitBootServices() fails. // We probably shouldn't allocate memory if ExitBootServices() failed, as that might change the memory map again. memory_map.descriptor_array_size = round_up_to_power_of_two(memory_map.descriptor_array_size + memory_map.descriptor_size * 10, PAGE_SIZE); if (memory_map.descriptor_array_size > EFI_MEMORY_MAP_MAX_SIZE) PANIC("EFI Memory map is too large: {} bytes (max: {} bytes)", memory_map.descriptor_array_size, EFI_MEMORY_MAP_MAX_SIZE); // We have to save the size here, as GetMemoryMap() overrides the value pointed to by the MemoryMap argument. memory_map.buffer_size = memory_map.descriptor_array_size; if (auto status = boot_services->allocate_pages(EFI::AllocateType::AnyPages, EFI::MemoryType::LoaderData, memory_map.buffer_size / PAGE_SIZE, &memory_map.descriptor_array_paddr); status != EFI::Status::Success) PANIC("Failed to allocate memory for the EFI memory map: {}", status); __builtin_memset(bit_cast(memory_map.descriptor_array_paddr), 0, memory_map.buffer_size); if (auto result = map_pages(root_page_table, EFI_MEMORY_MAP_VADDR, memory_map.descriptor_array_paddr, memory_map.buffer_size / PAGE_SIZE, Access::Read); result.is_error()) PANIC("Failed to map the EFI memory map: {}", result.release_error()); // Tell the kernel the location of the EFI memory map. memory_map.descriptor_array = bit_cast(EFI_MEMORY_MAP_VADDR); if (auto status = boot_services->get_memory_map(&memory_map.descriptor_array_size, bit_cast(memory_map.descriptor_array_paddr), &memory_map.map_key, &memory_map.descriptor_size, &memory_map.descriptor_version); status != EFI::Status::Success) PANIC("Failed to get the EFI memory map: {}", status); // A very crude memory leak detector // We should check for leaks before calling ExitBootServices(), as we have no way of freeing them after that. // Memory that should stay allocated has to directly be allocated via Allocate{Pages,Pool}(). kmalloc_stats stats; get_kmalloc_stats(stats); if (stats.kmalloc_call_count != stats.kfree_call_count) PANIC("Memory leak(s) detected! kmalloc call count: {}, kfree call count: {}", stats.kmalloc_call_count, stats.kfree_call_count); // From now on, we can't use any boot service or device-handle-based protocol anymore, even if ExitBootServices() failed. if (auto status = boot_services->exit_boot_services(g_efi_image_handle, memory_map.map_key); status == EFI::Status::InvalidParameter) { // We have to call GetMemoryMap() again, as the memory map changed between GetMemoryMap() and ExitBootServices(). // Memory allocation services are still allowed to be used if ExitBootServices() failed. memory_map.descriptor_array_size = memory_map.buffer_size; if (boot_services->get_memory_map(&memory_map.descriptor_array_size, bit_cast(memory_map.descriptor_array_paddr), &memory_map.map_key, &memory_map.descriptor_size, &memory_map.descriptor_version) != EFI::Status::Success) halt(); if (boot_services->exit_boot_services(g_efi_image_handle, memory_map.map_key) != EFI::Status::Success) halt(); } else if (status != EFI::Status::Success) { halt(); } } static void set_up_kernel_stack(void* root_page_table) { // Allocate pages for the kernel stack and map it to KERNEL_STACK_VADDR. // TODO: KASLR EFI::PhysicalAddress kernel_stack_paddr = 0; if (auto status = g_efi_system_table->boot_services->allocate_pages(EFI::AllocateType::AnyPages, EFI::MemoryType::LoaderData, KERNEL_STACK_SIZE / PAGE_SIZE, &kernel_stack_paddr); status != EFI::Status::Success) PANIC("Failed to allocate pages for the kernel stack: {}", status); __builtin_memset(bit_cast(kernel_stack_paddr), 0, KERNEL_STACK_SIZE); if (auto result = map_pages(root_page_table, KERNEL_STACK_VADDR, kernel_stack_paddr, KERNEL_STACK_SIZE / PAGE_SIZE, Access::Read | Access::Write); result.is_error()) PANIC("Failed to map the kernel stack: {}", result.release_error()); } static BootInfo* set_up_boot_info(void* root_page_table) { // Allocate pages for the boot info struct and map it to BOOT_INFO_VADDR. // TODO: KASLR EFI::PhysicalAddress boot_info_paddr = 0; if (auto status = g_efi_system_table->boot_services->allocate_pages(EFI::AllocateType::AnyPages, EFI::MemoryType::LoaderData, pages_needed(sizeof(BootInfo)), &boot_info_paddr); status != EFI::Status::Success) PANIC("Failed to allocate pages for the BootInfo struct: {}", status); if (auto result = map_pages(root_page_table, BOOT_INFO_VADDR, boot_info_paddr, pages_needed(sizeof(BootInfo)), Access::Read); result.is_error()) PANIC("Failed to map the BootInfo struct: {}", result.release_error()); auto* boot_info = bit_cast(boot_info_paddr); new (boot_info) BootInfo; boot_info->boot_method = BootMethod::EFI; boot_info->boot_method_specific.pre_init.~PreInitBootInfo(); new (&boot_info->boot_method_specific.efi) EFIBootInfo; return boot_info; } extern "C" EFIAPI EFI::Status init(EFI::Handle image_handle, EFI::SystemTable* system_table); extern "C" EFIAPI EFI::Status init(EFI::Handle image_handle, EFI::SystemTable* system_table) { // We use some EFI 1.10 functions from the System Table, so reject older versions. static constexpr u32 efi_version_1_10 = (1 << 16) | 10; if (system_table->hdr.signature != EFI::SystemTable::signature || system_table->hdr.revision < efi_version_1_10) return EFI::Status::Unsupported; g_efi_image_handle = image_handle; g_efi_system_table = system_table; auto* boot_services = system_table->boot_services; system_table->con_out->set_attribute(system_table->con_out, EFI::TextAttribute { .foreground_color = EFI::TextAttribute::ForegroundColor::White, .background_color = EFI::TextAttribute::BackgroundColor::Black, }); // Clear the screen. This also removes the manufacturer logo, if present. system_table->con_out->clear_screen(system_table->con_out); ucs2_dbgln(u"SerenityOS EFI Prekernel"); auto loaded_image_protocol_guid = EFI::LoadedImageProtocol::guid; EFI::LoadedImageProtocol* loaded_image_protocol; if (auto status = boot_services->handle_protocol(image_handle, &loaded_image_protocol_guid, reinterpret_cast(&loaded_image_protocol)); status != EFI::Status::Success) PANIC("Failed to get the loaded image protocol: {}", status); VERIFY(loaded_image_protocol->image_base == pe_image_base); VERIFY(bit_cast(loaded_image_protocol->image_base) % PAGE_SIZE == 0); auto maybe_root_page_table = allocate_empty_root_page_table(); if (maybe_root_page_table.is_error()) PANIC("Failed to allocate root page table: {}", maybe_root_page_table.release_error()); auto* root_page_table = maybe_root_page_table.value(); auto* boot_info = set_up_boot_info(root_page_table); auto kernel_image_paddr = bit_cast(&start_of_kernel_image); VERIFY(kernel_image_paddr % PAGE_SIZE == 0); auto kernel_image_size = bit_cast(&end_of_kernel_image) - kernel_image_paddr; Bytes kernel_elf_image_data { bit_cast(kernel_image_paddr), kernel_image_size }; ELF::Image kernel_elf_image { kernel_elf_image_data }; // TODO: KASLR FlatPtr default_kernel_load_base = KERNEL_MAPPING_BASE + 0x200000; boot_info->kernel_mapping_base = KERNEL_MAPPING_BASE; boot_info->kernel_load_base = default_kernel_load_base; boot_info->physical_to_virtual_offset = boot_info->kernel_load_base - kernel_image_paddr; // EFI_GRAPHICS_OUTPUT_PROTOCOL.SetMode() clears the screen, so do this as early as possible. init_gop_and_populate_framebuffer_boot_info(*boot_info); EFI::FileProtocol* root_directory = nullptr; auto root_directory_or_error = open_root_directory(loaded_image_protocol); if (root_directory_or_error.is_error()) dbgln("Failed to open the root directory: {}", root_directory_or_error.release_error()); else root_directory = root_directory_or_error.release_value(); dbgln("Mapping the kernel image..."); map_kernel_image(root_page_table, kernel_elf_image, kernel_elf_image_data, boot_info->kernel_load_base); dbgln("Performing relative relocations of the kernel image..."); perform_kernel_relocations(kernel_elf_image, kernel_elf_image_data, boot_info->kernel_load_base); set_up_kernel_stack(root_page_table); set_cmdline(root_directory, loaded_image_protocol, root_page_table, *boot_info); populate_firmware_boot_info(boot_info); if (root_directory != nullptr) (void)close_file(root_directory); arch_prepare_boot(root_page_table, *boot_info); get_memory_map_and_exit_boot_services(root_page_table, *boot_info); auto kernel_entry_vaddr = boot_info->kernel_load_base + kernel_elf_image.entry().get(); arch_enter_kernel(root_page_table, kernel_entry_vaddr, KERNEL_STACK_VADDR + KERNEL_STACK_SIZE, BOOT_INFO_VADDR); } } void __assertion_failed(char const* msg, char const* file, unsigned int line, char const* func) { dbgln("ASSERTION FAILED: {}", msg); dbgln("{}:{} in {}", file, line, func); Kernel::halt(); }