serenity/Kernel/Time/TimeManagement.cpp

/*
 * Copyright (c) 2020, Liav A. <[email protected]>
 * Copyright (c) 2022, Timon Kruiper <[email protected]>
 *
 * SPDX-License-Identifier: BSD-2-Clause
 */

#include <AK/NeverDestroyed.h>
#include <AK/Singleton.h>
#include <AK/StdLibExtras.h>
#include <AK/Time.h>
#if ARCH(X86_64)
#    include <Kernel/Arch/x86_64/Interrupts/APIC.h>
#    include <Kernel/Arch/x86_64/RTC.h>
#    include <Kernel/Arch/x86_64/Time/APICTimer.h>
#    include <Kernel/Arch/x86_64/Time/HPET.h>
#    include <Kernel/Arch/x86_64/Time/HPETComparator.h>
#    include <Kernel/Arch/x86_64/Time/PIT.h>
#    include <Kernel/Arch/x86_64/Time/RTC.h>
#    include <Kernel/Arch/x86_64/Time/TSC.h>
#elif ARCH(AARCH64)
#    include <Kernel/Arch/aarch64/RPi/Timer.h>
#    include <Kernel/Arch/aarch64/Time/ARMv8Timer.h>
#    include <Kernel/Arch/aarch64/Time/PL031.h>
#elif ARCH(RISCV64)
#    include <Kernel/Arch/riscv64/Timer.h>
#    include <Kernel/Devices/Time/GoldfishRTC.h>
#else
#    error Unknown architecture
#endif

#include <Kernel/Arch/CurrentTime.h>
#include <Kernel/Boot/CommandLine.h>
#include <Kernel/Firmware/ACPI/Parser.h>
#include <Kernel/Interrupts/InterruptDisabler.h>
#include <Kernel/Library/Panic.h>
#include <Kernel/Sections.h>
#include <Kernel/Tasks/PerformanceManager.h>
#include <Kernel/Tasks/Scheduler.h>
#include <Kernel/Time/HardwareTimer.h>
#include <Kernel/Time/TimeManagement.h>
#include <Kernel/Time/TimerQueue.h>

namespace Kernel {

static NeverDestroyed<Vector<NonnullLockRefPtr<HardwareTimerBase>>> s_hardware_timers;
static Singleton<TimeManagement> s_the;

bool TimeManagement::is_initialized()
{
    return s_the.is_initialized();
}

TimeManagement& TimeManagement::the()
{
    return *s_the;
}

ErrorOr<void> TimeManagement::register_hardware_timer(NonnullLockRefPtr<HardwareTimerBase> timer)
{
    // This function has to be called before TimeManagement is initialized,
    // as we do not support dynamic registration of timers.
    VERIFY(!is_initialized());

    return s_hardware_timers->try_append(move(timer));
}

// The s_scheduler_specific_current_time function provides a current time for scheduling purposes,
// which may not necessarily relate to wall time
static u64 (*s_scheduler_current_time)();

static u64 current_time_monotonic()
{
    // We always need a precise timestamp here, we cannot rely on a coarse timestamp
    return (u64)TimeManagement::the().monotonic_time(TimePrecision::Precise).nanoseconds();
}

u64 TimeManagement::scheduler_current_time()
{
    VERIFY(s_scheduler_current_time);
    return s_scheduler_current_time();
}

ErrorOr<void> TimeManagement::validate_clock_id(clockid_t clock_id)
{
    switch (clock_id) {
    case CLOCK_MONOTONIC:
    case CLOCK_MONOTONIC_COARSE:
    case CLOCK_MONOTONIC_RAW:
    case CLOCK_REALTIME:
    case CLOCK_REALTIME_COARSE:
        return {};
    default:
        return EINVAL;
    };
}

Duration TimeManagement::current_time(clockid_t clock_id) const
{
    switch (clock_id) {
    case CLOCK_MONOTONIC:
        return monotonic_time(TimePrecision::Precise).time_since_start({});
    case CLOCK_MONOTONIC_COARSE:
        return monotonic_time(TimePrecision::Coarse).time_since_start({});
    case CLOCK_MONOTONIC_RAW:
        return monotonic_time_raw().time_since_start({});
    case CLOCK_REALTIME:
        return epoch_time(TimePrecision::Precise).offset_to_epoch();
    case CLOCK_REALTIME_COARSE:
        return epoch_time(TimePrecision::Coarse).offset_to_epoch();
    default:
        // Syscall entrypoint is missing a is_valid_clock_id(..) check?
        VERIFY_NOT_REACHED();
    }
}

bool TimeManagement::is_system_timer(HardwareTimerBase const& timer) const
{
    return &timer == m_system_timer.ptr();
}

void TimeManagement::set_epoch_time(UnixDateTime ts)
{
    // FIXME: The interrupt disabler intends to enforce atomic update of epoch time and remaining adjustment,
    //        but that sort of assumption is known to break on SMP.
    InterruptDisabler disabler;
    m_epoch_time = ts;
    m_remaining_epoch_time_adjustment = {};
}

MonotonicTime TimeManagement::monotonic_time(TimePrecision precision) const
{
    // This is the time when last updated by an interrupt.
    u64 seconds;
    u32 ticks;

    bool do_query = precision == TimePrecision::Precise && m_can_query_precise_time.was_set();

    u32 update_iteration;
    do {
        update_iteration = m_update1.load(AK::MemoryOrder::memory_order_acquire);
        seconds = m_seconds_since_boot;
        ticks = m_ticks_this_second;

        if (do_query)
            m_read_time_from_hardware(seconds, ticks, true);
    } while (update_iteration != m_update2.load(AK::MemoryOrder::memory_order_acquire));

    VERIFY(m_time_ticks_per_second > 0);
    VERIFY(ticks < m_time_ticks_per_second);
    u64 ns = ((u64)ticks * 1000000000ull) / m_time_ticks_per_second;
    VERIFY(ns < 1000000000ull);
    return MonotonicTime::from_hardware_time({}, seconds, ns);
}

UnixDateTime TimeManagement::epoch_time(TimePrecision) const
{
    // TODO: Take into account precision
    UnixDateTime time;
    u32 update_iteration;
    do {
        update_iteration = m_update1.load(AK::MemoryOrder::memory_order_acquire);
        time = m_epoch_time;
    } while (update_iteration != m_update2.load(AK::MemoryOrder::memory_order_acquire));
    return time;
}

u64 TimeManagement::uptime_ms() const
{
    auto mtime = monotonic_time().time_since_start({}).to_timespec();
    // This overflows after 292 million years of uptime.
    // Since this is only used for performance timestamps and sys$times, that's probably enough.
    u64 ms = mtime.tv_sec * 1000ull;
    ms += mtime.tv_nsec / 1000000;
    return ms;
}

UNMAP_AFTER_INIT void TimeManagement::initialize([[maybe_unused]] u32 cpu)
{
    // Note: We must disable interrupts, because the timers interrupt might fire before
    //       the TimeManagement class is completely initialized.
    InterruptDisabler disabler;

#if ARCH(X86_64)
    if (cpu == 0) {
        VERIFY(!s_the.is_initialized());
        s_the.ensure_instance();

        if (APIC::initialized()) {
            // Initialize the APIC timers after the other timers as the
            // initialization needs to briefly enable interrupts, which then
            // would trigger a deadlock trying to get the s_the instance while
            // creating it.
            if (auto* apic_timer = APIC::the().initialize_timers(*s_the->m_system_timer)) {
                dmesgln("Duration: Using APIC timer as system timer");
                s_the->set_system_timer(*apic_timer);
            }
        }
    } else {
        VERIFY(s_the.is_initialized());
        if (auto* apic_timer = APIC::the().get_timer()) {
            dmesgln("Duration: Enable APIC timer on CPU #{}", cpu);
            apic_timer->enable_local_timer();
        }
    }
#elif ARCH(AARCH64)
    if (cpu == 0) {
        VERIFY(!s_the.is_initialized());
        s_the.ensure_instance();
    }
#elif ARCH(RISCV64)
    if (cpu == 0) {
        VERIFY(!s_the.is_initialized());
        s_the.ensure_instance();
    }
#else
#    error Unknown architecture
#endif
    auto* possible_arch_specific_current_time_function = optional_current_time();
    if (possible_arch_specific_current_time_function)
        s_scheduler_current_time = possible_arch_specific_current_time_function;
    else
        s_scheduler_current_time = current_time_monotonic;
}

void TimeManagement::set_system_timer(HardwareTimerBase& timer)
{
    VERIFY(Processor::is_bootstrap_processor()); // This should only be called on the BSP!
    auto original_callback = m_system_timer->set_callback(nullptr);
    m_system_timer->disable();
    timer.set_callback(move(original_callback));
    m_system_timer = timer;
}

time_t TimeManagement::ticks_per_second() const
{
    return m_time_keeper_timer->ticks_per_second();
}

UnixDateTime TimeManagement::boot_time()
{
#if ARCH(X86_64)
    return RTC::boot_time();
#elif ARCH(AARCH64)
    auto rtc = PL031::the();
    if (!rtc)
        return UnixDateTime::epoch();
    return rtc->boot_time();
#elif ARCH(RISCV64)
    auto rtc = GoldfishRTC::the();
    if (!rtc)
        return UnixDateTime::epoch();
    return rtc->boot_time();
#else
#    error Unknown architecture
#endif
}

Duration TimeManagement::clock_resolution() const
{
    long nanoseconds_per_tick = 1'000'000'000 / m_time_keeper_timer->ticks_per_second();
    return Duration::from_nanoseconds(nanoseconds_per_tick);
}

UNMAP_AFTER_INIT TimeManagement::TimeManagement()
    : m_time_page_region(MM.allocate_kernel_region(PAGE_SIZE, "Duration page"sv, Memory::Region::Access::ReadWrite, AllocationStrategy::AllocateNow).release_value_but_fixme_should_propagate_errors())
{
#if ARCH(X86_64)
    bool probe_non_legacy_hardware_timers = !(kernel_command_line().is_legacy_time_enabled());
    if (ACPI::is_enabled()) {
        if (!ACPI::Parser::the()->x86_specific_flags().cmos_rtc_not_present) {
            RTC::initialize();
            m_epoch_time += boot_time().offset_to_epoch();
        } else {
            dmesgln("ACPI: RTC CMOS Not present");
        }
    } else {
        // We just assume that we can access RTC CMOS, if ACPI isn't usable.
        RTC::initialize();
        m_epoch_time += boot_time().offset_to_epoch();
    }
    if (probe_non_legacy_hardware_timers) {
        if (!probe_and_set_x86_non_legacy_hardware_timers())
            if (!probe_and_set_x86_legacy_hardware_timers())
                VERIFY_NOT_REACHED();
    } else if (!probe_and_set_x86_legacy_hardware_timers()) {
        VERIFY_NOT_REACHED();
    }
#elif ARCH(AARCH64)
    auto rtc = PL031::the();
    if (rtc)
        m_epoch_time += boot_time().offset_to_epoch();
    probe_and_set_aarch64_hardware_timers();
#elif ARCH(RISCV64)
    auto rtc = GoldfishRTC::the();
    if (rtc)
        m_epoch_time += boot_time().offset_to_epoch();
    probe_and_set_riscv64_hardware_timers();
#else
#    error Unknown architecture
#endif
}

UnixDateTime TimeManagement::now()
{
    return s_the.ptr()->epoch_time();
}

UNMAP_AFTER_INIT Vector<HardwareTimerBase*> TimeManagement::scan_and_initialize_periodic_timers()
{
    bool should_enable = is_hpet_periodic_mode_allowed();
    dbgln("Duration: Scanning for periodic timers");
    Vector<HardwareTimerBase*> timers;
    for (auto& hardware_timer : *s_hardware_timers) {
        if (hardware_timer->is_periodic_capable()) {
            timers.try_append(hardware_timer).release_value_but_fixme_should_propagate_errors();
            if (should_enable)
                hardware_timer->set_periodic();
        }
    }
    return timers;
}

UNMAP_AFTER_INIT Vector<HardwareTimerBase*> TimeManagement::scan_for_non_periodic_timers()
{
    dbgln("Duration: Scanning for non-periodic timers");
    Vector<HardwareTimerBase*> timers;
    for (auto& hardware_timer : *s_hardware_timers) {
        if (!hardware_timer->is_periodic_capable())
            timers.try_append(hardware_timer).release_value_but_fixme_should_propagate_errors();
    }
    return timers;
}

bool TimeManagement::is_hpet_periodic_mode_allowed()
{
    switch (kernel_command_line().hpet_mode()) {
    case HPETMode::Periodic:
        return true;
    case HPETMode::NonPeriodic:
        return false;
    default:
        VERIFY_NOT_REACHED();
    }
}

#if ARCH(X86_64)
UNMAP_AFTER_INIT bool TimeManagement::probe_and_set_x86_non_legacy_hardware_timers()
{
    if (!ACPI::is_enabled())
        return false;
    if (!HPET::test_and_initialize())
        return false;
    if (!HPET::the().comparators().size()) {
        dbgln("HPET initialization aborted.");
        return false;
    }
    dbgln("HPET: Setting appropriate functions to timers.");

    for (auto& hpet_comparator : HPET::the().comparators())
        s_hardware_timers->try_append(hpet_comparator).release_value_but_fixme_should_propagate_errors();

    auto periodic_timers = scan_and_initialize_periodic_timers();
    auto non_periodic_timers = scan_for_non_periodic_timers();

    if (is_hpet_periodic_mode_allowed())
        VERIFY(!periodic_timers.is_empty());

    VERIFY(periodic_timers.size() + non_periodic_timers.size() > 0);

    size_t taken_periodic_timers_count = 0;
    size_t taken_non_periodic_timers_count = 0;

    if (periodic_timers.size() > taken_periodic_timers_count) {
        m_system_timer = periodic_timers[taken_periodic_timers_count];
        taken_periodic_timers_count += 1;
    } else if (non_periodic_timers.size() > taken_non_periodic_timers_count) {
        m_system_timer = non_periodic_timers[taken_non_periodic_timers_count];
        taken_non_periodic_timers_count += 1;
    }

    m_read_time_from_hardware = [](auto&&... args) -> u64 {
        return HPET::the().update_time(args...);
    };

    m_system_timer->set_callback([this]() { system_timer_callback(); });

    // Use the HPET main counter frequency for time purposes. This is likely
    // a much higher frequency than the interrupt itself and allows us to
    // keep a more accurate time
    m_can_query_precise_time.set();
    m_time_ticks_per_second = HPET::the().frequency();

    m_system_timer->try_to_set_frequency(m_system_timer->calculate_nearest_possible_frequency(OPTIMAL_TICKS_PER_SECOND_RATE));

    // We don't need an interrupt for time keeping purposes because we
    // can query the timer.
    m_time_keeper_timer = m_system_timer;

    if (TSC::is_supported_and_invariant()
        && TSC::the().calibrate(*m_system_timer)) {

        m_read_time_from_hardware = [](auto&&... args) -> u64 {
            return TSC::the().update_time(args...);
        };
        m_time_ticks_per_second = TSC::the().frequency();
    }

    if (periodic_timers.size() > taken_periodic_timers_count) {
        m_profile_timer = periodic_timers[taken_periodic_timers_count];
        taken_periodic_timers_count += 1;
    } else if (non_periodic_timers.size() > taken_non_periodic_timers_count) {
        m_profile_timer = non_periodic_timers[taken_non_periodic_timers_count];
        taken_non_periodic_timers_count += 1;
    }

    if (m_profile_timer) {
        m_profile_timer->set_callback(PerformanceManager::timer_tick);
        m_profile_timer->try_to_set_frequency(m_profile_timer->calculate_nearest_possible_frequency(1));
    }

    return true;
}

UNMAP_AFTER_INIT bool TimeManagement::probe_and_set_x86_legacy_hardware_timers()
{
    if (ACPI::is_enabled()) {
        if (ACPI::Parser::the()->x86_specific_flags().cmos_rtc_not_present) {
            dbgln("ACPI: CMOS RTC Not Present");
            return false;
        } else {
            dbgln("ACPI: CMOS RTC Present");
        }
    }

    VERIFY(s_hardware_timers->is_empty());
    s_hardware_timers->try_append(PIT::initialize([this]() { update_time(); })).release_value_but_fixme_should_propagate_errors();
    s_hardware_timers->try_append(RealTimeClock::create(TimeManagement::system_timer_tick)).release_value_but_fixme_should_propagate_errors();
    m_time_keeper_timer = (*s_hardware_timers)[0];
    m_system_timer = (*s_hardware_timers)[1];

    // The timer is only as accurate as the interrupts...
    m_time_ticks_per_second = m_time_keeper_timer->ticks_per_second();
    return true;
}
#elif ARCH(AARCH64)
UNMAP_AFTER_INIT bool TimeManagement::probe_and_set_aarch64_hardware_timers()
{
    if (s_hardware_timers->is_empty())
        PANIC("TimeManagement: No supported timer was found");

    // TODO: Use some kind of heuristic to decide which timer to use.
    m_system_timer = s_hardware_timers->last();
    dbgln("TimeManagement: System timer: {}", m_system_timer->model());

    m_time_ticks_per_second = m_system_timer->ticks_per_second();

    m_read_time_from_hardware = [this](auto&&... args) {
        if (m_system_timer->timer_type() == HardwareTimerType::RPiTimer)
            return static_cast<RPi::Timer*>(m_system_timer.ptr())->update_time(args...);
        if (m_system_timer->timer_type() == HardwareTimerType::ARMv8Timer)
            return static_cast<ARMv8Timer*>(m_system_timer.ptr())->update_time(args...);
        VERIFY_NOT_REACHED();
    };
    m_system_timer->set_callback([this]() { system_timer_callback(); });

    m_can_query_precise_time.set();
    m_time_keeper_timer = m_system_timer;

    return true;
}
#elif ARCH(RISCV64)
UNMAP_AFTER_INIT bool TimeManagement::probe_and_set_riscv64_hardware_timers()
{
    MUST(s_hardware_timers->try_append(RISCV64::Timer::initialize()));
    m_system_timer = (*s_hardware_timers)[0];
    m_time_ticks_per_second = m_system_timer->ticks_per_second();

    m_read_time_from_hardware = [this](auto&&... args) {
        return static_cast<RISCV64::Timer*>(m_system_timer.ptr())->update_time(args...);
    };
    m_system_timer->set_callback([this]() { system_timer_callback(); });

    m_can_query_precise_time.set();
    m_time_keeper_timer = m_system_timer;

    return true;
}
#else
#    error Unknown architecture
#endif

void TimeManagement::update_time()
{
    VERIFY(m_time_keeper_timer);
    // NOTE: m_seconds_since_boot and m_ticks_this_second are only ever
    // updated from here!

    if (m_can_query_precise_time.was_set()) {
        u32 update_iteration = m_update2.fetch_add(1, AK::MemoryOrder::memory_order_acquire);
        auto delta_ns = m_read_time_from_hardware(m_seconds_since_boot, m_ticks_this_second, false);
        // TODO: Apply m_remaining_epoch_time_adjustment
        m_epoch_time += Duration::from_nanoseconds(delta_ns);
        m_update1.store(update_iteration + 1, AK::MemoryOrder::memory_order_release);
    } else {
        increment_time_since_boot();
    }

    update_time_page();
}

void TimeManagement::increment_time_since_boot()
{
    VERIFY(!m_time_keeper_timer.is_null());

    // Compute time adjustment for adjtime. Let the clock run up to 1% fast or slow.
    // That way, adjtime can adjust up to 36 seconds per hour, without time getting very jumpy.
    // Once we have a smarter NTP service that also adjusts the frequency instead of just slewing time, maybe we can lower this.
    long nanos_per_tick = 1'000'000'000 / m_time_keeper_timer->ticks_per_second();
    time_t max_slew_nanos = nanos_per_tick / 100;

    u32 update_iteration = m_update2.fetch_add(1, AK::MemoryOrder::memory_order_acquire);

    auto slew_nanos = Duration::from_nanoseconds(
        clamp(m_remaining_epoch_time_adjustment.to_nanoseconds(), -max_slew_nanos, max_slew_nanos));
    m_remaining_epoch_time_adjustment -= slew_nanos;

    m_epoch_time += Duration::from_nanoseconds(nanos_per_tick + slew_nanos.to_nanoseconds());

    if (++m_ticks_this_second >= m_time_keeper_timer->ticks_per_second()) {
        // FIXME: Synchronize with other clock somehow to prevent drifting apart.
        ++m_seconds_since_boot;
        m_ticks_this_second = 0;
    }

    m_update1.store(update_iteration + 1, AK::MemoryOrder::memory_order_release);
}

void TimeManagement::system_timer_callback()
{
    if (Processor::is_bootstrap_processor()) {
        // `system_timer_callback()` is used when we have a single physical timer for both
        // the system and the time keeping. To avoid hammering the `update_time()`
        // function, only the bootstrap processor is doing the time keping.
        update_time();
    }
    system_timer_tick();
}

void TimeManagement::system_timer_tick()
{
    TimerQueue::the().fire();
    Scheduler::timer_tick();
}

bool TimeManagement::enable_profile_timer()
{
    if (!m_profile_timer)
        return false;
    if (m_profile_enable_count.fetch_add(1) == 0)
        return m_profile_timer->try_to_set_frequency(m_profile_timer->calculate_nearest_possible_frequency(OPTIMAL_PROFILE_TICKS_PER_SECOND_RATE));
    return true;
}

bool TimeManagement::disable_profile_timer()
{
    if (!m_profile_timer)
        return false;
    if (m_profile_enable_count.fetch_sub(1) == 1)
        return m_profile_timer->try_to_set_frequency(m_profile_timer->calculate_nearest_possible_frequency(1));
    return true;
}

void TimeManagement::update_time_page()
{
    auto& page = time_page();
    u32 update_iteration = AK::atomic_fetch_add(&page.update2, 1u, AK::MemoryOrder::memory_order_acquire);
    page.clocks[CLOCK_REALTIME_COARSE] = m_epoch_time.to_timespec();
    page.clocks[CLOCK_MONOTONIC_COARSE] = monotonic_time(TimePrecision::Coarse).time_since_start({}).to_timespec();
    AK::atomic_store(&page.update1, update_iteration + 1u, AK::MemoryOrder::memory_order_release);
}

TimePage& TimeManagement::time_page()
{
    return *static_cast<TimePage*>((void*)m_time_page_region->vaddr().as_ptr());
}

Memory::VMObject& TimeManagement::time_page_vmobject()
{
    return m_time_page_region->vmobject();
}

}