diff --git a/src/core/arm/arm_interface.cpp b/src/core/arm/arm_interface.cpp
index cef79b245a..cdf388fb9b 100644
--- a/src/core/arm/arm_interface.cpp
+++ b/src/core/arm/arm_interface.cpp
@@ -155,9 +155,10 @@ void ARM_Interface::Run() {
break;
}
- // Handle syscalls and scheduling (this may change the current thread)
+ // Handle syscalls and scheduling (this may change the current thread/core)
if (Has(hr, svc_call)) {
Kernel::Svc::Call(system, GetSvcNumber());
+ break;
}
if (Has(hr, break_loop) || !uses_wall_clock) {
break;
diff --git a/src/core/cpu_manager.cpp b/src/core/cpu_manager.cpp
index 37d3d83b98..4281941291 100644
--- a/src/core/cpu_manager.cpp
+++ b/src/core/cpu_manager.cpp
@@ -8,6 +8,7 @@
#include "core/core.h"
#include "core/core_timing.h"
#include "core/cpu_manager.h"
+#include "core/hle/kernel/k_interrupt_manager.h"
#include "core/hle/kernel/k_scheduler.h"
#include "core/hle/kernel/k_thread.h"
#include "core/hle/kernel/kernel.h"
@@ -41,6 +42,14 @@ void CpuManager::Shutdown() {
}
}
+void CpuManager::GuestActivateFunction() {
+ if (is_multicore) {
+ MultiCoreGuestActivate();
+ } else {
+ SingleCoreGuestActivate();
+ }
+}
+
void CpuManager::GuestThreadFunction() {
if (is_multicore) {
MultiCoreRunGuestThread();
@@ -49,36 +58,49 @@ void CpuManager::GuestThreadFunction() {
}
}
-void CpuManager::GuestRewindFunction() {
- if (is_multicore) {
- MultiCoreRunGuestLoop();
- } else {
- SingleCoreRunGuestLoop();
- }
-}
-
-void CpuManager::IdleThreadFunction() {
- if (is_multicore) {
- MultiCoreRunIdleThread();
- } else {
- SingleCoreRunIdleThread();
- }
-}
-
void CpuManager::ShutdownThreadFunction() {
ShutdownThread();
}
+void CpuManager::WaitForAndHandleInterrupt() {
+ auto& kernel = system.Kernel();
+ auto& physical_core = kernel.CurrentPhysicalCore();
+
+ ASSERT(Kernel::GetCurrentThread(kernel).GetDisableDispatchCount() == 1);
+
+ if (!physical_core.IsInterrupted()) {
+ physical_core.Idle();
+ }
+
+ HandleInterrupt();
+}
+
+void CpuManager::HandleInterrupt() {
+ auto& kernel = system.Kernel();
+ auto core_index = kernel.CurrentPhysicalCoreIndex();
+
+ Kernel::KInterruptManager::HandleInterrupt(kernel, static_cast<s32>(core_index));
+}
+
///////////////////////////////////////////////////////////////////////////////
/// MultiCore ///
///////////////////////////////////////////////////////////////////////////////
-void CpuManager::MultiCoreRunGuestThread() {
+void CpuManager::MultiCoreGuestActivate() {
+ // Similar to the HorizonKernelMain callback in HOS
auto& kernel = system.Kernel();
- kernel.CurrentScheduler()->OnThreadStart();
- auto* thread = kernel.CurrentScheduler()->GetSchedulerCurrentThread();
- auto& host_context = thread->GetHostContext();
- host_context->SetRewindPoint([this] { GuestRewindFunction(); });
+ auto* scheduler = kernel.CurrentScheduler();
+
+ scheduler->Activate();
+ UNREACHABLE();
+}
+
+void CpuManager::MultiCoreRunGuestThread() {
+ // Similar to UserModeThreadStarter in HOS
+ auto& kernel = system.Kernel();
+ auto* thread = kernel.GetCurrentEmuThread();
+ thread->EnableDispatch();
+
MultiCoreRunGuestLoop();
}
@@ -91,18 +113,8 @@ void CpuManager::MultiCoreRunGuestLoop() {
physical_core->Run();
physical_core = &kernel.CurrentPhysicalCore();
}
- {
- Kernel::KScopedDisableDispatch dd(kernel);
- physical_core->ArmInterface().ClearExclusiveState();
- }
- }
-}
-void CpuManager::MultiCoreRunIdleThread() {
- auto& kernel = system.Kernel();
- while (true) {
- Kernel::KScopedDisableDispatch dd(kernel);
- kernel.CurrentPhysicalCore().Idle();
+ HandleInterrupt();
}
}
@@ -110,83 +122,20 @@ void CpuManager::MultiCoreRunIdleThread() {
/// SingleCore ///
///////////////////////////////////////////////////////////////////////////////
-void CpuManager::SingleCoreRunGuestThread() {
- auto& kernel = system.Kernel();
- kernel.CurrentScheduler()->OnThreadStart();
- auto* thread = kernel.CurrentScheduler()->GetSchedulerCurrentThread();
- auto& host_context = thread->GetHostContext();
- host_context->SetRewindPoint([this] { GuestRewindFunction(); });
- SingleCoreRunGuestLoop();
-}
+void CpuManager::SingleCoreGuestActivate() {}
-void CpuManager::SingleCoreRunGuestLoop() {
- auto& kernel = system.Kernel();
- while (true) {
- auto* physical_core = &kernel.CurrentPhysicalCore();
- if (!physical_core->IsInterrupted()) {
- physical_core->Run();
- physical_core = &kernel.CurrentPhysicalCore();
- }
- kernel.SetIsPhantomModeForSingleCore(true);
- system.CoreTiming().Advance();
- kernel.SetIsPhantomModeForSingleCore(false);
- physical_core->ArmInterface().ClearExclusiveState();
- PreemptSingleCore();
- auto& scheduler = kernel.Scheduler(current_core);
- scheduler.RescheduleCurrentCore();
- }
-}
+void CpuManager::SingleCoreRunGuestThread() {}
-void CpuManager::SingleCoreRunIdleThread() {
- auto& kernel = system.Kernel();
- while (true) {
- auto& physical_core = kernel.CurrentPhysicalCore();
- PreemptSingleCore(false);
- system.CoreTiming().AddTicks(1000U);
- idle_count++;
- auto& scheduler = physical_core.Scheduler();
- scheduler.RescheduleCurrentCore();
- }
-}
+void CpuManager::SingleCoreRunGuestLoop() {}
-void CpuManager::PreemptSingleCore(bool from_running_enviroment) {
- {
- auto& kernel = system.Kernel();
- auto& scheduler = kernel.Scheduler(current_core);
- Kernel::KThread* current_thread = scheduler.GetSchedulerCurrentThread();
- if (idle_count >= 4 || from_running_enviroment) {
- if (!from_running_enviroment) {
- system.CoreTiming().Idle();
- idle_count = 0;
- }
- kernel.SetIsPhantomModeForSingleCore(true);
- system.CoreTiming().Advance();
- kernel.SetIsPhantomModeForSingleCore(false);
- }
- current_core.store((current_core + 1) % Core::Hardware::NUM_CPU_CORES);
- system.CoreTiming().ResetTicks();
- scheduler.Unload(scheduler.GetSchedulerCurrentThread());
-
- auto& next_scheduler = kernel.Scheduler(current_core);
- Common::Fiber::YieldTo(current_thread->GetHostContext(), *next_scheduler.ControlContext());
- }
-
- // May have changed scheduler
- {
- auto& scheduler = system.Kernel().Scheduler(current_core);
- scheduler.Reload(scheduler.GetSchedulerCurrentThread());
- if (!scheduler.IsIdle()) {
- idle_count = 0;
- }
- }
-}
+void CpuManager::PreemptSingleCore(bool from_running_enviroment) {}
void CpuManager::ShutdownThread() {
auto& kernel = system.Kernel();
+ auto* thread = kernel.GetCurrentEmuThread();
auto core = is_multicore ? kernel.CurrentPhysicalCoreIndex() : 0;
- auto* current_thread = kernel.GetCurrentEmuThread();
- Common::Fiber::YieldTo(current_thread->GetHostContext(), *core_data[core].host_context);
+ Common::Fiber::YieldTo(thread->GetHostContext(), *core_data[core].host_context);
UNREACHABLE();
}
@@ -218,9 +167,21 @@ void CpuManager::RunThread(std::size_t core) {
system.GPU().ObtainContext();
}
- auto* current_thread = system.Kernel().CurrentScheduler()->GetIdleThread();
- Kernel::SetCurrentThread(system.Kernel(), current_thread);
- Common::Fiber::YieldTo(data.host_context, *current_thread->GetHostContext());
+ auto& kernel = system.Kernel();
+
+ auto* main_thread = Kernel::KThread::Create(kernel);
+ main_thread->SetName(fmt::format("MainThread:{}", core));
+ ASSERT(Kernel::KThread::InitializeMainThread(system, main_thread, static_cast<s32>(core))
+ .IsSuccess());
+
+ auto* idle_thread = Kernel::KThread::Create(kernel);
+ ASSERT(Kernel::KThread::InitializeIdleThread(system, idle_thread, static_cast<s32>(core))
+ .IsSuccess());
+
+ kernel.SetCurrentEmuThread(main_thread);
+ kernel.CurrentScheduler()->Initialize(idle_thread);
+
+ Common::Fiber::YieldTo(data.host_context, *main_thread->GetHostContext());
}
} // namespace Core
diff --git a/src/core/cpu_manager.h b/src/core/cpu_manager.h
index 76dc58ee1e..8143424af2 100644
--- a/src/core/cpu_manager.h
+++ b/src/core/cpu_manager.h
@@ -5,6 +5,7 @@
#include <array>
#include <atomic>
+#include <csetjmp>
#include <functional>
#include <memory>
#include <thread>
@@ -47,10 +48,14 @@ public:
gpu_barrier->Sync();
}
+ void WaitForAndHandleInterrupt();
void Initialize();
void Shutdown();
- std::function<void()> GetGuestThreadStartFunc() {
+ std::function<void()> GetGuestActivateFunc() {
+ return [this] { GuestActivateFunction(); };
+ }
+ std::function<void()> GetGuestThreadFunc() {
return [this] { GuestThreadFunction(); };
}
std::function<void()> GetIdleThreadStartFunc() {
@@ -67,21 +72,22 @@ public:
}
private:
+ void GuestActivateFunction();
void GuestThreadFunction();
- void GuestRewindFunction();
void IdleThreadFunction();
void ShutdownThreadFunction();
+ void MultiCoreGuestActivate();
void MultiCoreRunGuestThread();
void MultiCoreRunGuestLoop();
- void MultiCoreRunIdleThread();
+ void SingleCoreGuestActivate();
void SingleCoreRunGuestThread();
void SingleCoreRunGuestLoop();
- void SingleCoreRunIdleThread();
static void ThreadStart(std::stop_token stop_token, CpuManager& cpu_manager, std::size_t core);
+ void HandleInterrupt();
void ShutdownThread();
void RunThread(std::size_t core);
diff --git a/src/core/hle/kernel/global_scheduler_context.cpp b/src/core/hle/kernel/global_scheduler_context.cpp
index 164436b261..21fd5cb678 100644
--- a/src/core/hle/kernel/global_scheduler_context.cpp
+++ b/src/core/hle/kernel/global_scheduler_context.cpp
@@ -41,7 +41,7 @@ void GlobalSchedulerContext::PreemptThreads() {
ASSERT(IsLocked());
for (u32 core_id = 0; core_id < Core::Hardware::NUM_CPU_CORES; core_id++) {
const u32 priority = preemption_priorities[core_id];
- kernel.Scheduler(core_id).RotateScheduledQueue(core_id, priority);
+ KScheduler::RotateScheduledQueue(kernel, core_id, priority);
// Signal an interrupt occurred. For core 3, this is a certainty, as preemption will result
// in the rotator thread being scheduled. For cores 0-2, this is to simulate or system
diff --git a/src/core/hle/kernel/k_interrupt_manager.cpp b/src/core/hle/kernel/k_interrupt_manager.cpp
index d606a7f865..1b577a5b3e 100644
--- a/src/core/hle/kernel/k_interrupt_manager.cpp
+++ b/src/core/hle/kernel/k_interrupt_manager.cpp
@@ -6,6 +6,7 @@
#include "core/hle/kernel/k_scheduler.h"
#include "core/hle/kernel/k_thread.h"
#include "core/hle/kernel/kernel.h"
+#include "core/hle/kernel/physical_core.h"
namespace Kernel::KInterruptManager {
@@ -15,6 +16,9 @@ void HandleInterrupt(KernelCore& kernel, s32 core_id) {
return;
}
+ // Acknowledge the interrupt.
+ kernel.PhysicalCore(core_id).ClearInterrupt();
+
auto& current_thread = GetCurrentThread(kernel);
// If the user disable count is set, we may need to pin the current thread.
@@ -27,6 +31,9 @@ void HandleInterrupt(KernelCore& kernel, s32 core_id) {
// Set the interrupt flag for the thread.
GetCurrentThread(kernel).SetInterruptFlag();
}
+
+ // Request interrupt scheduling.
+ kernel.CurrentScheduler()->RequestScheduleOnInterrupt();
}
} // namespace Kernel::KInterruptManager
diff --git a/src/core/hle/kernel/k_scheduler.cpp b/src/core/hle/kernel/k_scheduler.cpp
index d599d2bcb9..13915dbd99 100644
--- a/src/core/hle/kernel/k_scheduler.cpp
+++ b/src/core/hle/kernel/k_scheduler.cpp
@@ -27,69 +27,162 @@ static void IncrementScheduledCount(Kernel::KThread* thread) {
}
}
-void KScheduler::RescheduleCores(KernelCore& kernel, u64 cores_pending_reschedule) {
- auto scheduler = kernel.CurrentScheduler();
-
- u32 current_core{0xF};
- bool must_context_switch{};
- if (scheduler) {
- current_core = scheduler->core_id;
- // TODO(bunnei): Should be set to true when we deprecate single core
- must_context_switch = !kernel.IsPhantomModeForSingleCore();
- }
-
- while (cores_pending_reschedule != 0) {
- const auto core = static_cast<u32>(std::countr_zero(cores_pending_reschedule));
- ASSERT(core < Core::Hardware::NUM_CPU_CORES);
- if (!must_context_switch || core != current_core) {
- auto& phys_core = kernel.PhysicalCore(core);
- phys_core.Interrupt();
+KScheduler::KScheduler(KernelCore& kernel_) : kernel{kernel_} {
+ m_idle_stack = std::make_shared<Common::Fiber>([this] {
+ while (true) {
+ ScheduleImplOffStack();
}
- cores_pending_reschedule &= ~(1ULL << core);
- }
+ });
- for (std::size_t core_id = 0; core_id < Core::Hardware::NUM_CPU_CORES; ++core_id) {
- if (kernel.PhysicalCore(core_id).IsInterrupted()) {
- KInterruptManager::HandleInterrupt(kernel, static_cast<s32>(core_id));
- }
- }
+ m_state.needs_scheduling = true;
+}
- if (must_context_switch) {
- auto core_scheduler = kernel.CurrentScheduler();
- kernel.ExitSVCProfile();
- core_scheduler->RescheduleCurrentCore();
- kernel.EnterSVCProfile();
+KScheduler::~KScheduler() = default;
+
+void KScheduler::SetInterruptTaskRunnable() {
+ m_state.interrupt_task_runnable = true;
+ m_state.needs_scheduling = true;
+}
+
+void KScheduler::RequestScheduleOnInterrupt() {
+ m_state.needs_scheduling = true;
+
+ if (CanSchedule(kernel)) {
+ ScheduleOnInterrupt();
}
}
-u64 KScheduler::UpdateHighestPriorityThread(KThread* highest_thread) {
- KScopedSpinLock lk{guard};
- if (KThread* prev_highest_thread = state.highest_priority_thread;
- prev_highest_thread != highest_thread) {
- if (prev_highest_thread != nullptr) {
- IncrementScheduledCount(prev_highest_thread);
- prev_highest_thread->SetLastScheduledTick(system.CoreTiming().GetCPUTicks());
+void KScheduler::DisableScheduling(KernelCore& kernel) {
+ ASSERT(GetCurrentThread(kernel).GetDisableDispatchCount() >= 0);
+ GetCurrentThread(kernel).DisableDispatch();
+}
+
+void KScheduler::EnableScheduling(KernelCore& kernel, u64 cores_needing_scheduling) {
+ ASSERT(GetCurrentThread(kernel).GetDisableDispatchCount() >= 1);
+
+ auto* scheduler = kernel.CurrentScheduler();
+
+ if (!scheduler) {
+ // HACK: we cannot schedule from this thread, it is not a core thread
+ RescheduleCores(kernel, cores_needing_scheduling);
+ if (GetCurrentThread(kernel).GetDisableDispatchCount() == 1) {
+ // Special case to ensure dummy threads that are waiting block
+ GetCurrentThread(kernel).IfDummyThreadTryWait();
}
- if (state.should_count_idle) {
- if (highest_thread != nullptr) {
+ GetCurrentThread(kernel).EnableDispatch();
+ return;
+ }
+
+ scheduler->RescheduleOtherCores(cores_needing_scheduling);
+
+ if (GetCurrentThread(kernel).GetDisableDispatchCount() > 1) {
+ GetCurrentThread(kernel).EnableDispatch();
+ } else {
+ scheduler->RescheduleCurrentCore();
+ }
+}
+
+u64 KScheduler::UpdateHighestPriorityThreads(KernelCore& kernel) {
+ if (IsSchedulerUpdateNeeded(kernel)) {
+ return UpdateHighestPriorityThreadsImpl(kernel);
+ } else {
+ return 0;
+ }
+}
+
+void KScheduler::Schedule() {
+ ASSERT(GetCurrentThread(kernel).GetDisableDispatchCount() == 1);
+ ASSERT(m_core_id == GetCurrentCoreId(kernel));
+
+ ScheduleImpl();
+}
+
+void KScheduler::ScheduleOnInterrupt() {
+ GetCurrentThread(kernel).DisableDispatch();
+ Schedule();
+ GetCurrentThread(kernel).EnableDispatch();
+}
+
+void KScheduler::RescheduleCurrentCore() {
+ ASSERT(GetCurrentThread(kernel).GetDisableDispatchCount() == 1);
+
+ GetCurrentThread(kernel).EnableDispatch();
+
+ if (m_state.needs_scheduling.load()) {
+ // Disable interrupts, and then check again if rescheduling is needed.
+ // KScopedInterruptDisable intr_disable;
+
+ kernel.CurrentScheduler()->RescheduleCurrentCoreImpl();
+ }
+}
+
+void KScheduler::RescheduleCurrentCoreImpl() {
+ // Check that scheduling is needed.
+ if (m_state.needs_scheduling.load()) [[likely]] {
+ GetCurrentThread(kernel).DisableDispatch();
+ Schedule();
+ GetCurrentThread(kernel).EnableDispatch();
+ }
+}
+
+void KScheduler::Initialize(KThread* idle_thread) {
+ // Set core ID/idle thread/interrupt task manager.
+ m_core_id = GetCurrentCoreId(kernel);
+ m_idle_thread = idle_thread;
+ // m_state.idle_thread_stack = m_idle_thread->GetStackTop();
+ // m_state.interrupt_task_manager = &kernel.GetInterruptTaskManager();
+
+ // Insert the main thread into the priority queue.
+ // {
+ // KScopedSchedulerLock lk{kernel};
+ // GetPriorityQueue(kernel).PushBack(GetCurrentThreadPointer(kernel));
+ // SetSchedulerUpdateNeeded(kernel);
+ // }
+
+ // Bind interrupt handler.
+ // kernel.GetInterruptManager().BindHandler(
+ // GetSchedulerInterruptHandler(kernel), KInterruptName::Scheduler, m_core_id,
+ // KInterruptController::PriorityLevel_Scheduler, false, false);
+
+ // Set the current thread.
+ m_current_thread = GetCurrentThreadPointer(kernel);
+}
+
+void KScheduler::Activate() {
+ ASSERT(GetCurrentThread(kernel).GetDisableDispatchCount() == 1);
+
+ // m_state.should_count_idle = KTargetSystem::IsDebugMode();
+ m_is_active = true;
+ RescheduleCurrentCore();
+}
+
+u64 KScheduler::UpdateHighestPriorityThread(KThread* highest_thread) {
+ if (KThread* prev_highest_thread = m_state.highest_priority_thread;
+ prev_highest_thread != highest_thread) [[likely]] {
+ if (prev_highest_thread != nullptr) [[likely]] {
+ IncrementScheduledCount(prev_highest_thread);
+ prev_highest_thread->SetLastScheduledTick(kernel.System().CoreTiming().GetCPUTicks());
+ }
+ if (m_state.should_count_idle) {
+ if (highest_thread != nullptr) [[likely]] {
if (KProcess* process = highest_thread->GetOwnerProcess(); process != nullptr) {
- process->SetRunningThread(core_id, highest_thread, state.idle_count);
+ process->SetRunningThread(m_core_id, highest_thread, m_state.idle_count);
}
} else {
- state.idle_count++;
+ m_state.idle_count++;
}
}
- state.highest_priority_thread = highest_thread;
- state.needs_scheduling.store(true);
- return (1ULL << core_id);
+ m_state.highest_priority_thread = highest_thread;
+ m_state.needs_scheduling = true;
+ return (1ULL << m_core_id);
} else {
return 0;
}
}
u64 KScheduler::UpdateHighestPriorityThreadsImpl(KernelCore& kernel) {
- ASSERT(kernel.GlobalSchedulerContext().IsLocked());
+ ASSERT(IsSchedulerLockedByCurrentThread(kernel));
// Clear that we need to update.
ClearSchedulerUpdateNeeded(kernel);
@@ -98,18 +191,20 @@ u64 KScheduler::UpdateHighestPriorityThreadsImpl(KernelCore& kernel) {
KThread* top_threads[Core::Hardware::NUM_CPU_CORES];
auto& priority_queue = GetPriorityQueue(kernel);
- /// We want to go over all cores, finding the highest priority thread and determining if
- /// scheduling is needed for that core.
+ // We want to go over all cores, finding the highest priority thread and determining if
+ // scheduling is needed for that core.
for (size_t core_id = 0; core_id < Core::Hardware::NUM_CPU_CORES; core_id++) {
KThread* top_thread = priority_queue.GetScheduledFront(static_cast<s32>(core_id));
if (top_thread != nullptr) {
- // If the thread has no waiters, we need to check if the process has a thread pinned.
- if (top_thread->GetNumKernelWaiters() == 0) {
- if (KProcess* parent = top_thread->GetOwnerProcess(); parent != nullptr) {
- if (KThread* pinned = parent->GetPinnedThread(static_cast<s32>(core_id));
- pinned != nullptr && pinned != top_thread) {
- // We prefer our parent's pinned thread if possible. However, we also don't
- // want to schedule un-runnable threads.
+ // We need to check if the thread's process has a pinned thread.
+ if (KProcess* parent = top_thread->GetOwnerProcess()) {
+ // Check that there's a pinned thread other than the current top thread.
+ if (KThread* pinned = parent->GetPinnedThread(static_cast<s32>(core_id));
+ pinned != nullptr && pinned != top_thread) {
+ // We need to prefer threads with kernel waiters to the pinned thread.
+ if (top_thread->GetNumKernelWaiters() ==
+ 0 /* && top_thread != parent->GetExceptionThread() */) {
+ // If the pinned thread is runnable, use it.
if (pinned->GetRawState() == ThreadState::Runnable) {
top_thread = pinned;
} else {
@@ -129,7 +224,8 @@ u64 KScheduler::UpdateHighestPriorityThreadsImpl(KernelCore& kernel) {
// Idle cores are bad. We're going to try to migrate threads to each idle core in turn.
while (idle_cores != 0) {
- const auto core_id = static_cast<u32>(std::countr_zero(idle_cores));
+ const s32 core_id = static_cast<s32>(std::countr_zero(idle_cores));
+
if (KThread* suggested = priority_queue.GetSuggestedFront(core_id); suggested != nullptr) {
s32 migration_candidates[Core::Hardware::NUM_CPU_CORES];
size_t num_candidates = 0;
@@ -150,7 +246,6 @@ u64 KScheduler::UpdateHighestPriorityThreadsImpl(KernelCore& kernel) {
// The suggested thread isn't bound to its core, so we can migrate it!
suggested->SetActiveCore(core_id);
priority_queue.ChangeCore(suggested_core, suggested);
-
top_threads[core_id] = suggested;
cores_needing_scheduling |=
kernel.Scheduler(core_id).UpdateHighestPriorityThread(top_threads[core_id]);
@@ -183,7 +278,6 @@ u64 KScheduler::UpdateHighestPriorityThreadsImpl(KernelCore& kernel) {
// Perform the migration.
suggested->SetActiveCore(core_id);
priority_queue.ChangeCore(candidate_core, suggested);
-
top_threads[core_id] = suggested;
cores_needing_scheduling |=
kernel.Scheduler(core_id).UpdateHighestPriorityThread(
@@ -200,24 +294,223 @@ u64 KScheduler::UpdateHighestPriorityThreadsImpl(KernelCore& kernel) {
return cores_needing_scheduling;
}
+void KScheduler::SwitchThread(KThread* next_thread) {
+ KProcess* const cur_process = kernel.CurrentProcess();
+ KThread* const cur_thread = GetCurrentThreadPointer(kernel);
+
+ // We never want to schedule a null thread, so use the idle thread if we don't have a next.
+ if (next_thread == nullptr) {
+ next_thread = m_idle_thread;
+ }
+
+ if (next_thread->GetCurrentCore() != m_core_id) {
+ next_thread->SetCurrentCore(m_core_id);
+ }
+
+ // If we're not actually switching thread, there's nothing to do.
+ if (next_thread == cur_thread) {
+ return;
+ }
+
+ // Next thread is now known not to be nullptr, and must not be dispatchable.
+ ASSERT(next_thread->GetDisableDispatchCount() == 1);
+ ASSERT(!next_thread->IsDummyThread());
+
+ // Update the CPU time tracking variables.
+ const s64 prev_tick = m_last_context_switch_time;
+ const s64 cur_tick = kernel.System().CoreTiming().GetCPUTicks();
+ const s64 tick_diff = cur_tick - prev_tick;
+ cur_thread->AddCpuTime(m_core_id, tick_diff);
+ if (cur_process != nullptr) {
+ cur_process->UpdateCPUTimeTicks(tick_diff);
+ }
+ m_last_context_switch_time = cur_tick;
+
+ // Update our previous thread.
+ if (cur_process != nullptr) {
+ if (!cur_thread->IsTerminationRequested() && cur_thread->GetActiveCore() == m_core_id)
+ [[likely]] {
+ m_state.prev_thread = cur_thread;
+ } else {
+ m_state.prev_thread = nullptr;
+ }
+ }
+
+ // Switch the current process, if we're switching processes.
+ // if (KProcess *next_process = next_thread->GetOwnerProcess(); next_process != cur_process) {
+ // KProcess::Switch(cur_process, next_process);
+ // }
+
+ // Set the new thread.
+ SetCurrentThread(kernel, next_thread);
+ m_current_thread = next_thread;
+
+ // Set the new Thread Local region.
+ // cpu::SwitchThreadLocalRegion(GetInteger(next_thread->GetThreadLocalRegionAddress()));
+}
+
+void KScheduler::ScheduleImpl() {
+ // First, clear the needs scheduling bool.
+ m_state.needs_scheduling.store(false, std::memory_order_seq_cst);
+
+ // Load the appropriate thread pointers for scheduling.
+ KThread* const cur_thread{GetCurrentThreadPointer(kernel)};
+ KThread* highest_priority_thread{m_state.highest_priority_thread};
+
+ // Check whether there are runnable interrupt tasks.
+ if (m_state.interrupt_task_runnable) {
+ // The interrupt task is runnable.
+ // We want to switch to the interrupt task/idle thread.
+ highest_priority_thread = nullptr;
+ }
+
+ // If there aren't, we want to check if the highest priority thread is the same as the current
+ // thread.
+ if (highest_priority_thread == cur_thread) {
+ // If they're the same, then we can just return.
+ return;
+ }
+
+ // The highest priority thread is not the same as the current thread.
+ // Switch to the idle thread stack and continue executing from there.
+ m_idle_cur_thread = cur_thread;
+ m_idle_highest_priority_thread = highest_priority_thread;
+ Common::Fiber::YieldTo(cur_thread->host_context, *m_idle_stack);
+
+ // Returning from ScheduleImpl occurs after this thread has been scheduled again.
+}
+
+void KScheduler::ScheduleImplOffStack() {
+ KThread* const cur_thread{m_idle_cur_thread};
+ KThread* highest_priority_thread{m_idle_highest_priority_thread};
+
+ // Get a reference to the current thread's stack parameters.
+ auto& sp{cur_thread->GetStackParameters()};
+
+ // Save the original thread context.
+ {
+ auto& physical_core = kernel.System().CurrentPhysicalCore();
+ auto& cpu_core = physical_core.ArmInterface();
+ cpu_core.SaveContext(cur_thread->GetContext32());
+ cpu_core.SaveContext(cur_thread->GetContext64());
+ // Save the TPIDR_EL0 system register in case it was modified.
+ cur_thread->SetTPIDR_EL0(cpu_core.GetTPIDR_EL0());
+ cpu_core.ClearExclusiveState();
+ }
+
+ // Check if the thread is terminated by checking the DPC flags.
+ if ((sp.dpc_flags & static_cast<u32>(DpcFlag::Terminated)) == 0) {
+ // The thread isn't terminated, so we want to unlock it.
+ sp.m_lock.store(false, std::memory_order_seq_cst);
+ }
+
+ // The current thread's context has been entirely taken care of.
+ // Now we want to loop until we successfully switch the thread context.
+ while (true) {
+ // We're starting to try to do the context switch.
+ // Check if the highest priority thread is null.
+ if (!highest_priority_thread) {
+ // The next thread is nullptr!
+ // Switch to nullptr. This will actually switch to the idle thread.
+ SwitchThread(nullptr);
+
+ // We've switched to the idle thread, so we want to process interrupt tasks until we
+ // schedule a non-idle thread.
+ while (!m_state.interrupt_task_runnable) {
+ // Check if we need scheduling.
+ if (m_state.needs_scheduling.load(std::memory_order_seq_cst)) {
+ goto retry;
+ }
+
+ // Clear the previous thread.
+ m_state.prev_thread = nullptr;
+
+ // Wait for an interrupt before checking again.
+ kernel.System().GetCpuManager().WaitForAndHandleInterrupt();
+ }
+
+ // Execute any pending interrupt tasks.
+ // m_state.interrupt_task_manager->DoTasks();
+
+ // Clear the interrupt task thread as runnable.
+ m_state.interrupt_task_runnable = false;
+
+ // Retry the scheduling loop.
+ goto retry;
+ } else {
+ // We want to try to lock the highest priority thread's context.
+ // Try to take it.
+ bool expected{false};
+ while (!highest_priority_thread->stack_parameters.m_lock.compare_exchange_strong(
+ expected, true, std::memory_order_seq_cst)) {
+ // The highest priority thread's context is already locked.
+ // Check if we need scheduling. If we don't, we can retry directly.
+ if (m_state.needs_scheduling.load(std::memory_order_seq_cst)) {
+ // If we do, another core is interfering, and we must start again.
+ goto retry;
+ }
+ expected = false;
+ }
+
+ // It's time to switch the thread.
+ // Switch to the highest priority thread.
+ SwitchThread(highest_priority_thread);
+
+ // Check if we need scheduling. If we do, then we can't complete the switch and should
+ // retry.
+ if (m_state.needs_scheduling.load(std::memory_order_seq_cst)) {
+ // Our switch failed.
+ // We should unlock the thread context, and then retry.
+ highest_priority_thread->stack_parameters.m_lock.store(false,
+ std::memory_order_seq_cst);
+ goto retry;
+ } else {
+ break;
+ }
+ }
+
+ retry:
+
+ // We failed to successfully do the context switch, and need to retry.
+ // Clear needs_scheduling.
+ m_state.needs_scheduling.store(false, std::memory_order_seq_cst);
+
+ // Refresh the highest priority thread.
+ highest_priority_thread = m_state.highest_priority_thread;
+ }
+
+ // Reload the guest thread context.
+ {
+ auto& cpu_core = kernel.System().CurrentArmInterface();
+ cpu_core.LoadContext(highest_priority_thread->GetContext32());
+ cpu_core.LoadContext(highest_priority_thread->GetContext64());
+ cpu_core.SetTlsAddress(highest_priority_thread->GetTLSAddress());
+ cpu_core.SetTPIDR_EL0(highest_priority_thread->GetTPIDR_EL0());
+ cpu_core.LoadWatchpointArray(highest_priority_thread->GetOwnerProcess()->GetWatchpoints());
+ cpu_core.ClearExclusiveState();
+ }
+
+ // Reload the host thread.
+ Common::Fiber::YieldTo(m_idle_stack, *highest_priority_thread->host_context);
+}
+
void KScheduler::ClearPreviousThread(KernelCore& kernel, KThread* thread) {
- ASSERT(kernel.GlobalSchedulerContext().IsLocked());
+ ASSERT(IsSchedulerLockedByCurrentThread(kernel));
for (size_t i = 0; i < Core::Hardware::NUM_CPU_CORES; ++i) {
// Get an atomic reference to the core scheduler's previous thread.
- std::atomic_ref<KThread*> prev_thread(kernel.Scheduler(static_cast<s32>(i)).prev_thread);
- static_assert(std::atomic_ref<KThread*>::is_always_lock_free);
+ auto& prev_thread{kernel.Scheduler(i).m_state.prev_thread};
// Atomically clear the previous thread if it's our target.
KThread* compare = thread;
- prev_thread.compare_exchange_strong(compare, nullptr);
+ prev_thread.compare_exchange_strong(compare, nullptr, std::memory_order_seq_cst);
}
}
void KScheduler::OnThreadStateChanged(KernelCore& kernel, KThread* thread, ThreadState old_state) {
- ASSERT(kernel.GlobalSchedulerContext().IsLocked());
+ ASSERT(IsSchedulerLockedByCurrentThread(kernel));
// Check if the state has changed, because if it hasn't there's nothing to do.
- const auto cur_state = thread->GetRawState();
+ const ThreadState cur_state = thread->GetRawState();
if (cur_state == old_state) {
return;
}
@@ -237,12 +530,12 @@ void KScheduler::OnThreadStateChanged(KernelCore& kernel, KThread* thread, Threa
}
void KScheduler::OnThreadPriorityChanged(KernelCore& kernel, KThread* thread, s32 old_priority) {
- ASSERT(kernel.GlobalSchedulerContext().IsLocked());
+ ASSERT(IsSchedulerLockedByCurrentThread(kernel));
// If the thread is runnable, we want to change its priority in the queue.
if (thread->GetRawState() == ThreadState::Runnable) {
GetPriorityQueue(kernel).ChangePriority(old_priority,
- thread == kernel.GetCurrentEmuThread(), thread);
+ thread == GetCurrentThreadPointer(kernel), thread);
IncrementScheduledCount(thread);
SetSchedulerUpdateNeeded(kernel);
}
@@ -250,7 +543,7 @@ void KScheduler::OnThreadPriorityChanged(KernelCore& kernel, KThread* thread, s3
void KScheduler::OnThreadAffinityMaskChanged(KernelCore& kernel, KThread* thread,
const KAffinityMask& old_affinity, s32 old_core) {
- ASSERT(kernel.GlobalSchedulerContext().IsLocked());
+ ASSERT(IsSchedulerLockedByCurrentThread(kernel));
// If the thread is runnable, we want to change its affinity in the queue.
if (thread->GetRawState() == ThreadState::Runnable) {
@@ -260,15 +553,14 @@ void KScheduler::OnThreadAffinityMaskChanged(KernelCore& kernel, KThread* thread
}
}
-void KScheduler::RotateScheduledQueue(s32 cpu_core_id, s32 priority) {
- ASSERT(system.GlobalSchedulerContext().IsLocked());
+void KScheduler::RotateScheduledQueue(KernelCore& kernel, s32 core_id, s32 priority) {
+ ASSERT(IsSchedulerLockedByCurrentThread(kernel));
// Get a reference to the priority queue.
- auto& kernel = system.Kernel();
auto& priority_queue = GetPriorityQueue(kernel);
// Rotate the front of the queue to the end.
- KThread* top_thread = priority_queue.GetScheduledFront(cpu_core_id, priority);
+ KThread* top_thread = priority_queue.GetScheduledFront(core_id, priority);
KThread* next_thread = nullptr;
if (top_thread != nullptr) {
next_thread = priority_queue.MoveToScheduledBack(top_thread);
@@ -280,7 +572,7 @@ void KScheduler::RotateScheduledQueue(s32 cpu_core_id, s32 priority) {
// While we have a suggested thread, try to migrate it!
{
- KThread* suggested = priority_queue.GetSuggestedFront(cpu_core_id, priority);
+ KThread* suggested = priority_queue.GetSuggestedFront(core_id, priority);
while (suggested != nullptr) {
// Check if the suggested thread is the top thread on its core.
const s32 suggested_core = suggested->GetActiveCore();
@@ -301,7 +593,7 @@ void KScheduler::RotateScheduledQueue(s32 cpu_core_id, s32 priority) {
// to the front of the queue.
if (top_on_suggested_core == nullptr ||
top_on_suggested_core->GetPriority() >= HighestCoreMigrationAllowedPriority) {
- suggested->SetActiveCore(cpu_core_id);
+ suggested->SetActiveCore(core_id);
priority_queue.ChangeCore(suggested_core, suggested, true);
IncrementScheduledCount(suggested);
break;
@@ -309,22 +601,21 @@ void KScheduler::RotateScheduledQueue(s32 cpu_core_id, s32 priority) {
}
// Get the next suggestion.
- suggested = priority_queue.GetSamePriorityNext(cpu_core_id, suggested);
+ suggested = priority_queue.GetSamePriorityNext(core_id, suggested);
}
}
// Now that we might have migrated a thread with the same priority, check if we can do better.
-
{
- KThread* best_thread = priority_queue.GetScheduledFront(cpu_core_id);
+ KThread* best_thread = priority_queue.GetScheduledFront(core_id);
if (best_thread == GetCurrentThreadPointer(kernel)) {
- best_thread = priority_queue.GetScheduledNext(cpu_core_id, best_thread);
+ best_thread = priority_queue.GetScheduledNext(core_id, best_thread);
}
// If the best thread we can choose has a priority the same or worse than ours, try to
// migrate a higher priority thread.
if (best_thread != nullptr && best_thread->GetPriority() >= priority) {
- KThread* suggested = priority_queue.GetSuggestedFront(cpu_core_id);
+ KThread* suggested = priority_queue.GetSuggestedFront(core_id);
while (suggested != nullptr) {
// If the suggestion's priority is the same as ours, don't bother.
if (suggested->GetPriority() >= best_thread->GetPriority()) {
@@ -343,7 +634,7 @@ void KScheduler::RotateScheduledQueue(s32 cpu_core_id, s32 priority) {
if (top_on_suggested_core == nullptr ||
top_on_suggested_core->GetPriority() >=
HighestCoreMigrationAllowedPriority) {
- suggested->SetActiveCore(cpu_core_id);
+ suggested->SetActiveCore(core_id);
priority_queue.ChangeCore(suggested_core, suggested, true);
IncrementScheduledCount(suggested);
break;
@@ -351,7 +642,7 @@ void KScheduler::RotateScheduledQueue(s32 cpu_core_id, s32 priority) {
}
// Get the next suggestion.
- suggested = priority_queue.GetSuggestedNext(cpu_core_id, suggested);
+ suggested = priority_queue.GetSuggestedNext(core_id, suggested);
}
}
}
@@ -360,64 +651,6 @@ void KScheduler::RotateScheduledQueue(s32 cpu_core_id, s32 priority) {
SetSchedulerUpdateNeeded(kernel);
}
-bool KScheduler::CanSchedule(KernelCore& kernel) {
- return kernel.GetCurrentEmuThread()->GetDisableDispatchCount() <= 1;
-}
-
-bool KScheduler::IsSchedulerUpdateNeeded(const KernelCore& kernel) {
- return kernel.GlobalSchedulerContext().scheduler_update_needed.load(std::memory_order_acquire);
-}
-
-void KScheduler::SetSchedulerUpdateNeeded(KernelCore& kernel) {
- kernel.GlobalSchedulerContext().scheduler_update_needed.store(true, std::memory_order_release);
-}
-
-void KScheduler::ClearSchedulerUpdateNeeded(KernelCore& kernel) {
- kernel.GlobalSchedulerContext().scheduler_update_needed.store(false, std::memory_order_release);
-}
-
-void KScheduler::DisableScheduling(KernelCore& kernel) {
- // If we are shutting down the kernel, none of this is relevant anymore.
- if (kernel.IsShuttingDown()) {
- return;
- }
-
- ASSERT(GetCurrentThreadPointer(kernel)->GetDisableDispatchCount() >= 0);
- GetCurrentThreadPointer(kernel)->DisableDispatch();
-}
-
-void KScheduler::EnableScheduling(KernelCore& kernel, u64 cores_needing_scheduling) {
- // If we are shutting down the kernel, none of this is relevant anymore.
- if (kernel.IsShuttingDown()) {
- return;
- }
-
- auto* current_thread = GetCurrentThreadPointer(kernel);
-
- ASSERT(current_thread->GetDisableDispatchCount() >= 1);
-
- if (current_thread->GetDisableDispatchCount() > 1) {
- current_thread->EnableDispatch();
- } else {
- RescheduleCores(kernel, cores_needing_scheduling);
- }
-
- // Special case to ensure dummy threads that are waiting block.
- current_thread->IfDummyThreadTryWait();
-}
-
-u64 KScheduler::UpdateHighestPriorityThreads(KernelCore& kernel) {
- if (IsSchedulerUpdateNeeded(kernel)) {
- return UpdateHighestPriorityThreadsImpl(kernel);
- } else {
- return 0;
- }
-}
-
-KSchedulerPriorityQueue& KScheduler::GetPriorityQueue(KernelCore& kernel) {
- return kernel.GlobalSchedulerContext().priority_queue;
-}
-
void KScheduler::YieldWithoutCoreMigration(KernelCore& kernel) {
// Validate preconditions.
ASSERT(CanSchedule(kernel));
@@ -437,7 +670,7 @@ void KScheduler::YieldWithoutCoreMigration(KernelCore& kernel) {
// Perform the yield.
{
- KScopedSchedulerLock lock(kernel);
+ KScopedSchedulerLock sl{kernel};
const auto cur_state = cur_thread.GetRawState();
if (cur_state == ThreadState::Runnable) {
@@ -476,7 +709,7 @@ void KScheduler::YieldWithCoreMigration(KernelCore& kernel) {
// Perform the yield.
{
- KScopedSchedulerLock lock(kernel);
+ KScopedSchedulerLock sl{kernel};
const auto cur_state = cur_thread.GetRawState();
if (cur_state == ThreadState::Runnable) {
@@ -496,7 +729,7 @@ void KScheduler::YieldWithCoreMigration(KernelCore& kernel) {
if (KThread* running_on_suggested_core =
(suggested_core >= 0)
- ? kernel.Scheduler(suggested_core).state.highest_priority_thread
+ ? kernel.Scheduler(suggested_core).m_state.highest_priority_thread
: nullptr;
running_on_suggested_core != suggested) {
// If the current thread's priority is higher than our suggestion's we prefer
@@ -564,7 +797,7 @@ void KScheduler::YieldToAnyThread(KernelCore& kernel) {
// Perform the yield.
{
- KScopedSchedulerLock lock(kernel);
+ KScopedSchedulerLock sl{kernel};
const auto cur_state = cur_thread.GetRawState();
if (cur_state == ThreadState::Runnable) {
@@ -621,223 +854,19 @@ void KScheduler::YieldToAnyThread(KernelCore& kernel) {
}
}
-KScheduler::KScheduler(Core::System& system_, s32 core_id_) : system{system_}, core_id{core_id_} {
- switch_fiber = std::make_shared<Common::Fiber>([this] { SwitchToCurrent(); });
- state.needs_scheduling.store(true);
- state.interrupt_task_thread_runnable = false;
- state.should_count_idle = false;
- state.idle_count = 0;
- state.idle_thread_stack = nullptr;
- state.highest_priority_thread = nullptr;
-}
-
-void KScheduler::Finalize() {
- if (idle_thread) {
- idle_thread->Close();
- idle_thread = nullptr;
+void KScheduler::RescheduleOtherCores(u64 cores_needing_scheduling) {
+ if (const u64 core_mask = cores_needing_scheduling & ~(1ULL << m_core_id); core_mask != 0) {
+ RescheduleCores(kernel, core_mask);
}
}
-KScheduler::~KScheduler() {
- ASSERT(!idle_thread);
-}
-
-KThread* KScheduler::GetSchedulerCurrentThread() const {
- if (auto result = current_thread.load(); result) {
- return result;
- }
- return idle_thread;
-}
-
-u64 KScheduler::GetLastContextSwitchTicks() const {
- return last_context_switch_time;
-}
-
-void KScheduler::RescheduleCurrentCore() {
- ASSERT(GetCurrentThread(system.Kernel()).GetDisableDispatchCount() == 1);
-
- auto& phys_core = system.Kernel().PhysicalCore(core_id);
- if (phys_core.IsInterrupted()) {
- phys_core.ClearInterrupt();
- }
-
- guard.Lock();
- if (state.needs_scheduling.load()) {
- Schedule();
- } else {
- GetCurrentThread(system.Kernel()).EnableDispatch();
- guard.Unlock();
- }
-}
-
-void KScheduler::OnThreadStart() {
- SwitchContextStep2();
-}
-
-void KScheduler::Unload(KThread* thread) {
- ASSERT(thread);
-
- LOG_TRACE(Kernel, "core {}, unload thread {}", core_id, thread ? thread->GetName() : "nullptr");
-
- if (thread->IsCallingSvc()) {
- thread->ClearIsCallingSvc();
- }
-
- auto& physical_core = system.Kernel().PhysicalCore(core_id);
- if (!physical_core.IsInitialized()) {
- return;
- }
-
- Core::ARM_Interface& cpu_core = physical_core.ArmInterface();
- cpu_core.SaveContext(thread->GetContext32());
- cpu_core.SaveContext(thread->GetContext64());
- // Save the TPIDR_EL0 system register in case it was modified.
- thread->SetTPIDR_EL0(cpu_core.GetTPIDR_EL0());
- cpu_core.ClearExclusiveState();
-
- if (!thread->IsTerminationRequested() && thread->GetActiveCore() == core_id) {
- prev_thread = thread;
- } else {
- prev_thread = nullptr;
- }
-
- thread->context_guard.unlock();
-}
-
-void KScheduler::Reload(KThread* thread) {
- LOG_TRACE(Kernel, "core {}, reload thread {}", core_id, thread->GetName());
-
- Core::ARM_Interface& cpu_core = system.ArmInterface(core_id);
- cpu_core.LoadContext(thread->GetContext32());
- cpu_core.LoadContext(thread->GetContext64());
- cpu_core.LoadWatchpointArray(thread->GetOwnerProcess()->GetWatchpoints());
- cpu_core.SetTlsAddress(thread->GetTLSAddress());
- cpu_core.SetTPIDR_EL0(thread->GetTPIDR_EL0());
- cpu_core.ClearExclusiveState();
-}
-
-void KScheduler::SwitchContextStep2() {
- // Load context of new thread
- Reload(GetCurrentThreadPointer(system.Kernel()));
-
- RescheduleCurrentCore();
-}
-
-void KScheduler::Schedule() {
- ASSERT(GetCurrentThread(system.Kernel()).GetDisableDispatchCount() == 1);
- this->ScheduleImpl();
-}
-
-void KScheduler::ScheduleImpl() {
- KThread* previous_thread = GetCurrentThreadPointer(system.Kernel());
- KThread* next_thread = state.highest_priority_thread;
-
- state.needs_scheduling.store(false);
-
- // We never want to schedule a null thread, so use the idle thread if we don't have a next.
- if (next_thread == nullptr) {
- next_thread = idle_thread;
- }
-
- if (next_thread->GetCurrentCore() != core_id) {
- next_thread->SetCurrentCore(core_id);
- }
-
- // We never want to schedule a dummy thread, as these are only used by host threads for locking.
- if (next_thread->GetThreadType() == ThreadType::Dummy) {
- ASSERT_MSG(false, "Dummy threads should never be scheduled!");
- next_thread = idle_thread;
- }
-
- // If we're not actually switching thread, there's nothing to do.
- if (next_thread == current_thread.load()) {
- previous_thread->EnableDispatch();
- guard.Unlock();
- return;
- }
-
- // Update the CPU time tracking variables.
- KProcess* const previous_process = system.Kernel().CurrentProcess();
- UpdateLastContextSwitchTime(previous_thread, previous_process);
-
- // Save context for previous thread
- Unload(previous_thread);
-
- std::shared_ptr<Common::Fiber>* old_context;
- old_context = &previous_thread->GetHostContext();
-
- // Set the new thread.
- SetCurrentThread(system.Kernel(), next_thread);
- current_thread.store(next_thread);
-
- guard.Unlock();
-
- Common::Fiber::YieldTo(*old_context, *switch_fiber);
- /// When a thread wakes up, the scheduler may have changed to other in another core.
- auto& next_scheduler = *system.Kernel().CurrentScheduler();
- next_scheduler.SwitchContextStep2();
-}
-
-void KScheduler::SwitchToCurrent() {
- while (true) {
- {
- KScopedSpinLock lk{guard};
- current_thread.store(state.highest_priority_thread);
- state.needs_scheduling.store(false);
+void KScheduler::RescheduleCores(KernelCore& kernel, u64 core_mask) {
+ // Send IPI
+ for (size_t i = 0; i < Core::Hardware::NUM_CPU_CORES; i++) {
+ if (core_mask & (1ULL << i)) {
+ kernel.PhysicalCore(i).Interrupt();
}
- const auto is_switch_pending = [this] {
- KScopedSpinLock lk{guard};
- return state.needs_scheduling.load();
- };
- do {
- auto next_thread = current_thread.load();
- if (next_thread != nullptr) {
- const auto locked = next_thread->context_guard.try_lock();
- if (state.needs_scheduling.load()) {
- next_thread->context_guard.unlock();
- break;
- }
- if (next_thread->GetActiveCore() != core_id) {
- next_thread->context_guard.unlock();
- break;
- }
- if (!locked) {
- continue;
- }
- }
- auto thread = next_thread ? next_thread : idle_thread;
- SetCurrentThread(system.Kernel(), thread);
- Common::Fiber::YieldTo(switch_fiber, *thread->GetHostContext());
- } while (!is_switch_pending());
}
}
-void KScheduler::UpdateLastContextSwitchTime(KThread* thread, KProcess* process) {
- const u64 prev_switch_ticks = last_context_switch_time;
- const u64 most_recent_switch_ticks = system.CoreTiming().GetCPUTicks();
- const u64 update_ticks = most_recent_switch_ticks - prev_switch_ticks;
-
- if (thread != nullptr) {
- thread->AddCpuTime(core_id, update_ticks);
- }
-
- if (process != nullptr) {
- process->UpdateCPUTimeTicks(update_ticks);
- }
-
- last_context_switch_time = most_recent_switch_ticks;
-}
-
-void KScheduler::Initialize() {
- idle_thread = KThread::Create(system.Kernel());
- ASSERT(KThread::InitializeIdleThread(system, idle_thread, core_id).IsSuccess());
- idle_thread->SetName(fmt::format("IdleThread:{}", core_id));
- idle_thread->EnableDispatch();
-}
-
-KScopedSchedulerLock::KScopedSchedulerLock(KernelCore& kernel)
- : KScopedLock(kernel.GlobalSchedulerContext().SchedulerLock()) {}
-
-KScopedSchedulerLock::~KScopedSchedulerLock() = default;
-
} // namespace Kernel
diff --git a/src/core/hle/kernel/k_scheduler.h b/src/core/hle/kernel/k_scheduler.h
index 6a4760eca1..8f4eebf6ac 100644
--- a/src/core/hle/kernel/k_scheduler.h
+++ b/src/core/hle/kernel/k_scheduler.h
@@ -11,6 +11,7 @@
#include "core/hle/kernel/k_scheduler_lock.h"
#include "core/hle/kernel/k_scoped_lock.h"
#include "core/hle/kernel/k_spin_lock.h"
+#include "core/hle/kernel/k_thread.h"
namespace Common {
class Fiber;
@@ -23,184 +24,139 @@ class System;
namespace Kernel {
class KernelCore;
+class KInterruptTaskManager;
class KProcess;
-class SchedulerLock;
class KThread;
+class KScopedDisableDispatch;
+class KScopedSchedulerLock;
+class KScopedSchedulerLockAndSleep;
class KScheduler final {
public:
- explicit KScheduler(Core::System& system_, s32 core_id_);
+ YUZU_NON_COPYABLE(KScheduler);
+ YUZU_NON_MOVEABLE(KScheduler);
+
+ using LockType = KAbstractSchedulerLock<KScheduler>;
+
+ explicit KScheduler(KernelCore& kernel);
~KScheduler();
- void Finalize();
+ void Initialize(KThread* idle_thread);
+ void Activate();
- /// Reschedules to the next available thread (call after current thread is suspended)
- void RescheduleCurrentCore();
+ void SetInterruptTaskRunnable();
+ void RequestScheduleOnInterrupt();
- /// Reschedules cores pending reschedule, to be called on EnableScheduling.
- static void RescheduleCores(KernelCore& kernel, u64 cores_pending_reschedule);
-
- /// The next two are for SingleCore Only.
- /// Unload current thread before preempting core.
- void Unload(KThread* thread);
-
- /// Reload current thread after core preemption.
- void Reload(KThread* thread);
-
- /// Gets the current running thread
- [[nodiscard]] KThread* GetSchedulerCurrentThread() const;
-
- /// Gets the idle thread
- [[nodiscard]] KThread* GetIdleThread() const {
- return idle_thread;
+ u64 GetIdleCount() {
+ return m_state.idle_count;
}
- /// Returns true if the scheduler is idle
- [[nodiscard]] bool IsIdle() const {
- return GetSchedulerCurrentThread() == idle_thread;
+ KThread* GetIdleThread() const {
+ return m_idle_thread;
}
- /// Gets the timestamp for the last context switch in ticks.
- [[nodiscard]] u64 GetLastContextSwitchTicks() const;
-
- [[nodiscard]] bool ContextSwitchPending() const {
- return state.needs_scheduling.load(std::memory_order_relaxed);
+ KThread* GetPreviousThread() const {
+ return m_state.prev_thread;
}
- void Initialize();
-
- void OnThreadStart();
-
- [[nodiscard]] std::shared_ptr<Common::Fiber>& ControlContext() {
- return switch_fiber;
+ KThread* GetSchedulerCurrentThread() const {
+ return m_current_thread.load();
}
- [[nodiscard]] const std::shared_ptr<Common::Fiber>& ControlContext() const {
- return switch_fiber;
+ s64 GetLastContextSwitchTime() const {
+ return m_last_context_switch_time;
}
- [[nodiscard]] u64 UpdateHighestPriorityThread(KThread* highest_thread);
+ // Static public API.
+ static bool CanSchedule(KernelCore& kernel) {
+ return kernel.GetCurrentEmuThread()->GetDisableDispatchCount() == 0;
+ }
+ static bool IsSchedulerLockedByCurrentThread(KernelCore& kernel) {
+ return kernel.GlobalSchedulerContext().scheduler_lock.IsLockedByCurrentThread();
+ }
- /**
- * Takes a thread and moves it to the back of the it's priority list.
- *
- * @note This operation can be redundant and no scheduling is changed if marked as so.
- */
- static void YieldWithoutCoreMigration(KernelCore& kernel);
+ static bool IsSchedulerUpdateNeeded(KernelCore& kernel) {
+ return kernel.GlobalSchedulerContext().scheduler_update_needed;
+ }
+ static void SetSchedulerUpdateNeeded(KernelCore& kernel) {
+ kernel.GlobalSchedulerContext().scheduler_update_needed = true;
+ }
+ static void ClearSchedulerUpdateNeeded(KernelCore& kernel) {
+ kernel.GlobalSchedulerContext().scheduler_update_needed = false;
+ }
- /**
- * Takes a thread and moves it to the back of the it's priority list.
- * Afterwards, tries to pick a suggested thread from the suggested queue that has worse time or
- * a better priority than the next thread in the core.
- *
- * @note This operation can be redundant and no scheduling is changed if marked as so.
- */
- static void YieldWithCoreMigration(KernelCore& kernel);
+ static void DisableScheduling(KernelCore& kernel);
+ static void EnableScheduling(KernelCore& kernel, u64 cores_needing_scheduling);
- /**
- * Takes a thread and moves it out of the scheduling queue.
- * and into the suggested queue. If no thread can be scheduled afterwards in that core,
- * a suggested thread is obtained instead.
- *
- * @note This operation can be redundant and no scheduling is changed if marked as so.
- */
- static void YieldToAnyThread(KernelCore& kernel);
+ static u64 UpdateHighestPriorityThreads(KernelCore& kernel);
static void ClearPreviousThread(KernelCore& kernel, KThread* thread);
- /// Notify the scheduler a thread's status has changed.
static void OnThreadStateChanged(KernelCore& kernel, KThread* thread, ThreadState old_state);
-
- /// Notify the scheduler a thread's priority has changed.
static void OnThreadPriorityChanged(KernelCore& kernel, KThread* thread, s32 old_priority);
-
- /// Notify the scheduler a thread's core and/or affinity mask has changed.
static void OnThreadAffinityMaskChanged(KernelCore& kernel, KThread* thread,
const KAffinityMask& old_affinity, s32 old_core);
- static bool CanSchedule(KernelCore& kernel);
- static bool IsSchedulerUpdateNeeded(const KernelCore& kernel);
- static void SetSchedulerUpdateNeeded(KernelCore& kernel);
- static void ClearSchedulerUpdateNeeded(KernelCore& kernel);
- static void DisableScheduling(KernelCore& kernel);
- static void EnableScheduling(KernelCore& kernel, u64 cores_needing_scheduling);
- [[nodiscard]] static u64 UpdateHighestPriorityThreads(KernelCore& kernel);
+ static void RotateScheduledQueue(KernelCore& kernel, s32 core_id, s32 priority);
+ static void RescheduleCores(KernelCore& kernel, u64 cores_needing_scheduling);
+
+ static void YieldWithoutCoreMigration(KernelCore& kernel);
+ static void YieldWithCoreMigration(KernelCore& kernel);
+ static void YieldToAnyThread(KernelCore& kernel);
private:
- friend class GlobalSchedulerContext;
+ // Static private API.
+ static KSchedulerPriorityQueue& GetPriorityQueue(KernelCore& kernel) {
+ return kernel.GlobalSchedulerContext().priority_queue;
+ }
+ static u64 UpdateHighestPriorityThreadsImpl(KernelCore& kernel);
- /**
- * Takes care of selecting the new scheduled threads in three steps:
- *
- * 1. First a thread is selected from the top of the priority queue. If no thread
- * is obtained then we move to step two, else we are done.
- *
- * 2. Second we try to get a suggested thread that's not assigned to any core or
- * that is not the top thread in that core.
- *
- * 3. Third is no suggested thread is found, we do a second pass and pick a running
- * thread in another core and swap it with its current thread.
- *
- * returns the cores needing scheduling.
- */
- [[nodiscard]] static u64 UpdateHighestPriorityThreadsImpl(KernelCore& kernel);
-
- [[nodiscard]] static KSchedulerPriorityQueue& GetPriorityQueue(KernelCore& kernel);
-
- void RotateScheduledQueue(s32 cpu_core_id, s32 priority);
+ // Instanced private API.
+ void ScheduleImpl();
+ void ScheduleImplOffStack();
+ void SwitchThread(KThread* next_thread);
void Schedule();
+ void ScheduleOnInterrupt();
- /// Switches the CPU's active thread context to that of the specified thread
- void ScheduleImpl();
+ void RescheduleOtherCores(u64 cores_needing_scheduling);
+ void RescheduleCurrentCore();
+ void RescheduleCurrentCoreImpl();
- /// When a thread wakes up, it must run this through it's new scheduler
- void SwitchContextStep2();
+ u64 UpdateHighestPriorityThread(KThread* thread);
- /**
- * Called on every context switch to update the internal timestamp
- * This also updates the running time ticks for the given thread and
- * process using the following difference:
- *
- * ticks += most_recent_ticks - last_context_switch_ticks
- *
- * The internal tick timestamp for the scheduler is simply the
- * most recent tick count retrieved. No special arithmetic is
- * applied to it.
- */
- void UpdateLastContextSwitchTime(KThread* thread, KProcess* process);
-
- void SwitchToCurrent();
-
- KThread* prev_thread{};
- std::atomic<KThread*> current_thread{};
-
- KThread* idle_thread{};
-
- std::shared_ptr<Common::Fiber> switch_fiber{};
+private:
+ friend class KScopedDisableDispatch;
struct SchedulingState {
- std::atomic<bool> needs_scheduling{};
- bool interrupt_task_thread_runnable{};
- bool should_count_idle{};
- u64 idle_count{};
- KThread* highest_priority_thread{};
- void* idle_thread_stack{};
+ std::atomic<bool> needs_scheduling{false};
+ bool interrupt_task_runnable{false};
+ bool should_count_idle{false};
+ u64 idle_count{0};
+ KThread* highest_priority_thread{nullptr};
+ void* idle_thread_stack{nullptr};
+ std::atomic<KThread*> prev_thread{nullptr};
+ KInterruptTaskManager* interrupt_task_manager{nullptr};
};
- SchedulingState state;
+ KernelCore& kernel;
+ SchedulingState m_state;
+ bool m_is_active{false};
+ s32 m_core_id{0};
+ s64 m_last_context_switch_time{0};
+ KThread* m_idle_thread{nullptr};
+ std::atomic<KThread*> m_current_thread{nullptr};
- Core::System& system;
- u64 last_context_switch_time{};
- const s32 core_id;
-
- KSpinLock guard{};
+ std::shared_ptr<Common::Fiber> m_idle_stack{};
+ KThread* m_idle_cur_thread{};
+ KThread* m_idle_highest_priority_thread{};
};
-class [[nodiscard]] KScopedSchedulerLock : KScopedLock<GlobalSchedulerContext::LockType> {
+class KScopedSchedulerLock : public KScopedLock<KScheduler::LockType> {
public:
- explicit KScopedSchedulerLock(KernelCore& kernel);
- ~KScopedSchedulerLock();
+ explicit KScopedSchedulerLock(KernelCore& kernel)
+ : KScopedLock(kernel.GlobalSchedulerContext().scheduler_lock) {}
+ ~KScopedSchedulerLock() = default;
};
} // namespace Kernel
diff --git a/src/core/hle/kernel/k_scheduler_lock.h b/src/core/hle/kernel/k_scheduler_lock.h
index 4fa2569701..73314b45e9 100644
--- a/src/core/hle/kernel/k_scheduler_lock.h
+++ b/src/core/hle/kernel/k_scheduler_lock.h
@@ -5,9 +5,11 @@
#include <atomic>
#include "common/assert.h"
+#include "core/hle/kernel/k_interrupt_manager.h"
#include "core/hle/kernel/k_spin_lock.h"
#include "core/hle/kernel/k_thread.h"
#include "core/hle/kernel/kernel.h"
+#include "core/hle/kernel/physical_core.h"
namespace Kernel {
diff --git a/src/core/hle/kernel/k_thread.cpp b/src/core/hle/kernel/k_thread.cpp
index 90de867707..9daa589b5a 100644
--- a/src/core/hle/kernel/k_thread.cpp
+++ b/src/core/hle/kernel/k_thread.cpp
@@ -261,9 +261,14 @@ Result KThread::InitializeDummyThread(KThread* thread) {
return thread->Initialize({}, {}, {}, DummyThreadPriority, 3, {}, ThreadType::Dummy);
}
+Result KThread::InitializeMainThread(Core::System& system, KThread* thread, s32 virt_core) {
+ return InitializeThread(thread, {}, {}, {}, IdleThreadPriority, virt_core, {}, ThreadType::Main,
+ system.GetCpuManager().GetGuestActivateFunc());
+}
+
Result KThread::InitializeIdleThread(Core::System& system, KThread* thread, s32 virt_core) {
return InitializeThread(thread, {}, {}, {}, IdleThreadPriority, virt_core, {}, ThreadType::Main,
- system.GetCpuManager().GetIdleThreadStartFunc());
+ abort);
}
Result KThread::InitializeHighPriorityThread(Core::System& system, KThread* thread,
@@ -277,7 +282,7 @@ Result KThread::InitializeUserThread(Core::System& system, KThread* thread, KThr
KProcess* owner) {
system.Kernel().GlobalSchedulerContext().AddThread(thread);
return InitializeThread(thread, func, arg, user_stack_top, prio, virt_core, owner,
- ThreadType::User, system.GetCpuManager().GetGuestThreadStartFunc());
+ ThreadType::User, system.GetCpuManager().GetGuestThreadFunc());
}
void KThread::PostDestroy(uintptr_t arg) {
@@ -1058,6 +1063,8 @@ void KThread::Exit() {
// Register the thread as a work task.
KWorkerTaskManager::AddTask(kernel, KWorkerTaskManager::WorkerType::Exit, this);
}
+
+ UNREACHABLE_MSG("KThread::Exit() would return");
}
Result KThread::Sleep(s64 timeout) {
diff --git a/src/core/hle/kernel/k_thread.h b/src/core/hle/kernel/k_thread.h
index 28cd7ecb09..416a861a94 100644
--- a/src/core/hle/kernel/k_thread.h
+++ b/src/core/hle/kernel/k_thread.h
@@ -110,6 +110,7 @@ void SetCurrentThread(KernelCore& kernel, KThread* thread);
[[nodiscard]] KThread* GetCurrentThreadPointer(KernelCore& kernel);
[[nodiscard]] KThread& GetCurrentThread(KernelCore& kernel);
[[nodiscard]] s32 GetCurrentCoreId(KernelCore& kernel);
+size_t CaptureBacktrace(void** buffer, size_t max);
class KThread final : public KAutoObjectWithSlabHeapAndContainer<KThread, KWorkerTask>,
public boost::intrusive::list_base_hook<> {
@@ -413,6 +414,9 @@ public:
[[nodiscard]] static Result InitializeDummyThread(KThread* thread);
+ [[nodiscard]] static Result InitializeMainThread(Core::System& system, KThread* thread,
+ s32 virt_core);
+
[[nodiscard]] static Result InitializeIdleThread(Core::System& system, KThread* thread,
s32 virt_core);
@@ -435,6 +439,7 @@ public:
bool is_pinned;
s32 disable_count;
KThread* cur_thread;
+ std::atomic<bool> m_lock;
};
[[nodiscard]] StackParameters& GetStackParameters() {
diff --git a/src/core/hle/kernel/kernel.cpp b/src/core/hle/kernel/kernel.cpp
index 7307cf262b..10e1f47f65 100644
--- a/src/core/hle/kernel/kernel.cpp
+++ b/src/core/hle/kernel/kernel.cpp
@@ -75,7 +75,6 @@ struct KernelCore::Impl {
InitializeSystemResourceLimit(kernel, system.CoreTiming());
InitializeMemoryLayout();
Init::InitializeKPageBufferSlabHeap(system);
- InitializeSchedulers();
InitializeShutdownThreads();
InitializePreemption(kernel);
@@ -148,7 +147,6 @@ struct KernelCore::Impl {
shutdown_threads[core_id] = nullptr;
}
- schedulers[core_id]->Finalize();
schedulers[core_id].reset();
}
@@ -195,17 +193,11 @@ struct KernelCore::Impl {
exclusive_monitor =
Core::MakeExclusiveMonitor(system.Memory(), Core::Hardware::NUM_CPU_CORES);
for (u32 i = 0; i < Core::Hardware::NUM_CPU_CORES; i++) {
- schedulers[i] = std::make_unique<Kernel::KScheduler>(system, i);
+ schedulers[i] = std::make_unique<Kernel::KScheduler>(system.Kernel());
cores.emplace_back(i, system, *schedulers[i], interrupts);
}
}
- void InitializeSchedulers() {
- for (u32 i = 0; i < Core::Hardware::NUM_CPU_CORES; i++) {
- cores[i].Scheduler().Initialize();
- }
- }
-
// Creates the default system resource limit
void InitializeSystemResourceLimit(KernelCore& kernel,
const Core::Timing::CoreTiming& core_timing) {
diff --git a/src/core/hle/kernel/svc.cpp b/src/core/hle/kernel/svc.cpp
index 8655506b07..27e5a805db 100644
--- a/src/core/hle/kernel/svc.cpp
+++ b/src/core/hle/kernel/svc.cpp
@@ -887,7 +887,7 @@ static Result GetInfo(Core::System& system, u64* result, u64 info_id, Handle han
const auto* const current_thread = GetCurrentThreadPointer(system.Kernel());
const bool same_thread = current_thread == thread.GetPointerUnsafe();
- const u64 prev_ctx_ticks = scheduler.GetLastContextSwitchTicks();
+ const u64 prev_ctx_ticks = scheduler.GetLastContextSwitchTime();
u64 out_ticks = 0;
if (same_thread && info_sub_id == 0xFFFFFFFFFFFFFFFF) {
const u64 thread_ticks = current_thread->GetCpuTime();
@@ -3026,11 +3026,6 @@ void Call(Core::System& system, u32 immediate) {
}
kernel.ExitSVCProfile();
-
- if (!thread->IsCallingSvc()) {
- auto* host_context = thread->GetHostContext().get();
- host_context->Rewind();
- }
}
} // namespace Kernel::Svc