Integrate preemption state machine with Linux scheduler

Track when a processor is going to schedule "soon".

5 files changed, 22 insertions, 0 deletions

diff --git a/arch/arm/kernel/smp.c b/arch/arm/kernel/smp.c
index cca5b8758185..29a742bbf3ab 100644
--- a/arch/arm/kernel/smp.c
+++ b/arch/arm/kernel/smp.c
@@ -50,6 +50,8 @@
 #define CREATE_TRACE_POINTS
 #include <trace/events/ipi.h>
 
+#include <litmus/preempt.h>
+
 /*
  * as from 2.5, kernels no longer have an init_tasks structure
  * so we need some other way of telling a new secondary core
@@ -593,6 +595,8 @@ void handle_IPI(int ipinr, struct pt_regs *regs)
 #endif
 
         case IPI_RESCHEDULE:
+                /* LITMUS^RT: take action based on scheduler state */
+                sched_state_ipi();
                 scheduler_ipi();
                 break;
 
diff --git a/arch/x86/kernel/smp.c b/arch/x86/kernel/smp.c
index 7ce9fb210d55..431c3d29a667 100644
--- a/arch/x86/kernel/smp.c
+++ b/arch/x86/kernel/smp.c
@@ -24,6 +24,7 @@
 #include <linux/cpu.h>
 #include <linux/gfp.h>
 
+#include <litmus/preempt.h>
 #include <litmus/debug_trace.h>
 
 #include <asm/mtrr.h>
@@ -275,6 +276,11 @@ __visible void smp_reschedule_interrupt(struct pt_regs *regs)
         /*
          * KVM uses this interrupt to force a cpu out of guest mode
          */
+
+        /* LITMUS^RT: this IPI might need to trigger the sched state machine.
+         * Starting from 3.0 schedule_ipi() actually does something.  This may
+         * increase IPI latencies compared with previous versions. */
+        sched_state_ipi();
 }
 
 static inline void smp_entering_irq(void)
diff --git a/arch/x86/xen/smp.c b/arch/x86/xen/smp.c
index 86484384492e..517295f6b034 100644
--- a/arch/x86/xen/smp.c
+++ b/arch/x86/xen/smp.c
@@ -63,6 +63,8 @@ static irqreturn_t xen_reschedule_interrupt(int irq, void *dev_id)
         inc_irq_stat(irq_resched_count);
         scheduler_ipi();
 
+        sched_state_ipi();
+
         return IRQ_HANDLED;
 }
 
diff --git a/include/linux/sched.h b/include/linux/sched.h
index 3e6e669db92b..ade13badfacc 100644
--- a/include/linux/sched.h
+++ b/include/linux/sched.h
@@ -126,6 +126,7 @@ struct sched_attr {
 };
 
 #include <litmus/rt_param.h>
+#include <litmus/preempt.h>
 
 struct futex_pi_state;
 struct robust_list_head;
@@ -2784,6 +2785,7 @@ static inline int test_tsk_thread_flag(struct task_struct *tsk, int flag)
 static inline void set_tsk_need_resched(struct task_struct *tsk)
 {
         set_tsk_thread_flag(tsk,TIF_NEED_RESCHED);
+        sched_state_will_schedule(tsk);
 }
 
 static inline void clear_tsk_need_resched(struct task_struct *tsk)
diff --git a/kernel/sched/core.c b/kernel/sched/core.c
index 50f9f4c45ce9..377273d6cecc 100644
--- a/kernel/sched/core.c
+++ b/kernel/sched/core.c
@@ -2294,6 +2294,10 @@ asmlinkage __visible void schedule_tail(struct task_struct *prev)
         preempt_disable();
         rq = finish_task_switch(prev);
         post_schedule(rq);
+
+        if (sched_state_validate_switch())
+                litmus_reschedule_local();
+
         preempt_enable();
 
         if (current->set_child_tid)
@@ -2745,6 +2749,7 @@ static void __sched __schedule(void)
         int cpu;
 
         preempt_disable();
+        sched_state_entered_schedule();
         cpu = smp_processor_id();
         rq = cpu_rq(cpu);
         rcu_note_context_switch();
@@ -2819,6 +2824,9 @@ static void __sched __schedule(void)
 
         post_schedule(rq);
 
+        if (unlikely(sched_state_validate_switch()))
+                litmus_reschedule_local();
+
         sched_preempt_enable_no_resched();
 
         TS_SCHED2_END(prev);