Integrate preemption state machine with Linux scheduler

Track when a processor is going to schedule "soon".

5 files changed, 26 insertions, 1 deletions

diff --git a/arch/arm/kernel/smp.c b/arch/arm/kernel/smp.c
index 7dd14e8395e6..b89255cf8f24 100644
--- a/arch/arm/kernel/smp.c
+++ b/arch/arm/kernel/smp.c
@@ -51,6 +51,8 @@
 #define CREATE_TRACE_POINTS
 #include <trace/events/ipi.h>
 
+#include <litmus/preempt.h>
+
 /*
  * as from 2.5, kernels no longer have an init_tasks structure
  * so we need some other way of telling a new secondary core
@@ -614,6 +616,8 @@ void handle_IPI(int ipinr, struct pt_regs *regs)
 #endif
 
         case IPI_RESCHEDULE:
+                /* LITMUS^RT: take action based on scheduler state */
+                sched_state_ipi();
                 scheduler_ipi();
                 break;
 
diff --git a/arch/x86/kernel/smp.c b/arch/x86/kernel/smp.c
index c00cb64bc0a1..ec28c740cc55 100644
--- a/arch/x86/kernel/smp.c
+++ b/arch/x86/kernel/smp.c
@@ -24,6 +24,8 @@
 #include <linux/cpu.h>
 #include <linux/gfp.h>
 
+#include <litmus/preempt.h>
+
 #include <asm/mtrr.h>
 #include <asm/tlbflush.h>
 #include <asm/mmu_context.h>
@@ -218,7 +220,7 @@ static void native_stop_other_cpus(int wait)
                 while (num_online_cpus() > 1 && (wait || timeout--))
                         udelay(1);
         }
-        
+
         /* if the REBOOT_VECTOR didn't work, try with the NMI */
         if ((num_online_cpus() > 1) && (!smp_no_nmi_ipi))  {
                 if (register_nmi_handler(NMI_LOCAL, smp_stop_nmi_callback,
@@ -268,6 +270,11 @@ __visible void smp_reschedule_interrupt(struct pt_regs *regs)
         /*
          * KVM uses this interrupt to force a cpu out of guest mode
          */
+
+        /* LITMUS^RT: this IPI might need to trigger the sched state machine.
+         * Starting from 3.0 schedule_ipi() actually does something.  This may
+         * increase IPI latencies compared with previous versions. */
+        sched_state_ipi();
 }
 
 __visible void smp_trace_reschedule_interrupt(struct pt_regs *regs)
diff --git a/arch/x86/xen/smp.c b/arch/x86/xen/smp.c
index 137afbbd0590..ef65354b3faa 100644
--- a/arch/x86/xen/smp.c
+++ b/arch/x86/xen/smp.c
@@ -66,6 +66,8 @@ static irqreturn_t xen_reschedule_interrupt(int irq, void *dev_id)
         inc_irq_stat(irq_resched_count);
         scheduler_ipi();
 
+        sched_state_ipi();
+
         return IRQ_HANDLED;
 }
 
diff --git a/include/linux/sched.h b/include/linux/sched.h
index 78566173465b..52d2f5c2c926 100644
--- a/include/linux/sched.h
+++ b/include/linux/sched.h
@@ -127,6 +127,7 @@ struct sched_attr {
 };
 
 #include <litmus/rt_param.h>
+#include <litmus/preempt.h>
 
 struct futex_pi_state;
 struct robust_list_head;
@@ -3274,6 +3275,7 @@ static inline int test_tsk_thread_flag(struct task_struct *tsk, int flag)
 static inline void set_tsk_need_resched(struct task_struct *tsk)
 {
         set_tsk_thread_flag(tsk,TIF_NEED_RESCHED);
+        sched_state_will_schedule(tsk);
 }
 
 static inline void clear_tsk_need_resched(struct task_struct *tsk)
diff --git a/kernel/sched/core.c b/kernel/sched/core.c
index 32bffe174d75..f2f3a3000a1c 100644
--- a/kernel/sched/core.c
+++ b/kernel/sched/core.c
@@ -2857,6 +2857,10 @@ asmlinkage __visible void schedule_tail(struct task_struct *prev)
          */
 
         rq = finish_task_switch(prev);
+
+        if (unlikely(sched_state_validate_switch()))
+                litmus_reschedule_local();
+
         balance_callback(rq);
         preempt_enable();
 
@@ -3349,6 +3353,8 @@ static void __sched notrace __schedule(bool preempt)
 
         TS_SCHED_START;
 
+        sched_state_entered_schedule();
+
         cpu = smp_processor_id();
         rq = cpu_rq(cpu);
         prev = rq->curr;
@@ -3421,6 +3427,10 @@ static void __sched notrace __schedule(bool preempt)
         }
 
         TS_SCHED2_START(prev);
+
+        if (unlikely(sched_state_validate_switch()))
+                litmus_reschedule_local();
+
         balance_callback(rq);
         TS_SCHED2_END(prev);
 }