PGM: Boost priority of producers, not consumers.

This patch boosts the priority of PGM producers while
they are sending tokens instead of boosting the priority
of consumers while they are waiting for tokens. This improves
schedulability analysis.

4 files changed, 71 insertions, 38 deletions

diff --git a/include/litmus/pgm.h b/include/litmus/pgm.h
index 1e87e170e8c3..5682a76b3acb 100644
--- a/include/litmus/pgm.h
+++ b/include/litmus/pgm.h
@@ -4,8 +4,9 @@
 #include <litmus/litmus.h>
 
 #define is_pgm_waiting(t) (tsk_rt(t)->ctrl_page && tsk_rt(t)->ctrl_page->pgm_waiting)
+#define is_pgm_sending(t) (tsk_rt(t)->ctrl_page && tsk_rt(t)->ctrl_page->pgm_sending)
 #define is_pgm_satisfied(t) (tsk_rt(t)->ctrl_page && tsk_rt(t)->ctrl_page->pgm_satisfied)
 
-void setup_pgm_release(struct task_struct* t);
+int setup_pgm_release(struct task_struct* t);
 
 #endif
diff --git a/include/litmus/rt_param.h b/include/litmus/rt_param.h
index bc074c63c7ad..fe4b31320ac8 100644
--- a/include/litmus/rt_param.h
+++ b/include/litmus/rt_param.h
@@ -122,8 +122,9 @@ struct control_page {
                                      * started. */
 
         /* Flags from userspace signifying PGM wait states. */
-        volatile uint32_t       pgm_waiting;   /* waiting for tokens */
-        volatile uint32_t       pgm_satisfied; /* needed tokens acquired */
+        volatile uint32_t       pgm_waiting;    /* waiting for tokens */
+        volatile uint32_t       pgm_sending;    /* sending tokens */
+        volatile uint32_t       pgm_satisfied;  /* done waiting/sending */
 
         /* to be extended */
 };
diff --git a/litmus/pgm.c b/litmus/pgm.c
index 0bc190851718..db3378ff803d 100644
--- a/litmus/pgm.c
+++ b/litmus/pgm.c
@@ -12,18 +12,17 @@
    at 'now'. Adjustment threshold currently set to 200us. */
 #define ADJUSTMENT_THRESH_NS (200*1000LL)
 
-void setup_pgm_release(struct task_struct* t)
+int setup_pgm_release(struct task_struct* t)
 {
+        int shifted_release = 0;
+
         /* approximate time last predecessor gave us tokens */
         lt_t now = litmus_clock();
 
-        TRACE_TASK(t, "is starting a new PGM job: waiting:%d  satisfied:%d\n",
-                tsk_rt(t)->ctrl_page->pgm_waiting, tsk_rt(t)->ctrl_page->pgm_satisfied);
-
-        BUG_ON(!tsk_rt(t)->ctrl_page->pgm_waiting || !tsk_rt(t)->ctrl_page->pgm_satisfied);
+        TRACE_TASK(t, "is starting a new PGM job: waiting:%d\n",
+                tsk_rt(t)->ctrl_page->pgm_waiting);
 
-        tsk_rt(t)->ctrl_page->pgm_waiting = 0;
-        tsk_rt(t)->ctrl_page->pgm_satisfied = 0;
+        BUG_ON(!tsk_rt(t)->ctrl_page->pgm_waiting);
 
         /* Adjust release time if we got the last tokens after release of this job.
            This is possible since PGM jobs are early-released. Don't shift our
@@ -40,7 +39,7 @@ void setup_pgm_release(struct task_struct* t)
 
                         tsk_rt(t)->job_params.release = now;
                         tsk_rt(t)->job_params.deadline = adj_deadline;
-                        tsk_rt(t)->job_params.exec_time = 0; /* reset budget */
+                        shifted_release = 1;
                 }
                 else {
                         TRACE_TASK(t, "adjustment falls below threshold. %lld < %lld\n",
@@ -53,5 +52,10 @@ void setup_pgm_release(struct task_struct* t)
                         now, tsk_rt(t)->job_params.release);
         }
 
+        /* possible that there can be multiple instances of pgm_release logged.
+           analysis tools should filter out all but the last pgm_release for
+           a given job release */
         sched_trace_pgm_release(t);
+
+        return shifted_release;
 }
diff --git a/litmus/sched_cedf.c b/litmus/sched_cedf.c
index b6bed80133f7..8699f6d9d5b6 100644
--- a/litmus/sched_cedf.c
+++ b/litmus/sched_cedf.c
@@ -464,39 +464,62 @@ static struct task_struct* cedf_schedule(struct task_struct * prev)
                            entry->linked->comm, entry->linked->pid);
 
 #ifdef CONFIG_SCHED_PGM
-        if (exists && is_pgm_waiting(entry->scheduled)) {
-                if (!is_priority_boosted(entry->scheduled)) {
-                        TRACE_TASK(entry->scheduled, "is waiting for PGM tokens.\n");
-                        BUG_ON(is_pgm_satisfied(entry->scheduled));
-
-                        /* Boost priority so we'll be scheduled immediately
-                           when needed tokens arrive. */
-                        tsk_rt(entry->scheduled)->priority_boosted = 1;
-                        tsk_rt(entry->scheduled)->boost_start_time = litmus_clock();
-
-                        if (unlikely(!blocks)) {
-                                /* Task has probably blocked on an inbound token socket, but
-                                   if not, re-evaluate scheduling decisions */
-                                unlink(entry->scheduled);
-                                cedf_job_arrival(entry->scheduled);
+        if (exists) {
+                if (is_pgm_sending(entry->scheduled)) {
+                        if (!is_pgm_satisfied(entry->scheduled)) {
+                                if (!is_priority_boosted(entry->scheduled)) {
+                                        TRACE_TASK(entry->scheduled, "is sending PGM tokens and needs boosting.\n");
+                                        BUG_ON(is_pgm_satisfied(entry->scheduled));
+
+                                        /* We are either sending tokens or waiting for tokes.
+                                           If waiting: Boost priority so we'll be scheduled
+                                             immediately when needed tokens arrive.
+                                           If sending: Boost priority so no one (specifically, our
+                                             consumers) will preempt us while signalling the token
+                                             transmission.
+                                         */
+                                        tsk_rt(entry->scheduled)->priority_boosted = 1;
+                                        tsk_rt(entry->scheduled)->boost_start_time = litmus_clock();
+
+                                        if (likely(!blocks)) {
+                                                unlink(entry->scheduled);
+                                                cedf_job_arrival(entry->scheduled);
+                                        }
+                                }
+                        }
+                        else { /* sending is satisfied */
+                                tsk_rt(entry->scheduled)->ctrl_page->pgm_sending = 0;
+                                tsk_rt(entry->scheduled)->ctrl_page->pgm_satisfied = 0;
+
+                                if (is_priority_boosted(entry->scheduled)) {
+                                        TRACE_TASK(entry->scheduled,
+                                                        "is done sending PGM tokens must relinquish boosting.\n");
+                                        /* clear boosting */
+                                        tsk_rt(entry->scheduled)->priority_boosted = 0;
+                                        if(likely(!blocks)) {
+                                                /* recheck priority */
+                                                unlink(entry->scheduled);
+                                                cedf_job_arrival(entry->scheduled);
+                                        }
+                                }
                         }
                 }
-                else if (is_pgm_satisfied(entry->scheduled)) {
-                        TRACE_TASK(entry->scheduled, "is done waiting for PGM tokens.\n");
-                        BUG_ON(!is_priority_boosted(entry->scheduled));
-
-                        /* clear any boosting */
-                        tsk_rt(entry->scheduled)->priority_boosted = 0;
-                        setup_pgm_release(entry->scheduled);
-
-                        if (likely(!blocks)) {
-                                /* Task has probably called sched_yield(), so blocking is
-                                   unlikely. Re-evaluate scheduling decisions because we
-                                   still want to run. */
+#if 0
+                else if(is_pgm_waiting(entry->scheduled)) {
+                        int shifted_release;
+
+                        TRACE_TASK(entry->scheduled, "is waiting for PGM tokens.\n");
+                        /* release the next job if we have the tokens we need */
+                        shifted_release = setup_pgm_release(entry->scheduled);
+
+                        /* setup_pgm_release() can screw with our priority,
+                           so recheck it */
+                        if (shifted_release && likely(!blocks)) {
                                 unlink(entry->scheduled);
                                 cedf_job_arrival(entry->scheduled);
                         }
                 }
+#endif
         }
 #endif
 
@@ -641,6 +664,10 @@ static void cedf_task_wake_up(struct task_struct *task)
                 release_at(task, now);
                 sched_trace_task_release(task);
         }
+        if (is_pgm_waiting(task)) {
+                /* shift out release/deadline, if needed */
+                setup_pgm_release(task);
+        }
         cedf_job_arrival(task);
         raw_spin_unlock_irqrestore(&cluster->cluster_lock, flags);
 }