Per job blocking

5 files changed, 99 insertions, 145 deletions

diff --git a/include/litmus/rt_param.h b/include/litmus/rt_param.h
index 209a89d7b459..91fa107cafb5 100644
--- a/include/litmus/rt_param.h
+++ b/include/litmus/rt_param.h
@@ -71,6 +71,8 @@ struct control_page {
 
         /* locking overhead tracing: time stamp prior to system call */
         uint64_t ts_syscall_start; /* Feather-Trace cycles */
+
+        int colors_updated:8;
 };
 
 #ifndef __KERNEL__
diff --git a/include/litmus/rt_server.h b/include/litmus/rt_server.h
index 0f3147707a3b..0e2feb6c6b0e 100644
--- a/include/litmus/rt_server.h
+++ b/include/litmus/rt_server.h
@@ -9,8 +9,6 @@ struct rt_server;
 
 typedef int (*need_preempt_t)(rt_domain_t *rt, struct task_struct *t);
 typedef void (*server_update_t)(struct rt_server *srv);
-typedef void (*server_requeue_t)(struct rt_server *srv, struct task_struct *t);
-typedef struct task_struct* (*server_take_t)(struct rt_server *srv);
 
 struct rt_server {
         int sid;
@@ -23,17 +21,11 @@ struct rt_server {
         need_preempt_t          need_preempt;
         /* System state has changed, so should server */
         server_update_t         update;
-        /* Requeue task in domain */
-        server_requeue_t        requeue;
-        /* Take next task from domain */
-        server_take_t           take;
 };
 
 void init_rt_server(struct rt_server *server,
                     int sid, int cpu, rt_domain_t *domain,
                     need_preempt_t need_preempt,
-                    server_requeue_t requeue,
-                    server_update_t update,
-                    server_take_t take);
+                    server_update_t update);
 
 #endif
diff --git a/litmus/color.c b/litmus/color.c
index 8a9f68544e8f..ecc191137137 100644
--- a/litmus/color.c
+++ b/litmus/color.c
@@ -15,7 +15,7 @@
 #include <litmus/color.h>
 #include <litmus/litmus.h> /* for in_list(...) */
 
-#define PAGES_PER_COLOR 300
+#define PAGES_PER_COLOR 3072
 
 /*
  * This is used only to "trick" lockdep into permitting dynamically allocated
diff --git a/litmus/rt_server.c b/litmus/rt_server.c
index 818588a3d317..74d7c7b0f81a 100644
--- a/litmus/rt_server.c
+++ b/litmus/rt_server.c
@@ -1,11 +1,5 @@
 #include <litmus/rt_server.h>
 
-
-static struct task_struct* default_server_take(struct rt_server *srv)
-{
-        return __take_ready(srv->domain);
-}
-
 static void default_server_update(struct rt_server *srv)
 {
 }
@@ -13,18 +7,13 @@ static void default_server_update(struct rt_server *srv)
 void init_rt_server(struct rt_server *server,
                     int sid, int cpu, rt_domain_t *domain,
                     need_preempt_t need_preempt,
-                    server_requeue_t requeue,
-                    server_update_t update,
-                    server_take_t take)
+                    server_update_t update)
 {
-        if (!need_preempt || !requeue)
+        if (!need_preempt)
                 BUG_ON(1);
 
         server->need_preempt = need_preempt;
-        server->requeue = requeue;
-
         server->update = (update) ? update : default_server_update;
-        server->take = (take) ? take : default_server_take;
 
         server->sid = sid;
         server->cpu = cpu;
diff --git a/litmus/sched_color.c b/litmus/sched_color.c
index 8554fde49c0b..f095b302ddd6 100644
--- a/litmus/sched_color.c
+++ b/litmus/sched_color.c
@@ -55,15 +55,17 @@ static raw_spinlock_t dgl_lock;
 #define remote_entry(cpu)      (&per_cpu(color_cpus, cpu))
 #define task_entry(task)       remote_entry(get_partition(task))
 #define task_fserver(task)     (&task_entry(task)->fifo_server.server)
-#define entry_lock(entry)      (&entry->rm_domain.ready_lock)
+#define entry_lock(entry)      (&(entry)->rm_domain.ready_lock)
 
 
 #define task_dom(entry, task)  (is_be(task) ? &fifo_domain : &entry->rm_domain)
 #define task_lock(entry, task) (is_be(task) ? &fifo_lock   : entry_lock(entry))
-#define is_fifo_server(s)      (s->sid > num_online_cpus())
+#define is_fifo_server(s)      ((s)->sid > num_online_cpus())
+#define lock_if(lock, cond)    do { if (cond) raw_spin_lock(lock);} while(0)
+#define unlock_if(lock, cond)  do { if (cond) raw_spin_unlock(lock);} while(0)
 
 #ifdef CONFIG_NP_SECTION
-#define has_resources(t, c)    (tsk_rt(t)->req == group_lock.acquired[c])
+#define has_resources(t, c)   (tsk_rt(t)->req == group_lock.acquired[c])
 #else
 #define has_resources(t, c)   (1)
 #endif
@@ -212,6 +214,65 @@ static void link(struct rt_server *server, struct task_struct *task)
 }
 
 /*
+ * Triggers preemption on first FIFO server which is running NULL.
+ */
+static void check_for_fifo_preempt(void)
+{
+        int ret = 0, cpu;
+        struct cpu_entry *entry;
+        struct rt_server *cpu_server, *fifo_server;
+
+        TRACE("Checking for FIFO preempt\n");
+
+        for_each_online_cpu(cpu) {
+                entry = remote_entry(cpu);
+                cpu_server = &entry->server;
+                fifo_server = &entry->fifo_server.server;
+
+                raw_spin_lock(entry_lock(entry));
+                raw_spin_lock(&fifo_lock);
+
+                if (cpu_server->linked && is_server(cpu_server->linked) &&
+                    !fifo_server->linked) {
+                        litmus_reschedule(cpu);
+                        ret = 1;
+                }
+
+                raw_spin_unlock(&fifo_lock);
+                raw_spin_unlock(entry_lock(entry));
+
+                if (ret)
+                        break;
+        }
+}
+
+/*
+ * Rejoin a task into the system.
+ */
+static void job_arrival(struct task_struct *t)
+{
+        int i;
+        rt_domain_t *dom = task_dom(task_entry(t), t);
+        struct dgl_group_req *gr = tsk_rt(t)->req;
+        struct control_page  *cp = tsk_rt(t)->ctrl_page;
+        struct color_ctrl_page *ccp = tsk_rt(t)->color_ctrl_page;
+
+        /* Fill request */
+        if (cp && ccp && cp->colors_updated) {
+                cp->colors_updated = 0;
+                dgl_group_req_init(&group_lock, gr);
+                for (i = 0; ccp->pages[i]; ++i)
+                        set_req(&group_lock, gr, ccp->colors[i], ccp->pages[i]);
+        } else {
+                TRACE("Oh noz: %p %p %d\n", cp, ccp, ((cp) ? cp->colors_updated : -1));
+        }
+
+        lock_if(&fifo_lock, is_be(t));
+        requeue(dom, t);
+        unlock_if(&fifo_lock, is_be(t));
+}
+
+/*
  * Complete job for task linked to @server.
  */
 static void job_completion(struct rt_server *server)
@@ -243,7 +304,7 @@ static void job_completion(struct rt_server *server)
                 sched_trace_task_release(t);
 
         if (is_running(t))
-                server->requeue(server, t);
+                job_arrival(t);
 }
 
 /*
@@ -275,7 +336,6 @@ static struct task_struct* schedule_server(struct rt_server *server)
 {
         struct task_struct *next;
         struct rt_server *lserver;
-        int is_fifo = is_fifo_server(server);
 
         TRACE("Scheduling server %d\n", server->sid);
 
@@ -288,8 +348,7 @@ static struct task_struct* schedule_server(struct rt_server *server)
         }
 
         next = server->linked;
-        if (is_fifo)
-                raw_spin_lock(&fifo_lock);
+        lock_if(&fifo_lock, is_fifo_server(server));
         if ((!next || !is_np(next)) &&
              server->need_preempt(server->domain, next)) {
                 if (next) {
@@ -300,49 +359,12 @@ static struct task_struct* schedule_server(struct rt_server *server)
                 next = __take_ready(server->domain);
                 link(server, next);
         }
-        if (is_fifo)
-                raw_spin_unlock(&fifo_lock);
+        unlock_if(&fifo_lock, is_fifo_server(server));
 
         return next;
 }
 
 /*
- * Dumb requeue for PRM (CPU) servers.
- */
-static void rm_requeue(struct rt_server *server, struct task_struct *t)
-{
-        BUG_ON(is_be(t));
-        requeue(server->domain, t);
-}
-
-/*
- * Locking requeue for FIFO servers.
- */
-static void fifo_requeue(struct rt_server *server, struct task_struct *t)
-{
-        BUG_ON(!is_be(t));
-        raw_spin_lock(&fifo_lock);
-        requeue(server->domain, t);
-        raw_spin_unlock(&fifo_lock);
-}
-
-
-/*
- * Locking take for FIFO servers.
- * TODO: no longer necessary.
- */
-static struct task_struct* fifo_take(struct rt_server *server)
-{
-        struct task_struct *ret;
-
-        raw_spin_lock(&fifo_lock);
-        ret = __take_ready(server->domain);
-        raw_spin_unlock(&fifo_lock);
-
-        return ret;
-}
-
-/*
  * Update server state, including picking next running task and incrementing
  * server execution time.
  */
@@ -403,39 +425,6 @@ static void color_rm_release(rt_domain_t *rm, struct bheap *tasks)
         raw_spin_unlock_irqrestore(entry_lock(entry), flags);
 }
 
-/*
- * Triggers preemption on first FIFO server which is running NULL.
- */
-static void check_for_fifo_preempt(void)
-{
-        int ret = 0, cpu;
-        struct cpu_entry *entry;
-        struct rt_server *cpu_server, *fifo_server;
-
-        TRACE("Checking for FIFO preempt\n");
-
-        for_each_online_cpu(cpu) {
-                entry = remote_entry(cpu);
-                cpu_server = &entry->server;
-                fifo_server = &entry->fifo_server.server;
-
-                raw_spin_lock(entry_lock(entry));
-                raw_spin_lock(&fifo_lock);
-
-                if (cpu_server->linked && is_server(cpu_server->linked) &&
-                    !fifo_server->linked) {
-                        litmus_reschedule(cpu);
-                        ret = 1;
-                }
-
-                raw_spin_unlock(&fifo_lock);
-                raw_spin_unlock(entry_lock(entry));
-
-                if (ret)
-                        break;
-        }
-}
-
 static void color_fifo_release(rt_domain_t *dom, struct bheap *tasks)
 {
         unsigned long flags;
@@ -471,9 +460,7 @@ static struct task_struct* color_schedule(struct task_struct *prev)
 
         if (entry->scheduled && cpu_empty(entry, plink) && is_running(prev)) {
                 TRACE_TASK(prev, "Snuck in on new!\n");
-                raw_spin_lock(&fifo_lock);
-                requeue(task_dom(entry, prev), prev);
-                raw_spin_unlock(&fifo_lock);
+                job_arrival(entry->scheduled);
         }
 
         /* Pick next top-level task */
@@ -527,51 +514,38 @@ static struct task_struct* color_schedule(struct task_struct *prev)
 static void color_task_new(struct task_struct *t, int on_rq, int running)
 {
         unsigned long flags;
-        int i;
-        raw_spinlock_t *lock;
         struct cpu_entry *entry;
         struct dgl_group_req *req;
-        color_t *colors, *pages;
 
         TRACE_TASK(t, "New colored task\n");
-        local_irq_save(flags);
-
         entry = (is_be(t)) ? local_entry : task_entry(t);
-        lock  = task_lock(entry, t);
 
-        release_at(t, litmus_clock());
+        raw_spin_lock_irqsave(entry_lock(entry), flags);
 
         req = kmalloc(sizeof(*req), GFP_ATOMIC);
-        dgl_group_req_init(&group_lock, req);
         tsk_rt(t)->req = req;
-
-        tsk_rt(t)->max_exec_time = 0;
         tsk_rt(t)->tot_exec_time = 0;
+        tsk_rt(t)->max_exec_time = 0;
+        tsk_rt(t)->ctrl_page->colors_updated = 1;
 
-        /* Fill request */
-        if (tsk_rt(t)->color_ctrl_page) {
-                colors = tsk_rt(t)->color_ctrl_page->colors;
-                pages  = tsk_rt(t)->color_ctrl_page->pages;
-                for (i = 0; pages[i]; i++)
-                        set_req(&group_lock, req, colors[i], pages[i]);
-        }
+        release_at(t, litmus_clock());
 
-        /* Join system */
-        raw_spin_lock(lock);
         if (running) {
+                /* No need to lock with irqs disabled */
                 TRACE_TASK(t, "Already scheduled on %d\n", entry->server.cpu);
                 BUG_ON(entry->scheduled);
                 entry->scheduled = t;
                 tsk_rt(t)->scheduled_on = entry->server.cpu;
-        } else
-                requeue(task_dom(entry, t), t);
-        raw_spin_unlock(lock);
+        } else {
+                job_arrival(t);
+        }
+
+        raw_spin_unlock(entry_lock(entry));
 
-        /* Trigger necessary preemptions */
         if (is_be(t))
                 check_for_fifo_preempt();
         else
-                litmus_reschedule(entry->server.cpu);
+                litmus_reschedule_local();
 
         local_irq_restore(flags);
 }
@@ -579,13 +553,13 @@ static void color_task_new(struct task_struct *t, int on_rq, int running)
 static void color_task_wake_up(struct task_struct *task)
 {
         unsigned long flags;
-        struct cpu_entry* entry = task_entry(task);
-        raw_spinlock_t *lock = task_lock(entry, task);
+        struct cpu_entry* entry = local_entry;
+        int sched;
         lt_t now = litmus_clock();
 
         TRACE_TASK(task, "Wake up at %llu\n", now);
 
-        local_irq_save(flags);
+        raw_spin_lock_irqsave(entry_lock(entry), flags);
 
         /* Abuse sporadic model */
         if (is_tardy(task, now)) {
@@ -593,21 +567,20 @@ static void color_task_wake_up(struct task_struct *task)
                 sched_trace_task_release(task);
         }
 
-        /* Re-enter system */
-        if (entry->scheduled != task) {
-                raw_spin_lock(lock);
-                requeue(task_dom(entry, task), task);
-                raw_spin_unlock(lock);
-        } else {
+        sched = (entry->scheduled == task);
+
+        if (!sched)
+                job_arrival(task);
+        else
                 TRACE_TASK(task, "Is already scheduled on %d!\n",
                            entry->scheduled);
-        }
 
-        /* Trigger preemptions */
+        raw_spin_unlock(entry_lock(entry));
         if (is_be(task))
                 check_for_fifo_preempt();
         else
-                litmus_reschedule(entry->server.cpu);
+                litmus_reschedule_local();
+
 
         local_irq_restore(flags);
 }
@@ -619,7 +592,7 @@ static void color_task_block(struct task_struct *t)
         BUG_ON(is_queued(t));
 }
 
-static void color_task_exit(struct task_struct * t)
+static void color_task_exit(struct task_struct *t)
 {
         unsigned long flags;
         struct cpu_entry *entry = task_entry(t);
@@ -812,8 +785,7 @@ static int __init init_color(void)
                                cpu + num_online_cpus() + 1,
                                cpu,
                                &fifo_domain,
-                               fifo_preemption_needed,
-                               fifo_requeue, fifo_update, fifo_take);
+                               fifo_preemption_needed, fifo_update);
 
 
                 /* Create task struct for FIFO server */
@@ -833,8 +805,7 @@ static int __init init_color(void)
                 /* Create CPU server */
                 cpu_server = &entry->server;
                 init_rt_server(cpu_server, cpu + 1, cpu,
-                               &entry->rm_domain, rm_preemption_needed,
-                               rm_requeue, NULL, NULL);
+                               &entry->rm_domain, rm_preemption_needed, NULL);
                 cpu_server->running = 1;
 
                 init_enforcement_timer(&fifo_server->timer);