rt_domain: do not allocate/free memory in release queue handling

This fixes a bunch of bugs:
- a memory leak (some heap nodes were never reclaimed)
- a locking rules violation (memory allocation can wake up kswapd,
  which causes a circular locking dependency between the release queue lock
  and the run queue locks

Also, allocating memory in a hot path was never particularly clever.
This patch pre-allocates memory at task-creation time, but does not change
how the release queue works overall.

4 files changed, 33 insertions, 24 deletions

diff --git a/include/litmus/rt_domain.h b/include/litmus/rt_domain.h
index 7356ec7ca4..9323892a87 100644
--- a/include/litmus/rt_domain.h
+++ b/include/litmus/rt_domain.h
@@ -21,6 +21,10 @@ struct release_heap {
         struct list_head                list;
         lt_t                            release_time;
         struct heap                     heap;
+        /* the node used to enqueue this heap in a
+         * release queue
+         */
+        struct heap_node*               hn;
 };
 
 struct release_queue {
diff --git a/include/litmus/rt_param.h b/include/litmus/rt_param.h
index 155543ea93..a0d5de2aa5 100644
--- a/include/litmus/rt_param.h
+++ b/include/litmus/rt_param.h
@@ -40,6 +40,7 @@ struct rt_task {
 
 struct _rt_domain;
 struct heap_node;
+struct release_heap;
 
 struct rt_job {
         /* Time instant the the job was or will be released.  */
@@ -160,6 +161,7 @@ struct rt_param {
          *          implementation).
          */
         struct heap_node*       heap_node;
+        struct release_heap*    rel_heap;
 
         /* Used by rt_domain to queue task in release list.
          */
diff --git a/litmus/litmus.c b/litmus/litmus.c
index 8591e7c022..145da8e629 100644
--- a/litmus/litmus.c
+++ b/litmus/litmus.c
@@ -568,11 +568,22 @@ long litmus_admit_task(struct task_struct* tsk)
 
         /* allocate heap node for this task */
         tsk_rt(tsk)->heap_node    = heap_node_alloc(GFP_ATOMIC);
-        if (!tsk_rt(tsk)->heap_node) {
+        tsk_rt(tsk)->rel_heap = release_heap_alloc(GFP_ATOMIC);
+        tsk_rt(tsk)->rel_heap->hn = heap_node_alloc(GFP_ATOMIC);
+        if (!tsk_rt(tsk)->heap_node ||
+            !tsk_rt(tsk)->rel_heap ||
+            !tsk_rt(tsk)->rel_heap->hn) {
                 printk(KERN_WARNING "litmus: no more heap node memory!?\n");
                 retval = -ENOMEM;
-        } else
+                heap_node_free(tsk_rt(tsk)->heap_node);
+                if (tsk_rt(tsk)->rel_heap) {
+                        heap_node_free(tsk_rt(tsk)->rel_heap->hn);
+                        release_heap_free(tsk_rt(tsk)->rel_heap);
+                }
+        } else {
                 heap_node_init(&tsk_rt(tsk)->heap_node, tsk);
+                heap_node_init(&tsk_rt(tsk)->rel_heap->hn, tsk_rt(tsk)->rel_heap);
+        }
 
         if (!retval)
                 retval = litmus->admit_task(tsk);
@@ -595,6 +606,8 @@ void litmus_exit_task(struct task_struct* tsk)
                 litmus->task_exit(tsk);
                 BUG_ON(heap_node_in_heap(tsk_rt(tsk)->heap_node));
                 heap_node_free(tsk_rt(tsk)->heap_node);
+                heap_node_free(tsk_rt(tsk)->rel_heap->hn);
+                release_heap_free(tsk_rt(tsk)->rel_heap);
                 atomic_dec(&rt_task_count);
                 reinit_litmus_state(tsk, 1);
         }
diff --git a/litmus/rt_domain.c b/litmus/rt_domain.c
index a05ed9ca94..69e0019870 100644
--- a/litmus/rt_domain.c
+++ b/litmus/rt_domain.c
@@ -52,13 +52,13 @@ int heap_earlier_release(struct heap_node *_a, struct heap_node *_b)
  * Will return heap for given time. If no such heap exists prior to the invocation
  * it will be created.
  */
-static struct release_heap* get_release_heap(rt_domain_t *rt, lt_t release_time)
+static struct release_heap* get_release_heap(rt_domain_t *rt, struct task_struct* t)
 {
         struct list_head* pos;
         struct release_heap* heap = NULL;
         struct release_heap* rh;
+        lt_t release_time = get_release(t);
         unsigned int slot = time2slot(release_time);
-        int inserted;
 
         /* initialize pos for the case that the list is empty */
         pos = rt->release_queue.slot[slot].next;
@@ -78,23 +78,13 @@ static struct release_heap* get_release_heap(rt_domain_t *rt, lt_t release_time)
                 }
         }
         if (!heap) {
-                /* must create new node */
-                /* FIXME: use a kmemcache_t */
-                rh = kmalloc(sizeof(*rh), GFP_ATOMIC);
-                if (unlikely(!rh))
-                        /* Should be handled somehow.
-                         * For now, let's just hope there is
-                         * sufficient memory.
-                         */
-                        panic("rt_domain: no more memory?");
+                /* pre-allocated release heap */
+                rh = tsk_rt(t)->rel_heap;
                 rh->release_time = release_time;
                 heap_init(&rh->heap);
                 list_add(&rh->list, pos->prev);
-                inserted = heap_add(heap_earlier_release,
-                                    &rt->release_queue.rel_heap, rh,
-                                    GFP_ATOMIC);
-                if (unlikely(!inserted))
-                        panic("rt_domain: no more heap memory?");
+                heap_insert(heap_earlier_release,
+                            &rt->release_queue.rel_heap, rh->hn);
                 heap = rh;
         }
         return heap;
@@ -125,19 +115,19 @@ static enum hrtimer_restart on_release_timer(struct hrtimer *timer)
                         rh = list_entry(pos, struct release_heap, list);
                         heap_union(rt->order, &tasks, &rh->heap);
                         list_del(pos);
-                        kfree(rh);
                 }
 
                 /* call release callback */
-                rt->release_jobs(rt, &tasks);
+                if (!heap_empty(&tasks))
+                        rt->release_jobs(rt, &tasks);
 
 
                 spin_lock_irqsave(&rt->release_lock, flags);
                 while ((pending = next_release(rt, &release))) {
                         if (lt_before(release, litmus_clock())) {
                                 /* pick for release */
-                                rh = heap_take_del(heap_earlier_release,
-                                                   &rt->release_queue.rel_heap);
+                                rh = heap_take(heap_earlier_release,
+                                               &rt->release_queue.rel_heap)->value;
                                 list_move(&rh->list, &list);
                         } else
                                 break;
@@ -187,8 +177,8 @@ static void arm_release_timer(unsigned long _rt)
                 t = list_entry(pos, struct task_struct, rt_param.list);
                 sched_trace_task_release(t);
                 list_del(pos);
-                rh = get_release_heap(rt, get_release(t));
-                heap_add(rt->order, &rh->heap, t, GFP_ATOMIC);
+                rh = get_release_heap(rt, t);
+                heap_insert(rt->order, &rh->heap, tsk_rt(t)->heap_node);
         }
 
         next_release(rt, &release);