proto type

7 files changed, 396 insertions, 199 deletions

diff --git a/include/litmus/reservation.h b/include/litmus/reservation.h
index fc7e31918a60..0b9c08d6051e 100644
--- a/include/litmus/reservation.h
+++ b/include/litmus/reservation.h
@@ -201,19 +201,19 @@ struct reservation* sup_find_by_id(struct sup_reservation_environment* sup_env,
         unsigned int id);
         
 /* A global multiprocessor reservation environment. */
-/*
+
 typedef enum {
         EVENT_REPLENISH = 0,
         EVENT_DRAIN,
         EVENT_OTHERS,
 } event_type_t;
-*/
+
 
 struct next_timer_event {
         lt_t next_update;
         int timer_armed_on;
-        //unsigned int id;
-        //event_type_t type;
+        unsigned int id;
+        event_type_t type;
         struct list_head list;
 };
 
@@ -234,7 +234,7 @@ struct gmp_reservation_environment {
         struct list_head next_events;
         
         /* (schedule_now == true) means call gmp_dispatch() now */
-        bool schedule_now;
+        int schedule_now;
         /* set to true if a call to gmp_dispatch() is imminent */
         bool will_schedule;
 };
@@ -242,11 +242,11 @@ struct gmp_reservation_environment {
 void gmp_init(struct gmp_reservation_environment* gmp_env);
 void gmp_add_new_reservation(struct gmp_reservation_environment* gmp_env,
         struct reservation* new_res);
-void gmp_scheduler_update_after(struct gmp_reservation_environment* gmp_env,
-        lt_t timeout);
-bool gmp_update_time(struct gmp_reservation_environment* gmp_env, lt_t now);
+void gmp_add_event_after(struct gmp_reservation_environment* gmp_env,
+        lt_t timeout, unsigned int id, event_type_t type);
+int gmp_update_time(struct gmp_reservation_environment* gmp_env, lt_t now);
 struct task_struct* gmp_dispatch(struct gmp_reservation_environment* gmp_env);
-//struct next_timer_event* gmp_find_event_by_id(struct gmp_reservation_environment* gmp_env, unsigned int id);
+struct next_timer_event* gmp_find_event_by_id(struct gmp_reservation_environment* gmp_env, unsigned int id);
 struct next_timer_event* gmp_find_event_by_time(struct gmp_reservation_environment* gmp_env, lt_t when);
 struct reservation* gmp_find_by_id(struct gmp_reservation_environment* gmp_env,
         unsigned int id);
diff --git a/litmus/bank_proc.c b/litmus/bank_proc.c
index ec04626b43ec..07d572833b30 100644
--- a/litmus/bank_proc.c
+++ b/litmus/bank_proc.c
@@ -88,7 +88,7 @@ void add_page_to_color_list(struct page *page)
 
 static int do_add_pages(void)
 {
-        printk("LITMUS do add pages\n");
+        //printk("LITMUS do add pages\n");
         
         struct page *page, *page_tmp;
         LIST_HEAD(free_later);
@@ -122,6 +122,7 @@ out:
         return ret;
 }
 
+extern int l2_usable_sets;
 
 /*
  * provide pages for replacement 
@@ -131,9 +132,10 @@ out:
  * node = 3 for Level A, B tasks in Cpu 3
  * node = 4 for Level C tasks 
  */
+#if 1
 struct page *new_alloc_page(struct page *page, unsigned long node, int **x)
 {
-        printk("allocate new page node = %d\n", node);  
+        //printk("allocate new page node = %d\n", node);        
 //      return alloc_pages_exact_node(node, GFP_HIGHUSER_MOVABLE, 0);
         struct color_group *cgroup;
         struct page *rPage = NULL;
@@ -155,7 +157,20 @@ struct page *new_alloc_page(struct page *page, unsigned long node, int **x)
                 goto out;
         }
         */
-        
+        switch(node ){
+                case 0:
+                        color = (color % l2_usable_sets);
+                        break;
+                case 1: 
+                case 2: 
+                case 3:
+                case 4:
+                        color = (color% (16-l2_usable_sets)) + l2_usable_sets;
+                        break;
+                default:
+                        goto out;
+        }
+        /*
         switch(node ){
                 case 0:
                 case 1: 
@@ -171,14 +186,15 @@ struct page *new_alloc_page(struct page *page, unsigned long node, int **x)
                 default:
                         goto out;
         }
+        */
 
-
-        printk("allocate new page color = %d\n", color);
+        //printk("allocate new page color = %d\n", color);
+        //TRACE("allocate new page color = %d\n", color);
                 
         cgroup = &color_groups[color];
         spin_lock(&cgroup->lock);
         if (unlikely(!atomic_read(&cgroup->nr_pages))) {
-                TRACE_CUR("No free %lu colored pages.\n", color);
+                //TRACE_CUR("No free %lu colored pages.\n", color);
                 printk(KERN_WARNING "no free %lu colored pages.\n", color);
                 goto out_unlock;
         }
@@ -194,6 +210,7 @@ out:
         do_add_pages();
         return rPage;
 }
+#endif
 
 struct page *new_alloc_page_banknr(struct page *page, unsigned long banknr, int **x)
 {
diff --git a/litmus/cache_proc.c b/litmus/cache_proc.c
index cc818b9f1fc4..7b48d5cf2b98 100644
--- a/litmus/cache_proc.c
+++ b/litmus/cache_proc.c
@@ -49,6 +49,9 @@ struct mutex lockdown_proc;
 
 static int min_usable_ways = 0;
 static int max_usable_ways = 16;
+static int min_usable_sets = 1;
+static int max_usable_sets = 15;
+
 static int zero = 0;
 static int one = 1;
 
@@ -60,6 +63,7 @@ static int one = 1;
                         __cpu * L2X0_LOCKDOWN_STRIDE; __v; })
 
 int l2_usable_ways;
+int l2_usable_sets;
 int lock_all;
 int nr_lockregs;
 
@@ -176,6 +180,7 @@ int l2_usable_ways_handler(struct ctl_table *table, int write, void __user *buff
                 goto out;
                 
         TRACE_CUR("l2_usable_ways : %d\n", l2_usable_ways);
+        printk("l2_usable_ways : %d\n", l2_usable_ways);
         
         if (write) {
                 //for (i = 0; i < nr_lockregs;  i++) {
@@ -190,6 +195,31 @@ out:
         return ret;
 }
 
+int l2_usable_sets_handler(struct ctl_table *table, int write, void __user *buffer,
+                size_t *lenp, loff_t *ppos)
+{
+        int ret = 0;
+        
+        mutex_lock(&lockdown_proc);
+        
+        flush_cache_all();
+        
+        ret = proc_dointvec_minmax(table, write, buffer, lenp, ppos);
+        if (ret)
+                goto out;
+                
+        TRACE_CUR("l2_usable_sets : %d\n", l2_usable_sets);
+        printk("l2_usable_sets : %d\n", l2_usable_sets);
+        
+        if (write) {
+                ;
+        }
+
+out:
+        mutex_unlock(&lockdown_proc);
+        return ret;
+}
+
 static struct ctl_table cache_table[] =
 {
         {
@@ -202,6 +232,15 @@ static struct ctl_table cache_table[] =
                 .extra2         = &max_usable_ways,
         },
         {
+                .procname       = "l2_usable_sets",
+                .mode           = 0666,
+                .proc_handler   = l2_usable_sets_handler,
+                .data           = &l2_usable_sets,
+                .maxlen         = sizeof(l2_usable_sets),
+                .extra1         = &min_usable_sets,
+                .extra2         = &max_usable_sets,
+        },
+        {
                 .procname       = "lock_all",
                 .mode           = 0666,
                 .proc_handler   = lock_all_handler,
@@ -237,6 +276,7 @@ static int __init litmus_sysctl_init(void)
         }
 
         l2_usable_ways = 16;
+        l2_usable_sets = 5;
 
 out:
         return ret;
diff --git a/litmus/jobs.c b/litmus/jobs.c
index e523e29ea8a6..547222c3387a 100644
--- a/litmus/jobs.c
+++ b/litmus/jobs.c
@@ -45,7 +45,6 @@ void release_at(struct task_struct *t, lt_t start)
 {
         BUG_ON(!t);
         setup_release(t, start);
-        TRACE("RELEASE!!\n");
         tsk_rt(t)->completed = 0;
 }
 
diff --git a/litmus/litmus.c b/litmus/litmus.c
index dcb9ed58962c..4ff840dcb309 100644
--- a/litmus/litmus.c
+++ b/litmus/litmus.c
@@ -392,14 +392,16 @@ static struct page *walk_page_table(unsigned long addr)
 extern int isolate_lru_page(struct page *page);
 extern void putback_lru_page(struct page *page);
 
-extern struct page *new_alloc_page(struct page *page, unsigned long node, int **x);
-
 #if 0
 static struct page *new_alloc_page(struct page *page, unsigned long node, int **x)
 {
-        return alloc_pages_exact_node(node, GFP_HIGHUSER_MOVABLE, 0);
+        return alloc_pages_exact_node(0, GFP_HIGHUSER_MOVABLE, 0);
 }
+#else
+extern struct page *new_alloc_page(struct page *page, unsigned long node, int **x);
+
 #endif
+
 asmlinkage long sys_set_page_color(int cpu)
 {
         long ret = 0;
@@ -408,16 +410,18 @@ asmlinkage long sys_set_page_color(int cpu)
         struct page *page_itr = NULL;
         struct vm_area_struct *vma_itr = NULL;
         //struct task_page *entry = NULL;
-        int nr_pages = 0, nr_shared_pages = 0;
+        int nr_pages = 0, nr_shared_pages = 0, nr_failed = 0;
+        unsigned long node;
+        
         LIST_HEAD(pagelist);
         LIST_HEAD(shared_pagelist);
         
         down_read(&current->mm->mmap_sem);
-        printk(KERN_INFO "SYSCALL set_page_color\n");
+        TRACE_TASK(current, "SYSCALL set_page_color\n");
         vma_itr = current->mm->mmap;
         while (vma_itr != NULL) {
                 unsigned int num_pages = 0, i;
-                struct page *new_page = NULL, *old_page = NULL;
+                struct page *old_page = NULL;
                 /*
                 entry = kmalloc(sizeof(struct task_page), GFP_ATOMIC);
                 if (entry == NULL) {
@@ -428,8 +432,8 @@ asmlinkage long sys_set_page_color(int cpu)
                 */
                 num_pages = (vma_itr->vm_end - vma_itr->vm_start) / PAGE_SIZE;
                 // print vma flags
-                printk(KERN_INFO "flags: 0x%lx\n", vma_itr->vm_flags);
-                printk(KERN_INFO "start - end: 0x%lx - 0x%lx (%lu)\n", vma_itr->vm_start, vma_itr->vm_end, (vma_itr->vm_end - vma_itr->vm_start)/PAGE_SIZE);
+                //printk(KERN_INFO "flags: 0x%lx\n", vma_itr->vm_flags);
+                //printk(KERN_INFO "start - end: 0x%lx - 0x%lx (%lu)\n", vma_itr->vm_start, vma_itr->vm_end, (vma_itr->vm_end - vma_itr->vm_start)/PAGE_SIZE);
                 
                 for (i = 0; i < num_pages; i++) {
 /*
@@ -447,19 +451,22 @@ asmlinkage long sys_set_page_color(int cpu)
                                 continue;
 
                         if (PageReserved(old_page)) {
+                                TRACE("Reserved Page!\n");
                                 put_page(old_page);
                                 continue;
                         }
                         
-                        printk(KERN_INFO "addr: %lu, pfn: %lu, _mapcount: %d, _count: %d\n", vma_itr->vm_start + PAGE_SIZE*i, __page_to_pfn(old_page), page_mapcount(old_page), page_count(old_page));
+                        TRACE_TASK(current, "addr: %lu, pfn: %lu, _mapcount: %d, _count: %d\n", vma_itr->vm_start + PAGE_SIZE*i, __page_to_pfn(old_page), page_mapcount(old_page), page_count(old_page));
                         
-                        if (page_mapcount(old_page) == 1) {
+                        if (page_mapcount(old_page) != 0) {
                                 ret = isolate_lru_page(old_page);
-                                //if (pfn_valid(__page_to_pfn(old_page)))
                                 if (!ret) {
                                         list_add_tail(&old_page->lru, &pagelist);
                                         inc_zone_page_state(old_page, NR_ISOLATED_ANON + !PageSwapBacked(old_page));
                                         nr_pages++;
+                                } else {
+                                        TRACE_TASK(current, "isolate_lru_page failed\n");
+                                        nr_failed++;
                                 }
                                 put_page(old_page);
                         }
@@ -468,8 +475,9 @@ asmlinkage long sys_set_page_color(int cpu)
                                 if (!ret) {
                                         list_add_tail(&old_page->lru, &shared_pagelist);
                                         inc_zone_page_state(old_page, NR_ISOLATED_ANON + !PageSwapBacked(old_page));
-                                        nr_shared_pages++;
-                                }                                       
+                                        
+                                }       
+                                nr_shared_pages++;
                                 put_page(old_page);
                         }
                 }
@@ -485,10 +493,15 @@ asmlinkage long sys_set_page_color(int cpu)
 //      }
         
         ret = 0;
+        if (cpu == -1)
+                node = 4;
+        else
+                node = cpu;
+        
         if (!list_empty(&pagelist)) {
-                ret = migrate_pages(&pagelist, new_alloc_page, 4, MIGRATE_ASYNC, MR_SYSCALL);
+                ret = migrate_pages(&pagelist, new_alloc_page, node, MIGRATE_ASYNC, MR_SYSCALL);
+                TRACE_TASK(current, "%ld pages not migrated.\n", ret);
                 if (ret) {
-                        printk(KERN_INFO "%ld pages not migrated.\n", ret);
                         putback_lru_pages(&pagelist);
                 }
         }
@@ -507,7 +520,7 @@ asmlinkage long sys_set_page_color(int cpu)
         up_read(&current->mm->mmap_sem);
 
         list_for_each_entry(page_itr, &shared_pagelist, lru) {
-                printk(KERN_INFO "S Anon=%d, pfn = %lu, _mapcount = %d, _count = %d\n", PageAnon(page_itr), __page_to_pfn(page_itr), page_mapcount(page_itr), page_count(page_itr));
+                TRACE("S Anon=%d, pfn = %lu, _mapcount = %d, _count = %d\n", PageAnon(page_itr), __page_to_pfn(page_itr), page_mapcount(page_itr), page_count(page_itr));
         }
         
 /*      
@@ -517,7 +530,7 @@ asmlinkage long sys_set_page_color(int cpu)
                 kfree(task_page_itr);           
         }
 */      
-        printk(KERN_INFO "nr_pages = %d\n", nr_pages);
+        TRACE_TASK(current, "nr_pages = %d nr_failed = %d\n", nr_pages, nr_failed);
         return ret;
 }
 
@@ -888,6 +901,7 @@ static int __init _init_litmus(void)
 #endif
         
         color_mask = ((cache_info_sets << line_size_log) - 1) ^ (PAGE_SIZE - 1);
+        printk("Page color mask %08x\n", color_mask);
         return 0;
 }
 
diff --git a/litmus/reservation.c b/litmus/reservation.c
index e30892c72f4a..b0b13a9916ef 100644
--- a/litmus/reservation.c
+++ b/litmus/reservation.c
@@ -348,7 +348,7 @@ struct reservation* gmp_find_by_id(struct gmp_reservation_environment* gmp_env,
         return NULL;
 }
 
-/*
+
 struct next_timer_event* gmp_find_event_by_id(struct gmp_reservation_environment* gmp_env,
         unsigned int id)
 {
@@ -361,7 +361,7 @@ struct next_timer_event* gmp_find_event_by_id(struct gmp_reservation_environment
 
         return NULL;
 }
-*/
+
 
 struct next_timer_event* gmp_find_event_by_time(struct gmp_reservation_environment* gmp_env,
         lt_t when)
@@ -415,9 +415,9 @@ static void gmp_scheduler_update_at(
 */
 #define TIMER_RESOLUTION 100000L
 
-static void gmp_scheduler_update_at(
+static void gmp_add_event(
         struct gmp_reservation_environment* gmp_env,
-        lt_t when)
+        lt_t when, unsigned int id, event_type_t type)
 {
         struct next_timer_event *nevent, *queued;
         struct list_head *pos;
@@ -426,11 +426,13 @@ static void gmp_scheduler_update_at(
         //when = div64_u64(when, TIMER_RESOLUTION);
         //when *= TIMER_RESOLUTION;
         
-        nevent = gmp_find_event_by_time(gmp_env, when);
+        nevent = gmp_find_event_by_id(gmp_env, id);
         
         if (!nevent) {
-                nevent = kzalloc(sizeof(*nevent), GFP_KERNEL);
+                nevent = kzalloc(sizeof(*nevent), GFP_ATOMIC);
                 nevent->next_update = when;
+                nevent->id = id;
+                nevent->type = type;
                 nevent->timer_armed_on = NO_CPU;
                 
                 list_for_each(pos, &gmp_env->next_events) {
@@ -448,14 +450,14 @@ static void gmp_scheduler_update_at(
                         TRACE("NEXT_EVENT ADDED at %llu ADDED at HEAD\n", nevent->next_update);
                 }
         } else {
-                ; //TRACE("EVENT FOUND at %llu, NEW EVENT %llu\n", nevent->next_update, when);
+                TRACE("EVENT FOUND type=%d when=%llu, NEW EVENT type=%d when=%llu\n", nevent->type, nevent->next_update, type, when);
         }
 }
 
-void gmp_scheduler_update_after(
-        struct gmp_reservation_environment* gmp_env, lt_t timeout)
+void gmp_add_event_after(
+        struct gmp_reservation_environment* gmp_env, lt_t timeout, unsigned int id, event_type_t type)
 {
-        gmp_scheduler_update_at(gmp_env, gmp_env->env.current_time + timeout);
+        gmp_add_event(gmp_env, gmp_env->env.current_time + timeout, id, type);
 }
 
 static void gmp_queue_depleted(
@@ -468,7 +470,7 @@ static void gmp_queue_depleted(
 
         list_for_each(pos, &gmp_env->depleted_reservations) {
                 queued = list_entry(pos, struct reservation, list);
-                if (queued->next_replenishment > res->next_replenishment) {
+                if (queued && queued->next_replenishment > res->next_replenishment) {
                         list_add(&res->list, pos->prev);
                         found = 1;
                 }
@@ -477,7 +479,7 @@ static void gmp_queue_depleted(
         if (!found)
                 list_add_tail(&res->list, &gmp_env->depleted_reservations);
 
-        gmp_scheduler_update_at(gmp_env, res->next_replenishment);
+        gmp_add_event(gmp_env, res->next_replenishment, res->id, EVENT_REPLENISH);
 }
 
 static void gmp_queue_active(
@@ -502,16 +504,20 @@ static void gmp_queue_active(
                 list_add_tail(&res->list, &gmp_env->active_reservations);
 
         /* check for possible preemption */
-        if (res->state == RESERVATION_ACTIVE && !check_preempt)
-                gmp_env->schedule_now = true;
+        if (res->state == RESERVATION_ACTIVE && check_preempt)
+                gmp_env->schedule_now++;
         
-        gmp_scheduler_update_after(gmp_env, res->cur_budget);
+        gmp_add_event_after(gmp_env, res->cur_budget, res->id, EVENT_DRAIN);
 }
 
 static void gmp_queue_reservation(
         struct gmp_reservation_environment* gmp_env,
         struct reservation *res)
 {
+        if (res == NULL) {
+                BUG();
+                return;
+        }
         switch (res->state) {
                 case RESERVATION_INACTIVE:
                         list_add(&res->list, &gmp_env->inactive_reservations);
@@ -572,7 +578,7 @@ static void gmp_charge_budget(
                          * its remaining budget */
                          TRACE("requesting gmp_scheduler update for reservation %u in %llu nanoseconds\n",
                                 res->id, res->cur_budget);
-                         gmp_scheduler_update_after(gmp_env, res->cur_budget);
+                         gmp_add_event_after(gmp_env, res->cur_budget, res->id, EVENT_DRAIN);
                 }
                 //if (encountered_active == 2)
                         /* stop at the first ACTIVE reservation */
@@ -601,26 +607,22 @@ static void gmp_replenish_budgets(struct gmp_reservation_environment* gmp_env)
         res = list_first_entry_or_null(&gmp_env->depleted_reservations,
                 struct reservation, list);
         if (res)
-                gmp_scheduler_update_at(gmp_env, res->next_replenishment);
+                gmp_add_event(gmp_env, res->next_replenishment, res->id, EVENT_REPLENISH);
 }
 
 /* return schedule_now */
-bool gmp_update_time(
+int gmp_update_time(
         struct gmp_reservation_environment* gmp_env,
         lt_t now)
 {
         lt_t delta;
 
-        if (!gmp_env) {
-                TRACE("BUG****************************************\n");
-                return false;
-        }
         /* If the time didn't advance, there is nothing to do.
          * This check makes it safe to call sup_advance_time() potentially
          * multiple times (e.g., via different code paths. */
         //TRACE("(sup_update_time) now: %llu, current_time: %llu\n", now, sup_env->env.current_time);
         if (unlikely(now <= gmp_env->env.current_time))
-                return gmp_env->schedule_now;
+                return min(gmp_env->schedule_now, NR_CPUS);
 
         delta = now - gmp_env->env.current_time;
         gmp_env->env.current_time = now;
@@ -634,7 +636,7 @@ bool gmp_update_time(
         //TRACE("REPLENISH###\n");
         gmp_replenish_budgets(gmp_env);
         
-        return gmp_env->schedule_now;
+        return min(gmp_env->schedule_now, NR_CPUS);
 }
 
 static void gmp_res_change_state(
@@ -652,7 +654,7 @@ static void gmp_res_change_state(
         list_del(&res->list);
         /* check if we need to reschedule because we lost an active reservation */
         if (res->state == RESERVATION_ACTIVE && !gmp_env->will_schedule)
-                gmp_env->schedule_now = true;
+                gmp_env->schedule_now++;
         res->state = new_state;
         gmp_queue_reservation(gmp_env, res);
 }
@@ -668,7 +670,7 @@ void gmp_init(struct gmp_reservation_environment* gmp_env)
 
         gmp_env->env.change_state = gmp_res_change_state;
 
-        gmp_env->schedule_now = false;
+        gmp_env->schedule_now = 0;
         gmp_env->will_schedule = false;
         
         raw_spin_lock_init(&gmp_env->lock);
diff --git a/litmus/sched_mc2.c b/litmus/sched_mc2.c
index 6dee1ec2c99c..79fecd49080a 100644
--- a/litmus/sched_mc2.c
+++ b/litmus/sched_mc2.c
@@ -1,3 +1,12 @@
+/*
+ * litmus/sched_mc2.c
+ *
+ * Implementation of the Mixed-Criticality on MultiCore scheduler
+ *
+ * Thus plugin implements a scheduling algorithm proposed in 
+ * "Mixed-Criticality Real-Time Scheduling for Multicore System" paper.
+ */ 
+ 
 #include <linux/percpu.h>
 #include <linux/slab.h>
 #include <asm/uaccess.h>
@@ -10,21 +19,29 @@
 #include <litmus/jobs.h>
 #include <litmus/budget.h>
 #include <litmus/litmus_proc.h>
+#include <litmus/sched_trace.h>
 
 #include <litmus/mc2_common.h>
 #include <litmus/reservation.h>
 #include <litmus/polling_reservations.h>
 
+/* _global_env - reservation container for level-C tasks*/
 struct gmp_reservation_environment _global_env;
 
+/* cpu_entry - keep track of a running task on a cpu
+ * This state is used to decide the lowest priority cpu
+ */
 struct cpu_entry {
         struct task_struct *scheduled;
         lt_t deadline;
         int cpu;
         enum crit_level lv;
+        /* if will_schedule is true, this cpu is already selected and
+           call mc2_schedule() soon. */
         bool will_schedule;
 };
 
+/* cpu_priority - a global state for choosing the lowest priority CPU */
 struct cpu_priority {
         raw_spinlock_t lock;
         struct cpu_entry cpu_entries[NR_CPUS];
@@ -32,19 +49,26 @@ struct cpu_priority {
 
 struct cpu_priority _lowest_prio_cpu;
         
+/* mc2_task_state - a task state structure */
 struct mc2_task_state {
         struct task_client res_info;
+        /* if cpu == -1, this task is a global task (level C) */
         int cpu;
         bool has_departed;
         struct mc2_task mc2_param;
 };
 
+/* crit_entry - maintain the logically running job (ghost job) */
 struct crit_entry {
         enum crit_level level;
         struct task_struct *running;
         //struct hrtimer ghost_timer;
 };
 
+/* mc2_cpu_state - maintain the scheduled state and ghost jobs
+ * timer : timer for partitioned tasks (level A and B)
+ * g_timer : timer for global tasks (level C)
+ */
 struct mc2_cpu_state {
         raw_spinlock_t lock;
 
@@ -62,13 +86,22 @@ static DEFINE_PER_CPU(struct mc2_cpu_state, mc2_cpu_state);
 #define cpu_state_for(cpu_id)   (&per_cpu(mc2_cpu_state, cpu_id))
 #define local_cpu_state()       (&__get_cpu_var(mc2_cpu_state))
 
+/* get_mc2_state - get the task's state */
 static struct mc2_task_state* get_mc2_state(struct task_struct *tsk)
 {
-        return (struct mc2_task_state*) tsk_rt(tsk)->plugin_state;
+        struct mc2_task_state* tinfo;
+        
+        tinfo = (struct mc2_task_state*)tsk_rt(tsk)->plugin_state;
+        
+        if (tinfo)
+                return tinfo;
+        else
+                return NULL;
 }
+
+/* get_task_crit_level - return the criticaility level of a task */
 static enum crit_level get_task_crit_level(struct task_struct *tsk)
 {
-        //struct mc2_task_state *tinfo = get_mc2_state(tsk);
         struct mc2_task *mp;
         
         if (!tsk || !is_realtime(tsk))
@@ -82,7 +115,9 @@ static enum crit_level get_task_crit_level(struct task_struct *tsk)
                 return mp->crit;
 }
 
-static struct reservation* res_find_by_id(struct mc2_cpu_state *state, unsigned int id)
+/* res_find_by_id - find reservation by id */
+static struct reservation* res_find_by_id(struct mc2_cpu_state *state,
+                                          unsigned int id)
 {
         struct reservation *res;
 
@@ -93,7 +128,12 @@ static struct reservation* res_find_by_id(struct mc2_cpu_state *state, unsigned
         return res;
 }
 
-static void mc2_update_time(enum crit_level lv, struct mc2_cpu_state *state, lt_t time)
+/* mc2_update_time - update time for a given criticality level. 
+ *                   caller must hold a proper lock
+ *                   (cpu_state lock or global lock)
+ */
+static void mc2_update_time(enum crit_level lv, 
+                            struct mc2_cpu_state *state, lt_t time)
 {
         if (lv < CRIT_LEVEL_C)
                 sup_update_time(&state->sup_env, time);
@@ -103,6 +143,12 @@ static void mc2_update_time(enum crit_level lv, struct mc2_cpu_state *state, lt_
                 TRACE("update_time(): Criticality level error!!!!\n");
 }
 
+/* task_depart - remove a task from its reservation
+ *               If the job has remaining budget, convert it to a ghost job
+ *               and update crit_entries[]
+ *               
+ * @job_complete        indicate whether job completes or not              
+ */
 static void task_departs(struct task_struct *tsk, int job_complete)
 {
         struct mc2_task_state* tinfo = get_mc2_state(tsk);
@@ -110,28 +156,30 @@ static void task_departs(struct task_struct *tsk, int job_complete)
         struct reservation* res;
         struct reservation_client *client;
 
+        BUG_ON(!is_realtime(tsk));
+        
         res    = tinfo->res_info.client.reservation;
         client = &tinfo->res_info.client;
 
         res->ops->client_departs(res, client, job_complete);
         tinfo->has_departed = true;
         TRACE_TASK(tsk, "CLIENT DEPART with budget %llu\n", res->cur_budget);
+        
         if (job_complete && res->cur_budget) {
                 struct crit_entry* ce;
                 enum crit_level lv = tinfo->mc2_param.crit;
-                //lt_t now = litmus_clock();
                 
                 ce = &state->crit_entries[lv];
                 ce->running = tsk;
                 res->is_ghost = 1;
                 TRACE_TASK(tsk, "BECOME GHOST at %llu\n", litmus_clock());
                 
-                //BUG_ON(hrtimer_active(&ce->ghost_timer));
-                //TRACE("setting GHOST timer %llu\n", ns_to_ktime(now + res->cur_budget));
-                //__hrtimer_start_range_ns(&ce->ghost_timer, ns_to_ktime(now + res->cur_budget), 0, HRTIMER_MODE_ABS_PINNED, 0);
         }               
 }
 
+/* task_arrive - put a task into its reservation
+ *               If the job was a ghost job, remove it from crit_entries[]
+ */
 static void task_arrives(struct mc2_cpu_state *state, struct task_struct *tsk)
 {
         struct mc2_task_state* tinfo = get_mc2_state(tsk);
@@ -145,15 +193,22 @@ static void task_arrives(struct mc2_cpu_state *state, struct task_struct *tsk)
         tinfo->has_departed = false;
         res->ops->client_arrives(res, client);
         
+        sched_trace_task_release(tsk);
+        
         if (lv != NUM_CRIT_LEVELS) {
                 struct crit_entry *ce;
                 ce = &state->crit_entries[lv];
+                /* if the currrent task is a ghost job, remove it */
                 if (ce->running == tsk)
                         ce->running = NULL;
         }
 }
 
-/* return: NO_CPU - all CPUs are running tasks with higher priority than Level C */
+/* get_lowest_prio_cpu - return the lowest priority cpu
+ *                       This will be used for scheduling level-C tasks.
+ *                       If all CPUs are running tasks which has
+ *                       higher priority than level C, return NO_CPU.
+ */
 static int get_lowest_prio_cpu(void)
 {
         struct cpu_entry *ce;
@@ -163,11 +218,15 @@ static int get_lowest_prio_cpu(void)
         raw_spin_lock(&_lowest_prio_cpu.lock);
         for_each_online_cpu(cpu) {
                 ce = &_lowest_prio_cpu.cpu_entries[cpu];
+                /* If a CPU will call schedule() in the near future, we don't
+                   return that CPU. */
                 if (!ce->will_schedule) {
                         if (!ce->scheduled) {
+                                /* Idle cpu, return this. */
                                 raw_spin_unlock(&_lowest_prio_cpu.lock);
                                 return ce->cpu;
-                        } else if (ce->lv == CRIT_LEVEL_C && ce->deadline > latest_deadline) {
+                        } else if (ce->lv == CRIT_LEVEL_C && 
+                                   ce->deadline > latest_deadline) {
                                 latest_deadline = ce->deadline;
                                 ret = ce->cpu;
                         }
@@ -180,6 +239,14 @@ static int get_lowest_prio_cpu(void)
 }
 
 /* NOTE: drops state->lock */
+/* mc2_update_timer_and_unlock - set a timer and g_timer and unlock 
+ *                               Whenever res_env.current_time is updated,
+ *                               we check next_scheduler_update and set 
+ *                               a timer.
+ *                               If there exist a global event which is 
+ *                               not armed on any CPU and g_timer is not
+ *                               active, set a g_timer for that event.
+ */
 static void mc2_update_timer_and_unlock(struct mc2_cpu_state *state)
 {
         int local;
@@ -211,7 +278,8 @@ static void mc2_update_timer_and_unlock(struct mc2_cpu_state *state)
                 /* Reprogram only if not already set correctly. */
                 if (!hrtimer_active(&state->timer) ||
                     ktime_to_ns(hrtimer_get_expires(&state->timer)) != update) {
-                        TRACE("canceling timer...at %llu\n", ktime_to_ns(hrtimer_get_expires(&state->timer)));
+                        TRACE("canceling timer...at %llu\n", 
+                              ktime_to_ns(hrtimer_get_expires(&state->timer)));
                         hrtimer_cancel(&state->timer);
                         TRACE("setting scheduler timer for %llu\n", update);
                         /* We cannot use hrtimer_start() here because the
@@ -246,7 +314,8 @@ static void mc2_update_timer_and_unlock(struct mc2_cpu_state *state)
         raw_spin_lock(&_global_env.lock);
         list_for_each_entry_safe(event, next, &_global_env.next_events, list) {
                 if (event->timer_armed_on == NO_CPU) {
-                        found_event = 1;
+                        /* If the event time is already passed, we call schedule() on
+                           the lowest priority cpu */
                         if (event->next_update < litmus_clock()) {
                                 int cpu = get_lowest_prio_cpu();
                                 TRACE("GLOBAL EVENT PASSED!! poking CPU %d to reschedule\n", cpu);
@@ -260,7 +329,12 @@ static void mc2_update_timer_and_unlock(struct mc2_cpu_state *state)
                                 }
                         } else if (!hrtimer_active(&state->g_timer)) {
                                 int ret;
-                                TRACE("setting global scheduler timer for %llu\n", event->next_update);
+                        
+                                raw_spin_unlock(&_global_env.lock);
+                                found_event = 1;
+                        
+                                TRACE("setting global scheduler timer for %llu\n", 
+                                       event->next_update);
                                 ret = __hrtimer_start_range_ns(&state->g_timer,
                                                 ns_to_ktime(event->next_update),
                                                 0 /* timer coalescing slack */,
@@ -268,74 +342,76 @@ static void mc2_update_timer_and_unlock(struct mc2_cpu_state *state)
                                                 0 /* wakeup */);
                                 if (!ret) {
                                         event->timer_armed_on = state->cpu;
+                                        break;
                                 }
                         }                               
                 }
-        }       
-        raw_spin_unlock(&_global_env.lock);
+        }
+        if (found_event == 0)
+                raw_spin_unlock(&_global_env.lock);
 }
 
+/* mc2_update_ghost_state - Update crit_entries[] to track ghost jobs
+ *                          If the budget of a ghost is exhausted,
+ *                          clear is_ghost and reschedule
+ */
 static void mc2_update_ghost_state(struct mc2_cpu_state *state)
 {
         int lv = 0;
         struct crit_entry* ce;
         struct reservation *res;
         struct mc2_task_state *tinfo;
+
+        BUG_ON(!state);
         
         for (lv = 0; lv < NUM_CRIT_LEVELS; lv++) {
                 ce = &state->crit_entries[lv];
                 if (ce->running != NULL) {
                         tinfo = get_mc2_state(ce->running);
-                        /*
-                        if (lv != CRIT_LEVEL_C)
-                                res = sup_find_by_id(&state->sup_env, tinfo->mc2_param.res_id);
-                        else
+                        if (!tinfo)
                                 continue;
-                        */
+                        
                         res = res_find_by_id(state, tinfo->mc2_param.res_id);
-                        TRACE("LV %d running id %d budget %llu\n", lv, tinfo->mc2_param.res_id, res->cur_budget);
+                        TRACE("LV %d running id %d budget %llu\n", 
+                               lv, tinfo->mc2_param.res_id, res->cur_budget);
+                        /* If the budget is exhausted, clear is_ghost and reschedule */
                         if (!res->cur_budget) {
                                 struct sup_reservation_environment* sup_env = &state->sup_env;
                                 
-                                TRACE("GHOST FINISH id %d at %llu\n", tinfo->mc2_param.res_id, litmus_clock());
+                                TRACE("GHOST FINISH id %d at %llu\n", 
+                                      tinfo->mc2_param.res_id, litmus_clock());
                                 ce->running = NULL;
                                 res->is_ghost = 0;
-                                res = list_first_entry_or_null(&sup_env->active_reservations, struct reservation, list);
-                                if (res)
-                                        litmus_reschedule_local();
+                                
+                                if (lv < CRIT_LEVEL_C) {
+                                        res = list_first_entry_or_null(
+                                              &sup_env->active_reservations, 
+                                                  struct reservation, list);
+                                        if (res)
+                                                litmus_reschedule_local();
+                                } else {
+                                        res = list_first_entry_or_null(
+                                              &_global_env.active_reservations,
+                                                  struct reservation, list);
+                                        if (res)
+                                                litmus_reschedule(state->cpu);
+                                }
                         }
                 }
         }
 }                       
 
-/*
-static enum hrtimer_restart on_ghost_timer(struct hrtimer *timer)
-{
-        struct crit_entry *ce;
-        struct mc2_cpu_state *state;
-        
-        ce = container_of(timer, struct crit_entry, ghost_timer);
-        state = container_of(ce, struct mc2_cpu_state, crit_entries[ce->level]);
-        
-        TRACE("GHOST_TIMER FIRED at %llu\n", litmus_clock());
-        
-        raw_spin_lock(&state->lock);
-        sup_update_time(&state->sup_env, litmus_clock());
-        mc2_update_ghost_state(state);
-        
-        raw_spin_unlock(&state->lock);
-        
-        return HRTIMER_NORESTART;
-}
-*/
-
+/* update_cpu_prio - Update cpu's priority
+ *                   When a cpu picks a new task, call this function
+ *                   to update cpu priorities.
+ */
 static void update_cpu_prio(struct mc2_cpu_state *state)
 {
         struct cpu_entry *ce = &_lowest_prio_cpu.cpu_entries[state->cpu];
         enum crit_level lv = get_task_crit_level(state->scheduled);
         
         if (!state->scheduled) {
-                // cpu is idle.
+                /* cpu is idle. */
                 ce->scheduled = NULL;
                 ce->deadline = ULLONG_MAX;
                 ce->lv = NUM_CRIT_LEVELS;
@@ -344,24 +420,31 @@ static void update_cpu_prio(struct mc2_cpu_state *state)
                 ce->deadline = get_deadline(state->scheduled);
                 ce->lv = lv;
         } else if (lv < CRIT_LEVEL_C) {
+                /* If cpu is running level A or B tasks, it is not eligible
+                   to run level-C tasks */
                 ce->scheduled = state->scheduled;
                 ce->deadline = 0;
                 ce->lv = lv;
         }
 };
 
+/* on_global_scheduling_timer - Process the budget accounting (replenish
+ *                              and charge)
+ */                                                             
 static enum hrtimer_restart on_global_scheduling_timer(struct hrtimer *timer)
 {
         unsigned long flags;
         enum hrtimer_restart restart = HRTIMER_NORESTART;
         struct mc2_cpu_state *state;
         struct next_timer_event *event, *next;
-        bool schedule_now;
+        int schedule_now;
         lt_t update, now;
         int found_event = 0;
 
         state = container_of(timer, struct mc2_cpu_state, g_timer);
 
+        raw_spin_lock_irqsave(&state->lock, flags);
+        
         /* The scheduling timer should only fire on the local CPU, because
          * otherwise deadlocks via timer_cancel() are possible.
          * Note: this does not interfere with dedicated interrupt handling, as
@@ -372,11 +455,13 @@ static enum hrtimer_restart on_global_scheduling_timer(struct hrtimer *timer)
         if (state->cpu != raw_smp_processor_id())
                 TRACE("BUG!!!!!!!!!!!!! TIMER FIRED ON THE OTHER CPU\n");
 
-        raw_spin_lock_irqsave(&_global_env.lock, flags);
+        raw_spin_lock(&_global_env.lock);
         
         update = litmus_clock();
         TRACE("GLOBAL TIMER FIRED at %llu\n", update);
         
+        /* The event can be processed by the other cpus. So, if there is no 
+           events to process, we do nothing */
         list_for_each_entry_safe(event, next, &_global_env.next_events, list) {
                 if (event->next_update < update) {
                         found_event = 1;
@@ -387,22 +472,21 @@ static enum hrtimer_restart on_global_scheduling_timer(struct hrtimer *timer)
         }                       
         
         if (!found_event) {
-                raw_spin_unlock_irqrestore(&_global_env.lock, flags);
-                return restart;
+                goto unlock;
         }
         
+        /* gmp_update_timer returns how many tasks become ACTIVE */
         schedule_now = gmp_update_time(&_global_env, update);
         
-        raw_spin_lock(&state->lock);
         mc2_update_ghost_state(state);
-        raw_spin_unlock(&state->lock);
         
         now = _global_env.env.current_time;
         
         TRACE_CUR("on_global_scheduling_timer at %llu, upd:%llu (for cpu=%d) SCHEDULE_NOW = %d\n",
                 now, update, state->cpu, schedule_now);
 
-        if (schedule_now) {
+        /* Find the lowest cpu, and call reschedule */
+        while (schedule_now--) {
                 int cpu = get_lowest_prio_cpu();
                 if (cpu != NO_CPU) {
                         raw_spin_lock(&_lowest_prio_cpu.lock);
@@ -413,11 +497,15 @@ static enum hrtimer_restart on_global_scheduling_timer(struct hrtimer *timer)
                 }
         } 
 
-        raw_spin_unlock_irqrestore(&_global_env.lock, flags);
-
+unlock:
+        raw_spin_unlock(&_global_env.lock);
+        raw_spin_unlock_irqrestore(&state->lock, flags);
+        
         return restart;
 }
 
+/* on_scheduling_timer - timer event for partitioned tasks
+ */                       
 static enum hrtimer_restart on_scheduling_timer(struct hrtimer *timer)
 {
         unsigned long flags;
@@ -438,8 +526,9 @@ static enum hrtimer_restart on_scheduling_timer(struct hrtimer *timer)
         TRACE("TIMER FIRED at %llu\n", litmus_clock());
         raw_spin_lock_irqsave(&state->lock, flags);
         sup_update_time(&state->sup_env, litmus_clock());
+        raw_spin_lock(&_global_env.lock);
         mc2_update_ghost_state(state);
-        
+        raw_spin_unlock(&_global_env.lock);
         update = state->sup_env.next_scheduler_update;
         now = state->sup_env.env.current_time;
 
@@ -458,6 +547,8 @@ static enum hrtimer_restart on_scheduling_timer(struct hrtimer *timer)
         return restart;
 }
 
+/* mc2_dispatch - Select the next task to schedule.
+ */
 struct task_struct* mc2_dispatch(struct sup_reservation_environment* sup_env, struct mc2_cpu_state* state)
 {
         struct reservation *res, *next;
@@ -477,36 +568,38 @@ struct task_struct* mc2_dispatch(struct sup_reservation_environment* sup_env, st
                                 } else {
                                         ce = &state->crit_entries[lv];
                                         if (likely(!ce->running)) {
+                                                /* If we found the next task, clear all flags */
                                                 sup_scheduler_update_after(sup_env, res->cur_budget);
                                                 res->blocked_by_ghost = 0;
                                                 res->is_ghost = 0;
                                                 return tsk;
                                         } else {
+                                                /* We cannot schedule the same criticality task
+                                                   because the ghost job exists. Set blocked_by_ghost
+                                                   flag not to charge budget */
                                                 res->blocked_by_ghost = 1;
                                         }
                                 }
                         }
                 }
         }
-        // no level A or B tasks
         
+        /* no eligible level A or B tasks exists */
         list_for_each_entry_safe(res, next, &_global_env.active_reservations, list) {
                 if (res->state == RESERVATION_ACTIVE && res->scheduled_on == NO_CPU) {
                         tsk = res->ops->dispatch_client(res, &time_slice);
                         if (likely(tsk)) {
                                 lv = get_task_crit_level(tsk);
                                 if (lv == NUM_CRIT_LEVELS) {
-                                        gmp_scheduler_update_after(&_global_env, res->cur_budget);
-                                        //raw_spin_unlock(&_global_env.lock);
+                                        gmp_add_event_after(&_global_env, res->cur_budget, res->id, EVENT_DRAIN);
                                         return tsk;
                                 } else {
                                         ce = &state->crit_entries[lv];
                                         if (likely(!ce->running)) {
-                                                gmp_scheduler_update_after(&_global_env, res->cur_budget);
+                                                gmp_add_event_after(&_global_env, res->cur_budget, res->id, EVENT_DRAIN);
                                                 res->blocked_by_ghost = 0;
                                                 res->is_ghost = 0;
                                                 res->scheduled_on = state->cpu;
-                                                //raw_spin_unlock(&_global_env.lock);
                                                 return tsk;
                                         } else {
                                                 res->blocked_by_ghost = 1;
@@ -519,18 +612,43 @@ struct task_struct* mc2_dispatch(struct sup_reservation_environment* sup_env, st
         return NULL;
 }
 
+/* not used now */
+static void pre_schedule(struct task_struct *prev)
+{
+        enum crit_level lv;
+        if (!is_realtime(prev) || !prev)
+                return;
+        
+        lv = get_task_crit_level(prev);
+}
+
+/* not used now */
+static void post_schedule(struct task_struct *next)
+{
+        enum crit_level lv;
+        if (!is_realtime(next) || !next)
+                return;
+        
+        lv = get_task_crit_level(next);
+}
+
+/* mc2_schedule - main scheduler function. pick the next task to run
+ */
 static struct task_struct* mc2_schedule(struct task_struct * prev)
 {
         /* next == NULL means "schedule background work". */
         struct mc2_cpu_state *state = local_cpu_state();
         
+        pre_schedule(prev);
+        
         raw_spin_lock(&_lowest_prio_cpu.lock);
         if (_lowest_prio_cpu.cpu_entries[state->cpu].will_schedule == true)
                 _lowest_prio_cpu.cpu_entries[state->cpu].will_schedule = false;
         raw_spin_unlock(&_lowest_prio_cpu.lock);
         
         raw_spin_lock(&state->lock);
-
+        raw_spin_lock(&_global_env.lock);
+        
         //BUG_ON(state->scheduled && state->scheduled != prev);
         //BUG_ON(state->scheduled && !is_realtime(prev));
         if (state->scheduled && state->scheduled != prev)
@@ -540,19 +658,16 @@ static struct task_struct* mc2_schedule(struct task_struct * prev)
 
         /* update time */
         state->sup_env.will_schedule = true;
-        //TRACE_TASK(prev, "MC2_SCHEDULE sup_update_time ####\n");
+
         sup_update_time(&state->sup_env, litmus_clock());
-        
-        raw_spin_lock(&_global_env.lock);
         gmp_update_time(&_global_env, litmus_clock());
         
-        //TRACE_TASK(prev, "MC2_SCHEDULE sup_update_time !!!!\n");
         mc2_update_ghost_state(state);
         
         /* remove task from reservation if it blocks */
         if (is_realtime(prev) && !is_running(prev))
                 task_departs(prev, is_completed(prev));
-
+        
         /* figure out what to schedule next */
         state->scheduled = mc2_dispatch(&state->sup_env, state);
         if (state->scheduled && is_realtime(state->scheduled))
@@ -582,10 +697,10 @@ static struct task_struct* mc2_schedule(struct task_struct * prev)
         }
         if (state->scheduled) {
                 TRACE_TASK(state->scheduled, "scheduled.\n");
-                //tinfo = get_mc2_state(state->scheduled);
-                //state->run_level = tinfo->mc2_param.crit;
         }
         
+        post_schedule(state->scheduled);
+        
         return state->scheduled;
 }
 
@@ -599,13 +714,15 @@ static void resume_legacy_task_model_updates(struct task_struct *tsk)
                  * P-RES scheduler. */
 
                 now = litmus_clock();
-                if (is_tardy(tsk, now))
+                if (is_tardy(tsk, now)) {
                         release_at(tsk, now);
+                        sched_trace_task_release(tsk);
+                }
         }
 }
 
-/* Called when the state of tsk changes back to TASK_RUNNING.
- * We need to requeue the task.
+/* mc2_task_resume - Called when the state of tsk changes back to 
+ *                   TASK_RUNNING. We need to requeue the task.
  */
 static void mc2_task_resume(struct task_struct  *tsk)
 {
@@ -624,23 +741,25 @@ static void mc2_task_resume(struct task_struct  *tsk)
         /* Requeue only if self-suspension was already processed. */
         if (tinfo->has_departed)
         {
+                raw_spin_lock(&_global_env.lock);
                 /* Assumption: litmus_clock() is synchronized across cores,
                  * since we might not actually be executing on tinfo->cpu
                  * at the moment. */
                 if (tinfo->cpu != -1) {
                         sup_update_time(&state->sup_env, litmus_clock());
                 } else {
-                        raw_spin_lock(&_global_env.lock);
+                        //raw_spin_lock(&_global_env.lock);
                         TRACE("RESUME UPDATE ####\n");
                         gmp_update_time(&_global_env, litmus_clock());
                         TRACE("RESUME UPDATE $$$$\n");
-                        raw_spin_unlock(&_global_env.lock);
+                        //raw_spin_unlock(&_global_env.lock);
                 }
                         
                 mc2_update_ghost_state(state);
                 task_arrives(state, tsk);
                 /* NOTE: drops state->lock */
                 TRACE_TASK(tsk, "mc2_resume()\n");
+                raw_spin_unlock(&_global_env.lock);
                 mc2_update_timer_and_unlock(state);
                 local_irq_restore(flags);
         } else {
@@ -651,7 +770,8 @@ static void mc2_task_resume(struct task_struct  *tsk)
         resume_legacy_task_model_updates(tsk);
 }
 
-/* syscall backend for job completions */
+/* mc2_complete_job - syscall backend for job completions
+ */
 static long mc2_complete_job(void)
 {
         ktime_t next_release;
@@ -662,6 +782,8 @@ static long mc2_complete_job(void)
 
         tsk_rt(current)->completed = 1;
         
+        /* If this the first job instance, we need to reset replenish
+           time to the next release time */
         if (tsk_rt(current)->sporadic_release) {
                 struct mc2_cpu_state *state;
                 struct reservation_environment *env;
@@ -670,21 +792,27 @@ static long mc2_complete_job(void)
                 unsigned long flags;
 
                 local_irq_save(flags);
-        
-                state = local_cpu_state();
-                env = &(state->sup_env.env);
+                
                 tinfo = get_mc2_state(current);
                 
-                res = res_find_by_id(state, tsk_rt(current)->mc2_data->res_id);
+                if (get_task_crit_level(current) < CRIT_LEVEL_C)
+                        state = cpu_state_for(tinfo->cpu);
+                else
+                        state = local_cpu_state();
+                
+                raw_spin_lock(&state->lock);
+                env = &(state->sup_env.env);
+                
+                res = res_find_by_id(state, tinfo->mc2_param.res_id);
                 
                 if (get_task_crit_level(current) < CRIT_LEVEL_C) {
-                        raw_spin_lock(&state->lock);
                         env->time_zero = tsk_rt(current)->sporadic_release_time;
                 } else {
                         raw_spin_lock(&_global_env.lock);
                         _global_env.env.time_zero = tsk_rt(current)->sporadic_release_time;
                 }
                 
+                /* set next_replenishtime to synchronous release time */
                 res->next_replenishment = tsk_rt(current)->sporadic_release_time;
                 
                 if (get_task_crit_level(current) == CRIT_LEVEL_A) {
@@ -697,34 +825,44 @@ static long mc2_complete_job(void)
                 res->cur_budget = 0;
                 res->env->change_state(res->env, res, RESERVATION_DEPLETED);
                 
-                //TRACE_CUR("CHANGE NEXT_REP = %llu\n NEXT_UPDATE = %llu\n", res->next_replenishment, state->sup_env.next_scheduler_update);
-                if (get_task_crit_level(current) < CRIT_LEVEL_C) {
-                        raw_spin_unlock(&state->lock);
-                } else {
+                TRACE_CUR("CHANGE NEXT_REP = %llu\n NEXT_UPDATE = %llu\n", res->next_replenishment, state->sup_env.next_scheduler_update);
+                if (get_task_crit_level(current) == CRIT_LEVEL_C) {
                         raw_spin_unlock(&_global_env.lock);
                 }
+                raw_spin_unlock(&state->lock);
                 
                 local_irq_restore(flags);
         }
+        sched_trace_task_completion(current, 0);
         
+        /* update the next release time and deadline */
         prepare_for_next_period(current);
+        
         next_release = ns_to_ktime(get_release(current));
         preempt_disable();
         TRACE_CUR("next_release=%llu\n", get_release(current));
         if (get_release(current) > litmus_clock()) {
+                /* sleep until next_release */
                 set_current_state(TASK_INTERRUPTIBLE);
                 preempt_enable_no_resched();
                 err = schedule_hrtimeout(&next_release, HRTIMER_MODE_ABS);
+                if (get_task_crit_level(current) == CRIT_LEVEL_A)
+                        sched_trace_task_release(current);
         } else {
+                /* release the next job immediately */
                 err = 0;
                 TRACE_CUR("TARDY: release=%llu now=%llu\n", get_release(current), litmus_clock());
                 preempt_enable();
+                if (get_task_crit_level(current) == CRIT_LEVEL_A)
+                        sched_trace_task_release(current);
         }
 
         TRACE_CUR("mc2_complete_job returns at %llu\n", litmus_clock());
         return err;
 }
 
+/* mc2_admit_task - Setup mc2 task parameters
+ */
 static long mc2_admit_task(struct task_struct *tsk)
 {
         long err = -ESRCH;
@@ -752,15 +890,13 @@ static long mc2_admit_task(struct task_struct *tsk)
 
                 res = sup_find_by_id(&state->sup_env, mp->res_id);
 
-                /* found the appropriate reservation (or vCPU) */
+                /* found the appropriate reservation */
                 if (res) {
                         TRACE_TASK(tsk, "SUP FOUND RES ID\n");
                         tinfo->mc2_param.crit = mp->crit;
                         tinfo->mc2_param.res_id = mp->res_id;
-                        
-                        //kfree(tsk_rt(tsk)->plugin_state);
-                        //tsk_rt(tsk)->plugin_state = NULL;
-                        
+                
+                        /* initial values */
                         err = mc2_task_client_init(&tinfo->res_info, &tinfo->mc2_param, tsk, res);
                         tinfo->cpu = task_cpu(tsk);
                         tinfo->has_departed = true;
@@ -772,7 +908,10 @@ static long mc2_admit_task(struct task_struct *tsk)
 
                 raw_spin_unlock_irqrestore(&state->lock, flags);
         } else if (lv == CRIT_LEVEL_C) {
-                raw_spin_lock_irqsave(&_global_env.lock, flags);
+                local_irq_save(flags);
+                state = local_cpu_state();
+                raw_spin_lock(&state->lock);
+                raw_spin_lock(&_global_env.lock);
                 
                 res = gmp_find_by_id(&_global_env, mp->res_id);
 
@@ -782,9 +921,7 @@ static long mc2_admit_task(struct task_struct *tsk)
                         tinfo->mc2_param.crit = mp->crit;
                         tinfo->mc2_param.res_id = mp->res_id;
                         
-                        //kfree(tsk_rt(tsk)->plugin_state);
-                        //tsk_rt(tsk)->plugin_state = NULL;
-                        
+                        /* initial values */
                         err = mc2_task_client_init(&tinfo->res_info, &tinfo->mc2_param, tsk, res);
                         tinfo->cpu = -1;
                         tinfo->has_departed = true;
@@ -794,8 +931,9 @@ static long mc2_admit_task(struct task_struct *tsk)
                         tsk_rt(tsk)->task_params.budget_policy = NO_ENFORCEMENT;
                 }
 
-                raw_spin_unlock_irqrestore(&_global_env.lock, flags);
-                
+                raw_spin_unlock(&_global_env.lock);
+                raw_spin_unlock(&state->lock);
+                local_irq_restore(flags);
         }
         
         preempt_enable();
@@ -806,6 +944,9 @@ static long mc2_admit_task(struct task_struct *tsk)
         return err;
 }
 
+/* mc2_task_new - A new real-time job is arrived. Release the next job
+ *                at the next reservation replenish time
+ */
 static void mc2_task_new(struct task_struct *tsk, int on_runqueue,
                           int is_running)
 {
@@ -837,11 +978,12 @@ static void mc2_task_new(struct task_struct *tsk, int on_runqueue,
                  * [see comment in pres_task_resume()] */
                 raw_spin_lock(&_global_env.lock);
                 mc2_update_time(lv, state, litmus_clock());
-                raw_spin_unlock(&_global_env.lock);
                 mc2_update_ghost_state(state);
                 task_arrives(state, tsk);
                 /* NOTE: drops state->lock */
                 TRACE("mc2_new()\n");
+                raw_spin_unlock(&_global_env.lock);
+                
                 mc2_update_timer_and_unlock(state);
                 local_irq_restore(flags);
         } else
@@ -857,6 +999,8 @@ static void mc2_task_new(struct task_struct *tsk, int on_runqueue,
                 TRACE_TASK(tsk, "mc2_task_new() next_replenishment = NULL\n");
 }
 
+/* mc2_reservation_destroy - reservation_destroy system call backend
+ */
 static long mc2_reservation_destroy(unsigned int reservation_id, int cpu)
 {
         long ret = -EINVAL;
@@ -865,8 +1009,13 @@ static long mc2_reservation_destroy(unsigned int reservation_id, int cpu)
         struct sup_reservation_environment *sup_env;
         int found = 0;
         enum crit_level lv = get_task_crit_level(current);
+        unsigned long flags;
         
         if (cpu == -1) {
+                /* if the reservation is global reservation */
+                local_irq_save(flags);
+                state = local_cpu_state();
+                raw_spin_lock(&state->lock);
                 raw_spin_lock(&_global_env.lock);
         
                 list_for_each_entry_safe(res, next, &_global_env.depleted_reservations, list) {
@@ -901,34 +1050,16 @@ static long mc2_reservation_destroy(unsigned int reservation_id, int cpu)
                         }
                 }
 
-/*              
-list_for_each_entry(res, &_global_env.depleted_reservations, list) {
-        TRACE("DEPLETED LIST R%d\n", res->id);
-}
-list_for_each_entry(res, &_global_env.inactive_reservations, list) {
-        TRACE("INACTIVE LIST R%d\n", res->id);
-}
-list_for_each_entry(res, &_global_env.active_reservations, list) {
-        TRACE("ACTIVE LIST R%d\n", res->id);
-}
-*/
-                if (list_empty(&_global_env.active_reservations)) 
-                        INIT_LIST_HEAD(&_global_env.active_reservations);
-                if (list_empty(&_global_env.depleted_reservations)) 
-                        INIT_LIST_HEAD(&_global_env.depleted_reservations);
-                if (list_empty(&_global_env.inactive_reservations)) 
-                        INIT_LIST_HEAD(&_global_env.inactive_reservations);
-                if (list_empty(&_global_env.next_events)) 
-                        INIT_LIST_HEAD(&_global_env.next_events);
-                
                 raw_spin_unlock(&_global_env.lock);
+                raw_spin_unlock(&state->lock);
+                local_irq_restore(flags);
         } else {
+                /* if the reservation is partitioned reservation */
                 state = cpu_state_for(cpu);
-                raw_spin_lock(&state->lock);
+                raw_spin_lock_irqsave(&state->lock, flags);
                 
         //      res = sup_find_by_id(&state->sup_env, reservation_id);
                 sup_env = &state->sup_env;
-                //if (!res) {
                 list_for_each_entry_safe(res, next, &sup_env->depleted_reservations, list) {
                         if (res->id == reservation_id) {
                                 if (lv == CRIT_LEVEL_A) {
@@ -972,15 +1103,16 @@ list_for_each_entry(res, &_global_env.active_reservations, list) {
                                 }
                         }
                 }
-                //}
 
-                raw_spin_unlock(&state->lock);
+                raw_spin_unlock_irqrestore(&state->lock, flags);
         }
         
         TRACE("RESERVATION_DESTROY ret = %d\n", ret);
         return ret;
 }
 
+/* mc2_task_exit - Task became a normal task (not real-time task)
+ */
 static void mc2_task_exit(struct task_struct *tsk)
 {
         unsigned long flags;
@@ -1007,42 +1139,30 @@ static void mc2_task_exit(struct task_struct *tsk)
         if (is_running(tsk)) {
                 /* Assumption: litmus_clock() is synchronized across cores
                  * [see comment in pres_task_resume()] */
-                //if (lv < CRIT_LEVEL_C)
-                //      sup_update_time(&state->sup_env, litmus_clock());
+                
                 raw_spin_lock(&_global_env.lock);
+                /* update both global and partitioned */
                 mc2_update_time(lv, state, litmus_clock());
-                raw_spin_unlock(&_global_env.lock);
+                
                 mc2_update_ghost_state(state);
                 task_departs(tsk, 0);
                 
                 /* NOTE: drops state->lock */
                 TRACE("mc2_exit()\n");
+                raw_spin_unlock(&_global_env.lock);
                 mc2_update_timer_and_unlock(state);
                 local_irq_restore(flags);
         } else
                 raw_spin_unlock_irqrestore(&state->lock, flags);
-/*
-        if (tinfo->mc2_param.crit == CRIT_LEVEL_A) {
-                struct table_driven_reservation *td_res;
-                struct reservation *res;
-                res = sup_find_by_id(&state->sup_env, tinfo->mc2_param.res_id);
-                td_res = container_of(res, struct table_driven_reservation, res);
-                kfree(td_res->intervals);
-                //kfree(td_res);
-        } else if (tinfo->mc2_param.crit == CRIT_LEVEL_B) {
-                struct polling_reservation *pres;
-                struct reservation *res;
-                res = sup_find_by_id(&state->sup_env, tinfo->mc2_param.res_id);
-                pres = container_of(res, struct polling_reservation, res);
-                kfree(pres);
-        }
-*/
+
         kfree(tsk_rt(tsk)->plugin_state);
         tsk_rt(tsk)->plugin_state = NULL;
         kfree(tsk_rt(tsk)->mc2_data);
         tsk_rt(tsk)->mc2_data = NULL;
 }
 
+/* create_polling_reservation - create a new polling reservation
+ */
 static long create_polling_reservation(
         int res_type,
         struct reservation_config *config)
@@ -1055,6 +1175,7 @@ static long create_polling_reservation(
         int periodic =  res_type == PERIODIC_POLLING;
         long err = -EINVAL;
 
+        /* sanity checks */
         if (config->polling_params.budget >
             config->polling_params.period) {
                 printk(KERN_ERR "invalid polling reservation (%u): "
@@ -1138,6 +1259,8 @@ static long create_polling_reservation(
 
 #define MAX_INTERVALS 1024
 
+/* create_table_driven_reservation - create a table_driven reservation
+ */
 static long create_table_driven_reservation(
         struct reservation_config *config)
 {
@@ -1238,6 +1361,8 @@ static long create_table_driven_reservation(
         return err;
 }
 
+/* mc2_reservation_create - reservation_create system call backend
+ */
 static long mc2_reservation_create(int res_type, void* __user _config)
 {
         long ret = -EINVAL;