MC2 scheduling infrastructure

3 files changed, 204 insertions, 72 deletions

diff --git a/include/litmus/reservation.h b/include/litmus/reservation.h
index 0b9c08d6051e..7e022b34470f 100644
--- a/include/litmus/reservation.h
+++ b/include/litmus/reservation.h
@@ -129,6 +129,7 @@ struct reservation {
         
         /* for global env. */
         int scheduled_on;
+        int event_added;
         /* for blocked by ghost. Do not charge budget when ACTIVE */
         int blocked_by_ghost;
         /* ghost_job. If it is clear, do not charge budget when ACTIVE_IDLE */
@@ -244,6 +245,7 @@ void gmp_add_new_reservation(struct gmp_reservation_environment* gmp_env,
         struct reservation* new_res);
 void gmp_add_event_after(struct gmp_reservation_environment* gmp_env,
         lt_t timeout, unsigned int id, event_type_t type);
+void gmp_print_events(struct gmp_reservation_environment* gmp_env, lt_t now);
 int gmp_update_time(struct gmp_reservation_environment* gmp_env, lt_t now);
 struct task_struct* gmp_dispatch(struct gmp_reservation_environment* gmp_env);
 struct next_timer_event* gmp_find_event_by_id(struct gmp_reservation_environment* gmp_env, unsigned int id);
diff --git a/litmus/reservation.c b/litmus/reservation.c
index b0b13a9916ef..3ec18a23f588 100644
--- a/litmus/reservation.c
+++ b/litmus/reservation.c
@@ -428,7 +428,7 @@ static void gmp_add_event(
         
         nevent = gmp_find_event_by_id(gmp_env, id);
         
-        if (!nevent) {
+        if (!nevent || nevent->type != type) {
                 nevent = kzalloc(sizeof(*nevent), GFP_ATOMIC);
                 nevent->next_update = when;
                 nevent->id = id;
@@ -440,18 +440,22 @@ static void gmp_add_event(
                         if (queued->next_update > nevent->next_update) {
                                 list_add(&nevent->list, pos->prev);
                                 found = 1;
-                                TRACE("NEXT_EVENT at %llu ADDED before %llu\n", nevent->next_update, queued->next_update);
+                                TRACE("NEXT_EVENT id=%d type=%d update=%llu ADDED at before %llu\n", nevent->id, nevent->type, nevent->next_update, queued->next_update);
                                 break;
                         }
                 }
                 
                 if (!found) {
                         list_add_tail(&nevent->list, &gmp_env->next_events);
-                        TRACE("NEXT_EVENT ADDED at %llu ADDED at HEAD\n", nevent->next_update);
+                        TRACE("NEXT_EVENT id=%d type=%d update=%llu ADDED at TAIL\n", nevent->id, nevent->type, nevent->next_update);
                 }
         } else {
-                TRACE("EVENT FOUND type=%d when=%llu, NEW EVENT type=%d when=%llu\n", nevent->type, nevent->next_update, type, when);
+                TRACE("EVENT FOUND id = %d type=%d when=%llu, NEW EVENT type=%d when=%llu\n", nevent->id, nevent->type, nevent->next_update, type, when);
         }
+        
+        TRACE("======START PRINTING EVENT LIST======\n");
+        gmp_print_events(gmp_env, litmus_clock());
+        TRACE("======FINISH PRINTING EVENT LIST======\n");
 }
 
 void gmp_add_event_after(
@@ -478,8 +482,9 @@ static void gmp_queue_depleted(
 
         if (!found)
                 list_add_tail(&res->list, &gmp_env->depleted_reservations);
-
+        TRACE("R%d queued to depleted_list\n", res->id);
         gmp_add_event(gmp_env, res->next_replenishment, res->id, EVENT_REPLENISH);
+        res->event_added = 1;
 }
 
 static void gmp_queue_active(
@@ -508,6 +513,7 @@ static void gmp_queue_active(
                 gmp_env->schedule_now++;
         
         gmp_add_event_after(gmp_env, res->cur_budget, res->id, EVENT_DRAIN);
+        res->event_added = 1;
 }
 
 static void gmp_queue_reservation(
@@ -554,12 +560,14 @@ static void gmp_charge_budget(
                 /* charge all ACTIVE_IDLE up to the first ACTIVE reservation */
                 res = list_entry(pos, struct reservation, list);
                 if (res->state == RESERVATION_ACTIVE) {
-                        TRACE("gmp_charge_budget ACTIVE R%u drain %llu\n", res->id, delta);
+                        TRACE("gmp_charge_budget ACTIVE R%u scheduled_on=%d drain %llu\n", res->id, res->scheduled_on, delta);
                         if (res->scheduled_on != NO_CPU && res->blocked_by_ghost == 0) {
                                 TRACE("DRAIN !!\n");
                                 drained = 1;
                                 res->ops->drain_budget(res, delta);
-                        }                       
+                        } else {
+                                TRACE("NO DRAIN (not scheduled)!!\n");
+                        }
                 } else {
                         //BUG_ON(res->state != RESERVATION_ACTIVE_IDLE);
                         if (res->state != RESERVATION_ACTIVE_IDLE)
@@ -579,6 +587,7 @@ static void gmp_charge_budget(
                          TRACE("requesting gmp_scheduler update for reservation %u in %llu nanoseconds\n",
                                 res->id, res->cur_budget);
                          gmp_add_event_after(gmp_env, res->cur_budget, res->id, EVENT_DRAIN);
+                         res->event_added = 1;
                 }
                 //if (encountered_active == 2)
                         /* stop at the first ACTIVE reservation */
@@ -601,33 +610,49 @@ static void gmp_replenish_budgets(struct gmp_reservation_environment* gmp_env)
                         break;
                 }
         }
-        //TRACE("finished replenishing budgets\n");
+        TRACE("finished replenishing budgets\n");
 
         /* request a scheduler update at the next replenishment instant */
+        list_for_each_safe(pos, next, &gmp_env->depleted_reservations) {
+                res = list_entry(pos, struct reservation, list);
+                if (res->event_added == 0) {
+                        gmp_add_event(gmp_env, res->next_replenishment, res->id, EVENT_REPLENISH);
+                        res->event_added = 1;
+                }
+        }       
+        
+/*
         res = list_first_entry_or_null(&gmp_env->depleted_reservations,
                 struct reservation, list);
-        if (res)
+        if (res && res->event_added == 0) {
                 gmp_add_event(gmp_env, res->next_replenishment, res->id, EVENT_REPLENISH);
+                res->event_added = 1;
+        }
+*/
 }
 
+#define EPSILON 50
+
 /* return schedule_now */
 int gmp_update_time(
         struct gmp_reservation_environment* gmp_env,
         lt_t now)
 {
-        lt_t delta;
+        struct next_timer_event *event, *next;
+        lt_t delta, ret;
 
         /* If the time didn't advance, there is nothing to do.
          * This check makes it safe to call sup_advance_time() potentially
          * multiple times (e.g., via different code paths. */
-        //TRACE("(sup_update_time) now: %llu, current_time: %llu\n", now, sup_env->env.current_time);
-        if (unlikely(now <= gmp_env->env.current_time))
-                return min(gmp_env->schedule_now, NR_CPUS);
+        TRACE("(gmp_update_time) now: %llu, current_time: %llu\n", now, gmp_env->env.current_time);
+        if (unlikely(now <= gmp_env->env.current_time + EPSILON))
+                return 0;
 
         delta = now - gmp_env->env.current_time;
         gmp_env->env.current_time = now;
 
 
+        //gmp_print_events(gmp_env, now);
         /* deplete budgets by passage of time */
         //TRACE("CHARGE###\n");
         gmp_charge_budget(gmp_env, delta);
@@ -636,7 +661,30 @@ int gmp_update_time(
         //TRACE("REPLENISH###\n");
         gmp_replenish_budgets(gmp_env);
         
-        return min(gmp_env->schedule_now, NR_CPUS);
+        list_for_each_entry_safe(event, next, &gmp_env->next_events, list) {
+                if (event->next_update < now) {
+                        list_del(&event->list);
+                        TRACE("EVENT at %llu IS DELETED\n", event->next_update);
+                        kfree(event);
+                }
+        }               
+        
+        //gmp_print_events(gmp_env, litmus_clock());
+        
+        ret = min(gmp_env->schedule_now, NR_CPUS);
+        gmp_env->schedule_now = 0;
+        
+        return ret;
+}
+
+void gmp_print_events(struct gmp_reservation_environment* gmp_env, lt_t now)
+{
+        struct next_timer_event *event, *next;
+
+        TRACE("GLOBAL EVENTS now=%llu\n", now);
+        list_for_each_entry_safe(event, next, &gmp_env->next_events, list) {
+                TRACE("at %llu type=%d id=%d armed_on=%d\n", event->next_update, event->type, event->id, event->timer_armed_on);
+        }               
 }
 
 static void gmp_res_change_state(
@@ -653,7 +701,7 @@ static void gmp_res_change_state(
 
         list_del(&res->list);
         /* check if we need to reschedule because we lost an active reservation */
-        if (res->state == RESERVATION_ACTIVE && !gmp_env->will_schedule)
+        if (res->state == RESERVATION_ACTIVE)
                 gmp_env->schedule_now++;
         res->state = new_state;
         gmp_queue_reservation(gmp_env, res);
diff --git a/litmus/sched_mc2.c b/litmus/sched_mc2.c
index 79fecd49080a..b3390dc87d47 100644
--- a/litmus/sched_mc2.c
+++ b/litmus/sched_mc2.c
@@ -74,7 +74,7 @@ struct mc2_cpu_state {
 
         struct sup_reservation_environment sup_env;
         struct hrtimer timer;
-        struct hrtimer g_timer;
+        //struct hrtimer g_timer;
 
         int cpu;
         struct task_struct* scheduled;
@@ -209,21 +209,33 @@ static void task_arrives(struct mc2_cpu_state *state, struct task_struct *tsk)
  *                       If all CPUs are running tasks which has
  *                       higher priority than level C, return NO_CPU.
  */
-static int get_lowest_prio_cpu(void)
+static int get_lowest_prio_cpu(lt_t priority)
 {
         struct cpu_entry *ce;
         int cpu, ret = NO_CPU;
         lt_t latest_deadline = 0;
         
         raw_spin_lock(&_lowest_prio_cpu.lock);
+        ce = &_lowest_prio_cpu.cpu_entries[local_cpu_state()->cpu];
+        if (!ce->will_schedule && !ce->scheduled) {
+                raw_spin_unlock(&_lowest_prio_cpu.lock);
+                TRACE("CPU %d (local) is the lowest!\n", ce->cpu);
+                return ce->cpu;
+        }       
+
         for_each_online_cpu(cpu) {
                 ce = &_lowest_prio_cpu.cpu_entries[cpu];
                 /* If a CPU will call schedule() in the near future, we don't
                    return that CPU. */
+                TRACE("CPU %d will_schedule=%d, scheduled=(%s/%d:%d)\n", cpu, ce->will_schedule,
+              ce->scheduled ? (ce->scheduled)->comm : "null",
+              ce->scheduled ? (ce->scheduled)->pid : 0,
+              ce->scheduled ? (ce->scheduled)->rt_param.job_params.job_no : 0);
                 if (!ce->will_schedule) {
                         if (!ce->scheduled) {
                                 /* Idle cpu, return this. */
                                 raw_spin_unlock(&_lowest_prio_cpu.lock);
+                                TRACE("CPU %d is the lowest!\n", ce->cpu);
                                 return ce->cpu;
                         } else if (ce->lv == CRIT_LEVEL_C && 
                                    ce->deadline > latest_deadline) {
@@ -234,7 +246,12 @@ static int get_lowest_prio_cpu(void)
         }               
         
         raw_spin_unlock(&_lowest_prio_cpu.lock);
+
+        if (priority >= latest_deadline)
+                ret = NO_CPU;
         
+        TRACE("CPU %d is the lowest!\n", ret);
+
         return ret;
 }
 
@@ -253,7 +270,6 @@ static void mc2_update_timer_and_unlock(struct mc2_cpu_state *state)
         lt_t update, now;
         enum crit_level lv = get_task_crit_level(state->scheduled);
         struct next_timer_event *event, *next;
-        int found_event = 0;
         
         //TRACE_TASK(state->scheduled, "update_timer!\n");
         if (lv != NUM_CRIT_LEVELS)
@@ -268,10 +284,35 @@ static void mc2_update_timer_and_unlock(struct mc2_cpu_state *state)
          */
         local = local_cpu_state() == state;
 
+        list_for_each_entry_safe(event, next, &_global_env.next_events, list) {
+                /* If the event time is already passed, we call schedule() on
+                   the lowest priority cpu */
+                if (event->next_update < litmus_clock()) {
+                        if (event->timer_armed_on == NO_CPU) {
+                                struct reservation *res = gmp_find_by_id(&_global_env, event->id);
+                                int cpu = get_lowest_prio_cpu(res?res->priority:0);
+                                TRACE("GLOBAL EVENT PASSED!! poking CPU %d to reschedule\n", cpu);
+                                list_del(&event->list);
+                                kfree(event);
+                                if (cpu != NO_CPU) {
+                                        raw_spin_lock(&_lowest_prio_cpu.lock);
+                                        _lowest_prio_cpu.cpu_entries[cpu].will_schedule = true;
+                                        raw_spin_unlock(&_lowest_prio_cpu.lock);
+                                        litmus_reschedule(cpu);
+                                }
+                        }
+                } else if (event->next_update < update && event->timer_armed_on == NO_CPU) {
+                        event->timer_armed_on = state->cpu;
+                        update = event->next_update;
+                        break;
+                }
+        }
+        
         /* Must drop state lock before calling into hrtimer_start(), which
          * may raise a softirq, which in turn may wake ksoftirqd. */
         raw_spin_unlock(&state->lock);
-
+        raw_spin_unlock(&_global_env.lock);
+        
         if (update <= now) {
                 litmus_reschedule(state->cpu);
         } else if (likely(local && update != SUP_NO_SCHEDULER_UPDATE)) {
@@ -310,7 +351,8 @@ static void mc2_update_timer_and_unlock(struct mc2_cpu_state *state)
                         litmus_reschedule(state->cpu);
                 }
         }
-        
+
+#if 0   
         raw_spin_lock(&_global_env.lock);
         list_for_each_entry_safe(event, next, &_global_env.next_events, list) {
                 if (event->timer_armed_on == NO_CPU) {
@@ -349,6 +391,7 @@ static void mc2_update_timer_and_unlock(struct mc2_cpu_state *state)
         }
         if (found_event == 0)
                 raw_spin_unlock(&_global_env.lock);
+#endif  
 }
 
 /* mc2_update_ghost_state - Update crit_entries[] to track ghost jobs
@@ -396,6 +439,9 @@ static void mc2_update_ghost_state(struct mc2_cpu_state *state)
                                         if (res)
                                                 litmus_reschedule(state->cpu);
                                 }
+                        } else {
+                                TRACE("GHOST NOT FINISH id %d budget %llu\n", res->id, res->cur_budget);
+                                gmp_add_event_after(&_global_env, res->cur_budget, res->id, EVENT_DRAIN);
                         }
                 }
         }
@@ -428,6 +474,7 @@ static void update_cpu_prio(struct mc2_cpu_state *state)
         }
 };
 
+#if 0
 /* on_global_scheduling_timer - Process the budget accounting (replenish
  *                              and charge)
  */                                                             
@@ -503,6 +550,7 @@ unlock:
         
         return restart;
 }
+#endif
 
 /* on_scheduling_timer - timer event for partitioned tasks
  */                       
@@ -512,6 +560,7 @@ static enum hrtimer_restart on_scheduling_timer(struct hrtimer *timer)
         enum hrtimer_restart restart = HRTIMER_NORESTART;
         struct mc2_cpu_state *state;
         lt_t update, now;
+        int global_schedule_now;
 
         state = container_of(timer, struct mc2_cpu_state, timer);
 
@@ -524,16 +573,19 @@ static enum hrtimer_restart on_scheduling_timer(struct hrtimer *timer)
         BUG_ON(state->cpu != raw_smp_processor_id());
 
         TRACE("TIMER FIRED at %llu\n", litmus_clock());
-        raw_spin_lock_irqsave(&state->lock, flags);
+        raw_spin_lock_irqsave(&_global_env.lock, flags);
+        raw_spin_lock(&state->lock);
+        
         sup_update_time(&state->sup_env, litmus_clock());
-        raw_spin_lock(&_global_env.lock);
+        global_schedule_now = gmp_update_time(&_global_env, litmus_clock());
+        
         mc2_update_ghost_state(state);
-        raw_spin_unlock(&_global_env.lock);
+        
         update = state->sup_env.next_scheduler_update;
         now = state->sup_env.env.current_time;
 
-        TRACE_CUR("on_scheduling_timer at %llu, upd:%llu (for cpu=%d)\n",
-                now, update, state->cpu);
+        TRACE_CUR("on_scheduling_timer at %llu, upd:%llu (for cpu=%d) g_schedule_now:%d\n",
+                now, update, state->cpu, global_schedule_now);
 
         if (update <= now) {
                 litmus_reschedule_local();
@@ -542,7 +594,20 @@ static enum hrtimer_restart on_scheduling_timer(struct hrtimer *timer)
                 restart = HRTIMER_RESTART;
         }
 
-        raw_spin_unlock_irqrestore(&state->lock, flags);
+        /* Find the lowest cpu, and call reschedule */
+        while (global_schedule_now--) {
+                int cpu = get_lowest_prio_cpu(0);
+                if (cpu != NO_CPU) {
+                        raw_spin_lock(&_lowest_prio_cpu.lock);
+                        _lowest_prio_cpu.cpu_entries[cpu].will_schedule = true;
+                        raw_spin_unlock(&_lowest_prio_cpu.lock);
+                        TRACE("LOWEST CPU = P%d\n", cpu);
+                        litmus_reschedule(cpu);
+                }
+        } 
+        
+        raw_spin_unlock(&state->lock);
+        raw_spin_unlock_irqrestore(&_global_env.lock, flags);
 
         return restart;
 }
@@ -555,7 +620,7 @@ struct task_struct* mc2_dispatch(struct sup_reservation_environment* sup_env, st
         struct task_struct *tsk = NULL;
         struct crit_entry *ce;
         enum crit_level lv;
-        lt_t time_slice;
+        lt_t time_slice, cur_priority;
 
         list_for_each_entry_safe(res, next, &sup_env->active_reservations, list) {
                 if (res->state == RESERVATION_ACTIVE) {
@@ -578,6 +643,7 @@ struct task_struct* mc2_dispatch(struct sup_reservation_environment* sup_env, st
                                                    because the ghost job exists. Set blocked_by_ghost
                                                    flag not to charge budget */
                                                 res->blocked_by_ghost = 1;
+                                                TRACE_TASK(ce->running, " is GHOST\n");
                                         }
                                 }
                         }
@@ -585,24 +651,32 @@ struct task_struct* mc2_dispatch(struct sup_reservation_environment* sup_env, st
         }
         
         /* no eligible level A or B tasks exists */
+        cur_priority = _lowest_prio_cpu.cpu_entries[state->cpu].deadline;
+        
+        TRACE("****** ACTIVE LIST ******\n");
+        TRACE_TASK(_lowest_prio_cpu.cpu_entries[state->cpu].scheduled, "** CURRENT JOB deadline %llu **\n", cur_priority);
         list_for_each_entry_safe(res, next, &_global_env.active_reservations, list) {
+                TRACE("R%d deadline=%llu, scheduled_on=%d\n", res->id, res->priority, res->scheduled_on);
                 if (res->state == RESERVATION_ACTIVE && res->scheduled_on == NO_CPU) {
                         tsk = res->ops->dispatch_client(res, &time_slice);
                         if (likely(tsk)) {
                                 lv = get_task_crit_level(tsk);
                                 if (lv == NUM_CRIT_LEVELS) {
                                         gmp_add_event_after(&_global_env, res->cur_budget, res->id, EVENT_DRAIN);
+                                        res->event_added = 1;
                                         return tsk;
                                 } else {
                                         ce = &state->crit_entries[lv];
                                         if (likely(!ce->running)) {
                                                 gmp_add_event_after(&_global_env, res->cur_budget, res->id, EVENT_DRAIN);
+                                                res->event_added = 1;
                                                 res->blocked_by_ghost = 0;
                                                 res->is_ghost = 0;
                                                 res->scheduled_on = state->cpu;
                                                 return tsk;
                                         } else {
                                                 res->blocked_by_ghost = 1;
+                                                TRACE_TASK(ce->running, " is GHOST\n");
                                         }
                                 }
                         }
@@ -641,13 +715,8 @@ static struct task_struct* mc2_schedule(struct task_struct * prev)
         
         pre_schedule(prev);
         
-        raw_spin_lock(&_lowest_prio_cpu.lock);
-        if (_lowest_prio_cpu.cpu_entries[state->cpu].will_schedule == true)
-                _lowest_prio_cpu.cpu_entries[state->cpu].will_schedule = false;
-        raw_spin_unlock(&_lowest_prio_cpu.lock);
-        
-        raw_spin_lock(&state->lock);
         raw_spin_lock(&_global_env.lock);
+        raw_spin_lock(&state->lock);
         
         //BUG_ON(state->scheduled && state->scheduled != prev);
         //BUG_ON(state->scheduled && !is_realtime(prev));
@@ -668,12 +737,14 @@ static struct task_struct* mc2_schedule(struct task_struct * prev)
         if (is_realtime(prev) && !is_running(prev))
                 task_departs(prev, is_completed(prev));
         
+        raw_spin_lock(&_lowest_prio_cpu.lock);
+        _lowest_prio_cpu.cpu_entries[state->cpu].will_schedule = false;
+        
         /* figure out what to schedule next */
         state->scheduled = mc2_dispatch(&state->sup_env, state);
         if (state->scheduled && is_realtime(state->scheduled))
                 TRACE_TASK(state->scheduled, "mc2_dispatch picked me!\n");
         
-        raw_spin_lock(&_lowest_prio_cpu.lock);
         update_cpu_prio(state);
         raw_spin_unlock(&_lowest_prio_cpu.lock);
         
@@ -682,18 +753,29 @@ static struct task_struct* mc2_schedule(struct task_struct * prev)
 
         /* program scheduler timer */
         state->sup_env.will_schedule = false;
-        
-        raw_spin_unlock(&_global_env.lock);
-        
+                
         /* NOTE: drops state->lock */
         mc2_update_timer_and_unlock(state);
 
+
+        
         if (prev != state->scheduled && is_realtime(prev)) {
                 struct mc2_task_state* tinfo = get_mc2_state(prev);
                 struct reservation* res = tinfo->res_info.client.reservation;
                 TRACE_TASK(prev, "PREV JOB scheduled_on = P%d\n", res->scheduled_on);
                 res->scheduled_on = NO_CPU;
                 TRACE_TASK(prev, "descheduled.\n");
+                /* if prev is preempted and a global task, find the lowest cpu and reschedule */
+                if (tinfo->has_departed == false && get_task_crit_level(prev) == CRIT_LEVEL_C) {
+                        int cpu = get_lowest_prio_cpu(res?res->priority:0);
+                        TRACE("LEVEL-C TASK PREEMPTED!! poking CPU %d to reschedule\n", cpu);
+                        if (cpu != NO_CPU) {
+                                raw_spin_lock(&_lowest_prio_cpu.lock);
+                                _lowest_prio_cpu.cpu_entries[cpu].will_schedule = true;
+                                raw_spin_unlock(&_lowest_prio_cpu.lock);
+                                litmus_reschedule(cpu);
+                        }
+                }
         }
         if (state->scheduled) {
                 TRACE_TASK(state->scheduled, "scheduled.\n");
@@ -737,34 +819,31 @@ static void mc2_task_resume(struct task_struct  *tsk)
         else
                 state = local_cpu_state();
 
-        raw_spin_lock_irqsave(&state->lock, flags);
+        raw_spin_lock_irqsave(&_global_env.lock, flags);
         /* Requeue only if self-suspension was already processed. */
         if (tinfo->has_departed)
         {
-                raw_spin_lock(&_global_env.lock);
+                raw_spin_lock(&state->lock);
                 /* Assumption: litmus_clock() is synchronized across cores,
                  * since we might not actually be executing on tinfo->cpu
                  * at the moment. */
                 if (tinfo->cpu != -1) {
                         sup_update_time(&state->sup_env, litmus_clock());
                 } else {
-                        //raw_spin_lock(&_global_env.lock);
                         TRACE("RESUME UPDATE ####\n");
                         gmp_update_time(&_global_env, litmus_clock());
                         TRACE("RESUME UPDATE $$$$\n");
-                        //raw_spin_unlock(&_global_env.lock);
                 }
                         
                 mc2_update_ghost_state(state);
                 task_arrives(state, tsk);
                 /* NOTE: drops state->lock */
                 TRACE_TASK(tsk, "mc2_resume()\n");
-                raw_spin_unlock(&_global_env.lock);
                 mc2_update_timer_and_unlock(state);
                 local_irq_restore(flags);
         } else {
                 TRACE_TASK(tsk, "resume event ignored, still scheduled\n");
-                raw_spin_unlock_irqrestore(&state->lock, flags);
+                raw_spin_unlock_irqrestore(&_global_env.lock, flags);
         }
 
         resume_legacy_task_model_updates(tsk);
@@ -800,7 +879,9 @@ static long mc2_complete_job(void)
                 else
                         state = local_cpu_state();
                 
+                raw_spin_lock(&_global_env.lock);
                 raw_spin_lock(&state->lock);
+
                 env = &(state->sup_env.env);
                 
                 res = res_find_by_id(state, tinfo->mc2_param.res_id);
@@ -808,7 +889,6 @@ static long mc2_complete_job(void)
                 if (get_task_crit_level(current) < CRIT_LEVEL_C) {
                         env->time_zero = tsk_rt(current)->sporadic_release_time;
                 } else {
-                        raw_spin_lock(&_global_env.lock);
                         _global_env.env.time_zero = tsk_rt(current)->sporadic_release_time;
                 }
                 
@@ -826,10 +906,9 @@ static long mc2_complete_job(void)
                 res->env->change_state(res->env, res, RESERVATION_DEPLETED);
                 
                 TRACE_CUR("CHANGE NEXT_REP = %llu\n NEXT_UPDATE = %llu\n", res->next_replenishment, state->sup_env.next_scheduler_update);
-                if (get_task_crit_level(current) == CRIT_LEVEL_C) {
-                        raw_spin_unlock(&_global_env.lock);
-                }
+                
                 raw_spin_unlock(&state->lock);
+                raw_spin_unlock(&_global_env.lock);
                 
                 local_irq_restore(flags);
         }
@@ -908,10 +987,10 @@ static long mc2_admit_task(struct task_struct *tsk)
 
                 raw_spin_unlock_irqrestore(&state->lock, flags);
         } else if (lv == CRIT_LEVEL_C) {
-                local_irq_save(flags);
+                raw_spin_lock_irqsave(&_global_env.lock, flags);
                 state = local_cpu_state();
+                
                 raw_spin_lock(&state->lock);
-                raw_spin_lock(&_global_env.lock);
                 
                 res = gmp_find_by_id(&_global_env, mp->res_id);
 
@@ -931,9 +1010,8 @@ static long mc2_admit_task(struct task_struct *tsk)
                         tsk_rt(tsk)->task_params.budget_policy = NO_ENFORCEMENT;
                 }
 
-                raw_spin_unlock(&_global_env.lock);
                 raw_spin_unlock(&state->lock);
-                local_irq_restore(flags);
+                raw_spin_unlock_irqrestore(&_global_env.lock, flags);
         }
         
         preempt_enable();
@@ -965,8 +1043,9 @@ static void mc2_task_new(struct task_struct *tsk, int on_runqueue,
                 state = cpu_state_for(tinfo->cpu);
         
         /* acquire the lock protecting the state and disable interrupts */
-        raw_spin_lock_irqsave(&state->lock, flags);
-
+        raw_spin_lock_irqsave(&_global_env.lock, flags);
+        raw_spin_lock(&state->lock);
+        
         if (is_running) {
                 state->scheduled = tsk;
                 /* make sure this task should actually be running */
@@ -976,18 +1055,16 @@ static void mc2_task_new(struct task_struct *tsk, int on_runqueue,
         if (on_runqueue || is_running) {
                 /* Assumption: litmus_clock() is synchronized across cores
                  * [see comment in pres_task_resume()] */
-                raw_spin_lock(&_global_env.lock);
                 mc2_update_time(lv, state, litmus_clock());
                 mc2_update_ghost_state(state);
                 task_arrives(state, tsk);
                 /* NOTE: drops state->lock */
                 TRACE("mc2_new()\n");
-                raw_spin_unlock(&_global_env.lock);
                 
                 mc2_update_timer_and_unlock(state);
                 local_irq_restore(flags);
         } else
-                raw_spin_unlock_irqrestore(&state->lock, flags);
+                raw_spin_unlock_irqrestore(&_global_env.lock, flags);
 
         res = res_find_by_id(state, tinfo->mc2_param.res_id);
         
@@ -1015,9 +1092,9 @@ static long mc2_reservation_destroy(unsigned int reservation_id, int cpu)
                 /* if the reservation is global reservation */
                 local_irq_save(flags);
                 state = local_cpu_state();
-                raw_spin_lock(&state->lock);
                 raw_spin_lock(&_global_env.lock);
-        
+                raw_spin_lock(&state->lock);
+                
                 list_for_each_entry_safe(res, next, &_global_env.depleted_reservations, list) {
                         if (res->id == reservation_id) {
                                 TRACE("DESTROY RES FOUND!!!\n");
@@ -1050,8 +1127,8 @@ static long mc2_reservation_destroy(unsigned int reservation_id, int cpu)
                         }
                 }
 
-                raw_spin_unlock(&_global_env.lock);
                 raw_spin_unlock(&state->lock);
+                raw_spin_unlock(&_global_env.lock);
                 local_irq_restore(flags);
         } else {
                 /* if the reservation is partitioned reservation */
@@ -1125,9 +1202,10 @@ static void mc2_task_exit(struct task_struct *tsk)
                 state = cpu_state_for(tinfo->cpu);
         else
                 state = local_cpu_state();
+                
+        raw_spin_lock_irqsave(&_global_env.lock, flags);
+        raw_spin_lock(&state->lock);
         
-        raw_spin_lock_irqsave(&state->lock, flags);
-
         if (state->scheduled == tsk)
                 state->scheduled = NULL;
 
@@ -1140,20 +1218,18 @@ static void mc2_task_exit(struct task_struct *tsk)
                 /* Assumption: litmus_clock() is synchronized across cores
                  * [see comment in pres_task_resume()] */
                 
-                raw_spin_lock(&_global_env.lock);
                 /* update both global and partitioned */
                 mc2_update_time(lv, state, litmus_clock());
-                
                 mc2_update_ghost_state(state);
                 task_departs(tsk, 0);
                 
                 /* NOTE: drops state->lock */
                 TRACE("mc2_exit()\n");
-                raw_spin_unlock(&_global_env.lock);
+
                 mc2_update_timer_and_unlock(state);
                 local_irq_restore(flags);
         } else
-                raw_spin_unlock_irqrestore(&state->lock, flags);
+                raw_spin_unlock_irqrestore(&_global_env.lock, flags);
 
         kfree(tsk_rt(tsk)->plugin_state);
         tsk_rt(tsk)->plugin_state = NULL;
@@ -1204,8 +1280,10 @@ static long create_polling_reservation(
                 return -ENOMEM;
 
         if (config->cpu != -1) {
+                
+                raw_spin_lock_irqsave(&_global_env.lock, flags);
                 state = cpu_state_for(config->cpu);
-                raw_spin_lock_irqsave(&state->lock, flags);
+                raw_spin_lock(&state->lock);
 
                 res = sup_find_by_id(&state->sup_env, config->id);
                 if (!res) {
@@ -1225,7 +1303,9 @@ static long create_polling_reservation(
                         err = -EEXIST;
                 }
 
-                raw_spin_unlock_irqrestore(&state->lock, flags);
+                raw_spin_unlock(&state->lock);
+                raw_spin_unlock_irqrestore(&_global_env.lock, flags);
+
         } else {
                 raw_spin_lock_irqsave(&_global_env.lock, flags);
                 
@@ -1467,8 +1547,8 @@ static long mc2_activate_plugin(void)
                 hrtimer_init(&state->timer, CLOCK_MONOTONIC, HRTIMER_MODE_ABS_PINNED);
                 state->timer.function = on_scheduling_timer;
                 
-                hrtimer_init(&state->g_timer, CLOCK_MONOTONIC, HRTIMER_MODE_ABS_PINNED);
-                state->g_timer.function = on_global_scheduling_timer;
+//              hrtimer_init(&state->g_timer, CLOCK_MONOTONIC, HRTIMER_MODE_ABS_PINNED);
+//              state->g_timer.function = on_global_scheduling_timer;
         }
 
         mc2_setup_domain_proc();
@@ -1481,6 +1561,7 @@ static void mc2_finish_switch(struct task_struct *prev)
         struct mc2_cpu_state *state = local_cpu_state();
         
         state->scheduled = is_realtime(current) ? current : NULL;
+        TRACE_TASK(prev, "FINISH CXS! complete=%d\n", tsk_rt(prev)->completed);
 }
 
 static long mc2_deactivate_plugin(void)
@@ -1490,12 +1571,14 @@ static long mc2_deactivate_plugin(void)
         struct reservation *res;
         struct next_timer_event *event;
 
+        raw_spin_lock(&_global_env.lock);
+
         for_each_online_cpu(cpu) {
                 state = cpu_state_for(cpu);
                 raw_spin_lock(&state->lock);
 
                 hrtimer_cancel(&state->timer);
-                hrtimer_cancel(&state->g_timer);
+//              hrtimer_cancel(&state->g_timer);
 
                 /* Delete all reservations --- assumes struct reservation
                  * is prefix of containing struct. */
@@ -1527,8 +1610,7 @@ static long mc2_deactivate_plugin(void)
                 raw_spin_unlock(&state->lock);
         }
 
-        raw_spin_lock(&_global_env.lock);
-
+        
         while (!list_empty(&_global_env.active_reservations)) {
                 TRACE("RES FOUND!!!\n");
                 res = list_first_entry(