Working version

4 files changed, 187 insertions, 278 deletions

diff --git a/kernel/sched/litmus.c b/kernel/sched/litmus.c
index b84361f03f60..29cd69d5aa0c 100644
--- a/kernel/sched/litmus.c
+++ b/kernel/sched/litmus.c
@@ -73,7 +73,7 @@ litmus_schedule(struct rq *rq, struct task_struct *prev)
                         if (next->rt_param.stack_in_use == NO_CPU)
                                 TRACE_TASK(next,"descheduled. Proceeding.\n");
 
-                        if (lt_before(_maybe_deadlock + 1000000000L,
+                        if (lt_before(_maybe_deadlock + 5000000000L,
                                       litmus_clock())) {
                                 /* We've been spinning for 1s.
                                  * Something can't be right!
diff --git a/litmus/polling_reservations.c b/litmus/polling_reservations.c
index a3125ebf5273..df1aeb0a113f 100644
--- a/litmus/polling_reservations.c
+++ b/litmus/polling_reservations.c
@@ -383,11 +383,7 @@ static lt_t td_time_remaining_until_end(struct table_driven_reservation *tdres)
 {
         lt_t now = tdres->res.env->current_time;
         lt_t end = tdres->cur_interval.end;
-        TRACE("td_remaining(%u): start=%llu now=%llu end=%llu state=%d\n",
-                tdres->res.id,
-                tdres->cur_interval.start,
-                now, end,
-                tdres->res.state);
+        //TRACE("td_remaining(%u): start=%llu now=%llu end=%llu state=%d\n", tdres->res.id,     tdres->cur_interval.start, now, end, tdres->res.state);
         if (now >=  end)
                 return 0;
         else
@@ -400,24 +396,22 @@ static void td_replenish(
         struct table_driven_reservation *tdres =
                 container_of(res, struct table_driven_reservation, res);
 
-        TRACE("td_replenish(%u): expected_replenishment=%llu\n", res->id,
-                res->next_replenishment);
+        //TRACE("td_replenish(%u): expected_replenishment=%llu\n", res->id, res->next_replenishment);
 
         /* figure out current interval */
         tdres->cur_interval.start = tdres->major_cycle_start +
                 tdres->intervals[tdres->next_interval].start;
         tdres->cur_interval.end =  tdres->major_cycle_start +
                 tdres->intervals[tdres->next_interval].end;
-        TRACE("major_cycle_start=%llu => [%llu, %llu]\n",
+/*      TRACE("major_cycle_start=%llu => [%llu, %llu]\n",
                 tdres->major_cycle_start,
                 tdres->cur_interval.start,
                 tdres->cur_interval.end);
-
+*/
         /* reset budget */
         res->cur_budget = td_time_remaining_until_end(tdres);
         res->budget_consumed = 0;
-        TRACE("td_replenish(%u): %s budget=%llu\n", res->id,
-                res->cur_budget ? "" : "WARNING", res->cur_budget);
+        //TRACE("td_replenish(%u): %s budget=%llu\n", res->id, res->cur_budget ? "" : "WARNING", res->cur_budget);
 
         /* prepare next slot */
         tdres->next_interval = (tdres->next_interval + 1) % tdres->num_intervals;
@@ -428,8 +422,7 @@ static void td_replenish(
         /* determine next time this reservation becomes eligible to execute */
         res->next_replenishment  = tdres->major_cycle_start;
         res->next_replenishment += tdres->intervals[tdres->next_interval].start;
-        TRACE("td_replenish(%u): next_replenishment=%llu\n", res->id,
-                res->next_replenishment);
+        //TRACE("td_replenish(%u): next_replenishment=%llu\n", res->id, res->next_replenishment);
 
 
         switch (res->state) {
@@ -465,7 +458,7 @@ static void td_drain_budget(
          * how much time is left in this allocation interval. */
 
         /* sanity check: we should never try to drain from future slots */
-        TRACE("TD_DRAIN STATE(%d) [%llu,%llu]  %llu ?\n", res->state, tdres->cur_interval.start, tdres->cur_interval.end, res->env->current_time);
+        //TRACE("TD_DRAIN STATE(%d) [%llu,%llu]  %llu ?\n", res->state, tdres->cur_interval.start, tdres->cur_interval.end, res->env->current_time);
         //BUG_ON(tdres->cur_interval.start > res->env->current_time);
         if (tdres->cur_interval.start > res->env->current_time)
                 TRACE("TD_DRAIN BUG!!!!!!!!!!\n");
@@ -480,8 +473,7 @@ static void td_drain_budget(
                 case RESERVATION_ACTIVE_IDLE:
                 case RESERVATION_ACTIVE:
                         res->cur_budget = td_time_remaining_until_end(tdres);
-                        TRACE("td_drain_budget(%u): drained to budget=%llu\n",
-                                res->id, res->cur_budget);
+                        //TRACE("td_drain_budget(%u): drained to budget=%llu\n", res->id, res->cur_budget);
                         if (!res->cur_budget) {
                                 res->env->change_state(res->env, res,
                                         RESERVATION_DEPLETED);
@@ -489,6 +481,10 @@ static void td_drain_budget(
                                 /* sanity check budget calculation */
                                 //BUG_ON(res->env->current_time >= tdres->cur_interval.end);
                                 //BUG_ON(res->env->current_time < tdres->cur_interval.start);
+                                if (res->env->current_time >= tdres->cur_interval.end)
+                                        printk(KERN_ALERT "TD_DRAIN_BUDGET WARNING1\n");
+                                if (res->env->current_time < tdres->cur_interval.start)
+                                        printk(KERN_ALERT "TD_DRAIN_BUDGET WARNING2\n");
                         }
 
                         break;
diff --git a/litmus/reservation.c b/litmus/reservation.c
index 3ec18a23f588..86d2f6e65382 100644
--- a/litmus/reservation.c
+++ b/litmus/reservation.c
@@ -4,6 +4,9 @@
 #include <litmus/litmus.h>
 #include <litmus/reservation.h>
 
+#define TRACE(fmt, args...) do {} while (false)
+#define TRACE_TASK(fmt, args...) do {} while (false)
+
 void reservation_init(struct reservation *res)
 {
         memset(res, sizeof(*res), 0);
@@ -376,43 +379,6 @@ struct next_timer_event* gmp_find_event_by_time(struct gmp_reservation_environme
         return NULL;
 }
 
-/*
-static void gmp_scheduler_update_at(
-        struct gmp_reservation_environment* gmp_env, unsigned int id,
-        event_type_t type, lt_t when)
-{
-        struct next_timer_event *nevent, *queued;
-        struct list_head *pos;
-        int found = 0;
-        
-        nevent = gmp_find_event_by_id(gmp_env, id);
-        
-        if (!nevent) {
-                nevent = kzalloc(sizeof(*nevent), GFP_KERNEL);
-                nevent->next_update = when;
-                nevent->id = id;
-                nevent->timer_armed_on = NO_CPU;
-                nevent->type = type;
-                
-                list_for_each(pos, &gmp_env->next_events) {
-                        queued = list_entry(pos, struct next_timer_event, list);
-                        if (queued->next_update > nevent->next_update) {
-                                list_add(&nevent->list, pos->prev);
-                                found = 1;
-                                TRACE("NEXT_EVENT ADDED after %llu\n", queued->next_update);
-                                break;
-                        }
-                }
-                
-                if (!found) {
-                        list_add_tail(&nevent->list, &gmp_env->next_events);
-                        TRACE("NEXT_EVENT ADDED at [0]\n");
-                }
-        } else {
-                TRACE("EVENT FOUND at %llu T(%d), NEW EVENT %llu T(%d)\n", nevent->next_update, nevent->type, when, type);
-        }
-}
-*/
 #define TIMER_RESOLUTION 100000L
 
 static void gmp_add_event(
@@ -425,11 +391,12 @@ static void gmp_add_event(
 
         //when = div64_u64(when, TIMER_RESOLUTION);
         //when *= TIMER_RESOLUTION;
-        
+//printk(KERN_ALERT "GMP_ADD id=%d type=%d when=%llu\n", id, type, when);
         nevent = gmp_find_event_by_id(gmp_env, id);
         
         if (!nevent || nevent->type != type) {
                 nevent = kzalloc(sizeof(*nevent), GFP_ATOMIC);
+                BUG_ON(!nevent);
                 nevent->next_update = when;
                 nevent->id = id;
                 nevent->type = type;
@@ -450,17 +417,19 @@ static void gmp_add_event(
                         TRACE("NEXT_EVENT id=%d type=%d update=%llu ADDED at TAIL\n", nevent->id, nevent->type, nevent->next_update);
                 }
         } else {
-                TRACE("EVENT FOUND id = %d type=%d when=%llu, NEW EVENT type=%d when=%llu\n", nevent->id, nevent->type, nevent->next_update, type, when);
+                //TRACE("EVENT FOUND id = %d type=%d when=%llu, NEW EVENT type=%d when=%llu\n", nevent->id, nevent->type, nevent->next_update, type, when);
+; //printk(KERN_ALERT "EVENT FOUND id = %d type=%d when=%llu, NEW EVENT type=%d when=%llu\n", nevent->id, nevent->type, nevent->next_update, type, when);
         }
         
-        TRACE("======START PRINTING EVENT LIST======\n");
-        gmp_print_events(gmp_env, litmus_clock());
-        TRACE("======FINISH PRINTING EVENT LIST======\n");
+        //TRACE("======START PRINTING EVENT LIST======\n");
+        //gmp_print_events(gmp_env, litmus_clock());
+        //TRACE("======FINISH PRINTING EVENT LIST======\n");
 }
 
 void gmp_add_event_after(
         struct gmp_reservation_environment* gmp_env, lt_t timeout, unsigned int id, event_type_t type)
 {
+        //printk(KERN_ALERT "ADD_EVENT_AFTER id = %d\n", id);
         gmp_add_event(gmp_env, gmp_env->env.current_time + timeout, id, type);
 }
 
@@ -472,19 +441,24 @@ static void gmp_queue_depleted(
         struct reservation *queued;
         int found = 0;
 
+//printk(KERN_ALERT "R%d request to enqueue depleted_list\n", res->id);
+        
         list_for_each(pos, &gmp_env->depleted_reservations) {
                 queued = list_entry(pos, struct reservation, list);
-                if (queued && queued->next_replenishment > res->next_replenishment) {
+                if (queued && (queued->next_replenishment > res->next_replenishment)) {
+//printk(KERN_ALERT "QUEUED R%d %llu\n", queued->id, queued->next_replenishment);
                         list_add(&res->list, pos->prev);
                         found = 1;
+                        break;
                 }
         }
 
         if (!found)
                 list_add_tail(&res->list, &gmp_env->depleted_reservations);
+
         TRACE("R%d queued to depleted_list\n", res->id);
+//printk(KERN_ALERT "R%d queued to depleted_list\n", res->id);
         gmp_add_event(gmp_env, res->next_replenishment, res->id, EVENT_REPLENISH);
-        res->event_added = 1;
 }
 
 static void gmp_queue_active(
@@ -520,10 +494,8 @@ static void gmp_queue_reservation(
         struct gmp_reservation_environment* gmp_env,
         struct reservation *res)
 {
-        if (res == NULL) {
-                BUG();
-                return;
-        }
+
+//printk(KERN_ALERT "DEBUG: Passed %s %d %p R%d STATE %d\n",__FUNCTION__,__LINE__, gmp_env, res->id, res->state);
         switch (res->state) {
                 case RESERVATION_INACTIVE:
                         list_add(&res->list, &gmp_env->inactive_reservations);
@@ -584,8 +556,7 @@ static void gmp_charge_budget(
                 {
                         /* make sure scheduler is invoked when this reservation expires
                          * its remaining budget */
-                         TRACE("requesting gmp_scheduler update for reservation %u in %llu nanoseconds\n",
-                                res->id, res->cur_budget);
+                         TRACE("requesting gmp_scheduler update for reservation %u in %llu nanoseconds\n", res->id, res->cur_budget);
                          gmp_add_event_after(gmp_env, res->cur_budget, res->id, EVENT_DRAIN);
                          res->event_added = 1;
                 }
@@ -593,7 +564,7 @@ static void gmp_charge_budget(
                         /* stop at the first ACTIVE reservation */
                 //      break;
         }
-        //TRACE("finished charging budgets\n");
+        TRACE("finished charging budgets\n");
 }
 
 static void gmp_replenish_budgets(struct gmp_reservation_environment* gmp_env)
@@ -611,24 +582,6 @@ static void gmp_replenish_budgets(struct gmp_reservation_environment* gmp_env)
                 }
         }
         TRACE("finished replenishing budgets\n");
-
-        /* request a scheduler update at the next replenishment instant */
-        list_for_each_safe(pos, next, &gmp_env->depleted_reservations) {
-                res = list_entry(pos, struct reservation, list);
-                if (res->event_added == 0) {
-                        gmp_add_event(gmp_env, res->next_replenishment, res->id, EVENT_REPLENISH);
-                        res->event_added = 1;
-                }
-        }       
-        
-/*
-        res = list_first_entry_or_null(&gmp_env->depleted_reservations,
-                struct reservation, list);
-        if (res && res->event_added == 0) {
-                gmp_add_event(gmp_env, res->next_replenishment, res->id, EVENT_REPLENISH);
-                res->event_added = 1;
-        }
-*/
 }
 
 #define EPSILON 50
@@ -644,7 +597,7 @@ int gmp_update_time(
         /* If the time didn't advance, there is nothing to do.
          * This check makes it safe to call sup_advance_time() potentially
          * multiple times (e.g., via different code paths. */
-        TRACE("(gmp_update_time) now: %llu, current_time: %llu\n", now, gmp_env->env.current_time);
+        //TRACE("(gmp_update_time) now: %llu, current_time: %llu\n", now, gmp_env->env.current_time);
         if (unlikely(now <= gmp_env->env.current_time + EPSILON))
                 return 0;
 
@@ -660,12 +613,15 @@ int gmp_update_time(
         /* check if any budgets where replenished */
         //TRACE("REPLENISH###\n");
         gmp_replenish_budgets(gmp_env);
+
         
         list_for_each_entry_safe(event, next, &gmp_env->next_events, list) {
                 if (event->next_update < now) {
                         list_del(&event->list);
                         TRACE("EVENT at %llu IS DELETED\n", event->next_update);
                         kfree(event);
+                } else {
+                        break;
                 }
         }               
         
@@ -673,7 +629,7 @@ int gmp_update_time(
         
         ret = min(gmp_env->schedule_now, NR_CPUS);
         gmp_env->schedule_now = 0;
-        
+
         return ret;
 }
 
diff --git a/litmus/sched_mc2.c b/litmus/sched_mc2.c
index b3390dc87d47..f7758f2aff58 100644
--- a/litmus/sched_mc2.c
+++ b/litmus/sched_mc2.c
@@ -62,7 +62,6 @@ struct mc2_task_state {
 struct crit_entry {
         enum crit_level level;
         struct task_struct *running;
-        //struct hrtimer ghost_timer;
 };
 
 /* mc2_cpu_state - maintain the scheduled state and ghost jobs
@@ -74,7 +73,6 @@ struct mc2_cpu_state {
 
         struct sup_reservation_environment sup_env;
         struct hrtimer timer;
-        //struct hrtimer g_timer;
 
         int cpu;
         struct task_struct* scheduled;
@@ -221,7 +219,9 @@ static int get_lowest_prio_cpu(lt_t priority)
                 raw_spin_unlock(&_lowest_prio_cpu.lock);
                 TRACE("CPU %d (local) is the lowest!\n", ce->cpu);
                 return ce->cpu;
-        }       
+        } else {
+                TRACE("Local CPU will_schedule=%d, scheduled=(%s/%d)\n", ce->will_schedule, ce->scheduled ? (ce->scheduled)->comm : "null", ce->scheduled ? (ce->scheduled)->pid : 0);
+        }
 
         for_each_online_cpu(cpu) {
                 ce = &_lowest_prio_cpu.cpu_entries[cpu];
@@ -287,6 +287,10 @@ static void mc2_update_timer_and_unlock(struct mc2_cpu_state *state)
         list_for_each_entry_safe(event, next, &_global_env.next_events, list) {
                 /* If the event time is already passed, we call schedule() on
                    the lowest priority cpu */
+                if (event->next_update >= update) {
+                        break;
+                }
+                
                 if (event->next_update < litmus_clock()) {
                         if (event->timer_armed_on == NO_CPU) {
                                 struct reservation *res = gmp_find_by_id(&_global_env, event->id);
@@ -351,70 +355,33 @@ static void mc2_update_timer_and_unlock(struct mc2_cpu_state *state)
                         litmus_reschedule(state->cpu);
                 }
         }
-
-#if 0   
-        raw_spin_lock(&_global_env.lock);
-        list_for_each_entry_safe(event, next, &_global_env.next_events, list) {
-                if (event->timer_armed_on == NO_CPU) {
-                        /* If the event time is already passed, we call schedule() on
-                           the lowest priority cpu */
-                        if (event->next_update < litmus_clock()) {
-                                int cpu = get_lowest_prio_cpu();
-                                TRACE("GLOBAL EVENT PASSED!! poking CPU %d to reschedule\n", cpu);
-                                list_del(&event->list);
-                                kfree(event);
-                                if (cpu != NO_CPU) {
-                                        raw_spin_lock(&_lowest_prio_cpu.lock);
-                                        _lowest_prio_cpu.cpu_entries[cpu].will_schedule = true;
-                                        raw_spin_unlock(&_lowest_prio_cpu.lock);
-                                        litmus_reschedule(cpu);
-                                }
-                        } else if (!hrtimer_active(&state->g_timer)) {
-                                int ret;
-                        
-                                raw_spin_unlock(&_global_env.lock);
-                                found_event = 1;
-                        
-                                TRACE("setting global scheduler timer for %llu\n", 
-                                       event->next_update);
-                                ret = __hrtimer_start_range_ns(&state->g_timer,
-                                                ns_to_ktime(event->next_update),
-                                                0 /* timer coalescing slack */,
-                                                HRTIMER_MODE_ABS_PINNED,
-                                                0 /* wakeup */);
-                                if (!ret) {
-                                        event->timer_armed_on = state->cpu;
-                                        break;
-                                }
-                        }                               
-                }
-        }
-        if (found_event == 0)
-                raw_spin_unlock(&_global_env.lock);
-#endif  
 }
 
 /* mc2_update_ghost_state - Update crit_entries[] to track ghost jobs
  *                          If the budget of a ghost is exhausted,
  *                          clear is_ghost and reschedule
  */
-static void mc2_update_ghost_state(struct mc2_cpu_state *state)
+static lt_t mc2_update_ghost_state(struct mc2_cpu_state *state)
 {
         int lv = 0;
         struct crit_entry* ce;
         struct reservation *res;
         struct mc2_task_state *tinfo;
-
+        lt_t ret = ULLONG_MAX;
+        
         BUG_ON(!state);
         
         for (lv = 0; lv < NUM_CRIT_LEVELS; lv++) {
                 ce = &state->crit_entries[lv];
                 if (ce->running != NULL) {
+//printk(KERN_ALERT "P%d ce->running : %s/%d\n", state->cpu,  ce->running ? (ce->running)->comm : "null", ce->running ? (ce->running)->pid : 0);
                         tinfo = get_mc2_state(ce->running);
                         if (!tinfo)
                                 continue;
                         
                         res = res_find_by_id(state, tinfo->mc2_param.res_id);
+                        BUG_ON(!res);
+//printk(KERN_ALERT "R%d found!\n", res->id);                   
                         TRACE("LV %d running id %d budget %llu\n", 
                                lv, tinfo->mc2_param.res_id, res->cur_budget);
                         /* If the budget is exhausted, clear is_ghost and reschedule */
@@ -432,7 +399,7 @@ static void mc2_update_ghost_state(struct mc2_cpu_state *state)
                                                   struct reservation, list);
                                         if (res)
                                                 litmus_reschedule_local();
-                                } else {
+                                } else if (lv == CRIT_LEVEL_C) {
                                         res = list_first_entry_or_null(
                                               &_global_env.active_reservations,
                                                   struct reservation, list);
@@ -440,11 +407,16 @@ static void mc2_update_ghost_state(struct mc2_cpu_state *state)
                                                 litmus_reschedule(state->cpu);
                                 }
                         } else {
-                                TRACE("GHOST NOT FINISH id %d budget %llu\n", res->id, res->cur_budget);
-                                gmp_add_event_after(&_global_env, res->cur_budget, res->id, EVENT_DRAIN);
+                                //TRACE("GHOST NOT FINISH id %d budget %llu\n", res->id, res->cur_budget);
+                                //gmp_add_event_after(&_global_env, res->cur_budget, res->id, EVENT_DRAIN);
+                                if (ret > res->cur_budget) {
+                                        ret = res->cur_budget;
+                                }
                         }
                 }
         }
+        
+        return ret;
 }                       
 
 /* update_cpu_prio - Update cpu's priority
@@ -474,84 +446,6 @@ static void update_cpu_prio(struct mc2_cpu_state *state)
         }
 };
 
-#if 0
-/* on_global_scheduling_timer - Process the budget accounting (replenish
- *                              and charge)
- */                                                             
-static enum hrtimer_restart on_global_scheduling_timer(struct hrtimer *timer)
-{
-        unsigned long flags;
-        enum hrtimer_restart restart = HRTIMER_NORESTART;
-        struct mc2_cpu_state *state;
-        struct next_timer_event *event, *next;
-        int schedule_now;
-        lt_t update, now;
-        int found_event = 0;
-
-        state = container_of(timer, struct mc2_cpu_state, g_timer);
-
-        raw_spin_lock_irqsave(&state->lock, flags);
-        
-        /* The scheduling timer should only fire on the local CPU, because
-         * otherwise deadlocks via timer_cancel() are possible.
-         * Note: this does not interfere with dedicated interrupt handling, as
-         * even under dedicated interrupt handling scheduling timers for
-         * budget enforcement must occur locally on each CPU.
-         */
-        //BUG_ON(state->cpu != raw_smp_processor_id());
-        if (state->cpu != raw_smp_processor_id())
-                TRACE("BUG!!!!!!!!!!!!! TIMER FIRED ON THE OTHER CPU\n");
-
-        raw_spin_lock(&_global_env.lock);
-        
-        update = litmus_clock();
-        TRACE("GLOBAL TIMER FIRED at %llu\n", update);
-        
-        /* The event can be processed by the other cpus. So, if there is no 
-           events to process, we do nothing */
-        list_for_each_entry_safe(event, next, &_global_env.next_events, list) {
-                if (event->next_update < update) {
-                        found_event = 1;
-                        list_del(&event->list);
-                        TRACE("EVENT at %llu IS DELETED\n", event->next_update);
-                        kfree(event);
-                }
-        }                       
-        
-        if (!found_event) {
-                goto unlock;
-        }
-        
-        /* gmp_update_timer returns how many tasks become ACTIVE */
-        schedule_now = gmp_update_time(&_global_env, update);
-        
-        mc2_update_ghost_state(state);
-        
-        now = _global_env.env.current_time;
-        
-        TRACE_CUR("on_global_scheduling_timer at %llu, upd:%llu (for cpu=%d) SCHEDULE_NOW = %d\n",
-                now, update, state->cpu, schedule_now);
-
-        /* Find the lowest cpu, and call reschedule */
-        while (schedule_now--) {
-                int cpu = get_lowest_prio_cpu();
-                if (cpu != NO_CPU) {
-                        raw_spin_lock(&_lowest_prio_cpu.lock);
-                        _lowest_prio_cpu.cpu_entries[cpu].will_schedule = true;
-                        raw_spin_unlock(&_lowest_prio_cpu.lock);
-                        TRACE("LOWEST CPU = P%d\n", cpu);
-                        litmus_reschedule(cpu);
-                }
-        } 
-
-unlock:
-        raw_spin_unlock(&_global_env.lock);
-        raw_spin_unlock_irqrestore(&state->lock, flags);
-        
-        return restart;
-}
-#endif
-
 /* on_scheduling_timer - timer event for partitioned tasks
  */                       
 static enum hrtimer_restart on_scheduling_timer(struct hrtimer *timer)
@@ -561,7 +455,8 @@ static enum hrtimer_restart on_scheduling_timer(struct hrtimer *timer)
         struct mc2_cpu_state *state;
         lt_t update, now;
         int global_schedule_now;
-
+        lt_t remain_budget;
+        
         state = container_of(timer, struct mc2_cpu_state, timer);
 
         /* The scheduling timer should only fire on the local CPU, because
@@ -575,18 +470,22 @@ static enum hrtimer_restart on_scheduling_timer(struct hrtimer *timer)
         TRACE("TIMER FIRED at %llu\n", litmus_clock());
         raw_spin_lock_irqsave(&_global_env.lock, flags);
         raw_spin_lock(&state->lock);
-        
-        sup_update_time(&state->sup_env, litmus_clock());
-        global_schedule_now = gmp_update_time(&_global_env, litmus_clock());
-        
-        mc2_update_ghost_state(state);
+//printk(KERN_ALERT "P%d on_scheduling_timer() hold lock %s/%d\n", state->cpu, current ? (current)->comm : "null", current ? (current)->pid : 0);                       
+        now = litmus_clock();
+        sup_update_time(&state->sup_env, now);
+        global_schedule_now = gmp_update_time(&_global_env, now);
+//printk(KERN_ALERT "P%d update_time in timer() %s/%d\n", state->cpu, current ? (current)->comm : "null", current ? (current)->pid : 0);                        
+        remain_budget = mc2_update_ghost_state(state);
         
         update = state->sup_env.next_scheduler_update;
         now = state->sup_env.env.current_time;
 
-        TRACE_CUR("on_scheduling_timer at %llu, upd:%llu (for cpu=%d) g_schedule_now:%d\n",
-                now, update, state->cpu, global_schedule_now);
-
+        if (remain_budget != ULLONG_MAX && update > now + remain_budget) {
+                update = now + remain_budget;
+        }
+        
+        //TRACE_CUR("on_scheduling_timer at %llu, upd:%llu (for cpu=%d) g_schedule_now:%d\n", now, update, state->cpu, global_schedule_now);
+//printk(KERN_ALERT "on_scheduling_timer at %llu, upd:%llu (for cpu=%d) g_schedule_now:%d\n", now, update, state->cpu, global_schedule_now);
         if (update <= now) {
                 litmus_reschedule_local();
         } else if (update != SUP_NO_SCHEDULER_UPDATE) {
@@ -594,6 +493,8 @@ static enum hrtimer_restart on_scheduling_timer(struct hrtimer *timer)
                 restart = HRTIMER_RESTART;
         }
 
+        BUG_ON(global_schedule_now < 0 || global_schedule_now > 4);
+        
         /* Find the lowest cpu, and call reschedule */
         while (global_schedule_now--) {
                 int cpu = get_lowest_prio_cpu(0);
@@ -601,14 +502,14 @@ static enum hrtimer_restart on_scheduling_timer(struct hrtimer *timer)
                         raw_spin_lock(&_lowest_prio_cpu.lock);
                         _lowest_prio_cpu.cpu_entries[cpu].will_schedule = true;
                         raw_spin_unlock(&_lowest_prio_cpu.lock);
-                        TRACE("LOWEST CPU = P%d\n", cpu);
+                        //TRACE("LOWEST CPU = P%d\n", cpu);
                         litmus_reschedule(cpu);
                 }
         } 
         
         raw_spin_unlock(&state->lock);
         raw_spin_unlock_irqrestore(&_global_env.lock, flags);
-
+//printk(KERN_ALERT "P%d on_scheduling_timer() release lock %s/%d\n", state->cpu, current ? (current)->comm : "null", current ? (current)->pid : 0);    
         return restart;
 }
 
@@ -651,6 +552,13 @@ struct task_struct* mc2_dispatch(struct sup_reservation_environment* sup_env, st
         }
         
         /* no eligible level A or B tasks exists */
+        /* check the ghost job */
+        ce = &state->crit_entries[CRIT_LEVEL_C];
+        if (ce->running) {
+                TRACE_TASK(ce->running," is GHOST\n");
+                return NULL;
+        }
+        
         cur_priority = _lowest_prio_cpu.cpu_entries[state->cpu].deadline;
         
         TRACE("****** ACTIVE LIST ******\n");
@@ -663,21 +571,24 @@ struct task_struct* mc2_dispatch(struct sup_reservation_environment* sup_env, st
                                 lv = get_task_crit_level(tsk);
                                 if (lv == NUM_CRIT_LEVELS) {
                                         gmp_add_event_after(&_global_env, res->cur_budget, res->id, EVENT_DRAIN);
-                                        res->event_added = 1;
+                                        //res->event_added = 1;
                                         return tsk;
-                                } else {
-                                        ce = &state->crit_entries[lv];
-                                        if (likely(!ce->running)) {
+                                } else if (lv == CRIT_LEVEL_C) {
+                                        //ce = &state->crit_entries[lv];
+                                        //if (likely(!ce->running)) {
                                                 gmp_add_event_after(&_global_env, res->cur_budget, res->id, EVENT_DRAIN);
                                                 res->event_added = 1;
                                                 res->blocked_by_ghost = 0;
                                                 res->is_ghost = 0;
                                                 res->scheduled_on = state->cpu;
                                                 return tsk;
-                                        } else {
-                                                res->blocked_by_ghost = 1;
-                                                TRACE_TASK(ce->running, " is GHOST\n");
-                                        }
+                                        //} else {
+                                        //      res->blocked_by_ghost = 1;
+                                        //      TRACE_TASK(ce->running, " is GHOST\n");
+                                        //      return NULL;
+                                        //}
+                                } else {
+                                        BUG();
                                 }
                         }
                 }
@@ -711,8 +622,9 @@ static void post_schedule(struct task_struct *next)
 static struct task_struct* mc2_schedule(struct task_struct * prev)
 {
         /* next == NULL means "schedule background work". */
+        lt_t now;
         struct mc2_cpu_state *state = local_cpu_state();
-        
+
         pre_schedule(prev);
         
         raw_spin_lock(&_global_env.lock);
@@ -721,17 +633,18 @@ static struct task_struct* mc2_schedule(struct task_struct * prev)
         //BUG_ON(state->scheduled && state->scheduled != prev);
         //BUG_ON(state->scheduled && !is_realtime(prev));
         if (state->scheduled && state->scheduled != prev)
-                TRACE("BUG1!!!!!!!!\n");
+                printk(KERN_ALERT "BUG1!!!!!!!! %s %s\n", state->scheduled ? (state->scheduled)->comm : "null", prev ? (prev)->comm : "null");
         if (state->scheduled && !is_realtime(prev))
-                TRACE("BUG2!!!!!!!!\n");
+                printk(KERN_ALERT "BUG2!!!!!!!! \n");
 
         /* update time */
         state->sup_env.will_schedule = true;
 
-        sup_update_time(&state->sup_env, litmus_clock());
-        gmp_update_time(&_global_env, litmus_clock());
-        
-        mc2_update_ghost_state(state);
+        now = litmus_clock();
+        sup_update_time(&state->sup_env, now);
+        gmp_update_time(&_global_env, now);
+                
+        mc2_update_ghost_state(state);  
         
         /* remove task from reservation if it blocks */
         if (is_realtime(prev) && !is_running(prev))
@@ -767,14 +680,17 @@ static struct task_struct* mc2_schedule(struct task_struct * prev)
                 TRACE_TASK(prev, "descheduled.\n");
                 /* if prev is preempted and a global task, find the lowest cpu and reschedule */
                 if (tinfo->has_departed == false && get_task_crit_level(prev) == CRIT_LEVEL_C) {
-                        int cpu = get_lowest_prio_cpu(res?res->priority:0);
-                        TRACE("LEVEL-C TASK PREEMPTED!! poking CPU %d to reschedule\n", cpu);
+                        int cpu;
+                        raw_spin_lock(&_global_env.lock);
+                        cpu = get_lowest_prio_cpu(res?res->priority:0);
+                        //TRACE("LEVEL-C TASK PREEMPTED!! poking CPU %d to reschedule\n", cpu);
                         if (cpu != NO_CPU) {
                                 raw_spin_lock(&_lowest_prio_cpu.lock);
                                 _lowest_prio_cpu.cpu_entries[cpu].will_schedule = true;
                                 raw_spin_unlock(&_lowest_prio_cpu.lock);
                                 litmus_reschedule(cpu);
                         }
+                        raw_spin_unlock(&_global_env.lock);
                 }
         }
         if (state->scheduled) {
@@ -814,12 +730,14 @@ static void mc2_task_resume(struct task_struct  *tsk)
 
         TRACE_TASK(tsk, "thread wakes up at %llu\n", litmus_clock());
 
+        local_irq_save(flags);
         if (tinfo->cpu != -1)
                 state = cpu_state_for(tinfo->cpu);
         else
                 state = local_cpu_state();
 
-        raw_spin_lock_irqsave(&_global_env.lock, flags);
+        raw_spin_lock(&_global_env.lock);
+//printk(KERN_ALERT "P%d resume() hold lock\n", state->cpu);    
         /* Requeue only if self-suspension was already processed. */
         if (tinfo->has_departed)
         {
@@ -830,22 +748,25 @@ static void mc2_task_resume(struct task_struct  *tsk)
                 if (tinfo->cpu != -1) {
                         sup_update_time(&state->sup_env, litmus_clock());
                 } else {
-                        TRACE("RESUME UPDATE ####\n");
+                        //TRACE("RESUME UPDATE ####\n");
                         gmp_update_time(&_global_env, litmus_clock());
-                        TRACE("RESUME UPDATE $$$$\n");
+                        //TRACE("RESUME UPDATE $$$$\n");
                 }
                         
                 mc2_update_ghost_state(state);
                 task_arrives(state, tsk);
                 /* NOTE: drops state->lock */
                 TRACE_TASK(tsk, "mc2_resume()\n");
-                mc2_update_timer_and_unlock(state);
-                local_irq_restore(flags);
+                mc2_update_timer_and_unlock(state);     
+//printk(KERN_ALERT "P%d resume() dropped lock\n", state->cpu);                 
         } else {
                 TRACE_TASK(tsk, "resume event ignored, still scheduled\n");
-                raw_spin_unlock_irqrestore(&_global_env.lock, flags);
+                raw_spin_unlock(&_global_env.lock);
+//printk(KERN_ALERT "P%d resume() release lock\n", state->cpu);                 
         }
 
+        local_irq_restore(flags);
+        
         resume_legacy_task_model_updates(tsk);
 }
 
@@ -870,6 +791,7 @@ static long mc2_complete_job(void)
                 struct reservation *res;
                 unsigned long flags;
 
+                preempt_disable();
                 local_irq_save(flags);
                 
                 tinfo = get_mc2_state(current);
@@ -881,7 +803,7 @@ static long mc2_complete_job(void)
                 
                 raw_spin_lock(&_global_env.lock);
                 raw_spin_lock(&state->lock);
-
+//printk(KERN_ALERT "P%d complete() hold lock\n", state->cpu);
                 env = &(state->sup_env.env);
                 
                 res = res_find_by_id(state, tinfo->mc2_param.res_id);
@@ -905,12 +827,13 @@ static long mc2_complete_job(void)
                 res->cur_budget = 0;
                 res->env->change_state(res->env, res, RESERVATION_DEPLETED);
                 
-                TRACE_CUR("CHANGE NEXT_REP = %llu\n NEXT_UPDATE = %llu\n", res->next_replenishment, state->sup_env.next_scheduler_update);
+                //TRACE_CUR("CHANGE NEXT_REP = %llu\n NEXT_UPDATE = %llu\n", res->next_replenishment, state->sup_env.next_scheduler_update);
                 
                 raw_spin_unlock(&state->lock);
                 raw_spin_unlock(&_global_env.lock);
-                
+//printk(KERN_ALERT "P%d complete() release lock\n", state->cpu);                               
                 local_irq_restore(flags);
+                preempt_enable();
         }
         sched_trace_task_completion(current, 0);
         
@@ -937,6 +860,7 @@ static long mc2_complete_job(void)
         }
 
         TRACE_CUR("mc2_complete_job returns at %llu\n", litmus_clock());
+
         return err;
 }
 
@@ -988,6 +912,7 @@ static long mc2_admit_task(struct task_struct *tsk)
                 raw_spin_unlock_irqrestore(&state->lock, flags);
         } else if (lv == CRIT_LEVEL_C) {
                 raw_spin_lock_irqsave(&_global_env.lock, flags);
+//printk(KERN_ALERT "admit() hold lock\n");             
                 state = local_cpu_state();
                 
                 raw_spin_lock(&state->lock);
@@ -1012,6 +937,7 @@ static long mc2_admit_task(struct task_struct *tsk)
 
                 raw_spin_unlock(&state->lock);
                 raw_spin_unlock_irqrestore(&_global_env.lock, flags);
+//printk(KERN_ALERT "admit() release lock\n");          
         }
         
         preempt_enable();
@@ -1033,25 +959,30 @@ static void mc2_task_new(struct task_struct *tsk, int on_runqueue,
         struct mc2_cpu_state *state; // = cpu_state_for(tinfo->cpu);
         struct reservation *res;
         enum crit_level lv = get_task_crit_level(tsk);
+        lt_t release = 0;
 
         TRACE_TASK(tsk, "new RT task %llu (on_rq:%d, running:%d)\n",
                    litmus_clock(), on_runqueue, is_running);
 
+        local_irq_save(flags);
         if (tinfo->cpu == -1)
                 state = local_cpu_state();
         else 
                 state = cpu_state_for(tinfo->cpu);
         
         /* acquire the lock protecting the state and disable interrupts */
-        raw_spin_lock_irqsave(&_global_env.lock, flags);
+        raw_spin_lock(&_global_env.lock);
         raw_spin_lock(&state->lock);
-        
+//printk(KERN_ALERT "new() hold lock R%d\n", tinfo->mc2_param.res_id);  
         if (is_running) {
                 state->scheduled = tsk;
                 /* make sure this task should actually be running */
                 litmus_reschedule_local();
         }
-
+        
+        res = res_find_by_id(state, tinfo->mc2_param.res_id);
+        release = res->next_replenishment;
+        
         if (on_runqueue || is_running) {
                 /* Assumption: litmus_clock() is synchronized across cores
                  * [see comment in pres_task_resume()] */
@@ -1062,18 +993,20 @@ static void mc2_task_new(struct task_struct *tsk, int on_runqueue,
                 TRACE("mc2_new()\n");
                 
                 mc2_update_timer_and_unlock(state);
-                local_irq_restore(flags);
-        } else
-                raw_spin_unlock_irqrestore(&_global_env.lock, flags);
-
-        res = res_find_by_id(state, tinfo->mc2_param.res_id);
+//printk(KERN_ALERT "new() dropped lock R%d\n",tinfo->mc2_param.res_id);                
+        } else {
+                raw_spin_unlock(&state->lock);
+                raw_spin_unlock(&_global_env.lock);
+//printk(KERN_ALERT "new() release lock R%d\n",tinfo->mc2_param.res_id);                
+        }
+        local_irq_restore(flags);
         
-        if (res) {
-                TRACE_TASK(tsk, "mc2_task_new() next_replenishment = %llu\n", res->next_replenishment);
-                release_at(tsk, res->next_replenishment);
+        if (!release) {
+                TRACE_TASK(tsk, "mc2_task_new() next_release = %llu\n", release);
+                release_at(tsk, release);
         }
         else
-                TRACE_TASK(tsk, "mc2_task_new() next_replenishment = NULL\n");
+                TRACE_TASK(tsk, "mc2_task_new() next_release = NULL\n");
 }
 
 /* mc2_reservation_destroy - reservation_destroy system call backend
@@ -1196,14 +1129,16 @@ static void mc2_task_exit(struct task_struct *tsk)
         struct mc2_task_state* tinfo = get_mc2_state(tsk);
         struct mc2_cpu_state *state;
         enum crit_level lv = tinfo->mc2_param.crit;
-        struct crit_entry* ce;  
+        struct crit_entry* ce;
+        int cpu;
 
+        local_irq_save(flags);
         if (tinfo->cpu != -1)
                 state = cpu_state_for(tinfo->cpu);
         else
                 state = local_cpu_state();
                 
-        raw_spin_lock_irqsave(&_global_env.lock, flags);
+        raw_spin_lock(&_global_env.lock);
         raw_spin_lock(&state->lock);
         
         if (state->scheduled == tsk)
@@ -1226,11 +1161,30 @@ static void mc2_task_exit(struct task_struct *tsk)
                 /* NOTE: drops state->lock */
                 TRACE("mc2_exit()\n");
 
-                mc2_update_timer_and_unlock(state);
-                local_irq_restore(flags);
-        } else
-                raw_spin_unlock_irqrestore(&_global_env.lock, flags);
+                mc2_update_timer_and_unlock(state);     
+        } else {
+                raw_spin_unlock(&state->lock);
+                raw_spin_unlock(&_global_env.lock);
+        }
 
+        for_each_online_cpu(cpu) {
+                state = cpu_state_for(cpu);
+                if (state == local_cpu_state())
+                        continue;
+                raw_spin_lock(&state->lock);
+                
+                if (state->scheduled == tsk)
+                        state->scheduled = NULL;
+                
+                ce = &state->crit_entries[lv];
+                if (ce->running == tsk)
+                        ce->running = NULL;
+                
+                raw_spin_unlock(&state->lock);
+        }
+        
+        local_irq_restore(flags);
+        
         kfree(tsk_rt(tsk)->plugin_state);
         tsk_rt(tsk)->plugin_state = NULL;
         kfree(tsk_rt(tsk)->mc2_data);
@@ -1539,16 +1493,11 @@ static long mc2_activate_plugin(void)
                         struct crit_entry *cr_entry = &state->crit_entries[lv];
                         cr_entry->level = lv;
                         cr_entry->running = NULL;
-                        //hrtimer_init(&ce->ghost_timer, CLOCK_MONOTONIC, HRTIMER_MODE_ABS_PINNED);
-                        //ce->ghost_timer.function = on_ghost_timer;
                 }
                 sup_init(&state->sup_env);
 
                 hrtimer_init(&state->timer, CLOCK_MONOTONIC, HRTIMER_MODE_ABS_PINNED);
                 state->timer.function = on_scheduling_timer;
-                
-//              hrtimer_init(&state->g_timer, CLOCK_MONOTONIC, HRTIMER_MODE_ABS_PINNED);
-//              state->g_timer.function = on_global_scheduling_timer;
         }
 
         mc2_setup_domain_proc();
@@ -1561,7 +1510,7 @@ static void mc2_finish_switch(struct task_struct *prev)
         struct mc2_cpu_state *state = local_cpu_state();
         
         state->scheduled = is_realtime(current) ? current : NULL;
-        TRACE_TASK(prev, "FINISH CXS! complete=%d\n", tsk_rt(prev)->completed);
+        TRACE("FINISH CXS! from %s/%d to %s/%d\n", prev ? (prev)->comm : "null", prev ? (prev)->pid : 0, current ? (current)->comm : "null", current ? (current)->pid : 0);
 }
 
 static long mc2_deactivate_plugin(void)
@@ -1570,7 +1519,8 @@ static long mc2_deactivate_plugin(void)
         struct mc2_cpu_state *state;
         struct reservation *res;
         struct next_timer_event *event;
-
+        struct cpu_entry *ce;
+        
         raw_spin_lock(&_global_env.lock);
 
         for_each_online_cpu(cpu) {
@@ -1578,7 +1528,14 @@ static long mc2_deactivate_plugin(void)
                 raw_spin_lock(&state->lock);
 
                 hrtimer_cancel(&state->timer);
-//              hrtimer_cancel(&state->g_timer);
+
+                ce = &_lowest_prio_cpu.cpu_entries[cpu];
+                
+                ce->cpu = cpu;
+                ce->scheduled = NULL;
+                ce->deadline = ULLONG_MAX;
+                ce->lv = NUM_CRIT_LEVELS;
+                ce->will_schedule = false;
 
                 /* Delete all reservations --- assumes struct reservation
                  * is prefix of containing struct. */