path: root/litmus/sched_mc.c

/*
 * litmus/sched_mc.c
 *
 * Implementation of the Mixed Criticality scheduling algorithm.
 *
 * (Per Mollison, Erickson, Anderson, Baruah, Scoredos 2010)
 *
 * This version uses the simple approach and serializes all scheduling
 * decisions by the use of a queue lock. This is probably not the
 * best way to do it, but it should suffice for now.
 */

#include <linux/spinlock.h>
#include <linux/percpu.h>
#include <linux/sched.h>
#include <linux/hrtimer.h>
#include <linux/slab.h>

#include <litmus/litmus.h>
#include <litmus/jobs.h>
#include <litmus/sched_plugin.h>
#include <litmus/edf_common.h>
#include <litmus/sched_trace.h>

#include <litmus/bheap.h>

#include <linux/module.h>

#include <litmus/sched_mc.h>

/* Overview of MC operations.
 *
 * link_task_to_cpu(T, cpu) 	- Low-level operation to update the linkage
 *                                structure (NOT the actually scheduled
 *                                task). If there is another linked task To
 *                                already it will set To->linked_on = NO_CPU
 *                                (thereby removing its association with this
 *                                CPU). However, it will not requeue the
 *                                previously linked task (if any). It will set
 *                                T's state to RT_F_RUNNING and check whether
 *                                it is already running somewhere else. If T
 *                                is scheduled somewhere else it will link
 *                                it to that CPU instead (and pull the linked
 *                                task to cpu). T may be NULL.
 *
 * unlink(T)			- Unlink removes T from all scheduler data
 *                                structures. If it is linked to some CPU it
 *                                will link NULL to that CPU. If it is
 *                                currently queued in the mc queue it will
 *                                be removed from the rt_domain. It is safe to
 *                                call unlink(T) if T is not linked. T may not
 *                                be NULL.
 *
 * requeue(T)			- Requeue will insert T into the appropriate
 *                                queue. If the system is in real-time mode and
 *                                the T is released already, it will go into the
 *                                ready queue. If the system is not in
 *                                real-time mode is T, then T will go into the
 *                                release queue. If T's release time is in the
 *                                future, it will go into the release
 *                                queue. That means that T's release time/job
 *                                no/etc. has to be updated before requeu(T) is
 *                                called. It is not safe to call requeue(T)
 *                                when T is already queued. T may not be NULL.
 *
 * mc_job_arrival(T)	- This is the catch all function when T enters
 *                                the system after either a suspension or at a
 *                                job release. It will queue T (which means it
 *                                is not safe to call mc_job_arrival(T) if
 *                                T is already queued) and then check whether a
 *                                preemption is necessary. If a preemption is
 *                                necessary it will update the linkage
 *                                accordingly and cause scheduled to be called
 *                                (either with an IPI or need_resched). It is
 *                                safe to call mc_job_arrival(T) if T's
 *                                next job has not been actually released yet
 *                                (releast time in the future). T will be put
 *                                on the release queue in that case.
 *
 * job_completion(T)		- Take care of everything that needs to be done
 *                                to prepare T for its next release and place
 *                                it in the right queue with
 *                                mc_job_arrival().
 *
 *
 * When we now that T is linked to CPU then link_task_to_cpu(NULL, CPU) is
 * equivalent to unlink(T). Note that if you unlink a task from a CPU none of
 * the functions will automatically propagate pending task from the ready queue
 * to a linked task. This is the job of the calling function ( by means of
 * __take_ready).
 */


/* cpu_entry_t - maintain the linked and scheduled state
 */
typedef struct  {
	int 			cpu;
	struct task_struct*	linked;		/* only RT tasks */
	struct task_struct*	scheduled;	/* only RT tasks */
	atomic_t		will_schedule;	/* prevent unneeded IPIs */
	struct bheap_node*	hn_c;
	struct bheap_node*	hn_d;
	struct task_struct*	ghost_tasks[NUM_CRIT_LEVELS];
} cpu_entry_t;

/*This code is heavily based on Bjoern's budget enforcement code. */
struct watchdog_timer {
	/* The watchdog timers determine when ghost jobs finish. */
	struct hrtimer		timer;
	struct task_struct*	task;
};

DEFINE_PER_CPU(struct watchdog_timer[NUM_CRIT_LEVELS], ghost_timers);
#define ghost_timer(cpu, crit) (&(per_cpu(ghost_timers, cpu)[crit]))

DEFINE_PER_CPU(cpu_entry_t, mc_cpu_entries);

cpu_entry_t* mc_cpus[NR_CPUS];

#define set_will_schedule() \
	(atomic_set(&__get_cpu_var(mc_cpu_entries).will_schedule, 1))
#define clear_will_schedule() \
	(atomic_set(&__get_cpu_var(mc_cpu_entries).will_schedule, 0))
#define test_will_schedule(cpu) \
	(atomic_read(&per_cpu(mc_cpu_entries, cpu).will_schedule))
#define remote_cpu_entry(cpu)	(&per_cpu(mc_cpu_entries, cpu))

#define tsk_mc_data(t) (tsk_rt(t)->mc_data)
#define tsk_mc_crit(t) (tsk_mc_data(t)->mc_task.crit)

/* need to do a short-circuit null check on mc_data before checking is_ghost */
static inline int is_ghost(struct task_struct *t)
{
	struct mc_data *mc_data = tsk_mc_data(t);
	return mc_data && mc_data->mc_job.is_ghost;
}

/* the cpus queue themselves according to priority in here */
static struct bheap_node mc_heap_node_c[NR_CPUS], mc_heap_node_d[NR_CPUS];
static struct bheap      mc_cpu_heap_c, mc_cpu_heap_d;

/* Create per-CPU domains for criticality A */
DEFINE_PER_CPU(rt_domain_t, crit_a);
#define remote_a_queue(cpu)	(&per_cpu(crit_a, cpu))
#define local_a_queue		(&__get_cpu_var(crit_a))

/* Create per-CPU domains for criticality B */
DEFINE_PER_CPU(rt_domain_t, crit_b);
#define remote_b_queue(cpu)	(&per_cpu(crit_b, cpu))
#define local_b_queue		(&__get_cpu_var(crit_b))

/* Create global domains for criticalities C and D */
static rt_domain_t crit_c;
static rt_domain_t crit_d;

/* We use crit_c for shared globals */
#define global_lock (crit_c.ready_lock)
#define mc_release_master (crit_c.release_master)

/* BEGIN clone of edf_common.c to allow shared C/D run queue*/

static int mc_edf_higher_prio(struct task_struct* first, struct task_struct*
				second)
{
	/*Only differs from normal EDF when two tasks of differing criticality
	  are compared.*/
	if (first && second){
		enum crit_level first_crit = tsk_mc_crit(first);
		enum crit_level second_crit = tsk_mc_crit(second);
		/*Lower criticality numbers are higher priority*/
		if (first_crit < second_crit){
			return 1;
		}
		else if (second_crit < first_crit){
			return 0;
		}
	}
	return edf_higher_prio(first, second);
}

static int mc_edf_entry_higher_prio(cpu_entry_t* first, cpu_entry_t* second,
					enum crit_level crit)
{
	struct task_struct *first_active, *second_active;
	first_active = first->linked;
	second_active = second->linked;
	if (first->ghost_tasks[crit]){
		first_active = first->ghost_tasks[crit];
	}
	if (second->ghost_tasks[crit]){
		second_active = second->ghost_tasks[crit];
	}
	return mc_edf_higher_prio(first_active, second_active);
}

/* need_to_preempt - check whether the task t needs to be preempted
 *                   call only with irqs disabled and with  ready_lock acquired
 *                   THIS DOES NOT TAKE NON-PREEMPTIVE SECTIONS INTO ACCOUNT!
 */
static int mc_edf_preemption_needed(rt_domain_t* rt, enum crit_level crit,
					cpu_entry_t* entry)
{
	struct task_struct *active_task;

        /* we need the read lock for edf_ready_queue */
        /* no need to preempt if there is nothing pending */
        if (!__jobs_pending(rt))
                return 0;

	active_task = entry->linked;
	/* A ghost task can only exist if we haven't scheduled something above
	 * its level
	 */
	if (entry->ghost_tasks[crit]){
		active_task = entry->ghost_tasks[crit];
	}
        /* we need to reschedule if t doesn't exist */
        if (!active_task)
                return 1;

        /* NOTE: We cannot check for non-preemptibility since we
         *       don't know what address space we're currently in.
         */

        /* make sure to get non-rt stuff out of the way */
        return !is_realtime(active_task) ||
		mc_edf_higher_prio(__next_ready(rt), active_task);
}

static int mc_edf_ready_order(struct bheap_node* a, struct bheap_node* b)
{
        return mc_edf_higher_prio(bheap2task(a), bheap2task(b));
}

static void mc_edf_domain_init(rt_domain_t* rt, check_resched_needed_t resched,
                      release_jobs_t release)
{
        rt_domain_init(rt, mc_edf_ready_order, resched, release);
}


/* END clone of edf_common.c*/

/* Return the domain of a task */
static rt_domain_t* domain_of(struct task_struct* task)
{
	switch (tsk_mc_crit(task))
	{
		case CRIT_LEVEL_A:
			return remote_a_queue(get_partition(task));
			break;
		case CRIT_LEVEL_B:
			return remote_b_queue(get_partition(task));
			break;
		case CRIT_LEVEL_C:
			return &crit_c;
			break;
		case CRIT_LEVEL_D:
			return &crit_d;
			break;
		case NUM_CRIT_LEVELS:
		default:
			/*Should never get here*/
			BUG();
	}
}

/* Uncomment this if you want to see all scheduling decisions in the
 * TRACE() log.
#define WANT_ALL_SCHED_EVENTS
 */

/* Called by update_cpu_position and lowest_prio_cpu in bheap operations
 * 	Callers always have global lock
*/
static int cpu_lower_prio_c(struct bheap_node *_a, struct bheap_node *_b)
{
	cpu_entry_t *a, *b;
	a = _a->value;
	b = _b->value;
	/* Note that a and b are inverted: we want the lowest-priority CPU at
	 * the top of the heap.
	 */
	return mc_edf_entry_higher_prio(b, a, CRIT_LEVEL_C);
}

/* Called by update_cpu_position and lowest_prio_cpu in bheap operations
 * 	Callers always have global lock
*/
static int cpu_lower_prio_d(struct bheap_node *_a, struct bheap_node *_b)
{
	cpu_entry_t *a, *b;
	a = _a->value;
	b = _b->value;
	/* Note that a and b are inverted: we want the lowest-priority CPU at
	 * the top of the heap.
	 */
	return mc_edf_entry_higher_prio(b, a, CRIT_LEVEL_D);
}

/* update_cpu_position - Move the cpu entry to the correct place to maintain
 *                       order in the cpu queue. Caller must hold global lock.
 * Called from link_task_to_cpu, which holds global lock
 * link_task_to_cpu is the only way a CPU can get a new task, and hence have its
 *	priority change.
 */
static void update_cpu_position(cpu_entry_t *entry)
{
	if (likely(bheap_node_in_heap(entry->hn_c)))
		bheap_delete(cpu_lower_prio_c, &mc_cpu_heap_c, entry->hn_c);
	if (likely(bheap_node_in_heap(entry->hn_d)))
		bheap_delete(cpu_lower_prio_d, &mc_cpu_heap_d, entry->hn_d);
	bheap_insert(cpu_lower_prio_c, &mc_cpu_heap_c, entry->hn_c);
	bheap_insert(cpu_lower_prio_d, &mc_cpu_heap_d, entry->hn_d);
}

/* caller must hold global lock
 * Only called when checking for gedf preemptions by check_for_gedf_preemptions,
 * 	which always has global lock
 */
static cpu_entry_t* lowest_prio_cpu_c(void)
{
	struct bheap_node* hn;
	hn = bheap_peek(cpu_lower_prio_c, &mc_cpu_heap_c);
	return hn->value;
}

/* caller must hold global lock
 * Only called when checking for gedf preemptions by check_for_gedf_preemptions,
 * 	which always has global lock
 */
static cpu_entry_t* lowest_prio_cpu_d(void)
{
	struct bheap_node* hn;
	hn = bheap_peek(cpu_lower_prio_d, &mc_cpu_heap_d);
	return hn->value;
}

/* Forward Declarations*/
static noinline void unlink(struct task_struct* t);
static noinline void job_completion(struct task_struct *t, int forced);

/* update_ghost_time - Do time accounting for a ghost job.
 * Updates ghost budget and handles expired ghost budget.
 * Called from unlink(), mc_tick().
 * Caller holds global lock.
 */
static void update_ghost_time(struct task_struct *p)
{
	u64 delta;
	u64 clock;
	
	BUG_ON(!is_ghost(p));
	clock = litmus_clock();
	delta = clock - p->se.exec_start;
	if (unlikely ((s64)delta < 0)) {
		delta = 0;
		TRACE_TASK(p, "WARNING: negative time delta.\n");
	}
	if (tsk_mc_data(p)->mc_job.ghost_budget <= delta) {
		/*Currently will just set ghost budget to zero since
		 * task has already been queued.  Could probably do
		 * more efficiently with significant reworking.
		 */
		TRACE_TASK(p, "Ghost job could have ended\n");
		tsk_mc_data(p)->mc_job.ghost_budget = 0;
		p->se.exec_start = clock;
	}
	else{
		TRACE_TASK(p, "Ghost jub updated, but didn't finish\n");
		tsk_mc_data(p)->mc_job.ghost_budget -= delta;
		p->se.exec_start = clock;
	}
}


/* 
 * 
 */
static void cancel_watchdog_timer(struct watchdog_timer* wt)
{
	int ret;

	if (wt->task) {
		TRACE_TASK(wt->task, "Cancelling watchdog timer.\n");
		ret = hrtimer_try_to_cancel(&wt->timer);
		/*Should never be inactive.*/
		BUG_ON(ret == 0);
		/*Running concurrently is an unusual situation - log it. */
		/*TODO: is there a way to prevent this?  This probably means
		 * the timer task is waiting to acquire the lock while the
		 * cancellation attempt is happening.
		 *
		 * And are we even in a correct state when this happens?
		 */
		if (ret == -1)
			TRACE_TASK(wt->task, "Timer cancellation "
				"attempted while task completing\n");

		wt->task = NULL;
	}
}

/* link_task_to_cpu - Update the link of a CPU.
 *                    Handles the case where the to-be-linked task is already
 *                    scheduled on a different CPU.
 *		      Also handles ghost jobs and preemption of ghost jobs.
 *	Called from unlink(), prepare_preemption(), and mc_schedule()
 *	Callers hold global lock
 */
static noinline void link_task_to_cpu(struct task_struct* linked,
				      cpu_entry_t *entry)
{
	cpu_entry_t *sched;
	struct task_struct* tmp;
	int on_cpu;
	int i;
	struct watchdog_timer* timer;
	lt_t when_to_fire;

	BUG_ON(linked && !is_realtime(linked));
	BUG_ON(linked && is_realtime(linked) &&
		(tsk_mc_crit(linked) < CRIT_LEVEL_C) &&
		(tsk_rt(linked)->task_params.cpu != entry->cpu));

	if (linked && is_ghost(linked)) {
		TRACE_TASK(linked, "Linking ghost job to CPU %d.\n",
				entry->cpu);
		BUG_ON(entry->linked &&
			tsk_mc_crit(entry->linked) < tsk_mc_crit(linked));
		tmp = entry->ghost_tasks[tsk_mc_crit(linked)];
		if (tmp) {
			unlink(tmp);
		}
		/* We shouldn't link a ghost job that is already somewhere
		 * else (or here) - the caller is responsible for unlinking]
		 * first.
		 */
		BUG_ON(linked->rt_param.linked_on != NO_CPU);
		linked->rt_param.linked_on = entry->cpu;
		linked->se.exec_start = litmus_clock();
		entry->ghost_tasks[tsk_mc_crit(linked)] = linked;
		/* Set up the watchdog timer. */
		timer = ghost_timer(entry->cpu, tsk_mc_crit(linked));
		if (timer->task){
			cancel_watchdog_timer(timer);
		}
		when_to_fire = litmus_clock() +
			tsk_mc_data(linked)->mc_job.ghost_budget;
		timer->task = linked;
		__hrtimer_start_range_ns(&timer->timer,
			ns_to_ktime(when_to_fire),
			0 /* delta */,
			HRTIMER_MODE_ABS_PINNED,
			0 /* no wakeup */);
	}
	else{
		/* Currently linked task is set to be unlinked. */
		if (entry->linked) {
			entry->linked->rt_param.linked_on = NO_CPU;
		}

		/* Link new task to CPU. */
		if (linked) {
			set_rt_flags(linked, RT_F_RUNNING);
			/* handle task is already scheduled somewhere! */
			on_cpu = linked->rt_param.scheduled_on;
			if (on_cpu != NO_CPU) {
				sched = &per_cpu(mc_cpu_entries, on_cpu);
				/* this should only happen if not linked
				 * already
				 */
				BUG_ON(sched->linked == linked);

				/* If we are already scheduled on the CPU to
				 * which we wanted to link, we don't need to do
				 * the swap -- we just link ourselves to the
				 * CPU and depend on the caller to get things
				 * right.
				 *
				 * Also, we can only safely swap if neither
				 * task is partitioned.
				 */
				tmp = sched->linked;
				if (entry != sched && tsk_mc_crit(linked) >
						CRIT_LEVEL_B &&
					(!tmp || tsk_mc_crit(tmp)
						> CRIT_LEVEL_B)) {
					TRACE_TASK(linked,
						   "already scheduled on %d, updating link.\n",
						   sched->cpu);
					linked->rt_param.linked_on = sched->cpu;
					sched->linked = linked;
					for (i = tsk_mc_crit(linked);
						i < NUM_CRIT_LEVELS; i++) {
						if (sched->ghost_tasks[i]){
							unlink(sched->
								ghost_tasks[i]);
						}
					}
					update_cpu_position(sched);
					linked = tmp;
				}
			}
			if (linked) { /* might be NULL due to swap */
				linked->rt_param.linked_on = entry->cpu;
				for (i = tsk_mc_crit(linked);
					i < NUM_CRIT_LEVELS; i++){
					if (entry->ghost_tasks[i]){
						unlink(entry->ghost_tasks[i]);
						/* WARNING: it is up to the
						 * caller to requeue ghost jobs
						 */
					}
				}
			}
		}
		entry->linked = linked;
	}
#ifdef WANT_ALL_SCHED_EVENTS
		if (linked)
			TRACE_TASK(linked, "linked to %d.\n", entry->cpu);
		else
			TRACE("NULL linked to %d.\n", entry->cpu);
#endif
	update_cpu_position(entry);
}

/* unlink - Make sure a task is not linked any longer to a cpu entry
 *          where it was linked before.
 *          Can handle ghost jobs.
 *	Called by schedule, task_block, task_exit, and job_completion
 * 	Caller assumed to hold global lock
 */
static noinline void unlink(struct task_struct* t)
{
	int cpu;
    	cpu_entry_t *entry;
	struct watchdog_timer *timer;

	if (unlikely(!t)) {
		BUG_ON(1);
		return;
	}

	cpu = t->rt_param.linked_on;
	if (cpu != NO_CPU) {
		/* unlink */
		entry = remote_cpu_entry(cpu);
		t->rt_param.linked_on = NO_CPU;
		if (is_ghost(t)) {
			/* Clear the timer if it's set.
			 * It may be unset if we are called as a result of
			 * the watchdog timer triggering.
			 */
			timer = ghost_timer(cpu, tsk_mc_crit(t));
			if (timer->task) {
				/* Should already be watching task.*/
				BUG_ON(timer->task != t);
				cancel_watchdog_timer(timer);
			}
			if (tsk_mc_data(t)->mc_job.ghost_budget > 0) {
				/* Job isn't finished, so do accounting. */
				update_ghost_time(t);
				/* Just remove from CPU, even in the rare case
				 * of zero time left - it will be scheduled
				 * with an immediate timer fire.
				 */
				entry->ghost_tasks[tsk_mc_crit(t)] = NULL;
				/*TODO: maybe make more efficient by
				 * only updating on C/D completion?
				 */
				update_cpu_position(entry);
			}
			else{
				/* Job finished, so just remove */
				entry->ghost_tasks[tsk_mc_crit(t)] = NULL;
				update_cpu_position(entry);
			}
		}
		else {
			link_task_to_cpu(NULL, entry);
		}
	} else if (is_queued(t)) {
		/* This is an interesting situation: t is scheduled,
		 * but was just recently unlinked.  It cannot be
		 * linked anywhere else (because then it would have
		 * been relinked to this CPU), thus it must be in some
		 * queue. We must remove it from the list in this
		 * case.
		 */
		TRACE("Weird is_queued situation happened\n");
		remove(domain_of(t), t);
	}
}


/* preempt - force a CPU to reschedule
 * Just sets a Linux scheduler flag.
 */
static void preempt(cpu_entry_t *entry)
{
	preempt_if_preemptable(entry->scheduled, entry->cpu);
}

/* requeue - Put an unlinked task into the proper domain.
 *           Caller holds global lock.
 *           Called by mc_job_arrival() and prepare_preemption().
 */
static noinline void requeue(struct task_struct* task)
{
/*	BUG_ON(!task || !is_realtime(task));*/
	BUG_ON(!task);
	BUG_ON(!is_realtime(task));
	/* sanity check before insertion */
	BUG_ON(is_queued(task));

	if (is_released(task, litmus_clock()))
		__add_ready(domain_of(task), task);
	else {
		/* it has got to wait */
		add_release(domain_of(task), task);
	}
}

static void prepare_preemption(rt_domain_t *dom, cpu_entry_t *cpu,
			enum crit_level crit) {
	struct task_struct* task;
	int i;
	task = __take_ready(dom);
	TRACE("prepare_preemption: attempting to link task %d to %d\n",
		task->pid, cpu->cpu);
	if (is_ghost(task)){
		/* Changing ghost task only affects linked task at our level */
		if (cpu->linked && tsk_mc_crit(cpu->linked) == crit)
			requeue(cpu->linked);
		/* Can change ghost task at our level as well. */
		if (cpu->ghost_tasks[crit])
			requeue(cpu->ghost_tasks[crit]);
	}
	else {
		/* Changing linked tasks could affect both real and ghost
		 * tasks at multiple levels
		 */
		if (cpu->linked)
			requeue(cpu->linked);
		for (i = crit; i < NUM_CRIT_LEVELS; i++) {
			if (cpu->ghost_tasks[i])
				requeue(cpu->ghost_tasks[i]);
		}
	}
	link_task_to_cpu(task, cpu);
	preempt(cpu);
}

/* Callers always have global lock for functions in this section*/
static void check_for_c_preemptions(rt_domain_t *dom){
	cpu_entry_t* last;
	for (last = lowest_prio_cpu_c();
			mc_edf_preemption_needed(dom, CRIT_LEVEL_C, 
					last);
			last = lowest_prio_cpu_c()) {
		prepare_preemption(dom, last, CRIT_LEVEL_C);
	}
}

static void check_for_d_preemptions(rt_domain_t *dom){
	cpu_entry_t* last;
	for (last = lowest_prio_cpu_d();
			mc_edf_preemption_needed(dom, CRIT_LEVEL_D, 
					last);
			last = lowest_prio_cpu_d()) {
		prepare_preemption(dom, last, CRIT_LEVEL_D);
	}
}

static void check_for_a_preemption(rt_domain_t *dom, cpu_entry_t *cpu) {
	if (mc_edf_preemption_needed(dom, CRIT_LEVEL_A, cpu)) {
		prepare_preemption(dom, cpu, CRIT_LEVEL_A);
	}
}

static void check_for_b_preemption(rt_domain_t *dom, cpu_entry_t *cpu) {
	if (mc_edf_preemption_needed(dom, CRIT_LEVEL_B, cpu)) {
		prepare_preemption(dom, cpu, CRIT_LEVEL_B);
	}
}

/* mc_job_arrival: task is either resumed or released
 * Called from job_completion(), mc_task_new(), and mc_task_wake_up(), all
 *	of which have the global lock
 * Requeues task and checks for/triggers preemptions
 */
static noinline void mc_job_arrival(struct task_struct* task)
{
	enum crit_level task_crit_level;
	BUG_ON(!task);

	TRACE("mc_job_arrival triggered\n");
	task_crit_level = tsk_mc_crit(task);
	requeue(task);
	if (task_crit_level == CRIT_LEVEL_A){
		check_for_a_preemption(remote_a_queue(get_partition(task)),
			remote_cpu_entry(get_partition(task)));
	}
	else if (task_crit_level == CRIT_LEVEL_B){
		check_for_a_preemption(remote_b_queue(get_partition(task)),
			remote_cpu_entry(get_partition(task)));
	}
	else if (task_crit_level == CRIT_LEVEL_C){
		check_for_c_preemptions(&crit_c);
	}
	else if (task_crit_level == CRIT_LEVEL_D){
		check_for_d_preemptions(&crit_d);
	}
}

/* Called by the domain
 * Obtains global lock, merges ready tasks, checks for/triggers preemptions,
 *	and releases global lock
*/
static void mc_release_jobs(rt_domain_t* rt, struct bheap* tasks)
{
	unsigned long flags;
	int i;

	raw_spin_lock_irqsave(&global_lock, flags);
	TRACE("mc_release_jobs triggered\n");

	__merge_ready(rt, tasks);

	for (i = 0; i < NR_CPUS; i++){
		if (rt == remote_b_queue(i)){
			check_for_b_preemption(rt, remote_cpu_entry(i));
		}
		else if (rt == remote_a_queue(i)){
			check_for_a_preemption(rt, remote_cpu_entry(i));
		}
	}
	if (rt == &crit_c){
		check_for_c_preemptions(rt);
	}
	if (rt == &crit_d){
		check_for_d_preemptions(rt);
	}

	raw_spin_unlock_irqrestore(&global_lock, flags);
}

/* caller holds global_lock
 * Called only by mc_schedule() which holds global lock
 * Prepares task for next period, unlinks it, and calls mc_job_arrival
 * Converts jobs to ghost jobs as necessary, or finishes end of ghost jobs.
*/
static noinline void job_completion(struct task_struct *t, int forced)
{
	cpu_entry_t *cpu;
	BUG_ON(!t);

	sched_trace_task_completion(t, forced);

	TRACE_TASK(t, "job_completion().\n");

	/* set flags */
	set_rt_flags(t, RT_F_SLEEP);
	/* If it's not a ghost job, do ghost job conversion and return if
	 * needed.
	 */
	if (!is_ghost(t)) {
		TRACE_TASK(t, "Converting to ghost.\n");
		cpu = remote_cpu_entry(t->rt_param.scheduled_on);
		/*Unlink first while it's not a ghost job.*/
		unlink(t);
		tsk_mc_data(t)->mc_job.ghost_budget = budget_remaining(t);
		tsk_mc_data(t)->mc_job.is_ghost = 1;

		/* If we did just convert the job to ghost, we can safely
		 * reschedule it and then let schedule() determine a new
		 * job to run in the slack.
		 *
		 * If it actually needs to run as a ghost job, we'll do so
		 * here.
		 *
		 * If it doesn't need to, it will fall through and be handled
		 * properly as well.
		 */
		if (tsk_mc_data(t)->mc_job.ghost_budget > 0) {
			link_task_to_cpu(t, cpu);
			preempt(cpu);
			return;
		}
	}
	/* prepare for next period - we either just became ghost but with no
	 * budget left, or we were already ghost and the ghost job expired*/
	if (is_ghost(t)) {
		tsk_mc_data(t)->mc_job.ghost_budget = 0;
		/*Need to unlink here so prepare_for_next_period doesn't try
		 * to unlink us
		 */
		unlink(t);
		tsk_mc_data(t)->mc_job.is_ghost = 0;
		tsk_mc_data(t)->mc_job.ghost_budget = 0;
		prepare_for_next_period(t);
	}
	if (is_released(t, litmus_clock()))
		sched_trace_task_release(t);
	/* requeue
	 * But don't requeue a blocking task. */
	if (is_running(t))
		mc_job_arrival(t);
}

/* watchdog_timeout - this function is called when a watchdog timer expires.
 * 
 * Acquires global lock
 */

static enum hrtimer_restart watchdog_timeout(struct hrtimer *timer)
{
	struct watchdog_timer* wt = container_of(timer,
						 struct watchdog_timer,
						 timer);
	unsigned long flags;
	struct task_struct* task = wt->task;
	raw_spin_lock_irqsave(&global_lock, flags);
	/*If we have triggered, we know the budget must have expired.*/
	/*This needs to run first, so it doesn't look to job_completion like
	 * we have an active timer.
	 */
	wt->task = NULL;
	tsk_mc_data(task)->mc_job.ghost_budget = 0;
	job_completion(task, 0);
	TRACE_TASK(task, "Watchdog timeout\n");
	raw_spin_unlock_irqrestore(&global_lock, flags);
	return HRTIMER_NORESTART;
}


/* mc_tick - this function is called for every local timer
 *                         interrupt.
 *
 *                   checks whether the current task has expired and checks
 *                   whether we need to preempt it if it has not expired
 * Called from LITMUS core
 * Locks when calling update_ghost_time(t)
 * Just sets reschedule flags on task and CPU and request_exit_np flag on task
 */
static void mc_tick(struct task_struct* t)
{
	unsigned long flags;
	if (is_ghost(t)) {
		raw_spin_lock_irqsave(&global_lock, flags);
		update_ghost_time(t);
		raw_spin_unlock_irqrestore(&global_lock, flags);
	}
	if (is_realtime(t) && budget_enforced(t) && budget_exhausted(t)) {
		if (!is_np(t)) {
			/* np tasks will be preempted when they become
			 * preemptable again
			 */
			set_tsk_need_resched(t);
			set_will_schedule();
			TRACE("mc_scheduler_tick: "
			      "%d is preemptable "
			      " => FORCE_RESCHED\n", t->pid);
		} else if (is_user_np(t)) {
			TRACE("mc_scheduler_tick: "
			      "%d is non-preemptable, "
			      "preemption delayed.\n", t->pid);
			request_exit_np(t);
		}
	}
}

/* Getting schedule() right is a bit tricky. schedule() may not make any
 * assumptions on the state of the current task since it may be called for a
 * number of reasons. The reasons include a scheduler_tick() determined that it
 * was necessary, because sys_exit_np() was called, because some Linux
 * subsystem determined so, or even (in the worst case) because there is a bug
 * hidden somewhere. Thus, we must take extreme care to determine what the
 * current state is.
 *
 * The CPU could currently be scheduling a task (or not), be linked (or not).
 *
 * The following assertions for the scheduled task could hold:
 *
 *      - !is_running(scheduled)        // the job blocks
 *	- scheduled->timeslice == 0	// the job completed (forcefully)
 *	- get_rt_flag() == RT_F_SLEEP	// the job completed (by syscall)
 * 	- linked != scheduled		// we need to reschedule (for any reason)
 * 	- is_np(scheduled)		// rescheduling must be delayed,
 *					   sys_exit_np must be requested
 *
 * Any of these can occur together.
 *
 *
 * Called by LITMUS core
 * No lock required by caller
 * Obtains global lock
 * can call unlink(), request_exit_np(), job_completion(), __take_ready()
 * modifies next, scheduled->scheduled_on, linked->scheduled_on
 * Releases global lock
 */
static struct task_struct* mc_schedule(struct task_struct * prev)
{
	cpu_entry_t* entry = &__get_cpu_var(mc_cpu_entries);
	int out_of_time, sleep, preempt, np, exists, blocks;
	struct task_struct* next = NULL;
	struct task_struct* ready_task = NULL;
	enum crit_level ready_crit;
	int i;

#ifdef CONFIG_RELEASE_MASTER
	/* Bail out early if we are the release master.
	 * The release master never schedules any real-time tasks.
	 */
	if (mc_release_master == entry->cpu)
		return NULL;
#endif

	raw_spin_lock(&global_lock);
	clear_will_schedule();

	/* sanity checking */
	BUG_ON(entry->scheduled && entry->scheduled != prev);
	BUG_ON(entry->scheduled && !is_realtime(prev));
	BUG_ON(is_realtime(prev) && !entry->scheduled);

	/* (0) Determine state */
	exists      = entry->scheduled != NULL;
	blocks      = exists && !is_running(entry->scheduled);
	out_of_time = exists &&	budget_enforced(entry->scheduled) &&
				budget_exhausted(entry->scheduled);
	np 	    = exists && is_np(entry->scheduled);
	sleep	    = exists && get_rt_flags(entry->scheduled) == RT_F_SLEEP;
	preempt     = entry->scheduled != entry->linked;

#ifdef WANT_ALL_SCHED_EVENTS
	TRACE_TASK(prev, "invoked mc_schedule.\n");
#endif

	if (exists)
		TRACE_TASK(prev,
			   "blocks:%d out_of_time:%d np:%d sleep:%d preempt:%d "
			   "state:%d sig:%d\n",
			   blocks, out_of_time, np, sleep, preempt,
			   prev->state, signal_pending(prev));
	if (entry->linked && preempt)
		TRACE_TASK(prev, "will be preempted by %s/%d\n",
			   entry->linked->comm, entry->linked->pid);


	/* If a task blocks we have no choice but to reschedule.
	 */
	if (blocks)
		unlink(entry->scheduled);

	/* Request a sys_exit_np() call if we would like to preempt but cannot.
	 * We need to make sure to update the link structure anyway in case
	 * that we are still linked. Multiple calls to request_exit_np() don't
	 * hurt.
	 */
	if (np && (out_of_time || preempt || sleep)) {
		unlink(entry->scheduled);
		request_exit_np(entry->scheduled);
	}

	/* Any task that is preemptable and either exhausts its execution
	 * budget or wants to sleep completes. We may have to reschedule after
	 * this. Don't do a job completion if we block (can't have timers running
	 * for blocked jobs). Preemption go first for the same reason.
	 */
	if (!np && (out_of_time || sleep) && !blocks && !preempt)
		job_completion(entry->scheduled, !sleep);

	/* Link pending task if we became unlinked.
	 */
	if (!entry->linked){
		if (!entry->ghost_tasks[CRIT_LEVEL_A]) {
			ready_task = __take_ready(local_a_queue);
			ready_crit = CRIT_LEVEL_A;
			if (ready_task && is_ghost(ready_task)) {
				link_task_to_cpu(ready_task, entry);
				ready_task = NULL;
			}
		}
		if (!ready_task && !entry->ghost_tasks[CRIT_LEVEL_B]) {
			ready_task = __take_ready(local_b_queue);
			ready_crit = CRIT_LEVEL_B;
			if (ready_task && is_ghost(ready_task)) {
				link_task_to_cpu(ready_task, entry);
				ready_task = NULL;
			}
		}
		if (!ready_task && !entry->ghost_tasks[CRIT_LEVEL_C]) {
			ready_task = __take_ready(&crit_c);
			ready_crit = CRIT_LEVEL_C;
			if (ready_task && is_ghost(ready_task)) {
				link_task_to_cpu(ready_task, entry);
				ready_task = NULL;
			}
		}
		if (!ready_task && !entry->ghost_tasks[CRIT_LEVEL_D]) {
			ready_task = __take_ready(&crit_d);
			ready_crit = CRIT_LEVEL_D;
			if (ready_task && is_ghost(ready_task)) {
				link_task_to_cpu(ready_task, entry);
				ready_task = NULL;
			}
		}
		if (!ready_task) {
			/* set to something invalid? */
			ready_crit = NUM_CRIT_LEVELS;
		}
		for (i = ready_crit; i < NUM_CRIT_LEVELS; i++) {
			if (entry->ghost_tasks[i])
				requeue(entry->ghost_tasks[i]);
		}
		link_task_to_cpu(ready_task, entry);
		if (ready_task)
			TRACE_TASK(ready_task,
					"Linked task inside scheduler\n");
	}

	/* The final scheduling decision. Do we need to switch for some reason?
	 * If linked is different from scheduled, then select linked as next.
	 */
	if ((!np || blocks) &&
	    entry->linked != entry->scheduled) {
		/* Schedule a linked job? */
		if (entry->linked) {
			entry->linked->rt_param.scheduled_on = entry->cpu;
			next = entry->linked;
		}
		if (entry->scheduled) {
			/* not gonna be scheduled soon */
			entry->scheduled->rt_param.scheduled_on = NO_CPU;
			TRACE_TASK(entry->scheduled, "scheduled_on = NO_CPU\n");
		}
	} else
		/* Only override Linux scheduler if we have a real-time task
		 * scheduled that needs to continue.
		 */
		if (exists)
			next = prev;

	/*TODO: Update name of locking, reflect that we're locking all queues*/
	raw_spin_unlock(&global_lock);

#ifdef WANT_ALL_SCHED_EVENTS
	TRACE("global_lock released, next=0x%p\n", next);

	if (next)
		TRACE_TASK(next, "scheduled at %llu\n", litmus_clock());
	else if (exists && !next)
		TRACE("becomes idle at %llu.\n", litmus_clock());
#endif


	return next;
}


/* _finish_switch - we just finished the switch away from prev
 * Called by LITMUS core
 * No locks
 */
static void mc_finish_switch(struct task_struct *prev)
{
	cpu_entry_t* 	entry = &__get_cpu_var(mc_cpu_entries);

	entry->scheduled = is_realtime(current) ? current : NULL;
#ifdef WANT_ALL_SCHED_EVENTS
	TRACE_TASK(prev, "switched away from\n");
#endif
}


/*	Prepare a task for running in RT mode
 * 	Called by LITMUS core
 * 	No lock required by caller
 * 	Obtains lock and calls mc_job_arrival before releasing lock
 */
static void mc_task_new(struct task_struct * t, int on_rq, int running)
{
	unsigned long 		flags;
	cpu_entry_t* 		entry;

	TRACE("mixed crit: task new %d\n", t->pid);

	raw_spin_lock_irqsave(&global_lock, flags);

	/* setup job params */
	release_at(t, litmus_clock());
	tsk_mc_data(t)->mc_job.ghost_budget = 0;
	tsk_mc_data(t)->mc_job.is_ghost = 0;

	if (running) {
		entry = &per_cpu(mc_cpu_entries, task_cpu(t));
		BUG_ON(entry->scheduled);

#ifdef CONFIG_RELEASE_MASTER
		if (entry->cpu != mc_release_master) {
#endif
			entry->scheduled = t;
			tsk_rt(t)->scheduled_on = task_cpu(t);
#ifdef CONFIG_RELEASE_MASTER
		} else {
			/* do not schedule on release master */
			preempt(entry); /* force resched */
			tsk_rt(t)->scheduled_on = NO_CPU;
		}
#endif
	} else {
		t->rt_param.scheduled_on = NO_CPU;
	}
	t->rt_param.linked_on          = NO_CPU;

	mc_job_arrival(t);
	raw_spin_unlock_irqrestore(&global_lock, flags);
}

/* Called by LITMUS core
 * No lock required by caller
 * Obtains lock and calls mc_job_arrival before releasing lock
 */
static void mc_task_wake_up(struct task_struct *task)
{
	unsigned long flags;
	lt_t now;

	TRACE_TASK(task, "wake_up at %llu\n", litmus_clock());

	raw_spin_lock_irqsave(&global_lock, flags);
	/* We need to take suspensions because of semaphores into
	 * account! If a job resumes after being suspended due to acquiring
	 * a semaphore, it should never be treated as a new job release.
	 */
	if (get_rt_flags(task) == RT_F_EXIT_SEM) {
		set_rt_flags(task, RT_F_RUNNING);
	} else {
		now = litmus_clock();
		if (is_tardy(task, now)) {
			/* new sporadic release */
			release_at(task, now);
			sched_trace_task_release(task);
		}
		else {
			if (task->rt.time_slice) {
				/* came back in time before deadline
				*/
				set_rt_flags(task, RT_F_RUNNING);
			}
		}
	}
	/*Delay job arrival if we still have an active ghost job*/
	if (!is_ghost(task))
		mc_job_arrival(task);
	raw_spin_unlock_irqrestore(&global_lock, flags);
}

/* Called by LITMUS core
 * No lock required by caller
 * Obtains and releases global lock
 */
static void mc_task_block(struct task_struct *t)
{
	unsigned long flags;

	TRACE_TASK(t, "block at %llu\n", litmus_clock());

	/* unlink if necessary */
	raw_spin_lock_irqsave(&global_lock, flags);
	unlink(t);
	raw_spin_unlock_irqrestore(&global_lock, flags);

	BUG_ON(!is_realtime(t));
}


/* Called by LITMUS core
 * No lock required by caller
 * Obtains and releases global lock
 */
static void mc_task_exit(struct task_struct * t)
{
	unsigned long flags;

	/* unlink if necessary */
	raw_spin_lock_irqsave(&global_lock, flags);
	unlink(t);
	if (tsk_rt(t)->scheduled_on != NO_CPU) {
		mc_cpus[tsk_rt(t)->scheduled_on]->scheduled = NULL;
		tsk_rt(t)->scheduled_on = NO_CPU;
	}
	raw_spin_unlock_irqrestore(&global_lock, flags);

	BUG_ON(!is_realtime(t));
        TRACE_TASK(t, "RIP\n");
}

static long mc_admit_task(struct task_struct* tsk)
{
	if (!tsk_mc_data(tsk))
	{
		printk(KERN_WARNING "tried to admit task with no criticality "
			"level\n");
		return -EINVAL;
	}
	printk(KERN_INFO "admitted task with criticality level %d\n",
		tsk_mc_crit(tsk));
	return 0;
}

static long mc_activate_plugin(void)
{
	int cpu;
	cpu_entry_t *entry;

	bheap_init(&mc_cpu_heap_c);
	bheap_init(&mc_cpu_heap_d);
#ifdef CONFIG_RELEASE_MASTER
	crit_c.release_master = atomic_read(&release_master_cpu);
	crit_d.release_master = crit_c.release_master;
#endif

	for_each_online_cpu(cpu) {
		entry = &per_cpu(mc_cpu_entries, cpu);
		bheap_node_init(&entry->hn_c, entry);
		bheap_node_init(&entry->hn_d, entry);
		atomic_set(&entry->will_schedule, 0);
		entry->linked    = NULL;
		entry->scheduled = NULL;
#ifdef CONFIG_RELEASE_MASTER
		if (cpu != mc_release_master) {
#endif
			TRACE("MC: Initializing CPU #%d.\n", cpu);
			update_cpu_position(entry);
#ifdef CONFIG_RELEASE_MASTER
		} else {
			TRACE("MC: CPU %d is release master.\n", cpu);
		}
#endif
	}
	return 0;
}

/*	Plugin object	*/
static struct sched_plugin mc_plugin __cacheline_aligned_in_smp = {
	.plugin_name		= "MC",
	.finish_switch		= mc_finish_switch,
	.tick			= mc_tick,
	.task_new		= mc_task_new,
	.complete_job		= complete_job,
	.task_exit		= mc_task_exit,
	.schedule		= mc_schedule,
	.task_wake_up		= mc_task_wake_up,
	.task_block		= mc_task_block,
	.admit_task		= mc_admit_task,
	.activate_plugin	= mc_activate_plugin,
};


static int __init init_mc(void)
{
	int cpu;
	int i;
	cpu_entry_t *entry;
	struct watchdog_timer *timer;

	bheap_init(&mc_cpu_heap_c);
	bheap_init(&mc_cpu_heap_d);
	/* initialize CPU state */
	for (cpu = 0; cpu < NR_CPUS; cpu++)  {
		entry = &per_cpu(mc_cpu_entries, cpu);
		mc_cpus[cpu] = entry;
		atomic_set(&entry->will_schedule, 0);
		entry->cpu 	 = cpu;
		entry->hn_c      = &mc_heap_node_c[cpu];
		entry->hn_d      = &mc_heap_node_d[cpu];
		bheap_node_init(&entry->hn_c, entry);
		bheap_node_init(&entry->hn_d, entry);
		for (i = CRIT_LEVEL_A; i < NUM_CRIT_LEVELS; i++){
			timer = ghost_timer(cpu, i);
			hrtimer_init(&timer->timer, CLOCK_MONOTONIC,
				HRTIMER_MODE_ABS);
			timer->timer.function = watchdog_timeout;
		}
	}
	mc_edf_domain_init(&crit_c, NULL, mc_release_jobs);
	mc_edf_domain_init(&crit_d, NULL, mc_release_jobs);
	for (i = 0; i < NR_CPUS; i++){
		mc_edf_domain_init(remote_b_queue(i), NULL,
				   mc_release_jobs);
	}
	for (i = 0; i < NR_CPUS; i++){
		mc_edf_domain_init(remote_a_queue(i), NULL,
				   mc_release_jobs);
	}
	return register_sched_plugin(&mc_plugin);
}


module_init(init_mc);