path: root/litmus/sched_cedf.c

/*
 * litmus/sched_cedf.c
 *
 * Implementation of the C-EDF scheduling algorithm.
 *
 * This implementation is based on G-EDF:
 * - CPUs are clustered around L2 or L3 caches.
 * - Clusters topology is automatically detected (this is arch dependent
 *   and is working only on x86 at the moment --- and only with modern
 *   cpus that exports cpuid4 information)
 * - The plugins _does not_ attempt to put tasks in the right cluster i.e.
 *   the programmer needs to be aware of the topology to place tasks
 *   in the desired cluster
 * - default clustering is around L2 cache (cache index = 2)
 *   supported clusters are: L1 (private cache: pedf), L2, L3, ALL (all
 *   online_cpus are placed in a single cluster).
 *
 *   For details on functions, take a look at sched_gsn_edf.c
 *
 * Currently, we do not support changes in the number of online cpus.
 * If the num_online_cpus() dynamically changes, the plugin is broken.
 *
 * This version uses the simple approach and serializes all scheduling
 * decisions by the use of a queue lock. This is probably not the
 * best way to do it, but it should suffice for now.
 */

#include <linux/spinlock.h>
#include <linux/percpu.h>
#include <linux/sched.h>
#include <linux/slab.h>

#include <linux/module.h>

#include <litmus/litmus.h>
#include <litmus/wait.h>
#include <litmus/jobs.h>
#include <litmus/preempt.h>
#include <litmus/sched_plugin.h>
#include <litmus/edf_common.h>
#include <litmus/sched_trace.h>
#include <litmus/trace.h>

#include <litmus/clustered.h>

#include <litmus/bheap.h>

/* to configure the cluster size */
#include <litmus/litmus_proc.h>
#include <linux/uaccess.h>

/* Reference configuration variable. Determines which cache level is used to
 * group CPUs into clusters.  GLOBAL_CLUSTER, which is the default, means that
 * all CPUs form a single cluster (just like GSN-EDF).
 */
static enum cache_level cluster_config = GLOBAL_CLUSTER;

struct clusterdomain;

/* cpu_entry_t - maintain the linked and scheduled state
 *
 * A cpu also contains a pointer to the cedf_domain_t cluster
 * that owns it (struct clusterdomain*)
 */
typedef struct  {
	int 			cpu;
	struct clusterdomain*	cluster;	/* owning cluster */
	struct task_struct*	linked;		/* only RT tasks */
	struct task_struct*	scheduled;	/* only RT tasks */
	atomic_t		will_schedule;	/* prevent unneeded IPIs */
	struct bheap_node*	hn;
#ifdef CONFIG_LITMUS_LOCKING
	struct bheap_node*	pending_hn;
	struct task_struct*	pending;
#endif
} cpu_entry_t;

/* one cpu_entry_t per CPU */
DEFINE_PER_CPU(cpu_entry_t, cedf_cpu_entries);

/*
 * In C-EDF there is a cedf domain _per_ cluster
 * The number of clusters is dynamically determined accordingly to the
 * total cpu number and the cluster size
 */
typedef struct clusterdomain {
	/* rt_domain for this cluster */
	rt_domain_t	domain;
	/* cpus in this cluster */
	cpu_entry_t*	*cpus;
	/* map of this cluster cpus */
	cpumask_var_t	cpu_map;
	/* the cpus queue themselves according to priority in here */
	struct bheap_node *heap_node;
	struct bheap    cpu_heap;
#ifdef CONFIG_LITMUS_LOCKING
	struct bheap	pending_jobs;
	struct bheap	pending_cpus;
	struct bheap_node *pending_nodes;
#endif
	/* lock for this cluster */
#define cluster_lock domain.ready_lock
} cedf_domain_t;

/* a cedf_domain per cluster; allocation is done at init/activation time */
cedf_domain_t *cedf;

#define remote_cluster(cpu)	((cedf_domain_t *) per_cpu(cedf_cpu_entries, cpu).cluster)
#define task_cpu_cluster(task)	remote_cluster(get_partition(task))

/* Uncomment WANT_ALL_SCHED_EVENTS if you want to see all scheduling
 * decisions in the TRACE() log; uncomment VERBOSE_INIT for verbose
 * information during the initialization of the plugin (e.g., topology)
#define WANT_ALL_SCHED_EVENTS
 */
#define VERBOSE_INIT

static int cpu_lower_prio(struct bheap_node *_a, struct bheap_node *_b)
{
	cpu_entry_t *a, *b;
	a = _a->value;
	b = _b->value;
	/* Note that a and b are inverted: we want the lowest-priority CPU at
	 * the top of the heap.
	 */
	return edf_higher_prio(b->linked, a->linked);
}

/* update_cpu_position - Move the cpu entry to the correct place to maintain
 *                       order in the cpu queue. Caller must hold cedf lock.
 */
static void update_cpu_position(cpu_entry_t *entry)
{
	cedf_domain_t *cluster = entry->cluster;

	if (likely(bheap_node_in_heap(entry->hn)))
		bheap_delete(cpu_lower_prio,
				&cluster->cpu_heap,
				entry->hn);

	bheap_insert(cpu_lower_prio, &cluster->cpu_heap, entry->hn);
}

/* caller must hold cedf lock */
static cpu_entry_t* lowest_prio_cpu(cedf_domain_t *cluster)
{
	struct bheap_node* hn;
	hn = bheap_peek(cpu_lower_prio, &cluster->cpu_heap);
	return hn->value;
}


/* link_task_to_cpu - Update the link of a CPU.
 *                    Handles the case where the to-be-linked task is already
 *                    scheduled on a different CPU.
 */
static noinline void link_task_to_cpu(struct task_struct* linked,
				      cpu_entry_t *entry)
{
	cpu_entry_t *sched;
	struct task_struct* tmp;
	int on_cpu;

	BUG_ON(linked && !is_realtime(linked));

	/* Currently linked task is set to be unlinked. */
	if (entry->linked) {
		entry->linked->rt_param.linked_on = NO_CPU;
	}

	/* Link new task to CPU. */
	if (linked) {
		/* handle task is already scheduled somewhere! */
		on_cpu = linked->rt_param.scheduled_on;
		if (on_cpu != NO_CPU) {
			sched = &per_cpu(cedf_cpu_entries, on_cpu);
			/* this should only happen if not linked already */
			BUG_ON(sched->linked == linked);

			/* If we are already scheduled on the CPU to which we
			 * wanted to link, we don't need to do the swap --
			 * we just link ourselves to the CPU and depend on
			 * the caller to get things right.
			 */
			if (entry != sched) {
				TRACE_TASK(linked,
					   "already scheduled on %d, updating link.\n",
					   sched->cpu);
				tmp = sched->linked;
				linked->rt_param.linked_on = sched->cpu;
				sched->linked = linked;
				update_cpu_position(sched);
				linked = tmp;
			}
		}
		if (linked) /* might be NULL due to swap */
			linked->rt_param.linked_on = entry->cpu;
	}
	entry->linked = linked;
#ifdef WANT_ALL_SCHED_EVENTS
	if (linked)
		TRACE_TASK(linked, "linked to %d.\n", entry->cpu);
	else
		TRACE("NULL linked to %d.\n", entry->cpu);
#endif
	update_cpu_position(entry);
}

/* unlink - Make sure a task is not linked any longer to an entry
 *          where it was linked before. Must hold cedf_lock.
 */
static noinline void unlink(struct task_struct* t)
{
    	cpu_entry_t *entry;

	if (t->rt_param.linked_on != NO_CPU) {
		/* unlink */
		entry = &per_cpu(cedf_cpu_entries, t->rt_param.linked_on);
		t->rt_param.linked_on = NO_CPU;
		link_task_to_cpu(NULL, entry);
	} else if (is_queued(t)) {
		/* This is an interesting situation: t is scheduled,
		 * but was just recently unlinked.  It cannot be
		 * linked anywhere else (because then it would have
		 * been relinked to this CPU), thus it must be in some
		 * queue. We must remove it from the list in this
		 * case.
		 *
		 * in C-EDF case is should be somewhere in the queue for
		 * its domain, therefore and we can get the domain using
		 * task_cpu_cluster
		 */
		remove(&(task_cpu_cluster(t))->domain, t);
	}
}


/* preempt - force a CPU to reschedule
 */
static void preempt(cpu_entry_t *entry)
{
	preempt_if_preemptable(entry->scheduled, entry->cpu);
}

/* requeue - Put an unlinked task into gsn-edf domain.
 *           Caller must hold cedf_lock.
 */
static noinline void requeue(struct task_struct* task)
{
	cedf_domain_t *cluster = task_cpu_cluster(task);
	BUG_ON(!task);
	/* sanity check before insertion */
	BUG_ON(is_queued(task));

	if (is_released(task, litmus_clock()))
		__add_ready(&cluster->domain, task);
	else {
		/* it has got to wait */
		add_release(&cluster->domain, task);
	}
}

#ifdef CONFIG_LITMUS_LOCKING
static int update_pending_job(cedf_domain_t* cluster, struct task_struct* t);
#endif

/* check for any necessary preemptions */
static void check_for_preemptions(cedf_domain_t *cluster)
{
	struct task_struct *task;
	cpu_entry_t* last;

	for(last = lowest_prio_cpu(cluster);
	    edf_preemption_needed(&cluster->domain, last->linked);
	    last = lowest_prio_cpu(cluster)) {
		/* preemption necessary */

#ifdef CONFIG_LITMUS_LOCKING
		task = __peek_ready(&cluster->domain);
		if (update_pending_job(cluster, task)) {
			/* Something changed, re-evaluate priorites to
			 * see if we still need to preempt.
			 * */
			TRACE_TASK(task, "hitting continue\n");
			continue;
		}
#endif
		task = __take_ready(&cluster->domain);
		TRACE_TASK(task, "attempting to link task to P%d\n",
			   last->cpu);
		if (last->linked)
			requeue(last->linked);
		link_task_to_cpu(task, last);
		preempt(last);
	}
}

/* cedf_job_arrival: task is either resumed or released */
static noinline void cedf_job_arrival(struct task_struct* task)
{
	cedf_domain_t *cluster = task_cpu_cluster(task);
	BUG_ON(!task);

	requeue(task);
	check_for_preemptions(cluster);
}

#ifdef CONFIG_LITMUS_LOCKING

static inline int in_pending_heap(struct task_struct* t)
{
	return bheap_node_in_heap(tsk_rt(t)->pending_node);
}

/* has this task already been processed for pending */
static inline int is_pending(struct task_struct* t)
{
	return tsk_rt(t)->pending_on != NO_CPU ||
		in_pending_heap(t);
}


static int pending_lower_prio(struct bheap_node *_a, struct bheap_node *_b)
{
	cpu_entry_t *a, *b;
	a = _a->value;
	b = _b->value;
	/* Note that a and b are inverted: we want the lowest-priority CPU at
	 * the top of the heap.
	 */
	return edf_higher_base_prio(b->pending, a->pending);
}

/* update_cpu_position - Move the cpu entry to the correct place to maintain
 *                       order in the cpu queue. Caller must hold cedf lock.
 */
static void update_pending_position(cpu_entry_t *entry)
{
	cedf_domain_t *cluster = entry->cluster;

	if (likely(bheap_node_in_heap(entry->pending_hn)))
		bheap_delete(pending_lower_prio,
			     &cluster->pending_cpus,
			     entry->pending_hn);

	bheap_insert(pending_lower_prio, &cluster->pending_cpus, entry->pending_hn);
}

/* caller must hold cedf lock */
static cpu_entry_t* lowest_pending_cpu(cedf_domain_t *cluster)
{
	struct bheap_node* hn;
	hn = bheap_peek(pending_lower_prio, &cluster->pending_cpus);
	return hn->value;
}

static void priority_raised(struct task_struct* t)
{
	cedf_domain_t *cluster = task_cpu_cluster(t);
	int linked_on;

	linked_on  = tsk_rt(t)->linked_on;

	/* If it is scheduled, then we need to reorder the CPU heap. */
	if (linked_on != NO_CPU) {
		TRACE_TASK(t, "%s: linked  on %d\n",
			   __FUNCTION__, linked_on);
		/* Holder is scheduled; need to re-order CPUs.
		 * We can't use heap_decrease() here since
		 * the cpu_heap is ordered in reverse direction, so
		 * it is actually an increase. */
		bheap_delete(cpu_lower_prio, &cluster->cpu_heap,
			    cluster->cpus[linked_on]->hn);
		bheap_insert(cpu_lower_prio, &cluster->cpu_heap,
			    cluster->cpus[linked_on]->hn);
	} else {
		/* holder may be queued: first stop queue changes */
		raw_spin_lock(&cluster->domain.release_lock);
		if (is_queued(t)) {
			TRACE_TASK(t, "%s: is queued\n",
				   __FUNCTION__);
			bheap_decrease(edf_ready_order,
				       tsk_rt(t)->heap_node);
		} else {
			/* Nothing to do: if it is not queued and not linked
			 * then it is either sleeping or currently being moved
			 * by other code (e.g., a timer interrupt handler) that
			 * will use the correct priority when enqueuing the
			 * task. */
			TRACE_TASK(t, "%s: is NOT queued => Done.\n",
				   __FUNCTION__);
		}
		raw_spin_unlock(&cluster->domain.release_lock);
	}
}

static void priority_lowered(struct task_struct* t)
{
	/* assumption: t is not in a release heap */
	if (is_queued(t) || tsk_rt(t)->linked_on != NO_CPU) {
		unlink(t);
		requeue(t);
	}
}

static void donate_priority(struct task_struct* recipient, struct task_struct* donor)
{
	cedf_domain_t *cluster = task_cpu_cluster(donor);

	BUG_ON(task_cpu_cluster(recipient) != task_cpu_cluster(donor));
	BUG_ON(tsk_rt(donor)->is_donor);
	BUG_ON(tsk_rt(recipient)->is_donor);
	BUG_ON(tsk_rt(donor)->inh_task);
	BUG_ON(tsk_rt(recipient)->inh_task);

	TRACE_TASK(donor, "priodon: becomes priority donor for %s/%d\n",
		   recipient->comm, recipient->pid);

	/* swap priorities */
	tsk_rt(recipient)->inh_task = donor;
	tsk_rt(donor)->inh_task = recipient;
	tsk_rt(donor)->is_donor = 1;

	priority_lowered(donor);
	priority_raised(recipient);

	bheap_uncache_min(edf_ready_order,
			  &cluster->domain.ready_queue);
}

/* assumption: new_donor has a higher priority than old_donor */
static void switch_donor(struct task_struct* recipient,
			 struct task_struct* old_donor,
			 struct task_struct* new_donor)
{
	TRACE_TASK(new_donor, "becomes donor for %s/%d instead of %s/%d\n",
		   recipient->comm, recipient->pid, old_donor->comm, old_donor->pid);

	BUG_ON(tsk_rt(recipient)->inh_task != old_donor);
	BUG_ON(tsk_rt(old_donor)->inh_task != recipient);
	BUG_ON(tsk_rt(new_donor)->inh_task != NULL);
	BUG_ON(tsk_rt(new_donor)->is_donor);

	tsk_rt(old_donor)->inh_task = NULL;
	tsk_rt(old_donor)->is_donor = 0;

	tsk_rt(recipient)->inh_task = new_donor;
	tsk_rt(new_donor)->inh_task = recipient;
	tsk_rt(new_donor)->is_donor = 1;

	priority_raised(recipient);
	priority_raised(old_donor);
	priority_lowered(new_donor);
}

static void undonate_priority(struct task_struct* recipient, struct task_struct* donor)
{
	cedf_domain_t *cluster = task_cpu_cluster(donor);

	BUG_ON(tsk_rt(recipient)->inh_task != donor);
	BUG_ON(tsk_rt(donor)->inh_task != recipient);

	TRACE_TASK(donor, "priodon: is no longer priority donor of  %s/%d\n",
		   recipient->comm, recipient->pid);

	tsk_rt(recipient)->inh_task = NULL;
	tsk_rt(donor)->inh_task = NULL;
	tsk_rt(donor)->is_donor = 0;

	priority_lowered(recipient);
	priority_raised(donor);

	bheap_uncache_min(edf_ready_order,
			  &cluster->domain.ready_queue);
}

static inline void add_to_pending(cedf_domain_t* cluster, struct task_struct* t)
{
	TRACE_TASK(t, "priodon: adding to pending heap wait:%u donor:%u req:%u pend:%d\n",
		   tsk_rt(t)->waiting_eligible,
		   tsk_rt(t)->is_donor, tsk_rt(t)->request_incomplete,
		   tsk_rt(t)->pending_on);
	bheap_insert(edf_pending_order,
		     &cluster->pending_jobs,
		     tsk_rt(t)->pending_node);
}

static inline struct task_struct* take_pending(cedf_domain_t* cluster)
{
	struct bheap_node* node;
	node = bheap_take(edf_pending_order, &cluster->pending_jobs);
	return node ? (struct task_struct*) node->value : NULL;
}

static inline struct task_struct* peek_pending(cedf_domain_t* cluster)
{
	struct bheap_node* node;
	node = bheap_peek(edf_pending_order, &cluster->pending_jobs);
	return node ? (struct task_struct*) node->value : NULL;
}

static inline int fake_resume(struct task_struct* t)
{
	TRACE_TASK(t, "priodon: fake resume wait:%u donor:%u\n",
		   tsk_rt(t)->waiting_eligible, tsk_rt(t)->is_donor);
	/* Fake suspended. Let's resume it. */
	if (tsk_rt(t)->waiting_eligible) {
		tsk_rt(t)->waiting_eligible = 0;
		if (tsk_rt(t)->scheduled_on == NO_CPU) {
			/* it was removed from the queue */
			requeue(t);
			return 1;
		}
	}
	return 0;
}


/* Lazily update set of highest-priority pending jobs.
 * Returns 1 if priority recheck is required.
 */
static int update_pending_job(cedf_domain_t* cluster,
			      struct task_struct* to_be_linked)
{
	cpu_entry_t* entry;
	struct task_struct* lowest_hp; /* lowest-priority high-priority task */
	struct task_struct* highest_lp; /* highest-priority low-priority task */
	int reeval = 0;

	entry = lowest_pending_cpu(cluster);
	lowest_hp = entry->pending;

	if (to_be_linked && !is_pending(to_be_linked))
		/* not yet accounted for, stick in heap */
		add_to_pending(cluster, to_be_linked);

	highest_lp = peek_pending(cluster);
	if (edf_higher_base_prio(highest_lp, lowest_hp)) {
		/* yep, should be become of the c highest-prior pending jobs */

		TRACE_TASK(highest_lp,
			   "priodon: became one of the c highest-prio tasks (P%d, req:%u) X\n",
			   entry->cpu,
			   tsk_rt(highest_lp)->request_incomplete);

		/* get it out of the heap */
		highest_lp = take_pending(cluster);

		BUG_ON(highest_lp == lowest_hp);

		/* it should never be a priority donor at this point */
		BUG_ON(tsk_rt(highest_lp)->is_donor);

		entry->pending = highest_lp;
		update_pending_position(entry);
		tsk_rt(highest_lp)->pending_on = entry->cpu;

		/* things that could happen:
		 *
		 * 1) lowest_hp has no donor, but is in a request => highest_lp becomes donor
		 * 2) lowest_hp is donor => highest_lp becomes new donor, old donor is resumed if suspended
		 * 3) lowest_hp is not in a request, and highest_lp is waiting => highest_lp is resumed
		 * 4) lowest_hp is not in a request, and highest_lp is not waiting => nothing to do
		 * 5) highest_lp has a priority donor => resume its donor
		 */

		/* do we need to put it back? */
		if (lowest_hp) {
			TRACE_TASK(lowest_hp,
				   "priodon: no longer among c highest-prio tasks req:%u\n",
				   tsk_rt(lowest_hp)->request_incomplete);
			tsk_rt(lowest_hp)->pending_on = NO_CPU;
			add_to_pending(cluster, lowest_hp);


			if (tsk_rt(lowest_hp)->request_incomplete) {
				/* case 1) */
				donate_priority(lowest_hp, highest_lp);
				reeval = 1;
			} else if (tsk_rt(lowest_hp)->inh_task) {
				/* case 2) */
				switch_donor(tsk_rt(lowest_hp)->inh_task,
					     lowest_hp, highest_lp);
				fake_resume(lowest_hp);
				reeval = 1;
			}
		}


		if (!tsk_rt(highest_lp)->is_donor) {
			if (tsk_rt(highest_lp)->waiting_eligible) {
				/* case 3) */
				reeval = fake_resume(highest_lp);
				BUG_ON(tsk_rt(highest_lp)->inh_task);
			} else if (tsk_rt(highest_lp)->inh_task) {
				/* case 5 */
				struct task_struct* donor = tsk_rt(highest_lp)->inh_task;
				undonate_priority(highest_lp, donor);
				reeval = fake_resume(donor);
			}
		}
	}

	return reeval;
}

/* job has exited => no longer pending */
static void job_pending_exit(struct task_struct* t)
{
	cedf_domain_t *cluster;
	cpu_entry_t* entry;

	TRACE_TASK(t, "priodon: is no longer pending (pending_on:%d, queued:%d)\n",
		   tsk_rt(t)->pending_on, in_pending_heap(t));

	cluster = task_cpu_cluster(t);

	if (tsk_rt(t)->pending_on != NO_CPU) {
		entry = &per_cpu(cedf_cpu_entries, tsk_rt(t)->pending_on);
		tsk_rt(t)->pending_on = NO_CPU;
		entry->pending = NULL;
		update_pending_position(entry);

		/* let's see if anything changed */
		update_pending_job(cluster, NULL);
	} else if (in_pending_heap(t)) {
		bheap_delete(edf_pending_order, &cluster->pending_jobs,
			     tsk_rt(t)->pending_node);
	}
}

#endif

static void cedf_release_jobs(rt_domain_t* rt, struct bheap* tasks)
{
	cedf_domain_t* cluster = container_of(rt, cedf_domain_t, domain);
	unsigned long flags;

	raw_spin_lock_irqsave(&cluster->cluster_lock, flags);

#if 0
	while (!bheap_empty(tasks)) {
		hn = bheap_take(edf_ready_order, tasks);
		t = bheap2task(hn);
		TRACE_TASK(t, "released (part:%d)\n",
			   get_partition(t));
		new_pending_job(cluster, t);
		requeue(t);
	}
#else
	__merge_ready(&cluster->domain, tasks);
#endif
	check_for_preemptions(cluster);

	raw_spin_unlock_irqrestore(&cluster->cluster_lock, flags);
}

/* caller holds cedf_lock */
static noinline void job_completion(struct task_struct *t, int forced)
{
	BUG_ON(!t);

	sched_trace_task_completion(t, forced);

	TRACE_TASK(t, "job_completion().\n");

#ifdef CONFIG_LITMUS_LOCKING
	job_pending_exit(t);
#endif

	/* prepare for next period */
	prepare_for_next_period(t);
	if (is_released(t, litmus_clock()))
		sched_trace_task_release(t);
	/* unlink */
	unlink(t);
	/* requeue
	 * But don't requeue a blocking task. */
	set_rt_flags(t, RT_F_RUNNING);
	if (is_running(t))
		cedf_job_arrival(t);
}

/* cedf_tick - this function is called for every local timer
 *                         interrupt.
 *
 *                   checks whether the current task has expired and checks
 *                   whether we need to preempt it if it has not expired
 */
static void cedf_tick(struct task_struct* t)
{
	if (is_realtime(t) && budget_enforced(t) && budget_exhausted(t)) {
		if (!is_np(t)) {
			/* np tasks will be preempted when they become
			 * preemptable again
			 */
			litmus_reschedule_local();
			TRACE("cedf_scheduler_tick: "
			      "%d is preemptable "
			      " => FORCE_RESCHED\n", t->pid);
		} else if (is_user_np(t)) {
			TRACE("cedf_scheduler_tick: "
			      "%d is non-preemptable, "
			      "preemption delayed.\n", t->pid);
			request_exit_np(t);
		}
	}
}

/* Getting schedule() right is a bit tricky. schedule() may not make any
 * assumptions on the state of the current task since it may be called for a
 * number of reasons. The reasons include a scheduler_tick() determined that it
 * was necessary, because sys_exit_np() was called, because some Linux
 * subsystem determined so, or even (in the worst case) because there is a bug
 * hidden somewhere. Thus, we must take extreme care to determine what the
 * current state is.
 *
 * The CPU could currently be scheduling a task (or not), be linked (or not).
 *
 * The following assertions for the scheduled task could hold:
 *
 *      - !is_running(scheduled)        // the job blocks
 *	- scheduled->timeslice == 0	// the job completed (forcefully)
 *	- get_rt_flag() == RT_F_SLEEP	// the job completed (by syscall)
 * 	- linked != scheduled		// we need to reschedule (for any reason)
 * 	- is_np(scheduled)		// rescheduling must be delayed,
 *					   sys_exit_np must be requested
 *
 * Any of these can occur together.
 */
static struct task_struct* cedf_schedule(struct task_struct * prev)
{
	cpu_entry_t* entry = &__get_cpu_var(cedf_cpu_entries);
	cedf_domain_t *cluster = entry->cluster;
	int out_of_time, sleep, preempt, np, exists, blocks;
	struct task_struct* next = NULL;

#ifdef CONFIG_LITMUS_LOCKING
	int priodon;
#else
#define priodon 0
#endif

#ifdef CONFIG_RELEASE_MASTER
	/* Bail out early if we are the release master.
	 * The release master never schedules any real-time tasks.
	 */
	if (cluster->domain.release_master == entry->cpu) {
		sched_state_task_picked();
		return NULL;
	}
#endif

	raw_spin_lock(&cluster->cluster_lock);

	/* sanity checking */
	BUG_ON(entry->scheduled && entry->scheduled != prev);
	BUG_ON(entry->scheduled && !is_realtime(prev));
	BUG_ON(is_realtime(prev) && !entry->scheduled);

	/* (0) Determine state */
	exists      = entry->scheduled != NULL;
	blocks      = exists && !is_running(entry->scheduled);
	out_of_time = exists &&
				  budget_enforced(entry->scheduled) &&
				  budget_exhausted(entry->scheduled);
	np 	    = exists && is_np(entry->scheduled);
	sleep	    = exists && get_rt_flags(entry->scheduled) == RT_F_SLEEP;
	preempt     = entry->scheduled != entry->linked;

#ifdef CONFIG_LITMUS_LOCKING
	priodon = exists && (tsk_rt(entry->scheduled)->waiting_eligible ||
			     /* can't allow job to exit until request is over */
			     (tsk_rt(entry->scheduled)->is_donor && sleep));

	/* this should never happend together (at least we don't handle it atm) */
	BUG_ON(priodon && blocks);
#endif

#ifdef WANT_ALL_SCHED_EVENTS
	TRACE_TASK(prev, "invoked cedf_schedule.\n");
#endif

	if (exists)
		TRACE_TASK(prev,
			   "blocks:%d out_of_time:%d np:%d sleep:%d preempt:%d "
			   "state:%d sig:%d priodon:%d\n",
			   blocks, out_of_time, np, sleep, preempt,
			   prev->state, signal_pending(prev), priodon);
	if (entry->linked && preempt)
		TRACE_TASK(prev, "will be preempted by %s/%d\n",
			   entry->linked->comm, entry->linked->pid);


	/* If a task blocks we have no choice but to reschedule.
	 */
	if (blocks || priodon)
		unlink(entry->scheduled);

	/* Request a sys_exit_np() call if we would like to preempt but cannot.
	 * Do not unlink since entry->scheduled is currently in the ready queue.
	 * We don't process out_of_time and sleep until the job is preemptive again.
	 */
	if (np && (out_of_time || preempt || sleep)) {
		request_exit_np(entry->scheduled);
	}

	/* Any task that is preemptable and either exhausts its execution
	 * budget or wants to sleep completes. We may have to reschedule after
	 * this. Don't do a job completion if we block (can't have timers running
	 * for blocked jobs). Preemption go first for the same reason.
	 */
	if (!np && (out_of_time || sleep) && !blocks && !preempt
	    && !priodon)
		/* note: priority donation prevents job completion */
		job_completion(entry->scheduled, !sleep);

	/* Link pending task if we became unlinked.
	 */

	if (!entry->linked) {
		struct task_struct *pulled = __take_ready(&cluster->domain);
#ifdef CONFIG_LITMUS_LOCKING
		if (pulled && !is_pending(pulled)) {
			/* pulled an un-processed task from the ready queue
			 * It's certainly not among the top c highest-priority
			 * pending jobs: if it were, it would have showed up in
			 * check_for_preemptions() or in job_completion(). So
			 * just add in into the pending heap.
			 */
			TRACE_TASK(pulled, "pulled unprocessed\n");
			add_to_pending(cluster, pulled);
		}
#endif
		link_task_to_cpu(pulled, entry);
	}

	/* The final scheduling decision. Do we need to switch for some reason?
	 * If linked is different from scheduled, then select linked as next.
	 */
	if ((!np || blocks || priodon) &&
	    entry->linked != entry->scheduled) {
		/* Schedule a linked job? */
		if (entry->linked) {
			entry->linked->rt_param.scheduled_on = entry->cpu;
			next = entry->linked;
		}
		if (entry->scheduled) {
			/* not gonna be scheduled soon */
			entry->scheduled->rt_param.scheduled_on = NO_CPU;
			TRACE_TASK(entry->scheduled, "scheduled_on = NO_CPU\n");
		}
	} else
		/* Only override Linux scheduler if we have a real-time task
		 * scheduled that needs to continue.
		 */
		if (exists)
			next = prev;

	sched_state_task_picked();
	raw_spin_unlock(&cluster->cluster_lock);

#ifdef WANT_ALL_SCHED_EVENTS
	TRACE("cedf_lock released, next=0x%p\n", next);

	if (next)
		TRACE_TASK(next, "scheduled at %llu\n", litmus_clock());
	else if (exists && !next)
		TRACE("becomes idle at %llu.\n", litmus_clock());
#endif


	return next;
}


/* _finish_switch - we just finished the switch away from prev
 */
static void cedf_finish_switch(struct task_struct *prev)
{
	cpu_entry_t* 	entry = &__get_cpu_var(cedf_cpu_entries);

	entry->scheduled = is_realtime(current) ? current : NULL;
#ifdef WANT_ALL_SCHED_EVENTS
	TRACE_TASK(prev, "switched away from\n");
#endif
}


/*	Prepare a task for running in RT mode
 */
static void cedf_task_new(struct task_struct * t, int on_rq, int running)
{
	unsigned long 		flags;
	cpu_entry_t* 		entry;
	cedf_domain_t*		cluster;

	TRACE("gsn edf: task new %d\n", t->pid);

	/* the cluster doesn't change even if t is running */
	cluster = task_cpu_cluster(t);

	raw_spin_lock_irqsave(&cluster->cluster_lock, flags);

	/* setup job params */
	release_at(t, litmus_clock());

#ifdef CONFIG_LITMUS_LOCKING
	tsk_rt(t)->pending_node = bheap_node_alloc(GFP_ATOMIC | __GFP_NOFAIL);
	bheap_node_init(&tsk_rt(t)->pending_node, t);
	tsk_rt(t)->pending_on = NO_CPU;
	add_to_pending(cluster, t);
#endif

	if (running) {
		entry = &per_cpu(cedf_cpu_entries, task_cpu(t));
		BUG_ON(entry->scheduled);

#ifdef CONFIG_RELEASE_MASTER
		if (entry->cpu != cluster->domain.release_master) {
#endif
			entry->scheduled = t;
			tsk_rt(t)->scheduled_on = task_cpu(t);
#ifdef CONFIG_RELEASE_MASTER
		} else {
			/* do not schedule on release master */
			preempt(entry); /* force resched */
			tsk_rt(t)->scheduled_on = NO_CPU;
		}
#endif
	} else {
		t->rt_param.scheduled_on = NO_CPU;
	}
	t->rt_param.linked_on          = NO_CPU;

	cedf_job_arrival(t);
	raw_spin_unlock_irqrestore(&(cluster->cluster_lock), flags);
}

static void cedf_task_wake_up(struct task_struct *task)
{
	unsigned long flags;
	lt_t now;
	cedf_domain_t *cluster;

	TRACE_TASK(task, "wake_up at %llu\n", litmus_clock());

	cluster = task_cpu_cluster(task);

	raw_spin_lock_irqsave(&cluster->cluster_lock, flags);
	/* We need to take suspensions because of semaphores into
	 * account! If a job resumes after being suspended due to acquiring
	 * a semaphore, it should never be treated as a new job release.
	 */
	if (get_rt_flags(task) == RT_F_EXIT_SEM) {
		set_rt_flags(task, RT_F_RUNNING);
	} else {
		now = litmus_clock();
		if (is_tardy(task, now)) {
			/* new sporadic release */
			release_at(task, now);
			sched_trace_task_release(task);
		}
		else {
			if (task->rt.time_slice) {
				/* came back in time before deadline
				*/
				set_rt_flags(task, RT_F_RUNNING);
			}
		}
	}
	cedf_job_arrival(task);
	raw_spin_unlock_irqrestore(&cluster->cluster_lock, flags);
}

static void cedf_task_block(struct task_struct *t)
{
	unsigned long flags;
	cedf_domain_t *cluster;

	TRACE_TASK(t, "block at %llu\n", litmus_clock());

	cluster = task_cpu_cluster(t);

	/* unlink if necessary */
	raw_spin_lock_irqsave(&cluster->cluster_lock, flags);
	unlink(t);
	raw_spin_unlock_irqrestore(&cluster->cluster_lock, flags);

	BUG_ON(!is_realtime(t));
}


static void cedf_task_exit(struct task_struct * t)
{
	unsigned long flags;
	cedf_domain_t *cluster = task_cpu_cluster(t);


	/* unlink if necessary */
	raw_spin_lock_irqsave(&cluster->cluster_lock, flags);

	unlink(t);

#ifdef CONFIG_LITMUS_LOCKING
	/* make sure it's not pending anymore */
	job_pending_exit(t);
	bheap_node_free(tsk_rt(t)->pending_node);
#endif

	if (tsk_rt(t)->scheduled_on != NO_CPU) {
		cpu_entry_t *cpu;
		cpu = &per_cpu(cedf_cpu_entries, tsk_rt(t)->scheduled_on);
		cpu->scheduled = NULL;
		tsk_rt(t)->scheduled_on = NO_CPU;
	}
	raw_spin_unlock_irqrestore(&cluster->cluster_lock, flags);


	BUG_ON(!is_realtime(t));
        TRACE_TASK(t, "RIP\n");
}

#ifdef CONFIG_LITMUS_LOCKING

#include <litmus/fdso.h>
#include <litmus/locking.h>

/* NOTE: we use fake suspensions because we must wake the task from within the
 * scheduler */

/* suspend until the current task becomes eligible to issue a lock request */
static void priodon_become_eligible(void)
{
	struct task_struct* t = current;
	unsigned long flags;
	cedf_domain_t *cluster;

	cluster = task_cpu_cluster(t);

	do {
		TRACE_CUR("priodon: checking whether request may be issued\n");
		raw_spin_lock_irqsave(&cluster->cluster_lock, flags);

		if (tsk_rt(t)->pending_on == NO_CPU ||
		    tsk_rt(t)->is_donor) {
			/* nope, gotta wait */
			tsk_rt(t)->waiting_eligible = 1;
			TRACE_CUR("priodon: not eligible pend:%u donor:%u\n",
				  tsk_rt(t)->pending_on, tsk_rt(t)->is_donor);
		} else {
			/* alright! we are good to go! */
			tsk_rt(t)->request_incomplete = 1;
			TRACE_CUR("priodon: request issued\n");
		}

		raw_spin_unlock_irqrestore(&cluster->cluster_lock, flags);

		if (tsk_rt(t)->waiting_eligible) {
			TRACE_CUR("priodon: fake suspending\n");
			TS_LOCK_SUSPEND;
			schedule();
			TS_LOCK_RESUME;
		}

	} while (!tsk_rt(t)->request_incomplete);
}

/* current task has completed its request */
static void priodon_complete_request(void)
{
	struct task_struct* t = current;
	struct task_struct* donor;
	unsigned long flags;
	cedf_domain_t *cluster;

	cluster = task_cpu_cluster(t);

	preempt_disable();

	raw_spin_lock_irqsave(&cluster->cluster_lock, flags);

	TRACE_CUR("priodon: completing request\n");

	if (tsk_rt(t)->inh_task) {
		/* we have a donor job --- see if we need to wake it */
		donor = tsk_rt(t)->inh_task;
		undonate_priority(t, donor);

		if (fake_resume(donor))
			check_for_preemptions(cluster);
	}

	tsk_rt(t)->request_incomplete = 0;

	raw_spin_unlock_irqrestore(&cluster->cluster_lock, flags);

	preempt_enable();
}

/* struct for semaphore with priority inheritance */
struct omlp_semaphore {
	struct litmus_lock litmus_lock;

	/* current resource holder */
	struct task_struct *owner;

	/* FIFO queue of waiting tasks */
	wait_queue_head_t fifo_wait;
};

static inline struct omlp_semaphore* omlp_from_lock(struct litmus_lock* lock)
{
	return container_of(lock, struct omlp_semaphore, litmus_lock);
}

static int cedf_omlp_lock(struct litmus_lock* l)
{
	struct task_struct* t = current;
	struct omlp_semaphore *sem = omlp_from_lock(l);
	wait_queue_t wait;
	unsigned long flags;

	if (!is_realtime(t))
		return -EPERM;

	priodon_become_eligible();

	spin_lock_irqsave(&sem->fifo_wait.lock, flags);

	if (sem->owner) {
		/* resource is not free => must suspend and wait */

		init_waitqueue_entry(&wait, t);

		set_task_state(t, TASK_UNINTERRUPTIBLE);

		__add_wait_queue_tail_exclusive(&sem->fifo_wait, &wait);

		TS_LOCK_SUSPEND;

		spin_unlock_irqrestore(&sem->fifo_wait.lock, flags);

		schedule();

		TS_LOCK_RESUME;

		BUG_ON(sem->owner != t);
	} else {
		/* it's ours now */
		sem->owner = t;

		spin_unlock_irqrestore(&sem->fifo_wait.lock, flags);
	}

	return 0;
}

static int cedf_omlp_unlock(struct litmus_lock* l)
{
	struct task_struct *t = current, *next;
	struct omlp_semaphore *sem = omlp_from_lock(l);
	unsigned long flags;
	int err = 0;

	spin_lock_irqsave(&sem->fifo_wait.lock, flags);

	if (sem->owner != t) {
		err = -EINVAL;
		spin_unlock_irqrestore(&sem->fifo_wait.lock, flags);
		goto out;
	}

	/* check if there are jobs waiting for this resource */
	next = __waitqueue_remove_first(&sem->fifo_wait);
	if (next) {
		/* next becomes the resouce holder */
		sem->owner = next;
		TRACE_CUR("lock ownership passed to %s/%d\n", next->comm, next->pid);

		/* wake up next */
		wake_up_process(next);
	} else
		/* becomes available */
		sem->owner = NULL;

	spin_unlock_irqrestore(&sem->fifo_wait.lock, flags);

	priodon_complete_request();

out:
	return err;
}

static int cedf_omlp_close(struct litmus_lock* l)
{
	struct task_struct *t = current;
	struct omlp_semaphore *sem = omlp_from_lock(l);
	unsigned long flags;

	int owner;

	spin_lock_irqsave(&sem->fifo_wait.lock, flags);

	owner = sem->owner == t;

	spin_unlock_irqrestore(&sem->fifo_wait.lock, flags);

	if (owner)
		cedf_omlp_unlock(l);

	return 0;
}

static void cedf_omlp_free(struct litmus_lock* lock)
{
	kfree(omlp_from_lock(lock));
}

static struct litmus_lock_ops cedf_omlp_lock_ops = {
	.close  = cedf_omlp_close,
	.lock   = cedf_omlp_lock,
	.unlock = cedf_omlp_unlock,
	.deallocate = cedf_omlp_free,
};

static struct litmus_lock* cedf_new_omlp(void)
{
	struct omlp_semaphore* sem;

	sem = kmalloc(sizeof(*sem), GFP_KERNEL);
	if (!sem)
		return NULL;

	sem->owner   = NULL;
	init_waitqueue_head(&sem->fifo_wait);
	sem->litmus_lock.ops = &cedf_omlp_lock_ops;

	return &sem->litmus_lock;
}

static long cedf_allocate_lock(struct litmus_lock **lock, int type,
			       void* __user unused)
{
	int err = -ENXIO;

	switch (type) {

	case OMLP_SEM:
		/* O(m) Multiprocessor Locking Protocol */
		*lock  = cedf_new_omlp();
		if (*lock)
			err = 0;
		else
			err = -ENOMEM;
		break;

	};

	return err;
}


#endif

static long cedf_admit_task(struct task_struct* tsk)
{
	if (task_cpu(tsk) == tsk->rt_param.task_params.cpu) {
#ifdef CONFIG_LITMUS_LOCKING
		
#endif
		return 0;
	}
	else
		return -EINVAL;
}

/* total number of cluster */
static int num_clusters;
/* we do not support cluster of different sizes */
static unsigned int cluster_size;

#ifdef VERBOSE_INIT
static void print_cluster_topology(cpumask_var_t mask, int cpu)
{
	int chk;
	char buf[255];

	chk = cpulist_scnprintf(buf, 254, mask);
	buf[chk] = '\0';
	printk(KERN_INFO "CPU = %d, shared cpu(s) = %s\n", cpu, buf);

}
#endif

static int clusters_allocated = 0;

static void cleanup_cedf(void)
{
	int i;

	if (clusters_allocated) {
		for (i = 0; i < num_clusters; i++) {
			kfree(cedf[i].cpus);
			kfree(cedf[i].heap_node);
			free_cpumask_var(cedf[i].cpu_map);
		}

		kfree(cedf);
	}
}

static long cedf_activate_plugin(void)
{
	int i, j, cpu, ccpu, cpu_count;
	cpu_entry_t *entry;

	cpumask_var_t mask;
	int chk = 0;

	/* de-allocate old clusters, if any */
	cleanup_cedf();

	printk(KERN_INFO "C-EDF: Activate Plugin, cluster configuration = %d\n",
			cluster_config);

	/* need to get cluster_size first */
	if(!zalloc_cpumask_var(&mask, GFP_ATOMIC))
		return -ENOMEM;

	if (unlikely(cluster_config == GLOBAL_CLUSTER)) {
		cluster_size = num_online_cpus();
	} else {
		chk = get_shared_cpu_map(mask, 0, cluster_config);
		if (chk) {
			/* if chk != 0 then it is the max allowed index */
			printk(KERN_INFO "C-EDF: Cluster configuration = %d "
			       "is not supported on this hardware.\n",
			       cluster_config);
			/* User should notice that the configuration failed, so
			 * let's bail out. */
			return -EINVAL;
		}

		cluster_size = cpumask_weight(mask);
	}

	if ((num_online_cpus() % cluster_size) != 0) {
		/* this can't be right, some cpus are left out */
		printk(KERN_ERR "C-EDF: Trying to group %d cpus in %d!\n",
				num_online_cpus(), cluster_size);
		return -1;
	}

	num_clusters = num_online_cpus() / cluster_size;
	printk(KERN_INFO "C-EDF: %d cluster(s) of size = %d\n",
			num_clusters, cluster_size);

	/* initialize clusters */
	cedf = kmalloc(num_clusters * sizeof(cedf_domain_t), GFP_ATOMIC);
	for (i = 0; i < num_clusters; i++) {

		cedf[i].cpus = kmalloc(cluster_size * sizeof(cpu_entry_t),
				GFP_ATOMIC);
		cedf[i].heap_node = kmalloc(
				cluster_size * sizeof(struct bheap_node),
				GFP_ATOMIC);
		bheap_init(&(cedf[i].cpu_heap));
#ifdef CONFIG_LITMUS_LOCKING
		bheap_init(&(cedf[i].pending_jobs));
		bheap_init(&(cedf[i].pending_cpus));
		cedf[i].pending_nodes = kmalloc(
				cluster_size * sizeof(struct bheap_node),
				GFP_ATOMIC);
#endif
		edf_domain_init(&(cedf[i].domain), NULL, cedf_release_jobs);

		if(!zalloc_cpumask_var(&cedf[i].cpu_map, GFP_ATOMIC))
			return -ENOMEM;
#ifdef CONFIG_RELEASE_MASTER
		cedf[i].domain.release_master = atomic_read(&release_master_cpu);
#endif
	}

	/* cycle through cluster and add cpus to them */
	for (i = 0; i < num_clusters; i++) {

		for_each_online_cpu(cpu) {
			/* check if the cpu is already in a cluster */
			for (j = 0; j < num_clusters; j++)
				if (cpumask_test_cpu(cpu, cedf[j].cpu_map))
					break;
			/* if it is in a cluster go to next cpu */
			if (j < num_clusters &&
					cpumask_test_cpu(cpu, cedf[j].cpu_map))
				continue;

			/* this cpu isn't in any cluster */
			/* get the shared cpus */
			if (unlikely(cluster_config == GLOBAL_CLUSTER))
				cpumask_copy(mask, cpu_online_mask);
			else
				get_shared_cpu_map(mask, cpu, cluster_config);

			cpumask_copy(cedf[i].cpu_map, mask);
#ifdef VERBOSE_INIT
			print_cluster_topology(mask, cpu);
#endif
			/* add cpus to current cluster and init cpu_entry_t */
			cpu_count = 0;
			for_each_cpu(ccpu, cedf[i].cpu_map) {

				entry = &per_cpu(cedf_cpu_entries, ccpu);
				cedf[i].cpus[cpu_count] = entry;
				atomic_set(&entry->will_schedule, 0);
				entry->cpu = ccpu;
				entry->cluster = &cedf[i];
				entry->hn = &(cedf[i].heap_node[cpu_count]);
				bheap_node_init(&entry->hn, entry);

#ifdef CONFIG_LITMUS_LOCKING
				entry->pending_hn = &(cedf[i].pending_nodes[cpu_count]);
				bheap_node_init(&entry->pending_hn, entry);
				entry->pending = NULL;
#endif

				cpu_count++;

				entry->linked = NULL;
				entry->scheduled = NULL;
#ifdef CONFIG_RELEASE_MASTER
				/* only add CPUs that should schedule jobs */
				if (entry->cpu != entry->cluster->domain.release_master)
#endif
				{
					update_cpu_position(entry);
#ifdef CONFIG_LITMUS_LOCKING
					update_pending_position(entry);
#endif
				}
			}
			/* done with this cluster */
			break;
		}
	}

	free_cpumask_var(mask);
	clusters_allocated = 1;
	return 0;
}

/*	Plugin object	*/
static struct sched_plugin cedf_plugin __cacheline_aligned_in_smp = {
	.plugin_name		= "C-EDF",
	.finish_switch		= cedf_finish_switch,
	.tick			= cedf_tick,
	.task_new		= cedf_task_new,
	.complete_job		= complete_job,
	.task_exit		= cedf_task_exit,
	.schedule		= cedf_schedule,
	.task_wake_up		= cedf_task_wake_up,
	.task_block		= cedf_task_block,
	.admit_task		= cedf_admit_task,
	.activate_plugin	= cedf_activate_plugin,
#ifdef CONFIG_LITMUS_LOCKING
	.allocate_lock		= cedf_allocate_lock,
#endif
};

static struct proc_dir_entry *cluster_file = NULL, *cedf_dir = NULL;

static int __init init_cedf(void)
{
	int err, fs;

	err = register_sched_plugin(&cedf_plugin);
	if (!err) {
		fs = make_plugin_proc_dir(&cedf_plugin, &cedf_dir);
		if (!fs)
			cluster_file = create_cluster_file(cedf_dir, &cluster_config);
		else
			printk(KERN_ERR "Could not allocate C-EDF procfs dir.\n");
	}
	return err;
}

static void clean_cedf(void)
{
	cleanup_cedf();
	if (cluster_file)
		remove_proc_entry("cluster", cedf_dir);
	if (cedf_dir)
		remove_plugin_proc_dir(&cedf_plugin);
}

module_init(init_cedf);
module_exit(clean_cedf);