/* * litmus/sched_cedf.c * * Implementation of the C-EDF scheduling algorithm. * * This implementation is based on G-EDF: * - CPUs are clustered around L2 or L3 caches. * - Clusters topology is automatically detected (this is arch dependent * and is working only on x86 at the moment --- and only with modern * cpus that exports cpuid4 information) * - The plugins _does not_ attempt to put tasks in the right cluster i.e. * the programmer needs to be aware of the topology to place tasks * in the desired cluster * - default clustering is around L2 cache (cache index = 2) * supported clusters are: L1 (private cache: pedf), L2, L3, ALL (all * online_cpus are placed in a single cluster). * * For details on functions, take a look at sched_gsn_edf.c * * Currently, we do not support changes in the number of online cpus. * If the num_online_cpus() dynamically changes, the plugin is broken. * * This version uses the simple approach and serializes all scheduling * decisions by the use of a queue lock. This is probably not the * best way to do it, but it should suffice for now. */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include /* to configure the cluster size */ #include #include /* Reference configuration variable. Determines which cache level is used to * group CPUs into clusters. GLOBAL_CLUSTER, which is the default, means that * all CPUs form a single cluster (just like GSN-EDF). */ static enum cache_level cluster_config = GLOBAL_CLUSTER; struct clusterdomain; /* cpu_entry_t - maintain the linked and scheduled state * * A cpu also contains a pointer to the cedf_domain_t cluster * that owns it (struct clusterdomain*) */ typedef struct { int cpu; struct clusterdomain* cluster; /* owning cluster */ struct task_struct* linked; /* only RT tasks */ struct task_struct* scheduled; /* only RT tasks */ atomic_t will_schedule; /* prevent unneeded IPIs */ struct bheap_node* hn; #ifdef CONFIG_LITMUS_LOCKING struct bheap_node* pending_hn; struct task_struct* pending; #endif } cpu_entry_t; /* one cpu_entry_t per CPU */ DEFINE_PER_CPU(cpu_entry_t, cedf_cpu_entries); static struct bheap_node cpu_nodes[NR_CPUS]; #ifdef CONFIG_LITMUS_LOCKING static struct bheap_node pending_nodes[NR_CPUS]; #endif /* * In C-EDF there is a cedf domain _per_ cluster * The number of clusters is dynamically determined accordingly to the * total cpu number and the cluster size */ typedef struct clusterdomain { /* rt_domain for this cluster */ rt_domain_t domain; /* map of this cluster cpus */ cpumask_var_t cpu_map; unsigned int num_cpus; /* the cpus queue themselves according to priority in here */ struct bheap cpu_heap; #ifdef CONFIG_LITMUS_LOCKING struct bheap pending_jobs; struct bheap pending_cpus; #endif /* lock for this cluster */ #define cluster_lock domain.ready_lock } cedf_domain_t; /* a cedf_domain per cluster; allocation is done at init/activation time */ cedf_domain_t *cedf; #define remote_cpu(cpu) (&per_cpu(cedf_cpu_entries, cpu)) #define remote_cluster(cpu) ((cedf_domain_t *) per_cpu(cedf_cpu_entries, cpu).cluster) #define task_cpu_cluster(task) remote_cluster(get_partition(task)) /* Uncomment WANT_ALL_SCHED_EVENTS if you want to see all scheduling * decisions in the TRACE() log; uncomment VERBOSE_INIT for verbose * information during the initialization of the plugin (e.g., topology) #define WANT_ALL_SCHED_EVENTS */ #define VERBOSE_INIT static int cpu_lower_prio(struct bheap_node *_a, struct bheap_node *_b) { cpu_entry_t *a, *b; a = _a->value; b = _b->value; /* Note that a and b are inverted: we want the lowest-priority CPU at * the top of the heap. */ return edf_higher_prio(b->linked, a->linked); } /* update_cpu_position - Move the cpu entry to the correct place to maintain * order in the cpu queue. Caller must hold cedf lock. */ static void update_cpu_position(cpu_entry_t *entry) { cedf_domain_t *cluster = entry->cluster; if (likely(bheap_node_in_heap(entry->hn))) bheap_delete(cpu_lower_prio, &cluster->cpu_heap, entry->hn); bheap_insert(cpu_lower_prio, &cluster->cpu_heap, entry->hn); } /* caller must hold cedf lock */ static cpu_entry_t* lowest_prio_cpu(cedf_domain_t *cluster) { struct bheap_node* hn; hn = bheap_peek(cpu_lower_prio, &cluster->cpu_heap); return hn->value; } /* link_task_to_cpu - Update the link of a CPU. * Handles the case where the to-be-linked task is already * scheduled on a different CPU. */ static noinline void link_task_to_cpu(struct task_struct* linked, cpu_entry_t *entry) { cpu_entry_t *sched; struct task_struct* tmp; int on_cpu; BUG_ON(linked && !is_realtime(linked)); /* Currently linked task is set to be unlinked. */ if (entry->linked) { entry->linked->rt_param.linked_on = NO_CPU; } /* Link new task to CPU. */ if (linked) { /* handle task is already scheduled somewhere! */ on_cpu = linked->rt_param.scheduled_on; if (on_cpu != NO_CPU) { sched = &per_cpu(cedf_cpu_entries, on_cpu); /* this should only happen if not linked already */ BUG_ON(sched->linked == linked); /* If we are already scheduled on the CPU to which we * wanted to link, we don't need to do the swap -- * we just link ourselves to the CPU and depend on * the caller to get things right. */ if (entry != sched) { TRACE_TASK(linked, "already scheduled on %d, updating link.\n", sched->cpu); tmp = sched->linked; linked->rt_param.linked_on = sched->cpu; sched->linked = linked; update_cpu_position(sched); linked = tmp; } } if (linked) /* might be NULL due to swap */ linked->rt_param.linked_on = entry->cpu; } entry->linked = linked; #ifdef WANT_ALL_SCHED_EVENTS if (linked) TRACE_TASK(linked, "linked to %d.\n", entry->cpu); else TRACE("NULL linked to %d.\n", entry->cpu); #endif update_cpu_position(entry); } /* unlink - Make sure a task is not linked any longer to an entry * where it was linked before. Must hold cedf_lock. */ static noinline void unlink(struct task_struct* t) { cpu_entry_t *entry; if (t->rt_param.linked_on != NO_CPU) { /* unlink */ entry = &per_cpu(cedf_cpu_entries, t->rt_param.linked_on); t->rt_param.linked_on = NO_CPU; link_task_to_cpu(NULL, entry); } else if (is_queued(t)) { /* This is an interesting situation: t is scheduled, * but was just recently unlinked. It cannot be * linked anywhere else (because then it would have * been relinked to this CPU), thus it must be in some * queue. We must remove it from the list in this * case. * * in C-EDF case is should be somewhere in the queue for * its domain, therefore and we can get the domain using * task_cpu_cluster */ remove(&(task_cpu_cluster(t))->domain, t); } } /* preempt - force a CPU to reschedule */ static void preempt(cpu_entry_t *entry) { preempt_if_preemptable(entry->scheduled, entry->cpu); } #ifdef CONFIG_LITMUS_LOCKING static int update_pending_job(cedf_domain_t* cluster, struct task_struct* t); static void priodon_become_eligible(void); static void priodon_complete_request(void); static inline int in_pending_heap(struct task_struct* t) { return bheap_node_in_heap(tsk_rt(t)->pending_node); } /* has this task already been processed for pending */ static inline int is_pending(struct task_struct* t) { return tsk_rt(t)->pending_on != NO_CPU || in_pending_heap(t); } #endif /* requeue - Put an unlinked task into gsn-edf domain. * Caller must hold cedf_lock. */ static noinline void requeue(struct task_struct* task) { cedf_domain_t *cluster = task_cpu_cluster(task); BUG_ON(!task); /* sanity check before insertion */ BUG_ON(is_queued(task)); if (is_released(task, litmus_clock())) { #ifdef CONFIG_LITMUS_LOCKING if (!is_pending(task)) update_pending_job(cluster, task); #endif __add_ready(&cluster->domain, task); } else { /* it has got to wait */ add_release(&cluster->domain, task); } } /* check for any necessary preemptions */ static void check_for_preemptions(cedf_domain_t *cluster) { struct task_struct *task; cpu_entry_t* last; for(last = lowest_prio_cpu(cluster); edf_preemption_needed(&cluster->domain, last->linked); last = lowest_prio_cpu(cluster)) { /* preemption necessary */ #ifdef CONFIG_LITMUS_LOCKING task = __peek_ready(&cluster->domain); if (update_pending_job(cluster, task)) { /* Something changed, re-evaluate priorites to * see if we still need to preempt. * */ TRACE_TASK(task, "hitting continue\n"); continue; } #endif task = __take_ready(&cluster->domain); TRACE_TASK(task, "attempting to link task to P%d\n", last->cpu); if (last->linked) requeue(last->linked); link_task_to_cpu(task, last); preempt(last); } } #ifdef CONFIG_LITMUS_LOCKING static int pending_lower_prio(struct bheap_node *_a, struct bheap_node *_b) { cpu_entry_t *a, *b; a = _a->value; b = _b->value; /* Note that a and b are inverted: we want the lowest-priority CPU at * the top of the heap. */ return edf_higher_base_prio(b->pending, a->pending); } /* update_cpu_position - Move the cpu entry to the correct place to maintain * order in the cpu queue. Caller must hold cedf lock. */ static void update_pending_position(cpu_entry_t *entry) { cedf_domain_t *cluster = entry->cluster; if (likely(bheap_node_in_heap(entry->pending_hn))) bheap_delete(pending_lower_prio, &cluster->pending_cpus, entry->pending_hn); bheap_insert(pending_lower_prio, &cluster->pending_cpus, entry->pending_hn); } /* caller must hold cedf lock */ static cpu_entry_t* lowest_pending_cpu(cedf_domain_t *cluster) { struct bheap_node* hn; hn = bheap_peek(pending_lower_prio, &cluster->pending_cpus); return hn->value; } static void priority_raised(struct task_struct* t) { cedf_domain_t *cluster = task_cpu_cluster(t); int linked_on; linked_on = tsk_rt(t)->linked_on; /* If it is scheduled, then we need to reorder the CPU heap. */ if (linked_on != NO_CPU) { TRACE_TASK(t, "%s: linked on %d\n", __FUNCTION__, linked_on); /* Holder is scheduled; need to re-order CPUs. * We can't use heap_decrease() here since * the cpu_heap is ordered in reverse direction, so * it is actually an increase. */ bheap_delete(cpu_lower_prio, &cluster->cpu_heap, remote_cpu(linked_on)->hn); bheap_insert(cpu_lower_prio, &cluster->cpu_heap, remote_cpu(linked_on)->hn); } else { /* holder may be queued: first stop queue changes */ raw_spin_lock(&cluster->domain.release_lock); if (is_queued(t)) { TRACE_TASK(t, "%s: is queued\n", __FUNCTION__); bheap_decrease(edf_ready_order, tsk_rt(t)->heap_node); } else { /* Nothing to do: if it is not queued and not linked * then it is either sleeping or currently being moved * by other code (e.g., a timer interrupt handler) that * will use the correct priority when enqueuing the * task. */ TRACE_TASK(t, "%s: is NOT queued => Done.\n", __FUNCTION__); } raw_spin_unlock(&cluster->domain.release_lock); } } static void priority_lowered(struct task_struct* t) { /* assumption: t is not in a release heap */ if (is_queued(t) || tsk_rt(t)->linked_on != NO_CPU) { unlink(t); requeue(t); } } static void donate_priority(struct task_struct* recipient, struct task_struct* donor) { cedf_domain_t *cluster = task_cpu_cluster(donor); BUG_ON(task_cpu_cluster(recipient) != task_cpu_cluster(donor)); BUG_ON(tsk_rt(donor)->is_donor); BUG_ON(tsk_rt(recipient)->is_donor); BUG_ON(tsk_rt(donor)->inh_task); BUG_ON(tsk_rt(recipient)->inh_task); TRACE_TASK(donor, "priodon: becomes priority donor for %s/%d\n", recipient->comm, recipient->pid); /* swap priorities */ tsk_rt(recipient)->inh_task = donor; tsk_rt(donor)->inh_task = recipient; tsk_rt(donor)->is_donor = 1; priority_lowered(donor); priority_raised(recipient); bheap_uncache_min(edf_ready_order, &cluster->domain.ready_queue); } /* assumption: new_donor has a higher priority than old_donor */ static void switch_donor(struct task_struct* recipient, struct task_struct* old_donor, struct task_struct* new_donor) { TRACE_TASK(new_donor, "becomes donor for %s/%d instead of %s/%d\n", recipient->comm, recipient->pid, old_donor->comm, old_donor->pid); BUG_ON(tsk_rt(recipient)->inh_task != old_donor); BUG_ON(tsk_rt(old_donor)->inh_task != recipient); BUG_ON(tsk_rt(new_donor)->inh_task != NULL); BUG_ON(tsk_rt(new_donor)->is_donor); tsk_rt(old_donor)->inh_task = NULL; tsk_rt(old_donor)->is_donor = 0; tsk_rt(recipient)->inh_task = new_donor; tsk_rt(new_donor)->inh_task = recipient; tsk_rt(new_donor)->is_donor = 1; priority_raised(recipient); priority_raised(old_donor); priority_lowered(new_donor); } static void undonate_priority(struct task_struct* recipient, struct task_struct* donor) { cedf_domain_t *cluster = task_cpu_cluster(donor); BUG_ON(tsk_rt(recipient)->inh_task != donor); BUG_ON(tsk_rt(donor)->inh_task != recipient); TRACE_TASK(donor, "priodon: is no longer priority donor of %s/%d\n", recipient->comm, recipient->pid); tsk_rt(recipient)->inh_task = NULL; tsk_rt(donor)->inh_task = NULL; tsk_rt(donor)->is_donor = 0; priority_lowered(recipient); priority_raised(donor); bheap_uncache_min(edf_ready_order, &cluster->domain.ready_queue); } static inline void add_to_pending(cedf_domain_t* cluster, struct task_struct* t) { TRACE_TASK(t, "priodon: adding to pending heap wait:%u donor:%u req:%u pend:%d\n", tsk_rt(t)->waiting_eligible, tsk_rt(t)->is_donor, tsk_rt(t)->request_incomplete, tsk_rt(t)->pending_on); bheap_insert(edf_pending_order, &cluster->pending_jobs, tsk_rt(t)->pending_node); } static inline struct task_struct* take_pending(cedf_domain_t* cluster) { struct bheap_node* node; node = bheap_take(edf_pending_order, &cluster->pending_jobs); return node ? (struct task_struct*) node->value : NULL; } static inline struct task_struct* peek_pending(cedf_domain_t* cluster) { struct bheap_node* node; node = bheap_peek(edf_pending_order, &cluster->pending_jobs); return node ? (struct task_struct*) node->value : NULL; } static inline int fake_resume(struct task_struct* t) { TRACE_TASK(t, "priodon: fake resume wait:%u donor:%u\n", tsk_rt(t)->waiting_eligible, tsk_rt(t)->is_donor); /* Fake suspended. Let's resume it. */ if (tsk_rt(t)->waiting_eligible) { tsk_rt(t)->waiting_eligible = 0; if (tsk_rt(t)->scheduled_on == NO_CPU) { /* it was removed from the queue */ requeue(t); return 1; } } return 0; } /* Lazily update set of highest-priority pending jobs. * Returns 1 if priority recheck is required. */ static int update_pending_job(cedf_domain_t* cluster, struct task_struct* to_be_linked) { cpu_entry_t* entry; struct task_struct* lowest_hp; /* lowest-priority high-priority task */ struct task_struct* highest_lp; /* highest-priority low-priority task */ int reeval = 0; entry = lowest_pending_cpu(cluster); lowest_hp = entry->pending; if (to_be_linked && !is_pending(to_be_linked)) /* not yet accounted for, stick in heap */ add_to_pending(cluster, to_be_linked); highest_lp = peek_pending(cluster); if (edf_higher_base_prio(highest_lp, lowest_hp)) { /* yep, should be become of the c highest-prior pending jobs */ TRACE_TASK(highest_lp, "priodon: became one of the %u highest-prio tasks (P%d, req:%u) X\n", cluster->num_cpus, entry->cpu, tsk_rt(highest_lp)->request_incomplete); /* get it out of the heap */ highest_lp = take_pending(cluster); BUG_ON(highest_lp == lowest_hp); /* it should never be a priority donor at this point */ BUG_ON(tsk_rt(highest_lp)->is_donor); entry->pending = highest_lp; update_pending_position(entry); tsk_rt(highest_lp)->pending_on = entry->cpu; /* things that could happen: * * 1) lowest_hp has no donor, but is in a request => highest_lp becomes donor * 2) lowest_hp is donor => highest_lp becomes new donor, old donor is resumed if suspended * 3) lowest_hp is not in a request, and highest_lp is waiting => highest_lp is resumed * 4) lowest_hp is not in a request, and highest_lp is not waiting => nothing to do * 5) highest_lp has a priority donor => resume its donor */ /* do we need to put it back? */ if (lowest_hp) { TRACE_TASK(lowest_hp, "priodon: no longer among %u highest-prio tasks req:%u\n", cluster->num_cpus, tsk_rt(lowest_hp)->request_incomplete); tsk_rt(lowest_hp)->pending_on = NO_CPU; add_to_pending(cluster, lowest_hp); if (tsk_rt(lowest_hp)->request_incomplete) { /* case 1) */ donate_priority(lowest_hp, highest_lp); reeval = 1; } else if (tsk_rt(lowest_hp)->inh_task) { /* case 2) */ switch_donor(tsk_rt(lowest_hp)->inh_task, lowest_hp, highest_lp); fake_resume(lowest_hp); reeval = 1; } } if (!tsk_rt(highest_lp)->is_donor) { if (tsk_rt(highest_lp)->waiting_eligible) { /* case 3) */ reeval = fake_resume(highest_lp); BUG_ON(tsk_rt(highest_lp)->inh_task); } else if (tsk_rt(highest_lp)->inh_task) { /* case 5 */ struct task_struct* donor = tsk_rt(highest_lp)->inh_task; undonate_priority(highest_lp, donor); reeval = fake_resume(donor); } } } return reeval; } /* job has exited => no longer pending */ static void job_pending_exit(struct task_struct* t) { cedf_domain_t *cluster; cpu_entry_t* entry; TRACE_TASK(t, "priodon: is no longer pending (pending_on:%d, queued:%d)\n", tsk_rt(t)->pending_on, in_pending_heap(t)); cluster = task_cpu_cluster(t); if (tsk_rt(t)->pending_on != NO_CPU) { entry = &per_cpu(cedf_cpu_entries, tsk_rt(t)->pending_on); tsk_rt(t)->pending_on = NO_CPU; entry->pending = NULL; update_pending_position(entry); /* let's see if anything changed */ update_pending_job(cluster, NULL); } else if (in_pending_heap(t)) { bheap_delete(edf_pending_order, &cluster->pending_jobs, tsk_rt(t)->pending_node); } } #endif /* cedf_job_arrival: task is either resumed or released */ static noinline void cedf_job_arrival(struct task_struct* task) { cedf_domain_t *cluster = task_cpu_cluster(task); BUG_ON(!task); requeue(task); check_for_preemptions(cluster); } static void cedf_release_jobs(rt_domain_t* rt, struct bheap* tasks) { cedf_domain_t* cluster = container_of(rt, cedf_domain_t, domain); unsigned long flags; raw_spin_lock_irqsave(&cluster->cluster_lock, flags); __merge_ready(&cluster->domain, tasks); check_for_preemptions(cluster); raw_spin_unlock_irqrestore(&cluster->cluster_lock, flags); } /* caller holds cedf_lock */ static noinline void job_completion(struct task_struct *t, int forced) { BUG_ON(!t); sched_trace_task_completion(t, forced); TRACE_TASK(t, "job_completion().\n"); #ifdef CONFIG_LITMUS_LOCKING job_pending_exit(t); #endif /* prepare for next period */ prepare_for_next_period(t); if (is_released(t, litmus_clock())) sched_trace_task_release(t); /* unlink */ unlink(t); /* requeue * But don't requeue a blocking task. */ set_rt_flags(t, RT_F_RUNNING); if (is_running(t)) cedf_job_arrival(t); } /* cedf_tick - this function is called for every local timer * interrupt. * * checks whether the current task has expired and checks * whether we need to preempt it if it has not expired */ static void cedf_tick(struct task_struct* t) { if (is_realtime(t) && budget_enforced(t) && budget_exhausted(t)) { if (!is_np(t)) { /* np tasks will be preempted when they become * preemptable again */ litmus_reschedule_local(); TRACE("cedf_scheduler_tick: " "%d is preemptable " " => FORCE_RESCHED\n", t->pid); } else if (is_user_np(t)) { TRACE("cedf_scheduler_tick: " "%d is non-preemptable, " "preemption delayed.\n", t->pid); request_exit_np(t); } } } /* Getting schedule() right is a bit tricky. schedule() may not make any * assumptions on the state of the current task since it may be called for a * number of reasons. The reasons include a scheduler_tick() determined that it * was necessary, because sys_exit_np() was called, because some Linux * subsystem determined so, or even (in the worst case) because there is a bug * hidden somewhere. Thus, we must take extreme care to determine what the * current state is. * * The CPU could currently be scheduling a task (or not), be linked (or not). * * The following assertions for the scheduled task could hold: * * - !is_running(scheduled) // the job blocks * - scheduled->timeslice == 0 // the job completed (forcefully) * - get_rt_flag() == RT_F_SLEEP // the job completed (by syscall) * - linked != scheduled // we need to reschedule (for any reason) * - is_np(scheduled) // rescheduling must be delayed, * sys_exit_np must be requested * * Any of these can occur together. */ static struct task_struct* cedf_schedule(struct task_struct * prev) { cpu_entry_t* entry = &__get_cpu_var(cedf_cpu_entries); cedf_domain_t *cluster = entry->cluster; int out_of_time, sleep, preempt, np, exists, blocks; struct task_struct* next = NULL; #ifdef CONFIG_LITMUS_LOCKING int priodon, reeval; #else #define priodon 0 #endif #ifdef CONFIG_RELEASE_MASTER /* Bail out early if we are the release master. * The release master never schedules any real-time tasks. */ if (cluster->domain.release_master == entry->cpu) { sched_state_task_picked(); return NULL; } #endif raw_spin_lock(&cluster->cluster_lock); /* sanity checking */ BUG_ON(entry->scheduled && entry->scheduled != prev); BUG_ON(entry->scheduled && !is_realtime(prev)); BUG_ON(is_realtime(prev) && !entry->scheduled); /* (0) Determine state */ exists = entry->scheduled != NULL; blocks = exists && !is_running(entry->scheduled); out_of_time = exists && budget_enforced(entry->scheduled) && budget_exhausted(entry->scheduled); np = exists && is_np(entry->scheduled); sleep = exists && get_rt_flags(entry->scheduled) == RT_F_SLEEP; preempt = entry->scheduled != entry->linked; #ifdef CONFIG_LITMUS_LOCKING priodon = exists && (tsk_rt(entry->scheduled)->waiting_eligible || /* can't allow job to exit until request is over */ (tsk_rt(entry->scheduled)->is_donor && sleep)); /* this should never happend together (at least we don't handle it atm) */ BUG_ON(priodon && blocks); #endif #ifdef WANT_ALL_SCHED_EVENTS TRACE_TASK(prev, "invoked cedf_schedule.\n"); #endif if (exists) TRACE_TASK(prev, "blocks:%d out_of_time:%d np:%d sleep:%d preempt:%d " "state:%d sig:%d priodon:%d\n", blocks, out_of_time, np, sleep, preempt, prev->state, signal_pending(prev), priodon); if (entry->linked && preempt) TRACE_TASK(prev, "will be preempted by %s/%d\n", entry->linked->comm, entry->linked->pid); /* If a task blocks we have no choice but to reschedule. */ if (blocks || priodon) unlink(entry->scheduled); /* Request a sys_exit_np() call if we would like to preempt but cannot. * Do not unlink since entry->scheduled is currently in the ready queue. * We don't process out_of_time and sleep until the job is preemptive again. */ if (np && (out_of_time || preempt || sleep)) { request_exit_np(entry->scheduled); } /* Any task that is preemptable and either exhausts its execution * budget or wants to sleep completes. We may have to reschedule after * this. Don't do a job completion if we block (can't have timers running * for blocked jobs). Preemption go first for the same reason. */ if (!np && (out_of_time || sleep) && !blocks && !preempt && !priodon) /* note: priority donation prevents job completion */ job_completion(entry->scheduled, !sleep); /* Link pending task if we became unlinked. */ if (!entry->linked) { struct task_struct *pulled = __take_ready(&cluster->domain); #ifdef CONFIG_LITMUS_LOCKING if (pulled && !is_pending(pulled)) { /* Pulled an un-processed task from the ready queue. * Since this job was ready but not yet processed, it * cannot be a priority donor. So we should never * have to re-evaluate. */ TRACE_TASK(pulled, "pulled unprocessed\n"); reeval = update_pending_job(cluster, pulled); BUG_ON(reeval); } #endif link_task_to_cpu(pulled, entry); } /* The final scheduling decision. Do we need to switch for some reason? * If linked is different from scheduled, then select linked as next. */ if ((!np || blocks || priodon) && entry->linked != entry->scheduled) { /* Schedule a linked job? */ if (entry->linked) { entry->linked->rt_param.scheduled_on = entry->cpu; next = entry->linked; } if (entry->scheduled) { /* not gonna be scheduled soon */ entry->scheduled->rt_param.scheduled_on = NO_CPU; TRACE_TASK(entry->scheduled, "scheduled_on = NO_CPU\n"); } } else /* Only override Linux scheduler if we have a real-time task * scheduled that needs to continue. */ if (exists) next = prev; sched_state_task_picked(); raw_spin_unlock(&cluster->cluster_lock); #ifdef WANT_ALL_SCHED_EVENTS TRACE("cedf_lock released, next=0x%p\n", next); if (next) TRACE_TASK(next, "scheduled at %llu\n", litmus_clock()); else if (exists && !next) TRACE("becomes idle at %llu.\n", litmus_clock()); #endif return next; } /* _finish_switch - we just finished the switch away from prev */ static void cedf_finish_switch(struct task_struct *prev) { cpu_entry_t* entry = &__get_cpu_var(cedf_cpu_entries); entry->scheduled = is_realtime(current) ? current : NULL; #ifdef WANT_ALL_SCHED_EVENTS TRACE_TASK(prev, "switched away from\n"); #endif } /* Prepare a task for running in RT mode */ static void cedf_task_new(struct task_struct * t, int on_rq, int running) { unsigned long flags; cpu_entry_t* entry; cedf_domain_t* cluster; TRACE("gsn edf: task new %d\n", t->pid); /* the cluster doesn't change even if t is running */ cluster = task_cpu_cluster(t); raw_spin_lock_irqsave(&cluster->cluster_lock, flags); /* setup job params */ release_at(t, litmus_clock()); #ifdef CONFIG_LITMUS_LOCKING tsk_rt(t)->pending_node = bheap_node_alloc(GFP_ATOMIC | __GFP_NOFAIL); bheap_node_init(&tsk_rt(t)->pending_node, t); tsk_rt(t)->pending_on = NO_CPU; add_to_pending(cluster, t); #endif if (running) { entry = &per_cpu(cedf_cpu_entries, task_cpu(t)); BUG_ON(entry->scheduled); #ifdef CONFIG_RELEASE_MASTER if (entry->cpu != cluster->domain.release_master) { #endif entry->scheduled = t; tsk_rt(t)->scheduled_on = task_cpu(t); #ifdef CONFIG_RELEASE_MASTER } else { /* do not schedule on release master */ preempt(entry); /* force resched */ tsk_rt(t)->scheduled_on = NO_CPU; } #endif } else { t->rt_param.scheduled_on = NO_CPU; } t->rt_param.linked_on = NO_CPU; cedf_job_arrival(t); raw_spin_unlock_irqrestore(&(cluster->cluster_lock), flags); } static void cedf_task_wake_up(struct task_struct *task) { unsigned long flags; lt_t now; cedf_domain_t *cluster; TRACE_TASK(task, "wake_up at %llu\n", litmus_clock()); cluster = task_cpu_cluster(task); raw_spin_lock_irqsave(&cluster->cluster_lock, flags); /* We need to take suspensions because of semaphores into * account! If a job resumes after being suspended due to acquiring * a semaphore, it should never be treated as a new job release. */ if (get_rt_flags(task) == RT_F_EXIT_SEM) { set_rt_flags(task, RT_F_RUNNING); } else { now = litmus_clock(); if (is_tardy(task, now)) { /* new sporadic release */ release_at(task, now); sched_trace_task_release(task); } else { if (task->rt.time_slice) { /* came back in time before deadline */ set_rt_flags(task, RT_F_RUNNING); } } } cedf_job_arrival(task); raw_spin_unlock_irqrestore(&cluster->cluster_lock, flags); } static void cedf_task_block(struct task_struct *t) { unsigned long flags; cedf_domain_t *cluster; TRACE_TASK(t, "block at %llu\n", litmus_clock()); cluster = task_cpu_cluster(t); /* unlink if necessary */ raw_spin_lock_irqsave(&cluster->cluster_lock, flags); unlink(t); raw_spin_unlock_irqrestore(&cluster->cluster_lock, flags); BUG_ON(!is_realtime(t)); } #ifdef CONFIG_LITMUS_LOCKING static void cedf_pre_setsched(struct task_struct *t, int policy) { unsigned long flags; cedf_domain_t *cluster = task_cpu_cluster(t); int delay_donor_exit = 0; while (1) { raw_spin_lock_irqsave(&cluster->cluster_lock, flags); TRACE_CUR("cedf_pre_setsched wait:%u pend:%d donor:%u req:%u\n", tsk_rt(t)->waiting_eligible, tsk_rt(t)->pending_on, tsk_rt(t)->is_donor, tsk_rt(t)->request_incomplete); delay_donor_exit = tsk_rt(current)->is_donor; raw_spin_unlock_irqrestore(&cluster->cluster_lock, flags); if (!delay_donor_exit) break; TRACE_CUR("donor exit delay\n"); set_current_state(TASK_INTERRUPTIBLE); schedule_timeout(HZ); } } #endif static void cedf_task_exit(struct task_struct * t) { unsigned long flags; cedf_domain_t *cluster = task_cpu_cluster(t); /* unlink if necessary */ raw_spin_lock_irqsave(&cluster->cluster_lock, flags); unlink(t); #ifdef CONFIG_LITMUS_LOCKING /* make sure it's not pending anymore */ job_pending_exit(t); bheap_node_free(tsk_rt(t)->pending_node); #endif if (tsk_rt(t)->scheduled_on != NO_CPU) { cpu_entry_t *cpu; cpu = &per_cpu(cedf_cpu_entries, tsk_rt(t)->scheduled_on); cpu->scheduled = NULL; tsk_rt(t)->scheduled_on = NO_CPU; } raw_spin_unlock_irqrestore(&cluster->cluster_lock, flags); BUG_ON(!is_realtime(t)); TRACE_TASK(t, "RIP\n"); } #ifdef CONFIG_LITMUS_LOCKING #include #include /* NOTE: we use fake suspensions because we must wake the task from within the * scheduler */ /* suspend until the current task becomes eligible to issue a lock request */ static void priodon_become_eligible(void) { struct task_struct* t = current; unsigned long flags; cedf_domain_t *cluster; cluster = task_cpu_cluster(t); do { TRACE_CUR("priodon: checking whether request may be issued\n"); raw_spin_lock_irqsave(&cluster->cluster_lock, flags); if (tsk_rt(t)->pending_on == NO_CPU || tsk_rt(t)->is_donor) { /* nope, gotta wait */ tsk_rt(t)->waiting_eligible = 1; TRACE_CUR("priodon: not eligible pend:%u donor:%u\n", tsk_rt(t)->pending_on, tsk_rt(t)->is_donor); } else { /* alright! we are good to go! */ tsk_rt(t)->request_incomplete = 1; TRACE_CUR("priodon: request issued\n"); } raw_spin_unlock_irqrestore(&cluster->cluster_lock, flags); if (tsk_rt(t)->waiting_eligible) { TRACE_CUR("priodon: fake suspending\n"); TS_LOCK_SUSPEND; schedule(); TS_LOCK_RESUME; } } while (!tsk_rt(t)->request_incomplete); } /* current task has completed its request */ static void priodon_complete_request(void) { struct task_struct* t = current; struct task_struct* donor; unsigned long flags; cedf_domain_t *cluster; cluster = task_cpu_cluster(t); preempt_disable(); raw_spin_lock_irqsave(&cluster->cluster_lock, flags); TRACE_CUR("priodon: completing request\n"); if (tsk_rt(t)->inh_task) { /* we have a donor job --- see if we need to wake it */ donor = tsk_rt(t)->inh_task; undonate_priority(t, donor); if (fake_resume(donor)) check_for_preemptions(cluster); } tsk_rt(t)->request_incomplete = 0; raw_spin_unlock_irqrestore(&cluster->cluster_lock, flags); preempt_enable(); } /* struct for semaphore with priority inheritance */ struct omlp_semaphore { struct litmus_lock litmus_lock; /* current resource holder */ struct task_struct *owner; /* FIFO queue of waiting tasks */ wait_queue_head_t fifo_wait; }; static inline struct omlp_semaphore* omlp_from_lock(struct litmus_lock* lock) { return container_of(lock, struct omlp_semaphore, litmus_lock); } static int cedf_omlp_lock(struct litmus_lock* l) { struct task_struct* t = current; struct omlp_semaphore *sem = omlp_from_lock(l); wait_queue_t wait; unsigned long flags; if (!is_realtime(t)) return -EPERM; priodon_become_eligible(); spin_lock_irqsave(&sem->fifo_wait.lock, flags); if (sem->owner) { /* resource is not free => must suspend and wait */ init_waitqueue_entry(&wait, t); set_task_state(t, TASK_UNINTERRUPTIBLE); __add_wait_queue_tail_exclusive(&sem->fifo_wait, &wait); TS_LOCK_SUSPEND; spin_unlock_irqrestore(&sem->fifo_wait.lock, flags); schedule(); TS_LOCK_RESUME; BUG_ON(sem->owner != t); } else { /* it's ours now */ sem->owner = t; spin_unlock_irqrestore(&sem->fifo_wait.lock, flags); } return 0; } static int cedf_omlp_unlock(struct litmus_lock* l) { struct task_struct *t = current, *next; struct omlp_semaphore *sem = omlp_from_lock(l); unsigned long flags; int err = 0; spin_lock_irqsave(&sem->fifo_wait.lock, flags); if (sem->owner != t) { err = -EINVAL; spin_unlock_irqrestore(&sem->fifo_wait.lock, flags); goto out; } /* check if there are jobs waiting for this resource */ next = __waitqueue_remove_first(&sem->fifo_wait); if (next) { /* next becomes the resouce holder */ sem->owner = next; TRACE_CUR("lock ownership passed to %s/%d\n", next->comm, next->pid); /* wake up next */ wake_up_process(next); } else /* becomes available */ sem->owner = NULL; spin_unlock_irqrestore(&sem->fifo_wait.lock, flags); priodon_complete_request(); out: return err; } static int cedf_omlp_close(struct litmus_lock* l) { struct task_struct *t = current; struct omlp_semaphore *sem = omlp_from_lock(l); unsigned long flags; int owner; spin_lock_irqsave(&sem->fifo_wait.lock, flags); owner = sem->owner == t; spin_unlock_irqrestore(&sem->fifo_wait.lock, flags); if (owner) cedf_omlp_unlock(l); return 0; } static void cedf_omlp_free(struct litmus_lock* lock) { kfree(omlp_from_lock(lock)); } static struct litmus_lock_ops cedf_omlp_lock_ops = { .close = cedf_omlp_close, .lock = cedf_omlp_lock, .unlock = cedf_omlp_unlock, .deallocate = cedf_omlp_free, }; static struct litmus_lock* cedf_new_omlp(void) { struct omlp_semaphore* sem; sem = kmalloc(sizeof(*sem), GFP_KERNEL); if (!sem) return NULL; sem->owner = NULL; init_waitqueue_head(&sem->fifo_wait); sem->litmus_lock.ops = &cedf_omlp_lock_ops; return &sem->litmus_lock; } static long cedf_allocate_lock(struct litmus_lock **lock, int type, void* __user unused) { int err = -ENXIO; switch (type) { case OMLP_SEM: /* O(m) Multiprocessor Locking Protocol */ *lock = cedf_new_omlp(); if (*lock) err = 0; else err = -ENOMEM; break; }; return err; } #endif static long cedf_admit_task(struct task_struct* tsk) { if (task_cpu(tsk) == tsk->rt_param.task_params.cpu) { #ifdef CONFIG_LITMUS_LOCKING #endif return 0; } else return -EINVAL; } /* total number of cluster */ static int num_clusters; /* we do not support cluster of different sizes */ static unsigned int cluster_size; #ifdef VERBOSE_INIT static void print_cluster_topology(cpumask_var_t mask, int cpu) { int chk; char buf[255]; chk = cpulist_scnprintf(buf, 254, mask); buf[chk] = '\0'; printk(KERN_INFO "CPU = %d, shared cpu(s) = %s\n", cpu, buf); } #endif static int clusters_allocated = 0; static void cleanup_cedf(void) { int i; if (clusters_allocated) { for (i = 0; i < num_clusters; i++) { free_cpumask_var(cedf[i].cpu_map); } kfree(cedf); } } static long cedf_activate_plugin(void) { int i, j, cpu, ccpu, cpu_count; cpu_entry_t *entry; cpumask_var_t mask; int chk = 0; /* de-allocate old clusters, if any */ cleanup_cedf(); printk(KERN_INFO "C-EDF: Activate Plugin, cluster configuration = %d\n", cluster_config); /* need to get cluster_size first */ if(!zalloc_cpumask_var(&mask, GFP_ATOMIC)) return -ENOMEM; if (unlikely(cluster_config == GLOBAL_CLUSTER)) { cluster_size = num_online_cpus(); } else { chk = get_shared_cpu_map(mask, 0, cluster_config); if (chk) { /* if chk != 0 then it is the max allowed index */ printk(KERN_INFO "C-EDF: Cluster configuration = %d " "is not supported on this hardware.\n", cluster_config); /* User should notice that the configuration failed, so * let's bail out. */ return -EINVAL; } cluster_size = cpumask_weight(mask); } if ((num_online_cpus() % cluster_size) != 0) { /* this can't be right, some cpus are left out */ printk(KERN_ERR "C-EDF: Trying to group %d cpus in %d!\n", num_online_cpus(), cluster_size); return -1; } num_clusters = num_online_cpus() / cluster_size; printk(KERN_INFO "C-EDF: %d cluster(s) of size = %d\n", num_clusters, cluster_size); /* initialize clusters */ cedf = kmalloc(num_clusters * sizeof(cedf_domain_t), GFP_ATOMIC); for (i = 0; i < num_clusters; i++) { bheap_init(&(cedf[i].cpu_heap)); #ifdef CONFIG_LITMUS_LOCKING bheap_init(&(cedf[i].pending_jobs)); bheap_init(&(cedf[i].pending_cpus)); #endif edf_domain_init(&(cedf[i].domain), NULL, cedf_release_jobs); if(!zalloc_cpumask_var(&cedf[i].cpu_map, GFP_ATOMIC)) return -ENOMEM; #ifdef CONFIG_RELEASE_MASTER cedf[i].domain.release_master = atomic_read(&release_master_cpu); #endif } /* cycle through cluster and add cpus to them */ for (i = 0; i < num_clusters; i++) { for_each_online_cpu(cpu) { /* check if the cpu is already in a cluster */ for (j = 0; j < num_clusters; j++) if (cpumask_test_cpu(cpu, cedf[j].cpu_map)) break; /* if it is in a cluster go to next cpu */ if (j < num_clusters && cpumask_test_cpu(cpu, cedf[j].cpu_map)) continue; /* this cpu isn't in any cluster */ /* get the shared cpus */ if (unlikely(cluster_config == GLOBAL_CLUSTER)) cpumask_copy(mask, cpu_online_mask); else get_shared_cpu_map(mask, cpu, cluster_config); cpumask_copy(cedf[i].cpu_map, mask); #ifdef VERBOSE_INIT print_cluster_topology(mask, cpu); #endif /* add cpus to current cluster and init cpu_entry_t */ cpu_count = 0; cedf[i].num_cpus = 0; for_each_cpu(ccpu, cedf[i].cpu_map) { entry = &per_cpu(cedf_cpu_entries, ccpu); atomic_set(&entry->will_schedule, 0); entry->cpu = ccpu; entry->cluster = &cedf[i]; entry->hn = cpu_nodes + ccpu; bheap_node_init(&entry->hn, entry); #ifdef CONFIG_LITMUS_LOCKING entry->pending_hn = pending_nodes + ccpu; bheap_node_init(&entry->pending_hn, entry); entry->pending = NULL; #endif cpu_count++; entry->linked = NULL; entry->scheduled = NULL; #ifdef CONFIG_RELEASE_MASTER /* only add CPUs that should schedule jobs */ if (entry->cpu != entry->cluster->domain.release_master) #endif { cedf[i].num_cpus++; update_cpu_position(entry); #ifdef CONFIG_LITMUS_LOCKING update_pending_position(entry); #endif } } /* done with this cluster */ break; } } free_cpumask_var(mask); clusters_allocated = 1; return 0; } /* Plugin object */ static struct sched_plugin cedf_plugin __cacheline_aligned_in_smp = { .plugin_name = "C-EDF", .finish_switch = cedf_finish_switch, .tick = cedf_tick, .task_new = cedf_task_new, .complete_job = complete_job, .task_exit = cedf_task_exit, .schedule = cedf_schedule, .task_wake_up = cedf_task_wake_up, .task_block = cedf_task_block, .admit_task = cedf_admit_task, .activate_plugin = cedf_activate_plugin, #ifdef CONFIG_LITMUS_LOCKING .allocate_lock = cedf_allocate_lock, .pre_setsched = cedf_pre_setsched, #endif }; static struct proc_dir_entry *cluster_file = NULL, *cedf_dir = NULL; static int __init init_cedf(void) { int err, fs; err = register_sched_plugin(&cedf_plugin); if (!err) { fs = make_plugin_proc_dir(&cedf_plugin, &cedf_dir); if (!fs) cluster_file = create_cluster_file(cedf_dir, &cluster_config); else printk(KERN_ERR "Could not allocate C-EDF procfs dir.\n"); } return err; } static void clean_cedf(void) { cleanup_cedf(); if (cluster_file) remove_proc_entry("cluster", cedf_dir); if (cedf_dir) remove_plugin_proc_dir(&cedf_plugin); } module_init(init_cedf); module_exit(clean_cedf);