From 91ac0052eec03b8624ea3613acd50825c17981f1 Mon Sep 17 00:00:00 2001 From: Glenn Elliott Date: Wed, 9 Oct 2013 15:48:51 -0400 Subject: Remove C-RM support for now. Remove C-RM support. It will be easier to re-add C-RM after we've ported C-EDF forward. --- litmus/Kconfig | 10 - litmus/Makefile | 1 - litmus/rm_common.c | 300 ------ litmus/sched_crm.c | 2562 ---------------------------------------------------- 4 files changed, 2873 deletions(-) delete mode 100644 litmus/rm_common.c delete mode 100644 litmus/sched_crm.c diff --git a/litmus/Kconfig b/litmus/Kconfig index 03f31157abc7..3adfa1fe9800 100644 --- a/litmus/Kconfig +++ b/litmus/Kconfig @@ -12,16 +12,6 @@ config PLUGIN_CEDF On smaller platforms (e.g., ARM PB11MPCore), using C-EDF makes little sense since there aren't any shared caches. -config PLUGIN_CRM - bool "Clustered-RM" - depends on X86 && SYSFS - default y - help - Include the Clustered RM (C-RM) plugin in the kernel. - This is appropriate for large platforms with shared caches. - On smaller platforms (e.g., ARM PB11MPCore), using C-EDF - makes little sense since there aren't any shared caches. - config RECURSIVE_READYQ_LOCK bool "Recursive Ready Queue Lock" default n diff --git a/litmus/Makefile b/litmus/Makefile index 08ed4a663d8f..264640dd013b 100644 --- a/litmus/Makefile +++ b/litmus/Makefile @@ -25,7 +25,6 @@ obj-y = sched_plugin.o litmus.o \ sched_pfp.o obj-$(CONFIG_PLUGIN_CEDF) += sched_cedf.o -obj-$(CONFIG_PLUGIN_CRM) += rm_common.o sched_crm.o obj-$(CONFIG_PLUGIN_PFAIR) += sched_pfair.o obj-$(CONFIG_SCHED_CPU_AFFINITY) += affinity.o diff --git a/litmus/rm_common.c b/litmus/rm_common.c deleted file mode 100644 index 8d4cdf4c71cf..000000000000 --- a/litmus/rm_common.c +++ /dev/null @@ -1,300 +0,0 @@ -/* - * kernel/rm_common.c - * - * Common functions for EDF based scheduler. - */ - -#include -#include -#include - -#include -#include -#include - -#ifdef CONFIG_LITMUS_NESTED_LOCKING -#include -#endif - -#include - - -/* rm_higher_prio - returns true if first has a higher EDF priority - * than second. Deadline ties are broken by PID. - * - * both first and second may be NULL - */ -#ifdef CONFIG_LITMUS_NESTED_LOCKING -int __rm_higher_prio( - struct task_struct* first, comparison_mode_t first_mode, - struct task_struct* second, comparison_mode_t second_mode) -#else -int rm_higher_prio(struct task_struct* first, struct task_struct* second) -#endif -{ - struct task_struct *first_task = first; - struct task_struct *second_task = second; - - /* There is no point in comparing a task to itself. */ - if (first && first == second) { - TRACE_CUR("WARNING: pointless rm priority comparison: %s/%d\n", first->comm, first->pid); -// WARN_ON(1); - return 0; - } - - /* check for NULL tasks */ - if (!first || !second) { - return first && !second; - } - /* check for non-realtime */ - if (!is_realtime(first) || !is_realtime(second)) { - return is_realtime(first) && !is_realtime(second); - } - - /* There is some goofy stuff in this code here. There are three subclasses - * within the SCHED_LITMUS scheduling class: - * 1) Auxiliary tasks: COTS helper threads from the application level that - * are forced to be real-time. - * 2) klmirqd interrupt threads: Litmus threaded interrupt handlers. - * 3) Normal Litmus tasks. - * - * At their base priorities, #3 > #2 > #1. However, #1 and #2 threads might - * inherit a priority from a task of #3. - * - * The code proceeds in the following manner: - * 1) Make aux and klmirqd threads with base-priorities have low priorities. - * 2) Determine effective priorities. - * 3) Perform priority comparison. Favor #3 over #1 and #2 in case of tie. - */ - - -#if defined(CONFIG_REALTIME_AUX_TASK_PRIORITY_BOOSTED) - /* run aux tasks at max priority */ - if (tsk_rt(first)->is_aux_task != tsk_rt(second)->is_aux_task) { - return (tsk_rt(first)->is_aux_task > tsk_rt(second)->is_aux_task); - } -#elif defined(CONFIG_REALTIME_AUX_TASK_PRIORITY_INHERITANCE) - { - int first_lo_aux = tsk_rt(first)->is_aux_task && !tsk_rt(first)->inh_task; - int second_lo_aux = tsk_rt(second)->is_aux_task && !tsk_rt(second)->inh_task; - - /* prioritize aux tasks without inheritance below real-time tasks */ - if (first_lo_aux || second_lo_aux) { - // one of these is an aux task without inheritance. - if (first_lo_aux != second_lo_aux) { - int temp = (first_lo_aux < second_lo_aux); // non-lo-aux has higher priority. - return temp; - } - else { - /* both MUST be lo_aux. tie-break. */ - //TRACE_CUR("aux tie break!\n"); - goto aux_tie_break; - } - } - - if (tsk_rt(first)->is_aux_task && tsk_rt(second)->is_aux_task && - tsk_rt(first)->inh_task == tsk_rt(second)->inh_task) { - // inh_task is !NULL for both tasks since neither was a lo_aux task. - // Both aux tasks inherit from the same task, so tie-break - // by base priority of the aux tasks. - //TRACE_CUR("aux tie break!\n"); - goto aux_tie_break; - } - } -#endif - -#ifdef CONFIG_LITMUS_SOFTIRQD - { - int first_lo_klmirqd = tsk_rt(first)->is_interrupt_thread && !tsk_rt(first)->inh_task; - int second_lo_klmirqd = tsk_rt(second)->is_interrupt_thread && !tsk_rt(second)->inh_task; - - /* prioritize aux tasks without inheritance below real-time tasks */ - if (first_lo_klmirqd || second_lo_klmirqd) { - // one of these is an klmirqd thread without inheritance. - if (first_lo_klmirqd != second_lo_klmirqd) { - int temp = (first_lo_klmirqd < second_lo_klmirqd); // non-klmirqd has higher priority - return temp; - } - else { - /* both MUST be klmirqd. tie-break. */ - //TRACE_CUR("klmirqd tie break!\n"); - goto klmirqd_tie_break; - } - } - - if (tsk_rt(first)->is_interrupt_thread && tsk_rt(second)->is_interrupt_thread && - tsk_rt(first)->inh_task == tsk_rt(second)->inh_task) { - // inh_task is !NULL for both tasks since neither was a lo_klmirqd task. - // Both klmirqd tasks inherit from the same task, so tie-break - // by base priority of the klmirqd tasks. - //TRACE_CUR("klmirqd tie break!\n"); - goto klmirqd_tie_break; - } - } -#endif - - -#ifdef CONFIG_LITMUS_LOCKING - /* Check for EFFECTIVE priorities. Change task - * used for comparison in such a case. - */ - if (unlikely(tsk_rt(first)->inh_task) -#ifdef CONFIG_LITMUS_NESTED_LOCKING - && (first_mode == EFFECTIVE) -#endif - ) { - first_task = tsk_rt(first)->inh_task; - } - if (unlikely(tsk_rt(second)->inh_task) -#ifdef CONFIG_LITMUS_NESTED_LOCKING - && (second_mode == EFFECTIVE) -#endif - ) { - second_task = tsk_rt(second)->inh_task; - } - - /* Check for priority boosting. Tie-break by start of boosting. - */ - if (unlikely(is_priority_boosted(first_task))) { - /* first_task is boosted, how about second_task? */ - if (!is_priority_boosted(second_task) || - lt_before(get_boost_start(first_task), - get_boost_start(second_task))) { - return 1; - } - else { - return 0; - } - } - else if (unlikely(is_priority_boosted(second_task))) { - /* second_task is boosted, first is not*/ - return 0; - } - -#endif - -#ifdef CONFIG_REALTIME_AUX_TASK_PRIORITY_INHERITANCE -aux_tie_break: -#endif -#ifdef CONFIG_LITMUS_SOFTIRQD -klmirqd_tie_break: -#endif - - // KLUDGE! This is reverse of fp_common's implementation!!! - if (get_period(first_task) < get_period(second_task)) - return 1; - else if (get_period(first_task) == get_period(second_task)) { - if (first_task->pid < second_task->pid) - return 1; - else if (first_task->pid == second_task->pid) { - /* there is inheritance going on. consider inheritors. */ -#ifdef CONFIG_LITMUS_SOFTIRQD - /* non-interrupt thread gets prio */ - if (!tsk_rt(first)->is_interrupt_thread && tsk_rt(second)->is_interrupt_thread) - return 1; - else if (tsk_rt(first)->is_interrupt_thread == tsk_rt(second)->is_interrupt_thread) { -#endif - -#if defined(CONFIG_REALTIME_AUX_TASK_PRIORITY_INHERITANCE) - /* non-aux thread gets prio */ - if (!tsk_rt(first)->is_aux_task && tsk_rt(second)->is_aux_task) - return 1; - else if (tsk_rt(first_task)->is_aux_task == tsk_rt(second_task)->is_aux_task) { -#endif - /* if both tasks inherit from the same task */ - if (tsk_rt(first)->inh_task == tsk_rt(second)->inh_task) { - /* TODO: Make a recurive call to rm_higher_prio, - comparing base priorities. */ - return (first->pid < second->pid); - } - else { - /* At least one task must inherit */ - BUG_ON(!tsk_rt(first)->inh_task && - !tsk_rt(second)->inh_task); - - /* The task withOUT the inherited priority wins. */ - if (tsk_rt(second)->inh_task) { - return 1; - } - } -#if defined(CONFIG_REALTIME_AUX_TASK_PRIORITY_INHERITANCE) - } -#endif - -#ifdef CONFIG_LITMUS_SOFTIRQD - } -#endif - } - } - - return 0; /* fall-through. prio(second_task) > prio(first_task) */ -} - - -#ifdef CONFIG_LITMUS_NESTED_LOCKING -int rm_higher_prio(struct task_struct* first, struct task_struct* second) -{ - return __rm_higher_prio(first, EFFECTIVE, second, EFFECTIVE); -} - -int rm_max_heap_order(struct binheap_node *a, struct binheap_node *b) -{ - struct nested_info *l_a = (struct nested_info *)binheap_entry(a, struct nested_info, hp_binheap_node); - struct nested_info *l_b = (struct nested_info *)binheap_entry(b, struct nested_info, hp_binheap_node); - - return __rm_higher_prio(l_a->hp_waiter_eff_prio, EFFECTIVE, l_b->hp_waiter_eff_prio, EFFECTIVE); -} - -int rm_min_heap_order(struct binheap_node *a, struct binheap_node *b) -{ - return rm_max_heap_order(b, a); // swap comparison -} - -int rm_max_heap_base_priority_order(struct binheap_node *a, struct binheap_node *b) -{ - struct nested_info *l_a = (struct nested_info *)binheap_entry(a, struct nested_info, hp_binheap_node); - struct nested_info *l_b = (struct nested_info *)binheap_entry(b, struct nested_info, hp_binheap_node); - - return __rm_higher_prio(l_a->hp_waiter_eff_prio, BASE, l_b->hp_waiter_eff_prio, BASE); -} - -int rm_min_heap_base_priority_order(struct binheap_node *a, struct binheap_node *b) -{ - return rm_max_heap_base_priority_order(b, a); // swap comparison -} -#endif - - -int rm_ready_order(struct bheap_node* a, struct bheap_node* b) -{ - return rm_higher_prio(bheap2task(a), bheap2task(b)); -} - -void rm_domain_init(rt_domain_t* rt, check_resched_needed_t resched, - release_jobs_t release) -{ - rt_domain_init(rt, rm_ready_order, resched, release); -} - -/* need_to_preempt - check whether the task t needs to be preempted - * call only with irqs disabled and with ready_lock acquired - * THIS DOES NOT TAKE NON-PREEMPTIVE SECTIONS INTO ACCOUNT! - */ -int rm_preemption_needed(rt_domain_t* rt, struct task_struct *t) -{ - /* we need the read lock for rm_ready_queue */ - /* no need to preempt if there is nothing pending */ - if (!__jobs_pending(rt)) - return 0; - /* we need to reschedule if t doesn't exist */ - if (!t) - return 1; - - /* NOTE: We cannot check for non-preemptibility since we - * don't know what address space we're currently in. - */ - - /* make sure to get non-rt stuff out of the way */ - return !is_realtime(t) || rm_higher_prio(__next_ready(rt), t); -} - diff --git a/litmus/sched_crm.c b/litmus/sched_crm.c deleted file mode 100644 index 791b9979190e..000000000000 --- a/litmus/sched_crm.c +++ /dev/null @@ -1,2562 +0,0 @@ -/* - * litmus/sched_crm.c - * - * Implementation of the C-EDF scheduling algorithm. - * - * This implementation is based on G-EDF: - * - CPUs are clustered around L2 or L3 caches. - * - Clusters topology is automatically detected (this is arch dependent - * and is working only on x86 at the moment --- and only with modern - * cpus that exports cpuid4 information) - * - The plugins _does not_ attempt to put tasks in the right cluster i.e. - * the programmer needs to be aware of the topology to place tasks - * in the desired cluster - * - default clustering is around L2 cache (cache index = 2) - * supported clusters are: L1 (private cache: pedf), L2, L3, ALL (all - * online_cpus are placed in a single cluster). - * - * For details on functions, take a look at sched_gsn_edf.c - * - * Currently, we do not support changes in the number of online cpus. - * If the num_online_cpus() dynamically changes, the plugin is broken. - * - * This version uses the simple approach and serializes all scheduling - * decisions by the use of a queue lock. This is probably not the - * best way to do it, but it should suffice for now. - */ - -#include -#include -#include -#include -#include -#include - -#include -#include -#include -#include -#include -#include -#include - -#include - -#include -#include -#include - -#ifdef CONFIG_LITMUS_LOCKING -#include -#endif - -#ifdef CONFIG_LITMUS_NESTED_LOCKING -#include -#include -#include -#endif - -#ifdef CONFIG_SCHED_CPU_AFFINITY -#include -#endif - -#ifdef CONFIG_REALTIME_AUX_TASKS -#include -#endif - -/* to configure the cluster size */ -#include - -#ifdef CONFIG_SCHED_CPU_AFFINITY -#include -#endif - -#ifdef CONFIG_LITMUS_SOFTIRQD -#include -#endif - -#ifdef CONFIG_LITMUS_NVIDIA -#include -#endif - -#if defined(CONFIG_LITMUS_AFFINITY_LOCKING) && defined(CONFIG_LITMUS_NVIDIA) -#include -#endif - -/* Reference configuration variable. Determines which cache level is used to - * group CPUs into clusters. GLOBAL_CLUSTER, which is the default, means that - * all CPUs form a single cluster (just like GSN-EDF). - */ -static enum cache_level cluster_config = GLOBAL_CLUSTER; - -struct clusterdomain; - -/* cpu_entry_t - maintain the linked and scheduled state - * - * A cpu also contains a pointer to the crm_domain_t cluster - * that owns it (struct clusterdomain*) - */ -typedef struct { - int cpu; - struct clusterdomain* cluster; /* owning cluster */ - struct task_struct* linked; /* only RT tasks */ - struct task_struct* scheduled; /* only RT tasks */ - atomic_t will_schedule; /* prevent unneeded IPIs */ - struct binheap_node hn; -} cpu_entry_t; - -/* one cpu_entry_t per CPU */ -DEFINE_PER_CPU(cpu_entry_t, crm_cpu_entries); - -#define set_will_schedule() \ - (atomic_set(&__get_cpu_var(crm_cpu_entries).will_schedule, 1)) -#define clear_will_schedule() \ - (atomic_set(&__get_cpu_var(crm_cpu_entries).will_schedule, 0)) -#define test_will_schedule(cpu) \ - (atomic_read(&per_cpu(crm_cpu_entries, cpu).will_schedule)) - -/* - * In C-EDF there is a crm domain _per_ cluster - * The number of clusters is dynamically determined accordingly to the - * total cpu number and the cluster size - */ -typedef struct clusterdomain { - /* rt_domain for this cluster */ - rt_domain_t domain; - /* cpus in this cluster */ - cpu_entry_t* *cpus; - /* map of this cluster cpus */ - cpumask_var_t cpu_map; - /* the cpus queue themselves according to priority in here */ - struct binheap cpu_heap; - -#define cluster_lock domain.ready_lock - -#ifdef CONFIG_LITMUS_DGL_SUPPORT - raw_spinlock_t dgl_lock; -#endif - - int top_m_size; - struct binheap top_m; - struct binheap not_top_m; - -} crm_domain_t; - - -/* a crm_domain per cluster; allocation is done at init/activation time */ -crm_domain_t *crm; - -#define remote_cluster(cpu) ((crm_domain_t *) per_cpu(crm_cpu_entries, cpu).cluster) -#define task_cpu_cluster(task) remote_cluster(get_partition(task)) - -/* total number of cluster */ -static int num_clusters; -/* we do not support cluster of different sizes */ -static unsigned int cluster_size; - -static int clusters_allocated = 0; - - -#if defined(CONFIG_LITMUS_NVIDIA) && defined(CONFIG_LITMUS_SOFTIRQD) -static int num_gpu_clusters; -static unsigned int gpu_cluster_size; -#endif - -inline static struct task_struct* binheap_node_to_task(struct binheap_node *bn) -{ - struct budget_tracker *bt = binheap_entry(bn, struct budget_tracker, top_m_node); - struct task_struct *t = - container_of( - container_of(bt, struct rt_param, budget), - struct task_struct, - rt_param); - return t; -} - -static int crm_max_heap_base_priority_order(struct binheap_node *a, - struct binheap_node *b) -{ - struct task_struct* t_a = binheap_node_to_task(a); - struct task_struct* t_b = binheap_node_to_task(b); - return __rm_higher_prio(t_a, BASE, t_b, BASE); -} - -static int crm_min_heap_base_priority_order(struct binheap_node *a, - struct binheap_node *b) -{ - struct task_struct* t_a = binheap_node_to_task(a); - struct task_struct* t_b = binheap_node_to_task(b); - return __rm_higher_prio(t_b, BASE, t_a, BASE); -} - -static void crm_track_in_top_m(struct task_struct *t) -{ - /* cluster lock must be held */ - crm_domain_t *cluster = task_cpu_cluster(t); - struct budget_tracker *bt; - struct task_struct *mth_highest; - - //BUG_ON(binheap_is_in_heap(&tsk_rt(t)->budget.top_m_node)); - if (binheap_is_in_heap(&tsk_rt(t)->budget.top_m_node)) { -// TRACE_TASK(t, "apparently already being tracked. top-m?: %s\n", -// (bt_flag_is_set(t, BTF_IS_TOP_M)) ? "Yes":"No"); - return; - } - - /* TODO: do cluster_size-1 if release master is in this cluster */ - if (cluster->top_m_size < cluster_size) { -// TRACE_TASK(t, "unconditionally adding task to top-m.\n"); - binheap_add(&tsk_rt(t)->budget.top_m_node, &cluster->top_m, - struct budget_tracker, top_m_node); - ++cluster->top_m_size; - bt_flag_set(t, BTF_IS_TOP_M); - budget_state_machine(t,on_enter_top_m); - - return; - } - - BUG_ON(binheap_empty(&cluster->top_m)); - - bt = binheap_top_entry(&cluster->top_m, struct budget_tracker, top_m_node); - mth_highest = - container_of( - container_of(bt, struct rt_param, budget), - struct task_struct, - rt_param); - - if (__rm_higher_prio(t, BASE, mth_highest, BASE)) { -// TRACE_TASK(t, "adding to top-m (evicting %s/%d)\n", -// mth_highest->comm, mth_highest->pid); - - binheap_delete_root(&cluster->top_m, struct budget_tracker, top_m_node); - INIT_BINHEAP_NODE(&tsk_rt(mth_highest)->budget.top_m_node); - binheap_add(&tsk_rt(mth_highest)->budget.top_m_node, - &cluster->not_top_m, - struct budget_tracker, top_m_node); - budget_state_machine(mth_highest,on_exit_top_m); - bt_flag_clear(mth_highest, BTF_IS_TOP_M); - - binheap_add(&tsk_rt(t)->budget.top_m_node, &cluster->top_m, - struct budget_tracker, top_m_node); - bt_flag_set(t, BTF_IS_TOP_M); - budget_state_machine(t,on_enter_top_m); - } - else { -// TRACE_TASK(t, "adding to not-top-m\n"); - binheap_add(&tsk_rt(t)->budget.top_m_node, - &cluster->not_top_m, - struct budget_tracker, top_m_node); - } -} - -static void crm_untrack_in_top_m(struct task_struct *t) -{ - /* cluster lock must be held */ - crm_domain_t *cluster = task_cpu_cluster(t); - - if (!binheap_is_in_heap(&tsk_rt(t)->budget.top_m_node)) { -// TRACE_TASK(t, "is not being tracked\n"); /* BUG() on this case? */ - return; - } - - if (bt_flag_is_set(t, BTF_IS_TOP_M)) { -// TRACE_TASK(t, "removing task from top-m\n"); - - /* delete t's entry */ - binheap_delete(&tsk_rt(t)->budget.top_m_node, &cluster->top_m); - budget_state_machine(t,on_exit_top_m); - bt_flag_clear(t, BTF_IS_TOP_M); - - /* move a task over from the overflow heap */ - if(!binheap_empty(&cluster->not_top_m)) { - struct budget_tracker *bt = - binheap_top_entry(&cluster->not_top_m, struct budget_tracker, top_m_node); - struct task_struct *to_move = - container_of( - container_of(bt, struct rt_param, budget), - struct task_struct, - rt_param); - -// TRACE_TASK(to_move, "being promoted to top-m\n"); - - binheap_delete_root(&cluster->not_top_m, struct budget_tracker, top_m_node); - INIT_BINHEAP_NODE(&tsk_rt(to_move)->budget.top_m_node); - - binheap_add(&tsk_rt(to_move)->budget.top_m_node, - &cluster->top_m, - struct budget_tracker, top_m_node); - bt_flag_set(to_move, BTF_IS_TOP_M); - budget_state_machine(to_move,on_enter_top_m); - } - else { - --cluster->top_m_size; - } - } - else { -// TRACE_TASK(t, "removing task from not-top-m\n"); - binheap_delete(&tsk_rt(t)->budget.top_m_node, &cluster->not_top_m); - } -} - - -#ifdef CONFIG_LITMUS_DGL_SUPPORT -static raw_spinlock_t* crm_get_dgl_spinlock(struct task_struct *t) -{ - crm_domain_t *cluster = task_cpu_cluster(t); - return(&cluster->dgl_lock); -} -#endif - - -/* Uncomment WANT_ALL_SCHED_EVENTS if you want to see all scheduling - * decisions in the TRACE() log; uncomment VERBOSE_INIT for verbose - * information during the initialization of the plugin (e.g., topology) -#define WANT_ALL_SCHED_EVENTS - */ -#define VERBOSE_INIT - -static int cpu_lower_prio(struct binheap_node *_a, struct binheap_node *_b) -{ - cpu_entry_t *a = binheap_entry(_a, cpu_entry_t, hn); - cpu_entry_t *b = binheap_entry(_b, cpu_entry_t, hn); - - /* Note that a and b are inverted: we want the lowest-priority CPU at - * the top of the heap. - */ - return rm_higher_prio(b->linked, a->linked); -} - -/* update_cpu_position - Move the cpu entry to the correct place to maintain - * order in the cpu queue. Caller must hold crm lock. - */ -static void update_cpu_position(cpu_entry_t *entry) -{ - crm_domain_t *cluster = entry->cluster; - - if (likely(binheap_is_in_heap(&entry->hn))) { - binheap_delete(&entry->hn, &cluster->cpu_heap); - } - - binheap_add(&entry->hn, &cluster->cpu_heap, cpu_entry_t, hn); -} - -/* caller must hold crm lock */ -static cpu_entry_t* lowest_prio_cpu(crm_domain_t *cluster) -{ - return binheap_top_entry(&cluster->cpu_heap, cpu_entry_t, hn); -} - -static noinline void unlink(struct task_struct* t); - -/* link_task_to_cpu - Update the link of a CPU. - * Handles the case where the to-be-linked task is already - * scheduled on a different CPU. - */ -static noinline void link_task_to_cpu(struct task_struct* linked, - cpu_entry_t *entry) -{ - cpu_entry_t *sched; - struct task_struct* tmp; - int on_cpu; - - BUG_ON(linked && !is_realtime(linked)); - - /* Currently linked task is set to be unlinked. */ - if (entry->linked) { - entry->linked->rt_param.linked_on = NO_CPU; - -#ifdef CONFIG_LITMUS_LOCKING - if (tsk_rt(entry->linked)->inh_task) - clear_inh_task_linkback(entry->linked, tsk_rt(entry->linked)->inh_task); -#endif - } - - /* Link new task to CPU. */ - if (linked) { - /* handle task is already scheduled somewhere! */ - on_cpu = linked->rt_param.scheduled_on; - if (on_cpu != NO_CPU) { - sched = &per_cpu(crm_cpu_entries, on_cpu); - - BUG_ON(sched->linked == linked); - - /* If we are already scheduled on the CPU to which we - * wanted to link, we don't need to do the swap -- - * we just link ourselves to the CPU and depend on - * the caller to get things right. - */ - if (entry != sched) { - TRACE_TASK(linked, - "already scheduled on %d, updating link.\n", - sched->cpu); - tmp = sched->linked; - linked->rt_param.linked_on = sched->cpu; - sched->linked = linked; - update_cpu_position(sched); - linked = tmp; - } - } - if (linked) { /* might be NULL due to swap */ - linked->rt_param.linked_on = entry->cpu; - -#ifdef CONFIG_LITMUS_LOCKING - if (tsk_rt(linked)->inh_task) - set_inh_task_linkback(linked, tsk_rt(linked)->inh_task); -#endif - } - } - entry->linked = linked; -#ifdef WANT_ALL_SCHED_EVENTS - if (linked) - TRACE_TASK(linked, "linked to %d.\n", entry->cpu); - else - TRACE("NULL linked to %d.\n", entry->cpu); -#endif - update_cpu_position(entry); -} - -/* unlink - Make sure a task is not linked any longer to an entry - * where it was linked before. Must hold cluster_lock. - */ -static noinline void unlink(struct task_struct* t) -{ - if (t->rt_param.linked_on != NO_CPU) { - /* unlink */ - cpu_entry_t *entry = &per_cpu(crm_cpu_entries, t->rt_param.linked_on); - t->rt_param.linked_on = NO_CPU; - link_task_to_cpu(NULL, entry); - } else if (is_queued(t)) { - /* This is an interesting situation: t is scheduled, - * but was just recently unlinked. It cannot be - * linked anywhere else (because then it would have - * been relinked to this CPU), thus it must be in some - * queue. We must remove it from the list in this - * case. - * - * in C-EDF case is should be somewhere in the queue for - * its domain, therefore and we can get the domain using - * task_cpu_cluster - */ - remove(&(task_cpu_cluster(t))->domain, t); - } -} - - -/* preempt - force a CPU to reschedule - */ -static void preempt(cpu_entry_t *entry) -{ - preempt_if_preemptable(entry->scheduled, entry->cpu); -} - -/* requeue - Put an unlinked task into gsn-edf domain. - * Caller must hold cluster_lock. - */ -static noinline void requeue(struct task_struct* task) -{ - crm_domain_t *cluster = task_cpu_cluster(task); - BUG_ON(!task); - /* sanity check before insertion */ - BUG_ON(is_queued(task)); - - if (is_early_releasing(task) || is_released(task, litmus_clock()) || - tsk_rt(task)->job_params.is_backlogged_job) { -#ifdef CONFIG_REALTIME_AUX_TASKS - if (unlikely(tsk_rt(task)->is_aux_task && task->state != TASK_RUNNING && !tsk_rt(task)->aux_ready)) { - /* aux_task probably transitioned to real-time while it was blocked */ - TRACE_CUR("aux task %s/%d is not ready!\n", task->comm, task->pid); - tsk_rt(task)->aux_ready = 1; /* allow this to only happen once per aux task */ - } - else -#endif - __add_ready(&cluster->domain, task); - } - else { - TRACE_TASK(task, "not requeueing not-yet-released job\n"); - } -} - -#ifdef CONFIG_SCHED_CPU_AFFINITY -static cpu_entry_t* crm_get_nearest_available_cpu( - crm_domain_t *cluster, cpu_entry_t *start) -{ - cpu_entry_t *affinity; - - get_nearest_available_cpu(affinity, start, crm_cpu_entries, -#ifdef CONFIG_RELEASE_MASTER - cluster->domain.release_master -#else - NO_CPU -#endif - ); - - /* make sure CPU is in our cluster */ - if (affinity && cpu_isset(affinity->cpu, *cluster->cpu_map)) - return(affinity); - else - return(NULL); -} -#endif - - -/* check for any necessary preemptions */ -static void check_for_preemptions(crm_domain_t *cluster) -{ - struct task_struct *task; - cpu_entry_t *last; - - for(last = lowest_prio_cpu(cluster); - rm_preemption_needed(&cluster->domain, last->linked); - last = lowest_prio_cpu(cluster)) { - /* preemption necessary */ - task = __take_ready(&cluster->domain); - TRACE("check_for_preemptions: attempting to link task %d to %d\n", - task->pid, last->cpu); -#ifdef CONFIG_SCHED_CPU_AFFINITY - { - cpu_entry_t *affinity = - crm_get_nearest_available_cpu(cluster, - &per_cpu(crm_cpu_entries, task_cpu(task))); - if(affinity) - last = affinity; - else if(should_requeue_preempted_job(last->linked)) - requeue(last->linked); - } -#else - if (should_requeue_preempted_job(last->linked)) - requeue(last->linked); -#endif - link_task_to_cpu(task, last); - preempt(last); - } -} - -/* crm_job_arrival: task is either resumed or released */ -static noinline void crm_job_arrival(struct task_struct* task) -{ - crm_domain_t *cluster = task_cpu_cluster(task); - BUG_ON(!task); - - requeue(task); - check_for_preemptions(cluster); -} - -static void crm_track_on_release(struct bheap_node* n, void* dummy) -{ - struct task_struct* t = bheap2task(n); -// TRACE_TASK(t, "released\n"); - - crm_track_in_top_m(t); -} - -static void crm_release_jobs(rt_domain_t* rt, struct bheap* tasks) -{ - crm_domain_t* cluster = container_of(rt, crm_domain_t, domain); - unsigned long flags; - - raw_readyq_lock_irqsave(&cluster->cluster_lock, flags); - - bheap_for_each(tasks, crm_track_on_release, NULL); - - __merge_ready(&cluster->domain, tasks); - check_for_preemptions(cluster); - - raw_readyq_unlock_irqrestore(&cluster->cluster_lock, flags); -} - -/* caller holds cluster_lock */ -static noinline void job_completion(struct task_struct *t, int forced) -{ - int do_release = 0; - int backlogged = 0; - lt_t now; - - BUG_ON(!t); - - now = litmus_clock(); - - /* DO BACKLOG TRACKING */ - - /* job completed with budget remaining */ - if (get_release_policy(t) != SPORADIC) { - /* only jobs we know that will call sleep_next_job() can use backlogging */ - if (!forced) { - /* was it a backlogged job that completed? */ - if (tsk_rt(t)->job_params.is_backlogged_job) { - TRACE_TASK(t, "completed backlogged job\n"); - if (get_backlog(t)) { - --get_backlog(t); - /* is_backlogged_job remains asserted */ - } - else { - /* caught up completely */ - TRACE_TASK(t, "completely caught up.\n"); - tsk_rt(t)->job_params.is_backlogged_job = 0; - /* we now look like a normally completing job. */ - } - } - } - else { - ++get_backlog(t); - TRACE_TASK(t, "adding backlogged job\n"); - } - - backlogged = has_backlog(t); - TRACE_TASK(t, "number of backlogged jobs: %u\n", - get_backlog(t)); - } - - /* SETUP FOR THE NEXT JOB */ - - sched_trace_task_completion(t, forced); - - TRACE_TASK(t, "job_completion() at %llu (forced = %d).\n", now, forced); - - /* set flags */ - tsk_rt(t)->completed = 0; - -#if 0 - if (unlikely(!forced && backlogged)) { - /* Don't advance deadline/refresh budget. Use the remaining budget for - * the backlogged job. - * - * NOTE: Allowing backlogged jobs comsume remaining budget may affect - * blocking bound analysis. - */ - } - else if (unlikely(!forced && tsk_rt(t)->job_params.is_backlogged_job)) { - /* we've just about caught up, but we still have the job of this - * budget's allocation to do (even if it's for the future)... */ - TRACE_TASK(t, "Releasing final catch-up job.\n"); - backlogged = 1; - do_release = 1; - } - else { -#endif - crm_untrack_in_top_m(t); - prepare_for_next_period(t); - - do_release = (is_early_releasing(t) || is_released(t, now)); - - if (backlogged) { - TRACE_TASK(t, "refreshing budget with early " - "release for backlogged job.\n"); - } - if (do_release || backlogged) { - /* log here to capture overheads */ - sched_trace_task_release(t); - } -// } - - unlink(t); - - /* release or arm next job */ - if (is_running(t)) { - /* is our next job a backlogged job? */ - if (backlogged) { - TRACE_TASK(t, "next job is a backlogged job.\n"); - tsk_rt(t)->job_params.is_backlogged_job = 1; - } - else { - TRACE_TASK(t, "next job is a regular job.\n"); - tsk_rt(t)->job_params.is_backlogged_job = 0; - } - - if (do_release || backlogged) { - crm_track_in_top_m(t); - crm_job_arrival(t); - } - else { - add_release(&task_cpu_cluster(t)->domain, t); - } - } - else { - BUG_ON(!forced); - /* budget was refreshed and job early released */ - TRACE_TASK(t, "job exhausted budget while sleeping\n"); - crm_track_in_top_m(t); - } -} - -static enum hrtimer_restart crm_simple_on_exhausted(struct task_struct *t, int in_schedule) -{ - /* Assumption: t is scheduled on the CPU executing this callback */ - - if (in_schedule) { - BUG_ON(tsk_rt(t)->scheduled_on != smp_processor_id()); - if (budget_precisely_tracked(t) && cancel_enforcement_timer(t) < 0) { - TRACE_TASK(t, "raced with timer. deffering to timer.\n"); - goto out; - } - } - - if (budget_signalled(t) && !bt_flag_is_set(t, BTF_SIG_BUDGET_SENT)) { - /* signal exhaustion */ - send_sigbudget(t); /* will set BTF_SIG_BUDGET_SENT */ - } - - if (budget_enforced(t) && !bt_flag_test_and_set(t, BTF_BUDGET_EXHAUSTED)) { - if (likely(!is_np(t))) { - /* np tasks will be preempted when they become - * preemptable again - */ - if (!in_schedule) { - TRACE_TASK(t, "is preemptable => FORCE_RESCHED\n"); - litmus_reschedule_local(); - set_will_schedule(); - } - } else if (is_user_np(t)) { - TRACE_TASK(t, "is non-preemptable, preemption delayed.\n"); - request_exit_np(t); - } - } - -out: - return HRTIMER_NORESTART; -} - - -static enum hrtimer_restart crm_simple_io_on_exhausted(struct task_struct *t, int in_schedule) -{ - enum hrtimer_restart restart = HRTIMER_NORESTART; - - if (in_schedule) { - BUG_ON(tsk_rt(t)->scheduled_on != smp_processor_id()); - if (budget_precisely_tracked(t) && cancel_enforcement_timer(t) == -1) { - TRACE_TASK(t, "raced with timer. deffering to timer.\n"); - goto out; - } - } - - /* t may or may not be scheduled */ - - if (budget_signalled(t) && !bt_flag_is_set(t, BTF_SIG_BUDGET_SENT)) { - /* signal exhaustion */ - - /* Tasks should block SIG_BUDGET if they cannot gracefully respond to - * the signal while suspended. SIG_BUDGET is an rt-signal, so it will - * be queued and received when SIG_BUDGET is unblocked */ - send_sigbudget(t); /* will set BTF_SIG_BUDGET_SENT */ - } - - if (budget_enforced(t) && !bt_flag_is_set(t, BTF_BUDGET_EXHAUSTED)) { - int cpu = (tsk_rt(t)->linked_on != NO_CPU) ? - tsk_rt(t)->linked_on : tsk_rt(t)->scheduled_on; - - if (is_np(t) && is_user_np(t)) { - bt_flag_set(t, BTF_BUDGET_EXHAUSTED); - TRACE_TASK(t, "is non-preemptable, preemption delayed.\n"); - request_exit_np(t); - } - /* where do we need to call resched? */ - else if (cpu == smp_processor_id()) { - bt_flag_set(t, BTF_BUDGET_EXHAUSTED); - if (!in_schedule) { - TRACE_TASK(t, "is preemptable => FORCE_RESCHED\n"); - litmus_reschedule_local(); - set_will_schedule(); - } - } - else if (cpu != NO_CPU) { - bt_flag_set(t, BTF_BUDGET_EXHAUSTED); - if (!in_schedule) { - TRACE_TASK(t, "is preemptable on remote cpu (%d) => FORCE_RESCHED\n", cpu); - litmus_reschedule(cpu); - } - } - else if (unlikely(tsk_rt(t)->blocked_lock)) { - /* we shouldn't be draining while waiting for litmus lock, but we - * could have raced with the budget timer (?). */ - WARN_ON(1); - } - else { - lt_t remaining; - crm_domain_t *cluster; - unsigned long flags, kludge_flags; - - BUG_ON(in_schedule); - - cluster = task_cpu_cluster(t); - - // 1) refresh budget through job completion - // 2) if holds locks, tell the locking protocol to re-eval priority - // 3) -- the LP must undo any inheritance relations if appropriate - - /* force job completion */ - TRACE_TASK(t, "blocked, postponing deadline\n"); - - local_irq_save(kludge_flags); - - /* Outermost lock of the cluster. Recursive lock calls are - * possible on this code path. This should be the _ONLY_ - * scenario where recursive calls are made. */ -#ifdef CONFIG_LITMUS_DGL_SUPPORT - /* Unfortunately, we _might_ need to grab the DGL lock, so we - * must grab it every time since it must be take before the - * cluster lock. */ - raw_spin_lock_irqsave(&cluster->dgl_lock, flags); - raw_readyq_lock(&cluster->cluster_lock); -#else - raw_readyq_lock_irqsave(&cluster->cluster_lock, flags); -#endif - - job_completion(t, 1); /* refreshes budget and pushes out deadline */ - -#ifdef CONFIG_LITMUS_LOCKING - { - int i; - /* any linked task that inherits from 't' needs to have their - * cpu-position re-evaluated. we have to do this in two passes. - * pass 1: remove nodes from heap s.t. heap is in known good state. - * pass 2: re-add nodes. - * - */ - for (i = find_first_bit(&tsk_rt(t)->used_linkback_slots, BITS_PER_BYTE*sizeof(&tsk_rt(t)->used_linkback_slots)); - i < BITS_PER_LONG; - i = find_next_bit(&tsk_rt(t)->used_linkback_slots, BITS_PER_BYTE*sizeof(&tsk_rt(t)->used_linkback_slots), i+1)) - { - struct task_struct *to_update = tsk_rt(t)->inh_task_linkbacks[i]; - BUG_ON(!to_update); - if (tsk_rt(to_update)->linked_on != NO_CPU) { - cpu_entry_t *entry = &per_cpu(crm_cpu_entries, tsk_rt(to_update)->linked_on); - BUG_ON(!binheap_is_in_heap(&entry->hn)); - binheap_delete(&entry->hn, &cluster->cpu_heap); - } - } - for (i = find_first_bit(&tsk_rt(t)->used_linkback_slots, BITS_PER_BYTE*sizeof(&tsk_rt(t)->used_linkback_slots)); - i < BITS_PER_LONG; - i = find_next_bit(&tsk_rt(t)->used_linkback_slots, BITS_PER_BYTE*sizeof(&tsk_rt(t)->used_linkback_slots), i+1)) - { - struct task_struct *to_update = tsk_rt(t)->inh_task_linkbacks[i]; - BUG_ON(!to_update); - if (tsk_rt(to_update)->linked_on != NO_CPU) { - cpu_entry_t *entry = &per_cpu(crm_cpu_entries, tsk_rt(to_update)->linked_on); - binheap_add(&entry->hn, &cluster->cpu_heap, cpu_entry_t, hn); - } - } - } - - /* Check our inheritance and propagate any changes forward. */ - reevaluate_inheritance(t); -#endif - /* No need to recheck priority of AUX tasks. They will always - * inherit from 't' if they are enabled. Their prio change was - * captured by the cpu-heap operations above. */ - -#ifdef CONFIG_LITMUS_NVIDIA - /* Re-eval priority of GPU interrupt threads. */ - if(tsk_rt(t)->held_gpus && !tsk_rt(t)->hide_from_gpu) - gpu_owner_decrease_priority(t); -#endif - -#ifdef CONFIG_LITMUS_LOCKING - /* double-check that everything is okay */ - check_for_preemptions(cluster); -#endif - - /* should be the outermost unlock call */ -#ifdef CONFIG_LITMUS_DGL_SUPPORT - raw_readyq_unlock(&cluster->cluster_lock); - raw_spin_unlock_irqrestore(&cluster->dgl_lock, flags); -#else - raw_readyq_unlock_irqrestore(&cluster->cluster_lock, flags); -#endif - flush_pending_wakes(); - local_irq_restore(kludge_flags); - - /* we need to set up the budget timer since we're within the callback. */ - hrtimer_forward_now(&get_budget_timer(t).timer.timer, - ns_to_ktime(budget_remaining(t))); - remaining = hrtimer_get_expires_ns(&get_budget_timer(t).timer.timer); - - TRACE_TASK(t, "rearmed timer to %ld\n", remaining); - restart = HRTIMER_RESTART; - } - } - -out: - return restart; -} - - -#ifdef CONFIG_LITMUS_LOCKING -static void __crm_trigger_vunlock(struct task_struct *t) -{ - TRACE_TASK(t, "triggering virtual unlock of lock %d\n", - tsk_rt(t)->outermost_lock->ident); - tsk_rt(t)->outermost_lock->ops->omlp_virtual_unlock(tsk_rt(t)->outermost_lock, t); -} - -static void crm_trigger_vunlock(struct task_struct *t) -{ - crm_domain_t *cluster = task_cpu_cluster(t); -#ifdef CONFIG_LITMUS_DGL_SUPPORT - unsigned long flags; - - /* Unfortunately, we _might_ need to grab the DGL lock, so we - * must grab it every time since it must be take before the - * cluster lock. */ - raw_spin_lock_irqsave(&cluster->dgl_lock, flags); -#endif - - __crm_trigger_vunlock(t); - -#ifdef CONFIG_LITMUS_DGL_SUPPORT - raw_spin_unlock_irqrestore(&cluster->dgl_lock, flags); -#endif -} -#endif - -static enum hrtimer_restart crm_sobliv_on_exhausted(struct task_struct *t, int in_schedule) -{ - enum hrtimer_restart restart = HRTIMER_NORESTART; - - if (in_schedule) { - BUG_ON(tsk_rt(t)->scheduled_on != smp_processor_id()); - if (budget_precisely_tracked(t) && cancel_enforcement_timer(t) == -1) { - TRACE_TASK(t, "raced with timer. deffering to timer.\n"); - goto out; - } - } - - /* t may or may not be scheduled */ - - if (budget_signalled(t) && !bt_flag_is_set(t, BTF_SIG_BUDGET_SENT)) { - /* signal exhaustion */ - - /* Tasks should block SIG_BUDGET if they cannot gracefully respond to - * the signal while suspended. SIG_BUDGET is an rt-signal, so it will - * be queued and received when SIG_BUDGET is unblocked */ - send_sigbudget(t); /* will set BTF_SIG_BUDGET_SENT */ - } - - if (budget_enforced(t) && !bt_flag_is_set(t, BTF_BUDGET_EXHAUSTED)) { - int cpu = (tsk_rt(t)->linked_on != NO_CPU) ? - tsk_rt(t)->linked_on : tsk_rt(t)->scheduled_on; - -#ifdef CONFIG_LITMUS_LOCKING - /* if 't' running, trigger a virtual unlock of outermost held lock - * if supported. Case where 't' not running handled later in function. - */ - if (cpu != NO_CPU && - tsk_rt(t)->outermost_lock && - tsk_rt(t)->outermost_lock->ops->is_omlp_family) - crm_trigger_vunlock(t); -#endif - - if (is_np(t) && is_user_np(t)) { - TRACE_TASK(t, "is non-preemptable, preemption delayed.\n"); - bt_flag_set(t, BTF_BUDGET_EXHAUSTED); - request_exit_np(t); - } - /* where do we need to call resched? */ - else if (cpu == smp_processor_id()) { - bt_flag_set(t, BTF_BUDGET_EXHAUSTED); - if (!in_schedule) { - TRACE_TASK(t, "is preemptable => FORCE_RESCHED\n"); - litmus_reschedule_local(); - set_will_schedule(); - } - } - else if (cpu != NO_CPU) { - bt_flag_set(t, BTF_BUDGET_EXHAUSTED); - if (!in_schedule) { - litmus_reschedule(cpu); - TRACE_TASK(t, "is preemptable on remote cpu (%d) => FORCE_RESCHED\n", cpu); - } - } - else { - lt_t remaining; - crm_domain_t *cluster; - unsigned long flags, kludge_flags; - - BUG_ON(in_schedule); - - cluster = task_cpu_cluster(t); - - // 1) refresh budget through job completion - // 2) if holds locks, tell the locking protocol to re-eval priority - // 3) -- the LP must undo any inheritance relations if appropriate - - /* force job completion */ - TRACE_TASK(t, "blocked, postponing deadline\n"); - - /* Outermost lock of the cluster. Recursive lock calls are - * possible on this code path. This should be the _ONLY_ - * scenario where recursive calls are made. */ - local_irq_save(kludge_flags); -#ifdef CONFIG_LITMUS_DGL_SUPPORT - /* Unfortunately, we _might_ need to grab the DGL lock, so we - * must grab it every time since it must be take before the - * cluster lock. */ - raw_spin_lock_irqsave(&cluster->dgl_lock, flags); - raw_readyq_lock(&cluster->cluster_lock); -#else - raw_readyq_lock_irqsave(&cluster->cluster_lock, flags); -#endif - - job_completion(t, 1); /* refreshes budget and pushes out deadline */ - -#ifdef CONFIG_LITMUS_LOCKING - { - int i; - /* any linked task that inherits from 't' needs to have their - * cpu-position re-evaluated. we have to do this in two passes. - * pass 1: remove nodes from heap s.t. heap is in known good state. - * pass 2: re-add nodes. - * - */ - for (i = find_first_bit(&tsk_rt(t)->used_linkback_slots, BITS_PER_BYTE*sizeof(&tsk_rt(t)->used_linkback_slots)); - i < BITS_PER_LONG; - i = find_next_bit(&tsk_rt(t)->used_linkback_slots, BITS_PER_BYTE*sizeof(&tsk_rt(t)->used_linkback_slots), i+1)) - { - struct task_struct *to_update = tsk_rt(t)->inh_task_linkbacks[i]; - BUG_ON(!to_update); - if (tsk_rt(to_update)->linked_on != NO_CPU) { - cpu_entry_t *entry = &per_cpu(crm_cpu_entries, tsk_rt(to_update)->linked_on); - BUG_ON(!binheap_is_in_heap(&entry->hn)); - binheap_delete(&entry->hn, &cluster->cpu_heap); - } - } - for (i = find_first_bit(&tsk_rt(t)->used_linkback_slots, BITS_PER_BYTE*sizeof(&tsk_rt(t)->used_linkback_slots)); - i < BITS_PER_LONG; - i = find_next_bit(&tsk_rt(t)->used_linkback_slots, BITS_PER_BYTE*sizeof(&tsk_rt(t)->used_linkback_slots), i+1)) - { - struct task_struct *to_update = tsk_rt(t)->inh_task_linkbacks[i]; - BUG_ON(!to_update); - if (tsk_rt(to_update)->linked_on != NO_CPU) { - cpu_entry_t *entry = &per_cpu(crm_cpu_entries, tsk_rt(to_update)->linked_on); - binheap_add(&entry->hn, &cluster->cpu_heap, cpu_entry_t, hn); - } - } - } - - /* Check our inheritance and propagate any changes forward. */ - reevaluate_inheritance(t); - - if (tsk_rt(t)->outermost_lock && tsk_rt(t)->outermost_lock->ops->is_omlp_family) - __crm_trigger_vunlock(t); -#endif - /* No need to recheck priority of AUX tasks. They will always - * inherit from 't' if they are enabled. Their prio change was - * captured by the cpu-heap operations above. */ - -#ifdef CONFIG_LITMUS_NVIDIA - /* Re-eval priority of GPU interrupt threads. */ - if(tsk_rt(t)->held_gpus && !tsk_rt(t)->hide_from_gpu) - gpu_owner_decrease_priority(t); -#endif - -#ifdef CONFIG_LITMUS_LOCKING - /* double-check that everything is okay */ - check_for_preemptions(cluster); -#endif - - /* should be the outermost unlock call */ -#ifdef CONFIG_LITMUS_DGL_SUPPORT - raw_readyq_unlock(&cluster->cluster_lock); - raw_spin_unlock_irqrestore(&cluster->dgl_lock, flags); -#else - raw_readyq_unlock_irqrestore(&cluster->cluster_lock, flags); -#endif - flush_pending_wakes(); - local_irq_restore(kludge_flags); - - /* we need to set up the budget timer since we're within the callback. */ - if (bt_flag_is_set(t, BTF_IS_TOP_M)) { - hrtimer_forward_now(&get_budget_timer(t).timer.timer, - ns_to_ktime(budget_remaining(t))); - remaining = hrtimer_get_expires_ns(&get_budget_timer(t).timer.timer); - - TRACE_TASK(t, "rearmed timer to %ld\n", remaining); - restart = HRTIMER_RESTART; - } - } - } - -out: - return restart; -} - - -/* crm_tick - this function is called for every local timer - * interrupt. - * - * checks whether the current task has expired and checks - * whether we need to preempt it if it has not expired - */ -static void crm_tick(struct task_struct* t) -{ - if (is_realtime(t) && - tsk_rt(t)->budget.ops && budget_quantum_tracked(t) && - budget_exhausted(t)) { - TRACE_TASK(t, "budget exhausted\n"); - budget_state_machine2(t,on_exhausted,!IN_SCHEDULE); - } -} - -#ifdef CONFIG_LITMUS_LOCKING -static int __increase_priority_inheritance(struct task_struct* t, struct task_struct* prio_inh); -#endif - -/* Getting schedule() right is a bit tricky. schedule() may not make any - * assumptions on the state of the current task since it may be called for a - * number of reasons. The reasons include a scheduler_tick() determined that it - * was necessary, because sys_exit_np() was called, because some Linux - * subsystem determined so, or even (in the worst case) because there is a bug - * hidden somewhere. Thus, we must take extreme care to determine what the - * current state is. - * - * The CPU could currently be scheduling a task (or not), be linked (or not). - * - * The following assertions for the scheduled task could hold: - * - * - !is_running(scheduled) // the job blocks - * - scheduled->timeslice == 0 // the job completed (forcefully) - * - is_completed() // the job completed (by syscall) - * - linked != scheduled // we need to reschedule (for any reason) - * - is_np(scheduled) // rescheduling must be delayed, - * sys_exit_np must be requested - * - * Any of these can occur together. - */ -static struct task_struct* crm_schedule(struct task_struct * prev) -{ - cpu_entry_t* entry = &__get_cpu_var(crm_cpu_entries); - crm_domain_t *cluster = entry->cluster; - int out_of_time, sleep, preempt, np, exists, blocks; - struct task_struct* next = NULL; - -#ifdef CONFIG_LITMUS_NESTED_LOCKING - int recheck_inheritance; -#endif - -#ifdef CONFIG_RELEASE_MASTER - /* Bail out early if we are the release master. - * The release master never schedules any real-time tasks. - */ - if (unlikely(cluster->domain.release_master == entry->cpu)) { - sched_state_task_picked(); - return NULL; - } -#endif - - /* Detect and handle budget exhaustion if it hasn't already been done. - * Do this before acquring any locks. */ - if (prev && is_realtime(prev) && - budget_exhausted(prev) && - !is_completed(prev) && /* don't bother with jobs on their way out */ - ((budget_enforced(prev) && !bt_flag_is_set(prev, BTF_BUDGET_EXHAUSTED)) || - (budget_signalled(prev) && !bt_flag_is_set(prev, BTF_SIG_BUDGET_SENT))) ) { - TRACE_TASK(prev, "handling exhaustion in schedule() at %llu\n", litmus_clock()); - budget_state_machine2(prev,on_exhausted,IN_SCHEDULE); - } - -#ifdef CONFIG_LITMUS_NESTED_LOCKING - /* prevent updates to inheritance relations while we work with 'prev' */ - /* recheck inheritance if the task holds locks, is running, and will - * have its deadline pushed out by job_completion() */ - recheck_inheritance = - prev && - is_realtime(prev) && - holds_locks(prev) && - !is_np(prev) && - !is_completed(prev) && - is_running(prev) && - budget_enforced(prev) && - bt_flag_is_set(prev, BTF_BUDGET_EXHAUSTED); - if (recheck_inheritance) { -#ifdef CONFIG_LITMUS_DGL_SUPPORT - raw_spin_lock(&cluster->dgl_lock); -#endif - raw_spin_lock(&tsk_rt(prev)->hp_blocked_tasks_lock); - } -#endif - - raw_readyq_lock(&cluster->cluster_lock); - clear_will_schedule(); - - /* sanity checking */ - BUG_ON(entry->scheduled && entry->scheduled != prev); - BUG_ON(entry->scheduled && !is_realtime(prev)); - BUG_ON(is_realtime(prev) && !entry->scheduled); - - /* (0) Determine state */ - exists = entry->scheduled != NULL; - blocks = exists && !is_running(entry->scheduled); - out_of_time = exists && - budget_enforced(entry->scheduled) && - bt_flag_is_set(entry->scheduled, BTF_BUDGET_EXHAUSTED); - np = exists && is_np(entry->scheduled); - sleep = exists && is_completed(entry->scheduled); - preempt = entry->scheduled != entry->linked; - -#ifdef WANT_ALL_SCHED_EVENTS - TRACE_TASK(prev, "invoked crm_schedule.\n"); -#endif - - if (exists) { - TRACE_TASK(prev, - "blocks:%d out_of_time:%d np:%d completed:%d preempt:%d " - "state:%d sig:%d\n", - blocks, out_of_time, np, sleep, preempt, - prev->state, signal_pending(prev)); - } - if (entry->linked && preempt) - TRACE_TASK(prev, "will be preempted by %s/%d\n", - entry->linked->comm, entry->linked->pid); - -#ifdef CONFIG_REALTIME_AUX_TASKS - if (tsk_rt(prev)->is_aux_task && - (prev->state == TASK_INTERRUPTIBLE) && - !blocks) { - TRACE_TASK(prev, "Deferring descheduling of aux task %s/%d.\n", - prev->comm, prev->pid); - next = prev; /* allow prev to continue. */ - goto out_set_state; - } -#endif - - /* Do budget stuff */ - if (blocks) { - if (likely(!bt_flag_is_set(prev, BTF_WAITING_FOR_RELEASE))) - budget_state_machine(prev,on_blocked); - else { - /* waiting for release. 'exit' the scheduler. */ - crm_untrack_in_top_m(prev); - budget_state_machine(prev,on_exit); - } - } - else if (sleep) - budget_state_machine(prev,on_sleep); - else if (preempt) - budget_state_machine(prev,on_preempt); - - /* If a task blocks we have no choice but to reschedule. - */ - if (blocks) - unlink(entry->scheduled); - -#if defined(CONFIG_LITMUS_NVIDIA) && defined(CONFIG_LITMUS_AFFINITY_LOCKING) - if(exists && is_realtime(entry->scheduled) && tsk_rt(entry->scheduled)->held_gpus) { - if(!blocks || tsk_rt(entry->scheduled)->suspend_gpu_tracker_on_block) { - // don't track preemptions or locking protocol suspensions. - TRACE_TASK(entry->scheduled, "stopping GPU tracker.\n"); - stop_gpu_tracker(entry->scheduled); - } - else if(blocks && !tsk_rt(entry->scheduled)->suspend_gpu_tracker_on_block) { - TRACE_TASK(entry->scheduled, "GPU tracker remains on during suspension.\n"); - } - } -#endif - - /* Request a sys_exit_np() call if we would like to preempt but cannot. - * We need to make sure to update the link structure anyway in case - * that we are still linked. Multiple calls to request_exit_np() don't - * hurt. - */ - if (np && (out_of_time || preempt || sleep)) { - unlink(entry->scheduled); - request_exit_np(entry->scheduled); - } - - /* Any task that is preemptable and either exhausts its execution - * budget or wants to sleep completes. We may have to reschedule after - * this. Don't do a job completion if we block (can't have timers running - * for blocked jobs). - */ - if (!np && (out_of_time || sleep) && !blocks) { - job_completion(entry->scheduled, !sleep); -#ifdef CONFIG_LITMUS_NESTED_LOCKING - /* check if job completion enables an inheritance relation. no need to - * recheck if task already inherits a priority since job_completion() - * will not enable a higher-prio relation */ - if (unlikely(recheck_inheritance && !tsk_rt(entry->scheduled)->inh_task)) { - struct task_struct *hp_blocked; - TRACE_TASK(entry->scheduled, "rechecking inheritance.\n"); - hp_blocked = top_priority(&tsk_rt(entry->scheduled)->hp_blocked_tasks); - /* hp_blocked_tasks_lock is held */ - if (rm_higher_prio(hp_blocked, entry->scheduled)) - __increase_priority_inheritance(entry->scheduled, effective_priority(hp_blocked)); - } -#endif - } - - /* Link pending task if we became unlinked. - */ - if (!entry->linked) - link_task_to_cpu(__take_ready(&cluster->domain), entry); - - /* The final scheduling decision. Do we need to switch for some reason? - * If linked is different from scheduled, then select linked as next. - */ - if ((!np || blocks) && - entry->linked != entry->scheduled) { - /* Schedule a linked job? */ - if (entry->linked) { - entry->linked->rt_param.scheduled_on = entry->cpu; - next = entry->linked; - } - if (entry->scheduled) { - /* not gonna be scheduled soon */ - entry->scheduled->rt_param.scheduled_on = NO_CPU; - TRACE_TASK(entry->scheduled, "scheduled_on = NO_CPU\n"); - } - } - else { - /* Only override Linux scheduler if we have a real-time task - * scheduled that needs to continue. - */ - if (exists) { - next = prev; - } - } - -#ifdef CONFIG_REALTIME_AUX_TASKS -out_set_state: -#endif - - sched_state_task_picked(); - raw_readyq_unlock(&cluster->cluster_lock); - -#ifdef CONFIG_LITMUS_NESTED_LOCKING - if (recheck_inheritance) { - raw_spin_unlock(&tsk_rt(prev)->hp_blocked_tasks_lock); -#ifdef CONFIG_LITMUS_DGL_SUPPORT - raw_spin_unlock(&cluster->dgl_lock); -#endif - } -#endif - -#ifdef WANT_ALL_SCHED_EVENTS - TRACE("cluster_lock released, next=0x%p\n", next); - - if (next) - TRACE_TASK(next, "scheduled at %llu\n", litmus_clock()); - else if (exists && !next) - TRACE("becomes idle at %llu.\n", litmus_clock()); -#endif - - return next; -} - - -/* _finish_switch - we just finished the switch away from prev - */ -static void crm_finish_switch(struct task_struct *prev) -{ - cpu_entry_t* entry = &__get_cpu_var(crm_cpu_entries); - - entry->scheduled = is_realtime(current) ? current : NULL; -#ifdef WANT_ALL_SCHED_EVENTS - TRACE_TASK(prev, "switched away from\n"); -#endif -} - - -/* Prepare a task for running in RT mode - */ -static void crm_task_new(struct task_struct * t, int on_rq, int running) -{ - unsigned long flags; - cpu_entry_t* entry; - crm_domain_t* cluster; - - TRACE("c-fp: task new %d (param running = %d, is_running = %d)\n", t->pid, running, is_running(t)); - - /* the cluster doesn't change even if t is running */ - cluster = task_cpu_cluster(t); - - raw_readyq_lock_irqsave(&cluster->cluster_lock, flags); - - /* setup job params */ - release_at(t, litmus_clock()); - - t->rt_param.linked_on = NO_CPU; - - if (running) { - entry = &per_cpu(crm_cpu_entries, task_cpu(t)); - BUG_ON(entry->scheduled); - -#ifdef CONFIG_RELEASE_MASTER - if (entry->cpu != cluster->domain.release_master) { -#endif - entry->scheduled = t; - tsk_rt(t)->scheduled_on = task_cpu(t); -#ifdef CONFIG_RELEASE_MASTER - } else { - /* do not schedule on release master */ - preempt(entry); /* force resched */ - tsk_rt(t)->scheduled_on = NO_CPU; - } -#endif - } else { - t->rt_param.scheduled_on = NO_CPU; - } - - if (is_running(t)) { - crm_track_in_top_m(t); - crm_job_arrival(t); - } - - raw_readyq_unlock_irqrestore(&cluster->cluster_lock, flags); -} - -static void crm_task_wake_up(struct task_struct *t) -{ - unsigned long flags; - crm_domain_t *cluster; - lt_t now; - - cluster = task_cpu_cluster(t); - - raw_readyq_lock_irqsave(&cluster->cluster_lock, flags); - - now = litmus_clock(); - TRACE_TASK(t, "wake_up at %llu\n", now); - - if (is_sporadic(t) && is_tardy(t, now)) { - release_at(t, now); - sched_trace_task_release(t); - } - else { - /* periodic task model. don't force job to end. - * rely on user to say when jobs complete or when budget expires. */ - tsk_rt(t)->completed = 0; - } - -#ifdef CONFIG_REALTIME_AUX_TASKS - if (tsk_rt(t)->has_aux_tasks && !tsk_rt(t)->hide_from_aux_tasks) { - TRACE_CUR("%s/%d is ready so aux tasks may not inherit.\n", t->comm, t->pid); - disable_aux_task_owner(t); - } -#endif - -#ifdef CONFIG_LITMUS_NVIDIA - if (tsk_rt(t)->held_gpus && !tsk_rt(t)->hide_from_gpu) { - TRACE_CUR("%s/%d is ready so gpu klmirqd tasks may not inherit.\n", t->comm, t->pid); - disable_gpu_owner(t); - } -#endif - - budget_state_machine(t,on_wakeup); - crm_job_arrival(t); - - raw_readyq_unlock_irqrestore(&cluster->cluster_lock, flags); -} - -static void crm_task_block(struct task_struct *t) -{ - unsigned long flags; - crm_domain_t *cluster; - - TRACE_TASK(t, "block at %llu\n", litmus_clock()); - - cluster = task_cpu_cluster(t); - - /* unlink if necessary */ - raw_readyq_lock_irqsave(&cluster->cluster_lock, flags); - - unlink(t); - -#ifdef CONFIG_REALTIME_AUX_TASKS - if (tsk_rt(t)->has_aux_tasks && !tsk_rt(t)->hide_from_aux_tasks) { - - TRACE_CUR("%s/%d is blocked so aux tasks may inherit.\n", t->comm, t->pid); - enable_aux_task_owner(t); - } -#endif - -#ifdef CONFIG_LITMUS_NVIDIA - if (tsk_rt(t)->held_gpus && !tsk_rt(t)->hide_from_gpu) { - - TRACE_CUR("%s/%d is blocked so klmirqd threads may inherit.\n", t->comm, t->pid); - enable_gpu_owner(t); - } -#endif - - raw_readyq_unlock_irqrestore(&cluster->cluster_lock, flags); - - BUG_ON(!is_realtime(t)); -} - - -static void crm_task_exit(struct task_struct * t) -{ - unsigned long flags; - crm_domain_t *cluster = task_cpu_cluster(t); - - /* unlink if necessary */ - raw_readyq_lock_irqsave(&cluster->cluster_lock, flags); - - if (tsk_rt(t)->inh_task) { - WARN_ON(1); - clear_inh_task_linkback(t, tsk_rt(t)->inh_task); - } - - /* disable budget enforcement */ - crm_untrack_in_top_m(t); - budget_state_machine(t,on_exit); - -#ifdef CONFIG_REALTIME_AUX_TASKS - /* make sure we clean up on our way out */ - if (unlikely(tsk_rt(t)->is_aux_task)) - exit_aux_task(t); - else if(tsk_rt(t)->has_aux_tasks) - disable_aux_task_owner(t); -#endif - -#ifdef CONFIG_LITMUS_NVIDIA - /* make sure we clean up on our way out */ - if(tsk_rt(t)->held_gpus) - disable_gpu_owner(t); -#endif - - unlink(t); - if (tsk_rt(t)->scheduled_on != NO_CPU) { - cpu_entry_t *cpu; - cpu = &per_cpu(crm_cpu_entries, tsk_rt(t)->scheduled_on); - cpu->scheduled = NULL; - tsk_rt(t)->scheduled_on = NO_CPU; - } - raw_readyq_unlock_irqrestore(&cluster->cluster_lock, flags); - - BUG_ON(!is_realtime(t)); - TRACE_TASK(t, "RIP\n"); -} - - - - - - -static struct budget_tracker_ops crm_drain_simple_ops = -{ - .on_scheduled = simple_on_scheduled, - .on_blocked = simple_on_blocked, - .on_preempt = simple_on_preempt, - .on_sleep = simple_on_sleep, - .on_exit = simple_on_exit, - - .on_wakeup = NULL, - .on_inherit = NULL, - .on_disinherit = NULL, - .on_enter_top_m = NULL, - .on_exit_top_m = NULL, - - .on_exhausted = crm_simple_on_exhausted, -}; - -static struct budget_tracker_ops crm_drain_simple_io_ops = -{ - .on_scheduled = simple_io_on_scheduled, - .on_blocked = simple_io_on_blocked, - .on_preempt = simple_io_on_preempt, - .on_sleep = simple_io_on_sleep, - .on_exit = simple_io_on_exit, - - .on_wakeup = simple_io_on_wakeup, - .on_inherit = NULL, - .on_disinherit = NULL, - .on_enter_top_m = NULL, - .on_exit_top_m = NULL, - - .on_exhausted = crm_simple_io_on_exhausted, -}; - -static struct budget_tracker_ops crm_drain_sobliv_ops = -{ - .on_scheduled = NULL, - .on_preempt = NULL, - .on_sleep = NULL, - - .on_blocked = sobliv_on_blocked, - .on_wakeup = sobliv_on_wakeup, - .on_exit = sobliv_on_exit, - .on_inherit = sobliv_on_inherit, - .on_disinherit = sobliv_on_disinherit, - .on_enter_top_m = sobliv_on_enter_top_m, - .on_exit_top_m = sobliv_on_exit_top_m, - - .on_exhausted = crm_sobliv_on_exhausted, -}; - -static long crm_admit_task(struct task_struct* tsk) -{ - struct budget_tracker_ops* ops = NULL; - - if (remote_cluster(task_cpu(tsk)) != task_cpu_cluster(tsk)) { -// printk("rejected admit: incorrect cluster.\n"); -// return -EINVAL; - } - - if (budget_enforced(tsk) || budget_signalled(tsk)) { - switch(get_drain_policy(tsk)) { - case DRAIN_SIMPLE: - ops = &crm_drain_simple_ops; - break; - case DRAIN_SIMPLE_IO: - ops = &crm_drain_simple_io_ops; - break; - case DRAIN_SOBLIV: - /* budget_policy and budget_signal_policy cannot be quantum-based */ - if (!budget_quantum_tracked(tsk) && budget_precisely_tracked(tsk)) { - ops = &crm_drain_sobliv_ops; - } - else { - printk("rejected admit: QUANTUM_ENFORCEMENT and QUANTUM_SIGNALS is " - "unsupported with DRAIN_SOBLIV.\n"); - return -EINVAL; - } - break; - default: - printk("rejected admit: Unsupported budget draining mode.\n"); - return -EINVAL; - } - } - - /* always init the budget tracker, even if we're not using timers */ - init_budget_tracker(&tsk_rt(tsk)->budget, ops); - -#ifdef CONFIG_LITMUS_NESTED_LOCKING - INIT_BINHEAP_HANDLE(&tsk_rt(tsk)->hp_blocked_tasks, - rm_max_heap_base_priority_order); -#endif - - return 0; -} - - - -#ifdef CONFIG_LITMUS_LOCKING - -#include - -/* called with IRQs off */ -static int __increase_priority_inheritance(struct task_struct* t, - struct task_struct* prio_inh) -{ - int success = 1; - int linked_on; - int check_preempt = 0; - crm_domain_t* cluster; - struct task_struct* old_prio_inh = tsk_rt(t)->inh_task; - - if (prio_inh && prio_inh == effective_priority(t)) { - /* relationship already established. */ - TRACE_TASK(t, "already has effective priority of %s/%d\n", - prio_inh->comm, prio_inh->pid); - goto out; - } - - if (prio_inh && (effective_priority(prio_inh) != prio_inh)) { - TRACE_TASK(t, "Inheriting from %s/%d instead of the eff_prio = %s/%d!\n", - prio_inh->comm, prio_inh->pid, - effective_priority(prio_inh)->comm, - effective_priority(prio_inh)->pid); -#ifndef CONFIG_LITMUS_NESTED_LOCKING - /* Tasks should only inherit the base priority of a task. - If 't' inherits a priority, then tsk_rt(t)->inh_task should - be passed to this function instead. This includes transitive - inheritance relations (tsk_rt(tsk_rt(...)->inh_task)->inh_task). */ - BUG(); -#else - /* Not a bug with nested locking since inheritance propagation is - not atomic. */ - - /* TODO: Is the following 'helping' short-cut safe? - prio_inh = effective_priority(prio_inh); - */ -#endif - } - - cluster = task_cpu_cluster(t); - -#if 0 - if (prio_inh && task_cpu_cluster(prio_inh) != cluster) { - WARN_ONCE(1, "Illegal to inherit between clusters. " \ - "target (%s/%d) on cluster w/ CPU %d and " \ - "inh_prio (%s/%d) on w/ CPU %d\n", \ - t->comm, t->pid, cluster->cpus[0]->cpu, \ - prio_inh->comm, prio_inh->pid, \ - task_cpu_cluster(prio_inh)->cpus[0]->cpu); - return 1; - } -#endif - -#ifdef CONFIG_LITMUS_NESTED_LOCKING - /* this sanity check allows for weaker locking in protocols */ - /* TODO (klmirqd): Skip this check if 't' is a proxy thread (???) */ - if(__rm_higher_prio(prio_inh, BASE, t, EFFECTIVE)) { -#endif - sched_trace_eff_prio_change(t, prio_inh); - - /* clear out old inheritance relation */ - if (old_prio_inh) { - budget_state_machine_chgprio(t,old_prio_inh,on_disinherit); - clear_inh_task_linkback(t, old_prio_inh); - } - - TRACE_TASK(t, "inherits priority from %s/%d\n", - prio_inh->comm, prio_inh->pid); - tsk_rt(t)->inh_task = prio_inh; - - /* update inheritance relation */ - if (prio_inh) - budget_state_machine_chgprio(t,prio_inh,on_inherit); - - linked_on = tsk_rt(t)->linked_on; - - /* If it is scheduled, then we need to reorder the CPU heap. */ - if (linked_on != NO_CPU) { - TRACE_TASK(t, "%s: linked on %d\n", - __FUNCTION__, linked_on); - /* Holder is scheduled; need to re-order CPUs. - * We can't use heap_decrease() here since - * the cpu_heap is ordered in reverse direction, so - * it is actually an increase. */ - binheap_delete(&per_cpu(crm_cpu_entries, linked_on).hn, - &cluster->cpu_heap); - binheap_add(&per_cpu(crm_cpu_entries, linked_on).hn, - &cluster->cpu_heap, cpu_entry_t, hn); - - /* tell prio_inh that we're __running__ with its priority */ - set_inh_task_linkback(t, prio_inh); - } - else { - /* holder may be queued: first stop queue changes */ - raw_spin_lock(&cluster->domain.release_lock); - if (is_queued(t)) { - TRACE_TASK(t, "%s: is queued\n", - __FUNCTION__); - /* We need to update the position of holder in some - * heap. Note that this could be a release heap if we - * budget enforcement is used and this job overran. */ - check_preempt = - !bheap_decrease(rm_ready_order, tsk_rt(t)->heap_node); - } else { - /* Nothing to do: if it is not queued and not linked - * then it is either sleeping or currently being moved - * by other code (e.g., a timer interrupt handler) that - * will use the correct priority when enqueuing the - * task. */ - TRACE_TASK(t, "%s: is NOT queued => Done.\n", - __FUNCTION__); - } - raw_spin_unlock(&cluster->domain.release_lock); - -#ifdef CONFIG_REALTIME_AUX_TASKS - /* propagate to aux tasks */ - if (tsk_rt(t)->has_aux_tasks) { - aux_task_owner_increase_priority(t); - } -#endif - -#ifdef CONFIG_LITMUS_NVIDIA - /* propagate to gpu klmirqd */ - if (tsk_rt(t)->held_gpus) { - gpu_owner_increase_priority(t); - } -#endif - - /* If holder was enqueued in a release heap, then the following - * preemption check is pointless, but we can't easily detect - * that case. If you want to fix this, then consider that - * simply adding a state flag requires O(n) time to update when - * releasing n tasks, which conflicts with the goal to have - * O(log n) merges. */ - if (check_preempt) { - /* heap_decrease() hit the top level of the heap: make - * sure preemption checks get the right task, not the - * potentially stale cache. */ - bheap_uncache_min(rm_ready_order, - &cluster->domain.ready_queue); - check_for_preemptions(cluster); - } - } -#ifdef CONFIG_LITMUS_NESTED_LOCKING - } - else { - /* Occurance is okay under two scenarios: - * 1. Fine-grain nested locks (no compiled DGL support): Concurrent - * updates are chasing each other through the wait-for chain. - * 2. Budget exhausion caused the HP waiter to loose its priority, but - * the lock structure hasn't yet been updated (but soon will be). - */ - TRACE_TASK(t, "Spurious invalid priority increase. " - "Inheritance request: %s/%d [eff_prio = %s/%d] to inherit from %s/%d" - "Occurance is likely okay: probably due to (hopefully safe) concurrent priority updates.\n", - t->comm, t->pid, - effective_priority(t)->comm, effective_priority(t)->pid, - (prio_inh) ? prio_inh->comm : "null", - (prio_inh) ? prio_inh->pid : 0); - WARN_ON(!prio_inh); - success = 0; - } -#endif - -out: - return success; -} - -/* called with IRQs off */ -static void increase_priority_inheritance(struct task_struct* t, struct task_struct* prio_inh) -{ - crm_domain_t* cluster = task_cpu_cluster(t); - - raw_readyq_lock(&cluster->cluster_lock); - - TRACE_TASK(t, "to inherit from %s/%d\n", prio_inh->comm, prio_inh->pid); - - __increase_priority_inheritance(t, prio_inh); - - raw_readyq_unlock(&cluster->cluster_lock); -} - -/* called with IRQs off */ -static int __decrease_priority_inheritance(struct task_struct* t, - struct task_struct* prio_inh, - int budget_tiggered) -{ - crm_domain_t* cluster; - int success = 1; - struct task_struct* old_prio_inh = tsk_rt(t)->inh_task; - - if (prio_inh == old_prio_inh) { - /* relationship already established. */ - TRACE_TASK(t, "already inherits priority from %s/%d\n", - (prio_inh) ? prio_inh->comm : "(null)", - (prio_inh) ? prio_inh->pid : 0); - goto out; - } - - if (prio_inh && (effective_priority(prio_inh) != prio_inh)) { - TRACE_TASK(t, "Inheriting from %s/%d instead of the eff_prio = %s/%d!\n", - prio_inh->comm, prio_inh->pid, - effective_priority(prio_inh)->comm, - effective_priority(prio_inh)->pid); -#ifndef CONFIG_LITMUS_NESTED_LOCKING - /* Tasks should only inherit the base priority of a task. - If 't' inherits a priority, then tsk_rt(t)->inh_task should - be passed to this function instead. This includes transitive - inheritance relations (tsk_rt(tsk_rt(...)->inh_task)->inh_task). */ - BUG(); -#else - /* Not a bug with nested locking since inheritance propagation is - not atomic. */ - - /* TODO: Is the following 'helping' short-cut safe? - prio_inh = effective_priority(prio_inh); - */ -#endif - } - - cluster = task_cpu_cluster(t); - -#if 0 - if (prio_inh && task_cpu_cluster(prio_inh) != cluster) { - WARN_ONCE(1, "Illegal to inherit between clusters. " \ - "target (%s/%d) on cluster w/ CPU %d and " \ - "inh_prio (%s/%d) on w/ CPU %d\n", \ - t->comm, t->pid, cluster->cpus[0]->cpu, \ - prio_inh->comm, prio_inh->pid, \ - task_cpu_cluster(prio_inh)->cpus[0]->cpu); - return 1; - } -#endif - -#ifdef CONFIG_LITMUS_NESTED_LOCKING - if(budget_tiggered || __rm_higher_prio(t, EFFECTIVE, prio_inh, BASE)) { -#endif - sched_trace_eff_prio_change(t, prio_inh); - - if (budget_tiggered) { - BUG_ON(!old_prio_inh); - TRACE_TASK(t, "budget-triggered 'decrease' in priority. " - "%s/%d's budget should have just been exhuasted.\n", - old_prio_inh->comm, old_prio_inh->pid); - } - - /* clear out old inheritance relation */ - if (old_prio_inh) { - budget_state_machine_chgprio(t,old_prio_inh,on_disinherit); - clear_inh_task_linkback(t, old_prio_inh); - } - - /* A job only stops inheriting a priority when it releases a - * resource. Thus we can make the following assumption.*/ - if(prio_inh) - TRACE_TASK(t, "EFFECTIVE priority decreased to %s/%d\n", - prio_inh->comm, prio_inh->pid); - else - TRACE_TASK(t, "base priority restored.\n"); - - /* set up new inheritance relation */ - tsk_rt(t)->inh_task = prio_inh; - - if (prio_inh) - budget_state_machine_chgprio(t,prio_inh,on_inherit); - - if(tsk_rt(t)->scheduled_on != NO_CPU) { - TRACE_TASK(t, "is scheduled.\n"); - - /* link back to new inheritance */ - if (prio_inh) - set_inh_task_linkback(t, prio_inh); - - /* Check if rescheduling is necessary. We can't use heap_decrease() - * since the priority was effectively lowered. */ - unlink(t); - crm_job_arrival(t); - } - else { - /* task is queued */ - raw_spin_lock(&cluster->domain.release_lock); - if (is_queued(t)) { - TRACE_TASK(t, "is queued.\n"); - - BUG_ON( - !is_released(t, litmus_clock()) && - !tsk_rt(t)->job_params.is_backlogged_job && - !is_early_releasing(t)); - - unlink(t); - crm_job_arrival(t); - } - else { - TRACE_TASK(t, "is not in scheduler. Probably on wait queue somewhere.\n"); - } - raw_spin_unlock(&cluster->domain.release_lock); - } - -#ifdef CONFIG_REALTIME_AUX_TASKS - /* propagate to aux tasks */ - if (tsk_rt(t)->has_aux_tasks) - aux_task_owner_decrease_priority(t); -#endif - -#ifdef CONFIG_LITMUS_NVIDIA - /* propagate to gpu */ - if (tsk_rt(t)->held_gpus) - gpu_owner_decrease_priority(t); -#endif - -#ifdef CONFIG_LITMUS_NESTED_LOCKING - } - else { - TRACE_TASK(t, "Spurious invalid priority decrease. " - "Inheritance request: %s/%d [eff_prio = %s/%d] to inherit from %s/%d\n" - "Occurance is likely okay: probably due to (hopefully safe) concurrent priority updates.\n", - t->comm, t->pid, - effective_priority(t)->comm, effective_priority(t)->pid, - (prio_inh) ? prio_inh->comm : "null", - (prio_inh) ? prio_inh->pid : 0); - success = 0; - } -#endif - -out: - return success; -} - -static void decrease_priority_inheritance(struct task_struct* t, - struct task_struct* prio_inh, - int budget_tiggered) -{ - crm_domain_t* cluster = task_cpu_cluster(t); - - raw_readyq_lock(&cluster->cluster_lock); - - TRACE_TASK(t, "to inherit from %s/%d (decrease)\n", - (prio_inh) ? prio_inh->comm : "null", - (prio_inh) ? prio_inh->pid : 0); - - __decrease_priority_inheritance(t, prio_inh, budget_tiggered); - - raw_readyq_unlock(&cluster->cluster_lock); -} - - -#ifdef CONFIG_LITMUS_NESTED_LOCKING - -/* called with IRQs off */ -/* preconditions: - (1) The 'hp_blocked_tasks_lock' of task 't' is held. - (2) The lock 'to_unlock' is held. - */ -static void nested_increase_priority_inheritance(struct task_struct* t, - struct task_struct* prio_inh, - raw_spinlock_t *to_unlock, - unsigned long irqflags) -{ - struct litmus_lock *blocked_lock = tsk_rt(t)->blocked_lock; - - if(tsk_rt(t)->inh_task != prio_inh) { // shield redundent calls. - increase_priority_inheritance(t, prio_inh); // increase our prio. - } - - /* note: cluster lock is not held continuously during propagation, so there - may be momentary inconsistencies while nested priority propagation 'chases' - other updates. */ - - raw_spin_unlock(&tsk_rt(t)->hp_blocked_tasks_lock); // unlock the t's heap. - - if(blocked_lock) { - if(blocked_lock->ops->supports_nesting) { - TRACE_TASK(t, "Inheritor is blocked (...perhaps). Checking lock %d.\n", - blocked_lock->ident); - - // beware: recursion - blocked_lock->ops->propagate_increase_inheritance(blocked_lock, - t, to_unlock, - irqflags); - } - else { - TRACE_TASK(t, "Inheritor is blocked on litmus lock (%d) that does not support nesting!\n", - blocked_lock->ident); - unlock_fine_irqrestore(to_unlock, irqflags); - } - } - else { - TRACE_TASK(t, "is not blocked on litmus lock. No propagation.\n"); - unlock_fine_irqrestore(to_unlock, irqflags); - } -} - -/* called with IRQs off */ -/* preconditions: - (1) The 'hp_blocked_tasks_lock' of task 't' is held. - (2) The lock 'to_unlock' is held. - */ -static void nested_decrease_priority_inheritance(struct task_struct* t, - struct task_struct* prio_inh, - raw_spinlock_t *to_unlock, - unsigned long irqflags, - int budget_tiggered) -{ - struct litmus_lock *blocked_lock = tsk_rt(t)->blocked_lock; - decrease_priority_inheritance(t, prio_inh, budget_tiggered); - - raw_spin_unlock(&tsk_rt(t)->hp_blocked_tasks_lock); // unlock the t's heap. - - if(blocked_lock) { - if(blocked_lock->ops->supports_nesting) { - TRACE_TASK(t, "Inheritor is blocked (...perhaps). Checking lock %d.\n", - blocked_lock->ident); - // beware: recursion - blocked_lock->ops->propagate_decrease_inheritance(blocked_lock, t, - to_unlock, - irqflags, - budget_tiggered); - } - else { - TRACE_TASK(t, "Inheritor is blocked on lock (%p) that does not support nesting!\n", - blocked_lock); - unlock_fine_irqrestore(to_unlock, irqflags); - } - } - else { - TRACE_TASK(t, "is not blocked. No propagation.\n"); - unlock_fine_irqrestore(to_unlock, irqflags); - } -} - - -/* ******************** FIFO MUTEX ********************** */ - -static struct litmus_lock_ops crm_fifo_mutex_lock_ops = { - .lock = fifo_mutex_lock, - .unlock = fifo_mutex_unlock, - .should_yield_lock = fifo_mutex_should_yield_lock, - .close = fifo_mutex_close, - .deallocate = fifo_mutex_free, - - .budget_exhausted = fifo_mutex_budget_exhausted, - .propagate_increase_inheritance = fifo_mutex_propagate_increase_inheritance, - .propagate_decrease_inheritance = fifo_mutex_propagate_decrease_inheritance, - -#ifdef CONFIG_LITMUS_DGL_SUPPORT - .dgl_lock = fifo_mutex_dgl_lock, - .is_owner = fifo_mutex_is_owner, - .get_owner = fifo_mutex_get_owner, - .enable_priority = fifo_mutex_enable_priority, - - .dgl_can_quick_lock = NULL, - .dgl_quick_lock = NULL, - - .supports_dgl = 1, - .requires_atomic_dgl = 0, -#endif - .supports_nesting = 1, - .supports_budget_exhaustion = 1, - .is_omlp_family = 0, -}; - -static struct litmus_lock* crm_new_fifo_mutex(void) -{ - return fifo_mutex_new(&crm_fifo_mutex_lock_ops); -} - -/* ******************** PRIOQ MUTEX ********************** */ - -static struct litmus_lock_ops crm_prioq_mutex_lock_ops = { - .lock = prioq_mutex_lock, - .unlock = prioq_mutex_unlock, - .should_yield_lock = prioq_mutex_should_yield_lock, - .close = prioq_mutex_close, - .deallocate = prioq_mutex_free, - - .budget_exhausted = prioq_mutex_budget_exhausted, - .propagate_increase_inheritance = prioq_mutex_propagate_increase_inheritance, - .propagate_decrease_inheritance = prioq_mutex_propagate_decrease_inheritance, - -#ifdef CONFIG_LITMUS_DGL_SUPPORT - .dgl_lock = prioq_mutex_dgl_lock, - .is_owner = prioq_mutex_is_owner, - .get_owner = prioq_mutex_get_owner, - .enable_priority = prioq_mutex_enable_priority, - - .dgl_can_quick_lock = prioq_mutex_dgl_can_quick_lock, - .dgl_quick_lock = prioq_mutex_dgl_quick_lock, - - .supports_dgl = 1, - .requires_atomic_dgl = 1, -#endif - .supports_nesting = 1, - .supports_budget_exhaustion = 1, - .is_omlp_family = 0, -}; - -static struct litmus_lock* crm_new_prioq_mutex(void) -{ - return prioq_mutex_new(&crm_prioq_mutex_lock_ops); -} - -/* ******************** IKGLP ********************** */ - -static struct litmus_lock_ops crm_ikglp_lock_ops = { - .lock = ikglp_lock, - .unlock = ikglp_unlock, - .should_yield_lock = NULL, - .close = ikglp_close, - .deallocate = ikglp_free, - - .budget_exhausted = ikglp_budget_exhausted, - .omlp_virtual_unlock = ikglp_virtual_unlock, - - // ikglp can only be an outer-most lock. - .propagate_increase_inheritance = NULL, - .propagate_decrease_inheritance = NULL, - -#ifdef CONFIG_LITMUS_DGL_SUPPORT - .supports_dgl = 0, - .requires_atomic_dgl = 0, -#endif - .supports_nesting = 0, - .supports_budget_exhaustion = 1, - .is_omlp_family = 1, -}; - -static struct litmus_lock* crm_new_ikglp(void* __user arg) -{ - // assumes clusters of uniform size. - return ikglp_new(cluster_size, &crm_ikglp_lock_ops, arg); -} - - -/* ******************** KFMLP support ********************** */ - -static struct litmus_lock_ops crm_kfmlp_lock_ops = { - .lock = kfmlp_lock, - .unlock = kfmlp_unlock, - .should_yield_lock = NULL, - .close = kfmlp_close, - .deallocate = kfmlp_free, - - // kfmlp can only be an outer-most lock. - .propagate_increase_inheritance = NULL, - .propagate_decrease_inheritance = NULL, - -#ifdef CONFIG_LITMUS_DGL_SUPPORT - .supports_dgl = 0, - .requires_atomic_dgl = 0, -#endif - .supports_nesting = 0, - .supports_budget_exhaustion = 0, - .is_omlp_family = 0, -}; - - -static struct litmus_lock* crm_new_kfmlp(void* __user arg) -{ - return kfmlp_new(&crm_kfmlp_lock_ops, arg); -} - - -/* **** lock constructor **** */ - -static long crm_allocate_lock(struct litmus_lock **lock, int type, - void* __user args) -{ - int err; - - switch (type) { -#ifdef CONFIG_LITMUS_NESTED_LOCKING - case FIFO_MUTEX: - *lock = crm_new_fifo_mutex(); - break; - - case PRIOQ_MUTEX: - *lock = crm_new_prioq_mutex(); - break; - - case IKGLP_SEM: - *lock = crm_new_ikglp(args); - break; -#endif - case KFMLP_SEM: - *lock = crm_new_kfmlp(args); - break; - - default: - err = -ENXIO; - goto UNSUPPORTED_LOCK; - }; - - if (*lock) - err = 0; - else - err = -ENOMEM; - -UNSUPPORTED_LOCK: - return err; -} - -#endif // CONFIG_LITMUS_LOCKING - - -#ifdef CONFIG_LITMUS_AFFINITY_LOCKING -static struct affinity_observer_ops crm_kfmlp_affinity_ops __attribute__ ((unused)) = { - .close = kfmlp_aff_obs_close, - .deallocate = kfmlp_aff_obs_free, -}; - -#ifdef CONFIG_LITMUS_NESTED_LOCKING -static struct affinity_observer_ops crm_ikglp_affinity_ops __attribute__ ((unused)) = { - .close = ikglp_aff_obs_close, - .deallocate = ikglp_aff_obs_free, -}; -#endif - -static long crm_allocate_affinity_observer(struct affinity_observer **aff_obs, - int type, - void* __user args) -{ - int err; - - switch (type) { -#ifdef CONFIG_LITMUS_NVIDIA - case KFMLP_SIMPLE_GPU_AFF_OBS: - *aff_obs = kfmlp_simple_gpu_aff_obs_new(&crm_kfmlp_affinity_ops, args); - break; - - case KFMLP_GPU_AFF_OBS: - *aff_obs = kfmlp_gpu_aff_obs_new(&crm_kfmlp_affinity_ops, args); - break; - -#ifdef CONFIG_LITMUS_NESTED_LOCKING - case IKGLP_SIMPLE_GPU_AFF_OBS: - *aff_obs = ikglp_simple_gpu_aff_obs_new(&crm_ikglp_affinity_ops, args); - break; - - case IKGLP_GPU_AFF_OBS: - *aff_obs = ikglp_gpu_aff_obs_new(&crm_ikglp_affinity_ops, args); - break; -#endif -#endif - default: - err = -ENXIO; - goto UNSUPPORTED_AFF_OBS; - }; - - if (*aff_obs) - err = 0; - else - err = -ENOMEM; - -UNSUPPORTED_AFF_OBS: - return err; -} -#endif - - - -#endif // CONFIG_LITMUS_NESTED_LOCKING - - -#ifdef VERBOSE_INIT -static void print_cluster_topology(cpumask_var_t mask, int cpu) -{ - int chk; - char buf[255]; - - chk = cpulist_scnprintf(buf, 254, mask); - buf[chk] = '\0'; - printk(KERN_INFO "CPU = %d, shared cpu(s) = %s\n", cpu, buf); - -} -#endif - -static void cleanup_crm(void) -{ - int i; - - if (clusters_allocated) { - for (i = 0; i < num_clusters; i++) { - kfree(crm[i].cpus); - free_cpumask_var(crm[i].cpu_map); - } - - kfree(crm); - } -} - -#if defined(CONFIG_LITMUS_NVIDIA) && defined(CONFIG_LITMUS_SOFTIRQD) -static int crm_map_gpu_to_cpu(int gpu) -{ - int default_cpu; - int cpu_cluster = gpu / gpu_cluster_size; - - /* bonham-specific hack for the fully partitioned case (both CPUs and GPUs partitioned) */ - /* TODO: Make this aware of the NUMA topology generically */ - if(num_clusters == 12 && num_gpu_clusters == 8) { - if(gpu >= 4) { - cpu_cluster += 2; // assign the GPU to a CPU on the same NUMA node - } - } - - default_cpu = crm[cpu_cluster].cpus[0]->cpu; // first CPU in given cluster - - TRACE("CPU %d is default for GPU %d interrupt threads.\n", default_cpu, gpu); - - return default_cpu; -} -#endif - -static long crm_activate_plugin(void) -{ - int i, j, cpu, ccpu, cpu_count; - cpu_entry_t *entry; - - cpumask_var_t mask; - int chk = 0; - - /* de-allocate old clusters, if any */ - cleanup_crm(); - - - printk(KERN_INFO "C-RM: Activate Plugin, cluster configuration = %d\n", - cluster_config); - - /* need to get cluster_size first */ - if(!zalloc_cpumask_var(&mask, GFP_ATOMIC)) - return -ENOMEM; - - if (unlikely(cluster_config == GLOBAL_CLUSTER)) { - cluster_size = num_online_cpus(); - } else { - chk = get_shared_cpu_map(mask, 0, cluster_config); - if (chk) { - /* if chk != 0 then it is the max allowed index */ - printk(KERN_INFO "C-RM: Cluster configuration = %d " - "is not supported on this hardware.\n", - cluster_config); - /* User should notice that the configuration failed, so - * let's bail out. */ - return -EINVAL; - } - - cluster_size = cpumask_weight(mask); - } - - if ((num_online_cpus() % cluster_size) != 0) { - /* this can't be right, some cpus are left out */ - printk(KERN_ERR "C-RM: Trying to group %d cpus in %d!\n", - num_online_cpus(), cluster_size); - return -1; - } - - num_clusters = num_online_cpus() / cluster_size; - printk(KERN_INFO "C-RM: %d cluster(s) of size = %d\n", - num_clusters, cluster_size); - - -#if defined(CONFIG_LITMUS_NVIDIA) && defined(CONFIG_LITMUS_SOFTIRQD) - num_gpu_clusters = min(num_clusters, num_online_gpus()); - gpu_cluster_size = num_online_gpus() / num_gpu_clusters; - - if (((num_online_gpus() % gpu_cluster_size) != 0) || - (num_gpu_clusters != num_clusters)) { - printk(KERN_WARNING "C-RM: GPUs not uniformly distributed among CPU clusters.\n"); - } -#endif - - /* initialize clusters */ - crm = kmalloc(num_clusters * sizeof(crm_domain_t), GFP_ATOMIC); - for (i = 0; i < num_clusters; i++) { - - crm[i].cpus = kmalloc(cluster_size * sizeof(cpu_entry_t), - GFP_ATOMIC); - INIT_BINHEAP_HANDLE(&(crm[i].cpu_heap), cpu_lower_prio); - rm_domain_init(&(crm[i].domain), NULL, crm_release_jobs); - - if(!zalloc_cpumask_var(&crm[i].cpu_map, GFP_ATOMIC)) - return -ENOMEM; -#ifdef CONFIG_RELEASE_MASTER - crm[i].domain.release_master = atomic_read(&release_master_cpu); -#endif - } - - /* cycle through cluster and add cpus to them */ - for (i = 0; i < num_clusters; i++) { - -#ifdef CONFIG_LITMUS_DGL_SUPPORT - raw_spin_lock_init(&crm[i].dgl_lock); -#endif - -#ifdef RECURSIVE_READY_QUEUE_LOCK - crm[i].recursive_depth = 0; - atomic_set(&crm[i].owner_cpu, NO_CPU); -#endif - - crm[i].top_m_size = 0; - INIT_BINHEAP_HANDLE(&crm[i].top_m, crm_min_heap_base_priority_order); - INIT_BINHEAP_HANDLE(&crm[i].not_top_m, crm_max_heap_base_priority_order); - - for_each_online_cpu(cpu) { - /* check if the cpu is already in a cluster */ - for (j = 0; j < num_clusters; j++) - if (cpumask_test_cpu(cpu, crm[j].cpu_map)) - break; - /* if it is in a cluster go to next cpu */ - if (j < num_clusters && - cpumask_test_cpu(cpu, crm[j].cpu_map)) - continue; - - /* this cpu isn't in any cluster */ - /* get the shared cpus */ - if (unlikely(cluster_config == GLOBAL_CLUSTER)) - cpumask_copy(mask, cpu_online_mask); - else - get_shared_cpu_map(mask, cpu, cluster_config); - - cpumask_copy(crm[i].cpu_map, mask); -#ifdef VERBOSE_INIT - print_cluster_topology(mask, cpu); -#endif - /* add cpus to current cluster and init cpu_entry_t */ - cpu_count = 0; - for_each_cpu(ccpu, crm[i].cpu_map) { - - entry = &per_cpu(crm_cpu_entries, ccpu); - crm[i].cpus[cpu_count] = entry; - - memset(entry, 0, sizeof(*entry)); - entry->cpu = ccpu; - entry->cluster = &crm[i]; - INIT_BINHEAP_NODE(&entry->hn); - mb(); - - ++cpu_count; - -#ifdef CONFIG_RELEASE_MASTER - /* only add CPUs that should schedule jobs */ - if (entry->cpu != entry->cluster->domain.release_master) -#endif - update_cpu_position(entry); - } - /* done with this cluster */ - break; - } - } - -#ifdef CONFIG_LITMUS_SOFTIRQD - init_klmirqd(); -#endif - -#ifdef CONFIG_LITMUS_NVIDIA - init_nvidia_info(); -#endif - - init_wake_queues(); - - free_cpumask_var(mask); - clusters_allocated = 1; - return 0; -} - -/* Plugin object */ -static struct sched_plugin crm_plugin __cacheline_aligned_in_smp = { - .plugin_name = "C-RM", // for now - .finish_switch = crm_finish_switch, - .tick = crm_tick, - .task_new = crm_task_new, - .complete_job = complete_job, - .task_exit = crm_task_exit, - .schedule = crm_schedule, - .task_wake_up = crm_task_wake_up, - .task_block = crm_task_block, - .admit_task = crm_admit_task, - .activate_plugin = crm_activate_plugin, - .compare = rm_higher_prio, -#ifdef CONFIG_LITMUS_LOCKING - .allocate_lock = crm_allocate_lock, - .increase_prio = increase_priority_inheritance, - .decrease_prio = decrease_priority_inheritance, - .__increase_prio = __increase_priority_inheritance, - .__decrease_prio = __decrease_priority_inheritance, -#endif -#ifdef CONFIG_LITMUS_NESTED_LOCKING - .nested_increase_prio = nested_increase_priority_inheritance, - .nested_decrease_prio = nested_decrease_priority_inheritance, - .__compare = __rm_higher_prio, -#endif -#ifdef CONFIG_LITMUS_DGL_SUPPORT - .get_dgl_spinlock = crm_get_dgl_spinlock, -#endif -#ifdef CONFIG_LITMUS_AFFINITY_LOCKING - .allocate_aff_obs = crm_allocate_affinity_observer, -#endif -#if defined(CONFIG_LITMUS_NVIDIA) && defined(CONFIG_LITMUS_SOFTIRQD) - .map_gpu_to_cpu = crm_map_gpu_to_cpu, -#endif -}; - -static struct proc_dir_entry *cluster_file = NULL, *crm_dir = NULL; - -static int __init init_crm(void) -{ - int err, fs; - - err = register_sched_plugin(&crm_plugin); - if (!err) { - fs = make_plugin_proc_dir(&crm_plugin, &crm_dir); - if (!fs) - cluster_file = create_cluster_file(crm_dir, &cluster_config); - else - printk(KERN_ERR "Could not allocate C-RM procfs dir.\n"); - } - return err; -} - -static void clean_crm(void) -{ - cleanup_crm(); - if (cluster_file) - remove_proc_entry("cluster", crm_dir); - if (crm_dir) - remove_plugin_proc_dir(&crm_plugin); -} - -module_init(init_crm); -module_exit(clean_crm); -- cgit v1.2.2