aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorJonathan Herman <hermanjl@cs.unc.edu>2011-09-27 20:15:32 -0400
committerJonathan Herman <hermanjl@cs.unc.edu>2011-09-27 20:36:04 -0400
commit23a00b911b968c6290251913ecc34171836b4d32 (patch)
treef6c8289054d2961902931e89bdc11ccc01bc3a73
parentf21e1d0ef90c2e88ae6a563afc31ea601ed968c7 (diff)
parent609c45f71b7a2405230fd2f8436837d6389ec599 (diff)
Merged with ce domains
Diffstat
-rw-r--r--include/litmus/ce_domain.h23
-rw-r--r--include/litmus/event_group.h2
-rw-r--r--include/litmus/sched_mc.h76
-rw-r--r--litmus/ce_domain.c105
-rw-r--r--litmus/event_group.c62
-rw-r--r--litmus/rt_domain.c16
-rw-r--r--litmus/sched_mc.c224
-rw-r--r--litmus/sched_mc_ce.c365
8 files changed, 533 insertions, 340 deletions
diff --git a/include/litmus/ce_domain.h b/include/litmus/ce_domain.h
new file mode 100644
index 000000000000..373f3f5f78d3
--- /dev/null
+++ b/include/litmus/ce_domain.h
@@ -0,0 +1,23 @@
1#ifndef _LITMUS_CE_DOMAIN_H
2#define _LITMUS_CE_DOMAIN_H
3
4/*
5 * Functions that the MC plugin needs to call through a domain pointer.
6 */
7void ce_requeue(domain_t*, struct task_struct*);
8struct task_struct* ce_peek_and_take_ready(domain_t*);
9int ce_higher_prio(struct task_struct*, struct task_struct*);
10
11typedef enum hrtimer_restart (*ce_timer_callback_t)(struct hrtimer*);
12
13void ce_domain_init(domain_t*,
14 raw_spinlock_t*,
15 requeue_t,
16 peek_ready_t,
17 take_ready_t,
18 preempt_needed_t,
19 task_prio_t,
20 struct ce_dom_data*,
21 const int,
22 ce_timer_callback_t);
23#endif
diff --git a/include/litmus/event_group.h b/include/litmus/event_group.h
index 37d5012d770e..a2e4d4507738 100644
--- a/include/litmus/event_group.h
+++ b/include/litmus/event_group.h
@@ -66,7 +66,7 @@ void add_event(struct event_group* group, struct rt_event* e, lt_t time);
66/** 66/**
67 * cancel_event() - Remove event from the group. 67 * cancel_event() - Remove event from the group.
68 */ 68 */
69void cancel_event(struct rt_event*); 69void cancel_event(struct event_group *group, struct rt_event*);
70 70
71/** 71/**
72 * init_event() - Create an event. 72 * init_event() - Create an event.
diff --git a/include/litmus/sched_mc.h b/include/litmus/sched_mc.h
index 9ddf860c83a7..95cc367c8ade 100644
--- a/include/litmus/sched_mc.h
+++ b/include/litmus/sched_mc.h
@@ -21,7 +21,10 @@ struct mc_job {
21}; 21};
22 22
23#ifdef __KERNEL__ 23#ifdef __KERNEL__
24/* only used in the kernel (no user space) */ 24/*
25 * These are used only in the kernel. Userspace programs like RTSpin won't see
26 * them.
27 */
25 28
26struct mc_data { 29struct mc_data {
27 struct mc_task mc_task; 30 struct mc_task mc_task;
@@ -36,22 +39,15 @@ struct mc_data {
36 (t) ? t->pid : 1, \ 39 (t) ? t->pid : 1, \
37 (t) ? t->rt_param.job_params.job_no : 1, \ 40 (t) ? t->rt_param.job_params.job_no : 1, \
38 (t && get_task_domain(t)) ? get_task_domain(t)->name : "" 41 (t && get_task_domain(t)) ? get_task_domain(t)->name : ""
42#define STRACE(fmt, args...) \
43 sched_trace_log_message("%d P%d [%s@%s:%d]: " fmt, \
44 TRACE_ARGS, ## args)
39#define TRACE_MC_TASK(t, fmt, args...) \ 45#define TRACE_MC_TASK(t, fmt, args...) \
40 TRACE(TS " " fmt "\n", TA(t), ##args) 46 STRACE(TS " " fmt "\n", TA(t), ##args)
41 47
42/* 48/*
43 * Cache the budget along with the struct PID for a task so that we don't need 49 * The MC-CE scheduler uses this as domain data.
44 * to fetch its task_struct every time we check to see what should be
45 * scheduled.
46 */ 50 */
47struct ce_dom_pid_entry {
48 struct pid *pid;
49 lt_t budget;
50 /* accumulated (summed) budgets, including this one */
51 lt_t acc_time;
52 int expected_job;
53};
54
55struct ce_dom_data { 51struct ce_dom_data {
56 int cpu; 52 int cpu;
57 struct task_struct *scheduled, *should_schedule; 53 struct task_struct *scheduled, *should_schedule;
@@ -59,13 +55,61 @@ struct ce_dom_data {
59 * Each CPU needs a mapping of level A ID (integer) to struct pid so 55 * Each CPU needs a mapping of level A ID (integer) to struct pid so
60 * that we can get its task struct. 56 * that we can get its task struct.
61 */ 57 */
62 struct ce_dom_pid_entry pid_entries[CONFIG_PLUGIN_MC_LEVEL_A_MAX_TASKS];
63 int num_pid_entries;
64 lt_t cycle_time;
65 struct hrtimer_start_on_info timer_info; 58 struct hrtimer_start_on_info timer_info;
66 struct hrtimer timer; 59 struct hrtimer timer;
67}; 60};
68 61
62/**
63 * struct crit_entry - State of a CPU within each criticality level system.
64 * @level Criticality level of this entry
65 * @linked Logically running task, ghost or regular
66 * @domain Domain from which to draw tasks
67 * @usable False if a higher criticality task is running
68 * @timer For ghost task budget enforcement
69 * @node Used to sort crit_entries by preemptability in global domains
70 */
71struct crit_entry {
72 enum crit_level level;
73 struct task_struct* linked;
74 struct domain* domain;
75 int usable;
76#ifdef CONFIG_MERGE_TIMERS
77 struct rt_event event;
78#else
79 struct hrtimer timer;
80#endif
81 struct bheap_node* node;
82};
83
84/**
85 * struct domain_data - Wrap domains with related CPU state
86 * @domain A domain for a criticality level
87 * @heap The preemptable heap of crit entries (for global domains)
88 * @crit_entry The crit entry for this domain (for partitioned domains)
89 */
90struct domain_data {
91 struct domain domain;
92 struct bheap* heap;
93 struct crit_entry* crit_entry;
94};
95
96/*
97 * Functions that are used with the MC-CE plugin.
98 */
99long mc_ce_set_domains(const int, struct domain_data*[]);
100unsigned int mc_ce_get_expected_job(const int, const int);
101
102/*
103 * These functions are (lazily) inserted into the MC plugin code so that it
104 * manipulates the MC-CE state.
105 */
106long mc_ce_admit_task_common(struct task_struct*);
107void mc_ce_task_exit_common(struct task_struct*);
108void mc_ce_timer_callback_common(domain_t*, struct hrtimer*);
109void mc_ce_release_at_common(struct task_struct*, lt_t);
110long mc_ce_activate_plugin_common(void);
111long mc_ce_deactivate_plugin_common(void);
112
69#endif /* __KERNEL__ */ 113#endif /* __KERNEL__ */
70 114
71#endif 115#endif
diff --git a/litmus/ce_domain.c b/litmus/ce_domain.c
index 5b4fd1cb438f..ac6cc14d44f7 100644
--- a/litmus/ce_domain.c
+++ b/litmus/ce_domain.c
@@ -6,7 +6,9 @@
6#include <litmus/debug_trace.h> 6#include <litmus/debug_trace.h>
7#include <litmus/rt_param.h> 7#include <litmus/rt_param.h>
8#include <litmus/domain.h> 8#include <litmus/domain.h>
9#include <litmus/event_group.h>
9#include <litmus/sched_mc.h> 10#include <litmus/sched_mc.h>
11#include <litmus/ce_domain.h>
10 12
11/* 13/*
12 * Called for: 14 * Called for:
@@ -18,50 +20,26 @@ void ce_requeue(domain_t *dom, struct task_struct *ts)
18{ 20{
19 const struct ce_dom_data *ce_data = dom->data; 21 const struct ce_dom_data *ce_data = dom->data;
20 const int idx = tsk_mc_data(ts)->mc_task.lvl_a_id; 22 const int idx = tsk_mc_data(ts)->mc_task.lvl_a_id;
21 const struct ce_dom_pid_entry *pid_entry = 23 const unsigned int just_finished = tsk_rt(ts)->job_params.job_no;
22 &ce_data->pid_entries[idx]; 24 const unsigned int expected_job =
23 const int just_finished = tsk_rt(ts)->job_params.job_no; 25 mc_ce_get_expected_job(ce_data->cpu, idx);
24 const int expected_job = pid_entry->expected_job;
25 const int asleep = RT_F_SLEEP == get_rt_flags(ts); 26 const int asleep = RT_F_SLEEP == get_rt_flags(ts);
26 27
27 TRACE_TASK(ts, "entered ce_requeue. asleep: %d just_finished: %4d " 28 TRACE_MC_TASK(ts, "entered ce_requeue. asleep: %d just_finished: %3u "
28 "expected_job: %4d\n", 29 "expected_job: %3u",
29 asleep, just_finished, expected_job); 30 asleep, just_finished, expected_job);
30 31
31 /* When coming from job completion, the task will be asleep. */ 32 /* When coming from job completion, the task will be asleep. */
32 if (asleep && just_finished < expected_job) { 33 if (asleep && just_finished < expected_job) {
33 /* this job is running behind, so don't put it to sleep */ 34 TRACE_MC_TASK(ts, "appears behind");
34 set_rt_flags(ts, RT_F_RUNNING);
35 TRACE_TASK(ts, "appears behind, setting it to running again\n");
36 } else if (asleep && expected_job < just_finished) { 35 } else if (asleep && expected_job < just_finished) {
37 printk(KERN_CRIT "job %d completed in expected job %d which " 36 TRACE_MC_TASK(ts, "job %u completed in expected job %u which "
38 "seems too early\n", just_finished, 37 "seems too early", just_finished,
39 expected_job); 38 expected_job);
40 BUG();
41 } 39 }
42} 40}
43 41
44/* 42/*
45 * Called when a task exits the system.
46 */
47void ce_exit(domain_t *dom, struct task_struct *ts)
48{
49 struct ce_dom_data *ce_data = dom->data;
50 const int lvl_a_id = tsk_mc_data(ts)->mc_task.lvl_a_id;
51 struct pid *pid;
52
53 BUG_ON(task_cpu(ts) != get_partition(ts));
54 BUG_ON(CRIT_LEVEL_A != tsk_mc_crit(ts));
55 BUG_ON(lvl_a_id >= ce_data->num_pid_entries);
56 pid = ce_data->pid_entries[lvl_a_id].pid;
57 BUG_ON(!pid);
58 put_pid(pid);
59 ce_data->pid_entries[lvl_a_id].pid = NULL;
60 if (ce_data->should_schedule == ts)
61 ce_data->should_schedule = NULL;
62}
63
64/*
65 * ce_take_ready and ce_peek_ready 43 * ce_take_ready and ce_peek_ready
66 */ 44 */
67struct task_struct* ce_peek_and_take_ready(domain_t *dom) 45struct task_struct* ce_peek_and_take_ready(domain_t *dom)
@@ -71,57 +49,36 @@ struct task_struct* ce_peek_and_take_ready(domain_t *dom)
71 const int exists = NULL != ce_data->should_schedule; 49 const int exists = NULL != ce_data->should_schedule;
72 const int blocked = exists && !is_running(ce_data->should_schedule); 50 const int blocked = exists && !is_running(ce_data->should_schedule);
73 51
74 /* Return the task we should schedule if it is not blocked. If it is 52 /* Return the task we should schedule if it is not blocked or sleeping. */
75 * asleep, return it anyway, because the MC-scheduler might as about
76 * ghost jobs.
77 */
78 if (exists && !blocked) 53 if (exists && !blocked)
79 ret = ce_data->should_schedule; 54 ret = ce_data->should_schedule;
80 return ret; 55 return ret;
81} 56}
82 57
83int ce_higher_prio(domain_t *dom, struct task_struct *_a, 58int ce_higher_prio(struct task_struct *_a, struct task_struct *_b)
84 struct task_struct *_b)
85{ 59{
86 const struct task_struct *a = _a; 60 const struct task_struct *a = _a;
87 struct ce_dom_data *ce_data = dom->data; 61 const domain_t *dom = get_task_domain(a);
62 const struct ce_dom_data *ce_data = dom->data;
88 return (a == ce_data->should_schedule); 63 return (a == ce_data->should_schedule);
89} 64}
90 65
91void __mc_ce_timer_callback(struct hrtimer *timer); 66void ce_domain_init(domain_t *dom,
92static enum hrtimer_restart ce_timer_function(struct hrtimer *timer) 67 raw_spinlock_t *lock,
93{ 68 requeue_t requeue,
94 /* need to lock? */ 69 peek_ready_t peek_ready,
95 __mc_ce_timer_callback(timer); 70 take_ready_t take_ready,
96 return HRTIMER_RESTART; 71 preempt_needed_t preempt_needed,
97} 72 task_prio_t task_prio,
98 73 struct ce_dom_data *dom_data,
99void mc_ce_release_at(struct task_struct*, lt_t); 74 const int cpu,
100void ce_start(struct task_struct *ts, lt_t start) 75 ce_timer_callback_t ce_timer_callback)
101{
102 mc_ce_release_at(ts, start);
103}
104
105long mc_ce_activate_plugin(void);
106domain_t *ce_domain_for(int);
107long ce_activate_plugin(void)
108{
109 domain_t *dom;
110 struct ce_dom_data *ce_data;
111 int cpu;
112
113 /* first change the timer callback function */
114 for_each_online_cpu(cpu) {
115 dom = ce_domain_for(cpu);
116 ce_data = dom->data;
117 ce_data->timer.function = ce_timer_function;
118 }
119 /* then run the regular CE activate plugin */
120 return mc_ce_activate_plugin();
121}
122
123long mc_ce_deactivate_plugin(void);
124long ce_deactivate_plugin(void)
125{ 76{
126 return mc_ce_deactivate_plugin(); 77 domain_init(dom, lock, requeue, peek_ready, take_ready, preempt_needed,
78 task_prio);
79 dom->data = dom_data;
80 dom_data->cpu = cpu;
81 hrtimer_start_on_info_init(&dom_data->timer_info);
82 hrtimer_init(&dom_data->timer, CLOCK_MONOTONIC, HRTIMER_MODE_ABS);
83 dom_data->timer.function = ce_timer_callback;
127} 84}
diff --git a/litmus/event_group.c b/litmus/event_group.c
index b4521ab370d1..276ba5dd242d 100644
--- a/litmus/event_group.c
+++ b/litmus/event_group.c
@@ -6,7 +6,9 @@
6#include <litmus/event_group.h> 6#include <litmus/event_group.h>
7 7
8#if 1 8#if 1
9#define VTRACE TRACE 9#define VTRACE(fmt, args...) \
10sched_trace_log_message("%d P%d [%s@%s:%d]: " fmt, \
11 TRACE_ARGS, ## args)
10#else 12#else
11#define VTRACE(fmt, args...) 13#define VTRACE(fmt, args...)
12#endif 14#endif
@@ -46,8 +48,8 @@ static enum hrtimer_restart on_timer(struct hrtimer *timer)
46 list_for_each_safe(pos, safe, &list) { 48 list_for_each_safe(pos, safe, &list) {
47 num++; 49 num++;
48 e = list_entry(pos, struct rt_event, list); 50 e = list_entry(pos, struct rt_event, list);
49 TRACE("Dequeueing event with prio %d from 0x%p\n", 51 TRACE("Dequeueing event 0x%p with prio %d from 0x%p\n",
50 e->prio, el); 52 e, e->prio, el);
51 list_del_init(pos); 53 list_del_init(pos);
52 e->function(e); 54 e->function(e);
53 } 55 }
@@ -69,12 +71,14 @@ void insert_event(struct event_list *el, struct rt_event *e)
69 VTRACE("Inserting priority %d 0x%p before %d 0x%p " 71 VTRACE("Inserting priority %d 0x%p before %d 0x%p "
70 "in 0x%p, pos 0x%p\n", e->prio, &e->list, 72 "in 0x%p, pos 0x%p\n", e->prio, &e->list,
71 queued->prio, &queued->list, el, pos); 73 queued->prio, &queued->list, el, pos);
74 BUG_ON(!list_empty(&e->list));
72 list_add_tail(&e->list, pos); 75 list_add_tail(&e->list, pos);
73 return; 76 return;
74 } 77 }
75 } 78 }
76 VTRACE("Inserting priority %d 0x%p at end of 0x%p, last 0x%p\n", 79 VTRACE("Inserting priority %d 0x%p at end of 0x%p, last 0x%p\n",
77 e->prio, &el->list, el, last); 80 e->prio, &el->list, el, last);
81 BUG_ON(!list_empty(&e->list));
78 list_add(&e->list, (last) ? last : pos); 82 list_add(&e->list, (last) ? last : pos);
79} 83}
80 84
@@ -91,12 +95,14 @@ static struct event_list* get_event_list(struct event_group *group,
91 struct list_head* pos; 95 struct list_head* pos;
92 struct event_list *el = NULL, *tmp; 96 struct event_list *el = NULL, *tmp;
93 unsigned int slot = time2slot(fire); 97 unsigned int slot = time2slot(fire);
98 int remaining = 300;
94 99
95 VTRACE("Getting list for %llu\n", fire); 100 VTRACE("Getting list for %llu, event 0x%p\n", fire, e);
96 101
97 /* Initialize pos for the case that the list is empty */ 102 /* Initialize pos for the case that the list is empty */
98 pos = group->event_queue[slot].next; 103 pos = group->event_queue[slot].next;
99 list_for_each(pos, &group->event_queue[slot]) { 104 list_for_each(pos, &group->event_queue[slot]) {
105 BUG_ON(remaining-- < 0);
100 tmp = list_entry(pos, struct event_list, list); 106 tmp = list_entry(pos, struct event_list, list);
101 if (lt_after_eq(fire, tmp->fire_time) && 107 if (lt_after_eq(fire, tmp->fire_time) &&
102 lt_before(fire, tmp->fire_time + group->res)) { 108 lt_before(fire, tmp->fire_time + group->res)) {
@@ -121,10 +127,11 @@ static struct event_list* get_event_list(struct event_group *group,
121 tmp->fire_time = fire; 127 tmp->fire_time = fire;
122 tmp->group = group; 128 tmp->group = group;
123 /* Add to queue */ 129 /* Add to queue */
124 list_add(&tmp->list, pos->prev);
125 el = tmp;
126 VTRACE("Using list for priority %d and time %llu\n", 130 VTRACE("Using list for priority %d and time %llu\n",
127 e->prio, fire); 131 e->prio, fire);
132 BUG_ON(!list_empty(&tmp->list));
133 list_add(&tmp->list, pos->prev);
134 el = tmp;
128 } 135 }
129 return el; 136 return el;
130} 137}
@@ -135,8 +142,8 @@ static struct event_list* get_event_list(struct event_group *group,
135static void reinit_event_list(struct rt_event *e) 142static void reinit_event_list(struct rt_event *e)
136{ 143{
137 struct event_list *el = e->event_list; 144 struct event_list *el = e->event_list;
138 BUG_ON(hrtimer_cancel(&el->timer)); 145 VTRACE("Reinitting 0x%p for event 0x%p\n", el, e);
139 VTRACE("Reinitting 0x%p\n", el); 146 BUG_ON(hrtimer_try_to_cancel(&el->timer) == 1);
140 INIT_LIST_HEAD(&el->events); 147 INIT_LIST_HEAD(&el->events);
141 atomic_set(&el->info.state, HRTIMER_START_ON_INACTIVE); 148 atomic_set(&el->info.state, HRTIMER_START_ON_INACTIVE);
142} 149}
@@ -148,8 +155,8 @@ void add_event(struct event_group *group, struct rt_event *e, lt_t fire)
148{ 155{
149 struct event_list *el; 156 struct event_list *el;
150 157
151 VTRACE("Adding event with priority %d for time %llu\n", 158 VTRACE("Adding event 0x%p with priority %d for time %llu\n",
152 e->prio, fire); 159 e, e->prio, fire);
153 160
154 raw_spin_lock(&group->queue_lock); 161 raw_spin_lock(&group->queue_lock);
155 el = get_event_list(group, e, fire, 0); 162 el = get_event_list(group, e, fire, 0);
@@ -167,7 +174,7 @@ void add_event(struct event_group *group, struct rt_event *e, lt_t fire)
167 174
168 /* Arm timer if we are the owner */ 175 /* Arm timer if we are the owner */
169 if (el == e->event_list) { 176 if (el == e->event_list) {
170 VTRACE("Arming timer for %llu\n", fire); 177 VTRACE("Arming timer on event 0x%p for %llu\n", e, fire);
171 if (group->cpu == smp_processor_id()) { 178 if (group->cpu == smp_processor_id()) {
172 __hrtimer_start_range_ns(&el->timer, 179 __hrtimer_start_range_ns(&el->timer,
173 ns_to_ktime(el->fire_time), 180 ns_to_ktime(el->fire_time),
@@ -185,14 +192,37 @@ void add_event(struct event_group *group, struct rt_event *e, lt_t fire)
185/** 192/**
186 * cancel_event() - Remove event from the group. 193 * cancel_event() - Remove event from the group.
187 */ 194 */
188void cancel_event(struct rt_event *e) 195void cancel_event(struct event_group *group, struct rt_event *e)
189{ 196{
190 struct event_group *group; 197 struct list_head *swap = NULL;
198 struct rt_event *swappy;
199 struct event_list *tmp;
200
191 if (e->list.next != &e->list) { 201 if (e->list.next != &e->list) {
192 group = e->event_list->group;
193 raw_spin_lock(&group->queue_lock); 202 raw_spin_lock(&group->queue_lock);
194 VTRACE("Canceling event with priority %d\n", e->prio); 203 VTRACE("Canceling event 0x%p with priority %d\n", e, e->prio);
195 list_del_init(&e->list); 204
205 /* If somebody else is hooked up to our event list, swap
206 * with their event list and leave our old event list
207 * to execute.
208 */
209 if (!list_empty(&e->list)) {
210 swap = (e->list.next == &e->event_list->events) ?
211 (e->list.prev == &e->event_list->events) ?
212 NULL : e->list.prev : e->list.next;
213 list_del_init(&e->list);
214 }
215 if (swap) {
216 swappy = list_entry(swap, struct rt_event, list);
217 tmp = swappy->event_list;
218 swappy->event_list = e->event_list;
219 e->event_list = tmp;
220 VTRACE("Swapping with event 0x%p", swappy);
221 }
222
223 hrtimer_try_to_cancel(&e->event_list->timer);
224 list_del_init(&e->event_list->list);
225
196 raw_spin_unlock(&group->queue_lock); 226 raw_spin_unlock(&group->queue_lock);
197 } 227 }
198} 228}
diff --git a/litmus/rt_domain.c b/litmus/rt_domain.c
index 50a6abfd7676..db92e849f084 100644
--- a/litmus/rt_domain.c
+++ b/litmus/rt_domain.c
@@ -104,8 +104,8 @@ void release_heap_free(struct release_heap* rh)
104{ 104{
105 /* make sure timer is no longer in use */ 105 /* make sure timer is no longer in use */
106#ifdef CONFIG_MERGE_TIMERS 106#ifdef CONFIG_MERGE_TIMERS
107 if (rh->dom) 107 /* if (rh->dom) */
108 cancel_event(&rh->event); 108 /* cancel_event(&rh->event); */
109#else 109#else
110 hrtimer_cancel(&rh->timer); 110 hrtimer_cancel(&rh->timer);
111#endif 111#endif
@@ -166,7 +166,7 @@ static void reinit_release_heap(rt_domain_t *rt, struct task_struct* t)
166 166
167#ifdef CONFIG_MERGE_TIMERS 167#ifdef CONFIG_MERGE_TIMERS
168 rh->event.prio = rt->prio; 168 rh->event.prio = rt->prio;
169 cancel_event(&rh->event); 169 cancel_event(rt->event_group, &rh->event);
170#else 170#else
171 /* Make sure it is safe to use. The timer callback could still 171 /* Make sure it is safe to use. The timer callback could still
172 * be executing on another CPU; hrtimer_cancel() will wait 172 * be executing on another CPU; hrtimer_cancel() will wait
@@ -323,7 +323,7 @@ void rt_domain_init(rt_domain_t *rt,
323 */ 323 */
324void __add_ready(rt_domain_t* rt, struct task_struct *new) 324void __add_ready(rt_domain_t* rt, struct task_struct *new)
325{ 325{
326 TRACE("rt: adding %s/%d (%llu, %llu) rel=%llu to ready queue at %llu\n", 326 VTRACE("rt: adding %s/%d (%llu, %llu) rel=%llu to ready queue at %llu\n",
327 new->comm, new->pid, get_exec_cost(new), get_rt_period(new), 327 new->comm, new->pid, get_exec_cost(new), get_rt_period(new),
328 get_release(new), litmus_clock()); 328 get_release(new), litmus_clock());
329 329
@@ -348,7 +348,7 @@ void __merge_ready(rt_domain_t* rt, struct bheap* tasks)
348void __add_release_on(rt_domain_t* rt, struct task_struct *task, 348void __add_release_on(rt_domain_t* rt, struct task_struct *task,
349 int target_cpu) 349 int target_cpu)
350{ 350{
351 TRACE_TASK(task, "add_release_on(), rel=%llu, target=%d\n", 351 VTRACE_TASK(task, "add_release_on(), rel=%llu, target=%d\n",
352 get_release(task), target_cpu); 352 get_release(task), target_cpu);
353 list_add(&tsk_rt(task)->list, &rt->tobe_released); 353 list_add(&tsk_rt(task)->list, &rt->tobe_released);
354 task->rt_param.domain = rt; 354 task->rt_param.domain = rt;
@@ -367,7 +367,7 @@ void __add_release_on(rt_domain_t* rt, struct task_struct *task,
367 */ 367 */
368void __add_release(rt_domain_t* rt, struct task_struct *task) 368void __add_release(rt_domain_t* rt, struct task_struct *task)
369{ 369{
370 TRACE_TASK(task, "add_release(), rel=%llu\n", get_release(task)); 370 VTRACE_TASK(task, "add_release(), rel=%llu\n", get_release(task));
371 list_add(&tsk_rt(task)->list, &rt->tobe_released); 371 list_add(&tsk_rt(task)->list, &rt->tobe_released);
372 task->rt_param.domain = rt; 372 task->rt_param.domain = rt;
373 373
@@ -398,12 +398,12 @@ static void pd_requeue(domain_t *dom, struct task_struct *task)
398 398
399 if (is_released(task, litmus_clock())) { 399 if (is_released(task, litmus_clock())) {
400 __add_ready(domain, task); 400 __add_ready(domain, task);
401 TRACE("rt: adding %s/%d (%llu, %llu) rel=%llu to ready queue at %llu\n", 401 VTRACE("rt: adding %s/%d (%llu, %llu) rel=%llu to ready queue at %llu\n",
402 task->comm, task->pid, get_exec_cost(task), get_rt_period(task), 402 task->comm, task->pid, get_exec_cost(task), get_rt_period(task),
403 get_release(task), litmus_clock()); 403 get_release(task), litmus_clock());
404 } else { 404 } else {
405 /* task has to wait for next release */ 405 /* task has to wait for next release */
406 TRACE_TASK(task, "add release(), rel=%llu\n", get_release(task)); 406 VTRACE_TASK(task, "add release(), rel=%llu\n", get_release(task));
407 add_release(domain, task); 407 add_release(domain, task);
408 } 408 }
409} 409}
diff --git a/litmus/sched_mc.c b/litmus/sched_mc.c
index 30898246ea38..7b74958d1f4f 100644
--- a/litmus/sched_mc.c
+++ b/litmus/sched_mc.c
@@ -25,28 +25,7 @@
25#include <litmus/event_group.h> 25#include <litmus/event_group.h>
26 26
27#include <litmus/sched_mc.h> 27#include <litmus/sched_mc.h>
28 28#include <litmus/ce_domain.h>
29/**
30 * crit_entry_t - State of a CPU within each criticality level system.
31 * @level Criticality level of this entry
32 * @linked Logically running task, ghost or regular
33 * @domain Domain from which to draw tasks
34 * @usable False if a higher criticality task is running
35 * @timer For ghost task budget enforcement
36 * @node Used to sort crit_entries by preemptability in global domains
37 */
38struct crit_entry {
39 enum crit_level level;
40 struct task_struct* linked;
41 struct domain* domain;
42 int usable;
43#ifdef CONFIG_MERGE_TIMERS
44 struct rt_event event;
45#else
46 struct hrtimer timer;
47#endif
48 struct bheap_node* node;
49};
50 29
51/** 30/**
52 * struct cpu_entry - State of a CPU for the entire MC system 31 * struct cpu_entry - State of a CPU for the entire MC system
@@ -71,18 +50,6 @@ struct cpu_entry {
71#endif 50#endif
72}; 51};
73 52
74/**
75 * struct domain_data - Wrap domains with related CPU state
76 * @domain A domain for a criticality level
77 * @heap The preemptable heap of crit entries (for global domains)
78 * @crit_entry The crit entry for this domain (for partitioned domains)
79 */
80struct domain_data {
81 struct domain domain;
82 struct bheap* heap;
83 struct crit_entry* crit_entry;
84};
85
86DEFINE_PER_CPU(struct cpu_entry, cpus); 53DEFINE_PER_CPU(struct cpu_entry, cpus);
87#ifdef CONFIG_RELEASE_MASTER 54#ifdef CONFIG_RELEASE_MASTER
88static int interrupt_cpu; 55static int interrupt_cpu;
@@ -101,9 +68,9 @@ static struct event_group* global_group;
101#define crit_cpu(ce) \ 68#define crit_cpu(ce) \
102 (container_of((void*)((ce) - (ce)->level), struct cpu_entry, crit_entries)) 69 (container_of((void*)((ce) - (ce)->level), struct cpu_entry, crit_entries))
103#define TRACE_ENTRY(e, fmt, args...) \ 70#define TRACE_ENTRY(e, fmt, args...) \
104 TRACE("P%d, linked=" TS " " fmt "\n", e->cpu, TA(e->linked), ##args) 71 STRACE("P%d, linked=" TS " " fmt "\n", e->cpu, TA(e->linked), ##args)
105#define TRACE_CRIT_ENTRY(ce, fmt, args...) \ 72#define TRACE_CRIT_ENTRY(ce, fmt, args...) \
106 TRACE("%s P%d, linked=" TS " " fmt "\n", \ 73 STRACE("%s P%d, linked=" TS " " fmt "\n", \
107 (ce)->domain->name, crit_cpu(ce)->cpu, TA((ce)->linked), ##args) 74 (ce)->domain->name, crit_cpu(ce)->cpu, TA((ce)->linked), ##args)
108 75
109/* 76/*
@@ -162,7 +129,7 @@ static inline struct crit_entry* lowest_prio_cpu(struct domain *dom)
162static inline void cancel_ghost(struct crit_entry *ce) 129static inline void cancel_ghost(struct crit_entry *ce)
163{ 130{
164#ifdef CONFIG_MERGE_TIMERS 131#ifdef CONFIG_MERGE_TIMERS
165 cancel_event(&ce->event); 132 cancel_event(crit_cpu(ce)->event_group, &ce->event);
166#else 133#else
167 hrtimer_try_to_cancel(&ce->timer); 134 hrtimer_try_to_cancel(&ce->timer);
168#endif 135#endif
@@ -174,9 +141,7 @@ static inline void cancel_ghost(struct crit_entry *ce)
174static inline void arm_ghost(struct crit_entry *ce, lt_t fire) 141static inline void arm_ghost(struct crit_entry *ce, lt_t fire)
175{ 142{
176#ifdef CONFIG_MERGE_TIMERS 143#ifdef CONFIG_MERGE_TIMERS
177 struct event_group* group = (is_global(ce->domain)) ? 144 add_event(crit_cpu(ce)->event_group, &ce->event, fire);
178 global_group : crit_cpu(ce)->event_group;
179 add_event(group, &ce->event, fire);
180#else 145#else
181 __hrtimer_start_range_ns(&ce->timer, 146 __hrtimer_start_range_ns(&ce->timer,
182 ns_to_ktime(when_to_fire), 147 ns_to_ktime(when_to_fire),
@@ -243,10 +208,14 @@ static void link_task_to_crit(struct crit_entry *ce,
243 ce->linked = task; 208 ce->linked = task;
244 if (task) { 209 if (task) {
245 task->rt_param.linked_on = crit_cpu(ce)->cpu; 210 task->rt_param.linked_on = crit_cpu(ce)->cpu;
246 if (is_ghost(task)) { 211 if (is_ghost(task) && CRIT_LEVEL_A != tsk_mc_crit(task)) {
212 /* There is a level-A timer that will force a
213 * preemption, so we don't set this for level-A
214 * tasks.
215 */
247 /* Reset budget timer */ 216 /* Reset budget timer */
248 task->se.exec_start = litmus_clock(); 217 task->se.exec_start = litmus_clock();
249 when_to_fire = litmus_clock() + 218 when_to_fire = task->se.exec_start +
250 tsk_mc_data(task)->mc_job.ghost_budget; 219 tsk_mc_data(task)->mc_job.ghost_budget;
251 arm_ghost(ce, when_to_fire); 220 arm_ghost(ce, when_to_fire);
252 } 221 }
@@ -261,6 +230,7 @@ static void link_task_to_crit(struct crit_entry *ce,
261} 230}
262 231
263static void check_for_preempt(struct domain*); 232static void check_for_preempt(struct domain*);
233
264/** 234/**
265 * job_arrival() - Called when a task re-enters the system. 235 * job_arrival() - Called when a task re-enters the system.
266 * Caller must hold no locks. 236 * Caller must hold no locks.
@@ -330,7 +300,7 @@ static void fix_global_levels(void)
330 struct list_head *pos, *safe; 300 struct list_head *pos, *safe;
331 struct task_struct *t; 301 struct task_struct *t;
332 302
333 TRACE("Fixing global levels\n"); 303 STRACE("Fixing global levels");
334 for_each_online_cpu(c) { 304 for_each_online_cpu(c) {
335 e = &per_cpu(cpus, c); 305 e = &per_cpu(cpus, c);
336 raw_spin_lock(&e->redir_lock); 306 raw_spin_lock(&e->redir_lock);
@@ -527,6 +497,7 @@ static void remove_from_all(struct task_struct* task)
527 */ 497 */
528static void job_completion(struct task_struct *task, int forced) 498static void job_completion(struct task_struct *task, int forced)
529{ 499{
500 lt_t now;
530 TRACE_MC_TASK(task, "Completed"); 501 TRACE_MC_TASK(task, "Completed");
531 sched_trace_task_completion(task, forced); 502 sched_trace_task_completion(task, forced);
532 BUG_ON(!task); 503 BUG_ON(!task);
@@ -535,8 +506,11 @@ static void job_completion(struct task_struct *task, int forced)
535 set_rt_flags(task, RT_F_SLEEP); 506 set_rt_flags(task, RT_F_SLEEP);
536 remove_from_all(task); 507 remove_from_all(task);
537 508
509 now = litmus_clock();
510
538 /* If it's not a ghost job, do ghost job conversion */ 511 /* If it's not a ghost job, do ghost job conversion */
539 if (!is_ghost(task)) { 512 if (!is_ghost(task)) {
513 TRACE_MC_TASK(task, "is not a ghost task");
540 tsk_mc_data(task)->mc_job.ghost_budget = budget_remaining(task); 514 tsk_mc_data(task)->mc_job.ghost_budget = budget_remaining(task);
541 tsk_mc_data(task)->mc_job.is_ghost = 1; 515 tsk_mc_data(task)->mc_job.is_ghost = 1;
542 } 516 }
@@ -546,6 +520,7 @@ static void job_completion(struct task_struct *task, int forced)
546 * conversion. Revert back to a normal task and complete the period. 520 * conversion. Revert back to a normal task and complete the period.
547 */ 521 */
548 if (tsk_mc_data(task)->mc_job.ghost_budget == 0) { 522 if (tsk_mc_data(task)->mc_job.ghost_budget == 0) {
523 TRACE_MC_TASK(task, "has zero ghost budget");
549 tsk_mc_data(task)->mc_job.is_ghost = 0; 524 tsk_mc_data(task)->mc_job.is_ghost = 0;
550 prepare_for_next_period(task); 525 prepare_for_next_period(task);
551 if (is_released(task, litmus_clock())) 526 if (is_released(task, litmus_clock()))
@@ -573,9 +548,9 @@ static enum hrtimer_restart mc_ghost_exhausted(struct hrtimer *timer)
573 unsigned long flags; 548 unsigned long flags;
574 struct task_struct *tmp = NULL; 549 struct task_struct *tmp = NULL;
575 550
576
577 local_irq_save(flags); 551 local_irq_save(flags);
578 TRACE_CRIT_ENTRY(ce, "Ghost exhausted firing"); 552 TRACE("Ghost exhausted\n");
553 TRACE_CRIT_ENTRY(ce, "Firing here");
579 554
580 /* Due to race conditions, we cannot just set the linked 555 /* Due to race conditions, we cannot just set the linked
581 * task's budget to 0 as it may no longer be the task 556 * task's budget to 0 as it may no longer be the task
@@ -601,6 +576,52 @@ static enum hrtimer_restart mc_ghost_exhausted(struct hrtimer *timer)
601#endif 576#endif
602} 577}
603 578
579static enum hrtimer_restart ce_timer_function(struct hrtimer *timer)
580{
581 struct ce_dom_data *ce_data =
582 container_of(timer, struct ce_dom_data, timer);
583 struct crit_entry *ce = &per_cpu(cpus, ce_data->cpu).crit_entries[CRIT_LEVEL_A];
584 struct domain *dom = ce->domain;
585 struct task_struct *old_link = NULL;
586 unsigned long flags;
587
588 TRACE("MC level-A timer callback for CPU %d\n", ce_data->cpu);
589
590 local_irq_save(flags);
591
592 raw_spin_lock(dom->lock);
593
594 raw_spin_lock(&crit_cpu(ce)->lock);
595 if (ce->linked &&
596 ce->linked == ce_data->should_schedule &&
597 is_ghost(ce->linked))
598 {
599 old_link = ce->linked;
600 tsk_mc_data(ce->linked)->mc_job.ghost_budget = 0;
601 link_task_to_crit(ce, NULL);
602 }
603 raw_spin_unlock(&crit_cpu(ce)->lock);
604
605 mc_ce_timer_callback_common(dom, timer);
606
607 /* job completion will check for preemptions by means of calling job
608 * arrival if the task is not blocked */
609 if (NULL != old_link) {
610 STRACE("old_link " TS " so will call job completion\n", TA(old_link));
611 raw_spin_unlock(dom->lock);
612 job_completion(old_link, 0);
613 } else {
614 STRACE("old_link was null, so will call check for preempt\n");
615 raw_spin_unlock(dom->lock);
616 check_for_preempt(dom);
617 }
618
619 local_irq_restore(flags);
620
621 return HRTIMER_RESTART;
622}
623
624
604/** 625/**
605 * mc_release_jobs() - Add heap of tasks to the system, check for preemptions. 626 * mc_release_jobs() - Add heap of tasks to the system, check for preemptions.
606 */ 627 */
@@ -611,7 +632,7 @@ static void mc_release_jobs(rt_domain_t* rt, struct bheap* tasks)
611 struct domain *dom = get_task_domain(first); 632 struct domain *dom = get_task_domain(first);
612 633
613 raw_spin_lock_irqsave(dom->lock, flags); 634 raw_spin_lock_irqsave(dom->lock, flags);
614 TRACE_MC_TASK(first, "Jobs released"); 635 TRACE(TS "Jobs released\n", TA(first));
615 __merge_ready(rt, tasks); 636 __merge_ready(rt, tasks);
616 check_for_preempt(dom); 637 check_for_preempt(dom);
617 raw_spin_unlock_irqrestore(dom->lock, flags); 638 raw_spin_unlock_irqrestore(dom->lock, flags);
@@ -664,7 +685,7 @@ static void mc_task_wake_up(struct task_struct *task)
664 lt_t now = litmus_clock(); 685 lt_t now = litmus_clock();
665 local_irq_save(flags); 686 local_irq_save(flags);
666 687
667 TRACE_MC_TASK(task, "Wakes up"); 688 TRACE(TS " wakes up\n", TA(task));
668 if (is_tardy(task, now)) { 689 if (is_tardy(task, now)) {
669 /* Task missed its last release */ 690 /* Task missed its last release */
670 release_at(task, now); 691 release_at(task, now);
@@ -683,7 +704,7 @@ static void mc_task_block(struct task_struct *task)
683{ 704{
684 unsigned long flags; 705 unsigned long flags;
685 local_irq_save(flags); 706 local_irq_save(flags);
686 TRACE_MC_TASK(task, "Block at %llu", litmus_clock()); 707 TRACE(TS " blocks\n", TA(task));
687 remove_from_all(task); 708 remove_from_all(task);
688 local_irq_restore(flags); 709 local_irq_restore(flags);
689} 710}
@@ -696,7 +717,7 @@ static void mc_task_exit(struct task_struct *task)
696 unsigned long flags; 717 unsigned long flags;
697 local_irq_save(flags); 718 local_irq_save(flags);
698 BUG_ON(!is_realtime(task)); 719 BUG_ON(!is_realtime(task));
699 TRACE_MC_TASK(task, "RIP"); 720 TRACE(TS " RIP\n", TA(task));
700 721
701 remove_from_all(task); 722 remove_from_all(task);
702 if (tsk_rt(task)->scheduled_on != NO_CPU) { 723 if (tsk_rt(task)->scheduled_on != NO_CPU) {
@@ -704,6 +725,9 @@ static void mc_task_exit(struct task_struct *task)
704 tsk_rt(task)->scheduled_on = NO_CPU; 725 tsk_rt(task)->scheduled_on = NO_CPU;
705 } 726 }
706 727
728 if (CRIT_LEVEL_A == tsk_mc_crit(task))
729 mc_ce_task_exit_common(task);
730
707 local_irq_restore(flags); 731 local_irq_restore(flags);
708} 732}
709 733
@@ -713,19 +737,30 @@ static void mc_task_exit(struct task_struct *task)
713 */ 737 */
714static long mc_admit_task(struct task_struct* task) 738static long mc_admit_task(struct task_struct* task)
715{ 739{
740 const enum crit_level crit = tsk_mc_crit(task);
741 long ret;
716 if (!tsk_mc_data(task)) { 742 if (!tsk_mc_data(task)) {
717 printk(KERN_WARNING "Tried to admit task with no criticality " 743 printk(KERN_WARNING "Tried to admit task with no criticality "
718 "level\n"); 744 "level\n");
719 return -EINVAL; 745 ret = -EINVAL;
746 goto out;
720 } 747 }
721 if (tsk_mc_crit(task) < CRIT_LEVEL_C && get_partition(task) == NO_CPU) { 748 if (crit < CRIT_LEVEL_C && get_partition(task) == NO_CPU) {
722 printk(KERN_WARNING "Tried to admit partitioned task with no " 749 printk(KERN_WARNING "Tried to admit partitioned task with no "
723 "partition\n"); 750 "partition\n");
724 return -EINVAL; 751 ret = -EINVAL;
752 goto out;
753 }
754 if (crit == CRIT_LEVEL_A) {
755 ret = mc_ce_admit_task_common(task);
756 if (ret)
757 goto out;
725 } 758 }
726 printk(KERN_INFO "Admitted task with criticality level %d\n", 759 printk(KERN_INFO "Admitted task with criticality level %d\n",
727 tsk_mc_crit(task)); 760 tsk_mc_crit(task));
728 return 0; 761 ret = 0;
762out:
763 return ret;
729} 764}
730 765
731/** 766/**
@@ -761,11 +796,11 @@ static struct task_struct* mc_schedule(struct task_struct * prev)
761 796
762 if (exists) { 797 if (exists) {
763 entry->scheduled->rt_param.scheduled_on = NO_CPU; 798 entry->scheduled->rt_param.scheduled_on = NO_CPU;
764 TRACE_MC_TASK(prev, 799 TRACE(TS
765 "blocks:%d out_of_time:%d sleep:%d preempt:%d " 800 " blocks:%d out_of_time:%d sleep:%d preempt:%d "
766 "state:%d sig:%d global:%d", 801 "state:%d sig:%d global:%d\n", TA(prev),
767 blocks, out_of_time, sleep, preempt, 802 blocks, out_of_time, sleep, preempt,
768 prev->state, signal_pending(prev), global); 803 prev->state, signal_pending(prev), global);
769 } 804 }
770 raw_spin_unlock(&entry->lock); 805 raw_spin_unlock(&entry->lock);
771 806
@@ -781,7 +816,7 @@ static struct task_struct* mc_schedule(struct task_struct * prev)
781 /* Any task which exhausts its budget or sleeps waiting for its next 816 /* Any task which exhausts its budget or sleeps waiting for its next
782 * period completes unless its execution has been forcibly stopped. 817 * period completes unless its execution has been forcibly stopped.
783 */ 818 */
784 if ((out_of_time || sleep) && !blocks && !preempt) 819 if ((out_of_time || sleep) && !blocks)/* && !preempt)*/
785 job_completion(entry->scheduled, !sleep); 820 job_completion(entry->scheduled, !sleep);
786 /* Global scheduled tasks must wait for a deschedule before they 821 /* Global scheduled tasks must wait for a deschedule before they
787 * can rejoin the global state. Rejoin them here. 822 * can rejoin the global state. Rejoin them here.
@@ -836,10 +871,29 @@ static struct task_struct* mc_schedule(struct task_struct * prev)
836 if (next) 871 if (next)
837 TRACE_MC_TASK(next, "Scheduled at %llu", litmus_clock()); 872 TRACE_MC_TASK(next, "Scheduled at %llu", litmus_clock());
838 else if (exists && !next) 873 else if (exists && !next)
839 TRACE("Becomes idle at %llu\n", litmus_clock()); 874 TRACE_ENTRY(entry, "Becomes idle at %llu", litmus_clock());
840 return next; 875 return next;
841} 876}
842 877
878/*
879 * This is the plugin's release at function, called by the release task-set
880 * system call. Other places in the file use the generic LITMUS release_at(),
881 * which is not this.
882 */
883void mc_release_at(struct task_struct *ts, lt_t start)
884{
885 /* hack so that we can have CE timers start at the right time */
886 if (CRIT_LEVEL_A == tsk_mc_crit(ts))
887 mc_ce_release_at_common(ts, start);
888 else
889 release_at(ts, start);
890}
891
892long mc_deactivate_plugin(void)
893{
894 return mc_ce_deactivate_plugin_common();
895}
896
843/* ************************************************************************** 897/* **************************************************************************
844 * Initialization 898 * Initialization
845 * ************************************************************************** */ 899 * ************************************************************************** */
@@ -850,7 +904,8 @@ static struct task_struct* mc_schedule(struct task_struct * prev)
850 904
851/* LVL-A */ 905/* LVL-A */
852DEFINE_PER_CPU(struct domain_data, _mc_crit_a); 906DEFINE_PER_CPU(struct domain_data, _mc_crit_a);
853DEFINE_PER_CPU(rt_domain_t, _mc_crit_a_rt); 907DEFINE_PER_CPU(raw_spinlock_t, _mc_crit_a_lock);
908DEFINE_PER_CPU(struct ce_dom_data, _mc_crit_a_ce_data);
854/* LVL-B */ 909/* LVL-B */
855DEFINE_PER_CPU(struct domain_data, _mc_crit_b); 910DEFINE_PER_CPU(struct domain_data, _mc_crit_b);
856DEFINE_PER_CPU(rt_domain_t, _mc_crit_b_rt); 911DEFINE_PER_CPU(rt_domain_t, _mc_crit_b_rt);
@@ -870,12 +925,31 @@ DEFINE_PER_CPU(struct event_group, _mc_groups);
870 925
871static long mc_activate_plugin(void) 926static long mc_activate_plugin(void)
872{ 927{
928 struct domain_data *dom_data;
929 struct domain *dom;
930 struct domain_data *our_domains[NR_CPUS];
931 int cpu, n = 0;
932 long ret;
933
873#ifdef CONFIG_RELEASE_MASTER 934#ifdef CONFIG_RELEASE_MASTER
874 interrupt_cpu = atomic_read(&release_master_cpu); 935 interrupt_cpu = atomic_read(&release_master_cpu);
875 if (interrupt_cpu == NO_CPU) 936 if (interrupt_cpu == NO_CPU)
876 interrupt_cpu = 0; 937 interrupt_cpu = 0;
877#endif 938#endif
878 return 0; 939
940 for_each_online_cpu(cpu) {
941 BUG_ON(NR_CPUS <= n);
942 dom = per_cpu(cpus, cpu).crit_entries[CRIT_LEVEL_A].domain;
943 dom_data = domain_data(dom);
944 our_domains[cpu] = dom_data;
945 n++;
946 }
947 ret = mc_ce_set_domains(n, our_domains);
948 if (ret)
949 goto out;
950 ret = mc_ce_activate_plugin_common();
951out:
952 return ret;
879} 953}
880 954
881static struct sched_plugin mc_plugin __cacheline_aligned_in_smp = { 955static struct sched_plugin mc_plugin __cacheline_aligned_in_smp = {
@@ -888,6 +962,8 @@ static struct sched_plugin mc_plugin __cacheline_aligned_in_smp = {
888 .task_block = mc_task_block, 962 .task_block = mc_task_block,
889 .admit_task = mc_admit_task, 963 .admit_task = mc_admit_task,
890 .activate_plugin = mc_activate_plugin, 964 .activate_plugin = mc_activate_plugin,
965 .release_at = mc_release_at,
966 .deactivate_plugin = mc_deactivate_plugin,
891}; 967};
892 968
893static void init_crit_entry(struct crit_entry *ce, enum crit_level level, 969static void init_crit_entry(struct crit_entry *ce, enum crit_level level,
@@ -958,13 +1034,15 @@ static inline void init_edf_domain(struct domain *dom, rt_domain_t *rt,
958#endif 1034#endif
959} 1035}
960 1036
1037struct domain_data *ce_domain_for(int);
961static int __init init_mc(void) 1038static int __init init_mc(void)
962{ 1039{
963 int cpu; 1040 int cpu;
964 struct cpu_entry *entry; 1041 struct cpu_entry *entry;
965 struct domain_data *dom_data; 1042 struct domain_data *dom_data;
966 rt_domain_t *rt; 1043 rt_domain_t *rt;
967 raw_spinlock_t *a_dom, *b_dom, *c_dom; /* For lock debugger */ 1044 raw_spinlock_t *a_dom_lock, *b_dom_lock, *c_dom_lock; /* For lock debugger */
1045 struct ce_dom_data *ce_data;
968 1046
969 for_each_online_cpu(cpu) { 1047 for_each_online_cpu(cpu) {
970 entry = &per_cpu(cpus, cpu); 1048 entry = &per_cpu(cpus, cpu);
@@ -992,11 +1070,15 @@ static int __init init_mc(void)
992 1070
993 /* CRIT_LEVEL_A */ 1071 /* CRIT_LEVEL_A */
994 dom_data = &per_cpu(_mc_crit_a, cpu); 1072 dom_data = &per_cpu(_mc_crit_a, cpu);
995 rt = &per_cpu(_mc_crit_a_rt, cpu); 1073 ce_data = &per_cpu(_mc_crit_a_ce_data, cpu);
1074 a_dom_lock = &per_cpu(_mc_crit_a_lock, cpu);
1075 raw_spin_lock_init(a_dom_lock);
1076 ce_domain_init(&dom_data->domain,
1077 a_dom_lock, ce_requeue, ce_peek_and_take_ready,
1078 ce_peek_and_take_ready, mc_preempt_needed,
1079 ce_higher_prio, ce_data, cpu,
1080 ce_timer_function);
996 init_local_domain(entry, dom_data, CRIT_LEVEL_A); 1081 init_local_domain(entry, dom_data, CRIT_LEVEL_A);
997 init_edf_domain(&dom_data->domain, rt, cpu, CRIT_LEVEL_A);
998 a_dom = dom_data->domain.lock;
999 raw_spin_lock_init(a_dom);
1000 dom_data->domain.name = "LVL-A"; 1082 dom_data->domain.name = "LVL-A";
1001 1083
1002 /* CRIT_LEVEL_B */ 1084 /* CRIT_LEVEL_B */
@@ -1004,8 +1086,8 @@ static int __init init_mc(void)
1004 rt = &per_cpu(_mc_crit_b_rt, cpu); 1086 rt = &per_cpu(_mc_crit_b_rt, cpu);
1005 init_local_domain(entry, dom_data, CRIT_LEVEL_B); 1087 init_local_domain(entry, dom_data, CRIT_LEVEL_B);
1006 init_edf_domain(&dom_data->domain, rt, cpu, CRIT_LEVEL_B); 1088 init_edf_domain(&dom_data->domain, rt, cpu, CRIT_LEVEL_B);
1007 b_dom = dom_data->domain.lock; 1089 b_dom_lock = dom_data->domain.lock;
1008 raw_spin_lock_init(b_dom); 1090 raw_spin_lock_init(b_dom_lock);
1009 dom_data->domain.name = "LVL-B"; 1091 dom_data->domain.name = "LVL-B";
1010 } 1092 }
1011 1093
@@ -1022,8 +1104,8 @@ static int __init init_mc(void)
1022 init_global_domain(&_mc_crit_c, CRIT_LEVEL_C, 1104 init_global_domain(&_mc_crit_c, CRIT_LEVEL_C,
1023 &_mc_heap_c, _mc_nodes_c); 1105 &_mc_heap_c, _mc_nodes_c);
1024 init_edf_domain(&_mc_crit_c.domain, &_mc_crit_c_rt, 0, CRIT_LEVEL_C); 1106 init_edf_domain(&_mc_crit_c.domain, &_mc_crit_c_rt, 0, CRIT_LEVEL_C);
1025 c_dom = _mc_crit_c.domain.lock; 1107 c_dom_lock = _mc_crit_c.domain.lock;
1026 raw_spin_lock_init(c_dom); 1108 raw_spin_lock_init(c_dom_lock);
1027 _mc_crit_c.domain.name = "LVL-C"; 1109 _mc_crit_c.domain.name = "LVL-C";
1028 1110
1029 return register_sched_plugin(&mc_plugin); 1111 return register_sched_plugin(&mc_plugin);
diff --git a/litmus/sched_mc_ce.c b/litmus/sched_mc_ce.c
index dcb74f4ca67b..63b0470e1f52 100644
--- a/litmus/sched_mc_ce.c
+++ b/litmus/sched_mc_ce.c
@@ -23,27 +23,86 @@
23#include <litmus/sched_trace.h> 23#include <litmus/sched_trace.h>
24#include <litmus/jobs.h> 24#include <litmus/jobs.h>
25#include <litmus/sched_mc.h> 25#include <litmus/sched_mc.h>
26#include <litmus/ce_domain.h>
26 27
27static struct sched_plugin mc_ce_plugin __cacheline_aligned_in_smp; 28static struct sched_plugin mc_ce_plugin __cacheline_aligned_in_smp;
28 29
29#define is_active_plugin() (litmus == &mc_ce_plugin) 30#define using_linux_plugin() (litmus == &linux_sched_plugin)
31
32/* get a reference to struct domain for a CPU */
33#define get_domain_for(cpu) (&per_cpu(domains, cpu)->domain)
34
35#define get_pid_table(cpu) (&per_cpu(ce_pid_table, cpu))
36#define get_pid_entry(cpu, idx) (&(get_pid_table(cpu)->entries[idx]))
30 37
31static atomic_t start_time_set = ATOMIC_INIT(-1); 38static atomic_t start_time_set = ATOMIC_INIT(-1);
32static atomic64_t start_time = ATOMIC64_INIT(0); 39static atomic64_t start_time = ATOMIC64_INIT(0);
33static struct proc_dir_entry *mc_ce_dir = NULL, *ce_file = NULL; 40static struct proc_dir_entry *mc_ce_dir = NULL, *ce_file = NULL;
34 41
42/*
43 * Cache the budget along with the struct PID for a task so that we don't need
44 * to fetch its task_struct every time we check to see what should be
45 * scheduled.
46 */
47struct ce_pid_entry {
48 struct pid *pid;
49 lt_t budget;
50 /* accumulated (summed) budgets, including this one */
51 lt_t acc_time;
52 unsigned int expected_job;
53};
54
55struct ce_pid_table {
56 struct ce_pid_entry entries[CONFIG_PLUGIN_MC_LEVEL_A_MAX_TASKS];
57 int num_pid_entries;
58 lt_t cycle_time;
59};
60
61DEFINE_PER_CPU(struct ce_pid_table, ce_pid_table);
62
63/*
64 * How we get the domain for a given CPU locally. Set with the
65 * mc_ce_set_domains function. Must be done before activating plugins. Be
66 * careful when using domains as a variable elsewhere in this file.
67 */
68
69DEFINE_PER_CPU(struct domain_data*, domains);
35 70
36DEFINE_PER_CPU(domain_t, mc_ce_doms); 71/*
37DEFINE_PER_CPU(rt_domain_t, mc_ce_rts); 72 * The domains and other data used by the MC-CE plugin when it runs alone.
73 */
74DEFINE_PER_CPU(struct domain_data, _mc_ce_doms);
38DEFINE_PER_CPU(struct ce_dom_data, _mc_ce_dom_data); 75DEFINE_PER_CPU(struct ce_dom_data, _mc_ce_dom_data);
76DEFINE_PER_CPU(raw_spinlock_t, _mc_ce_dom_locks);
39 77
40/* Return the address of the domain_t for this CPU, used by the 78long mc_ce_set_domains(const int n, struct domain_data *domains_in[])
41 * mixed-criticality plugin. */
42domain_t *ce_domain_for(int cpu)
43{ 79{
44 return &per_cpu(mc_ce_doms, cpu); 80 const int max = (NR_CPUS < n) ? NR_CPUS : n;
81 struct domain_data *new_dom = NULL;
82 int i, ret;
83 if (!using_linux_plugin()) {
84 printk(KERN_WARNING "can't set MC-CE domains when not using "
85 "Linux scheduler.\n");
86 ret = -EINVAL;
87 goto out;
88 }
89 for (i = 0; i < max; ++i) {
90 new_dom = domains_in[i];
91 per_cpu(domains, i) = new_dom;
92 }
93 ret = 0;
94out:
95 return ret;
45} 96}
46 97
98unsigned int mc_ce_get_expected_job(const int cpu, const int idx)
99{
100 const struct ce_pid_table *pid_table = get_pid_table(cpu);
101 BUG_ON(0 > cpu);
102 BUG_ON(0 > idx);
103 BUG_ON(pid_table->num_pid_entries <= idx);
104 return pid_table->entries[idx].expected_job;
105}
47 106
48/* 107/*
49 * Get the offset into the cycle taking the start time into account. 108 * Get the offset into the cycle taking the start time into account.
@@ -63,16 +122,14 @@ static inline lt_t get_cycle_offset(const lt_t when, const lt_t cycle_time)
63 * 122 *
64 * Do not call prepare_for_next_period on Level-A tasks! 123 * Do not call prepare_for_next_period on Level-A tasks!
65 */ 124 */
66static void mc_ce_job_completion(struct task_struct *ts) 125static void mc_ce_job_completion(struct domain *dom, struct task_struct *ts)
67{ 126{
68 const domain_t *dom = &per_cpu(mc_ce_doms, smp_processor_id()); 127 const int cpu = task_cpu(ts);
69 const struct ce_dom_data *ce_data = dom->data;
70 const int idx = tsk_mc_data(ts)->mc_task.lvl_a_id; 128 const int idx = tsk_mc_data(ts)->mc_task.lvl_a_id;
71 const struct ce_dom_pid_entry *pid_entry = 129 const struct ce_pid_entry *pid_entry = get_pid_entry(cpu, idx);
72 &ce_data->pid_entries[idx]; 130 unsigned int just_finished;
73 int just_finished;
74 131
75 TRACE_TASK(ts, "completed\n"); 132 TRACE_TASK(ts, "Completed\n");
76 133
77 sched_trace_task_completion(ts, 0); 134 sched_trace_task_completion(ts, 0);
78 /* post-increment is important here */ 135 /* post-increment is important here */
@@ -85,11 +142,11 @@ static void mc_ce_job_completion(struct task_struct *ts)
85 if (just_finished < pid_entry->expected_job) { 142 if (just_finished < pid_entry->expected_job) {
86 /* this job is already released because it's running behind */ 143 /* this job is already released because it's running behind */
87 set_rt_flags(ts, RT_F_RUNNING); 144 set_rt_flags(ts, RT_F_RUNNING);
88 TRACE_TASK(ts, "appears behind: the expected job is %d but " 145 TRACE_TASK(ts, "appears behind: the expected job is %u but "
89 "job %d just completed\n", 146 "job %u just completed\n",
90 pid_entry->expected_job, just_finished); 147 pid_entry->expected_job, just_finished);
91 } else if (pid_entry->expected_job < just_finished) { 148 } else if (pid_entry->expected_job < just_finished) {
92 printk(KERN_CRIT "job %d completed in expected job %d which " 149 printk(KERN_CRIT "job %u completed in expected job %u which "
93 "seems too early\n", just_finished, 150 "seems too early\n", just_finished,
94 pid_entry->expected_job); 151 pid_entry->expected_job);
95 BUG(); 152 BUG();
@@ -104,31 +161,32 @@ static void mc_ce_job_completion(struct task_struct *ts)
104 * 161 *
105 * TODO Currently O(n) in the number of tasks on the CPU. Binary search? 162 * TODO Currently O(n) in the number of tasks on the CPU. Binary search?
106 */ 163 */
107static int mc_ce_schedule_at(const domain_t *dom, lt_t offset) 164static int mc_ce_schedule_at(const struct domain *dom, lt_t offset)
108{ 165{
109 const struct ce_dom_data *ce_data = dom->data; 166 const struct ce_dom_data *ce_data = dom->data;
110 const struct ce_dom_pid_entry *pid_entry = NULL; 167 struct ce_pid_table *pid_table = get_pid_table(ce_data->cpu);
111 int i; 168 const struct ce_pid_entry *pid_entry = NULL;
169 int idx;
112 170
113 BUG_ON(ce_data->cycle_time < 1); 171 BUG_ON(pid_table->cycle_time < 1);
114 BUG_ON(ce_data->num_pid_entries < 1); 172 BUG_ON(pid_table->num_pid_entries < 1);
115 173
116 for (i = 0; i < ce_data->num_pid_entries; ++i) { 174 for (idx = 0; idx < pid_table->num_pid_entries; ++idx) {
117 pid_entry = &ce_data->pid_entries[i]; 175 pid_entry = &pid_table->entries[idx];
118 if (offset < pid_entry->acc_time) { 176 if (offset < pid_entry->acc_time) {
119 /* found task to schedule in this window */ 177 /* found task to schedule in this window */
120 break; 178 break;
121 } 179 }
122 } 180 }
123 /* can only happen if cycle_time is not right */ 181 /* can only happen if cycle_time is not right */
124 BUG_ON(pid_entry->acc_time > ce_data->cycle_time); 182 BUG_ON(pid_entry->acc_time > pid_table->cycle_time);
125 TRACE("schedule at returned task %d for CPU %d\n", i, ce_data->cpu); 183 TRACE("schedule at returning task %d for CPU %d\n", idx, ce_data->cpu);
126 return i; 184 return idx;
127} 185}
128 186
129static struct task_struct *mc_ce_schedule(struct task_struct *prev) 187static struct task_struct *mc_ce_schedule(struct task_struct *prev)
130{ 188{
131 domain_t *dom = &per_cpu(mc_ce_doms, smp_processor_id()); 189 struct domain *dom = get_domain_for(smp_processor_id());
132 struct ce_dom_data *ce_data = dom->data; 190 struct ce_dom_data *ce_data = dom->data;
133 struct task_struct *next = NULL; 191 struct task_struct *next = NULL;
134 int exists, sleep, should_sched_exists, should_sched_blocked, 192 int exists, sleep, should_sched_exists, should_sched_blocked,
@@ -147,7 +205,7 @@ static struct task_struct *mc_ce_schedule(struct task_struct *prev)
147 TRACE("exists: %d, sleep: %d\n", exists, sleep); 205 TRACE("exists: %d, sleep: %d\n", exists, sleep);
148 206
149 if (sleep) 207 if (sleep)
150 mc_ce_job_completion(ce_data->scheduled); 208 mc_ce_job_completion(dom, ce_data->scheduled);
151 209
152 /* these checks must go after the call to mc_ce_job_completion in case 210 /* these checks must go after the call to mc_ce_job_completion in case
153 * a late task needs to be scheduled again right away and its the only 211 * a late task needs to be scheduled again right away and its the only
@@ -178,7 +236,7 @@ static struct task_struct *mc_ce_schedule(struct task_struct *prev)
178 236
179static void mc_ce_finish_switch(struct task_struct *prev) 237static void mc_ce_finish_switch(struct task_struct *prev)
180{ 238{
181 domain_t *dom = &per_cpu(mc_ce_doms, smp_processor_id()); 239 struct domain *dom = get_domain_for(smp_processor_id());
182 struct ce_dom_data *ce_data = dom->data; 240 struct ce_dom_data *ce_data = dom->data;
183 241
184 TRACE("finish switch\n"); 242 TRACE("finish switch\n");
@@ -190,41 +248,21 @@ static void mc_ce_finish_switch(struct task_struct *prev)
190} 248}
191 249
192/* 250/*
193 * Called for every local timer interrupt.
194 * Linux calls this with interrupts disabled, AFAIK.
195 */
196static void mc_ce_tick(struct task_struct *ts)
197{
198 domain_t *dom = &per_cpu(mc_ce_doms, smp_processor_id());
199 struct ce_dom_data *ce_data = dom->data;
200 struct task_struct *should_schedule;
201
202 if (is_realtime(ts) && CRIT_LEVEL_A == tsk_mc_crit(ts)) {
203 raw_spin_lock(dom->lock);
204 should_schedule = ce_data->should_schedule;
205 raw_spin_unlock(dom->lock);
206
207 if (!is_np(ts) && ts != should_schedule) {
208 litmus_reschedule_local();
209 } else if (is_user_np(ts)) {
210 request_exit_np(ts);
211 }
212 }
213}
214
215/*
216 * Admit task called to see if this task is permitted to enter the system. 251 * Admit task called to see if this task is permitted to enter the system.
217 * Here we look up the task's PID structure and save it in the proper slot on 252 * Here we look up the task's PID structure and save it in the proper slot on
218 * the CPU this task will run on. 253 * the CPU this task will run on.
219 */ 254 */
220static long __mc_ce_admit_task(struct task_struct *ts) 255long mc_ce_admit_task_common(struct task_struct *ts)
221{ 256{
222 domain_t *dom = &per_cpu(mc_ce_doms, get_partition(ts)); 257 struct domain *dom = get_domain_for(get_partition(ts));
223 struct ce_dom_data *ce_data = dom->data; 258 struct ce_dom_data *ce_data = dom->data;
224 struct mc_data *mcd = tsk_mc_data(ts); 259 struct mc_data *mcd = tsk_mc_data(ts);
225 struct pid *pid = NULL; 260 struct pid *pid = NULL;
226 long retval = -EINVAL; 261 long retval = -EINVAL;
227 const int lvl_a_id = mcd->mc_task.lvl_a_id; 262 const int lvl_a_id = mcd->mc_task.lvl_a_id;
263 struct ce_pid_table *pid_table = get_pid_table(ce_data->cpu);
264
265 BUG_ON(get_partition(ts) != ce_data->cpu);
228 266
229 /* check the task has migrated to the right CPU (like in sched_cedf) */ 267 /* check the task has migrated to the right CPU (like in sched_cedf) */
230 if (task_cpu(ts) != get_partition(ts)) { 268 if (task_cpu(ts) != get_partition(ts)) {
@@ -248,26 +286,26 @@ static long __mc_ce_admit_task(struct task_struct *ts)
248 goto out; 286 goto out;
249 } 287 }
250 288
251 if (lvl_a_id >= ce_data->num_pid_entries) { 289 if (lvl_a_id >= pid_table->num_pid_entries) {
252 printk(KERN_INFO "litmus: level A id greater than expected " 290 printk(KERN_INFO "litmus: level A id greater than expected "
253 "number of tasks %d for %d cpu %d\n", 291 "number of tasks %d for %d cpu %d\n",
254 ce_data->num_pid_entries, ts->pid, 292 pid_table->num_pid_entries, ts->pid,
255 get_partition(ts)); 293 get_partition(ts));
256 goto out_put_pid; 294 goto out_put_pid;
257 } 295 }
258 if (ce_data->pid_entries[lvl_a_id].pid) { 296 if (pid_table->entries[lvl_a_id].pid) {
259 printk(KERN_INFO "litmus: have saved pid info id: %d cpu: %d\n", 297 printk(KERN_INFO "litmus: have saved pid info id: %d cpu: %d\n",
260 lvl_a_id, get_partition(ts)); 298 lvl_a_id, get_partition(ts));
261 goto out_put_pid; 299 goto out_put_pid;
262 } 300 }
263 if (get_exec_cost(ts) >= ce_data->pid_entries[lvl_a_id].budget) { 301 if (get_exec_cost(ts) >= pid_table->entries[lvl_a_id].budget) {
264 printk(KERN_INFO "litmus: execution cost %llu is larger than " 302 printk(KERN_INFO "litmus: execution cost %llu is larger than "
265 "the budget %llu\n", 303 "the budget %llu\n",
266 get_exec_cost(ts), 304 get_exec_cost(ts),
267 ce_data->pid_entries[lvl_a_id].budget); 305 pid_table->entries[lvl_a_id].budget);
268 goto out_put_pid; 306 goto out_put_pid;
269 } 307 }
270 ce_data->pid_entries[lvl_a_id].pid = pid; 308 pid_table->entries[lvl_a_id].pid = pid;
271 retval = 0; 309 retval = 0;
272 /* don't call put_pid if we are successful */ 310 /* don't call put_pid if we are successful */
273 goto out; 311 goto out;
@@ -280,10 +318,10 @@ out:
280 318
281static long mc_ce_admit_task(struct task_struct *ts) 319static long mc_ce_admit_task(struct task_struct *ts)
282{ 320{
283 domain_t *dom = &per_cpu(mc_ce_doms, get_partition(ts)); 321 struct domain *dom = get_domain_for(get_partition(ts));
284 unsigned long flags, retval; 322 unsigned long flags, retval;
285 raw_spin_lock_irqsave(dom->lock, flags); 323 raw_spin_lock_irqsave(dom->lock, flags);
286 retval = __mc_ce_admit_task(ts); 324 retval = mc_ce_admit_task_common(ts);
287 raw_spin_unlock_irqrestore(dom->lock, flags); 325 raw_spin_unlock_irqrestore(dom->lock, flags);
288 return retval; 326 return retval;
289} 327}
@@ -295,26 +333,26 @@ static long mc_ce_admit_task(struct task_struct *ts)
295 */ 333 */
296static void mc_ce_task_new(struct task_struct *ts, int on_rq, int running) 334static void mc_ce_task_new(struct task_struct *ts, int on_rq, int running)
297{ 335{
298 domain_t *dom = &per_cpu(mc_ce_doms, task_cpu(ts)); 336 const int cpu = task_cpu(ts);
337 struct domain *dom = get_domain_for(cpu);
299 struct ce_dom_data *ce_data = dom->data; 338 struct ce_dom_data *ce_data = dom->data;
339 struct ce_pid_table *pid_table = get_pid_table(cpu);
300 struct pid *pid_should_be_running; 340 struct pid *pid_should_be_running;
301 struct ce_dom_pid_entry *pid_entry; 341 struct ce_pid_entry *pid_entry;
302 unsigned long flags; 342 unsigned long flags;
303 int idx, should_be_running; 343 int idx, should_be_running;
304 lt_t offset; 344 lt_t offset;
305 345
306 /* have to call mc_ce_schedule_at because the task only gets a PID
307 * entry after calling admit_task */
308
309 raw_spin_lock_irqsave(dom->lock, flags); 346 raw_spin_lock_irqsave(dom->lock, flags);
310 pid_entry = &ce_data->pid_entries[tsk_mc_data(ts)->mc_task.lvl_a_id]; 347 pid_entry = get_pid_entry(cpu, tsk_mc_data(ts)->mc_task.lvl_a_id);
311 /* initialize some task state */ 348 /* initialize some task state */
312 set_rt_flags(ts, RT_F_RUNNING); 349 set_rt_flags(ts, RT_F_RUNNING);
313 tsk_rt(ts)->job_params.job_no = 0;
314 350
315 offset = get_cycle_offset(litmus_clock(), ce_data->cycle_time); 351 /* have to call mc_ce_schedule_at because the task only gets a PID
352 * entry after calling admit_task */
353 offset = get_cycle_offset(litmus_clock(), pid_table->cycle_time);
316 idx = mc_ce_schedule_at(dom, offset); 354 idx = mc_ce_schedule_at(dom, offset);
317 pid_should_be_running = ce_data->pid_entries[idx].pid; 355 pid_should_be_running = get_pid_entry(cpu, idx)->pid;
318 rcu_read_lock(); 356 rcu_read_lock();
319 should_be_running = (ts == pid_task(pid_should_be_running, PIDTYPE_PID)); 357 should_be_running = (ts == pid_task(pid_should_be_running, PIDTYPE_PID));
320 rcu_read_unlock(); 358 rcu_read_unlock();
@@ -341,7 +379,7 @@ static void mc_ce_task_new(struct task_struct *ts, int on_rq, int running)
341 */ 379 */
342static void mc_ce_task_wake_up(struct task_struct *ts) 380static void mc_ce_task_wake_up(struct task_struct *ts)
343{ 381{
344 domain_t *dom = &per_cpu(mc_ce_doms, smp_processor_id()); 382 struct domain *dom = get_domain_for(get_partition(ts));
345 struct ce_dom_data *ce_data = dom->data; 383 struct ce_dom_data *ce_data = dom->data;
346 unsigned long flags; 384 unsigned long flags;
347 385
@@ -366,25 +404,25 @@ static void mc_ce_task_block(struct task_struct *ts)
366/* 404/*
367 * Called when a task switches from RT mode back to normal mode. 405 * Called when a task switches from RT mode back to normal mode.
368 */ 406 */
369static void mc_ce_task_exit(struct task_struct *ts) 407void mc_ce_task_exit_common(struct task_struct *ts)
370{ 408{
371 domain_t *dom = &per_cpu(mc_ce_doms, get_partition(ts)); 409 struct domain *dom = get_domain_for(get_partition(ts));
372 struct ce_dom_data *ce_data = dom->data; 410 struct ce_dom_data *ce_data = dom->data;
373 unsigned long flags; 411 unsigned long flags;
374 struct pid *pid; 412 struct pid *pid;
375 const int lvl_a_id = tsk_mc_data(ts)->mc_task.lvl_a_id; 413 const int lvl_a_id = tsk_mc_data(ts)->mc_task.lvl_a_id;
376 414 struct ce_pid_table *pid_table = get_pid_table(ce_data->cpu);
377 TRACE_TASK(ts, "exited\n");
378 415
379 BUG_ON(task_cpu(ts) != get_partition(ts)); 416 BUG_ON(task_cpu(ts) != get_partition(ts));
380 BUG_ON(CRIT_LEVEL_A != tsk_mc_crit(ts)); 417 BUG_ON(CRIT_LEVEL_A != tsk_mc_crit(ts));
381 BUG_ON(lvl_a_id >= ce_data->num_pid_entries); 418 BUG_ON(lvl_a_id >= pid_table->num_pid_entries);
419 BUG_ON(ce_data->cpu != task_cpu(ts));
382 420
383 raw_spin_lock_irqsave(dom->lock, flags); 421 raw_spin_lock_irqsave(dom->lock, flags);
384 pid = ce_data->pid_entries[lvl_a_id].pid; 422 pid = pid_table->entries[lvl_a_id].pid;
385 BUG_ON(!pid); 423 BUG_ON(!pid);
386 put_pid(pid); 424 put_pid(pid);
387 ce_data->pid_entries[lvl_a_id].pid = NULL; 425 pid_table->entries[lvl_a_id].pid = NULL;
388 if (ce_data->scheduled == ts) 426 if (ce_data->scheduled == ts)
389 ce_data->scheduled = NULL; 427 ce_data->scheduled = NULL;
390 if (ce_data->should_schedule == ts) 428 if (ce_data->should_schedule == ts)
@@ -396,32 +434,32 @@ static void mc_ce_task_exit(struct task_struct *ts)
396 * Timer stuff 434 * Timer stuff
397 **********************************************************/ 435 **********************************************************/
398 436
399void __mc_ce_timer_callback(struct hrtimer *timer) 437void mc_ce_timer_callback_common(struct domain *dom, struct hrtimer *timer)
400{ 438{
401 /* relative and absolute times for cycles */ 439 /* relative and absolute times for cycles */
402 lt_t now, offset_rel, cycle_start_abs, next_timer_abs; 440 lt_t now, offset_rel, cycle_start_abs, next_timer_abs;
403 struct task_struct *should_schedule; 441 struct task_struct *should_schedule;
404 struct ce_dom_pid_entry *pid_entry; 442 struct ce_pid_table *pid_table;
443 struct ce_pid_entry *pid_entry;
405 struct ce_dom_data *ce_data; 444 struct ce_dom_data *ce_data;
406 domain_t *dom;
407 int idx, budget_overrun; 445 int idx, budget_overrun;
408 446
409 ce_data = container_of(timer, struct ce_dom_data, timer); 447 ce_data = dom->data;
410 dom = container_of(((void*)ce_data), domain_t, data); 448 pid_table = get_pid_table(ce_data->cpu);
411 449
412 /* Based off of the current time, figure out the offset into the cycle 450 /* Based off of the current time, figure out the offset into the cycle
413 * and the cycle's start time, and determine what should be scheduled. 451 * and the cycle's start time, and determine what should be scheduled.
414 */ 452 */
415 now = litmus_clock(); 453 now = litmus_clock();
416 offset_rel = get_cycle_offset(now, ce_data->cycle_time); 454 offset_rel = get_cycle_offset(now, pid_table->cycle_time);
417 cycle_start_abs = now - offset_rel; 455 cycle_start_abs = now - offset_rel;
418 idx = mc_ce_schedule_at(dom, offset_rel); 456 idx = mc_ce_schedule_at(dom, offset_rel);
419 pid_entry = &ce_data->pid_entries[idx]; 457 pid_entry = get_pid_entry(ce_data->cpu, idx);
420 /* set the timer to fire at the next cycle start */ 458 /* set the timer to fire at the next cycle start */
421 next_timer_abs = cycle_start_abs + pid_entry->acc_time; 459 next_timer_abs = cycle_start_abs + pid_entry->acc_time;
422 hrtimer_set_expires(timer, ns_to_ktime(next_timer_abs)); 460 hrtimer_set_expires(timer, ns_to_ktime(next_timer_abs));
423 461
424 TRACE("timer: now: %llu offset_rel: %llu cycle_start_abs: %llu " 462 STRACE("timer: now: %llu offset_rel: %llu cycle_start_abs: %llu "
425 "next_timer_abs: %llu\n", now, offset_rel, 463 "next_timer_abs: %llu\n", now, offset_rel,
426 cycle_start_abs, next_timer_abs); 464 cycle_start_abs, next_timer_abs);
427 465
@@ -440,10 +478,11 @@ void __mc_ce_timer_callback(struct hrtimer *timer)
440 budget_overrun = pid_entry->expected_job != 478 budget_overrun = pid_entry->expected_job !=
441 tsk_rt(should_schedule)->job_params.job_no; 479 tsk_rt(should_schedule)->job_params.job_no;
442 if (budget_overrun) 480 if (budget_overrun)
443 TRACE_TASK(should_schedule, "timer expected job number: %d " 481 TRACE_MC_TASK(should_schedule,
444 "but current job: %d\n", 482 "timer expected job number: %u "
445 pid_entry->expected_job, 483 "but current job: %u",
446 tsk_rt(should_schedule)->job_params.job_no); 484 pid_entry->expected_job,
485 tsk_rt(should_schedule)->job_params.job_no);
447 } 486 }
448 487
449 if (ce_data->should_schedule) { 488 if (ce_data->should_schedule) {
@@ -466,15 +505,15 @@ static enum hrtimer_restart mc_ce_timer_callback(struct hrtimer *timer)
466{ 505{
467 struct ce_dom_data *ce_data; 506 struct ce_dom_data *ce_data;
468 unsigned long flags; 507 unsigned long flags;
469 domain_t *dom; 508 struct domain *dom;
470 509
471 ce_data = container_of(timer, struct ce_dom_data, timer); 510 ce_data = container_of(timer, struct ce_dom_data, timer);
472 dom = container_of(((void*)ce_data), domain_t, data); 511 dom = get_domain_for(ce_data->cpu);
473 512
474 TRACE("timer callback on CPU %d (before lock)\n", ce_data->cpu); 513 TRACE("timer callback on CPU %d (before lock)\n", ce_data->cpu);
475 514
476 raw_spin_lock_irqsave(dom->lock, flags); 515 raw_spin_lock_irqsave(dom->lock, flags);
477 __mc_ce_timer_callback(timer); 516 mc_ce_timer_callback_common(dom, timer);
478 517
479 if (ce_data->scheduled != ce_data->should_schedule) 518 if (ce_data->scheduled != ce_data->should_schedule)
480 preempt_if_preemptable(ce_data->scheduled, ce_data->cpu); 519 preempt_if_preemptable(ce_data->scheduled, ce_data->cpu);
@@ -490,13 +529,13 @@ static enum hrtimer_restart mc_ce_timer_callback(struct hrtimer *timer)
490static int cancel_all_timers(void) 529static int cancel_all_timers(void)
491{ 530{
492 struct ce_dom_data *ce_data; 531 struct ce_dom_data *ce_data;
493 domain_t *dom; 532 struct domain *dom;
494 int cpu, ret = 0, cancel_res; 533 int cpu, cancel_res, ret = 0;
495 534
496 TRACE("cancel all timers\n"); 535 TRACE("cancel all timers\n");
497 536
498 for_each_online_cpu(cpu) { 537 for_each_online_cpu(cpu) {
499 dom = &per_cpu(mc_ce_doms, cpu); 538 dom = get_domain_for(cpu);
500 ce_data = dom->data; 539 ce_data = dom->data;
501 ce_data->should_schedule = NULL; 540 ce_data->should_schedule = NULL;
502 cancel_res = hrtimer_cancel(&ce_data->timer); 541 cancel_res = hrtimer_cancel(&ce_data->timer);
@@ -514,20 +553,22 @@ static int cancel_all_timers(void)
514 */ 553 */
515static void arm_all_timers(void) 554static void arm_all_timers(void)
516{ 555{
556 struct domain *dom;
517 struct ce_dom_data *ce_data; 557 struct ce_dom_data *ce_data;
518 domain_t *dom; 558 struct ce_pid_table *pid_table;
519 int cpu, idx; 559 int cpu, idx;
520 const lt_t start = atomic64_read(&start_time); 560 const lt_t start = atomic64_read(&start_time);
521 561
522 TRACE("arm all timers\n"); 562 TRACE("arm all timers\n");
523 563
524 for_each_online_cpu(cpu) { 564 for_each_online_cpu(cpu) {
525 dom = &per_cpu(mc_ce_doms, cpu); 565 dom = get_domain_for(cpu);
526 ce_data = dom->data; 566 ce_data = dom->data;
527 if (0 == ce_data->num_pid_entries) 567 pid_table = get_pid_table(cpu);
568 if (0 == pid_table->num_pid_entries)
528 continue; 569 continue;
529 for (idx = 0; idx < ce_data->num_pid_entries; idx++) { 570 for (idx = 0; idx < pid_table->num_pid_entries; idx++) {
530 ce_data->pid_entries[idx].expected_job = -1; 571 pid_table->entries[idx].expected_job = 0;
531 } 572 }
532 TRACE("arming timer for CPU %d\n", cpu); 573 TRACE("arming timer for CPU %d\n", cpu);
533 hrtimer_start_on(cpu, &ce_data->timer_info, &ce_data->timer, 574 hrtimer_start_on(cpu, &ce_data->timer_info, &ce_data->timer,
@@ -540,7 +581,7 @@ static void arm_all_timers(void)
540 * call this. We can re-set our notion of the CE period start to make 581 * call this. We can re-set our notion of the CE period start to make
541 * the schedule look pretty. 582 * the schedule look pretty.
542 */ 583 */
543void mc_ce_release_at(struct task_struct *ts, lt_t start) 584void mc_ce_release_at_common(struct task_struct *ts, lt_t start)
544{ 585{
545 TRACE_TASK(ts, "release at\n"); 586 TRACE_TASK(ts, "release at\n");
546 if (atomic_inc_and_test(&start_time_set)) { 587 if (atomic_inc_and_test(&start_time_set)) {
@@ -552,14 +593,14 @@ void mc_ce_release_at(struct task_struct *ts, lt_t start)
552 atomic_dec(&start_time_set); 593 atomic_dec(&start_time_set);
553} 594}
554 595
555long mc_ce_activate_plugin(void) 596long mc_ce_activate_plugin_common(void)
556{ 597{
557 struct ce_dom_data *ce_data; 598 struct ce_dom_data *ce_data;
558 domain_t *dom; 599 struct domain *dom;
559 int cpu; 600 int cpu;
560 601
561 for_each_online_cpu(cpu) { 602 for_each_online_cpu(cpu) {
562 dom = &per_cpu(mc_ce_doms, cpu); 603 dom = get_domain_for(cpu);
563 ce_data = dom->data; 604 ce_data = dom->data;
564 ce_data->scheduled = NULL; 605 ce_data->scheduled = NULL;
565 ce_data->should_schedule = NULL; 606 ce_data->should_schedule = NULL;
@@ -572,33 +613,54 @@ long mc_ce_activate_plugin(void)
572 return 0; 613 return 0;
573} 614}
574 615
616static long mc_ce_activate_plugin(void)
617{
618 struct domain_data *our_domains[NR_CPUS];
619 int cpu, n = 0;
620 long ret;
621
622 for_each_online_cpu(cpu) {
623 BUG_ON(NR_CPUS <= n);
624 our_domains[cpu] = &per_cpu(_mc_ce_doms, cpu);
625 n++;
626 }
627 ret = mc_ce_set_domains(n, our_domains);
628 if (ret)
629 goto out;
630 ret = mc_ce_activate_plugin_common();
631out:
632 return ret;
633}
634
575static void clear_pid_entries(void) 635static void clear_pid_entries(void)
576{ 636{
637 struct ce_pid_table *pid_table = NULL;
577 int cpu, entry; 638 int cpu, entry;
578 domain_t *dom;
579 struct ce_dom_data *ce_data;
580 639
581 for_each_online_cpu(cpu) { 640 for_each_online_cpu(cpu) {
582 dom = &per_cpu(mc_ce_doms, cpu); 641 pid_table = get_pid_table(cpu);
583 ce_data = dom->data; 642 pid_table->num_pid_entries = 0;
584 ce_data->num_pid_entries = 0; 643 pid_table->cycle_time = 0;
585 ce_data->cycle_time = 0;
586 for (entry = 0; entry < CONFIG_PLUGIN_MC_LEVEL_A_MAX_TASKS; 644 for (entry = 0; entry < CONFIG_PLUGIN_MC_LEVEL_A_MAX_TASKS;
587 ++entry) { 645 ++entry) {
588 if (NULL != ce_data->pid_entries[entry].pid) { 646 if (NULL != pid_table->entries[entry].pid) {
589 put_pid(ce_data->pid_entries[entry].pid); 647 put_pid(pid_table->entries[entry].pid);
590 ce_data->pid_entries[entry].pid = NULL; 648 pid_table->entries[entry].pid = NULL;
591 } 649 }
592 ce_data->pid_entries[entry].budget = 0; 650 pid_table->entries[entry].budget = 0;
593 ce_data->pid_entries[entry].acc_time = 0; 651 pid_table->entries[entry].acc_time = 0;
594 ce_data->pid_entries[entry].expected_job = -1; 652 pid_table->entries[entry].expected_job = 0;
595 } 653 }
596 } 654 }
597} 655}
598 656
599long mc_ce_deactivate_plugin(void) 657long mc_ce_deactivate_plugin_common(void)
600{ 658{
659 int cpu;
601 cancel_all_timers(); 660 cancel_all_timers();
661 for_each_online_cpu(cpu) {
662 per_cpu(domains, cpu) = NULL;
663 }
602 return 0; 664 return 0;
603} 665}
604 666
@@ -608,35 +670,33 @@ static struct sched_plugin mc_ce_plugin __cacheline_aligned_in_smp = {
608 .admit_task = mc_ce_admit_task, 670 .admit_task = mc_ce_admit_task,
609 .task_new = mc_ce_task_new, 671 .task_new = mc_ce_task_new,
610 .complete_job = complete_job, 672 .complete_job = complete_job,
611 .release_at = mc_ce_release_at, 673 .release_at = mc_ce_release_at_common,
612 .task_exit = mc_ce_task_exit, 674 .task_exit = mc_ce_task_exit_common,
613 .schedule = mc_ce_schedule, 675 .schedule = mc_ce_schedule,
614 .finish_switch = mc_ce_finish_switch, 676 .finish_switch = mc_ce_finish_switch,
615 .tick = mc_ce_tick,
616 .task_wake_up = mc_ce_task_wake_up, 677 .task_wake_up = mc_ce_task_wake_up,
617 .task_block = mc_ce_task_block, 678 .task_block = mc_ce_task_block,
618 .activate_plugin = mc_ce_activate_plugin, 679 .activate_plugin = mc_ce_activate_plugin,
619 .deactivate_plugin = mc_ce_deactivate_plugin, 680 .deactivate_plugin = mc_ce_deactivate_plugin_common,
620}; 681};
621 682
622static int setup_proc(void); 683static int setup_proc(void);
623static int __init init_sched_mc_ce(void) 684static int __init init_sched_mc_ce(void)
624{ 685{
625 struct ce_dom_data *ce_data; 686 raw_spinlock_t *ce_lock;
626 domain_t *dom; 687 struct domain_data *dom_data;
627 rt_domain_t *rt; 688 struct domain *dom;
628 int cpu, err; 689 int cpu, err;
629 690
630 for_each_online_cpu(cpu) { 691 for_each_online_cpu(cpu) {
631 dom = &per_cpu(mc_ce_doms, cpu); 692 per_cpu(domains, cpu) = NULL;
632 rt = &per_cpu(mc_ce_rts, cpu); 693 ce_lock = &per_cpu(_mc_ce_dom_locks, cpu);
633 pd_domain_init(dom, rt, NULL, NULL, NULL, NULL, NULL); 694 raw_spin_lock_init(ce_lock);
634 dom->data = &per_cpu(_mc_ce_dom_data, cpu); 695 dom_data = &per_cpu(_mc_ce_doms, cpu);
635 ce_data = dom->data; 696 dom = &dom_data->domain;
636 hrtimer_init(&ce_data->timer, CLOCK_MONOTONIC, HRTIMER_MODE_ABS); 697 ce_domain_init(dom, ce_lock, NULL, NULL, NULL, NULL, NULL,
637 hrtimer_start_on_info_init(&ce_data->timer_info); 698 &per_cpu(_mc_ce_dom_data, cpu), cpu,
638 ce_data->cpu = cpu; 699 mc_ce_timer_callback);
639 ce_data->timer.function = mc_ce_timer_callback;
640 } 700 }
641 clear_pid_entries(); 701 clear_pid_entries();
642 err = setup_proc(); 702 err = setup_proc();
@@ -678,7 +738,7 @@ out:
678#define PID_SPACE 15 738#define PID_SPACE 15
679#define TASK_INFO_BUF (PID_SPACE + TASK_COMM_LEN) 739#define TASK_INFO_BUF (PID_SPACE + TASK_COMM_LEN)
680static int write_pid_entry(char *page, const int count, const int cpu, 740static int write_pid_entry(char *page, const int count, const int cpu,
681 const int task, struct ce_dom_pid_entry *pid_entry) 741 const int task, struct ce_pid_entry *pid_entry)
682{ 742{
683 static char task_info[TASK_INFO_BUF]; 743 static char task_info[TASK_INFO_BUF];
684 struct task_struct *ts; 744 struct task_struct *ts;
@@ -735,8 +795,7 @@ static int proc_read_ce_file(char *page, char **start, off_t off, int count,
735 int *eof, void *data) 795 int *eof, void *data)
736{ 796{
737 int n = 0, err, cpu, t; 797 int n = 0, err, cpu, t;
738 struct ce_dom_data *ce_data; 798 struct ce_pid_table *pid_table;
739 domain_t *dom;
740 799
741 if (off > 0) { 800 if (off > 0) {
742 printk(KERN_INFO "litmus: MC-CE called read with off > 0\n"); 801 printk(KERN_INFO "litmus: MC-CE called read with off > 0\n");
@@ -744,11 +803,10 @@ static int proc_read_ce_file(char *page, char **start, off_t off, int count,
744 } 803 }
745 804
746 for_each_online_cpu(cpu) { 805 for_each_online_cpu(cpu) {
747 dom = &per_cpu(mc_ce_doms, cpu); 806 pid_table = get_pid_table(cpu);
748 ce_data = dom->data; 807 for (t = 0; t < pid_table->num_pid_entries; ++t) {
749 for (t = 0; t < ce_data->num_pid_entries; ++t) {
750 err = write_pid_entry(page + n, count - n, 808 err = write_pid_entry(page + n, count - n,
751 cpu, t, &ce_data->pid_entries[t]); 809 cpu, t, get_pid_entry(cpu, t));
752 if (err < 0) { 810 if (err < 0) {
753 n = -ENOSPC; 811 n = -ENOSPC;
754 goto out; 812 goto out;
@@ -785,9 +843,8 @@ static int skip_comment(const char *buf, const unsigned long max)
785#define BUDGET_THRESHOLD 5000000ULL 843#define BUDGET_THRESHOLD 5000000ULL
786static int setup_pid_entry(const int cpu, const int task, const lt_t budget) 844static int setup_pid_entry(const int cpu, const int task, const lt_t budget)
787{ 845{
788 domain_t *dom = &per_cpu(mc_ce_doms, cpu); 846 struct ce_pid_table *pid_table = get_pid_table(cpu);
789 struct ce_dom_data *ce_data = dom->data; 847 struct ce_pid_entry *new_entry = NULL;
790 struct ce_dom_pid_entry *new_entry;
791 int err = 0; 848 int err = 0;
792 849
793 /* check the inputs */ 850 /* check the inputs */
@@ -805,20 +862,20 @@ static int setup_pid_entry(const int cpu, const int task, const lt_t budget)
805 "MC-CE task; that might be an issue.\n"); 862 "MC-CE task; that might be an issue.\n");
806 } 863 }
807 /* check that we have space for a new entry */ 864 /* check that we have space for a new entry */
808 if (CONFIG_PLUGIN_MC_LEVEL_A_MAX_TASKS <= ce_data->num_pid_entries) { 865 if (CONFIG_PLUGIN_MC_LEVEL_A_MAX_TASKS <= pid_table->num_pid_entries) {
809 printk(KERN_INFO "litmus: too many MC-CE tasks for cpu " 866 printk(KERN_INFO "litmus: too many MC-CE tasks for cpu "
810 "%d\n", cpu); 867 "%d\n", cpu);
811 err = -EINVAL; 868 err = -EINVAL;
812 goto out; 869 goto out;
813 } 870 }
814 /* add the new entry */ 871 /* add the new entry */
815 new_entry = &ce_data->pid_entries[ce_data->num_pid_entries]; 872 new_entry = get_pid_entry(cpu, pid_table->num_pid_entries);
816 BUG_ON(NULL != new_entry->pid); 873 BUG_ON(NULL != new_entry->pid);
817 new_entry->budget = budget; 874 new_entry->budget = budget;
818 new_entry->acc_time = ce_data->cycle_time + budget; 875 new_entry->acc_time = pid_table->cycle_time + budget;
819 /* update the domain entry */ 876 /* update the domain entry */
820 ce_data->cycle_time += budget; 877 pid_table->cycle_time += budget;
821 ce_data->num_pid_entries++; 878 pid_table->num_pid_entries++;
822out: 879out:
823 return err; 880 return err;
824} 881}
@@ -839,9 +896,9 @@ static int proc_write_ce_file(struct file *file, const char __user *buffer,
839 int cpu, task, cnt = 0, chars_read, converted, err; 896 int cpu, task, cnt = 0, chars_read, converted, err;
840 lt_t budget; 897 lt_t budget;
841 898
842 if (is_active_plugin()) { 899 if (!using_linux_plugin()) {
843 printk(KERN_INFO "litmus: can't edit MC-CE proc when plugin " 900 printk(KERN_INFO "litmus: can only edit MC-CE proc under Linux "
844 "active\n"); 901 "plugin\n");
845 cnt = -EINVAL; 902 cnt = -EINVAL;
846 goto out; 903 goto out;
847 } 904 }
generated by cgit v1.2.2 (git 2.25.0) at 2025-04-09 00:26:13 -0400