aboutsummaryrefslogtreecommitdiffstats
path: root/litmus
diff options
context:
space:
mode:
authorAndrea Bastoni <bastoni@cs.unc.edu>2010-05-29 23:35:01 -0400
committerAndrea Bastoni <bastoni@cs.unc.edu>2010-05-29 23:35:01 -0400
commit6ffc1fee98c4b995eb3a0285f4f8fb467cb0306e (patch)
tree69a05892a41e7f7400fa598ee0bdf8027c8f0fd6 /litmus
parente40152ee1e1c7a63f4777791863215e3faa37a86 (diff)
parent7c1ff4c544dd650cceff3cd69a04bcba60856678 (diff)
Merge branch 'master' into wip-merge-2.6.34
Simple merge between master and 2.6.34 with conflicts resolved. This commit does not compile, the following main problems are still unresolved: - spinlock -> raw_spinlock API changes - kfifo API changes - sched_class API changes Conflicts: Makefile arch/x86/include/asm/hw_irq.h arch/x86/include/asm/unistd_32.h arch/x86/kernel/syscall_table_32.S include/linux/hrtimer.h kernel/sched.c kernel/sched_fair.c
Diffstat (limited to 'litmus')
-rw-r--r--litmus/Kconfig85
-rw-r--r--litmus/Makefile23
-rw-r--r--litmus/bheap.c314
-rw-r--r--litmus/ctrldev.c150
-rw-r--r--litmus/edf_common.c102
-rw-r--r--litmus/fdso.c281
-rw-r--r--litmus/fmlp.c268
-rw-r--r--litmus/ft_event.c43
-rw-r--r--litmus/ftdev.c359
-rw-r--r--litmus/jobs.c43
-rw-r--r--litmus/litmus.c775
-rw-r--r--litmus/rt_domain.c310
-rw-r--r--litmus/sched_cedf.c756
-rw-r--r--litmus/sched_gsn_edf.c828
-rw-r--r--litmus/sched_litmus.c318
-rw-r--r--litmus/sched_pfair.c896
-rw-r--r--litmus/sched_plugin.c265
-rw-r--r--litmus/sched_psn_edf.c478
-rw-r--r--litmus/sched_task_trace.c204
-rw-r--r--litmus/sched_trace.c378
-rw-r--r--litmus/srp.c318
-rw-r--r--litmus/sync.c104
-rw-r--r--litmus/trace.c103
23 files changed, 7401 insertions, 0 deletions
diff --git a/litmus/Kconfig b/litmus/Kconfig
new file mode 100644
index 000000000000..874794f64af1
--- /dev/null
+++ b/litmus/Kconfig
@@ -0,0 +1,85 @@
1menu "LITMUS^RT"
2
3menu "Real-Time Synchronization"
4
5config NP_SECTION
6 bool "Non-preemptive section support"
7 default n
8 help
9 Allow tasks to become non-preemptable.
10 Note that plugins still need to explicitly support non-preemptivity.
11 Currently, only GSN-EDF and PSN-EDF have such support.
12
13 This is required to support the FMLP.
14 If disabled, all tasks will be considered preemptable at all times.
15
16config SRP
17 bool "Stack Resource Policy (SRP)"
18 default n
19 help
20 Include support for Baker's Stack Resource Policy.
21
22 Say Yes if you want FMLP local long critical section
23 synchronization support.
24
25config FMLP
26 bool "FMLP support"
27 depends on NP_SECTION
28 default n
29 help
30 Include support for deterministic multiprocessor real-time
31 synchronization support.
32
33 Say Yes if you want FMLP long critical section
34 synchronization support.
35
36endmenu
37
38menu "Tracing"
39
40config FEATHER_TRACE
41 bool "Feather-Trace Infrastructure"
42 default y
43 help
44 Feather-Trace basic tracing infrastructure. Includes device file
45 driver and instrumentation point support.
46
47
48config SCHED_TASK_TRACE
49 bool "Trace real-time tasks"
50 depends on FEATHER_TRACE
51 default y
52 help
53 Include support for the sched_trace_XXX() tracing functions. This
54 allows the collection of real-time task events such as job
55 completions, job releases, early completions, etc. This results in a
56 small overhead in the scheduling code. Disable if the overhead is not
57 acceptable (e.g., benchmarking).
58
59 Say Yes for debugging.
60 Say No for overhead tracing.
61
62config SCHED_OVERHEAD_TRACE
63 bool "Record timestamps for overhead measurements"
64 depends on FEATHER_TRACE
65 default n
66 help
67 Export event stream for overhead tracing.
68 Say Yes for overhead tracing.
69
70config SCHED_DEBUG_TRACE
71 bool "TRACE() debugging"
72 default y
73 help
74 Include support for sched_trace_log_messageg(), which is used to
75 implement TRACE(). If disabled, no TRACE() messages will be included
76 in the kernel, and no overheads due to debugging statements will be
77 incurred by the scheduler. Disable if the overhead is not acceptable
78 (e.g. benchmarking).
79
80 Say Yes for debugging.
81 Say No for overhead tracing.
82
83endmenu
84
85endmenu
diff --git a/litmus/Makefile b/litmus/Makefile
new file mode 100644
index 000000000000..0cc33e8bee51
--- /dev/null
+++ b/litmus/Makefile
@@ -0,0 +1,23 @@
1#
2# Makefile for LITMUS^RT
3#
4
5obj-y = sched_plugin.o litmus.o \
6 jobs.o \
7 sync.o \
8 rt_domain.o \
9 edf_common.o \
10 fdso.o \
11 srp.o \
12 fmlp.o \
13 bheap.o \
14 ctrldev.o \
15 sched_gsn_edf.o \
16 sched_psn_edf.o \
17 sched_cedf.o \
18 sched_pfair.o
19
20obj-$(CONFIG_FEATHER_TRACE) += ft_event.o ftdev.o
21obj-$(CONFIG_SCHED_TASK_TRACE) += sched_task_trace.o
22obj-$(CONFIG_SCHED_DEBUG_TRACE) += sched_trace.o
23obj-$(CONFIG_SCHED_OVERHEAD_TRACE) += trace.o
diff --git a/litmus/bheap.c b/litmus/bheap.c
new file mode 100644
index 000000000000..528af97f18a6
--- /dev/null
+++ b/litmus/bheap.c
@@ -0,0 +1,314 @@
1#include "linux/kernel.h"
2#include "litmus/bheap.h"
3
4void bheap_init(struct bheap* heap)
5{
6 heap->head = NULL;
7 heap->min = NULL;
8}
9
10void bheap_node_init(struct bheap_node** _h, void* value)
11{
12 struct bheap_node* h = *_h;
13 h->parent = NULL;
14 h->next = NULL;
15 h->child = NULL;
16 h->degree = NOT_IN_HEAP;
17 h->value = value;
18 h->ref = _h;
19}
20
21
22/* make child a subtree of root */
23static void __bheap_link(struct bheap_node* root,
24 struct bheap_node* child)
25{
26 child->parent = root;
27 child->next = root->child;
28 root->child = child;
29 root->degree++;
30}
31
32/* merge root lists */
33static struct bheap_node* __bheap_merge(struct bheap_node* a,
34 struct bheap_node* b)
35{
36 struct bheap_node* head = NULL;
37 struct bheap_node** pos = &head;
38
39 while (a && b) {
40 if (a->degree < b->degree) {
41 *pos = a;
42 a = a->next;
43 } else {
44 *pos = b;
45 b = b->next;
46 }
47 pos = &(*pos)->next;
48 }
49 if (a)
50 *pos = a;
51 else
52 *pos = b;
53 return head;
54}
55
56/* reverse a linked list of nodes. also clears parent pointer */
57static struct bheap_node* __bheap_reverse(struct bheap_node* h)
58{
59 struct bheap_node* tail = NULL;
60 struct bheap_node* next;
61
62 if (!h)
63 return h;
64
65 h->parent = NULL;
66 while (h->next) {
67 next = h->next;
68 h->next = tail;
69 tail = h;
70 h = next;
71 h->parent = NULL;
72 }
73 h->next = tail;
74 return h;
75}
76
77static void __bheap_min(bheap_prio_t higher_prio, struct bheap* heap,
78 struct bheap_node** prev, struct bheap_node** node)
79{
80 struct bheap_node *_prev, *cur;
81 *prev = NULL;
82
83 if (!heap->head) {
84 *node = NULL;
85 return;
86 }
87
88 *node = heap->head;
89 _prev = heap->head;
90 cur = heap->head->next;
91 while (cur) {
92 if (higher_prio(cur, *node)) {
93 *node = cur;
94 *prev = _prev;
95 }
96 _prev = cur;
97 cur = cur->next;
98 }
99}
100
101static void __bheap_union(bheap_prio_t higher_prio, struct bheap* heap,
102 struct bheap_node* h2)
103{
104 struct bheap_node* h1;
105 struct bheap_node *prev, *x, *next;
106 if (!h2)
107 return;
108 h1 = heap->head;
109 if (!h1) {
110 heap->head = h2;
111 return;
112 }
113 h1 = __bheap_merge(h1, h2);
114 prev = NULL;
115 x = h1;
116 next = x->next;
117 while (next) {
118 if (x->degree != next->degree ||
119 (next->next && next->next->degree == x->degree)) {
120 /* nothing to do, advance */
121 prev = x;
122 x = next;
123 } else if (higher_prio(x, next)) {
124 /* x becomes the root of next */
125 x->next = next->next;
126 __bheap_link(x, next);
127 } else {
128 /* next becomes the root of x */
129 if (prev)
130 prev->next = next;
131 else
132 h1 = next;
133 __bheap_link(next, x);
134 x = next;
135 }
136 next = x->next;
137 }
138 heap->head = h1;
139}
140
141static struct bheap_node* __bheap_extract_min(bheap_prio_t higher_prio,
142 struct bheap* heap)
143{
144 struct bheap_node *prev, *node;
145 __bheap_min(higher_prio, heap, &prev, &node);
146 if (!node)
147 return NULL;
148 if (prev)
149 prev->next = node->next;
150 else
151 heap->head = node->next;
152 __bheap_union(higher_prio, heap, __bheap_reverse(node->child));
153 return node;
154}
155
156/* insert (and reinitialize) a node into the heap */
157void bheap_insert(bheap_prio_t higher_prio, struct bheap* heap,
158 struct bheap_node* node)
159{
160 struct bheap_node *min;
161 node->child = NULL;
162 node->parent = NULL;
163 node->next = NULL;
164 node->degree = 0;
165 if (heap->min && higher_prio(node, heap->min)) {
166 /* swap min cache */
167 min = heap->min;
168 min->child = NULL;
169 min->parent = NULL;
170 min->next = NULL;
171 min->degree = 0;
172 __bheap_union(higher_prio, heap, min);
173 heap->min = node;
174 } else
175 __bheap_union(higher_prio, heap, node);
176}
177
178void bheap_uncache_min(bheap_prio_t higher_prio, struct bheap* heap)
179{
180 struct bheap_node* min;
181 if (heap->min) {
182 min = heap->min;
183 heap->min = NULL;
184 bheap_insert(higher_prio, heap, min);
185 }
186}
187
188/* merge addition into target */
189void bheap_union(bheap_prio_t higher_prio,
190 struct bheap* target, struct bheap* addition)
191{
192 /* first insert any cached minima, if necessary */
193 bheap_uncache_min(higher_prio, target);
194 bheap_uncache_min(higher_prio, addition);
195 __bheap_union(higher_prio, target, addition->head);
196 /* this is a destructive merge */
197 addition->head = NULL;
198}
199
200struct bheap_node* bheap_peek(bheap_prio_t higher_prio,
201 struct bheap* heap)
202{
203 if (!heap->min)
204 heap->min = __bheap_extract_min(higher_prio, heap);
205 return heap->min;
206}
207
208struct bheap_node* bheap_take(bheap_prio_t higher_prio,
209 struct bheap* heap)
210{
211 struct bheap_node *node;
212 if (!heap->min)
213 heap->min = __bheap_extract_min(higher_prio, heap);
214 node = heap->min;
215 heap->min = NULL;
216 if (node)
217 node->degree = NOT_IN_HEAP;
218 return node;
219}
220
221int bheap_decrease(bheap_prio_t higher_prio, struct bheap_node* node)
222{
223 struct bheap_node *parent;
224 struct bheap_node** tmp_ref;
225 void* tmp;
226
227 /* bubble up */
228 parent = node->parent;
229 while (parent && higher_prio(node, parent)) {
230 /* swap parent and node */
231 tmp = parent->value;
232 parent->value = node->value;
233 node->value = tmp;
234 /* swap references */
235 *(parent->ref) = node;
236 *(node->ref) = parent;
237 tmp_ref = parent->ref;
238 parent->ref = node->ref;
239 node->ref = tmp_ref;
240 /* step up */
241 node = parent;
242 parent = node->parent;
243 }
244
245 return parent != NULL;
246}
247
248void bheap_delete(bheap_prio_t higher_prio, struct bheap* heap,
249 struct bheap_node* node)
250{
251 struct bheap_node *parent, *prev, *pos;
252 struct bheap_node** tmp_ref;
253 void* tmp;
254
255 if (heap->min != node) {
256 /* bubble up */
257 parent = node->parent;
258 while (parent) {
259 /* swap parent and node */
260 tmp = parent->value;
261 parent->value = node->value;
262 node->value = tmp;
263 /* swap references */
264 *(parent->ref) = node;
265 *(node->ref) = parent;
266 tmp_ref = parent->ref;
267 parent->ref = node->ref;
268 node->ref = tmp_ref;
269 /* step up */
270 node = parent;
271 parent = node->parent;
272 }
273 /* now delete:
274 * first find prev */
275 prev = NULL;
276 pos = heap->head;
277 while (pos != node) {
278 prev = pos;
279 pos = pos->next;
280 }
281 /* we have prev, now remove node */
282 if (prev)
283 prev->next = node->next;
284 else
285 heap->head = node->next;
286 __bheap_union(higher_prio, heap, __bheap_reverse(node->child));
287 } else
288 heap->min = NULL;
289 node->degree = NOT_IN_HEAP;
290}
291
292/* allocate a heap node for value and insert into the heap */
293int bheap_add(bheap_prio_t higher_prio, struct bheap* heap,
294 void* value, int gfp_flags)
295{
296 struct bheap_node* hn = bheap_node_alloc(gfp_flags);
297 if (likely(hn)) {
298 bheap_node_init(&hn, value);
299 bheap_insert(higher_prio, heap, hn);
300 }
301 return hn != NULL;
302}
303
304void* bheap_take_del(bheap_prio_t higher_prio,
305 struct bheap* heap)
306{
307 struct bheap_node* hn = bheap_take(higher_prio, heap);
308 void* ret = NULL;
309 if (hn) {
310 ret = hn->value;
311 bheap_node_free(hn);
312 }
313 return ret;
314}
diff --git a/litmus/ctrldev.c b/litmus/ctrldev.c
new file mode 100644
index 000000000000..6677a67cc945
--- /dev/null
+++ b/litmus/ctrldev.c
@@ -0,0 +1,150 @@
1#include <linux/sched.h>
2#include <linux/mm.h>
3#include <linux/fs.h>
4#include <linux/miscdevice.h>
5#include <linux/module.h>
6
7#include <litmus/litmus.h>
8
9/* only one page for now, but we might want to add a RO version at some point */
10
11#define CTRL_NAME "litmus/ctrl"
12
13/* allocate t->rt_param.ctrl_page*/
14static int alloc_ctrl_page(struct task_struct *t)
15{
16 int err = 0;
17
18 /* only allocate if the task doesn't have one yet */
19 if (!tsk_rt(t)->ctrl_page) {
20 tsk_rt(t)->ctrl_page = (void*) get_zeroed_page(GFP_KERNEL);
21 if (!tsk_rt(t)->ctrl_page)
22 err = -ENOMEM;
23 /* will get de-allocated in task teardown */
24 TRACE_TASK(t, "%s ctrl_page = %p\n", __FUNCTION__,
25 tsk_rt(t)->ctrl_page);
26 }
27 return err;
28}
29
30static int map_ctrl_page(struct task_struct *t, struct vm_area_struct* vma)
31{
32 int err;
33 unsigned long pfn;
34
35 struct page* ctrl = virt_to_page(tsk_rt(t)->ctrl_page);
36
37 /* Increase ref count. Is decreased when vma is destroyed. */
38 get_page(ctrl);
39
40 /* compute page frame number */
41 pfn = page_to_pfn(ctrl);
42
43 TRACE_CUR(CTRL_NAME
44 ": mapping %p (pfn:%lx, %lx) to 0x%lx (prot:%lx)\n",
45 tsk_rt(t)->ctrl_page, pfn, page_to_pfn(ctrl), vma->vm_start,
46 vma->vm_page_prot);
47
48 /* Map it into the vma. Make sure to use PAGE_SHARED, otherwise
49 * userspace actually gets a copy-on-write page. */
50 err = remap_pfn_range(vma, vma->vm_start, pfn, PAGE_SIZE, PAGE_SHARED);
51
52 if (err)
53 TRACE_CUR(CTRL_NAME ": remap_pfn_range() failed (%d)\n", err);
54
55 return err;
56}
57
58static void litmus_ctrl_vm_close(struct vm_area_struct* vma)
59{
60 TRACE_CUR("%s flags=0x%x prot=0x%x\n", __FUNCTION__,
61 vma->vm_flags, vma->vm_page_prot);
62
63 TRACE_CUR(CTRL_NAME
64 ": %p:%p vma:%p vma->vm_private_data:%p closed.\n",
65 (void*) vma->vm_start, (void*) vma->vm_end, vma,
66 vma->vm_private_data, current->comm,
67 current->pid);
68}
69
70static int litmus_ctrl_vm_fault(struct vm_area_struct* vma,
71 struct vm_fault* vmf)
72{
73 /* This function should never be called, since
74 * all pages should have been mapped by mmap()
75 * already. */
76 TRACE_CUR("%s flags=0x%x\n", __FUNCTION__, vma->vm_flags);
77
78 /* nope, you only get one page */
79 return VM_FAULT_SIGBUS;
80}
81
82static struct vm_operations_struct litmus_ctrl_vm_ops = {
83 .close = litmus_ctrl_vm_close,
84 .fault = litmus_ctrl_vm_fault,
85};
86
87static int litmus_ctrl_mmap(struct file* filp, struct vm_area_struct* vma)
88{
89 int err = 0;
90
91 /* first make sure mapper knows what he's doing */
92
93 /* you can only get one page */
94 if (vma->vm_end - vma->vm_start != PAGE_SIZE)
95 return -EINVAL;
96
97 /* you can only map the "first" page */
98 if (vma->vm_pgoff != 0)
99 return -EINVAL;
100
101 /* you can't share it with anyone */
102 if (vma->vm_flags & (VM_MAYSHARE | VM_SHARED))
103 return -EINVAL;
104
105 vma->vm_ops = &litmus_ctrl_vm_ops;
106 /* this mapping should not be kept across forks,
107 * and cannot be expanded */
108 vma->vm_flags |= VM_DONTCOPY | VM_DONTEXPAND;
109
110 err = alloc_ctrl_page(current);
111 if (!err)
112 err = map_ctrl_page(current, vma);
113
114 TRACE_CUR("%s flags=0x%x prot=0x%lx\n",
115 __FUNCTION__, vma->vm_flags, vma->vm_page_prot);
116
117 return err;
118}
119
120static struct file_operations litmus_ctrl_fops = {
121 .owner = THIS_MODULE,
122 .mmap = litmus_ctrl_mmap,
123};
124
125static struct miscdevice litmus_ctrl_dev = {
126 .name = CTRL_NAME,
127 .minor = MISC_DYNAMIC_MINOR,
128 .fops = &litmus_ctrl_fops,
129};
130
131static int __init init_litmus_ctrl_dev(void)
132{
133 int err;
134
135 BUILD_BUG_ON(sizeof(struct control_page) > PAGE_SIZE);
136
137 printk("Initializing LITMUS^RT control device.\n");
138 err = misc_register(&litmus_ctrl_dev);
139 if (err)
140 printk("Could not allocate %s device (%d).\n", CTRL_NAME, err);
141 return err;
142}
143
144static void __exit exit_litmus_ctrl_dev(void)
145{
146 misc_deregister(&litmus_ctrl_dev);
147}
148
149module_init(init_litmus_ctrl_dev);
150module_exit(exit_litmus_ctrl_dev);
diff --git a/litmus/edf_common.c b/litmus/edf_common.c
new file mode 100644
index 000000000000..06daec66c984
--- /dev/null
+++ b/litmus/edf_common.c
@@ -0,0 +1,102 @@
1/*
2 * kernel/edf_common.c
3 *
4 * Common functions for EDF based scheduler.
5 */
6
7#include <linux/percpu.h>
8#include <linux/sched.h>
9#include <linux/list.h>
10
11#include <litmus/litmus.h>
12#include <litmus/sched_plugin.h>
13#include <litmus/sched_trace.h>
14
15#include <litmus/edf_common.h>
16
17/* edf_higher_prio - returns true if first has a higher EDF priority
18 * than second. Deadline ties are broken by PID.
19 *
20 * both first and second may be NULL
21 */
22int edf_higher_prio(struct task_struct* first,
23 struct task_struct* second)
24{
25 struct task_struct *first_task = first;
26 struct task_struct *second_task = second;
27
28 /* There is no point in comparing a task to itself. */
29 if (first && first == second) {
30 TRACE_TASK(first,
31 "WARNING: pointless edf priority comparison.\n");
32 return 0;
33 }
34
35
36 /* Check for inherited priorities. Change task
37 * used for comparison in such a case.
38 */
39 if (first && first->rt_param.inh_task)
40 first_task = first->rt_param.inh_task;
41 if (second && second->rt_param.inh_task)
42 second_task = second->rt_param.inh_task;
43
44 return
45 /* it has to exist in order to have higher priority */
46 first_task && (
47 /* does the second task exist and is it a real-time task? If
48 * not, the first task (which is a RT task) has higher
49 * priority.
50 */
51 !second_task || !is_realtime(second_task) ||
52
53 /* is the deadline of the first task earlier?
54 * Then it has higher priority.
55 */
56 earlier_deadline(first_task, second_task) ||
57
58 /* Do we have a deadline tie?
59 * Then break by PID.
60 */
61 (get_deadline(first_task) == get_deadline(second_task) &&
62 (first_task->pid < second_task->pid ||
63
64 /* If the PIDs are the same then the task with the inherited
65 * priority wins.
66 */
67 (first_task->pid == second_task->pid &&
68 !second->rt_param.inh_task))));
69}
70
71int edf_ready_order(struct bheap_node* a, struct bheap_node* b)
72{
73 return edf_higher_prio(bheap2task(a), bheap2task(b));
74}
75
76void edf_domain_init(rt_domain_t* rt, check_resched_needed_t resched,
77 release_jobs_t release)
78{
79 rt_domain_init(rt, edf_ready_order, resched, release);
80}
81
82/* need_to_preempt - check whether the task t needs to be preempted
83 * call only with irqs disabled and with ready_lock acquired
84 * THIS DOES NOT TAKE NON-PREEMPTIVE SECTIONS INTO ACCOUNT!
85 */
86int edf_preemption_needed(rt_domain_t* rt, struct task_struct *t)
87{
88 /* we need the read lock for edf_ready_queue */
89 /* no need to preempt if there is nothing pending */
90 if (!__jobs_pending(rt))
91 return 0;
92 /* we need to reschedule if t doesn't exist */
93 if (!t)
94 return 1;
95
96 /* NOTE: We cannot check for non-preemptibility since we
97 * don't know what address space we're currently in.
98 */
99
100 /* make sure to get non-rt stuff out of the way */
101 return !is_realtime(t) || edf_higher_prio(__next_ready(rt), t);
102}
diff --git a/litmus/fdso.c b/litmus/fdso.c
new file mode 100644
index 000000000000..85be716941d8
--- /dev/null
+++ b/litmus/fdso.c
@@ -0,0 +1,281 @@
1/* fdso.c - file descriptor attached shared objects
2 *
3 * (c) 2007 B. Brandenburg, LITMUS^RT project
4 *
5 * Notes:
6 * - objects descriptor (OD) tables are not cloned during a fork.
7 * - objects are created on-demand, and freed after the last reference
8 * is dropped.
9 * - for now, object types are hard coded.
10 * - As long as we have live objects, we keep a reference to the inode.
11 */
12
13#include <linux/errno.h>
14#include <linux/sched.h>
15#include <linux/mutex.h>
16#include <linux/file.h>
17#include <asm/uaccess.h>
18
19#include <litmus/fdso.h>
20
21extern struct fdso_ops fmlp_sem_ops;
22extern struct fdso_ops srp_sem_ops;
23
24static const struct fdso_ops* fdso_ops[] = {
25 &fmlp_sem_ops,
26 &srp_sem_ops,
27};
28
29static void* fdso_create(obj_type_t type)
30{
31 if (fdso_ops[type]->create)
32 return fdso_ops[type]->create();
33 else
34 return NULL;
35}
36
37static void fdso_destroy(obj_type_t type, void* obj)
38{
39 fdso_ops[type]->destroy(obj);
40}
41
42static int fdso_open(struct od_table_entry* entry, void* __user config)
43{
44 if (fdso_ops[entry->obj->type]->open)
45 return fdso_ops[entry->obj->type]->open(entry, config);
46 else
47 return 0;
48}
49
50static int fdso_close(struct od_table_entry* entry)
51{
52 if (fdso_ops[entry->obj->type]->close)
53 return fdso_ops[entry->obj->type]->close(entry);
54 else
55 return 0;
56}
57
58/* inode must be locked already */
59static struct inode_obj_id* alloc_inode_obj(struct inode* inode,
60 obj_type_t type,
61 unsigned int id)
62{
63 struct inode_obj_id* obj;
64 void* raw_obj;
65
66 raw_obj = fdso_create(type);
67 if (!raw_obj)
68 return NULL;
69
70 obj = kmalloc(sizeof(*obj), GFP_KERNEL);
71 if (!obj)
72 return NULL;
73 INIT_LIST_HEAD(&obj->list);
74 atomic_set(&obj->count, 1);
75 obj->type = type;
76 obj->id = id;
77 obj->obj = raw_obj;
78 obj->inode = inode;
79
80 list_add(&obj->list, &inode->i_obj_list);
81 atomic_inc(&inode->i_count);
82
83 printk(KERN_DEBUG "alloc_inode_obj(%p, %d, %d): object created\n", inode, type, id);
84 return obj;
85}
86
87/* inode must be locked already */
88static struct inode_obj_id* get_inode_obj(struct inode* inode,
89 obj_type_t type,
90 unsigned int id)
91{
92 struct list_head* pos;
93 struct inode_obj_id* obj = NULL;
94
95 list_for_each(pos, &inode->i_obj_list) {
96 obj = list_entry(pos, struct inode_obj_id, list);
97 if (obj->id == id && obj->type == type) {
98 atomic_inc(&obj->count);
99 return obj;
100 }
101 }
102 printk(KERN_DEBUG "get_inode_obj(%p, %d, %d): couldn't find object\n", inode, type, id);
103 return NULL;
104}
105
106
107static void put_inode_obj(struct inode_obj_id* obj)
108{
109 struct inode* inode;
110 int let_go = 0;
111
112 inode = obj->inode;
113 if (atomic_dec_and_test(&obj->count)) {
114
115 mutex_lock(&inode->i_obj_mutex);
116 /* no new references can be obtained */
117 if (!atomic_read(&obj->count)) {
118 list_del(&obj->list);
119 fdso_destroy(obj->type, obj->obj);
120 kfree(obj);
121 let_go = 1;
122 }
123 mutex_unlock(&inode->i_obj_mutex);
124 if (let_go)
125 iput(inode);
126 }
127}
128
129static struct od_table_entry* get_od_entry(struct task_struct* t)
130{
131 struct od_table_entry* table;
132 int i;
133
134
135 table = t->od_table;
136 if (!table) {
137 table = kzalloc(sizeof(*table) * MAX_OBJECT_DESCRIPTORS,
138 GFP_KERNEL);
139 t->od_table = table;
140 }
141
142 for (i = 0; table && i < MAX_OBJECT_DESCRIPTORS; i++)
143 if (!table[i].used) {
144 table[i].used = 1;
145 return table + i;
146 }
147 return NULL;
148}
149
150static int put_od_entry(struct od_table_entry* od)
151{
152 put_inode_obj(od->obj);
153 od->used = 0;
154 return 0;
155}
156
157void exit_od_table(struct task_struct* t)
158{
159 int i;
160
161 if (t->od_table) {
162 for (i = 0; i < MAX_OBJECT_DESCRIPTORS; i++)
163 if (t->od_table[i].used)
164 put_od_entry(t->od_table + i);
165 kfree(t->od_table);
166 t->od_table = NULL;
167 }
168}
169
170static int do_sys_od_open(struct file* file, obj_type_t type, int id,
171 void* __user config)
172{
173 int idx = 0, err;
174 struct inode* inode;
175 struct inode_obj_id* obj = NULL;
176 struct od_table_entry* entry;
177
178 inode = file->f_dentry->d_inode;
179
180 entry = get_od_entry(current);
181 if (!entry)
182 return -ENOMEM;
183
184 mutex_lock(&inode->i_obj_mutex);
185 obj = get_inode_obj(inode, type, id);
186 if (!obj)
187 obj = alloc_inode_obj(inode, type, id);
188 if (!obj) {
189 idx = -ENOMEM;
190 entry->used = 0;
191 } else {
192 entry->obj = obj;
193 entry->extra = NULL;
194 idx = entry - current->od_table;
195 }
196
197 mutex_unlock(&inode->i_obj_mutex);
198
199 err = fdso_open(entry, config);
200 if (err < 0) {
201 /* The class rejected the open call.
202 * We need to clean up and tell user space.
203 */
204 put_od_entry(entry);
205 idx = err;
206 }
207
208 return idx;
209}
210
211
212struct od_table_entry* __od_lookup(int od)
213{
214 struct task_struct *t = current;
215
216 if (!t->od_table)
217 return NULL;
218 if (od < 0 || od >= MAX_OBJECT_DESCRIPTORS)
219 return NULL;
220 if (!t->od_table[od].used)
221 return NULL;
222 return t->od_table + od;
223}
224
225
226asmlinkage long sys_od_open(int fd, int type, int obj_id, void* __user config)
227{
228 int ret = 0;
229 struct file* file;
230
231 /*
232 1) get file from fd, get inode from file
233 2) lock inode
234 3) try to lookup object
235 4) if not present create and enqueue object, inc inode refcnt
236 5) increment refcnt of object
237 6) alloc od_table_entry, setup ptrs
238 7) unlock inode
239 8) return offset in od_table as OD
240 */
241
242 if (type < MIN_OBJ_TYPE || type > MAX_OBJ_TYPE) {
243 ret = -EINVAL;
244 goto out;
245 }
246
247 file = fget(fd);
248 if (!file) {
249 ret = -EBADF;
250 goto out;
251 }
252
253 ret = do_sys_od_open(file, type, obj_id, config);
254
255 fput(file);
256
257out:
258 return ret;
259}
260
261
262asmlinkage long sys_od_close(int od)
263{
264 int ret = -EINVAL;
265 struct task_struct *t = current;
266
267 if (od < 0 || od >= MAX_OBJECT_DESCRIPTORS)
268 return ret;
269
270 if (!t->od_table || !t->od_table[od].used)
271 return ret;
272
273
274 /* give the class a chance to reject the close
275 */
276 ret = fdso_close(t->od_table + od);
277 if (ret == 0)
278 ret = put_od_entry(t->od_table + od);
279
280 return ret;
281}
diff --git a/litmus/fmlp.c b/litmus/fmlp.c
new file mode 100644
index 000000000000..03fa7358d5eb
--- /dev/null
+++ b/litmus/fmlp.c
@@ -0,0 +1,268 @@
1/*
2 * FMLP implementation.
3 * Much of the code here is borrowed from include/asm-i386/semaphore.h
4 */
5
6#include <asm/atomic.h>
7
8#include <linux/semaphore.h>
9#include <linux/sched.h>
10#include <linux/wait.h>
11#include <linux/spinlock.h>
12
13#include <litmus/litmus.h>
14#include <litmus/sched_plugin.h>
15#include <litmus/edf_common.h>
16
17#include <litmus/fdso.h>
18
19#include <litmus/trace.h>
20
21#ifdef CONFIG_FMLP
22
23static void* create_fmlp_semaphore(void)
24{
25 struct pi_semaphore* sem;
26 int i;
27
28 sem = kmalloc(sizeof(*sem), GFP_KERNEL);
29 if (!sem)
30 return NULL;
31 atomic_set(&sem->count, 1);
32 sem->sleepers = 0;
33 init_waitqueue_head(&sem->wait);
34 sem->hp.task = NULL;
35 sem->holder = NULL;
36 for (i = 0; i < NR_CPUS; i++)
37 sem->hp.cpu_task[i] = NULL;
38 return sem;
39}
40
41static int open_fmlp_semaphore(struct od_table_entry* entry, void* __user arg)
42{
43 if (!fmlp_active())
44 return -EBUSY;
45 return 0;
46}
47
48static void destroy_fmlp_semaphore(void* sem)
49{
50 /* XXX assert invariants */
51 kfree(sem);
52}
53
54struct fdso_ops fmlp_sem_ops = {
55 .create = create_fmlp_semaphore,
56 .open = open_fmlp_semaphore,
57 .destroy = destroy_fmlp_semaphore
58};
59
60struct wq_pair {
61 struct task_struct* tsk;
62 struct pi_semaphore* sem;
63};
64
65static int rt_pi_wake_up(wait_queue_t *wait, unsigned mode, int sync,
66 void *key)
67{
68 struct wq_pair* wqp = (struct wq_pair*) wait->private;
69 set_rt_flags(wqp->tsk, RT_F_EXIT_SEM);
70 litmus->inherit_priority(wqp->sem, wqp->tsk);
71 TRACE_TASK(wqp->tsk,
72 "woken up by rt_pi_wake_up() (RT_F_SEM_EXIT, PI)\n");
73 /* point to task for default_wake_function() */
74 wait->private = wqp->tsk;
75 default_wake_function(wait, mode, sync, key);
76
77 /* Always return true since we know that if we encountered a task
78 * that was already running the wake_up raced with the schedule in
79 * rt_pi_down(). In that case the task in rt_pi_down() will be scheduled
80 * immediately and own the lock. We must not wake up another task in
81 * any case.
82 */
83 return 1;
84}
85
86/* caller is responsible for locking */
87int edf_set_hp_task(struct pi_semaphore *sem)
88{
89 struct list_head *tmp, *next;
90 struct task_struct *queued;
91 int ret = 0;
92
93 sem->hp.task = NULL;
94 list_for_each_safe(tmp, next, &sem->wait.task_list) {
95 queued = ((struct wq_pair*)
96 list_entry(tmp, wait_queue_t,
97 task_list)->private)->tsk;
98
99 /* Compare task prios, find high prio task. */
100 if (edf_higher_prio(queued, sem->hp.task)) {
101 sem->hp.task = queued;
102 ret = 1;
103 }
104 }
105 return ret;
106}
107
108/* caller is responsible for locking */
109int edf_set_hp_cpu_task(struct pi_semaphore *sem, int cpu)
110{
111 struct list_head *tmp, *next;
112 struct task_struct *queued;
113 int ret = 0;
114
115 sem->hp.cpu_task[cpu] = NULL;
116 list_for_each_safe(tmp, next, &sem->wait.task_list) {
117 queued = ((struct wq_pair*)
118 list_entry(tmp, wait_queue_t,
119 task_list)->private)->tsk;
120
121 /* Compare task prios, find high prio task. */
122 if (get_partition(queued) == cpu &&
123 edf_higher_prio(queued, sem->hp.cpu_task[cpu])) {
124 sem->hp.cpu_task[cpu] = queued;
125 ret = 1;
126 }
127 }
128 return ret;
129}
130
131static int do_fmlp_down(struct pi_semaphore* sem)
132{
133 unsigned long flags;
134 struct task_struct *tsk = current;
135 struct wq_pair pair;
136 int suspended = 1;
137 wait_queue_t wait = {
138 .private = &pair,
139 .func = rt_pi_wake_up,
140 .task_list = {NULL, NULL}
141 };
142
143 pair.tsk = tsk;
144 pair.sem = sem;
145 spin_lock_irqsave(&sem->wait.lock, flags);
146
147 if (atomic_dec_return(&sem->count) < 0 ||
148 waitqueue_active(&sem->wait)) {
149 /* we need to suspend */
150 tsk->state = TASK_UNINTERRUPTIBLE;
151 add_wait_queue_exclusive_locked(&sem->wait, &wait);
152
153 TRACE_CUR("suspends on PI lock %p\n", sem);
154 litmus->pi_block(sem, tsk);
155
156 /* release lock before sleeping */
157 spin_unlock_irqrestore(&sem->wait.lock, flags);
158
159 TS_PI_DOWN_END;
160 preempt_enable_no_resched();
161
162
163 /* we depend on the FIFO order
164 * Thus, we don't need to recheck when we wake up, we
165 * are guaranteed to have the lock since there is only one
166 * wake up per release
167 */
168 schedule();
169
170 TRACE_CUR("woke up, now owns PI lock %p\n", sem);
171
172 /* try_to_wake_up() set our state to TASK_RUNNING,
173 * all we need to do is to remove our wait queue entry
174 */
175 remove_wait_queue(&sem->wait, &wait);
176 } else {
177 /* no priority inheritance necessary, since there are no queued
178 * tasks.
179 */
180 suspended = 0;
181 TRACE_CUR("acquired PI lock %p, no contention\n", sem);
182 sem->holder = tsk;
183
184 /* don't know if we're global or partitioned. */
185 sem->hp.task = tsk;
186 sem->hp.cpu_task[get_partition(tsk)] = tsk;
187
188 litmus->inherit_priority(sem, tsk);
189 spin_unlock_irqrestore(&sem->wait.lock, flags);
190 }
191 return suspended;
192}
193
194static void do_fmlp_up(struct pi_semaphore* sem)
195{
196 unsigned long flags;
197
198 spin_lock_irqsave(&sem->wait.lock, flags);
199
200 TRACE_CUR("releases PI lock %p\n", sem);
201 litmus->return_priority(sem);
202 sem->holder = NULL;
203 if (atomic_inc_return(&sem->count) < 1)
204 /* there is a task queued */
205 wake_up_locked(&sem->wait);
206
207 spin_unlock_irqrestore(&sem->wait.lock, flags);
208}
209
210asmlinkage long sys_fmlp_down(int sem_od)
211{
212 long ret = 0;
213 struct pi_semaphore * sem;
214 int suspended = 0;
215
216 preempt_disable();
217 TS_PI_DOWN_START;
218
219 sem = lookup_fmlp_sem(sem_od);
220 if (sem)
221 suspended = do_fmlp_down(sem);
222 else
223 ret = -EINVAL;
224
225 if (!suspended) {
226 TS_PI_DOWN_END;
227 preempt_enable();
228 }
229
230 return ret;
231}
232
233asmlinkage long sys_fmlp_up(int sem_od)
234{
235 long ret = 0;
236 struct pi_semaphore * sem;
237
238 preempt_disable();
239 TS_PI_UP_START;
240
241 sem = lookup_fmlp_sem(sem_od);
242 if (sem)
243 do_fmlp_up(sem);
244 else
245 ret = -EINVAL;
246
247
248 TS_PI_UP_END;
249 preempt_enable();
250
251 return ret;
252}
253
254#else
255
256struct fdso_ops fmlp_sem_ops = {};
257
258asmlinkage long sys_fmlp_down(int sem_od)
259{
260 return -ENOSYS;
261}
262
263asmlinkage long sys_fmlp_up(int sem_od)
264{
265 return -ENOSYS;
266}
267
268#endif
diff --git a/litmus/ft_event.c b/litmus/ft_event.c
new file mode 100644
index 000000000000..6084b6d6b364
--- /dev/null
+++ b/litmus/ft_event.c
@@ -0,0 +1,43 @@
1#include <linux/types.h>
2
3#include <litmus/feather_trace.h>
4
5#ifndef __ARCH_HAS_FEATHER_TRACE
6/* provide dummy implementation */
7
8int ft_events[MAX_EVENTS];
9
10int ft_enable_event(unsigned long id)
11{
12 if (id < MAX_EVENTS) {
13 ft_events[id]++;
14 return 1;
15 } else
16 return 0;
17}
18
19int ft_disable_event(unsigned long id)
20{
21 if (id < MAX_EVENTS && ft_events[id]) {
22 ft_events[id]--;
23 return 1;
24 } else
25 return 0;
26}
27
28int ft_disable_all_events(void)
29{
30 int i;
31
32 for (i = 0; i < MAX_EVENTS; i++)
33 ft_events[i] = 0;
34
35 return MAX_EVENTS;
36}
37
38int ft_is_event_enabled(unsigned long id)
39{
40 return id < MAX_EVENTS && ft_events[id];
41}
42
43#endif
diff --git a/litmus/ftdev.c b/litmus/ftdev.c
new file mode 100644
index 000000000000..8b2d74d816a2
--- /dev/null
+++ b/litmus/ftdev.c
@@ -0,0 +1,359 @@
1#include <linux/sched.h>
2#include <linux/fs.h>
3#include <linux/cdev.h>
4#include <asm/uaccess.h>
5#include <linux/module.h>
6
7#include <litmus/litmus.h>
8#include <litmus/feather_trace.h>
9#include <litmus/ftdev.h>
10
11struct ft_buffer* alloc_ft_buffer(unsigned int count, size_t size)
12{
13 struct ft_buffer* buf;
14 size_t total = (size + 1) * count;
15 char* mem;
16 int order = 0, pages = 1;
17
18 buf = kmalloc(sizeof(*buf), GFP_KERNEL);
19 if (!buf)
20 return NULL;
21
22 total = (total / PAGE_SIZE) + (total % PAGE_SIZE != 0);
23 while (pages < total) {
24 order++;
25 pages *= 2;
26 }
27
28 mem = (char*) __get_free_pages(GFP_KERNEL, order);
29 if (!mem) {
30 kfree(buf);
31 return NULL;
32 }
33
34 if (!init_ft_buffer(buf, count, size,
35 mem + (count * size), /* markers at the end */
36 mem)) { /* buffer objects */
37 free_pages((unsigned long) mem, order);
38 kfree(buf);
39 return NULL;
40 }
41 return buf;
42}
43
44void free_ft_buffer(struct ft_buffer* buf)
45{
46 int order = 0, pages = 1;
47 size_t total;
48
49 if (buf) {
50 total = (buf->slot_size + 1) * buf->slot_count;
51 total = (total / PAGE_SIZE) + (total % PAGE_SIZE != 0);
52 while (pages < total) {
53 order++;
54 pages *= 2;
55 }
56 free_pages((unsigned long) buf->buffer_mem, order);
57 kfree(buf);
58 }
59}
60
61struct ftdev_event {
62 int id;
63 struct ftdev_event* next;
64};
65
66static int activate(struct ftdev_event** chain, int id)
67{
68 struct ftdev_event* ev = kmalloc(sizeof(*ev), GFP_KERNEL);
69 if (ev) {
70 printk(KERN_INFO
71 "Enabling feather-trace event %d.\n", (int) id);
72 ft_enable_event(id);
73 ev->id = id;
74 ev->next = *chain;
75 *chain = ev;
76 }
77 return ev ? 0 : -ENOMEM;
78}
79
80static void deactivate(struct ftdev_event** chain, int id)
81{
82 struct ftdev_event **cur = chain;
83 struct ftdev_event *nxt;
84 while (*cur) {
85 if ((*cur)->id == id) {
86 nxt = (*cur)->next;
87 kfree(*cur);
88 *cur = nxt;
89 printk(KERN_INFO
90 "Disabling feather-trace event %d.\n", (int) id);
91 ft_disable_event(id);
92 break;
93 }
94 cur = &(*cur)->next;
95 }
96}
97
98static int ftdev_open(struct inode *in, struct file *filp)
99{
100 struct ftdev* ftdev;
101 struct ftdev_minor* ftdm;
102 unsigned int buf_idx = iminor(in);
103 int err = 0;
104
105 ftdev = container_of(in->i_cdev, struct ftdev, cdev);
106
107 if (buf_idx >= ftdev->minor_cnt) {
108 err = -ENODEV;
109 goto out;
110 }
111 if (ftdev->can_open && (err = ftdev->can_open(ftdev, buf_idx)))
112 goto out;
113
114 ftdm = ftdev->minor + buf_idx;
115 filp->private_data = ftdm;
116
117 if (mutex_lock_interruptible(&ftdm->lock)) {
118 err = -ERESTARTSYS;
119 goto out;
120 }
121
122 if (!ftdm->readers && ftdev->alloc)
123 err = ftdev->alloc(ftdev, buf_idx);
124 if (0 == err)
125 ftdm->readers++;
126
127 mutex_unlock(&ftdm->lock);
128out:
129 return err;
130}
131
132static int ftdev_release(struct inode *in, struct file *filp)
133{
134 struct ftdev* ftdev;
135 struct ftdev_minor* ftdm;
136 unsigned int buf_idx = iminor(in);
137 int err = 0;
138
139 ftdev = container_of(in->i_cdev, struct ftdev, cdev);
140
141 if (buf_idx >= ftdev->minor_cnt) {
142 err = -ENODEV;
143 goto out;
144 }
145 ftdm = ftdev->minor + buf_idx;
146
147 if (mutex_lock_interruptible(&ftdm->lock)) {
148 err = -ERESTARTSYS;
149 goto out;
150 }
151
152 if (ftdm->readers == 1) {
153 while (ftdm->events)
154 deactivate(&ftdm->events, ftdm->events->id);
155
156 /* wait for any pending events to complete */
157 set_current_state(TASK_UNINTERRUPTIBLE);
158 schedule_timeout(HZ);
159
160 printk(KERN_ALERT "Failed trace writes: %u\n",
161 ftdm->buf->failed_writes);
162
163 if (ftdev->free)
164 ftdev->free(ftdev, buf_idx);
165 }
166
167 ftdm->readers--;
168 mutex_unlock(&ftdm->lock);
169out:
170 return err;
171}
172
173/* based on ft_buffer_read
174 * @returns < 0 : page fault
175 * = 0 : no data available
176 * = 1 : one slot copied
177 */
178static int ft_buffer_copy_to_user(struct ft_buffer* buf, char __user *dest)
179{
180 unsigned int idx;
181 int err = 0;
182 if (buf->free_count != buf->slot_count) {
183 /* data available */
184 idx = buf->read_idx % buf->slot_count;
185 if (buf->slots[idx] == SLOT_READY) {
186 err = copy_to_user(dest, ((char*) buf->buffer_mem) +
187 idx * buf->slot_size,
188 buf->slot_size);
189 if (err == 0) {
190 /* copy ok */
191 buf->slots[idx] = SLOT_FREE;
192 buf->read_idx++;
193 fetch_and_inc(&buf->free_count);
194 err = 1;
195 }
196 }
197 }
198 return err;
199}
200
201static ssize_t ftdev_read(struct file *filp,
202 char __user *to, size_t len, loff_t *f_pos)
203{
204 /* we ignore f_pos, this is strictly sequential */
205
206 ssize_t err = 0;
207 size_t chunk;
208 int copied;
209 struct ftdev_minor* ftdm = filp->private_data;
210
211 if (mutex_lock_interruptible(&ftdm->lock)) {
212 err = -ERESTARTSYS;
213 goto out;
214 }
215
216
217 chunk = ftdm->buf->slot_size;
218 while (len >= chunk) {
219 copied = ft_buffer_copy_to_user(ftdm->buf, to);
220 if (copied == 1) {
221 len -= chunk;
222 to += chunk;
223 err += chunk;
224 } else if (err == 0 && copied == 0 && ftdm->events) {
225 /* Only wait if there are any events enabled and only
226 * if we haven't copied some data yet. We cannot wait
227 * here with copied data because that data would get
228 * lost if the task is interrupted (e.g., killed).
229 */
230 set_current_state(TASK_INTERRUPTIBLE);
231 schedule_timeout(50);
232 if (signal_pending(current)) {
233 if (err == 0)
234 /* nothing read yet, signal problem */
235 err = -ERESTARTSYS;
236 break;
237 }
238 } else if (copied < 0) {
239 /* page fault */
240 err = copied;
241 break;
242 } else
243 /* nothing left to get, return to user space */
244 break;
245 }
246 mutex_unlock(&ftdm->lock);
247out:
248 return err;
249}
250
251typedef uint32_t cmd_t;
252
253static ssize_t ftdev_write(struct file *filp, const char __user *from,
254 size_t len, loff_t *f_pos)
255{
256 struct ftdev_minor* ftdm = filp->private_data;
257 ssize_t err = -EINVAL;
258 cmd_t cmd;
259 cmd_t id;
260
261 if (len % sizeof(cmd) || len < 2 * sizeof(cmd))
262 goto out;
263
264 if (copy_from_user(&cmd, from, sizeof(cmd))) {
265 err = -EFAULT;
266 goto out;
267 }
268 len -= sizeof(cmd);
269 from += sizeof(cmd);
270
271 if (cmd != FTDEV_ENABLE_CMD && cmd != FTDEV_DISABLE_CMD)
272 goto out;
273
274 if (mutex_lock_interruptible(&ftdm->lock)) {
275 err = -ERESTARTSYS;
276 goto out;
277 }
278
279 err = sizeof(cmd);
280 while (len) {
281 if (copy_from_user(&id, from, sizeof(cmd))) {
282 err = -EFAULT;
283 goto out_unlock;
284 }
285 /* FIXME: check id against list of acceptable events */
286 len -= sizeof(cmd);
287 from += sizeof(cmd);
288 if (cmd == FTDEV_DISABLE_CMD)
289 deactivate(&ftdm->events, id);
290 else if (activate(&ftdm->events, id) != 0) {
291 err = -ENOMEM;
292 goto out_unlock;
293 }
294 err += sizeof(cmd);
295 }
296
297out_unlock:
298 mutex_unlock(&ftdm->lock);
299out:
300 return err;
301}
302
303struct file_operations ftdev_fops = {
304 .owner = THIS_MODULE,
305 .open = ftdev_open,
306 .release = ftdev_release,
307 .write = ftdev_write,
308 .read = ftdev_read,
309};
310
311
312void ftdev_init(struct ftdev* ftdev, struct module* owner)
313{
314 int i;
315 cdev_init(&ftdev->cdev, &ftdev_fops);
316 ftdev->cdev.owner = owner;
317 ftdev->cdev.ops = &ftdev_fops;
318 ftdev->minor_cnt = 0;
319 for (i = 0; i < MAX_FTDEV_MINORS; i++) {
320 mutex_init(&ftdev->minor[i].lock);
321 ftdev->minor[i].readers = 0;
322 ftdev->minor[i].buf = NULL;
323 ftdev->minor[i].events = NULL;
324 }
325 ftdev->alloc = NULL;
326 ftdev->free = NULL;
327 ftdev->can_open = NULL;
328}
329
330int register_ftdev(struct ftdev* ftdev, const char* name, int major)
331{
332 dev_t trace_dev;
333 int error = 0;
334
335 if(major) {
336 trace_dev = MKDEV(major, 0);
337 error = register_chrdev_region(trace_dev, ftdev->minor_cnt,
338 name);
339 } else {
340 error = alloc_chrdev_region(&trace_dev, 0, ftdev->minor_cnt,
341 name);
342 major = MAJOR(trace_dev);
343 }
344 if (error)
345 {
346 printk(KERN_WARNING "ftdev(%s): "
347 "Could not register major/minor number %d/%u\n",
348 name, major, ftdev->minor_cnt);
349 return error;
350 }
351 error = cdev_add(&ftdev->cdev, trace_dev, ftdev->minor_cnt);
352 if (error) {
353 printk(KERN_WARNING "ftdev(%s): "
354 "Could not add cdev for major/minor = %d/%u.\n",
355 name, major, ftdev->minor_cnt);
356 return error;
357 }
358 return error;
359}
diff --git a/litmus/jobs.c b/litmus/jobs.c
new file mode 100644
index 000000000000..36e314625d86
--- /dev/null
+++ b/litmus/jobs.c
@@ -0,0 +1,43 @@
1/* litmus/jobs.c - common job control code
2 */
3
4#include <linux/sched.h>
5
6#include <litmus/litmus.h>
7#include <litmus/jobs.h>
8
9void prepare_for_next_period(struct task_struct *t)
10{
11 BUG_ON(!t);
12 /* prepare next release */
13 t->rt_param.job_params.release = t->rt_param.job_params.deadline;
14 t->rt_param.job_params.deadline += get_rt_period(t);
15 t->rt_param.job_params.exec_time = 0;
16 /* update job sequence number */
17 t->rt_param.job_params.job_no++;
18
19 /* don't confuse Linux */
20 t->rt.time_slice = 1;
21}
22
23void release_at(struct task_struct *t, lt_t start)
24{
25 t->rt_param.job_params.deadline = start;
26 prepare_for_next_period(t);
27 set_rt_flags(t, RT_F_RUNNING);
28}
29
30
31/*
32 * Deactivate current task until the beginning of the next period.
33 */
34long complete_job(void)
35{
36 /* Mark that we do not excute anymore */
37 set_rt_flags(current, RT_F_SLEEP);
38 /* call schedule, this will return when a new job arrives
39 * it also takes care of preparing for the next release
40 */
41 schedule();
42 return 0;
43}
diff --git a/litmus/litmus.c b/litmus/litmus.c
new file mode 100644
index 000000000000..e43596a5104c
--- /dev/null
+++ b/litmus/litmus.c
@@ -0,0 +1,775 @@
1/*
2 * litmus.c -- Implementation of the LITMUS syscalls,
3 * the LITMUS intialization code,
4 * and the procfs interface..
5 */
6#include <asm/uaccess.h>
7#include <linux/uaccess.h>
8#include <linux/sysrq.h>
9
10#include <linux/module.h>
11#include <linux/proc_fs.h>
12#include <linux/slab.h>
13
14#include <litmus/litmus.h>
15#include <linux/sched.h>
16#include <litmus/sched_plugin.h>
17
18#include <litmus/bheap.h>
19
20#include <litmus/trace.h>
21
22#include <litmus/rt_domain.h>
23
24/* Number of RT tasks that exist in the system */
25atomic_t rt_task_count = ATOMIC_INIT(0);
26static DEFINE_SPINLOCK(task_transition_lock);
27/* synchronize plugin switching */
28atomic_t cannot_use_plugin = ATOMIC_INIT(0);
29
30/* Give log messages sequential IDs. */
31atomic_t __log_seq_no = ATOMIC_INIT(0);
32
33/* current master CPU for handling timer IRQs */
34atomic_t release_master_cpu = ATOMIC_INIT(NO_CPU);
35
36static struct kmem_cache * bheap_node_cache;
37extern struct kmem_cache * release_heap_cache;
38
39struct bheap_node* bheap_node_alloc(int gfp_flags)
40{
41 return kmem_cache_alloc(bheap_node_cache, gfp_flags);
42}
43
44void bheap_node_free(struct bheap_node* hn)
45{
46 kmem_cache_free(bheap_node_cache, hn);
47}
48
49struct release_heap* release_heap_alloc(int gfp_flags);
50void release_heap_free(struct release_heap* rh);
51
52/*
53 * sys_set_task_rt_param
54 * @pid: Pid of the task which scheduling parameters must be changed
55 * @param: New real-time extension parameters such as the execution cost and
56 * period
57 * Syscall for manipulating with task rt extension params
58 * Returns EFAULT if param is NULL.
59 * ESRCH if pid is not corrsponding
60 * to a valid task.
61 * EINVAL if either period or execution cost is <=0
62 * EPERM if pid is a real-time task
63 * 0 if success
64 *
65 * Only non-real-time tasks may be configured with this system call
66 * to avoid races with the scheduler. In practice, this means that a
67 * task's parameters must be set _before_ calling sys_prepare_rt_task()
68 *
69 * find_task_by_vpid() assumes that we are in the same namespace of the
70 * target.
71 */
72asmlinkage long sys_set_rt_task_param(pid_t pid, struct rt_task __user * param)
73{
74 struct rt_task tp;
75 struct task_struct *target;
76 int retval = -EINVAL;
77
78 printk("Setting up rt task parameters for process %d.\n", pid);
79
80 if (pid < 0 || param == 0) {
81 goto out;
82 }
83 if (copy_from_user(&tp, param, sizeof(tp))) {
84 retval = -EFAULT;
85 goto out;
86 }
87
88 /* Task search and manipulation must be protected */
89 read_lock_irq(&tasklist_lock);
90 if (!(target = find_task_by_vpid(pid))) {
91 retval = -ESRCH;
92 goto out_unlock;
93 }
94
95 if (is_realtime(target)) {
96 /* The task is already a real-time task.
97 * We cannot not allow parameter changes at this point.
98 */
99 retval = -EBUSY;
100 goto out_unlock;
101 }
102
103 if (tp.exec_cost <= 0)
104 goto out_unlock;
105 if (tp.period <= 0)
106 goto out_unlock;
107 if (!cpu_online(tp.cpu))
108 goto out_unlock;
109 if (tp.period < tp.exec_cost)
110 {
111 printk(KERN_INFO "litmus: real-time task %d rejected "
112 "because wcet > period\n", pid);
113 goto out_unlock;
114 }
115
116 target->rt_param.task_params = tp;
117
118 retval = 0;
119 out_unlock:
120 read_unlock_irq(&tasklist_lock);
121 out:
122 return retval;
123}
124
125/*
126 * Getter of task's RT params
127 * returns EINVAL if param or pid is NULL
128 * returns ESRCH if pid does not correspond to a valid task
129 * returns EFAULT if copying of parameters has failed.
130 *
131 * find_task_by_vpid() assumes that we are in the same namespace of the
132 * target.
133 */
134asmlinkage long sys_get_rt_task_param(pid_t pid, struct rt_task __user * param)
135{
136 int retval = -EINVAL;
137 struct task_struct *source;
138 struct rt_task lp;
139 if (param == 0 || pid < 0)
140 goto out;
141 read_lock(&tasklist_lock);
142 if (!(source = find_task_by_vpid(pid))) {
143 retval = -ESRCH;
144 goto out_unlock;
145 }
146 lp = source->rt_param.task_params;
147 read_unlock(&tasklist_lock);
148 /* Do copying outside the lock */
149 retval =
150 copy_to_user(param, &lp, sizeof(lp)) ? -EFAULT : 0;
151 return retval;
152 out_unlock:
153 read_unlock(&tasklist_lock);
154 out:
155 return retval;
156
157}
158
159/*
160 * This is the crucial function for periodic task implementation,
161 * It checks if a task is periodic, checks if such kind of sleep
162 * is permitted and calls plugin-specific sleep, which puts the
163 * task into a wait array.
164 * returns 0 on successful wakeup
165 * returns EPERM if current conditions do not permit such sleep
166 * returns EINVAL if current task is not able to go to sleep
167 */
168asmlinkage long sys_complete_job(void)
169{
170 int retval = -EPERM;
171 if (!is_realtime(current)) {
172 retval = -EINVAL;
173 goto out;
174 }
175 /* Task with negative or zero period cannot sleep */
176 if (get_rt_period(current) <= 0) {
177 retval = -EINVAL;
178 goto out;
179 }
180 /* The plugin has to put the task into an
181 * appropriate queue and call schedule
182 */
183 retval = litmus->complete_job();
184 out:
185 return retval;
186}
187
188/* This is an "improved" version of sys_complete_job that
189 * addresses the problem of unintentionally missing a job after
190 * an overrun.
191 *
192 * returns 0 on successful wakeup
193 * returns EPERM if current conditions do not permit such sleep
194 * returns EINVAL if current task is not able to go to sleep
195 */
196asmlinkage long sys_wait_for_job_release(unsigned int job)
197{
198 int retval = -EPERM;
199 if (!is_realtime(current)) {
200 retval = -EINVAL;
201 goto out;
202 }
203
204 /* Task with negative or zero period cannot sleep */
205 if (get_rt_period(current) <= 0) {
206 retval = -EINVAL;
207 goto out;
208 }
209
210 retval = 0;
211
212 /* first wait until we have "reached" the desired job
213 *
214 * This implementation has at least two problems:
215 *
216 * 1) It doesn't gracefully handle the wrap around of
217 * job_no. Since LITMUS is a prototype, this is not much
218 * of a problem right now.
219 *
220 * 2) It is theoretically racy if a job release occurs
221 * between checking job_no and calling sleep_next_period().
222 * A proper solution would requiring adding another callback
223 * in the plugin structure and testing the condition with
224 * interrupts disabled.
225 *
226 * FIXME: At least problem 2 should be taken care of eventually.
227 */
228 while (!retval && job > current->rt_param.job_params.job_no)
229 /* If the last job overran then job <= job_no and we
230 * don't send the task to sleep.
231 */
232 retval = litmus->complete_job();
233 out:
234 return retval;
235}
236
237/* This is a helper syscall to query the current job sequence number.
238 *
239 * returns 0 on successful query
240 * returns EPERM if task is not a real-time task.
241 * returns EFAULT if &job is not a valid pointer.
242 */
243asmlinkage long sys_query_job_no(unsigned int __user *job)
244{
245 int retval = -EPERM;
246 if (is_realtime(current))
247 retval = put_user(current->rt_param.job_params.job_no, job);
248
249 return retval;
250}
251
252/* sys_null_call() is only used for determining raw system call
253 * overheads (kernel entry, kernel exit). It has no useful side effects.
254 * If ts is non-NULL, then the current Feather-Trace time is recorded.
255 */
256asmlinkage long sys_null_call(cycles_t __user *ts)
257{
258 long ret = 0;
259 cycles_t now;
260
261 if (ts) {
262 now = get_cycles();
263 ret = put_user(now, ts);
264 }
265
266 return ret;
267}
268
269/* p is a real-time task. Re-init its state as a best-effort task. */
270static void reinit_litmus_state(struct task_struct* p, int restore)
271{
272 struct rt_task user_config = {};
273 void* ctrl_page = NULL;
274
275 if (restore) {
276 /* Safe user-space provided configuration data.
277 * and allocated page. */
278 user_config = p->rt_param.task_params;
279 ctrl_page = p->rt_param.ctrl_page;
280 }
281
282 /* We probably should not be inheriting any task's priority
283 * at this point in time.
284 */
285 WARN_ON(p->rt_param.inh_task);
286
287 /* We need to restore the priority of the task. */
288// __setscheduler(p, p->rt_param.old_policy, p->rt_param.old_prio); XXX why is this commented?
289
290 /* Cleanup everything else. */
291 memset(&p->rt_param, 0, sizeof(p->rt_param));
292
293 /* Restore preserved fields. */
294 if (restore) {
295 p->rt_param.task_params = user_config;
296 p->rt_param.ctrl_page = ctrl_page;
297 }
298}
299
300long litmus_admit_task(struct task_struct* tsk)
301{
302 long retval = 0;
303 unsigned long flags;
304
305 BUG_ON(is_realtime(tsk));
306
307 if (get_rt_period(tsk) == 0 ||
308 get_exec_cost(tsk) > get_rt_period(tsk)) {
309 TRACE_TASK(tsk, "litmus admit: invalid task parameters "
310 "(%lu, %lu)\n",
311 get_exec_cost(tsk), get_rt_period(tsk));
312 retval = -EINVAL;
313 goto out;
314 }
315
316 if (!cpu_online(get_partition(tsk))) {
317 TRACE_TASK(tsk, "litmus admit: cpu %d is not online\n",
318 get_partition(tsk));
319 retval = -EINVAL;
320 goto out;
321 }
322
323 INIT_LIST_HEAD(&tsk_rt(tsk)->list);
324
325 /* avoid scheduler plugin changing underneath us */
326 spin_lock_irqsave(&task_transition_lock, flags);
327
328 /* allocate heap node for this task */
329 tsk_rt(tsk)->heap_node = bheap_node_alloc(GFP_ATOMIC);
330 tsk_rt(tsk)->rel_heap = release_heap_alloc(GFP_ATOMIC);
331
332 if (!tsk_rt(tsk)->heap_node || !tsk_rt(tsk)->rel_heap) {
333 printk(KERN_WARNING "litmus: no more heap node memory!?\n");
334
335 bheap_node_free(tsk_rt(tsk)->heap_node);
336 release_heap_free(tsk_rt(tsk)->rel_heap);
337
338 retval = -ENOMEM;
339 goto out_unlock;
340 } else {
341 bheap_node_init(&tsk_rt(tsk)->heap_node, tsk);
342 }
343
344 retval = litmus->admit_task(tsk);
345
346 if (!retval) {
347 sched_trace_task_name(tsk);
348 sched_trace_task_param(tsk);
349 atomic_inc(&rt_task_count);
350 }
351
352out_unlock:
353 spin_unlock_irqrestore(&task_transition_lock, flags);
354out:
355 return retval;
356}
357
358void litmus_exit_task(struct task_struct* tsk)
359{
360 if (is_realtime(tsk)) {
361 sched_trace_task_completion(tsk, 1);
362
363 litmus->task_exit(tsk);
364
365 BUG_ON(bheap_node_in_heap(tsk_rt(tsk)->heap_node));
366 bheap_node_free(tsk_rt(tsk)->heap_node);
367 release_heap_free(tsk_rt(tsk)->rel_heap);
368
369 atomic_dec(&rt_task_count);
370 reinit_litmus_state(tsk, 1);
371 }
372}
373
374/* IPI callback to synchronize plugin switching */
375static void synch_on_plugin_switch(void* info)
376{
377 while (atomic_read(&cannot_use_plugin))
378 cpu_relax();
379}
380
381/* Switching a plugin in use is tricky.
382 * We must watch out that no real-time tasks exists
383 * (and that none is created in parallel) and that the plugin is not
384 * currently in use on any processor (in theory).
385 */
386int switch_sched_plugin(struct sched_plugin* plugin)
387{
388 unsigned long flags;
389 int ret = 0;
390
391 BUG_ON(!plugin);
392
393 /* forbid other cpus to use the plugin */
394 atomic_set(&cannot_use_plugin, 1);
395 /* send IPI to force other CPUs to synch with us */
396 smp_call_function(synch_on_plugin_switch, NULL, 0);
397
398 /* stop task transitions */
399 spin_lock_irqsave(&task_transition_lock, flags);
400
401 /* don't switch if there are active real-time tasks */
402 if (atomic_read(&rt_task_count) == 0) {
403 ret = litmus->deactivate_plugin();
404 if (0 != ret)
405 goto out;
406 ret = plugin->activate_plugin();
407 if (0 != ret) {
408 printk(KERN_INFO "Can't activate %s (%d).\n",
409 plugin->plugin_name, ret);
410 plugin = &linux_sched_plugin;
411 }
412 printk(KERN_INFO "Switching to LITMUS^RT plugin %s.\n", plugin->plugin_name);
413 litmus = plugin;
414 } else
415 ret = -EBUSY;
416out:
417 spin_unlock_irqrestore(&task_transition_lock, flags);
418 atomic_set(&cannot_use_plugin, 0);
419 return ret;
420}
421
422/* Called upon fork.
423 * p is the newly forked task.
424 */
425void litmus_fork(struct task_struct* p)
426{
427 if (is_realtime(p))
428 /* clean out any litmus related state, don't preserve anything */
429 reinit_litmus_state(p, 0);
430 else
431 /* non-rt tasks might have ctrl_page set */
432 tsk_rt(p)->ctrl_page = NULL;
433
434 /* od tables are never inherited across a fork */
435 p->od_table = NULL;
436}
437
438/* Called upon execve().
439 * current is doing the exec.
440 * Don't let address space specific stuff leak.
441 */
442void litmus_exec(void)
443{
444 struct task_struct* p = current;
445
446 if (is_realtime(p)) {
447 WARN_ON(p->rt_param.inh_task);
448 if (tsk_rt(p)->ctrl_page) {
449 free_page((unsigned long) tsk_rt(p)->ctrl_page);
450 tsk_rt(p)->ctrl_page = NULL;
451 }
452 }
453}
454
455void exit_litmus(struct task_struct *dead_tsk)
456{
457 /* We also allow non-RT tasks to
458 * allocate control pages to allow
459 * measurements with non-RT tasks.
460 * So check if we need to free the page
461 * in any case.
462 */
463 if (tsk_rt(dead_tsk)->ctrl_page) {
464 TRACE_TASK(dead_tsk,
465 "freeing ctrl_page %p\n",
466 tsk_rt(dead_tsk)->ctrl_page);
467 free_page((unsigned long) tsk_rt(dead_tsk)->ctrl_page);
468 }
469
470 /* main cleanup only for RT tasks */
471 if (is_realtime(dead_tsk))
472 litmus_exit_task(dead_tsk);
473}
474
475
476#ifdef CONFIG_MAGIC_SYSRQ
477int sys_kill(int pid, int sig);
478
479static void sysrq_handle_kill_rt_tasks(int key, struct tty_struct *tty)
480{
481 struct task_struct *t;
482 read_lock(&tasklist_lock);
483 for_each_process(t) {
484 if (is_realtime(t)) {
485 sys_kill(t->pid, SIGKILL);
486 }
487 }
488 read_unlock(&tasklist_lock);
489}
490
491static struct sysrq_key_op sysrq_kill_rt_tasks_op = {
492 .handler = sysrq_handle_kill_rt_tasks,
493 .help_msg = "quit-rt-tasks(X)",
494 .action_msg = "sent SIGKILL to all LITMUS^RT real-time tasks",
495};
496#endif
497
498/* in litmus/sync.c */
499int count_tasks_waiting_for_release(void);
500
501static int proc_read_stats(char *page, char **start,
502 off_t off, int count,
503 int *eof, void *data)
504{
505 int len;
506
507 len = snprintf(page, PAGE_SIZE,
508 "real-time tasks = %d\n"
509 "ready for release = %d\n",
510 atomic_read(&rt_task_count),
511 count_tasks_waiting_for_release());
512 return len;
513}
514
515static int proc_read_plugins(char *page, char **start,
516 off_t off, int count,
517 int *eof, void *data)
518{
519 int len;
520
521 len = print_sched_plugins(page, PAGE_SIZE);
522 return len;
523}
524
525static int proc_read_curr(char *page, char **start,
526 off_t off, int count,
527 int *eof, void *data)
528{
529 int len;
530
531 len = snprintf(page, PAGE_SIZE, "%s\n", litmus->plugin_name);
532 return len;
533}
534
535static int proc_write_curr(struct file *file,
536 const char *buffer,
537 unsigned long count,
538 void *data)
539{
540 int len, ret;
541 char name[65];
542 struct sched_plugin* found;
543
544 if(count > 64)
545 len = 64;
546 else
547 len = count;
548
549 if(copy_from_user(name, buffer, len))
550 return -EFAULT;
551
552 name[len] = '\0';
553 /* chomp name */
554 if (len > 1 && name[len - 1] == '\n')
555 name[len - 1] = '\0';
556
557 found = find_sched_plugin(name);
558
559 if (found) {
560 ret = switch_sched_plugin(found);
561 if (ret != 0)
562 printk(KERN_INFO "Could not switch plugin: %d\n", ret);
563 } else
564 printk(KERN_INFO "Plugin '%s' is unknown.\n", name);
565
566 return len;
567}
568
569static int proc_read_cluster_size(char *page, char **start,
570 off_t off, int count,
571 int *eof, void *data)
572{
573 int len;
574 if (cluster_cache_index == 2)
575 len = snprintf(page, PAGE_SIZE, "L2\n");
576 else if (cluster_cache_index == 3)
577 len = snprintf(page, PAGE_SIZE, "L3\n");
578 else /* (cluster_cache_index == 1) */
579 len = snprintf(page, PAGE_SIZE, "L1\n");
580
581 return len;
582}
583
584static int proc_write_cluster_size(struct file *file,
585 const char *buffer,
586 unsigned long count,
587 void *data)
588{
589 int len;
590 /* L2, L3 */
591 char cache_name[33];
592
593 if(count > 32)
594 len = 32;
595 else
596 len = count;
597
598 if(copy_from_user(cache_name, buffer, len))
599 return -EFAULT;
600
601 cache_name[len] = '\0';
602 /* chomp name */
603 if (len > 1 && cache_name[len - 1] == '\n')
604 cache_name[len - 1] = '\0';
605
606 /* do a quick and dirty comparison to find the cluster size */
607 if (!strcmp(cache_name, "L2"))
608 cluster_cache_index = 2;
609 else if (!strcmp(cache_name, "L3"))
610 cluster_cache_index = 3;
611 else if (!strcmp(cache_name, "L1"))
612 cluster_cache_index = 1;
613 else
614 printk(KERN_INFO "Cluster '%s' is unknown.\n", cache_name);
615
616 return len;
617}
618
619static int proc_read_release_master(char *page, char **start,
620 off_t off, int count,
621 int *eof, void *data)
622{
623 int len, master;
624 master = atomic_read(&release_master_cpu);
625 if (master == NO_CPU)
626 len = snprintf(page, PAGE_SIZE, "NO_CPU\n");
627 else
628 len = snprintf(page, PAGE_SIZE, "%d\n", master);
629 return len;
630}
631
632static int proc_write_release_master(struct file *file,
633 const char *buffer,
634 unsigned long count,
635 void *data)
636{
637 int cpu, err, online = 0;
638 char msg[64];
639
640 if (count > 63)
641 return -EINVAL;
642
643 if (copy_from_user(msg, buffer, count))
644 return -EFAULT;
645
646 /* terminate */
647 msg[count] = '\0';
648 /* chomp */
649 if (count > 1 && msg[count - 1] == '\n')
650 msg[count - 1] = '\0';
651
652 if (strcmp(msg, "NO_CPU") == 0) {
653 atomic_set(&release_master_cpu, NO_CPU);
654 return count;
655 } else {
656 err = sscanf(msg, "%d", &cpu);
657 if (err == 1 && cpu >= 0 && (online = cpu_online(cpu))) {
658 atomic_set(&release_master_cpu, cpu);
659 return count;
660 } else {
661 TRACE("invalid release master: '%s' "
662 "(err:%d cpu:%d online:%d)\n",
663 msg, err, cpu, online);
664 return -EINVAL;
665 }
666 }
667}
668
669static struct proc_dir_entry *litmus_dir = NULL,
670 *curr_file = NULL,
671 *stat_file = NULL,
672 *plugs_file = NULL,
673 *clus_cache_idx_file = NULL,
674 *release_master_file = NULL;
675
676static int __init init_litmus_proc(void)
677{
678 litmus_dir = proc_mkdir("litmus", NULL);
679 if (!litmus_dir) {
680 printk(KERN_ERR "Could not allocate LITMUS^RT procfs entry.\n");
681 return -ENOMEM;
682 }
683
684 curr_file = create_proc_entry("active_plugin",
685 0644, litmus_dir);
686 if (!curr_file) {
687 printk(KERN_ERR "Could not allocate active_plugin "
688 "procfs entry.\n");
689 return -ENOMEM;
690 }
691 curr_file->read_proc = proc_read_curr;
692 curr_file->write_proc = proc_write_curr;
693
694 release_master_file = create_proc_entry("release_master",
695 0644, litmus_dir);
696 if (!release_master_file) {
697 printk(KERN_ERR "Could not allocate release_master "
698 "procfs entry.\n");
699 return -ENOMEM;
700 }
701 release_master_file->read_proc = proc_read_release_master;
702 release_master_file->write_proc = proc_write_release_master;
703
704 clus_cache_idx_file = create_proc_entry("cluster_cache",
705 0644, litmus_dir);
706 if (!clus_cache_idx_file) {
707 printk(KERN_ERR "Could not allocate cluster_cache "
708 "procfs entry.\n");
709 return -ENOMEM;
710 }
711 clus_cache_idx_file->read_proc = proc_read_cluster_size;
712 clus_cache_idx_file->write_proc = proc_write_cluster_size;
713
714 stat_file = create_proc_read_entry("stats", 0444, litmus_dir,
715 proc_read_stats, NULL);
716
717 plugs_file = create_proc_read_entry("plugins", 0444, litmus_dir,
718 proc_read_plugins, NULL);
719
720 return 0;
721}
722
723static void exit_litmus_proc(void)
724{
725 if (plugs_file)
726 remove_proc_entry("plugins", litmus_dir);
727 if (stat_file)
728 remove_proc_entry("stats", litmus_dir);
729 if (curr_file)
730 remove_proc_entry("active_plugin", litmus_dir);
731 if (clus_cache_idx_file)
732 remove_proc_entry("cluster_cache", litmus_dir);
733 if (release_master_file)
734 remove_proc_entry("release_master", litmus_dir);
735 if (litmus_dir)
736 remove_proc_entry("litmus", NULL);
737}
738
739extern struct sched_plugin linux_sched_plugin;
740
741static int __init _init_litmus(void)
742{
743 /* Common initializers,
744 * mode change lock is used to enforce single mode change
745 * operation.
746 */
747 printk("Starting LITMUS^RT kernel\n");
748
749 register_sched_plugin(&linux_sched_plugin);
750
751 bheap_node_cache = KMEM_CACHE(bheap_node, SLAB_PANIC);
752 release_heap_cache = KMEM_CACHE(release_heap, SLAB_PANIC);
753
754#ifdef CONFIG_MAGIC_SYSRQ
755 /* offer some debugging help */
756 if (!register_sysrq_key('x', &sysrq_kill_rt_tasks_op))
757 printk("Registered kill rt tasks magic sysrq.\n");
758 else
759 printk("Could not register kill rt tasks magic sysrq.\n");
760#endif
761
762 init_litmus_proc();
763
764 return 0;
765}
766
767static void _exit_litmus(void)
768{
769 exit_litmus_proc();
770 kmem_cache_destroy(bheap_node_cache);
771 kmem_cache_destroy(release_heap_cache);
772}
773
774module_init(_init_litmus);
775module_exit(_exit_litmus);
diff --git a/litmus/rt_domain.c b/litmus/rt_domain.c
new file mode 100644
index 000000000000..609ff0f82abb
--- /dev/null
+++ b/litmus/rt_domain.c
@@ -0,0 +1,310 @@
1/*
2 * litmus/rt_domain.c
3 *
4 * LITMUS real-time infrastructure. This file contains the
5 * functions that manipulate RT domains. RT domains are an abstraction
6 * of a ready queue and a release queue.
7 */
8
9#include <linux/percpu.h>
10#include <linux/sched.h>
11#include <linux/list.h>
12#include <linux/slab.h>
13
14#include <litmus/litmus.h>
15#include <litmus/sched_plugin.h>
16#include <litmus/sched_trace.h>
17
18#include <litmus/rt_domain.h>
19
20#include <litmus/trace.h>
21
22#include <litmus/bheap.h>
23
24static int dummy_resched(rt_domain_t *rt)
25{
26 return 0;
27}
28
29static int dummy_order(struct bheap_node* a, struct bheap_node* b)
30{
31 return 0;
32}
33
34/* default implementation: use default lock */
35static void default_release_jobs(rt_domain_t* rt, struct bheap* tasks)
36{
37 merge_ready(rt, tasks);
38}
39
40static unsigned int time2slot(lt_t time)
41{
42 return (unsigned int) time2quanta(time, FLOOR) % RELEASE_QUEUE_SLOTS;
43}
44
45static enum hrtimer_restart on_release_timer(struct hrtimer *timer)
46{
47 unsigned long flags;
48 struct release_heap* rh;
49
50 TRACE("on_release_timer(0x%p) starts.\n", timer);
51
52 TS_RELEASE_START;
53
54 rh = container_of(timer, struct release_heap, timer);
55
56 spin_lock_irqsave(&rh->dom->release_lock, flags);
57 TRACE("CB has the release_lock 0x%p\n", &rh->dom->release_lock);
58 /* remove from release queue */
59 list_del(&rh->list);
60 spin_unlock_irqrestore(&rh->dom->release_lock, flags);
61 TRACE("CB returned release_lock 0x%p\n", &rh->dom->release_lock);
62
63 /* call release callback */
64 rh->dom->release_jobs(rh->dom, &rh->heap);
65 /* WARNING: rh can be referenced from other CPUs from now on. */
66
67 TS_RELEASE_END;
68
69 TRACE("on_release_timer(0x%p) ends.\n", timer);
70
71 return HRTIMER_NORESTART;
72}
73
74/* allocated in litmus.c */
75struct kmem_cache * release_heap_cache;
76
77struct release_heap* release_heap_alloc(int gfp_flags)
78{
79 struct release_heap* rh;
80 rh= kmem_cache_alloc(release_heap_cache, gfp_flags);
81 if (rh) {
82 /* initialize timer */
83 hrtimer_init(&rh->timer, CLOCK_MONOTONIC, HRTIMER_MODE_ABS);
84 rh->timer.function = on_release_timer;
85 }
86 return rh;
87}
88
89void release_heap_free(struct release_heap* rh)
90{
91 /* make sure timer is no longer in use */
92 hrtimer_cancel(&rh->timer);
93 kmem_cache_free(release_heap_cache, rh);
94}
95
96/* Caller must hold release lock.
97 * Will return heap for given time. If no such heap exists prior to
98 * the invocation it will be created.
99 */
100static struct release_heap* get_release_heap(rt_domain_t *rt,
101 struct task_struct* t,
102 int use_task_heap)
103{
104 struct list_head* pos;
105 struct release_heap* heap = NULL;
106 struct release_heap* rh;
107 lt_t release_time = get_release(t);
108 unsigned int slot = time2slot(release_time);
109
110 /* initialize pos for the case that the list is empty */
111 pos = rt->release_queue.slot[slot].next;
112 list_for_each(pos, &rt->release_queue.slot[slot]) {
113 rh = list_entry(pos, struct release_heap, list);
114 if (release_time == rh->release_time) {
115 /* perfect match -- this happens on hyperperiod
116 * boundaries
117 */
118 heap = rh;
119 break;
120 } else if (lt_before(release_time, rh->release_time)) {
121 /* we need to insert a new node since rh is
122 * already in the future
123 */
124 break;
125 }
126 }
127 if (!heap && use_task_heap) {
128 /* use pre-allocated release heap */
129 rh = tsk_rt(t)->rel_heap;
130
131 rh->dom = rt;
132 rh->release_time = release_time;
133
134 /* add to release queue */
135 list_add(&rh->list, pos->prev);
136 heap = rh;
137 }
138 return heap;
139}
140
141static void reinit_release_heap(struct task_struct* t)
142{
143 struct release_heap* rh;
144
145 /* use pre-allocated release heap */
146 rh = tsk_rt(t)->rel_heap;
147
148 /* Make sure it is safe to use. The timer callback could still
149 * be executing on another CPU; hrtimer_cancel() will wait
150 * until the timer callback has completed. However, under no
151 * circumstances should the timer be active (= yet to be
152 * triggered).
153 *
154 * WARNING: If the CPU still holds the release_lock at this point,
155 * deadlock may occur!
156 */
157 BUG_ON(hrtimer_cancel(&rh->timer));
158
159 /* initialize */
160 bheap_init(&rh->heap);
161 atomic_set(&rh->info.state, HRTIMER_START_ON_INACTIVE);
162}
163/* arm_release_timer() - start local release timer or trigger
164 * remote timer (pull timer)
165 *
166 * Called by add_release() with:
167 * - tobe_lock taken
168 * - IRQ disabled
169 */
170static void arm_release_timer(rt_domain_t *_rt)
171{
172 rt_domain_t *rt = _rt;
173 struct list_head list;
174 struct list_head *pos, *safe;
175 struct task_struct* t;
176 struct release_heap* rh;
177
178 TRACE("arm_release_timer() at %llu\n", litmus_clock());
179 list_replace_init(&rt->tobe_released, &list);
180
181 list_for_each_safe(pos, safe, &list) {
182 /* pick task of work list */
183 t = list_entry(pos, struct task_struct, rt_param.list);
184 sched_trace_task_release(t);
185 list_del(pos);
186
187 /* put into release heap while holding release_lock */
188 spin_lock(&rt->release_lock);
189 TRACE_TASK(t, "I have the release_lock 0x%p\n", &rt->release_lock);
190
191 rh = get_release_heap(rt, t, 0);
192 if (!rh) {
193 /* need to use our own, but drop lock first */
194 spin_unlock(&rt->release_lock);
195 TRACE_TASK(t, "Dropped release_lock 0x%p\n",
196 &rt->release_lock);
197
198 reinit_release_heap(t);
199 TRACE_TASK(t, "release_heap ready\n");
200
201 spin_lock(&rt->release_lock);
202 TRACE_TASK(t, "Re-acquired release_lock 0x%p\n",
203 &rt->release_lock);
204
205 rh = get_release_heap(rt, t, 1);
206 }
207 bheap_insert(rt->order, &rh->heap, tsk_rt(t)->heap_node);
208 TRACE_TASK(t, "arm_release_timer(): added to release heap\n");
209
210 spin_unlock(&rt->release_lock);
211 TRACE_TASK(t, "Returned the release_lock 0x%p\n", &rt->release_lock);
212
213 /* To avoid arming the timer multiple times, we only let the
214 * owner do the arming (which is the "first" task to reference
215 * this release_heap anyway).
216 */
217 if (rh == tsk_rt(t)->rel_heap) {
218 TRACE_TASK(t, "arming timer 0x%p\n", &rh->timer);
219 /* we cannot arm the timer using hrtimer_start()
220 * as it may deadlock on rq->lock
221 *
222 * PINNED mode is ok on both local and remote CPU
223 */
224 if (rt->release_master == NO_CPU)
225 __hrtimer_start_range_ns(&rh->timer,
226 ns_to_ktime(rh->release_time),
227 0, HRTIMER_MODE_ABS_PINNED, 0);
228 else
229 hrtimer_start_on(rt->release_master,
230 &rh->info, &rh->timer,
231 ns_to_ktime(rh->release_time),
232 HRTIMER_MODE_ABS_PINNED);
233 } else
234 TRACE_TASK(t, "0x%p is not my timer\n", &rh->timer);
235 }
236}
237
238void rt_domain_init(rt_domain_t *rt,
239 bheap_prio_t order,
240 check_resched_needed_t check,
241 release_jobs_t release
242 )
243{
244 int i;
245
246 BUG_ON(!rt);
247 if (!check)
248 check = dummy_resched;
249 if (!release)
250 release = default_release_jobs;
251 if (!order)
252 order = dummy_order;
253
254 rt->release_master = NO_CPU;
255
256 bheap_init(&rt->ready_queue);
257 INIT_LIST_HEAD(&rt->tobe_released);
258 for (i = 0; i < RELEASE_QUEUE_SLOTS; i++)
259 INIT_LIST_HEAD(&rt->release_queue.slot[i]);
260
261 spin_lock_init(&rt->ready_lock);
262 spin_lock_init(&rt->release_lock);
263 spin_lock_init(&rt->tobe_lock);
264
265 rt->check_resched = check;
266 rt->release_jobs = release;
267 rt->order = order;
268}
269
270/* add_ready - add a real-time task to the rt ready queue. It must be runnable.
271 * @new: the newly released task
272 */
273void __add_ready(rt_domain_t* rt, struct task_struct *new)
274{
275 TRACE("rt: adding %s/%d (%llu, %llu) rel=%llu to ready queue at %llu\n",
276 new->comm, new->pid, get_exec_cost(new), get_rt_period(new),
277 get_release(new), litmus_clock());
278
279 BUG_ON(bheap_node_in_heap(tsk_rt(new)->heap_node));
280
281 bheap_insert(rt->order, &rt->ready_queue, tsk_rt(new)->heap_node);
282 rt->check_resched(rt);
283}
284
285/* merge_ready - Add a sorted set of tasks to the rt ready queue. They must be runnable.
286 * @tasks - the newly released tasks
287 */
288void __merge_ready(rt_domain_t* rt, struct bheap* tasks)
289{
290 bheap_union(rt->order, &rt->ready_queue, tasks);
291 rt->check_resched(rt);
292}
293
294/* add_release - add a real-time task to the rt release queue.
295 * @task: the sleeping task
296 */
297void __add_release(rt_domain_t* rt, struct task_struct *task)
298{
299 TRACE_TASK(task, "add_release(), rel=%llu\n", get_release(task));
300 list_add(&tsk_rt(task)->list, &rt->tobe_released);
301 task->rt_param.domain = rt;
302
303 /* start release timer */
304 TS_SCHED2_START(task);
305
306 arm_release_timer(rt);
307
308 TS_SCHED2_END(task);
309}
310
diff --git a/litmus/sched_cedf.c b/litmus/sched_cedf.c
new file mode 100644
index 000000000000..da44b451c9ad
--- /dev/null
+++ b/litmus/sched_cedf.c
@@ -0,0 +1,756 @@
1/*
2 * litmus/sched_cedf.c
3 *
4 * Implementation of the C-EDF scheduling algorithm.
5 *
6 * This implementation is based on G-EDF:
7 * - CPUs are clustered around L2 or L3 caches.
8 * - Clusters topology is automatically detected (this is arch dependent
9 * and is working only on x86 at the moment --- and only with modern
10 * cpus that exports cpuid4 information)
11 * - The plugins _does not_ attempt to put tasks in the right cluster i.e.
12 * the programmer needs to be aware of the topology to place tasks
13 * in the desired cluster
14 * - default clustering is around L2 cache (cache index = 2)
15 * supported clusters are: L1 (private cache: pedf), L2, L3
16 *
17 * For details on functions, take a look at sched_gsn_edf.c
18 *
19 * This version uses the simple approach and serializes all scheduling
20 * decisions by the use of a queue lock. This is probably not the
21 * best way to do it, but it should suffice for now.
22 */
23
24#include <linux/spinlock.h>
25#include <linux/percpu.h>
26#include <linux/sched.h>
27
28#include <litmus/litmus.h>
29#include <litmus/jobs.h>
30#include <litmus/sched_plugin.h>
31#include <litmus/edf_common.h>
32#include <litmus/sched_trace.h>
33
34#include <litmus/bheap.h>
35
36#include <linux/module.h>
37
38/* forward declaration... a funny thing with C ;) */
39struct clusterdomain;
40
41/* cpu_entry_t - maintain the linked and scheduled state
42 *
43 * A cpu also contains a pointer to the cedf_domain_t cluster
44 * that owns it (struct clusterdomain*)
45 */
46typedef struct {
47 int cpu;
48 struct clusterdomain* cluster; /* owning cluster */
49 struct task_struct* linked; /* only RT tasks */
50 struct task_struct* scheduled; /* only RT tasks */
51 atomic_t will_schedule; /* prevent unneeded IPIs */
52 struct bheap_node* hn;
53} cpu_entry_t;
54
55/* one cpu_entry_t per CPU */
56DEFINE_PER_CPU(cpu_entry_t, cedf_cpu_entries);
57
58#define set_will_schedule() \
59 (atomic_set(&__get_cpu_var(cedf_cpu_entries).will_schedule, 1))
60#define clear_will_schedule() \
61 (atomic_set(&__get_cpu_var(cedf_cpu_entries).will_schedule, 0))
62#define test_will_schedule(cpu) \
63 (atomic_read(&per_cpu(cedf_cpu_entries, cpu).will_schedule))
64
65/*
66 * In C-EDF there is a cedf domain _per_ cluster
67 * The number of clusters is dynamically determined accordingly to the
68 * total cpu number and the cluster size
69 */
70typedef struct clusterdomain {
71 /* rt_domain for this cluster */
72 rt_domain_t domain;
73 /* cpus in this cluster */
74 cpu_entry_t* *cpus;
75 /* map of this cluster cpus */
76 cpumask_var_t cpu_map;
77 /* the cpus queue themselves according to priority in here */
78 struct bheap_node *heap_node;
79 struct bheap cpu_heap;
80 /* lock for this cluster */
81#define lock domain.ready_lock
82} cedf_domain_t;
83
84/* a cedf_domain per cluster; allocation is done at init/activation time */
85cedf_domain_t *cedf;
86
87#define remote_cluster(cpu) ((cedf_domain_t *) per_cpu(cedf_cpu_entries, cpu).cluster)
88#define task_cpu_cluster(task) remote_cluster(get_partition(task))
89
90/* Uncomment WANT_ALL_SCHED_EVENTS if you want to see all scheduling
91 * decisions in the TRACE() log; uncomment VERBOSE_INIT for verbose
92 * information during the initialization of the plugin (e.g., topology)
93#define WANT_ALL_SCHED_EVENTS
94 */
95#define VERBOSE_INIT
96
97static int cpu_lower_prio(struct bheap_node *_a, struct bheap_node *_b)
98{
99 cpu_entry_t *a, *b;
100 a = _a->value;
101 b = _b->value;
102 /* Note that a and b are inverted: we want the lowest-priority CPU at
103 * the top of the heap.
104 */
105 return edf_higher_prio(b->linked, a->linked);
106}
107
108/* update_cpu_position - Move the cpu entry to the correct place to maintain
109 * order in the cpu queue. Caller must hold cedf lock.
110 */
111static void update_cpu_position(cpu_entry_t *entry)
112{
113 cedf_domain_t *cluster = entry->cluster;
114
115 if (likely(bheap_node_in_heap(entry->hn)))
116 bheap_delete(cpu_lower_prio,
117 &cluster->cpu_heap,
118 entry->hn);
119
120 bheap_insert(cpu_lower_prio, &cluster->cpu_heap, entry->hn);
121}
122
123/* caller must hold cedf lock */
124static cpu_entry_t* lowest_prio_cpu(cedf_domain_t *cluster)
125{
126 struct bheap_node* hn;
127 hn = bheap_peek(cpu_lower_prio, &cluster->cpu_heap);
128 return hn->value;
129}
130
131
132/* link_task_to_cpu - Update the link of a CPU.
133 * Handles the case where the to-be-linked task is already
134 * scheduled on a different CPU.
135 */
136static noinline void link_task_to_cpu(struct task_struct* linked,
137 cpu_entry_t *entry)
138{
139 cpu_entry_t *sched;
140 struct task_struct* tmp;
141 int on_cpu;
142
143 BUG_ON(linked && !is_realtime(linked));
144
145 /* Currently linked task is set to be unlinked. */
146 if (entry->linked) {
147 entry->linked->rt_param.linked_on = NO_CPU;
148 }
149
150 /* Link new task to CPU. */
151 if (linked) {
152 set_rt_flags(linked, RT_F_RUNNING);
153 /* handle task is already scheduled somewhere! */
154 on_cpu = linked->rt_param.scheduled_on;
155 if (on_cpu != NO_CPU) {
156 sched = &per_cpu(cedf_cpu_entries, on_cpu);
157 /* this should only happen if not linked already */
158 BUG_ON(sched->linked == linked);
159
160 /* If we are already scheduled on the CPU to which we
161 * wanted to link, we don't need to do the swap --
162 * we just link ourselves to the CPU and depend on
163 * the caller to get things right.
164 */
165 if (entry != sched) {
166 TRACE_TASK(linked,
167 "already scheduled on %d, updating link.\n",
168 sched->cpu);
169 tmp = sched->linked;
170 linked->rt_param.linked_on = sched->cpu;
171 sched->linked = linked;
172 update_cpu_position(sched);
173 linked = tmp;
174 }
175 }
176 if (linked) /* might be NULL due to swap */
177 linked->rt_param.linked_on = entry->cpu;
178 }
179 entry->linked = linked;
180#ifdef WANT_ALL_SCHED_EVENTS
181 if (linked)
182 TRACE_TASK(linked, "linked to %d.\n", entry->cpu);
183 else
184 TRACE("NULL linked to %d.\n", entry->cpu);
185#endif
186 update_cpu_position(entry);
187}
188
189/* unlink - Make sure a task is not linked any longer to an entry
190 * where it was linked before. Must hold cedf_lock.
191 */
192static noinline void unlink(struct task_struct* t)
193{
194 cpu_entry_t *entry;
195
196 if (unlikely(!t)) {
197 TRACE_BUG_ON(!t);
198 return;
199 }
200
201
202 if (t->rt_param.linked_on != NO_CPU) {
203 /* unlink */
204 entry = &per_cpu(cedf_cpu_entries, t->rt_param.linked_on);
205 t->rt_param.linked_on = NO_CPU;
206 link_task_to_cpu(NULL, entry);
207 } else if (is_queued(t)) {
208 /* This is an interesting situation: t is scheduled,
209 * but was just recently unlinked. It cannot be
210 * linked anywhere else (because then it would have
211 * been relinked to this CPU), thus it must be in some
212 * queue. We must remove it from the list in this
213 * case.
214 *
215 * in C-EDF case is should be somewhere in the queue for
216 * its domain, therefore and we can get the domain using
217 * task_cpu_cluster
218 */
219 remove(&(task_cpu_cluster(t))->domain, t);
220 }
221}
222
223
224/* preempt - force a CPU to reschedule
225 */
226static void preempt(cpu_entry_t *entry)
227{
228 preempt_if_preemptable(entry->scheduled, entry->cpu);
229}
230
231/* requeue - Put an unlinked task into gsn-edf domain.
232 * Caller must hold cedf_lock.
233 */
234static noinline void requeue(struct task_struct* task)
235{
236 cedf_domain_t *cluster = task_cpu_cluster(task);
237 BUG_ON(!task);
238 /* sanity check before insertion */
239 BUG_ON(is_queued(task));
240
241 if (is_released(task, litmus_clock()))
242 __add_ready(&cluster->domain, task);
243 else {
244 /* it has got to wait */
245 add_release(&cluster->domain, task);
246 }
247}
248
249/* check for any necessary preemptions */
250static void check_for_preemptions(cedf_domain_t *cluster)
251{
252 struct task_struct *task;
253 cpu_entry_t* last;
254
255 for(last = lowest_prio_cpu(cluster);
256 edf_preemption_needed(&cluster->domain, last->linked);
257 last = lowest_prio_cpu(cluster)) {
258 /* preemption necessary */
259 task = __take_ready(&cluster->domain);
260 TRACE("check_for_preemptions: attempting to link task %d to %d\n",
261 task->pid, last->cpu);
262 if (last->linked)
263 requeue(last->linked);
264 link_task_to_cpu(task, last);
265 preempt(last);
266 }
267}
268
269/* cedf_job_arrival: task is either resumed or released */
270static noinline void cedf_job_arrival(struct task_struct* task)
271{
272 cedf_domain_t *cluster = task_cpu_cluster(task);
273 BUG_ON(!task);
274
275 requeue(task);
276 check_for_preemptions(cluster);
277}
278
279static void cedf_release_jobs(rt_domain_t* rt, struct bheap* tasks)
280{
281 cedf_domain_t* cluster = container_of(rt, cedf_domain_t, domain);
282 unsigned long flags;
283
284 spin_lock_irqsave(&cluster->lock, flags);
285
286 __merge_ready(&cluster->domain, tasks);
287 check_for_preemptions(cluster);
288
289 spin_unlock_irqrestore(&cluster->lock, flags);
290}
291
292/* caller holds cedf_lock */
293static noinline void job_completion(struct task_struct *t, int forced)
294{
295 BUG_ON(!t);
296
297 sched_trace_task_completion(t, forced);
298
299 TRACE_TASK(t, "job_completion().\n");
300
301 /* set flags */
302 set_rt_flags(t, RT_F_SLEEP);
303 /* prepare for next period */
304 prepare_for_next_period(t);
305 if (is_released(t, litmus_clock()))
306 sched_trace_task_release(t);
307 /* unlink */
308 unlink(t);
309 /* requeue
310 * But don't requeue a blocking task. */
311 if (is_running(t))
312 cedf_job_arrival(t);
313}
314
315/* cedf_tick - this function is called for every local timer
316 * interrupt.
317 *
318 * checks whether the current task has expired and checks
319 * whether we need to preempt it if it has not expired
320 */
321static void cedf_tick(struct task_struct* t)
322{
323 if (is_realtime(t) && budget_exhausted(t)) {
324 if (!is_np(t)) {
325 /* np tasks will be preempted when they become
326 * preemptable again
327 */
328 set_tsk_need_resched(t);
329 set_will_schedule();
330 TRACE("cedf_scheduler_tick: "
331 "%d is preemptable "
332 " => FORCE_RESCHED\n", t->pid);
333 } else if (is_user_np(t)) {
334 TRACE("cedf_scheduler_tick: "
335 "%d is non-preemptable, "
336 "preemption delayed.\n", t->pid);
337 request_exit_np(t);
338 }
339 }
340}
341
342/* Getting schedule() right is a bit tricky. schedule() may not make any
343 * assumptions on the state of the current task since it may be called for a
344 * number of reasons. The reasons include a scheduler_tick() determined that it
345 * was necessary, because sys_exit_np() was called, because some Linux
346 * subsystem determined so, or even (in the worst case) because there is a bug
347 * hidden somewhere. Thus, we must take extreme care to determine what the
348 * current state is.
349 *
350 * The CPU could currently be scheduling a task (or not), be linked (or not).
351 *
352 * The following assertions for the scheduled task could hold:
353 *
354 * - !is_running(scheduled) // the job blocks
355 * - scheduled->timeslice == 0 // the job completed (forcefully)
356 * - get_rt_flag() == RT_F_SLEEP // the job completed (by syscall)
357 * - linked != scheduled // we need to reschedule (for any reason)
358 * - is_np(scheduled) // rescheduling must be delayed,
359 * sys_exit_np must be requested
360 *
361 * Any of these can occur together.
362 */
363static struct task_struct* cedf_schedule(struct task_struct * prev)
364{
365 cpu_entry_t* entry = &__get_cpu_var(cedf_cpu_entries);
366 cedf_domain_t *cluster = entry->cluster;
367 int out_of_time, sleep, preempt, np, exists, blocks;
368 struct task_struct* next = NULL;
369
370 spin_lock(&cluster->lock);
371 clear_will_schedule();
372
373 /* sanity checking */
374 BUG_ON(entry->scheduled && entry->scheduled != prev);
375 BUG_ON(entry->scheduled && !is_realtime(prev));
376 BUG_ON(is_realtime(prev) && !entry->scheduled);
377
378 /* (0) Determine state */
379 exists = entry->scheduled != NULL;
380 blocks = exists && !is_running(entry->scheduled);
381 out_of_time = exists && budget_exhausted(entry->scheduled);
382 np = exists && is_np(entry->scheduled);
383 sleep = exists && get_rt_flags(entry->scheduled) == RT_F_SLEEP;
384 preempt = entry->scheduled != entry->linked;
385
386#ifdef WANT_ALL_SCHED_EVENTS
387 TRACE_TASK(prev, "invoked cedf_schedule.\n");
388#endif
389
390 if (exists)
391 TRACE_TASK(prev,
392 "blocks:%d out_of_time:%d np:%d sleep:%d preempt:%d "
393 "state:%d sig:%d\n",
394 blocks, out_of_time, np, sleep, preempt,
395 prev->state, signal_pending(prev));
396 if (entry->linked && preempt)
397 TRACE_TASK(prev, "will be preempted by %s/%d\n",
398 entry->linked->comm, entry->linked->pid);
399
400
401 /* If a task blocks we have no choice but to reschedule.
402 */
403 if (blocks)
404 unlink(entry->scheduled);
405
406 /* Request a sys_exit_np() call if we would like to preempt but cannot.
407 * We need to make sure to update the link structure anyway in case
408 * that we are still linked. Multiple calls to request_exit_np() don't
409 * hurt.
410 */
411 if (np && (out_of_time || preempt || sleep)) {
412 unlink(entry->scheduled);
413 request_exit_np(entry->scheduled);
414 }
415
416 /* Any task that is preemptable and either exhausts its execution
417 * budget or wants to sleep completes. We may have to reschedule after
418 * this. Don't do a job completion if we block (can't have timers running
419 * for blocked jobs). Preemption go first for the same reason.
420 */
421 if (!np && (out_of_time || sleep) && !blocks && !preempt)
422 job_completion(entry->scheduled, !sleep);
423
424 /* Link pending task if we became unlinked.
425 */
426 if (!entry->linked)
427 link_task_to_cpu(__take_ready(&cluster->domain), entry);
428
429 /* The final scheduling decision. Do we need to switch for some reason?
430 * If linked is different from scheduled, then select linked as next.
431 */
432 if ((!np || blocks) &&
433 entry->linked != entry->scheduled) {
434 /* Schedule a linked job? */
435 if (entry->linked) {
436 entry->linked->rt_param.scheduled_on = entry->cpu;
437 next = entry->linked;
438 }
439 if (entry->scheduled) {
440 /* not gonna be scheduled soon */
441 entry->scheduled->rt_param.scheduled_on = NO_CPU;
442 TRACE_TASK(entry->scheduled, "scheduled_on = NO_CPU\n");
443 }
444 } else
445 /* Only override Linux scheduler if we have a real-time task
446 * scheduled that needs to continue.
447 */
448 if (exists)
449 next = prev;
450
451 spin_unlock(&cluster->lock);
452
453#ifdef WANT_ALL_SCHED_EVENTS
454 TRACE("cedf_lock released, next=0x%p\n", next);
455
456 if (next)
457 TRACE_TASK(next, "scheduled at %llu\n", litmus_clock());
458 else if (exists && !next)
459 TRACE("becomes idle at %llu.\n", litmus_clock());
460#endif
461
462
463 return next;
464}
465
466
467/* _finish_switch - we just finished the switch away from prev
468 */
469static void cedf_finish_switch(struct task_struct *prev)
470{
471 cpu_entry_t* entry = &__get_cpu_var(cedf_cpu_entries);
472
473 entry->scheduled = is_realtime(current) ? current : NULL;
474#ifdef WANT_ALL_SCHED_EVENTS
475 TRACE_TASK(prev, "switched away from\n");
476#endif
477}
478
479
480/* Prepare a task for running in RT mode
481 */
482static void cedf_task_new(struct task_struct * t, int on_rq, int running)
483{
484 unsigned long flags;
485 cpu_entry_t* entry;
486 cedf_domain_t* cluster;
487
488 TRACE("gsn edf: task new %d\n", t->pid);
489
490 /* the cluster doesn't change even if t is running */
491 cluster = task_cpu_cluster(t);
492
493 spin_lock_irqsave(&cluster->domain.ready_lock, flags);
494
495 /* setup job params */
496 release_at(t, litmus_clock());
497
498 if (running) {
499 entry = &per_cpu(cedf_cpu_entries, task_cpu(t));
500 BUG_ON(entry->scheduled);
501
502 entry->scheduled = t;
503 tsk_rt(t)->scheduled_on = task_cpu(t);
504 } else {
505 t->rt_param.scheduled_on = NO_CPU;
506 }
507 t->rt_param.linked_on = NO_CPU;
508
509 cedf_job_arrival(t);
510 spin_unlock_irqrestore(&(cluster->domain.ready_lock), flags);
511}
512
513static void cedf_task_wake_up(struct task_struct *task)
514{
515 unsigned long flags;
516 lt_t now;
517 cedf_domain_t *cluster;
518
519 TRACE_TASK(task, "wake_up at %llu\n", litmus_clock());
520
521 cluster = task_cpu_cluster(task);
522
523 spin_lock_irqsave(&cluster->lock, flags);
524 /* We need to take suspensions because of semaphores into
525 * account! If a job resumes after being suspended due to acquiring
526 * a semaphore, it should never be treated as a new job release.
527 */
528 if (get_rt_flags(task) == RT_F_EXIT_SEM) {
529 set_rt_flags(task, RT_F_RUNNING);
530 } else {
531 now = litmus_clock();
532 if (is_tardy(task, now)) {
533 /* new sporadic release */
534 release_at(task, now);
535 sched_trace_task_release(task);
536 }
537 else {
538 if (task->rt.time_slice) {
539 /* came back in time before deadline
540 */
541 set_rt_flags(task, RT_F_RUNNING);
542 }
543 }
544 }
545 cedf_job_arrival(task);
546 spin_unlock_irqrestore(&cluster->lock, flags);
547}
548
549static void cedf_task_block(struct task_struct *t)
550{
551 unsigned long flags;
552 cedf_domain_t *cluster;
553
554 TRACE_TASK(t, "block at %llu\n", litmus_clock());
555
556 cluster = task_cpu_cluster(t);
557
558 /* unlink if necessary */
559 spin_lock_irqsave(&cluster->lock, flags);
560 unlink(t);
561 spin_unlock_irqrestore(&cluster->lock, flags);
562
563 BUG_ON(!is_realtime(t));
564}
565
566
567static void cedf_task_exit(struct task_struct * t)
568{
569 unsigned long flags;
570 cedf_domain_t *cluster = task_cpu_cluster(t);
571
572 /* unlink if necessary */
573 spin_lock_irqsave(&cluster->lock, flags);
574 unlink(t);
575 if (tsk_rt(t)->scheduled_on != NO_CPU) {
576 cluster->cpus[tsk_rt(t)->scheduled_on]->scheduled = NULL;
577 tsk_rt(t)->scheduled_on = NO_CPU;
578 }
579 spin_unlock_irqrestore(&cluster->lock, flags);
580
581 BUG_ON(!is_realtime(t));
582 TRACE_TASK(t, "RIP\n");
583}
584
585static long cedf_admit_task(struct task_struct* tsk)
586{
587 return task_cpu(tsk) == tsk->rt_param.task_params.cpu ? 0 : -EINVAL;
588}
589
590/* total number of cluster */
591static int num_clusters;
592/* we do not support cluster of different sizes */
593static unsigned int cluster_size;
594
595#ifdef VERBOSE_INIT
596static void print_cluster_topology(cpumask_var_t mask, int cpu)
597{
598 int chk;
599 char buf[255];
600
601 chk = cpulist_scnprintf(buf, 254, mask);
602 buf[chk] = '\0';
603 printk(KERN_INFO "CPU = %d, shared cpu(s) = %s\n", cpu, buf);
604
605}
606#endif
607
608static int clusters_allocated = 0;
609
610static void cleanup_cedf(void)
611{
612 int i;
613
614 if (clusters_allocated) {
615 for (i = 0; i < num_clusters; i++) {
616 kfree(cedf[i].cpus);
617 kfree(cedf[i].heap_node);
618 free_cpumask_var(cedf[i].cpu_map);
619 }
620
621 kfree(cedf);
622 }
623}
624
625static long cedf_activate_plugin(void)
626{
627 int i, j, cpu, ccpu, cpu_count;
628 cpu_entry_t *entry;
629
630 cpumask_var_t mask;
631 int chk = 0;
632
633 /* de-allocate old clusters, if any */
634 cleanup_cedf();
635
636 printk(KERN_INFO "C-EDF: Activate Plugin, cache index = %d\n",
637 cluster_cache_index);
638
639 /* need to get cluster_size first */
640 if(!zalloc_cpumask_var(&mask, GFP_ATOMIC))
641 return -ENOMEM;
642
643 chk = get_shared_cpu_map(mask, 0, cluster_cache_index);
644 if (chk) {
645 /* if chk != 0 then it is the max allowed index */
646 printk(KERN_INFO "C-EDF: Cannot support cache index = %d\n",
647 cluster_cache_index);
648 printk(KERN_INFO "C-EDF: Using cache index = %d\n",
649 chk);
650 cluster_cache_index = chk;
651 }
652
653 cluster_size = cpumask_weight(mask);
654
655 if ((num_online_cpus() % cluster_size) != 0) {
656 /* this can't be right, some cpus are left out */
657 printk(KERN_ERR "C-EDF: Trying to group %d cpus in %d!\n",
658 num_online_cpus(), cluster_size);
659 return -1;
660 }
661
662 num_clusters = num_online_cpus() / cluster_size;
663 printk(KERN_INFO "C-EDF: %d cluster(s) of size = %d\n",
664 num_clusters, cluster_size);
665
666 /* initialize clusters */
667 cedf = kmalloc(num_clusters * sizeof(cedf_domain_t), GFP_ATOMIC);
668 for (i = 0; i < num_clusters; i++) {
669
670 cedf[i].cpus = kmalloc(cluster_size * sizeof(cpu_entry_t),
671 GFP_ATOMIC);
672 cedf[i].heap_node = kmalloc(
673 cluster_size * sizeof(struct bheap_node),
674 GFP_ATOMIC);
675 bheap_init(&(cedf[i].cpu_heap));
676 edf_domain_init(&(cedf[i].domain), NULL, cedf_release_jobs);
677
678 if(!zalloc_cpumask_var(&cedf[i].cpu_map, GFP_ATOMIC))
679 return -ENOMEM;
680 }
681
682 /* cycle through cluster and add cpus to them */
683 for (i = 0; i < num_clusters; i++) {
684
685 for_each_online_cpu(cpu) {
686 /* check if the cpu is already in a cluster */
687 for (j = 0; j < num_clusters; j++)
688 if (cpumask_test_cpu(cpu, cedf[j].cpu_map))
689 break;
690 /* if it is in a cluster go to next cpu */
691 if (cpumask_test_cpu(cpu, cedf[j].cpu_map))
692 continue;
693
694 /* this cpu isn't in any cluster */
695 /* get the shared cpus */
696 get_shared_cpu_map(mask, cpu, cluster_cache_index);
697 cpumask_copy(cedf[i].cpu_map, mask);
698#ifdef VERBOSE_INIT
699 print_cluster_topology(mask, cpu);
700#endif
701 /* add cpus to current cluster and init cpu_entry_t */
702 cpu_count = 0;
703 for_each_cpu(ccpu, cedf[i].cpu_map) {
704
705 entry = &per_cpu(cedf_cpu_entries, ccpu);
706 cedf[i].cpus[cpu_count] = entry;
707 atomic_set(&entry->will_schedule, 0);
708 entry->cpu = ccpu;
709 entry->cluster = &cedf[i];
710 entry->hn = &(cedf[i].heap_node[cpu_count]);
711 bheap_node_init(&entry->hn, entry);
712
713 cpu_count++;
714
715 entry->linked = NULL;
716 entry->scheduled = NULL;
717 update_cpu_position(entry);
718 }
719 /* done with this cluster */
720 break;
721 }
722 }
723
724 free_cpumask_var(mask);
725 clusters_allocated = 1;
726 return 0;
727}
728
729/* Plugin object */
730static struct sched_plugin cedf_plugin __cacheline_aligned_in_smp = {
731 .plugin_name = "C-EDF",
732 .finish_switch = cedf_finish_switch,
733 .tick = cedf_tick,
734 .task_new = cedf_task_new,
735 .complete_job = complete_job,
736 .task_exit = cedf_task_exit,
737 .schedule = cedf_schedule,
738 .task_wake_up = cedf_task_wake_up,
739 .task_block = cedf_task_block,
740 .admit_task = cedf_admit_task,
741 .activate_plugin = cedf_activate_plugin,
742};
743
744
745static int __init init_cedf(void)
746{
747 return register_sched_plugin(&cedf_plugin);
748}
749
750static void clean_cedf(void)
751{
752 cleanup_cedf();
753}
754
755module_init(init_cedf);
756module_exit(clean_cedf);
diff --git a/litmus/sched_gsn_edf.c b/litmus/sched_gsn_edf.c
new file mode 100644
index 000000000000..b9310dd6f75c
--- /dev/null
+++ b/litmus/sched_gsn_edf.c
@@ -0,0 +1,828 @@
1/*
2 * litmus/sched_gsn_edf.c
3 *
4 * Implementation of the GSN-EDF scheduling algorithm.
5 *
6 * This version uses the simple approach and serializes all scheduling
7 * decisions by the use of a queue lock. This is probably not the
8 * best way to do it, but it should suffice for now.
9 */
10
11#include <linux/spinlock.h>
12#include <linux/percpu.h>
13#include <linux/sched.h>
14
15#include <litmus/litmus.h>
16#include <litmus/jobs.h>
17#include <litmus/sched_plugin.h>
18#include <litmus/edf_common.h>
19#include <litmus/sched_trace.h>
20
21#include <litmus/bheap.h>
22
23#include <linux/module.h>
24
25/* Overview of GSN-EDF operations.
26 *
27 * For a detailed explanation of GSN-EDF have a look at the FMLP paper. This
28 * description only covers how the individual operations are implemented in
29 * LITMUS.
30 *
31 * link_task_to_cpu(T, cpu) - Low-level operation to update the linkage
32 * structure (NOT the actually scheduled
33 * task). If there is another linked task To
34 * already it will set To->linked_on = NO_CPU
35 * (thereby removing its association with this
36 * CPU). However, it will not requeue the
37 * previously linked task (if any). It will set
38 * T's state to RT_F_RUNNING and check whether
39 * it is already running somewhere else. If T
40 * is scheduled somewhere else it will link
41 * it to that CPU instead (and pull the linked
42 * task to cpu). T may be NULL.
43 *
44 * unlink(T) - Unlink removes T from all scheduler data
45 * structures. If it is linked to some CPU it
46 * will link NULL to that CPU. If it is
47 * currently queued in the gsnedf queue it will
48 * be removed from the rt_domain. It is safe to
49 * call unlink(T) if T is not linked. T may not
50 * be NULL.
51 *
52 * requeue(T) - Requeue will insert T into the appropriate
53 * queue. If the system is in real-time mode and
54 * the T is released already, it will go into the
55 * ready queue. If the system is not in
56 * real-time mode is T, then T will go into the
57 * release queue. If T's release time is in the
58 * future, it will go into the release
59 * queue. That means that T's release time/job
60 * no/etc. has to be updated before requeu(T) is
61 * called. It is not safe to call requeue(T)
62 * when T is already queued. T may not be NULL.
63 *
64 * gsnedf_job_arrival(T) - This is the catch all function when T enters
65 * the system after either a suspension or at a
66 * job release. It will queue T (which means it
67 * is not safe to call gsnedf_job_arrival(T) if
68 * T is already queued) and then check whether a
69 * preemption is necessary. If a preemption is
70 * necessary it will update the linkage
71 * accordingly and cause scheduled to be called
72 * (either with an IPI or need_resched). It is
73 * safe to call gsnedf_job_arrival(T) if T's
74 * next job has not been actually released yet
75 * (releast time in the future). T will be put
76 * on the release queue in that case.
77 *
78 * job_completion(T) - Take care of everything that needs to be done
79 * to prepare T for its next release and place
80 * it in the right queue with
81 * gsnedf_job_arrival().
82 *
83 *
84 * When we now that T is linked to CPU then link_task_to_cpu(NULL, CPU) is
85 * equivalent to unlink(T). Note that if you unlink a task from a CPU none of
86 * the functions will automatically propagate pending task from the ready queue
87 * to a linked task. This is the job of the calling function ( by means of
88 * __take_ready).
89 */
90
91
92/* cpu_entry_t - maintain the linked and scheduled state
93 */
94typedef struct {
95 int cpu;
96 struct task_struct* linked; /* only RT tasks */
97 struct task_struct* scheduled; /* only RT tasks */
98 atomic_t will_schedule; /* prevent unneeded IPIs */
99 struct bheap_node* hn;
100} cpu_entry_t;
101DEFINE_PER_CPU(cpu_entry_t, gsnedf_cpu_entries);
102
103cpu_entry_t* gsnedf_cpus[NR_CPUS];
104
105#define set_will_schedule() \
106 (atomic_set(&__get_cpu_var(gsnedf_cpu_entries).will_schedule, 1))
107#define clear_will_schedule() \
108 (atomic_set(&__get_cpu_var(gsnedf_cpu_entries).will_schedule, 0))
109#define test_will_schedule(cpu) \
110 (atomic_read(&per_cpu(gsnedf_cpu_entries, cpu).will_schedule))
111
112
113/* the cpus queue themselves according to priority in here */
114static struct bheap_node gsnedf_heap_node[NR_CPUS];
115static struct bheap gsnedf_cpu_heap;
116
117static rt_domain_t gsnedf;
118#define gsnedf_lock (gsnedf.ready_lock)
119
120
121/* Uncomment this if you want to see all scheduling decisions in the
122 * TRACE() log.
123#define WANT_ALL_SCHED_EVENTS
124 */
125
126static int cpu_lower_prio(struct bheap_node *_a, struct bheap_node *_b)
127{
128 cpu_entry_t *a, *b;
129 a = _a->value;
130 b = _b->value;
131 /* Note that a and b are inverted: we want the lowest-priority CPU at
132 * the top of the heap.
133 */
134 return edf_higher_prio(b->linked, a->linked);
135}
136
137/* update_cpu_position - Move the cpu entry to the correct place to maintain
138 * order in the cpu queue. Caller must hold gsnedf lock.
139 */
140static void update_cpu_position(cpu_entry_t *entry)
141{
142 if (likely(bheap_node_in_heap(entry->hn)))
143 bheap_delete(cpu_lower_prio, &gsnedf_cpu_heap, entry->hn);
144 bheap_insert(cpu_lower_prio, &gsnedf_cpu_heap, entry->hn);
145}
146
147/* caller must hold gsnedf lock */
148static cpu_entry_t* lowest_prio_cpu(void)
149{
150 struct bheap_node* hn;
151 hn = bheap_peek(cpu_lower_prio, &gsnedf_cpu_heap);
152 return hn->value;
153}
154
155
156/* link_task_to_cpu - Update the link of a CPU.
157 * Handles the case where the to-be-linked task is already
158 * scheduled on a different CPU.
159 */
160static noinline void link_task_to_cpu(struct task_struct* linked,
161 cpu_entry_t *entry)
162{
163 cpu_entry_t *sched;
164 struct task_struct* tmp;
165 int on_cpu;
166
167 BUG_ON(linked && !is_realtime(linked));
168
169 /* Currently linked task is set to be unlinked. */
170 if (entry->linked) {
171 entry->linked->rt_param.linked_on = NO_CPU;
172 }
173
174 /* Link new task to CPU. */
175 if (linked) {
176 set_rt_flags(linked, RT_F_RUNNING);
177 /* handle task is already scheduled somewhere! */
178 on_cpu = linked->rt_param.scheduled_on;
179 if (on_cpu != NO_CPU) {
180 sched = &per_cpu(gsnedf_cpu_entries, on_cpu);
181 /* this should only happen if not linked already */
182 BUG_ON(sched->linked == linked);
183
184 /* If we are already scheduled on the CPU to which we
185 * wanted to link, we don't need to do the swap --
186 * we just link ourselves to the CPU and depend on
187 * the caller to get things right.
188 */
189 if (entry != sched) {
190 TRACE_TASK(linked,
191 "already scheduled on %d, updating link.\n",
192 sched->cpu);
193 tmp = sched->linked;
194 linked->rt_param.linked_on = sched->cpu;
195 sched->linked = linked;
196 update_cpu_position(sched);
197 linked = tmp;
198 }
199 }
200 if (linked) /* might be NULL due to swap */
201 linked->rt_param.linked_on = entry->cpu;
202 }
203 entry->linked = linked;
204#ifdef WANT_ALL_SCHED_EVENTS
205 if (linked)
206 TRACE_TASK(linked, "linked to %d.\n", entry->cpu);
207 else
208 TRACE("NULL linked to %d.\n", entry->cpu);
209#endif
210 update_cpu_position(entry);
211}
212
213/* unlink - Make sure a task is not linked any longer to an entry
214 * where it was linked before. Must hold gsnedf_lock.
215 */
216static noinline void unlink(struct task_struct* t)
217{
218 cpu_entry_t *entry;
219
220 if (unlikely(!t)) {
221 TRACE_BUG_ON(!t);
222 return;
223 }
224
225 if (t->rt_param.linked_on != NO_CPU) {
226 /* unlink */
227 entry = &per_cpu(gsnedf_cpu_entries, t->rt_param.linked_on);
228 t->rt_param.linked_on = NO_CPU;
229 link_task_to_cpu(NULL, entry);
230 } else if (is_queued(t)) {
231 /* This is an interesting situation: t is scheduled,
232 * but was just recently unlinked. It cannot be
233 * linked anywhere else (because then it would have
234 * been relinked to this CPU), thus it must be in some
235 * queue. We must remove it from the list in this
236 * case.
237 */
238 remove(&gsnedf, t);
239 }
240}
241
242
243/* preempt - force a CPU to reschedule
244 */
245static void preempt(cpu_entry_t *entry)
246{
247 preempt_if_preemptable(entry->scheduled, entry->cpu);
248}
249
250/* requeue - Put an unlinked task into gsn-edf domain.
251 * Caller must hold gsnedf_lock.
252 */
253static noinline void requeue(struct task_struct* task)
254{
255 BUG_ON(!task);
256 /* sanity check before insertion */
257 BUG_ON(is_queued(task));
258
259 if (is_released(task, litmus_clock()))
260 __add_ready(&gsnedf, task);
261 else {
262 /* it has got to wait */
263 add_release(&gsnedf, task);
264 }
265}
266
267/* check for any necessary preemptions */
268static void check_for_preemptions(void)
269{
270 struct task_struct *task;
271 cpu_entry_t* last;
272
273 for(last = lowest_prio_cpu();
274 edf_preemption_needed(&gsnedf, last->linked);
275 last = lowest_prio_cpu()) {
276 /* preemption necessary */
277 task = __take_ready(&gsnedf);
278 TRACE("check_for_preemptions: attempting to link task %d to %d\n",
279 task->pid, last->cpu);
280 if (last->linked)
281 requeue(last->linked);
282 link_task_to_cpu(task, last);
283 preempt(last);
284 }
285}
286
287/* gsnedf_job_arrival: task is either resumed or released */
288static noinline void gsnedf_job_arrival(struct task_struct* task)
289{
290 BUG_ON(!task);
291
292 requeue(task);
293 check_for_preemptions();
294}
295
296static void gsnedf_release_jobs(rt_domain_t* rt, struct bheap* tasks)
297{
298 unsigned long flags;
299
300 spin_lock_irqsave(&gsnedf_lock, flags);
301
302 __merge_ready(rt, tasks);
303 check_for_preemptions();
304
305 spin_unlock_irqrestore(&gsnedf_lock, flags);
306}
307
308/* caller holds gsnedf_lock */
309static noinline void job_completion(struct task_struct *t, int forced)
310{
311 BUG_ON(!t);
312
313 sched_trace_task_completion(t, forced);
314
315 TRACE_TASK(t, "job_completion().\n");
316
317 /* set flags */
318 set_rt_flags(t, RT_F_SLEEP);
319 /* prepare for next period */
320 prepare_for_next_period(t);
321 if (is_released(t, litmus_clock()))
322 sched_trace_task_release(t);
323 /* unlink */
324 unlink(t);
325 /* requeue
326 * But don't requeue a blocking task. */
327 if (is_running(t))
328 gsnedf_job_arrival(t);
329}
330
331/* gsnedf_tick - this function is called for every local timer
332 * interrupt.
333 *
334 * checks whether the current task has expired and checks
335 * whether we need to preempt it if it has not expired
336 */
337static void gsnedf_tick(struct task_struct* t)
338{
339 if (is_realtime(t) && budget_exhausted(t)) {
340 if (!is_np(t)) {
341 /* np tasks will be preempted when they become
342 * preemptable again
343 */
344 set_tsk_need_resched(t);
345 set_will_schedule();
346 TRACE("gsnedf_scheduler_tick: "
347 "%d is preemptable "
348 " => FORCE_RESCHED\n", t->pid);
349 } else if (is_user_np(t)) {
350 TRACE("gsnedf_scheduler_tick: "
351 "%d is non-preemptable, "
352 "preemption delayed.\n", t->pid);
353 request_exit_np(t);
354 }
355 }
356}
357
358/* Getting schedule() right is a bit tricky. schedule() may not make any
359 * assumptions on the state of the current task since it may be called for a
360 * number of reasons. The reasons include a scheduler_tick() determined that it
361 * was necessary, because sys_exit_np() was called, because some Linux
362 * subsystem determined so, or even (in the worst case) because there is a bug
363 * hidden somewhere. Thus, we must take extreme care to determine what the
364 * current state is.
365 *
366 * The CPU could currently be scheduling a task (or not), be linked (or not).
367 *
368 * The following assertions for the scheduled task could hold:
369 *
370 * - !is_running(scheduled) // the job blocks
371 * - scheduled->timeslice == 0 // the job completed (forcefully)
372 * - get_rt_flag() == RT_F_SLEEP // the job completed (by syscall)
373 * - linked != scheduled // we need to reschedule (for any reason)
374 * - is_np(scheduled) // rescheduling must be delayed,
375 * sys_exit_np must be requested
376 *
377 * Any of these can occur together.
378 */
379static struct task_struct* gsnedf_schedule(struct task_struct * prev)
380{
381 cpu_entry_t* entry = &__get_cpu_var(gsnedf_cpu_entries);
382 int out_of_time, sleep, preempt, np, exists, blocks;
383 struct task_struct* next = NULL;
384
385 /* Bail out early if we are the release master.
386 * The release master never schedules any real-time tasks.
387 */
388 if (gsnedf.release_master == entry->cpu)
389 return NULL;
390
391 spin_lock(&gsnedf_lock);
392 clear_will_schedule();
393
394 /* sanity checking */
395 BUG_ON(entry->scheduled && entry->scheduled != prev);
396 BUG_ON(entry->scheduled && !is_realtime(prev));
397 BUG_ON(is_realtime(prev) && !entry->scheduled);
398
399 /* (0) Determine state */
400 exists = entry->scheduled != NULL;
401 blocks = exists && !is_running(entry->scheduled);
402 out_of_time = exists && budget_exhausted(entry->scheduled);
403 np = exists && is_np(entry->scheduled);
404 sleep = exists && get_rt_flags(entry->scheduled) == RT_F_SLEEP;
405 preempt = entry->scheduled != entry->linked;
406
407#ifdef WANT_ALL_SCHED_EVENTS
408 TRACE_TASK(prev, "invoked gsnedf_schedule.\n");
409#endif
410
411 if (exists)
412 TRACE_TASK(prev,
413 "blocks:%d out_of_time:%d np:%d sleep:%d preempt:%d "
414 "state:%d sig:%d\n",
415 blocks, out_of_time, np, sleep, preempt,
416 prev->state, signal_pending(prev));
417 if (entry->linked && preempt)
418 TRACE_TASK(prev, "will be preempted by %s/%d\n",
419 entry->linked->comm, entry->linked->pid);
420
421
422 /* If a task blocks we have no choice but to reschedule.
423 */
424 if (blocks)
425 unlink(entry->scheduled);
426
427 /* Request a sys_exit_np() call if we would like to preempt but cannot.
428 * We need to make sure to update the link structure anyway in case
429 * that we are still linked. Multiple calls to request_exit_np() don't
430 * hurt.
431 */
432 if (np && (out_of_time || preempt || sleep)) {
433 unlink(entry->scheduled);
434 request_exit_np(entry->scheduled);
435 }
436
437 /* Any task that is preemptable and either exhausts its execution
438 * budget or wants to sleep completes. We may have to reschedule after
439 * this. Don't do a job completion if we block (can't have timers running
440 * for blocked jobs). Preemption go first for the same reason.
441 */
442 if (!np && (out_of_time || sleep) && !blocks && !preempt)
443 job_completion(entry->scheduled, !sleep);
444
445 /* Link pending task if we became unlinked.
446 */
447 if (!entry->linked)
448 link_task_to_cpu(__take_ready(&gsnedf), entry);
449
450 /* The final scheduling decision. Do we need to switch for some reason?
451 * If linked is different from scheduled, then select linked as next.
452 */
453 if ((!np || blocks) &&
454 entry->linked != entry->scheduled) {
455 /* Schedule a linked job? */
456 if (entry->linked) {
457 entry->linked->rt_param.scheduled_on = entry->cpu;
458 next = entry->linked;
459 }
460 if (entry->scheduled) {
461 /* not gonna be scheduled soon */
462 entry->scheduled->rt_param.scheduled_on = NO_CPU;
463 TRACE_TASK(entry->scheduled, "scheduled_on = NO_CPU\n");
464 }
465 } else
466 /* Only override Linux scheduler if we have a real-time task
467 * scheduled that needs to continue.
468 */
469 if (exists)
470 next = prev;
471
472 spin_unlock(&gsnedf_lock);
473
474#ifdef WANT_ALL_SCHED_EVENTS
475 TRACE("gsnedf_lock released, next=0x%p\n", next);
476
477 if (next)
478 TRACE_TASK(next, "scheduled at %llu\n", litmus_clock());
479 else if (exists && !next)
480 TRACE("becomes idle at %llu.\n", litmus_clock());
481#endif
482
483
484 return next;
485}
486
487
488/* _finish_switch - we just finished the switch away from prev
489 */
490static void gsnedf_finish_switch(struct task_struct *prev)
491{
492 cpu_entry_t* entry = &__get_cpu_var(gsnedf_cpu_entries);
493
494 entry->scheduled = is_realtime(current) ? current : NULL;
495#ifdef WANT_ALL_SCHED_EVENTS
496 TRACE_TASK(prev, "switched away from\n");
497#endif
498}
499
500
501/* Prepare a task for running in RT mode
502 */
503static void gsnedf_task_new(struct task_struct * t, int on_rq, int running)
504{
505 unsigned long flags;
506 cpu_entry_t* entry;
507
508 TRACE("gsn edf: task new %d\n", t->pid);
509
510 spin_lock_irqsave(&gsnedf_lock, flags);
511
512 /* setup job params */
513 release_at(t, litmus_clock());
514
515 if (running) {
516 entry = &per_cpu(gsnedf_cpu_entries, task_cpu(t));
517 BUG_ON(entry->scheduled);
518
519 if (entry->cpu != gsnedf.release_master) {
520 entry->scheduled = t;
521 tsk_rt(t)->scheduled_on = task_cpu(t);
522 } else {
523 /* do not schedule on release master */
524 preempt(entry); /* force resched */
525 tsk_rt(t)->scheduled_on = NO_CPU;
526 }
527 } else {
528 t->rt_param.scheduled_on = NO_CPU;
529 }
530 t->rt_param.linked_on = NO_CPU;
531
532 gsnedf_job_arrival(t);
533 spin_unlock_irqrestore(&gsnedf_lock, flags);
534}
535
536static void gsnedf_task_wake_up(struct task_struct *task)
537{
538 unsigned long flags;
539 lt_t now;
540
541 TRACE_TASK(task, "wake_up at %llu\n", litmus_clock());
542
543 spin_lock_irqsave(&gsnedf_lock, flags);
544 /* We need to take suspensions because of semaphores into
545 * account! If a job resumes after being suspended due to acquiring
546 * a semaphore, it should never be treated as a new job release.
547 */
548 if (get_rt_flags(task) == RT_F_EXIT_SEM) {
549 set_rt_flags(task, RT_F_RUNNING);
550 } else {
551 now = litmus_clock();
552 if (is_tardy(task, now)) {
553 /* new sporadic release */
554 release_at(task, now);
555 sched_trace_task_release(task);
556 }
557 else {
558 if (task->rt.time_slice) {
559 /* came back in time before deadline
560 */
561 set_rt_flags(task, RT_F_RUNNING);
562 }
563 }
564 }
565 gsnedf_job_arrival(task);
566 spin_unlock_irqrestore(&gsnedf_lock, flags);
567}
568
569static void gsnedf_task_block(struct task_struct *t)
570{
571 unsigned long flags;
572
573 TRACE_TASK(t, "block at %llu\n", litmus_clock());
574
575 /* unlink if necessary */
576 spin_lock_irqsave(&gsnedf_lock, flags);
577 unlink(t);
578 spin_unlock_irqrestore(&gsnedf_lock, flags);
579
580 BUG_ON(!is_realtime(t));
581}
582
583
584static void gsnedf_task_exit(struct task_struct * t)
585{
586 unsigned long flags;
587
588 /* unlink if necessary */
589 spin_lock_irqsave(&gsnedf_lock, flags);
590 unlink(t);
591 if (tsk_rt(t)->scheduled_on != NO_CPU) {
592 gsnedf_cpus[tsk_rt(t)->scheduled_on]->scheduled = NULL;
593 tsk_rt(t)->scheduled_on = NO_CPU;
594 }
595 spin_unlock_irqrestore(&gsnedf_lock, flags);
596
597 BUG_ON(!is_realtime(t));
598 TRACE_TASK(t, "RIP\n");
599}
600
601#ifdef CONFIG_FMLP
602
603/* Update the queue position of a task that got it's priority boosted via
604 * priority inheritance. */
605static void update_queue_position(struct task_struct *holder)
606{
607 /* We don't know whether holder is in the ready queue. It should, but
608 * on a budget overrun it may already be in a release queue. Hence,
609 * calling unlink() is not possible since it assumes that the task is
610 * not in a release queue. However, we can safely check whether
611 * sem->holder is currently in a queue or scheduled after locking both
612 * the release and the ready queue lock. */
613
614 /* Assumption: caller holds gsnedf_lock */
615
616 int check_preempt = 0;
617
618 if (tsk_rt(holder)->linked_on != NO_CPU) {
619 TRACE_TASK(holder, "%s: linked on %d\n",
620 __FUNCTION__, tsk_rt(holder)->linked_on);
621 /* Holder is scheduled; need to re-order CPUs.
622 * We can't use heap_decrease() here since
623 * the cpu_heap is ordered in reverse direction, so
624 * it is actually an increase. */
625 bheap_delete(cpu_lower_prio, &gsnedf_cpu_heap,
626 gsnedf_cpus[tsk_rt(holder)->linked_on]->hn);
627 bheap_insert(cpu_lower_prio, &gsnedf_cpu_heap,
628 gsnedf_cpus[tsk_rt(holder)->linked_on]->hn);
629 } else {
630 /* holder may be queued: first stop queue changes */
631 spin_lock(&gsnedf.release_lock);
632 if (is_queued(holder)) {
633 TRACE_TASK(holder, "%s: is queued\n",
634 __FUNCTION__);
635 /* We need to update the position
636 * of holder in some heap. Note that this
637 * may be a release heap. */
638 check_preempt =
639 !bheap_decrease(edf_ready_order,
640 tsk_rt(holder)->heap_node);
641 } else {
642 /* Nothing to do: if it is not queued and not linked
643 * then it is currently being moved by other code
644 * (e.g., a timer interrupt handler) that will use the
645 * correct priority when enqueuing the task. */
646 TRACE_TASK(holder, "%s: is NOT queued => Done.\n",
647 __FUNCTION__);
648 }
649 spin_unlock(&gsnedf.release_lock);
650
651 /* If holder was enqueued in a release heap, then the following
652 * preemption check is pointless, but we can't easily detect
653 * that case. If you want to fix this, then consider that
654 * simply adding a state flag requires O(n) time to update when
655 * releasing n tasks, which conflicts with the goal to have
656 * O(log n) merges. */
657 if (check_preempt) {
658 /* heap_decrease() hit the top level of the heap: make
659 * sure preemption checks get the right task, not the
660 * potentially stale cache. */
661 bheap_uncache_min(edf_ready_order,
662 &gsnedf.ready_queue);
663 check_for_preemptions();
664 }
665 }
666}
667
668static long gsnedf_pi_block(struct pi_semaphore *sem,
669 struct task_struct *new_waiter)
670{
671 /* This callback has to handle the situation where a new waiter is
672 * added to the wait queue of the semaphore.
673 *
674 * We must check if has a higher priority than the currently
675 * highest-priority task, and then potentially reschedule.
676 */
677
678 BUG_ON(!new_waiter);
679
680 if (edf_higher_prio(new_waiter, sem->hp.task)) {
681 TRACE_TASK(new_waiter, " boosts priority via %p\n", sem);
682 /* called with IRQs disabled */
683 spin_lock(&gsnedf_lock);
684 /* store new highest-priority task */
685 sem->hp.task = new_waiter;
686 if (sem->holder) {
687 TRACE_TASK(sem->holder,
688 " holds %p and will inherit from %s/%d\n",
689 sem,
690 new_waiter->comm, new_waiter->pid);
691 /* let holder inherit */
692 sem->holder->rt_param.inh_task = new_waiter;
693 update_queue_position(sem->holder);
694 }
695 spin_unlock(&gsnedf_lock);
696 }
697
698 return 0;
699}
700
701static long gsnedf_inherit_priority(struct pi_semaphore *sem,
702 struct task_struct *new_owner)
703{
704 /* We don't need to acquire the gsnedf_lock since at the time of this
705 * call new_owner isn't actually scheduled yet (it's still sleeping)
706 * and since the calling function already holds sem->wait.lock, which
707 * prevents concurrent sem->hp.task changes.
708 */
709
710 if (sem->hp.task && sem->hp.task != new_owner) {
711 new_owner->rt_param.inh_task = sem->hp.task;
712 TRACE_TASK(new_owner, "inherited priority from %s/%d\n",
713 sem->hp.task->comm, sem->hp.task->pid);
714 } else
715 TRACE_TASK(new_owner,
716 "cannot inherit priority, "
717 "no higher priority job waits.\n");
718 return 0;
719}
720
721/* This function is called on a semaphore release, and assumes that
722 * the current task is also the semaphore holder.
723 */
724static long gsnedf_return_priority(struct pi_semaphore *sem)
725{
726 struct task_struct* t = current;
727 int ret = 0;
728
729 /* Find new highest-priority semaphore task
730 * if holder task is the current hp.task.
731 *
732 * Calling function holds sem->wait.lock.
733 */
734 if (t == sem->hp.task)
735 edf_set_hp_task(sem);
736
737 TRACE_CUR("gsnedf_return_priority for lock %p\n", sem);
738
739 if (t->rt_param.inh_task) {
740 /* interrupts already disabled by PI code */
741 spin_lock(&gsnedf_lock);
742
743 /* Reset inh_task to NULL. */
744 t->rt_param.inh_task = NULL;
745
746 /* Check if rescheduling is necessary */
747 unlink(t);
748 gsnedf_job_arrival(t);
749 spin_unlock(&gsnedf_lock);
750 }
751
752 return ret;
753}
754
755#endif
756
757static long gsnedf_admit_task(struct task_struct* tsk)
758{
759 return 0;
760}
761
762static long gsnedf_activate_plugin(void)
763{
764 int cpu;
765 cpu_entry_t *entry;
766
767 bheap_init(&gsnedf_cpu_heap);
768 gsnedf.release_master = atomic_read(&release_master_cpu);
769
770 for_each_online_cpu(cpu) {
771 entry = &per_cpu(gsnedf_cpu_entries, cpu);
772 bheap_node_init(&entry->hn, entry);
773 atomic_set(&entry->will_schedule, 0);
774 entry->linked = NULL;
775 entry->scheduled = NULL;
776 if (cpu != gsnedf.release_master) {
777 TRACE("GSN-EDF: Initializing CPU #%d.\n", cpu);
778 update_cpu_position(entry);
779 } else {
780 TRACE("GSN-EDF: CPU %d is release master.\n", cpu);
781 }
782 }
783 return 0;
784}
785
786/* Plugin object */
787static struct sched_plugin gsn_edf_plugin __cacheline_aligned_in_smp = {
788 .plugin_name = "GSN-EDF",
789 .finish_switch = gsnedf_finish_switch,
790 .tick = gsnedf_tick,
791 .task_new = gsnedf_task_new,
792 .complete_job = complete_job,
793 .task_exit = gsnedf_task_exit,
794 .schedule = gsnedf_schedule,
795 .task_wake_up = gsnedf_task_wake_up,
796 .task_block = gsnedf_task_block,
797#ifdef CONFIG_FMLP
798 .fmlp_active = 1,
799 .pi_block = gsnedf_pi_block,
800 .inherit_priority = gsnedf_inherit_priority,
801 .return_priority = gsnedf_return_priority,
802#endif
803 .admit_task = gsnedf_admit_task,
804 .activate_plugin = gsnedf_activate_plugin,
805};
806
807
808static int __init init_gsn_edf(void)
809{
810 int cpu;
811 cpu_entry_t *entry;
812
813 bheap_init(&gsnedf_cpu_heap);
814 /* initialize CPU state */
815 for (cpu = 0; cpu < NR_CPUS; cpu++) {
816 entry = &per_cpu(gsnedf_cpu_entries, cpu);
817 gsnedf_cpus[cpu] = entry;
818 atomic_set(&entry->will_schedule, 0);
819 entry->cpu = cpu;
820 entry->hn = &gsnedf_heap_node[cpu];
821 bheap_node_init(&entry->hn, entry);
822 }
823 edf_domain_init(&gsnedf, NULL, gsnedf_release_jobs);
824 return register_sched_plugin(&gsn_edf_plugin);
825}
826
827
828module_init(init_gsn_edf);
diff --git a/litmus/sched_litmus.c b/litmus/sched_litmus.c
new file mode 100644
index 000000000000..c1fc7748e590
--- /dev/null
+++ b/litmus/sched_litmus.c
@@ -0,0 +1,318 @@
1/* This file is included from kernel/sched.c */
2
3#include <litmus/litmus.h>
4#include <litmus/sched_plugin.h>
5
6static void update_time_litmus(struct rq *rq, struct task_struct *p)
7{
8 u64 delta = rq->clock - p->se.exec_start;
9 if (unlikely((s64)delta < 0))
10 delta = 0;
11 /* per job counter */
12 p->rt_param.job_params.exec_time += delta;
13 /* task counter */
14 p->se.sum_exec_runtime += delta;
15 /* sched_clock() */
16 p->se.exec_start = rq->clock;
17 cpuacct_charge(p, delta);
18}
19
20static void double_rq_lock(struct rq *rq1, struct rq *rq2);
21static void double_rq_unlock(struct rq *rq1, struct rq *rq2);
22
23/*
24 * litmus_tick gets called by scheduler_tick() with HZ freq
25 * Interrupts are disabled
26 */
27static void litmus_tick(struct rq *rq, struct task_struct *p)
28{
29 TS_PLUGIN_TICK_START;
30
31 if (is_realtime(p))
32 update_time_litmus(rq, p);
33
34 /* plugin tick */
35 litmus->tick(p);
36
37 return;
38}
39
40static struct task_struct *
41litmus_schedule(struct rq *rq, struct task_struct *prev)
42{
43 struct rq* other_rq;
44 struct task_struct *next;
45
46 long was_running;
47 lt_t _maybe_deadlock = 0;
48
49 /* let the plugin schedule */
50 next = litmus->schedule(prev);
51
52 /* check if a global plugin pulled a task from a different RQ */
53 if (next && task_rq(next) != rq) {
54 /* we need to migrate the task */
55 other_rq = task_rq(next);
56 TRACE_TASK(next, "migrate from %d\n", other_rq->cpu);
57
58 /* while we drop the lock, the prev task could change its
59 * state
60 */
61 was_running = is_running(prev);
62 mb();
63 spin_unlock(&rq->lock);
64
65 /* Don't race with a concurrent switch. This could deadlock in
66 * the case of cross or circular migrations. It's the job of
67 * the plugin to make sure that doesn't happen.
68 */
69 TRACE_TASK(next, "stack_in_use=%d\n",
70 next->rt_param.stack_in_use);
71 if (next->rt_param.stack_in_use != NO_CPU) {
72 TRACE_TASK(next, "waiting to deschedule\n");
73 _maybe_deadlock = litmus_clock();
74 }
75 while (next->rt_param.stack_in_use != NO_CPU) {
76 cpu_relax();
77 mb();
78 if (next->rt_param.stack_in_use == NO_CPU)
79 TRACE_TASK(next,"descheduled. Proceeding.\n");
80
81 if (lt_before(_maybe_deadlock + 10000000,
82 litmus_clock())) {
83 /* We've been spinning for 10ms.
84 * Something can't be right!
85 * Let's abandon the task and bail out; at least
86 * we will have debug info instead of a hard
87 * deadlock.
88 */
89 TRACE_TASK(next,"stack too long in use. "
90 "Deadlock?\n");
91 next = NULL;
92
93 /* bail out */
94 spin_lock(&rq->lock);
95 return next;
96 }
97 }
98#ifdef __ARCH_WANT_UNLOCKED_CTXSW
99 if (next->oncpu)
100 TRACE_TASK(next, "waiting for !oncpu");
101 while (next->oncpu) {
102 cpu_relax();
103 mb();
104 }
105#endif
106 double_rq_lock(rq, other_rq);
107 mb();
108 if (is_realtime(prev) && is_running(prev) != was_running) {
109 TRACE_TASK(prev,
110 "state changed while we dropped"
111 " the lock: is_running=%d, was_running=%d\n",
112 is_running(prev), was_running);
113 if (is_running(prev) && !was_running) {
114 /* prev task became unblocked
115 * we need to simulate normal sequence of events
116 * to scheduler plugins.
117 */
118 litmus->task_block(prev);
119 litmus->task_wake_up(prev);
120 }
121 }
122
123 set_task_cpu(next, smp_processor_id());
124
125 /* DEBUG: now that we have the lock we need to make sure a
126 * couple of things still hold:
127 * - it is still a real-time task
128 * - it is still runnable (could have been stopped)
129 * If either is violated, then the active plugin is
130 * doing something wrong.
131 */
132 if (!is_realtime(next) || !is_running(next)) {
133 /* BAD BAD BAD */
134 TRACE_TASK(next,"BAD: migration invariant FAILED: "
135 "rt=%d running=%d\n",
136 is_realtime(next),
137 is_running(next));
138 /* drop the task */
139 next = NULL;
140 }
141 /* release the other CPU's runqueue, but keep ours */
142 spin_unlock(&other_rq->lock);
143 }
144 if (next) {
145 next->rt_param.stack_in_use = rq->cpu;
146 next->se.exec_start = rq->clock;
147 }
148
149 return next;
150}
151
152static void enqueue_task_litmus(struct rq *rq, struct task_struct *p,
153 int wakeup)
154{
155 if (wakeup) {
156 sched_trace_task_resume(p);
157 tsk_rt(p)->present = 1;
158 litmus->task_wake_up(p);
159
160 rq->litmus.nr_running++;
161 } else
162 TRACE_TASK(p, "ignoring an enqueue, not a wake up.\n");
163}
164
165static void dequeue_task_litmus(struct rq *rq, struct task_struct *p, int sleep)
166{
167 if (sleep) {
168 litmus->task_block(p);
169 tsk_rt(p)->present = 0;
170 sched_trace_task_block(p);
171
172 rq->litmus.nr_running--;
173 } else
174 TRACE_TASK(p, "ignoring a dequeue, not going to sleep.\n");
175}
176
177static void yield_task_litmus(struct rq *rq)
178{
179 BUG_ON(rq->curr != current);
180 /* sched_yield() is called to trigger delayed preemptions.
181 * Thus, mark the current task as needing to be rescheduled.
182 * This will cause the scheduler plugin to be invoked, which can
183 * then determine if a preemption is still required.
184 */
185 clear_exit_np(current);
186 set_tsk_need_resched(current);
187}
188
189/* Plugins are responsible for this.
190 */
191static void check_preempt_curr_litmus(struct rq *rq, struct task_struct *p, int flags)
192{
193}
194
195static void put_prev_task_litmus(struct rq *rq, struct task_struct *p)
196{
197}
198
199static void pre_schedule_litmus(struct rq *rq, struct task_struct *prev)
200{
201 update_time_litmus(rq, prev);
202 if (!is_running(prev))
203 tsk_rt(prev)->present = 0;
204}
205
206/* pick_next_task_litmus() - litmus_schedule() function
207 *
208 * return the next task to be scheduled
209 */
210static struct task_struct *pick_next_task_litmus(struct rq *rq)
211{
212 /* get the to-be-switched-out task (prev) */
213 struct task_struct *prev = rq->litmus.prev;
214 struct task_struct *next;
215
216 /* if not called from schedule() but from somewhere
217 * else (e.g., migration), return now!
218 */
219 if(!rq->litmus.prev)
220 return NULL;
221
222 rq->litmus.prev = NULL;
223
224 TS_PLUGIN_SCHED_START;
225 next = litmus_schedule(rq, prev);
226 TS_PLUGIN_SCHED_END;
227
228 return next;
229}
230
231static void task_tick_litmus(struct rq *rq, struct task_struct *p, int queued)
232{
233 /* nothing to do; tick related tasks are done by litmus_tick() */
234 return;
235}
236
237static void switched_to_litmus(struct rq *rq, struct task_struct *p, int running)
238{
239}
240
241static void prio_changed_litmus(struct rq *rq, struct task_struct *p,
242 int oldprio, int running)
243{
244}
245
246unsigned int get_rr_interval_litmus(struct task_struct *p)
247{
248 /* return infinity */
249 return 0;
250}
251
252/* This is called when a task became a real-time task, either due to a SCHED_*
253 * class transition or due to PI mutex inheritance. We don't handle Linux PI
254 * mutex inheritance yet (and probably never will). Use LITMUS provided
255 * synchronization primitives instead.
256 */
257static void set_curr_task_litmus(struct rq *rq)
258{
259 rq->curr->se.exec_start = rq->clock;
260}
261
262
263#ifdef CONFIG_SMP
264/* execve tries to rebalance task in this scheduling domain */
265static int select_task_rq_litmus(struct task_struct *p, int sd_flag, int flags)
266{
267 /* preemption is already disabled.
268 * We don't want to change cpu here
269 */
270 return smp_processor_id();
271}
272
273/* we don't repartition at runtime */
274
275static unsigned long
276load_balance_litmus(struct rq *this_rq, int this_cpu, struct rq *busiest,
277 unsigned long max_load_move,
278 struct sched_domain *sd, enum cpu_idle_type idle,
279 int *all_pinned, int *this_best_prio)
280{
281 return 0;
282}
283
284static int
285move_one_task_litmus(struct rq *this_rq, int this_cpu, struct rq *busiest,
286 struct sched_domain *sd, enum cpu_idle_type idle)
287{
288 return 0;
289}
290#endif
291
292const struct sched_class litmus_sched_class = {
293 .next = &rt_sched_class,
294 .enqueue_task = enqueue_task_litmus,
295 .dequeue_task = dequeue_task_litmus,
296 .yield_task = yield_task_litmus,
297
298 .check_preempt_curr = check_preempt_curr_litmus,
299
300 .pick_next_task = pick_next_task_litmus,
301 .put_prev_task = put_prev_task_litmus,
302
303#ifdef CONFIG_SMP
304 .select_task_rq = select_task_rq_litmus,
305
306 .load_balance = load_balance_litmus,
307 .move_one_task = move_one_task_litmus,
308 .pre_schedule = pre_schedule_litmus,
309#endif
310
311 .set_curr_task = set_curr_task_litmus,
312 .task_tick = task_tick_litmus,
313
314 .get_rr_interval = get_rr_interval_litmus,
315
316 .prio_changed = prio_changed_litmus,
317 .switched_to = switched_to_litmus,
318};
diff --git a/litmus/sched_pfair.c b/litmus/sched_pfair.c
new file mode 100644
index 000000000000..2ea39223e7f0
--- /dev/null
+++ b/litmus/sched_pfair.c
@@ -0,0 +1,896 @@
1/*
2 * kernel/sched_pfair.c
3 *
4 * Implementation of the (global) Pfair scheduling algorithm.
5 *
6 */
7
8#include <asm/div64.h>
9#include <linux/delay.h>
10#include <linux/module.h>
11#include <linux/spinlock.h>
12#include <linux/percpu.h>
13#include <linux/sched.h>
14#include <linux/list.h>
15
16#include <litmus/litmus.h>
17#include <litmus/jobs.h>
18#include <litmus/rt_domain.h>
19#include <litmus/sched_plugin.h>
20#include <litmus/sched_trace.h>
21
22#include <litmus/bheap.h>
23
24struct subtask {
25 /* measured in quanta relative to job release */
26 quanta_t release;
27 quanta_t deadline;
28 quanta_t overlap; /* called "b bit" by PD^2 */
29 quanta_t group_deadline;
30};
31
32struct pfair_param {
33 quanta_t quanta; /* number of subtasks */
34 quanta_t cur; /* index of current subtask */
35
36 quanta_t release; /* in quanta */
37 quanta_t period; /* in quanta */
38
39 quanta_t last_quantum; /* when scheduled last */
40 int last_cpu; /* where scheduled last */
41
42 unsigned int sporadic_release; /* On wakeup, new sporadic release? */
43
44 struct subtask subtasks[0]; /* allocate together with pfair_param */
45};
46
47#define tsk_pfair(tsk) ((tsk)->rt_param.pfair)
48
49struct pfair_state {
50 int cpu;
51 volatile quanta_t cur_tick; /* updated by the CPU that is advancing
52 * the time */
53 volatile quanta_t local_tick; /* What tick is the local CPU currently
54 * executing? Updated only by the local
55 * CPU. In QEMU, this may lag behind the
56 * current tick. In a real system, with
57 * proper timers and aligned quanta,
58 * that should only be the
59 * case for a very short time after the
60 * time advanced. With staggered quanta,
61 * it will lag for the duration of the
62 * offset.
63 */
64
65 struct task_struct* linked; /* the task that should be executing */
66 struct task_struct* local; /* the local copy of linked */
67 struct task_struct* scheduled; /* what is actually scheduled */
68
69 unsigned long missed_quanta;
70 lt_t offset; /* stagger offset */
71};
72
73/* Currently, we limit the maximum period of any task to 2000 quanta.
74 * The reason is that it makes the implementation easier since we do not
75 * need to reallocate the release wheel on task arrivals.
76 * In the future
77 */
78#define PFAIR_MAX_PERIOD 2000
79
80/* This is the release queue wheel. It is indexed by pfair_time %
81 * PFAIR_MAX_PERIOD. Each heap is ordered by PFAIR priority, so that it can be
82 * merged with the ready queue.
83 */
84static struct bheap release_queue[PFAIR_MAX_PERIOD];
85
86DEFINE_PER_CPU(struct pfair_state, pfair_state);
87struct pfair_state* *pstate; /* short cut */
88
89static quanta_t pfair_time = 0; /* the "official" PFAIR clock */
90static quanta_t merge_time = 0; /* Updated after the release queue has been
91 * merged. Used by drop_all_references().
92 */
93
94static rt_domain_t pfair;
95
96/* The pfair_lock is used to serialize all scheduling events.
97 */
98#define pfair_lock pfair.ready_lock
99
100/* Enable for lots of trace info.
101 * #define PFAIR_DEBUG
102 */
103
104#ifdef PFAIR_DEBUG
105#define PTRACE_TASK(t, f, args...) TRACE_TASK(t, f, ## args)
106#define PTRACE(f, args...) TRACE(f, ## args)
107#else
108#define PTRACE_TASK(t, f, args...)
109#define PTRACE(f, args...)
110#endif
111
112/* gcc will inline all of these accessor functions... */
113static struct subtask* cur_subtask(struct task_struct* t)
114{
115 return tsk_pfair(t)->subtasks + tsk_pfair(t)->cur;
116}
117
118static quanta_t cur_deadline(struct task_struct* t)
119{
120 return cur_subtask(t)->deadline + tsk_pfair(t)->release;
121}
122
123
124static quanta_t cur_sub_release(struct task_struct* t)
125{
126 return cur_subtask(t)->release + tsk_pfair(t)->release;
127}
128
129static quanta_t cur_release(struct task_struct* t)
130{
131#ifdef EARLY_RELEASE
132 /* only the release of the first subtask counts when we early
133 * release */
134 return tsk_pfair(t)->release;
135#else
136 return cur_sub_release(t);
137#endif
138}
139
140static quanta_t cur_overlap(struct task_struct* t)
141{
142 return cur_subtask(t)->overlap;
143}
144
145static quanta_t cur_group_deadline(struct task_struct* t)
146{
147 quanta_t gdl = cur_subtask(t)->group_deadline;
148 if (gdl)
149 return gdl + tsk_pfair(t)->release;
150 else
151 return gdl;
152}
153
154
155static int pfair_higher_prio(struct task_struct* first,
156 struct task_struct* second)
157{
158 return /* first task must exist */
159 first && (
160 /* Does the second task exist and is it a real-time task? If
161 * not, the first task (which is a RT task) has higher
162 * priority.
163 */
164 !second || !is_realtime(second) ||
165
166 /* Is the (subtask) deadline of the first task earlier?
167 * Then it has higher priority.
168 */
169 time_before(cur_deadline(first), cur_deadline(second)) ||
170
171 /* Do we have a deadline tie?
172 * Then break by B-bit.
173 */
174 (cur_deadline(first) == cur_deadline(second) &&
175 (cur_overlap(first) > cur_overlap(second) ||
176
177 /* Do we have a B-bit tie?
178 * Then break by group deadline.
179 */
180 (cur_overlap(first) == cur_overlap(second) &&
181 (time_after(cur_group_deadline(first),
182 cur_group_deadline(second)) ||
183
184 /* Do we have a group deadline tie?
185 * Then break by PID, which are unique.
186 */
187 (cur_group_deadline(first) ==
188 cur_group_deadline(second) &&
189 first->pid < second->pid))))));
190}
191
192int pfair_ready_order(struct bheap_node* a, struct bheap_node* b)
193{
194 return pfair_higher_prio(bheap2task(a), bheap2task(b));
195}
196
197/* return the proper release queue for time t */
198static struct bheap* relq(quanta_t t)
199{
200 struct bheap* rq = &release_queue[t % PFAIR_MAX_PERIOD];
201 return rq;
202}
203
204static void prepare_release(struct task_struct* t, quanta_t at)
205{
206 tsk_pfair(t)->release = at;
207 tsk_pfair(t)->cur = 0;
208}
209
210static void __pfair_add_release(struct task_struct* t, struct bheap* queue)
211{
212 bheap_insert(pfair_ready_order, queue,
213 tsk_rt(t)->heap_node);
214}
215
216static void pfair_add_release(struct task_struct* t)
217{
218 BUG_ON(bheap_node_in_heap(tsk_rt(t)->heap_node));
219 __pfair_add_release(t, relq(cur_release(t)));
220}
221
222/* pull released tasks from the release queue */
223static void poll_releases(quanta_t time)
224{
225 __merge_ready(&pfair, relq(time));
226 merge_time = time;
227}
228
229static void check_preempt(struct task_struct* t)
230{
231 int cpu = NO_CPU;
232 if (tsk_rt(t)->linked_on != tsk_rt(t)->scheduled_on &&
233 tsk_rt(t)->present) {
234 /* the task can be scheduled and
235 * is not scheduled where it ought to be scheduled
236 */
237 cpu = tsk_rt(t)->linked_on != NO_CPU ?
238 tsk_rt(t)->linked_on :
239 tsk_rt(t)->scheduled_on;
240 PTRACE_TASK(t, "linked_on:%d, scheduled_on:%d\n",
241 tsk_rt(t)->linked_on, tsk_rt(t)->scheduled_on);
242 /* preempt */
243 if (cpu == smp_processor_id())
244 set_tsk_need_resched(current);
245 else {
246 smp_send_reschedule(cpu);
247 }
248 }
249}
250
251/* caller must hold pfair_lock */
252static void drop_all_references(struct task_struct *t)
253{
254 int cpu;
255 struct pfair_state* s;
256 struct bheap* q;
257 if (bheap_node_in_heap(tsk_rt(t)->heap_node)) {
258 /* figure out what queue the node is in */
259 if (time_before_eq(cur_release(t), merge_time))
260 q = &pfair.ready_queue;
261 else
262 q = relq(cur_release(t));
263 bheap_delete(pfair_ready_order, q,
264 tsk_rt(t)->heap_node);
265 }
266 for (cpu = 0; cpu < num_online_cpus(); cpu++) {
267 s = &per_cpu(pfair_state, cpu);
268 if (s->linked == t)
269 s->linked = NULL;
270 if (s->local == t)
271 s->local = NULL;
272 if (s->scheduled == t)
273 s->scheduled = NULL;
274 }
275}
276
277/* returns 1 if the task needs to go the release queue */
278static int advance_subtask(quanta_t time, struct task_struct* t, int cpu)
279{
280 struct pfair_param* p = tsk_pfair(t);
281 int to_relq;
282 p->cur = (p->cur + 1) % p->quanta;
283 if (!p->cur) {
284 sched_trace_task_completion(t, 1);
285 if (tsk_rt(t)->present) {
286 /* we start a new job */
287 prepare_for_next_period(t);
288 sched_trace_task_release(t);
289 get_rt_flags(t) = RT_F_RUNNING;
290 p->release += p->period;
291 } else {
292 /* remove task from system until it wakes */
293 drop_all_references(t);
294 tsk_pfair(t)->sporadic_release = 1;
295 TRACE_TASK(t, "on %d advanced to subtask %lu (not present)\n",
296 cpu, p->cur);
297 return 0;
298 }
299 }
300 to_relq = time_after(cur_release(t), time);
301 TRACE_TASK(t, "on %d advanced to subtask %lu -> to_relq=%d\n",
302 cpu, p->cur, to_relq);
303 return to_relq;
304}
305
306static void advance_subtasks(quanta_t time)
307{
308 int cpu, missed;
309 struct task_struct* l;
310 struct pfair_param* p;
311
312 for_each_online_cpu(cpu) {
313 l = pstate[cpu]->linked;
314 missed = pstate[cpu]->linked != pstate[cpu]->local;
315 if (l) {
316 p = tsk_pfair(l);
317 p->last_quantum = time;
318 p->last_cpu = cpu;
319 if (advance_subtask(time, l, cpu)) {
320 pstate[cpu]->linked = NULL;
321 pfair_add_release(l);
322 }
323 }
324 }
325}
326
327static int target_cpu(quanta_t time, struct task_struct* t, int default_cpu)
328{
329 int cpu;
330 if (tsk_rt(t)->scheduled_on != NO_CPU) {
331 /* always observe scheduled_on linkage */
332 default_cpu = tsk_rt(t)->scheduled_on;
333 } else if (tsk_pfair(t)->last_quantum == time - 1) {
334 /* back2back quanta */
335 /* Only observe last_quantum if no scheduled_on is in the way.
336 * This should only kick in if a CPU missed quanta, and that
337 * *should* only happen in QEMU.
338 */
339 cpu = tsk_pfair(t)->last_cpu;
340 if (!pstate[cpu]->linked ||
341 tsk_rt(pstate[cpu]->linked)->scheduled_on != cpu) {
342 default_cpu = cpu;
343 }
344 }
345 return default_cpu;
346}
347
348/* returns one if linking was redirected */
349static int pfair_link(quanta_t time, int cpu,
350 struct task_struct* t)
351{
352 int target = target_cpu(time, t, cpu);
353 struct task_struct* prev = pstate[cpu]->linked;
354 struct task_struct* other;
355
356 if (target != cpu) {
357 other = pstate[target]->linked;
358 pstate[target]->linked = t;
359 tsk_rt(t)->linked_on = target;
360 if (!other)
361 /* linked ok, but reschedule this CPU */
362 return 1;
363 if (target < cpu) {
364 /* link other to cpu instead */
365 tsk_rt(other)->linked_on = cpu;
366 pstate[cpu]->linked = other;
367 if (prev) {
368 /* prev got pushed back into the ready queue */
369 tsk_rt(prev)->linked_on = NO_CPU;
370 __add_ready(&pfair, prev);
371 }
372 /* we are done with this cpu */
373 return 0;
374 } else {
375 /* re-add other, it's original CPU was not considered yet */
376 tsk_rt(other)->linked_on = NO_CPU;
377 __add_ready(&pfair, other);
378 /* reschedule this CPU */
379 return 1;
380 }
381 } else {
382 pstate[cpu]->linked = t;
383 tsk_rt(t)->linked_on = cpu;
384 if (prev) {
385 /* prev got pushed back into the ready queue */
386 tsk_rt(prev)->linked_on = NO_CPU;
387 __add_ready(&pfair, prev);
388 }
389 /* we are done with this CPU */
390 return 0;
391 }
392}
393
394static void schedule_subtasks(quanta_t time)
395{
396 int cpu, retry;
397
398 for_each_online_cpu(cpu) {
399 retry = 1;
400 while (retry) {
401 if (pfair_higher_prio(__peek_ready(&pfair),
402 pstate[cpu]->linked))
403 retry = pfair_link(time, cpu,
404 __take_ready(&pfair));
405 else
406 retry = 0;
407 }
408 }
409}
410
411static void schedule_next_quantum(quanta_t time)
412{
413 int cpu;
414
415 /* called with interrupts disabled */
416 PTRACE("--- Q %lu at %llu PRE-SPIN\n",
417 time, litmus_clock());
418 spin_lock(&pfair_lock);
419 PTRACE("<<< Q %lu at %llu\n",
420 time, litmus_clock());
421
422 sched_trace_quantum_boundary();
423
424 advance_subtasks(time);
425 poll_releases(time);
426 schedule_subtasks(time);
427
428 for (cpu = 0; cpu < num_online_cpus(); cpu++)
429 if (pstate[cpu]->linked)
430 PTRACE_TASK(pstate[cpu]->linked,
431 " linked on %d.\n", cpu);
432 else
433 PTRACE("(null) linked on %d.\n", cpu);
434
435 /* We are done. Advance time. */
436 mb();
437 for (cpu = 0; cpu < num_online_cpus(); cpu++) {
438 if (pstate[cpu]->local_tick != pstate[cpu]->cur_tick) {
439 TRACE("BAD Quantum not acked on %d "
440 "(l:%lu c:%lu p:%lu)\n",
441 cpu,
442 pstate[cpu]->local_tick,
443 pstate[cpu]->cur_tick,
444 pfair_time);
445 pstate[cpu]->missed_quanta++;
446 }
447 pstate[cpu]->cur_tick = time;
448 }
449 PTRACE(">>> Q %lu at %llu\n",
450 time, litmus_clock());
451 spin_unlock(&pfair_lock);
452}
453
454static noinline void wait_for_quantum(quanta_t q, struct pfair_state* state)
455{
456 quanta_t loc;
457
458 goto first; /* skip mb() on first iteration */
459 do {
460 cpu_relax();
461 mb();
462 first: loc = state->cur_tick;
463 /* FIXME: what if loc > cur? */
464 } while (time_before(loc, q));
465 PTRACE("observed cur_tick:%lu >= q:%lu\n",
466 loc, q);
467}
468
469static quanta_t current_quantum(struct pfair_state* state)
470{
471 lt_t t = litmus_clock() - state->offset;
472 return time2quanta(t, FLOOR);
473}
474
475static void catchup_quanta(quanta_t from, quanta_t target,
476 struct pfair_state* state)
477{
478 quanta_t cur = from, time;
479 TRACE("+++< BAD catching up quanta from %lu to %lu\n",
480 from, target);
481 while (time_before(cur, target)) {
482 wait_for_quantum(cur, state);
483 cur++;
484 time = cmpxchg(&pfair_time,
485 cur - 1, /* expected */
486 cur /* next */
487 );
488 if (time == cur - 1)
489 schedule_next_quantum(cur);
490 }
491 TRACE("+++> catching up done\n");
492}
493
494/* pfair_tick - this function is called for every local timer
495 * interrupt.
496 */
497static void pfair_tick(struct task_struct* t)
498{
499 struct pfair_state* state = &__get_cpu_var(pfair_state);
500 quanta_t time, cur;
501 int retry = 10;
502
503 do {
504 cur = current_quantum(state);
505 PTRACE("q %lu at %llu\n", cur, litmus_clock());
506
507 /* Attempt to advance time. First CPU to get here
508 * will prepare the next quantum.
509 */
510 time = cmpxchg(&pfair_time,
511 cur - 1, /* expected */
512 cur /* next */
513 );
514 if (time == cur - 1) {
515 /* exchange succeeded */
516 wait_for_quantum(cur - 1, state);
517 schedule_next_quantum(cur);
518 retry = 0;
519 } else if (time_before(time, cur - 1)) {
520 /* the whole system missed a tick !? */
521 catchup_quanta(time, cur, state);
522 retry--;
523 } else if (time_after(time, cur)) {
524 /* our timer lagging behind!? */
525 TRACE("BAD pfair_time:%lu > cur:%lu\n", time, cur);
526 retry--;
527 } else {
528 /* Some other CPU already started scheduling
529 * this quantum. Let it do its job and then update.
530 */
531 retry = 0;
532 }
533 } while (retry);
534
535 /* Spin locally until time advances. */
536 wait_for_quantum(cur, state);
537
538 /* copy assignment */
539 /* FIXME: what if we race with a future update? Corrupted state? */
540 state->local = state->linked;
541 /* signal that we are done */
542 mb();
543 state->local_tick = state->cur_tick;
544
545 if (state->local != current
546 && (is_realtime(current) || is_present(state->local)))
547 set_tsk_need_resched(current);
548}
549
550static int safe_to_schedule(struct task_struct* t, int cpu)
551{
552 int where = tsk_rt(t)->scheduled_on;
553 if (where != NO_CPU && where != cpu) {
554 TRACE_TASK(t, "BAD: can't be scheduled on %d, "
555 "scheduled already on %d.\n", cpu, where);
556 return 0;
557 } else
558 return tsk_rt(t)->present && get_rt_flags(t) == RT_F_RUNNING;
559}
560
561static struct task_struct* pfair_schedule(struct task_struct * prev)
562{
563 struct pfair_state* state = &__get_cpu_var(pfair_state);
564 int blocks;
565 struct task_struct* next = NULL;
566
567 spin_lock(&pfair_lock);
568
569 blocks = is_realtime(prev) && !is_running(prev);
570
571 if (state->local && safe_to_schedule(state->local, state->cpu))
572 next = state->local;
573
574 if (prev != next) {
575 tsk_rt(prev)->scheduled_on = NO_CPU;
576 if (next)
577 tsk_rt(next)->scheduled_on = state->cpu;
578 }
579
580 spin_unlock(&pfair_lock);
581
582 if (next)
583 TRACE_TASK(next, "scheduled rel=%lu at %lu (%llu)\n",
584 tsk_pfair(next)->release, pfair_time, litmus_clock());
585 else if (is_realtime(prev))
586 TRACE("Becomes idle at %lu (%llu)\n", pfair_time, litmus_clock());
587
588 return next;
589}
590
591static void pfair_task_new(struct task_struct * t, int on_rq, int running)
592{
593 unsigned long flags;
594
595 TRACE("pfair: task new %d state:%d\n", t->pid, t->state);
596
597 spin_lock_irqsave(&pfair_lock, flags);
598 if (running)
599 t->rt_param.scheduled_on = task_cpu(t);
600 else
601 t->rt_param.scheduled_on = NO_CPU;
602
603 prepare_release(t, pfair_time + 1);
604 tsk_pfair(t)->sporadic_release = 0;
605 pfair_add_release(t);
606 check_preempt(t);
607
608 spin_unlock_irqrestore(&pfair_lock, flags);
609}
610
611static void pfair_task_wake_up(struct task_struct *t)
612{
613 unsigned long flags;
614 lt_t now;
615
616 TRACE_TASK(t, "wakes at %llu, release=%lu, pfair_time:%lu\n",
617 litmus_clock(), cur_release(t), pfair_time);
618
619 spin_lock_irqsave(&pfair_lock, flags);
620
621 /* It is a little unclear how to deal with Pfair
622 * tasks that block for a while and then wake. For now,
623 * if a task blocks and wakes before its next job release,
624 * then it may resume if it is currently linked somewhere
625 * (as if it never blocked at all). Otherwise, we have a
626 * new sporadic job release.
627 */
628 if (tsk_pfair(t)->sporadic_release) {
629 now = litmus_clock();
630 release_at(t, now);
631 prepare_release(t, time2quanta(now, CEIL));
632 sched_trace_task_release(t);
633 /* FIXME: race with pfair_time advancing */
634 pfair_add_release(t);
635 tsk_pfair(t)->sporadic_release = 0;
636 }
637
638 check_preempt(t);
639
640 spin_unlock_irqrestore(&pfair_lock, flags);
641 TRACE_TASK(t, "wake up done at %llu\n", litmus_clock());
642}
643
644static void pfair_task_block(struct task_struct *t)
645{
646 BUG_ON(!is_realtime(t));
647 TRACE_TASK(t, "blocks at %llu, state:%d\n",
648 litmus_clock(), t->state);
649}
650
651static void pfair_task_exit(struct task_struct * t)
652{
653 unsigned long flags;
654
655 BUG_ON(!is_realtime(t));
656
657 /* Remote task from release or ready queue, and ensure
658 * that it is not the scheduled task for ANY CPU. We
659 * do this blanket check because occassionally when
660 * tasks exit while blocked, the task_cpu of the task
661 * might not be the same as the CPU that the PFAIR scheduler
662 * has chosen for it.
663 */
664 spin_lock_irqsave(&pfair_lock, flags);
665
666 TRACE_TASK(t, "RIP, state:%d\n", t->state);
667 drop_all_references(t);
668
669 spin_unlock_irqrestore(&pfair_lock, flags);
670
671 kfree(t->rt_param.pfair);
672 t->rt_param.pfair = NULL;
673}
674
675
676static void pfair_release_at(struct task_struct* task, lt_t start)
677{
678 unsigned long flags;
679 quanta_t release;
680
681 BUG_ON(!is_realtime(task));
682
683 spin_lock_irqsave(&pfair_lock, flags);
684 release_at(task, start);
685 release = time2quanta(start, CEIL);
686
687 if (release - pfair_time >= PFAIR_MAX_PERIOD)
688 release = pfair_time + PFAIR_MAX_PERIOD;
689
690 TRACE_TASK(task, "sys release at %lu\n", release);
691
692 drop_all_references(task);
693 prepare_release(task, release);
694 pfair_add_release(task);
695
696 /* Clear sporadic release flag, since this release subsumes any
697 * sporadic release on wake.
698 */
699 tsk_pfair(task)->sporadic_release = 0;
700
701 spin_unlock_irqrestore(&pfair_lock, flags);
702}
703
704static void init_subtask(struct subtask* sub, unsigned long i,
705 lt_t quanta, lt_t period)
706{
707 /* since i is zero-based, the formulas are shifted by one */
708 lt_t tmp;
709
710 /* release */
711 tmp = period * i;
712 do_div(tmp, quanta); /* floor */
713 sub->release = (quanta_t) tmp;
714
715 /* deadline */
716 tmp = period * (i + 1);
717 if (do_div(tmp, quanta)) /* ceil */
718 tmp++;
719 sub->deadline = (quanta_t) tmp;
720
721 /* next release */
722 tmp = period * (i + 1);
723 do_div(tmp, quanta); /* floor */
724 sub->overlap = sub->deadline - (quanta_t) tmp;
725
726 /* Group deadline.
727 * Based on the formula given in Uma's thesis.
728 */
729 if (2 * quanta >= period) {
730 /* heavy */
731 tmp = (sub->deadline - (i + 1)) * period;
732 if (period > quanta &&
733 do_div(tmp, (period - quanta))) /* ceil */
734 tmp++;
735 sub->group_deadline = (quanta_t) tmp;
736 } else
737 sub->group_deadline = 0;
738}
739
740static void dump_subtasks(struct task_struct* t)
741{
742 unsigned long i;
743 for (i = 0; i < t->rt_param.pfair->quanta; i++)
744 TRACE_TASK(t, "SUBTASK %lu: rel=%lu dl=%lu bbit:%lu gdl:%lu\n",
745 i + 1,
746 t->rt_param.pfair->subtasks[i].release,
747 t->rt_param.pfair->subtasks[i].deadline,
748 t->rt_param.pfair->subtasks[i].overlap,
749 t->rt_param.pfair->subtasks[i].group_deadline);
750}
751
752static long pfair_admit_task(struct task_struct* t)
753{
754 lt_t quanta;
755 lt_t period;
756 s64 quantum_length = ktime_to_ns(tick_period);
757 struct pfair_param* param;
758 unsigned long i;
759
760 /* Pfair is a tick-based method, so the time
761 * of interest is jiffies. Calculate tick-based
762 * times for everything.
763 * (Ceiling of exec cost, floor of period.)
764 */
765
766 quanta = get_exec_cost(t);
767 period = get_rt_period(t);
768
769 quanta = time2quanta(get_exec_cost(t), CEIL);
770
771 if (do_div(period, quantum_length))
772 printk(KERN_WARNING
773 "The period of %s/%d is not a multiple of %llu.\n",
774 t->comm, t->pid, (unsigned long long) quantum_length);
775
776 if (period >= PFAIR_MAX_PERIOD) {
777 printk(KERN_WARNING
778 "PFAIR: Rejecting task %s/%d; its period is too long.\n",
779 t->comm, t->pid);
780 return -EINVAL;
781 }
782
783 if (quanta == period) {
784 /* special case: task has weight 1.0 */
785 printk(KERN_INFO
786 "Admitting weight 1.0 task. (%s/%d, %llu, %llu).\n",
787 t->comm, t->pid, quanta, period);
788 quanta = 1;
789 period = 1;
790 }
791
792 param = kmalloc(sizeof(*param) +
793 quanta * sizeof(struct subtask), GFP_ATOMIC);
794
795 if (!param)
796 return -ENOMEM;
797
798 param->quanta = quanta;
799 param->cur = 0;
800 param->release = 0;
801 param->period = period;
802
803 for (i = 0; i < quanta; i++)
804 init_subtask(param->subtasks + i, i, quanta, period);
805
806 if (t->rt_param.pfair)
807 /* get rid of stale allocation */
808 kfree(t->rt_param.pfair);
809
810 t->rt_param.pfair = param;
811
812 /* spew out some debug info */
813 dump_subtasks(t);
814
815 return 0;
816}
817
818static long pfair_activate_plugin(void)
819{
820 int cpu;
821 struct pfair_state* state;
822
823 state = &__get_cpu_var(pfair_state);
824 pfair_time = current_quantum(state);
825
826 TRACE("Activating PFAIR at q=%lu\n", pfair_time);
827
828 for (cpu = 0; cpu < num_online_cpus(); cpu++) {
829 state = &per_cpu(pfair_state, cpu);
830 state->cur_tick = pfair_time;
831 state->local_tick = pfair_time;
832 state->missed_quanta = 0;
833 state->offset = cpu_stagger_offset(cpu);
834 }
835
836 return 0;
837}
838
839/* Plugin object */
840static struct sched_plugin pfair_plugin __cacheline_aligned_in_smp = {
841 .plugin_name = "PFAIR",
842 .tick = pfair_tick,
843 .task_new = pfair_task_new,
844 .task_exit = pfair_task_exit,
845 .schedule = pfair_schedule,
846 .task_wake_up = pfair_task_wake_up,
847 .task_block = pfair_task_block,
848 .admit_task = pfair_admit_task,
849 .release_at = pfair_release_at,
850 .complete_job = complete_job,
851 .activate_plugin = pfair_activate_plugin,
852};
853
854static int __init init_pfair(void)
855{
856 int cpu, i;
857 struct pfair_state *state;
858
859
860 /*
861 * initialize short_cut for per-cpu pfair state;
862 * there may be a problem here if someone removes a cpu
863 * while we are doing this initialization... and if cpus
864 * are added / removed later... is it a _real_ problem?
865 */
866 pstate = kmalloc(sizeof(struct pfair_state*) * num_online_cpus(), GFP_KERNEL);
867
868 /* initialize release queue */
869 for (i = 0; i < PFAIR_MAX_PERIOD; i++)
870 bheap_init(&release_queue[i]);
871
872 /* initialize CPU state */
873 for (cpu = 0; cpu < num_online_cpus(); cpu++) {
874 state = &per_cpu(pfair_state, cpu);
875 state->cpu = cpu;
876 state->cur_tick = 0;
877 state->local_tick = 0;
878 state->linked = NULL;
879 state->local = NULL;
880 state->scheduled = NULL;
881 state->missed_quanta = 0;
882 state->offset = cpu_stagger_offset(cpu);
883 pstate[cpu] = state;
884 }
885
886 rt_domain_init(&pfair, pfair_ready_order, NULL, NULL);
887 return register_sched_plugin(&pfair_plugin);
888}
889
890static void __exit clean_pfair(void)
891{
892 kfree(pstate);
893}
894
895module_init(init_pfair);
896module_exit(clean_pfair);
diff --git a/litmus/sched_plugin.c b/litmus/sched_plugin.c
new file mode 100644
index 000000000000..3767b30e610a
--- /dev/null
+++ b/litmus/sched_plugin.c
@@ -0,0 +1,265 @@
1/* sched_plugin.c -- core infrastructure for the scheduler plugin system
2 *
3 * This file includes the initialization of the plugin system, the no-op Linux
4 * scheduler plugin, some dummy functions, and some helper functions.
5 */
6
7#include <linux/list.h>
8#include <linux/spinlock.h>
9
10#include <litmus/litmus.h>
11#include <litmus/sched_plugin.h>
12
13#include <litmus/jobs.h>
14
15/*
16 * Generic function to trigger preemption on either local or remote cpu
17 * from scheduler plugins. The key feature is that this function is
18 * non-preemptive section aware and does not invoke the scheduler / send
19 * IPIs if the to-be-preempted task is actually non-preemptive.
20 */
21void preempt_if_preemptable(struct task_struct* t, int on_cpu)
22{
23 /* t is the real-time task executing on CPU on_cpu If t is NULL, then
24 * on_cpu is currently scheduling background work.
25 */
26
27 int send_ipi;
28
29 if (smp_processor_id() == on_cpu) {
30 /* local CPU case */
31 if (t) {
32 /* check if we need to poke userspace */
33 if (is_user_np(t))
34 /* yes, poke it */
35 request_exit_np(t);
36 else
37 /* no, see if we are allowed to preempt the
38 * currently-executing task */
39 if (!is_kernel_np(t))
40 set_tsk_need_resched(t);
41 } else
42 /* move non-real-time task out of the way */
43 set_tsk_need_resched(current);
44 } else {
45 /* remote CPU case */
46 if (!t)
47 /* currently schedules non-real-time work */
48 send_ipi = 1;
49 else {
50 /* currently schedules real-time work */
51 if (is_user_np(t)) {
52 /* need to notify user space of delayed
53 * preemption */
54
55 /* to avoid a race, set the flag, then test
56 * again */
57 request_exit_np(t);
58 /* make sure it got written */
59 mb();
60 }
61 /* Only send an ipi if remote task might have raced our
62 * request, i.e., send an IPI to make sure if it exited
63 * its critical section.
64 */
65 send_ipi = !is_np(t) && !is_kernel_np(t);
66 }
67 if (likely(send_ipi))
68 smp_send_reschedule(on_cpu);
69 }
70}
71
72
73/*************************************************************
74 * Dummy plugin functions *
75 *************************************************************/
76
77static void litmus_dummy_finish_switch(struct task_struct * prev)
78{
79}
80
81static struct task_struct* litmus_dummy_schedule(struct task_struct * prev)
82{
83 return NULL;
84}
85
86static void litmus_dummy_tick(struct task_struct* tsk)
87{
88}
89
90static long litmus_dummy_admit_task(struct task_struct* tsk)
91{
92 printk(KERN_CRIT "LITMUS^RT: Linux plugin rejects %s/%d.\n",
93 tsk->comm, tsk->pid);
94 return -EINVAL;
95}
96
97static void litmus_dummy_task_new(struct task_struct *t, int on_rq, int running)
98{
99}
100
101static void litmus_dummy_task_wake_up(struct task_struct *task)
102{
103}
104
105static void litmus_dummy_task_block(struct task_struct *task)
106{
107}
108
109static void litmus_dummy_task_exit(struct task_struct *task)
110{
111}
112
113static long litmus_dummy_complete_job(void)
114{
115 return -ENOSYS;
116}
117
118static long litmus_dummy_activate_plugin(void)
119{
120 return 0;
121}
122
123static long litmus_dummy_deactivate_plugin(void)
124{
125 return 0;
126}
127
128#ifdef CONFIG_FMLP
129
130static long litmus_dummy_inherit_priority(struct pi_semaphore *sem,
131 struct task_struct *new_owner)
132{
133 return -ENOSYS;
134}
135
136static long litmus_dummy_return_priority(struct pi_semaphore *sem)
137{
138 return -ENOSYS;
139}
140
141static long litmus_dummy_pi_block(struct pi_semaphore *sem,
142 struct task_struct *new_waiter)
143{
144 return -ENOSYS;
145}
146
147#endif
148
149
150/* The default scheduler plugin. It doesn't do anything and lets Linux do its
151 * job.
152 */
153struct sched_plugin linux_sched_plugin = {
154 .plugin_name = "Linux",
155 .tick = litmus_dummy_tick,
156 .task_new = litmus_dummy_task_new,
157 .task_exit = litmus_dummy_task_exit,
158 .task_wake_up = litmus_dummy_task_wake_up,
159 .task_block = litmus_dummy_task_block,
160 .complete_job = litmus_dummy_complete_job,
161 .schedule = litmus_dummy_schedule,
162 .finish_switch = litmus_dummy_finish_switch,
163 .activate_plugin = litmus_dummy_activate_plugin,
164 .deactivate_plugin = litmus_dummy_deactivate_plugin,
165#ifdef CONFIG_FMLP
166 .inherit_priority = litmus_dummy_inherit_priority,
167 .return_priority = litmus_dummy_return_priority,
168 .pi_block = litmus_dummy_pi_block,
169#endif
170 .admit_task = litmus_dummy_admit_task
171};
172
173/*
174 * The cluster size is needed in C-EDF: it makes sense only to cluster
175 * around L2 or L3, so if cluster_cache_index = 2 (default) we cluster
176 * all the CPUs that shares a L2 cache, while cluster_cache_index = 3
177 * we cluster all CPs that shares a L3 cache
178 */
179int cluster_cache_index = 2;
180
181/*
182 * The reference to current plugin that is used to schedule tasks within
183 * the system. It stores references to actual function implementations
184 * Should be initialized by calling "init_***_plugin()"
185 */
186struct sched_plugin *litmus = &linux_sched_plugin;
187
188/* the list of registered scheduling plugins */
189static LIST_HEAD(sched_plugins);
190static DEFINE_SPINLOCK(sched_plugins_lock);
191
192#define CHECK(func) {\
193 if (!plugin->func) \
194 plugin->func = litmus_dummy_ ## func;}
195
196/* FIXME: get reference to module */
197int register_sched_plugin(struct sched_plugin* plugin)
198{
199 printk(KERN_INFO "Registering LITMUS^RT plugin %s.\n",
200 plugin->plugin_name);
201
202 /* make sure we don't trip over null pointers later */
203 CHECK(finish_switch);
204 CHECK(schedule);
205 CHECK(tick);
206 CHECK(task_wake_up);
207 CHECK(task_exit);
208 CHECK(task_block);
209 CHECK(task_new);
210 CHECK(complete_job);
211 CHECK(activate_plugin);
212 CHECK(deactivate_plugin);
213#ifdef CONFIG_FMLP
214 CHECK(inherit_priority);
215 CHECK(return_priority);
216 CHECK(pi_block);
217#endif
218 CHECK(admit_task);
219
220 if (!plugin->release_at)
221 plugin->release_at = release_at;
222
223 spin_lock(&sched_plugins_lock);
224 list_add(&plugin->list, &sched_plugins);
225 spin_unlock(&sched_plugins_lock);
226
227 return 0;
228}
229
230
231/* FIXME: reference counting, etc. */
232struct sched_plugin* find_sched_plugin(const char* name)
233{
234 struct list_head *pos;
235 struct sched_plugin *plugin;
236
237 spin_lock(&sched_plugins_lock);
238 list_for_each(pos, &sched_plugins) {
239 plugin = list_entry(pos, struct sched_plugin, list);
240 if (!strcmp(plugin->plugin_name, name))
241 goto out_unlock;
242 }
243 plugin = NULL;
244
245out_unlock:
246 spin_unlock(&sched_plugins_lock);
247 return plugin;
248}
249
250int print_sched_plugins(char* buf, int max)
251{
252 int count = 0;
253 struct list_head *pos;
254 struct sched_plugin *plugin;
255
256 spin_lock(&sched_plugins_lock);
257 list_for_each(pos, &sched_plugins) {
258 plugin = list_entry(pos, struct sched_plugin, list);
259 count += snprintf(buf + count, max - count, "%s\n", plugin->plugin_name);
260 if (max - count <= 0)
261 break;
262 }
263 spin_unlock(&sched_plugins_lock);
264 return count;
265}
diff --git a/litmus/sched_psn_edf.c b/litmus/sched_psn_edf.c
new file mode 100644
index 000000000000..7f71ecfaaaae
--- /dev/null
+++ b/litmus/sched_psn_edf.c
@@ -0,0 +1,478 @@
1/*
2 * kernel/sched_psn_edf.c
3 *
4 * Implementation of the PSN-EDF scheduler plugin.
5 * Based on kern/sched_part_edf.c and kern/sched_gsn_edf.c.
6 *
7 * Suspensions and non-preemptable sections are supported.
8 * Priority inheritance is not supported.
9 */
10
11#include <linux/percpu.h>
12#include <linux/sched.h>
13#include <linux/list.h>
14#include <linux/spinlock.h>
15
16#include <linux/module.h>
17
18#include <litmus/litmus.h>
19#include <litmus/jobs.h>
20#include <litmus/sched_plugin.h>
21#include <litmus/edf_common.h>
22
23
24typedef struct {
25 rt_domain_t domain;
26 int cpu;
27 struct task_struct* scheduled; /* only RT tasks */
28/*
29 * scheduling lock slock
30 * protects the domain and serializes scheduling decisions
31 */
32#define slock domain.ready_lock
33
34} psnedf_domain_t;
35
36DEFINE_PER_CPU(psnedf_domain_t, psnedf_domains);
37
38#define local_edf (&__get_cpu_var(psnedf_domains).domain)
39#define local_pedf (&__get_cpu_var(psnedf_domains))
40#define remote_edf(cpu) (&per_cpu(psnedf_domains, cpu).domain)
41#define remote_pedf(cpu) (&per_cpu(psnedf_domains, cpu))
42#define task_edf(task) remote_edf(get_partition(task))
43#define task_pedf(task) remote_pedf(get_partition(task))
44
45
46static void psnedf_domain_init(psnedf_domain_t* pedf,
47 check_resched_needed_t check,
48 release_jobs_t release,
49 int cpu)
50{
51 edf_domain_init(&pedf->domain, check, release);
52 pedf->cpu = cpu;
53 pedf->scheduled = NULL;
54}
55
56static void requeue(struct task_struct* t, rt_domain_t *edf)
57{
58 if (t->state != TASK_RUNNING)
59 TRACE_TASK(t, "requeue: !TASK_RUNNING\n");
60
61 set_rt_flags(t, RT_F_RUNNING);
62 if (is_released(t, litmus_clock()))
63 __add_ready(edf, t);
64 else
65 add_release(edf, t); /* it has got to wait */
66}
67
68/* we assume the lock is being held */
69static void preempt(psnedf_domain_t *pedf)
70{
71 preempt_if_preemptable(pedf->scheduled, pedf->cpu);
72}
73
74/* This check is trivial in partioned systems as we only have to consider
75 * the CPU of the partition.
76 */
77static int psnedf_check_resched(rt_domain_t *edf)
78{
79 psnedf_domain_t *pedf = container_of(edf, psnedf_domain_t, domain);
80
81 /* because this is a callback from rt_domain_t we already hold
82 * the necessary lock for the ready queue
83 */
84 if (edf_preemption_needed(edf, pedf->scheduled)) {
85 preempt(pedf);
86 return 1;
87 } else
88 return 0;
89}
90
91static void job_completion(struct task_struct* t, int forced)
92{
93 sched_trace_task_completion(t,forced);
94 TRACE_TASK(t, "job_completion().\n");
95
96 set_rt_flags(t, RT_F_SLEEP);
97 prepare_for_next_period(t);
98}
99
100static void psnedf_tick(struct task_struct *t)
101{
102 psnedf_domain_t *pedf = local_pedf;
103
104 /* Check for inconsistency. We don't need the lock for this since
105 * ->scheduled is only changed in schedule, which obviously is not
106 * executing in parallel on this CPU
107 */
108 BUG_ON(is_realtime(t) && t != pedf->scheduled);
109
110 if (is_realtime(t) && budget_exhausted(t)) {
111 if (!is_np(t)) {
112 set_tsk_need_resched(t);
113 TRACE("psnedf_scheduler_tick: "
114 "%d is preemptable "
115 " => FORCE_RESCHED\n", t->pid);
116 } else if (is_user_np(t)) {
117 TRACE("psnedf_scheduler_tick: "
118 "%d is non-preemptable, "
119 "preemption delayed.\n", t->pid);
120 request_exit_np(t);
121 }
122 }
123}
124
125static struct task_struct* psnedf_schedule(struct task_struct * prev)
126{
127 psnedf_domain_t* pedf = local_pedf;
128 rt_domain_t* edf = &pedf->domain;
129 struct task_struct* next;
130
131 int out_of_time, sleep, preempt,
132 np, exists, blocks, resched;
133
134 spin_lock(&pedf->slock);
135
136 /* sanity checking
137 * differently from gedf, when a task exits (dead)
138 * pedf->schedule may be null and prev _is_ realtime
139 */
140 BUG_ON(pedf->scheduled && pedf->scheduled != prev);
141 BUG_ON(pedf->scheduled && !is_realtime(prev));
142
143 /* (0) Determine state */
144 exists = pedf->scheduled != NULL;
145 blocks = exists && !is_running(pedf->scheduled);
146 out_of_time = exists && budget_exhausted(pedf->scheduled);
147 np = exists && is_np(pedf->scheduled);
148 sleep = exists && get_rt_flags(pedf->scheduled) == RT_F_SLEEP;
149 preempt = edf_preemption_needed(edf, prev);
150
151 /* If we need to preempt do so.
152 * The following checks set resched to 1 in case of special
153 * circumstances.
154 */
155 resched = preempt;
156
157 /* If a task blocks we have no choice but to reschedule.
158 */
159 if (blocks)
160 resched = 1;
161
162 /* Request a sys_exit_np() call if we would like to preempt but cannot.
163 * Multiple calls to request_exit_np() don't hurt.
164 */
165 if (np && (out_of_time || preempt || sleep))
166 request_exit_np(pedf->scheduled);
167
168 /* Any task that is preemptable and either exhausts its execution
169 * budget or wants to sleep completes. We may have to reschedule after
170 * this.
171 */
172 if (!np && (out_of_time || sleep) && !blocks) {
173 job_completion(pedf->scheduled, !sleep);
174 resched = 1;
175 }
176
177 /* The final scheduling decision. Do we need to switch for some reason?
178 * Switch if we are in RT mode and have no task or if we need to
179 * resched.
180 */
181 next = NULL;
182 if ((!np || blocks) && (resched || !exists)) {
183 /* Take care of a previously scheduled
184 * job by taking it out of the Linux runqueue.
185 */
186 if (pedf->scheduled && !blocks)
187 requeue(pedf->scheduled, edf);
188 next = __take_ready(edf);
189 } else
190 /* Only override Linux scheduler if we have a real-time task
191 * scheduled that needs to continue.
192 */
193 if (exists)
194 next = prev;
195
196 if (next) {
197 TRACE_TASK(next, "scheduled at %llu\n", litmus_clock());
198 set_rt_flags(next, RT_F_RUNNING);
199 } else {
200 TRACE("becoming idle at %llu\n", litmus_clock());
201 }
202
203 pedf->scheduled = next;
204 spin_unlock(&pedf->slock);
205
206 return next;
207}
208
209
210/* Prepare a task for running in RT mode
211 */
212static void psnedf_task_new(struct task_struct * t, int on_rq, int running)
213{
214 rt_domain_t* edf = task_edf(t);
215 psnedf_domain_t* pedf = task_pedf(t);
216 unsigned long flags;
217
218 TRACE_TASK(t, "psn edf: task new, cpu = %d\n",
219 t->rt_param.task_params.cpu);
220
221 /* setup job parameters */
222 release_at(t, litmus_clock());
223
224 /* The task should be running in the queue, otherwise signal
225 * code will try to wake it up with fatal consequences.
226 */
227 spin_lock_irqsave(&pedf->slock, flags);
228 if (running) {
229 /* there shouldn't be anything else running at the time */
230 BUG_ON(pedf->scheduled);
231 pedf->scheduled = t;
232 } else {
233 requeue(t, edf);
234 /* maybe we have to reschedule */
235 preempt(pedf);
236 }
237 spin_unlock_irqrestore(&pedf->slock, flags);
238}
239
240static void psnedf_task_wake_up(struct task_struct *task)
241{
242 unsigned long flags;
243 psnedf_domain_t* pedf = task_pedf(task);
244 rt_domain_t* edf = task_edf(task);
245 lt_t now;
246
247 TRACE_TASK(task, "wake_up at %llu\n", litmus_clock());
248 spin_lock_irqsave(&pedf->slock, flags);
249 BUG_ON(is_queued(task));
250 /* We need to take suspensions because of semaphores into
251 * account! If a job resumes after being suspended due to acquiring
252 * a semaphore, it should never be treated as a new job release.
253 *
254 * FIXME: This should be done in some more predictable and userspace-controlled way.
255 */
256 now = litmus_clock();
257 if (is_tardy(task, now) &&
258 get_rt_flags(task) != RT_F_EXIT_SEM) {
259 /* new sporadic release */
260 release_at(task, now);
261 sched_trace_task_release(task);
262 }
263
264 /* Only add to ready queue if it is not the currently-scheduled
265 * task. This could be the case if a task was woken up concurrently
266 * on a remote CPU before the executing CPU got around to actually
267 * de-scheduling the task, i.e., wake_up() raced with schedule()
268 * and won.
269 */
270 if (pedf->scheduled != task)
271 requeue(task, edf);
272
273 spin_unlock_irqrestore(&pedf->slock, flags);
274 TRACE_TASK(task, "wake up done\n");
275}
276
277static void psnedf_task_block(struct task_struct *t)
278{
279 /* only running tasks can block, thus t is in no queue */
280 TRACE_TASK(t, "block at %llu, state=%d\n", litmus_clock(), t->state);
281
282 BUG_ON(!is_realtime(t));
283 BUG_ON(is_queued(t));
284}
285
286static void psnedf_task_exit(struct task_struct * t)
287{
288 unsigned long flags;
289 psnedf_domain_t* pedf = task_pedf(t);
290 rt_domain_t* edf;
291
292 spin_lock_irqsave(&pedf->slock, flags);
293 if (is_queued(t)) {
294 /* dequeue */
295 edf = task_edf(t);
296 remove(edf, t);
297 }
298 if (pedf->scheduled == t)
299 pedf->scheduled = NULL;
300
301 TRACE_TASK(t, "RIP, now reschedule\n");
302
303 preempt(pedf);
304 spin_unlock_irqrestore(&pedf->slock, flags);
305}
306
307#ifdef CONFIG_FMLP
308static long psnedf_pi_block(struct pi_semaphore *sem,
309 struct task_struct *new_waiter)
310{
311 psnedf_domain_t* pedf;
312 rt_domain_t* edf;
313 struct task_struct* t;
314 int cpu = get_partition(new_waiter);
315
316 BUG_ON(!new_waiter);
317
318 if (edf_higher_prio(new_waiter, sem->hp.cpu_task[cpu])) {
319 TRACE_TASK(new_waiter, " boosts priority\n");
320 pedf = task_pedf(new_waiter);
321 edf = task_edf(new_waiter);
322
323 /* interrupts already disabled */
324 spin_lock(&pedf->slock);
325
326 /* store new highest-priority task */
327 sem->hp.cpu_task[cpu] = new_waiter;
328 if (sem->holder &&
329 get_partition(sem->holder) == get_partition(new_waiter)) {
330 /* let holder inherit */
331 sem->holder->rt_param.inh_task = new_waiter;
332 t = sem->holder;
333 if (is_queued(t)) {
334 /* queued in domain*/
335 remove(edf, t);
336 /* readd to make priority change take place */
337 /* FIXME: this looks outdated */
338 if (is_released(t, litmus_clock()))
339 __add_ready(edf, t);
340 else
341 add_release(edf, t);
342 }
343 }
344
345 /* check if we need to reschedule */
346 if (edf_preemption_needed(edf, current))
347 preempt(pedf);
348
349 spin_unlock(&pedf->slock);
350 }
351
352 return 0;
353}
354
355static long psnedf_inherit_priority(struct pi_semaphore *sem,
356 struct task_struct *new_owner)
357{
358 int cpu = get_partition(new_owner);
359
360 new_owner->rt_param.inh_task = sem->hp.cpu_task[cpu];
361 if (sem->hp.cpu_task[cpu] && new_owner != sem->hp.cpu_task[cpu]) {
362 TRACE_TASK(new_owner,
363 "inherited priority from %s/%d\n",
364 sem->hp.cpu_task[cpu]->comm,
365 sem->hp.cpu_task[cpu]->pid);
366 } else
367 TRACE_TASK(new_owner,
368 "cannot inherit priority: "
369 "no higher priority job waits on this CPU!\n");
370 /* make new owner non-preemptable as required by FMLP under
371 * PSN-EDF.
372 */
373 make_np(new_owner);
374 return 0;
375}
376
377
378/* This function is called on a semaphore release, and assumes that
379 * the current task is also the semaphore holder.
380 */
381static long psnedf_return_priority(struct pi_semaphore *sem)
382{
383 struct task_struct* t = current;
384 psnedf_domain_t* pedf = task_pedf(t);
385 rt_domain_t* edf = task_edf(t);
386 int ret = 0;
387 int cpu = get_partition(current);
388 int still_np;
389
390
391 /* Find new highest-priority semaphore task
392 * if holder task is the current hp.cpu_task[cpu].
393 *
394 * Calling function holds sem->wait.lock.
395 */
396 if (t == sem->hp.cpu_task[cpu])
397 edf_set_hp_cpu_task(sem, cpu);
398
399 still_np = take_np(current);
400
401 /* Since we don't nest resources, this
402 * should always be zero */
403 BUG_ON(still_np);
404
405 if (current->rt_param.inh_task) {
406 TRACE_CUR("return priority of %s/%d\n",
407 current->rt_param.inh_task->comm,
408 current->rt_param.inh_task->pid);
409 } else
410 TRACE_CUR(" no priority to return %p\n", sem);
411
412
413 /* Always check for delayed preemptions that might have become
414 * necessary due to non-preemptive execution.
415 */
416 spin_lock(&pedf->slock);
417
418 /* Reset inh_task to NULL. */
419 current->rt_param.inh_task = NULL;
420
421 /* check if we need to reschedule */
422 if (edf_preemption_needed(edf, current))
423 preempt(pedf);
424
425 spin_unlock(&pedf->slock);
426
427
428 return ret;
429}
430
431#endif
432
433static long psnedf_admit_task(struct task_struct* tsk)
434{
435 return task_cpu(tsk) == tsk->rt_param.task_params.cpu ? 0 : -EINVAL;
436}
437
438/* Plugin object */
439static struct sched_plugin psn_edf_plugin __cacheline_aligned_in_smp = {
440 .plugin_name = "PSN-EDF",
441#ifdef CONFIG_SRP
442 .srp_active = 1,
443#endif
444 .tick = psnedf_tick,
445 .task_new = psnedf_task_new,
446 .complete_job = complete_job,
447 .task_exit = psnedf_task_exit,
448 .schedule = psnedf_schedule,
449 .task_wake_up = psnedf_task_wake_up,
450 .task_block = psnedf_task_block,
451#ifdef CONFIG_FMLP
452 .fmlp_active = 1,
453 .pi_block = psnedf_pi_block,
454 .inherit_priority = psnedf_inherit_priority,
455 .return_priority = psnedf_return_priority,
456#endif
457 .admit_task = psnedf_admit_task
458};
459
460
461static int __init init_psn_edf(void)
462{
463 int i;
464
465 /* We do not really want to support cpu hotplug, do we? ;)
466 * However, if we are so crazy to do so,
467 * we cannot use num_online_cpu()
468 */
469 for (i = 0; i < num_online_cpus(); i++) {
470 psnedf_domain_init(remote_pedf(i),
471 psnedf_check_resched,
472 NULL, i);
473 }
474 return register_sched_plugin(&psn_edf_plugin);
475}
476
477module_init(init_psn_edf);
478
diff --git a/litmus/sched_task_trace.c b/litmus/sched_task_trace.c
new file mode 100644
index 000000000000..39a543e22d41
--- /dev/null
+++ b/litmus/sched_task_trace.c
@@ -0,0 +1,204 @@
1/*
2 * sched_task_trace.c -- record scheduling events to a byte stream
3 */
4
5#define NO_TASK_TRACE_DECLS
6
7#include <linux/module.h>
8#include <linux/sched.h>
9#include <linux/percpu.h>
10
11#include <litmus/ftdev.h>
12#include <litmus/litmus.h>
13
14#include <litmus/sched_trace.h>
15#include <litmus/feather_trace.h>
16#include <litmus/ftdev.h>
17
18
19/* set MAJOR to 0 to have it dynamically assigned */
20#define FT_TASK_TRACE_MAJOR 253
21#define NO_EVENTS 4096 /* this is a buffer of 12 4k pages per CPU */
22
23#define now() litmus_clock()
24
25struct local_buffer {
26 struct st_event_record record[NO_EVENTS];
27 char flag[NO_EVENTS];
28 struct ft_buffer ftbuf;
29};
30
31DEFINE_PER_CPU(struct local_buffer, st_event_buffer);
32
33static struct ftdev st_dev;
34
35static int st_dev_can_open(struct ftdev *dev, unsigned int cpu)
36{
37 return cpu_online(cpu) ? 0 : -ENODEV;
38}
39
40static int __init init_sched_task_trace(void)
41{
42 struct local_buffer* buf;
43 int i, ok = 0;
44 ftdev_init(&st_dev, THIS_MODULE);
45 for (i = 0; i < NR_CPUS; i++) {
46 buf = &per_cpu(st_event_buffer, i);
47 ok += init_ft_buffer(&buf->ftbuf, NO_EVENTS,
48 sizeof(struct st_event_record),
49 buf->flag,
50 buf->record);
51 st_dev.minor[i].buf = &buf->ftbuf;
52 }
53 if (ok == NR_CPUS) {
54 st_dev.minor_cnt = NR_CPUS;
55 st_dev.can_open = st_dev_can_open;
56 return register_ftdev(&st_dev, "sched_trace", FT_TASK_TRACE_MAJOR);
57 } else {
58 return -EINVAL;
59 }
60}
61
62module_init(init_sched_task_trace);
63
64
65static inline struct st_event_record* get_record(u8 type, struct task_struct* t)
66{
67 struct st_event_record* rec = NULL;
68 struct local_buffer* buf;
69
70 buf = &get_cpu_var(st_event_buffer);
71 if (ft_buffer_start_write(&buf->ftbuf, (void**) &rec)) {
72 rec->hdr.type = type;
73 rec->hdr.cpu = smp_processor_id();
74 rec->hdr.pid = t ? t->pid : 0;
75 rec->hdr.job = t ? t->rt_param.job_params.job_no : 0;
76 } else {
77 put_cpu_var(st_event_buffer);
78 }
79 /* rec will be NULL if it failed */
80 return rec;
81}
82
83static inline void put_record(struct st_event_record* rec)
84{
85 struct local_buffer* buf;
86 buf = &__get_cpu_var(st_event_buffer);
87 ft_buffer_finish_write(&buf->ftbuf, rec);
88 put_cpu_var(st_event_buffer);
89}
90
91feather_callback void do_sched_trace_task_name(unsigned long id, unsigned long _task)
92{
93 struct task_struct *t = (struct task_struct*) _task;
94 struct st_event_record* rec = get_record(ST_NAME, t);
95 int i;
96 if (rec) {
97 for (i = 0; i < min(TASK_COMM_LEN, ST_NAME_LEN); i++)
98 rec->data.name.cmd[i] = t->comm[i];
99 put_record(rec);
100 }
101}
102
103feather_callback void do_sched_trace_task_param(unsigned long id, unsigned long _task)
104{
105 struct task_struct *t = (struct task_struct*) _task;
106 struct st_event_record* rec = get_record(ST_PARAM, t);
107 if (rec) {
108 rec->data.param.wcet = get_exec_cost(t);
109 rec->data.param.period = get_rt_period(t);
110 rec->data.param.phase = get_rt_phase(t);
111 rec->data.param.partition = get_partition(t);
112 put_record(rec);
113 }
114}
115
116feather_callback void do_sched_trace_task_release(unsigned long id, unsigned long _task)
117{
118 struct task_struct *t = (struct task_struct*) _task;
119 struct st_event_record* rec = get_record(ST_RELEASE, t);
120 if (rec) {
121 rec->data.release.release = get_release(t);
122 rec->data.release.deadline = get_deadline(t);
123 put_record(rec);
124 }
125}
126
127/* skipped: st_assigned_data, we don't use it atm */
128
129feather_callback void do_sched_trace_task_switch_to(unsigned long id,
130 unsigned long _task)
131{
132 struct task_struct *t = (struct task_struct*) _task;
133 struct st_event_record* rec;
134 if (is_realtime(t)) {
135 rec = get_record(ST_SWITCH_TO, t);
136 if (rec) {
137 rec->data.switch_to.when = now();
138 rec->data.switch_to.exec_time = get_exec_time(t);
139 put_record(rec);
140 }
141 }
142}
143
144feather_callback void do_sched_trace_task_switch_away(unsigned long id,
145 unsigned long _task)
146{
147 struct task_struct *t = (struct task_struct*) _task;
148 struct st_event_record* rec;
149 if (is_realtime(t)) {
150 rec = get_record(ST_SWITCH_AWAY, t);
151 if (rec) {
152 rec->data.switch_away.when = now();
153 rec->data.switch_away.exec_time = get_exec_time(t);
154 put_record(rec);
155 }
156 }
157}
158
159feather_callback void do_sched_trace_task_completion(unsigned long id,
160 unsigned long _task,
161 unsigned long forced)
162{
163 struct task_struct *t = (struct task_struct*) _task;
164 struct st_event_record* rec = get_record(ST_COMPLETION, t);
165 if (rec) {
166 rec->data.completion.when = now();
167 rec->data.completion.forced = forced;
168 put_record(rec);
169 }
170}
171
172feather_callback void do_sched_trace_task_block(unsigned long id,
173 unsigned long _task)
174{
175 struct task_struct *t = (struct task_struct*) _task;
176 struct st_event_record* rec = get_record(ST_BLOCK, t);
177 if (rec) {
178 rec->data.block.when = now();
179 put_record(rec);
180 }
181}
182
183feather_callback void do_sched_trace_task_resume(unsigned long id,
184 unsigned long _task)
185{
186 struct task_struct *t = (struct task_struct*) _task;
187 struct st_event_record* rec = get_record(ST_RESUME, t);
188 if (rec) {
189 rec->data.resume.when = now();
190 put_record(rec);
191 }
192}
193
194feather_callback void do_sched_trace_sys_release(unsigned long id,
195 unsigned long _start)
196{
197 lt_t *start = (lt_t*) _start;
198 struct st_event_record* rec = get_record(ST_SYS_RELEASE, NULL);
199 if (rec) {
200 rec->data.sys_release.when = now();
201 rec->data.sys_release.release = *start;
202 put_record(rec);
203 }
204}
diff --git a/litmus/sched_trace.c b/litmus/sched_trace.c
new file mode 100644
index 000000000000..ad0b138d4b01
--- /dev/null
+++ b/litmus/sched_trace.c
@@ -0,0 +1,378 @@
1/*
2 * sched_trace.c -- record scheduling events to a byte stream.
3 */
4#include <linux/spinlock.h>
5#include <linux/semaphore.h>
6
7#include <linux/fs.h>
8#include <linux/miscdevice.h>
9#include <asm/uaccess.h>
10#include <linux/module.h>
11#include <linux/sysrq.h>
12
13#include <linux/kfifo.h>
14
15#include <litmus/sched_trace.h>
16#include <litmus/litmus.h>
17
18#define SCHED_TRACE_NAME "litmus/log"
19
20/* Allocate a buffer of about 32k per CPU */
21#define LITMUS_TRACE_BUF_PAGES 8
22#define LITMUS_TRACE_BUF_SIZE (PAGE_SIZE * LITMUS_TRACE_BUF_PAGES * NR_CPUS)
23
24/* Max length of one read from the buffer */
25#define MAX_READ_LEN (64 * 1024)
26
27/* Max length for one write --- from kernel --- to the buffer */
28#define MSG_SIZE 255
29
30/* Inner ring buffer structure */
31typedef struct {
32 rwlock_t del_lock;
33
34 /* the buffer */
35 struct kfifo *kfifo;
36} ring_buffer_t;
37
38/* Main buffer structure */
39typedef struct {
40 ring_buffer_t buf;
41 atomic_t reader_cnt;
42 struct semaphore reader_mutex;
43} trace_buffer_t;
44
45
46/*
47 * Inner buffer management functions
48 */
49void rb_init(ring_buffer_t* buf)
50{
51 rwlock_init(&buf->del_lock);
52 buf->kfifo = NULL;
53}
54
55int rb_alloc_buf(ring_buffer_t* buf, unsigned int size)
56{
57 unsigned long flags;
58
59 write_lock_irqsave(&buf->del_lock, flags);
60
61 buf->kfifo = kfifo_alloc(size, GFP_ATOMIC, NULL);
62
63 write_unlock_irqrestore(&buf->del_lock, flags);
64
65 if(IS_ERR(buf->kfifo)) {
66 printk(KERN_ERR "kfifo_alloc failed\n");
67 return PTR_ERR(buf->kfifo);
68 }
69
70 return 0;
71}
72
73int rb_free_buf(ring_buffer_t* buf)
74{
75 unsigned long flags;
76
77 write_lock_irqsave(&buf->del_lock, flags);
78
79 BUG_ON(!buf->kfifo);
80 kfifo_free(buf->kfifo);
81
82 buf->kfifo = NULL;
83
84 write_unlock_irqrestore(&buf->del_lock, flags);
85
86 return 0;
87}
88
89/*
90 * Assumption: concurrent writes are serialized externally
91 *
92 * Will only succeed if there is enough space for all len bytes.
93 */
94int rb_put(ring_buffer_t* buf, char* mem, size_t len)
95{
96 unsigned long flags;
97 int error = 0;
98
99 read_lock_irqsave(&buf->del_lock, flags);
100
101 if (!buf->kfifo) {
102 error = -ENODEV;
103 goto out;
104 }
105
106 if((__kfifo_put(buf->kfifo, mem, len)) < len) {
107 error = -ENOMEM;
108 goto out;
109 }
110
111 out:
112 read_unlock_irqrestore(&buf->del_lock, flags);
113 return error;
114}
115
116/* Assumption: concurrent reads are serialized externally */
117int rb_get(ring_buffer_t* buf, char* mem, size_t len)
118{
119 unsigned long flags;
120 int error = 0;
121
122 read_lock_irqsave(&buf->del_lock, flags);
123 if (!buf->kfifo) {
124 error = -ENODEV;
125 goto out;
126 }
127
128 error = __kfifo_get(buf->kfifo, (unsigned char*)mem, len);
129
130 out:
131 read_unlock_irqrestore(&buf->del_lock, flags);
132 return error;
133}
134
135/*
136 * Device Driver management
137 */
138static spinlock_t log_buffer_lock = SPIN_LOCK_UNLOCKED;
139static trace_buffer_t log_buffer;
140
141static void init_log_buffer(void)
142{
143 rb_init(&log_buffer.buf);
144 atomic_set(&log_buffer.reader_cnt,0);
145 init_MUTEX(&log_buffer.reader_mutex);
146}
147
148static DEFINE_PER_CPU(char[MSG_SIZE], fmt_buffer);
149
150/*
151 * sched_trace_log_message - Write to the trace buffer (log_buffer)
152 *
153 * This is the only function accessing the log_buffer from inside the
154 * kernel for writing.
155 * Concurrent access to sched_trace_log_message must be serialized using
156 * log_buffer_lock
157 * The maximum length of a formatted message is 255
158 */
159void sched_trace_log_message(const char* fmt, ...)
160{
161 unsigned long flags;
162 va_list args;
163 size_t len;
164 char* buf;
165
166 va_start(args, fmt);
167 local_irq_save(flags);
168
169 /* format message */
170 buf = __get_cpu_var(fmt_buffer);
171 len = vscnprintf(buf, MSG_SIZE, fmt, args);
172
173 spin_lock(&log_buffer_lock);
174 /* Don't copy the trailing null byte, we don't want null bytes
175 * in a text file.
176 */
177 rb_put(&log_buffer.buf, buf, len);
178 spin_unlock(&log_buffer_lock);
179
180 local_irq_restore(flags);
181 va_end(args);
182}
183
184/*
185 * log_read - Read the trace buffer
186 *
187 * This function is called as a file operation from userspace.
188 * Readers can sleep. Access is serialized through reader_mutex
189 */
190static ssize_t log_read(struct file *filp, char __user *to, size_t len,
191 loff_t *f_pos)
192{
193 /* we ignore f_pos, this is strictly sequential */
194
195 ssize_t error = -EINVAL;
196 char* mem;
197 trace_buffer_t *tbuf = filp->private_data;
198
199 if (down_interruptible(&tbuf->reader_mutex)) {
200 error = -ERESTARTSYS;
201 goto out;
202 }
203
204 if (len > MAX_READ_LEN)
205 len = MAX_READ_LEN;
206
207 mem = kmalloc(len, GFP_KERNEL);
208 if (!mem) {
209 error = -ENOMEM;
210 goto out_unlock;
211 }
212
213 error = rb_get(&tbuf->buf, mem, len);
214 while (!error) {
215 set_current_state(TASK_INTERRUPTIBLE);
216 schedule_timeout(110);
217 if (signal_pending(current))
218 error = -ERESTARTSYS;
219 else
220 error = rb_get(&tbuf->buf, mem, len);
221 }
222
223 if (error > 0 && copy_to_user(to, mem, error))
224 error = -EFAULT;
225
226 kfree(mem);
227 out_unlock:
228 up(&tbuf->reader_mutex);
229 out:
230 return error;
231}
232
233/*
234 * Enable redirection of printk() messages to the trace buffer.
235 * Defined in kernel/printk.c
236 */
237extern int trace_override;
238extern int trace_recurse;
239
240/*
241 * log_open - open the global log message ring buffer.
242 */
243static int log_open(struct inode *in, struct file *filp)
244{
245 int error = -EINVAL;
246 trace_buffer_t* tbuf;
247
248 tbuf = &log_buffer;
249
250 if (down_interruptible(&tbuf->reader_mutex)) {
251 error = -ERESTARTSYS;
252 goto out;
253 }
254
255 /* first open must allocate buffers */
256 if (atomic_inc_return(&tbuf->reader_cnt) == 1) {
257 if ((error = rb_alloc_buf(&tbuf->buf, LITMUS_TRACE_BUF_SIZE)))
258 {
259 atomic_dec(&tbuf->reader_cnt);
260 goto out_unlock;
261 }
262 }
263
264 error = 0;
265 filp->private_data = tbuf;
266
267 printk(KERN_DEBUG
268 "sched_trace kfifo at 0x%p with buffer starting at: 0x%p\n",
269 tbuf->buf.kfifo, &((tbuf->buf.kfifo)->buffer));
270
271 /* override printk() */
272 trace_override++;
273
274 out_unlock:
275 up(&tbuf->reader_mutex);
276 out:
277 return error;
278}
279
280static int log_release(struct inode *in, struct file *filp)
281{
282 int error = -EINVAL;
283 trace_buffer_t* tbuf = filp->private_data;
284
285 BUG_ON(!filp->private_data);
286
287 if (down_interruptible(&tbuf->reader_mutex)) {
288 error = -ERESTARTSYS;
289 goto out;
290 }
291
292 /* last release must deallocate buffers */
293 if (atomic_dec_return(&tbuf->reader_cnt) == 0) {
294 error = rb_free_buf(&tbuf->buf);
295 }
296
297 /* release printk() overriding */
298 trace_override--;
299
300 printk(KERN_DEBUG "sched_trace kfifo released\n");
301
302 up(&tbuf->reader_mutex);
303 out:
304 return error;
305}
306
307/*
308 * log_fops - The file operations for accessing the global LITMUS log message
309 * buffer.
310 *
311 * Except for opening the device file it uses the same operations as trace_fops.
312 */
313static struct file_operations log_fops = {
314 .owner = THIS_MODULE,
315 .open = log_open,
316 .release = log_release,
317 .read = log_read,
318};
319
320static struct miscdevice litmus_log_dev = {
321 .name = SCHED_TRACE_NAME,
322 .minor = MISC_DYNAMIC_MINOR,
323 .fops = &log_fops,
324};
325
326#ifdef CONFIG_MAGIC_SYSRQ
327void dump_trace_buffer(int max)
328{
329 char line[80];
330 int len;
331 int count = 0;
332
333 /* potential, but very unlikely, race... */
334 trace_recurse = 1;
335 while ((max == 0 || count++ < max) &&
336 (len = rb_get(&log_buffer.buf, line, sizeof(line) - 1)) > 0) {
337 line[len] = '\0';
338 printk("%s", line);
339 }
340 trace_recurse = 0;
341}
342
343static void sysrq_dump_trace_buffer(int key, struct tty_struct *tty)
344{
345 dump_trace_buffer(100);
346}
347
348static struct sysrq_key_op sysrq_dump_trace_buffer_op = {
349 .handler = sysrq_dump_trace_buffer,
350 .help_msg = "dump-trace-buffer(Y)",
351 .action_msg = "writing content of TRACE() buffer",
352};
353#endif
354
355static int __init init_sched_trace(void)
356{
357 printk("Initializing TRACE() device\n");
358 init_log_buffer();
359
360#ifdef CONFIG_MAGIC_SYSRQ
361 /* offer some debugging help */
362 if (!register_sysrq_key('y', &sysrq_dump_trace_buffer_op))
363 printk("Registered dump-trace-buffer(Y) magic sysrq.\n");
364 else
365 printk("Could not register dump-trace-buffer(Y) magic sysrq.\n");
366#endif
367
368
369 return misc_register(&litmus_log_dev);
370}
371
372static void __exit exit_sched_trace(void)
373{
374 misc_deregister(&litmus_log_dev);
375}
376
377module_init(init_sched_trace);
378module_exit(exit_sched_trace);
diff --git a/litmus/srp.c b/litmus/srp.c
new file mode 100644
index 000000000000..71639b991630
--- /dev/null
+++ b/litmus/srp.c
@@ -0,0 +1,318 @@
1/* ************************************************************************** */
2/* STACK RESOURCE POLICY */
3/* ************************************************************************** */
4
5#include <asm/atomic.h>
6#include <linux/wait.h>
7#include <litmus/litmus.h>
8#include <litmus/sched_plugin.h>
9
10#include <litmus/fdso.h>
11
12#include <litmus/trace.h>
13
14
15#ifdef CONFIG_SRP
16
17struct srp_priority {
18 struct list_head list;
19 unsigned int period;
20 pid_t pid;
21};
22
23#define list2prio(l) list_entry(l, struct srp_priority, list)
24
25/* SRP task priority comparison function. Smaller periods have highest
26 * priority, tie-break is PID. Special case: period == 0 <=> no priority
27 */
28static int srp_higher_prio(struct srp_priority* first,
29 struct srp_priority* second)
30{
31 if (!first->period)
32 return 0;
33 else
34 return !second->period ||
35 first->period < second->period || (
36 first->period == second->period &&
37 first->pid < second->pid);
38}
39
40struct srp {
41 struct list_head ceiling;
42 wait_queue_head_t ceiling_blocked;
43};
44
45
46atomic_t srp_objects_in_use = ATOMIC_INIT(0);
47
48DEFINE_PER_CPU(struct srp, srp);
49
50
51/* Initialize SRP semaphores at boot time. */
52static int __init srp_init(void)
53{
54 int i;
55
56 printk("Initializing SRP per-CPU ceilings...");
57 for (i = 0; i < NR_CPUS; i++) {
58 init_waitqueue_head(&per_cpu(srp, i).ceiling_blocked);
59 INIT_LIST_HEAD(&per_cpu(srp, i).ceiling);
60 }
61 printk(" done!\n");
62
63 return 0;
64}
65module_init(srp_init);
66
67
68#define system_ceiling(srp) list2prio(srp->ceiling.next)
69
70
71#define UNDEF_SEM -2
72
73
74/* struct for uniprocessor SRP "semaphore" */
75struct srp_semaphore {
76 struct srp_priority ceiling;
77 struct task_struct* owner;
78 int cpu; /* cpu associated with this "semaphore" and resource */
79};
80
81#define ceiling2sem(c) container_of(c, struct srp_semaphore, ceiling)
82
83static int srp_exceeds_ceiling(struct task_struct* first,
84 struct srp* srp)
85{
86 return list_empty(&srp->ceiling) ||
87 get_rt_period(first) < system_ceiling(srp)->period ||
88 (get_rt_period(first) == system_ceiling(srp)->period &&
89 first->pid < system_ceiling(srp)->pid) ||
90 ceiling2sem(system_ceiling(srp))->owner == first;
91}
92
93static void srp_add_prio(struct srp* srp, struct srp_priority* prio)
94{
95 struct list_head *pos;
96 if (in_list(&prio->list)) {
97 printk(KERN_CRIT "WARNING: SRP violation detected, prio is already in "
98 "ceiling list! cpu=%d, srp=%p\n", smp_processor_id(), ceiling2sem(prio));
99 return;
100 }
101 list_for_each(pos, &srp->ceiling)
102 if (unlikely(srp_higher_prio(prio, list2prio(pos)))) {
103 __list_add(&prio->list, pos->prev, pos);
104 return;
105 }
106
107 list_add_tail(&prio->list, &srp->ceiling);
108}
109
110
111static void* create_srp_semaphore(void)
112{
113 struct srp_semaphore* sem;
114
115 sem = kmalloc(sizeof(*sem), GFP_KERNEL);
116 if (!sem)
117 return NULL;
118
119 INIT_LIST_HEAD(&sem->ceiling.list);
120 sem->ceiling.period = 0;
121 sem->cpu = UNDEF_SEM;
122 sem->owner = NULL;
123 atomic_inc(&srp_objects_in_use);
124 return sem;
125}
126
127static noinline int open_srp_semaphore(struct od_table_entry* entry, void* __user arg)
128{
129 struct srp_semaphore* sem = (struct srp_semaphore*) entry->obj->obj;
130 int ret = 0;
131 struct task_struct* t = current;
132 struct srp_priority t_prio;
133
134 TRACE("opening SRP semaphore %p, cpu=%d\n", sem, sem->cpu);
135 if (!srp_active())
136 return -EBUSY;
137
138 if (sem->cpu == UNDEF_SEM)
139 sem->cpu = get_partition(t);
140 else if (sem->cpu != get_partition(t))
141 ret = -EPERM;
142
143 if (ret == 0) {
144 t_prio.period = get_rt_period(t);
145 t_prio.pid = t->pid;
146 if (srp_higher_prio(&t_prio, &sem->ceiling)) {
147 sem->ceiling.period = t_prio.period;
148 sem->ceiling.pid = t_prio.pid;
149 }
150 }
151
152 return ret;
153}
154
155static void destroy_srp_semaphore(void* sem)
156{
157 /* XXX invariants */
158 atomic_dec(&srp_objects_in_use);
159 kfree(sem);
160}
161
162struct fdso_ops srp_sem_ops = {
163 .create = create_srp_semaphore,
164 .open = open_srp_semaphore,
165 .destroy = destroy_srp_semaphore
166};
167
168
169static void do_srp_down(struct srp_semaphore* sem)
170{
171 /* Update ceiling. */
172 srp_add_prio(&__get_cpu_var(srp), &sem->ceiling);
173 WARN_ON(sem->owner != NULL);
174 sem->owner = current;
175 TRACE_CUR("acquired srp 0x%p\n", sem);
176}
177
178static void do_srp_up(struct srp_semaphore* sem)
179{
180 /* Determine new system priority ceiling for this CPU. */
181 WARN_ON(!in_list(&sem->ceiling.list));
182 if (in_list(&sem->ceiling.list))
183 list_del(&sem->ceiling.list);
184
185 sem->owner = NULL;
186
187 /* Wake tasks on this CPU, if they exceed current ceiling. */
188 TRACE_CUR("released srp 0x%p\n", sem);
189 wake_up_all(&__get_cpu_var(srp).ceiling_blocked);
190}
191
192/* Adjust the system-wide priority ceiling if resource is claimed. */
193asmlinkage long sys_srp_down(int sem_od)
194{
195 int cpu;
196 int ret = -EINVAL;
197 struct srp_semaphore* sem;
198
199 /* disabling preemptions is sufficient protection since
200 * SRP is strictly per CPU and we don't interfere with any
201 * interrupt handlers
202 */
203 preempt_disable();
204 TS_SRP_DOWN_START;
205
206 cpu = smp_processor_id();
207 sem = lookup_srp_sem(sem_od);
208 if (sem && sem->cpu == cpu) {
209 do_srp_down(sem);
210 ret = 0;
211 }
212
213 TS_SRP_DOWN_END;
214 preempt_enable();
215 return ret;
216}
217
218/* Adjust the system-wide priority ceiling if resource is freed. */
219asmlinkage long sys_srp_up(int sem_od)
220{
221 int cpu;
222 int ret = -EINVAL;
223 struct srp_semaphore* sem;
224
225 preempt_disable();
226 TS_SRP_UP_START;
227
228 cpu = smp_processor_id();
229 sem = lookup_srp_sem(sem_od);
230
231 if (sem && sem->cpu == cpu) {
232 do_srp_up(sem);
233 ret = 0;
234 }
235
236 TS_SRP_UP_END;
237 preempt_enable();
238 return ret;
239}
240
241static int srp_wake_up(wait_queue_t *wait, unsigned mode, int sync,
242 void *key)
243{
244 int cpu = smp_processor_id();
245 struct task_struct *tsk = wait->private;
246 if (cpu != get_partition(tsk))
247 TRACE_TASK(tsk, "srp_wake_up on wrong cpu, partition is %d\b",
248 get_partition(tsk));
249 else if (srp_exceeds_ceiling(tsk, &__get_cpu_var(srp)))
250 return default_wake_function(wait, mode, sync, key);
251 return 0;
252}
253
254
255
256static void do_ceiling_block(struct task_struct *tsk)
257{
258 wait_queue_t wait = {
259 .private = tsk,
260 .func = srp_wake_up,
261 .task_list = {NULL, NULL}
262 };
263
264 tsk->state = TASK_UNINTERRUPTIBLE;
265 add_wait_queue(&__get_cpu_var(srp).ceiling_blocked, &wait);
266 tsk->rt_param.srp_non_recurse = 1;
267 preempt_enable_no_resched();
268 schedule();
269 preempt_disable();
270 tsk->rt_param.srp_non_recurse = 0;
271 remove_wait_queue(&__get_cpu_var(srp).ceiling_blocked, &wait);
272}
273
274/* Wait for current task priority to exceed system-wide priority ceiling.
275 */
276void srp_ceiling_block(void)
277{
278 struct task_struct *tsk = current;
279
280 /* Only applies to real-time tasks, but optimize for RT tasks. */
281 if (unlikely(!is_realtime(tsk)))
282 return;
283
284 /* Avoid recursive ceiling blocking. */
285 if (unlikely(tsk->rt_param.srp_non_recurse))
286 return;
287
288 /* Bail out early if there aren't any SRP resources around. */
289 if (likely(!atomic_read(&srp_objects_in_use)))
290 return;
291
292 preempt_disable();
293 if (!srp_exceeds_ceiling(tsk, &__get_cpu_var(srp))) {
294 TRACE_CUR("is priority ceiling blocked.\n");
295 while (!srp_exceeds_ceiling(tsk, &__get_cpu_var(srp)))
296 do_ceiling_block(tsk);
297 TRACE_CUR("finally exceeds system ceiling.\n");
298 } else
299 TRACE_CUR("is not priority ceiling blocked\n");
300 preempt_enable();
301}
302
303
304#else
305
306asmlinkage long sys_srp_down(int sem_od)
307{
308 return -ENOSYS;
309}
310
311asmlinkage long sys_srp_up(int sem_od)
312{
313 return -ENOSYS;
314}
315
316struct fdso_ops srp_sem_ops = {};
317
318#endif
diff --git a/litmus/sync.c b/litmus/sync.c
new file mode 100644
index 000000000000..bf75fde5450b
--- /dev/null
+++ b/litmus/sync.c
@@ -0,0 +1,104 @@
1/* litmus/sync.c - Support for synchronous and asynchronous task system releases.
2 *
3 *
4 */
5
6#include <asm/atomic.h>
7#include <asm/uaccess.h>
8#include <linux/spinlock.h>
9#include <linux/list.h>
10#include <linux/sched.h>
11#include <linux/completion.h>
12
13#include <litmus/litmus.h>
14#include <litmus/sched_plugin.h>
15#include <litmus/jobs.h>
16
17#include <litmus/sched_trace.h>
18
19static DECLARE_COMPLETION(ts_release);
20
21static long do_wait_for_ts_release(void)
22{
23 long ret = 0;
24
25 /* If the interruption races with a release, the completion object
26 * may have a non-zero counter. To avoid this problem, this should
27 * be replaced by wait_for_completion().
28 *
29 * For debugging purposes, this is interruptible for now.
30 */
31 ret = wait_for_completion_interruptible(&ts_release);
32
33 return ret;
34}
35
36int count_tasks_waiting_for_release(void)
37{
38 unsigned long flags;
39 int task_count = 0;
40 struct list_head *pos;
41
42 spin_lock_irqsave(&ts_release.wait.lock, flags);
43 list_for_each(pos, &ts_release.wait.task_list) {
44 task_count++;
45 }
46 spin_unlock_irqrestore(&ts_release.wait.lock, flags);
47
48 return task_count;
49}
50
51static long do_release_ts(lt_t start)
52{
53 int task_count = 0;
54 unsigned long flags;
55 struct list_head *pos;
56 struct task_struct *t;
57
58
59 spin_lock_irqsave(&ts_release.wait.lock, flags);
60 TRACE("<<<<<< synchronous task system release >>>>>>\n");
61
62 sched_trace_sys_release(&start);
63 list_for_each(pos, &ts_release.wait.task_list) {
64 t = (struct task_struct*) list_entry(pos,
65 struct __wait_queue,
66 task_list)->private;
67 task_count++;
68 litmus->release_at(t, start + t->rt_param.task_params.phase);
69 sched_trace_task_release(t);
70 }
71
72 spin_unlock_irqrestore(&ts_release.wait.lock, flags);
73
74 complete_n(&ts_release, task_count);
75
76 return task_count;
77}
78
79
80asmlinkage long sys_wait_for_ts_release(void)
81{
82 long ret = -EPERM;
83 struct task_struct *t = current;
84
85 if (is_realtime(t))
86 ret = do_wait_for_ts_release();
87
88 return ret;
89}
90
91
92asmlinkage long sys_release_ts(lt_t __user *__delay)
93{
94 long ret;
95 lt_t delay;
96
97 /* FIXME: check capabilities... */
98
99 ret = copy_from_user(&delay, __delay, sizeof(delay));
100 if (ret == 0)
101 ret = do_release_ts(litmus_clock() + delay);
102
103 return ret;
104}
diff --git a/litmus/trace.c b/litmus/trace.c
new file mode 100644
index 000000000000..440376998dc9
--- /dev/null
+++ b/litmus/trace.c
@@ -0,0 +1,103 @@
1#include <linux/module.h>
2
3#include <litmus/ftdev.h>
4#include <litmus/litmus.h>
5#include <litmus/trace.h>
6
7/******************************************************************************/
8/* Allocation */
9/******************************************************************************/
10
11static struct ftdev overhead_dev;
12
13#define trace_ts_buf overhead_dev.minor[0].buf
14
15static unsigned int ts_seq_no = 0;
16
17static inline void __save_timestamp_cpu(unsigned long event,
18 uint8_t type, uint8_t cpu)
19{
20 unsigned int seq_no;
21 struct timestamp *ts;
22 seq_no = fetch_and_inc((int *) &ts_seq_no);
23 if (ft_buffer_start_write(trace_ts_buf, (void**) &ts)) {
24 ts->event = event;
25 ts->timestamp = ft_timestamp();
26 ts->seq_no = seq_no;
27 ts->cpu = cpu;
28 ts->task_type = type;
29 ft_buffer_finish_write(trace_ts_buf, ts);
30 }
31}
32
33static inline void __save_timestamp(unsigned long event,
34 uint8_t type)
35{
36 __save_timestamp_cpu(event, type, raw_smp_processor_id());
37}
38
39feather_callback void save_timestamp(unsigned long event)
40{
41 __save_timestamp(event, TSK_UNKNOWN);
42}
43
44feather_callback void save_timestamp_def(unsigned long event,
45 unsigned long type)
46{
47 __save_timestamp(event, (uint8_t) type);
48}
49
50feather_callback void save_timestamp_task(unsigned long event,
51 unsigned long t_ptr)
52{
53 int rt = is_realtime((struct task_struct *) t_ptr);
54 __save_timestamp(event, rt ? TSK_RT : TSK_BE);
55}
56
57feather_callback void save_timestamp_cpu(unsigned long event,
58 unsigned long cpu)
59{
60 __save_timestamp_cpu(event, TSK_UNKNOWN, cpu);
61}
62
63/******************************************************************************/
64/* DEVICE FILE DRIVER */
65/******************************************************************************/
66
67/*
68 * should be 8M; it is the max we can ask to buddy system allocator (MAX_ORDER)
69 * and we might not get as much
70 */
71#define NO_TIMESTAMPS (2 << 11)
72
73/* set MAJOR to 0 to have it dynamically assigned */
74#define FT_TRACE_MAJOR 252
75
76static int alloc_timestamp_buffer(struct ftdev* ftdev, unsigned int idx)
77{
78 unsigned int count = NO_TIMESTAMPS;
79 while (count && !trace_ts_buf) {
80 printk("time stamp buffer: trying to allocate %u time stamps.\n", count);
81 ftdev->minor[idx].buf = alloc_ft_buffer(count, sizeof(struct timestamp));
82 count /= 2;
83 }
84 return ftdev->minor[idx].buf ? 0 : -ENOMEM;
85}
86
87static void free_timestamp_buffer(struct ftdev* ftdev, unsigned int idx)
88{
89 free_ft_buffer(ftdev->minor[idx].buf);
90 ftdev->minor[idx].buf = NULL;
91}
92
93static int __init init_ft_overhead_trace(void)
94{
95 printk("Initializing Feather-Trace overhead tracing device.\n");
96 ftdev_init(&overhead_dev, THIS_MODULE);
97 overhead_dev.minor_cnt = 1; /* only one buffer */
98 overhead_dev.alloc = alloc_timestamp_buffer;
99 overhead_dev.free = free_timestamp_buffer;
100 return register_ftdev(&overhead_dev, "ft_trace", FT_TRACE_MAJOR);
101}
102
103module_init(init_ft_overhead_trace);
generated by cgit v1.2.2 (git 2.25.0) at 2025-04-25 16:41:33 -0400