aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
Diffstat
-rw-r--r--litmus/Makefile4
-rw-r--r--litmus/sched_pfp.c1746
2 files changed, 1749 insertions, 1 deletions
diff --git a/litmus/Makefile b/litmus/Makefile
index 8428360eca09..9757399238de 100644
--- a/litmus/Makefile
+++ b/litmus/Makefile
@@ -19,7 +19,9 @@ obj-y = sched_plugin.o litmus.o \
19 binheap.o \ 19 binheap.o \
20 ctrldev.o \ 20 ctrldev.o \
21 sched_gsn_edf.o \ 21 sched_gsn_edf.o \
22 sched_psn_edf.o 22 sched_psn_edf.o \
23 sched_pfp.o
24
23 25
24obj-$(CONFIG_SCHED_CPU_AFFINITY) += affinity.o 26obj-$(CONFIG_SCHED_CPU_AFFINITY) += affinity.o
25 27
diff --git a/litmus/sched_pfp.c b/litmus/sched_pfp.c
new file mode 100644
index 000000000000..01ac97d7f161
--- /dev/null
+++ b/litmus/sched_pfp.c
@@ -0,0 +1,1746 @@
1/*
2 * litmus/sched_pfp.c
3 *
4 * Implementation of partitioned fixed-priority scheduling.
5 * Based on PSN-EDF.
6 */
7
8#include <linux/percpu.h>
9#include <linux/sched.h>
10#include <linux/list.h>
11#include <linux/spinlock.h>
12#include <linux/module.h>
13
14#include <litmus/litmus.h>
15#include <litmus/wait.h>
16#include <litmus/jobs.h>
17#include <litmus/preempt.h>
18#include <litmus/fp_common.h>
19#include <litmus/sched_plugin.h>
20#include <litmus/sched_trace.h>
21#include <litmus/trace.h>
22#include <litmus/budget.h>
23
24#include <linux/uaccess.h>
25
26
27typedef struct {
28 rt_domain_t domain;
29 struct fp_prio_queue ready_queue;
30 int cpu;
31 struct task_struct* scheduled; /* only RT tasks */
32/*
33 * scheduling lock slock
34 * protects the domain and serializes scheduling decisions
35 */
36#define slock domain.ready_lock
37
38} pfp_domain_t;
39
40DEFINE_PER_CPU(pfp_domain_t, pfp_domains);
41
42pfp_domain_t* pfp_doms[NR_CPUS];
43
44#define local_pfp (&__get_cpu_var(pfp_domains))
45#define remote_dom(cpu) (&per_cpu(pfp_domains, cpu).domain)
46#define remote_pfp(cpu) (&per_cpu(pfp_domains, cpu))
47#define task_dom(task) remote_dom(get_partition(task))
48#define task_pfp(task) remote_pfp(get_partition(task))
49
50/* we assume the lock is being held */
51static void preempt(pfp_domain_t *pfp)
52{
53 preempt_if_preemptable(pfp->scheduled, pfp->cpu);
54}
55
56static unsigned int priority_index(struct task_struct* t)
57{
58#ifdef CONFIG_LITMUS_LOCKING
59 if (unlikely(t->rt_param.inh_task))
60 /* use effective priority */
61 t = t->rt_param.inh_task;
62
63 if (is_priority_boosted(t)) {
64 /* zero is reserved for priority-boosted tasks */
65 return 0;
66 } else
67#endif
68 return get_priority(t);
69}
70
71
72static void pfp_release_jobs(rt_domain_t* rt, struct bheap* tasks)
73{
74 pfp_domain_t *pfp = container_of(rt, pfp_domain_t, domain);
75 unsigned long flags;
76 struct task_struct* t;
77 struct bheap_node* hn;
78
79 raw_spin_lock_irqsave(&pfp->slock, flags);
80
81 while (!bheap_empty(tasks)) {
82 hn = bheap_take(fp_ready_order, tasks);
83 t = bheap2task(hn);
84 TRACE_TASK(t, "released (part:%d prio:%d)\n",
85 get_partition(t), get_priority(t));
86 fp_prio_add(&pfp->ready_queue, t, priority_index(t));
87 }
88
89 /* do we need to preempt? */
90 if (fp_higher_prio(fp_prio_peek(&pfp->ready_queue), pfp->scheduled)) {
91 TRACE_CUR("preempted by new release\n");
92 preempt(pfp);
93 }
94
95 raw_spin_unlock_irqrestore(&pfp->slock, flags);
96}
97
98static void pfp_preempt_check(pfp_domain_t *pfp)
99{
100 if (fp_higher_prio(fp_prio_peek(&pfp->ready_queue), pfp->scheduled))
101 preempt(pfp);
102}
103
104static void pfp_domain_init(pfp_domain_t* pfp,
105 int cpu)
106{
107 fp_domain_init(&pfp->domain, NULL, pfp_release_jobs);
108 pfp->cpu = cpu;
109 pfp->scheduled = NULL;
110 fp_prio_queue_init(&pfp->ready_queue);
111}
112
113static void requeue(struct task_struct* t, pfp_domain_t *pfp)
114{
115 BUG_ON(!is_running(t));
116
117 tsk_rt(t)->completed = 0;
118 if (is_released(t, litmus_clock()))
119 fp_prio_add(&pfp->ready_queue, t, priority_index(t));
120 else
121 add_release(&pfp->domain, t); /* it has got to wait */
122}
123
124static void job_completion(struct task_struct* t, int forced)
125{
126 sched_trace_task_completion(t,forced);
127 TRACE_TASK(t, "job_completion().\n");
128
129 tsk_rt(t)->completed = 0;
130 prepare_for_next_period(t);
131 if (is_released(t, litmus_clock()))
132 sched_trace_task_release(t);
133}
134
135static void pfp_tick(struct task_struct *t)
136{
137 pfp_domain_t *pfp = local_pfp;
138
139 /* Check for inconsistency. We don't need the lock for this since
140 * ->scheduled is only changed in schedule, which obviously is not
141 * executing in parallel on this CPU
142 */
143 BUG_ON(is_realtime(t) && t != pfp->scheduled);
144
145 if (is_realtime(t) && budget_enforced(t) && budget_exhausted(t)) {
146 if (!is_np(t)) {
147 litmus_reschedule_local();
148 TRACE("pfp_scheduler_tick: "
149 "%d is preemptable "
150 " => FORCE_RESCHED\n", t->pid);
151 } else if (is_user_np(t)) {
152 TRACE("pfp_scheduler_tick: "
153 "%d is non-preemptable, "
154 "preemption delayed.\n", t->pid);
155 request_exit_np(t);
156 }
157 }
158}
159
160static struct task_struct* pfp_schedule(struct task_struct * prev)
161{
162 pfp_domain_t* pfp = local_pfp;
163 struct task_struct* next;
164
165 int out_of_time, sleep, preempt, np, exists, blocks, resched, migrate;
166
167 raw_spin_lock(&pfp->slock);
168
169 /* sanity checking
170 * differently from gedf, when a task exits (dead)
171 * pfp->schedule may be null and prev _is_ realtime
172 */
173 BUG_ON(pfp->scheduled && pfp->scheduled != prev);
174 BUG_ON(pfp->scheduled && !is_realtime(prev));
175
176 /* (0) Determine state */
177 exists = pfp->scheduled != NULL;
178 blocks = exists && !is_running(pfp->scheduled);
179 out_of_time = exists &&
180 budget_enforced(pfp->scheduled) &&
181 budget_exhausted(pfp->scheduled);
182 np = exists && is_np(pfp->scheduled);
183 sleep = exists && is_completed(pfp->scheduled);
184 migrate = exists && get_partition(pfp->scheduled) != pfp->cpu;
185 preempt = !blocks && (migrate || fp_preemption_needed(&pfp->ready_queue, prev));
186
187 /* If we need to preempt do so.
188 * The following checks set resched to 1 in case of special
189 * circumstances.
190 */
191 resched = preempt;
192
193 /* If a task blocks we have no choice but to reschedule.
194 */
195 if (blocks)
196 resched = 1;
197
198 /* Request a sys_exit_np() call if we would like to preempt but cannot.
199 * Multiple calls to request_exit_np() don't hurt.
200 */
201 if (np && (out_of_time || preempt || sleep))
202 request_exit_np(pfp->scheduled);
203
204 /* Any task that is preemptable and either exhausts its execution
205 * budget or wants to sleep completes. We may have to reschedule after
206 * this.
207 */
208 if (!np && (out_of_time || sleep) && !blocks && !migrate) {
209 job_completion(pfp->scheduled, !sleep);
210 resched = 1;
211 }
212
213 /* The final scheduling decision. Do we need to switch for some reason?
214 * Switch if we are in RT mode and have no task or if we need to
215 * resched.
216 */
217 next = NULL;
218 if ((!np || blocks) && (resched || !exists)) {
219 /* When preempting a task that does not block, then
220 * re-insert it into either the ready queue or the
221 * release queue (if it completed). requeue() picks
222 * the appropriate queue.
223 */
224 if (pfp->scheduled && !blocks && !migrate)
225 requeue(pfp->scheduled, pfp);
226 next = fp_prio_take(&pfp->ready_queue);
227 if (next == prev) {
228 struct task_struct *t = fp_prio_peek(&pfp->ready_queue);
229 TRACE_TASK(next, "next==prev sleep=%d oot=%d np=%d preempt=%d migrate=%d "
230 "boost=%d empty=%d prio-idx=%u prio=%u\n",
231 sleep, out_of_time, np, preempt, migrate,
232 is_priority_boosted(next),
233 t == NULL,
234 priority_index(next),
235 get_priority(next));
236 if (t)
237 TRACE_TASK(t, "waiter boost=%d prio-idx=%u prio=%u\n",
238 is_priority_boosted(t),
239 priority_index(t),
240 get_priority(t));
241 }
242 /* If preempt is set, we should not see the same task again. */
243 BUG_ON(preempt && next == prev);
244 /* Similarly, if preempt is set, then next may not be NULL,
245 * unless it's a migration. */
246 BUG_ON(preempt && !migrate && next == NULL);
247 } else
248 /* Only override Linux scheduler if we have a real-time task
249 * scheduled that needs to continue.
250 */
251 if (exists)
252 next = prev;
253
254 if (next) {
255 TRACE_TASK(next, "scheduled at %llu\n", litmus_clock());
256 } else {
257 TRACE("becoming idle at %llu\n", litmus_clock());
258 }
259
260 pfp->scheduled = next;
261 sched_state_task_picked();
262 raw_spin_unlock(&pfp->slock);
263
264 return next;
265}
266
267#ifdef CONFIG_LITMUS_LOCKING
268
269/* prev is no longer scheduled --- see if it needs to migrate */
270static void pfp_finish_switch(struct task_struct *prev)
271{
272 pfp_domain_t *to;
273
274 if (is_realtime(prev) &&
275 is_running(prev) &&
276 get_partition(prev) != smp_processor_id()) {
277 TRACE_TASK(prev, "needs to migrate from P%d to P%d\n",
278 smp_processor_id(), get_partition(prev));
279
280 to = task_pfp(prev);
281
282 raw_spin_lock(&to->slock);
283
284 TRACE_TASK(prev, "adding to queue on P%d\n", to->cpu);
285 requeue(prev, to);
286 if (fp_preemption_needed(&to->ready_queue, to->scheduled))
287 preempt(to);
288
289 raw_spin_unlock(&to->slock);
290
291 }
292}
293
294#endif
295
296/* Prepare a task for running in RT mode
297 */
298static void pfp_task_new(struct task_struct * t, int on_rq, int is_scheduled)
299{
300 pfp_domain_t* pfp = task_pfp(t);
301 unsigned long flags;
302
303 TRACE_TASK(t, "P-FP: task new, cpu = %d\n",
304 t->rt_param.task_params.cpu);
305
306 /* setup job parameters */
307 release_at(t, litmus_clock());
308
309 raw_spin_lock_irqsave(&pfp->slock, flags);
310 if (is_scheduled) {
311 /* there shouldn't be anything else running at the time */
312 BUG_ON(pfp->scheduled);
313 pfp->scheduled = t;
314 } else if (is_running(t)) {
315 requeue(t, pfp);
316 /* maybe we have to reschedule */
317 pfp_preempt_check(pfp);
318 }
319 raw_spin_unlock_irqrestore(&pfp->slock, flags);
320}
321
322static void pfp_task_wake_up(struct task_struct *task)
323{
324 unsigned long flags;
325 pfp_domain_t* pfp = task_pfp(task);
326 lt_t now;
327
328 TRACE_TASK(task, "wake_up at %llu\n", litmus_clock());
329 raw_spin_lock_irqsave(&pfp->slock, flags);
330
331#ifdef CONFIG_LITMUS_LOCKING
332 /* Should only be queued when processing a fake-wake up due to a
333 * migration-related state change. */
334 if (unlikely(is_queued(task))) {
335 TRACE_TASK(task, "WARNING: waking task still queued. Is this right?\n");
336 goto out_unlock;
337 }
338#else
339 BUG_ON(is_queued(task));
340#endif
341 now = litmus_clock();
342 if (is_sporadic(task) && is_tardy(task, now)
343#ifdef CONFIG_LITMUS_LOCKING
344 /* We need to take suspensions because of semaphores into
345 * account! If a job resumes after being suspended due to acquiring
346 * a semaphore, it should never be treated as a new job release.
347 */
348 && !is_priority_boosted(task)
349#endif
350 ) {
351 /* new sporadic release */
352 release_at(task, now);
353 sched_trace_task_release(task);
354 }
355
356 /* Only add to ready queue if it is not the currently-scheduled
357 * task. This could be the case if a task was woken up concurrently
358 * on a remote CPU before the executing CPU got around to actually
359 * de-scheduling the task, i.e., wake_up() raced with schedule()
360 * and won. Also, don't requeue if it is still queued, which can
361 * happen under the DPCP due wake-ups racing with migrations.
362 */
363 if (pfp->scheduled != task) {
364 requeue(task, pfp);
365 pfp_preempt_check(pfp);
366 }
367
368#ifdef CONFIG_LITMUS_LOCKING
369out_unlock:
370#endif
371 raw_spin_unlock_irqrestore(&pfp->slock, flags);
372 TRACE_TASK(task, "wake up done\n");
373}
374
375static void pfp_task_block(struct task_struct *t)
376{
377 /* only running tasks can block, thus t is in no queue */
378 TRACE_TASK(t, "block at %llu, state=%d\n", litmus_clock(), t->state);
379
380 BUG_ON(!is_realtime(t));
381
382 /* If this task blocked normally, it shouldn't be queued. The exception is
383 * if this is a simulated block()/wakeup() pair from the pull-migration code path.
384 * This should only happen if the DPCP is being used.
385 */
386#ifdef CONFIG_LITMUS_LOCKING
387 if (unlikely(is_queued(t)))
388 TRACE_TASK(t, "WARNING: blocking task still queued. Is this right?\n");
389#else
390 BUG_ON(is_queued(t));
391#endif
392}
393
394static void pfp_task_exit(struct task_struct * t)
395{
396 unsigned long flags;
397 pfp_domain_t* pfp = task_pfp(t);
398 rt_domain_t* dom;
399
400 raw_spin_lock_irqsave(&pfp->slock, flags);
401 if (is_queued(t)) {
402 BUG(); /* This currently doesn't work. */
403 /* dequeue */
404 dom = task_dom(t);
405 remove(dom, t);
406 }
407 if (pfp->scheduled == t) {
408 pfp->scheduled = NULL;
409 preempt(pfp);
410 }
411 TRACE_TASK(t, "RIP, now reschedule\n");
412
413 raw_spin_unlock_irqrestore(&pfp->slock, flags);
414}
415
416#ifdef CONFIG_LITMUS_LOCKING
417
418#include <litmus/fdso.h>
419#include <litmus/srp.h>
420
421static void fp_dequeue(pfp_domain_t* pfp, struct task_struct* t)
422{
423 BUG_ON(pfp->scheduled == t && is_queued(t));
424 if (is_queued(t))
425 fp_prio_remove(&pfp->ready_queue, t, priority_index(t));
426}
427
428static void fp_set_prio_inh(pfp_domain_t* pfp, struct task_struct* t,
429 struct task_struct* prio_inh)
430{
431 int requeue;
432
433 if (!t || t->rt_param.inh_task == prio_inh) {
434 /* no update required */
435 if (t)
436 TRACE_TASK(t, "no prio-inh update required\n");
437 return;
438 }
439
440 requeue = is_queued(t);
441 TRACE_TASK(t, "prio-inh: is_queued:%d\n", requeue);
442
443 if (requeue)
444 /* first remove */
445 fp_dequeue(pfp, t);
446
447 t->rt_param.inh_task = prio_inh;
448
449 if (requeue)
450 /* add again to the right queue */
451 fp_prio_add(&pfp->ready_queue, t, priority_index(t));
452}
453
454static int effective_agent_priority(int prio)
455{
456 /* make sure agents have higher priority */
457 return prio - LITMUS_MAX_PRIORITY;
458}
459
460static lt_t prio_point(int eprio)
461{
462 /* make sure we have non-negative prio points */
463 return eprio + LITMUS_MAX_PRIORITY;
464}
465
466static int prio_from_point(lt_t prio_point)
467{
468 return ((int) prio_point) - LITMUS_MAX_PRIORITY;
469}
470
471static void boost_priority(struct task_struct* t, lt_t priority_point)
472{
473 unsigned long flags;
474 pfp_domain_t* pfp = task_pfp(t);
475
476 raw_spin_lock_irqsave(&pfp->slock, flags);
477
478
479 TRACE_TASK(t, "priority boosted at %llu\n", litmus_clock());
480
481 tsk_rt(t)->priority_boosted = 1;
482 /* tie-break by protocol-specific priority point */
483 tsk_rt(t)->boost_start_time = priority_point;
484
485 /* Priority boosting currently only takes effect for already-scheduled
486 * tasks. This is sufficient since priority boosting only kicks in as
487 * part of lock acquisitions. */
488 BUG_ON(pfp->scheduled != t);
489
490 raw_spin_unlock_irqrestore(&pfp->slock, flags);
491}
492
493static void unboost_priority(struct task_struct* t)
494{
495 unsigned long flags;
496 pfp_domain_t* pfp = task_pfp(t);
497 lt_t now;
498
499 raw_spin_lock_irqsave(&pfp->slock, flags);
500 now = litmus_clock();
501
502 /* assumption: this only happens when the job is scheduled */
503 BUG_ON(pfp->scheduled != t);
504
505 TRACE_TASK(t, "priority restored at %llu\n", now);
506
507 /* priority boosted jobs must be scheduled */
508 BUG_ON(pfp->scheduled != t);
509
510 tsk_rt(t)->priority_boosted = 0;
511 tsk_rt(t)->boost_start_time = 0;
512
513 /* check if this changes anything */
514 if (fp_preemption_needed(&pfp->ready_queue, pfp->scheduled))
515 preempt(pfp);
516
517 raw_spin_unlock_irqrestore(&pfp->slock, flags);
518}
519
520/* ******************** SRP support ************************ */
521
522static unsigned int pfp_get_srp_prio(struct task_struct* t)
523{
524 return get_priority(t);
525}
526
527/* ******************** FMLP support ********************** */
528
529struct fmlp_semaphore {
530 struct litmus_lock litmus_lock;
531
532 /* current resource holder */
533 struct task_struct *owner;
534
535 /* FIFO queue of waiting tasks */
536 wait_queue_head_t wait;
537};
538
539static inline struct fmlp_semaphore* fmlp_from_lock(struct litmus_lock* lock)
540{
541 return container_of(lock, struct fmlp_semaphore, litmus_lock);
542}
543int pfp_fmlp_lock(struct litmus_lock* l)
544{
545 struct task_struct* t = current;
546 struct fmlp_semaphore *sem = fmlp_from_lock(l);
547 wait_queue_t wait;
548 unsigned long flags;
549 lt_t time_of_request;
550
551 if (!is_realtime(t))
552 return -EPERM;
553
554 /* prevent nested lock acquisition --- not supported by FMLP */
555 if (tsk_rt(t)->num_locks_held ||
556 tsk_rt(t)->num_local_locks_held)
557 return -EBUSY;
558
559 spin_lock_irqsave(&sem->wait.lock, flags);
560
561 /* tie-break by this point in time */
562 time_of_request = litmus_clock();
563
564 /* Priority-boost ourself *before* we suspend so that
565 * our priority is boosted when we resume. */
566 boost_priority(t, time_of_request);
567
568 if (sem->owner) {
569 /* resource is not free => must suspend and wait */
570
571 init_waitqueue_entry(&wait, t);
572
573 /* FIXME: interruptible would be nice some day */
574 set_task_state(t, TASK_UNINTERRUPTIBLE);
575
576 __add_wait_queue_tail_exclusive(&sem->wait, &wait);
577
578 TS_LOCK_SUSPEND;
579
580 /* release lock before sleeping */
581 spin_unlock_irqrestore(&sem->wait.lock, flags);
582
583 /* We depend on the FIFO order. Thus, we don't need to recheck
584 * when we wake up; we are guaranteed to have the lock since
585 * there is only one wake up per release.
586 */
587
588 schedule();
589
590 TS_LOCK_RESUME;
591
592 /* Since we hold the lock, no other task will change
593 * ->owner. We can thus check it without acquiring the spin
594 * lock. */
595 BUG_ON(sem->owner != t);
596 } else {
597 /* it's ours now */
598 sem->owner = t;
599
600 spin_unlock_irqrestore(&sem->wait.lock, flags);
601 }
602
603 tsk_rt(t)->num_locks_held++;
604
605 return 0;
606}
607
608int pfp_fmlp_unlock(struct litmus_lock* l)
609{
610 struct task_struct *t = current, *next;
611 struct fmlp_semaphore *sem = fmlp_from_lock(l);
612 unsigned long flags;
613 int err = 0;
614
615 spin_lock_irqsave(&sem->wait.lock, flags);
616
617 if (sem->owner != t) {
618 err = -EINVAL;
619 goto out;
620 }
621
622 tsk_rt(t)->num_locks_held--;
623
624 /* we lose the benefit of priority boosting */
625
626 unboost_priority(t);
627
628 /* check if there are jobs waiting for this resource */
629 next = __waitqueue_remove_first(&sem->wait);
630 if (next) {
631 /* next becomes the resouce holder */
632 sem->owner = next;
633
634 /* Wake up next. The waiting job is already priority-boosted. */
635 wake_up_process(next);
636 } else
637 /* resource becomes available */
638 sem->owner = NULL;
639
640out:
641 spin_unlock_irqrestore(&sem->wait.lock, flags);
642 return err;
643}
644
645int pfp_fmlp_close(struct litmus_lock* l)
646{
647 struct task_struct *t = current;
648 struct fmlp_semaphore *sem = fmlp_from_lock(l);
649 unsigned long flags;
650
651 int owner;
652
653 spin_lock_irqsave(&sem->wait.lock, flags);
654
655 owner = sem->owner == t;
656
657 spin_unlock_irqrestore(&sem->wait.lock, flags);
658
659 if (owner)
660 pfp_fmlp_unlock(l);
661
662 return 0;
663}
664
665void pfp_fmlp_free(struct litmus_lock* lock)
666{
667 kfree(fmlp_from_lock(lock));
668}
669
670static struct litmus_lock_ops pfp_fmlp_lock_ops = {
671 .close = pfp_fmlp_close,
672 .lock = pfp_fmlp_lock,
673 .unlock = pfp_fmlp_unlock,
674 .deallocate = pfp_fmlp_free,
675};
676
677static struct litmus_lock* pfp_new_fmlp(void)
678{
679 struct fmlp_semaphore* sem;
680
681 sem = kmalloc(sizeof(*sem), GFP_KERNEL);
682 if (!sem)
683 return NULL;
684
685 sem->owner = NULL;
686 init_waitqueue_head(&sem->wait);
687 sem->litmus_lock.ops = &pfp_fmlp_lock_ops;
688
689 return &sem->litmus_lock;
690}
691
692/* ******************** MPCP support ********************** */
693
694struct mpcp_semaphore {
695 struct litmus_lock litmus_lock;
696
697 /* current resource holder */
698 struct task_struct *owner;
699
700 /* priority queue of waiting tasks */
701 wait_queue_head_t wait;
702
703 /* priority ceiling per cpu */
704 unsigned int prio_ceiling[NR_CPUS];
705
706 /* should jobs spin "virtually" for this resource? */
707 int vspin;
708};
709
710#define OMEGA_CEILING UINT_MAX
711
712/* Since jobs spin "virtually" while waiting to acquire a lock,
713 * they first must aquire a local per-cpu resource.
714 */
715static DEFINE_PER_CPU(wait_queue_head_t, mpcpvs_vspin_wait);
716static DEFINE_PER_CPU(struct task_struct*, mpcpvs_vspin);
717
718/* called with preemptions off <=> no local modifications */
719static void mpcp_vspin_enter(void)
720{
721 struct task_struct* t = current;
722
723 while (1) {
724 if (__get_cpu_var(mpcpvs_vspin) == NULL) {
725 /* good, we get to issue our request */
726 __get_cpu_var(mpcpvs_vspin) = t;
727 break;
728 } else {
729 /* some job is spinning => enqueue in request queue */
730 prio_wait_queue_t wait;
731 wait_queue_head_t* vspin = &__get_cpu_var(mpcpvs_vspin_wait);
732 unsigned long flags;
733
734 /* ordered by regular priority */
735 init_prio_waitqueue_entry(&wait, t, prio_point(get_priority(t)));
736
737 spin_lock_irqsave(&vspin->lock, flags);
738
739 set_task_state(t, TASK_UNINTERRUPTIBLE);
740
741 __add_wait_queue_prio_exclusive(vspin, &wait);
742
743 spin_unlock_irqrestore(&vspin->lock, flags);
744
745 TS_LOCK_SUSPEND;
746
747 preempt_enable_no_resched();
748
749 schedule();
750
751 preempt_disable();
752
753 TS_LOCK_RESUME;
754 /* Recheck if we got it --- some higher-priority process might
755 * have swooped in. */
756 }
757 }
758 /* ok, now it is ours */
759}
760
761/* called with preemptions off */
762static void mpcp_vspin_exit(void)
763{
764 struct task_struct* t = current, *next;
765 unsigned long flags;
766 wait_queue_head_t* vspin = &__get_cpu_var(mpcpvs_vspin_wait);
767
768 BUG_ON(__get_cpu_var(mpcpvs_vspin) != t);
769
770 /* no spinning job */
771 __get_cpu_var(mpcpvs_vspin) = NULL;
772
773 /* see if anyone is waiting for us to stop "spinning" */
774 spin_lock_irqsave(&vspin->lock, flags);
775 next = __waitqueue_remove_first(vspin);
776
777 if (next)
778 wake_up_process(next);
779
780 spin_unlock_irqrestore(&vspin->lock, flags);
781}
782
783static inline struct mpcp_semaphore* mpcp_from_lock(struct litmus_lock* lock)
784{
785 return container_of(lock, struct mpcp_semaphore, litmus_lock);
786}
787
788int pfp_mpcp_lock(struct litmus_lock* l)
789{
790 struct task_struct* t = current;
791 struct mpcp_semaphore *sem = mpcp_from_lock(l);
792 prio_wait_queue_t wait;
793 unsigned long flags;
794
795 if (!is_realtime(t))
796 return -EPERM;
797
798 /* prevent nested lock acquisition */
799 if (tsk_rt(t)->num_locks_held ||
800 tsk_rt(t)->num_local_locks_held)
801 return -EBUSY;
802
803 preempt_disable();
804
805 if (sem->vspin)
806 mpcp_vspin_enter();
807
808 /* Priority-boost ourself *before* we suspend so that
809 * our priority is boosted when we resume. Use the priority
810 * ceiling for the local partition. */
811 boost_priority(t, sem->prio_ceiling[get_partition(t)]);
812
813 spin_lock_irqsave(&sem->wait.lock, flags);
814
815 preempt_enable_no_resched();
816
817 if (sem->owner) {
818 /* resource is not free => must suspend and wait */
819
820 /* ordered by regular priority */
821 init_prio_waitqueue_entry(&wait, t, prio_point(get_priority(t)));
822
823 /* FIXME: interruptible would be nice some day */
824 set_task_state(t, TASK_UNINTERRUPTIBLE);
825
826 __add_wait_queue_prio_exclusive(&sem->wait, &wait);
827
828 TS_LOCK_SUSPEND;
829
830 /* release lock before sleeping */
831 spin_unlock_irqrestore(&sem->wait.lock, flags);
832
833 /* We depend on the FIFO order. Thus, we don't need to recheck
834 * when we wake up; we are guaranteed to have the lock since
835 * there is only one wake up per release.
836 */
837
838 schedule();
839
840 TS_LOCK_RESUME;
841
842 /* Since we hold the lock, no other task will change
843 * ->owner. We can thus check it without acquiring the spin
844 * lock. */
845 BUG_ON(sem->owner != t);
846 } else {
847 /* it's ours now */
848 sem->owner = t;
849
850 spin_unlock_irqrestore(&sem->wait.lock, flags);
851 }
852
853 tsk_rt(t)->num_locks_held++;
854
855 return 0;
856}
857
858int pfp_mpcp_unlock(struct litmus_lock* l)
859{
860 struct task_struct *t = current, *next;
861 struct mpcp_semaphore *sem = mpcp_from_lock(l);
862 unsigned long flags;
863 int err = 0;
864
865 spin_lock_irqsave(&sem->wait.lock, flags);
866
867 if (sem->owner != t) {
868 err = -EINVAL;
869 goto out;
870 }
871
872
873 tsk_rt(t)->num_locks_held--;
874
875 /* we lose the benefit of priority boosting */
876
877 unboost_priority(t);
878
879 /* check if there are jobs waiting for this resource */
880 next = __waitqueue_remove_first(&sem->wait);
881 if (next) {
882 /* next becomes the resouce holder */
883 sem->owner = next;
884
885 /* Wake up next. The waiting job is already priority-boosted. */
886 wake_up_process(next);
887 } else
888 /* resource becomes available */
889 sem->owner = NULL;
890
891out:
892 spin_unlock_irqrestore(&sem->wait.lock, flags);
893
894 if (sem->vspin && err == 0) {
895 preempt_disable();
896 mpcp_vspin_exit();
897 preempt_enable();
898 }
899
900 return err;
901}
902
903int pfp_mpcp_open(struct litmus_lock* l, void* config)
904{
905 struct task_struct *t = current;
906 struct mpcp_semaphore *sem = mpcp_from_lock(l);
907 int cpu, local_cpu;
908 unsigned long flags;
909
910 if (!is_realtime(t))
911 /* we need to know the real-time priority */
912 return -EPERM;
913
914 local_cpu = get_partition(t);
915
916 spin_lock_irqsave(&sem->wait.lock, flags);
917
918 for (cpu = 0; cpu < NR_CPUS; cpu++)
919 if (cpu != local_cpu)
920 {
921 sem->prio_ceiling[cpu] = min(sem->prio_ceiling[cpu],
922 get_priority(t));
923 TRACE_CUR("priority ceiling for sem %p is now %d on cpu %d\n",
924 sem, sem->prio_ceiling[cpu], cpu);
925 }
926
927 spin_unlock_irqrestore(&sem->wait.lock, flags);
928
929 return 0;
930}
931
932int pfp_mpcp_close(struct litmus_lock* l)
933{
934 struct task_struct *t = current;
935 struct mpcp_semaphore *sem = mpcp_from_lock(l);
936 unsigned long flags;
937
938 int owner;
939
940 spin_lock_irqsave(&sem->wait.lock, flags);
941
942 owner = sem->owner == t;
943
944 spin_unlock_irqrestore(&sem->wait.lock, flags);
945
946 if (owner)
947 pfp_mpcp_unlock(l);
948
949 return 0;
950}
951
952void pfp_mpcp_free(struct litmus_lock* lock)
953{
954 kfree(mpcp_from_lock(lock));
955}
956
957static struct litmus_lock_ops pfp_mpcp_lock_ops = {
958 .close = pfp_mpcp_close,
959 .lock = pfp_mpcp_lock,
960 .open = pfp_mpcp_open,
961 .unlock = pfp_mpcp_unlock,
962 .deallocate = pfp_mpcp_free,
963};
964
965static struct litmus_lock* pfp_new_mpcp(int vspin)
966{
967 struct mpcp_semaphore* sem;
968 int cpu;
969
970 sem = kmalloc(sizeof(*sem), GFP_KERNEL);
971 if (!sem)
972 return NULL;
973
974 sem->owner = NULL;
975 init_waitqueue_head(&sem->wait);
976 sem->litmus_lock.ops = &pfp_mpcp_lock_ops;
977
978 for (cpu = 0; cpu < NR_CPUS; cpu++)
979 sem->prio_ceiling[cpu] = OMEGA_CEILING;
980
981 /* mark as virtual spinning */
982 sem->vspin = vspin;
983
984 return &sem->litmus_lock;
985}
986
987
988/* ******************** PCP support ********************** */
989
990
991struct pcp_semaphore {
992 struct litmus_lock litmus_lock;
993
994 struct list_head ceiling;
995
996 /* current resource holder */
997 struct task_struct *owner;
998
999 /* priority ceiling --- can be negative due to DPCP support */
1000 int prio_ceiling;
1001
1002 /* on which processor is this PCP semaphore allocated? */
1003 int on_cpu;
1004};
1005
1006static inline struct pcp_semaphore* pcp_from_lock(struct litmus_lock* lock)
1007{
1008 return container_of(lock, struct pcp_semaphore, litmus_lock);
1009}
1010
1011
1012struct pcp_state {
1013 struct list_head system_ceiling;
1014
1015 /* highest-priority waiting task */
1016 struct task_struct* hp_waiter;
1017
1018 /* list of jobs waiting to get past the system ceiling */
1019 wait_queue_head_t ceiling_blocked;
1020};
1021
1022static void pcp_init_state(struct pcp_state* s)
1023{
1024 INIT_LIST_HEAD(&s->system_ceiling);
1025 s->hp_waiter = NULL;
1026 init_waitqueue_head(&s->ceiling_blocked);
1027}
1028
1029static DEFINE_PER_CPU(struct pcp_state, pcp_state);
1030
1031/* assumes preemptions are off */
1032static struct pcp_semaphore* pcp_get_ceiling(void)
1033{
1034 struct list_head* top = __get_cpu_var(pcp_state).system_ceiling.next;
1035
1036 if (top)
1037 return list_entry(top, struct pcp_semaphore, ceiling);
1038 else
1039 return NULL;
1040}
1041
1042/* assumes preempt off */
1043static void pcp_add_ceiling(struct pcp_semaphore* sem)
1044{
1045 struct list_head *pos;
1046 struct list_head *in_use = &__get_cpu_var(pcp_state).system_ceiling;
1047 struct pcp_semaphore* held;
1048
1049 BUG_ON(sem->on_cpu != smp_processor_id());
1050 BUG_ON(in_list(&sem->ceiling));
1051
1052 list_for_each(pos, in_use) {
1053 held = list_entry(pos, struct pcp_semaphore, ceiling);
1054 if (held->prio_ceiling >= sem->prio_ceiling) {
1055 __list_add(&sem->ceiling, pos->prev, pos);
1056 return;
1057 }
1058 }
1059
1060 /* we hit the end of the list */
1061
1062 list_add_tail(&sem->ceiling, in_use);
1063}
1064
1065/* assumes preempt off */
1066static int pcp_exceeds_ceiling(struct pcp_semaphore* ceiling,
1067 struct task_struct* task,
1068 int effective_prio)
1069{
1070 return ceiling == NULL ||
1071 ceiling->prio_ceiling > effective_prio ||
1072 ceiling->owner == task;
1073}
1074
1075/* assumes preempt off */
1076static void pcp_priority_inheritance(void)
1077{
1078 unsigned long flags;
1079 pfp_domain_t* pfp = local_pfp;
1080
1081 struct pcp_semaphore* ceiling = pcp_get_ceiling();
1082 struct task_struct *blocker, *blocked;
1083
1084 blocker = ceiling ? ceiling->owner : NULL;
1085 blocked = __get_cpu_var(pcp_state).hp_waiter;
1086
1087 raw_spin_lock_irqsave(&pfp->slock, flags);
1088
1089 /* Current is no longer inheriting anything by default. This should be
1090 * the currently scheduled job, and hence not currently queued. */
1091 BUG_ON(current != pfp->scheduled);
1092
1093 fp_set_prio_inh(pfp, current, NULL);
1094 fp_set_prio_inh(pfp, blocked, NULL);
1095 fp_set_prio_inh(pfp, blocker, NULL);
1096
1097
1098 /* Let blocking job inherit priority of blocked job, if required. */
1099 if (blocker && blocked &&
1100 fp_higher_prio(blocked, blocker)) {
1101 TRACE_TASK(blocker, "PCP inherits from %s/%d (prio %u -> %u) \n",
1102 blocked->comm, blocked->pid,
1103 get_priority(blocker), get_priority(blocked));
1104 fp_set_prio_inh(pfp, blocker, blocked);
1105 }
1106
1107 /* Check if anything changed. If the blocked job is current, then it is
1108 * just blocking and hence is going to call the scheduler anyway. */
1109 if (blocked != current &&
1110 fp_higher_prio(fp_prio_peek(&pfp->ready_queue), pfp->scheduled))
1111 preempt(pfp);
1112
1113 raw_spin_unlock_irqrestore(&pfp->slock, flags);
1114}
1115
1116/* called with preemptions off */
1117static void pcp_raise_ceiling(struct pcp_semaphore* sem,
1118 int effective_prio)
1119{
1120 struct task_struct* t = current;
1121 struct pcp_semaphore* ceiling;
1122 prio_wait_queue_t wait;
1123 unsigned int waiting_higher_prio;
1124
1125 do {
1126 ceiling = pcp_get_ceiling();
1127 if (pcp_exceeds_ceiling(ceiling, t, effective_prio))
1128 break;
1129
1130 TRACE_CUR("PCP ceiling-blocked, wanted sem %p, but %s/%d has the ceiling \n",
1131 sem, ceiling->owner->comm, ceiling->owner->pid);
1132
1133 /* we need to wait until the ceiling is lowered */
1134
1135 /* enqueue in priority order */
1136 init_prio_waitqueue_entry(&wait, t, prio_point(effective_prio));
1137 set_task_state(t, TASK_UNINTERRUPTIBLE);
1138 waiting_higher_prio = add_wait_queue_prio_exclusive(
1139 &__get_cpu_var(pcp_state).ceiling_blocked, &wait);
1140
1141 if (waiting_higher_prio == 0) {
1142 TRACE_CUR("PCP new highest-prio waiter => prio inheritance\n");
1143
1144 /* we are the new highest-priority waiting job
1145 * => update inheritance */
1146 __get_cpu_var(pcp_state).hp_waiter = t;
1147 pcp_priority_inheritance();
1148 }
1149
1150 TS_LOCK_SUSPEND;
1151
1152 preempt_enable_no_resched();
1153 schedule();
1154 preempt_disable();
1155
1156 /* pcp_resume_unblocked() removed us from wait queue */
1157
1158 TS_LOCK_RESUME;
1159 } while(1);
1160
1161 TRACE_CUR("PCP got the ceiling and sem %p\n", sem);
1162
1163 /* We are good to go. The semaphore should be available. */
1164 BUG_ON(sem->owner != NULL);
1165
1166 sem->owner = t;
1167
1168 pcp_add_ceiling(sem);
1169}
1170
1171static void pcp_resume_unblocked(void)
1172{
1173 wait_queue_head_t *blocked = &__get_cpu_var(pcp_state).ceiling_blocked;
1174 unsigned long flags;
1175 prio_wait_queue_t* q;
1176 struct task_struct* t = NULL;
1177
1178 struct pcp_semaphore* ceiling = pcp_get_ceiling();
1179
1180 spin_lock_irqsave(&blocked->lock, flags);
1181
1182 while (waitqueue_active(blocked)) {
1183 /* check first == highest-priority waiting job */
1184 q = list_entry(blocked->task_list.next,
1185 prio_wait_queue_t, wq.task_list);
1186 t = (struct task_struct*) q->wq.private;
1187
1188 /* can it proceed now? => let it go */
1189 if (pcp_exceeds_ceiling(ceiling, t,
1190 prio_from_point(q->priority))) {
1191 __remove_wait_queue(blocked, &q->wq);
1192 wake_up_process(t);
1193 } else {
1194 /* We are done. Update highest-priority waiter. */
1195 __get_cpu_var(pcp_state).hp_waiter = t;
1196 goto out;
1197 }
1198 }
1199 /* If we get here, then there are no more waiting
1200 * jobs. */
1201 __get_cpu_var(pcp_state).hp_waiter = NULL;
1202out:
1203 spin_unlock_irqrestore(&blocked->lock, flags);
1204}
1205
1206/* assumes preempt off */
1207static void pcp_lower_ceiling(struct pcp_semaphore* sem)
1208{
1209 BUG_ON(!in_list(&sem->ceiling));
1210 BUG_ON(sem->owner != current);
1211 BUG_ON(sem->on_cpu != smp_processor_id());
1212
1213 /* remove from ceiling list */
1214 list_del(&sem->ceiling);
1215
1216 /* release */
1217 sem->owner = NULL;
1218
1219 TRACE_CUR("PCP released sem %p\n", sem);
1220
1221 pcp_priority_inheritance();
1222
1223 /* Wake up all ceiling-blocked jobs that now pass the ceiling. */
1224 pcp_resume_unblocked();
1225}
1226
1227static void pcp_update_prio_ceiling(struct pcp_semaphore* sem,
1228 int effective_prio)
1229{
1230 /* This needs to be synchronized on something.
1231 * Might as well use waitqueue lock for the processor.
1232 * We assume this happens only before the task set starts execution,
1233 * (i.e., during initialization), but it may happen on multiple processors
1234 * at the same time.
1235 */
1236 unsigned long flags;
1237
1238 struct pcp_state* s = &per_cpu(pcp_state, sem->on_cpu);
1239
1240 spin_lock_irqsave(&s->ceiling_blocked.lock, flags);
1241
1242 sem->prio_ceiling = min(sem->prio_ceiling, effective_prio);
1243
1244 spin_unlock_irqrestore(&s->ceiling_blocked.lock, flags);
1245}
1246
1247static void pcp_init_semaphore(struct pcp_semaphore* sem, int cpu)
1248{
1249 sem->owner = NULL;
1250 INIT_LIST_HEAD(&sem->ceiling);
1251 sem->prio_ceiling = INT_MAX;
1252 sem->on_cpu = cpu;
1253}
1254
1255int pfp_pcp_lock(struct litmus_lock* l)
1256{
1257 struct task_struct* t = current;
1258 struct pcp_semaphore *sem = pcp_from_lock(l);
1259
1260 int eprio = effective_agent_priority(get_priority(t));
1261 int from = get_partition(t);
1262 int to = sem->on_cpu;
1263
1264 if (!is_realtime(t) || from != to)
1265 return -EPERM;
1266
1267 /* prevent nested lock acquisition in global critical section */
1268 if (tsk_rt(t)->num_locks_held)
1269 return -EBUSY;
1270
1271 preempt_disable();
1272
1273 pcp_raise_ceiling(sem, eprio);
1274
1275 preempt_enable();
1276
1277 tsk_rt(t)->num_local_locks_held++;
1278
1279 return 0;
1280}
1281
1282int pfp_pcp_unlock(struct litmus_lock* l)
1283{
1284 struct task_struct *t = current;
1285 struct pcp_semaphore *sem = pcp_from_lock(l);
1286
1287 int err = 0;
1288
1289 preempt_disable();
1290
1291 if (sem->on_cpu != smp_processor_id() || sem->owner != t) {
1292 err = -EINVAL;
1293 goto out;
1294 }
1295
1296 tsk_rt(t)->num_local_locks_held--;
1297
1298 /* give it back */
1299 pcp_lower_ceiling(sem);
1300
1301out:
1302 preempt_enable();
1303
1304 return err;
1305}
1306
1307int pfp_pcp_open(struct litmus_lock* l, void* __user config)
1308{
1309 struct task_struct *t = current;
1310 struct pcp_semaphore *sem = pcp_from_lock(l);
1311
1312 int cpu, eprio;
1313
1314 if (!is_realtime(t))
1315 /* we need to know the real-time priority */
1316 return -EPERM;
1317
1318 if (!config)
1319 cpu = get_partition(t);
1320 else if (get_user(cpu, (int*) config))
1321 return -EFAULT;
1322
1323 /* make sure the resource location matches */
1324 if (cpu != sem->on_cpu)
1325 return -EINVAL;
1326
1327 eprio = effective_agent_priority(get_priority(t));
1328
1329 pcp_update_prio_ceiling(sem, eprio);
1330
1331 return 0;
1332}
1333
1334int pfp_pcp_close(struct litmus_lock* l)
1335{
1336 struct task_struct *t = current;
1337 struct pcp_semaphore *sem = pcp_from_lock(l);
1338
1339 int owner = 0;
1340
1341 preempt_disable();
1342
1343 if (sem->on_cpu == smp_processor_id())
1344 owner = sem->owner == t;
1345
1346 preempt_enable();
1347
1348 if (owner)
1349 pfp_pcp_unlock(l);
1350
1351 return 0;
1352}
1353
1354void pfp_pcp_free(struct litmus_lock* lock)
1355{
1356 kfree(pcp_from_lock(lock));
1357}
1358
1359
1360static struct litmus_lock_ops pfp_pcp_lock_ops = {
1361 .close = pfp_pcp_close,
1362 .lock = pfp_pcp_lock,
1363 .open = pfp_pcp_open,
1364 .unlock = pfp_pcp_unlock,
1365 .deallocate = pfp_pcp_free,
1366};
1367
1368
1369static struct litmus_lock* pfp_new_pcp(int on_cpu)
1370{
1371 struct pcp_semaphore* sem;
1372
1373 sem = kmalloc(sizeof(*sem), GFP_KERNEL);
1374 if (!sem)
1375 return NULL;
1376
1377 sem->litmus_lock.ops = &pfp_pcp_lock_ops;
1378 pcp_init_semaphore(sem, on_cpu);
1379
1380 return &sem->litmus_lock;
1381}
1382
1383/* ******************** DPCP support ********************** */
1384
1385struct dpcp_semaphore {
1386 struct litmus_lock litmus_lock;
1387 struct pcp_semaphore pcp;
1388 int owner_cpu;
1389};
1390
1391static inline struct dpcp_semaphore* dpcp_from_lock(struct litmus_lock* lock)
1392{
1393 return container_of(lock, struct dpcp_semaphore, litmus_lock);
1394}
1395
1396/* called with preemptions disabled */
1397static void pfp_migrate_to(int target_cpu)
1398{
1399 struct task_struct* t = current;
1400 pfp_domain_t *from;
1401
1402 if (get_partition(t) == target_cpu)
1403 return;
1404
1405 /* make sure target_cpu makes sense */
1406 BUG_ON(!cpu_online(target_cpu));
1407
1408 local_irq_disable();
1409
1410 /* scheduled task should not be in any ready or release queue */
1411 BUG_ON(is_queued(t));
1412
1413 /* lock both pfp domains in order of address */
1414 from = task_pfp(t);
1415
1416 raw_spin_lock(&from->slock);
1417
1418 /* switch partitions */
1419 tsk_rt(t)->task_params.cpu = target_cpu;
1420
1421 raw_spin_unlock(&from->slock);
1422
1423 /* Don't trace scheduler costs as part of
1424 * locking overhead. Scheduling costs are accounted for
1425 * explicitly. */
1426 TS_LOCK_SUSPEND;
1427
1428 local_irq_enable();
1429 preempt_enable_no_resched();
1430
1431 /* deschedule to be migrated */
1432 schedule();
1433
1434 /* we are now on the target processor */
1435 preempt_disable();
1436
1437 /* start recording costs again */
1438 TS_LOCK_RESUME;
1439
1440 BUG_ON(smp_processor_id() != target_cpu);
1441}
1442
1443int pfp_dpcp_lock(struct litmus_lock* l)
1444{
1445 struct task_struct* t = current;
1446 struct dpcp_semaphore *sem = dpcp_from_lock(l);
1447 int eprio = effective_agent_priority(get_priority(t));
1448 int from = get_partition(t);
1449 int to = sem->pcp.on_cpu;
1450
1451 if (!is_realtime(t))
1452 return -EPERM;
1453
1454 /* prevent nested lock accquisition */
1455 if (tsk_rt(t)->num_locks_held ||
1456 tsk_rt(t)->num_local_locks_held)
1457 return -EBUSY;
1458
1459 preempt_disable();
1460
1461 /* Priority-boost ourself *before* we suspend so that
1462 * our priority is boosted when we resume. */
1463
1464 boost_priority(t, get_priority(t));
1465
1466 pfp_migrate_to(to);
1467
1468 pcp_raise_ceiling(&sem->pcp, eprio);
1469
1470 /* yep, we got it => execute request */
1471 sem->owner_cpu = from;
1472
1473 preempt_enable();
1474
1475 tsk_rt(t)->num_locks_held++;
1476
1477 return 0;
1478}
1479
1480int pfp_dpcp_unlock(struct litmus_lock* l)
1481{
1482 struct task_struct *t = current;
1483 struct dpcp_semaphore *sem = dpcp_from_lock(l);
1484 int err = 0;
1485 int home;
1486
1487 preempt_disable();
1488
1489 if (sem->pcp.on_cpu != smp_processor_id() || sem->pcp.owner != t) {
1490 err = -EINVAL;
1491 goto out;
1492 }
1493
1494 tsk_rt(t)->num_locks_held--;
1495
1496 home = sem->owner_cpu;
1497
1498 /* give it back */
1499 pcp_lower_ceiling(&sem->pcp);
1500
1501 /* we lose the benefit of priority boosting */
1502 unboost_priority(t);
1503
1504 pfp_migrate_to(home);
1505
1506out:
1507 preempt_enable();
1508
1509 return err;
1510}
1511
1512int pfp_dpcp_open(struct litmus_lock* l, void* __user config)
1513{
1514 struct task_struct *t = current;
1515 struct dpcp_semaphore *sem = dpcp_from_lock(l);
1516 int cpu, eprio;
1517
1518 if (!is_realtime(t))
1519 /* we need to know the real-time priority */
1520 return -EPERM;
1521
1522 if (get_user(cpu, (int*) config))
1523 return -EFAULT;
1524
1525 /* make sure the resource location matches */
1526 if (cpu != sem->pcp.on_cpu)
1527 return -EINVAL;
1528
1529 eprio = effective_agent_priority(get_priority(t));
1530
1531 pcp_update_prio_ceiling(&sem->pcp, eprio);
1532
1533 return 0;
1534}
1535
1536int pfp_dpcp_close(struct litmus_lock* l)
1537{
1538 struct task_struct *t = current;
1539 struct dpcp_semaphore *sem = dpcp_from_lock(l);
1540 int owner = 0;
1541
1542 preempt_disable();
1543
1544 if (sem->pcp.on_cpu == smp_processor_id())
1545 owner = sem->pcp.owner == t;
1546
1547 preempt_enable();
1548
1549 if (owner)
1550 pfp_dpcp_unlock(l);
1551
1552 return 0;
1553}
1554
1555void pfp_dpcp_free(struct litmus_lock* lock)
1556{
1557 kfree(dpcp_from_lock(lock));
1558}
1559
1560static struct litmus_lock_ops pfp_dpcp_lock_ops = {
1561 .close = pfp_dpcp_close,
1562 .lock = pfp_dpcp_lock,
1563 .open = pfp_dpcp_open,
1564 .unlock = pfp_dpcp_unlock,
1565 .deallocate = pfp_dpcp_free,
1566};
1567
1568static struct litmus_lock* pfp_new_dpcp(int on_cpu)
1569{
1570 struct dpcp_semaphore* sem;
1571
1572 sem = kmalloc(sizeof(*sem), GFP_KERNEL);
1573 if (!sem)
1574 return NULL;
1575
1576 sem->litmus_lock.ops = &pfp_dpcp_lock_ops;
1577 sem->owner_cpu = NO_CPU;
1578 pcp_init_semaphore(&sem->pcp, on_cpu);
1579
1580 return &sem->litmus_lock;
1581}
1582
1583
1584/* **** lock constructor **** */
1585
1586
1587static long pfp_allocate_lock(struct litmus_lock **lock, int type,
1588 void* __user config)
1589{
1590 int err = -ENXIO, cpu;
1591 struct srp_semaphore* srp;
1592
1593 /* P-FP currently supports the SRP for local resources and the FMLP
1594 * for global resources. */
1595 switch (type) {
1596 case FMLP_SEM:
1597 /* FIFO Mutex Locking Protocol */
1598 *lock = pfp_new_fmlp();
1599 if (*lock)
1600 err = 0;
1601 else
1602 err = -ENOMEM;
1603 break;
1604
1605 case MPCP_SEM:
1606 /* Multiprocesor Priority Ceiling Protocol */
1607 *lock = pfp_new_mpcp(0);
1608 if (*lock)
1609 err = 0;
1610 else
1611 err = -ENOMEM;
1612 break;
1613
1614 case MPCP_VS_SEM:
1615 /* Multiprocesor Priority Ceiling Protocol with virtual spinning */
1616 *lock = pfp_new_mpcp(1);
1617 if (*lock)
1618 err = 0;
1619 else
1620 err = -ENOMEM;
1621 break;
1622
1623 case DPCP_SEM:
1624 /* Distributed Priority Ceiling Protocol */
1625 if (get_user(cpu, (int*) config))
1626 return -EFAULT;
1627
1628 if (!cpu_online(cpu))
1629 return -EINVAL;
1630
1631 *lock = pfp_new_dpcp(cpu);
1632 if (*lock)
1633 err = 0;
1634 else
1635 err = -ENOMEM;
1636 break;
1637
1638 case SRP_SEM:
1639 /* Baker's Stack Resource Policy */
1640 srp = allocate_srp_semaphore();
1641 if (srp) {
1642 *lock = &srp->litmus_lock;
1643 err = 0;
1644 } else
1645 err = -ENOMEM;
1646 break;
1647
1648 case PCP_SEM:
1649 /* Priority Ceiling Protocol */
1650 if (!config)
1651 cpu = get_partition(current);
1652 else if (get_user(cpu, (int*) config))
1653 return -EFAULT;
1654
1655 if (!cpu_online(cpu))
1656 return -EINVAL;
1657
1658 *lock = pfp_new_pcp(cpu);
1659 if (*lock)
1660 err = 0;
1661 else
1662 err = -ENOMEM;
1663 break;
1664 };
1665
1666 return err;
1667}
1668
1669#endif
1670
1671static long pfp_admit_task(struct task_struct* tsk)
1672{
1673 if (task_cpu(tsk) == tsk->rt_param.task_params.cpu &&
1674#ifdef CONFIG_RELEASE_MASTER
1675 /* don't allow tasks on release master CPU */
1676 task_cpu(tsk) != remote_dom(task_cpu(tsk))->release_master &&
1677#endif
1678 litmus_is_valid_fixed_prio(get_priority(tsk)))
1679 return 0;
1680 else
1681 return -EINVAL;
1682}
1683
1684static long pfp_activate_plugin(void)
1685{
1686#if defined(CONFIG_RELEASE_MASTER) || defined(CONFIG_LITMUS_LOCKING)
1687 int cpu;
1688#endif
1689
1690#ifdef CONFIG_RELEASE_MASTER
1691 for_each_online_cpu(cpu) {
1692 remote_dom(cpu)->release_master = atomic_read(&release_master_cpu);
1693 }
1694#endif
1695
1696#ifdef CONFIG_LITMUS_LOCKING
1697 get_srp_prio = pfp_get_srp_prio;
1698
1699 for_each_online_cpu(cpu) {
1700 init_waitqueue_head(&per_cpu(mpcpvs_vspin_wait, cpu));
1701 per_cpu(mpcpvs_vspin, cpu) = NULL;
1702
1703 pcp_init_state(&per_cpu(pcp_state, cpu));
1704 pfp_doms[cpu] = remote_pfp(cpu);
1705 }
1706
1707#endif
1708
1709 return 0;
1710}
1711
1712
1713/* Plugin object */
1714static struct sched_plugin pfp_plugin __cacheline_aligned_in_smp = {
1715 .plugin_name = "P-FP",
1716 .tick = pfp_tick,
1717 .task_new = pfp_task_new,
1718 .complete_job = complete_job,
1719 .task_exit = pfp_task_exit,
1720 .schedule = pfp_schedule,
1721 .task_wake_up = pfp_task_wake_up,
1722 .task_block = pfp_task_block,
1723 .admit_task = pfp_admit_task,
1724 .activate_plugin = pfp_activate_plugin,
1725#ifdef CONFIG_LITMUS_LOCKING
1726 .allocate_lock = pfp_allocate_lock,
1727 .finish_switch = pfp_finish_switch,
1728#endif
1729};
1730
1731
1732static int __init init_pfp(void)
1733{
1734 int i;
1735
1736 /* We do not really want to support cpu hotplug, do we? ;)
1737 * However, if we are so crazy to do so,
1738 * we cannot use num_online_cpu()
1739 */
1740 for (i = 0; i < num_online_cpus(); i++) {
1741 pfp_domain_init(remote_pfp(i), i);
1742 }
1743 return register_sched_plugin(&pfp_plugin);
1744}
1745
1746module_init(init_pfp);
generated by cgit v1.2.2 (git 2.25.0) at 2024-11-21 08:54:00 -0500