aboutsummaryrefslogtreecommitdiffstats
path: root/litmus/sched_cedf.c
diff options
context:
space:
mode:
Diffstat (limited to 'litmus/sched_cedf.c')
-rw-r--r--litmus/sched_cedf.c1109
1 files changed, 1055 insertions, 54 deletions
diff --git a/litmus/sched_cedf.c b/litmus/sched_cedf.c
index b0c16e34d2c5..d98de4579394 100644
--- a/litmus/sched_cedf.c
+++ b/litmus/sched_cedf.c
@@ -29,7 +29,7 @@
29#include <linux/percpu.h> 29#include <linux/percpu.h>
30#include <linux/sched.h> 30#include <linux/sched.h>
31#include <linux/slab.h> 31#include <linux/slab.h>
32 32#include <linux/uaccess.h>
33#include <linux/module.h> 33#include <linux/module.h>
34 34
35#include <litmus/litmus.h> 35#include <litmus/litmus.h>
@@ -43,6 +43,16 @@
43#include <litmus/clustered.h> 43#include <litmus/clustered.h>
44 44
45#include <litmus/bheap.h> 45#include <litmus/bheap.h>
46#include <litmus/binheap.h>
47
48#ifdef CONFIG_LITMUS_LOCKING
49#include <litmus/kfmlp_lock.h>
50#endif
51
52#ifdef CONFIG_LITMUS_NESTED_LOCKING
53#include <litmus/rsm_lock.h>
54#include <litmus/ikglp_lock.h>
55#endif
46 56
47#ifdef CONFIG_SCHED_CPU_AFFINITY 57#ifdef CONFIG_SCHED_CPU_AFFINITY
48#include <litmus/affinity.h> 58#include <litmus/affinity.h>
@@ -50,7 +60,27 @@
50 60
51/* to configure the cluster size */ 61/* to configure the cluster size */
52#include <litmus/litmus_proc.h> 62#include <litmus/litmus_proc.h>
53#include <linux/uaccess.h> 63
64#ifdef CONFIG_SCHED_CPU_AFFINITY
65#include <litmus/affinity.h>
66#endif
67
68#ifdef CONFIG_LITMUS_SOFTIRQD
69#include <litmus/litmus_softirq.h>
70#endif
71
72#ifdef CONFIG_LITMUS_PAI_SOFTIRQD
73#include <linux/interrupt.h>
74#include <litmus/trace.h>
75#endif
76
77#ifdef CONFIG_LITMUS_NVIDIA
78#include <litmus/nvidia_info.h>
79#endif
80
81#if defined(CONFIG_LITMUS_AFFINITY_LOCKING) && defined(CONFIG_LITMUS_NVIDIA)
82#include <litmus/gpu_affinity.h>
83#endif
54 84
55/* Reference configuration variable. Determines which cache level is used to 85/* Reference configuration variable. Determines which cache level is used to
56 * group CPUs into clusters. GLOBAL_CLUSTER, which is the default, means that 86 * group CPUs into clusters. GLOBAL_CLUSTER, which is the default, means that
@@ -71,7 +101,7 @@ typedef struct {
71 struct task_struct* linked; /* only RT tasks */ 101 struct task_struct* linked; /* only RT tasks */
72 struct task_struct* scheduled; /* only RT tasks */ 102 struct task_struct* scheduled; /* only RT tasks */
73 atomic_t will_schedule; /* prevent unneeded IPIs */ 103 atomic_t will_schedule; /* prevent unneeded IPIs */
74 struct bheap_node* hn; 104 struct binheap_node hn;
75} cpu_entry_t; 105} cpu_entry_t;
76 106
77/* one cpu_entry_t per CPU */ 107/* one cpu_entry_t per CPU */
@@ -84,6 +114,14 @@ DEFINE_PER_CPU(cpu_entry_t, cedf_cpu_entries);
84#define test_will_schedule(cpu) \ 114#define test_will_schedule(cpu) \
85 (atomic_read(&per_cpu(cedf_cpu_entries, cpu).will_schedule)) 115 (atomic_read(&per_cpu(cedf_cpu_entries, cpu).will_schedule))
86 116
117#ifdef CONFIG_LITMUS_PAI_SOFTIRQD
118struct tasklet_head
119{
120 struct tasklet_struct *head;
121 struct tasklet_struct **tail;
122};
123#endif
124
87/* 125/*
88 * In C-EDF there is a cedf domain _per_ cluster 126 * In C-EDF there is a cedf domain _per_ cluster
89 * The number of clusters is dynamically determined accordingly to the 127 * The number of clusters is dynamically determined accordingly to the
@@ -97,10 +135,17 @@ typedef struct clusterdomain {
97 /* map of this cluster cpus */ 135 /* map of this cluster cpus */
98 cpumask_var_t cpu_map; 136 cpumask_var_t cpu_map;
99 /* the cpus queue themselves according to priority in here */ 137 /* the cpus queue themselves according to priority in here */
100 struct bheap_node *heap_node; 138 struct binheap cpu_heap;
101 struct bheap cpu_heap;
102 /* lock for this cluster */ 139 /* lock for this cluster */
103#define cluster_lock domain.ready_lock 140#define cluster_lock domain.ready_lock
141
142#ifdef CONFIG_LITMUS_PAI_SOFTIRQD
143 struct tasklet_head pending_tasklets;
144#endif
145
146#ifdef CONFIG_LITMUS_DGL_SUPPORT
147 raw_spinlock_t dgl_lock;
148#endif
104} cedf_domain_t; 149} cedf_domain_t;
105 150
106/* a cedf_domain per cluster; allocation is done at init/activation time */ 151/* a cedf_domain per cluster; allocation is done at init/activation time */
@@ -109,6 +154,22 @@ cedf_domain_t *cedf;
109#define remote_cluster(cpu) ((cedf_domain_t *) per_cpu(cedf_cpu_entries, cpu).cluster) 154#define remote_cluster(cpu) ((cedf_domain_t *) per_cpu(cedf_cpu_entries, cpu).cluster)
110#define task_cpu_cluster(task) remote_cluster(get_partition(task)) 155#define task_cpu_cluster(task) remote_cluster(get_partition(task))
111 156
157/* total number of cluster */
158static int num_clusters;
159/* we do not support cluster of different sizes */
160static unsigned int cluster_size;
161
162static int clusters_allocated = 0;
163
164#ifdef CONFIG_LITMUS_DGL_SUPPORT
165static raw_spinlock_t* cedf_get_dgl_spinlock(struct task_struct *t)
166{
167 cedf_domain_t *cluster = task_cpu_cluster(t);
168 return(&cluster->dgl_lock);
169}
170#endif
171
172
112/* Uncomment WANT_ALL_SCHED_EVENTS if you want to see all scheduling 173/* Uncomment WANT_ALL_SCHED_EVENTS if you want to see all scheduling
113 * decisions in the TRACE() log; uncomment VERBOSE_INIT for verbose 174 * decisions in the TRACE() log; uncomment VERBOSE_INIT for verbose
114 * information during the initialization of the plugin (e.g., topology) 175 * information during the initialization of the plugin (e.g., topology)
@@ -116,11 +177,11 @@ cedf_domain_t *cedf;
116 */ 177 */
117#define VERBOSE_INIT 178#define VERBOSE_INIT
118 179
119static int cpu_lower_prio(struct bheap_node *_a, struct bheap_node *_b) 180static int cpu_lower_prio(struct binheap_node *_a, struct binheap_node *_b)
120{ 181{
121 cpu_entry_t *a, *b; 182 cpu_entry_t *a = binheap_entry(_a, cpu_entry_t, hn);
122 a = _a->value; 183 cpu_entry_t *b = binheap_entry(_b, cpu_entry_t, hn);
123 b = _b->value; 184
124 /* Note that a and b are inverted: we want the lowest-priority CPU at 185 /* Note that a and b are inverted: we want the lowest-priority CPU at
125 * the top of the heap. 186 * the top of the heap.
126 */ 187 */
@@ -134,20 +195,17 @@ static void update_cpu_position(cpu_entry_t *entry)
134{ 195{
135 cedf_domain_t *cluster = entry->cluster; 196 cedf_domain_t *cluster = entry->cluster;
136 197
137 if (likely(bheap_node_in_heap(entry->hn))) 198 if (likely(binheap_is_in_heap(&entry->hn))) {
138 bheap_delete(cpu_lower_prio, 199 binheap_delete(&entry->hn, &cluster->cpu_heap);
139 &cluster->cpu_heap, 200 }
140 entry->hn);
141 201
142 bheap_insert(cpu_lower_prio, &cluster->cpu_heap, entry->hn); 202 binheap_add(&entry->hn, &cluster->cpu_heap, cpu_entry_t, hn);
143} 203}
144 204
145/* caller must hold cedf lock */ 205/* caller must hold cedf lock */
146static cpu_entry_t* lowest_prio_cpu(cedf_domain_t *cluster) 206static cpu_entry_t* lowest_prio_cpu(cedf_domain_t *cluster)
147{ 207{
148 struct bheap_node* hn; 208 return binheap_top_entry(&cluster->cpu_heap, cpu_entry_t, hn);
149 hn = bheap_peek(cpu_lower_prio, &cluster->cpu_heap);
150 return hn->value;
151} 209}
152 210
153 211
@@ -209,7 +267,7 @@ static noinline void link_task_to_cpu(struct task_struct* linked,
209} 267}
210 268
211/* unlink - Make sure a task is not linked any longer to an entry 269/* unlink - Make sure a task is not linked any longer to an entry
212 * where it was linked before. Must hold cedf_lock. 270 * where it was linked before. Must hold cluster_lock.
213 */ 271 */
214static noinline void unlink(struct task_struct* t) 272static noinline void unlink(struct task_struct* t)
215{ 273{
@@ -245,7 +303,7 @@ static void preempt(cpu_entry_t *entry)
245} 303}
246 304
247/* requeue - Put an unlinked task into gsn-edf domain. 305/* requeue - Put an unlinked task into gsn-edf domain.
248 * Caller must hold cedf_lock. 306 * Caller must hold cluster_lock.
249 */ 307 */
250static noinline void requeue(struct task_struct* task) 308static noinline void requeue(struct task_struct* task)
251{ 309{
@@ -340,13 +398,17 @@ static void cedf_release_jobs(rt_domain_t* rt, struct bheap* tasks)
340 raw_spin_unlock_irqrestore(&cluster->cluster_lock, flags); 398 raw_spin_unlock_irqrestore(&cluster->cluster_lock, flags);
341} 399}
342 400
343/* caller holds cedf_lock */ 401/* caller holds cluster_lock */
344static noinline void job_completion(struct task_struct *t, int forced) 402static noinline void job_completion(struct task_struct *t, int forced)
345{ 403{
346 BUG_ON(!t); 404 BUG_ON(!t);
347 405
348 sched_trace_task_completion(t, forced); 406 sched_trace_task_completion(t, forced);
349 407
408#ifdef CONFIG_LITMUS_NVIDIA
409 atomic_set(&tsk_rt(t)->nv_int_count, 0);
410#endif
411
350 TRACE_TASK(t, "job_completion().\n"); 412 TRACE_TASK(t, "job_completion().\n");
351 413
352 /* set flags */ 414 /* set flags */
@@ -371,25 +433,341 @@ static noinline void job_completion(struct task_struct *t, int forced)
371 */ 433 */
372static void cedf_tick(struct task_struct* t) 434static void cedf_tick(struct task_struct* t)
373{ 435{
374 if (is_realtime(t) && budget_enforced(t) && budget_exhausted(t)) { 436 if (is_realtime(t) && budget_exhausted(t))
375 if (!is_np(t)) { 437 {
376 /* np tasks will be preempted when they become 438 if (budget_signalled(t) && !sigbudget_sent(t)) {
377 * preemptable again 439 /* signal exhaustion */
378 */ 440 send_sigbudget(t);
379 litmus_reschedule_local(); 441 }
380 set_will_schedule(); 442
381 TRACE("cedf_scheduler_tick: " 443 if (budget_enforced(t)) {
382 "%d is preemptable " 444 if (!is_np(t)) {
383 " => FORCE_RESCHED\n", t->pid); 445 /* np tasks will be preempted when they become
384 } else if (is_user_np(t)) { 446 * preemptable again
385 TRACE("cedf_scheduler_tick: " 447 */
386 "%d is non-preemptable, " 448 litmus_reschedule_local();
387 "preemption delayed.\n", t->pid); 449 set_will_schedule();
388 request_exit_np(t); 450 TRACE("cedf_scheduler_tick: "
451 "%d is preemptable "
452 " => FORCE_RESCHED\n", t->pid);
453 } else if (is_user_np(t)) {
454 TRACE("cedf_scheduler_tick: "
455 "%d is non-preemptable, "
456 "preemption delayed.\n", t->pid);
457 request_exit_np(t);
458 }
459 }
460 }
461}
462
463
464
465
466
467
468
469
470
471
472
473
474
475#ifdef CONFIG_LITMUS_PAI_SOFTIRQD
476
477
478static void __do_lit_tasklet(struct tasklet_struct* tasklet, unsigned long flushed)
479{
480 if (!atomic_read(&tasklet->count)) {
481 if(tasklet->owner) {
482 sched_trace_tasklet_begin(tasklet->owner);
483 }
484
485 if (!test_and_clear_bit(TASKLET_STATE_SCHED, &tasklet->state))
486 {
487 BUG();
488 }
489 TRACE("%s: Invoking tasklet with owner pid = %d (flushed = %d).\n",
490 __FUNCTION__,
491 (tasklet->owner) ? tasklet->owner->pid : -1,
492 (tasklet->owner) ? 0 : 1);
493 tasklet->func(tasklet->data);
494 tasklet_unlock(tasklet);
495
496 if(tasklet->owner) {
497 sched_trace_tasklet_end(tasklet->owner, flushed);
498 }
499 }
500 else {
501 BUG();
502 }
503}
504
505
506static void do_lit_tasklets(cedf_domain_t* cluster, struct task_struct* sched_task)
507{
508 int work_to_do = 1;
509 struct tasklet_struct *tasklet = NULL;
510 unsigned long flags;
511
512 while(work_to_do) {
513
514 TS_NV_SCHED_BOTISR_START;
515
516 raw_spin_lock_irqsave(&cluster->cluster_lock, flags);
517
518 if(cluster->pending_tasklets.head != NULL) {
519 // remove tasklet at head.
520 struct tasklet_struct *prev = NULL;
521 tasklet = cluster->pending_tasklets.head;
522
523 // find a tasklet with prio to execute; skip ones where
524 // sched_task has a higher priority.
525 // We use the '!edf' test instead of swaping function arguments since
526 // both sched_task and owner could be NULL. In this case, we want to
527 // still execute the tasklet.
528 while(tasklet && !edf_higher_prio(tasklet->owner, sched_task)) {
529 prev = tasklet;
530 tasklet = tasklet->next;
531 }
532
533 if(tasklet) { // found something to execuite
534 // remove the tasklet from the queue
535 if(prev) {
536 prev->next = tasklet->next;
537 if(prev->next == NULL) {
538 TRACE("%s: Tasklet for %d is the last element in tasklet queue.\n", __FUNCTION__, tasklet->owner->pid);
539 cluster->pending_tasklets.tail = &(prev);
540 }
541 }
542 else {
543 cluster->pending_tasklets.head = tasklet->next;
544 if(tasklet->next == NULL) {
545 TRACE("%s: Tasklet for %d is the last element in tasklet queue.\n", __FUNCTION__, tasklet->owner->pid);
546 cluster->pending_tasklets.tail = &(cluster->pending_tasklets.head);
547 }
548 }
549 }
550 else {
551 TRACE("%s: No tasklets with eligible priority.\n", __FUNCTION__);
552 }
553 }
554 else {
555 TRACE("%s: Tasklet queue is empty.\n", __FUNCTION__);
556 }
557
558 raw_spin_unlock_irqrestore(&cluster->cluster_lock, flags);
559
560 if(tasklet) {
561 __do_lit_tasklet(tasklet, 0ul);
562 tasklet = NULL;
563 }
564 else {
565 work_to_do = 0;
566 }
567
568 TS_NV_SCHED_BOTISR_END;
569 }
570}
571
572static void __add_pai_tasklet(struct tasklet_struct* tasklet, cedf_domain_t* cluster)
573{
574 struct tasklet_struct* step;
575
576 tasklet->next = NULL; // make sure there are no old values floating around
577
578 step = cluster->pending_tasklets.head;
579 if(step == NULL) {
580 TRACE("%s: tasklet queue empty. inserting tasklet for %d at head.\n", __FUNCTION__, tasklet->owner->pid);
581 // insert at tail.
582 *(cluster->pending_tasklets.tail) = tasklet;
583 cluster->pending_tasklets.tail = &(tasklet->next);
584 }
585 else if((*(cluster->pending_tasklets.tail) != NULL) &&
586 edf_higher_prio((*(cluster->pending_tasklets.tail))->owner, tasklet->owner)) {
587 // insert at tail.
588 TRACE("%s: tasklet belongs at end. inserting tasklet for %d at tail.\n", __FUNCTION__, tasklet->owner->pid);
589
590 *(cluster->pending_tasklets.tail) = tasklet;
591 cluster->pending_tasklets.tail = &(tasklet->next);
592 }
593 else {
594
595 // insert the tasklet somewhere in the middle.
596
597 TRACE("%s: tasklet belongs somewhere in the middle.\n", __FUNCTION__);
598
599 while(step->next && edf_higher_prio(step->next->owner, tasklet->owner)) {
600 step = step->next;
601 }
602
603 // insert tasklet right before step->next.
604
605 TRACE("%s: inserting tasklet for %d between %d and %d.\n", __FUNCTION__,
606 tasklet->owner->pid,
607 (step->owner) ?
608 step->owner->pid :
609 -1,
610 (step->next) ?
611 ((step->next->owner) ?
612 step->next->owner->pid :
613 -1) :
614 -1);
615
616 tasklet->next = step->next;
617 step->next = tasklet;
618
619 // patch up the head if needed.
620 if(cluster->pending_tasklets.head == step)
621 {
622 TRACE("%s: %d is the new tasklet queue head.\n", __FUNCTION__, tasklet->owner->pid);
623 cluster->pending_tasklets.head = tasklet;
389 } 624 }
390 } 625 }
391} 626}
392 627
628static void cedf_run_tasklets(struct task_struct* sched_task)
629{
630 cedf_domain_t* cluster;
631
632 preempt_disable();
633
634 cluster = (is_realtime(sched_task)) ?
635 task_cpu_cluster(sched_task) :
636 remote_cluster(smp_processor_id());
637
638 if(cluster && cluster->pending_tasklets.head != NULL) {
639 TRACE("%s: There are tasklets to process.\n", __FUNCTION__);
640 do_lit_tasklets(cluster, sched_task);
641 }
642
643 preempt_enable_no_resched();
644}
645
646
647
648static int cedf_enqueue_pai_tasklet(struct tasklet_struct* tasklet)
649{
650#if 0
651 cedf_domain_t *cluster = NULL;
652 cpu_entry_t *targetCPU = NULL;
653 int thisCPU;
654 int runLocal = 0;
655 int runNow = 0;
656 unsigned long flags;
657
658 if(unlikely((tasklet->owner == NULL) || !is_realtime(tasklet->owner)))
659 {
660 TRACE("%s: No owner associated with this tasklet!\n", __FUNCTION__);
661 return 0;
662 }
663
664 cluster = task_cpu_cluster(tasklet->owner);
665
666 raw_spin_lock_irqsave(&cluster->cluster_lock, flags);
667
668 thisCPU = smp_processor_id();
669
670#ifdef CONFIG_SCHED_CPU_AFFINITY
671 {
672 cpu_entry_t* affinity = NULL;
673
674 // use this CPU if it is in our cluster and isn't running any RT work.
675 if(cpu_isset(thisCPU, *cluster->cpu_map) && (__get_cpu_var(cedf_cpu_entries).linked == NULL)) {
676 affinity = &(__get_cpu_var(cedf_cpu_entries));
677 }
678 else {
679 // this CPU is busy or shouldn't run tasklet in this cluster.
680 // look for available near by CPUs.
681 // NOTE: Affinity towards owner and not this CPU. Is this right?
682 affinity =
683 cedf_get_nearest_available_cpu(cluster,
684 &per_cpu(cedf_cpu_entries, task_cpu(tasklet->owner)));
685 }
686
687 targetCPU = affinity;
688 }
689#endif
690
691 if (targetCPU == NULL) {
692 targetCPU = lowest_prio_cpu(cluster);
693 }
694
695 if (edf_higher_prio(tasklet->owner, targetCPU->linked)) {
696 if (thisCPU == targetCPU->cpu) {
697 TRACE("%s: Run tasklet locally (and now).\n", __FUNCTION__);
698 runLocal = 1;
699 runNow = 1;
700 }
701 else {
702 TRACE("%s: Run tasklet remotely (and now).\n", __FUNCTION__);
703 runLocal = 0;
704 runNow = 1;
705 }
706 }
707 else {
708 runLocal = 0;
709 runNow = 0;
710 }
711
712 if(!runLocal) {
713 // enqueue the tasklet
714 __add_pai_tasklet(tasklet, cluster);
715 }
716
717 raw_spin_unlock_irqrestore(&cluster->cluster_lock, flags);
718
719
720 if (runLocal /*&& runNow */) { // runNow == 1 is implied
721 TRACE("%s: Running tasklet on CPU where it was received.\n", __FUNCTION__);
722 __do_lit_tasklet(tasklet, 0ul);
723 }
724 else if (runNow /*&& !runLocal */) { // runLocal == 0 is implied
725 TRACE("%s: Triggering CPU %d to run tasklet.\n", __FUNCTION__, targetCPU->cpu);
726 preempt(targetCPU); // need to be protected by cluster_lock?
727 }
728 else {
729 TRACE("%s: Scheduling of tasklet was deferred.\n", __FUNCTION__);
730 }
731#else
732 TRACE("%s: Running tasklet on CPU where it was received.\n", __FUNCTION__);
733 __do_lit_tasklet(tasklet, 0ul);
734#endif
735 return(1); // success
736}
737
738static void cedf_change_prio_pai_tasklet(struct task_struct *old_prio,
739 struct task_struct *new_prio)
740{
741 struct tasklet_struct* step;
742 unsigned long flags;
743 cedf_domain_t *cluster;
744 struct task_struct *probe;
745
746 // identify the cluster by the assignment of these tasks. one should
747 // be non-NULL.
748 probe = (old_prio) ? old_prio : new_prio;
749
750 if(probe) {
751 cluster = task_cpu_cluster(probe);
752
753 if(cluster->pending_tasklets.head != NULL) {
754 raw_spin_lock_irqsave(&cluster->cluster_lock, flags);
755 for(step = cluster->pending_tasklets.head; step != NULL; step = step->next) {
756 if(step->owner == old_prio) {
757 TRACE("%s: Found tasklet to change: %d\n", __FUNCTION__, step->owner->pid);
758 step->owner = new_prio;
759 }
760 }
761 raw_spin_unlock_irqrestore(&cluster->cluster_lock, flags);
762 }
763 }
764 else {
765 TRACE("%s: Both priorities were NULL\n");
766 }
767}
768
769#endif // PAI
770
393/* Getting schedule() right is a bit tricky. schedule() may not make any 771/* Getting schedule() right is a bit tricky. schedule() may not make any
394 * assumptions on the state of the current task since it may be called for a 772 * assumptions on the state of the current task since it may be called for a
395 * number of reasons. The reasons include a scheduler_tick() determined that it 773 * number of reasons. The reasons include a scheduler_tick() determined that it
@@ -415,7 +793,7 @@ static struct task_struct* cedf_schedule(struct task_struct * prev)
415{ 793{
416 cpu_entry_t* entry = &__get_cpu_var(cedf_cpu_entries); 794 cpu_entry_t* entry = &__get_cpu_var(cedf_cpu_entries);
417 cedf_domain_t *cluster = entry->cluster; 795 cedf_domain_t *cluster = entry->cluster;
418 int out_of_time, sleep, preempt, np, exists, blocks; 796 int out_of_time, signal_budget, sleep, preempt, np, exists, blocks;
419 struct task_struct* next = NULL; 797 struct task_struct* next = NULL;
420 798
421#ifdef CONFIG_RELEASE_MASTER 799#ifdef CONFIG_RELEASE_MASTER
@@ -442,6 +820,10 @@ static struct task_struct* cedf_schedule(struct task_struct * prev)
442 out_of_time = exists && 820 out_of_time = exists &&
443 budget_enforced(entry->scheduled) && 821 budget_enforced(entry->scheduled) &&
444 budget_exhausted(entry->scheduled); 822 budget_exhausted(entry->scheduled);
823 signal_budget = exists &&
824 budget_signalled(entry->scheduled) &&
825 budget_exhausted(entry->scheduled) &&
826 !sigbudget_sent(entry->scheduled);
445 np = exists && is_np(entry->scheduled); 827 np = exists && is_np(entry->scheduled);
446 sleep = exists && get_rt_flags(entry->scheduled) == RT_F_SLEEP; 828 sleep = exists && get_rt_flags(entry->scheduled) == RT_F_SLEEP;
447 preempt = entry->scheduled != entry->linked; 829 preempt = entry->scheduled != entry->linked;
@@ -460,12 +842,28 @@ static struct task_struct* cedf_schedule(struct task_struct * prev)
460 TRACE_TASK(prev, "will be preempted by %s/%d\n", 842 TRACE_TASK(prev, "will be preempted by %s/%d\n",
461 entry->linked->comm, entry->linked->pid); 843 entry->linked->comm, entry->linked->pid);
462 844
845 /* Send the signal that the budget has been exhausted */
846 if (signal_budget)
847 send_sigbudget(entry->scheduled);
463 848
464 /* If a task blocks we have no choice but to reschedule. 849 /* If a task blocks we have no choice but to reschedule.
465 */ 850 */
466 if (blocks) 851 if (blocks)
467 unlink(entry->scheduled); 852 unlink(entry->scheduled);
468 853
854#if defined(CONFIG_LITMUS_NVIDIA) && defined(CONFIG_LITMUS_AFFINITY_LOCKING)
855 if(exists && is_realtime(entry->scheduled) && tsk_rt(entry->scheduled)->held_gpus) {
856 if(!blocks || tsk_rt(entry->scheduled)->suspend_gpu_tracker_on_block) {
857 // don't track preemptions or locking protocol suspensions.
858 TRACE_TASK(entry->scheduled, "stopping GPU tracker.\n");
859 stop_gpu_tracker(entry->scheduled);
860 }
861 else if(blocks && !tsk_rt(entry->scheduled)->suspend_gpu_tracker_on_block) {
862 TRACE_TASK(entry->scheduled, "GPU tracker remains on during suspension.\n");
863 }
864 }
865#endif
866
469 /* Request a sys_exit_np() call if we would like to preempt but cannot. 867 /* Request a sys_exit_np() call if we would like to preempt but cannot.
470 * We need to make sure to update the link structure anyway in case 868 * We need to make sure to update the link structure anyway in case
471 * that we are still linked. Multiple calls to request_exit_np() don't 869 * that we are still linked. Multiple calls to request_exit_np() don't
@@ -515,7 +913,7 @@ static struct task_struct* cedf_schedule(struct task_struct * prev)
515 raw_spin_unlock(&cluster->cluster_lock); 913 raw_spin_unlock(&cluster->cluster_lock);
516 914
517#ifdef WANT_ALL_SCHED_EVENTS 915#ifdef WANT_ALL_SCHED_EVENTS
518 TRACE("cedf_lock released, next=0x%p\n", next); 916 TRACE("cluster_lock released, next=0x%p\n", next);
519 917
520 if (next) 918 if (next)
521 TRACE_TASK(next, "scheduled at %llu\n", litmus_clock()); 919 TRACE_TASK(next, "scheduled at %llu\n", litmus_clock());
@@ -523,7 +921,6 @@ static struct task_struct* cedf_schedule(struct task_struct * prev)
523 TRACE("becomes idle at %llu.\n", litmus_clock()); 921 TRACE("becomes idle at %llu.\n", litmus_clock());
524#endif 922#endif
525 923
526
527 return next; 924 return next;
528} 925}
529 926
@@ -549,7 +946,7 @@ static void cedf_task_new(struct task_struct * t, int on_rq, int running)
549 cpu_entry_t* entry; 946 cpu_entry_t* entry;
550 cedf_domain_t* cluster; 947 cedf_domain_t* cluster;
551 948
552 TRACE("gsn edf: task new %d\n", t->pid); 949 TRACE("c-edf: task new %d\n", t->pid);
553 950
554 /* the cluster doesn't change even if t is running */ 951 /* the cluster doesn't change even if t is running */
555 cluster = task_cpu_cluster(t); 952 cluster = task_cpu_cluster(t);
@@ -587,7 +984,7 @@ static void cedf_task_new(struct task_struct * t, int on_rq, int running)
587static void cedf_task_wake_up(struct task_struct *task) 984static void cedf_task_wake_up(struct task_struct *task)
588{ 985{
589 unsigned long flags; 986 unsigned long flags;
590 lt_t now; 987 //lt_t now;
591 cedf_domain_t *cluster; 988 cedf_domain_t *cluster;
592 989
593 TRACE_TASK(task, "wake_up at %llu\n", litmus_clock()); 990 TRACE_TASK(task, "wake_up at %llu\n", litmus_clock());
@@ -595,6 +992,8 @@ static void cedf_task_wake_up(struct task_struct *task)
595 cluster = task_cpu_cluster(task); 992 cluster = task_cpu_cluster(task);
596 993
597 raw_spin_lock_irqsave(&cluster->cluster_lock, flags); 994 raw_spin_lock_irqsave(&cluster->cluster_lock, flags);
995
996#if 0 // sproadic task model
598 /* We need to take suspensions because of semaphores into 997 /* We need to take suspensions because of semaphores into
599 * account! If a job resumes after being suspended due to acquiring 998 * account! If a job resumes after being suspended due to acquiring
600 * a semaphore, it should never be treated as a new job release. 999 * a semaphore, it should never be treated as a new job release.
@@ -616,7 +1015,13 @@ static void cedf_task_wake_up(struct task_struct *task)
616 } 1015 }
617 } 1016 }
618 } 1017 }
619 cedf_job_arrival(task); 1018#else
1019 set_rt_flags(task, RT_F_RUNNING); // periodic model
1020#endif
1021
1022 if(tsk_rt(task)->linked_on == NO_CPU)
1023 cedf_job_arrival(task);
1024
620 raw_spin_unlock_irqrestore(&cluster->cluster_lock, flags); 1025 raw_spin_unlock_irqrestore(&cluster->cluster_lock, flags);
621} 1026}
622 1027
@@ -643,6 +1048,10 @@ static void cedf_task_exit(struct task_struct * t)
643 unsigned long flags; 1048 unsigned long flags;
644 cedf_domain_t *cluster = task_cpu_cluster(t); 1049 cedf_domain_t *cluster = task_cpu_cluster(t);
645 1050
1051#ifdef CONFIG_LITMUS_PAI_SOFTIRQD
1052 cedf_change_prio_pai_tasklet(t, NULL);
1053#endif
1054
646 /* unlink if necessary */ 1055 /* unlink if necessary */
647 raw_spin_lock_irqsave(&cluster->cluster_lock, flags); 1056 raw_spin_lock_irqsave(&cluster->cluster_lock, flags);
648 unlink(t); 1057 unlink(t);
@@ -660,13 +1069,536 @@ static void cedf_task_exit(struct task_struct * t)
660 1069
661static long cedf_admit_task(struct task_struct* tsk) 1070static long cedf_admit_task(struct task_struct* tsk)
662{ 1071{
1072#ifdef CONFIG_LITMUS_NESTED_LOCKING
1073 INIT_BINHEAP_HANDLE(&tsk_rt(tsk)->hp_blocked_tasks,
1074 edf_max_heap_base_priority_order);
1075#endif
1076
663 return task_cpu(tsk) == tsk->rt_param.task_params.cpu ? 0 : -EINVAL; 1077 return task_cpu(tsk) == tsk->rt_param.task_params.cpu ? 0 : -EINVAL;
664} 1078}
665 1079
666/* total number of cluster */ 1080
667static int num_clusters; 1081
668/* we do not support cluster of different sizes */ 1082#ifdef CONFIG_LITMUS_LOCKING
669static unsigned int cluster_size; 1083
1084#include <litmus/fdso.h>
1085
1086
1087
1088/* called with IRQs off */
1089static void __increase_priority_inheritance(struct task_struct* t,
1090 struct task_struct* prio_inh)
1091{
1092 int linked_on;
1093 int check_preempt = 0;
1094
1095 cedf_domain_t* cluster = task_cpu_cluster(t);
1096
1097#ifdef CONFIG_LITMUS_NESTED_LOCKING
1098 /* this sanity check allows for weaker locking in protocols */
1099 /* TODO (klitirqd): Skip this check if 't' is a proxy thread (???) */
1100 if(__edf_higher_prio(prio_inh, BASE, t, EFFECTIVE)) {
1101#endif
1102 TRACE_TASK(t, "inherits priority from %s/%d\n",
1103 prio_inh->comm, prio_inh->pid);
1104 tsk_rt(t)->inh_task = prio_inh;
1105
1106 linked_on = tsk_rt(t)->linked_on;
1107
1108 /* If it is scheduled, then we need to reorder the CPU heap. */
1109 if (linked_on != NO_CPU) {
1110 TRACE_TASK(t, "%s: linked on %d\n",
1111 __FUNCTION__, linked_on);
1112 /* Holder is scheduled; need to re-order CPUs.
1113 * We can't use heap_decrease() here since
1114 * the cpu_heap is ordered in reverse direction, so
1115 * it is actually an increase. */
1116 binheap_delete(&per_cpu(cedf_cpu_entries, linked_on).hn,
1117 &cluster->cpu_heap);
1118 binheap_add(&per_cpu(cedf_cpu_entries, linked_on).hn,
1119 &cluster->cpu_heap, cpu_entry_t, hn);
1120
1121 } else {
1122 /* holder may be queued: first stop queue changes */
1123 raw_spin_lock(&cluster->domain.release_lock);
1124 if (is_queued(t)) {
1125 TRACE_TASK(t, "%s: is queued\n",
1126 __FUNCTION__);
1127 /* We need to update the position of holder in some
1128 * heap. Note that this could be a release heap if we
1129 * budget enforcement is used and this job overran. */
1130 check_preempt =
1131 !bheap_decrease(edf_ready_order, tsk_rt(t)->heap_node);
1132 } else {
1133 /* Nothing to do: if it is not queued and not linked
1134 * then it is either sleeping or currently being moved
1135 * by other code (e.g., a timer interrupt handler) that
1136 * will use the correct priority when enqueuing the
1137 * task. */
1138 TRACE_TASK(t, "%s: is NOT queued => Done.\n",
1139 __FUNCTION__);
1140 }
1141 raw_spin_unlock(&cluster->domain.release_lock);
1142
1143 /* If holder was enqueued in a release heap, then the following
1144 * preemption check is pointless, but we can't easily detect
1145 * that case. If you want to fix this, then consider that
1146 * simply adding a state flag requires O(n) time to update when
1147 * releasing n tasks, which conflicts with the goal to have
1148 * O(log n) merges. */
1149 if (check_preempt) {
1150 /* heap_decrease() hit the top level of the heap: make
1151 * sure preemption checks get the right task, not the
1152 * potentially stale cache. */
1153 bheap_uncache_min(edf_ready_order,
1154 &cluster->domain.ready_queue);
1155 check_for_preemptions(cluster);
1156 }
1157 }
1158#ifdef CONFIG_LITMUS_NESTED_LOCKING
1159 }
1160 else {
1161 TRACE_TASK(t, "Spurious invalid priority increase. "
1162 "Inheritance request: %s/%d [eff_prio = %s/%d] to inherit from %s/%d\n"
1163 "Occurance is likely okay: probably due to (hopefully safe) concurrent priority updates.\n",
1164 t->comm, t->pid,
1165 effective_priority(t)->comm, effective_priority(t)->pid,
1166 (prio_inh) ? prio_inh->comm : "nil",
1167 (prio_inh) ? prio_inh->pid : -1);
1168 WARN_ON(!prio_inh);
1169 }
1170#endif
1171}
1172
1173/* called with IRQs off */
1174static void increase_priority_inheritance(struct task_struct* t, struct task_struct* prio_inh)
1175{
1176 cedf_domain_t* cluster = task_cpu_cluster(t);
1177
1178 raw_spin_lock(&cluster->cluster_lock);
1179
1180 __increase_priority_inheritance(t, prio_inh);
1181
1182#ifdef CONFIG_LITMUS_SOFTIRQD
1183 if(tsk_rt(t)->cur_klitirqd != NULL)
1184 {
1185 TRACE_TASK(t, "%s/%d inherits a new priority!\n",
1186 tsk_rt(t)->cur_klitirqd->comm, tsk_rt(t)->cur_klitirqd->pid);
1187
1188 __increase_priority_inheritance(tsk_rt(t)->cur_klitirqd, prio_inh);
1189 }
1190#endif
1191
1192 raw_spin_unlock(&cluster->cluster_lock);
1193
1194#if defined(CONFIG_LITMUS_PAI_SOFTIRQD) && defined(CONFIG_LITMUS_NVIDIA)
1195 if(tsk_rt(t)->held_gpus) {
1196 int i;
1197 for(i = find_first_bit(&tsk_rt(t)->held_gpus, sizeof(tsk_rt(t)->held_gpus));
1198 i < NV_DEVICE_NUM;
1199 i = find_next_bit(&tsk_rt(t)->held_gpus, sizeof(tsk_rt(t)->held_gpus), i+1)) {
1200 pai_check_priority_increase(t, i);
1201 }
1202 }
1203#endif
1204}
1205
1206/* called with IRQs off */
1207static void __decrease_priority_inheritance(struct task_struct* t,
1208 struct task_struct* prio_inh)
1209{
1210#ifdef CONFIG_LITMUS_NESTED_LOCKING
1211 if(__edf_higher_prio(t, EFFECTIVE, prio_inh, BASE)) {
1212#endif
1213 /* A job only stops inheriting a priority when it releases a
1214 * resource. Thus we can make the following assumption.*/
1215 if(prio_inh)
1216 TRACE_TASK(t, "EFFECTIVE priority decreased to %s/%d\n",
1217 prio_inh->comm, prio_inh->pid);
1218 else
1219 TRACE_TASK(t, "base priority restored.\n");
1220
1221 tsk_rt(t)->inh_task = prio_inh;
1222
1223 if(tsk_rt(t)->scheduled_on != NO_CPU) {
1224 TRACE_TASK(t, "is scheduled.\n");
1225
1226 /* Check if rescheduling is necessary. We can't use heap_decrease()
1227 * since the priority was effectively lowered. */
1228 unlink(t);
1229 cedf_job_arrival(t);
1230 }
1231 else {
1232 cedf_domain_t* cluster = task_cpu_cluster(t);
1233 /* task is queued */
1234 raw_spin_lock(&cluster->domain.release_lock);
1235 if (is_queued(t)) {
1236 TRACE_TASK(t, "is queued.\n");
1237
1238 /* decrease in priority, so we have to re-add to binomial heap */
1239 unlink(t);
1240 cedf_job_arrival(t);
1241 }
1242 else {
1243 TRACE_TASK(t, "is not in scheduler. Probably on wait queue somewhere.\n");
1244 }
1245 raw_spin_unlock(&cluster->domain.release_lock);
1246 }
1247#ifdef CONFIG_LITMUS_NESTED_LOCKING
1248 }
1249 else {
1250 TRACE_TASK(t, "Spurious invalid priority decrease. "
1251 "Inheritance request: %s/%d [eff_prio = %s/%d] to inherit from %s/%d\n"
1252 "Occurance is likely okay: probably due to (hopefully safe) concurrent priority updates.\n",
1253 t->comm, t->pid,
1254 effective_priority(t)->comm, effective_priority(t)->pid,
1255 (prio_inh) ? prio_inh->comm : "nil",
1256 (prio_inh) ? prio_inh->pid : -1);
1257 }
1258#endif
1259}
1260
1261static void decrease_priority_inheritance(struct task_struct* t,
1262 struct task_struct* prio_inh)
1263{
1264 cedf_domain_t* cluster = task_cpu_cluster(t);
1265
1266 raw_spin_lock(&cluster->cluster_lock);
1267 __decrease_priority_inheritance(t, prio_inh);
1268
1269#ifdef CONFIG_LITMUS_SOFTIRQD
1270 if(tsk_rt(t)->cur_klitirqd != NULL)
1271 {
1272 TRACE_TASK(t, "%s/%d decreases in priority!\n",
1273 tsk_rt(t)->cur_klitirqd->comm, tsk_rt(t)->cur_klitirqd->pid);
1274
1275 __decrease_priority_inheritance(tsk_rt(t)->cur_klitirqd, prio_inh);
1276 }
1277#endif
1278
1279 raw_spin_unlock(&cluster->cluster_lock);
1280
1281#if defined(CONFIG_LITMUS_PAI_SOFTIRQD) && defined(CONFIG_LITMUS_NVIDIA)
1282 if(tsk_rt(t)->held_gpus) {
1283 int i;
1284 for(i = find_first_bit(&tsk_rt(t)->held_gpus, sizeof(tsk_rt(t)->held_gpus));
1285 i < NV_DEVICE_NUM;
1286 i = find_next_bit(&tsk_rt(t)->held_gpus, sizeof(tsk_rt(t)->held_gpus), i+1)) {
1287 pai_check_priority_decrease(t, i);
1288 }
1289 }
1290#endif
1291}
1292
1293
1294
1295
1296
1297#ifdef CONFIG_LITMUS_SOFTIRQD
1298/* called with IRQs off */
1299static void increase_priority_inheritance_klitirqd(struct task_struct* klitirqd,
1300 struct task_struct* old_owner,
1301 struct task_struct* new_owner)
1302{
1303 cedf_domain_t* cluster = task_cpu_cluster(klitirqd);
1304
1305 BUG_ON(!(tsk_rt(klitirqd)->is_proxy_thread));
1306
1307 raw_spin_lock(&cluster->cluster_lock);
1308
1309 if(old_owner != new_owner)
1310 {
1311 if(old_owner)
1312 {
1313 // unreachable?
1314 tsk_rt(old_owner)->cur_klitirqd = NULL;
1315 }
1316
1317 TRACE_TASK(klitirqd, "giving ownership to %s/%d.\n",
1318 new_owner->comm, new_owner->pid);
1319
1320 tsk_rt(new_owner)->cur_klitirqd = klitirqd;
1321 }
1322
1323 __decrease_priority_inheritance(klitirqd, NULL); // kludge to clear out cur prio.
1324
1325 __increase_priority_inheritance(klitirqd,
1326 (tsk_rt(new_owner)->inh_task == NULL) ?
1327 new_owner :
1328 tsk_rt(new_owner)->inh_task);
1329
1330 raw_spin_unlock(&cluster->cluster_lock);
1331}
1332
1333
1334/* called with IRQs off */
1335static void decrease_priority_inheritance_klitirqd(struct task_struct* klitirqd,
1336 struct task_struct* old_owner,
1337 struct task_struct* new_owner)
1338{
1339 cedf_domain_t* cluster = task_cpu_cluster(klitirqd);
1340
1341 BUG_ON(!(tsk_rt(klitirqd)->is_proxy_thread));
1342
1343 raw_spin_lock(&cluster->cluster_lock);
1344
1345 TRACE_TASK(klitirqd, "priority restored\n");
1346
1347 __decrease_priority_inheritance(klitirqd, new_owner);
1348
1349 tsk_rt(old_owner)->cur_klitirqd = NULL;
1350
1351 raw_spin_unlock(&cluster->cluster_lock);
1352}
1353#endif // CONFIG_LITMUS_SOFTIRQD
1354
1355
1356
1357
1358
1359
1360
1361#ifdef CONFIG_LITMUS_NESTED_LOCKING
1362
1363/* called with IRQs off */
1364/* preconditions:
1365 (1) The 'hp_blocked_tasks_lock' of task 't' is held.
1366 (2) The lock 'to_unlock' is held.
1367 */
1368static void nested_increase_priority_inheritance(struct task_struct* t,
1369 struct task_struct* prio_inh,
1370 raw_spinlock_t *to_unlock,
1371 unsigned long irqflags)
1372{
1373 struct litmus_lock *blocked_lock = tsk_rt(t)->blocked_lock;
1374
1375 if(tsk_rt(t)->inh_task != prio_inh) { // shield redundent calls.
1376 increase_priority_inheritance(t, prio_inh); // increase our prio.
1377 }
1378
1379 raw_spin_unlock(&tsk_rt(t)->hp_blocked_tasks_lock); // unlock the t's heap.
1380
1381
1382 if(blocked_lock) {
1383 if(blocked_lock->ops->propagate_increase_inheritance) {
1384 TRACE_TASK(t, "Inheritor is blocked (...perhaps). Checking lock %d.\n",
1385 blocked_lock->ident);
1386
1387 // beware: recursion
1388 blocked_lock->ops->propagate_increase_inheritance(blocked_lock,
1389 t, to_unlock,
1390 irqflags);
1391 }
1392 else {
1393 TRACE_TASK(t, "Inheritor is blocked on lock (%d) that does not support nesting!\n",
1394 blocked_lock->ident);
1395 unlock_fine_irqrestore(to_unlock, irqflags);
1396 }
1397 }
1398 else {
1399 TRACE_TASK(t, "is not blocked. No propagation.\n");
1400 unlock_fine_irqrestore(to_unlock, irqflags);
1401 }
1402}
1403
1404/* called with IRQs off */
1405/* preconditions:
1406 (1) The 'hp_blocked_tasks_lock' of task 't' is held.
1407 (2) The lock 'to_unlock' is held.
1408 */
1409static void nested_decrease_priority_inheritance(struct task_struct* t,
1410 struct task_struct* prio_inh,
1411 raw_spinlock_t *to_unlock,
1412 unsigned long irqflags)
1413{
1414 struct litmus_lock *blocked_lock = tsk_rt(t)->blocked_lock;
1415 decrease_priority_inheritance(t, prio_inh);
1416
1417 raw_spin_unlock(&tsk_rt(t)->hp_blocked_tasks_lock); // unlock the t's heap.
1418
1419 if(blocked_lock) {
1420 if(blocked_lock->ops->propagate_decrease_inheritance) {
1421 TRACE_TASK(t, "Inheritor is blocked (...perhaps). Checking lock %d.\n",
1422 blocked_lock->ident);
1423
1424 // beware: recursion
1425 blocked_lock->ops->propagate_decrease_inheritance(blocked_lock, t,
1426 to_unlock,
1427 irqflags);
1428 }
1429 else {
1430 TRACE_TASK(t, "Inheritor is blocked on lock (%p) that does not support nesting!\n",
1431 blocked_lock);
1432 unlock_fine_irqrestore(to_unlock, irqflags);
1433 }
1434 }
1435 else {
1436 TRACE_TASK(t, "is not blocked. No propagation.\n");
1437 unlock_fine_irqrestore(to_unlock, irqflags);
1438 }
1439}
1440
1441
1442/* ******************** RSM MUTEX ********************** */
1443
1444static struct litmus_lock_ops cedf_rsm_mutex_lock_ops = {
1445 .lock = rsm_mutex_lock,
1446 .unlock = rsm_mutex_unlock,
1447 .close = rsm_mutex_close,
1448 .deallocate = rsm_mutex_free,
1449
1450 .propagate_increase_inheritance = rsm_mutex_propagate_increase_inheritance,
1451 .propagate_decrease_inheritance = rsm_mutex_propagate_decrease_inheritance,
1452
1453#ifdef CONFIG_LITMUS_DGL_SUPPORT
1454 .dgl_lock = rsm_mutex_dgl_lock,
1455 .is_owner = rsm_mutex_is_owner,
1456 .enable_priority = rsm_mutex_enable_priority,
1457#endif
1458};
1459
1460static struct litmus_lock* cedf_new_rsm_mutex(void)
1461{
1462 return rsm_mutex_new(&cedf_rsm_mutex_lock_ops);
1463}
1464
1465/* ******************** IKGLP ********************** */
1466
1467static struct litmus_lock_ops cedf_ikglp_lock_ops = {
1468 .lock = ikglp_lock,
1469 .unlock = ikglp_unlock,
1470 .close = ikglp_close,
1471 .deallocate = ikglp_free,
1472
1473 // ikglp can only be an outer-most lock.
1474 .propagate_increase_inheritance = NULL,
1475 .propagate_decrease_inheritance = NULL,
1476};
1477
1478static struct litmus_lock* cedf_new_ikglp(void* __user arg)
1479{
1480 // assumes clusters of uniform size.
1481 return ikglp_new(cluster_size/num_clusters, &cedf_ikglp_lock_ops, arg);
1482}
1483
1484#endif /* CONFIG_LITMUS_NESTED_LOCKING */
1485
1486
1487
1488
1489/* ******************** KFMLP support ********************** */
1490
1491static struct litmus_lock_ops cedf_kfmlp_lock_ops = {
1492 .lock = kfmlp_lock,
1493 .unlock = kfmlp_unlock,
1494 .close = kfmlp_close,
1495 .deallocate = kfmlp_free,
1496
1497 // kfmlp can only be an outer-most lock.
1498 .propagate_increase_inheritance = NULL,
1499 .propagate_decrease_inheritance = NULL,
1500};
1501
1502
1503static struct litmus_lock* cedf_new_kfmlp(void* __user arg)
1504{
1505 return kfmlp_new(&cedf_kfmlp_lock_ops, arg);
1506}
1507
1508
1509/* **** lock constructor **** */
1510
1511static long cedf_allocate_lock(struct litmus_lock **lock, int type,
1512 void* __user args)
1513{
1514 int err;
1515
1516 switch (type) {
1517#ifdef CONFIG_LITMUS_NESTED_LOCKING
1518 case RSM_MUTEX:
1519 *lock = cedf_new_rsm_mutex();
1520 break;
1521
1522 case IKGLP_SEM:
1523 *lock = cedf_new_ikglp(args);
1524 break;
1525#endif
1526 case KFMLP_SEM:
1527 *lock = cedf_new_kfmlp(args);
1528 break;
1529
1530 default:
1531 err = -ENXIO;
1532 goto UNSUPPORTED_LOCK;
1533 };
1534
1535 if (*lock)
1536 err = 0;
1537 else
1538 err = -ENOMEM;
1539
1540UNSUPPORTED_LOCK:
1541 return err;
1542}
1543
1544#endif // CONFIG_LITMUS_LOCKING
1545
1546
1547#ifdef CONFIG_LITMUS_AFFINITY_LOCKING
1548static struct affinity_observer_ops cedf_kfmlp_affinity_ops = {
1549 .close = kfmlp_aff_obs_close,
1550 .deallocate = kfmlp_aff_obs_free,
1551};
1552
1553#ifdef CONFIG_LITMUS_NESTED_LOCKING
1554static struct affinity_observer_ops cedf_ikglp_affinity_ops = {
1555 .close = ikglp_aff_obs_close,
1556 .deallocate = ikglp_aff_obs_free,
1557};
1558#endif
1559
1560static long cedf_allocate_affinity_observer(struct affinity_observer **aff_obs,
1561 int type,
1562 void* __user args)
1563{
1564 int err;
1565
1566 switch (type) {
1567
1568 case KFMLP_SIMPLE_GPU_AFF_OBS:
1569 *aff_obs = kfmlp_simple_gpu_aff_obs_new(&cedf_kfmlp_affinity_ops, args);
1570 break;
1571
1572 case KFMLP_GPU_AFF_OBS:
1573 *aff_obs = kfmlp_gpu_aff_obs_new(&cedf_kfmlp_affinity_ops, args);
1574 break;
1575
1576#ifdef CONFIG_LITMUS_NESTED_LOCKING
1577 case IKGLP_SIMPLE_GPU_AFF_OBS:
1578 *aff_obs = ikglp_simple_gpu_aff_obs_new(&cedf_ikglp_affinity_ops, args);
1579 break;
1580
1581 case IKGLP_GPU_AFF_OBS:
1582 *aff_obs = ikglp_gpu_aff_obs_new(&cedf_ikglp_affinity_ops, args);
1583 break;
1584#endif
1585 default:
1586 err = -ENXIO;
1587 goto UNSUPPORTED_AFF_OBS;
1588 };
1589
1590 if (*aff_obs)
1591 err = 0;
1592 else
1593 err = -ENOMEM;
1594
1595UNSUPPORTED_AFF_OBS:
1596 return err;
1597}
1598#endif
1599
1600
1601
670 1602
671#ifdef VERBOSE_INIT 1603#ifdef VERBOSE_INIT
672static void print_cluster_topology(cpumask_var_t mask, int cpu) 1604static void print_cluster_topology(cpumask_var_t mask, int cpu)
@@ -681,16 +1613,17 @@ static void print_cluster_topology(cpumask_var_t mask, int cpu)
681} 1613}
682#endif 1614#endif
683 1615
684static int clusters_allocated = 0;
685
686static void cleanup_cedf(void) 1616static void cleanup_cedf(void)
687{ 1617{
688 int i; 1618 int i;
689 1619
1620#ifdef CONFIG_LITMUS_NVIDIA
1621 shutdown_nvidia_info();
1622#endif
1623
690 if (clusters_allocated) { 1624 if (clusters_allocated) {
691 for (i = 0; i < num_clusters; i++) { 1625 for (i = 0; i < num_clusters; i++) {
692 kfree(cedf[i].cpus); 1626 kfree(cedf[i].cpus);
693 kfree(cedf[i].heap_node);
694 free_cpumask_var(cedf[i].cpu_map); 1627 free_cpumask_var(cedf[i].cpu_map);
695 } 1628 }
696 1629
@@ -750,12 +1683,16 @@ static long cedf_activate_plugin(void)
750 1683
751 cedf[i].cpus = kmalloc(cluster_size * sizeof(cpu_entry_t), 1684 cedf[i].cpus = kmalloc(cluster_size * sizeof(cpu_entry_t),
752 GFP_ATOMIC); 1685 GFP_ATOMIC);
753 cedf[i].heap_node = kmalloc( 1686 INIT_BINHEAP_HANDLE(&(cedf[i].cpu_heap), cpu_lower_prio);
754 cluster_size * sizeof(struct bheap_node),
755 GFP_ATOMIC);
756 bheap_init(&(cedf[i].cpu_heap));
757 edf_domain_init(&(cedf[i].domain), NULL, cedf_release_jobs); 1687 edf_domain_init(&(cedf[i].domain), NULL, cedf_release_jobs);
758 1688
1689
1690#ifdef CONFIG_LITMUS_PAI_SOFTIRQD
1691 cedf[i].pending_tasklets.head = NULL;
1692 cedf[i].pending_tasklets.tail = &(cedf[i].pending_tasklets.head);
1693#endif
1694
1695
759 if(!zalloc_cpumask_var(&cedf[i].cpu_map, GFP_ATOMIC)) 1696 if(!zalloc_cpumask_var(&cedf[i].cpu_map, GFP_ATOMIC))
760 return -ENOMEM; 1697 return -ENOMEM;
761#ifdef CONFIG_RELEASE_MASTER 1698#ifdef CONFIG_RELEASE_MASTER
@@ -766,6 +1703,10 @@ static long cedf_activate_plugin(void)
766 /* cycle through cluster and add cpus to them */ 1703 /* cycle through cluster and add cpus to them */
767 for (i = 0; i < num_clusters; i++) { 1704 for (i = 0; i < num_clusters; i++) {
768 1705
1706#ifdef CONFIG_LITMUS_DGL_SUPPORT
1707 raw_spin_lock_init(&cedf[i].dgl_lock);
1708#endif
1709
769 for_each_online_cpu(cpu) { 1710 for_each_online_cpu(cpu) {
770 /* check if the cpu is already in a cluster */ 1711 /* check if the cpu is already in a cluster */
771 for (j = 0; j < num_clusters; j++) 1712 for (j = 0; j < num_clusters; j++)
@@ -796,8 +1737,8 @@ static long cedf_activate_plugin(void)
796 atomic_set(&entry->will_schedule, 0); 1737 atomic_set(&entry->will_schedule, 0);
797 entry->cpu = ccpu; 1738 entry->cpu = ccpu;
798 entry->cluster = &cedf[i]; 1739 entry->cluster = &cedf[i];
799 entry->hn = &(cedf[i].heap_node[cpu_count]); 1740
800 bheap_node_init(&entry->hn, entry); 1741 INIT_BINHEAP_NODE(&entry->hn);
801 1742
802 cpu_count++; 1743 cpu_count++;
803 1744
@@ -814,6 +1755,40 @@ static long cedf_activate_plugin(void)
814 } 1755 }
815 } 1756 }
816 1757
1758#ifdef CONFIG_LITMUS_SOFTIRQD
1759 {
1760 /* distribute the daemons evenly across the clusters. */
1761 int* affinity = kmalloc(NR_LITMUS_SOFTIRQD * sizeof(int), GFP_ATOMIC);
1762 int num_daemons_per_cluster = NR_LITMUS_SOFTIRQD / num_clusters;
1763 int left_over = NR_LITMUS_SOFTIRQD % num_clusters;
1764
1765 int daemon = 0;
1766 for(i = 0; i < num_clusters; ++i)
1767 {
1768 int num_on_this_cluster = num_daemons_per_cluster;
1769 if(left_over)
1770 {
1771 ++num_on_this_cluster;
1772 --left_over;
1773 }
1774
1775 for(j = 0; j < num_on_this_cluster; ++j)
1776 {
1777 // first CPU of this cluster
1778 affinity[daemon++] = i*cluster_size;
1779 }
1780 }
1781
1782 spawn_klitirqd(affinity);
1783
1784 kfree(affinity);
1785 }
1786#endif
1787
1788#ifdef CONFIG_LITMUS_NVIDIA
1789 init_nvidia_info();
1790#endif
1791
817 free_cpumask_var(mask); 1792 free_cpumask_var(mask);
818 clusters_allocated = 1; 1793 clusters_allocated = 1;
819 return 0; 1794 return 0;
@@ -832,6 +1807,32 @@ static struct sched_plugin cedf_plugin __cacheline_aligned_in_smp = {
832 .task_block = cedf_task_block, 1807 .task_block = cedf_task_block,
833 .admit_task = cedf_admit_task, 1808 .admit_task = cedf_admit_task,
834 .activate_plugin = cedf_activate_plugin, 1809 .activate_plugin = cedf_activate_plugin,
1810 .compare = edf_higher_prio,
1811#ifdef CONFIG_LITMUS_LOCKING
1812 .allocate_lock = cedf_allocate_lock,
1813 .increase_prio = increase_priority_inheritance,
1814 .decrease_prio = decrease_priority_inheritance,
1815#endif
1816#ifdef CONFIG_LITMUS_NESTED_LOCKING
1817 .nested_increase_prio = nested_increase_priority_inheritance,
1818 .nested_decrease_prio = nested_decrease_priority_inheritance,
1819 .__compare = __edf_higher_prio,
1820#endif
1821#ifdef CONFIG_LITMUS_DGL_SUPPORT
1822 .get_dgl_spinlock = cedf_get_dgl_spinlock,
1823#endif
1824#ifdef CONFIG_LITMUS_AFFINITY_LOCKING
1825 .allocate_aff_obs = cedf_allocate_affinity_observer,
1826#endif
1827#ifdef CONFIG_LITMUS_SOFTIRQD
1828 .increase_prio_klitirqd = increase_priority_inheritance_klitirqd,
1829 .decrease_prio_klitirqd = decrease_priority_inheritance_klitirqd,
1830#endif
1831#ifdef CONFIG_LITMUS_PAI_SOFTIRQD
1832 .enqueue_pai_tasklet = cedf_enqueue_pai_tasklet,
1833 .change_prio_pai_tasklet = cedf_change_prio_pai_tasklet,
1834 .run_tasklets = cedf_run_tasklets,
1835#endif
835}; 1836};
836 1837
837static struct proc_dir_entry *cluster_file = NULL, *cedf_dir = NULL; 1838static struct proc_dir_entry *cluster_file = NULL, *cedf_dir = NULL;
generated by cgit v1.2.2 (git 2.25.0) at 2024-12-03 12:20:32 -0500