aboutsummaryrefslogtreecommitdiffstats
path: root/litmus/sched_cedf.c
diff options
context:
space:
mode:
Diffstat (limited to 'litmus/sched_cedf.c')
-rw-r--r--litmus/sched_cedf.c1275
1 files changed, 1267 insertions, 8 deletions
diff --git a/litmus/sched_cedf.c b/litmus/sched_cedf.c
index 480c62bc895b..3251fb1602f8 100644
--- a/litmus/sched_cedf.c
+++ b/litmus/sched_cedf.c
@@ -29,6 +29,7 @@
29#include <linux/percpu.h> 29#include <linux/percpu.h>
30#include <linux/sched.h> 30#include <linux/sched.h>
31#include <linux/slab.h> 31#include <linux/slab.h>
32#include <linux/uaccess.h>
32 33
33#include <linux/module.h> 34#include <linux/module.h>
34 35
@@ -49,7 +50,23 @@
49 50
50/* to configure the cluster size */ 51/* to configure the cluster size */
51#include <litmus/litmus_proc.h> 52#include <litmus/litmus_proc.h>
52#include <linux/uaccess.h> 53
54#ifdef CONFIG_SCHED_CPU_AFFINITY
55#include <litmus/affinity.h>
56#endif
57
58#ifdef CONFIG_LITMUS_SOFTIRQD
59#include <litmus/litmus_softirq.h>
60#endif
61
62#ifdef CONFIG_LITMUS_PAI_SOFTIRQD
63#include <linux/interrupt.h>
64#include <litmus/trace.h>
65#endif
66
67#ifdef CONFIG_LITMUS_NVIDIA
68#include <litmus/nvidia_info.h>
69#endif
53 70
54/* Reference configuration variable. Determines which cache level is used to 71/* Reference configuration variable. Determines which cache level is used to
55 * group CPUs into clusters. GLOBAL_CLUSTER, which is the default, means that 72 * group CPUs into clusters. GLOBAL_CLUSTER, which is the default, means that
@@ -83,6 +100,15 @@ DEFINE_PER_CPU(cpu_entry_t, cedf_cpu_entries);
83#define test_will_schedule(cpu) \ 100#define test_will_schedule(cpu) \
84 (atomic_read(&per_cpu(cedf_cpu_entries, cpu).will_schedule)) 101 (atomic_read(&per_cpu(cedf_cpu_entries, cpu).will_schedule))
85 102
103
104#ifdef CONFIG_LITMUS_PAI_SOFTIRQD
105struct tasklet_head
106{
107 struct tasklet_struct *head;
108 struct tasklet_struct **tail;
109};
110#endif
111
86/* 112/*
87 * In C-EDF there is a cedf domain _per_ cluster 113 * In C-EDF there is a cedf domain _per_ cluster
88 * The number of clusters is dynamically determined accordingly to the 114 * The number of clusters is dynamically determined accordingly to the
@@ -100,6 +126,10 @@ typedef struct clusterdomain {
100 struct bheap cpu_heap; 126 struct bheap cpu_heap;
101 /* lock for this cluster */ 127 /* lock for this cluster */
102#define cluster_lock domain.ready_lock 128#define cluster_lock domain.ready_lock
129
130#ifdef CONFIG_LITMUS_PAI_SOFTIRQD
131 struct tasklet_head pending_tasklets;
132#endif
103} cedf_domain_t; 133} cedf_domain_t;
104 134
105/* a cedf_domain per cluster; allocation is done at init/activation time */ 135/* a cedf_domain per cluster; allocation is done at init/activation time */
@@ -208,7 +238,7 @@ static noinline void link_task_to_cpu(struct task_struct* linked,
208} 238}
209 239
210/* unlink - Make sure a task is not linked any longer to an entry 240/* unlink - Make sure a task is not linked any longer to an entry
211 * where it was linked before. Must hold cedf_lock. 241 * where it was linked before. Must hold cluster_lock.
212 */ 242 */
213static noinline void unlink(struct task_struct* t) 243static noinline void unlink(struct task_struct* t)
214{ 244{
@@ -244,7 +274,7 @@ static void preempt(cpu_entry_t *entry)
244} 274}
245 275
246/* requeue - Put an unlinked task into gsn-edf domain. 276/* requeue - Put an unlinked task into gsn-edf domain.
247 * Caller must hold cedf_lock. 277 * Caller must hold cluster_lock.
248 */ 278 */
249static noinline void requeue(struct task_struct* task) 279static noinline void requeue(struct task_struct* task)
250{ 280{
@@ -339,13 +369,17 @@ static void cedf_release_jobs(rt_domain_t* rt, struct bheap* tasks)
339 raw_spin_unlock_irqrestore(&cluster->cluster_lock, flags); 369 raw_spin_unlock_irqrestore(&cluster->cluster_lock, flags);
340} 370}
341 371
342/* caller holds cedf_lock */ 372/* caller holds cluster_lock */
343static noinline void job_completion(struct task_struct *t, int forced) 373static noinline void job_completion(struct task_struct *t, int forced)
344{ 374{
345 BUG_ON(!t); 375 BUG_ON(!t);
346 376
347 sched_trace_task_completion(t, forced); 377 sched_trace_task_completion(t, forced);
348 378
379#ifdef CONFIG_LITMUS_NVIDIA
380 atomic_set(&tsk_rt(t)->nv_int_count, 0);
381#endif
382
349 TRACE_TASK(t, "job_completion().\n"); 383 TRACE_TASK(t, "job_completion().\n");
350 384
351 /* set flags */ 385 /* set flags */
@@ -389,6 +423,461 @@ static void cedf_tick(struct task_struct* t)
389 } 423 }
390} 424}
391 425
426
427
428
429
430
431
432
433
434
435
436
437
438#ifdef CONFIG_LITMUS_PAI_SOFTIRQD
439
440
441static void __do_lit_tasklet(struct tasklet_struct* tasklet, unsigned long flushed)
442{
443 if (!atomic_read(&tasklet->count)) {
444 if(tasklet->owner) {
445 sched_trace_tasklet_begin(tasklet->owner);
446 }
447
448 if (!test_and_clear_bit(TASKLET_STATE_SCHED, &tasklet->state))
449 {
450 BUG();
451 }
452 TRACE("%s: Invoking tasklet with owner pid = %d (flushed = %d).\n",
453 __FUNCTION__,
454 (tasklet->owner) ? tasklet->owner->pid : -1,
455 (tasklet->owner) ? 0 : 1);
456 tasklet->func(tasklet->data);
457 tasklet_unlock(tasklet);
458
459 if(tasklet->owner) {
460 sched_trace_tasklet_end(tasklet->owner, flushed);
461 }
462 }
463 else {
464 BUG();
465 }
466}
467
468
469static void __extract_tasklets(cedf_domain_t* cluster, struct task_struct* task, struct tasklet_head* task_tasklets)
470{
471 struct tasklet_struct* step;
472 struct tasklet_struct* tasklet;
473 struct tasklet_struct* prev;
474
475 task_tasklets->head = NULL;
476 task_tasklets->tail = &(task_tasklets->head);
477
478 prev = NULL;
479 for(step = cluster->pending_tasklets.head; step != NULL; step = step->next)
480 {
481 if(step->owner == task)
482 {
483 TRACE("%s: Found tasklet to flush: %d\n", __FUNCTION__, step->owner->pid);
484
485 tasklet = step;
486
487 if(prev) {
488 prev->next = tasklet->next;
489 }
490 else if(cluster->pending_tasklets.head == tasklet) {
491 // we're at the head.
492 cluster->pending_tasklets.head = tasklet->next;
493 }
494
495 if(cluster->pending_tasklets.tail == &tasklet) {
496 // we're at the tail
497 if(prev) {
498 cluster->pending_tasklets.tail = &prev;
499 }
500 else {
501 cluster->pending_tasklets.tail = &(cluster->pending_tasklets.head);
502 }
503 }
504
505 tasklet->next = NULL;
506 *(task_tasklets->tail) = tasklet;
507 task_tasklets->tail = &(tasklet->next);
508 }
509 else {
510 prev = step;
511 }
512 }
513}
514
515static void flush_tasklets(cedf_domain_t* cluster, struct task_struct* task)
516{
517#if 0
518 unsigned long flags;
519 struct tasklet_head task_tasklets;
520 struct tasklet_struct* step;
521
522 raw_spin_lock_irqsave(&cluster->cluster_lock, flags);
523 __extract_tasklets(cluster, task, &task_tasklets);
524 raw_spin_unlock_irqrestore(&cluster->cluster_lock, flags);
525
526 if(cluster->pending_tasklets.head != NULL) {
527 TRACE("%s: Flushing tasklets for %d...\n", __FUNCTION__, task->pid);
528 }
529
530 // now execute any flushed tasklets.
531 for(step = cluster->pending_tasklets.head; step != NULL; /**/)
532 {
533 struct tasklet_struct* temp = step->next;
534
535 step->next = NULL;
536 __do_lit_tasklet(step, 1ul);
537
538 step = temp;
539 }
540#endif
541
542 // lazy flushing.
543 // just change ownership to NULL and let an idle processor
544 // take care of it. :P
545
546 struct tasklet_struct* step;
547 unsigned long flags;
548
549 raw_spin_lock_irqsave(&cluster->cluster_lock, flags);
550
551 for(step = cluster->pending_tasklets.head; step != NULL; step = step->next)
552 {
553 if(step->owner == task)
554 {
555 TRACE("%s: Found tasklet to flush: %d\n", __FUNCTION__, step->owner->pid);
556 step->owner = NULL;
557 }
558 }
559
560 raw_spin_unlock_irqrestore(&cluster->cluster_lock, flags);
561}
562
563
564static void do_lit_tasklets(cedf_domain_t* cluster, struct task_struct* sched_task)
565{
566 int work_to_do = 1;
567 struct tasklet_struct *tasklet = NULL;
568 //struct tasklet_struct *step;
569 unsigned long flags;
570
571 while(work_to_do) {
572
573 TS_NV_SCHED_BOTISR_START;
574
575 // remove tasklet at head of list if it has higher priority.
576 raw_spin_lock_irqsave(&cluster->cluster_lock, flags);
577
578/*
579 step = cluster->pending_tasklets.head;
580 TRACE("%s: (BEFORE) dumping tasklet queue...\n", __FUNCTION__);
581 while(step != NULL){
582 TRACE("%s: %p (%d)\n", __FUNCTION__, step, step->owner->pid);
583 step = step->next;
584 }
585 TRACE("%s: tail = %p (%d)\n", __FUNCTION__, *(cluster->pending_tasklets.tail), (*(cluster->pending_tasklets.tail) != NULL) ? (*(cluster->pending_tasklets.tail))->owner->pid : -1);
586 TRACE("%s: done.\n", __FUNCTION__);
587 */
588
589 if(cluster->pending_tasklets.head != NULL) {
590 // remove tasklet at head.
591 tasklet = cluster->pending_tasklets.head;
592
593 if(edf_higher_prio(tasklet->owner, sched_task)) {
594
595 if(NULL == tasklet->next) {
596 // tasklet is at the head, list only has one element
597 TRACE("%s: Tasklet for %d is the last element in tasklet queue.\n", __FUNCTION__, (tasklet->owner) ? tasklet->owner->pid : -1);
598 cluster->pending_tasklets.tail = &(cluster->pending_tasklets.head);
599 }
600
601 // remove the tasklet from the queue
602 cluster->pending_tasklets.head = tasklet->next;
603
604 TRACE("%s: Removed tasklet for %d from tasklet queue.\n", __FUNCTION__, (tasklet->owner) ? tasklet->owner->pid : -1);
605 }
606 else {
607 TRACE("%s: Pending tasklet (%d) does not have priority to run on this CPU (%d).\n", __FUNCTION__, (tasklet->owner) ? tasklet->owner->pid : -1, smp_processor_id());
608 tasklet = NULL;
609 }
610 }
611 else {
612 TRACE("%s: Tasklet queue is empty.\n", __FUNCTION__);
613 }
614
615
616 /*
617 step = cluster->pending_tasklets.head;
618 TRACE("%s: (AFTER) dumping tasklet queue...\n", __FUNCTION__);
619 while(step != NULL){
620 TRACE("%s: %p (%d)\n", __FUNCTION__, step, step->owner->pid);
621 step = step->next;
622 }
623 TRACE("%s: tail = %p (%d)\n", __FUNCTION__, *(cluster->pending_tasklets.tail), (*(cluster->pending_tasklets.tail) != NULL) ? (*(cluster->pending_tasklets.tail))->owner->pid : -1);
624 TRACE("%s: done.\n", __FUNCTION__);
625 */
626
627 raw_spin_unlock_irqrestore(&cluster->cluster_lock, flags);
628
629
630 TS_NV_SCHED_BOTISR_END;
631
632 if(tasklet) {
633 __do_lit_tasklet(tasklet, 0ul);
634 tasklet = NULL;
635 }
636 else {
637 work_to_do = 0;
638 }
639 }
640
641 //TRACE("%s: exited.\n", __FUNCTION__);
642}
643
644
645static void run_tasklets(struct task_struct* sched_task)
646{
647 cedf_domain_t* cluster;
648
649#if 0
650 int task_is_rt = is_realtime(sched_task);
651 cedf_domain_t* cluster;
652
653 if(is_realtime(sched_task)) {
654 cluster = task_cpu_cluster(sched_task);
655 }
656 else {
657 cluster = remote_cluster(get_cpu());
658 }
659
660 if(cluster && cluster->pending_tasklets.head != NULL) {
661 TRACE("%s: There are tasklets to process.\n", __FUNCTION__);
662
663 do_lit_tasklets(cluster, sched_task);
664 }
665
666 if(!task_is_rt) {
667 put_cpu_no_resched();
668 }
669#else
670
671 preempt_disable();
672
673 cluster = (is_realtime(sched_task)) ?
674 task_cpu_cluster(sched_task) :
675 remote_cluster(smp_processor_id());
676
677 if(cluster && cluster->pending_tasklets.head != NULL) {
678 TRACE("%s: There are tasklets to process.\n", __FUNCTION__);
679 do_lit_tasklets(cluster, sched_task);
680 }
681
682 preempt_enable_no_resched();
683
684#endif
685}
686
687
688static void __add_pai_tasklet(struct tasklet_struct* tasklet, cedf_domain_t* cluster)
689{
690 struct tasklet_struct* step;
691
692 /*
693 step = cluster->pending_tasklets.head;
694 TRACE("%s: (BEFORE) dumping tasklet queue...\n", __FUNCTION__);
695 while(step != NULL){
696 TRACE("%s: %p (%d)\n", __FUNCTION__, step, step->owner->pid);
697 step = step->next;
698 }
699 TRACE("%s: tail = %p (%d)\n", __FUNCTION__, *(cluster->pending_tasklets.tail), (*(cluster->pending_tasklets.tail) != NULL) ? (*(cluster->pending_tasklets.tail))->owner->pid : -1);
700 TRACE("%s: done.\n", __FUNCTION__);
701 */
702
703
704 tasklet->next = NULL; // make sure there are no old values floating around
705
706 step = cluster->pending_tasklets.head;
707 if(step == NULL) {
708 TRACE("%s: tasklet queue empty. inserting tasklet for %d at head.\n", __FUNCTION__, tasklet->owner->pid);
709 // insert at tail.
710 *(cluster->pending_tasklets.tail) = tasklet;
711 cluster->pending_tasklets.tail = &(tasklet->next);
712 }
713 else if((*(cluster->pending_tasklets.tail) != NULL) &&
714 edf_higher_prio((*(cluster->pending_tasklets.tail))->owner, tasklet->owner)) {
715 // insert at tail.
716 TRACE("%s: tasklet belongs at end. inserting tasklet for %d at tail.\n", __FUNCTION__, tasklet->owner->pid);
717
718 *(cluster->pending_tasklets.tail) = tasklet;
719 cluster->pending_tasklets.tail = &(tasklet->next);
720 }
721 else {
722
723 //WARN_ON(1 == 1);
724
725 // insert the tasklet somewhere in the middle.
726
727 TRACE("%s: tasklet belongs somewhere in the middle.\n", __FUNCTION__);
728
729 while(step->next && edf_higher_prio(step->next->owner, tasklet->owner)) {
730 step = step->next;
731 }
732
733 // insert tasklet right before step->next.
734
735 TRACE("%s: inserting tasklet for %d between %d and %d.\n", __FUNCTION__,
736 tasklet->owner->pid,
737 (step->owner) ?
738 step->owner->pid :
739 -1,
740 (step->next) ?
741 ((step->next->owner) ?
742 step->next->owner->pid :
743 -1) :
744 -1);
745
746 tasklet->next = step->next;
747 step->next = tasklet;
748
749 // patch up the head if needed.
750 if(cluster->pending_tasklets.head == step)
751 {
752 TRACE("%s: %d is the new tasklet queue head.\n", __FUNCTION__, tasklet->owner->pid);
753 cluster->pending_tasklets.head = tasklet;
754 }
755 }
756
757 /*
758 step = cluster->pending_tasklets.head;
759 TRACE("%s: (AFTER) dumping tasklet queue...\n", __FUNCTION__);
760 while(step != NULL){
761 TRACE("%s: %p (%d)\n", __FUNCTION__, step, step->owner->pid);
762 step = step->next;
763 }
764 TRACE("%s: tail = %p (%d)\n", __FUNCTION__, *(cluster->pending_tasklets.tail), (*(cluster->pending_tasklets.tail) != NULL) ? (*(cluster->pending_tasklets.tail))->owner->pid : -1);
765 TRACE("%s: done.\n", __FUNCTION__);
766 */
767
768// TODO: Maintain this list in priority order.
769// tasklet->next = NULL;
770// *(cluster->pending_tasklets.tail) = tasklet;
771// cluster->pending_tasklets.tail = &tasklet->next;
772}
773
774static int enqueue_pai_tasklet(struct tasklet_struct* tasklet)
775{
776 cedf_domain_t *cluster = NULL;
777 cpu_entry_t *targetCPU = NULL;
778 int thisCPU;
779 int runLocal = 0;
780 int runNow = 0;
781 unsigned long flags;
782
783 if(unlikely((tasklet->owner == NULL) || !is_realtime(tasklet->owner)))
784 {
785 TRACE("%s: No owner associated with this tasklet!\n", __FUNCTION__);
786 return 0;
787 }
788
789 cluster = task_cpu_cluster(tasklet->owner);
790
791 raw_spin_lock_irqsave(&cluster->cluster_lock, flags);
792
793 thisCPU = smp_processor_id();
794
795#if 1
796#ifdef CONFIG_SCHED_CPU_AFFINITY
797 {
798 cpu_entry_t* affinity = NULL;
799
800 // use this CPU if it is in our cluster and isn't running any RT work.
801 if(cpu_isset(thisCPU, *cluster->cpu_map) && (__get_cpu_var(cedf_cpu_entries).linked == NULL)) {
802 affinity = &(__get_cpu_var(cedf_cpu_entries));
803 }
804 else {
805 // this CPU is busy or shouldn't run tasklet in this cluster.
806 // look for available near by CPUs.
807 // NOTE: Affinity towards owner and not this CPU. Is this right?
808 affinity =
809 cedf_get_nearest_available_cpu(cluster,
810 &per_cpu(cedf_cpu_entries, task_cpu(tasklet->owner)));
811 }
812
813 targetCPU = affinity;
814 }
815#endif
816#endif
817
818 if (targetCPU == NULL) {
819 targetCPU = lowest_prio_cpu(cluster);
820 }
821
822 if (edf_higher_prio(tasklet->owner, targetCPU->linked)) {
823 if (thisCPU == targetCPU->cpu) {
824 TRACE("%s: Run tasklet locally (and now).\n", __FUNCTION__);
825 runLocal = 1;
826 runNow = 1;
827 }
828 else {
829 TRACE("%s: Run tasklet remotely (and now).\n", __FUNCTION__);
830 runLocal = 0;
831 runNow = 1;
832 }
833 }
834 else {
835 runLocal = 0;
836 runNow = 0;
837 }
838
839 if(!runLocal) {
840 // enqueue the tasklet
841 __add_pai_tasklet(tasklet, cluster);
842 }
843
844 raw_spin_unlock_irqrestore(&cluster->cluster_lock, flags);
845
846
847 if (runLocal /*&& runNow */) { // runNow == 1 is implied
848 TRACE("%s: Running tasklet on CPU where it was received.\n", __FUNCTION__);
849 __do_lit_tasklet(tasklet, 0ul);
850 }
851 else if (runNow /*&& !runLocal */) { // runLocal == 0 is implied
852 TRACE("%s: Triggering CPU %d to run tasklet.\n", __FUNCTION__, targetCPU->cpu);
853 preempt(targetCPU); // need to be protected by cluster_lock?
854 }
855 else {
856 TRACE("%s: Scheduling of tasklet was deferred.\n", __FUNCTION__);
857 }
858
859 return(1); // success
860}
861
862
863#endif
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
392/* Getting schedule() right is a bit tricky. schedule() may not make any 881/* Getting schedule() right is a bit tricky. schedule() may not make any
393 * assumptions on the state of the current task since it may be called for a 882 * assumptions on the state of the current task since it may be called for a
394 * number of reasons. The reasons include a scheduler_tick() determined that it 883 * number of reasons. The reasons include a scheduler_tick() determined that it
@@ -514,7 +1003,7 @@ static struct task_struct* cedf_schedule(struct task_struct * prev)
514 raw_spin_unlock(&cluster->cluster_lock); 1003 raw_spin_unlock(&cluster->cluster_lock);
515 1004
516#ifdef WANT_ALL_SCHED_EVENTS 1005#ifdef WANT_ALL_SCHED_EVENTS
517 TRACE("cedf_lock released, next=0x%p\n", next); 1006 TRACE("cluster_lock released, next=0x%p\n", next);
518 1007
519 if (next) 1008 if (next)
520 TRACE_TASK(next, "scheduled at %llu\n", litmus_clock()); 1009 TRACE_TASK(next, "scheduled at %llu\n", litmus_clock());
@@ -522,7 +1011,6 @@ static struct task_struct* cedf_schedule(struct task_struct * prev)
522 TRACE("becomes idle at %llu.\n", litmus_clock()); 1011 TRACE("becomes idle at %llu.\n", litmus_clock());
523#endif 1012#endif
524 1013
525
526 return next; 1014 return next;
527} 1015}
528 1016
@@ -586,7 +1074,7 @@ static void cedf_task_new(struct task_struct * t, int on_rq, int running)
586static void cedf_task_wake_up(struct task_struct *task) 1074static void cedf_task_wake_up(struct task_struct *task)
587{ 1075{
588 unsigned long flags; 1076 unsigned long flags;
589 lt_t now; 1077 //lt_t now;
590 cedf_domain_t *cluster; 1078 cedf_domain_t *cluster;
591 1079
592 TRACE_TASK(task, "wake_up at %llu\n", litmus_clock()); 1080 TRACE_TASK(task, "wake_up at %llu\n", litmus_clock());
@@ -594,6 +1082,8 @@ static void cedf_task_wake_up(struct task_struct *task)
594 cluster = task_cpu_cluster(task); 1082 cluster = task_cpu_cluster(task);
595 1083
596 raw_spin_lock_irqsave(&cluster->cluster_lock, flags); 1084 raw_spin_lock_irqsave(&cluster->cluster_lock, flags);
1085
1086#if 0 // sproadic task model
597 /* We need to take suspensions because of semaphores into 1087 /* We need to take suspensions because of semaphores into
598 * account! If a job resumes after being suspended due to acquiring 1088 * account! If a job resumes after being suspended due to acquiring
599 * a semaphore, it should never be treated as a new job release. 1089 * a semaphore, it should never be treated as a new job release.
@@ -615,7 +1105,13 @@ static void cedf_task_wake_up(struct task_struct *task)
615 } 1105 }
616 } 1106 }
617 } 1107 }
618 cedf_job_arrival(task); 1108#endif
1109
1110 set_rt_flags(task, RT_F_RUNNING); // periodic model
1111
1112 if(tsk_rt(task)->linked_on == NO_CPU)
1113 cedf_job_arrival(task);
1114
619 raw_spin_unlock_irqrestore(&cluster->cluster_lock, flags); 1115 raw_spin_unlock_irqrestore(&cluster->cluster_lock, flags);
620} 1116}
621 1117
@@ -642,6 +1138,10 @@ static void cedf_task_exit(struct task_struct * t)
642 unsigned long flags; 1138 unsigned long flags;
643 cedf_domain_t *cluster = task_cpu_cluster(t); 1139 cedf_domain_t *cluster = task_cpu_cluster(t);
644 1140
1141#ifdef CONFIG_LITMUS_PAI_SOFTIRQD
1142 flush_tasklets(cluster, t);
1143#endif
1144
645 /* unlink if necessary */ 1145 /* unlink if necessary */
646 raw_spin_lock_irqsave(&cluster->cluster_lock, flags); 1146 raw_spin_lock_irqsave(&cluster->cluster_lock, flags);
647 unlink(t); 1147 unlink(t);
@@ -662,6 +1162,711 @@ static long cedf_admit_task(struct task_struct* tsk)
662 return task_cpu(tsk) == tsk->rt_param.task_params.cpu ? 0 : -EINVAL; 1162 return task_cpu(tsk) == tsk->rt_param.task_params.cpu ? 0 : -EINVAL;
663} 1163}
664 1164
1165
1166
1167#ifdef CONFIG_LITMUS_LOCKING
1168
1169#include <litmus/fdso.h>
1170
1171
1172static void __set_priority_inheritance(struct task_struct* t, struct task_struct* prio_inh)
1173{
1174 int linked_on;
1175 int check_preempt = 0;
1176
1177 cedf_domain_t* cluster = task_cpu_cluster(t);
1178
1179 if(prio_inh != NULL)
1180 TRACE_TASK(t, "inherits priority from %s/%d\n", prio_inh->comm, prio_inh->pid);
1181 else
1182 TRACE_TASK(t, "inherits priority from %p\n", prio_inh);
1183
1184 sched_trace_eff_prio_change(t, prio_inh);
1185
1186 tsk_rt(t)->inh_task = prio_inh;
1187
1188 linked_on = tsk_rt(t)->linked_on;
1189
1190 /* If it is scheduled, then we need to reorder the CPU heap. */
1191 if (linked_on != NO_CPU) {
1192 TRACE_TASK(t, "%s: linked on %d\n",
1193 __FUNCTION__, linked_on);
1194 /* Holder is scheduled; need to re-order CPUs.
1195 * We can't use heap_decrease() here since
1196 * the cpu_heap is ordered in reverse direction, so
1197 * it is actually an increase. */
1198 bheap_delete(cpu_lower_prio, &cluster->cpu_heap,
1199 per_cpu(cedf_cpu_entries, linked_on).hn);
1200 bheap_insert(cpu_lower_prio, &cluster->cpu_heap,
1201 per_cpu(cedf_cpu_entries, linked_on).hn);
1202 } else {
1203 /* holder may be queued: first stop queue changes */
1204 raw_spin_lock(&cluster->domain.release_lock);
1205 if (is_queued(t)) {
1206 TRACE_TASK(t, "%s: is queued\n", __FUNCTION__);
1207
1208 /* We need to update the position of holder in some
1209 * heap. Note that this could be a release heap if we
1210 * budget enforcement is used and this job overran. */
1211 check_preempt = !bheap_decrease(edf_ready_order, tsk_rt(t)->heap_node);
1212
1213 } else {
1214 /* Nothing to do: if it is not queued and not linked
1215 * then it is either sleeping or currently being moved
1216 * by other code (e.g., a timer interrupt handler) that
1217 * will use the correct priority when enqueuing the
1218 * task. */
1219 TRACE_TASK(t, "%s: is NOT queued => Done.\n", __FUNCTION__);
1220 }
1221 raw_spin_unlock(&cluster->domain.release_lock);
1222
1223 /* If holder was enqueued in a release heap, then the following
1224 * preemption check is pointless, but we can't easily detect
1225 * that case. If you want to fix this, then consider that
1226 * simply adding a state flag requires O(n) time to update when
1227 * releasing n tasks, which conflicts with the goal to have
1228 * O(log n) merges. */
1229 if (check_preempt) {
1230 /* heap_decrease() hit the top level of the heap: make
1231 * sure preemption checks get the right task, not the
1232 * potentially stale cache. */
1233 bheap_uncache_min(edf_ready_order, &cluster->domain.ready_queue);
1234 check_for_preemptions(cluster);
1235 }
1236 }
1237}
1238
1239/* called with IRQs off */
1240static void set_priority_inheritance(struct task_struct* t, struct task_struct* prio_inh)
1241{
1242 cedf_domain_t* cluster = task_cpu_cluster(t);
1243
1244 raw_spin_lock(&cluster->cluster_lock);
1245
1246 __set_priority_inheritance(t, prio_inh);
1247
1248#ifdef CONFIG_LITMUS_SOFTIRQD
1249 if(tsk_rt(t)->cur_klitirqd != NULL)
1250 {
1251 TRACE_TASK(t, "%s/%d inherits a new priority!\n",
1252 tsk_rt(t)->cur_klitirqd->comm, tsk_rt(t)->cur_klitirqd->pid);
1253
1254 __set_priority_inheritance(tsk_rt(t)->cur_klitirqd, prio_inh);
1255 }
1256#endif
1257
1258 raw_spin_unlock(&cluster->cluster_lock);
1259}
1260
1261
1262/* called with IRQs off */
1263static void __clear_priority_inheritance(struct task_struct* t)
1264{
1265 TRACE_TASK(t, "priority restored\n");
1266
1267 if(tsk_rt(t)->scheduled_on != NO_CPU)
1268 {
1269 sched_trace_eff_prio_change(t, NULL);
1270
1271 tsk_rt(t)->inh_task = NULL;
1272
1273 /* Check if rescheduling is necessary. We can't use heap_decrease()
1274 * since the priority was effectively lowered. */
1275 unlink(t);
1276 cedf_job_arrival(t);
1277 }
1278 else
1279 {
1280 __set_priority_inheritance(t, NULL);
1281 }
1282
1283#ifdef CONFIG_LITMUS_SOFTIRQD
1284 if(tsk_rt(t)->cur_klitirqd != NULL)
1285 {
1286 TRACE_TASK(t, "%s/%d inheritance set back to owner.\n",
1287 tsk_rt(t)->cur_klitirqd->comm, tsk_rt(t)->cur_klitirqd->pid);
1288
1289 if(tsk_rt(tsk_rt(t)->cur_klitirqd)->scheduled_on != NO_CPU)
1290 {
1291 sched_trace_eff_prio_change(tsk_rt(t)->cur_klitirqd, t);
1292
1293 tsk_rt(tsk_rt(t)->cur_klitirqd)->inh_task = t;
1294
1295 /* Check if rescheduling is necessary. We can't use heap_decrease()
1296 * since the priority was effectively lowered. */
1297 unlink(tsk_rt(t)->cur_klitirqd);
1298 cedf_job_arrival(tsk_rt(t)->cur_klitirqd);
1299 }
1300 else
1301 {
1302 __set_priority_inheritance(tsk_rt(t)->cur_klitirqd, t);
1303 }
1304 }
1305#endif
1306}
1307
1308/* called with IRQs off */
1309static void clear_priority_inheritance(struct task_struct* t)
1310{
1311 cedf_domain_t* cluster = task_cpu_cluster(t);
1312
1313 raw_spin_lock(&cluster->cluster_lock);
1314 __clear_priority_inheritance(t);
1315 raw_spin_unlock(&cluster->cluster_lock);
1316}
1317
1318
1319
1320#ifdef CONFIG_LITMUS_SOFTIRQD
1321/* called with IRQs off */
1322static void set_priority_inheritance_klitirqd(struct task_struct* klitirqd,
1323 struct task_struct* old_owner,
1324 struct task_struct* new_owner)
1325{
1326 cedf_domain_t* cluster = task_cpu_cluster(klitirqd);
1327
1328 BUG_ON(!(tsk_rt(klitirqd)->is_proxy_thread));
1329
1330 raw_spin_lock(&cluster->cluster_lock);
1331
1332 if(old_owner != new_owner)
1333 {
1334 if(old_owner)
1335 {
1336 // unreachable?
1337 tsk_rt(old_owner)->cur_klitirqd = NULL;
1338 }
1339
1340 TRACE_TASK(klitirqd, "giving ownership to %s/%d.\n",
1341 new_owner->comm, new_owner->pid);
1342
1343 tsk_rt(new_owner)->cur_klitirqd = klitirqd;
1344 }
1345
1346 __set_priority_inheritance(klitirqd,
1347 (tsk_rt(new_owner)->inh_task == NULL) ?
1348 new_owner :
1349 tsk_rt(new_owner)->inh_task);
1350
1351 raw_spin_unlock(&cluster->cluster_lock);
1352}
1353
1354/* called with IRQs off */
1355static void clear_priority_inheritance_klitirqd(struct task_struct* klitirqd,
1356 struct task_struct* old_owner)
1357{
1358 cedf_domain_t* cluster = task_cpu_cluster(klitirqd);
1359
1360 BUG_ON(!(tsk_rt(klitirqd)->is_proxy_thread));
1361
1362 raw_spin_lock(&cluster->cluster_lock);
1363
1364 TRACE_TASK(klitirqd, "priority restored\n");
1365
1366 if(tsk_rt(klitirqd)->scheduled_on != NO_CPU)
1367 {
1368 tsk_rt(klitirqd)->inh_task = NULL;
1369
1370 /* Check if rescheduling is necessary. We can't use heap_decrease()
1371 * since the priority was effectively lowered. */
1372 unlink(klitirqd);
1373 cedf_job_arrival(klitirqd);
1374 }
1375 else
1376 {
1377 __set_priority_inheritance(klitirqd, NULL);
1378 }
1379
1380 tsk_rt(old_owner)->cur_klitirqd = NULL;
1381
1382 raw_spin_unlock(&cluster->cluster_lock);
1383}
1384#endif // CONFIG_LITMUS_SOFTIRQD
1385
1386
1387/* ******************** KFMLP support ********************** */
1388
1389/* struct for semaphore with priority inheritance */
1390struct kfmlp_queue
1391{
1392 wait_queue_head_t wait;
1393 struct task_struct* owner;
1394 struct task_struct* hp_waiter;
1395 int count; /* number of waiters + holder */
1396};
1397
1398struct kfmlp_semaphore
1399{
1400 struct litmus_lock litmus_lock;
1401
1402 spinlock_t lock;
1403
1404 int num_resources; /* aka k */
1405 struct kfmlp_queue *queues; /* array */
1406 struct kfmlp_queue *shortest_queue; /* pointer to shortest queue */
1407};
1408
1409static inline struct kfmlp_semaphore* kfmlp_from_lock(struct litmus_lock* lock)
1410{
1411 return container_of(lock, struct kfmlp_semaphore, litmus_lock);
1412}
1413
1414static inline int kfmlp_get_idx(struct kfmlp_semaphore* sem,
1415 struct kfmlp_queue* queue)
1416{
1417 return (queue - &sem->queues[0]);
1418}
1419
1420static inline struct kfmlp_queue* kfmlp_get_queue(struct kfmlp_semaphore* sem,
1421 struct task_struct* holder)
1422{
1423 int i;
1424 for(i = 0; i < sem->num_resources; ++i)
1425 if(sem->queues[i].owner == holder)
1426 return(&sem->queues[i]);
1427 return(NULL);
1428}
1429
1430/* caller is responsible for locking */
1431static struct task_struct* kfmlp_find_hp_waiter(struct kfmlp_queue *kqueue,
1432 struct task_struct *skip)
1433{
1434 struct list_head *pos;
1435 struct task_struct *queued, *found = NULL;
1436
1437 list_for_each(pos, &kqueue->wait.task_list) {
1438 queued = (struct task_struct*) list_entry(pos, wait_queue_t,
1439 task_list)->private;
1440
1441 /* Compare task prios, find high prio task. */
1442 if (queued != skip && edf_higher_prio(queued, found))
1443 found = queued;
1444 }
1445 return found;
1446}
1447
1448static inline struct kfmlp_queue* kfmlp_find_shortest(
1449 struct kfmlp_semaphore* sem,
1450 struct kfmlp_queue* search_start)
1451{
1452 // we start our search at search_start instead of at the beginning of the
1453 // queue list to load-balance across all resources.
1454 struct kfmlp_queue* step = search_start;
1455 struct kfmlp_queue* shortest = sem->shortest_queue;
1456
1457 do
1458 {
1459 step = (step+1 != &sem->queues[sem->num_resources]) ?
1460 step+1 : &sem->queues[0];
1461 if(step->count < shortest->count)
1462 {
1463 shortest = step;
1464 if(step->count == 0)
1465 break; /* can't get any shorter */
1466 }
1467 }while(step != search_start);
1468
1469 return(shortest);
1470}
1471
1472static struct task_struct* kfmlp_remove_hp_waiter(struct kfmlp_semaphore* sem)
1473{
1474 /* must hold sem->lock */
1475
1476 struct kfmlp_queue *my_queue = NULL;
1477 struct task_struct *max_hp = NULL;
1478
1479
1480 struct list_head *pos;
1481 struct task_struct *queued;
1482 int i;
1483
1484 for(i = 0; i < sem->num_resources; ++i)
1485 {
1486 if( (sem->queues[i].count > 1) &&
1487 ((my_queue == NULL) ||
1488 (edf_higher_prio(sem->queues[i].hp_waiter, my_queue->hp_waiter))) )
1489 {
1490 my_queue = &sem->queues[i];
1491 }
1492 }
1493
1494 if(my_queue)
1495 {
1496 cedf_domain_t* cluster;
1497
1498 max_hp = my_queue->hp_waiter;
1499 BUG_ON(!max_hp);
1500
1501 TRACE_CUR("queue %d: stealing %s/%d from queue %d\n",
1502 kfmlp_get_idx(sem, my_queue),
1503 max_hp->comm, max_hp->pid,
1504 kfmlp_get_idx(sem, my_queue));
1505
1506 my_queue->hp_waiter = kfmlp_find_hp_waiter(my_queue, max_hp);
1507
1508 /*
1509 if(my_queue->hp_waiter)
1510 TRACE_CUR("queue %d: new hp_waiter is %s/%d\n",
1511 kfmlp_get_idx(sem, my_queue),
1512 my_queue->hp_waiter->comm,
1513 my_queue->hp_waiter->pid);
1514 else
1515 TRACE_CUR("queue %d: new hp_waiter is %p\n",
1516 kfmlp_get_idx(sem, my_queue), NULL);
1517 */
1518
1519 cluster = task_cpu_cluster(max_hp);
1520
1521 raw_spin_lock(&cluster->cluster_lock);
1522
1523 /*
1524 if(my_queue->owner)
1525 TRACE_CUR("queue %d: owner is %s/%d\n",
1526 kfmlp_get_idx(sem, my_queue),
1527 my_queue->owner->comm,
1528 my_queue->owner->pid);
1529 else
1530 TRACE_CUR("queue %d: owner is %p\n",
1531 kfmlp_get_idx(sem, my_queue),
1532 NULL);
1533 */
1534
1535 if(tsk_rt(my_queue->owner)->inh_task == max_hp)
1536 {
1537 __clear_priority_inheritance(my_queue->owner);
1538 if(my_queue->hp_waiter != NULL)
1539 {
1540 __set_priority_inheritance(my_queue->owner, my_queue->hp_waiter);
1541 }
1542 }
1543 raw_spin_unlock(&cluster->cluster_lock);
1544
1545 list_for_each(pos, &my_queue->wait.task_list)
1546 {
1547 queued = (struct task_struct*) list_entry(pos, wait_queue_t,
1548 task_list)->private;
1549 /* Compare task prios, find high prio task. */
1550 if (queued == max_hp)
1551 {
1552 /*
1553 TRACE_CUR("queue %d: found entry in wait queue. REMOVING!\n",
1554 kfmlp_get_idx(sem, my_queue));
1555 */
1556 __remove_wait_queue(&my_queue->wait,
1557 list_entry(pos, wait_queue_t, task_list));
1558 break;
1559 }
1560 }
1561 --(my_queue->count);
1562 }
1563
1564 return(max_hp);
1565}
1566
1567int cedf_kfmlp_lock(struct litmus_lock* l)
1568{
1569 struct task_struct* t = current;
1570 struct kfmlp_semaphore *sem = kfmlp_from_lock(l);
1571 struct kfmlp_queue* my_queue;
1572 wait_queue_t wait;
1573 unsigned long flags;
1574
1575 if (!is_realtime(t))
1576 return -EPERM;
1577
1578 spin_lock_irqsave(&sem->lock, flags);
1579
1580 my_queue = sem->shortest_queue;
1581
1582 if (my_queue->owner) {
1583 /* resource is not free => must suspend and wait */
1584 TRACE_CUR("queue %d: Resource is not free => must suspend and wait.\n",
1585 kfmlp_get_idx(sem, my_queue));
1586
1587 init_waitqueue_entry(&wait, t);
1588
1589 /* FIXME: interruptible would be nice some day */
1590 set_task_state(t, TASK_UNINTERRUPTIBLE);
1591
1592 __add_wait_queue_tail_exclusive(&my_queue->wait, &wait);
1593
1594 /* check if we need to activate priority inheritance */
1595 if (edf_higher_prio(t, my_queue->hp_waiter))
1596 {
1597 my_queue->hp_waiter = t;
1598 if (edf_higher_prio(t, my_queue->owner))
1599 {
1600 set_priority_inheritance(my_queue->owner, my_queue->hp_waiter);
1601 }
1602 }
1603
1604 ++(my_queue->count);
1605 sem->shortest_queue = kfmlp_find_shortest(sem, my_queue);
1606
1607 /* release lock before sleeping */
1608 spin_unlock_irqrestore(&sem->lock, flags);
1609
1610 /* We depend on the FIFO order. Thus, we don't need to recheck
1611 * when we wake up; we are guaranteed to have the lock since
1612 * there is only one wake up per release (or steal).
1613 */
1614 schedule();
1615
1616
1617 if(my_queue->owner == t)
1618 {
1619 TRACE_CUR("queue %d: acquired through waiting\n",
1620 kfmlp_get_idx(sem, my_queue));
1621 }
1622 else
1623 {
1624 /* this case may happen if our wait entry was stolen
1625 between queues. record where we went.*/
1626 my_queue = kfmlp_get_queue(sem, t);
1627 BUG_ON(!my_queue);
1628 TRACE_CUR("queue %d: acquired through stealing\n",
1629 kfmlp_get_idx(sem, my_queue));
1630 }
1631 }
1632 else
1633 {
1634 TRACE_CUR("queue %d: acquired immediately\n",
1635 kfmlp_get_idx(sem, my_queue));
1636
1637 my_queue->owner = t;
1638
1639 ++(my_queue->count);
1640 sem->shortest_queue = kfmlp_find_shortest(sem, my_queue);
1641
1642 spin_unlock_irqrestore(&sem->lock, flags);
1643 }
1644
1645 return kfmlp_get_idx(sem, my_queue);
1646}
1647
1648int cedf_kfmlp_unlock(struct litmus_lock* l)
1649{
1650 struct task_struct *t = current, *next;
1651 struct kfmlp_semaphore *sem = kfmlp_from_lock(l);
1652 struct kfmlp_queue *my_queue;
1653 unsigned long flags;
1654 int err = 0;
1655
1656 spin_lock_irqsave(&sem->lock, flags);
1657
1658 my_queue = kfmlp_get_queue(sem, t);
1659
1660 if (!my_queue) {
1661 err = -EINVAL;
1662 goto out;
1663 }
1664
1665 /* check if there are jobs waiting for this resource */
1666 next = __waitqueue_remove_first(&my_queue->wait);
1667 if (next) {
1668 /*
1669 TRACE_CUR("queue %d: ASSIGNING %s/%d as owner - next\n",
1670 kfmlp_get_idx(sem, my_queue),
1671 next->comm, next->pid);
1672 */
1673 /* next becomes the resouce holder */
1674 my_queue->owner = next;
1675
1676 --(my_queue->count);
1677 if(my_queue->count < sem->shortest_queue->count)
1678 {
1679 sem->shortest_queue = my_queue;
1680 }
1681
1682 TRACE_CUR("queue %d: lock ownership passed to %s/%d\n",
1683 kfmlp_get_idx(sem, my_queue), next->comm, next->pid);
1684
1685 /* determine new hp_waiter if necessary */
1686 if (next == my_queue->hp_waiter) {
1687 TRACE_TASK(next, "was highest-prio waiter\n");
1688 /* next has the highest priority --- it doesn't need to
1689 * inherit. However, we need to make sure that the
1690 * next-highest priority in the queue is reflected in
1691 * hp_waiter. */
1692 my_queue->hp_waiter = kfmlp_find_hp_waiter(my_queue, next);
1693 if (my_queue->hp_waiter)
1694 TRACE_TASK(my_queue->hp_waiter, "queue %d: is new highest-prio waiter\n", kfmlp_get_idx(sem, my_queue));
1695 else
1696 TRACE("queue %d: no further waiters\n", kfmlp_get_idx(sem, my_queue));
1697 } else {
1698 /* Well, if next is not the highest-priority waiter,
1699 * then it ought to inherit the highest-priority
1700 * waiter's priority. */
1701 set_priority_inheritance(next, my_queue->hp_waiter);
1702 }
1703
1704 /* wake up next */
1705 wake_up_process(next);
1706 }
1707 else
1708 {
1709 TRACE_CUR("queue %d: looking to steal someone...\n", kfmlp_get_idx(sem, my_queue));
1710
1711 next = kfmlp_remove_hp_waiter(sem); /* returns NULL if nothing to steal */
1712
1713 /*
1714 if(next)
1715 TRACE_CUR("queue %d: ASSIGNING %s/%d as owner - steal\n",
1716 kfmlp_get_idx(sem, my_queue),
1717 next->comm, next->pid);
1718 */
1719
1720 my_queue->owner = next;
1721
1722 if(next)
1723 {
1724 TRACE_CUR("queue %d: lock ownership passed to %s/%d (which was stolen)\n",
1725 kfmlp_get_idx(sem, my_queue),
1726 next->comm, next->pid);
1727
1728 /* wake up next */
1729 wake_up_process(next);
1730 }
1731 else
1732 {
1733 TRACE_CUR("queue %d: no one to steal.\n", kfmlp_get_idx(sem, my_queue));
1734
1735 --(my_queue->count);
1736 if(my_queue->count < sem->shortest_queue->count)
1737 {
1738 sem->shortest_queue = my_queue;
1739 }
1740 }
1741 }
1742
1743 /* we lose the benefit of priority inheritance (if any) */
1744 if (tsk_rt(t)->inh_task)
1745 clear_priority_inheritance(t);
1746
1747out:
1748 spin_unlock_irqrestore(&sem->lock, flags);
1749
1750 return err;
1751}
1752
1753int cedf_kfmlp_close(struct litmus_lock* l)
1754{
1755 struct task_struct *t = current;
1756 struct kfmlp_semaphore *sem = kfmlp_from_lock(l);
1757 struct kfmlp_queue *my_queue;
1758 unsigned long flags;
1759
1760 int owner;
1761
1762 spin_lock_irqsave(&sem->lock, flags);
1763
1764 my_queue = kfmlp_get_queue(sem, t);
1765 owner = (my_queue) ? (my_queue->owner == t) : 0;
1766
1767 spin_unlock_irqrestore(&sem->lock, flags);
1768
1769 if (owner)
1770 cedf_kfmlp_unlock(l);
1771
1772 return 0;
1773}
1774
1775void cedf_kfmlp_free(struct litmus_lock* l)
1776{
1777 struct kfmlp_semaphore *sem = kfmlp_from_lock(l);
1778 kfree(sem->queues);
1779 kfree(sem);
1780}
1781
1782static struct litmus_lock_ops cedf_kfmlp_lock_ops = {
1783 .close = cedf_kfmlp_close,
1784 .lock = cedf_kfmlp_lock,
1785 .unlock = cedf_kfmlp_unlock,
1786 .deallocate = cedf_kfmlp_free,
1787};
1788
1789static struct litmus_lock* cedf_new_kfmlp(void* __user arg, int* ret_code)
1790{
1791 struct kfmlp_semaphore* sem;
1792 int num_resources = 0;
1793 int i;
1794
1795 if(!access_ok(VERIFY_READ, arg, sizeof(num_resources)))
1796 {
1797 *ret_code = -EINVAL;
1798 return(NULL);
1799 }
1800 if(__copy_from_user(&num_resources, arg, sizeof(num_resources)))
1801 {
1802 *ret_code = -EINVAL;
1803 return(NULL);
1804 }
1805 if(num_resources < 1)
1806 {
1807 *ret_code = -EINVAL;
1808 return(NULL);
1809 }
1810
1811 sem = kmalloc(sizeof(*sem), GFP_KERNEL);
1812 if(!sem)
1813 {
1814 *ret_code = -ENOMEM;
1815 return NULL;
1816 }
1817
1818 sem->queues = kmalloc(sizeof(struct kfmlp_queue)*num_resources, GFP_KERNEL);
1819 if(!sem->queues)
1820 {
1821 kfree(sem);
1822 *ret_code = -ENOMEM;
1823 return NULL;
1824 }
1825
1826 sem->litmus_lock.ops = &cedf_kfmlp_lock_ops;
1827 spin_lock_init(&sem->lock);
1828 sem->num_resources = num_resources;
1829
1830 for(i = 0; i < num_resources; ++i)
1831 {
1832 sem->queues[i].owner = NULL;
1833 sem->queues[i].hp_waiter = NULL;
1834 init_waitqueue_head(&sem->queues[i].wait);
1835 sem->queues[i].count = 0;
1836 }
1837
1838 sem->shortest_queue = &sem->queues[0];
1839
1840 *ret_code = 0;
1841 return &sem->litmus_lock;
1842}
1843
1844
1845/* **** lock constructor **** */
1846
1847static long cedf_allocate_lock(struct litmus_lock **lock, int type,
1848 void* __user arg)
1849{
1850 int err = -ENXIO;
1851
1852 /* C-EDF currently only supports the FMLP for global resources
1853 WITHIN a given cluster. DO NOT USE CROSS-CLUSTER! */
1854 switch (type) {
1855 case KFMLP_SEM:
1856 *lock = cedf_new_kfmlp(arg, &err);
1857 break;
1858 };
1859
1860 return err;
1861}
1862
1863#endif // CONFIG_LITMUS_LOCKING
1864
1865
1866
1867
1868
1869
665/* total number of cluster */ 1870/* total number of cluster */
666static int num_clusters; 1871static int num_clusters;
667/* we do not support cluster of different sizes */ 1872/* we do not support cluster of different sizes */
@@ -755,6 +1960,13 @@ static long cedf_activate_plugin(void)
755 bheap_init(&(cedf[i].cpu_heap)); 1960 bheap_init(&(cedf[i].cpu_heap));
756 edf_domain_init(&(cedf[i].domain), NULL, cedf_release_jobs); 1961 edf_domain_init(&(cedf[i].domain), NULL, cedf_release_jobs);
757 1962
1963
1964#ifdef CONFIG_LITMUS_PAI_SOFTIRQD
1965 cedf[i].pending_tasklets.head = NULL;
1966 cedf[i].pending_tasklets.tail = &(cedf[i].pending_tasklets.head);
1967#endif
1968
1969
758 if(!zalloc_cpumask_var(&cedf[i].cpu_map, GFP_ATOMIC)) 1970 if(!zalloc_cpumask_var(&cedf[i].cpu_map, GFP_ATOMIC))
759 return -ENOMEM; 1971 return -ENOMEM;
760#ifdef CONFIG_RELEASE_MASTER 1972#ifdef CONFIG_RELEASE_MASTER
@@ -812,6 +2024,40 @@ static long cedf_activate_plugin(void)
812 break; 2024 break;
813 } 2025 }
814 } 2026 }
2027
2028#ifdef CONFIG_LITMUS_SOFTIRQD
2029 {
2030 /* distribute the daemons evenly across the clusters. */
2031 int* affinity = kmalloc(NR_LITMUS_SOFTIRQD * sizeof(int), GFP_ATOMIC);
2032 int num_daemons_per_cluster = NR_LITMUS_SOFTIRQD / num_clusters;
2033 int left_over = NR_LITMUS_SOFTIRQD % num_clusters;
2034
2035 int daemon = 0;
2036 for(i = 0; i < num_clusters; ++i)
2037 {
2038 int num_on_this_cluster = num_daemons_per_cluster;
2039 if(left_over)
2040 {
2041 ++num_on_this_cluster;
2042 --left_over;
2043 }
2044
2045 for(j = 0; j < num_on_this_cluster; ++j)
2046 {
2047 // first CPU of this cluster
2048 affinity[daemon++] = i*cluster_size;
2049 }
2050 }
2051
2052 spawn_klitirqd(affinity);
2053
2054 kfree(affinity);
2055 }
2056#endif
2057
2058#ifdef CONFIG_LITMUS_NVIDIA
2059 init_nvidia_info();
2060#endif
815 2061
816 free_cpumask_var(mask); 2062 free_cpumask_var(mask);
817 clusters_allocated = 1; 2063 clusters_allocated = 1;
@@ -831,6 +2077,19 @@ static struct sched_plugin cedf_plugin __cacheline_aligned_in_smp = {
831 .task_block = cedf_task_block, 2077 .task_block = cedf_task_block,
832 .admit_task = cedf_admit_task, 2078 .admit_task = cedf_admit_task,
833 .activate_plugin = cedf_activate_plugin, 2079 .activate_plugin = cedf_activate_plugin,
2080#ifdef CONFIG_LITMUS_LOCKING
2081 .allocate_lock = cedf_allocate_lock,
2082 .set_prio_inh = set_priority_inheritance,
2083 .clear_prio_inh = clear_priority_inheritance,
2084#endif
2085#ifdef CONFIG_LITMUS_SOFTIRQD
2086 .set_prio_inh_klitirqd = set_priority_inheritance_klitirqd,
2087 .clear_prio_inh_klitirqd = clear_priority_inheritance_klitirqd,
2088#endif
2089#ifdef CONFIG_LITMUS_PAI_SOFTIRQD
2090 .enqueue_pai_tasklet = enqueue_pai_tasklet,
2091 .run_tasklets = run_tasklets,
2092#endif
834}; 2093};
835 2094
836static struct proc_dir_entry *cluster_file = NULL, *cedf_dir = NULL; 2095static struct proc_dir_entry *cluster_file = NULL, *cedf_dir = NULL;
generated by cgit v1.2.2 (git 2.25.0) at 2024-11-09 14:53:14 -0500