aboutsummaryrefslogtreecommitdiffstats
path: root/litmus/sched_gfl_split_namechange.c
diff options
context:
space:
mode:
Diffstat (limited to 'litmus/sched_gfl_split_namechange.c')
-rw-r--r--litmus/sched_gfl_split_namechange.c1149
1 files changed, 1149 insertions, 0 deletions
diff --git a/litmus/sched_gfl_split_namechange.c b/litmus/sched_gfl_split_namechange.c
new file mode 100644
index 000000000000..c154b115a00e
--- /dev/null
+++ b/litmus/sched_gfl_split_namechange.c
@@ -0,0 +1,1149 @@
1/*
2 * litmus/sched_gfl_split.c
3 *
4 * Implementation of the G-FL with job splitting. See the Erickson/Anderson
5 * paper at ECRTS 2012 for a description of G-FL.
6 *
7 * This plugin is a modified version of the prior GSN-EDF-split plugin in
8 * litmus/sched_gsn_edf_split.c. Job splitting works the same way as in that
9 * plugin. The subjob "deadlines" (really priorities) are computed according
10 * to G-FL with respect to the post-split (smaller) jobs.
11 *
12 */
13
14#include <linux/spinlock.h>
15#include <linux/percpu.h>
16#include <linux/sched.h>
17#include <linux/slab.h>
18
19#include <litmus/litmus.h>
20#include <litmus/jobs.h>
21#include <litmus/sched_plugin.h>
22#include <litmus/edf_split_common.h>
23#include <litmus/sched_trace.h>
24#include <litmus/trace.h>
25
26#include <litmus/preempt.h>
27
28#include <litmus/bheap.h>
29
30#ifdef CONFIG_SCHED_CPU_AFFINITY
31#include <litmus/affinity.h>
32#endif
33
34#include <linux/module.h>
35
36/* cpu_entry_t - maintain the linked and scheduled state
37 */
38typedef struct {
39 int cpu;
40 struct task_struct* linked; /* only RT tasks */
41 struct task_struct* scheduled; /* only RT tasks */
42 struct bheap_node* hn;
43 struct hrtimer split_timer;
44 int timer_armed;
45} cpu_entry_t;
46DEFINE_PER_CPU(cpu_entry_t, gsnedf_cpu_entries);
47
48cpu_entry_t* gsnedf_cpus[NR_CPUS];
49
50/* the cpus queue themselves according to priority in here */
51static struct bheap_node gsnedf_heap_node[NR_CPUS];
52static struct bheap gsnedf_cpu_heap;
53
54static rt_domain_t gsnedf;
55#define gsnedf_lock (gsnedf.ready_lock)
56
57inline static int get_slice_num(struct task_struct* t)
58{
59 int basic = ((t->rt_param.job_params.exec_time *
60 t->rt_param.task_params.split) /
61 t->rt_param.task_params.exec_cost) + 1;
62 if (basic <= t->rt_param.task_params.split){
63 return basic;
64 }
65 else{
66 /*Since we don't police budget, just leave where it's at.*/
67 return t->rt_param.task_params.split;
68 }
69}
70
71/* Returns the appropriate subjob deadline.*/
72inline static lt_t get_proper_deadline(struct task_struct* t)
73{
74 unsigned int num_cpus = num_online_cpus();
75 return t->rt_param.job_params.release +
76 ((t->rt_param.task_params.period * get_slice_num(t))
77 / t->rt_param.task_params.split)
78 /* G-FL correction */
79 - (((num_cpus - 1) * t->rt_param.task_params.exec_cost)
80 / (num_cpus * t->rt_param.task_params.split));
81}
82
83/* Tells us if the current deadline is too small.*/
84inline static int needs_deadline_move(struct task_struct* t)
85{
86 BUG_ON(get_proper_deadline(t) < t->rt_param.job_params.subjob_deadline);
87#ifdef CONFIG_LITMUS_LOCKING
88 return !is_in_crit_section(t) &&
89 (get_proper_deadline(t) !=
90 tsk_rt(t)->job_params.subjob_deadline);
91#else
92 return get_proper_deadline(t) != tsk_rt(t)->job_params.subjob_deadline;
93#endif
94}
95
96/*Returns execution time until the next deadline move.
97 * 0 means the task has no more deadline moves
98 */
99inline static lt_t time_to_next_move(struct task_struct* t)
100{
101 if (get_slice_num(t) == t->rt_param.task_params.split){
102 return 0;
103 }
104 /* +1 upper bounds ceiling, since integer division is floor*/
105 return ((get_slice_num(t) * t->rt_param.task_params.exec_cost)
106 / t->rt_param.task_params.split) + 1
107 - t->rt_param.job_params.exec_time;
108}
109
110/* Timer stuff - similar to budget.c. */
111static enum hrtimer_restart on_split_timeout(struct hrtimer *timer)
112{
113 cpu_entry_t* st = container_of(timer,
114 cpu_entry_t,
115 split_timer);
116
117 unsigned long flags;
118
119 local_irq_save(flags);
120 TRACE("split timer fired.\n");
121 st->timer_armed = 0;
122 /* Activate scheduler */
123 litmus_reschedule_local();
124 local_irq_restore(flags);
125
126 return HRTIMER_NORESTART;
127}
128
129static void cancel_split_timer(cpu_entry_t* ce)
130{
131 int ret;
132
133 TRACE("cancelling split time.\n");
134
135 /* Since interrupts are disabled and et->timer_armed is only
136 * modified locally, we do not need any locks.
137 */
138
139 if (ce->timer_armed) {
140 ret = hrtimer_try_to_cancel(&ce->split_timer);
141 /* Should never be inactive. */
142 BUG_ON(ret == 0);
143 /* Should never be running concurrently.*/
144 BUG_ON(ret == -1);
145
146 ce->timer_armed = 0;
147 }
148}
149
150/* assumes called with IRQs off */
151static void arm_split_timer(cpu_entry_t *ce,
152 struct task_struct* t)
153{
154 lt_t when_to_fire;
155 lt_t time_to_move;
156 TRACE_TASK(t, "arming split timer.\n");
157
158 /* __hrtimer_start_range_ns() cancels the timer
159 * anyway, so we don't have to check whether it is still armed */
160
161 /*We won't do any new deadline moves if the budget has been exhausted*/
162 if (likely(!is_np(t) && (time_to_move = time_to_next_move(t)))) {
163 when_to_fire = litmus_clock() + time_to_move;
164 TRACE_TASK(t, "actually arming for %llu into the future\n",
165 time_to_move);
166 __hrtimer_start_range_ns(&ce->split_timer,
167 ns_to_ktime(when_to_fire),
168 0 /* delta */,
169 HRTIMER_MODE_ABS_PINNED,
170 0 /* no wakeup */);
171 ce->timer_armed = 1;
172 }
173}
174
175/* Uncomment this if you want to see all scheduling decisions in the
176 * TRACE() log.
177#define WANT_ALL_SCHED_EVENTS
178 */
179
180static int cpu_lower_prio(struct bheap_node *_a, struct bheap_node *_b)
181{
182 cpu_entry_t *a, *b;
183 a = _a->value;
184 b = _b->value;
185 /* Note that a and b are inverted: we want the lowest-priority CPU at
186 * the top of the heap.
187 */
188 return edf_split_higher_prio(b->linked, a->linked);
189}
190
191/* update_cpu_position - Move the cpu entry to the correct place to maintain
192 * order in the cpu queue. Caller must hold gsnedf lock.
193 */
194static void update_cpu_position(cpu_entry_t *entry)
195{
196 if (likely(bheap_node_in_heap(entry->hn)))
197 bheap_delete(cpu_lower_prio, &gsnedf_cpu_heap, entry->hn);
198 bheap_insert(cpu_lower_prio, &gsnedf_cpu_heap, entry->hn);
199}
200
201/* caller must hold gsnedf lock */
202static cpu_entry_t* lowest_prio_cpu(void)
203{
204 struct bheap_node* hn;
205 hn = bheap_peek(cpu_lower_prio, &gsnedf_cpu_heap);
206 return hn->value;
207}
208
209
210/* link_task_to_cpu - Update the link of a CPU.
211 * Handles the case where the to-be-linked task is already
212 * scheduled on a different CPU.
213 */
214static noinline void link_task_to_cpu(struct task_struct* linked,
215 cpu_entry_t *entry)
216{
217 cpu_entry_t *sched;
218 struct task_struct* tmp;
219 int on_cpu;
220
221 BUG_ON(linked && !is_realtime(linked));
222
223 /* Currently linked task is set to be unlinked. */
224 if (entry->linked) {
225 entry->linked->rt_param.linked_on = NO_CPU;
226 }
227
228 /* Link new task to CPU. */
229 if (linked) {
230 set_rt_flags(linked, RT_F_RUNNING);
231 /* handle task is already scheduled somewhere! */
232 on_cpu = linked->rt_param.scheduled_on;
233 if (on_cpu != NO_CPU) {
234 sched = &per_cpu(gsnedf_cpu_entries, on_cpu);
235 /* this should only happen if not linked already */
236 BUG_ON(sched->linked == linked);
237
238 /* If we are already scheduled on the CPU to which we
239 * wanted to link, we don't need to do the swap --
240 * we just link ourselves to the CPU and depend on
241 * the caller to get things right.
242 */
243 if (entry != sched) {
244 TRACE_TASK(linked,
245 "already scheduled on %d, updating link.\n",
246 sched->cpu);
247 tmp = sched->linked;
248 linked->rt_param.linked_on = sched->cpu;
249 sched->linked = linked;
250 update_cpu_position(sched);
251 linked = tmp;
252 }
253 }
254 if (linked) /* might be NULL due to swap */
255 linked->rt_param.linked_on = entry->cpu;
256 }
257 entry->linked = linked;
258#ifdef WANT_ALL_SCHED_EVENTS
259 if (linked)
260 TRACE_TASK(linked, "linked to %d.\n", entry->cpu);
261 else
262 TRACE("NULL linked to %d.\n", entry->cpu);
263#endif
264 update_cpu_position(entry);
265}
266
267/* unlink - Make sure a task is not linked any longer to an entry
268 * where it was linked before. Must hold gsnedf_lock.
269 */
270static noinline void unlink(struct task_struct* t)
271{
272 cpu_entry_t *entry;
273
274 if (t->rt_param.linked_on != NO_CPU) {
275 /* unlink */
276 entry = &per_cpu(gsnedf_cpu_entries, t->rt_param.linked_on);
277 t->rt_param.linked_on = NO_CPU;
278 link_task_to_cpu(NULL, entry);
279 } else if (is_queued(t)) {
280 /* This is an interesting situation: t is scheduled,
281 * but was just recently unlinked. It cannot be
282 * linked anywhere else (because then it would have
283 * been relinked to this CPU), thus it must be in some
284 * queue. We must remove it from the list in this
285 * case.
286 */
287 remove(&gsnedf, t);
288 }
289}
290
291
292/* preempt - force a CPU to reschedule
293 */
294static void preempt(cpu_entry_t *entry)
295{
296 preempt_if_preemptable(entry->scheduled, entry->cpu);
297}
298
299/* requeue - Put an unlinked task into gsn-edf domain.
300 * Caller must hold gsnedf_lock.
301 */
302static noinline void requeue(struct task_struct* task)
303{
304 BUG_ON(!task);
305 /* sanity check before insertion */
306 BUG_ON(is_queued(task));
307
308 if (is_released(task, litmus_clock()))
309 __add_ready(&gsnedf, task);
310 else {
311 /* it has got to wait */
312 add_release(&gsnedf, task);
313 }
314}
315
316#ifdef CONFIG_SCHED_CPU_AFFINITY
317static cpu_entry_t* gsnedf_get_nearest_available_cpu(cpu_entry_t *start)
318{
319 cpu_entry_t *affinity;
320
321 get_nearest_available_cpu(affinity, start, gsnedf_cpu_entries,
322#ifdef CONFIG_RELEASE_MASTER
323 gsnedf.release_master
324#else
325 NO_CPU
326#endif
327 );
328
329 return(affinity);
330}
331#endif
332
333/* check for any necessary preemptions */
334static void check_for_preemptions(void)
335{
336 struct task_struct *task;
337 cpu_entry_t *last;
338
339 for (last = lowest_prio_cpu();
340 edf_split_preemption_needed(&gsnedf, last->linked);
341 last = lowest_prio_cpu()) {
342 /* preemption necessary */
343 task = __take_ready(&gsnedf);
344 TRACE("check_for_preemptions: attempting to link task %d to %d\n",
345 task->pid, last->cpu);
346
347#ifdef CONFIG_SCHED_CPU_AFFINITY
348 {
349 cpu_entry_t *affinity =
350 gsnedf_get_nearest_available_cpu(
351 &per_cpu(gsnedf_cpu_entries,
352 task_cpu(task)));
353 if (affinity)
354 last = affinity;
355 else if (last->linked)
356 requeue(last->linked);
357 }
358#else
359 if (last->linked)
360 requeue(last->linked);
361#endif
362
363 link_task_to_cpu(task, last);
364 preempt(last);
365 }
366}
367
368/* gsnedf_job_arrival: task is either resumed or released */
369static noinline void gsnedf_job_arrival(struct task_struct* task)
370{
371 BUG_ON(!task);
372
373 requeue(task);
374 check_for_preemptions();
375}
376
377static void gsnedf_release_jobs(rt_domain_t* rt, struct bheap* tasks)
378{
379 unsigned long flags;
380
381 raw_spin_lock_irqsave(&gsnedf_lock, flags);
382
383 __merge_ready(rt, tasks);
384 check_for_preemptions();
385
386 raw_spin_unlock_irqrestore(&gsnedf_lock, flags);
387}
388
389/* caller holds gsnedf_lock */
390static noinline void job_completion(struct task_struct *t, int forced)
391{
392 BUG_ON(!t);
393
394 sched_trace_task_completion(t, forced);
395
396 TRACE_TASK(t, "job_completion().\n");
397
398 /* set flags */
399 set_rt_flags(t, RT_F_SLEEP);
400 /* prepare for next period */
401 /* prepare_for_next_period assumes implicit deadlines and no splitting,
402 * so we call it with the job deadline it expects.
403 */
404 t->rt_param.job_params.deadline = t->rt_param.job_params.release +
405 t->rt_param.task_params.period;
406 prepare_for_next_period(t);
407 /* We now set the subjob deadline to what it should be for scheduling
408 * priority.
409 */
410 t->rt_param.job_params.subjob_deadline = get_proper_deadline(t);
411 if (is_released(t, litmus_clock()))
412 sched_trace_task_release(t);
413 /* unlink */
414 unlink(t);
415 /* requeue
416 * But don't requeue a blocking task. */
417 if (is_running(t))
418 gsnedf_job_arrival(t);
419}
420
421static void move_deadline(struct task_struct *t)
422{
423 tsk_rt(t)->job_params.subjob_deadline = get_proper_deadline(t);
424 /* Check if rescheduling needed with lower priority. */
425 unlink(t);
426 gsnedf_job_arrival(t);
427}
428
429/* gsnedf_tick - this function is called for every local timer
430 * interrupt.
431 *
432 * checks whether the current task has expired and checks
433 * whether we need to preempt it if it has not expired
434 */
435static void gsnedf_tick(struct task_struct* t)
436{
437 if (is_realtime(t) && budget_enforced(t) && budget_exhausted(t)) {
438 if (!is_np(t)) {
439 /* np tasks will be preempted when they become
440 * preemptable again
441 */
442 litmus_reschedule_local();
443 TRACE("gsnedf_scheduler_tick: "
444 "%d is preemptable "
445 " => FORCE_RESCHED\n", t->pid);
446 } else if (is_user_np(t)) {
447 TRACE("gsnedf_scheduler_tick: "
448 "%d is non-preemptable, "
449 "preemption delayed.\n", t->pid);
450 request_exit_np(t);
451 }
452 }
453}
454
455/* Getting schedule() right is a bit tricky. schedule() may not make any
456 * assumptions on the state of the current task since it may be called for a
457 * number of reasons. The reasons include a scheduler_tick() determined that it
458 * was necessary, because sys_exit_np() was called, because some Linux
459 * subsystem determined so, or even (in the worst case) because there is a bug
460 * hidden somewhere. Thus, we must take extreme care to determine what the
461 * current state is.
462 *
463 * The CPU could currently be scheduling a task (or not), be linked (or not).
464 *
465 * The following assertions for the scheduled task could hold:
466 *
467 * - !is_running(scheduled) // the job blocks
468 * - scheduled->timeslice == 0 // the job completed (forcefully)
469 * - get_rt_flag() == RT_F_SLEEP // the job completed (by syscall)
470 * - linked != scheduled // we need to reschedule (for any reason)
471 * - is_np(scheduled) // rescheduling must be delayed,
472 * sys_exit_np must be requested
473 *
474 * Any of these can occur together.
475 */
476static struct task_struct* gsnedf_schedule(struct task_struct * prev)
477{
478 cpu_entry_t* entry = &__get_cpu_var(gsnedf_cpu_entries);
479 int out_of_time, sleep, preempt, np, exists, blocks, needs_move;
480 struct task_struct* next = NULL;
481
482#ifdef CONFIG_RELEASE_MASTER
483 /* Bail out early if we are the release master.
484 * The release master never schedules any real-time tasks.
485 */
486 if (unlikely(gsnedf.release_master == entry->cpu)) {
487 sched_state_task_picked();
488 return NULL;
489 }
490#endif
491
492 raw_spin_lock(&gsnedf_lock);
493
494 /* sanity checking */
495 BUG_ON(entry->scheduled && entry->scheduled != prev);
496 BUG_ON(entry->scheduled && !is_realtime(prev));
497 BUG_ON(is_realtime(prev) && !entry->scheduled);
498
499 /* (0) Determine state */
500 exists = entry->scheduled != NULL;
501 blocks = exists && !is_running(entry->scheduled);
502 out_of_time = exists &&
503 budget_enforced(entry->scheduled) &&
504 budget_exhausted(entry->scheduled);
505 needs_move = exists && needs_deadline_move(entry->scheduled);
506 np = exists && is_np(entry->scheduled);
507 sleep = exists && get_rt_flags(entry->scheduled) == RT_F_SLEEP;
508 preempt = entry->scheduled != entry->linked;
509
510#ifdef WANT_ALL_SCHED_EVENTS
511 TRACE_TASK(prev, "invoked gsnedf_schedule.\n");
512#endif
513
514 if (exists)
515 TRACE_TASK(prev,
516 "blocks:%d out_of_time:%d needs_move:%d np:%d"
517 " sleep:%d preempt:%d state:%d sig:%d\n",
518 blocks, out_of_time, needs_move, np, sleep, preempt,
519 prev->state, signal_pending(prev));
520 if (entry->linked && preempt)
521 TRACE_TASK(prev, "will be preempted by %s/%d\n",
522 entry->linked->comm, entry->linked->pid);
523
524
525 /* If a task blocks we have no choice but to reschedule.
526 */
527 if (blocks)
528 unlink(entry->scheduled);
529
530 /* Request a sys_exit_np() call if we would like to preempt but cannot.
531 * We need to make sure to update the link structure anyway in case
532 * that we are still linked. Multiple calls to request_exit_np() don't
533 * hurt.
534 *
535 * Job deadline moves handled similarly
536 */
537 if (np && (out_of_time || preempt || sleep)) {
538 unlink(entry->scheduled);
539 request_exit_np(entry->scheduled);
540 }
541 else if (np && needs_move) {
542 move_deadline(entry->scheduled);
543 }
544
545 /* Any task that is preemptable and either exhausts its execution
546 * budget or wants to sleep completes. We may have to reschedule after
547 * this. Don't do a job completion if we block (can't have timers running
548 * for blocked jobs). Preemption go first for the same reason.
549 */
550 if (!np && (out_of_time || sleep) && !blocks && !preempt)
551 job_completion(entry->scheduled, !sleep);
552 else if (!np && needs_move && !blocks && !preempt) {
553 move_deadline(entry->scheduled);
554 }
555
556 /* Link pending task if we became unlinked.
557 */
558 if (!entry->linked)
559 link_task_to_cpu(__take_ready(&gsnedf), entry);
560
561 /* The final scheduling decision. Do we need to switch for some reason?
562 * If linked is different from scheduled, then select linked as next.
563 */
564 if ((!np || blocks) &&
565 entry->linked != entry->scheduled) {
566 /* Schedule a linked job? */
567 if (entry->linked) {
568 entry->linked->rt_param.scheduled_on = entry->cpu;
569 next = entry->linked;
570 TRACE_TASK(next, "scheduled_on = P%d\n", smp_processor_id());
571 }
572 if (entry->scheduled) {
573 /* not gonna be scheduled soon */
574 entry->scheduled->rt_param.scheduled_on = NO_CPU;
575 TRACE_TASK(entry->scheduled, "scheduled_on = NO_CPU\n");
576 }
577 } else
578 /* Only override Linux scheduler if we have a real-time task
579 * scheduled that needs to continue.
580 */
581 if (exists)
582 next = prev;
583
584 sched_state_task_picked();
585
586 raw_spin_unlock(&gsnedf_lock);
587
588 if (next) {
589 arm_split_timer(entry, next);
590 }
591 else if (entry->timer_armed) {
592 cancel_split_timer(entry);
593 }
594
595#ifdef WANT_ALL_SCHED_EVENTS
596 TRACE("gsnedf_lock released, next=0x%p\n", next);
597
598 if (next)
599 TRACE_TASK(next, "scheduled at %llu\n", litmus_clock());
600 else if (exists && !next)
601 TRACE("becomes idle at %llu.\n", litmus_clock());
602#endif
603
604
605 return next;
606}
607
608
609/* _finish_switch - we just finished the switch away from prev
610 */
611static void gsnedf_finish_switch(struct task_struct *prev)
612{
613 cpu_entry_t* entry = &__get_cpu_var(gsnedf_cpu_entries);
614
615 entry->scheduled = is_realtime(current) ? current : NULL;
616#ifdef WANT_ALL_SCHED_EVENTS
617 TRACE_TASK(prev, "switched away from\n");
618#endif
619}
620
621static void gsnedf_release_at(struct task_struct *t, lt_t start)
622{
623 t->rt_param.job_params.deadline = start;
624 prepare_for_next_period(t);
625 t->rt_param.job_params.subjob_deadline = get_proper_deadline(t);
626 set_rt_flags(t, RT_F_RUNNING);
627}
628
629/* Prepare a task for running in RT mode
630 */
631static void gsnedf_task_new(struct task_struct * t, int on_rq, int running)
632{
633 unsigned long flags;
634 cpu_entry_t* entry;
635
636 TRACE("gsn edf: task new %d\n", t->pid);
637
638 raw_spin_lock_irqsave(&gsnedf_lock, flags);
639
640 /* setup job params */
641 gsnedf_release_at(t, litmus_clock());
642
643 if (running) {
644 entry = &per_cpu(gsnedf_cpu_entries, task_cpu(t));
645 BUG_ON(entry->scheduled);
646
647#ifdef CONFIG_RELEASE_MASTER
648 if (entry->cpu != gsnedf.release_master) {
649#endif
650 entry->scheduled = t;
651 tsk_rt(t)->scheduled_on = task_cpu(t);
652#ifdef CONFIG_RELEASE_MASTER
653 } else {
654 /* do not schedule on release master */
655 preempt(entry); /* force resched */
656 tsk_rt(t)->scheduled_on = NO_CPU;
657 }
658#endif
659 } else {
660 t->rt_param.scheduled_on = NO_CPU;
661 }
662 t->rt_param.linked_on = NO_CPU;
663
664 gsnedf_job_arrival(t);
665 raw_spin_unlock_irqrestore(&gsnedf_lock, flags);
666}
667
668static void gsnedf_task_wake_up(struct task_struct *task)
669{
670 unsigned long flags;
671 lt_t now;
672
673 TRACE_TASK(task, "wake_up at %llu\n", litmus_clock());
674
675 raw_spin_lock_irqsave(&gsnedf_lock, flags);
676 /* We need to take suspensions because of semaphores into
677 * account! If a job resumes after being suspended due to acquiring
678 * a semaphore, it should never be treated as a new job release.
679 */
680 if (get_rt_flags(task) == RT_F_EXIT_SEM) {
681 set_rt_flags(task, RT_F_RUNNING);
682 } else {
683 now = litmus_clock();
684 if (is_tardy(task, now)) {
685 /* new sporadic release */
686 gsnedf_release_at(task, now);
687 sched_trace_task_release(task);
688 }
689 else {
690 if (task->rt.time_slice) {
691 /* came back in time before deadline
692 */
693 set_rt_flags(task, RT_F_RUNNING);
694 }
695 }
696 }
697 gsnedf_job_arrival(task);
698 raw_spin_unlock_irqrestore(&gsnedf_lock, flags);
699}
700
701static void gsnedf_task_block(struct task_struct *t)
702{
703 unsigned long flags;
704
705 TRACE_TASK(t, "block at %llu\n", litmus_clock());
706
707 /* unlink if necessary */
708 raw_spin_lock_irqsave(&gsnedf_lock, flags);
709 unlink(t);
710 raw_spin_unlock_irqrestore(&gsnedf_lock, flags);
711
712 BUG_ON(!is_realtime(t));
713}
714
715
716static void gsnedf_task_exit(struct task_struct * t)
717{
718 unsigned long flags;
719
720 /* unlink if necessary */
721 raw_spin_lock_irqsave(&gsnedf_lock, flags);
722 unlink(t);
723 if (tsk_rt(t)->scheduled_on != NO_CPU) {
724 gsnedf_cpus[tsk_rt(t)->scheduled_on]->scheduled = NULL;
725 tsk_rt(t)->scheduled_on = NO_CPU;
726 }
727 raw_spin_unlock_irqrestore(&gsnedf_lock, flags);
728
729 BUG_ON(!is_realtime(t));
730 TRACE_TASK(t, "RIP\n");
731}
732
733
734static long gsnedf_admit_task(struct task_struct* tsk)
735{
736 return 0;
737}
738
739#ifdef CONFIG_LITMUS_LOCKING
740
741#include <litmus/fdso.h>
742
743/* called with IRQs off */
744static void set_priority_inheritance(struct task_struct* t, struct task_struct* prio_inh)
745{
746 int linked_on;
747 int check_preempt = 0;
748
749 raw_spin_lock(&gsnedf_lock);
750
751 TRACE_TASK(t, "inherits priority from %s/%d\n", prio_inh->comm, prio_inh->pid);
752 tsk_rt(t)->inh_task = prio_inh;
753
754 linked_on = tsk_rt(t)->linked_on;
755
756 /* If it is scheduled, then we need to reorder the CPU heap. */
757 if (linked_on != NO_CPU) {
758 TRACE_TASK(t, "%s: linked on %d\n",
759 __FUNCTION__, linked_on);
760 /* Holder is scheduled; need to re-order CPUs.
761 * We can't use heap_decrease() here since
762 * the cpu_heap is ordered in reverse direction, so
763 * it is actually an increase. */
764 bheap_delete(cpu_lower_prio, &gsnedf_cpu_heap,
765 gsnedf_cpus[linked_on]->hn);
766 bheap_insert(cpu_lower_prio, &gsnedf_cpu_heap,
767 gsnedf_cpus[linked_on]->hn);
768 } else {
769 /* holder may be queued: first stop queue changes */
770 raw_spin_lock(&gsnedf.release_lock);
771 if (is_queued(t)) {
772 TRACE_TASK(t, "%s: is queued\n",
773 __FUNCTION__);
774 /* We need to update the position of holder in some
775 * heap. Note that this could be a release heap if we
776 * budget enforcement is used and this job overran. */
777 check_preempt =
778 !bheap_decrease(edf_split_ready_order,
779 tsk_rt(t)->heap_node);
780 } else {
781 /* Nothing to do: if it is not queued and not linked
782 * then it is either sleeping or currently being moved
783 * by other code (e.g., a timer interrupt handler) that
784 * will use the correct priority when enqueuing the
785 * task. */
786 TRACE_TASK(t, "%s: is NOT queued => Done.\n",
787 __FUNCTION__);
788 }
789 raw_spin_unlock(&gsnedf.release_lock);
790
791 /* If holder was enqueued in a release heap, then the following
792 * preemption check is pointless, but we can't easily detect
793 * that case. If you want to fix this, then consider that
794 * simply adding a state flag requires O(n) time to update when
795 * releasing n tasks, which conflicts with the goal to have
796 * O(log n) merges. */
797 if (check_preempt) {
798 /* heap_decrease() hit the top level of the heap: make
799 * sure preemption checks get the right task, not the
800 * potentially stale cache. */
801 bheap_uncache_min(edf_split_ready_order,
802 &gsnedf.ready_queue);
803 check_for_preemptions();
804 }
805 }
806
807 raw_spin_unlock(&gsnedf_lock);
808}
809
810/* called with IRQs off */
811static void update_unlocked_priority(struct task_struct* t)
812{
813 raw_spin_lock(&gsnedf_lock);
814
815 /* A job only stops inheriting a priority when it releases a
816 * resource. Thus we can make the following assumption.*/
817 BUG_ON(tsk_rt(t)->scheduled_on == NO_CPU);
818
819 /* Clear priority inheritance */
820 TRACE_TASK(t, "priority restored\n");
821 tsk_rt(t)->inh_task = NULL;
822
823 /* Update splitting deadline */
824 tsk_rt(t)->job_params.subjob_deadline = get_proper_deadline(t);
825
826 /* Check if rescheduling is necessary. We can't use heap_decrease()
827 * since the priority was effectively lowered. */
828 unlink(t);
829 gsnedf_job_arrival(t);
830
831 raw_spin_unlock(&gsnedf_lock);
832}
833
834
835/* ******************** FMLP support ********************** */
836
837/* struct for semaphore with priority inheritance */
838struct fmlp_semaphore {
839 struct litmus_lock litmus_lock;
840
841 /* current resource holder */
842 struct task_struct *owner;
843
844 /* highest-priority waiter */
845 struct task_struct *hp_waiter;
846
847 /* FIFO queue of waiting tasks */
848 wait_queue_head_t wait;
849};
850
851static inline struct fmlp_semaphore* fmlp_from_lock(struct litmus_lock* lock)
852{
853 return container_of(lock, struct fmlp_semaphore, litmus_lock);
854}
855
856/* caller is responsible for locking */
857static struct task_struct* find_hp_waiter(struct fmlp_semaphore *sem,
858 struct task_struct* skip)
859{
860 struct list_head *pos;
861 struct task_struct *queued, *found = NULL;
862
863 list_for_each(pos, &sem->wait.task_list) {
864 queued = (struct task_struct*) list_entry(pos, wait_queue_t,
865 task_list)->private;
866
867 /* Compare task prios, find high prio task. */
868 if (queued != skip && edf_split_higher_prio(queued, found))
869 found = queued;
870 }
871 return found;
872}
873
874int gsnedf_fmlp_lock(struct litmus_lock* l)
875{
876 struct task_struct* t = current;
877 struct fmlp_semaphore *sem = fmlp_from_lock(l);
878 cpu_entry_t* entry;
879 wait_queue_t wait;
880 unsigned long flags;
881
882 if (!is_realtime(t))
883 return -EPERM;
884
885 spin_lock_irqsave(&sem->wait.lock, flags);
886 entry = &__get_cpu_var(gsnedf_cpu_entries);
887
888 tsk_rt(t)->in_crit_section = 1;
889 if (entry->timer_armed) {
890 cancel_split_timer(entry);
891 }
892
893 if (sem->owner) {
894 /* resource is not free => must suspend and wait */
895
896 init_waitqueue_entry(&wait, t);
897
898 /* FIXME: interruptible would be nice some day */
899 set_task_state(t, TASK_UNINTERRUPTIBLE);
900
901 __add_wait_queue_tail_exclusive(&sem->wait, &wait);
902
903 /* check if we need to activate priority inheritance */
904 if (edf_split_higher_prio(t, sem->hp_waiter)) {
905 sem->hp_waiter = t;
906 if (edf_split_higher_prio(t, sem->owner))
907 set_priority_inheritance(sem->owner, sem->hp_waiter);
908 }
909
910 TS_LOCK_SUSPEND;
911
912 /* release lock before sleeping */
913 spin_unlock_irqrestore(&sem->wait.lock, flags);
914
915 /* We depend on the FIFO order. Thus, we don't need to recheck
916 * when we wake up; we are guaranteed to have the lock since
917 * there is only one wake up per release.
918 */
919
920 schedule();
921
922 TS_LOCK_RESUME;
923
924 /* Since we hold the lock, no other task will change
925 * ->owner. We can thus check it without acquiring the spin
926 * lock. */
927 BUG_ON(sem->owner != t);
928 } else {
929 /* it's ours now */
930 sem->owner = t;
931
932 spin_unlock_irqrestore(&sem->wait.lock, flags);
933 }
934
935 return 0;
936}
937
938int gsnedf_fmlp_unlock(struct litmus_lock* l)
939{
940 struct task_struct *t = current, *next;
941 struct fmlp_semaphore *sem = fmlp_from_lock(l);
942 unsigned long flags;
943 int err = 0;
944
945 spin_lock_irqsave(&sem->wait.lock, flags);
946
947 if (sem->owner != t) {
948 err = -EINVAL;
949 goto out;
950 }
951
952 /* check if there are jobs waiting for this resource */
953 next = __waitqueue_remove_first(&sem->wait);
954 if (next) {
955 /* next becomes the resouce holder */
956 sem->owner = next;
957 TRACE_CUR("lock ownership passed to %s/%d\n", next->comm, next->pid);
958
959 /* determine new hp_waiter if necessary */
960 if (next == sem->hp_waiter) {
961 TRACE_TASK(next, "was highest-prio waiter\n");
962 /* next has the highest priority --- it doesn't need to
963 * inherit. However, we need to make sure that the
964 * next-highest priority in the queue is reflected in
965 * hp_waiter. */
966 sem->hp_waiter = find_hp_waiter(sem, next);
967 if (sem->hp_waiter)
968 TRACE_TASK(sem->hp_waiter, "is new highest-prio waiter\n");
969 else
970 TRACE("no further waiters\n");
971 } else {
972 /* Well, if next is not the highest-priority waiter,
973 * then it ought to inherit the highest-priority
974 * waiter's priority. */
975 set_priority_inheritance(next, sem->hp_waiter);
976 }
977
978 /* wake up next */
979 wake_up_process(next);
980 } else
981 /* becomes available */
982 sem->owner = NULL;
983
984 /* We are no longer in the critical section */
985 tsk_rt(t)->in_crit_section = 0;
986
987 /* we lose the benefit of priority inheritance (if any) and may need
988 * to move the deadline. In either case, may need to reschedule
989 * due to reduced priority. */
990 if (tsk_rt(t)->inh_task || needs_deadline_move(t))
991 update_unlocked_priority(t);
992 /* TODO: Check that schedule() gets called - it needs to arm the
993 * enforcement timer. Otherwise we should do it here or in
994 * update_unlocked_priority. */
995
996out:
997 spin_unlock_irqrestore(&sem->wait.lock, flags);
998
999 return err;
1000}
1001
1002int gsnedf_fmlp_close(struct litmus_lock* l)
1003{
1004 struct task_struct *t = current;
1005 struct fmlp_semaphore *sem = fmlp_from_lock(l);
1006 unsigned long flags;
1007
1008 int owner;
1009
1010 spin_lock_irqsave(&sem->wait.lock, flags);
1011
1012 owner = sem->owner == t;
1013
1014 spin_unlock_irqrestore(&sem->wait.lock, flags);
1015
1016 if (owner)
1017 gsnedf_fmlp_unlock(l);
1018
1019 return 0;
1020}
1021
1022void gsnedf_fmlp_free(struct litmus_lock* lock)
1023{
1024 kfree(fmlp_from_lock(lock));
1025}
1026
1027static struct litmus_lock_ops gsnedf_fmlp_lock_ops = {
1028 .close = gsnedf_fmlp_close,
1029 .lock = gsnedf_fmlp_lock,
1030 .unlock = gsnedf_fmlp_unlock,
1031 .deallocate = gsnedf_fmlp_free,
1032};
1033
1034static struct litmus_lock* gsnedf_new_fmlp(void)
1035{
1036 struct fmlp_semaphore* sem;
1037
1038 sem = kmalloc(sizeof(*sem), GFP_KERNEL);
1039 if (!sem)
1040 return NULL;
1041
1042 sem->owner = NULL;
1043 sem->hp_waiter = NULL;
1044 init_waitqueue_head(&sem->wait);
1045 sem->litmus_lock.ops = &gsnedf_fmlp_lock_ops;
1046
1047 return &sem->litmus_lock;
1048}
1049
1050/* **** lock constructor **** */
1051
1052
1053static long gsnedf_allocate_lock(struct litmus_lock **lock, int type,
1054 void* __user unused)
1055{
1056 int err = -ENXIO;
1057
1058 /* GSN-EDF currently only supports the FMLP for global resources. */
1059 switch (type) {
1060
1061 case FMLP_SEM:
1062 /* Flexible Multiprocessor Locking Protocol */
1063 *lock = gsnedf_new_fmlp();
1064 if (*lock)
1065 err = 0;
1066 else
1067 err = -ENOMEM;
1068 break;
1069
1070 };
1071
1072 return err;
1073}
1074
1075#endif
1076
1077
1078static long gsnedf_activate_plugin(void)
1079{
1080 int cpu;
1081 cpu_entry_t *entry;
1082
1083 bheap_init(&gsnedf_cpu_heap);
1084#ifdef CONFIG_RELEASE_MASTER
1085 gsnedf.release_master = atomic_read(&release_master_cpu);
1086#endif
1087
1088 for_each_online_cpu(cpu) {
1089 entry = &per_cpu(gsnedf_cpu_entries, cpu);
1090 bheap_node_init(&entry->hn, entry);
1091 entry->linked = NULL;
1092 entry->scheduled = NULL;
1093#ifdef CONFIG_RELEASE_MASTER
1094 if (cpu != gsnedf.release_master) {
1095#endif
1096 TRACE("GSN-EDF: Initializing CPU #%d.\n", cpu);
1097 update_cpu_position(entry);
1098#ifdef CONFIG_RELEASE_MASTER
1099 } else {
1100 TRACE("GSN-EDF: CPU %d is release master.\n", cpu);
1101 }
1102#endif
1103 }
1104 return 0;
1105}
1106
1107/* Plugin object */
1108static struct sched_plugin gfl_plugin __cacheline_aligned_in_smp = {
1109 .plugin_name = "GSN-EDF",
1110 .finish_switch = gsnedf_finish_switch,
1111 .tick = gsnedf_tick,
1112 .task_new = gsnedf_task_new,
1113 .complete_job = complete_job,
1114 .task_exit = gsnedf_task_exit,
1115 .schedule = gsnedf_schedule,
1116 .release_at = gsnedf_release_at,
1117 .task_wake_up = gsnedf_task_wake_up,
1118 .task_block = gsnedf_task_block,
1119 .admit_task = gsnedf_admit_task,
1120 .activate_plugin = gsnedf_activate_plugin,
1121#ifdef CONFIG_LITMUS_LOCKING
1122 .allocate_lock = gsnedf_allocate_lock,
1123#endif
1124};
1125
1126
1127static int __init init_gfl(void)
1128{
1129 int cpu;
1130 cpu_entry_t *entry;
1131
1132 bheap_init(&gsnedf_cpu_heap);
1133 /* initialize CPU state */
1134 for (cpu = 0; cpu < NR_CPUS; cpu++) {
1135 entry = &per_cpu(gsnedf_cpu_entries, cpu);
1136 gsnedf_cpus[cpu] = entry;
1137 entry->cpu = cpu;
1138 entry->hn = &gsnedf_heap_node[cpu];
1139 hrtimer_init(&entry->split_timer,
1140 CLOCK_MONOTONIC, HRTIMER_MODE_ABS);
1141 entry->split_timer.function = on_split_timeout;
1142 bheap_node_init(&entry->hn, entry);
1143 }
1144 edf_split_domain_init(&gsnedf, NULL, gsnedf_release_jobs);
1145 return register_sched_plugin(&gfl_plugin);
1146}
1147
1148
1149module_init(init_gfl);
generated by cgit v1.2.2 (git 2.25.0) at 2025-09-20 11:37:24 -0400