aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorGlenn Elliott <gelliott@cs.unc.edu>2011-06-02 16:06:05 -0400
committerGlenn Elliott <gelliott@cs.unc.edu>2011-06-02 16:06:05 -0400
commit3d5537c160c1484e8d562b9828baf679cc53f67a (patch)
treeb595364f1b0f94ac2426c8315bc5967debc7bbb0
parent7d754596756240fa918b94cd0c3011c77a638987 (diff)
Full patch for klitirqd with Nvidia GPU support.
-rw-r--r--arch/x86/kernel/irq.c14
-rw-r--r--arch/x86/kernel/syscall_table_32.S1
-rw-r--r--include/linux/completion.h1
-rw-r--r--include/linux/interrupt.h9
-rw-r--r--include/linux/mutex.h10
-rw-r--r--include/linux/semaphore.h9
-rw-r--r--include/linux/workqueue.h18
-rw-r--r--include/litmus/affinity.h78
-rw-r--r--include/litmus/fdso.h6
-rw-r--r--include/litmus/litmus.h1
-rw-r--r--include/litmus/litmus_softirq.h199
-rw-r--r--include/litmus/nvidia_info.h37
-rw-r--r--include/litmus/preempt.h1
-rw-r--r--include/litmus/rt_param.h44
-rw-r--r--include/litmus/sched_plugin.h22
-rw-r--r--include/litmus/sched_trace.h174
-rw-r--r--include/litmus/sched_trace_external.h42
-rw-r--r--include/litmus/unistd_32.h3
-rw-r--r--include/litmus/unistd_64.h5
-rw-r--r--kernel/lockdep.c3
-rw-r--r--kernel/mutex.c141
-rw-r--r--kernel/sched.c23
-rw-r--r--kernel/semaphore.c13
-rw-r--r--kernel/softirq.c278
-rw-r--r--kernel/workqueue.c70
-rw-r--r--litmus/Kconfig89
-rw-r--r--litmus/Makefile4
-rw-r--r--litmus/affinity.c49
-rw-r--r--litmus/edf_common.c6
-rw-r--r--litmus/fdso.c1
-rw-r--r--litmus/litmus.c82
-rw-r--r--litmus/litmus_proc.c17
-rw-r--r--litmus/litmus_softirq.c1579
-rw-r--r--litmus/locking.c1
-rw-r--r--litmus/nvidia_info.c526
-rw-r--r--litmus/preempt.c7
-rw-r--r--litmus/sched_cedf.c852
-rw-r--r--litmus/sched_gsn_edf.c756
-rw-r--r--litmus/sched_litmus.c2
-rw-r--r--litmus/sched_plugin.c29
-rw-r--r--litmus/sched_task_trace.c216
-rw-r--r--litmus/sched_trace_external.c45
42 files changed, 5325 insertions, 138 deletions
diff --git a/arch/x86/kernel/irq.c b/arch/x86/kernel/irq.c
index 91fd0c70a18a..50abbc6b7429 100644
--- a/arch/x86/kernel/irq.c
+++ b/arch/x86/kernel/irq.c
@@ -8,6 +8,10 @@
8#include <linux/smp.h> 8#include <linux/smp.h>
9#include <linux/ftrace.h> 9#include <linux/ftrace.h>
10 10
11#ifdef CONFIG_LITMUS_NVIDIA
12#include <litmus/sched_trace.h>
13#endif
14
11#include <asm/apic.h> 15#include <asm/apic.h>
12#include <asm/io_apic.h> 16#include <asm/io_apic.h>
13#include <asm/irq.h> 17#include <asm/irq.h>
@@ -244,7 +248,17 @@ unsigned int __irq_entry do_IRQ(struct pt_regs *regs)
244 __func__, smp_processor_id(), vector, irq); 248 __func__, smp_processor_id(), vector, irq);
245 } 249 }
246 250
251//#ifndef CONFIG_LITMUS_NVIDIA
247 irq_exit(); 252 irq_exit();
253//#else
254 /* skip softirqs if we're tracing an interrupt top-half */
255 /* comment out if-statement if we want to trace with bh on. */
256 //if(!is_interrupt_tracing_active())
257// irq_exit();
258
259
260// sched_trace_nv_interrupt_end();
261//#endif
248 262
249 set_irq_regs(old_regs); 263 set_irq_regs(old_regs);
250 return 1; 264 return 1;
diff --git a/arch/x86/kernel/syscall_table_32.S b/arch/x86/kernel/syscall_table_32.S
index 37702905f658..b5ddae40cee2 100644
--- a/arch/x86/kernel/syscall_table_32.S
+++ b/arch/x86/kernel/syscall_table_32.S
@@ -352,3 +352,4 @@ ENTRY(sys_call_table)
352 .long sys_wait_for_ts_release 352 .long sys_wait_for_ts_release
353 .long sys_release_ts 353 .long sys_release_ts
354 .long sys_null_call 354 .long sys_null_call
355 .long sys_register_nv_device
diff --git a/include/linux/completion.h b/include/linux/completion.h
index c63950e8a863..3ce20dd3086e 100644
--- a/include/linux/completion.h
+++ b/include/linux/completion.h
@@ -76,6 +76,7 @@ static inline void init_completion(struct completion *x)
76 init_waitqueue_head(&x->wait); 76 init_waitqueue_head(&x->wait);
77} 77}
78 78
79extern void __wait_for_completion_locked(struct completion *);
79extern void wait_for_completion(struct completion *); 80extern void wait_for_completion(struct completion *);
80extern int wait_for_completion_interruptible(struct completion *x); 81extern int wait_for_completion_interruptible(struct completion *x);
81extern int wait_for_completion_killable(struct completion *x); 82extern int wait_for_completion_killable(struct completion *x);
diff --git a/include/linux/interrupt.h b/include/linux/interrupt.h
index a0384a4d1e6f..5d22f5342376 100644
--- a/include/linux/interrupt.h
+++ b/include/linux/interrupt.h
@@ -459,6 +459,10 @@ struct tasklet_struct
459 atomic_t count; 459 atomic_t count;
460 void (*func)(unsigned long); 460 void (*func)(unsigned long);
461 unsigned long data; 461 unsigned long data;
462
463#ifdef CONFIG_LITMUS_SOFTIRQD
464 struct task_struct *owner;
465#endif
462}; 466};
463 467
464#define DECLARE_TASKLET(name, func, data) \ 468#define DECLARE_TASKLET(name, func, data) \
@@ -496,6 +500,7 @@ static inline void tasklet_unlock_wait(struct tasklet_struct *t)
496#define tasklet_unlock(t) do { } while (0) 500#define tasklet_unlock(t) do { } while (0)
497#endif 501#endif
498 502
503extern void ___tasklet_schedule(struct tasklet_struct *t);
499extern void __tasklet_schedule(struct tasklet_struct *t); 504extern void __tasklet_schedule(struct tasklet_struct *t);
500 505
501static inline void tasklet_schedule(struct tasklet_struct *t) 506static inline void tasklet_schedule(struct tasklet_struct *t)
@@ -504,6 +509,7 @@ static inline void tasklet_schedule(struct tasklet_struct *t)
504 __tasklet_schedule(t); 509 __tasklet_schedule(t);
505} 510}
506 511
512extern void ___tasklet_hi_schedule(struct tasklet_struct *t);
507extern void __tasklet_hi_schedule(struct tasklet_struct *t); 513extern void __tasklet_hi_schedule(struct tasklet_struct *t);
508 514
509static inline void tasklet_hi_schedule(struct tasklet_struct *t) 515static inline void tasklet_hi_schedule(struct tasklet_struct *t)
@@ -512,6 +518,7 @@ static inline void tasklet_hi_schedule(struct tasklet_struct *t)
512 __tasklet_hi_schedule(t); 518 __tasklet_hi_schedule(t);
513} 519}
514 520
521extern void ___tasklet_hi_schedule_first(struct tasklet_struct *t);
515extern void __tasklet_hi_schedule_first(struct tasklet_struct *t); 522extern void __tasklet_hi_schedule_first(struct tasklet_struct *t);
516 523
517/* 524/*
@@ -541,7 +548,7 @@ static inline void tasklet_disable(struct tasklet_struct *t)
541} 548}
542 549
543static inline void tasklet_enable(struct tasklet_struct *t) 550static inline void tasklet_enable(struct tasklet_struct *t)
544{ 551{
545 smp_mb__before_atomic_dec(); 552 smp_mb__before_atomic_dec();
546 atomic_dec(&t->count); 553 atomic_dec(&t->count);
547} 554}
diff --git a/include/linux/mutex.h b/include/linux/mutex.h
index f363bc8fdc74..9f3199571994 100644
--- a/include/linux/mutex.h
+++ b/include/linux/mutex.h
@@ -126,6 +126,15 @@ static inline int mutex_is_locked(struct mutex *lock)
126 return atomic_read(&lock->count) != 1; 126 return atomic_read(&lock->count) != 1;
127} 127}
128 128
129/* return non-zero to abort. only pre-side-effects may abort */
130typedef int (*side_effect_t)(unsigned long);
131extern void mutex_lock_sfx(struct mutex *lock,
132 side_effect_t pre, unsigned long pre_arg,
133 side_effect_t post, unsigned long post_arg);
134extern void mutex_unlock_sfx(struct mutex *lock,
135 side_effect_t pre, unsigned long pre_arg,
136 side_effect_t post, unsigned long post_arg);
137
129/* 138/*
130 * See kernel/mutex.c for detailed documentation of these APIs. 139 * See kernel/mutex.c for detailed documentation of these APIs.
131 * Also see Documentation/mutex-design.txt. 140 * Also see Documentation/mutex-design.txt.
@@ -145,6 +154,7 @@ extern void mutex_lock(struct mutex *lock);
145extern int __must_check mutex_lock_interruptible(struct mutex *lock); 154extern int __must_check mutex_lock_interruptible(struct mutex *lock);
146extern int __must_check mutex_lock_killable(struct mutex *lock); 155extern int __must_check mutex_lock_killable(struct mutex *lock);
147 156
157
148# define mutex_lock_nested(lock, subclass) mutex_lock(lock) 158# define mutex_lock_nested(lock, subclass) mutex_lock(lock)
149# define mutex_lock_interruptible_nested(lock, subclass) mutex_lock_interruptible(lock) 159# define mutex_lock_interruptible_nested(lock, subclass) mutex_lock_interruptible(lock)
150# define mutex_lock_killable_nested(lock, subclass) mutex_lock_killable(lock) 160# define mutex_lock_killable_nested(lock, subclass) mutex_lock_killable(lock)
diff --git a/include/linux/semaphore.h b/include/linux/semaphore.h
index 5310d27abd2a..69e3f57661ec 100644
--- a/include/linux/semaphore.h
+++ b/include/linux/semaphore.h
@@ -49,4 +49,13 @@ extern int __must_check down_trylock(struct semaphore *sem);
49extern int __must_check down_timeout(struct semaphore *sem, long jiffies); 49extern int __must_check down_timeout(struct semaphore *sem, long jiffies);
50extern void up(struct semaphore *sem); 50extern void up(struct semaphore *sem);
51 51
52extern void __down(struct semaphore *sem);
53extern void __up(struct semaphore *sem);
54
55struct semaphore_waiter {
56 struct list_head list;
57 struct task_struct *task;
58 int up;
59};
60
52#endif /* __LINUX_SEMAPHORE_H */ 61#endif /* __LINUX_SEMAPHORE_H */
diff --git a/include/linux/workqueue.h b/include/linux/workqueue.h
index 25e02c941bac..5fecfb375eeb 100644
--- a/include/linux/workqueue.h
+++ b/include/linux/workqueue.h
@@ -83,6 +83,9 @@ struct work_struct {
83#ifdef CONFIG_LOCKDEP 83#ifdef CONFIG_LOCKDEP
84 struct lockdep_map lockdep_map; 84 struct lockdep_map lockdep_map;
85#endif 85#endif
86#ifdef CONFIG_LITMUS_SOFTIRQD
87 struct task_struct *owner;
88#endif
86}; 89};
87 90
88#define WORK_DATA_INIT() ATOMIC_LONG_INIT(WORK_STRUCT_NO_CPU) 91#define WORK_DATA_INIT() ATOMIC_LONG_INIT(WORK_STRUCT_NO_CPU)
@@ -115,11 +118,25 @@ struct execute_work {
115#define __WORK_INIT_LOCKDEP_MAP(n, k) 118#define __WORK_INIT_LOCKDEP_MAP(n, k)
116#endif 119#endif
117 120
121#ifdef CONFIG_LITMUS_SOFTIRQD
122#define __WORK_INIT_OWNER() \
123 .owner = NULL,
124
125#define PREPARE_OWNER(_work, _owner) \
126 do { \
127 (_work)->owner = (_owner); \
128 } while(0)
129#else
130#define __WORK_INIT_OWNER()
131#define PREPARE_OWNER(_work, _owner)
132#endif
133
118#define __WORK_INITIALIZER(n, f) { \ 134#define __WORK_INITIALIZER(n, f) { \
119 .data = WORK_DATA_STATIC_INIT(), \ 135 .data = WORK_DATA_STATIC_INIT(), \
120 .entry = { &(n).entry, &(n).entry }, \ 136 .entry = { &(n).entry, &(n).entry }, \
121 .func = (f), \ 137 .func = (f), \
122 __WORK_INIT_LOCKDEP_MAP(#n, &(n)) \ 138 __WORK_INIT_LOCKDEP_MAP(#n, &(n)) \
139 __WORK_INIT_OWNER() \
123 } 140 }
124 141
125#define __DELAYED_WORK_INITIALIZER(n, f) { \ 142#define __DELAYED_WORK_INITIALIZER(n, f) { \
@@ -327,6 +344,7 @@ extern void flush_workqueue(struct workqueue_struct *wq);
327extern void flush_scheduled_work(void); 344extern void flush_scheduled_work(void);
328extern void flush_delayed_work(struct delayed_work *work); 345extern void flush_delayed_work(struct delayed_work *work);
329 346
347extern int __schedule_work(struct work_struct *work);
330extern int schedule_work(struct work_struct *work); 348extern int schedule_work(struct work_struct *work);
331extern int schedule_work_on(int cpu, struct work_struct *work); 349extern int schedule_work_on(int cpu, struct work_struct *work);
332extern int schedule_delayed_work(struct delayed_work *work, unsigned long delay); 350extern int schedule_delayed_work(struct delayed_work *work, unsigned long delay);
diff --git a/include/litmus/affinity.h b/include/litmus/affinity.h
new file mode 100644
index 000000000000..877b4099c6e2
--- /dev/null
+++ b/include/litmus/affinity.h
@@ -0,0 +1,78 @@
1#ifndef __LITMUS_AFFINITY_H
2#define __LITMUS_AFFINITY_H
3
4#include <linux/cpumask.h>
5
6/*
7 L1 (instr) = depth 0
8 L1 (data) = depth 1
9 L2 = depth 2
10 L3 = depth 3
11 */
12#define NUM_CACHE_LEVELS 4
13
14struct neighborhood
15{
16 unsigned int size[NUM_CACHE_LEVELS];
17 cpumask_var_t neighbors[NUM_CACHE_LEVELS];
18};
19
20/* topology info is stored redundently in a big array for fast lookups */
21extern struct neighborhood neigh_info[NR_CPUS];
22
23void init_topology(void); /* called by Litmus module's _init_litmus() */
24
25/* Works like:
26void get_nearest_available_cpu(cpu_entry_t* nearest, cpu_entry_t* start, cpu_entry_t* entries, int release_master)
27
28Set release_master = -1 for no RM.
29 */
30#define get_nearest_available_cpu(nearest, start, entries, release_master) \
31{ \
32 (nearest) = NULL; \
33 if(!(start)->linked) \
34 { \
35 (nearest) = (start); \
36 } \
37 else \
38 { \
39 int __level; \
40 int __cpu; \
41 struct neighborhood* __neighbors = &neigh_info[(start)->cpu]; \
42 \
43 for(__level = 0; (__level < NUM_CACHE_LEVELS) && !(nearest); ++__level) \
44 { \
45 if(__neighbors->size[__level] > 1) \
46 { \
47 for_each_cpu(__cpu, __neighbors->neighbors[__level]) \
48 { \
49 if(__cpu != (release_master)) \
50 { \
51 cpu_entry_t* __entry = &per_cpu((entries), __cpu); \
52 if(!__entry->linked) \
53 { \
54 (nearest) = __entry; \
55 break; \
56 } \
57 } \
58 } \
59 } \
60 else if(__neighbors->size[__level] == 0) \
61 { \
62 break; \
63 } \
64 } \
65 } \
66 \
67 if((nearest)) \
68 { \
69 TRACE("P%d is closest available CPU to P%d\n", (nearest)->cpu, (start)->cpu); \
70 } \
71 else \
72 { \
73 TRACE("Could not find an available CPU close to P%d\n", \
74 (start)->cpu); \
75 } \
76}
77
78#endif
diff --git a/include/litmus/fdso.h b/include/litmus/fdso.h
index caf2a1e6918c..c740e8fc3e88 100644
--- a/include/litmus/fdso.h
+++ b/include/litmus/fdso.h
@@ -18,9 +18,10 @@ typedef enum {
18 MIN_OBJ_TYPE = 0, 18 MIN_OBJ_TYPE = 0,
19 19
20 FMLP_SEM = 0, 20 FMLP_SEM = 0,
21 SRP_SEM = 1, 21 KFMLP_SEM = 1,
22 SRP_SEM = 2,
22 23
23 MAX_OBJ_TYPE = 1 24 MAX_OBJ_TYPE = SRP_SEM
24} obj_type_t; 25} obj_type_t;
25 26
26struct inode_obj_id { 27struct inode_obj_id {
@@ -64,6 +65,7 @@ static inline void* od_lookup(int od, obj_type_t type)
64} 65}
65 66
66#define lookup_fmlp_sem(od)((struct pi_semaphore*) od_lookup(od, FMLP_SEM)) 67#define lookup_fmlp_sem(od)((struct pi_semaphore*) od_lookup(od, FMLP_SEM))
68#define lookup_kfmlp_sem(od)((struct pi_semaphore*) od_lookup(od, KFMLP_SEM))
67#define lookup_srp_sem(od) ((struct srp_semaphore*) od_lookup(od, SRP_SEM)) 69#define lookup_srp_sem(od) ((struct srp_semaphore*) od_lookup(od, SRP_SEM))
68#define lookup_ics(od) ((struct ics*) od_lookup(od, ICS_ID)) 70#define lookup_ics(od) ((struct ics*) od_lookup(od, ICS_ID))
69 71
diff --git a/include/litmus/litmus.h b/include/litmus/litmus.h
index e7769ca36ec0..3df242bf272f 100644
--- a/include/litmus/litmus.h
+++ b/include/litmus/litmus.h
@@ -26,6 +26,7 @@ static inline int in_list(struct list_head* list)
26 ); 26 );
27} 27}
28 28
29
29struct task_struct* __waitqueue_remove_first(wait_queue_head_t *wq); 30struct task_struct* __waitqueue_remove_first(wait_queue_head_t *wq);
30 31
31#define NO_CPU 0xffffffff 32#define NO_CPU 0xffffffff
diff --git a/include/litmus/litmus_softirq.h b/include/litmus/litmus_softirq.h
new file mode 100644
index 000000000000..34287f3cbb8d
--- /dev/null
+++ b/include/litmus/litmus_softirq.h
@@ -0,0 +1,199 @@
1#ifndef __LITMUS_SOFTIRQ_H
2#define __LITMUS_SOFTIRQ_H
3
4#include <linux/interrupt.h>
5#include <linux/workqueue.h>
6
7/*
8 Threaded tasklet handling for Litmus. Tasklets
9 are scheduled with the priority of the tasklet's
10 owner---that is, the RT task on behalf the tasklet
11 runs.
12
13 Tasklets are current scheduled in FIFO order with
14 NO priority inheritance for "blocked" tasklets.
15
16 klitirqd assumes the priority of the owner of the
17 tasklet when the tasklet is next to execute.
18
19 Currently, hi-tasklets are scheduled before
20 low-tasklets, regardless of priority of low-tasklets.
21 And likewise, low-tasklets are scheduled before work
22 queue objects. This priority inversion probably needs
23 to be fixed, though it is not an issue if our work with
24 GPUs as GPUs are owned (and associated klitirqds) for
25 exclusive time periods, thus no inversions can
26 occur.
27 */
28
29
30
31#define NR_LITMUS_SOFTIRQD CONFIG_NR_LITMUS_SOFTIRQD
32
33/* Spawns NR_LITMUS_SOFTIRQD klitirqd daemons.
34 Actual launch of threads is deffered to kworker's
35 workqueue, so daemons will likely not be immediately
36 running when this function returns, though the required
37 data will be initialized.
38
39 @affinity_set: an array expressing the processor affinity
40 for each of the NR_LITMUS_SOFTIRQD daemons. May be set
41 to NULL for global scheduling.
42
43 - Examples -
44 8-CPU system with two CPU clusters:
45 affinity[] = {0, 0, 0, 0, 3, 3, 3, 3}
46 NOTE: Daemons not actually bound to specified CPU, but rather
47 cluster in which the CPU resides.
48
49 8-CPU system, partitioned:
50 affinity[] = {0, 1, 2, 3, 4, 5, 6, 7}
51
52 FIXME: change array to a CPU topology or array of cpumasks
53
54 */
55void spawn_klitirqd(int* affinity);
56
57
58/* Raises a flag to tell klitirqds to terminate.
59 Termination is async, so some threads may be running
60 after function return. */
61void kill_klitirqd(void);
62
63
64/* Returns 1 if all NR_LITMUS_SOFTIRQD klitirqs are ready
65 to handle tasklets. 0, otherwise.*/
66int klitirqd_is_ready(void);
67
68/* Returns 1 if no NR_LITMUS_SOFTIRQD klitirqs are ready
69 to handle tasklets. 0, otherwise.*/
70int klitirqd_is_dead(void);
71
72/* Flushes all pending work out to the OS for regular
73 * tasklet/work processing of the specified 'owner'
74 *
75 * PRECOND: klitirqd_thread must have a clear entry
76 * in the GPU registry, otherwise this call will become
77 * a no-op as work will loop back to the klitirqd_thread.
78 *
79 * Pass NULL for owner to flush ALL pending items.
80 */
81void flush_pending(struct task_struct* klitirqd_thread,
82 struct task_struct* owner);
83
84struct task_struct* get_klitirqd(unsigned int k_id);
85
86
87extern int __litmus_tasklet_schedule(
88 struct tasklet_struct *t,
89 unsigned int k_id);
90
91/* schedule a tasklet on klitirqd #k_id */
92static inline int litmus_tasklet_schedule(
93 struct tasklet_struct *t,
94 unsigned int k_id)
95{
96 int ret = 0;
97 if (!test_and_set_bit(TASKLET_STATE_SCHED, &t->state))
98 ret = __litmus_tasklet_schedule(t, k_id);
99 return(ret);
100}
101
102/* for use by __tasklet_schedule() */
103static inline int _litmus_tasklet_schedule(
104 struct tasklet_struct *t,
105 unsigned int k_id)
106{
107 return(__litmus_tasklet_schedule(t, k_id));
108}
109
110
111
112
113extern int __litmus_tasklet_hi_schedule(struct tasklet_struct *t,
114 unsigned int k_id);
115
116/* schedule a hi tasklet on klitirqd #k_id */
117static inline int litmus_tasklet_hi_schedule(struct tasklet_struct *t,
118 unsigned int k_id)
119{
120 int ret = 0;
121 if (!test_and_set_bit(TASKLET_STATE_SCHED, &t->state))
122 ret = __litmus_tasklet_hi_schedule(t, k_id);
123 return(ret);
124}
125
126/* for use by __tasklet_hi_schedule() */
127static inline int _litmus_tasklet_hi_schedule(struct tasklet_struct *t,
128 unsigned int k_id)
129{
130 return(__litmus_tasklet_hi_schedule(t, k_id));
131}
132
133
134
135
136
137extern int __litmus_tasklet_hi_schedule_first(
138 struct tasklet_struct *t,
139 unsigned int k_id);
140
141/* schedule a hi tasklet on klitirqd #k_id on next go-around */
142/* PRECONDITION: Interrupts must be disabled. */
143static inline int litmus_tasklet_hi_schedule_first(
144 struct tasklet_struct *t,
145 unsigned int k_id)
146{
147 int ret = 0;
148 if (!test_and_set_bit(TASKLET_STATE_SCHED, &t->state))
149 ret = __litmus_tasklet_hi_schedule_first(t, k_id);
150 return(ret);
151}
152
153/* for use by __tasklet_hi_schedule_first() */
154static inline int _litmus_tasklet_hi_schedule_first(
155 struct tasklet_struct *t,
156 unsigned int k_id)
157{
158 return(__litmus_tasklet_hi_schedule_first(t, k_id));
159}
160
161
162
163//////////////
164
165extern int __litmus_schedule_work(
166 struct work_struct* w,
167 unsigned int k_id);
168
169static inline int litmus_schedule_work(
170 struct work_struct* w,
171 unsigned int k_id)
172{
173 return(__litmus_schedule_work(w, k_id));
174}
175
176
177
178///////////// mutex operations for client threads.
179
180void down_and_set_stat(struct task_struct* t,
181 enum klitirqd_sem_status to_set,
182 struct mutex* sem);
183
184void __down_and_reset_and_set_stat(struct task_struct* t,
185 enum klitirqd_sem_status to_reset,
186 enum klitirqd_sem_status to_set,
187 struct mutex* sem);
188
189void up_and_set_stat(struct task_struct* t,
190 enum klitirqd_sem_status to_set,
191 struct mutex* sem);
192
193
194
195void release_klitirqd_lock(struct task_struct* t);
196
197int reacquire_klitirqd_lock(struct task_struct* t);
198
199#endif
diff --git a/include/litmus/nvidia_info.h b/include/litmus/nvidia_info.h
new file mode 100644
index 000000000000..579301d77cf5
--- /dev/null
+++ b/include/litmus/nvidia_info.h
@@ -0,0 +1,37 @@
1#ifndef __LITMUS_NVIDIA_H
2#define __LITMUS_NVIDIA_H
3
4#include <linux/interrupt.h>
5
6
7#include <litmus/litmus_softirq.h>
8
9
10#define NV_DEVICE_NUM NR_LITMUS_SOFTIRQD
11
12int init_nvidia_info(void);
13
14int is_nvidia_func(void* func_addr);
15
16void dump_nvidia_info(const struct tasklet_struct *t);
17
18
19// Returns the Nvidia device # associated with provided tasklet and work_struct.
20u32 get_tasklet_nv_device_num(const struct tasklet_struct *t);
21u32 get_work_nv_device_num(const struct work_struct *t);
22
23
24int init_nv_device_reg(void);
25//int get_nv_device_id(struct task_struct* owner);
26
27
28int reg_nv_device(int reg_device_id, int register_device);
29
30struct task_struct* get_nv_device_owner(u32 target_device_id);
31
32void lock_nv_registry(u32 reg_device_id, unsigned long* flags);
33void unlock_nv_registry(u32 reg_device_id, unsigned long* flags);
34
35void increment_nv_int_count(u32 device);
36
37#endif
diff --git a/include/litmus/preempt.h b/include/litmus/preempt.h
index 260c6fe17986..244924f93c48 100644
--- a/include/litmus/preempt.h
+++ b/include/litmus/preempt.h
@@ -26,6 +26,7 @@ const char* sched_state_name(int s);
26 (x), #x, __FUNCTION__); \ 26 (x), #x, __FUNCTION__); \
27 } while (0); 27 } while (0);
28 28
29//#define TRACE_SCHED_STATE_CHANGE(x, y, cpu) /* ignore */
29#define TRACE_SCHED_STATE_CHANGE(x, y, cpu) \ 30#define TRACE_SCHED_STATE_CHANGE(x, y, cpu) \
30 TRACE_STATE("[P%d] 0x%x (%s) -> 0x%x (%s)\n", \ 31 TRACE_STATE("[P%d] 0x%x (%s) -> 0x%x (%s)\n", \
31 cpu, (x), sched_state_name(x), \ 32 cpu, (x), sched_state_name(x), \
diff --git a/include/litmus/rt_param.h b/include/litmus/rt_param.h
index 5de422c742f6..53af3ce1d955 100644
--- a/include/litmus/rt_param.h
+++ b/include/litmus/rt_param.h
@@ -69,6 +69,8 @@ struct control_page {
69/* don't export internal data structures to user space (liblitmus) */ 69/* don't export internal data structures to user space (liblitmus) */
70#ifdef __KERNEL__ 70#ifdef __KERNEL__
71 71
72#include <linux/semaphore.h>
73
72struct _rt_domain; 74struct _rt_domain;
73struct bheap_node; 75struct bheap_node;
74struct release_heap; 76struct release_heap;
@@ -94,6 +96,14 @@ struct rt_job {
94 96
95struct pfair_param; 97struct pfair_param;
96 98
99enum klitirqd_sem_status
100{
101 NEED_TO_REACQUIRE,
102 REACQUIRING,
103 NOT_HELD,
104 HELD
105};
106
97/* RT task parameters for scheduling extensions 107/* RT task parameters for scheduling extensions
98 * These parameters are inherited during clone and therefore must 108 * These parameters are inherited during clone and therefore must
99 * be explicitly set up before the task set is launched. 109 * be explicitly set up before the task set is launched.
@@ -108,6 +118,38 @@ struct rt_param {
108 /* is the task present? (true if it can be scheduled) */ 118 /* is the task present? (true if it can be scheduled) */
109 unsigned int present:1; 119 unsigned int present:1;
110 120
121#ifdef CONFIG_LITMUS_SOFTIRQD
122 /* proxy threads have minimum priority by default */
123 unsigned int is_proxy_thread:1;
124
125 /* pointer to klitirqd currently working on this
126 task_struct's behalf. only set by the task pointed
127 to by klitirqd.
128
129 ptr only valid if is_proxy_thread == 0
130 */
131 struct task_struct* cur_klitirqd;
132
133 /* Used to implement mutual execution exclusion between
134 * job and klitirqd execution. Job must always hold
135 * it's klitirqd_sem to execute. klitirqd instance
136 * must hold the semaphore before executing on behalf
137 * of a job.
138 */
139 //struct semaphore klitirqd_sem;
140 struct mutex klitirqd_sem;
141
142 /* status of held klitirqd_sem, even if the held klitirqd_sem is from
143 another task (only proxy threads do this though).
144 */
145 atomic_t klitirqd_sem_stat;
146#endif
147
148#ifdef CONFIG_LITMUS_NVIDIA
149 /* number of top-half interrupts handled on behalf of current job */
150 atomic_t nv_int_count;
151#endif
152
111#ifdef CONFIG_LITMUS_LOCKING 153#ifdef CONFIG_LITMUS_LOCKING
112 /* Is the task being priority-boosted by a locking protocol? */ 154 /* Is the task being priority-boosted by a locking protocol? */
113 unsigned int priority_boosted:1; 155 unsigned int priority_boosted:1;
@@ -128,7 +170,7 @@ struct rt_param {
128 * an increased task priority. 170 * an increased task priority.
129 */ 171 */
130 struct task_struct* inh_task; 172 struct task_struct* inh_task;
131 173
132#ifdef CONFIG_NP_SECTION 174#ifdef CONFIG_NP_SECTION
133 /* For the FMLP under PSN-EDF, it is required to make the task 175 /* For the FMLP under PSN-EDF, it is required to make the task
134 * non-preemptive from kernel space. In order not to interfere with 176 * non-preemptive from kernel space. In order not to interfere with
diff --git a/include/litmus/sched_plugin.h b/include/litmus/sched_plugin.h
index 6e7cabdddae8..df50930d14a0 100644
--- a/include/litmus/sched_plugin.h
+++ b/include/litmus/sched_plugin.h
@@ -29,7 +29,6 @@ typedef struct task_struct* (*schedule_t)(struct task_struct * prev);
29 */ 29 */
30typedef void (*finish_switch_t)(struct task_struct *prev); 30typedef void (*finish_switch_t)(struct task_struct *prev);
31 31
32
33/********************* task state changes ********************/ 32/********************* task state changes ********************/
34 33
35/* Called to setup a new real-time task. 34/* Called to setup a new real-time task.
@@ -58,6 +57,17 @@ typedef void (*task_exit_t) (struct task_struct *);
58typedef long (*allocate_lock_t) (struct litmus_lock **lock, int type, 57typedef long (*allocate_lock_t) (struct litmus_lock **lock, int type,
59 void* __user config); 58 void* __user config);
60 59
60/* Called to change inheritance levels of given task */
61typedef void (*set_prio_inh_t)(struct task_struct* t,
62 struct task_struct* prio_inh);
63typedef void (*clear_prio_inh_t)(struct task_struct* t);
64
65
66typedef void (*set_prio_inh_klitirq_t)(struct task_struct* klitirqd,
67 struct task_struct* old_owner,
68 struct task_struct* new_owner);
69typedef void (*clear_prio_inh_klitirqd_t)(struct task_struct* klitirqd,
70 struct task_struct* old_owner);
61 71
62/********************* sys call backends ********************/ 72/********************* sys call backends ********************/
63/* This function causes the caller to sleep until the next release */ 73/* This function causes the caller to sleep until the next release */
@@ -88,7 +98,7 @@ struct sched_plugin {
88 /* task state changes */ 98 /* task state changes */
89 admit_task_t admit_task; 99 admit_task_t admit_task;
90 100
91 task_new_t task_new; 101 task_new_t task_new;
92 task_wake_up_t task_wake_up; 102 task_wake_up_t task_wake_up;
93 task_block_t task_block; 103 task_block_t task_block;
94 task_exit_t task_exit; 104 task_exit_t task_exit;
@@ -96,6 +106,14 @@ struct sched_plugin {
96#ifdef CONFIG_LITMUS_LOCKING 106#ifdef CONFIG_LITMUS_LOCKING
97 /* locking protocols */ 107 /* locking protocols */
98 allocate_lock_t allocate_lock; 108 allocate_lock_t allocate_lock;
109
110 set_prio_inh_t set_prio_inh;
111 clear_prio_inh_t clear_prio_inh;
112#endif
113
114#ifdef CONFIG_LITMUS_SOFTIRQD
115 set_prio_inh_klitirq_t set_prio_inh_klitirqd;
116 clear_prio_inh_klitirqd_t clear_prio_inh_klitirqd;
99#endif 117#endif
100} __attribute__ ((__aligned__(SMP_CACHE_BYTES))); 118} __attribute__ ((__aligned__(SMP_CACHE_BYTES)));
101 119
diff --git a/include/litmus/sched_trace.h b/include/litmus/sched_trace.h
index 7ca34cb13881..1486c778aff8 100644
--- a/include/litmus/sched_trace.h
+++ b/include/litmus/sched_trace.h
@@ -11,12 +11,12 @@ struct st_trace_header {
11 u8 cpu; /* On which CPU was it recorded? */ 11 u8 cpu; /* On which CPU was it recorded? */
12 u16 pid; /* PID of the task. */ 12 u16 pid; /* PID of the task. */
13 u32 job; /* The job sequence number. */ 13 u32 job; /* The job sequence number. */
14}; 14} __attribute__((packed));
15 15
16#define ST_NAME_LEN 16 16#define ST_NAME_LEN 16
17struct st_name_data { 17struct st_name_data {
18 char cmd[ST_NAME_LEN];/* The name of the executable of this process. */ 18 char cmd[ST_NAME_LEN];/* The name of the executable of this process. */
19}; 19} __attribute__((packed));
20 20
21struct st_param_data { /* regular params */ 21struct st_param_data { /* regular params */
22 u32 wcet; 22 u32 wcet;
@@ -25,30 +25,29 @@ struct st_param_data { /* regular params */
25 u8 partition; 25 u8 partition;
26 u8 class; 26 u8 class;
27 u8 __unused[2]; 27 u8 __unused[2];
28}; 28} __attribute__((packed));
29 29
30struct st_release_data { /* A job is was/is going to be released. */ 30struct st_release_data { /* A job is was/is going to be released. */
31 u64 release; /* What's the release time? */ 31 u64 release; /* What's the release time? */
32 u64 deadline; /* By when must it finish? */ 32 u64 deadline; /* By when must it finish? */
33}; 33} __attribute__((packed));
34 34
35struct st_assigned_data { /* A job was asigned to a CPU. */ 35struct st_assigned_data { /* A job was asigned to a CPU. */
36 u64 when; 36 u64 when;
37 u8 target; /* Where should it execute? */ 37 u8 target; /* Where should it execute? */
38 u8 __unused[7]; 38 u8 __unused[7];
39}; 39} __attribute__((packed));
40 40
41struct st_switch_to_data { /* A process was switched to on a given CPU. */ 41struct st_switch_to_data { /* A process was switched to on a given CPU. */
42 u64 when; /* When did this occur? */ 42 u64 when; /* When did this occur? */
43 u32 exec_time; /* Time the current job has executed. */ 43 u32 exec_time; /* Time the current job has executed. */
44 u8 __unused[4]; 44 u8 __unused[4];
45 45} __attribute__((packed));
46};
47 46
48struct st_switch_away_data { /* A process was switched away from on a given CPU. */ 47struct st_switch_away_data { /* A process was switched away from on a given CPU. */
49 u64 when; 48 u64 when;
50 u64 exec_time; 49 u64 exec_time;
51}; 50} __attribute__((packed));
52 51
53struct st_completion_data { /* A job completed. */ 52struct st_completion_data { /* A job completed. */
54 u64 when; 53 u64 when;
@@ -56,35 +55,92 @@ struct st_completion_data { /* A job completed. */
56 * next task automatically; set to 0 otherwise. 55 * next task automatically; set to 0 otherwise.
57 */ 56 */
58 u8 __uflags:7; 57 u8 __uflags:7;
59 u8 __unused[7]; 58 u16 nv_int_count;
60}; 59 u8 __unused[5];
60} __attribute__((packed));
61 61
62struct st_block_data { /* A task blocks. */ 62struct st_block_data { /* A task blocks. */
63 u64 when; 63 u64 when;
64 u64 __unused; 64 u64 __unused;
65}; 65} __attribute__((packed));
66 66
67struct st_resume_data { /* A task resumes. */ 67struct st_resume_data { /* A task resumes. */
68 u64 when; 68 u64 when;
69 u64 __unused; 69 u64 __unused;
70}; 70} __attribute__((packed));
71 71
72struct st_action_data { 72struct st_action_data {
73 u64 when; 73 u64 when;
74 u8 action; 74 u8 action;
75 u8 __unused[7]; 75 u8 __unused[7];
76}; 76} __attribute__((packed));
77 77
78struct st_sys_release_data { 78struct st_sys_release_data {
79 u64 when; 79 u64 when;
80 u64 release; 80 u64 release;
81}; 81} __attribute__((packed));
82
83
84struct st_tasklet_release_data {
85 u64 when;
86 u64 __unused;
87} __attribute__((packed));
88
89struct st_tasklet_begin_data {
90 u64 when;
91 u16 exe_pid;
92 u8 __unused[6];
93} __attribute__((packed));
94
95struct st_tasklet_end_data {
96 u64 when;
97 u16 exe_pid;
98 u8 flushed;
99 u8 __unused[5];
100} __attribute__((packed));
101
102
103struct st_work_release_data {
104 u64 when;
105 u64 __unused;
106} __attribute__((packed));
107
108struct st_work_begin_data {
109 u64 when;
110 u16 exe_pid;
111 u8 __unused[6];
112} __attribute__((packed));
113
114struct st_work_end_data {
115 u64 when;
116 u16 exe_pid;
117 u8 flushed;
118 u8 __unused[5];
119} __attribute__((packed));
120
121struct st_effective_priority_change_data {
122 u64 when;
123 u16 inh_pid;
124 u8 __unused[6];
125} __attribute__((packed));
126
127struct st_nv_interrupt_begin_data {
128 u64 when;
129 u32 device;
130 u8 __unused[4];
131} __attribute__((packed));
132
133struct st_nv_interrupt_end_data {
134 u64 when;
135 u32 device;
136 u8 __unused[4];
137} __attribute__((packed));
82 138
83#define DATA(x) struct st_ ## x ## _data x; 139#define DATA(x) struct st_ ## x ## _data x;
84 140
85typedef enum { 141typedef enum {
86 ST_NAME = 1, /* Start at one, so that we can spot 142 ST_NAME = 1, /* Start at one, so that we can spot
87 * uninitialized records. */ 143 * uninitialized records. */
88 ST_PARAM, 144 ST_PARAM,
89 ST_RELEASE, 145 ST_RELEASE,
90 ST_ASSIGNED, 146 ST_ASSIGNED,
@@ -94,7 +150,16 @@ typedef enum {
94 ST_BLOCK, 150 ST_BLOCK,
95 ST_RESUME, 151 ST_RESUME,
96 ST_ACTION, 152 ST_ACTION,
97 ST_SYS_RELEASE 153 ST_SYS_RELEASE,
154 ST_TASKLET_RELEASE,
155 ST_TASKLET_BEGIN,
156 ST_TASKLET_END,
157 ST_WORK_RELEASE,
158 ST_WORK_BEGIN,
159 ST_WORK_END,
160 ST_EFF_PRIO_CHANGE,
161 ST_NV_INTERRUPT_BEGIN,
162 ST_NV_INTERRUPT_END,
98} st_event_record_type_t; 163} st_event_record_type_t;
99 164
100struct st_event_record { 165struct st_event_record {
@@ -113,8 +178,17 @@ struct st_event_record {
113 DATA(resume); 178 DATA(resume);
114 DATA(action); 179 DATA(action);
115 DATA(sys_release); 180 DATA(sys_release);
181 DATA(tasklet_release);
182 DATA(tasklet_begin);
183 DATA(tasklet_end);
184 DATA(work_release);
185 DATA(work_begin);
186 DATA(work_end);
187 DATA(effective_priority_change);
188 DATA(nv_interrupt_begin);
189 DATA(nv_interrupt_end);
116 } data; 190 } data;
117}; 191} __attribute__((packed));
118 192
119#undef DATA 193#undef DATA
120 194
@@ -129,6 +203,8 @@ struct st_event_record {
129 ft_event1(id, callback, task) 203 ft_event1(id, callback, task)
130#define SCHED_TRACE2(id, callback, task, xtra) \ 204#define SCHED_TRACE2(id, callback, task, xtra) \
131 ft_event2(id, callback, task, xtra) 205 ft_event2(id, callback, task, xtra)
206#define SCHED_TRACE3(id, callback, task, xtra1, xtra2) \
207 ft_event3(id, callback, task, xtra1, xtra2)
132 208
133/* provide prototypes; needed on sparc64 */ 209/* provide prototypes; needed on sparc64 */
134#ifndef NO_TASK_TRACE_DECLS 210#ifndef NO_TASK_TRACE_DECLS
@@ -155,12 +231,45 @@ feather_callback void do_sched_trace_action(unsigned long id,
155feather_callback void do_sched_trace_sys_release(unsigned long id, 231feather_callback void do_sched_trace_sys_release(unsigned long id,
156 lt_t* start); 232 lt_t* start);
157 233
234
235feather_callback void do_sched_trace_tasklet_release(unsigned long id,
236 struct task_struct* owner);
237feather_callback void do_sched_trace_tasklet_begin(unsigned long id,
238 struct task_struct* owner);
239feather_callback void do_sched_trace_tasklet_end(unsigned long id,
240 struct task_struct* owner,
241 unsigned long flushed);
242
243feather_callback void do_sched_trace_work_release(unsigned long id,
244 struct task_struct* owner);
245feather_callback void do_sched_trace_work_begin(unsigned long id,
246 struct task_struct* owner,
247 struct task_struct* exe);
248feather_callback void do_sched_trace_work_end(unsigned long id,
249 struct task_struct* owner,
250 struct task_struct* exe,
251 unsigned long flushed);
252
253feather_callback void do_sched_trace_eff_prio_change(unsigned long id,
254 struct task_struct* task,
255 struct task_struct* inh);
256
257feather_callback void do_sched_trace_nv_interrupt_begin(unsigned long id,
258 u32 device);
259feather_callback void do_sched_trace_nv_interrupt_end(unsigned long id,
260 unsigned long unused);
261
262
263/* returns true if we're tracing an interrupt on current CPU */
264/* int is_interrupt_tracing_active(void); */
265
158#endif 266#endif
159 267
160#else 268#else
161 269
162#define SCHED_TRACE(id, callback, task) /* no tracing */ 270#define SCHED_TRACE(id, callback, task) /* no tracing */
163#define SCHED_TRACE2(id, callback, task, xtra) /* no tracing */ 271#define SCHED_TRACE2(id, callback, task, xtra) /* no tracing */
272#define SCHED_TRACE3(id, callback, task, xtra1, xtra2)
164 273
165#endif 274#endif
166 275
@@ -193,6 +302,35 @@ feather_callback void do_sched_trace_sys_release(unsigned long id,
193 SCHED_TRACE(SCHED_TRACE_BASE_ID + 10, do_sched_trace_sys_release, when) 302 SCHED_TRACE(SCHED_TRACE_BASE_ID + 10, do_sched_trace_sys_release, when)
194 303
195 304
305#define sched_trace_tasklet_release(t) \
306 SCHED_TRACE(SCHED_TRACE_BASE_ID + 11, do_sched_trace_tasklet_release, t)
307
308#define sched_trace_tasklet_begin(t) \
309 SCHED_TRACE(SCHED_TRACE_BASE_ID + 12, do_sched_trace_tasklet_begin, t)
310
311#define sched_trace_tasklet_end(t, flushed) \
312 SCHED_TRACE2(SCHED_TRACE_BASE_ID + 13, do_sched_trace_tasklet_end, t, flushed)
313
314
315#define sched_trace_work_release(t) \
316 SCHED_TRACE(SCHED_TRACE_BASE_ID + 14, do_sched_trace_work_release, t)
317
318#define sched_trace_work_begin(t, e) \
319 SCHED_TRACE2(SCHED_TRACE_BASE_ID + 15, do_sched_trace_work_begin, t, e)
320
321#define sched_trace_work_end(t, e, flushed) \
322 SCHED_TRACE3(SCHED_TRACE_BASE_ID + 16, do_sched_trace_work_end, t, e, flushed)
323
324
325#define sched_trace_eff_prio_change(t, inh) \
326 SCHED_TRACE2(SCHED_TRACE_BASE_ID + 17, do_sched_trace_eff_prio_change, t, inh)
327
328
329#define sched_trace_nv_interrupt_begin(d) \
330 SCHED_TRACE(SCHED_TRACE_BASE_ID + 18, do_sched_trace_nv_interrupt_begin, d)
331#define sched_trace_nv_interrupt_end() \
332 SCHED_TRACE(SCHED_TRACE_BASE_ID + 19, do_sched_trace_nv_interrupt_end, 0ul)
333
196#define sched_trace_quantum_boundary() /* NOT IMPLEMENTED */ 334#define sched_trace_quantum_boundary() /* NOT IMPLEMENTED */
197 335
198#endif /* __KERNEL__ */ 336#endif /* __KERNEL__ */
diff --git a/include/litmus/sched_trace_external.h b/include/litmus/sched_trace_external.h
new file mode 100644
index 000000000000..c2c872639880
--- /dev/null
+++ b/include/litmus/sched_trace_external.h
@@ -0,0 +1,42 @@
1/*
2 * sched_trace.h -- record scheduler events to a byte stream for offline analysis.
3 */
4#ifndef _LINUX_SCHED_TRACE_EXTERNAL_H_
5#define _LINUX_SCHED_TRACE_EXTERNAL_H_
6
7extern void __sched_trace_tasklet_begin_external(struct task_struct* t);
8static inline void sched_trace_tasklet_begin_external(struct task_struct* t)
9{
10 __sched_trace_tasklet_begin_external(t);
11}
12
13extern void __sched_trace_tasklet_end_external(struct task_struct* t, unsigned long flushed);
14static inline void sched_trace_tasklet_end_external(struct task_struct* t, unsigned long flushed)
15{
16 __sched_trace_tasklet_end_external(t, flushed);
17}
18
19extern void __sched_trace_work_begin_external(struct task_struct* t, struct task_struct* e);
20static inline void sched_trace_work_begin_external(struct task_struct* t, struct task_struct* e)
21{
22 __sched_trace_work_begin_external(t, e);
23}
24
25extern void __sched_trace_work_end_external(struct task_struct* t, struct task_struct* e, unsigned long f);
26static inline void sched_trace_work_end_external(struct task_struct* t, struct task_struct* e, unsigned long f)
27{
28 __sched_trace_work_end_external(t, e, f);
29}
30
31extern void __sched_trace_nv_interrupt_begin_external(u32 device);
32static inline void sched_trace_nv_interrupt_begin_external(u32 device)
33{
34 __sched_trace_nv_interrupt_begin_external(device);
35}
36
37extern void __sched_trace_nv_interrupt_end_external(void);
38static inline void sched_trace_nv_interrupt_end_external(void)
39{
40 __sched_trace_nv_interrupt_end_external();
41}
42#endif
diff --git a/include/litmus/unistd_32.h b/include/litmus/unistd_32.h
index 94264c27d9ac..c6efc4c40af2 100644
--- a/include/litmus/unistd_32.h
+++ b/include/litmus/unistd_32.h
@@ -17,5 +17,6 @@
17#define __NR_wait_for_ts_release __LSC(9) 17#define __NR_wait_for_ts_release __LSC(9)
18#define __NR_release_ts __LSC(10) 18#define __NR_release_ts __LSC(10)
19#define __NR_null_call __LSC(11) 19#define __NR_null_call __LSC(11)
20#define __NR_register_nv_device __LSC(12)
20 21
21#define NR_litmus_syscalls 12 22#define NR_litmus_syscalls 13
diff --git a/include/litmus/unistd_64.h b/include/litmus/unistd_64.h
index d5ced0d2642c..b44a7c33bdf8 100644
--- a/include/litmus/unistd_64.h
+++ b/include/litmus/unistd_64.h
@@ -29,5 +29,8 @@ __SYSCALL(__NR_wait_for_ts_release, sys_wait_for_ts_release)
29__SYSCALL(__NR_release_ts, sys_release_ts) 29__SYSCALL(__NR_release_ts, sys_release_ts)
30#define __NR_null_call __LSC(11) 30#define __NR_null_call __LSC(11)
31__SYSCALL(__NR_null_call, sys_null_call) 31__SYSCALL(__NR_null_call, sys_null_call)
32#define __NR_register_nv_device __LSC(12)
33__SYSCALL(__NR_register_nv_device, sys_register_nv_device)
32 34
33#define NR_litmus_syscalls 12 35
36#define NR_litmus_syscalls 13
diff --git a/kernel/lockdep.c b/kernel/lockdep.c
index f2852a510232..ebff2cf715c5 100644
--- a/kernel/lockdep.c
+++ b/kernel/lockdep.c
@@ -530,7 +530,7 @@ static void print_lock(struct held_lock *hlock)
530 print_ip_sym(hlock->acquire_ip); 530 print_ip_sym(hlock->acquire_ip);
531} 531}
532 532
533static void lockdep_print_held_locks(struct task_struct *curr) 533void lockdep_print_held_locks(struct task_struct *curr)
534{ 534{
535 int i, depth = curr->lockdep_depth; 535 int i, depth = curr->lockdep_depth;
536 536
@@ -546,6 +546,7 @@ static void lockdep_print_held_locks(struct task_struct *curr)
546 print_lock(curr->held_locks + i); 546 print_lock(curr->held_locks + i);
547 } 547 }
548} 548}
549EXPORT_SYMBOL(lockdep_print_held_locks);
549 550
550static void print_kernel_version(void) 551static void print_kernel_version(void)
551{ 552{
diff --git a/kernel/mutex.c b/kernel/mutex.c
index 200407c1502f..435685ecd068 100644
--- a/kernel/mutex.c
+++ b/kernel/mutex.c
@@ -496,3 +496,144 @@ int atomic_dec_and_mutex_lock(atomic_t *cnt, struct mutex *lock)
496 return 1; 496 return 1;
497} 497}
498EXPORT_SYMBOL(atomic_dec_and_mutex_lock); 498EXPORT_SYMBOL(atomic_dec_and_mutex_lock);
499
500
501
502
503
504
505
506
507//__mutex_lock_common(lock, TASK_UNINTERRUPTIBLE, 0, _RET_IP_);
508
509void mutex_lock_sfx(struct mutex *lock,
510 side_effect_t pre, unsigned long pre_arg,
511 side_effect_t post, unsigned long post_arg)
512{
513 long state = TASK_UNINTERRUPTIBLE;
514 unsigned int subclass = 0;
515 unsigned long ip = _RET_IP_;
516
517
518 struct task_struct *task = current;
519 struct mutex_waiter waiter;
520 unsigned long flags;
521
522 preempt_disable();
523 mutex_acquire(&lock->dep_map, subclass, 0, ip);
524
525 spin_lock_mutex(&lock->wait_lock, flags);
526
527 if(pre)
528 {
529 if(unlikely(pre(pre_arg)))
530 {
531 // this will fuck with lockdep's CONFIG_PROVE_LOCKING...
532 spin_unlock_mutex(&lock->wait_lock, flags);
533 preempt_enable();
534 return;
535 }
536 }
537
538 debug_mutex_lock_common(lock, &waiter);
539 debug_mutex_add_waiter(lock, &waiter, task_thread_info(task));
540
541 /* add waiting tasks to the end of the waitqueue (FIFO): */
542 list_add_tail(&waiter.list, &lock->wait_list);
543 waiter.task = task;
544
545 if (atomic_xchg(&lock->count, -1) == 1)
546 goto done;
547
548 lock_contended(&lock->dep_map, ip);
549
550 for (;;) {
551 /*
552 * Lets try to take the lock again - this is needed even if
553 * we get here for the first time (shortly after failing to
554 * acquire the lock), to make sure that we get a wakeup once
555 * it's unlocked. Later on, if we sleep, this is the
556 * operation that gives us the lock. We xchg it to -1, so
557 * that when we release the lock, we properly wake up the
558 * other waiters:
559 */
560 if (atomic_xchg(&lock->count, -1) == 1)
561 break;
562
563 __set_task_state(task, state);
564
565 /* didnt get the lock, go to sleep: */
566 spin_unlock_mutex(&lock->wait_lock, flags);
567 preempt_enable_no_resched();
568 schedule();
569 preempt_disable();
570 spin_lock_mutex(&lock->wait_lock, flags);
571 }
572
573done:
574 lock_acquired(&lock->dep_map, ip);
575 /* got the lock - rejoice! */
576 mutex_remove_waiter(lock, &waiter, current_thread_info());
577 mutex_set_owner(lock);
578
579 /* set it to 0 if there are no waiters left: */
580 if (likely(list_empty(&lock->wait_list)))
581 atomic_set(&lock->count, 0);
582
583 if(post)
584 post(post_arg);
585
586 spin_unlock_mutex(&lock->wait_lock, flags);
587
588 debug_mutex_free_waiter(&waiter);
589 preempt_enable();
590
591 //return 0;
592}
593EXPORT_SYMBOL(mutex_lock_sfx);
594
595
596
597//__mutex_unlock_common_slowpath(lock_count, 1);
598
599void mutex_unlock_sfx(struct mutex *lock,
600 side_effect_t pre, unsigned long pre_arg,
601 side_effect_t post, unsigned long post_arg)
602{
603 //struct mutex *lock = container_of(lock_count, struct mutex, count);
604 unsigned long flags;
605
606 spin_lock_mutex(&lock->wait_lock, flags);
607
608 if(pre)
609 pre(pre_arg);
610
611 //mutex_release(&lock->dep_map, nested, _RET_IP_);
612 mutex_release(&lock->dep_map, 1, _RET_IP_);
613 debug_mutex_unlock(lock);
614
615 /*
616 * some architectures leave the lock unlocked in the fastpath failure
617 * case, others need to leave it locked. In the later case we have to
618 * unlock it here
619 */
620 if (__mutex_slowpath_needs_to_unlock())
621 atomic_set(&lock->count, 1);
622
623 if (!list_empty(&lock->wait_list)) {
624 /* get the first entry from the wait-list: */
625 struct mutex_waiter *waiter =
626 list_entry(lock->wait_list.next,
627 struct mutex_waiter, list);
628
629 debug_mutex_wake_waiter(lock, waiter);
630
631 wake_up_process(waiter->task);
632 }
633
634 if(post)
635 post(post_arg);
636
637 spin_unlock_mutex(&lock->wait_lock, flags);
638}
639EXPORT_SYMBOL(mutex_unlock_sfx);
diff --git a/kernel/sched.c b/kernel/sched.c
index c5d775079027..3162605ffc91 100644
--- a/kernel/sched.c
+++ b/kernel/sched.c
@@ -82,6 +82,10 @@
82#include <litmus/sched_trace.h> 82#include <litmus/sched_trace.h>
83#include <litmus/trace.h> 83#include <litmus/trace.h>
84 84
85#ifdef CONFIG_LITMUS_SOFTIRQD
86#include <litmus/litmus_softirq.h>
87#endif
88
85static void litmus_tick(struct rq*, struct task_struct*); 89static void litmus_tick(struct rq*, struct task_struct*);
86 90
87#define CREATE_TRACE_POINTS 91#define CREATE_TRACE_POINTS
@@ -3789,6 +3793,7 @@ pick_next_task(struct rq *rq)
3789 } 3793 }
3790} 3794}
3791 3795
3796
3792/* 3797/*
3793 * schedule() is the main scheduler function. 3798 * schedule() is the main scheduler function.
3794 */ 3799 */
@@ -3807,6 +3812,10 @@ need_resched:
3807 rcu_note_context_switch(cpu); 3812 rcu_note_context_switch(cpu);
3808 prev = rq->curr; 3813 prev = rq->curr;
3809 3814
3815#ifdef CONFIG_LITMUS_SOFTIRQD
3816 release_klitirqd_lock(prev);
3817#endif
3818
3810 release_kernel_lock(prev); 3819 release_kernel_lock(prev);
3811need_resched_nonpreemptible: 3820need_resched_nonpreemptible:
3812 TS_SCHED_START; 3821 TS_SCHED_START;
@@ -3882,15 +3891,20 @@ need_resched_nonpreemptible:
3882 3891
3883 if (sched_state_validate_switch() || unlikely(reacquire_kernel_lock(prev))) 3892 if (sched_state_validate_switch() || unlikely(reacquire_kernel_lock(prev)))
3884 goto need_resched_nonpreemptible; 3893 goto need_resched_nonpreemptible;
3885 3894
3886 preempt_enable_no_resched(); 3895 preempt_enable_no_resched();
3896
3887 if (need_resched()) 3897 if (need_resched())
3888 goto need_resched; 3898 goto need_resched;
3889 3899
3900 reacquire_klitirqd_lock(prev);
3901
3890 srp_ceiling_block(); 3902 srp_ceiling_block();
3891} 3903}
3892EXPORT_SYMBOL(schedule); 3904EXPORT_SYMBOL(schedule);
3893 3905
3906
3907
3894#ifdef CONFIG_MUTEX_SPIN_ON_OWNER 3908#ifdef CONFIG_MUTEX_SPIN_ON_OWNER
3895/* 3909/*
3896 * Look out! "owner" is an entirely speculative pointer 3910 * Look out! "owner" is an entirely speculative pointer
@@ -4051,6 +4065,7 @@ static void __wake_up_common(wait_queue_head_t *q, unsigned int mode,
4051 } 4065 }
4052} 4066}
4053 4067
4068
4054/** 4069/**
4055 * __wake_up - wake up threads blocked on a waitqueue. 4070 * __wake_up - wake up threads blocked on a waitqueue.
4056 * @q: the waitqueue 4071 * @q: the waitqueue
@@ -4236,6 +4251,12 @@ void __sched wait_for_completion(struct completion *x)
4236} 4251}
4237EXPORT_SYMBOL(wait_for_completion); 4252EXPORT_SYMBOL(wait_for_completion);
4238 4253
4254void __sched __wait_for_completion_locked(struct completion *x)
4255{
4256 do_wait_for_common(x, MAX_SCHEDULE_TIMEOUT, TASK_UNINTERRUPTIBLE);
4257}
4258EXPORT_SYMBOL(__wait_for_completion_locked);
4259
4239/** 4260/**
4240 * wait_for_completion_timeout: - waits for completion of a task (w/timeout) 4261 * wait_for_completion_timeout: - waits for completion of a task (w/timeout)
4241 * @x: holds the state of this particular completion 4262 * @x: holds the state of this particular completion
diff --git a/kernel/semaphore.c b/kernel/semaphore.c
index 94a62c0d4ade..c947a046a6d7 100644
--- a/kernel/semaphore.c
+++ b/kernel/semaphore.c
@@ -33,11 +33,11 @@
33#include <linux/spinlock.h> 33#include <linux/spinlock.h>
34#include <linux/ftrace.h> 34#include <linux/ftrace.h>
35 35
36static noinline void __down(struct semaphore *sem); 36noinline void __down(struct semaphore *sem);
37static noinline int __down_interruptible(struct semaphore *sem); 37static noinline int __down_interruptible(struct semaphore *sem);
38static noinline int __down_killable(struct semaphore *sem); 38static noinline int __down_killable(struct semaphore *sem);
39static noinline int __down_timeout(struct semaphore *sem, long jiffies); 39static noinline int __down_timeout(struct semaphore *sem, long jiffies);
40static noinline void __up(struct semaphore *sem); 40noinline void __up(struct semaphore *sem);
41 41
42/** 42/**
43 * down - acquire the semaphore 43 * down - acquire the semaphore
@@ -190,11 +190,13 @@ EXPORT_SYMBOL(up);
190 190
191/* Functions for the contended case */ 191/* Functions for the contended case */
192 192
193/*
193struct semaphore_waiter { 194struct semaphore_waiter {
194 struct list_head list; 195 struct list_head list;
195 struct task_struct *task; 196 struct task_struct *task;
196 int up; 197 int up;
197}; 198};
199 */
198 200
199/* 201/*
200 * Because this function is inlined, the 'state' parameter will be 202 * Because this function is inlined, the 'state' parameter will be
@@ -233,10 +235,12 @@ static inline int __sched __down_common(struct semaphore *sem, long state,
233 return -EINTR; 235 return -EINTR;
234} 236}
235 237
236static noinline void __sched __down(struct semaphore *sem) 238noinline void __sched __down(struct semaphore *sem)
237{ 239{
238 __down_common(sem, TASK_UNINTERRUPTIBLE, MAX_SCHEDULE_TIMEOUT); 240 __down_common(sem, TASK_UNINTERRUPTIBLE, MAX_SCHEDULE_TIMEOUT);
239} 241}
242EXPORT_SYMBOL(__down);
243
240 244
241static noinline int __sched __down_interruptible(struct semaphore *sem) 245static noinline int __sched __down_interruptible(struct semaphore *sem)
242{ 246{
@@ -253,7 +257,7 @@ static noinline int __sched __down_timeout(struct semaphore *sem, long jiffies)
253 return __down_common(sem, TASK_UNINTERRUPTIBLE, jiffies); 257 return __down_common(sem, TASK_UNINTERRUPTIBLE, jiffies);
254} 258}
255 259
256static noinline void __sched __up(struct semaphore *sem) 260noinline void __sched __up(struct semaphore *sem)
257{ 261{
258 struct semaphore_waiter *waiter = list_first_entry(&sem->wait_list, 262 struct semaphore_waiter *waiter = list_first_entry(&sem->wait_list,
259 struct semaphore_waiter, list); 263 struct semaphore_waiter, list);
@@ -261,3 +265,4 @@ static noinline void __sched __up(struct semaphore *sem)
261 waiter->up = 1; 265 waiter->up = 1;
262 wake_up_process(waiter->task); 266 wake_up_process(waiter->task);
263} 267}
268EXPORT_SYMBOL(__up); \ No newline at end of file
diff --git a/kernel/softirq.c b/kernel/softirq.c
index 07b4f1b1a73a..be4b8fab3637 100644
--- a/kernel/softirq.c
+++ b/kernel/softirq.c
@@ -29,6 +29,14 @@
29#include <trace/events/irq.h> 29#include <trace/events/irq.h>
30 30
31#include <asm/irq.h> 31#include <asm/irq.h>
32
33#include <litmus/litmus.h>
34#include <litmus/sched_trace.h>
35
36#ifdef CONFIG_LITMUS_NVIDIA
37#include <litmus/nvidia_info.h>
38#endif
39
32/* 40/*
33 - No shared variables, all the data are CPU local. 41 - No shared variables, all the data are CPU local.
34 - If a softirq needs serialization, let it serialize itself 42 - If a softirq needs serialization, let it serialize itself
@@ -54,7 +62,7 @@ EXPORT_SYMBOL(irq_stat);
54 62
55static struct softirq_action softirq_vec[NR_SOFTIRQS] __cacheline_aligned_in_smp; 63static struct softirq_action softirq_vec[NR_SOFTIRQS] __cacheline_aligned_in_smp;
56 64
57static DEFINE_PER_CPU(struct task_struct *, ksoftirqd); 65static DEFINE_PER_CPU(struct task_struct *, ksoftirqd) = NULL;
58 66
59char *softirq_to_name[NR_SOFTIRQS] = { 67char *softirq_to_name[NR_SOFTIRQS] = {
60 "HI", "TIMER", "NET_TX", "NET_RX", "BLOCK", "BLOCK_IOPOLL", 68 "HI", "TIMER", "NET_TX", "NET_RX", "BLOCK", "BLOCK_IOPOLL",
@@ -177,6 +185,7 @@ void local_bh_enable_ip(unsigned long ip)
177} 185}
178EXPORT_SYMBOL(local_bh_enable_ip); 186EXPORT_SYMBOL(local_bh_enable_ip);
179 187
188
180/* 189/*
181 * We restart softirq processing MAX_SOFTIRQ_RESTART times, 190 * We restart softirq processing MAX_SOFTIRQ_RESTART times,
182 * and we fall back to softirqd after that. 191 * and we fall back to softirqd after that.
@@ -187,34 +196,30 @@ EXPORT_SYMBOL(local_bh_enable_ip);
187 * should not be able to lock up the box. 196 * should not be able to lock up the box.
188 */ 197 */
189#define MAX_SOFTIRQ_RESTART 10 198#define MAX_SOFTIRQ_RESTART 10
190 199static void ____do_softirq(void)
191asmlinkage void __do_softirq(void)
192{ 200{
193 struct softirq_action *h;
194 __u32 pending; 201 __u32 pending;
195 int max_restart = MAX_SOFTIRQ_RESTART; 202
203 struct softirq_action *h;
196 int cpu; 204 int cpu;
197 205
198 pending = local_softirq_pending(); 206 pending = local_softirq_pending();
207
199 account_system_vtime(current); 208 account_system_vtime(current);
200 209
201 __local_bh_disable((unsigned long)__builtin_return_address(0));
202 lockdep_softirq_enter();
203
204 cpu = smp_processor_id(); 210 cpu = smp_processor_id();
205restart:
206 /* Reset the pending bitmask before enabling irqs */
207 set_softirq_pending(0);
208 211
212 set_softirq_pending(0);
213
209 local_irq_enable(); 214 local_irq_enable();
210 215
211 h = softirq_vec; 216 h = softirq_vec;
212 217
213 do { 218 do {
214 if (pending & 1) { 219 if (pending & 1) {
215 int prev_count = preempt_count(); 220 int prev_count = preempt_count();
216 kstat_incr_softirqs_this_cpu(h - softirq_vec); 221 kstat_incr_softirqs_this_cpu(h - softirq_vec);
217 222
218 trace_softirq_entry(h, softirq_vec); 223 trace_softirq_entry(h, softirq_vec);
219 h->action(h); 224 h->action(h);
220 trace_softirq_exit(h, softirq_vec); 225 trace_softirq_exit(h, softirq_vec);
@@ -226,26 +231,70 @@ restart:
226 h->action, prev_count, preempt_count()); 231 h->action, prev_count, preempt_count());
227 preempt_count() = prev_count; 232 preempt_count() = prev_count;
228 } 233 }
229 234
230 rcu_bh_qs(cpu); 235 rcu_bh_qs(cpu);
231 } 236 }
232 h++; 237 h++;
233 pending >>= 1; 238 pending >>= 1;
234 } while (pending); 239 } while (pending);
235 240
236 local_irq_disable(); 241 local_irq_disable();
242}
243
244static void ___do_softirq(void)
245{
246 __u32 pending;
247
248 //struct softirq_action *h;
249 int max_restart = MAX_SOFTIRQ_RESTART;
250 //int cpu;
251
252 pending = local_softirq_pending();
253
254restart:
255 ____do_softirq();
237 256
238 pending = local_softirq_pending(); 257 pending = local_softirq_pending();
239 if (pending && --max_restart) 258 if (pending && --max_restart)
240 goto restart; 259 goto restart;
241 260
242 if (pending) 261 if (pending)
262 {
243 wakeup_softirqd(); 263 wakeup_softirqd();
264 }
265}
244 266
267asmlinkage void __do_softirq(void)
268{
269#ifdef LITMUS_THREAD_ALL_SOFTIRQ
270 /* Skip straight to wakeup_softirqd() if we're using
271 LITMUS_THREAD_ALL_SOFTIRQ (unless there's really high prio-stuff waiting.). */
272 struct task_struct *tsk = __get_cpu_var(ksoftirqd);
273
274 if(tsk)
275 {
276 __u32 pending = local_softirq_pending();
277 const __u32 high_prio_softirq = (1<<HI_SOFTIRQ) | (1<<TIMER_SOFTIRQ) | (1<<HRTIMER_SOFTIRQ);
278 if(pending && !(pending & high_prio_softirq))
279 {
280 wakeup_softirqd();
281 return;
282 }
283 }
284#endif
285
286 /*
287 * 'immediate' softirq execution:
288 */
289 __local_bh_disable((unsigned long)__builtin_return_address(0));
290 lockdep_softirq_enter();
291
292 ___do_softirq();
293
245 lockdep_softirq_exit(); 294 lockdep_softirq_exit();
246 295
247 account_system_vtime(current); 296 account_system_vtime(current);
248 _local_bh_enable(); 297 _local_bh_enable();
249} 298}
250 299
251#ifndef __ARCH_HAS_DO_SOFTIRQ 300#ifndef __ARCH_HAS_DO_SOFTIRQ
@@ -357,8 +406,64 @@ struct tasklet_head
357static DEFINE_PER_CPU(struct tasklet_head, tasklet_vec); 406static DEFINE_PER_CPU(struct tasklet_head, tasklet_vec);
358static DEFINE_PER_CPU(struct tasklet_head, tasklet_hi_vec); 407static DEFINE_PER_CPU(struct tasklet_head, tasklet_hi_vec);
359 408
409
360void __tasklet_schedule(struct tasklet_struct *t) 410void __tasklet_schedule(struct tasklet_struct *t)
361{ 411{
412#ifdef CONFIG_LITMUS_NVIDIA
413 if(is_nvidia_func(t->func))
414 {
415 u32 nvidia_device = get_tasklet_nv_device_num(t);
416 // TRACE("%s: Handling NVIDIA tasklet for device\t%u\tat\t%llu\n",
417 // __FUNCTION__, nvidia_device,litmus_clock());
418
419 unsigned long flags;
420 struct task_struct* device_owner;
421
422 lock_nv_registry(nvidia_device, &flags);
423
424 device_owner = get_nv_device_owner(nvidia_device);
425
426 if(device_owner==NULL)
427 {
428 t->owner = NULL;
429 }
430 else
431 {
432 if(is_realtime(device_owner))
433 {
434 TRACE("%s: Handling NVIDIA tasklet for device %u at %llu\n",
435 __FUNCTION__, nvidia_device,litmus_clock());
436 TRACE("%s: the owner task %d of NVIDIA Device %u is RT-task\n",
437 __FUNCTION__,device_owner->pid,nvidia_device);
438
439 t->owner = device_owner;
440 sched_trace_tasklet_release(t->owner);
441 if(likely(_litmus_tasklet_schedule(t,nvidia_device)))
442 {
443 unlock_nv_registry(nvidia_device, &flags);
444 return;
445 }
446 else
447 {
448 t->owner = NULL; /* fall through to normal scheduling */
449 }
450 }
451 else
452 {
453 t->owner = NULL;
454 }
455 }
456 unlock_nv_registry(nvidia_device, &flags);
457 }
458#endif
459
460 ___tasklet_schedule(t);
461}
462EXPORT_SYMBOL(__tasklet_schedule);
463
464
465void ___tasklet_schedule(struct tasklet_struct *t)
466{
362 unsigned long flags; 467 unsigned long flags;
363 468
364 local_irq_save(flags); 469 local_irq_save(flags);
@@ -368,11 +473,65 @@ void __tasklet_schedule(struct tasklet_struct *t)
368 raise_softirq_irqoff(TASKLET_SOFTIRQ); 473 raise_softirq_irqoff(TASKLET_SOFTIRQ);
369 local_irq_restore(flags); 474 local_irq_restore(flags);
370} 475}
476EXPORT_SYMBOL(___tasklet_schedule);
371 477
372EXPORT_SYMBOL(__tasklet_schedule);
373 478
374void __tasklet_hi_schedule(struct tasklet_struct *t) 479void __tasklet_hi_schedule(struct tasklet_struct *t)
375{ 480{
481#ifdef CONFIG_LITMUS_NVIDIA
482 if(is_nvidia_func(t->func))
483 {
484 u32 nvidia_device = get_tasklet_nv_device_num(t);
485 // TRACE("%s: Handling NVIDIA tasklet for device\t%u\tat\t%llu\n",
486 // __FUNCTION__, nvidia_device,litmus_clock());
487
488 unsigned long flags;
489 struct task_struct* device_owner;
490
491 lock_nv_registry(nvidia_device, &flags);
492
493 device_owner = get_nv_device_owner(nvidia_device);
494
495 if(device_owner==NULL)
496 {
497 t->owner = NULL;
498 }
499 else
500 {
501 if( is_realtime(device_owner))
502 {
503 TRACE("%s: Handling NVIDIA tasklet for device %u\tat %llu\n",
504 __FUNCTION__, nvidia_device,litmus_clock());
505 TRACE("%s: the owner task %d of NVIDIA Device %u is RT-task\n",
506 __FUNCTION__,device_owner->pid,nvidia_device);
507
508 t->owner = device_owner;
509 sched_trace_tasklet_release(t->owner);
510 if(likely(_litmus_tasklet_hi_schedule(t,nvidia_device)))
511 {
512 unlock_nv_registry(nvidia_device, &flags);
513 return;
514 }
515 else
516 {
517 t->owner = NULL; /* fall through to normal scheduling */
518 }
519 }
520 else
521 {
522 t->owner = NULL;
523 }
524 }
525 unlock_nv_registry(nvidia_device, &flags);
526 }
527#endif
528
529 ___tasklet_hi_schedule(t);
530}
531EXPORT_SYMBOL(__tasklet_hi_schedule);
532
533void ___tasklet_hi_schedule(struct tasklet_struct* t)
534{
376 unsigned long flags; 535 unsigned long flags;
377 536
378 local_irq_save(flags); 537 local_irq_save(flags);
@@ -382,19 +541,72 @@ void __tasklet_hi_schedule(struct tasklet_struct *t)
382 raise_softirq_irqoff(HI_SOFTIRQ); 541 raise_softirq_irqoff(HI_SOFTIRQ);
383 local_irq_restore(flags); 542 local_irq_restore(flags);
384} 543}
385 544EXPORT_SYMBOL(___tasklet_hi_schedule);
386EXPORT_SYMBOL(__tasklet_hi_schedule);
387 545
388void __tasklet_hi_schedule_first(struct tasklet_struct *t) 546void __tasklet_hi_schedule_first(struct tasklet_struct *t)
389{ 547{
390 BUG_ON(!irqs_disabled()); 548 BUG_ON(!irqs_disabled());
549#ifdef CONFIG_LITMUS_NVIDIA
550 if(is_nvidia_func(t->func))
551 {
552 u32 nvidia_device = get_tasklet_nv_device_num(t);
553 // TRACE("%s: Handling NVIDIA tasklet for device\t%u\tat\t%llu\n",
554 // __FUNCTION__, nvidia_device,litmus_clock());
555 unsigned long flags;
556 struct task_struct* device_owner;
557
558 lock_nv_registry(nvidia_device, &flags);
559
560 device_owner = get_nv_device_owner(nvidia_device);
561
562 if(device_owner==NULL)
563 {
564 t->owner = NULL;
565 }
566 else
567 {
568 if(is_realtime(device_owner))
569 {
570 TRACE("%s: Handling NVIDIA tasklet for device %u at %llu\n",
571 __FUNCTION__, nvidia_device,litmus_clock());
572
573 TRACE("%s: the owner task %d of NVIDIA Device %u is RT-task\n",
574 __FUNCTION__,device_owner->pid,nvidia_device);
575
576 t->owner = device_owner;
577 sched_trace_tasklet_release(t->owner);
578 if(likely(_litmus_tasklet_hi_schedule_first(t,nvidia_device)))
579 {
580 unlock_nv_registry(nvidia_device, &flags);
581 return;
582 }
583 else
584 {
585 t->owner = NULL; /* fall through to normal scheduling */
586 }
587 }
588 else
589 {
590 t->owner = NULL;
591 }
592 }
593 unlock_nv_registry(nvidia_device, &flags);
594 }
595#endif
596
597 ___tasklet_hi_schedule_first(t);
598}
599EXPORT_SYMBOL(__tasklet_hi_schedule_first);
600
601void ___tasklet_hi_schedule_first(struct tasklet_struct* t)
602{
603 BUG_ON(!irqs_disabled());
391 604
392 t->next = __get_cpu_var(tasklet_hi_vec).head; 605 t->next = __get_cpu_var(tasklet_hi_vec).head;
393 __get_cpu_var(tasklet_hi_vec).head = t; 606 __get_cpu_var(tasklet_hi_vec).head = t;
394 __raise_softirq_irqoff(HI_SOFTIRQ); 607 __raise_softirq_irqoff(HI_SOFTIRQ);
395} 608}
396 609EXPORT_SYMBOL(___tasklet_hi_schedule_first);
397EXPORT_SYMBOL(__tasklet_hi_schedule_first);
398 610
399static void tasklet_action(struct softirq_action *a) 611static void tasklet_action(struct softirq_action *a)
400{ 612{
@@ -450,6 +662,7 @@ static void tasklet_hi_action(struct softirq_action *a)
450 if (!atomic_read(&t->count)) { 662 if (!atomic_read(&t->count)) {
451 if (!test_and_clear_bit(TASKLET_STATE_SCHED, &t->state)) 663 if (!test_and_clear_bit(TASKLET_STATE_SCHED, &t->state))
452 BUG(); 664 BUG();
665
453 t->func(t->data); 666 t->func(t->data);
454 tasklet_unlock(t); 667 tasklet_unlock(t);
455 continue; 668 continue;
@@ -473,8 +686,13 @@ void tasklet_init(struct tasklet_struct *t,
473 t->next = NULL; 686 t->next = NULL;
474 t->state = 0; 687 t->state = 0;
475 atomic_set(&t->count, 0); 688 atomic_set(&t->count, 0);
689
476 t->func = func; 690 t->func = func;
477 t->data = data; 691 t->data = data;
692
693#ifdef CONFIG_LITMUS_SOFTIRQD
694 t->owner = NULL;
695#endif
478} 696}
479 697
480EXPORT_SYMBOL(tasklet_init); 698EXPORT_SYMBOL(tasklet_init);
@@ -489,6 +707,7 @@ void tasklet_kill(struct tasklet_struct *t)
489 yield(); 707 yield();
490 } while (test_bit(TASKLET_STATE_SCHED, &t->state)); 708 } while (test_bit(TASKLET_STATE_SCHED, &t->state));
491 } 709 }
710
492 tasklet_unlock_wait(t); 711 tasklet_unlock_wait(t);
493 clear_bit(TASKLET_STATE_SCHED, &t->state); 712 clear_bit(TASKLET_STATE_SCHED, &t->state);
494} 713}
@@ -694,6 +913,8 @@ void __init softirq_init(void)
694 913
695static int run_ksoftirqd(void * __bind_cpu) 914static int run_ksoftirqd(void * __bind_cpu)
696{ 915{
916 unsigned long flags;
917
697 set_current_state(TASK_INTERRUPTIBLE); 918 set_current_state(TASK_INTERRUPTIBLE);
698 919
699 while (!kthread_should_stop()) { 920 while (!kthread_should_stop()) {
@@ -712,7 +933,11 @@ static int run_ksoftirqd(void * __bind_cpu)
712 don't process */ 933 don't process */
713 if (cpu_is_offline((long)__bind_cpu)) 934 if (cpu_is_offline((long)__bind_cpu))
714 goto wait_to_die; 935 goto wait_to_die;
715 do_softirq(); 936
937 local_irq_save(flags);
938 ____do_softirq();
939 local_irq_restore(flags);
940
716 preempt_enable_no_resched(); 941 preempt_enable_no_resched();
717 cond_resched(); 942 cond_resched();
718 preempt_disable(); 943 preempt_disable();
@@ -760,6 +985,7 @@ void tasklet_kill_immediate(struct tasklet_struct *t, unsigned int cpu)
760 for (i = &per_cpu(tasklet_vec, cpu).head; *i; i = &(*i)->next) { 985 for (i = &per_cpu(tasklet_vec, cpu).head; *i; i = &(*i)->next) {
761 if (*i == t) { 986 if (*i == t) {
762 *i = t->next; 987 *i = t->next;
988
763 /* If this was the tail element, move the tail ptr */ 989 /* If this was the tail element, move the tail ptr */
764 if (*i == NULL) 990 if (*i == NULL)
765 per_cpu(tasklet_vec, cpu).tail = i; 991 per_cpu(tasklet_vec, cpu).tail = i;
diff --git a/kernel/workqueue.c b/kernel/workqueue.c
index f77afd939229..8139208eaee1 100644
--- a/kernel/workqueue.c
+++ b/kernel/workqueue.c
@@ -47,6 +47,13 @@
47 47
48#include "workqueue_sched.h" 48#include "workqueue_sched.h"
49 49
50#ifdef CONFIG_LITMUS_NVIDIA
51#include <litmus/litmus.h>
52#include <litmus/sched_trace.h>
53#include <litmus/nvidia_info.h>
54#endif
55
56
50enum { 57enum {
51 /* global_cwq flags */ 58 /* global_cwq flags */
52 GCWQ_MANAGE_WORKERS = 1 << 0, /* need to manage workers */ 59 GCWQ_MANAGE_WORKERS = 1 << 0, /* need to manage workers */
@@ -1010,9 +1017,7 @@ static void __queue_work(unsigned int cpu, struct workqueue_struct *wq,
1010 work_flags |= WORK_STRUCT_DELAYED; 1017 work_flags |= WORK_STRUCT_DELAYED;
1011 worklist = &cwq->delayed_works; 1018 worklist = &cwq->delayed_works;
1012 } 1019 }
1013
1014 insert_work(cwq, work, worklist, work_flags); 1020 insert_work(cwq, work, worklist, work_flags);
1015
1016 spin_unlock_irqrestore(&gcwq->lock, flags); 1021 spin_unlock_irqrestore(&gcwq->lock, flags);
1017} 1022}
1018 1023
@@ -2526,10 +2531,69 @@ EXPORT_SYMBOL(cancel_delayed_work_sync);
2526 */ 2531 */
2527int schedule_work(struct work_struct *work) 2532int schedule_work(struct work_struct *work)
2528{ 2533{
2529 return queue_work(system_wq, work); 2534#ifdef CONFIG_LITMUS_NVIDIA
2535 if(is_nvidia_func(work->func))
2536 {
2537 u32 nvidiaDevice = get_work_nv_device_num(work);
2538
2539 //1) Ask Litmus which task owns GPU <nvidiaDevice>. (API to be defined.)
2540 unsigned long flags;
2541 struct task_struct* device_owner;
2542
2543 lock_nv_registry(nvidiaDevice, &flags);
2544
2545 device_owner = get_nv_device_owner(nvidiaDevice);
2546
2547 //2) If there is an owner, set work->owner to the owner's task struct.
2548 if(device_owner==NULL)
2549 {
2550 work->owner = NULL;
2551 //TRACE("%s: the owner task of NVIDIA Device %u is NULL\n",__FUNCTION__,nvidiaDevice);
2552 }
2553 else
2554 {
2555 if( is_realtime(device_owner))
2556 {
2557 TRACE("%s: Handling NVIDIA work for device\t%u\tat\t%llu\n",
2558 __FUNCTION__, nvidiaDevice,litmus_clock());
2559 TRACE("%s: the owner task %d of NVIDIA Device %u is RT-task\n",
2560 __FUNCTION__,
2561 device_owner->pid,
2562 nvidiaDevice);
2563
2564 //3) Call litmus_schedule_work() and return (don't execute the rest
2565 // of schedule_schedule()).
2566 work->owner = device_owner;
2567 sched_trace_work_release(work->owner);
2568 if(likely(litmus_schedule_work(work, nvidiaDevice)))
2569 {
2570 unlock_nv_registry(nvidiaDevice, &flags);
2571 return 1;
2572 }
2573 else
2574 {
2575 work->owner = NULL; /* fall through to normal work scheduling */
2576 }
2577 }
2578 else
2579 {
2580 work->owner = NULL;
2581 }
2582 }
2583 unlock_nv_registry(nvidiaDevice, &flags);
2584 }
2585#endif
2586
2587 return(__schedule_work(work));
2530} 2588}
2531EXPORT_SYMBOL(schedule_work); 2589EXPORT_SYMBOL(schedule_work);
2532 2590
2591int __schedule_work(struct work_struct* work)
2592{
2593 return queue_work(system_wq, work);
2594}
2595EXPORT_SYMBOL(__schedule_work);
2596
2533/* 2597/*
2534 * schedule_work_on - put work task on a specific cpu 2598 * schedule_work_on - put work task on a specific cpu
2535 * @cpu: cpu to put the work task on 2599 * @cpu: cpu to put the work task on
diff --git a/litmus/Kconfig b/litmus/Kconfig
index ad8dc8308cf0..7e865d4dd703 100644
--- a/litmus/Kconfig
+++ b/litmus/Kconfig
@@ -62,6 +62,25 @@ config LITMUS_LOCKING
62 62
63endmenu 63endmenu
64 64
65menu "Performance Enhancements"
66
67config SCHED_CPU_AFFINITY
68 bool "Local Migration Affinity"
69 default y
70 help
71 Rescheduled tasks prefer CPUs near to their previously used CPU. This
72 may improve performance through possible preservation of cache affinity.
73
74 Warning: May make bugs ahrder to find since tasks may migrate less often.
75
76 NOTES:
77 * Pfair/PD^2 does not support this option.
78 * Only x86 currently supported.
79
80 Say Yes if unsure.
81
82endmenu
83
65menu "Tracing" 84menu "Tracing"
66 85
67config FEATHER_TRACE 86config FEATHER_TRACE
@@ -182,4 +201,74 @@ config SCHED_DEBUG_TRACE_CALLER
182 201
183endmenu 202endmenu
184 203
204menu "Interrupt Handling"
205
206config LITMUS_THREAD_ALL_SOFTIRQ
207 bool "Process all softirqs in ksoftirqd threads."
208 default n
209 help
210 (Experimental) Thread all softirqs to ksoftirqd
211 daemon threads, similar to PREEMPT_RT. I/O
212 throughput will will drop with this enabled, but
213 latencies due to interrupts will be reduced.
214
215 WARNING: Timer responsiveness will likely be
216 decreased as timer callbacks are also threaded.
217 This is unlike PREEEMPT_RTs hardirqs.
218
219 If unsure, say No.
220
221config LITMUS_SOFTIRQD
222 bool "Spawn klitirqd interrupt handling threads."
223 depends on LITMUS_LOCKING
224 default n
225 help
226 Create klitirqd interrupt handling threads. Work must be
227 specifically dispatched to these workers. (Softirqs for
228 Litmus tasks are not magically redirected to klitirqd.)
229
230 G-EDF ONLY for now!
231
232 If unsure, say No.
233
234config NR_LITMUS_SOFTIRQD
235 int "Number of klitirqd."
236 depends on LITMUS_SOFTIRQD
237 range 1 4096
238 default "1"
239 help
240 Should be <= to the number of CPUs in your system.
241
242config LITMUS_NVIDIA
243 bool "Litmus handling of NVIDIA interrupts."
244 depends on LITMUS_SOFTIRQD
245 default n
246 help
247 Direct tasklets from NVIDIA devices to Litmus's klitirqd.
248
249 If unsure, say No.
250
251choice
252 prompt "CUDA/Driver Version Support"
253 default CUDA_4_0
254 depends on LITMUS_NVIDIA
255 help
256 Select the version of CUDA/driver to support.
257
258config CUDA_4_0
259 bool "CUDA 4.0"
260 depends on LITMUS_NVIDIA
261 help
262 Support CUDA 4.0 RC2 (dev. driver version: x86_64-270.40)
263
264config CUDA_3_2
265 bool "CUDA 3.2"
266 depends on LITMUS_NVIDIA
267 help
268 Support CUDA 3.2 (dev. driver version: x86_64-260.24)
269
270endchoice
271
272endmenu
273
185endmenu 274endmenu
diff --git a/litmus/Makefile b/litmus/Makefile
index ad9936e07b83..892e01c2e1b3 100644
--- a/litmus/Makefile
+++ b/litmus/Makefile
@@ -21,8 +21,12 @@ obj-y = sched_plugin.o litmus.o \
21 21
22obj-$(CONFIG_PLUGIN_CEDF) += sched_cedf.o 22obj-$(CONFIG_PLUGIN_CEDF) += sched_cedf.o
23obj-$(CONFIG_PLUGIN_PFAIR) += sched_pfair.o 23obj-$(CONFIG_PLUGIN_PFAIR) += sched_pfair.o
24obj-$(CONFIG_SCHED_CPU_AFFINITY) += affinity.o
24 25
25obj-$(CONFIG_FEATHER_TRACE) += ft_event.o ftdev.o 26obj-$(CONFIG_FEATHER_TRACE) += ft_event.o ftdev.o
26obj-$(CONFIG_SCHED_TASK_TRACE) += sched_task_trace.o 27obj-$(CONFIG_SCHED_TASK_TRACE) += sched_task_trace.o
27obj-$(CONFIG_SCHED_DEBUG_TRACE) += sched_trace.o 28obj-$(CONFIG_SCHED_DEBUG_TRACE) += sched_trace.o
28obj-$(CONFIG_SCHED_OVERHEAD_TRACE) += trace.o 29obj-$(CONFIG_SCHED_OVERHEAD_TRACE) += trace.o
30
31obj-$(CONFIG_LITMUS_SOFTIRQD) += litmus_softirq.o
32obj-$(CONFIG_LITMUS_NVIDIA) += nvidia_info.o sched_trace_external.o
diff --git a/litmus/affinity.c b/litmus/affinity.c
new file mode 100644
index 000000000000..3b430d18885b
--- /dev/null
+++ b/litmus/affinity.c
@@ -0,0 +1,49 @@
1#include <linux/cpu.h>
2
3#include <litmus/affinity.h>
4
5struct neighborhood neigh_info[NR_CPUS];
6
7/* called by _init_litmus() */
8void init_topology(void)
9{
10 int cpu;
11 int i;
12 int chk;
13 int depth = num_cache_leaves;
14
15 if(depth > NUM_CACHE_LEVELS)
16 depth = NUM_CACHE_LEVELS;
17
18 for_each_online_cpu(cpu)
19 {
20 for(i = 0; i < depth; ++i)
21 {
22 long unsigned int firstbits;
23
24 chk = get_shared_cpu_map((struct cpumask *)&neigh_info[cpu].neighbors[i], cpu, i);
25 if(chk) /* failed */
26 {
27 neigh_info[cpu].size[i] = 0;
28 }
29 else
30 {
31 /* size = num bits in mask */
32 neigh_info[cpu].size[i] = cpumask_weight((struct cpumask *)&neigh_info[cpu].neighbors[i]);
33 }
34 firstbits = *neigh_info[cpu].neighbors[i]->bits;
35 printk("CPU %d has %d neighbors at level %d. (mask = %lx)\n",
36 cpu, neigh_info[cpu].size[i], i, firstbits);
37 }
38
39 /* set data for non-existent levels */
40 for(; i < NUM_CACHE_LEVELS; ++i)
41 {
42 neigh_info[cpu].size[i] = 0;
43
44 printk("CPU %d has %d neighbors at level %d. (mask = %lx)\n",
45 cpu, neigh_info[cpu].size[i], i, 0lu);
46 }
47 }
48}
49
diff --git a/litmus/edf_common.c b/litmus/edf_common.c
index 9b44dc2d8d1e..fbd67ab5f467 100644
--- a/litmus/edf_common.c
+++ b/litmus/edf_common.c
@@ -65,6 +65,12 @@ int edf_higher_prio(struct task_struct* first,
65 65
66 66
67 return !is_realtime(second_task) || 67 return !is_realtime(second_task) ||
68
69#ifdef CONFIG_LITMUS_SOFTIRQD
70 /* proxy threads always lose w/o inheritance. */
71 (first_task->rt_param.is_proxy_thread <
72 second_task->rt_param.is_proxy_thread) ||
73#endif
68 74
69 /* is the deadline of the first task earlier? 75 /* is the deadline of the first task earlier?
70 * Then it has higher priority. 76 * Then it has higher priority.
diff --git a/litmus/fdso.c b/litmus/fdso.c
index aa7b384264e3..2b7f9ba85857 100644
--- a/litmus/fdso.c
+++ b/litmus/fdso.c
@@ -22,6 +22,7 @@ extern struct fdso_ops generic_lock_ops;
22 22
23static const struct fdso_ops* fdso_ops[] = { 23static const struct fdso_ops* fdso_ops[] = {
24 &generic_lock_ops, /* FMLP_SEM */ 24 &generic_lock_ops, /* FMLP_SEM */
25 &generic_lock_ops, /* KFMLP_SEM */
25 &generic_lock_ops, /* SRP_SEM */ 26 &generic_lock_ops, /* SRP_SEM */
26}; 27};
27 28
diff --git a/litmus/litmus.c b/litmus/litmus.c
index 26938acacafc..29363c6ad565 100644
--- a/litmus/litmus.c
+++ b/litmus/litmus.c
@@ -17,6 +17,14 @@
17#include <litmus/litmus_proc.h> 17#include <litmus/litmus_proc.h>
18#include <litmus/sched_trace.h> 18#include <litmus/sched_trace.h>
19 19
20#ifdef CONFIG_SCHED_CPU_AFFINITY
21#include <litmus/affinity.h>
22#endif
23
24#ifdef CONFIG_LITMUS_NVIDIA
25#include <litmus/nvidia_info.h>
26#endif
27
20/* Number of RT tasks that exist in the system */ 28/* Number of RT tasks that exist in the system */
21atomic_t rt_task_count = ATOMIC_INIT(0); 29atomic_t rt_task_count = ATOMIC_INIT(0);
22static DEFINE_RAW_SPINLOCK(task_transition_lock); 30static DEFINE_RAW_SPINLOCK(task_transition_lock);
@@ -47,6 +55,28 @@ void bheap_node_free(struct bheap_node* hn)
47struct release_heap* release_heap_alloc(int gfp_flags); 55struct release_heap* release_heap_alloc(int gfp_flags);
48void release_heap_free(struct release_heap* rh); 56void release_heap_free(struct release_heap* rh);
49 57
58#ifdef CONFIG_LITMUS_NVIDIA
59/*
60 * sys_register_nv_device
61 * @nv_device_id: The Nvidia device id that the task want to register
62 * @reg_action: set to '1' to register the specified device. zero otherwise.
63 * Syscall for register task's designated nvidia device into NV_DEVICE_REG array
64 * Returns EFAULT if nv_device_id is out of range.
65 * 0 if success
66 */
67asmlinkage long sys_register_nv_device(int nv_device_id, int reg_action)
68{
69 /* register the device to caller (aka 'current') */
70 return(reg_nv_device(nv_device_id, reg_action));
71}
72#else
73asmlinkage long sys_register_nv_device(int nv_device_id, int reg_action)
74{
75 return(-EINVAL);
76}
77#endif
78
79
50/* 80/*
51 * sys_set_task_rt_param 81 * sys_set_task_rt_param
52 * @pid: Pid of the task which scheduling parameters must be changed 82 * @pid: Pid of the task which scheduling parameters must be changed
@@ -115,7 +145,7 @@ asmlinkage long sys_set_rt_task_param(pid_t pid, struct rt_task __user * param)
115 tp.cls != RT_CLASS_BEST_EFFORT) 145 tp.cls != RT_CLASS_BEST_EFFORT)
116 { 146 {
117 printk(KERN_INFO "litmus: real-time task %d rejected " 147 printk(KERN_INFO "litmus: real-time task %d rejected "
118 "because its class is invalid\n"); 148 "because its class is invalid\n", pid);
119 goto out_unlock; 149 goto out_unlock;
120 } 150 }
121 if (tp.budget_policy != NO_ENFORCEMENT && 151 if (tp.budget_policy != NO_ENFORCEMENT &&
@@ -131,6 +161,22 @@ asmlinkage long sys_set_rt_task_param(pid_t pid, struct rt_task __user * param)
131 161
132 target->rt_param.task_params = tp; 162 target->rt_param.task_params = tp;
133 163
164#ifdef CONFIG_LITMUS_SOFTIRQD
165 /* proxy thread off by default */
166 target->rt_param.is_proxy_thread = 0;
167 target->rt_param.cur_klitirqd = NULL;
168 //init_MUTEX(&target->rt_param.klitirqd_sem);
169 mutex_init(&target->rt_param.klitirqd_sem);
170 //init_completion(&target->rt_param.klitirqd_sem);
171 //target->rt_param.klitirqd_sem_stat = NOT_HELD;
172 atomic_set(&target->rt_param.klitirqd_sem_stat, NOT_HELD);
173#endif
174
175#ifdef CONFIG_LITMUS_NVIDIA
176 atomic_set(&target->rt_param.nv_int_count, 0);
177#endif
178
179
134 retval = 0; 180 retval = 0;
135 out_unlock: 181 out_unlock:
136 read_unlock_irq(&tasklist_lock); 182 read_unlock_irq(&tasklist_lock);
@@ -265,6 +311,7 @@ asmlinkage long sys_query_job_no(unsigned int __user *job)
265 return retval; 311 return retval;
266} 312}
267 313
314
268/* sys_null_call() is only used for determining raw system call 315/* sys_null_call() is only used for determining raw system call
269 * overheads (kernel entry, kernel exit). It has no useful side effects. 316 * overheads (kernel entry, kernel exit). It has no useful side effects.
270 * If ts is non-NULL, then the current Feather-Trace time is recorded. 317 * If ts is non-NULL, then the current Feather-Trace time is recorded.
@@ -278,7 +325,7 @@ asmlinkage long sys_null_call(cycles_t __user *ts)
278 now = get_cycles(); 325 now = get_cycles();
279 ret = put_user(now, ts); 326 ret = put_user(now, ts);
280 } 327 }
281 328
282 return ret; 329 return ret;
283} 330}
284 331
@@ -299,6 +346,20 @@ static void reinit_litmus_state(struct task_struct* p, int restore)
299 * at this point in time. 346 * at this point in time.
300 */ 347 */
301 WARN_ON(p->rt_param.inh_task); 348 WARN_ON(p->rt_param.inh_task);
349
350#ifdef CONFIG_LITMUS_SOFTIRQD
351 /* We probably should not have any tasklets executing for
352 * us at this time.
353 */
354 WARN_ON(p->rt_param.cur_klitirqd);
355 WARN_ON(atomic_read(&p->rt_param.klitirqd_sem_stat) == HELD);
356
357 if(p->rt_param.cur_klitirqd)
358 flush_pending(p->rt_param.cur_klitirqd, p);
359
360 if(atomic_read(&p->rt_param.klitirqd_sem_stat) == HELD)
361 up_and_set_stat(p, NOT_HELD, &p->rt_param.klitirqd_sem);
362#endif
302 363
303 /* Cleanup everything else. */ 364 /* Cleanup everything else. */
304 memset(&p->rt_param, 0, sizeof(p->rt_param)); 365 memset(&p->rt_param, 0, sizeof(p->rt_param));
@@ -399,7 +460,7 @@ static void synch_on_plugin_switch(void* info)
399 */ 460 */
400int switch_sched_plugin(struct sched_plugin* plugin) 461int switch_sched_plugin(struct sched_plugin* plugin)
401{ 462{
402 unsigned long flags; 463 //unsigned long flags;
403 int ret = 0; 464 int ret = 0;
404 465
405 BUG_ON(!plugin); 466 BUG_ON(!plugin);
@@ -413,8 +474,15 @@ int switch_sched_plugin(struct sched_plugin* plugin)
413 while (atomic_read(&cannot_use_plugin) < num_online_cpus()) 474 while (atomic_read(&cannot_use_plugin) < num_online_cpus())
414 cpu_relax(); 475 cpu_relax();
415 476
477#ifdef CONFIG_LITMUS_SOFTIRQD
478 if(!klitirqd_is_dead())
479 {
480 kill_klitirqd();
481 }
482#endif
483
416 /* stop task transitions */ 484 /* stop task transitions */
417 raw_spin_lock_irqsave(&task_transition_lock, flags); 485 //raw_spin_lock_irqsave(&task_transition_lock, flags);
418 486
419 /* don't switch if there are active real-time tasks */ 487 /* don't switch if there are active real-time tasks */
420 if (atomic_read(&rt_task_count) == 0) { 488 if (atomic_read(&rt_task_count) == 0) {
@@ -432,7 +500,7 @@ int switch_sched_plugin(struct sched_plugin* plugin)
432 } else 500 } else
433 ret = -EBUSY; 501 ret = -EBUSY;
434out: 502out:
435 raw_spin_unlock_irqrestore(&task_transition_lock, flags); 503 //raw_spin_unlock_irqrestore(&task_transition_lock, flags);
436 atomic_set(&cannot_use_plugin, 0); 504 atomic_set(&cannot_use_plugin, 0);
437 return ret; 505 return ret;
438} 506}
@@ -540,6 +608,10 @@ static int __init _init_litmus(void)
540 608
541 init_litmus_proc(); 609 init_litmus_proc();
542 610
611#ifdef CONFIG_SCHED_CPU_AFFINITY
612 init_topology();
613#endif
614
543 return 0; 615 return 0;
544} 616}
545 617
diff --git a/litmus/litmus_proc.c b/litmus/litmus_proc.c
index 4bf725a36c9c..381513366c7a 100644
--- a/litmus/litmus_proc.c
+++ b/litmus/litmus_proc.c
@@ -20,11 +20,18 @@ static struct proc_dir_entry *litmus_dir = NULL,
20#ifdef CONFIG_RELEASE_MASTER 20#ifdef CONFIG_RELEASE_MASTER
21 *release_master_file = NULL, 21 *release_master_file = NULL,
22#endif 22#endif
23#ifdef CONFIG_LITMUS_SOFTIRQD
24 *klitirqd_file = NULL,
25#endif
23 *plugs_file = NULL; 26 *plugs_file = NULL;
24 27
25/* in litmus/sync.c */ 28/* in litmus/sync.c */
26int count_tasks_waiting_for_release(void); 29int count_tasks_waiting_for_release(void);
27 30
31extern int proc_read_klitirqd_stats(char *page, char **start,
32 off_t off, int count,
33 int *eof, void *data);
34
28static int proc_read_stats(char *page, char **start, 35static int proc_read_stats(char *page, char **start,
29 off_t off, int count, 36 off_t off, int count,
30 int *eof, void *data) 37 int *eof, void *data)
@@ -161,6 +168,12 @@ int __init init_litmus_proc(void)
161 release_master_file->write_proc = proc_write_release_master; 168 release_master_file->write_proc = proc_write_release_master;
162#endif 169#endif
163 170
171#ifdef CONFIG_LITMUS_SOFTIRQD
172 klitirqd_file =
173 create_proc_read_entry("klitirqd_stats", 0444, litmus_dir,
174 proc_read_klitirqd_stats, NULL);
175#endif
176
164 stat_file = create_proc_read_entry("stats", 0444, litmus_dir, 177 stat_file = create_proc_read_entry("stats", 0444, litmus_dir,
165 proc_read_stats, NULL); 178 proc_read_stats, NULL);
166 179
@@ -187,6 +200,10 @@ void exit_litmus_proc(void)
187 remove_proc_entry("stats", litmus_dir); 200 remove_proc_entry("stats", litmus_dir);
188 if (curr_file) 201 if (curr_file)
189 remove_proc_entry("active_plugin", litmus_dir); 202 remove_proc_entry("active_plugin", litmus_dir);
203#ifdef CONFIG_LITMUS_SOFTIRQD
204 if (klitirqd_file)
205 remove_proc_entry("klitirqd_stats", litmus_dir);
206#endif
190#ifdef CONFIG_RELEASE_MASTER 207#ifdef CONFIG_RELEASE_MASTER
191 if (release_master_file) 208 if (release_master_file)
192 remove_proc_entry("release_master", litmus_dir); 209 remove_proc_entry("release_master", litmus_dir);
diff --git a/litmus/litmus_softirq.c b/litmus/litmus_softirq.c
new file mode 100644
index 000000000000..271e770dbaea
--- /dev/null
+++ b/litmus/litmus_softirq.c
@@ -0,0 +1,1579 @@
1#include <linux/interrupt.h>
2#include <linux/percpu.h>
3#include <linux/cpu.h>
4#include <linux/kthread.h>
5#include <linux/ftrace.h>
6#include <linux/smp.h>
7#include <linux/slab.h>
8#include <linux/mutex.h>
9
10#include <linux/sched.h>
11#include <linux/cpuset.h>
12
13#include <litmus/litmus.h>
14#include <litmus/sched_trace.h>
15#include <litmus/jobs.h>
16#include <litmus/sched_plugin.h>
17#include <litmus/litmus_softirq.h>
18
19/* TODO: Remove unneeded mb() and other barriers. */
20
21
22/* counts number of daemons ready to handle litmus irqs. */
23static atomic_t num_ready_klitirqds = ATOMIC_INIT(0);
24
25enum pending_flags
26{
27 LIT_TASKLET_LOW = 0x1,
28 LIT_TASKLET_HI = LIT_TASKLET_LOW<<1,
29 LIT_WORK = LIT_TASKLET_HI<<1
30};
31
32/* only support tasklet processing for now. */
33struct tasklet_head
34{
35 struct tasklet_struct *head;
36 struct tasklet_struct **tail;
37};
38
39struct klitirqd_info
40{
41 struct task_struct* klitirqd;
42 struct task_struct* current_owner;
43 int terminating;
44
45
46 raw_spinlock_t lock;
47
48 u32 pending;
49 atomic_t num_hi_pending;
50 atomic_t num_low_pending;
51 atomic_t num_work_pending;
52
53 /* in order of priority */
54 struct tasklet_head pending_tasklets_hi;
55 struct tasklet_head pending_tasklets;
56 struct list_head worklist;
57};
58
59/* one list for each klitirqd */
60static struct klitirqd_info klitirqds[NR_LITMUS_SOFTIRQD];
61
62
63
64
65
66int proc_read_klitirqd_stats(char *page, char **start,
67 off_t off, int count,
68 int *eof, void *data)
69{
70 int len = snprintf(page, PAGE_SIZE,
71 "num ready klitirqds: %d\n\n",
72 atomic_read(&num_ready_klitirqds));
73
74 if(klitirqd_is_ready())
75 {
76 int i;
77 for(i = 0; i < NR_LITMUS_SOFTIRQD; ++i)
78 {
79 len +=
80 snprintf(page + len - 1, PAGE_SIZE, /* -1 to strip off \0 */
81 "klitirqd_th%d: %s/%d\n"
82 "\tcurrent_owner: %s/%d\n"
83 "\tpending: %x\n"
84 "\tnum hi: %d\n"
85 "\tnum low: %d\n"
86 "\tnum work: %d\n\n",
87 i,
88 klitirqds[i].klitirqd->comm, klitirqds[i].klitirqd->pid,
89 (klitirqds[i].current_owner != NULL) ?
90 klitirqds[i].current_owner->comm : "(null)",
91 (klitirqds[i].current_owner != NULL) ?
92 klitirqds[i].current_owner->pid : 0,
93 klitirqds[i].pending,
94 atomic_read(&klitirqds[i].num_hi_pending),
95 atomic_read(&klitirqds[i].num_low_pending),
96 atomic_read(&klitirqds[i].num_work_pending));
97 }
98 }
99
100 return(len);
101}
102
103
104
105
106
107#if 0
108static atomic_t dump_id = ATOMIC_INIT(0);
109
110static void __dump_state(struct klitirqd_info* which, const char* caller)
111{
112 struct tasklet_struct* list;
113
114 int id = atomic_inc_return(&dump_id);
115
116 //if(in_interrupt())
117 {
118 if(which->current_owner)
119 {
120 TRACE("(id: %d caller: %s)\n"
121 "klitirqd: %s/%d\n"
122 "current owner: %s/%d\n"
123 "pending: %x\n",
124 id, caller,
125 which->klitirqd->comm, which->klitirqd->pid,
126 which->current_owner->comm, which->current_owner->pid,
127 which->pending);
128 }
129 else
130 {
131 TRACE("(id: %d caller: %s)\n"
132 "klitirqd: %s/%d\n"
133 "current owner: %p\n"
134 "pending: %x\n",
135 id, caller,
136 which->klitirqd->comm, which->klitirqd->pid,
137 NULL,
138 which->pending);
139 }
140
141 list = which->pending_tasklets.head;
142 while(list)
143 {
144 struct tasklet_struct *t = list;
145 list = list->next; /* advance */
146 if(t->owner)
147 TRACE("(id: %d caller: %s) Tasklet: %x, Owner = %s/%d\n", id, caller, t, t->owner->comm, t->owner->pid);
148 else
149 TRACE("(id: %d caller: %s) Tasklet: %x, Owner = %p\n", id, caller, t, NULL);
150 }
151 }
152}
153
154static void dump_state(struct klitirqd_info* which, const char* caller)
155{
156 unsigned long flags;
157
158 raw_spin_lock_irqsave(&which->lock, flags);
159 __dump_state(which, caller);
160 raw_spin_unlock_irqrestore(&which->lock, flags);
161}
162#endif
163
164
165/* forward declarations */
166static void ___litmus_tasklet_schedule(struct tasklet_struct *t,
167 struct klitirqd_info *which,
168 int wakeup);
169static void ___litmus_tasklet_hi_schedule(struct tasklet_struct *t,
170 struct klitirqd_info *which,
171 int wakeup);
172static void ___litmus_schedule_work(struct work_struct *w,
173 struct klitirqd_info *which,
174 int wakeup);
175
176
177
178inline unsigned int klitirqd_id(struct task_struct* tsk)
179{
180 int i;
181 for(i = 0; i < NR_LITMUS_SOFTIRQD; ++i)
182 {
183 if(klitirqds[i].klitirqd == tsk)
184 {
185 return i;
186 }
187 }
188
189 BUG();
190
191 return 0;
192}
193
194
195inline static u32 litirq_pending_hi_irqoff(struct klitirqd_info* which)
196{
197 return (which->pending & LIT_TASKLET_HI);
198}
199
200inline static u32 litirq_pending_low_irqoff(struct klitirqd_info* which)
201{
202 return (which->pending & LIT_TASKLET_LOW);
203}
204
205inline static u32 litirq_pending_work_irqoff(struct klitirqd_info* which)
206{
207 return (which->pending & LIT_WORK);
208}
209
210inline static u32 litirq_pending_irqoff(struct klitirqd_info* which)
211{
212 return(which->pending);
213}
214
215
216inline static u32 litirq_pending(struct klitirqd_info* which)
217{
218 unsigned long flags;
219 u32 pending;
220
221 raw_spin_lock_irqsave(&which->lock, flags);
222 pending = litirq_pending_irqoff(which);
223 raw_spin_unlock_irqrestore(&which->lock, flags);
224
225 return pending;
226};
227
228inline static u32 litirq_pending_with_owner(struct klitirqd_info* which, struct task_struct* owner)
229{
230 unsigned long flags;
231 u32 pending;
232
233 raw_spin_lock_irqsave(&which->lock, flags);
234 pending = litirq_pending_irqoff(which);
235 if(pending)
236 {
237 if(which->current_owner != owner)
238 {
239 pending = 0; // owner switch!
240 }
241 }
242 raw_spin_unlock_irqrestore(&which->lock, flags);
243
244 return pending;
245}
246
247
248inline static u32 litirq_pending_and_sem_and_owner(struct klitirqd_info* which,
249 struct mutex** sem,
250 struct task_struct** t)
251{
252 unsigned long flags;
253 u32 pending;
254
255 /* init values */
256 *sem = NULL;
257 *t = NULL;
258
259 raw_spin_lock_irqsave(&which->lock, flags);
260
261 pending = litirq_pending_irqoff(which);
262 if(pending)
263 {
264 if(which->current_owner != NULL)
265 {
266 *t = which->current_owner;
267 *sem = &tsk_rt(which->current_owner)->klitirqd_sem;
268 }
269 else
270 {
271 BUG();
272 }
273 }
274 raw_spin_unlock_irqrestore(&which->lock, flags);
275
276 if(likely(*sem))
277 {
278 return pending;
279 }
280 else
281 {
282 return 0;
283 }
284}
285
286/* returns true if the next piece of work to do is from a different owner.
287 */
288static int tasklet_ownership_change(
289 struct klitirqd_info* which,
290 enum pending_flags taskletQ)
291{
292 /* this function doesn't have to look at work objects since they have
293 priority below tasklets. */
294
295 unsigned long flags;
296 int ret = 0;
297
298 raw_spin_lock_irqsave(&which->lock, flags);
299
300 switch(taskletQ)
301 {
302 case LIT_TASKLET_HI:
303 if(litirq_pending_hi_irqoff(which))
304 {
305 ret = (which->pending_tasklets_hi.head->owner !=
306 which->current_owner);
307 }
308 break;
309 case LIT_TASKLET_LOW:
310 if(litirq_pending_low_irqoff(which))
311 {
312 ret = (which->pending_tasklets.head->owner !=
313 which->current_owner);
314 }
315 break;
316 default:
317 break;
318 }
319
320 raw_spin_unlock_irqrestore(&which->lock, flags);
321
322 TRACE_TASK(which->klitirqd, "ownership change needed: %d\n", ret);
323
324 return ret;
325}
326
327
328static void __reeval_prio(struct klitirqd_info* which)
329{
330 struct task_struct* next_owner = NULL;
331 struct task_struct* klitirqd = which->klitirqd;
332
333 /* Check in prio-order */
334 u32 pending = litirq_pending_irqoff(which);
335
336 //__dump_state(which, "__reeval_prio: before");
337
338 if(pending)
339 {
340 if(pending & LIT_TASKLET_HI)
341 {
342 next_owner = which->pending_tasklets_hi.head->owner;
343 }
344 else if(pending & LIT_TASKLET_LOW)
345 {
346 next_owner = which->pending_tasklets.head->owner;
347 }
348 else if(pending & LIT_WORK)
349 {
350 struct work_struct* work =
351 list_first_entry(&which->worklist, struct work_struct, entry);
352 next_owner = work->owner;
353 }
354 }
355
356 if(next_owner != which->current_owner)
357 {
358 struct task_struct* old_owner = which->current_owner;
359
360 /* bind the next owner. */
361 which->current_owner = next_owner;
362 mb();
363
364 if(next_owner != NULL)
365 {
366 if(!in_interrupt())
367 {
368 TRACE_CUR("%s: Ownership change: %s/%d to %s/%d\n", __FUNCTION__,
369 ((tsk_rt(klitirqd)->inh_task) ? tsk_rt(klitirqd)->inh_task : klitirqd)->comm,
370 ((tsk_rt(klitirqd)->inh_task) ? tsk_rt(klitirqd)->inh_task : klitirqd)->pid,
371 next_owner->comm, next_owner->pid);
372 }
373 else
374 {
375 TRACE("%s: Ownership change: %s/%d to %s/%d\n", __FUNCTION__,
376 ((tsk_rt(klitirqd)->inh_task) ? tsk_rt(klitirqd)->inh_task : klitirqd)->comm,
377 ((tsk_rt(klitirqd)->inh_task) ? tsk_rt(klitirqd)->inh_task : klitirqd)->pid,
378 next_owner->comm, next_owner->pid);
379 }
380
381 litmus->set_prio_inh_klitirqd(klitirqd, old_owner, next_owner);
382 }
383 else
384 {
385 if(likely(!in_interrupt()))
386 {
387 TRACE_CUR("%s: Ownership change: %s/%d to NULL (reverting)\n",
388 __FUNCTION__, klitirqd->comm, klitirqd->pid);
389 }
390 else
391 {
392 // is this a bug?
393 TRACE("%s: Ownership change: %s/%d to NULL (reverting)\n",
394 __FUNCTION__, klitirqd->comm, klitirqd->pid);
395 }
396
397 BUG_ON(pending != 0);
398 litmus->clear_prio_inh_klitirqd(klitirqd, old_owner);
399 }
400 }
401
402 //__dump_state(which, "__reeval_prio: after");
403}
404
405static void reeval_prio(struct klitirqd_info* which)
406{
407 unsigned long flags;
408
409 raw_spin_lock_irqsave(&which->lock, flags);
410 __reeval_prio(which);
411 raw_spin_unlock_irqrestore(&which->lock, flags);
412}
413
414
415static void wakeup_litirqd_locked(struct klitirqd_info* which)
416{
417 /* Interrupts are disabled: no need to stop preemption */
418 if (which && which->klitirqd)
419 {
420 __reeval_prio(which); /* configure the proper priority */
421
422 if(which->klitirqd->state != TASK_RUNNING)
423 {
424 TRACE("%s: Waking up klitirqd: %s/%d\n", __FUNCTION__,
425 which->klitirqd->comm, which->klitirqd->pid);
426
427 wake_up_process(which->klitirqd);
428 }
429 }
430}
431
432
433static void do_lit_tasklet(struct klitirqd_info* which,
434 struct tasklet_head* pending_tasklets)
435{
436 unsigned long flags;
437 struct tasklet_struct *list;
438 atomic_t* count;
439
440 raw_spin_lock_irqsave(&which->lock, flags);
441
442 //__dump_state(which, "do_lit_tasklet: before steal");
443
444 /* copy out the tasklets for our private use. */
445 list = pending_tasklets->head;
446 pending_tasklets->head = NULL;
447 pending_tasklets->tail = &pending_tasklets->head;
448
449 /* remove pending flag */
450 which->pending &= (pending_tasklets == &which->pending_tasklets) ?
451 ~LIT_TASKLET_LOW :
452 ~LIT_TASKLET_HI;
453
454 count = (pending_tasklets == &which->pending_tasklets) ?
455 &which->num_low_pending:
456 &which->num_hi_pending;
457
458 //__dump_state(which, "do_lit_tasklet: after steal");
459
460 raw_spin_unlock_irqrestore(&which->lock, flags);
461
462
463 while(list)
464 {
465 struct tasklet_struct *t = list;
466
467 /* advance, lest we forget */
468 list = list->next;
469
470 /* execute tasklet if it has my priority and is free */
471 if ((t->owner == which->current_owner) && tasklet_trylock(t)) {
472 if (!atomic_read(&t->count)) {
473 if (!test_and_clear_bit(TASKLET_STATE_SCHED, &t->state))
474 {
475 BUG();
476 }
477 TRACE_CUR("%s: Invoking tasklet.\n", __FUNCTION__);
478 t->func(t->data);
479 tasklet_unlock(t);
480
481 atomic_dec(count);
482
483 continue; /* process more tasklets */
484 }
485 tasklet_unlock(t);
486 }
487
488 TRACE_CUR("%s: Could not invoke tasklet. Requeuing.\n", __FUNCTION__);
489
490 /* couldn't process tasklet. put it back at the end of the queue. */
491 if(pending_tasklets == &which->pending_tasklets)
492 ___litmus_tasklet_schedule(t, which, 0);
493 else
494 ___litmus_tasklet_hi_schedule(t, which, 0);
495 }
496}
497
498
499// returns 1 if priorities need to be changed to continue processing
500// pending tasklets.
501static int do_litirq(struct klitirqd_info* which)
502{
503 u32 pending;
504 int resched = 0;
505
506 if(in_interrupt())
507 {
508 TRACE("%s: exiting early: in interrupt context!\n", __FUNCTION__);
509 return(0);
510 }
511
512 if(which->klitirqd != current)
513 {
514 TRACE_CUR("%s: exiting early: thread/info mismatch! Running %s/%d but given %s/%d.\n",
515 __FUNCTION__, current->comm, current->pid,
516 which->klitirqd->comm, which->klitirqd->pid);
517 return(0);
518 }
519
520 if(!is_realtime(current))
521 {
522 TRACE_CUR("%s: exiting early: klitirqd is not real-time. Sched Policy = %d\n",
523 __FUNCTION__, current->policy);
524 return(0);
525 }
526
527
528 /* We only handle tasklets & work objects, no need for RCU triggers? */
529
530 pending = litirq_pending(which);
531 if(pending)
532 {
533 /* extract the work to do and do it! */
534 if(pending & LIT_TASKLET_HI)
535 {
536 TRACE_CUR("%s: Invoking HI tasklets.\n", __FUNCTION__);
537 do_lit_tasklet(which, &which->pending_tasklets_hi);
538 resched = tasklet_ownership_change(which, LIT_TASKLET_HI);
539
540 if(resched)
541 {
542 TRACE_CUR("%s: HI tasklets of another owner remain. "
543 "Skipping any LOW tasklets.\n", __FUNCTION__);
544 }
545 }
546
547 if(!resched && (pending & LIT_TASKLET_LOW))
548 {
549 TRACE_CUR("%s: Invoking LOW tasklets.\n", __FUNCTION__);
550 do_lit_tasklet(which, &which->pending_tasklets);
551 resched = tasklet_ownership_change(which, LIT_TASKLET_LOW);
552
553 if(resched)
554 {
555 TRACE_CUR("%s: LOW tasklets of another owner remain. "
556 "Skipping any work objects.\n", __FUNCTION__);
557 }
558 }
559 }
560
561 return(resched);
562}
563
564
565static void do_work(struct klitirqd_info* which)
566{
567 unsigned long flags;
568 work_func_t f;
569 struct work_struct* work;
570
571 // only execute one work-queue item to yield to tasklets.
572 // ...is this a good idea, or should we just batch them?
573 raw_spin_lock_irqsave(&which->lock, flags);
574
575 if(!litirq_pending_work_irqoff(which))
576 {
577 raw_spin_unlock_irqrestore(&which->lock, flags);
578 goto no_work;
579 }
580
581 work = list_first_entry(&which->worklist, struct work_struct, entry);
582 list_del_init(&work->entry);
583
584 if(list_empty(&which->worklist))
585 {
586 which->pending &= ~LIT_WORK;
587 }
588
589 raw_spin_unlock_irqrestore(&which->lock, flags);
590
591
592
593 /* safe to read current_owner outside of lock since only this thread
594 may write to the pointer. */
595 if(work->owner == which->current_owner)
596 {
597 TRACE_CUR("%s: Invoking work object.\n", __FUNCTION__);
598 // do the work!
599 work_clear_pending(work);
600 f = work->func;
601 f(work); /* can't touch 'work' after this point,
602 the user may have freed it. */
603
604 atomic_dec(&which->num_work_pending);
605 }
606 else
607 {
608 TRACE_CUR("%s: Could not invoke work object. Requeuing.\n",
609 __FUNCTION__);
610 ___litmus_schedule_work(work, which, 0);
611 }
612
613no_work:
614 return;
615}
616
617
618static int set_litmus_daemon_sched(void)
619{
620 /* set up a daemon job that will never complete.
621 it should only ever run on behalf of another
622 real-time task.
623
624 TODO: Transition to a new job whenever a
625 new tasklet is handled */
626
627 int ret = 0;
628
629 struct rt_task tp = {
630 .exec_cost = 0,
631 .period = 1000000000, /* dummy 1 second period */
632 .phase = 0,
633 .cpu = task_cpu(current),
634 .budget_policy = NO_ENFORCEMENT,
635 .cls = RT_CLASS_BEST_EFFORT
636 };
637
638 struct sched_param param = { .sched_priority = 0};
639
640
641 /* set task params, mark as proxy thread, and init other data */
642 tsk_rt(current)->task_params = tp;
643 tsk_rt(current)->is_proxy_thread = 1;
644 tsk_rt(current)->cur_klitirqd = NULL;
645 //init_MUTEX(&tsk_rt(current)->klitirqd_sem);
646 mutex_init(&tsk_rt(current)->klitirqd_sem);
647 //init_completion(&tsk_rt(current)->klitirqd_sem);
648 atomic_set(&tsk_rt(current)->klitirqd_sem_stat, NOT_HELD);
649
650 /* inform the OS we're SCHED_LITMUS --
651 sched_setscheduler_nocheck() calls litmus_admit_task(). */
652 sched_setscheduler_nocheck(current, SCHED_LITMUS, &param);
653
654 return ret;
655}
656
657static void enter_execution_phase(struct klitirqd_info* which,
658 struct mutex* sem,
659 struct task_struct* t)
660{
661 TRACE_CUR("%s: Trying to enter execution phase. "
662 "Acquiring semaphore of %s/%d\n", __FUNCTION__,
663 t->comm, t->pid);
664 down_and_set_stat(current, HELD, sem);
665 TRACE_CUR("%s: Execution phase entered! "
666 "Acquired semaphore of %s/%d\n", __FUNCTION__,
667 t->comm, t->pid);
668}
669
670static void exit_execution_phase(struct klitirqd_info* which,
671 struct mutex* sem,
672 struct task_struct* t)
673{
674 TRACE_CUR("%s: Exiting execution phase. "
675 "Releasing semaphore of %s/%d\n", __FUNCTION__,
676 t->comm, t->pid);
677 if(atomic_read(&tsk_rt(current)->klitirqd_sem_stat) == HELD)
678 {
679 up_and_set_stat(current, NOT_HELD, sem);
680 TRACE_CUR("%s: Execution phase exited! "
681 "Released semaphore of %s/%d\n", __FUNCTION__,
682 t->comm, t->pid);
683 }
684 else
685 {
686 TRACE_CUR("%s: COULDN'T RELEASE SEMAPHORE BECAUSE ONE IS NOT HELD!\n", __FUNCTION__);
687 }
688}
689
690/* main loop for klitsoftirqd */
691static int run_klitirqd(void* unused)
692{
693 struct klitirqd_info* which = &klitirqds[klitirqd_id(current)];
694 struct mutex* sem;
695 struct task_struct* owner;
696
697 int rt_status = set_litmus_daemon_sched();
698
699 if(rt_status != 0)
700 {
701 TRACE_CUR("%s: Failed to transition to rt-task.\n", __FUNCTION__);
702 goto rt_failed;
703 }
704
705 atomic_inc(&num_ready_klitirqds);
706
707 set_current_state(TASK_INTERRUPTIBLE);
708
709 while (!kthread_should_stop())
710 {
711 preempt_disable();
712 if (!litirq_pending(which))
713 {
714 /* sleep for work */
715 TRACE_CUR("%s: No more tasklets or work objects. Going to sleep.\n",
716 __FUNCTION__);
717 preempt_enable_no_resched();
718 schedule();
719
720 if(kthread_should_stop()) /* bail out */
721 {
722 TRACE_CUR("%s:%d: Signaled to terminate.\n", __FUNCTION__, __LINE__);
723 continue;
724 }
725
726 preempt_disable();
727 }
728
729 __set_current_state(TASK_RUNNING);
730
731 while (litirq_pending_and_sem_and_owner(which, &sem, &owner))
732 {
733 int needs_resched = 0;
734
735 preempt_enable_no_resched();
736
737 BUG_ON(sem == NULL);
738
739 // wait to enter execution phase; wait for 'current_owner' to block.
740 enter_execution_phase(which, sem, owner);
741
742 if(kthread_should_stop())
743 {
744 TRACE_CUR("%s:%d: Signaled to terminate.\n", __FUNCTION__, __LINE__);
745 break;
746 }
747
748 preempt_disable();
749
750 /* Double check that there's still pending work and the owner hasn't
751 * changed. Pending items may have been flushed while we were sleeping.
752 */
753 if(litirq_pending_with_owner(which, owner))
754 {
755 TRACE_CUR("%s: Executing tasklets and/or work objects.\n",
756 __FUNCTION__);
757
758 needs_resched = do_litirq(which);
759
760 preempt_enable_no_resched();
761
762 // work objects are preemptible.
763 if(!needs_resched)
764 {
765 do_work(which);
766 }
767
768 // exit execution phase.
769 exit_execution_phase(which, sem, owner);
770
771 TRACE_CUR("%s: Setting up next priority.\n", __FUNCTION__);
772 reeval_prio(which); /* check if we need to change priority here */
773 }
774 else
775 {
776 TRACE_CUR("%s: Pending work was flushed! Prev owner was %s/%d\n",
777 __FUNCTION__,
778 owner->comm, owner->pid);
779 preempt_enable_no_resched();
780
781 // exit execution phase.
782 exit_execution_phase(which, sem, owner);
783 }
784
785 cond_resched();
786 preempt_disable();
787 }
788 preempt_enable();
789 set_current_state(TASK_INTERRUPTIBLE);
790 }
791 __set_current_state(TASK_RUNNING);
792
793 atomic_dec(&num_ready_klitirqds);
794
795rt_failed:
796 litmus_exit_task(current);
797
798 return rt_status;
799}
800
801
802struct klitirqd_launch_data
803{
804 int* cpu_affinity;
805 struct work_struct work;
806};
807
808/* executed by a kworker from workqueues */
809static void launch_klitirqd(struct work_struct *work)
810{
811 int i;
812
813 struct klitirqd_launch_data* launch_data =
814 container_of(work, struct klitirqd_launch_data, work);
815
816 TRACE("%s: Creating %d klitirqds\n", __FUNCTION__, NR_LITMUS_SOFTIRQD);
817
818 /* create the daemon threads */
819 for(i = 0; i < NR_LITMUS_SOFTIRQD; ++i)
820 {
821 if(launch_data->cpu_affinity)
822 {
823 klitirqds[i].klitirqd =
824 kthread_create(
825 run_klitirqd,
826 /* treat the affinity as a pointer, we'll cast it back later */
827 (void*)(long long)launch_data->cpu_affinity[i],
828 "klitirqd_th%d/%d",
829 i,
830 launch_data->cpu_affinity[i]);
831
832 /* litmus will put is in the right cluster. */
833 kthread_bind(klitirqds[i].klitirqd, launch_data->cpu_affinity[i]);
834 }
835 else
836 {
837 klitirqds[i].klitirqd =
838 kthread_create(
839 run_klitirqd,
840 /* treat the affinity as a pointer, we'll cast it back later */
841 (void*)(long long)(-1),
842 "klitirqd_th%d",
843 i);
844 }
845 }
846
847 TRACE("%s: Launching %d klitirqds\n", __FUNCTION__, NR_LITMUS_SOFTIRQD);
848
849 /* unleash the daemons */
850 for(i = 0; i < NR_LITMUS_SOFTIRQD; ++i)
851 {
852 wake_up_process(klitirqds[i].klitirqd);
853 }
854
855 if(launch_data->cpu_affinity)
856 kfree(launch_data->cpu_affinity);
857 kfree(launch_data);
858}
859
860
861void spawn_klitirqd(int* affinity)
862{
863 int i;
864 struct klitirqd_launch_data* delayed_launch;
865
866 if(atomic_read(&num_ready_klitirqds) != 0)
867 {
868 TRACE("%s: At least one klitirqd is already running! Need to call kill_klitirqd()?\n");
869 return;
870 }
871
872 /* init the tasklet & work queues */
873 for(i = 0; i < NR_LITMUS_SOFTIRQD; ++i)
874 {
875 klitirqds[i].terminating = 0;
876 klitirqds[i].pending = 0;
877
878 klitirqds[i].num_hi_pending.counter = 0;
879 klitirqds[i].num_low_pending.counter = 0;
880 klitirqds[i].num_work_pending.counter = 0;
881
882 klitirqds[i].pending_tasklets_hi.head = NULL;
883 klitirqds[i].pending_tasklets_hi.tail = &klitirqds[i].pending_tasklets_hi.head;
884
885 klitirqds[i].pending_tasklets.head = NULL;
886 klitirqds[i].pending_tasklets.tail = &klitirqds[i].pending_tasklets.head;
887
888 INIT_LIST_HEAD(&klitirqds[i].worklist);
889
890 raw_spin_lock_init(&klitirqds[i].lock);
891 }
892
893 /* wait to flush the initializations to memory since other threads
894 will access it. */
895 mb();
896
897 /* tell a work queue to launch the threads. we can't make scheduling
898 calls since we're in an atomic state. */
899 TRACE("%s: Setting callback up to launch klitirqds\n", __FUNCTION__);
900 delayed_launch = kmalloc(sizeof(struct klitirqd_launch_data), GFP_ATOMIC);
901 if(affinity)
902 {
903 delayed_launch->cpu_affinity =
904 kmalloc(sizeof(int)*NR_LITMUS_SOFTIRQD, GFP_ATOMIC);
905
906 memcpy(delayed_launch->cpu_affinity, affinity,
907 sizeof(int)*NR_LITMUS_SOFTIRQD);
908 }
909 else
910 {
911 delayed_launch->cpu_affinity = NULL;
912 }
913 INIT_WORK(&delayed_launch->work, launch_klitirqd);
914 schedule_work(&delayed_launch->work);
915}
916
917
918void kill_klitirqd(void)
919{
920 if(!klitirqd_is_dead())
921 {
922 int i;
923
924 TRACE("%s: Killing %d klitirqds\n", __FUNCTION__, NR_LITMUS_SOFTIRQD);
925
926 for(i = 0; i < NR_LITMUS_SOFTIRQD; ++i)
927 {
928 if(klitirqds[i].terminating != 1)
929 {
930 klitirqds[i].terminating = 1;
931 mb(); /* just to be sure? */
932 flush_pending(klitirqds[i].klitirqd, NULL);
933
934 /* signal termination */
935 kthread_stop(klitirqds[i].klitirqd);
936 }
937 }
938 }
939}
940
941
942int klitirqd_is_ready(void)
943{
944 return(atomic_read(&num_ready_klitirqds) == NR_LITMUS_SOFTIRQD);
945}
946
947int klitirqd_is_dead(void)
948{
949 return(atomic_read(&num_ready_klitirqds) == 0);
950}
951
952
953struct task_struct* get_klitirqd(unsigned int k_id)
954{
955 return(klitirqds[k_id].klitirqd);
956}
957
958
959void flush_pending(struct task_struct* klitirqd_thread,
960 struct task_struct* owner)
961{
962 unsigned int k_id = klitirqd_id(klitirqd_thread);
963 struct klitirqd_info *which = &klitirqds[k_id];
964
965 unsigned long flags;
966 struct tasklet_struct *list;
967
968 u32 work_flushed = 0;
969
970 raw_spin_lock_irqsave(&which->lock, flags);
971
972 //__dump_state(which, "flush_pending: before");
973
974 // flush hi tasklets.
975 if(litirq_pending_hi_irqoff(which))
976 {
977 which->pending &= ~LIT_TASKLET_HI;
978
979 list = which->pending_tasklets_hi.head;
980 which->pending_tasklets_hi.head = NULL;
981 which->pending_tasklets_hi.tail = &which->pending_tasklets_hi.head;
982
983 TRACE("%s: Handing HI tasklets back to Linux.\n", __FUNCTION__);
984
985 while(list)
986 {
987 struct tasklet_struct *t = list;
988 list = list->next;
989
990 if(likely((t->owner == owner) || (owner == NULL)))
991 {
992 if(unlikely(!test_and_clear_bit(TASKLET_STATE_SCHED, &t->state)))
993 {
994 BUG();
995 }
996
997 work_flushed |= LIT_TASKLET_HI;
998
999 t->owner = NULL;
1000
1001 // WTF?
1002 if(!test_and_set_bit(TASKLET_STATE_SCHED, &t->state))
1003 {
1004 atomic_dec(&which->num_hi_pending);
1005 ___tasklet_hi_schedule(t);
1006 }
1007 else
1008 {
1009 TRACE("%s: dropped hi tasklet??\n", __FUNCTION__);
1010 BUG();
1011 }
1012 }
1013 else
1014 {
1015 TRACE("%s: Could not flush a HI tasklet.\n", __FUNCTION__);
1016 // put back on queue.
1017 ___litmus_tasklet_hi_schedule(t, which, 0);
1018 }
1019 }
1020 }
1021
1022 // flush low tasklets.
1023 if(litirq_pending_low_irqoff(which))
1024 {
1025 which->pending &= ~LIT_TASKLET_LOW;
1026
1027 list = which->pending_tasklets.head;
1028 which->pending_tasklets.head = NULL;
1029 which->pending_tasklets.tail = &which->pending_tasklets.head;
1030
1031 TRACE("%s: Handing LOW tasklets back to Linux.\n", __FUNCTION__);
1032
1033 while(list)
1034 {
1035 struct tasklet_struct *t = list;
1036 list = list->next;
1037
1038 if(likely((t->owner == owner) || (owner == NULL)))
1039 {
1040 if(unlikely(!test_and_clear_bit(TASKLET_STATE_SCHED, &t->state)))
1041 {
1042 BUG();
1043 }
1044
1045 work_flushed |= LIT_TASKLET_LOW;
1046
1047 t->owner = NULL;
1048 sched_trace_tasklet_end(owner, 1ul);
1049
1050 if(!test_and_set_bit(TASKLET_STATE_SCHED, &t->state))
1051 {
1052 atomic_dec(&which->num_low_pending);
1053 ___tasklet_schedule(t);
1054 }
1055 else
1056 {
1057 TRACE("%s: dropped tasklet??\n", __FUNCTION__);
1058 BUG();
1059 }
1060 }
1061 else
1062 {
1063 TRACE("%s: Could not flush a LOW tasklet.\n", __FUNCTION__);
1064 // put back on queue
1065 ___litmus_tasklet_schedule(t, which, 0);
1066 }
1067 }
1068 }
1069
1070 // flush work objects
1071 if(litirq_pending_work_irqoff(which))
1072 {
1073 which->pending &= ~LIT_WORK;
1074
1075 TRACE("%s: Handing work objects back to Linux.\n", __FUNCTION__);
1076
1077 while(!list_empty(&which->worklist))
1078 {
1079 struct work_struct* work =
1080 list_first_entry(&which->worklist, struct work_struct, entry);
1081 list_del_init(&work->entry);
1082
1083 if(likely((work->owner == owner) || (owner == NULL)))
1084 {
1085 work_flushed |= LIT_WORK;
1086 atomic_dec(&which->num_work_pending);
1087
1088 work->owner = NULL;
1089 sched_trace_work_end(owner, current, 1ul);
1090 __schedule_work(work);
1091 }
1092 else
1093 {
1094 TRACE("%s: Could not flush a work object.\n", __FUNCTION__);
1095 // put back on queue
1096 ___litmus_schedule_work(work, which, 0);
1097 }
1098 }
1099 }
1100
1101 //__dump_state(which, "flush_pending: after (before reeval prio)");
1102
1103
1104 mb(); /* commit changes to pending flags */
1105
1106 /* reset the scheduling priority */
1107 if(work_flushed)
1108 {
1109 __reeval_prio(which);
1110
1111 /* Try to offload flushed tasklets to Linux's ksoftirqd. */
1112 if(work_flushed & (LIT_TASKLET_LOW | LIT_TASKLET_HI))
1113 {
1114 wakeup_softirqd();
1115 }
1116 }
1117 else
1118 {
1119 TRACE_CUR("%s: no work flushed, so __reeval_prio() skipped\n", __FUNCTION__);
1120 }
1121
1122 raw_spin_unlock_irqrestore(&which->lock, flags);
1123}
1124
1125
1126
1127
1128static void ___litmus_tasklet_schedule(struct tasklet_struct *t,
1129 struct klitirqd_info *which,
1130 int wakeup)
1131{
1132 unsigned long flags;
1133 u32 old_pending;
1134
1135 t->next = NULL;
1136
1137 raw_spin_lock_irqsave(&which->lock, flags);
1138
1139 //__dump_state(which, "___litmus_tasklet_schedule: before queuing");
1140
1141 *(which->pending_tasklets.tail) = t;
1142 which->pending_tasklets.tail = &t->next;
1143
1144 old_pending = which->pending;
1145 which->pending |= LIT_TASKLET_LOW;
1146
1147 atomic_inc(&which->num_low_pending);
1148
1149 mb();
1150
1151 if(!old_pending && wakeup)
1152 {
1153 wakeup_litirqd_locked(which); /* wake up the klitirqd */
1154 }
1155
1156 //__dump_state(which, "___litmus_tasklet_schedule: after queuing");
1157
1158 raw_spin_unlock_irqrestore(&which->lock, flags);
1159}
1160
1161int __litmus_tasklet_schedule(struct tasklet_struct *t, unsigned int k_id)
1162{
1163 int ret = 0; /* assume failure */
1164 if(unlikely((t->owner == NULL) || !is_realtime(t->owner)))
1165 {
1166 TRACE("%s: No owner associated with this tasklet!\n", __FUNCTION__);
1167 BUG();
1168 }
1169
1170 if(unlikely(k_id >= NR_LITMUS_SOFTIRQD))
1171 {
1172 TRACE("%s: No klitirqd_th%d!\n", __FUNCTION__, k_id);
1173 BUG();
1174 }
1175
1176 if(likely(!klitirqds[k_id].terminating))
1177 {
1178 /* Can't accept tasklets while we're processing a workqueue
1179 because they're handled by the same thread. This case is
1180 very RARE.
1181
1182 TODO: Use a separate thread for work objects!!!!!!
1183 */
1184 if(likely(atomic_read(&klitirqds[k_id].num_work_pending) == 0))
1185 {
1186 ret = 1;
1187 ___litmus_tasklet_schedule(t, &klitirqds[k_id], 1);
1188 }
1189 else
1190 {
1191 TRACE("%s: rejected tasklet because of pending work.\n",
1192 __FUNCTION__);
1193 }
1194 }
1195 return(ret);
1196}
1197
1198EXPORT_SYMBOL(__litmus_tasklet_schedule);
1199
1200
1201static void ___litmus_tasklet_hi_schedule(struct tasklet_struct *t,
1202 struct klitirqd_info *which,
1203 int wakeup)
1204{
1205 unsigned long flags;
1206 u32 old_pending;
1207
1208 t->next = NULL;
1209
1210 raw_spin_lock_irqsave(&which->lock, flags);
1211
1212 *(which->pending_tasklets_hi.tail) = t;
1213 which->pending_tasklets_hi.tail = &t->next;
1214
1215 old_pending = which->pending;
1216 which->pending |= LIT_TASKLET_HI;
1217
1218 atomic_inc(&which->num_hi_pending);
1219
1220 mb();
1221
1222 if(!old_pending && wakeup)
1223 {
1224 wakeup_litirqd_locked(which); /* wake up the klitirqd */
1225 }
1226
1227 raw_spin_unlock_irqrestore(&which->lock, flags);
1228}
1229
1230int __litmus_tasklet_hi_schedule(struct tasklet_struct *t, unsigned int k_id)
1231{
1232 int ret = 0; /* assume failure */
1233 if(unlikely((t->owner == NULL) || !is_realtime(t->owner)))
1234 {
1235 TRACE("%s: No owner associated with this tasklet!\n", __FUNCTION__);
1236 BUG();
1237 }
1238
1239 if(unlikely(k_id >= NR_LITMUS_SOFTIRQD))
1240 {
1241 TRACE("%s: No klitirqd_th%d!\n", __FUNCTION__, k_id);
1242 BUG();
1243 }
1244
1245 if(unlikely(!klitirqd_is_ready()))
1246 {
1247 TRACE("%s: klitirqd is not ready!\n", __FUNCTION__, k_id);
1248 BUG();
1249 }
1250
1251 if(likely(!klitirqds[k_id].terminating))
1252 {
1253 if(likely(atomic_read(&klitirqds[k_id].num_work_pending) == 0))
1254 {
1255 ret = 1;
1256 ___litmus_tasklet_hi_schedule(t, &klitirqds[k_id], 1);
1257 }
1258 else
1259 {
1260 TRACE("%s: rejected tasklet because of pending work.\n",
1261 __FUNCTION__);
1262 }
1263 }
1264 return(ret);
1265}
1266
1267EXPORT_SYMBOL(__litmus_tasklet_hi_schedule);
1268
1269
1270int __litmus_tasklet_hi_schedule_first(struct tasklet_struct *t, unsigned int k_id)
1271{
1272 int ret = 0; /* assume failure */
1273 u32 old_pending;
1274
1275 BUG_ON(!irqs_disabled());
1276
1277 if(unlikely((t->owner == NULL) || !is_realtime(t->owner)))
1278 {
1279 TRACE("%s: No owner associated with this tasklet!\n", __FUNCTION__);
1280 BUG();
1281 }
1282
1283 if(unlikely(k_id >= NR_LITMUS_SOFTIRQD))
1284 {
1285 TRACE("%s: No klitirqd_th%u!\n", __FUNCTION__, k_id);
1286 BUG();
1287 }
1288
1289 if(unlikely(!klitirqd_is_ready()))
1290 {
1291 TRACE("%s: klitirqd is not ready!\n", __FUNCTION__, k_id);
1292 BUG();
1293 }
1294
1295 if(likely(!klitirqds[k_id].terminating))
1296 {
1297 raw_spin_lock(&klitirqds[k_id].lock);
1298
1299 if(likely(atomic_read(&klitirqds[k_id].num_work_pending) == 0))
1300 {
1301 ret = 1; // success!
1302
1303 t->next = klitirqds[k_id].pending_tasklets_hi.head;
1304 klitirqds[k_id].pending_tasklets_hi.head = t;
1305
1306 old_pending = klitirqds[k_id].pending;
1307 klitirqds[k_id].pending |= LIT_TASKLET_HI;
1308
1309 atomic_inc(&klitirqds[k_id].num_hi_pending);
1310
1311 mb();
1312
1313 if(!old_pending)
1314 wakeup_litirqd_locked(&klitirqds[k_id]); /* wake up the klitirqd */
1315 }
1316 else
1317 {
1318 TRACE("%s: rejected tasklet because of pending work.\n",
1319 __FUNCTION__);
1320 }
1321
1322 raw_spin_unlock(&klitirqds[k_id].lock);
1323 }
1324 return(ret);
1325}
1326
1327EXPORT_SYMBOL(__litmus_tasklet_hi_schedule_first);
1328
1329
1330
1331static void ___litmus_schedule_work(struct work_struct *w,
1332 struct klitirqd_info *which,
1333 int wakeup)
1334{
1335 unsigned long flags;
1336 u32 old_pending;
1337
1338 raw_spin_lock_irqsave(&which->lock, flags);
1339
1340 work_pending(w);
1341 list_add_tail(&w->entry, &which->worklist);
1342
1343 old_pending = which->pending;
1344 which->pending |= LIT_WORK;
1345
1346 atomic_inc(&which->num_work_pending);
1347
1348 mb();
1349
1350 if(!old_pending && wakeup)
1351 {
1352 wakeup_litirqd_locked(which); /* wakeup the klitirqd */
1353 }
1354
1355 raw_spin_unlock_irqrestore(&which->lock, flags);
1356}
1357
1358int __litmus_schedule_work(struct work_struct *w, unsigned int k_id)
1359{
1360 int ret = 1; /* assume success */
1361 if(unlikely(w->owner == NULL) || !is_realtime(w->owner))
1362 {
1363 TRACE("%s: No owner associated with this work object!\n", __FUNCTION__);
1364 BUG();
1365 }
1366
1367 if(unlikely(k_id >= NR_LITMUS_SOFTIRQD))
1368 {
1369 TRACE("%s: No klitirqd_th%u!\n", k_id);
1370 BUG();
1371 }
1372
1373 if(unlikely(!klitirqd_is_ready()))
1374 {
1375 TRACE("%s: klitirqd is not ready!\n", __FUNCTION__, k_id);
1376 BUG();
1377 }
1378
1379 if(likely(!klitirqds[k_id].terminating))
1380 ___litmus_schedule_work(w, &klitirqds[k_id], 1);
1381 else
1382 ret = 0;
1383 return(ret);
1384}
1385EXPORT_SYMBOL(__litmus_schedule_work);
1386
1387
1388static int set_klitirqd_sem_status(unsigned long stat)
1389{
1390 TRACE_CUR("SETTING STATUS FROM %d TO %d\n",
1391 atomic_read(&tsk_rt(current)->klitirqd_sem_stat),
1392 stat);
1393 atomic_set(&tsk_rt(current)->klitirqd_sem_stat, stat);
1394 //mb();
1395
1396 return(0);
1397}
1398
1399static int set_klitirqd_sem_status_if_not_held(unsigned long stat)
1400{
1401 if(atomic_read(&tsk_rt(current)->klitirqd_sem_stat) != HELD)
1402 {
1403 return(set_klitirqd_sem_status(stat));
1404 }
1405 return(-1);
1406}
1407
1408
1409void __down_and_reset_and_set_stat(struct task_struct* t,
1410 enum klitirqd_sem_status to_reset,
1411 enum klitirqd_sem_status to_set,
1412 struct mutex* sem)
1413{
1414#if 0
1415 struct rt_param* param = container_of(sem, struct rt_param, klitirqd_sem);
1416 struct task_struct* task = container_of(param, struct task_struct, rt_param);
1417
1418 TRACE_CUR("%s: entered. Locking semaphore of %s/%d\n",
1419 __FUNCTION__, task->comm, task->pid);
1420#endif
1421
1422 mutex_lock_sfx(sem,
1423 set_klitirqd_sem_status_if_not_held, to_reset,
1424 set_klitirqd_sem_status, to_set);
1425#if 0
1426 TRACE_CUR("%s: exiting. Have semaphore of %s/%d\n",
1427 __FUNCTION__, task->comm, task->pid);
1428#endif
1429}
1430
1431void down_and_set_stat(struct task_struct* t,
1432 enum klitirqd_sem_status to_set,
1433 struct mutex* sem)
1434{
1435#if 0
1436 struct rt_param* param = container_of(sem, struct rt_param, klitirqd_sem);
1437 struct task_struct* task = container_of(param, struct task_struct, rt_param);
1438
1439 TRACE_CUR("%s: entered. Locking semaphore of %s/%d\n",
1440 __FUNCTION__, task->comm, task->pid);
1441#endif
1442
1443 mutex_lock_sfx(sem,
1444 NULL, 0,
1445 set_klitirqd_sem_status, to_set);
1446
1447#if 0
1448 TRACE_CUR("%s: exiting. Have semaphore of %s/%d\n",
1449 __FUNCTION__, task->comm, task->pid);
1450#endif
1451}
1452
1453
1454void up_and_set_stat(struct task_struct* t,
1455 enum klitirqd_sem_status to_set,
1456 struct mutex* sem)
1457{
1458#if 0
1459 struct rt_param* param = container_of(sem, struct rt_param, klitirqd_sem);
1460 struct task_struct* task = container_of(param, struct task_struct, rt_param);
1461
1462 TRACE_CUR("%s: entered. Unlocking semaphore of %s/%d\n",
1463 __FUNCTION__,
1464 task->comm, task->pid);
1465#endif
1466
1467 mutex_unlock_sfx(sem, NULL, 0,
1468 set_klitirqd_sem_status, to_set);
1469
1470#if 0
1471 TRACE_CUR("%s: exiting. Unlocked semaphore of %s/%d\n",
1472 __FUNCTION__,
1473 task->comm, task->pid);
1474#endif
1475}
1476
1477
1478
1479void release_klitirqd_lock(struct task_struct* t)
1480{
1481 if(is_realtime(t) && (atomic_read(&tsk_rt(t)->klitirqd_sem_stat) == HELD))
1482 {
1483 struct mutex* sem;
1484 struct task_struct* owner = t;
1485
1486 if(t->state == TASK_RUNNING)
1487 {
1488 TRACE_TASK(t, "NOT giving up klitirqd_sem because we're not blocked!\n");
1489 return;
1490 }
1491
1492 if(likely(!tsk_rt(t)->is_proxy_thread))
1493 {
1494 sem = &tsk_rt(t)->klitirqd_sem;
1495 }
1496 else
1497 {
1498 unsigned int k_id = klitirqd_id(t);
1499 owner = klitirqds[k_id].current_owner;
1500
1501 BUG_ON(t != klitirqds[k_id].klitirqd);
1502
1503 if(likely(owner))
1504 {
1505 sem = &tsk_rt(owner)->klitirqd_sem;
1506 }
1507 else
1508 {
1509 BUG();
1510
1511 // We had the rug pulled out from under us. Abort attempt
1512 // to reacquire the lock since our client no longer needs us.
1513 TRACE_CUR("HUH?! How did this happen?\n");
1514 atomic_set(&tsk_rt(t)->klitirqd_sem_stat, NOT_HELD);
1515 return;
1516 }
1517 }
1518
1519 //TRACE_CUR("Releasing semaphore of %s/%d...\n", owner->comm, owner->pid);
1520 up_and_set_stat(t, NEED_TO_REACQUIRE, sem);
1521 //TRACE_CUR("Semaphore of %s/%d released!\n", owner->comm, owner->pid);
1522 }
1523 /*
1524 else if(is_realtime(t))
1525 {
1526 TRACE_CUR("%s: Nothing to do. Stat = %d\n", __FUNCTION__, tsk_rt(t)->klitirqd_sem_stat);
1527 }
1528 */
1529}
1530
1531int reacquire_klitirqd_lock(struct task_struct* t)
1532{
1533 int ret = 0;
1534
1535 if(is_realtime(t) && (atomic_read(&tsk_rt(t)->klitirqd_sem_stat) == NEED_TO_REACQUIRE))
1536 {
1537 struct mutex* sem;
1538 struct task_struct* owner = t;
1539
1540 if(likely(!tsk_rt(t)->is_proxy_thread))
1541 {
1542 sem = &tsk_rt(t)->klitirqd_sem;
1543 }
1544 else
1545 {
1546 unsigned int k_id = klitirqd_id(t);
1547 //struct task_struct* owner = klitirqds[k_id].current_owner;
1548 owner = klitirqds[k_id].current_owner;
1549
1550 BUG_ON(t != klitirqds[k_id].klitirqd);
1551
1552 if(likely(owner))
1553 {
1554 sem = &tsk_rt(owner)->klitirqd_sem;
1555 }
1556 else
1557 {
1558 // We had the rug pulled out from under us. Abort attempt
1559 // to reacquire the lock since our client no longer needs us.
1560 TRACE_CUR("No longer needs to reacquire klitirqd_sem!\n");
1561 atomic_set(&tsk_rt(t)->klitirqd_sem_stat, NOT_HELD);
1562 return(0);
1563 }
1564 }
1565
1566 //TRACE_CUR("Trying to reacquire semaphore of %s/%d\n", owner->comm, owner->pid);
1567 __down_and_reset_and_set_stat(t, REACQUIRING, HELD, sem);
1568 //TRACE_CUR("Reacquired semaphore %s/%d\n", owner->comm, owner->pid);
1569 }
1570 /*
1571 else if(is_realtime(t))
1572 {
1573 TRACE_CUR("%s: Nothing to do. Stat = %d\n", __FUNCTION__, tsk_rt(t)->klitirqd_sem_stat);
1574 }
1575 */
1576
1577 return(ret);
1578}
1579
diff --git a/litmus/locking.c b/litmus/locking.c
index 2693f1aca859..cfce98e7480d 100644
--- a/litmus/locking.c
+++ b/litmus/locking.c
@@ -121,7 +121,6 @@ struct task_struct* __waitqueue_remove_first(wait_queue_head_t *wq)
121 return(t); 121 return(t);
122} 122}
123 123
124
125#else 124#else
126 125
127struct fdso_ops generic_lock_ops = {}; 126struct fdso_ops generic_lock_ops = {};
diff --git a/litmus/nvidia_info.c b/litmus/nvidia_info.c
new file mode 100644
index 000000000000..78f035244d21
--- /dev/null
+++ b/litmus/nvidia_info.c
@@ -0,0 +1,526 @@
1#include <linux/module.h>
2#include <linux/semaphore.h>
3#include <linux/pci.h>
4
5#include <litmus/sched_trace.h>
6#include <litmus/nvidia_info.h>
7#include <litmus/litmus.h>
8
9typedef unsigned char NvV8; /* "void": enumerated or multiple fields */
10typedef unsigned short NvV16; /* "void": enumerated or multiple fields */
11typedef unsigned char NvU8; /* 0 to 255 */
12typedef unsigned short NvU16; /* 0 to 65535 */
13typedef signed char NvS8; /* -128 to 127 */
14typedef signed short NvS16; /* -32768 to 32767 */
15typedef float NvF32; /* IEEE Single Precision (S1E8M23) */
16typedef double NvF64; /* IEEE Double Precision (S1E11M52) */
17typedef unsigned int NvV32; /* "void": enumerated or multiple fields */
18typedef unsigned int NvU32; /* 0 to 4294967295 */
19typedef unsigned long long NvU64; /* 0 to 18446744073709551615 */
20typedef union
21{
22 volatile NvV8 Reg008[1];
23 volatile NvV16 Reg016[1];
24 volatile NvV32 Reg032[1];
25} litmus_nv_hwreg_t, * litmus_nv_phwreg_t;
26
27typedef struct
28{
29 NvU64 address;
30 NvU64 size;
31 NvU32 offset;
32 NvU32 *map;
33 litmus_nv_phwreg_t map_u;
34} litmus_nv_aperture_t;
35
36typedef struct
37{
38 void *priv; /* private data */
39 void *os_state; /* os-specific device state */
40
41 int rmInitialized;
42 int flags;
43
44 /* PCI config info */
45 NvU32 domain;
46 NvU16 bus;
47 NvU16 slot;
48 NvU16 vendor_id;
49 NvU16 device_id;
50 NvU16 subsystem_id;
51 NvU32 gpu_id;
52 void *handle;
53
54 NvU32 pci_cfg_space[16];
55
56 /* physical characteristics */
57 litmus_nv_aperture_t bars[3];
58 litmus_nv_aperture_t *regs;
59 litmus_nv_aperture_t *fb, ud;
60 litmus_nv_aperture_t agp;
61
62 NvU32 interrupt_line;
63
64 NvU32 agp_config;
65 NvU32 agp_status;
66
67 NvU32 primary_vga;
68
69 NvU32 sim_env;
70
71 NvU32 rc_timer_enabled;
72
73 /* list of events allocated for this device */
74 void *event_list;
75
76 void *kern_mappings;
77
78} litmus_nv_state_t;
79
80typedef struct work_struct litmus_nv_task_t;
81
82typedef struct litmus_nv_work_s {
83 litmus_nv_task_t task;
84 void *data;
85} litmus_nv_work_t;
86
87typedef struct litmus_nv_linux_state_s {
88 litmus_nv_state_t nv_state;
89 atomic_t usage_count;
90
91 struct pci_dev *dev;
92 void *agp_bridge;
93 void *alloc_queue;
94
95 void *timer_sp;
96 void *isr_sp;
97 void *pci_cfgchk_sp;
98 void *isr_bh_sp;
99
100#ifdef CONFIG_CUDA_4_0
101 char registry_keys[512];
102#endif
103
104 /* keep track of any pending bottom halfes */
105 struct tasklet_struct tasklet;
106 litmus_nv_work_t work;
107
108 /* get a timer callback every second */
109 struct timer_list rc_timer;
110
111 /* lock for linux-specific data, not used by core rm */
112 struct semaphore ldata_lock;
113
114 /* lock for linux-specific alloc queue */
115 struct semaphore at_lock;
116
117#if 0
118#if defined(NV_USER_MAP)
119 /* list of user mappings */
120 struct nv_usermap_s *usermap_list;
121
122 /* lock for VMware-specific mapping list */
123 struct semaphore mt_lock;
124#endif /* defined(NV_USER_MAP) */
125#if defined(NV_PM_SUPPORT_OLD_STYLE_APM)
126 void *apm_nv_dev;
127#endif
128#endif
129
130 NvU32 device_num;
131 struct litmus_nv_linux_state_s *next;
132} litmus_nv_linux_state_t;
133
134void dump_nvidia_info(const struct tasklet_struct *t)
135{
136 litmus_nv_state_t* nvstate = NULL;
137 litmus_nv_linux_state_t* linuxstate = NULL;
138 struct pci_dev* pci = NULL;
139
140 nvstate = (litmus_nv_state_t*)(t->data);
141
142 if(nvstate)
143 {
144 TRACE("NV State:\n"
145 "\ttasklet ptr = %p\n"
146 "\tstate ptr = %p\n"
147 "\tprivate data ptr = %p\n"
148 "\tos state ptr = %p\n"
149 "\tdomain = %u\n"
150 "\tbus = %u\n"
151 "\tslot = %u\n"
152 "\tvender_id = %u\n"
153 "\tdevice_id = %u\n"
154 "\tsubsystem_id = %u\n"
155 "\tgpu_id = %u\n"
156 "\tinterrupt_line = %u\n",
157 t,
158 nvstate,
159 nvstate->priv,
160 nvstate->os_state,
161 nvstate->domain,
162 nvstate->bus,
163 nvstate->slot,
164 nvstate->vendor_id,
165 nvstate->device_id,
166 nvstate->subsystem_id,
167 nvstate->gpu_id,
168 nvstate->interrupt_line);
169
170 linuxstate = container_of(nvstate, litmus_nv_linux_state_t, nv_state);
171 }
172 else
173 {
174 TRACE("INVALID NVSTATE????\n");
175 }
176
177 if(linuxstate)
178 {
179 int ls_offset = (void*)(&(linuxstate->device_num)) - (void*)(linuxstate);
180 int ns_offset_raw = (void*)(&(linuxstate->device_num)) - (void*)(&(linuxstate->nv_state));
181 int ns_offset_desired = (void*)(&(linuxstate->device_num)) - (void*)(nvstate);
182
183
184 TRACE("LINUX NV State:\n"
185 "\tlinux nv state ptr: %p\n"
186 "\taddress of tasklet: %p\n"
187 "\taddress of work: %p\n"
188 "\tusage_count: %d\n"
189 "\tdevice_num: %u\n"
190 "\ttasklet addr == this tasklet: %d\n"
191 "\tpci: %p\n",
192 linuxstate,
193 &(linuxstate->tasklet),
194 &(linuxstate->work),
195 atomic_read(&(linuxstate->usage_count)),
196 linuxstate->device_num,
197 (t == &(linuxstate->tasklet)),
198 linuxstate->dev);
199
200 pci = linuxstate->dev;
201
202 TRACE("Offsets:\n"
203 "\tOffset from LinuxState: %d, %x\n"
204 "\tOffset from NVState: %d, %x\n"
205 "\tOffset from parameter: %d, %x\n"
206 "\tdevice_num: %u\n",
207 ls_offset, ls_offset,
208 ns_offset_raw, ns_offset_raw,
209 ns_offset_desired, ns_offset_desired,
210 *((u32*)((void*)nvstate + ns_offset_desired)));
211 }
212 else
213 {
214 TRACE("INVALID LINUXNVSTATE?????\n");
215 }
216
217#if 0
218 if(pci)
219 {
220 TRACE("PCI DEV Info:\n"
221 "pci device ptr: %p\n"
222 "\tdevfn = %d\n"
223 "\tvendor = %d\n"
224 "\tdevice = %d\n"
225 "\tsubsystem_vendor = %d\n"
226 "\tsubsystem_device = %d\n"
227 "\tslot # = %d\n",
228 pci,
229 pci->devfn,
230 pci->vendor,
231 pci->device,
232 pci->subsystem_vendor,
233 pci->subsystem_device,
234 pci->slot->number);
235 }
236 else
237 {
238 TRACE("INVALID PCIDEV PTR?????\n");
239 }
240#endif
241}
242
243static struct module* nvidia_mod = NULL;
244int init_nvidia_info(void)
245{
246 mutex_lock(&module_mutex);
247 nvidia_mod = find_module("nvidia");
248 mutex_unlock(&module_mutex);
249 if(nvidia_mod != NULL)
250 {
251 TRACE("%s : Found NVIDIA module. Core Code: %p to %p\n", __FUNCTION__,
252 (void*)(nvidia_mod->module_core),
253 (void*)(nvidia_mod->module_core) + nvidia_mod->core_size);
254 init_nv_device_reg();
255 return(0);
256 }
257 else
258 {
259 TRACE("%s : Could not find NVIDIA module! Loaded?\n", __FUNCTION__);
260 return(-1);
261 }
262}
263
264
265/* works with pointers to static data inside the module too. */
266int is_nvidia_func(void* func_addr)
267{
268 int ret = 0;
269 if(nvidia_mod)
270 {
271 ret = within_module_core((long unsigned int)func_addr, nvidia_mod);
272 /*
273 if(ret)
274 {
275 TRACE("%s : %p is in NVIDIA module: %d\n",
276 __FUNCTION__, func_addr, ret);
277 }*/
278 }
279
280 return(ret);
281}
282
283u32 get_tasklet_nv_device_num(const struct tasklet_struct *t)
284{
285 // life is too short to use hard-coded offsets. update this later.
286 litmus_nv_state_t* nvstate = (litmus_nv_state_t*)(t->data);
287 litmus_nv_linux_state_t* linuxstate = container_of(nvstate, litmus_nv_linux_state_t, nv_state);
288
289 BUG_ON(linuxstate->device_num >= NV_DEVICE_NUM);
290
291 return(linuxstate->device_num);
292
293 //int DEVICE_NUM_OFFSET = (void*)(&(linuxstate->device_num)) - (void*)(nvstate);
294
295#if 0
296 // offset determined though observed behavior of the NV driver.
297 //const int DEVICE_NUM_OFFSET = 0x480; // CUDA 4.0 RC1
298 //const int DEVICE_NUM_OFFSET = 0x510; // CUDA 4.0 RC2
299
300 void* state = (void*)(t->data);
301 void* device_num_ptr = state + DEVICE_NUM_OFFSET;
302
303 //dump_nvidia_info(t);
304 return(*((u32*)device_num_ptr));
305#endif
306}
307
308u32 get_work_nv_device_num(const struct work_struct *t)
309{
310 // offset determined though observed behavior of the NV driver.
311 const int DEVICE_NUM_OFFSET = sizeof(struct work_struct);
312 void* state = (void*)(t);
313 void** device_num_ptr = state + DEVICE_NUM_OFFSET;
314 return(*((u32*)(*device_num_ptr)));
315}
316
317
318
319typedef struct {
320 raw_spinlock_t lock;
321 struct task_struct *device_owner;
322}nv_device_registry_t;
323
324static nv_device_registry_t NV_DEVICE_REG[NV_DEVICE_NUM];
325
326int init_nv_device_reg(void)
327{
328 int i;
329
330 //memset(NV_DEVICE_REG, 0, sizeof(NV_DEVICE_REG));
331
332 for(i = 0; i < NV_DEVICE_NUM; ++i)
333 {
334 raw_spin_lock_init(&NV_DEVICE_REG[i].lock);
335 NV_DEVICE_REG[i].device_owner = NULL;
336 }
337
338 return(1);
339}
340
341/* use to get nv_device_id by given owner.
342 (if return -1, can't get the assocaite device id)*/
343/*
344int get_nv_device_id(struct task_struct* owner)
345{
346 int i;
347 if(!owner)
348 {
349 return(-1);
350 }
351 for(i = 0; i < NV_DEVICE_NUM; ++i)
352 {
353 if(NV_DEVICE_REG[i].device_owner == owner)
354 return(i);
355 }
356 return(-1);
357}
358*/
359
360
361
362static int __reg_nv_device(int reg_device_id)
363{
364 struct task_struct* old =
365 cmpxchg(&NV_DEVICE_REG[reg_device_id].device_owner,
366 NULL,
367 current);
368
369 mb();
370
371 if(likely(old == NULL))
372 {
373 down_and_set_stat(current, HELD, &tsk_rt(current)->klitirqd_sem);
374 TRACE_CUR("%s: device %d registered.\n", __FUNCTION__, reg_device_id);
375 return(0);
376 }
377 else
378 {
379 TRACE_CUR("%s: device %d is already in use!\n", __FUNCTION__, reg_device_id);
380 return(-EBUSY);
381 }
382
383#if 0
384 //unsigned long flags;
385 //raw_spin_lock_irqsave(&NV_DEVICE_REG[reg_device_id].lock, flags);
386 //lock_nv_registry(reg_device_id, &flags);
387
388 if(likely(NV_DEVICE_REG[reg_device_id].device_owner == NULL))
389 {
390 NV_DEVICE_REG[reg_device_id].device_owner = current;
391 mb(); // needed?
392
393 // release spin lock before chance of going to sleep.
394 //raw_spin_unlock_irqrestore(&NV_DEVICE_REG[reg_device_id].lock, flags);
395 //unlock_nv_registry(reg_device_id, &flags);
396
397 down_and_set_stat(current, HELD, &tsk_rt(current)->klitirqd_sem);
398 TRACE_CUR("%s: device %d registered.\n", __FUNCTION__, reg_device_id);
399 return(0);
400 }
401 else
402 {
403 //raw_spin_unlock_irqrestore(&NV_DEVICE_REG[reg_device_id].lock, flags);
404 //unlock_nv_registry(reg_device_id, &flags);
405
406 TRACE_CUR("%s: device %d is already in use!\n", __FUNCTION__, reg_device_id);
407 return(-EBUSY);
408 }
409#endif
410}
411
412static int __clear_reg_nv_device(int de_reg_device_id)
413{
414 int ret;
415 unsigned long flags;
416 struct task_struct* klitirqd_th = get_klitirqd(de_reg_device_id);
417 struct task_struct* old;
418
419 lock_nv_registry(de_reg_device_id, &flags);
420
421 old = cmpxchg(&NV_DEVICE_REG[de_reg_device_id].device_owner,
422 current,
423 NULL);
424
425 mb();
426
427 if(likely(old == current))
428 {
429 flush_pending(klitirqd_th, current);
430 //unlock_nv_registry(de_reg_device_id, &flags);
431
432 up_and_set_stat(current, NOT_HELD, &tsk_rt(current)->klitirqd_sem);
433
434 unlock_nv_registry(de_reg_device_id, &flags);
435 ret = 0;
436
437 TRACE_CUR("%s: semaphore released.\n",__FUNCTION__);
438 }
439 else
440 {
441 unlock_nv_registry(de_reg_device_id, &flags);
442 ret = -EINVAL;
443
444 if(old)
445 TRACE_CUR("%s: device %d is not registered for this process's use! %s/%d is!\n",
446 __FUNCTION__, de_reg_device_id, old->comm, old->pid);
447 else
448 TRACE_CUR("%s: device %d is not registered for this process's use! No one is!\n",
449 __FUNCTION__, de_reg_device_id);
450 }
451
452 return(ret);
453}
454
455
456int reg_nv_device(int reg_device_id, int reg_action)
457{
458 int ret;
459
460 if((reg_device_id < NV_DEVICE_NUM) && (reg_device_id >= 0))
461 {
462 if(reg_action)
463 ret = __reg_nv_device(reg_device_id);
464 else
465 ret = __clear_reg_nv_device(reg_device_id);
466 }
467 else
468 {
469 ret = -ENODEV;
470 }
471
472 return(ret);
473}
474
475/* use to get the owner of nv_device_id. */
476struct task_struct* get_nv_device_owner(u32 target_device_id)
477{
478 struct task_struct* owner;
479 BUG_ON(target_device_id >= NV_DEVICE_NUM);
480 owner = NV_DEVICE_REG[target_device_id].device_owner;
481 return(owner);
482}
483
484void lock_nv_registry(u32 target_device_id, unsigned long* flags)
485{
486 BUG_ON(target_device_id >= NV_DEVICE_NUM);
487
488 if(in_interrupt())
489 TRACE("Locking registry for %d.\n", target_device_id);
490 else
491 TRACE_CUR("Locking registry for %d.\n", target_device_id);
492
493 raw_spin_lock_irqsave(&NV_DEVICE_REG[target_device_id].lock, *flags);
494}
495
496void unlock_nv_registry(u32 target_device_id, unsigned long* flags)
497{
498 BUG_ON(target_device_id >= NV_DEVICE_NUM);
499
500 if(in_interrupt())
501 TRACE("Unlocking registry for %d.\n", target_device_id);
502 else
503 TRACE_CUR("Unlocking registry for %d.\n", target_device_id);
504
505 raw_spin_unlock_irqrestore(&NV_DEVICE_REG[target_device_id].lock, *flags);
506}
507
508
509void increment_nv_int_count(u32 device)
510{
511 unsigned long flags;
512 struct task_struct* owner;
513
514 lock_nv_registry(device, &flags);
515
516 owner = NV_DEVICE_REG[device].device_owner;
517 if(owner)
518 {
519 atomic_inc(&tsk_rt(owner)->nv_int_count);
520 }
521
522 unlock_nv_registry(device, &flags);
523}
524EXPORT_SYMBOL(increment_nv_int_count);
525
526
diff --git a/litmus/preempt.c b/litmus/preempt.c
index ebe2e3461895..08b98c3b57bf 100644
--- a/litmus/preempt.c
+++ b/litmus/preempt.c
@@ -30,8 +30,11 @@ void sched_state_will_schedule(struct task_struct* tsk)
30 /* Litmus tasks should never be subject to a remote 30 /* Litmus tasks should never be subject to a remote
31 * set_tsk_need_resched(). */ 31 * set_tsk_need_resched(). */
32 BUG_ON(is_realtime(tsk)); 32 BUG_ON(is_realtime(tsk));
33
34/*
33 TRACE_TASK(tsk, "set_tsk_need_resched() ret:%p\n", 35 TRACE_TASK(tsk, "set_tsk_need_resched() ret:%p\n",
34 __builtin_return_address(0)); 36 __builtin_return_address(0));
37*/
35} 38}
36 39
37/* Called by the IPI handler after another CPU called smp_send_resched(). */ 40/* Called by the IPI handler after another CPU called smp_send_resched(). */
@@ -43,13 +46,17 @@ void sched_state_ipi(void)
43 /* Cause scheduler to be invoked. 46 /* Cause scheduler to be invoked.
44 * This will cause a transition to WILL_SCHEDULE. */ 47 * This will cause a transition to WILL_SCHEDULE. */
45 set_tsk_need_resched(current); 48 set_tsk_need_resched(current);
49 /*
46 TRACE_STATE("IPI -> set_tsk_need_resched(%s/%d)\n", 50 TRACE_STATE("IPI -> set_tsk_need_resched(%s/%d)\n",
47 current->comm, current->pid); 51 current->comm, current->pid);
52 */
48 } else { 53 } else {
49 /* ignore */ 54 /* ignore */
55 /*
50 TRACE_STATE("ignoring IPI in state %x (%s)\n", 56 TRACE_STATE("ignoring IPI in state %x (%s)\n",
51 get_sched_state(), 57 get_sched_state(),
52 sched_state_name(get_sched_state())); 58 sched_state_name(get_sched_state()));
59 */
53 } 60 }
54} 61}
55 62
diff --git a/litmus/sched_cedf.c b/litmus/sched_cedf.c
index 73fe1c442a0d..9b0a8d3b624d 100644
--- a/litmus/sched_cedf.c
+++ b/litmus/sched_cedf.c
@@ -29,6 +29,7 @@
29#include <linux/percpu.h> 29#include <linux/percpu.h>
30#include <linux/sched.h> 30#include <linux/sched.h>
31#include <linux/slab.h> 31#include <linux/slab.h>
32#include <linux/uaccess.h>
32 33
33#include <linux/module.h> 34#include <linux/module.h>
34 35
@@ -45,7 +46,18 @@
45 46
46/* to configure the cluster size */ 47/* to configure the cluster size */
47#include <litmus/litmus_proc.h> 48#include <litmus/litmus_proc.h>
48#include <linux/uaccess.h> 49
50#ifdef CONFIG_SCHED_CPU_AFFINITY
51#include <litmus/affinity.h>
52#endif
53
54#ifdef CONFIG_LITMUS_SOFTIRQD
55#include <litmus/litmus_softirq.h>
56#endif
57
58#ifdef CONFIG_LITMUS_NVIDIA
59#include <litmus/nvidia_info.h>
60#endif
49 61
50/* Reference configuration variable. Determines which cache level is used to 62/* Reference configuration variable. Determines which cache level is used to
51 * group CPUs into clusters. GLOBAL_CLUSTER, which is the default, means that 63 * group CPUs into clusters. GLOBAL_CLUSTER, which is the default, means that
@@ -95,7 +107,7 @@ typedef struct clusterdomain {
95 struct bheap_node *heap_node; 107 struct bheap_node *heap_node;
96 struct bheap cpu_heap; 108 struct bheap cpu_heap;
97 /* lock for this cluster */ 109 /* lock for this cluster */
98#define lock domain.ready_lock 110#define cedf_lock domain.ready_lock
99} cedf_domain_t; 111} cedf_domain_t;
100 112
101/* a cedf_domain per cluster; allocation is done at init/activation time */ 113/* a cedf_domain per cluster; allocation is done at init/activation time */
@@ -257,21 +269,50 @@ static noinline void requeue(struct task_struct* task)
257 } 269 }
258} 270}
259 271
272#ifdef CONFIG_SCHED_CPU_AFFINITY
273static cpu_entry_t* cedf_get_nearest_available_cpu(
274 cedf_domain_t *cluster, cpu_entry_t* start)
275{
276 cpu_entry_t* affinity;
277
278 get_nearest_available_cpu(affinity, start, cedf_cpu_entries, -1);
279
280 /* make sure CPU is in our cluster */
281 if(affinity && cpu_isset(affinity->cpu, *cluster->cpu_map))
282 return(affinity);
283 else
284 return(NULL);
285}
286#endif
287
288
260/* check for any necessary preemptions */ 289/* check for any necessary preemptions */
261static void check_for_preemptions(cedf_domain_t *cluster) 290static void check_for_preemptions(cedf_domain_t *cluster)
262{ 291{
263 struct task_struct *task; 292 struct task_struct *task;
264 cpu_entry_t* last; 293 cpu_entry_t *last;
265 294
266 for(last = lowest_prio_cpu(cluster); 295 for(last = lowest_prio_cpu(cluster);
267 edf_preemption_needed(&cluster->domain, last->linked); 296 edf_preemption_needed(&cluster->domain, last->linked);
268 last = lowest_prio_cpu(cluster)) { 297 last = lowest_prio_cpu(cluster)) {
269 /* preemption necessary */ 298 /* preemption necessary */
270 task = __take_ready(&cluster->domain); 299 task = __take_ready(&cluster->domain);
271 TRACE("check_for_preemptions: attempting to link task %d to %d\n", 300#ifdef CONFIG_SCHED_CPU_AFFINITY
272 task->pid, last->cpu); 301 {
302 cpu_entry_t* affinity =
303 cedf_get_nearest_available_cpu(cluster,
304 &per_cpu(cedf_cpu_entries, task_cpu(task)));
305 if(affinity)
306 last = affinity;
307 else if(last->linked)
308 requeue(last->linked);
309 }
310#else
273 if (last->linked) 311 if (last->linked)
274 requeue(last->linked); 312 requeue(last->linked);
313#endif
314 TRACE("check_for_preemptions: attempting to link task %d to %d\n",
315 task->pid, last->cpu);
275 link_task_to_cpu(task, last); 316 link_task_to_cpu(task, last);
276 preempt(last); 317 preempt(last);
277 } 318 }
@@ -292,12 +333,12 @@ static void cedf_release_jobs(rt_domain_t* rt, struct bheap* tasks)
292 cedf_domain_t* cluster = container_of(rt, cedf_domain_t, domain); 333 cedf_domain_t* cluster = container_of(rt, cedf_domain_t, domain);
293 unsigned long flags; 334 unsigned long flags;
294 335
295 raw_spin_lock_irqsave(&cluster->lock, flags); 336 raw_spin_lock_irqsave(&cluster->cedf_lock, flags);
296 337
297 __merge_ready(&cluster->domain, tasks); 338 __merge_ready(&cluster->domain, tasks);
298 check_for_preemptions(cluster); 339 check_for_preemptions(cluster);
299 340
300 raw_spin_unlock_irqrestore(&cluster->lock, flags); 341 raw_spin_unlock_irqrestore(&cluster->cedf_lock, flags);
301} 342}
302 343
303/* caller holds cedf_lock */ 344/* caller holds cedf_lock */
@@ -307,6 +348,10 @@ static noinline void job_completion(struct task_struct *t, int forced)
307 348
308 sched_trace_task_completion(t, forced); 349 sched_trace_task_completion(t, forced);
309 350
351#ifdef CONFIG_LITMUS_NVIDIA
352 atomic_set(&tsk_rt(t)->nv_int_count, 0);
353#endif
354
310 TRACE_TASK(t, "job_completion().\n"); 355 TRACE_TASK(t, "job_completion().\n");
311 356
312 /* set flags */ 357 /* set flags */
@@ -378,7 +423,7 @@ static struct task_struct* cedf_schedule(struct task_struct * prev)
378 int out_of_time, sleep, preempt, np, exists, blocks; 423 int out_of_time, sleep, preempt, np, exists, blocks;
379 struct task_struct* next = NULL; 424 struct task_struct* next = NULL;
380 425
381 raw_spin_lock(&cluster->lock); 426 raw_spin_lock(&cluster->cedf_lock);
382 clear_will_schedule(); 427 clear_will_schedule();
383 428
384 /* sanity checking */ 429 /* sanity checking */
@@ -462,7 +507,7 @@ static struct task_struct* cedf_schedule(struct task_struct * prev)
462 next = prev; 507 next = prev;
463 508
464 sched_state_task_picked(); 509 sched_state_task_picked();
465 raw_spin_unlock(&cluster->lock); 510 raw_spin_unlock(&cluster->cedf_lock);
466 511
467#ifdef WANT_ALL_SCHED_EVENTS 512#ifdef WANT_ALL_SCHED_EVENTS
468 TRACE("cedf_lock released, next=0x%p\n", next); 513 TRACE("cedf_lock released, next=0x%p\n", next);
@@ -504,7 +549,7 @@ static void cedf_task_new(struct task_struct * t, int on_rq, int running)
504 /* the cluster doesn't change even if t is running */ 549 /* the cluster doesn't change even if t is running */
505 cluster = task_cpu_cluster(t); 550 cluster = task_cpu_cluster(t);
506 551
507 raw_spin_lock_irqsave(&cluster->domain.ready_lock, flags); 552 raw_spin_lock_irqsave(&cluster->cedf_lock, flags);
508 553
509 /* setup job params */ 554 /* setup job params */
510 release_at(t, litmus_clock()); 555 release_at(t, litmus_clock());
@@ -521,20 +566,22 @@ static void cedf_task_new(struct task_struct * t, int on_rq, int running)
521 t->rt_param.linked_on = NO_CPU; 566 t->rt_param.linked_on = NO_CPU;
522 567
523 cedf_job_arrival(t); 568 cedf_job_arrival(t);
524 raw_spin_unlock_irqrestore(&(cluster->domain.ready_lock), flags); 569 raw_spin_unlock_irqrestore(&cluster->cedf_lock, flags);
525} 570}
526 571
527static void cedf_task_wake_up(struct task_struct *task) 572static void cedf_task_wake_up(struct task_struct *task)
528{ 573{
529 unsigned long flags; 574 unsigned long flags;
530 lt_t now; 575 //lt_t now;
531 cedf_domain_t *cluster; 576 cedf_domain_t *cluster;
532 577
533 TRACE_TASK(task, "wake_up at %llu\n", litmus_clock()); 578 TRACE_TASK(task, "wake_up at %llu\n", litmus_clock());
534 579
535 cluster = task_cpu_cluster(task); 580 cluster = task_cpu_cluster(task);
536 581
537 raw_spin_lock_irqsave(&cluster->lock, flags); 582 raw_spin_lock_irqsave(&cluster->cedf_lock, flags);
583
584#if 0 // sporadic task model
538 /* We need to take suspensions because of semaphores into 585 /* We need to take suspensions because of semaphores into
539 * account! If a job resumes after being suspended due to acquiring 586 * account! If a job resumes after being suspended due to acquiring
540 * a semaphore, it should never be treated as a new job release. 587 * a semaphore, it should never be treated as a new job release.
@@ -556,8 +603,17 @@ static void cedf_task_wake_up(struct task_struct *task)
556 } 603 }
557 } 604 }
558 } 605 }
559 cedf_job_arrival(task); 606#endif
560 raw_spin_unlock_irqrestore(&cluster->lock, flags); 607
608 //BUG_ON(tsk_rt(task)->linked_on != NO_CPU);
609 set_rt_flags(task, RT_F_RUNNING); // periodic model
610
611 if(tsk_rt(task)->linked_on == NO_CPU)
612 cedf_job_arrival(task);
613 else
614 TRACE("WTF, mate?!\n");
615
616 raw_spin_unlock_irqrestore(&cluster->cedf_lock, flags);
561} 617}
562 618
563static void cedf_task_block(struct task_struct *t) 619static void cedf_task_block(struct task_struct *t)
@@ -570,9 +626,9 @@ static void cedf_task_block(struct task_struct *t)
570 cluster = task_cpu_cluster(t); 626 cluster = task_cpu_cluster(t);
571 627
572 /* unlink if necessary */ 628 /* unlink if necessary */
573 raw_spin_lock_irqsave(&cluster->lock, flags); 629 raw_spin_lock_irqsave(&cluster->cedf_lock, flags);
574 unlink(t); 630 unlink(t);
575 raw_spin_unlock_irqrestore(&cluster->lock, flags); 631 raw_spin_unlock_irqrestore(&cluster->cedf_lock, flags);
576 632
577 BUG_ON(!is_realtime(t)); 633 BUG_ON(!is_realtime(t));
578} 634}
@@ -584,7 +640,7 @@ static void cedf_task_exit(struct task_struct * t)
584 cedf_domain_t *cluster = task_cpu_cluster(t); 640 cedf_domain_t *cluster = task_cpu_cluster(t);
585 641
586 /* unlink if necessary */ 642 /* unlink if necessary */
587 raw_spin_lock_irqsave(&cluster->lock, flags); 643 raw_spin_lock_irqsave(&cluster->cedf_lock, flags);
588 unlink(t); 644 unlink(t);
589 if (tsk_rt(t)->scheduled_on != NO_CPU) { 645 if (tsk_rt(t)->scheduled_on != NO_CPU) {
590 cpu_entry_t *cpu; 646 cpu_entry_t *cpu;
@@ -592,7 +648,7 @@ static void cedf_task_exit(struct task_struct * t)
592 cpu->scheduled = NULL; 648 cpu->scheduled = NULL;
593 tsk_rt(t)->scheduled_on = NO_CPU; 649 tsk_rt(t)->scheduled_on = NO_CPU;
594 } 650 }
595 raw_spin_unlock_irqrestore(&cluster->lock, flags); 651 raw_spin_unlock_irqrestore(&cluster->cedf_lock, flags);
596 652
597 BUG_ON(!is_realtime(t)); 653 BUG_ON(!is_realtime(t));
598 TRACE_TASK(t, "RIP\n"); 654 TRACE_TASK(t, "RIP\n");
@@ -603,6 +659,721 @@ static long cedf_admit_task(struct task_struct* tsk)
603 return task_cpu(tsk) == tsk->rt_param.task_params.cpu ? 0 : -EINVAL; 659 return task_cpu(tsk) == tsk->rt_param.task_params.cpu ? 0 : -EINVAL;
604} 660}
605 661
662
663
664
665
666
667
668
669
670
671
672
673
674#ifdef CONFIG_LITMUS_LOCKING
675
676#include <litmus/fdso.h>
677
678
679static void __set_priority_inheritance(struct task_struct* t, struct task_struct* prio_inh)
680{
681 int linked_on;
682 int check_preempt = 0;
683
684 cedf_domain_t* cluster = task_cpu_cluster(t);
685
686 if(prio_inh != NULL)
687 TRACE_TASK(t, "inherits priority from %s/%d\n", prio_inh->comm, prio_inh->pid);
688 else
689 TRACE_TASK(t, "inherits priority from %p\n", prio_inh);
690
691 sched_trace_eff_prio_change(t, prio_inh);
692
693 tsk_rt(t)->inh_task = prio_inh;
694
695 linked_on = tsk_rt(t)->linked_on;
696
697 /* If it is scheduled, then we need to reorder the CPU heap. */
698 if (linked_on != NO_CPU) {
699 TRACE_TASK(t, "%s: linked on %d\n",
700 __FUNCTION__, linked_on);
701 /* Holder is scheduled; need to re-order CPUs.
702 * We can't use heap_decrease() here since
703 * the cpu_heap is ordered in reverse direction, so
704 * it is actually an increase. */
705 bheap_delete(cpu_lower_prio, &cluster->cpu_heap,
706 per_cpu(cedf_cpu_entries, linked_on).hn);
707 bheap_insert(cpu_lower_prio, &cluster->cpu_heap,
708 per_cpu(cedf_cpu_entries, linked_on).hn);
709 } else {
710 /* holder may be queued: first stop queue changes */
711 raw_spin_lock(&cluster->domain.release_lock);
712 if (is_queued(t)) {
713 TRACE_TASK(t, "%s: is queued\n", __FUNCTION__);
714
715 /* We need to update the position of holder in some
716 * heap. Note that this could be a release heap if we
717 * budget enforcement is used and this job overran. */
718 check_preempt = !bheap_decrease(edf_ready_order, tsk_rt(t)->heap_node);
719
720 } else {
721 /* Nothing to do: if it is not queued and not linked
722 * then it is either sleeping or currently being moved
723 * by other code (e.g., a timer interrupt handler) that
724 * will use the correct priority when enqueuing the
725 * task. */
726 TRACE_TASK(t, "%s: is NOT queued => Done.\n", __FUNCTION__);
727 }
728 raw_spin_unlock(&cluster->domain.release_lock);
729
730 /* If holder was enqueued in a release heap, then the following
731 * preemption check is pointless, but we can't easily detect
732 * that case. If you want to fix this, then consider that
733 * simply adding a state flag requires O(n) time to update when
734 * releasing n tasks, which conflicts with the goal to have
735 * O(log n) merges. */
736 if (check_preempt) {
737 /* heap_decrease() hit the top level of the heap: make
738 * sure preemption checks get the right task, not the
739 * potentially stale cache. */
740 bheap_uncache_min(edf_ready_order, &cluster->domain.ready_queue);
741 check_for_preemptions(cluster);
742 }
743 }
744}
745
746/* called with IRQs off */
747static void set_priority_inheritance(struct task_struct* t, struct task_struct* prio_inh)
748{
749 cedf_domain_t* cluster = task_cpu_cluster(t);
750
751 raw_spin_lock(&cluster->cedf_lock);
752
753 __set_priority_inheritance(t, prio_inh);
754
755#ifdef CONFIG_LITMUS_SOFTIRQD
756 if(tsk_rt(t)->cur_klitirqd != NULL)
757 {
758 TRACE_TASK(t, "%s/%d inherits a new priority!\n",
759 tsk_rt(t)->cur_klitirqd->comm, tsk_rt(t)->cur_klitirqd->pid);
760
761 __set_priority_inheritance(tsk_rt(t)->cur_klitirqd, prio_inh);
762 }
763#endif
764
765 raw_spin_unlock(&cluster->cedf_lock);
766}
767
768
769/* called with IRQs off */
770static void __clear_priority_inheritance(struct task_struct* t)
771{
772 TRACE_TASK(t, "priority restored\n");
773
774 if(tsk_rt(t)->scheduled_on != NO_CPU)
775 {
776 sched_trace_eff_prio_change(t, NULL);
777
778 tsk_rt(t)->inh_task = NULL;
779
780 /* Check if rescheduling is necessary. We can't use heap_decrease()
781 * since the priority was effectively lowered. */
782 unlink(t);
783 cedf_job_arrival(t);
784 }
785 else
786 {
787 __set_priority_inheritance(t, NULL);
788 }
789
790#ifdef CONFIG_LITMUS_SOFTIRQD
791 if(tsk_rt(t)->cur_klitirqd != NULL)
792 {
793 TRACE_TASK(t, "%s/%d inheritance set back to owner.\n",
794 tsk_rt(t)->cur_klitirqd->comm, tsk_rt(t)->cur_klitirqd->pid);
795
796 if(tsk_rt(tsk_rt(t)->cur_klitirqd)->scheduled_on != NO_CPU)
797 {
798 sched_trace_eff_prio_change(tsk_rt(t)->cur_klitirqd, t);
799
800 tsk_rt(tsk_rt(t)->cur_klitirqd)->inh_task = t;
801
802 /* Check if rescheduling is necessary. We can't use heap_decrease()
803 * since the priority was effectively lowered. */
804 unlink(tsk_rt(t)->cur_klitirqd);
805 cedf_job_arrival(tsk_rt(t)->cur_klitirqd);
806 }
807 else
808 {
809 __set_priority_inheritance(tsk_rt(t)->cur_klitirqd, t);
810 }
811 }
812#endif
813}
814
815/* called with IRQs off */
816static void clear_priority_inheritance(struct task_struct* t)
817{
818 cedf_domain_t* cluster = task_cpu_cluster(t);
819
820 raw_spin_lock(&cluster->cedf_lock);
821 __clear_priority_inheritance(t);
822 raw_spin_unlock(&cluster->cedf_lock);
823}
824
825
826
827#ifdef CONFIG_LITMUS_SOFTIRQD
828/* called with IRQs off */
829static void set_priority_inheritance_klitirqd(struct task_struct* klitirqd,
830 struct task_struct* old_owner,
831 struct task_struct* new_owner)
832{
833 cedf_domain_t* cluster = task_cpu_cluster(klitirqd);
834
835 BUG_ON(!(tsk_rt(klitirqd)->is_proxy_thread));
836
837 raw_spin_lock(&cluster->cedf_lock);
838
839 if(old_owner != new_owner)
840 {
841 if(old_owner)
842 {
843 // unreachable?
844 tsk_rt(old_owner)->cur_klitirqd = NULL;
845 }
846
847 TRACE_TASK(klitirqd, "giving ownership to %s/%d.\n",
848 new_owner->comm, new_owner->pid);
849
850 tsk_rt(new_owner)->cur_klitirqd = klitirqd;
851 }
852
853 __set_priority_inheritance(klitirqd,
854 (tsk_rt(new_owner)->inh_task == NULL) ?
855 new_owner :
856 tsk_rt(new_owner)->inh_task);
857
858 raw_spin_unlock(&cluster->cedf_lock);
859}
860
861/* called with IRQs off */
862static void clear_priority_inheritance_klitirqd(struct task_struct* klitirqd,
863 struct task_struct* old_owner)
864{
865 cedf_domain_t* cluster = task_cpu_cluster(klitirqd);
866
867 BUG_ON(!(tsk_rt(klitirqd)->is_proxy_thread));
868
869 raw_spin_lock(&cluster->cedf_lock);
870
871 TRACE_TASK(klitirqd, "priority restored\n");
872
873 if(tsk_rt(klitirqd)->scheduled_on != NO_CPU)
874 {
875 tsk_rt(klitirqd)->inh_task = NULL;
876
877 /* Check if rescheduling is necessary. We can't use heap_decrease()
878 * since the priority was effectively lowered. */
879 unlink(klitirqd);
880 cedf_job_arrival(klitirqd);
881 }
882 else
883 {
884 __set_priority_inheritance(klitirqd, NULL);
885 }
886
887 tsk_rt(old_owner)->cur_klitirqd = NULL;
888
889 raw_spin_unlock(&cluster->cedf_lock);
890}
891#endif // CONFIG_LITMUS_SOFTIRQD
892
893
894/* ******************** KFMLP support ********************** */
895
896/* struct for semaphore with priority inheritance */
897struct kfmlp_queue
898{
899 wait_queue_head_t wait;
900 struct task_struct* owner;
901 struct task_struct* hp_waiter;
902 int count; /* number of waiters + holder */
903};
904
905struct kfmlp_semaphore
906{
907 struct litmus_lock litmus_lock;
908
909 spinlock_t lock;
910
911 int num_resources; /* aka k */
912 struct kfmlp_queue *queues; /* array */
913 struct kfmlp_queue *shortest_queue; /* pointer to shortest queue */
914};
915
916static inline struct kfmlp_semaphore* kfmlp_from_lock(struct litmus_lock* lock)
917{
918 return container_of(lock, struct kfmlp_semaphore, litmus_lock);
919}
920
921static inline int kfmlp_get_idx(struct kfmlp_semaphore* sem,
922 struct kfmlp_queue* queue)
923{
924 return (queue - &sem->queues[0]);
925}
926
927static inline struct kfmlp_queue* kfmlp_get_queue(struct kfmlp_semaphore* sem,
928 struct task_struct* holder)
929{
930 int i;
931 for(i = 0; i < sem->num_resources; ++i)
932 if(sem->queues[i].owner == holder)
933 return(&sem->queues[i]);
934 return(NULL);
935}
936
937/* caller is responsible for locking */
938static struct task_struct* kfmlp_find_hp_waiter(struct kfmlp_queue *kqueue,
939 struct task_struct *skip)
940{
941 struct list_head *pos;
942 struct task_struct *queued, *found = NULL;
943
944 list_for_each(pos, &kqueue->wait.task_list) {
945 queued = (struct task_struct*) list_entry(pos, wait_queue_t,
946 task_list)->private;
947
948 /* Compare task prios, find high prio task. */
949 if (queued != skip && edf_higher_prio(queued, found))
950 found = queued;
951 }
952 return found;
953}
954
955static inline struct kfmlp_queue* kfmlp_find_shortest(
956 struct kfmlp_semaphore* sem,
957 struct kfmlp_queue* search_start)
958{
959 // we start our search at search_start instead of at the beginning of the
960 // queue list to load-balance across all resources.
961 struct kfmlp_queue* step = search_start;
962 struct kfmlp_queue* shortest = sem->shortest_queue;
963
964 do
965 {
966 step = (step+1 != &sem->queues[sem->num_resources]) ?
967 step+1 : &sem->queues[0];
968 if(step->count < shortest->count)
969 {
970 shortest = step;
971 if(step->count == 0)
972 break; /* can't get any shorter */
973 }
974 }while(step != search_start);
975
976 return(shortest);
977}
978
979static struct task_struct* kfmlp_remove_hp_waiter(struct kfmlp_semaphore* sem)
980{
981 /* must hold sem->lock */
982
983 struct kfmlp_queue *my_queue = NULL;
984 struct task_struct *max_hp = NULL;
985
986
987 struct list_head *pos;
988 struct task_struct *queued;
989 int i;
990
991 for(i = 0; i < sem->num_resources; ++i)
992 {
993 if( (sem->queues[i].count > 1) &&
994 ((my_queue == NULL) ||
995 (edf_higher_prio(sem->queues[i].hp_waiter, my_queue->hp_waiter))) )
996 {
997 my_queue = &sem->queues[i];
998 }
999 }
1000
1001 if(my_queue)
1002 {
1003 cedf_domain_t* cluster;
1004
1005 max_hp = my_queue->hp_waiter;
1006 BUG_ON(!max_hp);
1007
1008 TRACE_CUR("queue %d: stealing %s/%d from queue %d\n",
1009 kfmlp_get_idx(sem, my_queue),
1010 max_hp->comm, max_hp->pid,
1011 kfmlp_get_idx(sem, my_queue));
1012
1013 my_queue->hp_waiter = kfmlp_find_hp_waiter(my_queue, max_hp);
1014
1015 /*
1016 if(my_queue->hp_waiter)
1017 TRACE_CUR("queue %d: new hp_waiter is %s/%d\n",
1018 kfmlp_get_idx(sem, my_queue),
1019 my_queue->hp_waiter->comm,
1020 my_queue->hp_waiter->pid);
1021 else
1022 TRACE_CUR("queue %d: new hp_waiter is %p\n",
1023 kfmlp_get_idx(sem, my_queue), NULL);
1024 */
1025
1026 cluster = task_cpu_cluster(max_hp);
1027
1028 raw_spin_lock(&cluster->cedf_lock);
1029
1030 /*
1031 if(my_queue->owner)
1032 TRACE_CUR("queue %d: owner is %s/%d\n",
1033 kfmlp_get_idx(sem, my_queue),
1034 my_queue->owner->comm,
1035 my_queue->owner->pid);
1036 else
1037 TRACE_CUR("queue %d: owner is %p\n",
1038 kfmlp_get_idx(sem, my_queue),
1039 NULL);
1040 */
1041
1042 if(tsk_rt(my_queue->owner)->inh_task == max_hp)
1043 {
1044 __clear_priority_inheritance(my_queue->owner);
1045 if(my_queue->hp_waiter != NULL)
1046 {
1047 __set_priority_inheritance(my_queue->owner, my_queue->hp_waiter);
1048 }
1049 }
1050 raw_spin_unlock(&cluster->cedf_lock);
1051
1052 list_for_each(pos, &my_queue->wait.task_list)
1053 {
1054 queued = (struct task_struct*) list_entry(pos, wait_queue_t,
1055 task_list)->private;
1056 /* Compare task prios, find high prio task. */
1057 if (queued == max_hp)
1058 {
1059 /*
1060 TRACE_CUR("queue %d: found entry in wait queue. REMOVING!\n",
1061 kfmlp_get_idx(sem, my_queue));
1062 */
1063 __remove_wait_queue(&my_queue->wait,
1064 list_entry(pos, wait_queue_t, task_list));
1065 break;
1066 }
1067 }
1068 --(my_queue->count);
1069 }
1070
1071 return(max_hp);
1072}
1073
1074int cedf_kfmlp_lock(struct litmus_lock* l)
1075{
1076 struct task_struct* t = current;
1077 struct kfmlp_semaphore *sem = kfmlp_from_lock(l);
1078 struct kfmlp_queue* my_queue;
1079 wait_queue_t wait;
1080 unsigned long flags;
1081
1082 if (!is_realtime(t))
1083 return -EPERM;
1084
1085 spin_lock_irqsave(&sem->lock, flags);
1086
1087 my_queue = sem->shortest_queue;
1088
1089 if (my_queue->owner) {
1090 /* resource is not free => must suspend and wait */
1091 TRACE_CUR("queue %d: Resource is not free => must suspend and wait.\n",
1092 kfmlp_get_idx(sem, my_queue));
1093
1094 init_waitqueue_entry(&wait, t);
1095
1096 /* FIXME: interruptible would be nice some day */
1097 set_task_state(t, TASK_UNINTERRUPTIBLE);
1098
1099 __add_wait_queue_tail_exclusive(&my_queue->wait, &wait);
1100
1101 /* check if we need to activate priority inheritance */
1102 if (edf_higher_prio(t, my_queue->hp_waiter))
1103 {
1104 my_queue->hp_waiter = t;
1105 if (edf_higher_prio(t, my_queue->owner))
1106 {
1107 set_priority_inheritance(my_queue->owner, my_queue->hp_waiter);
1108 }
1109 }
1110
1111 ++(my_queue->count);
1112 sem->shortest_queue = kfmlp_find_shortest(sem, my_queue);
1113
1114 /* release lock before sleeping */
1115 spin_unlock_irqrestore(&sem->lock, flags);
1116
1117 /* We depend on the FIFO order. Thus, we don't need to recheck
1118 * when we wake up; we are guaranteed to have the lock since
1119 * there is only one wake up per release (or steal).
1120 */
1121 schedule();
1122
1123
1124 if(my_queue->owner == t)
1125 {
1126 TRACE_CUR("queue %d: acquired through waiting\n",
1127 kfmlp_get_idx(sem, my_queue));
1128 }
1129 else
1130 {
1131 /* this case may happen if our wait entry was stolen
1132 between queues. record where we went.*/
1133 my_queue = kfmlp_get_queue(sem, t);
1134 BUG_ON(!my_queue);
1135 TRACE_CUR("queue %d: acquired through stealing\n",
1136 kfmlp_get_idx(sem, my_queue));
1137 }
1138 }
1139 else
1140 {
1141 TRACE_CUR("queue %d: acquired immediately\n",
1142 kfmlp_get_idx(sem, my_queue));
1143
1144 my_queue->owner = t;
1145
1146 ++(my_queue->count);
1147 sem->shortest_queue = kfmlp_find_shortest(sem, my_queue);
1148
1149 spin_unlock_irqrestore(&sem->lock, flags);
1150 }
1151
1152 return kfmlp_get_idx(sem, my_queue);
1153}
1154
1155int cedf_kfmlp_unlock(struct litmus_lock* l)
1156{
1157 struct task_struct *t = current, *next;
1158 struct kfmlp_semaphore *sem = kfmlp_from_lock(l);
1159 struct kfmlp_queue *my_queue;
1160 unsigned long flags;
1161 int err = 0;
1162
1163 spin_lock_irqsave(&sem->lock, flags);
1164
1165 my_queue = kfmlp_get_queue(sem, t);
1166
1167 if (!my_queue) {
1168 err = -EINVAL;
1169 goto out;
1170 }
1171
1172 /* check if there are jobs waiting for this resource */
1173 next = __waitqueue_remove_first(&my_queue->wait);
1174 if (next) {
1175 /*
1176 TRACE_CUR("queue %d: ASSIGNING %s/%d as owner - next\n",
1177 kfmlp_get_idx(sem, my_queue),
1178 next->comm, next->pid);
1179 */
1180 /* next becomes the resouce holder */
1181 my_queue->owner = next;
1182
1183 --(my_queue->count);
1184 if(my_queue->count < sem->shortest_queue->count)
1185 {
1186 sem->shortest_queue = my_queue;
1187 }
1188
1189 TRACE_CUR("queue %d: lock ownership passed to %s/%d\n",
1190 kfmlp_get_idx(sem, my_queue), next->comm, next->pid);
1191
1192 /* determine new hp_waiter if necessary */
1193 if (next == my_queue->hp_waiter) {
1194 TRACE_TASK(next, "was highest-prio waiter\n");
1195 /* next has the highest priority --- it doesn't need to
1196 * inherit. However, we need to make sure that the
1197 * next-highest priority in the queue is reflected in
1198 * hp_waiter. */
1199 my_queue->hp_waiter = kfmlp_find_hp_waiter(my_queue, next);
1200 if (my_queue->hp_waiter)
1201 TRACE_TASK(my_queue->hp_waiter, "queue %d: is new highest-prio waiter\n", kfmlp_get_idx(sem, my_queue));
1202 else
1203 TRACE("queue %d: no further waiters\n", kfmlp_get_idx(sem, my_queue));
1204 } else {
1205 /* Well, if next is not the highest-priority waiter,
1206 * then it ought to inherit the highest-priority
1207 * waiter's priority. */
1208 set_priority_inheritance(next, my_queue->hp_waiter);
1209 }
1210
1211 /* wake up next */
1212 wake_up_process(next);
1213 }
1214 else
1215 {
1216 TRACE_CUR("queue %d: looking to steal someone...\n", kfmlp_get_idx(sem, my_queue));
1217
1218 next = kfmlp_remove_hp_waiter(sem); /* returns NULL if nothing to steal */
1219
1220 /*
1221 if(next)
1222 TRACE_CUR("queue %d: ASSIGNING %s/%d as owner - steal\n",
1223 kfmlp_get_idx(sem, my_queue),
1224 next->comm, next->pid);
1225 */
1226
1227 my_queue->owner = next;
1228
1229 if(next)
1230 {
1231 TRACE_CUR("queue %d: lock ownership passed to %s/%d (which was stolen)\n",
1232 kfmlp_get_idx(sem, my_queue),
1233 next->comm, next->pid);
1234
1235 /* wake up next */
1236 wake_up_process(next);
1237 }
1238 else
1239 {
1240 TRACE_CUR("queue %d: no one to steal.\n", kfmlp_get_idx(sem, my_queue));
1241
1242 --(my_queue->count);
1243 if(my_queue->count < sem->shortest_queue->count)
1244 {
1245 sem->shortest_queue = my_queue;
1246 }
1247 }
1248 }
1249
1250 /* we lose the benefit of priority inheritance (if any) */
1251 if (tsk_rt(t)->inh_task)
1252 clear_priority_inheritance(t);
1253
1254out:
1255 spin_unlock_irqrestore(&sem->lock, flags);
1256
1257 return err;
1258}
1259
1260int cedf_kfmlp_close(struct litmus_lock* l)
1261{
1262 struct task_struct *t = current;
1263 struct kfmlp_semaphore *sem = kfmlp_from_lock(l);
1264 struct kfmlp_queue *my_queue;
1265 unsigned long flags;
1266
1267 int owner;
1268
1269 spin_lock_irqsave(&sem->lock, flags);
1270
1271 my_queue = kfmlp_get_queue(sem, t);
1272 owner = (my_queue) ? (my_queue->owner == t) : 0;
1273
1274 spin_unlock_irqrestore(&sem->lock, flags);
1275
1276 if (owner)
1277 cedf_kfmlp_unlock(l);
1278
1279 return 0;
1280}
1281
1282void cedf_kfmlp_free(struct litmus_lock* l)
1283{
1284 struct kfmlp_semaphore *sem = kfmlp_from_lock(l);
1285 kfree(sem->queues);
1286 kfree(sem);
1287}
1288
1289static struct litmus_lock_ops cedf_kfmlp_lock_ops = {
1290 .close = cedf_kfmlp_close,
1291 .lock = cedf_kfmlp_lock,
1292 .unlock = cedf_kfmlp_unlock,
1293 .deallocate = cedf_kfmlp_free,
1294};
1295
1296static struct litmus_lock* cedf_new_kfmlp(void* __user arg, int* ret_code)
1297{
1298 struct kfmlp_semaphore* sem;
1299 int num_resources = 0;
1300 int i;
1301
1302 if(!access_ok(VERIFY_READ, arg, sizeof(num_resources)))
1303 {
1304 *ret_code = -EINVAL;
1305 return(NULL);
1306 }
1307 if(__copy_from_user(&num_resources, arg, sizeof(num_resources)))
1308 {
1309 *ret_code = -EINVAL;
1310 return(NULL);
1311 }
1312 if(num_resources < 1)
1313 {
1314 *ret_code = -EINVAL;
1315 return(NULL);
1316 }
1317
1318 sem = kmalloc(sizeof(*sem), GFP_KERNEL);
1319 if(!sem)
1320 {
1321 *ret_code = -ENOMEM;
1322 return NULL;
1323 }
1324
1325 sem->queues = kmalloc(sizeof(struct kfmlp_queue)*num_resources, GFP_KERNEL);
1326 if(!sem->queues)
1327 {
1328 kfree(sem);
1329 *ret_code = -ENOMEM;
1330 return NULL;
1331 }
1332
1333 sem->litmus_lock.ops = &cedf_kfmlp_lock_ops;
1334 spin_lock_init(&sem->lock);
1335 sem->num_resources = num_resources;
1336
1337 for(i = 0; i < num_resources; ++i)
1338 {
1339 sem->queues[i].owner = NULL;
1340 sem->queues[i].hp_waiter = NULL;
1341 init_waitqueue_head(&sem->queues[i].wait);
1342 sem->queues[i].count = 0;
1343 }
1344
1345 sem->shortest_queue = &sem->queues[0];
1346
1347 *ret_code = 0;
1348 return &sem->litmus_lock;
1349}
1350
1351
1352/* **** lock constructor **** */
1353
1354static long cedf_allocate_lock(struct litmus_lock **lock, int type,
1355 void* __user arg)
1356{
1357 int err = -ENXIO;
1358
1359 /* C-EDF currently only supports the FMLP for global resources
1360 WITHIN a given cluster. DO NOT USE CROSS-CLUSTER! */
1361 switch (type) {
1362 case KFMLP_SEM:
1363 *lock = cedf_new_kfmlp(arg, &err);
1364 break;
1365 };
1366
1367 return err;
1368}
1369
1370#endif // CONFIG_LITMUS_LOCKING
1371
1372
1373
1374
1375
1376
606/* total number of cluster */ 1377/* total number of cluster */
607static int num_clusters; 1378static int num_clusters;
608/* we do not support cluster of different sizes */ 1379/* we do not support cluster of different sizes */
@@ -746,6 +1517,40 @@ static long cedf_activate_plugin(void)
746 break; 1517 break;
747 } 1518 }
748 } 1519 }
1520
1521#ifdef CONFIG_LITMUS_SOFTIRQD
1522 {
1523 /* distribute the daemons evenly across the clusters. */
1524 int* affinity = kmalloc(NR_LITMUS_SOFTIRQD * sizeof(int), GFP_ATOMIC);
1525 int num_daemons_per_cluster = NR_LITMUS_SOFTIRQD / num_clusters;
1526 int left_over = NR_LITMUS_SOFTIRQD % num_clusters;
1527
1528 int daemon = 0;
1529 for(i = 0; i < num_clusters; ++i)
1530 {
1531 int num_on_this_cluster = num_daemons_per_cluster;
1532 if(left_over)
1533 {
1534 ++num_on_this_cluster;
1535 --left_over;
1536 }
1537
1538 for(j = 0; j < num_on_this_cluster; ++j)
1539 {
1540 // first CPU of this cluster
1541 affinity[daemon++] = i*cluster_size;
1542 }
1543 }
1544
1545 spawn_klitirqd(affinity);
1546
1547 kfree(affinity);
1548 }
1549#endif
1550
1551#ifdef CONFIG_LITMUS_NVIDIA
1552 init_nvidia_info();
1553#endif
749 1554
750 free_cpumask_var(mask); 1555 free_cpumask_var(mask);
751 clusters_allocated = 1; 1556 clusters_allocated = 1;
@@ -765,6 +1570,15 @@ static struct sched_plugin cedf_plugin __cacheline_aligned_in_smp = {
765 .task_block = cedf_task_block, 1570 .task_block = cedf_task_block,
766 .admit_task = cedf_admit_task, 1571 .admit_task = cedf_admit_task,
767 .activate_plugin = cedf_activate_plugin, 1572 .activate_plugin = cedf_activate_plugin,
1573#ifdef CONFIG_LITMUS_LOCKING
1574 .allocate_lock = cedf_allocate_lock,
1575 .set_prio_inh = set_priority_inheritance,
1576 .clear_prio_inh = clear_priority_inheritance,
1577#endif
1578#ifdef CONFIG_LITMUS_SOFTIRQD
1579 .set_prio_inh_klitirqd = set_priority_inheritance_klitirqd,
1580 .clear_prio_inh_klitirqd = clear_priority_inheritance_klitirqd,
1581#endif
768}; 1582};
769 1583
770static struct proc_dir_entry *cluster_file = NULL, *cedf_dir = NULL; 1584static struct proc_dir_entry *cluster_file = NULL, *cedf_dir = NULL;
diff --git a/litmus/sched_gsn_edf.c b/litmus/sched_gsn_edf.c
index 3092797480f8..d04e0703c154 100644
--- a/litmus/sched_gsn_edf.c
+++ b/litmus/sched_gsn_edf.c
@@ -12,6 +12,8 @@
12#include <linux/percpu.h> 12#include <linux/percpu.h>
13#include <linux/sched.h> 13#include <linux/sched.h>
14#include <linux/slab.h> 14#include <linux/slab.h>
15#include <linux/uaccess.h>
16
15 17
16#include <litmus/litmus.h> 18#include <litmus/litmus.h>
17#include <litmus/jobs.h> 19#include <litmus/jobs.h>
@@ -25,6 +27,19 @@
25 27
26#include <linux/module.h> 28#include <linux/module.h>
27 29
30#ifdef CONFIG_SCHED_CPU_AFFINITY
31#include <litmus/affinity.h>
32#endif
33
34#ifdef CONFIG_LITMUS_SOFTIRQD
35#include <litmus/litmus_softirq.h>
36#endif
37
38#ifdef CONFIG_LITMUS_NVIDIA
39#include <litmus/nvidia_info.h>
40#endif
41
42
28/* Overview of GSN-EDF operations. 43/* Overview of GSN-EDF operations.
29 * 44 *
30 * For a detailed explanation of GSN-EDF have a look at the FMLP paper. This 45 * For a detailed explanation of GSN-EDF have a look at the FMLP paper. This
@@ -253,21 +268,52 @@ static noinline void requeue(struct task_struct* task)
253 } 268 }
254} 269}
255 270
271#ifdef CONFIG_SCHED_CPU_AFFINITY
272static cpu_entry_t* gsnedf_get_nearest_available_cpu(cpu_entry_t* start)
273{
274 cpu_entry_t* affinity;
275
276 get_nearest_available_cpu(affinity, start, gsnedf_cpu_entries,
277#ifdef CONFIG_RELEASE_MASTER
278 gsnedf.release_master
279#else
280 -1
281#endif
282 );
283
284 return(affinity);
285}
286#endif
287
256/* check for any necessary preemptions */ 288/* check for any necessary preemptions */
257static void check_for_preemptions(void) 289static void check_for_preemptions(void)
258{ 290{
259 struct task_struct *task; 291 struct task_struct *task;
260 cpu_entry_t* last; 292 cpu_entry_t *last;
261 293
262 for(last = lowest_prio_cpu(); 294 for(last = lowest_prio_cpu();
263 edf_preemption_needed(&gsnedf, last->linked); 295 edf_preemption_needed(&gsnedf, last->linked);
264 last = lowest_prio_cpu()) { 296 last = lowest_prio_cpu()) {
265 /* preemption necessary */ 297 /* preemption necessary */
266 task = __take_ready(&gsnedf); 298 task = __take_ready(&gsnedf);
267 TRACE("check_for_preemptions: attempting to link task %d to %d\n", 299
268 task->pid, last->cpu); 300#ifdef CONFIG_SCHED_CPU_AFFINITY
301 {
302 cpu_entry_t* affinity = gsnedf_get_nearest_available_cpu(
303 &per_cpu(gsnedf_cpu_entries, task_cpu(task)));
304 if(affinity)
305 last = affinity;
306 else if(last->linked)
307 requeue(last->linked);
308 }
309#else
269 if (last->linked) 310 if (last->linked)
270 requeue(last->linked); 311 requeue(last->linked);
312#endif
313
314 TRACE("check_for_preemptions: attempting to link task %d to %d\n",
315 task->pid, last->cpu);
316
271 link_task_to_cpu(task, last); 317 link_task_to_cpu(task, last);
272 preempt(last); 318 preempt(last);
273 } 319 }
@@ -277,7 +323,7 @@ static void check_for_preemptions(void)
277static noinline void gsnedf_job_arrival(struct task_struct* task) 323static noinline void gsnedf_job_arrival(struct task_struct* task)
278{ 324{
279 BUG_ON(!task); 325 BUG_ON(!task);
280 326
281 requeue(task); 327 requeue(task);
282 check_for_preemptions(); 328 check_for_preemptions();
283} 329}
@@ -298,9 +344,13 @@ static void gsnedf_release_jobs(rt_domain_t* rt, struct bheap* tasks)
298static noinline void job_completion(struct task_struct *t, int forced) 344static noinline void job_completion(struct task_struct *t, int forced)
299{ 345{
300 BUG_ON(!t); 346 BUG_ON(!t);
301 347
302 sched_trace_task_completion(t, forced); 348 sched_trace_task_completion(t, forced);
303 349
350#ifdef CONFIG_LITMUS_NVIDIA
351 atomic_set(&tsk_rt(t)->nv_int_count, 0);
352#endif
353
304 TRACE_TASK(t, "job_completion().\n"); 354 TRACE_TASK(t, "job_completion().\n");
305 355
306 /* set flags */ 356 /* set flags */
@@ -401,17 +451,19 @@ static struct task_struct* gsnedf_schedule(struct task_struct * prev)
401 TRACE_TASK(prev, "invoked gsnedf_schedule.\n"); 451 TRACE_TASK(prev, "invoked gsnedf_schedule.\n");
402#endif 452#endif
403 453
454 /*
404 if (exists) 455 if (exists)
405 TRACE_TASK(prev, 456 TRACE_TASK(prev,
406 "blocks:%d out_of_time:%d np:%d sleep:%d preempt:%d " 457 "blocks:%d out_of_time:%d np:%d sleep:%d preempt:%d "
407 "state:%d sig:%d\n", 458 "state:%d sig:%d\n",
408 blocks, out_of_time, np, sleep, preempt, 459 blocks, out_of_time, np, sleep, preempt,
409 prev->state, signal_pending(prev)); 460 prev->state, signal_pending(prev));
461 */
462
410 if (entry->linked && preempt) 463 if (entry->linked && preempt)
411 TRACE_TASK(prev, "will be preempted by %s/%d\n", 464 TRACE_TASK(prev, "will be preempted by %s/%d\n",
412 entry->linked->comm, entry->linked->pid); 465 entry->linked->comm, entry->linked->pid);
413 466
414
415 /* If a task blocks we have no choice but to reschedule. 467 /* If a task blocks we have no choice but to reschedule.
416 */ 468 */
417 if (blocks) 469 if (blocks)
@@ -456,12 +508,15 @@ static struct task_struct* gsnedf_schedule(struct task_struct * prev)
456 entry->scheduled->rt_param.scheduled_on = NO_CPU; 508 entry->scheduled->rt_param.scheduled_on = NO_CPU;
457 TRACE_TASK(entry->scheduled, "scheduled_on = NO_CPU\n"); 509 TRACE_TASK(entry->scheduled, "scheduled_on = NO_CPU\n");
458 } 510 }
459 } else 511 }
512 else
513 {
460 /* Only override Linux scheduler if we have a real-time task 514 /* Only override Linux scheduler if we have a real-time task
461 * scheduled that needs to continue. 515 * scheduled that needs to continue.
462 */ 516 */
463 if (exists) 517 if (exists)
464 next = prev; 518 next = prev;
519 }
465 520
466 sched_state_task_picked(); 521 sched_state_task_picked();
467 522
@@ -486,8 +541,9 @@ static struct task_struct* gsnedf_schedule(struct task_struct * prev)
486static void gsnedf_finish_switch(struct task_struct *prev) 541static void gsnedf_finish_switch(struct task_struct *prev)
487{ 542{
488 cpu_entry_t* entry = &__get_cpu_var(gsnedf_cpu_entries); 543 cpu_entry_t* entry = &__get_cpu_var(gsnedf_cpu_entries);
489 544
490 entry->scheduled = is_realtime(current) ? current : NULL; 545 entry->scheduled = is_realtime(current) ? current : NULL;
546
491#ifdef WANT_ALL_SCHED_EVENTS 547#ifdef WANT_ALL_SCHED_EVENTS
492 TRACE_TASK(prev, "switched away from\n"); 548 TRACE_TASK(prev, "switched away from\n");
493#endif 549#endif
@@ -536,11 +592,14 @@ static void gsnedf_task_new(struct task_struct * t, int on_rq, int running)
536static void gsnedf_task_wake_up(struct task_struct *task) 592static void gsnedf_task_wake_up(struct task_struct *task)
537{ 593{
538 unsigned long flags; 594 unsigned long flags;
539 lt_t now; 595 lt_t now;
540 596
541 TRACE_TASK(task, "wake_up at %llu\n", litmus_clock()); 597 TRACE_TASK(task, "wake_up at %llu\n", litmus_clock());
542 598
543 raw_spin_lock_irqsave(&gsnedf_lock, flags); 599 raw_spin_lock_irqsave(&gsnedf_lock, flags);
600
601
602#if 0 // sporadic task model
544 /* We need to take suspensions because of semaphores into 603 /* We need to take suspensions because of semaphores into
545 * account! If a job resumes after being suspended due to acquiring 604 * account! If a job resumes after being suspended due to acquiring
546 * a semaphore, it should never be treated as a new job release. 605 * a semaphore, it should never be treated as a new job release.
@@ -562,19 +621,26 @@ static void gsnedf_task_wake_up(struct task_struct *task)
562 } 621 }
563 } 622 }
564 } 623 }
624#else // periodic task model
625 set_rt_flags(task, RT_F_RUNNING);
626#endif
627
565 gsnedf_job_arrival(task); 628 gsnedf_job_arrival(task);
566 raw_spin_unlock_irqrestore(&gsnedf_lock, flags); 629 raw_spin_unlock_irqrestore(&gsnedf_lock, flags);
567} 630}
568 631
569static void gsnedf_task_block(struct task_struct *t) 632static void gsnedf_task_block(struct task_struct *t)
570{ 633{
634 // TODO: is this called on preemption??
571 unsigned long flags; 635 unsigned long flags;
572 636
573 TRACE_TASK(t, "block at %llu\n", litmus_clock()); 637 TRACE_TASK(t, "block at %llu\n", litmus_clock());
574 638
575 /* unlink if necessary */ 639 /* unlink if necessary */
576 raw_spin_lock_irqsave(&gsnedf_lock, flags); 640 raw_spin_lock_irqsave(&gsnedf_lock, flags);
641
577 unlink(t); 642 unlink(t);
643
578 raw_spin_unlock_irqrestore(&gsnedf_lock, flags); 644 raw_spin_unlock_irqrestore(&gsnedf_lock, flags);
579 645
580 BUG_ON(!is_realtime(t)); 646 BUG_ON(!is_realtime(t));
@@ -608,51 +674,53 @@ static long gsnedf_admit_task(struct task_struct* tsk)
608 674
609#include <litmus/fdso.h> 675#include <litmus/fdso.h>
610 676
611/* called with IRQs off */ 677
612static void set_priority_inheritance(struct task_struct* t, struct task_struct* prio_inh) 678static void __set_priority_inheritance(struct task_struct* t, struct task_struct* prio_inh)
613{ 679{
614 int linked_on; 680 int linked_on;
615 int check_preempt = 0; 681 int check_preempt = 0;
616 682
617 raw_spin_lock(&gsnedf_lock); 683 if(prio_inh != NULL)
618 684 TRACE_TASK(t, "inherits priority from %s/%d\n", prio_inh->comm, prio_inh->pid);
619 TRACE_TASK(t, "inherits priority from %s/%d\n", prio_inh->comm, prio_inh->pid); 685 else
686 TRACE_TASK(t, "inherits priority from %p\n", prio_inh);
687
688 sched_trace_eff_prio_change(t, prio_inh);
689
620 tsk_rt(t)->inh_task = prio_inh; 690 tsk_rt(t)->inh_task = prio_inh;
621 691
622 linked_on = tsk_rt(t)->linked_on; 692 linked_on = tsk_rt(t)->linked_on;
623 693
624 /* If it is scheduled, then we need to reorder the CPU heap. */ 694 /* If it is scheduled, then we need to reorder the CPU heap. */
625 if (linked_on != NO_CPU) { 695 if (linked_on != NO_CPU) {
626 TRACE_TASK(t, "%s: linked on %d\n", 696 TRACE_TASK(t, "%s: linked on %d\n",
627 __FUNCTION__, linked_on); 697 __FUNCTION__, linked_on);
628 /* Holder is scheduled; need to re-order CPUs. 698 /* Holder is scheduled; need to re-order CPUs.
629 * We can't use heap_decrease() here since 699 * We can't use heap_decrease() here since
630 * the cpu_heap is ordered in reverse direction, so 700 * the cpu_heap is ordered in reverse direction, so
631 * it is actually an increase. */ 701 * it is actually an increase. */
632 bheap_delete(cpu_lower_prio, &gsnedf_cpu_heap, 702 bheap_delete(cpu_lower_prio, &gsnedf_cpu_heap,
633 gsnedf_cpus[linked_on]->hn); 703 gsnedf_cpus[linked_on]->hn);
634 bheap_insert(cpu_lower_prio, &gsnedf_cpu_heap, 704 bheap_insert(cpu_lower_prio, &gsnedf_cpu_heap,
635 gsnedf_cpus[linked_on]->hn); 705 gsnedf_cpus[linked_on]->hn);
636 } else { 706 } else {
637 /* holder may be queued: first stop queue changes */ 707 /* holder may be queued: first stop queue changes */
638 raw_spin_lock(&gsnedf.release_lock); 708 raw_spin_lock(&gsnedf.release_lock);
639 if (is_queued(t)) { 709 if (is_queued(t)) {
640 TRACE_TASK(t, "%s: is queued\n", 710 TRACE_TASK(t, "%s: is queued\n", __FUNCTION__);
641 __FUNCTION__); 711
642 /* We need to update the position of holder in some 712 /* We need to update the position of holder in some
643 * heap. Note that this could be a release heap if we 713 * heap. Note that this could be a release heap if we
644 * budget enforcement is used and this job overran. */ 714 * budget enforcement is used and this job overran. */
645 check_preempt = 715 check_preempt = !bheap_decrease(edf_ready_order, tsk_rt(t)->heap_node);
646 !bheap_decrease(edf_ready_order, 716
647 tsk_rt(t)->heap_node);
648 } else { 717 } else {
649 /* Nothing to do: if it is not queued and not linked 718 /* Nothing to do: if it is not queued and not linked
650 * then it is either sleeping or currently being moved 719 * then it is either sleeping or currently being moved
651 * by other code (e.g., a timer interrupt handler) that 720 * by other code (e.g., a timer interrupt handler) that
652 * will use the correct priority when enqueuing the 721 * will use the correct priority when enqueuing the
653 * task. */ 722 * task. */
654 TRACE_TASK(t, "%s: is NOT queued => Done.\n", 723 TRACE_TASK(t, "%s: is NOT queued => Done.\n", __FUNCTION__);
655 __FUNCTION__);
656 } 724 }
657 raw_spin_unlock(&gsnedf.release_lock); 725 raw_spin_unlock(&gsnedf.release_lock);
658 726
@@ -666,34 +734,148 @@ static void set_priority_inheritance(struct task_struct* t, struct task_struct*
666 /* heap_decrease() hit the top level of the heap: make 734 /* heap_decrease() hit the top level of the heap: make
667 * sure preemption checks get the right task, not the 735 * sure preemption checks get the right task, not the
668 * potentially stale cache. */ 736 * potentially stale cache. */
669 bheap_uncache_min(edf_ready_order, 737 bheap_uncache_min(edf_ready_order, &gsnedf.ready_queue);
670 &gsnedf.ready_queue);
671 check_for_preemptions(); 738 check_for_preemptions();
672 } 739 }
673 } 740 }
741}
674 742
743/* called with IRQs off */
744static void set_priority_inheritance(struct task_struct* t, struct task_struct* prio_inh)
745{
746 raw_spin_lock(&gsnedf_lock);
747
748 __set_priority_inheritance(t, prio_inh);
749
750#ifdef CONFIG_LITMUS_SOFTIRQD
751 if(tsk_rt(t)->cur_klitirqd != NULL)
752 {
753 TRACE_TASK(t, "%s/%d inherits a new priority!\n",
754 tsk_rt(t)->cur_klitirqd->comm, tsk_rt(t)->cur_klitirqd->pid);
755
756 __set_priority_inheritance(tsk_rt(t)->cur_klitirqd, prio_inh);
757 }
758#endif
759
675 raw_spin_unlock(&gsnedf_lock); 760 raw_spin_unlock(&gsnedf_lock);
676} 761}
677 762
763
764/* called with IRQs off */
765static void __clear_priority_inheritance(struct task_struct* t)
766{
767 TRACE_TASK(t, "priority restored\n");
768
769 if(tsk_rt(t)->scheduled_on != NO_CPU)
770 {
771 sched_trace_eff_prio_change(t, NULL);
772
773 tsk_rt(t)->inh_task = NULL;
774
775 /* Check if rescheduling is necessary. We can't use heap_decrease()
776 * since the priority was effectively lowered. */
777 unlink(t);
778 gsnedf_job_arrival(t);
779 }
780 else
781 {
782 __set_priority_inheritance(t, NULL);
783 }
784
785#ifdef CONFIG_LITMUS_SOFTIRQD
786 if(tsk_rt(t)->cur_klitirqd != NULL)
787 {
788 TRACE_TASK(t, "%s/%d inheritance set back to owner.\n",
789 tsk_rt(t)->cur_klitirqd->comm, tsk_rt(t)->cur_klitirqd->pid);
790
791 if(tsk_rt(tsk_rt(t)->cur_klitirqd)->scheduled_on != NO_CPU)
792 {
793 sched_trace_eff_prio_change(tsk_rt(t)->cur_klitirqd, t);
794
795 tsk_rt(tsk_rt(t)->cur_klitirqd)->inh_task = t;
796
797 /* Check if rescheduling is necessary. We can't use heap_decrease()
798 * since the priority was effectively lowered. */
799 unlink(tsk_rt(t)->cur_klitirqd);
800 gsnedf_job_arrival(tsk_rt(t)->cur_klitirqd);
801 }
802 else
803 {
804 __set_priority_inheritance(tsk_rt(t)->cur_klitirqd, t);
805 }
806 }
807#endif
808}
809
678/* called with IRQs off */ 810/* called with IRQs off */
679static void clear_priority_inheritance(struct task_struct* t) 811static void clear_priority_inheritance(struct task_struct* t)
680{ 812{
681 raw_spin_lock(&gsnedf_lock); 813 raw_spin_lock(&gsnedf_lock);
814 __clear_priority_inheritance(t);
815 raw_spin_unlock(&gsnedf_lock);
816}
682 817
683 /* A job only stops inheriting a priority when it releases a 818#ifdef CONFIG_LITMUS_SOFTIRQD
684 * resource. Thus we can make the following assumption.*/ 819/* called with IRQs off */
685 BUG_ON(tsk_rt(t)->scheduled_on == NO_CPU); 820static void set_priority_inheritance_klitirqd(struct task_struct* klitirqd,
686 821 struct task_struct* old_owner,
687 TRACE_TASK(t, "priority restored\n"); 822 struct task_struct* new_owner)
688 tsk_rt(t)->inh_task = NULL; 823{
824 BUG_ON(!(tsk_rt(klitirqd)->is_proxy_thread));
825
826 raw_spin_lock(&gsnedf_lock);
827
828 if(old_owner != new_owner)
829 {
830 if(old_owner)
831 {
832 // unreachable?
833 tsk_rt(old_owner)->cur_klitirqd = NULL;
834 }
835
836 TRACE_TASK(klitirqd, "giving ownership to %s/%d.\n",
837 new_owner->comm, new_owner->pid);
689 838
690 /* Check if rescheduling is necessary. We can't use heap_decrease() 839 tsk_rt(new_owner)->cur_klitirqd = klitirqd;
691 * since the priority was effectively lowered. */ 840 }
692 unlink(t); 841
693 gsnedf_job_arrival(t); 842 __set_priority_inheritance(klitirqd,
843 (tsk_rt(new_owner)->inh_task == NULL) ?
844 new_owner :
845 tsk_rt(new_owner)->inh_task);
846
847 raw_spin_unlock(&gsnedf_lock);
848}
694 849
850/* called with IRQs off */
851static void clear_priority_inheritance_klitirqd(struct task_struct* klitirqd,
852 struct task_struct* old_owner)
853{
854 BUG_ON(!(tsk_rt(klitirqd)->is_proxy_thread));
855
856 raw_spin_lock(&gsnedf_lock);
857
858 TRACE_TASK(klitirqd, "priority restored\n");
859
860 if(tsk_rt(klitirqd)->scheduled_on != NO_CPU)
861 {
862 tsk_rt(klitirqd)->inh_task = NULL;
863
864 /* Check if rescheduling is necessary. We can't use heap_decrease()
865 * since the priority was effectively lowered. */
866 unlink(klitirqd);
867 gsnedf_job_arrival(klitirqd);
868 }
869 else
870 {
871 __set_priority_inheritance(klitirqd, NULL);
872 }
873
874 tsk_rt(old_owner)->cur_klitirqd = NULL;
875
695 raw_spin_unlock(&gsnedf_lock); 876 raw_spin_unlock(&gsnedf_lock);
696} 877}
878#endif
697 879
698 880
699/* ******************** FMLP support ********************** */ 881/* ******************** FMLP support ********************** */
@@ -892,11 +1074,477 @@ static struct litmus_lock* gsnedf_new_fmlp(void)
892 return &sem->litmus_lock; 1074 return &sem->litmus_lock;
893} 1075}
894 1076
1077
1078
1079
1080
1081
1082
1083/* ******************** KFMLP support ********************** */
1084
1085/* struct for semaphore with priority inheritance */
1086struct kfmlp_queue
1087{
1088 wait_queue_head_t wait;
1089 struct task_struct* owner;
1090 struct task_struct* hp_waiter;
1091 int count; /* number of waiters + holder */
1092};
1093
1094struct kfmlp_semaphore
1095{
1096 struct litmus_lock litmus_lock;
1097
1098 spinlock_t lock;
1099
1100 int num_resources; /* aka k */
1101
1102 struct kfmlp_queue *queues; /* array */
1103 struct kfmlp_queue *shortest_queue; /* pointer to shortest queue */
1104};
1105
1106static inline struct kfmlp_semaphore* kfmlp_from_lock(struct litmus_lock* lock)
1107{
1108 return container_of(lock, struct kfmlp_semaphore, litmus_lock);
1109}
1110
1111static inline int kfmlp_get_idx(struct kfmlp_semaphore* sem,
1112 struct kfmlp_queue* queue)
1113{
1114 return (queue - &sem->queues[0]);
1115}
1116
1117static inline struct kfmlp_queue* kfmlp_get_queue(struct kfmlp_semaphore* sem,
1118 struct task_struct* holder)
1119{
1120 int i;
1121 for(i = 0; i < sem->num_resources; ++i)
1122 if(sem->queues[i].owner == holder)
1123 return(&sem->queues[i]);
1124 return(NULL);
1125}
1126
1127/* caller is responsible for locking */
1128static struct task_struct* kfmlp_find_hp_waiter(struct kfmlp_queue *kqueue,
1129 struct task_struct *skip)
1130{
1131 struct list_head *pos;
1132 struct task_struct *queued, *found = NULL;
1133
1134 list_for_each(pos, &kqueue->wait.task_list) {
1135 queued = (struct task_struct*) list_entry(pos, wait_queue_t,
1136 task_list)->private;
1137
1138 /* Compare task prios, find high prio task. */
1139 if (queued != skip && edf_higher_prio(queued, found))
1140 found = queued;
1141 }
1142 return found;
1143}
1144
1145static inline struct kfmlp_queue* kfmlp_find_shortest(
1146 struct kfmlp_semaphore* sem,
1147 struct kfmlp_queue* search_start)
1148{
1149 // we start our search at search_start instead of at the beginning of the
1150 // queue list to load-balance across all resources.
1151 struct kfmlp_queue* step = search_start;
1152 struct kfmlp_queue* shortest = sem->shortest_queue;
1153
1154 do
1155 {
1156 step = (step+1 != &sem->queues[sem->num_resources]) ?
1157 step+1 : &sem->queues[0];
1158 if(step->count < shortest->count)
1159 {
1160 shortest = step;
1161 if(step->count == 0)
1162 break; /* can't get any shorter */
1163 }
1164 }while(step != search_start);
1165
1166 return(shortest);
1167}
1168
1169static struct task_struct* kfmlp_remove_hp_waiter(struct kfmlp_semaphore* sem)
1170{
1171 /* must hold sem->lock */
1172
1173 struct kfmlp_queue *my_queue = NULL;
1174 struct task_struct *max_hp = NULL;
1175
1176
1177 struct list_head *pos;
1178 struct task_struct *queued;
1179 int i;
1180
1181 for(i = 0; i < sem->num_resources; ++i)
1182 {
1183 if( (sem->queues[i].count > 1) &&
1184 ((my_queue == NULL) ||
1185 (edf_higher_prio(sem->queues[i].hp_waiter, my_queue->hp_waiter))) )
1186 {
1187 my_queue = &sem->queues[i];
1188 }
1189 }
1190
1191 if(my_queue)
1192 {
1193 max_hp = my_queue->hp_waiter;
1194
1195 BUG_ON(!max_hp);
1196
1197 TRACE_CUR("queue %d: stealing %s/%d from queue %d\n",
1198 kfmlp_get_idx(sem, my_queue),
1199 max_hp->comm, max_hp->pid,
1200 kfmlp_get_idx(sem, my_queue));
1201
1202 my_queue->hp_waiter = kfmlp_find_hp_waiter(my_queue, max_hp);
1203
1204 /*
1205 if(my_queue->hp_waiter)
1206 TRACE_CUR("queue %d: new hp_waiter is %s/%d\n",
1207 kfmlp_get_idx(sem, my_queue),
1208 my_queue->hp_waiter->comm,
1209 my_queue->hp_waiter->pid);
1210 else
1211 TRACE_CUR("queue %d: new hp_waiter is %p\n",
1212 kfmlp_get_idx(sem, my_queue), NULL);
1213 */
1214
1215 raw_spin_lock(&gsnedf_lock);
1216
1217 /*
1218 if(my_queue->owner)
1219 TRACE_CUR("queue %d: owner is %s/%d\n",
1220 kfmlp_get_idx(sem, my_queue),
1221 my_queue->owner->comm,
1222 my_queue->owner->pid);
1223 else
1224 TRACE_CUR("queue %d: owner is %p\n",
1225 kfmlp_get_idx(sem, my_queue),
1226 NULL);
1227 */
1228
1229 if(tsk_rt(my_queue->owner)->inh_task == max_hp)
1230 {
1231 __clear_priority_inheritance(my_queue->owner);
1232 if(my_queue->hp_waiter != NULL)
1233 {
1234 __set_priority_inheritance(my_queue->owner, my_queue->hp_waiter);
1235 }
1236 }
1237 raw_spin_unlock(&gsnedf_lock);
1238
1239 list_for_each(pos, &my_queue->wait.task_list)
1240 {
1241 queued = (struct task_struct*) list_entry(pos, wait_queue_t,
1242 task_list)->private;
1243 /* Compare task prios, find high prio task. */
1244 if (queued == max_hp)
1245 {
1246 /*
1247 TRACE_CUR("queue %d: found entry in wait queue. REMOVING!\n",
1248 kfmlp_get_idx(sem, my_queue));
1249 */
1250 __remove_wait_queue(&my_queue->wait,
1251 list_entry(pos, wait_queue_t, task_list));
1252 break;
1253 }
1254 }
1255 --(my_queue->count);
1256 }
1257
1258 return(max_hp);
1259}
1260
1261int gsnedf_kfmlp_lock(struct litmus_lock* l)
1262{
1263 struct task_struct* t = current;
1264 struct kfmlp_semaphore *sem = kfmlp_from_lock(l);
1265 struct kfmlp_queue* my_queue;
1266 wait_queue_t wait;
1267 unsigned long flags;
1268
1269 if (!is_realtime(t))
1270 return -EPERM;
1271
1272 spin_lock_irqsave(&sem->lock, flags);
1273
1274 my_queue = sem->shortest_queue;
1275
1276 if (my_queue->owner) {
1277 /* resource is not free => must suspend and wait */
1278 TRACE_CUR("queue %d: Resource is not free => must suspend and wait.\n",
1279 kfmlp_get_idx(sem, my_queue));
1280
1281 init_waitqueue_entry(&wait, t);
1282
1283 /* FIXME: interruptible would be nice some day */
1284 set_task_state(t, TASK_UNINTERRUPTIBLE);
1285
1286 __add_wait_queue_tail_exclusive(&my_queue->wait, &wait);
1287
1288 /* check if we need to activate priority inheritance */
1289 if (edf_higher_prio(t, my_queue->hp_waiter))
1290 {
1291 my_queue->hp_waiter = t;
1292 if (edf_higher_prio(t, my_queue->owner))
1293 {
1294 set_priority_inheritance(my_queue->owner, my_queue->hp_waiter);
1295 }
1296 }
1297
1298 ++(my_queue->count);
1299 sem->shortest_queue = kfmlp_find_shortest(sem, my_queue);
1300
1301 /* release lock before sleeping */
1302 spin_unlock_irqrestore(&sem->lock, flags);
1303
1304 /* We depend on the FIFO order. Thus, we don't need to recheck
1305 * when we wake up; we are guaranteed to have the lock since
1306 * there is only one wake up per release (or steal).
1307 */
1308 schedule();
1309
1310
1311 if(my_queue->owner == t)
1312 {
1313 TRACE_CUR("queue %d: acquired through waiting\n",
1314 kfmlp_get_idx(sem, my_queue));
1315 }
1316 else
1317 {
1318 /* this case may happen if our wait entry was stolen
1319 between queues. record where we went. */
1320 my_queue = kfmlp_get_queue(sem, t);
1321
1322 BUG_ON(!my_queue);
1323 TRACE_CUR("queue %d: acquired through stealing\n",
1324 kfmlp_get_idx(sem, my_queue));
1325 }
1326 }
1327 else
1328 {
1329 TRACE_CUR("queue %d: acquired immediately\n",
1330 kfmlp_get_idx(sem, my_queue));
1331
1332 my_queue->owner = t;
1333
1334 ++(my_queue->count);
1335 sem->shortest_queue = kfmlp_find_shortest(sem, my_queue);
1336
1337 spin_unlock_irqrestore(&sem->lock, flags);
1338 }
1339
1340 return kfmlp_get_idx(sem, my_queue);
1341}
1342
1343int gsnedf_kfmlp_unlock(struct litmus_lock* l)
1344{
1345 struct task_struct *t = current, *next;
1346 struct kfmlp_semaphore *sem = kfmlp_from_lock(l);
1347 struct kfmlp_queue *my_queue;
1348 unsigned long flags;
1349 int err = 0;
1350
1351 spin_lock_irqsave(&sem->lock, flags);
1352
1353 my_queue = kfmlp_get_queue(sem, t);
1354
1355 if (!my_queue) {
1356 err = -EINVAL;
1357 goto out;
1358 }
1359
1360 /* check if there are jobs waiting for this resource */
1361 next = __waitqueue_remove_first(&my_queue->wait);
1362 if (next) {
1363 /*
1364 TRACE_CUR("queue %d: ASSIGNING %s/%d as owner - next\n",
1365 kfmlp_get_idx(sem, my_queue),
1366 next->comm, next->pid);
1367 */
1368 /* next becomes the resouce holder */
1369 my_queue->owner = next;
1370
1371 --(my_queue->count);
1372 if(my_queue->count < sem->shortest_queue->count)
1373 {
1374 sem->shortest_queue = my_queue;
1375 }
1376
1377 TRACE_CUR("queue %d: lock ownership passed to %s/%d\n",
1378 kfmlp_get_idx(sem, my_queue), next->comm, next->pid);
1379
1380 /* determine new hp_waiter if necessary */
1381 if (next == my_queue->hp_waiter) {
1382 TRACE_TASK(next, "was highest-prio waiter\n");
1383 /* next has the highest priority --- it doesn't need to
1384 * inherit. However, we need to make sure that the
1385 * next-highest priority in the queue is reflected in
1386 * hp_waiter. */
1387 my_queue->hp_waiter = kfmlp_find_hp_waiter(my_queue, next);
1388 if (my_queue->hp_waiter)
1389 TRACE_TASK(my_queue->hp_waiter, "queue %d: is new highest-prio waiter\n", kfmlp_get_idx(sem, my_queue));
1390 else
1391 TRACE("queue %d: no further waiters\n", kfmlp_get_idx(sem, my_queue));
1392 } else {
1393 /* Well, if next is not the highest-priority waiter,
1394 * then it ought to inherit the highest-priority
1395 * waiter's priority. */
1396 set_priority_inheritance(next, my_queue->hp_waiter);
1397 }
1398
1399 /* wake up next */
1400 wake_up_process(next);
1401 }
1402 else
1403 {
1404 TRACE_CUR("queue %d: looking to steal someone...\n", kfmlp_get_idx(sem, my_queue));
1405
1406 next = kfmlp_remove_hp_waiter(sem); /* returns NULL if nothing to steal */
1407
1408 /*
1409 if(next)
1410 TRACE_CUR("queue %d: ASSIGNING %s/%d as owner - steal\n",
1411 kfmlp_get_idx(sem, my_queue),
1412 next->comm, next->pid);
1413 */
1414
1415 my_queue->owner = next;
1416
1417 if(next)
1418 {
1419 TRACE_CUR("queue %d: lock ownership passed to %s/%d (which was stolen)\n",
1420 kfmlp_get_idx(sem, my_queue),
1421 next->comm, next->pid);
1422
1423 /* wake up next */
1424 wake_up_process(next);
1425 }
1426 else
1427 {
1428 TRACE_CUR("queue %d: no one to steal.\n", kfmlp_get_idx(sem, my_queue));
1429
1430 --(my_queue->count);
1431 if(my_queue->count < sem->shortest_queue->count)
1432 {
1433 sem->shortest_queue = my_queue;
1434 }
1435 }
1436 }
1437
1438 /* we lose the benefit of priority inheritance (if any) */
1439 if (tsk_rt(t)->inh_task)
1440 clear_priority_inheritance(t);
1441
1442out:
1443 spin_unlock_irqrestore(&sem->lock, flags);
1444
1445 return err;
1446}
1447
1448int gsnedf_kfmlp_close(struct litmus_lock* l)
1449{
1450 struct task_struct *t = current;
1451 struct kfmlp_semaphore *sem = kfmlp_from_lock(l);
1452 struct kfmlp_queue *my_queue;
1453 unsigned long flags;
1454
1455 int owner;
1456
1457 spin_lock_irqsave(&sem->lock, flags);
1458
1459 my_queue = kfmlp_get_queue(sem, t);
1460 owner = (my_queue) ? (my_queue->owner == t) : 0;
1461
1462 spin_unlock_irqrestore(&sem->lock, flags);
1463
1464 if (owner)
1465 gsnedf_kfmlp_unlock(l);
1466
1467 return 0;
1468}
1469
1470void gsnedf_kfmlp_free(struct litmus_lock* l)
1471{
1472 struct kfmlp_semaphore *sem = kfmlp_from_lock(l);
1473 kfree(sem->queues);
1474 kfree(sem);
1475}
1476
1477static struct litmus_lock_ops gsnedf_kfmlp_lock_ops = {
1478 .close = gsnedf_kfmlp_close,
1479 .lock = gsnedf_kfmlp_lock,
1480 .unlock = gsnedf_kfmlp_unlock,
1481 .deallocate = gsnedf_kfmlp_free,
1482};
1483
1484static struct litmus_lock* gsnedf_new_kfmlp(void* __user arg, int* ret_code)
1485{
1486 struct kfmlp_semaphore* sem;
1487 int num_resources = 0;
1488 int i;
1489
1490 if(!access_ok(VERIFY_READ, arg, sizeof(num_resources)))
1491 {
1492 *ret_code = -EINVAL;
1493 return(NULL);
1494 }
1495 if(__copy_from_user(&num_resources, arg, sizeof(num_resources)))
1496 {
1497 *ret_code = -EINVAL;
1498 return(NULL);
1499 }
1500 if(num_resources < 1)
1501 {
1502 *ret_code = -EINVAL;
1503 return(NULL);
1504 }
1505
1506 sem = kmalloc(sizeof(*sem), GFP_KERNEL);
1507 if(!sem)
1508 {
1509 *ret_code = -ENOMEM;
1510 return NULL;
1511 }
1512
1513 sem->queues = kmalloc(sizeof(struct kfmlp_queue)*num_resources, GFP_KERNEL);
1514 if(!sem->queues)
1515 {
1516 kfree(sem);
1517 *ret_code = -ENOMEM;
1518 return NULL;
1519 }
1520
1521 sem->litmus_lock.ops = &gsnedf_kfmlp_lock_ops;
1522 spin_lock_init(&sem->lock);
1523 sem->num_resources = num_resources;
1524
1525 for(i = 0; i < num_resources; ++i)
1526 {
1527 sem->queues[i].owner = NULL;
1528 sem->queues[i].hp_waiter = NULL;
1529 init_waitqueue_head(&sem->queues[i].wait);
1530 sem->queues[i].count = 0;
1531 }
1532
1533 sem->shortest_queue = &sem->queues[0];
1534
1535 *ret_code = 0;
1536 return &sem->litmus_lock;
1537}
1538
1539
1540
1541
1542
895/* **** lock constructor **** */ 1543/* **** lock constructor **** */
896 1544
897 1545
898static long gsnedf_allocate_lock(struct litmus_lock **lock, int type, 1546static long gsnedf_allocate_lock(struct litmus_lock **lock, int type,
899 void* __user unused) 1547 void* __user arg)
900{ 1548{
901 int err = -ENXIO; 1549 int err = -ENXIO;
902 1550
@@ -911,7 +1559,10 @@ static long gsnedf_allocate_lock(struct litmus_lock **lock, int type,
911 else 1559 else
912 err = -ENOMEM; 1560 err = -ENOMEM;
913 break; 1561 break;
914 1562
1563 case KFMLP_SEM:
1564 *lock = gsnedf_new_kfmlp(arg, &err);
1565 break;
915 }; 1566 };
916 1567
917 return err; 1568 return err;
@@ -919,7 +1570,6 @@ static long gsnedf_allocate_lock(struct litmus_lock **lock, int type,
919 1570
920#endif 1571#endif
921 1572
922
923static long gsnedf_activate_plugin(void) 1573static long gsnedf_activate_plugin(void)
924{ 1574{
925 int cpu; 1575 int cpu;
@@ -946,6 +1596,15 @@ static long gsnedf_activate_plugin(void)
946 } 1596 }
947#endif 1597#endif
948 } 1598 }
1599
1600#ifdef CONFIG_LITMUS_SOFTIRQD
1601 spawn_klitirqd(NULL);
1602#endif
1603
1604#ifdef CONFIG_LITMUS_NVIDIA
1605 init_nvidia_info();
1606#endif
1607
949 return 0; 1608 return 0;
950} 1609}
951 1610
@@ -963,8 +1622,15 @@ static struct sched_plugin gsn_edf_plugin __cacheline_aligned_in_smp = {
963 .admit_task = gsnedf_admit_task, 1622 .admit_task = gsnedf_admit_task,
964 .activate_plugin = gsnedf_activate_plugin, 1623 .activate_plugin = gsnedf_activate_plugin,
965#ifdef CONFIG_LITMUS_LOCKING 1624#ifdef CONFIG_LITMUS_LOCKING
966 .allocate_lock = gsnedf_allocate_lock, 1625 .allocate_lock = gsnedf_allocate_lock,
1626 .set_prio_inh = set_priority_inheritance,
1627 .clear_prio_inh = clear_priority_inheritance,
1628#endif
1629#ifdef CONFIG_LITMUS_SOFTIRQD
1630 .set_prio_inh_klitirqd = set_priority_inheritance_klitirqd,
1631 .clear_prio_inh_klitirqd = clear_priority_inheritance_klitirqd,
967#endif 1632#endif
1633
968}; 1634};
969 1635
970 1636
diff --git a/litmus/sched_litmus.c b/litmus/sched_litmus.c
index e6952896dc4b..1bca2e1a33cd 100644
--- a/litmus/sched_litmus.c
+++ b/litmus/sched_litmus.c
@@ -103,7 +103,9 @@ litmus_schedule(struct rq *rq, struct task_struct *prev)
103 } 103 }
104#ifdef __ARCH_WANT_UNLOCKED_CTXSW 104#ifdef __ARCH_WANT_UNLOCKED_CTXSW
105 if (next->oncpu) 105 if (next->oncpu)
106 {
106 TRACE_TASK(next, "waiting for !oncpu"); 107 TRACE_TASK(next, "waiting for !oncpu");
108 }
107 while (next->oncpu) { 109 while (next->oncpu) {
108 cpu_relax(); 110 cpu_relax();
109 mb(); 111 mb();
diff --git a/litmus/sched_plugin.c b/litmus/sched_plugin.c
index d54886df1f57..8802670a4b0b 100644
--- a/litmus/sched_plugin.c
+++ b/litmus/sched_plugin.c
@@ -129,6 +129,27 @@ static long litmus_dummy_allocate_lock(struct litmus_lock **lock, int type,
129 return -ENXIO; 129 return -ENXIO;
130} 130}
131 131
132static void litmus_dummy_set_prio_inh(struct task_struct* a, struct task_struct* b)
133{
134}
135
136static void litmus_dummy_clear_prio_inh(struct task_struct* t)
137{
138}
139
140#endif
141
142#ifdef CONFIG_LITMUS_SOFTIRQD
143static void litmus_dummy_set_prio_inh_klitirq(struct task_struct* klitirqd,
144 struct task_struct* old_owner,
145 struct task_struct* new_owner)
146{
147}
148
149static void litmus_dummy_clear_prio_inh_klitirqd(struct task_struct* klitirqd,
150 struct task_struct* old_owner)
151{
152}
132#endif 153#endif
133 154
134 155
@@ -149,6 +170,12 @@ struct sched_plugin linux_sched_plugin = {
149 .deactivate_plugin = litmus_dummy_deactivate_plugin, 170 .deactivate_plugin = litmus_dummy_deactivate_plugin,
150#ifdef CONFIG_LITMUS_LOCKING 171#ifdef CONFIG_LITMUS_LOCKING
151 .allocate_lock = litmus_dummy_allocate_lock, 172 .allocate_lock = litmus_dummy_allocate_lock,
173 .set_prio_inh = litmus_dummy_set_prio_inh,
174 .clear_prio_inh = litmus_dummy_clear_prio_inh,
175#endif
176#ifdef CONFIG_LITMUS_SOFTIRQD
177 .set_prio_inh_klitirqd = litmus_dummy_set_prio_inh_klitirq,
178 .clear_prio_inh_klitirqd = litmus_dummy_clear_prio_inh_klitirqd,
152#endif 179#endif
153 .admit_task = litmus_dummy_admit_task 180 .admit_task = litmus_dummy_admit_task
154}; 181};
@@ -187,6 +214,8 @@ int register_sched_plugin(struct sched_plugin* plugin)
187 CHECK(deactivate_plugin); 214 CHECK(deactivate_plugin);
188#ifdef CONFIG_LITMUS_LOCKING 215#ifdef CONFIG_LITMUS_LOCKING
189 CHECK(allocate_lock); 216 CHECK(allocate_lock);
217 CHECK(set_prio_inh);
218 CHECK(clear_prio_inh);
190#endif 219#endif
191 CHECK(admit_task); 220 CHECK(admit_task);
192 221
diff --git a/litmus/sched_task_trace.c b/litmus/sched_task_trace.c
index 5ef8d09ab41f..7aeb99b668d3 100644
--- a/litmus/sched_task_trace.c
+++ b/litmus/sched_task_trace.c
@@ -7,6 +7,7 @@
7#include <linux/module.h> 7#include <linux/module.h>
8#include <linux/sched.h> 8#include <linux/sched.h>
9#include <linux/percpu.h> 9#include <linux/percpu.h>
10#include <linux/hardirq.h>
10 11
11#include <litmus/ftdev.h> 12#include <litmus/ftdev.h>
12#include <litmus/litmus.h> 13#include <litmus/litmus.h>
@@ -16,13 +17,13 @@
16#include <litmus/ftdev.h> 17#include <litmus/ftdev.h>
17 18
18 19
19#define NO_EVENTS (1 << CONFIG_SCHED_TASK_TRACE_SHIFT) 20#define NUM_EVENTS (1 << (CONFIG_SCHED_TASK_TRACE_SHIFT+11))
20 21
21#define now() litmus_clock() 22#define now() litmus_clock()
22 23
23struct local_buffer { 24struct local_buffer {
24 struct st_event_record record[NO_EVENTS]; 25 struct st_event_record record[NUM_EVENTS];
25 char flag[NO_EVENTS]; 26 char flag[NUM_EVENTS];
26 struct ft_buffer ftbuf; 27 struct ft_buffer ftbuf;
27}; 28};
28 29
@@ -41,7 +42,7 @@ static int __init init_sched_task_trace(void)
41 int i, ok = 0, err; 42 int i, ok = 0, err;
42 printk("Allocated %u sched_trace_xxx() events per CPU " 43 printk("Allocated %u sched_trace_xxx() events per CPU "
43 "(buffer size: %d bytes)\n", 44 "(buffer size: %d bytes)\n",
44 NO_EVENTS, (int) sizeof(struct local_buffer)); 45 NUM_EVENTS, (int) sizeof(struct local_buffer));
45 46
46 err = ftdev_init(&st_dev, THIS_MODULE, 47 err = ftdev_init(&st_dev, THIS_MODULE,
47 num_online_cpus(), "sched_trace"); 48 num_online_cpus(), "sched_trace");
@@ -50,7 +51,7 @@ static int __init init_sched_task_trace(void)
50 51
51 for (i = 0; i < st_dev.minor_cnt; i++) { 52 for (i = 0; i < st_dev.minor_cnt; i++) {
52 buf = &per_cpu(st_event_buffer, i); 53 buf = &per_cpu(st_event_buffer, i);
53 ok += init_ft_buffer(&buf->ftbuf, NO_EVENTS, 54 ok += init_ft_buffer(&buf->ftbuf, NUM_EVENTS,
54 sizeof(struct st_event_record), 55 sizeof(struct st_event_record),
55 buf->flag, 56 buf->flag,
56 buf->record); 57 buf->record);
@@ -154,7 +155,8 @@ feather_callback void do_sched_trace_task_switch_to(unsigned long id,
154{ 155{
155 struct task_struct *t = (struct task_struct*) _task; 156 struct task_struct *t = (struct task_struct*) _task;
156 struct st_event_record* rec; 157 struct st_event_record* rec;
157 if (is_realtime(t)) { 158 //if (is_realtime(t)) /* comment out to trace EVERYTHING */
159 {
158 rec = get_record(ST_SWITCH_TO, t); 160 rec = get_record(ST_SWITCH_TO, t);
159 if (rec) { 161 if (rec) {
160 rec->data.switch_to.when = now(); 162 rec->data.switch_to.when = now();
@@ -169,7 +171,8 @@ feather_callback void do_sched_trace_task_switch_away(unsigned long id,
169{ 171{
170 struct task_struct *t = (struct task_struct*) _task; 172 struct task_struct *t = (struct task_struct*) _task;
171 struct st_event_record* rec; 173 struct st_event_record* rec;
172 if (is_realtime(t)) { 174 //if (is_realtime(t)) /* comment out to trace EVERYTHING */
175 {
173 rec = get_record(ST_SWITCH_AWAY, t); 176 rec = get_record(ST_SWITCH_AWAY, t);
174 if (rec) { 177 if (rec) {
175 rec->data.switch_away.when = now(); 178 rec->data.switch_away.when = now();
@@ -188,6 +191,7 @@ feather_callback void do_sched_trace_task_completion(unsigned long id,
188 if (rec) { 191 if (rec) {
189 rec->data.completion.when = now(); 192 rec->data.completion.when = now();
190 rec->data.completion.forced = forced; 193 rec->data.completion.forced = forced;
194 rec->data.completion.nv_int_count = (u16)atomic_read(&tsk_rt(t)->nv_int_count);
191 put_record(rec); 195 put_record(rec);
192 } 196 }
193} 197}
@@ -239,3 +243,201 @@ feather_callback void do_sched_trace_action(unsigned long id,
239 put_record(rec); 243 put_record(rec);
240 } 244 }
241} 245}
246
247
248feather_callback void do_sched_trace_tasklet_release(unsigned long id,
249 unsigned long _owner)
250{
251 struct task_struct *t = (struct task_struct*) _owner;
252 struct st_event_record *rec = get_record(ST_TASKLET_RELEASE, t);
253
254 if (rec) {
255 rec->data.tasklet_release.when = now();
256 put_record(rec);
257 }
258}
259
260
261feather_callback void do_sched_trace_tasklet_begin(unsigned long id,
262 unsigned long _owner)
263{
264 struct task_struct *t = (struct task_struct*) _owner;
265 struct st_event_record *rec = get_record(ST_TASKLET_BEGIN, t);
266
267 if (rec) {
268 rec->data.tasklet_begin.when = now();
269
270 if(!in_interrupt())
271 rec->data.tasklet_begin.exe_pid = current->pid;
272 else
273 rec->data.tasklet_begin.exe_pid = 0;
274
275 put_record(rec);
276 }
277}
278EXPORT_SYMBOL(do_sched_trace_tasklet_begin);
279
280
281feather_callback void do_sched_trace_tasklet_end(unsigned long id,
282 unsigned long _owner,
283 unsigned long _flushed)
284{
285 struct task_struct *t = (struct task_struct*) _owner;
286 struct st_event_record *rec = get_record(ST_TASKLET_END, t);
287
288 if (rec) {
289 rec->data.tasklet_end.when = now();
290 rec->data.tasklet_end.flushed = _flushed;
291
292 if(!in_interrupt())
293 rec->data.tasklet_end.exe_pid = current->pid;
294 else
295 rec->data.tasklet_end.exe_pid = 0;
296
297 put_record(rec);
298 }
299}
300EXPORT_SYMBOL(do_sched_trace_tasklet_end);
301
302
303feather_callback void do_sched_trace_work_release(unsigned long id,
304 unsigned long _owner)
305{
306 struct task_struct *t = (struct task_struct*) _owner;
307 struct st_event_record *rec = get_record(ST_WORK_RELEASE, t);
308
309 if (rec) {
310 rec->data.work_release.when = now();
311 put_record(rec);
312 }
313}
314
315
316feather_callback void do_sched_trace_work_begin(unsigned long id,
317 unsigned long _owner,
318 unsigned long _exe)
319{
320 struct task_struct *t = (struct task_struct*) _owner;
321 struct st_event_record *rec = get_record(ST_WORK_BEGIN, t);
322
323 if (rec) {
324 struct task_struct *exe = (struct task_struct*) _exe;
325 rec->data.work_begin.exe_pid = exe->pid;
326 rec->data.work_begin.when = now();
327 put_record(rec);
328 }
329}
330EXPORT_SYMBOL(do_sched_trace_work_begin);
331
332
333feather_callback void do_sched_trace_work_end(unsigned long id,
334 unsigned long _owner,
335 unsigned long _exe,
336 unsigned long _flushed)
337{
338 struct task_struct *t = (struct task_struct*) _owner;
339 struct st_event_record *rec = get_record(ST_WORK_END, t);
340
341 if (rec) {
342 struct task_struct *exe = (struct task_struct*) _exe;
343 rec->data.work_end.exe_pid = exe->pid;
344 rec->data.work_end.flushed = _flushed;
345 rec->data.work_end.when = now();
346 put_record(rec);
347 }
348}
349EXPORT_SYMBOL(do_sched_trace_work_end);
350
351
352feather_callback void do_sched_trace_eff_prio_change(unsigned long id,
353 unsigned long _task,
354 unsigned long _inh)
355{
356 struct task_struct *t = (struct task_struct*) _task;
357 struct st_event_record *rec = get_record(ST_EFF_PRIO_CHANGE, t);
358
359 if (rec) {
360 struct task_struct *inh = (struct task_struct*) _inh;
361 rec->data.effective_priority_change.when = now();
362 rec->data.effective_priority_change.inh_pid = (inh != NULL) ?
363 inh->pid :
364 0xffff;
365
366 put_record(rec);
367 }
368}
369
370
371/* pray for no nesting of nv interrupts on same CPU... */
372struct tracing_interrupt_map
373{
374 int active;
375 int count;
376 unsigned long data[128]; // assume nesting less than 128...
377};
378DEFINE_PER_CPU(struct tracing_interrupt_map, active_interrupt_tracing);
379
380feather_callback void do_sched_trace_nv_interrupt_begin(unsigned long id,
381 unsigned long _device)
382{
383 struct st_event_record *rec;
384
385 {
386 struct tracing_interrupt_map* int_map = &per_cpu(active_interrupt_tracing, smp_processor_id());
387 if(int_map->active == 0xcafebabe)
388 {
389 int_map->count++;
390 }
391 else
392 {
393 int_map->active = 0xcafebabe;
394 int_map->count = 1;
395 }
396 int_map->data[int_map->count-1] = _device;
397 }
398
399 rec = get_record(ST_NV_INTERRUPT_BEGIN, NULL);
400 if(rec) {
401 u32 device = _device;
402 rec->data.nv_interrupt_begin.when = now();
403 rec->data.nv_interrupt_begin.device = device;
404 put_record(rec);
405 }
406}
407EXPORT_SYMBOL(do_sched_trace_nv_interrupt_begin);
408
409/*
410int is_interrupt_tracing_active(void)
411{
412 struct tracing_interrupt_map* int_map = &per_cpu(active_interrupt_tracing, smp_processor_id());
413 if(int_map->active == 0xcafebabe)
414 return 1;
415 return 0;
416}
417*/
418
419feather_callback void do_sched_trace_nv_interrupt_end(unsigned long id, unsigned long unused)
420{
421 struct tracing_interrupt_map* int_map = &per_cpu(active_interrupt_tracing, smp_processor_id());
422 if(int_map->active == 0xcafebabe)
423 {
424 struct st_event_record *rec = get_record(ST_NV_INTERRUPT_END, NULL);
425
426 int_map->count--;
427 if(int_map->count == 0)
428 int_map->active = 0;
429
430 if(rec) {
431 rec->data.nv_interrupt_end.when = now();
432 rec->data.nv_interrupt_end.device = int_map->data[int_map->count];
433 put_record(rec);
434 }
435 }
436}
437EXPORT_SYMBOL(do_sched_trace_nv_interrupt_end);
438
439
440
441
442
443
diff --git a/litmus/sched_trace_external.c b/litmus/sched_trace_external.c
new file mode 100644
index 000000000000..d7d7d8bae298
--- /dev/null
+++ b/litmus/sched_trace_external.c
@@ -0,0 +1,45 @@
1#include <linux/module.h>
2
3#include <litmus/sched_trace.h>
4#include <litmus/litmus.h>
5
6void __sched_trace_tasklet_begin_external(struct task_struct* t)
7{
8 sched_trace_tasklet_begin(t);
9}
10EXPORT_SYMBOL(__sched_trace_tasklet_begin_external);
11
12void __sched_trace_tasklet_end_external(struct task_struct* t, unsigned long flushed)
13{
14 sched_trace_tasklet_end(t, flushed);
15}
16EXPORT_SYMBOL(__sched_trace_tasklet_end_external);
17
18
19
20void __sched_trace_work_begin_external(struct task_struct* t, struct task_struct* e)
21{
22 sched_trace_work_begin(t, e);
23}
24EXPORT_SYMBOL(__sched_trace_work_begin_external);
25
26void __sched_trace_work_end_external(struct task_struct* t, struct task_struct* e, unsigned long f)
27{
28 sched_trace_work_end(t, e, f);
29}
30EXPORT_SYMBOL(__sched_trace_work_end_external);
31
32
33
34void __sched_trace_nv_interrupt_begin_external(u32 device)
35{
36 unsigned long _device = device;
37 sched_trace_nv_interrupt_begin(_device);
38}
39EXPORT_SYMBOL(__sched_trace_nv_interrupt_begin_external);
40
41void __sched_trace_nv_interrupt_end_external(void)
42{
43 sched_trace_nv_interrupt_end();
44}
45EXPORT_SYMBOL(__sched_trace_nv_interrupt_end_external);