diff options
Diffstat (limited to 'kernel')
-rw-r--r-- | kernel/fork.c | 9 | ||||
-rw-r--r-- | kernel/hrtimer.c | 40 | ||||
-rw-r--r-- | kernel/itimer.c | 169 | ||||
-rw-r--r-- | kernel/posix-cpu-timers.c | 155 | ||||
-rw-r--r-- | kernel/sched.c | 9 | ||||
-rw-r--r-- | kernel/timer.c | 32 |
6 files changed, 245 insertions, 169 deletions
diff --git a/kernel/fork.c b/kernel/fork.c index 8f45b0ebdda7..51ad0b0b7266 100644 --- a/kernel/fork.c +++ b/kernel/fork.c | |||
@@ -63,6 +63,7 @@ | |||
63 | #include <linux/fs_struct.h> | 63 | #include <linux/fs_struct.h> |
64 | #include <linux/magic.h> | 64 | #include <linux/magic.h> |
65 | #include <linux/perf_event.h> | 65 | #include <linux/perf_event.h> |
66 | #include <linux/posix-timers.h> | ||
66 | 67 | ||
67 | #include <asm/pgtable.h> | 68 | #include <asm/pgtable.h> |
68 | #include <asm/pgalloc.h> | 69 | #include <asm/pgalloc.h> |
@@ -805,10 +806,10 @@ static void posix_cpu_timers_init_group(struct signal_struct *sig) | |||
805 | thread_group_cputime_init(sig); | 806 | thread_group_cputime_init(sig); |
806 | 807 | ||
807 | /* Expiration times and increments. */ | 808 | /* Expiration times and increments. */ |
808 | sig->it_virt_expires = cputime_zero; | 809 | sig->it[CPUCLOCK_PROF].expires = cputime_zero; |
809 | sig->it_virt_incr = cputime_zero; | 810 | sig->it[CPUCLOCK_PROF].incr = cputime_zero; |
810 | sig->it_prof_expires = cputime_zero; | 811 | sig->it[CPUCLOCK_VIRT].expires = cputime_zero; |
811 | sig->it_prof_incr = cputime_zero; | 812 | sig->it[CPUCLOCK_VIRT].incr = cputime_zero; |
812 | 813 | ||
813 | /* Cached expiration times. */ | 814 | /* Cached expiration times. */ |
814 | sig->cputime_expires.prof_exp = cputime_zero; | 815 | sig->cputime_expires.prof_exp = cputime_zero; |
diff --git a/kernel/hrtimer.c b/kernel/hrtimer.c index c03f221fee44..e5d98ce50f89 100644 --- a/kernel/hrtimer.c +++ b/kernel/hrtimer.c | |||
@@ -48,6 +48,8 @@ | |||
48 | 48 | ||
49 | #include <asm/uaccess.h> | 49 | #include <asm/uaccess.h> |
50 | 50 | ||
51 | #include <trace/events/timer.h> | ||
52 | |||
51 | /* | 53 | /* |
52 | * The timer bases: | 54 | * The timer bases: |
53 | * | 55 | * |
@@ -442,6 +444,26 @@ static inline void debug_hrtimer_activate(struct hrtimer *timer) { } | |||
442 | static inline void debug_hrtimer_deactivate(struct hrtimer *timer) { } | 444 | static inline void debug_hrtimer_deactivate(struct hrtimer *timer) { } |
443 | #endif | 445 | #endif |
444 | 446 | ||
447 | static inline void | ||
448 | debug_init(struct hrtimer *timer, clockid_t clockid, | ||
449 | enum hrtimer_mode mode) | ||
450 | { | ||
451 | debug_hrtimer_init(timer); | ||
452 | trace_hrtimer_init(timer, clockid, mode); | ||
453 | } | ||
454 | |||
455 | static inline void debug_activate(struct hrtimer *timer) | ||
456 | { | ||
457 | debug_hrtimer_activate(timer); | ||
458 | trace_hrtimer_start(timer); | ||
459 | } | ||
460 | |||
461 | static inline void debug_deactivate(struct hrtimer *timer) | ||
462 | { | ||
463 | debug_hrtimer_deactivate(timer); | ||
464 | trace_hrtimer_cancel(timer); | ||
465 | } | ||
466 | |||
445 | /* High resolution timer related functions */ | 467 | /* High resolution timer related functions */ |
446 | #ifdef CONFIG_HIGH_RES_TIMERS | 468 | #ifdef CONFIG_HIGH_RES_TIMERS |
447 | 469 | ||
@@ -798,7 +820,7 @@ static int enqueue_hrtimer(struct hrtimer *timer, | |||
798 | struct hrtimer *entry; | 820 | struct hrtimer *entry; |
799 | int leftmost = 1; | 821 | int leftmost = 1; |
800 | 822 | ||
801 | debug_hrtimer_activate(timer); | 823 | debug_activate(timer); |
802 | 824 | ||
803 | /* | 825 | /* |
804 | * Find the right place in the rbtree: | 826 | * Find the right place in the rbtree: |
@@ -884,7 +906,7 @@ remove_hrtimer(struct hrtimer *timer, struct hrtimer_clock_base *base) | |||
884 | * reprogramming happens in the interrupt handler. This is a | 906 | * reprogramming happens in the interrupt handler. This is a |
885 | * rare case and less expensive than a smp call. | 907 | * rare case and less expensive than a smp call. |
886 | */ | 908 | */ |
887 | debug_hrtimer_deactivate(timer); | 909 | debug_deactivate(timer); |
888 | timer_stats_hrtimer_clear_start_info(timer); | 910 | timer_stats_hrtimer_clear_start_info(timer); |
889 | reprogram = base->cpu_base == &__get_cpu_var(hrtimer_bases); | 911 | reprogram = base->cpu_base == &__get_cpu_var(hrtimer_bases); |
890 | __remove_hrtimer(timer, base, HRTIMER_STATE_INACTIVE, | 912 | __remove_hrtimer(timer, base, HRTIMER_STATE_INACTIVE, |
@@ -1117,7 +1139,7 @@ static void __hrtimer_init(struct hrtimer *timer, clockid_t clock_id, | |||
1117 | void hrtimer_init(struct hrtimer *timer, clockid_t clock_id, | 1139 | void hrtimer_init(struct hrtimer *timer, clockid_t clock_id, |
1118 | enum hrtimer_mode mode) | 1140 | enum hrtimer_mode mode) |
1119 | { | 1141 | { |
1120 | debug_hrtimer_init(timer); | 1142 | debug_init(timer, clock_id, mode); |
1121 | __hrtimer_init(timer, clock_id, mode); | 1143 | __hrtimer_init(timer, clock_id, mode); |
1122 | } | 1144 | } |
1123 | EXPORT_SYMBOL_GPL(hrtimer_init); | 1145 | EXPORT_SYMBOL_GPL(hrtimer_init); |
@@ -1141,7 +1163,7 @@ int hrtimer_get_res(const clockid_t which_clock, struct timespec *tp) | |||
1141 | } | 1163 | } |
1142 | EXPORT_SYMBOL_GPL(hrtimer_get_res); | 1164 | EXPORT_SYMBOL_GPL(hrtimer_get_res); |
1143 | 1165 | ||
1144 | static void __run_hrtimer(struct hrtimer *timer) | 1166 | static void __run_hrtimer(struct hrtimer *timer, ktime_t *now) |
1145 | { | 1167 | { |
1146 | struct hrtimer_clock_base *base = timer->base; | 1168 | struct hrtimer_clock_base *base = timer->base; |
1147 | struct hrtimer_cpu_base *cpu_base = base->cpu_base; | 1169 | struct hrtimer_cpu_base *cpu_base = base->cpu_base; |
@@ -1150,7 +1172,7 @@ static void __run_hrtimer(struct hrtimer *timer) | |||
1150 | 1172 | ||
1151 | WARN_ON(!irqs_disabled()); | 1173 | WARN_ON(!irqs_disabled()); |
1152 | 1174 | ||
1153 | debug_hrtimer_deactivate(timer); | 1175 | debug_deactivate(timer); |
1154 | __remove_hrtimer(timer, base, HRTIMER_STATE_CALLBACK, 0); | 1176 | __remove_hrtimer(timer, base, HRTIMER_STATE_CALLBACK, 0); |
1155 | timer_stats_account_hrtimer(timer); | 1177 | timer_stats_account_hrtimer(timer); |
1156 | fn = timer->function; | 1178 | fn = timer->function; |
@@ -1161,7 +1183,9 @@ static void __run_hrtimer(struct hrtimer *timer) | |||
1161 | * the timer base. | 1183 | * the timer base. |
1162 | */ | 1184 | */ |
1163 | spin_unlock(&cpu_base->lock); | 1185 | spin_unlock(&cpu_base->lock); |
1186 | trace_hrtimer_expire_entry(timer, now); | ||
1164 | restart = fn(timer); | 1187 | restart = fn(timer); |
1188 | trace_hrtimer_expire_exit(timer); | ||
1165 | spin_lock(&cpu_base->lock); | 1189 | spin_lock(&cpu_base->lock); |
1166 | 1190 | ||
1167 | /* | 1191 | /* |
@@ -1272,7 +1296,7 @@ void hrtimer_interrupt(struct clock_event_device *dev) | |||
1272 | break; | 1296 | break; |
1273 | } | 1297 | } |
1274 | 1298 | ||
1275 | __run_hrtimer(timer); | 1299 | __run_hrtimer(timer, &basenow); |
1276 | } | 1300 | } |
1277 | base++; | 1301 | base++; |
1278 | } | 1302 | } |
@@ -1394,7 +1418,7 @@ void hrtimer_run_queues(void) | |||
1394 | hrtimer_get_expires_tv64(timer)) | 1418 | hrtimer_get_expires_tv64(timer)) |
1395 | break; | 1419 | break; |
1396 | 1420 | ||
1397 | __run_hrtimer(timer); | 1421 | __run_hrtimer(timer, &base->softirq_time); |
1398 | } | 1422 | } |
1399 | spin_unlock(&cpu_base->lock); | 1423 | spin_unlock(&cpu_base->lock); |
1400 | } | 1424 | } |
@@ -1571,7 +1595,7 @@ static void migrate_hrtimer_list(struct hrtimer_clock_base *old_base, | |||
1571 | while ((node = rb_first(&old_base->active))) { | 1595 | while ((node = rb_first(&old_base->active))) { |
1572 | timer = rb_entry(node, struct hrtimer, node); | 1596 | timer = rb_entry(node, struct hrtimer, node); |
1573 | BUG_ON(hrtimer_callback_running(timer)); | 1597 | BUG_ON(hrtimer_callback_running(timer)); |
1574 | debug_hrtimer_deactivate(timer); | 1598 | debug_deactivate(timer); |
1575 | 1599 | ||
1576 | /* | 1600 | /* |
1577 | * Mark it as STATE_MIGRATE not INACTIVE otherwise the | 1601 | * Mark it as STATE_MIGRATE not INACTIVE otherwise the |
diff --git a/kernel/itimer.c b/kernel/itimer.c index 58762f7077ec..b03451ede528 100644 --- a/kernel/itimer.c +++ b/kernel/itimer.c | |||
@@ -12,6 +12,7 @@ | |||
12 | #include <linux/time.h> | 12 | #include <linux/time.h> |
13 | #include <linux/posix-timers.h> | 13 | #include <linux/posix-timers.h> |
14 | #include <linux/hrtimer.h> | 14 | #include <linux/hrtimer.h> |
15 | #include <trace/events/timer.h> | ||
15 | 16 | ||
16 | #include <asm/uaccess.h> | 17 | #include <asm/uaccess.h> |
17 | 18 | ||
@@ -41,10 +42,43 @@ static struct timeval itimer_get_remtime(struct hrtimer *timer) | |||
41 | return ktime_to_timeval(rem); | 42 | return ktime_to_timeval(rem); |
42 | } | 43 | } |
43 | 44 | ||
45 | static void get_cpu_itimer(struct task_struct *tsk, unsigned int clock_id, | ||
46 | struct itimerval *const value) | ||
47 | { | ||
48 | cputime_t cval, cinterval; | ||
49 | struct cpu_itimer *it = &tsk->signal->it[clock_id]; | ||
50 | |||
51 | spin_lock_irq(&tsk->sighand->siglock); | ||
52 | |||
53 | cval = it->expires; | ||
54 | cinterval = it->incr; | ||
55 | if (!cputime_eq(cval, cputime_zero)) { | ||
56 | struct task_cputime cputime; | ||
57 | cputime_t t; | ||
58 | |||
59 | thread_group_cputimer(tsk, &cputime); | ||
60 | if (clock_id == CPUCLOCK_PROF) | ||
61 | t = cputime_add(cputime.utime, cputime.stime); | ||
62 | else | ||
63 | /* CPUCLOCK_VIRT */ | ||
64 | t = cputime.utime; | ||
65 | |||
66 | if (cputime_le(cval, t)) | ||
67 | /* about to fire */ | ||
68 | cval = cputime_one_jiffy; | ||
69 | else | ||
70 | cval = cputime_sub(cval, t); | ||
71 | } | ||
72 | |||
73 | spin_unlock_irq(&tsk->sighand->siglock); | ||
74 | |||
75 | cputime_to_timeval(cval, &value->it_value); | ||
76 | cputime_to_timeval(cinterval, &value->it_interval); | ||
77 | } | ||
78 | |||
44 | int do_getitimer(int which, struct itimerval *value) | 79 | int do_getitimer(int which, struct itimerval *value) |
45 | { | 80 | { |
46 | struct task_struct *tsk = current; | 81 | struct task_struct *tsk = current; |
47 | cputime_t cinterval, cval; | ||
48 | 82 | ||
49 | switch (which) { | 83 | switch (which) { |
50 | case ITIMER_REAL: | 84 | case ITIMER_REAL: |
@@ -55,44 +89,10 @@ int do_getitimer(int which, struct itimerval *value) | |||
55 | spin_unlock_irq(&tsk->sighand->siglock); | 89 | spin_unlock_irq(&tsk->sighand->siglock); |
56 | break; | 90 | break; |
57 | case ITIMER_VIRTUAL: | 91 | case ITIMER_VIRTUAL: |
58 | spin_lock_irq(&tsk->sighand->siglock); | 92 | get_cpu_itimer(tsk, CPUCLOCK_VIRT, value); |
59 | cval = tsk->signal->it_virt_expires; | ||
60 | cinterval = tsk->signal->it_virt_incr; | ||
61 | if (!cputime_eq(cval, cputime_zero)) { | ||
62 | struct task_cputime cputime; | ||
63 | cputime_t utime; | ||
64 | |||
65 | thread_group_cputimer(tsk, &cputime); | ||
66 | utime = cputime.utime; | ||
67 | if (cputime_le(cval, utime)) { /* about to fire */ | ||
68 | cval = jiffies_to_cputime(1); | ||
69 | } else { | ||
70 | cval = cputime_sub(cval, utime); | ||
71 | } | ||
72 | } | ||
73 | spin_unlock_irq(&tsk->sighand->siglock); | ||
74 | cputime_to_timeval(cval, &value->it_value); | ||
75 | cputime_to_timeval(cinterval, &value->it_interval); | ||
76 | break; | 93 | break; |
77 | case ITIMER_PROF: | 94 | case ITIMER_PROF: |
78 | spin_lock_irq(&tsk->sighand->siglock); | 95 | get_cpu_itimer(tsk, CPUCLOCK_PROF, value); |
79 | cval = tsk->signal->it_prof_expires; | ||
80 | cinterval = tsk->signal->it_prof_incr; | ||
81 | if (!cputime_eq(cval, cputime_zero)) { | ||
82 | struct task_cputime times; | ||
83 | cputime_t ptime; | ||
84 | |||
85 | thread_group_cputimer(tsk, ×); | ||
86 | ptime = cputime_add(times.utime, times.stime); | ||
87 | if (cputime_le(cval, ptime)) { /* about to fire */ | ||
88 | cval = jiffies_to_cputime(1); | ||
89 | } else { | ||
90 | cval = cputime_sub(cval, ptime); | ||
91 | } | ||
92 | } | ||
93 | spin_unlock_irq(&tsk->sighand->siglock); | ||
94 | cputime_to_timeval(cval, &value->it_value); | ||
95 | cputime_to_timeval(cinterval, &value->it_interval); | ||
96 | break; | 96 | break; |
97 | default: | 97 | default: |
98 | return(-EINVAL); | 98 | return(-EINVAL); |
@@ -123,11 +123,62 @@ enum hrtimer_restart it_real_fn(struct hrtimer *timer) | |||
123 | struct signal_struct *sig = | 123 | struct signal_struct *sig = |
124 | container_of(timer, struct signal_struct, real_timer); | 124 | container_of(timer, struct signal_struct, real_timer); |
125 | 125 | ||
126 | trace_itimer_expire(ITIMER_REAL, sig->leader_pid, 0); | ||
126 | kill_pid_info(SIGALRM, SEND_SIG_PRIV, sig->leader_pid); | 127 | kill_pid_info(SIGALRM, SEND_SIG_PRIV, sig->leader_pid); |
127 | 128 | ||
128 | return HRTIMER_NORESTART; | 129 | return HRTIMER_NORESTART; |
129 | } | 130 | } |
130 | 131 | ||
132 | static inline u32 cputime_sub_ns(cputime_t ct, s64 real_ns) | ||
133 | { | ||
134 | struct timespec ts; | ||
135 | s64 cpu_ns; | ||
136 | |||
137 | cputime_to_timespec(ct, &ts); | ||
138 | cpu_ns = timespec_to_ns(&ts); | ||
139 | |||
140 | return (cpu_ns <= real_ns) ? 0 : cpu_ns - real_ns; | ||
141 | } | ||
142 | |||
143 | static void set_cpu_itimer(struct task_struct *tsk, unsigned int clock_id, | ||
144 | const struct itimerval *const value, | ||
145 | struct itimerval *const ovalue) | ||
146 | { | ||
147 | cputime_t cval, nval, cinterval, ninterval; | ||
148 | s64 ns_ninterval, ns_nval; | ||
149 | struct cpu_itimer *it = &tsk->signal->it[clock_id]; | ||
150 | |||
151 | nval = timeval_to_cputime(&value->it_value); | ||
152 | ns_nval = timeval_to_ns(&value->it_value); | ||
153 | ninterval = timeval_to_cputime(&value->it_interval); | ||
154 | ns_ninterval = timeval_to_ns(&value->it_interval); | ||
155 | |||
156 | it->incr_error = cputime_sub_ns(ninterval, ns_ninterval); | ||
157 | it->error = cputime_sub_ns(nval, ns_nval); | ||
158 | |||
159 | spin_lock_irq(&tsk->sighand->siglock); | ||
160 | |||
161 | cval = it->expires; | ||
162 | cinterval = it->incr; | ||
163 | if (!cputime_eq(cval, cputime_zero) || | ||
164 | !cputime_eq(nval, cputime_zero)) { | ||
165 | if (cputime_gt(nval, cputime_zero)) | ||
166 | nval = cputime_add(nval, cputime_one_jiffy); | ||
167 | set_process_cpu_timer(tsk, clock_id, &nval, &cval); | ||
168 | } | ||
169 | it->expires = nval; | ||
170 | it->incr = ninterval; | ||
171 | trace_itimer_state(clock_id == CPUCLOCK_VIRT ? | ||
172 | ITIMER_VIRTUAL : ITIMER_PROF, value, nval); | ||
173 | |||
174 | spin_unlock_irq(&tsk->sighand->siglock); | ||
175 | |||
176 | if (ovalue) { | ||
177 | cputime_to_timeval(cval, &ovalue->it_value); | ||
178 | cputime_to_timeval(cinterval, &ovalue->it_interval); | ||
179 | } | ||
180 | } | ||
181 | |||
131 | /* | 182 | /* |
132 | * Returns true if the timeval is in canonical form | 183 | * Returns true if the timeval is in canonical form |
133 | */ | 184 | */ |
@@ -139,7 +190,6 @@ int do_setitimer(int which, struct itimerval *value, struct itimerval *ovalue) | |||
139 | struct task_struct *tsk = current; | 190 | struct task_struct *tsk = current; |
140 | struct hrtimer *timer; | 191 | struct hrtimer *timer; |
141 | ktime_t expires; | 192 | ktime_t expires; |
142 | cputime_t cval, cinterval, nval, ninterval; | ||
143 | 193 | ||
144 | /* | 194 | /* |
145 | * Validate the timevals in value. | 195 | * Validate the timevals in value. |
@@ -171,51 +221,14 @@ again: | |||
171 | } else | 221 | } else |
172 | tsk->signal->it_real_incr.tv64 = 0; | 222 | tsk->signal->it_real_incr.tv64 = 0; |
173 | 223 | ||
224 | trace_itimer_state(ITIMER_REAL, value, 0); | ||
174 | spin_unlock_irq(&tsk->sighand->siglock); | 225 | spin_unlock_irq(&tsk->sighand->siglock); |
175 | break; | 226 | break; |
176 | case ITIMER_VIRTUAL: | 227 | case ITIMER_VIRTUAL: |
177 | nval = timeval_to_cputime(&value->it_value); | 228 | set_cpu_itimer(tsk, CPUCLOCK_VIRT, value, ovalue); |
178 | ninterval = timeval_to_cputime(&value->it_interval); | ||
179 | spin_lock_irq(&tsk->sighand->siglock); | ||
180 | cval = tsk->signal->it_virt_expires; | ||
181 | cinterval = tsk->signal->it_virt_incr; | ||
182 | if (!cputime_eq(cval, cputime_zero) || | ||
183 | !cputime_eq(nval, cputime_zero)) { | ||
184 | if (cputime_gt(nval, cputime_zero)) | ||
185 | nval = cputime_add(nval, | ||
186 | jiffies_to_cputime(1)); | ||
187 | set_process_cpu_timer(tsk, CPUCLOCK_VIRT, | ||
188 | &nval, &cval); | ||
189 | } | ||
190 | tsk->signal->it_virt_expires = nval; | ||
191 | tsk->signal->it_virt_incr = ninterval; | ||
192 | spin_unlock_irq(&tsk->sighand->siglock); | ||
193 | if (ovalue) { | ||
194 | cputime_to_timeval(cval, &ovalue->it_value); | ||
195 | cputime_to_timeval(cinterval, &ovalue->it_interval); | ||
196 | } | ||
197 | break; | 229 | break; |
198 | case ITIMER_PROF: | 230 | case ITIMER_PROF: |
199 | nval = timeval_to_cputime(&value->it_value); | 231 | set_cpu_itimer(tsk, CPUCLOCK_PROF, value, ovalue); |
200 | ninterval = timeval_to_cputime(&value->it_interval); | ||
201 | spin_lock_irq(&tsk->sighand->siglock); | ||
202 | cval = tsk->signal->it_prof_expires; | ||
203 | cinterval = tsk->signal->it_prof_incr; | ||
204 | if (!cputime_eq(cval, cputime_zero) || | ||
205 | !cputime_eq(nval, cputime_zero)) { | ||
206 | if (cputime_gt(nval, cputime_zero)) | ||
207 | nval = cputime_add(nval, | ||
208 | jiffies_to_cputime(1)); | ||
209 | set_process_cpu_timer(tsk, CPUCLOCK_PROF, | ||
210 | &nval, &cval); | ||
211 | } | ||
212 | tsk->signal->it_prof_expires = nval; | ||
213 | tsk->signal->it_prof_incr = ninterval; | ||
214 | spin_unlock_irq(&tsk->sighand->siglock); | ||
215 | if (ovalue) { | ||
216 | cputime_to_timeval(cval, &ovalue->it_value); | ||
217 | cputime_to_timeval(cinterval, &ovalue->it_interval); | ||
218 | } | ||
219 | break; | 232 | break; |
220 | default: | 233 | default: |
221 | return -EINVAL; | 234 | return -EINVAL; |
diff --git a/kernel/posix-cpu-timers.c b/kernel/posix-cpu-timers.c index e33a21cb9407..5c9dc228747b 100644 --- a/kernel/posix-cpu-timers.c +++ b/kernel/posix-cpu-timers.c | |||
@@ -8,17 +8,18 @@ | |||
8 | #include <linux/math64.h> | 8 | #include <linux/math64.h> |
9 | #include <asm/uaccess.h> | 9 | #include <asm/uaccess.h> |
10 | #include <linux/kernel_stat.h> | 10 | #include <linux/kernel_stat.h> |
11 | #include <trace/events/timer.h> | ||
11 | 12 | ||
12 | /* | 13 | /* |
13 | * Called after updating RLIMIT_CPU to set timer expiration if necessary. | 14 | * Called after updating RLIMIT_CPU to set timer expiration if necessary. |
14 | */ | 15 | */ |
15 | void update_rlimit_cpu(unsigned long rlim_new) | 16 | void update_rlimit_cpu(unsigned long rlim_new) |
16 | { | 17 | { |
17 | cputime_t cputime; | 18 | cputime_t cputime = secs_to_cputime(rlim_new); |
19 | struct signal_struct *const sig = current->signal; | ||
18 | 20 | ||
19 | cputime = secs_to_cputime(rlim_new); | 21 | if (cputime_eq(sig->it[CPUCLOCK_PROF].expires, cputime_zero) || |
20 | if (cputime_eq(current->signal->it_prof_expires, cputime_zero) || | 22 | cputime_gt(sig->it[CPUCLOCK_PROF].expires, cputime)) { |
21 | cputime_gt(current->signal->it_prof_expires, cputime)) { | ||
22 | spin_lock_irq(¤t->sighand->siglock); | 23 | spin_lock_irq(¤t->sighand->siglock); |
23 | set_process_cpu_timer(current, CPUCLOCK_PROF, &cputime, NULL); | 24 | set_process_cpu_timer(current, CPUCLOCK_PROF, &cputime, NULL); |
24 | spin_unlock_irq(¤t->sighand->siglock); | 25 | spin_unlock_irq(¤t->sighand->siglock); |
@@ -542,6 +543,17 @@ static void clear_dead_task(struct k_itimer *timer, union cpu_time_count now) | |||
542 | now); | 543 | now); |
543 | } | 544 | } |
544 | 545 | ||
546 | static inline int expires_gt(cputime_t expires, cputime_t new_exp) | ||
547 | { | ||
548 | return cputime_eq(expires, cputime_zero) || | ||
549 | cputime_gt(expires, new_exp); | ||
550 | } | ||
551 | |||
552 | static inline int expires_le(cputime_t expires, cputime_t new_exp) | ||
553 | { | ||
554 | return !cputime_eq(expires, cputime_zero) && | ||
555 | cputime_le(expires, new_exp); | ||
556 | } | ||
545 | /* | 557 | /* |
546 | * Insert the timer on the appropriate list before any timers that | 558 | * Insert the timer on the appropriate list before any timers that |
547 | * expire later. This must be called with the tasklist_lock held | 559 | * expire later. This must be called with the tasklist_lock held |
@@ -586,34 +598,32 @@ static void arm_timer(struct k_itimer *timer, union cpu_time_count now) | |||
586 | */ | 598 | */ |
587 | 599 | ||
588 | if (CPUCLOCK_PERTHREAD(timer->it_clock)) { | 600 | if (CPUCLOCK_PERTHREAD(timer->it_clock)) { |
601 | union cpu_time_count *exp = &nt->expires; | ||
602 | |||
589 | switch (CPUCLOCK_WHICH(timer->it_clock)) { | 603 | switch (CPUCLOCK_WHICH(timer->it_clock)) { |
590 | default: | 604 | default: |
591 | BUG(); | 605 | BUG(); |
592 | case CPUCLOCK_PROF: | 606 | case CPUCLOCK_PROF: |
593 | if (cputime_eq(p->cputime_expires.prof_exp, | 607 | if (expires_gt(p->cputime_expires.prof_exp, |
594 | cputime_zero) || | 608 | exp->cpu)) |
595 | cputime_gt(p->cputime_expires.prof_exp, | 609 | p->cputime_expires.prof_exp = exp->cpu; |
596 | nt->expires.cpu)) | ||
597 | p->cputime_expires.prof_exp = | ||
598 | nt->expires.cpu; | ||
599 | break; | 610 | break; |
600 | case CPUCLOCK_VIRT: | 611 | case CPUCLOCK_VIRT: |
601 | if (cputime_eq(p->cputime_expires.virt_exp, | 612 | if (expires_gt(p->cputime_expires.virt_exp, |
602 | cputime_zero) || | 613 | exp->cpu)) |
603 | cputime_gt(p->cputime_expires.virt_exp, | 614 | p->cputime_expires.virt_exp = exp->cpu; |
604 | nt->expires.cpu)) | ||
605 | p->cputime_expires.virt_exp = | ||
606 | nt->expires.cpu; | ||
607 | break; | 615 | break; |
608 | case CPUCLOCK_SCHED: | 616 | case CPUCLOCK_SCHED: |
609 | if (p->cputime_expires.sched_exp == 0 || | 617 | if (p->cputime_expires.sched_exp == 0 || |
610 | p->cputime_expires.sched_exp > | 618 | p->cputime_expires.sched_exp > exp->sched) |
611 | nt->expires.sched) | ||
612 | p->cputime_expires.sched_exp = | 619 | p->cputime_expires.sched_exp = |
613 | nt->expires.sched; | 620 | exp->sched; |
614 | break; | 621 | break; |
615 | } | 622 | } |
616 | } else { | 623 | } else { |
624 | struct signal_struct *const sig = p->signal; | ||
625 | union cpu_time_count *exp = &timer->it.cpu.expires; | ||
626 | |||
617 | /* | 627 | /* |
618 | * For a process timer, set the cached expiration time. | 628 | * For a process timer, set the cached expiration time. |
619 | */ | 629 | */ |
@@ -621,30 +631,23 @@ static void arm_timer(struct k_itimer *timer, union cpu_time_count now) | |||
621 | default: | 631 | default: |
622 | BUG(); | 632 | BUG(); |
623 | case CPUCLOCK_VIRT: | 633 | case CPUCLOCK_VIRT: |
624 | if (!cputime_eq(p->signal->it_virt_expires, | 634 | if (expires_le(sig->it[CPUCLOCK_VIRT].expires, |
625 | cputime_zero) && | 635 | exp->cpu)) |
626 | cputime_lt(p->signal->it_virt_expires, | ||
627 | timer->it.cpu.expires.cpu)) | ||
628 | break; | 636 | break; |
629 | p->signal->cputime_expires.virt_exp = | 637 | sig->cputime_expires.virt_exp = exp->cpu; |
630 | timer->it.cpu.expires.cpu; | ||
631 | break; | 638 | break; |
632 | case CPUCLOCK_PROF: | 639 | case CPUCLOCK_PROF: |
633 | if (!cputime_eq(p->signal->it_prof_expires, | 640 | if (expires_le(sig->it[CPUCLOCK_PROF].expires, |
634 | cputime_zero) && | 641 | exp->cpu)) |
635 | cputime_lt(p->signal->it_prof_expires, | ||
636 | timer->it.cpu.expires.cpu)) | ||
637 | break; | 642 | break; |
638 | i = p->signal->rlim[RLIMIT_CPU].rlim_cur; | 643 | i = sig->rlim[RLIMIT_CPU].rlim_cur; |
639 | if (i != RLIM_INFINITY && | 644 | if (i != RLIM_INFINITY && |
640 | i <= cputime_to_secs(timer->it.cpu.expires.cpu)) | 645 | i <= cputime_to_secs(exp->cpu)) |
641 | break; | 646 | break; |
642 | p->signal->cputime_expires.prof_exp = | 647 | sig->cputime_expires.prof_exp = exp->cpu; |
643 | timer->it.cpu.expires.cpu; | ||
644 | break; | 648 | break; |
645 | case CPUCLOCK_SCHED: | 649 | case CPUCLOCK_SCHED: |
646 | p->signal->cputime_expires.sched_exp = | 650 | sig->cputime_expires.sched_exp = exp->sched; |
647 | timer->it.cpu.expires.sched; | ||
648 | break; | 651 | break; |
649 | } | 652 | } |
650 | } | 653 | } |
@@ -1071,6 +1074,40 @@ static void stop_process_timers(struct task_struct *tsk) | |||
1071 | spin_unlock_irqrestore(&cputimer->lock, flags); | 1074 | spin_unlock_irqrestore(&cputimer->lock, flags); |
1072 | } | 1075 | } |
1073 | 1076 | ||
1077 | static u32 onecputick; | ||
1078 | |||
1079 | static void check_cpu_itimer(struct task_struct *tsk, struct cpu_itimer *it, | ||
1080 | cputime_t *expires, cputime_t cur_time, int signo) | ||
1081 | { | ||
1082 | if (cputime_eq(it->expires, cputime_zero)) | ||
1083 | return; | ||
1084 | |||
1085 | if (cputime_ge(cur_time, it->expires)) { | ||
1086 | if (!cputime_eq(it->incr, cputime_zero)) { | ||
1087 | it->expires = cputime_add(it->expires, it->incr); | ||
1088 | it->error += it->incr_error; | ||
1089 | if (it->error >= onecputick) { | ||
1090 | it->expires = cputime_sub(it->expires, | ||
1091 | cputime_one_jiffy); | ||
1092 | it->error -= onecputick; | ||
1093 | } | ||
1094 | } else { | ||
1095 | it->expires = cputime_zero; | ||
1096 | } | ||
1097 | |||
1098 | trace_itimer_expire(signo == SIGPROF ? | ||
1099 | ITIMER_PROF : ITIMER_VIRTUAL, | ||
1100 | tsk->signal->leader_pid, cur_time); | ||
1101 | __group_send_sig_info(signo, SEND_SIG_PRIV, tsk); | ||
1102 | } | ||
1103 | |||
1104 | if (!cputime_eq(it->expires, cputime_zero) && | ||
1105 | (cputime_eq(*expires, cputime_zero) || | ||
1106 | cputime_lt(it->expires, *expires))) { | ||
1107 | *expires = it->expires; | ||
1108 | } | ||
1109 | } | ||
1110 | |||
1074 | /* | 1111 | /* |
1075 | * Check for any per-thread CPU timers that have fired and move them | 1112 | * Check for any per-thread CPU timers that have fired and move them |
1076 | * off the tsk->*_timers list onto the firing list. Per-thread timers | 1113 | * off the tsk->*_timers list onto the firing list. Per-thread timers |
@@ -1090,10 +1127,10 @@ static void check_process_timers(struct task_struct *tsk, | |||
1090 | * Don't sample the current process CPU clocks if there are no timers. | 1127 | * Don't sample the current process CPU clocks if there are no timers. |
1091 | */ | 1128 | */ |
1092 | if (list_empty(&timers[CPUCLOCK_PROF]) && | 1129 | if (list_empty(&timers[CPUCLOCK_PROF]) && |
1093 | cputime_eq(sig->it_prof_expires, cputime_zero) && | 1130 | cputime_eq(sig->it[CPUCLOCK_PROF].expires, cputime_zero) && |
1094 | sig->rlim[RLIMIT_CPU].rlim_cur == RLIM_INFINITY && | 1131 | sig->rlim[RLIMIT_CPU].rlim_cur == RLIM_INFINITY && |
1095 | list_empty(&timers[CPUCLOCK_VIRT]) && | 1132 | list_empty(&timers[CPUCLOCK_VIRT]) && |
1096 | cputime_eq(sig->it_virt_expires, cputime_zero) && | 1133 | cputime_eq(sig->it[CPUCLOCK_VIRT].expires, cputime_zero) && |
1097 | list_empty(&timers[CPUCLOCK_SCHED])) { | 1134 | list_empty(&timers[CPUCLOCK_SCHED])) { |
1098 | stop_process_timers(tsk); | 1135 | stop_process_timers(tsk); |
1099 | return; | 1136 | return; |
@@ -1153,38 +1190,11 @@ static void check_process_timers(struct task_struct *tsk, | |||
1153 | /* | 1190 | /* |
1154 | * Check for the special case process timers. | 1191 | * Check for the special case process timers. |
1155 | */ | 1192 | */ |
1156 | if (!cputime_eq(sig->it_prof_expires, cputime_zero)) { | 1193 | check_cpu_itimer(tsk, &sig->it[CPUCLOCK_PROF], &prof_expires, ptime, |
1157 | if (cputime_ge(ptime, sig->it_prof_expires)) { | 1194 | SIGPROF); |
1158 | /* ITIMER_PROF fires and reloads. */ | 1195 | check_cpu_itimer(tsk, &sig->it[CPUCLOCK_VIRT], &virt_expires, utime, |
1159 | sig->it_prof_expires = sig->it_prof_incr; | 1196 | SIGVTALRM); |
1160 | if (!cputime_eq(sig->it_prof_expires, cputime_zero)) { | 1197 | |
1161 | sig->it_prof_expires = cputime_add( | ||
1162 | sig->it_prof_expires, ptime); | ||
1163 | } | ||
1164 | __group_send_sig_info(SIGPROF, SEND_SIG_PRIV, tsk); | ||
1165 | } | ||
1166 | if (!cputime_eq(sig->it_prof_expires, cputime_zero) && | ||
1167 | (cputime_eq(prof_expires, cputime_zero) || | ||
1168 | cputime_lt(sig->it_prof_expires, prof_expires))) { | ||
1169 | prof_expires = sig->it_prof_expires; | ||
1170 | } | ||
1171 | } | ||
1172 | if (!cputime_eq(sig->it_virt_expires, cputime_zero)) { | ||
1173 | if (cputime_ge(utime, sig->it_virt_expires)) { | ||
1174 | /* ITIMER_VIRTUAL fires and reloads. */ | ||
1175 | sig->it_virt_expires = sig->it_virt_incr; | ||
1176 | if (!cputime_eq(sig->it_virt_expires, cputime_zero)) { | ||
1177 | sig->it_virt_expires = cputime_add( | ||
1178 | sig->it_virt_expires, utime); | ||
1179 | } | ||
1180 | __group_send_sig_info(SIGVTALRM, SEND_SIG_PRIV, tsk); | ||
1181 | } | ||
1182 | if (!cputime_eq(sig->it_virt_expires, cputime_zero) && | ||
1183 | (cputime_eq(virt_expires, cputime_zero) || | ||
1184 | cputime_lt(sig->it_virt_expires, virt_expires))) { | ||
1185 | virt_expires = sig->it_virt_expires; | ||
1186 | } | ||
1187 | } | ||
1188 | if (sig->rlim[RLIMIT_CPU].rlim_cur != RLIM_INFINITY) { | 1198 | if (sig->rlim[RLIMIT_CPU].rlim_cur != RLIM_INFINITY) { |
1189 | unsigned long psecs = cputime_to_secs(ptime); | 1199 | unsigned long psecs = cputime_to_secs(ptime); |
1190 | cputime_t x; | 1200 | cputime_t x; |
@@ -1457,7 +1467,7 @@ void set_process_cpu_timer(struct task_struct *tsk, unsigned int clock_idx, | |||
1457 | if (!cputime_eq(*oldval, cputime_zero)) { | 1467 | if (!cputime_eq(*oldval, cputime_zero)) { |
1458 | if (cputime_le(*oldval, now.cpu)) { | 1468 | if (cputime_le(*oldval, now.cpu)) { |
1459 | /* Just about to fire. */ | 1469 | /* Just about to fire. */ |
1460 | *oldval = jiffies_to_cputime(1); | 1470 | *oldval = cputime_one_jiffy; |
1461 | } else { | 1471 | } else { |
1462 | *oldval = cputime_sub(*oldval, now.cpu); | 1472 | *oldval = cputime_sub(*oldval, now.cpu); |
1463 | } | 1473 | } |
@@ -1703,10 +1713,15 @@ static __init int init_posix_cpu_timers(void) | |||
1703 | .nsleep = thread_cpu_nsleep, | 1713 | .nsleep = thread_cpu_nsleep, |
1704 | .nsleep_restart = thread_cpu_nsleep_restart, | 1714 | .nsleep_restart = thread_cpu_nsleep_restart, |
1705 | }; | 1715 | }; |
1716 | struct timespec ts; | ||
1706 | 1717 | ||
1707 | register_posix_clock(CLOCK_PROCESS_CPUTIME_ID, &process); | 1718 | register_posix_clock(CLOCK_PROCESS_CPUTIME_ID, &process); |
1708 | register_posix_clock(CLOCK_THREAD_CPUTIME_ID, &thread); | 1719 | register_posix_clock(CLOCK_THREAD_CPUTIME_ID, &thread); |
1709 | 1720 | ||
1721 | cputime_to_timespec(cputime_one_jiffy, &ts); | ||
1722 | onecputick = ts.tv_nsec; | ||
1723 | WARN_ON(ts.tv_sec != 0); | ||
1724 | |||
1710 | return 0; | 1725 | return 0; |
1711 | } | 1726 | } |
1712 | __initcall(init_posix_cpu_timers); | 1727 | __initcall(init_posix_cpu_timers); |
diff --git a/kernel/sched.c b/kernel/sched.c index 0ac9053c21d6..2f76e06bea58 100644 --- a/kernel/sched.c +++ b/kernel/sched.c | |||
@@ -5092,17 +5092,16 @@ void account_idle_time(cputime_t cputime) | |||
5092 | */ | 5092 | */ |
5093 | void account_process_tick(struct task_struct *p, int user_tick) | 5093 | void account_process_tick(struct task_struct *p, int user_tick) |
5094 | { | 5094 | { |
5095 | cputime_t one_jiffy = jiffies_to_cputime(1); | 5095 | cputime_t one_jiffy_scaled = cputime_to_scaled(cputime_one_jiffy); |
5096 | cputime_t one_jiffy_scaled = cputime_to_scaled(one_jiffy); | ||
5097 | struct rq *rq = this_rq(); | 5096 | struct rq *rq = this_rq(); |
5098 | 5097 | ||
5099 | if (user_tick) | 5098 | if (user_tick) |
5100 | account_user_time(p, one_jiffy, one_jiffy_scaled); | 5099 | account_user_time(p, cputime_one_jiffy, one_jiffy_scaled); |
5101 | else if ((p != rq->idle) || (irq_count() != HARDIRQ_OFFSET)) | 5100 | else if ((p != rq->idle) || (irq_count() != HARDIRQ_OFFSET)) |
5102 | account_system_time(p, HARDIRQ_OFFSET, one_jiffy, | 5101 | account_system_time(p, HARDIRQ_OFFSET, cputime_one_jiffy, |
5103 | one_jiffy_scaled); | 5102 | one_jiffy_scaled); |
5104 | else | 5103 | else |
5105 | account_idle_time(one_jiffy); | 5104 | account_idle_time(cputime_one_jiffy); |
5106 | } | 5105 | } |
5107 | 5106 | ||
5108 | /* | 5107 | /* |
diff --git a/kernel/timer.c b/kernel/timer.c index 811e5c391456..5db5a8d26811 100644 --- a/kernel/timer.c +++ b/kernel/timer.c | |||
@@ -46,6 +46,9 @@ | |||
46 | #include <asm/timex.h> | 46 | #include <asm/timex.h> |
47 | #include <asm/io.h> | 47 | #include <asm/io.h> |
48 | 48 | ||
49 | #define CREATE_TRACE_POINTS | ||
50 | #include <trace/events/timer.h> | ||
51 | |||
49 | u64 jiffies_64 __cacheline_aligned_in_smp = INITIAL_JIFFIES; | 52 | u64 jiffies_64 __cacheline_aligned_in_smp = INITIAL_JIFFIES; |
50 | 53 | ||
51 | EXPORT_SYMBOL(jiffies_64); | 54 | EXPORT_SYMBOL(jiffies_64); |
@@ -521,6 +524,25 @@ static inline void debug_timer_activate(struct timer_list *timer) { } | |||
521 | static inline void debug_timer_deactivate(struct timer_list *timer) { } | 524 | static inline void debug_timer_deactivate(struct timer_list *timer) { } |
522 | #endif | 525 | #endif |
523 | 526 | ||
527 | static inline void debug_init(struct timer_list *timer) | ||
528 | { | ||
529 | debug_timer_init(timer); | ||
530 | trace_timer_init(timer); | ||
531 | } | ||
532 | |||
533 | static inline void | ||
534 | debug_activate(struct timer_list *timer, unsigned long expires) | ||
535 | { | ||
536 | debug_timer_activate(timer); | ||
537 | trace_timer_start(timer, expires); | ||
538 | } | ||
539 | |||
540 | static inline void debug_deactivate(struct timer_list *timer) | ||
541 | { | ||
542 | debug_timer_deactivate(timer); | ||
543 | trace_timer_cancel(timer); | ||
544 | } | ||
545 | |||
524 | static void __init_timer(struct timer_list *timer, | 546 | static void __init_timer(struct timer_list *timer, |
525 | const char *name, | 547 | const char *name, |
526 | struct lock_class_key *key) | 548 | struct lock_class_key *key) |
@@ -549,7 +571,7 @@ void init_timer_key(struct timer_list *timer, | |||
549 | const char *name, | 571 | const char *name, |
550 | struct lock_class_key *key) | 572 | struct lock_class_key *key) |
551 | { | 573 | { |
552 | debug_timer_init(timer); | 574 | debug_init(timer); |
553 | __init_timer(timer, name, key); | 575 | __init_timer(timer, name, key); |
554 | } | 576 | } |
555 | EXPORT_SYMBOL(init_timer_key); | 577 | EXPORT_SYMBOL(init_timer_key); |
@@ -568,7 +590,7 @@ static inline void detach_timer(struct timer_list *timer, | |||
568 | { | 590 | { |
569 | struct list_head *entry = &timer->entry; | 591 | struct list_head *entry = &timer->entry; |
570 | 592 | ||
571 | debug_timer_deactivate(timer); | 593 | debug_deactivate(timer); |
572 | 594 | ||
573 | __list_del(entry->prev, entry->next); | 595 | __list_del(entry->prev, entry->next); |
574 | if (clear_pending) | 596 | if (clear_pending) |
@@ -632,7 +654,7 @@ __mod_timer(struct timer_list *timer, unsigned long expires, | |||
632 | goto out_unlock; | 654 | goto out_unlock; |
633 | } | 655 | } |
634 | 656 | ||
635 | debug_timer_activate(timer); | 657 | debug_activate(timer, expires); |
636 | 658 | ||
637 | new_base = __get_cpu_var(tvec_bases); | 659 | new_base = __get_cpu_var(tvec_bases); |
638 | 660 | ||
@@ -787,7 +809,7 @@ void add_timer_on(struct timer_list *timer, int cpu) | |||
787 | BUG_ON(timer_pending(timer) || !timer->function); | 809 | BUG_ON(timer_pending(timer) || !timer->function); |
788 | spin_lock_irqsave(&base->lock, flags); | 810 | spin_lock_irqsave(&base->lock, flags); |
789 | timer_set_base(timer, base); | 811 | timer_set_base(timer, base); |
790 | debug_timer_activate(timer); | 812 | debug_activate(timer, timer->expires); |
791 | if (time_before(timer->expires, base->next_timer) && | 813 | if (time_before(timer->expires, base->next_timer) && |
792 | !tbase_get_deferrable(timer->base)) | 814 | !tbase_get_deferrable(timer->base)) |
793 | base->next_timer = timer->expires; | 815 | base->next_timer = timer->expires; |
@@ -1000,7 +1022,9 @@ static inline void __run_timers(struct tvec_base *base) | |||
1000 | */ | 1022 | */ |
1001 | lock_map_acquire(&lockdep_map); | 1023 | lock_map_acquire(&lockdep_map); |
1002 | 1024 | ||
1025 | trace_timer_expire_entry(timer); | ||
1003 | fn(data); | 1026 | fn(data); |
1027 | trace_timer_expire_exit(timer); | ||
1004 | 1028 | ||
1005 | lock_map_release(&lockdep_map); | 1029 | lock_map_release(&lockdep_map); |
1006 | 1030 | ||