diff options
| author | Thomas Gleixner <tglx@linutronix.de> | 2014-06-22 06:06:40 -0400 |
|---|---|---|
| committer | Thomas Gleixner <tglx@linutronix.de> | 2014-06-23 05:22:35 -0400 |
| commit | 5cee964597260237dd2cabb3ec22bba0da24b25d (patch) | |
| tree | f548efb4181a4cffb026adf43178e65330533e87 /kernel/time | |
| parent | 58394271c610e9c65dd0165a1c1f6dec75dc5f3e (diff) | |
time/timers: Move all time(r) related files into kernel/time
Except for Kconfig.HZ. That needs a separate treatment.
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Diffstat (limited to 'kernel/time')
| -rw-r--r-- | kernel/time/Makefile | 17 | ||||
| -rw-r--r-- | kernel/time/hrtimer.c | 1915 | ||||
| -rw-r--r-- | kernel/time/itimer.c | 301 | ||||
| -rw-r--r-- | kernel/time/posix-cpu-timers.c | 1490 | ||||
| -rw-r--r-- | kernel/time/posix-timers.c | 1121 | ||||
| -rw-r--r-- | kernel/time/time.c | 714 | ||||
| -rw-r--r-- | kernel/time/timeconst.bc | 108 | ||||
| -rw-r--r-- | kernel/time/timer.c | 1734 |
8 files changed, 7400 insertions, 0 deletions
diff --git a/kernel/time/Makefile b/kernel/time/Makefile index 57a413fd0ebf..e59ce8b1b550 100644 --- a/kernel/time/Makefile +++ b/kernel/time/Makefile | |||
| @@ -1,3 +1,4 @@ | |||
| 1 | obj-y += time.o timer.o hrtimer.o itimer.o posix-timers.o posix-cpu-timers.o | ||
| 1 | obj-y += timekeeping.o ntp.o clocksource.o jiffies.o timer_list.o | 2 | obj-y += timekeeping.o ntp.o clocksource.o jiffies.o timer_list.o |
| 2 | obj-y += timeconv.o posix-clock.o alarmtimer.o | 3 | obj-y += timeconv.o posix-clock.o alarmtimer.o |
| 3 | 4 | ||
| @@ -12,3 +13,19 @@ obj-$(CONFIG_TICK_ONESHOT) += tick-oneshot.o | |||
| 12 | obj-$(CONFIG_TICK_ONESHOT) += tick-sched.o | 13 | obj-$(CONFIG_TICK_ONESHOT) += tick-sched.o |
| 13 | obj-$(CONFIG_TIMER_STATS) += timer_stats.o | 14 | obj-$(CONFIG_TIMER_STATS) += timer_stats.o |
| 14 | obj-$(CONFIG_DEBUG_FS) += timekeeping_debug.o | 15 | obj-$(CONFIG_DEBUG_FS) += timekeeping_debug.o |
| 16 | |||
| 17 | $(obj)/time.o: $(obj)/timeconst.h | ||
| 18 | |||
| 19 | quiet_cmd_hzfile = HZFILE $@ | ||
| 20 | cmd_hzfile = echo "hz=$(CONFIG_HZ)" > $@ | ||
| 21 | |||
| 22 | targets += hz.bc | ||
| 23 | $(obj)/hz.bc: $(objtree)/include/config/hz.h FORCE | ||
| 24 | $(call if_changed,hzfile) | ||
| 25 | |||
| 26 | quiet_cmd_bc = BC $@ | ||
| 27 | cmd_bc = bc -q $(filter-out FORCE,$^) > $@ | ||
| 28 | |||
| 29 | targets += timeconst.h | ||
| 30 | $(obj)/timeconst.h: $(obj)/hz.bc $(src)/timeconst.bc FORCE | ||
| 31 | $(call if_changed,bc) | ||
diff --git a/kernel/time/hrtimer.c b/kernel/time/hrtimer.c new file mode 100644 index 000000000000..3ab28993f6e0 --- /dev/null +++ b/kernel/time/hrtimer.c | |||
| @@ -0,0 +1,1915 @@ | |||
| 1 | /* | ||
| 2 | * linux/kernel/hrtimer.c | ||
| 3 | * | ||
| 4 | * Copyright(C) 2005-2006, Thomas Gleixner <tglx@linutronix.de> | ||
| 5 | * Copyright(C) 2005-2007, Red Hat, Inc., Ingo Molnar | ||
| 6 | * Copyright(C) 2006-2007 Timesys Corp., Thomas Gleixner | ||
| 7 | * | ||
| 8 | * High-resolution kernel timers | ||
| 9 | * | ||
| 10 | * In contrast to the low-resolution timeout API implemented in | ||
| 11 | * kernel/timer.c, hrtimers provide finer resolution and accuracy | ||
| 12 | * depending on system configuration and capabilities. | ||
| 13 | * | ||
| 14 | * These timers are currently used for: | ||
| 15 | * - itimers | ||
| 16 | * - POSIX timers | ||
| 17 | * - nanosleep | ||
| 18 | * - precise in-kernel timing | ||
| 19 | * | ||
| 20 | * Started by: Thomas Gleixner and Ingo Molnar | ||
| 21 | * | ||
| 22 | * Credits: | ||
| 23 | * based on kernel/timer.c | ||
| 24 | * | ||
| 25 | * Help, testing, suggestions, bugfixes, improvements were | ||
| 26 | * provided by: | ||
| 27 | * | ||
| 28 | * George Anzinger, Andrew Morton, Steven Rostedt, Roman Zippel | ||
| 29 | * et. al. | ||
| 30 | * | ||
| 31 | * For licencing details see kernel-base/COPYING | ||
| 32 | */ | ||
| 33 | |||
| 34 | #include <linux/cpu.h> | ||
| 35 | #include <linux/export.h> | ||
| 36 | #include <linux/percpu.h> | ||
| 37 | #include <linux/hrtimer.h> | ||
| 38 | #include <linux/notifier.h> | ||
| 39 | #include <linux/syscalls.h> | ||
| 40 | #include <linux/kallsyms.h> | ||
| 41 | #include <linux/interrupt.h> | ||
| 42 | #include <linux/tick.h> | ||
| 43 | #include <linux/seq_file.h> | ||
| 44 | #include <linux/err.h> | ||
| 45 | #include <linux/debugobjects.h> | ||
| 46 | #include <linux/sched.h> | ||
| 47 | #include <linux/sched/sysctl.h> | ||
| 48 | #include <linux/sched/rt.h> | ||
| 49 | #include <linux/sched/deadline.h> | ||
| 50 | #include <linux/timer.h> | ||
| 51 | #include <linux/freezer.h> | ||
| 52 | |||
| 53 | #include <asm/uaccess.h> | ||
| 54 | |||
| 55 | #include <trace/events/timer.h> | ||
| 56 | |||
| 57 | /* | ||
| 58 | * The timer bases: | ||
| 59 | * | ||
| 60 | * There are more clockids then hrtimer bases. Thus, we index | ||
| 61 | * into the timer bases by the hrtimer_base_type enum. When trying | ||
| 62 | * to reach a base using a clockid, hrtimer_clockid_to_base() | ||
| 63 | * is used to convert from clockid to the proper hrtimer_base_type. | ||
| 64 | */ | ||
| 65 | DEFINE_PER_CPU(struct hrtimer_cpu_base, hrtimer_bases) = | ||
| 66 | { | ||
| 67 | |||
| 68 | .lock = __RAW_SPIN_LOCK_UNLOCKED(hrtimer_bases.lock), | ||
| 69 | .clock_base = | ||
| 70 | { | ||
| 71 | { | ||
| 72 | .index = HRTIMER_BASE_MONOTONIC, | ||
| 73 | .clockid = CLOCK_MONOTONIC, | ||
| 74 | .get_time = &ktime_get, | ||
| 75 | .resolution = KTIME_LOW_RES, | ||
| 76 | }, | ||
| 77 | { | ||
| 78 | .index = HRTIMER_BASE_REALTIME, | ||
| 79 | .clockid = CLOCK_REALTIME, | ||
| 80 | .get_time = &ktime_get_real, | ||
| 81 | .resolution = KTIME_LOW_RES, | ||
| 82 | }, | ||
| 83 | { | ||
| 84 | .index = HRTIMER_BASE_BOOTTIME, | ||
| 85 | .clockid = CLOCK_BOOTTIME, | ||
| 86 | .get_time = &ktime_get_boottime, | ||
| 87 | .resolution = KTIME_LOW_RES, | ||
| 88 | }, | ||
| 89 | { | ||
| 90 | .index = HRTIMER_BASE_TAI, | ||
| 91 | .clockid = CLOCK_TAI, | ||
| 92 | .get_time = &ktime_get_clocktai, | ||
| 93 | .resolution = KTIME_LOW_RES, | ||
| 94 | }, | ||
| 95 | } | ||
| 96 | }; | ||
| 97 | |||
| 98 | static const int hrtimer_clock_to_base_table[MAX_CLOCKS] = { | ||
| 99 | [CLOCK_REALTIME] = HRTIMER_BASE_REALTIME, | ||
| 100 | [CLOCK_MONOTONIC] = HRTIMER_BASE_MONOTONIC, | ||
| 101 | [CLOCK_BOOTTIME] = HRTIMER_BASE_BOOTTIME, | ||
| 102 | [CLOCK_TAI] = HRTIMER_BASE_TAI, | ||
| 103 | }; | ||
| 104 | |||
| 105 | static inline int hrtimer_clockid_to_base(clockid_t clock_id) | ||
| 106 | { | ||
| 107 | return hrtimer_clock_to_base_table[clock_id]; | ||
| 108 | } | ||
| 109 | |||
| 110 | |||
| 111 | /* | ||
| 112 | * Get the coarse grained time at the softirq based on xtime and | ||
| 113 | * wall_to_monotonic. | ||
| 114 | */ | ||
| 115 | static void hrtimer_get_softirq_time(struct hrtimer_cpu_base *base) | ||
| 116 | { | ||
| 117 | ktime_t xtim, mono, boot; | ||
| 118 | struct timespec xts, tom, slp; | ||
| 119 | s32 tai_offset; | ||
| 120 | |||
| 121 | get_xtime_and_monotonic_and_sleep_offset(&xts, &tom, &slp); | ||
| 122 | tai_offset = timekeeping_get_tai_offset(); | ||
| 123 | |||
| 124 | xtim = timespec_to_ktime(xts); | ||
| 125 | mono = ktime_add(xtim, timespec_to_ktime(tom)); | ||
| 126 | boot = ktime_add(mono, timespec_to_ktime(slp)); | ||
| 127 | base->clock_base[HRTIMER_BASE_REALTIME].softirq_time = xtim; | ||
| 128 | base->clock_base[HRTIMER_BASE_MONOTONIC].softirq_time = mono; | ||
| 129 | base->clock_base[HRTIMER_BASE_BOOTTIME].softirq_time = boot; | ||
| 130 | base->clock_base[HRTIMER_BASE_TAI].softirq_time = | ||
| 131 | ktime_add(xtim, ktime_set(tai_offset, 0)); | ||
| 132 | } | ||
| 133 | |||
| 134 | /* | ||
| 135 | * Functions and macros which are different for UP/SMP systems are kept in a | ||
| 136 | * single place | ||
| 137 | */ | ||
| 138 | #ifdef CONFIG_SMP | ||
| 139 | |||
| 140 | /* | ||
| 141 | * We are using hashed locking: holding per_cpu(hrtimer_bases)[n].lock | ||
| 142 | * means that all timers which are tied to this base via timer->base are | ||
| 143 | * locked, and the base itself is locked too. | ||
| 144 | * | ||
| 145 | * So __run_timers/migrate_timers can safely modify all timers which could | ||
| 146 | * be found on the lists/queues. | ||
| 147 | * | ||
| 148 | * When the timer's base is locked, and the timer removed from list, it is | ||
| 149 | * possible to set timer->base = NULL and drop the lock: the timer remains | ||
| 150 | * locked. | ||
| 151 | */ | ||
| 152 | static | ||
| 153 | struct hrtimer_clock_base *lock_hrtimer_base(const struct hrtimer *timer, | ||
| 154 | unsigned long *flags) | ||
| 155 | { | ||
| 156 | struct hrtimer_clock_base *base; | ||
| 157 | |||
| 158 | for (;;) { | ||
| 159 | base = timer->base; | ||
| 160 | if (likely(base != NULL)) { | ||
| 161 | raw_spin_lock_irqsave(&base->cpu_base->lock, *flags); | ||
| 162 | if (likely(base == timer->base)) | ||
| 163 | return base; | ||
| 164 | /* The timer has migrated to another CPU: */ | ||
| 165 | raw_spin_unlock_irqrestore(&base->cpu_base->lock, *flags); | ||
| 166 | } | ||
| 167 | cpu_relax(); | ||
| 168 | } | ||
| 169 | } | ||
| 170 | |||
| 171 | /* | ||
| 172 | * With HIGHRES=y we do not migrate the timer when it is expiring | ||
| 173 | * before the next event on the target cpu because we cannot reprogram | ||
| 174 | * the target cpu hardware and we would cause it to fire late. | ||
| 175 | * | ||
| 176 | * Called with cpu_base->lock of target cpu held. | ||
| 177 | */ | ||
| 178 | static int | ||
| 179 | hrtimer_check_target(struct hrtimer *timer, struct hrtimer_clock_base *new_base) | ||
| 180 | { | ||
| 181 | #ifdef CONFIG_HIGH_RES_TIMERS | ||
| 182 | ktime_t expires; | ||
| 183 | |||
| 184 | if (!new_base->cpu_base->hres_active) | ||
| 185 | return 0; | ||
| 186 | |||
| 187 | expires = ktime_sub(hrtimer_get_expires(timer), new_base->offset); | ||
| 188 | return expires.tv64 <= new_base->cpu_base->expires_next.tv64; | ||
| 189 | #else | ||
| 190 | return 0; | ||
| 191 | #endif | ||
| 192 | } | ||
| 193 | |||
| 194 | /* | ||
| 195 | * Switch the timer base to the current CPU when possible. | ||
| 196 | */ | ||
| 197 | static inline struct hrtimer_clock_base * | ||
| 198 | switch_hrtimer_base(struct hrtimer *timer, struct hrtimer_clock_base *base, | ||
| 199 | int pinned) | ||
| 200 | { | ||
| 201 | struct hrtimer_clock_base *new_base; | ||
| 202 | struct hrtimer_cpu_base *new_cpu_base; | ||
| 203 | int this_cpu = smp_processor_id(); | ||
| 204 | int cpu = get_nohz_timer_target(pinned); | ||
| 205 | int basenum = base->index; | ||
| 206 | |||
| 207 | again: | ||
| 208 | new_cpu_base = &per_cpu(hrtimer_bases, cpu); | ||
| 209 | new_base = &new_cpu_base->clock_base[basenum]; | ||
| 210 | |||
| 211 | if (base != new_base) { | ||
| 212 | /* | ||
| 213 | * We are trying to move timer to new_base. | ||
| 214 | * However we can't change timer's base while it is running, | ||
| 215 | * so we keep it on the same CPU. No hassle vs. reprogramming | ||
| 216 | * the event source in the high resolution case. The softirq | ||
| 217 | * code will take care of this when the timer function has | ||
| 218 | * completed. There is no conflict as we hold the lock until | ||
| 219 | * the timer is enqueued. | ||
| 220 | */ | ||
| 221 | if (unlikely(hrtimer_callback_running(timer))) | ||
| 222 | return base; | ||
| 223 | |||
| 224 | /* See the comment in lock_timer_base() */ | ||
| 225 | timer->base = NULL; | ||
| 226 | raw_spin_unlock(&base->cpu_base->lock); | ||
| 227 | raw_spin_lock(&new_base->cpu_base->lock); | ||
| 228 | |||
| 229 | if (cpu != this_cpu && hrtimer_check_target(timer, new_base)) { | ||
| 230 | cpu = this_cpu; | ||
| 231 | raw_spin_unlock(&new_base->cpu_base->lock); | ||
| 232 | raw_spin_lock(&base->cpu_base->lock); | ||
| 233 | timer->base = base; | ||
| 234 | goto again; | ||
| 235 | } | ||
| 236 | timer->base = new_base; | ||
| 237 | } else { | ||
| 238 | if (cpu != this_cpu && hrtimer_check_target(timer, new_base)) { | ||
| 239 | cpu = this_cpu; | ||
| 240 | goto again; | ||
| 241 | } | ||
| 242 | } | ||
| 243 | return new_base; | ||
| 244 | } | ||
| 245 | |||
| 246 | #else /* CONFIG_SMP */ | ||
| 247 | |||
| 248 | static inline struct hrtimer_clock_base * | ||
| 249 | lock_hrtimer_base(const struct hrtimer *timer, unsigned long *flags) | ||
| 250 | { | ||
| 251 | struct hrtimer_clock_base *base = timer->base; | ||
| 252 | |||
| 253 | raw_spin_lock_irqsave(&base->cpu_base->lock, *flags); | ||
| 254 | |||
| 255 | return base; | ||
| 256 | } | ||
| 257 | |||
| 258 | # define switch_hrtimer_base(t, b, p) (b) | ||
| 259 | |||
| 260 | #endif /* !CONFIG_SMP */ | ||
| 261 | |||
| 262 | /* | ||
| 263 | * Functions for the union type storage format of ktime_t which are | ||
| 264 | * too large for inlining: | ||
| 265 | */ | ||
| 266 | #if BITS_PER_LONG < 64 | ||
| 267 | # ifndef CONFIG_KTIME_SCALAR | ||
| 268 | /** | ||
| 269 | * ktime_add_ns - Add a scalar nanoseconds value to a ktime_t variable | ||
| 270 | * @kt: addend | ||
| 271 | * @nsec: the scalar nsec value to add | ||
| 272 | * | ||
| 273 | * Returns the sum of kt and nsec in ktime_t format | ||
| 274 | */ | ||
| 275 | ktime_t ktime_add_ns(const ktime_t kt, u64 nsec) | ||
| 276 | { | ||
| 277 | ktime_t tmp; | ||
| 278 | |||
| 279 | if (likely(nsec < NSEC_PER_SEC)) { | ||
| 280 | tmp.tv64 = nsec; | ||
| 281 | } else { | ||
| 282 | unsigned long rem = do_div(nsec, NSEC_PER_SEC); | ||
| 283 | |||
| 284 | /* Make sure nsec fits into long */ | ||
| 285 | if (unlikely(nsec > KTIME_SEC_MAX)) | ||
| 286 | return (ktime_t){ .tv64 = KTIME_MAX }; | ||
| 287 | |||
| 288 | tmp = ktime_set((long)nsec, rem); | ||
| 289 | } | ||
| 290 | |||
| 291 | return ktime_add(kt, tmp); | ||
| 292 | } | ||
| 293 | |||
| 294 | EXPORT_SYMBOL_GPL(ktime_add_ns); | ||
| 295 | |||
| 296 | /** | ||
| 297 | * ktime_sub_ns - Subtract a scalar nanoseconds value from a ktime_t variable | ||
| 298 | * @kt: minuend | ||
| 299 | * @nsec: the scalar nsec value to subtract | ||
| 300 | * | ||
| 301 | * Returns the subtraction of @nsec from @kt in ktime_t format | ||
| 302 | */ | ||
| 303 | ktime_t ktime_sub_ns(const ktime_t kt, u64 nsec) | ||
| 304 | { | ||
| 305 | ktime_t tmp; | ||
| 306 | |||
| 307 | if (likely(nsec < NSEC_PER_SEC)) { | ||
| 308 | tmp.tv64 = nsec; | ||
| 309 | } else { | ||
| 310 | unsigned long rem = do_div(nsec, NSEC_PER_SEC); | ||
| 311 | |||
| 312 | tmp = ktime_set((long)nsec, rem); | ||
| 313 | } | ||
| 314 | |||
| 315 | return ktime_sub(kt, tmp); | ||
| 316 | } | ||
| 317 | |||
| 318 | EXPORT_SYMBOL_GPL(ktime_sub_ns); | ||
| 319 | # endif /* !CONFIG_KTIME_SCALAR */ | ||
| 320 | |||
| 321 | /* | ||
| 322 | * Divide a ktime value by a nanosecond value | ||
| 323 | */ | ||
| 324 | u64 ktime_divns(const ktime_t kt, s64 div) | ||
| 325 | { | ||
| 326 | u64 dclc; | ||
| 327 | int sft = 0; | ||
| 328 | |||
| 329 | dclc = ktime_to_ns(kt); | ||
| 330 | /* Make sure the divisor is less than 2^32: */ | ||
| 331 | while (div >> 32) { | ||
| 332 | sft++; | ||
| 333 | div >>= 1; | ||
| 334 | } | ||
| 335 | dclc >>= sft; | ||
| 336 | do_div(dclc, (unsigned long) div); | ||
| 337 | |||
| 338 | return dclc; | ||
| 339 | } | ||
| 340 | #endif /* BITS_PER_LONG >= 64 */ | ||
| 341 | |||
| 342 | /* | ||
| 343 | * Add two ktime values and do a safety check for overflow: | ||
| 344 | */ | ||
| 345 | ktime_t ktime_add_safe(const ktime_t lhs, const ktime_t rhs) | ||
| 346 | { | ||
| 347 | ktime_t res = ktime_add(lhs, rhs); | ||
| 348 | |||
| 349 | /* | ||
| 350 | * We use KTIME_SEC_MAX here, the maximum timeout which we can | ||
| 351 | * return to user space in a timespec: | ||
| 352 | */ | ||
| 353 | if (res.tv64 < 0 || res.tv64 < lhs.tv64 || res.tv64 < rhs.tv64) | ||
| 354 | res = ktime_set(KTIME_SEC_MAX, 0); | ||
| 355 | |||
| 356 | return res; | ||
| 357 | } | ||
| 358 | |||
| 359 | EXPORT_SYMBOL_GPL(ktime_add_safe); | ||
| 360 | |||
| 361 | #ifdef CONFIG_DEBUG_OBJECTS_TIMERS | ||
| 362 | |||
| 363 | static struct debug_obj_descr hrtimer_debug_descr; | ||
| 364 | |||
| 365 | static void *hrtimer_debug_hint(void *addr) | ||
| 366 | { | ||
| 367 | return ((struct hrtimer *) addr)->function; | ||
| 368 | } | ||
| 369 | |||
| 370 | /* | ||
| 371 | * fixup_init is called when: | ||
| 372 | * - an active object is initialized | ||
| 373 | */ | ||
| 374 | static int hrtimer_fixup_init(void *addr, enum debug_obj_state state) | ||
| 375 | { | ||
| 376 | struct hrtimer *timer = addr; | ||
| 377 | |||
| 378 | switch (state) { | ||
| 379 | case ODEBUG_STATE_ACTIVE: | ||
| 380 | hrtimer_cancel(timer); | ||
| 381 | debug_object_init(timer, &hrtimer_debug_descr); | ||
| 382 | return 1; | ||
| 383 | default: | ||
| 384 | return 0; | ||
| 385 | } | ||
| 386 | } | ||
| 387 | |||
| 388 | /* | ||
| 389 | * fixup_activate is called when: | ||
| 390 | * - an active object is activated | ||
| 391 | * - an unknown object is activated (might be a statically initialized object) | ||
| 392 | */ | ||
| 393 | static int hrtimer_fixup_activate(void *addr, enum debug_obj_state state) | ||
| 394 | { | ||
| 395 | switch (state) { | ||
| 396 | |||
| 397 | case ODEBUG_STATE_NOTAVAILABLE: | ||
| 398 | WARN_ON_ONCE(1); | ||
| 399 | return 0; | ||
| 400 | |||
| 401 | case ODEBUG_STATE_ACTIVE: | ||
| 402 | WARN_ON(1); | ||
| 403 | |||
| 404 | default: | ||
| 405 | return 0; | ||
| 406 | } | ||
| 407 | } | ||
| 408 | |||
| 409 | /* | ||
| 410 | * fixup_free is called when: | ||
| 411 | * - an active object is freed | ||
| 412 | */ | ||
| 413 | static int hrtimer_fixup_free(void *addr, enum debug_obj_state state) | ||
| 414 | { | ||
| 415 | struct hrtimer *timer = addr; | ||
| 416 | |||
| 417 | switch (state) { | ||
| 418 | case ODEBUG_STATE_ACTIVE: | ||
| 419 | hrtimer_cancel(timer); | ||
| 420 | debug_object_free(timer, &hrtimer_debug_descr); | ||
| 421 | return 1; | ||
| 422 | default: | ||
| 423 | return 0; | ||
| 424 | } | ||
| 425 | } | ||
| 426 | |||
| 427 | static struct debug_obj_descr hrtimer_debug_descr = { | ||
| 428 | .name = "hrtimer", | ||
| 429 | .debug_hint = hrtimer_debug_hint, | ||
| 430 | .fixup_init = hrtimer_fixup_init, | ||
| 431 | .fixup_activate = hrtimer_fixup_activate, | ||
| 432 | .fixup_free = hrtimer_fixup_free, | ||
| 433 | }; | ||
| 434 | |||
| 435 | static inline void debug_hrtimer_init(struct hrtimer *timer) | ||
| 436 | { | ||
| 437 | debug_object_init(timer, &hrtimer_debug_descr); | ||
| 438 | } | ||
| 439 | |||
| 440 | static inline void debug_hrtimer_activate(struct hrtimer *timer) | ||
| 441 | { | ||
| 442 | debug_object_activate(timer, &hrtimer_debug_descr); | ||
| 443 | } | ||
| 444 | |||
| 445 | static inline void debug_hrtimer_deactivate(struct hrtimer *timer) | ||
| 446 | { | ||
| 447 | debug_object_deactivate(timer, &hrtimer_debug_descr); | ||
| 448 | } | ||
| 449 | |||
| 450 | static inline void debug_hrtimer_free(struct hrtimer *timer) | ||
| 451 | { | ||
| 452 | debug_object_free(timer, &hrtimer_debug_descr); | ||
| 453 | } | ||
| 454 | |||
| 455 | static void __hrtimer_init(struct hrtimer *timer, clockid_t clock_id, | ||
| 456 | enum hrtimer_mode mode); | ||
| 457 | |||
| 458 | void hrtimer_init_on_stack(struct hrtimer *timer, clockid_t clock_id, | ||
| 459 | enum hrtimer_mode mode) | ||
| 460 | { | ||
| 461 | debug_object_init_on_stack(timer, &hrtimer_debug_descr); | ||
| 462 | __hrtimer_init(timer, clock_id, mode); | ||
| 463 | } | ||
| 464 | EXPORT_SYMBOL_GPL(hrtimer_init_on_stack); | ||
| 465 | |||
| 466 | void destroy_hrtimer_on_stack(struct hrtimer *timer) | ||
| 467 | { | ||
| 468 | debug_object_free(timer, &hrtimer_debug_descr); | ||
| 469 | } | ||
| 470 | |||
| 471 | #else | ||
| 472 | static inline void debug_hrtimer_init(struct hrtimer *timer) { } | ||
| 473 | static inline void debug_hrtimer_activate(struct hrtimer *timer) { } | ||
| 474 | static inline void debug_hrtimer_deactivate(struct hrtimer *timer) { } | ||
| 475 | #endif | ||
| 476 | |||
| 477 | static inline void | ||
| 478 | debug_init(struct hrtimer *timer, clockid_t clockid, | ||
| 479 | enum hrtimer_mode mode) | ||
| 480 | { | ||
| 481 | debug_hrtimer_init(timer); | ||
| 482 | trace_hrtimer_init(timer, clockid, mode); | ||
| 483 | } | ||
| 484 | |||
| 485 | static inline void debug_activate(struct hrtimer *timer) | ||
| 486 | { | ||
| 487 | debug_hrtimer_activate(timer); | ||
| 488 | trace_hrtimer_start(timer); | ||
| 489 | } | ||
| 490 | |||
| 491 | static inline void debug_deactivate(struct hrtimer *timer) | ||
| 492 | { | ||
| 493 | debug_hrtimer_deactivate(timer); | ||
| 494 | trace_hrtimer_cancel(timer); | ||
| 495 | } | ||
| 496 | |||
| 497 | /* High resolution timer related functions */ | ||
| 498 | #ifdef CONFIG_HIGH_RES_TIMERS | ||
| 499 | |||
| 500 | /* | ||
| 501 | * High resolution timer enabled ? | ||
| 502 | */ | ||
| 503 | static int hrtimer_hres_enabled __read_mostly = 1; | ||
| 504 | |||
| 505 | /* | ||
| 506 | * Enable / Disable high resolution mode | ||
| 507 | */ | ||
| 508 | static int __init setup_hrtimer_hres(char *str) | ||
| 509 | { | ||
| 510 | if (!strcmp(str, "off")) | ||
| 511 | hrtimer_hres_enabled = 0; | ||
| 512 | else if (!strcmp(str, "on")) | ||
| 513 | hrtimer_hres_enabled = 1; | ||
| 514 | else | ||
| 515 | return 0; | ||
| 516 | return 1; | ||
| 517 | } | ||
| 518 | |||
| 519 | __setup("highres=", setup_hrtimer_hres); | ||
| 520 | |||
| 521 | /* | ||
| 522 | * hrtimer_high_res_enabled - query, if the highres mode is enabled | ||
| 523 | */ | ||
| 524 | static inline int hrtimer_is_hres_enabled(void) | ||
| 525 | { | ||
| 526 | return hrtimer_hres_enabled; | ||
| 527 | } | ||
| 528 | |||
| 529 | /* | ||
| 530 | * Is the high resolution mode active ? | ||
| 531 | */ | ||
| 532 | static inline int hrtimer_hres_active(void) | ||
| 533 | { | ||
| 534 | return __this_cpu_read(hrtimer_bases.hres_active); | ||
| 535 | } | ||
| 536 | |||
| 537 | /* | ||
| 538 | * Reprogram the event source with checking both queues for the | ||
| 539 | * next event | ||
| 540 | * Called with interrupts disabled and base->lock held | ||
| 541 | */ | ||
| 542 | static void | ||
| 543 | hrtimer_force_reprogram(struct hrtimer_cpu_base *cpu_base, int skip_equal) | ||
| 544 | { | ||
| 545 | int i; | ||
| 546 | struct hrtimer_clock_base *base = cpu_base->clock_base; | ||
| 547 | ktime_t expires, expires_next; | ||
| 548 | |||
| 549 | expires_next.tv64 = KTIME_MAX; | ||
| 550 | |||
| 551 | for (i = 0; i < HRTIMER_MAX_CLOCK_BASES; i++, base++) { | ||
| 552 | struct hrtimer *timer; | ||
| 553 | struct timerqueue_node *next; | ||
| 554 | |||
| 555 | next = timerqueue_getnext(&base->active); | ||
| 556 | if (!next) | ||
| 557 | continue; | ||
| 558 | timer = container_of(next, struct hrtimer, node); | ||
| 559 | |||
| 560 | expires = ktime_sub(hrtimer_get_expires(timer), base->offset); | ||
| 561 | /* | ||
| 562 | * clock_was_set() has changed base->offset so the | ||
| 563 | * result might be negative. Fix it up to prevent a | ||
| 564 | * false positive in clockevents_program_event() | ||
| 565 | */ | ||
| 566 | if (expires.tv64 < 0) | ||
| 567 | expires.tv64 = 0; | ||
| 568 | if (expires.tv64 < expires_next.tv64) | ||
| 569 | expires_next = expires; | ||
| 570 | } | ||
| 571 | |||
| 572 | if (skip_equal && expires_next.tv64 == cpu_base->expires_next.tv64) | ||
| 573 | return; | ||
| 574 | |||
| 575 | cpu_base->expires_next.tv64 = expires_next.tv64; | ||
| 576 | |||
| 577 | /* | ||
| 578 | * If a hang was detected in the last timer interrupt then we | ||
| 579 | * leave the hang delay active in the hardware. We want the | ||
| 580 | * system to make progress. That also prevents the following | ||
| 581 | * scenario: | ||
| 582 | * T1 expires 50ms from now | ||
| 583 | * T2 expires 5s from now | ||
| 584 | * | ||
| 585 | * T1 is removed, so this code is called and would reprogram | ||
| 586 | * the hardware to 5s from now. Any hrtimer_start after that | ||
| 587 | * will not reprogram the hardware due to hang_detected being | ||
| 588 | * set. So we'd effectivly block all timers until the T2 event | ||
| 589 | * fires. | ||
| 590 | */ | ||
| 591 | if (cpu_base->hang_detected) | ||
| 592 | return; | ||
| 593 | |||
| 594 | if (cpu_base->expires_next.tv64 != KTIME_MAX) | ||
| 595 | tick_program_event(cpu_base->expires_next, 1); | ||
| 596 | } | ||
| 597 | |||
| 598 | /* | ||
| 599 | * Shared reprogramming for clock_realtime and clock_monotonic | ||
| 600 | * | ||
| 601 | * When a timer is enqueued and expires earlier than the already enqueued | ||
| 602 | * timers, we have to check, whether it expires earlier than the timer for | ||
| 603 | * which the clock event device was armed. | ||
| 604 | * | ||
| 605 | * Called with interrupts disabled and base->cpu_base.lock held | ||
| 606 | */ | ||
| 607 | static int hrtimer_reprogram(struct hrtimer *timer, | ||
| 608 | struct hrtimer_clock_base *base) | ||
| 609 | { | ||
| 610 | struct hrtimer_cpu_base *cpu_base = &__get_cpu_var(hrtimer_bases); | ||
| 611 | ktime_t expires = ktime_sub(hrtimer_get_expires(timer), base->offset); | ||
| 612 | int res; | ||
| 613 | |||
| 614 | WARN_ON_ONCE(hrtimer_get_expires_tv64(timer) < 0); | ||
| 615 | |||
| 616 | /* | ||
| 617 | * When the callback is running, we do not reprogram the clock event | ||
| 618 | * device. The timer callback is either running on a different CPU or | ||
| 619 | * the callback is executed in the hrtimer_interrupt context. The | ||
| 620 | * reprogramming is handled either by the softirq, which called the | ||
| 621 | * callback or at the end of the hrtimer_interrupt. | ||
| 622 | */ | ||
| 623 | if (hrtimer_callback_running(timer)) | ||
| 624 | return 0; | ||
| 625 | |||
| 626 | /* | ||
| 627 | * CLOCK_REALTIME timer might be requested with an absolute | ||
| 628 | * expiry time which is less than base->offset. Nothing wrong | ||
| 629 | * about that, just avoid to call into the tick code, which | ||
| 630 | * has now objections against negative expiry values. | ||
| 631 | */ | ||
| 632 | if (expires.tv64 < 0) | ||
| 633 | return -ETIME; | ||
| 634 | |||
| 635 | if (expires.tv64 >= cpu_base->expires_next.tv64) | ||
| 636 | return 0; | ||
| 637 | |||
| 638 | /* | ||
| 639 | * If a hang was detected in the last timer interrupt then we | ||
| 640 | * do not schedule a timer which is earlier than the expiry | ||
| 641 | * which we enforced in the hang detection. We want the system | ||
| 642 | * to make progress. | ||
| 643 | */ | ||
| 644 | if (cpu_base->hang_detected) | ||
| 645 | return 0; | ||
| 646 | |||
| 647 | /* | ||
| 648 | * Clockevents returns -ETIME, when the event was in the past. | ||
| 649 | */ | ||
| 650 | res = tick_program_event(expires, 0); | ||
| 651 | if (!IS_ERR_VALUE(res)) | ||
| 652 | cpu_base->expires_next = expires; | ||
| 653 | return res; | ||
| 654 | } | ||
| 655 | |||
| 656 | /* | ||
| 657 | * Initialize the high resolution related parts of cpu_base | ||
| 658 | */ | ||
| 659 | static inline void hrtimer_init_hres(struct hrtimer_cpu_base *base) | ||
| 660 | { | ||
| 661 | base->expires_next.tv64 = KTIME_MAX; | ||
| 662 | base->hres_active = 0; | ||
| 663 | } | ||
| 664 | |||
| 665 | /* | ||
| 666 | * When High resolution timers are active, try to reprogram. Note, that in case | ||
| 667 | * the state has HRTIMER_STATE_CALLBACK set, no reprogramming and no expiry | ||
| 668 | * check happens. The timer gets enqueued into the rbtree. The reprogramming | ||
| 669 | * and expiry check is done in the hrtimer_interrupt or in the softirq. | ||
| 670 | */ | ||
| 671 | static inline int hrtimer_enqueue_reprogram(struct hrtimer *timer, | ||
| 672 | struct hrtimer_clock_base *base) | ||
| 673 | { | ||
| 674 | return base->cpu_base->hres_active && hrtimer_reprogram(timer, base); | ||
| 675 | } | ||
| 676 | |||
| 677 | static inline ktime_t hrtimer_update_base(struct hrtimer_cpu_base *base) | ||
| 678 | { | ||
| 679 | ktime_t *offs_real = &base->clock_base[HRTIMER_BASE_REALTIME].offset; | ||
| 680 | ktime_t *offs_boot = &base->clock_base[HRTIMER_BASE_BOOTTIME].offset; | ||
| 681 | ktime_t *offs_tai = &base->clock_base[HRTIMER_BASE_TAI].offset; | ||
| 682 | |||
| 683 | return ktime_get_update_offsets(offs_real, offs_boot, offs_tai); | ||
| 684 | } | ||
| 685 | |||
| 686 | /* | ||
| 687 | * Retrigger next event is called after clock was set | ||
| 688 | * | ||
| 689 | * Called with interrupts disabled via on_each_cpu() | ||
| 690 | */ | ||
| 691 | static void retrigger_next_event(void *arg) | ||
| 692 | { | ||
| 693 | struct hrtimer_cpu_base *base = &__get_cpu_var(hrtimer_bases); | ||
| 694 | |||
| 695 | if (!hrtimer_hres_active()) | ||
| 696 | return; | ||
| 697 | |||
| 698 | raw_spin_lock(&base->lock); | ||
| 699 | hrtimer_update_base(base); | ||
| 700 | hrtimer_force_reprogram(base, 0); | ||
| 701 | raw_spin_unlock(&base->lock); | ||
| 702 | } | ||
| 703 | |||
| 704 | /* | ||
| 705 | * Switch to high resolution mode | ||
| 706 | */ | ||
| 707 | static int hrtimer_switch_to_hres(void) | ||
| 708 | { | ||
| 709 | int i, cpu = smp_processor_id(); | ||
| 710 | struct hrtimer_cpu_base *base = &per_cpu(hrtimer_bases, cpu); | ||
| 711 | unsigned long flags; | ||
| 712 | |||
| 713 | if (base->hres_active) | ||
| 714 | return 1; | ||
| 715 | |||
| 716 | local_irq_save(flags); | ||
| 717 | |||
| 718 | if (tick_init_highres()) { | ||
| 719 | local_irq_restore(flags); | ||
| 720 | printk(KERN_WARNING "Could not switch to high resolution " | ||
| 721 | "mode on CPU %d\n", cpu); | ||
| 722 | return 0; | ||
| 723 | } | ||
| 724 | base->hres_active = 1; | ||
| 725 | for (i = 0; i < HRTIMER_MAX_CLOCK_BASES; i++) | ||
| 726 | base->clock_base[i].resolution = KTIME_HIGH_RES; | ||
| 727 | |||
| 728 | tick_setup_sched_timer(); | ||
| 729 | /* "Retrigger" the interrupt to get things going */ | ||
| 730 | retrigger_next_event(NULL); | ||
| 731 | local_irq_restore(flags); | ||
| 732 | return 1; | ||
| 733 | } | ||
| 734 | |||
| 735 | static void clock_was_set_work(struct work_struct *work) | ||
| 736 | { | ||
| 737 | clock_was_set(); | ||
| 738 | } | ||
| 739 | |||
| 740 | static DECLARE_WORK(hrtimer_work, clock_was_set_work); | ||
| 741 | |||
| 742 | /* | ||
| 743 | * Called from timekeeping and resume code to reprogramm the hrtimer | ||
| 744 | * interrupt device on all cpus. | ||
| 745 | */ | ||
| 746 | void clock_was_set_delayed(void) | ||
| 747 | { | ||
| 748 | schedule_work(&hrtimer_work); | ||
| 749 | } | ||
| 750 | |||
| 751 | #else | ||
| 752 | |||
| 753 | static inline int hrtimer_hres_active(void) { return 0; } | ||
| 754 | static inline int hrtimer_is_hres_enabled(void) { return 0; } | ||
| 755 | static inline int hrtimer_switch_to_hres(void) { return 0; } | ||
| 756 | static inline void | ||
| 757 | hrtimer_force_reprogram(struct hrtimer_cpu_base *base, int skip_equal) { } | ||
| 758 | static inline int hrtimer_enqueue_reprogram(struct hrtimer *timer, | ||
| 759 | struct hrtimer_clock_base *base) | ||
| 760 | { | ||
| 761 | return 0; | ||
| 762 | } | ||
| 763 | static inline void hrtimer_init_hres(struct hrtimer_cpu_base *base) { } | ||
| 764 | static inline void retrigger_next_event(void *arg) { } | ||
| 765 | |||
| 766 | #endif /* CONFIG_HIGH_RES_TIMERS */ | ||
| 767 | |||
| 768 | /* | ||
| 769 | * Clock realtime was set | ||
| 770 | * | ||
| 771 | * Change the offset of the realtime clock vs. the monotonic | ||
| 772 | * clock. | ||
| 773 | * | ||
| 774 | * We might have to reprogram the high resolution timer interrupt. On | ||
| 775 | * SMP we call the architecture specific code to retrigger _all_ high | ||
| 776 | * resolution timer interrupts. On UP we just disable interrupts and | ||
| 777 | * call the high resolution interrupt code. | ||
| 778 | */ | ||
| 779 | void clock_was_set(void) | ||
| 780 | { | ||
| 781 | #ifdef CONFIG_HIGH_RES_TIMERS | ||
| 782 | /* Retrigger the CPU local events everywhere */ | ||
| 783 | on_each_cpu(retrigger_next_event, NULL, 1); | ||
| 784 | #endif | ||
| 785 | timerfd_clock_was_set(); | ||
| 786 | } | ||
| 787 | |||
| 788 | /* | ||
| 789 | * During resume we might have to reprogram the high resolution timer | ||
| 790 | * interrupt on all online CPUs. However, all other CPUs will be | ||
| 791 | * stopped with IRQs interrupts disabled so the clock_was_set() call | ||
| 792 | * must be deferred. | ||
| 793 | */ | ||
| 794 | void hrtimers_resume(void) | ||
| 795 | { | ||
| 796 | WARN_ONCE(!irqs_disabled(), | ||
| 797 | KERN_INFO "hrtimers_resume() called with IRQs enabled!"); | ||
| 798 | |||
| 799 | /* Retrigger on the local CPU */ | ||
| 800 | retrigger_next_event(NULL); | ||
| 801 | /* And schedule a retrigger for all others */ | ||
| 802 | clock_was_set_delayed(); | ||
| 803 | } | ||
| 804 | |||
| 805 | static inline void timer_stats_hrtimer_set_start_info(struct hrtimer *timer) | ||
| 806 | { | ||
| 807 | #ifdef CONFIG_TIMER_STATS | ||
| 808 | if (timer->start_site) | ||
| 809 | return; | ||
| 810 | timer->start_site = __builtin_return_address(0); | ||
| 811 | memcpy(timer->start_comm, current->comm, TASK_COMM_LEN); | ||
| 812 | timer->start_pid = current->pid; | ||
| 813 | #endif | ||
| 814 | } | ||
| 815 | |||
| 816 | static inline void timer_stats_hrtimer_clear_start_info(struct hrtimer *timer) | ||
| 817 | { | ||
| 818 | #ifdef CONFIG_TIMER_STATS | ||
| 819 | timer->start_site = NULL; | ||
| 820 | #endif | ||
| 821 | } | ||
| 822 | |||
| 823 | static inline void timer_stats_account_hrtimer(struct hrtimer *timer) | ||
| 824 | { | ||
| 825 | #ifdef CONFIG_TIMER_STATS | ||
| 826 | if (likely(!timer_stats_active)) | ||
| 827 | return; | ||
| 828 | timer_stats_update_stats(timer, timer->start_pid, timer->start_site, | ||
| 829 | timer->function, timer->start_comm, 0); | ||
| 830 | #endif | ||
| 831 | } | ||
| 832 | |||
| 833 | /* | ||
| 834 | * Counterpart to lock_hrtimer_base above: | ||
| 835 | */ | ||
| 836 | static inline | ||
| 837 | void unlock_hrtimer_base(const struct hrtimer *timer, unsigned long *flags) | ||
| 838 | { | ||
| 839 | raw_spin_unlock_irqrestore(&timer->base->cpu_base->lock, *flags); | ||
| 840 | } | ||
| 841 | |||
| 842 | /** | ||
| 843 | * hrtimer_forward - forward the timer expiry | ||
| 844 | * @timer: hrtimer to forward | ||
| 845 | * @now: forward past this time | ||
| 846 | * @interval: the interval to forward | ||
| 847 | * | ||
| 848 | * Forward the timer expiry so it will expire in the future. | ||
| 849 | * Returns the number of overruns. | ||
| 850 | */ | ||
| 851 | u64 hrtimer_forward(struct hrtimer *timer, ktime_t now, ktime_t interval) | ||
| 852 | { | ||
| 853 | u64 orun = 1; | ||
| 854 | ktime_t delta; | ||
| 855 | |||
| 856 | delta = ktime_sub(now, hrtimer_get_expires(timer)); | ||
| 857 | |||
| 858 | if (delta.tv64 < 0) | ||
| 859 | return 0; | ||
| 860 | |||
| 861 | if (interval.tv64 < timer->base->resolution.tv64) | ||
| 862 | interval.tv64 = timer->base->resolution.tv64; | ||
| 863 | |||
| 864 | if (unlikely(delta.tv64 >= interval.tv64)) { | ||
| 865 | s64 incr = ktime_to_ns(interval); | ||
| 866 | |||
| 867 | orun = ktime_divns(delta, incr); | ||
| 868 | hrtimer_add_expires_ns(timer, incr * orun); | ||
| 869 | if (hrtimer_get_expires_tv64(timer) > now.tv64) | ||
| 870 | return orun; | ||
| 871 | /* | ||
| 872 | * This (and the ktime_add() below) is the | ||
| 873 | * correction for exact: | ||
| 874 | */ | ||
| 875 | orun++; | ||
| 876 | } | ||
| 877 | hrtimer_add_expires(timer, interval); | ||
| 878 | |||
| 879 | return orun; | ||
| 880 | } | ||
| 881 | EXPORT_SYMBOL_GPL(hrtimer_forward); | ||
| 882 | |||
| 883 | /* | ||
| 884 | * enqueue_hrtimer - internal function to (re)start a timer | ||
| 885 | * | ||
| 886 | * The timer is inserted in expiry order. Insertion into the | ||
| 887 | * red black tree is O(log(n)). Must hold the base lock. | ||
| 888 | * | ||
| 889 | * Returns 1 when the new timer is the leftmost timer in the tree. | ||
| 890 | */ | ||
| 891 | static int enqueue_hrtimer(struct hrtimer *timer, | ||
| 892 | struct hrtimer_clock_base *base) | ||
| 893 | { | ||
| 894 | debug_activate(timer); | ||
| 895 | |||
| 896 | timerqueue_add(&base->active, &timer->node); | ||
| 897 | base->cpu_base->active_bases |= 1 << base->index; | ||
| 898 | |||
| 899 | /* | ||
| 900 | * HRTIMER_STATE_ENQUEUED is or'ed to the current state to preserve the | ||
| 901 | * state of a possibly running callback. | ||
| 902 | */ | ||
| 903 | timer->state |= HRTIMER_STATE_ENQUEUED; | ||
| 904 | |||
| 905 | return (&timer->node == base->active.next); | ||
| 906 | } | ||
| 907 | |||
| 908 | /* | ||
| 909 | * __remove_hrtimer - internal function to remove a timer | ||
| 910 | * | ||
| 911 | * Caller must hold the base lock. | ||
| 912 | * | ||
| 913 | * High resolution timer mode reprograms the clock event device when the | ||
| 914 | * timer is the one which expires next. The caller can disable this by setting | ||
| 915 | * reprogram to zero. This is useful, when the context does a reprogramming | ||
| 916 | * anyway (e.g. timer interrupt) | ||
| 917 | */ | ||
| 918 | static void __remove_hrtimer(struct hrtimer *timer, | ||
| 919 | struct hrtimer_clock_base *base, | ||
| 920 | unsigned long newstate, int reprogram) | ||
| 921 | { | ||
| 922 | struct timerqueue_node *next_timer; | ||
| 923 | if (!(timer->state & HRTIMER_STATE_ENQUEUED)) | ||
| 924 | goto out; | ||
| 925 | |||
| 926 | next_timer = timerqueue_getnext(&base->active); | ||
| 927 | timerqueue_del(&base->active, &timer->node); | ||
| 928 | if (&timer->node == next_timer) { | ||
| 929 | #ifdef CONFIG_HIGH_RES_TIMERS | ||
| 930 | /* Reprogram the clock event device. if enabled */ | ||
| 931 | if (reprogram && hrtimer_hres_active()) { | ||
| 932 | ktime_t expires; | ||
| 933 | |||
| 934 | expires = ktime_sub(hrtimer_get_expires(timer), | ||
| 935 | base->offset); | ||
| 936 | if (base->cpu_base->expires_next.tv64 == expires.tv64) | ||
| 937 | hrtimer_force_reprogram(base->cpu_base, 1); | ||
| 938 | } | ||
| 939 | #endif | ||
| 940 | } | ||
| 941 | if (!timerqueue_getnext(&base->active)) | ||
| 942 | base->cpu_base->active_bases &= ~(1 << base->index); | ||
| 943 | out: | ||
| 944 | timer->state = newstate; | ||
| 945 | } | ||
| 946 | |||
| 947 | /* | ||
| 948 | * remove hrtimer, called with base lock held | ||
| 949 | */ | ||
| 950 | static inline int | ||
| 951 | remove_hrtimer(struct hrtimer *timer, struct hrtimer_clock_base *base) | ||
| 952 | { | ||
| 953 | if (hrtimer_is_queued(timer)) { | ||
| 954 | unsigned long state; | ||
| 955 | int reprogram; | ||
| 956 | |||
| 957 | /* | ||
| 958 | * Remove the timer and force reprogramming when high | ||
| 959 | * resolution mode is active and the timer is on the current | ||
| 960 | * CPU. If we remove a timer on another CPU, reprogramming is | ||
| 961 | * skipped. The interrupt event on this CPU is fired and | ||
| 962 | * reprogramming happens in the interrupt handler. This is a | ||
| 963 | * rare case and less expensive than a smp call. | ||
| 964 | */ | ||
| 965 | debug_deactivate(timer); | ||
| 966 | timer_stats_hrtimer_clear_start_info(timer); | ||
| 967 | reprogram = base->cpu_base == &__get_cpu_var(hrtimer_bases); | ||
| 968 | /* | ||
| 969 | * We must preserve the CALLBACK state flag here, | ||
| 970 | * otherwise we could move the timer base in | ||
| 971 | * switch_hrtimer_base. | ||
| 972 | */ | ||
| 973 | state = timer->state & HRTIMER_STATE_CALLBACK; | ||
| 974 | __remove_hrtimer(timer, base, state, reprogram); | ||
| 975 | return 1; | ||
| 976 | } | ||
| 977 | return 0; | ||
| 978 | } | ||
| 979 | |||
| 980 | int __hrtimer_start_range_ns(struct hrtimer *timer, ktime_t tim, | ||
| 981 | unsigned long delta_ns, const enum hrtimer_mode mode, | ||
| 982 | int wakeup) | ||
| 983 | { | ||
| 984 | struct hrtimer_clock_base *base, *new_base; | ||
| 985 | unsigned long flags; | ||
| 986 | int ret, leftmost; | ||
| 987 | |||
| 988 | base = lock_hrtimer_base(timer, &flags); | ||
| 989 | |||
| 990 | /* Remove an active timer from the queue: */ | ||
| 991 | ret = remove_hrtimer(timer, base); | ||
| 992 | |||
| 993 | if (mode & HRTIMER_MODE_REL) { | ||
| 994 | tim = ktime_add_safe(tim, base->get_time()); | ||
| 995 | /* | ||
| 996 | * CONFIG_TIME_LOW_RES is a temporary way for architectures | ||
| 997 | * to signal that they simply return xtime in | ||
| 998 | * do_gettimeoffset(). In this case we want to round up by | ||
| 999 | * resolution when starting a relative timer, to avoid short | ||
| 1000 | * timeouts. This will go away with the GTOD framework. | ||
| 1001 | */ | ||
| 1002 | #ifdef CONFIG_TIME_LOW_RES | ||
| 1003 | tim = ktime_add_safe(tim, base->resolution); | ||
| 1004 | #endif | ||
| 1005 | } | ||
| 1006 | |||
| 1007 | hrtimer_set_expires_range_ns(timer, tim, delta_ns); | ||
| 1008 | |||
| 1009 | /* Switch the timer base, if necessary: */ | ||
| 1010 | new_base = switch_hrtimer_base(timer, base, mode & HRTIMER_MODE_PINNED); | ||
| 1011 | |||
| 1012 | timer_stats_hrtimer_set_start_info(timer); | ||
| 1013 | |||
| 1014 | leftmost = enqueue_hrtimer(timer, new_base); | ||
| 1015 | |||
| 1016 | /* | ||
| 1017 | * Only allow reprogramming if the new base is on this CPU. | ||
| 1018 | * (it might still be on another CPU if the timer was pending) | ||
| 1019 | * | ||
| 1020 | * XXX send_remote_softirq() ? | ||
| 1021 | */ | ||
| 1022 | if (leftmost && new_base->cpu_base == &__get_cpu_var(hrtimer_bases) | ||
| 1023 | && hrtimer_enqueue_reprogram(timer, new_base)) { | ||
| 1024 | if (wakeup) { | ||
| 1025 | /* | ||
| 1026 | * We need to drop cpu_base->lock to avoid a | ||
| 1027 | * lock ordering issue vs. rq->lock. | ||
| 1028 | */ | ||
| 1029 | raw_spin_unlock(&new_base->cpu_base->lock); | ||
| 1030 | raise_softirq_irqoff(HRTIMER_SOFTIRQ); | ||
| 1031 | local_irq_restore(flags); | ||
| 1032 | return ret; | ||
| 1033 | } else { | ||
| 1034 | __raise_softirq_irqoff(HRTIMER_SOFTIRQ); | ||
| 1035 | } | ||
| 1036 | } | ||
| 1037 | |||
| 1038 | unlock_hrtimer_base(timer, &flags); | ||
| 1039 | |||
| 1040 | return ret; | ||
| 1041 | } | ||
| 1042 | EXPORT_SYMBOL_GPL(__hrtimer_start_range_ns); | ||
| 1043 | |||
| 1044 | /** | ||
| 1045 | * hrtimer_start_range_ns - (re)start an hrtimer on the current CPU | ||
| 1046 | * @timer: the timer to be added | ||
| 1047 | * @tim: expiry time | ||
| 1048 | * @delta_ns: "slack" range for the timer | ||
| 1049 | * @mode: expiry mode: absolute (HRTIMER_MODE_ABS) or | ||
| 1050 | * relative (HRTIMER_MODE_REL) | ||
| 1051 | * | ||
| 1052 | * Returns: | ||
| 1053 | * 0 on success | ||
| 1054 | * 1 when the timer was active | ||
| 1055 | */ | ||
| 1056 | int hrtimer_start_range_ns(struct hrtimer *timer, ktime_t tim, | ||
| 1057 | unsigned long delta_ns, const enum hrtimer_mode mode) | ||
| 1058 | { | ||
| 1059 | return __hrtimer_start_range_ns(timer, tim, delta_ns, mode, 1); | ||
| 1060 | } | ||
| 1061 | EXPORT_SYMBOL_GPL(hrtimer_start_range_ns); | ||
| 1062 | |||
| 1063 | /** | ||
| 1064 | * hrtimer_start - (re)start an hrtimer on the current CPU | ||
| 1065 | * @timer: the timer to be added | ||
| 1066 | * @tim: expiry time | ||
| 1067 | * @mode: expiry mode: absolute (HRTIMER_MODE_ABS) or | ||
| 1068 | * relative (HRTIMER_MODE_REL) | ||
| 1069 | * | ||
| 1070 | * Returns: | ||
| 1071 | * 0 on success | ||
| 1072 | * 1 when the timer was active | ||
| 1073 | */ | ||
| 1074 | int | ||
| 1075 | hrtimer_start(struct hrtimer *timer, ktime_t tim, const enum hrtimer_mode mode) | ||
| 1076 | { | ||
| 1077 | return __hrtimer_start_range_ns(timer, tim, 0, mode, 1); | ||
| 1078 | } | ||
| 1079 | EXPORT_SYMBOL_GPL(hrtimer_start); | ||
| 1080 | |||
| 1081 | |||
| 1082 | /** | ||
| 1083 | * hrtimer_try_to_cancel - try to deactivate a timer | ||
| 1084 | * @timer: hrtimer to stop | ||
| 1085 | * | ||
| 1086 | * Returns: | ||
| 1087 | * 0 when the timer was not active | ||
| 1088 | * 1 when the timer was active | ||
| 1089 | * -1 when the timer is currently excuting the callback function and | ||
| 1090 | * cannot be stopped | ||
| 1091 | */ | ||
| 1092 | int hrtimer_try_to_cancel(struct hrtimer *timer) | ||
| 1093 | { | ||
| 1094 | struct hrtimer_clock_base *base; | ||
| 1095 | unsigned long flags; | ||
| 1096 | int ret = -1; | ||
| 1097 | |||
| 1098 | base = lock_hrtimer_base(timer, &flags); | ||
| 1099 | |||
| 1100 | if (!hrtimer_callback_running(timer)) | ||
| 1101 | ret = remove_hrtimer(timer, base); | ||
| 1102 | |||
| 1103 | unlock_hrtimer_base(timer, &flags); | ||
| 1104 | |||
| 1105 | return ret; | ||
| 1106 | |||
| 1107 | } | ||
| 1108 | EXPORT_SYMBOL_GPL(hrtimer_try_to_cancel); | ||
| 1109 | |||
| 1110 | /** | ||
| 1111 | * hrtimer_cancel - cancel a timer and wait for the handler to finish. | ||
| 1112 | * @timer: the timer to be cancelled | ||
| 1113 | * | ||
| 1114 | * Returns: | ||
| 1115 | * 0 when the timer was not active | ||
| 1116 | * 1 when the timer was active | ||
| 1117 | */ | ||
| 1118 | int hrtimer_cancel(struct hrtimer *timer) | ||
| 1119 | { | ||
| 1120 | for (;;) { | ||
| 1121 | int ret = hrtimer_try_to_cancel(timer); | ||
| 1122 | |||
| 1123 | if (ret >= 0) | ||
| 1124 | return ret; | ||
| 1125 | cpu_relax(); | ||
| 1126 | } | ||
| 1127 | } | ||
| 1128 | EXPORT_SYMBOL_GPL(hrtimer_cancel); | ||
| 1129 | |||
| 1130 | /** | ||
| 1131 | * hrtimer_get_remaining - get remaining time for the timer | ||
| 1132 | * @timer: the timer to read | ||
| 1133 | */ | ||
| 1134 | ktime_t hrtimer_get_remaining(const struct hrtimer *timer) | ||
| 1135 | { | ||
| 1136 | unsigned long flags; | ||
| 1137 | ktime_t rem; | ||
| 1138 | |||
| 1139 | lock_hrtimer_base(timer, &flags); | ||
| 1140 | rem = hrtimer_expires_remaining(timer); | ||
| 1141 | unlock_hrtimer_base(timer, &flags); | ||
| 1142 | |||
| 1143 | return rem; | ||
| 1144 | } | ||
| 1145 | EXPORT_SYMBOL_GPL(hrtimer_get_remaining); | ||
| 1146 | |||
| 1147 | #ifdef CONFIG_NO_HZ_COMMON | ||
| 1148 | /** | ||
| 1149 | * hrtimer_get_next_event - get the time until next expiry event | ||
| 1150 | * | ||
| 1151 | * Returns the delta to the next expiry event or KTIME_MAX if no timer | ||
| 1152 | * is pending. | ||
| 1153 | */ | ||
| 1154 | ktime_t hrtimer_get_next_event(void) | ||
| 1155 | { | ||
| 1156 | struct hrtimer_cpu_base *cpu_base = &__get_cpu_var(hrtimer_bases); | ||
| 1157 | struct hrtimer_clock_base *base = cpu_base->clock_base; | ||
| 1158 | ktime_t delta, mindelta = { .tv64 = KTIME_MAX }; | ||
| 1159 | unsigned long flags; | ||
| 1160 | int i; | ||
| 1161 | |||
| 1162 | raw_spin_lock_irqsave(&cpu_base->lock, flags); | ||
| 1163 | |||
| 1164 | if (!hrtimer_hres_active()) { | ||
| 1165 | for (i = 0; i < HRTIMER_MAX_CLOCK_BASES; i++, base++) { | ||
| 1166 | struct hrtimer *timer; | ||
| 1167 | struct timerqueue_node *next; | ||
| 1168 | |||
| 1169 | next = timerqueue_getnext(&base->active); | ||
| 1170 | if (!next) | ||
| 1171 | continue; | ||
| 1172 | |||
| 1173 | timer = container_of(next, struct hrtimer, node); | ||
| 1174 | delta.tv64 = hrtimer_get_expires_tv64(timer); | ||
| 1175 | delta = ktime_sub(delta, base->get_time()); | ||
| 1176 | if (delta.tv64 < mindelta.tv64) | ||
| 1177 | mindelta.tv64 = delta.tv64; | ||
| 1178 | } | ||
| 1179 | } | ||
| 1180 | |||
| 1181 | raw_spin_unlock_irqrestore(&cpu_base->lock, flags); | ||
| 1182 | |||
| 1183 | if (mindelta.tv64 < 0) | ||
| 1184 | mindelta.tv64 = 0; | ||
| 1185 | return mindelta; | ||
| 1186 | } | ||
| 1187 | #endif | ||
| 1188 | |||
| 1189 | static void __hrtimer_init(struct hrtimer *timer, clockid_t clock_id, | ||
| 1190 | enum hrtimer_mode mode) | ||
| 1191 | { | ||
| 1192 | struct hrtimer_cpu_base *cpu_base; | ||
| 1193 | int base; | ||
| 1194 | |||
| 1195 | memset(timer, 0, sizeof(struct hrtimer)); | ||
| 1196 | |||
| 1197 | cpu_base = &__raw_get_cpu_var(hrtimer_bases); | ||
| 1198 | |||
| 1199 | if (clock_id == CLOCK_REALTIME && mode != HRTIMER_MODE_ABS) | ||
| 1200 | clock_id = CLOCK_MONOTONIC; | ||
| 1201 | |||
| 1202 | base = hrtimer_clockid_to_base(clock_id); | ||
| 1203 | timer->base = &cpu_base->clock_base[base]; | ||
| 1204 | timerqueue_init(&timer->node); | ||
| 1205 | |||
| 1206 | #ifdef CONFIG_TIMER_STATS | ||
| 1207 | timer->start_site = NULL; | ||
| 1208 | timer->start_pid = -1; | ||
| 1209 | memset(timer->start_comm, 0, TASK_COMM_LEN); | ||
| 1210 | #endif | ||
| 1211 | } | ||
| 1212 | |||
| 1213 | /** | ||
| 1214 | * hrtimer_init - initialize a timer to the given clock | ||
| 1215 | * @timer: the timer to be initialized | ||
| 1216 | * @clock_id: the clock to be used | ||
| 1217 | * @mode: timer mode abs/rel | ||
| 1218 | */ | ||
| 1219 | void hrtimer_init(struct hrtimer *timer, clockid_t clock_id, | ||
| 1220 | enum hrtimer_mode mode) | ||
| 1221 | { | ||
| 1222 | debug_init(timer, clock_id, mode); | ||
| 1223 | __hrtimer_init(timer, clock_id, mode); | ||
| 1224 | } | ||
| 1225 | EXPORT_SYMBOL_GPL(hrtimer_init); | ||
| 1226 | |||
| 1227 | /** | ||
| 1228 | * hrtimer_get_res - get the timer resolution for a clock | ||
| 1229 | * @which_clock: which clock to query | ||
| 1230 | * @tp: pointer to timespec variable to store the resolution | ||
| 1231 | * | ||
| 1232 | * Store the resolution of the clock selected by @which_clock in the | ||
| 1233 | * variable pointed to by @tp. | ||
| 1234 | */ | ||
| 1235 | int hrtimer_get_res(const clockid_t which_clock, struct timespec *tp) | ||
| 1236 | { | ||
| 1237 | struct hrtimer_cpu_base *cpu_base; | ||
| 1238 | int base = hrtimer_clockid_to_base(which_clock); | ||
| 1239 | |||
| 1240 | cpu_base = &__raw_get_cpu_var(hrtimer_bases); | ||
| 1241 | *tp = ktime_to_timespec(cpu_base->clock_base[base].resolution); | ||
| 1242 | |||
| 1243 | return 0; | ||
| 1244 | } | ||
| 1245 | EXPORT_SYMBOL_GPL(hrtimer_get_res); | ||
| 1246 | |||
| 1247 | static void __run_hrtimer(struct hrtimer *timer, ktime_t *now) | ||
| 1248 | { | ||
| 1249 | struct hrtimer_clock_base *base = timer->base; | ||
| 1250 | struct hrtimer_cpu_base *cpu_base = base->cpu_base; | ||
| 1251 | enum hrtimer_restart (*fn)(struct hrtimer *); | ||
| 1252 | int restart; | ||
| 1253 | |||
| 1254 | WARN_ON(!irqs_disabled()); | ||
| 1255 | |||
| 1256 | debug_deactivate(timer); | ||
| 1257 | __remove_hrtimer(timer, base, HRTIMER_STATE_CALLBACK, 0); | ||
| 1258 | timer_stats_account_hrtimer(timer); | ||
| 1259 | fn = timer->function; | ||
| 1260 | |||
| 1261 | /* | ||
| 1262 | * Because we run timers from hardirq context, there is no chance | ||
| 1263 | * they get migrated to another cpu, therefore its safe to unlock | ||
| 1264 | * the timer base. | ||
| 1265 | */ | ||
| 1266 | raw_spin_unlock(&cpu_base->lock); | ||
| 1267 | trace_hrtimer_expire_entry(timer, now); | ||
| 1268 | restart = fn(timer); | ||
| 1269 | trace_hrtimer_expire_exit(timer); | ||
| 1270 | raw_spin_lock(&cpu_base->lock); | ||
| 1271 | |||
| 1272 | /* | ||
| 1273 | * Note: We clear the CALLBACK bit after enqueue_hrtimer and | ||
| 1274 | * we do not reprogramm the event hardware. Happens either in | ||
| 1275 | * hrtimer_start_range_ns() or in hrtimer_interrupt() | ||
| 1276 | */ | ||
| 1277 | if (restart != HRTIMER_NORESTART) { | ||
| 1278 | BUG_ON(timer->state != HRTIMER_STATE_CALLBACK); | ||
| 1279 | enqueue_hrtimer(timer, base); | ||
| 1280 | } | ||
| 1281 | |||
| 1282 | WARN_ON_ONCE(!(timer->state & HRTIMER_STATE_CALLBACK)); | ||
| 1283 | |||
| 1284 | timer->state &= ~HRTIMER_STATE_CALLBACK; | ||
| 1285 | } | ||
| 1286 | |||
| 1287 | #ifdef CONFIG_HIGH_RES_TIMERS | ||
| 1288 | |||
| 1289 | /* | ||
| 1290 | * High resolution timer interrupt | ||
| 1291 | * Called with interrupts disabled | ||
| 1292 | */ | ||
| 1293 | void hrtimer_interrupt(struct clock_event_device *dev) | ||
| 1294 | { | ||
| 1295 | struct hrtimer_cpu_base *cpu_base = &__get_cpu_var(hrtimer_bases); | ||
| 1296 | ktime_t expires_next, now, entry_time, delta; | ||
| 1297 | int i, retries = 0; | ||
| 1298 | |||
| 1299 | BUG_ON(!cpu_base->hres_active); | ||
| 1300 | cpu_base->nr_events++; | ||
| 1301 | dev->next_event.tv64 = KTIME_MAX; | ||
| 1302 | |||
| 1303 | raw_spin_lock(&cpu_base->lock); | ||
| 1304 | entry_time = now = hrtimer_update_base(cpu_base); | ||
| 1305 | retry: | ||
| 1306 | expires_next.tv64 = KTIME_MAX; | ||
| 1307 | /* | ||
| 1308 | * We set expires_next to KTIME_MAX here with cpu_base->lock | ||
| 1309 | * held to prevent that a timer is enqueued in our queue via | ||
| 1310 | * the migration code. This does not affect enqueueing of | ||
| 1311 | * timers which run their callback and need to be requeued on | ||
| 1312 | * this CPU. | ||
| 1313 | */ | ||
| 1314 | cpu_base->expires_next.tv64 = KTIME_MAX; | ||
| 1315 | |||
| 1316 | for (i = 0; i < HRTIMER_MAX_CLOCK_BASES; i++) { | ||
| 1317 | struct hrtimer_clock_base *base; | ||
| 1318 | struct timerqueue_node *node; | ||
| 1319 | ktime_t basenow; | ||
| 1320 | |||
| 1321 | if (!(cpu_base->active_bases & (1 << i))) | ||
| 1322 | continue; | ||
| 1323 | |||
| 1324 | base = cpu_base->clock_base + i; | ||
| 1325 | basenow = ktime_add(now, base->offset); | ||
| 1326 | |||
| 1327 | while ((node = timerqueue_getnext(&base->active))) { | ||
| 1328 | struct hrtimer *timer; | ||
| 1329 | |||
| 1330 | timer = container_of(node, struct hrtimer, node); | ||
| 1331 | |||
| 1332 | /* | ||
| 1333 | * The immediate goal for using the softexpires is | ||
| 1334 | * minimizing wakeups, not running timers at the | ||
| 1335 | * earliest interrupt after their soft expiration. | ||
| 1336 | * This allows us to avoid using a Priority Search | ||
| 1337 | * Tree, which can answer a stabbing querry for | ||
| 1338 | * overlapping intervals and instead use the simple | ||
| 1339 | * BST we already have. | ||
| 1340 | * We don't add extra wakeups by delaying timers that | ||
| 1341 | * are right-of a not yet expired timer, because that | ||
| 1342 | * timer will have to trigger a wakeup anyway. | ||
| 1343 | */ | ||
| 1344 | |||
| 1345 | if (basenow.tv64 < hrtimer_get_softexpires_tv64(timer)) { | ||
| 1346 | ktime_t expires; | ||
| 1347 | |||
| 1348 | expires = ktime_sub(hrtimer_get_expires(timer), | ||
| 1349 | base->offset); | ||
| 1350 | if (expires.tv64 < 0) | ||
| 1351 | expires.tv64 = KTIME_MAX; | ||
| 1352 | if (expires.tv64 < expires_next.tv64) | ||
| 1353 | expires_next = expires; | ||
| 1354 | break; | ||
| 1355 | } | ||
| 1356 | |||
| 1357 | __run_hrtimer(timer, &basenow); | ||
| 1358 | } | ||
| 1359 | } | ||
| 1360 | |||
| 1361 | /* | ||
| 1362 | * Store the new expiry value so the migration code can verify | ||
| 1363 | * against it. | ||
| 1364 | */ | ||
| 1365 | cpu_base->expires_next = expires_next; | ||
| 1366 | raw_spin_unlock(&cpu_base->lock); | ||
| 1367 | |||
| 1368 | /* Reprogramming necessary ? */ | ||
| 1369 | if (expires_next.tv64 == KTIME_MAX || | ||
| 1370 | !tick_program_event(expires_next, 0)) { | ||
| 1371 | cpu_base->hang_detected = 0; | ||
| 1372 | return; | ||
| 1373 | } | ||
| 1374 | |||
| 1375 | /* | ||
| 1376 | * The next timer was already expired due to: | ||
| 1377 | * - tracing | ||
| 1378 | * - long lasting callbacks | ||
| 1379 | * - being scheduled away when running in a VM | ||
| 1380 | * | ||
| 1381 | * We need to prevent that we loop forever in the hrtimer | ||
| 1382 | * interrupt routine. We give it 3 attempts to avoid | ||
| 1383 | * overreacting on some spurious event. | ||
| 1384 | * | ||
| 1385 | * Acquire base lock for updating the offsets and retrieving | ||
| 1386 | * the current time. | ||
| 1387 | */ | ||
| 1388 | raw_spin_lock(&cpu_base->lock); | ||
| 1389 | now = hrtimer_update_base(cpu_base); | ||
| 1390 | cpu_base->nr_retries++; | ||
| 1391 | if (++retries < 3) | ||
| 1392 | goto retry; | ||
| 1393 | /* | ||
| 1394 | * Give the system a chance to do something else than looping | ||
| 1395 | * here. We stored the entry time, so we know exactly how long | ||
| 1396 | * we spent here. We schedule the next event this amount of | ||
| 1397 | * time away. | ||
| 1398 | */ | ||
| 1399 | cpu_base->nr_hangs++; | ||
| 1400 | cpu_base->hang_detected = 1; | ||
| 1401 | raw_spin_unlock(&cpu_base->lock); | ||
| 1402 | delta = ktime_sub(now, entry_time); | ||
| 1403 | if (delta.tv64 > cpu_base->max_hang_time.tv64) | ||
| 1404 | cpu_base->max_hang_time = delta; | ||
| 1405 | /* | ||
| 1406 | * Limit it to a sensible value as we enforce a longer | ||
| 1407 | * delay. Give the CPU at least 100ms to catch up. | ||
| 1408 | */ | ||
| 1409 | if (delta.tv64 > 100 * NSEC_PER_MSEC) | ||
| 1410 | expires_next = ktime_add_ns(now, 100 * NSEC_PER_MSEC); | ||
| 1411 | else | ||
| 1412 | expires_next = ktime_add(now, delta); | ||
| 1413 | tick_program_event(expires_next, 1); | ||
| 1414 | printk_once(KERN_WARNING "hrtimer: interrupt took %llu ns\n", | ||
| 1415 | ktime_to_ns(delta)); | ||
| 1416 | } | ||
| 1417 | |||
| 1418 | /* | ||
| 1419 | * local version of hrtimer_peek_ahead_timers() called with interrupts | ||
| 1420 | * disabled. | ||
| 1421 | */ | ||
| 1422 | static void __hrtimer_peek_ahead_timers(void) | ||
| 1423 | { | ||
| 1424 | struct tick_device *td; | ||
| 1425 | |||
| 1426 | if (!hrtimer_hres_active()) | ||
| 1427 | return; | ||
| 1428 | |||
| 1429 | td = &__get_cpu_var(tick_cpu_device); | ||
| 1430 | if (td && td->evtdev) | ||
| 1431 | hrtimer_interrupt(td->evtdev); | ||
| 1432 | } | ||
| 1433 | |||
| 1434 | /** | ||
| 1435 | * hrtimer_peek_ahead_timers -- run soft-expired timers now | ||
| 1436 | * | ||
| 1437 | * hrtimer_peek_ahead_timers will peek at the timer queue of | ||
| 1438 | * the current cpu and check if there are any timers for which | ||
| 1439 | * the soft expires time has passed. If any such timers exist, | ||
| 1440 | * they are run immediately and then removed from the timer queue. | ||
| 1441 | * | ||
| 1442 | */ | ||
| 1443 | void hrtimer_peek_ahead_timers(void) | ||
| 1444 | { | ||
| 1445 | unsigned long flags; | ||
| 1446 | |||
| 1447 | local_irq_save(flags); | ||
| 1448 | __hrtimer_peek_ahead_timers(); | ||
| 1449 | local_irq_restore(flags); | ||
| 1450 | } | ||
| 1451 | |||
| 1452 | static void run_hrtimer_softirq(struct softirq_action *h) | ||
| 1453 | { | ||
| 1454 | hrtimer_peek_ahead_timers(); | ||
| 1455 | } | ||
| 1456 | |||
| 1457 | #else /* CONFIG_HIGH_RES_TIMERS */ | ||
| 1458 | |||
| 1459 | static inline void __hrtimer_peek_ahead_timers(void) { } | ||
| 1460 | |||
| 1461 | #endif /* !CONFIG_HIGH_RES_TIMERS */ | ||
| 1462 | |||
| 1463 | /* | ||
| 1464 | * Called from timer softirq every jiffy, expire hrtimers: | ||
| 1465 | * | ||
| 1466 | * For HRT its the fall back code to run the softirq in the timer | ||
| 1467 | * softirq context in case the hrtimer initialization failed or has | ||
| 1468 | * not been done yet. | ||
| 1469 | */ | ||
| 1470 | void hrtimer_run_pending(void) | ||
| 1471 | { | ||
| 1472 | if (hrtimer_hres_active()) | ||
| 1473 | return; | ||
| 1474 | |||
| 1475 | /* | ||
| 1476 | * This _is_ ugly: We have to check in the softirq context, | ||
| 1477 | * whether we can switch to highres and / or nohz mode. The | ||
| 1478 | * clocksource switch happens in the timer interrupt with | ||
| 1479 | * xtime_lock held. Notification from there only sets the | ||
| 1480 | * check bit in the tick_oneshot code, otherwise we might | ||
| 1481 | * deadlock vs. xtime_lock. | ||
| 1482 | */ | ||
| 1483 | if (tick_check_oneshot_change(!hrtimer_is_hres_enabled())) | ||
| 1484 | hrtimer_switch_to_hres(); | ||
| 1485 | } | ||
| 1486 | |||
| 1487 | /* | ||
| 1488 | * Called from hardirq context every jiffy | ||
| 1489 | */ | ||
| 1490 | void hrtimer_run_queues(void) | ||
| 1491 | { | ||
| 1492 | struct timerqueue_node *node; | ||
| 1493 | struct hrtimer_cpu_base *cpu_base = &__get_cpu_var(hrtimer_bases); | ||
| 1494 | struct hrtimer_clock_base *base; | ||
| 1495 | int index, gettime = 1; | ||
| 1496 | |||
| 1497 | if (hrtimer_hres_active()) | ||
| 1498 | return; | ||
| 1499 | |||
| 1500 | for (index = 0; index < HRTIMER_MAX_CLOCK_BASES; index++) { | ||
| 1501 | base = &cpu_base->clock_base[index]; | ||
| 1502 | if (!timerqueue_getnext(&base->active)) | ||
| 1503 | continue; | ||
| 1504 | |||
| 1505 | if (gettime) { | ||
| 1506 | hrtimer_get_softirq_time(cpu_base); | ||
| 1507 | gettime = 0; | ||
| 1508 | } | ||
| 1509 | |||
| 1510 | raw_spin_lock(&cpu_base->lock); | ||
| 1511 | |||
| 1512 | while ((node = timerqueue_getnext(&base->active))) { | ||
| 1513 | struct hrtimer *timer; | ||
| 1514 | |||
| 1515 | timer = container_of(node, struct hrtimer, node); | ||
| 1516 | if (base->softirq_time.tv64 <= | ||
| 1517 | hrtimer_get_expires_tv64(timer)) | ||
| 1518 | break; | ||
| 1519 | |||
| 1520 | __run_hrtimer(timer, &base->softirq_time); | ||
| 1521 | } | ||
| 1522 | raw_spin_unlock(&cpu_base->lock); | ||
| 1523 | } | ||
| 1524 | } | ||
| 1525 | |||
| 1526 | /* | ||
| 1527 | * Sleep related functions: | ||
| 1528 | */ | ||
| 1529 | static enum hrtimer_restart hrtimer_wakeup(struct hrtimer *timer) | ||
| 1530 | { | ||
| 1531 | struct hrtimer_sleeper *t = | ||
| 1532 | container_of(timer, struct hrtimer_sleeper, timer); | ||
| 1533 | struct task_struct *task = t->task; | ||
| 1534 | |||
| 1535 | t->task = NULL; | ||
| 1536 | if (task) | ||
| 1537 | wake_up_process(task); | ||
| 1538 | |||
| 1539 | return HRTIMER_NORESTART; | ||
| 1540 | } | ||
| 1541 | |||
| 1542 | void hrtimer_init_sleeper(struct hrtimer_sleeper *sl, struct task_struct *task) | ||
| 1543 | { | ||
| 1544 | sl->timer.function = hrtimer_wakeup; | ||
| 1545 | sl->task = task; | ||
| 1546 | } | ||
| 1547 | EXPORT_SYMBOL_GPL(hrtimer_init_sleeper); | ||
| 1548 | |||
| 1549 | static int __sched do_nanosleep(struct hrtimer_sleeper *t, enum hrtimer_mode mode) | ||
| 1550 | { | ||
| 1551 | hrtimer_init_sleeper(t, current); | ||
| 1552 | |||
| 1553 | do { | ||
| 1554 | set_current_state(TASK_INTERRUPTIBLE); | ||
| 1555 | hrtimer_start_expires(&t->timer, mode); | ||
| 1556 | if (!hrtimer_active(&t->timer)) | ||
| 1557 | t->task = NULL; | ||
| 1558 | |||
| 1559 | if (likely(t->task)) | ||
| 1560 | freezable_schedule(); | ||
| 1561 | |||
| 1562 | hrtimer_cancel(&t->timer); | ||
| 1563 | mode = HRTIMER_MODE_ABS; | ||
| 1564 | |||
| 1565 | } while (t->task && !signal_pending(current)); | ||
| 1566 | |||
| 1567 | __set_current_state(TASK_RUNNING); | ||
| 1568 | |||
| 1569 | return t->task == NULL; | ||
| 1570 | } | ||
| 1571 | |||
| 1572 | static int update_rmtp(struct hrtimer *timer, struct timespec __user *rmtp) | ||
| 1573 | { | ||
| 1574 | struct timespec rmt; | ||
| 1575 | ktime_t rem; | ||
| 1576 | |||
| 1577 | rem = hrtimer_expires_remaining(timer); | ||
| 1578 | if (rem.tv64 <= 0) | ||
| 1579 | return 0; | ||
| 1580 | rmt = ktime_to_timespec(rem); | ||
| 1581 | |||
| 1582 | if (copy_to_user(rmtp, &rmt, sizeof(*rmtp))) | ||
| 1583 | return -EFAULT; | ||
| 1584 | |||
| 1585 | return 1; | ||
| 1586 | } | ||
| 1587 | |||
| 1588 | long __sched hrtimer_nanosleep_restart(struct restart_block *restart) | ||
| 1589 | { | ||
| 1590 | struct hrtimer_sleeper t; | ||
| 1591 | struct timespec __user *rmtp; | ||
| 1592 | int ret = 0; | ||
| 1593 | |||
| 1594 | hrtimer_init_on_stack(&t.timer, restart->nanosleep.clockid, | ||
| 1595 | HRTIMER_MODE_ABS); | ||
| 1596 | hrtimer_set_expires_tv64(&t.timer, restart->nanosleep.expires); | ||
| 1597 | |||
| 1598 | if (do_nanosleep(&t, HRTIMER_MODE_ABS)) | ||
| 1599 | goto out; | ||
| 1600 | |||
| 1601 | rmtp = restart->nanosleep.rmtp; | ||
| 1602 | if (rmtp) { | ||
| 1603 | ret = update_rmtp(&t.timer, rmtp); | ||
| 1604 | if (ret <= 0) | ||
| 1605 | goto out; | ||
| 1606 | } | ||
| 1607 | |||
| 1608 | /* The other values in restart are already filled in */ | ||
| 1609 | ret = -ERESTART_RESTARTBLOCK; | ||
| 1610 | out: | ||
| 1611 | destroy_hrtimer_on_stack(&t.timer); | ||
| 1612 | return ret; | ||
| 1613 | } | ||
| 1614 | |||
| 1615 | long hrtimer_nanosleep(struct timespec *rqtp, struct timespec __user *rmtp, | ||
| 1616 | const enum hrtimer_mode mode, const clockid_t clockid) | ||
| 1617 | { | ||
| 1618 | struct restart_block *restart; | ||
| 1619 | struct hrtimer_sleeper t; | ||
| 1620 | int ret = 0; | ||
| 1621 | unsigned long slack; | ||
| 1622 | |||
| 1623 | slack = current->timer_slack_ns; | ||
| 1624 | if (dl_task(current) || rt_task(current)) | ||
| 1625 | slack = 0; | ||
| 1626 | |||
| 1627 | hrtimer_init_on_stack(&t.timer, clockid, mode); | ||
| 1628 | hrtimer_set_expires_range_ns(&t.timer, timespec_to_ktime(*rqtp), slack); | ||
| 1629 | if (do_nanosleep(&t, mode)) | ||
| 1630 | goto out; | ||
| 1631 | |||
| 1632 | /* Absolute timers do not update the rmtp value and restart: */ | ||
| 1633 | if (mode == HRTIMER_MODE_ABS) { | ||
| 1634 | ret = -ERESTARTNOHAND; | ||
| 1635 | goto out; | ||
| 1636 | } | ||
| 1637 | |||
| 1638 | if (rmtp) { | ||
| 1639 | ret = update_rmtp(&t.timer, rmtp); | ||
| 1640 | if (ret <= 0) | ||
| 1641 | goto out; | ||
| 1642 | } | ||
| 1643 | |||
| 1644 | restart = ¤t_thread_info()->restart_block; | ||
| 1645 | restart->fn = hrtimer_nanosleep_restart; | ||
| 1646 | restart->nanosleep.clockid = t.timer.base->clockid; | ||
| 1647 | restart->nanosleep.rmtp = rmtp; | ||
| 1648 | restart->nanosleep.expires = hrtimer_get_expires_tv64(&t.timer); | ||
| 1649 | |||
| 1650 | ret = -ERESTART_RESTARTBLOCK; | ||
| 1651 | out: | ||
| 1652 | destroy_hrtimer_on_stack(&t.timer); | ||
| 1653 | return ret; | ||
| 1654 | } | ||
| 1655 | |||
| 1656 | SYSCALL_DEFINE2(nanosleep, struct timespec __user *, rqtp, | ||
| 1657 | struct timespec __user *, rmtp) | ||
| 1658 | { | ||
| 1659 | struct timespec tu; | ||
| 1660 | |||
| 1661 | if (copy_from_user(&tu, rqtp, sizeof(tu))) | ||
| 1662 | return -EFAULT; | ||
| 1663 | |||
| 1664 | if (!timespec_valid(&tu)) | ||
| 1665 | return -EINVAL; | ||
| 1666 | |||
| 1667 | return hrtimer_nanosleep(&tu, rmtp, HRTIMER_MODE_REL, CLOCK_MONOTONIC); | ||
| 1668 | } | ||
| 1669 | |||
| 1670 | /* | ||
| 1671 | * Functions related to boot-time initialization: | ||
| 1672 | */ | ||
| 1673 | static void init_hrtimers_cpu(int cpu) | ||
| 1674 | { | ||
| 1675 | struct hrtimer_cpu_base *cpu_base = &per_cpu(hrtimer_bases, cpu); | ||
| 1676 | int i; | ||
| 1677 | |||
| 1678 | for (i = 0; i < HRTIMER_MAX_CLOCK_BASES; i++) { | ||
| 1679 | cpu_base->clock_base[i].cpu_base = cpu_base; | ||
| 1680 | timerqueue_init_head(&cpu_base->clock_base[i].active); | ||
| 1681 | } | ||
| 1682 | |||
| 1683 | hrtimer_init_hres(cpu_base); | ||
| 1684 | } | ||
| 1685 | |||
| 1686 | #ifdef CONFIG_HOTPLUG_CPU | ||
| 1687 | |||
| 1688 | static void migrate_hrtimer_list(struct hrtimer_clock_base *old_base, | ||
| 1689 | struct hrtimer_clock_base *new_base) | ||
| 1690 | { | ||
| 1691 | struct hrtimer *timer; | ||
| 1692 | struct timerqueue_node *node; | ||
| 1693 | |||
| 1694 | while ((node = timerqueue_getnext(&old_base->active))) { | ||
| 1695 | timer = container_of(node, struct hrtimer, node); | ||
| 1696 | BUG_ON(hrtimer_callback_running(timer)); | ||
| 1697 | debug_deactivate(timer); | ||
| 1698 | |||
| 1699 | /* | ||
| 1700 | * Mark it as STATE_MIGRATE not INACTIVE otherwise the | ||
| 1701 | * timer could be seen as !active and just vanish away | ||
| 1702 | * under us on another CPU | ||
| 1703 | */ | ||
| 1704 | __remove_hrtimer(timer, old_base, HRTIMER_STATE_MIGRATE, 0); | ||
| 1705 | timer->base = new_base; | ||
| 1706 | /* | ||
| 1707 | * Enqueue the timers on the new cpu. This does not | ||
| 1708 | * reprogram the event device in case the timer | ||
| 1709 | * expires before the earliest on this CPU, but we run | ||
| 1710 | * hrtimer_interrupt after we migrated everything to | ||
| 1711 | * sort out already expired timers and reprogram the | ||
| 1712 | * event device. | ||
| 1713 | */ | ||
| 1714 | enqueue_hrtimer(timer, new_base); | ||
| 1715 | |||
| 1716 | /* Clear the migration state bit */ | ||
| 1717 | timer->state &= ~HRTIMER_STATE_MIGRATE; | ||
| 1718 | } | ||
| 1719 | } | ||
| 1720 | |||
| 1721 | static void migrate_hrtimers(int scpu) | ||
| 1722 | { | ||
| 1723 | struct hrtimer_cpu_base *old_base, *new_base; | ||
| 1724 | int i; | ||
| 1725 | |||
| 1726 | BUG_ON(cpu_online(scpu)); | ||
| 1727 | tick_cancel_sched_timer(scpu); | ||
| 1728 | |||
| 1729 | local_irq_disable(); | ||
| 1730 | old_base = &per_cpu(hrtimer_bases, scpu); | ||
| 1731 | new_base = &__get_cpu_var(hrtimer_bases); | ||
| 1732 | /* | ||
| 1733 | * The caller is globally serialized and nobody else | ||
| 1734 | * takes two locks at once, deadlock is not possible. | ||
| 1735 | */ | ||
| 1736 | raw_spin_lock(&new_base->lock); | ||
| 1737 | raw_spin_lock_nested(&old_base->lock, SINGLE_DEPTH_NESTING); | ||
| 1738 | |||
| 1739 | for (i = 0; i < HRTIMER_MAX_CLOCK_BASES; i++) { | ||
| 1740 | migrate_hrtimer_list(&old_base->clock_base[i], | ||
| 1741 | &new_base->clock_base[i]); | ||
| 1742 | } | ||
| 1743 | |||
| 1744 | raw_spin_unlock(&old_base->lock); | ||
| 1745 | raw_spin_unlock(&new_base->lock); | ||
| 1746 | |||
| 1747 | /* Check, if we got expired work to do */ | ||
| 1748 | __hrtimer_peek_ahead_timers(); | ||
| 1749 | local_irq_enable(); | ||
| 1750 | } | ||
| 1751 | |||
| 1752 | #endif /* CONFIG_HOTPLUG_CPU */ | ||
| 1753 | |||
| 1754 | static int hrtimer_cpu_notify(struct notifier_block *self, | ||
| 1755 | unsigned long action, void *hcpu) | ||
| 1756 | { | ||
| 1757 | int scpu = (long)hcpu; | ||
| 1758 | |||
| 1759 | switch (action) { | ||
| 1760 | |||
| 1761 | case CPU_UP_PREPARE: | ||
| 1762 | case CPU_UP_PREPARE_FROZEN: | ||
| 1763 | init_hrtimers_cpu(scpu); | ||
| 1764 | break; | ||
| 1765 | |||
| 1766 | #ifdef CONFIG_HOTPLUG_CPU | ||
| 1767 | case CPU_DYING: | ||
| 1768 | case CPU_DYING_FROZEN: | ||
| 1769 | clockevents_notify(CLOCK_EVT_NOTIFY_CPU_DYING, &scpu); | ||
| 1770 | break; | ||
| 1771 | case CPU_DEAD: | ||
| 1772 | case CPU_DEAD_FROZEN: | ||
| 1773 | { | ||
| 1774 | clockevents_notify(CLOCK_EVT_NOTIFY_CPU_DEAD, &scpu); | ||
| 1775 | migrate_hrtimers(scpu); | ||
| 1776 | break; | ||
| 1777 | } | ||
| 1778 | #endif | ||
| 1779 | |||
| 1780 | default: | ||
| 1781 | break; | ||
| 1782 | } | ||
| 1783 | |||
| 1784 | return NOTIFY_OK; | ||
| 1785 | } | ||
| 1786 | |||
| 1787 | static struct notifier_block hrtimers_nb = { | ||
| 1788 | .notifier_call = hrtimer_cpu_notify, | ||
| 1789 | }; | ||
| 1790 | |||
| 1791 | void __init hrtimers_init(void) | ||
| 1792 | { | ||
| 1793 | hrtimer_cpu_notify(&hrtimers_nb, (unsigned long)CPU_UP_PREPARE, | ||
| 1794 | (void *)(long)smp_processor_id()); | ||
| 1795 | register_cpu_notifier(&hrtimers_nb); | ||
| 1796 | #ifdef CONFIG_HIGH_RES_TIMERS | ||
| 1797 | open_softirq(HRTIMER_SOFTIRQ, run_hrtimer_softirq); | ||
| 1798 | #endif | ||
| 1799 | } | ||
| 1800 | |||
| 1801 | /** | ||
| 1802 | * schedule_hrtimeout_range_clock - sleep until timeout | ||
| 1803 | * @expires: timeout value (ktime_t) | ||
| 1804 | * @delta: slack in expires timeout (ktime_t) | ||
| 1805 | * @mode: timer mode, HRTIMER_MODE_ABS or HRTIMER_MODE_REL | ||
| 1806 | * @clock: timer clock, CLOCK_MONOTONIC or CLOCK_REALTIME | ||
| 1807 | */ | ||
| 1808 | int __sched | ||
| 1809 | schedule_hrtimeout_range_clock(ktime_t *expires, unsigned long delta, | ||
| 1810 | const enum hrtimer_mode mode, int clock) | ||
| 1811 | { | ||
| 1812 | struct hrtimer_sleeper t; | ||
| 1813 | |||
| 1814 | /* | ||
| 1815 | * Optimize when a zero timeout value is given. It does not | ||
| 1816 | * matter whether this is an absolute or a relative time. | ||
| 1817 | */ | ||
| 1818 | if (expires && !expires->tv64) { | ||
| 1819 | __set_current_state(TASK_RUNNING); | ||
| 1820 | return 0; | ||
| 1821 | } | ||
| 1822 | |||
| 1823 | /* | ||
| 1824 | * A NULL parameter means "infinite" | ||
| 1825 | */ | ||
| 1826 | if (!expires) { | ||
| 1827 | schedule(); | ||
| 1828 | __set_current_state(TASK_RUNNING); | ||
| 1829 | return -EINTR; | ||
| 1830 | } | ||
| 1831 | |||
| 1832 | hrtimer_init_on_stack(&t.timer, clock, mode); | ||
| 1833 | hrtimer_set_expires_range_ns(&t.timer, *expires, delta); | ||
| 1834 | |||
| 1835 | hrtimer_init_sleeper(&t, current); | ||
| 1836 | |||
| 1837 | hrtimer_start_expires(&t.timer, mode); | ||
| 1838 | if (!hrtimer_active(&t.timer)) | ||
| 1839 | t.task = NULL; | ||
| 1840 | |||
| 1841 | if (likely(t.task)) | ||
| 1842 | schedule(); | ||
| 1843 | |||
| 1844 | hrtimer_cancel(&t.timer); | ||
| 1845 | destroy_hrtimer_on_stack(&t.timer); | ||
| 1846 | |||
| 1847 | __set_current_state(TASK_RUNNING); | ||
| 1848 | |||
| 1849 | return !t.task ? 0 : -EINTR; | ||
| 1850 | } | ||
| 1851 | |||
| 1852 | /** | ||
| 1853 | * schedule_hrtimeout_range - sleep until timeout | ||
| 1854 | * @expires: timeout value (ktime_t) | ||
| 1855 | * @delta: slack in expires timeout (ktime_t) | ||
| 1856 | * @mode: timer mode, HRTIMER_MODE_ABS or HRTIMER_MODE_REL | ||
| 1857 | * | ||
| 1858 | * Make the current task sleep until the given expiry time has | ||
| 1859 | * elapsed. The routine will return immediately unless | ||
| 1860 | * the current task state has been set (see set_current_state()). | ||
| 1861 | * | ||
| 1862 | * The @delta argument gives the kernel the freedom to schedule the | ||
| 1863 | * actual wakeup to a time that is both power and performance friendly. | ||
| 1864 | * The kernel give the normal best effort behavior for "@expires+@delta", | ||
| 1865 | * but may decide to fire the timer earlier, but no earlier than @expires. | ||
| 1866 | * | ||
| 1867 | * You can set the task state as follows - | ||
| 1868 | * | ||
| 1869 | * %TASK_UNINTERRUPTIBLE - at least @timeout time is guaranteed to | ||
| 1870 | * pass before the routine returns. | ||
| 1871 | * | ||
| 1872 | * %TASK_INTERRUPTIBLE - the routine may return early if a signal is | ||
| 1873 | * delivered to the current task. | ||
| 1874 | * | ||
| 1875 | * The current task state is guaranteed to be TASK_RUNNING when this | ||
| 1876 | * routine returns. | ||
| 1877 | * | ||
| 1878 | * Returns 0 when the timer has expired otherwise -EINTR | ||
| 1879 | */ | ||
| 1880 | int __sched schedule_hrtimeout_range(ktime_t *expires, unsigned long delta, | ||
| 1881 | const enum hrtimer_mode mode) | ||
| 1882 | { | ||
| 1883 | return schedule_hrtimeout_range_clock(expires, delta, mode, | ||
| 1884 | CLOCK_MONOTONIC); | ||
| 1885 | } | ||
| 1886 | EXPORT_SYMBOL_GPL(schedule_hrtimeout_range); | ||
| 1887 | |||
| 1888 | /** | ||
| 1889 | * schedule_hrtimeout - sleep until timeout | ||
| 1890 | * @expires: timeout value (ktime_t) | ||
| 1891 | * @mode: timer mode, HRTIMER_MODE_ABS or HRTIMER_MODE_REL | ||
| 1892 | * | ||
| 1893 | * Make the current task sleep until the given expiry time has | ||
| 1894 | * elapsed. The routine will return immediately unless | ||
| 1895 | * the current task state has been set (see set_current_state()). | ||
| 1896 | * | ||
| 1897 | * You can set the task state as follows - | ||
| 1898 | * | ||
| 1899 | * %TASK_UNINTERRUPTIBLE - at least @timeout time is guaranteed to | ||
| 1900 | * pass before the routine returns. | ||
| 1901 | * | ||
| 1902 | * %TASK_INTERRUPTIBLE - the routine may return early if a signal is | ||
| 1903 | * delivered to the current task. | ||
| 1904 | * | ||
| 1905 | * The current task state is guaranteed to be TASK_RUNNING when this | ||
| 1906 | * routine returns. | ||
| 1907 | * | ||
| 1908 | * Returns 0 when the timer has expired otherwise -EINTR | ||
| 1909 | */ | ||
| 1910 | int __sched schedule_hrtimeout(ktime_t *expires, | ||
| 1911 | const enum hrtimer_mode mode) | ||
| 1912 | { | ||
| 1913 | return schedule_hrtimeout_range(expires, 0, mode); | ||
| 1914 | } | ||
| 1915 | EXPORT_SYMBOL_GPL(schedule_hrtimeout); | ||
diff --git a/kernel/time/itimer.c b/kernel/time/itimer.c new file mode 100644 index 000000000000..8d262b467573 --- /dev/null +++ b/kernel/time/itimer.c | |||
| @@ -0,0 +1,301 @@ | |||
| 1 | /* | ||
| 2 | * linux/kernel/itimer.c | ||
| 3 | * | ||
| 4 | * Copyright (C) 1992 Darren Senn | ||
| 5 | */ | ||
| 6 | |||
| 7 | /* These are all the functions necessary to implement itimers */ | ||
| 8 | |||
| 9 | #include <linux/mm.h> | ||
| 10 | #include <linux/interrupt.h> | ||
| 11 | #include <linux/syscalls.h> | ||
| 12 | #include <linux/time.h> | ||
| 13 | #include <linux/posix-timers.h> | ||
| 14 | #include <linux/hrtimer.h> | ||
| 15 | #include <trace/events/timer.h> | ||
| 16 | |||
| 17 | #include <asm/uaccess.h> | ||
| 18 | |||
| 19 | /** | ||
| 20 | * itimer_get_remtime - get remaining time for the timer | ||
| 21 | * | ||
| 22 | * @timer: the timer to read | ||
| 23 | * | ||
| 24 | * Returns the delta between the expiry time and now, which can be | ||
| 25 | * less than zero or 1usec for an pending expired timer | ||
| 26 | */ | ||
| 27 | static struct timeval itimer_get_remtime(struct hrtimer *timer) | ||
| 28 | { | ||
| 29 | ktime_t rem = hrtimer_get_remaining(timer); | ||
| 30 | |||
| 31 | /* | ||
| 32 | * Racy but safe: if the itimer expires after the above | ||
| 33 | * hrtimer_get_remtime() call but before this condition | ||
| 34 | * then we return 0 - which is correct. | ||
| 35 | */ | ||
| 36 | if (hrtimer_active(timer)) { | ||
| 37 | if (rem.tv64 <= 0) | ||
| 38 | rem.tv64 = NSEC_PER_USEC; | ||
| 39 | } else | ||
| 40 | rem.tv64 = 0; | ||
| 41 | |||
| 42 | return ktime_to_timeval(rem); | ||
| 43 | } | ||
| 44 | |||
| 45 | static void get_cpu_itimer(struct task_struct *tsk, unsigned int clock_id, | ||
| 46 | struct itimerval *const value) | ||
| 47 | { | ||
| 48 | cputime_t cval, cinterval; | ||
| 49 | struct cpu_itimer *it = &tsk->signal->it[clock_id]; | ||
| 50 | |||
| 51 | spin_lock_irq(&tsk->sighand->siglock); | ||
| 52 | |||
| 53 | cval = it->expires; | ||
| 54 | cinterval = it->incr; | ||
| 55 | if (cval) { | ||
| 56 | struct task_cputime cputime; | ||
| 57 | cputime_t t; | ||
| 58 | |||
| 59 | thread_group_cputimer(tsk, &cputime); | ||
| 60 | if (clock_id == CPUCLOCK_PROF) | ||
| 61 | t = cputime.utime + cputime.stime; | ||
| 62 | else | ||
| 63 | /* CPUCLOCK_VIRT */ | ||
| 64 | t = cputime.utime; | ||
| 65 | |||
| 66 | if (cval < t) | ||
| 67 | /* about to fire */ | ||
| 68 | cval = cputime_one_jiffy; | ||
| 69 | else | ||
| 70 | cval = cval - t; | ||
| 71 | } | ||
| 72 | |||
| 73 | spin_unlock_irq(&tsk->sighand->siglock); | ||
| 74 | |||
| 75 | cputime_to_timeval(cval, &value->it_value); | ||
| 76 | cputime_to_timeval(cinterval, &value->it_interval); | ||
| 77 | } | ||
| 78 | |||
| 79 | int do_getitimer(int which, struct itimerval *value) | ||
| 80 | { | ||
| 81 | struct task_struct *tsk = current; | ||
| 82 | |||
| 83 | switch (which) { | ||
| 84 | case ITIMER_REAL: | ||
| 85 | spin_lock_irq(&tsk->sighand->siglock); | ||
| 86 | value->it_value = itimer_get_remtime(&tsk->signal->real_timer); | ||
| 87 | value->it_interval = | ||
| 88 | ktime_to_timeval(tsk->signal->it_real_incr); | ||
| 89 | spin_unlock_irq(&tsk->sighand->siglock); | ||
| 90 | break; | ||
| 91 | case ITIMER_VIRTUAL: | ||
| 92 | get_cpu_itimer(tsk, CPUCLOCK_VIRT, value); | ||
| 93 | break; | ||
| 94 | case ITIMER_PROF: | ||
| 95 | get_cpu_itimer(tsk, CPUCLOCK_PROF, value); | ||
| 96 | break; | ||
| 97 | default: | ||
| 98 | return(-EINVAL); | ||
| 99 | } | ||
| 100 | return 0; | ||
| 101 | } | ||
| 102 | |||
| 103 | SYSCALL_DEFINE2(getitimer, int, which, struct itimerval __user *, value) | ||
| 104 | { | ||
| 105 | int error = -EFAULT; | ||
| 106 | struct itimerval get_buffer; | ||
| 107 | |||
| 108 | if (value) { | ||
| 109 | error = do_getitimer(which, &get_buffer); | ||
| 110 | if (!error && | ||
| 111 | copy_to_user(value, &get_buffer, sizeof(get_buffer))) | ||
| 112 | error = -EFAULT; | ||
| 113 | } | ||
| 114 | return error; | ||
| 115 | } | ||
| 116 | |||
| 117 | |||
| 118 | /* | ||
| 119 | * The timer is automagically restarted, when interval != 0 | ||
| 120 | */ | ||
| 121 | enum hrtimer_restart it_real_fn(struct hrtimer *timer) | ||
| 122 | { | ||
| 123 | struct signal_struct *sig = | ||
| 124 | container_of(timer, struct signal_struct, real_timer); | ||
| 125 | |||
| 126 | trace_itimer_expire(ITIMER_REAL, sig->leader_pid, 0); | ||
| 127 | kill_pid_info(SIGALRM, SEND_SIG_PRIV, sig->leader_pid); | ||
| 128 | |||
| 129 | return HRTIMER_NORESTART; | ||
| 130 | } | ||
| 131 | |||
| 132 | static inline u32 cputime_sub_ns(cputime_t ct, s64 real_ns) | ||
| 133 | { | ||
| 134 | struct timespec ts; | ||
| 135 | s64 cpu_ns; | ||
| 136 | |||
| 137 | cputime_to_timespec(ct, &ts); | ||
| 138 | cpu_ns = timespec_to_ns(&ts); | ||
| 139 | |||
| 140 | return (cpu_ns <= real_ns) ? 0 : cpu_ns - real_ns; | ||
| 141 | } | ||
| 142 | |||
| 143 | static void set_cpu_itimer(struct task_struct *tsk, unsigned int clock_id, | ||
| 144 | const struct itimerval *const value, | ||
| 145 | struct itimerval *const ovalue) | ||
| 146 | { | ||
| 147 | cputime_t cval, nval, cinterval, ninterval; | ||
| 148 | s64 ns_ninterval, ns_nval; | ||
| 149 | u32 error, incr_error; | ||
| 150 | struct cpu_itimer *it = &tsk->signal->it[clock_id]; | ||
| 151 | |||
| 152 | nval = timeval_to_cputime(&value->it_value); | ||
| 153 | ns_nval = timeval_to_ns(&value->it_value); | ||
| 154 | ninterval = timeval_to_cputime(&value->it_interval); | ||
| 155 | ns_ninterval = timeval_to_ns(&value->it_interval); | ||
| 156 | |||
| 157 | error = cputime_sub_ns(nval, ns_nval); | ||
| 158 | incr_error = cputime_sub_ns(ninterval, ns_ninterval); | ||
| 159 | |||
| 160 | spin_lock_irq(&tsk->sighand->siglock); | ||
| 161 | |||
| 162 | cval = it->expires; | ||
| 163 | cinterval = it->incr; | ||
| 164 | if (cval || nval) { | ||
| 165 | if (nval > 0) | ||
| 166 | nval += cputime_one_jiffy; | ||
| 167 | set_process_cpu_timer(tsk, clock_id, &nval, &cval); | ||
| 168 | } | ||
| 169 | it->expires = nval; | ||
| 170 | it->incr = ninterval; | ||
| 171 | it->error = error; | ||
| 172 | it->incr_error = incr_error; | ||
| 173 | trace_itimer_state(clock_id == CPUCLOCK_VIRT ? | ||
| 174 | ITIMER_VIRTUAL : ITIMER_PROF, value, nval); | ||
| 175 | |||
| 176 | spin_unlock_irq(&tsk->sighand->siglock); | ||
| 177 | |||
| 178 | if (ovalue) { | ||
| 179 | cputime_to_timeval(cval, &ovalue->it_value); | ||
| 180 | cputime_to_timeval(cinterval, &ovalue->it_interval); | ||
| 181 | } | ||
| 182 | } | ||
| 183 | |||
| 184 | /* | ||
| 185 | * Returns true if the timeval is in canonical form | ||
| 186 | */ | ||
| 187 | #define timeval_valid(t) \ | ||
| 188 | (((t)->tv_sec >= 0) && (((unsigned long) (t)->tv_usec) < USEC_PER_SEC)) | ||
| 189 | |||
| 190 | int do_setitimer(int which, struct itimerval *value, struct itimerval *ovalue) | ||
| 191 | { | ||
| 192 | struct task_struct *tsk = current; | ||
| 193 | struct hrtimer *timer; | ||
| 194 | ktime_t expires; | ||
| 195 | |||
| 196 | /* | ||
| 197 | * Validate the timevals in value. | ||
| 198 | */ | ||
| 199 | if (!timeval_valid(&value->it_value) || | ||
| 200 | !timeval_valid(&value->it_interval)) | ||
| 201 | return -EINVAL; | ||
| 202 | |||
| 203 | switch (which) { | ||
| 204 | case ITIMER_REAL: | ||
| 205 | again: | ||
| 206 | spin_lock_irq(&tsk->sighand->siglock); | ||
| 207 | timer = &tsk->signal->real_timer; | ||
| 208 | if (ovalue) { | ||
| 209 | ovalue->it_value = itimer_get_remtime(timer); | ||
| 210 | ovalue->it_interval | ||
| 211 | = ktime_to_timeval(tsk->signal->it_real_incr); | ||
| 212 | } | ||
| 213 | /* We are sharing ->siglock with it_real_fn() */ | ||
| 214 | if (hrtimer_try_to_cancel(timer) < 0) { | ||
| 215 | spin_unlock_irq(&tsk->sighand->siglock); | ||
| 216 | goto again; | ||
| 217 | } | ||
| 218 | expires = timeval_to_ktime(value->it_value); | ||
| 219 | if (expires.tv64 != 0) { | ||
| 220 | tsk->signal->it_real_incr = | ||
| 221 | timeval_to_ktime(value->it_interval); | ||
| 222 | hrtimer_start(timer, expires, HRTIMER_MODE_REL); | ||
| 223 | } else | ||
| 224 | tsk->signal->it_real_incr.tv64 = 0; | ||
| 225 | |||
| 226 | trace_itimer_state(ITIMER_REAL, value, 0); | ||
| 227 | spin_unlock_irq(&tsk->sighand->siglock); | ||
| 228 | break; | ||
| 229 | case ITIMER_VIRTUAL: | ||
| 230 | set_cpu_itimer(tsk, CPUCLOCK_VIRT, value, ovalue); | ||
| 231 | break; | ||
| 232 | case ITIMER_PROF: | ||
| 233 | set_cpu_itimer(tsk, CPUCLOCK_PROF, value, ovalue); | ||
| 234 | break; | ||
| 235 | default: | ||
| 236 | return -EINVAL; | ||
| 237 | } | ||
| 238 | return 0; | ||
| 239 | } | ||
| 240 | |||
| 241 | /** | ||
| 242 | * alarm_setitimer - set alarm in seconds | ||
| 243 | * | ||
| 244 | * @seconds: number of seconds until alarm | ||
| 245 | * 0 disables the alarm | ||
| 246 | * | ||
| 247 | * Returns the remaining time in seconds of a pending timer or 0 when | ||
| 248 | * the timer is not active. | ||
| 249 | * | ||
| 250 | * On 32 bit machines the seconds value is limited to (INT_MAX/2) to avoid | ||
| 251 | * negative timeval settings which would cause immediate expiry. | ||
| 252 | */ | ||
| 253 | unsigned int alarm_setitimer(unsigned int seconds) | ||
| 254 | { | ||
| 255 | struct itimerval it_new, it_old; | ||
| 256 | |||
| 257 | #if BITS_PER_LONG < 64 | ||
| 258 | if (seconds > INT_MAX) | ||
| 259 | seconds = INT_MAX; | ||
| 260 | #endif | ||
| 261 | it_new.it_value.tv_sec = seconds; | ||
| 262 | it_new.it_value.tv_usec = 0; | ||
| 263 | it_new.it_interval.tv_sec = it_new.it_interval.tv_usec = 0; | ||
| 264 | |||
| 265 | do_setitimer(ITIMER_REAL, &it_new, &it_old); | ||
| 266 | |||
| 267 | /* | ||
| 268 | * We can't return 0 if we have an alarm pending ... And we'd | ||
| 269 | * better return too much than too little anyway | ||
| 270 | */ | ||
| 271 | if ((!it_old.it_value.tv_sec && it_old.it_value.tv_usec) || | ||
| 272 | it_old.it_value.tv_usec >= 500000) | ||
| 273 | it_old.it_value.tv_sec++; | ||
| 274 | |||
| 275 | return it_old.it_value.tv_sec; | ||
| 276 | } | ||
| 277 | |||
| 278 | SYSCALL_DEFINE3(setitimer, int, which, struct itimerval __user *, value, | ||
| 279 | struct itimerval __user *, ovalue) | ||
| 280 | { | ||
| 281 | struct itimerval set_buffer, get_buffer; | ||
| 282 | int error; | ||
| 283 | |||
| 284 | if (value) { | ||
| 285 | if(copy_from_user(&set_buffer, value, sizeof(set_buffer))) | ||
| 286 | return -EFAULT; | ||
| 287 | } else { | ||
| 288 | memset(&set_buffer, 0, sizeof(set_buffer)); | ||
| 289 | printk_once(KERN_WARNING "%s calls setitimer() with new_value NULL pointer." | ||
| 290 | " Misfeature support will be removed\n", | ||
| 291 | current->comm); | ||
| 292 | } | ||
| 293 | |||
| 294 | error = do_setitimer(which, &set_buffer, ovalue ? &get_buffer : NULL); | ||
| 295 | if (error || !ovalue) | ||
| 296 | return error; | ||
| 297 | |||
| 298 | if (copy_to_user(ovalue, &get_buffer, sizeof(get_buffer))) | ||
| 299 | return -EFAULT; | ||
| 300 | return 0; | ||
| 301 | } | ||
diff --git a/kernel/time/posix-cpu-timers.c b/kernel/time/posix-cpu-timers.c new file mode 100644 index 000000000000..3b8946416a5f --- /dev/null +++ b/kernel/time/posix-cpu-timers.c | |||
| @@ -0,0 +1,1490 @@ | |||
| 1 | /* | ||
| 2 | * Implement CPU time clocks for the POSIX clock interface. | ||
| 3 | */ | ||
| 4 | |||
| 5 | #include <linux/sched.h> | ||
| 6 | #include <linux/posix-timers.h> | ||
| 7 | #include <linux/errno.h> | ||
| 8 | #include <linux/math64.h> | ||
| 9 | #include <asm/uaccess.h> | ||
| 10 | #include <linux/kernel_stat.h> | ||
| 11 | #include <trace/events/timer.h> | ||
| 12 | #include <linux/random.h> | ||
| 13 | #include <linux/tick.h> | ||
| 14 | #include <linux/workqueue.h> | ||
| 15 | |||
| 16 | /* | ||
| 17 | * Called after updating RLIMIT_CPU to run cpu timer and update | ||
| 18 | * tsk->signal->cputime_expires expiration cache if necessary. Needs | ||
| 19 | * siglock protection since other code may update expiration cache as | ||
| 20 | * well. | ||
| 21 | */ | ||
| 22 | void update_rlimit_cpu(struct task_struct *task, unsigned long rlim_new) | ||
| 23 | { | ||
| 24 | cputime_t cputime = secs_to_cputime(rlim_new); | ||
| 25 | |||
| 26 | spin_lock_irq(&task->sighand->siglock); | ||
| 27 | set_process_cpu_timer(task, CPUCLOCK_PROF, &cputime, NULL); | ||
| 28 | spin_unlock_irq(&task->sighand->siglock); | ||
| 29 | } | ||
| 30 | |||
| 31 | static int check_clock(const clockid_t which_clock) | ||
| 32 | { | ||
| 33 | int error = 0; | ||
| 34 | struct task_struct *p; | ||
| 35 | const pid_t pid = CPUCLOCK_PID(which_clock); | ||
| 36 | |||
| 37 | if (CPUCLOCK_WHICH(which_clock) >= CPUCLOCK_MAX) | ||
| 38 | return -EINVAL; | ||
| 39 | |||
| 40 | if (pid == 0) | ||
| 41 | return 0; | ||
| 42 | |||
| 43 | rcu_read_lock(); | ||
| 44 | p = find_task_by_vpid(pid); | ||
| 45 | if (!p || !(CPUCLOCK_PERTHREAD(which_clock) ? | ||
| 46 | same_thread_group(p, current) : has_group_leader_pid(p))) { | ||
| 47 | error = -EINVAL; | ||
| 48 | } | ||
| 49 | rcu_read_unlock(); | ||
| 50 | |||
| 51 | return error; | ||
| 52 | } | ||
| 53 | |||
| 54 | static inline unsigned long long | ||
| 55 | timespec_to_sample(const clockid_t which_clock, const struct timespec *tp) | ||
| 56 | { | ||
| 57 | unsigned long long ret; | ||
| 58 | |||
| 59 | ret = 0; /* high half always zero when .cpu used */ | ||
| 60 | if (CPUCLOCK_WHICH(which_clock) == CPUCLOCK_SCHED) { | ||
| 61 | ret = (unsigned long long)tp->tv_sec * NSEC_PER_SEC + tp->tv_nsec; | ||
| 62 | } else { | ||
| 63 | ret = cputime_to_expires(timespec_to_cputime(tp)); | ||
| 64 | } | ||
| 65 | return ret; | ||
| 66 | } | ||
| 67 | |||
| 68 | static void sample_to_timespec(const clockid_t which_clock, | ||
| 69 | unsigned long long expires, | ||
| 70 | struct timespec *tp) | ||
| 71 | { | ||
| 72 | if (CPUCLOCK_WHICH(which_clock) == CPUCLOCK_SCHED) | ||
| 73 | *tp = ns_to_timespec(expires); | ||
| 74 | else | ||
| 75 | cputime_to_timespec((__force cputime_t)expires, tp); | ||
| 76 | } | ||
| 77 | |||
| 78 | /* | ||
| 79 | * Update expiry time from increment, and increase overrun count, | ||
| 80 | * given the current clock sample. | ||
| 81 | */ | ||
| 82 | static void bump_cpu_timer(struct k_itimer *timer, | ||
| 83 | unsigned long long now) | ||
| 84 | { | ||
| 85 | int i; | ||
| 86 | unsigned long long delta, incr; | ||
| 87 | |||
| 88 | if (timer->it.cpu.incr == 0) | ||
| 89 | return; | ||
| 90 | |||
| 91 | if (now < timer->it.cpu.expires) | ||
| 92 | return; | ||
| 93 | |||
| 94 | incr = timer->it.cpu.incr; | ||
| 95 | delta = now + incr - timer->it.cpu.expires; | ||
| 96 | |||
| 97 | /* Don't use (incr*2 < delta), incr*2 might overflow. */ | ||
| 98 | for (i = 0; incr < delta - incr; i++) | ||
| 99 | incr = incr << 1; | ||
| 100 | |||
| 101 | for (; i >= 0; incr >>= 1, i--) { | ||
| 102 | if (delta < incr) | ||
| 103 | continue; | ||
| 104 | |||
| 105 | timer->it.cpu.expires += incr; | ||
| 106 | timer->it_overrun += 1 << i; | ||
| 107 | delta -= incr; | ||
| 108 | } | ||
| 109 | } | ||
| 110 | |||
| 111 | /** | ||
| 112 | * task_cputime_zero - Check a task_cputime struct for all zero fields. | ||
| 113 | * | ||
| 114 | * @cputime: The struct to compare. | ||
| 115 | * | ||
| 116 | * Checks @cputime to see if all fields are zero. Returns true if all fields | ||
| 117 | * are zero, false if any field is nonzero. | ||
| 118 | */ | ||
| 119 | static inline int task_cputime_zero(const struct task_cputime *cputime) | ||
| 120 | { | ||
| 121 | if (!cputime->utime && !cputime->stime && !cputime->sum_exec_runtime) | ||
| 122 | return 1; | ||
| 123 | return 0; | ||
| 124 | } | ||
| 125 | |||
| 126 | static inline unsigned long long prof_ticks(struct task_struct *p) | ||
| 127 | { | ||
| 128 | cputime_t utime, stime; | ||
| 129 | |||
| 130 | task_cputime(p, &utime, &stime); | ||
| 131 | |||
| 132 | return cputime_to_expires(utime + stime); | ||
| 133 | } | ||
| 134 | static inline unsigned long long virt_ticks(struct task_struct *p) | ||
| 135 | { | ||
| 136 | cputime_t utime; | ||
| 137 | |||
| 138 | task_cputime(p, &utime, NULL); | ||
| 139 | |||
| 140 | return cputime_to_expires(utime); | ||
| 141 | } | ||
| 142 | |||
| 143 | static int | ||
| 144 | posix_cpu_clock_getres(const clockid_t which_clock, struct timespec *tp) | ||
| 145 | { | ||
| 146 | int error = check_clock(which_clock); | ||
| 147 | if (!error) { | ||
| 148 | tp->tv_sec = 0; | ||
| 149 | tp->tv_nsec = ((NSEC_PER_SEC + HZ - 1) / HZ); | ||
| 150 | if (CPUCLOCK_WHICH(which_clock) == CPUCLOCK_SCHED) { | ||
| 151 | /* | ||
| 152 | * If sched_clock is using a cycle counter, we | ||
| 153 | * don't have any idea of its true resolution | ||
| 154 | * exported, but it is much more than 1s/HZ. | ||
| 155 | */ | ||
| 156 | tp->tv_nsec = 1; | ||
| 157 | } | ||
| 158 | } | ||
| 159 | return error; | ||
| 160 | } | ||
| 161 | |||
| 162 | static int | ||
| 163 | posix_cpu_clock_set(const clockid_t which_clock, const struct timespec *tp) | ||
| 164 | { | ||
| 165 | /* | ||
| 166 | * You can never reset a CPU clock, but we check for other errors | ||
| 167 | * in the call before failing with EPERM. | ||
| 168 | */ | ||
| 169 | int error = check_clock(which_clock); | ||
| 170 | if (error == 0) { | ||
| 171 | error = -EPERM; | ||
| 172 | } | ||
| 173 | return error; | ||
| 174 | } | ||
| 175 | |||
| 176 | |||
| 177 | /* | ||
| 178 | * Sample a per-thread clock for the given task. | ||
| 179 | */ | ||
| 180 | static int cpu_clock_sample(const clockid_t which_clock, struct task_struct *p, | ||
| 181 | unsigned long long *sample) | ||
| 182 | { | ||
| 183 | switch (CPUCLOCK_WHICH(which_clock)) { | ||
| 184 | default: | ||
| 185 | return -EINVAL; | ||
| 186 | case CPUCLOCK_PROF: | ||
| 187 | *sample = prof_ticks(p); | ||
| 188 | break; | ||
| 189 | case CPUCLOCK_VIRT: | ||
| 190 | *sample = virt_ticks(p); | ||
| 191 | break; | ||
| 192 | case CPUCLOCK_SCHED: | ||
| 193 | *sample = task_sched_runtime(p); | ||
| 194 | break; | ||
| 195 | } | ||
| 196 | return 0; | ||
| 197 | } | ||
| 198 | |||
| 199 | static void update_gt_cputime(struct task_cputime *a, struct task_cputime *b) | ||
| 200 | { | ||
| 201 | if (b->utime > a->utime) | ||
| 202 | a->utime = b->utime; | ||
| 203 | |||
| 204 | if (b->stime > a->stime) | ||
| 205 | a->stime = b->stime; | ||
| 206 | |||
| 207 | if (b->sum_exec_runtime > a->sum_exec_runtime) | ||
| 208 | a->sum_exec_runtime = b->sum_exec_runtime; | ||
| 209 | } | ||
| 210 | |||
| 211 | void thread_group_cputimer(struct task_struct *tsk, struct task_cputime *times) | ||
| 212 | { | ||
| 213 | struct thread_group_cputimer *cputimer = &tsk->signal->cputimer; | ||
| 214 | struct task_cputime sum; | ||
| 215 | unsigned long flags; | ||
| 216 | |||
| 217 | if (!cputimer->running) { | ||
| 218 | /* | ||
| 219 | * The POSIX timer interface allows for absolute time expiry | ||
| 220 | * values through the TIMER_ABSTIME flag, therefore we have | ||
| 221 | * to synchronize the timer to the clock every time we start | ||
| 222 | * it. | ||
| 223 | */ | ||
| 224 | thread_group_cputime(tsk, &sum); | ||
| 225 | raw_spin_lock_irqsave(&cputimer->lock, flags); | ||
| 226 | cputimer->running = 1; | ||
| 227 | update_gt_cputime(&cputimer->cputime, &sum); | ||
| 228 | } else | ||
| 229 | raw_spin_lock_irqsave(&cputimer->lock, flags); | ||
| 230 | *times = cputimer->cputime; | ||
| 231 | raw_spin_unlock_irqrestore(&cputimer->lock, flags); | ||
| 232 | } | ||
| 233 | |||
| 234 | /* | ||
| 235 | * Sample a process (thread group) clock for the given group_leader task. | ||
| 236 | * Must be called with task sighand lock held for safe while_each_thread() | ||
| 237 | * traversal. | ||
| 238 | */ | ||
| 239 | static int cpu_clock_sample_group(const clockid_t which_clock, | ||
| 240 | struct task_struct *p, | ||
| 241 | unsigned long long *sample) | ||
| 242 | { | ||
| 243 | struct task_cputime cputime; | ||
| 244 | |||
| 245 | switch (CPUCLOCK_WHICH(which_clock)) { | ||
| 246 | default: | ||
| 247 | return -EINVAL; | ||
| 248 | case CPUCLOCK_PROF: | ||
| 249 | thread_group_cputime(p, &cputime); | ||
| 250 | *sample = cputime_to_expires(cputime.utime + cputime.stime); | ||
| 251 | break; | ||
| 252 | case CPUCLOCK_VIRT: | ||
| 253 | thread_group_cputime(p, &cputime); | ||
| 254 | *sample = cputime_to_expires(cputime.utime); | ||
| 255 | break; | ||
| 256 | case CPUCLOCK_SCHED: | ||
| 257 | thread_group_cputime(p, &cputime); | ||
| 258 | *sample = cputime.sum_exec_runtime; | ||
| 259 | break; | ||
| 260 | } | ||
| 261 | return 0; | ||
| 262 | } | ||
| 263 | |||
| 264 | static int posix_cpu_clock_get_task(struct task_struct *tsk, | ||
| 265 | const clockid_t which_clock, | ||
| 266 | struct timespec *tp) | ||
| 267 | { | ||
| 268 | int err = -EINVAL; | ||
| 269 | unsigned long long rtn; | ||
| 270 | |||
| 271 | if (CPUCLOCK_PERTHREAD(which_clock)) { | ||
| 272 | if (same_thread_group(tsk, current)) | ||
| 273 | err = cpu_clock_sample(which_clock, tsk, &rtn); | ||
| 274 | } else { | ||
| 275 | unsigned long flags; | ||
| 276 | struct sighand_struct *sighand; | ||
| 277 | |||
| 278 | /* | ||
| 279 | * while_each_thread() is not yet entirely RCU safe, | ||
| 280 | * keep locking the group while sampling process | ||
| 281 | * clock for now. | ||
| 282 | */ | ||
| 283 | sighand = lock_task_sighand(tsk, &flags); | ||
| 284 | if (!sighand) | ||
| 285 | return err; | ||
| 286 | |||
| 287 | if (tsk == current || thread_group_leader(tsk)) | ||
| 288 | err = cpu_clock_sample_group(which_clock, tsk, &rtn); | ||
| 289 | |||
| 290 | unlock_task_sighand(tsk, &flags); | ||
| 291 | } | ||
| 292 | |||
| 293 | if (!err) | ||
| 294 | sample_to_timespec(which_clock, rtn, tp); | ||
| 295 | |||
| 296 | return err; | ||
| 297 | } | ||
| 298 | |||
| 299 | |||
| 300 | static int posix_cpu_clock_get(const clockid_t which_clock, struct timespec *tp) | ||
| 301 | { | ||
| 302 | const pid_t pid = CPUCLOCK_PID(which_clock); | ||
| 303 | int err = -EINVAL; | ||
| 304 | |||
| 305 | if (pid == 0) { | ||
| 306 | /* | ||
| 307 | * Special case constant value for our own clocks. | ||
| 308 | * We don't have to do any lookup to find ourselves. | ||
| 309 | */ | ||
| 310 | err = posix_cpu_clock_get_task(current, which_clock, tp); | ||
| 311 | } else { | ||
| 312 | /* | ||
| 313 | * Find the given PID, and validate that the caller | ||
| 314 | * should be able to see it. | ||
| 315 | */ | ||
| 316 | struct task_struct *p; | ||
| 317 | rcu_read_lock(); | ||
| 318 | p = find_task_by_vpid(pid); | ||
| 319 | if (p) | ||
| 320 | err = posix_cpu_clock_get_task(p, which_clock, tp); | ||
| 321 | rcu_read_unlock(); | ||
| 322 | } | ||
| 323 | |||
| 324 | return err; | ||
| 325 | } | ||
| 326 | |||
| 327 | |||
| 328 | /* | ||
| 329 | * Validate the clockid_t for a new CPU-clock timer, and initialize the timer. | ||
| 330 | * This is called from sys_timer_create() and do_cpu_nanosleep() with the | ||
| 331 | * new timer already all-zeros initialized. | ||
| 332 | */ | ||
| 333 | static int posix_cpu_timer_create(struct k_itimer *new_timer) | ||
| 334 | { | ||
| 335 | int ret = 0; | ||
| 336 | const pid_t pid = CPUCLOCK_PID(new_timer->it_clock); | ||
| 337 | struct task_struct *p; | ||
| 338 | |||
| 339 | if (CPUCLOCK_WHICH(new_timer->it_clock) >= CPUCLOCK_MAX) | ||
| 340 | return -EINVAL; | ||
| 341 | |||
| 342 | INIT_LIST_HEAD(&new_timer->it.cpu.entry); | ||
| 343 | |||
| 344 | rcu_read_lock(); | ||
| 345 | if (CPUCLOCK_PERTHREAD(new_timer->it_clock)) { | ||
| 346 | if (pid == 0) { | ||
| 347 | p = current; | ||
| 348 | } else { | ||
| 349 | p = find_task_by_vpid(pid); | ||
| 350 | if (p && !same_thread_group(p, current)) | ||
| 351 | p = NULL; | ||
| 352 | } | ||
| 353 | } else { | ||
| 354 | if (pid == 0) { | ||
| 355 | p = current->group_leader; | ||
| 356 | } else { | ||
| 357 | p = find_task_by_vpid(pid); | ||
| 358 | if (p && !has_group_leader_pid(p)) | ||
| 359 | p = NULL; | ||
| 360 | } | ||
| 361 | } | ||
| 362 | new_timer->it.cpu.task = p; | ||
| 363 | if (p) { | ||
| 364 | get_task_struct(p); | ||
| 365 | } else { | ||
| 366 | ret = -EINVAL; | ||
| 367 | } | ||
| 368 | rcu_read_unlock(); | ||
| 369 | |||
| 370 | return ret; | ||
| 371 | } | ||
| 372 | |||
| 373 | /* | ||
| 374 | * Clean up a CPU-clock timer that is about to be destroyed. | ||
| 375 | * This is called from timer deletion with the timer already locked. | ||
| 376 | * If we return TIMER_RETRY, it's necessary to release the timer's lock | ||
| 377 | * and try again. (This happens when the timer is in the middle of firing.) | ||
| 378 | */ | ||
| 379 | static int posix_cpu_timer_del(struct k_itimer *timer) | ||
| 380 | { | ||
| 381 | int ret = 0; | ||
| 382 | unsigned long flags; | ||
| 383 | struct sighand_struct *sighand; | ||
| 384 | struct task_struct *p = timer->it.cpu.task; | ||
| 385 | |||
| 386 | WARN_ON_ONCE(p == NULL); | ||
| 387 | |||
| 388 | /* | ||
| 389 | * Protect against sighand release/switch in exit/exec and process/ | ||
| 390 | * thread timer list entry concurrent read/writes. | ||
| 391 | */ | ||
| 392 | sighand = lock_task_sighand(p, &flags); | ||
| 393 | if (unlikely(sighand == NULL)) { | ||
| 394 | /* | ||
| 395 | * We raced with the reaping of the task. | ||
| 396 | * The deletion should have cleared us off the list. | ||
| 397 | */ | ||
| 398 | WARN_ON_ONCE(!list_empty(&timer->it.cpu.entry)); | ||
| 399 | } else { | ||
| 400 | if (timer->it.cpu.firing) | ||
| 401 | ret = TIMER_RETRY; | ||
| 402 | else | ||
| 403 | list_del(&timer->it.cpu.entry); | ||
| 404 | |||
| 405 | unlock_task_sighand(p, &flags); | ||
| 406 | } | ||
| 407 | |||
| 408 | if (!ret) | ||
| 409 | put_task_struct(p); | ||
| 410 | |||
| 411 | return ret; | ||
| 412 | } | ||
| 413 | |||
| 414 | static void cleanup_timers_list(struct list_head *head) | ||
| 415 | { | ||
| 416 | struct cpu_timer_list *timer, *next; | ||
| 417 | |||
| 418 | list_for_each_entry_safe(timer, next, head, entry) | ||
| 419 | list_del_init(&timer->entry); | ||
| 420 | } | ||
| 421 | |||
| 422 | /* | ||
| 423 | * Clean out CPU timers still ticking when a thread exited. The task | ||
| 424 | * pointer is cleared, and the expiry time is replaced with the residual | ||
| 425 | * time for later timer_gettime calls to return. | ||
| 426 | * This must be called with the siglock held. | ||
| 427 | */ | ||
| 428 | static void cleanup_timers(struct list_head *head) | ||
| 429 | { | ||
| 430 | cleanup_timers_list(head); | ||
| 431 | cleanup_timers_list(++head); | ||
| 432 | cleanup_timers_list(++head); | ||
| 433 | } | ||
| 434 | |||
| 435 | /* | ||
| 436 | * These are both called with the siglock held, when the current thread | ||
| 437 | * is being reaped. When the final (leader) thread in the group is reaped, | ||
| 438 | * posix_cpu_timers_exit_group will be called after posix_cpu_timers_exit. | ||
| 439 | */ | ||
| 440 | void posix_cpu_timers_exit(struct task_struct *tsk) | ||
| 441 | { | ||
| 442 | add_device_randomness((const void*) &tsk->se.sum_exec_runtime, | ||
| 443 | sizeof(unsigned long long)); | ||
| 444 | cleanup_timers(tsk->cpu_timers); | ||
| 445 | |||
| 446 | } | ||
| 447 | void posix_cpu_timers_exit_group(struct task_struct *tsk) | ||
| 448 | { | ||
| 449 | cleanup_timers(tsk->signal->cpu_timers); | ||
| 450 | } | ||
| 451 | |||
| 452 | static inline int expires_gt(cputime_t expires, cputime_t new_exp) | ||
| 453 | { | ||
| 454 | return expires == 0 || expires > new_exp; | ||
| 455 | } | ||
| 456 | |||
| 457 | /* | ||
| 458 | * Insert the timer on the appropriate list before any timers that | ||
| 459 | * expire later. This must be called with the sighand lock held. | ||
| 460 | */ | ||
| 461 | static void arm_timer(struct k_itimer *timer) | ||
| 462 | { | ||
| 463 | struct task_struct *p = timer->it.cpu.task; | ||
| 464 | struct list_head *head, *listpos; | ||
| 465 | struct task_cputime *cputime_expires; | ||
| 466 | struct cpu_timer_list *const nt = &timer->it.cpu; | ||
| 467 | struct cpu_timer_list *next; | ||
| 468 | |||
| 469 | if (CPUCLOCK_PERTHREAD(timer->it_clock)) { | ||
| 470 | head = p->cpu_timers; | ||
| 471 | cputime_expires = &p->cputime_expires; | ||
| 472 | } else { | ||
| 473 | head = p->signal->cpu_timers; | ||
| 474 | cputime_expires = &p->signal->cputime_expires; | ||
| 475 | } | ||
| 476 | head += CPUCLOCK_WHICH(timer->it_clock); | ||
| 477 | |||
| 478 | listpos = head; | ||
| 479 | list_for_each_entry(next, head, entry) { | ||
| 480 | if (nt->expires < next->expires) | ||
| 481 | break; | ||
| 482 | listpos = &next->entry; | ||
| 483 | } | ||
| 484 | list_add(&nt->entry, listpos); | ||
| 485 | |||
| 486 | if (listpos == head) { | ||
| 487 | unsigned long long exp = nt->expires; | ||
| 488 | |||
| 489 | /* | ||
| 490 | * We are the new earliest-expiring POSIX 1.b timer, hence | ||
| 491 | * need to update expiration cache. Take into account that | ||
| 492 | * for process timers we share expiration cache with itimers | ||
| 493 | * and RLIMIT_CPU and for thread timers with RLIMIT_RTTIME. | ||
| 494 | */ | ||
| 495 | |||
| 496 | switch (CPUCLOCK_WHICH(timer->it_clock)) { | ||
| 497 | case CPUCLOCK_PROF: | ||
| 498 | if (expires_gt(cputime_expires->prof_exp, expires_to_cputime(exp))) | ||
| 499 | cputime_expires->prof_exp = expires_to_cputime(exp); | ||
| 500 | break; | ||
| 501 | case CPUCLOCK_VIRT: | ||
| 502 | if (expires_gt(cputime_expires->virt_exp, expires_to_cputime(exp))) | ||
| 503 | cputime_expires->virt_exp = expires_to_cputime(exp); | ||
| 504 | break; | ||
| 505 | case CPUCLOCK_SCHED: | ||
| 506 | if (cputime_expires->sched_exp == 0 || | ||
| 507 | cputime_expires->sched_exp > exp) | ||
| 508 | cputime_expires->sched_exp = exp; | ||
| 509 | break; | ||
| 510 | } | ||
| 511 | } | ||
| 512 | } | ||
| 513 | |||
| 514 | /* | ||
| 515 | * The timer is locked, fire it and arrange for its reload. | ||
| 516 | */ | ||
| 517 | static void cpu_timer_fire(struct k_itimer *timer) | ||
| 518 | { | ||
| 519 | if ((timer->it_sigev_notify & ~SIGEV_THREAD_ID) == SIGEV_NONE) { | ||
| 520 | /* | ||
| 521 | * User don't want any signal. | ||
| 522 | */ | ||
| 523 | timer->it.cpu.expires = 0; | ||
| 524 | } else if (unlikely(timer->sigq == NULL)) { | ||
| 525 | /* | ||
| 526 | * This a special case for clock_nanosleep, | ||
| 527 | * not a normal timer from sys_timer_create. | ||
| 528 | */ | ||
| 529 | wake_up_process(timer->it_process); | ||
| 530 | timer->it.cpu.expires = 0; | ||
| 531 | } else if (timer->it.cpu.incr == 0) { | ||
| 532 | /* | ||
| 533 | * One-shot timer. Clear it as soon as it's fired. | ||
| 534 | */ | ||
| 535 | posix_timer_event(timer, 0); | ||
| 536 | timer->it.cpu.expires = 0; | ||
| 537 | } else if (posix_timer_event(timer, ++timer->it_requeue_pending)) { | ||
| 538 | /* | ||
| 539 | * The signal did not get queued because the signal | ||
| 540 | * was ignored, so we won't get any callback to | ||
| 541 | * reload the timer. But we need to keep it | ||
| 542 | * ticking in case the signal is deliverable next time. | ||
| 543 | */ | ||
| 544 | posix_cpu_timer_schedule(timer); | ||
| 545 | } | ||
| 546 | } | ||
| 547 | |||
| 548 | /* | ||
| 549 | * Sample a process (thread group) timer for the given group_leader task. | ||
| 550 | * Must be called with task sighand lock held for safe while_each_thread() | ||
| 551 | * traversal. | ||
| 552 | */ | ||
| 553 | static int cpu_timer_sample_group(const clockid_t which_clock, | ||
| 554 | struct task_struct *p, | ||
| 555 | unsigned long long *sample) | ||
| 556 | { | ||
| 557 | struct task_cputime cputime; | ||
| 558 | |||
| 559 | thread_group_cputimer(p, &cputime); | ||
| 560 | switch (CPUCLOCK_WHICH(which_clock)) { | ||
| 561 | default: | ||
| 562 | return -EINVAL; | ||
| 563 | case CPUCLOCK_PROF: | ||
| 564 | *sample = cputime_to_expires(cputime.utime + cputime.stime); | ||
| 565 | break; | ||
| 566 | case CPUCLOCK_VIRT: | ||
| 567 | *sample = cputime_to_expires(cputime.utime); | ||
| 568 | break; | ||
| 569 | case CPUCLOCK_SCHED: | ||
| 570 | *sample = cputime.sum_exec_runtime + task_delta_exec(p); | ||
| 571 | break; | ||
| 572 | } | ||
| 573 | return 0; | ||
| 574 | } | ||
| 575 | |||
| 576 | #ifdef CONFIG_NO_HZ_FULL | ||
| 577 | static void nohz_kick_work_fn(struct work_struct *work) | ||
| 578 | { | ||
| 579 | tick_nohz_full_kick_all(); | ||
| 580 | } | ||
| 581 | |||
| 582 | static DECLARE_WORK(nohz_kick_work, nohz_kick_work_fn); | ||
| 583 | |||
| 584 | /* | ||
| 585 | * We need the IPIs to be sent from sane process context. | ||
| 586 | * The posix cpu timers are always set with irqs disabled. | ||
| 587 | */ | ||
| 588 | static void posix_cpu_timer_kick_nohz(void) | ||
| 589 | { | ||
| 590 | if (context_tracking_is_enabled()) | ||
| 591 | schedule_work(&nohz_kick_work); | ||
| 592 | } | ||
| 593 | |||
| 594 | bool posix_cpu_timers_can_stop_tick(struct task_struct *tsk) | ||
| 595 | { | ||
| 596 | if (!task_cputime_zero(&tsk->cputime_expires)) | ||
| 597 | return false; | ||
| 598 | |||
| 599 | if (tsk->signal->cputimer.running) | ||
| 600 | return false; | ||
| 601 | |||
| 602 | return true; | ||
| 603 | } | ||
| 604 | #else | ||
| 605 | static inline void posix_cpu_timer_kick_nohz(void) { } | ||
| 606 | #endif | ||
| 607 | |||
| 608 | /* | ||
| 609 | * Guts of sys_timer_settime for CPU timers. | ||
| 610 | * This is called with the timer locked and interrupts disabled. | ||
| 611 | * If we return TIMER_RETRY, it's necessary to release the timer's lock | ||
| 612 | * and try again. (This happens when the timer is in the middle of firing.) | ||
| 613 | */ | ||
| 614 | static int posix_cpu_timer_set(struct k_itimer *timer, int timer_flags, | ||
| 615 | struct itimerspec *new, struct itimerspec *old) | ||
| 616 | { | ||
| 617 | unsigned long flags; | ||
| 618 | struct sighand_struct *sighand; | ||
| 619 | struct task_struct *p = timer->it.cpu.task; | ||
| 620 | unsigned long long old_expires, new_expires, old_incr, val; | ||
| 621 | int ret; | ||
| 622 | |||
| 623 | WARN_ON_ONCE(p == NULL); | ||
| 624 | |||
| 625 | new_expires = timespec_to_sample(timer->it_clock, &new->it_value); | ||
| 626 | |||
| 627 | /* | ||
| 628 | * Protect against sighand release/switch in exit/exec and p->cpu_timers | ||
| 629 | * and p->signal->cpu_timers read/write in arm_timer() | ||
| 630 | */ | ||
| 631 | sighand = lock_task_sighand(p, &flags); | ||
| 632 | /* | ||
| 633 | * If p has just been reaped, we can no | ||
| 634 | * longer get any information about it at all. | ||
| 635 | */ | ||
| 636 | if (unlikely(sighand == NULL)) { | ||
| 637 | return -ESRCH; | ||
| 638 | } | ||
| 639 | |||
| 640 | /* | ||
| 641 | * Disarm any old timer after extracting its expiry time. | ||
| 642 | */ | ||
| 643 | WARN_ON_ONCE(!irqs_disabled()); | ||
| 644 | |||
| 645 | ret = 0; | ||
| 646 | old_incr = timer->it.cpu.incr; | ||
| 647 | old_expires = timer->it.cpu.expires; | ||
| 648 | if (unlikely(timer->it.cpu.firing)) { | ||
| 649 | timer->it.cpu.firing = -1; | ||
| 650 | ret = TIMER_RETRY; | ||
| 651 | } else | ||
| 652 | list_del_init(&timer->it.cpu.entry); | ||
| 653 | |||
| 654 | /* | ||
| 655 | * We need to sample the current value to convert the new | ||
| 656 | * value from to relative and absolute, and to convert the | ||
| 657 | * old value from absolute to relative. To set a process | ||
| 658 | * timer, we need a sample to balance the thread expiry | ||
| 659 | * times (in arm_timer). With an absolute time, we must | ||
| 660 | * check if it's already passed. In short, we need a sample. | ||
| 661 | */ | ||
| 662 | if (CPUCLOCK_PERTHREAD(timer->it_clock)) { | ||
| 663 | cpu_clock_sample(timer->it_clock, p, &val); | ||
| 664 | } else { | ||
| 665 | cpu_timer_sample_group(timer->it_clock, p, &val); | ||
| 666 | } | ||
| 667 | |||
| 668 | if (old) { | ||
| 669 | if (old_expires == 0) { | ||
| 670 | old->it_value.tv_sec = 0; | ||
| 671 | old->it_value.tv_nsec = 0; | ||
| 672 | } else { | ||
| 673 | /* | ||
| 674 | * Update the timer in case it has | ||
| 675 | * overrun already. If it has, | ||
| 676 | * we'll report it as having overrun | ||
| 677 | * and with the next reloaded timer | ||
| 678 | * already ticking, though we are | ||
| 679 | * swallowing that pending | ||
| 680 | * notification here to install the | ||
| 681 | * new setting. | ||
| 682 | */ | ||
| 683 | bump_cpu_timer(timer, val); | ||
| 684 | if (val < timer->it.cpu.expires) { | ||
| 685 | old_expires = timer->it.cpu.expires - val; | ||
| 686 | sample_to_timespec(timer->it_clock, | ||
| 687 | old_expires, | ||
| 688 | &old->it_value); | ||
| 689 | } else { | ||
| 690 | old->it_value.tv_nsec = 1; | ||
| 691 | old->it_value.tv_sec = 0; | ||
| 692 | } | ||
| 693 | } | ||
| 694 | } | ||
| 695 | |||
| 696 | if (unlikely(ret)) { | ||
| 697 | /* | ||
| 698 | * We are colliding with the timer actually firing. | ||
| 699 | * Punt after filling in the timer's old value, and | ||
| 700 | * disable this firing since we are already reporting | ||
| 701 | * it as an overrun (thanks to bump_cpu_timer above). | ||
| 702 | */ | ||
| 703 | unlock_task_sighand(p, &flags); | ||
| 704 | goto out; | ||
| 705 | } | ||
| 706 | |||
| 707 | if (new_expires != 0 && !(timer_flags & TIMER_ABSTIME)) { | ||
| 708 | new_expires += val; | ||
| 709 | } | ||
| 710 | |||
| 711 | /* | ||
| 712 | * Install the new expiry time (or zero). | ||
| 713 | * For a timer with no notification action, we don't actually | ||
| 714 | * arm the timer (we'll just fake it for timer_gettime). | ||
| 715 | */ | ||
| 716 | timer->it.cpu.expires = new_expires; | ||
| 717 | if (new_expires != 0 && val < new_expires) { | ||
| 718 | arm_timer(timer); | ||
| 719 | } | ||
| 720 | |||
| 721 | unlock_task_sighand(p, &flags); | ||
| 722 | /* | ||
| 723 | * Install the new reload setting, and | ||
| 724 | * set up the signal and overrun bookkeeping. | ||
| 725 | */ | ||
| 726 | timer->it.cpu.incr = timespec_to_sample(timer->it_clock, | ||
| 727 | &new->it_interval); | ||
| 728 | |||
| 729 | /* | ||
| 730 | * This acts as a modification timestamp for the timer, | ||
| 731 | * so any automatic reload attempt will punt on seeing | ||
| 732 | * that we have reset the timer manually. | ||
| 733 | */ | ||
| 734 | timer->it_requeue_pending = (timer->it_requeue_pending + 2) & | ||
| 735 | ~REQUEUE_PENDING; | ||
| 736 | timer->it_overrun_last = 0; | ||
| 737 | timer->it_overrun = -1; | ||
| 738 | |||
| 739 | if (new_expires != 0 && !(val < new_expires)) { | ||
| 740 | /* | ||
| 741 | * The designated time already passed, so we notify | ||
| 742 | * immediately, even if the thread never runs to | ||
| 743 | * accumulate more time on this clock. | ||
| 744 | */ | ||
| 745 | cpu_timer_fire(timer); | ||
| 746 | } | ||
| 747 | |||
| 748 | ret = 0; | ||
| 749 | out: | ||
| 750 | if (old) { | ||
| 751 | sample_to_timespec(timer->it_clock, | ||
| 752 | old_incr, &old->it_interval); | ||
| 753 | } | ||
| 754 | if (!ret) | ||
| 755 | posix_cpu_timer_kick_nohz(); | ||
| 756 | return ret; | ||
| 757 | } | ||
| 758 | |||
| 759 | static void posix_cpu_timer_get(struct k_itimer *timer, struct itimerspec *itp) | ||
| 760 | { | ||
| 761 | unsigned long long now; | ||
| 762 | struct task_struct *p = timer->it.cpu.task; | ||
| 763 | |||
| 764 | WARN_ON_ONCE(p == NULL); | ||
| 765 | |||
| 766 | /* | ||
| 767 | * Easy part: convert the reload time. | ||
| 768 | */ | ||
| 769 | sample_to_timespec(timer->it_clock, | ||
| 770 | timer->it.cpu.incr, &itp->it_interval); | ||
| 771 | |||
| 772 | if (timer->it.cpu.expires == 0) { /* Timer not armed at all. */ | ||
| 773 | itp->it_value.tv_sec = itp->it_value.tv_nsec = 0; | ||
| 774 | return; | ||
| 775 | } | ||
| 776 | |||
| 777 | /* | ||
| 778 | * Sample the clock to take the difference with the expiry time. | ||
| 779 | */ | ||
| 780 | if (CPUCLOCK_PERTHREAD(timer->it_clock)) { | ||
| 781 | cpu_clock_sample(timer->it_clock, p, &now); | ||
| 782 | } else { | ||
| 783 | struct sighand_struct *sighand; | ||
| 784 | unsigned long flags; | ||
| 785 | |||
| 786 | /* | ||
| 787 | * Protect against sighand release/switch in exit/exec and | ||
| 788 | * also make timer sampling safe if it ends up calling | ||
| 789 | * thread_group_cputime(). | ||
| 790 | */ | ||
| 791 | sighand = lock_task_sighand(p, &flags); | ||
| 792 | if (unlikely(sighand == NULL)) { | ||
| 793 | /* | ||
| 794 | * The process has been reaped. | ||
| 795 | * We can't even collect a sample any more. | ||
| 796 | * Call the timer disarmed, nothing else to do. | ||
| 797 | */ | ||
| 798 | timer->it.cpu.expires = 0; | ||
| 799 | sample_to_timespec(timer->it_clock, timer->it.cpu.expires, | ||
| 800 | &itp->it_value); | ||
| 801 | } else { | ||
| 802 | cpu_timer_sample_group(timer->it_clock, p, &now); | ||
| 803 | unlock_task_sighand(p, &flags); | ||
| 804 | } | ||
| 805 | } | ||
| 806 | |||
| 807 | if (now < timer->it.cpu.expires) { | ||
| 808 | sample_to_timespec(timer->it_clock, | ||
| 809 | timer->it.cpu.expires - now, | ||
| 810 | &itp->it_value); | ||
| 811 | } else { | ||
| 812 | /* | ||
| 813 | * The timer should have expired already, but the firing | ||
| 814 | * hasn't taken place yet. Say it's just about to expire. | ||
| 815 | */ | ||
| 816 | itp->it_value.tv_nsec = 1; | ||
| 817 | itp->it_value.tv_sec = 0; | ||
| 818 | } | ||
| 819 | } | ||
| 820 | |||
| 821 | static unsigned long long | ||
| 822 | check_timers_list(struct list_head *timers, | ||
| 823 | struct list_head *firing, | ||
| 824 | unsigned long long curr) | ||
| 825 | { | ||
| 826 | int maxfire = 20; | ||
| 827 | |||
| 828 | while (!list_empty(timers)) { | ||
| 829 | struct cpu_timer_list *t; | ||
| 830 | |||
| 831 | t = list_first_entry(timers, struct cpu_timer_list, entry); | ||
| 832 | |||
| 833 | if (!--maxfire || curr < t->expires) | ||
| 834 | return t->expires; | ||
| 835 | |||
| 836 | t->firing = 1; | ||
| 837 | list_move_tail(&t->entry, firing); | ||
| 838 | } | ||
| 839 | |||
| 840 | return 0; | ||
| 841 | } | ||
| 842 | |||
| 843 | /* | ||
| 844 | * Check for any per-thread CPU timers that have fired and move them off | ||
| 845 | * the tsk->cpu_timers[N] list onto the firing list. Here we update the | ||
| 846 | * tsk->it_*_expires values to reflect the remaining thread CPU timers. | ||
| 847 | */ | ||
| 848 | static void check_thread_timers(struct task_struct *tsk, | ||
| 849 | struct list_head *firing) | ||
| 850 | { | ||
| 851 | struct list_head *timers = tsk->cpu_timers; | ||
| 852 | struct signal_struct *const sig = tsk->signal; | ||
| 853 | struct task_cputime *tsk_expires = &tsk->cputime_expires; | ||
| 854 | unsigned long long expires; | ||
| 855 | unsigned long soft; | ||
| 856 | |||
| 857 | expires = check_timers_list(timers, firing, prof_ticks(tsk)); | ||
| 858 | tsk_expires->prof_exp = expires_to_cputime(expires); | ||
| 859 | |||
| 860 | expires = check_timers_list(++timers, firing, virt_ticks(tsk)); | ||
| 861 | tsk_expires->virt_exp = expires_to_cputime(expires); | ||
| 862 | |||
| 863 | tsk_expires->sched_exp = check_timers_list(++timers, firing, | ||
| 864 | tsk->se.sum_exec_runtime); | ||
| 865 | |||
| 866 | /* | ||
| 867 | * Check for the special case thread timers. | ||
| 868 | */ | ||
| 869 | soft = ACCESS_ONCE(sig->rlim[RLIMIT_RTTIME].rlim_cur); | ||
| 870 | if (soft != RLIM_INFINITY) { | ||
| 871 | unsigned long hard = | ||
| 872 | ACCESS_ONCE(sig->rlim[RLIMIT_RTTIME].rlim_max); | ||
| 873 | |||
| 874 | if (hard != RLIM_INFINITY && | ||
| 875 | tsk->rt.timeout > DIV_ROUND_UP(hard, USEC_PER_SEC/HZ)) { | ||
| 876 | /* | ||
| 877 | * At the hard limit, we just die. | ||
| 878 | * No need to calculate anything else now. | ||
| 879 | */ | ||
| 880 | __group_send_sig_info(SIGKILL, SEND_SIG_PRIV, tsk); | ||
| 881 | return; | ||
| 882 | } | ||
| 883 | if (tsk->rt.timeout > DIV_ROUND_UP(soft, USEC_PER_SEC/HZ)) { | ||
| 884 | /* | ||
| 885 | * At the soft limit, send a SIGXCPU every second. | ||
| 886 | */ | ||
| 887 | if (soft < hard) { | ||
| 888 | soft += USEC_PER_SEC; | ||
| 889 | sig->rlim[RLIMIT_RTTIME].rlim_cur = soft; | ||
| 890 | } | ||
| 891 | printk(KERN_INFO | ||
| 892 | "RT Watchdog Timeout: %s[%d]\n", | ||
| 893 | tsk->comm, task_pid_nr(tsk)); | ||
| 894 | __group_send_sig_info(SIGXCPU, SEND_SIG_PRIV, tsk); | ||
| 895 | } | ||
| 896 | } | ||
| 897 | } | ||
| 898 | |||
| 899 | static void stop_process_timers(struct signal_struct *sig) | ||
| 900 | { | ||
| 901 | struct thread_group_cputimer *cputimer = &sig->cputimer; | ||
| 902 | unsigned long flags; | ||
| 903 | |||
| 904 | raw_spin_lock_irqsave(&cputimer->lock, flags); | ||
| 905 | cputimer->running = 0; | ||
| 906 | raw_spin_unlock_irqrestore(&cputimer->lock, flags); | ||
| 907 | } | ||
| 908 | |||
| 909 | static u32 onecputick; | ||
| 910 | |||
| 911 | static void check_cpu_itimer(struct task_struct *tsk, struct cpu_itimer *it, | ||
| 912 | unsigned long long *expires, | ||
| 913 | unsigned long long cur_time, int signo) | ||
| 914 | { | ||
| 915 | if (!it->expires) | ||
| 916 | return; | ||
| 917 | |||
| 918 | if (cur_time >= it->expires) { | ||
| 919 | if (it->incr) { | ||
| 920 | it->expires += it->incr; | ||
| 921 | it->error += it->incr_error; | ||
| 922 | if (it->error >= onecputick) { | ||
| 923 | it->expires -= cputime_one_jiffy; | ||
| 924 | it->error -= onecputick; | ||
| 925 | } | ||
| 926 | } else { | ||
| 927 | it->expires = 0; | ||
| 928 | } | ||
| 929 | |||
| 930 | trace_itimer_expire(signo == SIGPROF ? | ||
| 931 | ITIMER_PROF : ITIMER_VIRTUAL, | ||
| 932 | tsk->signal->leader_pid, cur_time); | ||
| 933 | __group_send_sig_info(signo, SEND_SIG_PRIV, tsk); | ||
| 934 | } | ||
| 935 | |||
| 936 | if (it->expires && (!*expires || it->expires < *expires)) { | ||
| 937 | *expires = it->expires; | ||
| 938 | } | ||
| 939 | } | ||
| 940 | |||
| 941 | /* | ||
| 942 | * Check for any per-thread CPU timers that have fired and move them | ||
| 943 | * off the tsk->*_timers list onto the firing list. Per-thread timers | ||
| 944 | * have already been taken off. | ||
| 945 | */ | ||
| 946 | static void check_process_timers(struct task_struct *tsk, | ||
| 947 | struct list_head *firing) | ||
| 948 | { | ||
| 949 | struct signal_struct *const sig = tsk->signal; | ||
| 950 | unsigned long long utime, ptime, virt_expires, prof_expires; | ||
| 951 | unsigned long long sum_sched_runtime, sched_expires; | ||
| 952 | struct list_head *timers = sig->cpu_timers; | ||
| 953 | struct task_cputime cputime; | ||
| 954 | unsigned long soft; | ||
| 955 | |||
| 956 | /* | ||
| 957 | * Collect the current process totals. | ||
| 958 | */ | ||
| 959 | thread_group_cputimer(tsk, &cputime); | ||
| 960 | utime = cputime_to_expires(cputime.utime); | ||
| 961 | ptime = utime + cputime_to_expires(cputime.stime); | ||
| 962 | sum_sched_runtime = cputime.sum_exec_runtime; | ||
| 963 | |||
| 964 | prof_expires = check_timers_list(timers, firing, ptime); | ||
| 965 | virt_expires = check_timers_list(++timers, firing, utime); | ||
| 966 | sched_expires = check_timers_list(++timers, firing, sum_sched_runtime); | ||
| 967 | |||
| 968 | /* | ||
| 969 | * Check for the special case process timers. | ||
| 970 | */ | ||
| 971 | check_cpu_itimer(tsk, &sig->it[CPUCLOCK_PROF], &prof_expires, ptime, | ||
| 972 | SIGPROF); | ||
| 973 | check_cpu_itimer(tsk, &sig->it[CPUCLOCK_VIRT], &virt_expires, utime, | ||
| 974 | SIGVTALRM); | ||
| 975 | soft = ACCESS_ONCE(sig->rlim[RLIMIT_CPU].rlim_cur); | ||
| 976 | if (soft != RLIM_INFINITY) { | ||
| 977 | unsigned long psecs = cputime_to_secs(ptime); | ||
| 978 | unsigned long hard = | ||
| 979 | ACCESS_ONCE(sig->rlim[RLIMIT_CPU].rlim_max); | ||
| 980 | cputime_t x; | ||
| 981 | if (psecs >= hard) { | ||
| 982 | /* | ||
| 983 | * At the hard limit, we just die. | ||
| 984 | * No need to calculate anything else now. | ||
| 985 | */ | ||
| 986 | __group_send_sig_info(SIGKILL, SEND_SIG_PRIV, tsk); | ||
| 987 | return; | ||
| 988 | } | ||
| 989 | if (psecs >= soft) { | ||
| 990 | /* | ||
| 991 | * At the soft limit, send a SIGXCPU every second. | ||
| 992 | */ | ||
| 993 | __group_send_sig_info(SIGXCPU, SEND_SIG_PRIV, tsk); | ||
| 994 | if (soft < hard) { | ||
| 995 | soft++; | ||
| 996 | sig->rlim[RLIMIT_CPU].rlim_cur = soft; | ||
| 997 | } | ||
| 998 | } | ||
| 999 | x = secs_to_cputime(soft); | ||
| 1000 | if (!prof_expires || x < prof_expires) { | ||
| 1001 | prof_expires = x; | ||
| 1002 | } | ||
| 1003 | } | ||
| 1004 | |||
| 1005 | sig->cputime_expires.prof_exp = expires_to_cputime(prof_expires); | ||
| 1006 | sig->cputime_expires.virt_exp = expires_to_cputime(virt_expires); | ||
| 1007 | sig->cputime_expires.sched_exp = sched_expires; | ||
| 1008 | if (task_cputime_zero(&sig->cputime_expires)) | ||
| 1009 | stop_process_timers(sig); | ||
| 1010 | } | ||
| 1011 | |||
| 1012 | /* | ||
| 1013 | * This is called from the signal code (via do_schedule_next_timer) | ||
| 1014 | * when the last timer signal was delivered and we have to reload the timer. | ||
| 1015 | */ | ||
| 1016 | void posix_cpu_timer_schedule(struct k_itimer *timer) | ||
| 1017 | { | ||
| 1018 | struct sighand_struct *sighand; | ||
| 1019 | unsigned long flags; | ||
| 1020 | struct task_struct *p = timer->it.cpu.task; | ||
| 1021 | unsigned long long now; | ||
| 1022 | |||
| 1023 | WARN_ON_ONCE(p == NULL); | ||
| 1024 | |||
| 1025 | /* | ||
| 1026 | * Fetch the current sample and update the timer's expiry time. | ||
| 1027 | */ | ||
| 1028 | if (CPUCLOCK_PERTHREAD(timer->it_clock)) { | ||
| 1029 | cpu_clock_sample(timer->it_clock, p, &now); | ||
| 1030 | bump_cpu_timer(timer, now); | ||
| 1031 | if (unlikely(p->exit_state)) | ||
| 1032 | goto out; | ||
| 1033 | |||
| 1034 | /* Protect timer list r/w in arm_timer() */ | ||
| 1035 | sighand = lock_task_sighand(p, &flags); | ||
| 1036 | if (!sighand) | ||
| 1037 | goto out; | ||
| 1038 | } else { | ||
| 1039 | /* | ||
| 1040 | * Protect arm_timer() and timer sampling in case of call to | ||
| 1041 | * thread_group_cputime(). | ||
| 1042 | */ | ||
| 1043 | sighand = lock_task_sighand(p, &flags); | ||
| 1044 | if (unlikely(sighand == NULL)) { | ||
| 1045 | /* | ||
| 1046 | * The process has been reaped. | ||
| 1047 | * We can't even collect a sample any more. | ||
| 1048 | */ | ||
| 1049 | timer->it.cpu.expires = 0; | ||
| 1050 | goto out; | ||
| 1051 | } else if (unlikely(p->exit_state) && thread_group_empty(p)) { | ||
| 1052 | unlock_task_sighand(p, &flags); | ||
| 1053 | /* Optimizations: if the process is dying, no need to rearm */ | ||
| 1054 | goto out; | ||
| 1055 | } | ||
| 1056 | cpu_timer_sample_group(timer->it_clock, p, &now); | ||
| 1057 | bump_cpu_timer(timer, now); | ||
| 1058 | /* Leave the sighand locked for the call below. */ | ||
| 1059 | } | ||
| 1060 | |||
| 1061 | /* | ||
| 1062 | * Now re-arm for the new expiry time. | ||
| 1063 | */ | ||
| 1064 | WARN_ON_ONCE(!irqs_disabled()); | ||
| 1065 | arm_timer(timer); | ||
| 1066 | unlock_task_sighand(p, &flags); | ||
| 1067 | |||
| 1068 | /* Kick full dynticks CPUs in case they need to tick on the new timer */ | ||
| 1069 | posix_cpu_timer_kick_nohz(); | ||
| 1070 | out: | ||
| 1071 | timer->it_overrun_last = timer->it_overrun; | ||
| 1072 | timer->it_overrun = -1; | ||
| 1073 | ++timer->it_requeue_pending; | ||
| 1074 | } | ||
| 1075 | |||
| 1076 | /** | ||
| 1077 | * task_cputime_expired - Compare two task_cputime entities. | ||
| 1078 | * | ||
| 1079 | * @sample: The task_cputime structure to be checked for expiration. | ||
| 1080 | * @expires: Expiration times, against which @sample will be checked. | ||
| 1081 | * | ||
| 1082 | * Checks @sample against @expires to see if any field of @sample has expired. | ||
| 1083 | * Returns true if any field of the former is greater than the corresponding | ||
| 1084 | * field of the latter if the latter field is set. Otherwise returns false. | ||
| 1085 | */ | ||
| 1086 | static inline int task_cputime_expired(const struct task_cputime *sample, | ||
| 1087 | const struct task_cputime *expires) | ||
| 1088 | { | ||
| 1089 | if (expires->utime && sample->utime >= expires->utime) | ||
| 1090 | return 1; | ||
| 1091 | if (expires->stime && sample->utime + sample->stime >= expires->stime) | ||
| 1092 | return 1; | ||
| 1093 | if (expires->sum_exec_runtime != 0 && | ||
| 1094 | sample->sum_exec_runtime >= expires->sum_exec_runtime) | ||
| 1095 | return 1; | ||
| 1096 | return 0; | ||
| 1097 | } | ||
| 1098 | |||
| 1099 | /** | ||
| 1100 | * fastpath_timer_check - POSIX CPU timers fast path. | ||
| 1101 | * | ||
| 1102 | * @tsk: The task (thread) being checked. | ||
| 1103 | * | ||
| 1104 | * Check the task and thread group timers. If both are zero (there are no | ||
| 1105 | * timers set) return false. Otherwise snapshot the task and thread group | ||
| 1106 | * timers and compare them with the corresponding expiration times. Return | ||
| 1107 | * true if a timer has expired, else return false. | ||
| 1108 | */ | ||
| 1109 | static inline int fastpath_timer_check(struct task_struct *tsk) | ||
| 1110 | { | ||
| 1111 | struct signal_struct *sig; | ||
| 1112 | cputime_t utime, stime; | ||
| 1113 | |||
| 1114 | task_cputime(tsk, &utime, &stime); | ||
| 1115 | |||
| 1116 | if (!task_cputime_zero(&tsk->cputime_expires)) { | ||
| 1117 | struct task_cputime task_sample = { | ||
| 1118 | .utime = utime, | ||
| 1119 | .stime = stime, | ||
| 1120 | .sum_exec_runtime = tsk->se.sum_exec_runtime | ||
| 1121 | }; | ||
| 1122 | |||
| 1123 | if (task_cputime_expired(&task_sample, &tsk->cputime_expires)) | ||
| 1124 | return 1; | ||
| 1125 | } | ||
| 1126 | |||
| 1127 | sig = tsk->signal; | ||
| 1128 | if (sig->cputimer.running) { | ||
| 1129 | struct task_cputime group_sample; | ||
| 1130 | |||
| 1131 | raw_spin_lock(&sig->cputimer.lock); | ||
| 1132 | group_sample = sig->cputimer.cputime; | ||
| 1133 | raw_spin_unlock(&sig->cputimer.lock); | ||
| 1134 | |||
| 1135 | if (task_cputime_expired(&group_sample, &sig->cputime_expires)) | ||
| 1136 | return 1; | ||
| 1137 | } | ||
| 1138 | |||
| 1139 | return 0; | ||
| 1140 | } | ||
| 1141 | |||
| 1142 | /* | ||
| 1143 | * This is called from the timer interrupt handler. The irq handler has | ||
| 1144 | * already updated our counts. We need to check if any timers fire now. | ||
| 1145 | * Interrupts are disabled. | ||
| 1146 | */ | ||
| 1147 | void run_posix_cpu_timers(struct task_struct *tsk) | ||
| 1148 | { | ||
| 1149 | LIST_HEAD(firing); | ||
| 1150 | struct k_itimer *timer, *next; | ||
| 1151 | unsigned long flags; | ||
| 1152 | |||
| 1153 | WARN_ON_ONCE(!irqs_disabled()); | ||
| 1154 | |||
| 1155 | /* | ||
| 1156 | * The fast path checks that there are no expired thread or thread | ||
| 1157 | * group timers. If that's so, just return. | ||
| 1158 | */ | ||
| 1159 | if (!fastpath_timer_check(tsk)) | ||
| 1160 | return; | ||
| 1161 | |||
| 1162 | if (!lock_task_sighand(tsk, &flags)) | ||
| 1163 | return; | ||
| 1164 | /* | ||
| 1165 | * Here we take off tsk->signal->cpu_timers[N] and | ||
| 1166 | * tsk->cpu_timers[N] all the timers that are firing, and | ||
| 1167 | * put them on the firing list. | ||
| 1168 | */ | ||
| 1169 | check_thread_timers(tsk, &firing); | ||
| 1170 | /* | ||
| 1171 | * If there are any active process wide timers (POSIX 1.b, itimers, | ||
| 1172 | * RLIMIT_CPU) cputimer must be running. | ||
| 1173 | */ | ||
| 1174 | if (tsk->signal->cputimer.running) | ||
| 1175 | check_process_timers(tsk, &firing); | ||
| 1176 | |||
| 1177 | /* | ||
| 1178 | * We must release these locks before taking any timer's lock. | ||
| 1179 | * There is a potential race with timer deletion here, as the | ||
| 1180 | * siglock now protects our private firing list. We have set | ||
| 1181 | * the firing flag in each timer, so that a deletion attempt | ||
| 1182 | * that gets the timer lock before we do will give it up and | ||
| 1183 | * spin until we've taken care of that timer below. | ||
| 1184 | */ | ||
| 1185 | unlock_task_sighand(tsk, &flags); | ||
| 1186 | |||
| 1187 | /* | ||
| 1188 | * Now that all the timers on our list have the firing flag, | ||
| 1189 | * no one will touch their list entries but us. We'll take | ||
| 1190 | * each timer's lock before clearing its firing flag, so no | ||
| 1191 | * timer call will interfere. | ||
| 1192 | */ | ||
| 1193 | list_for_each_entry_safe(timer, next, &firing, it.cpu.entry) { | ||
| 1194 | int cpu_firing; | ||
| 1195 | |||
| 1196 | spin_lock(&timer->it_lock); | ||
| 1197 | list_del_init(&timer->it.cpu.entry); | ||
| 1198 | cpu_firing = timer->it.cpu.firing; | ||
| 1199 | timer->it.cpu.firing = 0; | ||
| 1200 | /* | ||
| 1201 | * The firing flag is -1 if we collided with a reset | ||
| 1202 | * of the timer, which already reported this | ||
| 1203 | * almost-firing as an overrun. So don't generate an event. | ||
| 1204 | */ | ||
| 1205 | if (likely(cpu_firing >= 0)) | ||
| 1206 | cpu_timer_fire(timer); | ||
| 1207 | spin_unlock(&timer->it_lock); | ||
| 1208 | } | ||
| 1209 | } | ||
| 1210 | |||
| 1211 | /* | ||
| 1212 | * Set one of the process-wide special case CPU timers or RLIMIT_CPU. | ||
| 1213 | * The tsk->sighand->siglock must be held by the caller. | ||
| 1214 | */ | ||
| 1215 | void set_process_cpu_timer(struct task_struct *tsk, unsigned int clock_idx, | ||
| 1216 | cputime_t *newval, cputime_t *oldval) | ||
| 1217 | { | ||
| 1218 | unsigned long long now; | ||
| 1219 | |||
| 1220 | WARN_ON_ONCE(clock_idx == CPUCLOCK_SCHED); | ||
| 1221 | cpu_timer_sample_group(clock_idx, tsk, &now); | ||
| 1222 | |||
| 1223 | if (oldval) { | ||
| 1224 | /* | ||
| 1225 | * We are setting itimer. The *oldval is absolute and we update | ||
| 1226 | * it to be relative, *newval argument is relative and we update | ||
| 1227 | * it to be absolute. | ||
| 1228 | */ | ||
| 1229 | if (*oldval) { | ||
| 1230 | if (*oldval <= now) { | ||
| 1231 | /* Just about to fire. */ | ||
| 1232 | *oldval = cputime_one_jiffy; | ||
| 1233 | } else { | ||
| 1234 | *oldval -= now; | ||
| 1235 | } | ||
| 1236 | } | ||
| 1237 | |||
| 1238 | if (!*newval) | ||
| 1239 | goto out; | ||
| 1240 | *newval += now; | ||
| 1241 | } | ||
| 1242 | |||
| 1243 | /* | ||
| 1244 | * Update expiration cache if we are the earliest timer, or eventually | ||
| 1245 | * RLIMIT_CPU limit is earlier than prof_exp cpu timer expire. | ||
| 1246 | */ | ||
| 1247 | switch (clock_idx) { | ||
| 1248 | case CPUCLOCK_PROF: | ||
| 1249 | if (expires_gt(tsk->signal->cputime_expires.prof_exp, *newval)) | ||
| 1250 | tsk->signal->cputime_expires.prof_exp = *newval; | ||
| 1251 | break; | ||
| 1252 | case CPUCLOCK_VIRT: | ||
| 1253 | if (expires_gt(tsk->signal->cputime_expires.virt_exp, *newval)) | ||
| 1254 | tsk->signal->cputime_expires.virt_exp = *newval; | ||
| 1255 | break; | ||
| 1256 | } | ||
| 1257 | out: | ||
| 1258 | posix_cpu_timer_kick_nohz(); | ||
| 1259 | } | ||
| 1260 | |||
| 1261 | static int do_cpu_nanosleep(const clockid_t which_clock, int flags, | ||
| 1262 | struct timespec *rqtp, struct itimerspec *it) | ||
| 1263 | { | ||
| 1264 | struct k_itimer timer; | ||
| 1265 | int error; | ||
| 1266 | |||
| 1267 | /* | ||
| 1268 | * Set up a temporary timer and then wait for it to go off. | ||
| 1269 | */ | ||
| 1270 | memset(&timer, 0, sizeof timer); | ||
| 1271 | spin_lock_init(&timer.it_lock); | ||
| 1272 | timer.it_clock = which_clock; | ||
| 1273 | timer.it_overrun = -1; | ||
| 1274 | error = posix_cpu_timer_create(&timer); | ||
| 1275 | timer.it_process = current; | ||
| 1276 | if (!error) { | ||
| 1277 | static struct itimerspec zero_it; | ||
| 1278 | |||
| 1279 | memset(it, 0, sizeof *it); | ||
| 1280 | it->it_value = *rqtp; | ||
| 1281 | |||
| 1282 | spin_lock_irq(&timer.it_lock); | ||
| 1283 | error = posix_cpu_timer_set(&timer, flags, it, NULL); | ||
| 1284 | if (error) { | ||
| 1285 | spin_unlock_irq(&timer.it_lock); | ||
| 1286 | return error; | ||
| 1287 | } | ||
| 1288 | |||
| 1289 | while (!signal_pending(current)) { | ||
| 1290 | if (timer.it.cpu.expires == 0) { | ||
| 1291 | /* | ||
| 1292 | * Our timer fired and was reset, below | ||
| 1293 | * deletion can not fail. | ||
| 1294 | */ | ||
| 1295 | posix_cpu_timer_del(&timer); | ||
| 1296 | spin_unlock_irq(&timer.it_lock); | ||
| 1297 | return 0; | ||
| 1298 | } | ||
| 1299 | |||
| 1300 | /* | ||
| 1301 | * Block until cpu_timer_fire (or a signal) wakes us. | ||
| 1302 | */ | ||
| 1303 | __set_current_state(TASK_INTERRUPTIBLE); | ||
| 1304 | spin_unlock_irq(&timer.it_lock); | ||
| 1305 | schedule(); | ||
| 1306 | spin_lock_irq(&timer.it_lock); | ||
| 1307 | } | ||
| 1308 | |||
| 1309 | /* | ||
| 1310 | * We were interrupted by a signal. | ||
| 1311 | */ | ||
| 1312 | sample_to_timespec(which_clock, timer.it.cpu.expires, rqtp); | ||
| 1313 | error = posix_cpu_timer_set(&timer, 0, &zero_it, it); | ||
| 1314 | if (!error) { | ||
| 1315 | /* | ||
| 1316 | * Timer is now unarmed, deletion can not fail. | ||
| 1317 | */ | ||
| 1318 | posix_cpu_timer_del(&timer); | ||
| 1319 | } | ||
| 1320 | spin_unlock_irq(&timer.it_lock); | ||
| 1321 | |||
| 1322 | while (error == TIMER_RETRY) { | ||
| 1323 | /* | ||
| 1324 | * We need to handle case when timer was or is in the | ||
| 1325 | * middle of firing. In other cases we already freed | ||
| 1326 | * resources. | ||
| 1327 | */ | ||
| 1328 | spin_lock_irq(&timer.it_lock); | ||
| 1329 | error = posix_cpu_timer_del(&timer); | ||
| 1330 | spin_unlock_irq(&timer.it_lock); | ||
| 1331 | } | ||
| 1332 | |||
| 1333 | if ((it->it_value.tv_sec | it->it_value.tv_nsec) == 0) { | ||
| 1334 | /* | ||
| 1335 | * It actually did fire already. | ||
| 1336 | */ | ||
| 1337 | return 0; | ||
| 1338 | } | ||
| 1339 | |||
| 1340 | error = -ERESTART_RESTARTBLOCK; | ||
| 1341 | } | ||
| 1342 | |||
| 1343 | return error; | ||
| 1344 | } | ||
| 1345 | |||
| 1346 | static long posix_cpu_nsleep_restart(struct restart_block *restart_block); | ||
| 1347 | |||
| 1348 | static int posix_cpu_nsleep(const clockid_t which_clock, int flags, | ||
| 1349 | struct timespec *rqtp, struct timespec __user *rmtp) | ||
| 1350 | { | ||
| 1351 | struct restart_block *restart_block = | ||
| 1352 | ¤t_thread_info()->restart_block; | ||
| 1353 | struct itimerspec it; | ||
| 1354 | int error; | ||
| 1355 | |||
| 1356 | /* | ||
| 1357 | * Diagnose required errors first. | ||
| 1358 | */ | ||
| 1359 | if (CPUCLOCK_PERTHREAD(which_clock) && | ||
| 1360 | (CPUCLOCK_PID(which_clock) == 0 || | ||
| 1361 | CPUCLOCK_PID(which_clock) == current->pid)) | ||
| 1362 | return -EINVAL; | ||
| 1363 | |||
| 1364 | error = do_cpu_nanosleep(which_clock, flags, rqtp, &it); | ||
| 1365 | |||
| 1366 | if (error == -ERESTART_RESTARTBLOCK) { | ||
| 1367 | |||
| 1368 | if (flags & TIMER_ABSTIME) | ||
| 1369 | return -ERESTARTNOHAND; | ||
| 1370 | /* | ||
| 1371 | * Report back to the user the time still remaining. | ||
| 1372 | */ | ||
| 1373 | if (rmtp && copy_to_user(rmtp, &it.it_value, sizeof *rmtp)) | ||
| 1374 | return -EFAULT; | ||
| 1375 | |||
| 1376 | restart_block->fn = posix_cpu_nsleep_restart; | ||
| 1377 | restart_block->nanosleep.clockid = which_clock; | ||
| 1378 | restart_block->nanosleep.rmtp = rmtp; | ||
| 1379 | restart_block->nanosleep.expires = timespec_to_ns(rqtp); | ||
| 1380 | } | ||
| 1381 | return error; | ||
| 1382 | } | ||
| 1383 | |||
| 1384 | static long posix_cpu_nsleep_restart(struct restart_block *restart_block) | ||
| 1385 | { | ||
| 1386 | clockid_t which_clock = restart_block->nanosleep.clockid; | ||
| 1387 | struct timespec t; | ||
| 1388 | struct itimerspec it; | ||
| 1389 | int error; | ||
| 1390 | |||
| 1391 | t = ns_to_timespec(restart_block->nanosleep.expires); | ||
| 1392 | |||
| 1393 | error = do_cpu_nanosleep(which_clock, TIMER_ABSTIME, &t, &it); | ||
| 1394 | |||
| 1395 | if (error == -ERESTART_RESTARTBLOCK) { | ||
| 1396 | struct timespec __user *rmtp = restart_block->nanosleep.rmtp; | ||
| 1397 | /* | ||
| 1398 | * Report back to the user the time still remaining. | ||
| 1399 | */ | ||
| 1400 | if (rmtp && copy_to_user(rmtp, &it.it_value, sizeof *rmtp)) | ||
| 1401 | return -EFAULT; | ||
| 1402 | |||
| 1403 | restart_block->nanosleep.expires = timespec_to_ns(&t); | ||
| 1404 | } | ||
| 1405 | return error; | ||
| 1406 | |||
| 1407 | } | ||
| 1408 | |||
| 1409 | #define PROCESS_CLOCK MAKE_PROCESS_CPUCLOCK(0, CPUCLOCK_SCHED) | ||
| 1410 | #define THREAD_CLOCK MAKE_THREAD_CPUCLOCK(0, CPUCLOCK_SCHED) | ||
| 1411 | |||
| 1412 | static int process_cpu_clock_getres(const clockid_t which_clock, | ||
| 1413 | struct timespec *tp) | ||
| 1414 | { | ||
| 1415 | return posix_cpu_clock_getres(PROCESS_CLOCK, tp); | ||
| 1416 | } | ||
| 1417 | static int process_cpu_clock_get(const clockid_t which_clock, | ||
| 1418 | struct timespec *tp) | ||
| 1419 | { | ||
| 1420 | return posix_cpu_clock_get(PROCESS_CLOCK, tp); | ||
| 1421 | } | ||
| 1422 | static int process_cpu_timer_create(struct k_itimer *timer) | ||
| 1423 | { | ||
| 1424 | timer->it_clock = PROCESS_CLOCK; | ||
| 1425 | return posix_cpu_timer_create(timer); | ||
| 1426 | } | ||
| 1427 | static int process_cpu_nsleep(const clockid_t which_clock, int flags, | ||
| 1428 | struct timespec *rqtp, | ||
| 1429 | struct timespec __user *rmtp) | ||
| 1430 | { | ||
| 1431 | return posix_cpu_nsleep(PROCESS_CLOCK, flags, rqtp, rmtp); | ||
| 1432 | } | ||
| 1433 | static long process_cpu_nsleep_restart(struct restart_block *restart_block) | ||
| 1434 | { | ||
| 1435 | return -EINVAL; | ||
| 1436 | } | ||
| 1437 | static int thread_cpu_clock_getres(const clockid_t which_clock, | ||
| 1438 | struct timespec *tp) | ||
| 1439 | { | ||
| 1440 | return posix_cpu_clock_getres(THREAD_CLOCK, tp); | ||
| 1441 | } | ||
| 1442 | static int thread_cpu_clock_get(const clockid_t which_clock, | ||
| 1443 | struct timespec *tp) | ||
| 1444 | { | ||
| 1445 | return posix_cpu_clock_get(THREAD_CLOCK, tp); | ||
| 1446 | } | ||
| 1447 | static int thread_cpu_timer_create(struct k_itimer *timer) | ||
| 1448 | { | ||
| 1449 | timer->it_clock = THREAD_CLOCK; | ||
| 1450 | return posix_cpu_timer_create(timer); | ||
| 1451 | } | ||
| 1452 | |||
| 1453 | struct k_clock clock_posix_cpu = { | ||
| 1454 | .clock_getres = posix_cpu_clock_getres, | ||
| 1455 | .clock_set = posix_cpu_clock_set, | ||
| 1456 | .clock_get = posix_cpu_clock_get, | ||
| 1457 | .timer_create = posix_cpu_timer_create, | ||
| 1458 | .nsleep = posix_cpu_nsleep, | ||
| 1459 | .nsleep_restart = posix_cpu_nsleep_restart, | ||
| 1460 | .timer_set = posix_cpu_timer_set, | ||
| 1461 | .timer_del = posix_cpu_timer_del, | ||
| 1462 | .timer_get = posix_cpu_timer_get, | ||
| 1463 | }; | ||
| 1464 | |||
| 1465 | static __init int init_posix_cpu_timers(void) | ||
| 1466 | { | ||
| 1467 | struct k_clock process = { | ||
| 1468 | .clock_getres = process_cpu_clock_getres, | ||
| 1469 | .clock_get = process_cpu_clock_get, | ||
| 1470 | .timer_create = process_cpu_timer_create, | ||
| 1471 | .nsleep = process_cpu_nsleep, | ||
| 1472 | .nsleep_restart = process_cpu_nsleep_restart, | ||
| 1473 | }; | ||
| 1474 | struct k_clock thread = { | ||
| 1475 | .clock_getres = thread_cpu_clock_getres, | ||
| 1476 | .clock_get = thread_cpu_clock_get, | ||
| 1477 | .timer_create = thread_cpu_timer_create, | ||
| 1478 | }; | ||
| 1479 | struct timespec ts; | ||
| 1480 | |||
| 1481 | posix_timers_register_clock(CLOCK_PROCESS_CPUTIME_ID, &process); | ||
| 1482 | posix_timers_register_clock(CLOCK_THREAD_CPUTIME_ID, &thread); | ||
| 1483 | |||
| 1484 | cputime_to_timespec(cputime_one_jiffy, &ts); | ||
| 1485 | onecputick = ts.tv_nsec; | ||
| 1486 | WARN_ON(ts.tv_sec != 0); | ||
| 1487 | |||
| 1488 | return 0; | ||
| 1489 | } | ||
| 1490 | __initcall(init_posix_cpu_timers); | ||
diff --git a/kernel/time/posix-timers.c b/kernel/time/posix-timers.c new file mode 100644 index 000000000000..424c2d4265c9 --- /dev/null +++ b/kernel/time/posix-timers.c | |||
| @@ -0,0 +1,1121 @@ | |||
| 1 | /* | ||
| 2 | * linux/kernel/posix-timers.c | ||
| 3 | * | ||
| 4 | * | ||
| 5 | * 2002-10-15 Posix Clocks & timers | ||
| 6 | * by George Anzinger george@mvista.com | ||
| 7 | * | ||
| 8 | * Copyright (C) 2002 2003 by MontaVista Software. | ||
| 9 | * | ||
| 10 | * 2004-06-01 Fix CLOCK_REALTIME clock/timer TIMER_ABSTIME bug. | ||
| 11 | * Copyright (C) 2004 Boris Hu | ||
| 12 | * | ||
| 13 | * This program is free software; you can redistribute it and/or modify | ||
| 14 | * it under the terms of the GNU General Public License as published by | ||
| 15 | * the Free Software Foundation; either version 2 of the License, or (at | ||
| 16 | * your option) any later version. | ||
| 17 | * | ||
| 18 | * This program is distributed in the hope that it will be useful, but | ||
| 19 | * WITHOUT ANY WARRANTY; without even the implied warranty of | ||
| 20 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU | ||
| 21 | * General Public License for more details. | ||
| 22 | |||
| 23 | * You should have received a copy of the GNU General Public License | ||
| 24 | * along with this program; if not, write to the Free Software | ||
| 25 | * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. | ||
| 26 | * | ||
| 27 | * MontaVista Software | 1237 East Arques Avenue | Sunnyvale | CA 94085 | USA | ||
| 28 | */ | ||
| 29 | |||
| 30 | /* These are all the functions necessary to implement | ||
| 31 | * POSIX clocks & timers | ||
| 32 | */ | ||
| 33 | #include <linux/mm.h> | ||
| 34 | #include <linux/interrupt.h> | ||
| 35 | #include <linux/slab.h> | ||
| 36 | #include <linux/time.h> | ||
| 37 | #include <linux/mutex.h> | ||
| 38 | |||
| 39 | #include <asm/uaccess.h> | ||
| 40 | #include <linux/list.h> | ||
| 41 | #include <linux/init.h> | ||
| 42 | #include <linux/compiler.h> | ||
| 43 | #include <linux/hash.h> | ||
| 44 | #include <linux/posix-clock.h> | ||
| 45 | #include <linux/posix-timers.h> | ||
| 46 | #include <linux/syscalls.h> | ||
| 47 | #include <linux/wait.h> | ||
| 48 | #include <linux/workqueue.h> | ||
| 49 | #include <linux/export.h> | ||
| 50 | #include <linux/hashtable.h> | ||
| 51 | |||
| 52 | /* | ||
| 53 | * Management arrays for POSIX timers. Timers are now kept in static hash table | ||
| 54 | * with 512 entries. | ||
| 55 | * Timer ids are allocated by local routine, which selects proper hash head by | ||
| 56 | * key, constructed from current->signal address and per signal struct counter. | ||
| 57 | * This keeps timer ids unique per process, but now they can intersect between | ||
| 58 | * processes. | ||
| 59 | */ | ||
| 60 | |||
| 61 | /* | ||
| 62 | * Lets keep our timers in a slab cache :-) | ||
| 63 | */ | ||
| 64 | static struct kmem_cache *posix_timers_cache; | ||
| 65 | |||
| 66 | static DEFINE_HASHTABLE(posix_timers_hashtable, 9); | ||
| 67 | static DEFINE_SPINLOCK(hash_lock); | ||
| 68 | |||
| 69 | /* | ||
| 70 | * we assume that the new SIGEV_THREAD_ID shares no bits with the other | ||
| 71 | * SIGEV values. Here we put out an error if this assumption fails. | ||
| 72 | */ | ||
| 73 | #if SIGEV_THREAD_ID != (SIGEV_THREAD_ID & \ | ||
| 74 | ~(SIGEV_SIGNAL | SIGEV_NONE | SIGEV_THREAD)) | ||
| 75 | #error "SIGEV_THREAD_ID must not share bit with other SIGEV values!" | ||
| 76 | #endif | ||
| 77 | |||
| 78 | /* | ||
| 79 | * parisc wants ENOTSUP instead of EOPNOTSUPP | ||
| 80 | */ | ||
| 81 | #ifndef ENOTSUP | ||
| 82 | # define ENANOSLEEP_NOTSUP EOPNOTSUPP | ||
| 83 | #else | ||
| 84 | # define ENANOSLEEP_NOTSUP ENOTSUP | ||
| 85 | #endif | ||
| 86 | |||
| 87 | /* | ||
| 88 | * The timer ID is turned into a timer address by idr_find(). | ||
| 89 | * Verifying a valid ID consists of: | ||
| 90 | * | ||
| 91 | * a) checking that idr_find() returns other than -1. | ||
| 92 | * b) checking that the timer id matches the one in the timer itself. | ||
| 93 | * c) that the timer owner is in the callers thread group. | ||
| 94 | */ | ||
| 95 | |||
| 96 | /* | ||
| 97 | * CLOCKs: The POSIX standard calls for a couple of clocks and allows us | ||
| 98 | * to implement others. This structure defines the various | ||
| 99 | * clocks. | ||
| 100 | * | ||
| 101 | * RESOLUTION: Clock resolution is used to round up timer and interval | ||
| 102 | * times, NOT to report clock times, which are reported with as | ||
| 103 | * much resolution as the system can muster. In some cases this | ||
| 104 | * resolution may depend on the underlying clock hardware and | ||
| 105 | * may not be quantifiable until run time, and only then is the | ||
| 106 | * necessary code is written. The standard says we should say | ||
| 107 | * something about this issue in the documentation... | ||
| 108 | * | ||
| 109 | * FUNCTIONS: The CLOCKs structure defines possible functions to | ||
| 110 | * handle various clock functions. | ||
| 111 | * | ||
| 112 | * The standard POSIX timer management code assumes the | ||
| 113 | * following: 1.) The k_itimer struct (sched.h) is used for | ||
| 114 | * the timer. 2.) The list, it_lock, it_clock, it_id and | ||
| 115 | * it_pid fields are not modified by timer code. | ||
| 116 | * | ||
| 117 | * Permissions: It is assumed that the clock_settime() function defined | ||
| 118 | * for each clock will take care of permission checks. Some | ||
| 119 | * clocks may be set able by any user (i.e. local process | ||
| 120 | * clocks) others not. Currently the only set able clock we | ||
| 121 | * have is CLOCK_REALTIME and its high res counter part, both of | ||
| 122 | * which we beg off on and pass to do_sys_settimeofday(). | ||
| 123 | */ | ||
| 124 | |||
| 125 | static struct k_clock posix_clocks[MAX_CLOCKS]; | ||
| 126 | |||
| 127 | /* | ||
| 128 | * These ones are defined below. | ||
| 129 | */ | ||
| 130 | static int common_nsleep(const clockid_t, int flags, struct timespec *t, | ||
| 131 | struct timespec __user *rmtp); | ||
| 132 | static int common_timer_create(struct k_itimer *new_timer); | ||
| 133 | static void common_timer_get(struct k_itimer *, struct itimerspec *); | ||
| 134 | static int common_timer_set(struct k_itimer *, int, | ||
| 135 | struct itimerspec *, struct itimerspec *); | ||
| 136 | static int common_timer_del(struct k_itimer *timer); | ||
| 137 | |||
| 138 | static enum hrtimer_restart posix_timer_fn(struct hrtimer *data); | ||
| 139 | |||
| 140 | static struct k_itimer *__lock_timer(timer_t timer_id, unsigned long *flags); | ||
| 141 | |||
| 142 | #define lock_timer(tid, flags) \ | ||
| 143 | ({ struct k_itimer *__timr; \ | ||
| 144 | __cond_lock(&__timr->it_lock, __timr = __lock_timer(tid, flags)); \ | ||
| 145 | __timr; \ | ||
| 146 | }) | ||
| 147 | |||
| 148 | static int hash(struct signal_struct *sig, unsigned int nr) | ||
| 149 | { | ||
| 150 | return hash_32(hash32_ptr(sig) ^ nr, HASH_BITS(posix_timers_hashtable)); | ||
| 151 | } | ||
| 152 | |||
| 153 | static struct k_itimer *__posix_timers_find(struct hlist_head *head, | ||
| 154 | struct signal_struct *sig, | ||
| 155 | timer_t id) | ||
| 156 | { | ||
| 157 | struct k_itimer *timer; | ||
| 158 | |||
| 159 | hlist_for_each_entry_rcu(timer, head, t_hash) { | ||
| 160 | if ((timer->it_signal == sig) && (timer->it_id == id)) | ||
| 161 | return timer; | ||
| 162 | } | ||
| 163 | return NULL; | ||
| 164 | } | ||
| 165 | |||
| 166 | static struct k_itimer *posix_timer_by_id(timer_t id) | ||
| 167 | { | ||
| 168 | struct signal_struct *sig = current->signal; | ||
| 169 | struct hlist_head *head = &posix_timers_hashtable[hash(sig, id)]; | ||
| 170 | |||
| 171 | return __posix_timers_find(head, sig, id); | ||
| 172 | } | ||
| 173 | |||
| 174 | static int posix_timer_add(struct k_itimer *timer) | ||
| 175 | { | ||
| 176 | struct signal_struct *sig = current->signal; | ||
| 177 | int first_free_id = sig->posix_timer_id; | ||
| 178 | struct hlist_head *head; | ||
| 179 | int ret = -ENOENT; | ||
| 180 | |||
| 181 | do { | ||
| 182 | spin_lock(&hash_lock); | ||
| 183 | head = &posix_timers_hashtable[hash(sig, sig->posix_timer_id)]; | ||
| 184 | if (!__posix_timers_find(head, sig, sig->posix_timer_id)) { | ||
| 185 | hlist_add_head_rcu(&timer->t_hash, head); | ||
| 186 | ret = sig->posix_timer_id; | ||
| 187 | } | ||
| 188 | if (++sig->posix_timer_id < 0) | ||
| 189 | sig->posix_timer_id = 0; | ||
| 190 | if ((sig->posix_timer_id == first_free_id) && (ret == -ENOENT)) | ||
| 191 | /* Loop over all possible ids completed */ | ||
| 192 | ret = -EAGAIN; | ||
| 193 | spin_unlock(&hash_lock); | ||
| 194 | } while (ret == -ENOENT); | ||
| 195 | return ret; | ||
| 196 | } | ||
| 197 | |||
| 198 | static inline void unlock_timer(struct k_itimer *timr, unsigned long flags) | ||
| 199 | { | ||
| 200 | spin_unlock_irqrestore(&timr->it_lock, flags); | ||
| 201 | } | ||
| 202 | |||
| 203 | /* Get clock_realtime */ | ||
| 204 | static int posix_clock_realtime_get(clockid_t which_clock, struct timespec *tp) | ||
| 205 | { | ||
| 206 | ktime_get_real_ts(tp); | ||
| 207 | return 0; | ||
| 208 | } | ||
| 209 | |||
| 210 | /* Set clock_realtime */ | ||
| 211 | static int posix_clock_realtime_set(const clockid_t which_clock, | ||
| 212 | const struct timespec *tp) | ||
| 213 | { | ||
| 214 | return do_sys_settimeofday(tp, NULL); | ||
| 215 | } | ||
| 216 | |||
| 217 | static int posix_clock_realtime_adj(const clockid_t which_clock, | ||
| 218 | struct timex *t) | ||
| 219 | { | ||
| 220 | return do_adjtimex(t); | ||
| 221 | } | ||
| 222 | |||
| 223 | /* | ||
| 224 | * Get monotonic time for posix timers | ||
| 225 | */ | ||
| 226 | static int posix_ktime_get_ts(clockid_t which_clock, struct timespec *tp) | ||
| 227 | { | ||
| 228 | ktime_get_ts(tp); | ||
| 229 | return 0; | ||
| 230 | } | ||
| 231 | |||
| 232 | /* | ||
| 233 | * Get monotonic-raw time for posix timers | ||
| 234 | */ | ||
| 235 | static int posix_get_monotonic_raw(clockid_t which_clock, struct timespec *tp) | ||
| 236 | { | ||
| 237 | getrawmonotonic(tp); | ||
| 238 | return 0; | ||
| 239 | } | ||
| 240 | |||
| 241 | |||
| 242 | static int posix_get_realtime_coarse(clockid_t which_clock, struct timespec *tp) | ||
| 243 | { | ||
| 244 | *tp = current_kernel_time(); | ||
| 245 | return 0; | ||
| 246 | } | ||
| 247 | |||
| 248 | static int posix_get_monotonic_coarse(clockid_t which_clock, | ||
| 249 | struct timespec *tp) | ||
| 250 | { | ||
| 251 | *tp = get_monotonic_coarse(); | ||
| 252 | return 0; | ||
| 253 | } | ||
| 254 | |||
| 255 | static int posix_get_coarse_res(const clockid_t which_clock, struct timespec *tp) | ||
| 256 | { | ||
| 257 | *tp = ktime_to_timespec(KTIME_LOW_RES); | ||
| 258 | return 0; | ||
| 259 | } | ||
| 260 | |||
| 261 | static int posix_get_boottime(const clockid_t which_clock, struct timespec *tp) | ||
| 262 | { | ||
| 263 | get_monotonic_boottime(tp); | ||
| 264 | return 0; | ||
| 265 | } | ||
| 266 | |||
| 267 | static int posix_get_tai(clockid_t which_clock, struct timespec *tp) | ||
| 268 | { | ||
| 269 | timekeeping_clocktai(tp); | ||
| 270 | return 0; | ||
| 271 | } | ||
| 272 | |||
| 273 | /* | ||
| 274 | * Initialize everything, well, just everything in Posix clocks/timers ;) | ||
| 275 | */ | ||
| 276 | static __init int init_posix_timers(void) | ||
| 277 | { | ||
| 278 | struct k_clock clock_realtime = { | ||
| 279 | .clock_getres = hrtimer_get_res, | ||
| 280 | .clock_get = posix_clock_realtime_get, | ||
| 281 | .clock_set = posix_clock_realtime_set, | ||
| 282 | .clock_adj = posix_clock_realtime_adj, | ||
| 283 | .nsleep = common_nsleep, | ||
| 284 | .nsleep_restart = hrtimer_nanosleep_restart, | ||
| 285 | .timer_create = common_timer_create, | ||
| 286 | .timer_set = common_timer_set, | ||
| 287 | .timer_get = common_timer_get, | ||
| 288 | .timer_del = common_timer_del, | ||
| 289 | }; | ||
| 290 | struct k_clock clock_monotonic = { | ||
| 291 | .clock_getres = hrtimer_get_res, | ||
| 292 | .clock_get = posix_ktime_get_ts, | ||
| 293 | .nsleep = common_nsleep, | ||
| 294 | .nsleep_restart = hrtimer_nanosleep_restart, | ||
| 295 | .timer_create = common_timer_create, | ||
| 296 | .timer_set = common_timer_set, | ||
| 297 | .timer_get = common_timer_get, | ||
| 298 | .timer_del = common_timer_del, | ||
| 299 | }; | ||
| 300 | struct k_clock clock_monotonic_raw = { | ||
| 301 | .clock_getres = hrtimer_get_res, | ||
| 302 | .clock_get = posix_get_monotonic_raw, | ||
| 303 | }; | ||
| 304 | struct k_clock clock_realtime_coarse = { | ||
| 305 | .clock_getres = posix_get_coarse_res, | ||
| 306 | .clock_get = posix_get_realtime_coarse, | ||
| 307 | }; | ||
| 308 | struct k_clock clock_monotonic_coarse = { | ||
| 309 | .clock_getres = posix_get_coarse_res, | ||
| 310 | .clock_get = posix_get_monotonic_coarse, | ||
| 311 | }; | ||
| 312 | struct k_clock clock_tai = { | ||
| 313 | .clock_getres = hrtimer_get_res, | ||
| 314 | .clock_get = posix_get_tai, | ||
| 315 | .nsleep = common_nsleep, | ||
| 316 | .nsleep_restart = hrtimer_nanosleep_restart, | ||
| 317 | .timer_create = common_timer_create, | ||
| 318 | .timer_set = common_timer_set, | ||
| 319 | .timer_get = common_timer_get, | ||
| 320 | .timer_del = common_timer_del, | ||
| 321 | }; | ||
| 322 | struct k_clock clock_boottime = { | ||
| 323 | .clock_getres = hrtimer_get_res, | ||
| 324 | .clock_get = posix_get_boottime, | ||
| 325 | .nsleep = common_nsleep, | ||
| 326 | .nsleep_restart = hrtimer_nanosleep_restart, | ||
| 327 | .timer_create = common_timer_create, | ||
| 328 | .timer_set = common_timer_set, | ||
| 329 | .timer_get = common_timer_get, | ||
| 330 | .timer_del = common_timer_del, | ||
| 331 | }; | ||
| 332 | |||
| 333 | posix_timers_register_clock(CLOCK_REALTIME, &clock_realtime); | ||
| 334 | posix_timers_register_clock(CLOCK_MONOTONIC, &clock_monotonic); | ||
| 335 | posix_timers_register_clock(CLOCK_MONOTONIC_RAW, &clock_monotonic_raw); | ||
| 336 | posix_timers_register_clock(CLOCK_REALTIME_COARSE, &clock_realtime_coarse); | ||
| 337 | posix_timers_register_clock(CLOCK_MONOTONIC_COARSE, &clock_monotonic_coarse); | ||
| 338 | posix_timers_register_clock(CLOCK_BOOTTIME, &clock_boottime); | ||
| 339 | posix_timers_register_clock(CLOCK_TAI, &clock_tai); | ||
| 340 | |||
| 341 | posix_timers_cache = kmem_cache_create("posix_timers_cache", | ||
| 342 | sizeof (struct k_itimer), 0, SLAB_PANIC, | ||
| 343 | NULL); | ||
| 344 | return 0; | ||
| 345 | } | ||
| 346 | |||
| 347 | __initcall(init_posix_timers); | ||
| 348 | |||
| 349 | static void schedule_next_timer(struct k_itimer *timr) | ||
| 350 | { | ||
| 351 | struct hrtimer *timer = &timr->it.real.timer; | ||
| 352 | |||
| 353 | if (timr->it.real.interval.tv64 == 0) | ||
| 354 | return; | ||
| 355 | |||
| 356 | timr->it_overrun += (unsigned int) hrtimer_forward(timer, | ||
| 357 | timer->base->get_time(), | ||
| 358 | timr->it.real.interval); | ||
| 359 | |||
| 360 | timr->it_overrun_last = timr->it_overrun; | ||
| 361 | timr->it_overrun = -1; | ||
| 362 | ++timr->it_requeue_pending; | ||
| 363 | hrtimer_restart(timer); | ||
| 364 | } | ||
| 365 | |||
| 366 | /* | ||
| 367 | * This function is exported for use by the signal deliver code. It is | ||
| 368 | * called just prior to the info block being released and passes that | ||
| 369 | * block to us. It's function is to update the overrun entry AND to | ||
| 370 | * restart the timer. It should only be called if the timer is to be | ||
| 371 | * restarted (i.e. we have flagged this in the sys_private entry of the | ||
| 372 | * info block). | ||
| 373 | * | ||
| 374 | * To protect against the timer going away while the interrupt is queued, | ||
| 375 | * we require that the it_requeue_pending flag be set. | ||
| 376 | */ | ||
| 377 | void do_schedule_next_timer(struct siginfo *info) | ||
| 378 | { | ||
| 379 | struct k_itimer *timr; | ||
| 380 | unsigned long flags; | ||
| 381 | |||
| 382 | timr = lock_timer(info->si_tid, &flags); | ||
| 383 | |||
| 384 | if (timr && timr->it_requeue_pending == info->si_sys_private) { | ||
| 385 | if (timr->it_clock < 0) | ||
| 386 | posix_cpu_timer_schedule(timr); | ||
| 387 | else | ||
| 388 | schedule_next_timer(timr); | ||
| 389 | |||
| 390 | info->si_overrun += timr->it_overrun_last; | ||
| 391 | } | ||
| 392 | |||
| 393 | if (timr) | ||
| 394 | unlock_timer(timr, flags); | ||
| 395 | } | ||
| 396 | |||
| 397 | int posix_timer_event(struct k_itimer *timr, int si_private) | ||
| 398 | { | ||
| 399 | struct task_struct *task; | ||
| 400 | int shared, ret = -1; | ||
| 401 | /* | ||
| 402 | * FIXME: if ->sigq is queued we can race with | ||
| 403 | * dequeue_signal()->do_schedule_next_timer(). | ||
| 404 | * | ||
| 405 | * If dequeue_signal() sees the "right" value of | ||
| 406 | * si_sys_private it calls do_schedule_next_timer(). | ||
| 407 | * We re-queue ->sigq and drop ->it_lock(). | ||
| 408 | * do_schedule_next_timer() locks the timer | ||
| 409 | * and re-schedules it while ->sigq is pending. | ||
| 410 | * Not really bad, but not that we want. | ||
| 411 | */ | ||
| 412 | timr->sigq->info.si_sys_private = si_private; | ||
| 413 | |||
| 414 | rcu_read_lock(); | ||
| 415 | task = pid_task(timr->it_pid, PIDTYPE_PID); | ||
| 416 | if (task) { | ||
| 417 | shared = !(timr->it_sigev_notify & SIGEV_THREAD_ID); | ||
| 418 | ret = send_sigqueue(timr->sigq, task, shared); | ||
| 419 | } | ||
| 420 | rcu_read_unlock(); | ||
| 421 | /* If we failed to send the signal the timer stops. */ | ||
| 422 | return ret > 0; | ||
| 423 | } | ||
| 424 | EXPORT_SYMBOL_GPL(posix_timer_event); | ||
| 425 | |||
| 426 | /* | ||
| 427 | * This function gets called when a POSIX.1b interval timer expires. It | ||
| 428 | * is used as a callback from the kernel internal timer. The | ||
| 429 | * run_timer_list code ALWAYS calls with interrupts on. | ||
| 430 | |||
| 431 | * This code is for CLOCK_REALTIME* and CLOCK_MONOTONIC* timers. | ||
| 432 | */ | ||
| 433 | static enum hrtimer_restart posix_timer_fn(struct hrtimer *timer) | ||
| 434 | { | ||
| 435 | struct k_itimer *timr; | ||
| 436 | unsigned long flags; | ||
| 437 | int si_private = 0; | ||
| 438 | enum hrtimer_restart ret = HRTIMER_NORESTART; | ||
| 439 | |||
| 440 | timr = container_of(timer, struct k_itimer, it.real.timer); | ||
| 441 | spin_lock_irqsave(&timr->it_lock, flags); | ||
| 442 | |||
| 443 | if (timr->it.real.interval.tv64 != 0) | ||
| 444 | si_private = ++timr->it_requeue_pending; | ||
| 445 | |||
| 446 | if (posix_timer_event(timr, si_private)) { | ||
| 447 | /* | ||
| 448 | * signal was not sent because of sig_ignor | ||
| 449 | * we will not get a call back to restart it AND | ||
| 450 | * it should be restarted. | ||
| 451 | */ | ||
| 452 | if (timr->it.real.interval.tv64 != 0) { | ||
| 453 | ktime_t now = hrtimer_cb_get_time(timer); | ||
| 454 | |||
| 455 | /* | ||
| 456 | * FIXME: What we really want, is to stop this | ||
| 457 | * timer completely and restart it in case the | ||
| 458 | * SIG_IGN is removed. This is a non trivial | ||
| 459 | * change which involves sighand locking | ||
| 460 | * (sigh !), which we don't want to do late in | ||
| 461 | * the release cycle. | ||
| 462 | * | ||
| 463 | * For now we just let timers with an interval | ||
| 464 | * less than a jiffie expire every jiffie to | ||
| 465 | * avoid softirq starvation in case of SIG_IGN | ||
| 466 | * and a very small interval, which would put | ||
| 467 | * the timer right back on the softirq pending | ||
| 468 | * list. By moving now ahead of time we trick | ||
| 469 | * hrtimer_forward() to expire the timer | ||
| 470 | * later, while we still maintain the overrun | ||
| 471 | * accuracy, but have some inconsistency in | ||
| 472 | * the timer_gettime() case. This is at least | ||
| 473 | * better than a starved softirq. A more | ||
| 474 | * complex fix which solves also another related | ||
| 475 | * inconsistency is already in the pipeline. | ||
| 476 | */ | ||
| 477 | #ifdef CONFIG_HIGH_RES_TIMERS | ||
| 478 | { | ||
| 479 | ktime_t kj = ktime_set(0, NSEC_PER_SEC / HZ); | ||
| 480 | |||
| 481 | if (timr->it.real.interval.tv64 < kj.tv64) | ||
| 482 | now = ktime_add(now, kj); | ||
| 483 | } | ||
| 484 | #endif | ||
| 485 | timr->it_overrun += (unsigned int) | ||
| 486 | hrtimer_forward(timer, now, | ||
| 487 | timr->it.real.interval); | ||
| 488 | ret = HRTIMER_RESTART; | ||
| 489 | ++timr->it_requeue_pending; | ||
| 490 | } | ||
| 491 | } | ||
| 492 | |||
| 493 | unlock_timer(timr, flags); | ||
| 494 | return ret; | ||
| 495 | } | ||
| 496 | |||
| 497 | static struct pid *good_sigevent(sigevent_t * event) | ||
| 498 | { | ||
| 499 | struct task_struct *rtn = current->group_leader; | ||
| 500 | |||
| 501 | if ((event->sigev_notify & SIGEV_THREAD_ID ) && | ||
| 502 | (!(rtn = find_task_by_vpid(event->sigev_notify_thread_id)) || | ||
| 503 | !same_thread_group(rtn, current) || | ||
| 504 | (event->sigev_notify & ~SIGEV_THREAD_ID) != SIGEV_SIGNAL)) | ||
| 505 | return NULL; | ||
| 506 | |||
| 507 | if (((event->sigev_notify & ~SIGEV_THREAD_ID) != SIGEV_NONE) && | ||
| 508 | ((event->sigev_signo <= 0) || (event->sigev_signo > SIGRTMAX))) | ||
| 509 | return NULL; | ||
| 510 | |||
| 511 | return task_pid(rtn); | ||
| 512 | } | ||
| 513 | |||
| 514 | void posix_timers_register_clock(const clockid_t clock_id, | ||
| 515 | struct k_clock *new_clock) | ||
| 516 | { | ||
| 517 | if ((unsigned) clock_id >= MAX_CLOCKS) { | ||
| 518 | printk(KERN_WARNING "POSIX clock register failed for clock_id %d\n", | ||
| 519 | clock_id); | ||
| 520 | return; | ||
| 521 | } | ||
| 522 | |||
| 523 | if (!new_clock->clock_get) { | ||
| 524 | printk(KERN_WARNING "POSIX clock id %d lacks clock_get()\n", | ||
| 525 | clock_id); | ||
| 526 | return; | ||
| 527 | } | ||
| 528 | if (!new_clock->clock_getres) { | ||
| 529 | printk(KERN_WARNING "POSIX clock id %d lacks clock_getres()\n", | ||
| 530 | clock_id); | ||
| 531 | return; | ||
| 532 | } | ||
| 533 | |||
| 534 | posix_clocks[clock_id] = *new_clock; | ||
| 535 | } | ||
| 536 | EXPORT_SYMBOL_GPL(posix_timers_register_clock); | ||
| 537 | |||
| 538 | static struct k_itimer * alloc_posix_timer(void) | ||
| 539 | { | ||
| 540 | struct k_itimer *tmr; | ||
| 541 | tmr = kmem_cache_zalloc(posix_timers_cache, GFP_KERNEL); | ||
| 542 | if (!tmr) | ||
| 543 | return tmr; | ||
| 544 | if (unlikely(!(tmr->sigq = sigqueue_alloc()))) { | ||
| 545 | kmem_cache_free(posix_timers_cache, tmr); | ||
| 546 | return NULL; | ||
| 547 | } | ||
| 548 | memset(&tmr->sigq->info, 0, sizeof(siginfo_t)); | ||
| 549 | return tmr; | ||
| 550 | } | ||
| 551 | |||
| 552 | static void k_itimer_rcu_free(struct rcu_head *head) | ||
| 553 | { | ||
| 554 | struct k_itimer *tmr = container_of(head, struct k_itimer, it.rcu); | ||
| 555 | |||
| 556 | kmem_cache_free(posix_timers_cache, tmr); | ||
| 557 | } | ||
| 558 | |||
| 559 | #define IT_ID_SET 1 | ||
| 560 | #define IT_ID_NOT_SET 0 | ||
| 561 | static void release_posix_timer(struct k_itimer *tmr, int it_id_set) | ||
| 562 | { | ||
| 563 | if (it_id_set) { | ||
| 564 | unsigned long flags; | ||
| 565 | spin_lock_irqsave(&hash_lock, flags); | ||
| 566 | hlist_del_rcu(&tmr->t_hash); | ||
| 567 | spin_unlock_irqrestore(&hash_lock, flags); | ||
| 568 | } | ||
| 569 | put_pid(tmr->it_pid); | ||
| 570 | sigqueue_free(tmr->sigq); | ||
| 571 | call_rcu(&tmr->it.rcu, k_itimer_rcu_free); | ||
| 572 | } | ||
| 573 | |||
| 574 | static struct k_clock *clockid_to_kclock(const clockid_t id) | ||
| 575 | { | ||
| 576 | if (id < 0) | ||
| 577 | return (id & CLOCKFD_MASK) == CLOCKFD ? | ||
| 578 | &clock_posix_dynamic : &clock_posix_cpu; | ||
| 579 | |||
| 580 | if (id >= MAX_CLOCKS || !posix_clocks[id].clock_getres) | ||
| 581 | return NULL; | ||
| 582 | return &posix_clocks[id]; | ||
| 583 | } | ||
| 584 | |||
| 585 | static int common_timer_create(struct k_itimer *new_timer) | ||
| 586 | { | ||
| 587 | hrtimer_init(&new_timer->it.real.timer, new_timer->it_clock, 0); | ||
| 588 | return 0; | ||
| 589 | } | ||
| 590 | |||
| 591 | /* Create a POSIX.1b interval timer. */ | ||
| 592 | |||
| 593 | SYSCALL_DEFINE3(timer_create, const clockid_t, which_clock, | ||
| 594 | struct sigevent __user *, timer_event_spec, | ||
| 595 | timer_t __user *, created_timer_id) | ||
| 596 | { | ||
| 597 | struct k_clock *kc = clockid_to_kclock(which_clock); | ||
| 598 | struct k_itimer *new_timer; | ||
| 599 | int error, new_timer_id; | ||
| 600 | sigevent_t event; | ||
| 601 | int it_id_set = IT_ID_NOT_SET; | ||
| 602 | |||
| 603 | if (!kc) | ||
| 604 | return -EINVAL; | ||
| 605 | if (!kc->timer_create) | ||
| 606 | return -EOPNOTSUPP; | ||
| 607 | |||
| 608 | new_timer = alloc_posix_timer(); | ||
| 609 | if (unlikely(!new_timer)) | ||
| 610 | return -EAGAIN; | ||
| 611 | |||
| 612 | spin_lock_init(&new_timer->it_lock); | ||
| 613 | new_timer_id = posix_timer_add(new_timer); | ||
| 614 | if (new_timer_id < 0) { | ||
| 615 | error = new_timer_id; | ||
| 616 | goto out; | ||
| 617 | } | ||
| 618 | |||
| 619 | it_id_set = IT_ID_SET; | ||
| 620 | new_timer->it_id = (timer_t) new_timer_id; | ||
| 621 | new_timer->it_clock = which_clock; | ||
| 622 | new_timer->it_overrun = -1; | ||
| 623 | |||
| 624 | if (timer_event_spec) { | ||
| 625 | if (copy_from_user(&event, timer_event_spec, sizeof (event))) { | ||
| 626 | error = -EFAULT; | ||
| 627 | goto out; | ||
| 628 | } | ||
| 629 | rcu_read_lock(); | ||
| 630 | new_timer->it_pid = get_pid(good_sigevent(&event)); | ||
| 631 | rcu_read_unlock(); | ||
| 632 | if (!new_timer->it_pid) { | ||
| 633 | error = -EINVAL; | ||
| 634 | goto out; | ||
| 635 | } | ||
| 636 | } else { | ||
| 637 | event.sigev_notify = SIGEV_SIGNAL; | ||
| 638 | event.sigev_signo = SIGALRM; | ||
| 639 | event.sigev_value.sival_int = new_timer->it_id; | ||
| 640 | new_timer->it_pid = get_pid(task_tgid(current)); | ||
| 641 | } | ||
| 642 | |||
| 643 | new_timer->it_sigev_notify = event.sigev_notify; | ||
| 644 | new_timer->sigq->info.si_signo = event.sigev_signo; | ||
| 645 | new_timer->sigq->info.si_value = event.sigev_value; | ||
| 646 | new_timer->sigq->info.si_tid = new_timer->it_id; | ||
| 647 | new_timer->sigq->info.si_code = SI_TIMER; | ||
| 648 | |||
| 649 | if (copy_to_user(created_timer_id, | ||
| 650 | &new_timer_id, sizeof (new_timer_id))) { | ||
| 651 | error = -EFAULT; | ||
| 652 | goto out; | ||
| 653 | } | ||
| 654 | |||
| 655 | error = kc->timer_create(new_timer); | ||
| 656 | if (error) | ||
| 657 | goto out; | ||
| 658 | |||
| 659 | spin_lock_irq(¤t->sighand->siglock); | ||
| 660 | new_timer->it_signal = current->signal; | ||
| 661 | list_add(&new_timer->list, ¤t->signal->posix_timers); | ||
| 662 | spin_unlock_irq(¤t->sighand->siglock); | ||
| 663 | |||
| 664 | return 0; | ||
| 665 | /* | ||
| 666 | * In the case of the timer belonging to another task, after | ||
| 667 | * the task is unlocked, the timer is owned by the other task | ||
| 668 | * and may cease to exist at any time. Don't use or modify | ||
| 669 | * new_timer after the unlock call. | ||
| 670 | */ | ||
| 671 | out: | ||
| 672 | release_posix_timer(new_timer, it_id_set); | ||
| 673 | return error; | ||
| 674 | } | ||
| 675 | |||
| 676 | /* | ||
| 677 | * Locking issues: We need to protect the result of the id look up until | ||
| 678 | * we get the timer locked down so it is not deleted under us. The | ||
| 679 | * removal is done under the idr spinlock so we use that here to bridge | ||
| 680 | * the find to the timer lock. To avoid a dead lock, the timer id MUST | ||
| 681 | * be release with out holding the timer lock. | ||
| 682 | */ | ||
| 683 | static struct k_itimer *__lock_timer(timer_t timer_id, unsigned long *flags) | ||
| 684 | { | ||
| 685 | struct k_itimer *timr; | ||
| 686 | |||
| 687 | /* | ||
| 688 | * timer_t could be any type >= int and we want to make sure any | ||
| 689 | * @timer_id outside positive int range fails lookup. | ||
| 690 | */ | ||
| 691 | if ((unsigned long long)timer_id > INT_MAX) | ||
| 692 | return NULL; | ||
| 693 | |||
| 694 | rcu_read_lock(); | ||
| 695 | timr = posix_timer_by_id(timer_id); | ||
| 696 | if (timr) { | ||
| 697 | spin_lock_irqsave(&timr->it_lock, *flags); | ||
| 698 | if (timr->it_signal == current->signal) { | ||
| 699 | rcu_read_unlock(); | ||
| 700 | return timr; | ||
| 701 | } | ||
| 702 | spin_unlock_irqrestore(&timr->it_lock, *flags); | ||
| 703 | } | ||
| 704 | rcu_read_unlock(); | ||
| 705 | |||
| 706 | return NULL; | ||
| 707 | } | ||
| 708 | |||
| 709 | /* | ||
| 710 | * Get the time remaining on a POSIX.1b interval timer. This function | ||
| 711 | * is ALWAYS called with spin_lock_irq on the timer, thus it must not | ||
| 712 | * mess with irq. | ||
| 713 | * | ||
| 714 | * We have a couple of messes to clean up here. First there is the case | ||
| 715 | * of a timer that has a requeue pending. These timers should appear to | ||
| 716 | * be in the timer list with an expiry as if we were to requeue them | ||
| 717 | * now. | ||
| 718 | * | ||
| 719 | * The second issue is the SIGEV_NONE timer which may be active but is | ||
| 720 | * not really ever put in the timer list (to save system resources). | ||
| 721 | * This timer may be expired, and if so, we will do it here. Otherwise | ||
| 722 | * it is the same as a requeue pending timer WRT to what we should | ||
| 723 | * report. | ||
| 724 | */ | ||
| 725 | static void | ||
| 726 | common_timer_get(struct k_itimer *timr, struct itimerspec *cur_setting) | ||
| 727 | { | ||
| 728 | ktime_t now, remaining, iv; | ||
| 729 | struct hrtimer *timer = &timr->it.real.timer; | ||
| 730 | |||
| 731 | memset(cur_setting, 0, sizeof(struct itimerspec)); | ||
| 732 | |||
| 733 | iv = timr->it.real.interval; | ||
| 734 | |||
| 735 | /* interval timer ? */ | ||
| 736 | if (iv.tv64) | ||
| 737 | cur_setting->it_interval = ktime_to_timespec(iv); | ||
| 738 | else if (!hrtimer_active(timer) && | ||
| 739 | (timr->it_sigev_notify & ~SIGEV_THREAD_ID) != SIGEV_NONE) | ||
| 740 | return; | ||
| 741 | |||
| 742 | now = timer->base->get_time(); | ||
| 743 | |||
| 744 | /* | ||
| 745 | * When a requeue is pending or this is a SIGEV_NONE | ||
| 746 | * timer move the expiry time forward by intervals, so | ||
| 747 | * expiry is > now. | ||
| 748 | */ | ||
| 749 | if (iv.tv64 && (timr->it_requeue_pending & REQUEUE_PENDING || | ||
| 750 | (timr->it_sigev_notify & ~SIGEV_THREAD_ID) == SIGEV_NONE)) | ||
| 751 | timr->it_overrun += (unsigned int) hrtimer_forward(timer, now, iv); | ||
| 752 | |||
| 753 | remaining = ktime_sub(hrtimer_get_expires(timer), now); | ||
| 754 | /* Return 0 only, when the timer is expired and not pending */ | ||
| 755 | if (remaining.tv64 <= 0) { | ||
| 756 | /* | ||
| 757 | * A single shot SIGEV_NONE timer must return 0, when | ||
| 758 | * it is expired ! | ||
| 759 | */ | ||
| 760 | if ((timr->it_sigev_notify & ~SIGEV_THREAD_ID) != SIGEV_NONE) | ||
| 761 | cur_setting->it_value.tv_nsec = 1; | ||
| 762 | } else | ||
| 763 | cur_setting->it_value = ktime_to_timespec(remaining); | ||
| 764 | } | ||
| 765 | |||
| 766 | /* Get the time remaining on a POSIX.1b interval timer. */ | ||
| 767 | SYSCALL_DEFINE2(timer_gettime, timer_t, timer_id, | ||
| 768 | struct itimerspec __user *, setting) | ||
| 769 | { | ||
| 770 | struct itimerspec cur_setting; | ||
| 771 | struct k_itimer *timr; | ||
| 772 | struct k_clock *kc; | ||
| 773 | unsigned long flags; | ||
| 774 | int ret = 0; | ||
| 775 | |||
| 776 | timr = lock_timer(timer_id, &flags); | ||
| 777 | if (!timr) | ||
| 778 | return -EINVAL; | ||
| 779 | |||
| 780 | kc = clockid_to_kclock(timr->it_clock); | ||
| 781 | if (WARN_ON_ONCE(!kc || !kc->timer_get)) | ||
| 782 | ret = -EINVAL; | ||
| 783 | else | ||
| 784 | kc->timer_get(timr, &cur_setting); | ||
| 785 | |||
| 786 | unlock_timer(timr, flags); | ||
| 787 | |||
| 788 | if (!ret && copy_to_user(setting, &cur_setting, sizeof (cur_setting))) | ||
| 789 | return -EFAULT; | ||
| 790 | |||
| 791 | return ret; | ||
| 792 | } | ||
| 793 | |||
| 794 | /* | ||
| 795 | * Get the number of overruns of a POSIX.1b interval timer. This is to | ||
| 796 | * be the overrun of the timer last delivered. At the same time we are | ||
| 797 | * accumulating overruns on the next timer. The overrun is frozen when | ||
| 798 | * the signal is delivered, either at the notify time (if the info block | ||
| 799 | * is not queued) or at the actual delivery time (as we are informed by | ||
| 800 | * the call back to do_schedule_next_timer(). So all we need to do is | ||
| 801 | * to pick up the frozen overrun. | ||
| 802 | */ | ||
| 803 | SYSCALL_DEFINE1(timer_getoverrun, timer_t, timer_id) | ||
| 804 | { | ||
| 805 | struct k_itimer *timr; | ||
| 806 | int overrun; | ||
| 807 | unsigned long flags; | ||
| 808 | |||
| 809 | timr = lock_timer(timer_id, &flags); | ||
| 810 | if (!timr) | ||
| 811 | return -EINVAL; | ||
| 812 | |||
| 813 | overrun = timr->it_overrun_last; | ||
| 814 | unlock_timer(timr, flags); | ||
| 815 | |||
| 816 | return overrun; | ||
| 817 | } | ||
| 818 | |||
| 819 | /* Set a POSIX.1b interval timer. */ | ||
| 820 | /* timr->it_lock is taken. */ | ||
| 821 | static int | ||
| 822 | common_timer_set(struct k_itimer *timr, int flags, | ||
| 823 | struct itimerspec *new_setting, struct itimerspec *old_setting) | ||
| 824 | { | ||
| 825 | struct hrtimer *timer = &timr->it.real.timer; | ||
| 826 | enum hrtimer_mode mode; | ||
| 827 | |||
| 828 | if (old_setting) | ||
| 829 | common_timer_get(timr, old_setting); | ||
| 830 | |||
| 831 | /* disable the timer */ | ||
| 832 | timr->it.real.interval.tv64 = 0; | ||
| 833 | /* | ||
| 834 | * careful here. If smp we could be in the "fire" routine which will | ||
| 835 | * be spinning as we hold the lock. But this is ONLY an SMP issue. | ||
| 836 | */ | ||
| 837 | if (hrtimer_try_to_cancel(timer) < 0) | ||
| 838 | return TIMER_RETRY; | ||
| 839 | |||
| 840 | timr->it_requeue_pending = (timr->it_requeue_pending + 2) & | ||
| 841 | ~REQUEUE_PENDING; | ||
| 842 | timr->it_overrun_last = 0; | ||
| 843 | |||
| 844 | /* switch off the timer when it_value is zero */ | ||
| 845 | if (!new_setting->it_value.tv_sec && !new_setting->it_value.tv_nsec) | ||
| 846 | return 0; | ||
| 847 | |||
| 848 | mode = flags & TIMER_ABSTIME ? HRTIMER_MODE_ABS : HRTIMER_MODE_REL; | ||
| 849 | hrtimer_init(&timr->it.real.timer, timr->it_clock, mode); | ||
| 850 | timr->it.real.timer.function = posix_timer_fn; | ||
| 851 | |||
| 852 | hrtimer_set_expires(timer, timespec_to_ktime(new_setting->it_value)); | ||
| 853 | |||
| 854 | /* Convert interval */ | ||
| 855 | timr->it.real.interval = timespec_to_ktime(new_setting->it_interval); | ||
| 856 | |||
| 857 | /* SIGEV_NONE timers are not queued ! See common_timer_get */ | ||
| 858 | if (((timr->it_sigev_notify & ~SIGEV_THREAD_ID) == SIGEV_NONE)) { | ||
| 859 | /* Setup correct expiry time for relative timers */ | ||
| 860 | if (mode == HRTIMER_MODE_REL) { | ||
| 861 | hrtimer_add_expires(timer, timer->base->get_time()); | ||
| 862 | } | ||
| 863 | return 0; | ||
| 864 | } | ||
| 865 | |||
| 866 | hrtimer_start_expires(timer, mode); | ||
| 867 | return 0; | ||
| 868 | } | ||
| 869 | |||
| 870 | /* Set a POSIX.1b interval timer */ | ||
| 871 | SYSCALL_DEFINE4(timer_settime, timer_t, timer_id, int, flags, | ||
| 872 | const struct itimerspec __user *, new_setting, | ||
| 873 | struct itimerspec __user *, old_setting) | ||
| 874 | { | ||
| 875 | struct k_itimer *timr; | ||
| 876 | struct itimerspec new_spec, old_spec; | ||
| 877 | int error = 0; | ||
| 878 | unsigned long flag; | ||
| 879 | struct itimerspec *rtn = old_setting ? &old_spec : NULL; | ||
| 880 | struct k_clock *kc; | ||
| 881 | |||
| 882 | if (!new_setting) | ||
| 883 | return -EINVAL; | ||
| 884 | |||
| 885 | if (copy_from_user(&new_spec, new_setting, sizeof (new_spec))) | ||
| 886 | return -EFAULT; | ||
| 887 | |||
| 888 | if (!timespec_valid(&new_spec.it_interval) || | ||
| 889 | !timespec_valid(&new_spec.it_value)) | ||
| 890 | return -EINVAL; | ||
| 891 | retry: | ||
| 892 | timr = lock_timer(timer_id, &flag); | ||
| 893 | if (!timr) | ||
| 894 | return -EINVAL; | ||
| 895 | |||
| 896 | kc = clockid_to_kclock(timr->it_clock); | ||
| 897 | if (WARN_ON_ONCE(!kc || !kc->timer_set)) | ||
| 898 | error = -EINVAL; | ||
| 899 | else | ||
| 900 | error = kc->timer_set(timr, flags, &new_spec, rtn); | ||
| 901 | |||
| 902 | unlock_timer(timr, flag); | ||
| 903 | if (error == TIMER_RETRY) { | ||
| 904 | rtn = NULL; // We already got the old time... | ||
| 905 | goto retry; | ||
| 906 | } | ||
| 907 | |||
| 908 | if (old_setting && !error && | ||
| 909 | copy_to_user(old_setting, &old_spec, sizeof (old_spec))) | ||
| 910 | error = -EFAULT; | ||
| 911 | |||
| 912 | return error; | ||
| 913 | } | ||
| 914 | |||
| 915 | static int common_timer_del(struct k_itimer *timer) | ||
| 916 | { | ||
| 917 | timer->it.real.interval.tv64 = 0; | ||
| 918 | |||
| 919 | if (hrtimer_try_to_cancel(&timer->it.real.timer) < 0) | ||
| 920 | return TIMER_RETRY; | ||
| 921 | return 0; | ||
| 922 | } | ||
| 923 | |||
| 924 | static inline int timer_delete_hook(struct k_itimer *timer) | ||
| 925 | { | ||
| 926 | struct k_clock *kc = clockid_to_kclock(timer->it_clock); | ||
| 927 | |||
| 928 | if (WARN_ON_ONCE(!kc || !kc->timer_del)) | ||
| 929 | return -EINVAL; | ||
| 930 | return kc->timer_del(timer); | ||
| 931 | } | ||
| 932 | |||
| 933 | /* Delete a POSIX.1b interval timer. */ | ||
| 934 | SYSCALL_DEFINE1(timer_delete, timer_t, timer_id) | ||
| 935 | { | ||
| 936 | struct k_itimer *timer; | ||
| 937 | unsigned long flags; | ||
| 938 | |||
| 939 | retry_delete: | ||
| 940 | timer = lock_timer(timer_id, &flags); | ||
| 941 | if (!timer) | ||
| 942 | return -EINVAL; | ||
| 943 | |||
| 944 | if (timer_delete_hook(timer) == TIMER_RETRY) { | ||
| 945 | unlock_timer(timer, flags); | ||
| 946 | goto retry_delete; | ||
| 947 | } | ||
| 948 | |||
| 949 | spin_lock(¤t->sighand->siglock); | ||
| 950 | list_del(&timer->list); | ||
| 951 | spin_unlock(¤t->sighand->siglock); | ||
| 952 | /* | ||
| 953 | * This keeps any tasks waiting on the spin lock from thinking | ||
| 954 | * they got something (see the lock code above). | ||
| 955 | */ | ||
| 956 | timer->it_signal = NULL; | ||
| 957 | |||
| 958 | unlock_timer(timer, flags); | ||
| 959 | release_posix_timer(timer, IT_ID_SET); | ||
| 960 | return 0; | ||
| 961 | } | ||
| 962 | |||
| 963 | /* | ||
| 964 | * return timer owned by the process, used by exit_itimers | ||
| 965 | */ | ||
| 966 | static void itimer_delete(struct k_itimer *timer) | ||
| 967 | { | ||
| 968 | unsigned long flags; | ||
| 969 | |||
| 970 | retry_delete: | ||
| 971 | spin_lock_irqsave(&timer->it_lock, flags); | ||
| 972 | |||
| 973 | if (timer_delete_hook(timer) == TIMER_RETRY) { | ||
| 974 | unlock_timer(timer, flags); | ||
| 975 | goto retry_delete; | ||
| 976 | } | ||
| 977 | list_del(&timer->list); | ||
| 978 | /* | ||
| 979 | * This keeps any tasks waiting on the spin lock from thinking | ||
| 980 | * they got something (see the lock code above). | ||
| 981 | */ | ||
| 982 | timer->it_signal = NULL; | ||
| 983 | |||
| 984 | unlock_timer(timer, flags); | ||
| 985 | release_posix_timer(timer, IT_ID_SET); | ||
| 986 | } | ||
| 987 | |||
| 988 | /* | ||
| 989 | * This is called by do_exit or de_thread, only when there are no more | ||
| 990 | * references to the shared signal_struct. | ||
| 991 | */ | ||
| 992 | void exit_itimers(struct signal_struct *sig) | ||
| 993 | { | ||
| 994 | struct k_itimer *tmr; | ||
| 995 | |||
| 996 | while (!list_empty(&sig->posix_timers)) { | ||
| 997 | tmr = list_entry(sig->posix_timers.next, struct k_itimer, list); | ||
| 998 | itimer_delete(tmr); | ||
| 999 | } | ||
| 1000 | } | ||
| 1001 | |||
| 1002 | SYSCALL_DEFINE2(clock_settime, const clockid_t, which_clock, | ||
| 1003 | const struct timespec __user *, tp) | ||
| 1004 | { | ||
| 1005 | struct k_clock *kc = clockid_to_kclock(which_clock); | ||
| 1006 | struct timespec new_tp; | ||
| 1007 | |||
| 1008 | if (!kc || !kc->clock_set) | ||
| 1009 | return -EINVAL; | ||
| 1010 | |||
| 1011 | if (copy_from_user(&new_tp, tp, sizeof (*tp))) | ||
| 1012 | return -EFAULT; | ||
| 1013 | |||
| 1014 | return kc->clock_set(which_clock, &new_tp); | ||
| 1015 | } | ||
| 1016 | |||
| 1017 | SYSCALL_DEFINE2(clock_gettime, const clockid_t, which_clock, | ||
| 1018 | struct timespec __user *,tp) | ||
| 1019 | { | ||
| 1020 | struct k_clock *kc = clockid_to_kclock(which_clock); | ||
| 1021 | struct timespec kernel_tp; | ||
| 1022 | int error; | ||
| 1023 | |||
| 1024 | if (!kc) | ||
| 1025 | return -EINVAL; | ||
| 1026 | |||
| 1027 | error = kc->clock_get(which_clock, &kernel_tp); | ||
| 1028 | |||
| 1029 | if (!error && copy_to_user(tp, &kernel_tp, sizeof (kernel_tp))) | ||
| 1030 | error = -EFAULT; | ||
| 1031 | |||
| 1032 | return error; | ||
| 1033 | } | ||
| 1034 | |||
| 1035 | SYSCALL_DEFINE2(clock_adjtime, const clockid_t, which_clock, | ||
| 1036 | struct timex __user *, utx) | ||
| 1037 | { | ||
| 1038 | struct k_clock *kc = clockid_to_kclock(which_clock); | ||
| 1039 | struct timex ktx; | ||
| 1040 | int err; | ||
| 1041 | |||
| 1042 | if (!kc) | ||
| 1043 | return -EINVAL; | ||
| 1044 | if (!kc->clock_adj) | ||
| 1045 | return -EOPNOTSUPP; | ||
| 1046 | |||
| 1047 | if (copy_from_user(&ktx, utx, sizeof(ktx))) | ||
| 1048 | return -EFAULT; | ||
| 1049 | |||
| 1050 | err = kc->clock_adj(which_clock, &ktx); | ||
| 1051 | |||
| 1052 | if (err >= 0 && copy_to_user(utx, &ktx, sizeof(ktx))) | ||
| 1053 | return -EFAULT; | ||
| 1054 | |||
| 1055 | return err; | ||
| 1056 | } | ||
| 1057 | |||
| 1058 | SYSCALL_DEFINE2(clock_getres, const clockid_t, which_clock, | ||
| 1059 | struct timespec __user *, tp) | ||
| 1060 | { | ||
| 1061 | struct k_clock *kc = clockid_to_kclock(which_clock); | ||
| 1062 | struct timespec rtn_tp; | ||
| 1063 | int error; | ||
| 1064 | |||
| 1065 | if (!kc) | ||
| 1066 | return -EINVAL; | ||
| 1067 | |||
| 1068 | error = kc->clock_getres(which_clock, &rtn_tp); | ||
| 1069 | |||
| 1070 | if (!error && tp && copy_to_user(tp, &rtn_tp, sizeof (rtn_tp))) | ||
| 1071 | error = -EFAULT; | ||
| 1072 | |||
| 1073 | return error; | ||
| 1074 | } | ||
| 1075 | |||
| 1076 | /* | ||
| 1077 | * nanosleep for monotonic and realtime clocks | ||
| 1078 | */ | ||
| 1079 | static int common_nsleep(const clockid_t which_clock, int flags, | ||
| 1080 | struct timespec *tsave, struct timespec __user *rmtp) | ||
| 1081 | { | ||
| 1082 | return hrtimer_nanosleep(tsave, rmtp, flags & TIMER_ABSTIME ? | ||
| 1083 | HRTIMER_MODE_ABS : HRTIMER_MODE_REL, | ||
| 1084 | which_clock); | ||
| 1085 | } | ||
| 1086 | |||
| 1087 | SYSCALL_DEFINE4(clock_nanosleep, const clockid_t, which_clock, int, flags, | ||
| 1088 | const struct timespec __user *, rqtp, | ||
| 1089 | struct timespec __user *, rmtp) | ||
| 1090 | { | ||
| 1091 | struct k_clock *kc = clockid_to_kclock(which_clock); | ||
| 1092 | struct timespec t; | ||
| 1093 | |||
| 1094 | if (!kc) | ||
| 1095 | return -EINVAL; | ||
| 1096 | if (!kc->nsleep) | ||
| 1097 | return -ENANOSLEEP_NOTSUP; | ||
| 1098 | |||
| 1099 | if (copy_from_user(&t, rqtp, sizeof (struct timespec))) | ||
| 1100 | return -EFAULT; | ||
| 1101 | |||
| 1102 | if (!timespec_valid(&t)) | ||
| 1103 | return -EINVAL; | ||
| 1104 | |||
| 1105 | return kc->nsleep(which_clock, flags, &t, rmtp); | ||
| 1106 | } | ||
| 1107 | |||
| 1108 | /* | ||
| 1109 | * This will restart clock_nanosleep. This is required only by | ||
| 1110 | * compat_clock_nanosleep_restart for now. | ||
| 1111 | */ | ||
| 1112 | long clock_nanosleep_restart(struct restart_block *restart_block) | ||
| 1113 | { | ||
| 1114 | clockid_t which_clock = restart_block->nanosleep.clockid; | ||
| 1115 | struct k_clock *kc = clockid_to_kclock(which_clock); | ||
| 1116 | |||
| 1117 | if (WARN_ON_ONCE(!kc || !kc->nsleep_restart)) | ||
| 1118 | return -EINVAL; | ||
| 1119 | |||
| 1120 | return kc->nsleep_restart(restart_block); | ||
| 1121 | } | ||
diff --git a/kernel/time/time.c b/kernel/time/time.c new file mode 100644 index 000000000000..7c7964c33ae7 --- /dev/null +++ b/kernel/time/time.c | |||
| @@ -0,0 +1,714 @@ | |||
| 1 | /* | ||
| 2 | * linux/kernel/time.c | ||
| 3 | * | ||
| 4 | * Copyright (C) 1991, 1992 Linus Torvalds | ||
| 5 | * | ||
| 6 | * This file contains the interface functions for the various | ||
| 7 | * time related system calls: time, stime, gettimeofday, settimeofday, | ||
| 8 | * adjtime | ||
| 9 | */ | ||
| 10 | /* | ||
| 11 | * Modification history kernel/time.c | ||
| 12 | * | ||
| 13 | * 1993-09-02 Philip Gladstone | ||
| 14 | * Created file with time related functions from sched/core.c and adjtimex() | ||
| 15 | * 1993-10-08 Torsten Duwe | ||
| 16 | * adjtime interface update and CMOS clock write code | ||
| 17 | * 1995-08-13 Torsten Duwe | ||
| 18 | * kernel PLL updated to 1994-12-13 specs (rfc-1589) | ||
| 19 | * 1999-01-16 Ulrich Windl | ||
| 20 | * Introduced error checking for many cases in adjtimex(). | ||
| 21 | * Updated NTP code according to technical memorandum Jan '96 | ||
| 22 | * "A Kernel Model for Precision Timekeeping" by Dave Mills | ||
| 23 | * Allow time_constant larger than MAXTC(6) for NTP v4 (MAXTC == 10) | ||
| 24 | * (Even though the technical memorandum forbids it) | ||
| 25 | * 2004-07-14 Christoph Lameter | ||
| 26 | * Added getnstimeofday to allow the posix timer functions to return | ||
| 27 | * with nanosecond accuracy | ||
| 28 | */ | ||
| 29 | |||
| 30 | #include <linux/export.h> | ||
| 31 | #include <linux/timex.h> | ||
| 32 | #include <linux/capability.h> | ||
| 33 | #include <linux/timekeeper_internal.h> | ||
| 34 | #include <linux/errno.h> | ||
| 35 | #include <linux/syscalls.h> | ||
| 36 | #include <linux/security.h> | ||
| 37 | #include <linux/fs.h> | ||
| 38 | #include <linux/math64.h> | ||
| 39 | #include <linux/ptrace.h> | ||
| 40 | |||
| 41 | #include <asm/uaccess.h> | ||
| 42 | #include <asm/unistd.h> | ||
| 43 | |||
| 44 | #include "timeconst.h" | ||
| 45 | |||
| 46 | /* | ||
| 47 | * The timezone where the local system is located. Used as a default by some | ||
| 48 | * programs who obtain this value by using gettimeofday. | ||
| 49 | */ | ||
| 50 | struct timezone sys_tz; | ||
| 51 | |||
| 52 | EXPORT_SYMBOL(sys_tz); | ||
| 53 | |||
| 54 | #ifdef __ARCH_WANT_SYS_TIME | ||
| 55 | |||
| 56 | /* | ||
| 57 | * sys_time() can be implemented in user-level using | ||
| 58 | * sys_gettimeofday(). Is this for backwards compatibility? If so, | ||
| 59 | * why not move it into the appropriate arch directory (for those | ||
| 60 | * architectures that need it). | ||
| 61 | */ | ||
| 62 | SYSCALL_DEFINE1(time, time_t __user *, tloc) | ||
| 63 | { | ||
| 64 | time_t i = get_seconds(); | ||
| 65 | |||
| 66 | if (tloc) { | ||
| 67 | if (put_user(i,tloc)) | ||
| 68 | return -EFAULT; | ||
| 69 | } | ||
| 70 | force_successful_syscall_return(); | ||
| 71 | return i; | ||
| 72 | } | ||
| 73 | |||
| 74 | /* | ||
| 75 | * sys_stime() can be implemented in user-level using | ||
| 76 | * sys_settimeofday(). Is this for backwards compatibility? If so, | ||
| 77 | * why not move it into the appropriate arch directory (for those | ||
| 78 | * architectures that need it). | ||
| 79 | */ | ||
| 80 | |||
| 81 | SYSCALL_DEFINE1(stime, time_t __user *, tptr) | ||
| 82 | { | ||
| 83 | struct timespec tv; | ||
| 84 | int err; | ||
| 85 | |||
| 86 | if (get_user(tv.tv_sec, tptr)) | ||
| 87 | return -EFAULT; | ||
| 88 | |||
| 89 | tv.tv_nsec = 0; | ||
| 90 | |||
| 91 | err = security_settime(&tv, NULL); | ||
| 92 | if (err) | ||
| 93 | return err; | ||
| 94 | |||
| 95 | do_settimeofday(&tv); | ||
| 96 | return 0; | ||
| 97 | } | ||
| 98 | |||
| 99 | #endif /* __ARCH_WANT_SYS_TIME */ | ||
| 100 | |||
| 101 | SYSCALL_DEFINE2(gettimeofday, struct timeval __user *, tv, | ||
| 102 | struct timezone __user *, tz) | ||
| 103 | { | ||
| 104 | if (likely(tv != NULL)) { | ||
| 105 | struct timeval ktv; | ||
| 106 | do_gettimeofday(&ktv); | ||
| 107 | if (copy_to_user(tv, &ktv, sizeof(ktv))) | ||
| 108 | return -EFAULT; | ||
| 109 | } | ||
| 110 | if (unlikely(tz != NULL)) { | ||
| 111 | if (copy_to_user(tz, &sys_tz, sizeof(sys_tz))) | ||
| 112 | return -EFAULT; | ||
| 113 | } | ||
| 114 | return 0; | ||
| 115 | } | ||
| 116 | |||
| 117 | /* | ||
| 118 | * Indicates if there is an offset between the system clock and the hardware | ||
| 119 | * clock/persistent clock/rtc. | ||
| 120 | */ | ||
| 121 | int persistent_clock_is_local; | ||
| 122 | |||
| 123 | /* | ||
| 124 | * Adjust the time obtained from the CMOS to be UTC time instead of | ||
| 125 | * local time. | ||
| 126 | * | ||
| 127 | * This is ugly, but preferable to the alternatives. Otherwise we | ||
| 128 | * would either need to write a program to do it in /etc/rc (and risk | ||
| 129 | * confusion if the program gets run more than once; it would also be | ||
| 130 | * hard to make the program warp the clock precisely n hours) or | ||
| 131 | * compile in the timezone information into the kernel. Bad, bad.... | ||
| 132 | * | ||
| 133 | * - TYT, 1992-01-01 | ||
| 134 | * | ||
| 135 | * The best thing to do is to keep the CMOS clock in universal time (UTC) | ||
| 136 | * as real UNIX machines always do it. This avoids all headaches about | ||
| 137 | * daylight saving times and warping kernel clocks. | ||
| 138 | */ | ||
| 139 | static inline void warp_clock(void) | ||
| 140 | { | ||
| 141 | if (sys_tz.tz_minuteswest != 0) { | ||
| 142 | struct timespec adjust; | ||
| 143 | |||
| 144 | persistent_clock_is_local = 1; | ||
| 145 | adjust.tv_sec = sys_tz.tz_minuteswest * 60; | ||
| 146 | adjust.tv_nsec = 0; | ||
| 147 | timekeeping_inject_offset(&adjust); | ||
| 148 | } | ||
| 149 | } | ||
| 150 | |||
| 151 | /* | ||
| 152 | * In case for some reason the CMOS clock has not already been running | ||
| 153 | * in UTC, but in some local time: The first time we set the timezone, | ||
| 154 | * we will warp the clock so that it is ticking UTC time instead of | ||
| 155 | * local time. Presumably, if someone is setting the timezone then we | ||
| 156 | * are running in an environment where the programs understand about | ||
| 157 | * timezones. This should be done at boot time in the /etc/rc script, | ||
| 158 | * as soon as possible, so that the clock can be set right. Otherwise, | ||
| 159 | * various programs will get confused when the clock gets warped. | ||
| 160 | */ | ||
| 161 | |||
| 162 | int do_sys_settimeofday(const struct timespec *tv, const struct timezone *tz) | ||
| 163 | { | ||
| 164 | static int firsttime = 1; | ||
| 165 | int error = 0; | ||
| 166 | |||
| 167 | if (tv && !timespec_valid(tv)) | ||
| 168 | return -EINVAL; | ||
| 169 | |||
| 170 | error = security_settime(tv, tz); | ||
| 171 | if (error) | ||
| 172 | return error; | ||
| 173 | |||
| 174 | if (tz) { | ||
| 175 | sys_tz = *tz; | ||
| 176 | update_vsyscall_tz(); | ||
| 177 | if (firsttime) { | ||
| 178 | firsttime = 0; | ||
| 179 | if (!tv) | ||
| 180 | warp_clock(); | ||
| 181 | } | ||
| 182 | } | ||
| 183 | if (tv) | ||
| 184 | return do_settimeofday(tv); | ||
| 185 | return 0; | ||
| 186 | } | ||
| 187 | |||
| 188 | SYSCALL_DEFINE2(settimeofday, struct timeval __user *, tv, | ||
| 189 | struct timezone __user *, tz) | ||
| 190 | { | ||
| 191 | struct timeval user_tv; | ||
| 192 | struct timespec new_ts; | ||
| 193 | struct timezone new_tz; | ||
| 194 | |||
| 195 | if (tv) { | ||
| 196 | if (copy_from_user(&user_tv, tv, sizeof(*tv))) | ||
| 197 | return -EFAULT; | ||
| 198 | new_ts.tv_sec = user_tv.tv_sec; | ||
| 199 | new_ts.tv_nsec = user_tv.tv_usec * NSEC_PER_USEC; | ||
| 200 | } | ||
| 201 | if (tz) { | ||
| 202 | if (copy_from_user(&new_tz, tz, sizeof(*tz))) | ||
| 203 | return -EFAULT; | ||
| 204 | } | ||
| 205 | |||
| 206 | return do_sys_settimeofday(tv ? &new_ts : NULL, tz ? &new_tz : NULL); | ||
| 207 | } | ||
| 208 | |||
| 209 | SYSCALL_DEFINE1(adjtimex, struct timex __user *, txc_p) | ||
| 210 | { | ||
| 211 | struct timex txc; /* Local copy of parameter */ | ||
| 212 | int ret; | ||
| 213 | |||
| 214 | /* Copy the user data space into the kernel copy | ||
| 215 | * structure. But bear in mind that the structures | ||
| 216 | * may change | ||
| 217 | */ | ||
| 218 | if(copy_from_user(&txc, txc_p, sizeof(struct timex))) | ||
| 219 | return -EFAULT; | ||
| 220 | ret = do_adjtimex(&txc); | ||
| 221 | return copy_to_user(txc_p, &txc, sizeof(struct timex)) ? -EFAULT : ret; | ||
| 222 | } | ||
| 223 | |||
| 224 | /** | ||
| 225 | * current_fs_time - Return FS time | ||
| 226 | * @sb: Superblock. | ||
| 227 | * | ||
| 228 | * Return the current time truncated to the time granularity supported by | ||
| 229 | * the fs. | ||
| 230 | */ | ||
| 231 | struct timespec current_fs_time(struct super_block *sb) | ||
| 232 | { | ||
| 233 | struct timespec now = current_kernel_time(); | ||
| 234 | return timespec_trunc(now, sb->s_time_gran); | ||
| 235 | } | ||
| 236 | EXPORT_SYMBOL(current_fs_time); | ||
| 237 | |||
| 238 | /* | ||
| 239 | * Convert jiffies to milliseconds and back. | ||
| 240 | * | ||
| 241 | * Avoid unnecessary multiplications/divisions in the | ||
| 242 | * two most common HZ cases: | ||
| 243 | */ | ||
| 244 | unsigned int jiffies_to_msecs(const unsigned long j) | ||
| 245 | { | ||
| 246 | #if HZ <= MSEC_PER_SEC && !(MSEC_PER_SEC % HZ) | ||
| 247 | return (MSEC_PER_SEC / HZ) * j; | ||
| 248 | #elif HZ > MSEC_PER_SEC && !(HZ % MSEC_PER_SEC) | ||
| 249 | return (j + (HZ / MSEC_PER_SEC) - 1)/(HZ / MSEC_PER_SEC); | ||
| 250 | #else | ||
| 251 | # if BITS_PER_LONG == 32 | ||
| 252 | return (HZ_TO_MSEC_MUL32 * j) >> HZ_TO_MSEC_SHR32; | ||
| 253 | # else | ||
| 254 | return (j * HZ_TO_MSEC_NUM) / HZ_TO_MSEC_DEN; | ||
| 255 | # endif | ||
| 256 | #endif | ||
| 257 | } | ||
| 258 | EXPORT_SYMBOL(jiffies_to_msecs); | ||
| 259 | |||
| 260 | unsigned int jiffies_to_usecs(const unsigned long j) | ||
| 261 | { | ||
| 262 | #if HZ <= USEC_PER_SEC && !(USEC_PER_SEC % HZ) | ||
| 263 | return (USEC_PER_SEC / HZ) * j; | ||
| 264 | #elif HZ > USEC_PER_SEC && !(HZ % USEC_PER_SEC) | ||
| 265 | return (j + (HZ / USEC_PER_SEC) - 1)/(HZ / USEC_PER_SEC); | ||
| 266 | #else | ||
| 267 | # if BITS_PER_LONG == 32 | ||
| 268 | return (HZ_TO_USEC_MUL32 * j) >> HZ_TO_USEC_SHR32; | ||
| 269 | # else | ||
| 270 | return (j * HZ_TO_USEC_NUM) / HZ_TO_USEC_DEN; | ||
| 271 | # endif | ||
| 272 | #endif | ||
| 273 | } | ||
| 274 | EXPORT_SYMBOL(jiffies_to_usecs); | ||
| 275 | |||
| 276 | /** | ||
| 277 | * timespec_trunc - Truncate timespec to a granularity | ||
| 278 | * @t: Timespec | ||
| 279 | * @gran: Granularity in ns. | ||
| 280 | * | ||
| 281 | * Truncate a timespec to a granularity. gran must be smaller than a second. | ||
| 282 | * Always rounds down. | ||
| 283 | * | ||
| 284 | * This function should be only used for timestamps returned by | ||
| 285 | * current_kernel_time() or CURRENT_TIME, not with do_gettimeofday() because | ||
| 286 | * it doesn't handle the better resolution of the latter. | ||
| 287 | */ | ||
| 288 | struct timespec timespec_trunc(struct timespec t, unsigned gran) | ||
| 289 | { | ||
| 290 | /* | ||
| 291 | * Division is pretty slow so avoid it for common cases. | ||
| 292 | * Currently current_kernel_time() never returns better than | ||
| 293 | * jiffies resolution. Exploit that. | ||
| 294 | */ | ||
| 295 | if (gran <= jiffies_to_usecs(1) * 1000) { | ||
| 296 | /* nothing */ | ||
| 297 | } else if (gran == 1000000000) { | ||
| 298 | t.tv_nsec = 0; | ||
| 299 | } else { | ||
| 300 | t.tv_nsec -= t.tv_nsec % gran; | ||
| 301 | } | ||
| 302 | return t; | ||
| 303 | } | ||
| 304 | EXPORT_SYMBOL(timespec_trunc); | ||
| 305 | |||
| 306 | /* Converts Gregorian date to seconds since 1970-01-01 00:00:00. | ||
| 307 | * Assumes input in normal date format, i.e. 1980-12-31 23:59:59 | ||
| 308 | * => year=1980, mon=12, day=31, hour=23, min=59, sec=59. | ||
| 309 | * | ||
| 310 | * [For the Julian calendar (which was used in Russia before 1917, | ||
| 311 | * Britain & colonies before 1752, anywhere else before 1582, | ||
| 312 | * and is still in use by some communities) leave out the | ||
| 313 | * -year/100+year/400 terms, and add 10.] | ||
| 314 | * | ||
| 315 | * This algorithm was first published by Gauss (I think). | ||
| 316 | * | ||
| 317 | * WARNING: this function will overflow on 2106-02-07 06:28:16 on | ||
| 318 | * machines where long is 32-bit! (However, as time_t is signed, we | ||
| 319 | * will already get problems at other places on 2038-01-19 03:14:08) | ||
| 320 | */ | ||
| 321 | unsigned long | ||
| 322 | mktime(const unsigned int year0, const unsigned int mon0, | ||
| 323 | const unsigned int day, const unsigned int hour, | ||
| 324 | const unsigned int min, const unsigned int sec) | ||
| 325 | { | ||
| 326 | unsigned int mon = mon0, year = year0; | ||
| 327 | |||
| 328 | /* 1..12 -> 11,12,1..10 */ | ||
| 329 | if (0 >= (int) (mon -= 2)) { | ||
| 330 | mon += 12; /* Puts Feb last since it has leap day */ | ||
| 331 | year -= 1; | ||
| 332 | } | ||
| 333 | |||
| 334 | return ((((unsigned long) | ||
| 335 | (year/4 - year/100 + year/400 + 367*mon/12 + day) + | ||
| 336 | year*365 - 719499 | ||
| 337 | )*24 + hour /* now have hours */ | ||
| 338 | )*60 + min /* now have minutes */ | ||
| 339 | )*60 + sec; /* finally seconds */ | ||
| 340 | } | ||
| 341 | |||
| 342 | EXPORT_SYMBOL(mktime); | ||
| 343 | |||
| 344 | /** | ||
| 345 | * set_normalized_timespec - set timespec sec and nsec parts and normalize | ||
| 346 | * | ||
| 347 | * @ts: pointer to timespec variable to be set | ||
| 348 | * @sec: seconds to set | ||
| 349 | * @nsec: nanoseconds to set | ||
| 350 | * | ||
| 351 | * Set seconds and nanoseconds field of a timespec variable and | ||
| 352 | * normalize to the timespec storage format | ||
| 353 | * | ||
| 354 | * Note: The tv_nsec part is always in the range of | ||
| 355 | * 0 <= tv_nsec < NSEC_PER_SEC | ||
| 356 | * For negative values only the tv_sec field is negative ! | ||
| 357 | */ | ||
| 358 | void set_normalized_timespec(struct timespec *ts, time_t sec, s64 nsec) | ||
| 359 | { | ||
| 360 | while (nsec >= NSEC_PER_SEC) { | ||
| 361 | /* | ||
| 362 | * The following asm() prevents the compiler from | ||
| 363 | * optimising this loop into a modulo operation. See | ||
| 364 | * also __iter_div_u64_rem() in include/linux/time.h | ||
| 365 | */ | ||
| 366 | asm("" : "+rm"(nsec)); | ||
| 367 | nsec -= NSEC_PER_SEC; | ||
| 368 | ++sec; | ||
| 369 | } | ||
| 370 | while (nsec < 0) { | ||
| 371 | asm("" : "+rm"(nsec)); | ||
| 372 | nsec += NSEC_PER_SEC; | ||
| 373 | --sec; | ||
| 374 | } | ||
| 375 | ts->tv_sec = sec; | ||
| 376 | ts->tv_nsec = nsec; | ||
| 377 | } | ||
| 378 | EXPORT_SYMBOL(set_normalized_timespec); | ||
| 379 | |||
| 380 | /** | ||
| 381 | * ns_to_timespec - Convert nanoseconds to timespec | ||
| 382 | * @nsec: the nanoseconds value to be converted | ||
| 383 | * | ||
| 384 | * Returns the timespec representation of the nsec parameter. | ||
| 385 | */ | ||
| 386 | struct timespec ns_to_timespec(const s64 nsec) | ||
| 387 | { | ||
| 388 | struct timespec ts; | ||
| 389 | s32 rem; | ||
| 390 | |||
| 391 | if (!nsec) | ||
| 392 | return (struct timespec) {0, 0}; | ||
| 393 | |||
| 394 | ts.tv_sec = div_s64_rem(nsec, NSEC_PER_SEC, &rem); | ||
| 395 | if (unlikely(rem < 0)) { | ||
| 396 | ts.tv_sec--; | ||
| 397 | rem += NSEC_PER_SEC; | ||
| 398 | } | ||
| 399 | ts.tv_nsec = rem; | ||
| 400 | |||
| 401 | return ts; | ||
| 402 | } | ||
| 403 | EXPORT_SYMBOL(ns_to_timespec); | ||
| 404 | |||
| 405 | /** | ||
| 406 | * ns_to_timeval - Convert nanoseconds to timeval | ||
| 407 | * @nsec: the nanoseconds value to be converted | ||
| 408 | * | ||
| 409 | * Returns the timeval representation of the nsec parameter. | ||
| 410 | */ | ||
| 411 | struct timeval ns_to_timeval(const s64 nsec) | ||
| 412 | { | ||
| 413 | struct timespec ts = ns_to_timespec(nsec); | ||
| 414 | struct timeval tv; | ||
| 415 | |||
| 416 | tv.tv_sec = ts.tv_sec; | ||
| 417 | tv.tv_usec = (suseconds_t) ts.tv_nsec / 1000; | ||
| 418 | |||
| 419 | return tv; | ||
| 420 | } | ||
| 421 | EXPORT_SYMBOL(ns_to_timeval); | ||
| 422 | |||
| 423 | /* | ||
| 424 | * When we convert to jiffies then we interpret incoming values | ||
| 425 | * the following way: | ||
| 426 | * | ||
| 427 | * - negative values mean 'infinite timeout' (MAX_JIFFY_OFFSET) | ||
| 428 | * | ||
| 429 | * - 'too large' values [that would result in larger than | ||
| 430 | * MAX_JIFFY_OFFSET values] mean 'infinite timeout' too. | ||
| 431 | * | ||
| 432 | * - all other values are converted to jiffies by either multiplying | ||
| 433 | * the input value by a factor or dividing it with a factor | ||
| 434 | * | ||
| 435 | * We must also be careful about 32-bit overflows. | ||
| 436 | */ | ||
| 437 | unsigned long msecs_to_jiffies(const unsigned int m) | ||
| 438 | { | ||
| 439 | /* | ||
| 440 | * Negative value, means infinite timeout: | ||
| 441 | */ | ||
| 442 | if ((int)m < 0) | ||
| 443 | return MAX_JIFFY_OFFSET; | ||
| 444 | |||
| 445 | #if HZ <= MSEC_PER_SEC && !(MSEC_PER_SEC % HZ) | ||
| 446 | /* | ||
| 447 | * HZ is equal to or smaller than 1000, and 1000 is a nice | ||
| 448 | * round multiple of HZ, divide with the factor between them, | ||
| 449 | * but round upwards: | ||
| 450 | */ | ||
| 451 | return (m + (MSEC_PER_SEC / HZ) - 1) / (MSEC_PER_SEC / HZ); | ||
| 452 | #elif HZ > MSEC_PER_SEC && !(HZ % MSEC_PER_SEC) | ||
| 453 | /* | ||
| 454 | * HZ is larger than 1000, and HZ is a nice round multiple of | ||
| 455 | * 1000 - simply multiply with the factor between them. | ||
| 456 | * | ||
| 457 | * But first make sure the multiplication result cannot | ||
| 458 | * overflow: | ||
| 459 | */ | ||
| 460 | if (m > jiffies_to_msecs(MAX_JIFFY_OFFSET)) | ||
| 461 | return MAX_JIFFY_OFFSET; | ||
| 462 | |||
| 463 | return m * (HZ / MSEC_PER_SEC); | ||
| 464 | #else | ||
| 465 | /* | ||
| 466 | * Generic case - multiply, round and divide. But first | ||
| 467 | * check that if we are doing a net multiplication, that | ||
| 468 | * we wouldn't overflow: | ||
| 469 | */ | ||
| 470 | if (HZ > MSEC_PER_SEC && m > jiffies_to_msecs(MAX_JIFFY_OFFSET)) | ||
| 471 | return MAX_JIFFY_OFFSET; | ||
| 472 | |||
| 473 | return (MSEC_TO_HZ_MUL32 * m + MSEC_TO_HZ_ADJ32) | ||
| 474 | >> MSEC_TO_HZ_SHR32; | ||
| 475 | #endif | ||
| 476 | } | ||
| 477 | EXPORT_SYMBOL(msecs_to_jiffies); | ||
| 478 | |||
| 479 | unsigned long usecs_to_jiffies(const unsigned int u) | ||
| 480 | { | ||
| 481 | if (u > jiffies_to_usecs(MAX_JIFFY_OFFSET)) | ||
| 482 | return MAX_JIFFY_OFFSET; | ||
| 483 | #if HZ <= USEC_PER_SEC && !(USEC_PER_SEC % HZ) | ||
| 484 | return (u + (USEC_PER_SEC / HZ) - 1) / (USEC_PER_SEC / HZ); | ||
| 485 | #elif HZ > USEC_PER_SEC && !(HZ % USEC_PER_SEC) | ||
| 486 | return u * (HZ / USEC_PER_SEC); | ||
| 487 | #else | ||
| 488 | return (USEC_TO_HZ_MUL32 * u + USEC_TO_HZ_ADJ32) | ||
| 489 | >> USEC_TO_HZ_SHR32; | ||
| 490 | #endif | ||
| 491 | } | ||
| 492 | EXPORT_SYMBOL(usecs_to_jiffies); | ||
| 493 | |||
| 494 | /* | ||
| 495 | * The TICK_NSEC - 1 rounds up the value to the next resolution. Note | ||
| 496 | * that a remainder subtract here would not do the right thing as the | ||
| 497 | * resolution values don't fall on second boundries. I.e. the line: | ||
| 498 | * nsec -= nsec % TICK_NSEC; is NOT a correct resolution rounding. | ||
| 499 | * | ||
| 500 | * Rather, we just shift the bits off the right. | ||
| 501 | * | ||
| 502 | * The >> (NSEC_JIFFIE_SC - SEC_JIFFIE_SC) converts the scaled nsec | ||
| 503 | * value to a scaled second value. | ||
| 504 | */ | ||
| 505 | unsigned long | ||
| 506 | timespec_to_jiffies(const struct timespec *value) | ||
| 507 | { | ||
| 508 | unsigned long sec = value->tv_sec; | ||
| 509 | long nsec = value->tv_nsec + TICK_NSEC - 1; | ||
| 510 | |||
| 511 | if (sec >= MAX_SEC_IN_JIFFIES){ | ||
| 512 | sec = MAX_SEC_IN_JIFFIES; | ||
| 513 | nsec = 0; | ||
| 514 | } | ||
| 515 | return (((u64)sec * SEC_CONVERSION) + | ||
| 516 | (((u64)nsec * NSEC_CONVERSION) >> | ||
| 517 | (NSEC_JIFFIE_SC - SEC_JIFFIE_SC))) >> SEC_JIFFIE_SC; | ||
| 518 | |||
| 519 | } | ||
| 520 | EXPORT_SYMBOL(timespec_to_jiffies); | ||
| 521 | |||
| 522 | void | ||
| 523 | jiffies_to_timespec(const unsigned long jiffies, struct timespec *value) | ||
| 524 | { | ||
| 525 | /* | ||
| 526 | * Convert jiffies to nanoseconds and separate with | ||
| 527 | * one divide. | ||
| 528 | */ | ||
| 529 | u32 rem; | ||
| 530 | value->tv_sec = div_u64_rem((u64)jiffies * TICK_NSEC, | ||
| 531 | NSEC_PER_SEC, &rem); | ||
| 532 | value->tv_nsec = rem; | ||
| 533 | } | ||
| 534 | EXPORT_SYMBOL(jiffies_to_timespec); | ||
| 535 | |||
| 536 | /* Same for "timeval" | ||
| 537 | * | ||
| 538 | * Well, almost. The problem here is that the real system resolution is | ||
| 539 | * in nanoseconds and the value being converted is in micro seconds. | ||
| 540 | * Also for some machines (those that use HZ = 1024, in-particular), | ||
| 541 | * there is a LARGE error in the tick size in microseconds. | ||
| 542 | |||
| 543 | * The solution we use is to do the rounding AFTER we convert the | ||
| 544 | * microsecond part. Thus the USEC_ROUND, the bits to be shifted off. | ||
| 545 | * Instruction wise, this should cost only an additional add with carry | ||
| 546 | * instruction above the way it was done above. | ||
| 547 | */ | ||
| 548 | unsigned long | ||
| 549 | timeval_to_jiffies(const struct timeval *value) | ||
| 550 | { | ||
| 551 | unsigned long sec = value->tv_sec; | ||
| 552 | long usec = value->tv_usec; | ||
| 553 | |||
| 554 | if (sec >= MAX_SEC_IN_JIFFIES){ | ||
| 555 | sec = MAX_SEC_IN_JIFFIES; | ||
| 556 | usec = 0; | ||
| 557 | } | ||
| 558 | return (((u64)sec * SEC_CONVERSION) + | ||
| 559 | (((u64)usec * USEC_CONVERSION + USEC_ROUND) >> | ||
| 560 | (USEC_JIFFIE_SC - SEC_JIFFIE_SC))) >> SEC_JIFFIE_SC; | ||
| 561 | } | ||
| 562 | EXPORT_SYMBOL(timeval_to_jiffies); | ||
| 563 | |||
| 564 | void jiffies_to_timeval(const unsigned long jiffies, struct timeval *value) | ||
| 565 | { | ||
| 566 | /* | ||
| 567 | * Convert jiffies to nanoseconds and separate with | ||
| 568 | * one divide. | ||
| 569 | */ | ||
| 570 | u32 rem; | ||
| 571 | |||
| 572 | value->tv_sec = div_u64_rem((u64)jiffies * TICK_NSEC, | ||
| 573 | NSEC_PER_SEC, &rem); | ||
| 574 | value->tv_usec = rem / NSEC_PER_USEC; | ||
| 575 | } | ||
| 576 | EXPORT_SYMBOL(jiffies_to_timeval); | ||
| 577 | |||
| 578 | /* | ||
| 579 | * Convert jiffies/jiffies_64 to clock_t and back. | ||
| 580 | */ | ||
| 581 | clock_t jiffies_to_clock_t(unsigned long x) | ||
| 582 | { | ||
| 583 | #if (TICK_NSEC % (NSEC_PER_SEC / USER_HZ)) == 0 | ||
| 584 | # if HZ < USER_HZ | ||
| 585 | return x * (USER_HZ / HZ); | ||
| 586 | # else | ||
| 587 | return x / (HZ / USER_HZ); | ||
| 588 | # endif | ||
| 589 | #else | ||
| 590 | return div_u64((u64)x * TICK_NSEC, NSEC_PER_SEC / USER_HZ); | ||
| 591 | #endif | ||
| 592 | } | ||
| 593 | EXPORT_SYMBOL(jiffies_to_clock_t); | ||
| 594 | |||
| 595 | unsigned long clock_t_to_jiffies(unsigned long x) | ||
| 596 | { | ||
| 597 | #if (HZ % USER_HZ)==0 | ||
| 598 | if (x >= ~0UL / (HZ / USER_HZ)) | ||
| 599 | return ~0UL; | ||
| 600 | return x * (HZ / USER_HZ); | ||
| 601 | #else | ||
| 602 | /* Don't worry about loss of precision here .. */ | ||
| 603 | if (x >= ~0UL / HZ * USER_HZ) | ||
| 604 | return ~0UL; | ||
| 605 | |||
| 606 | /* .. but do try to contain it here */ | ||
| 607 | return div_u64((u64)x * HZ, USER_HZ); | ||
| 608 | #endif | ||
| 609 | } | ||
| 610 | EXPORT_SYMBOL(clock_t_to_jiffies); | ||
| 611 | |||
| 612 | u64 jiffies_64_to_clock_t(u64 x) | ||
| 613 | { | ||
| 614 | #if (TICK_NSEC % (NSEC_PER_SEC / USER_HZ)) == 0 | ||
| 615 | # if HZ < USER_HZ | ||
| 616 | x = div_u64(x * USER_HZ, HZ); | ||
| 617 | # elif HZ > USER_HZ | ||
| 618 | x = div_u64(x, HZ / USER_HZ); | ||
| 619 | # else | ||
| 620 | /* Nothing to do */ | ||
| 621 | # endif | ||
| 622 | #else | ||
| 623 | /* | ||
| 624 | * There are better ways that don't overflow early, | ||
| 625 | * but even this doesn't overflow in hundreds of years | ||
| 626 | * in 64 bits, so.. | ||
| 627 | */ | ||
| 628 | x = div_u64(x * TICK_NSEC, (NSEC_PER_SEC / USER_HZ)); | ||
| 629 | #endif | ||
| 630 | return x; | ||
| 631 | } | ||
| 632 | EXPORT_SYMBOL(jiffies_64_to_clock_t); | ||
| 633 | |||
| 634 | u64 nsec_to_clock_t(u64 x) | ||
| 635 | { | ||
| 636 | #if (NSEC_PER_SEC % USER_HZ) == 0 | ||
| 637 | return div_u64(x, NSEC_PER_SEC / USER_HZ); | ||
| 638 | #elif (USER_HZ % 512) == 0 | ||
| 639 | return div_u64(x * USER_HZ / 512, NSEC_PER_SEC / 512); | ||
| 640 | #else | ||
| 641 | /* | ||
| 642 | * max relative error 5.7e-8 (1.8s per year) for USER_HZ <= 1024, | ||
| 643 | * overflow after 64.99 years. | ||
| 644 | * exact for HZ=60, 72, 90, 120, 144, 180, 300, 600, 900, ... | ||
| 645 | */ | ||
| 646 | return div_u64(x * 9, (9ull * NSEC_PER_SEC + (USER_HZ / 2)) / USER_HZ); | ||
| 647 | #endif | ||
| 648 | } | ||
| 649 | |||
| 650 | /** | ||
| 651 | * nsecs_to_jiffies64 - Convert nsecs in u64 to jiffies64 | ||
| 652 | * | ||
| 653 | * @n: nsecs in u64 | ||
| 654 | * | ||
| 655 | * Unlike {m,u}secs_to_jiffies, type of input is not unsigned int but u64. | ||
| 656 | * And this doesn't return MAX_JIFFY_OFFSET since this function is designed | ||
| 657 | * for scheduler, not for use in device drivers to calculate timeout value. | ||
| 658 | * | ||
| 659 | * note: | ||
| 660 | * NSEC_PER_SEC = 10^9 = (5^9 * 2^9) = (1953125 * 512) | ||
| 661 | * ULLONG_MAX ns = 18446744073.709551615 secs = about 584 years | ||
| 662 | */ | ||
| 663 | u64 nsecs_to_jiffies64(u64 n) | ||
| 664 | { | ||
| 665 | #if (NSEC_PER_SEC % HZ) == 0 | ||
| 666 | /* Common case, HZ = 100, 128, 200, 250, 256, 500, 512, 1000 etc. */ | ||
| 667 | return div_u64(n, NSEC_PER_SEC / HZ); | ||
| 668 | #elif (HZ % 512) == 0 | ||
| 669 | /* overflow after 292 years if HZ = 1024 */ | ||
| 670 | return div_u64(n * HZ / 512, NSEC_PER_SEC / 512); | ||
| 671 | #else | ||
| 672 | /* | ||
| 673 | * Generic case - optimized for cases where HZ is a multiple of 3. | ||
| 674 | * overflow after 64.99 years, exact for HZ = 60, 72, 90, 120 etc. | ||
| 675 | */ | ||
| 676 | return div_u64(n * 9, (9ull * NSEC_PER_SEC + HZ / 2) / HZ); | ||
| 677 | #endif | ||
| 678 | } | ||
| 679 | |||
| 680 | /** | ||
| 681 | * nsecs_to_jiffies - Convert nsecs in u64 to jiffies | ||
| 682 | * | ||
| 683 | * @n: nsecs in u64 | ||
| 684 | * | ||
| 685 | * Unlike {m,u}secs_to_jiffies, type of input is not unsigned int but u64. | ||
| 686 | * And this doesn't return MAX_JIFFY_OFFSET since this function is designed | ||
| 687 | * for scheduler, not for use in device drivers to calculate timeout value. | ||
| 688 | * | ||
| 689 | * note: | ||
| 690 | * NSEC_PER_SEC = 10^9 = (5^9 * 2^9) = (1953125 * 512) | ||
| 691 | * ULLONG_MAX ns = 18446744073.709551615 secs = about 584 years | ||
| 692 | */ | ||
| 693 | unsigned long nsecs_to_jiffies(u64 n) | ||
| 694 | { | ||
| 695 | return (unsigned long)nsecs_to_jiffies64(n); | ||
| 696 | } | ||
| 697 | |||
| 698 | /* | ||
| 699 | * Add two timespec values and do a safety check for overflow. | ||
| 700 | * It's assumed that both values are valid (>= 0) | ||
| 701 | */ | ||
| 702 | struct timespec timespec_add_safe(const struct timespec lhs, | ||
| 703 | const struct timespec rhs) | ||
| 704 | { | ||
| 705 | struct timespec res; | ||
| 706 | |||
| 707 | set_normalized_timespec(&res, lhs.tv_sec + rhs.tv_sec, | ||
| 708 | lhs.tv_nsec + rhs.tv_nsec); | ||
| 709 | |||
| 710 | if (res.tv_sec < lhs.tv_sec || res.tv_sec < rhs.tv_sec) | ||
| 711 | res.tv_sec = TIME_T_MAX; | ||
| 712 | |||
| 713 | return res; | ||
| 714 | } | ||
diff --git a/kernel/time/timeconst.bc b/kernel/time/timeconst.bc new file mode 100644 index 000000000000..511bdf2cafda --- /dev/null +++ b/kernel/time/timeconst.bc | |||
| @@ -0,0 +1,108 @@ | |||
| 1 | scale=0 | ||
| 2 | |||
| 3 | define gcd(a,b) { | ||
| 4 | auto t; | ||
| 5 | while (b) { | ||
| 6 | t = b; | ||
| 7 | b = a % b; | ||
| 8 | a = t; | ||
| 9 | } | ||
| 10 | return a; | ||
| 11 | } | ||
| 12 | |||
| 13 | /* Division by reciprocal multiplication. */ | ||
| 14 | define fmul(b,n,d) { | ||
| 15 | return (2^b*n+d-1)/d; | ||
| 16 | } | ||
| 17 | |||
| 18 | /* Adjustment factor when a ceiling value is used. Use as: | ||
| 19 | (imul * n) + (fmulxx * n + fadjxx) >> xx) */ | ||
| 20 | define fadj(b,n,d) { | ||
| 21 | auto v; | ||
| 22 | d = d/gcd(n,d); | ||
| 23 | v = 2^b*(d-1)/d; | ||
| 24 | return v; | ||
| 25 | } | ||
| 26 | |||
| 27 | /* Compute the appropriate mul/adj values as well as a shift count, | ||
| 28 | which brings the mul value into the range 2^b-1 <= x < 2^b. Such | ||
| 29 | a shift value will be correct in the signed integer range and off | ||
| 30 | by at most one in the upper half of the unsigned range. */ | ||
| 31 | define fmuls(b,n,d) { | ||
| 32 | auto s, m; | ||
| 33 | for (s = 0; 1; s++) { | ||
| 34 | m = fmul(s,n,d); | ||
| 35 | if (m >= 2^(b-1)) | ||
| 36 | return s; | ||
| 37 | } | ||
| 38 | return 0; | ||
| 39 | } | ||
| 40 | |||
| 41 | define timeconst(hz) { | ||
| 42 | print "/* Automatically generated by kernel/timeconst.bc */\n" | ||
| 43 | print "/* Time conversion constants for HZ == ", hz, " */\n" | ||
| 44 | print "\n" | ||
| 45 | |||
| 46 | print "#ifndef KERNEL_TIMECONST_H\n" | ||
| 47 | print "#define KERNEL_TIMECONST_H\n\n" | ||
| 48 | |||
| 49 | print "#include <linux/param.h>\n" | ||
| 50 | print "#include <linux/types.h>\n\n" | ||
| 51 | |||
| 52 | print "#if HZ != ", hz, "\n" | ||
| 53 | print "#error \qkernel/timeconst.h has the wrong HZ value!\q\n" | ||
| 54 | print "#endif\n\n" | ||
| 55 | |||
| 56 | if (hz < 2) { | ||
| 57 | print "#error Totally bogus HZ value!\n" | ||
| 58 | } else { | ||
| 59 | s=fmuls(32,1000,hz) | ||
| 60 | obase=16 | ||
| 61 | print "#define HZ_TO_MSEC_MUL32\tU64_C(0x", fmul(s,1000,hz), ")\n" | ||
| 62 | print "#define HZ_TO_MSEC_ADJ32\tU64_C(0x", fadj(s,1000,hz), ")\n" | ||
| 63 | obase=10 | ||
| 64 | print "#define HZ_TO_MSEC_SHR32\t", s, "\n" | ||
| 65 | |||
| 66 | s=fmuls(32,hz,1000) | ||
| 67 | obase=16 | ||
| 68 | print "#define MSEC_TO_HZ_MUL32\tU64_C(0x", fmul(s,hz,1000), ")\n" | ||
| 69 | print "#define MSEC_TO_HZ_ADJ32\tU64_C(0x", fadj(s,hz,1000), ")\n" | ||
| 70 | obase=10 | ||
| 71 | print "#define MSEC_TO_HZ_SHR32\t", s, "\n" | ||
| 72 | |||
| 73 | obase=10 | ||
| 74 | cd=gcd(hz,1000) | ||
| 75 | print "#define HZ_TO_MSEC_NUM\t\t", 1000/cd, "\n" | ||
| 76 | print "#define HZ_TO_MSEC_DEN\t\t", hz/cd, "\n" | ||
| 77 | print "#define MSEC_TO_HZ_NUM\t\t", hz/cd, "\n" | ||
| 78 | print "#define MSEC_TO_HZ_DEN\t\t", 1000/cd, "\n" | ||
| 79 | print "\n" | ||
| 80 | |||
| 81 | s=fmuls(32,1000000,hz) | ||
| 82 | obase=16 | ||
| 83 | print "#define HZ_TO_USEC_MUL32\tU64_C(0x", fmul(s,1000000,hz), ")\n" | ||
| 84 | print "#define HZ_TO_USEC_ADJ32\tU64_C(0x", fadj(s,1000000,hz), ")\n" | ||
| 85 | obase=10 | ||
| 86 | print "#define HZ_TO_USEC_SHR32\t", s, "\n" | ||
| 87 | |||
| 88 | s=fmuls(32,hz,1000000) | ||
| 89 | obase=16 | ||
| 90 | print "#define USEC_TO_HZ_MUL32\tU64_C(0x", fmul(s,hz,1000000), ")\n" | ||
| 91 | print "#define USEC_TO_HZ_ADJ32\tU64_C(0x", fadj(s,hz,1000000), ")\n" | ||
| 92 | obase=10 | ||
| 93 | print "#define USEC_TO_HZ_SHR32\t", s, "\n" | ||
| 94 | |||
| 95 | obase=10 | ||
| 96 | cd=gcd(hz,1000000) | ||
| 97 | print "#define HZ_TO_USEC_NUM\t\t", 1000000/cd, "\n" | ||
| 98 | print "#define HZ_TO_USEC_DEN\t\t", hz/cd, "\n" | ||
| 99 | print "#define USEC_TO_HZ_NUM\t\t", hz/cd, "\n" | ||
| 100 | print "#define USEC_TO_HZ_DEN\t\t", 1000000/cd, "\n" | ||
| 101 | print "\n" | ||
| 102 | |||
| 103 | print "#endif /* KERNEL_TIMECONST_H */\n" | ||
| 104 | } | ||
| 105 | halt | ||
| 106 | } | ||
| 107 | |||
| 108 | timeconst(hz) | ||
diff --git a/kernel/time/timer.c b/kernel/time/timer.c new file mode 100644 index 000000000000..3bb01a323b2a --- /dev/null +++ b/kernel/time/timer.c | |||
| @@ -0,0 +1,1734 @@ | |||
| 1 | /* | ||
| 2 | * linux/kernel/timer.c | ||
| 3 | * | ||
| 4 | * Kernel internal timers | ||
| 5 | * | ||
| 6 | * Copyright (C) 1991, 1992 Linus Torvalds | ||
| 7 | * | ||
| 8 | * 1997-01-28 Modified by Finn Arne Gangstad to make timers scale better. | ||
| 9 | * | ||
| 10 | * 1997-09-10 Updated NTP code according to technical memorandum Jan '96 | ||
| 11 | * "A Kernel Model for Precision Timekeeping" by Dave Mills | ||
| 12 | * 1998-12-24 Fixed a xtime SMP race (we need the xtime_lock rw spinlock to | ||
| 13 | * serialize accesses to xtime/lost_ticks). | ||
| 14 | * Copyright (C) 1998 Andrea Arcangeli | ||
| 15 | * 1999-03-10 Improved NTP compatibility by Ulrich Windl | ||
| 16 | * 2002-05-31 Move sys_sysinfo here and make its locking sane, Robert Love | ||
| 17 | * 2000-10-05 Implemented scalable SMP per-CPU timer handling. | ||
| 18 | * Copyright (C) 2000, 2001, 2002 Ingo Molnar | ||
| 19 | * Designed by David S. Miller, Alexey Kuznetsov and Ingo Molnar | ||
| 20 | */ | ||
| 21 | |||
| 22 | #include <linux/kernel_stat.h> | ||
| 23 | #include <linux/export.h> | ||
| 24 | #include <linux/interrupt.h> | ||
| 25 | #include <linux/percpu.h> | ||
| 26 | #include <linux/init.h> | ||
| 27 | #include <linux/mm.h> | ||
| 28 | #include <linux/swap.h> | ||
| 29 | #include <linux/pid_namespace.h> | ||
| 30 | #include <linux/notifier.h> | ||
| 31 | #include <linux/thread_info.h> | ||
| 32 | #include <linux/time.h> | ||
| 33 | #include <linux/jiffies.h> | ||
| 34 | #include <linux/posix-timers.h> | ||
| 35 | #include <linux/cpu.h> | ||
| 36 | #include <linux/syscalls.h> | ||
| 37 | #include <linux/delay.h> | ||
| 38 | #include <linux/tick.h> | ||
| 39 | #include <linux/kallsyms.h> | ||
| 40 | #include <linux/irq_work.h> | ||
| 41 | #include <linux/sched.h> | ||
| 42 | #include <linux/sched/sysctl.h> | ||
| 43 | #include <linux/slab.h> | ||
| 44 | #include <linux/compat.h> | ||
| 45 | |||
| 46 | #include <asm/uaccess.h> | ||
| 47 | #include <asm/unistd.h> | ||
| 48 | #include <asm/div64.h> | ||
| 49 | #include <asm/timex.h> | ||
| 50 | #include <asm/io.h> | ||
| 51 | |||
| 52 | #define CREATE_TRACE_POINTS | ||
| 53 | #include <trace/events/timer.h> | ||
| 54 | |||
| 55 | __visible u64 jiffies_64 __cacheline_aligned_in_smp = INITIAL_JIFFIES; | ||
| 56 | |||
| 57 | EXPORT_SYMBOL(jiffies_64); | ||
| 58 | |||
| 59 | /* | ||
| 60 | * per-CPU timer vector definitions: | ||
| 61 | */ | ||
| 62 | #define TVN_BITS (CONFIG_BASE_SMALL ? 4 : 6) | ||
| 63 | #define TVR_BITS (CONFIG_BASE_SMALL ? 6 : 8) | ||
| 64 | #define TVN_SIZE (1 << TVN_BITS) | ||
| 65 | #define TVR_SIZE (1 << TVR_BITS) | ||
| 66 | #define TVN_MASK (TVN_SIZE - 1) | ||
| 67 | #define TVR_MASK (TVR_SIZE - 1) | ||
| 68 | #define MAX_TVAL ((unsigned long)((1ULL << (TVR_BITS + 4*TVN_BITS)) - 1)) | ||
| 69 | |||
| 70 | struct tvec { | ||
| 71 | struct list_head vec[TVN_SIZE]; | ||
| 72 | }; | ||
| 73 | |||
| 74 | struct tvec_root { | ||
| 75 | struct list_head vec[TVR_SIZE]; | ||
| 76 | }; | ||
| 77 | |||
| 78 | struct tvec_base { | ||
| 79 | spinlock_t lock; | ||
| 80 | struct timer_list *running_timer; | ||
| 81 | unsigned long timer_jiffies; | ||
| 82 | unsigned long next_timer; | ||
| 83 | unsigned long active_timers; | ||
| 84 | unsigned long all_timers; | ||
| 85 | struct tvec_root tv1; | ||
| 86 | struct tvec tv2; | ||
| 87 | struct tvec tv3; | ||
| 88 | struct tvec tv4; | ||
| 89 | struct tvec tv5; | ||
| 90 | } ____cacheline_aligned; | ||
| 91 | |||
| 92 | struct tvec_base boot_tvec_bases; | ||
| 93 | EXPORT_SYMBOL(boot_tvec_bases); | ||
| 94 | static DEFINE_PER_CPU(struct tvec_base *, tvec_bases) = &boot_tvec_bases; | ||
| 95 | |||
| 96 | /* Functions below help us manage 'deferrable' flag */ | ||
| 97 | static inline unsigned int tbase_get_deferrable(struct tvec_base *base) | ||
| 98 | { | ||
| 99 | return ((unsigned int)(unsigned long)base & TIMER_DEFERRABLE); | ||
| 100 | } | ||
| 101 | |||
| 102 | static inline unsigned int tbase_get_irqsafe(struct tvec_base *base) | ||
| 103 | { | ||
| 104 | return ((unsigned int)(unsigned long)base & TIMER_IRQSAFE); | ||
| 105 | } | ||
| 106 | |||
| 107 | static inline struct tvec_base *tbase_get_base(struct tvec_base *base) | ||
| 108 | { | ||
| 109 | return ((struct tvec_base *)((unsigned long)base & ~TIMER_FLAG_MASK)); | ||
| 110 | } | ||
| 111 | |||
| 112 | static inline void | ||
| 113 | timer_set_base(struct timer_list *timer, struct tvec_base *new_base) | ||
| 114 | { | ||
| 115 | unsigned long flags = (unsigned long)timer->base & TIMER_FLAG_MASK; | ||
| 116 | |||
| 117 | timer->base = (struct tvec_base *)((unsigned long)(new_base) | flags); | ||
| 118 | } | ||
| 119 | |||
| 120 | static unsigned long round_jiffies_common(unsigned long j, int cpu, | ||
| 121 | bool force_up) | ||
| 122 | { | ||
| 123 | int rem; | ||
| 124 | unsigned long original = j; | ||
| 125 | |||
| 126 | /* | ||
| 127 | * We don't want all cpus firing their timers at once hitting the | ||
| 128 | * same lock or cachelines, so we skew each extra cpu with an extra | ||
| 129 | * 3 jiffies. This 3 jiffies came originally from the mm/ code which | ||
| 130 | * already did this. | ||
| 131 | * The skew is done by adding 3*cpunr, then round, then subtract this | ||
| 132 | * extra offset again. | ||
| 133 | */ | ||
| 134 | j += cpu * 3; | ||
| 135 | |||
| 136 | rem = j % HZ; | ||
| 137 | |||
| 138 | /* | ||
| 139 | * If the target jiffie is just after a whole second (which can happen | ||
| 140 | * due to delays of the timer irq, long irq off times etc etc) then | ||
| 141 | * we should round down to the whole second, not up. Use 1/4th second | ||
| 142 | * as cutoff for this rounding as an extreme upper bound for this. | ||
| 143 | * But never round down if @force_up is set. | ||
| 144 | */ | ||
| 145 | if (rem < HZ/4 && !force_up) /* round down */ | ||
| 146 | j = j - rem; | ||
| 147 | else /* round up */ | ||
| 148 | j = j - rem + HZ; | ||
| 149 | |||
| 150 | /* now that we have rounded, subtract the extra skew again */ | ||
| 151 | j -= cpu * 3; | ||
| 152 | |||
| 153 | /* | ||
| 154 | * Make sure j is still in the future. Otherwise return the | ||
| 155 | * unmodified value. | ||
| 156 | */ | ||
| 157 | return time_is_after_jiffies(j) ? j : original; | ||
| 158 | } | ||
| 159 | |||
| 160 | /** | ||
| 161 | * __round_jiffies - function to round jiffies to a full second | ||
| 162 | * @j: the time in (absolute) jiffies that should be rounded | ||
| 163 | * @cpu: the processor number on which the timeout will happen | ||
| 164 | * | ||
| 165 | * __round_jiffies() rounds an absolute time in the future (in jiffies) | ||
| 166 | * up or down to (approximately) full seconds. This is useful for timers | ||
| 167 | * for which the exact time they fire does not matter too much, as long as | ||
| 168 | * they fire approximately every X seconds. | ||
| 169 | * | ||
| 170 | * By rounding these timers to whole seconds, all such timers will fire | ||
| 171 | * at the same time, rather than at various times spread out. The goal | ||
| 172 | * of this is to have the CPU wake up less, which saves power. | ||
| 173 | * | ||
| 174 | * The exact rounding is skewed for each processor to avoid all | ||
| 175 | * processors firing at the exact same time, which could lead | ||
| 176 | * to lock contention or spurious cache line bouncing. | ||
| 177 | * | ||
| 178 | * The return value is the rounded version of the @j parameter. | ||
| 179 | */ | ||
| 180 | unsigned long __round_jiffies(unsigned long j, int cpu) | ||
| 181 | { | ||
| 182 | return round_jiffies_common(j, cpu, false); | ||
| 183 | } | ||
| 184 | EXPORT_SYMBOL_GPL(__round_jiffies); | ||
| 185 | |||
| 186 | /** | ||
| 187 | * __round_jiffies_relative - function to round jiffies to a full second | ||
| 188 | * @j: the time in (relative) jiffies that should be rounded | ||
| 189 | * @cpu: the processor number on which the timeout will happen | ||
| 190 | * | ||
| 191 | * __round_jiffies_relative() rounds a time delta in the future (in jiffies) | ||
| 192 | * up or down to (approximately) full seconds. This is useful for timers | ||
| 193 | * for which the exact time they fire does not matter too much, as long as | ||
| 194 | * they fire approximately every X seconds. | ||
| 195 | * | ||
| 196 | * By rounding these timers to whole seconds, all such timers will fire | ||
| 197 | * at the same time, rather than at various times spread out. The goal | ||
| 198 | * of this is to have the CPU wake up less, which saves power. | ||
| 199 | * | ||
| 200 | * The exact rounding is skewed for each processor to avoid all | ||
| 201 | * processors firing at the exact same time, which could lead | ||
| 202 | * to lock contention or spurious cache line bouncing. | ||
| 203 | * | ||
| 204 | * The return value is the rounded version of the @j parameter. | ||
| 205 | */ | ||
| 206 | unsigned long __round_jiffies_relative(unsigned long j, int cpu) | ||
| 207 | { | ||
| 208 | unsigned long j0 = jiffies; | ||
| 209 | |||
| 210 | /* Use j0 because jiffies might change while we run */ | ||
| 211 | return round_jiffies_common(j + j0, cpu, false) - j0; | ||
| 212 | } | ||
| 213 | EXPORT_SYMBOL_GPL(__round_jiffies_relative); | ||
| 214 | |||
| 215 | /** | ||
| 216 | * round_jiffies - function to round jiffies to a full second | ||
| 217 | * @j: the time in (absolute) jiffies that should be rounded | ||
| 218 | * | ||
| 219 | * round_jiffies() rounds an absolute time in the future (in jiffies) | ||
| 220 | * up or down to (approximately) full seconds. This is useful for timers | ||
| 221 | * for which the exact time they fire does not matter too much, as long as | ||
| 222 | * they fire approximately every X seconds. | ||
| 223 | * | ||
| 224 | * By rounding these timers to whole seconds, all such timers will fire | ||
| 225 | * at the same time, rather than at various times spread out. The goal | ||
| 226 | * of this is to have the CPU wake up less, which saves power. | ||
| 227 | * | ||
| 228 | * The return value is the rounded version of the @j parameter. | ||
| 229 | */ | ||
| 230 | unsigned long round_jiffies(unsigned long j) | ||
| 231 | { | ||
| 232 | return round_jiffies_common(j, raw_smp_processor_id(), false); | ||
| 233 | } | ||
| 234 | EXPORT_SYMBOL_GPL(round_jiffies); | ||
| 235 | |||
| 236 | /** | ||
| 237 | * round_jiffies_relative - function to round jiffies to a full second | ||
| 238 | * @j: the time in (relative) jiffies that should be rounded | ||
| 239 | * | ||
| 240 | * round_jiffies_relative() rounds a time delta in the future (in jiffies) | ||
| 241 | * up or down to (approximately) full seconds. This is useful for timers | ||
| 242 | * for which the exact time they fire does not matter too much, as long as | ||
| 243 | * they fire approximately every X seconds. | ||
| 244 | * | ||
| 245 | * By rounding these timers to whole seconds, all such timers will fire | ||
| 246 | * at the same time, rather than at various times spread out. The goal | ||
| 247 | * of this is to have the CPU wake up less, which saves power. | ||
| 248 | * | ||
| 249 | * The return value is the rounded version of the @j parameter. | ||
| 250 | */ | ||
| 251 | unsigned long round_jiffies_relative(unsigned long j) | ||
| 252 | { | ||
| 253 | return __round_jiffies_relative(j, raw_smp_processor_id()); | ||
| 254 | } | ||
| 255 | EXPORT_SYMBOL_GPL(round_jiffies_relative); | ||
| 256 | |||
| 257 | /** | ||
| 258 | * __round_jiffies_up - function to round jiffies up to a full second | ||
| 259 | * @j: the time in (absolute) jiffies that should be rounded | ||
| 260 | * @cpu: the processor number on which the timeout will happen | ||
| 261 | * | ||
| 262 | * This is the same as __round_jiffies() except that it will never | ||
| 263 | * round down. This is useful for timeouts for which the exact time | ||
| 264 | * of firing does not matter too much, as long as they don't fire too | ||
| 265 | * early. | ||
| 266 | */ | ||
| 267 | unsigned long __round_jiffies_up(unsigned long j, int cpu) | ||
| 268 | { | ||
| 269 | return round_jiffies_common(j, cpu, true); | ||
| 270 | } | ||
| 271 | EXPORT_SYMBOL_GPL(__round_jiffies_up); | ||
| 272 | |||
| 273 | /** | ||
| 274 | * __round_jiffies_up_relative - function to round jiffies up to a full second | ||
| 275 | * @j: the time in (relative) jiffies that should be rounded | ||
| 276 | * @cpu: the processor number on which the timeout will happen | ||
| 277 | * | ||
| 278 | * This is the same as __round_jiffies_relative() except that it will never | ||
| 279 | * round down. This is useful for timeouts for which the exact time | ||
| 280 | * of firing does not matter too much, as long as they don't fire too | ||
| 281 | * early. | ||
| 282 | */ | ||
| 283 | unsigned long __round_jiffies_up_relative(unsigned long j, int cpu) | ||
| 284 | { | ||
| 285 | unsigned long j0 = jiffies; | ||
| 286 | |||
| 287 | /* Use j0 because jiffies might change while we run */ | ||
| 288 | return round_jiffies_common(j + j0, cpu, true) - j0; | ||
| 289 | } | ||
| 290 | EXPORT_SYMBOL_GPL(__round_jiffies_up_relative); | ||
| 291 | |||
| 292 | /** | ||
| 293 | * round_jiffies_up - function to round jiffies up to a full second | ||
| 294 | * @j: the time in (absolute) jiffies that should be rounded | ||
| 295 | * | ||
| 296 | * This is the same as round_jiffies() except that it will never | ||
| 297 | * round down. This is useful for timeouts for which the exact time | ||
| 298 | * of firing does not matter too much, as long as they don't fire too | ||
| 299 | * early. | ||
| 300 | */ | ||
| 301 | unsigned long round_jiffies_up(unsigned long j) | ||
| 302 | { | ||
| 303 | return round_jiffies_common(j, raw_smp_processor_id(), true); | ||
| 304 | } | ||
| 305 | EXPORT_SYMBOL_GPL(round_jiffies_up); | ||
| 306 | |||
| 307 | /** | ||
| 308 | * round_jiffies_up_relative - function to round jiffies up to a full second | ||
| 309 | * @j: the time in (relative) jiffies that should be rounded | ||
| 310 | * | ||
| 311 | * This is the same as round_jiffies_relative() except that it will never | ||
| 312 | * round down. This is useful for timeouts for which the exact time | ||
| 313 | * of firing does not matter too much, as long as they don't fire too | ||
| 314 | * early. | ||
| 315 | */ | ||
| 316 | unsigned long round_jiffies_up_relative(unsigned long j) | ||
| 317 | { | ||
| 318 | return __round_jiffies_up_relative(j, raw_smp_processor_id()); | ||
| 319 | } | ||
| 320 | EXPORT_SYMBOL_GPL(round_jiffies_up_relative); | ||
| 321 | |||
| 322 | /** | ||
| 323 | * set_timer_slack - set the allowed slack for a timer | ||
| 324 | * @timer: the timer to be modified | ||
| 325 | * @slack_hz: the amount of time (in jiffies) allowed for rounding | ||
| 326 | * | ||
| 327 | * Set the amount of time, in jiffies, that a certain timer has | ||
| 328 | * in terms of slack. By setting this value, the timer subsystem | ||
| 329 | * will schedule the actual timer somewhere between | ||
| 330 | * the time mod_timer() asks for, and that time plus the slack. | ||
| 331 | * | ||
| 332 | * By setting the slack to -1, a percentage of the delay is used | ||
| 333 | * instead. | ||
| 334 | */ | ||
| 335 | void set_timer_slack(struct timer_list *timer, int slack_hz) | ||
| 336 | { | ||
| 337 | timer->slack = slack_hz; | ||
| 338 | } | ||
| 339 | EXPORT_SYMBOL_GPL(set_timer_slack); | ||
| 340 | |||
| 341 | /* | ||
| 342 | * If the list is empty, catch up ->timer_jiffies to the current time. | ||
| 343 | * The caller must hold the tvec_base lock. Returns true if the list | ||
| 344 | * was empty and therefore ->timer_jiffies was updated. | ||
| 345 | */ | ||
| 346 | static bool catchup_timer_jiffies(struct tvec_base *base) | ||
| 347 | { | ||
| 348 | if (!base->all_timers) { | ||
| 349 | base->timer_jiffies = jiffies; | ||
| 350 | return true; | ||
| 351 | } | ||
| 352 | return false; | ||
| 353 | } | ||
| 354 | |||
| 355 | static void | ||
| 356 | __internal_add_timer(struct tvec_base *base, struct timer_list *timer) | ||
| 357 | { | ||
| 358 | unsigned long expires = timer->expires; | ||
| 359 | unsigned long idx = expires - base->timer_jiffies; | ||
| 360 | struct list_head *vec; | ||
| 361 | |||
| 362 | if (idx < TVR_SIZE) { | ||
| 363 | int i = expires & TVR_MASK; | ||
| 364 | vec = base->tv1.vec + i; | ||
| 365 | } else if (idx < 1 << (TVR_BITS + TVN_BITS)) { | ||
| 366 | int i = (expires >> TVR_BITS) & TVN_MASK; | ||
| 367 | vec = base->tv2.vec + i; | ||
| 368 | } else if (idx < 1 << (TVR_BITS + 2 * TVN_BITS)) { | ||
| 369 | int i = (expires >> (TVR_BITS + TVN_BITS)) & TVN_MASK; | ||
| 370 | vec = base->tv3.vec + i; | ||
| 371 | } else if (idx < 1 << (TVR_BITS + 3 * TVN_BITS)) { | ||
| 372 | int i = (expires >> (TVR_BITS + 2 * TVN_BITS)) & TVN_MASK; | ||
| 373 | vec = base->tv4.vec + i; | ||
| 374 | } else if ((signed long) idx < 0) { | ||
| 375 | /* | ||
| 376 | * Can happen if you add a timer with expires == jiffies, | ||
| 377 | * or you set a timer to go off in the past | ||
| 378 | */ | ||
| 379 | vec = base->tv1.vec + (base->timer_jiffies & TVR_MASK); | ||
| 380 | } else { | ||
| 381 | int i; | ||
| 382 | /* If the timeout is larger than MAX_TVAL (on 64-bit | ||
| 383 | * architectures or with CONFIG_BASE_SMALL=1) then we | ||
| 384 | * use the maximum timeout. | ||
| 385 | */ | ||
| 386 | if (idx > MAX_TVAL) { | ||
| 387 | idx = MAX_TVAL; | ||
| 388 | expires = idx + base->timer_jiffies; | ||
| 389 | } | ||
| 390 | i = (expires >> (TVR_BITS + 3 * TVN_BITS)) & TVN_MASK; | ||
| 391 | vec = base->tv5.vec + i; | ||
| 392 | } | ||
| 393 | /* | ||
| 394 | * Timers are FIFO: | ||
| 395 | */ | ||
| 396 | list_add_tail(&timer->entry, vec); | ||
| 397 | } | ||
| 398 | |||
| 399 | static void internal_add_timer(struct tvec_base *base, struct timer_list *timer) | ||
| 400 | { | ||
| 401 | (void)catchup_timer_jiffies(base); | ||
| 402 | __internal_add_timer(base, timer); | ||
| 403 | /* | ||
| 404 | * Update base->active_timers and base->next_timer | ||
| 405 | */ | ||
| 406 | if (!tbase_get_deferrable(timer->base)) { | ||
| 407 | if (!base->active_timers++ || | ||
| 408 | time_before(timer->expires, base->next_timer)) | ||
| 409 | base->next_timer = timer->expires; | ||
| 410 | } | ||
| 411 | base->all_timers++; | ||
| 412 | } | ||
| 413 | |||
| 414 | #ifdef CONFIG_TIMER_STATS | ||
| 415 | void __timer_stats_timer_set_start_info(struct timer_list *timer, void *addr) | ||
| 416 | { | ||
| 417 | if (timer->start_site) | ||
| 418 | return; | ||
| 419 | |||
| 420 | timer->start_site = addr; | ||
| 421 | memcpy(timer->start_comm, current->comm, TASK_COMM_LEN); | ||
| 422 | timer->start_pid = current->pid; | ||
| 423 | } | ||
| 424 | |||
| 425 | static void timer_stats_account_timer(struct timer_list *timer) | ||
| 426 | { | ||
| 427 | unsigned int flag = 0; | ||
| 428 | |||
| 429 | if (likely(!timer->start_site)) | ||
| 430 | return; | ||
| 431 | if (unlikely(tbase_get_deferrable(timer->base))) | ||
| 432 | flag |= TIMER_STATS_FLAG_DEFERRABLE; | ||
| 433 | |||
| 434 | timer_stats_update_stats(timer, timer->start_pid, timer->start_site, | ||
| 435 | timer->function, timer->start_comm, flag); | ||
| 436 | } | ||
| 437 | |||
| 438 | #else | ||
| 439 | static void timer_stats_account_timer(struct timer_list *timer) {} | ||
| 440 | #endif | ||
| 441 | |||
| 442 | #ifdef CONFIG_DEBUG_OBJECTS_TIMERS | ||
| 443 | |||
| 444 | static struct debug_obj_descr timer_debug_descr; | ||
| 445 | |||
| 446 | static void *timer_debug_hint(void *addr) | ||
| 447 | { | ||
| 448 | return ((struct timer_list *) addr)->function; | ||
| 449 | } | ||
| 450 | |||
| 451 | /* | ||
| 452 | * fixup_init is called when: | ||
| 453 | * - an active object is initialized | ||
| 454 | */ | ||
| 455 | static int timer_fixup_init(void *addr, enum debug_obj_state state) | ||
| 456 | { | ||
| 457 | struct timer_list *timer = addr; | ||
| 458 | |||
| 459 | switch (state) { | ||
| 460 | case ODEBUG_STATE_ACTIVE: | ||
| 461 | del_timer_sync(timer); | ||
| 462 | debug_object_init(timer, &timer_debug_descr); | ||
| 463 | return 1; | ||
| 464 | default: | ||
| 465 | return 0; | ||
| 466 | } | ||
| 467 | } | ||
| 468 | |||
| 469 | /* Stub timer callback for improperly used timers. */ | ||
| 470 | static void stub_timer(unsigned long data) | ||
| 471 | { | ||
| 472 | WARN_ON(1); | ||
| 473 | } | ||
| 474 | |||
| 475 | /* | ||
| 476 | * fixup_activate is called when: | ||
| 477 | * - an active object is activated | ||
| 478 | * - an unknown object is activated (might be a statically initialized object) | ||
| 479 | */ | ||
| 480 | static int timer_fixup_activate(void *addr, enum debug_obj_state state) | ||
| 481 | { | ||
| 482 | struct timer_list *timer = addr; | ||
| 483 | |||
| 484 | switch (state) { | ||
| 485 | |||
| 486 | case ODEBUG_STATE_NOTAVAILABLE: | ||
| 487 | /* | ||
| 488 | * This is not really a fixup. The timer was | ||
| 489 | * statically initialized. We just make sure that it | ||
| 490 | * is tracked in the object tracker. | ||
| 491 | */ | ||
| 492 | if (timer->entry.next == NULL && | ||
| 493 | timer->entry.prev == TIMER_ENTRY_STATIC) { | ||
| 494 | debug_object_init(timer, &timer_debug_descr); | ||
| 495 | debug_object_activate(timer, &timer_debug_descr); | ||
| 496 | return 0; | ||
| 497 | } else { | ||
| 498 | setup_timer(timer, stub_timer, 0); | ||
| 499 | return 1; | ||
| 500 | } | ||
| 501 | return 0; | ||
| 502 | |||
| 503 | case ODEBUG_STATE_ACTIVE: | ||
| 504 | WARN_ON(1); | ||
| 505 | |||
| 506 | default: | ||
| 507 | return 0; | ||
| 508 | } | ||
| 509 | } | ||
| 510 | |||
| 511 | /* | ||
| 512 | * fixup_free is called when: | ||
| 513 | * - an active object is freed | ||
| 514 | */ | ||
| 515 | static int timer_fixup_free(void *addr, enum debug_obj_state state) | ||
| 516 | { | ||
| 517 | struct timer_list *timer = addr; | ||
| 518 | |||
| 519 | switch (state) { | ||
| 520 | case ODEBUG_STATE_ACTIVE: | ||
| 521 | del_timer_sync(timer); | ||
| 522 | debug_object_free(timer, &timer_debug_descr); | ||
| 523 | return 1; | ||
| 524 | default: | ||
| 525 | return 0; | ||
| 526 | } | ||
| 527 | } | ||
| 528 | |||
| 529 | /* | ||
| 530 | * fixup_assert_init is called when: | ||
| 531 | * - an untracked/uninit-ed object is found | ||
| 532 | */ | ||
| 533 | static int timer_fixup_assert_init(void *addr, enum debug_obj_state state) | ||
| 534 | { | ||
| 535 | struct timer_list *timer = addr; | ||
| 536 | |||
| 537 | switch (state) { | ||
| 538 | case ODEBUG_STATE_NOTAVAILABLE: | ||
| 539 | if (timer->entry.prev == TIMER_ENTRY_STATIC) { | ||
| 540 | /* | ||
| 541 | * This is not really a fixup. The timer was | ||
| 542 | * statically initialized. We just make sure that it | ||
| 543 | * is tracked in the object tracker. | ||
| 544 | */ | ||
| 545 | debug_object_init(timer, &timer_debug_descr); | ||
| 546 | return 0; | ||
| 547 | } else { | ||
| 548 | setup_timer(timer, stub_timer, 0); | ||
| 549 | return 1; | ||
| 550 | } | ||
| 551 | default: | ||
| 552 | return 0; | ||
| 553 | } | ||
| 554 | } | ||
| 555 | |||
| 556 | static struct debug_obj_descr timer_debug_descr = { | ||
| 557 | .name = "timer_list", | ||
| 558 | .debug_hint = timer_debug_hint, | ||
| 559 | .fixup_init = timer_fixup_init, | ||
| 560 | .fixup_activate = timer_fixup_activate, | ||
| 561 | .fixup_free = timer_fixup_free, | ||
| 562 | .fixup_assert_init = timer_fixup_assert_init, | ||
| 563 | }; | ||
| 564 | |||
| 565 | static inline void debug_timer_init(struct timer_list *timer) | ||
| 566 | { | ||
| 567 | debug_object_init(timer, &timer_debug_descr); | ||
| 568 | } | ||
| 569 | |||
| 570 | static inline void debug_timer_activate(struct timer_list *timer) | ||
| 571 | { | ||
| 572 | debug_object_activate(timer, &timer_debug_descr); | ||
| 573 | } | ||
| 574 | |||
| 575 | static inline void debug_timer_deactivate(struct timer_list *timer) | ||
| 576 | { | ||
| 577 | debug_object_deactivate(timer, &timer_debug_descr); | ||
| 578 | } | ||
| 579 | |||
| 580 | static inline void debug_timer_free(struct timer_list *timer) | ||
| 581 | { | ||
| 582 | debug_object_free(timer, &timer_debug_descr); | ||
| 583 | } | ||
| 584 | |||
| 585 | static inline void debug_timer_assert_init(struct timer_list *timer) | ||
| 586 | { | ||
| 587 | debug_object_assert_init(timer, &timer_debug_descr); | ||
| 588 | } | ||
| 589 | |||
| 590 | static void do_init_timer(struct timer_list *timer, unsigned int flags, | ||
| 591 | const char *name, struct lock_class_key *key); | ||
| 592 | |||
| 593 | void init_timer_on_stack_key(struct timer_list *timer, unsigned int flags, | ||
| 594 | const char *name, struct lock_class_key *key) | ||
| 595 | { | ||
| 596 | debug_object_init_on_stack(timer, &timer_debug_descr); | ||
| 597 | do_init_timer(timer, flags, name, key); | ||
| 598 | } | ||
| 599 | EXPORT_SYMBOL_GPL(init_timer_on_stack_key); | ||
| 600 | |||
| 601 | void destroy_timer_on_stack(struct timer_list *timer) | ||
| 602 | { | ||
| 603 | debug_object_free(timer, &timer_debug_descr); | ||
| 604 | } | ||
| 605 | EXPORT_SYMBOL_GPL(destroy_timer_on_stack); | ||
| 606 | |||
| 607 | #else | ||
| 608 | static inline void debug_timer_init(struct timer_list *timer) { } | ||
| 609 | static inline void debug_timer_activate(struct timer_list *timer) { } | ||
| 610 | static inline void debug_timer_deactivate(struct timer_list *timer) { } | ||
| 611 | static inline void debug_timer_assert_init(struct timer_list *timer) { } | ||
| 612 | #endif | ||
| 613 | |||
| 614 | static inline void debug_init(struct timer_list *timer) | ||
| 615 | { | ||
| 616 | debug_timer_init(timer); | ||
| 617 | trace_timer_init(timer); | ||
| 618 | } | ||
| 619 | |||
| 620 | static inline void | ||
| 621 | debug_activate(struct timer_list *timer, unsigned long expires) | ||
| 622 | { | ||
| 623 | debug_timer_activate(timer); | ||
| 624 | trace_timer_start(timer, expires); | ||
| 625 | } | ||
| 626 | |||
| 627 | static inline void debug_deactivate(struct timer_list *timer) | ||
| 628 | { | ||
| 629 | debug_timer_deactivate(timer); | ||
| 630 | trace_timer_cancel(timer); | ||
| 631 | } | ||
| 632 | |||
| 633 | static inline void debug_assert_init(struct timer_list *timer) | ||
| 634 | { | ||
| 635 | debug_timer_assert_init(timer); | ||
| 636 | } | ||
| 637 | |||
| 638 | static void do_init_timer(struct timer_list *timer, unsigned int flags, | ||
| 639 | const char *name, struct lock_class_key *key) | ||
| 640 | { | ||
| 641 | struct tvec_base *base = __raw_get_cpu_var(tvec_bases); | ||
| 642 | |||
| 643 | timer->entry.next = NULL; | ||
| 644 | timer->base = (void *)((unsigned long)base | flags); | ||
| 645 | timer->slack = -1; | ||
| 646 | #ifdef CONFIG_TIMER_STATS | ||
| 647 | timer->start_site = NULL; | ||
| 648 | timer->start_pid = -1; | ||
| 649 | memset(timer->start_comm, 0, TASK_COMM_LEN); | ||
| 650 | #endif | ||
| 651 | lockdep_init_map(&timer->lockdep_map, name, key, 0); | ||
| 652 | } | ||
| 653 | |||
| 654 | /** | ||
| 655 | * init_timer_key - initialize a timer | ||
| 656 | * @timer: the timer to be initialized | ||
| 657 | * @flags: timer flags | ||
| 658 | * @name: name of the timer | ||
| 659 | * @key: lockdep class key of the fake lock used for tracking timer | ||
| 660 | * sync lock dependencies | ||
| 661 | * | ||
| 662 | * init_timer_key() must be done to a timer prior calling *any* of the | ||
| 663 | * other timer functions. | ||
| 664 | */ | ||
| 665 | void init_timer_key(struct timer_list *timer, unsigned int flags, | ||
| 666 | const char *name, struct lock_class_key *key) | ||
| 667 | { | ||
| 668 | debug_init(timer); | ||
| 669 | do_init_timer(timer, flags, name, key); | ||
| 670 | } | ||
| 671 | EXPORT_SYMBOL(init_timer_key); | ||
| 672 | |||
| 673 | static inline void detach_timer(struct timer_list *timer, bool clear_pending) | ||
| 674 | { | ||
| 675 | struct list_head *entry = &timer->entry; | ||
| 676 | |||
| 677 | debug_deactivate(timer); | ||
| 678 | |||
| 679 | __list_del(entry->prev, entry->next); | ||
| 680 | if (clear_pending) | ||
| 681 | entry->next = NULL; | ||
| 682 | entry->prev = LIST_POISON2; | ||
| 683 | } | ||
| 684 | |||
| 685 | static inline void | ||
| 686 | detach_expired_timer(struct timer_list *timer, struct tvec_base *base) | ||
| 687 | { | ||
| 688 | detach_timer(timer, true); | ||
| 689 | if (!tbase_get_deferrable(timer->base)) | ||
| 690 | base->active_timers--; | ||
| 691 | base->all_timers--; | ||
| 692 | (void)catchup_timer_jiffies(base); | ||
| 693 | } | ||
| 694 | |||
| 695 | static int detach_if_pending(struct timer_list *timer, struct tvec_base *base, | ||
| 696 | bool clear_pending) | ||
| 697 | { | ||
| 698 | if (!timer_pending(timer)) | ||
| 699 | return 0; | ||
| 700 | |||
| 701 | detach_timer(timer, clear_pending); | ||
| 702 | if (!tbase_get_deferrable(timer->base)) { | ||
| 703 | base->active_timers--; | ||
| 704 | if (timer->expires == base->next_timer) | ||
| 705 | base->next_timer = base->timer_jiffies; | ||
| 706 | } | ||
| 707 | base->all_timers--; | ||
| 708 | (void)catchup_timer_jiffies(base); | ||
| 709 | return 1; | ||
| 710 | } | ||
| 711 | |||
| 712 | /* | ||
| 713 | * We are using hashed locking: holding per_cpu(tvec_bases).lock | ||
| 714 | * means that all timers which are tied to this base via timer->base are | ||
| 715 | * locked, and the base itself is locked too. | ||
| 716 | * | ||
| 717 | * So __run_timers/migrate_timers can safely modify all timers which could | ||
| 718 | * be found on ->tvX lists. | ||
| 719 | * | ||
| 720 | * When the timer's base is locked, and the timer removed from list, it is | ||
| 721 | * possible to set timer->base = NULL and drop the lock: the timer remains | ||
| 722 | * locked. | ||
| 723 | */ | ||
| 724 | static struct tvec_base *lock_timer_base(struct timer_list *timer, | ||
| 725 | unsigned long *flags) | ||
| 726 | __acquires(timer->base->lock) | ||
| 727 | { | ||
| 728 | struct tvec_base *base; | ||
| 729 | |||
| 730 | for (;;) { | ||
| 731 | struct tvec_base *prelock_base = timer->base; | ||
| 732 | base = tbase_get_base(prelock_base); | ||
| 733 | if (likely(base != NULL)) { | ||
| 734 | spin_lock_irqsave(&base->lock, *flags); | ||
| 735 | if (likely(prelock_base == timer->base)) | ||
| 736 | return base; | ||
| 737 | /* The timer has migrated to another CPU */ | ||
| 738 | spin_unlock_irqrestore(&base->lock, *flags); | ||
| 739 | } | ||
| 740 | cpu_relax(); | ||
| 741 | } | ||
| 742 | } | ||
| 743 | |||
| 744 | static inline int | ||
| 745 | __mod_timer(struct timer_list *timer, unsigned long expires, | ||
| 746 | bool pending_only, int pinned) | ||
| 747 | { | ||
| 748 | struct tvec_base *base, *new_base; | ||
| 749 | unsigned long flags; | ||
| 750 | int ret = 0 , cpu; | ||
| 751 | |||
| 752 | timer_stats_timer_set_start_info(timer); | ||
| 753 | BUG_ON(!timer->function); | ||
| 754 | |||
| 755 | base = lock_timer_base(timer, &flags); | ||
| 756 | |||
| 757 | ret = detach_if_pending(timer, base, false); | ||
| 758 | if (!ret && pending_only) | ||
| 759 | goto out_unlock; | ||
| 760 | |||
| 761 | debug_activate(timer, expires); | ||
| 762 | |||
| 763 | cpu = get_nohz_timer_target(pinned); | ||
| 764 | new_base = per_cpu(tvec_bases, cpu); | ||
| 765 | |||
| 766 | if (base != new_base) { | ||
| 767 | /* | ||
| 768 | * We are trying to schedule the timer on the local CPU. | ||
| 769 | * However we can't change timer's base while it is running, | ||
| 770 | * otherwise del_timer_sync() can't detect that the timer's | ||
| 771 | * handler yet has not finished. This also guarantees that | ||
| 772 | * the timer is serialized wrt itself. | ||
| 773 | */ | ||
| 774 | if (likely(base->running_timer != timer)) { | ||
| 775 | /* See the comment in lock_timer_base() */ | ||
| 776 | timer_set_base(timer, NULL); | ||
| 777 | spin_unlock(&base->lock); | ||
| 778 | base = new_base; | ||
| 779 | spin_lock(&base->lock); | ||
| 780 | timer_set_base(timer, base); | ||
| 781 | } | ||
| 782 | } | ||
| 783 | |||
| 784 | timer->expires = expires; | ||
| 785 | internal_add_timer(base, timer); | ||
| 786 | |||
| 787 | out_unlock: | ||
| 788 | spin_unlock_irqrestore(&base->lock, flags); | ||
| 789 | |||
| 790 | return ret; | ||
| 791 | } | ||
| 792 | |||
| 793 | /** | ||
| 794 | * mod_timer_pending - modify a pending timer's timeout | ||
| 795 | * @timer: the pending timer to be modified | ||
| 796 | * @expires: new timeout in jiffies | ||
| 797 | * | ||
| 798 | * mod_timer_pending() is the same for pending timers as mod_timer(), | ||
| 799 | * but will not re-activate and modify already deleted timers. | ||
| 800 | * | ||
| 801 | * It is useful for unserialized use of timers. | ||
| 802 | */ | ||
| 803 | int mod_timer_pending(struct timer_list *timer, unsigned long expires) | ||
| 804 | { | ||
| 805 | return __mod_timer(timer, expires, true, TIMER_NOT_PINNED); | ||
| 806 | } | ||
| 807 | EXPORT_SYMBOL(mod_timer_pending); | ||
| 808 | |||
| 809 | /* | ||
| 810 | * Decide where to put the timer while taking the slack into account | ||
| 811 | * | ||
| 812 | * Algorithm: | ||
| 813 | * 1) calculate the maximum (absolute) time | ||
| 814 | * 2) calculate the highest bit where the expires and new max are different | ||
| 815 | * 3) use this bit to make a mask | ||
| 816 | * 4) use the bitmask to round down the maximum time, so that all last | ||
| 817 | * bits are zeros | ||
| 818 | */ | ||
| 819 | static inline | ||
| 820 | unsigned long apply_slack(struct timer_list *timer, unsigned long expires) | ||
| 821 | { | ||
| 822 | unsigned long expires_limit, mask; | ||
| 823 | int bit; | ||
| 824 | |||
| 825 | if (timer->slack >= 0) { | ||
| 826 | expires_limit = expires + timer->slack; | ||
| 827 | } else { | ||
| 828 | long delta = expires - jiffies; | ||
| 829 | |||
| 830 | if (delta < 256) | ||
| 831 | return expires; | ||
| 832 | |||
| 833 | expires_limit = expires + delta / 256; | ||
| 834 | } | ||
| 835 | mask = expires ^ expires_limit; | ||
| 836 | if (mask == 0) | ||
| 837 | return expires; | ||
| 838 | |||
| 839 | bit = find_last_bit(&mask, BITS_PER_LONG); | ||
| 840 | |||
| 841 | mask = (1UL << bit) - 1; | ||
| 842 | |||
| 843 | expires_limit = expires_limit & ~(mask); | ||
| 844 | |||
| 845 | return expires_limit; | ||
| 846 | } | ||
| 847 | |||
| 848 | /** | ||
| 849 | * mod_timer - modify a timer's timeout | ||
| 850 | * @timer: the timer to be modified | ||
| 851 | * @expires: new timeout in jiffies | ||
| 852 | * | ||
| 853 | * mod_timer() is a more efficient way to update the expire field of an | ||
| 854 | * active timer (if the timer is inactive it will be activated) | ||
| 855 | * | ||
| 856 | * mod_timer(timer, expires) is equivalent to: | ||
| 857 | * | ||
| 858 | * del_timer(timer); timer->expires = expires; add_timer(timer); | ||
| 859 | * | ||
| 860 | * Note that if there are multiple unserialized concurrent users of the | ||
| 861 | * same timer, then mod_timer() is the only safe way to modify the timeout, | ||
| 862 | * since add_timer() cannot modify an already running timer. | ||
| 863 | * | ||
| 864 | * The function returns whether it has modified a pending timer or not. | ||
| 865 | * (ie. mod_timer() of an inactive timer returns 0, mod_timer() of an | ||
| 866 | * active timer returns 1.) | ||
| 867 | */ | ||
| 868 | int mod_timer(struct timer_list *timer, unsigned long expires) | ||
| 869 | { | ||
| 870 | expires = apply_slack(timer, expires); | ||
| 871 | |||
| 872 | /* | ||
| 873 | * This is a common optimization triggered by the | ||
| 874 | * networking code - if the timer is re-modified | ||
| 875 | * to be the same thing then just return: | ||
| 876 | */ | ||
| 877 | if (timer_pending(timer) && timer->expires == expires) | ||
| 878 | return 1; | ||
| 879 | |||
| 880 | return __mod_timer(timer, expires, false, TIMER_NOT_PINNED); | ||
| 881 | } | ||
| 882 | EXPORT_SYMBOL(mod_timer); | ||
| 883 | |||
| 884 | /** | ||
| 885 | * mod_timer_pinned - modify a timer's timeout | ||
| 886 | * @timer: the timer to be modified | ||
| 887 | * @expires: new timeout in jiffies | ||
| 888 | * | ||
| 889 | * mod_timer_pinned() is a way to update the expire field of an | ||
| 890 | * active timer (if the timer is inactive it will be activated) | ||
| 891 | * and to ensure that the timer is scheduled on the current CPU. | ||
| 892 | * | ||
| 893 | * Note that this does not prevent the timer from being migrated | ||
| 894 | * when the current CPU goes offline. If this is a problem for | ||
| 895 | * you, use CPU-hotplug notifiers to handle it correctly, for | ||
| 896 | * example, cancelling the timer when the corresponding CPU goes | ||
| 897 | * offline. | ||
| 898 | * | ||
| 899 | * mod_timer_pinned(timer, expires) is equivalent to: | ||
| 900 | * | ||
| 901 | * del_timer(timer); timer->expires = expires; add_timer(timer); | ||
| 902 | */ | ||
| 903 | int mod_timer_pinned(struct timer_list *timer, unsigned long expires) | ||
| 904 | { | ||
| 905 | if (timer->expires == expires && timer_pending(timer)) | ||
| 906 | return 1; | ||
| 907 | |||
| 908 | return __mod_timer(timer, expires, false, TIMER_PINNED); | ||
| 909 | } | ||
| 910 | EXPORT_SYMBOL(mod_timer_pinned); | ||
| 911 | |||
| 912 | /** | ||
| 913 | * add_timer - start a timer | ||
| 914 | * @timer: the timer to be added | ||
| 915 | * | ||
| 916 | * The kernel will do a ->function(->data) callback from the | ||
| 917 | * timer interrupt at the ->expires point in the future. The | ||
| 918 | * current time is 'jiffies'. | ||
| 919 | * | ||
| 920 | * The timer's ->expires, ->function (and if the handler uses it, ->data) | ||
| 921 | * fields must be set prior calling this function. | ||
| 922 | * | ||
| 923 | * Timers with an ->expires field in the past will be executed in the next | ||
| 924 | * timer tick. | ||
| 925 | */ | ||
| 926 | void add_timer(struct timer_list *timer) | ||
| 927 | { | ||
| 928 | BUG_ON(timer_pending(timer)); | ||
| 929 | mod_timer(timer, timer->expires); | ||
| 930 | } | ||
| 931 | EXPORT_SYMBOL(add_timer); | ||
| 932 | |||
| 933 | /** | ||
| 934 | * add_timer_on - start a timer on a particular CPU | ||
| 935 | * @timer: the timer to be added | ||
| 936 | * @cpu: the CPU to start it on | ||
| 937 | * | ||
| 938 | * This is not very scalable on SMP. Double adds are not possible. | ||
| 939 | */ | ||
| 940 | void add_timer_on(struct timer_list *timer, int cpu) | ||
| 941 | { | ||
| 942 | struct tvec_base *base = per_cpu(tvec_bases, cpu); | ||
| 943 | unsigned long flags; | ||
| 944 | |||
| 945 | timer_stats_timer_set_start_info(timer); | ||
| 946 | BUG_ON(timer_pending(timer) || !timer->function); | ||
| 947 | spin_lock_irqsave(&base->lock, flags); | ||
| 948 | timer_set_base(timer, base); | ||
| 949 | debug_activate(timer, timer->expires); | ||
| 950 | internal_add_timer(base, timer); | ||
| 951 | /* | ||
| 952 | * Check whether the other CPU is in dynticks mode and needs | ||
| 953 | * to be triggered to reevaluate the timer wheel. | ||
| 954 | * We are protected against the other CPU fiddling | ||
| 955 | * with the timer by holding the timer base lock. This also | ||
| 956 | * makes sure that a CPU on the way to stop its tick can not | ||
| 957 | * evaluate the timer wheel. | ||
| 958 | * | ||
| 959 | * Spare the IPI for deferrable timers on idle targets though. | ||
| 960 | * The next busy ticks will take care of it. Except full dynticks | ||
| 961 | * require special care against races with idle_cpu(), lets deal | ||
| 962 | * with that later. | ||
| 963 | */ | ||
| 964 | if (!tbase_get_deferrable(timer->base) || tick_nohz_full_cpu(cpu)) | ||
| 965 | wake_up_nohz_cpu(cpu); | ||
| 966 | |||
| 967 | spin_unlock_irqrestore(&base->lock, flags); | ||
| 968 | } | ||
| 969 | EXPORT_SYMBOL_GPL(add_timer_on); | ||
| 970 | |||
| 971 | /** | ||
| 972 | * del_timer - deactive a timer. | ||
| 973 | * @timer: the timer to be deactivated | ||
| 974 | * | ||
| 975 | * del_timer() deactivates a timer - this works on both active and inactive | ||
| 976 | * timers. | ||
| 977 | * | ||
| 978 | * The function returns whether it has deactivated a pending timer or not. | ||
| 979 | * (ie. del_timer() of an inactive timer returns 0, del_timer() of an | ||
| 980 | * active timer returns 1.) | ||
| 981 | */ | ||
| 982 | int del_timer(struct timer_list *timer) | ||
| 983 | { | ||
| 984 | struct tvec_base *base; | ||
| 985 | unsigned long flags; | ||
| 986 | int ret = 0; | ||
| 987 | |||
| 988 | debug_assert_init(timer); | ||
| 989 | |||
| 990 | timer_stats_timer_clear_start_info(timer); | ||
| 991 | if (timer_pending(timer)) { | ||
| 992 | base = lock_timer_base(timer, &flags); | ||
| 993 | ret = detach_if_pending(timer, base, true); | ||
| 994 | spin_unlock_irqrestore(&base->lock, flags); | ||
| 995 | } | ||
| 996 | |||
| 997 | return ret; | ||
| 998 | } | ||
| 999 | EXPORT_SYMBOL(del_timer); | ||
| 1000 | |||
| 1001 | /** | ||
| 1002 | * try_to_del_timer_sync - Try to deactivate a timer | ||
| 1003 | * @timer: timer do del | ||
| 1004 | * | ||
| 1005 | * This function tries to deactivate a timer. Upon successful (ret >= 0) | ||
| 1006 | * exit the timer is not queued and the handler is not running on any CPU. | ||
| 1007 | */ | ||
| 1008 | int try_to_del_timer_sync(struct timer_list *timer) | ||
| 1009 | { | ||
| 1010 | struct tvec_base *base; | ||
| 1011 | unsigned long flags; | ||
| 1012 | int ret = -1; | ||
| 1013 | |||
| 1014 | debug_assert_init(timer); | ||
| 1015 | |||
| 1016 | base = lock_timer_base(timer, &flags); | ||
| 1017 | |||
| 1018 | if (base->running_timer != timer) { | ||
| 1019 | timer_stats_timer_clear_start_info(timer); | ||
| 1020 | ret = detach_if_pending(timer, base, true); | ||
| 1021 | } | ||
| 1022 | spin_unlock_irqrestore(&base->lock, flags); | ||
| 1023 | |||
| 1024 | return ret; | ||
| 1025 | } | ||
| 1026 | EXPORT_SYMBOL(try_to_del_timer_sync); | ||
| 1027 | |||
| 1028 | #ifdef CONFIG_SMP | ||
| 1029 | /** | ||
| 1030 | * del_timer_sync - deactivate a timer and wait for the handler to finish. | ||
| 1031 | * @timer: the timer to be deactivated | ||
| 1032 | * | ||
| 1033 | * This function only differs from del_timer() on SMP: besides deactivating | ||
| 1034 | * the timer it also makes sure the handler has finished executing on other | ||
| 1035 | * CPUs. | ||
| 1036 | * | ||
| 1037 | * Synchronization rules: Callers must prevent restarting of the timer, | ||
| 1038 | * otherwise this function is meaningless. It must not be called from | ||
| 1039 | * interrupt contexts unless the timer is an irqsafe one. The caller must | ||
| 1040 | * not hold locks which would prevent completion of the timer's | ||
| 1041 | * handler. The timer's handler must not call add_timer_on(). Upon exit the | ||
| 1042 | * timer is not queued and the handler is not running on any CPU. | ||
| 1043 | * | ||
| 1044 | * Note: For !irqsafe timers, you must not hold locks that are held in | ||
| 1045 | * interrupt context while calling this function. Even if the lock has | ||
| 1046 | * nothing to do with the timer in question. Here's why: | ||
| 1047 | * | ||
| 1048 | * CPU0 CPU1 | ||
| 1049 | * ---- ---- | ||
| 1050 | * <SOFTIRQ> | ||
| 1051 | * call_timer_fn(); | ||
| 1052 | * base->running_timer = mytimer; | ||
| 1053 | * spin_lock_irq(somelock); | ||
| 1054 | * <IRQ> | ||
| 1055 | * spin_lock(somelock); | ||
| 1056 | * del_timer_sync(mytimer); | ||
| 1057 | * while (base->running_timer == mytimer); | ||
| 1058 | * | ||
| 1059 | * Now del_timer_sync() will never return and never release somelock. | ||
| 1060 | * The interrupt on the other CPU is waiting to grab somelock but | ||
| 1061 | * it has interrupted the softirq that CPU0 is waiting to finish. | ||
| 1062 | * | ||
| 1063 | * The function returns whether it has deactivated a pending timer or not. | ||
| 1064 | */ | ||
| 1065 | int del_timer_sync(struct timer_list *timer) | ||
| 1066 | { | ||
| 1067 | #ifdef CONFIG_LOCKDEP | ||
| 1068 | unsigned long flags; | ||
| 1069 | |||
| 1070 | /* | ||
| 1071 | * If lockdep gives a backtrace here, please reference | ||
| 1072 | * the synchronization rules above. | ||
| 1073 | */ | ||
| 1074 | local_irq_save(flags); | ||
| 1075 | lock_map_acquire(&timer->lockdep_map); | ||
| 1076 | lock_map_release(&timer->lockdep_map); | ||
| 1077 | local_irq_restore(flags); | ||
| 1078 | #endif | ||
| 1079 | /* | ||
| 1080 | * don't use it in hardirq context, because it | ||
| 1081 | * could lead to deadlock. | ||
| 1082 | */ | ||
| 1083 | WARN_ON(in_irq() && !tbase_get_irqsafe(timer->base)); | ||
| 1084 | for (;;) { | ||
| 1085 | int ret = try_to_del_timer_sync(timer); | ||
| 1086 | if (ret >= 0) | ||
| 1087 | return ret; | ||
| 1088 | cpu_relax(); | ||
| 1089 | } | ||
| 1090 | } | ||
| 1091 | EXPORT_SYMBOL(del_timer_sync); | ||
| 1092 | #endif | ||
| 1093 | |||
| 1094 | static int cascade(struct tvec_base *base, struct tvec *tv, int index) | ||
| 1095 | { | ||
| 1096 | /* cascade all the timers from tv up one level */ | ||
| 1097 | struct timer_list *timer, *tmp; | ||
| 1098 | struct list_head tv_list; | ||
| 1099 | |||
| 1100 | list_replace_init(tv->vec + index, &tv_list); | ||
| 1101 | |||
| 1102 | /* | ||
| 1103 | * We are removing _all_ timers from the list, so we | ||
| 1104 | * don't have to detach them individually. | ||
| 1105 | */ | ||
| 1106 | list_for_each_entry_safe(timer, tmp, &tv_list, entry) { | ||
| 1107 | BUG_ON(tbase_get_base(timer->base) != base); | ||
| 1108 | /* No accounting, while moving them */ | ||
| 1109 | __internal_add_timer(base, timer); | ||
| 1110 | } | ||
| 1111 | |||
| 1112 | return index; | ||
| 1113 | } | ||
| 1114 | |||
| 1115 | static void call_timer_fn(struct timer_list *timer, void (*fn)(unsigned long), | ||
| 1116 | unsigned long data) | ||
| 1117 | { | ||
| 1118 | int count = preempt_count(); | ||
| 1119 | |||
| 1120 | #ifdef CONFIG_LOCKDEP | ||
| 1121 | /* | ||
| 1122 | * It is permissible to free the timer from inside the | ||
| 1123 | * function that is called from it, this we need to take into | ||
| 1124 | * account for lockdep too. To avoid bogus "held lock freed" | ||
| 1125 | * warnings as well as problems when looking into | ||
| 1126 | * timer->lockdep_map, make a copy and use that here. | ||
| 1127 | */ | ||
| 1128 | struct lockdep_map lockdep_map; | ||
| 1129 | |||
| 1130 | lockdep_copy_map(&lockdep_map, &timer->lockdep_map); | ||
| 1131 | #endif | ||
| 1132 | /* | ||
| 1133 | * Couple the lock chain with the lock chain at | ||
| 1134 | * del_timer_sync() by acquiring the lock_map around the fn() | ||
| 1135 | * call here and in del_timer_sync(). | ||
| 1136 | */ | ||
| 1137 | lock_map_acquire(&lockdep_map); | ||
| 1138 | |||
| 1139 | trace_timer_expire_entry(timer); | ||
| 1140 | fn(data); | ||
| 1141 | trace_timer_expire_exit(timer); | ||
| 1142 | |||
| 1143 | lock_map_release(&lockdep_map); | ||
| 1144 | |||
| 1145 | if (count != preempt_count()) { | ||
| 1146 | WARN_ONCE(1, "timer: %pF preempt leak: %08x -> %08x\n", | ||
| 1147 | fn, count, preempt_count()); | ||
| 1148 | /* | ||
| 1149 | * Restore the preempt count. That gives us a decent | ||
| 1150 | * chance to survive and extract information. If the | ||
| 1151 | * callback kept a lock held, bad luck, but not worse | ||
| 1152 | * than the BUG() we had. | ||
| 1153 | */ | ||
| 1154 | preempt_count_set(count); | ||
| 1155 | } | ||
| 1156 | } | ||
| 1157 | |||
| 1158 | #define INDEX(N) ((base->timer_jiffies >> (TVR_BITS + (N) * TVN_BITS)) & TVN_MASK) | ||
| 1159 | |||
| 1160 | /** | ||
| 1161 | * __run_timers - run all expired timers (if any) on this CPU. | ||
| 1162 | * @base: the timer vector to be processed. | ||
| 1163 | * | ||
| 1164 | * This function cascades all vectors and executes all expired timer | ||
| 1165 | * vectors. | ||
| 1166 | */ | ||
| 1167 | static inline void __run_timers(struct tvec_base *base) | ||
| 1168 | { | ||
| 1169 | struct timer_list *timer; | ||
| 1170 | |||
| 1171 | spin_lock_irq(&base->lock); | ||
| 1172 | if (catchup_timer_jiffies(base)) { | ||
| 1173 | spin_unlock_irq(&base->lock); | ||
| 1174 | return; | ||
| 1175 | } | ||
| 1176 | while (time_after_eq(jiffies, base->timer_jiffies)) { | ||
| 1177 | struct list_head work_list; | ||
| 1178 | struct list_head *head = &work_list; | ||
| 1179 | int index = base->timer_jiffies & TVR_MASK; | ||
| 1180 | |||
| 1181 | /* | ||
| 1182 | * Cascade timers: | ||
| 1183 | */ | ||
| 1184 | if (!index && | ||
| 1185 | (!cascade(base, &base->tv2, INDEX(0))) && | ||
| 1186 | (!cascade(base, &base->tv3, INDEX(1))) && | ||
| 1187 | !cascade(base, &base->tv4, INDEX(2))) | ||
| 1188 | cascade(base, &base->tv5, INDEX(3)); | ||
| 1189 | ++base->timer_jiffies; | ||
| 1190 | list_replace_init(base->tv1.vec + index, head); | ||
| 1191 | while (!list_empty(head)) { | ||
| 1192 | void (*fn)(unsigned long); | ||
| 1193 | unsigned long data; | ||
| 1194 | bool irqsafe; | ||
| 1195 | |||
| 1196 | timer = list_first_entry(head, struct timer_list,entry); | ||
| 1197 | fn = timer->function; | ||
| 1198 | data = timer->data; | ||
| 1199 | irqsafe = tbase_get_irqsafe(timer->base); | ||
| 1200 | |||
| 1201 | timer_stats_account_timer(timer); | ||
| 1202 | |||
| 1203 | base->running_timer = timer; | ||
| 1204 | detach_expired_timer(timer, base); | ||
| 1205 | |||
| 1206 | if (irqsafe) { | ||
| 1207 | spin_unlock(&base->lock); | ||
| 1208 | call_timer_fn(timer, fn, data); | ||
| 1209 | spin_lock(&base->lock); | ||
| 1210 | } else { | ||
| 1211 | spin_unlock_irq(&base->lock); | ||
| 1212 | call_timer_fn(timer, fn, data); | ||
| 1213 | spin_lock_irq(&base->lock); | ||
| 1214 | } | ||
| 1215 | } | ||
| 1216 | } | ||
| 1217 | base->running_timer = NULL; | ||
| 1218 | spin_unlock_irq(&base->lock); | ||
| 1219 | } | ||
| 1220 | |||
| 1221 | #ifdef CONFIG_NO_HZ_COMMON | ||
| 1222 | /* | ||
| 1223 | * Find out when the next timer event is due to happen. This | ||
| 1224 | * is used on S/390 to stop all activity when a CPU is idle. | ||
| 1225 | * This function needs to be called with interrupts disabled. | ||
| 1226 | */ | ||
| 1227 | static unsigned long __next_timer_interrupt(struct tvec_base *base) | ||
| 1228 | { | ||
| 1229 | unsigned long timer_jiffies = base->timer_jiffies; | ||
| 1230 | unsigned long expires = timer_jiffies + NEXT_TIMER_MAX_DELTA; | ||
| 1231 | int index, slot, array, found = 0; | ||
| 1232 | struct timer_list *nte; | ||
| 1233 | struct tvec *varray[4]; | ||
| 1234 | |||
| 1235 | /* Look for timer events in tv1. */ | ||
| 1236 | index = slot = timer_jiffies & TVR_MASK; | ||
| 1237 | do { | ||
| 1238 | list_for_each_entry(nte, base->tv1.vec + slot, entry) { | ||
| 1239 | if (tbase_get_deferrable(nte->base)) | ||
| 1240 | continue; | ||
| 1241 | |||
| 1242 | found = 1; | ||
| 1243 | expires = nte->expires; | ||
| 1244 | /* Look at the cascade bucket(s)? */ | ||
| 1245 | if (!index || slot < index) | ||
| 1246 | goto cascade; | ||
| 1247 | return expires; | ||
| 1248 | } | ||
| 1249 | slot = (slot + 1) & TVR_MASK; | ||
| 1250 | } while (slot != index); | ||
| 1251 | |||
| 1252 | cascade: | ||
| 1253 | /* Calculate the next cascade event */ | ||
| 1254 | if (index) | ||
| 1255 | timer_jiffies += TVR_SIZE - index; | ||
| 1256 | timer_jiffies >>= TVR_BITS; | ||
| 1257 | |||
| 1258 | /* Check tv2-tv5. */ | ||
| 1259 | varray[0] = &base->tv2; | ||
| 1260 | varray[1] = &base->tv3; | ||
| 1261 | varray[2] = &base->tv4; | ||
| 1262 | varray[3] = &base->tv5; | ||
| 1263 | |||
| 1264 | for (array = 0; array < 4; array++) { | ||
| 1265 | struct tvec *varp = varray[array]; | ||
| 1266 | |||
| 1267 | index = slot = timer_jiffies & TVN_MASK; | ||
| 1268 | do { | ||
| 1269 | list_for_each_entry(nte, varp->vec + slot, entry) { | ||
| 1270 | if (tbase_get_deferrable(nte->base)) | ||
| 1271 | continue; | ||
| 1272 | |||
| 1273 | found = 1; | ||
| 1274 | if (time_before(nte->expires, expires)) | ||
| 1275 | expires = nte->expires; | ||
| 1276 | } | ||
| 1277 | /* | ||
| 1278 | * Do we still search for the first timer or are | ||
| 1279 | * we looking up the cascade buckets ? | ||
| 1280 | */ | ||
| 1281 | if (found) { | ||
| 1282 | /* Look at the cascade bucket(s)? */ | ||
| 1283 | if (!index || slot < index) | ||
| 1284 | break; | ||
| 1285 | return expires; | ||
| 1286 | } | ||
| 1287 | slot = (slot + 1) & TVN_MASK; | ||
| 1288 | } while (slot != index); | ||
| 1289 | |||
| 1290 | if (index) | ||
| 1291 | timer_jiffies += TVN_SIZE - index; | ||
| 1292 | timer_jiffies >>= TVN_BITS; | ||
| 1293 | } | ||
| 1294 | return expires; | ||
| 1295 | } | ||
| 1296 | |||
| 1297 | /* | ||
| 1298 | * Check, if the next hrtimer event is before the next timer wheel | ||
| 1299 | * event: | ||
| 1300 | */ | ||
| 1301 | static unsigned long cmp_next_hrtimer_event(unsigned long now, | ||
| 1302 | unsigned long expires) | ||
| 1303 | { | ||
| 1304 | ktime_t hr_delta = hrtimer_get_next_event(); | ||
| 1305 | struct timespec tsdelta; | ||
| 1306 | unsigned long delta; | ||
| 1307 | |||
| 1308 | if (hr_delta.tv64 == KTIME_MAX) | ||
| 1309 | return expires; | ||
| 1310 | |||
| 1311 | /* | ||
| 1312 | * Expired timer available, let it expire in the next tick | ||
| 1313 | */ | ||
| 1314 | if (hr_delta.tv64 <= 0) | ||
| 1315 | return now + 1; | ||
| 1316 | |||
| 1317 | tsdelta = ktime_to_timespec(hr_delta); | ||
| 1318 | delta = timespec_to_jiffies(&tsdelta); | ||
| 1319 | |||
| 1320 | /* | ||
| 1321 | * Limit the delta to the max value, which is checked in | ||
| 1322 | * tick_nohz_stop_sched_tick(): | ||
| 1323 | */ | ||
| 1324 | if (delta > NEXT_TIMER_MAX_DELTA) | ||
| 1325 | delta = NEXT_TIMER_MAX_DELTA; | ||
| 1326 | |||
| 1327 | /* | ||
| 1328 | * Take rounding errors in to account and make sure, that it | ||
| 1329 | * expires in the next tick. Otherwise we go into an endless | ||
| 1330 | * ping pong due to tick_nohz_stop_sched_tick() retriggering | ||
| 1331 | * the timer softirq | ||
| 1332 | */ | ||
| 1333 | if (delta < 1) | ||
| 1334 | delta = 1; | ||
| 1335 | now += delta; | ||
| 1336 | if (time_before(now, expires)) | ||
| 1337 | return now; | ||
| 1338 | return expires; | ||
| 1339 | } | ||
| 1340 | |||
| 1341 | /** | ||
| 1342 | * get_next_timer_interrupt - return the jiffy of the next pending timer | ||
| 1343 | * @now: current time (in jiffies) | ||
| 1344 | */ | ||
| 1345 | unsigned long get_next_timer_interrupt(unsigned long now) | ||
| 1346 | { | ||
| 1347 | struct tvec_base *base = __this_cpu_read(tvec_bases); | ||
| 1348 | unsigned long expires = now + NEXT_TIMER_MAX_DELTA; | ||
| 1349 | |||
| 1350 | /* | ||
| 1351 | * Pretend that there is no timer pending if the cpu is offline. | ||
| 1352 | * Possible pending timers will be migrated later to an active cpu. | ||
| 1353 | */ | ||
| 1354 | if (cpu_is_offline(smp_processor_id())) | ||
| 1355 | return expires; | ||
| 1356 | |||
| 1357 | spin_lock(&base->lock); | ||
| 1358 | if (base->active_timers) { | ||
| 1359 | if (time_before_eq(base->next_timer, base->timer_jiffies)) | ||
| 1360 | base->next_timer = __next_timer_interrupt(base); | ||
| 1361 | expires = base->next_timer; | ||
| 1362 | } | ||
| 1363 | spin_unlock(&base->lock); | ||
| 1364 | |||
| 1365 | if (time_before_eq(expires, now)) | ||
| 1366 | return now; | ||
| 1367 | |||
| 1368 | return cmp_next_hrtimer_event(now, expires); | ||
| 1369 | } | ||
| 1370 | #endif | ||
| 1371 | |||
| 1372 | /* | ||
| 1373 | * Called from the timer interrupt handler to charge one tick to the current | ||
| 1374 | * process. user_tick is 1 if the tick is user time, 0 for system. | ||
| 1375 | */ | ||
| 1376 | void update_process_times(int user_tick) | ||
| 1377 | { | ||
| 1378 | struct task_struct *p = current; | ||
| 1379 | int cpu = smp_processor_id(); | ||
| 1380 | |||
| 1381 | /* Note: this timer irq context must be accounted for as well. */ | ||
| 1382 | account_process_tick(p, user_tick); | ||
| 1383 | run_local_timers(); | ||
| 1384 | rcu_check_callbacks(cpu, user_tick); | ||
| 1385 | #ifdef CONFIG_IRQ_WORK | ||
| 1386 | if (in_irq()) | ||
| 1387 | irq_work_run(); | ||
| 1388 | #endif | ||
| 1389 | scheduler_tick(); | ||
| 1390 | run_posix_cpu_timers(p); | ||
| 1391 | } | ||
| 1392 | |||
| 1393 | /* | ||
| 1394 | * This function runs timers and the timer-tq in bottom half context. | ||
| 1395 | */ | ||
| 1396 | static void run_timer_softirq(struct softirq_action *h) | ||
| 1397 | { | ||
| 1398 | struct tvec_base *base = __this_cpu_read(tvec_bases); | ||
| 1399 | |||
| 1400 | hrtimer_run_pending(); | ||
| 1401 | |||
| 1402 | if (time_after_eq(jiffies, base->timer_jiffies)) | ||
| 1403 | __run_timers(base); | ||
| 1404 | } | ||
| 1405 | |||
| 1406 | /* | ||
| 1407 | * Called by the local, per-CPU timer interrupt on SMP. | ||
| 1408 | */ | ||
| 1409 | void run_local_timers(void) | ||
| 1410 | { | ||
| 1411 | hrtimer_run_queues(); | ||
| 1412 | raise_softirq(TIMER_SOFTIRQ); | ||
| 1413 | } | ||
| 1414 | |||
| 1415 | #ifdef __ARCH_WANT_SYS_ALARM | ||
| 1416 | |||
| 1417 | /* | ||
| 1418 | * For backwards compatibility? This can be done in libc so Alpha | ||
| 1419 | * and all newer ports shouldn't need it. | ||
| 1420 | */ | ||
| 1421 | SYSCALL_DEFINE1(alarm, unsigned int, seconds) | ||
| 1422 | { | ||
| 1423 | return alarm_setitimer(seconds); | ||
| 1424 | } | ||
| 1425 | |||
| 1426 | #endif | ||
| 1427 | |||
| 1428 | static void process_timeout(unsigned long __data) | ||
| 1429 | { | ||
| 1430 | wake_up_process((struct task_struct *)__data); | ||
| 1431 | } | ||
| 1432 | |||
| 1433 | /** | ||
| 1434 | * schedule_timeout - sleep until timeout | ||
| 1435 | * @timeout: timeout value in jiffies | ||
| 1436 | * | ||
| 1437 | * Make the current task sleep until @timeout jiffies have | ||
| 1438 | * elapsed. The routine will return immediately unless | ||
| 1439 | * the current task state has been set (see set_current_state()). | ||
| 1440 | * | ||
| 1441 | * You can set the task state as follows - | ||
| 1442 | * | ||
| 1443 | * %TASK_UNINTERRUPTIBLE - at least @timeout jiffies are guaranteed to | ||
| 1444 | * pass before the routine returns. The routine will return 0 | ||
| 1445 | * | ||
| 1446 | * %TASK_INTERRUPTIBLE - the routine may return early if a signal is | ||
| 1447 | * delivered to the current task. In this case the remaining time | ||
| 1448 | * in jiffies will be returned, or 0 if the timer expired in time | ||
| 1449 | * | ||
| 1450 | * The current task state is guaranteed to be TASK_RUNNING when this | ||
| 1451 | * routine returns. | ||
| 1452 | * | ||
| 1453 | * Specifying a @timeout value of %MAX_SCHEDULE_TIMEOUT will schedule | ||
| 1454 | * the CPU away without a bound on the timeout. In this case the return | ||
| 1455 | * value will be %MAX_SCHEDULE_TIMEOUT. | ||
| 1456 | * | ||
| 1457 | * In all cases the return value is guaranteed to be non-negative. | ||
| 1458 | */ | ||
| 1459 | signed long __sched schedule_timeout(signed long timeout) | ||
| 1460 | { | ||
| 1461 | struct timer_list timer; | ||
| 1462 | unsigned long expire; | ||
| 1463 | |||
| 1464 | switch (timeout) | ||
| 1465 | { | ||
| 1466 | case MAX_SCHEDULE_TIMEOUT: | ||
| 1467 | /* | ||
| 1468 | * These two special cases are useful to be comfortable | ||
| 1469 | * in the caller. Nothing more. We could take | ||
| 1470 | * MAX_SCHEDULE_TIMEOUT from one of the negative value | ||
| 1471 | * but I' d like to return a valid offset (>=0) to allow | ||
| 1472 | * the caller to do everything it want with the retval. | ||
| 1473 | */ | ||
| 1474 | schedule(); | ||
| 1475 | goto out; | ||
| 1476 | default: | ||
| 1477 | /* | ||
| 1478 | * Another bit of PARANOID. Note that the retval will be | ||
| 1479 | * 0 since no piece of kernel is supposed to do a check | ||
| 1480 | * for a negative retval of schedule_timeout() (since it | ||
| 1481 | * should never happens anyway). You just have the printk() | ||
| 1482 | * that will tell you if something is gone wrong and where. | ||
| 1483 | */ | ||
| 1484 | if (timeout < 0) { | ||
| 1485 | printk(KERN_ERR "schedule_timeout: wrong timeout " | ||
| 1486 | "value %lx\n", timeout); | ||
| 1487 | dump_stack(); | ||
| 1488 | current->state = TASK_RUNNING; | ||
| 1489 | goto out; | ||
| 1490 | } | ||
| 1491 | } | ||
| 1492 | |||
| 1493 | expire = timeout + jiffies; | ||
| 1494 | |||
| 1495 | setup_timer_on_stack(&timer, process_timeout, (unsigned long)current); | ||
| 1496 | __mod_timer(&timer, expire, false, TIMER_NOT_PINNED); | ||
| 1497 | schedule(); | ||
| 1498 | del_singleshot_timer_sync(&timer); | ||
| 1499 | |||
| 1500 | /* Remove the timer from the object tracker */ | ||
| 1501 | destroy_timer_on_stack(&timer); | ||
| 1502 | |||
| 1503 | timeout = expire - jiffies; | ||
| 1504 | |||
| 1505 | out: | ||
| 1506 | return timeout < 0 ? 0 : timeout; | ||
| 1507 | } | ||
| 1508 | EXPORT_SYMBOL(schedule_timeout); | ||
| 1509 | |||
| 1510 | /* | ||
| 1511 | * We can use __set_current_state() here because schedule_timeout() calls | ||
| 1512 | * schedule() unconditionally. | ||
| 1513 | */ | ||
| 1514 | signed long __sched schedule_timeout_interruptible(signed long timeout) | ||
| 1515 | { | ||
| 1516 | __set_current_state(TASK_INTERRUPTIBLE); | ||
| 1517 | return schedule_timeout(timeout); | ||
| 1518 | } | ||
| 1519 | EXPORT_SYMBOL(schedule_timeout_interruptible); | ||
| 1520 | |||
| 1521 | signed long __sched schedule_timeout_killable(signed long timeout) | ||
| 1522 | { | ||
| 1523 | __set_current_state(TASK_KILLABLE); | ||
| 1524 | return schedule_timeout(timeout); | ||
| 1525 | } | ||
| 1526 | EXPORT_SYMBOL(schedule_timeout_killable); | ||
| 1527 | |||
| 1528 | signed long __sched schedule_timeout_uninterruptible(signed long timeout) | ||
| 1529 | { | ||
| 1530 | __set_current_state(TASK_UNINTERRUPTIBLE); | ||
| 1531 | return schedule_timeout(timeout); | ||
| 1532 | } | ||
| 1533 | EXPORT_SYMBOL(schedule_timeout_uninterruptible); | ||
| 1534 | |||
| 1535 | static int init_timers_cpu(int cpu) | ||
| 1536 | { | ||
| 1537 | int j; | ||
| 1538 | struct tvec_base *base; | ||
| 1539 | static char tvec_base_done[NR_CPUS]; | ||
| 1540 | |||
| 1541 | if (!tvec_base_done[cpu]) { | ||
| 1542 | static char boot_done; | ||
| 1543 | |||
| 1544 | if (boot_done) { | ||
| 1545 | /* | ||
| 1546 | * The APs use this path later in boot | ||
| 1547 | */ | ||
| 1548 | base = kzalloc_node(sizeof(*base), GFP_KERNEL, | ||
| 1549 | cpu_to_node(cpu)); | ||
| 1550 | if (!base) | ||
| 1551 | return -ENOMEM; | ||
| 1552 | |||
| 1553 | /* Make sure tvec_base has TIMER_FLAG_MASK bits free */ | ||
| 1554 | if (WARN_ON(base != tbase_get_base(base))) { | ||
| 1555 | kfree(base); | ||
| 1556 | return -ENOMEM; | ||
| 1557 | } | ||
| 1558 | per_cpu(tvec_bases, cpu) = base; | ||
| 1559 | } else { | ||
| 1560 | /* | ||
| 1561 | * This is for the boot CPU - we use compile-time | ||
| 1562 | * static initialisation because per-cpu memory isn't | ||
| 1563 | * ready yet and because the memory allocators are not | ||
| 1564 | * initialised either. | ||
| 1565 | */ | ||
| 1566 | boot_done = 1; | ||
| 1567 | base = &boot_tvec_bases; | ||
| 1568 | } | ||
| 1569 | spin_lock_init(&base->lock); | ||
| 1570 | tvec_base_done[cpu] = 1; | ||
| 1571 | } else { | ||
| 1572 | base = per_cpu(tvec_bases, cpu); | ||
| 1573 | } | ||
| 1574 | |||
| 1575 | |||
| 1576 | for (j = 0; j < TVN_SIZE; j++) { | ||
| 1577 | INIT_LIST_HEAD(base->tv5.vec + j); | ||
| 1578 | INIT_LIST_HEAD(base->tv4.vec + j); | ||
| 1579 | INIT_LIST_HEAD(base->tv3.vec + j); | ||
| 1580 | INIT_LIST_HEAD(base->tv2.vec + j); | ||
| 1581 | } | ||
| 1582 | for (j = 0; j < TVR_SIZE; j++) | ||
| 1583 | INIT_LIST_HEAD(base->tv1.vec + j); | ||
| 1584 | |||
| 1585 | base->timer_jiffies = jiffies; | ||
| 1586 | base->next_timer = base->timer_jiffies; | ||
| 1587 | base->active_timers = 0; | ||
| 1588 | base->all_timers = 0; | ||
| 1589 | return 0; | ||
| 1590 | } | ||
| 1591 | |||
| 1592 | #ifdef CONFIG_HOTPLUG_CPU | ||
| 1593 | static void migrate_timer_list(struct tvec_base *new_base, struct list_head *head) | ||
| 1594 | { | ||
| 1595 | struct timer_list *timer; | ||
| 1596 | |||
| 1597 | while (!list_empty(head)) { | ||
| 1598 | timer = list_first_entry(head, struct timer_list, entry); | ||
| 1599 | /* We ignore the accounting on the dying cpu */ | ||
| 1600 | detach_timer(timer, false); | ||
| 1601 | timer_set_base(timer, new_base); | ||
| 1602 | internal_add_timer(new_base, timer); | ||
| 1603 | } | ||
| 1604 | } | ||
| 1605 | |||
| 1606 | static void migrate_timers(int cpu) | ||
| 1607 | { | ||
| 1608 | struct tvec_base *old_base; | ||
| 1609 | struct tvec_base *new_base; | ||
| 1610 | int i; | ||
| 1611 | |||
| 1612 | BUG_ON(cpu_online(cpu)); | ||
| 1613 | old_base = per_cpu(tvec_bases, cpu); | ||
| 1614 | new_base = get_cpu_var(tvec_bases); | ||
| 1615 | /* | ||
| 1616 | * The caller is globally serialized and nobody else | ||
| 1617 | * takes two locks at once, deadlock is not possible. | ||
| 1618 | */ | ||
| 1619 | spin_lock_irq(&new_base->lock); | ||
| 1620 | spin_lock_nested(&old_base->lock, SINGLE_DEPTH_NESTING); | ||
| 1621 | |||
| 1622 | BUG_ON(old_base->running_timer); | ||
| 1623 | |||
| 1624 | for (i = 0; i < TVR_SIZE; i++) | ||
| 1625 | migrate_timer_list(new_base, old_base->tv1.vec + i); | ||
| 1626 | for (i = 0; i < TVN_SIZE; i++) { | ||
| 1627 | migrate_timer_list(new_base, old_base->tv2.vec + i); | ||
| 1628 | migrate_timer_list(new_base, old_base->tv3.vec + i); | ||
| 1629 | migrate_timer_list(new_base, old_base->tv4.vec + i); | ||
| 1630 | migrate_timer_list(new_base, old_base->tv5.vec + i); | ||
| 1631 | } | ||
| 1632 | |||
| 1633 | spin_unlock(&old_base->lock); | ||
| 1634 | spin_unlock_irq(&new_base->lock); | ||
| 1635 | put_cpu_var(tvec_bases); | ||
| 1636 | } | ||
| 1637 | #endif /* CONFIG_HOTPLUG_CPU */ | ||
| 1638 | |||
| 1639 | static int timer_cpu_notify(struct notifier_block *self, | ||
| 1640 | unsigned long action, void *hcpu) | ||
| 1641 | { | ||
| 1642 | long cpu = (long)hcpu; | ||
| 1643 | int err; | ||
| 1644 | |||
| 1645 | switch(action) { | ||
| 1646 | case CPU_UP_PREPARE: | ||
| 1647 | case CPU_UP_PREPARE_FROZEN: | ||
| 1648 | err = init_timers_cpu(cpu); | ||
| 1649 | if (err < 0) | ||
| 1650 | return notifier_from_errno(err); | ||
| 1651 | break; | ||
| 1652 | #ifdef CONFIG_HOTPLUG_CPU | ||
| 1653 | case CPU_DEAD: | ||
| 1654 | case CPU_DEAD_FROZEN: | ||
| 1655 | migrate_timers(cpu); | ||
| 1656 | break; | ||
| 1657 | #endif | ||
| 1658 | default: | ||
| 1659 | break; | ||
| 1660 | } | ||
| 1661 | return NOTIFY_OK; | ||
| 1662 | } | ||
| 1663 | |||
| 1664 | static struct notifier_block timers_nb = { | ||
| 1665 | .notifier_call = timer_cpu_notify, | ||
| 1666 | }; | ||
| 1667 | |||
| 1668 | |||
| 1669 | void __init init_timers(void) | ||
| 1670 | { | ||
| 1671 | int err; | ||
| 1672 | |||
| 1673 | /* ensure there are enough low bits for flags in timer->base pointer */ | ||
| 1674 | BUILD_BUG_ON(__alignof__(struct tvec_base) & TIMER_FLAG_MASK); | ||
| 1675 | |||
| 1676 | err = timer_cpu_notify(&timers_nb, (unsigned long)CPU_UP_PREPARE, | ||
| 1677 | (void *)(long)smp_processor_id()); | ||
| 1678 | BUG_ON(err != NOTIFY_OK); | ||
| 1679 | |||
| 1680 | init_timer_stats(); | ||
| 1681 | register_cpu_notifier(&timers_nb); | ||
| 1682 | open_softirq(TIMER_SOFTIRQ, run_timer_softirq); | ||
| 1683 | } | ||
| 1684 | |||
| 1685 | /** | ||
| 1686 | * msleep - sleep safely even with waitqueue interruptions | ||
| 1687 | * @msecs: Time in milliseconds to sleep for | ||
| 1688 | */ | ||
| 1689 | void msleep(unsigned int msecs) | ||
| 1690 | { | ||
| 1691 | unsigned long timeout = msecs_to_jiffies(msecs) + 1; | ||
| 1692 | |||
| 1693 | while (timeout) | ||
| 1694 | timeout = schedule_timeout_uninterruptible(timeout); | ||
| 1695 | } | ||
| 1696 | |||
| 1697 | EXPORT_SYMBOL(msleep); | ||
| 1698 | |||
| 1699 | /** | ||
| 1700 | * msleep_interruptible - sleep waiting for signals | ||
| 1701 | * @msecs: Time in milliseconds to sleep for | ||
| 1702 | */ | ||
| 1703 | unsigned long msleep_interruptible(unsigned int msecs) | ||
| 1704 | { | ||
| 1705 | unsigned long timeout = msecs_to_jiffies(msecs) + 1; | ||
| 1706 | |||
| 1707 | while (timeout && !signal_pending(current)) | ||
| 1708 | timeout = schedule_timeout_interruptible(timeout); | ||
| 1709 | return jiffies_to_msecs(timeout); | ||
| 1710 | } | ||
| 1711 | |||
| 1712 | EXPORT_SYMBOL(msleep_interruptible); | ||
| 1713 | |||
| 1714 | static int __sched do_usleep_range(unsigned long min, unsigned long max) | ||
| 1715 | { | ||
| 1716 | ktime_t kmin; | ||
| 1717 | unsigned long delta; | ||
| 1718 | |||
| 1719 | kmin = ktime_set(0, min * NSEC_PER_USEC); | ||
| 1720 | delta = (max - min) * NSEC_PER_USEC; | ||
| 1721 | return schedule_hrtimeout_range(&kmin, delta, HRTIMER_MODE_REL); | ||
| 1722 | } | ||
| 1723 | |||
| 1724 | /** | ||
| 1725 | * usleep_range - Drop in replacement for udelay where wakeup is flexible | ||
| 1726 | * @min: Minimum time in usecs to sleep | ||
| 1727 | * @max: Maximum time in usecs to sleep | ||
| 1728 | */ | ||
| 1729 | void usleep_range(unsigned long min, unsigned long max) | ||
| 1730 | { | ||
| 1731 | __set_current_state(TASK_UNINTERRUPTIBLE); | ||
| 1732 | do_usleep_range(min, max); | ||
| 1733 | } | ||
| 1734 | EXPORT_SYMBOL(usleep_range); | ||
