diff options
Diffstat (limited to 'kernel')
| -rw-r--r-- | kernel/debug/kdb/kdb_debugger.c | 4 | ||||
| -rw-r--r-- | kernel/debug/kdb/kdb_io.c | 11 | ||||
| -rw-r--r-- | kernel/debug/kdb/kdb_main.c | 15 | ||||
| -rw-r--r-- | kernel/events/callchain.c | 9 | ||||
| -rw-r--r-- | kernel/events/core.c | 30 | ||||
| -rw-r--r-- | kernel/events/internal.h | 3 | ||||
| -rw-r--r-- | kernel/futex.c | 17 | ||||
| -rw-r--r-- | kernel/irq/manage.c | 15 | ||||
| -rw-r--r-- | kernel/printk.c | 2 | ||||
| -rw-r--r-- | kernel/sched/core.c | 4 | ||||
| -rw-r--r-- | kernel/sched/cpupri.c | 10 | ||||
| -rw-r--r-- | kernel/sched/fair.c | 29 | ||||
| -rw-r--r-- | kernel/time/jiffies.c | 2 | ||||
| -rw-r--r-- | kernel/time/ntp.c | 2 | ||||
| -rw-r--r-- | kernel/time/timekeeping.c | 407 | ||||
| -rw-r--r-- | kernel/trace/trace_event_perf.c | 2 | ||||
| -rw-r--r-- | kernel/trace/trace_kprobe.c | 6 | ||||
| -rw-r--r-- | kernel/trace/trace_syscalls.c | 4 | ||||
| -rw-r--r-- | kernel/trace/trace_uprobe.c | 2 |
19 files changed, 321 insertions, 253 deletions
diff --git a/kernel/debug/kdb/kdb_debugger.c b/kernel/debug/kdb/kdb_debugger.c index 8b68ce78ff17..be7b33b73d30 100644 --- a/kernel/debug/kdb/kdb_debugger.c +++ b/kernel/debug/kdb/kdb_debugger.c | |||
| @@ -12,6 +12,7 @@ | |||
| 12 | #include <linux/kdb.h> | 12 | #include <linux/kdb.h> |
| 13 | #include <linux/kdebug.h> | 13 | #include <linux/kdebug.h> |
| 14 | #include <linux/export.h> | 14 | #include <linux/export.h> |
| 15 | #include <linux/hardirq.h> | ||
| 15 | #include "kdb_private.h" | 16 | #include "kdb_private.h" |
| 16 | #include "../debug_core.h" | 17 | #include "../debug_core.h" |
| 17 | 18 | ||
| @@ -52,6 +53,9 @@ int kdb_stub(struct kgdb_state *ks) | |||
| 52 | if (atomic_read(&kgdb_setting_breakpoint)) | 53 | if (atomic_read(&kgdb_setting_breakpoint)) |
| 53 | reason = KDB_REASON_KEYBOARD; | 54 | reason = KDB_REASON_KEYBOARD; |
| 54 | 55 | ||
| 56 | if (in_nmi()) | ||
| 57 | reason = KDB_REASON_NMI; | ||
| 58 | |||
| 55 | for (i = 0, bp = kdb_breakpoints; i < KDB_MAXBPT; i++, bp++) { | 59 | for (i = 0, bp = kdb_breakpoints; i < KDB_MAXBPT; i++, bp++) { |
| 56 | if ((bp->bp_enabled) && (bp->bp_addr == addr)) { | 60 | if ((bp->bp_enabled) && (bp->bp_addr == addr)) { |
| 57 | reason = KDB_REASON_BREAK; | 61 | reason = KDB_REASON_BREAK; |
diff --git a/kernel/debug/kdb/kdb_io.c b/kernel/debug/kdb/kdb_io.c index bb9520f0f6ff..0a69d2adc4f3 100644 --- a/kernel/debug/kdb/kdb_io.c +++ b/kernel/debug/kdb/kdb_io.c | |||
| @@ -715,9 +715,6 @@ kdb_printit: | |||
| 715 | /* check for having reached the LINES number of printed lines */ | 715 | /* check for having reached the LINES number of printed lines */ |
| 716 | if (kdb_nextline == linecount) { | 716 | if (kdb_nextline == linecount) { |
| 717 | char buf1[16] = ""; | 717 | char buf1[16] = ""; |
| 718 | #if defined(CONFIG_SMP) | ||
| 719 | char buf2[32]; | ||
| 720 | #endif | ||
| 721 | 718 | ||
| 722 | /* Watch out for recursion here. Any routine that calls | 719 | /* Watch out for recursion here. Any routine that calls |
| 723 | * kdb_printf will come back through here. And kdb_read | 720 | * kdb_printf will come back through here. And kdb_read |
| @@ -732,14 +729,6 @@ kdb_printit: | |||
| 732 | if (moreprompt == NULL) | 729 | if (moreprompt == NULL) |
| 733 | moreprompt = "more> "; | 730 | moreprompt = "more> "; |
| 734 | 731 | ||
| 735 | #if defined(CONFIG_SMP) | ||
| 736 | if (strchr(moreprompt, '%')) { | ||
| 737 | sprintf(buf2, moreprompt, get_cpu()); | ||
| 738 | put_cpu(); | ||
| 739 | moreprompt = buf2; | ||
| 740 | } | ||
| 741 | #endif | ||
| 742 | |||
| 743 | kdb_input_flush(); | 732 | kdb_input_flush(); |
| 744 | c = console_drivers; | 733 | c = console_drivers; |
| 745 | 734 | ||
diff --git a/kernel/debug/kdb/kdb_main.c b/kernel/debug/kdb/kdb_main.c index 1f91413edb87..31df1706b9a9 100644 --- a/kernel/debug/kdb/kdb_main.c +++ b/kernel/debug/kdb/kdb_main.c | |||
| @@ -139,11 +139,10 @@ static const int __nkdb_err = sizeof(kdbmsgs) / sizeof(kdbmsg_t); | |||
| 139 | static char *__env[] = { | 139 | static char *__env[] = { |
| 140 | #if defined(CONFIG_SMP) | 140 | #if defined(CONFIG_SMP) |
| 141 | "PROMPT=[%d]kdb> ", | 141 | "PROMPT=[%d]kdb> ", |
| 142 | "MOREPROMPT=[%d]more> ", | ||
| 143 | #else | 142 | #else |
| 144 | "PROMPT=kdb> ", | 143 | "PROMPT=kdb> ", |
| 145 | "MOREPROMPT=more> ", | ||
| 146 | #endif | 144 | #endif |
| 145 | "MOREPROMPT=more> ", | ||
| 147 | "RADIX=16", | 146 | "RADIX=16", |
| 148 | "MDCOUNT=8", /* lines of md output */ | 147 | "MDCOUNT=8", /* lines of md output */ |
| 149 | KDB_PLATFORM_ENV, | 148 | KDB_PLATFORM_ENV, |
| @@ -1236,18 +1235,6 @@ static int kdb_local(kdb_reason_t reason, int error, struct pt_regs *regs, | |||
| 1236 | *cmdbuf = '\0'; | 1235 | *cmdbuf = '\0'; |
| 1237 | *(cmd_hist[cmd_head]) = '\0'; | 1236 | *(cmd_hist[cmd_head]) = '\0'; |
| 1238 | 1237 | ||
| 1239 | if (KDB_FLAG(ONLY_DO_DUMP)) { | ||
| 1240 | /* kdb is off but a catastrophic error requires a dump. | ||
| 1241 | * Take the dump and reboot. | ||
| 1242 | * Turn on logging so the kdb output appears in the log | ||
| 1243 | * buffer in the dump. | ||
| 1244 | */ | ||
| 1245 | const char *setargs[] = { "set", "LOGGING", "1" }; | ||
| 1246 | kdb_set(2, setargs); | ||
| 1247 | kdb_reboot(0, NULL); | ||
| 1248 | /*NOTREACHED*/ | ||
| 1249 | } | ||
| 1250 | |||
| 1251 | do_full_getstr: | 1238 | do_full_getstr: |
| 1252 | #if defined(CONFIG_SMP) | 1239 | #if defined(CONFIG_SMP) |
| 1253 | snprintf(kdb_prompt_str, CMD_BUFLEN, kdbgetenv("PROMPT"), | 1240 | snprintf(kdb_prompt_str, CMD_BUFLEN, kdbgetenv("PROMPT"), |
diff --git a/kernel/events/callchain.c b/kernel/events/callchain.c index 6581a040f399..98d4597f43d6 100644 --- a/kernel/events/callchain.c +++ b/kernel/events/callchain.c | |||
| @@ -153,7 +153,8 @@ put_callchain_entry(int rctx) | |||
| 153 | put_recursion_context(__get_cpu_var(callchain_recursion), rctx); | 153 | put_recursion_context(__get_cpu_var(callchain_recursion), rctx); |
| 154 | } | 154 | } |
| 155 | 155 | ||
| 156 | struct perf_callchain_entry *perf_callchain(struct pt_regs *regs) | 156 | struct perf_callchain_entry * |
| 157 | perf_callchain(struct perf_event *event, struct pt_regs *regs) | ||
| 157 | { | 158 | { |
| 158 | int rctx; | 159 | int rctx; |
| 159 | struct perf_callchain_entry *entry; | 160 | struct perf_callchain_entry *entry; |
| @@ -178,6 +179,12 @@ struct perf_callchain_entry *perf_callchain(struct pt_regs *regs) | |||
| 178 | } | 179 | } |
| 179 | 180 | ||
| 180 | if (regs) { | 181 | if (regs) { |
| 182 | /* | ||
| 183 | * Disallow cross-task user callchains. | ||
| 184 | */ | ||
| 185 | if (event->ctx->task && event->ctx->task != current) | ||
| 186 | goto exit_put; | ||
| 187 | |||
| 181 | perf_callchain_store(entry, PERF_CONTEXT_USER); | 188 | perf_callchain_store(entry, PERF_CONTEXT_USER); |
| 182 | perf_callchain_user(entry, regs); | 189 | perf_callchain_user(entry, regs); |
| 183 | } | 190 | } |
diff --git a/kernel/events/core.c b/kernel/events/core.c index f1cf0edeb39a..b7935fcec7d9 100644 --- a/kernel/events/core.c +++ b/kernel/events/core.c | |||
| @@ -4039,7 +4039,7 @@ void perf_prepare_sample(struct perf_event_header *header, | |||
| 4039 | if (sample_type & PERF_SAMPLE_CALLCHAIN) { | 4039 | if (sample_type & PERF_SAMPLE_CALLCHAIN) { |
| 4040 | int size = 1; | 4040 | int size = 1; |
| 4041 | 4041 | ||
| 4042 | data->callchain = perf_callchain(regs); | 4042 | data->callchain = perf_callchain(event, regs); |
| 4043 | 4043 | ||
| 4044 | if (data->callchain) | 4044 | if (data->callchain) |
| 4045 | size += data->callchain->nr; | 4045 | size += data->callchain->nr; |
| @@ -5209,7 +5209,8 @@ static int perf_tp_event_match(struct perf_event *event, | |||
| 5209 | } | 5209 | } |
| 5210 | 5210 | ||
| 5211 | void perf_tp_event(u64 addr, u64 count, void *record, int entry_size, | 5211 | void perf_tp_event(u64 addr, u64 count, void *record, int entry_size, |
| 5212 | struct pt_regs *regs, struct hlist_head *head, int rctx) | 5212 | struct pt_regs *regs, struct hlist_head *head, int rctx, |
| 5213 | struct task_struct *task) | ||
| 5213 | { | 5214 | { |
| 5214 | struct perf_sample_data data; | 5215 | struct perf_sample_data data; |
| 5215 | struct perf_event *event; | 5216 | struct perf_event *event; |
| @@ -5228,6 +5229,31 @@ void perf_tp_event(u64 addr, u64 count, void *record, int entry_size, | |||
| 5228 | perf_swevent_event(event, count, &data, regs); | 5229 | perf_swevent_event(event, count, &data, regs); |
| 5229 | } | 5230 | } |
| 5230 | 5231 | ||
| 5232 | /* | ||
| 5233 | * If we got specified a target task, also iterate its context and | ||
| 5234 | * deliver this event there too. | ||
| 5235 | */ | ||
| 5236 | if (task && task != current) { | ||
| 5237 | struct perf_event_context *ctx; | ||
| 5238 | struct trace_entry *entry = record; | ||
| 5239 | |||
| 5240 | rcu_read_lock(); | ||
| 5241 | ctx = rcu_dereference(task->perf_event_ctxp[perf_sw_context]); | ||
| 5242 | if (!ctx) | ||
| 5243 | goto unlock; | ||
| 5244 | |||
| 5245 | list_for_each_entry_rcu(event, &ctx->event_list, event_entry) { | ||
| 5246 | if (event->attr.type != PERF_TYPE_TRACEPOINT) | ||
| 5247 | continue; | ||
| 5248 | if (event->attr.config != entry->type) | ||
| 5249 | continue; | ||
| 5250 | if (perf_tp_event_match(event, &data, regs)) | ||
| 5251 | perf_swevent_event(event, count, &data, regs); | ||
| 5252 | } | ||
| 5253 | unlock: | ||
| 5254 | rcu_read_unlock(); | ||
| 5255 | } | ||
| 5256 | |||
| 5231 | perf_swevent_put_recursion_context(rctx); | 5257 | perf_swevent_put_recursion_context(rctx); |
| 5232 | } | 5258 | } |
| 5233 | EXPORT_SYMBOL_GPL(perf_tp_event); | 5259 | EXPORT_SYMBOL_GPL(perf_tp_event); |
diff --git a/kernel/events/internal.h b/kernel/events/internal.h index b0b107f90afc..a096c19f2c2a 100644 --- a/kernel/events/internal.h +++ b/kernel/events/internal.h | |||
| @@ -101,7 +101,8 @@ __output_copy(struct perf_output_handle *handle, | |||
| 101 | } | 101 | } |
| 102 | 102 | ||
| 103 | /* Callchain handling */ | 103 | /* Callchain handling */ |
| 104 | extern struct perf_callchain_entry *perf_callchain(struct pt_regs *regs); | 104 | extern struct perf_callchain_entry * |
| 105 | perf_callchain(struct perf_event *event, struct pt_regs *regs); | ||
| 105 | extern int get_callchain_buffers(void); | 106 | extern int get_callchain_buffers(void); |
| 106 | extern void put_callchain_buffers(void); | 107 | extern void put_callchain_buffers(void); |
| 107 | 108 | ||
diff --git a/kernel/futex.c b/kernel/futex.c index e2b0fb9a0b3b..3717e7b306e0 100644 --- a/kernel/futex.c +++ b/kernel/futex.c | |||
| @@ -2231,11 +2231,11 @@ int handle_early_requeue_pi_wakeup(struct futex_hash_bucket *hb, | |||
| 2231 | * @uaddr2: the pi futex we will take prior to returning to user-space | 2231 | * @uaddr2: the pi futex we will take prior to returning to user-space |
| 2232 | * | 2232 | * |
| 2233 | * The caller will wait on uaddr and will be requeued by futex_requeue() to | 2233 | * The caller will wait on uaddr and will be requeued by futex_requeue() to |
| 2234 | * uaddr2 which must be PI aware. Normal wakeup will wake on uaddr2 and | 2234 | * uaddr2 which must be PI aware and unique from uaddr. Normal wakeup will wake |
| 2235 | * complete the acquisition of the rt_mutex prior to returning to userspace. | 2235 | * on uaddr2 and complete the acquisition of the rt_mutex prior to returning to |
| 2236 | * This ensures the rt_mutex maintains an owner when it has waiters; without | 2236 | * userspace. This ensures the rt_mutex maintains an owner when it has waiters; |
| 2237 | * one, the pi logic wouldn't know which task to boost/deboost, if there was a | 2237 | * without one, the pi logic would not know which task to boost/deboost, if |
| 2238 | * need to. | 2238 | * there was a need to. |
| 2239 | * | 2239 | * |
| 2240 | * We call schedule in futex_wait_queue_me() when we enqueue and return there | 2240 | * We call schedule in futex_wait_queue_me() when we enqueue and return there |
| 2241 | * via the following: | 2241 | * via the following: |
| @@ -2272,6 +2272,9 @@ static int futex_wait_requeue_pi(u32 __user *uaddr, unsigned int flags, | |||
| 2272 | struct futex_q q = futex_q_init; | 2272 | struct futex_q q = futex_q_init; |
| 2273 | int res, ret; | 2273 | int res, ret; |
| 2274 | 2274 | ||
| 2275 | if (uaddr == uaddr2) | ||
| 2276 | return -EINVAL; | ||
| 2277 | |||
| 2275 | if (!bitset) | 2278 | if (!bitset) |
| 2276 | return -EINVAL; | 2279 | return -EINVAL; |
| 2277 | 2280 | ||
| @@ -2343,7 +2346,7 @@ static int futex_wait_requeue_pi(u32 __user *uaddr, unsigned int flags, | |||
| 2343 | * signal. futex_unlock_pi() will not destroy the lock_ptr nor | 2346 | * signal. futex_unlock_pi() will not destroy the lock_ptr nor |
| 2344 | * the pi_state. | 2347 | * the pi_state. |
| 2345 | */ | 2348 | */ |
| 2346 | WARN_ON(!&q.pi_state); | 2349 | WARN_ON(!q.pi_state); |
| 2347 | pi_mutex = &q.pi_state->pi_mutex; | 2350 | pi_mutex = &q.pi_state->pi_mutex; |
| 2348 | ret = rt_mutex_finish_proxy_lock(pi_mutex, to, &rt_waiter, 1); | 2351 | ret = rt_mutex_finish_proxy_lock(pi_mutex, to, &rt_waiter, 1); |
| 2349 | debug_rt_mutex_free_waiter(&rt_waiter); | 2352 | debug_rt_mutex_free_waiter(&rt_waiter); |
| @@ -2370,7 +2373,7 @@ static int futex_wait_requeue_pi(u32 __user *uaddr, unsigned int flags, | |||
| 2370 | * fault, unlock the rt_mutex and return the fault to userspace. | 2373 | * fault, unlock the rt_mutex and return the fault to userspace. |
| 2371 | */ | 2374 | */ |
| 2372 | if (ret == -EFAULT) { | 2375 | if (ret == -EFAULT) { |
| 2373 | if (rt_mutex_owner(pi_mutex) == current) | 2376 | if (pi_mutex && rt_mutex_owner(pi_mutex) == current) |
| 2374 | rt_mutex_unlock(pi_mutex); | 2377 | rt_mutex_unlock(pi_mutex); |
| 2375 | } else if (ret == -EINTR) { | 2378 | } else if (ret == -EINTR) { |
| 2376 | /* | 2379 | /* |
diff --git a/kernel/irq/manage.c b/kernel/irq/manage.c index 0a8e8f059627..4c69326aa773 100644 --- a/kernel/irq/manage.c +++ b/kernel/irq/manage.c | |||
| @@ -944,6 +944,18 @@ __setup_irq(unsigned int irq, struct irq_desc *desc, struct irqaction *new) | |||
| 944 | } | 944 | } |
| 945 | 945 | ||
| 946 | /* | 946 | /* |
| 947 | * Drivers are often written to work w/o knowledge about the | ||
| 948 | * underlying irq chip implementation, so a request for a | ||
| 949 | * threaded irq without a primary hard irq context handler | ||
| 950 | * requires the ONESHOT flag to be set. Some irq chips like | ||
| 951 | * MSI based interrupts are per se one shot safe. Check the | ||
| 952 | * chip flags, so we can avoid the unmask dance at the end of | ||
| 953 | * the threaded handler for those. | ||
| 954 | */ | ||
| 955 | if (desc->irq_data.chip->flags & IRQCHIP_ONESHOT_SAFE) | ||
| 956 | new->flags &= ~IRQF_ONESHOT; | ||
| 957 | |||
| 958 | /* | ||
| 947 | * The following block of code has to be executed atomically | 959 | * The following block of code has to be executed atomically |
| 948 | */ | 960 | */ |
| 949 | raw_spin_lock_irqsave(&desc->lock, flags); | 961 | raw_spin_lock_irqsave(&desc->lock, flags); |
| @@ -1017,7 +1029,8 @@ __setup_irq(unsigned int irq, struct irq_desc *desc, struct irqaction *new) | |||
| 1017 | */ | 1029 | */ |
| 1018 | new->thread_mask = 1 << ffz(thread_mask); | 1030 | new->thread_mask = 1 << ffz(thread_mask); |
| 1019 | 1031 | ||
| 1020 | } else if (new->handler == irq_default_primary_handler) { | 1032 | } else if (new->handler == irq_default_primary_handler && |
| 1033 | !(desc->irq_data.chip->flags & IRQCHIP_ONESHOT_SAFE)) { | ||
| 1021 | /* | 1034 | /* |
| 1022 | * The interrupt was requested with handler = NULL, so | 1035 | * The interrupt was requested with handler = NULL, so |
| 1023 | * we use the default primary handler for it. But it | 1036 | * we use the default primary handler for it. But it |
diff --git a/kernel/printk.c b/kernel/printk.c index 6a76ab9d4476..66a2ea37b576 100644 --- a/kernel/printk.c +++ b/kernel/printk.c | |||
| @@ -1034,6 +1034,7 @@ static int syslog_print_all(char __user *buf, int size, bool clear) | |||
| 1034 | struct log *msg = log_from_idx(idx); | 1034 | struct log *msg = log_from_idx(idx); |
| 1035 | 1035 | ||
| 1036 | len += msg_print_text(msg, prev, true, NULL, 0); | 1036 | len += msg_print_text(msg, prev, true, NULL, 0); |
| 1037 | prev = msg->flags; | ||
| 1037 | idx = log_next(idx); | 1038 | idx = log_next(idx); |
| 1038 | seq++; | 1039 | seq++; |
| 1039 | } | 1040 | } |
| @@ -1046,6 +1047,7 @@ static int syslog_print_all(char __user *buf, int size, bool clear) | |||
| 1046 | struct log *msg = log_from_idx(idx); | 1047 | struct log *msg = log_from_idx(idx); |
| 1047 | 1048 | ||
| 1048 | len -= msg_print_text(msg, prev, true, NULL, 0); | 1049 | len -= msg_print_text(msg, prev, true, NULL, 0); |
| 1050 | prev = msg->flags; | ||
| 1049 | idx = log_next(idx); | 1051 | idx = log_next(idx); |
| 1050 | seq++; | 1052 | seq++; |
| 1051 | } | 1053 | } |
diff --git a/kernel/sched/core.c b/kernel/sched/core.c index d325c4b2dcbb..82ad284f823b 100644 --- a/kernel/sched/core.c +++ b/kernel/sched/core.c | |||
| @@ -4340,9 +4340,7 @@ recheck: | |||
| 4340 | */ | 4340 | */ |
| 4341 | if (unlikely(policy == p->policy && (!rt_policy(policy) || | 4341 | if (unlikely(policy == p->policy && (!rt_policy(policy) || |
| 4342 | param->sched_priority == p->rt_priority))) { | 4342 | param->sched_priority == p->rt_priority))) { |
| 4343 | 4343 | task_rq_unlock(rq, p, &flags); | |
| 4344 | __task_rq_unlock(rq); | ||
| 4345 | raw_spin_unlock_irqrestore(&p->pi_lock, flags); | ||
| 4346 | return 0; | 4344 | return 0; |
| 4347 | } | 4345 | } |
| 4348 | 4346 | ||
diff --git a/kernel/sched/cpupri.c b/kernel/sched/cpupri.c index d72586fdf660..23aa789c53ee 100644 --- a/kernel/sched/cpupri.c +++ b/kernel/sched/cpupri.c | |||
| @@ -65,8 +65,8 @@ static int convert_prio(int prio) | |||
| 65 | int cpupri_find(struct cpupri *cp, struct task_struct *p, | 65 | int cpupri_find(struct cpupri *cp, struct task_struct *p, |
| 66 | struct cpumask *lowest_mask) | 66 | struct cpumask *lowest_mask) |
| 67 | { | 67 | { |
| 68 | int idx = 0; | 68 | int idx = 0; |
| 69 | int task_pri = convert_prio(p->prio); | 69 | int task_pri = convert_prio(p->prio); |
| 70 | 70 | ||
| 71 | if (task_pri >= MAX_RT_PRIO) | 71 | if (task_pri >= MAX_RT_PRIO) |
| 72 | return 0; | 72 | return 0; |
| @@ -137,9 +137,9 @@ int cpupri_find(struct cpupri *cp, struct task_struct *p, | |||
| 137 | */ | 137 | */ |
| 138 | void cpupri_set(struct cpupri *cp, int cpu, int newpri) | 138 | void cpupri_set(struct cpupri *cp, int cpu, int newpri) |
| 139 | { | 139 | { |
| 140 | int *currpri = &cp->cpu_to_pri[cpu]; | 140 | int *currpri = &cp->cpu_to_pri[cpu]; |
| 141 | int oldpri = *currpri; | 141 | int oldpri = *currpri; |
| 142 | int do_mb = 0; | 142 | int do_mb = 0; |
| 143 | 143 | ||
| 144 | newpri = convert_prio(newpri); | 144 | newpri = convert_prio(newpri); |
| 145 | 145 | ||
diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c index 22321db64952..d0cc03b3e70b 100644 --- a/kernel/sched/fair.c +++ b/kernel/sched/fair.c | |||
| @@ -3069,6 +3069,9 @@ struct lb_env { | |||
| 3069 | int new_dst_cpu; | 3069 | int new_dst_cpu; |
| 3070 | enum cpu_idle_type idle; | 3070 | enum cpu_idle_type idle; |
| 3071 | long imbalance; | 3071 | long imbalance; |
| 3072 | /* The set of CPUs under consideration for load-balancing */ | ||
| 3073 | struct cpumask *cpus; | ||
| 3074 | |||
| 3072 | unsigned int flags; | 3075 | unsigned int flags; |
| 3073 | 3076 | ||
| 3074 | unsigned int loop; | 3077 | unsigned int loop; |
| @@ -3653,8 +3656,7 @@ fix_small_capacity(struct sched_domain *sd, struct sched_group *group) | |||
| 3653 | */ | 3656 | */ |
| 3654 | static inline void update_sg_lb_stats(struct lb_env *env, | 3657 | static inline void update_sg_lb_stats(struct lb_env *env, |
| 3655 | struct sched_group *group, int load_idx, | 3658 | struct sched_group *group, int load_idx, |
| 3656 | int local_group, const struct cpumask *cpus, | 3659 | int local_group, int *balance, struct sg_lb_stats *sgs) |
| 3657 | int *balance, struct sg_lb_stats *sgs) | ||
| 3658 | { | 3660 | { |
| 3659 | unsigned long nr_running, max_nr_running, min_nr_running; | 3661 | unsigned long nr_running, max_nr_running, min_nr_running; |
| 3660 | unsigned long load, max_cpu_load, min_cpu_load; | 3662 | unsigned long load, max_cpu_load, min_cpu_load; |
| @@ -3671,7 +3673,7 @@ static inline void update_sg_lb_stats(struct lb_env *env, | |||
| 3671 | max_nr_running = 0; | 3673 | max_nr_running = 0; |
| 3672 | min_nr_running = ~0UL; | 3674 | min_nr_running = ~0UL; |
| 3673 | 3675 | ||
| 3674 | for_each_cpu_and(i, sched_group_cpus(group), cpus) { | 3676 | for_each_cpu_and(i, sched_group_cpus(group), env->cpus) { |
| 3675 | struct rq *rq = cpu_rq(i); | 3677 | struct rq *rq = cpu_rq(i); |
| 3676 | 3678 | ||
| 3677 | nr_running = rq->nr_running; | 3679 | nr_running = rq->nr_running; |
| @@ -3800,8 +3802,7 @@ static bool update_sd_pick_busiest(struct lb_env *env, | |||
| 3800 | * @sds: variable to hold the statistics for this sched_domain. | 3802 | * @sds: variable to hold the statistics for this sched_domain. |
| 3801 | */ | 3803 | */ |
| 3802 | static inline void update_sd_lb_stats(struct lb_env *env, | 3804 | static inline void update_sd_lb_stats(struct lb_env *env, |
| 3803 | const struct cpumask *cpus, | 3805 | int *balance, struct sd_lb_stats *sds) |
| 3804 | int *balance, struct sd_lb_stats *sds) | ||
| 3805 | { | 3806 | { |
| 3806 | struct sched_domain *child = env->sd->child; | 3807 | struct sched_domain *child = env->sd->child; |
| 3807 | struct sched_group *sg = env->sd->groups; | 3808 | struct sched_group *sg = env->sd->groups; |
| @@ -3818,8 +3819,7 @@ static inline void update_sd_lb_stats(struct lb_env *env, | |||
| 3818 | 3819 | ||
| 3819 | local_group = cpumask_test_cpu(env->dst_cpu, sched_group_cpus(sg)); | 3820 | local_group = cpumask_test_cpu(env->dst_cpu, sched_group_cpus(sg)); |
| 3820 | memset(&sgs, 0, sizeof(sgs)); | 3821 | memset(&sgs, 0, sizeof(sgs)); |
| 3821 | update_sg_lb_stats(env, sg, load_idx, local_group, | 3822 | update_sg_lb_stats(env, sg, load_idx, local_group, balance, &sgs); |
| 3822 | cpus, balance, &sgs); | ||
| 3823 | 3823 | ||
| 3824 | if (local_group && !(*balance)) | 3824 | if (local_group && !(*balance)) |
| 3825 | return; | 3825 | return; |
| @@ -4055,7 +4055,6 @@ static inline void calculate_imbalance(struct lb_env *env, struct sd_lb_stats *s | |||
| 4055 | * to restore balance. | 4055 | * to restore balance. |
| 4056 | * | 4056 | * |
| 4057 | * @env: The load balancing environment. | 4057 | * @env: The load balancing environment. |
| 4058 | * @cpus: The set of CPUs under consideration for load-balancing. | ||
| 4059 | * @balance: Pointer to a variable indicating if this_cpu | 4058 | * @balance: Pointer to a variable indicating if this_cpu |
| 4060 | * is the appropriate cpu to perform load balancing at this_level. | 4059 | * is the appropriate cpu to perform load balancing at this_level. |
| 4061 | * | 4060 | * |
| @@ -4065,7 +4064,7 @@ static inline void calculate_imbalance(struct lb_env *env, struct sd_lb_stats *s | |||
| 4065 | * put to idle by rebalancing its tasks onto our group. | 4064 | * put to idle by rebalancing its tasks onto our group. |
| 4066 | */ | 4065 | */ |
| 4067 | static struct sched_group * | 4066 | static struct sched_group * |
| 4068 | find_busiest_group(struct lb_env *env, const struct cpumask *cpus, int *balance) | 4067 | find_busiest_group(struct lb_env *env, int *balance) |
| 4069 | { | 4068 | { |
| 4070 | struct sd_lb_stats sds; | 4069 | struct sd_lb_stats sds; |
| 4071 | 4070 | ||
| @@ -4075,7 +4074,7 @@ find_busiest_group(struct lb_env *env, const struct cpumask *cpus, int *balance) | |||
| 4075 | * Compute the various statistics relavent for load balancing at | 4074 | * Compute the various statistics relavent for load balancing at |
| 4076 | * this level. | 4075 | * this level. |
| 4077 | */ | 4076 | */ |
| 4078 | update_sd_lb_stats(env, cpus, balance, &sds); | 4077 | update_sd_lb_stats(env, balance, &sds); |
| 4079 | 4078 | ||
| 4080 | /* | 4079 | /* |
| 4081 | * this_cpu is not the appropriate cpu to perform load balancing at | 4080 | * this_cpu is not the appropriate cpu to perform load balancing at |
| @@ -4155,8 +4154,7 @@ ret: | |||
| 4155 | * find_busiest_queue - find the busiest runqueue among the cpus in group. | 4154 | * find_busiest_queue - find the busiest runqueue among the cpus in group. |
| 4156 | */ | 4155 | */ |
| 4157 | static struct rq *find_busiest_queue(struct lb_env *env, | 4156 | static struct rq *find_busiest_queue(struct lb_env *env, |
| 4158 | struct sched_group *group, | 4157 | struct sched_group *group) |
| 4159 | const struct cpumask *cpus) | ||
| 4160 | { | 4158 | { |
| 4161 | struct rq *busiest = NULL, *rq; | 4159 | struct rq *busiest = NULL, *rq; |
| 4162 | unsigned long max_load = 0; | 4160 | unsigned long max_load = 0; |
| @@ -4171,7 +4169,7 @@ static struct rq *find_busiest_queue(struct lb_env *env, | |||
| 4171 | if (!capacity) | 4169 | if (!capacity) |
| 4172 | capacity = fix_small_capacity(env->sd, group); | 4170 | capacity = fix_small_capacity(env->sd, group); |
| 4173 | 4171 | ||
| 4174 | if (!cpumask_test_cpu(i, cpus)) | 4172 | if (!cpumask_test_cpu(i, env->cpus)) |
| 4175 | continue; | 4173 | continue; |
| 4176 | 4174 | ||
| 4177 | rq = cpu_rq(i); | 4175 | rq = cpu_rq(i); |
| @@ -4252,6 +4250,7 @@ static int load_balance(int this_cpu, struct rq *this_rq, | |||
| 4252 | .dst_grpmask = sched_group_cpus(sd->groups), | 4250 | .dst_grpmask = sched_group_cpus(sd->groups), |
| 4253 | .idle = idle, | 4251 | .idle = idle, |
| 4254 | .loop_break = sched_nr_migrate_break, | 4252 | .loop_break = sched_nr_migrate_break, |
| 4253 | .cpus = cpus, | ||
| 4255 | }; | 4254 | }; |
| 4256 | 4255 | ||
| 4257 | cpumask_copy(cpus, cpu_active_mask); | 4256 | cpumask_copy(cpus, cpu_active_mask); |
| @@ -4260,7 +4259,7 @@ static int load_balance(int this_cpu, struct rq *this_rq, | |||
| 4260 | schedstat_inc(sd, lb_count[idle]); | 4259 | schedstat_inc(sd, lb_count[idle]); |
| 4261 | 4260 | ||
| 4262 | redo: | 4261 | redo: |
| 4263 | group = find_busiest_group(&env, cpus, balance); | 4262 | group = find_busiest_group(&env, balance); |
| 4264 | 4263 | ||
| 4265 | if (*balance == 0) | 4264 | if (*balance == 0) |
| 4266 | goto out_balanced; | 4265 | goto out_balanced; |
| @@ -4270,7 +4269,7 @@ redo: | |||
| 4270 | goto out_balanced; | 4269 | goto out_balanced; |
| 4271 | } | 4270 | } |
| 4272 | 4271 | ||
| 4273 | busiest = find_busiest_queue(&env, group, cpus); | 4272 | busiest = find_busiest_queue(&env, group); |
| 4274 | if (!busiest) { | 4273 | if (!busiest) { |
| 4275 | schedstat_inc(sd, lb_nobusyq[idle]); | 4274 | schedstat_inc(sd, lb_nobusyq[idle]); |
| 4276 | goto out_balanced; | 4275 | goto out_balanced; |
diff --git a/kernel/time/jiffies.c b/kernel/time/jiffies.c index a470154e0408..46da0537c10b 100644 --- a/kernel/time/jiffies.c +++ b/kernel/time/jiffies.c | |||
| @@ -37,7 +37,7 @@ | |||
| 37 | * requested HZ value. It is also not recommended | 37 | * requested HZ value. It is also not recommended |
| 38 | * for "tick-less" systems. | 38 | * for "tick-less" systems. |
| 39 | */ | 39 | */ |
| 40 | #define NSEC_PER_JIFFY ((u32)((((u64)NSEC_PER_SEC)<<8)/ACTHZ)) | 40 | #define NSEC_PER_JIFFY ((u32)((((u64)NSEC_PER_SEC)<<8)/SHIFTED_HZ)) |
| 41 | 41 | ||
| 42 | /* Since jiffies uses a simple NSEC_PER_JIFFY multiplier | 42 | /* Since jiffies uses a simple NSEC_PER_JIFFY multiplier |
| 43 | * conversion, the .shift value could be zero. However | 43 | * conversion, the .shift value could be zero. However |
diff --git a/kernel/time/ntp.c b/kernel/time/ntp.c index b7fbadc5c973..24174b4d669b 100644 --- a/kernel/time/ntp.c +++ b/kernel/time/ntp.c | |||
| @@ -28,7 +28,7 @@ DEFINE_SPINLOCK(ntp_lock); | |||
| 28 | /* USER_HZ period (usecs): */ | 28 | /* USER_HZ period (usecs): */ |
| 29 | unsigned long tick_usec = TICK_USEC; | 29 | unsigned long tick_usec = TICK_USEC; |
| 30 | 30 | ||
| 31 | /* ACTHZ period (nsecs): */ | 31 | /* SHIFTED_HZ period (nsecs): */ |
| 32 | unsigned long tick_nsec; | 32 | unsigned long tick_nsec; |
| 33 | 33 | ||
| 34 | static u64 tick_length; | 34 | static u64 tick_length; |
diff --git a/kernel/time/timekeeping.c b/kernel/time/timekeeping.c index f045cc50832d..e16af197a2bc 100644 --- a/kernel/time/timekeeping.c +++ b/kernel/time/timekeeping.c | |||
| @@ -65,14 +65,14 @@ struct timekeeper { | |||
| 65 | * used instead. | 65 | * used instead. |
| 66 | */ | 66 | */ |
| 67 | struct timespec wall_to_monotonic; | 67 | struct timespec wall_to_monotonic; |
| 68 | /* time spent in suspend */ | ||
| 69 | struct timespec total_sleep_time; | ||
| 70 | /* The raw monotonic time for the CLOCK_MONOTONIC_RAW posix clock. */ | ||
| 71 | struct timespec raw_time; | ||
| 72 | /* Offset clock monotonic -> clock realtime */ | 68 | /* Offset clock monotonic -> clock realtime */ |
| 73 | ktime_t offs_real; | 69 | ktime_t offs_real; |
| 70 | /* time spent in suspend */ | ||
| 71 | struct timespec total_sleep_time; | ||
| 74 | /* Offset clock monotonic -> clock boottime */ | 72 | /* Offset clock monotonic -> clock boottime */ |
| 75 | ktime_t offs_boot; | 73 | ktime_t offs_boot; |
| 74 | /* The raw monotonic time for the CLOCK_MONOTONIC_RAW posix clock. */ | ||
| 75 | struct timespec raw_time; | ||
| 76 | /* Seqlock for all timekeeper values */ | 76 | /* Seqlock for all timekeeper values */ |
| 77 | seqlock_t lock; | 77 | seqlock_t lock; |
| 78 | }; | 78 | }; |
| @@ -108,13 +108,38 @@ static struct timespec tk_xtime(struct timekeeper *tk) | |||
| 108 | static void tk_set_xtime(struct timekeeper *tk, const struct timespec *ts) | 108 | static void tk_set_xtime(struct timekeeper *tk, const struct timespec *ts) |
| 109 | { | 109 | { |
| 110 | tk->xtime_sec = ts->tv_sec; | 110 | tk->xtime_sec = ts->tv_sec; |
| 111 | tk->xtime_nsec = ts->tv_nsec << tk->shift; | 111 | tk->xtime_nsec = (u64)ts->tv_nsec << tk->shift; |
| 112 | } | 112 | } |
| 113 | 113 | ||
| 114 | static void tk_xtime_add(struct timekeeper *tk, const struct timespec *ts) | 114 | static void tk_xtime_add(struct timekeeper *tk, const struct timespec *ts) |
| 115 | { | 115 | { |
| 116 | tk->xtime_sec += ts->tv_sec; | 116 | tk->xtime_sec += ts->tv_sec; |
| 117 | tk->xtime_nsec += ts->tv_nsec << tk->shift; | 117 | tk->xtime_nsec += (u64)ts->tv_nsec << tk->shift; |
| 118 | } | ||
| 119 | |||
| 120 | static void tk_set_wall_to_mono(struct timekeeper *tk, struct timespec wtm) | ||
| 121 | { | ||
| 122 | struct timespec tmp; | ||
| 123 | |||
| 124 | /* | ||
| 125 | * Verify consistency of: offset_real = -wall_to_monotonic | ||
| 126 | * before modifying anything | ||
| 127 | */ | ||
| 128 | set_normalized_timespec(&tmp, -tk->wall_to_monotonic.tv_sec, | ||
| 129 | -tk->wall_to_monotonic.tv_nsec); | ||
| 130 | WARN_ON_ONCE(tk->offs_real.tv64 != timespec_to_ktime(tmp).tv64); | ||
| 131 | tk->wall_to_monotonic = wtm; | ||
| 132 | set_normalized_timespec(&tmp, -wtm.tv_sec, -wtm.tv_nsec); | ||
| 133 | tk->offs_real = timespec_to_ktime(tmp); | ||
| 134 | } | ||
| 135 | |||
| 136 | static void tk_set_sleep_time(struct timekeeper *tk, struct timespec t) | ||
| 137 | { | ||
| 138 | /* Verify consistency before modifying */ | ||
| 139 | WARN_ON_ONCE(tk->offs_boot.tv64 != timespec_to_ktime(tk->total_sleep_time).tv64); | ||
| 140 | |||
| 141 | tk->total_sleep_time = t; | ||
| 142 | tk->offs_boot = timespec_to_ktime(t); | ||
| 118 | } | 143 | } |
| 119 | 144 | ||
| 120 | /** | 145 | /** |
| @@ -217,14 +242,6 @@ static inline s64 timekeeping_get_ns_raw(struct timekeeper *tk) | |||
| 217 | return nsec + arch_gettimeoffset(); | 242 | return nsec + arch_gettimeoffset(); |
| 218 | } | 243 | } |
| 219 | 244 | ||
| 220 | static void update_rt_offset(struct timekeeper *tk) | ||
| 221 | { | ||
| 222 | struct timespec tmp, *wtm = &tk->wall_to_monotonic; | ||
| 223 | |||
| 224 | set_normalized_timespec(&tmp, -wtm->tv_sec, -wtm->tv_nsec); | ||
| 225 | tk->offs_real = timespec_to_ktime(tmp); | ||
| 226 | } | ||
| 227 | |||
| 228 | /* must hold write on timekeeper.lock */ | 245 | /* must hold write on timekeeper.lock */ |
| 229 | static void timekeeping_update(struct timekeeper *tk, bool clearntp) | 246 | static void timekeeping_update(struct timekeeper *tk, bool clearntp) |
| 230 | { | 247 | { |
| @@ -234,12 +251,10 @@ static void timekeeping_update(struct timekeeper *tk, bool clearntp) | |||
| 234 | tk->ntp_error = 0; | 251 | tk->ntp_error = 0; |
| 235 | ntp_clear(); | 252 | ntp_clear(); |
| 236 | } | 253 | } |
| 237 | update_rt_offset(tk); | ||
| 238 | xt = tk_xtime(tk); | 254 | xt = tk_xtime(tk); |
| 239 | update_vsyscall(&xt, &tk->wall_to_monotonic, tk->clock, tk->mult); | 255 | update_vsyscall(&xt, &tk->wall_to_monotonic, tk->clock, tk->mult); |
| 240 | } | 256 | } |
| 241 | 257 | ||
| 242 | |||
| 243 | /** | 258 | /** |
| 244 | * timekeeping_forward_now - update clock to the current time | 259 | * timekeeping_forward_now - update clock to the current time |
| 245 | * | 260 | * |
| @@ -277,18 +292,19 @@ static void timekeeping_forward_now(struct timekeeper *tk) | |||
| 277 | */ | 292 | */ |
| 278 | void getnstimeofday(struct timespec *ts) | 293 | void getnstimeofday(struct timespec *ts) |
| 279 | { | 294 | { |
| 295 | struct timekeeper *tk = &timekeeper; | ||
| 280 | unsigned long seq; | 296 | unsigned long seq; |
| 281 | s64 nsecs = 0; | 297 | s64 nsecs = 0; |
| 282 | 298 | ||
| 283 | WARN_ON(timekeeping_suspended); | 299 | WARN_ON(timekeeping_suspended); |
| 284 | 300 | ||
| 285 | do { | 301 | do { |
| 286 | seq = read_seqbegin(&timekeeper.lock); | 302 | seq = read_seqbegin(&tk->lock); |
| 287 | 303 | ||
| 288 | ts->tv_sec = timekeeper.xtime_sec; | 304 | ts->tv_sec = tk->xtime_sec; |
| 289 | ts->tv_nsec = timekeeping_get_ns(&timekeeper); | 305 | ts->tv_nsec = timekeeping_get_ns(tk); |
| 290 | 306 | ||
| 291 | } while (read_seqretry(&timekeeper.lock, seq)); | 307 | } while (read_seqretry(&tk->lock, seq)); |
| 292 | 308 | ||
| 293 | timespec_add_ns(ts, nsecs); | 309 | timespec_add_ns(ts, nsecs); |
| 294 | } | 310 | } |
| @@ -296,19 +312,18 @@ EXPORT_SYMBOL(getnstimeofday); | |||
| 296 | 312 | ||
| 297 | ktime_t ktime_get(void) | 313 | ktime_t ktime_get(void) |
| 298 | { | 314 | { |
| 315 | struct timekeeper *tk = &timekeeper; | ||
| 299 | unsigned int seq; | 316 | unsigned int seq; |
| 300 | s64 secs, nsecs; | 317 | s64 secs, nsecs; |
| 301 | 318 | ||
| 302 | WARN_ON(timekeeping_suspended); | 319 | WARN_ON(timekeeping_suspended); |
| 303 | 320 | ||
| 304 | do { | 321 | do { |
| 305 | seq = read_seqbegin(&timekeeper.lock); | 322 | seq = read_seqbegin(&tk->lock); |
| 306 | secs = timekeeper.xtime_sec + | 323 | secs = tk->xtime_sec + tk->wall_to_monotonic.tv_sec; |
| 307 | timekeeper.wall_to_monotonic.tv_sec; | 324 | nsecs = timekeeping_get_ns(tk) + tk->wall_to_monotonic.tv_nsec; |
| 308 | nsecs = timekeeping_get_ns(&timekeeper) + | ||
| 309 | timekeeper.wall_to_monotonic.tv_nsec; | ||
| 310 | 325 | ||
| 311 | } while (read_seqretry(&timekeeper.lock, seq)); | 326 | } while (read_seqretry(&tk->lock, seq)); |
| 312 | /* | 327 | /* |
| 313 | * Use ktime_set/ktime_add_ns to create a proper ktime on | 328 | * Use ktime_set/ktime_add_ns to create a proper ktime on |
| 314 | * 32-bit architectures without CONFIG_KTIME_SCALAR. | 329 | * 32-bit architectures without CONFIG_KTIME_SCALAR. |
| @@ -327,18 +342,19 @@ EXPORT_SYMBOL_GPL(ktime_get); | |||
| 327 | */ | 342 | */ |
| 328 | void ktime_get_ts(struct timespec *ts) | 343 | void ktime_get_ts(struct timespec *ts) |
| 329 | { | 344 | { |
| 345 | struct timekeeper *tk = &timekeeper; | ||
| 330 | struct timespec tomono; | 346 | struct timespec tomono; |
| 331 | unsigned int seq; | 347 | unsigned int seq; |
| 332 | 348 | ||
| 333 | WARN_ON(timekeeping_suspended); | 349 | WARN_ON(timekeeping_suspended); |
| 334 | 350 | ||
| 335 | do { | 351 | do { |
| 336 | seq = read_seqbegin(&timekeeper.lock); | 352 | seq = read_seqbegin(&tk->lock); |
| 337 | ts->tv_sec = timekeeper.xtime_sec; | 353 | ts->tv_sec = tk->xtime_sec; |
| 338 | ts->tv_nsec = timekeeping_get_ns(&timekeeper); | 354 | ts->tv_nsec = timekeeping_get_ns(tk); |
| 339 | tomono = timekeeper.wall_to_monotonic; | 355 | tomono = tk->wall_to_monotonic; |
| 340 | 356 | ||
| 341 | } while (read_seqretry(&timekeeper.lock, seq)); | 357 | } while (read_seqretry(&tk->lock, seq)); |
| 342 | 358 | ||
| 343 | set_normalized_timespec(ts, ts->tv_sec + tomono.tv_sec, | 359 | set_normalized_timespec(ts, ts->tv_sec + tomono.tv_sec, |
| 344 | ts->tv_nsec + tomono.tv_nsec); | 360 | ts->tv_nsec + tomono.tv_nsec); |
| @@ -358,22 +374,23 @@ EXPORT_SYMBOL_GPL(ktime_get_ts); | |||
| 358 | */ | 374 | */ |
| 359 | void getnstime_raw_and_real(struct timespec *ts_raw, struct timespec *ts_real) | 375 | void getnstime_raw_and_real(struct timespec *ts_raw, struct timespec *ts_real) |
| 360 | { | 376 | { |
| 377 | struct timekeeper *tk = &timekeeper; | ||
| 361 | unsigned long seq; | 378 | unsigned long seq; |
| 362 | s64 nsecs_raw, nsecs_real; | 379 | s64 nsecs_raw, nsecs_real; |
| 363 | 380 | ||
| 364 | WARN_ON_ONCE(timekeeping_suspended); | 381 | WARN_ON_ONCE(timekeeping_suspended); |
| 365 | 382 | ||
| 366 | do { | 383 | do { |
| 367 | seq = read_seqbegin(&timekeeper.lock); | 384 | seq = read_seqbegin(&tk->lock); |
| 368 | 385 | ||
| 369 | *ts_raw = timekeeper.raw_time; | 386 | *ts_raw = tk->raw_time; |
| 370 | ts_real->tv_sec = timekeeper.xtime_sec; | 387 | ts_real->tv_sec = tk->xtime_sec; |
| 371 | ts_real->tv_nsec = 0; | 388 | ts_real->tv_nsec = 0; |
| 372 | 389 | ||
| 373 | nsecs_raw = timekeeping_get_ns_raw(&timekeeper); | 390 | nsecs_raw = timekeeping_get_ns_raw(tk); |
| 374 | nsecs_real = timekeeping_get_ns(&timekeeper); | 391 | nsecs_real = timekeeping_get_ns(tk); |
| 375 | 392 | ||
| 376 | } while (read_seqretry(&timekeeper.lock, seq)); | 393 | } while (read_seqretry(&tk->lock, seq)); |
| 377 | 394 | ||
| 378 | timespec_add_ns(ts_raw, nsecs_raw); | 395 | timespec_add_ns(ts_raw, nsecs_raw); |
| 379 | timespec_add_ns(ts_real, nsecs_real); | 396 | timespec_add_ns(ts_real, nsecs_real); |
| @@ -406,28 +423,28 @@ EXPORT_SYMBOL(do_gettimeofday); | |||
| 406 | */ | 423 | */ |
| 407 | int do_settimeofday(const struct timespec *tv) | 424 | int do_settimeofday(const struct timespec *tv) |
| 408 | { | 425 | { |
| 426 | struct timekeeper *tk = &timekeeper; | ||
| 409 | struct timespec ts_delta, xt; | 427 | struct timespec ts_delta, xt; |
| 410 | unsigned long flags; | 428 | unsigned long flags; |
| 411 | 429 | ||
| 412 | if ((unsigned long)tv->tv_nsec >= NSEC_PER_SEC) | 430 | if ((unsigned long)tv->tv_nsec >= NSEC_PER_SEC) |
| 413 | return -EINVAL; | 431 | return -EINVAL; |
| 414 | 432 | ||
| 415 | write_seqlock_irqsave(&timekeeper.lock, flags); | 433 | write_seqlock_irqsave(&tk->lock, flags); |
| 416 | 434 | ||
| 417 | timekeeping_forward_now(&timekeeper); | 435 | timekeeping_forward_now(tk); |
| 418 | 436 | ||
| 419 | xt = tk_xtime(&timekeeper); | 437 | xt = tk_xtime(tk); |
| 420 | ts_delta.tv_sec = tv->tv_sec - xt.tv_sec; | 438 | ts_delta.tv_sec = tv->tv_sec - xt.tv_sec; |
| 421 | ts_delta.tv_nsec = tv->tv_nsec - xt.tv_nsec; | 439 | ts_delta.tv_nsec = tv->tv_nsec - xt.tv_nsec; |
| 422 | 440 | ||
| 423 | timekeeper.wall_to_monotonic = | 441 | tk_set_wall_to_mono(tk, timespec_sub(tk->wall_to_monotonic, ts_delta)); |
| 424 | timespec_sub(timekeeper.wall_to_monotonic, ts_delta); | ||
| 425 | 442 | ||
| 426 | tk_set_xtime(&timekeeper, tv); | 443 | tk_set_xtime(tk, tv); |
| 427 | 444 | ||
| 428 | timekeeping_update(&timekeeper, true); | 445 | timekeeping_update(tk, true); |
| 429 | 446 | ||
| 430 | write_sequnlock_irqrestore(&timekeeper.lock, flags); | 447 | write_sequnlock_irqrestore(&tk->lock, flags); |
| 431 | 448 | ||
| 432 | /* signal hrtimers about time change */ | 449 | /* signal hrtimers about time change */ |
| 433 | clock_was_set(); | 450 | clock_was_set(); |
| @@ -436,7 +453,6 @@ int do_settimeofday(const struct timespec *tv) | |||
| 436 | } | 453 | } |
| 437 | EXPORT_SYMBOL(do_settimeofday); | 454 | EXPORT_SYMBOL(do_settimeofday); |
| 438 | 455 | ||
| 439 | |||
| 440 | /** | 456 | /** |
| 441 | * timekeeping_inject_offset - Adds or subtracts from the current time. | 457 | * timekeeping_inject_offset - Adds or subtracts from the current time. |
| 442 | * @tv: pointer to the timespec variable containing the offset | 458 | * @tv: pointer to the timespec variable containing the offset |
| @@ -445,23 +461,23 @@ EXPORT_SYMBOL(do_settimeofday); | |||
| 445 | */ | 461 | */ |
| 446 | int timekeeping_inject_offset(struct timespec *ts) | 462 | int timekeeping_inject_offset(struct timespec *ts) |
| 447 | { | 463 | { |
| 464 | struct timekeeper *tk = &timekeeper; | ||
| 448 | unsigned long flags; | 465 | unsigned long flags; |
| 449 | 466 | ||
| 450 | if ((unsigned long)ts->tv_nsec >= NSEC_PER_SEC) | 467 | if ((unsigned long)ts->tv_nsec >= NSEC_PER_SEC) |
| 451 | return -EINVAL; | 468 | return -EINVAL; |
| 452 | 469 | ||
| 453 | write_seqlock_irqsave(&timekeeper.lock, flags); | 470 | write_seqlock_irqsave(&tk->lock, flags); |
| 454 | 471 | ||
| 455 | timekeeping_forward_now(&timekeeper); | 472 | timekeeping_forward_now(tk); |
| 456 | 473 | ||
| 457 | 474 | ||
| 458 | tk_xtime_add(&timekeeper, ts); | 475 | tk_xtime_add(tk, ts); |
| 459 | timekeeper.wall_to_monotonic = | 476 | tk_set_wall_to_mono(tk, timespec_sub(tk->wall_to_monotonic, *ts)); |
| 460 | timespec_sub(timekeeper.wall_to_monotonic, *ts); | ||
| 461 | 477 | ||
| 462 | timekeeping_update(&timekeeper, true); | 478 | timekeeping_update(tk, true); |
| 463 | 479 | ||
| 464 | write_sequnlock_irqrestore(&timekeeper.lock, flags); | 480 | write_sequnlock_irqrestore(&tk->lock, flags); |
| 465 | 481 | ||
| 466 | /* signal hrtimers about time change */ | 482 | /* signal hrtimers about time change */ |
| 467 | clock_was_set(); | 483 | clock_was_set(); |
| @@ -477,23 +493,24 @@ EXPORT_SYMBOL(timekeeping_inject_offset); | |||
| 477 | */ | 493 | */ |
| 478 | static int change_clocksource(void *data) | 494 | static int change_clocksource(void *data) |
| 479 | { | 495 | { |
| 496 | struct timekeeper *tk = &timekeeper; | ||
| 480 | struct clocksource *new, *old; | 497 | struct clocksource *new, *old; |
| 481 | unsigned long flags; | 498 | unsigned long flags; |
| 482 | 499 | ||
| 483 | new = (struct clocksource *) data; | 500 | new = (struct clocksource *) data; |
| 484 | 501 | ||
| 485 | write_seqlock_irqsave(&timekeeper.lock, flags); | 502 | write_seqlock_irqsave(&tk->lock, flags); |
| 486 | 503 | ||
| 487 | timekeeping_forward_now(&timekeeper); | 504 | timekeeping_forward_now(tk); |
| 488 | if (!new->enable || new->enable(new) == 0) { | 505 | if (!new->enable || new->enable(new) == 0) { |
| 489 | old = timekeeper.clock; | 506 | old = tk->clock; |
| 490 | tk_setup_internals(&timekeeper, new); | 507 | tk_setup_internals(tk, new); |
| 491 | if (old->disable) | 508 | if (old->disable) |
| 492 | old->disable(old); | 509 | old->disable(old); |
| 493 | } | 510 | } |
| 494 | timekeeping_update(&timekeeper, true); | 511 | timekeeping_update(tk, true); |
| 495 | 512 | ||
| 496 | write_sequnlock_irqrestore(&timekeeper.lock, flags); | 513 | write_sequnlock_irqrestore(&tk->lock, flags); |
| 497 | 514 | ||
| 498 | return 0; | 515 | return 0; |
| 499 | } | 516 | } |
| @@ -507,7 +524,9 @@ static int change_clocksource(void *data) | |||
| 507 | */ | 524 | */ |
| 508 | void timekeeping_notify(struct clocksource *clock) | 525 | void timekeeping_notify(struct clocksource *clock) |
| 509 | { | 526 | { |
| 510 | if (timekeeper.clock == clock) | 527 | struct timekeeper *tk = &timekeeper; |
| 528 | |||
| 529 | if (tk->clock == clock) | ||
| 511 | return; | 530 | return; |
| 512 | stop_machine(change_clocksource, clock, NULL); | 531 | stop_machine(change_clocksource, clock, NULL); |
| 513 | tick_clock_notify(); | 532 | tick_clock_notify(); |
| @@ -536,35 +555,36 @@ EXPORT_SYMBOL_GPL(ktime_get_real); | |||
| 536 | */ | 555 | */ |
| 537 | void getrawmonotonic(struct timespec *ts) | 556 | void getrawmonotonic(struct timespec *ts) |
| 538 | { | 557 | { |
| 558 | struct timekeeper *tk = &timekeeper; | ||
| 539 | unsigned long seq; | 559 | unsigned long seq; |
| 540 | s64 nsecs; | 560 | s64 nsecs; |
| 541 | 561 | ||
| 542 | do { | 562 | do { |
| 543 | seq = read_seqbegin(&timekeeper.lock); | 563 | seq = read_seqbegin(&tk->lock); |
| 544 | nsecs = timekeeping_get_ns_raw(&timekeeper); | 564 | nsecs = timekeeping_get_ns_raw(tk); |
| 545 | *ts = timekeeper.raw_time; | 565 | *ts = tk->raw_time; |
| 546 | 566 | ||
| 547 | } while (read_seqretry(&timekeeper.lock, seq)); | 567 | } while (read_seqretry(&tk->lock, seq)); |
| 548 | 568 | ||
| 549 | timespec_add_ns(ts, nsecs); | 569 | timespec_add_ns(ts, nsecs); |
| 550 | } | 570 | } |
| 551 | EXPORT_SYMBOL(getrawmonotonic); | 571 | EXPORT_SYMBOL(getrawmonotonic); |
| 552 | 572 | ||
| 553 | |||
| 554 | /** | 573 | /** |
| 555 | * timekeeping_valid_for_hres - Check if timekeeping is suitable for hres | 574 | * timekeeping_valid_for_hres - Check if timekeeping is suitable for hres |
| 556 | */ | 575 | */ |
| 557 | int timekeeping_valid_for_hres(void) | 576 | int timekeeping_valid_for_hres(void) |
| 558 | { | 577 | { |
| 578 | struct timekeeper *tk = &timekeeper; | ||
| 559 | unsigned long seq; | 579 | unsigned long seq; |
| 560 | int ret; | 580 | int ret; |
| 561 | 581 | ||
| 562 | do { | 582 | do { |
| 563 | seq = read_seqbegin(&timekeeper.lock); | 583 | seq = read_seqbegin(&tk->lock); |
| 564 | 584 | ||
| 565 | ret = timekeeper.clock->flags & CLOCK_SOURCE_VALID_FOR_HRES; | 585 | ret = tk->clock->flags & CLOCK_SOURCE_VALID_FOR_HRES; |
| 566 | 586 | ||
| 567 | } while (read_seqretry(&timekeeper.lock, seq)); | 587 | } while (read_seqretry(&tk->lock, seq)); |
| 568 | 588 | ||
| 569 | return ret; | 589 | return ret; |
| 570 | } | 590 | } |
| @@ -574,15 +594,16 @@ int timekeeping_valid_for_hres(void) | |||
| 574 | */ | 594 | */ |
| 575 | u64 timekeeping_max_deferment(void) | 595 | u64 timekeeping_max_deferment(void) |
| 576 | { | 596 | { |
| 597 | struct timekeeper *tk = &timekeeper; | ||
| 577 | unsigned long seq; | 598 | unsigned long seq; |
| 578 | u64 ret; | 599 | u64 ret; |
| 579 | 600 | ||
| 580 | do { | 601 | do { |
| 581 | seq = read_seqbegin(&timekeeper.lock); | 602 | seq = read_seqbegin(&tk->lock); |
| 582 | 603 | ||
| 583 | ret = timekeeper.clock->max_idle_ns; | 604 | ret = tk->clock->max_idle_ns; |
| 584 | 605 | ||
| 585 | } while (read_seqretry(&timekeeper.lock, seq)); | 606 | } while (read_seqretry(&tk->lock, seq)); |
| 586 | 607 | ||
| 587 | return ret; | 608 | return ret; |
| 588 | } | 609 | } |
| @@ -622,46 +643,43 @@ void __attribute__((weak)) read_boot_clock(struct timespec *ts) | |||
| 622 | */ | 643 | */ |
| 623 | void __init timekeeping_init(void) | 644 | void __init timekeeping_init(void) |
| 624 | { | 645 | { |
| 646 | struct timekeeper *tk = &timekeeper; | ||
| 625 | struct clocksource *clock; | 647 | struct clocksource *clock; |
| 626 | unsigned long flags; | 648 | unsigned long flags; |
| 627 | struct timespec now, boot; | 649 | struct timespec now, boot, tmp; |
| 628 | 650 | ||
| 629 | read_persistent_clock(&now); | 651 | read_persistent_clock(&now); |
| 630 | read_boot_clock(&boot); | 652 | read_boot_clock(&boot); |
| 631 | 653 | ||
| 632 | seqlock_init(&timekeeper.lock); | 654 | seqlock_init(&tk->lock); |
| 633 | 655 | ||
| 634 | ntp_init(); | 656 | ntp_init(); |
| 635 | 657 | ||
| 636 | write_seqlock_irqsave(&timekeeper.lock, flags); | 658 | write_seqlock_irqsave(&tk->lock, flags); |
| 637 | clock = clocksource_default_clock(); | 659 | clock = clocksource_default_clock(); |
| 638 | if (clock->enable) | 660 | if (clock->enable) |
| 639 | clock->enable(clock); | 661 | clock->enable(clock); |
| 640 | tk_setup_internals(&timekeeper, clock); | 662 | tk_setup_internals(tk, clock); |
| 641 | 663 | ||
| 642 | tk_set_xtime(&timekeeper, &now); | 664 | tk_set_xtime(tk, &now); |
| 643 | timekeeper.raw_time.tv_sec = 0; | 665 | tk->raw_time.tv_sec = 0; |
| 644 | timekeeper.raw_time.tv_nsec = 0; | 666 | tk->raw_time.tv_nsec = 0; |
| 645 | if (boot.tv_sec == 0 && boot.tv_nsec == 0) | 667 | if (boot.tv_sec == 0 && boot.tv_nsec == 0) |
| 646 | boot = tk_xtime(&timekeeper); | 668 | boot = tk_xtime(tk); |
| 647 | 669 | ||
| 648 | set_normalized_timespec(&timekeeper.wall_to_monotonic, | 670 | set_normalized_timespec(&tmp, -boot.tv_sec, -boot.tv_nsec); |
| 649 | -boot.tv_sec, -boot.tv_nsec); | 671 | tk_set_wall_to_mono(tk, tmp); |
| 650 | update_rt_offset(&timekeeper); | 672 | |
| 651 | timekeeper.total_sleep_time.tv_sec = 0; | 673 | tmp.tv_sec = 0; |
| 652 | timekeeper.total_sleep_time.tv_nsec = 0; | 674 | tmp.tv_nsec = 0; |
| 653 | write_sequnlock_irqrestore(&timekeeper.lock, flags); | 675 | tk_set_sleep_time(tk, tmp); |
| 676 | |||
| 677 | write_sequnlock_irqrestore(&tk->lock, flags); | ||
| 654 | } | 678 | } |
| 655 | 679 | ||
| 656 | /* time in seconds when suspend began */ | 680 | /* time in seconds when suspend began */ |
| 657 | static struct timespec timekeeping_suspend_time; | 681 | static struct timespec timekeeping_suspend_time; |
| 658 | 682 | ||
| 659 | static void update_sleep_time(struct timespec t) | ||
| 660 | { | ||
| 661 | timekeeper.total_sleep_time = t; | ||
| 662 | timekeeper.offs_boot = timespec_to_ktime(t); | ||
| 663 | } | ||
| 664 | |||
| 665 | /** | 683 | /** |
| 666 | * __timekeeping_inject_sleeptime - Internal function to add sleep interval | 684 | * __timekeeping_inject_sleeptime - Internal function to add sleep interval |
| 667 | * @delta: pointer to a timespec delta value | 685 | * @delta: pointer to a timespec delta value |
| @@ -677,13 +695,11 @@ static void __timekeeping_inject_sleeptime(struct timekeeper *tk, | |||
| 677 | "sleep delta value!\n"); | 695 | "sleep delta value!\n"); |
| 678 | return; | 696 | return; |
| 679 | } | 697 | } |
| 680 | |||
| 681 | tk_xtime_add(tk, delta); | 698 | tk_xtime_add(tk, delta); |
| 682 | tk->wall_to_monotonic = timespec_sub(tk->wall_to_monotonic, *delta); | 699 | tk_set_wall_to_mono(tk, timespec_sub(tk->wall_to_monotonic, *delta)); |
| 683 | update_sleep_time(timespec_add(tk->total_sleep_time, *delta)); | 700 | tk_set_sleep_time(tk, timespec_add(tk->total_sleep_time, *delta)); |
| 684 | } | 701 | } |
| 685 | 702 | ||
| 686 | |||
| 687 | /** | 703 | /** |
| 688 | * timekeeping_inject_sleeptime - Adds suspend interval to timeekeeping values | 704 | * timekeeping_inject_sleeptime - Adds suspend interval to timeekeeping values |
| 689 | * @delta: pointer to a timespec delta value | 705 | * @delta: pointer to a timespec delta value |
| @@ -696,6 +712,7 @@ static void __timekeeping_inject_sleeptime(struct timekeeper *tk, | |||
| 696 | */ | 712 | */ |
| 697 | void timekeeping_inject_sleeptime(struct timespec *delta) | 713 | void timekeeping_inject_sleeptime(struct timespec *delta) |
| 698 | { | 714 | { |
| 715 | struct timekeeper *tk = &timekeeper; | ||
| 699 | unsigned long flags; | 716 | unsigned long flags; |
| 700 | struct timespec ts; | 717 | struct timespec ts; |
| 701 | 718 | ||
| @@ -704,21 +721,20 @@ void timekeeping_inject_sleeptime(struct timespec *delta) | |||
| 704 | if (!(ts.tv_sec == 0 && ts.tv_nsec == 0)) | 721 | if (!(ts.tv_sec == 0 && ts.tv_nsec == 0)) |
| 705 | return; | 722 | return; |
| 706 | 723 | ||
| 707 | write_seqlock_irqsave(&timekeeper.lock, flags); | 724 | write_seqlock_irqsave(&tk->lock, flags); |
| 708 | 725 | ||
| 709 | timekeeping_forward_now(&timekeeper); | 726 | timekeeping_forward_now(tk); |
| 710 | 727 | ||
| 711 | __timekeeping_inject_sleeptime(&timekeeper, delta); | 728 | __timekeeping_inject_sleeptime(tk, delta); |
| 712 | 729 | ||
| 713 | timekeeping_update(&timekeeper, true); | 730 | timekeeping_update(tk, true); |
| 714 | 731 | ||
| 715 | write_sequnlock_irqrestore(&timekeeper.lock, flags); | 732 | write_sequnlock_irqrestore(&tk->lock, flags); |
| 716 | 733 | ||
| 717 | /* signal hrtimers about time change */ | 734 | /* signal hrtimers about time change */ |
| 718 | clock_was_set(); | 735 | clock_was_set(); |
| 719 | } | 736 | } |
| 720 | 737 | ||
| 721 | |||
| 722 | /** | 738 | /** |
| 723 | * timekeeping_resume - Resumes the generic timekeeping subsystem. | 739 | * timekeeping_resume - Resumes the generic timekeeping subsystem. |
| 724 | * | 740 | * |
| @@ -728,6 +744,7 @@ void timekeeping_inject_sleeptime(struct timespec *delta) | |||
| 728 | */ | 744 | */ |
| 729 | static void timekeeping_resume(void) | 745 | static void timekeeping_resume(void) |
| 730 | { | 746 | { |
| 747 | struct timekeeper *tk = &timekeeper; | ||
| 731 | unsigned long flags; | 748 | unsigned long flags; |
| 732 | struct timespec ts; | 749 | struct timespec ts; |
| 733 | 750 | ||
| @@ -735,18 +752,18 @@ static void timekeeping_resume(void) | |||
| 735 | 752 | ||
| 736 | clocksource_resume(); | 753 | clocksource_resume(); |
| 737 | 754 | ||
| 738 | write_seqlock_irqsave(&timekeeper.lock, flags); | 755 | write_seqlock_irqsave(&tk->lock, flags); |
| 739 | 756 | ||
| 740 | if (timespec_compare(&ts, &timekeeping_suspend_time) > 0) { | 757 | if (timespec_compare(&ts, &timekeeping_suspend_time) > 0) { |
| 741 | ts = timespec_sub(ts, timekeeping_suspend_time); | 758 | ts = timespec_sub(ts, timekeeping_suspend_time); |
| 742 | __timekeeping_inject_sleeptime(&timekeeper, &ts); | 759 | __timekeeping_inject_sleeptime(tk, &ts); |
| 743 | } | 760 | } |
| 744 | /* re-base the last cycle value */ | 761 | /* re-base the last cycle value */ |
| 745 | timekeeper.clock->cycle_last = timekeeper.clock->read(timekeeper.clock); | 762 | tk->clock->cycle_last = tk->clock->read(tk->clock); |
| 746 | timekeeper.ntp_error = 0; | 763 | tk->ntp_error = 0; |
| 747 | timekeeping_suspended = 0; | 764 | timekeeping_suspended = 0; |
| 748 | timekeeping_update(&timekeeper, false); | 765 | timekeeping_update(tk, false); |
| 749 | write_sequnlock_irqrestore(&timekeeper.lock, flags); | 766 | write_sequnlock_irqrestore(&tk->lock, flags); |
| 750 | 767 | ||
| 751 | touch_softlockup_watchdog(); | 768 | touch_softlockup_watchdog(); |
| 752 | 769 | ||
| @@ -758,14 +775,15 @@ static void timekeeping_resume(void) | |||
| 758 | 775 | ||
| 759 | static int timekeeping_suspend(void) | 776 | static int timekeeping_suspend(void) |
| 760 | { | 777 | { |
| 778 | struct timekeeper *tk = &timekeeper; | ||
| 761 | unsigned long flags; | 779 | unsigned long flags; |
| 762 | struct timespec delta, delta_delta; | 780 | struct timespec delta, delta_delta; |
| 763 | static struct timespec old_delta; | 781 | static struct timespec old_delta; |
| 764 | 782 | ||
| 765 | read_persistent_clock(&timekeeping_suspend_time); | 783 | read_persistent_clock(&timekeeping_suspend_time); |
| 766 | 784 | ||
| 767 | write_seqlock_irqsave(&timekeeper.lock, flags); | 785 | write_seqlock_irqsave(&tk->lock, flags); |
| 768 | timekeeping_forward_now(&timekeeper); | 786 | timekeeping_forward_now(tk); |
| 769 | timekeeping_suspended = 1; | 787 | timekeeping_suspended = 1; |
| 770 | 788 | ||
| 771 | /* | 789 | /* |
| @@ -774,7 +792,7 @@ static int timekeeping_suspend(void) | |||
| 774 | * try to compensate so the difference in system time | 792 | * try to compensate so the difference in system time |
| 775 | * and persistent_clock time stays close to constant. | 793 | * and persistent_clock time stays close to constant. |
| 776 | */ | 794 | */ |
| 777 | delta = timespec_sub(tk_xtime(&timekeeper), timekeeping_suspend_time); | 795 | delta = timespec_sub(tk_xtime(tk), timekeeping_suspend_time); |
| 778 | delta_delta = timespec_sub(delta, old_delta); | 796 | delta_delta = timespec_sub(delta, old_delta); |
| 779 | if (abs(delta_delta.tv_sec) >= 2) { | 797 | if (abs(delta_delta.tv_sec) >= 2) { |
| 780 | /* | 798 | /* |
| @@ -787,7 +805,7 @@ static int timekeeping_suspend(void) | |||
| 787 | timekeeping_suspend_time = | 805 | timekeeping_suspend_time = |
| 788 | timespec_add(timekeeping_suspend_time, delta_delta); | 806 | timespec_add(timekeeping_suspend_time, delta_delta); |
| 789 | } | 807 | } |
| 790 | write_sequnlock_irqrestore(&timekeeper.lock, flags); | 808 | write_sequnlock_irqrestore(&tk->lock, flags); |
| 791 | 809 | ||
| 792 | clockevents_notify(CLOCK_EVT_NOTIFY_SUSPEND, NULL); | 810 | clockevents_notify(CLOCK_EVT_NOTIFY_SUSPEND, NULL); |
| 793 | clocksource_suspend(); | 811 | clocksource_suspend(); |
| @@ -898,27 +916,29 @@ static void timekeeping_adjust(struct timekeeper *tk, s64 offset) | |||
| 898 | * the error. This causes the likely below to be unlikely. | 916 | * the error. This causes the likely below to be unlikely. |
| 899 | * | 917 | * |
| 900 | * The proper fix is to avoid rounding up by using | 918 | * The proper fix is to avoid rounding up by using |
| 901 | * the high precision timekeeper.xtime_nsec instead of | 919 | * the high precision tk->xtime_nsec instead of |
| 902 | * xtime.tv_nsec everywhere. Fixing this will take some | 920 | * xtime.tv_nsec everywhere. Fixing this will take some |
| 903 | * time. | 921 | * time. |
| 904 | */ | 922 | */ |
| 905 | if (likely(error <= interval)) | 923 | if (likely(error <= interval)) |
| 906 | adj = 1; | 924 | adj = 1; |
| 907 | else | 925 | else |
| 908 | adj = timekeeping_bigadjust(tk, error, &interval, | 926 | adj = timekeeping_bigadjust(tk, error, &interval, &offset); |
| 909 | &offset); | 927 | } else { |
| 910 | } else if (error < -interval) { | 928 | if (error < -interval) { |
| 911 | /* See comment above, this is just switched for the negative */ | 929 | /* See comment above, this is just switched for the negative */ |
| 912 | error >>= 2; | 930 | error >>= 2; |
| 913 | if (likely(error >= -interval)) { | 931 | if (likely(error >= -interval)) { |
| 914 | adj = -1; | 932 | adj = -1; |
| 915 | interval = -interval; | 933 | interval = -interval; |
| 916 | offset = -offset; | 934 | offset = -offset; |
| 917 | } else | 935 | } else { |
| 918 | adj = timekeeping_bigadjust(tk, error, &interval, | 936 | adj = timekeeping_bigadjust(tk, error, &interval, &offset); |
| 919 | &offset); | 937 | } |
| 920 | } else | 938 | } else { |
| 921 | return; | 939 | goto out_adjust; |
| 940 | } | ||
| 941 | } | ||
| 922 | 942 | ||
| 923 | if (unlikely(tk->clock->maxadj && | 943 | if (unlikely(tk->clock->maxadj && |
| 924 | (tk->mult + adj > tk->clock->mult + tk->clock->maxadj))) { | 944 | (tk->mult + adj > tk->clock->mult + tk->clock->maxadj))) { |
| @@ -981,6 +1001,7 @@ static void timekeeping_adjust(struct timekeeper *tk, s64 offset) | |||
| 981 | tk->xtime_nsec -= offset; | 1001 | tk->xtime_nsec -= offset; |
| 982 | tk->ntp_error -= (interval - offset) << tk->ntp_error_shift; | 1002 | tk->ntp_error -= (interval - offset) << tk->ntp_error_shift; |
| 983 | 1003 | ||
| 1004 | out_adjust: | ||
| 984 | /* | 1005 | /* |
| 985 | * It may be possible that when we entered this function, xtime_nsec | 1006 | * It may be possible that when we entered this function, xtime_nsec |
| 986 | * was very small. Further, if we're slightly speeding the clocksource | 1007 | * was very small. Further, if we're slightly speeding the clocksource |
| @@ -1003,7 +1024,6 @@ static void timekeeping_adjust(struct timekeeper *tk, s64 offset) | |||
| 1003 | 1024 | ||
| 1004 | } | 1025 | } |
| 1005 | 1026 | ||
| 1006 | |||
| 1007 | /** | 1027 | /** |
| 1008 | * accumulate_nsecs_to_secs - Accumulates nsecs into secs | 1028 | * accumulate_nsecs_to_secs - Accumulates nsecs into secs |
| 1009 | * | 1029 | * |
| @@ -1024,15 +1044,21 @@ static inline void accumulate_nsecs_to_secs(struct timekeeper *tk) | |||
| 1024 | 1044 | ||
| 1025 | /* Figure out if its a leap sec and apply if needed */ | 1045 | /* Figure out if its a leap sec and apply if needed */ |
| 1026 | leap = second_overflow(tk->xtime_sec); | 1046 | leap = second_overflow(tk->xtime_sec); |
| 1027 | tk->xtime_sec += leap; | 1047 | if (unlikely(leap)) { |
| 1028 | tk->wall_to_monotonic.tv_sec -= leap; | 1048 | struct timespec ts; |
| 1029 | if (leap) | 1049 | |
| 1030 | clock_was_set_delayed(); | 1050 | tk->xtime_sec += leap; |
| 1051 | |||
| 1052 | ts.tv_sec = leap; | ||
| 1053 | ts.tv_nsec = 0; | ||
| 1054 | tk_set_wall_to_mono(tk, | ||
| 1055 | timespec_sub(tk->wall_to_monotonic, ts)); | ||
| 1031 | 1056 | ||
| 1057 | clock_was_set_delayed(); | ||
| 1058 | } | ||
| 1032 | } | 1059 | } |
| 1033 | } | 1060 | } |
| 1034 | 1061 | ||
| 1035 | |||
| 1036 | /** | 1062 | /** |
| 1037 | * logarithmic_accumulation - shifted accumulation of cycles | 1063 | * logarithmic_accumulation - shifted accumulation of cycles |
| 1038 | * | 1064 | * |
| @@ -1076,7 +1102,6 @@ static cycle_t logarithmic_accumulation(struct timekeeper *tk, cycle_t offset, | |||
| 1076 | return offset; | 1102 | return offset; |
| 1077 | } | 1103 | } |
| 1078 | 1104 | ||
| 1079 | |||
| 1080 | /** | 1105 | /** |
| 1081 | * update_wall_time - Uses the current clocksource to increment the wall time | 1106 | * update_wall_time - Uses the current clocksource to increment the wall time |
| 1082 | * | 1107 | * |
| @@ -1084,21 +1109,22 @@ static cycle_t logarithmic_accumulation(struct timekeeper *tk, cycle_t offset, | |||
| 1084 | static void update_wall_time(void) | 1109 | static void update_wall_time(void) |
| 1085 | { | 1110 | { |
| 1086 | struct clocksource *clock; | 1111 | struct clocksource *clock; |
| 1112 | struct timekeeper *tk = &timekeeper; | ||
| 1087 | cycle_t offset; | 1113 | cycle_t offset; |
| 1088 | int shift = 0, maxshift; | 1114 | int shift = 0, maxshift; |
| 1089 | unsigned long flags; | 1115 | unsigned long flags; |
| 1090 | s64 remainder; | 1116 | s64 remainder; |
| 1091 | 1117 | ||
| 1092 | write_seqlock_irqsave(&timekeeper.lock, flags); | 1118 | write_seqlock_irqsave(&tk->lock, flags); |
| 1093 | 1119 | ||
| 1094 | /* Make sure we're fully resumed: */ | 1120 | /* Make sure we're fully resumed: */ |
| 1095 | if (unlikely(timekeeping_suspended)) | 1121 | if (unlikely(timekeeping_suspended)) |
| 1096 | goto out; | 1122 | goto out; |
| 1097 | 1123 | ||
| 1098 | clock = timekeeper.clock; | 1124 | clock = tk->clock; |
| 1099 | 1125 | ||
| 1100 | #ifdef CONFIG_ARCH_USES_GETTIMEOFFSET | 1126 | #ifdef CONFIG_ARCH_USES_GETTIMEOFFSET |
| 1101 | offset = timekeeper.cycle_interval; | 1127 | offset = tk->cycle_interval; |
| 1102 | #else | 1128 | #else |
| 1103 | offset = (clock->read(clock) - clock->cycle_last) & clock->mask; | 1129 | offset = (clock->read(clock) - clock->cycle_last) & clock->mask; |
| 1104 | #endif | 1130 | #endif |
| @@ -1111,19 +1137,19 @@ static void update_wall_time(void) | |||
| 1111 | * chunk in one go, and then try to consume the next smaller | 1137 | * chunk in one go, and then try to consume the next smaller |
| 1112 | * doubled multiple. | 1138 | * doubled multiple. |
| 1113 | */ | 1139 | */ |
| 1114 | shift = ilog2(offset) - ilog2(timekeeper.cycle_interval); | 1140 | shift = ilog2(offset) - ilog2(tk->cycle_interval); |
| 1115 | shift = max(0, shift); | 1141 | shift = max(0, shift); |
| 1116 | /* Bound shift to one less than what overflows tick_length */ | 1142 | /* Bound shift to one less than what overflows tick_length */ |
| 1117 | maxshift = (64 - (ilog2(ntp_tick_length())+1)) - 1; | 1143 | maxshift = (64 - (ilog2(ntp_tick_length())+1)) - 1; |
| 1118 | shift = min(shift, maxshift); | 1144 | shift = min(shift, maxshift); |
| 1119 | while (offset >= timekeeper.cycle_interval) { | 1145 | while (offset >= tk->cycle_interval) { |
| 1120 | offset = logarithmic_accumulation(&timekeeper, offset, shift); | 1146 | offset = logarithmic_accumulation(tk, offset, shift); |
| 1121 | if(offset < timekeeper.cycle_interval<<shift) | 1147 | if (offset < tk->cycle_interval<<shift) |
| 1122 | shift--; | 1148 | shift--; |
| 1123 | } | 1149 | } |
| 1124 | 1150 | ||
| 1125 | /* correct the clock when NTP error is too big */ | 1151 | /* correct the clock when NTP error is too big */ |
| 1126 | timekeeping_adjust(&timekeeper, offset); | 1152 | timekeeping_adjust(tk, offset); |
| 1127 | 1153 | ||
| 1128 | 1154 | ||
| 1129 | /* | 1155 | /* |
| @@ -1135,21 +1161,21 @@ static void update_wall_time(void) | |||
| 1135 | * the vsyscall implementations are converted to use xtime_nsec | 1161 | * the vsyscall implementations are converted to use xtime_nsec |
| 1136 | * (shifted nanoseconds), this can be killed. | 1162 | * (shifted nanoseconds), this can be killed. |
| 1137 | */ | 1163 | */ |
| 1138 | remainder = timekeeper.xtime_nsec & ((1 << timekeeper.shift) - 1); | 1164 | remainder = tk->xtime_nsec & ((1 << tk->shift) - 1); |
| 1139 | timekeeper.xtime_nsec -= remainder; | 1165 | tk->xtime_nsec -= remainder; |
| 1140 | timekeeper.xtime_nsec += 1 << timekeeper.shift; | 1166 | tk->xtime_nsec += 1 << tk->shift; |
| 1141 | timekeeper.ntp_error += remainder << timekeeper.ntp_error_shift; | 1167 | tk->ntp_error += remainder << tk->ntp_error_shift; |
| 1142 | 1168 | ||
| 1143 | /* | 1169 | /* |
| 1144 | * Finally, make sure that after the rounding | 1170 | * Finally, make sure that after the rounding |
| 1145 | * xtime_nsec isn't larger than NSEC_PER_SEC | 1171 | * xtime_nsec isn't larger than NSEC_PER_SEC |
| 1146 | */ | 1172 | */ |
| 1147 | accumulate_nsecs_to_secs(&timekeeper); | 1173 | accumulate_nsecs_to_secs(tk); |
| 1148 | 1174 | ||
| 1149 | timekeeping_update(&timekeeper, false); | 1175 | timekeeping_update(tk, false); |
| 1150 | 1176 | ||
| 1151 | out: | 1177 | out: |
| 1152 | write_sequnlock_irqrestore(&timekeeper.lock, flags); | 1178 | write_sequnlock_irqrestore(&tk->lock, flags); |
| 1153 | 1179 | ||
| 1154 | } | 1180 | } |
| 1155 | 1181 | ||
| @@ -1166,18 +1192,18 @@ out: | |||
| 1166 | */ | 1192 | */ |
| 1167 | void getboottime(struct timespec *ts) | 1193 | void getboottime(struct timespec *ts) |
| 1168 | { | 1194 | { |
| 1195 | struct timekeeper *tk = &timekeeper; | ||
| 1169 | struct timespec boottime = { | 1196 | struct timespec boottime = { |
| 1170 | .tv_sec = timekeeper.wall_to_monotonic.tv_sec + | 1197 | .tv_sec = tk->wall_to_monotonic.tv_sec + |
| 1171 | timekeeper.total_sleep_time.tv_sec, | 1198 | tk->total_sleep_time.tv_sec, |
| 1172 | .tv_nsec = timekeeper.wall_to_monotonic.tv_nsec + | 1199 | .tv_nsec = tk->wall_to_monotonic.tv_nsec + |
| 1173 | timekeeper.total_sleep_time.tv_nsec | 1200 | tk->total_sleep_time.tv_nsec |
| 1174 | }; | 1201 | }; |
| 1175 | 1202 | ||
| 1176 | set_normalized_timespec(ts, -boottime.tv_sec, -boottime.tv_nsec); | 1203 | set_normalized_timespec(ts, -boottime.tv_sec, -boottime.tv_nsec); |
| 1177 | } | 1204 | } |
| 1178 | EXPORT_SYMBOL_GPL(getboottime); | 1205 | EXPORT_SYMBOL_GPL(getboottime); |
| 1179 | 1206 | ||
| 1180 | |||
| 1181 | /** | 1207 | /** |
| 1182 | * get_monotonic_boottime - Returns monotonic time since boot | 1208 | * get_monotonic_boottime - Returns monotonic time since boot |
| 1183 | * @ts: pointer to the timespec to be set | 1209 | * @ts: pointer to the timespec to be set |
| @@ -1189,19 +1215,20 @@ EXPORT_SYMBOL_GPL(getboottime); | |||
| 1189 | */ | 1215 | */ |
| 1190 | void get_monotonic_boottime(struct timespec *ts) | 1216 | void get_monotonic_boottime(struct timespec *ts) |
| 1191 | { | 1217 | { |
| 1218 | struct timekeeper *tk = &timekeeper; | ||
| 1192 | struct timespec tomono, sleep; | 1219 | struct timespec tomono, sleep; |
| 1193 | unsigned int seq; | 1220 | unsigned int seq; |
| 1194 | 1221 | ||
| 1195 | WARN_ON(timekeeping_suspended); | 1222 | WARN_ON(timekeeping_suspended); |
| 1196 | 1223 | ||
| 1197 | do { | 1224 | do { |
| 1198 | seq = read_seqbegin(&timekeeper.lock); | 1225 | seq = read_seqbegin(&tk->lock); |
| 1199 | ts->tv_sec = timekeeper.xtime_sec; | 1226 | ts->tv_sec = tk->xtime_sec; |
| 1200 | ts->tv_nsec = timekeeping_get_ns(&timekeeper); | 1227 | ts->tv_nsec = timekeeping_get_ns(tk); |
| 1201 | tomono = timekeeper.wall_to_monotonic; | 1228 | tomono = tk->wall_to_monotonic; |
| 1202 | sleep = timekeeper.total_sleep_time; | 1229 | sleep = tk->total_sleep_time; |
| 1203 | 1230 | ||
| 1204 | } while (read_seqretry(&timekeeper.lock, seq)); | 1231 | } while (read_seqretry(&tk->lock, seq)); |
| 1205 | 1232 | ||
| 1206 | set_normalized_timespec(ts, ts->tv_sec + tomono.tv_sec + sleep.tv_sec, | 1233 | set_normalized_timespec(ts, ts->tv_sec + tomono.tv_sec + sleep.tv_sec, |
| 1207 | ts->tv_nsec + tomono.tv_nsec + sleep.tv_nsec); | 1234 | ts->tv_nsec + tomono.tv_nsec + sleep.tv_nsec); |
| @@ -1231,31 +1258,38 @@ EXPORT_SYMBOL_GPL(ktime_get_boottime); | |||
| 1231 | */ | 1258 | */ |
| 1232 | void monotonic_to_bootbased(struct timespec *ts) | 1259 | void monotonic_to_bootbased(struct timespec *ts) |
| 1233 | { | 1260 | { |
| 1234 | *ts = timespec_add(*ts, timekeeper.total_sleep_time); | 1261 | struct timekeeper *tk = &timekeeper; |
| 1262 | |||
| 1263 | *ts = timespec_add(*ts, tk->total_sleep_time); | ||
| 1235 | } | 1264 | } |
| 1236 | EXPORT_SYMBOL_GPL(monotonic_to_bootbased); | 1265 | EXPORT_SYMBOL_GPL(monotonic_to_bootbased); |
| 1237 | 1266 | ||
| 1238 | unsigned long get_seconds(void) | 1267 | unsigned long get_seconds(void) |
| 1239 | { | 1268 | { |
| 1240 | return timekeeper.xtime_sec; | 1269 | struct timekeeper *tk = &timekeeper; |
| 1270 | |||
| 1271 | return tk->xtime_sec; | ||
| 1241 | } | 1272 | } |
| 1242 | EXPORT_SYMBOL(get_seconds); | 1273 | EXPORT_SYMBOL(get_seconds); |
| 1243 | 1274 | ||
| 1244 | struct timespec __current_kernel_time(void) | 1275 | struct timespec __current_kernel_time(void) |
| 1245 | { | 1276 | { |
| 1246 | return tk_xtime(&timekeeper); | 1277 | struct timekeeper *tk = &timekeeper; |
| 1278 | |||
| 1279 | return tk_xtime(tk); | ||
| 1247 | } | 1280 | } |
| 1248 | 1281 | ||
| 1249 | struct timespec current_kernel_time(void) | 1282 | struct timespec current_kernel_time(void) |
| 1250 | { | 1283 | { |
| 1284 | struct timekeeper *tk = &timekeeper; | ||
| 1251 | struct timespec now; | 1285 | struct timespec now; |
| 1252 | unsigned long seq; | 1286 | unsigned long seq; |
| 1253 | 1287 | ||
| 1254 | do { | 1288 | do { |
| 1255 | seq = read_seqbegin(&timekeeper.lock); | 1289 | seq = read_seqbegin(&tk->lock); |
| 1256 | 1290 | ||
| 1257 | now = tk_xtime(&timekeeper); | 1291 | now = tk_xtime(tk); |
| 1258 | } while (read_seqretry(&timekeeper.lock, seq)); | 1292 | } while (read_seqretry(&tk->lock, seq)); |
| 1259 | 1293 | ||
| 1260 | return now; | 1294 | return now; |
| 1261 | } | 1295 | } |
| @@ -1263,15 +1297,16 @@ EXPORT_SYMBOL(current_kernel_time); | |||
| 1263 | 1297 | ||
| 1264 | struct timespec get_monotonic_coarse(void) | 1298 | struct timespec get_monotonic_coarse(void) |
| 1265 | { | 1299 | { |
| 1300 | struct timekeeper *tk = &timekeeper; | ||
| 1266 | struct timespec now, mono; | 1301 | struct timespec now, mono; |
| 1267 | unsigned long seq; | 1302 | unsigned long seq; |
| 1268 | 1303 | ||
| 1269 | do { | 1304 | do { |
| 1270 | seq = read_seqbegin(&timekeeper.lock); | 1305 | seq = read_seqbegin(&tk->lock); |
| 1271 | 1306 | ||
| 1272 | now = tk_xtime(&timekeeper); | 1307 | now = tk_xtime(tk); |
| 1273 | mono = timekeeper.wall_to_monotonic; | 1308 | mono = tk->wall_to_monotonic; |
| 1274 | } while (read_seqretry(&timekeeper.lock, seq)); | 1309 | } while (read_seqretry(&tk->lock, seq)); |
| 1275 | 1310 | ||
| 1276 | set_normalized_timespec(&now, now.tv_sec + mono.tv_sec, | 1311 | set_normalized_timespec(&now, now.tv_sec + mono.tv_sec, |
| 1277 | now.tv_nsec + mono.tv_nsec); | 1312 | now.tv_nsec + mono.tv_nsec); |
| @@ -1300,14 +1335,15 @@ void do_timer(unsigned long ticks) | |||
| 1300 | void get_xtime_and_monotonic_and_sleep_offset(struct timespec *xtim, | 1335 | void get_xtime_and_monotonic_and_sleep_offset(struct timespec *xtim, |
| 1301 | struct timespec *wtom, struct timespec *sleep) | 1336 | struct timespec *wtom, struct timespec *sleep) |
| 1302 | { | 1337 | { |
| 1338 | struct timekeeper *tk = &timekeeper; | ||
| 1303 | unsigned long seq; | 1339 | unsigned long seq; |
| 1304 | 1340 | ||
| 1305 | do { | 1341 | do { |
| 1306 | seq = read_seqbegin(&timekeeper.lock); | 1342 | seq = read_seqbegin(&tk->lock); |
| 1307 | *xtim = tk_xtime(&timekeeper); | 1343 | *xtim = tk_xtime(tk); |
| 1308 | *wtom = timekeeper.wall_to_monotonic; | 1344 | *wtom = tk->wall_to_monotonic; |
| 1309 | *sleep = timekeeper.total_sleep_time; | 1345 | *sleep = tk->total_sleep_time; |
| 1310 | } while (read_seqretry(&timekeeper.lock, seq)); | 1346 | } while (read_seqretry(&tk->lock, seq)); |
| 1311 | } | 1347 | } |
| 1312 | 1348 | ||
| 1313 | #ifdef CONFIG_HIGH_RES_TIMERS | 1349 | #ifdef CONFIG_HIGH_RES_TIMERS |
| @@ -1321,19 +1357,20 @@ void get_xtime_and_monotonic_and_sleep_offset(struct timespec *xtim, | |||
| 1321 | */ | 1357 | */ |
| 1322 | ktime_t ktime_get_update_offsets(ktime_t *offs_real, ktime_t *offs_boot) | 1358 | ktime_t ktime_get_update_offsets(ktime_t *offs_real, ktime_t *offs_boot) |
| 1323 | { | 1359 | { |
| 1360 | struct timekeeper *tk = &timekeeper; | ||
| 1324 | ktime_t now; | 1361 | ktime_t now; |
| 1325 | unsigned int seq; | 1362 | unsigned int seq; |
| 1326 | u64 secs, nsecs; | 1363 | u64 secs, nsecs; |
| 1327 | 1364 | ||
| 1328 | do { | 1365 | do { |
| 1329 | seq = read_seqbegin(&timekeeper.lock); | 1366 | seq = read_seqbegin(&tk->lock); |
| 1330 | 1367 | ||
| 1331 | secs = timekeeper.xtime_sec; | 1368 | secs = tk->xtime_sec; |
| 1332 | nsecs = timekeeping_get_ns(&timekeeper); | 1369 | nsecs = timekeeping_get_ns(tk); |
| 1333 | 1370 | ||
| 1334 | *offs_real = timekeeper.offs_real; | 1371 | *offs_real = tk->offs_real; |
| 1335 | *offs_boot = timekeeper.offs_boot; | 1372 | *offs_boot = tk->offs_boot; |
| 1336 | } while (read_seqretry(&timekeeper.lock, seq)); | 1373 | } while (read_seqretry(&tk->lock, seq)); |
| 1337 | 1374 | ||
| 1338 | now = ktime_add_ns(ktime_set(secs, 0), nsecs); | 1375 | now = ktime_add_ns(ktime_set(secs, 0), nsecs); |
| 1339 | now = ktime_sub(now, *offs_real); | 1376 | now = ktime_sub(now, *offs_real); |
| @@ -1346,19 +1383,19 @@ ktime_t ktime_get_update_offsets(ktime_t *offs_real, ktime_t *offs_boot) | |||
| 1346 | */ | 1383 | */ |
| 1347 | ktime_t ktime_get_monotonic_offset(void) | 1384 | ktime_t ktime_get_monotonic_offset(void) |
| 1348 | { | 1385 | { |
| 1386 | struct timekeeper *tk = &timekeeper; | ||
| 1349 | unsigned long seq; | 1387 | unsigned long seq; |
| 1350 | struct timespec wtom; | 1388 | struct timespec wtom; |
| 1351 | 1389 | ||
| 1352 | do { | 1390 | do { |
| 1353 | seq = read_seqbegin(&timekeeper.lock); | 1391 | seq = read_seqbegin(&tk->lock); |
| 1354 | wtom = timekeeper.wall_to_monotonic; | 1392 | wtom = tk->wall_to_monotonic; |
| 1355 | } while (read_seqretry(&timekeeper.lock, seq)); | 1393 | } while (read_seqretry(&tk->lock, seq)); |
| 1356 | 1394 | ||
| 1357 | return timespec_to_ktime(wtom); | 1395 | return timespec_to_ktime(wtom); |
| 1358 | } | 1396 | } |
| 1359 | EXPORT_SYMBOL_GPL(ktime_get_monotonic_offset); | 1397 | EXPORT_SYMBOL_GPL(ktime_get_monotonic_offset); |
| 1360 | 1398 | ||
| 1361 | |||
| 1362 | /** | 1399 | /** |
| 1363 | * xtime_update() - advances the timekeeping infrastructure | 1400 | * xtime_update() - advances the timekeeping infrastructure |
| 1364 | * @ticks: number of ticks, that have elapsed since the last call. | 1401 | * @ticks: number of ticks, that have elapsed since the last call. |
diff --git a/kernel/trace/trace_event_perf.c b/kernel/trace/trace_event_perf.c index fee3752ae8f6..8a6d2ee2086c 100644 --- a/kernel/trace/trace_event_perf.c +++ b/kernel/trace/trace_event_perf.c | |||
| @@ -281,7 +281,7 @@ perf_ftrace_function_call(unsigned long ip, unsigned long parent_ip) | |||
| 281 | 281 | ||
| 282 | head = this_cpu_ptr(event_function.perf_events); | 282 | head = this_cpu_ptr(event_function.perf_events); |
| 283 | perf_trace_buf_submit(entry, ENTRY_SIZE, rctx, 0, | 283 | perf_trace_buf_submit(entry, ENTRY_SIZE, rctx, 0, |
| 284 | 1, ®s, head); | 284 | 1, ®s, head, NULL); |
| 285 | 285 | ||
| 286 | #undef ENTRY_SIZE | 286 | #undef ENTRY_SIZE |
| 287 | } | 287 | } |
diff --git a/kernel/trace/trace_kprobe.c b/kernel/trace/trace_kprobe.c index b31d3d5699fe..1a2117043bb1 100644 --- a/kernel/trace/trace_kprobe.c +++ b/kernel/trace/trace_kprobe.c | |||
| @@ -1002,7 +1002,8 @@ static __kprobes void kprobe_perf_func(struct kprobe *kp, | |||
| 1002 | store_trace_args(sizeof(*entry), tp, regs, (u8 *)&entry[1], dsize); | 1002 | store_trace_args(sizeof(*entry), tp, regs, (u8 *)&entry[1], dsize); |
| 1003 | 1003 | ||
| 1004 | head = this_cpu_ptr(call->perf_events); | 1004 | head = this_cpu_ptr(call->perf_events); |
| 1005 | perf_trace_buf_submit(entry, size, rctx, entry->ip, 1, regs, head); | 1005 | perf_trace_buf_submit(entry, size, rctx, |
| 1006 | entry->ip, 1, regs, head, NULL); | ||
| 1006 | } | 1007 | } |
| 1007 | 1008 | ||
| 1008 | /* Kretprobe profile handler */ | 1009 | /* Kretprobe profile handler */ |
| @@ -1033,7 +1034,8 @@ static __kprobes void kretprobe_perf_func(struct kretprobe_instance *ri, | |||
| 1033 | store_trace_args(sizeof(*entry), tp, regs, (u8 *)&entry[1], dsize); | 1034 | store_trace_args(sizeof(*entry), tp, regs, (u8 *)&entry[1], dsize); |
| 1034 | 1035 | ||
| 1035 | head = this_cpu_ptr(call->perf_events); | 1036 | head = this_cpu_ptr(call->perf_events); |
| 1036 | perf_trace_buf_submit(entry, size, rctx, entry->ret_ip, 1, regs, head); | 1037 | perf_trace_buf_submit(entry, size, rctx, |
| 1038 | entry->ret_ip, 1, regs, head, NULL); | ||
| 1037 | } | 1039 | } |
| 1038 | #endif /* CONFIG_PERF_EVENTS */ | 1040 | #endif /* CONFIG_PERF_EVENTS */ |
| 1039 | 1041 | ||
diff --git a/kernel/trace/trace_syscalls.c b/kernel/trace/trace_syscalls.c index 96fc73369099..60e4d7875672 100644 --- a/kernel/trace/trace_syscalls.c +++ b/kernel/trace/trace_syscalls.c | |||
| @@ -532,7 +532,7 @@ static void perf_syscall_enter(void *ignore, struct pt_regs *regs, long id) | |||
| 532 | (unsigned long *)&rec->args); | 532 | (unsigned long *)&rec->args); |
| 533 | 533 | ||
| 534 | head = this_cpu_ptr(sys_data->enter_event->perf_events); | 534 | head = this_cpu_ptr(sys_data->enter_event->perf_events); |
| 535 | perf_trace_buf_submit(rec, size, rctx, 0, 1, regs, head); | 535 | perf_trace_buf_submit(rec, size, rctx, 0, 1, regs, head, NULL); |
| 536 | } | 536 | } |
| 537 | 537 | ||
| 538 | int perf_sysenter_enable(struct ftrace_event_call *call) | 538 | int perf_sysenter_enable(struct ftrace_event_call *call) |
| @@ -608,7 +608,7 @@ static void perf_syscall_exit(void *ignore, struct pt_regs *regs, long ret) | |||
| 608 | rec->ret = syscall_get_return_value(current, regs); | 608 | rec->ret = syscall_get_return_value(current, regs); |
| 609 | 609 | ||
| 610 | head = this_cpu_ptr(sys_data->exit_event->perf_events); | 610 | head = this_cpu_ptr(sys_data->exit_event->perf_events); |
| 611 | perf_trace_buf_submit(rec, size, rctx, 0, 1, regs, head); | 611 | perf_trace_buf_submit(rec, size, rctx, 0, 1, regs, head, NULL); |
| 612 | } | 612 | } |
| 613 | 613 | ||
| 614 | int perf_sysexit_enable(struct ftrace_event_call *call) | 614 | int perf_sysexit_enable(struct ftrace_event_call *call) |
diff --git a/kernel/trace/trace_uprobe.c b/kernel/trace/trace_uprobe.c index 2b36ac68549e..03003cd7dd96 100644 --- a/kernel/trace/trace_uprobe.c +++ b/kernel/trace/trace_uprobe.c | |||
| @@ -670,7 +670,7 @@ static void uprobe_perf_func(struct trace_uprobe *tu, struct pt_regs *regs) | |||
| 670 | call_fetch(&tu->args[i].fetch, regs, data + tu->args[i].offset); | 670 | call_fetch(&tu->args[i].fetch, regs, data + tu->args[i].offset); |
| 671 | 671 | ||
| 672 | head = this_cpu_ptr(call->perf_events); | 672 | head = this_cpu_ptr(call->perf_events); |
| 673 | perf_trace_buf_submit(entry, size, rctx, entry->ip, 1, regs, head); | 673 | perf_trace_buf_submit(entry, size, rctx, entry->ip, 1, regs, head, NULL); |
| 674 | 674 | ||
| 675 | out: | 675 | out: |
| 676 | preempt_enable(); | 676 | preempt_enable(); |
