diff options
Diffstat (limited to 'kernel')
91 files changed, 9260 insertions, 4188 deletions
diff --git a/kernel/Kconfig.locks b/kernel/Kconfig.locks new file mode 100644 index 000000000000..88c92fb44618 --- /dev/null +++ b/kernel/Kconfig.locks | |||
@@ -0,0 +1,202 @@ | |||
1 | # | ||
2 | # The ARCH_INLINE foo is necessary because select ignores "depends on" | ||
3 | # | ||
4 | config ARCH_INLINE_SPIN_TRYLOCK | ||
5 | bool | ||
6 | |||
7 | config ARCH_INLINE_SPIN_TRYLOCK_BH | ||
8 | bool | ||
9 | |||
10 | config ARCH_INLINE_SPIN_LOCK | ||
11 | bool | ||
12 | |||
13 | config ARCH_INLINE_SPIN_LOCK_BH | ||
14 | bool | ||
15 | |||
16 | config ARCH_INLINE_SPIN_LOCK_IRQ | ||
17 | bool | ||
18 | |||
19 | config ARCH_INLINE_SPIN_LOCK_IRQSAVE | ||
20 | bool | ||
21 | |||
22 | config ARCH_INLINE_SPIN_UNLOCK | ||
23 | bool | ||
24 | |||
25 | config ARCH_INLINE_SPIN_UNLOCK_BH | ||
26 | bool | ||
27 | |||
28 | config ARCH_INLINE_SPIN_UNLOCK_IRQ | ||
29 | bool | ||
30 | |||
31 | config ARCH_INLINE_SPIN_UNLOCK_IRQRESTORE | ||
32 | bool | ||
33 | |||
34 | |||
35 | config ARCH_INLINE_READ_TRYLOCK | ||
36 | bool | ||
37 | |||
38 | config ARCH_INLINE_READ_LOCK | ||
39 | bool | ||
40 | |||
41 | config ARCH_INLINE_READ_LOCK_BH | ||
42 | bool | ||
43 | |||
44 | config ARCH_INLINE_READ_LOCK_IRQ | ||
45 | bool | ||
46 | |||
47 | config ARCH_INLINE_READ_LOCK_IRQSAVE | ||
48 | bool | ||
49 | |||
50 | config ARCH_INLINE_READ_UNLOCK | ||
51 | bool | ||
52 | |||
53 | config ARCH_INLINE_READ_UNLOCK_BH | ||
54 | bool | ||
55 | |||
56 | config ARCH_INLINE_READ_UNLOCK_IRQ | ||
57 | bool | ||
58 | |||
59 | config ARCH_INLINE_READ_UNLOCK_IRQRESTORE | ||
60 | bool | ||
61 | |||
62 | |||
63 | config ARCH_INLINE_WRITE_TRYLOCK | ||
64 | bool | ||
65 | |||
66 | config ARCH_INLINE_WRITE_LOCK | ||
67 | bool | ||
68 | |||
69 | config ARCH_INLINE_WRITE_LOCK_BH | ||
70 | bool | ||
71 | |||
72 | config ARCH_INLINE_WRITE_LOCK_IRQ | ||
73 | bool | ||
74 | |||
75 | config ARCH_INLINE_WRITE_LOCK_IRQSAVE | ||
76 | bool | ||
77 | |||
78 | config ARCH_INLINE_WRITE_UNLOCK | ||
79 | bool | ||
80 | |||
81 | config ARCH_INLINE_WRITE_UNLOCK_BH | ||
82 | bool | ||
83 | |||
84 | config ARCH_INLINE_WRITE_UNLOCK_IRQ | ||
85 | bool | ||
86 | |||
87 | config ARCH_INLINE_WRITE_UNLOCK_IRQRESTORE | ||
88 | bool | ||
89 | |||
90 | # | ||
91 | # lock_* functions are inlined when: | ||
92 | # - DEBUG_SPINLOCK=n and GENERIC_LOCKBREAK=n and ARCH_INLINE_*LOCK=y | ||
93 | # | ||
94 | # trylock_* functions are inlined when: | ||
95 | # - DEBUG_SPINLOCK=n and ARCH_INLINE_*LOCK=y | ||
96 | # | ||
97 | # unlock and unlock_irq functions are inlined when: | ||
98 | # - DEBUG_SPINLOCK=n and ARCH_INLINE_*LOCK=y | ||
99 | # or | ||
100 | # - DEBUG_SPINLOCK=n and PREEMPT=n | ||
101 | # | ||
102 | # unlock_bh and unlock_irqrestore functions are inlined when: | ||
103 | # - DEBUG_SPINLOCK=n and ARCH_INLINE_*LOCK=y | ||
104 | # | ||
105 | |||
106 | config INLINE_SPIN_TRYLOCK | ||
107 | def_bool !DEBUG_SPINLOCK && ARCH_INLINE_SPIN_TRYLOCK | ||
108 | |||
109 | config INLINE_SPIN_TRYLOCK_BH | ||
110 | def_bool !DEBUG_SPINLOCK && ARCH_INLINE_SPIN_TRYLOCK_BH | ||
111 | |||
112 | config INLINE_SPIN_LOCK | ||
113 | def_bool !DEBUG_SPINLOCK && !GENERIC_LOCKBREAK && ARCH_INLINE_SPIN_LOCK | ||
114 | |||
115 | config INLINE_SPIN_LOCK_BH | ||
116 | def_bool !DEBUG_SPINLOCK && !GENERIC_LOCKBREAK && \ | ||
117 | ARCH_INLINE_SPIN_LOCK_BH | ||
118 | |||
119 | config INLINE_SPIN_LOCK_IRQ | ||
120 | def_bool !DEBUG_SPINLOCK && !GENERIC_LOCKBREAK && \ | ||
121 | ARCH_INLINE_SPIN_LOCK_IRQ | ||
122 | |||
123 | config INLINE_SPIN_LOCK_IRQSAVE | ||
124 | def_bool !DEBUG_SPINLOCK && !GENERIC_LOCKBREAK && \ | ||
125 | ARCH_INLINE_SPIN_LOCK_IRQSAVE | ||
126 | |||
127 | config INLINE_SPIN_UNLOCK | ||
128 | def_bool !DEBUG_SPINLOCK && (!PREEMPT || ARCH_INLINE_SPIN_UNLOCK) | ||
129 | |||
130 | config INLINE_SPIN_UNLOCK_BH | ||
131 | def_bool !DEBUG_SPINLOCK && ARCH_INLINE_SPIN_UNLOCK_BH | ||
132 | |||
133 | config INLINE_SPIN_UNLOCK_IRQ | ||
134 | def_bool !DEBUG_SPINLOCK && (!PREEMPT || ARCH_INLINE_SPIN_UNLOCK_BH) | ||
135 | |||
136 | config INLINE_SPIN_UNLOCK_IRQRESTORE | ||
137 | def_bool !DEBUG_SPINLOCK && ARCH_INLINE_SPIN_UNLOCK_IRQRESTORE | ||
138 | |||
139 | |||
140 | config INLINE_READ_TRYLOCK | ||
141 | def_bool !DEBUG_SPINLOCK && ARCH_INLINE_READ_TRYLOCK | ||
142 | |||
143 | config INLINE_READ_LOCK | ||
144 | def_bool !DEBUG_SPINLOCK && !GENERIC_LOCKBREAK && ARCH_INLINE_READ_LOCK | ||
145 | |||
146 | config INLINE_READ_LOCK_BH | ||
147 | def_bool !DEBUG_SPINLOCK && !GENERIC_LOCKBREAK && \ | ||
148 | ARCH_INLINE_READ_LOCK_BH | ||
149 | |||
150 | config INLINE_READ_LOCK_IRQ | ||
151 | def_bool !DEBUG_SPINLOCK && !GENERIC_LOCKBREAK && \ | ||
152 | ARCH_INLINE_READ_LOCK_IRQ | ||
153 | |||
154 | config INLINE_READ_LOCK_IRQSAVE | ||
155 | def_bool !DEBUG_SPINLOCK && !GENERIC_LOCKBREAK && \ | ||
156 | ARCH_INLINE_READ_LOCK_IRQSAVE | ||
157 | |||
158 | config INLINE_READ_UNLOCK | ||
159 | def_bool !DEBUG_SPINLOCK && (!PREEMPT || ARCH_INLINE_READ_UNLOCK) | ||
160 | |||
161 | config INLINE_READ_UNLOCK_BH | ||
162 | def_bool !DEBUG_SPINLOCK && ARCH_INLINE_READ_UNLOCK_BH | ||
163 | |||
164 | config INLINE_READ_UNLOCK_IRQ | ||
165 | def_bool !DEBUG_SPINLOCK && (!PREEMPT || ARCH_INLINE_READ_UNLOCK_BH) | ||
166 | |||
167 | config INLINE_READ_UNLOCK_IRQRESTORE | ||
168 | def_bool !DEBUG_SPINLOCK && ARCH_INLINE_READ_UNLOCK_IRQRESTORE | ||
169 | |||
170 | |||
171 | config INLINE_WRITE_TRYLOCK | ||
172 | def_bool !DEBUG_SPINLOCK && ARCH_INLINE_WRITE_TRYLOCK | ||
173 | |||
174 | config INLINE_WRITE_LOCK | ||
175 | def_bool !DEBUG_SPINLOCK && !GENERIC_LOCKBREAK && ARCH_INLINE_WRITE_LOCK | ||
176 | |||
177 | config INLINE_WRITE_LOCK_BH | ||
178 | def_bool !DEBUG_SPINLOCK && !GENERIC_LOCKBREAK && \ | ||
179 | ARCH_INLINE_WRITE_LOCK_BH | ||
180 | |||
181 | config INLINE_WRITE_LOCK_IRQ | ||
182 | def_bool !DEBUG_SPINLOCK && !GENERIC_LOCKBREAK && \ | ||
183 | ARCH_INLINE_WRITE_LOCK_IRQ | ||
184 | |||
185 | config INLINE_WRITE_LOCK_IRQSAVE | ||
186 | def_bool !DEBUG_SPINLOCK && !GENERIC_LOCKBREAK && \ | ||
187 | ARCH_INLINE_WRITE_LOCK_IRQSAVE | ||
188 | |||
189 | config INLINE_WRITE_UNLOCK | ||
190 | def_bool !DEBUG_SPINLOCK && (!PREEMPT || ARCH_INLINE_WRITE_UNLOCK) | ||
191 | |||
192 | config INLINE_WRITE_UNLOCK_BH | ||
193 | def_bool !DEBUG_SPINLOCK && ARCH_INLINE_WRITE_UNLOCK_BH | ||
194 | |||
195 | config INLINE_WRITE_UNLOCK_IRQ | ||
196 | def_bool !DEBUG_SPINLOCK && (!PREEMPT || ARCH_INLINE_WRITE_UNLOCK_BH) | ||
197 | |||
198 | config INLINE_WRITE_UNLOCK_IRQRESTORE | ||
199 | def_bool !DEBUG_SPINLOCK && ARCH_INLINE_WRITE_UNLOCK_IRQRESTORE | ||
200 | |||
201 | config MUTEX_SPIN_ON_OWNER | ||
202 | def_bool SMP && !DEBUG_MUTEXES && !HAVE_DEFAULT_NO_SPIN_MUTEXES | ||
diff --git a/kernel/Makefile b/kernel/Makefile index b8d4cd8ac0b9..864ff75d65f2 100644 --- a/kernel/Makefile +++ b/kernel/Makefile | |||
@@ -4,7 +4,7 @@ | |||
4 | 4 | ||
5 | obj-y = sched.o fork.o exec_domain.o panic.o printk.o \ | 5 | obj-y = sched.o fork.o exec_domain.o panic.o printk.o \ |
6 | cpu.o exit.o itimer.o time.o softirq.o resource.o \ | 6 | cpu.o exit.o itimer.o time.o softirq.o resource.o \ |
7 | sysctl.o capability.o ptrace.o timer.o user.o \ | 7 | sysctl.o sysctl_binary.o capability.o ptrace.o timer.o user.o \ |
8 | signal.o sys.o kmod.o workqueue.o pid.o \ | 8 | signal.o sys.o kmod.o workqueue.o pid.o \ |
9 | rcupdate.o extable.o params.o posix-timers.o \ | 9 | rcupdate.o extable.o params.o posix-timers.o \ |
10 | kthread.o wait.o kfifo.o sys_ni.o posix-cpu-timers.o mutex.o \ | 10 | kthread.o wait.o kfifo.o sys_ni.o posix-cpu-timers.o mutex.o \ |
@@ -21,6 +21,7 @@ CFLAGS_REMOVE_mutex-debug.o = -pg | |||
21 | CFLAGS_REMOVE_rtmutex-debug.o = -pg | 21 | CFLAGS_REMOVE_rtmutex-debug.o = -pg |
22 | CFLAGS_REMOVE_cgroup-debug.o = -pg | 22 | CFLAGS_REMOVE_cgroup-debug.o = -pg |
23 | CFLAGS_REMOVE_sched_clock.o = -pg | 23 | CFLAGS_REMOVE_sched_clock.o = -pg |
24 | CFLAGS_REMOVE_perf_event.o = -pg | ||
24 | endif | 25 | endif |
25 | 26 | ||
26 | obj-$(CONFIG_FREEZER) += freezer.o | 27 | obj-$(CONFIG_FREEZER) += freezer.o |
@@ -82,6 +83,7 @@ obj-$(CONFIG_RCU_TORTURE_TEST) += rcutorture.o | |||
82 | obj-$(CONFIG_TREE_RCU) += rcutree.o | 83 | obj-$(CONFIG_TREE_RCU) += rcutree.o |
83 | obj-$(CONFIG_TREE_PREEMPT_RCU) += rcutree.o | 84 | obj-$(CONFIG_TREE_PREEMPT_RCU) += rcutree.o |
84 | obj-$(CONFIG_TREE_RCU_TRACE) += rcutree_trace.o | 85 | obj-$(CONFIG_TREE_RCU_TRACE) += rcutree_trace.o |
86 | obj-$(CONFIG_TINY_RCU) += rcutiny.o | ||
85 | obj-$(CONFIG_RELAY) += relay.o | 87 | obj-$(CONFIG_RELAY) += relay.o |
86 | obj-$(CONFIG_SYSCTL) += utsname_sysctl.o | 88 | obj-$(CONFIG_SYSCTL) += utsname_sysctl.o |
87 | obj-$(CONFIG_TASK_DELAY_ACCT) += delayacct.o | 89 | obj-$(CONFIG_TASK_DELAY_ACCT) += delayacct.o |
@@ -94,7 +96,10 @@ obj-$(CONFIG_X86_DS) += trace/ | |||
94 | obj-$(CONFIG_RING_BUFFER) += trace/ | 96 | obj-$(CONFIG_RING_BUFFER) += trace/ |
95 | obj-$(CONFIG_SMP) += sched_cpupri.o | 97 | obj-$(CONFIG_SMP) += sched_cpupri.o |
96 | obj-$(CONFIG_SLOW_WORK) += slow-work.o | 98 | obj-$(CONFIG_SLOW_WORK) += slow-work.o |
99 | obj-$(CONFIG_SLOW_WORK_DEBUG) += slow-work-debugfs.o | ||
97 | obj-$(CONFIG_PERF_EVENTS) += perf_event.o | 100 | obj-$(CONFIG_PERF_EVENTS) += perf_event.o |
101 | obj-$(CONFIG_HAVE_HW_BREAKPOINT) += hw_breakpoint.o | ||
102 | obj-$(CONFIG_USER_RETURN_NOTIFIER) += user-return-notifier.o | ||
98 | 103 | ||
99 | ifneq ($(CONFIG_SCHED_OMIT_FRAME_POINTER),y) | 104 | ifneq ($(CONFIG_SCHED_OMIT_FRAME_POINTER),y) |
100 | # According to Alan Modra <alan@linuxcare.com.au>, the -fno-omit-frame-pointer is | 105 | # According to Alan Modra <alan@linuxcare.com.au>, the -fno-omit-frame-pointer is |
diff --git a/kernel/capability.c b/kernel/capability.c index 4e17041963f5..7f876e60521f 100644 --- a/kernel/capability.c +++ b/kernel/capability.c | |||
@@ -29,7 +29,6 @@ EXPORT_SYMBOL(__cap_empty_set); | |||
29 | EXPORT_SYMBOL(__cap_full_set); | 29 | EXPORT_SYMBOL(__cap_full_set); |
30 | EXPORT_SYMBOL(__cap_init_eff_set); | 30 | EXPORT_SYMBOL(__cap_init_eff_set); |
31 | 31 | ||
32 | #ifdef CONFIG_SECURITY_FILE_CAPABILITIES | ||
33 | int file_caps_enabled = 1; | 32 | int file_caps_enabled = 1; |
34 | 33 | ||
35 | static int __init file_caps_disable(char *str) | 34 | static int __init file_caps_disable(char *str) |
@@ -38,7 +37,6 @@ static int __init file_caps_disable(char *str) | |||
38 | return 1; | 37 | return 1; |
39 | } | 38 | } |
40 | __setup("no_file_caps", file_caps_disable); | 39 | __setup("no_file_caps", file_caps_disable); |
41 | #endif | ||
42 | 40 | ||
43 | /* | 41 | /* |
44 | * More recent versions of libcap are available from: | 42 | * More recent versions of libcap are available from: |
@@ -169,8 +167,8 @@ SYSCALL_DEFINE2(capget, cap_user_header_t, header, cap_user_data_t, dataptr) | |||
169 | kernel_cap_t pE, pI, pP; | 167 | kernel_cap_t pE, pI, pP; |
170 | 168 | ||
171 | ret = cap_validate_magic(header, &tocopy); | 169 | ret = cap_validate_magic(header, &tocopy); |
172 | if (ret != 0) | 170 | if ((dataptr == NULL) || (ret != 0)) |
173 | return ret; | 171 | return ((dataptr == NULL) && (ret == -EINVAL)) ? 0 : ret; |
174 | 172 | ||
175 | if (get_user(pid, &header->pid)) | 173 | if (get_user(pid, &header->pid)) |
176 | return -EFAULT; | 174 | return -EFAULT; |
@@ -238,7 +236,7 @@ SYSCALL_DEFINE2(capget, cap_user_header_t, header, cap_user_data_t, dataptr) | |||
238 | SYSCALL_DEFINE2(capset, cap_user_header_t, header, const cap_user_data_t, data) | 236 | SYSCALL_DEFINE2(capset, cap_user_header_t, header, const cap_user_data_t, data) |
239 | { | 237 | { |
240 | struct __user_cap_data_struct kdata[_KERNEL_CAPABILITY_U32S]; | 238 | struct __user_cap_data_struct kdata[_KERNEL_CAPABILITY_U32S]; |
241 | unsigned i, tocopy; | 239 | unsigned i, tocopy, copybytes; |
242 | kernel_cap_t inheritable, permitted, effective; | 240 | kernel_cap_t inheritable, permitted, effective; |
243 | struct cred *new; | 241 | struct cred *new; |
244 | int ret; | 242 | int ret; |
@@ -255,8 +253,11 @@ SYSCALL_DEFINE2(capset, cap_user_header_t, header, const cap_user_data_t, data) | |||
255 | if (pid != 0 && pid != task_pid_vnr(current)) | 253 | if (pid != 0 && pid != task_pid_vnr(current)) |
256 | return -EPERM; | 254 | return -EPERM; |
257 | 255 | ||
258 | if (copy_from_user(&kdata, data, | 256 | copybytes = tocopy * sizeof(struct __user_cap_data_struct); |
259 | tocopy * sizeof(struct __user_cap_data_struct))) | 257 | if (copybytes > sizeof(kdata)) |
258 | return -EFAULT; | ||
259 | |||
260 | if (copy_from_user(&kdata, data, copybytes)) | ||
260 | return -EFAULT; | 261 | return -EFAULT; |
261 | 262 | ||
262 | for (i = 0; i < tocopy; i++) { | 263 | for (i = 0; i < tocopy; i++) { |
diff --git a/kernel/cgroup.c b/kernel/cgroup.c index ca83b73fba19..0249f4be9b5c 100644 --- a/kernel/cgroup.c +++ b/kernel/cgroup.c | |||
@@ -1710,14 +1710,13 @@ static ssize_t cgroup_write_X64(struct cgroup *cgrp, struct cftype *cft, | |||
1710 | return -EFAULT; | 1710 | return -EFAULT; |
1711 | 1711 | ||
1712 | buffer[nbytes] = 0; /* nul-terminate */ | 1712 | buffer[nbytes] = 0; /* nul-terminate */ |
1713 | strstrip(buffer); | ||
1714 | if (cft->write_u64) { | 1713 | if (cft->write_u64) { |
1715 | u64 val = simple_strtoull(buffer, &end, 0); | 1714 | u64 val = simple_strtoull(strstrip(buffer), &end, 0); |
1716 | if (*end) | 1715 | if (*end) |
1717 | return -EINVAL; | 1716 | return -EINVAL; |
1718 | retval = cft->write_u64(cgrp, cft, val); | 1717 | retval = cft->write_u64(cgrp, cft, val); |
1719 | } else { | 1718 | } else { |
1720 | s64 val = simple_strtoll(buffer, &end, 0); | 1719 | s64 val = simple_strtoll(strstrip(buffer), &end, 0); |
1721 | if (*end) | 1720 | if (*end) |
1722 | return -EINVAL; | 1721 | return -EINVAL; |
1723 | retval = cft->write_s64(cgrp, cft, val); | 1722 | retval = cft->write_s64(cgrp, cft, val); |
@@ -1753,8 +1752,7 @@ static ssize_t cgroup_write_string(struct cgroup *cgrp, struct cftype *cft, | |||
1753 | } | 1752 | } |
1754 | 1753 | ||
1755 | buffer[nbytes] = 0; /* nul-terminate */ | 1754 | buffer[nbytes] = 0; /* nul-terminate */ |
1756 | strstrip(buffer); | 1755 | retval = cft->write_string(cgrp, cft, strstrip(buffer)); |
1757 | retval = cft->write_string(cgrp, cft, buffer); | ||
1758 | if (!retval) | 1756 | if (!retval) |
1759 | retval = nbytes; | 1757 | retval = nbytes; |
1760 | out: | 1758 | out: |
diff --git a/kernel/cpu.c b/kernel/cpu.c index 6ba0f1ecb212..7c4e2713df0a 100644 --- a/kernel/cpu.c +++ b/kernel/cpu.c | |||
@@ -392,10 +392,9 @@ int disable_nonboot_cpus(void) | |||
392 | if (cpu == first_cpu) | 392 | if (cpu == first_cpu) |
393 | continue; | 393 | continue; |
394 | error = _cpu_down(cpu, 1); | 394 | error = _cpu_down(cpu, 1); |
395 | if (!error) { | 395 | if (!error) |
396 | cpumask_set_cpu(cpu, frozen_cpus); | 396 | cpumask_set_cpu(cpu, frozen_cpus); |
397 | printk("CPU%d is down\n", cpu); | 397 | else { |
398 | } else { | ||
399 | printk(KERN_ERR "Error taking CPU%d down: %d\n", | 398 | printk(KERN_ERR "Error taking CPU%d down: %d\n", |
400 | cpu, error); | 399 | cpu, error); |
401 | break; | 400 | break; |
diff --git a/kernel/cpuset.c b/kernel/cpuset.c index b5cb469d2545..3cf2183b472d 100644 --- a/kernel/cpuset.c +++ b/kernel/cpuset.c | |||
@@ -537,8 +537,7 @@ update_domain_attr_tree(struct sched_domain_attr *dattr, struct cpuset *c) | |||
537 | * element of the partition (one sched domain) to be passed to | 537 | * element of the partition (one sched domain) to be passed to |
538 | * partition_sched_domains(). | 538 | * partition_sched_domains(). |
539 | */ | 539 | */ |
540 | /* FIXME: see the FIXME in partition_sched_domains() */ | 540 | static int generate_sched_domains(cpumask_var_t **domains, |
541 | static int generate_sched_domains(struct cpumask **domains, | ||
542 | struct sched_domain_attr **attributes) | 541 | struct sched_domain_attr **attributes) |
543 | { | 542 | { |
544 | LIST_HEAD(q); /* queue of cpusets to be scanned */ | 543 | LIST_HEAD(q); /* queue of cpusets to be scanned */ |
@@ -546,7 +545,7 @@ static int generate_sched_domains(struct cpumask **domains, | |||
546 | struct cpuset **csa; /* array of all cpuset ptrs */ | 545 | struct cpuset **csa; /* array of all cpuset ptrs */ |
547 | int csn; /* how many cpuset ptrs in csa so far */ | 546 | int csn; /* how many cpuset ptrs in csa so far */ |
548 | int i, j, k; /* indices for partition finding loops */ | 547 | int i, j, k; /* indices for partition finding loops */ |
549 | struct cpumask *doms; /* resulting partition; i.e. sched domains */ | 548 | cpumask_var_t *doms; /* resulting partition; i.e. sched domains */ |
550 | struct sched_domain_attr *dattr; /* attributes for custom domains */ | 549 | struct sched_domain_attr *dattr; /* attributes for custom domains */ |
551 | int ndoms = 0; /* number of sched domains in result */ | 550 | int ndoms = 0; /* number of sched domains in result */ |
552 | int nslot; /* next empty doms[] struct cpumask slot */ | 551 | int nslot; /* next empty doms[] struct cpumask slot */ |
@@ -557,7 +556,8 @@ static int generate_sched_domains(struct cpumask **domains, | |||
557 | 556 | ||
558 | /* Special case for the 99% of systems with one, full, sched domain */ | 557 | /* Special case for the 99% of systems with one, full, sched domain */ |
559 | if (is_sched_load_balance(&top_cpuset)) { | 558 | if (is_sched_load_balance(&top_cpuset)) { |
560 | doms = kmalloc(cpumask_size(), GFP_KERNEL); | 559 | ndoms = 1; |
560 | doms = alloc_sched_domains(ndoms); | ||
561 | if (!doms) | 561 | if (!doms) |
562 | goto done; | 562 | goto done; |
563 | 563 | ||
@@ -566,9 +566,8 @@ static int generate_sched_domains(struct cpumask **domains, | |||
566 | *dattr = SD_ATTR_INIT; | 566 | *dattr = SD_ATTR_INIT; |
567 | update_domain_attr_tree(dattr, &top_cpuset); | 567 | update_domain_attr_tree(dattr, &top_cpuset); |
568 | } | 568 | } |
569 | cpumask_copy(doms, top_cpuset.cpus_allowed); | 569 | cpumask_copy(doms[0], top_cpuset.cpus_allowed); |
570 | 570 | ||
571 | ndoms = 1; | ||
572 | goto done; | 571 | goto done; |
573 | } | 572 | } |
574 | 573 | ||
@@ -636,7 +635,7 @@ restart: | |||
636 | * Now we know how many domains to create. | 635 | * Now we know how many domains to create. |
637 | * Convert <csn, csa> to <ndoms, doms> and populate cpu masks. | 636 | * Convert <csn, csa> to <ndoms, doms> and populate cpu masks. |
638 | */ | 637 | */ |
639 | doms = kmalloc(ndoms * cpumask_size(), GFP_KERNEL); | 638 | doms = alloc_sched_domains(ndoms); |
640 | if (!doms) | 639 | if (!doms) |
641 | goto done; | 640 | goto done; |
642 | 641 | ||
@@ -656,7 +655,7 @@ restart: | |||
656 | continue; | 655 | continue; |
657 | } | 656 | } |
658 | 657 | ||
659 | dp = doms + nslot; | 658 | dp = doms[nslot]; |
660 | 659 | ||
661 | if (nslot == ndoms) { | 660 | if (nslot == ndoms) { |
662 | static int warnings = 10; | 661 | static int warnings = 10; |
@@ -718,7 +717,7 @@ done: | |||
718 | static void do_rebuild_sched_domains(struct work_struct *unused) | 717 | static void do_rebuild_sched_domains(struct work_struct *unused) |
719 | { | 718 | { |
720 | struct sched_domain_attr *attr; | 719 | struct sched_domain_attr *attr; |
721 | struct cpumask *doms; | 720 | cpumask_var_t *doms; |
722 | int ndoms; | 721 | int ndoms; |
723 | 722 | ||
724 | get_online_cpus(); | 723 | get_online_cpus(); |
@@ -2052,7 +2051,7 @@ static int cpuset_track_online_cpus(struct notifier_block *unused_nb, | |||
2052 | unsigned long phase, void *unused_cpu) | 2051 | unsigned long phase, void *unused_cpu) |
2053 | { | 2052 | { |
2054 | struct sched_domain_attr *attr; | 2053 | struct sched_domain_attr *attr; |
2055 | struct cpumask *doms; | 2054 | cpumask_var_t *doms; |
2056 | int ndoms; | 2055 | int ndoms; |
2057 | 2056 | ||
2058 | switch (phase) { | 2057 | switch (phase) { |
@@ -2537,15 +2536,9 @@ const struct file_operations proc_cpuset_operations = { | |||
2537 | }; | 2536 | }; |
2538 | #endif /* CONFIG_PROC_PID_CPUSET */ | 2537 | #endif /* CONFIG_PROC_PID_CPUSET */ |
2539 | 2538 | ||
2540 | /* Display task cpus_allowed, mems_allowed in /proc/<pid>/status file. */ | 2539 | /* Display task mems_allowed in /proc/<pid>/status file. */ |
2541 | void cpuset_task_status_allowed(struct seq_file *m, struct task_struct *task) | 2540 | void cpuset_task_status_allowed(struct seq_file *m, struct task_struct *task) |
2542 | { | 2541 | { |
2543 | seq_printf(m, "Cpus_allowed:\t"); | ||
2544 | seq_cpumask(m, &task->cpus_allowed); | ||
2545 | seq_printf(m, "\n"); | ||
2546 | seq_printf(m, "Cpus_allowed_list:\t"); | ||
2547 | seq_cpumask_list(m, &task->cpus_allowed); | ||
2548 | seq_printf(m, "\n"); | ||
2549 | seq_printf(m, "Mems_allowed:\t"); | 2542 | seq_printf(m, "Mems_allowed:\t"); |
2550 | seq_nodemask(m, &task->mems_allowed); | 2543 | seq_nodemask(m, &task->mems_allowed); |
2551 | seq_printf(m, "\n"); | 2544 | seq_printf(m, "\n"); |
diff --git a/kernel/exit.c b/kernel/exit.c index e61891f80123..1143012951e9 100644 --- a/kernel/exit.c +++ b/kernel/exit.c | |||
@@ -49,6 +49,7 @@ | |||
49 | #include <linux/init_task.h> | 49 | #include <linux/init_task.h> |
50 | #include <linux/perf_event.h> | 50 | #include <linux/perf_event.h> |
51 | #include <trace/events/sched.h> | 51 | #include <trace/events/sched.h> |
52 | #include <linux/hw_breakpoint.h> | ||
52 | 53 | ||
53 | #include <asm/uaccess.h> | 54 | #include <asm/uaccess.h> |
54 | #include <asm/unistd.h> | 55 | #include <asm/unistd.h> |
@@ -110,9 +111,9 @@ static void __exit_signal(struct task_struct *tsk) | |||
110 | * We won't ever get here for the group leader, since it | 111 | * We won't ever get here for the group leader, since it |
111 | * will have been the last reference on the signal_struct. | 112 | * will have been the last reference on the signal_struct. |
112 | */ | 113 | */ |
113 | sig->utime = cputime_add(sig->utime, task_utime(tsk)); | 114 | sig->utime = cputime_add(sig->utime, tsk->utime); |
114 | sig->stime = cputime_add(sig->stime, task_stime(tsk)); | 115 | sig->stime = cputime_add(sig->stime, tsk->stime); |
115 | sig->gtime = cputime_add(sig->gtime, task_gtime(tsk)); | 116 | sig->gtime = cputime_add(sig->gtime, tsk->gtime); |
116 | sig->min_flt += tsk->min_flt; | 117 | sig->min_flt += tsk->min_flt; |
117 | sig->maj_flt += tsk->maj_flt; | 118 | sig->maj_flt += tsk->maj_flt; |
118 | sig->nvcsw += tsk->nvcsw; | 119 | sig->nvcsw += tsk->nvcsw; |
@@ -359,10 +360,8 @@ void __set_special_pids(struct pid *pid) | |||
359 | { | 360 | { |
360 | struct task_struct *curr = current->group_leader; | 361 | struct task_struct *curr = current->group_leader; |
361 | 362 | ||
362 | if (task_session(curr) != pid) { | 363 | if (task_session(curr) != pid) |
363 | change_pid(curr, PIDTYPE_SID, pid); | 364 | change_pid(curr, PIDTYPE_SID, pid); |
364 | proc_sid_connector(curr); | ||
365 | } | ||
366 | 365 | ||
367 | if (task_pgrp(curr) != pid) | 366 | if (task_pgrp(curr) != pid) |
368 | change_pid(curr, PIDTYPE_PGID, pid); | 367 | change_pid(curr, PIDTYPE_PGID, pid); |
@@ -980,6 +979,10 @@ NORET_TYPE void do_exit(long code) | |||
980 | proc_exit_connector(tsk); | 979 | proc_exit_connector(tsk); |
981 | 980 | ||
982 | /* | 981 | /* |
982 | * FIXME: do that only when needed, using sched_exit tracepoint | ||
983 | */ | ||
984 | flush_ptrace_hw_breakpoint(tsk); | ||
985 | /* | ||
983 | * Flush inherited counters to the parent - before the parent | 986 | * Flush inherited counters to the parent - before the parent |
984 | * gets woken up by child-exit notifications. | 987 | * gets woken up by child-exit notifications. |
985 | */ | 988 | */ |
@@ -1006,7 +1009,7 @@ NORET_TYPE void do_exit(long code) | |||
1006 | tsk->flags |= PF_EXITPIDONE; | 1009 | tsk->flags |= PF_EXITPIDONE; |
1007 | 1010 | ||
1008 | if (tsk->io_context) | 1011 | if (tsk->io_context) |
1009 | exit_io_context(); | 1012 | exit_io_context(tsk); |
1010 | 1013 | ||
1011 | if (tsk->splice_pipe) | 1014 | if (tsk->splice_pipe) |
1012 | __free_pipe_info(tsk->splice_pipe); | 1015 | __free_pipe_info(tsk->splice_pipe); |
@@ -1207,6 +1210,7 @@ static int wait_task_zombie(struct wait_opts *wo, struct task_struct *p) | |||
1207 | struct signal_struct *psig; | 1210 | struct signal_struct *psig; |
1208 | struct signal_struct *sig; | 1211 | struct signal_struct *sig; |
1209 | unsigned long maxrss; | 1212 | unsigned long maxrss; |
1213 | cputime_t tgutime, tgstime; | ||
1210 | 1214 | ||
1211 | /* | 1215 | /* |
1212 | * The resource counters for the group leader are in its | 1216 | * The resource counters for the group leader are in its |
@@ -1222,20 +1226,23 @@ static int wait_task_zombie(struct wait_opts *wo, struct task_struct *p) | |||
1222 | * need to protect the access to parent->signal fields, | 1226 | * need to protect the access to parent->signal fields, |
1223 | * as other threads in the parent group can be right | 1227 | * as other threads in the parent group can be right |
1224 | * here reaping other children at the same time. | 1228 | * here reaping other children at the same time. |
1229 | * | ||
1230 | * We use thread_group_times() to get times for the thread | ||
1231 | * group, which consolidates times for all threads in the | ||
1232 | * group including the group leader. | ||
1225 | */ | 1233 | */ |
1234 | thread_group_times(p, &tgutime, &tgstime); | ||
1226 | spin_lock_irq(&p->real_parent->sighand->siglock); | 1235 | spin_lock_irq(&p->real_parent->sighand->siglock); |
1227 | psig = p->real_parent->signal; | 1236 | psig = p->real_parent->signal; |
1228 | sig = p->signal; | 1237 | sig = p->signal; |
1229 | psig->cutime = | 1238 | psig->cutime = |
1230 | cputime_add(psig->cutime, | 1239 | cputime_add(psig->cutime, |
1231 | cputime_add(p->utime, | 1240 | cputime_add(tgutime, |
1232 | cputime_add(sig->utime, | 1241 | sig->cutime)); |
1233 | sig->cutime))); | ||
1234 | psig->cstime = | 1242 | psig->cstime = |
1235 | cputime_add(psig->cstime, | 1243 | cputime_add(psig->cstime, |
1236 | cputime_add(p->stime, | 1244 | cputime_add(tgstime, |
1237 | cputime_add(sig->stime, | 1245 | sig->cstime)); |
1238 | sig->cstime))); | ||
1239 | psig->cgtime = | 1246 | psig->cgtime = |
1240 | cputime_add(psig->cgtime, | 1247 | cputime_add(psig->cgtime, |
1241 | cputime_add(p->gtime, | 1248 | cputime_add(p->gtime, |
diff --git a/kernel/fork.c b/kernel/fork.c index 4c20fff8c13a..1415dc4598ae 100644 --- a/kernel/fork.c +++ b/kernel/fork.c | |||
@@ -64,6 +64,7 @@ | |||
64 | #include <linux/magic.h> | 64 | #include <linux/magic.h> |
65 | #include <linux/perf_event.h> | 65 | #include <linux/perf_event.h> |
66 | #include <linux/posix-timers.h> | 66 | #include <linux/posix-timers.h> |
67 | #include <linux/user-return-notifier.h> | ||
67 | 68 | ||
68 | #include <asm/pgtable.h> | 69 | #include <asm/pgtable.h> |
69 | #include <asm/pgalloc.h> | 70 | #include <asm/pgalloc.h> |
@@ -91,7 +92,7 @@ int nr_processes(void) | |||
91 | int cpu; | 92 | int cpu; |
92 | int total = 0; | 93 | int total = 0; |
93 | 94 | ||
94 | for_each_online_cpu(cpu) | 95 | for_each_possible_cpu(cpu) |
95 | total += per_cpu(process_counts, cpu); | 96 | total += per_cpu(process_counts, cpu); |
96 | 97 | ||
97 | return total; | 98 | return total; |
@@ -249,6 +250,7 @@ static struct task_struct *dup_task_struct(struct task_struct *orig) | |||
249 | goto out; | 250 | goto out; |
250 | 251 | ||
251 | setup_thread_stack(tsk, orig); | 252 | setup_thread_stack(tsk, orig); |
253 | clear_user_return_notifier(tsk); | ||
252 | stackend = end_of_stack(tsk); | 254 | stackend = end_of_stack(tsk); |
253 | *stackend = STACK_END_MAGIC; /* for overflow detection */ | 255 | *stackend = STACK_END_MAGIC; /* for overflow detection */ |
254 | 256 | ||
@@ -884,6 +886,9 @@ static int copy_signal(unsigned long clone_flags, struct task_struct *tsk) | |||
884 | sig->utime = sig->stime = sig->cutime = sig->cstime = cputime_zero; | 886 | sig->utime = sig->stime = sig->cutime = sig->cstime = cputime_zero; |
885 | sig->gtime = cputime_zero; | 887 | sig->gtime = cputime_zero; |
886 | sig->cgtime = cputime_zero; | 888 | sig->cgtime = cputime_zero; |
889 | #ifndef CONFIG_VIRT_CPU_ACCOUNTING | ||
890 | sig->prev_utime = sig->prev_stime = cputime_zero; | ||
891 | #endif | ||
887 | sig->nvcsw = sig->nivcsw = sig->cnvcsw = sig->cnivcsw = 0; | 892 | sig->nvcsw = sig->nivcsw = sig->cnvcsw = sig->cnivcsw = 0; |
888 | sig->min_flt = sig->maj_flt = sig->cmin_flt = sig->cmaj_flt = 0; | 893 | sig->min_flt = sig->maj_flt = sig->cmin_flt = sig->cmaj_flt = 0; |
889 | sig->inblock = sig->oublock = sig->cinblock = sig->coublock = 0; | 894 | sig->inblock = sig->oublock = sig->cinblock = sig->coublock = 0; |
@@ -1066,8 +1071,10 @@ static struct task_struct *copy_process(unsigned long clone_flags, | |||
1066 | p->gtime = cputime_zero; | 1071 | p->gtime = cputime_zero; |
1067 | p->utimescaled = cputime_zero; | 1072 | p->utimescaled = cputime_zero; |
1068 | p->stimescaled = cputime_zero; | 1073 | p->stimescaled = cputime_zero; |
1074 | #ifndef CONFIG_VIRT_CPU_ACCOUNTING | ||
1069 | p->prev_utime = cputime_zero; | 1075 | p->prev_utime = cputime_zero; |
1070 | p->prev_stime = cputime_zero; | 1076 | p->prev_stime = cputime_zero; |
1077 | #endif | ||
1071 | 1078 | ||
1072 | p->default_timer_slack_ns = current->timer_slack_ns; | 1079 | p->default_timer_slack_ns = current->timer_slack_ns; |
1073 | 1080 | ||
@@ -1310,7 +1317,8 @@ bad_fork_free_pid: | |||
1310 | if (pid != &init_struct_pid) | 1317 | if (pid != &init_struct_pid) |
1311 | free_pid(pid); | 1318 | free_pid(pid); |
1312 | bad_fork_cleanup_io: | 1319 | bad_fork_cleanup_io: |
1313 | put_io_context(p->io_context); | 1320 | if (p->io_context) |
1321 | exit_io_context(p); | ||
1314 | bad_fork_cleanup_namespaces: | 1322 | bad_fork_cleanup_namespaces: |
1315 | exit_task_namespaces(p); | 1323 | exit_task_namespaces(p); |
1316 | bad_fork_cleanup_mm: | 1324 | bad_fork_cleanup_mm: |
diff --git a/kernel/futex.c b/kernel/futex.c index 4949d336d88d..fb65e822fc41 100644 --- a/kernel/futex.c +++ b/kernel/futex.c | |||
@@ -150,7 +150,8 @@ static struct futex_hash_bucket *hash_futex(union futex_key *key) | |||
150 | */ | 150 | */ |
151 | static inline int match_futex(union futex_key *key1, union futex_key *key2) | 151 | static inline int match_futex(union futex_key *key1, union futex_key *key2) |
152 | { | 152 | { |
153 | return (key1->both.word == key2->both.word | 153 | return (key1 && key2 |
154 | && key1->both.word == key2->both.word | ||
154 | && key1->both.ptr == key2->both.ptr | 155 | && key1->both.ptr == key2->both.ptr |
155 | && key1->both.offset == key2->both.offset); | 156 | && key1->both.offset == key2->both.offset); |
156 | } | 157 | } |
@@ -1028,7 +1029,6 @@ static inline | |||
1028 | void requeue_pi_wake_futex(struct futex_q *q, union futex_key *key, | 1029 | void requeue_pi_wake_futex(struct futex_q *q, union futex_key *key, |
1029 | struct futex_hash_bucket *hb) | 1030 | struct futex_hash_bucket *hb) |
1030 | { | 1031 | { |
1031 | drop_futex_key_refs(&q->key); | ||
1032 | get_futex_key_refs(key); | 1032 | get_futex_key_refs(key); |
1033 | q->key = *key; | 1033 | q->key = *key; |
1034 | 1034 | ||
@@ -1226,6 +1226,7 @@ retry_private: | |||
1226 | */ | 1226 | */ |
1227 | if (ret == 1) { | 1227 | if (ret == 1) { |
1228 | WARN_ON(pi_state); | 1228 | WARN_ON(pi_state); |
1229 | drop_count++; | ||
1229 | task_count++; | 1230 | task_count++; |
1230 | ret = get_futex_value_locked(&curval2, uaddr2); | 1231 | ret = get_futex_value_locked(&curval2, uaddr2); |
1231 | if (!ret) | 1232 | if (!ret) |
@@ -1304,6 +1305,7 @@ retry_private: | |||
1304 | if (ret == 1) { | 1305 | if (ret == 1) { |
1305 | /* We got the lock. */ | 1306 | /* We got the lock. */ |
1306 | requeue_pi_wake_futex(this, &key2, hb2); | 1307 | requeue_pi_wake_futex(this, &key2, hb2); |
1308 | drop_count++; | ||
1307 | continue; | 1309 | continue; |
1308 | } else if (ret) { | 1310 | } else if (ret) { |
1309 | /* -EDEADLK */ | 1311 | /* -EDEADLK */ |
@@ -1791,6 +1793,7 @@ static int futex_wait(u32 __user *uaddr, int fshared, | |||
1791 | current->timer_slack_ns); | 1793 | current->timer_slack_ns); |
1792 | } | 1794 | } |
1793 | 1795 | ||
1796 | retry: | ||
1794 | /* Prepare to wait on uaddr. */ | 1797 | /* Prepare to wait on uaddr. */ |
1795 | ret = futex_wait_setup(uaddr, val, fshared, &q, &hb); | 1798 | ret = futex_wait_setup(uaddr, val, fshared, &q, &hb); |
1796 | if (ret) | 1799 | if (ret) |
@@ -1808,9 +1811,14 @@ static int futex_wait(u32 __user *uaddr, int fshared, | |||
1808 | goto out_put_key; | 1811 | goto out_put_key; |
1809 | 1812 | ||
1810 | /* | 1813 | /* |
1811 | * We expect signal_pending(current), but another thread may | 1814 | * We expect signal_pending(current), but we might be the |
1812 | * have handled it for us already. | 1815 | * victim of a spurious wakeup as well. |
1813 | */ | 1816 | */ |
1817 | if (!signal_pending(current)) { | ||
1818 | put_futex_key(fshared, &q.key); | ||
1819 | goto retry; | ||
1820 | } | ||
1821 | |||
1814 | ret = -ERESTARTSYS; | 1822 | ret = -ERESTARTSYS; |
1815 | if (!abs_time) | 1823 | if (!abs_time) |
1816 | goto out_put_key; | 1824 | goto out_put_key; |
@@ -2118,9 +2126,11 @@ int handle_early_requeue_pi_wakeup(struct futex_hash_bucket *hb, | |||
2118 | */ | 2126 | */ |
2119 | plist_del(&q->list, &q->list.plist); | 2127 | plist_del(&q->list, &q->list.plist); |
2120 | 2128 | ||
2129 | /* Handle spurious wakeups gracefully */ | ||
2130 | ret = -EWOULDBLOCK; | ||
2121 | if (timeout && !timeout->task) | 2131 | if (timeout && !timeout->task) |
2122 | ret = -ETIMEDOUT; | 2132 | ret = -ETIMEDOUT; |
2123 | else | 2133 | else if (signal_pending(current)) |
2124 | ret = -ERESTARTNOINTR; | 2134 | ret = -ERESTARTNOINTR; |
2125 | } | 2135 | } |
2126 | return ret; | 2136 | return ret; |
diff --git a/kernel/hrtimer.c b/kernel/hrtimer.c index 3e1c36e7998f..ede527708123 100644 --- a/kernel/hrtimer.c +++ b/kernel/hrtimer.c | |||
@@ -1238,7 +1238,8 @@ hrtimer_interrupt_hanging(struct clock_event_device *dev, | |||
1238 | force_clock_reprogram = 1; | 1238 | force_clock_reprogram = 1; |
1239 | dev->min_delta_ns = (unsigned long)try_time.tv64 * 3; | 1239 | dev->min_delta_ns = (unsigned long)try_time.tv64 * 3; |
1240 | printk(KERN_WARNING "hrtimer: interrupt too slow, " | 1240 | printk(KERN_WARNING "hrtimer: interrupt too slow, " |
1241 | "forcing clock min delta to %lu ns\n", dev->min_delta_ns); | 1241 | "forcing clock min delta to %llu ns\n", |
1242 | (unsigned long long) dev->min_delta_ns); | ||
1242 | } | 1243 | } |
1243 | /* | 1244 | /* |
1244 | * High resolution timer interrupt | 1245 | * High resolution timer interrupt |
diff --git a/kernel/hung_task.c b/kernel/hung_task.c index d4e841747400..0c642d51aac2 100644 --- a/kernel/hung_task.c +++ b/kernel/hung_task.c | |||
@@ -144,7 +144,7 @@ static void check_hung_uninterruptible_tasks(unsigned long timeout) | |||
144 | 144 | ||
145 | rcu_read_lock(); | 145 | rcu_read_lock(); |
146 | do_each_thread(g, t) { | 146 | do_each_thread(g, t) { |
147 | if (!--max_count) | 147 | if (!max_count--) |
148 | goto unlock; | 148 | goto unlock; |
149 | if (!--batch_count) { | 149 | if (!--batch_count) { |
150 | batch_count = HUNG_TASK_BATCHING; | 150 | batch_count = HUNG_TASK_BATCHING; |
diff --git a/kernel/hw_breakpoint.c b/kernel/hw_breakpoint.c new file mode 100644 index 000000000000..cf5ee1628411 --- /dev/null +++ b/kernel/hw_breakpoint.c | |||
@@ -0,0 +1,423 @@ | |||
1 | /* | ||
2 | * This program is free software; you can redistribute it and/or modify | ||
3 | * it under the terms of the GNU General Public License as published by | ||
4 | * the Free Software Foundation; either version 2 of the License, or | ||
5 | * (at your option) any later version. | ||
6 | * | ||
7 | * This program is distributed in the hope that it will be useful, | ||
8 | * but WITHOUT ANY WARRANTY; without even the implied warranty of | ||
9 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | ||
10 | * GNU General Public License for more details. | ||
11 | * | ||
12 | * You should have received a copy of the GNU General Public License | ||
13 | * along with this program; if not, write to the Free Software | ||
14 | * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. | ||
15 | * | ||
16 | * Copyright (C) 2007 Alan Stern | ||
17 | * Copyright (C) IBM Corporation, 2009 | ||
18 | * Copyright (C) 2009, Frederic Weisbecker <fweisbec@gmail.com> | ||
19 | * | ||
20 | * Thanks to Ingo Molnar for his many suggestions. | ||
21 | * | ||
22 | * Authors: Alan Stern <stern@rowland.harvard.edu> | ||
23 | * K.Prasad <prasad@linux.vnet.ibm.com> | ||
24 | * Frederic Weisbecker <fweisbec@gmail.com> | ||
25 | */ | ||
26 | |||
27 | /* | ||
28 | * HW_breakpoint: a unified kernel/user-space hardware breakpoint facility, | ||
29 | * using the CPU's debug registers. | ||
30 | * This file contains the arch-independent routines. | ||
31 | */ | ||
32 | |||
33 | #include <linux/irqflags.h> | ||
34 | #include <linux/kallsyms.h> | ||
35 | #include <linux/notifier.h> | ||
36 | #include <linux/kprobes.h> | ||
37 | #include <linux/kdebug.h> | ||
38 | #include <linux/kernel.h> | ||
39 | #include <linux/module.h> | ||
40 | #include <linux/percpu.h> | ||
41 | #include <linux/sched.h> | ||
42 | #include <linux/init.h> | ||
43 | #include <linux/smp.h> | ||
44 | |||
45 | #include <linux/hw_breakpoint.h> | ||
46 | |||
47 | /* | ||
48 | * Constraints data | ||
49 | */ | ||
50 | |||
51 | /* Number of pinned cpu breakpoints in a cpu */ | ||
52 | static DEFINE_PER_CPU(unsigned int, nr_cpu_bp_pinned); | ||
53 | |||
54 | /* Number of pinned task breakpoints in a cpu */ | ||
55 | static DEFINE_PER_CPU(unsigned int, task_bp_pinned[HBP_NUM]); | ||
56 | |||
57 | /* Number of non-pinned cpu/task breakpoints in a cpu */ | ||
58 | static DEFINE_PER_CPU(unsigned int, nr_bp_flexible); | ||
59 | |||
60 | /* Gather the number of total pinned and un-pinned bp in a cpuset */ | ||
61 | struct bp_busy_slots { | ||
62 | unsigned int pinned; | ||
63 | unsigned int flexible; | ||
64 | }; | ||
65 | |||
66 | /* Serialize accesses to the above constraints */ | ||
67 | static DEFINE_MUTEX(nr_bp_mutex); | ||
68 | |||
69 | /* | ||
70 | * Report the maximum number of pinned breakpoints a task | ||
71 | * have in this cpu | ||
72 | */ | ||
73 | static unsigned int max_task_bp_pinned(int cpu) | ||
74 | { | ||
75 | int i; | ||
76 | unsigned int *tsk_pinned = per_cpu(task_bp_pinned, cpu); | ||
77 | |||
78 | for (i = HBP_NUM -1; i >= 0; i--) { | ||
79 | if (tsk_pinned[i] > 0) | ||
80 | return i + 1; | ||
81 | } | ||
82 | |||
83 | return 0; | ||
84 | } | ||
85 | |||
86 | /* | ||
87 | * Report the number of pinned/un-pinned breakpoints we have in | ||
88 | * a given cpu (cpu > -1) or in all of them (cpu = -1). | ||
89 | */ | ||
90 | static void fetch_bp_busy_slots(struct bp_busy_slots *slots, int cpu) | ||
91 | { | ||
92 | if (cpu >= 0) { | ||
93 | slots->pinned = per_cpu(nr_cpu_bp_pinned, cpu); | ||
94 | slots->pinned += max_task_bp_pinned(cpu); | ||
95 | slots->flexible = per_cpu(nr_bp_flexible, cpu); | ||
96 | |||
97 | return; | ||
98 | } | ||
99 | |||
100 | for_each_online_cpu(cpu) { | ||
101 | unsigned int nr; | ||
102 | |||
103 | nr = per_cpu(nr_cpu_bp_pinned, cpu); | ||
104 | nr += max_task_bp_pinned(cpu); | ||
105 | |||
106 | if (nr > slots->pinned) | ||
107 | slots->pinned = nr; | ||
108 | |||
109 | nr = per_cpu(nr_bp_flexible, cpu); | ||
110 | |||
111 | if (nr > slots->flexible) | ||
112 | slots->flexible = nr; | ||
113 | } | ||
114 | } | ||
115 | |||
116 | /* | ||
117 | * Add a pinned breakpoint for the given task in our constraint table | ||
118 | */ | ||
119 | static void toggle_bp_task_slot(struct task_struct *tsk, int cpu, bool enable) | ||
120 | { | ||
121 | int count = 0; | ||
122 | struct perf_event *bp; | ||
123 | struct perf_event_context *ctx = tsk->perf_event_ctxp; | ||
124 | unsigned int *tsk_pinned; | ||
125 | struct list_head *list; | ||
126 | unsigned long flags; | ||
127 | |||
128 | if (WARN_ONCE(!ctx, "No perf context for this task")) | ||
129 | return; | ||
130 | |||
131 | list = &ctx->event_list; | ||
132 | |||
133 | spin_lock_irqsave(&ctx->lock, flags); | ||
134 | |||
135 | /* | ||
136 | * The current breakpoint counter is not included in the list | ||
137 | * at the open() callback time | ||
138 | */ | ||
139 | list_for_each_entry(bp, list, event_entry) { | ||
140 | if (bp->attr.type == PERF_TYPE_BREAKPOINT) | ||
141 | count++; | ||
142 | } | ||
143 | |||
144 | spin_unlock_irqrestore(&ctx->lock, flags); | ||
145 | |||
146 | if (WARN_ONCE(count < 0, "No breakpoint counter found in the counter list")) | ||
147 | return; | ||
148 | |||
149 | tsk_pinned = per_cpu(task_bp_pinned, cpu); | ||
150 | if (enable) { | ||
151 | tsk_pinned[count]++; | ||
152 | if (count > 0) | ||
153 | tsk_pinned[count-1]--; | ||
154 | } else { | ||
155 | tsk_pinned[count]--; | ||
156 | if (count > 0) | ||
157 | tsk_pinned[count-1]++; | ||
158 | } | ||
159 | } | ||
160 | |||
161 | /* | ||
162 | * Add/remove the given breakpoint in our constraint table | ||
163 | */ | ||
164 | static void toggle_bp_slot(struct perf_event *bp, bool enable) | ||
165 | { | ||
166 | int cpu = bp->cpu; | ||
167 | struct task_struct *tsk = bp->ctx->task; | ||
168 | |||
169 | /* Pinned counter task profiling */ | ||
170 | if (tsk) { | ||
171 | if (cpu >= 0) { | ||
172 | toggle_bp_task_slot(tsk, cpu, enable); | ||
173 | return; | ||
174 | } | ||
175 | |||
176 | for_each_online_cpu(cpu) | ||
177 | toggle_bp_task_slot(tsk, cpu, enable); | ||
178 | return; | ||
179 | } | ||
180 | |||
181 | /* Pinned counter cpu profiling */ | ||
182 | if (enable) | ||
183 | per_cpu(nr_cpu_bp_pinned, bp->cpu)++; | ||
184 | else | ||
185 | per_cpu(nr_cpu_bp_pinned, bp->cpu)--; | ||
186 | } | ||
187 | |||
188 | /* | ||
189 | * Contraints to check before allowing this new breakpoint counter: | ||
190 | * | ||
191 | * == Non-pinned counter == (Considered as pinned for now) | ||
192 | * | ||
193 | * - If attached to a single cpu, check: | ||
194 | * | ||
195 | * (per_cpu(nr_bp_flexible, cpu) || (per_cpu(nr_cpu_bp_pinned, cpu) | ||
196 | * + max(per_cpu(task_bp_pinned, cpu)))) < HBP_NUM | ||
197 | * | ||
198 | * -> If there are already non-pinned counters in this cpu, it means | ||
199 | * there is already a free slot for them. | ||
200 | * Otherwise, we check that the maximum number of per task | ||
201 | * breakpoints (for this cpu) plus the number of per cpu breakpoint | ||
202 | * (for this cpu) doesn't cover every registers. | ||
203 | * | ||
204 | * - If attached to every cpus, check: | ||
205 | * | ||
206 | * (per_cpu(nr_bp_flexible, *) || (max(per_cpu(nr_cpu_bp_pinned, *)) | ||
207 | * + max(per_cpu(task_bp_pinned, *)))) < HBP_NUM | ||
208 | * | ||
209 | * -> This is roughly the same, except we check the number of per cpu | ||
210 | * bp for every cpu and we keep the max one. Same for the per tasks | ||
211 | * breakpoints. | ||
212 | * | ||
213 | * | ||
214 | * == Pinned counter == | ||
215 | * | ||
216 | * - If attached to a single cpu, check: | ||
217 | * | ||
218 | * ((per_cpu(nr_bp_flexible, cpu) > 1) + per_cpu(nr_cpu_bp_pinned, cpu) | ||
219 | * + max(per_cpu(task_bp_pinned, cpu))) < HBP_NUM | ||
220 | * | ||
221 | * -> Same checks as before. But now the nr_bp_flexible, if any, must keep | ||
222 | * one register at least (or they will never be fed). | ||
223 | * | ||
224 | * - If attached to every cpus, check: | ||
225 | * | ||
226 | * ((per_cpu(nr_bp_flexible, *) > 1) + max(per_cpu(nr_cpu_bp_pinned, *)) | ||
227 | * + max(per_cpu(task_bp_pinned, *))) < HBP_NUM | ||
228 | */ | ||
229 | int reserve_bp_slot(struct perf_event *bp) | ||
230 | { | ||
231 | struct bp_busy_slots slots = {0}; | ||
232 | int ret = 0; | ||
233 | |||
234 | mutex_lock(&nr_bp_mutex); | ||
235 | |||
236 | fetch_bp_busy_slots(&slots, bp->cpu); | ||
237 | |||
238 | /* Flexible counters need to keep at least one slot */ | ||
239 | if (slots.pinned + (!!slots.flexible) == HBP_NUM) { | ||
240 | ret = -ENOSPC; | ||
241 | goto end; | ||
242 | } | ||
243 | |||
244 | toggle_bp_slot(bp, true); | ||
245 | |||
246 | end: | ||
247 | mutex_unlock(&nr_bp_mutex); | ||
248 | |||
249 | return ret; | ||
250 | } | ||
251 | |||
252 | void release_bp_slot(struct perf_event *bp) | ||
253 | { | ||
254 | mutex_lock(&nr_bp_mutex); | ||
255 | |||
256 | toggle_bp_slot(bp, false); | ||
257 | |||
258 | mutex_unlock(&nr_bp_mutex); | ||
259 | } | ||
260 | |||
261 | |||
262 | int __register_perf_hw_breakpoint(struct perf_event *bp) | ||
263 | { | ||
264 | int ret; | ||
265 | |||
266 | ret = reserve_bp_slot(bp); | ||
267 | if (ret) | ||
268 | return ret; | ||
269 | |||
270 | /* | ||
271 | * Ptrace breakpoints can be temporary perf events only | ||
272 | * meant to reserve a slot. In this case, it is created disabled and | ||
273 | * we don't want to check the params right now (as we put a null addr) | ||
274 | * But perf tools create events as disabled and we want to check | ||
275 | * the params for them. | ||
276 | * This is a quick hack that will be removed soon, once we remove | ||
277 | * the tmp breakpoints from ptrace | ||
278 | */ | ||
279 | if (!bp->attr.disabled || bp->callback == perf_bp_event) | ||
280 | ret = arch_validate_hwbkpt_settings(bp, bp->ctx->task); | ||
281 | |||
282 | return ret; | ||
283 | } | ||
284 | |||
285 | int register_perf_hw_breakpoint(struct perf_event *bp) | ||
286 | { | ||
287 | bp->callback = perf_bp_event; | ||
288 | |||
289 | return __register_perf_hw_breakpoint(bp); | ||
290 | } | ||
291 | |||
292 | /** | ||
293 | * register_user_hw_breakpoint - register a hardware breakpoint for user space | ||
294 | * @attr: breakpoint attributes | ||
295 | * @triggered: callback to trigger when we hit the breakpoint | ||
296 | * @tsk: pointer to 'task_struct' of the process to which the address belongs | ||
297 | */ | ||
298 | struct perf_event * | ||
299 | register_user_hw_breakpoint(struct perf_event_attr *attr, | ||
300 | perf_callback_t triggered, | ||
301 | struct task_struct *tsk) | ||
302 | { | ||
303 | return perf_event_create_kernel_counter(attr, -1, tsk->pid, triggered); | ||
304 | } | ||
305 | EXPORT_SYMBOL_GPL(register_user_hw_breakpoint); | ||
306 | |||
307 | /** | ||
308 | * modify_user_hw_breakpoint - modify a user-space hardware breakpoint | ||
309 | * @bp: the breakpoint structure to modify | ||
310 | * @attr: new breakpoint attributes | ||
311 | * @triggered: callback to trigger when we hit the breakpoint | ||
312 | * @tsk: pointer to 'task_struct' of the process to which the address belongs | ||
313 | */ | ||
314 | struct perf_event * | ||
315 | modify_user_hw_breakpoint(struct perf_event *bp, struct perf_event_attr *attr, | ||
316 | perf_callback_t triggered, | ||
317 | struct task_struct *tsk) | ||
318 | { | ||
319 | /* | ||
320 | * FIXME: do it without unregistering | ||
321 | * - We don't want to lose our slot | ||
322 | * - If the new bp is incorrect, don't lose the older one | ||
323 | */ | ||
324 | unregister_hw_breakpoint(bp); | ||
325 | |||
326 | return perf_event_create_kernel_counter(attr, -1, tsk->pid, triggered); | ||
327 | } | ||
328 | EXPORT_SYMBOL_GPL(modify_user_hw_breakpoint); | ||
329 | |||
330 | /** | ||
331 | * unregister_hw_breakpoint - unregister a user-space hardware breakpoint | ||
332 | * @bp: the breakpoint structure to unregister | ||
333 | */ | ||
334 | void unregister_hw_breakpoint(struct perf_event *bp) | ||
335 | { | ||
336 | if (!bp) | ||
337 | return; | ||
338 | perf_event_release_kernel(bp); | ||
339 | } | ||
340 | EXPORT_SYMBOL_GPL(unregister_hw_breakpoint); | ||
341 | |||
342 | /** | ||
343 | * register_wide_hw_breakpoint - register a wide breakpoint in the kernel | ||
344 | * @attr: breakpoint attributes | ||
345 | * @triggered: callback to trigger when we hit the breakpoint | ||
346 | * | ||
347 | * @return a set of per_cpu pointers to perf events | ||
348 | */ | ||
349 | struct perf_event ** | ||
350 | register_wide_hw_breakpoint(struct perf_event_attr *attr, | ||
351 | perf_callback_t triggered) | ||
352 | { | ||
353 | struct perf_event **cpu_events, **pevent, *bp; | ||
354 | long err; | ||
355 | int cpu; | ||
356 | |||
357 | cpu_events = alloc_percpu(typeof(*cpu_events)); | ||
358 | if (!cpu_events) | ||
359 | return ERR_PTR(-ENOMEM); | ||
360 | |||
361 | for_each_possible_cpu(cpu) { | ||
362 | pevent = per_cpu_ptr(cpu_events, cpu); | ||
363 | bp = perf_event_create_kernel_counter(attr, cpu, -1, triggered); | ||
364 | |||
365 | *pevent = bp; | ||
366 | |||
367 | if (IS_ERR(bp)) { | ||
368 | err = PTR_ERR(bp); | ||
369 | goto fail; | ||
370 | } | ||
371 | } | ||
372 | |||
373 | return cpu_events; | ||
374 | |||
375 | fail: | ||
376 | for_each_possible_cpu(cpu) { | ||
377 | pevent = per_cpu_ptr(cpu_events, cpu); | ||
378 | if (IS_ERR(*pevent)) | ||
379 | break; | ||
380 | unregister_hw_breakpoint(*pevent); | ||
381 | } | ||
382 | free_percpu(cpu_events); | ||
383 | /* return the error if any */ | ||
384 | return ERR_PTR(err); | ||
385 | } | ||
386 | EXPORT_SYMBOL_GPL(register_wide_hw_breakpoint); | ||
387 | |||
388 | /** | ||
389 | * unregister_wide_hw_breakpoint - unregister a wide breakpoint in the kernel | ||
390 | * @cpu_events: the per cpu set of events to unregister | ||
391 | */ | ||
392 | void unregister_wide_hw_breakpoint(struct perf_event **cpu_events) | ||
393 | { | ||
394 | int cpu; | ||
395 | struct perf_event **pevent; | ||
396 | |||
397 | for_each_possible_cpu(cpu) { | ||
398 | pevent = per_cpu_ptr(cpu_events, cpu); | ||
399 | unregister_hw_breakpoint(*pevent); | ||
400 | } | ||
401 | free_percpu(cpu_events); | ||
402 | } | ||
403 | EXPORT_SYMBOL_GPL(unregister_wide_hw_breakpoint); | ||
404 | |||
405 | static struct notifier_block hw_breakpoint_exceptions_nb = { | ||
406 | .notifier_call = hw_breakpoint_exceptions_notify, | ||
407 | /* we need to be notified first */ | ||
408 | .priority = 0x7fffffff | ||
409 | }; | ||
410 | |||
411 | static int __init init_hw_breakpoint(void) | ||
412 | { | ||
413 | return register_die_notifier(&hw_breakpoint_exceptions_nb); | ||
414 | } | ||
415 | core_initcall(init_hw_breakpoint); | ||
416 | |||
417 | |||
418 | struct pmu perf_ops_bp = { | ||
419 | .enable = arch_install_hw_breakpoint, | ||
420 | .disable = arch_uninstall_hw_breakpoint, | ||
421 | .read = hw_breakpoint_pmu_read, | ||
422 | .unthrottle = hw_breakpoint_pmu_unthrottle | ||
423 | }; | ||
diff --git a/kernel/irq/chip.c b/kernel/irq/chip.c index c1660194d115..ba566c261adc 100644 --- a/kernel/irq/chip.c +++ b/kernel/irq/chip.c | |||
@@ -166,11 +166,11 @@ int set_irq_data(unsigned int irq, void *data) | |||
166 | EXPORT_SYMBOL(set_irq_data); | 166 | EXPORT_SYMBOL(set_irq_data); |
167 | 167 | ||
168 | /** | 168 | /** |
169 | * set_irq_data - set irq type data for an irq | 169 | * set_irq_msi - set MSI descriptor data for an irq |
170 | * @irq: Interrupt number | 170 | * @irq: Interrupt number |
171 | * @entry: Pointer to MSI descriptor data | 171 | * @entry: Pointer to MSI descriptor data |
172 | * | 172 | * |
173 | * Set the hardware irq controller data for an irq | 173 | * Set the MSI descriptor entry for an irq |
174 | */ | 174 | */ |
175 | int set_irq_msi(unsigned int irq, struct msi_desc *entry) | 175 | int set_irq_msi(unsigned int irq, struct msi_desc *entry) |
176 | { | 176 | { |
@@ -590,7 +590,7 @@ out_unlock: | |||
590 | } | 590 | } |
591 | 591 | ||
592 | /** | 592 | /** |
593 | * handle_percpu_IRQ - Per CPU local irq handler | 593 | * handle_percpu_irq - Per CPU local irq handler |
594 | * @irq: the interrupt number | 594 | * @irq: the interrupt number |
595 | * @desc: the interrupt description structure for this irq | 595 | * @desc: the interrupt description structure for this irq |
596 | * | 596 | * |
diff --git a/kernel/irq/manage.c b/kernel/irq/manage.c index bde4c667d24d..7305b297d1eb 100644 --- a/kernel/irq/manage.c +++ b/kernel/irq/manage.c | |||
@@ -1067,7 +1067,7 @@ int request_threaded_irq(unsigned int irq, irq_handler_t handler, | |||
1067 | kfree(action); | 1067 | kfree(action); |
1068 | 1068 | ||
1069 | #ifdef CONFIG_DEBUG_SHIRQ | 1069 | #ifdef CONFIG_DEBUG_SHIRQ |
1070 | if (irqflags & IRQF_SHARED) { | 1070 | if (!retval && (irqflags & IRQF_SHARED)) { |
1071 | /* | 1071 | /* |
1072 | * It's a shared IRQ -- the driver ought to be prepared for it | 1072 | * It's a shared IRQ -- the driver ought to be prepared for it |
1073 | * to happen immediately, so let's make sure.... | 1073 | * to happen immediately, so let's make sure.... |
diff --git a/kernel/irq/proc.c b/kernel/irq/proc.c index 692363dd591f..0832145fea97 100644 --- a/kernel/irq/proc.c +++ b/kernel/irq/proc.c | |||
@@ -136,7 +136,7 @@ out: | |||
136 | 136 | ||
137 | static int default_affinity_open(struct inode *inode, struct file *file) | 137 | static int default_affinity_open(struct inode *inode, struct file *file) |
138 | { | 138 | { |
139 | return single_open(file, default_affinity_show, NULL); | 139 | return single_open(file, default_affinity_show, PDE(inode)->data); |
140 | } | 140 | } |
141 | 141 | ||
142 | static const struct file_operations default_affinity_proc_fops = { | 142 | static const struct file_operations default_affinity_proc_fops = { |
@@ -148,18 +148,28 @@ static const struct file_operations default_affinity_proc_fops = { | |||
148 | }; | 148 | }; |
149 | #endif | 149 | #endif |
150 | 150 | ||
151 | static int irq_spurious_read(char *page, char **start, off_t off, | 151 | static int irq_spurious_proc_show(struct seq_file *m, void *v) |
152 | int count, int *eof, void *data) | ||
153 | { | 152 | { |
154 | struct irq_desc *desc = irq_to_desc((long) data); | 153 | struct irq_desc *desc = irq_to_desc((long) m->private); |
155 | return sprintf(page, "count %u\n" | 154 | |
156 | "unhandled %u\n" | 155 | seq_printf(m, "count %u\n" "unhandled %u\n" "last_unhandled %u ms\n", |
157 | "last_unhandled %u ms\n", | 156 | desc->irq_count, desc->irqs_unhandled, |
158 | desc->irq_count, | 157 | jiffies_to_msecs(desc->last_unhandled)); |
159 | desc->irqs_unhandled, | 158 | return 0; |
160 | jiffies_to_msecs(desc->last_unhandled)); | 159 | } |
160 | |||
161 | static int irq_spurious_proc_open(struct inode *inode, struct file *file) | ||
162 | { | ||
163 | return single_open(file, irq_spurious_proc_show, NULL); | ||
161 | } | 164 | } |
162 | 165 | ||
166 | static const struct file_operations irq_spurious_proc_fops = { | ||
167 | .open = irq_spurious_proc_open, | ||
168 | .read = seq_read, | ||
169 | .llseek = seq_lseek, | ||
170 | .release = single_release, | ||
171 | }; | ||
172 | |||
163 | #define MAX_NAMELEN 128 | 173 | #define MAX_NAMELEN 128 |
164 | 174 | ||
165 | static int name_unique(unsigned int irq, struct irqaction *new_action) | 175 | static int name_unique(unsigned int irq, struct irqaction *new_action) |
@@ -204,7 +214,6 @@ void register_handler_proc(unsigned int irq, struct irqaction *action) | |||
204 | void register_irq_proc(unsigned int irq, struct irq_desc *desc) | 214 | void register_irq_proc(unsigned int irq, struct irq_desc *desc) |
205 | { | 215 | { |
206 | char name [MAX_NAMELEN]; | 216 | char name [MAX_NAMELEN]; |
207 | struct proc_dir_entry *entry; | ||
208 | 217 | ||
209 | if (!root_irq_dir || (desc->chip == &no_irq_chip) || desc->dir) | 218 | if (!root_irq_dir || (desc->chip == &no_irq_chip) || desc->dir) |
210 | return; | 219 | return; |
@@ -214,6 +223,8 @@ void register_irq_proc(unsigned int irq, struct irq_desc *desc) | |||
214 | 223 | ||
215 | /* create /proc/irq/1234 */ | 224 | /* create /proc/irq/1234 */ |
216 | desc->dir = proc_mkdir(name, root_irq_dir); | 225 | desc->dir = proc_mkdir(name, root_irq_dir); |
226 | if (!desc->dir) | ||
227 | return; | ||
217 | 228 | ||
218 | #ifdef CONFIG_SMP | 229 | #ifdef CONFIG_SMP |
219 | /* create /proc/irq/<irq>/smp_affinity */ | 230 | /* create /proc/irq/<irq>/smp_affinity */ |
@@ -221,11 +232,8 @@ void register_irq_proc(unsigned int irq, struct irq_desc *desc) | |||
221 | &irq_affinity_proc_fops, (void *)(long)irq); | 232 | &irq_affinity_proc_fops, (void *)(long)irq); |
222 | #endif | 233 | #endif |
223 | 234 | ||
224 | entry = create_proc_entry("spurious", 0444, desc->dir); | 235 | proc_create_data("spurious", 0444, desc->dir, |
225 | if (entry) { | 236 | &irq_spurious_proc_fops, (void *)(long)irq); |
226 | entry->data = (void *)(long)irq; | ||
227 | entry->read_proc = irq_spurious_read; | ||
228 | } | ||
229 | } | 237 | } |
230 | 238 | ||
231 | #undef MAX_NAMELEN | 239 | #undef MAX_NAMELEN |
diff --git a/kernel/irq/spurious.c b/kernel/irq/spurious.c index 114e704760fe..e49ea1c5232d 100644 --- a/kernel/irq/spurious.c +++ b/kernel/irq/spurious.c | |||
@@ -104,7 +104,7 @@ static int misrouted_irq(int irq) | |||
104 | return ok; | 104 | return ok; |
105 | } | 105 | } |
106 | 106 | ||
107 | static void poll_all_shared_irqs(void) | 107 | static void poll_spurious_irqs(unsigned long dummy) |
108 | { | 108 | { |
109 | struct irq_desc *desc; | 109 | struct irq_desc *desc; |
110 | int i; | 110 | int i; |
@@ -121,25 +121,15 @@ static void poll_all_shared_irqs(void) | |||
121 | if (!(status & IRQ_SPURIOUS_DISABLED)) | 121 | if (!(status & IRQ_SPURIOUS_DISABLED)) |
122 | continue; | 122 | continue; |
123 | 123 | ||
124 | local_irq_disable(); | ||
124 | try_one_irq(i, desc); | 125 | try_one_irq(i, desc); |
126 | local_irq_enable(); | ||
125 | } | 127 | } |
126 | } | ||
127 | |||
128 | static void poll_spurious_irqs(unsigned long dummy) | ||
129 | { | ||
130 | poll_all_shared_irqs(); | ||
131 | 128 | ||
132 | mod_timer(&poll_spurious_irq_timer, | 129 | mod_timer(&poll_spurious_irq_timer, |
133 | jiffies + POLL_SPURIOUS_IRQ_INTERVAL); | 130 | jiffies + POLL_SPURIOUS_IRQ_INTERVAL); |
134 | } | 131 | } |
135 | 132 | ||
136 | #ifdef CONFIG_DEBUG_SHIRQ | ||
137 | void debug_poll_all_shared_irqs(void) | ||
138 | { | ||
139 | poll_all_shared_irqs(); | ||
140 | } | ||
141 | #endif | ||
142 | |||
143 | /* | 133 | /* |
144 | * If 99,900 of the previous 100,000 interrupts have not been handled | 134 | * If 99,900 of the previous 100,000 interrupts have not been handled |
145 | * then assume that the IRQ is stuck in some manner. Drop a diagnostic | 135 | * then assume that the IRQ is stuck in some manner. Drop a diagnostic |
@@ -230,7 +220,7 @@ void note_interrupt(unsigned int irq, struct irq_desc *desc, | |||
230 | /* | 220 | /* |
231 | * If we are seeing only the odd spurious IRQ caused by | 221 | * If we are seeing only the odd spurious IRQ caused by |
232 | * bus asynchronicity then don't eventually trigger an error, | 222 | * bus asynchronicity then don't eventually trigger an error, |
233 | * otherwise the couter becomes a doomsday timer for otherwise | 223 | * otherwise the counter becomes a doomsday timer for otherwise |
234 | * working systems | 224 | * working systems |
235 | */ | 225 | */ |
236 | if (time_after(jiffies, desc->last_unhandled + HZ/10)) | 226 | if (time_after(jiffies, desc->last_unhandled + HZ/10)) |
diff --git a/kernel/itimer.c b/kernel/itimer.c index b03451ede528..d802883153da 100644 --- a/kernel/itimer.c +++ b/kernel/itimer.c | |||
@@ -146,6 +146,7 @@ static void set_cpu_itimer(struct task_struct *tsk, unsigned int clock_id, | |||
146 | { | 146 | { |
147 | cputime_t cval, nval, cinterval, ninterval; | 147 | cputime_t cval, nval, cinterval, ninterval; |
148 | s64 ns_ninterval, ns_nval; | 148 | s64 ns_ninterval, ns_nval; |
149 | u32 error, incr_error; | ||
149 | struct cpu_itimer *it = &tsk->signal->it[clock_id]; | 150 | struct cpu_itimer *it = &tsk->signal->it[clock_id]; |
150 | 151 | ||
151 | nval = timeval_to_cputime(&value->it_value); | 152 | nval = timeval_to_cputime(&value->it_value); |
@@ -153,8 +154,8 @@ static void set_cpu_itimer(struct task_struct *tsk, unsigned int clock_id, | |||
153 | ninterval = timeval_to_cputime(&value->it_interval); | 154 | ninterval = timeval_to_cputime(&value->it_interval); |
154 | ns_ninterval = timeval_to_ns(&value->it_interval); | 155 | ns_ninterval = timeval_to_ns(&value->it_interval); |
155 | 156 | ||
156 | it->incr_error = cputime_sub_ns(ninterval, ns_ninterval); | 157 | error = cputime_sub_ns(nval, ns_nval); |
157 | it->error = cputime_sub_ns(nval, ns_nval); | 158 | incr_error = cputime_sub_ns(ninterval, ns_ninterval); |
158 | 159 | ||
159 | spin_lock_irq(&tsk->sighand->siglock); | 160 | spin_lock_irq(&tsk->sighand->siglock); |
160 | 161 | ||
@@ -168,6 +169,8 @@ static void set_cpu_itimer(struct task_struct *tsk, unsigned int clock_id, | |||
168 | } | 169 | } |
169 | it->expires = nval; | 170 | it->expires = nval; |
170 | it->incr = ninterval; | 171 | it->incr = ninterval; |
172 | it->error = error; | ||
173 | it->incr_error = incr_error; | ||
171 | trace_itimer_state(clock_id == CPUCLOCK_VIRT ? | 174 | trace_itimer_state(clock_id == CPUCLOCK_VIRT ? |
172 | ITIMER_VIRTUAL : ITIMER_PROF, value, nval); | 175 | ITIMER_VIRTUAL : ITIMER_PROF, value, nval); |
173 | 176 | ||
diff --git a/kernel/kallsyms.c b/kernel/kallsyms.c index 8b6b8b697c68..8e5288a8a355 100644 --- a/kernel/kallsyms.c +++ b/kernel/kallsyms.c | |||
@@ -181,6 +181,7 @@ unsigned long kallsyms_lookup_name(const char *name) | |||
181 | } | 181 | } |
182 | return module_kallsyms_lookup_name(name); | 182 | return module_kallsyms_lookup_name(name); |
183 | } | 183 | } |
184 | EXPORT_SYMBOL_GPL(kallsyms_lookup_name); | ||
184 | 185 | ||
185 | int kallsyms_on_each_symbol(int (*fn)(void *, const char *, struct module *, | 186 | int kallsyms_on_each_symbol(int (*fn)(void *, const char *, struct module *, |
186 | unsigned long), | 187 | unsigned long), |
diff --git a/kernel/kgdb.c b/kernel/kgdb.c index 9147a3190c9d..7d7014634022 100644 --- a/kernel/kgdb.c +++ b/kernel/kgdb.c | |||
@@ -870,7 +870,7 @@ static void gdb_cmd_getregs(struct kgdb_state *ks) | |||
870 | 870 | ||
871 | /* | 871 | /* |
872 | * All threads that don't have debuggerinfo should be | 872 | * All threads that don't have debuggerinfo should be |
873 | * in __schedule() sleeping, since all other CPUs | 873 | * in schedule() sleeping, since all other CPUs |
874 | * are in kgdb_wait, and thus have debuggerinfo. | 874 | * are in kgdb_wait, and thus have debuggerinfo. |
875 | */ | 875 | */ |
876 | if (local_debuggerinfo) { | 876 | if (local_debuggerinfo) { |
diff --git a/kernel/kmod.c b/kernel/kmod.c index 9fcb53a11f87..25b103190364 100644 --- a/kernel/kmod.c +++ b/kernel/kmod.c | |||
@@ -80,16 +80,16 @@ int __request_module(bool wait, const char *fmt, ...) | |||
80 | #define MAX_KMOD_CONCURRENT 50 /* Completely arbitrary value - KAO */ | 80 | #define MAX_KMOD_CONCURRENT 50 /* Completely arbitrary value - KAO */ |
81 | static int kmod_loop_msg; | 81 | static int kmod_loop_msg; |
82 | 82 | ||
83 | ret = security_kernel_module_request(); | ||
84 | if (ret) | ||
85 | return ret; | ||
86 | |||
87 | va_start(args, fmt); | 83 | va_start(args, fmt); |
88 | ret = vsnprintf(module_name, MODULE_NAME_LEN, fmt, args); | 84 | ret = vsnprintf(module_name, MODULE_NAME_LEN, fmt, args); |
89 | va_end(args); | 85 | va_end(args); |
90 | if (ret >= MODULE_NAME_LEN) | 86 | if (ret >= MODULE_NAME_LEN) |
91 | return -ENAMETOOLONG; | 87 | return -ENAMETOOLONG; |
92 | 88 | ||
89 | ret = security_kernel_module_request(module_name); | ||
90 | if (ret) | ||
91 | return ret; | ||
92 | |||
93 | /* If modprobe needs a service that is in a module, we get a recursive | 93 | /* If modprobe needs a service that is in a module, we get a recursive |
94 | * loop. Limit the number of running kmod threads to max_threads/2 or | 94 | * loop. Limit the number of running kmod threads to max_threads/2 or |
95 | * MAX_KMOD_CONCURRENT, whichever is the smaller. A cleaner method | 95 | * MAX_KMOD_CONCURRENT, whichever is the smaller. A cleaner method |
diff --git a/kernel/kprobes.c b/kernel/kprobes.c index 5240d75f4c60..e5342a344c43 100644 --- a/kernel/kprobes.c +++ b/kernel/kprobes.c | |||
@@ -90,6 +90,9 @@ static spinlock_t *kretprobe_table_lock_ptr(unsigned long hash) | |||
90 | */ | 90 | */ |
91 | static struct kprobe_blackpoint kprobe_blacklist[] = { | 91 | static struct kprobe_blackpoint kprobe_blacklist[] = { |
92 | {"preempt_schedule",}, | 92 | {"preempt_schedule",}, |
93 | {"native_get_debugreg",}, | ||
94 | {"irq_entries_start",}, | ||
95 | {"common_interrupt",}, | ||
93 | {NULL} /* Terminator */ | 96 | {NULL} /* Terminator */ |
94 | }; | 97 | }; |
95 | 98 | ||
@@ -673,6 +676,40 @@ static kprobe_opcode_t __kprobes *kprobe_addr(struct kprobe *p) | |||
673 | return (kprobe_opcode_t *)(((char *)addr) + p->offset); | 676 | return (kprobe_opcode_t *)(((char *)addr) + p->offset); |
674 | } | 677 | } |
675 | 678 | ||
679 | /* Check passed kprobe is valid and return kprobe in kprobe_table. */ | ||
680 | static struct kprobe * __kprobes __get_valid_kprobe(struct kprobe *p) | ||
681 | { | ||
682 | struct kprobe *old_p, *list_p; | ||
683 | |||
684 | old_p = get_kprobe(p->addr); | ||
685 | if (unlikely(!old_p)) | ||
686 | return NULL; | ||
687 | |||
688 | if (p != old_p) { | ||
689 | list_for_each_entry_rcu(list_p, &old_p->list, list) | ||
690 | if (list_p == p) | ||
691 | /* kprobe p is a valid probe */ | ||
692 | goto valid; | ||
693 | return NULL; | ||
694 | } | ||
695 | valid: | ||
696 | return old_p; | ||
697 | } | ||
698 | |||
699 | /* Return error if the kprobe is being re-registered */ | ||
700 | static inline int check_kprobe_rereg(struct kprobe *p) | ||
701 | { | ||
702 | int ret = 0; | ||
703 | struct kprobe *old_p; | ||
704 | |||
705 | mutex_lock(&kprobe_mutex); | ||
706 | old_p = __get_valid_kprobe(p); | ||
707 | if (old_p) | ||
708 | ret = -EINVAL; | ||
709 | mutex_unlock(&kprobe_mutex); | ||
710 | return ret; | ||
711 | } | ||
712 | |||
676 | int __kprobes register_kprobe(struct kprobe *p) | 713 | int __kprobes register_kprobe(struct kprobe *p) |
677 | { | 714 | { |
678 | int ret = 0; | 715 | int ret = 0; |
@@ -685,6 +722,10 @@ int __kprobes register_kprobe(struct kprobe *p) | |||
685 | return -EINVAL; | 722 | return -EINVAL; |
686 | p->addr = addr; | 723 | p->addr = addr; |
687 | 724 | ||
725 | ret = check_kprobe_rereg(p); | ||
726 | if (ret) | ||
727 | return ret; | ||
728 | |||
688 | preempt_disable(); | 729 | preempt_disable(); |
689 | if (!kernel_text_address((unsigned long) p->addr) || | 730 | if (!kernel_text_address((unsigned long) p->addr) || |
690 | in_kprobes_functions((unsigned long) p->addr)) { | 731 | in_kprobes_functions((unsigned long) p->addr)) { |
@@ -754,26 +795,6 @@ out: | |||
754 | } | 795 | } |
755 | EXPORT_SYMBOL_GPL(register_kprobe); | 796 | EXPORT_SYMBOL_GPL(register_kprobe); |
756 | 797 | ||
757 | /* Check passed kprobe is valid and return kprobe in kprobe_table. */ | ||
758 | static struct kprobe * __kprobes __get_valid_kprobe(struct kprobe *p) | ||
759 | { | ||
760 | struct kprobe *old_p, *list_p; | ||
761 | |||
762 | old_p = get_kprobe(p->addr); | ||
763 | if (unlikely(!old_p)) | ||
764 | return NULL; | ||
765 | |||
766 | if (p != old_p) { | ||
767 | list_for_each_entry_rcu(list_p, &old_p->list, list) | ||
768 | if (list_p == p) | ||
769 | /* kprobe p is a valid probe */ | ||
770 | goto valid; | ||
771 | return NULL; | ||
772 | } | ||
773 | valid: | ||
774 | return old_p; | ||
775 | } | ||
776 | |||
777 | /* | 798 | /* |
778 | * Unregister a kprobe without a scheduler synchronization. | 799 | * Unregister a kprobe without a scheduler synchronization. |
779 | */ | 800 | */ |
@@ -1014,9 +1035,9 @@ int __kprobes register_kretprobe(struct kretprobe *rp) | |||
1014 | /* Pre-allocate memory for max kretprobe instances */ | 1035 | /* Pre-allocate memory for max kretprobe instances */ |
1015 | if (rp->maxactive <= 0) { | 1036 | if (rp->maxactive <= 0) { |
1016 | #ifdef CONFIG_PREEMPT | 1037 | #ifdef CONFIG_PREEMPT |
1017 | rp->maxactive = max(10, 2 * NR_CPUS); | 1038 | rp->maxactive = max(10, 2 * num_possible_cpus()); |
1018 | #else | 1039 | #else |
1019 | rp->maxactive = NR_CPUS; | 1040 | rp->maxactive = num_possible_cpus(); |
1020 | #endif | 1041 | #endif |
1021 | } | 1042 | } |
1022 | spin_lock_init(&rp->lock); | 1043 | spin_lock_init(&rp->lock); |
@@ -1141,6 +1162,13 @@ static void __kprobes kill_kprobe(struct kprobe *p) | |||
1141 | arch_remove_kprobe(p); | 1162 | arch_remove_kprobe(p); |
1142 | } | 1163 | } |
1143 | 1164 | ||
1165 | void __kprobes dump_kprobe(struct kprobe *kp) | ||
1166 | { | ||
1167 | printk(KERN_WARNING "Dumping kprobe:\n"); | ||
1168 | printk(KERN_WARNING "Name: %s\nAddress: %p\nOffset: %x\n", | ||
1169 | kp->symbol_name, kp->addr, kp->offset); | ||
1170 | } | ||
1171 | |||
1144 | /* Module notifier call back, checking kprobes on the module */ | 1172 | /* Module notifier call back, checking kprobes on the module */ |
1145 | static int __kprobes kprobes_module_callback(struct notifier_block *nb, | 1173 | static int __kprobes kprobes_module_callback(struct notifier_block *nb, |
1146 | unsigned long val, void *data) | 1174 | unsigned long val, void *data) |
diff --git a/kernel/kthread.c b/kernel/kthread.c index 5fe709982caa..ab7ae57773e1 100644 --- a/kernel/kthread.c +++ b/kernel/kthread.c | |||
@@ -150,29 +150,6 @@ struct task_struct *kthread_create(int (*threadfn)(void *data), | |||
150 | EXPORT_SYMBOL(kthread_create); | 150 | EXPORT_SYMBOL(kthread_create); |
151 | 151 | ||
152 | /** | 152 | /** |
153 | * kthread_bind - bind a just-created kthread to a cpu. | ||
154 | * @k: thread created by kthread_create(). | ||
155 | * @cpu: cpu (might not be online, must be possible) for @k to run on. | ||
156 | * | ||
157 | * Description: This function is equivalent to set_cpus_allowed(), | ||
158 | * except that @cpu doesn't need to be online, and the thread must be | ||
159 | * stopped (i.e., just returned from kthread_create()). | ||
160 | */ | ||
161 | void kthread_bind(struct task_struct *k, unsigned int cpu) | ||
162 | { | ||
163 | /* Must have done schedule() in kthread() before we set_task_cpu */ | ||
164 | if (!wait_task_inactive(k, TASK_UNINTERRUPTIBLE)) { | ||
165 | WARN_ON(1); | ||
166 | return; | ||
167 | } | ||
168 | set_task_cpu(k, cpu); | ||
169 | k->cpus_allowed = cpumask_of_cpu(cpu); | ||
170 | k->rt.nr_cpus_allowed = 1; | ||
171 | k->flags |= PF_THREAD_BOUND; | ||
172 | } | ||
173 | EXPORT_SYMBOL(kthread_bind); | ||
174 | |||
175 | /** | ||
176 | * kthread_stop - stop a thread created by kthread_create(). | 153 | * kthread_stop - stop a thread created by kthread_create(). |
177 | * @k: thread created by kthread_create(). | 154 | * @k: thread created by kthread_create(). |
178 | * | 155 | * |
diff --git a/kernel/lockdep.c b/kernel/lockdep.c index 9af56723c096..f5dcd36d3151 100644 --- a/kernel/lockdep.c +++ b/kernel/lockdep.c | |||
@@ -49,7 +49,7 @@ | |||
49 | #include "lockdep_internals.h" | 49 | #include "lockdep_internals.h" |
50 | 50 | ||
51 | #define CREATE_TRACE_POINTS | 51 | #define CREATE_TRACE_POINTS |
52 | #include <trace/events/lockdep.h> | 52 | #include <trace/events/lock.h> |
53 | 53 | ||
54 | #ifdef CONFIG_PROVE_LOCKING | 54 | #ifdef CONFIG_PROVE_LOCKING |
55 | int prove_locking = 1; | 55 | int prove_locking = 1; |
diff --git a/kernel/module.c b/kernel/module.c index 8b7d8805819d..5842a71cf052 100644 --- a/kernel/module.c +++ b/kernel/module.c | |||
@@ -1187,7 +1187,8 @@ static void add_sect_attrs(struct module *mod, unsigned int nsect, | |||
1187 | 1187 | ||
1188 | /* Count loaded sections and allocate structures */ | 1188 | /* Count loaded sections and allocate structures */ |
1189 | for (i = 0; i < nsect; i++) | 1189 | for (i = 0; i < nsect; i++) |
1190 | if (sechdrs[i].sh_flags & SHF_ALLOC) | 1190 | if (sechdrs[i].sh_flags & SHF_ALLOC |
1191 | && sechdrs[i].sh_size) | ||
1191 | nloaded++; | 1192 | nloaded++; |
1192 | size[0] = ALIGN(sizeof(*sect_attrs) | 1193 | size[0] = ALIGN(sizeof(*sect_attrs) |
1193 | + nloaded * sizeof(sect_attrs->attrs[0]), | 1194 | + nloaded * sizeof(sect_attrs->attrs[0]), |
@@ -1207,6 +1208,8 @@ static void add_sect_attrs(struct module *mod, unsigned int nsect, | |||
1207 | for (i = 0; i < nsect; i++) { | 1208 | for (i = 0; i < nsect; i++) { |
1208 | if (! (sechdrs[i].sh_flags & SHF_ALLOC)) | 1209 | if (! (sechdrs[i].sh_flags & SHF_ALLOC)) |
1209 | continue; | 1210 | continue; |
1211 | if (!sechdrs[i].sh_size) | ||
1212 | continue; | ||
1210 | sattr->address = sechdrs[i].sh_addr; | 1213 | sattr->address = sechdrs[i].sh_addr; |
1211 | sattr->name = kstrdup(secstrings + sechdrs[i].sh_name, | 1214 | sattr->name = kstrdup(secstrings + sechdrs[i].sh_name, |
1212 | GFP_KERNEL); | 1215 | GFP_KERNEL); |
diff --git a/kernel/mutex.c b/kernel/mutex.c index 947b3ad551f8..632f04c57d82 100644 --- a/kernel/mutex.c +++ b/kernel/mutex.c | |||
@@ -148,8 +148,8 @@ __mutex_lock_common(struct mutex *lock, long state, unsigned int subclass, | |||
148 | 148 | ||
149 | preempt_disable(); | 149 | preempt_disable(); |
150 | mutex_acquire(&lock->dep_map, subclass, 0, ip); | 150 | mutex_acquire(&lock->dep_map, subclass, 0, ip); |
151 | #if defined(CONFIG_SMP) && !defined(CONFIG_DEBUG_MUTEXES) && \ | 151 | |
152 | !defined(CONFIG_HAVE_DEFAULT_NO_SPIN_MUTEXES) | 152 | #ifdef CONFIG_MUTEX_SPIN_ON_OWNER |
153 | /* | 153 | /* |
154 | * Optimistic spinning. | 154 | * Optimistic spinning. |
155 | * | 155 | * |
diff --git a/kernel/notifier.c b/kernel/notifier.c index 61d5aa5eced3..acd24e7643eb 100644 --- a/kernel/notifier.c +++ b/kernel/notifier.c | |||
@@ -558,7 +558,7 @@ EXPORT_SYMBOL(unregister_reboot_notifier); | |||
558 | 558 | ||
559 | static ATOMIC_NOTIFIER_HEAD(die_chain); | 559 | static ATOMIC_NOTIFIER_HEAD(die_chain); |
560 | 560 | ||
561 | int notrace notify_die(enum die_val val, const char *str, | 561 | int notrace __kprobes notify_die(enum die_val val, const char *str, |
562 | struct pt_regs *regs, long err, int trap, int sig) | 562 | struct pt_regs *regs, long err, int trap, int sig) |
563 | { | 563 | { |
564 | struct die_args args = { | 564 | struct die_args args = { |
diff --git a/kernel/params.c b/kernel/params.c index 9da58eabdcb2..d656c276508d 100644 --- a/kernel/params.c +++ b/kernel/params.c | |||
@@ -218,15 +218,11 @@ int param_set_charp(const char *val, struct kernel_param *kp) | |||
218 | return -ENOSPC; | 218 | return -ENOSPC; |
219 | } | 219 | } |
220 | 220 | ||
221 | if (kp->flags & KPARAM_KMALLOCED) | ||
222 | kfree(*(char **)kp->arg); | ||
223 | |||
224 | /* This is a hack. We can't need to strdup in early boot, and we | 221 | /* This is a hack. We can't need to strdup in early boot, and we |
225 | * don't need to; this mangled commandline is preserved. */ | 222 | * don't need to; this mangled commandline is preserved. */ |
226 | if (slab_is_available()) { | 223 | if (slab_is_available()) { |
227 | kp->flags |= KPARAM_KMALLOCED; | ||
228 | *(char **)kp->arg = kstrdup(val, GFP_KERNEL); | 224 | *(char **)kp->arg = kstrdup(val, GFP_KERNEL); |
229 | if (!kp->arg) | 225 | if (!*(char **)kp->arg) |
230 | return -ENOMEM; | 226 | return -ENOMEM; |
231 | } else | 227 | } else |
232 | *(const char **)kp->arg = val; | 228 | *(const char **)kp->arg = val; |
@@ -304,6 +300,7 @@ static int param_array(const char *name, | |||
304 | unsigned int min, unsigned int max, | 300 | unsigned int min, unsigned int max, |
305 | void *elem, int elemsize, | 301 | void *elem, int elemsize, |
306 | int (*set)(const char *, struct kernel_param *kp), | 302 | int (*set)(const char *, struct kernel_param *kp), |
303 | u16 flags, | ||
307 | unsigned int *num) | 304 | unsigned int *num) |
308 | { | 305 | { |
309 | int ret; | 306 | int ret; |
@@ -313,6 +310,7 @@ static int param_array(const char *name, | |||
313 | /* Get the name right for errors. */ | 310 | /* Get the name right for errors. */ |
314 | kp.name = name; | 311 | kp.name = name; |
315 | kp.arg = elem; | 312 | kp.arg = elem; |
313 | kp.flags = flags; | ||
316 | 314 | ||
317 | /* No equals sign? */ | 315 | /* No equals sign? */ |
318 | if (!val) { | 316 | if (!val) { |
@@ -358,7 +356,8 @@ int param_array_set(const char *val, struct kernel_param *kp) | |||
358 | unsigned int temp_num; | 356 | unsigned int temp_num; |
359 | 357 | ||
360 | return param_array(kp->name, val, 1, arr->max, arr->elem, | 358 | return param_array(kp->name, val, 1, arr->max, arr->elem, |
361 | arr->elemsize, arr->set, arr->num ?: &temp_num); | 359 | arr->elemsize, arr->set, kp->flags, |
360 | arr->num ?: &temp_num); | ||
362 | } | 361 | } |
363 | 362 | ||
364 | int param_array_get(char *buffer, struct kernel_param *kp) | 363 | int param_array_get(char *buffer, struct kernel_param *kp) |
@@ -605,11 +604,7 @@ void module_param_sysfs_remove(struct module *mod) | |||
605 | 604 | ||
606 | void destroy_params(const struct kernel_param *params, unsigned num) | 605 | void destroy_params(const struct kernel_param *params, unsigned num) |
607 | { | 606 | { |
608 | unsigned int i; | 607 | /* FIXME: This should free kmalloced charp parameters. It doesn't. */ |
609 | |||
610 | for (i = 0; i < num; i++) | ||
611 | if (params[i].flags & KPARAM_KMALLOCED) | ||
612 | kfree(*(char **)params[i].arg); | ||
613 | } | 608 | } |
614 | 609 | ||
615 | static void __init kernel_add_sysfs_param(const char *name, | 610 | static void __init kernel_add_sysfs_param(const char *name, |
diff --git a/kernel/perf_event.c b/kernel/perf_event.c index 9d0b5c665883..40a996ec39fa 100644 --- a/kernel/perf_event.c +++ b/kernel/perf_event.c | |||
@@ -28,6 +28,8 @@ | |||
28 | #include <linux/anon_inodes.h> | 28 | #include <linux/anon_inodes.h> |
29 | #include <linux/kernel_stat.h> | 29 | #include <linux/kernel_stat.h> |
30 | #include <linux/perf_event.h> | 30 | #include <linux/perf_event.h> |
31 | #include <linux/ftrace_event.h> | ||
32 | #include <linux/hw_breakpoint.h> | ||
31 | 33 | ||
32 | #include <asm/irq_regs.h> | 34 | #include <asm/irq_regs.h> |
33 | 35 | ||
@@ -244,6 +246,49 @@ static void perf_unpin_context(struct perf_event_context *ctx) | |||
244 | put_ctx(ctx); | 246 | put_ctx(ctx); |
245 | } | 247 | } |
246 | 248 | ||
249 | static inline u64 perf_clock(void) | ||
250 | { | ||
251 | return cpu_clock(smp_processor_id()); | ||
252 | } | ||
253 | |||
254 | /* | ||
255 | * Update the record of the current time in a context. | ||
256 | */ | ||
257 | static void update_context_time(struct perf_event_context *ctx) | ||
258 | { | ||
259 | u64 now = perf_clock(); | ||
260 | |||
261 | ctx->time += now - ctx->timestamp; | ||
262 | ctx->timestamp = now; | ||
263 | } | ||
264 | |||
265 | /* | ||
266 | * Update the total_time_enabled and total_time_running fields for a event. | ||
267 | */ | ||
268 | static void update_event_times(struct perf_event *event) | ||
269 | { | ||
270 | struct perf_event_context *ctx = event->ctx; | ||
271 | u64 run_end; | ||
272 | |||
273 | if (event->state < PERF_EVENT_STATE_INACTIVE || | ||
274 | event->group_leader->state < PERF_EVENT_STATE_INACTIVE) | ||
275 | return; | ||
276 | |||
277 | if (ctx->is_active) | ||
278 | run_end = ctx->time; | ||
279 | else | ||
280 | run_end = event->tstamp_stopped; | ||
281 | |||
282 | event->total_time_enabled = run_end - event->tstamp_enabled; | ||
283 | |||
284 | if (event->state == PERF_EVENT_STATE_INACTIVE) | ||
285 | run_end = event->tstamp_stopped; | ||
286 | else | ||
287 | run_end = ctx->time; | ||
288 | |||
289 | event->total_time_running = run_end - event->tstamp_running; | ||
290 | } | ||
291 | |||
247 | /* | 292 | /* |
248 | * Add a event from the lists for its context. | 293 | * Add a event from the lists for its context. |
249 | * Must be called with ctx->mutex and ctx->lock held. | 294 | * Must be called with ctx->mutex and ctx->lock held. |
@@ -292,6 +337,18 @@ list_del_event(struct perf_event *event, struct perf_event_context *ctx) | |||
292 | if (event->group_leader != event) | 337 | if (event->group_leader != event) |
293 | event->group_leader->nr_siblings--; | 338 | event->group_leader->nr_siblings--; |
294 | 339 | ||
340 | update_event_times(event); | ||
341 | |||
342 | /* | ||
343 | * If event was in error state, then keep it | ||
344 | * that way, otherwise bogus counts will be | ||
345 | * returned on read(). The only way to get out | ||
346 | * of error state is by explicit re-enabling | ||
347 | * of the event | ||
348 | */ | ||
349 | if (event->state > PERF_EVENT_STATE_OFF) | ||
350 | event->state = PERF_EVENT_STATE_OFF; | ||
351 | |||
295 | /* | 352 | /* |
296 | * If this was a group event with sibling events then | 353 | * If this was a group event with sibling events then |
297 | * upgrade the siblings to singleton events by adding them | 354 | * upgrade the siblings to singleton events by adding them |
@@ -419,7 +476,7 @@ static void perf_event_remove_from_context(struct perf_event *event) | |||
419 | if (!task) { | 476 | if (!task) { |
420 | /* | 477 | /* |
421 | * Per cpu events are removed via an smp call and | 478 | * Per cpu events are removed via an smp call and |
422 | * the removal is always sucessful. | 479 | * the removal is always successful. |
423 | */ | 480 | */ |
424 | smp_call_function_single(event->cpu, | 481 | smp_call_function_single(event->cpu, |
425 | __perf_event_remove_from_context, | 482 | __perf_event_remove_from_context, |
@@ -445,50 +502,11 @@ retry: | |||
445 | * can remove the event safely, if the call above did not | 502 | * can remove the event safely, if the call above did not |
446 | * succeed. | 503 | * succeed. |
447 | */ | 504 | */ |
448 | if (!list_empty(&event->group_entry)) { | 505 | if (!list_empty(&event->group_entry)) |
449 | list_del_event(event, ctx); | 506 | list_del_event(event, ctx); |
450 | } | ||
451 | spin_unlock_irq(&ctx->lock); | 507 | spin_unlock_irq(&ctx->lock); |
452 | } | 508 | } |
453 | 509 | ||
454 | static inline u64 perf_clock(void) | ||
455 | { | ||
456 | return cpu_clock(smp_processor_id()); | ||
457 | } | ||
458 | |||
459 | /* | ||
460 | * Update the record of the current time in a context. | ||
461 | */ | ||
462 | static void update_context_time(struct perf_event_context *ctx) | ||
463 | { | ||
464 | u64 now = perf_clock(); | ||
465 | |||
466 | ctx->time += now - ctx->timestamp; | ||
467 | ctx->timestamp = now; | ||
468 | } | ||
469 | |||
470 | /* | ||
471 | * Update the total_time_enabled and total_time_running fields for a event. | ||
472 | */ | ||
473 | static void update_event_times(struct perf_event *event) | ||
474 | { | ||
475 | struct perf_event_context *ctx = event->ctx; | ||
476 | u64 run_end; | ||
477 | |||
478 | if (event->state < PERF_EVENT_STATE_INACTIVE || | ||
479 | event->group_leader->state < PERF_EVENT_STATE_INACTIVE) | ||
480 | return; | ||
481 | |||
482 | event->total_time_enabled = ctx->time - event->tstamp_enabled; | ||
483 | |||
484 | if (event->state == PERF_EVENT_STATE_INACTIVE) | ||
485 | run_end = event->tstamp_stopped; | ||
486 | else | ||
487 | run_end = ctx->time; | ||
488 | |||
489 | event->total_time_running = run_end - event->tstamp_running; | ||
490 | } | ||
491 | |||
492 | /* | 510 | /* |
493 | * Update total_time_enabled and total_time_running for all events in a group. | 511 | * Update total_time_enabled and total_time_running for all events in a group. |
494 | */ | 512 | */ |
@@ -827,7 +845,7 @@ perf_install_in_context(struct perf_event_context *ctx, | |||
827 | if (!task) { | 845 | if (!task) { |
828 | /* | 846 | /* |
829 | * Per cpu events are installed via an smp call and | 847 | * Per cpu events are installed via an smp call and |
830 | * the install is always sucessful. | 848 | * the install is always successful. |
831 | */ | 849 | */ |
832 | smp_call_function_single(cpu, __perf_install_in_context, | 850 | smp_call_function_single(cpu, __perf_install_in_context, |
833 | event, 1); | 851 | event, 1); |
@@ -1031,10 +1049,10 @@ void __perf_event_sched_out(struct perf_event_context *ctx, | |||
1031 | update_context_time(ctx); | 1049 | update_context_time(ctx); |
1032 | 1050 | ||
1033 | perf_disable(); | 1051 | perf_disable(); |
1034 | if (ctx->nr_active) | 1052 | if (ctx->nr_active) { |
1035 | list_for_each_entry(event, &ctx->group_list, group_entry) | 1053 | list_for_each_entry(event, &ctx->group_list, group_entry) |
1036 | group_sched_out(event, cpuctx, ctx); | 1054 | group_sched_out(event, cpuctx, ctx); |
1037 | 1055 | } | |
1038 | perf_enable(); | 1056 | perf_enable(); |
1039 | out: | 1057 | out: |
1040 | spin_unlock(&ctx->lock); | 1058 | spin_unlock(&ctx->lock); |
@@ -1059,8 +1077,6 @@ static int context_equiv(struct perf_event_context *ctx1, | |||
1059 | && !ctx1->pin_count && !ctx2->pin_count; | 1077 | && !ctx1->pin_count && !ctx2->pin_count; |
1060 | } | 1078 | } |
1061 | 1079 | ||
1062 | static void __perf_event_read(void *event); | ||
1063 | |||
1064 | static void __perf_event_sync_stat(struct perf_event *event, | 1080 | static void __perf_event_sync_stat(struct perf_event *event, |
1065 | struct perf_event *next_event) | 1081 | struct perf_event *next_event) |
1066 | { | 1082 | { |
@@ -1078,8 +1094,8 @@ static void __perf_event_sync_stat(struct perf_event *event, | |||
1078 | */ | 1094 | */ |
1079 | switch (event->state) { | 1095 | switch (event->state) { |
1080 | case PERF_EVENT_STATE_ACTIVE: | 1096 | case PERF_EVENT_STATE_ACTIVE: |
1081 | __perf_event_read(event); | 1097 | event->pmu->read(event); |
1082 | break; | 1098 | /* fall-through */ |
1083 | 1099 | ||
1084 | case PERF_EVENT_STATE_INACTIVE: | 1100 | case PERF_EVENT_STATE_INACTIVE: |
1085 | update_event_times(event); | 1101 | update_event_times(event); |
@@ -1118,6 +1134,8 @@ static void perf_event_sync_stat(struct perf_event_context *ctx, | |||
1118 | if (!ctx->nr_stat) | 1134 | if (!ctx->nr_stat) |
1119 | return; | 1135 | return; |
1120 | 1136 | ||
1137 | update_context_time(ctx); | ||
1138 | |||
1121 | event = list_first_entry(&ctx->event_list, | 1139 | event = list_first_entry(&ctx->event_list, |
1122 | struct perf_event, event_entry); | 1140 | struct perf_event, event_entry); |
1123 | 1141 | ||
@@ -1161,8 +1179,6 @@ void perf_event_task_sched_out(struct task_struct *task, | |||
1161 | if (likely(!ctx || !cpuctx->task_ctx)) | 1179 | if (likely(!ctx || !cpuctx->task_ctx)) |
1162 | return; | 1180 | return; |
1163 | 1181 | ||
1164 | update_context_time(ctx); | ||
1165 | |||
1166 | rcu_read_lock(); | 1182 | rcu_read_lock(); |
1167 | parent = rcu_dereference(ctx->parent_ctx); | 1183 | parent = rcu_dereference(ctx->parent_ctx); |
1168 | next_ctx = next->perf_event_ctxp; | 1184 | next_ctx = next->perf_event_ctxp; |
@@ -1355,7 +1371,7 @@ static void perf_ctx_adjust_freq(struct perf_event_context *ctx) | |||
1355 | u64 interrupts, freq; | 1371 | u64 interrupts, freq; |
1356 | 1372 | ||
1357 | spin_lock(&ctx->lock); | 1373 | spin_lock(&ctx->lock); |
1358 | list_for_each_entry(event, &ctx->group_list, group_entry) { | 1374 | list_for_each_entry_rcu(event, &ctx->event_list, event_entry) { |
1359 | if (event->state != PERF_EVENT_STATE_ACTIVE) | 1375 | if (event->state != PERF_EVENT_STATE_ACTIVE) |
1360 | continue; | 1376 | continue; |
1361 | 1377 | ||
@@ -1515,7 +1531,6 @@ static void __perf_event_read(void *info) | |||
1515 | struct perf_cpu_context *cpuctx = &__get_cpu_var(perf_cpu_context); | 1531 | struct perf_cpu_context *cpuctx = &__get_cpu_var(perf_cpu_context); |
1516 | struct perf_event *event = info; | 1532 | struct perf_event *event = info; |
1517 | struct perf_event_context *ctx = event->ctx; | 1533 | struct perf_event_context *ctx = event->ctx; |
1518 | unsigned long flags; | ||
1519 | 1534 | ||
1520 | /* | 1535 | /* |
1521 | * If this is a task context, we need to check whether it is | 1536 | * If this is a task context, we need to check whether it is |
@@ -1527,12 +1542,12 @@ static void __perf_event_read(void *info) | |||
1527 | if (ctx->task && cpuctx->task_ctx != ctx) | 1542 | if (ctx->task && cpuctx->task_ctx != ctx) |
1528 | return; | 1543 | return; |
1529 | 1544 | ||
1530 | local_irq_save(flags); | 1545 | spin_lock(&ctx->lock); |
1531 | if (ctx->is_active) | 1546 | update_context_time(ctx); |
1532 | update_context_time(ctx); | ||
1533 | event->pmu->read(event); | ||
1534 | update_event_times(event); | 1547 | update_event_times(event); |
1535 | local_irq_restore(flags); | 1548 | spin_unlock(&ctx->lock); |
1549 | |||
1550 | event->pmu->read(event); | ||
1536 | } | 1551 | } |
1537 | 1552 | ||
1538 | static u64 perf_event_read(struct perf_event *event) | 1553 | static u64 perf_event_read(struct perf_event *event) |
@@ -1545,7 +1560,13 @@ static u64 perf_event_read(struct perf_event *event) | |||
1545 | smp_call_function_single(event->oncpu, | 1560 | smp_call_function_single(event->oncpu, |
1546 | __perf_event_read, event, 1); | 1561 | __perf_event_read, event, 1); |
1547 | } else if (event->state == PERF_EVENT_STATE_INACTIVE) { | 1562 | } else if (event->state == PERF_EVENT_STATE_INACTIVE) { |
1563 | struct perf_event_context *ctx = event->ctx; | ||
1564 | unsigned long flags; | ||
1565 | |||
1566 | spin_lock_irqsave(&ctx->lock, flags); | ||
1567 | update_context_time(ctx); | ||
1548 | update_event_times(event); | 1568 | update_event_times(event); |
1569 | spin_unlock_irqrestore(&ctx->lock, flags); | ||
1549 | } | 1570 | } |
1550 | 1571 | ||
1551 | return atomic64_read(&event->count); | 1572 | return atomic64_read(&event->count); |
@@ -1658,6 +1679,8 @@ static struct perf_event_context *find_get_context(pid_t pid, int cpu) | |||
1658 | return ERR_PTR(err); | 1679 | return ERR_PTR(err); |
1659 | } | 1680 | } |
1660 | 1681 | ||
1682 | static void perf_event_free_filter(struct perf_event *event); | ||
1683 | |||
1661 | static void free_event_rcu(struct rcu_head *head) | 1684 | static void free_event_rcu(struct rcu_head *head) |
1662 | { | 1685 | { |
1663 | struct perf_event *event; | 1686 | struct perf_event *event; |
@@ -1665,6 +1688,7 @@ static void free_event_rcu(struct rcu_head *head) | |||
1665 | event = container_of(head, struct perf_event, rcu_head); | 1688 | event = container_of(head, struct perf_event, rcu_head); |
1666 | if (event->ns) | 1689 | if (event->ns) |
1667 | put_pid_ns(event->ns); | 1690 | put_pid_ns(event->ns); |
1691 | perf_event_free_filter(event); | ||
1668 | kfree(event); | 1692 | kfree(event); |
1669 | } | 1693 | } |
1670 | 1694 | ||
@@ -1696,16 +1720,10 @@ static void free_event(struct perf_event *event) | |||
1696 | call_rcu(&event->rcu_head, free_event_rcu); | 1720 | call_rcu(&event->rcu_head, free_event_rcu); |
1697 | } | 1721 | } |
1698 | 1722 | ||
1699 | /* | 1723 | int perf_event_release_kernel(struct perf_event *event) |
1700 | * Called when the last reference to the file is gone. | ||
1701 | */ | ||
1702 | static int perf_release(struct inode *inode, struct file *file) | ||
1703 | { | 1724 | { |
1704 | struct perf_event *event = file->private_data; | ||
1705 | struct perf_event_context *ctx = event->ctx; | 1725 | struct perf_event_context *ctx = event->ctx; |
1706 | 1726 | ||
1707 | file->private_data = NULL; | ||
1708 | |||
1709 | WARN_ON_ONCE(ctx->parent_ctx); | 1727 | WARN_ON_ONCE(ctx->parent_ctx); |
1710 | mutex_lock(&ctx->mutex); | 1728 | mutex_lock(&ctx->mutex); |
1711 | perf_event_remove_from_context(event); | 1729 | perf_event_remove_from_context(event); |
@@ -1720,6 +1738,19 @@ static int perf_release(struct inode *inode, struct file *file) | |||
1720 | 1738 | ||
1721 | return 0; | 1739 | return 0; |
1722 | } | 1740 | } |
1741 | EXPORT_SYMBOL_GPL(perf_event_release_kernel); | ||
1742 | |||
1743 | /* | ||
1744 | * Called when the last reference to the file is gone. | ||
1745 | */ | ||
1746 | static int perf_release(struct inode *inode, struct file *file) | ||
1747 | { | ||
1748 | struct perf_event *event = file->private_data; | ||
1749 | |||
1750 | file->private_data = NULL; | ||
1751 | |||
1752 | return perf_event_release_kernel(event); | ||
1753 | } | ||
1723 | 1754 | ||
1724 | static int perf_event_read_size(struct perf_event *event) | 1755 | static int perf_event_read_size(struct perf_event *event) |
1725 | { | 1756 | { |
@@ -1746,91 +1777,94 @@ static int perf_event_read_size(struct perf_event *event) | |||
1746 | return size; | 1777 | return size; |
1747 | } | 1778 | } |
1748 | 1779 | ||
1749 | static u64 perf_event_read_value(struct perf_event *event) | 1780 | u64 perf_event_read_value(struct perf_event *event, u64 *enabled, u64 *running) |
1750 | { | 1781 | { |
1751 | struct perf_event *child; | 1782 | struct perf_event *child; |
1752 | u64 total = 0; | 1783 | u64 total = 0; |
1753 | 1784 | ||
1785 | *enabled = 0; | ||
1786 | *running = 0; | ||
1787 | |||
1788 | mutex_lock(&event->child_mutex); | ||
1754 | total += perf_event_read(event); | 1789 | total += perf_event_read(event); |
1755 | list_for_each_entry(child, &event->child_list, child_list) | 1790 | *enabled += event->total_time_enabled + |
1791 | atomic64_read(&event->child_total_time_enabled); | ||
1792 | *running += event->total_time_running + | ||
1793 | atomic64_read(&event->child_total_time_running); | ||
1794 | |||
1795 | list_for_each_entry(child, &event->child_list, child_list) { | ||
1756 | total += perf_event_read(child); | 1796 | total += perf_event_read(child); |
1797 | *enabled += child->total_time_enabled; | ||
1798 | *running += child->total_time_running; | ||
1799 | } | ||
1800 | mutex_unlock(&event->child_mutex); | ||
1757 | 1801 | ||
1758 | return total; | 1802 | return total; |
1759 | } | 1803 | } |
1760 | 1804 | EXPORT_SYMBOL_GPL(perf_event_read_value); | |
1761 | static int perf_event_read_entry(struct perf_event *event, | ||
1762 | u64 read_format, char __user *buf) | ||
1763 | { | ||
1764 | int n = 0, count = 0; | ||
1765 | u64 values[2]; | ||
1766 | |||
1767 | values[n++] = perf_event_read_value(event); | ||
1768 | if (read_format & PERF_FORMAT_ID) | ||
1769 | values[n++] = primary_event_id(event); | ||
1770 | |||
1771 | count = n * sizeof(u64); | ||
1772 | |||
1773 | if (copy_to_user(buf, values, count)) | ||
1774 | return -EFAULT; | ||
1775 | |||
1776 | return count; | ||
1777 | } | ||
1778 | 1805 | ||
1779 | static int perf_event_read_group(struct perf_event *event, | 1806 | static int perf_event_read_group(struct perf_event *event, |
1780 | u64 read_format, char __user *buf) | 1807 | u64 read_format, char __user *buf) |
1781 | { | 1808 | { |
1782 | struct perf_event *leader = event->group_leader, *sub; | 1809 | struct perf_event *leader = event->group_leader, *sub; |
1783 | int n = 0, size = 0, err = -EFAULT; | 1810 | int n = 0, size = 0, ret = -EFAULT; |
1784 | u64 values[3]; | 1811 | struct perf_event_context *ctx = leader->ctx; |
1812 | u64 values[5]; | ||
1813 | u64 count, enabled, running; | ||
1814 | |||
1815 | mutex_lock(&ctx->mutex); | ||
1816 | count = perf_event_read_value(leader, &enabled, &running); | ||
1785 | 1817 | ||
1786 | values[n++] = 1 + leader->nr_siblings; | 1818 | values[n++] = 1 + leader->nr_siblings; |
1787 | if (read_format & PERF_FORMAT_TOTAL_TIME_ENABLED) { | 1819 | if (read_format & PERF_FORMAT_TOTAL_TIME_ENABLED) |
1788 | values[n++] = leader->total_time_enabled + | 1820 | values[n++] = enabled; |
1789 | atomic64_read(&leader->child_total_time_enabled); | 1821 | if (read_format & PERF_FORMAT_TOTAL_TIME_RUNNING) |
1790 | } | 1822 | values[n++] = running; |
1791 | if (read_format & PERF_FORMAT_TOTAL_TIME_RUNNING) { | 1823 | values[n++] = count; |
1792 | values[n++] = leader->total_time_running + | 1824 | if (read_format & PERF_FORMAT_ID) |
1793 | atomic64_read(&leader->child_total_time_running); | 1825 | values[n++] = primary_event_id(leader); |
1794 | } | ||
1795 | 1826 | ||
1796 | size = n * sizeof(u64); | 1827 | size = n * sizeof(u64); |
1797 | 1828 | ||
1798 | if (copy_to_user(buf, values, size)) | 1829 | if (copy_to_user(buf, values, size)) |
1799 | return -EFAULT; | 1830 | goto unlock; |
1800 | |||
1801 | err = perf_event_read_entry(leader, read_format, buf + size); | ||
1802 | if (err < 0) | ||
1803 | return err; | ||
1804 | 1831 | ||
1805 | size += err; | 1832 | ret = size; |
1806 | 1833 | ||
1807 | list_for_each_entry(sub, &leader->sibling_list, group_entry) { | 1834 | list_for_each_entry(sub, &leader->sibling_list, group_entry) { |
1808 | err = perf_event_read_entry(sub, read_format, | 1835 | n = 0; |
1809 | buf + size); | 1836 | |
1810 | if (err < 0) | 1837 | values[n++] = perf_event_read_value(sub, &enabled, &running); |
1811 | return err; | 1838 | if (read_format & PERF_FORMAT_ID) |
1839 | values[n++] = primary_event_id(sub); | ||
1840 | |||
1841 | size = n * sizeof(u64); | ||
1842 | |||
1843 | if (copy_to_user(buf + ret, values, size)) { | ||
1844 | ret = -EFAULT; | ||
1845 | goto unlock; | ||
1846 | } | ||
1812 | 1847 | ||
1813 | size += err; | 1848 | ret += size; |
1814 | } | 1849 | } |
1850 | unlock: | ||
1851 | mutex_unlock(&ctx->mutex); | ||
1815 | 1852 | ||
1816 | return size; | 1853 | return ret; |
1817 | } | 1854 | } |
1818 | 1855 | ||
1819 | static int perf_event_read_one(struct perf_event *event, | 1856 | static int perf_event_read_one(struct perf_event *event, |
1820 | u64 read_format, char __user *buf) | 1857 | u64 read_format, char __user *buf) |
1821 | { | 1858 | { |
1859 | u64 enabled, running; | ||
1822 | u64 values[4]; | 1860 | u64 values[4]; |
1823 | int n = 0; | 1861 | int n = 0; |
1824 | 1862 | ||
1825 | values[n++] = perf_event_read_value(event); | 1863 | values[n++] = perf_event_read_value(event, &enabled, &running); |
1826 | if (read_format & PERF_FORMAT_TOTAL_TIME_ENABLED) { | 1864 | if (read_format & PERF_FORMAT_TOTAL_TIME_ENABLED) |
1827 | values[n++] = event->total_time_enabled + | 1865 | values[n++] = enabled; |
1828 | atomic64_read(&event->child_total_time_enabled); | 1866 | if (read_format & PERF_FORMAT_TOTAL_TIME_RUNNING) |
1829 | } | 1867 | values[n++] = running; |
1830 | if (read_format & PERF_FORMAT_TOTAL_TIME_RUNNING) { | ||
1831 | values[n++] = event->total_time_running + | ||
1832 | atomic64_read(&event->child_total_time_running); | ||
1833 | } | ||
1834 | if (read_format & PERF_FORMAT_ID) | 1868 | if (read_format & PERF_FORMAT_ID) |
1835 | values[n++] = primary_event_id(event); | 1869 | values[n++] = primary_event_id(event); |
1836 | 1870 | ||
@@ -1861,12 +1895,10 @@ perf_read_hw(struct perf_event *event, char __user *buf, size_t count) | |||
1861 | return -ENOSPC; | 1895 | return -ENOSPC; |
1862 | 1896 | ||
1863 | WARN_ON_ONCE(event->ctx->parent_ctx); | 1897 | WARN_ON_ONCE(event->ctx->parent_ctx); |
1864 | mutex_lock(&event->child_mutex); | ||
1865 | if (read_format & PERF_FORMAT_GROUP) | 1898 | if (read_format & PERF_FORMAT_GROUP) |
1866 | ret = perf_event_read_group(event, read_format, buf); | 1899 | ret = perf_event_read_group(event, read_format, buf); |
1867 | else | 1900 | else |
1868 | ret = perf_event_read_one(event, read_format, buf); | 1901 | ret = perf_event_read_one(event, read_format, buf); |
1869 | mutex_unlock(&event->child_mutex); | ||
1870 | 1902 | ||
1871 | return ret; | 1903 | return ret; |
1872 | } | 1904 | } |
@@ -1974,7 +2006,8 @@ unlock: | |||
1974 | return ret; | 2006 | return ret; |
1975 | } | 2007 | } |
1976 | 2008 | ||
1977 | int perf_event_set_output(struct perf_event *event, int output_fd); | 2009 | static int perf_event_set_output(struct perf_event *event, int output_fd); |
2010 | static int perf_event_set_filter(struct perf_event *event, void __user *arg); | ||
1978 | 2011 | ||
1979 | static long perf_ioctl(struct file *file, unsigned int cmd, unsigned long arg) | 2012 | static long perf_ioctl(struct file *file, unsigned int cmd, unsigned long arg) |
1980 | { | 2013 | { |
@@ -2002,6 +2035,9 @@ static long perf_ioctl(struct file *file, unsigned int cmd, unsigned long arg) | |||
2002 | case PERF_EVENT_IOC_SET_OUTPUT: | 2035 | case PERF_EVENT_IOC_SET_OUTPUT: |
2003 | return perf_event_set_output(event, arg); | 2036 | return perf_event_set_output(event, arg); |
2004 | 2037 | ||
2038 | case PERF_EVENT_IOC_SET_FILTER: | ||
2039 | return perf_event_set_filter(event, (void __user *)arg); | ||
2040 | |||
2005 | default: | 2041 | default: |
2006 | return -ENOTTY; | 2042 | return -ENOTTY; |
2007 | } | 2043 | } |
@@ -2174,6 +2210,7 @@ static void perf_mmap_data_free(struct perf_mmap_data *data) | |||
2174 | perf_mmap_free_page((unsigned long)data->user_page); | 2210 | perf_mmap_free_page((unsigned long)data->user_page); |
2175 | for (i = 0; i < data->nr_pages; i++) | 2211 | for (i = 0; i < data->nr_pages; i++) |
2176 | perf_mmap_free_page((unsigned long)data->data_pages[i]); | 2212 | perf_mmap_free_page((unsigned long)data->data_pages[i]); |
2213 | kfree(data); | ||
2177 | } | 2214 | } |
2178 | 2215 | ||
2179 | #else | 2216 | #else |
@@ -2214,6 +2251,7 @@ static void perf_mmap_data_free_work(struct work_struct *work) | |||
2214 | perf_mmap_unmark_page(base + (i * PAGE_SIZE)); | 2251 | perf_mmap_unmark_page(base + (i * PAGE_SIZE)); |
2215 | 2252 | ||
2216 | vfree(base); | 2253 | vfree(base); |
2254 | kfree(data); | ||
2217 | } | 2255 | } |
2218 | 2256 | ||
2219 | static void perf_mmap_data_free(struct perf_mmap_data *data) | 2257 | static void perf_mmap_data_free(struct perf_mmap_data *data) |
@@ -2307,7 +2345,7 @@ perf_mmap_data_init(struct perf_event *event, struct perf_mmap_data *data) | |||
2307 | } | 2345 | } |
2308 | 2346 | ||
2309 | if (!data->watermark) | 2347 | if (!data->watermark) |
2310 | data->watermark = max_t(long, PAGE_SIZE, max_size / 2); | 2348 | data->watermark = max_size / 2; |
2311 | 2349 | ||
2312 | 2350 | ||
2313 | rcu_assign_pointer(event->data, data); | 2351 | rcu_assign_pointer(event->data, data); |
@@ -2319,7 +2357,6 @@ static void perf_mmap_data_free_rcu(struct rcu_head *rcu_head) | |||
2319 | 2357 | ||
2320 | data = container_of(rcu_head, struct perf_mmap_data, rcu_head); | 2358 | data = container_of(rcu_head, struct perf_mmap_data, rcu_head); |
2321 | perf_mmap_data_free(data); | 2359 | perf_mmap_data_free(data); |
2322 | kfree(data); | ||
2323 | } | 2360 | } |
2324 | 2361 | ||
2325 | static void perf_mmap_data_release(struct perf_event *event) | 2362 | static void perf_mmap_data_release(struct perf_event *event) |
@@ -2666,20 +2703,21 @@ static void perf_output_wakeup(struct perf_output_handle *handle) | |||
2666 | static void perf_output_lock(struct perf_output_handle *handle) | 2703 | static void perf_output_lock(struct perf_output_handle *handle) |
2667 | { | 2704 | { |
2668 | struct perf_mmap_data *data = handle->data; | 2705 | struct perf_mmap_data *data = handle->data; |
2669 | int cpu; | 2706 | int cur, cpu = get_cpu(); |
2670 | 2707 | ||
2671 | handle->locked = 0; | 2708 | handle->locked = 0; |
2672 | 2709 | ||
2673 | local_irq_save(handle->flags); | 2710 | for (;;) { |
2674 | cpu = smp_processor_id(); | 2711 | cur = atomic_cmpxchg(&data->lock, -1, cpu); |
2675 | 2712 | if (cur == -1) { | |
2676 | if (in_nmi() && atomic_read(&data->lock) == cpu) | 2713 | handle->locked = 1; |
2677 | return; | 2714 | break; |
2715 | } | ||
2716 | if (cur == cpu) | ||
2717 | break; | ||
2678 | 2718 | ||
2679 | while (atomic_cmpxchg(&data->lock, -1, cpu) != -1) | ||
2680 | cpu_relax(); | 2719 | cpu_relax(); |
2681 | 2720 | } | |
2682 | handle->locked = 1; | ||
2683 | } | 2721 | } |
2684 | 2722 | ||
2685 | static void perf_output_unlock(struct perf_output_handle *handle) | 2723 | static void perf_output_unlock(struct perf_output_handle *handle) |
@@ -2725,7 +2763,7 @@ again: | |||
2725 | if (atomic_xchg(&data->wakeup, 0)) | 2763 | if (atomic_xchg(&data->wakeup, 0)) |
2726 | perf_output_wakeup(handle); | 2764 | perf_output_wakeup(handle); |
2727 | out: | 2765 | out: |
2728 | local_irq_restore(handle->flags); | 2766 | put_cpu(); |
2729 | } | 2767 | } |
2730 | 2768 | ||
2731 | void perf_output_copy(struct perf_output_handle *handle, | 2769 | void perf_output_copy(struct perf_output_handle *handle, |
@@ -3236,15 +3274,10 @@ static void perf_event_task_ctx(struct perf_event_context *ctx, | |||
3236 | { | 3274 | { |
3237 | struct perf_event *event; | 3275 | struct perf_event *event; |
3238 | 3276 | ||
3239 | if (system_state != SYSTEM_RUNNING || list_empty(&ctx->event_list)) | ||
3240 | return; | ||
3241 | |||
3242 | rcu_read_lock(); | ||
3243 | list_for_each_entry_rcu(event, &ctx->event_list, event_entry) { | 3277 | list_for_each_entry_rcu(event, &ctx->event_list, event_entry) { |
3244 | if (perf_event_task_match(event)) | 3278 | if (perf_event_task_match(event)) |
3245 | perf_event_task_output(event, task_event); | 3279 | perf_event_task_output(event, task_event); |
3246 | } | 3280 | } |
3247 | rcu_read_unlock(); | ||
3248 | } | 3281 | } |
3249 | 3282 | ||
3250 | static void perf_event_task_event(struct perf_task_event *task_event) | 3283 | static void perf_event_task_event(struct perf_task_event *task_event) |
@@ -3252,11 +3285,11 @@ static void perf_event_task_event(struct perf_task_event *task_event) | |||
3252 | struct perf_cpu_context *cpuctx; | 3285 | struct perf_cpu_context *cpuctx; |
3253 | struct perf_event_context *ctx = task_event->task_ctx; | 3286 | struct perf_event_context *ctx = task_event->task_ctx; |
3254 | 3287 | ||
3288 | rcu_read_lock(); | ||
3255 | cpuctx = &get_cpu_var(perf_cpu_context); | 3289 | cpuctx = &get_cpu_var(perf_cpu_context); |
3256 | perf_event_task_ctx(&cpuctx->ctx, task_event); | 3290 | perf_event_task_ctx(&cpuctx->ctx, task_event); |
3257 | put_cpu_var(perf_cpu_context); | 3291 | put_cpu_var(perf_cpu_context); |
3258 | 3292 | ||
3259 | rcu_read_lock(); | ||
3260 | if (!ctx) | 3293 | if (!ctx) |
3261 | ctx = rcu_dereference(task_event->task->perf_event_ctxp); | 3294 | ctx = rcu_dereference(task_event->task->perf_event_ctxp); |
3262 | if (ctx) | 3295 | if (ctx) |
@@ -3348,15 +3381,10 @@ static void perf_event_comm_ctx(struct perf_event_context *ctx, | |||
3348 | { | 3381 | { |
3349 | struct perf_event *event; | 3382 | struct perf_event *event; |
3350 | 3383 | ||
3351 | if (system_state != SYSTEM_RUNNING || list_empty(&ctx->event_list)) | ||
3352 | return; | ||
3353 | |||
3354 | rcu_read_lock(); | ||
3355 | list_for_each_entry_rcu(event, &ctx->event_list, event_entry) { | 3384 | list_for_each_entry_rcu(event, &ctx->event_list, event_entry) { |
3356 | if (perf_event_comm_match(event)) | 3385 | if (perf_event_comm_match(event)) |
3357 | perf_event_comm_output(event, comm_event); | 3386 | perf_event_comm_output(event, comm_event); |
3358 | } | 3387 | } |
3359 | rcu_read_unlock(); | ||
3360 | } | 3388 | } |
3361 | 3389 | ||
3362 | static void perf_event_comm_event(struct perf_comm_event *comm_event) | 3390 | static void perf_event_comm_event(struct perf_comm_event *comm_event) |
@@ -3367,7 +3395,7 @@ static void perf_event_comm_event(struct perf_comm_event *comm_event) | |||
3367 | char comm[TASK_COMM_LEN]; | 3395 | char comm[TASK_COMM_LEN]; |
3368 | 3396 | ||
3369 | memset(comm, 0, sizeof(comm)); | 3397 | memset(comm, 0, sizeof(comm)); |
3370 | strncpy(comm, comm_event->task->comm, sizeof(comm)); | 3398 | strlcpy(comm, comm_event->task->comm, sizeof(comm)); |
3371 | size = ALIGN(strlen(comm)+1, sizeof(u64)); | 3399 | size = ALIGN(strlen(comm)+1, sizeof(u64)); |
3372 | 3400 | ||
3373 | comm_event->comm = comm; | 3401 | comm_event->comm = comm; |
@@ -3375,11 +3403,11 @@ static void perf_event_comm_event(struct perf_comm_event *comm_event) | |||
3375 | 3403 | ||
3376 | comm_event->event_id.header.size = sizeof(comm_event->event_id) + size; | 3404 | comm_event->event_id.header.size = sizeof(comm_event->event_id) + size; |
3377 | 3405 | ||
3406 | rcu_read_lock(); | ||
3378 | cpuctx = &get_cpu_var(perf_cpu_context); | 3407 | cpuctx = &get_cpu_var(perf_cpu_context); |
3379 | perf_event_comm_ctx(&cpuctx->ctx, comm_event); | 3408 | perf_event_comm_ctx(&cpuctx->ctx, comm_event); |
3380 | put_cpu_var(perf_cpu_context); | 3409 | put_cpu_var(perf_cpu_context); |
3381 | 3410 | ||
3382 | rcu_read_lock(); | ||
3383 | /* | 3411 | /* |
3384 | * doesn't really matter which of the child contexts the | 3412 | * doesn't really matter which of the child contexts the |
3385 | * events ends up in. | 3413 | * events ends up in. |
@@ -3472,15 +3500,10 @@ static void perf_event_mmap_ctx(struct perf_event_context *ctx, | |||
3472 | { | 3500 | { |
3473 | struct perf_event *event; | 3501 | struct perf_event *event; |
3474 | 3502 | ||
3475 | if (system_state != SYSTEM_RUNNING || list_empty(&ctx->event_list)) | ||
3476 | return; | ||
3477 | |||
3478 | rcu_read_lock(); | ||
3479 | list_for_each_entry_rcu(event, &ctx->event_list, event_entry) { | 3503 | list_for_each_entry_rcu(event, &ctx->event_list, event_entry) { |
3480 | if (perf_event_mmap_match(event, mmap_event)) | 3504 | if (perf_event_mmap_match(event, mmap_event)) |
3481 | perf_event_mmap_output(event, mmap_event); | 3505 | perf_event_mmap_output(event, mmap_event); |
3482 | } | 3506 | } |
3483 | rcu_read_unlock(); | ||
3484 | } | 3507 | } |
3485 | 3508 | ||
3486 | static void perf_event_mmap_event(struct perf_mmap_event *mmap_event) | 3509 | static void perf_event_mmap_event(struct perf_mmap_event *mmap_event) |
@@ -3536,11 +3559,11 @@ got_name: | |||
3536 | 3559 | ||
3537 | mmap_event->event_id.header.size = sizeof(mmap_event->event_id) + size; | 3560 | mmap_event->event_id.header.size = sizeof(mmap_event->event_id) + size; |
3538 | 3561 | ||
3562 | rcu_read_lock(); | ||
3539 | cpuctx = &get_cpu_var(perf_cpu_context); | 3563 | cpuctx = &get_cpu_var(perf_cpu_context); |
3540 | perf_event_mmap_ctx(&cpuctx->ctx, mmap_event); | 3564 | perf_event_mmap_ctx(&cpuctx->ctx, mmap_event); |
3541 | put_cpu_var(perf_cpu_context); | 3565 | put_cpu_var(perf_cpu_context); |
3542 | 3566 | ||
3543 | rcu_read_lock(); | ||
3544 | /* | 3567 | /* |
3545 | * doesn't really matter which of the child contexts the | 3568 | * doesn't really matter which of the child contexts the |
3546 | * events ends up in. | 3569 | * events ends up in. |
@@ -3679,7 +3702,11 @@ static int __perf_event_overflow(struct perf_event *event, int nmi, | |||
3679 | perf_event_disable(event); | 3702 | perf_event_disable(event); |
3680 | } | 3703 | } |
3681 | 3704 | ||
3682 | perf_event_output(event, nmi, data, regs); | 3705 | if (event->overflow_handler) |
3706 | event->overflow_handler(event, nmi, data, regs); | ||
3707 | else | ||
3708 | perf_event_output(event, nmi, data, regs); | ||
3709 | |||
3683 | return ret; | 3710 | return ret; |
3684 | } | 3711 | } |
3685 | 3712 | ||
@@ -3724,16 +3751,16 @@ again: | |||
3724 | return nr; | 3751 | return nr; |
3725 | } | 3752 | } |
3726 | 3753 | ||
3727 | static void perf_swevent_overflow(struct perf_event *event, | 3754 | static void perf_swevent_overflow(struct perf_event *event, u64 overflow, |
3728 | int nmi, struct perf_sample_data *data, | 3755 | int nmi, struct perf_sample_data *data, |
3729 | struct pt_regs *regs) | 3756 | struct pt_regs *regs) |
3730 | { | 3757 | { |
3731 | struct hw_perf_event *hwc = &event->hw; | 3758 | struct hw_perf_event *hwc = &event->hw; |
3732 | int throttle = 0; | 3759 | int throttle = 0; |
3733 | u64 overflow; | ||
3734 | 3760 | ||
3735 | data->period = event->hw.last_period; | 3761 | data->period = event->hw.last_period; |
3736 | overflow = perf_swevent_set_period(event); | 3762 | if (!overflow) |
3763 | overflow = perf_swevent_set_period(event); | ||
3737 | 3764 | ||
3738 | if (hwc->interrupts == MAX_INTERRUPTS) | 3765 | if (hwc->interrupts == MAX_INTERRUPTS) |
3739 | return; | 3766 | return; |
@@ -3766,14 +3793,19 @@ static void perf_swevent_add(struct perf_event *event, u64 nr, | |||
3766 | 3793 | ||
3767 | atomic64_add(nr, &event->count); | 3794 | atomic64_add(nr, &event->count); |
3768 | 3795 | ||
3796 | if (!regs) | ||
3797 | return; | ||
3798 | |||
3769 | if (!hwc->sample_period) | 3799 | if (!hwc->sample_period) |
3770 | return; | 3800 | return; |
3771 | 3801 | ||
3772 | if (!regs) | 3802 | if (nr == 1 && hwc->sample_period == 1 && !event->attr.freq) |
3803 | return perf_swevent_overflow(event, 1, nmi, data, regs); | ||
3804 | |||
3805 | if (atomic64_add_negative(nr, &hwc->period_left)) | ||
3773 | return; | 3806 | return; |
3774 | 3807 | ||
3775 | if (!atomic64_add_negative(nr, &hwc->period_left)) | 3808 | perf_swevent_overflow(event, 0, nmi, data, regs); |
3776 | perf_swevent_overflow(event, nmi, data, regs); | ||
3777 | } | 3809 | } |
3778 | 3810 | ||
3779 | static int perf_swevent_is_counting(struct perf_event *event) | 3811 | static int perf_swevent_is_counting(struct perf_event *event) |
@@ -3806,25 +3838,44 @@ static int perf_swevent_is_counting(struct perf_event *event) | |||
3806 | return 1; | 3838 | return 1; |
3807 | } | 3839 | } |
3808 | 3840 | ||
3841 | static int perf_tp_event_match(struct perf_event *event, | ||
3842 | struct perf_sample_data *data); | ||
3843 | |||
3844 | static int perf_exclude_event(struct perf_event *event, | ||
3845 | struct pt_regs *regs) | ||
3846 | { | ||
3847 | if (regs) { | ||
3848 | if (event->attr.exclude_user && user_mode(regs)) | ||
3849 | return 1; | ||
3850 | |||
3851 | if (event->attr.exclude_kernel && !user_mode(regs)) | ||
3852 | return 1; | ||
3853 | } | ||
3854 | |||
3855 | return 0; | ||
3856 | } | ||
3857 | |||
3809 | static int perf_swevent_match(struct perf_event *event, | 3858 | static int perf_swevent_match(struct perf_event *event, |
3810 | enum perf_type_id type, | 3859 | enum perf_type_id type, |
3811 | u32 event_id, struct pt_regs *regs) | 3860 | u32 event_id, |
3861 | struct perf_sample_data *data, | ||
3862 | struct pt_regs *regs) | ||
3812 | { | 3863 | { |
3813 | if (!perf_swevent_is_counting(event)) | 3864 | if (!perf_swevent_is_counting(event)) |
3814 | return 0; | 3865 | return 0; |
3815 | 3866 | ||
3816 | if (event->attr.type != type) | 3867 | if (event->attr.type != type) |
3817 | return 0; | 3868 | return 0; |
3869 | |||
3818 | if (event->attr.config != event_id) | 3870 | if (event->attr.config != event_id) |
3819 | return 0; | 3871 | return 0; |
3820 | 3872 | ||
3821 | if (regs) { | 3873 | if (perf_exclude_event(event, regs)) |
3822 | if (event->attr.exclude_user && user_mode(regs)) | 3874 | return 0; |
3823 | return 0; | ||
3824 | 3875 | ||
3825 | if (event->attr.exclude_kernel && !user_mode(regs)) | 3876 | if (event->attr.type == PERF_TYPE_TRACEPOINT && |
3826 | return 0; | 3877 | !perf_tp_event_match(event, data)) |
3827 | } | 3878 | return 0; |
3828 | 3879 | ||
3829 | return 1; | 3880 | return 1; |
3830 | } | 3881 | } |
@@ -3837,49 +3888,59 @@ static void perf_swevent_ctx_event(struct perf_event_context *ctx, | |||
3837 | { | 3888 | { |
3838 | struct perf_event *event; | 3889 | struct perf_event *event; |
3839 | 3890 | ||
3840 | if (system_state != SYSTEM_RUNNING || list_empty(&ctx->event_list)) | ||
3841 | return; | ||
3842 | |||
3843 | rcu_read_lock(); | ||
3844 | list_for_each_entry_rcu(event, &ctx->event_list, event_entry) { | 3891 | list_for_each_entry_rcu(event, &ctx->event_list, event_entry) { |
3845 | if (perf_swevent_match(event, type, event_id, regs)) | 3892 | if (perf_swevent_match(event, type, event_id, data, regs)) |
3846 | perf_swevent_add(event, nr, nmi, data, regs); | 3893 | perf_swevent_add(event, nr, nmi, data, regs); |
3847 | } | 3894 | } |
3848 | rcu_read_unlock(); | ||
3849 | } | 3895 | } |
3850 | 3896 | ||
3851 | static int *perf_swevent_recursion_context(struct perf_cpu_context *cpuctx) | 3897 | int perf_swevent_get_recursion_context(void) |
3852 | { | 3898 | { |
3899 | struct perf_cpu_context *cpuctx = &get_cpu_var(perf_cpu_context); | ||
3900 | int rctx; | ||
3901 | |||
3853 | if (in_nmi()) | 3902 | if (in_nmi()) |
3854 | return &cpuctx->recursion[3]; | 3903 | rctx = 3; |
3904 | else if (in_irq()) | ||
3905 | rctx = 2; | ||
3906 | else if (in_softirq()) | ||
3907 | rctx = 1; | ||
3908 | else | ||
3909 | rctx = 0; | ||
3855 | 3910 | ||
3856 | if (in_irq()) | 3911 | if (cpuctx->recursion[rctx]) { |
3857 | return &cpuctx->recursion[2]; | 3912 | put_cpu_var(perf_cpu_context); |
3913 | return -1; | ||
3914 | } | ||
3858 | 3915 | ||
3859 | if (in_softirq()) | 3916 | cpuctx->recursion[rctx]++; |
3860 | return &cpuctx->recursion[1]; | 3917 | barrier(); |
3861 | 3918 | ||
3862 | return &cpuctx->recursion[0]; | 3919 | return rctx; |
3920 | } | ||
3921 | EXPORT_SYMBOL_GPL(perf_swevent_get_recursion_context); | ||
3922 | |||
3923 | void perf_swevent_put_recursion_context(int rctx) | ||
3924 | { | ||
3925 | struct perf_cpu_context *cpuctx = &__get_cpu_var(perf_cpu_context); | ||
3926 | barrier(); | ||
3927 | cpuctx->recursion[rctx]--; | ||
3928 | put_cpu_var(perf_cpu_context); | ||
3863 | } | 3929 | } |
3930 | EXPORT_SYMBOL_GPL(perf_swevent_put_recursion_context); | ||
3864 | 3931 | ||
3865 | static void do_perf_sw_event(enum perf_type_id type, u32 event_id, | 3932 | static void do_perf_sw_event(enum perf_type_id type, u32 event_id, |
3866 | u64 nr, int nmi, | 3933 | u64 nr, int nmi, |
3867 | struct perf_sample_data *data, | 3934 | struct perf_sample_data *data, |
3868 | struct pt_regs *regs) | 3935 | struct pt_regs *regs) |
3869 | { | 3936 | { |
3870 | struct perf_cpu_context *cpuctx = &get_cpu_var(perf_cpu_context); | 3937 | struct perf_cpu_context *cpuctx; |
3871 | int *recursion = perf_swevent_recursion_context(cpuctx); | ||
3872 | struct perf_event_context *ctx; | 3938 | struct perf_event_context *ctx; |
3873 | 3939 | ||
3874 | if (*recursion) | 3940 | cpuctx = &__get_cpu_var(perf_cpu_context); |
3875 | goto out; | 3941 | rcu_read_lock(); |
3876 | |||
3877 | (*recursion)++; | ||
3878 | barrier(); | ||
3879 | |||
3880 | perf_swevent_ctx_event(&cpuctx->ctx, type, event_id, | 3942 | perf_swevent_ctx_event(&cpuctx->ctx, type, event_id, |
3881 | nr, nmi, data, regs); | 3943 | nr, nmi, data, regs); |
3882 | rcu_read_lock(); | ||
3883 | /* | 3944 | /* |
3884 | * doesn't really matter which of the child contexts the | 3945 | * doesn't really matter which of the child contexts the |
3885 | * events ends up in. | 3946 | * events ends up in. |
@@ -3888,23 +3949,24 @@ static void do_perf_sw_event(enum perf_type_id type, u32 event_id, | |||
3888 | if (ctx) | 3949 | if (ctx) |
3889 | perf_swevent_ctx_event(ctx, type, event_id, nr, nmi, data, regs); | 3950 | perf_swevent_ctx_event(ctx, type, event_id, nr, nmi, data, regs); |
3890 | rcu_read_unlock(); | 3951 | rcu_read_unlock(); |
3891 | |||
3892 | barrier(); | ||
3893 | (*recursion)--; | ||
3894 | |||
3895 | out: | ||
3896 | put_cpu_var(perf_cpu_context); | ||
3897 | } | 3952 | } |
3898 | 3953 | ||
3899 | void __perf_sw_event(u32 event_id, u64 nr, int nmi, | 3954 | void __perf_sw_event(u32 event_id, u64 nr, int nmi, |
3900 | struct pt_regs *regs, u64 addr) | 3955 | struct pt_regs *regs, u64 addr) |
3901 | { | 3956 | { |
3902 | struct perf_sample_data data = { | 3957 | struct perf_sample_data data; |
3903 | .addr = addr, | 3958 | int rctx; |
3904 | }; | ||
3905 | 3959 | ||
3906 | do_perf_sw_event(PERF_TYPE_SOFTWARE, event_id, nr, nmi, | 3960 | rctx = perf_swevent_get_recursion_context(); |
3907 | &data, regs); | 3961 | if (rctx < 0) |
3962 | return; | ||
3963 | |||
3964 | data.addr = addr; | ||
3965 | data.raw = NULL; | ||
3966 | |||
3967 | do_perf_sw_event(PERF_TYPE_SOFTWARE, event_id, nr, nmi, &data, regs); | ||
3968 | |||
3969 | perf_swevent_put_recursion_context(rctx); | ||
3908 | } | 3970 | } |
3909 | 3971 | ||
3910 | static void perf_swevent_read(struct perf_event *event) | 3972 | static void perf_swevent_read(struct perf_event *event) |
@@ -3949,6 +4011,7 @@ static enum hrtimer_restart perf_swevent_hrtimer(struct hrtimer *hrtimer) | |||
3949 | event->pmu->read(event); | 4011 | event->pmu->read(event); |
3950 | 4012 | ||
3951 | data.addr = 0; | 4013 | data.addr = 0; |
4014 | data.period = event->hw.last_period; | ||
3952 | regs = get_irq_regs(); | 4015 | regs = get_irq_regs(); |
3953 | /* | 4016 | /* |
3954 | * In case we exclude kernel IPs or are somehow not in interrupt | 4017 | * In case we exclude kernel IPs or are somehow not in interrupt |
@@ -3959,8 +4022,9 @@ static enum hrtimer_restart perf_swevent_hrtimer(struct hrtimer *hrtimer) | |||
3959 | regs = task_pt_regs(current); | 4022 | regs = task_pt_regs(current); |
3960 | 4023 | ||
3961 | if (regs) { | 4024 | if (regs) { |
3962 | if (perf_event_overflow(event, 0, &data, regs)) | 4025 | if (!(event->attr.exclude_idle && current->pid == 0)) |
3963 | ret = HRTIMER_NORESTART; | 4026 | if (perf_event_overflow(event, 0, &data, regs)) |
4027 | ret = HRTIMER_NORESTART; | ||
3964 | } | 4028 | } |
3965 | 4029 | ||
3966 | period = max_t(u64, 10000, event->hw.sample_period); | 4030 | period = max_t(u64, 10000, event->hw.sample_period); |
@@ -3969,6 +4033,42 @@ static enum hrtimer_restart perf_swevent_hrtimer(struct hrtimer *hrtimer) | |||
3969 | return ret; | 4033 | return ret; |
3970 | } | 4034 | } |
3971 | 4035 | ||
4036 | static void perf_swevent_start_hrtimer(struct perf_event *event) | ||
4037 | { | ||
4038 | struct hw_perf_event *hwc = &event->hw; | ||
4039 | |||
4040 | hrtimer_init(&hwc->hrtimer, CLOCK_MONOTONIC, HRTIMER_MODE_REL); | ||
4041 | hwc->hrtimer.function = perf_swevent_hrtimer; | ||
4042 | if (hwc->sample_period) { | ||
4043 | u64 period; | ||
4044 | |||
4045 | if (hwc->remaining) { | ||
4046 | if (hwc->remaining < 0) | ||
4047 | period = 10000; | ||
4048 | else | ||
4049 | period = hwc->remaining; | ||
4050 | hwc->remaining = 0; | ||
4051 | } else { | ||
4052 | period = max_t(u64, 10000, hwc->sample_period); | ||
4053 | } | ||
4054 | __hrtimer_start_range_ns(&hwc->hrtimer, | ||
4055 | ns_to_ktime(period), 0, | ||
4056 | HRTIMER_MODE_REL, 0); | ||
4057 | } | ||
4058 | } | ||
4059 | |||
4060 | static void perf_swevent_cancel_hrtimer(struct perf_event *event) | ||
4061 | { | ||
4062 | struct hw_perf_event *hwc = &event->hw; | ||
4063 | |||
4064 | if (hwc->sample_period) { | ||
4065 | ktime_t remaining = hrtimer_get_remaining(&hwc->hrtimer); | ||
4066 | hwc->remaining = ktime_to_ns(remaining); | ||
4067 | |||
4068 | hrtimer_cancel(&hwc->hrtimer); | ||
4069 | } | ||
4070 | } | ||
4071 | |||
3972 | /* | 4072 | /* |
3973 | * Software event: cpu wall time clock | 4073 | * Software event: cpu wall time clock |
3974 | */ | 4074 | */ |
@@ -3991,22 +4091,14 @@ static int cpu_clock_perf_event_enable(struct perf_event *event) | |||
3991 | int cpu = raw_smp_processor_id(); | 4091 | int cpu = raw_smp_processor_id(); |
3992 | 4092 | ||
3993 | atomic64_set(&hwc->prev_count, cpu_clock(cpu)); | 4093 | atomic64_set(&hwc->prev_count, cpu_clock(cpu)); |
3994 | hrtimer_init(&hwc->hrtimer, CLOCK_MONOTONIC, HRTIMER_MODE_REL); | 4094 | perf_swevent_start_hrtimer(event); |
3995 | hwc->hrtimer.function = perf_swevent_hrtimer; | ||
3996 | if (hwc->sample_period) { | ||
3997 | u64 period = max_t(u64, 10000, hwc->sample_period); | ||
3998 | __hrtimer_start_range_ns(&hwc->hrtimer, | ||
3999 | ns_to_ktime(period), 0, | ||
4000 | HRTIMER_MODE_REL, 0); | ||
4001 | } | ||
4002 | 4095 | ||
4003 | return 0; | 4096 | return 0; |
4004 | } | 4097 | } |
4005 | 4098 | ||
4006 | static void cpu_clock_perf_event_disable(struct perf_event *event) | 4099 | static void cpu_clock_perf_event_disable(struct perf_event *event) |
4007 | { | 4100 | { |
4008 | if (event->hw.sample_period) | 4101 | perf_swevent_cancel_hrtimer(event); |
4009 | hrtimer_cancel(&event->hw.hrtimer); | ||
4010 | cpu_clock_perf_event_update(event); | 4102 | cpu_clock_perf_event_update(event); |
4011 | } | 4103 | } |
4012 | 4104 | ||
@@ -4043,22 +4135,15 @@ static int task_clock_perf_event_enable(struct perf_event *event) | |||
4043 | now = event->ctx->time; | 4135 | now = event->ctx->time; |
4044 | 4136 | ||
4045 | atomic64_set(&hwc->prev_count, now); | 4137 | atomic64_set(&hwc->prev_count, now); |
4046 | hrtimer_init(&hwc->hrtimer, CLOCK_MONOTONIC, HRTIMER_MODE_REL); | 4138 | |
4047 | hwc->hrtimer.function = perf_swevent_hrtimer; | 4139 | perf_swevent_start_hrtimer(event); |
4048 | if (hwc->sample_period) { | ||
4049 | u64 period = max_t(u64, 10000, hwc->sample_period); | ||
4050 | __hrtimer_start_range_ns(&hwc->hrtimer, | ||
4051 | ns_to_ktime(period), 0, | ||
4052 | HRTIMER_MODE_REL, 0); | ||
4053 | } | ||
4054 | 4140 | ||
4055 | return 0; | 4141 | return 0; |
4056 | } | 4142 | } |
4057 | 4143 | ||
4058 | static void task_clock_perf_event_disable(struct perf_event *event) | 4144 | static void task_clock_perf_event_disable(struct perf_event *event) |
4059 | { | 4145 | { |
4060 | if (event->hw.sample_period) | 4146 | perf_swevent_cancel_hrtimer(event); |
4061 | hrtimer_cancel(&event->hw.hrtimer); | ||
4062 | task_clock_perf_event_update(event, event->ctx->time); | 4147 | task_clock_perf_event_update(event, event->ctx->time); |
4063 | 4148 | ||
4064 | } | 4149 | } |
@@ -4086,6 +4171,7 @@ static const struct pmu perf_ops_task_clock = { | |||
4086 | }; | 4171 | }; |
4087 | 4172 | ||
4088 | #ifdef CONFIG_EVENT_PROFILE | 4173 | #ifdef CONFIG_EVENT_PROFILE |
4174 | |||
4089 | void perf_tp_event(int event_id, u64 addr, u64 count, void *record, | 4175 | void perf_tp_event(int event_id, u64 addr, u64 count, void *record, |
4090 | int entry_size) | 4176 | int entry_size) |
4091 | { | 4177 | { |
@@ -4104,13 +4190,21 @@ void perf_tp_event(int event_id, u64 addr, u64 count, void *record, | |||
4104 | if (!regs) | 4190 | if (!regs) |
4105 | regs = task_pt_regs(current); | 4191 | regs = task_pt_regs(current); |
4106 | 4192 | ||
4193 | /* Trace events already protected against recursion */ | ||
4107 | do_perf_sw_event(PERF_TYPE_TRACEPOINT, event_id, count, 1, | 4194 | do_perf_sw_event(PERF_TYPE_TRACEPOINT, event_id, count, 1, |
4108 | &data, regs); | 4195 | &data, regs); |
4109 | } | 4196 | } |
4110 | EXPORT_SYMBOL_GPL(perf_tp_event); | 4197 | EXPORT_SYMBOL_GPL(perf_tp_event); |
4111 | 4198 | ||
4112 | extern int ftrace_profile_enable(int); | 4199 | static int perf_tp_event_match(struct perf_event *event, |
4113 | extern void ftrace_profile_disable(int); | 4200 | struct perf_sample_data *data) |
4201 | { | ||
4202 | void *record = data->raw->data; | ||
4203 | |||
4204 | if (likely(!event->filter) || filter_match_preds(event->filter, record)) | ||
4205 | return 1; | ||
4206 | return 0; | ||
4207 | } | ||
4114 | 4208 | ||
4115 | static void tp_perf_event_destroy(struct perf_event *event) | 4209 | static void tp_perf_event_destroy(struct perf_event *event) |
4116 | { | 4210 | { |
@@ -4135,11 +4229,99 @@ static const struct pmu *tp_perf_event_init(struct perf_event *event) | |||
4135 | 4229 | ||
4136 | return &perf_ops_generic; | 4230 | return &perf_ops_generic; |
4137 | } | 4231 | } |
4232 | |||
4233 | static int perf_event_set_filter(struct perf_event *event, void __user *arg) | ||
4234 | { | ||
4235 | char *filter_str; | ||
4236 | int ret; | ||
4237 | |||
4238 | if (event->attr.type != PERF_TYPE_TRACEPOINT) | ||
4239 | return -EINVAL; | ||
4240 | |||
4241 | filter_str = strndup_user(arg, PAGE_SIZE); | ||
4242 | if (IS_ERR(filter_str)) | ||
4243 | return PTR_ERR(filter_str); | ||
4244 | |||
4245 | ret = ftrace_profile_set_filter(event, event->attr.config, filter_str); | ||
4246 | |||
4247 | kfree(filter_str); | ||
4248 | return ret; | ||
4249 | } | ||
4250 | |||
4251 | static void perf_event_free_filter(struct perf_event *event) | ||
4252 | { | ||
4253 | ftrace_profile_free_filter(event); | ||
4254 | } | ||
4255 | |||
4138 | #else | 4256 | #else |
4257 | |||
4258 | static int perf_tp_event_match(struct perf_event *event, | ||
4259 | struct perf_sample_data *data) | ||
4260 | { | ||
4261 | return 1; | ||
4262 | } | ||
4263 | |||
4139 | static const struct pmu *tp_perf_event_init(struct perf_event *event) | 4264 | static const struct pmu *tp_perf_event_init(struct perf_event *event) |
4140 | { | 4265 | { |
4141 | return NULL; | 4266 | return NULL; |
4142 | } | 4267 | } |
4268 | |||
4269 | static int perf_event_set_filter(struct perf_event *event, void __user *arg) | ||
4270 | { | ||
4271 | return -ENOENT; | ||
4272 | } | ||
4273 | |||
4274 | static void perf_event_free_filter(struct perf_event *event) | ||
4275 | { | ||
4276 | } | ||
4277 | |||
4278 | #endif /* CONFIG_EVENT_PROFILE */ | ||
4279 | |||
4280 | #ifdef CONFIG_HAVE_HW_BREAKPOINT | ||
4281 | static void bp_perf_event_destroy(struct perf_event *event) | ||
4282 | { | ||
4283 | release_bp_slot(event); | ||
4284 | } | ||
4285 | |||
4286 | static const struct pmu *bp_perf_event_init(struct perf_event *bp) | ||
4287 | { | ||
4288 | int err; | ||
4289 | /* | ||
4290 | * The breakpoint is already filled if we haven't created the counter | ||
4291 | * through perf syscall | ||
4292 | * FIXME: manage to get trigerred to NULL if it comes from syscalls | ||
4293 | */ | ||
4294 | if (!bp->callback) | ||
4295 | err = register_perf_hw_breakpoint(bp); | ||
4296 | else | ||
4297 | err = __register_perf_hw_breakpoint(bp); | ||
4298 | if (err) | ||
4299 | return ERR_PTR(err); | ||
4300 | |||
4301 | bp->destroy = bp_perf_event_destroy; | ||
4302 | |||
4303 | return &perf_ops_bp; | ||
4304 | } | ||
4305 | |||
4306 | void perf_bp_event(struct perf_event *bp, void *data) | ||
4307 | { | ||
4308 | struct perf_sample_data sample; | ||
4309 | struct pt_regs *regs = data; | ||
4310 | |||
4311 | sample.addr = bp->attr.bp_addr; | ||
4312 | |||
4313 | if (!perf_exclude_event(bp, regs)) | ||
4314 | perf_swevent_add(bp, 1, 1, &sample, regs); | ||
4315 | } | ||
4316 | #else | ||
4317 | static const struct pmu *bp_perf_event_init(struct perf_event *bp) | ||
4318 | { | ||
4319 | return NULL; | ||
4320 | } | ||
4321 | |||
4322 | void perf_bp_event(struct perf_event *bp, void *regs) | ||
4323 | { | ||
4324 | } | ||
4143 | #endif | 4325 | #endif |
4144 | 4326 | ||
4145 | atomic_t perf_swevent_enabled[PERF_COUNT_SW_MAX]; | 4327 | atomic_t perf_swevent_enabled[PERF_COUNT_SW_MAX]; |
@@ -4186,6 +4368,8 @@ static const struct pmu *sw_perf_event_init(struct perf_event *event) | |||
4186 | case PERF_COUNT_SW_PAGE_FAULTS_MAJ: | 4368 | case PERF_COUNT_SW_PAGE_FAULTS_MAJ: |
4187 | case PERF_COUNT_SW_CONTEXT_SWITCHES: | 4369 | case PERF_COUNT_SW_CONTEXT_SWITCHES: |
4188 | case PERF_COUNT_SW_CPU_MIGRATIONS: | 4370 | case PERF_COUNT_SW_CPU_MIGRATIONS: |
4371 | case PERF_COUNT_SW_ALIGNMENT_FAULTS: | ||
4372 | case PERF_COUNT_SW_EMULATION_FAULTS: | ||
4189 | if (!event->parent) { | 4373 | if (!event->parent) { |
4190 | atomic_inc(&perf_swevent_enabled[event_id]); | 4374 | atomic_inc(&perf_swevent_enabled[event_id]); |
4191 | event->destroy = sw_perf_event_destroy; | 4375 | event->destroy = sw_perf_event_destroy; |
@@ -4206,6 +4390,7 @@ perf_event_alloc(struct perf_event_attr *attr, | |||
4206 | struct perf_event_context *ctx, | 4390 | struct perf_event_context *ctx, |
4207 | struct perf_event *group_leader, | 4391 | struct perf_event *group_leader, |
4208 | struct perf_event *parent_event, | 4392 | struct perf_event *parent_event, |
4393 | perf_callback_t callback, | ||
4209 | gfp_t gfpflags) | 4394 | gfp_t gfpflags) |
4210 | { | 4395 | { |
4211 | const struct pmu *pmu; | 4396 | const struct pmu *pmu; |
@@ -4248,6 +4433,11 @@ perf_event_alloc(struct perf_event_attr *attr, | |||
4248 | 4433 | ||
4249 | event->state = PERF_EVENT_STATE_INACTIVE; | 4434 | event->state = PERF_EVENT_STATE_INACTIVE; |
4250 | 4435 | ||
4436 | if (!callback && parent_event) | ||
4437 | callback = parent_event->callback; | ||
4438 | |||
4439 | event->callback = callback; | ||
4440 | |||
4251 | if (attr->disabled) | 4441 | if (attr->disabled) |
4252 | event->state = PERF_EVENT_STATE_OFF; | 4442 | event->state = PERF_EVENT_STATE_OFF; |
4253 | 4443 | ||
@@ -4282,6 +4472,11 @@ perf_event_alloc(struct perf_event_attr *attr, | |||
4282 | pmu = tp_perf_event_init(event); | 4472 | pmu = tp_perf_event_init(event); |
4283 | break; | 4473 | break; |
4284 | 4474 | ||
4475 | case PERF_TYPE_BREAKPOINT: | ||
4476 | pmu = bp_perf_event_init(event); | ||
4477 | break; | ||
4478 | |||
4479 | |||
4285 | default: | 4480 | default: |
4286 | break; | 4481 | break; |
4287 | } | 4482 | } |
@@ -4394,7 +4589,7 @@ err_size: | |||
4394 | goto out; | 4589 | goto out; |
4395 | } | 4590 | } |
4396 | 4591 | ||
4397 | int perf_event_set_output(struct perf_event *event, int output_fd) | 4592 | static int perf_event_set_output(struct perf_event *event, int output_fd) |
4398 | { | 4593 | { |
4399 | struct perf_event *output_event = NULL; | 4594 | struct perf_event *output_event = NULL; |
4400 | struct file *output_file = NULL; | 4595 | struct file *output_file = NULL; |
@@ -4524,7 +4719,7 @@ SYSCALL_DEFINE5(perf_event_open, | |||
4524 | } | 4719 | } |
4525 | 4720 | ||
4526 | event = perf_event_alloc(&attr, cpu, ctx, group_leader, | 4721 | event = perf_event_alloc(&attr, cpu, ctx, group_leader, |
4527 | NULL, GFP_KERNEL); | 4722 | NULL, NULL, GFP_KERNEL); |
4528 | err = PTR_ERR(event); | 4723 | err = PTR_ERR(event); |
4529 | if (IS_ERR(event)) | 4724 | if (IS_ERR(event)) |
4530 | goto err_put_context; | 4725 | goto err_put_context; |
@@ -4572,6 +4767,60 @@ err_put_context: | |||
4572 | return err; | 4767 | return err; |
4573 | } | 4768 | } |
4574 | 4769 | ||
4770 | /** | ||
4771 | * perf_event_create_kernel_counter | ||
4772 | * | ||
4773 | * @attr: attributes of the counter to create | ||
4774 | * @cpu: cpu in which the counter is bound | ||
4775 | * @pid: task to profile | ||
4776 | */ | ||
4777 | struct perf_event * | ||
4778 | perf_event_create_kernel_counter(struct perf_event_attr *attr, int cpu, | ||
4779 | pid_t pid, perf_callback_t callback) | ||
4780 | { | ||
4781 | struct perf_event *event; | ||
4782 | struct perf_event_context *ctx; | ||
4783 | int err; | ||
4784 | |||
4785 | /* | ||
4786 | * Get the target context (task or percpu): | ||
4787 | */ | ||
4788 | |||
4789 | ctx = find_get_context(pid, cpu); | ||
4790 | if (IS_ERR(ctx)) { | ||
4791 | err = PTR_ERR(ctx); | ||
4792 | goto err_exit; | ||
4793 | } | ||
4794 | |||
4795 | event = perf_event_alloc(attr, cpu, ctx, NULL, | ||
4796 | NULL, callback, GFP_KERNEL); | ||
4797 | if (IS_ERR(event)) { | ||
4798 | err = PTR_ERR(event); | ||
4799 | goto err_put_context; | ||
4800 | } | ||
4801 | |||
4802 | event->filp = NULL; | ||
4803 | WARN_ON_ONCE(ctx->parent_ctx); | ||
4804 | mutex_lock(&ctx->mutex); | ||
4805 | perf_install_in_context(ctx, event, cpu); | ||
4806 | ++ctx->generation; | ||
4807 | mutex_unlock(&ctx->mutex); | ||
4808 | |||
4809 | event->owner = current; | ||
4810 | get_task_struct(current); | ||
4811 | mutex_lock(¤t->perf_event_mutex); | ||
4812 | list_add_tail(&event->owner_entry, ¤t->perf_event_list); | ||
4813 | mutex_unlock(¤t->perf_event_mutex); | ||
4814 | |||
4815 | return event; | ||
4816 | |||
4817 | err_put_context: | ||
4818 | put_ctx(ctx); | ||
4819 | err_exit: | ||
4820 | return ERR_PTR(err); | ||
4821 | } | ||
4822 | EXPORT_SYMBOL_GPL(perf_event_create_kernel_counter); | ||
4823 | |||
4575 | /* | 4824 | /* |
4576 | * inherit a event from parent task to child task: | 4825 | * inherit a event from parent task to child task: |
4577 | */ | 4826 | */ |
@@ -4597,7 +4846,7 @@ inherit_event(struct perf_event *parent_event, | |||
4597 | child_event = perf_event_alloc(&parent_event->attr, | 4846 | child_event = perf_event_alloc(&parent_event->attr, |
4598 | parent_event->cpu, child_ctx, | 4847 | parent_event->cpu, child_ctx, |
4599 | group_leader, parent_event, | 4848 | group_leader, parent_event, |
4600 | GFP_KERNEL); | 4849 | NULL, GFP_KERNEL); |
4601 | if (IS_ERR(child_event)) | 4850 | if (IS_ERR(child_event)) |
4602 | return child_event; | 4851 | return child_event; |
4603 | get_ctx(child_ctx); | 4852 | get_ctx(child_ctx); |
@@ -4615,6 +4864,8 @@ inherit_event(struct perf_event *parent_event, | |||
4615 | if (parent_event->attr.freq) | 4864 | if (parent_event->attr.freq) |
4616 | child_event->hw.sample_period = parent_event->hw.sample_period; | 4865 | child_event->hw.sample_period = parent_event->hw.sample_period; |
4617 | 4866 | ||
4867 | child_event->overflow_handler = parent_event->overflow_handler; | ||
4868 | |||
4618 | /* | 4869 | /* |
4619 | * Link it up in the child's context: | 4870 | * Link it up in the child's context: |
4620 | */ | 4871 | */ |
@@ -4704,7 +4955,6 @@ __perf_event_exit_task(struct perf_event *child_event, | |||
4704 | { | 4955 | { |
4705 | struct perf_event *parent_event; | 4956 | struct perf_event *parent_event; |
4706 | 4957 | ||
4707 | update_event_times(child_event); | ||
4708 | perf_event_remove_from_context(child_event); | 4958 | perf_event_remove_from_context(child_event); |
4709 | 4959 | ||
4710 | parent_event = child_event->parent; | 4960 | parent_event = child_event->parent; |
@@ -4756,6 +5006,7 @@ void perf_event_exit_task(struct task_struct *child) | |||
4756 | * the events from it. | 5006 | * the events from it. |
4757 | */ | 5007 | */ |
4758 | unclone_ctx(child_ctx); | 5008 | unclone_ctx(child_ctx); |
5009 | update_context_time(child_ctx); | ||
4759 | spin_unlock_irqrestore(&child_ctx->lock, flags); | 5010 | spin_unlock_irqrestore(&child_ctx->lock, flags); |
4760 | 5011 | ||
4761 | /* | 5012 | /* |
diff --git a/kernel/pm_qos_params.c b/kernel/pm_qos_params.c index dfdec524d1b7..3db49b9ca374 100644 --- a/kernel/pm_qos_params.c +++ b/kernel/pm_qos_params.c | |||
@@ -29,7 +29,6 @@ | |||
29 | 29 | ||
30 | #include <linux/pm_qos_params.h> | 30 | #include <linux/pm_qos_params.h> |
31 | #include <linux/sched.h> | 31 | #include <linux/sched.h> |
32 | #include <linux/smp_lock.h> | ||
33 | #include <linux/spinlock.h> | 32 | #include <linux/spinlock.h> |
34 | #include <linux/slab.h> | 33 | #include <linux/slab.h> |
35 | #include <linux/time.h> | 34 | #include <linux/time.h> |
@@ -344,37 +343,33 @@ int pm_qos_remove_notifier(int pm_qos_class, struct notifier_block *notifier) | |||
344 | } | 343 | } |
345 | EXPORT_SYMBOL_GPL(pm_qos_remove_notifier); | 344 | EXPORT_SYMBOL_GPL(pm_qos_remove_notifier); |
346 | 345 | ||
347 | #define PID_NAME_LEN sizeof("process_1234567890") | 346 | #define PID_NAME_LEN 32 |
348 | static char name[PID_NAME_LEN]; | ||
349 | 347 | ||
350 | static int pm_qos_power_open(struct inode *inode, struct file *filp) | 348 | static int pm_qos_power_open(struct inode *inode, struct file *filp) |
351 | { | 349 | { |
352 | int ret; | 350 | int ret; |
353 | long pm_qos_class; | 351 | long pm_qos_class; |
352 | char name[PID_NAME_LEN]; | ||
354 | 353 | ||
355 | lock_kernel(); | ||
356 | pm_qos_class = find_pm_qos_object_by_minor(iminor(inode)); | 354 | pm_qos_class = find_pm_qos_object_by_minor(iminor(inode)); |
357 | if (pm_qos_class >= 0) { | 355 | if (pm_qos_class >= 0) { |
358 | filp->private_data = (void *)pm_qos_class; | 356 | filp->private_data = (void *)pm_qos_class; |
359 | sprintf(name, "process_%d", current->pid); | 357 | snprintf(name, PID_NAME_LEN, "process_%d", current->pid); |
360 | ret = pm_qos_add_requirement(pm_qos_class, name, | 358 | ret = pm_qos_add_requirement(pm_qos_class, name, |
361 | PM_QOS_DEFAULT_VALUE); | 359 | PM_QOS_DEFAULT_VALUE); |
362 | if (ret >= 0) { | 360 | if (ret >= 0) |
363 | unlock_kernel(); | ||
364 | return 0; | 361 | return 0; |
365 | } | ||
366 | } | 362 | } |
367 | unlock_kernel(); | ||
368 | |||
369 | return -EPERM; | 363 | return -EPERM; |
370 | } | 364 | } |
371 | 365 | ||
372 | static int pm_qos_power_release(struct inode *inode, struct file *filp) | 366 | static int pm_qos_power_release(struct inode *inode, struct file *filp) |
373 | { | 367 | { |
374 | int pm_qos_class; | 368 | int pm_qos_class; |
369 | char name[PID_NAME_LEN]; | ||
375 | 370 | ||
376 | pm_qos_class = (long)filp->private_data; | 371 | pm_qos_class = (long)filp->private_data; |
377 | sprintf(name, "process_%d", current->pid); | 372 | snprintf(name, PID_NAME_LEN, "process_%d", current->pid); |
378 | pm_qos_remove_requirement(pm_qos_class, name); | 373 | pm_qos_remove_requirement(pm_qos_class, name); |
379 | 374 | ||
380 | return 0; | 375 | return 0; |
@@ -385,13 +380,14 @@ static ssize_t pm_qos_power_write(struct file *filp, const char __user *buf, | |||
385 | { | 380 | { |
386 | s32 value; | 381 | s32 value; |
387 | int pm_qos_class; | 382 | int pm_qos_class; |
383 | char name[PID_NAME_LEN]; | ||
388 | 384 | ||
389 | pm_qos_class = (long)filp->private_data; | 385 | pm_qos_class = (long)filp->private_data; |
390 | if (count != sizeof(s32)) | 386 | if (count != sizeof(s32)) |
391 | return -EINVAL; | 387 | return -EINVAL; |
392 | if (copy_from_user(&value, buf, sizeof(s32))) | 388 | if (copy_from_user(&value, buf, sizeof(s32))) |
393 | return -EFAULT; | 389 | return -EFAULT; |
394 | sprintf(name, "process_%d", current->pid); | 390 | snprintf(name, PID_NAME_LEN, "process_%d", current->pid); |
395 | pm_qos_update_requirement(pm_qos_class, name, value); | 391 | pm_qos_update_requirement(pm_qos_class, name, value); |
396 | 392 | ||
397 | return sizeof(s32); | 393 | return sizeof(s32); |
diff --git a/kernel/posix-cpu-timers.c b/kernel/posix-cpu-timers.c index 5c9dc228747b..438ff4523513 100644 --- a/kernel/posix-cpu-timers.c +++ b/kernel/posix-cpu-timers.c | |||
@@ -384,7 +384,8 @@ int posix_cpu_clock_get(const clockid_t which_clock, struct timespec *tp) | |||
384 | 384 | ||
385 | /* | 385 | /* |
386 | * Validate the clockid_t for a new CPU-clock timer, and initialize the timer. | 386 | * Validate the clockid_t for a new CPU-clock timer, and initialize the timer. |
387 | * This is called from sys_timer_create with the new timer already locked. | 387 | * This is called from sys_timer_create() and do_cpu_nanosleep() with the |
388 | * new timer already all-zeros initialized. | ||
388 | */ | 389 | */ |
389 | int posix_cpu_timer_create(struct k_itimer *new_timer) | 390 | int posix_cpu_timer_create(struct k_itimer *new_timer) |
390 | { | 391 | { |
@@ -396,8 +397,6 @@ int posix_cpu_timer_create(struct k_itimer *new_timer) | |||
396 | return -EINVAL; | 397 | return -EINVAL; |
397 | 398 | ||
398 | INIT_LIST_HEAD(&new_timer->it.cpu.entry); | 399 | INIT_LIST_HEAD(&new_timer->it.cpu.entry); |
399 | new_timer->it.cpu.incr.sched = 0; | ||
400 | new_timer->it.cpu.expires.sched = 0; | ||
401 | 400 | ||
402 | read_lock(&tasklist_lock); | 401 | read_lock(&tasklist_lock); |
403 | if (CPUCLOCK_PERTHREAD(new_timer->it_clock)) { | 402 | if (CPUCLOCK_PERTHREAD(new_timer->it_clock)) { |
diff --git a/kernel/power/Makefile b/kernel/power/Makefile index c3b81c30e5d5..43191815f874 100644 --- a/kernel/power/Makefile +++ b/kernel/power/Makefile | |||
@@ -8,7 +8,7 @@ obj-$(CONFIG_PM_SLEEP) += console.o | |||
8 | obj-$(CONFIG_FREEZER) += process.o | 8 | obj-$(CONFIG_FREEZER) += process.o |
9 | obj-$(CONFIG_SUSPEND) += suspend.o | 9 | obj-$(CONFIG_SUSPEND) += suspend.o |
10 | obj-$(CONFIG_PM_TEST_SUSPEND) += suspend_test.o | 10 | obj-$(CONFIG_PM_TEST_SUSPEND) += suspend_test.o |
11 | obj-$(CONFIG_HIBERNATION) += swsusp.o hibernate.o snapshot.o swap.o user.o | 11 | obj-$(CONFIG_HIBERNATION) += hibernate.o snapshot.o swap.o user.o |
12 | obj-$(CONFIG_HIBERNATION_NVS) += hibernate_nvs.o | 12 | obj-$(CONFIG_HIBERNATION_NVS) += hibernate_nvs.o |
13 | 13 | ||
14 | obj-$(CONFIG_MAGIC_SYSRQ) += poweroff.o | 14 | obj-$(CONFIG_MAGIC_SYSRQ) += poweroff.o |
diff --git a/kernel/power/hibernate.c b/kernel/power/hibernate.c index 04b3a83d686f..bbfe472d7524 100644 --- a/kernel/power/hibernate.c +++ b/kernel/power/hibernate.c | |||
@@ -32,6 +32,7 @@ static int noresume = 0; | |||
32 | static char resume_file[256] = CONFIG_PM_STD_PARTITION; | 32 | static char resume_file[256] = CONFIG_PM_STD_PARTITION; |
33 | dev_t swsusp_resume_device; | 33 | dev_t swsusp_resume_device; |
34 | sector_t swsusp_resume_block; | 34 | sector_t swsusp_resume_block; |
35 | int in_suspend __nosavedata = 0; | ||
35 | 36 | ||
36 | enum { | 37 | enum { |
37 | HIBERNATION_INVALID, | 38 | HIBERNATION_INVALID, |
@@ -202,6 +203,35 @@ static void platform_recover(int platform_mode) | |||
202 | } | 203 | } |
203 | 204 | ||
204 | /** | 205 | /** |
206 | * swsusp_show_speed - print the time elapsed between two events. | ||
207 | * @start: Starting event. | ||
208 | * @stop: Final event. | ||
209 | * @nr_pages - number of pages processed between @start and @stop | ||
210 | * @msg - introductory message to print | ||
211 | */ | ||
212 | |||
213 | void swsusp_show_speed(struct timeval *start, struct timeval *stop, | ||
214 | unsigned nr_pages, char *msg) | ||
215 | { | ||
216 | s64 elapsed_centisecs64; | ||
217 | int centisecs; | ||
218 | int k; | ||
219 | int kps; | ||
220 | |||
221 | elapsed_centisecs64 = timeval_to_ns(stop) - timeval_to_ns(start); | ||
222 | do_div(elapsed_centisecs64, NSEC_PER_SEC / 100); | ||
223 | centisecs = elapsed_centisecs64; | ||
224 | if (centisecs == 0) | ||
225 | centisecs = 1; /* avoid div-by-zero */ | ||
226 | k = nr_pages * (PAGE_SIZE / 1024); | ||
227 | kps = (k * 100) / centisecs; | ||
228 | printk(KERN_INFO "PM: %s %d kbytes in %d.%02d seconds (%d.%02d MB/s)\n", | ||
229 | msg, k, | ||
230 | centisecs / 100, centisecs % 100, | ||
231 | kps / 1000, (kps % 1000) / 10); | ||
232 | } | ||
233 | |||
234 | /** | ||
205 | * create_image - freeze devices that need to be frozen with interrupts | 235 | * create_image - freeze devices that need to be frozen with interrupts |
206 | * off, create the hibernation image and thaw those devices. Control | 236 | * off, create the hibernation image and thaw those devices. Control |
207 | * reappears in this routine after a restore. | 237 | * reappears in this routine after a restore. |
@@ -693,21 +723,22 @@ static int software_resume(void) | |||
693 | /* The snapshot device should not be opened while we're running */ | 723 | /* The snapshot device should not be opened while we're running */ |
694 | if (!atomic_add_unless(&snapshot_device_available, -1, 0)) { | 724 | if (!atomic_add_unless(&snapshot_device_available, -1, 0)) { |
695 | error = -EBUSY; | 725 | error = -EBUSY; |
726 | swsusp_close(FMODE_READ); | ||
696 | goto Unlock; | 727 | goto Unlock; |
697 | } | 728 | } |
698 | 729 | ||
699 | pm_prepare_console(); | 730 | pm_prepare_console(); |
700 | error = pm_notifier_call_chain(PM_RESTORE_PREPARE); | 731 | error = pm_notifier_call_chain(PM_RESTORE_PREPARE); |
701 | if (error) | 732 | if (error) |
702 | goto Finish; | 733 | goto close_finish; |
703 | 734 | ||
704 | error = usermodehelper_disable(); | 735 | error = usermodehelper_disable(); |
705 | if (error) | 736 | if (error) |
706 | goto Finish; | 737 | goto close_finish; |
707 | 738 | ||
708 | error = create_basic_memory_bitmaps(); | 739 | error = create_basic_memory_bitmaps(); |
709 | if (error) | 740 | if (error) |
710 | goto Finish; | 741 | goto close_finish; |
711 | 742 | ||
712 | pr_debug("PM: Preparing processes for restore.\n"); | 743 | pr_debug("PM: Preparing processes for restore.\n"); |
713 | error = prepare_processes(); | 744 | error = prepare_processes(); |
@@ -719,6 +750,7 @@ static int software_resume(void) | |||
719 | pr_debug("PM: Reading hibernation image.\n"); | 750 | pr_debug("PM: Reading hibernation image.\n"); |
720 | 751 | ||
721 | error = swsusp_read(&flags); | 752 | error = swsusp_read(&flags); |
753 | swsusp_close(FMODE_READ); | ||
722 | if (!error) | 754 | if (!error) |
723 | hibernation_restore(flags & SF_PLATFORM_MODE); | 755 | hibernation_restore(flags & SF_PLATFORM_MODE); |
724 | 756 | ||
@@ -737,6 +769,9 @@ static int software_resume(void) | |||
737 | mutex_unlock(&pm_mutex); | 769 | mutex_unlock(&pm_mutex); |
738 | pr_debug("PM: Resume from disk failed.\n"); | 770 | pr_debug("PM: Resume from disk failed.\n"); |
739 | return error; | 771 | return error; |
772 | close_finish: | ||
773 | swsusp_close(FMODE_READ); | ||
774 | goto Finish; | ||
740 | } | 775 | } |
741 | 776 | ||
742 | late_initcall(software_resume); | 777 | late_initcall(software_resume); |
diff --git a/kernel/power/main.c b/kernel/power/main.c index 347d2cc88cd0..0998c7139053 100644 --- a/kernel/power/main.c +++ b/kernel/power/main.c | |||
@@ -220,6 +220,7 @@ static struct attribute_group attr_group = { | |||
220 | 220 | ||
221 | #ifdef CONFIG_PM_RUNTIME | 221 | #ifdef CONFIG_PM_RUNTIME |
222 | struct workqueue_struct *pm_wq; | 222 | struct workqueue_struct *pm_wq; |
223 | EXPORT_SYMBOL_GPL(pm_wq); | ||
223 | 224 | ||
224 | static int __init pm_start_workqueue(void) | 225 | static int __init pm_start_workqueue(void) |
225 | { | 226 | { |
diff --git a/kernel/power/process.c b/kernel/power/process.c index cc2e55373b68..5ade1bdcf366 100644 --- a/kernel/power/process.c +++ b/kernel/power/process.c | |||
@@ -14,6 +14,7 @@ | |||
14 | #include <linux/module.h> | 14 | #include <linux/module.h> |
15 | #include <linux/syscalls.h> | 15 | #include <linux/syscalls.h> |
16 | #include <linux/freezer.h> | 16 | #include <linux/freezer.h> |
17 | #include <linux/delay.h> | ||
17 | 18 | ||
18 | /* | 19 | /* |
19 | * Timeout for stopping processes | 20 | * Timeout for stopping processes |
@@ -41,7 +42,7 @@ static int try_to_freeze_tasks(bool sig_only) | |||
41 | do_gettimeofday(&start); | 42 | do_gettimeofday(&start); |
42 | 43 | ||
43 | end_time = jiffies + TIMEOUT; | 44 | end_time = jiffies + TIMEOUT; |
44 | do { | 45 | while (true) { |
45 | todo = 0; | 46 | todo = 0; |
46 | read_lock(&tasklist_lock); | 47 | read_lock(&tasklist_lock); |
47 | do_each_thread(g, p) { | 48 | do_each_thread(g, p) { |
@@ -62,10 +63,15 @@ static int try_to_freeze_tasks(bool sig_only) | |||
62 | todo++; | 63 | todo++; |
63 | } while_each_thread(g, p); | 64 | } while_each_thread(g, p); |
64 | read_unlock(&tasklist_lock); | 65 | read_unlock(&tasklist_lock); |
65 | yield(); /* Yield is okay here */ | 66 | if (!todo || time_after(jiffies, end_time)) |
66 | if (time_after(jiffies, end_time)) | ||
67 | break; | 67 | break; |
68 | } while (todo); | 68 | |
69 | /* | ||
70 | * We need to retry, but first give the freezing tasks some | ||
71 | * time to enter the regrigerator. | ||
72 | */ | ||
73 | msleep(10); | ||
74 | } | ||
69 | 75 | ||
70 | do_gettimeofday(&end); | 76 | do_gettimeofday(&end); |
71 | elapsed_csecs64 = timeval_to_ns(&end) - timeval_to_ns(&start); | 77 | elapsed_csecs64 = timeval_to_ns(&end) - timeval_to_ns(&start); |
diff --git a/kernel/power/swap.c b/kernel/power/swap.c index b101cdc4df3f..09b2b0ae9e9d 100644 --- a/kernel/power/swap.c +++ b/kernel/power/swap.c | |||
@@ -38,6 +38,107 @@ struct swsusp_header { | |||
38 | 38 | ||
39 | static struct swsusp_header *swsusp_header; | 39 | static struct swsusp_header *swsusp_header; |
40 | 40 | ||
41 | /** | ||
42 | * The following functions are used for tracing the allocated | ||
43 | * swap pages, so that they can be freed in case of an error. | ||
44 | */ | ||
45 | |||
46 | struct swsusp_extent { | ||
47 | struct rb_node node; | ||
48 | unsigned long start; | ||
49 | unsigned long end; | ||
50 | }; | ||
51 | |||
52 | static struct rb_root swsusp_extents = RB_ROOT; | ||
53 | |||
54 | static int swsusp_extents_insert(unsigned long swap_offset) | ||
55 | { | ||
56 | struct rb_node **new = &(swsusp_extents.rb_node); | ||
57 | struct rb_node *parent = NULL; | ||
58 | struct swsusp_extent *ext; | ||
59 | |||
60 | /* Figure out where to put the new node */ | ||
61 | while (*new) { | ||
62 | ext = container_of(*new, struct swsusp_extent, node); | ||
63 | parent = *new; | ||
64 | if (swap_offset < ext->start) { | ||
65 | /* Try to merge */ | ||
66 | if (swap_offset == ext->start - 1) { | ||
67 | ext->start--; | ||
68 | return 0; | ||
69 | } | ||
70 | new = &((*new)->rb_left); | ||
71 | } else if (swap_offset > ext->end) { | ||
72 | /* Try to merge */ | ||
73 | if (swap_offset == ext->end + 1) { | ||
74 | ext->end++; | ||
75 | return 0; | ||
76 | } | ||
77 | new = &((*new)->rb_right); | ||
78 | } else { | ||
79 | /* It already is in the tree */ | ||
80 | return -EINVAL; | ||
81 | } | ||
82 | } | ||
83 | /* Add the new node and rebalance the tree. */ | ||
84 | ext = kzalloc(sizeof(struct swsusp_extent), GFP_KERNEL); | ||
85 | if (!ext) | ||
86 | return -ENOMEM; | ||
87 | |||
88 | ext->start = swap_offset; | ||
89 | ext->end = swap_offset; | ||
90 | rb_link_node(&ext->node, parent, new); | ||
91 | rb_insert_color(&ext->node, &swsusp_extents); | ||
92 | return 0; | ||
93 | } | ||
94 | |||
95 | /** | ||
96 | * alloc_swapdev_block - allocate a swap page and register that it has | ||
97 | * been allocated, so that it can be freed in case of an error. | ||
98 | */ | ||
99 | |||
100 | sector_t alloc_swapdev_block(int swap) | ||
101 | { | ||
102 | unsigned long offset; | ||
103 | |||
104 | offset = swp_offset(get_swap_page_of_type(swap)); | ||
105 | if (offset) { | ||
106 | if (swsusp_extents_insert(offset)) | ||
107 | swap_free(swp_entry(swap, offset)); | ||
108 | else | ||
109 | return swapdev_block(swap, offset); | ||
110 | } | ||
111 | return 0; | ||
112 | } | ||
113 | |||
114 | /** | ||
115 | * free_all_swap_pages - free swap pages allocated for saving image data. | ||
116 | * It also frees the extents used to register which swap entres had been | ||
117 | * allocated. | ||
118 | */ | ||
119 | |||
120 | void free_all_swap_pages(int swap) | ||
121 | { | ||
122 | struct rb_node *node; | ||
123 | |||
124 | while ((node = swsusp_extents.rb_node)) { | ||
125 | struct swsusp_extent *ext; | ||
126 | unsigned long offset; | ||
127 | |||
128 | ext = container_of(node, struct swsusp_extent, node); | ||
129 | rb_erase(node, &swsusp_extents); | ||
130 | for (offset = ext->start; offset <= ext->end; offset++) | ||
131 | swap_free(swp_entry(swap, offset)); | ||
132 | |||
133 | kfree(ext); | ||
134 | } | ||
135 | } | ||
136 | |||
137 | int swsusp_swap_in_use(void) | ||
138 | { | ||
139 | return (swsusp_extents.rb_node != NULL); | ||
140 | } | ||
141 | |||
41 | /* | 142 | /* |
42 | * General things | 143 | * General things |
43 | */ | 144 | */ |
@@ -314,7 +415,6 @@ static int save_image(struct swap_map_handle *handle, | |||
314 | { | 415 | { |
315 | unsigned int m; | 416 | unsigned int m; |
316 | int ret; | 417 | int ret; |
317 | int error = 0; | ||
318 | int nr_pages; | 418 | int nr_pages; |
319 | int err2; | 419 | int err2; |
320 | struct bio *bio; | 420 | struct bio *bio; |
@@ -329,26 +429,27 @@ static int save_image(struct swap_map_handle *handle, | |||
329 | nr_pages = 0; | 429 | nr_pages = 0; |
330 | bio = NULL; | 430 | bio = NULL; |
331 | do_gettimeofday(&start); | 431 | do_gettimeofday(&start); |
332 | do { | 432 | while (1) { |
333 | ret = snapshot_read_next(snapshot, PAGE_SIZE); | 433 | ret = snapshot_read_next(snapshot, PAGE_SIZE); |
334 | if (ret > 0) { | 434 | if (ret <= 0) |
335 | error = swap_write_page(handle, data_of(*snapshot), | 435 | break; |
336 | &bio); | 436 | ret = swap_write_page(handle, data_of(*snapshot), &bio); |
337 | if (error) | 437 | if (ret) |
338 | break; | 438 | break; |
339 | if (!(nr_pages % m)) | 439 | if (!(nr_pages % m)) |
340 | printk("\b\b\b\b%3d%%", nr_pages / m); | 440 | printk(KERN_CONT "\b\b\b\b%3d%%", nr_pages / m); |
341 | nr_pages++; | 441 | nr_pages++; |
342 | } | 442 | } |
343 | } while (ret > 0); | ||
344 | err2 = wait_on_bio_chain(&bio); | 443 | err2 = wait_on_bio_chain(&bio); |
345 | do_gettimeofday(&stop); | 444 | do_gettimeofday(&stop); |
346 | if (!error) | 445 | if (!ret) |
347 | error = err2; | 446 | ret = err2; |
348 | if (!error) | 447 | if (!ret) |
349 | printk("\b\b\b\bdone\n"); | 448 | printk(KERN_CONT "\b\b\b\bdone\n"); |
449 | else | ||
450 | printk(KERN_CONT "\n"); | ||
350 | swsusp_show_speed(&start, &stop, nr_to_write, "Wrote"); | 451 | swsusp_show_speed(&start, &stop, nr_to_write, "Wrote"); |
351 | return error; | 452 | return ret; |
352 | } | 453 | } |
353 | 454 | ||
354 | /** | 455 | /** |
@@ -536,7 +637,8 @@ static int load_image(struct swap_map_handle *handle, | |||
536 | snapshot_write_finalize(snapshot); | 637 | snapshot_write_finalize(snapshot); |
537 | if (!snapshot_image_loaded(snapshot)) | 638 | if (!snapshot_image_loaded(snapshot)) |
538 | error = -ENODATA; | 639 | error = -ENODATA; |
539 | } | 640 | } else |
641 | printk("\n"); | ||
540 | swsusp_show_speed(&start, &stop, nr_to_read, "Read"); | 642 | swsusp_show_speed(&start, &stop, nr_to_read, "Read"); |
541 | return error; | 643 | return error; |
542 | } | 644 | } |
@@ -572,8 +674,6 @@ int swsusp_read(unsigned int *flags_p) | |||
572 | error = load_image(&handle, &snapshot, header->pages - 1); | 674 | error = load_image(&handle, &snapshot, header->pages - 1); |
573 | release_swap_reader(&handle); | 675 | release_swap_reader(&handle); |
574 | 676 | ||
575 | blkdev_put(resume_bdev, FMODE_READ); | ||
576 | |||
577 | if (!error) | 677 | if (!error) |
578 | pr_debug("PM: Image successfully loaded\n"); | 678 | pr_debug("PM: Image successfully loaded\n"); |
579 | else | 679 | else |
@@ -596,7 +696,7 @@ int swsusp_check(void) | |||
596 | error = bio_read_page(swsusp_resume_block, | 696 | error = bio_read_page(swsusp_resume_block, |
597 | swsusp_header, NULL); | 697 | swsusp_header, NULL); |
598 | if (error) | 698 | if (error) |
599 | return error; | 699 | goto put; |
600 | 700 | ||
601 | if (!memcmp(SWSUSP_SIG, swsusp_header->sig, 10)) { | 701 | if (!memcmp(SWSUSP_SIG, swsusp_header->sig, 10)) { |
602 | memcpy(swsusp_header->sig, swsusp_header->orig_sig, 10); | 702 | memcpy(swsusp_header->sig, swsusp_header->orig_sig, 10); |
@@ -604,8 +704,10 @@ int swsusp_check(void) | |||
604 | error = bio_write_page(swsusp_resume_block, | 704 | error = bio_write_page(swsusp_resume_block, |
605 | swsusp_header, NULL); | 705 | swsusp_header, NULL); |
606 | } else { | 706 | } else { |
607 | return -EINVAL; | 707 | error = -EINVAL; |
608 | } | 708 | } |
709 | |||
710 | put: | ||
609 | if (error) | 711 | if (error) |
610 | blkdev_put(resume_bdev, FMODE_READ); | 712 | blkdev_put(resume_bdev, FMODE_READ); |
611 | else | 713 | else |
diff --git a/kernel/power/swsusp.c b/kernel/power/swsusp.c index 6a07f4dbf2f8..5b3601bd1893 100644 --- a/kernel/power/swsusp.c +++ b/kernel/power/swsusp.c | |||
@@ -56,133 +56,3 @@ | |||
56 | #include "power.h" | 56 | #include "power.h" |
57 | 57 | ||
58 | int in_suspend __nosavedata = 0; | 58 | int in_suspend __nosavedata = 0; |
59 | |||
60 | /** | ||
61 | * The following functions are used for tracing the allocated | ||
62 | * swap pages, so that they can be freed in case of an error. | ||
63 | */ | ||
64 | |||
65 | struct swsusp_extent { | ||
66 | struct rb_node node; | ||
67 | unsigned long start; | ||
68 | unsigned long end; | ||
69 | }; | ||
70 | |||
71 | static struct rb_root swsusp_extents = RB_ROOT; | ||
72 | |||
73 | static int swsusp_extents_insert(unsigned long swap_offset) | ||
74 | { | ||
75 | struct rb_node **new = &(swsusp_extents.rb_node); | ||
76 | struct rb_node *parent = NULL; | ||
77 | struct swsusp_extent *ext; | ||
78 | |||
79 | /* Figure out where to put the new node */ | ||
80 | while (*new) { | ||
81 | ext = container_of(*new, struct swsusp_extent, node); | ||
82 | parent = *new; | ||
83 | if (swap_offset < ext->start) { | ||
84 | /* Try to merge */ | ||
85 | if (swap_offset == ext->start - 1) { | ||
86 | ext->start--; | ||
87 | return 0; | ||
88 | } | ||
89 | new = &((*new)->rb_left); | ||
90 | } else if (swap_offset > ext->end) { | ||
91 | /* Try to merge */ | ||
92 | if (swap_offset == ext->end + 1) { | ||
93 | ext->end++; | ||
94 | return 0; | ||
95 | } | ||
96 | new = &((*new)->rb_right); | ||
97 | } else { | ||
98 | /* It already is in the tree */ | ||
99 | return -EINVAL; | ||
100 | } | ||
101 | } | ||
102 | /* Add the new node and rebalance the tree. */ | ||
103 | ext = kzalloc(sizeof(struct swsusp_extent), GFP_KERNEL); | ||
104 | if (!ext) | ||
105 | return -ENOMEM; | ||
106 | |||
107 | ext->start = swap_offset; | ||
108 | ext->end = swap_offset; | ||
109 | rb_link_node(&ext->node, parent, new); | ||
110 | rb_insert_color(&ext->node, &swsusp_extents); | ||
111 | return 0; | ||
112 | } | ||
113 | |||
114 | /** | ||
115 | * alloc_swapdev_block - allocate a swap page and register that it has | ||
116 | * been allocated, so that it can be freed in case of an error. | ||
117 | */ | ||
118 | |||
119 | sector_t alloc_swapdev_block(int swap) | ||
120 | { | ||
121 | unsigned long offset; | ||
122 | |||
123 | offset = swp_offset(get_swap_page_of_type(swap)); | ||
124 | if (offset) { | ||
125 | if (swsusp_extents_insert(offset)) | ||
126 | swap_free(swp_entry(swap, offset)); | ||
127 | else | ||
128 | return swapdev_block(swap, offset); | ||
129 | } | ||
130 | return 0; | ||
131 | } | ||
132 | |||
133 | /** | ||
134 | * free_all_swap_pages - free swap pages allocated for saving image data. | ||
135 | * It also frees the extents used to register which swap entres had been | ||
136 | * allocated. | ||
137 | */ | ||
138 | |||
139 | void free_all_swap_pages(int swap) | ||
140 | { | ||
141 | struct rb_node *node; | ||
142 | |||
143 | while ((node = swsusp_extents.rb_node)) { | ||
144 | struct swsusp_extent *ext; | ||
145 | unsigned long offset; | ||
146 | |||
147 | ext = container_of(node, struct swsusp_extent, node); | ||
148 | rb_erase(node, &swsusp_extents); | ||
149 | for (offset = ext->start; offset <= ext->end; offset++) | ||
150 | swap_free(swp_entry(swap, offset)); | ||
151 | |||
152 | kfree(ext); | ||
153 | } | ||
154 | } | ||
155 | |||
156 | int swsusp_swap_in_use(void) | ||
157 | { | ||
158 | return (swsusp_extents.rb_node != NULL); | ||
159 | } | ||
160 | |||
161 | /** | ||
162 | * swsusp_show_speed - print the time elapsed between two events represented by | ||
163 | * @start and @stop | ||
164 | * | ||
165 | * @nr_pages - number of pages processed between @start and @stop | ||
166 | * @msg - introductory message to print | ||
167 | */ | ||
168 | |||
169 | void swsusp_show_speed(struct timeval *start, struct timeval *stop, | ||
170 | unsigned nr_pages, char *msg) | ||
171 | { | ||
172 | s64 elapsed_centisecs64; | ||
173 | int centisecs; | ||
174 | int k; | ||
175 | int kps; | ||
176 | |||
177 | elapsed_centisecs64 = timeval_to_ns(stop) - timeval_to_ns(start); | ||
178 | do_div(elapsed_centisecs64, NSEC_PER_SEC / 100); | ||
179 | centisecs = elapsed_centisecs64; | ||
180 | if (centisecs == 0) | ||
181 | centisecs = 1; /* avoid div-by-zero */ | ||
182 | k = nr_pages * (PAGE_SIZE / 1024); | ||
183 | kps = (k * 100) / centisecs; | ||
184 | printk(KERN_INFO "PM: %s %d kbytes in %d.%02d seconds (%d.%02d MB/s)\n", | ||
185 | msg, k, | ||
186 | centisecs / 100, centisecs % 100, | ||
187 | kps / 1000, (kps % 1000) / 10); | ||
188 | } | ||
diff --git a/kernel/printk.c b/kernel/printk.c index f38b07f78a4e..b5ac4d99c667 100644 --- a/kernel/printk.c +++ b/kernel/printk.c | |||
@@ -33,6 +33,7 @@ | |||
33 | #include <linux/bootmem.h> | 33 | #include <linux/bootmem.h> |
34 | #include <linux/syscalls.h> | 34 | #include <linux/syscalls.h> |
35 | #include <linux/kexec.h> | 35 | #include <linux/kexec.h> |
36 | #include <linux/ratelimit.h> | ||
36 | 37 | ||
37 | #include <asm/uaccess.h> | 38 | #include <asm/uaccess.h> |
38 | 39 | ||
@@ -1376,11 +1377,11 @@ late_initcall(disable_boot_consoles); | |||
1376 | */ | 1377 | */ |
1377 | DEFINE_RATELIMIT_STATE(printk_ratelimit_state, 5 * HZ, 10); | 1378 | DEFINE_RATELIMIT_STATE(printk_ratelimit_state, 5 * HZ, 10); |
1378 | 1379 | ||
1379 | int printk_ratelimit(void) | 1380 | int __printk_ratelimit(const char *func) |
1380 | { | 1381 | { |
1381 | return __ratelimit(&printk_ratelimit_state); | 1382 | return ___ratelimit(&printk_ratelimit_state, func); |
1382 | } | 1383 | } |
1383 | EXPORT_SYMBOL(printk_ratelimit); | 1384 | EXPORT_SYMBOL(__printk_ratelimit); |
1384 | 1385 | ||
1385 | /** | 1386 | /** |
1386 | * printk_timed_ratelimit - caller-controlled printk ratelimiting | 1387 | * printk_timed_ratelimit - caller-controlled printk ratelimiting |
diff --git a/kernel/rcupdate.c b/kernel/rcupdate.c index 400183346ad2..9b7fd4723878 100644 --- a/kernel/rcupdate.c +++ b/kernel/rcupdate.c | |||
@@ -44,7 +44,6 @@ | |||
44 | #include <linux/cpu.h> | 44 | #include <linux/cpu.h> |
45 | #include <linux/mutex.h> | 45 | #include <linux/mutex.h> |
46 | #include <linux/module.h> | 46 | #include <linux/module.h> |
47 | #include <linux/kernel_stat.h> | ||
48 | 47 | ||
49 | #ifdef CONFIG_DEBUG_LOCK_ALLOC | 48 | #ifdef CONFIG_DEBUG_LOCK_ALLOC |
50 | static struct lock_class_key rcu_lock_key; | 49 | static struct lock_class_key rcu_lock_key; |
@@ -53,8 +52,6 @@ struct lockdep_map rcu_lock_map = | |||
53 | EXPORT_SYMBOL_GPL(rcu_lock_map); | 52 | EXPORT_SYMBOL_GPL(rcu_lock_map); |
54 | #endif | 53 | #endif |
55 | 54 | ||
56 | int rcu_scheduler_active __read_mostly; | ||
57 | |||
58 | /* | 55 | /* |
59 | * Awaken the corresponding synchronize_rcu() instance now that a | 56 | * Awaken the corresponding synchronize_rcu() instance now that a |
60 | * grace period has elapsed. | 57 | * grace period has elapsed. |
@@ -66,122 +63,3 @@ void wakeme_after_rcu(struct rcu_head *head) | |||
66 | rcu = container_of(head, struct rcu_synchronize, head); | 63 | rcu = container_of(head, struct rcu_synchronize, head); |
67 | complete(&rcu->completion); | 64 | complete(&rcu->completion); |
68 | } | 65 | } |
69 | |||
70 | #ifdef CONFIG_TREE_PREEMPT_RCU | ||
71 | |||
72 | /** | ||
73 | * synchronize_rcu - wait until a grace period has elapsed. | ||
74 | * | ||
75 | * Control will return to the caller some time after a full grace | ||
76 | * period has elapsed, in other words after all currently executing RCU | ||
77 | * read-side critical sections have completed. RCU read-side critical | ||
78 | * sections are delimited by rcu_read_lock() and rcu_read_unlock(), | ||
79 | * and may be nested. | ||
80 | */ | ||
81 | void synchronize_rcu(void) | ||
82 | { | ||
83 | struct rcu_synchronize rcu; | ||
84 | |||
85 | if (!rcu_scheduler_active) | ||
86 | return; | ||
87 | |||
88 | init_completion(&rcu.completion); | ||
89 | /* Will wake me after RCU finished. */ | ||
90 | call_rcu(&rcu.head, wakeme_after_rcu); | ||
91 | /* Wait for it. */ | ||
92 | wait_for_completion(&rcu.completion); | ||
93 | } | ||
94 | EXPORT_SYMBOL_GPL(synchronize_rcu); | ||
95 | |||
96 | #endif /* #ifdef CONFIG_TREE_PREEMPT_RCU */ | ||
97 | |||
98 | /** | ||
99 | * synchronize_sched - wait until an rcu-sched grace period has elapsed. | ||
100 | * | ||
101 | * Control will return to the caller some time after a full rcu-sched | ||
102 | * grace period has elapsed, in other words after all currently executing | ||
103 | * rcu-sched read-side critical sections have completed. These read-side | ||
104 | * critical sections are delimited by rcu_read_lock_sched() and | ||
105 | * rcu_read_unlock_sched(), and may be nested. Note that preempt_disable(), | ||
106 | * local_irq_disable(), and so on may be used in place of | ||
107 | * rcu_read_lock_sched(). | ||
108 | * | ||
109 | * This means that all preempt_disable code sequences, including NMI and | ||
110 | * hardware-interrupt handlers, in progress on entry will have completed | ||
111 | * before this primitive returns. However, this does not guarantee that | ||
112 | * softirq handlers will have completed, since in some kernels, these | ||
113 | * handlers can run in process context, and can block. | ||
114 | * | ||
115 | * This primitive provides the guarantees made by the (now removed) | ||
116 | * synchronize_kernel() API. In contrast, synchronize_rcu() only | ||
117 | * guarantees that rcu_read_lock() sections will have completed. | ||
118 | * In "classic RCU", these two guarantees happen to be one and | ||
119 | * the same, but can differ in realtime RCU implementations. | ||
120 | */ | ||
121 | void synchronize_sched(void) | ||
122 | { | ||
123 | struct rcu_synchronize rcu; | ||
124 | |||
125 | if (rcu_blocking_is_gp()) | ||
126 | return; | ||
127 | |||
128 | init_completion(&rcu.completion); | ||
129 | /* Will wake me after RCU finished. */ | ||
130 | call_rcu_sched(&rcu.head, wakeme_after_rcu); | ||
131 | /* Wait for it. */ | ||
132 | wait_for_completion(&rcu.completion); | ||
133 | } | ||
134 | EXPORT_SYMBOL_GPL(synchronize_sched); | ||
135 | |||
136 | /** | ||
137 | * synchronize_rcu_bh - wait until an rcu_bh grace period has elapsed. | ||
138 | * | ||
139 | * Control will return to the caller some time after a full rcu_bh grace | ||
140 | * period has elapsed, in other words after all currently executing rcu_bh | ||
141 | * read-side critical sections have completed. RCU read-side critical | ||
142 | * sections are delimited by rcu_read_lock_bh() and rcu_read_unlock_bh(), | ||
143 | * and may be nested. | ||
144 | */ | ||
145 | void synchronize_rcu_bh(void) | ||
146 | { | ||
147 | struct rcu_synchronize rcu; | ||
148 | |||
149 | if (rcu_blocking_is_gp()) | ||
150 | return; | ||
151 | |||
152 | init_completion(&rcu.completion); | ||
153 | /* Will wake me after RCU finished. */ | ||
154 | call_rcu_bh(&rcu.head, wakeme_after_rcu); | ||
155 | /* Wait for it. */ | ||
156 | wait_for_completion(&rcu.completion); | ||
157 | } | ||
158 | EXPORT_SYMBOL_GPL(synchronize_rcu_bh); | ||
159 | |||
160 | static int __cpuinit rcu_barrier_cpu_hotplug(struct notifier_block *self, | ||
161 | unsigned long action, void *hcpu) | ||
162 | { | ||
163 | return rcu_cpu_notify(self, action, hcpu); | ||
164 | } | ||
165 | |||
166 | void __init rcu_init(void) | ||
167 | { | ||
168 | int i; | ||
169 | |||
170 | __rcu_init(); | ||
171 | cpu_notifier(rcu_barrier_cpu_hotplug, 0); | ||
172 | |||
173 | /* | ||
174 | * We don't need protection against CPU-hotplug here because | ||
175 | * this is called early in boot, before either interrupts | ||
176 | * or the scheduler are operational. | ||
177 | */ | ||
178 | for_each_online_cpu(i) | ||
179 | rcu_barrier_cpu_hotplug(NULL, CPU_UP_PREPARE, (void *)(long)i); | ||
180 | } | ||
181 | |||
182 | void rcu_scheduler_starting(void) | ||
183 | { | ||
184 | WARN_ON(num_online_cpus() != 1); | ||
185 | WARN_ON(nr_context_switches() > 0); | ||
186 | rcu_scheduler_active = 1; | ||
187 | } | ||
diff --git a/kernel/rcutiny.c b/kernel/rcutiny.c new file mode 100644 index 000000000000..9f6d9ff2572c --- /dev/null +++ b/kernel/rcutiny.c | |||
@@ -0,0 +1,282 @@ | |||
1 | /* | ||
2 | * Read-Copy Update mechanism for mutual exclusion, the Bloatwatch edition. | ||
3 | * | ||
4 | * This program is free software; you can redistribute it and/or modify | ||
5 | * it under the terms of the GNU General Public License as published by | ||
6 | * the Free Software Foundation; either version 2 of the License, or | ||
7 | * (at your option) any later version. | ||
8 | * | ||
9 | * This program is distributed in the hope that it will be useful, | ||
10 | * but WITHOUT ANY WARRANTY; without even the implied warranty of | ||
11 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | ||
12 | * GNU General Public License for more details. | ||
13 | * | ||
14 | * You should have received a copy of the GNU General Public License | ||
15 | * along with this program; if not, write to the Free Software | ||
16 | * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. | ||
17 | * | ||
18 | * Copyright IBM Corporation, 2008 | ||
19 | * | ||
20 | * Author: Paul E. McKenney <paulmck@linux.vnet.ibm.com> | ||
21 | * | ||
22 | * For detailed explanation of Read-Copy Update mechanism see - | ||
23 | * Documentation/RCU | ||
24 | */ | ||
25 | #include <linux/moduleparam.h> | ||
26 | #include <linux/completion.h> | ||
27 | #include <linux/interrupt.h> | ||
28 | #include <linux/notifier.h> | ||
29 | #include <linux/rcupdate.h> | ||
30 | #include <linux/kernel.h> | ||
31 | #include <linux/module.h> | ||
32 | #include <linux/mutex.h> | ||
33 | #include <linux/sched.h> | ||
34 | #include <linux/types.h> | ||
35 | #include <linux/init.h> | ||
36 | #include <linux/time.h> | ||
37 | #include <linux/cpu.h> | ||
38 | |||
39 | /* Global control variables for rcupdate callback mechanism. */ | ||
40 | struct rcu_ctrlblk { | ||
41 | struct rcu_head *rcucblist; /* List of pending callbacks (CBs). */ | ||
42 | struct rcu_head **donetail; /* ->next pointer of last "done" CB. */ | ||
43 | struct rcu_head **curtail; /* ->next pointer of last CB. */ | ||
44 | }; | ||
45 | |||
46 | /* Definition for rcupdate control block. */ | ||
47 | static struct rcu_ctrlblk rcu_ctrlblk = { | ||
48 | .donetail = &rcu_ctrlblk.rcucblist, | ||
49 | .curtail = &rcu_ctrlblk.rcucblist, | ||
50 | }; | ||
51 | |||
52 | static struct rcu_ctrlblk rcu_bh_ctrlblk = { | ||
53 | .donetail = &rcu_bh_ctrlblk.rcucblist, | ||
54 | .curtail = &rcu_bh_ctrlblk.rcucblist, | ||
55 | }; | ||
56 | |||
57 | #ifdef CONFIG_NO_HZ | ||
58 | |||
59 | static long rcu_dynticks_nesting = 1; | ||
60 | |||
61 | /* | ||
62 | * Enter dynticks-idle mode, which is an extended quiescent state | ||
63 | * if we have fully entered that mode (i.e., if the new value of | ||
64 | * dynticks_nesting is zero). | ||
65 | */ | ||
66 | void rcu_enter_nohz(void) | ||
67 | { | ||
68 | if (--rcu_dynticks_nesting == 0) | ||
69 | rcu_sched_qs(0); /* implies rcu_bh_qsctr_inc(0) */ | ||
70 | } | ||
71 | |||
72 | /* | ||
73 | * Exit dynticks-idle mode, so that we are no longer in an extended | ||
74 | * quiescent state. | ||
75 | */ | ||
76 | void rcu_exit_nohz(void) | ||
77 | { | ||
78 | rcu_dynticks_nesting++; | ||
79 | } | ||
80 | |||
81 | #endif /* #ifdef CONFIG_NO_HZ */ | ||
82 | |||
83 | /* | ||
84 | * Helper function for rcu_qsctr_inc() and rcu_bh_qsctr_inc(). | ||
85 | * Also disable irqs to avoid confusion due to interrupt handlers | ||
86 | * invoking call_rcu(). | ||
87 | */ | ||
88 | static int rcu_qsctr_help(struct rcu_ctrlblk *rcp) | ||
89 | { | ||
90 | unsigned long flags; | ||
91 | |||
92 | local_irq_save(flags); | ||
93 | if (rcp->rcucblist != NULL && | ||
94 | rcp->donetail != rcp->curtail) { | ||
95 | rcp->donetail = rcp->curtail; | ||
96 | local_irq_restore(flags); | ||
97 | return 1; | ||
98 | } | ||
99 | local_irq_restore(flags); | ||
100 | |||
101 | return 0; | ||
102 | } | ||
103 | |||
104 | /* | ||
105 | * Record an rcu quiescent state. And an rcu_bh quiescent state while we | ||
106 | * are at it, given that any rcu quiescent state is also an rcu_bh | ||
107 | * quiescent state. Use "+" instead of "||" to defeat short circuiting. | ||
108 | */ | ||
109 | void rcu_sched_qs(int cpu) | ||
110 | { | ||
111 | if (rcu_qsctr_help(&rcu_ctrlblk) + rcu_qsctr_help(&rcu_bh_ctrlblk)) | ||
112 | raise_softirq(RCU_SOFTIRQ); | ||
113 | } | ||
114 | |||
115 | /* | ||
116 | * Record an rcu_bh quiescent state. | ||
117 | */ | ||
118 | void rcu_bh_qs(int cpu) | ||
119 | { | ||
120 | if (rcu_qsctr_help(&rcu_bh_ctrlblk)) | ||
121 | raise_softirq(RCU_SOFTIRQ); | ||
122 | } | ||
123 | |||
124 | /* | ||
125 | * Check to see if the scheduling-clock interrupt came from an extended | ||
126 | * quiescent state, and, if so, tell RCU about it. | ||
127 | */ | ||
128 | void rcu_check_callbacks(int cpu, int user) | ||
129 | { | ||
130 | if (user || | ||
131 | (idle_cpu(cpu) && | ||
132 | !in_softirq() && | ||
133 | hardirq_count() <= (1 << HARDIRQ_SHIFT))) | ||
134 | rcu_sched_qs(cpu); | ||
135 | else if (!in_softirq()) | ||
136 | rcu_bh_qs(cpu); | ||
137 | } | ||
138 | |||
139 | /* | ||
140 | * Helper function for rcu_process_callbacks() that operates on the | ||
141 | * specified rcu_ctrlkblk structure. | ||
142 | */ | ||
143 | static void __rcu_process_callbacks(struct rcu_ctrlblk *rcp) | ||
144 | { | ||
145 | struct rcu_head *next, *list; | ||
146 | unsigned long flags; | ||
147 | |||
148 | /* If no RCU callbacks ready to invoke, just return. */ | ||
149 | if (&rcp->rcucblist == rcp->donetail) | ||
150 | return; | ||
151 | |||
152 | /* Move the ready-to-invoke callbacks to a local list. */ | ||
153 | local_irq_save(flags); | ||
154 | list = rcp->rcucblist; | ||
155 | rcp->rcucblist = *rcp->donetail; | ||
156 | *rcp->donetail = NULL; | ||
157 | if (rcp->curtail == rcp->donetail) | ||
158 | rcp->curtail = &rcp->rcucblist; | ||
159 | rcp->donetail = &rcp->rcucblist; | ||
160 | local_irq_restore(flags); | ||
161 | |||
162 | /* Invoke the callbacks on the local list. */ | ||
163 | while (list) { | ||
164 | next = list->next; | ||
165 | prefetch(next); | ||
166 | list->func(list); | ||
167 | list = next; | ||
168 | } | ||
169 | } | ||
170 | |||
171 | /* | ||
172 | * Invoke any callbacks whose grace period has completed. | ||
173 | */ | ||
174 | static void rcu_process_callbacks(struct softirq_action *unused) | ||
175 | { | ||
176 | __rcu_process_callbacks(&rcu_ctrlblk); | ||
177 | __rcu_process_callbacks(&rcu_bh_ctrlblk); | ||
178 | } | ||
179 | |||
180 | /* | ||
181 | * Wait for a grace period to elapse. But it is illegal to invoke | ||
182 | * synchronize_sched() from within an RCU read-side critical section. | ||
183 | * Therefore, any legal call to synchronize_sched() is a quiescent | ||
184 | * state, and so on a UP system, synchronize_sched() need do nothing. | ||
185 | * Ditto for synchronize_rcu_bh(). (But Lai Jiangshan points out the | ||
186 | * benefits of doing might_sleep() to reduce latency.) | ||
187 | * | ||
188 | * Cool, huh? (Due to Josh Triplett.) | ||
189 | * | ||
190 | * But we want to make this a static inline later. | ||
191 | */ | ||
192 | void synchronize_sched(void) | ||
193 | { | ||
194 | cond_resched(); | ||
195 | } | ||
196 | EXPORT_SYMBOL_GPL(synchronize_sched); | ||
197 | |||
198 | void synchronize_rcu_bh(void) | ||
199 | { | ||
200 | synchronize_sched(); | ||
201 | } | ||
202 | EXPORT_SYMBOL_GPL(synchronize_rcu_bh); | ||
203 | |||
204 | /* | ||
205 | * Helper function for call_rcu() and call_rcu_bh(). | ||
206 | */ | ||
207 | static void __call_rcu(struct rcu_head *head, | ||
208 | void (*func)(struct rcu_head *rcu), | ||
209 | struct rcu_ctrlblk *rcp) | ||
210 | { | ||
211 | unsigned long flags; | ||
212 | |||
213 | head->func = func; | ||
214 | head->next = NULL; | ||
215 | |||
216 | local_irq_save(flags); | ||
217 | *rcp->curtail = head; | ||
218 | rcp->curtail = &head->next; | ||
219 | local_irq_restore(flags); | ||
220 | } | ||
221 | |||
222 | /* | ||
223 | * Post an RCU callback to be invoked after the end of an RCU grace | ||
224 | * period. But since we have but one CPU, that would be after any | ||
225 | * quiescent state. | ||
226 | */ | ||
227 | void call_rcu(struct rcu_head *head, void (*func)(struct rcu_head *rcu)) | ||
228 | { | ||
229 | __call_rcu(head, func, &rcu_ctrlblk); | ||
230 | } | ||
231 | EXPORT_SYMBOL_GPL(call_rcu); | ||
232 | |||
233 | /* | ||
234 | * Post an RCU bottom-half callback to be invoked after any subsequent | ||
235 | * quiescent state. | ||
236 | */ | ||
237 | void call_rcu_bh(struct rcu_head *head, void (*func)(struct rcu_head *rcu)) | ||
238 | { | ||
239 | __call_rcu(head, func, &rcu_bh_ctrlblk); | ||
240 | } | ||
241 | EXPORT_SYMBOL_GPL(call_rcu_bh); | ||
242 | |||
243 | void rcu_barrier(void) | ||
244 | { | ||
245 | struct rcu_synchronize rcu; | ||
246 | |||
247 | init_completion(&rcu.completion); | ||
248 | /* Will wake me after RCU finished. */ | ||
249 | call_rcu(&rcu.head, wakeme_after_rcu); | ||
250 | /* Wait for it. */ | ||
251 | wait_for_completion(&rcu.completion); | ||
252 | } | ||
253 | EXPORT_SYMBOL_GPL(rcu_barrier); | ||
254 | |||
255 | void rcu_barrier_bh(void) | ||
256 | { | ||
257 | struct rcu_synchronize rcu; | ||
258 | |||
259 | init_completion(&rcu.completion); | ||
260 | /* Will wake me after RCU finished. */ | ||
261 | call_rcu_bh(&rcu.head, wakeme_after_rcu); | ||
262 | /* Wait for it. */ | ||
263 | wait_for_completion(&rcu.completion); | ||
264 | } | ||
265 | EXPORT_SYMBOL_GPL(rcu_barrier_bh); | ||
266 | |||
267 | void rcu_barrier_sched(void) | ||
268 | { | ||
269 | struct rcu_synchronize rcu; | ||
270 | |||
271 | init_completion(&rcu.completion); | ||
272 | /* Will wake me after RCU finished. */ | ||
273 | call_rcu_sched(&rcu.head, wakeme_after_rcu); | ||
274 | /* Wait for it. */ | ||
275 | wait_for_completion(&rcu.completion); | ||
276 | } | ||
277 | EXPORT_SYMBOL_GPL(rcu_barrier_sched); | ||
278 | |||
279 | void __init rcu_init(void) | ||
280 | { | ||
281 | open_softirq(RCU_SOFTIRQ, rcu_process_callbacks); | ||
282 | } | ||
diff --git a/kernel/rcutorture.c b/kernel/rcutorture.c index 697c0a0229d4..a621a67ef4e3 100644 --- a/kernel/rcutorture.c +++ b/kernel/rcutorture.c | |||
@@ -327,6 +327,11 @@ rcu_torture_cb(struct rcu_head *p) | |||
327 | cur_ops->deferred_free(rp); | 327 | cur_ops->deferred_free(rp); |
328 | } | 328 | } |
329 | 329 | ||
330 | static int rcu_no_completed(void) | ||
331 | { | ||
332 | return 0; | ||
333 | } | ||
334 | |||
330 | static void rcu_torture_deferred_free(struct rcu_torture *p) | 335 | static void rcu_torture_deferred_free(struct rcu_torture *p) |
331 | { | 336 | { |
332 | call_rcu(&p->rtort_rcu, rcu_torture_cb); | 337 | call_rcu(&p->rtort_rcu, rcu_torture_cb); |
@@ -388,6 +393,21 @@ static struct rcu_torture_ops rcu_sync_ops = { | |||
388 | .name = "rcu_sync" | 393 | .name = "rcu_sync" |
389 | }; | 394 | }; |
390 | 395 | ||
396 | static struct rcu_torture_ops rcu_expedited_ops = { | ||
397 | .init = rcu_sync_torture_init, | ||
398 | .cleanup = NULL, | ||
399 | .readlock = rcu_torture_read_lock, | ||
400 | .read_delay = rcu_read_delay, /* just reuse rcu's version. */ | ||
401 | .readunlock = rcu_torture_read_unlock, | ||
402 | .completed = rcu_no_completed, | ||
403 | .deferred_free = rcu_sync_torture_deferred_free, | ||
404 | .sync = synchronize_rcu_expedited, | ||
405 | .cb_barrier = NULL, | ||
406 | .stats = NULL, | ||
407 | .irq_capable = 1, | ||
408 | .name = "rcu_expedited" | ||
409 | }; | ||
410 | |||
391 | /* | 411 | /* |
392 | * Definitions for rcu_bh torture testing. | 412 | * Definitions for rcu_bh torture testing. |
393 | */ | 413 | */ |
@@ -547,6 +567,25 @@ static struct rcu_torture_ops srcu_ops = { | |||
547 | .name = "srcu" | 567 | .name = "srcu" |
548 | }; | 568 | }; |
549 | 569 | ||
570 | static void srcu_torture_synchronize_expedited(void) | ||
571 | { | ||
572 | synchronize_srcu_expedited(&srcu_ctl); | ||
573 | } | ||
574 | |||
575 | static struct rcu_torture_ops srcu_expedited_ops = { | ||
576 | .init = srcu_torture_init, | ||
577 | .cleanup = srcu_torture_cleanup, | ||
578 | .readlock = srcu_torture_read_lock, | ||
579 | .read_delay = srcu_read_delay, | ||
580 | .readunlock = srcu_torture_read_unlock, | ||
581 | .completed = srcu_torture_completed, | ||
582 | .deferred_free = rcu_sync_torture_deferred_free, | ||
583 | .sync = srcu_torture_synchronize_expedited, | ||
584 | .cb_barrier = NULL, | ||
585 | .stats = srcu_torture_stats, | ||
586 | .name = "srcu_expedited" | ||
587 | }; | ||
588 | |||
550 | /* | 589 | /* |
551 | * Definitions for sched torture testing. | 590 | * Definitions for sched torture testing. |
552 | */ | 591 | */ |
@@ -562,11 +601,6 @@ static void sched_torture_read_unlock(int idx) | |||
562 | preempt_enable(); | 601 | preempt_enable(); |
563 | } | 602 | } |
564 | 603 | ||
565 | static int sched_torture_completed(void) | ||
566 | { | ||
567 | return 0; | ||
568 | } | ||
569 | |||
570 | static void rcu_sched_torture_deferred_free(struct rcu_torture *p) | 604 | static void rcu_sched_torture_deferred_free(struct rcu_torture *p) |
571 | { | 605 | { |
572 | call_rcu_sched(&p->rtort_rcu, rcu_torture_cb); | 606 | call_rcu_sched(&p->rtort_rcu, rcu_torture_cb); |
@@ -583,7 +617,7 @@ static struct rcu_torture_ops sched_ops = { | |||
583 | .readlock = sched_torture_read_lock, | 617 | .readlock = sched_torture_read_lock, |
584 | .read_delay = rcu_read_delay, /* just reuse rcu's version. */ | 618 | .read_delay = rcu_read_delay, /* just reuse rcu's version. */ |
585 | .readunlock = sched_torture_read_unlock, | 619 | .readunlock = sched_torture_read_unlock, |
586 | .completed = sched_torture_completed, | 620 | .completed = rcu_no_completed, |
587 | .deferred_free = rcu_sched_torture_deferred_free, | 621 | .deferred_free = rcu_sched_torture_deferred_free, |
588 | .sync = sched_torture_synchronize, | 622 | .sync = sched_torture_synchronize, |
589 | .cb_barrier = rcu_barrier_sched, | 623 | .cb_barrier = rcu_barrier_sched, |
@@ -592,13 +626,13 @@ static struct rcu_torture_ops sched_ops = { | |||
592 | .name = "sched" | 626 | .name = "sched" |
593 | }; | 627 | }; |
594 | 628 | ||
595 | static struct rcu_torture_ops sched_ops_sync = { | 629 | static struct rcu_torture_ops sched_sync_ops = { |
596 | .init = rcu_sync_torture_init, | 630 | .init = rcu_sync_torture_init, |
597 | .cleanup = NULL, | 631 | .cleanup = NULL, |
598 | .readlock = sched_torture_read_lock, | 632 | .readlock = sched_torture_read_lock, |
599 | .read_delay = rcu_read_delay, /* just reuse rcu's version. */ | 633 | .read_delay = rcu_read_delay, /* just reuse rcu's version. */ |
600 | .readunlock = sched_torture_read_unlock, | 634 | .readunlock = sched_torture_read_unlock, |
601 | .completed = sched_torture_completed, | 635 | .completed = rcu_no_completed, |
602 | .deferred_free = rcu_sync_torture_deferred_free, | 636 | .deferred_free = rcu_sync_torture_deferred_free, |
603 | .sync = sched_torture_synchronize, | 637 | .sync = sched_torture_synchronize, |
604 | .cb_barrier = NULL, | 638 | .cb_barrier = NULL, |
@@ -612,7 +646,7 @@ static struct rcu_torture_ops sched_expedited_ops = { | |||
612 | .readlock = sched_torture_read_lock, | 646 | .readlock = sched_torture_read_lock, |
613 | .read_delay = rcu_read_delay, /* just reuse rcu's version. */ | 647 | .read_delay = rcu_read_delay, /* just reuse rcu's version. */ |
614 | .readunlock = sched_torture_read_unlock, | 648 | .readunlock = sched_torture_read_unlock, |
615 | .completed = sched_torture_completed, | 649 | .completed = rcu_no_completed, |
616 | .deferred_free = rcu_sync_torture_deferred_free, | 650 | .deferred_free = rcu_sync_torture_deferred_free, |
617 | .sync = synchronize_sched_expedited, | 651 | .sync = synchronize_sched_expedited, |
618 | .cb_barrier = NULL, | 652 | .cb_barrier = NULL, |
@@ -1097,9 +1131,10 @@ rcu_torture_init(void) | |||
1097 | int cpu; | 1131 | int cpu; |
1098 | int firsterr = 0; | 1132 | int firsterr = 0; |
1099 | static struct rcu_torture_ops *torture_ops[] = | 1133 | static struct rcu_torture_ops *torture_ops[] = |
1100 | { &rcu_ops, &rcu_sync_ops, &rcu_bh_ops, &rcu_bh_sync_ops, | 1134 | { &rcu_ops, &rcu_sync_ops, &rcu_expedited_ops, |
1101 | &sched_expedited_ops, | 1135 | &rcu_bh_ops, &rcu_bh_sync_ops, |
1102 | &srcu_ops, &sched_ops, &sched_ops_sync, }; | 1136 | &srcu_ops, &srcu_expedited_ops, |
1137 | &sched_ops, &sched_sync_ops, &sched_expedited_ops, }; | ||
1103 | 1138 | ||
1104 | mutex_lock(&fullstop_mutex); | 1139 | mutex_lock(&fullstop_mutex); |
1105 | 1140 | ||
@@ -1110,8 +1145,12 @@ rcu_torture_init(void) | |||
1110 | break; | 1145 | break; |
1111 | } | 1146 | } |
1112 | if (i == ARRAY_SIZE(torture_ops)) { | 1147 | if (i == ARRAY_SIZE(torture_ops)) { |
1113 | printk(KERN_ALERT "rcutorture: invalid torture type: \"%s\"\n", | 1148 | printk(KERN_ALERT "rcu-torture: invalid torture type: \"%s\"\n", |
1114 | torture_type); | 1149 | torture_type); |
1150 | printk(KERN_ALERT "rcu-torture types:"); | ||
1151 | for (i = 0; i < ARRAY_SIZE(torture_ops); i++) | ||
1152 | printk(KERN_ALERT " %s", torture_ops[i]->name); | ||
1153 | printk(KERN_ALERT "\n"); | ||
1115 | mutex_unlock(&fullstop_mutex); | 1154 | mutex_unlock(&fullstop_mutex); |
1116 | return -EINVAL; | 1155 | return -EINVAL; |
1117 | } | 1156 | } |
diff --git a/kernel/rcutree.c b/kernel/rcutree.c index 705f02ac7433..53ae9598f798 100644 --- a/kernel/rcutree.c +++ b/kernel/rcutree.c | |||
@@ -46,20 +46,24 @@ | |||
46 | #include <linux/cpu.h> | 46 | #include <linux/cpu.h> |
47 | #include <linux/mutex.h> | 47 | #include <linux/mutex.h> |
48 | #include <linux/time.h> | 48 | #include <linux/time.h> |
49 | #include <linux/kernel_stat.h> | ||
49 | 50 | ||
50 | #include "rcutree.h" | 51 | #include "rcutree.h" |
51 | 52 | ||
52 | /* Data structures. */ | 53 | /* Data structures. */ |
53 | 54 | ||
55 | static struct lock_class_key rcu_node_class[NUM_RCU_LVLS]; | ||
56 | |||
54 | #define RCU_STATE_INITIALIZER(name) { \ | 57 | #define RCU_STATE_INITIALIZER(name) { \ |
55 | .level = { &name.node[0] }, \ | 58 | .level = { &name.node[0] }, \ |
56 | .levelcnt = { \ | 59 | .levelcnt = { \ |
57 | NUM_RCU_LVL_0, /* root of hierarchy. */ \ | 60 | NUM_RCU_LVL_0, /* root of hierarchy. */ \ |
58 | NUM_RCU_LVL_1, \ | 61 | NUM_RCU_LVL_1, \ |
59 | NUM_RCU_LVL_2, \ | 62 | NUM_RCU_LVL_2, \ |
60 | NUM_RCU_LVL_3, /* == MAX_RCU_LVLS */ \ | 63 | NUM_RCU_LVL_3, \ |
64 | NUM_RCU_LVL_4, /* == MAX_RCU_LVLS */ \ | ||
61 | }, \ | 65 | }, \ |
62 | .signaled = RCU_SIGNAL_INIT, \ | 66 | .signaled = RCU_GP_IDLE, \ |
63 | .gpnum = -300, \ | 67 | .gpnum = -300, \ |
64 | .completed = -300, \ | 68 | .completed = -300, \ |
65 | .onofflock = __SPIN_LOCK_UNLOCKED(&name.onofflock), \ | 69 | .onofflock = __SPIN_LOCK_UNLOCKED(&name.onofflock), \ |
@@ -77,6 +81,8 @@ DEFINE_PER_CPU(struct rcu_data, rcu_sched_data); | |||
77 | struct rcu_state rcu_bh_state = RCU_STATE_INITIALIZER(rcu_bh_state); | 81 | struct rcu_state rcu_bh_state = RCU_STATE_INITIALIZER(rcu_bh_state); |
78 | DEFINE_PER_CPU(struct rcu_data, rcu_bh_data); | 82 | DEFINE_PER_CPU(struct rcu_data, rcu_bh_data); |
79 | 83 | ||
84 | static int rcu_scheduler_active __read_mostly; | ||
85 | |||
80 | 86 | ||
81 | /* | 87 | /* |
82 | * Return true if an RCU grace period is in progress. The ACCESS_ONCE()s | 88 | * Return true if an RCU grace period is in progress. The ACCESS_ONCE()s |
@@ -98,7 +104,7 @@ void rcu_sched_qs(int cpu) | |||
98 | struct rcu_data *rdp; | 104 | struct rcu_data *rdp; |
99 | 105 | ||
100 | rdp = &per_cpu(rcu_sched_data, cpu); | 106 | rdp = &per_cpu(rcu_sched_data, cpu); |
101 | rdp->passed_quiesc_completed = rdp->completed; | 107 | rdp->passed_quiesc_completed = rdp->gpnum - 1; |
102 | barrier(); | 108 | barrier(); |
103 | rdp->passed_quiesc = 1; | 109 | rdp->passed_quiesc = 1; |
104 | rcu_preempt_note_context_switch(cpu); | 110 | rcu_preempt_note_context_switch(cpu); |
@@ -109,7 +115,7 @@ void rcu_bh_qs(int cpu) | |||
109 | struct rcu_data *rdp; | 115 | struct rcu_data *rdp; |
110 | 116 | ||
111 | rdp = &per_cpu(rcu_bh_data, cpu); | 117 | rdp = &per_cpu(rcu_bh_data, cpu); |
112 | rdp->passed_quiesc_completed = rdp->completed; | 118 | rdp->passed_quiesc_completed = rdp->gpnum - 1; |
113 | barrier(); | 119 | barrier(); |
114 | rdp->passed_quiesc = 1; | 120 | rdp->passed_quiesc = 1; |
115 | } | 121 | } |
@@ -335,28 +341,9 @@ void rcu_irq_exit(void) | |||
335 | set_need_resched(); | 341 | set_need_resched(); |
336 | } | 342 | } |
337 | 343 | ||
338 | /* | ||
339 | * Record the specified "completed" value, which is later used to validate | ||
340 | * dynticks counter manipulations. Specify "rsp->completed - 1" to | ||
341 | * unconditionally invalidate any future dynticks manipulations (which is | ||
342 | * useful at the beginning of a grace period). | ||
343 | */ | ||
344 | static void dyntick_record_completed(struct rcu_state *rsp, long comp) | ||
345 | { | ||
346 | rsp->dynticks_completed = comp; | ||
347 | } | ||
348 | |||
349 | #ifdef CONFIG_SMP | 344 | #ifdef CONFIG_SMP |
350 | 345 | ||
351 | /* | 346 | /* |
352 | * Recall the previously recorded value of the completion for dynticks. | ||
353 | */ | ||
354 | static long dyntick_recall_completed(struct rcu_state *rsp) | ||
355 | { | ||
356 | return rsp->dynticks_completed; | ||
357 | } | ||
358 | |||
359 | /* | ||
360 | * Snapshot the specified CPU's dynticks counter so that we can later | 347 | * Snapshot the specified CPU's dynticks counter so that we can later |
361 | * credit them with an implicit quiescent state. Return 1 if this CPU | 348 | * credit them with an implicit quiescent state. Return 1 if this CPU |
362 | * is in dynticks idle mode, which is an extended quiescent state. | 349 | * is in dynticks idle mode, which is an extended quiescent state. |
@@ -419,24 +406,8 @@ static int rcu_implicit_dynticks_qs(struct rcu_data *rdp) | |||
419 | 406 | ||
420 | #else /* #ifdef CONFIG_NO_HZ */ | 407 | #else /* #ifdef CONFIG_NO_HZ */ |
421 | 408 | ||
422 | static void dyntick_record_completed(struct rcu_state *rsp, long comp) | ||
423 | { | ||
424 | } | ||
425 | |||
426 | #ifdef CONFIG_SMP | 409 | #ifdef CONFIG_SMP |
427 | 410 | ||
428 | /* | ||
429 | * If there are no dynticks, then the only way that a CPU can passively | ||
430 | * be in a quiescent state is to be offline. Unlike dynticks idle, which | ||
431 | * is a point in time during the prior (already finished) grace period, | ||
432 | * an offline CPU is always in a quiescent state, and thus can be | ||
433 | * unconditionally applied. So just return the current value of completed. | ||
434 | */ | ||
435 | static long dyntick_recall_completed(struct rcu_state *rsp) | ||
436 | { | ||
437 | return rsp->completed; | ||
438 | } | ||
439 | |||
440 | static int dyntick_save_progress_counter(struct rcu_data *rdp) | 411 | static int dyntick_save_progress_counter(struct rcu_data *rdp) |
441 | { | 412 | { |
442 | return 0; | 413 | return 0; |
@@ -553,13 +524,33 @@ static void check_cpu_stall(struct rcu_state *rsp, struct rcu_data *rdp) | |||
553 | /* | 524 | /* |
554 | * Update CPU-local rcu_data state to record the newly noticed grace period. | 525 | * Update CPU-local rcu_data state to record the newly noticed grace period. |
555 | * This is used both when we started the grace period and when we notice | 526 | * This is used both when we started the grace period and when we notice |
556 | * that someone else started the grace period. | 527 | * that someone else started the grace period. The caller must hold the |
528 | * ->lock of the leaf rcu_node structure corresponding to the current CPU, | ||
529 | * and must have irqs disabled. | ||
557 | */ | 530 | */ |
531 | static void __note_new_gpnum(struct rcu_state *rsp, struct rcu_node *rnp, struct rcu_data *rdp) | ||
532 | { | ||
533 | if (rdp->gpnum != rnp->gpnum) { | ||
534 | rdp->qs_pending = 1; | ||
535 | rdp->passed_quiesc = 0; | ||
536 | rdp->gpnum = rnp->gpnum; | ||
537 | } | ||
538 | } | ||
539 | |||
558 | static void note_new_gpnum(struct rcu_state *rsp, struct rcu_data *rdp) | 540 | static void note_new_gpnum(struct rcu_state *rsp, struct rcu_data *rdp) |
559 | { | 541 | { |
560 | rdp->qs_pending = 1; | 542 | unsigned long flags; |
561 | rdp->passed_quiesc = 0; | 543 | struct rcu_node *rnp; |
562 | rdp->gpnum = rsp->gpnum; | 544 | |
545 | local_irq_save(flags); | ||
546 | rnp = rdp->mynode; | ||
547 | if (rdp->gpnum == ACCESS_ONCE(rnp->gpnum) || /* outside lock. */ | ||
548 | !spin_trylock(&rnp->lock)) { /* irqs already off, retry later. */ | ||
549 | local_irq_restore(flags); | ||
550 | return; | ||
551 | } | ||
552 | __note_new_gpnum(rsp, rnp, rdp); | ||
553 | spin_unlock_irqrestore(&rnp->lock, flags); | ||
563 | } | 554 | } |
564 | 555 | ||
565 | /* | 556 | /* |
@@ -583,6 +574,79 @@ check_for_new_grace_period(struct rcu_state *rsp, struct rcu_data *rdp) | |||
583 | } | 574 | } |
584 | 575 | ||
585 | /* | 576 | /* |
577 | * Advance this CPU's callbacks, but only if the current grace period | ||
578 | * has ended. This may be called only from the CPU to whom the rdp | ||
579 | * belongs. In addition, the corresponding leaf rcu_node structure's | ||
580 | * ->lock must be held by the caller, with irqs disabled. | ||
581 | */ | ||
582 | static void | ||
583 | __rcu_process_gp_end(struct rcu_state *rsp, struct rcu_node *rnp, struct rcu_data *rdp) | ||
584 | { | ||
585 | /* Did another grace period end? */ | ||
586 | if (rdp->completed != rnp->completed) { | ||
587 | |||
588 | /* Advance callbacks. No harm if list empty. */ | ||
589 | rdp->nxttail[RCU_DONE_TAIL] = rdp->nxttail[RCU_WAIT_TAIL]; | ||
590 | rdp->nxttail[RCU_WAIT_TAIL] = rdp->nxttail[RCU_NEXT_READY_TAIL]; | ||
591 | rdp->nxttail[RCU_NEXT_READY_TAIL] = rdp->nxttail[RCU_NEXT_TAIL]; | ||
592 | |||
593 | /* Remember that we saw this grace-period completion. */ | ||
594 | rdp->completed = rnp->completed; | ||
595 | } | ||
596 | } | ||
597 | |||
598 | /* | ||
599 | * Advance this CPU's callbacks, but only if the current grace period | ||
600 | * has ended. This may be called only from the CPU to whom the rdp | ||
601 | * belongs. | ||
602 | */ | ||
603 | static void | ||
604 | rcu_process_gp_end(struct rcu_state *rsp, struct rcu_data *rdp) | ||
605 | { | ||
606 | unsigned long flags; | ||
607 | struct rcu_node *rnp; | ||
608 | |||
609 | local_irq_save(flags); | ||
610 | rnp = rdp->mynode; | ||
611 | if (rdp->completed == ACCESS_ONCE(rnp->completed) || /* outside lock. */ | ||
612 | !spin_trylock(&rnp->lock)) { /* irqs already off, retry later. */ | ||
613 | local_irq_restore(flags); | ||
614 | return; | ||
615 | } | ||
616 | __rcu_process_gp_end(rsp, rnp, rdp); | ||
617 | spin_unlock_irqrestore(&rnp->lock, flags); | ||
618 | } | ||
619 | |||
620 | /* | ||
621 | * Do per-CPU grace-period initialization for running CPU. The caller | ||
622 | * must hold the lock of the leaf rcu_node structure corresponding to | ||
623 | * this CPU. | ||
624 | */ | ||
625 | static void | ||
626 | rcu_start_gp_per_cpu(struct rcu_state *rsp, struct rcu_node *rnp, struct rcu_data *rdp) | ||
627 | { | ||
628 | /* Prior grace period ended, so advance callbacks for current CPU. */ | ||
629 | __rcu_process_gp_end(rsp, rnp, rdp); | ||
630 | |||
631 | /* | ||
632 | * Because this CPU just now started the new grace period, we know | ||
633 | * that all of its callbacks will be covered by this upcoming grace | ||
634 | * period, even the ones that were registered arbitrarily recently. | ||
635 | * Therefore, advance all outstanding callbacks to RCU_WAIT_TAIL. | ||
636 | * | ||
637 | * Other CPUs cannot be sure exactly when the grace period started. | ||
638 | * Therefore, their recently registered callbacks must pass through | ||
639 | * an additional RCU_NEXT_READY stage, so that they will be handled | ||
640 | * by the next RCU grace period. | ||
641 | */ | ||
642 | rdp->nxttail[RCU_NEXT_READY_TAIL] = rdp->nxttail[RCU_NEXT_TAIL]; | ||
643 | rdp->nxttail[RCU_WAIT_TAIL] = rdp->nxttail[RCU_NEXT_TAIL]; | ||
644 | |||
645 | /* Set state so that this CPU will detect the next quiescent state. */ | ||
646 | __note_new_gpnum(rsp, rnp, rdp); | ||
647 | } | ||
648 | |||
649 | /* | ||
586 | * Start a new RCU grace period if warranted, re-initializing the hierarchy | 650 | * Start a new RCU grace period if warranted, re-initializing the hierarchy |
587 | * in preparation for detecting the next grace period. The caller must hold | 651 | * in preparation for detecting the next grace period. The caller must hold |
588 | * the root node's ->lock, which is released before return. Hard irqs must | 652 | * the root node's ->lock, which is released before return. Hard irqs must |
@@ -596,7 +660,23 @@ rcu_start_gp(struct rcu_state *rsp, unsigned long flags) | |||
596 | struct rcu_node *rnp = rcu_get_root(rsp); | 660 | struct rcu_node *rnp = rcu_get_root(rsp); |
597 | 661 | ||
598 | if (!cpu_needs_another_gp(rsp, rdp)) { | 662 | if (!cpu_needs_another_gp(rsp, rdp)) { |
599 | spin_unlock_irqrestore(&rnp->lock, flags); | 663 | if (rnp->completed == rsp->completed) { |
664 | spin_unlock_irqrestore(&rnp->lock, flags); | ||
665 | return; | ||
666 | } | ||
667 | spin_unlock(&rnp->lock); /* irqs remain disabled. */ | ||
668 | |||
669 | /* | ||
670 | * Propagate new ->completed value to rcu_node structures | ||
671 | * so that other CPUs don't have to wait until the start | ||
672 | * of the next grace period to process their callbacks. | ||
673 | */ | ||
674 | rcu_for_each_node_breadth_first(rsp, rnp) { | ||
675 | spin_lock(&rnp->lock); /* irqs already disabled. */ | ||
676 | rnp->completed = rsp->completed; | ||
677 | spin_unlock(&rnp->lock); /* irqs remain disabled. */ | ||
678 | } | ||
679 | local_irq_restore(flags); | ||
600 | return; | 680 | return; |
601 | } | 681 | } |
602 | 682 | ||
@@ -606,29 +686,15 @@ rcu_start_gp(struct rcu_state *rsp, unsigned long flags) | |||
606 | rsp->signaled = RCU_GP_INIT; /* Hold off force_quiescent_state. */ | 686 | rsp->signaled = RCU_GP_INIT; /* Hold off force_quiescent_state. */ |
607 | rsp->jiffies_force_qs = jiffies + RCU_JIFFIES_TILL_FORCE_QS; | 687 | rsp->jiffies_force_qs = jiffies + RCU_JIFFIES_TILL_FORCE_QS; |
608 | record_gp_stall_check_time(rsp); | 688 | record_gp_stall_check_time(rsp); |
609 | dyntick_record_completed(rsp, rsp->completed - 1); | ||
610 | note_new_gpnum(rsp, rdp); | ||
611 | |||
612 | /* | ||
613 | * Because this CPU just now started the new grace period, we know | ||
614 | * that all of its callbacks will be covered by this upcoming grace | ||
615 | * period, even the ones that were registered arbitrarily recently. | ||
616 | * Therefore, advance all outstanding callbacks to RCU_WAIT_TAIL. | ||
617 | * | ||
618 | * Other CPUs cannot be sure exactly when the grace period started. | ||
619 | * Therefore, their recently registered callbacks must pass through | ||
620 | * an additional RCU_NEXT_READY stage, so that they will be handled | ||
621 | * by the next RCU grace period. | ||
622 | */ | ||
623 | rdp->nxttail[RCU_NEXT_READY_TAIL] = rdp->nxttail[RCU_NEXT_TAIL]; | ||
624 | rdp->nxttail[RCU_WAIT_TAIL] = rdp->nxttail[RCU_NEXT_TAIL]; | ||
625 | 689 | ||
626 | /* Special-case the common single-level case. */ | 690 | /* Special-case the common single-level case. */ |
627 | if (NUM_RCU_NODES == 1) { | 691 | if (NUM_RCU_NODES == 1) { |
628 | rcu_preempt_check_blocked_tasks(rnp); | 692 | rcu_preempt_check_blocked_tasks(rnp); |
629 | rnp->qsmask = rnp->qsmaskinit; | 693 | rnp->qsmask = rnp->qsmaskinit; |
630 | rnp->gpnum = rsp->gpnum; | 694 | rnp->gpnum = rsp->gpnum; |
695 | rnp->completed = rsp->completed; | ||
631 | rsp->signaled = RCU_SIGNAL_INIT; /* force_quiescent_state OK. */ | 696 | rsp->signaled = RCU_SIGNAL_INIT; /* force_quiescent_state OK. */ |
697 | rcu_start_gp_per_cpu(rsp, rnp, rdp); | ||
632 | spin_unlock_irqrestore(&rnp->lock, flags); | 698 | spin_unlock_irqrestore(&rnp->lock, flags); |
633 | return; | 699 | return; |
634 | } | 700 | } |
@@ -657,69 +723,50 @@ rcu_start_gp(struct rcu_state *rsp, unsigned long flags) | |||
657 | * irqs disabled. | 723 | * irqs disabled. |
658 | */ | 724 | */ |
659 | rcu_for_each_node_breadth_first(rsp, rnp) { | 725 | rcu_for_each_node_breadth_first(rsp, rnp) { |
660 | spin_lock(&rnp->lock); /* irqs already disabled. */ | 726 | spin_lock(&rnp->lock); /* irqs already disabled. */ |
661 | rcu_preempt_check_blocked_tasks(rnp); | 727 | rcu_preempt_check_blocked_tasks(rnp); |
662 | rnp->qsmask = rnp->qsmaskinit; | 728 | rnp->qsmask = rnp->qsmaskinit; |
663 | rnp->gpnum = rsp->gpnum; | 729 | rnp->gpnum = rsp->gpnum; |
664 | spin_unlock(&rnp->lock); /* irqs already disabled. */ | 730 | rnp->completed = rsp->completed; |
731 | if (rnp == rdp->mynode) | ||
732 | rcu_start_gp_per_cpu(rsp, rnp, rdp); | ||
733 | spin_unlock(&rnp->lock); /* irqs remain disabled. */ | ||
665 | } | 734 | } |
666 | 735 | ||
736 | rnp = rcu_get_root(rsp); | ||
737 | spin_lock(&rnp->lock); /* irqs already disabled. */ | ||
667 | rsp->signaled = RCU_SIGNAL_INIT; /* force_quiescent_state now OK. */ | 738 | rsp->signaled = RCU_SIGNAL_INIT; /* force_quiescent_state now OK. */ |
739 | spin_unlock(&rnp->lock); /* irqs remain disabled. */ | ||
668 | spin_unlock_irqrestore(&rsp->onofflock, flags); | 740 | spin_unlock_irqrestore(&rsp->onofflock, flags); |
669 | } | 741 | } |
670 | 742 | ||
671 | /* | 743 | /* |
672 | * Advance this CPU's callbacks, but only if the current grace period | 744 | * Report a full set of quiescent states to the specified rcu_state |
673 | * has ended. This may be called only from the CPU to whom the rdp | 745 | * data structure. This involves cleaning up after the prior grace |
674 | * belongs. | 746 | * period and letting rcu_start_gp() start up the next grace period |
675 | */ | 747 | * if one is needed. Note that the caller must hold rnp->lock, as |
676 | static void | 748 | * required by rcu_start_gp(), which will release it. |
677 | rcu_process_gp_end(struct rcu_state *rsp, struct rcu_data *rdp) | ||
678 | { | ||
679 | long completed_snap; | ||
680 | unsigned long flags; | ||
681 | |||
682 | local_irq_save(flags); | ||
683 | completed_snap = ACCESS_ONCE(rsp->completed); /* outside of lock. */ | ||
684 | |||
685 | /* Did another grace period end? */ | ||
686 | if (rdp->completed != completed_snap) { | ||
687 | |||
688 | /* Advance callbacks. No harm if list empty. */ | ||
689 | rdp->nxttail[RCU_DONE_TAIL] = rdp->nxttail[RCU_WAIT_TAIL]; | ||
690 | rdp->nxttail[RCU_WAIT_TAIL] = rdp->nxttail[RCU_NEXT_READY_TAIL]; | ||
691 | rdp->nxttail[RCU_NEXT_READY_TAIL] = rdp->nxttail[RCU_NEXT_TAIL]; | ||
692 | |||
693 | /* Remember that we saw this grace-period completion. */ | ||
694 | rdp->completed = completed_snap; | ||
695 | } | ||
696 | local_irq_restore(flags); | ||
697 | } | ||
698 | |||
699 | /* | ||
700 | * Clean up after the prior grace period and let rcu_start_gp() start up | ||
701 | * the next grace period if one is needed. Note that the caller must | ||
702 | * hold rnp->lock, as required by rcu_start_gp(), which will release it. | ||
703 | */ | 749 | */ |
704 | static void cpu_quiet_msk_finish(struct rcu_state *rsp, unsigned long flags) | 750 | static void rcu_report_qs_rsp(struct rcu_state *rsp, unsigned long flags) |
705 | __releases(rcu_get_root(rsp)->lock) | 751 | __releases(rcu_get_root(rsp)->lock) |
706 | { | 752 | { |
707 | WARN_ON_ONCE(!rcu_gp_in_progress(rsp)); | 753 | WARN_ON_ONCE(!rcu_gp_in_progress(rsp)); |
708 | rsp->completed = rsp->gpnum; | 754 | rsp->completed = rsp->gpnum; |
709 | rcu_process_gp_end(rsp, rsp->rda[smp_processor_id()]); | 755 | rsp->signaled = RCU_GP_IDLE; |
710 | rcu_start_gp(rsp, flags); /* releases root node's rnp->lock. */ | 756 | rcu_start_gp(rsp, flags); /* releases root node's rnp->lock. */ |
711 | } | 757 | } |
712 | 758 | ||
713 | /* | 759 | /* |
714 | * Similar to cpu_quiet(), for which it is a helper function. Allows | 760 | * Similar to rcu_report_qs_rdp(), for which it is a helper function. |
715 | * a group of CPUs to be quieted at one go, though all the CPUs in the | 761 | * Allows quiescent states for a group of CPUs to be reported at one go |
716 | * group must be represented by the same leaf rcu_node structure. | 762 | * to the specified rcu_node structure, though all the CPUs in the group |
717 | * That structure's lock must be held upon entry, and it is released | 763 | * must be represented by the same rcu_node structure (which need not be |
718 | * before return. | 764 | * a leaf rcu_node structure, though it often will be). That structure's |
765 | * lock must be held upon entry, and it is released before return. | ||
719 | */ | 766 | */ |
720 | static void | 767 | static void |
721 | cpu_quiet_msk(unsigned long mask, struct rcu_state *rsp, struct rcu_node *rnp, | 768 | rcu_report_qs_rnp(unsigned long mask, struct rcu_state *rsp, |
722 | unsigned long flags) | 769 | struct rcu_node *rnp, unsigned long flags) |
723 | __releases(rnp->lock) | 770 | __releases(rnp->lock) |
724 | { | 771 | { |
725 | struct rcu_node *rnp_c; | 772 | struct rcu_node *rnp_c; |
@@ -755,21 +802,23 @@ cpu_quiet_msk(unsigned long mask, struct rcu_state *rsp, struct rcu_node *rnp, | |||
755 | 802 | ||
756 | /* | 803 | /* |
757 | * Get here if we are the last CPU to pass through a quiescent | 804 | * Get here if we are the last CPU to pass through a quiescent |
758 | * state for this grace period. Invoke cpu_quiet_msk_finish() | 805 | * state for this grace period. Invoke rcu_report_qs_rsp() |
759 | * to clean up and start the next grace period if one is needed. | 806 | * to clean up and start the next grace period if one is needed. |
760 | */ | 807 | */ |
761 | cpu_quiet_msk_finish(rsp, flags); /* releases rnp->lock. */ | 808 | rcu_report_qs_rsp(rsp, flags); /* releases rnp->lock. */ |
762 | } | 809 | } |
763 | 810 | ||
764 | /* | 811 | /* |
765 | * Record a quiescent state for the specified CPU, which must either be | 812 | * Record a quiescent state for the specified CPU to that CPU's rcu_data |
766 | * the current CPU. The lastcomp argument is used to make sure we are | 813 | * structure. This must be either called from the specified CPU, or |
767 | * still in the grace period of interest. We don't want to end the current | 814 | * called when the specified CPU is known to be offline (and when it is |
768 | * grace period based on quiescent states detected in an earlier grace | 815 | * also known that no other CPU is concurrently trying to help the offline |
769 | * period! | 816 | * CPU). The lastcomp argument is used to make sure we are still in the |
817 | * grace period of interest. We don't want to end the current grace period | ||
818 | * based on quiescent states detected in an earlier grace period! | ||
770 | */ | 819 | */ |
771 | static void | 820 | static void |
772 | cpu_quiet(int cpu, struct rcu_state *rsp, struct rcu_data *rdp, long lastcomp) | 821 | rcu_report_qs_rdp(int cpu, struct rcu_state *rsp, struct rcu_data *rdp, long lastcomp) |
773 | { | 822 | { |
774 | unsigned long flags; | 823 | unsigned long flags; |
775 | unsigned long mask; | 824 | unsigned long mask; |
@@ -777,15 +826,15 @@ cpu_quiet(int cpu, struct rcu_state *rsp, struct rcu_data *rdp, long lastcomp) | |||
777 | 826 | ||
778 | rnp = rdp->mynode; | 827 | rnp = rdp->mynode; |
779 | spin_lock_irqsave(&rnp->lock, flags); | 828 | spin_lock_irqsave(&rnp->lock, flags); |
780 | if (lastcomp != ACCESS_ONCE(rsp->completed)) { | 829 | if (lastcomp != rnp->completed) { |
781 | 830 | ||
782 | /* | 831 | /* |
783 | * Someone beat us to it for this grace period, so leave. | 832 | * Someone beat us to it for this grace period, so leave. |
784 | * The race with GP start is resolved by the fact that we | 833 | * The race with GP start is resolved by the fact that we |
785 | * hold the leaf rcu_node lock, so that the per-CPU bits | 834 | * hold the leaf rcu_node lock, so that the per-CPU bits |
786 | * cannot yet be initialized -- so we would simply find our | 835 | * cannot yet be initialized -- so we would simply find our |
787 | * CPU's bit already cleared in cpu_quiet_msk() if this race | 836 | * CPU's bit already cleared in rcu_report_qs_rnp() if this |
788 | * occurred. | 837 | * race occurred. |
789 | */ | 838 | */ |
790 | rdp->passed_quiesc = 0; /* try again later! */ | 839 | rdp->passed_quiesc = 0; /* try again later! */ |
791 | spin_unlock_irqrestore(&rnp->lock, flags); | 840 | spin_unlock_irqrestore(&rnp->lock, flags); |
@@ -803,7 +852,7 @@ cpu_quiet(int cpu, struct rcu_state *rsp, struct rcu_data *rdp, long lastcomp) | |||
803 | */ | 852 | */ |
804 | rdp->nxttail[RCU_NEXT_READY_TAIL] = rdp->nxttail[RCU_NEXT_TAIL]; | 853 | rdp->nxttail[RCU_NEXT_READY_TAIL] = rdp->nxttail[RCU_NEXT_TAIL]; |
805 | 854 | ||
806 | cpu_quiet_msk(mask, rsp, rnp, flags); /* releases rnp->lock */ | 855 | rcu_report_qs_rnp(mask, rsp, rnp, flags); /* rlses rnp->lock */ |
807 | } | 856 | } |
808 | } | 857 | } |
809 | 858 | ||
@@ -834,8 +883,11 @@ rcu_check_quiescent_state(struct rcu_state *rsp, struct rcu_data *rdp) | |||
834 | if (!rdp->passed_quiesc) | 883 | if (!rdp->passed_quiesc) |
835 | return; | 884 | return; |
836 | 885 | ||
837 | /* Tell RCU we are done (but cpu_quiet() will be the judge of that). */ | 886 | /* |
838 | cpu_quiet(rdp->cpu, rsp, rdp, rdp->passed_quiesc_completed); | 887 | * Tell RCU we are done (but rcu_report_qs_rdp() will be the |
888 | * judge of that). | ||
889 | */ | ||
890 | rcu_report_qs_rdp(rdp->cpu, rsp, rdp, rdp->passed_quiesc_completed); | ||
839 | } | 891 | } |
840 | 892 | ||
841 | #ifdef CONFIG_HOTPLUG_CPU | 893 | #ifdef CONFIG_HOTPLUG_CPU |
@@ -895,8 +947,8 @@ static void rcu_adopt_orphan_cbs(struct rcu_state *rsp) | |||
895 | static void __rcu_offline_cpu(int cpu, struct rcu_state *rsp) | 947 | static void __rcu_offline_cpu(int cpu, struct rcu_state *rsp) |
896 | { | 948 | { |
897 | unsigned long flags; | 949 | unsigned long flags; |
898 | long lastcomp; | ||
899 | unsigned long mask; | 950 | unsigned long mask; |
951 | int need_report = 0; | ||
900 | struct rcu_data *rdp = rsp->rda[cpu]; | 952 | struct rcu_data *rdp = rsp->rda[cpu]; |
901 | struct rcu_node *rnp; | 953 | struct rcu_node *rnp; |
902 | 954 | ||
@@ -910,17 +962,32 @@ static void __rcu_offline_cpu(int cpu, struct rcu_state *rsp) | |||
910 | spin_lock(&rnp->lock); /* irqs already disabled. */ | 962 | spin_lock(&rnp->lock); /* irqs already disabled. */ |
911 | rnp->qsmaskinit &= ~mask; | 963 | rnp->qsmaskinit &= ~mask; |
912 | if (rnp->qsmaskinit != 0) { | 964 | if (rnp->qsmaskinit != 0) { |
913 | spin_unlock(&rnp->lock); /* irqs remain disabled. */ | 965 | if (rnp != rdp->mynode) |
966 | spin_unlock(&rnp->lock); /* irqs remain disabled. */ | ||
914 | break; | 967 | break; |
915 | } | 968 | } |
916 | rcu_preempt_offline_tasks(rsp, rnp, rdp); | 969 | if (rnp == rdp->mynode) |
970 | need_report = rcu_preempt_offline_tasks(rsp, rnp, rdp); | ||
971 | else | ||
972 | spin_unlock(&rnp->lock); /* irqs remain disabled. */ | ||
917 | mask = rnp->grpmask; | 973 | mask = rnp->grpmask; |
918 | spin_unlock(&rnp->lock); /* irqs remain disabled. */ | ||
919 | rnp = rnp->parent; | 974 | rnp = rnp->parent; |
920 | } while (rnp != NULL); | 975 | } while (rnp != NULL); |
921 | lastcomp = rsp->completed; | ||
922 | 976 | ||
923 | spin_unlock_irqrestore(&rsp->onofflock, flags); | 977 | /* |
978 | * We still hold the leaf rcu_node structure lock here, and | ||
979 | * irqs are still disabled. The reason for this subterfuge is | ||
980 | * because invoking rcu_report_unblock_qs_rnp() with ->onofflock | ||
981 | * held leads to deadlock. | ||
982 | */ | ||
983 | spin_unlock(&rsp->onofflock); /* irqs remain disabled. */ | ||
984 | rnp = rdp->mynode; | ||
985 | if (need_report & RCU_OFL_TASKS_NORM_GP) | ||
986 | rcu_report_unblock_qs_rnp(rnp, flags); | ||
987 | else | ||
988 | spin_unlock_irqrestore(&rnp->lock, flags); | ||
989 | if (need_report & RCU_OFL_TASKS_EXP_GP) | ||
990 | rcu_report_exp_rnp(rsp, rnp); | ||
924 | 991 | ||
925 | rcu_adopt_orphan_cbs(rsp); | 992 | rcu_adopt_orphan_cbs(rsp); |
926 | } | 993 | } |
@@ -958,7 +1025,7 @@ static void rcu_offline_cpu(int cpu) | |||
958 | * Invoke any RCU callbacks that have made it to the end of their grace | 1025 | * Invoke any RCU callbacks that have made it to the end of their grace |
959 | * period. Thottle as specified by rdp->blimit. | 1026 | * period. Thottle as specified by rdp->blimit. |
960 | */ | 1027 | */ |
961 | static void rcu_do_batch(struct rcu_data *rdp) | 1028 | static void rcu_do_batch(struct rcu_state *rsp, struct rcu_data *rdp) |
962 | { | 1029 | { |
963 | unsigned long flags; | 1030 | unsigned long flags; |
964 | struct rcu_head *next, *list, **tail; | 1031 | struct rcu_head *next, *list, **tail; |
@@ -1011,6 +1078,13 @@ static void rcu_do_batch(struct rcu_data *rdp) | |||
1011 | if (rdp->blimit == LONG_MAX && rdp->qlen <= qlowmark) | 1078 | if (rdp->blimit == LONG_MAX && rdp->qlen <= qlowmark) |
1012 | rdp->blimit = blimit; | 1079 | rdp->blimit = blimit; |
1013 | 1080 | ||
1081 | /* Reset ->qlen_last_fqs_check trigger if enough CBs have drained. */ | ||
1082 | if (rdp->qlen == 0 && rdp->qlen_last_fqs_check != 0) { | ||
1083 | rdp->qlen_last_fqs_check = 0; | ||
1084 | rdp->n_force_qs_snap = rsp->n_force_qs; | ||
1085 | } else if (rdp->qlen < rdp->qlen_last_fqs_check - qhimark) | ||
1086 | rdp->qlen_last_fqs_check = rdp->qlen; | ||
1087 | |||
1014 | local_irq_restore(flags); | 1088 | local_irq_restore(flags); |
1015 | 1089 | ||
1016 | /* Re-raise the RCU softirq if there are callbacks remaining. */ | 1090 | /* Re-raise the RCU softirq if there are callbacks remaining. */ |
@@ -1085,7 +1159,7 @@ static int rcu_process_dyntick(struct rcu_state *rsp, long lastcomp, | |||
1085 | rcu_for_each_leaf_node(rsp, rnp) { | 1159 | rcu_for_each_leaf_node(rsp, rnp) { |
1086 | mask = 0; | 1160 | mask = 0; |
1087 | spin_lock_irqsave(&rnp->lock, flags); | 1161 | spin_lock_irqsave(&rnp->lock, flags); |
1088 | if (rsp->completed != lastcomp) { | 1162 | if (rnp->completed != lastcomp) { |
1089 | spin_unlock_irqrestore(&rnp->lock, flags); | 1163 | spin_unlock_irqrestore(&rnp->lock, flags); |
1090 | return 1; | 1164 | return 1; |
1091 | } | 1165 | } |
@@ -1099,10 +1173,10 @@ static int rcu_process_dyntick(struct rcu_state *rsp, long lastcomp, | |||
1099 | if ((rnp->qsmask & bit) != 0 && f(rsp->rda[cpu])) | 1173 | if ((rnp->qsmask & bit) != 0 && f(rsp->rda[cpu])) |
1100 | mask |= bit; | 1174 | mask |= bit; |
1101 | } | 1175 | } |
1102 | if (mask != 0 && rsp->completed == lastcomp) { | 1176 | if (mask != 0 && rnp->completed == lastcomp) { |
1103 | 1177 | ||
1104 | /* cpu_quiet_msk() releases rnp->lock. */ | 1178 | /* rcu_report_qs_rnp() releases rnp->lock. */ |
1105 | cpu_quiet_msk(mask, rsp, rnp, flags); | 1179 | rcu_report_qs_rnp(mask, rsp, rnp, flags); |
1106 | continue; | 1180 | continue; |
1107 | } | 1181 | } |
1108 | spin_unlock_irqrestore(&rnp->lock, flags); | 1182 | spin_unlock_irqrestore(&rnp->lock, flags); |
@@ -1120,6 +1194,7 @@ static void force_quiescent_state(struct rcu_state *rsp, int relaxed) | |||
1120 | long lastcomp; | 1194 | long lastcomp; |
1121 | struct rcu_node *rnp = rcu_get_root(rsp); | 1195 | struct rcu_node *rnp = rcu_get_root(rsp); |
1122 | u8 signaled; | 1196 | u8 signaled; |
1197 | u8 forcenow; | ||
1123 | 1198 | ||
1124 | if (!rcu_gp_in_progress(rsp)) | 1199 | if (!rcu_gp_in_progress(rsp)) |
1125 | return; /* No grace period in progress, nothing to force. */ | 1200 | return; /* No grace period in progress, nothing to force. */ |
@@ -1132,19 +1207,20 @@ static void force_quiescent_state(struct rcu_state *rsp, int relaxed) | |||
1132 | goto unlock_ret; /* no emergency and done recently. */ | 1207 | goto unlock_ret; /* no emergency and done recently. */ |
1133 | rsp->n_force_qs++; | 1208 | rsp->n_force_qs++; |
1134 | spin_lock(&rnp->lock); | 1209 | spin_lock(&rnp->lock); |
1135 | lastcomp = rsp->completed; | 1210 | lastcomp = rsp->gpnum - 1; |
1136 | signaled = rsp->signaled; | 1211 | signaled = rsp->signaled; |
1137 | rsp->jiffies_force_qs = jiffies + RCU_JIFFIES_TILL_FORCE_QS; | 1212 | rsp->jiffies_force_qs = jiffies + RCU_JIFFIES_TILL_FORCE_QS; |
1138 | if (lastcomp == rsp->gpnum) { | 1213 | if(!rcu_gp_in_progress(rsp)) { |
1139 | rsp->n_force_qs_ngp++; | 1214 | rsp->n_force_qs_ngp++; |
1140 | spin_unlock(&rnp->lock); | 1215 | spin_unlock(&rnp->lock); |
1141 | goto unlock_ret; /* no GP in progress, time updated. */ | 1216 | goto unlock_ret; /* no GP in progress, time updated. */ |
1142 | } | 1217 | } |
1143 | spin_unlock(&rnp->lock); | 1218 | spin_unlock(&rnp->lock); |
1144 | switch (signaled) { | 1219 | switch (signaled) { |
1220 | case RCU_GP_IDLE: | ||
1145 | case RCU_GP_INIT: | 1221 | case RCU_GP_INIT: |
1146 | 1222 | ||
1147 | break; /* grace period still initializing, ignore. */ | 1223 | break; /* grace period idle or initializing, ignore. */ |
1148 | 1224 | ||
1149 | case RCU_SAVE_DYNTICK: | 1225 | case RCU_SAVE_DYNTICK: |
1150 | 1226 | ||
@@ -1155,20 +1231,29 @@ static void force_quiescent_state(struct rcu_state *rsp, int relaxed) | |||
1155 | if (rcu_process_dyntick(rsp, lastcomp, | 1231 | if (rcu_process_dyntick(rsp, lastcomp, |
1156 | dyntick_save_progress_counter)) | 1232 | dyntick_save_progress_counter)) |
1157 | goto unlock_ret; | 1233 | goto unlock_ret; |
1234 | /* fall into next case. */ | ||
1235 | |||
1236 | case RCU_SAVE_COMPLETED: | ||
1158 | 1237 | ||
1159 | /* Update state, record completion counter. */ | 1238 | /* Update state, record completion counter. */ |
1239 | forcenow = 0; | ||
1160 | spin_lock(&rnp->lock); | 1240 | spin_lock(&rnp->lock); |
1161 | if (lastcomp == rsp->completed) { | 1241 | if (lastcomp + 1 == rsp->gpnum && |
1242 | lastcomp == rsp->completed && | ||
1243 | rsp->signaled == signaled) { | ||
1162 | rsp->signaled = RCU_FORCE_QS; | 1244 | rsp->signaled = RCU_FORCE_QS; |
1163 | dyntick_record_completed(rsp, lastcomp); | 1245 | rsp->completed_fqs = lastcomp; |
1246 | forcenow = signaled == RCU_SAVE_COMPLETED; | ||
1164 | } | 1247 | } |
1165 | spin_unlock(&rnp->lock); | 1248 | spin_unlock(&rnp->lock); |
1166 | break; | 1249 | if (!forcenow) |
1250 | break; | ||
1251 | /* fall into next case. */ | ||
1167 | 1252 | ||
1168 | case RCU_FORCE_QS: | 1253 | case RCU_FORCE_QS: |
1169 | 1254 | ||
1170 | /* Check dyntick-idle state, send IPI to laggarts. */ | 1255 | /* Check dyntick-idle state, send IPI to laggarts. */ |
1171 | if (rcu_process_dyntick(rsp, dyntick_recall_completed(rsp), | 1256 | if (rcu_process_dyntick(rsp, rsp->completed_fqs, |
1172 | rcu_implicit_dynticks_qs)) | 1257 | rcu_implicit_dynticks_qs)) |
1173 | goto unlock_ret; | 1258 | goto unlock_ret; |
1174 | 1259 | ||
@@ -1224,7 +1309,7 @@ __rcu_process_callbacks(struct rcu_state *rsp, struct rcu_data *rdp) | |||
1224 | } | 1309 | } |
1225 | 1310 | ||
1226 | /* If there are callbacks ready, invoke them. */ | 1311 | /* If there are callbacks ready, invoke them. */ |
1227 | rcu_do_batch(rdp); | 1312 | rcu_do_batch(rsp, rdp); |
1228 | } | 1313 | } |
1229 | 1314 | ||
1230 | /* | 1315 | /* |
@@ -1288,10 +1373,20 @@ __call_rcu(struct rcu_head *head, void (*func)(struct rcu_head *rcu), | |||
1288 | rcu_start_gp(rsp, nestflag); /* releases rnp_root->lock. */ | 1373 | rcu_start_gp(rsp, nestflag); /* releases rnp_root->lock. */ |
1289 | } | 1374 | } |
1290 | 1375 | ||
1291 | /* Force the grace period if too many callbacks or too long waiting. */ | 1376 | /* |
1292 | if (unlikely(++rdp->qlen > qhimark)) { | 1377 | * Force the grace period if too many callbacks or too long waiting. |
1378 | * Enforce hysteresis, and don't invoke force_quiescent_state() | ||
1379 | * if some other CPU has recently done so. Also, don't bother | ||
1380 | * invoking force_quiescent_state() if the newly enqueued callback | ||
1381 | * is the only one waiting for a grace period to complete. | ||
1382 | */ | ||
1383 | if (unlikely(++rdp->qlen > rdp->qlen_last_fqs_check + qhimark)) { | ||
1293 | rdp->blimit = LONG_MAX; | 1384 | rdp->blimit = LONG_MAX; |
1294 | force_quiescent_state(rsp, 0); | 1385 | if (rsp->n_force_qs == rdp->n_force_qs_snap && |
1386 | *rdp->nxttail[RCU_DONE_TAIL] != head) | ||
1387 | force_quiescent_state(rsp, 0); | ||
1388 | rdp->n_force_qs_snap = rsp->n_force_qs; | ||
1389 | rdp->qlen_last_fqs_check = rdp->qlen; | ||
1295 | } else if ((long)(ACCESS_ONCE(rsp->jiffies_force_qs) - jiffies) < 0) | 1390 | } else if ((long)(ACCESS_ONCE(rsp->jiffies_force_qs) - jiffies) < 0) |
1296 | force_quiescent_state(rsp, 1); | 1391 | force_quiescent_state(rsp, 1); |
1297 | local_irq_restore(flags); | 1392 | local_irq_restore(flags); |
@@ -1315,6 +1410,68 @@ void call_rcu_bh(struct rcu_head *head, void (*func)(struct rcu_head *rcu)) | |||
1315 | } | 1410 | } |
1316 | EXPORT_SYMBOL_GPL(call_rcu_bh); | 1411 | EXPORT_SYMBOL_GPL(call_rcu_bh); |
1317 | 1412 | ||
1413 | /** | ||
1414 | * synchronize_sched - wait until an rcu-sched grace period has elapsed. | ||
1415 | * | ||
1416 | * Control will return to the caller some time after a full rcu-sched | ||
1417 | * grace period has elapsed, in other words after all currently executing | ||
1418 | * rcu-sched read-side critical sections have completed. These read-side | ||
1419 | * critical sections are delimited by rcu_read_lock_sched() and | ||
1420 | * rcu_read_unlock_sched(), and may be nested. Note that preempt_disable(), | ||
1421 | * local_irq_disable(), and so on may be used in place of | ||
1422 | * rcu_read_lock_sched(). | ||
1423 | * | ||
1424 | * This means that all preempt_disable code sequences, including NMI and | ||
1425 | * hardware-interrupt handlers, in progress on entry will have completed | ||
1426 | * before this primitive returns. However, this does not guarantee that | ||
1427 | * softirq handlers will have completed, since in some kernels, these | ||
1428 | * handlers can run in process context, and can block. | ||
1429 | * | ||
1430 | * This primitive provides the guarantees made by the (now removed) | ||
1431 | * synchronize_kernel() API. In contrast, synchronize_rcu() only | ||
1432 | * guarantees that rcu_read_lock() sections will have completed. | ||
1433 | * In "classic RCU", these two guarantees happen to be one and | ||
1434 | * the same, but can differ in realtime RCU implementations. | ||
1435 | */ | ||
1436 | void synchronize_sched(void) | ||
1437 | { | ||
1438 | struct rcu_synchronize rcu; | ||
1439 | |||
1440 | if (rcu_blocking_is_gp()) | ||
1441 | return; | ||
1442 | |||
1443 | init_completion(&rcu.completion); | ||
1444 | /* Will wake me after RCU finished. */ | ||
1445 | call_rcu_sched(&rcu.head, wakeme_after_rcu); | ||
1446 | /* Wait for it. */ | ||
1447 | wait_for_completion(&rcu.completion); | ||
1448 | } | ||
1449 | EXPORT_SYMBOL_GPL(synchronize_sched); | ||
1450 | |||
1451 | /** | ||
1452 | * synchronize_rcu_bh - wait until an rcu_bh grace period has elapsed. | ||
1453 | * | ||
1454 | * Control will return to the caller some time after a full rcu_bh grace | ||
1455 | * period has elapsed, in other words after all currently executing rcu_bh | ||
1456 | * read-side critical sections have completed. RCU read-side critical | ||
1457 | * sections are delimited by rcu_read_lock_bh() and rcu_read_unlock_bh(), | ||
1458 | * and may be nested. | ||
1459 | */ | ||
1460 | void synchronize_rcu_bh(void) | ||
1461 | { | ||
1462 | struct rcu_synchronize rcu; | ||
1463 | |||
1464 | if (rcu_blocking_is_gp()) | ||
1465 | return; | ||
1466 | |||
1467 | init_completion(&rcu.completion); | ||
1468 | /* Will wake me after RCU finished. */ | ||
1469 | call_rcu_bh(&rcu.head, wakeme_after_rcu); | ||
1470 | /* Wait for it. */ | ||
1471 | wait_for_completion(&rcu.completion); | ||
1472 | } | ||
1473 | EXPORT_SYMBOL_GPL(synchronize_rcu_bh); | ||
1474 | |||
1318 | /* | 1475 | /* |
1319 | * Check to see if there is any immediate RCU-related work to be done | 1476 | * Check to see if there is any immediate RCU-related work to be done |
1320 | * by the current CPU, for the specified type of RCU, returning 1 if so. | 1477 | * by the current CPU, for the specified type of RCU, returning 1 if so. |
@@ -1324,6 +1481,8 @@ EXPORT_SYMBOL_GPL(call_rcu_bh); | |||
1324 | */ | 1481 | */ |
1325 | static int __rcu_pending(struct rcu_state *rsp, struct rcu_data *rdp) | 1482 | static int __rcu_pending(struct rcu_state *rsp, struct rcu_data *rdp) |
1326 | { | 1483 | { |
1484 | struct rcu_node *rnp = rdp->mynode; | ||
1485 | |||
1327 | rdp->n_rcu_pending++; | 1486 | rdp->n_rcu_pending++; |
1328 | 1487 | ||
1329 | /* Check for CPU stalls, if enabled. */ | 1488 | /* Check for CPU stalls, if enabled. */ |
@@ -1348,13 +1507,13 @@ static int __rcu_pending(struct rcu_state *rsp, struct rcu_data *rdp) | |||
1348 | } | 1507 | } |
1349 | 1508 | ||
1350 | /* Has another RCU grace period completed? */ | 1509 | /* Has another RCU grace period completed? */ |
1351 | if (ACCESS_ONCE(rsp->completed) != rdp->completed) { /* outside lock */ | 1510 | if (ACCESS_ONCE(rnp->completed) != rdp->completed) { /* outside lock */ |
1352 | rdp->n_rp_gp_completed++; | 1511 | rdp->n_rp_gp_completed++; |
1353 | return 1; | 1512 | return 1; |
1354 | } | 1513 | } |
1355 | 1514 | ||
1356 | /* Has a new RCU grace period started? */ | 1515 | /* Has a new RCU grace period started? */ |
1357 | if (ACCESS_ONCE(rsp->gpnum) != rdp->gpnum) { /* outside lock */ | 1516 | if (ACCESS_ONCE(rnp->gpnum) != rdp->gpnum) { /* outside lock */ |
1358 | rdp->n_rp_gp_started++; | 1517 | rdp->n_rp_gp_started++; |
1359 | return 1; | 1518 | return 1; |
1360 | } | 1519 | } |
@@ -1397,6 +1556,21 @@ int rcu_needs_cpu(int cpu) | |||
1397 | rcu_preempt_needs_cpu(cpu); | 1556 | rcu_preempt_needs_cpu(cpu); |
1398 | } | 1557 | } |
1399 | 1558 | ||
1559 | /* | ||
1560 | * This function is invoked towards the end of the scheduler's initialization | ||
1561 | * process. Before this is called, the idle task might contain | ||
1562 | * RCU read-side critical sections (during which time, this idle | ||
1563 | * task is booting the system). After this function is called, the | ||
1564 | * idle tasks are prohibited from containing RCU read-side critical | ||
1565 | * sections. | ||
1566 | */ | ||
1567 | void rcu_scheduler_starting(void) | ||
1568 | { | ||
1569 | WARN_ON(num_online_cpus() != 1); | ||
1570 | WARN_ON(nr_context_switches() > 0); | ||
1571 | rcu_scheduler_active = 1; | ||
1572 | } | ||
1573 | |||
1400 | static DEFINE_PER_CPU(struct rcu_head, rcu_barrier_head) = {NULL}; | 1574 | static DEFINE_PER_CPU(struct rcu_head, rcu_barrier_head) = {NULL}; |
1401 | static atomic_t rcu_barrier_cpu_count; | 1575 | static atomic_t rcu_barrier_cpu_count; |
1402 | static DEFINE_MUTEX(rcu_barrier_mutex); | 1576 | static DEFINE_MUTEX(rcu_barrier_mutex); |
@@ -1508,21 +1682,18 @@ static void __cpuinit | |||
1508 | rcu_init_percpu_data(int cpu, struct rcu_state *rsp, int preemptable) | 1682 | rcu_init_percpu_data(int cpu, struct rcu_state *rsp, int preemptable) |
1509 | { | 1683 | { |
1510 | unsigned long flags; | 1684 | unsigned long flags; |
1511 | long lastcomp; | ||
1512 | unsigned long mask; | 1685 | unsigned long mask; |
1513 | struct rcu_data *rdp = rsp->rda[cpu]; | 1686 | struct rcu_data *rdp = rsp->rda[cpu]; |
1514 | struct rcu_node *rnp = rcu_get_root(rsp); | 1687 | struct rcu_node *rnp = rcu_get_root(rsp); |
1515 | 1688 | ||
1516 | /* Set up local state, ensuring consistent view of global state. */ | 1689 | /* Set up local state, ensuring consistent view of global state. */ |
1517 | spin_lock_irqsave(&rnp->lock, flags); | 1690 | spin_lock_irqsave(&rnp->lock, flags); |
1518 | lastcomp = rsp->completed; | ||
1519 | rdp->completed = lastcomp; | ||
1520 | rdp->gpnum = lastcomp; | ||
1521 | rdp->passed_quiesc = 0; /* We could be racing with new GP, */ | 1691 | rdp->passed_quiesc = 0; /* We could be racing with new GP, */ |
1522 | rdp->qs_pending = 1; /* so set up to respond to current GP. */ | 1692 | rdp->qs_pending = 1; /* so set up to respond to current GP. */ |
1523 | rdp->beenonline = 1; /* We have now been online. */ | 1693 | rdp->beenonline = 1; /* We have now been online. */ |
1524 | rdp->preemptable = preemptable; | 1694 | rdp->preemptable = preemptable; |
1525 | rdp->passed_quiesc_completed = lastcomp - 1; | 1695 | rdp->qlen_last_fqs_check = 0; |
1696 | rdp->n_force_qs_snap = rsp->n_force_qs; | ||
1526 | rdp->blimit = blimit; | 1697 | rdp->blimit = blimit; |
1527 | spin_unlock(&rnp->lock); /* irqs remain disabled. */ | 1698 | spin_unlock(&rnp->lock); /* irqs remain disabled. */ |
1528 | 1699 | ||
@@ -1542,6 +1713,11 @@ rcu_init_percpu_data(int cpu, struct rcu_state *rsp, int preemptable) | |||
1542 | spin_lock(&rnp->lock); /* irqs already disabled. */ | 1713 | spin_lock(&rnp->lock); /* irqs already disabled. */ |
1543 | rnp->qsmaskinit |= mask; | 1714 | rnp->qsmaskinit |= mask; |
1544 | mask = rnp->grpmask; | 1715 | mask = rnp->grpmask; |
1716 | if (rnp == rdp->mynode) { | ||
1717 | rdp->gpnum = rnp->completed; /* if GP in progress... */ | ||
1718 | rdp->completed = rnp->completed; | ||
1719 | rdp->passed_quiesc_completed = rnp->completed - 1; | ||
1720 | } | ||
1545 | spin_unlock(&rnp->lock); /* irqs already disabled. */ | 1721 | spin_unlock(&rnp->lock); /* irqs already disabled. */ |
1546 | rnp = rnp->parent; | 1722 | rnp = rnp->parent; |
1547 | } while (rnp != NULL && !(rnp->qsmaskinit & mask)); | 1723 | } while (rnp != NULL && !(rnp->qsmaskinit & mask)); |
@@ -1559,8 +1735,8 @@ static void __cpuinit rcu_online_cpu(int cpu) | |||
1559 | /* | 1735 | /* |
1560 | * Handle CPU online/offline notification events. | 1736 | * Handle CPU online/offline notification events. |
1561 | */ | 1737 | */ |
1562 | int __cpuinit rcu_cpu_notify(struct notifier_block *self, | 1738 | static int __cpuinit rcu_cpu_notify(struct notifier_block *self, |
1563 | unsigned long action, void *hcpu) | 1739 | unsigned long action, void *hcpu) |
1564 | { | 1740 | { |
1565 | long cpu = (long)hcpu; | 1741 | long cpu = (long)hcpu; |
1566 | 1742 | ||
@@ -1647,8 +1823,8 @@ static void __init rcu_init_one(struct rcu_state *rsp) | |||
1647 | cpustride *= rsp->levelspread[i]; | 1823 | cpustride *= rsp->levelspread[i]; |
1648 | rnp = rsp->level[i]; | 1824 | rnp = rsp->level[i]; |
1649 | for (j = 0; j < rsp->levelcnt[i]; j++, rnp++) { | 1825 | for (j = 0; j < rsp->levelcnt[i]; j++, rnp++) { |
1650 | if (rnp != rcu_get_root(rsp)) | 1826 | spin_lock_init(&rnp->lock); |
1651 | spin_lock_init(&rnp->lock); | 1827 | lockdep_set_class(&rnp->lock, &rcu_node_class[i]); |
1652 | rnp->gpnum = 0; | 1828 | rnp->gpnum = 0; |
1653 | rnp->qsmask = 0; | 1829 | rnp->qsmask = 0; |
1654 | rnp->qsmaskinit = 0; | 1830 | rnp->qsmaskinit = 0; |
@@ -1669,9 +1845,10 @@ static void __init rcu_init_one(struct rcu_state *rsp) | |||
1669 | rnp->level = i; | 1845 | rnp->level = i; |
1670 | INIT_LIST_HEAD(&rnp->blocked_tasks[0]); | 1846 | INIT_LIST_HEAD(&rnp->blocked_tasks[0]); |
1671 | INIT_LIST_HEAD(&rnp->blocked_tasks[1]); | 1847 | INIT_LIST_HEAD(&rnp->blocked_tasks[1]); |
1848 | INIT_LIST_HEAD(&rnp->blocked_tasks[2]); | ||
1849 | INIT_LIST_HEAD(&rnp->blocked_tasks[3]); | ||
1672 | } | 1850 | } |
1673 | } | 1851 | } |
1674 | spin_lock_init(&rcu_get_root(rsp)->lock); | ||
1675 | } | 1852 | } |
1676 | 1853 | ||
1677 | /* | 1854 | /* |
@@ -1697,16 +1874,30 @@ do { \ | |||
1697 | } \ | 1874 | } \ |
1698 | } while (0) | 1875 | } while (0) |
1699 | 1876 | ||
1700 | void __init __rcu_init(void) | 1877 | void __init rcu_init(void) |
1701 | { | 1878 | { |
1879 | int i; | ||
1880 | |||
1702 | rcu_bootup_announce(); | 1881 | rcu_bootup_announce(); |
1703 | #ifdef CONFIG_RCU_CPU_STALL_DETECTOR | 1882 | #ifdef CONFIG_RCU_CPU_STALL_DETECTOR |
1704 | printk(KERN_INFO "RCU-based detection of stalled CPUs is enabled.\n"); | 1883 | printk(KERN_INFO "RCU-based detection of stalled CPUs is enabled.\n"); |
1705 | #endif /* #ifdef CONFIG_RCU_CPU_STALL_DETECTOR */ | 1884 | #endif /* #ifdef CONFIG_RCU_CPU_STALL_DETECTOR */ |
1885 | #if NUM_RCU_LVL_4 != 0 | ||
1886 | printk(KERN_INFO "Experimental four-level hierarchy is enabled.\n"); | ||
1887 | #endif /* #if NUM_RCU_LVL_4 != 0 */ | ||
1706 | RCU_INIT_FLAVOR(&rcu_sched_state, rcu_sched_data); | 1888 | RCU_INIT_FLAVOR(&rcu_sched_state, rcu_sched_data); |
1707 | RCU_INIT_FLAVOR(&rcu_bh_state, rcu_bh_data); | 1889 | RCU_INIT_FLAVOR(&rcu_bh_state, rcu_bh_data); |
1708 | __rcu_init_preempt(); | 1890 | __rcu_init_preempt(); |
1709 | open_softirq(RCU_SOFTIRQ, rcu_process_callbacks); | 1891 | open_softirq(RCU_SOFTIRQ, rcu_process_callbacks); |
1892 | |||
1893 | /* | ||
1894 | * We don't need protection against CPU-hotplug here because | ||
1895 | * this is called early in boot, before either interrupts | ||
1896 | * or the scheduler are operational. | ||
1897 | */ | ||
1898 | cpu_notifier(rcu_cpu_notify, 0); | ||
1899 | for_each_online_cpu(i) | ||
1900 | rcu_cpu_notify(NULL, CPU_UP_PREPARE, (void *)(long)i); | ||
1710 | } | 1901 | } |
1711 | 1902 | ||
1712 | #include "rcutree_plugin.h" | 1903 | #include "rcutree_plugin.h" |
diff --git a/kernel/rcutree.h b/kernel/rcutree.h index b40ac5706040..d2a0046f63b2 100644 --- a/kernel/rcutree.h +++ b/kernel/rcutree.h | |||
@@ -34,10 +34,11 @@ | |||
34 | * In practice, this has not been tested, so there is probably some | 34 | * In practice, this has not been tested, so there is probably some |
35 | * bug somewhere. | 35 | * bug somewhere. |
36 | */ | 36 | */ |
37 | #define MAX_RCU_LVLS 3 | 37 | #define MAX_RCU_LVLS 4 |
38 | #define RCU_FANOUT (CONFIG_RCU_FANOUT) | 38 | #define RCU_FANOUT (CONFIG_RCU_FANOUT) |
39 | #define RCU_FANOUT_SQ (RCU_FANOUT * RCU_FANOUT) | 39 | #define RCU_FANOUT_SQ (RCU_FANOUT * RCU_FANOUT) |
40 | #define RCU_FANOUT_CUBE (RCU_FANOUT_SQ * RCU_FANOUT) | 40 | #define RCU_FANOUT_CUBE (RCU_FANOUT_SQ * RCU_FANOUT) |
41 | #define RCU_FANOUT_FOURTH (RCU_FANOUT_CUBE * RCU_FANOUT) | ||
41 | 42 | ||
42 | #if NR_CPUS <= RCU_FANOUT | 43 | #if NR_CPUS <= RCU_FANOUT |
43 | # define NUM_RCU_LVLS 1 | 44 | # define NUM_RCU_LVLS 1 |
@@ -45,23 +46,33 @@ | |||
45 | # define NUM_RCU_LVL_1 (NR_CPUS) | 46 | # define NUM_RCU_LVL_1 (NR_CPUS) |
46 | # define NUM_RCU_LVL_2 0 | 47 | # define NUM_RCU_LVL_2 0 |
47 | # define NUM_RCU_LVL_3 0 | 48 | # define NUM_RCU_LVL_3 0 |
49 | # define NUM_RCU_LVL_4 0 | ||
48 | #elif NR_CPUS <= RCU_FANOUT_SQ | 50 | #elif NR_CPUS <= RCU_FANOUT_SQ |
49 | # define NUM_RCU_LVLS 2 | 51 | # define NUM_RCU_LVLS 2 |
50 | # define NUM_RCU_LVL_0 1 | 52 | # define NUM_RCU_LVL_0 1 |
51 | # define NUM_RCU_LVL_1 DIV_ROUND_UP(NR_CPUS, RCU_FANOUT) | 53 | # define NUM_RCU_LVL_1 DIV_ROUND_UP(NR_CPUS, RCU_FANOUT) |
52 | # define NUM_RCU_LVL_2 (NR_CPUS) | 54 | # define NUM_RCU_LVL_2 (NR_CPUS) |
53 | # define NUM_RCU_LVL_3 0 | 55 | # define NUM_RCU_LVL_3 0 |
56 | # define NUM_RCU_LVL_4 0 | ||
54 | #elif NR_CPUS <= RCU_FANOUT_CUBE | 57 | #elif NR_CPUS <= RCU_FANOUT_CUBE |
55 | # define NUM_RCU_LVLS 3 | 58 | # define NUM_RCU_LVLS 3 |
56 | # define NUM_RCU_LVL_0 1 | 59 | # define NUM_RCU_LVL_0 1 |
57 | # define NUM_RCU_LVL_1 DIV_ROUND_UP(NR_CPUS, RCU_FANOUT_SQ) | 60 | # define NUM_RCU_LVL_1 DIV_ROUND_UP(NR_CPUS, RCU_FANOUT_SQ) |
58 | # define NUM_RCU_LVL_2 DIV_ROUND_UP(NR_CPUS, RCU_FANOUT) | 61 | # define NUM_RCU_LVL_2 DIV_ROUND_UP(NR_CPUS, RCU_FANOUT) |
59 | # define NUM_RCU_LVL_3 NR_CPUS | 62 | # define NUM_RCU_LVL_3 NR_CPUS |
63 | # define NUM_RCU_LVL_4 0 | ||
64 | #elif NR_CPUS <= RCU_FANOUT_FOURTH | ||
65 | # define NUM_RCU_LVLS 4 | ||
66 | # define NUM_RCU_LVL_0 1 | ||
67 | # define NUM_RCU_LVL_1 DIV_ROUND_UP(NR_CPUS, RCU_FANOUT_CUBE) | ||
68 | # define NUM_RCU_LVL_2 DIV_ROUND_UP(NR_CPUS, RCU_FANOUT_SQ) | ||
69 | # define NUM_RCU_LVL_3 DIV_ROUND_UP(NR_CPUS, RCU_FANOUT) | ||
70 | # define NUM_RCU_LVL_4 NR_CPUS | ||
60 | #else | 71 | #else |
61 | # error "CONFIG_RCU_FANOUT insufficient for NR_CPUS" | 72 | # error "CONFIG_RCU_FANOUT insufficient for NR_CPUS" |
62 | #endif /* #if (NR_CPUS) <= RCU_FANOUT */ | 73 | #endif /* #if (NR_CPUS) <= RCU_FANOUT */ |
63 | 74 | ||
64 | #define RCU_SUM (NUM_RCU_LVL_0 + NUM_RCU_LVL_1 + NUM_RCU_LVL_2 + NUM_RCU_LVL_3) | 75 | #define RCU_SUM (NUM_RCU_LVL_0 + NUM_RCU_LVL_1 + NUM_RCU_LVL_2 + NUM_RCU_LVL_3 + NUM_RCU_LVL_4) |
65 | #define NUM_RCU_NODES (RCU_SUM - NR_CPUS) | 76 | #define NUM_RCU_NODES (RCU_SUM - NR_CPUS) |
66 | 77 | ||
67 | /* | 78 | /* |
@@ -84,14 +95,21 @@ struct rcu_node { | |||
84 | long gpnum; /* Current grace period for this node. */ | 95 | long gpnum; /* Current grace period for this node. */ |
85 | /* This will either be equal to or one */ | 96 | /* This will either be equal to or one */ |
86 | /* behind the root rcu_node's gpnum. */ | 97 | /* behind the root rcu_node's gpnum. */ |
98 | long completed; /* Last grace period completed for this node. */ | ||
99 | /* This will either be equal to or one */ | ||
100 | /* behind the root rcu_node's gpnum. */ | ||
87 | unsigned long qsmask; /* CPUs or groups that need to switch in */ | 101 | unsigned long qsmask; /* CPUs or groups that need to switch in */ |
88 | /* order for current grace period to proceed.*/ | 102 | /* order for current grace period to proceed.*/ |
89 | /* In leaf rcu_node, each bit corresponds to */ | 103 | /* In leaf rcu_node, each bit corresponds to */ |
90 | /* an rcu_data structure, otherwise, each */ | 104 | /* an rcu_data structure, otherwise, each */ |
91 | /* bit corresponds to a child rcu_node */ | 105 | /* bit corresponds to a child rcu_node */ |
92 | /* structure. */ | 106 | /* structure. */ |
107 | unsigned long expmask; /* Groups that have ->blocked_tasks[] */ | ||
108 | /* elements that need to drain to allow the */ | ||
109 | /* current expedited grace period to */ | ||
110 | /* complete (only for TREE_PREEMPT_RCU). */ | ||
93 | unsigned long qsmaskinit; | 111 | unsigned long qsmaskinit; |
94 | /* Per-GP initialization for qsmask. */ | 112 | /* Per-GP initial value for qsmask & expmask. */ |
95 | unsigned long grpmask; /* Mask to apply to parent qsmask. */ | 113 | unsigned long grpmask; /* Mask to apply to parent qsmask. */ |
96 | /* Only one bit will be set in this mask. */ | 114 | /* Only one bit will be set in this mask. */ |
97 | int grplo; /* lowest-numbered CPU or group here. */ | 115 | int grplo; /* lowest-numbered CPU or group here. */ |
@@ -99,7 +117,7 @@ struct rcu_node { | |||
99 | u8 grpnum; /* CPU/group number for next level up. */ | 117 | u8 grpnum; /* CPU/group number for next level up. */ |
100 | u8 level; /* root is at level 0. */ | 118 | u8 level; /* root is at level 0. */ |
101 | struct rcu_node *parent; | 119 | struct rcu_node *parent; |
102 | struct list_head blocked_tasks[2]; | 120 | struct list_head blocked_tasks[4]; |
103 | /* Tasks blocked in RCU read-side critsect. */ | 121 | /* Tasks blocked in RCU read-side critsect. */ |
104 | /* Grace period number (->gpnum) x blocked */ | 122 | /* Grace period number (->gpnum) x blocked */ |
105 | /* by tasks on the (x & 0x1) element of the */ | 123 | /* by tasks on the (x & 0x1) element of the */ |
@@ -114,6 +132,21 @@ struct rcu_node { | |||
114 | for ((rnp) = &(rsp)->node[0]; \ | 132 | for ((rnp) = &(rsp)->node[0]; \ |
115 | (rnp) < &(rsp)->node[NUM_RCU_NODES]; (rnp)++) | 133 | (rnp) < &(rsp)->node[NUM_RCU_NODES]; (rnp)++) |
116 | 134 | ||
135 | /* | ||
136 | * Do a breadth-first scan of the non-leaf rcu_node structures for the | ||
137 | * specified rcu_state structure. Note that if there is a singleton | ||
138 | * rcu_node tree with but one rcu_node structure, this loop is a no-op. | ||
139 | */ | ||
140 | #define rcu_for_each_nonleaf_node_breadth_first(rsp, rnp) \ | ||
141 | for ((rnp) = &(rsp)->node[0]; \ | ||
142 | (rnp) < (rsp)->level[NUM_RCU_LVLS - 1]; (rnp)++) | ||
143 | |||
144 | /* | ||
145 | * Scan the leaves of the rcu_node hierarchy for the specified rcu_state | ||
146 | * structure. Note that if there is a singleton rcu_node tree with but | ||
147 | * one rcu_node structure, this loop -will- visit the rcu_node structure. | ||
148 | * It is still a leaf node, even if it is also the root node. | ||
149 | */ | ||
117 | #define rcu_for_each_leaf_node(rsp, rnp) \ | 150 | #define rcu_for_each_leaf_node(rsp, rnp) \ |
118 | for ((rnp) = (rsp)->level[NUM_RCU_LVLS - 1]; \ | 151 | for ((rnp) = (rsp)->level[NUM_RCU_LVLS - 1]; \ |
119 | (rnp) < &(rsp)->node[NUM_RCU_NODES]; (rnp)++) | 152 | (rnp) < &(rsp)->node[NUM_RCU_NODES]; (rnp)++) |
@@ -167,6 +200,10 @@ struct rcu_data { | |||
167 | struct rcu_head *nxtlist; | 200 | struct rcu_head *nxtlist; |
168 | struct rcu_head **nxttail[RCU_NEXT_SIZE]; | 201 | struct rcu_head **nxttail[RCU_NEXT_SIZE]; |
169 | long qlen; /* # of queued callbacks */ | 202 | long qlen; /* # of queued callbacks */ |
203 | long qlen_last_fqs_check; | ||
204 | /* qlen at last check for QS forcing */ | ||
205 | unsigned long n_force_qs_snap; | ||
206 | /* did other CPU force QS recently? */ | ||
170 | long blimit; /* Upper limit on a processed batch */ | 207 | long blimit; /* Upper limit on a processed batch */ |
171 | 208 | ||
172 | #ifdef CONFIG_NO_HZ | 209 | #ifdef CONFIG_NO_HZ |
@@ -197,13 +234,15 @@ struct rcu_data { | |||
197 | }; | 234 | }; |
198 | 235 | ||
199 | /* Values for signaled field in struct rcu_state. */ | 236 | /* Values for signaled field in struct rcu_state. */ |
200 | #define RCU_GP_INIT 0 /* Grace period being initialized. */ | 237 | #define RCU_GP_IDLE 0 /* No grace period in progress. */ |
201 | #define RCU_SAVE_DYNTICK 1 /* Need to scan dyntick state. */ | 238 | #define RCU_GP_INIT 1 /* Grace period being initialized. */ |
202 | #define RCU_FORCE_QS 2 /* Need to force quiescent state. */ | 239 | #define RCU_SAVE_DYNTICK 2 /* Need to scan dyntick state. */ |
240 | #define RCU_SAVE_COMPLETED 3 /* Need to save rsp->completed. */ | ||
241 | #define RCU_FORCE_QS 4 /* Need to force quiescent state. */ | ||
203 | #ifdef CONFIG_NO_HZ | 242 | #ifdef CONFIG_NO_HZ |
204 | #define RCU_SIGNAL_INIT RCU_SAVE_DYNTICK | 243 | #define RCU_SIGNAL_INIT RCU_SAVE_DYNTICK |
205 | #else /* #ifdef CONFIG_NO_HZ */ | 244 | #else /* #ifdef CONFIG_NO_HZ */ |
206 | #define RCU_SIGNAL_INIT RCU_FORCE_QS | 245 | #define RCU_SIGNAL_INIT RCU_SAVE_COMPLETED |
207 | #endif /* #else #ifdef CONFIG_NO_HZ */ | 246 | #endif /* #else #ifdef CONFIG_NO_HZ */ |
208 | 247 | ||
209 | #define RCU_JIFFIES_TILL_FORCE_QS 3 /* for rsp->jiffies_force_qs */ | 248 | #define RCU_JIFFIES_TILL_FORCE_QS 3 /* for rsp->jiffies_force_qs */ |
@@ -241,7 +280,7 @@ struct rcu_state { | |||
241 | long gpnum; /* Current gp number. */ | 280 | long gpnum; /* Current gp number. */ |
242 | long completed; /* # of last completed gp. */ | 281 | long completed; /* # of last completed gp. */ |
243 | 282 | ||
244 | /* End of fields guarded by root rcu_node's lock. */ | 283 | /* End of fields guarded by root rcu_node's lock. */ |
245 | 284 | ||
246 | spinlock_t onofflock; /* exclude on/offline and */ | 285 | spinlock_t onofflock; /* exclude on/offline and */ |
247 | /* starting new GP. Also */ | 286 | /* starting new GP. Also */ |
@@ -255,6 +294,8 @@ struct rcu_state { | |||
255 | long orphan_qlen; /* Number of orphaned cbs. */ | 294 | long orphan_qlen; /* Number of orphaned cbs. */ |
256 | spinlock_t fqslock; /* Only one task forcing */ | 295 | spinlock_t fqslock; /* Only one task forcing */ |
257 | /* quiescent states. */ | 296 | /* quiescent states. */ |
297 | long completed_fqs; /* Value of completed @ snap. */ | ||
298 | /* Protected by fqslock. */ | ||
258 | unsigned long jiffies_force_qs; /* Time at which to invoke */ | 299 | unsigned long jiffies_force_qs; /* Time at which to invoke */ |
259 | /* force_quiescent_state(). */ | 300 | /* force_quiescent_state(). */ |
260 | unsigned long n_force_qs; /* Number of calls to */ | 301 | unsigned long n_force_qs; /* Number of calls to */ |
@@ -269,11 +310,15 @@ struct rcu_state { | |||
269 | unsigned long jiffies_stall; /* Time at which to check */ | 310 | unsigned long jiffies_stall; /* Time at which to check */ |
270 | /* for CPU stalls. */ | 311 | /* for CPU stalls. */ |
271 | #endif /* #ifdef CONFIG_RCU_CPU_STALL_DETECTOR */ | 312 | #endif /* #ifdef CONFIG_RCU_CPU_STALL_DETECTOR */ |
272 | #ifdef CONFIG_NO_HZ | ||
273 | long dynticks_completed; /* Value of completed @ snap. */ | ||
274 | #endif /* #ifdef CONFIG_NO_HZ */ | ||
275 | }; | 313 | }; |
276 | 314 | ||
315 | /* Return values for rcu_preempt_offline_tasks(). */ | ||
316 | |||
317 | #define RCU_OFL_TASKS_NORM_GP 0x1 /* Tasks blocking normal */ | ||
318 | /* GP were moved to root. */ | ||
319 | #define RCU_OFL_TASKS_EXP_GP 0x2 /* Tasks blocking expedited */ | ||
320 | /* GP were moved to root. */ | ||
321 | |||
277 | #ifdef RCU_TREE_NONCORE | 322 | #ifdef RCU_TREE_NONCORE |
278 | 323 | ||
279 | /* | 324 | /* |
@@ -293,23 +338,30 @@ DECLARE_PER_CPU(struct rcu_data, rcu_preempt_data); | |||
293 | #else /* #ifdef RCU_TREE_NONCORE */ | 338 | #else /* #ifdef RCU_TREE_NONCORE */ |
294 | 339 | ||
295 | /* Forward declarations for rcutree_plugin.h */ | 340 | /* Forward declarations for rcutree_plugin.h */ |
296 | static inline void rcu_bootup_announce(void); | 341 | static void rcu_bootup_announce(void); |
297 | long rcu_batches_completed(void); | 342 | long rcu_batches_completed(void); |
298 | static void rcu_preempt_note_context_switch(int cpu); | 343 | static void rcu_preempt_note_context_switch(int cpu); |
299 | static int rcu_preempted_readers(struct rcu_node *rnp); | 344 | static int rcu_preempted_readers(struct rcu_node *rnp); |
345 | #ifdef CONFIG_HOTPLUG_CPU | ||
346 | static void rcu_report_unblock_qs_rnp(struct rcu_node *rnp, | ||
347 | unsigned long flags); | ||
348 | #endif /* #ifdef CONFIG_HOTPLUG_CPU */ | ||
300 | #ifdef CONFIG_RCU_CPU_STALL_DETECTOR | 349 | #ifdef CONFIG_RCU_CPU_STALL_DETECTOR |
301 | static void rcu_print_task_stall(struct rcu_node *rnp); | 350 | static void rcu_print_task_stall(struct rcu_node *rnp); |
302 | #endif /* #ifdef CONFIG_RCU_CPU_STALL_DETECTOR */ | 351 | #endif /* #ifdef CONFIG_RCU_CPU_STALL_DETECTOR */ |
303 | static void rcu_preempt_check_blocked_tasks(struct rcu_node *rnp); | 352 | static void rcu_preempt_check_blocked_tasks(struct rcu_node *rnp); |
304 | #ifdef CONFIG_HOTPLUG_CPU | 353 | #ifdef CONFIG_HOTPLUG_CPU |
305 | static void rcu_preempt_offline_tasks(struct rcu_state *rsp, | 354 | static int rcu_preempt_offline_tasks(struct rcu_state *rsp, |
306 | struct rcu_node *rnp, | 355 | struct rcu_node *rnp, |
307 | struct rcu_data *rdp); | 356 | struct rcu_data *rdp); |
308 | static void rcu_preempt_offline_cpu(int cpu); | 357 | static void rcu_preempt_offline_cpu(int cpu); |
309 | #endif /* #ifdef CONFIG_HOTPLUG_CPU */ | 358 | #endif /* #ifdef CONFIG_HOTPLUG_CPU */ |
310 | static void rcu_preempt_check_callbacks(int cpu); | 359 | static void rcu_preempt_check_callbacks(int cpu); |
311 | static void rcu_preempt_process_callbacks(void); | 360 | static void rcu_preempt_process_callbacks(void); |
312 | void call_rcu(struct rcu_head *head, void (*func)(struct rcu_head *rcu)); | 361 | void call_rcu(struct rcu_head *head, void (*func)(struct rcu_head *rcu)); |
362 | #if defined(CONFIG_HOTPLUG_CPU) || defined(CONFIG_TREE_PREEMPT_RCU) | ||
363 | static void rcu_report_exp_rnp(struct rcu_state *rsp, struct rcu_node *rnp); | ||
364 | #endif /* #if defined(CONFIG_HOTPLUG_CPU) || defined(CONFIG_TREE_PREEMPT_RCU) */ | ||
313 | static int rcu_preempt_pending(int cpu); | 365 | static int rcu_preempt_pending(int cpu); |
314 | static int rcu_preempt_needs_cpu(int cpu); | 366 | static int rcu_preempt_needs_cpu(int cpu); |
315 | static void __cpuinit rcu_preempt_init_percpu_data(int cpu); | 367 | static void __cpuinit rcu_preempt_init_percpu_data(int cpu); |
diff --git a/kernel/rcutree_plugin.h b/kernel/rcutree_plugin.h index c0cb783aa16a..37fbccdf41d5 100644 --- a/kernel/rcutree_plugin.h +++ b/kernel/rcutree_plugin.h | |||
@@ -24,16 +24,19 @@ | |||
24 | * Paul E. McKenney <paulmck@linux.vnet.ibm.com> | 24 | * Paul E. McKenney <paulmck@linux.vnet.ibm.com> |
25 | */ | 25 | */ |
26 | 26 | ||
27 | #include <linux/delay.h> | ||
27 | 28 | ||
28 | #ifdef CONFIG_TREE_PREEMPT_RCU | 29 | #ifdef CONFIG_TREE_PREEMPT_RCU |
29 | 30 | ||
30 | struct rcu_state rcu_preempt_state = RCU_STATE_INITIALIZER(rcu_preempt_state); | 31 | struct rcu_state rcu_preempt_state = RCU_STATE_INITIALIZER(rcu_preempt_state); |
31 | DEFINE_PER_CPU(struct rcu_data, rcu_preempt_data); | 32 | DEFINE_PER_CPU(struct rcu_data, rcu_preempt_data); |
32 | 33 | ||
34 | static int rcu_preempted_readers_exp(struct rcu_node *rnp); | ||
35 | |||
33 | /* | 36 | /* |
34 | * Tell them what RCU they are running. | 37 | * Tell them what RCU they are running. |
35 | */ | 38 | */ |
36 | static inline void rcu_bootup_announce(void) | 39 | static void __init rcu_bootup_announce(void) |
37 | { | 40 | { |
38 | printk(KERN_INFO | 41 | printk(KERN_INFO |
39 | "Experimental preemptable hierarchical RCU implementation.\n"); | 42 | "Experimental preemptable hierarchical RCU implementation.\n"); |
@@ -67,7 +70,7 @@ EXPORT_SYMBOL_GPL(rcu_batches_completed); | |||
67 | static void rcu_preempt_qs(int cpu) | 70 | static void rcu_preempt_qs(int cpu) |
68 | { | 71 | { |
69 | struct rcu_data *rdp = &per_cpu(rcu_preempt_data, cpu); | 72 | struct rcu_data *rdp = &per_cpu(rcu_preempt_data, cpu); |
70 | rdp->passed_quiesc_completed = rdp->completed; | 73 | rdp->passed_quiesc_completed = rdp->gpnum - 1; |
71 | barrier(); | 74 | barrier(); |
72 | rdp->passed_quiesc = 1; | 75 | rdp->passed_quiesc = 1; |
73 | } | 76 | } |
@@ -157,14 +160,58 @@ EXPORT_SYMBOL_GPL(__rcu_read_lock); | |||
157 | */ | 160 | */ |
158 | static int rcu_preempted_readers(struct rcu_node *rnp) | 161 | static int rcu_preempted_readers(struct rcu_node *rnp) |
159 | { | 162 | { |
160 | return !list_empty(&rnp->blocked_tasks[rnp->gpnum & 0x1]); | 163 | int phase = rnp->gpnum & 0x1; |
164 | |||
165 | return !list_empty(&rnp->blocked_tasks[phase]) || | ||
166 | !list_empty(&rnp->blocked_tasks[phase + 2]); | ||
167 | } | ||
168 | |||
169 | /* | ||
170 | * Record a quiescent state for all tasks that were previously queued | ||
171 | * on the specified rcu_node structure and that were blocking the current | ||
172 | * RCU grace period. The caller must hold the specified rnp->lock with | ||
173 | * irqs disabled, and this lock is released upon return, but irqs remain | ||
174 | * disabled. | ||
175 | */ | ||
176 | static void rcu_report_unblock_qs_rnp(struct rcu_node *rnp, unsigned long flags) | ||
177 | __releases(rnp->lock) | ||
178 | { | ||
179 | unsigned long mask; | ||
180 | struct rcu_node *rnp_p; | ||
181 | |||
182 | if (rnp->qsmask != 0 || rcu_preempted_readers(rnp)) { | ||
183 | spin_unlock_irqrestore(&rnp->lock, flags); | ||
184 | return; /* Still need more quiescent states! */ | ||
185 | } | ||
186 | |||
187 | rnp_p = rnp->parent; | ||
188 | if (rnp_p == NULL) { | ||
189 | /* | ||
190 | * Either there is only one rcu_node in the tree, | ||
191 | * or tasks were kicked up to root rcu_node due to | ||
192 | * CPUs going offline. | ||
193 | */ | ||
194 | rcu_report_qs_rsp(&rcu_preempt_state, flags); | ||
195 | return; | ||
196 | } | ||
197 | |||
198 | /* Report up the rest of the hierarchy. */ | ||
199 | mask = rnp->grpmask; | ||
200 | spin_unlock(&rnp->lock); /* irqs remain disabled. */ | ||
201 | spin_lock(&rnp_p->lock); /* irqs already disabled. */ | ||
202 | rcu_report_qs_rnp(mask, &rcu_preempt_state, rnp_p, flags); | ||
161 | } | 203 | } |
162 | 204 | ||
205 | /* | ||
206 | * Handle special cases during rcu_read_unlock(), such as needing to | ||
207 | * notify RCU core processing or task having blocked during the RCU | ||
208 | * read-side critical section. | ||
209 | */ | ||
163 | static void rcu_read_unlock_special(struct task_struct *t) | 210 | static void rcu_read_unlock_special(struct task_struct *t) |
164 | { | 211 | { |
165 | int empty; | 212 | int empty; |
213 | int empty_exp; | ||
166 | unsigned long flags; | 214 | unsigned long flags; |
167 | unsigned long mask; | ||
168 | struct rcu_node *rnp; | 215 | struct rcu_node *rnp; |
169 | int special; | 216 | int special; |
170 | 217 | ||
@@ -207,36 +254,30 @@ static void rcu_read_unlock_special(struct task_struct *t) | |||
207 | spin_unlock(&rnp->lock); /* irqs remain disabled. */ | 254 | spin_unlock(&rnp->lock); /* irqs remain disabled. */ |
208 | } | 255 | } |
209 | empty = !rcu_preempted_readers(rnp); | 256 | empty = !rcu_preempted_readers(rnp); |
257 | empty_exp = !rcu_preempted_readers_exp(rnp); | ||
258 | smp_mb(); /* ensure expedited fastpath sees end of RCU c-s. */ | ||
210 | list_del_init(&t->rcu_node_entry); | 259 | list_del_init(&t->rcu_node_entry); |
211 | t->rcu_blocked_node = NULL; | 260 | t->rcu_blocked_node = NULL; |
212 | 261 | ||
213 | /* | 262 | /* |
214 | * If this was the last task on the current list, and if | 263 | * If this was the last task on the current list, and if |
215 | * we aren't waiting on any CPUs, report the quiescent state. | 264 | * we aren't waiting on any CPUs, report the quiescent state. |
216 | * Note that both cpu_quiet_msk_finish() and cpu_quiet_msk() | 265 | * Note that rcu_report_unblock_qs_rnp() releases rnp->lock. |
217 | * drop rnp->lock and restore irq. | ||
218 | */ | 266 | */ |
219 | if (!empty && rnp->qsmask == 0 && | 267 | if (empty) |
220 | !rcu_preempted_readers(rnp)) { | ||
221 | struct rcu_node *rnp_p; | ||
222 | |||
223 | if (rnp->parent == NULL) { | ||
224 | /* Only one rcu_node in the tree. */ | ||
225 | cpu_quiet_msk_finish(&rcu_preempt_state, flags); | ||
226 | return; | ||
227 | } | ||
228 | /* Report up the rest of the hierarchy. */ | ||
229 | mask = rnp->grpmask; | ||
230 | spin_unlock_irqrestore(&rnp->lock, flags); | 268 | spin_unlock_irqrestore(&rnp->lock, flags); |
231 | rnp_p = rnp->parent; | 269 | else |
232 | spin_lock_irqsave(&rnp_p->lock, flags); | 270 | rcu_report_unblock_qs_rnp(rnp, flags); |
233 | WARN_ON_ONCE(rnp->qsmask); | 271 | |
234 | cpu_quiet_msk(mask, &rcu_preempt_state, rnp_p, flags); | 272 | /* |
235 | return; | 273 | * If this was the last task on the expedited lists, |
236 | } | 274 | * then we need to report up the rcu_node hierarchy. |
237 | spin_unlock(&rnp->lock); | 275 | */ |
276 | if (!empty_exp && !rcu_preempted_readers_exp(rnp)) | ||
277 | rcu_report_exp_rnp(&rcu_preempt_state, rnp); | ||
278 | } else { | ||
279 | local_irq_restore(flags); | ||
238 | } | 280 | } |
239 | local_irq_restore(flags); | ||
240 | } | 281 | } |
241 | 282 | ||
242 | /* | 283 | /* |
@@ -303,26 +344,34 @@ static void rcu_preempt_check_blocked_tasks(struct rcu_node *rnp) | |||
303 | * rcu_node. The reason for not just moving them to the immediate | 344 | * rcu_node. The reason for not just moving them to the immediate |
304 | * parent is to remove the need for rcu_read_unlock_special() to | 345 | * parent is to remove the need for rcu_read_unlock_special() to |
305 | * make more than two attempts to acquire the target rcu_node's lock. | 346 | * make more than two attempts to acquire the target rcu_node's lock. |
347 | * Returns true if there were tasks blocking the current RCU grace | ||
348 | * period. | ||
349 | * | ||
350 | * Returns 1 if there was previously a task blocking the current grace | ||
351 | * period on the specified rcu_node structure. | ||
306 | * | 352 | * |
307 | * The caller must hold rnp->lock with irqs disabled. | 353 | * The caller must hold rnp->lock with irqs disabled. |
308 | */ | 354 | */ |
309 | static void rcu_preempt_offline_tasks(struct rcu_state *rsp, | 355 | static int rcu_preempt_offline_tasks(struct rcu_state *rsp, |
310 | struct rcu_node *rnp, | 356 | struct rcu_node *rnp, |
311 | struct rcu_data *rdp) | 357 | struct rcu_data *rdp) |
312 | { | 358 | { |
313 | int i; | 359 | int i; |
314 | struct list_head *lp; | 360 | struct list_head *lp; |
315 | struct list_head *lp_root; | 361 | struct list_head *lp_root; |
362 | int retval = 0; | ||
316 | struct rcu_node *rnp_root = rcu_get_root(rsp); | 363 | struct rcu_node *rnp_root = rcu_get_root(rsp); |
317 | struct task_struct *tp; | 364 | struct task_struct *tp; |
318 | 365 | ||
319 | if (rnp == rnp_root) { | 366 | if (rnp == rnp_root) { |
320 | WARN_ONCE(1, "Last CPU thought to be offlined?"); | 367 | WARN_ONCE(1, "Last CPU thought to be offlined?"); |
321 | return; /* Shouldn't happen: at least one CPU online. */ | 368 | return 0; /* Shouldn't happen: at least one CPU online. */ |
322 | } | 369 | } |
323 | WARN_ON_ONCE(rnp != rdp->mynode && | 370 | WARN_ON_ONCE(rnp != rdp->mynode && |
324 | (!list_empty(&rnp->blocked_tasks[0]) || | 371 | (!list_empty(&rnp->blocked_tasks[0]) || |
325 | !list_empty(&rnp->blocked_tasks[1]))); | 372 | !list_empty(&rnp->blocked_tasks[1]) || |
373 | !list_empty(&rnp->blocked_tasks[2]) || | ||
374 | !list_empty(&rnp->blocked_tasks[3]))); | ||
326 | 375 | ||
327 | /* | 376 | /* |
328 | * Move tasks up to root rcu_node. Rely on the fact that the | 377 | * Move tasks up to root rcu_node. Rely on the fact that the |
@@ -330,7 +379,11 @@ static void rcu_preempt_offline_tasks(struct rcu_state *rsp, | |||
330 | * rcu_nodes in terms of gp_num value. This fact allows us to | 379 | * rcu_nodes in terms of gp_num value. This fact allows us to |
331 | * move the blocked_tasks[] array directly, element by element. | 380 | * move the blocked_tasks[] array directly, element by element. |
332 | */ | 381 | */ |
333 | for (i = 0; i < 2; i++) { | 382 | if (rcu_preempted_readers(rnp)) |
383 | retval |= RCU_OFL_TASKS_NORM_GP; | ||
384 | if (rcu_preempted_readers_exp(rnp)) | ||
385 | retval |= RCU_OFL_TASKS_EXP_GP; | ||
386 | for (i = 0; i < 4; i++) { | ||
334 | lp = &rnp->blocked_tasks[i]; | 387 | lp = &rnp->blocked_tasks[i]; |
335 | lp_root = &rnp_root->blocked_tasks[i]; | 388 | lp_root = &rnp_root->blocked_tasks[i]; |
336 | while (!list_empty(lp)) { | 389 | while (!list_empty(lp)) { |
@@ -342,6 +395,7 @@ static void rcu_preempt_offline_tasks(struct rcu_state *rsp, | |||
342 | spin_unlock(&rnp_root->lock); /* irqs remain disabled */ | 395 | spin_unlock(&rnp_root->lock); /* irqs remain disabled */ |
343 | } | 396 | } |
344 | } | 397 | } |
398 | return retval; | ||
345 | } | 399 | } |
346 | 400 | ||
347 | /* | 401 | /* |
@@ -392,6 +446,186 @@ void call_rcu(struct rcu_head *head, void (*func)(struct rcu_head *rcu)) | |||
392 | } | 446 | } |
393 | EXPORT_SYMBOL_GPL(call_rcu); | 447 | EXPORT_SYMBOL_GPL(call_rcu); |
394 | 448 | ||
449 | /** | ||
450 | * synchronize_rcu - wait until a grace period has elapsed. | ||
451 | * | ||
452 | * Control will return to the caller some time after a full grace | ||
453 | * period has elapsed, in other words after all currently executing RCU | ||
454 | * read-side critical sections have completed. RCU read-side critical | ||
455 | * sections are delimited by rcu_read_lock() and rcu_read_unlock(), | ||
456 | * and may be nested. | ||
457 | */ | ||
458 | void synchronize_rcu(void) | ||
459 | { | ||
460 | struct rcu_synchronize rcu; | ||
461 | |||
462 | if (!rcu_scheduler_active) | ||
463 | return; | ||
464 | |||
465 | init_completion(&rcu.completion); | ||
466 | /* Will wake me after RCU finished. */ | ||
467 | call_rcu(&rcu.head, wakeme_after_rcu); | ||
468 | /* Wait for it. */ | ||
469 | wait_for_completion(&rcu.completion); | ||
470 | } | ||
471 | EXPORT_SYMBOL_GPL(synchronize_rcu); | ||
472 | |||
473 | static DECLARE_WAIT_QUEUE_HEAD(sync_rcu_preempt_exp_wq); | ||
474 | static long sync_rcu_preempt_exp_count; | ||
475 | static DEFINE_MUTEX(sync_rcu_preempt_exp_mutex); | ||
476 | |||
477 | /* | ||
478 | * Return non-zero if there are any tasks in RCU read-side critical | ||
479 | * sections blocking the current preemptible-RCU expedited grace period. | ||
480 | * If there is no preemptible-RCU expedited grace period currently in | ||
481 | * progress, returns zero unconditionally. | ||
482 | */ | ||
483 | static int rcu_preempted_readers_exp(struct rcu_node *rnp) | ||
484 | { | ||
485 | return !list_empty(&rnp->blocked_tasks[2]) || | ||
486 | !list_empty(&rnp->blocked_tasks[3]); | ||
487 | } | ||
488 | |||
489 | /* | ||
490 | * return non-zero if there is no RCU expedited grace period in progress | ||
491 | * for the specified rcu_node structure, in other words, if all CPUs and | ||
492 | * tasks covered by the specified rcu_node structure have done their bit | ||
493 | * for the current expedited grace period. Works only for preemptible | ||
494 | * RCU -- other RCU implementation use other means. | ||
495 | * | ||
496 | * Caller must hold sync_rcu_preempt_exp_mutex. | ||
497 | */ | ||
498 | static int sync_rcu_preempt_exp_done(struct rcu_node *rnp) | ||
499 | { | ||
500 | return !rcu_preempted_readers_exp(rnp) && | ||
501 | ACCESS_ONCE(rnp->expmask) == 0; | ||
502 | } | ||
503 | |||
504 | /* | ||
505 | * Report the exit from RCU read-side critical section for the last task | ||
506 | * that queued itself during or before the current expedited preemptible-RCU | ||
507 | * grace period. This event is reported either to the rcu_node structure on | ||
508 | * which the task was queued or to one of that rcu_node structure's ancestors, | ||
509 | * recursively up the tree. (Calm down, calm down, we do the recursion | ||
510 | * iteratively!) | ||
511 | * | ||
512 | * Caller must hold sync_rcu_preempt_exp_mutex. | ||
513 | */ | ||
514 | static void rcu_report_exp_rnp(struct rcu_state *rsp, struct rcu_node *rnp) | ||
515 | { | ||
516 | unsigned long flags; | ||
517 | unsigned long mask; | ||
518 | |||
519 | spin_lock_irqsave(&rnp->lock, flags); | ||
520 | for (;;) { | ||
521 | if (!sync_rcu_preempt_exp_done(rnp)) | ||
522 | break; | ||
523 | if (rnp->parent == NULL) { | ||
524 | wake_up(&sync_rcu_preempt_exp_wq); | ||
525 | break; | ||
526 | } | ||
527 | mask = rnp->grpmask; | ||
528 | spin_unlock(&rnp->lock); /* irqs remain disabled */ | ||
529 | rnp = rnp->parent; | ||
530 | spin_lock(&rnp->lock); /* irqs already disabled */ | ||
531 | rnp->expmask &= ~mask; | ||
532 | } | ||
533 | spin_unlock_irqrestore(&rnp->lock, flags); | ||
534 | } | ||
535 | |||
536 | /* | ||
537 | * Snapshot the tasks blocking the newly started preemptible-RCU expedited | ||
538 | * grace period for the specified rcu_node structure. If there are no such | ||
539 | * tasks, report it up the rcu_node hierarchy. | ||
540 | * | ||
541 | * Caller must hold sync_rcu_preempt_exp_mutex and rsp->onofflock. | ||
542 | */ | ||
543 | static void | ||
544 | sync_rcu_preempt_exp_init(struct rcu_state *rsp, struct rcu_node *rnp) | ||
545 | { | ||
546 | int must_wait; | ||
547 | |||
548 | spin_lock(&rnp->lock); /* irqs already disabled */ | ||
549 | list_splice_init(&rnp->blocked_tasks[0], &rnp->blocked_tasks[2]); | ||
550 | list_splice_init(&rnp->blocked_tasks[1], &rnp->blocked_tasks[3]); | ||
551 | must_wait = rcu_preempted_readers_exp(rnp); | ||
552 | spin_unlock(&rnp->lock); /* irqs remain disabled */ | ||
553 | if (!must_wait) | ||
554 | rcu_report_exp_rnp(rsp, rnp); | ||
555 | } | ||
556 | |||
557 | /* | ||
558 | * Wait for an rcu-preempt grace period, but expedite it. The basic idea | ||
559 | * is to invoke synchronize_sched_expedited() to push all the tasks to | ||
560 | * the ->blocked_tasks[] lists, move all entries from the first set of | ||
561 | * ->blocked_tasks[] lists to the second set, and finally wait for this | ||
562 | * second set to drain. | ||
563 | */ | ||
564 | void synchronize_rcu_expedited(void) | ||
565 | { | ||
566 | unsigned long flags; | ||
567 | struct rcu_node *rnp; | ||
568 | struct rcu_state *rsp = &rcu_preempt_state; | ||
569 | long snap; | ||
570 | int trycount = 0; | ||
571 | |||
572 | smp_mb(); /* Caller's modifications seen first by other CPUs. */ | ||
573 | snap = ACCESS_ONCE(sync_rcu_preempt_exp_count) + 1; | ||
574 | smp_mb(); /* Above access cannot bleed into critical section. */ | ||
575 | |||
576 | /* | ||
577 | * Acquire lock, falling back to synchronize_rcu() if too many | ||
578 | * lock-acquisition failures. Of course, if someone does the | ||
579 | * expedited grace period for us, just leave. | ||
580 | */ | ||
581 | while (!mutex_trylock(&sync_rcu_preempt_exp_mutex)) { | ||
582 | if (trycount++ < 10) | ||
583 | udelay(trycount * num_online_cpus()); | ||
584 | else { | ||
585 | synchronize_rcu(); | ||
586 | return; | ||
587 | } | ||
588 | if ((ACCESS_ONCE(sync_rcu_preempt_exp_count) - snap) > 0) | ||
589 | goto mb_ret; /* Others did our work for us. */ | ||
590 | } | ||
591 | if ((ACCESS_ONCE(sync_rcu_preempt_exp_count) - snap) > 0) | ||
592 | goto unlock_mb_ret; /* Others did our work for us. */ | ||
593 | |||
594 | /* force all RCU readers onto blocked_tasks[]. */ | ||
595 | synchronize_sched_expedited(); | ||
596 | |||
597 | spin_lock_irqsave(&rsp->onofflock, flags); | ||
598 | |||
599 | /* Initialize ->expmask for all non-leaf rcu_node structures. */ | ||
600 | rcu_for_each_nonleaf_node_breadth_first(rsp, rnp) { | ||
601 | spin_lock(&rnp->lock); /* irqs already disabled. */ | ||
602 | rnp->expmask = rnp->qsmaskinit; | ||
603 | spin_unlock(&rnp->lock); /* irqs remain disabled. */ | ||
604 | } | ||
605 | |||
606 | /* Snapshot current state of ->blocked_tasks[] lists. */ | ||
607 | rcu_for_each_leaf_node(rsp, rnp) | ||
608 | sync_rcu_preempt_exp_init(rsp, rnp); | ||
609 | if (NUM_RCU_NODES > 1) | ||
610 | sync_rcu_preempt_exp_init(rsp, rcu_get_root(rsp)); | ||
611 | |||
612 | spin_unlock_irqrestore(&rsp->onofflock, flags); | ||
613 | |||
614 | /* Wait for snapshotted ->blocked_tasks[] lists to drain. */ | ||
615 | rnp = rcu_get_root(rsp); | ||
616 | wait_event(sync_rcu_preempt_exp_wq, | ||
617 | sync_rcu_preempt_exp_done(rnp)); | ||
618 | |||
619 | /* Clean up and exit. */ | ||
620 | smp_mb(); /* ensure expedited GP seen before counter increment. */ | ||
621 | ACCESS_ONCE(sync_rcu_preempt_exp_count)++; | ||
622 | unlock_mb_ret: | ||
623 | mutex_unlock(&sync_rcu_preempt_exp_mutex); | ||
624 | mb_ret: | ||
625 | smp_mb(); /* ensure subsequent action seen after grace period. */ | ||
626 | } | ||
627 | EXPORT_SYMBOL_GPL(synchronize_rcu_expedited); | ||
628 | |||
395 | /* | 629 | /* |
396 | * Check to see if there is any immediate preemptable-RCU-related work | 630 | * Check to see if there is any immediate preemptable-RCU-related work |
397 | * to be done. | 631 | * to be done. |
@@ -464,7 +698,7 @@ void exit_rcu(void) | |||
464 | /* | 698 | /* |
465 | * Tell them what RCU they are running. | 699 | * Tell them what RCU they are running. |
466 | */ | 700 | */ |
467 | static inline void rcu_bootup_announce(void) | 701 | static void __init rcu_bootup_announce(void) |
468 | { | 702 | { |
469 | printk(KERN_INFO "Hierarchical RCU implementation.\n"); | 703 | printk(KERN_INFO "Hierarchical RCU implementation.\n"); |
470 | } | 704 | } |
@@ -495,6 +729,16 @@ static int rcu_preempted_readers(struct rcu_node *rnp) | |||
495 | return 0; | 729 | return 0; |
496 | } | 730 | } |
497 | 731 | ||
732 | #ifdef CONFIG_HOTPLUG_CPU | ||
733 | |||
734 | /* Because preemptible RCU does not exist, no quieting of tasks. */ | ||
735 | static void rcu_report_unblock_qs_rnp(struct rcu_node *rnp, unsigned long flags) | ||
736 | { | ||
737 | spin_unlock_irqrestore(&rnp->lock, flags); | ||
738 | } | ||
739 | |||
740 | #endif /* #ifdef CONFIG_HOTPLUG_CPU */ | ||
741 | |||
498 | #ifdef CONFIG_RCU_CPU_STALL_DETECTOR | 742 | #ifdef CONFIG_RCU_CPU_STALL_DETECTOR |
499 | 743 | ||
500 | /* | 744 | /* |
@@ -521,12 +765,15 @@ static void rcu_preempt_check_blocked_tasks(struct rcu_node *rnp) | |||
521 | 765 | ||
522 | /* | 766 | /* |
523 | * Because preemptable RCU does not exist, it never needs to migrate | 767 | * Because preemptable RCU does not exist, it never needs to migrate |
524 | * tasks that were blocked within RCU read-side critical sections. | 768 | * tasks that were blocked within RCU read-side critical sections, and |
769 | * such non-existent tasks cannot possibly have been blocking the current | ||
770 | * grace period. | ||
525 | */ | 771 | */ |
526 | static void rcu_preempt_offline_tasks(struct rcu_state *rsp, | 772 | static int rcu_preempt_offline_tasks(struct rcu_state *rsp, |
527 | struct rcu_node *rnp, | 773 | struct rcu_node *rnp, |
528 | struct rcu_data *rdp) | 774 | struct rcu_data *rdp) |
529 | { | 775 | { |
776 | return 0; | ||
530 | } | 777 | } |
531 | 778 | ||
532 | /* | 779 | /* |
@@ -565,6 +812,30 @@ void call_rcu(struct rcu_head *head, void (*func)(struct rcu_head *rcu)) | |||
565 | EXPORT_SYMBOL_GPL(call_rcu); | 812 | EXPORT_SYMBOL_GPL(call_rcu); |
566 | 813 | ||
567 | /* | 814 | /* |
815 | * Wait for an rcu-preempt grace period, but make it happen quickly. | ||
816 | * But because preemptable RCU does not exist, map to rcu-sched. | ||
817 | */ | ||
818 | void synchronize_rcu_expedited(void) | ||
819 | { | ||
820 | synchronize_sched_expedited(); | ||
821 | } | ||
822 | EXPORT_SYMBOL_GPL(synchronize_rcu_expedited); | ||
823 | |||
824 | #ifdef CONFIG_HOTPLUG_CPU | ||
825 | |||
826 | /* | ||
827 | * Because preemptable RCU does not exist, there is never any need to | ||
828 | * report on tasks preempted in RCU read-side critical sections during | ||
829 | * expedited RCU grace periods. | ||
830 | */ | ||
831 | static void rcu_report_exp_rnp(struct rcu_state *rsp, struct rcu_node *rnp) | ||
832 | { | ||
833 | return; | ||
834 | } | ||
835 | |||
836 | #endif /* #ifdef CONFIG_HOTPLUG_CPU */ | ||
837 | |||
838 | /* | ||
568 | * Because preemptable RCU does not exist, it never has any work to do. | 839 | * Because preemptable RCU does not exist, it never has any work to do. |
569 | */ | 840 | */ |
570 | static int rcu_preempt_pending(int cpu) | 841 | static int rcu_preempt_pending(int cpu) |
diff --git a/kernel/rcutree_trace.c b/kernel/rcutree_trace.c index 4b31c779e62e..9d2c88423b31 100644 --- a/kernel/rcutree_trace.c +++ b/kernel/rcutree_trace.c | |||
@@ -155,12 +155,15 @@ static const struct file_operations rcudata_csv_fops = { | |||
155 | 155 | ||
156 | static void print_one_rcu_state(struct seq_file *m, struct rcu_state *rsp) | 156 | static void print_one_rcu_state(struct seq_file *m, struct rcu_state *rsp) |
157 | { | 157 | { |
158 | long gpnum; | ||
158 | int level = 0; | 159 | int level = 0; |
160 | int phase; | ||
159 | struct rcu_node *rnp; | 161 | struct rcu_node *rnp; |
160 | 162 | ||
163 | gpnum = rsp->gpnum; | ||
161 | seq_printf(m, "c=%ld g=%ld s=%d jfq=%ld j=%x " | 164 | seq_printf(m, "c=%ld g=%ld s=%d jfq=%ld j=%x " |
162 | "nfqs=%lu/nfqsng=%lu(%lu) fqlh=%lu oqlen=%ld\n", | 165 | "nfqs=%lu/nfqsng=%lu(%lu) fqlh=%lu oqlen=%ld\n", |
163 | rsp->completed, rsp->gpnum, rsp->signaled, | 166 | rsp->completed, gpnum, rsp->signaled, |
164 | (long)(rsp->jiffies_force_qs - jiffies), | 167 | (long)(rsp->jiffies_force_qs - jiffies), |
165 | (int)(jiffies & 0xffff), | 168 | (int)(jiffies & 0xffff), |
166 | rsp->n_force_qs, rsp->n_force_qs_ngp, | 169 | rsp->n_force_qs, rsp->n_force_qs_ngp, |
@@ -171,8 +174,13 @@ static void print_one_rcu_state(struct seq_file *m, struct rcu_state *rsp) | |||
171 | seq_puts(m, "\n"); | 174 | seq_puts(m, "\n"); |
172 | level = rnp->level; | 175 | level = rnp->level; |
173 | } | 176 | } |
174 | seq_printf(m, "%lx/%lx %d:%d ^%d ", | 177 | phase = gpnum & 0x1; |
178 | seq_printf(m, "%lx/%lx %c%c>%c%c %d:%d ^%d ", | ||
175 | rnp->qsmask, rnp->qsmaskinit, | 179 | rnp->qsmask, rnp->qsmaskinit, |
180 | "T."[list_empty(&rnp->blocked_tasks[phase])], | ||
181 | "E."[list_empty(&rnp->blocked_tasks[phase + 2])], | ||
182 | "T."[list_empty(&rnp->blocked_tasks[!phase])], | ||
183 | "E."[list_empty(&rnp->blocked_tasks[!phase + 2])], | ||
176 | rnp->grplo, rnp->grphi, rnp->grpnum); | 184 | rnp->grplo, rnp->grphi, rnp->grpnum); |
177 | } | 185 | } |
178 | seq_puts(m, "\n"); | 186 | seq_puts(m, "\n"); |
diff --git a/kernel/sched.c b/kernel/sched.c index e88689522e66..e7f2cfa6a257 100644 --- a/kernel/sched.c +++ b/kernel/sched.c | |||
@@ -309,6 +309,8 @@ static DEFINE_PER_CPU_SHARED_ALIGNED(struct rt_rq, init_rt_rq); | |||
309 | */ | 309 | */ |
310 | static DEFINE_SPINLOCK(task_group_lock); | 310 | static DEFINE_SPINLOCK(task_group_lock); |
311 | 311 | ||
312 | #ifdef CONFIG_FAIR_GROUP_SCHED | ||
313 | |||
312 | #ifdef CONFIG_SMP | 314 | #ifdef CONFIG_SMP |
313 | static int root_task_group_empty(void) | 315 | static int root_task_group_empty(void) |
314 | { | 316 | { |
@@ -316,7 +318,6 @@ static int root_task_group_empty(void) | |||
316 | } | 318 | } |
317 | #endif | 319 | #endif |
318 | 320 | ||
319 | #ifdef CONFIG_FAIR_GROUP_SCHED | ||
320 | #ifdef CONFIG_USER_SCHED | 321 | #ifdef CONFIG_USER_SCHED |
321 | # define INIT_TASK_GROUP_LOAD (2*NICE_0_LOAD) | 322 | # define INIT_TASK_GROUP_LOAD (2*NICE_0_LOAD) |
322 | #else /* !CONFIG_USER_SCHED */ | 323 | #else /* !CONFIG_USER_SCHED */ |
@@ -534,14 +535,12 @@ struct rq { | |||
534 | #define CPU_LOAD_IDX_MAX 5 | 535 | #define CPU_LOAD_IDX_MAX 5 |
535 | unsigned long cpu_load[CPU_LOAD_IDX_MAX]; | 536 | unsigned long cpu_load[CPU_LOAD_IDX_MAX]; |
536 | #ifdef CONFIG_NO_HZ | 537 | #ifdef CONFIG_NO_HZ |
537 | unsigned long last_tick_seen; | ||
538 | unsigned char in_nohz_recently; | 538 | unsigned char in_nohz_recently; |
539 | #endif | 539 | #endif |
540 | /* capture load from *all* tasks on this cpu: */ | 540 | /* capture load from *all* tasks on this cpu: */ |
541 | struct load_weight load; | 541 | struct load_weight load; |
542 | unsigned long nr_load_updates; | 542 | unsigned long nr_load_updates; |
543 | u64 nr_switches; | 543 | u64 nr_switches; |
544 | u64 nr_migrations_in; | ||
545 | 544 | ||
546 | struct cfs_rq cfs; | 545 | struct cfs_rq cfs; |
547 | struct rt_rq rt; | 546 | struct rt_rq rt; |
@@ -590,6 +589,8 @@ struct rq { | |||
590 | 589 | ||
591 | u64 rt_avg; | 590 | u64 rt_avg; |
592 | u64 age_stamp; | 591 | u64 age_stamp; |
592 | u64 idle_stamp; | ||
593 | u64 avg_idle; | ||
593 | #endif | 594 | #endif |
594 | 595 | ||
595 | /* calc_load related fields */ | 596 | /* calc_load related fields */ |
@@ -771,7 +772,7 @@ sched_feat_write(struct file *filp, const char __user *ubuf, | |||
771 | if (!sched_feat_names[i]) | 772 | if (!sched_feat_names[i]) |
772 | return -EINVAL; | 773 | return -EINVAL; |
773 | 774 | ||
774 | filp->f_pos += cnt; | 775 | *ppos += cnt; |
775 | 776 | ||
776 | return cnt; | 777 | return cnt; |
777 | } | 778 | } |
@@ -1564,11 +1565,7 @@ static unsigned long cpu_avg_load_per_task(int cpu) | |||
1564 | 1565 | ||
1565 | #ifdef CONFIG_FAIR_GROUP_SCHED | 1566 | #ifdef CONFIG_FAIR_GROUP_SCHED |
1566 | 1567 | ||
1567 | struct update_shares_data { | 1568 | static __read_mostly unsigned long *update_shares_data; |
1568 | unsigned long rq_weight[NR_CPUS]; | ||
1569 | }; | ||
1570 | |||
1571 | static DEFINE_PER_CPU(struct update_shares_data, update_shares_data); | ||
1572 | 1569 | ||
1573 | static void __set_se_shares(struct sched_entity *se, unsigned long shares); | 1570 | static void __set_se_shares(struct sched_entity *se, unsigned long shares); |
1574 | 1571 | ||
@@ -1578,12 +1575,12 @@ static void __set_se_shares(struct sched_entity *se, unsigned long shares); | |||
1578 | static void update_group_shares_cpu(struct task_group *tg, int cpu, | 1575 | static void update_group_shares_cpu(struct task_group *tg, int cpu, |
1579 | unsigned long sd_shares, | 1576 | unsigned long sd_shares, |
1580 | unsigned long sd_rq_weight, | 1577 | unsigned long sd_rq_weight, |
1581 | struct update_shares_data *usd) | 1578 | unsigned long *usd_rq_weight) |
1582 | { | 1579 | { |
1583 | unsigned long shares, rq_weight; | 1580 | unsigned long shares, rq_weight; |
1584 | int boost = 0; | 1581 | int boost = 0; |
1585 | 1582 | ||
1586 | rq_weight = usd->rq_weight[cpu]; | 1583 | rq_weight = usd_rq_weight[cpu]; |
1587 | if (!rq_weight) { | 1584 | if (!rq_weight) { |
1588 | boost = 1; | 1585 | boost = 1; |
1589 | rq_weight = NICE_0_LOAD; | 1586 | rq_weight = NICE_0_LOAD; |
@@ -1618,7 +1615,7 @@ static void update_group_shares_cpu(struct task_group *tg, int cpu, | |||
1618 | static int tg_shares_up(struct task_group *tg, void *data) | 1615 | static int tg_shares_up(struct task_group *tg, void *data) |
1619 | { | 1616 | { |
1620 | unsigned long weight, rq_weight = 0, shares = 0; | 1617 | unsigned long weight, rq_weight = 0, shares = 0; |
1621 | struct update_shares_data *usd; | 1618 | unsigned long *usd_rq_weight; |
1622 | struct sched_domain *sd = data; | 1619 | struct sched_domain *sd = data; |
1623 | unsigned long flags; | 1620 | unsigned long flags; |
1624 | int i; | 1621 | int i; |
@@ -1627,11 +1624,11 @@ static int tg_shares_up(struct task_group *tg, void *data) | |||
1627 | return 0; | 1624 | return 0; |
1628 | 1625 | ||
1629 | local_irq_save(flags); | 1626 | local_irq_save(flags); |
1630 | usd = &__get_cpu_var(update_shares_data); | 1627 | usd_rq_weight = per_cpu_ptr(update_shares_data, smp_processor_id()); |
1631 | 1628 | ||
1632 | for_each_cpu(i, sched_domain_span(sd)) { | 1629 | for_each_cpu(i, sched_domain_span(sd)) { |
1633 | weight = tg->cfs_rq[i]->load.weight; | 1630 | weight = tg->cfs_rq[i]->load.weight; |
1634 | usd->rq_weight[i] = weight; | 1631 | usd_rq_weight[i] = weight; |
1635 | 1632 | ||
1636 | /* | 1633 | /* |
1637 | * If there are currently no tasks on the cpu pretend there | 1634 | * If there are currently no tasks on the cpu pretend there |
@@ -1652,7 +1649,7 @@ static int tg_shares_up(struct task_group *tg, void *data) | |||
1652 | shares = tg->shares; | 1649 | shares = tg->shares; |
1653 | 1650 | ||
1654 | for_each_cpu(i, sched_domain_span(sd)) | 1651 | for_each_cpu(i, sched_domain_span(sd)) |
1655 | update_group_shares_cpu(tg, i, shares, rq_weight, usd); | 1652 | update_group_shares_cpu(tg, i, shares, rq_weight, usd_rq_weight); |
1656 | 1653 | ||
1657 | local_irq_restore(flags); | 1654 | local_irq_restore(flags); |
1658 | 1655 | ||
@@ -1996,6 +1993,39 @@ static inline void check_class_changed(struct rq *rq, struct task_struct *p, | |||
1996 | p->sched_class->prio_changed(rq, p, oldprio, running); | 1993 | p->sched_class->prio_changed(rq, p, oldprio, running); |
1997 | } | 1994 | } |
1998 | 1995 | ||
1996 | /** | ||
1997 | * kthread_bind - bind a just-created kthread to a cpu. | ||
1998 | * @p: thread created by kthread_create(). | ||
1999 | * @cpu: cpu (might not be online, must be possible) for @k to run on. | ||
2000 | * | ||
2001 | * Description: This function is equivalent to set_cpus_allowed(), | ||
2002 | * except that @cpu doesn't need to be online, and the thread must be | ||
2003 | * stopped (i.e., just returned from kthread_create()). | ||
2004 | * | ||
2005 | * Function lives here instead of kthread.c because it messes with | ||
2006 | * scheduler internals which require locking. | ||
2007 | */ | ||
2008 | void kthread_bind(struct task_struct *p, unsigned int cpu) | ||
2009 | { | ||
2010 | struct rq *rq = cpu_rq(cpu); | ||
2011 | unsigned long flags; | ||
2012 | |||
2013 | /* Must have done schedule() in kthread() before we set_task_cpu */ | ||
2014 | if (!wait_task_inactive(p, TASK_UNINTERRUPTIBLE)) { | ||
2015 | WARN_ON(1); | ||
2016 | return; | ||
2017 | } | ||
2018 | |||
2019 | spin_lock_irqsave(&rq->lock, flags); | ||
2020 | update_rq_clock(rq); | ||
2021 | set_task_cpu(p, cpu); | ||
2022 | p->cpus_allowed = cpumask_of_cpu(cpu); | ||
2023 | p->rt.nr_cpus_allowed = 1; | ||
2024 | p->flags |= PF_THREAD_BOUND; | ||
2025 | spin_unlock_irqrestore(&rq->lock, flags); | ||
2026 | } | ||
2027 | EXPORT_SYMBOL(kthread_bind); | ||
2028 | |||
1999 | #ifdef CONFIG_SMP | 2029 | #ifdef CONFIG_SMP |
2000 | /* | 2030 | /* |
2001 | * Is this task likely cache-hot: | 2031 | * Is this task likely cache-hot: |
@@ -2008,7 +2038,7 @@ task_hot(struct task_struct *p, u64 now, struct sched_domain *sd) | |||
2008 | /* | 2038 | /* |
2009 | * Buddy candidates are cache hot: | 2039 | * Buddy candidates are cache hot: |
2010 | */ | 2040 | */ |
2011 | if (sched_feat(CACHE_HOT_BUDDY) && | 2041 | if (sched_feat(CACHE_HOT_BUDDY) && this_rq()->nr_running && |
2012 | (&p->se == cfs_rq_of(&p->se)->next || | 2042 | (&p->se == cfs_rq_of(&p->se)->next || |
2013 | &p->se == cfs_rq_of(&p->se)->last)) | 2043 | &p->se == cfs_rq_of(&p->se)->last)) |
2014 | return 1; | 2044 | return 1; |
@@ -2049,7 +2079,6 @@ void set_task_cpu(struct task_struct *p, unsigned int new_cpu) | |||
2049 | #endif | 2079 | #endif |
2050 | if (old_cpu != new_cpu) { | 2080 | if (old_cpu != new_cpu) { |
2051 | p->se.nr_migrations++; | 2081 | p->se.nr_migrations++; |
2052 | new_rq->nr_migrations_in++; | ||
2053 | #ifdef CONFIG_SCHEDSTATS | 2082 | #ifdef CONFIG_SCHEDSTATS |
2054 | if (task_hot(p, old_rq->clock, NULL)) | 2083 | if (task_hot(p, old_rq->clock, NULL)) |
2055 | schedstat_inc(p, se.nr_forced2_migrations); | 2084 | schedstat_inc(p, se.nr_forced2_migrations); |
@@ -2086,6 +2115,7 @@ migrate_task(struct task_struct *p, int dest_cpu, struct migration_req *req) | |||
2086 | * it is sufficient to simply update the task's cpu field. | 2115 | * it is sufficient to simply update the task's cpu field. |
2087 | */ | 2116 | */ |
2088 | if (!p->se.on_rq && !task_running(rq, p)) { | 2117 | if (!p->se.on_rq && !task_running(rq, p)) { |
2118 | update_rq_clock(rq); | ||
2089 | set_task_cpu(p, dest_cpu); | 2119 | set_task_cpu(p, dest_cpu); |
2090 | return 0; | 2120 | return 0; |
2091 | } | 2121 | } |
@@ -2347,14 +2377,15 @@ static int try_to_wake_up(struct task_struct *p, unsigned int state, | |||
2347 | task_rq_unlock(rq, &flags); | 2377 | task_rq_unlock(rq, &flags); |
2348 | 2378 | ||
2349 | cpu = p->sched_class->select_task_rq(p, SD_BALANCE_WAKE, wake_flags); | 2379 | cpu = p->sched_class->select_task_rq(p, SD_BALANCE_WAKE, wake_flags); |
2350 | if (cpu != orig_cpu) | 2380 | if (cpu != orig_cpu) { |
2381 | local_irq_save(flags); | ||
2382 | rq = cpu_rq(cpu); | ||
2383 | update_rq_clock(rq); | ||
2351 | set_task_cpu(p, cpu); | 2384 | set_task_cpu(p, cpu); |
2352 | 2385 | local_irq_restore(flags); | |
2386 | } | ||
2353 | rq = task_rq_lock(p, &flags); | 2387 | rq = task_rq_lock(p, &flags); |
2354 | 2388 | ||
2355 | if (rq != orig_rq) | ||
2356 | update_rq_clock(rq); | ||
2357 | |||
2358 | WARN_ON(p->state != TASK_WAKING); | 2389 | WARN_ON(p->state != TASK_WAKING); |
2359 | cpu = task_cpu(p); | 2390 | cpu = task_cpu(p); |
2360 | 2391 | ||
@@ -2411,6 +2442,17 @@ out_running: | |||
2411 | #ifdef CONFIG_SMP | 2442 | #ifdef CONFIG_SMP |
2412 | if (p->sched_class->task_wake_up) | 2443 | if (p->sched_class->task_wake_up) |
2413 | p->sched_class->task_wake_up(rq, p); | 2444 | p->sched_class->task_wake_up(rq, p); |
2445 | |||
2446 | if (unlikely(rq->idle_stamp)) { | ||
2447 | u64 delta = rq->clock - rq->idle_stamp; | ||
2448 | u64 max = 2*sysctl_sched_migration_cost; | ||
2449 | |||
2450 | if (delta > max) | ||
2451 | rq->avg_idle = max; | ||
2452 | else | ||
2453 | update_avg(&rq->avg_idle, delta); | ||
2454 | rq->idle_stamp = 0; | ||
2455 | } | ||
2414 | #endif | 2456 | #endif |
2415 | out: | 2457 | out: |
2416 | task_rq_unlock(rq, &flags); | 2458 | task_rq_unlock(rq, &flags); |
@@ -2516,6 +2558,7 @@ static void __sched_fork(struct task_struct *p) | |||
2516 | void sched_fork(struct task_struct *p, int clone_flags) | 2558 | void sched_fork(struct task_struct *p, int clone_flags) |
2517 | { | 2559 | { |
2518 | int cpu = get_cpu(); | 2560 | int cpu = get_cpu(); |
2561 | unsigned long flags; | ||
2519 | 2562 | ||
2520 | __sched_fork(p); | 2563 | __sched_fork(p); |
2521 | 2564 | ||
@@ -2552,7 +2595,10 @@ void sched_fork(struct task_struct *p, int clone_flags) | |||
2552 | #ifdef CONFIG_SMP | 2595 | #ifdef CONFIG_SMP |
2553 | cpu = p->sched_class->select_task_rq(p, SD_BALANCE_FORK, 0); | 2596 | cpu = p->sched_class->select_task_rq(p, SD_BALANCE_FORK, 0); |
2554 | #endif | 2597 | #endif |
2598 | local_irq_save(flags); | ||
2599 | update_rq_clock(cpu_rq(cpu)); | ||
2555 | set_task_cpu(p, cpu); | 2600 | set_task_cpu(p, cpu); |
2601 | local_irq_restore(flags); | ||
2556 | 2602 | ||
2557 | #if defined(CONFIG_SCHEDSTATS) || defined(CONFIG_TASK_DELAY_ACCT) | 2603 | #if defined(CONFIG_SCHEDSTATS) || defined(CONFIG_TASK_DELAY_ACCT) |
2558 | if (likely(sched_info_on())) | 2604 | if (likely(sched_info_on())) |
@@ -2819,14 +2865,14 @@ context_switch(struct rq *rq, struct task_struct *prev, | |||
2819 | */ | 2865 | */ |
2820 | arch_start_context_switch(prev); | 2866 | arch_start_context_switch(prev); |
2821 | 2867 | ||
2822 | if (unlikely(!mm)) { | 2868 | if (likely(!mm)) { |
2823 | next->active_mm = oldmm; | 2869 | next->active_mm = oldmm; |
2824 | atomic_inc(&oldmm->mm_count); | 2870 | atomic_inc(&oldmm->mm_count); |
2825 | enter_lazy_tlb(oldmm, next); | 2871 | enter_lazy_tlb(oldmm, next); |
2826 | } else | 2872 | } else |
2827 | switch_mm(oldmm, mm, next); | 2873 | switch_mm(oldmm, mm, next); |
2828 | 2874 | ||
2829 | if (unlikely(!prev->mm)) { | 2875 | if (likely(!prev->mm)) { |
2830 | prev->active_mm = NULL; | 2876 | prev->active_mm = NULL; |
2831 | rq->prev_mm = oldmm; | 2877 | rq->prev_mm = oldmm; |
2832 | } | 2878 | } |
@@ -2989,15 +3035,6 @@ static void calc_load_account_active(struct rq *this_rq) | |||
2989 | } | 3035 | } |
2990 | 3036 | ||
2991 | /* | 3037 | /* |
2992 | * Externally visible per-cpu scheduler statistics: | ||
2993 | * cpu_nr_migrations(cpu) - number of migrations into that cpu | ||
2994 | */ | ||
2995 | u64 cpu_nr_migrations(int cpu) | ||
2996 | { | ||
2997 | return cpu_rq(cpu)->nr_migrations_in; | ||
2998 | } | ||
2999 | |||
3000 | /* | ||
3001 | * Update rq->cpu_load[] statistics. This function is usually called every | 3038 | * Update rq->cpu_load[] statistics. This function is usually called every |
3002 | * scheduler tick (TICK_NSEC). | 3039 | * scheduler tick (TICK_NSEC). |
3003 | */ | 3040 | */ |
@@ -4097,7 +4134,7 @@ static int load_balance(int this_cpu, struct rq *this_rq, | |||
4097 | unsigned long flags; | 4134 | unsigned long flags; |
4098 | struct cpumask *cpus = __get_cpu_var(load_balance_tmpmask); | 4135 | struct cpumask *cpus = __get_cpu_var(load_balance_tmpmask); |
4099 | 4136 | ||
4100 | cpumask_setall(cpus); | 4137 | cpumask_copy(cpus, cpu_online_mask); |
4101 | 4138 | ||
4102 | /* | 4139 | /* |
4103 | * When power savings policy is enabled for the parent domain, idle | 4140 | * When power savings policy is enabled for the parent domain, idle |
@@ -4260,7 +4297,7 @@ load_balance_newidle(int this_cpu, struct rq *this_rq, struct sched_domain *sd) | |||
4260 | int all_pinned = 0; | 4297 | int all_pinned = 0; |
4261 | struct cpumask *cpus = __get_cpu_var(load_balance_tmpmask); | 4298 | struct cpumask *cpus = __get_cpu_var(load_balance_tmpmask); |
4262 | 4299 | ||
4263 | cpumask_setall(cpus); | 4300 | cpumask_copy(cpus, cpu_online_mask); |
4264 | 4301 | ||
4265 | /* | 4302 | /* |
4266 | * When power savings policy is enabled for the parent domain, idle | 4303 | * When power savings policy is enabled for the parent domain, idle |
@@ -4400,6 +4437,11 @@ static void idle_balance(int this_cpu, struct rq *this_rq) | |||
4400 | int pulled_task = 0; | 4437 | int pulled_task = 0; |
4401 | unsigned long next_balance = jiffies + HZ; | 4438 | unsigned long next_balance = jiffies + HZ; |
4402 | 4439 | ||
4440 | this_rq->idle_stamp = this_rq->clock; | ||
4441 | |||
4442 | if (this_rq->avg_idle < sysctl_sched_migration_cost) | ||
4443 | return; | ||
4444 | |||
4403 | for_each_domain(this_cpu, sd) { | 4445 | for_each_domain(this_cpu, sd) { |
4404 | unsigned long interval; | 4446 | unsigned long interval; |
4405 | 4447 | ||
@@ -4414,8 +4456,10 @@ static void idle_balance(int this_cpu, struct rq *this_rq) | |||
4414 | interval = msecs_to_jiffies(sd->balance_interval); | 4456 | interval = msecs_to_jiffies(sd->balance_interval); |
4415 | if (time_after(next_balance, sd->last_balance + interval)) | 4457 | if (time_after(next_balance, sd->last_balance + interval)) |
4416 | next_balance = sd->last_balance + interval; | 4458 | next_balance = sd->last_balance + interval; |
4417 | if (pulled_task) | 4459 | if (pulled_task) { |
4460 | this_rq->idle_stamp = 0; | ||
4418 | break; | 4461 | break; |
4462 | } | ||
4419 | } | 4463 | } |
4420 | if (pulled_task || time_after(jiffies, this_rq->next_balance)) { | 4464 | if (pulled_task || time_after(jiffies, this_rq->next_balance)) { |
4421 | /* | 4465 | /* |
@@ -5017,8 +5061,13 @@ static void account_guest_time(struct task_struct *p, cputime_t cputime, | |||
5017 | p->gtime = cputime_add(p->gtime, cputime); | 5061 | p->gtime = cputime_add(p->gtime, cputime); |
5018 | 5062 | ||
5019 | /* Add guest time to cpustat. */ | 5063 | /* Add guest time to cpustat. */ |
5020 | cpustat->user = cputime64_add(cpustat->user, tmp); | 5064 | if (TASK_NICE(p) > 0) { |
5021 | cpustat->guest = cputime64_add(cpustat->guest, tmp); | 5065 | cpustat->nice = cputime64_add(cpustat->nice, tmp); |
5066 | cpustat->guest_nice = cputime64_add(cpustat->guest_nice, tmp); | ||
5067 | } else { | ||
5068 | cpustat->user = cputime64_add(cpustat->user, tmp); | ||
5069 | cpustat->guest = cputime64_add(cpustat->guest, tmp); | ||
5070 | } | ||
5022 | } | 5071 | } |
5023 | 5072 | ||
5024 | /* | 5073 | /* |
@@ -5133,60 +5182,86 @@ void account_idle_ticks(unsigned long ticks) | |||
5133 | * Use precise platform statistics if available: | 5182 | * Use precise platform statistics if available: |
5134 | */ | 5183 | */ |
5135 | #ifdef CONFIG_VIRT_CPU_ACCOUNTING | 5184 | #ifdef CONFIG_VIRT_CPU_ACCOUNTING |
5136 | cputime_t task_utime(struct task_struct *p) | 5185 | void task_times(struct task_struct *p, cputime_t *ut, cputime_t *st) |
5137 | { | 5186 | { |
5138 | return p->utime; | 5187 | *ut = p->utime; |
5188 | *st = p->stime; | ||
5139 | } | 5189 | } |
5140 | 5190 | ||
5141 | cputime_t task_stime(struct task_struct *p) | 5191 | void thread_group_times(struct task_struct *p, cputime_t *ut, cputime_t *st) |
5142 | { | 5192 | { |
5143 | return p->stime; | 5193 | struct task_cputime cputime; |
5194 | |||
5195 | thread_group_cputime(p, &cputime); | ||
5196 | |||
5197 | *ut = cputime.utime; | ||
5198 | *st = cputime.stime; | ||
5144 | } | 5199 | } |
5145 | #else | 5200 | #else |
5146 | cputime_t task_utime(struct task_struct *p) | 5201 | |
5202 | #ifndef nsecs_to_cputime | ||
5203 | # define nsecs_to_cputime(__nsecs) nsecs_to_jiffies(__nsecs) | ||
5204 | #endif | ||
5205 | |||
5206 | void task_times(struct task_struct *p, cputime_t *ut, cputime_t *st) | ||
5147 | { | 5207 | { |
5148 | clock_t utime = cputime_to_clock_t(p->utime), | 5208 | cputime_t rtime, utime = p->utime, total = cputime_add(utime, p->stime); |
5149 | total = utime + cputime_to_clock_t(p->stime); | ||
5150 | u64 temp; | ||
5151 | 5209 | ||
5152 | /* | 5210 | /* |
5153 | * Use CFS's precise accounting: | 5211 | * Use CFS's precise accounting: |
5154 | */ | 5212 | */ |
5155 | temp = (u64)nsec_to_clock_t(p->se.sum_exec_runtime); | 5213 | rtime = nsecs_to_cputime(p->se.sum_exec_runtime); |
5156 | 5214 | ||
5157 | if (total) { | 5215 | if (total) { |
5158 | temp *= utime; | 5216 | u64 temp; |
5217 | |||
5218 | temp = (u64)(rtime * utime); | ||
5159 | do_div(temp, total); | 5219 | do_div(temp, total); |
5160 | } | 5220 | utime = (cputime_t)temp; |
5161 | utime = (clock_t)temp; | 5221 | } else |
5222 | utime = rtime; | ||
5223 | |||
5224 | /* | ||
5225 | * Compare with previous values, to keep monotonicity: | ||
5226 | */ | ||
5227 | p->prev_utime = max(p->prev_utime, utime); | ||
5228 | p->prev_stime = max(p->prev_stime, cputime_sub(rtime, p->prev_utime)); | ||
5162 | 5229 | ||
5163 | p->prev_utime = max(p->prev_utime, clock_t_to_cputime(utime)); | 5230 | *ut = p->prev_utime; |
5164 | return p->prev_utime; | 5231 | *st = p->prev_stime; |
5165 | } | 5232 | } |
5166 | 5233 | ||
5167 | cputime_t task_stime(struct task_struct *p) | 5234 | /* |
5235 | * Must be called with siglock held. | ||
5236 | */ | ||
5237 | void thread_group_times(struct task_struct *p, cputime_t *ut, cputime_t *st) | ||
5168 | { | 5238 | { |
5169 | clock_t stime; | 5239 | struct signal_struct *sig = p->signal; |
5240 | struct task_cputime cputime; | ||
5241 | cputime_t rtime, utime, total; | ||
5170 | 5242 | ||
5171 | /* | 5243 | thread_group_cputime(p, &cputime); |
5172 | * Use CFS's precise accounting. (we subtract utime from | ||
5173 | * the total, to make sure the total observed by userspace | ||
5174 | * grows monotonically - apps rely on that): | ||
5175 | */ | ||
5176 | stime = nsec_to_clock_t(p->se.sum_exec_runtime) - | ||
5177 | cputime_to_clock_t(task_utime(p)); | ||
5178 | 5244 | ||
5179 | if (stime >= 0) | 5245 | total = cputime_add(cputime.utime, cputime.stime); |
5180 | p->prev_stime = max(p->prev_stime, clock_t_to_cputime(stime)); | 5246 | rtime = nsecs_to_cputime(cputime.sum_exec_runtime); |
5181 | 5247 | ||
5182 | return p->prev_stime; | 5248 | if (total) { |
5183 | } | 5249 | u64 temp; |
5184 | #endif | ||
5185 | 5250 | ||
5186 | inline cputime_t task_gtime(struct task_struct *p) | 5251 | temp = (u64)(rtime * cputime.utime); |
5187 | { | 5252 | do_div(temp, total); |
5188 | return p->gtime; | 5253 | utime = (cputime_t)temp; |
5254 | } else | ||
5255 | utime = rtime; | ||
5256 | |||
5257 | sig->prev_utime = max(sig->prev_utime, utime); | ||
5258 | sig->prev_stime = max(sig->prev_stime, | ||
5259 | cputime_sub(rtime, sig->prev_utime)); | ||
5260 | |||
5261 | *ut = sig->prev_utime; | ||
5262 | *st = sig->prev_stime; | ||
5189 | } | 5263 | } |
5264 | #endif | ||
5190 | 5265 | ||
5191 | /* | 5266 | /* |
5192 | * This function gets called by the timer code, with HZ frequency. | 5267 | * This function gets called by the timer code, with HZ frequency. |
@@ -5452,7 +5527,7 @@ need_resched_nonpreemptible: | |||
5452 | } | 5527 | } |
5453 | EXPORT_SYMBOL(schedule); | 5528 | EXPORT_SYMBOL(schedule); |
5454 | 5529 | ||
5455 | #ifdef CONFIG_SMP | 5530 | #ifdef CONFIG_MUTEX_SPIN_ON_OWNER |
5456 | /* | 5531 | /* |
5457 | * Look out! "owner" is an entirely speculative pointer | 5532 | * Look out! "owner" is an entirely speculative pointer |
5458 | * access and not reliable. | 5533 | * access and not reliable. |
@@ -6146,22 +6221,14 @@ __setscheduler(struct rq *rq, struct task_struct *p, int policy, int prio) | |||
6146 | BUG_ON(p->se.on_rq); | 6221 | BUG_ON(p->se.on_rq); |
6147 | 6222 | ||
6148 | p->policy = policy; | 6223 | p->policy = policy; |
6149 | switch (p->policy) { | ||
6150 | case SCHED_NORMAL: | ||
6151 | case SCHED_BATCH: | ||
6152 | case SCHED_IDLE: | ||
6153 | p->sched_class = &fair_sched_class; | ||
6154 | break; | ||
6155 | case SCHED_FIFO: | ||
6156 | case SCHED_RR: | ||
6157 | p->sched_class = &rt_sched_class; | ||
6158 | break; | ||
6159 | } | ||
6160 | |||
6161 | p->rt_priority = prio; | 6224 | p->rt_priority = prio; |
6162 | p->normal_prio = normal_prio(p); | 6225 | p->normal_prio = normal_prio(p); |
6163 | /* we are holding p->pi_lock already */ | 6226 | /* we are holding p->pi_lock already */ |
6164 | p->prio = rt_mutex_getprio(p); | 6227 | p->prio = rt_mutex_getprio(p); |
6228 | if (rt_prio(p->prio)) | ||
6229 | p->sched_class = &rt_sched_class; | ||
6230 | else | ||
6231 | p->sched_class = &fair_sched_class; | ||
6165 | set_load_weight(p); | 6232 | set_load_weight(p); |
6166 | } | 6233 | } |
6167 | 6234 | ||
@@ -6906,7 +6973,7 @@ void show_state_filter(unsigned long state_filter) | |||
6906 | /* | 6973 | /* |
6907 | * Only show locks if all tasks are dumped: | 6974 | * Only show locks if all tasks are dumped: |
6908 | */ | 6975 | */ |
6909 | if (state_filter == -1) | 6976 | if (!state_filter) |
6910 | debug_show_all_locks(); | 6977 | debug_show_all_locks(); |
6911 | } | 6978 | } |
6912 | 6979 | ||
@@ -7377,17 +7444,16 @@ static struct ctl_table sd_ctl_dir[] = { | |||
7377 | .procname = "sched_domain", | 7444 | .procname = "sched_domain", |
7378 | .mode = 0555, | 7445 | .mode = 0555, |
7379 | }, | 7446 | }, |
7380 | {0, }, | 7447 | {} |
7381 | }; | 7448 | }; |
7382 | 7449 | ||
7383 | static struct ctl_table sd_ctl_root[] = { | 7450 | static struct ctl_table sd_ctl_root[] = { |
7384 | { | 7451 | { |
7385 | .ctl_name = CTL_KERN, | ||
7386 | .procname = "kernel", | 7452 | .procname = "kernel", |
7387 | .mode = 0555, | 7453 | .mode = 0555, |
7388 | .child = sd_ctl_dir, | 7454 | .child = sd_ctl_dir, |
7389 | }, | 7455 | }, |
7390 | {0, }, | 7456 | {} |
7391 | }; | 7457 | }; |
7392 | 7458 | ||
7393 | static struct ctl_table *sd_alloc_ctl_entry(int n) | 7459 | static struct ctl_table *sd_alloc_ctl_entry(int n) |
@@ -7711,6 +7777,16 @@ early_initcall(migration_init); | |||
7711 | 7777 | ||
7712 | #ifdef CONFIG_SCHED_DEBUG | 7778 | #ifdef CONFIG_SCHED_DEBUG |
7713 | 7779 | ||
7780 | static __read_mostly int sched_domain_debug_enabled; | ||
7781 | |||
7782 | static int __init sched_domain_debug_setup(char *str) | ||
7783 | { | ||
7784 | sched_domain_debug_enabled = 1; | ||
7785 | |||
7786 | return 0; | ||
7787 | } | ||
7788 | early_param("sched_debug", sched_domain_debug_setup); | ||
7789 | |||
7714 | static int sched_domain_debug_one(struct sched_domain *sd, int cpu, int level, | 7790 | static int sched_domain_debug_one(struct sched_domain *sd, int cpu, int level, |
7715 | struct cpumask *groupmask) | 7791 | struct cpumask *groupmask) |
7716 | { | 7792 | { |
@@ -7797,6 +7873,9 @@ static void sched_domain_debug(struct sched_domain *sd, int cpu) | |||
7797 | cpumask_var_t groupmask; | 7873 | cpumask_var_t groupmask; |
7798 | int level = 0; | 7874 | int level = 0; |
7799 | 7875 | ||
7876 | if (!sched_domain_debug_enabled) | ||
7877 | return; | ||
7878 | |||
7800 | if (!sd) { | 7879 | if (!sd) { |
7801 | printk(KERN_DEBUG "CPU%d attaching NULL sched-domain.\n", cpu); | 7880 | printk(KERN_DEBUG "CPU%d attaching NULL sched-domain.\n", cpu); |
7802 | return; | 7881 | return; |
@@ -7876,6 +7955,8 @@ sd_parent_degenerate(struct sched_domain *sd, struct sched_domain *parent) | |||
7876 | 7955 | ||
7877 | static void free_rootdomain(struct root_domain *rd) | 7956 | static void free_rootdomain(struct root_domain *rd) |
7878 | { | 7957 | { |
7958 | synchronize_sched(); | ||
7959 | |||
7879 | cpupri_cleanup(&rd->cpupri); | 7960 | cpupri_cleanup(&rd->cpupri); |
7880 | 7961 | ||
7881 | free_cpumask_var(rd->rto_mask); | 7962 | free_cpumask_var(rd->rto_mask); |
@@ -8016,6 +8097,7 @@ static cpumask_var_t cpu_isolated_map; | |||
8016 | /* Setup the mask of cpus configured for isolated domains */ | 8097 | /* Setup the mask of cpus configured for isolated domains */ |
8017 | static int __init isolated_cpu_setup(char *str) | 8098 | static int __init isolated_cpu_setup(char *str) |
8018 | { | 8099 | { |
8100 | alloc_bootmem_cpumask_var(&cpu_isolated_map); | ||
8019 | cpulist_parse(str, cpu_isolated_map); | 8101 | cpulist_parse(str, cpu_isolated_map); |
8020 | return 1; | 8102 | return 1; |
8021 | } | 8103 | } |
@@ -8852,7 +8934,7 @@ static int build_sched_domains(const struct cpumask *cpu_map) | |||
8852 | return __build_sched_domains(cpu_map, NULL); | 8934 | return __build_sched_domains(cpu_map, NULL); |
8853 | } | 8935 | } |
8854 | 8936 | ||
8855 | static struct cpumask *doms_cur; /* current sched domains */ | 8937 | static cpumask_var_t *doms_cur; /* current sched domains */ |
8856 | static int ndoms_cur; /* number of sched domains in 'doms_cur' */ | 8938 | static int ndoms_cur; /* number of sched domains in 'doms_cur' */ |
8857 | static struct sched_domain_attr *dattr_cur; | 8939 | static struct sched_domain_attr *dattr_cur; |
8858 | /* attribues of custom domains in 'doms_cur' */ | 8940 | /* attribues of custom domains in 'doms_cur' */ |
@@ -8874,6 +8956,31 @@ int __attribute__((weak)) arch_update_cpu_topology(void) | |||
8874 | return 0; | 8956 | return 0; |
8875 | } | 8957 | } |
8876 | 8958 | ||
8959 | cpumask_var_t *alloc_sched_domains(unsigned int ndoms) | ||
8960 | { | ||
8961 | int i; | ||
8962 | cpumask_var_t *doms; | ||
8963 | |||
8964 | doms = kmalloc(sizeof(*doms) * ndoms, GFP_KERNEL); | ||
8965 | if (!doms) | ||
8966 | return NULL; | ||
8967 | for (i = 0; i < ndoms; i++) { | ||
8968 | if (!alloc_cpumask_var(&doms[i], GFP_KERNEL)) { | ||
8969 | free_sched_domains(doms, i); | ||
8970 | return NULL; | ||
8971 | } | ||
8972 | } | ||
8973 | return doms; | ||
8974 | } | ||
8975 | |||
8976 | void free_sched_domains(cpumask_var_t doms[], unsigned int ndoms) | ||
8977 | { | ||
8978 | unsigned int i; | ||
8979 | for (i = 0; i < ndoms; i++) | ||
8980 | free_cpumask_var(doms[i]); | ||
8981 | kfree(doms); | ||
8982 | } | ||
8983 | |||
8877 | /* | 8984 | /* |
8878 | * Set up scheduler domains and groups. Callers must hold the hotplug lock. | 8985 | * Set up scheduler domains and groups. Callers must hold the hotplug lock. |
8879 | * For now this just excludes isolated cpus, but could be used to | 8986 | * For now this just excludes isolated cpus, but could be used to |
@@ -8885,12 +8992,12 @@ static int arch_init_sched_domains(const struct cpumask *cpu_map) | |||
8885 | 8992 | ||
8886 | arch_update_cpu_topology(); | 8993 | arch_update_cpu_topology(); |
8887 | ndoms_cur = 1; | 8994 | ndoms_cur = 1; |
8888 | doms_cur = kmalloc(cpumask_size(), GFP_KERNEL); | 8995 | doms_cur = alloc_sched_domains(ndoms_cur); |
8889 | if (!doms_cur) | 8996 | if (!doms_cur) |
8890 | doms_cur = fallback_doms; | 8997 | doms_cur = &fallback_doms; |
8891 | cpumask_andnot(doms_cur, cpu_map, cpu_isolated_map); | 8998 | cpumask_andnot(doms_cur[0], cpu_map, cpu_isolated_map); |
8892 | dattr_cur = NULL; | 8999 | dattr_cur = NULL; |
8893 | err = build_sched_domains(doms_cur); | 9000 | err = build_sched_domains(doms_cur[0]); |
8894 | register_sched_domain_sysctl(); | 9001 | register_sched_domain_sysctl(); |
8895 | 9002 | ||
8896 | return err; | 9003 | return err; |
@@ -8940,19 +9047,19 @@ static int dattrs_equal(struct sched_domain_attr *cur, int idx_cur, | |||
8940 | * doms_new[] to the current sched domain partitioning, doms_cur[]. | 9047 | * doms_new[] to the current sched domain partitioning, doms_cur[]. |
8941 | * It destroys each deleted domain and builds each new domain. | 9048 | * It destroys each deleted domain and builds each new domain. |
8942 | * | 9049 | * |
8943 | * 'doms_new' is an array of cpumask's of length 'ndoms_new'. | 9050 | * 'doms_new' is an array of cpumask_var_t's of length 'ndoms_new'. |
8944 | * The masks don't intersect (don't overlap.) We should setup one | 9051 | * The masks don't intersect (don't overlap.) We should setup one |
8945 | * sched domain for each mask. CPUs not in any of the cpumasks will | 9052 | * sched domain for each mask. CPUs not in any of the cpumasks will |
8946 | * not be load balanced. If the same cpumask appears both in the | 9053 | * not be load balanced. If the same cpumask appears both in the |
8947 | * current 'doms_cur' domains and in the new 'doms_new', we can leave | 9054 | * current 'doms_cur' domains and in the new 'doms_new', we can leave |
8948 | * it as it is. | 9055 | * it as it is. |
8949 | * | 9056 | * |
8950 | * The passed in 'doms_new' should be kmalloc'd. This routine takes | 9057 | * The passed in 'doms_new' should be allocated using |
8951 | * ownership of it and will kfree it when done with it. If the caller | 9058 | * alloc_sched_domains. This routine takes ownership of it and will |
8952 | * failed the kmalloc call, then it can pass in doms_new == NULL && | 9059 | * free_sched_domains it when done with it. If the caller failed the |
8953 | * ndoms_new == 1, and partition_sched_domains() will fallback to | 9060 | * alloc call, then it can pass in doms_new == NULL && ndoms_new == 1, |
8954 | * the single partition 'fallback_doms', it also forces the domains | 9061 | * and partition_sched_domains() will fallback to the single partition |
8955 | * to be rebuilt. | 9062 | * 'fallback_doms', it also forces the domains to be rebuilt. |
8956 | * | 9063 | * |
8957 | * If doms_new == NULL it will be replaced with cpu_online_mask. | 9064 | * If doms_new == NULL it will be replaced with cpu_online_mask. |
8958 | * ndoms_new == 0 is a special case for destroying existing domains, | 9065 | * ndoms_new == 0 is a special case for destroying existing domains, |
@@ -8960,8 +9067,7 @@ static int dattrs_equal(struct sched_domain_attr *cur, int idx_cur, | |||
8960 | * | 9067 | * |
8961 | * Call with hotplug lock held | 9068 | * Call with hotplug lock held |
8962 | */ | 9069 | */ |
8963 | /* FIXME: Change to struct cpumask *doms_new[] */ | 9070 | void partition_sched_domains(int ndoms_new, cpumask_var_t doms_new[], |
8964 | void partition_sched_domains(int ndoms_new, struct cpumask *doms_new, | ||
8965 | struct sched_domain_attr *dattr_new) | 9071 | struct sched_domain_attr *dattr_new) |
8966 | { | 9072 | { |
8967 | int i, j, n; | 9073 | int i, j, n; |
@@ -8980,40 +9086,40 @@ void partition_sched_domains(int ndoms_new, struct cpumask *doms_new, | |||
8980 | /* Destroy deleted domains */ | 9086 | /* Destroy deleted domains */ |
8981 | for (i = 0; i < ndoms_cur; i++) { | 9087 | for (i = 0; i < ndoms_cur; i++) { |
8982 | for (j = 0; j < n && !new_topology; j++) { | 9088 | for (j = 0; j < n && !new_topology; j++) { |
8983 | if (cpumask_equal(&doms_cur[i], &doms_new[j]) | 9089 | if (cpumask_equal(doms_cur[i], doms_new[j]) |
8984 | && dattrs_equal(dattr_cur, i, dattr_new, j)) | 9090 | && dattrs_equal(dattr_cur, i, dattr_new, j)) |
8985 | goto match1; | 9091 | goto match1; |
8986 | } | 9092 | } |
8987 | /* no match - a current sched domain not in new doms_new[] */ | 9093 | /* no match - a current sched domain not in new doms_new[] */ |
8988 | detach_destroy_domains(doms_cur + i); | 9094 | detach_destroy_domains(doms_cur[i]); |
8989 | match1: | 9095 | match1: |
8990 | ; | 9096 | ; |
8991 | } | 9097 | } |
8992 | 9098 | ||
8993 | if (doms_new == NULL) { | 9099 | if (doms_new == NULL) { |
8994 | ndoms_cur = 0; | 9100 | ndoms_cur = 0; |
8995 | doms_new = fallback_doms; | 9101 | doms_new = &fallback_doms; |
8996 | cpumask_andnot(&doms_new[0], cpu_online_mask, cpu_isolated_map); | 9102 | cpumask_andnot(doms_new[0], cpu_online_mask, cpu_isolated_map); |
8997 | WARN_ON_ONCE(dattr_new); | 9103 | WARN_ON_ONCE(dattr_new); |
8998 | } | 9104 | } |
8999 | 9105 | ||
9000 | /* Build new domains */ | 9106 | /* Build new domains */ |
9001 | for (i = 0; i < ndoms_new; i++) { | 9107 | for (i = 0; i < ndoms_new; i++) { |
9002 | for (j = 0; j < ndoms_cur && !new_topology; j++) { | 9108 | for (j = 0; j < ndoms_cur && !new_topology; j++) { |
9003 | if (cpumask_equal(&doms_new[i], &doms_cur[j]) | 9109 | if (cpumask_equal(doms_new[i], doms_cur[j]) |
9004 | && dattrs_equal(dattr_new, i, dattr_cur, j)) | 9110 | && dattrs_equal(dattr_new, i, dattr_cur, j)) |
9005 | goto match2; | 9111 | goto match2; |
9006 | } | 9112 | } |
9007 | /* no match - add a new doms_new */ | 9113 | /* no match - add a new doms_new */ |
9008 | __build_sched_domains(doms_new + i, | 9114 | __build_sched_domains(doms_new[i], |
9009 | dattr_new ? dattr_new + i : NULL); | 9115 | dattr_new ? dattr_new + i : NULL); |
9010 | match2: | 9116 | match2: |
9011 | ; | 9117 | ; |
9012 | } | 9118 | } |
9013 | 9119 | ||
9014 | /* Remember the new sched domains */ | 9120 | /* Remember the new sched domains */ |
9015 | if (doms_cur != fallback_doms) | 9121 | if (doms_cur != &fallback_doms) |
9016 | kfree(doms_cur); | 9122 | free_sched_domains(doms_cur, ndoms_cur); |
9017 | kfree(dattr_cur); /* kfree(NULL) is safe */ | 9123 | kfree(dattr_cur); /* kfree(NULL) is safe */ |
9018 | doms_cur = doms_new; | 9124 | doms_cur = doms_new; |
9019 | dattr_cur = dattr_new; | 9125 | dattr_cur = dattr_new; |
@@ -9335,10 +9441,6 @@ void __init sched_init(void) | |||
9335 | #ifdef CONFIG_CPUMASK_OFFSTACK | 9441 | #ifdef CONFIG_CPUMASK_OFFSTACK |
9336 | alloc_size += num_possible_cpus() * cpumask_size(); | 9442 | alloc_size += num_possible_cpus() * cpumask_size(); |
9337 | #endif | 9443 | #endif |
9338 | /* | ||
9339 | * As sched_init() is called before page_alloc is setup, | ||
9340 | * we use alloc_bootmem(). | ||
9341 | */ | ||
9342 | if (alloc_size) { | 9444 | if (alloc_size) { |
9343 | ptr = (unsigned long)kzalloc(alloc_size, GFP_NOWAIT); | 9445 | ptr = (unsigned long)kzalloc(alloc_size, GFP_NOWAIT); |
9344 | 9446 | ||
@@ -9407,6 +9509,10 @@ void __init sched_init(void) | |||
9407 | #endif /* CONFIG_USER_SCHED */ | 9509 | #endif /* CONFIG_USER_SCHED */ |
9408 | #endif /* CONFIG_GROUP_SCHED */ | 9510 | #endif /* CONFIG_GROUP_SCHED */ |
9409 | 9511 | ||
9512 | #if defined CONFIG_FAIR_GROUP_SCHED && defined CONFIG_SMP | ||
9513 | update_shares_data = __alloc_percpu(nr_cpu_ids * sizeof(unsigned long), | ||
9514 | __alignof__(unsigned long)); | ||
9515 | #endif | ||
9410 | for_each_possible_cpu(i) { | 9516 | for_each_possible_cpu(i) { |
9411 | struct rq *rq; | 9517 | struct rq *rq; |
9412 | 9518 | ||
@@ -9489,6 +9595,8 @@ void __init sched_init(void) | |||
9489 | rq->cpu = i; | 9595 | rq->cpu = i; |
9490 | rq->online = 0; | 9596 | rq->online = 0; |
9491 | rq->migration_thread = NULL; | 9597 | rq->migration_thread = NULL; |
9598 | rq->idle_stamp = 0; | ||
9599 | rq->avg_idle = 2*sysctl_sched_migration_cost; | ||
9492 | INIT_LIST_HEAD(&rq->migration_queue); | 9600 | INIT_LIST_HEAD(&rq->migration_queue); |
9493 | rq_attach_root(rq, &def_root_domain); | 9601 | rq_attach_root(rq, &def_root_domain); |
9494 | #endif | 9602 | #endif |
@@ -9532,13 +9640,15 @@ void __init sched_init(void) | |||
9532 | current->sched_class = &fair_sched_class; | 9640 | current->sched_class = &fair_sched_class; |
9533 | 9641 | ||
9534 | /* Allocate the nohz_cpu_mask if CONFIG_CPUMASK_OFFSTACK */ | 9642 | /* Allocate the nohz_cpu_mask if CONFIG_CPUMASK_OFFSTACK */ |
9535 | alloc_cpumask_var(&nohz_cpu_mask, GFP_NOWAIT); | 9643 | zalloc_cpumask_var(&nohz_cpu_mask, GFP_NOWAIT); |
9536 | #ifdef CONFIG_SMP | 9644 | #ifdef CONFIG_SMP |
9537 | #ifdef CONFIG_NO_HZ | 9645 | #ifdef CONFIG_NO_HZ |
9538 | alloc_cpumask_var(&nohz.cpu_mask, GFP_NOWAIT); | 9646 | zalloc_cpumask_var(&nohz.cpu_mask, GFP_NOWAIT); |
9539 | alloc_cpumask_var(&nohz.ilb_grp_nohz_mask, GFP_NOWAIT); | 9647 | alloc_cpumask_var(&nohz.ilb_grp_nohz_mask, GFP_NOWAIT); |
9540 | #endif | 9648 | #endif |
9541 | alloc_cpumask_var(&cpu_isolated_map, GFP_NOWAIT); | 9649 | /* May be allocated at isolcpus cmdline parse time */ |
9650 | if (cpu_isolated_map == NULL) | ||
9651 | zalloc_cpumask_var(&cpu_isolated_map, GFP_NOWAIT); | ||
9542 | #endif /* SMP */ | 9652 | #endif /* SMP */ |
9543 | 9653 | ||
9544 | perf_event_init(); | 9654 | perf_event_init(); |
@@ -10868,6 +10978,7 @@ void synchronize_sched_expedited(void) | |||
10868 | spin_unlock_irqrestore(&rq->lock, flags); | 10978 | spin_unlock_irqrestore(&rq->lock, flags); |
10869 | } | 10979 | } |
10870 | rcu_expedited_state = RCU_EXPEDITED_STATE_IDLE; | 10980 | rcu_expedited_state = RCU_EXPEDITED_STATE_IDLE; |
10981 | synchronize_sched_expedited_count++; | ||
10871 | mutex_unlock(&rcu_sched_expedited_mutex); | 10982 | mutex_unlock(&rcu_sched_expedited_mutex); |
10872 | put_online_cpus(); | 10983 | put_online_cpus(); |
10873 | if (need_full_sync) | 10984 | if (need_full_sync) |
diff --git a/kernel/sched_debug.c b/kernel/sched_debug.c index efb84409bc43..6988cf08f705 100644 --- a/kernel/sched_debug.c +++ b/kernel/sched_debug.c | |||
@@ -285,12 +285,16 @@ static void print_cpu(struct seq_file *m, int cpu) | |||
285 | 285 | ||
286 | #ifdef CONFIG_SCHEDSTATS | 286 | #ifdef CONFIG_SCHEDSTATS |
287 | #define P(n) SEQ_printf(m, " .%-30s: %d\n", #n, rq->n); | 287 | #define P(n) SEQ_printf(m, " .%-30s: %d\n", #n, rq->n); |
288 | #define P64(n) SEQ_printf(m, " .%-30s: %Ld\n", #n, rq->n); | ||
288 | 289 | ||
289 | P(yld_count); | 290 | P(yld_count); |
290 | 291 | ||
291 | P(sched_switch); | 292 | P(sched_switch); |
292 | P(sched_count); | 293 | P(sched_count); |
293 | P(sched_goidle); | 294 | P(sched_goidle); |
295 | #ifdef CONFIG_SMP | ||
296 | P64(avg_idle); | ||
297 | #endif | ||
294 | 298 | ||
295 | P(ttwu_count); | 299 | P(ttwu_count); |
296 | P(ttwu_local); | 300 | P(ttwu_local); |
diff --git a/kernel/sched_fair.c b/kernel/sched_fair.c index 4e777b47eeda..f61837ad336d 100644 --- a/kernel/sched_fair.c +++ b/kernel/sched_fair.c | |||
@@ -822,6 +822,26 @@ check_preempt_tick(struct cfs_rq *cfs_rq, struct sched_entity *curr) | |||
822 | * re-elected due to buddy favours. | 822 | * re-elected due to buddy favours. |
823 | */ | 823 | */ |
824 | clear_buddies(cfs_rq, curr); | 824 | clear_buddies(cfs_rq, curr); |
825 | return; | ||
826 | } | ||
827 | |||
828 | /* | ||
829 | * Ensure that a task that missed wakeup preemption by a | ||
830 | * narrow margin doesn't have to wait for a full slice. | ||
831 | * This also mitigates buddy induced latencies under load. | ||
832 | */ | ||
833 | if (!sched_feat(WAKEUP_PREEMPT)) | ||
834 | return; | ||
835 | |||
836 | if (delta_exec < sysctl_sched_min_granularity) | ||
837 | return; | ||
838 | |||
839 | if (cfs_rq->nr_running > 1) { | ||
840 | struct sched_entity *se = __pick_next_entity(cfs_rq); | ||
841 | s64 delta = curr->vruntime - se->vruntime; | ||
842 | |||
843 | if (delta > ideal_runtime) | ||
844 | resched_task(rq_of(cfs_rq)->curr); | ||
825 | } | 845 | } |
826 | } | 846 | } |
827 | 847 | ||
@@ -861,12 +881,18 @@ wakeup_preempt_entity(struct sched_entity *curr, struct sched_entity *se); | |||
861 | static struct sched_entity *pick_next_entity(struct cfs_rq *cfs_rq) | 881 | static struct sched_entity *pick_next_entity(struct cfs_rq *cfs_rq) |
862 | { | 882 | { |
863 | struct sched_entity *se = __pick_next_entity(cfs_rq); | 883 | struct sched_entity *se = __pick_next_entity(cfs_rq); |
884 | struct sched_entity *left = se; | ||
864 | 885 | ||
865 | if (cfs_rq->next && wakeup_preempt_entity(cfs_rq->next, se) < 1) | 886 | if (cfs_rq->next && wakeup_preempt_entity(cfs_rq->next, left) < 1) |
866 | return cfs_rq->next; | 887 | se = cfs_rq->next; |
867 | 888 | ||
868 | if (cfs_rq->last && wakeup_preempt_entity(cfs_rq->last, se) < 1) | 889 | /* |
869 | return cfs_rq->last; | 890 | * Prefer last buddy, try to return the CPU to a preempted task. |
891 | */ | ||
892 | if (cfs_rq->last && wakeup_preempt_entity(cfs_rq->last, left) < 1) | ||
893 | se = cfs_rq->last; | ||
894 | |||
895 | clear_buddies(cfs_rq, se); | ||
870 | 896 | ||
871 | return se; | 897 | return se; |
872 | } | 898 | } |
@@ -1319,6 +1345,37 @@ find_idlest_cpu(struct sched_group *group, struct task_struct *p, int this_cpu) | |||
1319 | } | 1345 | } |
1320 | 1346 | ||
1321 | /* | 1347 | /* |
1348 | * Try and locate an idle CPU in the sched_domain. | ||
1349 | */ | ||
1350 | static int | ||
1351 | select_idle_sibling(struct task_struct *p, struct sched_domain *sd, int target) | ||
1352 | { | ||
1353 | int cpu = smp_processor_id(); | ||
1354 | int prev_cpu = task_cpu(p); | ||
1355 | int i; | ||
1356 | |||
1357 | /* | ||
1358 | * If this domain spans both cpu and prev_cpu (see the SD_WAKE_AFFINE | ||
1359 | * test in select_task_rq_fair) and the prev_cpu is idle then that's | ||
1360 | * always a better target than the current cpu. | ||
1361 | */ | ||
1362 | if (target == cpu && !cpu_rq(prev_cpu)->cfs.nr_running) | ||
1363 | return prev_cpu; | ||
1364 | |||
1365 | /* | ||
1366 | * Otherwise, iterate the domain and find an elegible idle cpu. | ||
1367 | */ | ||
1368 | for_each_cpu_and(i, sched_domain_span(sd), &p->cpus_allowed) { | ||
1369 | if (!cpu_rq(i)->cfs.nr_running) { | ||
1370 | target = i; | ||
1371 | break; | ||
1372 | } | ||
1373 | } | ||
1374 | |||
1375 | return target; | ||
1376 | } | ||
1377 | |||
1378 | /* | ||
1322 | * sched_balance_self: balance the current task (running on cpu) in domains | 1379 | * sched_balance_self: balance the current task (running on cpu) in domains |
1323 | * that have the 'flag' flag set. In practice, this is SD_BALANCE_FORK and | 1380 | * that have the 'flag' flag set. In practice, this is SD_BALANCE_FORK and |
1324 | * SD_BALANCE_EXEC. | 1381 | * SD_BALANCE_EXEC. |
@@ -1372,11 +1429,35 @@ static int select_task_rq_fair(struct task_struct *p, int sd_flag, int wake_flag | |||
1372 | want_sd = 0; | 1429 | want_sd = 0; |
1373 | } | 1430 | } |
1374 | 1431 | ||
1375 | if (want_affine && (tmp->flags & SD_WAKE_AFFINE) && | 1432 | /* |
1376 | cpumask_test_cpu(prev_cpu, sched_domain_span(tmp))) { | 1433 | * While iterating the domains looking for a spanning |
1434 | * WAKE_AFFINE domain, adjust the affine target to any idle cpu | ||
1435 | * in cache sharing domains along the way. | ||
1436 | */ | ||
1437 | if (want_affine) { | ||
1438 | int target = -1; | ||
1377 | 1439 | ||
1378 | affine_sd = tmp; | 1440 | /* |
1379 | want_affine = 0; | 1441 | * If both cpu and prev_cpu are part of this domain, |
1442 | * cpu is a valid SD_WAKE_AFFINE target. | ||
1443 | */ | ||
1444 | if (cpumask_test_cpu(prev_cpu, sched_domain_span(tmp))) | ||
1445 | target = cpu; | ||
1446 | |||
1447 | /* | ||
1448 | * If there's an idle sibling in this domain, make that | ||
1449 | * the wake_affine target instead of the current cpu. | ||
1450 | */ | ||
1451 | if (tmp->flags & SD_PREFER_SIBLING) | ||
1452 | target = select_idle_sibling(p, tmp, target); | ||
1453 | |||
1454 | if (target >= 0) { | ||
1455 | if (tmp->flags & SD_WAKE_AFFINE) { | ||
1456 | affine_sd = tmp; | ||
1457 | want_affine = 0; | ||
1458 | } | ||
1459 | cpu = target; | ||
1460 | } | ||
1380 | } | 1461 | } |
1381 | 1462 | ||
1382 | if (!want_sd && !want_affine) | 1463 | if (!want_sd && !want_affine) |
@@ -1568,6 +1649,7 @@ static void check_preempt_wakeup(struct rq *rq, struct task_struct *p, int wake_ | |||
1568 | struct sched_entity *se = &curr->se, *pse = &p->se; | 1649 | struct sched_entity *se = &curr->se, *pse = &p->se; |
1569 | struct cfs_rq *cfs_rq = task_cfs_rq(curr); | 1650 | struct cfs_rq *cfs_rq = task_cfs_rq(curr); |
1570 | int sync = wake_flags & WF_SYNC; | 1651 | int sync = wake_flags & WF_SYNC; |
1652 | int scale = cfs_rq->nr_running >= sched_nr_latency; | ||
1571 | 1653 | ||
1572 | update_curr(cfs_rq); | 1654 | update_curr(cfs_rq); |
1573 | 1655 | ||
@@ -1582,18 +1664,7 @@ static void check_preempt_wakeup(struct rq *rq, struct task_struct *p, int wake_ | |||
1582 | if (unlikely(se == pse)) | 1664 | if (unlikely(se == pse)) |
1583 | return; | 1665 | return; |
1584 | 1666 | ||
1585 | /* | 1667 | if (sched_feat(NEXT_BUDDY) && scale && !(wake_flags & WF_FORK)) |
1586 | * Only set the backward buddy when the current task is still on the | ||
1587 | * rq. This can happen when a wakeup gets interleaved with schedule on | ||
1588 | * the ->pre_schedule() or idle_balance() point, either of which can | ||
1589 | * drop the rq lock. | ||
1590 | * | ||
1591 | * Also, during early boot the idle thread is in the fair class, for | ||
1592 | * obvious reasons its a bad idea to schedule back to the idle thread. | ||
1593 | */ | ||
1594 | if (sched_feat(LAST_BUDDY) && likely(se->on_rq && curr != rq->idle)) | ||
1595 | set_last_buddy(se); | ||
1596 | if (sched_feat(NEXT_BUDDY) && !(wake_flags & WF_FORK)) | ||
1597 | set_next_buddy(pse); | 1668 | set_next_buddy(pse); |
1598 | 1669 | ||
1599 | /* | 1670 | /* |
@@ -1639,8 +1710,22 @@ static void check_preempt_wakeup(struct rq *rq, struct task_struct *p, int wake_ | |||
1639 | 1710 | ||
1640 | BUG_ON(!pse); | 1711 | BUG_ON(!pse); |
1641 | 1712 | ||
1642 | if (wakeup_preempt_entity(se, pse) == 1) | 1713 | if (wakeup_preempt_entity(se, pse) == 1) { |
1643 | resched_task(curr); | 1714 | resched_task(curr); |
1715 | /* | ||
1716 | * Only set the backward buddy when the current task is still | ||
1717 | * on the rq. This can happen when a wakeup gets interleaved | ||
1718 | * with schedule on the ->pre_schedule() or idle_balance() | ||
1719 | * point, either of which can * drop the rq lock. | ||
1720 | * | ||
1721 | * Also, during early boot the idle thread is in the fair class, | ||
1722 | * for obvious reasons its a bad idea to schedule back to it. | ||
1723 | */ | ||
1724 | if (unlikely(!se->on_rq || curr == rq->idle)) | ||
1725 | return; | ||
1726 | if (sched_feat(LAST_BUDDY) && scale && entity_is_task(se)) | ||
1727 | set_last_buddy(se); | ||
1728 | } | ||
1644 | } | 1729 | } |
1645 | 1730 | ||
1646 | static struct task_struct *pick_next_task_fair(struct rq *rq) | 1731 | static struct task_struct *pick_next_task_fair(struct rq *rq) |
@@ -1649,21 +1734,11 @@ static struct task_struct *pick_next_task_fair(struct rq *rq) | |||
1649 | struct cfs_rq *cfs_rq = &rq->cfs; | 1734 | struct cfs_rq *cfs_rq = &rq->cfs; |
1650 | struct sched_entity *se; | 1735 | struct sched_entity *se; |
1651 | 1736 | ||
1652 | if (unlikely(!cfs_rq->nr_running)) | 1737 | if (!cfs_rq->nr_running) |
1653 | return NULL; | 1738 | return NULL; |
1654 | 1739 | ||
1655 | do { | 1740 | do { |
1656 | se = pick_next_entity(cfs_rq); | 1741 | se = pick_next_entity(cfs_rq); |
1657 | /* | ||
1658 | * If se was a buddy, clear it so that it will have to earn | ||
1659 | * the favour again. | ||
1660 | * | ||
1661 | * If se was not a buddy, clear the buddies because neither | ||
1662 | * was elegible to run, let them earn it again. | ||
1663 | * | ||
1664 | * IOW. unconditionally clear buddies. | ||
1665 | */ | ||
1666 | __clear_buddies(cfs_rq, NULL); | ||
1667 | set_next_entity(cfs_rq, se); | 1742 | set_next_entity(cfs_rq, se); |
1668 | cfs_rq = group_cfs_rq(se); | 1743 | cfs_rq = group_cfs_rq(se); |
1669 | } while (cfs_rq); | 1744 | } while (cfs_rq); |
diff --git a/kernel/sched_rt.c b/kernel/sched_rt.c index a4d790cddb19..5c5fef378415 100644 --- a/kernel/sched_rt.c +++ b/kernel/sched_rt.c | |||
@@ -1153,29 +1153,12 @@ static struct task_struct *pick_next_highest_task_rt(struct rq *rq, int cpu) | |||
1153 | 1153 | ||
1154 | static DEFINE_PER_CPU(cpumask_var_t, local_cpu_mask); | 1154 | static DEFINE_PER_CPU(cpumask_var_t, local_cpu_mask); |
1155 | 1155 | ||
1156 | static inline int pick_optimal_cpu(int this_cpu, | ||
1157 | const struct cpumask *mask) | ||
1158 | { | ||
1159 | int first; | ||
1160 | |||
1161 | /* "this_cpu" is cheaper to preempt than a remote processor */ | ||
1162 | if ((this_cpu != -1) && cpumask_test_cpu(this_cpu, mask)) | ||
1163 | return this_cpu; | ||
1164 | |||
1165 | first = cpumask_first(mask); | ||
1166 | if (first < nr_cpu_ids) | ||
1167 | return first; | ||
1168 | |||
1169 | return -1; | ||
1170 | } | ||
1171 | |||
1172 | static int find_lowest_rq(struct task_struct *task) | 1156 | static int find_lowest_rq(struct task_struct *task) |
1173 | { | 1157 | { |
1174 | struct sched_domain *sd; | 1158 | struct sched_domain *sd; |
1175 | struct cpumask *lowest_mask = __get_cpu_var(local_cpu_mask); | 1159 | struct cpumask *lowest_mask = __get_cpu_var(local_cpu_mask); |
1176 | int this_cpu = smp_processor_id(); | 1160 | int this_cpu = smp_processor_id(); |
1177 | int cpu = task_cpu(task); | 1161 | int cpu = task_cpu(task); |
1178 | cpumask_var_t domain_mask; | ||
1179 | 1162 | ||
1180 | if (task->rt.nr_cpus_allowed == 1) | 1163 | if (task->rt.nr_cpus_allowed == 1) |
1181 | return -1; /* No other targets possible */ | 1164 | return -1; /* No other targets possible */ |
@@ -1198,28 +1181,26 @@ static int find_lowest_rq(struct task_struct *task) | |||
1198 | * Otherwise, we consult the sched_domains span maps to figure | 1181 | * Otherwise, we consult the sched_domains span maps to figure |
1199 | * out which cpu is logically closest to our hot cache data. | 1182 | * out which cpu is logically closest to our hot cache data. |
1200 | */ | 1183 | */ |
1201 | if (this_cpu == cpu) | 1184 | if (!cpumask_test_cpu(this_cpu, lowest_mask)) |
1202 | this_cpu = -1; /* Skip this_cpu opt if the same */ | 1185 | this_cpu = -1; /* Skip this_cpu opt if not among lowest */ |
1203 | |||
1204 | if (alloc_cpumask_var(&domain_mask, GFP_ATOMIC)) { | ||
1205 | for_each_domain(cpu, sd) { | ||
1206 | if (sd->flags & SD_WAKE_AFFINE) { | ||
1207 | int best_cpu; | ||
1208 | 1186 | ||
1209 | cpumask_and(domain_mask, | 1187 | for_each_domain(cpu, sd) { |
1210 | sched_domain_span(sd), | 1188 | if (sd->flags & SD_WAKE_AFFINE) { |
1211 | lowest_mask); | 1189 | int best_cpu; |
1212 | 1190 | ||
1213 | best_cpu = pick_optimal_cpu(this_cpu, | 1191 | /* |
1214 | domain_mask); | 1192 | * "this_cpu" is cheaper to preempt than a |
1215 | 1193 | * remote processor. | |
1216 | if (best_cpu != -1) { | 1194 | */ |
1217 | free_cpumask_var(domain_mask); | 1195 | if (this_cpu != -1 && |
1218 | return best_cpu; | 1196 | cpumask_test_cpu(this_cpu, sched_domain_span(sd))) |
1219 | } | 1197 | return this_cpu; |
1220 | } | 1198 | |
1199 | best_cpu = cpumask_first_and(lowest_mask, | ||
1200 | sched_domain_span(sd)); | ||
1201 | if (best_cpu < nr_cpu_ids) | ||
1202 | return best_cpu; | ||
1221 | } | 1203 | } |
1222 | free_cpumask_var(domain_mask); | ||
1223 | } | 1204 | } |
1224 | 1205 | ||
1225 | /* | 1206 | /* |
@@ -1227,7 +1208,13 @@ static int find_lowest_rq(struct task_struct *task) | |||
1227 | * just give the caller *something* to work with from the compatible | 1208 | * just give the caller *something* to work with from the compatible |
1228 | * locations. | 1209 | * locations. |
1229 | */ | 1210 | */ |
1230 | return pick_optimal_cpu(this_cpu, lowest_mask); | 1211 | if (this_cpu != -1) |
1212 | return this_cpu; | ||
1213 | |||
1214 | cpu = cpumask_any(lowest_mask); | ||
1215 | if (cpu < nr_cpu_ids) | ||
1216 | return cpu; | ||
1217 | return -1; | ||
1231 | } | 1218 | } |
1232 | 1219 | ||
1233 | /* Will lock the rq it finds */ | 1220 | /* Will lock the rq it finds */ |
diff --git a/kernel/signal.c b/kernel/signal.c index 6705320784fd..6b982f2cf524 100644 --- a/kernel/signal.c +++ b/kernel/signal.c | |||
@@ -22,12 +22,14 @@ | |||
22 | #include <linux/ptrace.h> | 22 | #include <linux/ptrace.h> |
23 | #include <linux/signal.h> | 23 | #include <linux/signal.h> |
24 | #include <linux/signalfd.h> | 24 | #include <linux/signalfd.h> |
25 | #include <linux/ratelimit.h> | ||
25 | #include <linux/tracehook.h> | 26 | #include <linux/tracehook.h> |
26 | #include <linux/capability.h> | 27 | #include <linux/capability.h> |
27 | #include <linux/freezer.h> | 28 | #include <linux/freezer.h> |
28 | #include <linux/pid_namespace.h> | 29 | #include <linux/pid_namespace.h> |
29 | #include <linux/nsproxy.h> | 30 | #include <linux/nsproxy.h> |
30 | #include <trace/events/sched.h> | 31 | #define CREATE_TRACE_POINTS |
32 | #include <trace/events/signal.h> | ||
31 | 33 | ||
32 | #include <asm/param.h> | 34 | #include <asm/param.h> |
33 | #include <asm/uaccess.h> | 35 | #include <asm/uaccess.h> |
@@ -41,6 +43,8 @@ | |||
41 | 43 | ||
42 | static struct kmem_cache *sigqueue_cachep; | 44 | static struct kmem_cache *sigqueue_cachep; |
43 | 45 | ||
46 | int print_fatal_signals __read_mostly; | ||
47 | |||
44 | static void __user *sig_handler(struct task_struct *t, int sig) | 48 | static void __user *sig_handler(struct task_struct *t, int sig) |
45 | { | 49 | { |
46 | return t->sighand->action[sig - 1].sa.sa_handler; | 50 | return t->sighand->action[sig - 1].sa.sa_handler; |
@@ -159,7 +163,7 @@ int next_signal(struct sigpending *pending, sigset_t *mask) | |||
159 | { | 163 | { |
160 | unsigned long i, *s, *m, x; | 164 | unsigned long i, *s, *m, x; |
161 | int sig = 0; | 165 | int sig = 0; |
162 | 166 | ||
163 | s = pending->signal.sig; | 167 | s = pending->signal.sig; |
164 | m = mask->sig; | 168 | m = mask->sig; |
165 | switch (_NSIG_WORDS) { | 169 | switch (_NSIG_WORDS) { |
@@ -184,17 +188,31 @@ int next_signal(struct sigpending *pending, sigset_t *mask) | |||
184 | sig = ffz(~x) + 1; | 188 | sig = ffz(~x) + 1; |
185 | break; | 189 | break; |
186 | } | 190 | } |
187 | 191 | ||
188 | return sig; | 192 | return sig; |
189 | } | 193 | } |
190 | 194 | ||
195 | static inline void print_dropped_signal(int sig) | ||
196 | { | ||
197 | static DEFINE_RATELIMIT_STATE(ratelimit_state, 5 * HZ, 10); | ||
198 | |||
199 | if (!print_fatal_signals) | ||
200 | return; | ||
201 | |||
202 | if (!__ratelimit(&ratelimit_state)) | ||
203 | return; | ||
204 | |||
205 | printk(KERN_INFO "%s/%d: reached RLIMIT_SIGPENDING, dropped signal %d\n", | ||
206 | current->comm, current->pid, sig); | ||
207 | } | ||
208 | |||
191 | /* | 209 | /* |
192 | * allocate a new signal queue record | 210 | * allocate a new signal queue record |
193 | * - this may be called without locks if and only if t == current, otherwise an | 211 | * - this may be called without locks if and only if t == current, otherwise an |
194 | * appopriate lock must be held to stop the target task from exiting | 212 | * appopriate lock must be held to stop the target task from exiting |
195 | */ | 213 | */ |
196 | static struct sigqueue *__sigqueue_alloc(struct task_struct *t, gfp_t flags, | 214 | static struct sigqueue * |
197 | int override_rlimit) | 215 | __sigqueue_alloc(int sig, struct task_struct *t, gfp_t flags, int override_rlimit) |
198 | { | 216 | { |
199 | struct sigqueue *q = NULL; | 217 | struct sigqueue *q = NULL; |
200 | struct user_struct *user; | 218 | struct user_struct *user; |
@@ -207,10 +225,15 @@ static struct sigqueue *__sigqueue_alloc(struct task_struct *t, gfp_t flags, | |||
207 | */ | 225 | */ |
208 | user = get_uid(__task_cred(t)->user); | 226 | user = get_uid(__task_cred(t)->user); |
209 | atomic_inc(&user->sigpending); | 227 | atomic_inc(&user->sigpending); |
228 | |||
210 | if (override_rlimit || | 229 | if (override_rlimit || |
211 | atomic_read(&user->sigpending) <= | 230 | atomic_read(&user->sigpending) <= |
212 | t->signal->rlim[RLIMIT_SIGPENDING].rlim_cur) | 231 | t->signal->rlim[RLIMIT_SIGPENDING].rlim_cur) { |
213 | q = kmem_cache_alloc(sigqueue_cachep, flags); | 232 | q = kmem_cache_alloc(sigqueue_cachep, flags); |
233 | } else { | ||
234 | print_dropped_signal(sig); | ||
235 | } | ||
236 | |||
214 | if (unlikely(q == NULL)) { | 237 | if (unlikely(q == NULL)) { |
215 | atomic_dec(&user->sigpending); | 238 | atomic_dec(&user->sigpending); |
216 | free_uid(user); | 239 | free_uid(user); |
@@ -834,7 +857,7 @@ static int __send_signal(int sig, struct siginfo *info, struct task_struct *t, | |||
834 | struct sigqueue *q; | 857 | struct sigqueue *q; |
835 | int override_rlimit; | 858 | int override_rlimit; |
836 | 859 | ||
837 | trace_sched_signal_send(sig, t); | 860 | trace_signal_generate(sig, info, t); |
838 | 861 | ||
839 | assert_spin_locked(&t->sighand->siglock); | 862 | assert_spin_locked(&t->sighand->siglock); |
840 | 863 | ||
@@ -869,7 +892,7 @@ static int __send_signal(int sig, struct siginfo *info, struct task_struct *t, | |||
869 | else | 892 | else |
870 | override_rlimit = 0; | 893 | override_rlimit = 0; |
871 | 894 | ||
872 | q = __sigqueue_alloc(t, GFP_ATOMIC | __GFP_NOTRACK_FALSE_POSITIVE, | 895 | q = __sigqueue_alloc(sig, t, GFP_ATOMIC | __GFP_NOTRACK_FALSE_POSITIVE, |
873 | override_rlimit); | 896 | override_rlimit); |
874 | if (q) { | 897 | if (q) { |
875 | list_add_tail(&q->list, &pending->list); | 898 | list_add_tail(&q->list, &pending->list); |
@@ -896,12 +919,21 @@ static int __send_signal(int sig, struct siginfo *info, struct task_struct *t, | |||
896 | break; | 919 | break; |
897 | } | 920 | } |
898 | } else if (!is_si_special(info)) { | 921 | } else if (!is_si_special(info)) { |
899 | if (sig >= SIGRTMIN && info->si_code != SI_USER) | 922 | if (sig >= SIGRTMIN && info->si_code != SI_USER) { |
900 | /* | 923 | /* |
901 | * Queue overflow, abort. We may abort if the signal was rt | 924 | * Queue overflow, abort. We may abort if the |
902 | * and sent by user using something other than kill(). | 925 | * signal was rt and sent by user using something |
903 | */ | 926 | * other than kill(). |
927 | */ | ||
928 | trace_signal_overflow_fail(sig, group, info); | ||
904 | return -EAGAIN; | 929 | return -EAGAIN; |
930 | } else { | ||
931 | /* | ||
932 | * This is a silent loss of information. We still | ||
933 | * send the signal, but the *info bits are lost. | ||
934 | */ | ||
935 | trace_signal_lose_info(sig, group, info); | ||
936 | } | ||
905 | } | 937 | } |
906 | 938 | ||
907 | out_set: | 939 | out_set: |
@@ -925,8 +957,6 @@ static int send_signal(int sig, struct siginfo *info, struct task_struct *t, | |||
925 | return __send_signal(sig, info, t, group, from_ancestor_ns); | 957 | return __send_signal(sig, info, t, group, from_ancestor_ns); |
926 | } | 958 | } |
927 | 959 | ||
928 | int print_fatal_signals; | ||
929 | |||
930 | static void print_fatal_signal(struct pt_regs *regs, int signr) | 960 | static void print_fatal_signal(struct pt_regs *regs, int signr) |
931 | { | 961 | { |
932 | printk("%s/%d: potentially unexpected fatal signal %d.\n", | 962 | printk("%s/%d: potentially unexpected fatal signal %d.\n", |
@@ -1293,19 +1323,19 @@ EXPORT_SYMBOL(kill_pid); | |||
1293 | * These functions support sending signals using preallocated sigqueue | 1323 | * These functions support sending signals using preallocated sigqueue |
1294 | * structures. This is needed "because realtime applications cannot | 1324 | * structures. This is needed "because realtime applications cannot |
1295 | * afford to lose notifications of asynchronous events, like timer | 1325 | * afford to lose notifications of asynchronous events, like timer |
1296 | * expirations or I/O completions". In the case of Posix Timers | 1326 | * expirations or I/O completions". In the case of Posix Timers |
1297 | * we allocate the sigqueue structure from the timer_create. If this | 1327 | * we allocate the sigqueue structure from the timer_create. If this |
1298 | * allocation fails we are able to report the failure to the application | 1328 | * allocation fails we are able to report the failure to the application |
1299 | * with an EAGAIN error. | 1329 | * with an EAGAIN error. |
1300 | */ | 1330 | */ |
1301 | |||
1302 | struct sigqueue *sigqueue_alloc(void) | 1331 | struct sigqueue *sigqueue_alloc(void) |
1303 | { | 1332 | { |
1304 | struct sigqueue *q; | 1333 | struct sigqueue *q = __sigqueue_alloc(-1, current, GFP_KERNEL, 0); |
1305 | 1334 | ||
1306 | if ((q = __sigqueue_alloc(current, GFP_KERNEL, 0))) | 1335 | if (q) |
1307 | q->flags |= SIGQUEUE_PREALLOC; | 1336 | q->flags |= SIGQUEUE_PREALLOC; |
1308 | return(q); | 1337 | |
1338 | return q; | ||
1309 | } | 1339 | } |
1310 | 1340 | ||
1311 | void sigqueue_free(struct sigqueue *q) | 1341 | void sigqueue_free(struct sigqueue *q) |
@@ -1839,6 +1869,9 @@ relock: | |||
1839 | ka = &sighand->action[signr-1]; | 1869 | ka = &sighand->action[signr-1]; |
1840 | } | 1870 | } |
1841 | 1871 | ||
1872 | /* Trace actually delivered signals. */ | ||
1873 | trace_signal_deliver(signr, info, ka); | ||
1874 | |||
1842 | if (ka->sa.sa_handler == SIG_IGN) /* Do nothing. */ | 1875 | if (ka->sa.sa_handler == SIG_IGN) /* Do nothing. */ |
1843 | continue; | 1876 | continue; |
1844 | if (ka->sa.sa_handler != SIG_DFL) { | 1877 | if (ka->sa.sa_handler != SIG_DFL) { |
diff --git a/kernel/slow-work-debugfs.c b/kernel/slow-work-debugfs.c new file mode 100644 index 000000000000..e45c43645298 --- /dev/null +++ b/kernel/slow-work-debugfs.c | |||
@@ -0,0 +1,227 @@ | |||
1 | /* Slow work debugging | ||
2 | * | ||
3 | * Copyright (C) 2009 Red Hat, Inc. All Rights Reserved. | ||
4 | * Written by David Howells (dhowells@redhat.com) | ||
5 | * | ||
6 | * This program is free software; you can redistribute it and/or | ||
7 | * modify it under the terms of the GNU General Public Licence | ||
8 | * as published by the Free Software Foundation; either version | ||
9 | * 2 of the Licence, or (at your option) any later version. | ||
10 | */ | ||
11 | |||
12 | #include <linux/module.h> | ||
13 | #include <linux/slow-work.h> | ||
14 | #include <linux/fs.h> | ||
15 | #include <linux/time.h> | ||
16 | #include <linux/seq_file.h> | ||
17 | #include "slow-work.h" | ||
18 | |||
19 | #define ITERATOR_SHIFT (BITS_PER_LONG - 4) | ||
20 | #define ITERATOR_SELECTOR (0xfUL << ITERATOR_SHIFT) | ||
21 | #define ITERATOR_COUNTER (~ITERATOR_SELECTOR) | ||
22 | |||
23 | void slow_work_new_thread_desc(struct slow_work *work, struct seq_file *m) | ||
24 | { | ||
25 | seq_puts(m, "Slow-work: New thread"); | ||
26 | } | ||
27 | |||
28 | /* | ||
29 | * Render the time mark field on a work item into a 5-char time with units plus | ||
30 | * a space | ||
31 | */ | ||
32 | static void slow_work_print_mark(struct seq_file *m, struct slow_work *work) | ||
33 | { | ||
34 | struct timespec now, diff; | ||
35 | |||
36 | now = CURRENT_TIME; | ||
37 | diff = timespec_sub(now, work->mark); | ||
38 | |||
39 | if (diff.tv_sec < 0) | ||
40 | seq_puts(m, " -ve "); | ||
41 | else if (diff.tv_sec == 0 && diff.tv_nsec < 1000) | ||
42 | seq_printf(m, "%3luns ", diff.tv_nsec); | ||
43 | else if (diff.tv_sec == 0 && diff.tv_nsec < 1000000) | ||
44 | seq_printf(m, "%3luus ", diff.tv_nsec / 1000); | ||
45 | else if (diff.tv_sec == 0 && diff.tv_nsec < 1000000000) | ||
46 | seq_printf(m, "%3lums ", diff.tv_nsec / 1000000); | ||
47 | else if (diff.tv_sec <= 1) | ||
48 | seq_puts(m, " 1s "); | ||
49 | else if (diff.tv_sec < 60) | ||
50 | seq_printf(m, "%4lus ", diff.tv_sec); | ||
51 | else if (diff.tv_sec < 60 * 60) | ||
52 | seq_printf(m, "%4lum ", diff.tv_sec / 60); | ||
53 | else if (diff.tv_sec < 60 * 60 * 24) | ||
54 | seq_printf(m, "%4luh ", diff.tv_sec / 3600); | ||
55 | else | ||
56 | seq_puts(m, "exces "); | ||
57 | } | ||
58 | |||
59 | /* | ||
60 | * Describe a slow work item for debugfs | ||
61 | */ | ||
62 | static int slow_work_runqueue_show(struct seq_file *m, void *v) | ||
63 | { | ||
64 | struct slow_work *work; | ||
65 | struct list_head *p = v; | ||
66 | unsigned long id; | ||
67 | |||
68 | switch ((unsigned long) v) { | ||
69 | case 1: | ||
70 | seq_puts(m, "THR PID ITEM ADDR FL MARK DESC\n"); | ||
71 | return 0; | ||
72 | case 2: | ||
73 | seq_puts(m, "=== ===== ================ == ===== ==========\n"); | ||
74 | return 0; | ||
75 | |||
76 | case 3 ... 3 + SLOW_WORK_THREAD_LIMIT - 1: | ||
77 | id = (unsigned long) v - 3; | ||
78 | |||
79 | read_lock(&slow_work_execs_lock); | ||
80 | work = slow_work_execs[id]; | ||
81 | if (work) { | ||
82 | smp_read_barrier_depends(); | ||
83 | |||
84 | seq_printf(m, "%3lu %5d %16p %2lx ", | ||
85 | id, slow_work_pids[id], work, work->flags); | ||
86 | slow_work_print_mark(m, work); | ||
87 | |||
88 | if (work->ops->desc) | ||
89 | work->ops->desc(work, m); | ||
90 | seq_putc(m, '\n'); | ||
91 | } | ||
92 | read_unlock(&slow_work_execs_lock); | ||
93 | return 0; | ||
94 | |||
95 | default: | ||
96 | work = list_entry(p, struct slow_work, link); | ||
97 | seq_printf(m, "%3s - %16p %2lx ", | ||
98 | work->flags & SLOW_WORK_VERY_SLOW ? "vsq" : "sq", | ||
99 | work, work->flags); | ||
100 | slow_work_print_mark(m, work); | ||
101 | |||
102 | if (work->ops->desc) | ||
103 | work->ops->desc(work, m); | ||
104 | seq_putc(m, '\n'); | ||
105 | return 0; | ||
106 | } | ||
107 | } | ||
108 | |||
109 | /* | ||
110 | * map the iterator to a work item | ||
111 | */ | ||
112 | static void *slow_work_runqueue_index(struct seq_file *m, loff_t *_pos) | ||
113 | { | ||
114 | struct list_head *p; | ||
115 | unsigned long count, id; | ||
116 | |||
117 | switch (*_pos >> ITERATOR_SHIFT) { | ||
118 | case 0x0: | ||
119 | if (*_pos == 0) | ||
120 | *_pos = 1; | ||
121 | if (*_pos < 3) | ||
122 | return (void *)(unsigned long) *_pos; | ||
123 | if (*_pos < 3 + SLOW_WORK_THREAD_LIMIT) | ||
124 | for (id = *_pos - 3; | ||
125 | id < SLOW_WORK_THREAD_LIMIT; | ||
126 | id++, (*_pos)++) | ||
127 | if (slow_work_execs[id]) | ||
128 | return (void *)(unsigned long) *_pos; | ||
129 | *_pos = 0x1UL << ITERATOR_SHIFT; | ||
130 | |||
131 | case 0x1: | ||
132 | count = *_pos & ITERATOR_COUNTER; | ||
133 | list_for_each(p, &slow_work_queue) { | ||
134 | if (count == 0) | ||
135 | return p; | ||
136 | count--; | ||
137 | } | ||
138 | *_pos = 0x2UL << ITERATOR_SHIFT; | ||
139 | |||
140 | case 0x2: | ||
141 | count = *_pos & ITERATOR_COUNTER; | ||
142 | list_for_each(p, &vslow_work_queue) { | ||
143 | if (count == 0) | ||
144 | return p; | ||
145 | count--; | ||
146 | } | ||
147 | *_pos = 0x3UL << ITERATOR_SHIFT; | ||
148 | |||
149 | default: | ||
150 | return NULL; | ||
151 | } | ||
152 | } | ||
153 | |||
154 | /* | ||
155 | * set up the iterator to start reading from the first line | ||
156 | */ | ||
157 | static void *slow_work_runqueue_start(struct seq_file *m, loff_t *_pos) | ||
158 | { | ||
159 | spin_lock_irq(&slow_work_queue_lock); | ||
160 | return slow_work_runqueue_index(m, _pos); | ||
161 | } | ||
162 | |||
163 | /* | ||
164 | * move to the next line | ||
165 | */ | ||
166 | static void *slow_work_runqueue_next(struct seq_file *m, void *v, loff_t *_pos) | ||
167 | { | ||
168 | struct list_head *p = v; | ||
169 | unsigned long selector = *_pos >> ITERATOR_SHIFT; | ||
170 | |||
171 | (*_pos)++; | ||
172 | switch (selector) { | ||
173 | case 0x0: | ||
174 | return slow_work_runqueue_index(m, _pos); | ||
175 | |||
176 | case 0x1: | ||
177 | if (*_pos >> ITERATOR_SHIFT == 0x1) { | ||
178 | p = p->next; | ||
179 | if (p != &slow_work_queue) | ||
180 | return p; | ||
181 | } | ||
182 | *_pos = 0x2UL << ITERATOR_SHIFT; | ||
183 | p = &vslow_work_queue; | ||
184 | |||
185 | case 0x2: | ||
186 | if (*_pos >> ITERATOR_SHIFT == 0x2) { | ||
187 | p = p->next; | ||
188 | if (p != &vslow_work_queue) | ||
189 | return p; | ||
190 | } | ||
191 | *_pos = 0x3UL << ITERATOR_SHIFT; | ||
192 | |||
193 | default: | ||
194 | return NULL; | ||
195 | } | ||
196 | } | ||
197 | |||
198 | /* | ||
199 | * clean up after reading | ||
200 | */ | ||
201 | static void slow_work_runqueue_stop(struct seq_file *m, void *v) | ||
202 | { | ||
203 | spin_unlock_irq(&slow_work_queue_lock); | ||
204 | } | ||
205 | |||
206 | static const struct seq_operations slow_work_runqueue_ops = { | ||
207 | .start = slow_work_runqueue_start, | ||
208 | .stop = slow_work_runqueue_stop, | ||
209 | .next = slow_work_runqueue_next, | ||
210 | .show = slow_work_runqueue_show, | ||
211 | }; | ||
212 | |||
213 | /* | ||
214 | * open "/sys/kernel/debug/slow_work/runqueue" to list queue contents | ||
215 | */ | ||
216 | static int slow_work_runqueue_open(struct inode *inode, struct file *file) | ||
217 | { | ||
218 | return seq_open(file, &slow_work_runqueue_ops); | ||
219 | } | ||
220 | |||
221 | const struct file_operations slow_work_runqueue_fops = { | ||
222 | .owner = THIS_MODULE, | ||
223 | .open = slow_work_runqueue_open, | ||
224 | .read = seq_read, | ||
225 | .llseek = seq_lseek, | ||
226 | .release = seq_release, | ||
227 | }; | ||
diff --git a/kernel/slow-work.c b/kernel/slow-work.c index 0d31135efbf4..7494bbf5a270 100644 --- a/kernel/slow-work.c +++ b/kernel/slow-work.c | |||
@@ -16,11 +16,8 @@ | |||
16 | #include <linux/kthread.h> | 16 | #include <linux/kthread.h> |
17 | #include <linux/freezer.h> | 17 | #include <linux/freezer.h> |
18 | #include <linux/wait.h> | 18 | #include <linux/wait.h> |
19 | 19 | #include <linux/debugfs.h> | |
20 | #define SLOW_WORK_CULL_TIMEOUT (5 * HZ) /* cull threads 5s after running out of | 20 | #include "slow-work.h" |
21 | * things to do */ | ||
22 | #define SLOW_WORK_OOM_TIMEOUT (5 * HZ) /* can't start new threads for 5s after | ||
23 | * OOM */ | ||
24 | 21 | ||
25 | static void slow_work_cull_timeout(unsigned long); | 22 | static void slow_work_cull_timeout(unsigned long); |
26 | static void slow_work_oom_timeout(unsigned long); | 23 | static void slow_work_oom_timeout(unsigned long); |
@@ -46,13 +43,12 @@ static unsigned vslow_work_proportion = 50; /* % of threads that may process | |||
46 | 43 | ||
47 | #ifdef CONFIG_SYSCTL | 44 | #ifdef CONFIG_SYSCTL |
48 | static const int slow_work_min_min_threads = 2; | 45 | static const int slow_work_min_min_threads = 2; |
49 | static int slow_work_max_max_threads = 255; | 46 | static int slow_work_max_max_threads = SLOW_WORK_THREAD_LIMIT; |
50 | static const int slow_work_min_vslow = 1; | 47 | static const int slow_work_min_vslow = 1; |
51 | static const int slow_work_max_vslow = 99; | 48 | static const int slow_work_max_vslow = 99; |
52 | 49 | ||
53 | ctl_table slow_work_sysctls[] = { | 50 | ctl_table slow_work_sysctls[] = { |
54 | { | 51 | { |
55 | .ctl_name = CTL_UNNUMBERED, | ||
56 | .procname = "min-threads", | 52 | .procname = "min-threads", |
57 | .data = &slow_work_min_threads, | 53 | .data = &slow_work_min_threads, |
58 | .maxlen = sizeof(unsigned), | 54 | .maxlen = sizeof(unsigned), |
@@ -62,7 +58,6 @@ ctl_table slow_work_sysctls[] = { | |||
62 | .extra2 = &slow_work_max_threads, | 58 | .extra2 = &slow_work_max_threads, |
63 | }, | 59 | }, |
64 | { | 60 | { |
65 | .ctl_name = CTL_UNNUMBERED, | ||
66 | .procname = "max-threads", | 61 | .procname = "max-threads", |
67 | .data = &slow_work_max_threads, | 62 | .data = &slow_work_max_threads, |
68 | .maxlen = sizeof(unsigned), | 63 | .maxlen = sizeof(unsigned), |
@@ -72,16 +67,15 @@ ctl_table slow_work_sysctls[] = { | |||
72 | .extra2 = (void *) &slow_work_max_max_threads, | 67 | .extra2 = (void *) &slow_work_max_max_threads, |
73 | }, | 68 | }, |
74 | { | 69 | { |
75 | .ctl_name = CTL_UNNUMBERED, | ||
76 | .procname = "vslow-percentage", | 70 | .procname = "vslow-percentage", |
77 | .data = &vslow_work_proportion, | 71 | .data = &vslow_work_proportion, |
78 | .maxlen = sizeof(unsigned), | 72 | .maxlen = sizeof(unsigned), |
79 | .mode = 0644, | 73 | .mode = 0644, |
80 | .proc_handler = &proc_dointvec_minmax, | 74 | .proc_handler = proc_dointvec_minmax, |
81 | .extra1 = (void *) &slow_work_min_vslow, | 75 | .extra1 = (void *) &slow_work_min_vslow, |
82 | .extra2 = (void *) &slow_work_max_vslow, | 76 | .extra2 = (void *) &slow_work_max_vslow, |
83 | }, | 77 | }, |
84 | { .ctl_name = 0 } | 78 | {} |
85 | }; | 79 | }; |
86 | #endif | 80 | #endif |
87 | 81 | ||
@@ -98,6 +92,56 @@ static DEFINE_TIMER(slow_work_oom_timer, slow_work_oom_timeout, 0, 0); | |||
98 | static struct slow_work slow_work_new_thread; /* new thread starter */ | 92 | static struct slow_work slow_work_new_thread; /* new thread starter */ |
99 | 93 | ||
100 | /* | 94 | /* |
95 | * slow work ID allocation (use slow_work_queue_lock) | ||
96 | */ | ||
97 | static DECLARE_BITMAP(slow_work_ids, SLOW_WORK_THREAD_LIMIT); | ||
98 | |||
99 | /* | ||
100 | * Unregistration tracking to prevent put_ref() from disappearing during module | ||
101 | * unload | ||
102 | */ | ||
103 | #ifdef CONFIG_MODULES | ||
104 | static struct module *slow_work_thread_processing[SLOW_WORK_THREAD_LIMIT]; | ||
105 | static struct module *slow_work_unreg_module; | ||
106 | static struct slow_work *slow_work_unreg_work_item; | ||
107 | static DECLARE_WAIT_QUEUE_HEAD(slow_work_unreg_wq); | ||
108 | static DEFINE_MUTEX(slow_work_unreg_sync_lock); | ||
109 | |||
110 | static void slow_work_set_thread_processing(int id, struct slow_work *work) | ||
111 | { | ||
112 | if (work) | ||
113 | slow_work_thread_processing[id] = work->owner; | ||
114 | } | ||
115 | static void slow_work_done_thread_processing(int id, struct slow_work *work) | ||
116 | { | ||
117 | struct module *module = slow_work_thread_processing[id]; | ||
118 | |||
119 | slow_work_thread_processing[id] = NULL; | ||
120 | smp_mb(); | ||
121 | if (slow_work_unreg_work_item == work || | ||
122 | slow_work_unreg_module == module) | ||
123 | wake_up_all(&slow_work_unreg_wq); | ||
124 | } | ||
125 | static void slow_work_clear_thread_processing(int id) | ||
126 | { | ||
127 | slow_work_thread_processing[id] = NULL; | ||
128 | } | ||
129 | #else | ||
130 | static void slow_work_set_thread_processing(int id, struct slow_work *work) {} | ||
131 | static void slow_work_done_thread_processing(int id, struct slow_work *work) {} | ||
132 | static void slow_work_clear_thread_processing(int id) {} | ||
133 | #endif | ||
134 | |||
135 | /* | ||
136 | * Data for tracking currently executing items for indication through /proc | ||
137 | */ | ||
138 | #ifdef CONFIG_SLOW_WORK_DEBUG | ||
139 | struct slow_work *slow_work_execs[SLOW_WORK_THREAD_LIMIT]; | ||
140 | pid_t slow_work_pids[SLOW_WORK_THREAD_LIMIT]; | ||
141 | DEFINE_RWLOCK(slow_work_execs_lock); | ||
142 | #endif | ||
143 | |||
144 | /* | ||
101 | * The queues of work items and the lock governing access to them. These are | 145 | * The queues of work items and the lock governing access to them. These are |
102 | * shared between all the CPUs. It doesn't make sense to have per-CPU queues | 146 | * shared between all the CPUs. It doesn't make sense to have per-CPU queues |
103 | * as the number of threads bears no relation to the number of CPUs. | 147 | * as the number of threads bears no relation to the number of CPUs. |
@@ -105,9 +149,18 @@ static struct slow_work slow_work_new_thread; /* new thread starter */ | |||
105 | * There are two queues of work items: one for slow work items, and one for | 149 | * There are two queues of work items: one for slow work items, and one for |
106 | * very slow work items. | 150 | * very slow work items. |
107 | */ | 151 | */ |
108 | static LIST_HEAD(slow_work_queue); | 152 | LIST_HEAD(slow_work_queue); |
109 | static LIST_HEAD(vslow_work_queue); | 153 | LIST_HEAD(vslow_work_queue); |
110 | static DEFINE_SPINLOCK(slow_work_queue_lock); | 154 | DEFINE_SPINLOCK(slow_work_queue_lock); |
155 | |||
156 | /* | ||
157 | * The following are two wait queues that get pinged when a work item is placed | ||
158 | * on an empty queue. These allow work items that are hogging a thread by | ||
159 | * sleeping in a way that could be deferred to yield their thread and enqueue | ||
160 | * themselves. | ||
161 | */ | ||
162 | static DECLARE_WAIT_QUEUE_HEAD(slow_work_queue_waits_for_occupation); | ||
163 | static DECLARE_WAIT_QUEUE_HEAD(vslow_work_queue_waits_for_occupation); | ||
111 | 164 | ||
112 | /* | 165 | /* |
113 | * The thread controls. A variable used to signal to the threads that they | 166 | * The thread controls. A variable used to signal to the threads that they |
@@ -126,6 +179,20 @@ static DECLARE_COMPLETION(slow_work_last_thread_exited); | |||
126 | static int slow_work_user_count; | 179 | static int slow_work_user_count; |
127 | static DEFINE_MUTEX(slow_work_user_lock); | 180 | static DEFINE_MUTEX(slow_work_user_lock); |
128 | 181 | ||
182 | static inline int slow_work_get_ref(struct slow_work *work) | ||
183 | { | ||
184 | if (work->ops->get_ref) | ||
185 | return work->ops->get_ref(work); | ||
186 | |||
187 | return 0; | ||
188 | } | ||
189 | |||
190 | static inline void slow_work_put_ref(struct slow_work *work) | ||
191 | { | ||
192 | if (work->ops->put_ref) | ||
193 | work->ops->put_ref(work); | ||
194 | } | ||
195 | |||
129 | /* | 196 | /* |
130 | * Calculate the maximum number of active threads in the pool that are | 197 | * Calculate the maximum number of active threads in the pool that are |
131 | * permitted to process very slow work items. | 198 | * permitted to process very slow work items. |
@@ -149,7 +216,7 @@ static unsigned slow_work_calc_vsmax(void) | |||
149 | * Attempt to execute stuff queued on a slow thread. Return true if we managed | 216 | * Attempt to execute stuff queued on a slow thread. Return true if we managed |
150 | * it, false if there was nothing to do. | 217 | * it, false if there was nothing to do. |
151 | */ | 218 | */ |
152 | static bool slow_work_execute(void) | 219 | static noinline bool slow_work_execute(int id) |
153 | { | 220 | { |
154 | struct slow_work *work = NULL; | 221 | struct slow_work *work = NULL; |
155 | unsigned vsmax; | 222 | unsigned vsmax; |
@@ -186,6 +253,13 @@ static bool slow_work_execute(void) | |||
186 | } else { | 253 | } else { |
187 | very_slow = false; /* avoid the compiler warning */ | 254 | very_slow = false; /* avoid the compiler warning */ |
188 | } | 255 | } |
256 | |||
257 | slow_work_set_thread_processing(id, work); | ||
258 | if (work) { | ||
259 | slow_work_mark_time(work); | ||
260 | slow_work_begin_exec(id, work); | ||
261 | } | ||
262 | |||
189 | spin_unlock_irq(&slow_work_queue_lock); | 263 | spin_unlock_irq(&slow_work_queue_lock); |
190 | 264 | ||
191 | if (!work) | 265 | if (!work) |
@@ -194,12 +268,19 @@ static bool slow_work_execute(void) | |||
194 | if (!test_and_clear_bit(SLOW_WORK_PENDING, &work->flags)) | 268 | if (!test_and_clear_bit(SLOW_WORK_PENDING, &work->flags)) |
195 | BUG(); | 269 | BUG(); |
196 | 270 | ||
197 | work->ops->execute(work); | 271 | /* don't execute if the work is in the process of being cancelled */ |
272 | if (!test_bit(SLOW_WORK_CANCELLING, &work->flags)) | ||
273 | work->ops->execute(work); | ||
198 | 274 | ||
199 | if (very_slow) | 275 | if (very_slow) |
200 | atomic_dec(&vslow_work_executing_count); | 276 | atomic_dec(&vslow_work_executing_count); |
201 | clear_bit_unlock(SLOW_WORK_EXECUTING, &work->flags); | 277 | clear_bit_unlock(SLOW_WORK_EXECUTING, &work->flags); |
202 | 278 | ||
279 | /* wake up anyone waiting for this work to be complete */ | ||
280 | wake_up_bit(&work->flags, SLOW_WORK_EXECUTING); | ||
281 | |||
282 | slow_work_end_exec(id, work); | ||
283 | |||
203 | /* if someone tried to enqueue the item whilst we were executing it, | 284 | /* if someone tried to enqueue the item whilst we were executing it, |
204 | * then it'll be left unenqueued to avoid multiple threads trying to | 285 | * then it'll be left unenqueued to avoid multiple threads trying to |
205 | * execute it simultaneously | 286 | * execute it simultaneously |
@@ -219,7 +300,10 @@ static bool slow_work_execute(void) | |||
219 | spin_unlock_irq(&slow_work_queue_lock); | 300 | spin_unlock_irq(&slow_work_queue_lock); |
220 | } | 301 | } |
221 | 302 | ||
222 | work->ops->put_ref(work); | 303 | /* sort out the race between module unloading and put_ref() */ |
304 | slow_work_put_ref(work); | ||
305 | slow_work_done_thread_processing(id, work); | ||
306 | |||
223 | return true; | 307 | return true; |
224 | 308 | ||
225 | auto_requeue: | 309 | auto_requeue: |
@@ -227,15 +311,61 @@ auto_requeue: | |||
227 | * - we transfer our ref on the item back to the appropriate queue | 311 | * - we transfer our ref on the item back to the appropriate queue |
228 | * - don't wake another thread up as we're awake already | 312 | * - don't wake another thread up as we're awake already |
229 | */ | 313 | */ |
314 | slow_work_mark_time(work); | ||
230 | if (test_bit(SLOW_WORK_VERY_SLOW, &work->flags)) | 315 | if (test_bit(SLOW_WORK_VERY_SLOW, &work->flags)) |
231 | list_add_tail(&work->link, &vslow_work_queue); | 316 | list_add_tail(&work->link, &vslow_work_queue); |
232 | else | 317 | else |
233 | list_add_tail(&work->link, &slow_work_queue); | 318 | list_add_tail(&work->link, &slow_work_queue); |
234 | spin_unlock_irq(&slow_work_queue_lock); | 319 | spin_unlock_irq(&slow_work_queue_lock); |
320 | slow_work_clear_thread_processing(id); | ||
235 | return true; | 321 | return true; |
236 | } | 322 | } |
237 | 323 | ||
238 | /** | 324 | /** |
325 | * slow_work_sleep_till_thread_needed - Sleep till thread needed by other work | ||
326 | * work: The work item under execution that wants to sleep | ||
327 | * _timeout: Scheduler sleep timeout | ||
328 | * | ||
329 | * Allow a requeueable work item to sleep on a slow-work processor thread until | ||
330 | * that thread is needed to do some other work or the sleep is interrupted by | ||
331 | * some other event. | ||
332 | * | ||
333 | * The caller must set up a wake up event before calling this and must have set | ||
334 | * the appropriate sleep mode (such as TASK_UNINTERRUPTIBLE) and tested its own | ||
335 | * condition before calling this function as no test is made here. | ||
336 | * | ||
337 | * False is returned if there is nothing on the queue; true is returned if the | ||
338 | * work item should be requeued | ||
339 | */ | ||
340 | bool slow_work_sleep_till_thread_needed(struct slow_work *work, | ||
341 | signed long *_timeout) | ||
342 | { | ||
343 | wait_queue_head_t *wfo_wq; | ||
344 | struct list_head *queue; | ||
345 | |||
346 | DEFINE_WAIT(wait); | ||
347 | |||
348 | if (test_bit(SLOW_WORK_VERY_SLOW, &work->flags)) { | ||
349 | wfo_wq = &vslow_work_queue_waits_for_occupation; | ||
350 | queue = &vslow_work_queue; | ||
351 | } else { | ||
352 | wfo_wq = &slow_work_queue_waits_for_occupation; | ||
353 | queue = &slow_work_queue; | ||
354 | } | ||
355 | |||
356 | if (!list_empty(queue)) | ||
357 | return true; | ||
358 | |||
359 | add_wait_queue_exclusive(wfo_wq, &wait); | ||
360 | if (list_empty(queue)) | ||
361 | *_timeout = schedule_timeout(*_timeout); | ||
362 | finish_wait(wfo_wq, &wait); | ||
363 | |||
364 | return !list_empty(queue); | ||
365 | } | ||
366 | EXPORT_SYMBOL(slow_work_sleep_till_thread_needed); | ||
367 | |||
368 | /** | ||
239 | * slow_work_enqueue - Schedule a slow work item for processing | 369 | * slow_work_enqueue - Schedule a slow work item for processing |
240 | * @work: The work item to queue | 370 | * @work: The work item to queue |
241 | * | 371 | * |
@@ -260,16 +390,22 @@ auto_requeue: | |||
260 | * allowed to pick items to execute. This ensures that very slow items won't | 390 | * allowed to pick items to execute. This ensures that very slow items won't |
261 | * overly block ones that are just ordinarily slow. | 391 | * overly block ones that are just ordinarily slow. |
262 | * | 392 | * |
263 | * Returns 0 if successful, -EAGAIN if not. | 393 | * Returns 0 if successful, -EAGAIN if not (or -ECANCELED if cancelled work is |
394 | * attempted queued) | ||
264 | */ | 395 | */ |
265 | int slow_work_enqueue(struct slow_work *work) | 396 | int slow_work_enqueue(struct slow_work *work) |
266 | { | 397 | { |
398 | wait_queue_head_t *wfo_wq; | ||
399 | struct list_head *queue; | ||
267 | unsigned long flags; | 400 | unsigned long flags; |
401 | int ret; | ||
402 | |||
403 | if (test_bit(SLOW_WORK_CANCELLING, &work->flags)) | ||
404 | return -ECANCELED; | ||
268 | 405 | ||
269 | BUG_ON(slow_work_user_count <= 0); | 406 | BUG_ON(slow_work_user_count <= 0); |
270 | BUG_ON(!work); | 407 | BUG_ON(!work); |
271 | BUG_ON(!work->ops); | 408 | BUG_ON(!work->ops); |
272 | BUG_ON(!work->ops->get_ref); | ||
273 | 409 | ||
274 | /* when honouring an enqueue request, we only promise that we will run | 410 | /* when honouring an enqueue request, we only promise that we will run |
275 | * the work function in the future; we do not promise to run it once | 411 | * the work function in the future; we do not promise to run it once |
@@ -280,8 +416,19 @@ int slow_work_enqueue(struct slow_work *work) | |||
280 | * maintaining our promise | 416 | * maintaining our promise |
281 | */ | 417 | */ |
282 | if (!test_and_set_bit_lock(SLOW_WORK_PENDING, &work->flags)) { | 418 | if (!test_and_set_bit_lock(SLOW_WORK_PENDING, &work->flags)) { |
419 | if (test_bit(SLOW_WORK_VERY_SLOW, &work->flags)) { | ||
420 | wfo_wq = &vslow_work_queue_waits_for_occupation; | ||
421 | queue = &vslow_work_queue; | ||
422 | } else { | ||
423 | wfo_wq = &slow_work_queue_waits_for_occupation; | ||
424 | queue = &slow_work_queue; | ||
425 | } | ||
426 | |||
283 | spin_lock_irqsave(&slow_work_queue_lock, flags); | 427 | spin_lock_irqsave(&slow_work_queue_lock, flags); |
284 | 428 | ||
429 | if (unlikely(test_bit(SLOW_WORK_CANCELLING, &work->flags))) | ||
430 | goto cancelled; | ||
431 | |||
285 | /* we promise that we will not attempt to execute the work | 432 | /* we promise that we will not attempt to execute the work |
286 | * function in more than one thread simultaneously | 433 | * function in more than one thread simultaneously |
287 | * | 434 | * |
@@ -299,25 +446,221 @@ int slow_work_enqueue(struct slow_work *work) | |||
299 | if (test_bit(SLOW_WORK_EXECUTING, &work->flags)) { | 446 | if (test_bit(SLOW_WORK_EXECUTING, &work->flags)) { |
300 | set_bit(SLOW_WORK_ENQ_DEFERRED, &work->flags); | 447 | set_bit(SLOW_WORK_ENQ_DEFERRED, &work->flags); |
301 | } else { | 448 | } else { |
302 | if (work->ops->get_ref(work) < 0) | 449 | ret = slow_work_get_ref(work); |
303 | goto cant_get_ref; | 450 | if (ret < 0) |
304 | if (test_bit(SLOW_WORK_VERY_SLOW, &work->flags)) | 451 | goto failed; |
305 | list_add_tail(&work->link, &vslow_work_queue); | 452 | slow_work_mark_time(work); |
306 | else | 453 | list_add_tail(&work->link, queue); |
307 | list_add_tail(&work->link, &slow_work_queue); | ||
308 | wake_up(&slow_work_thread_wq); | 454 | wake_up(&slow_work_thread_wq); |
455 | |||
456 | /* if someone who could be requeued is sleeping on a | ||
457 | * thread, then ask them to yield their thread */ | ||
458 | if (work->link.prev == queue) | ||
459 | wake_up(wfo_wq); | ||
309 | } | 460 | } |
310 | 461 | ||
311 | spin_unlock_irqrestore(&slow_work_queue_lock, flags); | 462 | spin_unlock_irqrestore(&slow_work_queue_lock, flags); |
312 | } | 463 | } |
313 | return 0; | 464 | return 0; |
314 | 465 | ||
315 | cant_get_ref: | 466 | cancelled: |
467 | ret = -ECANCELED; | ||
468 | failed: | ||
316 | spin_unlock_irqrestore(&slow_work_queue_lock, flags); | 469 | spin_unlock_irqrestore(&slow_work_queue_lock, flags); |
317 | return -EAGAIN; | 470 | return ret; |
318 | } | 471 | } |
319 | EXPORT_SYMBOL(slow_work_enqueue); | 472 | EXPORT_SYMBOL(slow_work_enqueue); |
320 | 473 | ||
474 | static int slow_work_wait(void *word) | ||
475 | { | ||
476 | schedule(); | ||
477 | return 0; | ||
478 | } | ||
479 | |||
480 | /** | ||
481 | * slow_work_cancel - Cancel a slow work item | ||
482 | * @work: The work item to cancel | ||
483 | * | ||
484 | * This function will cancel a previously enqueued work item. If we cannot | ||
485 | * cancel the work item, it is guarenteed to have run when this function | ||
486 | * returns. | ||
487 | */ | ||
488 | void slow_work_cancel(struct slow_work *work) | ||
489 | { | ||
490 | bool wait = true, put = false; | ||
491 | |||
492 | set_bit(SLOW_WORK_CANCELLING, &work->flags); | ||
493 | smp_mb(); | ||
494 | |||
495 | /* if the work item is a delayed work item with an active timer, we | ||
496 | * need to wait for the timer to finish _before_ getting the spinlock, | ||
497 | * lest we deadlock against the timer routine | ||
498 | * | ||
499 | * the timer routine will leave DELAYED set if it notices the | ||
500 | * CANCELLING flag in time | ||
501 | */ | ||
502 | if (test_bit(SLOW_WORK_DELAYED, &work->flags)) { | ||
503 | struct delayed_slow_work *dwork = | ||
504 | container_of(work, struct delayed_slow_work, work); | ||
505 | del_timer_sync(&dwork->timer); | ||
506 | } | ||
507 | |||
508 | spin_lock_irq(&slow_work_queue_lock); | ||
509 | |||
510 | if (test_bit(SLOW_WORK_DELAYED, &work->flags)) { | ||
511 | /* the timer routine aborted or never happened, so we are left | ||
512 | * holding the timer's reference on the item and should just | ||
513 | * drop the pending flag and wait for any ongoing execution to | ||
514 | * finish */ | ||
515 | struct delayed_slow_work *dwork = | ||
516 | container_of(work, struct delayed_slow_work, work); | ||
517 | |||
518 | BUG_ON(timer_pending(&dwork->timer)); | ||
519 | BUG_ON(!list_empty(&work->link)); | ||
520 | |||
521 | clear_bit(SLOW_WORK_DELAYED, &work->flags); | ||
522 | put = true; | ||
523 | clear_bit(SLOW_WORK_PENDING, &work->flags); | ||
524 | |||
525 | } else if (test_bit(SLOW_WORK_PENDING, &work->flags) && | ||
526 | !list_empty(&work->link)) { | ||
527 | /* the link in the pending queue holds a reference on the item | ||
528 | * that we will need to release */ | ||
529 | list_del_init(&work->link); | ||
530 | wait = false; | ||
531 | put = true; | ||
532 | clear_bit(SLOW_WORK_PENDING, &work->flags); | ||
533 | |||
534 | } else if (test_and_clear_bit(SLOW_WORK_ENQ_DEFERRED, &work->flags)) { | ||
535 | /* the executor is holding our only reference on the item, so | ||
536 | * we merely need to wait for it to finish executing */ | ||
537 | clear_bit(SLOW_WORK_PENDING, &work->flags); | ||
538 | } | ||
539 | |||
540 | spin_unlock_irq(&slow_work_queue_lock); | ||
541 | |||
542 | /* the EXECUTING flag is set by the executor whilst the spinlock is set | ||
543 | * and before the item is dequeued - so assuming the above doesn't | ||
544 | * actually dequeue it, simply waiting for the EXECUTING flag to be | ||
545 | * released here should be sufficient */ | ||
546 | if (wait) | ||
547 | wait_on_bit(&work->flags, SLOW_WORK_EXECUTING, slow_work_wait, | ||
548 | TASK_UNINTERRUPTIBLE); | ||
549 | |||
550 | clear_bit(SLOW_WORK_CANCELLING, &work->flags); | ||
551 | if (put) | ||
552 | slow_work_put_ref(work); | ||
553 | } | ||
554 | EXPORT_SYMBOL(slow_work_cancel); | ||
555 | |||
556 | /* | ||
557 | * Handle expiry of the delay timer, indicating that a delayed slow work item | ||
558 | * should now be queued if not cancelled | ||
559 | */ | ||
560 | static void delayed_slow_work_timer(unsigned long data) | ||
561 | { | ||
562 | wait_queue_head_t *wfo_wq; | ||
563 | struct list_head *queue; | ||
564 | struct slow_work *work = (struct slow_work *) data; | ||
565 | unsigned long flags; | ||
566 | bool queued = false, put = false, first = false; | ||
567 | |||
568 | if (test_bit(SLOW_WORK_VERY_SLOW, &work->flags)) { | ||
569 | wfo_wq = &vslow_work_queue_waits_for_occupation; | ||
570 | queue = &vslow_work_queue; | ||
571 | } else { | ||
572 | wfo_wq = &slow_work_queue_waits_for_occupation; | ||
573 | queue = &slow_work_queue; | ||
574 | } | ||
575 | |||
576 | spin_lock_irqsave(&slow_work_queue_lock, flags); | ||
577 | if (likely(!test_bit(SLOW_WORK_CANCELLING, &work->flags))) { | ||
578 | clear_bit(SLOW_WORK_DELAYED, &work->flags); | ||
579 | |||
580 | if (test_bit(SLOW_WORK_EXECUTING, &work->flags)) { | ||
581 | /* we discard the reference the timer was holding in | ||
582 | * favour of the one the executor holds */ | ||
583 | set_bit(SLOW_WORK_ENQ_DEFERRED, &work->flags); | ||
584 | put = true; | ||
585 | } else { | ||
586 | slow_work_mark_time(work); | ||
587 | list_add_tail(&work->link, queue); | ||
588 | queued = true; | ||
589 | if (work->link.prev == queue) | ||
590 | first = true; | ||
591 | } | ||
592 | } | ||
593 | |||
594 | spin_unlock_irqrestore(&slow_work_queue_lock, flags); | ||
595 | if (put) | ||
596 | slow_work_put_ref(work); | ||
597 | if (first) | ||
598 | wake_up(wfo_wq); | ||
599 | if (queued) | ||
600 | wake_up(&slow_work_thread_wq); | ||
601 | } | ||
602 | |||
603 | /** | ||
604 | * delayed_slow_work_enqueue - Schedule a delayed slow work item for processing | ||
605 | * @dwork: The delayed work item to queue | ||
606 | * @delay: When to start executing the work, in jiffies from now | ||
607 | * | ||
608 | * This is similar to slow_work_enqueue(), but it adds a delay before the work | ||
609 | * is actually queued for processing. | ||
610 | * | ||
611 | * The item can have delayed processing requested on it whilst it is being | ||
612 | * executed. The delay will begin immediately, and if it expires before the | ||
613 | * item finishes executing, the item will be placed back on the queue when it | ||
614 | * has done executing. | ||
615 | */ | ||
616 | int delayed_slow_work_enqueue(struct delayed_slow_work *dwork, | ||
617 | unsigned long delay) | ||
618 | { | ||
619 | struct slow_work *work = &dwork->work; | ||
620 | unsigned long flags; | ||
621 | int ret; | ||
622 | |||
623 | if (delay == 0) | ||
624 | return slow_work_enqueue(&dwork->work); | ||
625 | |||
626 | BUG_ON(slow_work_user_count <= 0); | ||
627 | BUG_ON(!work); | ||
628 | BUG_ON(!work->ops); | ||
629 | |||
630 | if (test_bit(SLOW_WORK_CANCELLING, &work->flags)) | ||
631 | return -ECANCELED; | ||
632 | |||
633 | if (!test_and_set_bit_lock(SLOW_WORK_PENDING, &work->flags)) { | ||
634 | spin_lock_irqsave(&slow_work_queue_lock, flags); | ||
635 | |||
636 | if (test_bit(SLOW_WORK_CANCELLING, &work->flags)) | ||
637 | goto cancelled; | ||
638 | |||
639 | /* the timer holds a reference whilst it is pending */ | ||
640 | ret = work->ops->get_ref(work); | ||
641 | if (ret < 0) | ||
642 | goto cant_get_ref; | ||
643 | |||
644 | if (test_and_set_bit(SLOW_WORK_DELAYED, &work->flags)) | ||
645 | BUG(); | ||
646 | dwork->timer.expires = jiffies + delay; | ||
647 | dwork->timer.data = (unsigned long) work; | ||
648 | dwork->timer.function = delayed_slow_work_timer; | ||
649 | add_timer(&dwork->timer); | ||
650 | |||
651 | spin_unlock_irqrestore(&slow_work_queue_lock, flags); | ||
652 | } | ||
653 | |||
654 | return 0; | ||
655 | |||
656 | cancelled: | ||
657 | ret = -ECANCELED; | ||
658 | cant_get_ref: | ||
659 | spin_unlock_irqrestore(&slow_work_queue_lock, flags); | ||
660 | return ret; | ||
661 | } | ||
662 | EXPORT_SYMBOL(delayed_slow_work_enqueue); | ||
663 | |||
321 | /* | 664 | /* |
322 | * Schedule a cull of the thread pool at some time in the near future | 665 | * Schedule a cull of the thread pool at some time in the near future |
323 | */ | 666 | */ |
@@ -368,13 +711,23 @@ static inline bool slow_work_available(int vsmax) | |||
368 | */ | 711 | */ |
369 | static int slow_work_thread(void *_data) | 712 | static int slow_work_thread(void *_data) |
370 | { | 713 | { |
371 | int vsmax; | 714 | int vsmax, id; |
372 | 715 | ||
373 | DEFINE_WAIT(wait); | 716 | DEFINE_WAIT(wait); |
374 | 717 | ||
375 | set_freezable(); | 718 | set_freezable(); |
376 | set_user_nice(current, -5); | 719 | set_user_nice(current, -5); |
377 | 720 | ||
721 | /* allocate ourselves an ID */ | ||
722 | spin_lock_irq(&slow_work_queue_lock); | ||
723 | id = find_first_zero_bit(slow_work_ids, SLOW_WORK_THREAD_LIMIT); | ||
724 | BUG_ON(id < 0 || id >= SLOW_WORK_THREAD_LIMIT); | ||
725 | __set_bit(id, slow_work_ids); | ||
726 | slow_work_set_thread_pid(id, current->pid); | ||
727 | spin_unlock_irq(&slow_work_queue_lock); | ||
728 | |||
729 | sprintf(current->comm, "kslowd%03u", id); | ||
730 | |||
378 | for (;;) { | 731 | for (;;) { |
379 | vsmax = vslow_work_proportion; | 732 | vsmax = vslow_work_proportion; |
380 | vsmax *= atomic_read(&slow_work_thread_count); | 733 | vsmax *= atomic_read(&slow_work_thread_count); |
@@ -395,7 +748,7 @@ static int slow_work_thread(void *_data) | |||
395 | vsmax *= atomic_read(&slow_work_thread_count); | 748 | vsmax *= atomic_read(&slow_work_thread_count); |
396 | vsmax /= 100; | 749 | vsmax /= 100; |
397 | 750 | ||
398 | if (slow_work_available(vsmax) && slow_work_execute()) { | 751 | if (slow_work_available(vsmax) && slow_work_execute(id)) { |
399 | cond_resched(); | 752 | cond_resched(); |
400 | if (list_empty(&slow_work_queue) && | 753 | if (list_empty(&slow_work_queue) && |
401 | list_empty(&vslow_work_queue) && | 754 | list_empty(&vslow_work_queue) && |
@@ -412,6 +765,11 @@ static int slow_work_thread(void *_data) | |||
412 | break; | 765 | break; |
413 | } | 766 | } |
414 | 767 | ||
768 | spin_lock_irq(&slow_work_queue_lock); | ||
769 | slow_work_set_thread_pid(id, 0); | ||
770 | __clear_bit(id, slow_work_ids); | ||
771 | spin_unlock_irq(&slow_work_queue_lock); | ||
772 | |||
415 | if (atomic_dec_and_test(&slow_work_thread_count)) | 773 | if (atomic_dec_and_test(&slow_work_thread_count)) |
416 | complete_and_exit(&slow_work_last_thread_exited, 0); | 774 | complete_and_exit(&slow_work_last_thread_exited, 0); |
417 | return 0; | 775 | return 0; |
@@ -427,21 +785,6 @@ static void slow_work_cull_timeout(unsigned long data) | |||
427 | } | 785 | } |
428 | 786 | ||
429 | /* | 787 | /* |
430 | * Get a reference on slow work thread starter | ||
431 | */ | ||
432 | static int slow_work_new_thread_get_ref(struct slow_work *work) | ||
433 | { | ||
434 | return 0; | ||
435 | } | ||
436 | |||
437 | /* | ||
438 | * Drop a reference on slow work thread starter | ||
439 | */ | ||
440 | static void slow_work_new_thread_put_ref(struct slow_work *work) | ||
441 | { | ||
442 | } | ||
443 | |||
444 | /* | ||
445 | * Start a new slow work thread | 788 | * Start a new slow work thread |
446 | */ | 789 | */ |
447 | static void slow_work_new_thread_execute(struct slow_work *work) | 790 | static void slow_work_new_thread_execute(struct slow_work *work) |
@@ -475,9 +818,11 @@ static void slow_work_new_thread_execute(struct slow_work *work) | |||
475 | } | 818 | } |
476 | 819 | ||
477 | static const struct slow_work_ops slow_work_new_thread_ops = { | 820 | static const struct slow_work_ops slow_work_new_thread_ops = { |
478 | .get_ref = slow_work_new_thread_get_ref, | 821 | .owner = THIS_MODULE, |
479 | .put_ref = slow_work_new_thread_put_ref, | ||
480 | .execute = slow_work_new_thread_execute, | 822 | .execute = slow_work_new_thread_execute, |
823 | #ifdef CONFIG_SLOW_WORK_DEBUG | ||
824 | .desc = slow_work_new_thread_desc, | ||
825 | #endif | ||
481 | }; | 826 | }; |
482 | 827 | ||
483 | /* | 828 | /* |
@@ -546,12 +891,13 @@ static int slow_work_max_threads_sysctl(struct ctl_table *table, int write, | |||
546 | 891 | ||
547 | /** | 892 | /** |
548 | * slow_work_register_user - Register a user of the facility | 893 | * slow_work_register_user - Register a user of the facility |
894 | * @module: The module about to make use of the facility | ||
549 | * | 895 | * |
550 | * Register a user of the facility, starting up the initial threads if there | 896 | * Register a user of the facility, starting up the initial threads if there |
551 | * aren't any other users at this point. This will return 0 if successful, or | 897 | * aren't any other users at this point. This will return 0 if successful, or |
552 | * an error if not. | 898 | * an error if not. |
553 | */ | 899 | */ |
554 | int slow_work_register_user(void) | 900 | int slow_work_register_user(struct module *module) |
555 | { | 901 | { |
556 | struct task_struct *p; | 902 | struct task_struct *p; |
557 | int loop; | 903 | int loop; |
@@ -598,14 +944,81 @@ error: | |||
598 | } | 944 | } |
599 | EXPORT_SYMBOL(slow_work_register_user); | 945 | EXPORT_SYMBOL(slow_work_register_user); |
600 | 946 | ||
947 | /* | ||
948 | * wait for all outstanding items from the calling module to complete | ||
949 | * - note that more items may be queued whilst we're waiting | ||
950 | */ | ||
951 | static void slow_work_wait_for_items(struct module *module) | ||
952 | { | ||
953 | #ifdef CONFIG_MODULES | ||
954 | DECLARE_WAITQUEUE(myself, current); | ||
955 | struct slow_work *work; | ||
956 | int loop; | ||
957 | |||
958 | mutex_lock(&slow_work_unreg_sync_lock); | ||
959 | add_wait_queue(&slow_work_unreg_wq, &myself); | ||
960 | |||
961 | for (;;) { | ||
962 | spin_lock_irq(&slow_work_queue_lock); | ||
963 | |||
964 | /* first of all, we wait for the last queued item in each list | ||
965 | * to be processed */ | ||
966 | list_for_each_entry_reverse(work, &vslow_work_queue, link) { | ||
967 | if (work->owner == module) { | ||
968 | set_current_state(TASK_UNINTERRUPTIBLE); | ||
969 | slow_work_unreg_work_item = work; | ||
970 | goto do_wait; | ||
971 | } | ||
972 | } | ||
973 | list_for_each_entry_reverse(work, &slow_work_queue, link) { | ||
974 | if (work->owner == module) { | ||
975 | set_current_state(TASK_UNINTERRUPTIBLE); | ||
976 | slow_work_unreg_work_item = work; | ||
977 | goto do_wait; | ||
978 | } | ||
979 | } | ||
980 | |||
981 | /* then we wait for the items being processed to finish */ | ||
982 | slow_work_unreg_module = module; | ||
983 | smp_mb(); | ||
984 | for (loop = 0; loop < SLOW_WORK_THREAD_LIMIT; loop++) { | ||
985 | if (slow_work_thread_processing[loop] == module) | ||
986 | goto do_wait; | ||
987 | } | ||
988 | spin_unlock_irq(&slow_work_queue_lock); | ||
989 | break; /* okay, we're done */ | ||
990 | |||
991 | do_wait: | ||
992 | spin_unlock_irq(&slow_work_queue_lock); | ||
993 | schedule(); | ||
994 | slow_work_unreg_work_item = NULL; | ||
995 | slow_work_unreg_module = NULL; | ||
996 | } | ||
997 | |||
998 | remove_wait_queue(&slow_work_unreg_wq, &myself); | ||
999 | mutex_unlock(&slow_work_unreg_sync_lock); | ||
1000 | #endif /* CONFIG_MODULES */ | ||
1001 | } | ||
1002 | |||
601 | /** | 1003 | /** |
602 | * slow_work_unregister_user - Unregister a user of the facility | 1004 | * slow_work_unregister_user - Unregister a user of the facility |
1005 | * @module: The module whose items should be cleared | ||
603 | * | 1006 | * |
604 | * Unregister a user of the facility, killing all the threads if this was the | 1007 | * Unregister a user of the facility, killing all the threads if this was the |
605 | * last one. | 1008 | * last one. |
1009 | * | ||
1010 | * This waits for all the work items belonging to the nominated module to go | ||
1011 | * away before proceeding. | ||
606 | */ | 1012 | */ |
607 | void slow_work_unregister_user(void) | 1013 | void slow_work_unregister_user(struct module *module) |
608 | { | 1014 | { |
1015 | /* first of all, wait for all outstanding items from the calling module | ||
1016 | * to complete */ | ||
1017 | if (module) | ||
1018 | slow_work_wait_for_items(module); | ||
1019 | |||
1020 | /* then we can actually go about shutting down the facility if need | ||
1021 | * be */ | ||
609 | mutex_lock(&slow_work_user_lock); | 1022 | mutex_lock(&slow_work_user_lock); |
610 | 1023 | ||
611 | BUG_ON(slow_work_user_count <= 0); | 1024 | BUG_ON(slow_work_user_count <= 0); |
@@ -639,6 +1052,16 @@ static int __init init_slow_work(void) | |||
639 | if (slow_work_max_max_threads < nr_cpus * 2) | 1052 | if (slow_work_max_max_threads < nr_cpus * 2) |
640 | slow_work_max_max_threads = nr_cpus * 2; | 1053 | slow_work_max_max_threads = nr_cpus * 2; |
641 | #endif | 1054 | #endif |
1055 | #ifdef CONFIG_SLOW_WORK_DEBUG | ||
1056 | { | ||
1057 | struct dentry *dbdir; | ||
1058 | |||
1059 | dbdir = debugfs_create_dir("slow_work", NULL); | ||
1060 | if (dbdir && !IS_ERR(dbdir)) | ||
1061 | debugfs_create_file("runqueue", S_IFREG | 0400, dbdir, | ||
1062 | NULL, &slow_work_runqueue_fops); | ||
1063 | } | ||
1064 | #endif | ||
642 | return 0; | 1065 | return 0; |
643 | } | 1066 | } |
644 | 1067 | ||
diff --git a/kernel/slow-work.h b/kernel/slow-work.h new file mode 100644 index 000000000000..321f3c59d732 --- /dev/null +++ b/kernel/slow-work.h | |||
@@ -0,0 +1,72 @@ | |||
1 | /* Slow work private definitions | ||
2 | * | ||
3 | * Copyright (C) 2009 Red Hat, Inc. All Rights Reserved. | ||
4 | * Written by David Howells (dhowells@redhat.com) | ||
5 | * | ||
6 | * This program is free software; you can redistribute it and/or | ||
7 | * modify it under the terms of the GNU General Public Licence | ||
8 | * as published by the Free Software Foundation; either version | ||
9 | * 2 of the Licence, or (at your option) any later version. | ||
10 | */ | ||
11 | |||
12 | #define SLOW_WORK_CULL_TIMEOUT (5 * HZ) /* cull threads 5s after running out of | ||
13 | * things to do */ | ||
14 | #define SLOW_WORK_OOM_TIMEOUT (5 * HZ) /* can't start new threads for 5s after | ||
15 | * OOM */ | ||
16 | |||
17 | #define SLOW_WORK_THREAD_LIMIT 255 /* abs maximum number of slow-work threads */ | ||
18 | |||
19 | /* | ||
20 | * slow-work.c | ||
21 | */ | ||
22 | #ifdef CONFIG_SLOW_WORK_DEBUG | ||
23 | extern struct slow_work *slow_work_execs[]; | ||
24 | extern pid_t slow_work_pids[]; | ||
25 | extern rwlock_t slow_work_execs_lock; | ||
26 | #endif | ||
27 | |||
28 | extern struct list_head slow_work_queue; | ||
29 | extern struct list_head vslow_work_queue; | ||
30 | extern spinlock_t slow_work_queue_lock; | ||
31 | |||
32 | /* | ||
33 | * slow-work-debugfs.c | ||
34 | */ | ||
35 | #ifdef CONFIG_SLOW_WORK_DEBUG | ||
36 | extern const struct file_operations slow_work_runqueue_fops; | ||
37 | |||
38 | extern void slow_work_new_thread_desc(struct slow_work *, struct seq_file *); | ||
39 | #endif | ||
40 | |||
41 | /* | ||
42 | * Helper functions | ||
43 | */ | ||
44 | static inline void slow_work_set_thread_pid(int id, pid_t pid) | ||
45 | { | ||
46 | #ifdef CONFIG_SLOW_WORK_PROC | ||
47 | slow_work_pids[id] = pid; | ||
48 | #endif | ||
49 | } | ||
50 | |||
51 | static inline void slow_work_mark_time(struct slow_work *work) | ||
52 | { | ||
53 | #ifdef CONFIG_SLOW_WORK_PROC | ||
54 | work->mark = CURRENT_TIME; | ||
55 | #endif | ||
56 | } | ||
57 | |||
58 | static inline void slow_work_begin_exec(int id, struct slow_work *work) | ||
59 | { | ||
60 | #ifdef CONFIG_SLOW_WORK_PROC | ||
61 | slow_work_execs[id] = work; | ||
62 | #endif | ||
63 | } | ||
64 | |||
65 | static inline void slow_work_end_exec(int id, struct slow_work *work) | ||
66 | { | ||
67 | #ifdef CONFIG_SLOW_WORK_PROC | ||
68 | write_lock(&slow_work_execs_lock); | ||
69 | slow_work_execs[id] = NULL; | ||
70 | write_unlock(&slow_work_execs_lock); | ||
71 | #endif | ||
72 | } | ||
diff --git a/kernel/smp.c b/kernel/smp.c index c9d1c7835c2f..a8c76069cf50 100644 --- a/kernel/smp.c +++ b/kernel/smp.c | |||
@@ -265,9 +265,7 @@ static DEFINE_PER_CPU(struct call_single_data, csd_data); | |||
265 | * @info: An arbitrary pointer to pass to the function. | 265 | * @info: An arbitrary pointer to pass to the function. |
266 | * @wait: If true, wait until function has completed on other CPUs. | 266 | * @wait: If true, wait until function has completed on other CPUs. |
267 | * | 267 | * |
268 | * Returns 0 on success, else a negative status code. Note that @wait | 268 | * Returns 0 on success, else a negative status code. |
269 | * will be implicitly turned on in case of allocation failures, since | ||
270 | * we fall back to on-stack allocation. | ||
271 | */ | 269 | */ |
272 | int smp_call_function_single(int cpu, void (*func) (void *info), void *info, | 270 | int smp_call_function_single(int cpu, void (*func) (void *info), void *info, |
273 | int wait) | 271 | int wait) |
@@ -321,6 +319,51 @@ int smp_call_function_single(int cpu, void (*func) (void *info), void *info, | |||
321 | } | 319 | } |
322 | EXPORT_SYMBOL(smp_call_function_single); | 320 | EXPORT_SYMBOL(smp_call_function_single); |
323 | 321 | ||
322 | /* | ||
323 | * smp_call_function_any - Run a function on any of the given cpus | ||
324 | * @mask: The mask of cpus it can run on. | ||
325 | * @func: The function to run. This must be fast and non-blocking. | ||
326 | * @info: An arbitrary pointer to pass to the function. | ||
327 | * @wait: If true, wait until function has completed. | ||
328 | * | ||
329 | * Returns 0 on success, else a negative status code (if no cpus were online). | ||
330 | * Note that @wait will be implicitly turned on in case of allocation failures, | ||
331 | * since we fall back to on-stack allocation. | ||
332 | * | ||
333 | * Selection preference: | ||
334 | * 1) current cpu if in @mask | ||
335 | * 2) any cpu of current node if in @mask | ||
336 | * 3) any other online cpu in @mask | ||
337 | */ | ||
338 | int smp_call_function_any(const struct cpumask *mask, | ||
339 | void (*func)(void *info), void *info, int wait) | ||
340 | { | ||
341 | unsigned int cpu; | ||
342 | const struct cpumask *nodemask; | ||
343 | int ret; | ||
344 | |||
345 | /* Try for same CPU (cheapest) */ | ||
346 | cpu = get_cpu(); | ||
347 | if (cpumask_test_cpu(cpu, mask)) | ||
348 | goto call; | ||
349 | |||
350 | /* Try for same node. */ | ||
351 | nodemask = cpumask_of_node(cpu); | ||
352 | for (cpu = cpumask_first_and(nodemask, mask); cpu < nr_cpu_ids; | ||
353 | cpu = cpumask_next_and(cpu, nodemask, mask)) { | ||
354 | if (cpu_online(cpu)) | ||
355 | goto call; | ||
356 | } | ||
357 | |||
358 | /* Any online will do: smp_call_function_single handles nr_cpu_ids. */ | ||
359 | cpu = cpumask_any_and(mask, cpu_online_mask); | ||
360 | call: | ||
361 | ret = smp_call_function_single(cpu, func, info, wait); | ||
362 | put_cpu(); | ||
363 | return ret; | ||
364 | } | ||
365 | EXPORT_SYMBOL_GPL(smp_call_function_any); | ||
366 | |||
324 | /** | 367 | /** |
325 | * __smp_call_function_single(): Run a function on another CPU | 368 | * __smp_call_function_single(): Run a function on another CPU |
326 | * @cpu: The CPU to run on. | 369 | * @cpu: The CPU to run on. |
@@ -355,9 +398,7 @@ void __smp_call_function_single(int cpu, struct call_single_data *data, | |||
355 | * @wait: If true, wait (atomically) until function has completed | 398 | * @wait: If true, wait (atomically) until function has completed |
356 | * on other CPUs. | 399 | * on other CPUs. |
357 | * | 400 | * |
358 | * If @wait is true, then returns once @func has returned. Note that @wait | 401 | * If @wait is true, then returns once @func has returned. |
359 | * will be implicitly turned on in case of allocation failures, since | ||
360 | * we fall back to on-stack allocation. | ||
361 | * | 402 | * |
362 | * You must not call this function with disabled interrupts or from a | 403 | * You must not call this function with disabled interrupts or from a |
363 | * hardware interrupt handler or from a bottom half handler. Preemption | 404 | * hardware interrupt handler or from a bottom half handler. Preemption |
@@ -443,8 +484,7 @@ EXPORT_SYMBOL(smp_call_function_many); | |||
443 | * Returns 0. | 484 | * Returns 0. |
444 | * | 485 | * |
445 | * If @wait is true, then returns once @func has returned; otherwise | 486 | * If @wait is true, then returns once @func has returned; otherwise |
446 | * it returns just before the target cpu calls @func. In case of allocation | 487 | * it returns just before the target cpu calls @func. |
447 | * failure, @wait will be implicitly turned on. | ||
448 | * | 488 | * |
449 | * You must not call this function with disabled interrupts or from a | 489 | * You must not call this function with disabled interrupts or from a |
450 | * hardware interrupt handler or from a bottom half handler. | 490 | * hardware interrupt handler or from a bottom half handler. |
diff --git a/kernel/softirq.c b/kernel/softirq.c index f8749e5216e0..21939d9e830e 100644 --- a/kernel/softirq.c +++ b/kernel/softirq.c | |||
@@ -302,9 +302,9 @@ void irq_exit(void) | |||
302 | if (!in_interrupt() && local_softirq_pending()) | 302 | if (!in_interrupt() && local_softirq_pending()) |
303 | invoke_softirq(); | 303 | invoke_softirq(); |
304 | 304 | ||
305 | rcu_irq_exit(); | ||
305 | #ifdef CONFIG_NO_HZ | 306 | #ifdef CONFIG_NO_HZ |
306 | /* Make sure that timer wheel updates are propagated */ | 307 | /* Make sure that timer wheel updates are propagated */ |
307 | rcu_irq_exit(); | ||
308 | if (idle_cpu(smp_processor_id()) && !in_interrupt() && !need_resched()) | 308 | if (idle_cpu(smp_processor_id()) && !in_interrupt() && !need_resched()) |
309 | tick_nohz_stop_sched_tick(0); | 309 | tick_nohz_stop_sched_tick(0); |
310 | #endif | 310 | #endif |
diff --git a/kernel/spinlock.c b/kernel/spinlock.c index 5ddab730cb2f..41e042219ff6 100644 --- a/kernel/spinlock.c +++ b/kernel/spinlock.c | |||
@@ -21,145 +21,28 @@ | |||
21 | #include <linux/debug_locks.h> | 21 | #include <linux/debug_locks.h> |
22 | #include <linux/module.h> | 22 | #include <linux/module.h> |
23 | 23 | ||
24 | #ifndef _spin_trylock | ||
25 | int __lockfunc _spin_trylock(spinlock_t *lock) | ||
26 | { | ||
27 | return __spin_trylock(lock); | ||
28 | } | ||
29 | EXPORT_SYMBOL(_spin_trylock); | ||
30 | #endif | ||
31 | |||
32 | #ifndef _read_trylock | ||
33 | int __lockfunc _read_trylock(rwlock_t *lock) | ||
34 | { | ||
35 | return __read_trylock(lock); | ||
36 | } | ||
37 | EXPORT_SYMBOL(_read_trylock); | ||
38 | #endif | ||
39 | |||
40 | #ifndef _write_trylock | ||
41 | int __lockfunc _write_trylock(rwlock_t *lock) | ||
42 | { | ||
43 | return __write_trylock(lock); | ||
44 | } | ||
45 | EXPORT_SYMBOL(_write_trylock); | ||
46 | #endif | ||
47 | |||
48 | /* | 24 | /* |
49 | * If lockdep is enabled then we use the non-preemption spin-ops | 25 | * If lockdep is enabled then we use the non-preemption spin-ops |
50 | * even on CONFIG_PREEMPT, because lockdep assumes that interrupts are | 26 | * even on CONFIG_PREEMPT, because lockdep assumes that interrupts are |
51 | * not re-enabled during lock-acquire (which the preempt-spin-ops do): | 27 | * not re-enabled during lock-acquire (which the preempt-spin-ops do): |
52 | */ | 28 | */ |
53 | #if !defined(CONFIG_GENERIC_LOCKBREAK) || defined(CONFIG_DEBUG_LOCK_ALLOC) | 29 | #if !defined(CONFIG_GENERIC_LOCKBREAK) || defined(CONFIG_DEBUG_LOCK_ALLOC) |
54 | |||
55 | #ifndef _read_lock | ||
56 | void __lockfunc _read_lock(rwlock_t *lock) | ||
57 | { | ||
58 | __read_lock(lock); | ||
59 | } | ||
60 | EXPORT_SYMBOL(_read_lock); | ||
61 | #endif | ||
62 | |||
63 | #ifndef _spin_lock_irqsave | ||
64 | unsigned long __lockfunc _spin_lock_irqsave(spinlock_t *lock) | ||
65 | { | ||
66 | return __spin_lock_irqsave(lock); | ||
67 | } | ||
68 | EXPORT_SYMBOL(_spin_lock_irqsave); | ||
69 | #endif | ||
70 | |||
71 | #ifndef _spin_lock_irq | ||
72 | void __lockfunc _spin_lock_irq(spinlock_t *lock) | ||
73 | { | ||
74 | __spin_lock_irq(lock); | ||
75 | } | ||
76 | EXPORT_SYMBOL(_spin_lock_irq); | ||
77 | #endif | ||
78 | |||
79 | #ifndef _spin_lock_bh | ||
80 | void __lockfunc _spin_lock_bh(spinlock_t *lock) | ||
81 | { | ||
82 | __spin_lock_bh(lock); | ||
83 | } | ||
84 | EXPORT_SYMBOL(_spin_lock_bh); | ||
85 | #endif | ||
86 | |||
87 | #ifndef _read_lock_irqsave | ||
88 | unsigned long __lockfunc _read_lock_irqsave(rwlock_t *lock) | ||
89 | { | ||
90 | return __read_lock_irqsave(lock); | ||
91 | } | ||
92 | EXPORT_SYMBOL(_read_lock_irqsave); | ||
93 | #endif | ||
94 | |||
95 | #ifndef _read_lock_irq | ||
96 | void __lockfunc _read_lock_irq(rwlock_t *lock) | ||
97 | { | ||
98 | __read_lock_irq(lock); | ||
99 | } | ||
100 | EXPORT_SYMBOL(_read_lock_irq); | ||
101 | #endif | ||
102 | |||
103 | #ifndef _read_lock_bh | ||
104 | void __lockfunc _read_lock_bh(rwlock_t *lock) | ||
105 | { | ||
106 | __read_lock_bh(lock); | ||
107 | } | ||
108 | EXPORT_SYMBOL(_read_lock_bh); | ||
109 | #endif | ||
110 | |||
111 | #ifndef _write_lock_irqsave | ||
112 | unsigned long __lockfunc _write_lock_irqsave(rwlock_t *lock) | ||
113 | { | ||
114 | return __write_lock_irqsave(lock); | ||
115 | } | ||
116 | EXPORT_SYMBOL(_write_lock_irqsave); | ||
117 | #endif | ||
118 | |||
119 | #ifndef _write_lock_irq | ||
120 | void __lockfunc _write_lock_irq(rwlock_t *lock) | ||
121 | { | ||
122 | __write_lock_irq(lock); | ||
123 | } | ||
124 | EXPORT_SYMBOL(_write_lock_irq); | ||
125 | #endif | ||
126 | |||
127 | #ifndef _write_lock_bh | ||
128 | void __lockfunc _write_lock_bh(rwlock_t *lock) | ||
129 | { | ||
130 | __write_lock_bh(lock); | ||
131 | } | ||
132 | EXPORT_SYMBOL(_write_lock_bh); | ||
133 | #endif | ||
134 | |||
135 | #ifndef _spin_lock | ||
136 | void __lockfunc _spin_lock(spinlock_t *lock) | ||
137 | { | ||
138 | __spin_lock(lock); | ||
139 | } | ||
140 | EXPORT_SYMBOL(_spin_lock); | ||
141 | #endif | ||
142 | |||
143 | #ifndef _write_lock | ||
144 | void __lockfunc _write_lock(rwlock_t *lock) | ||
145 | { | ||
146 | __write_lock(lock); | ||
147 | } | ||
148 | EXPORT_SYMBOL(_write_lock); | ||
149 | #endif | ||
150 | |||
151 | #else /* CONFIG_PREEMPT: */ | ||
152 | |||
153 | /* | 30 | /* |
31 | * The __lock_function inlines are taken from | ||
32 | * include/linux/spinlock_api_smp.h | ||
33 | */ | ||
34 | #else | ||
35 | /* | ||
36 | * We build the __lock_function inlines here. They are too large for | ||
37 | * inlining all over the place, but here is only one user per function | ||
38 | * which embedds them into the calling _lock_function below. | ||
39 | * | ||
154 | * This could be a long-held lock. We both prepare to spin for a long | 40 | * This could be a long-held lock. We both prepare to spin for a long |
155 | * time (making _this_ CPU preemptable if possible), and we also signal | 41 | * time (making _this_ CPU preemptable if possible), and we also signal |
156 | * towards that other CPU that it should break the lock ASAP. | 42 | * towards that other CPU that it should break the lock ASAP. |
157 | * | ||
158 | * (We do this in a function because inlining it would be excessive.) | ||
159 | */ | 43 | */ |
160 | |||
161 | #define BUILD_LOCK_OPS(op, locktype) \ | 44 | #define BUILD_LOCK_OPS(op, locktype) \ |
162 | void __lockfunc _##op##_lock(locktype##_t *lock) \ | 45 | void __lockfunc __##op##_lock(locktype##_t *lock) \ |
163 | { \ | 46 | { \ |
164 | for (;;) { \ | 47 | for (;;) { \ |
165 | preempt_disable(); \ | 48 | preempt_disable(); \ |
@@ -175,9 +58,7 @@ void __lockfunc _##op##_lock(locktype##_t *lock) \ | |||
175 | (lock)->break_lock = 0; \ | 58 | (lock)->break_lock = 0; \ |
176 | } \ | 59 | } \ |
177 | \ | 60 | \ |
178 | EXPORT_SYMBOL(_##op##_lock); \ | 61 | unsigned long __lockfunc __##op##_lock_irqsave(locktype##_t *lock) \ |
179 | \ | ||
180 | unsigned long __lockfunc _##op##_lock_irqsave(locktype##_t *lock) \ | ||
181 | { \ | 62 | { \ |
182 | unsigned long flags; \ | 63 | unsigned long flags; \ |
183 | \ | 64 | \ |
@@ -198,16 +79,12 @@ unsigned long __lockfunc _##op##_lock_irqsave(locktype##_t *lock) \ | |||
198 | return flags; \ | 79 | return flags; \ |
199 | } \ | 80 | } \ |
200 | \ | 81 | \ |
201 | EXPORT_SYMBOL(_##op##_lock_irqsave); \ | 82 | void __lockfunc __##op##_lock_irq(locktype##_t *lock) \ |
202 | \ | ||
203 | void __lockfunc _##op##_lock_irq(locktype##_t *lock) \ | ||
204 | { \ | 83 | { \ |
205 | _##op##_lock_irqsave(lock); \ | 84 | _##op##_lock_irqsave(lock); \ |
206 | } \ | 85 | } \ |
207 | \ | 86 | \ |
208 | EXPORT_SYMBOL(_##op##_lock_irq); \ | 87 | void __lockfunc __##op##_lock_bh(locktype##_t *lock) \ |
209 | \ | ||
210 | void __lockfunc _##op##_lock_bh(locktype##_t *lock) \ | ||
211 | { \ | 88 | { \ |
212 | unsigned long flags; \ | 89 | unsigned long flags; \ |
213 | \ | 90 | \ |
@@ -220,23 +97,21 @@ void __lockfunc _##op##_lock_bh(locktype##_t *lock) \ | |||
220 | local_bh_disable(); \ | 97 | local_bh_disable(); \ |
221 | local_irq_restore(flags); \ | 98 | local_irq_restore(flags); \ |
222 | } \ | 99 | } \ |
223 | \ | ||
224 | EXPORT_SYMBOL(_##op##_lock_bh) | ||
225 | 100 | ||
226 | /* | 101 | /* |
227 | * Build preemption-friendly versions of the following | 102 | * Build preemption-friendly versions of the following |
228 | * lock-spinning functions: | 103 | * lock-spinning functions: |
229 | * | 104 | * |
230 | * _[spin|read|write]_lock() | 105 | * __[spin|read|write]_lock() |
231 | * _[spin|read|write]_lock_irq() | 106 | * __[spin|read|write]_lock_irq() |
232 | * _[spin|read|write]_lock_irqsave() | 107 | * __[spin|read|write]_lock_irqsave() |
233 | * _[spin|read|write]_lock_bh() | 108 | * __[spin|read|write]_lock_bh() |
234 | */ | 109 | */ |
235 | BUILD_LOCK_OPS(spin, spinlock); | 110 | BUILD_LOCK_OPS(spin, spinlock); |
236 | BUILD_LOCK_OPS(read, rwlock); | 111 | BUILD_LOCK_OPS(read, rwlock); |
237 | BUILD_LOCK_OPS(write, rwlock); | 112 | BUILD_LOCK_OPS(write, rwlock); |
238 | 113 | ||
239 | #endif /* CONFIG_PREEMPT */ | 114 | #endif |
240 | 115 | ||
241 | #ifdef CONFIG_DEBUG_LOCK_ALLOC | 116 | #ifdef CONFIG_DEBUG_LOCK_ALLOC |
242 | 117 | ||
@@ -248,7 +123,8 @@ void __lockfunc _spin_lock_nested(spinlock_t *lock, int subclass) | |||
248 | } | 123 | } |
249 | EXPORT_SYMBOL(_spin_lock_nested); | 124 | EXPORT_SYMBOL(_spin_lock_nested); |
250 | 125 | ||
251 | unsigned long __lockfunc _spin_lock_irqsave_nested(spinlock_t *lock, int subclass) | 126 | unsigned long __lockfunc _spin_lock_irqsave_nested(spinlock_t *lock, |
127 | int subclass) | ||
252 | { | 128 | { |
253 | unsigned long flags; | 129 | unsigned long flags; |
254 | 130 | ||
@@ -272,7 +148,127 @@ EXPORT_SYMBOL(_spin_lock_nest_lock); | |||
272 | 148 | ||
273 | #endif | 149 | #endif |
274 | 150 | ||
275 | #ifndef _spin_unlock | 151 | #ifndef CONFIG_INLINE_SPIN_TRYLOCK |
152 | int __lockfunc _spin_trylock(spinlock_t *lock) | ||
153 | { | ||
154 | return __spin_trylock(lock); | ||
155 | } | ||
156 | EXPORT_SYMBOL(_spin_trylock); | ||
157 | #endif | ||
158 | |||
159 | #ifndef CONFIG_INLINE_READ_TRYLOCK | ||
160 | int __lockfunc _read_trylock(rwlock_t *lock) | ||
161 | { | ||
162 | return __read_trylock(lock); | ||
163 | } | ||
164 | EXPORT_SYMBOL(_read_trylock); | ||
165 | #endif | ||
166 | |||
167 | #ifndef CONFIG_INLINE_WRITE_TRYLOCK | ||
168 | int __lockfunc _write_trylock(rwlock_t *lock) | ||
169 | { | ||
170 | return __write_trylock(lock); | ||
171 | } | ||
172 | EXPORT_SYMBOL(_write_trylock); | ||
173 | #endif | ||
174 | |||
175 | #ifndef CONFIG_INLINE_READ_LOCK | ||
176 | void __lockfunc _read_lock(rwlock_t *lock) | ||
177 | { | ||
178 | __read_lock(lock); | ||
179 | } | ||
180 | EXPORT_SYMBOL(_read_lock); | ||
181 | #endif | ||
182 | |||
183 | #ifndef CONFIG_INLINE_SPIN_LOCK_IRQSAVE | ||
184 | unsigned long __lockfunc _spin_lock_irqsave(spinlock_t *lock) | ||
185 | { | ||
186 | return __spin_lock_irqsave(lock); | ||
187 | } | ||
188 | EXPORT_SYMBOL(_spin_lock_irqsave); | ||
189 | #endif | ||
190 | |||
191 | #ifndef CONFIG_INLINE_SPIN_LOCK_IRQ | ||
192 | void __lockfunc _spin_lock_irq(spinlock_t *lock) | ||
193 | { | ||
194 | __spin_lock_irq(lock); | ||
195 | } | ||
196 | EXPORT_SYMBOL(_spin_lock_irq); | ||
197 | #endif | ||
198 | |||
199 | #ifndef CONFIG_INLINE_SPIN_LOCK_BH | ||
200 | void __lockfunc _spin_lock_bh(spinlock_t *lock) | ||
201 | { | ||
202 | __spin_lock_bh(lock); | ||
203 | } | ||
204 | EXPORT_SYMBOL(_spin_lock_bh); | ||
205 | #endif | ||
206 | |||
207 | #ifndef CONFIG_INLINE_READ_LOCK_IRQSAVE | ||
208 | unsigned long __lockfunc _read_lock_irqsave(rwlock_t *lock) | ||
209 | { | ||
210 | return __read_lock_irqsave(lock); | ||
211 | } | ||
212 | EXPORT_SYMBOL(_read_lock_irqsave); | ||
213 | #endif | ||
214 | |||
215 | #ifndef CONFIG_INLINE_READ_LOCK_IRQ | ||
216 | void __lockfunc _read_lock_irq(rwlock_t *lock) | ||
217 | { | ||
218 | __read_lock_irq(lock); | ||
219 | } | ||
220 | EXPORT_SYMBOL(_read_lock_irq); | ||
221 | #endif | ||
222 | |||
223 | #ifndef CONFIG_INLINE_READ_LOCK_BH | ||
224 | void __lockfunc _read_lock_bh(rwlock_t *lock) | ||
225 | { | ||
226 | __read_lock_bh(lock); | ||
227 | } | ||
228 | EXPORT_SYMBOL(_read_lock_bh); | ||
229 | #endif | ||
230 | |||
231 | #ifndef CONFIG_INLINE_WRITE_LOCK_IRQSAVE | ||
232 | unsigned long __lockfunc _write_lock_irqsave(rwlock_t *lock) | ||
233 | { | ||
234 | return __write_lock_irqsave(lock); | ||
235 | } | ||
236 | EXPORT_SYMBOL(_write_lock_irqsave); | ||
237 | #endif | ||
238 | |||
239 | #ifndef CONFIG_INLINE_WRITE_LOCK_IRQ | ||
240 | void __lockfunc _write_lock_irq(rwlock_t *lock) | ||
241 | { | ||
242 | __write_lock_irq(lock); | ||
243 | } | ||
244 | EXPORT_SYMBOL(_write_lock_irq); | ||
245 | #endif | ||
246 | |||
247 | #ifndef CONFIG_INLINE_WRITE_LOCK_BH | ||
248 | void __lockfunc _write_lock_bh(rwlock_t *lock) | ||
249 | { | ||
250 | __write_lock_bh(lock); | ||
251 | } | ||
252 | EXPORT_SYMBOL(_write_lock_bh); | ||
253 | #endif | ||
254 | |||
255 | #ifndef CONFIG_INLINE_SPIN_LOCK | ||
256 | void __lockfunc _spin_lock(spinlock_t *lock) | ||
257 | { | ||
258 | __spin_lock(lock); | ||
259 | } | ||
260 | EXPORT_SYMBOL(_spin_lock); | ||
261 | #endif | ||
262 | |||
263 | #ifndef CONFIG_INLINE_WRITE_LOCK | ||
264 | void __lockfunc _write_lock(rwlock_t *lock) | ||
265 | { | ||
266 | __write_lock(lock); | ||
267 | } | ||
268 | EXPORT_SYMBOL(_write_lock); | ||
269 | #endif | ||
270 | |||
271 | #ifndef CONFIG_INLINE_SPIN_UNLOCK | ||
276 | void __lockfunc _spin_unlock(spinlock_t *lock) | 272 | void __lockfunc _spin_unlock(spinlock_t *lock) |
277 | { | 273 | { |
278 | __spin_unlock(lock); | 274 | __spin_unlock(lock); |
@@ -280,7 +276,7 @@ void __lockfunc _spin_unlock(spinlock_t *lock) | |||
280 | EXPORT_SYMBOL(_spin_unlock); | 276 | EXPORT_SYMBOL(_spin_unlock); |
281 | #endif | 277 | #endif |
282 | 278 | ||
283 | #ifndef _write_unlock | 279 | #ifndef CONFIG_INLINE_WRITE_UNLOCK |
284 | void __lockfunc _write_unlock(rwlock_t *lock) | 280 | void __lockfunc _write_unlock(rwlock_t *lock) |
285 | { | 281 | { |
286 | __write_unlock(lock); | 282 | __write_unlock(lock); |
@@ -288,7 +284,7 @@ void __lockfunc _write_unlock(rwlock_t *lock) | |||
288 | EXPORT_SYMBOL(_write_unlock); | 284 | EXPORT_SYMBOL(_write_unlock); |
289 | #endif | 285 | #endif |
290 | 286 | ||
291 | #ifndef _read_unlock | 287 | #ifndef CONFIG_INLINE_READ_UNLOCK |
292 | void __lockfunc _read_unlock(rwlock_t *lock) | 288 | void __lockfunc _read_unlock(rwlock_t *lock) |
293 | { | 289 | { |
294 | __read_unlock(lock); | 290 | __read_unlock(lock); |
@@ -296,7 +292,7 @@ void __lockfunc _read_unlock(rwlock_t *lock) | |||
296 | EXPORT_SYMBOL(_read_unlock); | 292 | EXPORT_SYMBOL(_read_unlock); |
297 | #endif | 293 | #endif |
298 | 294 | ||
299 | #ifndef _spin_unlock_irqrestore | 295 | #ifndef CONFIG_INLINE_SPIN_UNLOCK_IRQRESTORE |
300 | void __lockfunc _spin_unlock_irqrestore(spinlock_t *lock, unsigned long flags) | 296 | void __lockfunc _spin_unlock_irqrestore(spinlock_t *lock, unsigned long flags) |
301 | { | 297 | { |
302 | __spin_unlock_irqrestore(lock, flags); | 298 | __spin_unlock_irqrestore(lock, flags); |
@@ -304,7 +300,7 @@ void __lockfunc _spin_unlock_irqrestore(spinlock_t *lock, unsigned long flags) | |||
304 | EXPORT_SYMBOL(_spin_unlock_irqrestore); | 300 | EXPORT_SYMBOL(_spin_unlock_irqrestore); |
305 | #endif | 301 | #endif |
306 | 302 | ||
307 | #ifndef _spin_unlock_irq | 303 | #ifndef CONFIG_INLINE_SPIN_UNLOCK_IRQ |
308 | void __lockfunc _spin_unlock_irq(spinlock_t *lock) | 304 | void __lockfunc _spin_unlock_irq(spinlock_t *lock) |
309 | { | 305 | { |
310 | __spin_unlock_irq(lock); | 306 | __spin_unlock_irq(lock); |
@@ -312,7 +308,7 @@ void __lockfunc _spin_unlock_irq(spinlock_t *lock) | |||
312 | EXPORT_SYMBOL(_spin_unlock_irq); | 308 | EXPORT_SYMBOL(_spin_unlock_irq); |
313 | #endif | 309 | #endif |
314 | 310 | ||
315 | #ifndef _spin_unlock_bh | 311 | #ifndef CONFIG_INLINE_SPIN_UNLOCK_BH |
316 | void __lockfunc _spin_unlock_bh(spinlock_t *lock) | 312 | void __lockfunc _spin_unlock_bh(spinlock_t *lock) |
317 | { | 313 | { |
318 | __spin_unlock_bh(lock); | 314 | __spin_unlock_bh(lock); |
@@ -320,7 +316,7 @@ void __lockfunc _spin_unlock_bh(spinlock_t *lock) | |||
320 | EXPORT_SYMBOL(_spin_unlock_bh); | 316 | EXPORT_SYMBOL(_spin_unlock_bh); |
321 | #endif | 317 | #endif |
322 | 318 | ||
323 | #ifndef _read_unlock_irqrestore | 319 | #ifndef CONFIG_INLINE_READ_UNLOCK_IRQRESTORE |
324 | void __lockfunc _read_unlock_irqrestore(rwlock_t *lock, unsigned long flags) | 320 | void __lockfunc _read_unlock_irqrestore(rwlock_t *lock, unsigned long flags) |
325 | { | 321 | { |
326 | __read_unlock_irqrestore(lock, flags); | 322 | __read_unlock_irqrestore(lock, flags); |
@@ -328,7 +324,7 @@ void __lockfunc _read_unlock_irqrestore(rwlock_t *lock, unsigned long flags) | |||
328 | EXPORT_SYMBOL(_read_unlock_irqrestore); | 324 | EXPORT_SYMBOL(_read_unlock_irqrestore); |
329 | #endif | 325 | #endif |
330 | 326 | ||
331 | #ifndef _read_unlock_irq | 327 | #ifndef CONFIG_INLINE_READ_UNLOCK_IRQ |
332 | void __lockfunc _read_unlock_irq(rwlock_t *lock) | 328 | void __lockfunc _read_unlock_irq(rwlock_t *lock) |
333 | { | 329 | { |
334 | __read_unlock_irq(lock); | 330 | __read_unlock_irq(lock); |
@@ -336,7 +332,7 @@ void __lockfunc _read_unlock_irq(rwlock_t *lock) | |||
336 | EXPORT_SYMBOL(_read_unlock_irq); | 332 | EXPORT_SYMBOL(_read_unlock_irq); |
337 | #endif | 333 | #endif |
338 | 334 | ||
339 | #ifndef _read_unlock_bh | 335 | #ifndef CONFIG_INLINE_READ_UNLOCK_BH |
340 | void __lockfunc _read_unlock_bh(rwlock_t *lock) | 336 | void __lockfunc _read_unlock_bh(rwlock_t *lock) |
341 | { | 337 | { |
342 | __read_unlock_bh(lock); | 338 | __read_unlock_bh(lock); |
@@ -344,7 +340,7 @@ void __lockfunc _read_unlock_bh(rwlock_t *lock) | |||
344 | EXPORT_SYMBOL(_read_unlock_bh); | 340 | EXPORT_SYMBOL(_read_unlock_bh); |
345 | #endif | 341 | #endif |
346 | 342 | ||
347 | #ifndef _write_unlock_irqrestore | 343 | #ifndef CONFIG_INLINE_WRITE_UNLOCK_IRQRESTORE |
348 | void __lockfunc _write_unlock_irqrestore(rwlock_t *lock, unsigned long flags) | 344 | void __lockfunc _write_unlock_irqrestore(rwlock_t *lock, unsigned long flags) |
349 | { | 345 | { |
350 | __write_unlock_irqrestore(lock, flags); | 346 | __write_unlock_irqrestore(lock, flags); |
@@ -352,7 +348,7 @@ void __lockfunc _write_unlock_irqrestore(rwlock_t *lock, unsigned long flags) | |||
352 | EXPORT_SYMBOL(_write_unlock_irqrestore); | 348 | EXPORT_SYMBOL(_write_unlock_irqrestore); |
353 | #endif | 349 | #endif |
354 | 350 | ||
355 | #ifndef _write_unlock_irq | 351 | #ifndef CONFIG_INLINE_WRITE_UNLOCK_IRQ |
356 | void __lockfunc _write_unlock_irq(rwlock_t *lock) | 352 | void __lockfunc _write_unlock_irq(rwlock_t *lock) |
357 | { | 353 | { |
358 | __write_unlock_irq(lock); | 354 | __write_unlock_irq(lock); |
@@ -360,7 +356,7 @@ void __lockfunc _write_unlock_irq(rwlock_t *lock) | |||
360 | EXPORT_SYMBOL(_write_unlock_irq); | 356 | EXPORT_SYMBOL(_write_unlock_irq); |
361 | #endif | 357 | #endif |
362 | 358 | ||
363 | #ifndef _write_unlock_bh | 359 | #ifndef CONFIG_INLINE_WRITE_UNLOCK_BH |
364 | void __lockfunc _write_unlock_bh(rwlock_t *lock) | 360 | void __lockfunc _write_unlock_bh(rwlock_t *lock) |
365 | { | 361 | { |
366 | __write_unlock_bh(lock); | 362 | __write_unlock_bh(lock); |
@@ -368,7 +364,7 @@ void __lockfunc _write_unlock_bh(rwlock_t *lock) | |||
368 | EXPORT_SYMBOL(_write_unlock_bh); | 364 | EXPORT_SYMBOL(_write_unlock_bh); |
369 | #endif | 365 | #endif |
370 | 366 | ||
371 | #ifndef _spin_trylock_bh | 367 | #ifndef CONFIG_INLINE_SPIN_TRYLOCK_BH |
372 | int __lockfunc _spin_trylock_bh(spinlock_t *lock) | 368 | int __lockfunc _spin_trylock_bh(spinlock_t *lock) |
373 | { | 369 | { |
374 | return __spin_trylock_bh(lock); | 370 | return __spin_trylock_bh(lock); |
diff --git a/kernel/srcu.c b/kernel/srcu.c index b0aeeaf22ce4..818d7d9aa03c 100644 --- a/kernel/srcu.c +++ b/kernel/srcu.c | |||
@@ -49,6 +49,7 @@ int init_srcu_struct(struct srcu_struct *sp) | |||
49 | sp->per_cpu_ref = alloc_percpu(struct srcu_struct_array); | 49 | sp->per_cpu_ref = alloc_percpu(struct srcu_struct_array); |
50 | return (sp->per_cpu_ref ? 0 : -ENOMEM); | 50 | return (sp->per_cpu_ref ? 0 : -ENOMEM); |
51 | } | 51 | } |
52 | EXPORT_SYMBOL_GPL(init_srcu_struct); | ||
52 | 53 | ||
53 | /* | 54 | /* |
54 | * srcu_readers_active_idx -- returns approximate number of readers | 55 | * srcu_readers_active_idx -- returns approximate number of readers |
@@ -97,6 +98,7 @@ void cleanup_srcu_struct(struct srcu_struct *sp) | |||
97 | free_percpu(sp->per_cpu_ref); | 98 | free_percpu(sp->per_cpu_ref); |
98 | sp->per_cpu_ref = NULL; | 99 | sp->per_cpu_ref = NULL; |
99 | } | 100 | } |
101 | EXPORT_SYMBOL_GPL(cleanup_srcu_struct); | ||
100 | 102 | ||
101 | /** | 103 | /** |
102 | * srcu_read_lock - register a new reader for an SRCU-protected structure. | 104 | * srcu_read_lock - register a new reader for an SRCU-protected structure. |
@@ -118,6 +120,7 @@ int srcu_read_lock(struct srcu_struct *sp) | |||
118 | preempt_enable(); | 120 | preempt_enable(); |
119 | return idx; | 121 | return idx; |
120 | } | 122 | } |
123 | EXPORT_SYMBOL_GPL(srcu_read_lock); | ||
121 | 124 | ||
122 | /** | 125 | /** |
123 | * srcu_read_unlock - unregister a old reader from an SRCU-protected structure. | 126 | * srcu_read_unlock - unregister a old reader from an SRCU-protected structure. |
@@ -136,22 +139,12 @@ void srcu_read_unlock(struct srcu_struct *sp, int idx) | |||
136 | per_cpu_ptr(sp->per_cpu_ref, smp_processor_id())->c[idx]--; | 139 | per_cpu_ptr(sp->per_cpu_ref, smp_processor_id())->c[idx]--; |
137 | preempt_enable(); | 140 | preempt_enable(); |
138 | } | 141 | } |
142 | EXPORT_SYMBOL_GPL(srcu_read_unlock); | ||
139 | 143 | ||
140 | /** | 144 | /* |
141 | * synchronize_srcu - wait for prior SRCU read-side critical-section completion | 145 | * Helper function for synchronize_srcu() and synchronize_srcu_expedited(). |
142 | * @sp: srcu_struct with which to synchronize. | ||
143 | * | ||
144 | * Flip the completed counter, and wait for the old count to drain to zero. | ||
145 | * As with classic RCU, the updater must use some separate means of | ||
146 | * synchronizing concurrent updates. Can block; must be called from | ||
147 | * process context. | ||
148 | * | ||
149 | * Note that it is illegal to call synchornize_srcu() from the corresponding | ||
150 | * SRCU read-side critical section; doing so will result in deadlock. | ||
151 | * However, it is perfectly legal to call synchronize_srcu() on one | ||
152 | * srcu_struct from some other srcu_struct's read-side critical section. | ||
153 | */ | 146 | */ |
154 | void synchronize_srcu(struct srcu_struct *sp) | 147 | void __synchronize_srcu(struct srcu_struct *sp, void (*sync_func)(void)) |
155 | { | 148 | { |
156 | int idx; | 149 | int idx; |
157 | 150 | ||
@@ -173,7 +166,7 @@ void synchronize_srcu(struct srcu_struct *sp) | |||
173 | return; | 166 | return; |
174 | } | 167 | } |
175 | 168 | ||
176 | synchronize_sched(); /* Force memory barrier on all CPUs. */ | 169 | sync_func(); /* Force memory barrier on all CPUs. */ |
177 | 170 | ||
178 | /* | 171 | /* |
179 | * The preceding synchronize_sched() ensures that any CPU that | 172 | * The preceding synchronize_sched() ensures that any CPU that |
@@ -190,7 +183,7 @@ void synchronize_srcu(struct srcu_struct *sp) | |||
190 | idx = sp->completed & 0x1; | 183 | idx = sp->completed & 0x1; |
191 | sp->completed++; | 184 | sp->completed++; |
192 | 185 | ||
193 | synchronize_sched(); /* Force memory barrier on all CPUs. */ | 186 | sync_func(); /* Force memory barrier on all CPUs. */ |
194 | 187 | ||
195 | /* | 188 | /* |
196 | * At this point, because of the preceding synchronize_sched(), | 189 | * At this point, because of the preceding synchronize_sched(), |
@@ -203,7 +196,7 @@ void synchronize_srcu(struct srcu_struct *sp) | |||
203 | while (srcu_readers_active_idx(sp, idx)) | 196 | while (srcu_readers_active_idx(sp, idx)) |
204 | schedule_timeout_interruptible(1); | 197 | schedule_timeout_interruptible(1); |
205 | 198 | ||
206 | synchronize_sched(); /* Force memory barrier on all CPUs. */ | 199 | sync_func(); /* Force memory barrier on all CPUs. */ |
207 | 200 | ||
208 | /* | 201 | /* |
209 | * The preceding synchronize_sched() forces all srcu_read_unlock() | 202 | * The preceding synchronize_sched() forces all srcu_read_unlock() |
@@ -237,6 +230,47 @@ void synchronize_srcu(struct srcu_struct *sp) | |||
237 | } | 230 | } |
238 | 231 | ||
239 | /** | 232 | /** |
233 | * synchronize_srcu - wait for prior SRCU read-side critical-section completion | ||
234 | * @sp: srcu_struct with which to synchronize. | ||
235 | * | ||
236 | * Flip the completed counter, and wait for the old count to drain to zero. | ||
237 | * As with classic RCU, the updater must use some separate means of | ||
238 | * synchronizing concurrent updates. Can block; must be called from | ||
239 | * process context. | ||
240 | * | ||
241 | * Note that it is illegal to call synchronize_srcu() from the corresponding | ||
242 | * SRCU read-side critical section; doing so will result in deadlock. | ||
243 | * However, it is perfectly legal to call synchronize_srcu() on one | ||
244 | * srcu_struct from some other srcu_struct's read-side critical section. | ||
245 | */ | ||
246 | void synchronize_srcu(struct srcu_struct *sp) | ||
247 | { | ||
248 | __synchronize_srcu(sp, synchronize_sched); | ||
249 | } | ||
250 | EXPORT_SYMBOL_GPL(synchronize_srcu); | ||
251 | |||
252 | /** | ||
253 | * synchronize_srcu_expedited - like synchronize_srcu, but less patient | ||
254 | * @sp: srcu_struct with which to synchronize. | ||
255 | * | ||
256 | * Flip the completed counter, and wait for the old count to drain to zero. | ||
257 | * As with classic RCU, the updater must use some separate means of | ||
258 | * synchronizing concurrent updates. Can block; must be called from | ||
259 | * process context. | ||
260 | * | ||
261 | * Note that it is illegal to call synchronize_srcu_expedited() | ||
262 | * from the corresponding SRCU read-side critical section; doing so | ||
263 | * will result in deadlock. However, it is perfectly legal to call | ||
264 | * synchronize_srcu_expedited() on one srcu_struct from some other | ||
265 | * srcu_struct's read-side critical section. | ||
266 | */ | ||
267 | void synchronize_srcu_expedited(struct srcu_struct *sp) | ||
268 | { | ||
269 | __synchronize_srcu(sp, synchronize_sched_expedited); | ||
270 | } | ||
271 | EXPORT_SYMBOL_GPL(synchronize_srcu_expedited); | ||
272 | |||
273 | /** | ||
240 | * srcu_batches_completed - return batches completed. | 274 | * srcu_batches_completed - return batches completed. |
241 | * @sp: srcu_struct on which to report batch completion. | 275 | * @sp: srcu_struct on which to report batch completion. |
242 | * | 276 | * |
@@ -248,10 +282,4 @@ long srcu_batches_completed(struct srcu_struct *sp) | |||
248 | { | 282 | { |
249 | return sp->completed; | 283 | return sp->completed; |
250 | } | 284 | } |
251 | |||
252 | EXPORT_SYMBOL_GPL(init_srcu_struct); | ||
253 | EXPORT_SYMBOL_GPL(cleanup_srcu_struct); | ||
254 | EXPORT_SYMBOL_GPL(srcu_read_lock); | ||
255 | EXPORT_SYMBOL_GPL(srcu_read_unlock); | ||
256 | EXPORT_SYMBOL_GPL(synchronize_srcu); | ||
257 | EXPORT_SYMBOL_GPL(srcu_batches_completed); | 285 | EXPORT_SYMBOL_GPL(srcu_batches_completed); |
diff --git a/kernel/sys.c b/kernel/sys.c index 255475d163e0..585d6cd10040 100644 --- a/kernel/sys.c +++ b/kernel/sys.c | |||
@@ -8,7 +8,6 @@ | |||
8 | #include <linux/mm.h> | 8 | #include <linux/mm.h> |
9 | #include <linux/utsname.h> | 9 | #include <linux/utsname.h> |
10 | #include <linux/mman.h> | 10 | #include <linux/mman.h> |
11 | #include <linux/smp_lock.h> | ||
12 | #include <linux/notifier.h> | 11 | #include <linux/notifier.h> |
13 | #include <linux/reboot.h> | 12 | #include <linux/reboot.h> |
14 | #include <linux/prctl.h> | 13 | #include <linux/prctl.h> |
@@ -349,6 +348,9 @@ void kernel_power_off(void) | |||
349 | machine_power_off(); | 348 | machine_power_off(); |
350 | } | 349 | } |
351 | EXPORT_SYMBOL_GPL(kernel_power_off); | 350 | EXPORT_SYMBOL_GPL(kernel_power_off); |
351 | |||
352 | static DEFINE_MUTEX(reboot_mutex); | ||
353 | |||
352 | /* | 354 | /* |
353 | * Reboot system call: for obvious reasons only root may call it, | 355 | * Reboot system call: for obvious reasons only root may call it, |
354 | * and even root needs to set up some magic numbers in the registers | 356 | * and even root needs to set up some magic numbers in the registers |
@@ -381,7 +383,7 @@ SYSCALL_DEFINE4(reboot, int, magic1, int, magic2, unsigned int, cmd, | |||
381 | if ((cmd == LINUX_REBOOT_CMD_POWER_OFF) && !pm_power_off) | 383 | if ((cmd == LINUX_REBOOT_CMD_POWER_OFF) && !pm_power_off) |
382 | cmd = LINUX_REBOOT_CMD_HALT; | 384 | cmd = LINUX_REBOOT_CMD_HALT; |
383 | 385 | ||
384 | lock_kernel(); | 386 | mutex_lock(&reboot_mutex); |
385 | switch (cmd) { | 387 | switch (cmd) { |
386 | case LINUX_REBOOT_CMD_RESTART: | 388 | case LINUX_REBOOT_CMD_RESTART: |
387 | kernel_restart(NULL); | 389 | kernel_restart(NULL); |
@@ -397,20 +399,18 @@ SYSCALL_DEFINE4(reboot, int, magic1, int, magic2, unsigned int, cmd, | |||
397 | 399 | ||
398 | case LINUX_REBOOT_CMD_HALT: | 400 | case LINUX_REBOOT_CMD_HALT: |
399 | kernel_halt(); | 401 | kernel_halt(); |
400 | unlock_kernel(); | ||
401 | do_exit(0); | 402 | do_exit(0); |
402 | panic("cannot halt"); | 403 | panic("cannot halt"); |
403 | 404 | ||
404 | case LINUX_REBOOT_CMD_POWER_OFF: | 405 | case LINUX_REBOOT_CMD_POWER_OFF: |
405 | kernel_power_off(); | 406 | kernel_power_off(); |
406 | unlock_kernel(); | ||
407 | do_exit(0); | 407 | do_exit(0); |
408 | break; | 408 | break; |
409 | 409 | ||
410 | case LINUX_REBOOT_CMD_RESTART2: | 410 | case LINUX_REBOOT_CMD_RESTART2: |
411 | if (strncpy_from_user(&buffer[0], arg, sizeof(buffer) - 1) < 0) { | 411 | if (strncpy_from_user(&buffer[0], arg, sizeof(buffer) - 1) < 0) { |
412 | unlock_kernel(); | 412 | ret = -EFAULT; |
413 | return -EFAULT; | 413 | break; |
414 | } | 414 | } |
415 | buffer[sizeof(buffer) - 1] = '\0'; | 415 | buffer[sizeof(buffer) - 1] = '\0'; |
416 | 416 | ||
@@ -433,7 +433,7 @@ SYSCALL_DEFINE4(reboot, int, magic1, int, magic2, unsigned int, cmd, | |||
433 | ret = -EINVAL; | 433 | ret = -EINVAL; |
434 | break; | 434 | break; |
435 | } | 435 | } |
436 | unlock_kernel(); | 436 | mutex_unlock(&reboot_mutex); |
437 | return ret; | 437 | return ret; |
438 | } | 438 | } |
439 | 439 | ||
@@ -911,16 +911,15 @@ change_okay: | |||
911 | 911 | ||
912 | void do_sys_times(struct tms *tms) | 912 | void do_sys_times(struct tms *tms) |
913 | { | 913 | { |
914 | struct task_cputime cputime; | 914 | cputime_t tgutime, tgstime, cutime, cstime; |
915 | cputime_t cutime, cstime; | ||
916 | 915 | ||
917 | thread_group_cputime(current, &cputime); | ||
918 | spin_lock_irq(¤t->sighand->siglock); | 916 | spin_lock_irq(¤t->sighand->siglock); |
917 | thread_group_times(current, &tgutime, &tgstime); | ||
919 | cutime = current->signal->cutime; | 918 | cutime = current->signal->cutime; |
920 | cstime = current->signal->cstime; | 919 | cstime = current->signal->cstime; |
921 | spin_unlock_irq(¤t->sighand->siglock); | 920 | spin_unlock_irq(¤t->sighand->siglock); |
922 | tms->tms_utime = cputime_to_clock_t(cputime.utime); | 921 | tms->tms_utime = cputime_to_clock_t(tgutime); |
923 | tms->tms_stime = cputime_to_clock_t(cputime.stime); | 922 | tms->tms_stime = cputime_to_clock_t(tgstime); |
924 | tms->tms_cutime = cputime_to_clock_t(cutime); | 923 | tms->tms_cutime = cputime_to_clock_t(cutime); |
925 | tms->tms_cstime = cputime_to_clock_t(cstime); | 924 | tms->tms_cstime = cputime_to_clock_t(cstime); |
926 | } | 925 | } |
@@ -1110,6 +1109,8 @@ SYSCALL_DEFINE0(setsid) | |||
1110 | err = session; | 1109 | err = session; |
1111 | out: | 1110 | out: |
1112 | write_unlock_irq(&tasklist_lock); | 1111 | write_unlock_irq(&tasklist_lock); |
1112 | if (err > 0) | ||
1113 | proc_sid_connector(group_leader); | ||
1113 | return err; | 1114 | return err; |
1114 | } | 1115 | } |
1115 | 1116 | ||
@@ -1336,16 +1337,14 @@ static void k_getrusage(struct task_struct *p, int who, struct rusage *r) | |||
1336 | { | 1337 | { |
1337 | struct task_struct *t; | 1338 | struct task_struct *t; |
1338 | unsigned long flags; | 1339 | unsigned long flags; |
1339 | cputime_t utime, stime; | 1340 | cputime_t tgutime, tgstime, utime, stime; |
1340 | struct task_cputime cputime; | ||
1341 | unsigned long maxrss = 0; | 1341 | unsigned long maxrss = 0; |
1342 | 1342 | ||
1343 | memset((char *) r, 0, sizeof *r); | 1343 | memset((char *) r, 0, sizeof *r); |
1344 | utime = stime = cputime_zero; | 1344 | utime = stime = cputime_zero; |
1345 | 1345 | ||
1346 | if (who == RUSAGE_THREAD) { | 1346 | if (who == RUSAGE_THREAD) { |
1347 | utime = task_utime(current); | 1347 | task_times(current, &utime, &stime); |
1348 | stime = task_stime(current); | ||
1349 | accumulate_thread_rusage(p, r); | 1348 | accumulate_thread_rusage(p, r); |
1350 | maxrss = p->signal->maxrss; | 1349 | maxrss = p->signal->maxrss; |
1351 | goto out; | 1350 | goto out; |
@@ -1371,9 +1370,9 @@ static void k_getrusage(struct task_struct *p, int who, struct rusage *r) | |||
1371 | break; | 1370 | break; |
1372 | 1371 | ||
1373 | case RUSAGE_SELF: | 1372 | case RUSAGE_SELF: |
1374 | thread_group_cputime(p, &cputime); | 1373 | thread_group_times(p, &tgutime, &tgstime); |
1375 | utime = cputime_add(utime, cputime.utime); | 1374 | utime = cputime_add(utime, tgutime); |
1376 | stime = cputime_add(stime, cputime.stime); | 1375 | stime = cputime_add(stime, tgstime); |
1377 | r->ru_nvcsw += p->signal->nvcsw; | 1376 | r->ru_nvcsw += p->signal->nvcsw; |
1378 | r->ru_nivcsw += p->signal->nivcsw; | 1377 | r->ru_nivcsw += p->signal->nivcsw; |
1379 | r->ru_minflt += p->signal->min_flt; | 1378 | r->ru_minflt += p->signal->min_flt; |
@@ -1546,24 +1545,37 @@ SYSCALL_DEFINE5(prctl, int, option, unsigned long, arg2, unsigned long, arg3, | |||
1546 | if (arg4 | arg5) | 1545 | if (arg4 | arg5) |
1547 | return -EINVAL; | 1546 | return -EINVAL; |
1548 | switch (arg2) { | 1547 | switch (arg2) { |
1549 | case 0: | 1548 | case PR_MCE_KILL_CLEAR: |
1550 | if (arg3 != 0) | 1549 | if (arg3 != 0) |
1551 | return -EINVAL; | 1550 | return -EINVAL; |
1552 | current->flags &= ~PF_MCE_PROCESS; | 1551 | current->flags &= ~PF_MCE_PROCESS; |
1553 | break; | 1552 | break; |
1554 | case 1: | 1553 | case PR_MCE_KILL_SET: |
1555 | current->flags |= PF_MCE_PROCESS; | 1554 | current->flags |= PF_MCE_PROCESS; |
1556 | if (arg3 != 0) | 1555 | if (arg3 == PR_MCE_KILL_EARLY) |
1557 | current->flags |= PF_MCE_EARLY; | 1556 | current->flags |= PF_MCE_EARLY; |
1558 | else | 1557 | else if (arg3 == PR_MCE_KILL_LATE) |
1559 | current->flags &= ~PF_MCE_EARLY; | 1558 | current->flags &= ~PF_MCE_EARLY; |
1559 | else if (arg3 == PR_MCE_KILL_DEFAULT) | ||
1560 | current->flags &= | ||
1561 | ~(PF_MCE_EARLY|PF_MCE_PROCESS); | ||
1562 | else | ||
1563 | return -EINVAL; | ||
1560 | break; | 1564 | break; |
1561 | default: | 1565 | default: |
1562 | return -EINVAL; | 1566 | return -EINVAL; |
1563 | } | 1567 | } |
1564 | error = 0; | 1568 | error = 0; |
1565 | break; | 1569 | break; |
1566 | 1570 | case PR_MCE_KILL_GET: | |
1571 | if (arg2 | arg3 | arg4 | arg5) | ||
1572 | return -EINVAL; | ||
1573 | if (current->flags & PF_MCE_PROCESS) | ||
1574 | error = (current->flags & PF_MCE_EARLY) ? | ||
1575 | PR_MCE_KILL_EARLY : PR_MCE_KILL_LATE; | ||
1576 | else | ||
1577 | error = PR_MCE_KILL_DEFAULT; | ||
1578 | break; | ||
1567 | default: | 1579 | default: |
1568 | error = -EINVAL; | 1580 | error = -EINVAL; |
1569 | break; | 1581 | break; |
diff --git a/kernel/sys_ni.c b/kernel/sys_ni.c index e06d0b8d1951..695384f12a7d 100644 --- a/kernel/sys_ni.c +++ b/kernel/sys_ni.c | |||
@@ -48,8 +48,10 @@ cond_syscall(sys_shutdown); | |||
48 | cond_syscall(sys_sendmsg); | 48 | cond_syscall(sys_sendmsg); |
49 | cond_syscall(compat_sys_sendmsg); | 49 | cond_syscall(compat_sys_sendmsg); |
50 | cond_syscall(sys_recvmsg); | 50 | cond_syscall(sys_recvmsg); |
51 | cond_syscall(sys_recvmmsg); | ||
51 | cond_syscall(compat_sys_recvmsg); | 52 | cond_syscall(compat_sys_recvmsg); |
52 | cond_syscall(compat_sys_recvfrom); | 53 | cond_syscall(compat_sys_recvfrom); |
54 | cond_syscall(compat_sys_recvmmsg); | ||
53 | cond_syscall(sys_socketcall); | 55 | cond_syscall(sys_socketcall); |
54 | cond_syscall(sys_futex); | 56 | cond_syscall(sys_futex); |
55 | cond_syscall(compat_sys_futex); | 57 | cond_syscall(compat_sys_futex); |
@@ -139,7 +141,6 @@ cond_syscall(sys_pciconfig_read); | |||
139 | cond_syscall(sys_pciconfig_write); | 141 | cond_syscall(sys_pciconfig_write); |
140 | cond_syscall(sys_pciconfig_iobase); | 142 | cond_syscall(sys_pciconfig_iobase); |
141 | cond_syscall(sys32_ipc); | 143 | cond_syscall(sys32_ipc); |
142 | cond_syscall(sys32_sysctl); | ||
143 | cond_syscall(ppc_rtas); | 144 | cond_syscall(ppc_rtas); |
144 | cond_syscall(sys_spu_run); | 145 | cond_syscall(sys_spu_run); |
145 | cond_syscall(sys_spu_create); | 146 | cond_syscall(sys_spu_create); |
diff --git a/kernel/sysctl.c b/kernel/sysctl.c index 0d949c517412..9327a26765c5 100644 --- a/kernel/sysctl.c +++ b/kernel/sysctl.c | |||
@@ -27,7 +27,6 @@ | |||
27 | #include <linux/security.h> | 27 | #include <linux/security.h> |
28 | #include <linux/ctype.h> | 28 | #include <linux/ctype.h> |
29 | #include <linux/kmemcheck.h> | 29 | #include <linux/kmemcheck.h> |
30 | #include <linux/smp_lock.h> | ||
31 | #include <linux/fs.h> | 30 | #include <linux/fs.h> |
32 | #include <linux/init.h> | 31 | #include <linux/init.h> |
33 | #include <linux/kernel.h> | 32 | #include <linux/kernel.h> |
@@ -36,6 +35,7 @@ | |||
36 | #include <linux/sysrq.h> | 35 | #include <linux/sysrq.h> |
37 | #include <linux/highuid.h> | 36 | #include <linux/highuid.h> |
38 | #include <linux/writeback.h> | 37 | #include <linux/writeback.h> |
38 | #include <linux/ratelimit.h> | ||
39 | #include <linux/hugetlb.h> | 39 | #include <linux/hugetlb.h> |
40 | #include <linux/initrd.h> | 40 | #include <linux/initrd.h> |
41 | #include <linux/key.h> | 41 | #include <linux/key.h> |
@@ -60,7 +60,6 @@ | |||
60 | #include <asm/io.h> | 60 | #include <asm/io.h> |
61 | #endif | 61 | #endif |
62 | 62 | ||
63 | static int deprecated_sysctl_warning(struct __sysctl_args *args); | ||
64 | 63 | ||
65 | #if defined(CONFIG_SYSCTL) | 64 | #if defined(CONFIG_SYSCTL) |
66 | 65 | ||
@@ -158,6 +157,8 @@ extern int no_unaligned_warning; | |||
158 | extern int unaligned_dump_stack; | 157 | extern int unaligned_dump_stack; |
159 | #endif | 158 | #endif |
160 | 159 | ||
160 | extern struct ratelimit_state printk_ratelimit_state; | ||
161 | |||
161 | #ifdef CONFIG_RT_MUTEXES | 162 | #ifdef CONFIG_RT_MUTEXES |
162 | extern int max_lock_depth; | 163 | extern int max_lock_depth; |
163 | #endif | 164 | #endif |
@@ -207,31 +208,26 @@ extern int lock_stat; | |||
207 | 208 | ||
208 | static struct ctl_table root_table[] = { | 209 | static struct ctl_table root_table[] = { |
209 | { | 210 | { |
210 | .ctl_name = CTL_KERN, | ||
211 | .procname = "kernel", | 211 | .procname = "kernel", |
212 | .mode = 0555, | 212 | .mode = 0555, |
213 | .child = kern_table, | 213 | .child = kern_table, |
214 | }, | 214 | }, |
215 | { | 215 | { |
216 | .ctl_name = CTL_VM, | ||
217 | .procname = "vm", | 216 | .procname = "vm", |
218 | .mode = 0555, | 217 | .mode = 0555, |
219 | .child = vm_table, | 218 | .child = vm_table, |
220 | }, | 219 | }, |
221 | { | 220 | { |
222 | .ctl_name = CTL_FS, | ||
223 | .procname = "fs", | 221 | .procname = "fs", |
224 | .mode = 0555, | 222 | .mode = 0555, |
225 | .child = fs_table, | 223 | .child = fs_table, |
226 | }, | 224 | }, |
227 | { | 225 | { |
228 | .ctl_name = CTL_DEBUG, | ||
229 | .procname = "debug", | 226 | .procname = "debug", |
230 | .mode = 0555, | 227 | .mode = 0555, |
231 | .child = debug_table, | 228 | .child = debug_table, |
232 | }, | 229 | }, |
233 | { | 230 | { |
234 | .ctl_name = CTL_DEV, | ||
235 | .procname = "dev", | 231 | .procname = "dev", |
236 | .mode = 0555, | 232 | .mode = 0555, |
237 | .child = dev_table, | 233 | .child = dev_table, |
@@ -240,7 +236,7 @@ static struct ctl_table root_table[] = { | |||
240 | * NOTE: do not add new entries to this table unless you have read | 236 | * NOTE: do not add new entries to this table unless you have read |
241 | * Documentation/sysctl/ctl_unnumbered.txt | 237 | * Documentation/sysctl/ctl_unnumbered.txt |
242 | */ | 238 | */ |
243 | { .ctl_name = 0 } | 239 | { } |
244 | }; | 240 | }; |
245 | 241 | ||
246 | #ifdef CONFIG_SCHED_DEBUG | 242 | #ifdef CONFIG_SCHED_DEBUG |
@@ -252,192 +248,166 @@ static int max_wakeup_granularity_ns = NSEC_PER_SEC; /* 1 second */ | |||
252 | 248 | ||
253 | static struct ctl_table kern_table[] = { | 249 | static struct ctl_table kern_table[] = { |
254 | { | 250 | { |
255 | .ctl_name = CTL_UNNUMBERED, | ||
256 | .procname = "sched_child_runs_first", | 251 | .procname = "sched_child_runs_first", |
257 | .data = &sysctl_sched_child_runs_first, | 252 | .data = &sysctl_sched_child_runs_first, |
258 | .maxlen = sizeof(unsigned int), | 253 | .maxlen = sizeof(unsigned int), |
259 | .mode = 0644, | 254 | .mode = 0644, |
260 | .proc_handler = &proc_dointvec, | 255 | .proc_handler = proc_dointvec, |
261 | }, | 256 | }, |
262 | #ifdef CONFIG_SCHED_DEBUG | 257 | #ifdef CONFIG_SCHED_DEBUG |
263 | { | 258 | { |
264 | .ctl_name = CTL_UNNUMBERED, | ||
265 | .procname = "sched_min_granularity_ns", | 259 | .procname = "sched_min_granularity_ns", |
266 | .data = &sysctl_sched_min_granularity, | 260 | .data = &sysctl_sched_min_granularity, |
267 | .maxlen = sizeof(unsigned int), | 261 | .maxlen = sizeof(unsigned int), |
268 | .mode = 0644, | 262 | .mode = 0644, |
269 | .proc_handler = &sched_nr_latency_handler, | 263 | .proc_handler = sched_nr_latency_handler, |
270 | .strategy = &sysctl_intvec, | ||
271 | .extra1 = &min_sched_granularity_ns, | 264 | .extra1 = &min_sched_granularity_ns, |
272 | .extra2 = &max_sched_granularity_ns, | 265 | .extra2 = &max_sched_granularity_ns, |
273 | }, | 266 | }, |
274 | { | 267 | { |
275 | .ctl_name = CTL_UNNUMBERED, | ||
276 | .procname = "sched_latency_ns", | 268 | .procname = "sched_latency_ns", |
277 | .data = &sysctl_sched_latency, | 269 | .data = &sysctl_sched_latency, |
278 | .maxlen = sizeof(unsigned int), | 270 | .maxlen = sizeof(unsigned int), |
279 | .mode = 0644, | 271 | .mode = 0644, |
280 | .proc_handler = &sched_nr_latency_handler, | 272 | .proc_handler = sched_nr_latency_handler, |
281 | .strategy = &sysctl_intvec, | ||
282 | .extra1 = &min_sched_granularity_ns, | 273 | .extra1 = &min_sched_granularity_ns, |
283 | .extra2 = &max_sched_granularity_ns, | 274 | .extra2 = &max_sched_granularity_ns, |
284 | }, | 275 | }, |
285 | { | 276 | { |
286 | .ctl_name = CTL_UNNUMBERED, | ||
287 | .procname = "sched_wakeup_granularity_ns", | 277 | .procname = "sched_wakeup_granularity_ns", |
288 | .data = &sysctl_sched_wakeup_granularity, | 278 | .data = &sysctl_sched_wakeup_granularity, |
289 | .maxlen = sizeof(unsigned int), | 279 | .maxlen = sizeof(unsigned int), |
290 | .mode = 0644, | 280 | .mode = 0644, |
291 | .proc_handler = &proc_dointvec_minmax, | 281 | .proc_handler = proc_dointvec_minmax, |
292 | .strategy = &sysctl_intvec, | ||
293 | .extra1 = &min_wakeup_granularity_ns, | 282 | .extra1 = &min_wakeup_granularity_ns, |
294 | .extra2 = &max_wakeup_granularity_ns, | 283 | .extra2 = &max_wakeup_granularity_ns, |
295 | }, | 284 | }, |
296 | { | 285 | { |
297 | .ctl_name = CTL_UNNUMBERED, | ||
298 | .procname = "sched_shares_ratelimit", | 286 | .procname = "sched_shares_ratelimit", |
299 | .data = &sysctl_sched_shares_ratelimit, | 287 | .data = &sysctl_sched_shares_ratelimit, |
300 | .maxlen = sizeof(unsigned int), | 288 | .maxlen = sizeof(unsigned int), |
301 | .mode = 0644, | 289 | .mode = 0644, |
302 | .proc_handler = &proc_dointvec, | 290 | .proc_handler = proc_dointvec, |
303 | }, | 291 | }, |
304 | { | 292 | { |
305 | .ctl_name = CTL_UNNUMBERED, | ||
306 | .procname = "sched_shares_thresh", | 293 | .procname = "sched_shares_thresh", |
307 | .data = &sysctl_sched_shares_thresh, | 294 | .data = &sysctl_sched_shares_thresh, |
308 | .maxlen = sizeof(unsigned int), | 295 | .maxlen = sizeof(unsigned int), |
309 | .mode = 0644, | 296 | .mode = 0644, |
310 | .proc_handler = &proc_dointvec_minmax, | 297 | .proc_handler = proc_dointvec_minmax, |
311 | .strategy = &sysctl_intvec, | ||
312 | .extra1 = &zero, | 298 | .extra1 = &zero, |
313 | }, | 299 | }, |
314 | { | 300 | { |
315 | .ctl_name = CTL_UNNUMBERED, | ||
316 | .procname = "sched_features", | 301 | .procname = "sched_features", |
317 | .data = &sysctl_sched_features, | 302 | .data = &sysctl_sched_features, |
318 | .maxlen = sizeof(unsigned int), | 303 | .maxlen = sizeof(unsigned int), |
319 | .mode = 0644, | 304 | .mode = 0644, |
320 | .proc_handler = &proc_dointvec, | 305 | .proc_handler = proc_dointvec, |
321 | }, | 306 | }, |
322 | { | 307 | { |
323 | .ctl_name = CTL_UNNUMBERED, | ||
324 | .procname = "sched_migration_cost", | 308 | .procname = "sched_migration_cost", |
325 | .data = &sysctl_sched_migration_cost, | 309 | .data = &sysctl_sched_migration_cost, |
326 | .maxlen = sizeof(unsigned int), | 310 | .maxlen = sizeof(unsigned int), |
327 | .mode = 0644, | 311 | .mode = 0644, |
328 | .proc_handler = &proc_dointvec, | 312 | .proc_handler = proc_dointvec, |
329 | }, | 313 | }, |
330 | { | 314 | { |
331 | .ctl_name = CTL_UNNUMBERED, | ||
332 | .procname = "sched_nr_migrate", | 315 | .procname = "sched_nr_migrate", |
333 | .data = &sysctl_sched_nr_migrate, | 316 | .data = &sysctl_sched_nr_migrate, |
334 | .maxlen = sizeof(unsigned int), | 317 | .maxlen = sizeof(unsigned int), |
335 | .mode = 0644, | 318 | .mode = 0644, |
336 | .proc_handler = &proc_dointvec, | 319 | .proc_handler = proc_dointvec, |
337 | }, | 320 | }, |
338 | { | 321 | { |
339 | .ctl_name = CTL_UNNUMBERED, | ||
340 | .procname = "sched_time_avg", | 322 | .procname = "sched_time_avg", |
341 | .data = &sysctl_sched_time_avg, | 323 | .data = &sysctl_sched_time_avg, |
342 | .maxlen = sizeof(unsigned int), | 324 | .maxlen = sizeof(unsigned int), |
343 | .mode = 0644, | 325 | .mode = 0644, |
344 | .proc_handler = &proc_dointvec, | 326 | .proc_handler = proc_dointvec, |
345 | }, | 327 | }, |
346 | { | 328 | { |
347 | .ctl_name = CTL_UNNUMBERED, | ||
348 | .procname = "timer_migration", | 329 | .procname = "timer_migration", |
349 | .data = &sysctl_timer_migration, | 330 | .data = &sysctl_timer_migration, |
350 | .maxlen = sizeof(unsigned int), | 331 | .maxlen = sizeof(unsigned int), |
351 | .mode = 0644, | 332 | .mode = 0644, |
352 | .proc_handler = &proc_dointvec_minmax, | 333 | .proc_handler = proc_dointvec_minmax, |
353 | .strategy = &sysctl_intvec, | ||
354 | .extra1 = &zero, | 334 | .extra1 = &zero, |
355 | .extra2 = &one, | 335 | .extra2 = &one, |
356 | }, | 336 | }, |
357 | #endif | 337 | #endif |
358 | { | 338 | { |
359 | .ctl_name = CTL_UNNUMBERED, | ||
360 | .procname = "sched_rt_period_us", | 339 | .procname = "sched_rt_period_us", |
361 | .data = &sysctl_sched_rt_period, | 340 | .data = &sysctl_sched_rt_period, |
362 | .maxlen = sizeof(unsigned int), | 341 | .maxlen = sizeof(unsigned int), |
363 | .mode = 0644, | 342 | .mode = 0644, |
364 | .proc_handler = &sched_rt_handler, | 343 | .proc_handler = sched_rt_handler, |
365 | }, | 344 | }, |
366 | { | 345 | { |
367 | .ctl_name = CTL_UNNUMBERED, | ||
368 | .procname = "sched_rt_runtime_us", | 346 | .procname = "sched_rt_runtime_us", |
369 | .data = &sysctl_sched_rt_runtime, | 347 | .data = &sysctl_sched_rt_runtime, |
370 | .maxlen = sizeof(int), | 348 | .maxlen = sizeof(int), |
371 | .mode = 0644, | 349 | .mode = 0644, |
372 | .proc_handler = &sched_rt_handler, | 350 | .proc_handler = sched_rt_handler, |
373 | }, | 351 | }, |
374 | { | 352 | { |
375 | .ctl_name = CTL_UNNUMBERED, | ||
376 | .procname = "sched_compat_yield", | 353 | .procname = "sched_compat_yield", |
377 | .data = &sysctl_sched_compat_yield, | 354 | .data = &sysctl_sched_compat_yield, |
378 | .maxlen = sizeof(unsigned int), | 355 | .maxlen = sizeof(unsigned int), |
379 | .mode = 0644, | 356 | .mode = 0644, |
380 | .proc_handler = &proc_dointvec, | 357 | .proc_handler = proc_dointvec, |
381 | }, | 358 | }, |
382 | #ifdef CONFIG_PROVE_LOCKING | 359 | #ifdef CONFIG_PROVE_LOCKING |
383 | { | 360 | { |
384 | .ctl_name = CTL_UNNUMBERED, | ||
385 | .procname = "prove_locking", | 361 | .procname = "prove_locking", |
386 | .data = &prove_locking, | 362 | .data = &prove_locking, |
387 | .maxlen = sizeof(int), | 363 | .maxlen = sizeof(int), |
388 | .mode = 0644, | 364 | .mode = 0644, |
389 | .proc_handler = &proc_dointvec, | 365 | .proc_handler = proc_dointvec, |
390 | }, | 366 | }, |
391 | #endif | 367 | #endif |
392 | #ifdef CONFIG_LOCK_STAT | 368 | #ifdef CONFIG_LOCK_STAT |
393 | { | 369 | { |
394 | .ctl_name = CTL_UNNUMBERED, | ||
395 | .procname = "lock_stat", | 370 | .procname = "lock_stat", |
396 | .data = &lock_stat, | 371 | .data = &lock_stat, |
397 | .maxlen = sizeof(int), | 372 | .maxlen = sizeof(int), |
398 | .mode = 0644, | 373 | .mode = 0644, |
399 | .proc_handler = &proc_dointvec, | 374 | .proc_handler = proc_dointvec, |
400 | }, | 375 | }, |
401 | #endif | 376 | #endif |
402 | { | 377 | { |
403 | .ctl_name = KERN_PANIC, | ||
404 | .procname = "panic", | 378 | .procname = "panic", |
405 | .data = &panic_timeout, | 379 | .data = &panic_timeout, |
406 | .maxlen = sizeof(int), | 380 | .maxlen = sizeof(int), |
407 | .mode = 0644, | 381 | .mode = 0644, |
408 | .proc_handler = &proc_dointvec, | 382 | .proc_handler = proc_dointvec, |
409 | }, | 383 | }, |
410 | { | 384 | { |
411 | .ctl_name = KERN_CORE_USES_PID, | ||
412 | .procname = "core_uses_pid", | 385 | .procname = "core_uses_pid", |
413 | .data = &core_uses_pid, | 386 | .data = &core_uses_pid, |
414 | .maxlen = sizeof(int), | 387 | .maxlen = sizeof(int), |
415 | .mode = 0644, | 388 | .mode = 0644, |
416 | .proc_handler = &proc_dointvec, | 389 | .proc_handler = proc_dointvec, |
417 | }, | 390 | }, |
418 | { | 391 | { |
419 | .ctl_name = KERN_CORE_PATTERN, | ||
420 | .procname = "core_pattern", | 392 | .procname = "core_pattern", |
421 | .data = core_pattern, | 393 | .data = core_pattern, |
422 | .maxlen = CORENAME_MAX_SIZE, | 394 | .maxlen = CORENAME_MAX_SIZE, |
423 | .mode = 0644, | 395 | .mode = 0644, |
424 | .proc_handler = &proc_dostring, | 396 | .proc_handler = proc_dostring, |
425 | .strategy = &sysctl_string, | ||
426 | }, | 397 | }, |
427 | { | 398 | { |
428 | .ctl_name = CTL_UNNUMBERED, | ||
429 | .procname = "core_pipe_limit", | 399 | .procname = "core_pipe_limit", |
430 | .data = &core_pipe_limit, | 400 | .data = &core_pipe_limit, |
431 | .maxlen = sizeof(unsigned int), | 401 | .maxlen = sizeof(unsigned int), |
432 | .mode = 0644, | 402 | .mode = 0644, |
433 | .proc_handler = &proc_dointvec, | 403 | .proc_handler = proc_dointvec, |
434 | }, | 404 | }, |
435 | #ifdef CONFIG_PROC_SYSCTL | 405 | #ifdef CONFIG_PROC_SYSCTL |
436 | { | 406 | { |
437 | .procname = "tainted", | 407 | .procname = "tainted", |
438 | .maxlen = sizeof(long), | 408 | .maxlen = sizeof(long), |
439 | .mode = 0644, | 409 | .mode = 0644, |
440 | .proc_handler = &proc_taint, | 410 | .proc_handler = proc_taint, |
441 | }, | 411 | }, |
442 | #endif | 412 | #endif |
443 | #ifdef CONFIG_LATENCYTOP | 413 | #ifdef CONFIG_LATENCYTOP |
@@ -446,181 +416,160 @@ static struct ctl_table kern_table[] = { | |||
446 | .data = &latencytop_enabled, | 416 | .data = &latencytop_enabled, |
447 | .maxlen = sizeof(int), | 417 | .maxlen = sizeof(int), |
448 | .mode = 0644, | 418 | .mode = 0644, |
449 | .proc_handler = &proc_dointvec, | 419 | .proc_handler = proc_dointvec, |
450 | }, | 420 | }, |
451 | #endif | 421 | #endif |
452 | #ifdef CONFIG_BLK_DEV_INITRD | 422 | #ifdef CONFIG_BLK_DEV_INITRD |
453 | { | 423 | { |
454 | .ctl_name = KERN_REALROOTDEV, | ||
455 | .procname = "real-root-dev", | 424 | .procname = "real-root-dev", |
456 | .data = &real_root_dev, | 425 | .data = &real_root_dev, |
457 | .maxlen = sizeof(int), | 426 | .maxlen = sizeof(int), |
458 | .mode = 0644, | 427 | .mode = 0644, |
459 | .proc_handler = &proc_dointvec, | 428 | .proc_handler = proc_dointvec, |
460 | }, | 429 | }, |
461 | #endif | 430 | #endif |
462 | { | 431 | { |
463 | .ctl_name = CTL_UNNUMBERED, | ||
464 | .procname = "print-fatal-signals", | 432 | .procname = "print-fatal-signals", |
465 | .data = &print_fatal_signals, | 433 | .data = &print_fatal_signals, |
466 | .maxlen = sizeof(int), | 434 | .maxlen = sizeof(int), |
467 | .mode = 0644, | 435 | .mode = 0644, |
468 | .proc_handler = &proc_dointvec, | 436 | .proc_handler = proc_dointvec, |
469 | }, | 437 | }, |
470 | #ifdef CONFIG_SPARC | 438 | #ifdef CONFIG_SPARC |
471 | { | 439 | { |
472 | .ctl_name = KERN_SPARC_REBOOT, | ||
473 | .procname = "reboot-cmd", | 440 | .procname = "reboot-cmd", |
474 | .data = reboot_command, | 441 | .data = reboot_command, |
475 | .maxlen = 256, | 442 | .maxlen = 256, |
476 | .mode = 0644, | 443 | .mode = 0644, |
477 | .proc_handler = &proc_dostring, | 444 | .proc_handler = proc_dostring, |
478 | .strategy = &sysctl_string, | ||
479 | }, | 445 | }, |
480 | { | 446 | { |
481 | .ctl_name = KERN_SPARC_STOP_A, | ||
482 | .procname = "stop-a", | 447 | .procname = "stop-a", |
483 | .data = &stop_a_enabled, | 448 | .data = &stop_a_enabled, |
484 | .maxlen = sizeof (int), | 449 | .maxlen = sizeof (int), |
485 | .mode = 0644, | 450 | .mode = 0644, |
486 | .proc_handler = &proc_dointvec, | 451 | .proc_handler = proc_dointvec, |
487 | }, | 452 | }, |
488 | { | 453 | { |
489 | .ctl_name = KERN_SPARC_SCONS_PWROFF, | ||
490 | .procname = "scons-poweroff", | 454 | .procname = "scons-poweroff", |
491 | .data = &scons_pwroff, | 455 | .data = &scons_pwroff, |
492 | .maxlen = sizeof (int), | 456 | .maxlen = sizeof (int), |
493 | .mode = 0644, | 457 | .mode = 0644, |
494 | .proc_handler = &proc_dointvec, | 458 | .proc_handler = proc_dointvec, |
495 | }, | 459 | }, |
496 | #endif | 460 | #endif |
497 | #ifdef CONFIG_SPARC64 | 461 | #ifdef CONFIG_SPARC64 |
498 | { | 462 | { |
499 | .ctl_name = CTL_UNNUMBERED, | ||
500 | .procname = "tsb-ratio", | 463 | .procname = "tsb-ratio", |
501 | .data = &sysctl_tsb_ratio, | 464 | .data = &sysctl_tsb_ratio, |
502 | .maxlen = sizeof (int), | 465 | .maxlen = sizeof (int), |
503 | .mode = 0644, | 466 | .mode = 0644, |
504 | .proc_handler = &proc_dointvec, | 467 | .proc_handler = proc_dointvec, |
505 | }, | 468 | }, |
506 | #endif | 469 | #endif |
507 | #ifdef __hppa__ | 470 | #ifdef __hppa__ |
508 | { | 471 | { |
509 | .ctl_name = KERN_HPPA_PWRSW, | ||
510 | .procname = "soft-power", | 472 | .procname = "soft-power", |
511 | .data = &pwrsw_enabled, | 473 | .data = &pwrsw_enabled, |
512 | .maxlen = sizeof (int), | 474 | .maxlen = sizeof (int), |
513 | .mode = 0644, | 475 | .mode = 0644, |
514 | .proc_handler = &proc_dointvec, | 476 | .proc_handler = proc_dointvec, |
515 | }, | 477 | }, |
516 | { | 478 | { |
517 | .ctl_name = KERN_HPPA_UNALIGNED, | ||
518 | .procname = "unaligned-trap", | 479 | .procname = "unaligned-trap", |
519 | .data = &unaligned_enabled, | 480 | .data = &unaligned_enabled, |
520 | .maxlen = sizeof (int), | 481 | .maxlen = sizeof (int), |
521 | .mode = 0644, | 482 | .mode = 0644, |
522 | .proc_handler = &proc_dointvec, | 483 | .proc_handler = proc_dointvec, |
523 | }, | 484 | }, |
524 | #endif | 485 | #endif |
525 | { | 486 | { |
526 | .ctl_name = KERN_CTLALTDEL, | ||
527 | .procname = "ctrl-alt-del", | 487 | .procname = "ctrl-alt-del", |
528 | .data = &C_A_D, | 488 | .data = &C_A_D, |
529 | .maxlen = sizeof(int), | 489 | .maxlen = sizeof(int), |
530 | .mode = 0644, | 490 | .mode = 0644, |
531 | .proc_handler = &proc_dointvec, | 491 | .proc_handler = proc_dointvec, |
532 | }, | 492 | }, |
533 | #ifdef CONFIG_FUNCTION_TRACER | 493 | #ifdef CONFIG_FUNCTION_TRACER |
534 | { | 494 | { |
535 | .ctl_name = CTL_UNNUMBERED, | ||
536 | .procname = "ftrace_enabled", | 495 | .procname = "ftrace_enabled", |
537 | .data = &ftrace_enabled, | 496 | .data = &ftrace_enabled, |
538 | .maxlen = sizeof(int), | 497 | .maxlen = sizeof(int), |
539 | .mode = 0644, | 498 | .mode = 0644, |
540 | .proc_handler = &ftrace_enable_sysctl, | 499 | .proc_handler = ftrace_enable_sysctl, |
541 | }, | 500 | }, |
542 | #endif | 501 | #endif |
543 | #ifdef CONFIG_STACK_TRACER | 502 | #ifdef CONFIG_STACK_TRACER |
544 | { | 503 | { |
545 | .ctl_name = CTL_UNNUMBERED, | ||
546 | .procname = "stack_tracer_enabled", | 504 | .procname = "stack_tracer_enabled", |
547 | .data = &stack_tracer_enabled, | 505 | .data = &stack_tracer_enabled, |
548 | .maxlen = sizeof(int), | 506 | .maxlen = sizeof(int), |
549 | .mode = 0644, | 507 | .mode = 0644, |
550 | .proc_handler = &stack_trace_sysctl, | 508 | .proc_handler = stack_trace_sysctl, |
551 | }, | 509 | }, |
552 | #endif | 510 | #endif |
553 | #ifdef CONFIG_TRACING | 511 | #ifdef CONFIG_TRACING |
554 | { | 512 | { |
555 | .ctl_name = CTL_UNNUMBERED, | ||
556 | .procname = "ftrace_dump_on_oops", | 513 | .procname = "ftrace_dump_on_oops", |
557 | .data = &ftrace_dump_on_oops, | 514 | .data = &ftrace_dump_on_oops, |
558 | .maxlen = sizeof(int), | 515 | .maxlen = sizeof(int), |
559 | .mode = 0644, | 516 | .mode = 0644, |
560 | .proc_handler = &proc_dointvec, | 517 | .proc_handler = proc_dointvec, |
561 | }, | 518 | }, |
562 | #endif | 519 | #endif |
563 | #ifdef CONFIG_MODULES | 520 | #ifdef CONFIG_MODULES |
564 | { | 521 | { |
565 | .ctl_name = KERN_MODPROBE, | ||
566 | .procname = "modprobe", | 522 | .procname = "modprobe", |
567 | .data = &modprobe_path, | 523 | .data = &modprobe_path, |
568 | .maxlen = KMOD_PATH_LEN, | 524 | .maxlen = KMOD_PATH_LEN, |
569 | .mode = 0644, | 525 | .mode = 0644, |
570 | .proc_handler = &proc_dostring, | 526 | .proc_handler = proc_dostring, |
571 | .strategy = &sysctl_string, | ||
572 | }, | 527 | }, |
573 | { | 528 | { |
574 | .ctl_name = CTL_UNNUMBERED, | ||
575 | .procname = "modules_disabled", | 529 | .procname = "modules_disabled", |
576 | .data = &modules_disabled, | 530 | .data = &modules_disabled, |
577 | .maxlen = sizeof(int), | 531 | .maxlen = sizeof(int), |
578 | .mode = 0644, | 532 | .mode = 0644, |
579 | /* only handle a transition from default "0" to "1" */ | 533 | /* only handle a transition from default "0" to "1" */ |
580 | .proc_handler = &proc_dointvec_minmax, | 534 | .proc_handler = proc_dointvec_minmax, |
581 | .extra1 = &one, | 535 | .extra1 = &one, |
582 | .extra2 = &one, | 536 | .extra2 = &one, |
583 | }, | 537 | }, |
584 | #endif | 538 | #endif |
585 | #if defined(CONFIG_HOTPLUG) && defined(CONFIG_NET) | 539 | #if defined(CONFIG_HOTPLUG) && defined(CONFIG_NET) |
586 | { | 540 | { |
587 | .ctl_name = KERN_HOTPLUG, | ||
588 | .procname = "hotplug", | 541 | .procname = "hotplug", |
589 | .data = &uevent_helper, | 542 | .data = &uevent_helper, |
590 | .maxlen = UEVENT_HELPER_PATH_LEN, | 543 | .maxlen = UEVENT_HELPER_PATH_LEN, |
591 | .mode = 0644, | 544 | .mode = 0644, |
592 | .proc_handler = &proc_dostring, | 545 | .proc_handler = proc_dostring, |
593 | .strategy = &sysctl_string, | ||
594 | }, | 546 | }, |
595 | #endif | 547 | #endif |
596 | #ifdef CONFIG_CHR_DEV_SG | 548 | #ifdef CONFIG_CHR_DEV_SG |
597 | { | 549 | { |
598 | .ctl_name = KERN_SG_BIG_BUFF, | ||
599 | .procname = "sg-big-buff", | 550 | .procname = "sg-big-buff", |
600 | .data = &sg_big_buff, | 551 | .data = &sg_big_buff, |
601 | .maxlen = sizeof (int), | 552 | .maxlen = sizeof (int), |
602 | .mode = 0444, | 553 | .mode = 0444, |
603 | .proc_handler = &proc_dointvec, | 554 | .proc_handler = proc_dointvec, |
604 | }, | 555 | }, |
605 | #endif | 556 | #endif |
606 | #ifdef CONFIG_BSD_PROCESS_ACCT | 557 | #ifdef CONFIG_BSD_PROCESS_ACCT |
607 | { | 558 | { |
608 | .ctl_name = KERN_ACCT, | ||
609 | .procname = "acct", | 559 | .procname = "acct", |
610 | .data = &acct_parm, | 560 | .data = &acct_parm, |
611 | .maxlen = 3*sizeof(int), | 561 | .maxlen = 3*sizeof(int), |
612 | .mode = 0644, | 562 | .mode = 0644, |
613 | .proc_handler = &proc_dointvec, | 563 | .proc_handler = proc_dointvec, |
614 | }, | 564 | }, |
615 | #endif | 565 | #endif |
616 | #ifdef CONFIG_MAGIC_SYSRQ | 566 | #ifdef CONFIG_MAGIC_SYSRQ |
617 | { | 567 | { |
618 | .ctl_name = KERN_SYSRQ, | ||
619 | .procname = "sysrq", | 568 | .procname = "sysrq", |
620 | .data = &__sysrq_enabled, | 569 | .data = &__sysrq_enabled, |
621 | .maxlen = sizeof (int), | 570 | .maxlen = sizeof (int), |
622 | .mode = 0644, | 571 | .mode = 0644, |
623 | .proc_handler = &proc_dointvec, | 572 | .proc_handler = proc_dointvec, |
624 | }, | 573 | }, |
625 | #endif | 574 | #endif |
626 | #ifdef CONFIG_PROC_SYSCTL | 575 | #ifdef CONFIG_PROC_SYSCTL |
@@ -629,215 +578,188 @@ static struct ctl_table kern_table[] = { | |||
629 | .data = NULL, | 578 | .data = NULL, |
630 | .maxlen = sizeof (int), | 579 | .maxlen = sizeof (int), |
631 | .mode = 0600, | 580 | .mode = 0600, |
632 | .proc_handler = &proc_do_cad_pid, | 581 | .proc_handler = proc_do_cad_pid, |
633 | }, | 582 | }, |
634 | #endif | 583 | #endif |
635 | { | 584 | { |
636 | .ctl_name = KERN_MAX_THREADS, | ||
637 | .procname = "threads-max", | 585 | .procname = "threads-max", |
638 | .data = &max_threads, | 586 | .data = &max_threads, |
639 | .maxlen = sizeof(int), | 587 | .maxlen = sizeof(int), |
640 | .mode = 0644, | 588 | .mode = 0644, |
641 | .proc_handler = &proc_dointvec, | 589 | .proc_handler = proc_dointvec, |
642 | }, | 590 | }, |
643 | { | 591 | { |
644 | .ctl_name = KERN_RANDOM, | ||
645 | .procname = "random", | 592 | .procname = "random", |
646 | .mode = 0555, | 593 | .mode = 0555, |
647 | .child = random_table, | 594 | .child = random_table, |
648 | }, | 595 | }, |
649 | { | 596 | { |
650 | .ctl_name = KERN_OVERFLOWUID, | ||
651 | .procname = "overflowuid", | 597 | .procname = "overflowuid", |
652 | .data = &overflowuid, | 598 | .data = &overflowuid, |
653 | .maxlen = sizeof(int), | 599 | .maxlen = sizeof(int), |
654 | .mode = 0644, | 600 | .mode = 0644, |
655 | .proc_handler = &proc_dointvec_minmax, | 601 | .proc_handler = proc_dointvec_minmax, |
656 | .strategy = &sysctl_intvec, | ||
657 | .extra1 = &minolduid, | 602 | .extra1 = &minolduid, |
658 | .extra2 = &maxolduid, | 603 | .extra2 = &maxolduid, |
659 | }, | 604 | }, |
660 | { | 605 | { |
661 | .ctl_name = KERN_OVERFLOWGID, | ||
662 | .procname = "overflowgid", | 606 | .procname = "overflowgid", |
663 | .data = &overflowgid, | 607 | .data = &overflowgid, |
664 | .maxlen = sizeof(int), | 608 | .maxlen = sizeof(int), |
665 | .mode = 0644, | 609 | .mode = 0644, |
666 | .proc_handler = &proc_dointvec_minmax, | 610 | .proc_handler = proc_dointvec_minmax, |
667 | .strategy = &sysctl_intvec, | ||
668 | .extra1 = &minolduid, | 611 | .extra1 = &minolduid, |
669 | .extra2 = &maxolduid, | 612 | .extra2 = &maxolduid, |
670 | }, | 613 | }, |
671 | #ifdef CONFIG_S390 | 614 | #ifdef CONFIG_S390 |
672 | #ifdef CONFIG_MATHEMU | 615 | #ifdef CONFIG_MATHEMU |
673 | { | 616 | { |
674 | .ctl_name = KERN_IEEE_EMULATION_WARNINGS, | ||
675 | .procname = "ieee_emulation_warnings", | 617 | .procname = "ieee_emulation_warnings", |
676 | .data = &sysctl_ieee_emulation_warnings, | 618 | .data = &sysctl_ieee_emulation_warnings, |
677 | .maxlen = sizeof(int), | 619 | .maxlen = sizeof(int), |
678 | .mode = 0644, | 620 | .mode = 0644, |
679 | .proc_handler = &proc_dointvec, | 621 | .proc_handler = proc_dointvec, |
680 | }, | 622 | }, |
681 | #endif | 623 | #endif |
682 | { | 624 | { |
683 | .ctl_name = KERN_S390_USER_DEBUG_LOGGING, | ||
684 | .procname = "userprocess_debug", | 625 | .procname = "userprocess_debug", |
685 | .data = &sysctl_userprocess_debug, | 626 | .data = &sysctl_userprocess_debug, |
686 | .maxlen = sizeof(int), | 627 | .maxlen = sizeof(int), |
687 | .mode = 0644, | 628 | .mode = 0644, |
688 | .proc_handler = &proc_dointvec, | 629 | .proc_handler = proc_dointvec, |
689 | }, | 630 | }, |
690 | #endif | 631 | #endif |
691 | { | 632 | { |
692 | .ctl_name = KERN_PIDMAX, | ||
693 | .procname = "pid_max", | 633 | .procname = "pid_max", |
694 | .data = &pid_max, | 634 | .data = &pid_max, |
695 | .maxlen = sizeof (int), | 635 | .maxlen = sizeof (int), |
696 | .mode = 0644, | 636 | .mode = 0644, |
697 | .proc_handler = &proc_dointvec_minmax, | 637 | .proc_handler = proc_dointvec_minmax, |
698 | .strategy = sysctl_intvec, | ||
699 | .extra1 = &pid_max_min, | 638 | .extra1 = &pid_max_min, |
700 | .extra2 = &pid_max_max, | 639 | .extra2 = &pid_max_max, |
701 | }, | 640 | }, |
702 | { | 641 | { |
703 | .ctl_name = KERN_PANIC_ON_OOPS, | ||
704 | .procname = "panic_on_oops", | 642 | .procname = "panic_on_oops", |
705 | .data = &panic_on_oops, | 643 | .data = &panic_on_oops, |
706 | .maxlen = sizeof(int), | 644 | .maxlen = sizeof(int), |
707 | .mode = 0644, | 645 | .mode = 0644, |
708 | .proc_handler = &proc_dointvec, | 646 | .proc_handler = proc_dointvec, |
709 | }, | 647 | }, |
710 | #if defined CONFIG_PRINTK | 648 | #if defined CONFIG_PRINTK |
711 | { | 649 | { |
712 | .ctl_name = KERN_PRINTK, | ||
713 | .procname = "printk", | 650 | .procname = "printk", |
714 | .data = &console_loglevel, | 651 | .data = &console_loglevel, |
715 | .maxlen = 4*sizeof(int), | 652 | .maxlen = 4*sizeof(int), |
716 | .mode = 0644, | 653 | .mode = 0644, |
717 | .proc_handler = &proc_dointvec, | 654 | .proc_handler = proc_dointvec, |
718 | }, | 655 | }, |
719 | { | 656 | { |
720 | .ctl_name = KERN_PRINTK_RATELIMIT, | ||
721 | .procname = "printk_ratelimit", | 657 | .procname = "printk_ratelimit", |
722 | .data = &printk_ratelimit_state.interval, | 658 | .data = &printk_ratelimit_state.interval, |
723 | .maxlen = sizeof(int), | 659 | .maxlen = sizeof(int), |
724 | .mode = 0644, | 660 | .mode = 0644, |
725 | .proc_handler = &proc_dointvec_jiffies, | 661 | .proc_handler = proc_dointvec_jiffies, |
726 | .strategy = &sysctl_jiffies, | ||
727 | }, | 662 | }, |
728 | { | 663 | { |
729 | .ctl_name = KERN_PRINTK_RATELIMIT_BURST, | ||
730 | .procname = "printk_ratelimit_burst", | 664 | .procname = "printk_ratelimit_burst", |
731 | .data = &printk_ratelimit_state.burst, | 665 | .data = &printk_ratelimit_state.burst, |
732 | .maxlen = sizeof(int), | 666 | .maxlen = sizeof(int), |
733 | .mode = 0644, | 667 | .mode = 0644, |
734 | .proc_handler = &proc_dointvec, | 668 | .proc_handler = proc_dointvec, |
735 | }, | 669 | }, |
736 | { | 670 | { |
737 | .ctl_name = CTL_UNNUMBERED, | ||
738 | .procname = "printk_delay", | 671 | .procname = "printk_delay", |
739 | .data = &printk_delay_msec, | 672 | .data = &printk_delay_msec, |
740 | .maxlen = sizeof(int), | 673 | .maxlen = sizeof(int), |
741 | .mode = 0644, | 674 | .mode = 0644, |
742 | .proc_handler = &proc_dointvec_minmax, | 675 | .proc_handler = proc_dointvec_minmax, |
743 | .strategy = &sysctl_intvec, | ||
744 | .extra1 = &zero, | 676 | .extra1 = &zero, |
745 | .extra2 = &ten_thousand, | 677 | .extra2 = &ten_thousand, |
746 | }, | 678 | }, |
747 | #endif | 679 | #endif |
748 | { | 680 | { |
749 | .ctl_name = KERN_NGROUPS_MAX, | ||
750 | .procname = "ngroups_max", | 681 | .procname = "ngroups_max", |
751 | .data = &ngroups_max, | 682 | .data = &ngroups_max, |
752 | .maxlen = sizeof (int), | 683 | .maxlen = sizeof (int), |
753 | .mode = 0444, | 684 | .mode = 0444, |
754 | .proc_handler = &proc_dointvec, | 685 | .proc_handler = proc_dointvec, |
755 | }, | 686 | }, |
756 | #if defined(CONFIG_X86_LOCAL_APIC) && defined(CONFIG_X86) | 687 | #if defined(CONFIG_X86_LOCAL_APIC) && defined(CONFIG_X86) |
757 | { | 688 | { |
758 | .ctl_name = KERN_UNKNOWN_NMI_PANIC, | ||
759 | .procname = "unknown_nmi_panic", | 689 | .procname = "unknown_nmi_panic", |
760 | .data = &unknown_nmi_panic, | 690 | .data = &unknown_nmi_panic, |
761 | .maxlen = sizeof (int), | 691 | .maxlen = sizeof (int), |
762 | .mode = 0644, | 692 | .mode = 0644, |
763 | .proc_handler = &proc_dointvec, | 693 | .proc_handler = proc_dointvec, |
764 | }, | 694 | }, |
765 | { | 695 | { |
766 | .procname = "nmi_watchdog", | 696 | .procname = "nmi_watchdog", |
767 | .data = &nmi_watchdog_enabled, | 697 | .data = &nmi_watchdog_enabled, |
768 | .maxlen = sizeof (int), | 698 | .maxlen = sizeof (int), |
769 | .mode = 0644, | 699 | .mode = 0644, |
770 | .proc_handler = &proc_nmi_enabled, | 700 | .proc_handler = proc_nmi_enabled, |
771 | }, | 701 | }, |
772 | #endif | 702 | #endif |
773 | #if defined(CONFIG_X86) | 703 | #if defined(CONFIG_X86) |
774 | { | 704 | { |
775 | .ctl_name = KERN_PANIC_ON_NMI, | ||
776 | .procname = "panic_on_unrecovered_nmi", | 705 | .procname = "panic_on_unrecovered_nmi", |
777 | .data = &panic_on_unrecovered_nmi, | 706 | .data = &panic_on_unrecovered_nmi, |
778 | .maxlen = sizeof(int), | 707 | .maxlen = sizeof(int), |
779 | .mode = 0644, | 708 | .mode = 0644, |
780 | .proc_handler = &proc_dointvec, | 709 | .proc_handler = proc_dointvec, |
781 | }, | 710 | }, |
782 | { | 711 | { |
783 | .ctl_name = CTL_UNNUMBERED, | ||
784 | .procname = "panic_on_io_nmi", | 712 | .procname = "panic_on_io_nmi", |
785 | .data = &panic_on_io_nmi, | 713 | .data = &panic_on_io_nmi, |
786 | .maxlen = sizeof(int), | 714 | .maxlen = sizeof(int), |
787 | .mode = 0644, | 715 | .mode = 0644, |
788 | .proc_handler = &proc_dointvec, | 716 | .proc_handler = proc_dointvec, |
789 | }, | 717 | }, |
790 | { | 718 | { |
791 | .ctl_name = KERN_BOOTLOADER_TYPE, | ||
792 | .procname = "bootloader_type", | 719 | .procname = "bootloader_type", |
793 | .data = &bootloader_type, | 720 | .data = &bootloader_type, |
794 | .maxlen = sizeof (int), | 721 | .maxlen = sizeof (int), |
795 | .mode = 0444, | 722 | .mode = 0444, |
796 | .proc_handler = &proc_dointvec, | 723 | .proc_handler = proc_dointvec, |
797 | }, | 724 | }, |
798 | { | 725 | { |
799 | .ctl_name = CTL_UNNUMBERED, | ||
800 | .procname = "bootloader_version", | 726 | .procname = "bootloader_version", |
801 | .data = &bootloader_version, | 727 | .data = &bootloader_version, |
802 | .maxlen = sizeof (int), | 728 | .maxlen = sizeof (int), |
803 | .mode = 0444, | 729 | .mode = 0444, |
804 | .proc_handler = &proc_dointvec, | 730 | .proc_handler = proc_dointvec, |
805 | }, | 731 | }, |
806 | { | 732 | { |
807 | .ctl_name = CTL_UNNUMBERED, | ||
808 | .procname = "kstack_depth_to_print", | 733 | .procname = "kstack_depth_to_print", |
809 | .data = &kstack_depth_to_print, | 734 | .data = &kstack_depth_to_print, |
810 | .maxlen = sizeof(int), | 735 | .maxlen = sizeof(int), |
811 | .mode = 0644, | 736 | .mode = 0644, |
812 | .proc_handler = &proc_dointvec, | 737 | .proc_handler = proc_dointvec, |
813 | }, | 738 | }, |
814 | { | 739 | { |
815 | .ctl_name = CTL_UNNUMBERED, | ||
816 | .procname = "io_delay_type", | 740 | .procname = "io_delay_type", |
817 | .data = &io_delay_type, | 741 | .data = &io_delay_type, |
818 | .maxlen = sizeof(int), | 742 | .maxlen = sizeof(int), |
819 | .mode = 0644, | 743 | .mode = 0644, |
820 | .proc_handler = &proc_dointvec, | 744 | .proc_handler = proc_dointvec, |
821 | }, | 745 | }, |
822 | #endif | 746 | #endif |
823 | #if defined(CONFIG_MMU) | 747 | #if defined(CONFIG_MMU) |
824 | { | 748 | { |
825 | .ctl_name = KERN_RANDOMIZE, | ||
826 | .procname = "randomize_va_space", | 749 | .procname = "randomize_va_space", |
827 | .data = &randomize_va_space, | 750 | .data = &randomize_va_space, |
828 | .maxlen = sizeof(int), | 751 | .maxlen = sizeof(int), |
829 | .mode = 0644, | 752 | .mode = 0644, |
830 | .proc_handler = &proc_dointvec, | 753 | .proc_handler = proc_dointvec, |
831 | }, | 754 | }, |
832 | #endif | 755 | #endif |
833 | #if defined(CONFIG_S390) && defined(CONFIG_SMP) | 756 | #if defined(CONFIG_S390) && defined(CONFIG_SMP) |
834 | { | 757 | { |
835 | .ctl_name = KERN_SPIN_RETRY, | ||
836 | .procname = "spin_retry", | 758 | .procname = "spin_retry", |
837 | .data = &spin_retry, | 759 | .data = &spin_retry, |
838 | .maxlen = sizeof (int), | 760 | .maxlen = sizeof (int), |
839 | .mode = 0644, | 761 | .mode = 0644, |
840 | .proc_handler = &proc_dointvec, | 762 | .proc_handler = proc_dointvec, |
841 | }, | 763 | }, |
842 | #endif | 764 | #endif |
843 | #if defined(CONFIG_ACPI_SLEEP) && defined(CONFIG_X86) | 765 | #if defined(CONFIG_ACPI_SLEEP) && defined(CONFIG_X86) |
@@ -846,123 +768,104 @@ static struct ctl_table kern_table[] = { | |||
846 | .data = &acpi_realmode_flags, | 768 | .data = &acpi_realmode_flags, |
847 | .maxlen = sizeof (unsigned long), | 769 | .maxlen = sizeof (unsigned long), |
848 | .mode = 0644, | 770 | .mode = 0644, |
849 | .proc_handler = &proc_doulongvec_minmax, | 771 | .proc_handler = proc_doulongvec_minmax, |
850 | }, | 772 | }, |
851 | #endif | 773 | #endif |
852 | #ifdef CONFIG_IA64 | 774 | #ifdef CONFIG_IA64 |
853 | { | 775 | { |
854 | .ctl_name = KERN_IA64_UNALIGNED, | ||
855 | .procname = "ignore-unaligned-usertrap", | 776 | .procname = "ignore-unaligned-usertrap", |
856 | .data = &no_unaligned_warning, | 777 | .data = &no_unaligned_warning, |
857 | .maxlen = sizeof (int), | 778 | .maxlen = sizeof (int), |
858 | .mode = 0644, | 779 | .mode = 0644, |
859 | .proc_handler = &proc_dointvec, | 780 | .proc_handler = proc_dointvec, |
860 | }, | 781 | }, |
861 | { | 782 | { |
862 | .ctl_name = CTL_UNNUMBERED, | ||
863 | .procname = "unaligned-dump-stack", | 783 | .procname = "unaligned-dump-stack", |
864 | .data = &unaligned_dump_stack, | 784 | .data = &unaligned_dump_stack, |
865 | .maxlen = sizeof (int), | 785 | .maxlen = sizeof (int), |
866 | .mode = 0644, | 786 | .mode = 0644, |
867 | .proc_handler = &proc_dointvec, | 787 | .proc_handler = proc_dointvec, |
868 | }, | 788 | }, |
869 | #endif | 789 | #endif |
870 | #ifdef CONFIG_DETECT_SOFTLOCKUP | 790 | #ifdef CONFIG_DETECT_SOFTLOCKUP |
871 | { | 791 | { |
872 | .ctl_name = CTL_UNNUMBERED, | ||
873 | .procname = "softlockup_panic", | 792 | .procname = "softlockup_panic", |
874 | .data = &softlockup_panic, | 793 | .data = &softlockup_panic, |
875 | .maxlen = sizeof(int), | 794 | .maxlen = sizeof(int), |
876 | .mode = 0644, | 795 | .mode = 0644, |
877 | .proc_handler = &proc_dointvec_minmax, | 796 | .proc_handler = proc_dointvec_minmax, |
878 | .strategy = &sysctl_intvec, | ||
879 | .extra1 = &zero, | 797 | .extra1 = &zero, |
880 | .extra2 = &one, | 798 | .extra2 = &one, |
881 | }, | 799 | }, |
882 | { | 800 | { |
883 | .ctl_name = CTL_UNNUMBERED, | ||
884 | .procname = "softlockup_thresh", | 801 | .procname = "softlockup_thresh", |
885 | .data = &softlockup_thresh, | 802 | .data = &softlockup_thresh, |
886 | .maxlen = sizeof(int), | 803 | .maxlen = sizeof(int), |
887 | .mode = 0644, | 804 | .mode = 0644, |
888 | .proc_handler = &proc_dosoftlockup_thresh, | 805 | .proc_handler = proc_dosoftlockup_thresh, |
889 | .strategy = &sysctl_intvec, | ||
890 | .extra1 = &neg_one, | 806 | .extra1 = &neg_one, |
891 | .extra2 = &sixty, | 807 | .extra2 = &sixty, |
892 | }, | 808 | }, |
893 | #endif | 809 | #endif |
894 | #ifdef CONFIG_DETECT_HUNG_TASK | 810 | #ifdef CONFIG_DETECT_HUNG_TASK |
895 | { | 811 | { |
896 | .ctl_name = CTL_UNNUMBERED, | ||
897 | .procname = "hung_task_panic", | 812 | .procname = "hung_task_panic", |
898 | .data = &sysctl_hung_task_panic, | 813 | .data = &sysctl_hung_task_panic, |
899 | .maxlen = sizeof(int), | 814 | .maxlen = sizeof(int), |
900 | .mode = 0644, | 815 | .mode = 0644, |
901 | .proc_handler = &proc_dointvec_minmax, | 816 | .proc_handler = proc_dointvec_minmax, |
902 | .strategy = &sysctl_intvec, | ||
903 | .extra1 = &zero, | 817 | .extra1 = &zero, |
904 | .extra2 = &one, | 818 | .extra2 = &one, |
905 | }, | 819 | }, |
906 | { | 820 | { |
907 | .ctl_name = CTL_UNNUMBERED, | ||
908 | .procname = "hung_task_check_count", | 821 | .procname = "hung_task_check_count", |
909 | .data = &sysctl_hung_task_check_count, | 822 | .data = &sysctl_hung_task_check_count, |
910 | .maxlen = sizeof(unsigned long), | 823 | .maxlen = sizeof(unsigned long), |
911 | .mode = 0644, | 824 | .mode = 0644, |
912 | .proc_handler = &proc_doulongvec_minmax, | 825 | .proc_handler = proc_doulongvec_minmax, |
913 | .strategy = &sysctl_intvec, | ||
914 | }, | 826 | }, |
915 | { | 827 | { |
916 | .ctl_name = CTL_UNNUMBERED, | ||
917 | .procname = "hung_task_timeout_secs", | 828 | .procname = "hung_task_timeout_secs", |
918 | .data = &sysctl_hung_task_timeout_secs, | 829 | .data = &sysctl_hung_task_timeout_secs, |
919 | .maxlen = sizeof(unsigned long), | 830 | .maxlen = sizeof(unsigned long), |
920 | .mode = 0644, | 831 | .mode = 0644, |
921 | .proc_handler = &proc_dohung_task_timeout_secs, | 832 | .proc_handler = proc_dohung_task_timeout_secs, |
922 | .strategy = &sysctl_intvec, | ||
923 | }, | 833 | }, |
924 | { | 834 | { |
925 | .ctl_name = CTL_UNNUMBERED, | ||
926 | .procname = "hung_task_warnings", | 835 | .procname = "hung_task_warnings", |
927 | .data = &sysctl_hung_task_warnings, | 836 | .data = &sysctl_hung_task_warnings, |
928 | .maxlen = sizeof(unsigned long), | 837 | .maxlen = sizeof(unsigned long), |
929 | .mode = 0644, | 838 | .mode = 0644, |
930 | .proc_handler = &proc_doulongvec_minmax, | 839 | .proc_handler = proc_doulongvec_minmax, |
931 | .strategy = &sysctl_intvec, | ||
932 | }, | 840 | }, |
933 | #endif | 841 | #endif |
934 | #ifdef CONFIG_COMPAT | 842 | #ifdef CONFIG_COMPAT |
935 | { | 843 | { |
936 | .ctl_name = KERN_COMPAT_LOG, | ||
937 | .procname = "compat-log", | 844 | .procname = "compat-log", |
938 | .data = &compat_log, | 845 | .data = &compat_log, |
939 | .maxlen = sizeof (int), | 846 | .maxlen = sizeof (int), |
940 | .mode = 0644, | 847 | .mode = 0644, |
941 | .proc_handler = &proc_dointvec, | 848 | .proc_handler = proc_dointvec, |
942 | }, | 849 | }, |
943 | #endif | 850 | #endif |
944 | #ifdef CONFIG_RT_MUTEXES | 851 | #ifdef CONFIG_RT_MUTEXES |
945 | { | 852 | { |
946 | .ctl_name = KERN_MAX_LOCK_DEPTH, | ||
947 | .procname = "max_lock_depth", | 853 | .procname = "max_lock_depth", |
948 | .data = &max_lock_depth, | 854 | .data = &max_lock_depth, |
949 | .maxlen = sizeof(int), | 855 | .maxlen = sizeof(int), |
950 | .mode = 0644, | 856 | .mode = 0644, |
951 | .proc_handler = &proc_dointvec, | 857 | .proc_handler = proc_dointvec, |
952 | }, | 858 | }, |
953 | #endif | 859 | #endif |
954 | { | 860 | { |
955 | .ctl_name = CTL_UNNUMBERED, | ||
956 | .procname = "poweroff_cmd", | 861 | .procname = "poweroff_cmd", |
957 | .data = &poweroff_cmd, | 862 | .data = &poweroff_cmd, |
958 | .maxlen = POWEROFF_CMD_PATH_LEN, | 863 | .maxlen = POWEROFF_CMD_PATH_LEN, |
959 | .mode = 0644, | 864 | .mode = 0644, |
960 | .proc_handler = &proc_dostring, | 865 | .proc_handler = proc_dostring, |
961 | .strategy = &sysctl_string, | ||
962 | }, | 866 | }, |
963 | #ifdef CONFIG_KEYS | 867 | #ifdef CONFIG_KEYS |
964 | { | 868 | { |
965 | .ctl_name = CTL_UNNUMBERED, | ||
966 | .procname = "keys", | 869 | .procname = "keys", |
967 | .mode = 0555, | 870 | .mode = 0555, |
968 | .child = key_sysctls, | 871 | .child = key_sysctls, |
@@ -970,17 +873,15 @@ static struct ctl_table kern_table[] = { | |||
970 | #endif | 873 | #endif |
971 | #ifdef CONFIG_RCU_TORTURE_TEST | 874 | #ifdef CONFIG_RCU_TORTURE_TEST |
972 | { | 875 | { |
973 | .ctl_name = CTL_UNNUMBERED, | ||
974 | .procname = "rcutorture_runnable", | 876 | .procname = "rcutorture_runnable", |
975 | .data = &rcutorture_runnable, | 877 | .data = &rcutorture_runnable, |
976 | .maxlen = sizeof(int), | 878 | .maxlen = sizeof(int), |
977 | .mode = 0644, | 879 | .mode = 0644, |
978 | .proc_handler = &proc_dointvec, | 880 | .proc_handler = proc_dointvec, |
979 | }, | 881 | }, |
980 | #endif | 882 | #endif |
981 | #ifdef CONFIG_SLOW_WORK | 883 | #ifdef CONFIG_SLOW_WORK |
982 | { | 884 | { |
983 | .ctl_name = CTL_UNNUMBERED, | ||
984 | .procname = "slow-work", | 885 | .procname = "slow-work", |
985 | .mode = 0555, | 886 | .mode = 0555, |
986 | .child = slow_work_sysctls, | 887 | .child = slow_work_sysctls, |
@@ -988,146 +889,127 @@ static struct ctl_table kern_table[] = { | |||
988 | #endif | 889 | #endif |
989 | #ifdef CONFIG_PERF_EVENTS | 890 | #ifdef CONFIG_PERF_EVENTS |
990 | { | 891 | { |
991 | .ctl_name = CTL_UNNUMBERED, | ||
992 | .procname = "perf_event_paranoid", | 892 | .procname = "perf_event_paranoid", |
993 | .data = &sysctl_perf_event_paranoid, | 893 | .data = &sysctl_perf_event_paranoid, |
994 | .maxlen = sizeof(sysctl_perf_event_paranoid), | 894 | .maxlen = sizeof(sysctl_perf_event_paranoid), |
995 | .mode = 0644, | 895 | .mode = 0644, |
996 | .proc_handler = &proc_dointvec, | 896 | .proc_handler = proc_dointvec, |
997 | }, | 897 | }, |
998 | { | 898 | { |
999 | .ctl_name = CTL_UNNUMBERED, | ||
1000 | .procname = "perf_event_mlock_kb", | 899 | .procname = "perf_event_mlock_kb", |
1001 | .data = &sysctl_perf_event_mlock, | 900 | .data = &sysctl_perf_event_mlock, |
1002 | .maxlen = sizeof(sysctl_perf_event_mlock), | 901 | .maxlen = sizeof(sysctl_perf_event_mlock), |
1003 | .mode = 0644, | 902 | .mode = 0644, |
1004 | .proc_handler = &proc_dointvec, | 903 | .proc_handler = proc_dointvec, |
1005 | }, | 904 | }, |
1006 | { | 905 | { |
1007 | .ctl_name = CTL_UNNUMBERED, | ||
1008 | .procname = "perf_event_max_sample_rate", | 906 | .procname = "perf_event_max_sample_rate", |
1009 | .data = &sysctl_perf_event_sample_rate, | 907 | .data = &sysctl_perf_event_sample_rate, |
1010 | .maxlen = sizeof(sysctl_perf_event_sample_rate), | 908 | .maxlen = sizeof(sysctl_perf_event_sample_rate), |
1011 | .mode = 0644, | 909 | .mode = 0644, |
1012 | .proc_handler = &proc_dointvec, | 910 | .proc_handler = proc_dointvec, |
1013 | }, | 911 | }, |
1014 | #endif | 912 | #endif |
1015 | #ifdef CONFIG_KMEMCHECK | 913 | #ifdef CONFIG_KMEMCHECK |
1016 | { | 914 | { |
1017 | .ctl_name = CTL_UNNUMBERED, | ||
1018 | .procname = "kmemcheck", | 915 | .procname = "kmemcheck", |
1019 | .data = &kmemcheck_enabled, | 916 | .data = &kmemcheck_enabled, |
1020 | .maxlen = sizeof(int), | 917 | .maxlen = sizeof(int), |
1021 | .mode = 0644, | 918 | .mode = 0644, |
1022 | .proc_handler = &proc_dointvec, | 919 | .proc_handler = proc_dointvec, |
1023 | }, | 920 | }, |
1024 | #endif | 921 | #endif |
1025 | #ifdef CONFIG_BLOCK | 922 | #ifdef CONFIG_BLOCK |
1026 | { | 923 | { |
1027 | .ctl_name = CTL_UNNUMBERED, | ||
1028 | .procname = "blk_iopoll", | 924 | .procname = "blk_iopoll", |
1029 | .data = &blk_iopoll_enabled, | 925 | .data = &blk_iopoll_enabled, |
1030 | .maxlen = sizeof(int), | 926 | .maxlen = sizeof(int), |
1031 | .mode = 0644, | 927 | .mode = 0644, |
1032 | .proc_handler = &proc_dointvec, | 928 | .proc_handler = proc_dointvec, |
1033 | }, | 929 | }, |
1034 | #endif | 930 | #endif |
1035 | /* | 931 | /* |
1036 | * NOTE: do not add new entries to this table unless you have read | 932 | * NOTE: do not add new entries to this table unless you have read |
1037 | * Documentation/sysctl/ctl_unnumbered.txt | 933 | * Documentation/sysctl/ctl_unnumbered.txt |
1038 | */ | 934 | */ |
1039 | { .ctl_name = 0 } | 935 | { } |
1040 | }; | 936 | }; |
1041 | 937 | ||
1042 | static struct ctl_table vm_table[] = { | 938 | static struct ctl_table vm_table[] = { |
1043 | { | 939 | { |
1044 | .ctl_name = VM_OVERCOMMIT_MEMORY, | ||
1045 | .procname = "overcommit_memory", | 940 | .procname = "overcommit_memory", |
1046 | .data = &sysctl_overcommit_memory, | 941 | .data = &sysctl_overcommit_memory, |
1047 | .maxlen = sizeof(sysctl_overcommit_memory), | 942 | .maxlen = sizeof(sysctl_overcommit_memory), |
1048 | .mode = 0644, | 943 | .mode = 0644, |
1049 | .proc_handler = &proc_dointvec, | 944 | .proc_handler = proc_dointvec, |
1050 | }, | 945 | }, |
1051 | { | 946 | { |
1052 | .ctl_name = VM_PANIC_ON_OOM, | ||
1053 | .procname = "panic_on_oom", | 947 | .procname = "panic_on_oom", |
1054 | .data = &sysctl_panic_on_oom, | 948 | .data = &sysctl_panic_on_oom, |
1055 | .maxlen = sizeof(sysctl_panic_on_oom), | 949 | .maxlen = sizeof(sysctl_panic_on_oom), |
1056 | .mode = 0644, | 950 | .mode = 0644, |
1057 | .proc_handler = &proc_dointvec, | 951 | .proc_handler = proc_dointvec, |
1058 | }, | 952 | }, |
1059 | { | 953 | { |
1060 | .ctl_name = CTL_UNNUMBERED, | ||
1061 | .procname = "oom_kill_allocating_task", | 954 | .procname = "oom_kill_allocating_task", |
1062 | .data = &sysctl_oom_kill_allocating_task, | 955 | .data = &sysctl_oom_kill_allocating_task, |
1063 | .maxlen = sizeof(sysctl_oom_kill_allocating_task), | 956 | .maxlen = sizeof(sysctl_oom_kill_allocating_task), |
1064 | .mode = 0644, | 957 | .mode = 0644, |
1065 | .proc_handler = &proc_dointvec, | 958 | .proc_handler = proc_dointvec, |
1066 | }, | 959 | }, |
1067 | { | 960 | { |
1068 | .ctl_name = CTL_UNNUMBERED, | ||
1069 | .procname = "oom_dump_tasks", | 961 | .procname = "oom_dump_tasks", |
1070 | .data = &sysctl_oom_dump_tasks, | 962 | .data = &sysctl_oom_dump_tasks, |
1071 | .maxlen = sizeof(sysctl_oom_dump_tasks), | 963 | .maxlen = sizeof(sysctl_oom_dump_tasks), |
1072 | .mode = 0644, | 964 | .mode = 0644, |
1073 | .proc_handler = &proc_dointvec, | 965 | .proc_handler = proc_dointvec, |
1074 | }, | 966 | }, |
1075 | { | 967 | { |
1076 | .ctl_name = VM_OVERCOMMIT_RATIO, | ||
1077 | .procname = "overcommit_ratio", | 968 | .procname = "overcommit_ratio", |
1078 | .data = &sysctl_overcommit_ratio, | 969 | .data = &sysctl_overcommit_ratio, |
1079 | .maxlen = sizeof(sysctl_overcommit_ratio), | 970 | .maxlen = sizeof(sysctl_overcommit_ratio), |
1080 | .mode = 0644, | 971 | .mode = 0644, |
1081 | .proc_handler = &proc_dointvec, | 972 | .proc_handler = proc_dointvec, |
1082 | }, | 973 | }, |
1083 | { | 974 | { |
1084 | .ctl_name = VM_PAGE_CLUSTER, | ||
1085 | .procname = "page-cluster", | 975 | .procname = "page-cluster", |
1086 | .data = &page_cluster, | 976 | .data = &page_cluster, |
1087 | .maxlen = sizeof(int), | 977 | .maxlen = sizeof(int), |
1088 | .mode = 0644, | 978 | .mode = 0644, |
1089 | .proc_handler = &proc_dointvec, | 979 | .proc_handler = proc_dointvec, |
1090 | }, | 980 | }, |
1091 | { | 981 | { |
1092 | .ctl_name = VM_DIRTY_BACKGROUND, | ||
1093 | .procname = "dirty_background_ratio", | 982 | .procname = "dirty_background_ratio", |
1094 | .data = &dirty_background_ratio, | 983 | .data = &dirty_background_ratio, |
1095 | .maxlen = sizeof(dirty_background_ratio), | 984 | .maxlen = sizeof(dirty_background_ratio), |
1096 | .mode = 0644, | 985 | .mode = 0644, |
1097 | .proc_handler = &dirty_background_ratio_handler, | 986 | .proc_handler = dirty_background_ratio_handler, |
1098 | .strategy = &sysctl_intvec, | ||
1099 | .extra1 = &zero, | 987 | .extra1 = &zero, |
1100 | .extra2 = &one_hundred, | 988 | .extra2 = &one_hundred, |
1101 | }, | 989 | }, |
1102 | { | 990 | { |
1103 | .ctl_name = CTL_UNNUMBERED, | ||
1104 | .procname = "dirty_background_bytes", | 991 | .procname = "dirty_background_bytes", |
1105 | .data = &dirty_background_bytes, | 992 | .data = &dirty_background_bytes, |
1106 | .maxlen = sizeof(dirty_background_bytes), | 993 | .maxlen = sizeof(dirty_background_bytes), |
1107 | .mode = 0644, | 994 | .mode = 0644, |
1108 | .proc_handler = &dirty_background_bytes_handler, | 995 | .proc_handler = dirty_background_bytes_handler, |
1109 | .strategy = &sysctl_intvec, | ||
1110 | .extra1 = &one_ul, | 996 | .extra1 = &one_ul, |
1111 | }, | 997 | }, |
1112 | { | 998 | { |
1113 | .ctl_name = VM_DIRTY_RATIO, | ||
1114 | .procname = "dirty_ratio", | 999 | .procname = "dirty_ratio", |
1115 | .data = &vm_dirty_ratio, | 1000 | .data = &vm_dirty_ratio, |
1116 | .maxlen = sizeof(vm_dirty_ratio), | 1001 | .maxlen = sizeof(vm_dirty_ratio), |
1117 | .mode = 0644, | 1002 | .mode = 0644, |
1118 | .proc_handler = &dirty_ratio_handler, | 1003 | .proc_handler = dirty_ratio_handler, |
1119 | .strategy = &sysctl_intvec, | ||
1120 | .extra1 = &zero, | 1004 | .extra1 = &zero, |
1121 | .extra2 = &one_hundred, | 1005 | .extra2 = &one_hundred, |
1122 | }, | 1006 | }, |
1123 | { | 1007 | { |
1124 | .ctl_name = CTL_UNNUMBERED, | ||
1125 | .procname = "dirty_bytes", | 1008 | .procname = "dirty_bytes", |
1126 | .data = &vm_dirty_bytes, | 1009 | .data = &vm_dirty_bytes, |
1127 | .maxlen = sizeof(vm_dirty_bytes), | 1010 | .maxlen = sizeof(vm_dirty_bytes), |
1128 | .mode = 0644, | 1011 | .mode = 0644, |
1129 | .proc_handler = &dirty_bytes_handler, | 1012 | .proc_handler = dirty_bytes_handler, |
1130 | .strategy = &sysctl_intvec, | ||
1131 | .extra1 = &dirty_bytes_min, | 1013 | .extra1 = &dirty_bytes_min, |
1132 | }, | 1014 | }, |
1133 | { | 1015 | { |
@@ -1135,31 +1017,28 @@ static struct ctl_table vm_table[] = { | |||
1135 | .data = &dirty_writeback_interval, | 1017 | .data = &dirty_writeback_interval, |
1136 | .maxlen = sizeof(dirty_writeback_interval), | 1018 | .maxlen = sizeof(dirty_writeback_interval), |
1137 | .mode = 0644, | 1019 | .mode = 0644, |
1138 | .proc_handler = &dirty_writeback_centisecs_handler, | 1020 | .proc_handler = dirty_writeback_centisecs_handler, |
1139 | }, | 1021 | }, |
1140 | { | 1022 | { |
1141 | .procname = "dirty_expire_centisecs", | 1023 | .procname = "dirty_expire_centisecs", |
1142 | .data = &dirty_expire_interval, | 1024 | .data = &dirty_expire_interval, |
1143 | .maxlen = sizeof(dirty_expire_interval), | 1025 | .maxlen = sizeof(dirty_expire_interval), |
1144 | .mode = 0644, | 1026 | .mode = 0644, |
1145 | .proc_handler = &proc_dointvec, | 1027 | .proc_handler = proc_dointvec, |
1146 | }, | 1028 | }, |
1147 | { | 1029 | { |
1148 | .ctl_name = VM_NR_PDFLUSH_THREADS, | ||
1149 | .procname = "nr_pdflush_threads", | 1030 | .procname = "nr_pdflush_threads", |
1150 | .data = &nr_pdflush_threads, | 1031 | .data = &nr_pdflush_threads, |
1151 | .maxlen = sizeof nr_pdflush_threads, | 1032 | .maxlen = sizeof nr_pdflush_threads, |
1152 | .mode = 0444 /* read-only*/, | 1033 | .mode = 0444 /* read-only*/, |
1153 | .proc_handler = &proc_dointvec, | 1034 | .proc_handler = proc_dointvec, |
1154 | }, | 1035 | }, |
1155 | { | 1036 | { |
1156 | .ctl_name = VM_SWAPPINESS, | ||
1157 | .procname = "swappiness", | 1037 | .procname = "swappiness", |
1158 | .data = &vm_swappiness, | 1038 | .data = &vm_swappiness, |
1159 | .maxlen = sizeof(vm_swappiness), | 1039 | .maxlen = sizeof(vm_swappiness), |
1160 | .mode = 0644, | 1040 | .mode = 0644, |
1161 | .proc_handler = &proc_dointvec_minmax, | 1041 | .proc_handler = proc_dointvec_minmax, |
1162 | .strategy = &sysctl_intvec, | ||
1163 | .extra1 = &zero, | 1042 | .extra1 = &zero, |
1164 | .extra2 = &one_hundred, | 1043 | .extra2 = &one_hundred, |
1165 | }, | 1044 | }, |
@@ -1169,255 +1048,213 @@ static struct ctl_table vm_table[] = { | |||
1169 | .data = NULL, | 1048 | .data = NULL, |
1170 | .maxlen = sizeof(unsigned long), | 1049 | .maxlen = sizeof(unsigned long), |
1171 | .mode = 0644, | 1050 | .mode = 0644, |
1172 | .proc_handler = &hugetlb_sysctl_handler, | 1051 | .proc_handler = hugetlb_sysctl_handler, |
1173 | .extra1 = (void *)&hugetlb_zero, | 1052 | .extra1 = (void *)&hugetlb_zero, |
1174 | .extra2 = (void *)&hugetlb_infinity, | 1053 | .extra2 = (void *)&hugetlb_infinity, |
1175 | }, | 1054 | }, |
1176 | { | 1055 | { |
1177 | .ctl_name = VM_HUGETLB_GROUP, | ||
1178 | .procname = "hugetlb_shm_group", | 1056 | .procname = "hugetlb_shm_group", |
1179 | .data = &sysctl_hugetlb_shm_group, | 1057 | .data = &sysctl_hugetlb_shm_group, |
1180 | .maxlen = sizeof(gid_t), | 1058 | .maxlen = sizeof(gid_t), |
1181 | .mode = 0644, | 1059 | .mode = 0644, |
1182 | .proc_handler = &proc_dointvec, | 1060 | .proc_handler = proc_dointvec, |
1183 | }, | 1061 | }, |
1184 | { | 1062 | { |
1185 | .ctl_name = CTL_UNNUMBERED, | ||
1186 | .procname = "hugepages_treat_as_movable", | 1063 | .procname = "hugepages_treat_as_movable", |
1187 | .data = &hugepages_treat_as_movable, | 1064 | .data = &hugepages_treat_as_movable, |
1188 | .maxlen = sizeof(int), | 1065 | .maxlen = sizeof(int), |
1189 | .mode = 0644, | 1066 | .mode = 0644, |
1190 | .proc_handler = &hugetlb_treat_movable_handler, | 1067 | .proc_handler = hugetlb_treat_movable_handler, |
1191 | }, | 1068 | }, |
1192 | { | 1069 | { |
1193 | .ctl_name = CTL_UNNUMBERED, | ||
1194 | .procname = "nr_overcommit_hugepages", | 1070 | .procname = "nr_overcommit_hugepages", |
1195 | .data = NULL, | 1071 | .data = NULL, |
1196 | .maxlen = sizeof(unsigned long), | 1072 | .maxlen = sizeof(unsigned long), |
1197 | .mode = 0644, | 1073 | .mode = 0644, |
1198 | .proc_handler = &hugetlb_overcommit_handler, | 1074 | .proc_handler = hugetlb_overcommit_handler, |
1199 | .extra1 = (void *)&hugetlb_zero, | 1075 | .extra1 = (void *)&hugetlb_zero, |
1200 | .extra2 = (void *)&hugetlb_infinity, | 1076 | .extra2 = (void *)&hugetlb_infinity, |
1201 | }, | 1077 | }, |
1202 | #endif | 1078 | #endif |
1203 | { | 1079 | { |
1204 | .ctl_name = VM_LOWMEM_RESERVE_RATIO, | ||
1205 | .procname = "lowmem_reserve_ratio", | 1080 | .procname = "lowmem_reserve_ratio", |
1206 | .data = &sysctl_lowmem_reserve_ratio, | 1081 | .data = &sysctl_lowmem_reserve_ratio, |
1207 | .maxlen = sizeof(sysctl_lowmem_reserve_ratio), | 1082 | .maxlen = sizeof(sysctl_lowmem_reserve_ratio), |
1208 | .mode = 0644, | 1083 | .mode = 0644, |
1209 | .proc_handler = &lowmem_reserve_ratio_sysctl_handler, | 1084 | .proc_handler = lowmem_reserve_ratio_sysctl_handler, |
1210 | .strategy = &sysctl_intvec, | ||
1211 | }, | 1085 | }, |
1212 | { | 1086 | { |
1213 | .ctl_name = VM_DROP_PAGECACHE, | ||
1214 | .procname = "drop_caches", | 1087 | .procname = "drop_caches", |
1215 | .data = &sysctl_drop_caches, | 1088 | .data = &sysctl_drop_caches, |
1216 | .maxlen = sizeof(int), | 1089 | .maxlen = sizeof(int), |
1217 | .mode = 0644, | 1090 | .mode = 0644, |
1218 | .proc_handler = drop_caches_sysctl_handler, | 1091 | .proc_handler = drop_caches_sysctl_handler, |
1219 | .strategy = &sysctl_intvec, | ||
1220 | }, | 1092 | }, |
1221 | { | 1093 | { |
1222 | .ctl_name = VM_MIN_FREE_KBYTES, | ||
1223 | .procname = "min_free_kbytes", | 1094 | .procname = "min_free_kbytes", |
1224 | .data = &min_free_kbytes, | 1095 | .data = &min_free_kbytes, |
1225 | .maxlen = sizeof(min_free_kbytes), | 1096 | .maxlen = sizeof(min_free_kbytes), |
1226 | .mode = 0644, | 1097 | .mode = 0644, |
1227 | .proc_handler = &min_free_kbytes_sysctl_handler, | 1098 | .proc_handler = min_free_kbytes_sysctl_handler, |
1228 | .strategy = &sysctl_intvec, | ||
1229 | .extra1 = &zero, | 1099 | .extra1 = &zero, |
1230 | }, | 1100 | }, |
1231 | { | 1101 | { |
1232 | .ctl_name = VM_PERCPU_PAGELIST_FRACTION, | ||
1233 | .procname = "percpu_pagelist_fraction", | 1102 | .procname = "percpu_pagelist_fraction", |
1234 | .data = &percpu_pagelist_fraction, | 1103 | .data = &percpu_pagelist_fraction, |
1235 | .maxlen = sizeof(percpu_pagelist_fraction), | 1104 | .maxlen = sizeof(percpu_pagelist_fraction), |
1236 | .mode = 0644, | 1105 | .mode = 0644, |
1237 | .proc_handler = &percpu_pagelist_fraction_sysctl_handler, | 1106 | .proc_handler = percpu_pagelist_fraction_sysctl_handler, |
1238 | .strategy = &sysctl_intvec, | ||
1239 | .extra1 = &min_percpu_pagelist_fract, | 1107 | .extra1 = &min_percpu_pagelist_fract, |
1240 | }, | 1108 | }, |
1241 | #ifdef CONFIG_MMU | 1109 | #ifdef CONFIG_MMU |
1242 | { | 1110 | { |
1243 | .ctl_name = VM_MAX_MAP_COUNT, | ||
1244 | .procname = "max_map_count", | 1111 | .procname = "max_map_count", |
1245 | .data = &sysctl_max_map_count, | 1112 | .data = &sysctl_max_map_count, |
1246 | .maxlen = sizeof(sysctl_max_map_count), | 1113 | .maxlen = sizeof(sysctl_max_map_count), |
1247 | .mode = 0644, | 1114 | .mode = 0644, |
1248 | .proc_handler = &proc_dointvec | 1115 | .proc_handler = proc_dointvec |
1249 | }, | 1116 | }, |
1250 | #else | 1117 | #else |
1251 | { | 1118 | { |
1252 | .ctl_name = CTL_UNNUMBERED, | ||
1253 | .procname = "nr_trim_pages", | 1119 | .procname = "nr_trim_pages", |
1254 | .data = &sysctl_nr_trim_pages, | 1120 | .data = &sysctl_nr_trim_pages, |
1255 | .maxlen = sizeof(sysctl_nr_trim_pages), | 1121 | .maxlen = sizeof(sysctl_nr_trim_pages), |
1256 | .mode = 0644, | 1122 | .mode = 0644, |
1257 | .proc_handler = &proc_dointvec_minmax, | 1123 | .proc_handler = proc_dointvec_minmax, |
1258 | .strategy = &sysctl_intvec, | ||
1259 | .extra1 = &zero, | 1124 | .extra1 = &zero, |
1260 | }, | 1125 | }, |
1261 | #endif | 1126 | #endif |
1262 | { | 1127 | { |
1263 | .ctl_name = VM_LAPTOP_MODE, | ||
1264 | .procname = "laptop_mode", | 1128 | .procname = "laptop_mode", |
1265 | .data = &laptop_mode, | 1129 | .data = &laptop_mode, |
1266 | .maxlen = sizeof(laptop_mode), | 1130 | .maxlen = sizeof(laptop_mode), |
1267 | .mode = 0644, | 1131 | .mode = 0644, |
1268 | .proc_handler = &proc_dointvec_jiffies, | 1132 | .proc_handler = proc_dointvec_jiffies, |
1269 | .strategy = &sysctl_jiffies, | ||
1270 | }, | 1133 | }, |
1271 | { | 1134 | { |
1272 | .ctl_name = VM_BLOCK_DUMP, | ||
1273 | .procname = "block_dump", | 1135 | .procname = "block_dump", |
1274 | .data = &block_dump, | 1136 | .data = &block_dump, |
1275 | .maxlen = sizeof(block_dump), | 1137 | .maxlen = sizeof(block_dump), |
1276 | .mode = 0644, | 1138 | .mode = 0644, |
1277 | .proc_handler = &proc_dointvec, | 1139 | .proc_handler = proc_dointvec, |
1278 | .strategy = &sysctl_intvec, | ||
1279 | .extra1 = &zero, | 1140 | .extra1 = &zero, |
1280 | }, | 1141 | }, |
1281 | { | 1142 | { |
1282 | .ctl_name = VM_VFS_CACHE_PRESSURE, | ||
1283 | .procname = "vfs_cache_pressure", | 1143 | .procname = "vfs_cache_pressure", |
1284 | .data = &sysctl_vfs_cache_pressure, | 1144 | .data = &sysctl_vfs_cache_pressure, |
1285 | .maxlen = sizeof(sysctl_vfs_cache_pressure), | 1145 | .maxlen = sizeof(sysctl_vfs_cache_pressure), |
1286 | .mode = 0644, | 1146 | .mode = 0644, |
1287 | .proc_handler = &proc_dointvec, | 1147 | .proc_handler = proc_dointvec, |
1288 | .strategy = &sysctl_intvec, | ||
1289 | .extra1 = &zero, | 1148 | .extra1 = &zero, |
1290 | }, | 1149 | }, |
1291 | #ifdef HAVE_ARCH_PICK_MMAP_LAYOUT | 1150 | #ifdef HAVE_ARCH_PICK_MMAP_LAYOUT |
1292 | { | 1151 | { |
1293 | .ctl_name = VM_LEGACY_VA_LAYOUT, | ||
1294 | .procname = "legacy_va_layout", | 1152 | .procname = "legacy_va_layout", |
1295 | .data = &sysctl_legacy_va_layout, | 1153 | .data = &sysctl_legacy_va_layout, |
1296 | .maxlen = sizeof(sysctl_legacy_va_layout), | 1154 | .maxlen = sizeof(sysctl_legacy_va_layout), |
1297 | .mode = 0644, | 1155 | .mode = 0644, |
1298 | .proc_handler = &proc_dointvec, | 1156 | .proc_handler = proc_dointvec, |
1299 | .strategy = &sysctl_intvec, | ||
1300 | .extra1 = &zero, | 1157 | .extra1 = &zero, |
1301 | }, | 1158 | }, |
1302 | #endif | 1159 | #endif |
1303 | #ifdef CONFIG_NUMA | 1160 | #ifdef CONFIG_NUMA |
1304 | { | 1161 | { |
1305 | .ctl_name = VM_ZONE_RECLAIM_MODE, | ||
1306 | .procname = "zone_reclaim_mode", | 1162 | .procname = "zone_reclaim_mode", |
1307 | .data = &zone_reclaim_mode, | 1163 | .data = &zone_reclaim_mode, |
1308 | .maxlen = sizeof(zone_reclaim_mode), | 1164 | .maxlen = sizeof(zone_reclaim_mode), |
1309 | .mode = 0644, | 1165 | .mode = 0644, |
1310 | .proc_handler = &proc_dointvec, | 1166 | .proc_handler = proc_dointvec, |
1311 | .strategy = &sysctl_intvec, | ||
1312 | .extra1 = &zero, | 1167 | .extra1 = &zero, |
1313 | }, | 1168 | }, |
1314 | { | 1169 | { |
1315 | .ctl_name = VM_MIN_UNMAPPED, | ||
1316 | .procname = "min_unmapped_ratio", | 1170 | .procname = "min_unmapped_ratio", |
1317 | .data = &sysctl_min_unmapped_ratio, | 1171 | .data = &sysctl_min_unmapped_ratio, |
1318 | .maxlen = sizeof(sysctl_min_unmapped_ratio), | 1172 | .maxlen = sizeof(sysctl_min_unmapped_ratio), |
1319 | .mode = 0644, | 1173 | .mode = 0644, |
1320 | .proc_handler = &sysctl_min_unmapped_ratio_sysctl_handler, | 1174 | .proc_handler = sysctl_min_unmapped_ratio_sysctl_handler, |
1321 | .strategy = &sysctl_intvec, | ||
1322 | .extra1 = &zero, | 1175 | .extra1 = &zero, |
1323 | .extra2 = &one_hundred, | 1176 | .extra2 = &one_hundred, |
1324 | }, | 1177 | }, |
1325 | { | 1178 | { |
1326 | .ctl_name = VM_MIN_SLAB, | ||
1327 | .procname = "min_slab_ratio", | 1179 | .procname = "min_slab_ratio", |
1328 | .data = &sysctl_min_slab_ratio, | 1180 | .data = &sysctl_min_slab_ratio, |
1329 | .maxlen = sizeof(sysctl_min_slab_ratio), | 1181 | .maxlen = sizeof(sysctl_min_slab_ratio), |
1330 | .mode = 0644, | 1182 | .mode = 0644, |
1331 | .proc_handler = &sysctl_min_slab_ratio_sysctl_handler, | 1183 | .proc_handler = sysctl_min_slab_ratio_sysctl_handler, |
1332 | .strategy = &sysctl_intvec, | ||
1333 | .extra1 = &zero, | 1184 | .extra1 = &zero, |
1334 | .extra2 = &one_hundred, | 1185 | .extra2 = &one_hundred, |
1335 | }, | 1186 | }, |
1336 | #endif | 1187 | #endif |
1337 | #ifdef CONFIG_SMP | 1188 | #ifdef CONFIG_SMP |
1338 | { | 1189 | { |
1339 | .ctl_name = CTL_UNNUMBERED, | ||
1340 | .procname = "stat_interval", | 1190 | .procname = "stat_interval", |
1341 | .data = &sysctl_stat_interval, | 1191 | .data = &sysctl_stat_interval, |
1342 | .maxlen = sizeof(sysctl_stat_interval), | 1192 | .maxlen = sizeof(sysctl_stat_interval), |
1343 | .mode = 0644, | 1193 | .mode = 0644, |
1344 | .proc_handler = &proc_dointvec_jiffies, | 1194 | .proc_handler = proc_dointvec_jiffies, |
1345 | .strategy = &sysctl_jiffies, | ||
1346 | }, | 1195 | }, |
1347 | #endif | 1196 | #endif |
1348 | { | 1197 | { |
1349 | .ctl_name = CTL_UNNUMBERED, | ||
1350 | .procname = "mmap_min_addr", | 1198 | .procname = "mmap_min_addr", |
1351 | .data = &dac_mmap_min_addr, | 1199 | .data = &dac_mmap_min_addr, |
1352 | .maxlen = sizeof(unsigned long), | 1200 | .maxlen = sizeof(unsigned long), |
1353 | .mode = 0644, | 1201 | .mode = 0644, |
1354 | .proc_handler = &mmap_min_addr_handler, | 1202 | .proc_handler = mmap_min_addr_handler, |
1355 | }, | 1203 | }, |
1356 | #ifdef CONFIG_NUMA | 1204 | #ifdef CONFIG_NUMA |
1357 | { | 1205 | { |
1358 | .ctl_name = CTL_UNNUMBERED, | ||
1359 | .procname = "numa_zonelist_order", | 1206 | .procname = "numa_zonelist_order", |
1360 | .data = &numa_zonelist_order, | 1207 | .data = &numa_zonelist_order, |
1361 | .maxlen = NUMA_ZONELIST_ORDER_LEN, | 1208 | .maxlen = NUMA_ZONELIST_ORDER_LEN, |
1362 | .mode = 0644, | 1209 | .mode = 0644, |
1363 | .proc_handler = &numa_zonelist_order_handler, | 1210 | .proc_handler = numa_zonelist_order_handler, |
1364 | .strategy = &sysctl_string, | ||
1365 | }, | 1211 | }, |
1366 | #endif | 1212 | #endif |
1367 | #if (defined(CONFIG_X86_32) && !defined(CONFIG_UML))|| \ | 1213 | #if (defined(CONFIG_X86_32) && !defined(CONFIG_UML))|| \ |
1368 | (defined(CONFIG_SUPERH) && defined(CONFIG_VSYSCALL)) | 1214 | (defined(CONFIG_SUPERH) && defined(CONFIG_VSYSCALL)) |
1369 | { | 1215 | { |
1370 | .ctl_name = VM_VDSO_ENABLED, | ||
1371 | .procname = "vdso_enabled", | 1216 | .procname = "vdso_enabled", |
1372 | .data = &vdso_enabled, | 1217 | .data = &vdso_enabled, |
1373 | .maxlen = sizeof(vdso_enabled), | 1218 | .maxlen = sizeof(vdso_enabled), |
1374 | .mode = 0644, | 1219 | .mode = 0644, |
1375 | .proc_handler = &proc_dointvec, | 1220 | .proc_handler = proc_dointvec, |
1376 | .strategy = &sysctl_intvec, | ||
1377 | .extra1 = &zero, | 1221 | .extra1 = &zero, |
1378 | }, | 1222 | }, |
1379 | #endif | 1223 | #endif |
1380 | #ifdef CONFIG_HIGHMEM | 1224 | #ifdef CONFIG_HIGHMEM |
1381 | { | 1225 | { |
1382 | .ctl_name = CTL_UNNUMBERED, | ||
1383 | .procname = "highmem_is_dirtyable", | 1226 | .procname = "highmem_is_dirtyable", |
1384 | .data = &vm_highmem_is_dirtyable, | 1227 | .data = &vm_highmem_is_dirtyable, |
1385 | .maxlen = sizeof(vm_highmem_is_dirtyable), | 1228 | .maxlen = sizeof(vm_highmem_is_dirtyable), |
1386 | .mode = 0644, | 1229 | .mode = 0644, |
1387 | .proc_handler = &proc_dointvec_minmax, | 1230 | .proc_handler = proc_dointvec_minmax, |
1388 | .strategy = &sysctl_intvec, | ||
1389 | .extra1 = &zero, | 1231 | .extra1 = &zero, |
1390 | .extra2 = &one, | 1232 | .extra2 = &one, |
1391 | }, | 1233 | }, |
1392 | #endif | 1234 | #endif |
1393 | { | 1235 | { |
1394 | .ctl_name = CTL_UNNUMBERED, | ||
1395 | .procname = "scan_unevictable_pages", | 1236 | .procname = "scan_unevictable_pages", |
1396 | .data = &scan_unevictable_pages, | 1237 | .data = &scan_unevictable_pages, |
1397 | .maxlen = sizeof(scan_unevictable_pages), | 1238 | .maxlen = sizeof(scan_unevictable_pages), |
1398 | .mode = 0644, | 1239 | .mode = 0644, |
1399 | .proc_handler = &scan_unevictable_handler, | 1240 | .proc_handler = scan_unevictable_handler, |
1400 | }, | 1241 | }, |
1401 | #ifdef CONFIG_MEMORY_FAILURE | 1242 | #ifdef CONFIG_MEMORY_FAILURE |
1402 | { | 1243 | { |
1403 | .ctl_name = CTL_UNNUMBERED, | ||
1404 | .procname = "memory_failure_early_kill", | 1244 | .procname = "memory_failure_early_kill", |
1405 | .data = &sysctl_memory_failure_early_kill, | 1245 | .data = &sysctl_memory_failure_early_kill, |
1406 | .maxlen = sizeof(sysctl_memory_failure_early_kill), | 1246 | .maxlen = sizeof(sysctl_memory_failure_early_kill), |
1407 | .mode = 0644, | 1247 | .mode = 0644, |
1408 | .proc_handler = &proc_dointvec_minmax, | 1248 | .proc_handler = proc_dointvec_minmax, |
1409 | .strategy = &sysctl_intvec, | ||
1410 | .extra1 = &zero, | 1249 | .extra1 = &zero, |
1411 | .extra2 = &one, | 1250 | .extra2 = &one, |
1412 | }, | 1251 | }, |
1413 | { | 1252 | { |
1414 | .ctl_name = CTL_UNNUMBERED, | ||
1415 | .procname = "memory_failure_recovery", | 1253 | .procname = "memory_failure_recovery", |
1416 | .data = &sysctl_memory_failure_recovery, | 1254 | .data = &sysctl_memory_failure_recovery, |
1417 | .maxlen = sizeof(sysctl_memory_failure_recovery), | 1255 | .maxlen = sizeof(sysctl_memory_failure_recovery), |
1418 | .mode = 0644, | 1256 | .mode = 0644, |
1419 | .proc_handler = &proc_dointvec_minmax, | 1257 | .proc_handler = proc_dointvec_minmax, |
1420 | .strategy = &sysctl_intvec, | ||
1421 | .extra1 = &zero, | 1258 | .extra1 = &zero, |
1422 | .extra2 = &one, | 1259 | .extra2 = &one, |
1423 | }, | 1260 | }, |
@@ -1427,116 +1264,104 @@ static struct ctl_table vm_table[] = { | |||
1427 | * NOTE: do not add new entries to this table unless you have read | 1264 | * NOTE: do not add new entries to this table unless you have read |
1428 | * Documentation/sysctl/ctl_unnumbered.txt | 1265 | * Documentation/sysctl/ctl_unnumbered.txt |
1429 | */ | 1266 | */ |
1430 | { .ctl_name = 0 } | 1267 | { } |
1431 | }; | 1268 | }; |
1432 | 1269 | ||
1433 | #if defined(CONFIG_BINFMT_MISC) || defined(CONFIG_BINFMT_MISC_MODULE) | 1270 | #if defined(CONFIG_BINFMT_MISC) || defined(CONFIG_BINFMT_MISC_MODULE) |
1434 | static struct ctl_table binfmt_misc_table[] = { | 1271 | static struct ctl_table binfmt_misc_table[] = { |
1435 | { .ctl_name = 0 } | 1272 | { } |
1436 | }; | 1273 | }; |
1437 | #endif | 1274 | #endif |
1438 | 1275 | ||
1439 | static struct ctl_table fs_table[] = { | 1276 | static struct ctl_table fs_table[] = { |
1440 | { | 1277 | { |
1441 | .ctl_name = FS_NRINODE, | ||
1442 | .procname = "inode-nr", | 1278 | .procname = "inode-nr", |
1443 | .data = &inodes_stat, | 1279 | .data = &inodes_stat, |
1444 | .maxlen = 2*sizeof(int), | 1280 | .maxlen = 2*sizeof(int), |
1445 | .mode = 0444, | 1281 | .mode = 0444, |
1446 | .proc_handler = &proc_dointvec, | 1282 | .proc_handler = proc_dointvec, |
1447 | }, | 1283 | }, |
1448 | { | 1284 | { |
1449 | .ctl_name = FS_STATINODE, | ||
1450 | .procname = "inode-state", | 1285 | .procname = "inode-state", |
1451 | .data = &inodes_stat, | 1286 | .data = &inodes_stat, |
1452 | .maxlen = 7*sizeof(int), | 1287 | .maxlen = 7*sizeof(int), |
1453 | .mode = 0444, | 1288 | .mode = 0444, |
1454 | .proc_handler = &proc_dointvec, | 1289 | .proc_handler = proc_dointvec, |
1455 | }, | 1290 | }, |
1456 | { | 1291 | { |
1457 | .procname = "file-nr", | 1292 | .procname = "file-nr", |
1458 | .data = &files_stat, | 1293 | .data = &files_stat, |
1459 | .maxlen = 3*sizeof(int), | 1294 | .maxlen = 3*sizeof(int), |
1460 | .mode = 0444, | 1295 | .mode = 0444, |
1461 | .proc_handler = &proc_nr_files, | 1296 | .proc_handler = proc_nr_files, |
1462 | }, | 1297 | }, |
1463 | { | 1298 | { |
1464 | .ctl_name = FS_MAXFILE, | ||
1465 | .procname = "file-max", | 1299 | .procname = "file-max", |
1466 | .data = &files_stat.max_files, | 1300 | .data = &files_stat.max_files, |
1467 | .maxlen = sizeof(int), | 1301 | .maxlen = sizeof(int), |
1468 | .mode = 0644, | 1302 | .mode = 0644, |
1469 | .proc_handler = &proc_dointvec, | 1303 | .proc_handler = proc_dointvec, |
1470 | }, | 1304 | }, |
1471 | { | 1305 | { |
1472 | .ctl_name = CTL_UNNUMBERED, | ||
1473 | .procname = "nr_open", | 1306 | .procname = "nr_open", |
1474 | .data = &sysctl_nr_open, | 1307 | .data = &sysctl_nr_open, |
1475 | .maxlen = sizeof(int), | 1308 | .maxlen = sizeof(int), |
1476 | .mode = 0644, | 1309 | .mode = 0644, |
1477 | .proc_handler = &proc_dointvec_minmax, | 1310 | .proc_handler = proc_dointvec_minmax, |
1478 | .extra1 = &sysctl_nr_open_min, | 1311 | .extra1 = &sysctl_nr_open_min, |
1479 | .extra2 = &sysctl_nr_open_max, | 1312 | .extra2 = &sysctl_nr_open_max, |
1480 | }, | 1313 | }, |
1481 | { | 1314 | { |
1482 | .ctl_name = FS_DENTRY, | ||
1483 | .procname = "dentry-state", | 1315 | .procname = "dentry-state", |
1484 | .data = &dentry_stat, | 1316 | .data = &dentry_stat, |
1485 | .maxlen = 6*sizeof(int), | 1317 | .maxlen = 6*sizeof(int), |
1486 | .mode = 0444, | 1318 | .mode = 0444, |
1487 | .proc_handler = &proc_dointvec, | 1319 | .proc_handler = proc_dointvec, |
1488 | }, | 1320 | }, |
1489 | { | 1321 | { |
1490 | .ctl_name = FS_OVERFLOWUID, | ||
1491 | .procname = "overflowuid", | 1322 | .procname = "overflowuid", |
1492 | .data = &fs_overflowuid, | 1323 | .data = &fs_overflowuid, |
1493 | .maxlen = sizeof(int), | 1324 | .maxlen = sizeof(int), |
1494 | .mode = 0644, | 1325 | .mode = 0644, |
1495 | .proc_handler = &proc_dointvec_minmax, | 1326 | .proc_handler = proc_dointvec_minmax, |
1496 | .strategy = &sysctl_intvec, | ||
1497 | .extra1 = &minolduid, | 1327 | .extra1 = &minolduid, |
1498 | .extra2 = &maxolduid, | 1328 | .extra2 = &maxolduid, |
1499 | }, | 1329 | }, |
1500 | { | 1330 | { |
1501 | .ctl_name = FS_OVERFLOWGID, | ||
1502 | .procname = "overflowgid", | 1331 | .procname = "overflowgid", |
1503 | .data = &fs_overflowgid, | 1332 | .data = &fs_overflowgid, |
1504 | .maxlen = sizeof(int), | 1333 | .maxlen = sizeof(int), |
1505 | .mode = 0644, | 1334 | .mode = 0644, |
1506 | .proc_handler = &proc_dointvec_minmax, | 1335 | .proc_handler = proc_dointvec_minmax, |
1507 | .strategy = &sysctl_intvec, | ||
1508 | .extra1 = &minolduid, | 1336 | .extra1 = &minolduid, |
1509 | .extra2 = &maxolduid, | 1337 | .extra2 = &maxolduid, |
1510 | }, | 1338 | }, |
1511 | #ifdef CONFIG_FILE_LOCKING | 1339 | #ifdef CONFIG_FILE_LOCKING |
1512 | { | 1340 | { |
1513 | .ctl_name = FS_LEASES, | ||
1514 | .procname = "leases-enable", | 1341 | .procname = "leases-enable", |
1515 | .data = &leases_enable, | 1342 | .data = &leases_enable, |
1516 | .maxlen = sizeof(int), | 1343 | .maxlen = sizeof(int), |
1517 | .mode = 0644, | 1344 | .mode = 0644, |
1518 | .proc_handler = &proc_dointvec, | 1345 | .proc_handler = proc_dointvec, |
1519 | }, | 1346 | }, |
1520 | #endif | 1347 | #endif |
1521 | #ifdef CONFIG_DNOTIFY | 1348 | #ifdef CONFIG_DNOTIFY |
1522 | { | 1349 | { |
1523 | .ctl_name = FS_DIR_NOTIFY, | ||
1524 | .procname = "dir-notify-enable", | 1350 | .procname = "dir-notify-enable", |
1525 | .data = &dir_notify_enable, | 1351 | .data = &dir_notify_enable, |
1526 | .maxlen = sizeof(int), | 1352 | .maxlen = sizeof(int), |
1527 | .mode = 0644, | 1353 | .mode = 0644, |
1528 | .proc_handler = &proc_dointvec, | 1354 | .proc_handler = proc_dointvec, |
1529 | }, | 1355 | }, |
1530 | #endif | 1356 | #endif |
1531 | #ifdef CONFIG_MMU | 1357 | #ifdef CONFIG_MMU |
1532 | #ifdef CONFIG_FILE_LOCKING | 1358 | #ifdef CONFIG_FILE_LOCKING |
1533 | { | 1359 | { |
1534 | .ctl_name = FS_LEASE_TIME, | ||
1535 | .procname = "lease-break-time", | 1360 | .procname = "lease-break-time", |
1536 | .data = &lease_break_time, | 1361 | .data = &lease_break_time, |
1537 | .maxlen = sizeof(int), | 1362 | .maxlen = sizeof(int), |
1538 | .mode = 0644, | 1363 | .mode = 0644, |
1539 | .proc_handler = &proc_dointvec, | 1364 | .proc_handler = proc_dointvec, |
1540 | }, | 1365 | }, |
1541 | #endif | 1366 | #endif |
1542 | #ifdef CONFIG_AIO | 1367 | #ifdef CONFIG_AIO |
@@ -1545,19 +1370,18 @@ static struct ctl_table fs_table[] = { | |||
1545 | .data = &aio_nr, | 1370 | .data = &aio_nr, |
1546 | .maxlen = sizeof(aio_nr), | 1371 | .maxlen = sizeof(aio_nr), |
1547 | .mode = 0444, | 1372 | .mode = 0444, |
1548 | .proc_handler = &proc_doulongvec_minmax, | 1373 | .proc_handler = proc_doulongvec_minmax, |
1549 | }, | 1374 | }, |
1550 | { | 1375 | { |
1551 | .procname = "aio-max-nr", | 1376 | .procname = "aio-max-nr", |
1552 | .data = &aio_max_nr, | 1377 | .data = &aio_max_nr, |
1553 | .maxlen = sizeof(aio_max_nr), | 1378 | .maxlen = sizeof(aio_max_nr), |
1554 | .mode = 0644, | 1379 | .mode = 0644, |
1555 | .proc_handler = &proc_doulongvec_minmax, | 1380 | .proc_handler = proc_doulongvec_minmax, |
1556 | }, | 1381 | }, |
1557 | #endif /* CONFIG_AIO */ | 1382 | #endif /* CONFIG_AIO */ |
1558 | #ifdef CONFIG_INOTIFY_USER | 1383 | #ifdef CONFIG_INOTIFY_USER |
1559 | { | 1384 | { |
1560 | .ctl_name = FS_INOTIFY, | ||
1561 | .procname = "inotify", | 1385 | .procname = "inotify", |
1562 | .mode = 0555, | 1386 | .mode = 0555, |
1563 | .child = inotify_table, | 1387 | .child = inotify_table, |
@@ -1572,19 +1396,16 @@ static struct ctl_table fs_table[] = { | |||
1572 | #endif | 1396 | #endif |
1573 | #endif | 1397 | #endif |
1574 | { | 1398 | { |
1575 | .ctl_name = KERN_SETUID_DUMPABLE, | ||
1576 | .procname = "suid_dumpable", | 1399 | .procname = "suid_dumpable", |
1577 | .data = &suid_dumpable, | 1400 | .data = &suid_dumpable, |
1578 | .maxlen = sizeof(int), | 1401 | .maxlen = sizeof(int), |
1579 | .mode = 0644, | 1402 | .mode = 0644, |
1580 | .proc_handler = &proc_dointvec_minmax, | 1403 | .proc_handler = proc_dointvec_minmax, |
1581 | .strategy = &sysctl_intvec, | ||
1582 | .extra1 = &zero, | 1404 | .extra1 = &zero, |
1583 | .extra2 = &two, | 1405 | .extra2 = &two, |
1584 | }, | 1406 | }, |
1585 | #if defined(CONFIG_BINFMT_MISC) || defined(CONFIG_BINFMT_MISC_MODULE) | 1407 | #if defined(CONFIG_BINFMT_MISC) || defined(CONFIG_BINFMT_MISC_MODULE) |
1586 | { | 1408 | { |
1587 | .ctl_name = CTL_UNNUMBERED, | ||
1588 | .procname = "binfmt_misc", | 1409 | .procname = "binfmt_misc", |
1589 | .mode = 0555, | 1410 | .mode = 0555, |
1590 | .child = binfmt_misc_table, | 1411 | .child = binfmt_misc_table, |
@@ -1594,13 +1415,12 @@ static struct ctl_table fs_table[] = { | |||
1594 | * NOTE: do not add new entries to this table unless you have read | 1415 | * NOTE: do not add new entries to this table unless you have read |
1595 | * Documentation/sysctl/ctl_unnumbered.txt | 1416 | * Documentation/sysctl/ctl_unnumbered.txt |
1596 | */ | 1417 | */ |
1597 | { .ctl_name = 0 } | 1418 | { } |
1598 | }; | 1419 | }; |
1599 | 1420 | ||
1600 | static struct ctl_table debug_table[] = { | 1421 | static struct ctl_table debug_table[] = { |
1601 | #if defined(CONFIG_X86) || defined(CONFIG_PPC) | 1422 | #if defined(CONFIG_X86) || defined(CONFIG_PPC) |
1602 | { | 1423 | { |
1603 | .ctl_name = CTL_UNNUMBERED, | ||
1604 | .procname = "exception-trace", | 1424 | .procname = "exception-trace", |
1605 | .data = &show_unhandled_signals, | 1425 | .data = &show_unhandled_signals, |
1606 | .maxlen = sizeof(int), | 1426 | .maxlen = sizeof(int), |
@@ -1608,11 +1428,11 @@ static struct ctl_table debug_table[] = { | |||
1608 | .proc_handler = proc_dointvec | 1428 | .proc_handler = proc_dointvec |
1609 | }, | 1429 | }, |
1610 | #endif | 1430 | #endif |
1611 | { .ctl_name = 0 } | 1431 | { } |
1612 | }; | 1432 | }; |
1613 | 1433 | ||
1614 | static struct ctl_table dev_table[] = { | 1434 | static struct ctl_table dev_table[] = { |
1615 | { .ctl_name = 0 } | 1435 | { } |
1616 | }; | 1436 | }; |
1617 | 1437 | ||
1618 | static DEFINE_SPINLOCK(sysctl_lock); | 1438 | static DEFINE_SPINLOCK(sysctl_lock); |
@@ -1766,122 +1586,6 @@ void register_sysctl_root(struct ctl_table_root *root) | |||
1766 | spin_unlock(&sysctl_lock); | 1586 | spin_unlock(&sysctl_lock); |
1767 | } | 1587 | } |
1768 | 1588 | ||
1769 | #ifdef CONFIG_SYSCTL_SYSCALL | ||
1770 | /* Perform the actual read/write of a sysctl table entry. */ | ||
1771 | static int do_sysctl_strategy(struct ctl_table_root *root, | ||
1772 | struct ctl_table *table, | ||
1773 | void __user *oldval, size_t __user *oldlenp, | ||
1774 | void __user *newval, size_t newlen) | ||
1775 | { | ||
1776 | int op = 0, rc; | ||
1777 | |||
1778 | if (oldval) | ||
1779 | op |= MAY_READ; | ||
1780 | if (newval) | ||
1781 | op |= MAY_WRITE; | ||
1782 | if (sysctl_perm(root, table, op)) | ||
1783 | return -EPERM; | ||
1784 | |||
1785 | if (table->strategy) { | ||
1786 | rc = table->strategy(table, oldval, oldlenp, newval, newlen); | ||
1787 | if (rc < 0) | ||
1788 | return rc; | ||
1789 | if (rc > 0) | ||
1790 | return 0; | ||
1791 | } | ||
1792 | |||
1793 | /* If there is no strategy routine, or if the strategy returns | ||
1794 | * zero, proceed with automatic r/w */ | ||
1795 | if (table->data && table->maxlen) { | ||
1796 | rc = sysctl_data(table, oldval, oldlenp, newval, newlen); | ||
1797 | if (rc < 0) | ||
1798 | return rc; | ||
1799 | } | ||
1800 | return 0; | ||
1801 | } | ||
1802 | |||
1803 | static int parse_table(int __user *name, int nlen, | ||
1804 | void __user *oldval, size_t __user *oldlenp, | ||
1805 | void __user *newval, size_t newlen, | ||
1806 | struct ctl_table_root *root, | ||
1807 | struct ctl_table *table) | ||
1808 | { | ||
1809 | int n; | ||
1810 | repeat: | ||
1811 | if (!nlen) | ||
1812 | return -ENOTDIR; | ||
1813 | if (get_user(n, name)) | ||
1814 | return -EFAULT; | ||
1815 | for ( ; table->ctl_name || table->procname; table++) { | ||
1816 | if (!table->ctl_name) | ||
1817 | continue; | ||
1818 | if (n == table->ctl_name) { | ||
1819 | int error; | ||
1820 | if (table->child) { | ||
1821 | if (sysctl_perm(root, table, MAY_EXEC)) | ||
1822 | return -EPERM; | ||
1823 | name++; | ||
1824 | nlen--; | ||
1825 | table = table->child; | ||
1826 | goto repeat; | ||
1827 | } | ||
1828 | error = do_sysctl_strategy(root, table, | ||
1829 | oldval, oldlenp, | ||
1830 | newval, newlen); | ||
1831 | return error; | ||
1832 | } | ||
1833 | } | ||
1834 | return -ENOTDIR; | ||
1835 | } | ||
1836 | |||
1837 | int do_sysctl(int __user *name, int nlen, void __user *oldval, size_t __user *oldlenp, | ||
1838 | void __user *newval, size_t newlen) | ||
1839 | { | ||
1840 | struct ctl_table_header *head; | ||
1841 | int error = -ENOTDIR; | ||
1842 | |||
1843 | if (nlen <= 0 || nlen >= CTL_MAXNAME) | ||
1844 | return -ENOTDIR; | ||
1845 | if (oldval) { | ||
1846 | int old_len; | ||
1847 | if (!oldlenp || get_user(old_len, oldlenp)) | ||
1848 | return -EFAULT; | ||
1849 | } | ||
1850 | |||
1851 | for (head = sysctl_head_next(NULL); head; | ||
1852 | head = sysctl_head_next(head)) { | ||
1853 | error = parse_table(name, nlen, oldval, oldlenp, | ||
1854 | newval, newlen, | ||
1855 | head->root, head->ctl_table); | ||
1856 | if (error != -ENOTDIR) { | ||
1857 | sysctl_head_finish(head); | ||
1858 | break; | ||
1859 | } | ||
1860 | } | ||
1861 | return error; | ||
1862 | } | ||
1863 | |||
1864 | SYSCALL_DEFINE1(sysctl, struct __sysctl_args __user *, args) | ||
1865 | { | ||
1866 | struct __sysctl_args tmp; | ||
1867 | int error; | ||
1868 | |||
1869 | if (copy_from_user(&tmp, args, sizeof(tmp))) | ||
1870 | return -EFAULT; | ||
1871 | |||
1872 | error = deprecated_sysctl_warning(&tmp); | ||
1873 | if (error) | ||
1874 | goto out; | ||
1875 | |||
1876 | lock_kernel(); | ||
1877 | error = do_sysctl(tmp.name, tmp.nlen, tmp.oldval, tmp.oldlenp, | ||
1878 | tmp.newval, tmp.newlen); | ||
1879 | unlock_kernel(); | ||
1880 | out: | ||
1881 | return error; | ||
1882 | } | ||
1883 | #endif /* CONFIG_SYSCTL_SYSCALL */ | ||
1884 | |||
1885 | /* | 1589 | /* |
1886 | * sysctl_perm does NOT grant the superuser all rights automatically, because | 1590 | * sysctl_perm does NOT grant the superuser all rights automatically, because |
1887 | * some sysctl variables are readonly even to root. | 1591 | * some sysctl variables are readonly even to root. |
@@ -1917,7 +1621,7 @@ int sysctl_perm(struct ctl_table_root *root, struct ctl_table *table, int op) | |||
1917 | 1621 | ||
1918 | static void sysctl_set_parent(struct ctl_table *parent, struct ctl_table *table) | 1622 | static void sysctl_set_parent(struct ctl_table *parent, struct ctl_table *table) |
1919 | { | 1623 | { |
1920 | for (; table->ctl_name || table->procname; table++) { | 1624 | for (; table->procname; table++) { |
1921 | table->parent = parent; | 1625 | table->parent = parent; |
1922 | if (table->child) | 1626 | if (table->child) |
1923 | sysctl_set_parent(table, table->child); | 1627 | sysctl_set_parent(table, table->child); |
@@ -1949,11 +1653,11 @@ static struct ctl_table *is_branch_in(struct ctl_table *branch, | |||
1949 | return NULL; | 1653 | return NULL; |
1950 | 1654 | ||
1951 | /* ... and nothing else */ | 1655 | /* ... and nothing else */ |
1952 | if (branch[1].procname || branch[1].ctl_name) | 1656 | if (branch[1].procname) |
1953 | return NULL; | 1657 | return NULL; |
1954 | 1658 | ||
1955 | /* table should contain subdirectory with the same name */ | 1659 | /* table should contain subdirectory with the same name */ |
1956 | for (p = table; p->procname || p->ctl_name; p++) { | 1660 | for (p = table; p->procname; p++) { |
1957 | if (!p->child) | 1661 | if (!p->child) |
1958 | continue; | 1662 | continue; |
1959 | if (p->procname && strcmp(p->procname, s) == 0) | 1663 | if (p->procname && strcmp(p->procname, s) == 0) |
@@ -1998,9 +1702,6 @@ static void try_attach(struct ctl_table_header *p, struct ctl_table_header *q) | |||
1998 | * | 1702 | * |
1999 | * The members of the &struct ctl_table structure are used as follows: | 1703 | * The members of the &struct ctl_table structure are used as follows: |
2000 | * | 1704 | * |
2001 | * ctl_name - This is the numeric sysctl value used by sysctl(2). The number | ||
2002 | * must be unique within that level of sysctl | ||
2003 | * | ||
2004 | * procname - the name of the sysctl file under /proc/sys. Set to %NULL to not | 1705 | * procname - the name of the sysctl file under /proc/sys. Set to %NULL to not |
2005 | * enter a sysctl file | 1706 | * enter a sysctl file |
2006 | * | 1707 | * |
@@ -2015,8 +1716,6 @@ static void try_attach(struct ctl_table_header *p, struct ctl_table_header *q) | |||
2015 | * | 1716 | * |
2016 | * proc_handler - the text handler routine (described below) | 1717 | * proc_handler - the text handler routine (described below) |
2017 | * | 1718 | * |
2018 | * strategy - the strategy routine (described below) | ||
2019 | * | ||
2020 | * de - for internal use by the sysctl routines | 1719 | * de - for internal use by the sysctl routines |
2021 | * | 1720 | * |
2022 | * extra1, extra2 - extra pointers usable by the proc handler routines | 1721 | * extra1, extra2 - extra pointers usable by the proc handler routines |
@@ -2029,19 +1728,6 @@ static void try_attach(struct ctl_table_header *p, struct ctl_table_header *q) | |||
2029 | * struct enable minimal validation of the values being written to be | 1728 | * struct enable minimal validation of the values being written to be |
2030 | * performed, and the mode field allows minimal authentication. | 1729 | * performed, and the mode field allows minimal authentication. |
2031 | * | 1730 | * |
2032 | * More sophisticated management can be enabled by the provision of a | ||
2033 | * strategy routine with the table entry. This will be called before | ||
2034 | * any automatic read or write of the data is performed. | ||
2035 | * | ||
2036 | * The strategy routine may return | ||
2037 | * | ||
2038 | * < 0 - Error occurred (error is passed to user process) | ||
2039 | * | ||
2040 | * 0 - OK - proceed with automatic read or write. | ||
2041 | * | ||
2042 | * > 0 - OK - read or write has been done by the strategy routine, so | ||
2043 | * return immediately. | ||
2044 | * | ||
2045 | * There must be a proc_handler routine for any terminal nodes | 1731 | * There must be a proc_handler routine for any terminal nodes |
2046 | * mirrored under /proc/sys (non-terminals are handled by a built-in | 1732 | * mirrored under /proc/sys (non-terminals are handled by a built-in |
2047 | * directory handler). Several default handlers are available to | 1733 | * directory handler). Several default handlers are available to |
@@ -2068,13 +1754,13 @@ struct ctl_table_header *__register_sysctl_paths( | |||
2068 | struct ctl_table_set *set; | 1754 | struct ctl_table_set *set; |
2069 | 1755 | ||
2070 | /* Count the path components */ | 1756 | /* Count the path components */ |
2071 | for (npath = 0; path[npath].ctl_name || path[npath].procname; ++npath) | 1757 | for (npath = 0; path[npath].procname; ++npath) |
2072 | ; | 1758 | ; |
2073 | 1759 | ||
2074 | /* | 1760 | /* |
2075 | * For each path component, allocate a 2-element ctl_table array. | 1761 | * For each path component, allocate a 2-element ctl_table array. |
2076 | * The first array element will be filled with the sysctl entry | 1762 | * The first array element will be filled with the sysctl entry |
2077 | * for this, the second will be the sentinel (ctl_name == 0). | 1763 | * for this, the second will be the sentinel (procname == 0). |
2078 | * | 1764 | * |
2079 | * We allocate everything in one go so that we don't have to | 1765 | * We allocate everything in one go so that we don't have to |
2080 | * worry about freeing additional memory in unregister_sysctl_table. | 1766 | * worry about freeing additional memory in unregister_sysctl_table. |
@@ -2091,7 +1777,6 @@ struct ctl_table_header *__register_sysctl_paths( | |||
2091 | for (n = 0; n < npath; ++n, ++path) { | 1777 | for (n = 0; n < npath; ++n, ++path) { |
2092 | /* Copy the procname */ | 1778 | /* Copy the procname */ |
2093 | new->procname = path->procname; | 1779 | new->procname = path->procname; |
2094 | new->ctl_name = path->ctl_name; | ||
2095 | new->mode = 0555; | 1780 | new->mode = 0555; |
2096 | 1781 | ||
2097 | *prevp = new; | 1782 | *prevp = new; |
@@ -2953,286 +2638,6 @@ int proc_doulongvec_ms_jiffies_minmax(struct ctl_table *table, int write, | |||
2953 | 2638 | ||
2954 | #endif /* CONFIG_PROC_FS */ | 2639 | #endif /* CONFIG_PROC_FS */ |
2955 | 2640 | ||
2956 | |||
2957 | #ifdef CONFIG_SYSCTL_SYSCALL | ||
2958 | /* | ||
2959 | * General sysctl support routines | ||
2960 | */ | ||
2961 | |||
2962 | /* The generic sysctl data routine (used if no strategy routine supplied) */ | ||
2963 | int sysctl_data(struct ctl_table *table, | ||
2964 | void __user *oldval, size_t __user *oldlenp, | ||
2965 | void __user *newval, size_t newlen) | ||
2966 | { | ||
2967 | size_t len; | ||
2968 | |||
2969 | /* Get out of I don't have a variable */ | ||
2970 | if (!table->data || !table->maxlen) | ||
2971 | return -ENOTDIR; | ||
2972 | |||
2973 | if (oldval && oldlenp) { | ||
2974 | if (get_user(len, oldlenp)) | ||
2975 | return -EFAULT; | ||
2976 | if (len) { | ||
2977 | if (len > table->maxlen) | ||
2978 | len = table->maxlen; | ||
2979 | if (copy_to_user(oldval, table->data, len)) | ||
2980 | return -EFAULT; | ||
2981 | if (put_user(len, oldlenp)) | ||
2982 | return -EFAULT; | ||
2983 | } | ||
2984 | } | ||
2985 | |||
2986 | if (newval && newlen) { | ||
2987 | if (newlen > table->maxlen) | ||
2988 | newlen = table->maxlen; | ||
2989 | |||
2990 | if (copy_from_user(table->data, newval, newlen)) | ||
2991 | return -EFAULT; | ||
2992 | } | ||
2993 | return 1; | ||
2994 | } | ||
2995 | |||
2996 | /* The generic string strategy routine: */ | ||
2997 | int sysctl_string(struct ctl_table *table, | ||
2998 | void __user *oldval, size_t __user *oldlenp, | ||
2999 | void __user *newval, size_t newlen) | ||
3000 | { | ||
3001 | if (!table->data || !table->maxlen) | ||
3002 | return -ENOTDIR; | ||
3003 | |||
3004 | if (oldval && oldlenp) { | ||
3005 | size_t bufsize; | ||
3006 | if (get_user(bufsize, oldlenp)) | ||
3007 | return -EFAULT; | ||
3008 | if (bufsize) { | ||
3009 | size_t len = strlen(table->data), copied; | ||
3010 | |||
3011 | /* This shouldn't trigger for a well-formed sysctl */ | ||
3012 | if (len > table->maxlen) | ||
3013 | len = table->maxlen; | ||
3014 | |||
3015 | /* Copy up to a max of bufsize-1 bytes of the string */ | ||
3016 | copied = (len >= bufsize) ? bufsize - 1 : len; | ||
3017 | |||
3018 | if (copy_to_user(oldval, table->data, copied) || | ||
3019 | put_user(0, (char __user *)(oldval + copied))) | ||
3020 | return -EFAULT; | ||
3021 | if (put_user(len, oldlenp)) | ||
3022 | return -EFAULT; | ||
3023 | } | ||
3024 | } | ||
3025 | if (newval && newlen) { | ||
3026 | size_t len = newlen; | ||
3027 | if (len > table->maxlen) | ||
3028 | len = table->maxlen; | ||
3029 | if(copy_from_user(table->data, newval, len)) | ||
3030 | return -EFAULT; | ||
3031 | if (len == table->maxlen) | ||
3032 | len--; | ||
3033 | ((char *) table->data)[len] = 0; | ||
3034 | } | ||
3035 | return 1; | ||
3036 | } | ||
3037 | |||
3038 | /* | ||
3039 | * This function makes sure that all of the integers in the vector | ||
3040 | * are between the minimum and maximum values given in the arrays | ||
3041 | * table->extra1 and table->extra2, respectively. | ||
3042 | */ | ||
3043 | int sysctl_intvec(struct ctl_table *table, | ||
3044 | void __user *oldval, size_t __user *oldlenp, | ||
3045 | void __user *newval, size_t newlen) | ||
3046 | { | ||
3047 | |||
3048 | if (newval && newlen) { | ||
3049 | int __user *vec = (int __user *) newval; | ||
3050 | int *min = (int *) table->extra1; | ||
3051 | int *max = (int *) table->extra2; | ||
3052 | size_t length; | ||
3053 | int i; | ||
3054 | |||
3055 | if (newlen % sizeof(int) != 0) | ||
3056 | return -EINVAL; | ||
3057 | |||
3058 | if (!table->extra1 && !table->extra2) | ||
3059 | return 0; | ||
3060 | |||
3061 | if (newlen > table->maxlen) | ||
3062 | newlen = table->maxlen; | ||
3063 | length = newlen / sizeof(int); | ||
3064 | |||
3065 | for (i = 0; i < length; i++) { | ||
3066 | int value; | ||
3067 | if (get_user(value, vec + i)) | ||
3068 | return -EFAULT; | ||
3069 | if (min && value < min[i]) | ||
3070 | return -EINVAL; | ||
3071 | if (max && value > max[i]) | ||
3072 | return -EINVAL; | ||
3073 | } | ||
3074 | } | ||
3075 | return 0; | ||
3076 | } | ||
3077 | |||
3078 | /* Strategy function to convert jiffies to seconds */ | ||
3079 | int sysctl_jiffies(struct ctl_table *table, | ||
3080 | void __user *oldval, size_t __user *oldlenp, | ||
3081 | void __user *newval, size_t newlen) | ||
3082 | { | ||
3083 | if (oldval && oldlenp) { | ||
3084 | size_t olen; | ||
3085 | |||
3086 | if (get_user(olen, oldlenp)) | ||
3087 | return -EFAULT; | ||
3088 | if (olen) { | ||
3089 | int val; | ||
3090 | |||
3091 | if (olen < sizeof(int)) | ||
3092 | return -EINVAL; | ||
3093 | |||
3094 | val = *(int *)(table->data) / HZ; | ||
3095 | if (put_user(val, (int __user *)oldval)) | ||
3096 | return -EFAULT; | ||
3097 | if (put_user(sizeof(int), oldlenp)) | ||
3098 | return -EFAULT; | ||
3099 | } | ||
3100 | } | ||
3101 | if (newval && newlen) { | ||
3102 | int new; | ||
3103 | if (newlen != sizeof(int)) | ||
3104 | return -EINVAL; | ||
3105 | if (get_user(new, (int __user *)newval)) | ||
3106 | return -EFAULT; | ||
3107 | *(int *)(table->data) = new*HZ; | ||
3108 | } | ||
3109 | return 1; | ||
3110 | } | ||
3111 | |||
3112 | /* Strategy function to convert jiffies to seconds */ | ||
3113 | int sysctl_ms_jiffies(struct ctl_table *table, | ||
3114 | void __user *oldval, size_t __user *oldlenp, | ||
3115 | void __user *newval, size_t newlen) | ||
3116 | { | ||
3117 | if (oldval && oldlenp) { | ||
3118 | size_t olen; | ||
3119 | |||
3120 | if (get_user(olen, oldlenp)) | ||
3121 | return -EFAULT; | ||
3122 | if (olen) { | ||
3123 | int val; | ||
3124 | |||
3125 | if (olen < sizeof(int)) | ||
3126 | return -EINVAL; | ||
3127 | |||
3128 | val = jiffies_to_msecs(*(int *)(table->data)); | ||
3129 | if (put_user(val, (int __user *)oldval)) | ||
3130 | return -EFAULT; | ||
3131 | if (put_user(sizeof(int), oldlenp)) | ||
3132 | return -EFAULT; | ||
3133 | } | ||
3134 | } | ||
3135 | if (newval && newlen) { | ||
3136 | int new; | ||
3137 | if (newlen != sizeof(int)) | ||
3138 | return -EINVAL; | ||
3139 | if (get_user(new, (int __user *)newval)) | ||
3140 | return -EFAULT; | ||
3141 | *(int *)(table->data) = msecs_to_jiffies(new); | ||
3142 | } | ||
3143 | return 1; | ||
3144 | } | ||
3145 | |||
3146 | |||
3147 | |||
3148 | #else /* CONFIG_SYSCTL_SYSCALL */ | ||
3149 | |||
3150 | |||
3151 | SYSCALL_DEFINE1(sysctl, struct __sysctl_args __user *, args) | ||
3152 | { | ||
3153 | struct __sysctl_args tmp; | ||
3154 | int error; | ||
3155 | |||
3156 | if (copy_from_user(&tmp, args, sizeof(tmp))) | ||
3157 | return -EFAULT; | ||
3158 | |||
3159 | error = deprecated_sysctl_warning(&tmp); | ||
3160 | |||
3161 | /* If no error reading the parameters then just -ENOSYS ... */ | ||
3162 | if (!error) | ||
3163 | error = -ENOSYS; | ||
3164 | |||
3165 | return error; | ||
3166 | } | ||
3167 | |||
3168 | int sysctl_data(struct ctl_table *table, | ||
3169 | void __user *oldval, size_t __user *oldlenp, | ||
3170 | void __user *newval, size_t newlen) | ||
3171 | { | ||
3172 | return -ENOSYS; | ||
3173 | } | ||
3174 | |||
3175 | int sysctl_string(struct ctl_table *table, | ||
3176 | void __user *oldval, size_t __user *oldlenp, | ||
3177 | void __user *newval, size_t newlen) | ||
3178 | { | ||
3179 | return -ENOSYS; | ||
3180 | } | ||
3181 | |||
3182 | int sysctl_intvec(struct ctl_table *table, | ||
3183 | void __user *oldval, size_t __user *oldlenp, | ||
3184 | void __user *newval, size_t newlen) | ||
3185 | { | ||
3186 | return -ENOSYS; | ||
3187 | } | ||
3188 | |||
3189 | int sysctl_jiffies(struct ctl_table *table, | ||
3190 | void __user *oldval, size_t __user *oldlenp, | ||
3191 | void __user *newval, size_t newlen) | ||
3192 | { | ||
3193 | return -ENOSYS; | ||
3194 | } | ||
3195 | |||
3196 | int sysctl_ms_jiffies(struct ctl_table *table, | ||
3197 | void __user *oldval, size_t __user *oldlenp, | ||
3198 | void __user *newval, size_t newlen) | ||
3199 | { | ||
3200 | return -ENOSYS; | ||
3201 | } | ||
3202 | |||
3203 | #endif /* CONFIG_SYSCTL_SYSCALL */ | ||
3204 | |||
3205 | static int deprecated_sysctl_warning(struct __sysctl_args *args) | ||
3206 | { | ||
3207 | static int msg_count; | ||
3208 | int name[CTL_MAXNAME]; | ||
3209 | int i; | ||
3210 | |||
3211 | /* Check args->nlen. */ | ||
3212 | if (args->nlen < 0 || args->nlen > CTL_MAXNAME) | ||
3213 | return -ENOTDIR; | ||
3214 | |||
3215 | /* Read in the sysctl name for better debug message logging */ | ||
3216 | for (i = 0; i < args->nlen; i++) | ||
3217 | if (get_user(name[i], args->name + i)) | ||
3218 | return -EFAULT; | ||
3219 | |||
3220 | /* Ignore accesses to kernel.version */ | ||
3221 | if ((args->nlen == 2) && (name[0] == CTL_KERN) && (name[1] == KERN_VERSION)) | ||
3222 | return 0; | ||
3223 | |||
3224 | if (msg_count < 5) { | ||
3225 | msg_count++; | ||
3226 | printk(KERN_INFO | ||
3227 | "warning: process `%s' used the deprecated sysctl " | ||
3228 | "system call with ", current->comm); | ||
3229 | for (i = 0; i < args->nlen; i++) | ||
3230 | printk("%d.", name[i]); | ||
3231 | printk("\n"); | ||
3232 | } | ||
3233 | return 0; | ||
3234 | } | ||
3235 | |||
3236 | /* | 2641 | /* |
3237 | * No sense putting this after each symbol definition, twice, | 2642 | * No sense putting this after each symbol definition, twice, |
3238 | * exception granted :-) | 2643 | * exception granted :-) |
@@ -3247,9 +2652,4 @@ EXPORT_SYMBOL(proc_doulongvec_minmax); | |||
3247 | EXPORT_SYMBOL(proc_doulongvec_ms_jiffies_minmax); | 2652 | EXPORT_SYMBOL(proc_doulongvec_ms_jiffies_minmax); |
3248 | EXPORT_SYMBOL(register_sysctl_table); | 2653 | EXPORT_SYMBOL(register_sysctl_table); |
3249 | EXPORT_SYMBOL(register_sysctl_paths); | 2654 | EXPORT_SYMBOL(register_sysctl_paths); |
3250 | EXPORT_SYMBOL(sysctl_intvec); | ||
3251 | EXPORT_SYMBOL(sysctl_jiffies); | ||
3252 | EXPORT_SYMBOL(sysctl_ms_jiffies); | ||
3253 | EXPORT_SYMBOL(sysctl_string); | ||
3254 | EXPORT_SYMBOL(sysctl_data); | ||
3255 | EXPORT_SYMBOL(unregister_sysctl_table); | 2655 | EXPORT_SYMBOL(unregister_sysctl_table); |
diff --git a/kernel/sysctl_binary.c b/kernel/sysctl_binary.c new file mode 100644 index 000000000000..b75dbf40f573 --- /dev/null +++ b/kernel/sysctl_binary.c | |||
@@ -0,0 +1,1507 @@ | |||
1 | #include <linux/stat.h> | ||
2 | #include <linux/sysctl.h> | ||
3 | #include "../fs/xfs/linux-2.6/xfs_sysctl.h" | ||
4 | #include <linux/sunrpc/debug.h> | ||
5 | #include <linux/string.h> | ||
6 | #include <net/ip_vs.h> | ||
7 | #include <linux/syscalls.h> | ||
8 | #include <linux/namei.h> | ||
9 | #include <linux/mount.h> | ||
10 | #include <linux/fs.h> | ||
11 | #include <linux/nsproxy.h> | ||
12 | #include <linux/pid_namespace.h> | ||
13 | #include <linux/file.h> | ||
14 | #include <linux/ctype.h> | ||
15 | #include <linux/netdevice.h> | ||
16 | |||
17 | #ifdef CONFIG_SYSCTL_SYSCALL | ||
18 | |||
19 | struct bin_table; | ||
20 | typedef ssize_t bin_convert_t(struct file *file, | ||
21 | void __user *oldval, size_t oldlen, void __user *newval, size_t newlen); | ||
22 | |||
23 | static bin_convert_t bin_dir; | ||
24 | static bin_convert_t bin_string; | ||
25 | static bin_convert_t bin_intvec; | ||
26 | static bin_convert_t bin_ulongvec; | ||
27 | static bin_convert_t bin_uuid; | ||
28 | static bin_convert_t bin_dn_node_address; | ||
29 | |||
30 | #define CTL_DIR bin_dir | ||
31 | #define CTL_STR bin_string | ||
32 | #define CTL_INT bin_intvec | ||
33 | #define CTL_ULONG bin_ulongvec | ||
34 | #define CTL_UUID bin_uuid | ||
35 | #define CTL_DNADR bin_dn_node_address | ||
36 | |||
37 | #define BUFSZ 256 | ||
38 | |||
39 | struct bin_table { | ||
40 | bin_convert_t *convert; | ||
41 | int ctl_name; | ||
42 | const char *procname; | ||
43 | const struct bin_table *child; | ||
44 | }; | ||
45 | |||
46 | static const struct bin_table bin_random_table[] = { | ||
47 | { CTL_INT, RANDOM_POOLSIZE, "poolsize" }, | ||
48 | { CTL_INT, RANDOM_ENTROPY_COUNT, "entropy_avail" }, | ||
49 | { CTL_INT, RANDOM_READ_THRESH, "read_wakeup_threshold" }, | ||
50 | { CTL_INT, RANDOM_WRITE_THRESH, "write_wakeup_threshold" }, | ||
51 | { CTL_UUID, RANDOM_BOOT_ID, "boot_id" }, | ||
52 | { CTL_UUID, RANDOM_UUID, "uuid" }, | ||
53 | {} | ||
54 | }; | ||
55 | |||
56 | static const struct bin_table bin_pty_table[] = { | ||
57 | { CTL_INT, PTY_MAX, "max" }, | ||
58 | { CTL_INT, PTY_NR, "nr" }, | ||
59 | {} | ||
60 | }; | ||
61 | |||
62 | static const struct bin_table bin_kern_table[] = { | ||
63 | { CTL_STR, KERN_OSTYPE, "ostype" }, | ||
64 | { CTL_STR, KERN_OSRELEASE, "osrelease" }, | ||
65 | /* KERN_OSREV not used */ | ||
66 | { CTL_STR, KERN_VERSION, "version" }, | ||
67 | /* KERN_SECUREMASK not used */ | ||
68 | /* KERN_PROF not used */ | ||
69 | { CTL_STR, KERN_NODENAME, "hostname" }, | ||
70 | { CTL_STR, KERN_DOMAINNAME, "domainname" }, | ||
71 | |||
72 | { CTL_INT, KERN_PANIC, "panic" }, | ||
73 | { CTL_INT, KERN_REALROOTDEV, "real-root-dev" }, | ||
74 | |||
75 | { CTL_STR, KERN_SPARC_REBOOT, "reboot-cmd" }, | ||
76 | { CTL_INT, KERN_CTLALTDEL, "ctrl-alt-del" }, | ||
77 | { CTL_INT, KERN_PRINTK, "printk" }, | ||
78 | |||
79 | /* KERN_NAMETRANS not used */ | ||
80 | /* KERN_PPC_HTABRECLAIM not used */ | ||
81 | /* KERN_PPC_ZEROPAGED not used */ | ||
82 | { CTL_INT, KERN_PPC_POWERSAVE_NAP, "powersave-nap" }, | ||
83 | |||
84 | { CTL_STR, KERN_MODPROBE, "modprobe" }, | ||
85 | { CTL_INT, KERN_SG_BIG_BUFF, "sg-big-buff" }, | ||
86 | { CTL_INT, KERN_ACCT, "acct" }, | ||
87 | /* KERN_PPC_L2CR "l2cr" no longer used */ | ||
88 | |||
89 | /* KERN_RTSIGNR not used */ | ||
90 | /* KERN_RTSIGMAX not used */ | ||
91 | |||
92 | { CTL_ULONG, KERN_SHMMAX, "shmmax" }, | ||
93 | { CTL_INT, KERN_MSGMAX, "msgmax" }, | ||
94 | { CTL_INT, KERN_MSGMNB, "msgmnb" }, | ||
95 | /* KERN_MSGPOOL not used*/ | ||
96 | { CTL_INT, KERN_SYSRQ, "sysrq" }, | ||
97 | { CTL_INT, KERN_MAX_THREADS, "threads-max" }, | ||
98 | { CTL_DIR, KERN_RANDOM, "random", bin_random_table }, | ||
99 | { CTL_ULONG, KERN_SHMALL, "shmall" }, | ||
100 | { CTL_INT, KERN_MSGMNI, "msgmni" }, | ||
101 | { CTL_INT, KERN_SEM, "sem" }, | ||
102 | { CTL_INT, KERN_SPARC_STOP_A, "stop-a" }, | ||
103 | { CTL_INT, KERN_SHMMNI, "shmmni" }, | ||
104 | |||
105 | { CTL_INT, KERN_OVERFLOWUID, "overflowuid" }, | ||
106 | { CTL_INT, KERN_OVERFLOWGID, "overflowgid" }, | ||
107 | |||
108 | { CTL_STR, KERN_HOTPLUG, "hotplug", }, | ||
109 | { CTL_INT, KERN_IEEE_EMULATION_WARNINGS, "ieee_emulation_warnings" }, | ||
110 | |||
111 | { CTL_INT, KERN_S390_USER_DEBUG_LOGGING, "userprocess_debug" }, | ||
112 | { CTL_INT, KERN_CORE_USES_PID, "core_uses_pid" }, | ||
113 | /* KERN_TAINTED "tainted" no longer used */ | ||
114 | { CTL_INT, KERN_CADPID, "cad_pid" }, | ||
115 | { CTL_INT, KERN_PIDMAX, "pid_max" }, | ||
116 | { CTL_STR, KERN_CORE_PATTERN, "core_pattern" }, | ||
117 | { CTL_INT, KERN_PANIC_ON_OOPS, "panic_on_oops" }, | ||
118 | { CTL_INT, KERN_HPPA_PWRSW, "soft-power" }, | ||
119 | { CTL_INT, KERN_HPPA_UNALIGNED, "unaligned-trap" }, | ||
120 | |||
121 | { CTL_INT, KERN_PRINTK_RATELIMIT, "printk_ratelimit" }, | ||
122 | { CTL_INT, KERN_PRINTK_RATELIMIT_BURST, "printk_ratelimit_burst" }, | ||
123 | |||
124 | { CTL_DIR, KERN_PTY, "pty", bin_pty_table }, | ||
125 | { CTL_INT, KERN_NGROUPS_MAX, "ngroups_max" }, | ||
126 | { CTL_INT, KERN_SPARC_SCONS_PWROFF, "scons-poweroff" }, | ||
127 | /* KERN_HZ_TIMER "hz_timer" no longer used */ | ||
128 | { CTL_INT, KERN_UNKNOWN_NMI_PANIC, "unknown_nmi_panic" }, | ||
129 | { CTL_INT, KERN_BOOTLOADER_TYPE, "bootloader_type" }, | ||
130 | { CTL_INT, KERN_RANDOMIZE, "randomize_va_space" }, | ||
131 | |||
132 | { CTL_INT, KERN_SPIN_RETRY, "spin_retry" }, | ||
133 | /* KERN_ACPI_VIDEO_FLAGS "acpi_video_flags" no longer used */ | ||
134 | { CTL_INT, KERN_IA64_UNALIGNED, "ignore-unaligned-usertrap" }, | ||
135 | { CTL_INT, KERN_COMPAT_LOG, "compat-log" }, | ||
136 | { CTL_INT, KERN_MAX_LOCK_DEPTH, "max_lock_depth" }, | ||
137 | { CTL_INT, KERN_NMI_WATCHDOG, "nmi_watchdog" }, | ||
138 | { CTL_INT, KERN_PANIC_ON_NMI, "panic_on_unrecovered_nmi" }, | ||
139 | {} | ||
140 | }; | ||
141 | |||
142 | static const struct bin_table bin_vm_table[] = { | ||
143 | { CTL_INT, VM_OVERCOMMIT_MEMORY, "overcommit_memory" }, | ||
144 | { CTL_INT, VM_PAGE_CLUSTER, "page-cluster" }, | ||
145 | { CTL_INT, VM_DIRTY_BACKGROUND, "dirty_background_ratio" }, | ||
146 | { CTL_INT, VM_DIRTY_RATIO, "dirty_ratio" }, | ||
147 | /* VM_DIRTY_WB_CS "dirty_writeback_centisecs" no longer used */ | ||
148 | /* VM_DIRTY_EXPIRE_CS "dirty_expire_centisecs" no longer used */ | ||
149 | { CTL_INT, VM_NR_PDFLUSH_THREADS, "nr_pdflush_threads" }, | ||
150 | { CTL_INT, VM_OVERCOMMIT_RATIO, "overcommit_ratio" }, | ||
151 | /* VM_PAGEBUF unused */ | ||
152 | /* VM_HUGETLB_PAGES "nr_hugepages" no longer used */ | ||
153 | { CTL_INT, VM_SWAPPINESS, "swappiness" }, | ||
154 | { CTL_INT, VM_LOWMEM_RESERVE_RATIO, "lowmem_reserve_ratio" }, | ||
155 | { CTL_INT, VM_MIN_FREE_KBYTES, "min_free_kbytes" }, | ||
156 | { CTL_INT, VM_MAX_MAP_COUNT, "max_map_count" }, | ||
157 | { CTL_INT, VM_LAPTOP_MODE, "laptop_mode" }, | ||
158 | { CTL_INT, VM_BLOCK_DUMP, "block_dump" }, | ||
159 | { CTL_INT, VM_HUGETLB_GROUP, "hugetlb_shm_group" }, | ||
160 | { CTL_INT, VM_VFS_CACHE_PRESSURE, "vfs_cache_pressure" }, | ||
161 | { CTL_INT, VM_LEGACY_VA_LAYOUT, "legacy_va_layout" }, | ||
162 | /* VM_SWAP_TOKEN_TIMEOUT unused */ | ||
163 | { CTL_INT, VM_DROP_PAGECACHE, "drop_caches" }, | ||
164 | { CTL_INT, VM_PERCPU_PAGELIST_FRACTION, "percpu_pagelist_fraction" }, | ||
165 | { CTL_INT, VM_ZONE_RECLAIM_MODE, "zone_reclaim_mode" }, | ||
166 | { CTL_INT, VM_MIN_UNMAPPED, "min_unmapped_ratio" }, | ||
167 | { CTL_INT, VM_PANIC_ON_OOM, "panic_on_oom" }, | ||
168 | { CTL_INT, VM_VDSO_ENABLED, "vdso_enabled" }, | ||
169 | { CTL_INT, VM_MIN_SLAB, "min_slab_ratio" }, | ||
170 | |||
171 | {} | ||
172 | }; | ||
173 | |||
174 | static const struct bin_table bin_net_core_table[] = { | ||
175 | { CTL_INT, NET_CORE_WMEM_MAX, "wmem_max" }, | ||
176 | { CTL_INT, NET_CORE_RMEM_MAX, "rmem_max" }, | ||
177 | { CTL_INT, NET_CORE_WMEM_DEFAULT, "wmem_default" }, | ||
178 | { CTL_INT, NET_CORE_RMEM_DEFAULT, "rmem_default" }, | ||
179 | /* NET_CORE_DESTROY_DELAY unused */ | ||
180 | { CTL_INT, NET_CORE_MAX_BACKLOG, "netdev_max_backlog" }, | ||
181 | /* NET_CORE_FASTROUTE unused */ | ||
182 | { CTL_INT, NET_CORE_MSG_COST, "message_cost" }, | ||
183 | { CTL_INT, NET_CORE_MSG_BURST, "message_burst" }, | ||
184 | { CTL_INT, NET_CORE_OPTMEM_MAX, "optmem_max" }, | ||
185 | /* NET_CORE_HOT_LIST_LENGTH unused */ | ||
186 | /* NET_CORE_DIVERT_VERSION unused */ | ||
187 | /* NET_CORE_NO_CONG_THRESH unused */ | ||
188 | /* NET_CORE_NO_CONG unused */ | ||
189 | /* NET_CORE_LO_CONG unused */ | ||
190 | /* NET_CORE_MOD_CONG unused */ | ||
191 | { CTL_INT, NET_CORE_DEV_WEIGHT, "dev_weight" }, | ||
192 | { CTL_INT, NET_CORE_SOMAXCONN, "somaxconn" }, | ||
193 | { CTL_INT, NET_CORE_BUDGET, "netdev_budget" }, | ||
194 | { CTL_INT, NET_CORE_AEVENT_ETIME, "xfrm_aevent_etime" }, | ||
195 | { CTL_INT, NET_CORE_AEVENT_RSEQTH, "xfrm_aevent_rseqth" }, | ||
196 | { CTL_INT, NET_CORE_WARNINGS, "warnings" }, | ||
197 | {}, | ||
198 | }; | ||
199 | |||
200 | static const struct bin_table bin_net_unix_table[] = { | ||
201 | /* NET_UNIX_DESTROY_DELAY unused */ | ||
202 | /* NET_UNIX_DELETE_DELAY unused */ | ||
203 | { CTL_INT, NET_UNIX_MAX_DGRAM_QLEN, "max_dgram_qlen" }, | ||
204 | {} | ||
205 | }; | ||
206 | |||
207 | static const struct bin_table bin_net_ipv4_route_table[] = { | ||
208 | { CTL_INT, NET_IPV4_ROUTE_FLUSH, "flush" }, | ||
209 | /* NET_IPV4_ROUTE_MIN_DELAY "min_delay" no longer used */ | ||
210 | /* NET_IPV4_ROUTE_MAX_DELAY "max_delay" no longer used */ | ||
211 | { CTL_INT, NET_IPV4_ROUTE_GC_THRESH, "gc_thresh" }, | ||
212 | { CTL_INT, NET_IPV4_ROUTE_MAX_SIZE, "max_size" }, | ||
213 | { CTL_INT, NET_IPV4_ROUTE_GC_MIN_INTERVAL, "gc_min_interval" }, | ||
214 | { CTL_INT, NET_IPV4_ROUTE_GC_MIN_INTERVAL_MS, "gc_min_interval_ms" }, | ||
215 | { CTL_INT, NET_IPV4_ROUTE_GC_TIMEOUT, "gc_timeout" }, | ||
216 | { CTL_INT, NET_IPV4_ROUTE_GC_INTERVAL, "gc_interval" }, | ||
217 | { CTL_INT, NET_IPV4_ROUTE_REDIRECT_LOAD, "redirect_load" }, | ||
218 | { CTL_INT, NET_IPV4_ROUTE_REDIRECT_NUMBER, "redirect_number" }, | ||
219 | { CTL_INT, NET_IPV4_ROUTE_REDIRECT_SILENCE, "redirect_silence" }, | ||
220 | { CTL_INT, NET_IPV4_ROUTE_ERROR_COST, "error_cost" }, | ||
221 | { CTL_INT, NET_IPV4_ROUTE_ERROR_BURST, "error_burst" }, | ||
222 | { CTL_INT, NET_IPV4_ROUTE_GC_ELASTICITY, "gc_elasticity" }, | ||
223 | { CTL_INT, NET_IPV4_ROUTE_MTU_EXPIRES, "mtu_expires" }, | ||
224 | { CTL_INT, NET_IPV4_ROUTE_MIN_PMTU, "min_pmtu" }, | ||
225 | { CTL_INT, NET_IPV4_ROUTE_MIN_ADVMSS, "min_adv_mss" }, | ||
226 | { CTL_INT, NET_IPV4_ROUTE_SECRET_INTERVAL, "secret_interval" }, | ||
227 | {} | ||
228 | }; | ||
229 | |||
230 | static const struct bin_table bin_net_ipv4_conf_vars_table[] = { | ||
231 | { CTL_INT, NET_IPV4_CONF_FORWARDING, "forwarding" }, | ||
232 | { CTL_INT, NET_IPV4_CONF_MC_FORWARDING, "mc_forwarding" }, | ||
233 | |||
234 | { CTL_INT, NET_IPV4_CONF_ACCEPT_REDIRECTS, "accept_redirects" }, | ||
235 | { CTL_INT, NET_IPV4_CONF_SECURE_REDIRECTS, "secure_redirects" }, | ||
236 | { CTL_INT, NET_IPV4_CONF_SEND_REDIRECTS, "send_redirects" }, | ||
237 | { CTL_INT, NET_IPV4_CONF_SHARED_MEDIA, "shared_media" }, | ||
238 | { CTL_INT, NET_IPV4_CONF_RP_FILTER, "rp_filter" }, | ||
239 | { CTL_INT, NET_IPV4_CONF_ACCEPT_SOURCE_ROUTE, "accept_source_route" }, | ||
240 | { CTL_INT, NET_IPV4_CONF_PROXY_ARP, "proxy_arp" }, | ||
241 | { CTL_INT, NET_IPV4_CONF_MEDIUM_ID, "medium_id" }, | ||
242 | { CTL_INT, NET_IPV4_CONF_BOOTP_RELAY, "bootp_relay" }, | ||
243 | { CTL_INT, NET_IPV4_CONF_LOG_MARTIANS, "log_martians" }, | ||
244 | { CTL_INT, NET_IPV4_CONF_TAG, "tag" }, | ||
245 | { CTL_INT, NET_IPV4_CONF_ARPFILTER, "arp_filter" }, | ||
246 | { CTL_INT, NET_IPV4_CONF_ARP_ANNOUNCE, "arp_announce" }, | ||
247 | { CTL_INT, NET_IPV4_CONF_ARP_IGNORE, "arp_ignore" }, | ||
248 | { CTL_INT, NET_IPV4_CONF_ARP_ACCEPT, "arp_accept" }, | ||
249 | { CTL_INT, NET_IPV4_CONF_ARP_NOTIFY, "arp_notify" }, | ||
250 | |||
251 | { CTL_INT, NET_IPV4_CONF_NOXFRM, "disable_xfrm" }, | ||
252 | { CTL_INT, NET_IPV4_CONF_NOPOLICY, "disable_policy" }, | ||
253 | { CTL_INT, NET_IPV4_CONF_FORCE_IGMP_VERSION, "force_igmp_version" }, | ||
254 | { CTL_INT, NET_IPV4_CONF_PROMOTE_SECONDARIES, "promote_secondaries" }, | ||
255 | {} | ||
256 | }; | ||
257 | |||
258 | static const struct bin_table bin_net_ipv4_conf_table[] = { | ||
259 | { CTL_DIR, NET_PROTO_CONF_ALL, "all", bin_net_ipv4_conf_vars_table }, | ||
260 | { CTL_DIR, NET_PROTO_CONF_DEFAULT, "default", bin_net_ipv4_conf_vars_table }, | ||
261 | { CTL_DIR, 0, NULL, bin_net_ipv4_conf_vars_table }, | ||
262 | {} | ||
263 | }; | ||
264 | |||
265 | static const struct bin_table bin_net_neigh_vars_table[] = { | ||
266 | { CTL_INT, NET_NEIGH_MCAST_SOLICIT, "mcast_solicit" }, | ||
267 | { CTL_INT, NET_NEIGH_UCAST_SOLICIT, "ucast_solicit" }, | ||
268 | { CTL_INT, NET_NEIGH_APP_SOLICIT, "app_solicit" }, | ||
269 | /* NET_NEIGH_RETRANS_TIME "retrans_time" no longer used */ | ||
270 | { CTL_INT, NET_NEIGH_REACHABLE_TIME, "base_reachable_time" }, | ||
271 | { CTL_INT, NET_NEIGH_DELAY_PROBE_TIME, "delay_first_probe_time" }, | ||
272 | { CTL_INT, NET_NEIGH_GC_STALE_TIME, "gc_stale_time" }, | ||
273 | { CTL_INT, NET_NEIGH_UNRES_QLEN, "unres_qlen" }, | ||
274 | { CTL_INT, NET_NEIGH_PROXY_QLEN, "proxy_qlen" }, | ||
275 | /* NET_NEIGH_ANYCAST_DELAY "anycast_delay" no longer used */ | ||
276 | /* NET_NEIGH_PROXY_DELAY "proxy_delay" no longer used */ | ||
277 | /* NET_NEIGH_LOCKTIME "locktime" no longer used */ | ||
278 | { CTL_INT, NET_NEIGH_GC_INTERVAL, "gc_interval" }, | ||
279 | { CTL_INT, NET_NEIGH_GC_THRESH1, "gc_thresh1" }, | ||
280 | { CTL_INT, NET_NEIGH_GC_THRESH2, "gc_thresh2" }, | ||
281 | { CTL_INT, NET_NEIGH_GC_THRESH3, "gc_thresh3" }, | ||
282 | { CTL_INT, NET_NEIGH_RETRANS_TIME_MS, "retrans_time_ms" }, | ||
283 | { CTL_INT, NET_NEIGH_REACHABLE_TIME_MS, "base_reachable_time_ms" }, | ||
284 | {} | ||
285 | }; | ||
286 | |||
287 | static const struct bin_table bin_net_neigh_table[] = { | ||
288 | { CTL_DIR, NET_PROTO_CONF_DEFAULT, "default", bin_net_neigh_vars_table }, | ||
289 | { CTL_DIR, 0, NULL, bin_net_neigh_vars_table }, | ||
290 | {} | ||
291 | }; | ||
292 | |||
293 | static const struct bin_table bin_net_ipv4_netfilter_table[] = { | ||
294 | { CTL_INT, NET_IPV4_NF_CONNTRACK_MAX, "ip_conntrack_max" }, | ||
295 | |||
296 | /* NET_IPV4_NF_CONNTRACK_TCP_TIMEOUT_SYN_SENT "ip_conntrack_tcp_timeout_syn_sent" no longer used */ | ||
297 | /* NET_IPV4_NF_CONNTRACK_TCP_TIMEOUT_SYN_RECV "ip_conntrack_tcp_timeout_syn_recv" no longer used */ | ||
298 | /* NET_IPV4_NF_CONNTRACK_TCP_TIMEOUT_ESTABLISHED "ip_conntrack_tcp_timeout_established" no longer used */ | ||
299 | /* NET_IPV4_NF_CONNTRACK_TCP_TIMEOUT_FIN_WAIT "ip_conntrack_tcp_timeout_fin_wait" no longer used */ | ||
300 | /* NET_IPV4_NF_CONNTRACK_TCP_TIMEOUT_CLOSE_WAIT "ip_conntrack_tcp_timeout_close_wait" no longer used */ | ||
301 | /* NET_IPV4_NF_CONNTRACK_TCP_TIMEOUT_LAST_ACK "ip_conntrack_tcp_timeout_last_ack" no longer used */ | ||
302 | /* NET_IPV4_NF_CONNTRACK_TCP_TIMEOUT_TIME_WAIT "ip_conntrack_tcp_timeout_time_wait" no longer used */ | ||
303 | /* NET_IPV4_NF_CONNTRACK_TCP_TIMEOUT_CLOSE "ip_conntrack_tcp_timeout_close" no longer used */ | ||
304 | |||
305 | /* NET_IPV4_NF_CONNTRACK_UDP_TIMEOUT "ip_conntrack_udp_timeout" no longer used */ | ||
306 | /* NET_IPV4_NF_CONNTRACK_UDP_TIMEOUT_STREAM "ip_conntrack_udp_timeout_stream" no longer used */ | ||
307 | /* NET_IPV4_NF_CONNTRACK_ICMP_TIMEOUT "ip_conntrack_icmp_timeout" no longer used */ | ||
308 | /* NET_IPV4_NF_CONNTRACK_GENERIC_TIMEOUT "ip_conntrack_generic_timeout" no longer used */ | ||
309 | |||
310 | { CTL_INT, NET_IPV4_NF_CONNTRACK_BUCKETS, "ip_conntrack_buckets" }, | ||
311 | { CTL_INT, NET_IPV4_NF_CONNTRACK_LOG_INVALID, "ip_conntrack_log_invalid" }, | ||
312 | /* NET_IPV4_NF_CONNTRACK_TCP_TIMEOUT_MAX_RETRANS "ip_conntrack_tcp_timeout_max_retrans" no longer used */ | ||
313 | { CTL_INT, NET_IPV4_NF_CONNTRACK_TCP_LOOSE, "ip_conntrack_tcp_loose" }, | ||
314 | { CTL_INT, NET_IPV4_NF_CONNTRACK_TCP_BE_LIBERAL, "ip_conntrack_tcp_be_liberal" }, | ||
315 | { CTL_INT, NET_IPV4_NF_CONNTRACK_TCP_MAX_RETRANS, "ip_conntrack_tcp_max_retrans" }, | ||
316 | |||
317 | /* NET_IPV4_NF_CONNTRACK_SCTP_TIMEOUT_CLOSED "ip_conntrack_sctp_timeout_closed" no longer used */ | ||
318 | /* NET_IPV4_NF_CONNTRACK_SCTP_TIMEOUT_COOKIE_WAIT "ip_conntrack_sctp_timeout_cookie_wait" no longer used */ | ||
319 | /* NET_IPV4_NF_CONNTRACK_SCTP_TIMEOUT_COOKIE_ECHOED "ip_conntrack_sctp_timeout_cookie_echoed" no longer used */ | ||
320 | /* NET_IPV4_NF_CONNTRACK_SCTP_TIMEOUT_ESTABLISHED "ip_conntrack_sctp_timeout_established" no longer used */ | ||
321 | /* NET_IPV4_NF_CONNTRACK_SCTP_TIMEOUT_SHUTDOWN_SENT "ip_conntrack_sctp_timeout_shutdown_sent" no longer used */ | ||
322 | /* NET_IPV4_NF_CONNTRACK_SCTP_TIMEOUT_SHUTDOWN_RECD "ip_conntrack_sctp_timeout_shutdown_recd" no longer used */ | ||
323 | /* NET_IPV4_NF_CONNTRACK_SCTP_TIMEOUT_SHUTDOWN_ACK_SENT "ip_conntrack_sctp_timeout_shutdown_ack_sent" no longer used */ | ||
324 | |||
325 | { CTL_INT, NET_IPV4_NF_CONNTRACK_COUNT, "ip_conntrack_count" }, | ||
326 | { CTL_INT, NET_IPV4_NF_CONNTRACK_CHECKSUM, "ip_conntrack_checksum" }, | ||
327 | {} | ||
328 | }; | ||
329 | |||
330 | static const struct bin_table bin_net_ipv4_table[] = { | ||
331 | {CTL_INT, NET_IPV4_FORWARD, "ip_forward" }, | ||
332 | |||
333 | { CTL_DIR, NET_IPV4_CONF, "conf", bin_net_ipv4_conf_table }, | ||
334 | { CTL_DIR, NET_IPV4_NEIGH, "neigh", bin_net_neigh_table }, | ||
335 | { CTL_DIR, NET_IPV4_ROUTE, "route", bin_net_ipv4_route_table }, | ||
336 | /* NET_IPV4_FIB_HASH unused */ | ||
337 | { CTL_DIR, NET_IPV4_NETFILTER, "netfilter", bin_net_ipv4_netfilter_table }, | ||
338 | |||
339 | { CTL_INT, NET_IPV4_TCP_TIMESTAMPS, "tcp_timestamps" }, | ||
340 | { CTL_INT, NET_IPV4_TCP_WINDOW_SCALING, "tcp_window_scaling" }, | ||
341 | { CTL_INT, NET_IPV4_TCP_SACK, "tcp_sack" }, | ||
342 | { CTL_INT, NET_IPV4_TCP_RETRANS_COLLAPSE, "tcp_retrans_collapse" }, | ||
343 | { CTL_INT, NET_IPV4_DEFAULT_TTL, "ip_default_ttl" }, | ||
344 | /* NET_IPV4_AUTOCONFIG unused */ | ||
345 | { CTL_INT, NET_IPV4_NO_PMTU_DISC, "ip_no_pmtu_disc" }, | ||
346 | { CTL_INT, NET_IPV4_NONLOCAL_BIND, "ip_nonlocal_bind" }, | ||
347 | { CTL_INT, NET_IPV4_TCP_SYN_RETRIES, "tcp_syn_retries" }, | ||
348 | { CTL_INT, NET_TCP_SYNACK_RETRIES, "tcp_synack_retries" }, | ||
349 | { CTL_INT, NET_TCP_MAX_ORPHANS, "tcp_max_orphans" }, | ||
350 | { CTL_INT, NET_TCP_MAX_TW_BUCKETS, "tcp_max_tw_buckets" }, | ||
351 | { CTL_INT, NET_IPV4_DYNADDR, "ip_dynaddr" }, | ||
352 | { CTL_INT, NET_IPV4_TCP_KEEPALIVE_TIME, "tcp_keepalive_time" }, | ||
353 | { CTL_INT, NET_IPV4_TCP_KEEPALIVE_PROBES, "tcp_keepalive_probes" }, | ||
354 | { CTL_INT, NET_IPV4_TCP_KEEPALIVE_INTVL, "tcp_keepalive_intvl" }, | ||
355 | { CTL_INT, NET_IPV4_TCP_RETRIES1, "tcp_retries1" }, | ||
356 | { CTL_INT, NET_IPV4_TCP_RETRIES2, "tcp_retries2" }, | ||
357 | { CTL_INT, NET_IPV4_TCP_FIN_TIMEOUT, "tcp_fin_timeout" }, | ||
358 | { CTL_INT, NET_TCP_SYNCOOKIES, "tcp_syncookies" }, | ||
359 | { CTL_INT, NET_TCP_TW_RECYCLE, "tcp_tw_recycle" }, | ||
360 | { CTL_INT, NET_TCP_ABORT_ON_OVERFLOW, "tcp_abort_on_overflow" }, | ||
361 | { CTL_INT, NET_TCP_STDURG, "tcp_stdurg" }, | ||
362 | { CTL_INT, NET_TCP_RFC1337, "tcp_rfc1337" }, | ||
363 | { CTL_INT, NET_TCP_MAX_SYN_BACKLOG, "tcp_max_syn_backlog" }, | ||
364 | { CTL_INT, NET_IPV4_LOCAL_PORT_RANGE, "ip_local_port_range" }, | ||
365 | { CTL_INT, NET_IPV4_IGMP_MAX_MEMBERSHIPS, "igmp_max_memberships" }, | ||
366 | { CTL_INT, NET_IPV4_IGMP_MAX_MSF, "igmp_max_msf" }, | ||
367 | { CTL_INT, NET_IPV4_INET_PEER_THRESHOLD, "inet_peer_threshold" }, | ||
368 | { CTL_INT, NET_IPV4_INET_PEER_MINTTL, "inet_peer_minttl" }, | ||
369 | { CTL_INT, NET_IPV4_INET_PEER_MAXTTL, "inet_peer_maxttl" }, | ||
370 | { CTL_INT, NET_IPV4_INET_PEER_GC_MINTIME, "inet_peer_gc_mintime" }, | ||
371 | { CTL_INT, NET_IPV4_INET_PEER_GC_MAXTIME, "inet_peer_gc_maxtime" }, | ||
372 | { CTL_INT, NET_TCP_ORPHAN_RETRIES, "tcp_orphan_retries" }, | ||
373 | { CTL_INT, NET_TCP_FACK, "tcp_fack" }, | ||
374 | { CTL_INT, NET_TCP_REORDERING, "tcp_reordering" }, | ||
375 | { CTL_INT, NET_TCP_ECN, "tcp_ecn" }, | ||
376 | { CTL_INT, NET_TCP_DSACK, "tcp_dsack" }, | ||
377 | { CTL_INT, NET_TCP_MEM, "tcp_mem" }, | ||
378 | { CTL_INT, NET_TCP_WMEM, "tcp_wmem" }, | ||
379 | { CTL_INT, NET_TCP_RMEM, "tcp_rmem" }, | ||
380 | { CTL_INT, NET_TCP_APP_WIN, "tcp_app_win" }, | ||
381 | { CTL_INT, NET_TCP_ADV_WIN_SCALE, "tcp_adv_win_scale" }, | ||
382 | { CTL_INT, NET_TCP_TW_REUSE, "tcp_tw_reuse" }, | ||
383 | { CTL_INT, NET_TCP_FRTO, "tcp_frto" }, | ||
384 | { CTL_INT, NET_TCP_FRTO_RESPONSE, "tcp_frto_response" }, | ||
385 | { CTL_INT, NET_TCP_LOW_LATENCY, "tcp_low_latency" }, | ||
386 | { CTL_INT, NET_TCP_NO_METRICS_SAVE, "tcp_no_metrics_save" }, | ||
387 | { CTL_INT, NET_TCP_MODERATE_RCVBUF, "tcp_moderate_rcvbuf" }, | ||
388 | { CTL_INT, NET_TCP_TSO_WIN_DIVISOR, "tcp_tso_win_divisor" }, | ||
389 | { CTL_STR, NET_TCP_CONG_CONTROL, "tcp_congestion_control" }, | ||
390 | { CTL_INT, NET_TCP_ABC, "tcp_abc" }, | ||
391 | { CTL_INT, NET_TCP_MTU_PROBING, "tcp_mtu_probing" }, | ||
392 | { CTL_INT, NET_TCP_BASE_MSS, "tcp_base_mss" }, | ||
393 | { CTL_INT, NET_IPV4_TCP_WORKAROUND_SIGNED_WINDOWS, "tcp_workaround_signed_windows" }, | ||
394 | { CTL_INT, NET_TCP_DMA_COPYBREAK, "tcp_dma_copybreak" }, | ||
395 | { CTL_INT, NET_TCP_SLOW_START_AFTER_IDLE, "tcp_slow_start_after_idle" }, | ||
396 | { CTL_INT, NET_CIPSOV4_CACHE_ENABLE, "cipso_cache_enable" }, | ||
397 | { CTL_INT, NET_CIPSOV4_CACHE_BUCKET_SIZE, "cipso_cache_bucket_size" }, | ||
398 | { CTL_INT, NET_CIPSOV4_RBM_OPTFMT, "cipso_rbm_optfmt" }, | ||
399 | { CTL_INT, NET_CIPSOV4_RBM_STRICTVALID, "cipso_rbm_strictvalid" }, | ||
400 | /* NET_TCP_AVAIL_CONG_CONTROL "tcp_available_congestion_control" no longer used */ | ||
401 | { CTL_STR, NET_TCP_ALLOWED_CONG_CONTROL, "tcp_allowed_congestion_control" }, | ||
402 | { CTL_INT, NET_TCP_MAX_SSTHRESH, "tcp_max_ssthresh" }, | ||
403 | |||
404 | { CTL_INT, NET_IPV4_ICMP_ECHO_IGNORE_ALL, "icmp_echo_ignore_all" }, | ||
405 | { CTL_INT, NET_IPV4_ICMP_ECHO_IGNORE_BROADCASTS, "icmp_echo_ignore_broadcasts" }, | ||
406 | { CTL_INT, NET_IPV4_ICMP_IGNORE_BOGUS_ERROR_RESPONSES, "icmp_ignore_bogus_error_responses" }, | ||
407 | { CTL_INT, NET_IPV4_ICMP_ERRORS_USE_INBOUND_IFADDR, "icmp_errors_use_inbound_ifaddr" }, | ||
408 | { CTL_INT, NET_IPV4_ICMP_RATELIMIT, "icmp_ratelimit" }, | ||
409 | { CTL_INT, NET_IPV4_ICMP_RATEMASK, "icmp_ratemask" }, | ||
410 | |||
411 | { CTL_INT, NET_IPV4_IPFRAG_HIGH_THRESH, "ipfrag_high_thresh" }, | ||
412 | { CTL_INT, NET_IPV4_IPFRAG_LOW_THRESH, "ipfrag_low_thresh" }, | ||
413 | { CTL_INT, NET_IPV4_IPFRAG_TIME, "ipfrag_time" }, | ||
414 | |||
415 | { CTL_INT, NET_IPV4_IPFRAG_SECRET_INTERVAL, "ipfrag_secret_interval" }, | ||
416 | /* NET_IPV4_IPFRAG_MAX_DIST "ipfrag_max_dist" no longer used */ | ||
417 | |||
418 | { CTL_INT, 2088 /* NET_IPQ_QMAX */, "ip_queue_maxlen" }, | ||
419 | |||
420 | /* NET_TCP_DEFAULT_WIN_SCALE unused */ | ||
421 | /* NET_TCP_BIC_BETA unused */ | ||
422 | /* NET_IPV4_TCP_MAX_KA_PROBES unused */ | ||
423 | /* NET_IPV4_IP_MASQ_DEBUG unused */ | ||
424 | /* NET_TCP_SYN_TAILDROP unused */ | ||
425 | /* NET_IPV4_ICMP_SOURCEQUENCH_RATE unused */ | ||
426 | /* NET_IPV4_ICMP_DESTUNREACH_RATE unused */ | ||
427 | /* NET_IPV4_ICMP_TIMEEXCEED_RATE unused */ | ||
428 | /* NET_IPV4_ICMP_PARAMPROB_RATE unused */ | ||
429 | /* NET_IPV4_ICMP_ECHOREPLY_RATE unused */ | ||
430 | /* NET_IPV4_ALWAYS_DEFRAG unused */ | ||
431 | {} | ||
432 | }; | ||
433 | |||
434 | static const struct bin_table bin_net_ipx_table[] = { | ||
435 | { CTL_INT, NET_IPX_PPROP_BROADCASTING, "ipx_pprop_broadcasting" }, | ||
436 | /* NET_IPX_FORWARDING unused */ | ||
437 | {} | ||
438 | }; | ||
439 | |||
440 | static const struct bin_table bin_net_atalk_table[] = { | ||
441 | { CTL_INT, NET_ATALK_AARP_EXPIRY_TIME, "aarp-expiry-time" }, | ||
442 | { CTL_INT, NET_ATALK_AARP_TICK_TIME, "aarp-tick-time" }, | ||
443 | { CTL_INT, NET_ATALK_AARP_RETRANSMIT_LIMIT, "aarp-retransmit-limit" }, | ||
444 | { CTL_INT, NET_ATALK_AARP_RESOLVE_TIME, "aarp-resolve-time" }, | ||
445 | {}, | ||
446 | }; | ||
447 | |||
448 | static const struct bin_table bin_net_netrom_table[] = { | ||
449 | { CTL_INT, NET_NETROM_DEFAULT_PATH_QUALITY, "default_path_quality" }, | ||
450 | { CTL_INT, NET_NETROM_OBSOLESCENCE_COUNT_INITIALISER, "obsolescence_count_initialiser" }, | ||
451 | { CTL_INT, NET_NETROM_NETWORK_TTL_INITIALISER, "network_ttl_initialiser" }, | ||
452 | { CTL_INT, NET_NETROM_TRANSPORT_TIMEOUT, "transport_timeout" }, | ||
453 | { CTL_INT, NET_NETROM_TRANSPORT_MAXIMUM_TRIES, "transport_maximum_tries" }, | ||
454 | { CTL_INT, NET_NETROM_TRANSPORT_ACKNOWLEDGE_DELAY, "transport_acknowledge_delay" }, | ||
455 | { CTL_INT, NET_NETROM_TRANSPORT_BUSY_DELAY, "transport_busy_delay" }, | ||
456 | { CTL_INT, NET_NETROM_TRANSPORT_REQUESTED_WINDOW_SIZE, "transport_requested_window_size" }, | ||
457 | { CTL_INT, NET_NETROM_TRANSPORT_NO_ACTIVITY_TIMEOUT, "transport_no_activity_timeout" }, | ||
458 | { CTL_INT, NET_NETROM_ROUTING_CONTROL, "routing_control" }, | ||
459 | { CTL_INT, NET_NETROM_LINK_FAILS_COUNT, "link_fails_count" }, | ||
460 | { CTL_INT, NET_NETROM_RESET, "reset" }, | ||
461 | {} | ||
462 | }; | ||
463 | |||
464 | static const struct bin_table bin_net_ax25_param_table[] = { | ||
465 | { CTL_INT, NET_AX25_IP_DEFAULT_MODE, "ip_default_mode" }, | ||
466 | { CTL_INT, NET_AX25_DEFAULT_MODE, "ax25_default_mode" }, | ||
467 | { CTL_INT, NET_AX25_BACKOFF_TYPE, "backoff_type" }, | ||
468 | { CTL_INT, NET_AX25_CONNECT_MODE, "connect_mode" }, | ||
469 | { CTL_INT, NET_AX25_STANDARD_WINDOW, "standard_window_size" }, | ||
470 | { CTL_INT, NET_AX25_EXTENDED_WINDOW, "extended_window_size" }, | ||
471 | { CTL_INT, NET_AX25_T1_TIMEOUT, "t1_timeout" }, | ||
472 | { CTL_INT, NET_AX25_T2_TIMEOUT, "t2_timeout" }, | ||
473 | { CTL_INT, NET_AX25_T3_TIMEOUT, "t3_timeout" }, | ||
474 | { CTL_INT, NET_AX25_IDLE_TIMEOUT, "idle_timeout" }, | ||
475 | { CTL_INT, NET_AX25_N2, "maximum_retry_count" }, | ||
476 | { CTL_INT, NET_AX25_PACLEN, "maximum_packet_length" }, | ||
477 | { CTL_INT, NET_AX25_PROTOCOL, "protocol" }, | ||
478 | { CTL_INT, NET_AX25_DAMA_SLAVE_TIMEOUT, "dama_slave_timeout" }, | ||
479 | {} | ||
480 | }; | ||
481 | |||
482 | static const struct bin_table bin_net_ax25_table[] = { | ||
483 | { CTL_DIR, 0, NULL, bin_net_ax25_param_table }, | ||
484 | {} | ||
485 | }; | ||
486 | |||
487 | static const struct bin_table bin_net_rose_table[] = { | ||
488 | { CTL_INT, NET_ROSE_RESTART_REQUEST_TIMEOUT, "restart_request_timeout" }, | ||
489 | { CTL_INT, NET_ROSE_CALL_REQUEST_TIMEOUT, "call_request_timeout" }, | ||
490 | { CTL_INT, NET_ROSE_RESET_REQUEST_TIMEOUT, "reset_request_timeout" }, | ||
491 | { CTL_INT, NET_ROSE_CLEAR_REQUEST_TIMEOUT, "clear_request_timeout" }, | ||
492 | { CTL_INT, NET_ROSE_ACK_HOLD_BACK_TIMEOUT, "acknowledge_hold_back_timeout" }, | ||
493 | { CTL_INT, NET_ROSE_ROUTING_CONTROL, "routing_control" }, | ||
494 | { CTL_INT, NET_ROSE_LINK_FAIL_TIMEOUT, "link_fail_timeout" }, | ||
495 | { CTL_INT, NET_ROSE_MAX_VCS, "maximum_virtual_circuits" }, | ||
496 | { CTL_INT, NET_ROSE_WINDOW_SIZE, "window_size" }, | ||
497 | { CTL_INT, NET_ROSE_NO_ACTIVITY_TIMEOUT, "no_activity_timeout" }, | ||
498 | {} | ||
499 | }; | ||
500 | |||
501 | static const struct bin_table bin_net_ipv6_conf_var_table[] = { | ||
502 | { CTL_INT, NET_IPV6_FORWARDING, "forwarding" }, | ||
503 | { CTL_INT, NET_IPV6_HOP_LIMIT, "hop_limit" }, | ||
504 | { CTL_INT, NET_IPV6_MTU, "mtu" }, | ||
505 | { CTL_INT, NET_IPV6_ACCEPT_RA, "accept_ra" }, | ||
506 | { CTL_INT, NET_IPV6_ACCEPT_REDIRECTS, "accept_redirects" }, | ||
507 | { CTL_INT, NET_IPV6_AUTOCONF, "autoconf" }, | ||
508 | { CTL_INT, NET_IPV6_DAD_TRANSMITS, "dad_transmits" }, | ||
509 | { CTL_INT, NET_IPV6_RTR_SOLICITS, "router_solicitations" }, | ||
510 | { CTL_INT, NET_IPV6_RTR_SOLICIT_INTERVAL, "router_solicitation_interval" }, | ||
511 | { CTL_INT, NET_IPV6_RTR_SOLICIT_DELAY, "router_solicitation_delay" }, | ||
512 | { CTL_INT, NET_IPV6_USE_TEMPADDR, "use_tempaddr" }, | ||
513 | { CTL_INT, NET_IPV6_TEMP_VALID_LFT, "temp_valid_lft" }, | ||
514 | { CTL_INT, NET_IPV6_TEMP_PREFERED_LFT, "temp_prefered_lft" }, | ||
515 | { CTL_INT, NET_IPV6_REGEN_MAX_RETRY, "regen_max_retry" }, | ||
516 | { CTL_INT, NET_IPV6_MAX_DESYNC_FACTOR, "max_desync_factor" }, | ||
517 | { CTL_INT, NET_IPV6_MAX_ADDRESSES, "max_addresses" }, | ||
518 | { CTL_INT, NET_IPV6_FORCE_MLD_VERSION, "force_mld_version" }, | ||
519 | { CTL_INT, NET_IPV6_ACCEPT_RA_DEFRTR, "accept_ra_defrtr" }, | ||
520 | { CTL_INT, NET_IPV6_ACCEPT_RA_PINFO, "accept_ra_pinfo" }, | ||
521 | { CTL_INT, NET_IPV6_ACCEPT_RA_RTR_PREF, "accept_ra_rtr_pref" }, | ||
522 | { CTL_INT, NET_IPV6_RTR_PROBE_INTERVAL, "router_probe_interval" }, | ||
523 | { CTL_INT, NET_IPV6_ACCEPT_RA_RT_INFO_MAX_PLEN, "accept_ra_rt_info_max_plen" }, | ||
524 | { CTL_INT, NET_IPV6_PROXY_NDP, "proxy_ndp" }, | ||
525 | { CTL_INT, NET_IPV6_ACCEPT_SOURCE_ROUTE, "accept_source_route" }, | ||
526 | {} | ||
527 | }; | ||
528 | |||
529 | static const struct bin_table bin_net_ipv6_conf_table[] = { | ||
530 | { CTL_DIR, NET_PROTO_CONF_ALL, "all", bin_net_ipv6_conf_var_table }, | ||
531 | { CTL_DIR, NET_PROTO_CONF_DEFAULT, "default", bin_net_ipv6_conf_var_table }, | ||
532 | { CTL_DIR, 0, NULL, bin_net_ipv6_conf_var_table }, | ||
533 | {} | ||
534 | }; | ||
535 | |||
536 | static const struct bin_table bin_net_ipv6_route_table[] = { | ||
537 | /* NET_IPV6_ROUTE_FLUSH "flush" no longer used */ | ||
538 | { CTL_INT, NET_IPV6_ROUTE_GC_THRESH, "gc_thresh" }, | ||
539 | { CTL_INT, NET_IPV6_ROUTE_MAX_SIZE, "max_size" }, | ||
540 | { CTL_INT, NET_IPV6_ROUTE_GC_MIN_INTERVAL, "gc_min_interval" }, | ||
541 | { CTL_INT, NET_IPV6_ROUTE_GC_TIMEOUT, "gc_timeout" }, | ||
542 | { CTL_INT, NET_IPV6_ROUTE_GC_INTERVAL, "gc_interval" }, | ||
543 | { CTL_INT, NET_IPV6_ROUTE_GC_ELASTICITY, "gc_elasticity" }, | ||
544 | { CTL_INT, NET_IPV6_ROUTE_MTU_EXPIRES, "mtu_expires" }, | ||
545 | { CTL_INT, NET_IPV6_ROUTE_MIN_ADVMSS, "min_adv_mss" }, | ||
546 | { CTL_INT, NET_IPV6_ROUTE_GC_MIN_INTERVAL_MS, "gc_min_interval_ms" }, | ||
547 | {} | ||
548 | }; | ||
549 | |||
550 | static const struct bin_table bin_net_ipv6_icmp_table[] = { | ||
551 | { CTL_INT, NET_IPV6_ICMP_RATELIMIT, "ratelimit" }, | ||
552 | {} | ||
553 | }; | ||
554 | |||
555 | static const struct bin_table bin_net_ipv6_table[] = { | ||
556 | { CTL_DIR, NET_IPV6_CONF, "conf", bin_net_ipv6_conf_table }, | ||
557 | { CTL_DIR, NET_IPV6_NEIGH, "neigh", bin_net_neigh_table }, | ||
558 | { CTL_DIR, NET_IPV6_ROUTE, "route", bin_net_ipv6_route_table }, | ||
559 | { CTL_DIR, NET_IPV6_ICMP, "icmp", bin_net_ipv6_icmp_table }, | ||
560 | { CTL_INT, NET_IPV6_BINDV6ONLY, "bindv6only" }, | ||
561 | { CTL_INT, NET_IPV6_IP6FRAG_HIGH_THRESH, "ip6frag_high_thresh" }, | ||
562 | { CTL_INT, NET_IPV6_IP6FRAG_LOW_THRESH, "ip6frag_low_thresh" }, | ||
563 | { CTL_INT, NET_IPV6_IP6FRAG_TIME, "ip6frag_time" }, | ||
564 | { CTL_INT, NET_IPV6_IP6FRAG_SECRET_INTERVAL, "ip6frag_secret_interval" }, | ||
565 | { CTL_INT, NET_IPV6_MLD_MAX_MSF, "mld_max_msf" }, | ||
566 | { CTL_INT, 2088 /* IPQ_QMAX */, "ip6_queue_maxlen" }, | ||
567 | {} | ||
568 | }; | ||
569 | |||
570 | static const struct bin_table bin_net_x25_table[] = { | ||
571 | { CTL_INT, NET_X25_RESTART_REQUEST_TIMEOUT, "restart_request_timeout" }, | ||
572 | { CTL_INT, NET_X25_CALL_REQUEST_TIMEOUT, "call_request_timeout" }, | ||
573 | { CTL_INT, NET_X25_RESET_REQUEST_TIMEOUT, "reset_request_timeout" }, | ||
574 | { CTL_INT, NET_X25_CLEAR_REQUEST_TIMEOUT, "clear_request_timeout" }, | ||
575 | { CTL_INT, NET_X25_ACK_HOLD_BACK_TIMEOUT, "acknowledgement_hold_back_timeout" }, | ||
576 | { CTL_INT, NET_X25_FORWARD, "x25_forward" }, | ||
577 | {} | ||
578 | }; | ||
579 | |||
580 | static const struct bin_table bin_net_tr_table[] = { | ||
581 | { CTL_INT, NET_TR_RIF_TIMEOUT, "rif_timeout" }, | ||
582 | {} | ||
583 | }; | ||
584 | |||
585 | |||
586 | static const struct bin_table bin_net_decnet_conf_vars[] = { | ||
587 | { CTL_INT, NET_DECNET_CONF_DEV_FORWARDING, "forwarding" }, | ||
588 | { CTL_INT, NET_DECNET_CONF_DEV_PRIORITY, "priority" }, | ||
589 | { CTL_INT, NET_DECNET_CONF_DEV_T2, "t2" }, | ||
590 | { CTL_INT, NET_DECNET_CONF_DEV_T3, "t3" }, | ||
591 | {} | ||
592 | }; | ||
593 | |||
594 | static const struct bin_table bin_net_decnet_conf[] = { | ||
595 | { CTL_DIR, NET_DECNET_CONF_ETHER, "ethernet", bin_net_decnet_conf_vars }, | ||
596 | { CTL_DIR, NET_DECNET_CONF_GRE, "ipgre", bin_net_decnet_conf_vars }, | ||
597 | { CTL_DIR, NET_DECNET_CONF_X25, "x25", bin_net_decnet_conf_vars }, | ||
598 | { CTL_DIR, NET_DECNET_CONF_PPP, "ppp", bin_net_decnet_conf_vars }, | ||
599 | { CTL_DIR, NET_DECNET_CONF_DDCMP, "ddcmp", bin_net_decnet_conf_vars }, | ||
600 | { CTL_DIR, NET_DECNET_CONF_LOOPBACK, "loopback", bin_net_decnet_conf_vars }, | ||
601 | { CTL_DIR, 0, NULL, bin_net_decnet_conf_vars }, | ||
602 | {} | ||
603 | }; | ||
604 | |||
605 | static const struct bin_table bin_net_decnet_table[] = { | ||
606 | { CTL_DIR, NET_DECNET_CONF, "conf", bin_net_decnet_conf }, | ||
607 | { CTL_DNADR, NET_DECNET_NODE_ADDRESS, "node_address" }, | ||
608 | { CTL_STR, NET_DECNET_NODE_NAME, "node_name" }, | ||
609 | { CTL_STR, NET_DECNET_DEFAULT_DEVICE, "default_device" }, | ||
610 | { CTL_INT, NET_DECNET_TIME_WAIT, "time_wait" }, | ||
611 | { CTL_INT, NET_DECNET_DN_COUNT, "dn_count" }, | ||
612 | { CTL_INT, NET_DECNET_DI_COUNT, "di_count" }, | ||
613 | { CTL_INT, NET_DECNET_DR_COUNT, "dr_count" }, | ||
614 | { CTL_INT, NET_DECNET_DST_GC_INTERVAL, "dst_gc_interval" }, | ||
615 | { CTL_INT, NET_DECNET_NO_FC_MAX_CWND, "no_fc_max_cwnd" }, | ||
616 | { CTL_INT, NET_DECNET_MEM, "decnet_mem" }, | ||
617 | { CTL_INT, NET_DECNET_RMEM, "decnet_rmem" }, | ||
618 | { CTL_INT, NET_DECNET_WMEM, "decnet_wmem" }, | ||
619 | { CTL_INT, NET_DECNET_DEBUG_LEVEL, "debug" }, | ||
620 | {} | ||
621 | }; | ||
622 | |||
623 | static const struct bin_table bin_net_sctp_table[] = { | ||
624 | { CTL_INT, NET_SCTP_RTO_INITIAL, "rto_initial" }, | ||
625 | { CTL_INT, NET_SCTP_RTO_MIN, "rto_min" }, | ||
626 | { CTL_INT, NET_SCTP_RTO_MAX, "rto_max" }, | ||
627 | { CTL_INT, NET_SCTP_RTO_ALPHA, "rto_alpha_exp_divisor" }, | ||
628 | { CTL_INT, NET_SCTP_RTO_BETA, "rto_beta_exp_divisor" }, | ||
629 | { CTL_INT, NET_SCTP_VALID_COOKIE_LIFE, "valid_cookie_life" }, | ||
630 | { CTL_INT, NET_SCTP_ASSOCIATION_MAX_RETRANS, "association_max_retrans" }, | ||
631 | { CTL_INT, NET_SCTP_PATH_MAX_RETRANS, "path_max_retrans" }, | ||
632 | { CTL_INT, NET_SCTP_MAX_INIT_RETRANSMITS, "max_init_retransmits" }, | ||
633 | { CTL_INT, NET_SCTP_HB_INTERVAL, "hb_interval" }, | ||
634 | { CTL_INT, NET_SCTP_PRESERVE_ENABLE, "cookie_preserve_enable" }, | ||
635 | { CTL_INT, NET_SCTP_MAX_BURST, "max_burst" }, | ||
636 | { CTL_INT, NET_SCTP_ADDIP_ENABLE, "addip_enable" }, | ||
637 | { CTL_INT, NET_SCTP_PRSCTP_ENABLE, "prsctp_enable" }, | ||
638 | { CTL_INT, NET_SCTP_SNDBUF_POLICY, "sndbuf_policy" }, | ||
639 | { CTL_INT, NET_SCTP_SACK_TIMEOUT, "sack_timeout" }, | ||
640 | { CTL_INT, NET_SCTP_RCVBUF_POLICY, "rcvbuf_policy" }, | ||
641 | {} | ||
642 | }; | ||
643 | |||
644 | static const struct bin_table bin_net_llc_llc2_timeout_table[] = { | ||
645 | { CTL_INT, NET_LLC2_ACK_TIMEOUT, "ack" }, | ||
646 | { CTL_INT, NET_LLC2_P_TIMEOUT, "p" }, | ||
647 | { CTL_INT, NET_LLC2_REJ_TIMEOUT, "rej" }, | ||
648 | { CTL_INT, NET_LLC2_BUSY_TIMEOUT, "busy" }, | ||
649 | {} | ||
650 | }; | ||
651 | |||
652 | static const struct bin_table bin_net_llc_station_table[] = { | ||
653 | { CTL_INT, NET_LLC_STATION_ACK_TIMEOUT, "ack_timeout" }, | ||
654 | {} | ||
655 | }; | ||
656 | |||
657 | static const struct bin_table bin_net_llc_llc2_table[] = { | ||
658 | { CTL_DIR, NET_LLC2, "timeout", bin_net_llc_llc2_timeout_table }, | ||
659 | {} | ||
660 | }; | ||
661 | |||
662 | static const struct bin_table bin_net_llc_table[] = { | ||
663 | { CTL_DIR, NET_LLC2, "llc2", bin_net_llc_llc2_table }, | ||
664 | { CTL_DIR, NET_LLC_STATION, "station", bin_net_llc_station_table }, | ||
665 | {} | ||
666 | }; | ||
667 | |||
668 | static const struct bin_table bin_net_netfilter_table[] = { | ||
669 | { CTL_INT, NET_NF_CONNTRACK_MAX, "nf_conntrack_max" }, | ||
670 | /* NET_NF_CONNTRACK_TCP_TIMEOUT_SYN_SENT "nf_conntrack_tcp_timeout_syn_sent" no longer used */ | ||
671 | /* NET_NF_CONNTRACK_TCP_TIMEOUT_SYN_RECV "nf_conntrack_tcp_timeout_syn_recv" no longer used */ | ||
672 | /* NET_NF_CONNTRACK_TCP_TIMEOUT_ESTABLISHED "nf_conntrack_tcp_timeout_established" no longer used */ | ||
673 | /* NET_NF_CONNTRACK_TCP_TIMEOUT_FIN_WAIT "nf_conntrack_tcp_timeout_fin_wait" no longer used */ | ||
674 | /* NET_NF_CONNTRACK_TCP_TIMEOUT_CLOSE_WAIT "nf_conntrack_tcp_timeout_close_wait" no longer used */ | ||
675 | /* NET_NF_CONNTRACK_TCP_TIMEOUT_LAST_ACK "nf_conntrack_tcp_timeout_last_ack" no longer used */ | ||
676 | /* NET_NF_CONNTRACK_TCP_TIMEOUT_TIME_WAIT "nf_conntrack_tcp_timeout_time_wait" no longer used */ | ||
677 | /* NET_NF_CONNTRACK_TCP_TIMEOUT_CLOSE "nf_conntrack_tcp_timeout_close" no longer used */ | ||
678 | /* NET_NF_CONNTRACK_UDP_TIMEOUT "nf_conntrack_udp_timeout" no longer used */ | ||
679 | /* NET_NF_CONNTRACK_UDP_TIMEOUT_STREAM "nf_conntrack_udp_timeout_stream" no longer used */ | ||
680 | /* NET_NF_CONNTRACK_ICMP_TIMEOUT "nf_conntrack_icmp_timeout" no longer used */ | ||
681 | /* NET_NF_CONNTRACK_GENERIC_TIMEOUT "nf_conntrack_generic_timeout" no longer used */ | ||
682 | { CTL_INT, NET_NF_CONNTRACK_BUCKETS, "nf_conntrack_buckets" }, | ||
683 | { CTL_INT, NET_NF_CONNTRACK_LOG_INVALID, "nf_conntrack_log_invalid" }, | ||
684 | /* NET_NF_CONNTRACK_TCP_TIMEOUT_MAX_RETRANS "nf_conntrack_tcp_timeout_max_retrans" no longer used */ | ||
685 | { CTL_INT, NET_NF_CONNTRACK_TCP_LOOSE, "nf_conntrack_tcp_loose" }, | ||
686 | { CTL_INT, NET_NF_CONNTRACK_TCP_BE_LIBERAL, "nf_conntrack_tcp_be_liberal" }, | ||
687 | { CTL_INT, NET_NF_CONNTRACK_TCP_MAX_RETRANS, "nf_conntrack_tcp_max_retrans" }, | ||
688 | /* NET_NF_CONNTRACK_SCTP_TIMEOUT_CLOSED "nf_conntrack_sctp_timeout_closed" no longer used */ | ||
689 | /* NET_NF_CONNTRACK_SCTP_TIMEOUT_COOKIE_WAIT "nf_conntrack_sctp_timeout_cookie_wait" no longer used */ | ||
690 | /* NET_NF_CONNTRACK_SCTP_TIMEOUT_COOKIE_ECHOED "nf_conntrack_sctp_timeout_cookie_echoed" no longer used */ | ||
691 | /* NET_NF_CONNTRACK_SCTP_TIMEOUT_ESTABLISHED "nf_conntrack_sctp_timeout_established" no longer used */ | ||
692 | /* NET_NF_CONNTRACK_SCTP_TIMEOUT_SHUTDOWN_SENT "nf_conntrack_sctp_timeout_shutdown_sent" no longer used */ | ||
693 | /* NET_NF_CONNTRACK_SCTP_TIMEOUT_SHUTDOWN_RECD "nf_conntrack_sctp_timeout_shutdown_recd" no longer used */ | ||
694 | /* NET_NF_CONNTRACK_SCTP_TIMEOUT_SHUTDOWN_ACK_SENT "nf_conntrack_sctp_timeout_shutdown_ack_sent" no longer used */ | ||
695 | { CTL_INT, NET_NF_CONNTRACK_COUNT, "nf_conntrack_count" }, | ||
696 | /* NET_NF_CONNTRACK_ICMPV6_TIMEOUT "nf_conntrack_icmpv6_timeout" no longer used */ | ||
697 | /* NET_NF_CONNTRACK_FRAG6_TIMEOUT "nf_conntrack_frag6_timeout" no longer used */ | ||
698 | { CTL_INT, NET_NF_CONNTRACK_FRAG6_LOW_THRESH, "nf_conntrack_frag6_low_thresh" }, | ||
699 | { CTL_INT, NET_NF_CONNTRACK_FRAG6_HIGH_THRESH, "nf_conntrack_frag6_high_thresh" }, | ||
700 | { CTL_INT, NET_NF_CONNTRACK_CHECKSUM, "nf_conntrack_checksum" }, | ||
701 | |||
702 | {} | ||
703 | }; | ||
704 | |||
705 | static const struct bin_table bin_net_irda_table[] = { | ||
706 | { CTL_INT, NET_IRDA_DISCOVERY, "discovery" }, | ||
707 | { CTL_STR, NET_IRDA_DEVNAME, "devname" }, | ||
708 | { CTL_INT, NET_IRDA_DEBUG, "debug" }, | ||
709 | { CTL_INT, NET_IRDA_FAST_POLL, "fast_poll_increase" }, | ||
710 | { CTL_INT, NET_IRDA_DISCOVERY_SLOTS, "discovery_slots" }, | ||
711 | { CTL_INT, NET_IRDA_DISCOVERY_TIMEOUT, "discovery_timeout" }, | ||
712 | { CTL_INT, NET_IRDA_SLOT_TIMEOUT, "slot_timeout" }, | ||
713 | { CTL_INT, NET_IRDA_MAX_BAUD_RATE, "max_baud_rate" }, | ||
714 | { CTL_INT, NET_IRDA_MIN_TX_TURN_TIME, "min_tx_turn_time" }, | ||
715 | { CTL_INT, NET_IRDA_MAX_TX_DATA_SIZE, "max_tx_data_size" }, | ||
716 | { CTL_INT, NET_IRDA_MAX_TX_WINDOW, "max_tx_window" }, | ||
717 | { CTL_INT, NET_IRDA_MAX_NOREPLY_TIME, "max_noreply_time" }, | ||
718 | { CTL_INT, NET_IRDA_WARN_NOREPLY_TIME, "warn_noreply_time" }, | ||
719 | { CTL_INT, NET_IRDA_LAP_KEEPALIVE_TIME, "lap_keepalive_time" }, | ||
720 | {} | ||
721 | }; | ||
722 | |||
723 | static const struct bin_table bin_net_table[] = { | ||
724 | { CTL_DIR, NET_CORE, "core", bin_net_core_table }, | ||
725 | /* NET_ETHER not used */ | ||
726 | /* NET_802 not used */ | ||
727 | { CTL_DIR, NET_UNIX, "unix", bin_net_unix_table }, | ||
728 | { CTL_DIR, NET_IPV4, "ipv4", bin_net_ipv4_table }, | ||
729 | { CTL_DIR, NET_IPX, "ipx", bin_net_ipx_table }, | ||
730 | { CTL_DIR, NET_ATALK, "appletalk", bin_net_atalk_table }, | ||
731 | { CTL_DIR, NET_NETROM, "netrom", bin_net_netrom_table }, | ||
732 | { CTL_DIR, NET_AX25, "ax25", bin_net_ax25_table }, | ||
733 | /* NET_BRIDGE "bridge" no longer used */ | ||
734 | { CTL_DIR, NET_ROSE, "rose", bin_net_rose_table }, | ||
735 | { CTL_DIR, NET_IPV6, "ipv6", bin_net_ipv6_table }, | ||
736 | { CTL_DIR, NET_X25, "x25", bin_net_x25_table }, | ||
737 | { CTL_DIR, NET_TR, "token-ring", bin_net_tr_table }, | ||
738 | { CTL_DIR, NET_DECNET, "decnet", bin_net_decnet_table }, | ||
739 | /* NET_ECONET not used */ | ||
740 | { CTL_DIR, NET_SCTP, "sctp", bin_net_sctp_table }, | ||
741 | { CTL_DIR, NET_LLC, "llc", bin_net_llc_table }, | ||
742 | { CTL_DIR, NET_NETFILTER, "netfilter", bin_net_netfilter_table }, | ||
743 | /* NET_DCCP "dccp" no longer used */ | ||
744 | { CTL_DIR, NET_IRDA, "irda", bin_net_irda_table }, | ||
745 | { CTL_INT, 2089, "nf_conntrack_max" }, | ||
746 | {} | ||
747 | }; | ||
748 | |||
749 | static const struct bin_table bin_fs_quota_table[] = { | ||
750 | { CTL_INT, FS_DQ_LOOKUPS, "lookups" }, | ||
751 | { CTL_INT, FS_DQ_DROPS, "drops" }, | ||
752 | { CTL_INT, FS_DQ_READS, "reads" }, | ||
753 | { CTL_INT, FS_DQ_WRITES, "writes" }, | ||
754 | { CTL_INT, FS_DQ_CACHE_HITS, "cache_hits" }, | ||
755 | { CTL_INT, FS_DQ_ALLOCATED, "allocated_dquots" }, | ||
756 | { CTL_INT, FS_DQ_FREE, "free_dquots" }, | ||
757 | { CTL_INT, FS_DQ_SYNCS, "syncs" }, | ||
758 | { CTL_INT, FS_DQ_WARNINGS, "warnings" }, | ||
759 | {} | ||
760 | }; | ||
761 | |||
762 | static const struct bin_table bin_fs_xfs_table[] = { | ||
763 | { CTL_INT, XFS_SGID_INHERIT, "irix_sgid_inherit" }, | ||
764 | { CTL_INT, XFS_SYMLINK_MODE, "irix_symlink_mode" }, | ||
765 | { CTL_INT, XFS_PANIC_MASK, "panic_mask" }, | ||
766 | |||
767 | { CTL_INT, XFS_ERRLEVEL, "error_level" }, | ||
768 | { CTL_INT, XFS_SYNCD_TIMER, "xfssyncd_centisecs" }, | ||
769 | { CTL_INT, XFS_INHERIT_SYNC, "inherit_sync" }, | ||
770 | { CTL_INT, XFS_INHERIT_NODUMP, "inherit_nodump" }, | ||
771 | { CTL_INT, XFS_INHERIT_NOATIME, "inherit_noatime" }, | ||
772 | { CTL_INT, XFS_BUF_TIMER, "xfsbufd_centisecs" }, | ||
773 | { CTL_INT, XFS_BUF_AGE, "age_buffer_centisecs" }, | ||
774 | { CTL_INT, XFS_INHERIT_NOSYM, "inherit_nosymlinks" }, | ||
775 | { CTL_INT, XFS_ROTORSTEP, "rotorstep" }, | ||
776 | { CTL_INT, XFS_INHERIT_NODFRG, "inherit_nodefrag" }, | ||
777 | { CTL_INT, XFS_FILESTREAM_TIMER, "filestream_centisecs" }, | ||
778 | { CTL_INT, XFS_STATS_CLEAR, "stats_clear" }, | ||
779 | {} | ||
780 | }; | ||
781 | |||
782 | static const struct bin_table bin_fs_ocfs2_nm_table[] = { | ||
783 | { CTL_STR, 1, "hb_ctl_path" }, | ||
784 | {} | ||
785 | }; | ||
786 | |||
787 | static const struct bin_table bin_fs_ocfs2_table[] = { | ||
788 | { CTL_DIR, 1, "nm", bin_fs_ocfs2_nm_table }, | ||
789 | {} | ||
790 | }; | ||
791 | |||
792 | static const struct bin_table bin_inotify_table[] = { | ||
793 | { CTL_INT, INOTIFY_MAX_USER_INSTANCES, "max_user_instances" }, | ||
794 | { CTL_INT, INOTIFY_MAX_USER_WATCHES, "max_user_watches" }, | ||
795 | { CTL_INT, INOTIFY_MAX_QUEUED_EVENTS, "max_queued_events" }, | ||
796 | {} | ||
797 | }; | ||
798 | |||
799 | static const struct bin_table bin_fs_table[] = { | ||
800 | { CTL_INT, FS_NRINODE, "inode-nr" }, | ||
801 | { CTL_INT, FS_STATINODE, "inode-state" }, | ||
802 | /* FS_MAXINODE unused */ | ||
803 | /* FS_NRDQUOT unused */ | ||
804 | /* FS_MAXDQUOT unused */ | ||
805 | /* FS_NRFILE "file-nr" no longer used */ | ||
806 | { CTL_INT, FS_MAXFILE, "file-max" }, | ||
807 | { CTL_INT, FS_DENTRY, "dentry-state" }, | ||
808 | /* FS_NRSUPER unused */ | ||
809 | /* FS_MAXUPSER unused */ | ||
810 | { CTL_INT, FS_OVERFLOWUID, "overflowuid" }, | ||
811 | { CTL_INT, FS_OVERFLOWGID, "overflowgid" }, | ||
812 | { CTL_INT, FS_LEASES, "leases-enable" }, | ||
813 | { CTL_INT, FS_DIR_NOTIFY, "dir-notify-enable" }, | ||
814 | { CTL_INT, FS_LEASE_TIME, "lease-break-time" }, | ||
815 | { CTL_DIR, FS_DQSTATS, "quota", bin_fs_quota_table }, | ||
816 | { CTL_DIR, FS_XFS, "xfs", bin_fs_xfs_table }, | ||
817 | { CTL_ULONG, FS_AIO_NR, "aio-nr" }, | ||
818 | { CTL_ULONG, FS_AIO_MAX_NR, "aio-max-nr" }, | ||
819 | { CTL_DIR, FS_INOTIFY, "inotify", bin_inotify_table }, | ||
820 | { CTL_DIR, FS_OCFS2, "ocfs2", bin_fs_ocfs2_table }, | ||
821 | { CTL_INT, KERN_SETUID_DUMPABLE, "suid_dumpable" }, | ||
822 | {} | ||
823 | }; | ||
824 | |||
825 | static const struct bin_table bin_ipmi_table[] = { | ||
826 | { CTL_INT, DEV_IPMI_POWEROFF_POWERCYCLE, "poweroff_powercycle" }, | ||
827 | {} | ||
828 | }; | ||
829 | |||
830 | static const struct bin_table bin_mac_hid_files[] = { | ||
831 | /* DEV_MAC_HID_KEYBOARD_SENDS_LINUX_KEYCODES unused */ | ||
832 | /* DEV_MAC_HID_KEYBOARD_LOCK_KEYCODES unused */ | ||
833 | { CTL_INT, DEV_MAC_HID_MOUSE_BUTTON_EMULATION, "mouse_button_emulation" }, | ||
834 | { CTL_INT, DEV_MAC_HID_MOUSE_BUTTON2_KEYCODE, "mouse_button2_keycode" }, | ||
835 | { CTL_INT, DEV_MAC_HID_MOUSE_BUTTON3_KEYCODE, "mouse_button3_keycode" }, | ||
836 | /* DEV_MAC_HID_ADB_MOUSE_SENDS_KEYCODES unused */ | ||
837 | {} | ||
838 | }; | ||
839 | |||
840 | static const struct bin_table bin_raid_table[] = { | ||
841 | { CTL_INT, DEV_RAID_SPEED_LIMIT_MIN, "speed_limit_min" }, | ||
842 | { CTL_INT, DEV_RAID_SPEED_LIMIT_MAX, "speed_limit_max" }, | ||
843 | {} | ||
844 | }; | ||
845 | |||
846 | static const struct bin_table bin_scsi_table[] = { | ||
847 | { CTL_INT, DEV_SCSI_LOGGING_LEVEL, "logging_level" }, | ||
848 | {} | ||
849 | }; | ||
850 | |||
851 | static const struct bin_table bin_dev_table[] = { | ||
852 | /* DEV_CDROM "cdrom" no longer used */ | ||
853 | /* DEV_HWMON unused */ | ||
854 | /* DEV_PARPORT "parport" no longer used */ | ||
855 | { CTL_DIR, DEV_RAID, "raid", bin_raid_table }, | ||
856 | { CTL_DIR, DEV_MAC_HID, "mac_hid", bin_mac_hid_files }, | ||
857 | { CTL_DIR, DEV_SCSI, "scsi", bin_scsi_table }, | ||
858 | { CTL_DIR, DEV_IPMI, "ipmi", bin_ipmi_table }, | ||
859 | {} | ||
860 | }; | ||
861 | |||
862 | static const struct bin_table bin_bus_isa_table[] = { | ||
863 | { CTL_INT, BUS_ISA_MEM_BASE, "membase" }, | ||
864 | { CTL_INT, BUS_ISA_PORT_BASE, "portbase" }, | ||
865 | { CTL_INT, BUS_ISA_PORT_SHIFT, "portshift" }, | ||
866 | {} | ||
867 | }; | ||
868 | |||
869 | static const struct bin_table bin_bus_table[] = { | ||
870 | { CTL_DIR, CTL_BUS_ISA, "isa", bin_bus_isa_table }, | ||
871 | {} | ||
872 | }; | ||
873 | |||
874 | |||
875 | static const struct bin_table bin_s390dbf_table[] = { | ||
876 | { CTL_INT, 5678 /* CTL_S390DBF_STOPPABLE */, "debug_stoppable" }, | ||
877 | { CTL_INT, 5679 /* CTL_S390DBF_ACTIVE */, "debug_active" }, | ||
878 | {} | ||
879 | }; | ||
880 | |||
881 | static const struct bin_table bin_sunrpc_table[] = { | ||
882 | /* CTL_RPCDEBUG "rpc_debug" no longer used */ | ||
883 | /* CTL_NFSDEBUG "nfs_debug" no longer used */ | ||
884 | /* CTL_NFSDDEBUG "nfsd_debug" no longer used */ | ||
885 | /* CTL_NLMDEBUG "nlm_debug" no longer used */ | ||
886 | |||
887 | { CTL_INT, CTL_SLOTTABLE_UDP, "udp_slot_table_entries" }, | ||
888 | { CTL_INT, CTL_SLOTTABLE_TCP, "tcp_slot_table_entries" }, | ||
889 | { CTL_INT, CTL_MIN_RESVPORT, "min_resvport" }, | ||
890 | { CTL_INT, CTL_MAX_RESVPORT, "max_resvport" }, | ||
891 | {} | ||
892 | }; | ||
893 | |||
894 | static const struct bin_table bin_pm_table[] = { | ||
895 | /* frv specific */ | ||
896 | /* 1 == CTL_PM_SUSPEND "suspend" no longer used" */ | ||
897 | { CTL_INT, 2 /* CTL_PM_CMODE */, "cmode" }, | ||
898 | { CTL_INT, 3 /* CTL_PM_P0 */, "p0" }, | ||
899 | { CTL_INT, 4 /* CTL_PM_CM */, "cm" }, | ||
900 | {} | ||
901 | }; | ||
902 | |||
903 | static const struct bin_table bin_root_table[] = { | ||
904 | { CTL_DIR, CTL_KERN, "kernel", bin_kern_table }, | ||
905 | { CTL_DIR, CTL_VM, "vm", bin_vm_table }, | ||
906 | { CTL_DIR, CTL_NET, "net", bin_net_table }, | ||
907 | /* CTL_PROC not used */ | ||
908 | { CTL_DIR, CTL_FS, "fs", bin_fs_table }, | ||
909 | /* CTL_DEBUG "debug" no longer used */ | ||
910 | { CTL_DIR, CTL_DEV, "dev", bin_dev_table }, | ||
911 | { CTL_DIR, CTL_BUS, "bus", bin_bus_table }, | ||
912 | { CTL_DIR, CTL_ABI, "abi" }, | ||
913 | /* CTL_CPU not used */ | ||
914 | /* CTL_ARLAN "arlan" no longer used */ | ||
915 | { CTL_DIR, CTL_S390DBF, "s390dbf", bin_s390dbf_table }, | ||
916 | { CTL_DIR, CTL_SUNRPC, "sunrpc", bin_sunrpc_table }, | ||
917 | { CTL_DIR, CTL_PM, "pm", bin_pm_table }, | ||
918 | {} | ||
919 | }; | ||
920 | |||
921 | static ssize_t bin_dir(struct file *file, | ||
922 | void __user *oldval, size_t oldlen, void __user *newval, size_t newlen) | ||
923 | { | ||
924 | return -ENOTDIR; | ||
925 | } | ||
926 | |||
927 | |||
928 | static ssize_t bin_string(struct file *file, | ||
929 | void __user *oldval, size_t oldlen, void __user *newval, size_t newlen) | ||
930 | { | ||
931 | ssize_t result, copied = 0; | ||
932 | |||
933 | if (oldval && oldlen) { | ||
934 | char __user *lastp; | ||
935 | loff_t pos = 0; | ||
936 | int ch; | ||
937 | |||
938 | result = vfs_read(file, oldval, oldlen, &pos); | ||
939 | if (result < 0) | ||
940 | goto out; | ||
941 | |||
942 | copied = result; | ||
943 | lastp = oldval + copied - 1; | ||
944 | |||
945 | result = -EFAULT; | ||
946 | if (get_user(ch, lastp)) | ||
947 | goto out; | ||
948 | |||
949 | /* Trim off the trailing newline */ | ||
950 | if (ch == '\n') { | ||
951 | result = -EFAULT; | ||
952 | if (put_user('\0', lastp)) | ||
953 | goto out; | ||
954 | copied -= 1; | ||
955 | } | ||
956 | } | ||
957 | |||
958 | if (newval && newlen) { | ||
959 | loff_t pos = 0; | ||
960 | |||
961 | result = vfs_write(file, newval, newlen, &pos); | ||
962 | if (result < 0) | ||
963 | goto out; | ||
964 | } | ||
965 | |||
966 | result = copied; | ||
967 | out: | ||
968 | return result; | ||
969 | } | ||
970 | |||
971 | static ssize_t bin_intvec(struct file *file, | ||
972 | void __user *oldval, size_t oldlen, void __user *newval, size_t newlen) | ||
973 | { | ||
974 | mm_segment_t old_fs = get_fs(); | ||
975 | ssize_t copied = 0; | ||
976 | char *buffer; | ||
977 | ssize_t result; | ||
978 | |||
979 | result = -ENOMEM; | ||
980 | buffer = kmalloc(BUFSZ, GFP_KERNEL); | ||
981 | if (!buffer) | ||
982 | goto out; | ||
983 | |||
984 | if (oldval && oldlen) { | ||
985 | unsigned __user *vec = oldval; | ||
986 | size_t length = oldlen / sizeof(*vec); | ||
987 | loff_t pos = 0; | ||
988 | char *str, *end; | ||
989 | int i; | ||
990 | |||
991 | set_fs(KERNEL_DS); | ||
992 | result = vfs_read(file, buffer, BUFSZ - 1, &pos); | ||
993 | set_fs(old_fs); | ||
994 | if (result < 0) | ||
995 | goto out_kfree; | ||
996 | |||
997 | str = buffer; | ||
998 | end = str + result; | ||
999 | *end++ = '\0'; | ||
1000 | for (i = 0; i < length; i++) { | ||
1001 | unsigned long value; | ||
1002 | |||
1003 | value = simple_strtoul(str, &str, 10); | ||
1004 | while (isspace(*str)) | ||
1005 | str++; | ||
1006 | |||
1007 | result = -EFAULT; | ||
1008 | if (put_user(value, vec + i)) | ||
1009 | goto out_kfree; | ||
1010 | |||
1011 | copied += sizeof(*vec); | ||
1012 | if (!isdigit(*str)) | ||
1013 | break; | ||
1014 | } | ||
1015 | } | ||
1016 | |||
1017 | if (newval && newlen) { | ||
1018 | unsigned __user *vec = newval; | ||
1019 | size_t length = newlen / sizeof(*vec); | ||
1020 | loff_t pos = 0; | ||
1021 | char *str, *end; | ||
1022 | int i; | ||
1023 | |||
1024 | str = buffer; | ||
1025 | end = str + BUFSZ; | ||
1026 | for (i = 0; i < length; i++) { | ||
1027 | unsigned long value; | ||
1028 | |||
1029 | result = -EFAULT; | ||
1030 | if (get_user(value, vec + i)) | ||
1031 | goto out_kfree; | ||
1032 | |||
1033 | str += snprintf(str, end - str, "%lu\t", value); | ||
1034 | } | ||
1035 | |||
1036 | set_fs(KERNEL_DS); | ||
1037 | result = vfs_write(file, buffer, str - buffer, &pos); | ||
1038 | set_fs(old_fs); | ||
1039 | if (result < 0) | ||
1040 | goto out_kfree; | ||
1041 | } | ||
1042 | result = copied; | ||
1043 | out_kfree: | ||
1044 | kfree(buffer); | ||
1045 | out: | ||
1046 | return result; | ||
1047 | } | ||
1048 | |||
1049 | static ssize_t bin_ulongvec(struct file *file, | ||
1050 | void __user *oldval, size_t oldlen, void __user *newval, size_t newlen) | ||
1051 | { | ||
1052 | mm_segment_t old_fs = get_fs(); | ||
1053 | ssize_t copied = 0; | ||
1054 | char *buffer; | ||
1055 | ssize_t result; | ||
1056 | |||
1057 | result = -ENOMEM; | ||
1058 | buffer = kmalloc(BUFSZ, GFP_KERNEL); | ||
1059 | if (!buffer) | ||
1060 | goto out; | ||
1061 | |||
1062 | if (oldval && oldlen) { | ||
1063 | unsigned long __user *vec = oldval; | ||
1064 | size_t length = oldlen / sizeof(*vec); | ||
1065 | loff_t pos = 0; | ||
1066 | char *str, *end; | ||
1067 | int i; | ||
1068 | |||
1069 | set_fs(KERNEL_DS); | ||
1070 | result = vfs_read(file, buffer, BUFSZ - 1, &pos); | ||
1071 | set_fs(old_fs); | ||
1072 | if (result < 0) | ||
1073 | goto out_kfree; | ||
1074 | |||
1075 | str = buffer; | ||
1076 | end = str + result; | ||
1077 | *end++ = '\0'; | ||
1078 | for (i = 0; i < length; i++) { | ||
1079 | unsigned long value; | ||
1080 | |||
1081 | value = simple_strtoul(str, &str, 10); | ||
1082 | while (isspace(*str)) | ||
1083 | str++; | ||
1084 | |||
1085 | result = -EFAULT; | ||
1086 | if (put_user(value, vec + i)) | ||
1087 | goto out_kfree; | ||
1088 | |||
1089 | copied += sizeof(*vec); | ||
1090 | if (!isdigit(*str)) | ||
1091 | break; | ||
1092 | } | ||
1093 | } | ||
1094 | |||
1095 | if (newval && newlen) { | ||
1096 | unsigned long __user *vec = newval; | ||
1097 | size_t length = newlen / sizeof(*vec); | ||
1098 | loff_t pos = 0; | ||
1099 | char *str, *end; | ||
1100 | int i; | ||
1101 | |||
1102 | str = buffer; | ||
1103 | end = str + BUFSZ; | ||
1104 | for (i = 0; i < length; i++) { | ||
1105 | unsigned long value; | ||
1106 | |||
1107 | result = -EFAULT; | ||
1108 | if (get_user(value, vec + i)) | ||
1109 | goto out_kfree; | ||
1110 | |||
1111 | str += snprintf(str, end - str, "%lu\t", value); | ||
1112 | } | ||
1113 | |||
1114 | set_fs(KERNEL_DS); | ||
1115 | result = vfs_write(file, buffer, str - buffer, &pos); | ||
1116 | set_fs(old_fs); | ||
1117 | if (result < 0) | ||
1118 | goto out_kfree; | ||
1119 | } | ||
1120 | result = copied; | ||
1121 | out_kfree: | ||
1122 | kfree(buffer); | ||
1123 | out: | ||
1124 | return result; | ||
1125 | } | ||
1126 | |||
1127 | static unsigned hex_value(int ch) | ||
1128 | { | ||
1129 | return isdigit(ch) ? ch - '0' : ((ch | 0x20) - 'a') + 10; | ||
1130 | } | ||
1131 | |||
1132 | static ssize_t bin_uuid(struct file *file, | ||
1133 | void __user *oldval, size_t oldlen, void __user *newval, size_t newlen) | ||
1134 | { | ||
1135 | mm_segment_t old_fs = get_fs(); | ||
1136 | ssize_t result, copied = 0; | ||
1137 | |||
1138 | /* Only supports reads */ | ||
1139 | if (oldval && oldlen) { | ||
1140 | loff_t pos = 0; | ||
1141 | char buf[40], *str = buf; | ||
1142 | unsigned char uuid[16]; | ||
1143 | int i; | ||
1144 | |||
1145 | set_fs(KERNEL_DS); | ||
1146 | result = vfs_read(file, buf, sizeof(buf) - 1, &pos); | ||
1147 | set_fs(old_fs); | ||
1148 | if (result < 0) | ||
1149 | goto out; | ||
1150 | |||
1151 | buf[result] = '\0'; | ||
1152 | |||
1153 | /* Convert the uuid to from a string to binary */ | ||
1154 | for (i = 0; i < 16; i++) { | ||
1155 | result = -EIO; | ||
1156 | if (!isxdigit(str[0]) || !isxdigit(str[1])) | ||
1157 | goto out; | ||
1158 | |||
1159 | uuid[i] = (hex_value(str[0]) << 4) | hex_value(str[1]); | ||
1160 | str += 2; | ||
1161 | if (*str == '-') | ||
1162 | str++; | ||
1163 | } | ||
1164 | |||
1165 | if (oldlen > 16) | ||
1166 | oldlen = 16; | ||
1167 | |||
1168 | result = -EFAULT; | ||
1169 | if (copy_to_user(oldval, uuid, oldlen)) | ||
1170 | goto out; | ||
1171 | |||
1172 | copied = oldlen; | ||
1173 | } | ||
1174 | result = copied; | ||
1175 | out: | ||
1176 | return result; | ||
1177 | } | ||
1178 | |||
1179 | static ssize_t bin_dn_node_address(struct file *file, | ||
1180 | void __user *oldval, size_t oldlen, void __user *newval, size_t newlen) | ||
1181 | { | ||
1182 | mm_segment_t old_fs = get_fs(); | ||
1183 | ssize_t result, copied = 0; | ||
1184 | |||
1185 | if (oldval && oldlen) { | ||
1186 | loff_t pos = 0; | ||
1187 | char buf[15], *nodep; | ||
1188 | unsigned long area, node; | ||
1189 | __le16 dnaddr; | ||
1190 | |||
1191 | set_fs(KERNEL_DS); | ||
1192 | result = vfs_read(file, buf, sizeof(buf) - 1, &pos); | ||
1193 | set_fs(old_fs); | ||
1194 | if (result < 0) | ||
1195 | goto out; | ||
1196 | |||
1197 | buf[result] = '\0'; | ||
1198 | |||
1199 | /* Convert the decnet addresss to binary */ | ||
1200 | result = -EIO; | ||
1201 | nodep = strchr(buf, '.') + 1; | ||
1202 | if (!nodep) | ||
1203 | goto out; | ||
1204 | |||
1205 | area = simple_strtoul(buf, NULL, 10); | ||
1206 | node = simple_strtoul(nodep, NULL, 10); | ||
1207 | |||
1208 | result = -EIO; | ||
1209 | if ((area > 63)||(node > 1023)) | ||
1210 | goto out; | ||
1211 | |||
1212 | dnaddr = cpu_to_le16((area << 10) | node); | ||
1213 | |||
1214 | result = -EFAULT; | ||
1215 | if (put_user(dnaddr, (__le16 __user *)oldval)) | ||
1216 | goto out; | ||
1217 | |||
1218 | copied = sizeof(dnaddr); | ||
1219 | } | ||
1220 | |||
1221 | if (newval && newlen) { | ||
1222 | loff_t pos = 0; | ||
1223 | __le16 dnaddr; | ||
1224 | char buf[15]; | ||
1225 | int len; | ||
1226 | |||
1227 | result = -EINVAL; | ||
1228 | if (newlen != sizeof(dnaddr)) | ||
1229 | goto out; | ||
1230 | |||
1231 | result = -EFAULT; | ||
1232 | if (get_user(dnaddr, (__le16 __user *)newval)) | ||
1233 | goto out; | ||
1234 | |||
1235 | len = snprintf(buf, sizeof(buf), "%hu.%hu", | ||
1236 | le16_to_cpu(dnaddr) >> 10, | ||
1237 | le16_to_cpu(dnaddr) & 0x3ff); | ||
1238 | |||
1239 | set_fs(KERNEL_DS); | ||
1240 | result = vfs_write(file, buf, len, &pos); | ||
1241 | set_fs(old_fs); | ||
1242 | if (result < 0) | ||
1243 | goto out; | ||
1244 | } | ||
1245 | |||
1246 | result = copied; | ||
1247 | out: | ||
1248 | return result; | ||
1249 | } | ||
1250 | |||
1251 | static const struct bin_table *get_sysctl(const int *name, int nlen, char *path) | ||
1252 | { | ||
1253 | const struct bin_table *table = &bin_root_table[0]; | ||
1254 | int ctl_name; | ||
1255 | |||
1256 | /* The binary sysctl tables have a small maximum depth so | ||
1257 | * there is no danger of overflowing our path as it PATH_MAX | ||
1258 | * bytes long. | ||
1259 | */ | ||
1260 | memcpy(path, "sys/", 4); | ||
1261 | path += 4; | ||
1262 | |||
1263 | repeat: | ||
1264 | if (!nlen) | ||
1265 | return ERR_PTR(-ENOTDIR); | ||
1266 | ctl_name = *name; | ||
1267 | name++; | ||
1268 | nlen--; | ||
1269 | for ( ; table->convert; table++) { | ||
1270 | int len = 0; | ||
1271 | |||
1272 | /* | ||
1273 | * For a wild card entry map from ifindex to network | ||
1274 | * device name. | ||
1275 | */ | ||
1276 | if (!table->ctl_name) { | ||
1277 | #ifdef CONFIG_NET | ||
1278 | struct net *net = current->nsproxy->net_ns; | ||
1279 | struct net_device *dev; | ||
1280 | dev = dev_get_by_index(net, ctl_name); | ||
1281 | if (dev) { | ||
1282 | len = strlen(dev->name); | ||
1283 | memcpy(path, dev->name, len); | ||
1284 | dev_put(dev); | ||
1285 | } | ||
1286 | #endif | ||
1287 | /* Use the well known sysctl number to proc name mapping */ | ||
1288 | } else if (ctl_name == table->ctl_name) { | ||
1289 | len = strlen(table->procname); | ||
1290 | memcpy(path, table->procname, len); | ||
1291 | } | ||
1292 | if (len) { | ||
1293 | path += len; | ||
1294 | if (table->child) { | ||
1295 | *path++ = '/'; | ||
1296 | table = table->child; | ||
1297 | goto repeat; | ||
1298 | } | ||
1299 | *path = '\0'; | ||
1300 | return table; | ||
1301 | } | ||
1302 | } | ||
1303 | return ERR_PTR(-ENOTDIR); | ||
1304 | } | ||
1305 | |||
1306 | static char *sysctl_getname(const int *name, int nlen, const struct bin_table **tablep) | ||
1307 | { | ||
1308 | char *tmp, *result; | ||
1309 | |||
1310 | result = ERR_PTR(-ENOMEM); | ||
1311 | tmp = __getname(); | ||
1312 | if (tmp) { | ||
1313 | const struct bin_table *table = get_sysctl(name, nlen, tmp); | ||
1314 | result = tmp; | ||
1315 | *tablep = table; | ||
1316 | if (IS_ERR(table)) { | ||
1317 | __putname(tmp); | ||
1318 | result = ERR_CAST(table); | ||
1319 | } | ||
1320 | } | ||
1321 | return result; | ||
1322 | } | ||
1323 | |||
1324 | static ssize_t binary_sysctl(const int *name, int nlen, | ||
1325 | void __user *oldval, size_t oldlen, void __user *newval, size_t newlen) | ||
1326 | { | ||
1327 | const struct bin_table *table = NULL; | ||
1328 | struct nameidata nd; | ||
1329 | struct vfsmount *mnt; | ||
1330 | struct file *file; | ||
1331 | ssize_t result; | ||
1332 | char *pathname; | ||
1333 | int flags; | ||
1334 | int acc_mode, fmode; | ||
1335 | |||
1336 | pathname = sysctl_getname(name, nlen, &table); | ||
1337 | result = PTR_ERR(pathname); | ||
1338 | if (IS_ERR(pathname)) | ||
1339 | goto out; | ||
1340 | |||
1341 | /* How should the sysctl be accessed? */ | ||
1342 | if (oldval && oldlen && newval && newlen) { | ||
1343 | flags = O_RDWR; | ||
1344 | acc_mode = MAY_READ | MAY_WRITE; | ||
1345 | fmode = FMODE_READ | FMODE_WRITE; | ||
1346 | } else if (newval && newlen) { | ||
1347 | flags = O_WRONLY; | ||
1348 | acc_mode = MAY_WRITE; | ||
1349 | fmode = FMODE_WRITE; | ||
1350 | } else if (oldval && oldlen) { | ||
1351 | flags = O_RDONLY; | ||
1352 | acc_mode = MAY_READ; | ||
1353 | fmode = FMODE_READ; | ||
1354 | } else { | ||
1355 | result = 0; | ||
1356 | goto out_putname; | ||
1357 | } | ||
1358 | |||
1359 | mnt = current->nsproxy->pid_ns->proc_mnt; | ||
1360 | result = vfs_path_lookup(mnt->mnt_root, mnt, pathname, 0, &nd); | ||
1361 | if (result) | ||
1362 | goto out_putname; | ||
1363 | |||
1364 | result = may_open(&nd.path, acc_mode, fmode); | ||
1365 | if (result) | ||
1366 | goto out_putpath; | ||
1367 | |||
1368 | file = dentry_open(nd.path.dentry, nd.path.mnt, flags, current_cred()); | ||
1369 | result = PTR_ERR(file); | ||
1370 | if (IS_ERR(file)) | ||
1371 | goto out_putname; | ||
1372 | |||
1373 | result = table->convert(file, oldval, oldlen, newval, newlen); | ||
1374 | |||
1375 | fput(file); | ||
1376 | out_putname: | ||
1377 | putname(pathname); | ||
1378 | out: | ||
1379 | return result; | ||
1380 | |||
1381 | out_putpath: | ||
1382 | path_put(&nd.path); | ||
1383 | goto out_putname; | ||
1384 | } | ||
1385 | |||
1386 | |||
1387 | #else /* CONFIG_SYSCTL_SYSCALL */ | ||
1388 | |||
1389 | static ssize_t binary_sysctl(const int *name, int nlen, | ||
1390 | void __user *oldval, size_t oldlen, void __user *newval, size_t newlen) | ||
1391 | { | ||
1392 | return -ENOSYS; | ||
1393 | } | ||
1394 | |||
1395 | #endif /* CONFIG_SYSCTL_SYSCALL */ | ||
1396 | |||
1397 | |||
1398 | static void deprecated_sysctl_warning(const int *name, int nlen) | ||
1399 | { | ||
1400 | int i; | ||
1401 | |||
1402 | if (printk_ratelimit()) { | ||
1403 | printk(KERN_INFO | ||
1404 | "warning: process `%s' used the deprecated sysctl " | ||
1405 | "system call with ", current->comm); | ||
1406 | for (i = 0; i < nlen; i++) | ||
1407 | printk("%d.", name[i]); | ||
1408 | printk("\n"); | ||
1409 | } | ||
1410 | return; | ||
1411 | } | ||
1412 | |||
1413 | static ssize_t do_sysctl(int __user *args_name, int nlen, | ||
1414 | void __user *oldval, size_t oldlen, void __user *newval, size_t newlen) | ||
1415 | { | ||
1416 | int name[CTL_MAXNAME]; | ||
1417 | int i; | ||
1418 | |||
1419 | /* Check args->nlen. */ | ||
1420 | if (nlen < 0 || nlen > CTL_MAXNAME) | ||
1421 | return -ENOTDIR; | ||
1422 | /* Read in the sysctl name for simplicity */ | ||
1423 | for (i = 0; i < nlen; i++) | ||
1424 | if (get_user(name[i], args_name + i)) | ||
1425 | return -EFAULT; | ||
1426 | |||
1427 | deprecated_sysctl_warning(name, nlen); | ||
1428 | |||
1429 | return binary_sysctl(name, nlen, oldval, oldlen, newval, newlen); | ||
1430 | } | ||
1431 | |||
1432 | SYSCALL_DEFINE1(sysctl, struct __sysctl_args __user *, args) | ||
1433 | { | ||
1434 | struct __sysctl_args tmp; | ||
1435 | size_t oldlen = 0; | ||
1436 | ssize_t result; | ||
1437 | |||
1438 | if (copy_from_user(&tmp, args, sizeof(tmp))) | ||
1439 | return -EFAULT; | ||
1440 | |||
1441 | if (tmp.oldval && !tmp.oldlenp) | ||
1442 | return -EFAULT; | ||
1443 | |||
1444 | if (tmp.oldlenp && get_user(oldlen, tmp.oldlenp)) | ||
1445 | return -EFAULT; | ||
1446 | |||
1447 | result = do_sysctl(tmp.name, tmp.nlen, tmp.oldval, oldlen, | ||
1448 | tmp.newval, tmp.newlen); | ||
1449 | |||
1450 | if (result >= 0) { | ||
1451 | oldlen = result; | ||
1452 | result = 0; | ||
1453 | } | ||
1454 | |||
1455 | if (tmp.oldlenp && put_user(oldlen, tmp.oldlenp)) | ||
1456 | return -EFAULT; | ||
1457 | |||
1458 | return result; | ||
1459 | } | ||
1460 | |||
1461 | |||
1462 | #ifdef CONFIG_COMPAT | ||
1463 | #include <asm/compat.h> | ||
1464 | |||
1465 | struct compat_sysctl_args { | ||
1466 | compat_uptr_t name; | ||
1467 | int nlen; | ||
1468 | compat_uptr_t oldval; | ||
1469 | compat_uptr_t oldlenp; | ||
1470 | compat_uptr_t newval; | ||
1471 | compat_size_t newlen; | ||
1472 | compat_ulong_t __unused[4]; | ||
1473 | }; | ||
1474 | |||
1475 | asmlinkage long compat_sys_sysctl(struct compat_sysctl_args __user *args) | ||
1476 | { | ||
1477 | struct compat_sysctl_args tmp; | ||
1478 | compat_size_t __user *compat_oldlenp; | ||
1479 | size_t oldlen = 0; | ||
1480 | ssize_t result; | ||
1481 | |||
1482 | if (copy_from_user(&tmp, args, sizeof(tmp))) | ||
1483 | return -EFAULT; | ||
1484 | |||
1485 | if (tmp.oldval && !tmp.oldlenp) | ||
1486 | return -EFAULT; | ||
1487 | |||
1488 | compat_oldlenp = compat_ptr(tmp.oldlenp); | ||
1489 | if (compat_oldlenp && get_user(oldlen, compat_oldlenp)) | ||
1490 | return -EFAULT; | ||
1491 | |||
1492 | result = do_sysctl(compat_ptr(tmp.name), tmp.nlen, | ||
1493 | compat_ptr(tmp.oldval), oldlen, | ||
1494 | compat_ptr(tmp.newval), tmp.newlen); | ||
1495 | |||
1496 | if (result >= 0) { | ||
1497 | oldlen = result; | ||
1498 | result = 0; | ||
1499 | } | ||
1500 | |||
1501 | if (compat_oldlenp && put_user(oldlen, compat_oldlenp)) | ||
1502 | return -EFAULT; | ||
1503 | |||
1504 | return result; | ||
1505 | } | ||
1506 | |||
1507 | #endif /* CONFIG_COMPAT */ | ||
diff --git a/kernel/sysctl_check.c b/kernel/sysctl_check.c index b38423ca711a..04cdcf72c827 100644 --- a/kernel/sysctl_check.c +++ b/kernel/sysctl_check.c | |||
@@ -5,1239 +5,6 @@ | |||
5 | #include <linux/string.h> | 5 | #include <linux/string.h> |
6 | #include <net/ip_vs.h> | 6 | #include <net/ip_vs.h> |
7 | 7 | ||
8 | struct trans_ctl_table { | ||
9 | int ctl_name; | ||
10 | const char *procname; | ||
11 | const struct trans_ctl_table *child; | ||
12 | }; | ||
13 | |||
14 | static const struct trans_ctl_table trans_random_table[] = { | ||
15 | { RANDOM_POOLSIZE, "poolsize" }, | ||
16 | { RANDOM_ENTROPY_COUNT, "entropy_avail" }, | ||
17 | { RANDOM_READ_THRESH, "read_wakeup_threshold" }, | ||
18 | { RANDOM_WRITE_THRESH, "write_wakeup_threshold" }, | ||
19 | { RANDOM_BOOT_ID, "boot_id" }, | ||
20 | { RANDOM_UUID, "uuid" }, | ||
21 | {} | ||
22 | }; | ||
23 | |||
24 | static const struct trans_ctl_table trans_pty_table[] = { | ||
25 | { PTY_MAX, "max" }, | ||
26 | { PTY_NR, "nr" }, | ||
27 | {} | ||
28 | }; | ||
29 | |||
30 | static const struct trans_ctl_table trans_kern_table[] = { | ||
31 | { KERN_OSTYPE, "ostype" }, | ||
32 | { KERN_OSRELEASE, "osrelease" }, | ||
33 | /* KERN_OSREV not used */ | ||
34 | { KERN_VERSION, "version" }, | ||
35 | /* KERN_SECUREMASK not used */ | ||
36 | /* KERN_PROF not used */ | ||
37 | { KERN_NODENAME, "hostname" }, | ||
38 | { KERN_DOMAINNAME, "domainname" }, | ||
39 | |||
40 | { KERN_PANIC, "panic" }, | ||
41 | { KERN_REALROOTDEV, "real-root-dev" }, | ||
42 | |||
43 | { KERN_SPARC_REBOOT, "reboot-cmd" }, | ||
44 | { KERN_CTLALTDEL, "ctrl-alt-del" }, | ||
45 | { KERN_PRINTK, "printk" }, | ||
46 | |||
47 | /* KERN_NAMETRANS not used */ | ||
48 | /* KERN_PPC_HTABRECLAIM not used */ | ||
49 | /* KERN_PPC_ZEROPAGED not used */ | ||
50 | { KERN_PPC_POWERSAVE_NAP, "powersave-nap" }, | ||
51 | |||
52 | { KERN_MODPROBE, "modprobe" }, | ||
53 | { KERN_SG_BIG_BUFF, "sg-big-buff" }, | ||
54 | { KERN_ACCT, "acct" }, | ||
55 | { KERN_PPC_L2CR, "l2cr" }, | ||
56 | |||
57 | /* KERN_RTSIGNR not used */ | ||
58 | /* KERN_RTSIGMAX not used */ | ||
59 | |||
60 | { KERN_SHMMAX, "shmmax" }, | ||
61 | { KERN_MSGMAX, "msgmax" }, | ||
62 | { KERN_MSGMNB, "msgmnb" }, | ||
63 | /* KERN_MSGPOOL not used*/ | ||
64 | { KERN_SYSRQ, "sysrq" }, | ||
65 | { KERN_MAX_THREADS, "threads-max" }, | ||
66 | { KERN_RANDOM, "random", trans_random_table }, | ||
67 | { KERN_SHMALL, "shmall" }, | ||
68 | { KERN_MSGMNI, "msgmni" }, | ||
69 | { KERN_SEM, "sem" }, | ||
70 | { KERN_SPARC_STOP_A, "stop-a" }, | ||
71 | { KERN_SHMMNI, "shmmni" }, | ||
72 | |||
73 | { KERN_OVERFLOWUID, "overflowuid" }, | ||
74 | { KERN_OVERFLOWGID, "overflowgid" }, | ||
75 | |||
76 | { KERN_HOTPLUG, "hotplug", }, | ||
77 | { KERN_IEEE_EMULATION_WARNINGS, "ieee_emulation_warnings" }, | ||
78 | |||
79 | { KERN_S390_USER_DEBUG_LOGGING, "userprocess_debug" }, | ||
80 | { KERN_CORE_USES_PID, "core_uses_pid" }, | ||
81 | { KERN_TAINTED, "tainted" }, | ||
82 | { KERN_CADPID, "cad_pid" }, | ||
83 | { KERN_PIDMAX, "pid_max" }, | ||
84 | { KERN_CORE_PATTERN, "core_pattern" }, | ||
85 | { KERN_PANIC_ON_OOPS, "panic_on_oops" }, | ||
86 | { KERN_HPPA_PWRSW, "soft-power" }, | ||
87 | { KERN_HPPA_UNALIGNED, "unaligned-trap" }, | ||
88 | |||
89 | { KERN_PRINTK_RATELIMIT, "printk_ratelimit" }, | ||
90 | { KERN_PRINTK_RATELIMIT_BURST, "printk_ratelimit_burst" }, | ||
91 | |||
92 | { KERN_PTY, "pty", trans_pty_table }, | ||
93 | { KERN_NGROUPS_MAX, "ngroups_max" }, | ||
94 | { KERN_SPARC_SCONS_PWROFF, "scons-poweroff" }, | ||
95 | { KERN_HZ_TIMER, "hz_timer" }, | ||
96 | { KERN_UNKNOWN_NMI_PANIC, "unknown_nmi_panic" }, | ||
97 | { KERN_BOOTLOADER_TYPE, "bootloader_type" }, | ||
98 | { KERN_RANDOMIZE, "randomize_va_space" }, | ||
99 | |||
100 | { KERN_SPIN_RETRY, "spin_retry" }, | ||
101 | { KERN_ACPI_VIDEO_FLAGS, "acpi_video_flags" }, | ||
102 | { KERN_IA64_UNALIGNED, "ignore-unaligned-usertrap" }, | ||
103 | { KERN_COMPAT_LOG, "compat-log" }, | ||
104 | { KERN_MAX_LOCK_DEPTH, "max_lock_depth" }, | ||
105 | { KERN_NMI_WATCHDOG, "nmi_watchdog" }, | ||
106 | { KERN_PANIC_ON_NMI, "panic_on_unrecovered_nmi" }, | ||
107 | {} | ||
108 | }; | ||
109 | |||
110 | static const struct trans_ctl_table trans_vm_table[] = { | ||
111 | { VM_OVERCOMMIT_MEMORY, "overcommit_memory" }, | ||
112 | { VM_PAGE_CLUSTER, "page-cluster" }, | ||
113 | { VM_DIRTY_BACKGROUND, "dirty_background_ratio" }, | ||
114 | { VM_DIRTY_RATIO, "dirty_ratio" }, | ||
115 | { VM_DIRTY_WB_CS, "dirty_writeback_centisecs" }, | ||
116 | { VM_DIRTY_EXPIRE_CS, "dirty_expire_centisecs" }, | ||
117 | { VM_NR_PDFLUSH_THREADS, "nr_pdflush_threads" }, | ||
118 | { VM_OVERCOMMIT_RATIO, "overcommit_ratio" }, | ||
119 | /* VM_PAGEBUF unused */ | ||
120 | { VM_HUGETLB_PAGES, "nr_hugepages" }, | ||
121 | { VM_SWAPPINESS, "swappiness" }, | ||
122 | { VM_LOWMEM_RESERVE_RATIO, "lowmem_reserve_ratio" }, | ||
123 | { VM_MIN_FREE_KBYTES, "min_free_kbytes" }, | ||
124 | { VM_MAX_MAP_COUNT, "max_map_count" }, | ||
125 | { VM_LAPTOP_MODE, "laptop_mode" }, | ||
126 | { VM_BLOCK_DUMP, "block_dump" }, | ||
127 | { VM_HUGETLB_GROUP, "hugetlb_shm_group" }, | ||
128 | { VM_VFS_CACHE_PRESSURE, "vfs_cache_pressure" }, | ||
129 | { VM_LEGACY_VA_LAYOUT, "legacy_va_layout" }, | ||
130 | /* VM_SWAP_TOKEN_TIMEOUT unused */ | ||
131 | { VM_DROP_PAGECACHE, "drop_caches" }, | ||
132 | { VM_PERCPU_PAGELIST_FRACTION, "percpu_pagelist_fraction" }, | ||
133 | { VM_ZONE_RECLAIM_MODE, "zone_reclaim_mode" }, | ||
134 | { VM_MIN_UNMAPPED, "min_unmapped_ratio" }, | ||
135 | { VM_PANIC_ON_OOM, "panic_on_oom" }, | ||
136 | { VM_VDSO_ENABLED, "vdso_enabled" }, | ||
137 | { VM_MIN_SLAB, "min_slab_ratio" }, | ||
138 | |||
139 | {} | ||
140 | }; | ||
141 | |||
142 | static const struct trans_ctl_table trans_net_core_table[] = { | ||
143 | { NET_CORE_WMEM_MAX, "wmem_max" }, | ||
144 | { NET_CORE_RMEM_MAX, "rmem_max" }, | ||
145 | { NET_CORE_WMEM_DEFAULT, "wmem_default" }, | ||
146 | { NET_CORE_RMEM_DEFAULT, "rmem_default" }, | ||
147 | /* NET_CORE_DESTROY_DELAY unused */ | ||
148 | { NET_CORE_MAX_BACKLOG, "netdev_max_backlog" }, | ||
149 | /* NET_CORE_FASTROUTE unused */ | ||
150 | { NET_CORE_MSG_COST, "message_cost" }, | ||
151 | { NET_CORE_MSG_BURST, "message_burst" }, | ||
152 | { NET_CORE_OPTMEM_MAX, "optmem_max" }, | ||
153 | /* NET_CORE_HOT_LIST_LENGTH unused */ | ||
154 | /* NET_CORE_DIVERT_VERSION unused */ | ||
155 | /* NET_CORE_NO_CONG_THRESH unused */ | ||
156 | /* NET_CORE_NO_CONG unused */ | ||
157 | /* NET_CORE_LO_CONG unused */ | ||
158 | /* NET_CORE_MOD_CONG unused */ | ||
159 | { NET_CORE_DEV_WEIGHT, "dev_weight" }, | ||
160 | { NET_CORE_SOMAXCONN, "somaxconn" }, | ||
161 | { NET_CORE_BUDGET, "netdev_budget" }, | ||
162 | { NET_CORE_AEVENT_ETIME, "xfrm_aevent_etime" }, | ||
163 | { NET_CORE_AEVENT_RSEQTH, "xfrm_aevent_rseqth" }, | ||
164 | { NET_CORE_WARNINGS, "warnings" }, | ||
165 | {}, | ||
166 | }; | ||
167 | |||
168 | static const struct trans_ctl_table trans_net_unix_table[] = { | ||
169 | /* NET_UNIX_DESTROY_DELAY unused */ | ||
170 | /* NET_UNIX_DELETE_DELAY unused */ | ||
171 | { NET_UNIX_MAX_DGRAM_QLEN, "max_dgram_qlen" }, | ||
172 | {} | ||
173 | }; | ||
174 | |||
175 | static const struct trans_ctl_table trans_net_ipv4_route_table[] = { | ||
176 | { NET_IPV4_ROUTE_FLUSH, "flush" }, | ||
177 | { NET_IPV4_ROUTE_MIN_DELAY, "min_delay" }, | ||
178 | { NET_IPV4_ROUTE_MAX_DELAY, "max_delay" }, | ||
179 | { NET_IPV4_ROUTE_GC_THRESH, "gc_thresh" }, | ||
180 | { NET_IPV4_ROUTE_MAX_SIZE, "max_size" }, | ||
181 | { NET_IPV4_ROUTE_GC_MIN_INTERVAL, "gc_min_interval" }, | ||
182 | { NET_IPV4_ROUTE_GC_TIMEOUT, "gc_timeout" }, | ||
183 | { NET_IPV4_ROUTE_GC_INTERVAL, "gc_interval" }, | ||
184 | { NET_IPV4_ROUTE_REDIRECT_LOAD, "redirect_load" }, | ||
185 | { NET_IPV4_ROUTE_REDIRECT_NUMBER, "redirect_number" }, | ||
186 | { NET_IPV4_ROUTE_REDIRECT_SILENCE, "redirect_silence" }, | ||
187 | { NET_IPV4_ROUTE_ERROR_COST, "error_cost" }, | ||
188 | { NET_IPV4_ROUTE_ERROR_BURST, "error_burst" }, | ||
189 | { NET_IPV4_ROUTE_GC_ELASTICITY, "gc_elasticity" }, | ||
190 | { NET_IPV4_ROUTE_MTU_EXPIRES, "mtu_expires" }, | ||
191 | { NET_IPV4_ROUTE_MIN_PMTU, "min_pmtu" }, | ||
192 | { NET_IPV4_ROUTE_MIN_ADVMSS, "min_adv_mss" }, | ||
193 | { NET_IPV4_ROUTE_SECRET_INTERVAL, "secret_interval" }, | ||
194 | { NET_IPV4_ROUTE_GC_MIN_INTERVAL_MS, "gc_min_interval_ms" }, | ||
195 | {} | ||
196 | }; | ||
197 | |||
198 | static const struct trans_ctl_table trans_net_ipv4_conf_vars_table[] = { | ||
199 | { NET_IPV4_CONF_FORWARDING, "forwarding" }, | ||
200 | { NET_IPV4_CONF_MC_FORWARDING, "mc_forwarding" }, | ||
201 | |||
202 | { NET_IPV4_CONF_PROXY_ARP, "proxy_arp" }, | ||
203 | { NET_IPV4_CONF_ACCEPT_REDIRECTS, "accept_redirects" }, | ||
204 | { NET_IPV4_CONF_SECURE_REDIRECTS, "secure_redirects" }, | ||
205 | { NET_IPV4_CONF_SEND_REDIRECTS, "send_redirects" }, | ||
206 | { NET_IPV4_CONF_SHARED_MEDIA, "shared_media" }, | ||
207 | { NET_IPV4_CONF_RP_FILTER, "rp_filter" }, | ||
208 | { NET_IPV4_CONF_ACCEPT_SOURCE_ROUTE, "accept_source_route" }, | ||
209 | { NET_IPV4_CONF_BOOTP_RELAY, "bootp_relay" }, | ||
210 | { NET_IPV4_CONF_LOG_MARTIANS, "log_martians" }, | ||
211 | { NET_IPV4_CONF_TAG, "tag" }, | ||
212 | { NET_IPV4_CONF_ARPFILTER, "arp_filter" }, | ||
213 | { NET_IPV4_CONF_MEDIUM_ID, "medium_id" }, | ||
214 | { NET_IPV4_CONF_NOXFRM, "disable_xfrm" }, | ||
215 | { NET_IPV4_CONF_NOPOLICY, "disable_policy" }, | ||
216 | { NET_IPV4_CONF_FORCE_IGMP_VERSION, "force_igmp_version" }, | ||
217 | |||
218 | { NET_IPV4_CONF_ARP_ANNOUNCE, "arp_announce" }, | ||
219 | { NET_IPV4_CONF_ARP_IGNORE, "arp_ignore" }, | ||
220 | { NET_IPV4_CONF_PROMOTE_SECONDARIES, "promote_secondaries" }, | ||
221 | { NET_IPV4_CONF_ARP_ACCEPT, "arp_accept" }, | ||
222 | { NET_IPV4_CONF_ARP_NOTIFY, "arp_notify" }, | ||
223 | {} | ||
224 | }; | ||
225 | |||
226 | static const struct trans_ctl_table trans_net_ipv4_conf_table[] = { | ||
227 | { NET_PROTO_CONF_ALL, "all", trans_net_ipv4_conf_vars_table }, | ||
228 | { NET_PROTO_CONF_DEFAULT, "default", trans_net_ipv4_conf_vars_table }, | ||
229 | { 0, NULL, trans_net_ipv4_conf_vars_table }, | ||
230 | {} | ||
231 | }; | ||
232 | |||
233 | static const struct trans_ctl_table trans_net_neigh_vars_table[] = { | ||
234 | { NET_NEIGH_MCAST_SOLICIT, "mcast_solicit" }, | ||
235 | { NET_NEIGH_UCAST_SOLICIT, "ucast_solicit" }, | ||
236 | { NET_NEIGH_APP_SOLICIT, "app_solicit" }, | ||
237 | { NET_NEIGH_RETRANS_TIME, "retrans_time" }, | ||
238 | { NET_NEIGH_REACHABLE_TIME, "base_reachable_time" }, | ||
239 | { NET_NEIGH_DELAY_PROBE_TIME, "delay_first_probe_time" }, | ||
240 | { NET_NEIGH_GC_STALE_TIME, "gc_stale_time" }, | ||
241 | { NET_NEIGH_UNRES_QLEN, "unres_qlen" }, | ||
242 | { NET_NEIGH_PROXY_QLEN, "proxy_qlen" }, | ||
243 | { NET_NEIGH_ANYCAST_DELAY, "anycast_delay" }, | ||
244 | { NET_NEIGH_PROXY_DELAY, "proxy_delay" }, | ||
245 | { NET_NEIGH_LOCKTIME, "locktime" }, | ||
246 | { NET_NEIGH_GC_INTERVAL, "gc_interval" }, | ||
247 | { NET_NEIGH_GC_THRESH1, "gc_thresh1" }, | ||
248 | { NET_NEIGH_GC_THRESH2, "gc_thresh2" }, | ||
249 | { NET_NEIGH_GC_THRESH3, "gc_thresh3" }, | ||
250 | { NET_NEIGH_RETRANS_TIME_MS, "retrans_time_ms" }, | ||
251 | { NET_NEIGH_REACHABLE_TIME_MS, "base_reachable_time_ms" }, | ||
252 | {} | ||
253 | }; | ||
254 | |||
255 | static const struct trans_ctl_table trans_net_neigh_table[] = { | ||
256 | { NET_PROTO_CONF_DEFAULT, "default", trans_net_neigh_vars_table }, | ||
257 | { 0, NULL, trans_net_neigh_vars_table }, | ||
258 | {} | ||
259 | }; | ||
260 | |||
261 | static const struct trans_ctl_table trans_net_ipv4_netfilter_table[] = { | ||
262 | { NET_IPV4_NF_CONNTRACK_MAX, "ip_conntrack_max" }, | ||
263 | |||
264 | { NET_IPV4_NF_CONNTRACK_TCP_TIMEOUT_SYN_SENT, "ip_conntrack_tcp_timeout_syn_sent" }, | ||
265 | { NET_IPV4_NF_CONNTRACK_TCP_TIMEOUT_SYN_RECV, "ip_conntrack_tcp_timeout_syn_recv" }, | ||
266 | { NET_IPV4_NF_CONNTRACK_TCP_TIMEOUT_ESTABLISHED, "ip_conntrack_tcp_timeout_established" }, | ||
267 | { NET_IPV4_NF_CONNTRACK_TCP_TIMEOUT_FIN_WAIT, "ip_conntrack_tcp_timeout_fin_wait" }, | ||
268 | { NET_IPV4_NF_CONNTRACK_TCP_TIMEOUT_CLOSE_WAIT, "ip_conntrack_tcp_timeout_close_wait" }, | ||
269 | { NET_IPV4_NF_CONNTRACK_TCP_TIMEOUT_LAST_ACK, "ip_conntrack_tcp_timeout_last_ack" }, | ||
270 | { NET_IPV4_NF_CONNTRACK_TCP_TIMEOUT_TIME_WAIT, "ip_conntrack_tcp_timeout_time_wait" }, | ||
271 | { NET_IPV4_NF_CONNTRACK_TCP_TIMEOUT_CLOSE, "ip_conntrack_tcp_timeout_close" }, | ||
272 | |||
273 | { NET_IPV4_NF_CONNTRACK_UDP_TIMEOUT, "ip_conntrack_udp_timeout" }, | ||
274 | { NET_IPV4_NF_CONNTRACK_UDP_TIMEOUT_STREAM, "ip_conntrack_udp_timeout_stream" }, | ||
275 | { NET_IPV4_NF_CONNTRACK_ICMP_TIMEOUT, "ip_conntrack_icmp_timeout" }, | ||
276 | { NET_IPV4_NF_CONNTRACK_GENERIC_TIMEOUT, "ip_conntrack_generic_timeout" }, | ||
277 | |||
278 | { NET_IPV4_NF_CONNTRACK_BUCKETS, "ip_conntrack_buckets" }, | ||
279 | { NET_IPV4_NF_CONNTRACK_LOG_INVALID, "ip_conntrack_log_invalid" }, | ||
280 | { NET_IPV4_NF_CONNTRACK_TCP_TIMEOUT_MAX_RETRANS, "ip_conntrack_tcp_timeout_max_retrans" }, | ||
281 | { NET_IPV4_NF_CONNTRACK_TCP_LOOSE, "ip_conntrack_tcp_loose" }, | ||
282 | { NET_IPV4_NF_CONNTRACK_TCP_BE_LIBERAL, "ip_conntrack_tcp_be_liberal" }, | ||
283 | { NET_IPV4_NF_CONNTRACK_TCP_MAX_RETRANS, "ip_conntrack_tcp_max_retrans" }, | ||
284 | |||
285 | { NET_IPV4_NF_CONNTRACK_SCTP_TIMEOUT_CLOSED, "ip_conntrack_sctp_timeout_closed" }, | ||
286 | { NET_IPV4_NF_CONNTRACK_SCTP_TIMEOUT_COOKIE_WAIT, "ip_conntrack_sctp_timeout_cookie_wait" }, | ||
287 | { NET_IPV4_NF_CONNTRACK_SCTP_TIMEOUT_COOKIE_ECHOED, "ip_conntrack_sctp_timeout_cookie_echoed" }, | ||
288 | { NET_IPV4_NF_CONNTRACK_SCTP_TIMEOUT_ESTABLISHED, "ip_conntrack_sctp_timeout_established" }, | ||
289 | { NET_IPV4_NF_CONNTRACK_SCTP_TIMEOUT_SHUTDOWN_SENT, "ip_conntrack_sctp_timeout_shutdown_sent" }, | ||
290 | { NET_IPV4_NF_CONNTRACK_SCTP_TIMEOUT_SHUTDOWN_RECD, "ip_conntrack_sctp_timeout_shutdown_recd" }, | ||
291 | { NET_IPV4_NF_CONNTRACK_SCTP_TIMEOUT_SHUTDOWN_ACK_SENT, "ip_conntrack_sctp_timeout_shutdown_ack_sent" }, | ||
292 | |||
293 | { NET_IPV4_NF_CONNTRACK_COUNT, "ip_conntrack_count" }, | ||
294 | { NET_IPV4_NF_CONNTRACK_CHECKSUM, "ip_conntrack_checksum" }, | ||
295 | {} | ||
296 | }; | ||
297 | |||
298 | static const struct trans_ctl_table trans_net_ipv4_table[] = { | ||
299 | { NET_IPV4_FORWARD, "ip_forward" }, | ||
300 | { NET_IPV4_DYNADDR, "ip_dynaddr" }, | ||
301 | |||
302 | { NET_IPV4_CONF, "conf", trans_net_ipv4_conf_table }, | ||
303 | { NET_IPV4_NEIGH, "neigh", trans_net_neigh_table }, | ||
304 | { NET_IPV4_ROUTE, "route", trans_net_ipv4_route_table }, | ||
305 | /* NET_IPV4_FIB_HASH unused */ | ||
306 | { NET_IPV4_NETFILTER, "netfilter", trans_net_ipv4_netfilter_table }, | ||
307 | |||
308 | { NET_IPV4_TCP_TIMESTAMPS, "tcp_timestamps" }, | ||
309 | { NET_IPV4_TCP_WINDOW_SCALING, "tcp_window_scaling" }, | ||
310 | { NET_IPV4_TCP_SACK, "tcp_sack" }, | ||
311 | { NET_IPV4_TCP_RETRANS_COLLAPSE, "tcp_retrans_collapse" }, | ||
312 | { NET_IPV4_DEFAULT_TTL, "ip_default_ttl" }, | ||
313 | /* NET_IPV4_AUTOCONFIG unused */ | ||
314 | { NET_IPV4_NO_PMTU_DISC, "ip_no_pmtu_disc" }, | ||
315 | { NET_IPV4_TCP_SYN_RETRIES, "tcp_syn_retries" }, | ||
316 | { NET_IPV4_IPFRAG_HIGH_THRESH, "ipfrag_high_thresh" }, | ||
317 | { NET_IPV4_IPFRAG_LOW_THRESH, "ipfrag_low_thresh" }, | ||
318 | { NET_IPV4_IPFRAG_TIME, "ipfrag_time" }, | ||
319 | /* NET_IPV4_TCP_MAX_KA_PROBES unused */ | ||
320 | { NET_IPV4_TCP_KEEPALIVE_TIME, "tcp_keepalive_time" }, | ||
321 | { NET_IPV4_TCP_KEEPALIVE_PROBES, "tcp_keepalive_probes" }, | ||
322 | { NET_IPV4_TCP_RETRIES1, "tcp_retries1" }, | ||
323 | { NET_IPV4_TCP_RETRIES2, "tcp_retries2" }, | ||
324 | { NET_IPV4_TCP_FIN_TIMEOUT, "tcp_fin_timeout" }, | ||
325 | /* NET_IPV4_IP_MASQ_DEBUG unused */ | ||
326 | { NET_TCP_SYNCOOKIES, "tcp_syncookies" }, | ||
327 | { NET_TCP_STDURG, "tcp_stdurg" }, | ||
328 | { NET_TCP_RFC1337, "tcp_rfc1337" }, | ||
329 | /* NET_TCP_SYN_TAILDROP unused */ | ||
330 | { NET_TCP_MAX_SYN_BACKLOG, "tcp_max_syn_backlog" }, | ||
331 | { NET_IPV4_LOCAL_PORT_RANGE, "ip_local_port_range" }, | ||
332 | { NET_IPV4_ICMP_ECHO_IGNORE_ALL, "icmp_echo_ignore_all" }, | ||
333 | { NET_IPV4_ICMP_ECHO_IGNORE_BROADCASTS, "icmp_echo_ignore_broadcasts" }, | ||
334 | /* NET_IPV4_ICMP_SOURCEQUENCH_RATE unused */ | ||
335 | /* NET_IPV4_ICMP_DESTUNREACH_RATE unused */ | ||
336 | /* NET_IPV4_ICMP_TIMEEXCEED_RATE unused */ | ||
337 | /* NET_IPV4_ICMP_PARAMPROB_RATE unused */ | ||
338 | /* NET_IPV4_ICMP_ECHOREPLY_RATE unused */ | ||
339 | { NET_IPV4_ICMP_IGNORE_BOGUS_ERROR_RESPONSES, "icmp_ignore_bogus_error_responses" }, | ||
340 | { NET_IPV4_IGMP_MAX_MEMBERSHIPS, "igmp_max_memberships" }, | ||
341 | { NET_TCP_TW_RECYCLE, "tcp_tw_recycle" }, | ||
342 | /* NET_IPV4_ALWAYS_DEFRAG unused */ | ||
343 | { NET_IPV4_TCP_KEEPALIVE_INTVL, "tcp_keepalive_intvl" }, | ||
344 | { NET_IPV4_INET_PEER_THRESHOLD, "inet_peer_threshold" }, | ||
345 | { NET_IPV4_INET_PEER_MINTTL, "inet_peer_minttl" }, | ||
346 | { NET_IPV4_INET_PEER_MAXTTL, "inet_peer_maxttl" }, | ||
347 | { NET_IPV4_INET_PEER_GC_MINTIME, "inet_peer_gc_mintime" }, | ||
348 | { NET_IPV4_INET_PEER_GC_MAXTIME, "inet_peer_gc_maxtime" }, | ||
349 | { NET_TCP_ORPHAN_RETRIES, "tcp_orphan_retries" }, | ||
350 | { NET_TCP_ABORT_ON_OVERFLOW, "tcp_abort_on_overflow" }, | ||
351 | { NET_TCP_SYNACK_RETRIES, "tcp_synack_retries" }, | ||
352 | { NET_TCP_MAX_ORPHANS, "tcp_max_orphans" }, | ||
353 | { NET_TCP_MAX_TW_BUCKETS, "tcp_max_tw_buckets" }, | ||
354 | { NET_TCP_FACK, "tcp_fack" }, | ||
355 | { NET_TCP_REORDERING, "tcp_reordering" }, | ||
356 | { NET_TCP_ECN, "tcp_ecn" }, | ||
357 | { NET_TCP_DSACK, "tcp_dsack" }, | ||
358 | { NET_TCP_MEM, "tcp_mem" }, | ||
359 | { NET_TCP_WMEM, "tcp_wmem" }, | ||
360 | { NET_TCP_RMEM, "tcp_rmem" }, | ||
361 | { NET_TCP_APP_WIN, "tcp_app_win" }, | ||
362 | { NET_TCP_ADV_WIN_SCALE, "tcp_adv_win_scale" }, | ||
363 | { NET_IPV4_NONLOCAL_BIND, "ip_nonlocal_bind" }, | ||
364 | { NET_IPV4_ICMP_RATELIMIT, "icmp_ratelimit" }, | ||
365 | { NET_IPV4_ICMP_RATEMASK, "icmp_ratemask" }, | ||
366 | { NET_TCP_TW_REUSE, "tcp_tw_reuse" }, | ||
367 | { NET_TCP_FRTO, "tcp_frto" }, | ||
368 | { NET_TCP_LOW_LATENCY, "tcp_low_latency" }, | ||
369 | { NET_IPV4_IPFRAG_SECRET_INTERVAL, "ipfrag_secret_interval" }, | ||
370 | { NET_IPV4_IGMP_MAX_MSF, "igmp_max_msf" }, | ||
371 | { NET_TCP_NO_METRICS_SAVE, "tcp_no_metrics_save" }, | ||
372 | /* NET_TCP_DEFAULT_WIN_SCALE unused */ | ||
373 | { NET_TCP_MODERATE_RCVBUF, "tcp_moderate_rcvbuf" }, | ||
374 | { NET_TCP_TSO_WIN_DIVISOR, "tcp_tso_win_divisor" }, | ||
375 | /* NET_TCP_BIC_BETA unused */ | ||
376 | { NET_IPV4_ICMP_ERRORS_USE_INBOUND_IFADDR, "icmp_errors_use_inbound_ifaddr" }, | ||
377 | { NET_TCP_CONG_CONTROL, "tcp_congestion_control" }, | ||
378 | { NET_TCP_ABC, "tcp_abc" }, | ||
379 | { NET_IPV4_IPFRAG_MAX_DIST, "ipfrag_max_dist" }, | ||
380 | { NET_TCP_MTU_PROBING, "tcp_mtu_probing" }, | ||
381 | { NET_TCP_BASE_MSS, "tcp_base_mss" }, | ||
382 | { NET_IPV4_TCP_WORKAROUND_SIGNED_WINDOWS, "tcp_workaround_signed_windows" }, | ||
383 | { NET_TCP_DMA_COPYBREAK, "tcp_dma_copybreak" }, | ||
384 | { NET_TCP_SLOW_START_AFTER_IDLE, "tcp_slow_start_after_idle" }, | ||
385 | { NET_CIPSOV4_CACHE_ENABLE, "cipso_cache_enable" }, | ||
386 | { NET_CIPSOV4_CACHE_BUCKET_SIZE, "cipso_cache_bucket_size" }, | ||
387 | { NET_CIPSOV4_RBM_OPTFMT, "cipso_rbm_optfmt" }, | ||
388 | { NET_CIPSOV4_RBM_STRICTVALID, "cipso_rbm_strictvalid" }, | ||
389 | { NET_TCP_AVAIL_CONG_CONTROL, "tcp_available_congestion_control" }, | ||
390 | { NET_TCP_ALLOWED_CONG_CONTROL, "tcp_allowed_congestion_control" }, | ||
391 | { NET_TCP_MAX_SSTHRESH, "tcp_max_ssthresh" }, | ||
392 | { NET_TCP_FRTO_RESPONSE, "tcp_frto_response" }, | ||
393 | { 2088 /* NET_IPQ_QMAX */, "ip_queue_maxlen" }, | ||
394 | {} | ||
395 | }; | ||
396 | |||
397 | static const struct trans_ctl_table trans_net_ipx_table[] = { | ||
398 | { NET_IPX_PPROP_BROADCASTING, "ipx_pprop_broadcasting" }, | ||
399 | /* NET_IPX_FORWARDING unused */ | ||
400 | {} | ||
401 | }; | ||
402 | |||
403 | static const struct trans_ctl_table trans_net_atalk_table[] = { | ||
404 | { NET_ATALK_AARP_EXPIRY_TIME, "aarp-expiry-time" }, | ||
405 | { NET_ATALK_AARP_TICK_TIME, "aarp-tick-time" }, | ||
406 | { NET_ATALK_AARP_RETRANSMIT_LIMIT, "aarp-retransmit-limit" }, | ||
407 | { NET_ATALK_AARP_RESOLVE_TIME, "aarp-resolve-time" }, | ||
408 | {}, | ||
409 | }; | ||
410 | |||
411 | static const struct trans_ctl_table trans_net_netrom_table[] = { | ||
412 | { NET_NETROM_DEFAULT_PATH_QUALITY, "default_path_quality" }, | ||
413 | { NET_NETROM_OBSOLESCENCE_COUNT_INITIALISER, "obsolescence_count_initialiser" }, | ||
414 | { NET_NETROM_NETWORK_TTL_INITIALISER, "network_ttl_initialiser" }, | ||
415 | { NET_NETROM_TRANSPORT_TIMEOUT, "transport_timeout" }, | ||
416 | { NET_NETROM_TRANSPORT_MAXIMUM_TRIES, "transport_maximum_tries" }, | ||
417 | { NET_NETROM_TRANSPORT_ACKNOWLEDGE_DELAY, "transport_acknowledge_delay" }, | ||
418 | { NET_NETROM_TRANSPORT_BUSY_DELAY, "transport_busy_delay" }, | ||
419 | { NET_NETROM_TRANSPORT_REQUESTED_WINDOW_SIZE, "transport_requested_window_size" }, | ||
420 | { NET_NETROM_TRANSPORT_NO_ACTIVITY_TIMEOUT, "transport_no_activity_timeout" }, | ||
421 | { NET_NETROM_ROUTING_CONTROL, "routing_control" }, | ||
422 | { NET_NETROM_LINK_FAILS_COUNT, "link_fails_count" }, | ||
423 | { NET_NETROM_RESET, "reset" }, | ||
424 | {} | ||
425 | }; | ||
426 | |||
427 | static const struct trans_ctl_table trans_net_ax25_param_table[] = { | ||
428 | { NET_AX25_IP_DEFAULT_MODE, "ip_default_mode" }, | ||
429 | { NET_AX25_DEFAULT_MODE, "ax25_default_mode" }, | ||
430 | { NET_AX25_BACKOFF_TYPE, "backoff_type" }, | ||
431 | { NET_AX25_CONNECT_MODE, "connect_mode" }, | ||
432 | { NET_AX25_STANDARD_WINDOW, "standard_window_size" }, | ||
433 | { NET_AX25_EXTENDED_WINDOW, "extended_window_size" }, | ||
434 | { NET_AX25_T1_TIMEOUT, "t1_timeout" }, | ||
435 | { NET_AX25_T2_TIMEOUT, "t2_timeout" }, | ||
436 | { NET_AX25_T3_TIMEOUT, "t3_timeout" }, | ||
437 | { NET_AX25_IDLE_TIMEOUT, "idle_timeout" }, | ||
438 | { NET_AX25_N2, "maximum_retry_count" }, | ||
439 | { NET_AX25_PACLEN, "maximum_packet_length" }, | ||
440 | { NET_AX25_PROTOCOL, "protocol" }, | ||
441 | { NET_AX25_DAMA_SLAVE_TIMEOUT, "dama_slave_timeout" }, | ||
442 | {} | ||
443 | }; | ||
444 | |||
445 | static const struct trans_ctl_table trans_net_ax25_table[] = { | ||
446 | { 0, NULL, trans_net_ax25_param_table }, | ||
447 | {} | ||
448 | }; | ||
449 | |||
450 | static const struct trans_ctl_table trans_net_bridge_table[] = { | ||
451 | { NET_BRIDGE_NF_CALL_ARPTABLES, "bridge-nf-call-arptables" }, | ||
452 | { NET_BRIDGE_NF_CALL_IPTABLES, "bridge-nf-call-iptables" }, | ||
453 | { NET_BRIDGE_NF_CALL_IP6TABLES, "bridge-nf-call-ip6tables" }, | ||
454 | { NET_BRIDGE_NF_FILTER_VLAN_TAGGED, "bridge-nf-filter-vlan-tagged" }, | ||
455 | { NET_BRIDGE_NF_FILTER_PPPOE_TAGGED, "bridge-nf-filter-pppoe-tagged" }, | ||
456 | {} | ||
457 | }; | ||
458 | |||
459 | static const struct trans_ctl_table trans_net_rose_table[] = { | ||
460 | { NET_ROSE_RESTART_REQUEST_TIMEOUT, "restart_request_timeout" }, | ||
461 | { NET_ROSE_CALL_REQUEST_TIMEOUT, "call_request_timeout" }, | ||
462 | { NET_ROSE_RESET_REQUEST_TIMEOUT, "reset_request_timeout" }, | ||
463 | { NET_ROSE_CLEAR_REQUEST_TIMEOUT, "clear_request_timeout" }, | ||
464 | { NET_ROSE_ACK_HOLD_BACK_TIMEOUT, "acknowledge_hold_back_timeout" }, | ||
465 | { NET_ROSE_ROUTING_CONTROL, "routing_control" }, | ||
466 | { NET_ROSE_LINK_FAIL_TIMEOUT, "link_fail_timeout" }, | ||
467 | { NET_ROSE_MAX_VCS, "maximum_virtual_circuits" }, | ||
468 | { NET_ROSE_WINDOW_SIZE, "window_size" }, | ||
469 | { NET_ROSE_NO_ACTIVITY_TIMEOUT, "no_activity_timeout" }, | ||
470 | {} | ||
471 | }; | ||
472 | |||
473 | static const struct trans_ctl_table trans_net_ipv6_conf_var_table[] = { | ||
474 | { NET_IPV6_FORWARDING, "forwarding" }, | ||
475 | { NET_IPV6_HOP_LIMIT, "hop_limit" }, | ||
476 | { NET_IPV6_MTU, "mtu" }, | ||
477 | { NET_IPV6_ACCEPT_RA, "accept_ra" }, | ||
478 | { NET_IPV6_ACCEPT_REDIRECTS, "accept_redirects" }, | ||
479 | { NET_IPV6_AUTOCONF, "autoconf" }, | ||
480 | { NET_IPV6_DAD_TRANSMITS, "dad_transmits" }, | ||
481 | { NET_IPV6_RTR_SOLICITS, "router_solicitations" }, | ||
482 | { NET_IPV6_RTR_SOLICIT_INTERVAL, "router_solicitation_interval" }, | ||
483 | { NET_IPV6_RTR_SOLICIT_DELAY, "router_solicitation_delay" }, | ||
484 | { NET_IPV6_USE_TEMPADDR, "use_tempaddr" }, | ||
485 | { NET_IPV6_TEMP_VALID_LFT, "temp_valid_lft" }, | ||
486 | { NET_IPV6_TEMP_PREFERED_LFT, "temp_prefered_lft" }, | ||
487 | { NET_IPV6_REGEN_MAX_RETRY, "regen_max_retry" }, | ||
488 | { NET_IPV6_MAX_DESYNC_FACTOR, "max_desync_factor" }, | ||
489 | { NET_IPV6_MAX_ADDRESSES, "max_addresses" }, | ||
490 | { NET_IPV6_FORCE_MLD_VERSION, "force_mld_version" }, | ||
491 | { NET_IPV6_ACCEPT_RA_DEFRTR, "accept_ra_defrtr" }, | ||
492 | { NET_IPV6_ACCEPT_RA_PINFO, "accept_ra_pinfo" }, | ||
493 | { NET_IPV6_ACCEPT_RA_RTR_PREF, "accept_ra_rtr_pref" }, | ||
494 | { NET_IPV6_RTR_PROBE_INTERVAL, "router_probe_interval" }, | ||
495 | { NET_IPV6_ACCEPT_RA_RT_INFO_MAX_PLEN, "accept_ra_rt_info_max_plen" }, | ||
496 | { NET_IPV6_PROXY_NDP, "proxy_ndp" }, | ||
497 | { NET_IPV6_ACCEPT_SOURCE_ROUTE, "accept_source_route" }, | ||
498 | {} | ||
499 | }; | ||
500 | |||
501 | static const struct trans_ctl_table trans_net_ipv6_conf_table[] = { | ||
502 | { NET_PROTO_CONF_ALL, "all", trans_net_ipv6_conf_var_table }, | ||
503 | { NET_PROTO_CONF_DEFAULT, "default", trans_net_ipv6_conf_var_table }, | ||
504 | { 0, NULL, trans_net_ipv6_conf_var_table }, | ||
505 | {} | ||
506 | }; | ||
507 | |||
508 | static const struct trans_ctl_table trans_net_ipv6_route_table[] = { | ||
509 | { NET_IPV6_ROUTE_FLUSH, "flush" }, | ||
510 | { NET_IPV6_ROUTE_GC_THRESH, "gc_thresh" }, | ||
511 | { NET_IPV6_ROUTE_MAX_SIZE, "max_size" }, | ||
512 | { NET_IPV6_ROUTE_GC_MIN_INTERVAL, "gc_min_interval" }, | ||
513 | { NET_IPV6_ROUTE_GC_TIMEOUT, "gc_timeout" }, | ||
514 | { NET_IPV6_ROUTE_GC_INTERVAL, "gc_interval" }, | ||
515 | { NET_IPV6_ROUTE_GC_ELASTICITY, "gc_elasticity" }, | ||
516 | { NET_IPV6_ROUTE_MTU_EXPIRES, "mtu_expires" }, | ||
517 | { NET_IPV6_ROUTE_MIN_ADVMSS, "min_adv_mss" }, | ||
518 | { NET_IPV6_ROUTE_GC_MIN_INTERVAL_MS, "gc_min_interval_ms" }, | ||
519 | {} | ||
520 | }; | ||
521 | |||
522 | static const struct trans_ctl_table trans_net_ipv6_icmp_table[] = { | ||
523 | { NET_IPV6_ICMP_RATELIMIT, "ratelimit" }, | ||
524 | {} | ||
525 | }; | ||
526 | |||
527 | static const struct trans_ctl_table trans_net_ipv6_table[] = { | ||
528 | { NET_IPV6_CONF, "conf", trans_net_ipv6_conf_table }, | ||
529 | { NET_IPV6_NEIGH, "neigh", trans_net_neigh_table }, | ||
530 | { NET_IPV6_ROUTE, "route", trans_net_ipv6_route_table }, | ||
531 | { NET_IPV6_ICMP, "icmp", trans_net_ipv6_icmp_table }, | ||
532 | { NET_IPV6_BINDV6ONLY, "bindv6only" }, | ||
533 | { NET_IPV6_IP6FRAG_HIGH_THRESH, "ip6frag_high_thresh" }, | ||
534 | { NET_IPV6_IP6FRAG_LOW_THRESH, "ip6frag_low_thresh" }, | ||
535 | { NET_IPV6_IP6FRAG_TIME, "ip6frag_time" }, | ||
536 | { NET_IPV6_IP6FRAG_SECRET_INTERVAL, "ip6frag_secret_interval" }, | ||
537 | { NET_IPV6_MLD_MAX_MSF, "mld_max_msf" }, | ||
538 | { 2088 /* IPQ_QMAX */, "ip6_queue_maxlen" }, | ||
539 | {} | ||
540 | }; | ||
541 | |||
542 | static const struct trans_ctl_table trans_net_x25_table[] = { | ||
543 | { NET_X25_RESTART_REQUEST_TIMEOUT, "restart_request_timeout" }, | ||
544 | { NET_X25_CALL_REQUEST_TIMEOUT, "call_request_timeout" }, | ||
545 | { NET_X25_RESET_REQUEST_TIMEOUT, "reset_request_timeout" }, | ||
546 | { NET_X25_CLEAR_REQUEST_TIMEOUT, "clear_request_timeout" }, | ||
547 | { NET_X25_ACK_HOLD_BACK_TIMEOUT, "acknowledgement_hold_back_timeout" }, | ||
548 | { NET_X25_FORWARD, "x25_forward" }, | ||
549 | {} | ||
550 | }; | ||
551 | |||
552 | static const struct trans_ctl_table trans_net_tr_table[] = { | ||
553 | { NET_TR_RIF_TIMEOUT, "rif_timeout" }, | ||
554 | {} | ||
555 | }; | ||
556 | |||
557 | |||
558 | static const struct trans_ctl_table trans_net_decnet_conf_vars[] = { | ||
559 | { NET_DECNET_CONF_DEV_FORWARDING, "forwarding" }, | ||
560 | { NET_DECNET_CONF_DEV_PRIORITY, "priority" }, | ||
561 | { NET_DECNET_CONF_DEV_T2, "t2" }, | ||
562 | { NET_DECNET_CONF_DEV_T3, "t3" }, | ||
563 | {} | ||
564 | }; | ||
565 | |||
566 | static const struct trans_ctl_table trans_net_decnet_conf[] = { | ||
567 | { 0, NULL, trans_net_decnet_conf_vars }, | ||
568 | {} | ||
569 | }; | ||
570 | |||
571 | static const struct trans_ctl_table trans_net_decnet_table[] = { | ||
572 | { NET_DECNET_CONF, "conf", trans_net_decnet_conf }, | ||
573 | { NET_DECNET_NODE_ADDRESS, "node_address" }, | ||
574 | { NET_DECNET_NODE_NAME, "node_name" }, | ||
575 | { NET_DECNET_DEFAULT_DEVICE, "default_device" }, | ||
576 | { NET_DECNET_TIME_WAIT, "time_wait" }, | ||
577 | { NET_DECNET_DN_COUNT, "dn_count" }, | ||
578 | { NET_DECNET_DI_COUNT, "di_count" }, | ||
579 | { NET_DECNET_DR_COUNT, "dr_count" }, | ||
580 | { NET_DECNET_DST_GC_INTERVAL, "dst_gc_interval" }, | ||
581 | { NET_DECNET_NO_FC_MAX_CWND, "no_fc_max_cwnd" }, | ||
582 | { NET_DECNET_MEM, "decnet_mem" }, | ||
583 | { NET_DECNET_RMEM, "decnet_rmem" }, | ||
584 | { NET_DECNET_WMEM, "decnet_wmem" }, | ||
585 | { NET_DECNET_DEBUG_LEVEL, "debug" }, | ||
586 | {} | ||
587 | }; | ||
588 | |||
589 | static const struct trans_ctl_table trans_net_sctp_table[] = { | ||
590 | { NET_SCTP_RTO_INITIAL, "rto_initial" }, | ||
591 | { NET_SCTP_RTO_MIN, "rto_min" }, | ||
592 | { NET_SCTP_RTO_MAX, "rto_max" }, | ||
593 | { NET_SCTP_RTO_ALPHA, "rto_alpha_exp_divisor" }, | ||
594 | { NET_SCTP_RTO_BETA, "rto_beta_exp_divisor" }, | ||
595 | { NET_SCTP_VALID_COOKIE_LIFE, "valid_cookie_life" }, | ||
596 | { NET_SCTP_ASSOCIATION_MAX_RETRANS, "association_max_retrans" }, | ||
597 | { NET_SCTP_PATH_MAX_RETRANS, "path_max_retrans" }, | ||
598 | { NET_SCTP_MAX_INIT_RETRANSMITS, "max_init_retransmits" }, | ||
599 | { NET_SCTP_HB_INTERVAL, "hb_interval" }, | ||
600 | { NET_SCTP_PRESERVE_ENABLE, "cookie_preserve_enable" }, | ||
601 | { NET_SCTP_MAX_BURST, "max_burst" }, | ||
602 | { NET_SCTP_ADDIP_ENABLE, "addip_enable" }, | ||
603 | { NET_SCTP_PRSCTP_ENABLE, "prsctp_enable" }, | ||
604 | { NET_SCTP_SNDBUF_POLICY, "sndbuf_policy" }, | ||
605 | { NET_SCTP_SACK_TIMEOUT, "sack_timeout" }, | ||
606 | { NET_SCTP_RCVBUF_POLICY, "rcvbuf_policy" }, | ||
607 | {} | ||
608 | }; | ||
609 | |||
610 | static const struct trans_ctl_table trans_net_llc_llc2_timeout_table[] = { | ||
611 | { NET_LLC2_ACK_TIMEOUT, "ack" }, | ||
612 | { NET_LLC2_P_TIMEOUT, "p" }, | ||
613 | { NET_LLC2_REJ_TIMEOUT, "rej" }, | ||
614 | { NET_LLC2_BUSY_TIMEOUT, "busy" }, | ||
615 | {} | ||
616 | }; | ||
617 | |||
618 | static const struct trans_ctl_table trans_net_llc_station_table[] = { | ||
619 | { NET_LLC_STATION_ACK_TIMEOUT, "ack_timeout" }, | ||
620 | {} | ||
621 | }; | ||
622 | |||
623 | static const struct trans_ctl_table trans_net_llc_llc2_table[] = { | ||
624 | { NET_LLC2, "timeout", trans_net_llc_llc2_timeout_table }, | ||
625 | {} | ||
626 | }; | ||
627 | |||
628 | static const struct trans_ctl_table trans_net_llc_table[] = { | ||
629 | { NET_LLC2, "llc2", trans_net_llc_llc2_table }, | ||
630 | { NET_LLC_STATION, "station", trans_net_llc_station_table }, | ||
631 | {} | ||
632 | }; | ||
633 | |||
634 | static const struct trans_ctl_table trans_net_netfilter_table[] = { | ||
635 | { NET_NF_CONNTRACK_MAX, "nf_conntrack_max" }, | ||
636 | { NET_NF_CONNTRACK_TCP_TIMEOUT_SYN_SENT, "nf_conntrack_tcp_timeout_syn_sent" }, | ||
637 | { NET_NF_CONNTRACK_TCP_TIMEOUT_SYN_RECV, "nf_conntrack_tcp_timeout_syn_recv" }, | ||
638 | { NET_NF_CONNTRACK_TCP_TIMEOUT_ESTABLISHED, "nf_conntrack_tcp_timeout_established" }, | ||
639 | { NET_NF_CONNTRACK_TCP_TIMEOUT_FIN_WAIT, "nf_conntrack_tcp_timeout_fin_wait" }, | ||
640 | { NET_NF_CONNTRACK_TCP_TIMEOUT_CLOSE_WAIT, "nf_conntrack_tcp_timeout_close_wait" }, | ||
641 | { NET_NF_CONNTRACK_TCP_TIMEOUT_LAST_ACK, "nf_conntrack_tcp_timeout_last_ack" }, | ||
642 | { NET_NF_CONNTRACK_TCP_TIMEOUT_TIME_WAIT, "nf_conntrack_tcp_timeout_time_wait" }, | ||
643 | { NET_NF_CONNTRACK_TCP_TIMEOUT_CLOSE, "nf_conntrack_tcp_timeout_close" }, | ||
644 | { NET_NF_CONNTRACK_UDP_TIMEOUT, "nf_conntrack_udp_timeout" }, | ||
645 | { NET_NF_CONNTRACK_UDP_TIMEOUT_STREAM, "nf_conntrack_udp_timeout_stream" }, | ||
646 | { NET_NF_CONNTRACK_ICMP_TIMEOUT, "nf_conntrack_icmp_timeout" }, | ||
647 | { NET_NF_CONNTRACK_GENERIC_TIMEOUT, "nf_conntrack_generic_timeout" }, | ||
648 | { NET_NF_CONNTRACK_BUCKETS, "nf_conntrack_buckets" }, | ||
649 | { NET_NF_CONNTRACK_LOG_INVALID, "nf_conntrack_log_invalid" }, | ||
650 | { NET_NF_CONNTRACK_TCP_TIMEOUT_MAX_RETRANS, "nf_conntrack_tcp_timeout_max_retrans" }, | ||
651 | { NET_NF_CONNTRACK_TCP_LOOSE, "nf_conntrack_tcp_loose" }, | ||
652 | { NET_NF_CONNTRACK_TCP_BE_LIBERAL, "nf_conntrack_tcp_be_liberal" }, | ||
653 | { NET_NF_CONNTRACK_TCP_MAX_RETRANS, "nf_conntrack_tcp_max_retrans" }, | ||
654 | { NET_NF_CONNTRACK_SCTP_TIMEOUT_CLOSED, "nf_conntrack_sctp_timeout_closed" }, | ||
655 | { NET_NF_CONNTRACK_SCTP_TIMEOUT_COOKIE_WAIT, "nf_conntrack_sctp_timeout_cookie_wait" }, | ||
656 | { NET_NF_CONNTRACK_SCTP_TIMEOUT_COOKIE_ECHOED, "nf_conntrack_sctp_timeout_cookie_echoed" }, | ||
657 | { NET_NF_CONNTRACK_SCTP_TIMEOUT_ESTABLISHED, "nf_conntrack_sctp_timeout_established" }, | ||
658 | { NET_NF_CONNTRACK_SCTP_TIMEOUT_SHUTDOWN_SENT, "nf_conntrack_sctp_timeout_shutdown_sent" }, | ||
659 | { NET_NF_CONNTRACK_SCTP_TIMEOUT_SHUTDOWN_RECD, "nf_conntrack_sctp_timeout_shutdown_recd" }, | ||
660 | { NET_NF_CONNTRACK_SCTP_TIMEOUT_SHUTDOWN_ACK_SENT, "nf_conntrack_sctp_timeout_shutdown_ack_sent" }, | ||
661 | { NET_NF_CONNTRACK_COUNT, "nf_conntrack_count" }, | ||
662 | { NET_NF_CONNTRACK_ICMPV6_TIMEOUT, "nf_conntrack_icmpv6_timeout" }, | ||
663 | { NET_NF_CONNTRACK_FRAG6_TIMEOUT, "nf_conntrack_frag6_timeout" }, | ||
664 | { NET_NF_CONNTRACK_FRAG6_LOW_THRESH, "nf_conntrack_frag6_low_thresh" }, | ||
665 | { NET_NF_CONNTRACK_FRAG6_HIGH_THRESH, "nf_conntrack_frag6_high_thresh" }, | ||
666 | { NET_NF_CONNTRACK_CHECKSUM, "nf_conntrack_checksum" }, | ||
667 | |||
668 | {} | ||
669 | }; | ||
670 | |||
671 | static const struct trans_ctl_table trans_net_dccp_table[] = { | ||
672 | { NET_DCCP_DEFAULT, "default" }, | ||
673 | {} | ||
674 | }; | ||
675 | |||
676 | static const struct trans_ctl_table trans_net_irda_table[] = { | ||
677 | { NET_IRDA_DISCOVERY, "discovery" }, | ||
678 | { NET_IRDA_DEVNAME, "devname" }, | ||
679 | { NET_IRDA_DEBUG, "debug" }, | ||
680 | { NET_IRDA_FAST_POLL, "fast_poll_increase" }, | ||
681 | { NET_IRDA_DISCOVERY_SLOTS, "discovery_slots" }, | ||
682 | { NET_IRDA_DISCOVERY_TIMEOUT, "discovery_timeout" }, | ||
683 | { NET_IRDA_SLOT_TIMEOUT, "slot_timeout" }, | ||
684 | { NET_IRDA_MAX_BAUD_RATE, "max_baud_rate" }, | ||
685 | { NET_IRDA_MIN_TX_TURN_TIME, "min_tx_turn_time" }, | ||
686 | { NET_IRDA_MAX_TX_DATA_SIZE, "max_tx_data_size" }, | ||
687 | { NET_IRDA_MAX_TX_WINDOW, "max_tx_window" }, | ||
688 | { NET_IRDA_MAX_NOREPLY_TIME, "max_noreply_time" }, | ||
689 | { NET_IRDA_WARN_NOREPLY_TIME, "warn_noreply_time" }, | ||
690 | { NET_IRDA_LAP_KEEPALIVE_TIME, "lap_keepalive_time" }, | ||
691 | {} | ||
692 | }; | ||
693 | |||
694 | static const struct trans_ctl_table trans_net_table[] = { | ||
695 | { NET_CORE, "core", trans_net_core_table }, | ||
696 | /* NET_ETHER not used */ | ||
697 | /* NET_802 not used */ | ||
698 | { NET_UNIX, "unix", trans_net_unix_table }, | ||
699 | { NET_IPV4, "ipv4", trans_net_ipv4_table }, | ||
700 | { NET_IPX, "ipx", trans_net_ipx_table }, | ||
701 | { NET_ATALK, "appletalk", trans_net_atalk_table }, | ||
702 | { NET_NETROM, "netrom", trans_net_netrom_table }, | ||
703 | { NET_AX25, "ax25", trans_net_ax25_table }, | ||
704 | { NET_BRIDGE, "bridge", trans_net_bridge_table }, | ||
705 | { NET_ROSE, "rose", trans_net_rose_table }, | ||
706 | { NET_IPV6, "ipv6", trans_net_ipv6_table }, | ||
707 | { NET_X25, "x25", trans_net_x25_table }, | ||
708 | { NET_TR, "token-ring", trans_net_tr_table }, | ||
709 | { NET_DECNET, "decnet", trans_net_decnet_table }, | ||
710 | /* NET_ECONET not used */ | ||
711 | { NET_SCTP, "sctp", trans_net_sctp_table }, | ||
712 | { NET_LLC, "llc", trans_net_llc_table }, | ||
713 | { NET_NETFILTER, "netfilter", trans_net_netfilter_table }, | ||
714 | { NET_DCCP, "dccp", trans_net_dccp_table }, | ||
715 | { NET_IRDA, "irda", trans_net_irda_table }, | ||
716 | { 2089, "nf_conntrack_max" }, | ||
717 | {} | ||
718 | }; | ||
719 | |||
720 | static const struct trans_ctl_table trans_fs_quota_table[] = { | ||
721 | { FS_DQ_LOOKUPS, "lookups" }, | ||
722 | { FS_DQ_DROPS, "drops" }, | ||
723 | { FS_DQ_READS, "reads" }, | ||
724 | { FS_DQ_WRITES, "writes" }, | ||
725 | { FS_DQ_CACHE_HITS, "cache_hits" }, | ||
726 | { FS_DQ_ALLOCATED, "allocated_dquots" }, | ||
727 | { FS_DQ_FREE, "free_dquots" }, | ||
728 | { FS_DQ_SYNCS, "syncs" }, | ||
729 | { FS_DQ_WARNINGS, "warnings" }, | ||
730 | {} | ||
731 | }; | ||
732 | |||
733 | static const struct trans_ctl_table trans_fs_xfs_table[] = { | ||
734 | { XFS_SGID_INHERIT, "irix_sgid_inherit" }, | ||
735 | { XFS_SYMLINK_MODE, "irix_symlink_mode" }, | ||
736 | { XFS_PANIC_MASK, "panic_mask" }, | ||
737 | |||
738 | { XFS_ERRLEVEL, "error_level" }, | ||
739 | { XFS_SYNCD_TIMER, "xfssyncd_centisecs" }, | ||
740 | { XFS_INHERIT_SYNC, "inherit_sync" }, | ||
741 | { XFS_INHERIT_NODUMP, "inherit_nodump" }, | ||
742 | { XFS_INHERIT_NOATIME, "inherit_noatime" }, | ||
743 | { XFS_BUF_TIMER, "xfsbufd_centisecs" }, | ||
744 | { XFS_BUF_AGE, "age_buffer_centisecs" }, | ||
745 | { XFS_INHERIT_NOSYM, "inherit_nosymlinks" }, | ||
746 | { XFS_ROTORSTEP, "rotorstep" }, | ||
747 | { XFS_INHERIT_NODFRG, "inherit_nodefrag" }, | ||
748 | { XFS_FILESTREAM_TIMER, "filestream_centisecs" }, | ||
749 | { XFS_STATS_CLEAR, "stats_clear" }, | ||
750 | {} | ||
751 | }; | ||
752 | |||
753 | static const struct trans_ctl_table trans_fs_ocfs2_nm_table[] = { | ||
754 | { 1, "hb_ctl_path" }, | ||
755 | {} | ||
756 | }; | ||
757 | |||
758 | static const struct trans_ctl_table trans_fs_ocfs2_table[] = { | ||
759 | { 1, "nm", trans_fs_ocfs2_nm_table }, | ||
760 | {} | ||
761 | }; | ||
762 | |||
763 | static const struct trans_ctl_table trans_inotify_table[] = { | ||
764 | { INOTIFY_MAX_USER_INSTANCES, "max_user_instances" }, | ||
765 | { INOTIFY_MAX_USER_WATCHES, "max_user_watches" }, | ||
766 | { INOTIFY_MAX_QUEUED_EVENTS, "max_queued_events" }, | ||
767 | {} | ||
768 | }; | ||
769 | |||
770 | static const struct trans_ctl_table trans_fs_table[] = { | ||
771 | { FS_NRINODE, "inode-nr" }, | ||
772 | { FS_STATINODE, "inode-state" }, | ||
773 | /* FS_MAXINODE unused */ | ||
774 | /* FS_NRDQUOT unused */ | ||
775 | /* FS_MAXDQUOT unused */ | ||
776 | { FS_NRFILE, "file-nr" }, | ||
777 | { FS_MAXFILE, "file-max" }, | ||
778 | { FS_DENTRY, "dentry-state" }, | ||
779 | /* FS_NRSUPER unused */ | ||
780 | /* FS_MAXUPSER unused */ | ||
781 | { FS_OVERFLOWUID, "overflowuid" }, | ||
782 | { FS_OVERFLOWGID, "overflowgid" }, | ||
783 | { FS_LEASES, "leases-enable" }, | ||
784 | { FS_DIR_NOTIFY, "dir-notify-enable" }, | ||
785 | { FS_LEASE_TIME, "lease-break-time" }, | ||
786 | { FS_DQSTATS, "quota", trans_fs_quota_table }, | ||
787 | { FS_XFS, "xfs", trans_fs_xfs_table }, | ||
788 | { FS_AIO_NR, "aio-nr" }, | ||
789 | { FS_AIO_MAX_NR, "aio-max-nr" }, | ||
790 | { FS_INOTIFY, "inotify", trans_inotify_table }, | ||
791 | { FS_OCFS2, "ocfs2", trans_fs_ocfs2_table }, | ||
792 | { KERN_SETUID_DUMPABLE, "suid_dumpable" }, | ||
793 | {} | ||
794 | }; | ||
795 | |||
796 | static const struct trans_ctl_table trans_debug_table[] = { | ||
797 | {} | ||
798 | }; | ||
799 | |||
800 | static const struct trans_ctl_table trans_cdrom_table[] = { | ||
801 | { DEV_CDROM_INFO, "info" }, | ||
802 | { DEV_CDROM_AUTOCLOSE, "autoclose" }, | ||
803 | { DEV_CDROM_AUTOEJECT, "autoeject" }, | ||
804 | { DEV_CDROM_DEBUG, "debug" }, | ||
805 | { DEV_CDROM_LOCK, "lock" }, | ||
806 | { DEV_CDROM_CHECK_MEDIA, "check_media" }, | ||
807 | {} | ||
808 | }; | ||
809 | |||
810 | static const struct trans_ctl_table trans_ipmi_table[] = { | ||
811 | { DEV_IPMI_POWEROFF_POWERCYCLE, "poweroff_powercycle" }, | ||
812 | {} | ||
813 | }; | ||
814 | |||
815 | static const struct trans_ctl_table trans_mac_hid_files[] = { | ||
816 | /* DEV_MAC_HID_KEYBOARD_SENDS_LINUX_KEYCODES unused */ | ||
817 | /* DEV_MAC_HID_KEYBOARD_LOCK_KEYCODES unused */ | ||
818 | { DEV_MAC_HID_MOUSE_BUTTON_EMULATION, "mouse_button_emulation" }, | ||
819 | { DEV_MAC_HID_MOUSE_BUTTON2_KEYCODE, "mouse_button2_keycode" }, | ||
820 | { DEV_MAC_HID_MOUSE_BUTTON3_KEYCODE, "mouse_button3_keycode" }, | ||
821 | /* DEV_MAC_HID_ADB_MOUSE_SENDS_KEYCODES unused */ | ||
822 | {} | ||
823 | }; | ||
824 | |||
825 | static const struct trans_ctl_table trans_raid_table[] = { | ||
826 | { DEV_RAID_SPEED_LIMIT_MIN, "speed_limit_min" }, | ||
827 | { DEV_RAID_SPEED_LIMIT_MAX, "speed_limit_max" }, | ||
828 | {} | ||
829 | }; | ||
830 | |||
831 | static const struct trans_ctl_table trans_scsi_table[] = { | ||
832 | { DEV_SCSI_LOGGING_LEVEL, "logging_level" }, | ||
833 | {} | ||
834 | }; | ||
835 | |||
836 | static const struct trans_ctl_table trans_parport_default_table[] = { | ||
837 | { DEV_PARPORT_DEFAULT_TIMESLICE, "timeslice" }, | ||
838 | { DEV_PARPORT_DEFAULT_SPINTIME, "spintime" }, | ||
839 | {} | ||
840 | }; | ||
841 | |||
842 | static const struct trans_ctl_table trans_parport_device_table[] = { | ||
843 | { DEV_PARPORT_DEVICE_TIMESLICE, "timeslice" }, | ||
844 | {} | ||
845 | }; | ||
846 | |||
847 | static const struct trans_ctl_table trans_parport_devices_table[] = { | ||
848 | { DEV_PARPORT_DEVICES_ACTIVE, "active" }, | ||
849 | { 0, NULL, trans_parport_device_table }, | ||
850 | {} | ||
851 | }; | ||
852 | |||
853 | static const struct trans_ctl_table trans_parport_parport_table[] = { | ||
854 | { DEV_PARPORT_SPINTIME, "spintime" }, | ||
855 | { DEV_PARPORT_BASE_ADDR, "base-addr" }, | ||
856 | { DEV_PARPORT_IRQ, "irq" }, | ||
857 | { DEV_PARPORT_DMA, "dma" }, | ||
858 | { DEV_PARPORT_MODES, "modes" }, | ||
859 | { DEV_PARPORT_DEVICES, "devices", trans_parport_devices_table }, | ||
860 | { DEV_PARPORT_AUTOPROBE, "autoprobe" }, | ||
861 | { DEV_PARPORT_AUTOPROBE + 1, "autoprobe0" }, | ||
862 | { DEV_PARPORT_AUTOPROBE + 2, "autoprobe1" }, | ||
863 | { DEV_PARPORT_AUTOPROBE + 3, "autoprobe2" }, | ||
864 | { DEV_PARPORT_AUTOPROBE + 4, "autoprobe3" }, | ||
865 | {} | ||
866 | }; | ||
867 | static const struct trans_ctl_table trans_parport_table[] = { | ||
868 | { DEV_PARPORT_DEFAULT, "default", trans_parport_default_table }, | ||
869 | { 0, NULL, trans_parport_parport_table }, | ||
870 | {} | ||
871 | }; | ||
872 | |||
873 | static const struct trans_ctl_table trans_dev_table[] = { | ||
874 | { DEV_CDROM, "cdrom", trans_cdrom_table }, | ||
875 | /* DEV_HWMON unused */ | ||
876 | { DEV_PARPORT, "parport", trans_parport_table }, | ||
877 | { DEV_RAID, "raid", trans_raid_table }, | ||
878 | { DEV_MAC_HID, "mac_hid", trans_mac_hid_files }, | ||
879 | { DEV_SCSI, "scsi", trans_scsi_table }, | ||
880 | { DEV_IPMI, "ipmi", trans_ipmi_table }, | ||
881 | {} | ||
882 | }; | ||
883 | |||
884 | static const struct trans_ctl_table trans_bus_isa_table[] = { | ||
885 | { BUS_ISA_MEM_BASE, "membase" }, | ||
886 | { BUS_ISA_PORT_BASE, "portbase" }, | ||
887 | { BUS_ISA_PORT_SHIFT, "portshift" }, | ||
888 | {} | ||
889 | }; | ||
890 | |||
891 | static const struct trans_ctl_table trans_bus_table[] = { | ||
892 | { CTL_BUS_ISA, "isa", trans_bus_isa_table }, | ||
893 | {} | ||
894 | }; | ||
895 | |||
896 | static const struct trans_ctl_table trans_arlan_conf_table0[] = { | ||
897 | { 1, "spreadingCode" }, | ||
898 | { 2, "channelNumber" }, | ||
899 | { 3, "scramblingDisable" }, | ||
900 | { 4, "txAttenuation" }, | ||
901 | { 5, "systemId" }, | ||
902 | { 6, "maxDatagramSize" }, | ||
903 | { 7, "maxFrameSize" }, | ||
904 | { 8, "maxRetries" }, | ||
905 | { 9, "receiveMode" }, | ||
906 | { 10, "priority" }, | ||
907 | { 11, "rootOrRepeater" }, | ||
908 | { 12, "SID" }, | ||
909 | { 13, "registrationMode" }, | ||
910 | { 14, "registrationFill" }, | ||
911 | { 15, "localTalkAddress" }, | ||
912 | { 16, "codeFormat" }, | ||
913 | { 17, "numChannels" }, | ||
914 | { 18, "channel1" }, | ||
915 | { 19, "channel2" }, | ||
916 | { 20, "channel3" }, | ||
917 | { 21, "channel4" }, | ||
918 | { 22, "txClear" }, | ||
919 | { 23, "txRetries" }, | ||
920 | { 24, "txRouting" }, | ||
921 | { 25, "txScrambled" }, | ||
922 | { 26, "rxParameter" }, | ||
923 | { 27, "txTimeoutMs" }, | ||
924 | { 28, "waitCardTimeout" }, | ||
925 | { 29, "channelSet" }, | ||
926 | { 30, "name" }, | ||
927 | { 31, "waitTime" }, | ||
928 | { 32, "lParameter" }, | ||
929 | { 33, "_15" }, | ||
930 | { 34, "headerSize" }, | ||
931 | { 36, "tx_delay_ms" }, | ||
932 | { 37, "retries" }, | ||
933 | { 38, "ReTransmitPacketMaxSize" }, | ||
934 | { 39, "waitReTransmitPacketMaxSize" }, | ||
935 | { 40, "fastReTransCount" }, | ||
936 | { 41, "driverRetransmissions" }, | ||
937 | { 42, "txAckTimeoutMs" }, | ||
938 | { 43, "registrationInterrupts" }, | ||
939 | { 44, "hardwareType" }, | ||
940 | { 45, "radioType" }, | ||
941 | { 46, "writeEEPROM" }, | ||
942 | { 47, "writeRadioType" }, | ||
943 | { 48, "entry_exit_debug" }, | ||
944 | { 49, "debug" }, | ||
945 | { 50, "in_speed" }, | ||
946 | { 51, "out_speed" }, | ||
947 | { 52, "in_speed10" }, | ||
948 | { 53, "out_speed10" }, | ||
949 | { 54, "in_speed_max" }, | ||
950 | { 55, "out_speed_max" }, | ||
951 | { 56, "measure_rate" }, | ||
952 | { 57, "pre_Command_Wait" }, | ||
953 | { 58, "rx_tweak1" }, | ||
954 | { 59, "rx_tweak2" }, | ||
955 | { 60, "tx_queue_len" }, | ||
956 | |||
957 | { 150, "arlan0-txRing" }, | ||
958 | { 151, "arlan0-rxRing" }, | ||
959 | { 152, "arlan0-18" }, | ||
960 | { 153, "arlan0-ring" }, | ||
961 | { 154, "arlan0-shm-cpy" }, | ||
962 | { 155, "config0" }, | ||
963 | { 156, "reset0" }, | ||
964 | {} | ||
965 | }; | ||
966 | |||
967 | static const struct trans_ctl_table trans_arlan_conf_table1[] = { | ||
968 | { 1, "spreadingCode" }, | ||
969 | { 2, "channelNumber" }, | ||
970 | { 3, "scramblingDisable" }, | ||
971 | { 4, "txAttenuation" }, | ||
972 | { 5, "systemId" }, | ||
973 | { 6, "maxDatagramSize" }, | ||
974 | { 7, "maxFrameSize" }, | ||
975 | { 8, "maxRetries" }, | ||
976 | { 9, "receiveMode" }, | ||
977 | { 10, "priority" }, | ||
978 | { 11, "rootOrRepeater" }, | ||
979 | { 12, "SID" }, | ||
980 | { 13, "registrationMode" }, | ||
981 | { 14, "registrationFill" }, | ||
982 | { 15, "localTalkAddress" }, | ||
983 | { 16, "codeFormat" }, | ||
984 | { 17, "numChannels" }, | ||
985 | { 18, "channel1" }, | ||
986 | { 19, "channel2" }, | ||
987 | { 20, "channel3" }, | ||
988 | { 21, "channel4" }, | ||
989 | { 22, "txClear" }, | ||
990 | { 23, "txRetries" }, | ||
991 | { 24, "txRouting" }, | ||
992 | { 25, "txScrambled" }, | ||
993 | { 26, "rxParameter" }, | ||
994 | { 27, "txTimeoutMs" }, | ||
995 | { 28, "waitCardTimeout" }, | ||
996 | { 29, "channelSet" }, | ||
997 | { 30, "name" }, | ||
998 | { 31, "waitTime" }, | ||
999 | { 32, "lParameter" }, | ||
1000 | { 33, "_15" }, | ||
1001 | { 34, "headerSize" }, | ||
1002 | { 36, "tx_delay_ms" }, | ||
1003 | { 37, "retries" }, | ||
1004 | { 38, "ReTransmitPacketMaxSize" }, | ||
1005 | { 39, "waitReTransmitPacketMaxSize" }, | ||
1006 | { 40, "fastReTransCount" }, | ||
1007 | { 41, "driverRetransmissions" }, | ||
1008 | { 42, "txAckTimeoutMs" }, | ||
1009 | { 43, "registrationInterrupts" }, | ||
1010 | { 44, "hardwareType" }, | ||
1011 | { 45, "radioType" }, | ||
1012 | { 46, "writeEEPROM" }, | ||
1013 | { 47, "writeRadioType" }, | ||
1014 | { 48, "entry_exit_debug" }, | ||
1015 | { 49, "debug" }, | ||
1016 | { 50, "in_speed" }, | ||
1017 | { 51, "out_speed" }, | ||
1018 | { 52, "in_speed10" }, | ||
1019 | { 53, "out_speed10" }, | ||
1020 | { 54, "in_speed_max" }, | ||
1021 | { 55, "out_speed_max" }, | ||
1022 | { 56, "measure_rate" }, | ||
1023 | { 57, "pre_Command_Wait" }, | ||
1024 | { 58, "rx_tweak1" }, | ||
1025 | { 59, "rx_tweak2" }, | ||
1026 | { 60, "tx_queue_len" }, | ||
1027 | |||
1028 | { 150, "arlan1-txRing" }, | ||
1029 | { 151, "arlan1-rxRing" }, | ||
1030 | { 152, "arlan1-18" }, | ||
1031 | { 153, "arlan1-ring" }, | ||
1032 | { 154, "arlan1-shm-cpy" }, | ||
1033 | { 155, "config1" }, | ||
1034 | { 156, "reset1" }, | ||
1035 | {} | ||
1036 | }; | ||
1037 | |||
1038 | static const struct trans_ctl_table trans_arlan_conf_table2[] = { | ||
1039 | { 1, "spreadingCode" }, | ||
1040 | { 2, "channelNumber" }, | ||
1041 | { 3, "scramblingDisable" }, | ||
1042 | { 4, "txAttenuation" }, | ||
1043 | { 5, "systemId" }, | ||
1044 | { 6, "maxDatagramSize" }, | ||
1045 | { 7, "maxFrameSize" }, | ||
1046 | { 8, "maxRetries" }, | ||
1047 | { 9, "receiveMode" }, | ||
1048 | { 10, "priority" }, | ||
1049 | { 11, "rootOrRepeater" }, | ||
1050 | { 12, "SID" }, | ||
1051 | { 13, "registrationMode" }, | ||
1052 | { 14, "registrationFill" }, | ||
1053 | { 15, "localTalkAddress" }, | ||
1054 | { 16, "codeFormat" }, | ||
1055 | { 17, "numChannels" }, | ||
1056 | { 18, "channel1" }, | ||
1057 | { 19, "channel2" }, | ||
1058 | { 20, "channel3" }, | ||
1059 | { 21, "channel4" }, | ||
1060 | { 22, "txClear" }, | ||
1061 | { 23, "txRetries" }, | ||
1062 | { 24, "txRouting" }, | ||
1063 | { 25, "txScrambled" }, | ||
1064 | { 26, "rxParameter" }, | ||
1065 | { 27, "txTimeoutMs" }, | ||
1066 | { 28, "waitCardTimeout" }, | ||
1067 | { 29, "channelSet" }, | ||
1068 | { 30, "name" }, | ||
1069 | { 31, "waitTime" }, | ||
1070 | { 32, "lParameter" }, | ||
1071 | { 33, "_15" }, | ||
1072 | { 34, "headerSize" }, | ||
1073 | { 36, "tx_delay_ms" }, | ||
1074 | { 37, "retries" }, | ||
1075 | { 38, "ReTransmitPacketMaxSize" }, | ||
1076 | { 39, "waitReTransmitPacketMaxSize" }, | ||
1077 | { 40, "fastReTransCount" }, | ||
1078 | { 41, "driverRetransmissions" }, | ||
1079 | { 42, "txAckTimeoutMs" }, | ||
1080 | { 43, "registrationInterrupts" }, | ||
1081 | { 44, "hardwareType" }, | ||
1082 | { 45, "radioType" }, | ||
1083 | { 46, "writeEEPROM" }, | ||
1084 | { 47, "writeRadioType" }, | ||
1085 | { 48, "entry_exit_debug" }, | ||
1086 | { 49, "debug" }, | ||
1087 | { 50, "in_speed" }, | ||
1088 | { 51, "out_speed" }, | ||
1089 | { 52, "in_speed10" }, | ||
1090 | { 53, "out_speed10" }, | ||
1091 | { 54, "in_speed_max" }, | ||
1092 | { 55, "out_speed_max" }, | ||
1093 | { 56, "measure_rate" }, | ||
1094 | { 57, "pre_Command_Wait" }, | ||
1095 | { 58, "rx_tweak1" }, | ||
1096 | { 59, "rx_tweak2" }, | ||
1097 | { 60, "tx_queue_len" }, | ||
1098 | |||
1099 | { 150, "arlan2-txRing" }, | ||
1100 | { 151, "arlan2-rxRing" }, | ||
1101 | { 152, "arlan2-18" }, | ||
1102 | { 153, "arlan2-ring" }, | ||
1103 | { 154, "arlan2-shm-cpy" }, | ||
1104 | { 155, "config2" }, | ||
1105 | { 156, "reset2" }, | ||
1106 | {} | ||
1107 | }; | ||
1108 | |||
1109 | static const struct trans_ctl_table trans_arlan_conf_table3[] = { | ||
1110 | { 1, "spreadingCode" }, | ||
1111 | { 2, "channelNumber" }, | ||
1112 | { 3, "scramblingDisable" }, | ||
1113 | { 4, "txAttenuation" }, | ||
1114 | { 5, "systemId" }, | ||
1115 | { 6, "maxDatagramSize" }, | ||
1116 | { 7, "maxFrameSize" }, | ||
1117 | { 8, "maxRetries" }, | ||
1118 | { 9, "receiveMode" }, | ||
1119 | { 10, "priority" }, | ||
1120 | { 11, "rootOrRepeater" }, | ||
1121 | { 12, "SID" }, | ||
1122 | { 13, "registrationMode" }, | ||
1123 | { 14, "registrationFill" }, | ||
1124 | { 15, "localTalkAddress" }, | ||
1125 | { 16, "codeFormat" }, | ||
1126 | { 17, "numChannels" }, | ||
1127 | { 18, "channel1" }, | ||
1128 | { 19, "channel2" }, | ||
1129 | { 20, "channel3" }, | ||
1130 | { 21, "channel4" }, | ||
1131 | { 22, "txClear" }, | ||
1132 | { 23, "txRetries" }, | ||
1133 | { 24, "txRouting" }, | ||
1134 | { 25, "txScrambled" }, | ||
1135 | { 26, "rxParameter" }, | ||
1136 | { 27, "txTimeoutMs" }, | ||
1137 | { 28, "waitCardTimeout" }, | ||
1138 | { 29, "channelSet" }, | ||
1139 | { 30, "name" }, | ||
1140 | { 31, "waitTime" }, | ||
1141 | { 32, "lParameter" }, | ||
1142 | { 33, "_15" }, | ||
1143 | { 34, "headerSize" }, | ||
1144 | { 36, "tx_delay_ms" }, | ||
1145 | { 37, "retries" }, | ||
1146 | { 38, "ReTransmitPacketMaxSize" }, | ||
1147 | { 39, "waitReTransmitPacketMaxSize" }, | ||
1148 | { 40, "fastReTransCount" }, | ||
1149 | { 41, "driverRetransmissions" }, | ||
1150 | { 42, "txAckTimeoutMs" }, | ||
1151 | { 43, "registrationInterrupts" }, | ||
1152 | { 44, "hardwareType" }, | ||
1153 | { 45, "radioType" }, | ||
1154 | { 46, "writeEEPROM" }, | ||
1155 | { 47, "writeRadioType" }, | ||
1156 | { 48, "entry_exit_debug" }, | ||
1157 | { 49, "debug" }, | ||
1158 | { 50, "in_speed" }, | ||
1159 | { 51, "out_speed" }, | ||
1160 | { 52, "in_speed10" }, | ||
1161 | { 53, "out_speed10" }, | ||
1162 | { 54, "in_speed_max" }, | ||
1163 | { 55, "out_speed_max" }, | ||
1164 | { 56, "measure_rate" }, | ||
1165 | { 57, "pre_Command_Wait" }, | ||
1166 | { 58, "rx_tweak1" }, | ||
1167 | { 59, "rx_tweak2" }, | ||
1168 | { 60, "tx_queue_len" }, | ||
1169 | |||
1170 | { 150, "arlan3-txRing" }, | ||
1171 | { 151, "arlan3-rxRing" }, | ||
1172 | { 152, "arlan3-18" }, | ||
1173 | { 153, "arlan3-ring" }, | ||
1174 | { 154, "arlan3-shm-cpy" }, | ||
1175 | { 155, "config3" }, | ||
1176 | { 156, "reset3" }, | ||
1177 | {} | ||
1178 | }; | ||
1179 | |||
1180 | static const struct trans_ctl_table trans_arlan_table[] = { | ||
1181 | { 1, "arlan0", trans_arlan_conf_table0 }, | ||
1182 | { 2, "arlan1", trans_arlan_conf_table1 }, | ||
1183 | { 3, "arlan2", trans_arlan_conf_table2 }, | ||
1184 | { 4, "arlan3", trans_arlan_conf_table3 }, | ||
1185 | {} | ||
1186 | }; | ||
1187 | |||
1188 | static const struct trans_ctl_table trans_s390dbf_table[] = { | ||
1189 | { 5678 /* CTL_S390DBF_STOPPABLE */, "debug_stoppable" }, | ||
1190 | { 5679 /* CTL_S390DBF_ACTIVE */, "debug_active" }, | ||
1191 | {} | ||
1192 | }; | ||
1193 | |||
1194 | static const struct trans_ctl_table trans_sunrpc_table[] = { | ||
1195 | { CTL_RPCDEBUG, "rpc_debug" }, | ||
1196 | { CTL_NFSDEBUG, "nfs_debug" }, | ||
1197 | { CTL_NFSDDEBUG, "nfsd_debug" }, | ||
1198 | { CTL_NLMDEBUG, "nlm_debug" }, | ||
1199 | { CTL_SLOTTABLE_UDP, "udp_slot_table_entries" }, | ||
1200 | { CTL_SLOTTABLE_TCP, "tcp_slot_table_entries" }, | ||
1201 | { CTL_MIN_RESVPORT, "min_resvport" }, | ||
1202 | { CTL_MAX_RESVPORT, "max_resvport" }, | ||
1203 | {} | ||
1204 | }; | ||
1205 | |||
1206 | static const struct trans_ctl_table trans_pm_table[] = { | ||
1207 | { 1 /* CTL_PM_SUSPEND */, "suspend" }, | ||
1208 | { 2 /* CTL_PM_CMODE */, "cmode" }, | ||
1209 | { 3 /* CTL_PM_P0 */, "p0" }, | ||
1210 | { 4 /* CTL_PM_CM */, "cm" }, | ||
1211 | {} | ||
1212 | }; | ||
1213 | |||
1214 | static const struct trans_ctl_table trans_frv_table[] = { | ||
1215 | { 1, "cache-mode" }, | ||
1216 | { 2, "pin-cxnr" }, | ||
1217 | {} | ||
1218 | }; | ||
1219 | |||
1220 | static const struct trans_ctl_table trans_root_table[] = { | ||
1221 | { CTL_KERN, "kernel", trans_kern_table }, | ||
1222 | { CTL_VM, "vm", trans_vm_table }, | ||
1223 | { CTL_NET, "net", trans_net_table }, | ||
1224 | /* CTL_PROC not used */ | ||
1225 | { CTL_FS, "fs", trans_fs_table }, | ||
1226 | { CTL_DEBUG, "debug", trans_debug_table }, | ||
1227 | { CTL_DEV, "dev", trans_dev_table }, | ||
1228 | { CTL_BUS, "bus", trans_bus_table }, | ||
1229 | { CTL_ABI, "abi" }, | ||
1230 | /* CTL_CPU not used */ | ||
1231 | { CTL_ARLAN, "arlan", trans_arlan_table }, | ||
1232 | { CTL_S390DBF, "s390dbf", trans_s390dbf_table }, | ||
1233 | { CTL_SUNRPC, "sunrpc", trans_sunrpc_table }, | ||
1234 | { CTL_PM, "pm", trans_pm_table }, | ||
1235 | { CTL_FRV, "frv", trans_frv_table }, | ||
1236 | {} | ||
1237 | }; | ||
1238 | |||
1239 | |||
1240 | |||
1241 | 8 | ||
1242 | static int sysctl_depth(struct ctl_table *table) | 9 | static int sysctl_depth(struct ctl_table *table) |
1243 | { | 10 | { |
@@ -1261,47 +28,6 @@ static struct ctl_table *sysctl_parent(struct ctl_table *table, int n) | |||
1261 | return table; | 28 | return table; |
1262 | } | 29 | } |
1263 | 30 | ||
1264 | static const struct trans_ctl_table *sysctl_binary_lookup(struct ctl_table *table) | ||
1265 | { | ||
1266 | struct ctl_table *test; | ||
1267 | const struct trans_ctl_table *ref; | ||
1268 | int cur_depth; | ||
1269 | |||
1270 | cur_depth = sysctl_depth(table); | ||
1271 | |||
1272 | ref = trans_root_table; | ||
1273 | repeat: | ||
1274 | test = sysctl_parent(table, cur_depth); | ||
1275 | for (; ref->ctl_name || ref->procname || ref->child; ref++) { | ||
1276 | int match = 0; | ||
1277 | |||
1278 | if (cur_depth && !ref->child) | ||
1279 | continue; | ||
1280 | |||
1281 | if (test->procname && ref->procname && | ||
1282 | (strcmp(test->procname, ref->procname) == 0)) | ||
1283 | match++; | ||
1284 | |||
1285 | if (test->ctl_name && ref->ctl_name && | ||
1286 | (test->ctl_name == ref->ctl_name)) | ||
1287 | match++; | ||
1288 | |||
1289 | if (!ref->ctl_name && !ref->procname) | ||
1290 | match++; | ||
1291 | |||
1292 | if (match) { | ||
1293 | if (cur_depth != 0) { | ||
1294 | cur_depth--; | ||
1295 | ref = ref->child; | ||
1296 | goto repeat; | ||
1297 | } | ||
1298 | goto out; | ||
1299 | } | ||
1300 | } | ||
1301 | ref = NULL; | ||
1302 | out: | ||
1303 | return ref; | ||
1304 | } | ||
1305 | 31 | ||
1306 | static void sysctl_print_path(struct ctl_table *table) | 32 | static void sysctl_print_path(struct ctl_table *table) |
1307 | { | 33 | { |
@@ -1315,26 +41,6 @@ static void sysctl_print_path(struct ctl_table *table) | |||
1315 | } | 41 | } |
1316 | } | 42 | } |
1317 | printk(" "); | 43 | printk(" "); |
1318 | if (table->ctl_name) { | ||
1319 | for (i = depth; i >= 0; i--) { | ||
1320 | tmp = sysctl_parent(table, i); | ||
1321 | printk(".%d", tmp->ctl_name); | ||
1322 | } | ||
1323 | } | ||
1324 | } | ||
1325 | |||
1326 | static void sysctl_repair_table(struct ctl_table *table) | ||
1327 | { | ||
1328 | /* Don't complain about the classic default | ||
1329 | * sysctl strategy routine. Maybe later we | ||
1330 | * can get the tables fixed and complain about | ||
1331 | * this. | ||
1332 | */ | ||
1333 | if (table->ctl_name && table->procname && | ||
1334 | (table->proc_handler == proc_dointvec) && | ||
1335 | (!table->strategy)) { | ||
1336 | table->strategy = sysctl_data; | ||
1337 | } | ||
1338 | } | 44 | } |
1339 | 45 | ||
1340 | static struct ctl_table *sysctl_check_lookup(struct nsproxy *namespaces, | 46 | static struct ctl_table *sysctl_check_lookup(struct nsproxy *namespaces, |
@@ -1352,7 +58,7 @@ static struct ctl_table *sysctl_check_lookup(struct nsproxy *namespaces, | |||
1352 | ref = head->ctl_table; | 58 | ref = head->ctl_table; |
1353 | repeat: | 59 | repeat: |
1354 | test = sysctl_parent(table, cur_depth); | 60 | test = sysctl_parent(table, cur_depth); |
1355 | for (; ref->ctl_name || ref->procname; ref++) { | 61 | for (; ref->procname; ref++) { |
1356 | int match = 0; | 62 | int match = 0; |
1357 | if (cur_depth && !ref->child) | 63 | if (cur_depth && !ref->child) |
1358 | continue; | 64 | continue; |
@@ -1361,10 +67,6 @@ repeat: | |||
1361 | (strcmp(test->procname, ref->procname) == 0)) | 67 | (strcmp(test->procname, ref->procname) == 0)) |
1362 | match++; | 68 | match++; |
1363 | 69 | ||
1364 | if (test->ctl_name && ref->ctl_name && | ||
1365 | (test->ctl_name == ref->ctl_name)) | ||
1366 | match++; | ||
1367 | |||
1368 | if (match) { | 70 | if (match) { |
1369 | if (cur_depth != 0) { | 71 | if (cur_depth != 0) { |
1370 | cur_depth--; | 72 | cur_depth--; |
@@ -1392,38 +94,6 @@ static void set_fail(const char **fail, struct ctl_table *table, const char *str | |||
1392 | *fail = str; | 94 | *fail = str; |
1393 | } | 95 | } |
1394 | 96 | ||
1395 | static int sysctl_check_dir(struct nsproxy *namespaces, | ||
1396 | struct ctl_table *table) | ||
1397 | { | ||
1398 | struct ctl_table *ref; | ||
1399 | int error; | ||
1400 | |||
1401 | error = 0; | ||
1402 | ref = sysctl_check_lookup(namespaces, table); | ||
1403 | if (ref) { | ||
1404 | int match = 0; | ||
1405 | if ((!table->procname && !ref->procname) || | ||
1406 | (table->procname && ref->procname && | ||
1407 | (strcmp(table->procname, ref->procname) == 0))) | ||
1408 | match++; | ||
1409 | |||
1410 | if ((!table->ctl_name && !ref->ctl_name) || | ||
1411 | (table->ctl_name && ref->ctl_name && | ||
1412 | (table->ctl_name == ref->ctl_name))) | ||
1413 | match++; | ||
1414 | |||
1415 | if (match != 2) { | ||
1416 | printk(KERN_ERR "%s: failed: ", __func__); | ||
1417 | sysctl_print_path(table); | ||
1418 | printk(" ref: "); | ||
1419 | sysctl_print_path(ref); | ||
1420 | printk("\n"); | ||
1421 | error = -EINVAL; | ||
1422 | } | ||
1423 | } | ||
1424 | return error; | ||
1425 | } | ||
1426 | |||
1427 | static void sysctl_check_leaf(struct nsproxy *namespaces, | 97 | static void sysctl_check_leaf(struct nsproxy *namespaces, |
1428 | struct ctl_table *table, const char **fail) | 98 | struct ctl_table *table, const char **fail) |
1429 | { | 99 | { |
@@ -1434,37 +104,15 @@ static void sysctl_check_leaf(struct nsproxy *namespaces, | |||
1434 | set_fail(fail, table, "Sysctl already exists"); | 104 | set_fail(fail, table, "Sysctl already exists"); |
1435 | } | 105 | } |
1436 | 106 | ||
1437 | static void sysctl_check_bin_path(struct ctl_table *table, const char **fail) | ||
1438 | { | ||
1439 | const struct trans_ctl_table *ref; | ||
1440 | |||
1441 | ref = sysctl_binary_lookup(table); | ||
1442 | if (table->ctl_name && !ref) | ||
1443 | set_fail(fail, table, "Unknown sysctl binary path"); | ||
1444 | if (ref) { | ||
1445 | if (ref->procname && | ||
1446 | (!table->procname || | ||
1447 | (strcmp(table->procname, ref->procname) != 0))) | ||
1448 | set_fail(fail, table, "procname does not match binary path procname"); | ||
1449 | |||
1450 | if (ref->ctl_name && table->ctl_name && | ||
1451 | (table->ctl_name != ref->ctl_name)) | ||
1452 | set_fail(fail, table, "ctl_name does not match binary path ctl_name"); | ||
1453 | } | ||
1454 | } | ||
1455 | |||
1456 | int sysctl_check_table(struct nsproxy *namespaces, struct ctl_table *table) | 107 | int sysctl_check_table(struct nsproxy *namespaces, struct ctl_table *table) |
1457 | { | 108 | { |
1458 | int error = 0; | 109 | int error = 0; |
1459 | for (; table->ctl_name || table->procname; table++) { | 110 | for (; table->procname; table++) { |
1460 | const char *fail = NULL; | 111 | const char *fail = NULL; |
1461 | 112 | ||
1462 | sysctl_repair_table(table); | ||
1463 | if (table->parent) { | 113 | if (table->parent) { |
1464 | if (table->procname && !table->parent->procname) | 114 | if (table->procname && !table->parent->procname) |
1465 | set_fail(&fail, table, "Parent without procname"); | 115 | set_fail(&fail, table, "Parent without procname"); |
1466 | if (table->ctl_name && !table->parent->ctl_name) | ||
1467 | set_fail(&fail, table, "Parent without ctl_name"); | ||
1468 | } | 116 | } |
1469 | if (!table->procname) | 117 | if (!table->procname) |
1470 | set_fail(&fail, table, "No procname"); | 118 | set_fail(&fail, table, "No procname"); |
@@ -1477,21 +125,12 @@ int sysctl_check_table(struct nsproxy *namespaces, struct ctl_table *table) | |||
1477 | set_fail(&fail, table, "Writable sysctl directory"); | 125 | set_fail(&fail, table, "Writable sysctl directory"); |
1478 | if (table->proc_handler) | 126 | if (table->proc_handler) |
1479 | set_fail(&fail, table, "Directory with proc_handler"); | 127 | set_fail(&fail, table, "Directory with proc_handler"); |
1480 | if (table->strategy) | ||
1481 | set_fail(&fail, table, "Directory with strategy"); | ||
1482 | if (table->extra1) | 128 | if (table->extra1) |
1483 | set_fail(&fail, table, "Directory with extra1"); | 129 | set_fail(&fail, table, "Directory with extra1"); |
1484 | if (table->extra2) | 130 | if (table->extra2) |
1485 | set_fail(&fail, table, "Directory with extra2"); | 131 | set_fail(&fail, table, "Directory with extra2"); |
1486 | if (sysctl_check_dir(namespaces, table)) | ||
1487 | set_fail(&fail, table, "Inconsistent directory names"); | ||
1488 | } else { | 132 | } else { |
1489 | if ((table->strategy == sysctl_data) || | 133 | if ((table->proc_handler == proc_dostring) || |
1490 | (table->strategy == sysctl_string) || | ||
1491 | (table->strategy == sysctl_intvec) || | ||
1492 | (table->strategy == sysctl_jiffies) || | ||
1493 | (table->strategy == sysctl_ms_jiffies) || | ||
1494 | (table->proc_handler == proc_dostring) || | ||
1495 | (table->proc_handler == proc_dointvec) || | 134 | (table->proc_handler == proc_dointvec) || |
1496 | (table->proc_handler == proc_dointvec_minmax) || | 135 | (table->proc_handler == proc_dointvec_minmax) || |
1497 | (table->proc_handler == proc_dointvec_jiffies) || | 136 | (table->proc_handler == proc_dointvec_jiffies) || |
@@ -1513,15 +152,7 @@ int sysctl_check_table(struct nsproxy *namespaces, struct ctl_table *table) | |||
1513 | set_fail(&fail, table, "No max"); | 152 | set_fail(&fail, table, "No max"); |
1514 | } | 153 | } |
1515 | } | 154 | } |
1516 | #ifdef CONFIG_SYSCTL_SYSCALL | 155 | #ifdef CONFIG_PROC_SYSCTL |
1517 | if (table->ctl_name && !table->strategy) | ||
1518 | set_fail(&fail, table, "Missing strategy"); | ||
1519 | #endif | ||
1520 | #if 0 | ||
1521 | if (!table->ctl_name && table->strategy) | ||
1522 | set_fail(&fail, table, "Strategy without ctl_name"); | ||
1523 | #endif | ||
1524 | #ifdef CONFIG_PROC_FS | ||
1525 | if (table->procname && !table->proc_handler) | 156 | if (table->procname && !table->proc_handler) |
1526 | set_fail(&fail, table, "No proc_handler"); | 157 | set_fail(&fail, table, "No proc_handler"); |
1527 | #endif | 158 | #endif |
@@ -1531,7 +162,6 @@ int sysctl_check_table(struct nsproxy *namespaces, struct ctl_table *table) | |||
1531 | #endif | 162 | #endif |
1532 | sysctl_check_leaf(namespaces, table, &fail); | 163 | sysctl_check_leaf(namespaces, table, &fail); |
1533 | } | 164 | } |
1534 | sysctl_check_bin_path(table, &fail); | ||
1535 | if (table->mode > 0777) | 165 | if (table->mode > 0777) |
1536 | set_fail(&fail, table, "bogus .mode"); | 166 | set_fail(&fail, table, "bogus .mode"); |
1537 | if (fail) { | 167 | if (fail) { |
diff --git a/kernel/time.c b/kernel/time.c index 2e2e469a7fec..c6324d96009e 100644 --- a/kernel/time.c +++ b/kernel/time.c | |||
@@ -136,7 +136,6 @@ static inline void warp_clock(void) | |||
136 | write_seqlock_irq(&xtime_lock); | 136 | write_seqlock_irq(&xtime_lock); |
137 | wall_to_monotonic.tv_sec -= sys_tz.tz_minuteswest * 60; | 137 | wall_to_monotonic.tv_sec -= sys_tz.tz_minuteswest * 60; |
138 | xtime.tv_sec += sys_tz.tz_minuteswest * 60; | 138 | xtime.tv_sec += sys_tz.tz_minuteswest * 60; |
139 | update_xtime_cache(0); | ||
140 | write_sequnlock_irq(&xtime_lock); | 139 | write_sequnlock_irq(&xtime_lock); |
141 | clock_was_set(); | 140 | clock_was_set(); |
142 | } | 141 | } |
@@ -662,6 +661,36 @@ u64 nsec_to_clock_t(u64 x) | |||
662 | #endif | 661 | #endif |
663 | } | 662 | } |
664 | 663 | ||
664 | /** | ||
665 | * nsecs_to_jiffies - Convert nsecs in u64 to jiffies | ||
666 | * | ||
667 | * @n: nsecs in u64 | ||
668 | * | ||
669 | * Unlike {m,u}secs_to_jiffies, type of input is not unsigned int but u64. | ||
670 | * And this doesn't return MAX_JIFFY_OFFSET since this function is designed | ||
671 | * for scheduler, not for use in device drivers to calculate timeout value. | ||
672 | * | ||
673 | * note: | ||
674 | * NSEC_PER_SEC = 10^9 = (5^9 * 2^9) = (1953125 * 512) | ||
675 | * ULLONG_MAX ns = 18446744073.709551615 secs = about 584 years | ||
676 | */ | ||
677 | unsigned long nsecs_to_jiffies(u64 n) | ||
678 | { | ||
679 | #if (NSEC_PER_SEC % HZ) == 0 | ||
680 | /* Common case, HZ = 100, 128, 200, 250, 256, 500, 512, 1000 etc. */ | ||
681 | return div_u64(n, NSEC_PER_SEC / HZ); | ||
682 | #elif (HZ % 512) == 0 | ||
683 | /* overflow after 292 years if HZ = 1024 */ | ||
684 | return div_u64(n * HZ / 512, NSEC_PER_SEC / 512); | ||
685 | #else | ||
686 | /* | ||
687 | * Generic case - optimized for cases where HZ is a multiple of 3. | ||
688 | * overflow after 64.99 years, exact for HZ = 60, 72, 90, 120 etc. | ||
689 | */ | ||
690 | return div_u64(n * 9, (9ull * NSEC_PER_SEC + HZ / 2) / HZ); | ||
691 | #endif | ||
692 | } | ||
693 | |||
665 | #if (BITS_PER_LONG < 64) | 694 | #if (BITS_PER_LONG < 64) |
666 | u64 get_jiffies_64(void) | 695 | u64 get_jiffies_64(void) |
667 | { | 696 | { |
diff --git a/kernel/time/clockevents.c b/kernel/time/clockevents.c index 620b58abdc32..20a8920029ee 100644 --- a/kernel/time/clockevents.c +++ b/kernel/time/clockevents.c | |||
@@ -20,6 +20,8 @@ | |||
20 | #include <linux/sysdev.h> | 20 | #include <linux/sysdev.h> |
21 | #include <linux/tick.h> | 21 | #include <linux/tick.h> |
22 | 22 | ||
23 | #include "tick-internal.h" | ||
24 | |||
23 | /* The registered clock event devices */ | 25 | /* The registered clock event devices */ |
24 | static LIST_HEAD(clockevent_devices); | 26 | static LIST_HEAD(clockevent_devices); |
25 | static LIST_HEAD(clockevents_released); | 27 | static LIST_HEAD(clockevents_released); |
@@ -37,10 +39,9 @@ static DEFINE_SPINLOCK(clockevents_lock); | |||
37 | * | 39 | * |
38 | * Math helper, returns latch value converted to nanoseconds (bound checked) | 40 | * Math helper, returns latch value converted to nanoseconds (bound checked) |
39 | */ | 41 | */ |
40 | unsigned long clockevent_delta2ns(unsigned long latch, | 42 | u64 clockevent_delta2ns(unsigned long latch, struct clock_event_device *evt) |
41 | struct clock_event_device *evt) | ||
42 | { | 43 | { |
43 | u64 clc = ((u64) latch << evt->shift); | 44 | u64 clc = (u64) latch << evt->shift; |
44 | 45 | ||
45 | if (unlikely(!evt->mult)) { | 46 | if (unlikely(!evt->mult)) { |
46 | evt->mult = 1; | 47 | evt->mult = 1; |
@@ -50,10 +51,10 @@ unsigned long clockevent_delta2ns(unsigned long latch, | |||
50 | do_div(clc, evt->mult); | 51 | do_div(clc, evt->mult); |
51 | if (clc < 1000) | 52 | if (clc < 1000) |
52 | clc = 1000; | 53 | clc = 1000; |
53 | if (clc > LONG_MAX) | 54 | if (clc > KTIME_MAX) |
54 | clc = LONG_MAX; | 55 | clc = KTIME_MAX; |
55 | 56 | ||
56 | return (unsigned long) clc; | 57 | return clc; |
57 | } | 58 | } |
58 | EXPORT_SYMBOL_GPL(clockevent_delta2ns); | 59 | EXPORT_SYMBOL_GPL(clockevent_delta2ns); |
59 | 60 | ||
diff --git a/kernel/time/clocksource.c b/kernel/time/clocksource.c index 5e18c6ab2c6a..e85c23404d34 100644 --- a/kernel/time/clocksource.c +++ b/kernel/time/clocksource.c | |||
@@ -39,7 +39,7 @@ void timecounter_init(struct timecounter *tc, | |||
39 | tc->cycle_last = cc->read(cc); | 39 | tc->cycle_last = cc->read(cc); |
40 | tc->nsec = start_tstamp; | 40 | tc->nsec = start_tstamp; |
41 | } | 41 | } |
42 | EXPORT_SYMBOL(timecounter_init); | 42 | EXPORT_SYMBOL_GPL(timecounter_init); |
43 | 43 | ||
44 | /** | 44 | /** |
45 | * timecounter_read_delta - get nanoseconds since last call of this function | 45 | * timecounter_read_delta - get nanoseconds since last call of this function |
@@ -83,7 +83,7 @@ u64 timecounter_read(struct timecounter *tc) | |||
83 | 83 | ||
84 | return nsec; | 84 | return nsec; |
85 | } | 85 | } |
86 | EXPORT_SYMBOL(timecounter_read); | 86 | EXPORT_SYMBOL_GPL(timecounter_read); |
87 | 87 | ||
88 | u64 timecounter_cyc2time(struct timecounter *tc, | 88 | u64 timecounter_cyc2time(struct timecounter *tc, |
89 | cycle_t cycle_tstamp) | 89 | cycle_t cycle_tstamp) |
@@ -105,7 +105,60 @@ u64 timecounter_cyc2time(struct timecounter *tc, | |||
105 | 105 | ||
106 | return nsec; | 106 | return nsec; |
107 | } | 107 | } |
108 | EXPORT_SYMBOL(timecounter_cyc2time); | 108 | EXPORT_SYMBOL_GPL(timecounter_cyc2time); |
109 | |||
110 | /** | ||
111 | * clocks_calc_mult_shift - calculate mult/shift factors for scaled math of clocks | ||
112 | * @mult: pointer to mult variable | ||
113 | * @shift: pointer to shift variable | ||
114 | * @from: frequency to convert from | ||
115 | * @to: frequency to convert to | ||
116 | * @minsec: guaranteed runtime conversion range in seconds | ||
117 | * | ||
118 | * The function evaluates the shift/mult pair for the scaled math | ||
119 | * operations of clocksources and clockevents. | ||
120 | * | ||
121 | * @to and @from are frequency values in HZ. For clock sources @to is | ||
122 | * NSEC_PER_SEC == 1GHz and @from is the counter frequency. For clock | ||
123 | * event @to is the counter frequency and @from is NSEC_PER_SEC. | ||
124 | * | ||
125 | * The @minsec conversion range argument controls the time frame in | ||
126 | * seconds which must be covered by the runtime conversion with the | ||
127 | * calculated mult and shift factors. This guarantees that no 64bit | ||
128 | * overflow happens when the input value of the conversion is | ||
129 | * multiplied with the calculated mult factor. Larger ranges may | ||
130 | * reduce the conversion accuracy by chosing smaller mult and shift | ||
131 | * factors. | ||
132 | */ | ||
133 | void | ||
134 | clocks_calc_mult_shift(u32 *mult, u32 *shift, u32 from, u32 to, u32 minsec) | ||
135 | { | ||
136 | u64 tmp; | ||
137 | u32 sft, sftacc= 32; | ||
138 | |||
139 | /* | ||
140 | * Calculate the shift factor which is limiting the conversion | ||
141 | * range: | ||
142 | */ | ||
143 | tmp = ((u64)minsec * from) >> 32; | ||
144 | while (tmp) { | ||
145 | tmp >>=1; | ||
146 | sftacc--; | ||
147 | } | ||
148 | |||
149 | /* | ||
150 | * Find the conversion shift/mult pair which has the best | ||
151 | * accuracy and fits the maxsec conversion range: | ||
152 | */ | ||
153 | for (sft = 32; sft > 0; sft--) { | ||
154 | tmp = (u64) to << sft; | ||
155 | do_div(tmp, from); | ||
156 | if ((tmp >> sftacc) == 0) | ||
157 | break; | ||
158 | } | ||
159 | *mult = tmp; | ||
160 | *shift = sft; | ||
161 | } | ||
109 | 162 | ||
110 | /*[Clocksource internal variables]--------- | 163 | /*[Clocksource internal variables]--------- |
111 | * curr_clocksource: | 164 | * curr_clocksource: |
@@ -413,6 +466,47 @@ void clocksource_touch_watchdog(void) | |||
413 | clocksource_resume_watchdog(); | 466 | clocksource_resume_watchdog(); |
414 | } | 467 | } |
415 | 468 | ||
469 | /** | ||
470 | * clocksource_max_deferment - Returns max time the clocksource can be deferred | ||
471 | * @cs: Pointer to clocksource | ||
472 | * | ||
473 | */ | ||
474 | static u64 clocksource_max_deferment(struct clocksource *cs) | ||
475 | { | ||
476 | u64 max_nsecs, max_cycles; | ||
477 | |||
478 | /* | ||
479 | * Calculate the maximum number of cycles that we can pass to the | ||
480 | * cyc2ns function without overflowing a 64-bit signed result. The | ||
481 | * maximum number of cycles is equal to ULLONG_MAX/cs->mult which | ||
482 | * is equivalent to the below. | ||
483 | * max_cycles < (2^63)/cs->mult | ||
484 | * max_cycles < 2^(log2((2^63)/cs->mult)) | ||
485 | * max_cycles < 2^(log2(2^63) - log2(cs->mult)) | ||
486 | * max_cycles < 2^(63 - log2(cs->mult)) | ||
487 | * max_cycles < 1 << (63 - log2(cs->mult)) | ||
488 | * Please note that we add 1 to the result of the log2 to account for | ||
489 | * any rounding errors, ensure the above inequality is satisfied and | ||
490 | * no overflow will occur. | ||
491 | */ | ||
492 | max_cycles = 1ULL << (63 - (ilog2(cs->mult) + 1)); | ||
493 | |||
494 | /* | ||
495 | * The actual maximum number of cycles we can defer the clocksource is | ||
496 | * determined by the minimum of max_cycles and cs->mask. | ||
497 | */ | ||
498 | max_cycles = min_t(u64, max_cycles, (u64) cs->mask); | ||
499 | max_nsecs = clocksource_cyc2ns(max_cycles, cs->mult, cs->shift); | ||
500 | |||
501 | /* | ||
502 | * To ensure that the clocksource does not wrap whilst we are idle, | ||
503 | * limit the time the clocksource can be deferred by 12.5%. Please | ||
504 | * note a margin of 12.5% is used because this can be computed with | ||
505 | * a shift, versus say 10% which would require division. | ||
506 | */ | ||
507 | return max_nsecs - (max_nsecs >> 5); | ||
508 | } | ||
509 | |||
416 | #ifdef CONFIG_GENERIC_TIME | 510 | #ifdef CONFIG_GENERIC_TIME |
417 | 511 | ||
418 | /** | 512 | /** |
@@ -511,6 +605,9 @@ static void clocksource_enqueue(struct clocksource *cs) | |||
511 | */ | 605 | */ |
512 | int clocksource_register(struct clocksource *cs) | 606 | int clocksource_register(struct clocksource *cs) |
513 | { | 607 | { |
608 | /* calculate max idle time permitted for this clocksource */ | ||
609 | cs->max_idle_ns = clocksource_max_deferment(cs); | ||
610 | |||
514 | mutex_lock(&clocksource_mutex); | 611 | mutex_lock(&clocksource_mutex); |
515 | clocksource_enqueue(cs); | 612 | clocksource_enqueue(cs); |
516 | clocksource_select(); | 613 | clocksource_select(); |
@@ -580,7 +677,7 @@ sysfs_show_current_clocksources(struct sys_device *dev, | |||
580 | * @count: length of buffer | 677 | * @count: length of buffer |
581 | * | 678 | * |
582 | * Takes input from sysfs interface for manually overriding the default | 679 | * Takes input from sysfs interface for manually overriding the default |
583 | * clocksource selction. | 680 | * clocksource selection. |
584 | */ | 681 | */ |
585 | static ssize_t sysfs_override_clocksource(struct sys_device *dev, | 682 | static ssize_t sysfs_override_clocksource(struct sys_device *dev, |
586 | struct sysdev_attribute *attr, | 683 | struct sysdev_attribute *attr, |
diff --git a/kernel/time/tick-oneshot.c b/kernel/time/tick-oneshot.c index a96c0e2b89cf..0a8a213016f0 100644 --- a/kernel/time/tick-oneshot.c +++ b/kernel/time/tick-oneshot.c | |||
@@ -50,9 +50,9 @@ int tick_dev_program_event(struct clock_event_device *dev, ktime_t expires, | |||
50 | dev->min_delta_ns += dev->min_delta_ns >> 1; | 50 | dev->min_delta_ns += dev->min_delta_ns >> 1; |
51 | 51 | ||
52 | printk(KERN_WARNING | 52 | printk(KERN_WARNING |
53 | "CE: %s increasing min_delta_ns to %lu nsec\n", | 53 | "CE: %s increasing min_delta_ns to %llu nsec\n", |
54 | dev->name ? dev->name : "?", | 54 | dev->name ? dev->name : "?", |
55 | dev->min_delta_ns << 1); | 55 | (unsigned long long) dev->min_delta_ns << 1); |
56 | 56 | ||
57 | i = 0; | 57 | i = 0; |
58 | } | 58 | } |
diff --git a/kernel/time/tick-sched.c b/kernel/time/tick-sched.c index 89aed5933ed4..f992762d7f51 100644 --- a/kernel/time/tick-sched.c +++ b/kernel/time/tick-sched.c | |||
@@ -134,18 +134,13 @@ __setup("nohz=", setup_tick_nohz); | |||
134 | * value. We do this unconditionally on any cpu, as we don't know whether the | 134 | * value. We do this unconditionally on any cpu, as we don't know whether the |
135 | * cpu, which has the update task assigned is in a long sleep. | 135 | * cpu, which has the update task assigned is in a long sleep. |
136 | */ | 136 | */ |
137 | static void tick_nohz_update_jiffies(void) | 137 | static void tick_nohz_update_jiffies(ktime_t now) |
138 | { | 138 | { |
139 | int cpu = smp_processor_id(); | 139 | int cpu = smp_processor_id(); |
140 | struct tick_sched *ts = &per_cpu(tick_cpu_sched, cpu); | 140 | struct tick_sched *ts = &per_cpu(tick_cpu_sched, cpu); |
141 | unsigned long flags; | 141 | unsigned long flags; |
142 | ktime_t now; | ||
143 | |||
144 | if (!ts->tick_stopped) | ||
145 | return; | ||
146 | 142 | ||
147 | cpumask_clear_cpu(cpu, nohz_cpu_mask); | 143 | cpumask_clear_cpu(cpu, nohz_cpu_mask); |
148 | now = ktime_get(); | ||
149 | ts->idle_waketime = now; | 144 | ts->idle_waketime = now; |
150 | 145 | ||
151 | local_irq_save(flags); | 146 | local_irq_save(flags); |
@@ -155,20 +150,17 @@ static void tick_nohz_update_jiffies(void) | |||
155 | touch_softlockup_watchdog(); | 150 | touch_softlockup_watchdog(); |
156 | } | 151 | } |
157 | 152 | ||
158 | static void tick_nohz_stop_idle(int cpu) | 153 | static void tick_nohz_stop_idle(int cpu, ktime_t now) |
159 | { | 154 | { |
160 | struct tick_sched *ts = &per_cpu(tick_cpu_sched, cpu); | 155 | struct tick_sched *ts = &per_cpu(tick_cpu_sched, cpu); |
156 | ktime_t delta; | ||
161 | 157 | ||
162 | if (ts->idle_active) { | 158 | delta = ktime_sub(now, ts->idle_entrytime); |
163 | ktime_t now, delta; | 159 | ts->idle_lastupdate = now; |
164 | now = ktime_get(); | 160 | ts->idle_sleeptime = ktime_add(ts->idle_sleeptime, delta); |
165 | delta = ktime_sub(now, ts->idle_entrytime); | 161 | ts->idle_active = 0; |
166 | ts->idle_lastupdate = now; | ||
167 | ts->idle_sleeptime = ktime_add(ts->idle_sleeptime, delta); | ||
168 | ts->idle_active = 0; | ||
169 | 162 | ||
170 | sched_clock_idle_wakeup_event(0); | 163 | sched_clock_idle_wakeup_event(0); |
171 | } | ||
172 | } | 164 | } |
173 | 165 | ||
174 | static ktime_t tick_nohz_start_idle(struct tick_sched *ts) | 166 | static ktime_t tick_nohz_start_idle(struct tick_sched *ts) |
@@ -216,6 +208,7 @@ void tick_nohz_stop_sched_tick(int inidle) | |||
216 | struct tick_sched *ts; | 208 | struct tick_sched *ts; |
217 | ktime_t last_update, expires, now; | 209 | ktime_t last_update, expires, now; |
218 | struct clock_event_device *dev = __get_cpu_var(tick_cpu_device).evtdev; | 210 | struct clock_event_device *dev = __get_cpu_var(tick_cpu_device).evtdev; |
211 | u64 time_delta; | ||
219 | int cpu; | 212 | int cpu; |
220 | 213 | ||
221 | local_irq_save(flags); | 214 | local_irq_save(flags); |
@@ -263,7 +256,7 @@ void tick_nohz_stop_sched_tick(int inidle) | |||
263 | 256 | ||
264 | if (ratelimit < 10) { | 257 | if (ratelimit < 10) { |
265 | printk(KERN_ERR "NOHZ: local_softirq_pending %02x\n", | 258 | printk(KERN_ERR "NOHZ: local_softirq_pending %02x\n", |
266 | local_softirq_pending()); | 259 | (unsigned int) local_softirq_pending()); |
267 | ratelimit++; | 260 | ratelimit++; |
268 | } | 261 | } |
269 | goto end; | 262 | goto end; |
@@ -275,14 +268,18 @@ void tick_nohz_stop_sched_tick(int inidle) | |||
275 | seq = read_seqbegin(&xtime_lock); | 268 | seq = read_seqbegin(&xtime_lock); |
276 | last_update = last_jiffies_update; | 269 | last_update = last_jiffies_update; |
277 | last_jiffies = jiffies; | 270 | last_jiffies = jiffies; |
271 | time_delta = timekeeping_max_deferment(); | ||
278 | } while (read_seqretry(&xtime_lock, seq)); | 272 | } while (read_seqretry(&xtime_lock, seq)); |
279 | 273 | ||
280 | /* Get the next timer wheel timer */ | 274 | if (rcu_needs_cpu(cpu) || printk_needs_cpu(cpu) || |
281 | next_jiffies = get_next_timer_interrupt(last_jiffies); | 275 | arch_needs_cpu(cpu)) { |
282 | delta_jiffies = next_jiffies - last_jiffies; | 276 | next_jiffies = last_jiffies + 1; |
283 | |||
284 | if (rcu_needs_cpu(cpu) || printk_needs_cpu(cpu)) | ||
285 | delta_jiffies = 1; | 277 | delta_jiffies = 1; |
278 | } else { | ||
279 | /* Get the next timer wheel timer */ | ||
280 | next_jiffies = get_next_timer_interrupt(last_jiffies); | ||
281 | delta_jiffies = next_jiffies - last_jiffies; | ||
282 | } | ||
286 | /* | 283 | /* |
287 | * Do not stop the tick, if we are only one off | 284 | * Do not stop the tick, if we are only one off |
288 | * or if the cpu is required for rcu | 285 | * or if the cpu is required for rcu |
@@ -294,22 +291,51 @@ void tick_nohz_stop_sched_tick(int inidle) | |||
294 | if ((long)delta_jiffies >= 1) { | 291 | if ((long)delta_jiffies >= 1) { |
295 | 292 | ||
296 | /* | 293 | /* |
297 | * calculate the expiry time for the next timer wheel | ||
298 | * timer | ||
299 | */ | ||
300 | expires = ktime_add_ns(last_update, tick_period.tv64 * | ||
301 | delta_jiffies); | ||
302 | |||
303 | /* | ||
304 | * If this cpu is the one which updates jiffies, then | 294 | * If this cpu is the one which updates jiffies, then |
305 | * give up the assignment and let it be taken by the | 295 | * give up the assignment and let it be taken by the |
306 | * cpu which runs the tick timer next, which might be | 296 | * cpu which runs the tick timer next, which might be |
307 | * this cpu as well. If we don't drop this here the | 297 | * this cpu as well. If we don't drop this here the |
308 | * jiffies might be stale and do_timer() never | 298 | * jiffies might be stale and do_timer() never |
309 | * invoked. | 299 | * invoked. Keep track of the fact that it was the one |
300 | * which had the do_timer() duty last. If this cpu is | ||
301 | * the one which had the do_timer() duty last, we | ||
302 | * limit the sleep time to the timekeeping | ||
303 | * max_deferement value which we retrieved | ||
304 | * above. Otherwise we can sleep as long as we want. | ||
310 | */ | 305 | */ |
311 | if (cpu == tick_do_timer_cpu) | 306 | if (cpu == tick_do_timer_cpu) { |
312 | tick_do_timer_cpu = TICK_DO_TIMER_NONE; | 307 | tick_do_timer_cpu = TICK_DO_TIMER_NONE; |
308 | ts->do_timer_last = 1; | ||
309 | } else if (tick_do_timer_cpu != TICK_DO_TIMER_NONE) { | ||
310 | time_delta = KTIME_MAX; | ||
311 | ts->do_timer_last = 0; | ||
312 | } else if (!ts->do_timer_last) { | ||
313 | time_delta = KTIME_MAX; | ||
314 | } | ||
315 | |||
316 | /* | ||
317 | * calculate the expiry time for the next timer wheel | ||
318 | * timer. delta_jiffies >= NEXT_TIMER_MAX_DELTA signals | ||
319 | * that there is no timer pending or at least extremely | ||
320 | * far into the future (12 days for HZ=1000). In this | ||
321 | * case we set the expiry to the end of time. | ||
322 | */ | ||
323 | if (likely(delta_jiffies < NEXT_TIMER_MAX_DELTA)) { | ||
324 | /* | ||
325 | * Calculate the time delta for the next timer event. | ||
326 | * If the time delta exceeds the maximum time delta | ||
327 | * permitted by the current clocksource then adjust | ||
328 | * the time delta accordingly to ensure the | ||
329 | * clocksource does not wrap. | ||
330 | */ | ||
331 | time_delta = min_t(u64, time_delta, | ||
332 | tick_period.tv64 * delta_jiffies); | ||
333 | } | ||
334 | |||
335 | if (time_delta < KTIME_MAX) | ||
336 | expires = ktime_add_ns(last_update, time_delta); | ||
337 | else | ||
338 | expires.tv64 = KTIME_MAX; | ||
313 | 339 | ||
314 | if (delta_jiffies > 1) | 340 | if (delta_jiffies > 1) |
315 | cpumask_set_cpu(cpu, nohz_cpu_mask); | 341 | cpumask_set_cpu(cpu, nohz_cpu_mask); |
@@ -342,22 +368,19 @@ void tick_nohz_stop_sched_tick(int inidle) | |||
342 | 368 | ||
343 | ts->idle_sleeps++; | 369 | ts->idle_sleeps++; |
344 | 370 | ||
371 | /* Mark expires */ | ||
372 | ts->idle_expires = expires; | ||
373 | |||
345 | /* | 374 | /* |
346 | * delta_jiffies >= NEXT_TIMER_MAX_DELTA signals that | 375 | * If the expiration time == KTIME_MAX, then |
347 | * there is no timer pending or at least extremly far | 376 | * in this case we simply stop the tick timer. |
348 | * into the future (12 days for HZ=1000). In this case | ||
349 | * we simply stop the tick timer: | ||
350 | */ | 377 | */ |
351 | if (unlikely(delta_jiffies >= NEXT_TIMER_MAX_DELTA)) { | 378 | if (unlikely(expires.tv64 == KTIME_MAX)) { |
352 | ts->idle_expires.tv64 = KTIME_MAX; | ||
353 | if (ts->nohz_mode == NOHZ_MODE_HIGHRES) | 379 | if (ts->nohz_mode == NOHZ_MODE_HIGHRES) |
354 | hrtimer_cancel(&ts->sched_timer); | 380 | hrtimer_cancel(&ts->sched_timer); |
355 | goto out; | 381 | goto out; |
356 | } | 382 | } |
357 | 383 | ||
358 | /* Mark expiries */ | ||
359 | ts->idle_expires = expires; | ||
360 | |||
361 | if (ts->nohz_mode == NOHZ_MODE_HIGHRES) { | 384 | if (ts->nohz_mode == NOHZ_MODE_HIGHRES) { |
362 | hrtimer_start(&ts->sched_timer, expires, | 385 | hrtimer_start(&ts->sched_timer, expires, |
363 | HRTIMER_MODE_ABS_PINNED); | 386 | HRTIMER_MODE_ABS_PINNED); |
@@ -436,7 +459,11 @@ void tick_nohz_restart_sched_tick(void) | |||
436 | ktime_t now; | 459 | ktime_t now; |
437 | 460 | ||
438 | local_irq_disable(); | 461 | local_irq_disable(); |
439 | tick_nohz_stop_idle(cpu); | 462 | if (ts->idle_active || (ts->inidle && ts->tick_stopped)) |
463 | now = ktime_get(); | ||
464 | |||
465 | if (ts->idle_active) | ||
466 | tick_nohz_stop_idle(cpu, now); | ||
440 | 467 | ||
441 | if (!ts->inidle || !ts->tick_stopped) { | 468 | if (!ts->inidle || !ts->tick_stopped) { |
442 | ts->inidle = 0; | 469 | ts->inidle = 0; |
@@ -450,7 +477,6 @@ void tick_nohz_restart_sched_tick(void) | |||
450 | 477 | ||
451 | /* Update jiffies first */ | 478 | /* Update jiffies first */ |
452 | select_nohz_load_balancer(0); | 479 | select_nohz_load_balancer(0); |
453 | now = ktime_get(); | ||
454 | tick_do_update_jiffies64(now); | 480 | tick_do_update_jiffies64(now); |
455 | cpumask_clear_cpu(cpu, nohz_cpu_mask); | 481 | cpumask_clear_cpu(cpu, nohz_cpu_mask); |
456 | 482 | ||
@@ -584,22 +610,18 @@ static void tick_nohz_switch_to_nohz(void) | |||
584 | * timer and do not touch the other magic bits which need to be done | 610 | * timer and do not touch the other magic bits which need to be done |
585 | * when idle is left. | 611 | * when idle is left. |
586 | */ | 612 | */ |
587 | static void tick_nohz_kick_tick(int cpu) | 613 | static void tick_nohz_kick_tick(int cpu, ktime_t now) |
588 | { | 614 | { |
589 | #if 0 | 615 | #if 0 |
590 | /* Switch back to 2.6.27 behaviour */ | 616 | /* Switch back to 2.6.27 behaviour */ |
591 | 617 | ||
592 | struct tick_sched *ts = &per_cpu(tick_cpu_sched, cpu); | 618 | struct tick_sched *ts = &per_cpu(tick_cpu_sched, cpu); |
593 | ktime_t delta, now; | 619 | ktime_t delta; |
594 | |||
595 | if (!ts->tick_stopped) | ||
596 | return; | ||
597 | 620 | ||
598 | /* | 621 | /* |
599 | * Do not touch the tick device, when the next expiry is either | 622 | * Do not touch the tick device, when the next expiry is either |
600 | * already reached or less/equal than the tick period. | 623 | * already reached or less/equal than the tick period. |
601 | */ | 624 | */ |
602 | now = ktime_get(); | ||
603 | delta = ktime_sub(hrtimer_get_expires(&ts->sched_timer), now); | 625 | delta = ktime_sub(hrtimer_get_expires(&ts->sched_timer), now); |
604 | if (delta.tv64 <= tick_period.tv64) | 626 | if (delta.tv64 <= tick_period.tv64) |
605 | return; | 627 | return; |
@@ -608,9 +630,26 @@ static void tick_nohz_kick_tick(int cpu) | |||
608 | #endif | 630 | #endif |
609 | } | 631 | } |
610 | 632 | ||
633 | static inline void tick_check_nohz(int cpu) | ||
634 | { | ||
635 | struct tick_sched *ts = &per_cpu(tick_cpu_sched, cpu); | ||
636 | ktime_t now; | ||
637 | |||
638 | if (!ts->idle_active && !ts->tick_stopped) | ||
639 | return; | ||
640 | now = ktime_get(); | ||
641 | if (ts->idle_active) | ||
642 | tick_nohz_stop_idle(cpu, now); | ||
643 | if (ts->tick_stopped) { | ||
644 | tick_nohz_update_jiffies(now); | ||
645 | tick_nohz_kick_tick(cpu, now); | ||
646 | } | ||
647 | } | ||
648 | |||
611 | #else | 649 | #else |
612 | 650 | ||
613 | static inline void tick_nohz_switch_to_nohz(void) { } | 651 | static inline void tick_nohz_switch_to_nohz(void) { } |
652 | static inline void tick_check_nohz(int cpu) { } | ||
614 | 653 | ||
615 | #endif /* NO_HZ */ | 654 | #endif /* NO_HZ */ |
616 | 655 | ||
@@ -620,11 +659,7 @@ static inline void tick_nohz_switch_to_nohz(void) { } | |||
620 | void tick_check_idle(int cpu) | 659 | void tick_check_idle(int cpu) |
621 | { | 660 | { |
622 | tick_check_oneshot_broadcast(cpu); | 661 | tick_check_oneshot_broadcast(cpu); |
623 | #ifdef CONFIG_NO_HZ | 662 | tick_check_nohz(cpu); |
624 | tick_nohz_stop_idle(cpu); | ||
625 | tick_nohz_update_jiffies(); | ||
626 | tick_nohz_kick_tick(cpu); | ||
627 | #endif | ||
628 | } | 663 | } |
629 | 664 | ||
630 | /* | 665 | /* |
diff --git a/kernel/time/timecompare.c b/kernel/time/timecompare.c index 71e7f1a19156..96ff643a5a59 100644 --- a/kernel/time/timecompare.c +++ b/kernel/time/timecompare.c | |||
@@ -40,7 +40,7 @@ ktime_t timecompare_transform(struct timecompare *sync, | |||
40 | 40 | ||
41 | return ns_to_ktime(nsec); | 41 | return ns_to_ktime(nsec); |
42 | } | 42 | } |
43 | EXPORT_SYMBOL(timecompare_transform); | 43 | EXPORT_SYMBOL_GPL(timecompare_transform); |
44 | 44 | ||
45 | int timecompare_offset(struct timecompare *sync, | 45 | int timecompare_offset(struct timecompare *sync, |
46 | s64 *offset, | 46 | s64 *offset, |
@@ -131,7 +131,7 @@ int timecompare_offset(struct timecompare *sync, | |||
131 | 131 | ||
132 | return used; | 132 | return used; |
133 | } | 133 | } |
134 | EXPORT_SYMBOL(timecompare_offset); | 134 | EXPORT_SYMBOL_GPL(timecompare_offset); |
135 | 135 | ||
136 | void __timecompare_update(struct timecompare *sync, | 136 | void __timecompare_update(struct timecompare *sync, |
137 | u64 source_tstamp) | 137 | u64 source_tstamp) |
@@ -188,4 +188,4 @@ void __timecompare_update(struct timecompare *sync, | |||
188 | } | 188 | } |
189 | } | 189 | } |
190 | } | 190 | } |
191 | EXPORT_SYMBOL(__timecompare_update); | 191 | EXPORT_SYMBOL_GPL(__timecompare_update); |
diff --git a/kernel/time/timekeeping.c b/kernel/time/timekeeping.c index c3a4e2907eaa..af4135f05825 100644 --- a/kernel/time/timekeeping.c +++ b/kernel/time/timekeeping.c | |||
@@ -165,19 +165,12 @@ struct timespec raw_time; | |||
165 | /* flag for if timekeeping is suspended */ | 165 | /* flag for if timekeeping is suspended */ |
166 | int __read_mostly timekeeping_suspended; | 166 | int __read_mostly timekeeping_suspended; |
167 | 167 | ||
168 | static struct timespec xtime_cache __attribute__ ((aligned (16))); | ||
169 | void update_xtime_cache(u64 nsec) | ||
170 | { | ||
171 | xtime_cache = xtime; | ||
172 | timespec_add_ns(&xtime_cache, nsec); | ||
173 | } | ||
174 | |||
175 | /* must hold xtime_lock */ | 168 | /* must hold xtime_lock */ |
176 | void timekeeping_leap_insert(int leapsecond) | 169 | void timekeeping_leap_insert(int leapsecond) |
177 | { | 170 | { |
178 | xtime.tv_sec += leapsecond; | 171 | xtime.tv_sec += leapsecond; |
179 | wall_to_monotonic.tv_sec -= leapsecond; | 172 | wall_to_monotonic.tv_sec -= leapsecond; |
180 | update_vsyscall(&xtime, timekeeper.clock); | 173 | update_vsyscall(&xtime, timekeeper.clock, timekeeper.mult); |
181 | } | 174 | } |
182 | 175 | ||
183 | #ifdef CONFIG_GENERIC_TIME | 176 | #ifdef CONFIG_GENERIC_TIME |
@@ -332,12 +325,10 @@ int do_settimeofday(struct timespec *tv) | |||
332 | 325 | ||
333 | xtime = *tv; | 326 | xtime = *tv; |
334 | 327 | ||
335 | update_xtime_cache(0); | ||
336 | |||
337 | timekeeper.ntp_error = 0; | 328 | timekeeper.ntp_error = 0; |
338 | ntp_clear(); | 329 | ntp_clear(); |
339 | 330 | ||
340 | update_vsyscall(&xtime, timekeeper.clock); | 331 | update_vsyscall(&xtime, timekeeper.clock, timekeeper.mult); |
341 | 332 | ||
342 | write_sequnlock_irqrestore(&xtime_lock, flags); | 333 | write_sequnlock_irqrestore(&xtime_lock, flags); |
343 | 334 | ||
@@ -488,6 +479,17 @@ int timekeeping_valid_for_hres(void) | |||
488 | } | 479 | } |
489 | 480 | ||
490 | /** | 481 | /** |
482 | * timekeeping_max_deferment - Returns max time the clocksource can be deferred | ||
483 | * | ||
484 | * Caller must observe xtime_lock via read_seqbegin/read_seqretry to | ||
485 | * ensure that the clocksource does not change! | ||
486 | */ | ||
487 | u64 timekeeping_max_deferment(void) | ||
488 | { | ||
489 | return timekeeper.clock->max_idle_ns; | ||
490 | } | ||
491 | |||
492 | /** | ||
491 | * read_persistent_clock - Return time from the persistent clock. | 493 | * read_persistent_clock - Return time from the persistent clock. |
492 | * | 494 | * |
493 | * Weak dummy function for arches that do not yet support it. | 495 | * Weak dummy function for arches that do not yet support it. |
@@ -548,7 +550,6 @@ void __init timekeeping_init(void) | |||
548 | } | 550 | } |
549 | set_normalized_timespec(&wall_to_monotonic, | 551 | set_normalized_timespec(&wall_to_monotonic, |
550 | -boot.tv_sec, -boot.tv_nsec); | 552 | -boot.tv_sec, -boot.tv_nsec); |
551 | update_xtime_cache(0); | ||
552 | total_sleep_time.tv_sec = 0; | 553 | total_sleep_time.tv_sec = 0; |
553 | total_sleep_time.tv_nsec = 0; | 554 | total_sleep_time.tv_nsec = 0; |
554 | write_sequnlock_irqrestore(&xtime_lock, flags); | 555 | write_sequnlock_irqrestore(&xtime_lock, flags); |
@@ -582,7 +583,6 @@ static int timekeeping_resume(struct sys_device *dev) | |||
582 | wall_to_monotonic = timespec_sub(wall_to_monotonic, ts); | 583 | wall_to_monotonic = timespec_sub(wall_to_monotonic, ts); |
583 | total_sleep_time = timespec_add_safe(total_sleep_time, ts); | 584 | total_sleep_time = timespec_add_safe(total_sleep_time, ts); |
584 | } | 585 | } |
585 | update_xtime_cache(0); | ||
586 | /* re-base the last cycle value */ | 586 | /* re-base the last cycle value */ |
587 | timekeeper.clock->cycle_last = timekeeper.clock->read(timekeeper.clock); | 587 | timekeeper.clock->cycle_last = timekeeper.clock->read(timekeeper.clock); |
588 | timekeeper.ntp_error = 0; | 588 | timekeeper.ntp_error = 0; |
@@ -723,6 +723,49 @@ static void timekeeping_adjust(s64 offset) | |||
723 | } | 723 | } |
724 | 724 | ||
725 | /** | 725 | /** |
726 | * logarithmic_accumulation - shifted accumulation of cycles | ||
727 | * | ||
728 | * This functions accumulates a shifted interval of cycles into | ||
729 | * into a shifted interval nanoseconds. Allows for O(log) accumulation | ||
730 | * loop. | ||
731 | * | ||
732 | * Returns the unconsumed cycles. | ||
733 | */ | ||
734 | static cycle_t logarithmic_accumulation(cycle_t offset, int shift) | ||
735 | { | ||
736 | u64 nsecps = (u64)NSEC_PER_SEC << timekeeper.shift; | ||
737 | |||
738 | /* If the offset is smaller then a shifted interval, do nothing */ | ||
739 | if (offset < timekeeper.cycle_interval<<shift) | ||
740 | return offset; | ||
741 | |||
742 | /* Accumulate one shifted interval */ | ||
743 | offset -= timekeeper.cycle_interval << shift; | ||
744 | timekeeper.clock->cycle_last += timekeeper.cycle_interval << shift; | ||
745 | |||
746 | timekeeper.xtime_nsec += timekeeper.xtime_interval << shift; | ||
747 | while (timekeeper.xtime_nsec >= nsecps) { | ||
748 | timekeeper.xtime_nsec -= nsecps; | ||
749 | xtime.tv_sec++; | ||
750 | second_overflow(); | ||
751 | } | ||
752 | |||
753 | /* Accumulate into raw time */ | ||
754 | raw_time.tv_nsec += timekeeper.raw_interval << shift;; | ||
755 | while (raw_time.tv_nsec >= NSEC_PER_SEC) { | ||
756 | raw_time.tv_nsec -= NSEC_PER_SEC; | ||
757 | raw_time.tv_sec++; | ||
758 | } | ||
759 | |||
760 | /* Accumulate error between NTP and clock interval */ | ||
761 | timekeeper.ntp_error += tick_length << shift; | ||
762 | timekeeper.ntp_error -= timekeeper.xtime_interval << | ||
763 | (timekeeper.ntp_error_shift + shift); | ||
764 | |||
765 | return offset; | ||
766 | } | ||
767 | |||
768 | /** | ||
726 | * update_wall_time - Uses the current clocksource to increment the wall time | 769 | * update_wall_time - Uses the current clocksource to increment the wall time |
727 | * | 770 | * |
728 | * Called from the timer interrupt, must hold a write on xtime_lock. | 771 | * Called from the timer interrupt, must hold a write on xtime_lock. |
@@ -731,7 +774,7 @@ void update_wall_time(void) | |||
731 | { | 774 | { |
732 | struct clocksource *clock; | 775 | struct clocksource *clock; |
733 | cycle_t offset; | 776 | cycle_t offset; |
734 | u64 nsecs; | 777 | int shift = 0, maxshift; |
735 | 778 | ||
736 | /* Make sure we're fully resumed: */ | 779 | /* Make sure we're fully resumed: */ |
737 | if (unlikely(timekeeping_suspended)) | 780 | if (unlikely(timekeeping_suspended)) |
@@ -745,33 +788,22 @@ void update_wall_time(void) | |||
745 | #endif | 788 | #endif |
746 | timekeeper.xtime_nsec = (s64)xtime.tv_nsec << timekeeper.shift; | 789 | timekeeper.xtime_nsec = (s64)xtime.tv_nsec << timekeeper.shift; |
747 | 790 | ||
748 | /* normally this loop will run just once, however in the | 791 | /* |
749 | * case of lost or late ticks, it will accumulate correctly. | 792 | * With NO_HZ we may have to accumulate many cycle_intervals |
793 | * (think "ticks") worth of time at once. To do this efficiently, | ||
794 | * we calculate the largest doubling multiple of cycle_intervals | ||
795 | * that is smaller then the offset. We then accumulate that | ||
796 | * chunk in one go, and then try to consume the next smaller | ||
797 | * doubled multiple. | ||
750 | */ | 798 | */ |
799 | shift = ilog2(offset) - ilog2(timekeeper.cycle_interval); | ||
800 | shift = max(0, shift); | ||
801 | /* Bound shift to one less then what overflows tick_length */ | ||
802 | maxshift = (8*sizeof(tick_length) - (ilog2(tick_length)+1)) - 1; | ||
803 | shift = min(shift, maxshift); | ||
751 | while (offset >= timekeeper.cycle_interval) { | 804 | while (offset >= timekeeper.cycle_interval) { |
752 | u64 nsecps = (u64)NSEC_PER_SEC << timekeeper.shift; | 805 | offset = logarithmic_accumulation(offset, shift); |
753 | 806 | shift--; | |
754 | /* accumulate one interval */ | ||
755 | offset -= timekeeper.cycle_interval; | ||
756 | clock->cycle_last += timekeeper.cycle_interval; | ||
757 | |||
758 | timekeeper.xtime_nsec += timekeeper.xtime_interval; | ||
759 | if (timekeeper.xtime_nsec >= nsecps) { | ||
760 | timekeeper.xtime_nsec -= nsecps; | ||
761 | xtime.tv_sec++; | ||
762 | second_overflow(); | ||
763 | } | ||
764 | |||
765 | raw_time.tv_nsec += timekeeper.raw_interval; | ||
766 | if (raw_time.tv_nsec >= NSEC_PER_SEC) { | ||
767 | raw_time.tv_nsec -= NSEC_PER_SEC; | ||
768 | raw_time.tv_sec++; | ||
769 | } | ||
770 | |||
771 | /* accumulate error between NTP and clock interval */ | ||
772 | timekeeper.ntp_error += tick_length; | ||
773 | timekeeper.ntp_error -= timekeeper.xtime_interval << | ||
774 | timekeeper.ntp_error_shift; | ||
775 | } | 807 | } |
776 | 808 | ||
777 | /* correct the clock when NTP error is too big */ | 809 | /* correct the clock when NTP error is too big */ |
@@ -807,11 +839,8 @@ void update_wall_time(void) | |||
807 | timekeeper.ntp_error += timekeeper.xtime_nsec << | 839 | timekeeper.ntp_error += timekeeper.xtime_nsec << |
808 | timekeeper.ntp_error_shift; | 840 | timekeeper.ntp_error_shift; |
809 | 841 | ||
810 | nsecs = clocksource_cyc2ns(offset, timekeeper.mult, timekeeper.shift); | ||
811 | update_xtime_cache(nsecs); | ||
812 | |||
813 | /* check to see if there is a new clocksource to use */ | 842 | /* check to see if there is a new clocksource to use */ |
814 | update_vsyscall(&xtime, timekeeper.clock); | 843 | update_vsyscall(&xtime, timekeeper.clock, timekeeper.mult); |
815 | } | 844 | } |
816 | 845 | ||
817 | /** | 846 | /** |
@@ -846,13 +875,13 @@ void monotonic_to_bootbased(struct timespec *ts) | |||
846 | 875 | ||
847 | unsigned long get_seconds(void) | 876 | unsigned long get_seconds(void) |
848 | { | 877 | { |
849 | return xtime_cache.tv_sec; | 878 | return xtime.tv_sec; |
850 | } | 879 | } |
851 | EXPORT_SYMBOL(get_seconds); | 880 | EXPORT_SYMBOL(get_seconds); |
852 | 881 | ||
853 | struct timespec __current_kernel_time(void) | 882 | struct timespec __current_kernel_time(void) |
854 | { | 883 | { |
855 | return xtime_cache; | 884 | return xtime; |
856 | } | 885 | } |
857 | 886 | ||
858 | struct timespec current_kernel_time(void) | 887 | struct timespec current_kernel_time(void) |
@@ -862,8 +891,7 @@ struct timespec current_kernel_time(void) | |||
862 | 891 | ||
863 | do { | 892 | do { |
864 | seq = read_seqbegin(&xtime_lock); | 893 | seq = read_seqbegin(&xtime_lock); |
865 | 894 | now = xtime; | |
866 | now = xtime_cache; | ||
867 | } while (read_seqretry(&xtime_lock, seq)); | 895 | } while (read_seqretry(&xtime_lock, seq)); |
868 | 896 | ||
869 | return now; | 897 | return now; |
@@ -877,8 +905,7 @@ struct timespec get_monotonic_coarse(void) | |||
877 | 905 | ||
878 | do { | 906 | do { |
879 | seq = read_seqbegin(&xtime_lock); | 907 | seq = read_seqbegin(&xtime_lock); |
880 | 908 | now = xtime; | |
881 | now = xtime_cache; | ||
882 | mono = wall_to_monotonic; | 909 | mono = wall_to_monotonic; |
883 | } while (read_seqretry(&xtime_lock, seq)); | 910 | } while (read_seqretry(&xtime_lock, seq)); |
884 | 911 | ||
diff --git a/kernel/time/timer_list.c b/kernel/time/timer_list.c index 1b5b7aa2fdfd..665c76edbf17 100644 --- a/kernel/time/timer_list.c +++ b/kernel/time/timer_list.c | |||
@@ -204,10 +204,12 @@ print_tickdevice(struct seq_file *m, struct tick_device *td, int cpu) | |||
204 | return; | 204 | return; |
205 | } | 205 | } |
206 | SEQ_printf(m, "%s\n", dev->name); | 206 | SEQ_printf(m, "%s\n", dev->name); |
207 | SEQ_printf(m, " max_delta_ns: %lu\n", dev->max_delta_ns); | 207 | SEQ_printf(m, " max_delta_ns: %llu\n", |
208 | SEQ_printf(m, " min_delta_ns: %lu\n", dev->min_delta_ns); | 208 | (unsigned long long) dev->max_delta_ns); |
209 | SEQ_printf(m, " mult: %lu\n", dev->mult); | 209 | SEQ_printf(m, " min_delta_ns: %llu\n", |
210 | SEQ_printf(m, " shift: %d\n", dev->shift); | 210 | (unsigned long long) dev->min_delta_ns); |
211 | SEQ_printf(m, " mult: %u\n", dev->mult); | ||
212 | SEQ_printf(m, " shift: %u\n", dev->shift); | ||
211 | SEQ_printf(m, " mode: %d\n", dev->mode); | 213 | SEQ_printf(m, " mode: %d\n", dev->mode); |
212 | SEQ_printf(m, " next_event: %Ld nsecs\n", | 214 | SEQ_printf(m, " next_event: %Ld nsecs\n", |
213 | (unsigned long long) ktime_to_ns(dev->next_event)); | 215 | (unsigned long long) ktime_to_ns(dev->next_event)); |
diff --git a/kernel/trace/Kconfig b/kernel/trace/Kconfig index b416512ad17f..d006554888dc 100644 --- a/kernel/trace/Kconfig +++ b/kernel/trace/Kconfig | |||
@@ -339,6 +339,27 @@ config POWER_TRACER | |||
339 | power management decisions, specifically the C-state and P-state | 339 | power management decisions, specifically the C-state and P-state |
340 | behavior. | 340 | behavior. |
341 | 341 | ||
342 | config KSYM_TRACER | ||
343 | bool "Trace read and write access on kernel memory locations" | ||
344 | depends on HAVE_HW_BREAKPOINT | ||
345 | select TRACING | ||
346 | help | ||
347 | This tracer helps find read and write operations on any given kernel | ||
348 | symbol i.e. /proc/kallsyms. | ||
349 | |||
350 | config PROFILE_KSYM_TRACER | ||
351 | bool "Profile all kernel memory accesses on 'watched' variables" | ||
352 | depends on KSYM_TRACER | ||
353 | help | ||
354 | This tracer profiles kernel accesses on variables watched through the | ||
355 | ksym tracer ftrace plugin. Depending upon the hardware, all read | ||
356 | and write operations on kernel variables can be monitored for | ||
357 | accesses. | ||
358 | |||
359 | The results will be displayed in: | ||
360 | /debugfs/tracing/profile_ksym | ||
361 | |||
362 | Say N if unsure. | ||
342 | 363 | ||
343 | config STACK_TRACER | 364 | config STACK_TRACER |
344 | bool "Trace max stack" | 365 | bool "Trace max stack" |
@@ -428,6 +449,23 @@ config BLK_DEV_IO_TRACE | |||
428 | 449 | ||
429 | If unsure, say N. | 450 | If unsure, say N. |
430 | 451 | ||
452 | config KPROBE_EVENT | ||
453 | depends on KPROBES | ||
454 | depends on X86 | ||
455 | bool "Enable kprobes-based dynamic events" | ||
456 | select TRACING | ||
457 | default y | ||
458 | help | ||
459 | This allows the user to add tracing events (similar to tracepoints) on the fly | ||
460 | via the ftrace interface. See Documentation/trace/kprobetrace.txt | ||
461 | for more details. | ||
462 | |||
463 | Those events can be inserted wherever kprobes can probe, and record | ||
464 | various register and memory values. | ||
465 | |||
466 | This option is also required by perf-probe subcommand of perf tools. If | ||
467 | you want to use perf tools, this option is strongly recommended. | ||
468 | |||
431 | config DYNAMIC_FTRACE | 469 | config DYNAMIC_FTRACE |
432 | bool "enable/disable ftrace tracepoints dynamically" | 470 | bool "enable/disable ftrace tracepoints dynamically" |
433 | depends on FUNCTION_TRACER | 471 | depends on FUNCTION_TRACER |
diff --git a/kernel/trace/Makefile b/kernel/trace/Makefile index 26f03ac07c2b..cd9ecd89ec77 100644 --- a/kernel/trace/Makefile +++ b/kernel/trace/Makefile | |||
@@ -53,6 +53,8 @@ obj-$(CONFIG_EVENT_TRACING) += trace_export.o | |||
53 | obj-$(CONFIG_FTRACE_SYSCALLS) += trace_syscalls.o | 53 | obj-$(CONFIG_FTRACE_SYSCALLS) += trace_syscalls.o |
54 | obj-$(CONFIG_EVENT_PROFILE) += trace_event_profile.o | 54 | obj-$(CONFIG_EVENT_PROFILE) += trace_event_profile.o |
55 | obj-$(CONFIG_EVENT_TRACING) += trace_events_filter.o | 55 | obj-$(CONFIG_EVENT_TRACING) += trace_events_filter.o |
56 | obj-$(CONFIG_KPROBE_EVENT) += trace_kprobe.o | ||
57 | obj-$(CONFIG_KSYM_TRACER) += trace_ksym.o | ||
56 | obj-$(CONFIG_EVENT_TRACING) += power-traces.o | 58 | obj-$(CONFIG_EVENT_TRACING) += power-traces.o |
57 | 59 | ||
58 | libftrace-y := ftrace.o | 60 | libftrace-y := ftrace.o |
diff --git a/kernel/trace/ftrace.c b/kernel/trace/ftrace.c index 37ba67e33265..e51a1bcb7bed 100644 --- a/kernel/trace/ftrace.c +++ b/kernel/trace/ftrace.c | |||
@@ -60,6 +60,13 @@ static int last_ftrace_enabled; | |||
60 | /* Quick disabling of function tracer. */ | 60 | /* Quick disabling of function tracer. */ |
61 | int function_trace_stop; | 61 | int function_trace_stop; |
62 | 62 | ||
63 | /* List for set_ftrace_pid's pids. */ | ||
64 | LIST_HEAD(ftrace_pids); | ||
65 | struct ftrace_pid { | ||
66 | struct list_head list; | ||
67 | struct pid *pid; | ||
68 | }; | ||
69 | |||
63 | /* | 70 | /* |
64 | * ftrace_disabled is set when an anomaly is discovered. | 71 | * ftrace_disabled is set when an anomaly is discovered. |
65 | * ftrace_disabled is much stronger than ftrace_enabled. | 72 | * ftrace_disabled is much stronger than ftrace_enabled. |
@@ -78,6 +85,10 @@ ftrace_func_t ftrace_trace_function __read_mostly = ftrace_stub; | |||
78 | ftrace_func_t __ftrace_trace_function __read_mostly = ftrace_stub; | 85 | ftrace_func_t __ftrace_trace_function __read_mostly = ftrace_stub; |
79 | ftrace_func_t ftrace_pid_function __read_mostly = ftrace_stub; | 86 | ftrace_func_t ftrace_pid_function __read_mostly = ftrace_stub; |
80 | 87 | ||
88 | #ifdef CONFIG_FUNCTION_GRAPH_TRACER | ||
89 | static int ftrace_set_func(unsigned long *array, int *idx, char *buffer); | ||
90 | #endif | ||
91 | |||
81 | static void ftrace_list_func(unsigned long ip, unsigned long parent_ip) | 92 | static void ftrace_list_func(unsigned long ip, unsigned long parent_ip) |
82 | { | 93 | { |
83 | struct ftrace_ops *op = ftrace_list; | 94 | struct ftrace_ops *op = ftrace_list; |
@@ -155,7 +166,7 @@ static int __register_ftrace_function(struct ftrace_ops *ops) | |||
155 | else | 166 | else |
156 | func = ftrace_list_func; | 167 | func = ftrace_list_func; |
157 | 168 | ||
158 | if (ftrace_pid_trace) { | 169 | if (!list_empty(&ftrace_pids)) { |
159 | set_ftrace_pid_function(func); | 170 | set_ftrace_pid_function(func); |
160 | func = ftrace_pid_func; | 171 | func = ftrace_pid_func; |
161 | } | 172 | } |
@@ -203,7 +214,7 @@ static int __unregister_ftrace_function(struct ftrace_ops *ops) | |||
203 | if (ftrace_list->next == &ftrace_list_end) { | 214 | if (ftrace_list->next == &ftrace_list_end) { |
204 | ftrace_func_t func = ftrace_list->func; | 215 | ftrace_func_t func = ftrace_list->func; |
205 | 216 | ||
206 | if (ftrace_pid_trace) { | 217 | if (!list_empty(&ftrace_pids)) { |
207 | set_ftrace_pid_function(func); | 218 | set_ftrace_pid_function(func); |
208 | func = ftrace_pid_func; | 219 | func = ftrace_pid_func; |
209 | } | 220 | } |
@@ -231,7 +242,7 @@ static void ftrace_update_pid_func(void) | |||
231 | func = __ftrace_trace_function; | 242 | func = __ftrace_trace_function; |
232 | #endif | 243 | #endif |
233 | 244 | ||
234 | if (ftrace_pid_trace) { | 245 | if (!list_empty(&ftrace_pids)) { |
235 | set_ftrace_pid_function(func); | 246 | set_ftrace_pid_function(func); |
236 | func = ftrace_pid_func; | 247 | func = ftrace_pid_func; |
237 | } else { | 248 | } else { |
@@ -740,7 +751,7 @@ ftrace_profile_write(struct file *filp, const char __user *ubuf, | |||
740 | out: | 751 | out: |
741 | mutex_unlock(&ftrace_profile_lock); | 752 | mutex_unlock(&ftrace_profile_lock); |
742 | 753 | ||
743 | filp->f_pos += cnt; | 754 | *ppos += cnt; |
744 | 755 | ||
745 | return cnt; | 756 | return cnt; |
746 | } | 757 | } |
@@ -821,8 +832,6 @@ static __init void ftrace_profile_debugfs(struct dentry *d_tracer) | |||
821 | } | 832 | } |
822 | #endif /* CONFIG_FUNCTION_PROFILER */ | 833 | #endif /* CONFIG_FUNCTION_PROFILER */ |
823 | 834 | ||
824 | /* set when tracing only a pid */ | ||
825 | struct pid *ftrace_pid_trace; | ||
826 | static struct pid * const ftrace_swapper_pid = &init_struct_pid; | 835 | static struct pid * const ftrace_swapper_pid = &init_struct_pid; |
827 | 836 | ||
828 | #ifdef CONFIG_DYNAMIC_FTRACE | 837 | #ifdef CONFIG_DYNAMIC_FTRACE |
@@ -1261,12 +1270,34 @@ static int ftrace_update_code(struct module *mod) | |||
1261 | ftrace_new_addrs = p->newlist; | 1270 | ftrace_new_addrs = p->newlist; |
1262 | p->flags = 0L; | 1271 | p->flags = 0L; |
1263 | 1272 | ||
1264 | /* convert record (i.e, patch mcount-call with NOP) */ | 1273 | /* |
1265 | if (ftrace_code_disable(mod, p)) { | 1274 | * Do the initial record convertion from mcount jump |
1266 | p->flags |= FTRACE_FL_CONVERTED; | 1275 | * to the NOP instructions. |
1267 | ftrace_update_cnt++; | 1276 | */ |
1268 | } else | 1277 | if (!ftrace_code_disable(mod, p)) { |
1269 | ftrace_free_rec(p); | 1278 | ftrace_free_rec(p); |
1279 | continue; | ||
1280 | } | ||
1281 | |||
1282 | p->flags |= FTRACE_FL_CONVERTED; | ||
1283 | ftrace_update_cnt++; | ||
1284 | |||
1285 | /* | ||
1286 | * If the tracing is enabled, go ahead and enable the record. | ||
1287 | * | ||
1288 | * The reason not to enable the record immediatelly is the | ||
1289 | * inherent check of ftrace_make_nop/ftrace_make_call for | ||
1290 | * correct previous instructions. Making first the NOP | ||
1291 | * conversion puts the module to the correct state, thus | ||
1292 | * passing the ftrace_make_call check. | ||
1293 | */ | ||
1294 | if (ftrace_start_up) { | ||
1295 | int failed = __ftrace_replace_code(p, 1); | ||
1296 | if (failed) { | ||
1297 | ftrace_bug(failed, p->ip); | ||
1298 | ftrace_free_rec(p); | ||
1299 | } | ||
1300 | } | ||
1270 | } | 1301 | } |
1271 | 1302 | ||
1272 | stop = ftrace_now(raw_smp_processor_id()); | 1303 | stop = ftrace_now(raw_smp_processor_id()); |
@@ -1656,60 +1687,6 @@ ftrace_regex_lseek(struct file *file, loff_t offset, int origin) | |||
1656 | return ret; | 1687 | return ret; |
1657 | } | 1688 | } |
1658 | 1689 | ||
1659 | enum { | ||
1660 | MATCH_FULL, | ||
1661 | MATCH_FRONT_ONLY, | ||
1662 | MATCH_MIDDLE_ONLY, | ||
1663 | MATCH_END_ONLY, | ||
1664 | }; | ||
1665 | |||
1666 | /* | ||
1667 | * (static function - no need for kernel doc) | ||
1668 | * | ||
1669 | * Pass in a buffer containing a glob and this function will | ||
1670 | * set search to point to the search part of the buffer and | ||
1671 | * return the type of search it is (see enum above). | ||
1672 | * This does modify buff. | ||
1673 | * | ||
1674 | * Returns enum type. | ||
1675 | * search returns the pointer to use for comparison. | ||
1676 | * not returns 1 if buff started with a '!' | ||
1677 | * 0 otherwise. | ||
1678 | */ | ||
1679 | static int | ||
1680 | ftrace_setup_glob(char *buff, int len, char **search, int *not) | ||
1681 | { | ||
1682 | int type = MATCH_FULL; | ||
1683 | int i; | ||
1684 | |||
1685 | if (buff[0] == '!') { | ||
1686 | *not = 1; | ||
1687 | buff++; | ||
1688 | len--; | ||
1689 | } else | ||
1690 | *not = 0; | ||
1691 | |||
1692 | *search = buff; | ||
1693 | |||
1694 | for (i = 0; i < len; i++) { | ||
1695 | if (buff[i] == '*') { | ||
1696 | if (!i) { | ||
1697 | *search = buff + 1; | ||
1698 | type = MATCH_END_ONLY; | ||
1699 | } else { | ||
1700 | if (type == MATCH_END_ONLY) | ||
1701 | type = MATCH_MIDDLE_ONLY; | ||
1702 | else | ||
1703 | type = MATCH_FRONT_ONLY; | ||
1704 | buff[i] = 0; | ||
1705 | break; | ||
1706 | } | ||
1707 | } | ||
1708 | } | ||
1709 | |||
1710 | return type; | ||
1711 | } | ||
1712 | |||
1713 | static int ftrace_match(char *str, char *regex, int len, int type) | 1690 | static int ftrace_match(char *str, char *regex, int len, int type) |
1714 | { | 1691 | { |
1715 | int matched = 0; | 1692 | int matched = 0; |
@@ -1758,7 +1735,7 @@ static void ftrace_match_records(char *buff, int len, int enable) | |||
1758 | int not; | 1735 | int not; |
1759 | 1736 | ||
1760 | flag = enable ? FTRACE_FL_FILTER : FTRACE_FL_NOTRACE; | 1737 | flag = enable ? FTRACE_FL_FILTER : FTRACE_FL_NOTRACE; |
1761 | type = ftrace_setup_glob(buff, len, &search, ¬); | 1738 | type = filter_parse_regex(buff, len, &search, ¬); |
1762 | 1739 | ||
1763 | search_len = strlen(search); | 1740 | search_len = strlen(search); |
1764 | 1741 | ||
@@ -1826,7 +1803,7 @@ static void ftrace_match_module_records(char *buff, char *mod, int enable) | |||
1826 | } | 1803 | } |
1827 | 1804 | ||
1828 | if (strlen(buff)) { | 1805 | if (strlen(buff)) { |
1829 | type = ftrace_setup_glob(buff, strlen(buff), &search, ¬); | 1806 | type = filter_parse_regex(buff, strlen(buff), &search, ¬); |
1830 | search_len = strlen(search); | 1807 | search_len = strlen(search); |
1831 | } | 1808 | } |
1832 | 1809 | ||
@@ -1991,7 +1968,7 @@ register_ftrace_function_probe(char *glob, struct ftrace_probe_ops *ops, | |||
1991 | int count = 0; | 1968 | int count = 0; |
1992 | char *search; | 1969 | char *search; |
1993 | 1970 | ||
1994 | type = ftrace_setup_glob(glob, strlen(glob), &search, ¬); | 1971 | type = filter_parse_regex(glob, strlen(glob), &search, ¬); |
1995 | len = strlen(search); | 1972 | len = strlen(search); |
1996 | 1973 | ||
1997 | /* we do not support '!' for function probes */ | 1974 | /* we do not support '!' for function probes */ |
@@ -2068,7 +2045,7 @@ __unregister_ftrace_function_probe(char *glob, struct ftrace_probe_ops *ops, | |||
2068 | else if (glob) { | 2045 | else if (glob) { |
2069 | int not; | 2046 | int not; |
2070 | 2047 | ||
2071 | type = ftrace_setup_glob(glob, strlen(glob), &search, ¬); | 2048 | type = filter_parse_regex(glob, strlen(glob), &search, ¬); |
2072 | len = strlen(search); | 2049 | len = strlen(search); |
2073 | 2050 | ||
2074 | /* we do not support '!' for function probes */ | 2051 | /* we do not support '!' for function probes */ |
@@ -2222,15 +2199,15 @@ ftrace_regex_write(struct file *file, const char __user *ubuf, | |||
2222 | ret = ftrace_process_regex(parser->buffer, | 2199 | ret = ftrace_process_regex(parser->buffer, |
2223 | parser->idx, enable); | 2200 | parser->idx, enable); |
2224 | if (ret) | 2201 | if (ret) |
2225 | goto out; | 2202 | goto out_unlock; |
2226 | 2203 | ||
2227 | trace_parser_clear(parser); | 2204 | trace_parser_clear(parser); |
2228 | } | 2205 | } |
2229 | 2206 | ||
2230 | ret = read; | 2207 | ret = read; |
2231 | 2208 | out_unlock: | |
2232 | mutex_unlock(&ftrace_regex_lock); | 2209 | mutex_unlock(&ftrace_regex_lock); |
2233 | out: | 2210 | |
2234 | return ret; | 2211 | return ret; |
2235 | } | 2212 | } |
2236 | 2213 | ||
@@ -2312,6 +2289,32 @@ static int __init set_ftrace_filter(char *str) | |||
2312 | } | 2289 | } |
2313 | __setup("ftrace_filter=", set_ftrace_filter); | 2290 | __setup("ftrace_filter=", set_ftrace_filter); |
2314 | 2291 | ||
2292 | #ifdef CONFIG_FUNCTION_GRAPH_TRACER | ||
2293 | static char ftrace_graph_buf[FTRACE_FILTER_SIZE] __initdata; | ||
2294 | static int __init set_graph_function(char *str) | ||
2295 | { | ||
2296 | strlcpy(ftrace_graph_buf, str, FTRACE_FILTER_SIZE); | ||
2297 | return 1; | ||
2298 | } | ||
2299 | __setup("ftrace_graph_filter=", set_graph_function); | ||
2300 | |||
2301 | static void __init set_ftrace_early_graph(char *buf) | ||
2302 | { | ||
2303 | int ret; | ||
2304 | char *func; | ||
2305 | |||
2306 | while (buf) { | ||
2307 | func = strsep(&buf, ","); | ||
2308 | /* we allow only one expression at a time */ | ||
2309 | ret = ftrace_set_func(ftrace_graph_funcs, &ftrace_graph_count, | ||
2310 | func); | ||
2311 | if (ret) | ||
2312 | printk(KERN_DEBUG "ftrace: function %s not " | ||
2313 | "traceable\n", func); | ||
2314 | } | ||
2315 | } | ||
2316 | #endif /* CONFIG_FUNCTION_GRAPH_TRACER */ | ||
2317 | |||
2315 | static void __init set_ftrace_early_filter(char *buf, int enable) | 2318 | static void __init set_ftrace_early_filter(char *buf, int enable) |
2316 | { | 2319 | { |
2317 | char *func; | 2320 | char *func; |
@@ -2328,6 +2331,10 @@ static void __init set_ftrace_early_filters(void) | |||
2328 | set_ftrace_early_filter(ftrace_filter_buf, 1); | 2331 | set_ftrace_early_filter(ftrace_filter_buf, 1); |
2329 | if (ftrace_notrace_buf[0]) | 2332 | if (ftrace_notrace_buf[0]) |
2330 | set_ftrace_early_filter(ftrace_notrace_buf, 0); | 2333 | set_ftrace_early_filter(ftrace_notrace_buf, 0); |
2334 | #ifdef CONFIG_FUNCTION_GRAPH_TRACER | ||
2335 | if (ftrace_graph_buf[0]) | ||
2336 | set_ftrace_early_graph(ftrace_graph_buf); | ||
2337 | #endif /* CONFIG_FUNCTION_GRAPH_TRACER */ | ||
2331 | } | 2338 | } |
2332 | 2339 | ||
2333 | static int | 2340 | static int |
@@ -2513,7 +2520,7 @@ ftrace_set_func(unsigned long *array, int *idx, char *buffer) | |||
2513 | return -ENODEV; | 2520 | return -ENODEV; |
2514 | 2521 | ||
2515 | /* decode regex */ | 2522 | /* decode regex */ |
2516 | type = ftrace_setup_glob(buffer, strlen(buffer), &search, ¬); | 2523 | type = filter_parse_regex(buffer, strlen(buffer), &search, ¬); |
2517 | if (not) | 2524 | if (not) |
2518 | return -EINVAL; | 2525 | return -EINVAL; |
2519 | 2526 | ||
@@ -2624,7 +2631,7 @@ static __init int ftrace_init_dyn_debugfs(struct dentry *d_tracer) | |||
2624 | return 0; | 2631 | return 0; |
2625 | } | 2632 | } |
2626 | 2633 | ||
2627 | static int ftrace_convert_nops(struct module *mod, | 2634 | static int ftrace_process_locs(struct module *mod, |
2628 | unsigned long *start, | 2635 | unsigned long *start, |
2629 | unsigned long *end) | 2636 | unsigned long *end) |
2630 | { | 2637 | { |
@@ -2684,7 +2691,7 @@ static void ftrace_init_module(struct module *mod, | |||
2684 | { | 2691 | { |
2685 | if (ftrace_disabled || start == end) | 2692 | if (ftrace_disabled || start == end) |
2686 | return; | 2693 | return; |
2687 | ftrace_convert_nops(mod, start, end); | 2694 | ftrace_process_locs(mod, start, end); |
2688 | } | 2695 | } |
2689 | 2696 | ||
2690 | static int ftrace_module_notify(struct notifier_block *self, | 2697 | static int ftrace_module_notify(struct notifier_block *self, |
@@ -2745,7 +2752,7 @@ void __init ftrace_init(void) | |||
2745 | 2752 | ||
2746 | last_ftrace_enabled = ftrace_enabled = 1; | 2753 | last_ftrace_enabled = ftrace_enabled = 1; |
2747 | 2754 | ||
2748 | ret = ftrace_convert_nops(NULL, | 2755 | ret = ftrace_process_locs(NULL, |
2749 | __start_mcount_loc, | 2756 | __start_mcount_loc, |
2750 | __stop_mcount_loc); | 2757 | __stop_mcount_loc); |
2751 | 2758 | ||
@@ -2778,23 +2785,6 @@ static inline void ftrace_startup_enable(int command) { } | |||
2778 | # define ftrace_shutdown_sysctl() do { } while (0) | 2785 | # define ftrace_shutdown_sysctl() do { } while (0) |
2779 | #endif /* CONFIG_DYNAMIC_FTRACE */ | 2786 | #endif /* CONFIG_DYNAMIC_FTRACE */ |
2780 | 2787 | ||
2781 | static ssize_t | ||
2782 | ftrace_pid_read(struct file *file, char __user *ubuf, | ||
2783 | size_t cnt, loff_t *ppos) | ||
2784 | { | ||
2785 | char buf[64]; | ||
2786 | int r; | ||
2787 | |||
2788 | if (ftrace_pid_trace == ftrace_swapper_pid) | ||
2789 | r = sprintf(buf, "swapper tasks\n"); | ||
2790 | else if (ftrace_pid_trace) | ||
2791 | r = sprintf(buf, "%u\n", pid_vnr(ftrace_pid_trace)); | ||
2792 | else | ||
2793 | r = sprintf(buf, "no pid\n"); | ||
2794 | |||
2795 | return simple_read_from_buffer(ubuf, cnt, ppos, buf, r); | ||
2796 | } | ||
2797 | |||
2798 | static void clear_ftrace_swapper(void) | 2788 | static void clear_ftrace_swapper(void) |
2799 | { | 2789 | { |
2800 | struct task_struct *p; | 2790 | struct task_struct *p; |
@@ -2845,14 +2835,12 @@ static void set_ftrace_pid(struct pid *pid) | |||
2845 | rcu_read_unlock(); | 2835 | rcu_read_unlock(); |
2846 | } | 2836 | } |
2847 | 2837 | ||
2848 | static void clear_ftrace_pid_task(struct pid **pid) | 2838 | static void clear_ftrace_pid_task(struct pid *pid) |
2849 | { | 2839 | { |
2850 | if (*pid == ftrace_swapper_pid) | 2840 | if (pid == ftrace_swapper_pid) |
2851 | clear_ftrace_swapper(); | 2841 | clear_ftrace_swapper(); |
2852 | else | 2842 | else |
2853 | clear_ftrace_pid(*pid); | 2843 | clear_ftrace_pid(pid); |
2854 | |||
2855 | *pid = NULL; | ||
2856 | } | 2844 | } |
2857 | 2845 | ||
2858 | static void set_ftrace_pid_task(struct pid *pid) | 2846 | static void set_ftrace_pid_task(struct pid *pid) |
@@ -2863,74 +2851,184 @@ static void set_ftrace_pid_task(struct pid *pid) | |||
2863 | set_ftrace_pid(pid); | 2851 | set_ftrace_pid(pid); |
2864 | } | 2852 | } |
2865 | 2853 | ||
2866 | static ssize_t | 2854 | static int ftrace_pid_add(int p) |
2867 | ftrace_pid_write(struct file *filp, const char __user *ubuf, | ||
2868 | size_t cnt, loff_t *ppos) | ||
2869 | { | 2855 | { |
2870 | struct pid *pid; | 2856 | struct pid *pid; |
2871 | char buf[64]; | 2857 | struct ftrace_pid *fpid; |
2872 | long val; | 2858 | int ret = -EINVAL; |
2873 | int ret; | ||
2874 | 2859 | ||
2875 | if (cnt >= sizeof(buf)) | 2860 | mutex_lock(&ftrace_lock); |
2876 | return -EINVAL; | ||
2877 | 2861 | ||
2878 | if (copy_from_user(&buf, ubuf, cnt)) | 2862 | if (!p) |
2879 | return -EFAULT; | 2863 | pid = ftrace_swapper_pid; |
2864 | else | ||
2865 | pid = find_get_pid(p); | ||
2880 | 2866 | ||
2881 | buf[cnt] = 0; | 2867 | if (!pid) |
2868 | goto out; | ||
2882 | 2869 | ||
2883 | ret = strict_strtol(buf, 10, &val); | 2870 | ret = 0; |
2884 | if (ret < 0) | ||
2885 | return ret; | ||
2886 | 2871 | ||
2887 | mutex_lock(&ftrace_lock); | 2872 | list_for_each_entry(fpid, &ftrace_pids, list) |
2888 | if (val < 0) { | 2873 | if (fpid->pid == pid) |
2889 | /* disable pid tracing */ | 2874 | goto out_put; |
2890 | if (!ftrace_pid_trace) | ||
2891 | goto out; | ||
2892 | 2875 | ||
2893 | clear_ftrace_pid_task(&ftrace_pid_trace); | 2876 | ret = -ENOMEM; |
2894 | 2877 | ||
2895 | } else { | 2878 | fpid = kmalloc(sizeof(*fpid), GFP_KERNEL); |
2896 | /* swapper task is special */ | 2879 | if (!fpid) |
2897 | if (!val) { | 2880 | goto out_put; |
2898 | pid = ftrace_swapper_pid; | ||
2899 | if (pid == ftrace_pid_trace) | ||
2900 | goto out; | ||
2901 | } else { | ||
2902 | pid = find_get_pid(val); | ||
2903 | 2881 | ||
2904 | if (pid == ftrace_pid_trace) { | 2882 | list_add(&fpid->list, &ftrace_pids); |
2905 | put_pid(pid); | 2883 | fpid->pid = pid; |
2906 | goto out; | ||
2907 | } | ||
2908 | } | ||
2909 | 2884 | ||
2910 | if (ftrace_pid_trace) | 2885 | set_ftrace_pid_task(pid); |
2911 | clear_ftrace_pid_task(&ftrace_pid_trace); | ||
2912 | 2886 | ||
2913 | if (!pid) | 2887 | ftrace_update_pid_func(); |
2914 | goto out; | 2888 | ftrace_startup_enable(0); |
2889 | |||
2890 | mutex_unlock(&ftrace_lock); | ||
2891 | return 0; | ||
2892 | |||
2893 | out_put: | ||
2894 | if (pid != ftrace_swapper_pid) | ||
2895 | put_pid(pid); | ||
2896 | |||
2897 | out: | ||
2898 | mutex_unlock(&ftrace_lock); | ||
2899 | return ret; | ||
2900 | } | ||
2901 | |||
2902 | static void ftrace_pid_reset(void) | ||
2903 | { | ||
2904 | struct ftrace_pid *fpid, *safe; | ||
2905 | |||
2906 | mutex_lock(&ftrace_lock); | ||
2907 | list_for_each_entry_safe(fpid, safe, &ftrace_pids, list) { | ||
2908 | struct pid *pid = fpid->pid; | ||
2915 | 2909 | ||
2916 | ftrace_pid_trace = pid; | 2910 | clear_ftrace_pid_task(pid); |
2917 | 2911 | ||
2918 | set_ftrace_pid_task(ftrace_pid_trace); | 2912 | list_del(&fpid->list); |
2913 | kfree(fpid); | ||
2919 | } | 2914 | } |
2920 | 2915 | ||
2921 | /* update the function call */ | ||
2922 | ftrace_update_pid_func(); | 2916 | ftrace_update_pid_func(); |
2923 | ftrace_startup_enable(0); | 2917 | ftrace_startup_enable(0); |
2924 | 2918 | ||
2925 | out: | ||
2926 | mutex_unlock(&ftrace_lock); | 2919 | mutex_unlock(&ftrace_lock); |
2920 | } | ||
2927 | 2921 | ||
2928 | return cnt; | 2922 | static void *fpid_start(struct seq_file *m, loff_t *pos) |
2923 | { | ||
2924 | mutex_lock(&ftrace_lock); | ||
2925 | |||
2926 | if (list_empty(&ftrace_pids) && (!*pos)) | ||
2927 | return (void *) 1; | ||
2928 | |||
2929 | return seq_list_start(&ftrace_pids, *pos); | ||
2930 | } | ||
2931 | |||
2932 | static void *fpid_next(struct seq_file *m, void *v, loff_t *pos) | ||
2933 | { | ||
2934 | if (v == (void *)1) | ||
2935 | return NULL; | ||
2936 | |||
2937 | return seq_list_next(v, &ftrace_pids, pos); | ||
2938 | } | ||
2939 | |||
2940 | static void fpid_stop(struct seq_file *m, void *p) | ||
2941 | { | ||
2942 | mutex_unlock(&ftrace_lock); | ||
2943 | } | ||
2944 | |||
2945 | static int fpid_show(struct seq_file *m, void *v) | ||
2946 | { | ||
2947 | const struct ftrace_pid *fpid = list_entry(v, struct ftrace_pid, list); | ||
2948 | |||
2949 | if (v == (void *)1) { | ||
2950 | seq_printf(m, "no pid\n"); | ||
2951 | return 0; | ||
2952 | } | ||
2953 | |||
2954 | if (fpid->pid == ftrace_swapper_pid) | ||
2955 | seq_printf(m, "swapper tasks\n"); | ||
2956 | else | ||
2957 | seq_printf(m, "%u\n", pid_vnr(fpid->pid)); | ||
2958 | |||
2959 | return 0; | ||
2960 | } | ||
2961 | |||
2962 | static const struct seq_operations ftrace_pid_sops = { | ||
2963 | .start = fpid_start, | ||
2964 | .next = fpid_next, | ||
2965 | .stop = fpid_stop, | ||
2966 | .show = fpid_show, | ||
2967 | }; | ||
2968 | |||
2969 | static int | ||
2970 | ftrace_pid_open(struct inode *inode, struct file *file) | ||
2971 | { | ||
2972 | int ret = 0; | ||
2973 | |||
2974 | if ((file->f_mode & FMODE_WRITE) && | ||
2975 | (file->f_flags & O_TRUNC)) | ||
2976 | ftrace_pid_reset(); | ||
2977 | |||
2978 | if (file->f_mode & FMODE_READ) | ||
2979 | ret = seq_open(file, &ftrace_pid_sops); | ||
2980 | |||
2981 | return ret; | ||
2982 | } | ||
2983 | |||
2984 | static ssize_t | ||
2985 | ftrace_pid_write(struct file *filp, const char __user *ubuf, | ||
2986 | size_t cnt, loff_t *ppos) | ||
2987 | { | ||
2988 | char buf[64], *tmp; | ||
2989 | long val; | ||
2990 | int ret; | ||
2991 | |||
2992 | if (cnt >= sizeof(buf)) | ||
2993 | return -EINVAL; | ||
2994 | |||
2995 | if (copy_from_user(&buf, ubuf, cnt)) | ||
2996 | return -EFAULT; | ||
2997 | |||
2998 | buf[cnt] = 0; | ||
2999 | |||
3000 | /* | ||
3001 | * Allow "echo > set_ftrace_pid" or "echo -n '' > set_ftrace_pid" | ||
3002 | * to clean the filter quietly. | ||
3003 | */ | ||
3004 | tmp = strstrip(buf); | ||
3005 | if (strlen(tmp) == 0) | ||
3006 | return 1; | ||
3007 | |||
3008 | ret = strict_strtol(tmp, 10, &val); | ||
3009 | if (ret < 0) | ||
3010 | return ret; | ||
3011 | |||
3012 | ret = ftrace_pid_add(val); | ||
3013 | |||
3014 | return ret ? ret : cnt; | ||
3015 | } | ||
3016 | |||
3017 | static int | ||
3018 | ftrace_pid_release(struct inode *inode, struct file *file) | ||
3019 | { | ||
3020 | if (file->f_mode & FMODE_READ) | ||
3021 | seq_release(inode, file); | ||
3022 | |||
3023 | return 0; | ||
2929 | } | 3024 | } |
2930 | 3025 | ||
2931 | static const struct file_operations ftrace_pid_fops = { | 3026 | static const struct file_operations ftrace_pid_fops = { |
2932 | .read = ftrace_pid_read, | 3027 | .open = ftrace_pid_open, |
2933 | .write = ftrace_pid_write, | 3028 | .write = ftrace_pid_write, |
3029 | .read = seq_read, | ||
3030 | .llseek = seq_lseek, | ||
3031 | .release = ftrace_pid_release, | ||
2934 | }; | 3032 | }; |
2935 | 3033 | ||
2936 | static __init int ftrace_init_debugfs(void) | 3034 | static __init int ftrace_init_debugfs(void) |
@@ -3293,4 +3391,3 @@ void ftrace_graph_stop(void) | |||
3293 | ftrace_stop(); | 3391 | ftrace_stop(); |
3294 | } | 3392 | } |
3295 | #endif | 3393 | #endif |
3296 | |||
diff --git a/kernel/trace/ring_buffer.c b/kernel/trace/ring_buffer.c index d4ff01970547..a1ca4956ab5e 100644 --- a/kernel/trace/ring_buffer.c +++ b/kernel/trace/ring_buffer.c | |||
@@ -397,18 +397,21 @@ int ring_buffer_print_page_header(struct trace_seq *s) | |||
397 | int ret; | 397 | int ret; |
398 | 398 | ||
399 | ret = trace_seq_printf(s, "\tfield: u64 timestamp;\t" | 399 | ret = trace_seq_printf(s, "\tfield: u64 timestamp;\t" |
400 | "offset:0;\tsize:%u;\n", | 400 | "offset:0;\tsize:%u;\tsigned:%u;\n", |
401 | (unsigned int)sizeof(field.time_stamp)); | 401 | (unsigned int)sizeof(field.time_stamp), |
402 | (unsigned int)is_signed_type(u64)); | ||
402 | 403 | ||
403 | ret = trace_seq_printf(s, "\tfield: local_t commit;\t" | 404 | ret = trace_seq_printf(s, "\tfield: local_t commit;\t" |
404 | "offset:%u;\tsize:%u;\n", | 405 | "offset:%u;\tsize:%u;\tsigned:%u;\n", |
405 | (unsigned int)offsetof(typeof(field), commit), | 406 | (unsigned int)offsetof(typeof(field), commit), |
406 | (unsigned int)sizeof(field.commit)); | 407 | (unsigned int)sizeof(field.commit), |
408 | (unsigned int)is_signed_type(long)); | ||
407 | 409 | ||
408 | ret = trace_seq_printf(s, "\tfield: char data;\t" | 410 | ret = trace_seq_printf(s, "\tfield: char data;\t" |
409 | "offset:%u;\tsize:%u;\n", | 411 | "offset:%u;\tsize:%u;\tsigned:%u;\n", |
410 | (unsigned int)offsetof(typeof(field), data), | 412 | (unsigned int)offsetof(typeof(field), data), |
411 | (unsigned int)BUF_PAGE_SIZE); | 413 | (unsigned int)BUF_PAGE_SIZE, |
414 | (unsigned int)is_signed_type(char)); | ||
412 | 415 | ||
413 | return ret; | 416 | return ret; |
414 | } | 417 | } |
@@ -483,7 +486,7 @@ struct ring_buffer_iter { | |||
483 | /* Up this if you want to test the TIME_EXTENTS and normalization */ | 486 | /* Up this if you want to test the TIME_EXTENTS and normalization */ |
484 | #define DEBUG_SHIFT 0 | 487 | #define DEBUG_SHIFT 0 |
485 | 488 | ||
486 | static inline u64 rb_time_stamp(struct ring_buffer *buffer, int cpu) | 489 | static inline u64 rb_time_stamp(struct ring_buffer *buffer) |
487 | { | 490 | { |
488 | /* shift to debug/test normalization and TIME_EXTENTS */ | 491 | /* shift to debug/test normalization and TIME_EXTENTS */ |
489 | return buffer->clock() << DEBUG_SHIFT; | 492 | return buffer->clock() << DEBUG_SHIFT; |
@@ -494,7 +497,7 @@ u64 ring_buffer_time_stamp(struct ring_buffer *buffer, int cpu) | |||
494 | u64 time; | 497 | u64 time; |
495 | 498 | ||
496 | preempt_disable_notrace(); | 499 | preempt_disable_notrace(); |
497 | time = rb_time_stamp(buffer, cpu); | 500 | time = rb_time_stamp(buffer); |
498 | preempt_enable_no_resched_notrace(); | 501 | preempt_enable_no_resched_notrace(); |
499 | 502 | ||
500 | return time; | 503 | return time; |
@@ -599,7 +602,7 @@ static struct list_head *rb_list_head(struct list_head *list) | |||
599 | } | 602 | } |
600 | 603 | ||
601 | /* | 604 | /* |
602 | * rb_is_head_page - test if the give page is the head page | 605 | * rb_is_head_page - test if the given page is the head page |
603 | * | 606 | * |
604 | * Because the reader may move the head_page pointer, we can | 607 | * Because the reader may move the head_page pointer, we can |
605 | * not trust what the head page is (it may be pointing to | 608 | * not trust what the head page is (it may be pointing to |
@@ -1193,6 +1196,7 @@ rb_remove_pages(struct ring_buffer_per_cpu *cpu_buffer, unsigned nr_pages) | |||
1193 | atomic_inc(&cpu_buffer->record_disabled); | 1196 | atomic_inc(&cpu_buffer->record_disabled); |
1194 | synchronize_sched(); | 1197 | synchronize_sched(); |
1195 | 1198 | ||
1199 | spin_lock_irq(&cpu_buffer->reader_lock); | ||
1196 | rb_head_page_deactivate(cpu_buffer); | 1200 | rb_head_page_deactivate(cpu_buffer); |
1197 | 1201 | ||
1198 | for (i = 0; i < nr_pages; i++) { | 1202 | for (i = 0; i < nr_pages; i++) { |
@@ -1207,6 +1211,7 @@ rb_remove_pages(struct ring_buffer_per_cpu *cpu_buffer, unsigned nr_pages) | |||
1207 | return; | 1211 | return; |
1208 | 1212 | ||
1209 | rb_reset_cpu(cpu_buffer); | 1213 | rb_reset_cpu(cpu_buffer); |
1214 | spin_unlock_irq(&cpu_buffer->reader_lock); | ||
1210 | 1215 | ||
1211 | rb_check_pages(cpu_buffer); | 1216 | rb_check_pages(cpu_buffer); |
1212 | 1217 | ||
@@ -1785,9 +1790,9 @@ rb_reset_tail(struct ring_buffer_per_cpu *cpu_buffer, | |||
1785 | static struct ring_buffer_event * | 1790 | static struct ring_buffer_event * |
1786 | rb_move_tail(struct ring_buffer_per_cpu *cpu_buffer, | 1791 | rb_move_tail(struct ring_buffer_per_cpu *cpu_buffer, |
1787 | unsigned long length, unsigned long tail, | 1792 | unsigned long length, unsigned long tail, |
1788 | struct buffer_page *commit_page, | ||
1789 | struct buffer_page *tail_page, u64 *ts) | 1793 | struct buffer_page *tail_page, u64 *ts) |
1790 | { | 1794 | { |
1795 | struct buffer_page *commit_page = cpu_buffer->commit_page; | ||
1791 | struct ring_buffer *buffer = cpu_buffer->buffer; | 1796 | struct ring_buffer *buffer = cpu_buffer->buffer; |
1792 | struct buffer_page *next_page; | 1797 | struct buffer_page *next_page; |
1793 | int ret; | 1798 | int ret; |
@@ -1868,7 +1873,7 @@ rb_move_tail(struct ring_buffer_per_cpu *cpu_buffer, | |||
1868 | * Nested commits always have zero deltas, so | 1873 | * Nested commits always have zero deltas, so |
1869 | * just reread the time stamp | 1874 | * just reread the time stamp |
1870 | */ | 1875 | */ |
1871 | *ts = rb_time_stamp(buffer, cpu_buffer->cpu); | 1876 | *ts = rb_time_stamp(buffer); |
1872 | next_page->page->time_stamp = *ts; | 1877 | next_page->page->time_stamp = *ts; |
1873 | } | 1878 | } |
1874 | 1879 | ||
@@ -1890,13 +1895,10 @@ static struct ring_buffer_event * | |||
1890 | __rb_reserve_next(struct ring_buffer_per_cpu *cpu_buffer, | 1895 | __rb_reserve_next(struct ring_buffer_per_cpu *cpu_buffer, |
1891 | unsigned type, unsigned long length, u64 *ts) | 1896 | unsigned type, unsigned long length, u64 *ts) |
1892 | { | 1897 | { |
1893 | struct buffer_page *tail_page, *commit_page; | 1898 | struct buffer_page *tail_page; |
1894 | struct ring_buffer_event *event; | 1899 | struct ring_buffer_event *event; |
1895 | unsigned long tail, write; | 1900 | unsigned long tail, write; |
1896 | 1901 | ||
1897 | commit_page = cpu_buffer->commit_page; | ||
1898 | /* we just need to protect against interrupts */ | ||
1899 | barrier(); | ||
1900 | tail_page = cpu_buffer->tail_page; | 1902 | tail_page = cpu_buffer->tail_page; |
1901 | write = local_add_return(length, &tail_page->write); | 1903 | write = local_add_return(length, &tail_page->write); |
1902 | 1904 | ||
@@ -1907,7 +1909,7 @@ __rb_reserve_next(struct ring_buffer_per_cpu *cpu_buffer, | |||
1907 | /* See if we shot pass the end of this buffer page */ | 1909 | /* See if we shot pass the end of this buffer page */ |
1908 | if (write > BUF_PAGE_SIZE) | 1910 | if (write > BUF_PAGE_SIZE) |
1909 | return rb_move_tail(cpu_buffer, length, tail, | 1911 | return rb_move_tail(cpu_buffer, length, tail, |
1910 | commit_page, tail_page, ts); | 1912 | tail_page, ts); |
1911 | 1913 | ||
1912 | /* We reserved something on the buffer */ | 1914 | /* We reserved something on the buffer */ |
1913 | 1915 | ||
@@ -2111,7 +2113,7 @@ rb_reserve_next_event(struct ring_buffer *buffer, | |||
2111 | if (RB_WARN_ON(cpu_buffer, ++nr_loops > 1000)) | 2113 | if (RB_WARN_ON(cpu_buffer, ++nr_loops > 1000)) |
2112 | goto out_fail; | 2114 | goto out_fail; |
2113 | 2115 | ||
2114 | ts = rb_time_stamp(cpu_buffer->buffer, cpu_buffer->cpu); | 2116 | ts = rb_time_stamp(cpu_buffer->buffer); |
2115 | 2117 | ||
2116 | /* | 2118 | /* |
2117 | * Only the first commit can update the timestamp. | 2119 | * Only the first commit can update the timestamp. |
@@ -2681,7 +2683,7 @@ unsigned long ring_buffer_entries(struct ring_buffer *buffer) | |||
2681 | EXPORT_SYMBOL_GPL(ring_buffer_entries); | 2683 | EXPORT_SYMBOL_GPL(ring_buffer_entries); |
2682 | 2684 | ||
2683 | /** | 2685 | /** |
2684 | * ring_buffer_overrun_cpu - get the number of overruns in buffer | 2686 | * ring_buffer_overruns - get the number of overruns in buffer |
2685 | * @buffer: The ring buffer | 2687 | * @buffer: The ring buffer |
2686 | * | 2688 | * |
2687 | * Returns the total number of overruns in the ring buffer | 2689 | * Returns the total number of overruns in the ring buffer |
diff --git a/kernel/trace/ring_buffer_benchmark.c b/kernel/trace/ring_buffer_benchmark.c index 573d3cc762c3..b2477caf09c2 100644 --- a/kernel/trace/ring_buffer_benchmark.c +++ b/kernel/trace/ring_buffer_benchmark.c | |||
@@ -35,6 +35,28 @@ static int disable_reader; | |||
35 | module_param(disable_reader, uint, 0644); | 35 | module_param(disable_reader, uint, 0644); |
36 | MODULE_PARM_DESC(disable_reader, "only run producer"); | 36 | MODULE_PARM_DESC(disable_reader, "only run producer"); |
37 | 37 | ||
38 | static int write_iteration = 50; | ||
39 | module_param(write_iteration, uint, 0644); | ||
40 | MODULE_PARM_DESC(write_iteration, "# of writes between timestamp readings"); | ||
41 | |||
42 | static int producer_nice = 19; | ||
43 | static int consumer_nice = 19; | ||
44 | |||
45 | static int producer_fifo = -1; | ||
46 | static int consumer_fifo = -1; | ||
47 | |||
48 | module_param(producer_nice, uint, 0644); | ||
49 | MODULE_PARM_DESC(producer_nice, "nice prio for producer"); | ||
50 | |||
51 | module_param(consumer_nice, uint, 0644); | ||
52 | MODULE_PARM_DESC(consumer_nice, "nice prio for consumer"); | ||
53 | |||
54 | module_param(producer_fifo, uint, 0644); | ||
55 | MODULE_PARM_DESC(producer_fifo, "fifo prio for producer"); | ||
56 | |||
57 | module_param(consumer_fifo, uint, 0644); | ||
58 | MODULE_PARM_DESC(consumer_fifo, "fifo prio for consumer"); | ||
59 | |||
38 | static int read_events; | 60 | static int read_events; |
39 | 61 | ||
40 | static int kill_test; | 62 | static int kill_test; |
@@ -208,15 +230,18 @@ static void ring_buffer_producer(void) | |||
208 | do { | 230 | do { |
209 | struct ring_buffer_event *event; | 231 | struct ring_buffer_event *event; |
210 | int *entry; | 232 | int *entry; |
211 | 233 | int i; | |
212 | event = ring_buffer_lock_reserve(buffer, 10); | 234 | |
213 | if (!event) { | 235 | for (i = 0; i < write_iteration; i++) { |
214 | missed++; | 236 | event = ring_buffer_lock_reserve(buffer, 10); |
215 | } else { | 237 | if (!event) { |
216 | hit++; | 238 | missed++; |
217 | entry = ring_buffer_event_data(event); | 239 | } else { |
218 | *entry = smp_processor_id(); | 240 | hit++; |
219 | ring_buffer_unlock_commit(buffer, event); | 241 | entry = ring_buffer_event_data(event); |
242 | *entry = smp_processor_id(); | ||
243 | ring_buffer_unlock_commit(buffer, event); | ||
244 | } | ||
220 | } | 245 | } |
221 | do_gettimeofday(&end_tv); | 246 | do_gettimeofday(&end_tv); |
222 | 247 | ||
@@ -263,6 +288,27 @@ static void ring_buffer_producer(void) | |||
263 | 288 | ||
264 | if (kill_test) | 289 | if (kill_test) |
265 | trace_printk("ERROR!\n"); | 290 | trace_printk("ERROR!\n"); |
291 | |||
292 | if (!disable_reader) { | ||
293 | if (consumer_fifo < 0) | ||
294 | trace_printk("Running Consumer at nice: %d\n", | ||
295 | consumer_nice); | ||
296 | else | ||
297 | trace_printk("Running Consumer at SCHED_FIFO %d\n", | ||
298 | consumer_fifo); | ||
299 | } | ||
300 | if (producer_fifo < 0) | ||
301 | trace_printk("Running Producer at nice: %d\n", | ||
302 | producer_nice); | ||
303 | else | ||
304 | trace_printk("Running Producer at SCHED_FIFO %d\n", | ||
305 | producer_fifo); | ||
306 | |||
307 | /* Let the user know that the test is running at low priority */ | ||
308 | if (producer_fifo < 0 && consumer_fifo < 0 && | ||
309 | producer_nice == 19 && consumer_nice == 19) | ||
310 | trace_printk("WARNING!!! This test is running at lowest priority.\n"); | ||
311 | |||
266 | trace_printk("Time: %lld (usecs)\n", time); | 312 | trace_printk("Time: %lld (usecs)\n", time); |
267 | trace_printk("Overruns: %lld\n", overruns); | 313 | trace_printk("Overruns: %lld\n", overruns); |
268 | if (disable_reader) | 314 | if (disable_reader) |
@@ -392,6 +438,27 @@ static int __init ring_buffer_benchmark_init(void) | |||
392 | if (IS_ERR(producer)) | 438 | if (IS_ERR(producer)) |
393 | goto out_kill; | 439 | goto out_kill; |
394 | 440 | ||
441 | /* | ||
442 | * Run them as low-prio background tasks by default: | ||
443 | */ | ||
444 | if (!disable_reader) { | ||
445 | if (consumer_fifo >= 0) { | ||
446 | struct sched_param param = { | ||
447 | .sched_priority = consumer_fifo | ||
448 | }; | ||
449 | sched_setscheduler(consumer, SCHED_FIFO, ¶m); | ||
450 | } else | ||
451 | set_user_nice(consumer, consumer_nice); | ||
452 | } | ||
453 | |||
454 | if (producer_fifo >= 0) { | ||
455 | struct sched_param param = { | ||
456 | .sched_priority = consumer_fifo | ||
457 | }; | ||
458 | sched_setscheduler(producer, SCHED_FIFO, ¶m); | ||
459 | } else | ||
460 | set_user_nice(producer, producer_nice); | ||
461 | |||
395 | return 0; | 462 | return 0; |
396 | 463 | ||
397 | out_kill: | 464 | out_kill: |
diff --git a/kernel/trace/trace.c b/kernel/trace/trace.c index c820b0310a12..874f2893cff0 100644 --- a/kernel/trace/trace.c +++ b/kernel/trace/trace.c | |||
@@ -129,7 +129,7 @@ static int tracing_set_tracer(const char *buf); | |||
129 | static char bootup_tracer_buf[MAX_TRACER_SIZE] __initdata; | 129 | static char bootup_tracer_buf[MAX_TRACER_SIZE] __initdata; |
130 | static char *default_bootup_tracer; | 130 | static char *default_bootup_tracer; |
131 | 131 | ||
132 | static int __init set_ftrace(char *str) | 132 | static int __init set_cmdline_ftrace(char *str) |
133 | { | 133 | { |
134 | strncpy(bootup_tracer_buf, str, MAX_TRACER_SIZE); | 134 | strncpy(bootup_tracer_buf, str, MAX_TRACER_SIZE); |
135 | default_bootup_tracer = bootup_tracer_buf; | 135 | default_bootup_tracer = bootup_tracer_buf; |
@@ -137,7 +137,7 @@ static int __init set_ftrace(char *str) | |||
137 | ring_buffer_expanded = 1; | 137 | ring_buffer_expanded = 1; |
138 | return 1; | 138 | return 1; |
139 | } | 139 | } |
140 | __setup("ftrace=", set_ftrace); | 140 | __setup("ftrace=", set_cmdline_ftrace); |
141 | 141 | ||
142 | static int __init set_ftrace_dump_on_oops(char *str) | 142 | static int __init set_ftrace_dump_on_oops(char *str) |
143 | { | 143 | { |
@@ -1361,10 +1361,11 @@ int trace_array_vprintk(struct trace_array *tr, | |||
1361 | pause_graph_tracing(); | 1361 | pause_graph_tracing(); |
1362 | raw_local_irq_save(irq_flags); | 1362 | raw_local_irq_save(irq_flags); |
1363 | __raw_spin_lock(&trace_buf_lock); | 1363 | __raw_spin_lock(&trace_buf_lock); |
1364 | len = vsnprintf(trace_buf, TRACE_BUF_SIZE, fmt, args); | 1364 | if (args == NULL) { |
1365 | 1365 | strncpy(trace_buf, fmt, TRACE_BUF_SIZE); | |
1366 | len = min(len, TRACE_BUF_SIZE-1); | 1366 | len = strlen(trace_buf); |
1367 | trace_buf[len] = 0; | 1367 | } else |
1368 | len = vsnprintf(trace_buf, TRACE_BUF_SIZE, fmt, args); | ||
1368 | 1369 | ||
1369 | size = sizeof(*entry) + len + 1; | 1370 | size = sizeof(*entry) + len + 1; |
1370 | buffer = tr->buffer; | 1371 | buffer = tr->buffer; |
@@ -1373,10 +1374,10 @@ int trace_array_vprintk(struct trace_array *tr, | |||
1373 | if (!event) | 1374 | if (!event) |
1374 | goto out_unlock; | 1375 | goto out_unlock; |
1375 | entry = ring_buffer_event_data(event); | 1376 | entry = ring_buffer_event_data(event); |
1376 | entry->ip = ip; | 1377 | entry->ip = ip; |
1377 | 1378 | ||
1378 | memcpy(&entry->buf, trace_buf, len); | 1379 | memcpy(&entry->buf, trace_buf, len); |
1379 | entry->buf[len] = 0; | 1380 | entry->buf[len] = '\0'; |
1380 | if (!filter_check_discard(call, entry, buffer, event)) | 1381 | if (!filter_check_discard(call, entry, buffer, event)) |
1381 | ring_buffer_unlock_commit(buffer, event); | 1382 | ring_buffer_unlock_commit(buffer, event); |
1382 | 1383 | ||
@@ -2440,7 +2441,7 @@ tracing_trace_options_write(struct file *filp, const char __user *ubuf, | |||
2440 | return ret; | 2441 | return ret; |
2441 | } | 2442 | } |
2442 | 2443 | ||
2443 | filp->f_pos += cnt; | 2444 | *ppos += cnt; |
2444 | 2445 | ||
2445 | return cnt; | 2446 | return cnt; |
2446 | } | 2447 | } |
@@ -2582,7 +2583,7 @@ tracing_ctrl_write(struct file *filp, const char __user *ubuf, | |||
2582 | } | 2583 | } |
2583 | mutex_unlock(&trace_types_lock); | 2584 | mutex_unlock(&trace_types_lock); |
2584 | 2585 | ||
2585 | filp->f_pos += cnt; | 2586 | *ppos += cnt; |
2586 | 2587 | ||
2587 | return cnt; | 2588 | return cnt; |
2588 | } | 2589 | } |
@@ -2764,7 +2765,7 @@ tracing_set_trace_write(struct file *filp, const char __user *ubuf, | |||
2764 | if (err) | 2765 | if (err) |
2765 | return err; | 2766 | return err; |
2766 | 2767 | ||
2767 | filp->f_pos += ret; | 2768 | *ppos += ret; |
2768 | 2769 | ||
2769 | return ret; | 2770 | return ret; |
2770 | } | 2771 | } |
@@ -3299,7 +3300,7 @@ tracing_entries_write(struct file *filp, const char __user *ubuf, | |||
3299 | } | 3300 | } |
3300 | } | 3301 | } |
3301 | 3302 | ||
3302 | filp->f_pos += cnt; | 3303 | *ppos += cnt; |
3303 | 3304 | ||
3304 | /* If check pages failed, return ENOMEM */ | 3305 | /* If check pages failed, return ENOMEM */ |
3305 | if (tracing_disabled) | 3306 | if (tracing_disabled) |
@@ -3319,22 +3320,11 @@ tracing_entries_write(struct file *filp, const char __user *ubuf, | |||
3319 | return cnt; | 3320 | return cnt; |
3320 | } | 3321 | } |
3321 | 3322 | ||
3322 | static int mark_printk(const char *fmt, ...) | ||
3323 | { | ||
3324 | int ret; | ||
3325 | va_list args; | ||
3326 | va_start(args, fmt); | ||
3327 | ret = trace_vprintk(0, fmt, args); | ||
3328 | va_end(args); | ||
3329 | return ret; | ||
3330 | } | ||
3331 | |||
3332 | static ssize_t | 3323 | static ssize_t |
3333 | tracing_mark_write(struct file *filp, const char __user *ubuf, | 3324 | tracing_mark_write(struct file *filp, const char __user *ubuf, |
3334 | size_t cnt, loff_t *fpos) | 3325 | size_t cnt, loff_t *fpos) |
3335 | { | 3326 | { |
3336 | char *buf; | 3327 | char *buf; |
3337 | char *end; | ||
3338 | 3328 | ||
3339 | if (tracing_disabled) | 3329 | if (tracing_disabled) |
3340 | return -EINVAL; | 3330 | return -EINVAL; |
@@ -3342,7 +3332,7 @@ tracing_mark_write(struct file *filp, const char __user *ubuf, | |||
3342 | if (cnt > TRACE_BUF_SIZE) | 3332 | if (cnt > TRACE_BUF_SIZE) |
3343 | cnt = TRACE_BUF_SIZE; | 3333 | cnt = TRACE_BUF_SIZE; |
3344 | 3334 | ||
3345 | buf = kmalloc(cnt + 1, GFP_KERNEL); | 3335 | buf = kmalloc(cnt + 2, GFP_KERNEL); |
3346 | if (buf == NULL) | 3336 | if (buf == NULL) |
3347 | return -ENOMEM; | 3337 | return -ENOMEM; |
3348 | 3338 | ||
@@ -3350,14 +3340,13 @@ tracing_mark_write(struct file *filp, const char __user *ubuf, | |||
3350 | kfree(buf); | 3340 | kfree(buf); |
3351 | return -EFAULT; | 3341 | return -EFAULT; |
3352 | } | 3342 | } |
3343 | if (buf[cnt-1] != '\n') { | ||
3344 | buf[cnt] = '\n'; | ||
3345 | buf[cnt+1] = '\0'; | ||
3346 | } else | ||
3347 | buf[cnt] = '\0'; | ||
3353 | 3348 | ||
3354 | /* Cut from the first nil or newline. */ | 3349 | cnt = trace_vprintk(0, buf, NULL); |
3355 | buf[cnt] = '\0'; | ||
3356 | end = strchr(buf, '\n'); | ||
3357 | if (end) | ||
3358 | *end = '\0'; | ||
3359 | |||
3360 | cnt = mark_printk("%s\n", buf); | ||
3361 | kfree(buf); | 3350 | kfree(buf); |
3362 | *fpos += cnt; | 3351 | *fpos += cnt; |
3363 | 3352 | ||
@@ -3730,7 +3719,7 @@ tracing_stats_read(struct file *filp, char __user *ubuf, | |||
3730 | 3719 | ||
3731 | s = kmalloc(sizeof(*s), GFP_KERNEL); | 3720 | s = kmalloc(sizeof(*s), GFP_KERNEL); |
3732 | if (!s) | 3721 | if (!s) |
3733 | return ENOMEM; | 3722 | return -ENOMEM; |
3734 | 3723 | ||
3735 | trace_seq_init(s); | 3724 | trace_seq_init(s); |
3736 | 3725 | ||
diff --git a/kernel/trace/trace.h b/kernel/trace/trace.h index 405cb850b75d..1d7f4830a80d 100644 --- a/kernel/trace/trace.h +++ b/kernel/trace/trace.h | |||
@@ -11,6 +11,7 @@ | |||
11 | #include <linux/ftrace.h> | 11 | #include <linux/ftrace.h> |
12 | #include <trace/boot.h> | 12 | #include <trace/boot.h> |
13 | #include <linux/kmemtrace.h> | 13 | #include <linux/kmemtrace.h> |
14 | #include <linux/hw_breakpoint.h> | ||
14 | 15 | ||
15 | #include <linux/trace_seq.h> | 16 | #include <linux/trace_seq.h> |
16 | #include <linux/ftrace_event.h> | 17 | #include <linux/ftrace_event.h> |
@@ -37,6 +38,7 @@ enum trace_type { | |||
37 | TRACE_KMEM_ALLOC, | 38 | TRACE_KMEM_ALLOC, |
38 | TRACE_KMEM_FREE, | 39 | TRACE_KMEM_FREE, |
39 | TRACE_BLK, | 40 | TRACE_BLK, |
41 | TRACE_KSYM, | ||
40 | 42 | ||
41 | __TRACE_LAST_TYPE, | 43 | __TRACE_LAST_TYPE, |
42 | }; | 44 | }; |
@@ -98,9 +100,32 @@ struct syscall_trace_enter { | |||
98 | struct syscall_trace_exit { | 100 | struct syscall_trace_exit { |
99 | struct trace_entry ent; | 101 | struct trace_entry ent; |
100 | int nr; | 102 | int nr; |
101 | unsigned long ret; | 103 | long ret; |
102 | }; | 104 | }; |
103 | 105 | ||
106 | struct kprobe_trace_entry { | ||
107 | struct trace_entry ent; | ||
108 | unsigned long ip; | ||
109 | int nargs; | ||
110 | unsigned long args[]; | ||
111 | }; | ||
112 | |||
113 | #define SIZEOF_KPROBE_TRACE_ENTRY(n) \ | ||
114 | (offsetof(struct kprobe_trace_entry, args) + \ | ||
115 | (sizeof(unsigned long) * (n))) | ||
116 | |||
117 | struct kretprobe_trace_entry { | ||
118 | struct trace_entry ent; | ||
119 | unsigned long func; | ||
120 | unsigned long ret_ip; | ||
121 | int nargs; | ||
122 | unsigned long args[]; | ||
123 | }; | ||
124 | |||
125 | #define SIZEOF_KRETPROBE_TRACE_ENTRY(n) \ | ||
126 | (offsetof(struct kretprobe_trace_entry, args) + \ | ||
127 | (sizeof(unsigned long) * (n))) | ||
128 | |||
104 | /* | 129 | /* |
105 | * trace_flag_type is an enumeration that holds different | 130 | * trace_flag_type is an enumeration that holds different |
106 | * states when a trace occurs. These are: | 131 | * states when a trace occurs. These are: |
@@ -209,6 +234,7 @@ extern void __ftrace_bad_type(void); | |||
209 | TRACE_KMEM_ALLOC); \ | 234 | TRACE_KMEM_ALLOC); \ |
210 | IF_ASSIGN(var, ent, struct kmemtrace_free_entry, \ | 235 | IF_ASSIGN(var, ent, struct kmemtrace_free_entry, \ |
211 | TRACE_KMEM_FREE); \ | 236 | TRACE_KMEM_FREE); \ |
237 | IF_ASSIGN(var, ent, struct ksym_trace_entry, TRACE_KSYM);\ | ||
212 | __ftrace_bad_type(); \ | 238 | __ftrace_bad_type(); \ |
213 | } while (0) | 239 | } while (0) |
214 | 240 | ||
@@ -364,6 +390,8 @@ int register_tracer(struct tracer *type); | |||
364 | void unregister_tracer(struct tracer *type); | 390 | void unregister_tracer(struct tracer *type); |
365 | int is_tracing_stopped(void); | 391 | int is_tracing_stopped(void); |
366 | 392 | ||
393 | extern int process_new_ksym_entry(char *ksymname, int op, unsigned long addr); | ||
394 | |||
367 | extern unsigned long nsecs_to_usecs(unsigned long nsecs); | 395 | extern unsigned long nsecs_to_usecs(unsigned long nsecs); |
368 | 396 | ||
369 | #ifdef CONFIG_TRACER_MAX_TRACE | 397 | #ifdef CONFIG_TRACER_MAX_TRACE |
@@ -438,6 +466,8 @@ extern int trace_selftest_startup_branch(struct tracer *trace, | |||
438 | struct trace_array *tr); | 466 | struct trace_array *tr); |
439 | extern int trace_selftest_startup_hw_branches(struct tracer *trace, | 467 | extern int trace_selftest_startup_hw_branches(struct tracer *trace, |
440 | struct trace_array *tr); | 468 | struct trace_array *tr); |
469 | extern int trace_selftest_startup_ksym(struct tracer *trace, | ||
470 | struct trace_array *tr); | ||
441 | #endif /* CONFIG_FTRACE_STARTUP_TEST */ | 471 | #endif /* CONFIG_FTRACE_STARTUP_TEST */ |
442 | 472 | ||
443 | extern void *head_page(struct trace_array_cpu *data); | 473 | extern void *head_page(struct trace_array_cpu *data); |
@@ -483,10 +513,6 @@ static inline int ftrace_graph_addr(unsigned long addr) | |||
483 | return 0; | 513 | return 0; |
484 | } | 514 | } |
485 | #else | 515 | #else |
486 | static inline int ftrace_trace_addr(unsigned long addr) | ||
487 | { | ||
488 | return 1; | ||
489 | } | ||
490 | static inline int ftrace_graph_addr(unsigned long addr) | 516 | static inline int ftrace_graph_addr(unsigned long addr) |
491 | { | 517 | { |
492 | return 1; | 518 | return 1; |
@@ -500,12 +526,12 @@ print_graph_function(struct trace_iterator *iter) | |||
500 | } | 526 | } |
501 | #endif /* CONFIG_FUNCTION_GRAPH_TRACER */ | 527 | #endif /* CONFIG_FUNCTION_GRAPH_TRACER */ |
502 | 528 | ||
503 | extern struct pid *ftrace_pid_trace; | 529 | extern struct list_head ftrace_pids; |
504 | 530 | ||
505 | #ifdef CONFIG_FUNCTION_TRACER | 531 | #ifdef CONFIG_FUNCTION_TRACER |
506 | static inline int ftrace_trace_task(struct task_struct *task) | 532 | static inline int ftrace_trace_task(struct task_struct *task) |
507 | { | 533 | { |
508 | if (!ftrace_pid_trace) | 534 | if (list_empty(&ftrace_pids)) |
509 | return 1; | 535 | return 1; |
510 | 536 | ||
511 | return test_tsk_trace_trace(task); | 537 | return test_tsk_trace_trace(task); |
@@ -687,7 +713,6 @@ struct event_filter { | |||
687 | int n_preds; | 713 | int n_preds; |
688 | struct filter_pred **preds; | 714 | struct filter_pred **preds; |
689 | char *filter_string; | 715 | char *filter_string; |
690 | bool no_reset; | ||
691 | }; | 716 | }; |
692 | 717 | ||
693 | struct event_subsystem { | 718 | struct event_subsystem { |
@@ -699,22 +724,40 @@ struct event_subsystem { | |||
699 | }; | 724 | }; |
700 | 725 | ||
701 | struct filter_pred; | 726 | struct filter_pred; |
727 | struct regex; | ||
702 | 728 | ||
703 | typedef int (*filter_pred_fn_t) (struct filter_pred *pred, void *event, | 729 | typedef int (*filter_pred_fn_t) (struct filter_pred *pred, void *event, |
704 | int val1, int val2); | 730 | int val1, int val2); |
705 | 731 | ||
732 | typedef int (*regex_match_func)(char *str, struct regex *r, int len); | ||
733 | |||
734 | enum regex_type { | ||
735 | MATCH_FULL = 0, | ||
736 | MATCH_FRONT_ONLY, | ||
737 | MATCH_MIDDLE_ONLY, | ||
738 | MATCH_END_ONLY, | ||
739 | }; | ||
740 | |||
741 | struct regex { | ||
742 | char pattern[MAX_FILTER_STR_VAL]; | ||
743 | int len; | ||
744 | int field_len; | ||
745 | regex_match_func match; | ||
746 | }; | ||
747 | |||
706 | struct filter_pred { | 748 | struct filter_pred { |
707 | filter_pred_fn_t fn; | 749 | filter_pred_fn_t fn; |
708 | u64 val; | 750 | u64 val; |
709 | char str_val[MAX_FILTER_STR_VAL]; | 751 | struct regex regex; |
710 | int str_len; | 752 | char *field_name; |
711 | char *field_name; | 753 | int offset; |
712 | int offset; | 754 | int not; |
713 | int not; | 755 | int op; |
714 | int op; | 756 | int pop_n; |
715 | int pop_n; | ||
716 | }; | 757 | }; |
717 | 758 | ||
759 | extern enum regex_type | ||
760 | filter_parse_regex(char *buff, int len, char **search, int *not); | ||
718 | extern void print_event_filter(struct ftrace_event_call *call, | 761 | extern void print_event_filter(struct ftrace_event_call *call, |
719 | struct trace_seq *s); | 762 | struct trace_seq *s); |
720 | extern int apply_event_filter(struct ftrace_event_call *call, | 763 | extern int apply_event_filter(struct ftrace_event_call *call, |
@@ -730,7 +773,8 @@ filter_check_discard(struct ftrace_event_call *call, void *rec, | |||
730 | struct ring_buffer *buffer, | 773 | struct ring_buffer *buffer, |
731 | struct ring_buffer_event *event) | 774 | struct ring_buffer_event *event) |
732 | { | 775 | { |
733 | if (unlikely(call->filter_active) && !filter_match_preds(call, rec)) { | 776 | if (unlikely(call->filter_active) && |
777 | !filter_match_preds(call->filter, rec)) { | ||
734 | ring_buffer_discard_commit(buffer, event); | 778 | ring_buffer_discard_commit(buffer, event); |
735 | return 1; | 779 | return 1; |
736 | } | 780 | } |
diff --git a/kernel/trace/trace_clock.c b/kernel/trace/trace_clock.c index 20c5f92e28a8..878c03f386ba 100644 --- a/kernel/trace/trace_clock.c +++ b/kernel/trace/trace_clock.c | |||
@@ -20,6 +20,8 @@ | |||
20 | #include <linux/ktime.h> | 20 | #include <linux/ktime.h> |
21 | #include <linux/trace_clock.h> | 21 | #include <linux/trace_clock.h> |
22 | 22 | ||
23 | #include "trace.h" | ||
24 | |||
23 | /* | 25 | /* |
24 | * trace_clock_local(): the simplest and least coherent tracing clock. | 26 | * trace_clock_local(): the simplest and least coherent tracing clock. |
25 | * | 27 | * |
@@ -28,17 +30,17 @@ | |||
28 | */ | 30 | */ |
29 | u64 notrace trace_clock_local(void) | 31 | u64 notrace trace_clock_local(void) |
30 | { | 32 | { |
31 | unsigned long flags; | ||
32 | u64 clock; | 33 | u64 clock; |
34 | int resched; | ||
33 | 35 | ||
34 | /* | 36 | /* |
35 | * sched_clock() is an architecture implemented, fast, scalable, | 37 | * sched_clock() is an architecture implemented, fast, scalable, |
36 | * lockless clock. It is not guaranteed to be coherent across | 38 | * lockless clock. It is not guaranteed to be coherent across |
37 | * CPUs, nor across CPU idle events. | 39 | * CPUs, nor across CPU idle events. |
38 | */ | 40 | */ |
39 | raw_local_irq_save(flags); | 41 | resched = ftrace_preempt_disable(); |
40 | clock = sched_clock(); | 42 | clock = sched_clock(); |
41 | raw_local_irq_restore(flags); | 43 | ftrace_preempt_enable(resched); |
42 | 44 | ||
43 | return clock; | 45 | return clock; |
44 | } | 46 | } |
diff --git a/kernel/trace/trace_entries.h b/kernel/trace/trace_entries.h index ead3d724599d..c16a08f399df 100644 --- a/kernel/trace/trace_entries.h +++ b/kernel/trace/trace_entries.h | |||
@@ -364,3 +364,19 @@ FTRACE_ENTRY(kmem_free, kmemtrace_free_entry, | |||
364 | F_printk("type:%u call_site:%lx ptr:%p", | 364 | F_printk("type:%u call_site:%lx ptr:%p", |
365 | __entry->type_id, __entry->call_site, __entry->ptr) | 365 | __entry->type_id, __entry->call_site, __entry->ptr) |
366 | ); | 366 | ); |
367 | |||
368 | FTRACE_ENTRY(ksym_trace, ksym_trace_entry, | ||
369 | |||
370 | TRACE_KSYM, | ||
371 | |||
372 | F_STRUCT( | ||
373 | __field( unsigned long, ip ) | ||
374 | __field( unsigned char, type ) | ||
375 | __array( char , cmd, TASK_COMM_LEN ) | ||
376 | __field( unsigned long, addr ) | ||
377 | ), | ||
378 | |||
379 | F_printk("ip: %pF type: %d ksym_name: %pS cmd: %s", | ||
380 | (void *)__entry->ip, (unsigned int)__entry->type, | ||
381 | (void *)__entry->addr, __entry->cmd) | ||
382 | ); | ||
diff --git a/kernel/trace/trace_event_profile.c b/kernel/trace/trace_event_profile.c index 8d5c171cc998..d9c60f80aa0d 100644 --- a/kernel/trace/trace_event_profile.c +++ b/kernel/trace/trace_event_profile.c | |||
@@ -8,17 +8,14 @@ | |||
8 | #include <linux/module.h> | 8 | #include <linux/module.h> |
9 | #include "trace.h" | 9 | #include "trace.h" |
10 | 10 | ||
11 | /* | ||
12 | * We can't use a size but a type in alloc_percpu() | ||
13 | * So let's create a dummy type that matches the desired size | ||
14 | */ | ||
15 | typedef struct {char buf[FTRACE_MAX_PROFILE_SIZE];} profile_buf_t; | ||
16 | 11 | ||
17 | char *trace_profile_buf; | 12 | char *perf_trace_buf; |
18 | EXPORT_SYMBOL_GPL(trace_profile_buf); | 13 | EXPORT_SYMBOL_GPL(perf_trace_buf); |
14 | |||
15 | char *perf_trace_buf_nmi; | ||
16 | EXPORT_SYMBOL_GPL(perf_trace_buf_nmi); | ||
19 | 17 | ||
20 | char *trace_profile_buf_nmi; | 18 | typedef typeof(char [FTRACE_MAX_PROFILE_SIZE]) perf_trace_t ; |
21 | EXPORT_SYMBOL_GPL(trace_profile_buf_nmi); | ||
22 | 19 | ||
23 | /* Count the events in use (per event id, not per instance) */ | 20 | /* Count the events in use (per event id, not per instance) */ |
24 | static int total_profile_count; | 21 | static int total_profile_count; |
@@ -32,20 +29,20 @@ static int ftrace_profile_enable_event(struct ftrace_event_call *event) | |||
32 | return 0; | 29 | return 0; |
33 | 30 | ||
34 | if (!total_profile_count) { | 31 | if (!total_profile_count) { |
35 | buf = (char *)alloc_percpu(profile_buf_t); | 32 | buf = (char *)alloc_percpu(perf_trace_t); |
36 | if (!buf) | 33 | if (!buf) |
37 | goto fail_buf; | 34 | goto fail_buf; |
38 | 35 | ||
39 | rcu_assign_pointer(trace_profile_buf, buf); | 36 | rcu_assign_pointer(perf_trace_buf, buf); |
40 | 37 | ||
41 | buf = (char *)alloc_percpu(profile_buf_t); | 38 | buf = (char *)alloc_percpu(perf_trace_t); |
42 | if (!buf) | 39 | if (!buf) |
43 | goto fail_buf_nmi; | 40 | goto fail_buf_nmi; |
44 | 41 | ||
45 | rcu_assign_pointer(trace_profile_buf_nmi, buf); | 42 | rcu_assign_pointer(perf_trace_buf_nmi, buf); |
46 | } | 43 | } |
47 | 44 | ||
48 | ret = event->profile_enable(); | 45 | ret = event->profile_enable(event); |
49 | if (!ret) { | 46 | if (!ret) { |
50 | total_profile_count++; | 47 | total_profile_count++; |
51 | return 0; | 48 | return 0; |
@@ -53,10 +50,10 @@ static int ftrace_profile_enable_event(struct ftrace_event_call *event) | |||
53 | 50 | ||
54 | fail_buf_nmi: | 51 | fail_buf_nmi: |
55 | if (!total_profile_count) { | 52 | if (!total_profile_count) { |
56 | free_percpu(trace_profile_buf_nmi); | 53 | free_percpu(perf_trace_buf_nmi); |
57 | free_percpu(trace_profile_buf); | 54 | free_percpu(perf_trace_buf); |
58 | trace_profile_buf_nmi = NULL; | 55 | perf_trace_buf_nmi = NULL; |
59 | trace_profile_buf = NULL; | 56 | perf_trace_buf = NULL; |
60 | } | 57 | } |
61 | fail_buf: | 58 | fail_buf: |
62 | atomic_dec(&event->profile_count); | 59 | atomic_dec(&event->profile_count); |
@@ -89,14 +86,14 @@ static void ftrace_profile_disable_event(struct ftrace_event_call *event) | |||
89 | if (!atomic_add_negative(-1, &event->profile_count)) | 86 | if (!atomic_add_negative(-1, &event->profile_count)) |
90 | return; | 87 | return; |
91 | 88 | ||
92 | event->profile_disable(); | 89 | event->profile_disable(event); |
93 | 90 | ||
94 | if (!--total_profile_count) { | 91 | if (!--total_profile_count) { |
95 | buf = trace_profile_buf; | 92 | buf = perf_trace_buf; |
96 | rcu_assign_pointer(trace_profile_buf, NULL); | 93 | rcu_assign_pointer(perf_trace_buf, NULL); |
97 | 94 | ||
98 | nmi_buf = trace_profile_buf_nmi; | 95 | nmi_buf = perf_trace_buf_nmi; |
99 | rcu_assign_pointer(trace_profile_buf_nmi, NULL); | 96 | rcu_assign_pointer(perf_trace_buf_nmi, NULL); |
100 | 97 | ||
101 | /* | 98 | /* |
102 | * Ensure every events in profiling have finished before | 99 | * Ensure every events in profiling have finished before |
diff --git a/kernel/trace/trace_events.c b/kernel/trace/trace_events.c index d128f65778e6..1d18315dc836 100644 --- a/kernel/trace/trace_events.c +++ b/kernel/trace/trace_events.c | |||
@@ -93,9 +93,7 @@ int trace_define_common_fields(struct ftrace_event_call *call) | |||
93 | } | 93 | } |
94 | EXPORT_SYMBOL_GPL(trace_define_common_fields); | 94 | EXPORT_SYMBOL_GPL(trace_define_common_fields); |
95 | 95 | ||
96 | #ifdef CONFIG_MODULES | 96 | void trace_destroy_fields(struct ftrace_event_call *call) |
97 | |||
98 | static void trace_destroy_fields(struct ftrace_event_call *call) | ||
99 | { | 97 | { |
100 | struct ftrace_event_field *field, *next; | 98 | struct ftrace_event_field *field, *next; |
101 | 99 | ||
@@ -107,8 +105,6 @@ static void trace_destroy_fields(struct ftrace_event_call *call) | |||
107 | } | 105 | } |
108 | } | 106 | } |
109 | 107 | ||
110 | #endif /* CONFIG_MODULES */ | ||
111 | |||
112 | static void ftrace_event_enable_disable(struct ftrace_event_call *call, | 108 | static void ftrace_event_enable_disable(struct ftrace_event_call *call, |
113 | int enable) | 109 | int enable) |
114 | { | 110 | { |
@@ -117,14 +113,14 @@ static void ftrace_event_enable_disable(struct ftrace_event_call *call, | |||
117 | if (call->enabled) { | 113 | if (call->enabled) { |
118 | call->enabled = 0; | 114 | call->enabled = 0; |
119 | tracing_stop_cmdline_record(); | 115 | tracing_stop_cmdline_record(); |
120 | call->unregfunc(call->data); | 116 | call->unregfunc(call); |
121 | } | 117 | } |
122 | break; | 118 | break; |
123 | case 1: | 119 | case 1: |
124 | if (!call->enabled) { | 120 | if (!call->enabled) { |
125 | call->enabled = 1; | 121 | call->enabled = 1; |
126 | tracing_start_cmdline_record(); | 122 | tracing_start_cmdline_record(); |
127 | call->regfunc(call->data); | 123 | call->regfunc(call); |
128 | } | 124 | } |
129 | break; | 125 | break; |
130 | } | 126 | } |
@@ -507,7 +503,7 @@ extern char *__bad_type_size(void); | |||
507 | #define FIELD(type, name) \ | 503 | #define FIELD(type, name) \ |
508 | sizeof(type) != sizeof(field.name) ? __bad_type_size() : \ | 504 | sizeof(type) != sizeof(field.name) ? __bad_type_size() : \ |
509 | #type, "common_" #name, offsetof(typeof(field), name), \ | 505 | #type, "common_" #name, offsetof(typeof(field), name), \ |
510 | sizeof(field.name) | 506 | sizeof(field.name), is_signed_type(type) |
511 | 507 | ||
512 | static int trace_write_header(struct trace_seq *s) | 508 | static int trace_write_header(struct trace_seq *s) |
513 | { | 509 | { |
@@ -515,17 +511,17 @@ static int trace_write_header(struct trace_seq *s) | |||
515 | 511 | ||
516 | /* struct trace_entry */ | 512 | /* struct trace_entry */ |
517 | return trace_seq_printf(s, | 513 | return trace_seq_printf(s, |
518 | "\tfield:%s %s;\toffset:%zu;\tsize:%zu;\n" | 514 | "\tfield:%s %s;\toffset:%zu;\tsize:%zu;\tsigned:%u;\n" |
519 | "\tfield:%s %s;\toffset:%zu;\tsize:%zu;\n" | 515 | "\tfield:%s %s;\toffset:%zu;\tsize:%zu;\tsigned:%u;\n" |
520 | "\tfield:%s %s;\toffset:%zu;\tsize:%zu;\n" | 516 | "\tfield:%s %s;\toffset:%zu;\tsize:%zu;\tsigned:%u;\n" |
521 | "\tfield:%s %s;\toffset:%zu;\tsize:%zu;\n" | 517 | "\tfield:%s %s;\toffset:%zu;\tsize:%zu;\tsigned:%u;\n" |
522 | "\tfield:%s %s;\toffset:%zu;\tsize:%zu;\n" | 518 | "\tfield:%s %s;\toffset:%zu;\tsize:%zu;\tsigned:%u;\n" |
523 | "\n", | 519 | "\n", |
524 | FIELD(unsigned short, type), | 520 | FIELD(unsigned short, type), |
525 | FIELD(unsigned char, flags), | 521 | FIELD(unsigned char, flags), |
526 | FIELD(unsigned char, preempt_count), | 522 | FIELD(unsigned char, preempt_count), |
527 | FIELD(int, pid), | 523 | FIELD(int, pid), |
528 | FIELD(int, lock_depth)); | 524 | FIELD(int, lock_depth)); |
529 | } | 525 | } |
530 | 526 | ||
531 | static ssize_t | 527 | static ssize_t |
@@ -878,9 +874,9 @@ event_subsystem_dir(const char *name, struct dentry *d_events) | |||
878 | "'%s/filter' entry\n", name); | 874 | "'%s/filter' entry\n", name); |
879 | } | 875 | } |
880 | 876 | ||
881 | entry = trace_create_file("enable", 0644, system->entry, | 877 | trace_create_file("enable", 0644, system->entry, |
882 | (void *)system->name, | 878 | (void *)system->name, |
883 | &ftrace_system_enable_fops); | 879 | &ftrace_system_enable_fops); |
884 | 880 | ||
885 | return system->entry; | 881 | return system->entry; |
886 | } | 882 | } |
@@ -892,7 +888,6 @@ event_create_dir(struct ftrace_event_call *call, struct dentry *d_events, | |||
892 | const struct file_operations *filter, | 888 | const struct file_operations *filter, |
893 | const struct file_operations *format) | 889 | const struct file_operations *format) |
894 | { | 890 | { |
895 | struct dentry *entry; | ||
896 | int ret; | 891 | int ret; |
897 | 892 | ||
898 | /* | 893 | /* |
@@ -910,12 +905,12 @@ event_create_dir(struct ftrace_event_call *call, struct dentry *d_events, | |||
910 | } | 905 | } |
911 | 906 | ||
912 | if (call->regfunc) | 907 | if (call->regfunc) |
913 | entry = trace_create_file("enable", 0644, call->dir, call, | 908 | trace_create_file("enable", 0644, call->dir, call, |
914 | enable); | 909 | enable); |
915 | 910 | ||
916 | if (call->id && call->profile_enable) | 911 | if (call->id && call->profile_enable) |
917 | entry = trace_create_file("id", 0444, call->dir, call, | 912 | trace_create_file("id", 0444, call->dir, call, |
918 | id); | 913 | id); |
919 | 914 | ||
920 | if (call->define_fields) { | 915 | if (call->define_fields) { |
921 | ret = call->define_fields(call); | 916 | ret = call->define_fields(call); |
@@ -924,41 +919,60 @@ event_create_dir(struct ftrace_event_call *call, struct dentry *d_events, | |||
924 | " events/%s\n", call->name); | 919 | " events/%s\n", call->name); |
925 | return ret; | 920 | return ret; |
926 | } | 921 | } |
927 | entry = trace_create_file("filter", 0644, call->dir, call, | 922 | trace_create_file("filter", 0644, call->dir, call, |
928 | filter); | 923 | filter); |
929 | } | 924 | } |
930 | 925 | ||
931 | /* A trace may not want to export its format */ | 926 | /* A trace may not want to export its format */ |
932 | if (!call->show_format) | 927 | if (!call->show_format) |
933 | return 0; | 928 | return 0; |
934 | 929 | ||
935 | entry = trace_create_file("format", 0444, call->dir, call, | 930 | trace_create_file("format", 0444, call->dir, call, |
936 | format); | 931 | format); |
937 | 932 | ||
938 | return 0; | 933 | return 0; |
939 | } | 934 | } |
940 | 935 | ||
941 | #define for_each_event(event, start, end) \ | 936 | static int __trace_add_event_call(struct ftrace_event_call *call) |
942 | for (event = start; \ | 937 | { |
943 | (unsigned long)event < (unsigned long)end; \ | 938 | struct dentry *d_events; |
944 | event++) | 939 | int ret; |
945 | 940 | ||
946 | #ifdef CONFIG_MODULES | 941 | if (!call->name) |
942 | return -EINVAL; | ||
947 | 943 | ||
948 | static LIST_HEAD(ftrace_module_file_list); | 944 | if (call->raw_init) { |
945 | ret = call->raw_init(call); | ||
946 | if (ret < 0) { | ||
947 | if (ret != -ENOSYS) | ||
948 | pr_warning("Could not initialize trace " | ||
949 | "events/%s\n", call->name); | ||
950 | return ret; | ||
951 | } | ||
952 | } | ||
949 | 953 | ||
950 | /* | 954 | d_events = event_trace_events_dir(); |
951 | * Modules must own their file_operations to keep up with | 955 | if (!d_events) |
952 | * reference counting. | 956 | return -ENOENT; |
953 | */ | 957 | |
954 | struct ftrace_module_file_ops { | 958 | ret = event_create_dir(call, d_events, &ftrace_event_id_fops, |
955 | struct list_head list; | 959 | &ftrace_enable_fops, &ftrace_event_filter_fops, |
956 | struct module *mod; | 960 | &ftrace_event_format_fops); |
957 | struct file_operations id; | 961 | if (!ret) |
958 | struct file_operations enable; | 962 | list_add(&call->list, &ftrace_events); |
959 | struct file_operations format; | 963 | |
960 | struct file_operations filter; | 964 | return ret; |
961 | }; | 965 | } |
966 | |||
967 | /* Add an additional event_call dynamically */ | ||
968 | int trace_add_event_call(struct ftrace_event_call *call) | ||
969 | { | ||
970 | int ret; | ||
971 | mutex_lock(&event_mutex); | ||
972 | ret = __trace_add_event_call(call); | ||
973 | mutex_unlock(&event_mutex); | ||
974 | return ret; | ||
975 | } | ||
962 | 976 | ||
963 | static void remove_subsystem_dir(const char *name) | 977 | static void remove_subsystem_dir(const char *name) |
964 | { | 978 | { |
@@ -986,6 +1000,53 @@ static void remove_subsystem_dir(const char *name) | |||
986 | } | 1000 | } |
987 | } | 1001 | } |
988 | 1002 | ||
1003 | /* | ||
1004 | * Must be called under locking both of event_mutex and trace_event_mutex. | ||
1005 | */ | ||
1006 | static void __trace_remove_event_call(struct ftrace_event_call *call) | ||
1007 | { | ||
1008 | ftrace_event_enable_disable(call, 0); | ||
1009 | if (call->event) | ||
1010 | __unregister_ftrace_event(call->event); | ||
1011 | debugfs_remove_recursive(call->dir); | ||
1012 | list_del(&call->list); | ||
1013 | trace_destroy_fields(call); | ||
1014 | destroy_preds(call); | ||
1015 | remove_subsystem_dir(call->system); | ||
1016 | } | ||
1017 | |||
1018 | /* Remove an event_call */ | ||
1019 | void trace_remove_event_call(struct ftrace_event_call *call) | ||
1020 | { | ||
1021 | mutex_lock(&event_mutex); | ||
1022 | down_write(&trace_event_mutex); | ||
1023 | __trace_remove_event_call(call); | ||
1024 | up_write(&trace_event_mutex); | ||
1025 | mutex_unlock(&event_mutex); | ||
1026 | } | ||
1027 | |||
1028 | #define for_each_event(event, start, end) \ | ||
1029 | for (event = start; \ | ||
1030 | (unsigned long)event < (unsigned long)end; \ | ||
1031 | event++) | ||
1032 | |||
1033 | #ifdef CONFIG_MODULES | ||
1034 | |||
1035 | static LIST_HEAD(ftrace_module_file_list); | ||
1036 | |||
1037 | /* | ||
1038 | * Modules must own their file_operations to keep up with | ||
1039 | * reference counting. | ||
1040 | */ | ||
1041 | struct ftrace_module_file_ops { | ||
1042 | struct list_head list; | ||
1043 | struct module *mod; | ||
1044 | struct file_operations id; | ||
1045 | struct file_operations enable; | ||
1046 | struct file_operations format; | ||
1047 | struct file_operations filter; | ||
1048 | }; | ||
1049 | |||
989 | static struct ftrace_module_file_ops * | 1050 | static struct ftrace_module_file_ops * |
990 | trace_create_file_ops(struct module *mod) | 1051 | trace_create_file_ops(struct module *mod) |
991 | { | 1052 | { |
@@ -1043,7 +1104,7 @@ static void trace_module_add_events(struct module *mod) | |||
1043 | if (!call->name) | 1104 | if (!call->name) |
1044 | continue; | 1105 | continue; |
1045 | if (call->raw_init) { | 1106 | if (call->raw_init) { |
1046 | ret = call->raw_init(); | 1107 | ret = call->raw_init(call); |
1047 | if (ret < 0) { | 1108 | if (ret < 0) { |
1048 | if (ret != -ENOSYS) | 1109 | if (ret != -ENOSYS) |
1049 | pr_warning("Could not initialize trace " | 1110 | pr_warning("Could not initialize trace " |
@@ -1061,10 +1122,11 @@ static void trace_module_add_events(struct module *mod) | |||
1061 | return; | 1122 | return; |
1062 | } | 1123 | } |
1063 | call->mod = mod; | 1124 | call->mod = mod; |
1064 | list_add(&call->list, &ftrace_events); | 1125 | ret = event_create_dir(call, d_events, |
1065 | event_create_dir(call, d_events, | 1126 | &file_ops->id, &file_ops->enable, |
1066 | &file_ops->id, &file_ops->enable, | 1127 | &file_ops->filter, &file_ops->format); |
1067 | &file_ops->filter, &file_ops->format); | 1128 | if (!ret) |
1129 | list_add(&call->list, &ftrace_events); | ||
1068 | } | 1130 | } |
1069 | } | 1131 | } |
1070 | 1132 | ||
@@ -1078,14 +1140,7 @@ static void trace_module_remove_events(struct module *mod) | |||
1078 | list_for_each_entry_safe(call, p, &ftrace_events, list) { | 1140 | list_for_each_entry_safe(call, p, &ftrace_events, list) { |
1079 | if (call->mod == mod) { | 1141 | if (call->mod == mod) { |
1080 | found = true; | 1142 | found = true; |
1081 | ftrace_event_enable_disable(call, 0); | 1143 | __trace_remove_event_call(call); |
1082 | if (call->event) | ||
1083 | __unregister_ftrace_event(call->event); | ||
1084 | debugfs_remove_recursive(call->dir); | ||
1085 | list_del(&call->list); | ||
1086 | trace_destroy_fields(call); | ||
1087 | destroy_preds(call); | ||
1088 | remove_subsystem_dir(call->system); | ||
1089 | } | 1144 | } |
1090 | } | 1145 | } |
1091 | 1146 | ||
@@ -1203,7 +1258,7 @@ static __init int event_trace_init(void) | |||
1203 | if (!call->name) | 1258 | if (!call->name) |
1204 | continue; | 1259 | continue; |
1205 | if (call->raw_init) { | 1260 | if (call->raw_init) { |
1206 | ret = call->raw_init(); | 1261 | ret = call->raw_init(call); |
1207 | if (ret < 0) { | 1262 | if (ret < 0) { |
1208 | if (ret != -ENOSYS) | 1263 | if (ret != -ENOSYS) |
1209 | pr_warning("Could not initialize trace " | 1264 | pr_warning("Could not initialize trace " |
@@ -1211,10 +1266,12 @@ static __init int event_trace_init(void) | |||
1211 | continue; | 1266 | continue; |
1212 | } | 1267 | } |
1213 | } | 1268 | } |
1214 | list_add(&call->list, &ftrace_events); | 1269 | ret = event_create_dir(call, d_events, &ftrace_event_id_fops, |
1215 | event_create_dir(call, d_events, &ftrace_event_id_fops, | 1270 | &ftrace_enable_fops, |
1216 | &ftrace_enable_fops, &ftrace_event_filter_fops, | 1271 | &ftrace_event_filter_fops, |
1217 | &ftrace_event_format_fops); | 1272 | &ftrace_event_format_fops); |
1273 | if (!ret) | ||
1274 | list_add(&call->list, &ftrace_events); | ||
1218 | } | 1275 | } |
1219 | 1276 | ||
1220 | while (true) { | 1277 | while (true) { |
diff --git a/kernel/trace/trace_events_filter.c b/kernel/trace/trace_events_filter.c index 98a6cc5c64ed..50504cb228de 100644 --- a/kernel/trace/trace_events_filter.c +++ b/kernel/trace/trace_events_filter.c | |||
@@ -18,11 +18,10 @@ | |||
18 | * Copyright (C) 2009 Tom Zanussi <tzanussi@gmail.com> | 18 | * Copyright (C) 2009 Tom Zanussi <tzanussi@gmail.com> |
19 | */ | 19 | */ |
20 | 20 | ||
21 | #include <linux/debugfs.h> | ||
22 | #include <linux/uaccess.h> | ||
23 | #include <linux/module.h> | 21 | #include <linux/module.h> |
24 | #include <linux/ctype.h> | 22 | #include <linux/ctype.h> |
25 | #include <linux/mutex.h> | 23 | #include <linux/mutex.h> |
24 | #include <linux/perf_event.h> | ||
26 | 25 | ||
27 | #include "trace.h" | 26 | #include "trace.h" |
28 | #include "trace_output.h" | 27 | #include "trace_output.h" |
@@ -31,6 +30,7 @@ enum filter_op_ids | |||
31 | { | 30 | { |
32 | OP_OR, | 31 | OP_OR, |
33 | OP_AND, | 32 | OP_AND, |
33 | OP_GLOB, | ||
34 | OP_NE, | 34 | OP_NE, |
35 | OP_EQ, | 35 | OP_EQ, |
36 | OP_LT, | 36 | OP_LT, |
@@ -48,16 +48,17 @@ struct filter_op { | |||
48 | }; | 48 | }; |
49 | 49 | ||
50 | static struct filter_op filter_ops[] = { | 50 | static struct filter_op filter_ops[] = { |
51 | { OP_OR, "||", 1 }, | 51 | { OP_OR, "||", 1 }, |
52 | { OP_AND, "&&", 2 }, | 52 | { OP_AND, "&&", 2 }, |
53 | { OP_NE, "!=", 4 }, | 53 | { OP_GLOB, "~", 4 }, |
54 | { OP_EQ, "==", 4 }, | 54 | { OP_NE, "!=", 4 }, |
55 | { OP_LT, "<", 5 }, | 55 | { OP_EQ, "==", 4 }, |
56 | { OP_LE, "<=", 5 }, | 56 | { OP_LT, "<", 5 }, |
57 | { OP_GT, ">", 5 }, | 57 | { OP_LE, "<=", 5 }, |
58 | { OP_GE, ">=", 5 }, | 58 | { OP_GT, ">", 5 }, |
59 | { OP_NONE, "OP_NONE", 0 }, | 59 | { OP_GE, ">=", 5 }, |
60 | { OP_OPEN_PAREN, "(", 0 }, | 60 | { OP_NONE, "OP_NONE", 0 }, |
61 | { OP_OPEN_PAREN, "(", 0 }, | ||
61 | }; | 62 | }; |
62 | 63 | ||
63 | enum { | 64 | enum { |
@@ -197,9 +198,9 @@ static int filter_pred_string(struct filter_pred *pred, void *event, | |||
197 | char *addr = (char *)(event + pred->offset); | 198 | char *addr = (char *)(event + pred->offset); |
198 | int cmp, match; | 199 | int cmp, match; |
199 | 200 | ||
200 | cmp = strncmp(addr, pred->str_val, pred->str_len); | 201 | cmp = pred->regex.match(addr, &pred->regex, pred->regex.field_len); |
201 | 202 | ||
202 | match = (!cmp) ^ pred->not; | 203 | match = cmp ^ pred->not; |
203 | 204 | ||
204 | return match; | 205 | return match; |
205 | } | 206 | } |
@@ -211,9 +212,9 @@ static int filter_pred_pchar(struct filter_pred *pred, void *event, | |||
211 | char **addr = (char **)(event + pred->offset); | 212 | char **addr = (char **)(event + pred->offset); |
212 | int cmp, match; | 213 | int cmp, match; |
213 | 214 | ||
214 | cmp = strncmp(*addr, pred->str_val, pred->str_len); | 215 | cmp = pred->regex.match(*addr, &pred->regex, pred->regex.field_len); |
215 | 216 | ||
216 | match = (!cmp) ^ pred->not; | 217 | match = cmp ^ pred->not; |
217 | 218 | ||
218 | return match; | 219 | return match; |
219 | } | 220 | } |
@@ -237,9 +238,9 @@ static int filter_pred_strloc(struct filter_pred *pred, void *event, | |||
237 | char *addr = (char *)(event + str_loc); | 238 | char *addr = (char *)(event + str_loc); |
238 | int cmp, match; | 239 | int cmp, match; |
239 | 240 | ||
240 | cmp = strncmp(addr, pred->str_val, str_len); | 241 | cmp = pred->regex.match(addr, &pred->regex, str_len); |
241 | 242 | ||
242 | match = (!cmp) ^ pred->not; | 243 | match = cmp ^ pred->not; |
243 | 244 | ||
244 | return match; | 245 | return match; |
245 | } | 246 | } |
@@ -250,10 +251,121 @@ static int filter_pred_none(struct filter_pred *pred, void *event, | |||
250 | return 0; | 251 | return 0; |
251 | } | 252 | } |
252 | 253 | ||
254 | /* Basic regex callbacks */ | ||
255 | static int regex_match_full(char *str, struct regex *r, int len) | ||
256 | { | ||
257 | if (strncmp(str, r->pattern, len) == 0) | ||
258 | return 1; | ||
259 | return 0; | ||
260 | } | ||
261 | |||
262 | static int regex_match_front(char *str, struct regex *r, int len) | ||
263 | { | ||
264 | if (strncmp(str, r->pattern, len) == 0) | ||
265 | return 1; | ||
266 | return 0; | ||
267 | } | ||
268 | |||
269 | static int regex_match_middle(char *str, struct regex *r, int len) | ||
270 | { | ||
271 | if (strstr(str, r->pattern)) | ||
272 | return 1; | ||
273 | return 0; | ||
274 | } | ||
275 | |||
276 | static int regex_match_end(char *str, struct regex *r, int len) | ||
277 | { | ||
278 | char *ptr = strstr(str, r->pattern); | ||
279 | |||
280 | if (ptr && (ptr[r->len] == 0)) | ||
281 | return 1; | ||
282 | return 0; | ||
283 | } | ||
284 | |||
285 | /** | ||
286 | * filter_parse_regex - parse a basic regex | ||
287 | * @buff: the raw regex | ||
288 | * @len: length of the regex | ||
289 | * @search: will point to the beginning of the string to compare | ||
290 | * @not: tell whether the match will have to be inverted | ||
291 | * | ||
292 | * This passes in a buffer containing a regex and this function will | ||
293 | * set search to point to the search part of the buffer and | ||
294 | * return the type of search it is (see enum above). | ||
295 | * This does modify buff. | ||
296 | * | ||
297 | * Returns enum type. | ||
298 | * search returns the pointer to use for comparison. | ||
299 | * not returns 1 if buff started with a '!' | ||
300 | * 0 otherwise. | ||
301 | */ | ||
302 | enum regex_type filter_parse_regex(char *buff, int len, char **search, int *not) | ||
303 | { | ||
304 | int type = MATCH_FULL; | ||
305 | int i; | ||
306 | |||
307 | if (buff[0] == '!') { | ||
308 | *not = 1; | ||
309 | buff++; | ||
310 | len--; | ||
311 | } else | ||
312 | *not = 0; | ||
313 | |||
314 | *search = buff; | ||
315 | |||
316 | for (i = 0; i < len; i++) { | ||
317 | if (buff[i] == '*') { | ||
318 | if (!i) { | ||
319 | *search = buff + 1; | ||
320 | type = MATCH_END_ONLY; | ||
321 | } else { | ||
322 | if (type == MATCH_END_ONLY) | ||
323 | type = MATCH_MIDDLE_ONLY; | ||
324 | else | ||
325 | type = MATCH_FRONT_ONLY; | ||
326 | buff[i] = 0; | ||
327 | break; | ||
328 | } | ||
329 | } | ||
330 | } | ||
331 | |||
332 | return type; | ||
333 | } | ||
334 | |||
335 | static void filter_build_regex(struct filter_pred *pred) | ||
336 | { | ||
337 | struct regex *r = &pred->regex; | ||
338 | char *search; | ||
339 | enum regex_type type = MATCH_FULL; | ||
340 | int not = 0; | ||
341 | |||
342 | if (pred->op == OP_GLOB) { | ||
343 | type = filter_parse_regex(r->pattern, r->len, &search, ¬); | ||
344 | r->len = strlen(search); | ||
345 | memmove(r->pattern, search, r->len+1); | ||
346 | } | ||
347 | |||
348 | switch (type) { | ||
349 | case MATCH_FULL: | ||
350 | r->match = regex_match_full; | ||
351 | break; | ||
352 | case MATCH_FRONT_ONLY: | ||
353 | r->match = regex_match_front; | ||
354 | break; | ||
355 | case MATCH_MIDDLE_ONLY: | ||
356 | r->match = regex_match_middle; | ||
357 | break; | ||
358 | case MATCH_END_ONLY: | ||
359 | r->match = regex_match_end; | ||
360 | break; | ||
361 | } | ||
362 | |||
363 | pred->not ^= not; | ||
364 | } | ||
365 | |||
253 | /* return 1 if event matches, 0 otherwise (discard) */ | 366 | /* return 1 if event matches, 0 otherwise (discard) */ |
254 | int filter_match_preds(struct ftrace_event_call *call, void *rec) | 367 | int filter_match_preds(struct event_filter *filter, void *rec) |
255 | { | 368 | { |
256 | struct event_filter *filter = call->filter; | ||
257 | int match, top = 0, val1 = 0, val2 = 0; | 369 | int match, top = 0, val1 = 0, val2 = 0; |
258 | int stack[MAX_FILTER_PRED]; | 370 | int stack[MAX_FILTER_PRED]; |
259 | struct filter_pred *pred; | 371 | struct filter_pred *pred; |
@@ -396,7 +508,7 @@ static void filter_clear_pred(struct filter_pred *pred) | |||
396 | { | 508 | { |
397 | kfree(pred->field_name); | 509 | kfree(pred->field_name); |
398 | pred->field_name = NULL; | 510 | pred->field_name = NULL; |
399 | pred->str_len = 0; | 511 | pred->regex.len = 0; |
400 | } | 512 | } |
401 | 513 | ||
402 | static int filter_set_pred(struct filter_pred *dest, | 514 | static int filter_set_pred(struct filter_pred *dest, |
@@ -426,9 +538,8 @@ static void filter_disable_preds(struct ftrace_event_call *call) | |||
426 | filter->preds[i]->fn = filter_pred_none; | 538 | filter->preds[i]->fn = filter_pred_none; |
427 | } | 539 | } |
428 | 540 | ||
429 | void destroy_preds(struct ftrace_event_call *call) | 541 | static void __free_preds(struct event_filter *filter) |
430 | { | 542 | { |
431 | struct event_filter *filter = call->filter; | ||
432 | int i; | 543 | int i; |
433 | 544 | ||
434 | if (!filter) | 545 | if (!filter) |
@@ -441,21 +552,24 @@ void destroy_preds(struct ftrace_event_call *call) | |||
441 | kfree(filter->preds); | 552 | kfree(filter->preds); |
442 | kfree(filter->filter_string); | 553 | kfree(filter->filter_string); |
443 | kfree(filter); | 554 | kfree(filter); |
555 | } | ||
556 | |||
557 | void destroy_preds(struct ftrace_event_call *call) | ||
558 | { | ||
559 | __free_preds(call->filter); | ||
444 | call->filter = NULL; | 560 | call->filter = NULL; |
561 | call->filter_active = 0; | ||
445 | } | 562 | } |
446 | 563 | ||
447 | static int init_preds(struct ftrace_event_call *call) | 564 | static struct event_filter *__alloc_preds(void) |
448 | { | 565 | { |
449 | struct event_filter *filter; | 566 | struct event_filter *filter; |
450 | struct filter_pred *pred; | 567 | struct filter_pred *pred; |
451 | int i; | 568 | int i; |
452 | 569 | ||
453 | if (call->filter) | 570 | filter = kzalloc(sizeof(*filter), GFP_KERNEL); |
454 | return 0; | 571 | if (!filter) |
455 | 572 | return ERR_PTR(-ENOMEM); | |
456 | filter = call->filter = kzalloc(sizeof(*filter), GFP_KERNEL); | ||
457 | if (!call->filter) | ||
458 | return -ENOMEM; | ||
459 | 573 | ||
460 | filter->n_preds = 0; | 574 | filter->n_preds = 0; |
461 | 575 | ||
@@ -471,12 +585,24 @@ static int init_preds(struct ftrace_event_call *call) | |||
471 | filter->preds[i] = pred; | 585 | filter->preds[i] = pred; |
472 | } | 586 | } |
473 | 587 | ||
474 | return 0; | 588 | return filter; |
475 | 589 | ||
476 | oom: | 590 | oom: |
477 | destroy_preds(call); | 591 | __free_preds(filter); |
592 | return ERR_PTR(-ENOMEM); | ||
593 | } | ||
478 | 594 | ||
479 | return -ENOMEM; | 595 | static int init_preds(struct ftrace_event_call *call) |
596 | { | ||
597 | if (call->filter) | ||
598 | return 0; | ||
599 | |||
600 | call->filter_active = 0; | ||
601 | call->filter = __alloc_preds(); | ||
602 | if (IS_ERR(call->filter)) | ||
603 | return PTR_ERR(call->filter); | ||
604 | |||
605 | return 0; | ||
480 | } | 606 | } |
481 | 607 | ||
482 | static int init_subsystem_preds(struct event_subsystem *system) | 608 | static int init_subsystem_preds(struct event_subsystem *system) |
@@ -499,14 +625,7 @@ static int init_subsystem_preds(struct event_subsystem *system) | |||
499 | return 0; | 625 | return 0; |
500 | } | 626 | } |
501 | 627 | ||
502 | enum { | 628 | static void filter_free_subsystem_preds(struct event_subsystem *system) |
503 | FILTER_DISABLE_ALL, | ||
504 | FILTER_INIT_NO_RESET, | ||
505 | FILTER_SKIP_NO_RESET, | ||
506 | }; | ||
507 | |||
508 | static void filter_free_subsystem_preds(struct event_subsystem *system, | ||
509 | int flag) | ||
510 | { | 629 | { |
511 | struct ftrace_event_call *call; | 630 | struct ftrace_event_call *call; |
512 | 631 | ||
@@ -517,14 +636,6 @@ static void filter_free_subsystem_preds(struct event_subsystem *system, | |||
517 | if (strcmp(call->system, system->name) != 0) | 636 | if (strcmp(call->system, system->name) != 0) |
518 | continue; | 637 | continue; |
519 | 638 | ||
520 | if (flag == FILTER_INIT_NO_RESET) { | ||
521 | call->filter->no_reset = false; | ||
522 | continue; | ||
523 | } | ||
524 | |||
525 | if (flag == FILTER_SKIP_NO_RESET && call->filter->no_reset) | ||
526 | continue; | ||
527 | |||
528 | filter_disable_preds(call); | 639 | filter_disable_preds(call); |
529 | remove_filter_string(call->filter); | 640 | remove_filter_string(call->filter); |
530 | } | 641 | } |
@@ -532,10 +643,10 @@ static void filter_free_subsystem_preds(struct event_subsystem *system, | |||
532 | 643 | ||
533 | static int filter_add_pred_fn(struct filter_parse_state *ps, | 644 | static int filter_add_pred_fn(struct filter_parse_state *ps, |
534 | struct ftrace_event_call *call, | 645 | struct ftrace_event_call *call, |
646 | struct event_filter *filter, | ||
535 | struct filter_pred *pred, | 647 | struct filter_pred *pred, |
536 | filter_pred_fn_t fn) | 648 | filter_pred_fn_t fn) |
537 | { | 649 | { |
538 | struct event_filter *filter = call->filter; | ||
539 | int idx, err; | 650 | int idx, err; |
540 | 651 | ||
541 | if (filter->n_preds == MAX_FILTER_PRED) { | 652 | if (filter->n_preds == MAX_FILTER_PRED) { |
@@ -550,7 +661,6 @@ static int filter_add_pred_fn(struct filter_parse_state *ps, | |||
550 | return err; | 661 | return err; |
551 | 662 | ||
552 | filter->n_preds++; | 663 | filter->n_preds++; |
553 | call->filter_active = 1; | ||
554 | 664 | ||
555 | return 0; | 665 | return 0; |
556 | } | 666 | } |
@@ -575,7 +685,10 @@ static bool is_string_field(struct ftrace_event_field *field) | |||
575 | 685 | ||
576 | static int is_legal_op(struct ftrace_event_field *field, int op) | 686 | static int is_legal_op(struct ftrace_event_field *field, int op) |
577 | { | 687 | { |
578 | if (is_string_field(field) && (op != OP_EQ && op != OP_NE)) | 688 | if (is_string_field(field) && |
689 | (op != OP_EQ && op != OP_NE && op != OP_GLOB)) | ||
690 | return 0; | ||
691 | if (!is_string_field(field) && op == OP_GLOB) | ||
579 | return 0; | 692 | return 0; |
580 | 693 | ||
581 | return 1; | 694 | return 1; |
@@ -626,6 +739,7 @@ static filter_pred_fn_t select_comparison_fn(int op, int field_size, | |||
626 | 739 | ||
627 | static int filter_add_pred(struct filter_parse_state *ps, | 740 | static int filter_add_pred(struct filter_parse_state *ps, |
628 | struct ftrace_event_call *call, | 741 | struct ftrace_event_call *call, |
742 | struct event_filter *filter, | ||
629 | struct filter_pred *pred, | 743 | struct filter_pred *pred, |
630 | bool dry_run) | 744 | bool dry_run) |
631 | { | 745 | { |
@@ -660,21 +774,22 @@ static int filter_add_pred(struct filter_parse_state *ps, | |||
660 | } | 774 | } |
661 | 775 | ||
662 | if (is_string_field(field)) { | 776 | if (is_string_field(field)) { |
663 | pred->str_len = field->size; | 777 | filter_build_regex(pred); |
664 | 778 | ||
665 | if (field->filter_type == FILTER_STATIC_STRING) | 779 | if (field->filter_type == FILTER_STATIC_STRING) { |
666 | fn = filter_pred_string; | 780 | fn = filter_pred_string; |
667 | else if (field->filter_type == FILTER_DYN_STRING) | 781 | pred->regex.field_len = field->size; |
782 | } else if (field->filter_type == FILTER_DYN_STRING) | ||
668 | fn = filter_pred_strloc; | 783 | fn = filter_pred_strloc; |
669 | else { | 784 | else { |
670 | fn = filter_pred_pchar; | 785 | fn = filter_pred_pchar; |
671 | pred->str_len = strlen(pred->str_val); | 786 | pred->regex.field_len = strlen(pred->regex.pattern); |
672 | } | 787 | } |
673 | } else { | 788 | } else { |
674 | if (field->is_signed) | 789 | if (field->is_signed) |
675 | ret = strict_strtoll(pred->str_val, 0, &val); | 790 | ret = strict_strtoll(pred->regex.pattern, 0, &val); |
676 | else | 791 | else |
677 | ret = strict_strtoull(pred->str_val, 0, &val); | 792 | ret = strict_strtoull(pred->regex.pattern, 0, &val); |
678 | if (ret) { | 793 | if (ret) { |
679 | parse_error(ps, FILT_ERR_ILLEGAL_INTVAL, 0); | 794 | parse_error(ps, FILT_ERR_ILLEGAL_INTVAL, 0); |
680 | return -EINVAL; | 795 | return -EINVAL; |
@@ -694,45 +809,7 @@ static int filter_add_pred(struct filter_parse_state *ps, | |||
694 | 809 | ||
695 | add_pred_fn: | 810 | add_pred_fn: |
696 | if (!dry_run) | 811 | if (!dry_run) |
697 | return filter_add_pred_fn(ps, call, pred, fn); | 812 | return filter_add_pred_fn(ps, call, filter, pred, fn); |
698 | return 0; | ||
699 | } | ||
700 | |||
701 | static int filter_add_subsystem_pred(struct filter_parse_state *ps, | ||
702 | struct event_subsystem *system, | ||
703 | struct filter_pred *pred, | ||
704 | char *filter_string, | ||
705 | bool dry_run) | ||
706 | { | ||
707 | struct ftrace_event_call *call; | ||
708 | int err = 0; | ||
709 | bool fail = true; | ||
710 | |||
711 | list_for_each_entry(call, &ftrace_events, list) { | ||
712 | |||
713 | if (!call->define_fields) | ||
714 | continue; | ||
715 | |||
716 | if (strcmp(call->system, system->name)) | ||
717 | continue; | ||
718 | |||
719 | if (call->filter->no_reset) | ||
720 | continue; | ||
721 | |||
722 | err = filter_add_pred(ps, call, pred, dry_run); | ||
723 | if (err) | ||
724 | call->filter->no_reset = true; | ||
725 | else | ||
726 | fail = false; | ||
727 | |||
728 | if (!dry_run) | ||
729 | replace_filter_string(call->filter, filter_string); | ||
730 | } | ||
731 | |||
732 | if (fail) { | ||
733 | parse_error(ps, FILT_ERR_BAD_SUBSYS_FILTER, 0); | ||
734 | return err; | ||
735 | } | ||
736 | return 0; | 813 | return 0; |
737 | } | 814 | } |
738 | 815 | ||
@@ -1045,8 +1122,8 @@ static struct filter_pred *create_pred(int op, char *operand1, char *operand2) | |||
1045 | return NULL; | 1122 | return NULL; |
1046 | } | 1123 | } |
1047 | 1124 | ||
1048 | strcpy(pred->str_val, operand2); | 1125 | strcpy(pred->regex.pattern, operand2); |
1049 | pred->str_len = strlen(operand2); | 1126 | pred->regex.len = strlen(pred->regex.pattern); |
1050 | 1127 | ||
1051 | pred->op = op; | 1128 | pred->op = op; |
1052 | 1129 | ||
@@ -1090,8 +1167,8 @@ static int check_preds(struct filter_parse_state *ps) | |||
1090 | return 0; | 1167 | return 0; |
1091 | } | 1168 | } |
1092 | 1169 | ||
1093 | static int replace_preds(struct event_subsystem *system, | 1170 | static int replace_preds(struct ftrace_event_call *call, |
1094 | struct ftrace_event_call *call, | 1171 | struct event_filter *filter, |
1095 | struct filter_parse_state *ps, | 1172 | struct filter_parse_state *ps, |
1096 | char *filter_string, | 1173 | char *filter_string, |
1097 | bool dry_run) | 1174 | bool dry_run) |
@@ -1138,11 +1215,7 @@ static int replace_preds(struct event_subsystem *system, | |||
1138 | add_pred: | 1215 | add_pred: |
1139 | if (!pred) | 1216 | if (!pred) |
1140 | return -ENOMEM; | 1217 | return -ENOMEM; |
1141 | if (call) | 1218 | err = filter_add_pred(ps, call, filter, pred, dry_run); |
1142 | err = filter_add_pred(ps, call, pred, false); | ||
1143 | else | ||
1144 | err = filter_add_subsystem_pred(ps, system, pred, | ||
1145 | filter_string, dry_run); | ||
1146 | filter_free_pred(pred); | 1219 | filter_free_pred(pred); |
1147 | if (err) | 1220 | if (err) |
1148 | return err; | 1221 | return err; |
@@ -1153,10 +1226,50 @@ add_pred: | |||
1153 | return 0; | 1226 | return 0; |
1154 | } | 1227 | } |
1155 | 1228 | ||
1156 | int apply_event_filter(struct ftrace_event_call *call, char *filter_string) | 1229 | static int replace_system_preds(struct event_subsystem *system, |
1230 | struct filter_parse_state *ps, | ||
1231 | char *filter_string) | ||
1157 | { | 1232 | { |
1233 | struct ftrace_event_call *call; | ||
1234 | bool fail = true; | ||
1158 | int err; | 1235 | int err; |
1159 | 1236 | ||
1237 | list_for_each_entry(call, &ftrace_events, list) { | ||
1238 | struct event_filter *filter = call->filter; | ||
1239 | |||
1240 | if (!call->define_fields) | ||
1241 | continue; | ||
1242 | |||
1243 | if (strcmp(call->system, system->name) != 0) | ||
1244 | continue; | ||
1245 | |||
1246 | /* try to see if the filter can be applied */ | ||
1247 | err = replace_preds(call, filter, ps, filter_string, true); | ||
1248 | if (err) | ||
1249 | continue; | ||
1250 | |||
1251 | /* really apply the filter */ | ||
1252 | filter_disable_preds(call); | ||
1253 | err = replace_preds(call, filter, ps, filter_string, false); | ||
1254 | if (err) | ||
1255 | filter_disable_preds(call); | ||
1256 | else { | ||
1257 | call->filter_active = 1; | ||
1258 | replace_filter_string(filter, filter_string); | ||
1259 | } | ||
1260 | fail = false; | ||
1261 | } | ||
1262 | |||
1263 | if (fail) { | ||
1264 | parse_error(ps, FILT_ERR_BAD_SUBSYS_FILTER, 0); | ||
1265 | return -EINVAL; | ||
1266 | } | ||
1267 | return 0; | ||
1268 | } | ||
1269 | |||
1270 | int apply_event_filter(struct ftrace_event_call *call, char *filter_string) | ||
1271 | { | ||
1272 | int err; | ||
1160 | struct filter_parse_state *ps; | 1273 | struct filter_parse_state *ps; |
1161 | 1274 | ||
1162 | mutex_lock(&event_mutex); | 1275 | mutex_lock(&event_mutex); |
@@ -1168,8 +1281,7 @@ int apply_event_filter(struct ftrace_event_call *call, char *filter_string) | |||
1168 | if (!strcmp(strstrip(filter_string), "0")) { | 1281 | if (!strcmp(strstrip(filter_string), "0")) { |
1169 | filter_disable_preds(call); | 1282 | filter_disable_preds(call); |
1170 | remove_filter_string(call->filter); | 1283 | remove_filter_string(call->filter); |
1171 | mutex_unlock(&event_mutex); | 1284 | goto out_unlock; |
1172 | return 0; | ||
1173 | } | 1285 | } |
1174 | 1286 | ||
1175 | err = -ENOMEM; | 1287 | err = -ENOMEM; |
@@ -1187,10 +1299,11 @@ int apply_event_filter(struct ftrace_event_call *call, char *filter_string) | |||
1187 | goto out; | 1299 | goto out; |
1188 | } | 1300 | } |
1189 | 1301 | ||
1190 | err = replace_preds(NULL, call, ps, filter_string, false); | 1302 | err = replace_preds(call, call->filter, ps, filter_string, false); |
1191 | if (err) | 1303 | if (err) |
1192 | append_filter_err(ps, call->filter); | 1304 | append_filter_err(ps, call->filter); |
1193 | 1305 | else | |
1306 | call->filter_active = 1; | ||
1194 | out: | 1307 | out: |
1195 | filter_opstack_clear(ps); | 1308 | filter_opstack_clear(ps); |
1196 | postfix_clear(ps); | 1309 | postfix_clear(ps); |
@@ -1205,7 +1318,6 @@ int apply_subsystem_event_filter(struct event_subsystem *system, | |||
1205 | char *filter_string) | 1318 | char *filter_string) |
1206 | { | 1319 | { |
1207 | int err; | 1320 | int err; |
1208 | |||
1209 | struct filter_parse_state *ps; | 1321 | struct filter_parse_state *ps; |
1210 | 1322 | ||
1211 | mutex_lock(&event_mutex); | 1323 | mutex_lock(&event_mutex); |
@@ -1215,10 +1327,9 @@ int apply_subsystem_event_filter(struct event_subsystem *system, | |||
1215 | goto out_unlock; | 1327 | goto out_unlock; |
1216 | 1328 | ||
1217 | if (!strcmp(strstrip(filter_string), "0")) { | 1329 | if (!strcmp(strstrip(filter_string), "0")) { |
1218 | filter_free_subsystem_preds(system, FILTER_DISABLE_ALL); | 1330 | filter_free_subsystem_preds(system); |
1219 | remove_filter_string(system->filter); | 1331 | remove_filter_string(system->filter); |
1220 | mutex_unlock(&event_mutex); | 1332 | goto out_unlock; |
1221 | return 0; | ||
1222 | } | 1333 | } |
1223 | 1334 | ||
1224 | err = -ENOMEM; | 1335 | err = -ENOMEM; |
@@ -1235,31 +1346,87 @@ int apply_subsystem_event_filter(struct event_subsystem *system, | |||
1235 | goto out; | 1346 | goto out; |
1236 | } | 1347 | } |
1237 | 1348 | ||
1238 | filter_free_subsystem_preds(system, FILTER_INIT_NO_RESET); | 1349 | err = replace_system_preds(system, ps, filter_string); |
1239 | 1350 | if (err) | |
1240 | /* try to see the filter can be applied to which events */ | ||
1241 | err = replace_preds(system, NULL, ps, filter_string, true); | ||
1242 | if (err) { | ||
1243 | append_filter_err(ps, system->filter); | 1351 | append_filter_err(ps, system->filter); |
1244 | goto out; | 1352 | |
1353 | out: | ||
1354 | filter_opstack_clear(ps); | ||
1355 | postfix_clear(ps); | ||
1356 | kfree(ps); | ||
1357 | out_unlock: | ||
1358 | mutex_unlock(&event_mutex); | ||
1359 | |||
1360 | return err; | ||
1361 | } | ||
1362 | |||
1363 | #ifdef CONFIG_EVENT_PROFILE | ||
1364 | |||
1365 | void ftrace_profile_free_filter(struct perf_event *event) | ||
1366 | { | ||
1367 | struct event_filter *filter = event->filter; | ||
1368 | |||
1369 | event->filter = NULL; | ||
1370 | __free_preds(filter); | ||
1371 | } | ||
1372 | |||
1373 | int ftrace_profile_set_filter(struct perf_event *event, int event_id, | ||
1374 | char *filter_str) | ||
1375 | { | ||
1376 | int err; | ||
1377 | struct event_filter *filter; | ||
1378 | struct filter_parse_state *ps; | ||
1379 | struct ftrace_event_call *call = NULL; | ||
1380 | |||
1381 | mutex_lock(&event_mutex); | ||
1382 | |||
1383 | list_for_each_entry(call, &ftrace_events, list) { | ||
1384 | if (call->id == event_id) | ||
1385 | break; | ||
1245 | } | 1386 | } |
1246 | 1387 | ||
1247 | filter_free_subsystem_preds(system, FILTER_SKIP_NO_RESET); | 1388 | err = -EINVAL; |
1389 | if (!call) | ||
1390 | goto out_unlock; | ||
1248 | 1391 | ||
1249 | /* really apply the filter to the events */ | 1392 | err = -EEXIST; |
1250 | err = replace_preds(system, NULL, ps, filter_string, false); | 1393 | if (event->filter) |
1251 | if (err) { | 1394 | goto out_unlock; |
1252 | append_filter_err(ps, system->filter); | 1395 | |
1253 | filter_free_subsystem_preds(system, 2); | 1396 | filter = __alloc_preds(); |
1397 | if (IS_ERR(filter)) { | ||
1398 | err = PTR_ERR(filter); | ||
1399 | goto out_unlock; | ||
1254 | } | 1400 | } |
1255 | 1401 | ||
1256 | out: | 1402 | err = -ENOMEM; |
1403 | ps = kzalloc(sizeof(*ps), GFP_KERNEL); | ||
1404 | if (!ps) | ||
1405 | goto free_preds; | ||
1406 | |||
1407 | parse_init(ps, filter_ops, filter_str); | ||
1408 | err = filter_parse(ps); | ||
1409 | if (err) | ||
1410 | goto free_ps; | ||
1411 | |||
1412 | err = replace_preds(call, filter, ps, filter_str, false); | ||
1413 | if (!err) | ||
1414 | event->filter = filter; | ||
1415 | |||
1416 | free_ps: | ||
1257 | filter_opstack_clear(ps); | 1417 | filter_opstack_clear(ps); |
1258 | postfix_clear(ps); | 1418 | postfix_clear(ps); |
1259 | kfree(ps); | 1419 | kfree(ps); |
1420 | |||
1421 | free_preds: | ||
1422 | if (err) | ||
1423 | __free_preds(filter); | ||
1424 | |||
1260 | out_unlock: | 1425 | out_unlock: |
1261 | mutex_unlock(&event_mutex); | 1426 | mutex_unlock(&event_mutex); |
1262 | 1427 | ||
1263 | return err; | 1428 | return err; |
1264 | } | 1429 | } |
1265 | 1430 | ||
1431 | #endif /* CONFIG_EVENT_PROFILE */ | ||
1432 | |||
diff --git a/kernel/trace/trace_export.c b/kernel/trace/trace_export.c index 9753fcc61bc5..dff8c84ddf17 100644 --- a/kernel/trace/trace_export.c +++ b/kernel/trace/trace_export.c | |||
@@ -48,11 +48,11 @@ | |||
48 | struct ____ftrace_##name { \ | 48 | struct ____ftrace_##name { \ |
49 | tstruct \ | 49 | tstruct \ |
50 | }; \ | 50 | }; \ |
51 | static void __used ____ftrace_check_##name(void) \ | 51 | static void __always_unused ____ftrace_check_##name(void) \ |
52 | { \ | 52 | { \ |
53 | struct ____ftrace_##name *__entry = NULL; \ | 53 | struct ____ftrace_##name *__entry = NULL; \ |
54 | \ | 54 | \ |
55 | /* force cmpile-time check on F_printk() */ \ | 55 | /* force compile-time check on F_printk() */ \ |
56 | printk(print); \ | 56 | printk(print); \ |
57 | } | 57 | } |
58 | 58 | ||
@@ -66,44 +66,47 @@ static void __used ____ftrace_check_##name(void) \ | |||
66 | #undef __field | 66 | #undef __field |
67 | #define __field(type, item) \ | 67 | #define __field(type, item) \ |
68 | ret = trace_seq_printf(s, "\tfield:" #type " " #item ";\t" \ | 68 | ret = trace_seq_printf(s, "\tfield:" #type " " #item ";\t" \ |
69 | "offset:%zu;\tsize:%zu;\n", \ | 69 | "offset:%zu;\tsize:%zu;\tsigned:%u;\n", \ |
70 | offsetof(typeof(field), item), \ | 70 | offsetof(typeof(field), item), \ |
71 | sizeof(field.item)); \ | 71 | sizeof(field.item), is_signed_type(type)); \ |
72 | if (!ret) \ | 72 | if (!ret) \ |
73 | return 0; | 73 | return 0; |
74 | 74 | ||
75 | #undef __field_desc | 75 | #undef __field_desc |
76 | #define __field_desc(type, container, item) \ | 76 | #define __field_desc(type, container, item) \ |
77 | ret = trace_seq_printf(s, "\tfield:" #type " " #item ";\t" \ | 77 | ret = trace_seq_printf(s, "\tfield:" #type " " #item ";\t" \ |
78 | "offset:%zu;\tsize:%zu;\n", \ | 78 | "offset:%zu;\tsize:%zu;\tsigned:%u;\n", \ |
79 | offsetof(typeof(field), container.item), \ | 79 | offsetof(typeof(field), container.item), \ |
80 | sizeof(field.container.item)); \ | 80 | sizeof(field.container.item), \ |
81 | is_signed_type(type)); \ | ||
81 | if (!ret) \ | 82 | if (!ret) \ |
82 | return 0; | 83 | return 0; |
83 | 84 | ||
84 | #undef __array | 85 | #undef __array |
85 | #define __array(type, item, len) \ | 86 | #define __array(type, item, len) \ |
86 | ret = trace_seq_printf(s, "\tfield:" #type " " #item "[" #len "];\t" \ | 87 | ret = trace_seq_printf(s, "\tfield:" #type " " #item "[" #len "];\t" \ |
87 | "offset:%zu;\tsize:%zu;\n", \ | 88 | "offset:%zu;\tsize:%zu;\tsigned:%u;\n", \ |
88 | offsetof(typeof(field), item), \ | 89 | offsetof(typeof(field), item), \ |
89 | sizeof(field.item)); \ | 90 | sizeof(field.item), is_signed_type(type)); \ |
90 | if (!ret) \ | 91 | if (!ret) \ |
91 | return 0; | 92 | return 0; |
92 | 93 | ||
93 | #undef __array_desc | 94 | #undef __array_desc |
94 | #define __array_desc(type, container, item, len) \ | 95 | #define __array_desc(type, container, item, len) \ |
95 | ret = trace_seq_printf(s, "\tfield:" #type " " #item "[" #len "];\t" \ | 96 | ret = trace_seq_printf(s, "\tfield:" #type " " #item "[" #len "];\t" \ |
96 | "offset:%zu;\tsize:%zu;\n", \ | 97 | "offset:%zu;\tsize:%zu;\tsigned:%u;\n", \ |
97 | offsetof(typeof(field), container.item), \ | 98 | offsetof(typeof(field), container.item), \ |
98 | sizeof(field.container.item)); \ | 99 | sizeof(field.container.item), \ |
100 | is_signed_type(type)); \ | ||
99 | if (!ret) \ | 101 | if (!ret) \ |
100 | return 0; | 102 | return 0; |
101 | 103 | ||
102 | #undef __dynamic_array | 104 | #undef __dynamic_array |
103 | #define __dynamic_array(type, item) \ | 105 | #define __dynamic_array(type, item) \ |
104 | ret = trace_seq_printf(s, "\tfield:" #type " " #item ";\t" \ | 106 | ret = trace_seq_printf(s, "\tfield:" #type " " #item ";\t" \ |
105 | "offset:%zu;\tsize:0;\n", \ | 107 | "offset:%zu;\tsize:0;\tsigned:%u;\n", \ |
106 | offsetof(typeof(field), item)); \ | 108 | offsetof(typeof(field), item), \ |
109 | is_signed_type(type)); \ | ||
107 | if (!ret) \ | 110 | if (!ret) \ |
108 | return 0; | 111 | return 0; |
109 | 112 | ||
@@ -131,7 +134,6 @@ ftrace_format_##name(struct ftrace_event_call *unused, \ | |||
131 | 134 | ||
132 | #include "trace_entries.h" | 135 | #include "trace_entries.h" |
133 | 136 | ||
134 | |||
135 | #undef __field | 137 | #undef __field |
136 | #define __field(type, item) \ | 138 | #define __field(type, item) \ |
137 | ret = trace_define_field(event_call, #type, #item, \ | 139 | ret = trace_define_field(event_call, #type, #item, \ |
@@ -193,6 +195,11 @@ ftrace_define_fields_##name(struct ftrace_event_call *event_call) \ | |||
193 | 195 | ||
194 | #include "trace_entries.h" | 196 | #include "trace_entries.h" |
195 | 197 | ||
198 | static int ftrace_raw_init_event(struct ftrace_event_call *call) | ||
199 | { | ||
200 | INIT_LIST_HEAD(&call->fields); | ||
201 | return 0; | ||
202 | } | ||
196 | 203 | ||
197 | #undef __field | 204 | #undef __field |
198 | #define __field(type, item) | 205 | #define __field(type, item) |
@@ -211,7 +218,6 @@ ftrace_define_fields_##name(struct ftrace_event_call *event_call) \ | |||
211 | 218 | ||
212 | #undef FTRACE_ENTRY | 219 | #undef FTRACE_ENTRY |
213 | #define FTRACE_ENTRY(call, struct_name, type, tstruct, print) \ | 220 | #define FTRACE_ENTRY(call, struct_name, type, tstruct, print) \ |
214 | static int ftrace_raw_init_event_##call(void); \ | ||
215 | \ | 221 | \ |
216 | struct ftrace_event_call __used \ | 222 | struct ftrace_event_call __used \ |
217 | __attribute__((__aligned__(4))) \ | 223 | __attribute__((__aligned__(4))) \ |
@@ -219,14 +225,9 @@ __attribute__((section("_ftrace_events"))) event_##call = { \ | |||
219 | .name = #call, \ | 225 | .name = #call, \ |
220 | .id = type, \ | 226 | .id = type, \ |
221 | .system = __stringify(TRACE_SYSTEM), \ | 227 | .system = __stringify(TRACE_SYSTEM), \ |
222 | .raw_init = ftrace_raw_init_event_##call, \ | 228 | .raw_init = ftrace_raw_init_event, \ |
223 | .show_format = ftrace_format_##call, \ | 229 | .show_format = ftrace_format_##call, \ |
224 | .define_fields = ftrace_define_fields_##call, \ | 230 | .define_fields = ftrace_define_fields_##call, \ |
225 | }; \ | 231 | }; \ |
226 | static int ftrace_raw_init_event_##call(void) \ | ||
227 | { \ | ||
228 | INIT_LIST_HEAD(&event_##call.fields); \ | ||
229 | return 0; \ | ||
230 | } \ | ||
231 | 232 | ||
232 | #include "trace_entries.h" | 233 | #include "trace_entries.h" |
diff --git a/kernel/trace/trace_kprobe.c b/kernel/trace/trace_kprobe.c new file mode 100644 index 000000000000..aff5f80b59b8 --- /dev/null +++ b/kernel/trace/trace_kprobe.c | |||
@@ -0,0 +1,1523 @@ | |||
1 | /* | ||
2 | * Kprobes-based tracing events | ||
3 | * | ||
4 | * Created by Masami Hiramatsu <mhiramat@redhat.com> | ||
5 | * | ||
6 | * This program is free software; you can redistribute it and/or modify | ||
7 | * it under the terms of the GNU General Public License version 2 as | ||
8 | * published by the Free Software Foundation. | ||
9 | * | ||
10 | * This program is distributed in the hope that it will be useful, | ||
11 | * but WITHOUT ANY WARRANTY; without even the implied warranty of | ||
12 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | ||
13 | * GNU General Public License for more details. | ||
14 | * | ||
15 | * You should have received a copy of the GNU General Public License | ||
16 | * along with this program; if not, write to the Free Software | ||
17 | * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA | ||
18 | */ | ||
19 | |||
20 | #include <linux/module.h> | ||
21 | #include <linux/uaccess.h> | ||
22 | #include <linux/kprobes.h> | ||
23 | #include <linux/seq_file.h> | ||
24 | #include <linux/slab.h> | ||
25 | #include <linux/smp.h> | ||
26 | #include <linux/debugfs.h> | ||
27 | #include <linux/types.h> | ||
28 | #include <linux/string.h> | ||
29 | #include <linux/ctype.h> | ||
30 | #include <linux/ptrace.h> | ||
31 | #include <linux/perf_event.h> | ||
32 | |||
33 | #include "trace.h" | ||
34 | #include "trace_output.h" | ||
35 | |||
36 | #define MAX_TRACE_ARGS 128 | ||
37 | #define MAX_ARGSTR_LEN 63 | ||
38 | #define MAX_EVENT_NAME_LEN 64 | ||
39 | #define KPROBE_EVENT_SYSTEM "kprobes" | ||
40 | |||
41 | /* Reserved field names */ | ||
42 | #define FIELD_STRING_IP "__probe_ip" | ||
43 | #define FIELD_STRING_NARGS "__probe_nargs" | ||
44 | #define FIELD_STRING_RETIP "__probe_ret_ip" | ||
45 | #define FIELD_STRING_FUNC "__probe_func" | ||
46 | |||
47 | const char *reserved_field_names[] = { | ||
48 | "common_type", | ||
49 | "common_flags", | ||
50 | "common_preempt_count", | ||
51 | "common_pid", | ||
52 | "common_tgid", | ||
53 | "common_lock_depth", | ||
54 | FIELD_STRING_IP, | ||
55 | FIELD_STRING_NARGS, | ||
56 | FIELD_STRING_RETIP, | ||
57 | FIELD_STRING_FUNC, | ||
58 | }; | ||
59 | |||
60 | struct fetch_func { | ||
61 | unsigned long (*func)(struct pt_regs *, void *); | ||
62 | void *data; | ||
63 | }; | ||
64 | |||
65 | static __kprobes unsigned long call_fetch(struct fetch_func *f, | ||
66 | struct pt_regs *regs) | ||
67 | { | ||
68 | return f->func(regs, f->data); | ||
69 | } | ||
70 | |||
71 | /* fetch handlers */ | ||
72 | static __kprobes unsigned long fetch_register(struct pt_regs *regs, | ||
73 | void *offset) | ||
74 | { | ||
75 | return regs_get_register(regs, (unsigned int)((unsigned long)offset)); | ||
76 | } | ||
77 | |||
78 | static __kprobes unsigned long fetch_stack(struct pt_regs *regs, | ||
79 | void *num) | ||
80 | { | ||
81 | return regs_get_kernel_stack_nth(regs, | ||
82 | (unsigned int)((unsigned long)num)); | ||
83 | } | ||
84 | |||
85 | static __kprobes unsigned long fetch_memory(struct pt_regs *regs, void *addr) | ||
86 | { | ||
87 | unsigned long retval; | ||
88 | |||
89 | if (probe_kernel_address(addr, retval)) | ||
90 | return 0; | ||
91 | return retval; | ||
92 | } | ||
93 | |||
94 | static __kprobes unsigned long fetch_argument(struct pt_regs *regs, void *num) | ||
95 | { | ||
96 | return regs_get_argument_nth(regs, (unsigned int)((unsigned long)num)); | ||
97 | } | ||
98 | |||
99 | static __kprobes unsigned long fetch_retvalue(struct pt_regs *regs, | ||
100 | void *dummy) | ||
101 | { | ||
102 | return regs_return_value(regs); | ||
103 | } | ||
104 | |||
105 | static __kprobes unsigned long fetch_stack_address(struct pt_regs *regs, | ||
106 | void *dummy) | ||
107 | { | ||
108 | return kernel_stack_pointer(regs); | ||
109 | } | ||
110 | |||
111 | /* Memory fetching by symbol */ | ||
112 | struct symbol_cache { | ||
113 | char *symbol; | ||
114 | long offset; | ||
115 | unsigned long addr; | ||
116 | }; | ||
117 | |||
118 | static unsigned long update_symbol_cache(struct symbol_cache *sc) | ||
119 | { | ||
120 | sc->addr = (unsigned long)kallsyms_lookup_name(sc->symbol); | ||
121 | if (sc->addr) | ||
122 | sc->addr += sc->offset; | ||
123 | return sc->addr; | ||
124 | } | ||
125 | |||
126 | static void free_symbol_cache(struct symbol_cache *sc) | ||
127 | { | ||
128 | kfree(sc->symbol); | ||
129 | kfree(sc); | ||
130 | } | ||
131 | |||
132 | static struct symbol_cache *alloc_symbol_cache(const char *sym, long offset) | ||
133 | { | ||
134 | struct symbol_cache *sc; | ||
135 | |||
136 | if (!sym || strlen(sym) == 0) | ||
137 | return NULL; | ||
138 | sc = kzalloc(sizeof(struct symbol_cache), GFP_KERNEL); | ||
139 | if (!sc) | ||
140 | return NULL; | ||
141 | |||
142 | sc->symbol = kstrdup(sym, GFP_KERNEL); | ||
143 | if (!sc->symbol) { | ||
144 | kfree(sc); | ||
145 | return NULL; | ||
146 | } | ||
147 | sc->offset = offset; | ||
148 | |||
149 | update_symbol_cache(sc); | ||
150 | return sc; | ||
151 | } | ||
152 | |||
153 | static __kprobes unsigned long fetch_symbol(struct pt_regs *regs, void *data) | ||
154 | { | ||
155 | struct symbol_cache *sc = data; | ||
156 | |||
157 | if (sc->addr) | ||
158 | return fetch_memory(regs, (void *)sc->addr); | ||
159 | else | ||
160 | return 0; | ||
161 | } | ||
162 | |||
163 | /* Special indirect memory access interface */ | ||
164 | struct indirect_fetch_data { | ||
165 | struct fetch_func orig; | ||
166 | long offset; | ||
167 | }; | ||
168 | |||
169 | static __kprobes unsigned long fetch_indirect(struct pt_regs *regs, void *data) | ||
170 | { | ||
171 | struct indirect_fetch_data *ind = data; | ||
172 | unsigned long addr; | ||
173 | |||
174 | addr = call_fetch(&ind->orig, regs); | ||
175 | if (addr) { | ||
176 | addr += ind->offset; | ||
177 | return fetch_memory(regs, (void *)addr); | ||
178 | } else | ||
179 | return 0; | ||
180 | } | ||
181 | |||
182 | static __kprobes void free_indirect_fetch_data(struct indirect_fetch_data *data) | ||
183 | { | ||
184 | if (data->orig.func == fetch_indirect) | ||
185 | free_indirect_fetch_data(data->orig.data); | ||
186 | else if (data->orig.func == fetch_symbol) | ||
187 | free_symbol_cache(data->orig.data); | ||
188 | kfree(data); | ||
189 | } | ||
190 | |||
191 | /** | ||
192 | * Kprobe event core functions | ||
193 | */ | ||
194 | |||
195 | struct probe_arg { | ||
196 | struct fetch_func fetch; | ||
197 | const char *name; | ||
198 | }; | ||
199 | |||
200 | /* Flags for trace_probe */ | ||
201 | #define TP_FLAG_TRACE 1 | ||
202 | #define TP_FLAG_PROFILE 2 | ||
203 | |||
204 | struct trace_probe { | ||
205 | struct list_head list; | ||
206 | struct kretprobe rp; /* Use rp.kp for kprobe use */ | ||
207 | unsigned long nhit; | ||
208 | unsigned int flags; /* For TP_FLAG_* */ | ||
209 | const char *symbol; /* symbol name */ | ||
210 | struct ftrace_event_call call; | ||
211 | struct trace_event event; | ||
212 | unsigned int nr_args; | ||
213 | struct probe_arg args[]; | ||
214 | }; | ||
215 | |||
216 | #define SIZEOF_TRACE_PROBE(n) \ | ||
217 | (offsetof(struct trace_probe, args) + \ | ||
218 | (sizeof(struct probe_arg) * (n))) | ||
219 | |||
220 | static __kprobes int probe_is_return(struct trace_probe *tp) | ||
221 | { | ||
222 | return tp->rp.handler != NULL; | ||
223 | } | ||
224 | |||
225 | static __kprobes const char *probe_symbol(struct trace_probe *tp) | ||
226 | { | ||
227 | return tp->symbol ? tp->symbol : "unknown"; | ||
228 | } | ||
229 | |||
230 | static int probe_arg_string(char *buf, size_t n, struct fetch_func *ff) | ||
231 | { | ||
232 | int ret = -EINVAL; | ||
233 | |||
234 | if (ff->func == fetch_argument) | ||
235 | ret = snprintf(buf, n, "$arg%lu", (unsigned long)ff->data); | ||
236 | else if (ff->func == fetch_register) { | ||
237 | const char *name; | ||
238 | name = regs_query_register_name((unsigned int)((long)ff->data)); | ||
239 | ret = snprintf(buf, n, "%%%s", name); | ||
240 | } else if (ff->func == fetch_stack) | ||
241 | ret = snprintf(buf, n, "$stack%lu", (unsigned long)ff->data); | ||
242 | else if (ff->func == fetch_memory) | ||
243 | ret = snprintf(buf, n, "@0x%p", ff->data); | ||
244 | else if (ff->func == fetch_symbol) { | ||
245 | struct symbol_cache *sc = ff->data; | ||
246 | if (sc->offset) | ||
247 | ret = snprintf(buf, n, "@%s%+ld", sc->symbol, | ||
248 | sc->offset); | ||
249 | else | ||
250 | ret = snprintf(buf, n, "@%s", sc->symbol); | ||
251 | } else if (ff->func == fetch_retvalue) | ||
252 | ret = snprintf(buf, n, "$retval"); | ||
253 | else if (ff->func == fetch_stack_address) | ||
254 | ret = snprintf(buf, n, "$stack"); | ||
255 | else if (ff->func == fetch_indirect) { | ||
256 | struct indirect_fetch_data *id = ff->data; | ||
257 | size_t l = 0; | ||
258 | ret = snprintf(buf, n, "%+ld(", id->offset); | ||
259 | if (ret >= n) | ||
260 | goto end; | ||
261 | l += ret; | ||
262 | ret = probe_arg_string(buf + l, n - l, &id->orig); | ||
263 | if (ret < 0) | ||
264 | goto end; | ||
265 | l += ret; | ||
266 | ret = snprintf(buf + l, n - l, ")"); | ||
267 | ret += l; | ||
268 | } | ||
269 | end: | ||
270 | if (ret >= n) | ||
271 | return -ENOSPC; | ||
272 | return ret; | ||
273 | } | ||
274 | |||
275 | static int register_probe_event(struct trace_probe *tp); | ||
276 | static void unregister_probe_event(struct trace_probe *tp); | ||
277 | |||
278 | static DEFINE_MUTEX(probe_lock); | ||
279 | static LIST_HEAD(probe_list); | ||
280 | |||
281 | static int kprobe_dispatcher(struct kprobe *kp, struct pt_regs *regs); | ||
282 | static int kretprobe_dispatcher(struct kretprobe_instance *ri, | ||
283 | struct pt_regs *regs); | ||
284 | |||
285 | /* | ||
286 | * Allocate new trace_probe and initialize it (including kprobes). | ||
287 | */ | ||
288 | static struct trace_probe *alloc_trace_probe(const char *group, | ||
289 | const char *event, | ||
290 | void *addr, | ||
291 | const char *symbol, | ||
292 | unsigned long offs, | ||
293 | int nargs, int is_return) | ||
294 | { | ||
295 | struct trace_probe *tp; | ||
296 | |||
297 | tp = kzalloc(SIZEOF_TRACE_PROBE(nargs), GFP_KERNEL); | ||
298 | if (!tp) | ||
299 | return ERR_PTR(-ENOMEM); | ||
300 | |||
301 | if (symbol) { | ||
302 | tp->symbol = kstrdup(symbol, GFP_KERNEL); | ||
303 | if (!tp->symbol) | ||
304 | goto error; | ||
305 | tp->rp.kp.symbol_name = tp->symbol; | ||
306 | tp->rp.kp.offset = offs; | ||
307 | } else | ||
308 | tp->rp.kp.addr = addr; | ||
309 | |||
310 | if (is_return) | ||
311 | tp->rp.handler = kretprobe_dispatcher; | ||
312 | else | ||
313 | tp->rp.kp.pre_handler = kprobe_dispatcher; | ||
314 | |||
315 | if (!event) | ||
316 | goto error; | ||
317 | tp->call.name = kstrdup(event, GFP_KERNEL); | ||
318 | if (!tp->call.name) | ||
319 | goto error; | ||
320 | |||
321 | if (!group) | ||
322 | goto error; | ||
323 | tp->call.system = kstrdup(group, GFP_KERNEL); | ||
324 | if (!tp->call.system) | ||
325 | goto error; | ||
326 | |||
327 | INIT_LIST_HEAD(&tp->list); | ||
328 | return tp; | ||
329 | error: | ||
330 | kfree(tp->call.name); | ||
331 | kfree(tp->symbol); | ||
332 | kfree(tp); | ||
333 | return ERR_PTR(-ENOMEM); | ||
334 | } | ||
335 | |||
336 | static void free_probe_arg(struct probe_arg *arg) | ||
337 | { | ||
338 | if (arg->fetch.func == fetch_symbol) | ||
339 | free_symbol_cache(arg->fetch.data); | ||
340 | else if (arg->fetch.func == fetch_indirect) | ||
341 | free_indirect_fetch_data(arg->fetch.data); | ||
342 | kfree(arg->name); | ||
343 | } | ||
344 | |||
345 | static void free_trace_probe(struct trace_probe *tp) | ||
346 | { | ||
347 | int i; | ||
348 | |||
349 | for (i = 0; i < tp->nr_args; i++) | ||
350 | free_probe_arg(&tp->args[i]); | ||
351 | |||
352 | kfree(tp->call.system); | ||
353 | kfree(tp->call.name); | ||
354 | kfree(tp->symbol); | ||
355 | kfree(tp); | ||
356 | } | ||
357 | |||
358 | static struct trace_probe *find_probe_event(const char *event, | ||
359 | const char *group) | ||
360 | { | ||
361 | struct trace_probe *tp; | ||
362 | |||
363 | list_for_each_entry(tp, &probe_list, list) | ||
364 | if (strcmp(tp->call.name, event) == 0 && | ||
365 | strcmp(tp->call.system, group) == 0) | ||
366 | return tp; | ||
367 | return NULL; | ||
368 | } | ||
369 | |||
370 | /* Unregister a trace_probe and probe_event: call with locking probe_lock */ | ||
371 | static void unregister_trace_probe(struct trace_probe *tp) | ||
372 | { | ||
373 | if (probe_is_return(tp)) | ||
374 | unregister_kretprobe(&tp->rp); | ||
375 | else | ||
376 | unregister_kprobe(&tp->rp.kp); | ||
377 | list_del(&tp->list); | ||
378 | unregister_probe_event(tp); | ||
379 | } | ||
380 | |||
381 | /* Register a trace_probe and probe_event */ | ||
382 | static int register_trace_probe(struct trace_probe *tp) | ||
383 | { | ||
384 | struct trace_probe *old_tp; | ||
385 | int ret; | ||
386 | |||
387 | mutex_lock(&probe_lock); | ||
388 | |||
389 | /* register as an event */ | ||
390 | old_tp = find_probe_event(tp->call.name, tp->call.system); | ||
391 | if (old_tp) { | ||
392 | /* delete old event */ | ||
393 | unregister_trace_probe(old_tp); | ||
394 | free_trace_probe(old_tp); | ||
395 | } | ||
396 | ret = register_probe_event(tp); | ||
397 | if (ret) { | ||
398 | pr_warning("Faild to register probe event(%d)\n", ret); | ||
399 | goto end; | ||
400 | } | ||
401 | |||
402 | tp->rp.kp.flags |= KPROBE_FLAG_DISABLED; | ||
403 | if (probe_is_return(tp)) | ||
404 | ret = register_kretprobe(&tp->rp); | ||
405 | else | ||
406 | ret = register_kprobe(&tp->rp.kp); | ||
407 | |||
408 | if (ret) { | ||
409 | pr_warning("Could not insert probe(%d)\n", ret); | ||
410 | if (ret == -EILSEQ) { | ||
411 | pr_warning("Probing address(0x%p) is not an " | ||
412 | "instruction boundary.\n", | ||
413 | tp->rp.kp.addr); | ||
414 | ret = -EINVAL; | ||
415 | } | ||
416 | unregister_probe_event(tp); | ||
417 | } else | ||
418 | list_add_tail(&tp->list, &probe_list); | ||
419 | end: | ||
420 | mutex_unlock(&probe_lock); | ||
421 | return ret; | ||
422 | } | ||
423 | |||
424 | /* Split symbol and offset. */ | ||
425 | static int split_symbol_offset(char *symbol, unsigned long *offset) | ||
426 | { | ||
427 | char *tmp; | ||
428 | int ret; | ||
429 | |||
430 | if (!offset) | ||
431 | return -EINVAL; | ||
432 | |||
433 | tmp = strchr(symbol, '+'); | ||
434 | if (tmp) { | ||
435 | /* skip sign because strict_strtol doesn't accept '+' */ | ||
436 | ret = strict_strtoul(tmp + 1, 0, offset); | ||
437 | if (ret) | ||
438 | return ret; | ||
439 | *tmp = '\0'; | ||
440 | } else | ||
441 | *offset = 0; | ||
442 | return 0; | ||
443 | } | ||
444 | |||
445 | #define PARAM_MAX_ARGS 16 | ||
446 | #define PARAM_MAX_STACK (THREAD_SIZE / sizeof(unsigned long)) | ||
447 | |||
448 | static int parse_probe_vars(char *arg, struct fetch_func *ff, int is_return) | ||
449 | { | ||
450 | int ret = 0; | ||
451 | unsigned long param; | ||
452 | |||
453 | if (strcmp(arg, "retval") == 0) { | ||
454 | if (is_return) { | ||
455 | ff->func = fetch_retvalue; | ||
456 | ff->data = NULL; | ||
457 | } else | ||
458 | ret = -EINVAL; | ||
459 | } else if (strncmp(arg, "stack", 5) == 0) { | ||
460 | if (arg[5] == '\0') { | ||
461 | ff->func = fetch_stack_address; | ||
462 | ff->data = NULL; | ||
463 | } else if (isdigit(arg[5])) { | ||
464 | ret = strict_strtoul(arg + 5, 10, ¶m); | ||
465 | if (ret || param > PARAM_MAX_STACK) | ||
466 | ret = -EINVAL; | ||
467 | else { | ||
468 | ff->func = fetch_stack; | ||
469 | ff->data = (void *)param; | ||
470 | } | ||
471 | } else | ||
472 | ret = -EINVAL; | ||
473 | } else if (strncmp(arg, "arg", 3) == 0 && isdigit(arg[3])) { | ||
474 | ret = strict_strtoul(arg + 3, 10, ¶m); | ||
475 | if (ret || param > PARAM_MAX_ARGS) | ||
476 | ret = -EINVAL; | ||
477 | else { | ||
478 | ff->func = fetch_argument; | ||
479 | ff->data = (void *)param; | ||
480 | } | ||
481 | } else | ||
482 | ret = -EINVAL; | ||
483 | return ret; | ||
484 | } | ||
485 | |||
486 | /* Recursive argument parser */ | ||
487 | static int __parse_probe_arg(char *arg, struct fetch_func *ff, int is_return) | ||
488 | { | ||
489 | int ret = 0; | ||
490 | unsigned long param; | ||
491 | long offset; | ||
492 | char *tmp; | ||
493 | |||
494 | switch (arg[0]) { | ||
495 | case '$': | ||
496 | ret = parse_probe_vars(arg + 1, ff, is_return); | ||
497 | break; | ||
498 | case '%': /* named register */ | ||
499 | ret = regs_query_register_offset(arg + 1); | ||
500 | if (ret >= 0) { | ||
501 | ff->func = fetch_register; | ||
502 | ff->data = (void *)(unsigned long)ret; | ||
503 | ret = 0; | ||
504 | } | ||
505 | break; | ||
506 | case '@': /* memory or symbol */ | ||
507 | if (isdigit(arg[1])) { | ||
508 | ret = strict_strtoul(arg + 1, 0, ¶m); | ||
509 | if (ret) | ||
510 | break; | ||
511 | ff->func = fetch_memory; | ||
512 | ff->data = (void *)param; | ||
513 | } else { | ||
514 | ret = split_symbol_offset(arg + 1, &offset); | ||
515 | if (ret) | ||
516 | break; | ||
517 | ff->data = alloc_symbol_cache(arg + 1, offset); | ||
518 | if (ff->data) | ||
519 | ff->func = fetch_symbol; | ||
520 | else | ||
521 | ret = -EINVAL; | ||
522 | } | ||
523 | break; | ||
524 | case '+': /* indirect memory */ | ||
525 | case '-': | ||
526 | tmp = strchr(arg, '('); | ||
527 | if (!tmp) { | ||
528 | ret = -EINVAL; | ||
529 | break; | ||
530 | } | ||
531 | *tmp = '\0'; | ||
532 | ret = strict_strtol(arg + 1, 0, &offset); | ||
533 | if (ret) | ||
534 | break; | ||
535 | if (arg[0] == '-') | ||
536 | offset = -offset; | ||
537 | arg = tmp + 1; | ||
538 | tmp = strrchr(arg, ')'); | ||
539 | if (tmp) { | ||
540 | struct indirect_fetch_data *id; | ||
541 | *tmp = '\0'; | ||
542 | id = kzalloc(sizeof(struct indirect_fetch_data), | ||
543 | GFP_KERNEL); | ||
544 | if (!id) | ||
545 | return -ENOMEM; | ||
546 | id->offset = offset; | ||
547 | ret = __parse_probe_arg(arg, &id->orig, is_return); | ||
548 | if (ret) | ||
549 | kfree(id); | ||
550 | else { | ||
551 | ff->func = fetch_indirect; | ||
552 | ff->data = (void *)id; | ||
553 | } | ||
554 | } else | ||
555 | ret = -EINVAL; | ||
556 | break; | ||
557 | default: | ||
558 | /* TODO: support custom handler */ | ||
559 | ret = -EINVAL; | ||
560 | } | ||
561 | return ret; | ||
562 | } | ||
563 | |||
564 | /* String length checking wrapper */ | ||
565 | static int parse_probe_arg(char *arg, struct fetch_func *ff, int is_return) | ||
566 | { | ||
567 | if (strlen(arg) > MAX_ARGSTR_LEN) { | ||
568 | pr_info("Argument is too long.: %s\n", arg); | ||
569 | return -ENOSPC; | ||
570 | } | ||
571 | return __parse_probe_arg(arg, ff, is_return); | ||
572 | } | ||
573 | |||
574 | /* Return 1 if name is reserved or already used by another argument */ | ||
575 | static int conflict_field_name(const char *name, | ||
576 | struct probe_arg *args, int narg) | ||
577 | { | ||
578 | int i; | ||
579 | for (i = 0; i < ARRAY_SIZE(reserved_field_names); i++) | ||
580 | if (strcmp(reserved_field_names[i], name) == 0) | ||
581 | return 1; | ||
582 | for (i = 0; i < narg; i++) | ||
583 | if (strcmp(args[i].name, name) == 0) | ||
584 | return 1; | ||
585 | return 0; | ||
586 | } | ||
587 | |||
588 | static int create_trace_probe(int argc, char **argv) | ||
589 | { | ||
590 | /* | ||
591 | * Argument syntax: | ||
592 | * - Add kprobe: p[:[GRP/]EVENT] KSYM[+OFFS]|KADDR [FETCHARGS] | ||
593 | * - Add kretprobe: r[:[GRP/]EVENT] KSYM[+0] [FETCHARGS] | ||
594 | * Fetch args: | ||
595 | * $argN : fetch Nth of function argument. (N:0-) | ||
596 | * $retval : fetch return value | ||
597 | * $stack : fetch stack address | ||
598 | * $stackN : fetch Nth of stack (N:0-) | ||
599 | * @ADDR : fetch memory at ADDR (ADDR should be in kernel) | ||
600 | * @SYM[+|-offs] : fetch memory at SYM +|- offs (SYM is a data symbol) | ||
601 | * %REG : fetch register REG | ||
602 | * Indirect memory fetch: | ||
603 | * +|-offs(ARG) : fetch memory at ARG +|- offs address. | ||
604 | * Alias name of args: | ||
605 | * NAME=FETCHARG : set NAME as alias of FETCHARG. | ||
606 | */ | ||
607 | struct trace_probe *tp; | ||
608 | int i, ret = 0; | ||
609 | int is_return = 0; | ||
610 | char *symbol = NULL, *event = NULL, *arg = NULL, *group = NULL; | ||
611 | unsigned long offset = 0; | ||
612 | void *addr = NULL; | ||
613 | char buf[MAX_EVENT_NAME_LEN]; | ||
614 | |||
615 | if (argc < 2) { | ||
616 | pr_info("Probe point is not specified.\n"); | ||
617 | return -EINVAL; | ||
618 | } | ||
619 | |||
620 | if (argv[0][0] == 'p') | ||
621 | is_return = 0; | ||
622 | else if (argv[0][0] == 'r') | ||
623 | is_return = 1; | ||
624 | else { | ||
625 | pr_info("Probe definition must be started with 'p' or 'r'.\n"); | ||
626 | return -EINVAL; | ||
627 | } | ||
628 | |||
629 | if (argv[0][1] == ':') { | ||
630 | event = &argv[0][2]; | ||
631 | if (strchr(event, '/')) { | ||
632 | group = event; | ||
633 | event = strchr(group, '/') + 1; | ||
634 | event[-1] = '\0'; | ||
635 | if (strlen(group) == 0) { | ||
636 | pr_info("Group name is not specifiled\n"); | ||
637 | return -EINVAL; | ||
638 | } | ||
639 | } | ||
640 | if (strlen(event) == 0) { | ||
641 | pr_info("Event name is not specifiled\n"); | ||
642 | return -EINVAL; | ||
643 | } | ||
644 | } | ||
645 | |||
646 | if (isdigit(argv[1][0])) { | ||
647 | if (is_return) { | ||
648 | pr_info("Return probe point must be a symbol.\n"); | ||
649 | return -EINVAL; | ||
650 | } | ||
651 | /* an address specified */ | ||
652 | ret = strict_strtoul(&argv[0][2], 0, (unsigned long *)&addr); | ||
653 | if (ret) { | ||
654 | pr_info("Failed to parse address.\n"); | ||
655 | return ret; | ||
656 | } | ||
657 | } else { | ||
658 | /* a symbol specified */ | ||
659 | symbol = argv[1]; | ||
660 | /* TODO: support .init module functions */ | ||
661 | ret = split_symbol_offset(symbol, &offset); | ||
662 | if (ret) { | ||
663 | pr_info("Failed to parse symbol.\n"); | ||
664 | return ret; | ||
665 | } | ||
666 | if (offset && is_return) { | ||
667 | pr_info("Return probe must be used without offset.\n"); | ||
668 | return -EINVAL; | ||
669 | } | ||
670 | } | ||
671 | argc -= 2; argv += 2; | ||
672 | |||
673 | /* setup a probe */ | ||
674 | if (!group) | ||
675 | group = KPROBE_EVENT_SYSTEM; | ||
676 | if (!event) { | ||
677 | /* Make a new event name */ | ||
678 | if (symbol) | ||
679 | snprintf(buf, MAX_EVENT_NAME_LEN, "%c@%s%+ld", | ||
680 | is_return ? 'r' : 'p', symbol, offset); | ||
681 | else | ||
682 | snprintf(buf, MAX_EVENT_NAME_LEN, "%c@0x%p", | ||
683 | is_return ? 'r' : 'p', addr); | ||
684 | event = buf; | ||
685 | } | ||
686 | tp = alloc_trace_probe(group, event, addr, symbol, offset, argc, | ||
687 | is_return); | ||
688 | if (IS_ERR(tp)) { | ||
689 | pr_info("Failed to allocate trace_probe.(%d)\n", | ||
690 | (int)PTR_ERR(tp)); | ||
691 | return PTR_ERR(tp); | ||
692 | } | ||
693 | |||
694 | /* parse arguments */ | ||
695 | ret = 0; | ||
696 | for (i = 0; i < argc && i < MAX_TRACE_ARGS; i++) { | ||
697 | /* Parse argument name */ | ||
698 | arg = strchr(argv[i], '='); | ||
699 | if (arg) | ||
700 | *arg++ = '\0'; | ||
701 | else | ||
702 | arg = argv[i]; | ||
703 | |||
704 | if (conflict_field_name(argv[i], tp->args, i)) { | ||
705 | pr_info("Argument%d name '%s' conflicts with " | ||
706 | "another field.\n", i, argv[i]); | ||
707 | ret = -EINVAL; | ||
708 | goto error; | ||
709 | } | ||
710 | |||
711 | tp->args[i].name = kstrdup(argv[i], GFP_KERNEL); | ||
712 | if (!tp->args[i].name) { | ||
713 | pr_info("Failed to allocate argument%d name '%s'.\n", | ||
714 | i, argv[i]); | ||
715 | ret = -ENOMEM; | ||
716 | goto error; | ||
717 | } | ||
718 | |||
719 | /* Parse fetch argument */ | ||
720 | ret = parse_probe_arg(arg, &tp->args[i].fetch, is_return); | ||
721 | if (ret) { | ||
722 | pr_info("Parse error at argument%d. (%d)\n", i, ret); | ||
723 | kfree(tp->args[i].name); | ||
724 | goto error; | ||
725 | } | ||
726 | |||
727 | tp->nr_args++; | ||
728 | } | ||
729 | |||
730 | ret = register_trace_probe(tp); | ||
731 | if (ret) | ||
732 | goto error; | ||
733 | return 0; | ||
734 | |||
735 | error: | ||
736 | free_trace_probe(tp); | ||
737 | return ret; | ||
738 | } | ||
739 | |||
740 | static void cleanup_all_probes(void) | ||
741 | { | ||
742 | struct trace_probe *tp; | ||
743 | |||
744 | mutex_lock(&probe_lock); | ||
745 | /* TODO: Use batch unregistration */ | ||
746 | while (!list_empty(&probe_list)) { | ||
747 | tp = list_entry(probe_list.next, struct trace_probe, list); | ||
748 | unregister_trace_probe(tp); | ||
749 | free_trace_probe(tp); | ||
750 | } | ||
751 | mutex_unlock(&probe_lock); | ||
752 | } | ||
753 | |||
754 | |||
755 | /* Probes listing interfaces */ | ||
756 | static void *probes_seq_start(struct seq_file *m, loff_t *pos) | ||
757 | { | ||
758 | mutex_lock(&probe_lock); | ||
759 | return seq_list_start(&probe_list, *pos); | ||
760 | } | ||
761 | |||
762 | static void *probes_seq_next(struct seq_file *m, void *v, loff_t *pos) | ||
763 | { | ||
764 | return seq_list_next(v, &probe_list, pos); | ||
765 | } | ||
766 | |||
767 | static void probes_seq_stop(struct seq_file *m, void *v) | ||
768 | { | ||
769 | mutex_unlock(&probe_lock); | ||
770 | } | ||
771 | |||
772 | static int probes_seq_show(struct seq_file *m, void *v) | ||
773 | { | ||
774 | struct trace_probe *tp = v; | ||
775 | int i, ret; | ||
776 | char buf[MAX_ARGSTR_LEN + 1]; | ||
777 | |||
778 | seq_printf(m, "%c", probe_is_return(tp) ? 'r' : 'p'); | ||
779 | seq_printf(m, ":%s/%s", tp->call.system, tp->call.name); | ||
780 | |||
781 | if (!tp->symbol) | ||
782 | seq_printf(m, " 0x%p", tp->rp.kp.addr); | ||
783 | else if (tp->rp.kp.offset) | ||
784 | seq_printf(m, " %s+%u", probe_symbol(tp), tp->rp.kp.offset); | ||
785 | else | ||
786 | seq_printf(m, " %s", probe_symbol(tp)); | ||
787 | |||
788 | for (i = 0; i < tp->nr_args; i++) { | ||
789 | ret = probe_arg_string(buf, MAX_ARGSTR_LEN, &tp->args[i].fetch); | ||
790 | if (ret < 0) { | ||
791 | pr_warning("Argument%d decoding error(%d).\n", i, ret); | ||
792 | return ret; | ||
793 | } | ||
794 | seq_printf(m, " %s=%s", tp->args[i].name, buf); | ||
795 | } | ||
796 | seq_printf(m, "\n"); | ||
797 | return 0; | ||
798 | } | ||
799 | |||
800 | static const struct seq_operations probes_seq_op = { | ||
801 | .start = probes_seq_start, | ||
802 | .next = probes_seq_next, | ||
803 | .stop = probes_seq_stop, | ||
804 | .show = probes_seq_show | ||
805 | }; | ||
806 | |||
807 | static int probes_open(struct inode *inode, struct file *file) | ||
808 | { | ||
809 | if ((file->f_mode & FMODE_WRITE) && | ||
810 | (file->f_flags & O_TRUNC)) | ||
811 | cleanup_all_probes(); | ||
812 | |||
813 | return seq_open(file, &probes_seq_op); | ||
814 | } | ||
815 | |||
816 | static int command_trace_probe(const char *buf) | ||
817 | { | ||
818 | char **argv; | ||
819 | int argc = 0, ret = 0; | ||
820 | |||
821 | argv = argv_split(GFP_KERNEL, buf, &argc); | ||
822 | if (!argv) | ||
823 | return -ENOMEM; | ||
824 | |||
825 | if (argc) | ||
826 | ret = create_trace_probe(argc, argv); | ||
827 | |||
828 | argv_free(argv); | ||
829 | return ret; | ||
830 | } | ||
831 | |||
832 | #define WRITE_BUFSIZE 128 | ||
833 | |||
834 | static ssize_t probes_write(struct file *file, const char __user *buffer, | ||
835 | size_t count, loff_t *ppos) | ||
836 | { | ||
837 | char *kbuf, *tmp; | ||
838 | int ret; | ||
839 | size_t done; | ||
840 | size_t size; | ||
841 | |||
842 | kbuf = kmalloc(WRITE_BUFSIZE, GFP_KERNEL); | ||
843 | if (!kbuf) | ||
844 | return -ENOMEM; | ||
845 | |||
846 | ret = done = 0; | ||
847 | while (done < count) { | ||
848 | size = count - done; | ||
849 | if (size >= WRITE_BUFSIZE) | ||
850 | size = WRITE_BUFSIZE - 1; | ||
851 | if (copy_from_user(kbuf, buffer + done, size)) { | ||
852 | ret = -EFAULT; | ||
853 | goto out; | ||
854 | } | ||
855 | kbuf[size] = '\0'; | ||
856 | tmp = strchr(kbuf, '\n'); | ||
857 | if (tmp) { | ||
858 | *tmp = '\0'; | ||
859 | size = tmp - kbuf + 1; | ||
860 | } else if (done + size < count) { | ||
861 | pr_warning("Line length is too long: " | ||
862 | "Should be less than %d.", WRITE_BUFSIZE); | ||
863 | ret = -EINVAL; | ||
864 | goto out; | ||
865 | } | ||
866 | done += size; | ||
867 | /* Remove comments */ | ||
868 | tmp = strchr(kbuf, '#'); | ||
869 | if (tmp) | ||
870 | *tmp = '\0'; | ||
871 | |||
872 | ret = command_trace_probe(kbuf); | ||
873 | if (ret) | ||
874 | goto out; | ||
875 | } | ||
876 | ret = done; | ||
877 | out: | ||
878 | kfree(kbuf); | ||
879 | return ret; | ||
880 | } | ||
881 | |||
882 | static const struct file_operations kprobe_events_ops = { | ||
883 | .owner = THIS_MODULE, | ||
884 | .open = probes_open, | ||
885 | .read = seq_read, | ||
886 | .llseek = seq_lseek, | ||
887 | .release = seq_release, | ||
888 | .write = probes_write, | ||
889 | }; | ||
890 | |||
891 | /* Probes profiling interfaces */ | ||
892 | static int probes_profile_seq_show(struct seq_file *m, void *v) | ||
893 | { | ||
894 | struct trace_probe *tp = v; | ||
895 | |||
896 | seq_printf(m, " %-44s %15lu %15lu\n", tp->call.name, tp->nhit, | ||
897 | tp->rp.kp.nmissed); | ||
898 | |||
899 | return 0; | ||
900 | } | ||
901 | |||
902 | static const struct seq_operations profile_seq_op = { | ||
903 | .start = probes_seq_start, | ||
904 | .next = probes_seq_next, | ||
905 | .stop = probes_seq_stop, | ||
906 | .show = probes_profile_seq_show | ||
907 | }; | ||
908 | |||
909 | static int profile_open(struct inode *inode, struct file *file) | ||
910 | { | ||
911 | return seq_open(file, &profile_seq_op); | ||
912 | } | ||
913 | |||
914 | static const struct file_operations kprobe_profile_ops = { | ||
915 | .owner = THIS_MODULE, | ||
916 | .open = profile_open, | ||
917 | .read = seq_read, | ||
918 | .llseek = seq_lseek, | ||
919 | .release = seq_release, | ||
920 | }; | ||
921 | |||
922 | /* Kprobe handler */ | ||
923 | static __kprobes int kprobe_trace_func(struct kprobe *kp, struct pt_regs *regs) | ||
924 | { | ||
925 | struct trace_probe *tp = container_of(kp, struct trace_probe, rp.kp); | ||
926 | struct kprobe_trace_entry *entry; | ||
927 | struct ring_buffer_event *event; | ||
928 | struct ring_buffer *buffer; | ||
929 | int size, i, pc; | ||
930 | unsigned long irq_flags; | ||
931 | struct ftrace_event_call *call = &tp->call; | ||
932 | |||
933 | tp->nhit++; | ||
934 | |||
935 | local_save_flags(irq_flags); | ||
936 | pc = preempt_count(); | ||
937 | |||
938 | size = SIZEOF_KPROBE_TRACE_ENTRY(tp->nr_args); | ||
939 | |||
940 | event = trace_current_buffer_lock_reserve(&buffer, call->id, size, | ||
941 | irq_flags, pc); | ||
942 | if (!event) | ||
943 | return 0; | ||
944 | |||
945 | entry = ring_buffer_event_data(event); | ||
946 | entry->nargs = tp->nr_args; | ||
947 | entry->ip = (unsigned long)kp->addr; | ||
948 | for (i = 0; i < tp->nr_args; i++) | ||
949 | entry->args[i] = call_fetch(&tp->args[i].fetch, regs); | ||
950 | |||
951 | if (!filter_current_check_discard(buffer, call, entry, event)) | ||
952 | trace_nowake_buffer_unlock_commit(buffer, event, irq_flags, pc); | ||
953 | return 0; | ||
954 | } | ||
955 | |||
956 | /* Kretprobe handler */ | ||
957 | static __kprobes int kretprobe_trace_func(struct kretprobe_instance *ri, | ||
958 | struct pt_regs *regs) | ||
959 | { | ||
960 | struct trace_probe *tp = container_of(ri->rp, struct trace_probe, rp); | ||
961 | struct kretprobe_trace_entry *entry; | ||
962 | struct ring_buffer_event *event; | ||
963 | struct ring_buffer *buffer; | ||
964 | int size, i, pc; | ||
965 | unsigned long irq_flags; | ||
966 | struct ftrace_event_call *call = &tp->call; | ||
967 | |||
968 | local_save_flags(irq_flags); | ||
969 | pc = preempt_count(); | ||
970 | |||
971 | size = SIZEOF_KRETPROBE_TRACE_ENTRY(tp->nr_args); | ||
972 | |||
973 | event = trace_current_buffer_lock_reserve(&buffer, call->id, size, | ||
974 | irq_flags, pc); | ||
975 | if (!event) | ||
976 | return 0; | ||
977 | |||
978 | entry = ring_buffer_event_data(event); | ||
979 | entry->nargs = tp->nr_args; | ||
980 | entry->func = (unsigned long)tp->rp.kp.addr; | ||
981 | entry->ret_ip = (unsigned long)ri->ret_addr; | ||
982 | for (i = 0; i < tp->nr_args; i++) | ||
983 | entry->args[i] = call_fetch(&tp->args[i].fetch, regs); | ||
984 | |||
985 | if (!filter_current_check_discard(buffer, call, entry, event)) | ||
986 | trace_nowake_buffer_unlock_commit(buffer, event, irq_flags, pc); | ||
987 | |||
988 | return 0; | ||
989 | } | ||
990 | |||
991 | /* Event entry printers */ | ||
992 | enum print_line_t | ||
993 | print_kprobe_event(struct trace_iterator *iter, int flags) | ||
994 | { | ||
995 | struct kprobe_trace_entry *field; | ||
996 | struct trace_seq *s = &iter->seq; | ||
997 | struct trace_event *event; | ||
998 | struct trace_probe *tp; | ||
999 | int i; | ||
1000 | |||
1001 | field = (struct kprobe_trace_entry *)iter->ent; | ||
1002 | event = ftrace_find_event(field->ent.type); | ||
1003 | tp = container_of(event, struct trace_probe, event); | ||
1004 | |||
1005 | if (!trace_seq_printf(s, "%s: (", tp->call.name)) | ||
1006 | goto partial; | ||
1007 | |||
1008 | if (!seq_print_ip_sym(s, field->ip, flags | TRACE_ITER_SYM_OFFSET)) | ||
1009 | goto partial; | ||
1010 | |||
1011 | if (!trace_seq_puts(s, ")")) | ||
1012 | goto partial; | ||
1013 | |||
1014 | for (i = 0; i < field->nargs; i++) | ||
1015 | if (!trace_seq_printf(s, " %s=%lx", | ||
1016 | tp->args[i].name, field->args[i])) | ||
1017 | goto partial; | ||
1018 | |||
1019 | if (!trace_seq_puts(s, "\n")) | ||
1020 | goto partial; | ||
1021 | |||
1022 | return TRACE_TYPE_HANDLED; | ||
1023 | partial: | ||
1024 | return TRACE_TYPE_PARTIAL_LINE; | ||
1025 | } | ||
1026 | |||
1027 | enum print_line_t | ||
1028 | print_kretprobe_event(struct trace_iterator *iter, int flags) | ||
1029 | { | ||
1030 | struct kretprobe_trace_entry *field; | ||
1031 | struct trace_seq *s = &iter->seq; | ||
1032 | struct trace_event *event; | ||
1033 | struct trace_probe *tp; | ||
1034 | int i; | ||
1035 | |||
1036 | field = (struct kretprobe_trace_entry *)iter->ent; | ||
1037 | event = ftrace_find_event(field->ent.type); | ||
1038 | tp = container_of(event, struct trace_probe, event); | ||
1039 | |||
1040 | if (!trace_seq_printf(s, "%s: (", tp->call.name)) | ||
1041 | goto partial; | ||
1042 | |||
1043 | if (!seq_print_ip_sym(s, field->ret_ip, flags | TRACE_ITER_SYM_OFFSET)) | ||
1044 | goto partial; | ||
1045 | |||
1046 | if (!trace_seq_puts(s, " <- ")) | ||
1047 | goto partial; | ||
1048 | |||
1049 | if (!seq_print_ip_sym(s, field->func, flags & ~TRACE_ITER_SYM_OFFSET)) | ||
1050 | goto partial; | ||
1051 | |||
1052 | if (!trace_seq_puts(s, ")")) | ||
1053 | goto partial; | ||
1054 | |||
1055 | for (i = 0; i < field->nargs; i++) | ||
1056 | if (!trace_seq_printf(s, " %s=%lx", | ||
1057 | tp->args[i].name, field->args[i])) | ||
1058 | goto partial; | ||
1059 | |||
1060 | if (!trace_seq_puts(s, "\n")) | ||
1061 | goto partial; | ||
1062 | |||
1063 | return TRACE_TYPE_HANDLED; | ||
1064 | partial: | ||
1065 | return TRACE_TYPE_PARTIAL_LINE; | ||
1066 | } | ||
1067 | |||
1068 | static int probe_event_enable(struct ftrace_event_call *call) | ||
1069 | { | ||
1070 | struct trace_probe *tp = (struct trace_probe *)call->data; | ||
1071 | |||
1072 | tp->flags |= TP_FLAG_TRACE; | ||
1073 | if (probe_is_return(tp)) | ||
1074 | return enable_kretprobe(&tp->rp); | ||
1075 | else | ||
1076 | return enable_kprobe(&tp->rp.kp); | ||
1077 | } | ||
1078 | |||
1079 | static void probe_event_disable(struct ftrace_event_call *call) | ||
1080 | { | ||
1081 | struct trace_probe *tp = (struct trace_probe *)call->data; | ||
1082 | |||
1083 | tp->flags &= ~TP_FLAG_TRACE; | ||
1084 | if (!(tp->flags & (TP_FLAG_TRACE | TP_FLAG_PROFILE))) { | ||
1085 | if (probe_is_return(tp)) | ||
1086 | disable_kretprobe(&tp->rp); | ||
1087 | else | ||
1088 | disable_kprobe(&tp->rp.kp); | ||
1089 | } | ||
1090 | } | ||
1091 | |||
1092 | static int probe_event_raw_init(struct ftrace_event_call *event_call) | ||
1093 | { | ||
1094 | INIT_LIST_HEAD(&event_call->fields); | ||
1095 | |||
1096 | return 0; | ||
1097 | } | ||
1098 | |||
1099 | #undef DEFINE_FIELD | ||
1100 | #define DEFINE_FIELD(type, item, name, is_signed) \ | ||
1101 | do { \ | ||
1102 | ret = trace_define_field(event_call, #type, name, \ | ||
1103 | offsetof(typeof(field), item), \ | ||
1104 | sizeof(field.item), is_signed, \ | ||
1105 | FILTER_OTHER); \ | ||
1106 | if (ret) \ | ||
1107 | return ret; \ | ||
1108 | } while (0) | ||
1109 | |||
1110 | static int kprobe_event_define_fields(struct ftrace_event_call *event_call) | ||
1111 | { | ||
1112 | int ret, i; | ||
1113 | struct kprobe_trace_entry field; | ||
1114 | struct trace_probe *tp = (struct trace_probe *)event_call->data; | ||
1115 | |||
1116 | ret = trace_define_common_fields(event_call); | ||
1117 | if (!ret) | ||
1118 | return ret; | ||
1119 | |||
1120 | DEFINE_FIELD(unsigned long, ip, FIELD_STRING_IP, 0); | ||
1121 | DEFINE_FIELD(int, nargs, FIELD_STRING_NARGS, 1); | ||
1122 | /* Set argument names as fields */ | ||
1123 | for (i = 0; i < tp->nr_args; i++) | ||
1124 | DEFINE_FIELD(unsigned long, args[i], tp->args[i].name, 0); | ||
1125 | return 0; | ||
1126 | } | ||
1127 | |||
1128 | static int kretprobe_event_define_fields(struct ftrace_event_call *event_call) | ||
1129 | { | ||
1130 | int ret, i; | ||
1131 | struct kretprobe_trace_entry field; | ||
1132 | struct trace_probe *tp = (struct trace_probe *)event_call->data; | ||
1133 | |||
1134 | ret = trace_define_common_fields(event_call); | ||
1135 | if (!ret) | ||
1136 | return ret; | ||
1137 | |||
1138 | DEFINE_FIELD(unsigned long, func, FIELD_STRING_FUNC, 0); | ||
1139 | DEFINE_FIELD(unsigned long, ret_ip, FIELD_STRING_RETIP, 0); | ||
1140 | DEFINE_FIELD(int, nargs, FIELD_STRING_NARGS, 1); | ||
1141 | /* Set argument names as fields */ | ||
1142 | for (i = 0; i < tp->nr_args; i++) | ||
1143 | DEFINE_FIELD(unsigned long, args[i], tp->args[i].name, 0); | ||
1144 | return 0; | ||
1145 | } | ||
1146 | |||
1147 | static int __probe_event_show_format(struct trace_seq *s, | ||
1148 | struct trace_probe *tp, const char *fmt, | ||
1149 | const char *arg) | ||
1150 | { | ||
1151 | int i; | ||
1152 | |||
1153 | /* Show format */ | ||
1154 | if (!trace_seq_printf(s, "\nprint fmt: \"%s", fmt)) | ||
1155 | return 0; | ||
1156 | |||
1157 | for (i = 0; i < tp->nr_args; i++) | ||
1158 | if (!trace_seq_printf(s, " %s=%%lx", tp->args[i].name)) | ||
1159 | return 0; | ||
1160 | |||
1161 | if (!trace_seq_printf(s, "\", %s", arg)) | ||
1162 | return 0; | ||
1163 | |||
1164 | for (i = 0; i < tp->nr_args; i++) | ||
1165 | if (!trace_seq_printf(s, ", REC->%s", tp->args[i].name)) | ||
1166 | return 0; | ||
1167 | |||
1168 | return trace_seq_puts(s, "\n"); | ||
1169 | } | ||
1170 | |||
1171 | #undef SHOW_FIELD | ||
1172 | #define SHOW_FIELD(type, item, name) \ | ||
1173 | do { \ | ||
1174 | ret = trace_seq_printf(s, "\tfield: " #type " %s;\t" \ | ||
1175 | "offset:%u;\tsize:%u;\n", name, \ | ||
1176 | (unsigned int)offsetof(typeof(field), item),\ | ||
1177 | (unsigned int)sizeof(type)); \ | ||
1178 | if (!ret) \ | ||
1179 | return 0; \ | ||
1180 | } while (0) | ||
1181 | |||
1182 | static int kprobe_event_show_format(struct ftrace_event_call *call, | ||
1183 | struct trace_seq *s) | ||
1184 | { | ||
1185 | struct kprobe_trace_entry field __attribute__((unused)); | ||
1186 | int ret, i; | ||
1187 | struct trace_probe *tp = (struct trace_probe *)call->data; | ||
1188 | |||
1189 | SHOW_FIELD(unsigned long, ip, FIELD_STRING_IP); | ||
1190 | SHOW_FIELD(int, nargs, FIELD_STRING_NARGS); | ||
1191 | |||
1192 | /* Show fields */ | ||
1193 | for (i = 0; i < tp->nr_args; i++) | ||
1194 | SHOW_FIELD(unsigned long, args[i], tp->args[i].name); | ||
1195 | trace_seq_puts(s, "\n"); | ||
1196 | |||
1197 | return __probe_event_show_format(s, tp, "(%lx)", | ||
1198 | "REC->" FIELD_STRING_IP); | ||
1199 | } | ||
1200 | |||
1201 | static int kretprobe_event_show_format(struct ftrace_event_call *call, | ||
1202 | struct trace_seq *s) | ||
1203 | { | ||
1204 | struct kretprobe_trace_entry field __attribute__((unused)); | ||
1205 | int ret, i; | ||
1206 | struct trace_probe *tp = (struct trace_probe *)call->data; | ||
1207 | |||
1208 | SHOW_FIELD(unsigned long, func, FIELD_STRING_FUNC); | ||
1209 | SHOW_FIELD(unsigned long, ret_ip, FIELD_STRING_RETIP); | ||
1210 | SHOW_FIELD(int, nargs, FIELD_STRING_NARGS); | ||
1211 | |||
1212 | /* Show fields */ | ||
1213 | for (i = 0; i < tp->nr_args; i++) | ||
1214 | SHOW_FIELD(unsigned long, args[i], tp->args[i].name); | ||
1215 | trace_seq_puts(s, "\n"); | ||
1216 | |||
1217 | return __probe_event_show_format(s, tp, "(%lx <- %lx)", | ||
1218 | "REC->" FIELD_STRING_FUNC | ||
1219 | ", REC->" FIELD_STRING_RETIP); | ||
1220 | } | ||
1221 | |||
1222 | #ifdef CONFIG_EVENT_PROFILE | ||
1223 | |||
1224 | /* Kprobe profile handler */ | ||
1225 | static __kprobes int kprobe_profile_func(struct kprobe *kp, | ||
1226 | struct pt_regs *regs) | ||
1227 | { | ||
1228 | struct trace_probe *tp = container_of(kp, struct trace_probe, rp.kp); | ||
1229 | struct ftrace_event_call *call = &tp->call; | ||
1230 | struct kprobe_trace_entry *entry; | ||
1231 | struct trace_entry *ent; | ||
1232 | int size, __size, i, pc, __cpu; | ||
1233 | unsigned long irq_flags; | ||
1234 | char *trace_buf; | ||
1235 | char *raw_data; | ||
1236 | int rctx; | ||
1237 | |||
1238 | pc = preempt_count(); | ||
1239 | __size = SIZEOF_KPROBE_TRACE_ENTRY(tp->nr_args); | ||
1240 | size = ALIGN(__size + sizeof(u32), sizeof(u64)); | ||
1241 | size -= sizeof(u32); | ||
1242 | if (WARN_ONCE(size > FTRACE_MAX_PROFILE_SIZE, | ||
1243 | "profile buffer not large enough")) | ||
1244 | return 0; | ||
1245 | |||
1246 | /* | ||
1247 | * Protect the non nmi buffer | ||
1248 | * This also protects the rcu read side | ||
1249 | */ | ||
1250 | local_irq_save(irq_flags); | ||
1251 | |||
1252 | rctx = perf_swevent_get_recursion_context(); | ||
1253 | if (rctx < 0) | ||
1254 | goto end_recursion; | ||
1255 | |||
1256 | __cpu = smp_processor_id(); | ||
1257 | |||
1258 | if (in_nmi()) | ||
1259 | trace_buf = rcu_dereference(perf_trace_buf_nmi); | ||
1260 | else | ||
1261 | trace_buf = rcu_dereference(perf_trace_buf); | ||
1262 | |||
1263 | if (!trace_buf) | ||
1264 | goto end; | ||
1265 | |||
1266 | raw_data = per_cpu_ptr(trace_buf, __cpu); | ||
1267 | |||
1268 | /* Zero dead bytes from alignment to avoid buffer leak to userspace */ | ||
1269 | *(u64 *)(&raw_data[size - sizeof(u64)]) = 0ULL; | ||
1270 | entry = (struct kprobe_trace_entry *)raw_data; | ||
1271 | ent = &entry->ent; | ||
1272 | |||
1273 | tracing_generic_entry_update(ent, irq_flags, pc); | ||
1274 | ent->type = call->id; | ||
1275 | entry->nargs = tp->nr_args; | ||
1276 | entry->ip = (unsigned long)kp->addr; | ||
1277 | for (i = 0; i < tp->nr_args; i++) | ||
1278 | entry->args[i] = call_fetch(&tp->args[i].fetch, regs); | ||
1279 | perf_tp_event(call->id, entry->ip, 1, entry, size); | ||
1280 | |||
1281 | end: | ||
1282 | perf_swevent_put_recursion_context(rctx); | ||
1283 | end_recursion: | ||
1284 | local_irq_restore(irq_flags); | ||
1285 | |||
1286 | return 0; | ||
1287 | } | ||
1288 | |||
1289 | /* Kretprobe profile handler */ | ||
1290 | static __kprobes int kretprobe_profile_func(struct kretprobe_instance *ri, | ||
1291 | struct pt_regs *regs) | ||
1292 | { | ||
1293 | struct trace_probe *tp = container_of(ri->rp, struct trace_probe, rp); | ||
1294 | struct ftrace_event_call *call = &tp->call; | ||
1295 | struct kretprobe_trace_entry *entry; | ||
1296 | struct trace_entry *ent; | ||
1297 | int size, __size, i, pc, __cpu; | ||
1298 | unsigned long irq_flags; | ||
1299 | char *trace_buf; | ||
1300 | char *raw_data; | ||
1301 | int rctx; | ||
1302 | |||
1303 | pc = preempt_count(); | ||
1304 | __size = SIZEOF_KRETPROBE_TRACE_ENTRY(tp->nr_args); | ||
1305 | size = ALIGN(__size + sizeof(u32), sizeof(u64)); | ||
1306 | size -= sizeof(u32); | ||
1307 | if (WARN_ONCE(size > FTRACE_MAX_PROFILE_SIZE, | ||
1308 | "profile buffer not large enough")) | ||
1309 | return 0; | ||
1310 | |||
1311 | /* | ||
1312 | * Protect the non nmi buffer | ||
1313 | * This also protects the rcu read side | ||
1314 | */ | ||
1315 | local_irq_save(irq_flags); | ||
1316 | |||
1317 | rctx = perf_swevent_get_recursion_context(); | ||
1318 | if (rctx < 0) | ||
1319 | goto end_recursion; | ||
1320 | |||
1321 | __cpu = smp_processor_id(); | ||
1322 | |||
1323 | if (in_nmi()) | ||
1324 | trace_buf = rcu_dereference(perf_trace_buf_nmi); | ||
1325 | else | ||
1326 | trace_buf = rcu_dereference(perf_trace_buf); | ||
1327 | |||
1328 | if (!trace_buf) | ||
1329 | goto end; | ||
1330 | |||
1331 | raw_data = per_cpu_ptr(trace_buf, __cpu); | ||
1332 | |||
1333 | /* Zero dead bytes from alignment to avoid buffer leak to userspace */ | ||
1334 | *(u64 *)(&raw_data[size - sizeof(u64)]) = 0ULL; | ||
1335 | entry = (struct kretprobe_trace_entry *)raw_data; | ||
1336 | ent = &entry->ent; | ||
1337 | |||
1338 | tracing_generic_entry_update(ent, irq_flags, pc); | ||
1339 | ent->type = call->id; | ||
1340 | entry->nargs = tp->nr_args; | ||
1341 | entry->func = (unsigned long)tp->rp.kp.addr; | ||
1342 | entry->ret_ip = (unsigned long)ri->ret_addr; | ||
1343 | for (i = 0; i < tp->nr_args; i++) | ||
1344 | entry->args[i] = call_fetch(&tp->args[i].fetch, regs); | ||
1345 | perf_tp_event(call->id, entry->ret_ip, 1, entry, size); | ||
1346 | |||
1347 | end: | ||
1348 | perf_swevent_put_recursion_context(rctx); | ||
1349 | end_recursion: | ||
1350 | local_irq_restore(irq_flags); | ||
1351 | |||
1352 | return 0; | ||
1353 | } | ||
1354 | |||
1355 | static int probe_profile_enable(struct ftrace_event_call *call) | ||
1356 | { | ||
1357 | struct trace_probe *tp = (struct trace_probe *)call->data; | ||
1358 | |||
1359 | tp->flags |= TP_FLAG_PROFILE; | ||
1360 | |||
1361 | if (probe_is_return(tp)) | ||
1362 | return enable_kretprobe(&tp->rp); | ||
1363 | else | ||
1364 | return enable_kprobe(&tp->rp.kp); | ||
1365 | } | ||
1366 | |||
1367 | static void probe_profile_disable(struct ftrace_event_call *call) | ||
1368 | { | ||
1369 | struct trace_probe *tp = (struct trace_probe *)call->data; | ||
1370 | |||
1371 | tp->flags &= ~TP_FLAG_PROFILE; | ||
1372 | |||
1373 | if (!(tp->flags & TP_FLAG_TRACE)) { | ||
1374 | if (probe_is_return(tp)) | ||
1375 | disable_kretprobe(&tp->rp); | ||
1376 | else | ||
1377 | disable_kprobe(&tp->rp.kp); | ||
1378 | } | ||
1379 | } | ||
1380 | #endif /* CONFIG_EVENT_PROFILE */ | ||
1381 | |||
1382 | |||
1383 | static __kprobes | ||
1384 | int kprobe_dispatcher(struct kprobe *kp, struct pt_regs *regs) | ||
1385 | { | ||
1386 | struct trace_probe *tp = container_of(kp, struct trace_probe, rp.kp); | ||
1387 | |||
1388 | if (tp->flags & TP_FLAG_TRACE) | ||
1389 | kprobe_trace_func(kp, regs); | ||
1390 | #ifdef CONFIG_EVENT_PROFILE | ||
1391 | if (tp->flags & TP_FLAG_PROFILE) | ||
1392 | kprobe_profile_func(kp, regs); | ||
1393 | #endif /* CONFIG_EVENT_PROFILE */ | ||
1394 | return 0; /* We don't tweek kernel, so just return 0 */ | ||
1395 | } | ||
1396 | |||
1397 | static __kprobes | ||
1398 | int kretprobe_dispatcher(struct kretprobe_instance *ri, struct pt_regs *regs) | ||
1399 | { | ||
1400 | struct trace_probe *tp = container_of(ri->rp, struct trace_probe, rp); | ||
1401 | |||
1402 | if (tp->flags & TP_FLAG_TRACE) | ||
1403 | kretprobe_trace_func(ri, regs); | ||
1404 | #ifdef CONFIG_EVENT_PROFILE | ||
1405 | if (tp->flags & TP_FLAG_PROFILE) | ||
1406 | kretprobe_profile_func(ri, regs); | ||
1407 | #endif /* CONFIG_EVENT_PROFILE */ | ||
1408 | return 0; /* We don't tweek kernel, so just return 0 */ | ||
1409 | } | ||
1410 | |||
1411 | static int register_probe_event(struct trace_probe *tp) | ||
1412 | { | ||
1413 | struct ftrace_event_call *call = &tp->call; | ||
1414 | int ret; | ||
1415 | |||
1416 | /* Initialize ftrace_event_call */ | ||
1417 | if (probe_is_return(tp)) { | ||
1418 | tp->event.trace = print_kretprobe_event; | ||
1419 | call->raw_init = probe_event_raw_init; | ||
1420 | call->show_format = kretprobe_event_show_format; | ||
1421 | call->define_fields = kretprobe_event_define_fields; | ||
1422 | } else { | ||
1423 | tp->event.trace = print_kprobe_event; | ||
1424 | call->raw_init = probe_event_raw_init; | ||
1425 | call->show_format = kprobe_event_show_format; | ||
1426 | call->define_fields = kprobe_event_define_fields; | ||
1427 | } | ||
1428 | call->event = &tp->event; | ||
1429 | call->id = register_ftrace_event(&tp->event); | ||
1430 | if (!call->id) | ||
1431 | return -ENODEV; | ||
1432 | call->enabled = 0; | ||
1433 | call->regfunc = probe_event_enable; | ||
1434 | call->unregfunc = probe_event_disable; | ||
1435 | |||
1436 | #ifdef CONFIG_EVENT_PROFILE | ||
1437 | atomic_set(&call->profile_count, -1); | ||
1438 | call->profile_enable = probe_profile_enable; | ||
1439 | call->profile_disable = probe_profile_disable; | ||
1440 | #endif | ||
1441 | call->data = tp; | ||
1442 | ret = trace_add_event_call(call); | ||
1443 | if (ret) { | ||
1444 | pr_info("Failed to register kprobe event: %s\n", call->name); | ||
1445 | unregister_ftrace_event(&tp->event); | ||
1446 | } | ||
1447 | return ret; | ||
1448 | } | ||
1449 | |||
1450 | static void unregister_probe_event(struct trace_probe *tp) | ||
1451 | { | ||
1452 | /* tp->event is unregistered in trace_remove_event_call() */ | ||
1453 | trace_remove_event_call(&tp->call); | ||
1454 | } | ||
1455 | |||
1456 | /* Make a debugfs interface for controling probe points */ | ||
1457 | static __init int init_kprobe_trace(void) | ||
1458 | { | ||
1459 | struct dentry *d_tracer; | ||
1460 | struct dentry *entry; | ||
1461 | |||
1462 | d_tracer = tracing_init_dentry(); | ||
1463 | if (!d_tracer) | ||
1464 | return 0; | ||
1465 | |||
1466 | entry = debugfs_create_file("kprobe_events", 0644, d_tracer, | ||
1467 | NULL, &kprobe_events_ops); | ||
1468 | |||
1469 | /* Event list interface */ | ||
1470 | if (!entry) | ||
1471 | pr_warning("Could not create debugfs " | ||
1472 | "'kprobe_events' entry\n"); | ||
1473 | |||
1474 | /* Profile interface */ | ||
1475 | entry = debugfs_create_file("kprobe_profile", 0444, d_tracer, | ||
1476 | NULL, &kprobe_profile_ops); | ||
1477 | |||
1478 | if (!entry) | ||
1479 | pr_warning("Could not create debugfs " | ||
1480 | "'kprobe_profile' entry\n"); | ||
1481 | return 0; | ||
1482 | } | ||
1483 | fs_initcall(init_kprobe_trace); | ||
1484 | |||
1485 | |||
1486 | #ifdef CONFIG_FTRACE_STARTUP_TEST | ||
1487 | |||
1488 | static int kprobe_trace_selftest_target(int a1, int a2, int a3, | ||
1489 | int a4, int a5, int a6) | ||
1490 | { | ||
1491 | return a1 + a2 + a3 + a4 + a5 + a6; | ||
1492 | } | ||
1493 | |||
1494 | static __init int kprobe_trace_self_tests_init(void) | ||
1495 | { | ||
1496 | int ret; | ||
1497 | int (*target)(int, int, int, int, int, int); | ||
1498 | |||
1499 | target = kprobe_trace_selftest_target; | ||
1500 | |||
1501 | pr_info("Testing kprobe tracing: "); | ||
1502 | |||
1503 | ret = command_trace_probe("p:testprobe kprobe_trace_selftest_target " | ||
1504 | "$arg1 $arg2 $arg3 $arg4 $stack $stack0"); | ||
1505 | if (WARN_ON_ONCE(ret)) | ||
1506 | pr_warning("error enabling function entry\n"); | ||
1507 | |||
1508 | ret = command_trace_probe("r:testprobe2 kprobe_trace_selftest_target " | ||
1509 | "$retval"); | ||
1510 | if (WARN_ON_ONCE(ret)) | ||
1511 | pr_warning("error enabling function return\n"); | ||
1512 | |||
1513 | ret = target(1, 2, 3, 4, 5, 6); | ||
1514 | |||
1515 | cleanup_all_probes(); | ||
1516 | |||
1517 | pr_cont("OK\n"); | ||
1518 | return 0; | ||
1519 | } | ||
1520 | |||
1521 | late_initcall(kprobe_trace_self_tests_init); | ||
1522 | |||
1523 | #endif | ||
diff --git a/kernel/trace/trace_ksym.c b/kernel/trace/trace_ksym.c new file mode 100644 index 000000000000..ddfa0fd43bc0 --- /dev/null +++ b/kernel/trace/trace_ksym.c | |||
@@ -0,0 +1,550 @@ | |||
1 | /* | ||
2 | * trace_ksym.c - Kernel Symbol Tracer | ||
3 | * | ||
4 | * This program is free software; you can redistribute it and/or modify | ||
5 | * it under the terms of the GNU General Public License as published by | ||
6 | * the Free Software Foundation; either version 2 of the License, or | ||
7 | * (at your option) any later version. | ||
8 | * | ||
9 | * This program is distributed in the hope that it will be useful, | ||
10 | * but WITHOUT ANY WARRANTY; without even the implied warranty of | ||
11 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | ||
12 | * GNU General Public License for more details. | ||
13 | * | ||
14 | * You should have received a copy of the GNU General Public License | ||
15 | * along with this program; if not, write to the Free Software | ||
16 | * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. | ||
17 | * | ||
18 | * Copyright (C) IBM Corporation, 2009 | ||
19 | */ | ||
20 | |||
21 | #include <linux/kallsyms.h> | ||
22 | #include <linux/uaccess.h> | ||
23 | #include <linux/debugfs.h> | ||
24 | #include <linux/ftrace.h> | ||
25 | #include <linux/module.h> | ||
26 | #include <linux/fs.h> | ||
27 | |||
28 | #include "trace_output.h" | ||
29 | #include "trace_stat.h" | ||
30 | #include "trace.h" | ||
31 | |||
32 | #include <linux/hw_breakpoint.h> | ||
33 | #include <asm/hw_breakpoint.h> | ||
34 | |||
35 | /* | ||
36 | * For now, let us restrict the no. of symbols traced simultaneously to number | ||
37 | * of available hardware breakpoint registers. | ||
38 | */ | ||
39 | #define KSYM_TRACER_MAX HBP_NUM | ||
40 | |||
41 | #define KSYM_TRACER_OP_LEN 3 /* rw- */ | ||
42 | |||
43 | struct trace_ksym { | ||
44 | struct perf_event **ksym_hbp; | ||
45 | struct perf_event_attr attr; | ||
46 | #ifdef CONFIG_PROFILE_KSYM_TRACER | ||
47 | unsigned long counter; | ||
48 | #endif | ||
49 | struct hlist_node ksym_hlist; | ||
50 | }; | ||
51 | |||
52 | static struct trace_array *ksym_trace_array; | ||
53 | |||
54 | static unsigned int ksym_filter_entry_count; | ||
55 | static unsigned int ksym_tracing_enabled; | ||
56 | |||
57 | static HLIST_HEAD(ksym_filter_head); | ||
58 | |||
59 | static DEFINE_MUTEX(ksym_tracer_mutex); | ||
60 | |||
61 | #ifdef CONFIG_PROFILE_KSYM_TRACER | ||
62 | |||
63 | #define MAX_UL_INT 0xffffffff | ||
64 | |||
65 | void ksym_collect_stats(unsigned long hbp_hit_addr) | ||
66 | { | ||
67 | struct hlist_node *node; | ||
68 | struct trace_ksym *entry; | ||
69 | |||
70 | rcu_read_lock(); | ||
71 | hlist_for_each_entry_rcu(entry, node, &ksym_filter_head, ksym_hlist) { | ||
72 | if ((entry->attr.bp_addr == hbp_hit_addr) && | ||
73 | (entry->counter <= MAX_UL_INT)) { | ||
74 | entry->counter++; | ||
75 | break; | ||
76 | } | ||
77 | } | ||
78 | rcu_read_unlock(); | ||
79 | } | ||
80 | #endif /* CONFIG_PROFILE_KSYM_TRACER */ | ||
81 | |||
82 | void ksym_hbp_handler(struct perf_event *hbp, void *data) | ||
83 | { | ||
84 | struct ring_buffer_event *event; | ||
85 | struct ksym_trace_entry *entry; | ||
86 | struct pt_regs *regs = data; | ||
87 | struct ring_buffer *buffer; | ||
88 | int pc; | ||
89 | |||
90 | if (!ksym_tracing_enabled) | ||
91 | return; | ||
92 | |||
93 | buffer = ksym_trace_array->buffer; | ||
94 | |||
95 | pc = preempt_count(); | ||
96 | |||
97 | event = trace_buffer_lock_reserve(buffer, TRACE_KSYM, | ||
98 | sizeof(*entry), 0, pc); | ||
99 | if (!event) | ||
100 | return; | ||
101 | |||
102 | entry = ring_buffer_event_data(event); | ||
103 | entry->ip = instruction_pointer(regs); | ||
104 | entry->type = hw_breakpoint_type(hbp); | ||
105 | entry->addr = hw_breakpoint_addr(hbp); | ||
106 | strlcpy(entry->cmd, current->comm, TASK_COMM_LEN); | ||
107 | |||
108 | #ifdef CONFIG_PROFILE_KSYM_TRACER | ||
109 | ksym_collect_stats(hw_breakpoint_addr(hbp)); | ||
110 | #endif /* CONFIG_PROFILE_KSYM_TRACER */ | ||
111 | |||
112 | trace_buffer_unlock_commit(buffer, event, 0, pc); | ||
113 | } | ||
114 | |||
115 | /* Valid access types are represented as | ||
116 | * | ||
117 | * rw- : Set Read/Write Access Breakpoint | ||
118 | * -w- : Set Write Access Breakpoint | ||
119 | * --- : Clear Breakpoints | ||
120 | * --x : Set Execution Break points (Not available yet) | ||
121 | * | ||
122 | */ | ||
123 | static int ksym_trace_get_access_type(char *str) | ||
124 | { | ||
125 | int access = 0; | ||
126 | |||
127 | if (str[0] == 'r') | ||
128 | access |= HW_BREAKPOINT_R; | ||
129 | |||
130 | if (str[1] == 'w') | ||
131 | access |= HW_BREAKPOINT_W; | ||
132 | |||
133 | if (str[2] == 'x') | ||
134 | access |= HW_BREAKPOINT_X; | ||
135 | |||
136 | switch (access) { | ||
137 | case HW_BREAKPOINT_R: | ||
138 | case HW_BREAKPOINT_W: | ||
139 | case HW_BREAKPOINT_W | HW_BREAKPOINT_R: | ||
140 | return access; | ||
141 | default: | ||
142 | return -EINVAL; | ||
143 | } | ||
144 | } | ||
145 | |||
146 | /* | ||
147 | * There can be several possible malformed requests and we attempt to capture | ||
148 | * all of them. We enumerate some of the rules | ||
149 | * 1. We will not allow kernel symbols with ':' since it is used as a delimiter. | ||
150 | * i.e. multiple ':' symbols disallowed. Possible uses are of the form | ||
151 | * <module>:<ksym_name>:<op>. | ||
152 | * 2. No delimiter symbol ':' in the input string | ||
153 | * 3. Spurious operator symbols or symbols not in their respective positions | ||
154 | * 4. <ksym_name>:--- i.e. clear breakpoint request when ksym_name not in file | ||
155 | * 5. Kernel symbol not a part of /proc/kallsyms | ||
156 | * 6. Duplicate requests | ||
157 | */ | ||
158 | static int parse_ksym_trace_str(char *input_string, char **ksymname, | ||
159 | unsigned long *addr) | ||
160 | { | ||
161 | int ret; | ||
162 | |||
163 | *ksymname = strsep(&input_string, ":"); | ||
164 | *addr = kallsyms_lookup_name(*ksymname); | ||
165 | |||
166 | /* Check for malformed request: (2), (1) and (5) */ | ||
167 | if ((!input_string) || | ||
168 | (strlen(input_string) != KSYM_TRACER_OP_LEN) || | ||
169 | (*addr == 0)) | ||
170 | return -EINVAL;; | ||
171 | |||
172 | ret = ksym_trace_get_access_type(input_string); | ||
173 | |||
174 | return ret; | ||
175 | } | ||
176 | |||
177 | int process_new_ksym_entry(char *ksymname, int op, unsigned long addr) | ||
178 | { | ||
179 | struct trace_ksym *entry; | ||
180 | int ret = -ENOMEM; | ||
181 | |||
182 | if (ksym_filter_entry_count >= KSYM_TRACER_MAX) { | ||
183 | printk(KERN_ERR "ksym_tracer: Maximum limit:(%d) reached. No" | ||
184 | " new requests for tracing can be accepted now.\n", | ||
185 | KSYM_TRACER_MAX); | ||
186 | return -ENOSPC; | ||
187 | } | ||
188 | |||
189 | entry = kzalloc(sizeof(struct trace_ksym), GFP_KERNEL); | ||
190 | if (!entry) | ||
191 | return -ENOMEM; | ||
192 | |||
193 | hw_breakpoint_init(&entry->attr); | ||
194 | |||
195 | entry->attr.bp_type = op; | ||
196 | entry->attr.bp_addr = addr; | ||
197 | entry->attr.bp_len = HW_BREAKPOINT_LEN_4; | ||
198 | |||
199 | ret = -EAGAIN; | ||
200 | entry->ksym_hbp = register_wide_hw_breakpoint(&entry->attr, | ||
201 | ksym_hbp_handler); | ||
202 | |||
203 | if (IS_ERR(entry->ksym_hbp)) { | ||
204 | ret = PTR_ERR(entry->ksym_hbp); | ||
205 | printk(KERN_INFO "ksym_tracer request failed. Try again" | ||
206 | " later!!\n"); | ||
207 | goto err; | ||
208 | } | ||
209 | |||
210 | hlist_add_head_rcu(&(entry->ksym_hlist), &ksym_filter_head); | ||
211 | ksym_filter_entry_count++; | ||
212 | |||
213 | return 0; | ||
214 | |||
215 | err: | ||
216 | kfree(entry); | ||
217 | |||
218 | return ret; | ||
219 | } | ||
220 | |||
221 | static ssize_t ksym_trace_filter_read(struct file *filp, char __user *ubuf, | ||
222 | size_t count, loff_t *ppos) | ||
223 | { | ||
224 | struct trace_ksym *entry; | ||
225 | struct hlist_node *node; | ||
226 | struct trace_seq *s; | ||
227 | ssize_t cnt = 0; | ||
228 | int ret; | ||
229 | |||
230 | s = kmalloc(sizeof(*s), GFP_KERNEL); | ||
231 | if (!s) | ||
232 | return -ENOMEM; | ||
233 | trace_seq_init(s); | ||
234 | |||
235 | mutex_lock(&ksym_tracer_mutex); | ||
236 | |||
237 | hlist_for_each_entry(entry, node, &ksym_filter_head, ksym_hlist) { | ||
238 | ret = trace_seq_printf(s, "%pS:", (void *)entry->attr.bp_addr); | ||
239 | if (entry->attr.bp_type == HW_BREAKPOINT_R) | ||
240 | ret = trace_seq_puts(s, "r--\n"); | ||
241 | else if (entry->attr.bp_type == HW_BREAKPOINT_W) | ||
242 | ret = trace_seq_puts(s, "-w-\n"); | ||
243 | else if (entry->attr.bp_type == (HW_BREAKPOINT_W | HW_BREAKPOINT_R)) | ||
244 | ret = trace_seq_puts(s, "rw-\n"); | ||
245 | WARN_ON_ONCE(!ret); | ||
246 | } | ||
247 | |||
248 | cnt = simple_read_from_buffer(ubuf, count, ppos, s->buffer, s->len); | ||
249 | |||
250 | mutex_unlock(&ksym_tracer_mutex); | ||
251 | |||
252 | kfree(s); | ||
253 | |||
254 | return cnt; | ||
255 | } | ||
256 | |||
257 | static void __ksym_trace_reset(void) | ||
258 | { | ||
259 | struct trace_ksym *entry; | ||
260 | struct hlist_node *node, *node1; | ||
261 | |||
262 | mutex_lock(&ksym_tracer_mutex); | ||
263 | hlist_for_each_entry_safe(entry, node, node1, &ksym_filter_head, | ||
264 | ksym_hlist) { | ||
265 | unregister_wide_hw_breakpoint(entry->ksym_hbp); | ||
266 | ksym_filter_entry_count--; | ||
267 | hlist_del_rcu(&(entry->ksym_hlist)); | ||
268 | synchronize_rcu(); | ||
269 | kfree(entry); | ||
270 | } | ||
271 | mutex_unlock(&ksym_tracer_mutex); | ||
272 | } | ||
273 | |||
274 | static ssize_t ksym_trace_filter_write(struct file *file, | ||
275 | const char __user *buffer, | ||
276 | size_t count, loff_t *ppos) | ||
277 | { | ||
278 | struct trace_ksym *entry; | ||
279 | struct hlist_node *node; | ||
280 | char *input_string, *ksymname = NULL; | ||
281 | unsigned long ksym_addr = 0; | ||
282 | int ret, op, changed = 0; | ||
283 | |||
284 | input_string = kzalloc(count + 1, GFP_KERNEL); | ||
285 | if (!input_string) | ||
286 | return -ENOMEM; | ||
287 | |||
288 | if (copy_from_user(input_string, buffer, count)) { | ||
289 | kfree(input_string); | ||
290 | return -EFAULT; | ||
291 | } | ||
292 | input_string[count] = '\0'; | ||
293 | |||
294 | strstrip(input_string); | ||
295 | |||
296 | /* | ||
297 | * Clear all breakpoints if: | ||
298 | * 1: echo > ksym_trace_filter | ||
299 | * 2: echo 0 > ksym_trace_filter | ||
300 | * 3: echo "*:---" > ksym_trace_filter | ||
301 | */ | ||
302 | if (!input_string[0] || !strcmp(input_string, "0") || | ||
303 | !strcmp(input_string, "*:---")) { | ||
304 | __ksym_trace_reset(); | ||
305 | kfree(input_string); | ||
306 | return count; | ||
307 | } | ||
308 | |||
309 | ret = op = parse_ksym_trace_str(input_string, &ksymname, &ksym_addr); | ||
310 | if (ret < 0) { | ||
311 | kfree(input_string); | ||
312 | return ret; | ||
313 | } | ||
314 | |||
315 | mutex_lock(&ksym_tracer_mutex); | ||
316 | |||
317 | ret = -EINVAL; | ||
318 | hlist_for_each_entry(entry, node, &ksym_filter_head, ksym_hlist) { | ||
319 | if (entry->attr.bp_addr == ksym_addr) { | ||
320 | /* Check for malformed request: (6) */ | ||
321 | if (entry->attr.bp_type != op) | ||
322 | changed = 1; | ||
323 | else | ||
324 | goto out; | ||
325 | break; | ||
326 | } | ||
327 | } | ||
328 | if (changed) { | ||
329 | unregister_wide_hw_breakpoint(entry->ksym_hbp); | ||
330 | entry->attr.bp_type = op; | ||
331 | ret = 0; | ||
332 | if (op > 0) { | ||
333 | entry->ksym_hbp = | ||
334 | register_wide_hw_breakpoint(&entry->attr, | ||
335 | ksym_hbp_handler); | ||
336 | if (IS_ERR(entry->ksym_hbp)) | ||
337 | ret = PTR_ERR(entry->ksym_hbp); | ||
338 | else | ||
339 | goto out; | ||
340 | } | ||
341 | /* Error or "symbol:---" case: drop it */ | ||
342 | ksym_filter_entry_count--; | ||
343 | hlist_del_rcu(&(entry->ksym_hlist)); | ||
344 | synchronize_rcu(); | ||
345 | kfree(entry); | ||
346 | goto out; | ||
347 | } else { | ||
348 | /* Check for malformed request: (4) */ | ||
349 | if (op == 0) | ||
350 | goto out; | ||
351 | ret = process_new_ksym_entry(ksymname, op, ksym_addr); | ||
352 | } | ||
353 | out: | ||
354 | mutex_unlock(&ksym_tracer_mutex); | ||
355 | |||
356 | kfree(input_string); | ||
357 | |||
358 | if (!ret) | ||
359 | ret = count; | ||
360 | return ret; | ||
361 | } | ||
362 | |||
363 | static const struct file_operations ksym_tracing_fops = { | ||
364 | .open = tracing_open_generic, | ||
365 | .read = ksym_trace_filter_read, | ||
366 | .write = ksym_trace_filter_write, | ||
367 | }; | ||
368 | |||
369 | static void ksym_trace_reset(struct trace_array *tr) | ||
370 | { | ||
371 | ksym_tracing_enabled = 0; | ||
372 | __ksym_trace_reset(); | ||
373 | } | ||
374 | |||
375 | static int ksym_trace_init(struct trace_array *tr) | ||
376 | { | ||
377 | int cpu, ret = 0; | ||
378 | |||
379 | for_each_online_cpu(cpu) | ||
380 | tracing_reset(tr, cpu); | ||
381 | ksym_tracing_enabled = 1; | ||
382 | ksym_trace_array = tr; | ||
383 | |||
384 | return ret; | ||
385 | } | ||
386 | |||
387 | static void ksym_trace_print_header(struct seq_file *m) | ||
388 | { | ||
389 | seq_puts(m, | ||
390 | "# TASK-PID CPU# Symbol " | ||
391 | "Type Function\n"); | ||
392 | seq_puts(m, | ||
393 | "# | | | " | ||
394 | " | |\n"); | ||
395 | } | ||
396 | |||
397 | static enum print_line_t ksym_trace_output(struct trace_iterator *iter) | ||
398 | { | ||
399 | struct trace_entry *entry = iter->ent; | ||
400 | struct trace_seq *s = &iter->seq; | ||
401 | struct ksym_trace_entry *field; | ||
402 | char str[KSYM_SYMBOL_LEN]; | ||
403 | int ret; | ||
404 | |||
405 | if (entry->type != TRACE_KSYM) | ||
406 | return TRACE_TYPE_UNHANDLED; | ||
407 | |||
408 | trace_assign_type(field, entry); | ||
409 | |||
410 | ret = trace_seq_printf(s, "%11s-%-5d [%03d] %pS", field->cmd, | ||
411 | entry->pid, iter->cpu, (char *)field->addr); | ||
412 | if (!ret) | ||
413 | return TRACE_TYPE_PARTIAL_LINE; | ||
414 | |||
415 | switch (field->type) { | ||
416 | case HW_BREAKPOINT_R: | ||
417 | ret = trace_seq_printf(s, " R "); | ||
418 | break; | ||
419 | case HW_BREAKPOINT_W: | ||
420 | ret = trace_seq_printf(s, " W "); | ||
421 | break; | ||
422 | case HW_BREAKPOINT_R | HW_BREAKPOINT_W: | ||
423 | ret = trace_seq_printf(s, " RW "); | ||
424 | break; | ||
425 | default: | ||
426 | return TRACE_TYPE_PARTIAL_LINE; | ||
427 | } | ||
428 | |||
429 | if (!ret) | ||
430 | return TRACE_TYPE_PARTIAL_LINE; | ||
431 | |||
432 | sprint_symbol(str, field->ip); | ||
433 | ret = trace_seq_printf(s, "%s\n", str); | ||
434 | if (!ret) | ||
435 | return TRACE_TYPE_PARTIAL_LINE; | ||
436 | |||
437 | return TRACE_TYPE_HANDLED; | ||
438 | } | ||
439 | |||
440 | struct tracer ksym_tracer __read_mostly = | ||
441 | { | ||
442 | .name = "ksym_tracer", | ||
443 | .init = ksym_trace_init, | ||
444 | .reset = ksym_trace_reset, | ||
445 | #ifdef CONFIG_FTRACE_SELFTEST | ||
446 | .selftest = trace_selftest_startup_ksym, | ||
447 | #endif | ||
448 | .print_header = ksym_trace_print_header, | ||
449 | .print_line = ksym_trace_output | ||
450 | }; | ||
451 | |||
452 | __init static int init_ksym_trace(void) | ||
453 | { | ||
454 | struct dentry *d_tracer; | ||
455 | struct dentry *entry; | ||
456 | |||
457 | d_tracer = tracing_init_dentry(); | ||
458 | ksym_filter_entry_count = 0; | ||
459 | |||
460 | entry = debugfs_create_file("ksym_trace_filter", 0644, d_tracer, | ||
461 | NULL, &ksym_tracing_fops); | ||
462 | if (!entry) | ||
463 | pr_warning("Could not create debugfs " | ||
464 | "'ksym_trace_filter' file\n"); | ||
465 | |||
466 | return register_tracer(&ksym_tracer); | ||
467 | } | ||
468 | device_initcall(init_ksym_trace); | ||
469 | |||
470 | |||
471 | #ifdef CONFIG_PROFILE_KSYM_TRACER | ||
472 | static int ksym_tracer_stat_headers(struct seq_file *m) | ||
473 | { | ||
474 | seq_puts(m, " Access Type "); | ||
475 | seq_puts(m, " Symbol Counter\n"); | ||
476 | seq_puts(m, " ----------- "); | ||
477 | seq_puts(m, " ------ -------\n"); | ||
478 | return 0; | ||
479 | } | ||
480 | |||
481 | static int ksym_tracer_stat_show(struct seq_file *m, void *v) | ||
482 | { | ||
483 | struct hlist_node *stat = v; | ||
484 | struct trace_ksym *entry; | ||
485 | int access_type = 0; | ||
486 | char fn_name[KSYM_NAME_LEN]; | ||
487 | |||
488 | entry = hlist_entry(stat, struct trace_ksym, ksym_hlist); | ||
489 | |||
490 | access_type = entry->attr.bp_type; | ||
491 | |||
492 | switch (access_type) { | ||
493 | case HW_BREAKPOINT_R: | ||
494 | seq_puts(m, " R "); | ||
495 | break; | ||
496 | case HW_BREAKPOINT_W: | ||
497 | seq_puts(m, " W "); | ||
498 | break; | ||
499 | case HW_BREAKPOINT_R | HW_BREAKPOINT_W: | ||
500 | seq_puts(m, " RW "); | ||
501 | break; | ||
502 | default: | ||
503 | seq_puts(m, " NA "); | ||
504 | } | ||
505 | |||
506 | if (lookup_symbol_name(entry->attr.bp_addr, fn_name) >= 0) | ||
507 | seq_printf(m, " %-36s", fn_name); | ||
508 | else | ||
509 | seq_printf(m, " %-36s", "<NA>"); | ||
510 | seq_printf(m, " %15lu\n", entry->counter); | ||
511 | |||
512 | return 0; | ||
513 | } | ||
514 | |||
515 | static void *ksym_tracer_stat_start(struct tracer_stat *trace) | ||
516 | { | ||
517 | return ksym_filter_head.first; | ||
518 | } | ||
519 | |||
520 | static void * | ||
521 | ksym_tracer_stat_next(void *v, int idx) | ||
522 | { | ||
523 | struct hlist_node *stat = v; | ||
524 | |||
525 | return stat->next; | ||
526 | } | ||
527 | |||
528 | static struct tracer_stat ksym_tracer_stats = { | ||
529 | .name = "ksym_tracer", | ||
530 | .stat_start = ksym_tracer_stat_start, | ||
531 | .stat_next = ksym_tracer_stat_next, | ||
532 | .stat_headers = ksym_tracer_stat_headers, | ||
533 | .stat_show = ksym_tracer_stat_show | ||
534 | }; | ||
535 | |||
536 | __init static int ksym_tracer_stat_init(void) | ||
537 | { | ||
538 | int ret; | ||
539 | |||
540 | ret = register_stat_tracer(&ksym_tracer_stats); | ||
541 | if (ret) { | ||
542 | printk(KERN_WARNING "Warning: could not register " | ||
543 | "ksym tracer stats\n"); | ||
544 | return 1; | ||
545 | } | ||
546 | |||
547 | return 0; | ||
548 | } | ||
549 | fs_initcall(ksym_tracer_stat_init); | ||
550 | #endif /* CONFIG_PROFILE_KSYM_TRACER */ | ||
diff --git a/kernel/trace/trace_output.c b/kernel/trace/trace_output.c index ed17565826b0..b6c12c6a1bcd 100644 --- a/kernel/trace/trace_output.c +++ b/kernel/trace/trace_output.c | |||
@@ -69,6 +69,9 @@ enum print_line_t trace_print_printk_msg_only(struct trace_iterator *iter) | |||
69 | * @s: trace sequence descriptor | 69 | * @s: trace sequence descriptor |
70 | * @fmt: printf format string | 70 | * @fmt: printf format string |
71 | * | 71 | * |
72 | * It returns 0 if the trace oversizes the buffer's free | ||
73 | * space, 1 otherwise. | ||
74 | * | ||
72 | * The tracer may use either sequence operations or its own | 75 | * The tracer may use either sequence operations or its own |
73 | * copy to user routines. To simplify formating of a trace | 76 | * copy to user routines. To simplify formating of a trace |
74 | * trace_seq_printf is used to store strings into a special | 77 | * trace_seq_printf is used to store strings into a special |
@@ -95,7 +98,7 @@ trace_seq_printf(struct trace_seq *s, const char *fmt, ...) | |||
95 | 98 | ||
96 | s->len += ret; | 99 | s->len += ret; |
97 | 100 | ||
98 | return len; | 101 | return 1; |
99 | } | 102 | } |
100 | EXPORT_SYMBOL_GPL(trace_seq_printf); | 103 | EXPORT_SYMBOL_GPL(trace_seq_printf); |
101 | 104 | ||
diff --git a/kernel/trace/trace_selftest.c b/kernel/trace/trace_selftest.c index d2cdbabb4ead..dc98309e839a 100644 --- a/kernel/trace/trace_selftest.c +++ b/kernel/trace/trace_selftest.c | |||
@@ -17,6 +17,7 @@ static inline int trace_valid_entry(struct trace_entry *entry) | |||
17 | case TRACE_GRAPH_ENT: | 17 | case TRACE_GRAPH_ENT: |
18 | case TRACE_GRAPH_RET: | 18 | case TRACE_GRAPH_RET: |
19 | case TRACE_HW_BRANCHES: | 19 | case TRACE_HW_BRANCHES: |
20 | case TRACE_KSYM: | ||
20 | return 1; | 21 | return 1; |
21 | } | 22 | } |
22 | return 0; | 23 | return 0; |
@@ -808,3 +809,57 @@ trace_selftest_startup_hw_branches(struct tracer *trace, | |||
808 | return ret; | 809 | return ret; |
809 | } | 810 | } |
810 | #endif /* CONFIG_HW_BRANCH_TRACER */ | 811 | #endif /* CONFIG_HW_BRANCH_TRACER */ |
812 | |||
813 | #ifdef CONFIG_KSYM_TRACER | ||
814 | static int ksym_selftest_dummy; | ||
815 | |||
816 | int | ||
817 | trace_selftest_startup_ksym(struct tracer *trace, struct trace_array *tr) | ||
818 | { | ||
819 | unsigned long count; | ||
820 | int ret; | ||
821 | |||
822 | /* start the tracing */ | ||
823 | ret = tracer_init(trace, tr); | ||
824 | if (ret) { | ||
825 | warn_failed_init_tracer(trace, ret); | ||
826 | return ret; | ||
827 | } | ||
828 | |||
829 | ksym_selftest_dummy = 0; | ||
830 | /* Register the read-write tracing request */ | ||
831 | |||
832 | ret = process_new_ksym_entry("ksym_selftest_dummy", | ||
833 | HW_BREAKPOINT_R | HW_BREAKPOINT_W, | ||
834 | (unsigned long)(&ksym_selftest_dummy)); | ||
835 | |||
836 | if (ret < 0) { | ||
837 | printk(KERN_CONT "ksym_trace read-write startup test failed\n"); | ||
838 | goto ret_path; | ||
839 | } | ||
840 | /* Perform a read and a write operation over the dummy variable to | ||
841 | * trigger the tracer | ||
842 | */ | ||
843 | if (ksym_selftest_dummy == 0) | ||
844 | ksym_selftest_dummy++; | ||
845 | |||
846 | /* stop the tracing. */ | ||
847 | tracing_stop(); | ||
848 | /* check the trace buffer */ | ||
849 | ret = trace_test_buffer(tr, &count); | ||
850 | trace->reset(tr); | ||
851 | tracing_start(); | ||
852 | |||
853 | /* read & write operations - one each is performed on the dummy variable | ||
854 | * triggering two entries in the trace buffer | ||
855 | */ | ||
856 | if (!ret && count != 2) { | ||
857 | printk(KERN_CONT "Ksym tracer startup test failed"); | ||
858 | ret = -1; | ||
859 | } | ||
860 | |||
861 | ret_path: | ||
862 | return ret; | ||
863 | } | ||
864 | #endif /* CONFIG_KSYM_TRACER */ | ||
865 | |||
diff --git a/kernel/trace/trace_syscalls.c b/kernel/trace/trace_syscalls.c index 527e17eae575..57501d90096a 100644 --- a/kernel/trace/trace_syscalls.c +++ b/kernel/trace/trace_syscalls.c | |||
@@ -14,6 +14,43 @@ static int sys_refcount_exit; | |||
14 | static DECLARE_BITMAP(enabled_enter_syscalls, NR_syscalls); | 14 | static DECLARE_BITMAP(enabled_enter_syscalls, NR_syscalls); |
15 | static DECLARE_BITMAP(enabled_exit_syscalls, NR_syscalls); | 15 | static DECLARE_BITMAP(enabled_exit_syscalls, NR_syscalls); |
16 | 16 | ||
17 | extern unsigned long __start_syscalls_metadata[]; | ||
18 | extern unsigned long __stop_syscalls_metadata[]; | ||
19 | |||
20 | static struct syscall_metadata **syscalls_metadata; | ||
21 | |||
22 | static struct syscall_metadata *find_syscall_meta(unsigned long syscall) | ||
23 | { | ||
24 | struct syscall_metadata *start; | ||
25 | struct syscall_metadata *stop; | ||
26 | char str[KSYM_SYMBOL_LEN]; | ||
27 | |||
28 | |||
29 | start = (struct syscall_metadata *)__start_syscalls_metadata; | ||
30 | stop = (struct syscall_metadata *)__stop_syscalls_metadata; | ||
31 | kallsyms_lookup(syscall, NULL, NULL, NULL, str); | ||
32 | |||
33 | for ( ; start < stop; start++) { | ||
34 | /* | ||
35 | * Only compare after the "sys" prefix. Archs that use | ||
36 | * syscall wrappers may have syscalls symbols aliases prefixed | ||
37 | * with "SyS" instead of "sys", leading to an unwanted | ||
38 | * mismatch. | ||
39 | */ | ||
40 | if (start->name && !strcmp(start->name + 3, str + 3)) | ||
41 | return start; | ||
42 | } | ||
43 | return NULL; | ||
44 | } | ||
45 | |||
46 | static struct syscall_metadata *syscall_nr_to_meta(int nr) | ||
47 | { | ||
48 | if (!syscalls_metadata || nr >= NR_syscalls || nr < 0) | ||
49 | return NULL; | ||
50 | |||
51 | return syscalls_metadata[nr]; | ||
52 | } | ||
53 | |||
17 | enum print_line_t | 54 | enum print_line_t |
18 | print_syscall_enter(struct trace_iterator *iter, int flags) | 55 | print_syscall_enter(struct trace_iterator *iter, int flags) |
19 | { | 56 | { |
@@ -30,7 +67,7 @@ print_syscall_enter(struct trace_iterator *iter, int flags) | |||
30 | if (!entry) | 67 | if (!entry) |
31 | goto end; | 68 | goto end; |
32 | 69 | ||
33 | if (entry->enter_id != ent->type) { | 70 | if (entry->enter_event->id != ent->type) { |
34 | WARN_ON_ONCE(1); | 71 | WARN_ON_ONCE(1); |
35 | goto end; | 72 | goto end; |
36 | } | 73 | } |
@@ -85,7 +122,7 @@ print_syscall_exit(struct trace_iterator *iter, int flags) | |||
85 | return TRACE_TYPE_HANDLED; | 122 | return TRACE_TYPE_HANDLED; |
86 | } | 123 | } |
87 | 124 | ||
88 | if (entry->exit_id != ent->type) { | 125 | if (entry->exit_event->id != ent->type) { |
89 | WARN_ON_ONCE(1); | 126 | WARN_ON_ONCE(1); |
90 | return TRACE_TYPE_UNHANDLED; | 127 | return TRACE_TYPE_UNHANDLED; |
91 | } | 128 | } |
@@ -103,24 +140,19 @@ extern char *__bad_type_size(void); | |||
103 | #define SYSCALL_FIELD(type, name) \ | 140 | #define SYSCALL_FIELD(type, name) \ |
104 | sizeof(type) != sizeof(trace.name) ? \ | 141 | sizeof(type) != sizeof(trace.name) ? \ |
105 | __bad_type_size() : \ | 142 | __bad_type_size() : \ |
106 | #type, #name, offsetof(typeof(trace), name), sizeof(trace.name) | 143 | #type, #name, offsetof(typeof(trace), name), \ |
144 | sizeof(trace.name), is_signed_type(type) | ||
107 | 145 | ||
108 | int syscall_enter_format(struct ftrace_event_call *call, struct trace_seq *s) | 146 | int syscall_enter_format(struct ftrace_event_call *call, struct trace_seq *s) |
109 | { | 147 | { |
110 | int i; | 148 | int i; |
111 | int nr; | ||
112 | int ret; | 149 | int ret; |
113 | struct syscall_metadata *entry; | 150 | struct syscall_metadata *entry = call->data; |
114 | struct syscall_trace_enter trace; | 151 | struct syscall_trace_enter trace; |
115 | int offset = offsetof(struct syscall_trace_enter, args); | 152 | int offset = offsetof(struct syscall_trace_enter, args); |
116 | 153 | ||
117 | nr = syscall_name_to_nr(call->data); | 154 | ret = trace_seq_printf(s, "\tfield:%s %s;\toffset:%zu;\tsize:%zu;" |
118 | entry = syscall_nr_to_meta(nr); | 155 | "\tsigned:%u;\n", |
119 | |||
120 | if (!entry) | ||
121 | return 0; | ||
122 | |||
123 | ret = trace_seq_printf(s, "\tfield:%s %s;\toffset:%zu;\tsize:%zu;\n", | ||
124 | SYSCALL_FIELD(int, nr)); | 156 | SYSCALL_FIELD(int, nr)); |
125 | if (!ret) | 157 | if (!ret) |
126 | return 0; | 158 | return 0; |
@@ -130,8 +162,10 @@ int syscall_enter_format(struct ftrace_event_call *call, struct trace_seq *s) | |||
130 | entry->args[i]); | 162 | entry->args[i]); |
131 | if (!ret) | 163 | if (!ret) |
132 | return 0; | 164 | return 0; |
133 | ret = trace_seq_printf(s, "\toffset:%d;\tsize:%zu;\n", offset, | 165 | ret = trace_seq_printf(s, "\toffset:%d;\tsize:%zu;" |
134 | sizeof(unsigned long)); | 166 | "\tsigned:%u;\n", offset, |
167 | sizeof(unsigned long), | ||
168 | is_signed_type(unsigned long)); | ||
135 | if (!ret) | 169 | if (!ret) |
136 | return 0; | 170 | return 0; |
137 | offset += sizeof(unsigned long); | 171 | offset += sizeof(unsigned long); |
@@ -163,8 +197,10 @@ int syscall_exit_format(struct ftrace_event_call *call, struct trace_seq *s) | |||
163 | struct syscall_trace_exit trace; | 197 | struct syscall_trace_exit trace; |
164 | 198 | ||
165 | ret = trace_seq_printf(s, | 199 | ret = trace_seq_printf(s, |
166 | "\tfield:%s %s;\toffset:%zu;\tsize:%zu;\n" | 200 | "\tfield:%s %s;\toffset:%zu;\tsize:%zu;" |
167 | "\tfield:%s %s;\toffset:%zu;\tsize:%zu;\n", | 201 | "\tsigned:%u;\n" |
202 | "\tfield:%s %s;\toffset:%zu;\tsize:%zu;" | ||
203 | "\tsigned:%u;\n", | ||
168 | SYSCALL_FIELD(int, nr), | 204 | SYSCALL_FIELD(int, nr), |
169 | SYSCALL_FIELD(long, ret)); | 205 | SYSCALL_FIELD(long, ret)); |
170 | if (!ret) | 206 | if (!ret) |
@@ -176,22 +212,19 @@ int syscall_exit_format(struct ftrace_event_call *call, struct trace_seq *s) | |||
176 | int syscall_enter_define_fields(struct ftrace_event_call *call) | 212 | int syscall_enter_define_fields(struct ftrace_event_call *call) |
177 | { | 213 | { |
178 | struct syscall_trace_enter trace; | 214 | struct syscall_trace_enter trace; |
179 | struct syscall_metadata *meta; | 215 | struct syscall_metadata *meta = call->data; |
180 | int ret; | 216 | int ret; |
181 | int nr; | ||
182 | int i; | 217 | int i; |
183 | int offset = offsetof(typeof(trace), args); | 218 | int offset = offsetof(typeof(trace), args); |
184 | 219 | ||
185 | nr = syscall_name_to_nr(call->data); | ||
186 | meta = syscall_nr_to_meta(nr); | ||
187 | |||
188 | if (!meta) | ||
189 | return 0; | ||
190 | |||
191 | ret = trace_define_common_fields(call); | 220 | ret = trace_define_common_fields(call); |
192 | if (ret) | 221 | if (ret) |
193 | return ret; | 222 | return ret; |
194 | 223 | ||
224 | ret = trace_define_field(call, SYSCALL_FIELD(int, nr), FILTER_OTHER); | ||
225 | if (ret) | ||
226 | return ret; | ||
227 | |||
195 | for (i = 0; i < meta->nb_args; i++) { | 228 | for (i = 0; i < meta->nb_args; i++) { |
196 | ret = trace_define_field(call, meta->types[i], | 229 | ret = trace_define_field(call, meta->types[i], |
197 | meta->args[i], offset, | 230 | meta->args[i], offset, |
@@ -212,7 +245,11 @@ int syscall_exit_define_fields(struct ftrace_event_call *call) | |||
212 | if (ret) | 245 | if (ret) |
213 | return ret; | 246 | return ret; |
214 | 247 | ||
215 | ret = trace_define_field(call, SYSCALL_FIELD(long, ret), 0, | 248 | ret = trace_define_field(call, SYSCALL_FIELD(int, nr), FILTER_OTHER); |
249 | if (ret) | ||
250 | return ret; | ||
251 | |||
252 | ret = trace_define_field(call, SYSCALL_FIELD(long, ret), | ||
216 | FILTER_OTHER); | 253 | FILTER_OTHER); |
217 | 254 | ||
218 | return ret; | 255 | return ret; |
@@ -239,8 +276,8 @@ void ftrace_syscall_enter(struct pt_regs *regs, long id) | |||
239 | 276 | ||
240 | size = sizeof(*entry) + sizeof(unsigned long) * sys_data->nb_args; | 277 | size = sizeof(*entry) + sizeof(unsigned long) * sys_data->nb_args; |
241 | 278 | ||
242 | event = trace_current_buffer_lock_reserve(&buffer, sys_data->enter_id, | 279 | event = trace_current_buffer_lock_reserve(&buffer, |
243 | size, 0, 0); | 280 | sys_data->enter_event->id, size, 0, 0); |
244 | if (!event) | 281 | if (!event) |
245 | return; | 282 | return; |
246 | 283 | ||
@@ -271,8 +308,8 @@ void ftrace_syscall_exit(struct pt_regs *regs, long ret) | |||
271 | if (!sys_data) | 308 | if (!sys_data) |
272 | return; | 309 | return; |
273 | 310 | ||
274 | event = trace_current_buffer_lock_reserve(&buffer, sys_data->exit_id, | 311 | event = trace_current_buffer_lock_reserve(&buffer, |
275 | sizeof(*entry), 0, 0); | 312 | sys_data->exit_event->id, sizeof(*entry), 0, 0); |
276 | if (!event) | 313 | if (!event) |
277 | return; | 314 | return; |
278 | 315 | ||
@@ -285,14 +322,12 @@ void ftrace_syscall_exit(struct pt_regs *regs, long ret) | |||
285 | trace_current_buffer_unlock_commit(buffer, event, 0, 0); | 322 | trace_current_buffer_unlock_commit(buffer, event, 0, 0); |
286 | } | 323 | } |
287 | 324 | ||
288 | int reg_event_syscall_enter(void *ptr) | 325 | int reg_event_syscall_enter(struct ftrace_event_call *call) |
289 | { | 326 | { |
290 | int ret = 0; | 327 | int ret = 0; |
291 | int num; | 328 | int num; |
292 | char *name; | ||
293 | 329 | ||
294 | name = (char *)ptr; | 330 | num = ((struct syscall_metadata *)call->data)->syscall_nr; |
295 | num = syscall_name_to_nr(name); | ||
296 | if (num < 0 || num >= NR_syscalls) | 331 | if (num < 0 || num >= NR_syscalls) |
297 | return -ENOSYS; | 332 | return -ENOSYS; |
298 | mutex_lock(&syscall_trace_lock); | 333 | mutex_lock(&syscall_trace_lock); |
@@ -309,13 +344,11 @@ int reg_event_syscall_enter(void *ptr) | |||
309 | return ret; | 344 | return ret; |
310 | } | 345 | } |
311 | 346 | ||
312 | void unreg_event_syscall_enter(void *ptr) | 347 | void unreg_event_syscall_enter(struct ftrace_event_call *call) |
313 | { | 348 | { |
314 | int num; | 349 | int num; |
315 | char *name; | ||
316 | 350 | ||
317 | name = (char *)ptr; | 351 | num = ((struct syscall_metadata *)call->data)->syscall_nr; |
318 | num = syscall_name_to_nr(name); | ||
319 | if (num < 0 || num >= NR_syscalls) | 352 | if (num < 0 || num >= NR_syscalls) |
320 | return; | 353 | return; |
321 | mutex_lock(&syscall_trace_lock); | 354 | mutex_lock(&syscall_trace_lock); |
@@ -326,14 +359,12 @@ void unreg_event_syscall_enter(void *ptr) | |||
326 | mutex_unlock(&syscall_trace_lock); | 359 | mutex_unlock(&syscall_trace_lock); |
327 | } | 360 | } |
328 | 361 | ||
329 | int reg_event_syscall_exit(void *ptr) | 362 | int reg_event_syscall_exit(struct ftrace_event_call *call) |
330 | { | 363 | { |
331 | int ret = 0; | 364 | int ret = 0; |
332 | int num; | 365 | int num; |
333 | char *name; | ||
334 | 366 | ||
335 | name = (char *)ptr; | 367 | num = ((struct syscall_metadata *)call->data)->syscall_nr; |
336 | num = syscall_name_to_nr(name); | ||
337 | if (num < 0 || num >= NR_syscalls) | 368 | if (num < 0 || num >= NR_syscalls) |
338 | return -ENOSYS; | 369 | return -ENOSYS; |
339 | mutex_lock(&syscall_trace_lock); | 370 | mutex_lock(&syscall_trace_lock); |
@@ -350,13 +381,11 @@ int reg_event_syscall_exit(void *ptr) | |||
350 | return ret; | 381 | return ret; |
351 | } | 382 | } |
352 | 383 | ||
353 | void unreg_event_syscall_exit(void *ptr) | 384 | void unreg_event_syscall_exit(struct ftrace_event_call *call) |
354 | { | 385 | { |
355 | int num; | 386 | int num; |
356 | char *name; | ||
357 | 387 | ||
358 | name = (char *)ptr; | 388 | num = ((struct syscall_metadata *)call->data)->syscall_nr; |
359 | num = syscall_name_to_nr(name); | ||
360 | if (num < 0 || num >= NR_syscalls) | 389 | if (num < 0 || num >= NR_syscalls) |
361 | return; | 390 | return; |
362 | mutex_lock(&syscall_trace_lock); | 391 | mutex_lock(&syscall_trace_lock); |
@@ -367,13 +396,44 @@ void unreg_event_syscall_exit(void *ptr) | |||
367 | mutex_unlock(&syscall_trace_lock); | 396 | mutex_unlock(&syscall_trace_lock); |
368 | } | 397 | } |
369 | 398 | ||
370 | struct trace_event event_syscall_enter = { | 399 | int init_syscall_trace(struct ftrace_event_call *call) |
371 | .trace = print_syscall_enter, | 400 | { |
372 | }; | 401 | int id; |
402 | |||
403 | id = register_ftrace_event(call->event); | ||
404 | if (!id) | ||
405 | return -ENODEV; | ||
406 | call->id = id; | ||
407 | INIT_LIST_HEAD(&call->fields); | ||
408 | return 0; | ||
409 | } | ||
410 | |||
411 | int __init init_ftrace_syscalls(void) | ||
412 | { | ||
413 | struct syscall_metadata *meta; | ||
414 | unsigned long addr; | ||
415 | int i; | ||
416 | |||
417 | syscalls_metadata = kzalloc(sizeof(*syscalls_metadata) * | ||
418 | NR_syscalls, GFP_KERNEL); | ||
419 | if (!syscalls_metadata) { | ||
420 | WARN_ON(1); | ||
421 | return -ENOMEM; | ||
422 | } | ||
423 | |||
424 | for (i = 0; i < NR_syscalls; i++) { | ||
425 | addr = arch_syscall_addr(i); | ||
426 | meta = find_syscall_meta(addr); | ||
427 | if (!meta) | ||
428 | continue; | ||
429 | |||
430 | meta->syscall_nr = i; | ||
431 | syscalls_metadata[i] = meta; | ||
432 | } | ||
373 | 433 | ||
374 | struct trace_event event_syscall_exit = { | 434 | return 0; |
375 | .trace = print_syscall_exit, | 435 | } |
376 | }; | 436 | core_initcall(init_ftrace_syscalls); |
377 | 437 | ||
378 | #ifdef CONFIG_EVENT_PROFILE | 438 | #ifdef CONFIG_EVENT_PROFILE |
379 | 439 | ||
@@ -387,8 +447,10 @@ static void prof_syscall_enter(struct pt_regs *regs, long id) | |||
387 | struct syscall_metadata *sys_data; | 447 | struct syscall_metadata *sys_data; |
388 | struct syscall_trace_enter *rec; | 448 | struct syscall_trace_enter *rec; |
389 | unsigned long flags; | 449 | unsigned long flags; |
450 | char *trace_buf; | ||
390 | char *raw_data; | 451 | char *raw_data; |
391 | int syscall_nr; | 452 | int syscall_nr; |
453 | int rctx; | ||
392 | int size; | 454 | int size; |
393 | int cpu; | 455 | int cpu; |
394 | 456 | ||
@@ -412,41 +474,42 @@ static void prof_syscall_enter(struct pt_regs *regs, long id) | |||
412 | /* Protect the per cpu buffer, begin the rcu read side */ | 474 | /* Protect the per cpu buffer, begin the rcu read side */ |
413 | local_irq_save(flags); | 475 | local_irq_save(flags); |
414 | 476 | ||
477 | rctx = perf_swevent_get_recursion_context(); | ||
478 | if (rctx < 0) | ||
479 | goto end_recursion; | ||
480 | |||
415 | cpu = smp_processor_id(); | 481 | cpu = smp_processor_id(); |
416 | 482 | ||
417 | if (in_nmi()) | 483 | trace_buf = rcu_dereference(perf_trace_buf); |
418 | raw_data = rcu_dereference(trace_profile_buf_nmi); | ||
419 | else | ||
420 | raw_data = rcu_dereference(trace_profile_buf); | ||
421 | 484 | ||
422 | if (!raw_data) | 485 | if (!trace_buf) |
423 | goto end; | 486 | goto end; |
424 | 487 | ||
425 | raw_data = per_cpu_ptr(raw_data, cpu); | 488 | raw_data = per_cpu_ptr(trace_buf, cpu); |
426 | 489 | ||
427 | /* zero the dead bytes from align to not leak stack to user */ | 490 | /* zero the dead bytes from align to not leak stack to user */ |
428 | *(u64 *)(&raw_data[size - sizeof(u64)]) = 0ULL; | 491 | *(u64 *)(&raw_data[size - sizeof(u64)]) = 0ULL; |
429 | 492 | ||
430 | rec = (struct syscall_trace_enter *) raw_data; | 493 | rec = (struct syscall_trace_enter *) raw_data; |
431 | tracing_generic_entry_update(&rec->ent, 0, 0); | 494 | tracing_generic_entry_update(&rec->ent, 0, 0); |
432 | rec->ent.type = sys_data->enter_id; | 495 | rec->ent.type = sys_data->enter_event->id; |
433 | rec->nr = syscall_nr; | 496 | rec->nr = syscall_nr; |
434 | syscall_get_arguments(current, regs, 0, sys_data->nb_args, | 497 | syscall_get_arguments(current, regs, 0, sys_data->nb_args, |
435 | (unsigned long *)&rec->args); | 498 | (unsigned long *)&rec->args); |
436 | perf_tp_event(sys_data->enter_id, 0, 1, rec, size); | 499 | perf_tp_event(sys_data->enter_event->id, 0, 1, rec, size); |
437 | 500 | ||
438 | end: | 501 | end: |
502 | perf_swevent_put_recursion_context(rctx); | ||
503 | end_recursion: | ||
439 | local_irq_restore(flags); | 504 | local_irq_restore(flags); |
440 | } | 505 | } |
441 | 506 | ||
442 | int reg_prof_syscall_enter(char *name) | 507 | int prof_sysenter_enable(struct ftrace_event_call *call) |
443 | { | 508 | { |
444 | int ret = 0; | 509 | int ret = 0; |
445 | int num; | 510 | int num; |
446 | 511 | ||
447 | num = syscall_name_to_nr(name); | 512 | num = ((struct syscall_metadata *)call->data)->syscall_nr; |
448 | if (num < 0 || num >= NR_syscalls) | ||
449 | return -ENOSYS; | ||
450 | 513 | ||
451 | mutex_lock(&syscall_trace_lock); | 514 | mutex_lock(&syscall_trace_lock); |
452 | if (!sys_prof_refcount_enter) | 515 | if (!sys_prof_refcount_enter) |
@@ -462,13 +525,11 @@ int reg_prof_syscall_enter(char *name) | |||
462 | return ret; | 525 | return ret; |
463 | } | 526 | } |
464 | 527 | ||
465 | void unreg_prof_syscall_enter(char *name) | 528 | void prof_sysenter_disable(struct ftrace_event_call *call) |
466 | { | 529 | { |
467 | int num; | 530 | int num; |
468 | 531 | ||
469 | num = syscall_name_to_nr(name); | 532 | num = ((struct syscall_metadata *)call->data)->syscall_nr; |
470 | if (num < 0 || num >= NR_syscalls) | ||
471 | return; | ||
472 | 533 | ||
473 | mutex_lock(&syscall_trace_lock); | 534 | mutex_lock(&syscall_trace_lock); |
474 | sys_prof_refcount_enter--; | 535 | sys_prof_refcount_enter--; |
@@ -484,7 +545,9 @@ static void prof_syscall_exit(struct pt_regs *regs, long ret) | |||
484 | struct syscall_trace_exit *rec; | 545 | struct syscall_trace_exit *rec; |
485 | unsigned long flags; | 546 | unsigned long flags; |
486 | int syscall_nr; | 547 | int syscall_nr; |
548 | char *trace_buf; | ||
487 | char *raw_data; | 549 | char *raw_data; |
550 | int rctx; | ||
488 | int size; | 551 | int size; |
489 | int cpu; | 552 | int cpu; |
490 | 553 | ||
@@ -510,17 +573,19 @@ static void prof_syscall_exit(struct pt_regs *regs, long ret) | |||
510 | 573 | ||
511 | /* Protect the per cpu buffer, begin the rcu read side */ | 574 | /* Protect the per cpu buffer, begin the rcu read side */ |
512 | local_irq_save(flags); | 575 | local_irq_save(flags); |
576 | |||
577 | rctx = perf_swevent_get_recursion_context(); | ||
578 | if (rctx < 0) | ||
579 | goto end_recursion; | ||
580 | |||
513 | cpu = smp_processor_id(); | 581 | cpu = smp_processor_id(); |
514 | 582 | ||
515 | if (in_nmi()) | 583 | trace_buf = rcu_dereference(perf_trace_buf); |
516 | raw_data = rcu_dereference(trace_profile_buf_nmi); | ||
517 | else | ||
518 | raw_data = rcu_dereference(trace_profile_buf); | ||
519 | 584 | ||
520 | if (!raw_data) | 585 | if (!trace_buf) |
521 | goto end; | 586 | goto end; |
522 | 587 | ||
523 | raw_data = per_cpu_ptr(raw_data, cpu); | 588 | raw_data = per_cpu_ptr(trace_buf, cpu); |
524 | 589 | ||
525 | /* zero the dead bytes from align to not leak stack to user */ | 590 | /* zero the dead bytes from align to not leak stack to user */ |
526 | *(u64 *)(&raw_data[size - sizeof(u64)]) = 0ULL; | 591 | *(u64 *)(&raw_data[size - sizeof(u64)]) = 0ULL; |
@@ -528,24 +593,24 @@ static void prof_syscall_exit(struct pt_regs *regs, long ret) | |||
528 | rec = (struct syscall_trace_exit *)raw_data; | 593 | rec = (struct syscall_trace_exit *)raw_data; |
529 | 594 | ||
530 | tracing_generic_entry_update(&rec->ent, 0, 0); | 595 | tracing_generic_entry_update(&rec->ent, 0, 0); |
531 | rec->ent.type = sys_data->exit_id; | 596 | rec->ent.type = sys_data->exit_event->id; |
532 | rec->nr = syscall_nr; | 597 | rec->nr = syscall_nr; |
533 | rec->ret = syscall_get_return_value(current, regs); | 598 | rec->ret = syscall_get_return_value(current, regs); |
534 | 599 | ||
535 | perf_tp_event(sys_data->exit_id, 0, 1, rec, size); | 600 | perf_tp_event(sys_data->exit_event->id, 0, 1, rec, size); |
536 | 601 | ||
537 | end: | 602 | end: |
603 | perf_swevent_put_recursion_context(rctx); | ||
604 | end_recursion: | ||
538 | local_irq_restore(flags); | 605 | local_irq_restore(flags); |
539 | } | 606 | } |
540 | 607 | ||
541 | int reg_prof_syscall_exit(char *name) | 608 | int prof_sysexit_enable(struct ftrace_event_call *call) |
542 | { | 609 | { |
543 | int ret = 0; | 610 | int ret = 0; |
544 | int num; | 611 | int num; |
545 | 612 | ||
546 | num = syscall_name_to_nr(name); | 613 | num = ((struct syscall_metadata *)call->data)->syscall_nr; |
547 | if (num < 0 || num >= NR_syscalls) | ||
548 | return -ENOSYS; | ||
549 | 614 | ||
550 | mutex_lock(&syscall_trace_lock); | 615 | mutex_lock(&syscall_trace_lock); |
551 | if (!sys_prof_refcount_exit) | 616 | if (!sys_prof_refcount_exit) |
@@ -561,13 +626,11 @@ int reg_prof_syscall_exit(char *name) | |||
561 | return ret; | 626 | return ret; |
562 | } | 627 | } |
563 | 628 | ||
564 | void unreg_prof_syscall_exit(char *name) | 629 | void prof_sysexit_disable(struct ftrace_event_call *call) |
565 | { | 630 | { |
566 | int num; | 631 | int num; |
567 | 632 | ||
568 | num = syscall_name_to_nr(name); | 633 | num = ((struct syscall_metadata *)call->data)->syscall_nr; |
569 | if (num < 0 || num >= NR_syscalls) | ||
570 | return; | ||
571 | 634 | ||
572 | mutex_lock(&syscall_trace_lock); | 635 | mutex_lock(&syscall_trace_lock); |
573 | sys_prof_refcount_exit--; | 636 | sys_prof_refcount_exit--; |
diff --git a/kernel/user-return-notifier.c b/kernel/user-return-notifier.c new file mode 100644 index 000000000000..eb27fd3430a2 --- /dev/null +++ b/kernel/user-return-notifier.c | |||
@@ -0,0 +1,44 @@ | |||
1 | |||
2 | #include <linux/user-return-notifier.h> | ||
3 | #include <linux/percpu.h> | ||
4 | #include <linux/sched.h> | ||
5 | #include <linux/module.h> | ||
6 | |||
7 | static DEFINE_PER_CPU(struct hlist_head, return_notifier_list); | ||
8 | |||
9 | /* | ||
10 | * Request a notification when the current cpu returns to userspace. Must be | ||
11 | * called in atomic context. The notifier will also be called in atomic | ||
12 | * context. | ||
13 | */ | ||
14 | void user_return_notifier_register(struct user_return_notifier *urn) | ||
15 | { | ||
16 | set_tsk_thread_flag(current, TIF_USER_RETURN_NOTIFY); | ||
17 | hlist_add_head(&urn->link, &__get_cpu_var(return_notifier_list)); | ||
18 | } | ||
19 | EXPORT_SYMBOL_GPL(user_return_notifier_register); | ||
20 | |||
21 | /* | ||
22 | * Removes a registered user return notifier. Must be called from atomic | ||
23 | * context, and from the same cpu registration occured in. | ||
24 | */ | ||
25 | void user_return_notifier_unregister(struct user_return_notifier *urn) | ||
26 | { | ||
27 | hlist_del(&urn->link); | ||
28 | if (hlist_empty(&__get_cpu_var(return_notifier_list))) | ||
29 | clear_tsk_thread_flag(current, TIF_USER_RETURN_NOTIFY); | ||
30 | } | ||
31 | EXPORT_SYMBOL_GPL(user_return_notifier_unregister); | ||
32 | |||
33 | /* Calls registered user return notifiers */ | ||
34 | void fire_user_return_notifiers(void) | ||
35 | { | ||
36 | struct user_return_notifier *urn; | ||
37 | struct hlist_node *tmp1, *tmp2; | ||
38 | struct hlist_head *head; | ||
39 | |||
40 | head = &get_cpu_var(return_notifier_list); | ||
41 | hlist_for_each_entry_safe(urn, tmp1, tmp2, head, link) | ||
42 | urn->on_user_return(urn); | ||
43 | put_cpu_var(return_notifier_list); | ||
44 | } | ||
diff --git a/kernel/user.c b/kernel/user.c index 2c000e7132ac..46d0165ca70c 100644 --- a/kernel/user.c +++ b/kernel/user.c | |||
@@ -330,9 +330,9 @@ done: | |||
330 | */ | 330 | */ |
331 | static void free_user(struct user_struct *up, unsigned long flags) | 331 | static void free_user(struct user_struct *up, unsigned long flags) |
332 | { | 332 | { |
333 | spin_unlock_irqrestore(&uidhash_lock, flags); | ||
334 | INIT_DELAYED_WORK(&up->work, cleanup_user_struct); | 333 | INIT_DELAYED_WORK(&up->work, cleanup_user_struct); |
335 | schedule_delayed_work(&up->work, msecs_to_jiffies(1000)); | 334 | schedule_delayed_work(&up->work, msecs_to_jiffies(1000)); |
335 | spin_unlock_irqrestore(&uidhash_lock, flags); | ||
336 | } | 336 | } |
337 | 337 | ||
338 | #else /* CONFIG_USER_SCHED && CONFIG_SYSFS */ | 338 | #else /* CONFIG_USER_SCHED && CONFIG_SYSFS */ |
diff --git a/kernel/utsname_sysctl.c b/kernel/utsname_sysctl.c index 69eae358a726..a2cd77e70d4d 100644 --- a/kernel/utsname_sysctl.c +++ b/kernel/utsname_sysctl.c | |||
@@ -57,78 +57,47 @@ static int proc_do_uts_string(ctl_table *table, int write, | |||
57 | #define proc_do_uts_string NULL | 57 | #define proc_do_uts_string NULL |
58 | #endif | 58 | #endif |
59 | 59 | ||
60 | |||
61 | #ifdef CONFIG_SYSCTL_SYSCALL | ||
62 | /* The generic string strategy routine: */ | ||
63 | static int sysctl_uts_string(ctl_table *table, | ||
64 | void __user *oldval, size_t __user *oldlenp, | ||
65 | void __user *newval, size_t newlen) | ||
66 | { | ||
67 | struct ctl_table uts_table; | ||
68 | int r, write; | ||
69 | write = newval && newlen; | ||
70 | memcpy(&uts_table, table, sizeof(uts_table)); | ||
71 | uts_table.data = get_uts(table, write); | ||
72 | r = sysctl_string(&uts_table, oldval, oldlenp, newval, newlen); | ||
73 | put_uts(table, write, uts_table.data); | ||
74 | return r; | ||
75 | } | ||
76 | #else | ||
77 | #define sysctl_uts_string NULL | ||
78 | #endif | ||
79 | |||
80 | static struct ctl_table uts_kern_table[] = { | 60 | static struct ctl_table uts_kern_table[] = { |
81 | { | 61 | { |
82 | .ctl_name = KERN_OSTYPE, | ||
83 | .procname = "ostype", | 62 | .procname = "ostype", |
84 | .data = init_uts_ns.name.sysname, | 63 | .data = init_uts_ns.name.sysname, |
85 | .maxlen = sizeof(init_uts_ns.name.sysname), | 64 | .maxlen = sizeof(init_uts_ns.name.sysname), |
86 | .mode = 0444, | 65 | .mode = 0444, |
87 | .proc_handler = proc_do_uts_string, | 66 | .proc_handler = proc_do_uts_string, |
88 | .strategy = sysctl_uts_string, | ||
89 | }, | 67 | }, |
90 | { | 68 | { |
91 | .ctl_name = KERN_OSRELEASE, | ||
92 | .procname = "osrelease", | 69 | .procname = "osrelease", |
93 | .data = init_uts_ns.name.release, | 70 | .data = init_uts_ns.name.release, |
94 | .maxlen = sizeof(init_uts_ns.name.release), | 71 | .maxlen = sizeof(init_uts_ns.name.release), |
95 | .mode = 0444, | 72 | .mode = 0444, |
96 | .proc_handler = proc_do_uts_string, | 73 | .proc_handler = proc_do_uts_string, |
97 | .strategy = sysctl_uts_string, | ||
98 | }, | 74 | }, |
99 | { | 75 | { |
100 | .ctl_name = KERN_VERSION, | ||
101 | .procname = "version", | 76 | .procname = "version", |
102 | .data = init_uts_ns.name.version, | 77 | .data = init_uts_ns.name.version, |
103 | .maxlen = sizeof(init_uts_ns.name.version), | 78 | .maxlen = sizeof(init_uts_ns.name.version), |
104 | .mode = 0444, | 79 | .mode = 0444, |
105 | .proc_handler = proc_do_uts_string, | 80 | .proc_handler = proc_do_uts_string, |
106 | .strategy = sysctl_uts_string, | ||
107 | }, | 81 | }, |
108 | { | 82 | { |
109 | .ctl_name = KERN_NODENAME, | ||
110 | .procname = "hostname", | 83 | .procname = "hostname", |
111 | .data = init_uts_ns.name.nodename, | 84 | .data = init_uts_ns.name.nodename, |
112 | .maxlen = sizeof(init_uts_ns.name.nodename), | 85 | .maxlen = sizeof(init_uts_ns.name.nodename), |
113 | .mode = 0644, | 86 | .mode = 0644, |
114 | .proc_handler = proc_do_uts_string, | 87 | .proc_handler = proc_do_uts_string, |
115 | .strategy = sysctl_uts_string, | ||
116 | }, | 88 | }, |
117 | { | 89 | { |
118 | .ctl_name = KERN_DOMAINNAME, | ||
119 | .procname = "domainname", | 90 | .procname = "domainname", |
120 | .data = init_uts_ns.name.domainname, | 91 | .data = init_uts_ns.name.domainname, |
121 | .maxlen = sizeof(init_uts_ns.name.domainname), | 92 | .maxlen = sizeof(init_uts_ns.name.domainname), |
122 | .mode = 0644, | 93 | .mode = 0644, |
123 | .proc_handler = proc_do_uts_string, | 94 | .proc_handler = proc_do_uts_string, |
124 | .strategy = sysctl_uts_string, | ||
125 | }, | 95 | }, |
126 | {} | 96 | {} |
127 | }; | 97 | }; |
128 | 98 | ||
129 | static struct ctl_table uts_root_table[] = { | 99 | static struct ctl_table uts_root_table[] = { |
130 | { | 100 | { |
131 | .ctl_name = CTL_KERN, | ||
132 | .procname = "kernel", | 101 | .procname = "kernel", |
133 | .mode = 0555, | 102 | .mode = 0555, |
134 | .child = uts_kern_table, | 103 | .child = uts_kern_table, |
diff --git a/kernel/workqueue.c b/kernel/workqueue.c index 47cdd7e76f2b..67e526b6ae81 100644 --- a/kernel/workqueue.c +++ b/kernel/workqueue.c | |||
@@ -685,6 +685,7 @@ EXPORT_SYMBOL(schedule_delayed_work_on); | |||
685 | int schedule_on_each_cpu(work_func_t func) | 685 | int schedule_on_each_cpu(work_func_t func) |
686 | { | 686 | { |
687 | int cpu; | 687 | int cpu; |
688 | int orig = -1; | ||
688 | struct work_struct *works; | 689 | struct work_struct *works; |
689 | 690 | ||
690 | works = alloc_percpu(struct work_struct); | 691 | works = alloc_percpu(struct work_struct); |
@@ -692,14 +693,28 @@ int schedule_on_each_cpu(work_func_t func) | |||
692 | return -ENOMEM; | 693 | return -ENOMEM; |
693 | 694 | ||
694 | get_online_cpus(); | 695 | get_online_cpus(); |
696 | |||
697 | /* | ||
698 | * When running in keventd don't schedule a work item on | ||
699 | * itself. Can just call directly because the work queue is | ||
700 | * already bound. This also is faster. | ||
701 | */ | ||
702 | if (current_is_keventd()) | ||
703 | orig = raw_smp_processor_id(); | ||
704 | |||
695 | for_each_online_cpu(cpu) { | 705 | for_each_online_cpu(cpu) { |
696 | struct work_struct *work = per_cpu_ptr(works, cpu); | 706 | struct work_struct *work = per_cpu_ptr(works, cpu); |
697 | 707 | ||
698 | INIT_WORK(work, func); | 708 | INIT_WORK(work, func); |
699 | schedule_work_on(cpu, work); | 709 | if (cpu != orig) |
710 | schedule_work_on(cpu, work); | ||
700 | } | 711 | } |
712 | if (orig >= 0) | ||
713 | func(per_cpu_ptr(works, orig)); | ||
714 | |||
701 | for_each_online_cpu(cpu) | 715 | for_each_online_cpu(cpu) |
702 | flush_work(per_cpu_ptr(works, cpu)); | 716 | flush_work(per_cpu_ptr(works, cpu)); |
717 | |||
703 | put_online_cpus(); | 718 | put_online_cpus(); |
704 | free_percpu(works); | 719 | free_percpu(works); |
705 | return 0; | 720 | return 0; |