diff options
83 files changed, 534 insertions, 148 deletions
diff --git a/Documentation/DocBook/kernel-locking.tmpl b/Documentation/DocBook/kernel-locking.tmpl index 09e884e5b9f5..19f2a5a5a5b4 100644 --- a/Documentation/DocBook/kernel-locking.tmpl +++ b/Documentation/DocBook/kernel-locking.tmpl | |||
@@ -1958,7 +1958,7 @@ machines due to caching. | |||
1958 | <chapter id="apiref-mutex"> | 1958 | <chapter id="apiref-mutex"> |
1959 | <title>Mutex API reference</title> | 1959 | <title>Mutex API reference</title> |
1960 | !Iinclude/linux/mutex.h | 1960 | !Iinclude/linux/mutex.h |
1961 | !Ekernel/mutex.c | 1961 | !Ekernel/locking/mutex.c |
1962 | </chapter> | 1962 | </chapter> |
1963 | 1963 | ||
1964 | <chapter id="apiref-futex"> | 1964 | <chapter id="apiref-futex"> |
diff --git a/Documentation/lockstat.txt b/Documentation/lockstat.txt index dd2f7b26ca30..72d010689751 100644 --- a/Documentation/lockstat.txt +++ b/Documentation/lockstat.txt | |||
@@ -46,16 +46,14 @@ With these hooks we provide the following statistics: | |||
46 | contentions - number of lock acquisitions that had to wait | 46 | contentions - number of lock acquisitions that had to wait |
47 | wait time min - shortest (non-0) time we ever had to wait for a lock | 47 | wait time min - shortest (non-0) time we ever had to wait for a lock |
48 | max - longest time we ever had to wait for a lock | 48 | max - longest time we ever had to wait for a lock |
49 | total - total time we spend waiting on this lock | 49 | total - total time we spend waiting on this lock |
50 | avg - average time spent waiting on this lock | ||
50 | acq-bounces - number of lock acquisitions that involved x-cpu data | 51 | acq-bounces - number of lock acquisitions that involved x-cpu data |
51 | acquisitions - number of times we took the lock | 52 | acquisitions - number of times we took the lock |
52 | hold time min - shortest (non-0) time we ever held the lock | 53 | hold time min - shortest (non-0) time we ever held the lock |
53 | max - longest time we ever held the lock | 54 | max - longest time we ever held the lock |
54 | total - total time this lock was held | 55 | total - total time this lock was held |
55 | 56 | avg - average time this lock was held | |
56 | From these number various other statistics can be derived, such as: | ||
57 | |||
58 | hold time average = hold time total / acquisitions | ||
59 | 57 | ||
60 | These numbers are gathered per lock class, per read/write state (when | 58 | These numbers are gathered per lock class, per read/write state (when |
61 | applicable). | 59 | applicable). |
@@ -84,37 +82,38 @@ Look at the current lock statistics: | |||
84 | 82 | ||
85 | # less /proc/lock_stat | 83 | # less /proc/lock_stat |
86 | 84 | ||
87 | 01 lock_stat version 0.3 | 85 | 01 lock_stat version 0.4 |
88 | 02 ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | 86 | 02----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
89 | 03 class name con-bounces contentions waittime-min waittime-max waittime-total acq-bounces acquisitions holdtime-min holdtime-max holdtime-total | 87 | 03 class name con-bounces contentions waittime-min waittime-max waittime-total waittime-avg acq-bounces acquisitions holdtime-min holdtime-max holdtime-total holdtime-avg |
90 | 04 ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | 88 | 04----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
91 | 05 | 89 | 05 |
92 | 06 &mm->mmap_sem-W: 233 538 18446744073708 22924.27 607243.51 1342 45806 1.71 8595.89 1180582.34 | 90 | 06 &mm->mmap_sem-W: 46 84 0.26 939.10 16371.53 194.90 47291 2922365 0.16 2220301.69 17464026916.32 5975.99 |
93 | 07 &mm->mmap_sem-R: 205 587 18446744073708 28403.36 731975.00 1940 412426 0.58 187825.45 6307502.88 | 91 | 07 &mm->mmap_sem-R: 37 100 1.31 299502.61 325629.52 3256.30 212344 34316685 0.10 7744.91 95016910.20 2.77 |
94 | 08 --------------- | 92 | 08 --------------- |
95 | 09 &mm->mmap_sem 487 [<ffffffff8053491f>] do_page_fault+0x466/0x928 | 93 | 09 &mm->mmap_sem 1 [<ffffffff811502a7>] khugepaged_scan_mm_slot+0x57/0x280 |
96 | 10 &mm->mmap_sem 179 [<ffffffff802a6200>] sys_mprotect+0xcd/0x21d | 94 | 19 &mm->mmap_sem 96 [<ffffffff815351c4>] __do_page_fault+0x1d4/0x510 |
97 | 11 &mm->mmap_sem 279 [<ffffffff80210a57>] sys_mmap+0x75/0xce | 95 | 11 &mm->mmap_sem 34 [<ffffffff81113d77>] vm_mmap_pgoff+0x87/0xd0 |
98 | 12 &mm->mmap_sem 76 [<ffffffff802a490b>] sys_munmap+0x32/0x59 | 96 | 12 &mm->mmap_sem 17 [<ffffffff81127e71>] vm_munmap+0x41/0x80 |
99 | 13 --------------- | 97 | 13 --------------- |
100 | 14 &mm->mmap_sem 270 [<ffffffff80210a57>] sys_mmap+0x75/0xce | 98 | 14 &mm->mmap_sem 1 [<ffffffff81046fda>] dup_mmap+0x2a/0x3f0 |
101 | 15 &mm->mmap_sem 431 [<ffffffff8053491f>] do_page_fault+0x466/0x928 | 99 | 15 &mm->mmap_sem 60 [<ffffffff81129e29>] SyS_mprotect+0xe9/0x250 |
102 | 16 &mm->mmap_sem 138 [<ffffffff802a490b>] sys_munmap+0x32/0x59 | 100 | 16 &mm->mmap_sem 41 [<ffffffff815351c4>] __do_page_fault+0x1d4/0x510 |
103 | 17 &mm->mmap_sem 145 [<ffffffff802a6200>] sys_mprotect+0xcd/0x21d | 101 | 17 &mm->mmap_sem 68 [<ffffffff81113d77>] vm_mmap_pgoff+0x87/0xd0 |
104 | 18 | 102 | 18 |
105 | 19 ............................................................................................................................................................................................... | 103 | 19............................................................................................................................................................................................................................. |
106 | 20 | 104 | 20 |
107 | 21 dcache_lock: 621 623 0.52 118.26 1053.02 6745 91930 0.29 316.29 118423.41 | 105 | 21 unix_table_lock: 110 112 0.21 49.24 163.91 1.46 21094 66312 0.12 624.42 31589.81 0.48 |
108 | 22 ----------- | 106 | 22 --------------- |
109 | 23 dcache_lock 179 [<ffffffff80378274>] _atomic_dec_and_lock+0x34/0x54 | 107 | 23 unix_table_lock 45 [<ffffffff8150ad8e>] unix_create1+0x16e/0x1b0 |
110 | 24 dcache_lock 113 [<ffffffff802cc17b>] d_alloc+0x19a/0x1eb | 108 | 24 unix_table_lock 47 [<ffffffff8150b111>] unix_release_sock+0x31/0x250 |
111 | 25 dcache_lock 99 [<ffffffff802ca0dc>] d_rehash+0x1b/0x44 | 109 | 25 unix_table_lock 15 [<ffffffff8150ca37>] unix_find_other+0x117/0x230 |
112 | 26 dcache_lock 104 [<ffffffff802cbca0>] d_instantiate+0x36/0x8a | 110 | 26 unix_table_lock 5 [<ffffffff8150a09f>] unix_autobind+0x11f/0x1b0 |
113 | 27 ----------- | 111 | 27 --------------- |
114 | 28 dcache_lock 192 [<ffffffff80378274>] _atomic_dec_and_lock+0x34/0x54 | 112 | 28 unix_table_lock 39 [<ffffffff8150b111>] unix_release_sock+0x31/0x250 |
115 | 29 dcache_lock 98 [<ffffffff802ca0dc>] d_rehash+0x1b/0x44 | 113 | 29 unix_table_lock 49 [<ffffffff8150ad8e>] unix_create1+0x16e/0x1b0 |
116 | 30 dcache_lock 72 [<ffffffff802cc17b>] d_alloc+0x19a/0x1eb | 114 | 30 unix_table_lock 20 [<ffffffff8150ca37>] unix_find_other+0x117/0x230 |
117 | 31 dcache_lock 112 [<ffffffff802cbca0>] d_instantiate+0x36/0x8a | 115 | 31 unix_table_lock 4 [<ffffffff8150a09f>] unix_autobind+0x11f/0x1b0 |
116 | |||
118 | 117 | ||
119 | This excerpt shows the first two lock class statistics. Line 01 shows the | 118 | This excerpt shows the first two lock class statistics. Line 01 shows the |
120 | output version - each time the format changes this will be updated. Line 02-04 | 119 | output version - each time the format changes this will be updated. Line 02-04 |
@@ -131,30 +130,30 @@ The integer part of the time values is in us. | |||
131 | 130 | ||
132 | Dealing with nested locks, subclasses may appear: | 131 | Dealing with nested locks, subclasses may appear: |
133 | 132 | ||
134 | 32............................................................................................................................................................................................... | 133 | 32........................................................................................................................................................................................................................... |
135 | 33 | 134 | 33 |
136 | 34 &rq->lock: 13128 13128 0.43 190.53 103881.26 97454 3453404 0.00 401.11 13224683.11 | 135 | 34 &rq->lock: 13128 13128 0.43 190.53 103881.26 7.91 97454 3453404 0.00 401.11 13224683.11 3.82 |
137 | 35 --------- | 136 | 35 --------- |
138 | 36 &rq->lock 645 [<ffffffff8103bfc4>] task_rq_lock+0x43/0x75 | 137 | 36 &rq->lock 645 [<ffffffff8103bfc4>] task_rq_lock+0x43/0x75 |
139 | 37 &rq->lock 297 [<ffffffff8104ba65>] try_to_wake_up+0x127/0x25a | 138 | 37 &rq->lock 297 [<ffffffff8104ba65>] try_to_wake_up+0x127/0x25a |
140 | 38 &rq->lock 360 [<ffffffff8103c4c5>] select_task_rq_fair+0x1f0/0x74a | 139 | 38 &rq->lock 360 [<ffffffff8103c4c5>] select_task_rq_fair+0x1f0/0x74a |
141 | 39 &rq->lock 428 [<ffffffff81045f98>] scheduler_tick+0x46/0x1fb | 140 | 39 &rq->lock 428 [<ffffffff81045f98>] scheduler_tick+0x46/0x1fb |
142 | 40 --------- | 141 | 40 --------- |
143 | 41 &rq->lock 77 [<ffffffff8103bfc4>] task_rq_lock+0x43/0x75 | 142 | 41 &rq->lock 77 [<ffffffff8103bfc4>] task_rq_lock+0x43/0x75 |
144 | 42 &rq->lock 174 [<ffffffff8104ba65>] try_to_wake_up+0x127/0x25a | 143 | 42 &rq->lock 174 [<ffffffff8104ba65>] try_to_wake_up+0x127/0x25a |
145 | 43 &rq->lock 4715 [<ffffffff8103ed4b>] double_rq_lock+0x42/0x54 | 144 | 43 &rq->lock 4715 [<ffffffff8103ed4b>] double_rq_lock+0x42/0x54 |
146 | 44 &rq->lock 893 [<ffffffff81340524>] schedule+0x157/0x7b8 | 145 | 44 &rq->lock 893 [<ffffffff81340524>] schedule+0x157/0x7b8 |
147 | 45 | 146 | 45 |
148 | 46............................................................................................................................................................................................... | 147 | 46........................................................................................................................................................................................................................... |
149 | 47 | 148 | 47 |
150 | 48 &rq->lock/1: 11526 11488 0.33 388.73 136294.31 21461 38404 0.00 37.93 109388.53 | 149 | 48 &rq->lock/1: 1526 11488 0.33 388.73 136294.31 11.86 21461 38404 0.00 37.93 109388.53 2.84 |
151 | 49 ----------- | 150 | 49 ----------- |
152 | 50 &rq->lock/1 11526 [<ffffffff8103ed58>] double_rq_lock+0x4f/0x54 | 151 | 50 &rq->lock/1 11526 [<ffffffff8103ed58>] double_rq_lock+0x4f/0x54 |
153 | 51 ----------- | 152 | 51 ----------- |
154 | 52 &rq->lock/1 5645 [<ffffffff8103ed4b>] double_rq_lock+0x42/0x54 | 153 | 52 &rq->lock/1 5645 [<ffffffff8103ed4b>] double_rq_lock+0x42/0x54 |
155 | 53 &rq->lock/1 1224 [<ffffffff81340524>] schedule+0x157/0x7b8 | 154 | 53 &rq->lock/1 1224 [<ffffffff81340524>] schedule+0x157/0x7b8 |
156 | 54 &rq->lock/1 4336 [<ffffffff8103ed58>] double_rq_lock+0x4f/0x54 | 155 | 54 &rq->lock/1 4336 [<ffffffff8103ed58>] double_rq_lock+0x4f/0x54 |
157 | 55 &rq->lock/1 181 [<ffffffff8104ba65>] try_to_wake_up+0x127/0x25a | 156 | 55 &rq->lock/1 181 [<ffffffff8104ba65>] try_to_wake_up+0x127/0x25a |
158 | 157 | ||
159 | Line 48 shows statistics for the second subclass (/1) of &rq->lock class | 158 | Line 48 shows statistics for the second subclass (/1) of &rq->lock class |
160 | (subclass starts from 0), since in this case, as line 50 suggests, | 159 | (subclass starts from 0), since in this case, as line 50 suggests, |
@@ -163,16 +162,16 @@ double_rq_lock actually acquires a nested lock of two spinlocks. | |||
163 | View the top contending locks: | 162 | View the top contending locks: |
164 | 163 | ||
165 | # grep : /proc/lock_stat | head | 164 | # grep : /proc/lock_stat | head |
166 | &inode->i_data.tree_lock-W: 15 21657 0.18 1093295.30 11547131054.85 58 10415 0.16 87.51 6387.60 | 165 | clockevents_lock: 2926159 2947636 0.15 46882.81 1784540466.34 605.41 3381345 3879161 0.00 2260.97 53178395.68 13.71 |
167 | &inode->i_data.tree_lock-R: 0 0 0.00 0.00 0.00 23302 231198 0.25 8.45 98023.38 | 166 | tick_broadcast_lock: 346460 346717 0.18 2257.43 39364622.71 113.54 3642919 4242696 0.00 2263.79 49173646.60 11.59 |
168 | dcache_lock: 1037 1161 0.38 45.32 774.51 6611 243371 0.15 306.48 77387.24 | 167 | &mapping->i_mmap_mutex: 203896 203899 3.36 645530.05 31767507988.39 155800.21 3361776 8893984 0.17 2254.15 14110121.02 1.59 |
169 | &inode->i_mutex: 161 286 18446744073709 62882.54 1244614.55 3653 20598 18446744073709 62318.60 1693822.74 | 168 | &rq->lock: 135014 136909 0.18 606.09 842160.68 6.15 1540728 10436146 0.00 728.72 17606683.41 1.69 |
170 | &zone->lru_lock: 94 94 0.53 7.33 92.10 4366 32690 0.29 59.81 16350.06 | 169 | &(&zone->lru_lock)->rlock: 93000 94934 0.16 59.18 188253.78 1.98 1199912 3809894 0.15 391.40 3559518.81 0.93 |
171 | &inode->i_data.i_mmap_mutex: 79 79 0.40 3.77 53.03 11779 87755 0.28 116.93 29898.44 | 170 | tasklist_lock-W: 40667 41130 0.23 1189.42 428980.51 10.43 270278 510106 0.16 653.51 3939674.91 7.72 |
172 | &q->__queue_lock: 48 50 0.52 31.62 86.31 774 13131 0.17 113.08 12277.52 | 171 | tasklist_lock-R: 21298 21305 0.20 1310.05 215511.12 10.12 186204 241258 0.14 1162.33 1179779.23 4.89 |
173 | &rq->rq_lock_key: 43 47 0.74 68.50 170.63 3706 33929 0.22 107.99 17460.62 | 172 | rcu_node_1: 47656 49022 0.16 635.41 193616.41 3.95 844888 1865423 0.00 764.26 1656226.96 0.89 |
174 | &rq->rq_lock_key#2: 39 46 0.75 6.68 49.03 2979 32292 0.17 125.17 17137.63 | 173 | &(&dentry->d_lockref.lock)->rlock: 39791 40179 0.15 1302.08 88851.96 2.21 2790851 12527025 0.10 1910.75 3379714.27 0.27 |
175 | tasklist_lock-W: 15 15 1.45 10.87 32.70 1201 7390 0.58 62.55 13648.47 | 174 | rcu_node_0: 29203 30064 0.16 786.55 1555573.00 51.74 88963 244254 0.00 398.87 428872.51 1.76 |
176 | 175 | ||
177 | Clear the statistics: | 176 | Clear the statistics: |
178 | 177 | ||
diff --git a/Documentation/mutex-design.txt b/Documentation/mutex-design.txt index 38c10fd7f411..1dfe62c3641d 100644 --- a/Documentation/mutex-design.txt +++ b/Documentation/mutex-design.txt | |||
@@ -116,11 +116,11 @@ using mutexes at the moment, please let me know if you find any. ] | |||
116 | Implementation of mutexes | 116 | Implementation of mutexes |
117 | ------------------------- | 117 | ------------------------- |
118 | 118 | ||
119 | 'struct mutex' is the new mutex type, defined in include/linux/mutex.h | 119 | 'struct mutex' is the new mutex type, defined in include/linux/mutex.h and |
120 | and implemented in kernel/mutex.c. It is a counter-based mutex with a | 120 | implemented in kernel/locking/mutex.c. It is a counter-based mutex with a |
121 | spinlock and a wait-list. The counter has 3 states: 1 for "unlocked", | 121 | spinlock and a wait-list. The counter has 3 states: 1 for "unlocked", 0 for |
122 | 0 for "locked" and negative numbers (usually -1) for "locked, potential | 122 | "locked" and negative numbers (usually -1) for "locked, potential waiters |
123 | waiters queued". | 123 | queued". |
124 | 124 | ||
125 | the APIs of 'struct mutex' have been streamlined: | 125 | the APIs of 'struct mutex' have been streamlined: |
126 | 126 | ||
diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig index 14dc9c797abb..6e3e1cb3f6a0 100644 --- a/arch/x86/Kconfig +++ b/arch/x86/Kconfig | |||
@@ -635,10 +635,10 @@ config PARAVIRT_SPINLOCKS | |||
635 | spinlock implementation with something virtualization-friendly | 635 | spinlock implementation with something virtualization-friendly |
636 | (for example, block the virtual CPU rather than spinning). | 636 | (for example, block the virtual CPU rather than spinning). |
637 | 637 | ||
638 | Unfortunately the downside is an up to 5% performance hit on | 638 | It has a minimal impact on native kernels and gives a nice performance |
639 | native kernels, with various workloads. | 639 | benefit on paravirtualized KVM / Xen kernels. |
640 | 640 | ||
641 | If you are unsure how to answer this question, answer N. | 641 | If you are unsure how to answer this question, answer Y. |
642 | 642 | ||
643 | source "arch/x86/xen/Kconfig" | 643 | source "arch/x86/xen/Kconfig" |
644 | 644 | ||
diff --git a/arch/x86/kernel/alternative.c b/arch/x86/kernel/alternative.c index 15e8563e5c24..df94598ad05a 100644 --- a/arch/x86/kernel/alternative.c +++ b/arch/x86/kernel/alternative.c | |||
@@ -402,17 +402,6 @@ void alternatives_enable_smp(void) | |||
402 | { | 402 | { |
403 | struct smp_alt_module *mod; | 403 | struct smp_alt_module *mod; |
404 | 404 | ||
405 | #ifdef CONFIG_LOCKDEP | ||
406 | /* | ||
407 | * Older binutils section handling bug prevented | ||
408 | * alternatives-replacement from working reliably. | ||
409 | * | ||
410 | * If this still occurs then you should see a hang | ||
411 | * or crash shortly after this line: | ||
412 | */ | ||
413 | pr_info("lockdep: fixing up alternatives\n"); | ||
414 | #endif | ||
415 | |||
416 | /* Why bother if there are no other CPUs? */ | 405 | /* Why bother if there are no other CPUs? */ |
417 | BUG_ON(num_possible_cpus() == 1); | 406 | BUG_ON(num_possible_cpus() == 1); |
418 | 407 | ||
diff --git a/arch/x86/vdso/vclock_gettime.c b/arch/x86/vdso/vclock_gettime.c index 72074d528400..2ada505067cc 100644 --- a/arch/x86/vdso/vclock_gettime.c +++ b/arch/x86/vdso/vclock_gettime.c | |||
@@ -178,7 +178,7 @@ notrace static int __always_inline do_realtime(struct timespec *ts) | |||
178 | 178 | ||
179 | ts->tv_nsec = 0; | 179 | ts->tv_nsec = 0; |
180 | do { | 180 | do { |
181 | seq = read_seqcount_begin(>od->seq); | 181 | seq = read_seqcount_begin_no_lockdep(>od->seq); |
182 | mode = gtod->clock.vclock_mode; | 182 | mode = gtod->clock.vclock_mode; |
183 | ts->tv_sec = gtod->wall_time_sec; | 183 | ts->tv_sec = gtod->wall_time_sec; |
184 | ns = gtod->wall_time_snsec; | 184 | ns = gtod->wall_time_snsec; |
@@ -198,7 +198,7 @@ notrace static int do_monotonic(struct timespec *ts) | |||
198 | 198 | ||
199 | ts->tv_nsec = 0; | 199 | ts->tv_nsec = 0; |
200 | do { | 200 | do { |
201 | seq = read_seqcount_begin(>od->seq); | 201 | seq = read_seqcount_begin_no_lockdep(>od->seq); |
202 | mode = gtod->clock.vclock_mode; | 202 | mode = gtod->clock.vclock_mode; |
203 | ts->tv_sec = gtod->monotonic_time_sec; | 203 | ts->tv_sec = gtod->monotonic_time_sec; |
204 | ns = gtod->monotonic_time_snsec; | 204 | ns = gtod->monotonic_time_snsec; |
@@ -214,7 +214,7 @@ notrace static int do_realtime_coarse(struct timespec *ts) | |||
214 | { | 214 | { |
215 | unsigned long seq; | 215 | unsigned long seq; |
216 | do { | 216 | do { |
217 | seq = read_seqcount_begin(>od->seq); | 217 | seq = read_seqcount_begin_no_lockdep(>od->seq); |
218 | ts->tv_sec = gtod->wall_time_coarse.tv_sec; | 218 | ts->tv_sec = gtod->wall_time_coarse.tv_sec; |
219 | ts->tv_nsec = gtod->wall_time_coarse.tv_nsec; | 219 | ts->tv_nsec = gtod->wall_time_coarse.tv_nsec; |
220 | } while (unlikely(read_seqcount_retry(>od->seq, seq))); | 220 | } while (unlikely(read_seqcount_retry(>od->seq, seq))); |
@@ -225,7 +225,7 @@ notrace static int do_monotonic_coarse(struct timespec *ts) | |||
225 | { | 225 | { |
226 | unsigned long seq; | 226 | unsigned long seq; |
227 | do { | 227 | do { |
228 | seq = read_seqcount_begin(>od->seq); | 228 | seq = read_seqcount_begin_no_lockdep(>od->seq); |
229 | ts->tv_sec = gtod->monotonic_time_coarse.tv_sec; | 229 | ts->tv_sec = gtod->monotonic_time_coarse.tv_sec; |
230 | ts->tv_nsec = gtod->monotonic_time_coarse.tv_nsec; | 230 | ts->tv_nsec = gtod->monotonic_time_coarse.tv_nsec; |
231 | } while (unlikely(read_seqcount_retry(>od->seq, seq))); | 231 | } while (unlikely(read_seqcount_retry(>od->seq, seq))); |
diff --git a/block/blk-cgroup.h b/block/blk-cgroup.h index ae6969a7ffd4..1610b22edf09 100644 --- a/block/blk-cgroup.h +++ b/block/blk-cgroup.h | |||
@@ -402,6 +402,11 @@ struct request_list *__blk_queue_next_rl(struct request_list *rl, | |||
402 | #define blk_queue_for_each_rl(rl, q) \ | 402 | #define blk_queue_for_each_rl(rl, q) \ |
403 | for ((rl) = &(q)->root_rl; (rl); (rl) = __blk_queue_next_rl((rl), (q))) | 403 | for ((rl) = &(q)->root_rl; (rl); (rl) = __blk_queue_next_rl((rl), (q))) |
404 | 404 | ||
405 | static inline void blkg_stat_init(struct blkg_stat *stat) | ||
406 | { | ||
407 | u64_stats_init(&stat->syncp); | ||
408 | } | ||
409 | |||
405 | /** | 410 | /** |
406 | * blkg_stat_add - add a value to a blkg_stat | 411 | * blkg_stat_add - add a value to a blkg_stat |
407 | * @stat: target blkg_stat | 412 | * @stat: target blkg_stat |
@@ -458,6 +463,11 @@ static inline void blkg_stat_merge(struct blkg_stat *to, struct blkg_stat *from) | |||
458 | blkg_stat_add(to, blkg_stat_read(from)); | 463 | blkg_stat_add(to, blkg_stat_read(from)); |
459 | } | 464 | } |
460 | 465 | ||
466 | static inline void blkg_rwstat_init(struct blkg_rwstat *rwstat) | ||
467 | { | ||
468 | u64_stats_init(&rwstat->syncp); | ||
469 | } | ||
470 | |||
461 | /** | 471 | /** |
462 | * blkg_rwstat_add - add a value to a blkg_rwstat | 472 | * blkg_rwstat_add - add a value to a blkg_rwstat |
463 | * @rwstat: target blkg_rwstat | 473 | * @rwstat: target blkg_rwstat |
diff --git a/block/blk-throttle.c b/block/blk-throttle.c index 8331aba9426f..06534049afba 100644 --- a/block/blk-throttle.c +++ b/block/blk-throttle.c | |||
@@ -256,6 +256,12 @@ static struct throtl_data *sq_to_td(struct throtl_service_queue *sq) | |||
256 | } \ | 256 | } \ |
257 | } while (0) | 257 | } while (0) |
258 | 258 | ||
259 | static void tg_stats_init(struct tg_stats_cpu *tg_stats) | ||
260 | { | ||
261 | blkg_rwstat_init(&tg_stats->service_bytes); | ||
262 | blkg_rwstat_init(&tg_stats->serviced); | ||
263 | } | ||
264 | |||
259 | /* | 265 | /* |
260 | * Worker for allocating per cpu stat for tgs. This is scheduled on the | 266 | * Worker for allocating per cpu stat for tgs. This is scheduled on the |
261 | * system_wq once there are some groups on the alloc_list waiting for | 267 | * system_wq once there are some groups on the alloc_list waiting for |
@@ -269,12 +275,16 @@ static void tg_stats_alloc_fn(struct work_struct *work) | |||
269 | 275 | ||
270 | alloc_stats: | 276 | alloc_stats: |
271 | if (!stats_cpu) { | 277 | if (!stats_cpu) { |
278 | int cpu; | ||
279 | |||
272 | stats_cpu = alloc_percpu(struct tg_stats_cpu); | 280 | stats_cpu = alloc_percpu(struct tg_stats_cpu); |
273 | if (!stats_cpu) { | 281 | if (!stats_cpu) { |
274 | /* allocation failed, try again after some time */ | 282 | /* allocation failed, try again after some time */ |
275 | schedule_delayed_work(dwork, msecs_to_jiffies(10)); | 283 | schedule_delayed_work(dwork, msecs_to_jiffies(10)); |
276 | return; | 284 | return; |
277 | } | 285 | } |
286 | for_each_possible_cpu(cpu) | ||
287 | tg_stats_init(per_cpu_ptr(stats_cpu, cpu)); | ||
278 | } | 288 | } |
279 | 289 | ||
280 | spin_lock_irq(&tg_stats_alloc_lock); | 290 | spin_lock_irq(&tg_stats_alloc_lock); |
diff --git a/block/cfq-iosched.c b/block/cfq-iosched.c index 434944cbd761..4d5cec1ad80d 100644 --- a/block/cfq-iosched.c +++ b/block/cfq-iosched.c | |||
@@ -1508,6 +1508,29 @@ static void cfq_init_cfqg_base(struct cfq_group *cfqg) | |||
1508 | } | 1508 | } |
1509 | 1509 | ||
1510 | #ifdef CONFIG_CFQ_GROUP_IOSCHED | 1510 | #ifdef CONFIG_CFQ_GROUP_IOSCHED |
1511 | static void cfqg_stats_init(struct cfqg_stats *stats) | ||
1512 | { | ||
1513 | blkg_rwstat_init(&stats->service_bytes); | ||
1514 | blkg_rwstat_init(&stats->serviced); | ||
1515 | blkg_rwstat_init(&stats->merged); | ||
1516 | blkg_rwstat_init(&stats->service_time); | ||
1517 | blkg_rwstat_init(&stats->wait_time); | ||
1518 | blkg_rwstat_init(&stats->queued); | ||
1519 | |||
1520 | blkg_stat_init(&stats->sectors); | ||
1521 | blkg_stat_init(&stats->time); | ||
1522 | |||
1523 | #ifdef CONFIG_DEBUG_BLK_CGROUP | ||
1524 | blkg_stat_init(&stats->unaccounted_time); | ||
1525 | blkg_stat_init(&stats->avg_queue_size_sum); | ||
1526 | blkg_stat_init(&stats->avg_queue_size_samples); | ||
1527 | blkg_stat_init(&stats->dequeue); | ||
1528 | blkg_stat_init(&stats->group_wait_time); | ||
1529 | blkg_stat_init(&stats->idle_time); | ||
1530 | blkg_stat_init(&stats->empty_time); | ||
1531 | #endif | ||
1532 | } | ||
1533 | |||
1511 | static void cfq_pd_init(struct blkcg_gq *blkg) | 1534 | static void cfq_pd_init(struct blkcg_gq *blkg) |
1512 | { | 1535 | { |
1513 | struct cfq_group *cfqg = blkg_to_cfqg(blkg); | 1536 | struct cfq_group *cfqg = blkg_to_cfqg(blkg); |
@@ -1515,6 +1538,8 @@ static void cfq_pd_init(struct blkcg_gq *blkg) | |||
1515 | cfq_init_cfqg_base(cfqg); | 1538 | cfq_init_cfqg_base(cfqg); |
1516 | cfqg->weight = blkg->blkcg->cfq_weight; | 1539 | cfqg->weight = blkg->blkcg->cfq_weight; |
1517 | cfqg->leaf_weight = blkg->blkcg->cfq_leaf_weight; | 1540 | cfqg->leaf_weight = blkg->blkcg->cfq_leaf_weight; |
1541 | cfqg_stats_init(&cfqg->stats); | ||
1542 | cfqg_stats_init(&cfqg->dead_stats); | ||
1518 | } | 1543 | } |
1519 | 1544 | ||
1520 | static void cfq_pd_offline(struct blkcg_gq *blkg) | 1545 | static void cfq_pd_offline(struct blkcg_gq *blkg) |
diff --git a/drivers/net/dummy.c b/drivers/net/dummy.c index b710c6b2d659..bd8f84b0b894 100644 --- a/drivers/net/dummy.c +++ b/drivers/net/dummy.c | |||
@@ -88,10 +88,16 @@ static netdev_tx_t dummy_xmit(struct sk_buff *skb, struct net_device *dev) | |||
88 | 88 | ||
89 | static int dummy_dev_init(struct net_device *dev) | 89 | static int dummy_dev_init(struct net_device *dev) |
90 | { | 90 | { |
91 | int i; | ||
91 | dev->dstats = alloc_percpu(struct pcpu_dstats); | 92 | dev->dstats = alloc_percpu(struct pcpu_dstats); |
92 | if (!dev->dstats) | 93 | if (!dev->dstats) |
93 | return -ENOMEM; | 94 | return -ENOMEM; |
94 | 95 | ||
96 | for_each_possible_cpu(i) { | ||
97 | struct pcpu_dstats *dstats; | ||
98 | dstats = per_cpu_ptr(dev->dstats, i); | ||
99 | u64_stats_init(&dstats->syncp); | ||
100 | } | ||
95 | return 0; | 101 | return 0; |
96 | } | 102 | } |
97 | 103 | ||
diff --git a/drivers/net/ethernet/emulex/benet/be_main.c b/drivers/net/ethernet/emulex/benet/be_main.c index cb2bb6fccbc8..eaecaadfa8c5 100644 --- a/drivers/net/ethernet/emulex/benet/be_main.c +++ b/drivers/net/ethernet/emulex/benet/be_main.c | |||
@@ -2148,6 +2148,9 @@ static int be_tx_qs_create(struct be_adapter *adapter) | |||
2148 | if (status) | 2148 | if (status) |
2149 | return status; | 2149 | return status; |
2150 | 2150 | ||
2151 | u64_stats_init(&txo->stats.sync); | ||
2152 | u64_stats_init(&txo->stats.sync_compl); | ||
2153 | |||
2151 | /* If num_evt_qs is less than num_tx_qs, then more than | 2154 | /* If num_evt_qs is less than num_tx_qs, then more than |
2152 | * one txq share an eq | 2155 | * one txq share an eq |
2153 | */ | 2156 | */ |
@@ -2209,6 +2212,7 @@ static int be_rx_cqs_create(struct be_adapter *adapter) | |||
2209 | if (rc) | 2212 | if (rc) |
2210 | return rc; | 2213 | return rc; |
2211 | 2214 | ||
2215 | u64_stats_init(&rxo->stats.sync); | ||
2212 | eq = &adapter->eq_obj[i % adapter->num_evt_qs].q; | 2216 | eq = &adapter->eq_obj[i % adapter->num_evt_qs].q; |
2213 | rc = be_cmd_cq_create(adapter, cq, eq, false, 3); | 2217 | rc = be_cmd_cq_create(adapter, cq, eq, false, 3); |
2214 | if (rc) | 2218 | if (rc) |
diff --git a/drivers/net/ethernet/intel/igb/igb_main.c b/drivers/net/ethernet/intel/igb/igb_main.c index 2ac14bdd5fbb..025e5f4b7481 100644 --- a/drivers/net/ethernet/intel/igb/igb_main.c +++ b/drivers/net/ethernet/intel/igb/igb_main.c | |||
@@ -1224,6 +1224,9 @@ static int igb_alloc_q_vector(struct igb_adapter *adapter, | |||
1224 | ring->count = adapter->tx_ring_count; | 1224 | ring->count = adapter->tx_ring_count; |
1225 | ring->queue_index = txr_idx; | 1225 | ring->queue_index = txr_idx; |
1226 | 1226 | ||
1227 | u64_stats_init(&ring->tx_syncp); | ||
1228 | u64_stats_init(&ring->tx_syncp2); | ||
1229 | |||
1227 | /* assign ring to adapter */ | 1230 | /* assign ring to adapter */ |
1228 | adapter->tx_ring[txr_idx] = ring; | 1231 | adapter->tx_ring[txr_idx] = ring; |
1229 | 1232 | ||
@@ -1257,6 +1260,8 @@ static int igb_alloc_q_vector(struct igb_adapter *adapter, | |||
1257 | ring->count = adapter->rx_ring_count; | 1260 | ring->count = adapter->rx_ring_count; |
1258 | ring->queue_index = rxr_idx; | 1261 | ring->queue_index = rxr_idx; |
1259 | 1262 | ||
1263 | u64_stats_init(&ring->rx_syncp); | ||
1264 | |||
1260 | /* assign ring to adapter */ | 1265 | /* assign ring to adapter */ |
1261 | adapter->rx_ring[rxr_idx] = ring; | 1266 | adapter->rx_ring[rxr_idx] = ring; |
1262 | } | 1267 | } |
diff --git a/drivers/net/ethernet/intel/ixgbe/ixgbe_main.c b/drivers/net/ethernet/intel/ixgbe/ixgbe_main.c index 0066f0aefbfa..0c55079ebee3 100644 --- a/drivers/net/ethernet/intel/ixgbe/ixgbe_main.c +++ b/drivers/net/ethernet/intel/ixgbe/ixgbe_main.c | |||
@@ -5085,6 +5085,8 @@ int ixgbe_setup_tx_resources(struct ixgbe_ring *tx_ring) | |||
5085 | if (!tx_ring->tx_buffer_info) | 5085 | if (!tx_ring->tx_buffer_info) |
5086 | goto err; | 5086 | goto err; |
5087 | 5087 | ||
5088 | u64_stats_init(&tx_ring->syncp); | ||
5089 | |||
5088 | /* round up to nearest 4K */ | 5090 | /* round up to nearest 4K */ |
5089 | tx_ring->size = tx_ring->count * sizeof(union ixgbe_adv_tx_desc); | 5091 | tx_ring->size = tx_ring->count * sizeof(union ixgbe_adv_tx_desc); |
5090 | tx_ring->size = ALIGN(tx_ring->size, 4096); | 5092 | tx_ring->size = ALIGN(tx_ring->size, 4096); |
@@ -5167,6 +5169,8 @@ int ixgbe_setup_rx_resources(struct ixgbe_ring *rx_ring) | |||
5167 | if (!rx_ring->rx_buffer_info) | 5169 | if (!rx_ring->rx_buffer_info) |
5168 | goto err; | 5170 | goto err; |
5169 | 5171 | ||
5172 | u64_stats_init(&rx_ring->syncp); | ||
5173 | |||
5170 | /* Round up to nearest 4K */ | 5174 | /* Round up to nearest 4K */ |
5171 | rx_ring->size = rx_ring->count * sizeof(union ixgbe_adv_rx_desc); | 5175 | rx_ring->size = rx_ring->count * sizeof(union ixgbe_adv_rx_desc); |
5172 | rx_ring->size = ALIGN(rx_ring->size, 4096); | 5176 | rx_ring->size = ALIGN(rx_ring->size, 4096); |
diff --git a/drivers/net/ethernet/marvell/mvneta.c b/drivers/net/ethernet/marvell/mvneta.c index 7d99e695a110..b8e232b4ea2d 100644 --- a/drivers/net/ethernet/marvell/mvneta.c +++ b/drivers/net/ethernet/marvell/mvneta.c | |||
@@ -2792,6 +2792,9 @@ static int mvneta_probe(struct platform_device *pdev) | |||
2792 | 2792 | ||
2793 | pp = netdev_priv(dev); | 2793 | pp = netdev_priv(dev); |
2794 | 2794 | ||
2795 | u64_stats_init(&pp->tx_stats.syncp); | ||
2796 | u64_stats_init(&pp->rx_stats.syncp); | ||
2797 | |||
2795 | pp->weight = MVNETA_RX_POLL_WEIGHT; | 2798 | pp->weight = MVNETA_RX_POLL_WEIGHT; |
2796 | pp->phy_node = phy_node; | 2799 | pp->phy_node = phy_node; |
2797 | pp->phy_interface = phy_mode; | 2800 | pp->phy_interface = phy_mode; |
diff --git a/drivers/net/ethernet/marvell/sky2.c b/drivers/net/ethernet/marvell/sky2.c index a7df981d2123..43aa7acd84a6 100644 --- a/drivers/net/ethernet/marvell/sky2.c +++ b/drivers/net/ethernet/marvell/sky2.c | |||
@@ -4763,6 +4763,9 @@ static struct net_device *sky2_init_netdev(struct sky2_hw *hw, unsigned port, | |||
4763 | sky2->hw = hw; | 4763 | sky2->hw = hw; |
4764 | sky2->msg_enable = netif_msg_init(debug, default_msg); | 4764 | sky2->msg_enable = netif_msg_init(debug, default_msg); |
4765 | 4765 | ||
4766 | u64_stats_init(&sky2->tx_stats.syncp); | ||
4767 | u64_stats_init(&sky2->rx_stats.syncp); | ||
4768 | |||
4766 | /* Auto speed and flow control */ | 4769 | /* Auto speed and flow control */ |
4767 | sky2->flags = SKY2_FLAG_AUTO_SPEED | SKY2_FLAG_AUTO_PAUSE; | 4770 | sky2->flags = SKY2_FLAG_AUTO_SPEED | SKY2_FLAG_AUTO_PAUSE; |
4768 | if (hw->chip_id != CHIP_ID_YUKON_XL) | 4771 | if (hw->chip_id != CHIP_ID_YUKON_XL) |
diff --git a/drivers/net/ethernet/neterion/vxge/vxge-main.c b/drivers/net/ethernet/neterion/vxge/vxge-main.c index 8614eeb7de81..f9876ea8c8bf 100644 --- a/drivers/net/ethernet/neterion/vxge/vxge-main.c +++ b/drivers/net/ethernet/neterion/vxge/vxge-main.c | |||
@@ -2072,6 +2072,10 @@ static int vxge_open_vpaths(struct vxgedev *vdev) | |||
2072 | vdev->config.tx_steering_type; | 2072 | vdev->config.tx_steering_type; |
2073 | vpath->fifo.ndev = vdev->ndev; | 2073 | vpath->fifo.ndev = vdev->ndev; |
2074 | vpath->fifo.pdev = vdev->pdev; | 2074 | vpath->fifo.pdev = vdev->pdev; |
2075 | |||
2076 | u64_stats_init(&vpath->fifo.stats.syncp); | ||
2077 | u64_stats_init(&vpath->ring.stats.syncp); | ||
2078 | |||
2075 | if (vdev->config.tx_steering_type) | 2079 | if (vdev->config.tx_steering_type) |
2076 | vpath->fifo.txq = | 2080 | vpath->fifo.txq = |
2077 | netdev_get_tx_queue(vdev->ndev, i); | 2081 | netdev_get_tx_queue(vdev->ndev, i); |
diff --git a/drivers/net/ethernet/nvidia/forcedeth.c b/drivers/net/ethernet/nvidia/forcedeth.c index 098b96dad66f..2d045be4b5cf 100644 --- a/drivers/net/ethernet/nvidia/forcedeth.c +++ b/drivers/net/ethernet/nvidia/forcedeth.c | |||
@@ -5619,6 +5619,8 @@ static int nv_probe(struct pci_dev *pci_dev, const struct pci_device_id *id) | |||
5619 | spin_lock_init(&np->lock); | 5619 | spin_lock_init(&np->lock); |
5620 | spin_lock_init(&np->hwstats_lock); | 5620 | spin_lock_init(&np->hwstats_lock); |
5621 | SET_NETDEV_DEV(dev, &pci_dev->dev); | 5621 | SET_NETDEV_DEV(dev, &pci_dev->dev); |
5622 | u64_stats_init(&np->swstats_rx_syncp); | ||
5623 | u64_stats_init(&np->swstats_tx_syncp); | ||
5622 | 5624 | ||
5623 | init_timer(&np->oom_kick); | 5625 | init_timer(&np->oom_kick); |
5624 | np->oom_kick.data = (unsigned long) dev; | 5626 | np->oom_kick.data = (unsigned long) dev; |
diff --git a/drivers/net/ethernet/realtek/8139too.c b/drivers/net/ethernet/realtek/8139too.c index 50a92104dd0a..da5972eefdd2 100644 --- a/drivers/net/ethernet/realtek/8139too.c +++ b/drivers/net/ethernet/realtek/8139too.c | |||
@@ -790,6 +790,9 @@ static struct net_device *rtl8139_init_board(struct pci_dev *pdev) | |||
790 | 790 | ||
791 | pci_set_master (pdev); | 791 | pci_set_master (pdev); |
792 | 792 | ||
793 | u64_stats_init(&tp->rx_stats.syncp); | ||
794 | u64_stats_init(&tp->tx_stats.syncp); | ||
795 | |||
793 | retry: | 796 | retry: |
794 | /* PIO bar register comes first. */ | 797 | /* PIO bar register comes first. */ |
795 | bar = !use_io; | 798 | bar = !use_io; |
diff --git a/drivers/net/ethernet/tile/tilepro.c b/drivers/net/ethernet/tile/tilepro.c index 106be47716e7..edb2e12a0fe2 100644 --- a/drivers/net/ethernet/tile/tilepro.c +++ b/drivers/net/ethernet/tile/tilepro.c | |||
@@ -1008,6 +1008,8 @@ static void tile_net_register(void *dev_ptr) | |||
1008 | info->egress_timer.data = (long)info; | 1008 | info->egress_timer.data = (long)info; |
1009 | info->egress_timer.function = tile_net_handle_egress_timer; | 1009 | info->egress_timer.function = tile_net_handle_egress_timer; |
1010 | 1010 | ||
1011 | u64_stats_init(&info->stats.syncp); | ||
1012 | |||
1011 | priv->cpu[my_cpu] = info; | 1013 | priv->cpu[my_cpu] = info; |
1012 | 1014 | ||
1013 | /* | 1015 | /* |
diff --git a/drivers/net/ethernet/via/via-rhine.c b/drivers/net/ethernet/via/via-rhine.c index 4a7293ed95e9..cce6c4bc556a 100644 --- a/drivers/net/ethernet/via/via-rhine.c +++ b/drivers/net/ethernet/via/via-rhine.c | |||
@@ -987,6 +987,9 @@ static int rhine_init_one(struct pci_dev *pdev, const struct pci_device_id *ent) | |||
987 | 987 | ||
988 | rp->base = ioaddr; | 988 | rp->base = ioaddr; |
989 | 989 | ||
990 | u64_stats_init(&rp->tx_stats.syncp); | ||
991 | u64_stats_init(&rp->rx_stats.syncp); | ||
992 | |||
990 | /* Get chip registers into a sane state */ | 993 | /* Get chip registers into a sane state */ |
991 | rhine_power_init(dev); | 994 | rhine_power_init(dev); |
992 | rhine_hw_init(dev, pioaddr); | 995 | rhine_hw_init(dev, pioaddr); |
diff --git a/drivers/net/ifb.c b/drivers/net/ifb.c index a3bed28197d2..c14d39bf32d0 100644 --- a/drivers/net/ifb.c +++ b/drivers/net/ifb.c | |||
@@ -265,6 +265,7 @@ MODULE_PARM_DESC(numifbs, "Number of ifb devices"); | |||
265 | static int __init ifb_init_one(int index) | 265 | static int __init ifb_init_one(int index) |
266 | { | 266 | { |
267 | struct net_device *dev_ifb; | 267 | struct net_device *dev_ifb; |
268 | struct ifb_private *dp; | ||
268 | int err; | 269 | int err; |
269 | 270 | ||
270 | dev_ifb = alloc_netdev(sizeof(struct ifb_private), | 271 | dev_ifb = alloc_netdev(sizeof(struct ifb_private), |
@@ -273,6 +274,10 @@ static int __init ifb_init_one(int index) | |||
273 | if (!dev_ifb) | 274 | if (!dev_ifb) |
274 | return -ENOMEM; | 275 | return -ENOMEM; |
275 | 276 | ||
277 | dp = netdev_priv(dev_ifb); | ||
278 | u64_stats_init(&dp->rsync); | ||
279 | u64_stats_init(&dp->tsync); | ||
280 | |||
276 | dev_ifb->rtnl_link_ops = &ifb_link_ops; | 281 | dev_ifb->rtnl_link_ops = &ifb_link_ops; |
277 | err = register_netdevice(dev_ifb); | 282 | err = register_netdevice(dev_ifb); |
278 | if (err < 0) | 283 | if (err < 0) |
diff --git a/drivers/net/loopback.c b/drivers/net/loopback.c index a17d85a331f1..ac24c27b4b2d 100644 --- a/drivers/net/loopback.c +++ b/drivers/net/loopback.c | |||
@@ -137,10 +137,16 @@ static const struct ethtool_ops loopback_ethtool_ops = { | |||
137 | 137 | ||
138 | static int loopback_dev_init(struct net_device *dev) | 138 | static int loopback_dev_init(struct net_device *dev) |
139 | { | 139 | { |
140 | int i; | ||
140 | dev->lstats = alloc_percpu(struct pcpu_lstats); | 141 | dev->lstats = alloc_percpu(struct pcpu_lstats); |
141 | if (!dev->lstats) | 142 | if (!dev->lstats) |
142 | return -ENOMEM; | 143 | return -ENOMEM; |
143 | 144 | ||
145 | for_each_possible_cpu(i) { | ||
146 | struct pcpu_lstats *lb_stats; | ||
147 | lb_stats = per_cpu_ptr(dev->lstats, i); | ||
148 | u64_stats_init(&lb_stats->syncp); | ||
149 | } | ||
144 | return 0; | 150 | return 0; |
145 | } | 151 | } |
146 | 152 | ||
diff --git a/drivers/net/macvlan.c b/drivers/net/macvlan.c index af4aaa5893ff..acf93798dc67 100644 --- a/drivers/net/macvlan.c +++ b/drivers/net/macvlan.c | |||
@@ -534,6 +534,7 @@ static int macvlan_init(struct net_device *dev) | |||
534 | { | 534 | { |
535 | struct macvlan_dev *vlan = netdev_priv(dev); | 535 | struct macvlan_dev *vlan = netdev_priv(dev); |
536 | const struct net_device *lowerdev = vlan->lowerdev; | 536 | const struct net_device *lowerdev = vlan->lowerdev; |
537 | int i; | ||
537 | 538 | ||
538 | dev->state = (dev->state & ~MACVLAN_STATE_MASK) | | 539 | dev->state = (dev->state & ~MACVLAN_STATE_MASK) | |
539 | (lowerdev->state & MACVLAN_STATE_MASK); | 540 | (lowerdev->state & MACVLAN_STATE_MASK); |
@@ -549,6 +550,12 @@ static int macvlan_init(struct net_device *dev) | |||
549 | if (!vlan->pcpu_stats) | 550 | if (!vlan->pcpu_stats) |
550 | return -ENOMEM; | 551 | return -ENOMEM; |
551 | 552 | ||
553 | for_each_possible_cpu(i) { | ||
554 | struct macvlan_pcpu_stats *mvlstats; | ||
555 | mvlstats = per_cpu_ptr(vlan->pcpu_stats, i); | ||
556 | u64_stats_init(&mvlstats->syncp); | ||
557 | } | ||
558 | |||
552 | return 0; | 559 | return 0; |
553 | } | 560 | } |
554 | 561 | ||
diff --git a/drivers/net/nlmon.c b/drivers/net/nlmon.c index b57ce5f48962..d2bb12bfabd5 100644 --- a/drivers/net/nlmon.c +++ b/drivers/net/nlmon.c | |||
@@ -47,8 +47,16 @@ static int nlmon_change_mtu(struct net_device *dev, int new_mtu) | |||
47 | 47 | ||
48 | static int nlmon_dev_init(struct net_device *dev) | 48 | static int nlmon_dev_init(struct net_device *dev) |
49 | { | 49 | { |
50 | int i; | ||
51 | |||
50 | dev->lstats = alloc_percpu(struct pcpu_lstats); | 52 | dev->lstats = alloc_percpu(struct pcpu_lstats); |
51 | 53 | ||
54 | for_each_possible_cpu(i) { | ||
55 | struct pcpu_lstats *nlmstats; | ||
56 | nlmstats = per_cpu_ptr(dev->lstats, i); | ||
57 | u64_stats_init(&nlmstats->syncp); | ||
58 | } | ||
59 | |||
52 | return dev->lstats == NULL ? -ENOMEM : 0; | 60 | return dev->lstats == NULL ? -ENOMEM : 0; |
53 | } | 61 | } |
54 | 62 | ||
diff --git a/drivers/net/team/team.c b/drivers/net/team/team.c index 50e43e64d51d..6574eb8766f9 100644 --- a/drivers/net/team/team.c +++ b/drivers/net/team/team.c | |||
@@ -1540,6 +1540,12 @@ static int team_init(struct net_device *dev) | |||
1540 | if (!team->pcpu_stats) | 1540 | if (!team->pcpu_stats) |
1541 | return -ENOMEM; | 1541 | return -ENOMEM; |
1542 | 1542 | ||
1543 | for_each_possible_cpu(i) { | ||
1544 | struct team_pcpu_stats *team_stats; | ||
1545 | team_stats = per_cpu_ptr(team->pcpu_stats, i); | ||
1546 | u64_stats_init(&team_stats->syncp); | ||
1547 | } | ||
1548 | |||
1543 | for (i = 0; i < TEAM_PORT_HASHENTRIES; i++) | 1549 | for (i = 0; i < TEAM_PORT_HASHENTRIES; i++) |
1544 | INIT_HLIST_HEAD(&team->en_port_hlist[i]); | 1550 | INIT_HLIST_HEAD(&team->en_port_hlist[i]); |
1545 | INIT_LIST_HEAD(&team->port_list); | 1551 | INIT_LIST_HEAD(&team->port_list); |
diff --git a/drivers/net/team/team_mode_loadbalance.c b/drivers/net/team/team_mode_loadbalance.c index 829a9cd2b4da..d671fc3ac5ac 100644 --- a/drivers/net/team/team_mode_loadbalance.c +++ b/drivers/net/team/team_mode_loadbalance.c | |||
@@ -570,7 +570,7 @@ static int lb_init(struct team *team) | |||
570 | { | 570 | { |
571 | struct lb_priv *lb_priv = get_lb_priv(team); | 571 | struct lb_priv *lb_priv = get_lb_priv(team); |
572 | lb_select_tx_port_func_t *func; | 572 | lb_select_tx_port_func_t *func; |
573 | int err; | 573 | int i, err; |
574 | 574 | ||
575 | /* set default tx port selector */ | 575 | /* set default tx port selector */ |
576 | func = lb_select_tx_port_get_func("hash"); | 576 | func = lb_select_tx_port_get_func("hash"); |
@@ -588,6 +588,13 @@ static int lb_init(struct team *team) | |||
588 | goto err_alloc_pcpu_stats; | 588 | goto err_alloc_pcpu_stats; |
589 | } | 589 | } |
590 | 590 | ||
591 | for_each_possible_cpu(i) { | ||
592 | struct lb_pcpu_stats *team_lb_stats; | ||
593 | team_lb_stats = per_cpu_ptr(lb_priv->pcpu_stats, i); | ||
594 | u64_stats_init(&team_lb_stats->syncp); | ||
595 | } | ||
596 | |||
597 | |||
591 | INIT_DELAYED_WORK(&lb_priv->ex->stats.refresh_dw, lb_stats_refresh); | 598 | INIT_DELAYED_WORK(&lb_priv->ex->stats.refresh_dw, lb_stats_refresh); |
592 | 599 | ||
593 | err = team_options_register(team, lb_options, ARRAY_SIZE(lb_options)); | 600 | err = team_options_register(team, lb_options, ARRAY_SIZE(lb_options)); |
diff --git a/drivers/net/veth.c b/drivers/net/veth.c index b24db7acbf12..2ec2041b62d4 100644 --- a/drivers/net/veth.c +++ b/drivers/net/veth.c | |||
@@ -235,10 +235,18 @@ static int veth_change_mtu(struct net_device *dev, int new_mtu) | |||
235 | 235 | ||
236 | static int veth_dev_init(struct net_device *dev) | 236 | static int veth_dev_init(struct net_device *dev) |
237 | { | 237 | { |
238 | int i; | ||
239 | |||
238 | dev->vstats = alloc_percpu(struct pcpu_vstats); | 240 | dev->vstats = alloc_percpu(struct pcpu_vstats); |
239 | if (!dev->vstats) | 241 | if (!dev->vstats) |
240 | return -ENOMEM; | 242 | return -ENOMEM; |
241 | 243 | ||
244 | for_each_possible_cpu(i) { | ||
245 | struct pcpu_vstats *veth_stats; | ||
246 | veth_stats = per_cpu_ptr(dev->vstats, i); | ||
247 | u64_stats_init(&veth_stats->syncp); | ||
248 | } | ||
249 | |||
242 | return 0; | 250 | return 0; |
243 | } | 251 | } |
244 | 252 | ||
diff --git a/drivers/net/virtio_net.c b/drivers/net/virtio_net.c index 01f4eb5c8b78..bf7c734259ad 100644 --- a/drivers/net/virtio_net.c +++ b/drivers/net/virtio_net.c | |||
@@ -1576,6 +1576,13 @@ static int virtnet_probe(struct virtio_device *vdev) | |||
1576 | if (vi->stats == NULL) | 1576 | if (vi->stats == NULL) |
1577 | goto free; | 1577 | goto free; |
1578 | 1578 | ||
1579 | for_each_possible_cpu(i) { | ||
1580 | struct virtnet_stats *virtnet_stats; | ||
1581 | virtnet_stats = per_cpu_ptr(vi->stats, i); | ||
1582 | u64_stats_init(&virtnet_stats->tx_syncp); | ||
1583 | u64_stats_init(&virtnet_stats->rx_syncp); | ||
1584 | } | ||
1585 | |||
1579 | mutex_init(&vi->config_lock); | 1586 | mutex_init(&vi->config_lock); |
1580 | vi->config_enable = true; | 1587 | vi->config_enable = true; |
1581 | INIT_WORK(&vi->config_work, virtnet_config_changed_work); | 1588 | INIT_WORK(&vi->config_work, virtnet_config_changed_work); |
diff --git a/drivers/net/vxlan.c b/drivers/net/vxlan.c index 78df8f39e57c..0358c07f7669 100644 --- a/drivers/net/vxlan.c +++ b/drivers/net/vxlan.c | |||
@@ -1880,11 +1880,19 @@ static int vxlan_init(struct net_device *dev) | |||
1880 | struct vxlan_dev *vxlan = netdev_priv(dev); | 1880 | struct vxlan_dev *vxlan = netdev_priv(dev); |
1881 | struct vxlan_net *vn = net_generic(dev_net(dev), vxlan_net_id); | 1881 | struct vxlan_net *vn = net_generic(dev_net(dev), vxlan_net_id); |
1882 | struct vxlan_sock *vs; | 1882 | struct vxlan_sock *vs; |
1883 | int i; | ||
1883 | 1884 | ||
1884 | dev->tstats = alloc_percpu(struct pcpu_tstats); | 1885 | dev->tstats = alloc_percpu(struct pcpu_tstats); |
1885 | if (!dev->tstats) | 1886 | if (!dev->tstats) |
1886 | return -ENOMEM; | 1887 | return -ENOMEM; |
1887 | 1888 | ||
1889 | for_each_possible_cpu(i) { | ||
1890 | struct pcpu_tstats *vxlan_stats; | ||
1891 | vxlan_stats = per_cpu_ptr(dev->tstats, i); | ||
1892 | u64_stats_init(&vxlan_stats->syncp); | ||
1893 | } | ||
1894 | |||
1895 | |||
1888 | spin_lock(&vn->sock_lock); | 1896 | spin_lock(&vn->sock_lock); |
1889 | vs = vxlan_find_sock(dev_net(dev), vxlan->dst_port); | 1897 | vs = vxlan_find_sock(dev_net(dev), vxlan->dst_port); |
1890 | if (vs) { | 1898 | if (vs) { |
diff --git a/drivers/net/xen-netfront.c b/drivers/net/xen-netfront.c index dd1011e55cb5..d85e66979711 100644 --- a/drivers/net/xen-netfront.c +++ b/drivers/net/xen-netfront.c | |||
@@ -1340,6 +1340,12 @@ static struct net_device *xennet_create_dev(struct xenbus_device *dev) | |||
1340 | if (np->stats == NULL) | 1340 | if (np->stats == NULL) |
1341 | goto exit; | 1341 | goto exit; |
1342 | 1342 | ||
1343 | for_each_possible_cpu(i) { | ||
1344 | struct netfront_stats *xen_nf_stats; | ||
1345 | xen_nf_stats = per_cpu_ptr(np->stats, i); | ||
1346 | u64_stats_init(&xen_nf_stats->syncp); | ||
1347 | } | ||
1348 | |||
1343 | /* Initialise tx_skbs as a free chain containing every entry. */ | 1349 | /* Initialise tx_skbs as a free chain containing every entry. */ |
1344 | np->tx_skb_freelist = 0; | 1350 | np->tx_skb_freelist = 0; |
1345 | for (i = 0; i < NET_TX_RING_SIZE; i++) { | 1351 | for (i = 0; i < NET_TX_RING_SIZE; i++) { |
diff --git a/fs/dcache.c b/fs/dcache.c index a9dd384c5e80..0a38ef8d7f00 100644 --- a/fs/dcache.c +++ b/fs/dcache.c | |||
@@ -2606,7 +2606,7 @@ static void __d_move(struct dentry * dentry, struct dentry * target) | |||
2606 | dentry_lock_for_move(dentry, target); | 2606 | dentry_lock_for_move(dentry, target); |
2607 | 2607 | ||
2608 | write_seqcount_begin(&dentry->d_seq); | 2608 | write_seqcount_begin(&dentry->d_seq); |
2609 | write_seqcount_begin(&target->d_seq); | 2609 | write_seqcount_begin_nested(&target->d_seq, DENTRY_D_LOCK_NESTED); |
2610 | 2610 | ||
2611 | /* __d_drop does write_seqcount_barrier, but they're OK to nest. */ | 2611 | /* __d_drop does write_seqcount_barrier, but they're OK to nest. */ |
2612 | 2612 | ||
@@ -2738,7 +2738,7 @@ static void __d_materialise_dentry(struct dentry *dentry, struct dentry *anon) | |||
2738 | dentry_lock_for_move(anon, dentry); | 2738 | dentry_lock_for_move(anon, dentry); |
2739 | 2739 | ||
2740 | write_seqcount_begin(&dentry->d_seq); | 2740 | write_seqcount_begin(&dentry->d_seq); |
2741 | write_seqcount_begin(&anon->d_seq); | 2741 | write_seqcount_begin_nested(&anon->d_seq, DENTRY_D_LOCK_NESTED); |
2742 | 2742 | ||
2743 | dparent = dentry->d_parent; | 2743 | dparent = dentry->d_parent; |
2744 | 2744 | ||
diff --git a/fs/fs_struct.c b/fs/fs_struct.c index d8ac61d0c932..7dca743b2ce1 100644 --- a/fs/fs_struct.c +++ b/fs/fs_struct.c | |||
@@ -161,6 +161,6 @@ EXPORT_SYMBOL(current_umask); | |||
161 | struct fs_struct init_fs = { | 161 | struct fs_struct init_fs = { |
162 | .users = 1, | 162 | .users = 1, |
163 | .lock = __SPIN_LOCK_UNLOCKED(init_fs.lock), | 163 | .lock = __SPIN_LOCK_UNLOCKED(init_fs.lock), |
164 | .seq = SEQCNT_ZERO, | 164 | .seq = SEQCNT_ZERO(init_fs.seq), |
165 | .umask = 0022, | 165 | .umask = 0022, |
166 | }; | 166 | }; |
diff --git a/include/linux/cpuset.h b/include/linux/cpuset.h index cc1b01cf2035..3fe661fe96d1 100644 --- a/include/linux/cpuset.h +++ b/include/linux/cpuset.h | |||
@@ -110,10 +110,14 @@ static inline bool put_mems_allowed(unsigned int seq) | |||
110 | 110 | ||
111 | static inline void set_mems_allowed(nodemask_t nodemask) | 111 | static inline void set_mems_allowed(nodemask_t nodemask) |
112 | { | 112 | { |
113 | unsigned long flags; | ||
114 | |||
113 | task_lock(current); | 115 | task_lock(current); |
116 | local_irq_save(flags); | ||
114 | write_seqcount_begin(¤t->mems_allowed_seq); | 117 | write_seqcount_begin(¤t->mems_allowed_seq); |
115 | current->mems_allowed = nodemask; | 118 | current->mems_allowed = nodemask; |
116 | write_seqcount_end(¤t->mems_allowed_seq); | 119 | write_seqcount_end(¤t->mems_allowed_seq); |
120 | local_irq_restore(flags); | ||
117 | task_unlock(current); | 121 | task_unlock(current); |
118 | } | 122 | } |
119 | 123 | ||
diff --git a/include/linux/init_task.h b/include/linux/init_task.h index 5cd0f0949927..b0ed422e4e4a 100644 --- a/include/linux/init_task.h +++ b/include/linux/init_task.h | |||
@@ -32,10 +32,10 @@ extern struct fs_struct init_fs; | |||
32 | #endif | 32 | #endif |
33 | 33 | ||
34 | #ifdef CONFIG_CPUSETS | 34 | #ifdef CONFIG_CPUSETS |
35 | #define INIT_CPUSET_SEQ \ | 35 | #define INIT_CPUSET_SEQ(tsk) \ |
36 | .mems_allowed_seq = SEQCNT_ZERO, | 36 | .mems_allowed_seq = SEQCNT_ZERO(tsk.mems_allowed_seq), |
37 | #else | 37 | #else |
38 | #define INIT_CPUSET_SEQ | 38 | #define INIT_CPUSET_SEQ(tsk) |
39 | #endif | 39 | #endif |
40 | 40 | ||
41 | #define INIT_SIGNALS(sig) { \ | 41 | #define INIT_SIGNALS(sig) { \ |
@@ -220,7 +220,7 @@ extern struct task_group root_task_group; | |||
220 | INIT_FTRACE_GRAPH \ | 220 | INIT_FTRACE_GRAPH \ |
221 | INIT_TRACE_RECURSION \ | 221 | INIT_TRACE_RECURSION \ |
222 | INIT_TASK_RCU_PREEMPT(tsk) \ | 222 | INIT_TASK_RCU_PREEMPT(tsk) \ |
223 | INIT_CPUSET_SEQ \ | 223 | INIT_CPUSET_SEQ(tsk) \ |
224 | INIT_VTIME(tsk) \ | 224 | INIT_VTIME(tsk) \ |
225 | } | 225 | } |
226 | 226 | ||
diff --git a/include/linux/lockdep.h b/include/linux/lockdep.h index cfc2f119779a..92b1bfc5da60 100644 --- a/include/linux/lockdep.h +++ b/include/linux/lockdep.h | |||
@@ -497,6 +497,10 @@ static inline void print_irqtrace_events(struct task_struct *curr) | |||
497 | #define rwlock_acquire_read(l, s, t, i) lock_acquire_shared_recursive(l, s, t, NULL, i) | 497 | #define rwlock_acquire_read(l, s, t, i) lock_acquire_shared_recursive(l, s, t, NULL, i) |
498 | #define rwlock_release(l, n, i) lock_release(l, n, i) | 498 | #define rwlock_release(l, n, i) lock_release(l, n, i) |
499 | 499 | ||
500 | #define seqcount_acquire(l, s, t, i) lock_acquire_exclusive(l, s, t, NULL, i) | ||
501 | #define seqcount_acquire_read(l, s, t, i) lock_acquire_shared_recursive(l, s, t, NULL, i) | ||
502 | #define seqcount_release(l, n, i) lock_release(l, n, i) | ||
503 | |||
500 | #define mutex_acquire(l, s, t, i) lock_acquire_exclusive(l, s, t, NULL, i) | 504 | #define mutex_acquire(l, s, t, i) lock_acquire_exclusive(l, s, t, NULL, i) |
501 | #define mutex_acquire_nest(l, s, t, n, i) lock_acquire_exclusive(l, s, t, n, i) | 505 | #define mutex_acquire_nest(l, s, t, n, i) lock_acquire_exclusive(l, s, t, n, i) |
502 | #define mutex_release(l, n, i) lock_release(l, n, i) | 506 | #define mutex_release(l, n, i) lock_release(l, n, i) |
@@ -504,11 +508,11 @@ static inline void print_irqtrace_events(struct task_struct *curr) | |||
504 | #define rwsem_acquire(l, s, t, i) lock_acquire_exclusive(l, s, t, NULL, i) | 508 | #define rwsem_acquire(l, s, t, i) lock_acquire_exclusive(l, s, t, NULL, i) |
505 | #define rwsem_acquire_nest(l, s, t, n, i) lock_acquire_exclusive(l, s, t, n, i) | 509 | #define rwsem_acquire_nest(l, s, t, n, i) lock_acquire_exclusive(l, s, t, n, i) |
506 | #define rwsem_acquire_read(l, s, t, i) lock_acquire_shared(l, s, t, NULL, i) | 510 | #define rwsem_acquire_read(l, s, t, i) lock_acquire_shared(l, s, t, NULL, i) |
507 | # define rwsem_release(l, n, i) lock_release(l, n, i) | 511 | #define rwsem_release(l, n, i) lock_release(l, n, i) |
508 | 512 | ||
509 | #define lock_map_acquire(l) lock_acquire_exclusive(l, 0, 0, NULL, _THIS_IP_) | 513 | #define lock_map_acquire(l) lock_acquire_exclusive(l, 0, 0, NULL, _THIS_IP_) |
510 | #define lock_map_acquire_read(l) lock_acquire_shared_recursive(l, 0, 0, NULL, _THIS_IP_) | 514 | #define lock_map_acquire_read(l) lock_acquire_shared_recursive(l, 0, 0, NULL, _THIS_IP_) |
511 | # define lock_map_release(l) lock_release(l, 1, _THIS_IP_) | 515 | #define lock_map_release(l) lock_release(l, 1, _THIS_IP_) |
512 | 516 | ||
513 | #ifdef CONFIG_PROVE_LOCKING | 517 | #ifdef CONFIG_PROVE_LOCKING |
514 | # define might_lock(lock) \ | 518 | # define might_lock(lock) \ |
diff --git a/include/linux/mutex.h b/include/linux/mutex.h index bab49da8a0f0..d3181936c138 100644 --- a/include/linux/mutex.h +++ b/include/linux/mutex.h | |||
@@ -131,7 +131,7 @@ static inline int mutex_is_locked(struct mutex *lock) | |||
131 | } | 131 | } |
132 | 132 | ||
133 | /* | 133 | /* |
134 | * See kernel/mutex.c for detailed documentation of these APIs. | 134 | * See kernel/locking/mutex.c for detailed documentation of these APIs. |
135 | * Also see Documentation/mutex-design.txt. | 135 | * Also see Documentation/mutex-design.txt. |
136 | */ | 136 | */ |
137 | #ifdef CONFIG_DEBUG_LOCK_ALLOC | 137 | #ifdef CONFIG_DEBUG_LOCK_ALLOC |
diff --git a/include/linux/sched/sysctl.h b/include/linux/sched/sysctl.h index 10d16c4fbe89..41467f8ff8ec 100644 --- a/include/linux/sched/sysctl.h +++ b/include/linux/sched/sysctl.h | |||
@@ -2,8 +2,8 @@ | |||
2 | #define _SCHED_SYSCTL_H | 2 | #define _SCHED_SYSCTL_H |
3 | 3 | ||
4 | #ifdef CONFIG_DETECT_HUNG_TASK | 4 | #ifdef CONFIG_DETECT_HUNG_TASK |
5 | extern int sysctl_hung_task_check_count; | ||
5 | extern unsigned int sysctl_hung_task_panic; | 6 | extern unsigned int sysctl_hung_task_panic; |
6 | extern unsigned long sysctl_hung_task_check_count; | ||
7 | extern unsigned long sysctl_hung_task_timeout_secs; | 7 | extern unsigned long sysctl_hung_task_timeout_secs; |
8 | extern unsigned long sysctl_hung_task_warnings; | 8 | extern unsigned long sysctl_hung_task_warnings; |
9 | extern int proc_dohung_task_timeout_secs(struct ctl_table *table, int write, | 9 | extern int proc_dohung_task_timeout_secs(struct ctl_table *table, int write, |
diff --git a/include/linux/seqlock.h b/include/linux/seqlock.h index 21a209336e79..1e8a8b6e837d 100644 --- a/include/linux/seqlock.h +++ b/include/linux/seqlock.h | |||
@@ -34,6 +34,7 @@ | |||
34 | 34 | ||
35 | #include <linux/spinlock.h> | 35 | #include <linux/spinlock.h> |
36 | #include <linux/preempt.h> | 36 | #include <linux/preempt.h> |
37 | #include <linux/lockdep.h> | ||
37 | #include <asm/processor.h> | 38 | #include <asm/processor.h> |
38 | 39 | ||
39 | /* | 40 | /* |
@@ -44,10 +45,50 @@ | |||
44 | */ | 45 | */ |
45 | typedef struct seqcount { | 46 | typedef struct seqcount { |
46 | unsigned sequence; | 47 | unsigned sequence; |
48 | #ifdef CONFIG_DEBUG_LOCK_ALLOC | ||
49 | struct lockdep_map dep_map; | ||
50 | #endif | ||
47 | } seqcount_t; | 51 | } seqcount_t; |
48 | 52 | ||
49 | #define SEQCNT_ZERO { 0 } | 53 | static inline void __seqcount_init(seqcount_t *s, const char *name, |
50 | #define seqcount_init(x) do { *(x) = (seqcount_t) SEQCNT_ZERO; } while (0) | 54 | struct lock_class_key *key) |
55 | { | ||
56 | /* | ||
57 | * Make sure we are not reinitializing a held lock: | ||
58 | */ | ||
59 | lockdep_init_map(&s->dep_map, name, key, 0); | ||
60 | s->sequence = 0; | ||
61 | } | ||
62 | |||
63 | #ifdef CONFIG_DEBUG_LOCK_ALLOC | ||
64 | # define SEQCOUNT_DEP_MAP_INIT(lockname) \ | ||
65 | .dep_map = { .name = #lockname } \ | ||
66 | |||
67 | # define seqcount_init(s) \ | ||
68 | do { \ | ||
69 | static struct lock_class_key __key; \ | ||
70 | __seqcount_init((s), #s, &__key); \ | ||
71 | } while (0) | ||
72 | |||
73 | static inline void seqcount_lockdep_reader_access(const seqcount_t *s) | ||
74 | { | ||
75 | seqcount_t *l = (seqcount_t *)s; | ||
76 | unsigned long flags; | ||
77 | |||
78 | local_irq_save(flags); | ||
79 | seqcount_acquire_read(&l->dep_map, 0, 0, _RET_IP_); | ||
80 | seqcount_release(&l->dep_map, 1, _RET_IP_); | ||
81 | local_irq_restore(flags); | ||
82 | } | ||
83 | |||
84 | #else | ||
85 | # define SEQCOUNT_DEP_MAP_INIT(lockname) | ||
86 | # define seqcount_init(s) __seqcount_init(s, NULL, NULL) | ||
87 | # define seqcount_lockdep_reader_access(x) | ||
88 | #endif | ||
89 | |||
90 | #define SEQCNT_ZERO(lockname) { .sequence = 0, SEQCOUNT_DEP_MAP_INIT(lockname)} | ||
91 | |||
51 | 92 | ||
52 | /** | 93 | /** |
53 | * __read_seqcount_begin - begin a seq-read critical section (without barrier) | 94 | * __read_seqcount_begin - begin a seq-read critical section (without barrier) |
@@ -76,6 +117,22 @@ repeat: | |||
76 | } | 117 | } |
77 | 118 | ||
78 | /** | 119 | /** |
120 | * read_seqcount_begin_no_lockdep - start seq-read critical section w/o lockdep | ||
121 | * @s: pointer to seqcount_t | ||
122 | * Returns: count to be passed to read_seqcount_retry | ||
123 | * | ||
124 | * read_seqcount_begin_no_lockdep opens a read critical section of the given | ||
125 | * seqcount, but without any lockdep checking. Validity of the critical | ||
126 | * section is tested by checking read_seqcount_retry function. | ||
127 | */ | ||
128 | static inline unsigned read_seqcount_begin_no_lockdep(const seqcount_t *s) | ||
129 | { | ||
130 | unsigned ret = __read_seqcount_begin(s); | ||
131 | smp_rmb(); | ||
132 | return ret; | ||
133 | } | ||
134 | |||
135 | /** | ||
79 | * read_seqcount_begin - begin a seq-read critical section | 136 | * read_seqcount_begin - begin a seq-read critical section |
80 | * @s: pointer to seqcount_t | 137 | * @s: pointer to seqcount_t |
81 | * Returns: count to be passed to read_seqcount_retry | 138 | * Returns: count to be passed to read_seqcount_retry |
@@ -86,9 +143,8 @@ repeat: | |||
86 | */ | 143 | */ |
87 | static inline unsigned read_seqcount_begin(const seqcount_t *s) | 144 | static inline unsigned read_seqcount_begin(const seqcount_t *s) |
88 | { | 145 | { |
89 | unsigned ret = __read_seqcount_begin(s); | 146 | seqcount_lockdep_reader_access(s); |
90 | smp_rmb(); | 147 | return read_seqcount_begin_no_lockdep(s); |
91 | return ret; | ||
92 | } | 148 | } |
93 | 149 | ||
94 | /** | 150 | /** |
@@ -108,6 +164,8 @@ static inline unsigned read_seqcount_begin(const seqcount_t *s) | |||
108 | static inline unsigned raw_seqcount_begin(const seqcount_t *s) | 164 | static inline unsigned raw_seqcount_begin(const seqcount_t *s) |
109 | { | 165 | { |
110 | unsigned ret = ACCESS_ONCE(s->sequence); | 166 | unsigned ret = ACCESS_ONCE(s->sequence); |
167 | |||
168 | seqcount_lockdep_reader_access(s); | ||
111 | smp_rmb(); | 169 | smp_rmb(); |
112 | return ret & ~1; | 170 | return ret & ~1; |
113 | } | 171 | } |
@@ -152,14 +210,21 @@ static inline int read_seqcount_retry(const seqcount_t *s, unsigned start) | |||
152 | * Sequence counter only version assumes that callers are using their | 210 | * Sequence counter only version assumes that callers are using their |
153 | * own mutexing. | 211 | * own mutexing. |
154 | */ | 212 | */ |
155 | static inline void write_seqcount_begin(seqcount_t *s) | 213 | static inline void write_seqcount_begin_nested(seqcount_t *s, int subclass) |
156 | { | 214 | { |
157 | s->sequence++; | 215 | s->sequence++; |
158 | smp_wmb(); | 216 | smp_wmb(); |
217 | seqcount_acquire(&s->dep_map, subclass, 0, _RET_IP_); | ||
218 | } | ||
219 | |||
220 | static inline void write_seqcount_begin(seqcount_t *s) | ||
221 | { | ||
222 | write_seqcount_begin_nested(s, 0); | ||
159 | } | 223 | } |
160 | 224 | ||
161 | static inline void write_seqcount_end(seqcount_t *s) | 225 | static inline void write_seqcount_end(seqcount_t *s) |
162 | { | 226 | { |
227 | seqcount_release(&s->dep_map, 1, _RET_IP_); | ||
163 | smp_wmb(); | 228 | smp_wmb(); |
164 | s->sequence++; | 229 | s->sequence++; |
165 | } | 230 | } |
@@ -188,7 +253,7 @@ typedef struct { | |||
188 | */ | 253 | */ |
189 | #define __SEQLOCK_UNLOCKED(lockname) \ | 254 | #define __SEQLOCK_UNLOCKED(lockname) \ |
190 | { \ | 255 | { \ |
191 | .seqcount = SEQCNT_ZERO, \ | 256 | .seqcount = SEQCNT_ZERO(lockname), \ |
192 | .lock = __SPIN_LOCK_UNLOCKED(lockname) \ | 257 | .lock = __SPIN_LOCK_UNLOCKED(lockname) \ |
193 | } | 258 | } |
194 | 259 | ||
diff --git a/include/linux/u64_stats_sync.h b/include/linux/u64_stats_sync.h index 8da8c4e87da3..7bfabd20204c 100644 --- a/include/linux/u64_stats_sync.h +++ b/include/linux/u64_stats_sync.h | |||
@@ -67,6 +67,13 @@ struct u64_stats_sync { | |||
67 | #endif | 67 | #endif |
68 | }; | 68 | }; |
69 | 69 | ||
70 | |||
71 | #if BITS_PER_LONG == 32 && defined(CONFIG_SMP) | ||
72 | # define u64_stats_init(syncp) seqcount_init(syncp.seq) | ||
73 | #else | ||
74 | # define u64_stats_init(syncp) do { } while (0) | ||
75 | #endif | ||
76 | |||
70 | static inline void u64_stats_update_begin(struct u64_stats_sync *syncp) | 77 | static inline void u64_stats_update_begin(struct u64_stats_sync *syncp) |
71 | { | 78 | { |
72 | #if BITS_PER_LONG==32 && defined(CONFIG_SMP) | 79 | #if BITS_PER_LONG==32 && defined(CONFIG_SMP) |
diff --git a/include/trace/events/sched.h b/include/trace/events/sched.h index 613381bcde40..04c308413a5d 100644 --- a/include/trace/events/sched.h +++ b/include/trace/events/sched.h | |||
@@ -424,6 +424,25 @@ TRACE_EVENT(sched_pi_setprio, | |||
424 | __entry->oldprio, __entry->newprio) | 424 | __entry->oldprio, __entry->newprio) |
425 | ); | 425 | ); |
426 | 426 | ||
427 | #ifdef CONFIG_DETECT_HUNG_TASK | ||
428 | TRACE_EVENT(sched_process_hang, | ||
429 | TP_PROTO(struct task_struct *tsk), | ||
430 | TP_ARGS(tsk), | ||
431 | |||
432 | TP_STRUCT__entry( | ||
433 | __array( char, comm, TASK_COMM_LEN ) | ||
434 | __field( pid_t, pid ) | ||
435 | ), | ||
436 | |||
437 | TP_fast_assign( | ||
438 | memcpy(__entry->comm, tsk->comm, TASK_COMM_LEN); | ||
439 | __entry->pid = tsk->pid; | ||
440 | ), | ||
441 | |||
442 | TP_printk("comm=%s pid=%d", __entry->comm, __entry->pid) | ||
443 | ); | ||
444 | #endif /* CONFIG_DETECT_HUNG_TASK */ | ||
445 | |||
427 | #endif /* _TRACE_SCHED_H */ | 446 | #endif /* _TRACE_SCHED_H */ |
428 | 447 | ||
429 | /* This part must be outside protection */ | 448 | /* This part must be outside protection */ |
diff --git a/kernel/Makefile b/kernel/Makefile index a4d1aa8da9bc..09a9c94f42bd 100644 --- a/kernel/Makefile +++ b/kernel/Makefile | |||
@@ -7,22 +7,19 @@ obj-y = fork.o exec_domain.o panic.o \ | |||
7 | sysctl.o sysctl_binary.o capability.o ptrace.o timer.o user.o \ | 7 | sysctl.o sysctl_binary.o capability.o ptrace.o timer.o user.o \ |
8 | signal.o sys.o kmod.o workqueue.o pid.o task_work.o \ | 8 | signal.o sys.o kmod.o workqueue.o pid.o task_work.o \ |
9 | extable.o params.o posix-timers.o \ | 9 | extable.o params.o posix-timers.o \ |
10 | kthread.o sys_ni.o posix-cpu-timers.o mutex.o \ | 10 | kthread.o sys_ni.o posix-cpu-timers.o \ |
11 | hrtimer.o rwsem.o nsproxy.o semaphore.o \ | 11 | hrtimer.o nsproxy.o \ |
12 | notifier.o ksysfs.o cred.o reboot.o \ | 12 | notifier.o ksysfs.o cred.o reboot.o \ |
13 | async.o range.o groups.o lglock.o smpboot.o | 13 | async.o range.o groups.o smpboot.o |
14 | 14 | ||
15 | ifdef CONFIG_FUNCTION_TRACER | 15 | ifdef CONFIG_FUNCTION_TRACER |
16 | # Do not trace debug files and internal ftrace files | 16 | # Do not trace debug files and internal ftrace files |
17 | CFLAGS_REMOVE_lockdep.o = -pg | ||
18 | CFLAGS_REMOVE_lockdep_proc.o = -pg | ||
19 | CFLAGS_REMOVE_mutex-debug.o = -pg | ||
20 | CFLAGS_REMOVE_rtmutex-debug.o = -pg | ||
21 | CFLAGS_REMOVE_cgroup-debug.o = -pg | 17 | CFLAGS_REMOVE_cgroup-debug.o = -pg |
22 | CFLAGS_REMOVE_irq_work.o = -pg | 18 | CFLAGS_REMOVE_irq_work.o = -pg |
23 | endif | 19 | endif |
24 | 20 | ||
25 | obj-y += sched/ | 21 | obj-y += sched/ |
22 | obj-y += locking/ | ||
26 | obj-y += power/ | 23 | obj-y += power/ |
27 | obj-y += printk/ | 24 | obj-y += printk/ |
28 | obj-y += cpu/ | 25 | obj-y += cpu/ |
@@ -34,26 +31,15 @@ obj-$(CONFIG_FREEZER) += freezer.o | |||
34 | obj-$(CONFIG_PROFILING) += profile.o | 31 | obj-$(CONFIG_PROFILING) += profile.o |
35 | obj-$(CONFIG_STACKTRACE) += stacktrace.o | 32 | obj-$(CONFIG_STACKTRACE) += stacktrace.o |
36 | obj-y += time/ | 33 | obj-y += time/ |
37 | obj-$(CONFIG_DEBUG_MUTEXES) += mutex-debug.o | ||
38 | obj-$(CONFIG_LOCKDEP) += lockdep.o | ||
39 | ifeq ($(CONFIG_PROC_FS),y) | ||
40 | obj-$(CONFIG_LOCKDEP) += lockdep_proc.o | ||
41 | endif | ||
42 | obj-$(CONFIG_FUTEX) += futex.o | 34 | obj-$(CONFIG_FUTEX) += futex.o |
43 | ifeq ($(CONFIG_COMPAT),y) | 35 | ifeq ($(CONFIG_COMPAT),y) |
44 | obj-$(CONFIG_FUTEX) += futex_compat.o | 36 | obj-$(CONFIG_FUTEX) += futex_compat.o |
45 | endif | 37 | endif |
46 | obj-$(CONFIG_RT_MUTEXES) += rtmutex.o | ||
47 | obj-$(CONFIG_DEBUG_RT_MUTEXES) += rtmutex-debug.o | ||
48 | obj-$(CONFIG_RT_MUTEX_TESTER) += rtmutex-tester.o | ||
49 | obj-$(CONFIG_GENERIC_ISA_DMA) += dma.o | 38 | obj-$(CONFIG_GENERIC_ISA_DMA) += dma.o |
50 | obj-$(CONFIG_SMP) += smp.o | 39 | obj-$(CONFIG_SMP) += smp.o |
51 | ifneq ($(CONFIG_SMP),y) | 40 | ifneq ($(CONFIG_SMP),y) |
52 | obj-y += up.o | 41 | obj-y += up.o |
53 | endif | 42 | endif |
54 | obj-$(CONFIG_SMP) += spinlock.o | ||
55 | obj-$(CONFIG_DEBUG_SPINLOCK) += spinlock.o | ||
56 | obj-$(CONFIG_PROVE_LOCKING) += spinlock.o | ||
57 | obj-$(CONFIG_UID16) += uid16.o | 43 | obj-$(CONFIG_UID16) += uid16.o |
58 | obj-$(CONFIG_MODULES) += module.o | 44 | obj-$(CONFIG_MODULES) += module.o |
59 | obj-$(CONFIG_MODULE_SIG) += module_signing.o modsign_pubkey.o modsign_certificate.o | 45 | obj-$(CONFIG_MODULE_SIG) += module_signing.o modsign_pubkey.o modsign_certificate.o |
diff --git a/kernel/futex.c b/kernel/futex.c index c3a1a55a5214..80ba086f021d 100644 --- a/kernel/futex.c +++ b/kernel/futex.c | |||
@@ -66,7 +66,7 @@ | |||
66 | 66 | ||
67 | #include <asm/futex.h> | 67 | #include <asm/futex.h> |
68 | 68 | ||
69 | #include "rtmutex_common.h" | 69 | #include "locking/rtmutex_common.h" |
70 | 70 | ||
71 | int __read_mostly futex_cmpxchg_enabled; | 71 | int __read_mostly futex_cmpxchg_enabled; |
72 | 72 | ||
diff --git a/kernel/hung_task.c b/kernel/hung_task.c index 3e97fb126e6b..8807061ca004 100644 --- a/kernel/hung_task.c +++ b/kernel/hung_task.c | |||
@@ -16,11 +16,12 @@ | |||
16 | #include <linux/export.h> | 16 | #include <linux/export.h> |
17 | #include <linux/sysctl.h> | 17 | #include <linux/sysctl.h> |
18 | #include <linux/utsname.h> | 18 | #include <linux/utsname.h> |
19 | #include <trace/events/sched.h> | ||
19 | 20 | ||
20 | /* | 21 | /* |
21 | * The number of tasks checked: | 22 | * The number of tasks checked: |
22 | */ | 23 | */ |
23 | unsigned long __read_mostly sysctl_hung_task_check_count = PID_MAX_LIMIT; | 24 | int __read_mostly sysctl_hung_task_check_count = PID_MAX_LIMIT; |
24 | 25 | ||
25 | /* | 26 | /* |
26 | * Limit number of tasks checked in a batch. | 27 | * Limit number of tasks checked in a batch. |
@@ -92,6 +93,9 @@ static void check_hung_task(struct task_struct *t, unsigned long timeout) | |||
92 | t->last_switch_count = switch_count; | 93 | t->last_switch_count = switch_count; |
93 | return; | 94 | return; |
94 | } | 95 | } |
96 | |||
97 | trace_sched_process_hang(t); | ||
98 | |||
95 | if (!sysctl_hung_task_warnings) | 99 | if (!sysctl_hung_task_warnings) |
96 | return; | 100 | return; |
97 | sysctl_hung_task_warnings--; | 101 | sysctl_hung_task_warnings--; |
diff --git a/kernel/locking/Makefile b/kernel/locking/Makefile new file mode 100644 index 000000000000..baab8e5e7f66 --- /dev/null +++ b/kernel/locking/Makefile | |||
@@ -0,0 +1,25 @@ | |||
1 | |||
2 | obj-y += mutex.o semaphore.o rwsem.o lglock.o | ||
3 | |||
4 | ifdef CONFIG_FUNCTION_TRACER | ||
5 | CFLAGS_REMOVE_lockdep.o = -pg | ||
6 | CFLAGS_REMOVE_lockdep_proc.o = -pg | ||
7 | CFLAGS_REMOVE_mutex-debug.o = -pg | ||
8 | CFLAGS_REMOVE_rtmutex-debug.o = -pg | ||
9 | endif | ||
10 | |||
11 | obj-$(CONFIG_DEBUG_MUTEXES) += mutex-debug.o | ||
12 | obj-$(CONFIG_LOCKDEP) += lockdep.o | ||
13 | ifeq ($(CONFIG_PROC_FS),y) | ||
14 | obj-$(CONFIG_LOCKDEP) += lockdep_proc.o | ||
15 | endif | ||
16 | obj-$(CONFIG_SMP) += spinlock.o | ||
17 | obj-$(CONFIG_PROVE_LOCKING) += spinlock.o | ||
18 | obj-$(CONFIG_RT_MUTEXES) += rtmutex.o | ||
19 | obj-$(CONFIG_DEBUG_RT_MUTEXES) += rtmutex-debug.o | ||
20 | obj-$(CONFIG_RT_MUTEX_TESTER) += rtmutex-tester.o | ||
21 | obj-$(CONFIG_DEBUG_SPINLOCK) += spinlock.o | ||
22 | obj-$(CONFIG_DEBUG_SPINLOCK) += spinlock_debug.o | ||
23 | obj-$(CONFIG_RWSEM_GENERIC_SPINLOCK) += rwsem-spinlock.o | ||
24 | obj-$(CONFIG_RWSEM_XCHGADD_ALGORITHM) += rwsem-xadd.o | ||
25 | obj-$(CONFIG_PERCPU_RWSEM) += percpu-rwsem.o | ||
diff --git a/kernel/lglock.c b/kernel/locking/lglock.c index 86ae2aebf004..86ae2aebf004 100644 --- a/kernel/lglock.c +++ b/kernel/locking/lglock.c | |||
diff --git a/kernel/lockdep.c b/kernel/locking/lockdep.c index 4e8e14c34e42..576ba756a32d 100644 --- a/kernel/lockdep.c +++ b/kernel/locking/lockdep.c | |||
@@ -1232,7 +1232,7 @@ static int noop_count(struct lock_list *entry, void *data) | |||
1232 | return 0; | 1232 | return 0; |
1233 | } | 1233 | } |
1234 | 1234 | ||
1235 | unsigned long __lockdep_count_forward_deps(struct lock_list *this) | 1235 | static unsigned long __lockdep_count_forward_deps(struct lock_list *this) |
1236 | { | 1236 | { |
1237 | unsigned long count = 0; | 1237 | unsigned long count = 0; |
1238 | struct lock_list *uninitialized_var(target_entry); | 1238 | struct lock_list *uninitialized_var(target_entry); |
@@ -1258,7 +1258,7 @@ unsigned long lockdep_count_forward_deps(struct lock_class *class) | |||
1258 | return ret; | 1258 | return ret; |
1259 | } | 1259 | } |
1260 | 1260 | ||
1261 | unsigned long __lockdep_count_backward_deps(struct lock_list *this) | 1261 | static unsigned long __lockdep_count_backward_deps(struct lock_list *this) |
1262 | { | 1262 | { |
1263 | unsigned long count = 0; | 1263 | unsigned long count = 0; |
1264 | struct lock_list *uninitialized_var(target_entry); | 1264 | struct lock_list *uninitialized_var(target_entry); |
diff --git a/kernel/lockdep_internals.h b/kernel/locking/lockdep_internals.h index 4f560cfedc8f..4f560cfedc8f 100644 --- a/kernel/lockdep_internals.h +++ b/kernel/locking/lockdep_internals.h | |||
diff --git a/kernel/lockdep_proc.c b/kernel/locking/lockdep_proc.c index b2c71c5873e4..ef43ac4bafb5 100644 --- a/kernel/lockdep_proc.c +++ b/kernel/locking/lockdep_proc.c | |||
@@ -421,6 +421,7 @@ static void seq_lock_time(struct seq_file *m, struct lock_time *lt) | |||
421 | seq_time(m, lt->min); | 421 | seq_time(m, lt->min); |
422 | seq_time(m, lt->max); | 422 | seq_time(m, lt->max); |
423 | seq_time(m, lt->total); | 423 | seq_time(m, lt->total); |
424 | seq_time(m, lt->nr ? div_s64(lt->total, lt->nr) : 0); | ||
424 | } | 425 | } |
425 | 426 | ||
426 | static void seq_stats(struct seq_file *m, struct lock_stat_data *data) | 427 | static void seq_stats(struct seq_file *m, struct lock_stat_data *data) |
@@ -518,20 +519,20 @@ static void seq_stats(struct seq_file *m, struct lock_stat_data *data) | |||
518 | } | 519 | } |
519 | if (i) { | 520 | if (i) { |
520 | seq_puts(m, "\n"); | 521 | seq_puts(m, "\n"); |
521 | seq_line(m, '.', 0, 40 + 1 + 10 * (14 + 1)); | 522 | seq_line(m, '.', 0, 40 + 1 + 12 * (14 + 1)); |
522 | seq_puts(m, "\n"); | 523 | seq_puts(m, "\n"); |
523 | } | 524 | } |
524 | } | 525 | } |
525 | 526 | ||
526 | static void seq_header(struct seq_file *m) | 527 | static void seq_header(struct seq_file *m) |
527 | { | 528 | { |
528 | seq_printf(m, "lock_stat version 0.3\n"); | 529 | seq_puts(m, "lock_stat version 0.4\n"); |
529 | 530 | ||
530 | if (unlikely(!debug_locks)) | 531 | if (unlikely(!debug_locks)) |
531 | seq_printf(m, "*WARNING* lock debugging disabled!! - possibly due to a lockdep warning\n"); | 532 | seq_printf(m, "*WARNING* lock debugging disabled!! - possibly due to a lockdep warning\n"); |
532 | 533 | ||
533 | seq_line(m, '-', 0, 40 + 1 + 10 * (14 + 1)); | 534 | seq_line(m, '-', 0, 40 + 1 + 12 * (14 + 1)); |
534 | seq_printf(m, "%40s %14s %14s %14s %14s %14s %14s %14s %14s " | 535 | seq_printf(m, "%40s %14s %14s %14s %14s %14s %14s %14s %14s %14s %14s " |
535 | "%14s %14s\n", | 536 | "%14s %14s\n", |
536 | "class name", | 537 | "class name", |
537 | "con-bounces", | 538 | "con-bounces", |
@@ -539,12 +540,14 @@ static void seq_header(struct seq_file *m) | |||
539 | "waittime-min", | 540 | "waittime-min", |
540 | "waittime-max", | 541 | "waittime-max", |
541 | "waittime-total", | 542 | "waittime-total", |
543 | "waittime-avg", | ||
542 | "acq-bounces", | 544 | "acq-bounces", |
543 | "acquisitions", | 545 | "acquisitions", |
544 | "holdtime-min", | 546 | "holdtime-min", |
545 | "holdtime-max", | 547 | "holdtime-max", |
546 | "holdtime-total"); | 548 | "holdtime-total", |
547 | seq_line(m, '-', 0, 40 + 1 + 10 * (14 + 1)); | 549 | "holdtime-avg"); |
550 | seq_line(m, '-', 0, 40 + 1 + 12 * (14 + 1)); | ||
548 | seq_printf(m, "\n"); | 551 | seq_printf(m, "\n"); |
549 | } | 552 | } |
550 | 553 | ||
diff --git a/kernel/lockdep_states.h b/kernel/locking/lockdep_states.h index 995b0cc2b84c..995b0cc2b84c 100644 --- a/kernel/lockdep_states.h +++ b/kernel/locking/lockdep_states.h | |||
diff --git a/kernel/mutex-debug.c b/kernel/locking/mutex-debug.c index 7e3443fe1f48..7e3443fe1f48 100644 --- a/kernel/mutex-debug.c +++ b/kernel/locking/mutex-debug.c | |||
diff --git a/kernel/mutex-debug.h b/kernel/locking/mutex-debug.h index 0799fd3e4cfa..0799fd3e4cfa 100644 --- a/kernel/mutex-debug.h +++ b/kernel/locking/mutex-debug.h | |||
diff --git a/kernel/mutex.c b/kernel/locking/mutex.c index d24105b1b794..4dd6e4c219de 100644 --- a/kernel/mutex.c +++ b/kernel/locking/mutex.c | |||
@@ -1,5 +1,5 @@ | |||
1 | /* | 1 | /* |
2 | * kernel/mutex.c | 2 | * kernel/locking/mutex.c |
3 | * | 3 | * |
4 | * Mutexes: blocking mutual exclusion locks | 4 | * Mutexes: blocking mutual exclusion locks |
5 | * | 5 | * |
diff --git a/kernel/mutex.h b/kernel/locking/mutex.h index 4115fbf83b12..4115fbf83b12 100644 --- a/kernel/mutex.h +++ b/kernel/locking/mutex.h | |||
diff --git a/lib/percpu-rwsem.c b/kernel/locking/percpu-rwsem.c index 652a8ee8efe9..652a8ee8efe9 100644 --- a/lib/percpu-rwsem.c +++ b/kernel/locking/percpu-rwsem.c | |||
diff --git a/kernel/rtmutex-debug.c b/kernel/locking/rtmutex-debug.c index 13b243a323fa..13b243a323fa 100644 --- a/kernel/rtmutex-debug.c +++ b/kernel/locking/rtmutex-debug.c | |||
diff --git a/kernel/rtmutex-debug.h b/kernel/locking/rtmutex-debug.h index 14193d596d78..14193d596d78 100644 --- a/kernel/rtmutex-debug.h +++ b/kernel/locking/rtmutex-debug.h | |||
diff --git a/kernel/rtmutex-tester.c b/kernel/locking/rtmutex-tester.c index 1d96dd0d93c1..1d96dd0d93c1 100644 --- a/kernel/rtmutex-tester.c +++ b/kernel/locking/rtmutex-tester.c | |||
diff --git a/kernel/rtmutex.c b/kernel/locking/rtmutex.c index 0dd6aec1cb6a..0dd6aec1cb6a 100644 --- a/kernel/rtmutex.c +++ b/kernel/locking/rtmutex.c | |||
diff --git a/kernel/rtmutex.h b/kernel/locking/rtmutex.h index a1a1dd06421d..a1a1dd06421d 100644 --- a/kernel/rtmutex.h +++ b/kernel/locking/rtmutex.h | |||
diff --git a/kernel/rtmutex_common.h b/kernel/locking/rtmutex_common.h index 53a66c85261b..53a66c85261b 100644 --- a/kernel/rtmutex_common.h +++ b/kernel/locking/rtmutex_common.h | |||
diff --git a/lib/rwsem-spinlock.c b/kernel/locking/rwsem-spinlock.c index 9be8a9144978..9be8a9144978 100644 --- a/lib/rwsem-spinlock.c +++ b/kernel/locking/rwsem-spinlock.c | |||
diff --git a/lib/rwsem.c b/kernel/locking/rwsem-xadd.c index 19c5fa95e0b4..19c5fa95e0b4 100644 --- a/lib/rwsem.c +++ b/kernel/locking/rwsem-xadd.c | |||
diff --git a/kernel/rwsem.c b/kernel/locking/rwsem.c index cfff1435bdfb..cfff1435bdfb 100644 --- a/kernel/rwsem.c +++ b/kernel/locking/rwsem.c | |||
diff --git a/kernel/semaphore.c b/kernel/locking/semaphore.c index 6815171a4fff..6815171a4fff 100644 --- a/kernel/semaphore.c +++ b/kernel/locking/semaphore.c | |||
diff --git a/kernel/spinlock.c b/kernel/locking/spinlock.c index 4b082b5cac9e..4b082b5cac9e 100644 --- a/kernel/spinlock.c +++ b/kernel/locking/spinlock.c | |||
diff --git a/lib/spinlock_debug.c b/kernel/locking/spinlock_debug.c index 0374a596cffa..0374a596cffa 100644 --- a/lib/spinlock_debug.c +++ b/kernel/locking/spinlock_debug.c | |||
diff --git a/kernel/rcu/tree_plugin.h b/kernel/rcu/tree_plugin.h index 3822ac0c4b27..6abb03dff5c0 100644 --- a/kernel/rcu/tree_plugin.h +++ b/kernel/rcu/tree_plugin.h | |||
@@ -1133,7 +1133,7 @@ void exit_rcu(void) | |||
1133 | 1133 | ||
1134 | #ifdef CONFIG_RCU_BOOST | 1134 | #ifdef CONFIG_RCU_BOOST |
1135 | 1135 | ||
1136 | #include "../rtmutex_common.h" | 1136 | #include "../locking/rtmutex_common.h" |
1137 | 1137 | ||
1138 | #ifdef CONFIG_RCU_TRACE | 1138 | #ifdef CONFIG_RCU_TRACE |
1139 | 1139 | ||
diff --git a/kernel/sysctl.c b/kernel/sysctl.c index d37d9dd8f463..34a604726d0b 100644 --- a/kernel/sysctl.c +++ b/kernel/sysctl.c | |||
@@ -969,9 +969,10 @@ static struct ctl_table kern_table[] = { | |||
969 | { | 969 | { |
970 | .procname = "hung_task_check_count", | 970 | .procname = "hung_task_check_count", |
971 | .data = &sysctl_hung_task_check_count, | 971 | .data = &sysctl_hung_task_check_count, |
972 | .maxlen = sizeof(unsigned long), | 972 | .maxlen = sizeof(int), |
973 | .mode = 0644, | 973 | .mode = 0644, |
974 | .proc_handler = proc_doulongvec_minmax, | 974 | .proc_handler = proc_dointvec_minmax, |
975 | .extra1 = &zero, | ||
975 | }, | 976 | }, |
976 | { | 977 | { |
977 | .procname = "hung_task_timeout_secs", | 978 | .procname = "hung_task_timeout_secs", |
diff --git a/lib/Makefile b/lib/Makefile index bb016e116ba4..d480a8c92385 100644 --- a/lib/Makefile +++ b/lib/Makefile | |||
@@ -42,10 +42,6 @@ obj-$(CONFIG_GENERIC_PCI_IOMAP) += pci_iomap.o | |||
42 | obj-$(CONFIG_HAS_IOMEM) += iomap_copy.o devres.o | 42 | obj-$(CONFIG_HAS_IOMEM) += iomap_copy.o devres.o |
43 | obj-$(CONFIG_CHECK_SIGNATURE) += check_signature.o | 43 | obj-$(CONFIG_CHECK_SIGNATURE) += check_signature.o |
44 | obj-$(CONFIG_DEBUG_LOCKING_API_SELFTESTS) += locking-selftest.o | 44 | obj-$(CONFIG_DEBUG_LOCKING_API_SELFTESTS) += locking-selftest.o |
45 | obj-$(CONFIG_DEBUG_SPINLOCK) += spinlock_debug.o | ||
46 | lib-$(CONFIG_RWSEM_GENERIC_SPINLOCK) += rwsem-spinlock.o | ||
47 | lib-$(CONFIG_RWSEM_XCHGADD_ALGORITHM) += rwsem.o | ||
48 | lib-$(CONFIG_PERCPU_RWSEM) += percpu-rwsem.o | ||
49 | 45 | ||
50 | CFLAGS_hweight.o = $(subst $(quote),,$(CONFIG_ARCH_HWEIGHT_CFLAGS)) | 46 | CFLAGS_hweight.o = $(subst $(quote),,$(CONFIG_ARCH_HWEIGHT_CFLAGS)) |
51 | obj-$(CONFIG_GENERIC_HWEIGHT) += hweight.o | 47 | obj-$(CONFIG_GENERIC_HWEIGHT) += hweight.o |
diff --git a/mm/filemap_xip.c b/mm/filemap_xip.c index 28fe26b64f8a..d8d9fe3f685c 100644 --- a/mm/filemap_xip.c +++ b/mm/filemap_xip.c | |||
@@ -26,7 +26,7 @@ | |||
26 | * of ZERO_PAGE(), such as /dev/zero | 26 | * of ZERO_PAGE(), such as /dev/zero |
27 | */ | 27 | */ |
28 | static DEFINE_MUTEX(xip_sparse_mutex); | 28 | static DEFINE_MUTEX(xip_sparse_mutex); |
29 | static seqcount_t xip_sparse_seq = SEQCNT_ZERO; | 29 | static seqcount_t xip_sparse_seq = SEQCNT_ZERO(xip_sparse_seq); |
30 | static struct page *__xip_sparse_page; | 30 | static struct page *__xip_sparse_page; |
31 | 31 | ||
32 | /* called under xip_sparse_mutex */ | 32 | /* called under xip_sparse_mutex */ |
diff --git a/net/8021q/vlan_dev.c b/net/8021q/vlan_dev.c index 8db1b985dbf1..762896ebfcf5 100644 --- a/net/8021q/vlan_dev.c +++ b/net/8021q/vlan_dev.c | |||
@@ -539,7 +539,7 @@ static const struct net_device_ops vlan_netdev_ops; | |||
539 | static int vlan_dev_init(struct net_device *dev) | 539 | static int vlan_dev_init(struct net_device *dev) |
540 | { | 540 | { |
541 | struct net_device *real_dev = vlan_dev_priv(dev)->real_dev; | 541 | struct net_device *real_dev = vlan_dev_priv(dev)->real_dev; |
542 | int subclass = 0; | 542 | int subclass = 0, i; |
543 | 543 | ||
544 | netif_carrier_off(dev); | 544 | netif_carrier_off(dev); |
545 | 545 | ||
@@ -593,6 +593,13 @@ static int vlan_dev_init(struct net_device *dev) | |||
593 | if (!vlan_dev_priv(dev)->vlan_pcpu_stats) | 593 | if (!vlan_dev_priv(dev)->vlan_pcpu_stats) |
594 | return -ENOMEM; | 594 | return -ENOMEM; |
595 | 595 | ||
596 | for_each_possible_cpu(i) { | ||
597 | struct vlan_pcpu_stats *vlan_stat; | ||
598 | vlan_stat = per_cpu_ptr(vlan_dev_priv(dev)->vlan_pcpu_stats, i); | ||
599 | u64_stats_init(&vlan_stat->syncp); | ||
600 | } | ||
601 | |||
602 | |||
596 | return 0; | 603 | return 0; |
597 | } | 604 | } |
598 | 605 | ||
diff --git a/net/bridge/br_device.c b/net/bridge/br_device.c index e6b7fecb3af1..f00cfd2a0143 100644 --- a/net/bridge/br_device.c +++ b/net/bridge/br_device.c | |||
@@ -88,11 +88,18 @@ out: | |||
88 | static int br_dev_init(struct net_device *dev) | 88 | static int br_dev_init(struct net_device *dev) |
89 | { | 89 | { |
90 | struct net_bridge *br = netdev_priv(dev); | 90 | struct net_bridge *br = netdev_priv(dev); |
91 | int i; | ||
91 | 92 | ||
92 | br->stats = alloc_percpu(struct br_cpu_netstats); | 93 | br->stats = alloc_percpu(struct br_cpu_netstats); |
93 | if (!br->stats) | 94 | if (!br->stats) |
94 | return -ENOMEM; | 95 | return -ENOMEM; |
95 | 96 | ||
97 | for_each_possible_cpu(i) { | ||
98 | struct br_cpu_netstats *br_dev_stats; | ||
99 | br_dev_stats = per_cpu_ptr(br->stats, i); | ||
100 | u64_stats_init(&br_dev_stats->syncp); | ||
101 | } | ||
102 | |||
96 | return 0; | 103 | return 0; |
97 | } | 104 | } |
98 | 105 | ||
diff --git a/net/ipv4/af_inet.c b/net/ipv4/af_inet.c index 68af9aac91d0..70011e029ac1 100644 --- a/net/ipv4/af_inet.c +++ b/net/ipv4/af_inet.c | |||
@@ -1503,6 +1503,7 @@ int snmp_mib_init(void __percpu *ptr[2], size_t mibsize, size_t align) | |||
1503 | ptr[0] = __alloc_percpu(mibsize, align); | 1503 | ptr[0] = __alloc_percpu(mibsize, align); |
1504 | if (!ptr[0]) | 1504 | if (!ptr[0]) |
1505 | return -ENOMEM; | 1505 | return -ENOMEM; |
1506 | |||
1506 | #if SNMP_ARRAY_SZ == 2 | 1507 | #if SNMP_ARRAY_SZ == 2 |
1507 | ptr[1] = __alloc_percpu(mibsize, align); | 1508 | ptr[1] = __alloc_percpu(mibsize, align); |
1508 | if (!ptr[1]) { | 1509 | if (!ptr[1]) { |
@@ -1547,6 +1548,8 @@ static const struct net_protocol icmp_protocol = { | |||
1547 | 1548 | ||
1548 | static __net_init int ipv4_mib_init_net(struct net *net) | 1549 | static __net_init int ipv4_mib_init_net(struct net *net) |
1549 | { | 1550 | { |
1551 | int i; | ||
1552 | |||
1550 | if (snmp_mib_init((void __percpu **)net->mib.tcp_statistics, | 1553 | if (snmp_mib_init((void __percpu **)net->mib.tcp_statistics, |
1551 | sizeof(struct tcp_mib), | 1554 | sizeof(struct tcp_mib), |
1552 | __alignof__(struct tcp_mib)) < 0) | 1555 | __alignof__(struct tcp_mib)) < 0) |
@@ -1555,6 +1558,17 @@ static __net_init int ipv4_mib_init_net(struct net *net) | |||
1555 | sizeof(struct ipstats_mib), | 1558 | sizeof(struct ipstats_mib), |
1556 | __alignof__(struct ipstats_mib)) < 0) | 1559 | __alignof__(struct ipstats_mib)) < 0) |
1557 | goto err_ip_mib; | 1560 | goto err_ip_mib; |
1561 | |||
1562 | for_each_possible_cpu(i) { | ||
1563 | struct ipstats_mib *af_inet_stats; | ||
1564 | af_inet_stats = per_cpu_ptr(net->mib.ip_statistics[0], i); | ||
1565 | u64_stats_init(&af_inet_stats->syncp); | ||
1566 | #if SNMP_ARRAY_SZ == 2 | ||
1567 | af_inet_stats = per_cpu_ptr(net->mib.ip_statistics[1], i); | ||
1568 | u64_stats_init(&af_inet_stats->syncp); | ||
1569 | #endif | ||
1570 | } | ||
1571 | |||
1558 | if (snmp_mib_init((void __percpu **)net->mib.net_statistics, | 1572 | if (snmp_mib_init((void __percpu **)net->mib.net_statistics, |
1559 | sizeof(struct linux_mib), | 1573 | sizeof(struct linux_mib), |
1560 | __alignof__(struct linux_mib)) < 0) | 1574 | __alignof__(struct linux_mib)) < 0) |
diff --git a/net/ipv4/ip_tunnel.c b/net/ipv4/ip_tunnel.c index 63a6d6d6b875..caf01176a5e4 100644 --- a/net/ipv4/ip_tunnel.c +++ b/net/ipv4/ip_tunnel.c | |||
@@ -976,13 +976,19 @@ int ip_tunnel_init(struct net_device *dev) | |||
976 | { | 976 | { |
977 | struct ip_tunnel *tunnel = netdev_priv(dev); | 977 | struct ip_tunnel *tunnel = netdev_priv(dev); |
978 | struct iphdr *iph = &tunnel->parms.iph; | 978 | struct iphdr *iph = &tunnel->parms.iph; |
979 | int err; | 979 | int i, err; |
980 | 980 | ||
981 | dev->destructor = ip_tunnel_dev_free; | 981 | dev->destructor = ip_tunnel_dev_free; |
982 | dev->tstats = alloc_percpu(struct pcpu_tstats); | 982 | dev->tstats = alloc_percpu(struct pcpu_tstats); |
983 | if (!dev->tstats) | 983 | if (!dev->tstats) |
984 | return -ENOMEM; | 984 | return -ENOMEM; |
985 | 985 | ||
986 | for_each_possible_cpu(i) { | ||
987 | struct pcpu_tstats *ipt_stats; | ||
988 | ipt_stats = per_cpu_ptr(dev->tstats, i); | ||
989 | u64_stats_init(&ipt_stats->syncp); | ||
990 | } | ||
991 | |||
986 | err = gro_cells_init(&tunnel->gro_cells, dev); | 992 | err = gro_cells_init(&tunnel->gro_cells, dev); |
987 | if (err) { | 993 | if (err) { |
988 | free_percpu(dev->tstats); | 994 | free_percpu(dev->tstats); |
diff --git a/net/ipv6/addrconf.c b/net/ipv6/addrconf.c index 542d09561ed6..5658d9d51637 100644 --- a/net/ipv6/addrconf.c +++ b/net/ipv6/addrconf.c | |||
@@ -271,10 +271,24 @@ static void addrconf_mod_dad_timer(struct inet6_ifaddr *ifp, | |||
271 | 271 | ||
272 | static int snmp6_alloc_dev(struct inet6_dev *idev) | 272 | static int snmp6_alloc_dev(struct inet6_dev *idev) |
273 | { | 273 | { |
274 | int i; | ||
275 | |||
274 | if (snmp_mib_init((void __percpu **)idev->stats.ipv6, | 276 | if (snmp_mib_init((void __percpu **)idev->stats.ipv6, |
275 | sizeof(struct ipstats_mib), | 277 | sizeof(struct ipstats_mib), |
276 | __alignof__(struct ipstats_mib)) < 0) | 278 | __alignof__(struct ipstats_mib)) < 0) |
277 | goto err_ip; | 279 | goto err_ip; |
280 | |||
281 | for_each_possible_cpu(i) { | ||
282 | struct ipstats_mib *addrconf_stats; | ||
283 | addrconf_stats = per_cpu_ptr(idev->stats.ipv6[0], i); | ||
284 | u64_stats_init(&addrconf_stats->syncp); | ||
285 | #if SNMP_ARRAY_SZ == 2 | ||
286 | addrconf_stats = per_cpu_ptr(idev->stats.ipv6[1], i); | ||
287 | u64_stats_init(&addrconf_stats->syncp); | ||
288 | #endif | ||
289 | } | ||
290 | |||
291 | |||
278 | idev->stats.icmpv6dev = kzalloc(sizeof(struct icmpv6_mib_device), | 292 | idev->stats.icmpv6dev = kzalloc(sizeof(struct icmpv6_mib_device), |
279 | GFP_KERNEL); | 293 | GFP_KERNEL); |
280 | if (!idev->stats.icmpv6dev) | 294 | if (!idev->stats.icmpv6dev) |
diff --git a/net/ipv6/af_inet6.c b/net/ipv6/af_inet6.c index 6468bda1f2b9..ff75313f27a8 100644 --- a/net/ipv6/af_inet6.c +++ b/net/ipv6/af_inet6.c | |||
@@ -714,6 +714,8 @@ static void ipv6_packet_cleanup(void) | |||
714 | 714 | ||
715 | static int __net_init ipv6_init_mibs(struct net *net) | 715 | static int __net_init ipv6_init_mibs(struct net *net) |
716 | { | 716 | { |
717 | int i; | ||
718 | |||
717 | if (snmp_mib_init((void __percpu **)net->mib.udp_stats_in6, | 719 | if (snmp_mib_init((void __percpu **)net->mib.udp_stats_in6, |
718 | sizeof(struct udp_mib), | 720 | sizeof(struct udp_mib), |
719 | __alignof__(struct udp_mib)) < 0) | 721 | __alignof__(struct udp_mib)) < 0) |
@@ -726,6 +728,18 @@ static int __net_init ipv6_init_mibs(struct net *net) | |||
726 | sizeof(struct ipstats_mib), | 728 | sizeof(struct ipstats_mib), |
727 | __alignof__(struct ipstats_mib)) < 0) | 729 | __alignof__(struct ipstats_mib)) < 0) |
728 | goto err_ip_mib; | 730 | goto err_ip_mib; |
731 | |||
732 | for_each_possible_cpu(i) { | ||
733 | struct ipstats_mib *af_inet6_stats; | ||
734 | af_inet6_stats = per_cpu_ptr(net->mib.ipv6_statistics[0], i); | ||
735 | u64_stats_init(&af_inet6_stats->syncp); | ||
736 | #if SNMP_ARRAY_SZ == 2 | ||
737 | af_inet6_stats = per_cpu_ptr(net->mib.ipv6_statistics[1], i); | ||
738 | u64_stats_init(&af_inet6_stats->syncp); | ||
739 | #endif | ||
740 | } | ||
741 | |||
742 | |||
729 | if (snmp_mib_init((void __percpu **)net->mib.icmpv6_statistics, | 743 | if (snmp_mib_init((void __percpu **)net->mib.icmpv6_statistics, |
730 | sizeof(struct icmpv6_mib), | 744 | sizeof(struct icmpv6_mib), |
731 | __alignof__(struct icmpv6_mib)) < 0) | 745 | __alignof__(struct icmpv6_mib)) < 0) |
diff --git a/net/ipv6/ip6_gre.c b/net/ipv6/ip6_gre.c index bf4a9a084de5..8acb28621f9c 100644 --- a/net/ipv6/ip6_gre.c +++ b/net/ipv6/ip6_gre.c | |||
@@ -1252,6 +1252,7 @@ static void ip6gre_tunnel_setup(struct net_device *dev) | |||
1252 | static int ip6gre_tunnel_init(struct net_device *dev) | 1252 | static int ip6gre_tunnel_init(struct net_device *dev) |
1253 | { | 1253 | { |
1254 | struct ip6_tnl *tunnel; | 1254 | struct ip6_tnl *tunnel; |
1255 | int i; | ||
1255 | 1256 | ||
1256 | tunnel = netdev_priv(dev); | 1257 | tunnel = netdev_priv(dev); |
1257 | 1258 | ||
@@ -1269,6 +1270,13 @@ static int ip6gre_tunnel_init(struct net_device *dev) | |||
1269 | if (!dev->tstats) | 1270 | if (!dev->tstats) |
1270 | return -ENOMEM; | 1271 | return -ENOMEM; |
1271 | 1272 | ||
1273 | for_each_possible_cpu(i) { | ||
1274 | struct pcpu_tstats *ip6gre_tunnel_stats; | ||
1275 | ip6gre_tunnel_stats = per_cpu_ptr(dev->tstats, i); | ||
1276 | u64_stats_init(&ip6gre_tunnel_stats->syncp); | ||
1277 | } | ||
1278 | |||
1279 | |||
1272 | return 0; | 1280 | return 0; |
1273 | } | 1281 | } |
1274 | 1282 | ||
@@ -1449,6 +1457,7 @@ static void ip6gre_netlink_parms(struct nlattr *data[], | |||
1449 | static int ip6gre_tap_init(struct net_device *dev) | 1457 | static int ip6gre_tap_init(struct net_device *dev) |
1450 | { | 1458 | { |
1451 | struct ip6_tnl *tunnel; | 1459 | struct ip6_tnl *tunnel; |
1460 | int i; | ||
1452 | 1461 | ||
1453 | tunnel = netdev_priv(dev); | 1462 | tunnel = netdev_priv(dev); |
1454 | 1463 | ||
@@ -1462,6 +1471,12 @@ static int ip6gre_tap_init(struct net_device *dev) | |||
1462 | if (!dev->tstats) | 1471 | if (!dev->tstats) |
1463 | return -ENOMEM; | 1472 | return -ENOMEM; |
1464 | 1473 | ||
1474 | for_each_possible_cpu(i) { | ||
1475 | struct pcpu_tstats *ip6gre_tap_stats; | ||
1476 | ip6gre_tap_stats = per_cpu_ptr(dev->tstats, i); | ||
1477 | u64_stats_init(&ip6gre_tap_stats->syncp); | ||
1478 | } | ||
1479 | |||
1465 | return 0; | 1480 | return 0; |
1466 | } | 1481 | } |
1467 | 1482 | ||
diff --git a/net/ipv6/ip6_output.c b/net/ipv6/ip6_output.c index 5e31a909a2b0..59df872e2f4d 100644 --- a/net/ipv6/ip6_output.c +++ b/net/ipv6/ip6_output.c | |||
@@ -910,7 +910,7 @@ static int ip6_dst_lookup_tail(struct sock *sk, | |||
910 | 910 | ||
911 | out_err_release: | 911 | out_err_release: |
912 | if (err == -ENETUNREACH) | 912 | if (err == -ENETUNREACH) |
913 | IP6_INC_STATS_BH(net, NULL, IPSTATS_MIB_OUTNOROUTES); | 913 | IP6_INC_STATS(net, NULL, IPSTATS_MIB_OUTNOROUTES); |
914 | dst_release(*dst); | 914 | dst_release(*dst); |
915 | *dst = NULL; | 915 | *dst = NULL; |
916 | return err; | 916 | return err; |
diff --git a/net/ipv6/ip6_tunnel.c b/net/ipv6/ip6_tunnel.c index 583b77e2f69b..df1fa58528c6 100644 --- a/net/ipv6/ip6_tunnel.c +++ b/net/ipv6/ip6_tunnel.c | |||
@@ -1494,12 +1494,19 @@ static inline int | |||
1494 | ip6_tnl_dev_init_gen(struct net_device *dev) | 1494 | ip6_tnl_dev_init_gen(struct net_device *dev) |
1495 | { | 1495 | { |
1496 | struct ip6_tnl *t = netdev_priv(dev); | 1496 | struct ip6_tnl *t = netdev_priv(dev); |
1497 | int i; | ||
1497 | 1498 | ||
1498 | t->dev = dev; | 1499 | t->dev = dev; |
1499 | t->net = dev_net(dev); | 1500 | t->net = dev_net(dev); |
1500 | dev->tstats = alloc_percpu(struct pcpu_tstats); | 1501 | dev->tstats = alloc_percpu(struct pcpu_tstats); |
1501 | if (!dev->tstats) | 1502 | if (!dev->tstats) |
1502 | return -ENOMEM; | 1503 | return -ENOMEM; |
1504 | |||
1505 | for_each_possible_cpu(i) { | ||
1506 | struct pcpu_tstats *ip6_tnl_stats; | ||
1507 | ip6_tnl_stats = per_cpu_ptr(dev->tstats, i); | ||
1508 | u64_stats_init(&ip6_tnl_stats->syncp); | ||
1509 | } | ||
1503 | return 0; | 1510 | return 0; |
1504 | } | 1511 | } |
1505 | 1512 | ||
diff --git a/net/ipv6/sit.c b/net/ipv6/sit.c index 3a9038dd818d..bfc6fcea3841 100644 --- a/net/ipv6/sit.c +++ b/net/ipv6/sit.c | |||
@@ -1320,6 +1320,7 @@ static void ipip6_tunnel_setup(struct net_device *dev) | |||
1320 | static int ipip6_tunnel_init(struct net_device *dev) | 1320 | static int ipip6_tunnel_init(struct net_device *dev) |
1321 | { | 1321 | { |
1322 | struct ip_tunnel *tunnel = netdev_priv(dev); | 1322 | struct ip_tunnel *tunnel = netdev_priv(dev); |
1323 | int i; | ||
1323 | 1324 | ||
1324 | tunnel->dev = dev; | 1325 | tunnel->dev = dev; |
1325 | tunnel->net = dev_net(dev); | 1326 | tunnel->net = dev_net(dev); |
@@ -1332,6 +1333,12 @@ static int ipip6_tunnel_init(struct net_device *dev) | |||
1332 | if (!dev->tstats) | 1333 | if (!dev->tstats) |
1333 | return -ENOMEM; | 1334 | return -ENOMEM; |
1334 | 1335 | ||
1336 | for_each_possible_cpu(i) { | ||
1337 | struct pcpu_tstats *ipip6_tunnel_stats; | ||
1338 | ipip6_tunnel_stats = per_cpu_ptr(dev->tstats, i); | ||
1339 | u64_stats_init(&ipip6_tunnel_stats->syncp); | ||
1340 | } | ||
1341 | |||
1335 | return 0; | 1342 | return 0; |
1336 | } | 1343 | } |
1337 | 1344 | ||
@@ -1341,6 +1348,7 @@ static int __net_init ipip6_fb_tunnel_init(struct net_device *dev) | |||
1341 | struct iphdr *iph = &tunnel->parms.iph; | 1348 | struct iphdr *iph = &tunnel->parms.iph; |
1342 | struct net *net = dev_net(dev); | 1349 | struct net *net = dev_net(dev); |
1343 | struct sit_net *sitn = net_generic(net, sit_net_id); | 1350 | struct sit_net *sitn = net_generic(net, sit_net_id); |
1351 | int i; | ||
1344 | 1352 | ||
1345 | tunnel->dev = dev; | 1353 | tunnel->dev = dev; |
1346 | tunnel->net = dev_net(dev); | 1354 | tunnel->net = dev_net(dev); |
@@ -1354,6 +1362,13 @@ static int __net_init ipip6_fb_tunnel_init(struct net_device *dev) | |||
1354 | dev->tstats = alloc_percpu(struct pcpu_tstats); | 1362 | dev->tstats = alloc_percpu(struct pcpu_tstats); |
1355 | if (!dev->tstats) | 1363 | if (!dev->tstats) |
1356 | return -ENOMEM; | 1364 | return -ENOMEM; |
1365 | |||
1366 | for_each_possible_cpu(i) { | ||
1367 | struct pcpu_tstats *ipip6_fb_stats; | ||
1368 | ipip6_fb_stats = per_cpu_ptr(dev->tstats, i); | ||
1369 | u64_stats_init(&ipip6_fb_stats->syncp); | ||
1370 | } | ||
1371 | |||
1357 | dev_hold(dev); | 1372 | dev_hold(dev); |
1358 | rcu_assign_pointer(sitn->tunnels_wc[0], tunnel); | 1373 | rcu_assign_pointer(sitn->tunnels_wc[0], tunnel); |
1359 | return 0; | 1374 | return 0; |
diff --git a/net/netfilter/ipvs/ip_vs_ctl.c b/net/netfilter/ipvs/ip_vs_ctl.c index 62786a495cea..1ded5c6d268c 100644 --- a/net/netfilter/ipvs/ip_vs_ctl.c +++ b/net/netfilter/ipvs/ip_vs_ctl.c | |||
@@ -842,7 +842,7 @@ ip_vs_new_dest(struct ip_vs_service *svc, struct ip_vs_dest_user_kern *udest, | |||
842 | struct ip_vs_dest **dest_p) | 842 | struct ip_vs_dest **dest_p) |
843 | { | 843 | { |
844 | struct ip_vs_dest *dest; | 844 | struct ip_vs_dest *dest; |
845 | unsigned int atype; | 845 | unsigned int atype, i; |
846 | 846 | ||
847 | EnterFunction(2); | 847 | EnterFunction(2); |
848 | 848 | ||
@@ -869,6 +869,12 @@ ip_vs_new_dest(struct ip_vs_service *svc, struct ip_vs_dest_user_kern *udest, | |||
869 | if (!dest->stats.cpustats) | 869 | if (!dest->stats.cpustats) |
870 | goto err_alloc; | 870 | goto err_alloc; |
871 | 871 | ||
872 | for_each_possible_cpu(i) { | ||
873 | struct ip_vs_cpu_stats *ip_vs_dest_stats; | ||
874 | ip_vs_dest_stats = per_cpu_ptr(dest->stats.cpustats, i); | ||
875 | u64_stats_init(&ip_vs_dest_stats->syncp); | ||
876 | } | ||
877 | |||
872 | dest->af = svc->af; | 878 | dest->af = svc->af; |
873 | dest->protocol = svc->protocol; | 879 | dest->protocol = svc->protocol; |
874 | dest->vaddr = svc->addr; | 880 | dest->vaddr = svc->addr; |
@@ -1134,7 +1140,7 @@ static int | |||
1134 | ip_vs_add_service(struct net *net, struct ip_vs_service_user_kern *u, | 1140 | ip_vs_add_service(struct net *net, struct ip_vs_service_user_kern *u, |
1135 | struct ip_vs_service **svc_p) | 1141 | struct ip_vs_service **svc_p) |
1136 | { | 1142 | { |
1137 | int ret = 0; | 1143 | int ret = 0, i; |
1138 | struct ip_vs_scheduler *sched = NULL; | 1144 | struct ip_vs_scheduler *sched = NULL; |
1139 | struct ip_vs_pe *pe = NULL; | 1145 | struct ip_vs_pe *pe = NULL; |
1140 | struct ip_vs_service *svc = NULL; | 1146 | struct ip_vs_service *svc = NULL; |
@@ -1184,6 +1190,13 @@ ip_vs_add_service(struct net *net, struct ip_vs_service_user_kern *u, | |||
1184 | goto out_err; | 1190 | goto out_err; |
1185 | } | 1191 | } |
1186 | 1192 | ||
1193 | for_each_possible_cpu(i) { | ||
1194 | struct ip_vs_cpu_stats *ip_vs_stats; | ||
1195 | ip_vs_stats = per_cpu_ptr(svc->stats.cpustats, i); | ||
1196 | u64_stats_init(&ip_vs_stats->syncp); | ||
1197 | } | ||
1198 | |||
1199 | |||
1187 | /* I'm the first user of the service */ | 1200 | /* I'm the first user of the service */ |
1188 | atomic_set(&svc->refcnt, 0); | 1201 | atomic_set(&svc->refcnt, 0); |
1189 | 1202 | ||
@@ -3780,7 +3793,7 @@ static struct notifier_block ip_vs_dst_notifier = { | |||
3780 | 3793 | ||
3781 | int __net_init ip_vs_control_net_init(struct net *net) | 3794 | int __net_init ip_vs_control_net_init(struct net *net) |
3782 | { | 3795 | { |
3783 | int idx; | 3796 | int i, idx; |
3784 | struct netns_ipvs *ipvs = net_ipvs(net); | 3797 | struct netns_ipvs *ipvs = net_ipvs(net); |
3785 | 3798 | ||
3786 | /* Initialize rs_table */ | 3799 | /* Initialize rs_table */ |
@@ -3799,6 +3812,12 @@ int __net_init ip_vs_control_net_init(struct net *net) | |||
3799 | if (!ipvs->tot_stats.cpustats) | 3812 | if (!ipvs->tot_stats.cpustats) |
3800 | return -ENOMEM; | 3813 | return -ENOMEM; |
3801 | 3814 | ||
3815 | for_each_possible_cpu(i) { | ||
3816 | struct ip_vs_cpu_stats *ipvs_tot_stats; | ||
3817 | ipvs_tot_stats = per_cpu_ptr(ipvs->tot_stats.cpustats, i); | ||
3818 | u64_stats_init(&ipvs_tot_stats->syncp); | ||
3819 | } | ||
3820 | |||
3802 | spin_lock_init(&ipvs->tot_stats.lock); | 3821 | spin_lock_init(&ipvs->tot_stats.lock); |
3803 | 3822 | ||
3804 | proc_create("ip_vs", 0, net->proc_net, &ip_vs_info_fops); | 3823 | proc_create("ip_vs", 0, net->proc_net, &ip_vs_info_fops); |
diff --git a/net/openvswitch/datapath.c b/net/openvswitch/datapath.c index 1408adc2a2a7..449e0776a2c0 100644 --- a/net/openvswitch/datapath.c +++ b/net/openvswitch/datapath.c | |||
@@ -1199,6 +1199,12 @@ static int ovs_dp_cmd_new(struct sk_buff *skb, struct genl_info *info) | |||
1199 | goto err_destroy_table; | 1199 | goto err_destroy_table; |
1200 | } | 1200 | } |
1201 | 1201 | ||
1202 | for_each_possible_cpu(i) { | ||
1203 | struct dp_stats_percpu *dpath_stats; | ||
1204 | dpath_stats = per_cpu_ptr(dp->stats_percpu, i); | ||
1205 | u64_stats_init(&dpath_stats->sync); | ||
1206 | } | ||
1207 | |||
1202 | dp->ports = kmalloc(DP_VPORT_HASH_BUCKETS * sizeof(struct hlist_head), | 1208 | dp->ports = kmalloc(DP_VPORT_HASH_BUCKETS * sizeof(struct hlist_head), |
1203 | GFP_KERNEL); | 1209 | GFP_KERNEL); |
1204 | if (!dp->ports) { | 1210 | if (!dp->ports) { |
diff --git a/net/openvswitch/vport.c b/net/openvswitch/vport.c index 6f65dbe13812..d830a95f03a4 100644 --- a/net/openvswitch/vport.c +++ b/net/openvswitch/vport.c | |||
@@ -118,6 +118,7 @@ struct vport *ovs_vport_alloc(int priv_size, const struct vport_ops *ops, | |||
118 | { | 118 | { |
119 | struct vport *vport; | 119 | struct vport *vport; |
120 | size_t alloc_size; | 120 | size_t alloc_size; |
121 | int i; | ||
121 | 122 | ||
122 | alloc_size = sizeof(struct vport); | 123 | alloc_size = sizeof(struct vport); |
123 | if (priv_size) { | 124 | if (priv_size) { |
@@ -141,6 +142,13 @@ struct vport *ovs_vport_alloc(int priv_size, const struct vport_ops *ops, | |||
141 | return ERR_PTR(-ENOMEM); | 142 | return ERR_PTR(-ENOMEM); |
142 | } | 143 | } |
143 | 144 | ||
145 | for_each_possible_cpu(i) { | ||
146 | struct pcpu_tstats *vport_stats; | ||
147 | vport_stats = per_cpu_ptr(vport->percpu_stats, i); | ||
148 | u64_stats_init(&vport_stats->syncp); | ||
149 | } | ||
150 | |||
151 | |||
144 | spin_lock_init(&vport->stats_lock); | 152 | spin_lock_init(&vport->stats_lock); |
145 | 153 | ||
146 | return vport; | 154 | return vport; |