aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
-rw-r--r--Documentation/DocBook/kernel-locking.tmpl2
-rw-r--r--Documentation/lockstat.txt123
-rw-r--r--Documentation/mutex-design.txt10
-rw-r--r--arch/x86/Kconfig6
-rw-r--r--arch/x86/kernel/alternative.c11
-rw-r--r--arch/x86/vdso/vclock_gettime.c8
-rw-r--r--block/blk-cgroup.h10
-rw-r--r--block/blk-throttle.c10
-rw-r--r--block/cfq-iosched.c25
-rw-r--r--drivers/net/dummy.c6
-rw-r--r--drivers/net/ethernet/emulex/benet/be_main.c4
-rw-r--r--drivers/net/ethernet/intel/igb/igb_main.c5
-rw-r--r--drivers/net/ethernet/intel/ixgbe/ixgbe_main.c4
-rw-r--r--drivers/net/ethernet/marvell/mvneta.c3
-rw-r--r--drivers/net/ethernet/marvell/sky2.c3
-rw-r--r--drivers/net/ethernet/neterion/vxge/vxge-main.c4
-rw-r--r--drivers/net/ethernet/nvidia/forcedeth.c2
-rw-r--r--drivers/net/ethernet/realtek/8139too.c3
-rw-r--r--drivers/net/ethernet/tile/tilepro.c2
-rw-r--r--drivers/net/ethernet/via/via-rhine.c3
-rw-r--r--drivers/net/ifb.c5
-rw-r--r--drivers/net/loopback.c6
-rw-r--r--drivers/net/macvlan.c7
-rw-r--r--drivers/net/nlmon.c8
-rw-r--r--drivers/net/team/team.c6
-rw-r--r--drivers/net/team/team_mode_loadbalance.c9
-rw-r--r--drivers/net/veth.c8
-rw-r--r--drivers/net/virtio_net.c7
-rw-r--r--drivers/net/vxlan.c8
-rw-r--r--drivers/net/xen-netfront.c6
-rw-r--r--fs/dcache.c4
-rw-r--r--fs/fs_struct.c2
-rw-r--r--include/linux/cpuset.h4
-rw-r--r--include/linux/init_task.h8
-rw-r--r--include/linux/lockdep.h8
-rw-r--r--include/linux/mutex.h2
-rw-r--r--include/linux/sched/sysctl.h2
-rw-r--r--include/linux/seqlock.h79
-rw-r--r--include/linux/u64_stats_sync.h7
-rw-r--r--include/trace/events/sched.h19
-rw-r--r--kernel/Makefile22
-rw-r--r--kernel/futex.c2
-rw-r--r--kernel/hung_task.c6
-rw-r--r--kernel/locking/Makefile25
-rw-r--r--kernel/locking/lglock.c (renamed from kernel/lglock.c)0
-rw-r--r--kernel/locking/lockdep.c (renamed from kernel/lockdep.c)4
-rw-r--r--kernel/locking/lockdep_internals.h (renamed from kernel/lockdep_internals.h)0
-rw-r--r--kernel/locking/lockdep_proc.c (renamed from kernel/lockdep_proc.c)15
-rw-r--r--kernel/locking/lockdep_states.h (renamed from kernel/lockdep_states.h)0
-rw-r--r--kernel/locking/mutex-debug.c (renamed from kernel/mutex-debug.c)0
-rw-r--r--kernel/locking/mutex-debug.h (renamed from kernel/mutex-debug.h)0
-rw-r--r--kernel/locking/mutex.c (renamed from kernel/mutex.c)2
-rw-r--r--kernel/locking/mutex.h (renamed from kernel/mutex.h)0
-rw-r--r--kernel/locking/percpu-rwsem.c (renamed from lib/percpu-rwsem.c)0
-rw-r--r--kernel/locking/rtmutex-debug.c (renamed from kernel/rtmutex-debug.c)0
-rw-r--r--kernel/locking/rtmutex-debug.h (renamed from kernel/rtmutex-debug.h)0
-rw-r--r--kernel/locking/rtmutex-tester.c (renamed from kernel/rtmutex-tester.c)0
-rw-r--r--kernel/locking/rtmutex.c (renamed from kernel/rtmutex.c)0
-rw-r--r--kernel/locking/rtmutex.h (renamed from kernel/rtmutex.h)0
-rw-r--r--kernel/locking/rtmutex_common.h (renamed from kernel/rtmutex_common.h)0
-rw-r--r--kernel/locking/rwsem-spinlock.c (renamed from lib/rwsem-spinlock.c)0
-rw-r--r--kernel/locking/rwsem-xadd.c (renamed from lib/rwsem.c)0
-rw-r--r--kernel/locking/rwsem.c (renamed from kernel/rwsem.c)0
-rw-r--r--kernel/locking/semaphore.c (renamed from kernel/semaphore.c)0
-rw-r--r--kernel/locking/spinlock.c (renamed from kernel/spinlock.c)0
-rw-r--r--kernel/locking/spinlock_debug.c (renamed from lib/spinlock_debug.c)0
-rw-r--r--kernel/rcu/tree_plugin.h2
-rw-r--r--kernel/sysctl.c5
-rw-r--r--lib/Makefile4
-rw-r--r--mm/filemap_xip.c2
-rw-r--r--net/8021q/vlan_dev.c9
-rw-r--r--net/bridge/br_device.c7
-rw-r--r--net/ipv4/af_inet.c14
-rw-r--r--net/ipv4/ip_tunnel.c8
-rw-r--r--net/ipv6/addrconf.c14
-rw-r--r--net/ipv6/af_inet6.c14
-rw-r--r--net/ipv6/ip6_gre.c15
-rw-r--r--net/ipv6/ip6_output.c2
-rw-r--r--net/ipv6/ip6_tunnel.c7
-rw-r--r--net/ipv6/sit.c15
-rw-r--r--net/netfilter/ipvs/ip_vs_ctl.c25
-rw-r--r--net/openvswitch/datapath.c6
-rw-r--r--net/openvswitch/vport.c8
83 files changed, 534 insertions, 148 deletions
diff --git a/Documentation/DocBook/kernel-locking.tmpl b/Documentation/DocBook/kernel-locking.tmpl
index 09e884e5b9f5..19f2a5a5a5b4 100644
--- a/Documentation/DocBook/kernel-locking.tmpl
+++ b/Documentation/DocBook/kernel-locking.tmpl
@@ -1958,7 +1958,7 @@ machines due to caching.
1958 <chapter id="apiref-mutex"> 1958 <chapter id="apiref-mutex">
1959 <title>Mutex API reference</title> 1959 <title>Mutex API reference</title>
1960!Iinclude/linux/mutex.h 1960!Iinclude/linux/mutex.h
1961!Ekernel/mutex.c 1961!Ekernel/locking/mutex.c
1962 </chapter> 1962 </chapter>
1963 1963
1964 <chapter id="apiref-futex"> 1964 <chapter id="apiref-futex">
diff --git a/Documentation/lockstat.txt b/Documentation/lockstat.txt
index dd2f7b26ca30..72d010689751 100644
--- a/Documentation/lockstat.txt
+++ b/Documentation/lockstat.txt
@@ -46,16 +46,14 @@ With these hooks we provide the following statistics:
46 contentions - number of lock acquisitions that had to wait 46 contentions - number of lock acquisitions that had to wait
47 wait time min - shortest (non-0) time we ever had to wait for a lock 47 wait time min - shortest (non-0) time we ever had to wait for a lock
48 max - longest time we ever had to wait for a lock 48 max - longest time we ever had to wait for a lock
49 total - total time we spend waiting on this lock 49 total - total time we spend waiting on this lock
50 avg - average time spent waiting on this lock
50 acq-bounces - number of lock acquisitions that involved x-cpu data 51 acq-bounces - number of lock acquisitions that involved x-cpu data
51 acquisitions - number of times we took the lock 52 acquisitions - number of times we took the lock
52 hold time min - shortest (non-0) time we ever held the lock 53 hold time min - shortest (non-0) time we ever held the lock
53 max - longest time we ever held the lock 54 max - longest time we ever held the lock
54 total - total time this lock was held 55 total - total time this lock was held
55 56 avg - average time this lock was held
56From these number various other statistics can be derived, such as:
57
58 hold time average = hold time total / acquisitions
59 57
60These numbers are gathered per lock class, per read/write state (when 58These numbers are gathered per lock class, per read/write state (when
61applicable). 59applicable).
@@ -84,37 +82,38 @@ Look at the current lock statistics:
84 82
85# less /proc/lock_stat 83# less /proc/lock_stat
86 84
8701 lock_stat version 0.3 8501 lock_stat version 0.4
8802 ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 8602-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
8903 class name con-bounces contentions waittime-min waittime-max waittime-total acq-bounces acquisitions holdtime-min holdtime-max holdtime-total 8703 class name con-bounces contentions waittime-min waittime-max waittime-total waittime-avg acq-bounces acquisitions holdtime-min holdtime-max holdtime-total holdtime-avg
9004 ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 8804-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
9105 8905
9206 &mm->mmap_sem-W: 233 538 18446744073708 22924.27 607243.51 1342 45806 1.71 8595.89 1180582.34 9006 &mm->mmap_sem-W: 46 84 0.26 939.10 16371.53 194.90 47291 2922365 0.16 2220301.69 17464026916.32 5975.99
9307 &mm->mmap_sem-R: 205 587 18446744073708 28403.36 731975.00 1940 412426 0.58 187825.45 6307502.88 9107 &mm->mmap_sem-R: 37 100 1.31 299502.61 325629.52 3256.30 212344 34316685 0.10 7744.91 95016910.20 2.77
9408 --------------- 9208 ---------------
9509 &mm->mmap_sem 487 [<ffffffff8053491f>] do_page_fault+0x466/0x928 9309 &mm->mmap_sem 1 [<ffffffff811502a7>] khugepaged_scan_mm_slot+0x57/0x280
9610 &mm->mmap_sem 179 [<ffffffff802a6200>] sys_mprotect+0xcd/0x21d 9419 &mm->mmap_sem 96 [<ffffffff815351c4>] __do_page_fault+0x1d4/0x510
9711 &mm->mmap_sem 279 [<ffffffff80210a57>] sys_mmap+0x75/0xce 9511 &mm->mmap_sem 34 [<ffffffff81113d77>] vm_mmap_pgoff+0x87/0xd0
9812 &mm->mmap_sem 76 [<ffffffff802a490b>] sys_munmap+0x32/0x59 9612 &mm->mmap_sem 17 [<ffffffff81127e71>] vm_munmap+0x41/0x80
9913 --------------- 9713 ---------------
10014 &mm->mmap_sem 270 [<ffffffff80210a57>] sys_mmap+0x75/0xce 9814 &mm->mmap_sem 1 [<ffffffff81046fda>] dup_mmap+0x2a/0x3f0
10115 &mm->mmap_sem 431 [<ffffffff8053491f>] do_page_fault+0x466/0x928 9915 &mm->mmap_sem 60 [<ffffffff81129e29>] SyS_mprotect+0xe9/0x250
10216 &mm->mmap_sem 138 [<ffffffff802a490b>] sys_munmap+0x32/0x59 10016 &mm->mmap_sem 41 [<ffffffff815351c4>] __do_page_fault+0x1d4/0x510
10317 &mm->mmap_sem 145 [<ffffffff802a6200>] sys_mprotect+0xcd/0x21d 10117 &mm->mmap_sem 68 [<ffffffff81113d77>] vm_mmap_pgoff+0x87/0xd0
10418 10218
10519 ............................................................................................................................................................................................... 10319.............................................................................................................................................................................................................................
10620 10420
10721 dcache_lock: 621 623 0.52 118.26 1053.02 6745 91930 0.29 316.29 118423.41 10521 unix_table_lock: 110 112 0.21 49.24 163.91 1.46 21094 66312 0.12 624.42 31589.81 0.48
10822 ----------- 10622 ---------------
10923 dcache_lock 179 [<ffffffff80378274>] _atomic_dec_and_lock+0x34/0x54 10723 unix_table_lock 45 [<ffffffff8150ad8e>] unix_create1+0x16e/0x1b0
11024 dcache_lock 113 [<ffffffff802cc17b>] d_alloc+0x19a/0x1eb 10824 unix_table_lock 47 [<ffffffff8150b111>] unix_release_sock+0x31/0x250
11125 dcache_lock 99 [<ffffffff802ca0dc>] d_rehash+0x1b/0x44 10925 unix_table_lock 15 [<ffffffff8150ca37>] unix_find_other+0x117/0x230
11226 dcache_lock 104 [<ffffffff802cbca0>] d_instantiate+0x36/0x8a 11026 unix_table_lock 5 [<ffffffff8150a09f>] unix_autobind+0x11f/0x1b0
11327 ----------- 11127 ---------------
11428 dcache_lock 192 [<ffffffff80378274>] _atomic_dec_and_lock+0x34/0x54 11228 unix_table_lock 39 [<ffffffff8150b111>] unix_release_sock+0x31/0x250
11529 dcache_lock 98 [<ffffffff802ca0dc>] d_rehash+0x1b/0x44 11329 unix_table_lock 49 [<ffffffff8150ad8e>] unix_create1+0x16e/0x1b0
11630 dcache_lock 72 [<ffffffff802cc17b>] d_alloc+0x19a/0x1eb 11430 unix_table_lock 20 [<ffffffff8150ca37>] unix_find_other+0x117/0x230
11731 dcache_lock 112 [<ffffffff802cbca0>] d_instantiate+0x36/0x8a 11531 unix_table_lock 4 [<ffffffff8150a09f>] unix_autobind+0x11f/0x1b0
116
118 117
119This excerpt shows the first two lock class statistics. Line 01 shows the 118This excerpt shows the first two lock class statistics. Line 01 shows the
120output version - each time the format changes this will be updated. Line 02-04 119output version - each time the format changes this will be updated. Line 02-04
@@ -131,30 +130,30 @@ The integer part of the time values is in us.
131 130
132Dealing with nested locks, subclasses may appear: 131Dealing with nested locks, subclasses may appear:
133 132
13432............................................................................................................................................................................................... 13332...........................................................................................................................................................................................................................
13533 13433
13634 &rq->lock: 13128 13128 0.43 190.53 103881.26 97454 3453404 0.00 401.11 13224683.11 13534 &rq->lock: 13128 13128 0.43 190.53 103881.26 7.91 97454 3453404 0.00 401.11 13224683.11 3.82
13735 --------- 13635 ---------
13836 &rq->lock 645 [<ffffffff8103bfc4>] task_rq_lock+0x43/0x75 13736 &rq->lock 645 [<ffffffff8103bfc4>] task_rq_lock+0x43/0x75
13937 &rq->lock 297 [<ffffffff8104ba65>] try_to_wake_up+0x127/0x25a 13837 &rq->lock 297 [<ffffffff8104ba65>] try_to_wake_up+0x127/0x25a
14038 &rq->lock 360 [<ffffffff8103c4c5>] select_task_rq_fair+0x1f0/0x74a 13938 &rq->lock 360 [<ffffffff8103c4c5>] select_task_rq_fair+0x1f0/0x74a
14139 &rq->lock 428 [<ffffffff81045f98>] scheduler_tick+0x46/0x1fb 14039 &rq->lock 428 [<ffffffff81045f98>] scheduler_tick+0x46/0x1fb
14240 --------- 14140 ---------
14341 &rq->lock 77 [<ffffffff8103bfc4>] task_rq_lock+0x43/0x75 14241 &rq->lock 77 [<ffffffff8103bfc4>] task_rq_lock+0x43/0x75
14442 &rq->lock 174 [<ffffffff8104ba65>] try_to_wake_up+0x127/0x25a 14342 &rq->lock 174 [<ffffffff8104ba65>] try_to_wake_up+0x127/0x25a
14543 &rq->lock 4715 [<ffffffff8103ed4b>] double_rq_lock+0x42/0x54 14443 &rq->lock 4715 [<ffffffff8103ed4b>] double_rq_lock+0x42/0x54
14644 &rq->lock 893 [<ffffffff81340524>] schedule+0x157/0x7b8 14544 &rq->lock 893 [<ffffffff81340524>] schedule+0x157/0x7b8
14745 14645
14846............................................................................................................................................................................................... 14746...........................................................................................................................................................................................................................
14947 14847
15048 &rq->lock/1: 11526 11488 0.33 388.73 136294.31 21461 38404 0.00 37.93 109388.53 14948 &rq->lock/1: 1526 11488 0.33 388.73 136294.31 11.86 21461 38404 0.00 37.93 109388.53 2.84
15149 ----------- 15049 -----------
15250 &rq->lock/1 11526 [<ffffffff8103ed58>] double_rq_lock+0x4f/0x54 15150 &rq->lock/1 11526 [<ffffffff8103ed58>] double_rq_lock+0x4f/0x54
15351 ----------- 15251 -----------
15452 &rq->lock/1 5645 [<ffffffff8103ed4b>] double_rq_lock+0x42/0x54 15352 &rq->lock/1 5645 [<ffffffff8103ed4b>] double_rq_lock+0x42/0x54
15553 &rq->lock/1 1224 [<ffffffff81340524>] schedule+0x157/0x7b8 15453 &rq->lock/1 1224 [<ffffffff81340524>] schedule+0x157/0x7b8
15654 &rq->lock/1 4336 [<ffffffff8103ed58>] double_rq_lock+0x4f/0x54 15554 &rq->lock/1 4336 [<ffffffff8103ed58>] double_rq_lock+0x4f/0x54
15755 &rq->lock/1 181 [<ffffffff8104ba65>] try_to_wake_up+0x127/0x25a 15655 &rq->lock/1 181 [<ffffffff8104ba65>] try_to_wake_up+0x127/0x25a
158 157
159Line 48 shows statistics for the second subclass (/1) of &rq->lock class 158Line 48 shows statistics for the second subclass (/1) of &rq->lock class
160(subclass starts from 0), since in this case, as line 50 suggests, 159(subclass starts from 0), since in this case, as line 50 suggests,
@@ -163,16 +162,16 @@ double_rq_lock actually acquires a nested lock of two spinlocks.
163View the top contending locks: 162View the top contending locks:
164 163
165# grep : /proc/lock_stat | head 164# grep : /proc/lock_stat | head
166 &inode->i_data.tree_lock-W: 15 21657 0.18 1093295.30 11547131054.85 58 10415 0.16 87.51 6387.60 165 clockevents_lock: 2926159 2947636 0.15 46882.81 1784540466.34 605.41 3381345 3879161 0.00 2260.97 53178395.68 13.71
167 &inode->i_data.tree_lock-R: 0 0 0.00 0.00 0.00 23302 231198 0.25 8.45 98023.38 166 tick_broadcast_lock: 346460 346717 0.18 2257.43 39364622.71 113.54 3642919 4242696 0.00 2263.79 49173646.60 11.59
168 dcache_lock: 1037 1161 0.38 45.32 774.51 6611 243371 0.15 306.48 77387.24 167 &mapping->i_mmap_mutex: 203896 203899 3.36 645530.05 31767507988.39 155800.21 3361776 8893984 0.17 2254.15 14110121.02 1.59
169 &inode->i_mutex: 161 286 18446744073709 62882.54 1244614.55 3653 20598 18446744073709 62318.60 1693822.74 168 &rq->lock: 135014 136909 0.18 606.09 842160.68 6.15 1540728 10436146 0.00 728.72 17606683.41 1.69
170 &zone->lru_lock: 94 94 0.53 7.33 92.10 4366 32690 0.29 59.81 16350.06 169 &(&zone->lru_lock)->rlock: 93000 94934 0.16 59.18 188253.78 1.98 1199912 3809894 0.15 391.40 3559518.81 0.93
171 &inode->i_data.i_mmap_mutex: 79 79 0.40 3.77 53.03 11779 87755 0.28 116.93 29898.44 170 tasklist_lock-W: 40667 41130 0.23 1189.42 428980.51 10.43 270278 510106 0.16 653.51 3939674.91 7.72
172 &q->__queue_lock: 48 50 0.52 31.62 86.31 774 13131 0.17 113.08 12277.52 171 tasklist_lock-R: 21298 21305 0.20 1310.05 215511.12 10.12 186204 241258 0.14 1162.33 1179779.23 4.89
173 &rq->rq_lock_key: 43 47 0.74 68.50 170.63 3706 33929 0.22 107.99 17460.62 172 rcu_node_1: 47656 49022 0.16 635.41 193616.41 3.95 844888 1865423 0.00 764.26 1656226.96 0.89
174 &rq->rq_lock_key#2: 39 46 0.75 6.68 49.03 2979 32292 0.17 125.17 17137.63 173 &(&dentry->d_lockref.lock)->rlock: 39791 40179 0.15 1302.08 88851.96 2.21 2790851 12527025 0.10 1910.75 3379714.27 0.27
175 tasklist_lock-W: 15 15 1.45 10.87 32.70 1201 7390 0.58 62.55 13648.47 174 rcu_node_0: 29203 30064 0.16 786.55 1555573.00 51.74 88963 244254 0.00 398.87 428872.51 1.76
176 175
177Clear the statistics: 176Clear the statistics:
178 177
diff --git a/Documentation/mutex-design.txt b/Documentation/mutex-design.txt
index 38c10fd7f411..1dfe62c3641d 100644
--- a/Documentation/mutex-design.txt
+++ b/Documentation/mutex-design.txt
@@ -116,11 +116,11 @@ using mutexes at the moment, please let me know if you find any. ]
116Implementation of mutexes 116Implementation of mutexes
117------------------------- 117-------------------------
118 118
119'struct mutex' is the new mutex type, defined in include/linux/mutex.h 119'struct mutex' is the new mutex type, defined in include/linux/mutex.h and
120and implemented in kernel/mutex.c. It is a counter-based mutex with a 120implemented in kernel/locking/mutex.c. It is a counter-based mutex with a
121spinlock and a wait-list. The counter has 3 states: 1 for "unlocked", 121spinlock and a wait-list. The counter has 3 states: 1 for "unlocked", 0 for
1220 for "locked" and negative numbers (usually -1) for "locked, potential 122"locked" and negative numbers (usually -1) for "locked, potential waiters
123waiters queued". 123queued".
124 124
125the APIs of 'struct mutex' have been streamlined: 125the APIs of 'struct mutex' have been streamlined:
126 126
diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig
index 14dc9c797abb..6e3e1cb3f6a0 100644
--- a/arch/x86/Kconfig
+++ b/arch/x86/Kconfig
@@ -635,10 +635,10 @@ config PARAVIRT_SPINLOCKS
635 spinlock implementation with something virtualization-friendly 635 spinlock implementation with something virtualization-friendly
636 (for example, block the virtual CPU rather than spinning). 636 (for example, block the virtual CPU rather than spinning).
637 637
638 Unfortunately the downside is an up to 5% performance hit on 638 It has a minimal impact on native kernels and gives a nice performance
639 native kernels, with various workloads. 639 benefit on paravirtualized KVM / Xen kernels.
640 640
641 If you are unsure how to answer this question, answer N. 641 If you are unsure how to answer this question, answer Y.
642 642
643source "arch/x86/xen/Kconfig" 643source "arch/x86/xen/Kconfig"
644 644
diff --git a/arch/x86/kernel/alternative.c b/arch/x86/kernel/alternative.c
index 15e8563e5c24..df94598ad05a 100644
--- a/arch/x86/kernel/alternative.c
+++ b/arch/x86/kernel/alternative.c
@@ -402,17 +402,6 @@ void alternatives_enable_smp(void)
402{ 402{
403 struct smp_alt_module *mod; 403 struct smp_alt_module *mod;
404 404
405#ifdef CONFIG_LOCKDEP
406 /*
407 * Older binutils section handling bug prevented
408 * alternatives-replacement from working reliably.
409 *
410 * If this still occurs then you should see a hang
411 * or crash shortly after this line:
412 */
413 pr_info("lockdep: fixing up alternatives\n");
414#endif
415
416 /* Why bother if there are no other CPUs? */ 405 /* Why bother if there are no other CPUs? */
417 BUG_ON(num_possible_cpus() == 1); 406 BUG_ON(num_possible_cpus() == 1);
418 407
diff --git a/arch/x86/vdso/vclock_gettime.c b/arch/x86/vdso/vclock_gettime.c
index 72074d528400..2ada505067cc 100644
--- a/arch/x86/vdso/vclock_gettime.c
+++ b/arch/x86/vdso/vclock_gettime.c
@@ -178,7 +178,7 @@ notrace static int __always_inline do_realtime(struct timespec *ts)
178 178
179 ts->tv_nsec = 0; 179 ts->tv_nsec = 0;
180 do { 180 do {
181 seq = read_seqcount_begin(&gtod->seq); 181 seq = read_seqcount_begin_no_lockdep(&gtod->seq);
182 mode = gtod->clock.vclock_mode; 182 mode = gtod->clock.vclock_mode;
183 ts->tv_sec = gtod->wall_time_sec; 183 ts->tv_sec = gtod->wall_time_sec;
184 ns = gtod->wall_time_snsec; 184 ns = gtod->wall_time_snsec;
@@ -198,7 +198,7 @@ notrace static int do_monotonic(struct timespec *ts)
198 198
199 ts->tv_nsec = 0; 199 ts->tv_nsec = 0;
200 do { 200 do {
201 seq = read_seqcount_begin(&gtod->seq); 201 seq = read_seqcount_begin_no_lockdep(&gtod->seq);
202 mode = gtod->clock.vclock_mode; 202 mode = gtod->clock.vclock_mode;
203 ts->tv_sec = gtod->monotonic_time_sec; 203 ts->tv_sec = gtod->monotonic_time_sec;
204 ns = gtod->monotonic_time_snsec; 204 ns = gtod->monotonic_time_snsec;
@@ -214,7 +214,7 @@ notrace static int do_realtime_coarse(struct timespec *ts)
214{ 214{
215 unsigned long seq; 215 unsigned long seq;
216 do { 216 do {
217 seq = read_seqcount_begin(&gtod->seq); 217 seq = read_seqcount_begin_no_lockdep(&gtod->seq);
218 ts->tv_sec = gtod->wall_time_coarse.tv_sec; 218 ts->tv_sec = gtod->wall_time_coarse.tv_sec;
219 ts->tv_nsec = gtod->wall_time_coarse.tv_nsec; 219 ts->tv_nsec = gtod->wall_time_coarse.tv_nsec;
220 } while (unlikely(read_seqcount_retry(&gtod->seq, seq))); 220 } while (unlikely(read_seqcount_retry(&gtod->seq, seq)));
@@ -225,7 +225,7 @@ notrace static int do_monotonic_coarse(struct timespec *ts)
225{ 225{
226 unsigned long seq; 226 unsigned long seq;
227 do { 227 do {
228 seq = read_seqcount_begin(&gtod->seq); 228 seq = read_seqcount_begin_no_lockdep(&gtod->seq);
229 ts->tv_sec = gtod->monotonic_time_coarse.tv_sec; 229 ts->tv_sec = gtod->monotonic_time_coarse.tv_sec;
230 ts->tv_nsec = gtod->monotonic_time_coarse.tv_nsec; 230 ts->tv_nsec = gtod->monotonic_time_coarse.tv_nsec;
231 } while (unlikely(read_seqcount_retry(&gtod->seq, seq))); 231 } while (unlikely(read_seqcount_retry(&gtod->seq, seq)));
diff --git a/block/blk-cgroup.h b/block/blk-cgroup.h
index ae6969a7ffd4..1610b22edf09 100644
--- a/block/blk-cgroup.h
+++ b/block/blk-cgroup.h
@@ -402,6 +402,11 @@ struct request_list *__blk_queue_next_rl(struct request_list *rl,
402#define blk_queue_for_each_rl(rl, q) \ 402#define blk_queue_for_each_rl(rl, q) \
403 for ((rl) = &(q)->root_rl; (rl); (rl) = __blk_queue_next_rl((rl), (q))) 403 for ((rl) = &(q)->root_rl; (rl); (rl) = __blk_queue_next_rl((rl), (q)))
404 404
405static inline void blkg_stat_init(struct blkg_stat *stat)
406{
407 u64_stats_init(&stat->syncp);
408}
409
405/** 410/**
406 * blkg_stat_add - add a value to a blkg_stat 411 * blkg_stat_add - add a value to a blkg_stat
407 * @stat: target blkg_stat 412 * @stat: target blkg_stat
@@ -458,6 +463,11 @@ static inline void blkg_stat_merge(struct blkg_stat *to, struct blkg_stat *from)
458 blkg_stat_add(to, blkg_stat_read(from)); 463 blkg_stat_add(to, blkg_stat_read(from));
459} 464}
460 465
466static inline void blkg_rwstat_init(struct blkg_rwstat *rwstat)
467{
468 u64_stats_init(&rwstat->syncp);
469}
470
461/** 471/**
462 * blkg_rwstat_add - add a value to a blkg_rwstat 472 * blkg_rwstat_add - add a value to a blkg_rwstat
463 * @rwstat: target blkg_rwstat 473 * @rwstat: target blkg_rwstat
diff --git a/block/blk-throttle.c b/block/blk-throttle.c
index 8331aba9426f..06534049afba 100644
--- a/block/blk-throttle.c
+++ b/block/blk-throttle.c
@@ -256,6 +256,12 @@ static struct throtl_data *sq_to_td(struct throtl_service_queue *sq)
256 } \ 256 } \
257} while (0) 257} while (0)
258 258
259static void tg_stats_init(struct tg_stats_cpu *tg_stats)
260{
261 blkg_rwstat_init(&tg_stats->service_bytes);
262 blkg_rwstat_init(&tg_stats->serviced);
263}
264
259/* 265/*
260 * Worker for allocating per cpu stat for tgs. This is scheduled on the 266 * Worker for allocating per cpu stat for tgs. This is scheduled on the
261 * system_wq once there are some groups on the alloc_list waiting for 267 * system_wq once there are some groups on the alloc_list waiting for
@@ -269,12 +275,16 @@ static void tg_stats_alloc_fn(struct work_struct *work)
269 275
270alloc_stats: 276alloc_stats:
271 if (!stats_cpu) { 277 if (!stats_cpu) {
278 int cpu;
279
272 stats_cpu = alloc_percpu(struct tg_stats_cpu); 280 stats_cpu = alloc_percpu(struct tg_stats_cpu);
273 if (!stats_cpu) { 281 if (!stats_cpu) {
274 /* allocation failed, try again after some time */ 282 /* allocation failed, try again after some time */
275 schedule_delayed_work(dwork, msecs_to_jiffies(10)); 283 schedule_delayed_work(dwork, msecs_to_jiffies(10));
276 return; 284 return;
277 } 285 }
286 for_each_possible_cpu(cpu)
287 tg_stats_init(per_cpu_ptr(stats_cpu, cpu));
278 } 288 }
279 289
280 spin_lock_irq(&tg_stats_alloc_lock); 290 spin_lock_irq(&tg_stats_alloc_lock);
diff --git a/block/cfq-iosched.c b/block/cfq-iosched.c
index 434944cbd761..4d5cec1ad80d 100644
--- a/block/cfq-iosched.c
+++ b/block/cfq-iosched.c
@@ -1508,6 +1508,29 @@ static void cfq_init_cfqg_base(struct cfq_group *cfqg)
1508} 1508}
1509 1509
1510#ifdef CONFIG_CFQ_GROUP_IOSCHED 1510#ifdef CONFIG_CFQ_GROUP_IOSCHED
1511static void cfqg_stats_init(struct cfqg_stats *stats)
1512{
1513 blkg_rwstat_init(&stats->service_bytes);
1514 blkg_rwstat_init(&stats->serviced);
1515 blkg_rwstat_init(&stats->merged);
1516 blkg_rwstat_init(&stats->service_time);
1517 blkg_rwstat_init(&stats->wait_time);
1518 blkg_rwstat_init(&stats->queued);
1519
1520 blkg_stat_init(&stats->sectors);
1521 blkg_stat_init(&stats->time);
1522
1523#ifdef CONFIG_DEBUG_BLK_CGROUP
1524 blkg_stat_init(&stats->unaccounted_time);
1525 blkg_stat_init(&stats->avg_queue_size_sum);
1526 blkg_stat_init(&stats->avg_queue_size_samples);
1527 blkg_stat_init(&stats->dequeue);
1528 blkg_stat_init(&stats->group_wait_time);
1529 blkg_stat_init(&stats->idle_time);
1530 blkg_stat_init(&stats->empty_time);
1531#endif
1532}
1533
1511static void cfq_pd_init(struct blkcg_gq *blkg) 1534static void cfq_pd_init(struct blkcg_gq *blkg)
1512{ 1535{
1513 struct cfq_group *cfqg = blkg_to_cfqg(blkg); 1536 struct cfq_group *cfqg = blkg_to_cfqg(blkg);
@@ -1515,6 +1538,8 @@ static void cfq_pd_init(struct blkcg_gq *blkg)
1515 cfq_init_cfqg_base(cfqg); 1538 cfq_init_cfqg_base(cfqg);
1516 cfqg->weight = blkg->blkcg->cfq_weight; 1539 cfqg->weight = blkg->blkcg->cfq_weight;
1517 cfqg->leaf_weight = blkg->blkcg->cfq_leaf_weight; 1540 cfqg->leaf_weight = blkg->blkcg->cfq_leaf_weight;
1541 cfqg_stats_init(&cfqg->stats);
1542 cfqg_stats_init(&cfqg->dead_stats);
1518} 1543}
1519 1544
1520static void cfq_pd_offline(struct blkcg_gq *blkg) 1545static void cfq_pd_offline(struct blkcg_gq *blkg)
diff --git a/drivers/net/dummy.c b/drivers/net/dummy.c
index b710c6b2d659..bd8f84b0b894 100644
--- a/drivers/net/dummy.c
+++ b/drivers/net/dummy.c
@@ -88,10 +88,16 @@ static netdev_tx_t dummy_xmit(struct sk_buff *skb, struct net_device *dev)
88 88
89static int dummy_dev_init(struct net_device *dev) 89static int dummy_dev_init(struct net_device *dev)
90{ 90{
91 int i;
91 dev->dstats = alloc_percpu(struct pcpu_dstats); 92 dev->dstats = alloc_percpu(struct pcpu_dstats);
92 if (!dev->dstats) 93 if (!dev->dstats)
93 return -ENOMEM; 94 return -ENOMEM;
94 95
96 for_each_possible_cpu(i) {
97 struct pcpu_dstats *dstats;
98 dstats = per_cpu_ptr(dev->dstats, i);
99 u64_stats_init(&dstats->syncp);
100 }
95 return 0; 101 return 0;
96} 102}
97 103
diff --git a/drivers/net/ethernet/emulex/benet/be_main.c b/drivers/net/ethernet/emulex/benet/be_main.c
index cb2bb6fccbc8..eaecaadfa8c5 100644
--- a/drivers/net/ethernet/emulex/benet/be_main.c
+++ b/drivers/net/ethernet/emulex/benet/be_main.c
@@ -2148,6 +2148,9 @@ static int be_tx_qs_create(struct be_adapter *adapter)
2148 if (status) 2148 if (status)
2149 return status; 2149 return status;
2150 2150
2151 u64_stats_init(&txo->stats.sync);
2152 u64_stats_init(&txo->stats.sync_compl);
2153
2151 /* If num_evt_qs is less than num_tx_qs, then more than 2154 /* If num_evt_qs is less than num_tx_qs, then more than
2152 * one txq share an eq 2155 * one txq share an eq
2153 */ 2156 */
@@ -2209,6 +2212,7 @@ static int be_rx_cqs_create(struct be_adapter *adapter)
2209 if (rc) 2212 if (rc)
2210 return rc; 2213 return rc;
2211 2214
2215 u64_stats_init(&rxo->stats.sync);
2212 eq = &adapter->eq_obj[i % adapter->num_evt_qs].q; 2216 eq = &adapter->eq_obj[i % adapter->num_evt_qs].q;
2213 rc = be_cmd_cq_create(adapter, cq, eq, false, 3); 2217 rc = be_cmd_cq_create(adapter, cq, eq, false, 3);
2214 if (rc) 2218 if (rc)
diff --git a/drivers/net/ethernet/intel/igb/igb_main.c b/drivers/net/ethernet/intel/igb/igb_main.c
index 2ac14bdd5fbb..025e5f4b7481 100644
--- a/drivers/net/ethernet/intel/igb/igb_main.c
+++ b/drivers/net/ethernet/intel/igb/igb_main.c
@@ -1224,6 +1224,9 @@ static int igb_alloc_q_vector(struct igb_adapter *adapter,
1224 ring->count = adapter->tx_ring_count; 1224 ring->count = adapter->tx_ring_count;
1225 ring->queue_index = txr_idx; 1225 ring->queue_index = txr_idx;
1226 1226
1227 u64_stats_init(&ring->tx_syncp);
1228 u64_stats_init(&ring->tx_syncp2);
1229
1227 /* assign ring to adapter */ 1230 /* assign ring to adapter */
1228 adapter->tx_ring[txr_idx] = ring; 1231 adapter->tx_ring[txr_idx] = ring;
1229 1232
@@ -1257,6 +1260,8 @@ static int igb_alloc_q_vector(struct igb_adapter *adapter,
1257 ring->count = adapter->rx_ring_count; 1260 ring->count = adapter->rx_ring_count;
1258 ring->queue_index = rxr_idx; 1261 ring->queue_index = rxr_idx;
1259 1262
1263 u64_stats_init(&ring->rx_syncp);
1264
1260 /* assign ring to adapter */ 1265 /* assign ring to adapter */
1261 adapter->rx_ring[rxr_idx] = ring; 1266 adapter->rx_ring[rxr_idx] = ring;
1262 } 1267 }
diff --git a/drivers/net/ethernet/intel/ixgbe/ixgbe_main.c b/drivers/net/ethernet/intel/ixgbe/ixgbe_main.c
index 0066f0aefbfa..0c55079ebee3 100644
--- a/drivers/net/ethernet/intel/ixgbe/ixgbe_main.c
+++ b/drivers/net/ethernet/intel/ixgbe/ixgbe_main.c
@@ -5085,6 +5085,8 @@ int ixgbe_setup_tx_resources(struct ixgbe_ring *tx_ring)
5085 if (!tx_ring->tx_buffer_info) 5085 if (!tx_ring->tx_buffer_info)
5086 goto err; 5086 goto err;
5087 5087
5088 u64_stats_init(&tx_ring->syncp);
5089
5088 /* round up to nearest 4K */ 5090 /* round up to nearest 4K */
5089 tx_ring->size = tx_ring->count * sizeof(union ixgbe_adv_tx_desc); 5091 tx_ring->size = tx_ring->count * sizeof(union ixgbe_adv_tx_desc);
5090 tx_ring->size = ALIGN(tx_ring->size, 4096); 5092 tx_ring->size = ALIGN(tx_ring->size, 4096);
@@ -5167,6 +5169,8 @@ int ixgbe_setup_rx_resources(struct ixgbe_ring *rx_ring)
5167 if (!rx_ring->rx_buffer_info) 5169 if (!rx_ring->rx_buffer_info)
5168 goto err; 5170 goto err;
5169 5171
5172 u64_stats_init(&rx_ring->syncp);
5173
5170 /* Round up to nearest 4K */ 5174 /* Round up to nearest 4K */
5171 rx_ring->size = rx_ring->count * sizeof(union ixgbe_adv_rx_desc); 5175 rx_ring->size = rx_ring->count * sizeof(union ixgbe_adv_rx_desc);
5172 rx_ring->size = ALIGN(rx_ring->size, 4096); 5176 rx_ring->size = ALIGN(rx_ring->size, 4096);
diff --git a/drivers/net/ethernet/marvell/mvneta.c b/drivers/net/ethernet/marvell/mvneta.c
index 7d99e695a110..b8e232b4ea2d 100644
--- a/drivers/net/ethernet/marvell/mvneta.c
+++ b/drivers/net/ethernet/marvell/mvneta.c
@@ -2792,6 +2792,9 @@ static int mvneta_probe(struct platform_device *pdev)
2792 2792
2793 pp = netdev_priv(dev); 2793 pp = netdev_priv(dev);
2794 2794
2795 u64_stats_init(&pp->tx_stats.syncp);
2796 u64_stats_init(&pp->rx_stats.syncp);
2797
2795 pp->weight = MVNETA_RX_POLL_WEIGHT; 2798 pp->weight = MVNETA_RX_POLL_WEIGHT;
2796 pp->phy_node = phy_node; 2799 pp->phy_node = phy_node;
2797 pp->phy_interface = phy_mode; 2800 pp->phy_interface = phy_mode;
diff --git a/drivers/net/ethernet/marvell/sky2.c b/drivers/net/ethernet/marvell/sky2.c
index a7df981d2123..43aa7acd84a6 100644
--- a/drivers/net/ethernet/marvell/sky2.c
+++ b/drivers/net/ethernet/marvell/sky2.c
@@ -4763,6 +4763,9 @@ static struct net_device *sky2_init_netdev(struct sky2_hw *hw, unsigned port,
4763 sky2->hw = hw; 4763 sky2->hw = hw;
4764 sky2->msg_enable = netif_msg_init(debug, default_msg); 4764 sky2->msg_enable = netif_msg_init(debug, default_msg);
4765 4765
4766 u64_stats_init(&sky2->tx_stats.syncp);
4767 u64_stats_init(&sky2->rx_stats.syncp);
4768
4766 /* Auto speed and flow control */ 4769 /* Auto speed and flow control */
4767 sky2->flags = SKY2_FLAG_AUTO_SPEED | SKY2_FLAG_AUTO_PAUSE; 4770 sky2->flags = SKY2_FLAG_AUTO_SPEED | SKY2_FLAG_AUTO_PAUSE;
4768 if (hw->chip_id != CHIP_ID_YUKON_XL) 4771 if (hw->chip_id != CHIP_ID_YUKON_XL)
diff --git a/drivers/net/ethernet/neterion/vxge/vxge-main.c b/drivers/net/ethernet/neterion/vxge/vxge-main.c
index 8614eeb7de81..f9876ea8c8bf 100644
--- a/drivers/net/ethernet/neterion/vxge/vxge-main.c
+++ b/drivers/net/ethernet/neterion/vxge/vxge-main.c
@@ -2072,6 +2072,10 @@ static int vxge_open_vpaths(struct vxgedev *vdev)
2072 vdev->config.tx_steering_type; 2072 vdev->config.tx_steering_type;
2073 vpath->fifo.ndev = vdev->ndev; 2073 vpath->fifo.ndev = vdev->ndev;
2074 vpath->fifo.pdev = vdev->pdev; 2074 vpath->fifo.pdev = vdev->pdev;
2075
2076 u64_stats_init(&vpath->fifo.stats.syncp);
2077 u64_stats_init(&vpath->ring.stats.syncp);
2078
2075 if (vdev->config.tx_steering_type) 2079 if (vdev->config.tx_steering_type)
2076 vpath->fifo.txq = 2080 vpath->fifo.txq =
2077 netdev_get_tx_queue(vdev->ndev, i); 2081 netdev_get_tx_queue(vdev->ndev, i);
diff --git a/drivers/net/ethernet/nvidia/forcedeth.c b/drivers/net/ethernet/nvidia/forcedeth.c
index 098b96dad66f..2d045be4b5cf 100644
--- a/drivers/net/ethernet/nvidia/forcedeth.c
+++ b/drivers/net/ethernet/nvidia/forcedeth.c
@@ -5619,6 +5619,8 @@ static int nv_probe(struct pci_dev *pci_dev, const struct pci_device_id *id)
5619 spin_lock_init(&np->lock); 5619 spin_lock_init(&np->lock);
5620 spin_lock_init(&np->hwstats_lock); 5620 spin_lock_init(&np->hwstats_lock);
5621 SET_NETDEV_DEV(dev, &pci_dev->dev); 5621 SET_NETDEV_DEV(dev, &pci_dev->dev);
5622 u64_stats_init(&np->swstats_rx_syncp);
5623 u64_stats_init(&np->swstats_tx_syncp);
5622 5624
5623 init_timer(&np->oom_kick); 5625 init_timer(&np->oom_kick);
5624 np->oom_kick.data = (unsigned long) dev; 5626 np->oom_kick.data = (unsigned long) dev;
diff --git a/drivers/net/ethernet/realtek/8139too.c b/drivers/net/ethernet/realtek/8139too.c
index 50a92104dd0a..da5972eefdd2 100644
--- a/drivers/net/ethernet/realtek/8139too.c
+++ b/drivers/net/ethernet/realtek/8139too.c
@@ -790,6 +790,9 @@ static struct net_device *rtl8139_init_board(struct pci_dev *pdev)
790 790
791 pci_set_master (pdev); 791 pci_set_master (pdev);
792 792
793 u64_stats_init(&tp->rx_stats.syncp);
794 u64_stats_init(&tp->tx_stats.syncp);
795
793retry: 796retry:
794 /* PIO bar register comes first. */ 797 /* PIO bar register comes first. */
795 bar = !use_io; 798 bar = !use_io;
diff --git a/drivers/net/ethernet/tile/tilepro.c b/drivers/net/ethernet/tile/tilepro.c
index 106be47716e7..edb2e12a0fe2 100644
--- a/drivers/net/ethernet/tile/tilepro.c
+++ b/drivers/net/ethernet/tile/tilepro.c
@@ -1008,6 +1008,8 @@ static void tile_net_register(void *dev_ptr)
1008 info->egress_timer.data = (long)info; 1008 info->egress_timer.data = (long)info;
1009 info->egress_timer.function = tile_net_handle_egress_timer; 1009 info->egress_timer.function = tile_net_handle_egress_timer;
1010 1010
1011 u64_stats_init(&info->stats.syncp);
1012
1011 priv->cpu[my_cpu] = info; 1013 priv->cpu[my_cpu] = info;
1012 1014
1013 /* 1015 /*
diff --git a/drivers/net/ethernet/via/via-rhine.c b/drivers/net/ethernet/via/via-rhine.c
index 4a7293ed95e9..cce6c4bc556a 100644
--- a/drivers/net/ethernet/via/via-rhine.c
+++ b/drivers/net/ethernet/via/via-rhine.c
@@ -987,6 +987,9 @@ static int rhine_init_one(struct pci_dev *pdev, const struct pci_device_id *ent)
987 987
988 rp->base = ioaddr; 988 rp->base = ioaddr;
989 989
990 u64_stats_init(&rp->tx_stats.syncp);
991 u64_stats_init(&rp->rx_stats.syncp);
992
990 /* Get chip registers into a sane state */ 993 /* Get chip registers into a sane state */
991 rhine_power_init(dev); 994 rhine_power_init(dev);
992 rhine_hw_init(dev, pioaddr); 995 rhine_hw_init(dev, pioaddr);
diff --git a/drivers/net/ifb.c b/drivers/net/ifb.c
index a3bed28197d2..c14d39bf32d0 100644
--- a/drivers/net/ifb.c
+++ b/drivers/net/ifb.c
@@ -265,6 +265,7 @@ MODULE_PARM_DESC(numifbs, "Number of ifb devices");
265static int __init ifb_init_one(int index) 265static int __init ifb_init_one(int index)
266{ 266{
267 struct net_device *dev_ifb; 267 struct net_device *dev_ifb;
268 struct ifb_private *dp;
268 int err; 269 int err;
269 270
270 dev_ifb = alloc_netdev(sizeof(struct ifb_private), 271 dev_ifb = alloc_netdev(sizeof(struct ifb_private),
@@ -273,6 +274,10 @@ static int __init ifb_init_one(int index)
273 if (!dev_ifb) 274 if (!dev_ifb)
274 return -ENOMEM; 275 return -ENOMEM;
275 276
277 dp = netdev_priv(dev_ifb);
278 u64_stats_init(&dp->rsync);
279 u64_stats_init(&dp->tsync);
280
276 dev_ifb->rtnl_link_ops = &ifb_link_ops; 281 dev_ifb->rtnl_link_ops = &ifb_link_ops;
277 err = register_netdevice(dev_ifb); 282 err = register_netdevice(dev_ifb);
278 if (err < 0) 283 if (err < 0)
diff --git a/drivers/net/loopback.c b/drivers/net/loopback.c
index a17d85a331f1..ac24c27b4b2d 100644
--- a/drivers/net/loopback.c
+++ b/drivers/net/loopback.c
@@ -137,10 +137,16 @@ static const struct ethtool_ops loopback_ethtool_ops = {
137 137
138static int loopback_dev_init(struct net_device *dev) 138static int loopback_dev_init(struct net_device *dev)
139{ 139{
140 int i;
140 dev->lstats = alloc_percpu(struct pcpu_lstats); 141 dev->lstats = alloc_percpu(struct pcpu_lstats);
141 if (!dev->lstats) 142 if (!dev->lstats)
142 return -ENOMEM; 143 return -ENOMEM;
143 144
145 for_each_possible_cpu(i) {
146 struct pcpu_lstats *lb_stats;
147 lb_stats = per_cpu_ptr(dev->lstats, i);
148 u64_stats_init(&lb_stats->syncp);
149 }
144 return 0; 150 return 0;
145} 151}
146 152
diff --git a/drivers/net/macvlan.c b/drivers/net/macvlan.c
index af4aaa5893ff..acf93798dc67 100644
--- a/drivers/net/macvlan.c
+++ b/drivers/net/macvlan.c
@@ -534,6 +534,7 @@ static int macvlan_init(struct net_device *dev)
534{ 534{
535 struct macvlan_dev *vlan = netdev_priv(dev); 535 struct macvlan_dev *vlan = netdev_priv(dev);
536 const struct net_device *lowerdev = vlan->lowerdev; 536 const struct net_device *lowerdev = vlan->lowerdev;
537 int i;
537 538
538 dev->state = (dev->state & ~MACVLAN_STATE_MASK) | 539 dev->state = (dev->state & ~MACVLAN_STATE_MASK) |
539 (lowerdev->state & MACVLAN_STATE_MASK); 540 (lowerdev->state & MACVLAN_STATE_MASK);
@@ -549,6 +550,12 @@ static int macvlan_init(struct net_device *dev)
549 if (!vlan->pcpu_stats) 550 if (!vlan->pcpu_stats)
550 return -ENOMEM; 551 return -ENOMEM;
551 552
553 for_each_possible_cpu(i) {
554 struct macvlan_pcpu_stats *mvlstats;
555 mvlstats = per_cpu_ptr(vlan->pcpu_stats, i);
556 u64_stats_init(&mvlstats->syncp);
557 }
558
552 return 0; 559 return 0;
553} 560}
554 561
diff --git a/drivers/net/nlmon.c b/drivers/net/nlmon.c
index b57ce5f48962..d2bb12bfabd5 100644
--- a/drivers/net/nlmon.c
+++ b/drivers/net/nlmon.c
@@ -47,8 +47,16 @@ static int nlmon_change_mtu(struct net_device *dev, int new_mtu)
47 47
48static int nlmon_dev_init(struct net_device *dev) 48static int nlmon_dev_init(struct net_device *dev)
49{ 49{
50 int i;
51
50 dev->lstats = alloc_percpu(struct pcpu_lstats); 52 dev->lstats = alloc_percpu(struct pcpu_lstats);
51 53
54 for_each_possible_cpu(i) {
55 struct pcpu_lstats *nlmstats;
56 nlmstats = per_cpu_ptr(dev->lstats, i);
57 u64_stats_init(&nlmstats->syncp);
58 }
59
52 return dev->lstats == NULL ? -ENOMEM : 0; 60 return dev->lstats == NULL ? -ENOMEM : 0;
53} 61}
54 62
diff --git a/drivers/net/team/team.c b/drivers/net/team/team.c
index 50e43e64d51d..6574eb8766f9 100644
--- a/drivers/net/team/team.c
+++ b/drivers/net/team/team.c
@@ -1540,6 +1540,12 @@ static int team_init(struct net_device *dev)
1540 if (!team->pcpu_stats) 1540 if (!team->pcpu_stats)
1541 return -ENOMEM; 1541 return -ENOMEM;
1542 1542
1543 for_each_possible_cpu(i) {
1544 struct team_pcpu_stats *team_stats;
1545 team_stats = per_cpu_ptr(team->pcpu_stats, i);
1546 u64_stats_init(&team_stats->syncp);
1547 }
1548
1543 for (i = 0; i < TEAM_PORT_HASHENTRIES; i++) 1549 for (i = 0; i < TEAM_PORT_HASHENTRIES; i++)
1544 INIT_HLIST_HEAD(&team->en_port_hlist[i]); 1550 INIT_HLIST_HEAD(&team->en_port_hlist[i]);
1545 INIT_LIST_HEAD(&team->port_list); 1551 INIT_LIST_HEAD(&team->port_list);
diff --git a/drivers/net/team/team_mode_loadbalance.c b/drivers/net/team/team_mode_loadbalance.c
index 829a9cd2b4da..d671fc3ac5ac 100644
--- a/drivers/net/team/team_mode_loadbalance.c
+++ b/drivers/net/team/team_mode_loadbalance.c
@@ -570,7 +570,7 @@ static int lb_init(struct team *team)
570{ 570{
571 struct lb_priv *lb_priv = get_lb_priv(team); 571 struct lb_priv *lb_priv = get_lb_priv(team);
572 lb_select_tx_port_func_t *func; 572 lb_select_tx_port_func_t *func;
573 int err; 573 int i, err;
574 574
575 /* set default tx port selector */ 575 /* set default tx port selector */
576 func = lb_select_tx_port_get_func("hash"); 576 func = lb_select_tx_port_get_func("hash");
@@ -588,6 +588,13 @@ static int lb_init(struct team *team)
588 goto err_alloc_pcpu_stats; 588 goto err_alloc_pcpu_stats;
589 } 589 }
590 590
591 for_each_possible_cpu(i) {
592 struct lb_pcpu_stats *team_lb_stats;
593 team_lb_stats = per_cpu_ptr(lb_priv->pcpu_stats, i);
594 u64_stats_init(&team_lb_stats->syncp);
595 }
596
597
591 INIT_DELAYED_WORK(&lb_priv->ex->stats.refresh_dw, lb_stats_refresh); 598 INIT_DELAYED_WORK(&lb_priv->ex->stats.refresh_dw, lb_stats_refresh);
592 599
593 err = team_options_register(team, lb_options, ARRAY_SIZE(lb_options)); 600 err = team_options_register(team, lb_options, ARRAY_SIZE(lb_options));
diff --git a/drivers/net/veth.c b/drivers/net/veth.c
index b24db7acbf12..2ec2041b62d4 100644
--- a/drivers/net/veth.c
+++ b/drivers/net/veth.c
@@ -235,10 +235,18 @@ static int veth_change_mtu(struct net_device *dev, int new_mtu)
235 235
236static int veth_dev_init(struct net_device *dev) 236static int veth_dev_init(struct net_device *dev)
237{ 237{
238 int i;
239
238 dev->vstats = alloc_percpu(struct pcpu_vstats); 240 dev->vstats = alloc_percpu(struct pcpu_vstats);
239 if (!dev->vstats) 241 if (!dev->vstats)
240 return -ENOMEM; 242 return -ENOMEM;
241 243
244 for_each_possible_cpu(i) {
245 struct pcpu_vstats *veth_stats;
246 veth_stats = per_cpu_ptr(dev->vstats, i);
247 u64_stats_init(&veth_stats->syncp);
248 }
249
242 return 0; 250 return 0;
243} 251}
244 252
diff --git a/drivers/net/virtio_net.c b/drivers/net/virtio_net.c
index 01f4eb5c8b78..bf7c734259ad 100644
--- a/drivers/net/virtio_net.c
+++ b/drivers/net/virtio_net.c
@@ -1576,6 +1576,13 @@ static int virtnet_probe(struct virtio_device *vdev)
1576 if (vi->stats == NULL) 1576 if (vi->stats == NULL)
1577 goto free; 1577 goto free;
1578 1578
1579 for_each_possible_cpu(i) {
1580 struct virtnet_stats *virtnet_stats;
1581 virtnet_stats = per_cpu_ptr(vi->stats, i);
1582 u64_stats_init(&virtnet_stats->tx_syncp);
1583 u64_stats_init(&virtnet_stats->rx_syncp);
1584 }
1585
1579 mutex_init(&vi->config_lock); 1586 mutex_init(&vi->config_lock);
1580 vi->config_enable = true; 1587 vi->config_enable = true;
1581 INIT_WORK(&vi->config_work, virtnet_config_changed_work); 1588 INIT_WORK(&vi->config_work, virtnet_config_changed_work);
diff --git a/drivers/net/vxlan.c b/drivers/net/vxlan.c
index 78df8f39e57c..0358c07f7669 100644
--- a/drivers/net/vxlan.c
+++ b/drivers/net/vxlan.c
@@ -1880,11 +1880,19 @@ static int vxlan_init(struct net_device *dev)
1880 struct vxlan_dev *vxlan = netdev_priv(dev); 1880 struct vxlan_dev *vxlan = netdev_priv(dev);
1881 struct vxlan_net *vn = net_generic(dev_net(dev), vxlan_net_id); 1881 struct vxlan_net *vn = net_generic(dev_net(dev), vxlan_net_id);
1882 struct vxlan_sock *vs; 1882 struct vxlan_sock *vs;
1883 int i;
1883 1884
1884 dev->tstats = alloc_percpu(struct pcpu_tstats); 1885 dev->tstats = alloc_percpu(struct pcpu_tstats);
1885 if (!dev->tstats) 1886 if (!dev->tstats)
1886 return -ENOMEM; 1887 return -ENOMEM;
1887 1888
1889 for_each_possible_cpu(i) {
1890 struct pcpu_tstats *vxlan_stats;
1891 vxlan_stats = per_cpu_ptr(dev->tstats, i);
1892 u64_stats_init(&vxlan_stats->syncp);
1893 }
1894
1895
1888 spin_lock(&vn->sock_lock); 1896 spin_lock(&vn->sock_lock);
1889 vs = vxlan_find_sock(dev_net(dev), vxlan->dst_port); 1897 vs = vxlan_find_sock(dev_net(dev), vxlan->dst_port);
1890 if (vs) { 1898 if (vs) {
diff --git a/drivers/net/xen-netfront.c b/drivers/net/xen-netfront.c
index dd1011e55cb5..d85e66979711 100644
--- a/drivers/net/xen-netfront.c
+++ b/drivers/net/xen-netfront.c
@@ -1340,6 +1340,12 @@ static struct net_device *xennet_create_dev(struct xenbus_device *dev)
1340 if (np->stats == NULL) 1340 if (np->stats == NULL)
1341 goto exit; 1341 goto exit;
1342 1342
1343 for_each_possible_cpu(i) {
1344 struct netfront_stats *xen_nf_stats;
1345 xen_nf_stats = per_cpu_ptr(np->stats, i);
1346 u64_stats_init(&xen_nf_stats->syncp);
1347 }
1348
1343 /* Initialise tx_skbs as a free chain containing every entry. */ 1349 /* Initialise tx_skbs as a free chain containing every entry. */
1344 np->tx_skb_freelist = 0; 1350 np->tx_skb_freelist = 0;
1345 for (i = 0; i < NET_TX_RING_SIZE; i++) { 1351 for (i = 0; i < NET_TX_RING_SIZE; i++) {
diff --git a/fs/dcache.c b/fs/dcache.c
index a9dd384c5e80..0a38ef8d7f00 100644
--- a/fs/dcache.c
+++ b/fs/dcache.c
@@ -2606,7 +2606,7 @@ static void __d_move(struct dentry * dentry, struct dentry * target)
2606 dentry_lock_for_move(dentry, target); 2606 dentry_lock_for_move(dentry, target);
2607 2607
2608 write_seqcount_begin(&dentry->d_seq); 2608 write_seqcount_begin(&dentry->d_seq);
2609 write_seqcount_begin(&target->d_seq); 2609 write_seqcount_begin_nested(&target->d_seq, DENTRY_D_LOCK_NESTED);
2610 2610
2611 /* __d_drop does write_seqcount_barrier, but they're OK to nest. */ 2611 /* __d_drop does write_seqcount_barrier, but they're OK to nest. */
2612 2612
@@ -2738,7 +2738,7 @@ static void __d_materialise_dentry(struct dentry *dentry, struct dentry *anon)
2738 dentry_lock_for_move(anon, dentry); 2738 dentry_lock_for_move(anon, dentry);
2739 2739
2740 write_seqcount_begin(&dentry->d_seq); 2740 write_seqcount_begin(&dentry->d_seq);
2741 write_seqcount_begin(&anon->d_seq); 2741 write_seqcount_begin_nested(&anon->d_seq, DENTRY_D_LOCK_NESTED);
2742 2742
2743 dparent = dentry->d_parent; 2743 dparent = dentry->d_parent;
2744 2744
diff --git a/fs/fs_struct.c b/fs/fs_struct.c
index d8ac61d0c932..7dca743b2ce1 100644
--- a/fs/fs_struct.c
+++ b/fs/fs_struct.c
@@ -161,6 +161,6 @@ EXPORT_SYMBOL(current_umask);
161struct fs_struct init_fs = { 161struct fs_struct init_fs = {
162 .users = 1, 162 .users = 1,
163 .lock = __SPIN_LOCK_UNLOCKED(init_fs.lock), 163 .lock = __SPIN_LOCK_UNLOCKED(init_fs.lock),
164 .seq = SEQCNT_ZERO, 164 .seq = SEQCNT_ZERO(init_fs.seq),
165 .umask = 0022, 165 .umask = 0022,
166}; 166};
diff --git a/include/linux/cpuset.h b/include/linux/cpuset.h
index cc1b01cf2035..3fe661fe96d1 100644
--- a/include/linux/cpuset.h
+++ b/include/linux/cpuset.h
@@ -110,10 +110,14 @@ static inline bool put_mems_allowed(unsigned int seq)
110 110
111static inline void set_mems_allowed(nodemask_t nodemask) 111static inline void set_mems_allowed(nodemask_t nodemask)
112{ 112{
113 unsigned long flags;
114
113 task_lock(current); 115 task_lock(current);
116 local_irq_save(flags);
114 write_seqcount_begin(&current->mems_allowed_seq); 117 write_seqcount_begin(&current->mems_allowed_seq);
115 current->mems_allowed = nodemask; 118 current->mems_allowed = nodemask;
116 write_seqcount_end(&current->mems_allowed_seq); 119 write_seqcount_end(&current->mems_allowed_seq);
120 local_irq_restore(flags);
117 task_unlock(current); 121 task_unlock(current);
118} 122}
119 123
diff --git a/include/linux/init_task.h b/include/linux/init_task.h
index 5cd0f0949927..b0ed422e4e4a 100644
--- a/include/linux/init_task.h
+++ b/include/linux/init_task.h
@@ -32,10 +32,10 @@ extern struct fs_struct init_fs;
32#endif 32#endif
33 33
34#ifdef CONFIG_CPUSETS 34#ifdef CONFIG_CPUSETS
35#define INIT_CPUSET_SEQ \ 35#define INIT_CPUSET_SEQ(tsk) \
36 .mems_allowed_seq = SEQCNT_ZERO, 36 .mems_allowed_seq = SEQCNT_ZERO(tsk.mems_allowed_seq),
37#else 37#else
38#define INIT_CPUSET_SEQ 38#define INIT_CPUSET_SEQ(tsk)
39#endif 39#endif
40 40
41#define INIT_SIGNALS(sig) { \ 41#define INIT_SIGNALS(sig) { \
@@ -220,7 +220,7 @@ extern struct task_group root_task_group;
220 INIT_FTRACE_GRAPH \ 220 INIT_FTRACE_GRAPH \
221 INIT_TRACE_RECURSION \ 221 INIT_TRACE_RECURSION \
222 INIT_TASK_RCU_PREEMPT(tsk) \ 222 INIT_TASK_RCU_PREEMPT(tsk) \
223 INIT_CPUSET_SEQ \ 223 INIT_CPUSET_SEQ(tsk) \
224 INIT_VTIME(tsk) \ 224 INIT_VTIME(tsk) \
225} 225}
226 226
diff --git a/include/linux/lockdep.h b/include/linux/lockdep.h
index cfc2f119779a..92b1bfc5da60 100644
--- a/include/linux/lockdep.h
+++ b/include/linux/lockdep.h
@@ -497,6 +497,10 @@ static inline void print_irqtrace_events(struct task_struct *curr)
497#define rwlock_acquire_read(l, s, t, i) lock_acquire_shared_recursive(l, s, t, NULL, i) 497#define rwlock_acquire_read(l, s, t, i) lock_acquire_shared_recursive(l, s, t, NULL, i)
498#define rwlock_release(l, n, i) lock_release(l, n, i) 498#define rwlock_release(l, n, i) lock_release(l, n, i)
499 499
500#define seqcount_acquire(l, s, t, i) lock_acquire_exclusive(l, s, t, NULL, i)
501#define seqcount_acquire_read(l, s, t, i) lock_acquire_shared_recursive(l, s, t, NULL, i)
502#define seqcount_release(l, n, i) lock_release(l, n, i)
503
500#define mutex_acquire(l, s, t, i) lock_acquire_exclusive(l, s, t, NULL, i) 504#define mutex_acquire(l, s, t, i) lock_acquire_exclusive(l, s, t, NULL, i)
501#define mutex_acquire_nest(l, s, t, n, i) lock_acquire_exclusive(l, s, t, n, i) 505#define mutex_acquire_nest(l, s, t, n, i) lock_acquire_exclusive(l, s, t, n, i)
502#define mutex_release(l, n, i) lock_release(l, n, i) 506#define mutex_release(l, n, i) lock_release(l, n, i)
@@ -504,11 +508,11 @@ static inline void print_irqtrace_events(struct task_struct *curr)
504#define rwsem_acquire(l, s, t, i) lock_acquire_exclusive(l, s, t, NULL, i) 508#define rwsem_acquire(l, s, t, i) lock_acquire_exclusive(l, s, t, NULL, i)
505#define rwsem_acquire_nest(l, s, t, n, i) lock_acquire_exclusive(l, s, t, n, i) 509#define rwsem_acquire_nest(l, s, t, n, i) lock_acquire_exclusive(l, s, t, n, i)
506#define rwsem_acquire_read(l, s, t, i) lock_acquire_shared(l, s, t, NULL, i) 510#define rwsem_acquire_read(l, s, t, i) lock_acquire_shared(l, s, t, NULL, i)
507# define rwsem_release(l, n, i) lock_release(l, n, i) 511#define rwsem_release(l, n, i) lock_release(l, n, i)
508 512
509#define lock_map_acquire(l) lock_acquire_exclusive(l, 0, 0, NULL, _THIS_IP_) 513#define lock_map_acquire(l) lock_acquire_exclusive(l, 0, 0, NULL, _THIS_IP_)
510#define lock_map_acquire_read(l) lock_acquire_shared_recursive(l, 0, 0, NULL, _THIS_IP_) 514#define lock_map_acquire_read(l) lock_acquire_shared_recursive(l, 0, 0, NULL, _THIS_IP_)
511# define lock_map_release(l) lock_release(l, 1, _THIS_IP_) 515#define lock_map_release(l) lock_release(l, 1, _THIS_IP_)
512 516
513#ifdef CONFIG_PROVE_LOCKING 517#ifdef CONFIG_PROVE_LOCKING
514# define might_lock(lock) \ 518# define might_lock(lock) \
diff --git a/include/linux/mutex.h b/include/linux/mutex.h
index bab49da8a0f0..d3181936c138 100644
--- a/include/linux/mutex.h
+++ b/include/linux/mutex.h
@@ -131,7 +131,7 @@ static inline int mutex_is_locked(struct mutex *lock)
131} 131}
132 132
133/* 133/*
134 * See kernel/mutex.c for detailed documentation of these APIs. 134 * See kernel/locking/mutex.c for detailed documentation of these APIs.
135 * Also see Documentation/mutex-design.txt. 135 * Also see Documentation/mutex-design.txt.
136 */ 136 */
137#ifdef CONFIG_DEBUG_LOCK_ALLOC 137#ifdef CONFIG_DEBUG_LOCK_ALLOC
diff --git a/include/linux/sched/sysctl.h b/include/linux/sched/sysctl.h
index 10d16c4fbe89..41467f8ff8ec 100644
--- a/include/linux/sched/sysctl.h
+++ b/include/linux/sched/sysctl.h
@@ -2,8 +2,8 @@
2#define _SCHED_SYSCTL_H 2#define _SCHED_SYSCTL_H
3 3
4#ifdef CONFIG_DETECT_HUNG_TASK 4#ifdef CONFIG_DETECT_HUNG_TASK
5extern int sysctl_hung_task_check_count;
5extern unsigned int sysctl_hung_task_panic; 6extern unsigned int sysctl_hung_task_panic;
6extern unsigned long sysctl_hung_task_check_count;
7extern unsigned long sysctl_hung_task_timeout_secs; 7extern unsigned long sysctl_hung_task_timeout_secs;
8extern unsigned long sysctl_hung_task_warnings; 8extern unsigned long sysctl_hung_task_warnings;
9extern int proc_dohung_task_timeout_secs(struct ctl_table *table, int write, 9extern int proc_dohung_task_timeout_secs(struct ctl_table *table, int write,
diff --git a/include/linux/seqlock.h b/include/linux/seqlock.h
index 21a209336e79..1e8a8b6e837d 100644
--- a/include/linux/seqlock.h
+++ b/include/linux/seqlock.h
@@ -34,6 +34,7 @@
34 34
35#include <linux/spinlock.h> 35#include <linux/spinlock.h>
36#include <linux/preempt.h> 36#include <linux/preempt.h>
37#include <linux/lockdep.h>
37#include <asm/processor.h> 38#include <asm/processor.h>
38 39
39/* 40/*
@@ -44,10 +45,50 @@
44 */ 45 */
45typedef struct seqcount { 46typedef struct seqcount {
46 unsigned sequence; 47 unsigned sequence;
48#ifdef CONFIG_DEBUG_LOCK_ALLOC
49 struct lockdep_map dep_map;
50#endif
47} seqcount_t; 51} seqcount_t;
48 52
49#define SEQCNT_ZERO { 0 } 53static inline void __seqcount_init(seqcount_t *s, const char *name,
50#define seqcount_init(x) do { *(x) = (seqcount_t) SEQCNT_ZERO; } while (0) 54 struct lock_class_key *key)
55{
56 /*
57 * Make sure we are not reinitializing a held lock:
58 */
59 lockdep_init_map(&s->dep_map, name, key, 0);
60 s->sequence = 0;
61}
62
63#ifdef CONFIG_DEBUG_LOCK_ALLOC
64# define SEQCOUNT_DEP_MAP_INIT(lockname) \
65 .dep_map = { .name = #lockname } \
66
67# define seqcount_init(s) \
68 do { \
69 static struct lock_class_key __key; \
70 __seqcount_init((s), #s, &__key); \
71 } while (0)
72
73static inline void seqcount_lockdep_reader_access(const seqcount_t *s)
74{
75 seqcount_t *l = (seqcount_t *)s;
76 unsigned long flags;
77
78 local_irq_save(flags);
79 seqcount_acquire_read(&l->dep_map, 0, 0, _RET_IP_);
80 seqcount_release(&l->dep_map, 1, _RET_IP_);
81 local_irq_restore(flags);
82}
83
84#else
85# define SEQCOUNT_DEP_MAP_INIT(lockname)
86# define seqcount_init(s) __seqcount_init(s, NULL, NULL)
87# define seqcount_lockdep_reader_access(x)
88#endif
89
90#define SEQCNT_ZERO(lockname) { .sequence = 0, SEQCOUNT_DEP_MAP_INIT(lockname)}
91
51 92
52/** 93/**
53 * __read_seqcount_begin - begin a seq-read critical section (without barrier) 94 * __read_seqcount_begin - begin a seq-read critical section (without barrier)
@@ -76,6 +117,22 @@ repeat:
76} 117}
77 118
78/** 119/**
120 * read_seqcount_begin_no_lockdep - start seq-read critical section w/o lockdep
121 * @s: pointer to seqcount_t
122 * Returns: count to be passed to read_seqcount_retry
123 *
124 * read_seqcount_begin_no_lockdep opens a read critical section of the given
125 * seqcount, but without any lockdep checking. Validity of the critical
126 * section is tested by checking read_seqcount_retry function.
127 */
128static inline unsigned read_seqcount_begin_no_lockdep(const seqcount_t *s)
129{
130 unsigned ret = __read_seqcount_begin(s);
131 smp_rmb();
132 return ret;
133}
134
135/**
79 * read_seqcount_begin - begin a seq-read critical section 136 * read_seqcount_begin - begin a seq-read critical section
80 * @s: pointer to seqcount_t 137 * @s: pointer to seqcount_t
81 * Returns: count to be passed to read_seqcount_retry 138 * Returns: count to be passed to read_seqcount_retry
@@ -86,9 +143,8 @@ repeat:
86 */ 143 */
87static inline unsigned read_seqcount_begin(const seqcount_t *s) 144static inline unsigned read_seqcount_begin(const seqcount_t *s)
88{ 145{
89 unsigned ret = __read_seqcount_begin(s); 146 seqcount_lockdep_reader_access(s);
90 smp_rmb(); 147 return read_seqcount_begin_no_lockdep(s);
91 return ret;
92} 148}
93 149
94/** 150/**
@@ -108,6 +164,8 @@ static inline unsigned read_seqcount_begin(const seqcount_t *s)
108static inline unsigned raw_seqcount_begin(const seqcount_t *s) 164static inline unsigned raw_seqcount_begin(const seqcount_t *s)
109{ 165{
110 unsigned ret = ACCESS_ONCE(s->sequence); 166 unsigned ret = ACCESS_ONCE(s->sequence);
167
168 seqcount_lockdep_reader_access(s);
111 smp_rmb(); 169 smp_rmb();
112 return ret & ~1; 170 return ret & ~1;
113} 171}
@@ -152,14 +210,21 @@ static inline int read_seqcount_retry(const seqcount_t *s, unsigned start)
152 * Sequence counter only version assumes that callers are using their 210 * Sequence counter only version assumes that callers are using their
153 * own mutexing. 211 * own mutexing.
154 */ 212 */
155static inline void write_seqcount_begin(seqcount_t *s) 213static inline void write_seqcount_begin_nested(seqcount_t *s, int subclass)
156{ 214{
157 s->sequence++; 215 s->sequence++;
158 smp_wmb(); 216 smp_wmb();
217 seqcount_acquire(&s->dep_map, subclass, 0, _RET_IP_);
218}
219
220static inline void write_seqcount_begin(seqcount_t *s)
221{
222 write_seqcount_begin_nested(s, 0);
159} 223}
160 224
161static inline void write_seqcount_end(seqcount_t *s) 225static inline void write_seqcount_end(seqcount_t *s)
162{ 226{
227 seqcount_release(&s->dep_map, 1, _RET_IP_);
163 smp_wmb(); 228 smp_wmb();
164 s->sequence++; 229 s->sequence++;
165} 230}
@@ -188,7 +253,7 @@ typedef struct {
188 */ 253 */
189#define __SEQLOCK_UNLOCKED(lockname) \ 254#define __SEQLOCK_UNLOCKED(lockname) \
190 { \ 255 { \
191 .seqcount = SEQCNT_ZERO, \ 256 .seqcount = SEQCNT_ZERO(lockname), \
192 .lock = __SPIN_LOCK_UNLOCKED(lockname) \ 257 .lock = __SPIN_LOCK_UNLOCKED(lockname) \
193 } 258 }
194 259
diff --git a/include/linux/u64_stats_sync.h b/include/linux/u64_stats_sync.h
index 8da8c4e87da3..7bfabd20204c 100644
--- a/include/linux/u64_stats_sync.h
+++ b/include/linux/u64_stats_sync.h
@@ -67,6 +67,13 @@ struct u64_stats_sync {
67#endif 67#endif
68}; 68};
69 69
70
71#if BITS_PER_LONG == 32 && defined(CONFIG_SMP)
72# define u64_stats_init(syncp) seqcount_init(syncp.seq)
73#else
74# define u64_stats_init(syncp) do { } while (0)
75#endif
76
70static inline void u64_stats_update_begin(struct u64_stats_sync *syncp) 77static inline void u64_stats_update_begin(struct u64_stats_sync *syncp)
71{ 78{
72#if BITS_PER_LONG==32 && defined(CONFIG_SMP) 79#if BITS_PER_LONG==32 && defined(CONFIG_SMP)
diff --git a/include/trace/events/sched.h b/include/trace/events/sched.h
index 613381bcde40..04c308413a5d 100644
--- a/include/trace/events/sched.h
+++ b/include/trace/events/sched.h
@@ -424,6 +424,25 @@ TRACE_EVENT(sched_pi_setprio,
424 __entry->oldprio, __entry->newprio) 424 __entry->oldprio, __entry->newprio)
425); 425);
426 426
427#ifdef CONFIG_DETECT_HUNG_TASK
428TRACE_EVENT(sched_process_hang,
429 TP_PROTO(struct task_struct *tsk),
430 TP_ARGS(tsk),
431
432 TP_STRUCT__entry(
433 __array( char, comm, TASK_COMM_LEN )
434 __field( pid_t, pid )
435 ),
436
437 TP_fast_assign(
438 memcpy(__entry->comm, tsk->comm, TASK_COMM_LEN);
439 __entry->pid = tsk->pid;
440 ),
441
442 TP_printk("comm=%s pid=%d", __entry->comm, __entry->pid)
443);
444#endif /* CONFIG_DETECT_HUNG_TASK */
445
427#endif /* _TRACE_SCHED_H */ 446#endif /* _TRACE_SCHED_H */
428 447
429/* This part must be outside protection */ 448/* This part must be outside protection */
diff --git a/kernel/Makefile b/kernel/Makefile
index a4d1aa8da9bc..09a9c94f42bd 100644
--- a/kernel/Makefile
+++ b/kernel/Makefile
@@ -7,22 +7,19 @@ obj-y = fork.o exec_domain.o panic.o \
7 sysctl.o sysctl_binary.o capability.o ptrace.o timer.o user.o \ 7 sysctl.o sysctl_binary.o capability.o ptrace.o timer.o user.o \
8 signal.o sys.o kmod.o workqueue.o pid.o task_work.o \ 8 signal.o sys.o kmod.o workqueue.o pid.o task_work.o \
9 extable.o params.o posix-timers.o \ 9 extable.o params.o posix-timers.o \
10 kthread.o sys_ni.o posix-cpu-timers.o mutex.o \ 10 kthread.o sys_ni.o posix-cpu-timers.o \
11 hrtimer.o rwsem.o nsproxy.o semaphore.o \ 11 hrtimer.o nsproxy.o \
12 notifier.o ksysfs.o cred.o reboot.o \ 12 notifier.o ksysfs.o cred.o reboot.o \
13 async.o range.o groups.o lglock.o smpboot.o 13 async.o range.o groups.o smpboot.o
14 14
15ifdef CONFIG_FUNCTION_TRACER 15ifdef CONFIG_FUNCTION_TRACER
16# Do not trace debug files and internal ftrace files 16# Do not trace debug files and internal ftrace files
17CFLAGS_REMOVE_lockdep.o = -pg
18CFLAGS_REMOVE_lockdep_proc.o = -pg
19CFLAGS_REMOVE_mutex-debug.o = -pg
20CFLAGS_REMOVE_rtmutex-debug.o = -pg
21CFLAGS_REMOVE_cgroup-debug.o = -pg 17CFLAGS_REMOVE_cgroup-debug.o = -pg
22CFLAGS_REMOVE_irq_work.o = -pg 18CFLAGS_REMOVE_irq_work.o = -pg
23endif 19endif
24 20
25obj-y += sched/ 21obj-y += sched/
22obj-y += locking/
26obj-y += power/ 23obj-y += power/
27obj-y += printk/ 24obj-y += printk/
28obj-y += cpu/ 25obj-y += cpu/
@@ -34,26 +31,15 @@ obj-$(CONFIG_FREEZER) += freezer.o
34obj-$(CONFIG_PROFILING) += profile.o 31obj-$(CONFIG_PROFILING) += profile.o
35obj-$(CONFIG_STACKTRACE) += stacktrace.o 32obj-$(CONFIG_STACKTRACE) += stacktrace.o
36obj-y += time/ 33obj-y += time/
37obj-$(CONFIG_DEBUG_MUTEXES) += mutex-debug.o
38obj-$(CONFIG_LOCKDEP) += lockdep.o
39ifeq ($(CONFIG_PROC_FS),y)
40obj-$(CONFIG_LOCKDEP) += lockdep_proc.o
41endif
42obj-$(CONFIG_FUTEX) += futex.o 34obj-$(CONFIG_FUTEX) += futex.o
43ifeq ($(CONFIG_COMPAT),y) 35ifeq ($(CONFIG_COMPAT),y)
44obj-$(CONFIG_FUTEX) += futex_compat.o 36obj-$(CONFIG_FUTEX) += futex_compat.o
45endif 37endif
46obj-$(CONFIG_RT_MUTEXES) += rtmutex.o
47obj-$(CONFIG_DEBUG_RT_MUTEXES) += rtmutex-debug.o
48obj-$(CONFIG_RT_MUTEX_TESTER) += rtmutex-tester.o
49obj-$(CONFIG_GENERIC_ISA_DMA) += dma.o 38obj-$(CONFIG_GENERIC_ISA_DMA) += dma.o
50obj-$(CONFIG_SMP) += smp.o 39obj-$(CONFIG_SMP) += smp.o
51ifneq ($(CONFIG_SMP),y) 40ifneq ($(CONFIG_SMP),y)
52obj-y += up.o 41obj-y += up.o
53endif 42endif
54obj-$(CONFIG_SMP) += spinlock.o
55obj-$(CONFIG_DEBUG_SPINLOCK) += spinlock.o
56obj-$(CONFIG_PROVE_LOCKING) += spinlock.o
57obj-$(CONFIG_UID16) += uid16.o 43obj-$(CONFIG_UID16) += uid16.o
58obj-$(CONFIG_MODULES) += module.o 44obj-$(CONFIG_MODULES) += module.o
59obj-$(CONFIG_MODULE_SIG) += module_signing.o modsign_pubkey.o modsign_certificate.o 45obj-$(CONFIG_MODULE_SIG) += module_signing.o modsign_pubkey.o modsign_certificate.o
diff --git a/kernel/futex.c b/kernel/futex.c
index c3a1a55a5214..80ba086f021d 100644
--- a/kernel/futex.c
+++ b/kernel/futex.c
@@ -66,7 +66,7 @@
66 66
67#include <asm/futex.h> 67#include <asm/futex.h>
68 68
69#include "rtmutex_common.h" 69#include "locking/rtmutex_common.h"
70 70
71int __read_mostly futex_cmpxchg_enabled; 71int __read_mostly futex_cmpxchg_enabled;
72 72
diff --git a/kernel/hung_task.c b/kernel/hung_task.c
index 3e97fb126e6b..8807061ca004 100644
--- a/kernel/hung_task.c
+++ b/kernel/hung_task.c
@@ -16,11 +16,12 @@
16#include <linux/export.h> 16#include <linux/export.h>
17#include <linux/sysctl.h> 17#include <linux/sysctl.h>
18#include <linux/utsname.h> 18#include <linux/utsname.h>
19#include <trace/events/sched.h>
19 20
20/* 21/*
21 * The number of tasks checked: 22 * The number of tasks checked:
22 */ 23 */
23unsigned long __read_mostly sysctl_hung_task_check_count = PID_MAX_LIMIT; 24int __read_mostly sysctl_hung_task_check_count = PID_MAX_LIMIT;
24 25
25/* 26/*
26 * Limit number of tasks checked in a batch. 27 * Limit number of tasks checked in a batch.
@@ -92,6 +93,9 @@ static void check_hung_task(struct task_struct *t, unsigned long timeout)
92 t->last_switch_count = switch_count; 93 t->last_switch_count = switch_count;
93 return; 94 return;
94 } 95 }
96
97 trace_sched_process_hang(t);
98
95 if (!sysctl_hung_task_warnings) 99 if (!sysctl_hung_task_warnings)
96 return; 100 return;
97 sysctl_hung_task_warnings--; 101 sysctl_hung_task_warnings--;
diff --git a/kernel/locking/Makefile b/kernel/locking/Makefile
new file mode 100644
index 000000000000..baab8e5e7f66
--- /dev/null
+++ b/kernel/locking/Makefile
@@ -0,0 +1,25 @@
1
2obj-y += mutex.o semaphore.o rwsem.o lglock.o
3
4ifdef CONFIG_FUNCTION_TRACER
5CFLAGS_REMOVE_lockdep.o = -pg
6CFLAGS_REMOVE_lockdep_proc.o = -pg
7CFLAGS_REMOVE_mutex-debug.o = -pg
8CFLAGS_REMOVE_rtmutex-debug.o = -pg
9endif
10
11obj-$(CONFIG_DEBUG_MUTEXES) += mutex-debug.o
12obj-$(CONFIG_LOCKDEP) += lockdep.o
13ifeq ($(CONFIG_PROC_FS),y)
14obj-$(CONFIG_LOCKDEP) += lockdep_proc.o
15endif
16obj-$(CONFIG_SMP) += spinlock.o
17obj-$(CONFIG_PROVE_LOCKING) += spinlock.o
18obj-$(CONFIG_RT_MUTEXES) += rtmutex.o
19obj-$(CONFIG_DEBUG_RT_MUTEXES) += rtmutex-debug.o
20obj-$(CONFIG_RT_MUTEX_TESTER) += rtmutex-tester.o
21obj-$(CONFIG_DEBUG_SPINLOCK) += spinlock.o
22obj-$(CONFIG_DEBUG_SPINLOCK) += spinlock_debug.o
23obj-$(CONFIG_RWSEM_GENERIC_SPINLOCK) += rwsem-spinlock.o
24obj-$(CONFIG_RWSEM_XCHGADD_ALGORITHM) += rwsem-xadd.o
25obj-$(CONFIG_PERCPU_RWSEM) += percpu-rwsem.o
diff --git a/kernel/lglock.c b/kernel/locking/lglock.c
index 86ae2aebf004..86ae2aebf004 100644
--- a/kernel/lglock.c
+++ b/kernel/locking/lglock.c
diff --git a/kernel/lockdep.c b/kernel/locking/lockdep.c
index 4e8e14c34e42..576ba756a32d 100644
--- a/kernel/lockdep.c
+++ b/kernel/locking/lockdep.c
@@ -1232,7 +1232,7 @@ static int noop_count(struct lock_list *entry, void *data)
1232 return 0; 1232 return 0;
1233} 1233}
1234 1234
1235unsigned long __lockdep_count_forward_deps(struct lock_list *this) 1235static unsigned long __lockdep_count_forward_deps(struct lock_list *this)
1236{ 1236{
1237 unsigned long count = 0; 1237 unsigned long count = 0;
1238 struct lock_list *uninitialized_var(target_entry); 1238 struct lock_list *uninitialized_var(target_entry);
@@ -1258,7 +1258,7 @@ unsigned long lockdep_count_forward_deps(struct lock_class *class)
1258 return ret; 1258 return ret;
1259} 1259}
1260 1260
1261unsigned long __lockdep_count_backward_deps(struct lock_list *this) 1261static unsigned long __lockdep_count_backward_deps(struct lock_list *this)
1262{ 1262{
1263 unsigned long count = 0; 1263 unsigned long count = 0;
1264 struct lock_list *uninitialized_var(target_entry); 1264 struct lock_list *uninitialized_var(target_entry);
diff --git a/kernel/lockdep_internals.h b/kernel/locking/lockdep_internals.h
index 4f560cfedc8f..4f560cfedc8f 100644
--- a/kernel/lockdep_internals.h
+++ b/kernel/locking/lockdep_internals.h
diff --git a/kernel/lockdep_proc.c b/kernel/locking/lockdep_proc.c
index b2c71c5873e4..ef43ac4bafb5 100644
--- a/kernel/lockdep_proc.c
+++ b/kernel/locking/lockdep_proc.c
@@ -421,6 +421,7 @@ static void seq_lock_time(struct seq_file *m, struct lock_time *lt)
421 seq_time(m, lt->min); 421 seq_time(m, lt->min);
422 seq_time(m, lt->max); 422 seq_time(m, lt->max);
423 seq_time(m, lt->total); 423 seq_time(m, lt->total);
424 seq_time(m, lt->nr ? div_s64(lt->total, lt->nr) : 0);
424} 425}
425 426
426static void seq_stats(struct seq_file *m, struct lock_stat_data *data) 427static void seq_stats(struct seq_file *m, struct lock_stat_data *data)
@@ -518,20 +519,20 @@ static void seq_stats(struct seq_file *m, struct lock_stat_data *data)
518 } 519 }
519 if (i) { 520 if (i) {
520 seq_puts(m, "\n"); 521 seq_puts(m, "\n");
521 seq_line(m, '.', 0, 40 + 1 + 10 * (14 + 1)); 522 seq_line(m, '.', 0, 40 + 1 + 12 * (14 + 1));
522 seq_puts(m, "\n"); 523 seq_puts(m, "\n");
523 } 524 }
524} 525}
525 526
526static void seq_header(struct seq_file *m) 527static void seq_header(struct seq_file *m)
527{ 528{
528 seq_printf(m, "lock_stat version 0.3\n"); 529 seq_puts(m, "lock_stat version 0.4\n");
529 530
530 if (unlikely(!debug_locks)) 531 if (unlikely(!debug_locks))
531 seq_printf(m, "*WARNING* lock debugging disabled!! - possibly due to a lockdep warning\n"); 532 seq_printf(m, "*WARNING* lock debugging disabled!! - possibly due to a lockdep warning\n");
532 533
533 seq_line(m, '-', 0, 40 + 1 + 10 * (14 + 1)); 534 seq_line(m, '-', 0, 40 + 1 + 12 * (14 + 1));
534 seq_printf(m, "%40s %14s %14s %14s %14s %14s %14s %14s %14s " 535 seq_printf(m, "%40s %14s %14s %14s %14s %14s %14s %14s %14s %14s %14s "
535 "%14s %14s\n", 536 "%14s %14s\n",
536 "class name", 537 "class name",
537 "con-bounces", 538 "con-bounces",
@@ -539,12 +540,14 @@ static void seq_header(struct seq_file *m)
539 "waittime-min", 540 "waittime-min",
540 "waittime-max", 541 "waittime-max",
541 "waittime-total", 542 "waittime-total",
543 "waittime-avg",
542 "acq-bounces", 544 "acq-bounces",
543 "acquisitions", 545 "acquisitions",
544 "holdtime-min", 546 "holdtime-min",
545 "holdtime-max", 547 "holdtime-max",
546 "holdtime-total"); 548 "holdtime-total",
547 seq_line(m, '-', 0, 40 + 1 + 10 * (14 + 1)); 549 "holdtime-avg");
550 seq_line(m, '-', 0, 40 + 1 + 12 * (14 + 1));
548 seq_printf(m, "\n"); 551 seq_printf(m, "\n");
549} 552}
550 553
diff --git a/kernel/lockdep_states.h b/kernel/locking/lockdep_states.h
index 995b0cc2b84c..995b0cc2b84c 100644
--- a/kernel/lockdep_states.h
+++ b/kernel/locking/lockdep_states.h
diff --git a/kernel/mutex-debug.c b/kernel/locking/mutex-debug.c
index 7e3443fe1f48..7e3443fe1f48 100644
--- a/kernel/mutex-debug.c
+++ b/kernel/locking/mutex-debug.c
diff --git a/kernel/mutex-debug.h b/kernel/locking/mutex-debug.h
index 0799fd3e4cfa..0799fd3e4cfa 100644
--- a/kernel/mutex-debug.h
+++ b/kernel/locking/mutex-debug.h
diff --git a/kernel/mutex.c b/kernel/locking/mutex.c
index d24105b1b794..4dd6e4c219de 100644
--- a/kernel/mutex.c
+++ b/kernel/locking/mutex.c
@@ -1,5 +1,5 @@
1/* 1/*
2 * kernel/mutex.c 2 * kernel/locking/mutex.c
3 * 3 *
4 * Mutexes: blocking mutual exclusion locks 4 * Mutexes: blocking mutual exclusion locks
5 * 5 *
diff --git a/kernel/mutex.h b/kernel/locking/mutex.h
index 4115fbf83b12..4115fbf83b12 100644
--- a/kernel/mutex.h
+++ b/kernel/locking/mutex.h
diff --git a/lib/percpu-rwsem.c b/kernel/locking/percpu-rwsem.c
index 652a8ee8efe9..652a8ee8efe9 100644
--- a/lib/percpu-rwsem.c
+++ b/kernel/locking/percpu-rwsem.c
diff --git a/kernel/rtmutex-debug.c b/kernel/locking/rtmutex-debug.c
index 13b243a323fa..13b243a323fa 100644
--- a/kernel/rtmutex-debug.c
+++ b/kernel/locking/rtmutex-debug.c
diff --git a/kernel/rtmutex-debug.h b/kernel/locking/rtmutex-debug.h
index 14193d596d78..14193d596d78 100644
--- a/kernel/rtmutex-debug.h
+++ b/kernel/locking/rtmutex-debug.h
diff --git a/kernel/rtmutex-tester.c b/kernel/locking/rtmutex-tester.c
index 1d96dd0d93c1..1d96dd0d93c1 100644
--- a/kernel/rtmutex-tester.c
+++ b/kernel/locking/rtmutex-tester.c
diff --git a/kernel/rtmutex.c b/kernel/locking/rtmutex.c
index 0dd6aec1cb6a..0dd6aec1cb6a 100644
--- a/kernel/rtmutex.c
+++ b/kernel/locking/rtmutex.c
diff --git a/kernel/rtmutex.h b/kernel/locking/rtmutex.h
index a1a1dd06421d..a1a1dd06421d 100644
--- a/kernel/rtmutex.h
+++ b/kernel/locking/rtmutex.h
diff --git a/kernel/rtmutex_common.h b/kernel/locking/rtmutex_common.h
index 53a66c85261b..53a66c85261b 100644
--- a/kernel/rtmutex_common.h
+++ b/kernel/locking/rtmutex_common.h
diff --git a/lib/rwsem-spinlock.c b/kernel/locking/rwsem-spinlock.c
index 9be8a9144978..9be8a9144978 100644
--- a/lib/rwsem-spinlock.c
+++ b/kernel/locking/rwsem-spinlock.c
diff --git a/lib/rwsem.c b/kernel/locking/rwsem-xadd.c
index 19c5fa95e0b4..19c5fa95e0b4 100644
--- a/lib/rwsem.c
+++ b/kernel/locking/rwsem-xadd.c
diff --git a/kernel/rwsem.c b/kernel/locking/rwsem.c
index cfff1435bdfb..cfff1435bdfb 100644
--- a/kernel/rwsem.c
+++ b/kernel/locking/rwsem.c
diff --git a/kernel/semaphore.c b/kernel/locking/semaphore.c
index 6815171a4fff..6815171a4fff 100644
--- a/kernel/semaphore.c
+++ b/kernel/locking/semaphore.c
diff --git a/kernel/spinlock.c b/kernel/locking/spinlock.c
index 4b082b5cac9e..4b082b5cac9e 100644
--- a/kernel/spinlock.c
+++ b/kernel/locking/spinlock.c
diff --git a/lib/spinlock_debug.c b/kernel/locking/spinlock_debug.c
index 0374a596cffa..0374a596cffa 100644
--- a/lib/spinlock_debug.c
+++ b/kernel/locking/spinlock_debug.c
diff --git a/kernel/rcu/tree_plugin.h b/kernel/rcu/tree_plugin.h
index 3822ac0c4b27..6abb03dff5c0 100644
--- a/kernel/rcu/tree_plugin.h
+++ b/kernel/rcu/tree_plugin.h
@@ -1133,7 +1133,7 @@ void exit_rcu(void)
1133 1133
1134#ifdef CONFIG_RCU_BOOST 1134#ifdef CONFIG_RCU_BOOST
1135 1135
1136#include "../rtmutex_common.h" 1136#include "../locking/rtmutex_common.h"
1137 1137
1138#ifdef CONFIG_RCU_TRACE 1138#ifdef CONFIG_RCU_TRACE
1139 1139
diff --git a/kernel/sysctl.c b/kernel/sysctl.c
index d37d9dd8f463..34a604726d0b 100644
--- a/kernel/sysctl.c
+++ b/kernel/sysctl.c
@@ -969,9 +969,10 @@ static struct ctl_table kern_table[] = {
969 { 969 {
970 .procname = "hung_task_check_count", 970 .procname = "hung_task_check_count",
971 .data = &sysctl_hung_task_check_count, 971 .data = &sysctl_hung_task_check_count,
972 .maxlen = sizeof(unsigned long), 972 .maxlen = sizeof(int),
973 .mode = 0644, 973 .mode = 0644,
974 .proc_handler = proc_doulongvec_minmax, 974 .proc_handler = proc_dointvec_minmax,
975 .extra1 = &zero,
975 }, 976 },
976 { 977 {
977 .procname = "hung_task_timeout_secs", 978 .procname = "hung_task_timeout_secs",
diff --git a/lib/Makefile b/lib/Makefile
index bb016e116ba4..d480a8c92385 100644
--- a/lib/Makefile
+++ b/lib/Makefile
@@ -42,10 +42,6 @@ obj-$(CONFIG_GENERIC_PCI_IOMAP) += pci_iomap.o
42obj-$(CONFIG_HAS_IOMEM) += iomap_copy.o devres.o 42obj-$(CONFIG_HAS_IOMEM) += iomap_copy.o devres.o
43obj-$(CONFIG_CHECK_SIGNATURE) += check_signature.o 43obj-$(CONFIG_CHECK_SIGNATURE) += check_signature.o
44obj-$(CONFIG_DEBUG_LOCKING_API_SELFTESTS) += locking-selftest.o 44obj-$(CONFIG_DEBUG_LOCKING_API_SELFTESTS) += locking-selftest.o
45obj-$(CONFIG_DEBUG_SPINLOCK) += spinlock_debug.o
46lib-$(CONFIG_RWSEM_GENERIC_SPINLOCK) += rwsem-spinlock.o
47lib-$(CONFIG_RWSEM_XCHGADD_ALGORITHM) += rwsem.o
48lib-$(CONFIG_PERCPU_RWSEM) += percpu-rwsem.o
49 45
50CFLAGS_hweight.o = $(subst $(quote),,$(CONFIG_ARCH_HWEIGHT_CFLAGS)) 46CFLAGS_hweight.o = $(subst $(quote),,$(CONFIG_ARCH_HWEIGHT_CFLAGS))
51obj-$(CONFIG_GENERIC_HWEIGHT) += hweight.o 47obj-$(CONFIG_GENERIC_HWEIGHT) += hweight.o
diff --git a/mm/filemap_xip.c b/mm/filemap_xip.c
index 28fe26b64f8a..d8d9fe3f685c 100644
--- a/mm/filemap_xip.c
+++ b/mm/filemap_xip.c
@@ -26,7 +26,7 @@
26 * of ZERO_PAGE(), such as /dev/zero 26 * of ZERO_PAGE(), such as /dev/zero
27 */ 27 */
28static DEFINE_MUTEX(xip_sparse_mutex); 28static DEFINE_MUTEX(xip_sparse_mutex);
29static seqcount_t xip_sparse_seq = SEQCNT_ZERO; 29static seqcount_t xip_sparse_seq = SEQCNT_ZERO(xip_sparse_seq);
30static struct page *__xip_sparse_page; 30static struct page *__xip_sparse_page;
31 31
32/* called under xip_sparse_mutex */ 32/* called under xip_sparse_mutex */
diff --git a/net/8021q/vlan_dev.c b/net/8021q/vlan_dev.c
index 8db1b985dbf1..762896ebfcf5 100644
--- a/net/8021q/vlan_dev.c
+++ b/net/8021q/vlan_dev.c
@@ -539,7 +539,7 @@ static const struct net_device_ops vlan_netdev_ops;
539static int vlan_dev_init(struct net_device *dev) 539static int vlan_dev_init(struct net_device *dev)
540{ 540{
541 struct net_device *real_dev = vlan_dev_priv(dev)->real_dev; 541 struct net_device *real_dev = vlan_dev_priv(dev)->real_dev;
542 int subclass = 0; 542 int subclass = 0, i;
543 543
544 netif_carrier_off(dev); 544 netif_carrier_off(dev);
545 545
@@ -593,6 +593,13 @@ static int vlan_dev_init(struct net_device *dev)
593 if (!vlan_dev_priv(dev)->vlan_pcpu_stats) 593 if (!vlan_dev_priv(dev)->vlan_pcpu_stats)
594 return -ENOMEM; 594 return -ENOMEM;
595 595
596 for_each_possible_cpu(i) {
597 struct vlan_pcpu_stats *vlan_stat;
598 vlan_stat = per_cpu_ptr(vlan_dev_priv(dev)->vlan_pcpu_stats, i);
599 u64_stats_init(&vlan_stat->syncp);
600 }
601
602
596 return 0; 603 return 0;
597} 604}
598 605
diff --git a/net/bridge/br_device.c b/net/bridge/br_device.c
index e6b7fecb3af1..f00cfd2a0143 100644
--- a/net/bridge/br_device.c
+++ b/net/bridge/br_device.c
@@ -88,11 +88,18 @@ out:
88static int br_dev_init(struct net_device *dev) 88static int br_dev_init(struct net_device *dev)
89{ 89{
90 struct net_bridge *br = netdev_priv(dev); 90 struct net_bridge *br = netdev_priv(dev);
91 int i;
91 92
92 br->stats = alloc_percpu(struct br_cpu_netstats); 93 br->stats = alloc_percpu(struct br_cpu_netstats);
93 if (!br->stats) 94 if (!br->stats)
94 return -ENOMEM; 95 return -ENOMEM;
95 96
97 for_each_possible_cpu(i) {
98 struct br_cpu_netstats *br_dev_stats;
99 br_dev_stats = per_cpu_ptr(br->stats, i);
100 u64_stats_init(&br_dev_stats->syncp);
101 }
102
96 return 0; 103 return 0;
97} 104}
98 105
diff --git a/net/ipv4/af_inet.c b/net/ipv4/af_inet.c
index 68af9aac91d0..70011e029ac1 100644
--- a/net/ipv4/af_inet.c
+++ b/net/ipv4/af_inet.c
@@ -1503,6 +1503,7 @@ int snmp_mib_init(void __percpu *ptr[2], size_t mibsize, size_t align)
1503 ptr[0] = __alloc_percpu(mibsize, align); 1503 ptr[0] = __alloc_percpu(mibsize, align);
1504 if (!ptr[0]) 1504 if (!ptr[0])
1505 return -ENOMEM; 1505 return -ENOMEM;
1506
1506#if SNMP_ARRAY_SZ == 2 1507#if SNMP_ARRAY_SZ == 2
1507 ptr[1] = __alloc_percpu(mibsize, align); 1508 ptr[1] = __alloc_percpu(mibsize, align);
1508 if (!ptr[1]) { 1509 if (!ptr[1]) {
@@ -1547,6 +1548,8 @@ static const struct net_protocol icmp_protocol = {
1547 1548
1548static __net_init int ipv4_mib_init_net(struct net *net) 1549static __net_init int ipv4_mib_init_net(struct net *net)
1549{ 1550{
1551 int i;
1552
1550 if (snmp_mib_init((void __percpu **)net->mib.tcp_statistics, 1553 if (snmp_mib_init((void __percpu **)net->mib.tcp_statistics,
1551 sizeof(struct tcp_mib), 1554 sizeof(struct tcp_mib),
1552 __alignof__(struct tcp_mib)) < 0) 1555 __alignof__(struct tcp_mib)) < 0)
@@ -1555,6 +1558,17 @@ static __net_init int ipv4_mib_init_net(struct net *net)
1555 sizeof(struct ipstats_mib), 1558 sizeof(struct ipstats_mib),
1556 __alignof__(struct ipstats_mib)) < 0) 1559 __alignof__(struct ipstats_mib)) < 0)
1557 goto err_ip_mib; 1560 goto err_ip_mib;
1561
1562 for_each_possible_cpu(i) {
1563 struct ipstats_mib *af_inet_stats;
1564 af_inet_stats = per_cpu_ptr(net->mib.ip_statistics[0], i);
1565 u64_stats_init(&af_inet_stats->syncp);
1566#if SNMP_ARRAY_SZ == 2
1567 af_inet_stats = per_cpu_ptr(net->mib.ip_statistics[1], i);
1568 u64_stats_init(&af_inet_stats->syncp);
1569#endif
1570 }
1571
1558 if (snmp_mib_init((void __percpu **)net->mib.net_statistics, 1572 if (snmp_mib_init((void __percpu **)net->mib.net_statistics,
1559 sizeof(struct linux_mib), 1573 sizeof(struct linux_mib),
1560 __alignof__(struct linux_mib)) < 0) 1574 __alignof__(struct linux_mib)) < 0)
diff --git a/net/ipv4/ip_tunnel.c b/net/ipv4/ip_tunnel.c
index 63a6d6d6b875..caf01176a5e4 100644
--- a/net/ipv4/ip_tunnel.c
+++ b/net/ipv4/ip_tunnel.c
@@ -976,13 +976,19 @@ int ip_tunnel_init(struct net_device *dev)
976{ 976{
977 struct ip_tunnel *tunnel = netdev_priv(dev); 977 struct ip_tunnel *tunnel = netdev_priv(dev);
978 struct iphdr *iph = &tunnel->parms.iph; 978 struct iphdr *iph = &tunnel->parms.iph;
979 int err; 979 int i, err;
980 980
981 dev->destructor = ip_tunnel_dev_free; 981 dev->destructor = ip_tunnel_dev_free;
982 dev->tstats = alloc_percpu(struct pcpu_tstats); 982 dev->tstats = alloc_percpu(struct pcpu_tstats);
983 if (!dev->tstats) 983 if (!dev->tstats)
984 return -ENOMEM; 984 return -ENOMEM;
985 985
986 for_each_possible_cpu(i) {
987 struct pcpu_tstats *ipt_stats;
988 ipt_stats = per_cpu_ptr(dev->tstats, i);
989 u64_stats_init(&ipt_stats->syncp);
990 }
991
986 err = gro_cells_init(&tunnel->gro_cells, dev); 992 err = gro_cells_init(&tunnel->gro_cells, dev);
987 if (err) { 993 if (err) {
988 free_percpu(dev->tstats); 994 free_percpu(dev->tstats);
diff --git a/net/ipv6/addrconf.c b/net/ipv6/addrconf.c
index 542d09561ed6..5658d9d51637 100644
--- a/net/ipv6/addrconf.c
+++ b/net/ipv6/addrconf.c
@@ -271,10 +271,24 @@ static void addrconf_mod_dad_timer(struct inet6_ifaddr *ifp,
271 271
272static int snmp6_alloc_dev(struct inet6_dev *idev) 272static int snmp6_alloc_dev(struct inet6_dev *idev)
273{ 273{
274 int i;
275
274 if (snmp_mib_init((void __percpu **)idev->stats.ipv6, 276 if (snmp_mib_init((void __percpu **)idev->stats.ipv6,
275 sizeof(struct ipstats_mib), 277 sizeof(struct ipstats_mib),
276 __alignof__(struct ipstats_mib)) < 0) 278 __alignof__(struct ipstats_mib)) < 0)
277 goto err_ip; 279 goto err_ip;
280
281 for_each_possible_cpu(i) {
282 struct ipstats_mib *addrconf_stats;
283 addrconf_stats = per_cpu_ptr(idev->stats.ipv6[0], i);
284 u64_stats_init(&addrconf_stats->syncp);
285#if SNMP_ARRAY_SZ == 2
286 addrconf_stats = per_cpu_ptr(idev->stats.ipv6[1], i);
287 u64_stats_init(&addrconf_stats->syncp);
288#endif
289 }
290
291
278 idev->stats.icmpv6dev = kzalloc(sizeof(struct icmpv6_mib_device), 292 idev->stats.icmpv6dev = kzalloc(sizeof(struct icmpv6_mib_device),
279 GFP_KERNEL); 293 GFP_KERNEL);
280 if (!idev->stats.icmpv6dev) 294 if (!idev->stats.icmpv6dev)
diff --git a/net/ipv6/af_inet6.c b/net/ipv6/af_inet6.c
index 6468bda1f2b9..ff75313f27a8 100644
--- a/net/ipv6/af_inet6.c
+++ b/net/ipv6/af_inet6.c
@@ -714,6 +714,8 @@ static void ipv6_packet_cleanup(void)
714 714
715static int __net_init ipv6_init_mibs(struct net *net) 715static int __net_init ipv6_init_mibs(struct net *net)
716{ 716{
717 int i;
718
717 if (snmp_mib_init((void __percpu **)net->mib.udp_stats_in6, 719 if (snmp_mib_init((void __percpu **)net->mib.udp_stats_in6,
718 sizeof(struct udp_mib), 720 sizeof(struct udp_mib),
719 __alignof__(struct udp_mib)) < 0) 721 __alignof__(struct udp_mib)) < 0)
@@ -726,6 +728,18 @@ static int __net_init ipv6_init_mibs(struct net *net)
726 sizeof(struct ipstats_mib), 728 sizeof(struct ipstats_mib),
727 __alignof__(struct ipstats_mib)) < 0) 729 __alignof__(struct ipstats_mib)) < 0)
728 goto err_ip_mib; 730 goto err_ip_mib;
731
732 for_each_possible_cpu(i) {
733 struct ipstats_mib *af_inet6_stats;
734 af_inet6_stats = per_cpu_ptr(net->mib.ipv6_statistics[0], i);
735 u64_stats_init(&af_inet6_stats->syncp);
736#if SNMP_ARRAY_SZ == 2
737 af_inet6_stats = per_cpu_ptr(net->mib.ipv6_statistics[1], i);
738 u64_stats_init(&af_inet6_stats->syncp);
739#endif
740 }
741
742
729 if (snmp_mib_init((void __percpu **)net->mib.icmpv6_statistics, 743 if (snmp_mib_init((void __percpu **)net->mib.icmpv6_statistics,
730 sizeof(struct icmpv6_mib), 744 sizeof(struct icmpv6_mib),
731 __alignof__(struct icmpv6_mib)) < 0) 745 __alignof__(struct icmpv6_mib)) < 0)
diff --git a/net/ipv6/ip6_gre.c b/net/ipv6/ip6_gre.c
index bf4a9a084de5..8acb28621f9c 100644
--- a/net/ipv6/ip6_gre.c
+++ b/net/ipv6/ip6_gre.c
@@ -1252,6 +1252,7 @@ static void ip6gre_tunnel_setup(struct net_device *dev)
1252static int ip6gre_tunnel_init(struct net_device *dev) 1252static int ip6gre_tunnel_init(struct net_device *dev)
1253{ 1253{
1254 struct ip6_tnl *tunnel; 1254 struct ip6_tnl *tunnel;
1255 int i;
1255 1256
1256 tunnel = netdev_priv(dev); 1257 tunnel = netdev_priv(dev);
1257 1258
@@ -1269,6 +1270,13 @@ static int ip6gre_tunnel_init(struct net_device *dev)
1269 if (!dev->tstats) 1270 if (!dev->tstats)
1270 return -ENOMEM; 1271 return -ENOMEM;
1271 1272
1273 for_each_possible_cpu(i) {
1274 struct pcpu_tstats *ip6gre_tunnel_stats;
1275 ip6gre_tunnel_stats = per_cpu_ptr(dev->tstats, i);
1276 u64_stats_init(&ip6gre_tunnel_stats->syncp);
1277 }
1278
1279
1272 return 0; 1280 return 0;
1273} 1281}
1274 1282
@@ -1449,6 +1457,7 @@ static void ip6gre_netlink_parms(struct nlattr *data[],
1449static int ip6gre_tap_init(struct net_device *dev) 1457static int ip6gre_tap_init(struct net_device *dev)
1450{ 1458{
1451 struct ip6_tnl *tunnel; 1459 struct ip6_tnl *tunnel;
1460 int i;
1452 1461
1453 tunnel = netdev_priv(dev); 1462 tunnel = netdev_priv(dev);
1454 1463
@@ -1462,6 +1471,12 @@ static int ip6gre_tap_init(struct net_device *dev)
1462 if (!dev->tstats) 1471 if (!dev->tstats)
1463 return -ENOMEM; 1472 return -ENOMEM;
1464 1473
1474 for_each_possible_cpu(i) {
1475 struct pcpu_tstats *ip6gre_tap_stats;
1476 ip6gre_tap_stats = per_cpu_ptr(dev->tstats, i);
1477 u64_stats_init(&ip6gre_tap_stats->syncp);
1478 }
1479
1465 return 0; 1480 return 0;
1466} 1481}
1467 1482
diff --git a/net/ipv6/ip6_output.c b/net/ipv6/ip6_output.c
index 5e31a909a2b0..59df872e2f4d 100644
--- a/net/ipv6/ip6_output.c
+++ b/net/ipv6/ip6_output.c
@@ -910,7 +910,7 @@ static int ip6_dst_lookup_tail(struct sock *sk,
910 910
911out_err_release: 911out_err_release:
912 if (err == -ENETUNREACH) 912 if (err == -ENETUNREACH)
913 IP6_INC_STATS_BH(net, NULL, IPSTATS_MIB_OUTNOROUTES); 913 IP6_INC_STATS(net, NULL, IPSTATS_MIB_OUTNOROUTES);
914 dst_release(*dst); 914 dst_release(*dst);
915 *dst = NULL; 915 *dst = NULL;
916 return err; 916 return err;
diff --git a/net/ipv6/ip6_tunnel.c b/net/ipv6/ip6_tunnel.c
index 583b77e2f69b..df1fa58528c6 100644
--- a/net/ipv6/ip6_tunnel.c
+++ b/net/ipv6/ip6_tunnel.c
@@ -1494,12 +1494,19 @@ static inline int
1494ip6_tnl_dev_init_gen(struct net_device *dev) 1494ip6_tnl_dev_init_gen(struct net_device *dev)
1495{ 1495{
1496 struct ip6_tnl *t = netdev_priv(dev); 1496 struct ip6_tnl *t = netdev_priv(dev);
1497 int i;
1497 1498
1498 t->dev = dev; 1499 t->dev = dev;
1499 t->net = dev_net(dev); 1500 t->net = dev_net(dev);
1500 dev->tstats = alloc_percpu(struct pcpu_tstats); 1501 dev->tstats = alloc_percpu(struct pcpu_tstats);
1501 if (!dev->tstats) 1502 if (!dev->tstats)
1502 return -ENOMEM; 1503 return -ENOMEM;
1504
1505 for_each_possible_cpu(i) {
1506 struct pcpu_tstats *ip6_tnl_stats;
1507 ip6_tnl_stats = per_cpu_ptr(dev->tstats, i);
1508 u64_stats_init(&ip6_tnl_stats->syncp);
1509 }
1503 return 0; 1510 return 0;
1504} 1511}
1505 1512
diff --git a/net/ipv6/sit.c b/net/ipv6/sit.c
index 3a9038dd818d..bfc6fcea3841 100644
--- a/net/ipv6/sit.c
+++ b/net/ipv6/sit.c
@@ -1320,6 +1320,7 @@ static void ipip6_tunnel_setup(struct net_device *dev)
1320static int ipip6_tunnel_init(struct net_device *dev) 1320static int ipip6_tunnel_init(struct net_device *dev)
1321{ 1321{
1322 struct ip_tunnel *tunnel = netdev_priv(dev); 1322 struct ip_tunnel *tunnel = netdev_priv(dev);
1323 int i;
1323 1324
1324 tunnel->dev = dev; 1325 tunnel->dev = dev;
1325 tunnel->net = dev_net(dev); 1326 tunnel->net = dev_net(dev);
@@ -1332,6 +1333,12 @@ static int ipip6_tunnel_init(struct net_device *dev)
1332 if (!dev->tstats) 1333 if (!dev->tstats)
1333 return -ENOMEM; 1334 return -ENOMEM;
1334 1335
1336 for_each_possible_cpu(i) {
1337 struct pcpu_tstats *ipip6_tunnel_stats;
1338 ipip6_tunnel_stats = per_cpu_ptr(dev->tstats, i);
1339 u64_stats_init(&ipip6_tunnel_stats->syncp);
1340 }
1341
1335 return 0; 1342 return 0;
1336} 1343}
1337 1344
@@ -1341,6 +1348,7 @@ static int __net_init ipip6_fb_tunnel_init(struct net_device *dev)
1341 struct iphdr *iph = &tunnel->parms.iph; 1348 struct iphdr *iph = &tunnel->parms.iph;
1342 struct net *net = dev_net(dev); 1349 struct net *net = dev_net(dev);
1343 struct sit_net *sitn = net_generic(net, sit_net_id); 1350 struct sit_net *sitn = net_generic(net, sit_net_id);
1351 int i;
1344 1352
1345 tunnel->dev = dev; 1353 tunnel->dev = dev;
1346 tunnel->net = dev_net(dev); 1354 tunnel->net = dev_net(dev);
@@ -1354,6 +1362,13 @@ static int __net_init ipip6_fb_tunnel_init(struct net_device *dev)
1354 dev->tstats = alloc_percpu(struct pcpu_tstats); 1362 dev->tstats = alloc_percpu(struct pcpu_tstats);
1355 if (!dev->tstats) 1363 if (!dev->tstats)
1356 return -ENOMEM; 1364 return -ENOMEM;
1365
1366 for_each_possible_cpu(i) {
1367 struct pcpu_tstats *ipip6_fb_stats;
1368 ipip6_fb_stats = per_cpu_ptr(dev->tstats, i);
1369 u64_stats_init(&ipip6_fb_stats->syncp);
1370 }
1371
1357 dev_hold(dev); 1372 dev_hold(dev);
1358 rcu_assign_pointer(sitn->tunnels_wc[0], tunnel); 1373 rcu_assign_pointer(sitn->tunnels_wc[0], tunnel);
1359 return 0; 1374 return 0;
diff --git a/net/netfilter/ipvs/ip_vs_ctl.c b/net/netfilter/ipvs/ip_vs_ctl.c
index 62786a495cea..1ded5c6d268c 100644
--- a/net/netfilter/ipvs/ip_vs_ctl.c
+++ b/net/netfilter/ipvs/ip_vs_ctl.c
@@ -842,7 +842,7 @@ ip_vs_new_dest(struct ip_vs_service *svc, struct ip_vs_dest_user_kern *udest,
842 struct ip_vs_dest **dest_p) 842 struct ip_vs_dest **dest_p)
843{ 843{
844 struct ip_vs_dest *dest; 844 struct ip_vs_dest *dest;
845 unsigned int atype; 845 unsigned int atype, i;
846 846
847 EnterFunction(2); 847 EnterFunction(2);
848 848
@@ -869,6 +869,12 @@ ip_vs_new_dest(struct ip_vs_service *svc, struct ip_vs_dest_user_kern *udest,
869 if (!dest->stats.cpustats) 869 if (!dest->stats.cpustats)
870 goto err_alloc; 870 goto err_alloc;
871 871
872 for_each_possible_cpu(i) {
873 struct ip_vs_cpu_stats *ip_vs_dest_stats;
874 ip_vs_dest_stats = per_cpu_ptr(dest->stats.cpustats, i);
875 u64_stats_init(&ip_vs_dest_stats->syncp);
876 }
877
872 dest->af = svc->af; 878 dest->af = svc->af;
873 dest->protocol = svc->protocol; 879 dest->protocol = svc->protocol;
874 dest->vaddr = svc->addr; 880 dest->vaddr = svc->addr;
@@ -1134,7 +1140,7 @@ static int
1134ip_vs_add_service(struct net *net, struct ip_vs_service_user_kern *u, 1140ip_vs_add_service(struct net *net, struct ip_vs_service_user_kern *u,
1135 struct ip_vs_service **svc_p) 1141 struct ip_vs_service **svc_p)
1136{ 1142{
1137 int ret = 0; 1143 int ret = 0, i;
1138 struct ip_vs_scheduler *sched = NULL; 1144 struct ip_vs_scheduler *sched = NULL;
1139 struct ip_vs_pe *pe = NULL; 1145 struct ip_vs_pe *pe = NULL;
1140 struct ip_vs_service *svc = NULL; 1146 struct ip_vs_service *svc = NULL;
@@ -1184,6 +1190,13 @@ ip_vs_add_service(struct net *net, struct ip_vs_service_user_kern *u,
1184 goto out_err; 1190 goto out_err;
1185 } 1191 }
1186 1192
1193 for_each_possible_cpu(i) {
1194 struct ip_vs_cpu_stats *ip_vs_stats;
1195 ip_vs_stats = per_cpu_ptr(svc->stats.cpustats, i);
1196 u64_stats_init(&ip_vs_stats->syncp);
1197 }
1198
1199
1187 /* I'm the first user of the service */ 1200 /* I'm the first user of the service */
1188 atomic_set(&svc->refcnt, 0); 1201 atomic_set(&svc->refcnt, 0);
1189 1202
@@ -3780,7 +3793,7 @@ static struct notifier_block ip_vs_dst_notifier = {
3780 3793
3781int __net_init ip_vs_control_net_init(struct net *net) 3794int __net_init ip_vs_control_net_init(struct net *net)
3782{ 3795{
3783 int idx; 3796 int i, idx;
3784 struct netns_ipvs *ipvs = net_ipvs(net); 3797 struct netns_ipvs *ipvs = net_ipvs(net);
3785 3798
3786 /* Initialize rs_table */ 3799 /* Initialize rs_table */
@@ -3799,6 +3812,12 @@ int __net_init ip_vs_control_net_init(struct net *net)
3799 if (!ipvs->tot_stats.cpustats) 3812 if (!ipvs->tot_stats.cpustats)
3800 return -ENOMEM; 3813 return -ENOMEM;
3801 3814
3815 for_each_possible_cpu(i) {
3816 struct ip_vs_cpu_stats *ipvs_tot_stats;
3817 ipvs_tot_stats = per_cpu_ptr(ipvs->tot_stats.cpustats, i);
3818 u64_stats_init(&ipvs_tot_stats->syncp);
3819 }
3820
3802 spin_lock_init(&ipvs->tot_stats.lock); 3821 spin_lock_init(&ipvs->tot_stats.lock);
3803 3822
3804 proc_create("ip_vs", 0, net->proc_net, &ip_vs_info_fops); 3823 proc_create("ip_vs", 0, net->proc_net, &ip_vs_info_fops);
diff --git a/net/openvswitch/datapath.c b/net/openvswitch/datapath.c
index 1408adc2a2a7..449e0776a2c0 100644
--- a/net/openvswitch/datapath.c
+++ b/net/openvswitch/datapath.c
@@ -1199,6 +1199,12 @@ static int ovs_dp_cmd_new(struct sk_buff *skb, struct genl_info *info)
1199 goto err_destroy_table; 1199 goto err_destroy_table;
1200 } 1200 }
1201 1201
1202 for_each_possible_cpu(i) {
1203 struct dp_stats_percpu *dpath_stats;
1204 dpath_stats = per_cpu_ptr(dp->stats_percpu, i);
1205 u64_stats_init(&dpath_stats->sync);
1206 }
1207
1202 dp->ports = kmalloc(DP_VPORT_HASH_BUCKETS * sizeof(struct hlist_head), 1208 dp->ports = kmalloc(DP_VPORT_HASH_BUCKETS * sizeof(struct hlist_head),
1203 GFP_KERNEL); 1209 GFP_KERNEL);
1204 if (!dp->ports) { 1210 if (!dp->ports) {
diff --git a/net/openvswitch/vport.c b/net/openvswitch/vport.c
index 6f65dbe13812..d830a95f03a4 100644
--- a/net/openvswitch/vport.c
+++ b/net/openvswitch/vport.c
@@ -118,6 +118,7 @@ struct vport *ovs_vport_alloc(int priv_size, const struct vport_ops *ops,
118{ 118{
119 struct vport *vport; 119 struct vport *vport;
120 size_t alloc_size; 120 size_t alloc_size;
121 int i;
121 122
122 alloc_size = sizeof(struct vport); 123 alloc_size = sizeof(struct vport);
123 if (priv_size) { 124 if (priv_size) {
@@ -141,6 +142,13 @@ struct vport *ovs_vport_alloc(int priv_size, const struct vport_ops *ops,
141 return ERR_PTR(-ENOMEM); 142 return ERR_PTR(-ENOMEM);
142 } 143 }
143 144
145 for_each_possible_cpu(i) {
146 struct pcpu_tstats *vport_stats;
147 vport_stats = per_cpu_ptr(vport->percpu_stats, i);
148 u64_stats_init(&vport_stats->syncp);
149 }
150
151
144 spin_lock_init(&vport->stats_lock); 152 spin_lock_init(&vport->stats_lock);
145 153
146 return vport; 154 return vport;