aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
Diffstat
-rw-r--r--Documentation/RCU/stallwarn.txt16
-rw-r--r--Documentation/RCU/trace.txt32
-rw-r--r--Documentation/memory-barriers.txt46
-rw-r--r--arch/arm/kvm/Kconfig1
-rw-r--r--arch/arm64/kvm/Kconfig1
-rw-r--r--arch/mips/kvm/Kconfig1
-rw-r--r--arch/powerpc/kvm/Kconfig1
-rw-r--r--arch/s390/kvm/Kconfig1
-rw-r--r--arch/tile/kvm/Kconfig1
-rw-r--r--arch/x86/Kconfig1
-rw-r--r--arch/x86/kvm/Kconfig1
-rw-r--r--drivers/clk/Kconfig1
-rw-r--r--drivers/cpufreq/Kconfig1
-rw-r--r--drivers/devfreq/Kconfig1
-rw-r--r--drivers/md/Kconfig1
-rw-r--r--drivers/net/Kconfig1
-rw-r--r--fs/btrfs/Kconfig1
-rw-r--r--fs/notify/Kconfig1
-rw-r--r--fs/quota/Kconfig1
-rw-r--r--include/linux/compiler.h2
-rw-r--r--include/linux/rculist.h16
-rw-r--r--include/linux/rcupdate.h13
-rw-r--r--include/linux/rcutiny.h5
-rw-r--r--include/linux/rcutree.h2
-rw-r--r--include/linux/srcu.h14
-rw-r--r--init/Kconfig18
-rw-r--r--kernel/cpu.c56
-rw-r--r--kernel/notifier.c3
-rw-r--r--kernel/power/Kconfig1
-rw-r--r--kernel/rcu/Makefile3
-rw-r--r--kernel/rcu/rcu.h6
-rw-r--r--kernel/rcu/srcu.c2
-rw-r--r--kernel/rcu/tiny.c113
-rw-r--r--kernel/rcu/tiny_plugin.h9
-rw-r--r--kernel/rcu/tree.c245
-rw-r--r--kernel/rcu/tree.h61
-rw-r--r--kernel/rcu/tree_plugin.h243
-rw-r--r--kernel/rcu/tree_trace.c8
-rw-r--r--kernel/softirq.c3
-rw-r--r--lib/Kconfig.debug3
-rw-r--r--mm/Kconfig1
-rw-r--r--security/tomoyo/Kconfig1
-rwxr-xr-xtools/testing/selftests/rcutorture/bin/parse-console.sh2
43 files changed, 451 insertions, 489 deletions
diff --git a/Documentation/RCU/stallwarn.txt b/Documentation/RCU/stallwarn.txt
index ed186a902d31..b57c0c1cdac6 100644
--- a/Documentation/RCU/stallwarn.txt
+++ b/Documentation/RCU/stallwarn.txt
@@ -15,7 +15,7 @@ CONFIG_RCU_CPU_STALL_TIMEOUT
15 21 seconds. 15 21 seconds.
16 16
17 This configuration parameter may be changed at runtime via the 17 This configuration parameter may be changed at runtime via the
18 /sys/module/rcutree/parameters/rcu_cpu_stall_timeout, however 18 /sys/module/rcupdate/parameters/rcu_cpu_stall_timeout, however
19 this parameter is checked only at the beginning of a cycle. 19 this parameter is checked only at the beginning of a cycle.
20 So if you are 10 seconds into a 40-second stall, setting this 20 So if you are 10 seconds into a 40-second stall, setting this
21 sysfs parameter to (say) five will shorten the timeout for the 21 sysfs parameter to (say) five will shorten the timeout for the
@@ -152,6 +152,15 @@ no non-lazy callbacks ("." is printed otherwise, as shown above) and
152"D" indicates that dyntick-idle processing is enabled ("." is printed 152"D" indicates that dyntick-idle processing is enabled ("." is printed
153otherwise, for example, if disabled via the "nohz=" kernel boot parameter). 153otherwise, for example, if disabled via the "nohz=" kernel boot parameter).
154 154
155If the relevant grace-period kthread has been unable to run prior to
156the stall warning, the following additional line is printed:
157
158 rcu_preempt kthread starved for 2023 jiffies!
159
160Starving the grace-period kthreads of CPU time can of course result in
161RCU CPU stall warnings even when all CPUs and tasks have passed through
162the required quiescent states.
163
155 164
156Multiple Warnings From One Stall 165Multiple Warnings From One Stall
157 166
@@ -187,6 +196,11 @@ o For !CONFIG_PREEMPT kernels, a CPU looping anywhere in the
187 behavior, you might need to replace some of the cond_resched() 196 behavior, you might need to replace some of the cond_resched()
188 calls with calls to cond_resched_rcu_qs(). 197 calls with calls to cond_resched_rcu_qs().
189 198
199o Anything that prevents RCU's grace-period kthreads from running.
200 This can result in the "All QSes seen" console-log message.
201 This message will include information on when the kthread last
202 ran and how often it should be expected to run.
203
190o A CPU-bound real-time task in a CONFIG_PREEMPT kernel, which might 204o A CPU-bound real-time task in a CONFIG_PREEMPT kernel, which might
191 happen to preempt a low-priority task in the middle of an RCU 205 happen to preempt a low-priority task in the middle of an RCU
192 read-side critical section. This is especially damaging if 206 read-side critical section. This is especially damaging if
diff --git a/Documentation/RCU/trace.txt b/Documentation/RCU/trace.txt
index b63b9bb3bc0c..08651da15448 100644
--- a/Documentation/RCU/trace.txt
+++ b/Documentation/RCU/trace.txt
@@ -56,14 +56,14 @@ rcuboost:
56 56
57The output of "cat rcu/rcu_preempt/rcudata" looks as follows: 57The output of "cat rcu/rcu_preempt/rcudata" looks as follows:
58 58
59 0!c=30455 g=30456 pq=1 qp=1 dt=126535/140000000000000/0 df=2002 of=4 ql=0/0 qs=N... b=10 ci=74572 nci=0 co=1131 ca=716 59 0!c=30455 g=30456 pq=1/0 qp=1 dt=126535/140000000000000/0 df=2002 of=4 ql=0/0 qs=N... b=10 ci=74572 nci=0 co=1131 ca=716
60 1!c=30719 g=30720 pq=1 qp=0 dt=132007/140000000000000/0 df=1874 of=10 ql=0/0 qs=N... b=10 ci=123209 nci=0 co=685 ca=982 60 1!c=30719 g=30720 pq=1/0 qp=0 dt=132007/140000000000000/0 df=1874 of=10 ql=0/0 qs=N... b=10 ci=123209 nci=0 co=685 ca=982
61 2!c=30150 g=30151 pq=1 qp=1 dt=138537/140000000000000/0 df=1707 of=8 ql=0/0 qs=N... b=10 ci=80132 nci=0 co=1328 ca=1458 61 2!c=30150 g=30151 pq=1/1 qp=1 dt=138537/140000000000000/0 df=1707 of=8 ql=0/0 qs=N... b=10 ci=80132 nci=0 co=1328 ca=1458
62 3 c=31249 g=31250 pq=1 qp=0 dt=107255/140000000000000/0 df=1749 of=6 ql=0/450 qs=NRW. b=10 ci=151700 nci=0 co=509 ca=622 62 3 c=31249 g=31250 pq=1/1 qp=0 dt=107255/140000000000000/0 df=1749 of=6 ql=0/450 qs=NRW. b=10 ci=151700 nci=0 co=509 ca=622
63 4!c=29502 g=29503 pq=1 qp=1 dt=83647/140000000000000/0 df=965 of=5 ql=0/0 qs=N... b=10 ci=65643 nci=0 co=1373 ca=1521 63 4!c=29502 g=29503 pq=1/0 qp=1 dt=83647/140000000000000/0 df=965 of=5 ql=0/0 qs=N... b=10 ci=65643 nci=0 co=1373 ca=1521
64 5 c=31201 g=31202 pq=1 qp=1 dt=70422/0/0 df=535 of=7 ql=0/0 qs=.... b=10 ci=58500 nci=0 co=764 ca=698 64 5 c=31201 g=31202 pq=1/0 qp=1 dt=70422/0/0 df=535 of=7 ql=0/0 qs=.... b=10 ci=58500 nci=0 co=764 ca=698
65 6!c=30253 g=30254 pq=1 qp=1 dt=95363/140000000000000/0 df=780 of=5 ql=0/0 qs=N... b=10 ci=100607 nci=0 co=1414 ca=1353 65 6!c=30253 g=30254 pq=1/0 qp=1 dt=95363/140000000000000/0 df=780 of=5 ql=0/0 qs=N... b=10 ci=100607 nci=0 co=1414 ca=1353
66 7 c=31178 g=31178 pq=1 qp=0 dt=91536/0/0 df=547 of=4 ql=0/0 qs=.... b=10 ci=109819 nci=0 co=1115 ca=969 66 7 c=31178 g=31178 pq=1/0 qp=0 dt=91536/0/0 df=547 of=4 ql=0/0 qs=.... b=10 ci=109819 nci=0 co=1115 ca=969
67 67
68This file has one line per CPU, or eight for this 8-CPU system. 68This file has one line per CPU, or eight for this 8-CPU system.
69The fields are as follows: 69The fields are as follows:
@@ -188,14 +188,14 @@ o "ca" is the number of RCU callbacks that have been adopted by this
188Kernels compiled with CONFIG_RCU_BOOST=y display the following from 188Kernels compiled with CONFIG_RCU_BOOST=y display the following from
189/debug/rcu/rcu_preempt/rcudata: 189/debug/rcu/rcu_preempt/rcudata:
190 190
191 0!c=12865 g=12866 pq=1 qp=1 dt=83113/140000000000000/0 df=288 of=11 ql=0/0 qs=N... kt=0/O ktl=944 b=10 ci=60709 nci=0 co=748 ca=871 191 0!c=12865 g=12866 pq=1/0 qp=1 dt=83113/140000000000000/0 df=288 of=11 ql=0/0 qs=N... kt=0/O ktl=944 b=10 ci=60709 nci=0 co=748 ca=871
192 1 c=14407 g=14408 pq=1 qp=0 dt=100679/140000000000000/0 df=378 of=7 ql=0/119 qs=NRW. kt=0/W ktl=9b6 b=10 ci=109740 nci=0 co=589 ca=485 192 1 c=14407 g=14408 pq=1/0 qp=0 dt=100679/140000000000000/0 df=378 of=7 ql=0/119 qs=NRW. kt=0/W ktl=9b6 b=10 ci=109740 nci=0 co=589 ca=485
193 2 c=14407 g=14408 pq=1 qp=0 dt=105486/0/0 df=90 of=9 ql=0/89 qs=NRW. kt=0/W ktl=c0c b=10 ci=83113 nci=0 co=533 ca=490 193 2 c=14407 g=14408 pq=1/0 qp=0 dt=105486/0/0 df=90 of=9 ql=0/89 qs=NRW. kt=0/W ktl=c0c b=10 ci=83113 nci=0 co=533 ca=490
194 3 c=14407 g=14408 pq=1 qp=0 dt=107138/0/0 df=142 of=8 ql=0/188 qs=NRW. kt=0/W ktl=b96 b=10 ci=121114 nci=0 co=426 ca=290 194 3 c=14407 g=14408 pq=1/0 qp=0 dt=107138/0/0 df=142 of=8 ql=0/188 qs=NRW. kt=0/W ktl=b96 b=10 ci=121114 nci=0 co=426 ca=290
195 4 c=14405 g=14406 pq=1 qp=1 dt=50238/0/0 df=706 of=7 ql=0/0 qs=.... kt=0/W ktl=812 b=10 ci=34929 nci=0 co=643 ca=114 195 4 c=14405 g=14406 pq=1/0 qp=1 dt=50238/0/0 df=706 of=7 ql=0/0 qs=.... kt=0/W ktl=812 b=10 ci=34929 nci=0 co=643 ca=114
196 5!c=14168 g=14169 pq=1 qp=0 dt=45465/140000000000000/0 df=161 of=11 ql=0/0 qs=N... kt=0/O ktl=b4d b=10 ci=47712 nci=0 co=677 ca=722 196 5!c=14168 g=14169 pq=1/0 qp=0 dt=45465/140000000000000/0 df=161 of=11 ql=0/0 qs=N... kt=0/O ktl=b4d b=10 ci=47712 nci=0 co=677 ca=722
197 6 c=14404 g=14405 pq=1 qp=0 dt=59454/0/0 df=94 of=6 ql=0/0 qs=.... kt=0/W ktl=e57 b=10 ci=55597 nci=0 co=701 ca=811 197 6 c=14404 g=14405 pq=1/0 qp=0 dt=59454/0/0 df=94 of=6 ql=0/0 qs=.... kt=0/W ktl=e57 b=10 ci=55597 nci=0 co=701 ca=811
198 7 c=14407 g=14408 pq=1 qp=1 dt=68850/0/0 df=31 of=8 ql=0/0 qs=.... kt=0/W ktl=14bd b=10 ci=77475 nci=0 co=508 ca=1042 198 7 c=14407 g=14408 pq=1/0 qp=1 dt=68850/0/0 df=31 of=8 ql=0/0 qs=.... kt=0/W ktl=14bd b=10 ci=77475 nci=0 co=508 ca=1042
199 199
200This is similar to the output discussed above, but contains the following 200This is similar to the output discussed above, but contains the following
201additional fields: 201additional fields:
diff --git a/Documentation/memory-barriers.txt b/Documentation/memory-barriers.txt
index 70a09f8a0383..ca2387ef27ab 100644
--- a/Documentation/memory-barriers.txt
+++ b/Documentation/memory-barriers.txt
@@ -269,6 +269,50 @@ And there are a number of things that _must_ or _must_not_ be assumed:
269 STORE *(A + 4) = Y; STORE *A = X; 269 STORE *(A + 4) = Y; STORE *A = X;
270 STORE {*A, *(A + 4) } = {X, Y}; 270 STORE {*A, *(A + 4) } = {X, Y};
271 271
272And there are anti-guarantees:
273
274 (*) These guarantees do not apply to bitfields, because compilers often
275 generate code to modify these using non-atomic read-modify-write
276 sequences. Do not attempt to use bitfields to synchronize parallel
277 algorithms.
278
279 (*) Even in cases where bitfields are protected by locks, all fields
280 in a given bitfield must be protected by one lock. If two fields
281 in a given bitfield are protected by different locks, the compiler's
282 non-atomic read-modify-write sequences can cause an update to one
283 field to corrupt the value of an adjacent field.
284
285 (*) These guarantees apply only to properly aligned and sized scalar
286 variables. "Properly sized" currently means variables that are
287 the same size as "char", "short", "int" and "long". "Properly
288 aligned" means the natural alignment, thus no constraints for
289 "char", two-byte alignment for "short", four-byte alignment for
290 "int", and either four-byte or eight-byte alignment for "long",
291 on 32-bit and 64-bit systems, respectively. Note that these
292 guarantees were introduced into the C11 standard, so beware when
293 using older pre-C11 compilers (for example, gcc 4.6). The portion
294 of the standard containing this guarantee is Section 3.14, which
295 defines "memory location" as follows:
296
297 memory location
298 either an object of scalar type, or a maximal sequence
299 of adjacent bit-fields all having nonzero width
300
301 NOTE 1: Two threads of execution can update and access
302 separate memory locations without interfering with
303 each other.
304
305 NOTE 2: A bit-field and an adjacent non-bit-field member
306 are in separate memory locations. The same applies
307 to two bit-fields, if one is declared inside a nested
308 structure declaration and the other is not, or if the two
309 are separated by a zero-length bit-field declaration,
310 or if they are separated by a non-bit-field member
311 declaration. It is not safe to concurrently update two
312 bit-fields in the same structure if all members declared
313 between them are also bit-fields, no matter what the
314 sizes of those intervening bit-fields happen to be.
315
272 316
273========================= 317=========================
274WHAT ARE MEMORY BARRIERS? 318WHAT ARE MEMORY BARRIERS?
@@ -750,7 +794,7 @@ In summary:
750 However, they do -not- guarantee any other sort of ordering: 794 However, they do -not- guarantee any other sort of ordering:
751 Not prior loads against later loads, nor prior stores against 795 Not prior loads against later loads, nor prior stores against
752 later anything. If you need these other forms of ordering, 796 later anything. If you need these other forms of ordering,
753 use smb_rmb(), smp_wmb(), or, in the case of prior stores and 797 use smp_rmb(), smp_wmb(), or, in the case of prior stores and
754 later loads, smp_mb(). 798 later loads, smp_mb().
755 799
756 (*) If both legs of the "if" statement begin with identical stores 800 (*) If both legs of the "if" statement begin with identical stores
diff --git a/arch/arm/kvm/Kconfig b/arch/arm/kvm/Kconfig
index 466bd299b1a8..3afee5f40f4f 100644
--- a/arch/arm/kvm/Kconfig
+++ b/arch/arm/kvm/Kconfig
@@ -23,6 +23,7 @@ config KVM
23 select HAVE_KVM_CPU_RELAX_INTERCEPT 23 select HAVE_KVM_CPU_RELAX_INTERCEPT
24 select KVM_MMIO 24 select KVM_MMIO
25 select KVM_ARM_HOST 25 select KVM_ARM_HOST
26 select SRCU
26 depends on ARM_VIRT_EXT && ARM_LPAE 27 depends on ARM_VIRT_EXT && ARM_LPAE
27 ---help--- 28 ---help---
28 Support hosting virtualized guest machines. You will also 29 Support hosting virtualized guest machines. You will also
diff --git a/arch/arm64/kvm/Kconfig b/arch/arm64/kvm/Kconfig
index 8ba85e9ea388..b334084d3675 100644
--- a/arch/arm64/kvm/Kconfig
+++ b/arch/arm64/kvm/Kconfig
@@ -26,6 +26,7 @@ config KVM
26 select KVM_ARM_HOST 26 select KVM_ARM_HOST
27 select KVM_ARM_VGIC 27 select KVM_ARM_VGIC
28 select KVM_ARM_TIMER 28 select KVM_ARM_TIMER
29 select SRCU
29 ---help--- 30 ---help---
30 Support hosting virtualized guest machines. 31 Support hosting virtualized guest machines.
31 32
diff --git a/arch/mips/kvm/Kconfig b/arch/mips/kvm/Kconfig
index 30e334e823bd..2ae12825529f 100644
--- a/arch/mips/kvm/Kconfig
+++ b/arch/mips/kvm/Kconfig
@@ -20,6 +20,7 @@ config KVM
20 select PREEMPT_NOTIFIERS 20 select PREEMPT_NOTIFIERS
21 select ANON_INODES 21 select ANON_INODES
22 select KVM_MMIO 22 select KVM_MMIO
23 select SRCU
23 ---help--- 24 ---help---
24 Support for hosting Guest kernels. 25 Support for hosting Guest kernels.
25 Currently supported on MIPS32 processors. 26 Currently supported on MIPS32 processors.
diff --git a/arch/powerpc/kvm/Kconfig b/arch/powerpc/kvm/Kconfig
index f5769f19ae25..11850f310fb4 100644
--- a/arch/powerpc/kvm/Kconfig
+++ b/arch/powerpc/kvm/Kconfig
@@ -21,6 +21,7 @@ config KVM
21 select PREEMPT_NOTIFIERS 21 select PREEMPT_NOTIFIERS
22 select ANON_INODES 22 select ANON_INODES
23 select HAVE_KVM_EVENTFD 23 select HAVE_KVM_EVENTFD
24 select SRCU
24 25
25config KVM_BOOK3S_HANDLER 26config KVM_BOOK3S_HANDLER
26 bool 27 bool
diff --git a/arch/s390/kvm/Kconfig b/arch/s390/kvm/Kconfig
index 646db9c467d1..5fce52cf0e57 100644
--- a/arch/s390/kvm/Kconfig
+++ b/arch/s390/kvm/Kconfig
@@ -28,6 +28,7 @@ config KVM
28 select HAVE_KVM_IRQCHIP 28 select HAVE_KVM_IRQCHIP
29 select HAVE_KVM_IRQFD 29 select HAVE_KVM_IRQFD
30 select HAVE_KVM_IRQ_ROUTING 30 select HAVE_KVM_IRQ_ROUTING
31 select SRCU
31 ---help--- 32 ---help---
32 Support hosting paravirtualized guest machines using the SIE 33 Support hosting paravirtualized guest machines using the SIE
33 virtualization capability on the mainframe. This should work 34 virtualization capability on the mainframe. This should work
diff --git a/arch/tile/kvm/Kconfig b/arch/tile/kvm/Kconfig
index 2298cb1daff7..1e968f7550dc 100644
--- a/arch/tile/kvm/Kconfig
+++ b/arch/tile/kvm/Kconfig
@@ -21,6 +21,7 @@ config KVM
21 depends on HAVE_KVM && MODULES 21 depends on HAVE_KVM && MODULES
22 select PREEMPT_NOTIFIERS 22 select PREEMPT_NOTIFIERS
23 select ANON_INODES 23 select ANON_INODES
24 select SRCU
24 ---help--- 25 ---help---
25 Support hosting paravirtualized guest machines. 26 Support hosting paravirtualized guest machines.
26 27
diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig
index ba397bde7948..661269953c1a 100644
--- a/arch/x86/Kconfig
+++ b/arch/x86/Kconfig
@@ -138,6 +138,7 @@ config X86
138 select HAVE_ACPI_APEI_NMI if ACPI 138 select HAVE_ACPI_APEI_NMI if ACPI
139 select ACPI_LEGACY_TABLES_LOOKUP if ACPI 139 select ACPI_LEGACY_TABLES_LOOKUP if ACPI
140 select X86_FEATURE_NAMES if PROC_FS 140 select X86_FEATURE_NAMES if PROC_FS
141 select SRCU
141 142
142config INSTRUCTION_DECODER 143config INSTRUCTION_DECODER
143 def_bool y 144 def_bool y
diff --git a/arch/x86/kvm/Kconfig b/arch/x86/kvm/Kconfig
index f9d16ff56c6b..7dc7ba577ecd 100644
--- a/arch/x86/kvm/Kconfig
+++ b/arch/x86/kvm/Kconfig
@@ -40,6 +40,7 @@ config KVM
40 select HAVE_KVM_MSI 40 select HAVE_KVM_MSI
41 select HAVE_KVM_CPU_RELAX_INTERCEPT 41 select HAVE_KVM_CPU_RELAX_INTERCEPT
42 select KVM_VFIO 42 select KVM_VFIO
43 select SRCU
43 ---help--- 44 ---help---
44 Support hosting fully virtualized guest machines using hardware 45 Support hosting fully virtualized guest machines using hardware
45 virtualization extensions. You will need a fairly recent 46 virtualization extensions. You will need a fairly recent
diff --git a/drivers/clk/Kconfig b/drivers/clk/Kconfig
index 3f44f292d066..91f86131bb7a 100644
--- a/drivers/clk/Kconfig
+++ b/drivers/clk/Kconfig
@@ -13,6 +13,7 @@ config COMMON_CLK
13 bool 13 bool
14 select HAVE_CLK_PREPARE 14 select HAVE_CLK_PREPARE
15 select CLKDEV_LOOKUP 15 select CLKDEV_LOOKUP
16 select SRCU
16 ---help--- 17 ---help---
17 The common clock framework is a single definition of struct 18 The common clock framework is a single definition of struct
18 clk, useful across many platforms, as well as an 19 clk, useful across many platforms, as well as an
diff --git a/drivers/cpufreq/Kconfig b/drivers/cpufreq/Kconfig
index 29b2ef5a68b9..a171fef2c2b6 100644
--- a/drivers/cpufreq/Kconfig
+++ b/drivers/cpufreq/Kconfig
@@ -2,6 +2,7 @@ menu "CPU Frequency scaling"
2 2
3config CPU_FREQ 3config CPU_FREQ
4 bool "CPU Frequency scaling" 4 bool "CPU Frequency scaling"
5 select SRCU
5 help 6 help
6 CPU Frequency scaling allows you to change the clock speed of 7 CPU Frequency scaling allows you to change the clock speed of
7 CPUs on the fly. This is a nice method to save power, because 8 CPUs on the fly. This is a nice method to save power, because
diff --git a/drivers/devfreq/Kconfig b/drivers/devfreq/Kconfig
index faf4e70c42e0..3891f6781298 100644
--- a/drivers/devfreq/Kconfig
+++ b/drivers/devfreq/Kconfig
@@ -1,5 +1,6 @@
1menuconfig PM_DEVFREQ 1menuconfig PM_DEVFREQ
2 bool "Generic Dynamic Voltage and Frequency Scaling (DVFS) support" 2 bool "Generic Dynamic Voltage and Frequency Scaling (DVFS) support"
3 select SRCU
3 help 4 help
4 A device may have a list of frequencies and voltages available. 5 A device may have a list of frequencies and voltages available.
5 devfreq, a generic DVFS framework can be registered for a device 6 devfreq, a generic DVFS framework can be registered for a device
diff --git a/drivers/md/Kconfig b/drivers/md/Kconfig
index 5bdedf6df153..c355a226a024 100644
--- a/drivers/md/Kconfig
+++ b/drivers/md/Kconfig
@@ -5,6 +5,7 @@
5menuconfig MD 5menuconfig MD
6 bool "Multiple devices driver support (RAID and LVM)" 6 bool "Multiple devices driver support (RAID and LVM)"
7 depends on BLOCK 7 depends on BLOCK
8 select SRCU
8 help 9 help
9 Support multiple physical spindles through a single logical device. 10 Support multiple physical spindles through a single logical device.
10 Required for RAID and logical volume management. 11 Required for RAID and logical volume management.
diff --git a/drivers/net/Kconfig b/drivers/net/Kconfig
index d6607ee9c855..84673ebcf428 100644
--- a/drivers/net/Kconfig
+++ b/drivers/net/Kconfig
@@ -197,6 +197,7 @@ config NETCONSOLE_DYNAMIC
197 197
198config NETPOLL 198config NETPOLL
199 def_bool NETCONSOLE 199 def_bool NETCONSOLE
200 select SRCU
200 201
201config NET_POLL_CONTROLLER 202config NET_POLL_CONTROLLER
202 def_bool NETPOLL 203 def_bool NETPOLL
diff --git a/fs/btrfs/Kconfig b/fs/btrfs/Kconfig
index a66768ebc8d1..80e9c18ea64f 100644
--- a/fs/btrfs/Kconfig
+++ b/fs/btrfs/Kconfig
@@ -8,6 +8,7 @@ config BTRFS_FS
8 select LZO_DECOMPRESS 8 select LZO_DECOMPRESS
9 select RAID6_PQ 9 select RAID6_PQ
10 select XOR_BLOCKS 10 select XOR_BLOCKS
11 select SRCU
11 12
12 help 13 help
13 Btrfs is a general purpose copy-on-write filesystem with extents, 14 Btrfs is a general purpose copy-on-write filesystem with extents,
diff --git a/fs/notify/Kconfig b/fs/notify/Kconfig
index 22c629eedd82..2a24249b30af 100644
--- a/fs/notify/Kconfig
+++ b/fs/notify/Kconfig
@@ -1,5 +1,6 @@
1config FSNOTIFY 1config FSNOTIFY
2 def_bool n 2 def_bool n
3 select SRCU
3 4
4source "fs/notify/dnotify/Kconfig" 5source "fs/notify/dnotify/Kconfig"
5source "fs/notify/inotify/Kconfig" 6source "fs/notify/inotify/Kconfig"
diff --git a/fs/quota/Kconfig b/fs/quota/Kconfig
index c51df1dd237e..4a09975aac90 100644
--- a/fs/quota/Kconfig
+++ b/fs/quota/Kconfig
@@ -5,6 +5,7 @@
5config QUOTA 5config QUOTA
6 bool "Quota support" 6 bool "Quota support"
7 select QUOTACTL 7 select QUOTACTL
8 select SRCU
8 help 9 help
9 If you say Y here, you will be able to set per user limits for disk 10 If you say Y here, you will be able to set per user limits for disk
10 usage (also called disk quotas). Currently, it works for the 11 usage (also called disk quotas). Currently, it works for the
diff --git a/include/linux/compiler.h b/include/linux/compiler.h
index a1c81f80978e..49811cdddaa5 100644
--- a/include/linux/compiler.h
+++ b/include/linux/compiler.h
@@ -385,7 +385,7 @@ static __always_inline void __assign_once_size(volatile void *p, void *res, int
385 385
386/* Is this type a native word size -- useful for atomic operations */ 386/* Is this type a native word size -- useful for atomic operations */
387#ifndef __native_word 387#ifndef __native_word
388# define __native_word(t) (sizeof(t) == sizeof(int) || sizeof(t) == sizeof(long)) 388# define __native_word(t) (sizeof(t) == sizeof(char) || sizeof(t) == sizeof(short) || sizeof(t) == sizeof(int) || sizeof(t) == sizeof(long))
389#endif 389#endif
390 390
391/* Compile time object size, -1 for unknown */ 391/* Compile time object size, -1 for unknown */
diff --git a/include/linux/rculist.h b/include/linux/rculist.h
index 529bc946f450..a18b16f1dc0e 100644
--- a/include/linux/rculist.h
+++ b/include/linux/rculist.h
@@ -524,11 +524,11 @@ static inline void hlist_add_behind_rcu(struct hlist_node *n,
524 * @member: the name of the hlist_node within the struct. 524 * @member: the name of the hlist_node within the struct.
525 */ 525 */
526#define hlist_for_each_entry_continue_rcu(pos, member) \ 526#define hlist_for_each_entry_continue_rcu(pos, member) \
527 for (pos = hlist_entry_safe(rcu_dereference((pos)->member.next),\ 527 for (pos = hlist_entry_safe(rcu_dereference_raw(hlist_next_rcu( \
528 typeof(*(pos)), member); \ 528 &(pos)->member)), typeof(*(pos)), member); \
529 pos; \ 529 pos; \
530 pos = hlist_entry_safe(rcu_dereference((pos)->member.next),\ 530 pos = hlist_entry_safe(rcu_dereference_raw(hlist_next_rcu( \
531 typeof(*(pos)), member)) 531 &(pos)->member)), typeof(*(pos)), member))
532 532
533/** 533/**
534 * hlist_for_each_entry_continue_rcu_bh - iterate over a hlist continuing after current point 534 * hlist_for_each_entry_continue_rcu_bh - iterate over a hlist continuing after current point
@@ -536,11 +536,11 @@ static inline void hlist_add_behind_rcu(struct hlist_node *n,
536 * @member: the name of the hlist_node within the struct. 536 * @member: the name of the hlist_node within the struct.
537 */ 537 */
538#define hlist_for_each_entry_continue_rcu_bh(pos, member) \ 538#define hlist_for_each_entry_continue_rcu_bh(pos, member) \
539 for (pos = hlist_entry_safe(rcu_dereference_bh((pos)->member.next),\ 539 for (pos = hlist_entry_safe(rcu_dereference_bh(hlist_next_rcu( \
540 typeof(*(pos)), member); \ 540 &(pos)->member)), typeof(*(pos)), member); \
541 pos; \ 541 pos; \
542 pos = hlist_entry_safe(rcu_dereference_bh((pos)->member.next),\ 542 pos = hlist_entry_safe(rcu_dereference_bh(hlist_next_rcu( \
543 typeof(*(pos)), member)) 543 &(pos)->member)), typeof(*(pos)), member))
544 544
545/** 545/**
546 * hlist_for_each_entry_from_rcu - iterate over a hlist continuing from current point 546 * hlist_for_each_entry_from_rcu - iterate over a hlist continuing from current point
diff --git a/include/linux/rcupdate.h b/include/linux/rcupdate.h
index ed4f5939a452..78097491cd99 100644
--- a/include/linux/rcupdate.h
+++ b/include/linux/rcupdate.h
@@ -331,12 +331,13 @@ static inline void rcu_init_nohz(void)
331extern struct srcu_struct tasks_rcu_exit_srcu; 331extern struct srcu_struct tasks_rcu_exit_srcu;
332#define rcu_note_voluntary_context_switch(t) \ 332#define rcu_note_voluntary_context_switch(t) \
333 do { \ 333 do { \
334 rcu_all_qs(); \
334 if (ACCESS_ONCE((t)->rcu_tasks_holdout)) \ 335 if (ACCESS_ONCE((t)->rcu_tasks_holdout)) \
335 ACCESS_ONCE((t)->rcu_tasks_holdout) = false; \ 336 ACCESS_ONCE((t)->rcu_tasks_holdout) = false; \
336 } while (0) 337 } while (0)
337#else /* #ifdef CONFIG_TASKS_RCU */ 338#else /* #ifdef CONFIG_TASKS_RCU */
338#define TASKS_RCU(x) do { } while (0) 339#define TASKS_RCU(x) do { } while (0)
339#define rcu_note_voluntary_context_switch(t) do { } while (0) 340#define rcu_note_voluntary_context_switch(t) rcu_all_qs()
340#endif /* #else #ifdef CONFIG_TASKS_RCU */ 341#endif /* #else #ifdef CONFIG_TASKS_RCU */
341 342
342/** 343/**
@@ -582,11 +583,11 @@ static inline void rcu_preempt_sleep_check(void)
582}) 583})
583#define __rcu_dereference_check(p, c, space) \ 584#define __rcu_dereference_check(p, c, space) \
584({ \ 585({ \
585 typeof(*p) *_________p1 = (typeof(*p) *__force)ACCESS_ONCE(p); \ 586 /* Dependency order vs. p above. */ \
587 typeof(*p) *________p1 = (typeof(*p) *__force)lockless_dereference(p); \
586 rcu_lockdep_assert(c, "suspicious rcu_dereference_check() usage"); \ 588 rcu_lockdep_assert(c, "suspicious rcu_dereference_check() usage"); \
587 rcu_dereference_sparse(p, space); \ 589 rcu_dereference_sparse(p, space); \
588 smp_read_barrier_depends(); /* Dependency order vs. p above. */ \ 590 ((typeof(*p) __force __kernel *)(________p1)); \
589 ((typeof(*p) __force __kernel *)(_________p1)); \
590}) 591})
591#define __rcu_dereference_protected(p, c, space) \ 592#define __rcu_dereference_protected(p, c, space) \
592({ \ 593({ \
@@ -603,10 +604,10 @@ static inline void rcu_preempt_sleep_check(void)
603}) 604})
604#define __rcu_dereference_index_check(p, c) \ 605#define __rcu_dereference_index_check(p, c) \
605({ \ 606({ \
606 typeof(p) _________p1 = ACCESS_ONCE(p); \ 607 /* Dependency order vs. p above. */ \
608 typeof(p) _________p1 = lockless_dereference(p); \
607 rcu_lockdep_assert(c, \ 609 rcu_lockdep_assert(c, \
608 "suspicious rcu_dereference_index_check() usage"); \ 610 "suspicious rcu_dereference_index_check() usage"); \
609 smp_read_barrier_depends(); /* Dependency order vs. p above. */ \
610 (_________p1); \ 611 (_________p1); \
611}) 612})
612 613
diff --git a/include/linux/rcutiny.h b/include/linux/rcutiny.h
index 984192160e9b..937edaeb150d 100644
--- a/include/linux/rcutiny.h
+++ b/include/linux/rcutiny.h
@@ -186,7 +186,10 @@ static inline bool rcu_is_watching(void)
186 return true; 186 return true;
187} 187}
188 188
189
190#endif /* #else defined(CONFIG_DEBUG_LOCK_ALLOC) || defined(CONFIG_RCU_TRACE) */ 189#endif /* #else defined(CONFIG_DEBUG_LOCK_ALLOC) || defined(CONFIG_RCU_TRACE) */
191 190
191static inline void rcu_all_qs(void)
192{
193}
194
192#endif /* __LINUX_RCUTINY_H */ 195#endif /* __LINUX_RCUTINY_H */
diff --git a/include/linux/rcutree.h b/include/linux/rcutree.h
index c0dd124e69ec..d2e583a6aaca 100644
--- a/include/linux/rcutree.h
+++ b/include/linux/rcutree.h
@@ -100,4 +100,6 @@ extern int rcu_scheduler_active __read_mostly;
100 100
101bool rcu_is_watching(void); 101bool rcu_is_watching(void);
102 102
103void rcu_all_qs(void);
104
103#endif /* __LINUX_RCUTREE_H */ 105#endif /* __LINUX_RCUTREE_H */
diff --git a/include/linux/srcu.h b/include/linux/srcu.h
index a2783cb5d275..9cfd9623fb03 100644
--- a/include/linux/srcu.h
+++ b/include/linux/srcu.h
@@ -45,7 +45,7 @@ struct rcu_batch {
45#define RCU_BATCH_INIT(name) { NULL, &(name.head) } 45#define RCU_BATCH_INIT(name) { NULL, &(name.head) }
46 46
47struct srcu_struct { 47struct srcu_struct {
48 unsigned completed; 48 unsigned long completed;
49 struct srcu_struct_array __percpu *per_cpu_ref; 49 struct srcu_struct_array __percpu *per_cpu_ref;
50 spinlock_t queue_lock; /* protect ->batch_queue, ->running */ 50 spinlock_t queue_lock; /* protect ->batch_queue, ->running */
51 bool running; 51 bool running;
@@ -102,13 +102,11 @@ void process_srcu(struct work_struct *work);
102 * define and init a srcu struct at build time. 102 * define and init a srcu struct at build time.
103 * dont't call init_srcu_struct() nor cleanup_srcu_struct() on it. 103 * dont't call init_srcu_struct() nor cleanup_srcu_struct() on it.
104 */ 104 */
105#define DEFINE_SRCU(name) \ 105#define __DEFINE_SRCU(name, is_static) \
106 static DEFINE_PER_CPU(struct srcu_struct_array, name##_srcu_array);\ 106 static DEFINE_PER_CPU(struct srcu_struct_array, name##_srcu_array);\
107 struct srcu_struct name = __SRCU_STRUCT_INIT(name); 107 is_static struct srcu_struct name = __SRCU_STRUCT_INIT(name)
108 108#define DEFINE_SRCU(name) __DEFINE_SRCU(name, /* not static */)
109#define DEFINE_STATIC_SRCU(name) \ 109#define DEFINE_STATIC_SRCU(name) __DEFINE_SRCU(name, static)
110 static DEFINE_PER_CPU(struct srcu_struct_array, name##_srcu_array);\
111 static struct srcu_struct name = __SRCU_STRUCT_INIT(name);
112 110
113/** 111/**
114 * call_srcu() - Queue a callback for invocation after an SRCU grace period 112 * call_srcu() - Queue a callback for invocation after an SRCU grace period
@@ -135,7 +133,7 @@ int __srcu_read_lock(struct srcu_struct *sp) __acquires(sp);
135void __srcu_read_unlock(struct srcu_struct *sp, int idx) __releases(sp); 133void __srcu_read_unlock(struct srcu_struct *sp, int idx) __releases(sp);
136void synchronize_srcu(struct srcu_struct *sp); 134void synchronize_srcu(struct srcu_struct *sp);
137void synchronize_srcu_expedited(struct srcu_struct *sp); 135void synchronize_srcu_expedited(struct srcu_struct *sp);
138long srcu_batches_completed(struct srcu_struct *sp); 136unsigned long srcu_batches_completed(struct srcu_struct *sp);
139void srcu_barrier(struct srcu_struct *sp); 137void srcu_barrier(struct srcu_struct *sp);
140 138
141#ifdef CONFIG_DEBUG_LOCK_ALLOC 139#ifdef CONFIG_DEBUG_LOCK_ALLOC
diff --git a/init/Kconfig b/init/Kconfig
index 9afb971497f4..1354ac09b516 100644
--- a/init/Kconfig
+++ b/init/Kconfig
@@ -470,7 +470,6 @@ choice
470config TREE_RCU 470config TREE_RCU
471 bool "Tree-based hierarchical RCU" 471 bool "Tree-based hierarchical RCU"
472 depends on !PREEMPT && SMP 472 depends on !PREEMPT && SMP
473 select IRQ_WORK
474 help 473 help
475 This option selects the RCU implementation that is 474 This option selects the RCU implementation that is
476 designed for very large SMP system with hundreds or 475 designed for very large SMP system with hundreds or
@@ -480,7 +479,6 @@ config TREE_RCU
480config PREEMPT_RCU 479config PREEMPT_RCU
481 bool "Preemptible tree-based hierarchical RCU" 480 bool "Preemptible tree-based hierarchical RCU"
482 depends on PREEMPT 481 depends on PREEMPT
483 select IRQ_WORK
484 help 482 help
485 This option selects the RCU implementation that is 483 This option selects the RCU implementation that is
486 designed for very large SMP systems with hundreds or 484 designed for very large SMP systems with hundreds or
@@ -501,9 +499,17 @@ config TINY_RCU
501 499
502endchoice 500endchoice
503 501
502config SRCU
503 bool
504 help
505 This option selects the sleepable version of RCU. This version
506 permits arbitrary sleeping or blocking within RCU read-side critical
507 sections.
508
504config TASKS_RCU 509config TASKS_RCU
505 bool "Task_based RCU implementation using voluntary context switch" 510 bool "Task_based RCU implementation using voluntary context switch"
506 default n 511 default n
512 select SRCU
507 help 513 help
508 This option enables a task-based RCU implementation that uses 514 This option enables a task-based RCU implementation that uses
509 only voluntary context switch (not preemption!), idle, and 515 only voluntary context switch (not preemption!), idle, and
@@ -668,9 +674,10 @@ config RCU_BOOST
668 674
669config RCU_KTHREAD_PRIO 675config RCU_KTHREAD_PRIO
670 int "Real-time priority to use for RCU worker threads" 676 int "Real-time priority to use for RCU worker threads"
671 range 1 99 677 range 1 99 if RCU_BOOST
672 depends on RCU_BOOST 678 range 0 99 if !RCU_BOOST
673 default 1 679 default 1 if RCU_BOOST
680 default 0 if !RCU_BOOST
674 help 681 help
675 This option specifies the SCHED_FIFO priority value that will be 682 This option specifies the SCHED_FIFO priority value that will be
676 assigned to the rcuc/n and rcub/n threads and is also the value 683 assigned to the rcuc/n and rcub/n threads and is also the value
@@ -1595,6 +1602,7 @@ config PERF_EVENTS
1595 depends on HAVE_PERF_EVENTS 1602 depends on HAVE_PERF_EVENTS
1596 select ANON_INODES 1603 select ANON_INODES
1597 select IRQ_WORK 1604 select IRQ_WORK
1605 select SRCU
1598 help 1606 help
1599 Enable kernel support for various performance events provided 1607 Enable kernel support for various performance events provided
1600 by software and hardware. 1608 by software and hardware.
diff --git a/kernel/cpu.c b/kernel/cpu.c
index 5d220234b3ca..1972b161c61e 100644
--- a/kernel/cpu.c
+++ b/kernel/cpu.c
@@ -58,22 +58,23 @@ static int cpu_hotplug_disabled;
58 58
59static struct { 59static struct {
60 struct task_struct *active_writer; 60 struct task_struct *active_writer;
61 struct mutex lock; /* Synchronizes accesses to refcount, */ 61 /* wait queue to wake up the active_writer */
62 wait_queue_head_t wq;
63 /* verifies that no writer will get active while readers are active */
64 struct mutex lock;
62 /* 65 /*
63 * Also blocks the new readers during 66 * Also blocks the new readers during
64 * an ongoing cpu hotplug operation. 67 * an ongoing cpu hotplug operation.
65 */ 68 */
66 int refcount; 69 atomic_t refcount;
67 /* And allows lockless put_online_cpus(). */
68 atomic_t puts_pending;
69 70
70#ifdef CONFIG_DEBUG_LOCK_ALLOC 71#ifdef CONFIG_DEBUG_LOCK_ALLOC
71 struct lockdep_map dep_map; 72 struct lockdep_map dep_map;
72#endif 73#endif
73} cpu_hotplug = { 74} cpu_hotplug = {
74 .active_writer = NULL, 75 .active_writer = NULL,
76 .wq = __WAIT_QUEUE_HEAD_INITIALIZER(cpu_hotplug.wq),
75 .lock = __MUTEX_INITIALIZER(cpu_hotplug.lock), 77 .lock = __MUTEX_INITIALIZER(cpu_hotplug.lock),
76 .refcount = 0,
77#ifdef CONFIG_DEBUG_LOCK_ALLOC 78#ifdef CONFIG_DEBUG_LOCK_ALLOC
78 .dep_map = {.name = "cpu_hotplug.lock" }, 79 .dep_map = {.name = "cpu_hotplug.lock" },
79#endif 80#endif
@@ -86,15 +87,6 @@ static struct {
86#define cpuhp_lock_acquire() lock_map_acquire(&cpu_hotplug.dep_map) 87#define cpuhp_lock_acquire() lock_map_acquire(&cpu_hotplug.dep_map)
87#define cpuhp_lock_release() lock_map_release(&cpu_hotplug.dep_map) 88#define cpuhp_lock_release() lock_map_release(&cpu_hotplug.dep_map)
88 89
89static void apply_puts_pending(int max)
90{
91 int delta;
92
93 if (atomic_read(&cpu_hotplug.puts_pending) >= max) {
94 delta = atomic_xchg(&cpu_hotplug.puts_pending, 0);
95 cpu_hotplug.refcount -= delta;
96 }
97}
98 90
99void get_online_cpus(void) 91void get_online_cpus(void)
100{ 92{
@@ -103,8 +95,7 @@ void get_online_cpus(void)
103 return; 95 return;
104 cpuhp_lock_acquire_read(); 96 cpuhp_lock_acquire_read();
105 mutex_lock(&cpu_hotplug.lock); 97 mutex_lock(&cpu_hotplug.lock);
106 apply_puts_pending(65536); 98 atomic_inc(&cpu_hotplug.refcount);
107 cpu_hotplug.refcount++;
108 mutex_unlock(&cpu_hotplug.lock); 99 mutex_unlock(&cpu_hotplug.lock);
109} 100}
110EXPORT_SYMBOL_GPL(get_online_cpus); 101EXPORT_SYMBOL_GPL(get_online_cpus);
@@ -116,8 +107,7 @@ bool try_get_online_cpus(void)
116 if (!mutex_trylock(&cpu_hotplug.lock)) 107 if (!mutex_trylock(&cpu_hotplug.lock))
117 return false; 108 return false;
118 cpuhp_lock_acquire_tryread(); 109 cpuhp_lock_acquire_tryread();
119 apply_puts_pending(65536); 110 atomic_inc(&cpu_hotplug.refcount);
120 cpu_hotplug.refcount++;
121 mutex_unlock(&cpu_hotplug.lock); 111 mutex_unlock(&cpu_hotplug.lock);
122 return true; 112 return true;
123} 113}
@@ -125,20 +115,18 @@ EXPORT_SYMBOL_GPL(try_get_online_cpus);
125 115
126void put_online_cpus(void) 116void put_online_cpus(void)
127{ 117{
118 int refcount;
119
128 if (cpu_hotplug.active_writer == current) 120 if (cpu_hotplug.active_writer == current)
129 return; 121 return;
130 if (!mutex_trylock(&cpu_hotplug.lock)) {
131 atomic_inc(&cpu_hotplug.puts_pending);
132 cpuhp_lock_release();
133 return;
134 }
135 122
136 if (WARN_ON(!cpu_hotplug.refcount)) 123 refcount = atomic_dec_return(&cpu_hotplug.refcount);
137 cpu_hotplug.refcount++; /* try to fix things up */ 124 if (WARN_ON(refcount < 0)) /* try to fix things up */
125 atomic_inc(&cpu_hotplug.refcount);
126
127 if (refcount <= 0 && waitqueue_active(&cpu_hotplug.wq))
128 wake_up(&cpu_hotplug.wq);
138 129
139 if (!--cpu_hotplug.refcount && unlikely(cpu_hotplug.active_writer))
140 wake_up_process(cpu_hotplug.active_writer);
141 mutex_unlock(&cpu_hotplug.lock);
142 cpuhp_lock_release(); 130 cpuhp_lock_release();
143 131
144} 132}
@@ -168,18 +156,20 @@ EXPORT_SYMBOL_GPL(put_online_cpus);
168 */ 156 */
169void cpu_hotplug_begin(void) 157void cpu_hotplug_begin(void)
170{ 158{
171 cpu_hotplug.active_writer = current; 159 DEFINE_WAIT(wait);
172 160
161 cpu_hotplug.active_writer = current;
173 cpuhp_lock_acquire(); 162 cpuhp_lock_acquire();
163
174 for (;;) { 164 for (;;) {
175 mutex_lock(&cpu_hotplug.lock); 165 mutex_lock(&cpu_hotplug.lock);
176 apply_puts_pending(1); 166 prepare_to_wait(&cpu_hotplug.wq, &wait, TASK_UNINTERRUPTIBLE);
177 if (likely(!cpu_hotplug.refcount)) 167 if (likely(!atomic_read(&cpu_hotplug.refcount)))
178 break; 168 break;
179 __set_current_state(TASK_UNINTERRUPTIBLE);
180 mutex_unlock(&cpu_hotplug.lock); 169 mutex_unlock(&cpu_hotplug.lock);
181 schedule(); 170 schedule();
182 } 171 }
172 finish_wait(&cpu_hotplug.wq, &wait);
183} 173}
184 174
185void cpu_hotplug_done(void) 175void cpu_hotplug_done(void)
diff --git a/kernel/notifier.c b/kernel/notifier.c
index 4803da6eab62..ae9fc7cc360e 100644
--- a/kernel/notifier.c
+++ b/kernel/notifier.c
@@ -402,6 +402,7 @@ int raw_notifier_call_chain(struct raw_notifier_head *nh,
402} 402}
403EXPORT_SYMBOL_GPL(raw_notifier_call_chain); 403EXPORT_SYMBOL_GPL(raw_notifier_call_chain);
404 404
405#ifdef CONFIG_SRCU
405/* 406/*
406 * SRCU notifier chain routines. Registration and unregistration 407 * SRCU notifier chain routines. Registration and unregistration
407 * use a mutex, and call_chain is synchronized by SRCU (no locks). 408 * use a mutex, and call_chain is synchronized by SRCU (no locks).
@@ -528,6 +529,8 @@ void srcu_init_notifier_head(struct srcu_notifier_head *nh)
528} 529}
529EXPORT_SYMBOL_GPL(srcu_init_notifier_head); 530EXPORT_SYMBOL_GPL(srcu_init_notifier_head);
530 531
532#endif /* CONFIG_SRCU */
533
531static ATOMIC_NOTIFIER_HEAD(die_chain); 534static ATOMIC_NOTIFIER_HEAD(die_chain);
532 535
533int notrace notify_die(enum die_val val, const char *str, 536int notrace notify_die(enum die_val val, const char *str,
diff --git a/kernel/power/Kconfig b/kernel/power/Kconfig
index 48b28d387c7f..7e01f78f0417 100644
--- a/kernel/power/Kconfig
+++ b/kernel/power/Kconfig
@@ -251,6 +251,7 @@ config APM_EMULATION
251 251
252config PM_OPP 252config PM_OPP
253 bool 253 bool
254 select SRCU
254 ---help--- 255 ---help---
255 SOCs have a standard set of tuples consisting of frequency and 256 SOCs have a standard set of tuples consisting of frequency and
256 voltage pairs that the device will support per voltage domain. This 257 voltage pairs that the device will support per voltage domain. This
diff --git a/kernel/rcu/Makefile b/kernel/rcu/Makefile
index e6fae503d1bc..50a808424b06 100644
--- a/kernel/rcu/Makefile
+++ b/kernel/rcu/Makefile
@@ -1,4 +1,5 @@
1obj-y += update.o srcu.o 1obj-y += update.o
2obj-$(CONFIG_SRCU) += srcu.o
2obj-$(CONFIG_RCU_TORTURE_TEST) += rcutorture.o 3obj-$(CONFIG_RCU_TORTURE_TEST) += rcutorture.o
3obj-$(CONFIG_TREE_RCU) += tree.o 4obj-$(CONFIG_TREE_RCU) += tree.o
4obj-$(CONFIG_PREEMPT_RCU) += tree.o 5obj-$(CONFIG_PREEMPT_RCU) += tree.o
diff --git a/kernel/rcu/rcu.h b/kernel/rcu/rcu.h
index 07bb02eda844..80adef7d4c3d 100644
--- a/kernel/rcu/rcu.h
+++ b/kernel/rcu/rcu.h
@@ -137,4 +137,10 @@ int rcu_jiffies_till_stall_check(void);
137 137
138void rcu_early_boot_tests(void); 138void rcu_early_boot_tests(void);
139 139
140/*
141 * This function really isn't for public consumption, but RCU is special in
142 * that context switches can allow the state machine to make progress.
143 */
144extern void resched_cpu(int cpu);
145
140#endif /* __LINUX_RCU_H */ 146#endif /* __LINUX_RCU_H */
diff --git a/kernel/rcu/srcu.c b/kernel/rcu/srcu.c
index e037f3eb2f7b..445bf8ffe3fb 100644
--- a/kernel/rcu/srcu.c
+++ b/kernel/rcu/srcu.c
@@ -546,7 +546,7 @@ EXPORT_SYMBOL_GPL(srcu_barrier);
546 * Report the number of batches, correlated with, but not necessarily 546 * Report the number of batches, correlated with, but not necessarily
547 * precisely the same as, the number of grace periods that have elapsed. 547 * precisely the same as, the number of grace periods that have elapsed.
548 */ 548 */
549long srcu_batches_completed(struct srcu_struct *sp) 549unsigned long srcu_batches_completed(struct srcu_struct *sp)
550{ 550{
551 return sp->completed; 551 return sp->completed;
552} 552}
diff --git a/kernel/rcu/tiny.c b/kernel/rcu/tiny.c
index 0db5649f8817..cc9ceca7bde1 100644
--- a/kernel/rcu/tiny.c
+++ b/kernel/rcu/tiny.c
@@ -47,54 +47,14 @@ static void __call_rcu(struct rcu_head *head,
47 void (*func)(struct rcu_head *rcu), 47 void (*func)(struct rcu_head *rcu),
48 struct rcu_ctrlblk *rcp); 48 struct rcu_ctrlblk *rcp);
49 49
50static long long rcu_dynticks_nesting = DYNTICK_TASK_EXIT_IDLE;
51
52#include "tiny_plugin.h" 50#include "tiny_plugin.h"
53 51
54/* Common code for rcu_idle_enter() and rcu_irq_exit(), see kernel/rcu/tree.c. */
55static void rcu_idle_enter_common(long long newval)
56{
57 if (newval) {
58 RCU_TRACE(trace_rcu_dyntick(TPS("--="),
59 rcu_dynticks_nesting, newval));
60 rcu_dynticks_nesting = newval;
61 return;
62 }
63 RCU_TRACE(trace_rcu_dyntick(TPS("Start"),
64 rcu_dynticks_nesting, newval));
65 if (IS_ENABLED(CONFIG_RCU_TRACE) && !is_idle_task(current)) {
66 struct task_struct *idle __maybe_unused = idle_task(smp_processor_id());
67
68 RCU_TRACE(trace_rcu_dyntick(TPS("Entry error: not idle task"),
69 rcu_dynticks_nesting, newval));
70 ftrace_dump(DUMP_ALL);
71 WARN_ONCE(1, "Current pid: %d comm: %s / Idle pid: %d comm: %s",
72 current->pid, current->comm,
73 idle->pid, idle->comm); /* must be idle task! */
74 }
75 rcu_sched_qs(); /* implies rcu_bh_inc() */
76 barrier();
77 rcu_dynticks_nesting = newval;
78}
79
80/* 52/*
81 * Enter idle, which is an extended quiescent state if we have fully 53 * Enter idle, which is an extended quiescent state if we have fully
82 * entered that mode (i.e., if the new value of dynticks_nesting is zero). 54 * entered that mode.
83 */ 55 */
84void rcu_idle_enter(void) 56void rcu_idle_enter(void)
85{ 57{
86 unsigned long flags;
87 long long newval;
88
89 local_irq_save(flags);
90 WARN_ON_ONCE((rcu_dynticks_nesting & DYNTICK_TASK_NEST_MASK) == 0);
91 if ((rcu_dynticks_nesting & DYNTICK_TASK_NEST_MASK) ==
92 DYNTICK_TASK_NEST_VALUE)
93 newval = 0;
94 else
95 newval = rcu_dynticks_nesting - DYNTICK_TASK_NEST_VALUE;
96 rcu_idle_enter_common(newval);
97 local_irq_restore(flags);
98} 58}
99EXPORT_SYMBOL_GPL(rcu_idle_enter); 59EXPORT_SYMBOL_GPL(rcu_idle_enter);
100 60
@@ -103,55 +63,14 @@ EXPORT_SYMBOL_GPL(rcu_idle_enter);
103 */ 63 */
104void rcu_irq_exit(void) 64void rcu_irq_exit(void)
105{ 65{
106 unsigned long flags;
107 long long newval;
108
109 local_irq_save(flags);
110 newval = rcu_dynticks_nesting - 1;
111 WARN_ON_ONCE(newval < 0);
112 rcu_idle_enter_common(newval);
113 local_irq_restore(flags);
114} 66}
115EXPORT_SYMBOL_GPL(rcu_irq_exit); 67EXPORT_SYMBOL_GPL(rcu_irq_exit);
116 68
117/* Common code for rcu_idle_exit() and rcu_irq_enter(), see kernel/rcu/tree.c. */
118static void rcu_idle_exit_common(long long oldval)
119{
120 if (oldval) {
121 RCU_TRACE(trace_rcu_dyntick(TPS("++="),
122 oldval, rcu_dynticks_nesting));
123 return;
124 }
125 RCU_TRACE(trace_rcu_dyntick(TPS("End"), oldval, rcu_dynticks_nesting));
126 if (IS_ENABLED(CONFIG_RCU_TRACE) && !is_idle_task(current)) {
127 struct task_struct *idle __maybe_unused = idle_task(smp_processor_id());
128
129 RCU_TRACE(trace_rcu_dyntick(TPS("Exit error: not idle task"),
130 oldval, rcu_dynticks_nesting));
131 ftrace_dump(DUMP_ALL);
132 WARN_ONCE(1, "Current pid: %d comm: %s / Idle pid: %d comm: %s",
133 current->pid, current->comm,
134 idle->pid, idle->comm); /* must be idle task! */
135 }
136}
137
138/* 69/*
139 * Exit idle, so that we are no longer in an extended quiescent state. 70 * Exit idle, so that we are no longer in an extended quiescent state.
140 */ 71 */
141void rcu_idle_exit(void) 72void rcu_idle_exit(void)
142{ 73{
143 unsigned long flags;
144 long long oldval;
145
146 local_irq_save(flags);
147 oldval = rcu_dynticks_nesting;
148 WARN_ON_ONCE(rcu_dynticks_nesting < 0);
149 if (rcu_dynticks_nesting & DYNTICK_TASK_NEST_MASK)
150 rcu_dynticks_nesting += DYNTICK_TASK_NEST_VALUE;
151 else
152 rcu_dynticks_nesting = DYNTICK_TASK_EXIT_IDLE;
153 rcu_idle_exit_common(oldval);
154 local_irq_restore(flags);
155} 74}
156EXPORT_SYMBOL_GPL(rcu_idle_exit); 75EXPORT_SYMBOL_GPL(rcu_idle_exit);
157 76
@@ -160,15 +79,6 @@ EXPORT_SYMBOL_GPL(rcu_idle_exit);
160 */ 79 */
161void rcu_irq_enter(void) 80void rcu_irq_enter(void)
162{ 81{
163 unsigned long flags;
164 long long oldval;
165
166 local_irq_save(flags);
167 oldval = rcu_dynticks_nesting;
168 rcu_dynticks_nesting++;
169 WARN_ON_ONCE(rcu_dynticks_nesting == 0);
170 rcu_idle_exit_common(oldval);
171 local_irq_restore(flags);
172} 82}
173EXPORT_SYMBOL_GPL(rcu_irq_enter); 83EXPORT_SYMBOL_GPL(rcu_irq_enter);
174 84
@@ -179,23 +89,13 @@ EXPORT_SYMBOL_GPL(rcu_irq_enter);
179 */ 89 */
180bool notrace __rcu_is_watching(void) 90bool notrace __rcu_is_watching(void)
181{ 91{
182 return rcu_dynticks_nesting; 92 return true;
183} 93}
184EXPORT_SYMBOL(__rcu_is_watching); 94EXPORT_SYMBOL(__rcu_is_watching);
185 95
186#endif /* defined(CONFIG_DEBUG_LOCK_ALLOC) || defined(CONFIG_RCU_TRACE) */ 96#endif /* defined(CONFIG_DEBUG_LOCK_ALLOC) || defined(CONFIG_RCU_TRACE) */
187 97
188/* 98/*
189 * Test whether the current CPU was interrupted from idle. Nested
190 * interrupts don't count, we must be running at the first interrupt
191 * level.
192 */
193static int rcu_is_cpu_rrupt_from_idle(void)
194{
195 return rcu_dynticks_nesting <= 1;
196}
197
198/*
199 * Helper function for rcu_sched_qs() and rcu_bh_qs(). 99 * Helper function for rcu_sched_qs() and rcu_bh_qs().
200 * Also irqs are disabled to avoid confusion due to interrupt handlers 100 * Also irqs are disabled to avoid confusion due to interrupt handlers
201 * invoking call_rcu(). 101 * invoking call_rcu().
@@ -250,7 +150,7 @@ void rcu_bh_qs(void)
250void rcu_check_callbacks(int user) 150void rcu_check_callbacks(int user)
251{ 151{
252 RCU_TRACE(check_cpu_stalls()); 152 RCU_TRACE(check_cpu_stalls());
253 if (user || rcu_is_cpu_rrupt_from_idle()) 153 if (user)
254 rcu_sched_qs(); 154 rcu_sched_qs();
255 else if (!in_softirq()) 155 else if (!in_softirq())
256 rcu_bh_qs(); 156 rcu_bh_qs();
@@ -357,6 +257,11 @@ static void __call_rcu(struct rcu_head *head,
357 rcp->curtail = &head->next; 257 rcp->curtail = &head->next;
358 RCU_TRACE(rcp->qlen++); 258 RCU_TRACE(rcp->qlen++);
359 local_irq_restore(flags); 259 local_irq_restore(flags);
260
261 if (unlikely(is_idle_task(current))) {
262 /* force scheduling for rcu_sched_qs() */
263 resched_cpu(0);
264 }
360} 265}
361 266
362/* 267/*
@@ -383,6 +288,8 @@ EXPORT_SYMBOL_GPL(call_rcu_bh);
383void __init rcu_init(void) 288void __init rcu_init(void)
384{ 289{
385 open_softirq(RCU_SOFTIRQ, rcu_process_callbacks); 290 open_softirq(RCU_SOFTIRQ, rcu_process_callbacks);
291 RCU_TRACE(reset_cpu_stall_ticks(&rcu_sched_ctrlblk));
292 RCU_TRACE(reset_cpu_stall_ticks(&rcu_bh_ctrlblk));
386 293
387 rcu_early_boot_tests(); 294 rcu_early_boot_tests();
388} 295}
diff --git a/kernel/rcu/tiny_plugin.h b/kernel/rcu/tiny_plugin.h
index 858c56569127..f94e209a10d6 100644
--- a/kernel/rcu/tiny_plugin.h
+++ b/kernel/rcu/tiny_plugin.h
@@ -145,17 +145,16 @@ static void check_cpu_stall(struct rcu_ctrlblk *rcp)
145 rcp->ticks_this_gp++; 145 rcp->ticks_this_gp++;
146 j = jiffies; 146 j = jiffies;
147 js = ACCESS_ONCE(rcp->jiffies_stall); 147 js = ACCESS_ONCE(rcp->jiffies_stall);
148 if (*rcp->curtail && ULONG_CMP_GE(j, js)) { 148 if (rcp->rcucblist && ULONG_CMP_GE(j, js)) {
149 pr_err("INFO: %s stall on CPU (%lu ticks this GP) idle=%llx (t=%lu jiffies q=%ld)\n", 149 pr_err("INFO: %s stall on CPU (%lu ticks this GP) idle=%llx (t=%lu jiffies q=%ld)\n",
150 rcp->name, rcp->ticks_this_gp, rcu_dynticks_nesting, 150 rcp->name, rcp->ticks_this_gp, DYNTICK_TASK_EXIT_IDLE,
151 jiffies - rcp->gp_start, rcp->qlen); 151 jiffies - rcp->gp_start, rcp->qlen);
152 dump_stack(); 152 dump_stack();
153 }
154 if (*rcp->curtail && ULONG_CMP_GE(j, js))
155 ACCESS_ONCE(rcp->jiffies_stall) = jiffies + 153 ACCESS_ONCE(rcp->jiffies_stall) = jiffies +
156 3 * rcu_jiffies_till_stall_check() + 3; 154 3 * rcu_jiffies_till_stall_check() + 3;
157 else if (ULONG_CMP_GE(j, js)) 155 } else if (ULONG_CMP_GE(j, js)) {
158 ACCESS_ONCE(rcp->jiffies_stall) = jiffies + rcu_jiffies_till_stall_check(); 156 ACCESS_ONCE(rcp->jiffies_stall) = jiffies + rcu_jiffies_till_stall_check();
157 }
159} 158}
160 159
161static void reset_cpu_stall_ticks(struct rcu_ctrlblk *rcp) 160static void reset_cpu_stall_ticks(struct rcu_ctrlblk *rcp)
diff --git a/kernel/rcu/tree.c b/kernel/rcu/tree.c
index c0faad51ae87..48d640ca1a05 100644
--- a/kernel/rcu/tree.c
+++ b/kernel/rcu/tree.c
@@ -156,6 +156,10 @@ static void rcu_boost_kthread_setaffinity(struct rcu_node *rnp, int outgoingcpu)
156static void invoke_rcu_core(void); 156static void invoke_rcu_core(void);
157static void invoke_rcu_callbacks(struct rcu_state *rsp, struct rcu_data *rdp); 157static void invoke_rcu_callbacks(struct rcu_state *rsp, struct rcu_data *rdp);
158 158
159/* rcuc/rcub kthread realtime priority */
160static int kthread_prio = CONFIG_RCU_KTHREAD_PRIO;
161module_param(kthread_prio, int, 0644);
162
159/* 163/*
160 * Track the rcutorture test sequence number and the update version 164 * Track the rcutorture test sequence number and the update version
161 * number within a given test. The rcutorture_testseq is incremented 165 * number within a given test. The rcutorture_testseq is incremented
@@ -215,6 +219,9 @@ static DEFINE_PER_CPU(struct rcu_dynticks, rcu_dynticks) = {
215#endif /* #ifdef CONFIG_NO_HZ_FULL_SYSIDLE */ 219#endif /* #ifdef CONFIG_NO_HZ_FULL_SYSIDLE */
216}; 220};
217 221
222DEFINE_PER_CPU_SHARED_ALIGNED(unsigned long, rcu_qs_ctr);
223EXPORT_PER_CPU_SYMBOL_GPL(rcu_qs_ctr);
224
218/* 225/*
219 * Let the RCU core know that this CPU has gone through the scheduler, 226 * Let the RCU core know that this CPU has gone through the scheduler,
220 * which is a quiescent state. This is called when the need for a 227 * which is a quiescent state. This is called when the need for a
@@ -284,6 +291,22 @@ void rcu_note_context_switch(void)
284} 291}
285EXPORT_SYMBOL_GPL(rcu_note_context_switch); 292EXPORT_SYMBOL_GPL(rcu_note_context_switch);
286 293
294/*
295 * Register a quiesecent state for all RCU flavors. If there is an
296 * emergency, invoke rcu_momentary_dyntick_idle() to do a heavy-weight
297 * dyntick-idle quiescent state visible to other CPUs (but only for those
298 * RCU flavors in desparate need of a quiescent state, which will normally
299 * be none of them). Either way, do a lightweight quiescent state for
300 * all RCU flavors.
301 */
302void rcu_all_qs(void)
303{
304 if (unlikely(raw_cpu_read(rcu_sched_qs_mask)))
305 rcu_momentary_dyntick_idle();
306 this_cpu_inc(rcu_qs_ctr);
307}
308EXPORT_SYMBOL_GPL(rcu_all_qs);
309
287static long blimit = 10; /* Maximum callbacks per rcu_do_batch. */ 310static long blimit = 10; /* Maximum callbacks per rcu_do_batch. */
288static long qhimark = 10000; /* If this many pending, ignore blimit. */ 311static long qhimark = 10000; /* If this many pending, ignore blimit. */
289static long qlowmark = 100; /* Once only this many pending, use blimit. */ 312static long qlowmark = 100; /* Once only this many pending, use blimit. */
@@ -966,17 +989,14 @@ static int dyntick_save_progress_counter(struct rcu_data *rdp,
966 trace_rcu_fqs(rdp->rsp->name, rdp->gpnum, rdp->cpu, TPS("dti")); 989 trace_rcu_fqs(rdp->rsp->name, rdp->gpnum, rdp->cpu, TPS("dti"));
967 return 1; 990 return 1;
968 } else { 991 } else {
992 if (ULONG_CMP_LT(ACCESS_ONCE(rdp->gpnum) + ULONG_MAX / 4,
993 rdp->mynode->gpnum))
994 ACCESS_ONCE(rdp->gpwrap) = true;
969 return 0; 995 return 0;
970 } 996 }
971} 997}
972 998
973/* 999/*
974 * This function really isn't for public consumption, but RCU is special in
975 * that context switches can allow the state machine to make progress.
976 */
977extern void resched_cpu(int cpu);
978
979/*
980 * Return true if the specified CPU has passed through a quiescent 1000 * Return true if the specified CPU has passed through a quiescent
981 * state by virtue of being in or having passed through an dynticks 1001 * state by virtue of being in or having passed through an dynticks
982 * idle state since the last call to dyntick_save_progress_counter() 1002 * idle state since the last call to dyntick_save_progress_counter()
@@ -1079,6 +1099,22 @@ static void record_gp_stall_check_time(struct rcu_state *rsp)
1079 j1 = rcu_jiffies_till_stall_check(); 1099 j1 = rcu_jiffies_till_stall_check();
1080 ACCESS_ONCE(rsp->jiffies_stall) = j + j1; 1100 ACCESS_ONCE(rsp->jiffies_stall) = j + j1;
1081 rsp->jiffies_resched = j + j1 / 2; 1101 rsp->jiffies_resched = j + j1 / 2;
1102 rsp->n_force_qs_gpstart = ACCESS_ONCE(rsp->n_force_qs);
1103}
1104
1105/*
1106 * Complain about starvation of grace-period kthread.
1107 */
1108static void rcu_check_gp_kthread_starvation(struct rcu_state *rsp)
1109{
1110 unsigned long gpa;
1111 unsigned long j;
1112
1113 j = jiffies;
1114 gpa = ACCESS_ONCE(rsp->gp_activity);
1115 if (j - gpa > 2 * HZ)
1116 pr_err("%s kthread starved for %ld jiffies!\n",
1117 rsp->name, j - gpa);
1082} 1118}
1083 1119
1084/* 1120/*
@@ -1101,11 +1137,13 @@ static void rcu_dump_cpu_stacks(struct rcu_state *rsp)
1101 } 1137 }
1102} 1138}
1103 1139
1104static void print_other_cpu_stall(struct rcu_state *rsp) 1140static void print_other_cpu_stall(struct rcu_state *rsp, unsigned long gpnum)
1105{ 1141{
1106 int cpu; 1142 int cpu;
1107 long delta; 1143 long delta;
1108 unsigned long flags; 1144 unsigned long flags;
1145 unsigned long gpa;
1146 unsigned long j;
1109 int ndetected = 0; 1147 int ndetected = 0;
1110 struct rcu_node *rnp = rcu_get_root(rsp); 1148 struct rcu_node *rnp = rcu_get_root(rsp);
1111 long totqlen = 0; 1149 long totqlen = 0;
@@ -1143,30 +1181,34 @@ static void print_other_cpu_stall(struct rcu_state *rsp)
1143 raw_spin_unlock_irqrestore(&rnp->lock, flags); 1181 raw_spin_unlock_irqrestore(&rnp->lock, flags);
1144 } 1182 }
1145 1183
1146 /*
1147 * Now rat on any tasks that got kicked up to the root rcu_node
1148 * due to CPU offlining.
1149 */
1150 rnp = rcu_get_root(rsp);
1151 raw_spin_lock_irqsave(&rnp->lock, flags);
1152 ndetected += rcu_print_task_stall(rnp);
1153 raw_spin_unlock_irqrestore(&rnp->lock, flags);
1154
1155 print_cpu_stall_info_end(); 1184 print_cpu_stall_info_end();
1156 for_each_possible_cpu(cpu) 1185 for_each_possible_cpu(cpu)
1157 totqlen += per_cpu_ptr(rsp->rda, cpu)->qlen; 1186 totqlen += per_cpu_ptr(rsp->rda, cpu)->qlen;
1158 pr_cont("(detected by %d, t=%ld jiffies, g=%ld, c=%ld, q=%lu)\n", 1187 pr_cont("(detected by %d, t=%ld jiffies, g=%ld, c=%ld, q=%lu)\n",
1159 smp_processor_id(), (long)(jiffies - rsp->gp_start), 1188 smp_processor_id(), (long)(jiffies - rsp->gp_start),
1160 (long)rsp->gpnum, (long)rsp->completed, totqlen); 1189 (long)rsp->gpnum, (long)rsp->completed, totqlen);
1161 if (ndetected == 0) 1190 if (ndetected) {
1162 pr_err("INFO: Stall ended before state dump start\n");
1163 else
1164 rcu_dump_cpu_stacks(rsp); 1191 rcu_dump_cpu_stacks(rsp);
1192 } else {
1193 if (ACCESS_ONCE(rsp->gpnum) != gpnum ||
1194 ACCESS_ONCE(rsp->completed) == gpnum) {
1195 pr_err("INFO: Stall ended before state dump start\n");
1196 } else {
1197 j = jiffies;
1198 gpa = ACCESS_ONCE(rsp->gp_activity);
1199 pr_err("All QSes seen, last %s kthread activity %ld (%ld-%ld), jiffies_till_next_fqs=%ld\n",
1200 rsp->name, j - gpa, j, gpa,
1201 jiffies_till_next_fqs);
1202 /* In this case, the current CPU might be at fault. */
1203 sched_show_task(current);
1204 }
1205 }
1165 1206
1166 /* Complain about tasks blocking the grace period. */ 1207 /* Complain about tasks blocking the grace period. */
1167
1168 rcu_print_detail_task_stall(rsp); 1208 rcu_print_detail_task_stall(rsp);
1169 1209
1210 rcu_check_gp_kthread_starvation(rsp);
1211
1170 force_quiescent_state(rsp); /* Kick them all. */ 1212 force_quiescent_state(rsp); /* Kick them all. */
1171} 1213}
1172 1214
@@ -1191,6 +1233,9 @@ static void print_cpu_stall(struct rcu_state *rsp)
1191 pr_cont(" (t=%lu jiffies g=%ld c=%ld q=%lu)\n", 1233 pr_cont(" (t=%lu jiffies g=%ld c=%ld q=%lu)\n",
1192 jiffies - rsp->gp_start, 1234 jiffies - rsp->gp_start,
1193 (long)rsp->gpnum, (long)rsp->completed, totqlen); 1235 (long)rsp->gpnum, (long)rsp->completed, totqlen);
1236
1237 rcu_check_gp_kthread_starvation(rsp);
1238
1194 rcu_dump_cpu_stacks(rsp); 1239 rcu_dump_cpu_stacks(rsp);
1195 1240
1196 raw_spin_lock_irqsave(&rnp->lock, flags); 1241 raw_spin_lock_irqsave(&rnp->lock, flags);
@@ -1261,7 +1306,7 @@ static void check_cpu_stall(struct rcu_state *rsp, struct rcu_data *rdp)
1261 ULONG_CMP_GE(j, js + RCU_STALL_RAT_DELAY)) { 1306 ULONG_CMP_GE(j, js + RCU_STALL_RAT_DELAY)) {
1262 1307
1263 /* They had a few time units to dump stack, so complain. */ 1308 /* They had a few time units to dump stack, so complain. */
1264 print_other_cpu_stall(rsp); 1309 print_other_cpu_stall(rsp, gpnum);
1265 } 1310 }
1266} 1311}
1267 1312
@@ -1598,7 +1643,8 @@ static bool __note_gp_changes(struct rcu_state *rsp, struct rcu_node *rnp,
1598 bool ret; 1643 bool ret;
1599 1644
1600 /* Handle the ends of any preceding grace periods first. */ 1645 /* Handle the ends of any preceding grace periods first. */
1601 if (rdp->completed == rnp->completed) { 1646 if (rdp->completed == rnp->completed &&
1647 !unlikely(ACCESS_ONCE(rdp->gpwrap))) {
1602 1648
1603 /* No grace period end, so just accelerate recent callbacks. */ 1649 /* No grace period end, so just accelerate recent callbacks. */
1604 ret = rcu_accelerate_cbs(rsp, rnp, rdp); 1650 ret = rcu_accelerate_cbs(rsp, rnp, rdp);
@@ -1613,7 +1659,7 @@ static bool __note_gp_changes(struct rcu_state *rsp, struct rcu_node *rnp,
1613 trace_rcu_grace_period(rsp->name, rdp->gpnum, TPS("cpuend")); 1659 trace_rcu_grace_period(rsp->name, rdp->gpnum, TPS("cpuend"));
1614 } 1660 }
1615 1661
1616 if (rdp->gpnum != rnp->gpnum) { 1662 if (rdp->gpnum != rnp->gpnum || unlikely(ACCESS_ONCE(rdp->gpwrap))) {
1617 /* 1663 /*
1618 * If the current grace period is waiting for this CPU, 1664 * If the current grace period is waiting for this CPU,
1619 * set up to detect a quiescent state, otherwise don't 1665 * set up to detect a quiescent state, otherwise don't
@@ -1622,8 +1668,10 @@ static bool __note_gp_changes(struct rcu_state *rsp, struct rcu_node *rnp,
1622 rdp->gpnum = rnp->gpnum; 1668 rdp->gpnum = rnp->gpnum;
1623 trace_rcu_grace_period(rsp->name, rdp->gpnum, TPS("cpustart")); 1669 trace_rcu_grace_period(rsp->name, rdp->gpnum, TPS("cpustart"));
1624 rdp->passed_quiesce = 0; 1670 rdp->passed_quiesce = 0;
1671 rdp->rcu_qs_ctr_snap = __this_cpu_read(rcu_qs_ctr);
1625 rdp->qs_pending = !!(rnp->qsmask & rdp->grpmask); 1672 rdp->qs_pending = !!(rnp->qsmask & rdp->grpmask);
1626 zero_cpu_stall_ticks(rdp); 1673 zero_cpu_stall_ticks(rdp);
1674 ACCESS_ONCE(rdp->gpwrap) = false;
1627 } 1675 }
1628 return ret; 1676 return ret;
1629} 1677}
@@ -1637,7 +1685,8 @@ static void note_gp_changes(struct rcu_state *rsp, struct rcu_data *rdp)
1637 local_irq_save(flags); 1685 local_irq_save(flags);
1638 rnp = rdp->mynode; 1686 rnp = rdp->mynode;
1639 if ((rdp->gpnum == ACCESS_ONCE(rnp->gpnum) && 1687 if ((rdp->gpnum == ACCESS_ONCE(rnp->gpnum) &&
1640 rdp->completed == ACCESS_ONCE(rnp->completed)) || /* w/out lock. */ 1688 rdp->completed == ACCESS_ONCE(rnp->completed) &&
1689 !unlikely(ACCESS_ONCE(rdp->gpwrap))) || /* w/out lock. */
1641 !raw_spin_trylock(&rnp->lock)) { /* irqs already off, so later. */ 1690 !raw_spin_trylock(&rnp->lock)) { /* irqs already off, so later. */
1642 local_irq_restore(flags); 1691 local_irq_restore(flags);
1643 return; 1692 return;
@@ -1657,6 +1706,7 @@ static int rcu_gp_init(struct rcu_state *rsp)
1657 struct rcu_data *rdp; 1706 struct rcu_data *rdp;
1658 struct rcu_node *rnp = rcu_get_root(rsp); 1707 struct rcu_node *rnp = rcu_get_root(rsp);
1659 1708
1709 ACCESS_ONCE(rsp->gp_activity) = jiffies;
1660 rcu_bind_gp_kthread(); 1710 rcu_bind_gp_kthread();
1661 raw_spin_lock_irq(&rnp->lock); 1711 raw_spin_lock_irq(&rnp->lock);
1662 smp_mb__after_unlock_lock(); 1712 smp_mb__after_unlock_lock();
@@ -1717,6 +1767,7 @@ static int rcu_gp_init(struct rcu_state *rsp)
1717 rnp->grphi, rnp->qsmask); 1767 rnp->grphi, rnp->qsmask);
1718 raw_spin_unlock_irq(&rnp->lock); 1768 raw_spin_unlock_irq(&rnp->lock);
1719 cond_resched_rcu_qs(); 1769 cond_resched_rcu_qs();
1770 ACCESS_ONCE(rsp->gp_activity) = jiffies;
1720 } 1771 }
1721 1772
1722 mutex_unlock(&rsp->onoff_mutex); 1773 mutex_unlock(&rsp->onoff_mutex);
@@ -1733,6 +1784,7 @@ static int rcu_gp_fqs(struct rcu_state *rsp, int fqs_state_in)
1733 unsigned long maxj; 1784 unsigned long maxj;
1734 struct rcu_node *rnp = rcu_get_root(rsp); 1785 struct rcu_node *rnp = rcu_get_root(rsp);
1735 1786
1787 ACCESS_ONCE(rsp->gp_activity) = jiffies;
1736 rsp->n_force_qs++; 1788 rsp->n_force_qs++;
1737 if (fqs_state == RCU_SAVE_DYNTICK) { 1789 if (fqs_state == RCU_SAVE_DYNTICK) {
1738 /* Collect dyntick-idle snapshots. */ 1790 /* Collect dyntick-idle snapshots. */
@@ -1771,6 +1823,7 @@ static void rcu_gp_cleanup(struct rcu_state *rsp)
1771 struct rcu_data *rdp; 1823 struct rcu_data *rdp;
1772 struct rcu_node *rnp = rcu_get_root(rsp); 1824 struct rcu_node *rnp = rcu_get_root(rsp);
1773 1825
1826 ACCESS_ONCE(rsp->gp_activity) = jiffies;
1774 raw_spin_lock_irq(&rnp->lock); 1827 raw_spin_lock_irq(&rnp->lock);
1775 smp_mb__after_unlock_lock(); 1828 smp_mb__after_unlock_lock();
1776 gp_duration = jiffies - rsp->gp_start; 1829 gp_duration = jiffies - rsp->gp_start;
@@ -1807,6 +1860,7 @@ static void rcu_gp_cleanup(struct rcu_state *rsp)
1807 nocb += rcu_future_gp_cleanup(rsp, rnp); 1860 nocb += rcu_future_gp_cleanup(rsp, rnp);
1808 raw_spin_unlock_irq(&rnp->lock); 1861 raw_spin_unlock_irq(&rnp->lock);
1809 cond_resched_rcu_qs(); 1862 cond_resched_rcu_qs();
1863 ACCESS_ONCE(rsp->gp_activity) = jiffies;
1810 } 1864 }
1811 rnp = rcu_get_root(rsp); 1865 rnp = rcu_get_root(rsp);
1812 raw_spin_lock_irq(&rnp->lock); 1866 raw_spin_lock_irq(&rnp->lock);
@@ -1856,6 +1910,7 @@ static int __noreturn rcu_gp_kthread(void *arg)
1856 if (rcu_gp_init(rsp)) 1910 if (rcu_gp_init(rsp))
1857 break; 1911 break;
1858 cond_resched_rcu_qs(); 1912 cond_resched_rcu_qs();
1913 ACCESS_ONCE(rsp->gp_activity) = jiffies;
1859 WARN_ON(signal_pending(current)); 1914 WARN_ON(signal_pending(current));
1860 trace_rcu_grace_period(rsp->name, 1915 trace_rcu_grace_period(rsp->name,
1861 ACCESS_ONCE(rsp->gpnum), 1916 ACCESS_ONCE(rsp->gpnum),
@@ -1899,9 +1954,11 @@ static int __noreturn rcu_gp_kthread(void *arg)
1899 ACCESS_ONCE(rsp->gpnum), 1954 ACCESS_ONCE(rsp->gpnum),
1900 TPS("fqsend")); 1955 TPS("fqsend"));
1901 cond_resched_rcu_qs(); 1956 cond_resched_rcu_qs();
1957 ACCESS_ONCE(rsp->gp_activity) = jiffies;
1902 } else { 1958 } else {
1903 /* Deal with stray signal. */ 1959 /* Deal with stray signal. */
1904 cond_resched_rcu_qs(); 1960 cond_resched_rcu_qs();
1961 ACCESS_ONCE(rsp->gp_activity) = jiffies;
1905 WARN_ON(signal_pending(current)); 1962 WARN_ON(signal_pending(current));
1906 trace_rcu_grace_period(rsp->name, 1963 trace_rcu_grace_period(rsp->name,
1907 ACCESS_ONCE(rsp->gpnum), 1964 ACCESS_ONCE(rsp->gpnum),
@@ -2078,8 +2135,10 @@ rcu_report_qs_rdp(int cpu, struct rcu_state *rsp, struct rcu_data *rdp)
2078 rnp = rdp->mynode; 2135 rnp = rdp->mynode;
2079 raw_spin_lock_irqsave(&rnp->lock, flags); 2136 raw_spin_lock_irqsave(&rnp->lock, flags);
2080 smp_mb__after_unlock_lock(); 2137 smp_mb__after_unlock_lock();
2081 if (rdp->passed_quiesce == 0 || rdp->gpnum != rnp->gpnum || 2138 if ((rdp->passed_quiesce == 0 &&
2082 rnp->completed == rnp->gpnum) { 2139 rdp->rcu_qs_ctr_snap == __this_cpu_read(rcu_qs_ctr)) ||
2140 rdp->gpnum != rnp->gpnum || rnp->completed == rnp->gpnum ||
2141 rdp->gpwrap) {
2083 2142
2084 /* 2143 /*
2085 * The grace period in which this quiescent state was 2144 * The grace period in which this quiescent state was
@@ -2088,6 +2147,7 @@ rcu_report_qs_rdp(int cpu, struct rcu_state *rsp, struct rcu_data *rdp)
2088 * within the current grace period. 2147 * within the current grace period.
2089 */ 2148 */
2090 rdp->passed_quiesce = 0; /* need qs for new gp. */ 2149 rdp->passed_quiesce = 0; /* need qs for new gp. */
2150 rdp->rcu_qs_ctr_snap = __this_cpu_read(rcu_qs_ctr);
2091 raw_spin_unlock_irqrestore(&rnp->lock, flags); 2151 raw_spin_unlock_irqrestore(&rnp->lock, flags);
2092 return; 2152 return;
2093 } 2153 }
@@ -2132,7 +2192,8 @@ rcu_check_quiescent_state(struct rcu_state *rsp, struct rcu_data *rdp)
2132 * Was there a quiescent state since the beginning of the grace 2192 * Was there a quiescent state since the beginning of the grace
2133 * period? If no, then exit and wait for the next call. 2193 * period? If no, then exit and wait for the next call.
2134 */ 2194 */
2135 if (!rdp->passed_quiesce) 2195 if (!rdp->passed_quiesce &&
2196 rdp->rcu_qs_ctr_snap == __this_cpu_read(rcu_qs_ctr))
2136 return; 2197 return;
2137 2198
2138 /* 2199 /*
@@ -2263,6 +2324,46 @@ static void rcu_cleanup_dying_cpu(struct rcu_state *rsp)
2263} 2324}
2264 2325
2265/* 2326/*
2327 * All CPUs for the specified rcu_node structure have gone offline,
2328 * and all tasks that were preempted within an RCU read-side critical
2329 * section while running on one of those CPUs have since exited their RCU
2330 * read-side critical section. Some other CPU is reporting this fact with
2331 * the specified rcu_node structure's ->lock held and interrupts disabled.
2332 * This function therefore goes up the tree of rcu_node structures,
2333 * clearing the corresponding bits in the ->qsmaskinit fields. Note that
2334 * the leaf rcu_node structure's ->qsmaskinit field has already been
2335 * updated
2336 *
2337 * This function does check that the specified rcu_node structure has
2338 * all CPUs offline and no blocked tasks, so it is OK to invoke it
2339 * prematurely. That said, invoking it after the fact will cost you
2340 * a needless lock acquisition. So once it has done its work, don't
2341 * invoke it again.
2342 */
2343static void rcu_cleanup_dead_rnp(struct rcu_node *rnp_leaf)
2344{
2345 long mask;
2346 struct rcu_node *rnp = rnp_leaf;
2347
2348 if (rnp->qsmaskinit || rcu_preempt_has_tasks(rnp))
2349 return;
2350 for (;;) {
2351 mask = rnp->grpmask;
2352 rnp = rnp->parent;
2353 if (!rnp)
2354 break;
2355 raw_spin_lock(&rnp->lock); /* irqs already disabled. */
2356 smp_mb__after_unlock_lock(); /* GP memory ordering. */
2357 rnp->qsmaskinit &= ~mask;
2358 if (rnp->qsmaskinit) {
2359 raw_spin_unlock(&rnp->lock); /* irqs remain disabled. */
2360 return;
2361 }
2362 raw_spin_unlock(&rnp->lock); /* irqs remain disabled. */
2363 }
2364}
2365
2366/*
2266 * The CPU has been completely removed, and some other CPU is reporting 2367 * The CPU has been completely removed, and some other CPU is reporting
2267 * this fact from process context. Do the remainder of the cleanup, 2368 * this fact from process context. Do the remainder of the cleanup,
2268 * including orphaning the outgoing CPU's RCU callbacks, and also 2369 * including orphaning the outgoing CPU's RCU callbacks, and also
@@ -2272,8 +2373,6 @@ static void rcu_cleanup_dying_cpu(struct rcu_state *rsp)
2272static void rcu_cleanup_dead_cpu(int cpu, struct rcu_state *rsp) 2373static void rcu_cleanup_dead_cpu(int cpu, struct rcu_state *rsp)
2273{ 2374{
2274 unsigned long flags; 2375 unsigned long flags;
2275 unsigned long mask;
2276 int need_report = 0;
2277 struct rcu_data *rdp = per_cpu_ptr(rsp->rda, cpu); 2376 struct rcu_data *rdp = per_cpu_ptr(rsp->rda, cpu);
2278 struct rcu_node *rnp = rdp->mynode; /* Outgoing CPU's rdp & rnp. */ 2377 struct rcu_node *rnp = rdp->mynode; /* Outgoing CPU's rdp & rnp. */
2279 2378
@@ -2287,40 +2386,15 @@ static void rcu_cleanup_dead_cpu(int cpu, struct rcu_state *rsp)
2287 /* Orphan the dead CPU's callbacks, and adopt them if appropriate. */ 2386 /* Orphan the dead CPU's callbacks, and adopt them if appropriate. */
2288 rcu_send_cbs_to_orphanage(cpu, rsp, rnp, rdp); 2387 rcu_send_cbs_to_orphanage(cpu, rsp, rnp, rdp);
2289 rcu_adopt_orphan_cbs(rsp, flags); 2388 rcu_adopt_orphan_cbs(rsp, flags);
2389 raw_spin_unlock_irqrestore(&rsp->orphan_lock, flags);
2290 2390
2291 /* Remove the outgoing CPU from the masks in the rcu_node hierarchy. */ 2391 /* Remove outgoing CPU from mask in the leaf rcu_node structure. */
2292 mask = rdp->grpmask; /* rnp->grplo is constant. */ 2392 raw_spin_lock_irqsave(&rnp->lock, flags);
2293 do { 2393 smp_mb__after_unlock_lock(); /* Enforce GP memory-order guarantee. */
2294 raw_spin_lock(&rnp->lock); /* irqs already disabled. */ 2394 rnp->qsmaskinit &= ~rdp->grpmask;
2295 smp_mb__after_unlock_lock(); 2395 if (rnp->qsmaskinit == 0 && !rcu_preempt_has_tasks(rnp))
2296 rnp->qsmaskinit &= ~mask; 2396 rcu_cleanup_dead_rnp(rnp);
2297 if (rnp->qsmaskinit != 0) { 2397 rcu_report_qs_rnp(rdp->grpmask, rsp, rnp, flags); /* Rlses rnp->lock. */
2298 if (rnp != rdp->mynode)
2299 raw_spin_unlock(&rnp->lock); /* irqs remain disabled. */
2300 break;
2301 }
2302 if (rnp == rdp->mynode)
2303 need_report = rcu_preempt_offline_tasks(rsp, rnp, rdp);
2304 else
2305 raw_spin_unlock(&rnp->lock); /* irqs remain disabled. */
2306 mask = rnp->grpmask;
2307 rnp = rnp->parent;
2308 } while (rnp != NULL);
2309
2310 /*
2311 * We still hold the leaf rcu_node structure lock here, and
2312 * irqs are still disabled. The reason for this subterfuge is
2313 * because invoking rcu_report_unblock_qs_rnp() with ->orphan_lock
2314 * held leads to deadlock.
2315 */
2316 raw_spin_unlock(&rsp->orphan_lock); /* irqs remain disabled. */
2317 rnp = rdp->mynode;
2318 if (need_report & RCU_OFL_TASKS_NORM_GP)
2319 rcu_report_unblock_qs_rnp(rnp, flags);
2320 else
2321 raw_spin_unlock_irqrestore(&rnp->lock, flags);
2322 if (need_report & RCU_OFL_TASKS_EXP_GP)
2323 rcu_report_exp_rnp(rsp, rnp, true);
2324 WARN_ONCE(rdp->qlen != 0 || rdp->nxtlist != NULL, 2398 WARN_ONCE(rdp->qlen != 0 || rdp->nxtlist != NULL,
2325 "rcu_cleanup_dead_cpu: Callbacks on offline CPU %d: qlen=%lu, nxtlist=%p\n", 2399 "rcu_cleanup_dead_cpu: Callbacks on offline CPU %d: qlen=%lu, nxtlist=%p\n",
2326 cpu, rdp->qlen, rdp->nxtlist); 2400 cpu, rdp->qlen, rdp->nxtlist);
@@ -2336,6 +2410,10 @@ static void rcu_cleanup_dying_cpu(struct rcu_state *rsp)
2336{ 2410{
2337} 2411}
2338 2412
2413static void __maybe_unused rcu_cleanup_dead_rnp(struct rcu_node *rnp_leaf)
2414{
2415}
2416
2339static void rcu_cleanup_dead_cpu(int cpu, struct rcu_state *rsp) 2417static void rcu_cleanup_dead_cpu(int cpu, struct rcu_state *rsp)
2340{ 2418{
2341} 2419}
@@ -2532,12 +2610,6 @@ static void force_qs_rnp(struct rcu_state *rsp,
2532 } 2610 }
2533 raw_spin_unlock_irqrestore(&rnp->lock, flags); 2611 raw_spin_unlock_irqrestore(&rnp->lock, flags);
2534 } 2612 }
2535 rnp = rcu_get_root(rsp);
2536 if (rnp->qsmask == 0) {
2537 raw_spin_lock_irqsave(&rnp->lock, flags);
2538 smp_mb__after_unlock_lock();
2539 rcu_initiate_boost(rnp, flags); /* releases rnp->lock. */
2540 }
2541} 2613}
2542 2614
2543/* 2615/*
@@ -2637,7 +2709,7 @@ static void rcu_process_callbacks(struct softirq_action *unused)
2637 * Schedule RCU callback invocation. If the specified type of RCU 2709 * Schedule RCU callback invocation. If the specified type of RCU
2638 * does not support RCU priority boosting, just do a direct call, 2710 * does not support RCU priority boosting, just do a direct call,
2639 * otherwise wake up the per-CPU kernel kthread. Note that because we 2711 * otherwise wake up the per-CPU kernel kthread. Note that because we
2640 * are running on the current CPU with interrupts disabled, the 2712 * are running on the current CPU with softirqs disabled, the
2641 * rcu_cpu_kthread_task cannot disappear out from under us. 2713 * rcu_cpu_kthread_task cannot disappear out from under us.
2642 */ 2714 */
2643static void invoke_rcu_callbacks(struct rcu_state *rsp, struct rcu_data *rdp) 2715static void invoke_rcu_callbacks(struct rcu_state *rsp, struct rcu_data *rdp)
@@ -3177,9 +3249,12 @@ static int __rcu_pending(struct rcu_state *rsp, struct rcu_data *rdp)
3177 3249
3178 /* Is the RCU core waiting for a quiescent state from this CPU? */ 3250 /* Is the RCU core waiting for a quiescent state from this CPU? */
3179 if (rcu_scheduler_fully_active && 3251 if (rcu_scheduler_fully_active &&
3180 rdp->qs_pending && !rdp->passed_quiesce) { 3252 rdp->qs_pending && !rdp->passed_quiesce &&
3253 rdp->rcu_qs_ctr_snap == __this_cpu_read(rcu_qs_ctr)) {
3181 rdp->n_rp_qs_pending++; 3254 rdp->n_rp_qs_pending++;
3182 } else if (rdp->qs_pending && rdp->passed_quiesce) { 3255 } else if (rdp->qs_pending &&
3256 (rdp->passed_quiesce ||
3257 rdp->rcu_qs_ctr_snap != __this_cpu_read(rcu_qs_ctr))) {
3183 rdp->n_rp_report_qs++; 3258 rdp->n_rp_report_qs++;
3184 return 1; 3259 return 1;
3185 } 3260 }
@@ -3203,7 +3278,8 @@ static int __rcu_pending(struct rcu_state *rsp, struct rcu_data *rdp)
3203 } 3278 }
3204 3279
3205 /* Has a new RCU grace period started? */ 3280 /* Has a new RCU grace period started? */
3206 if (ACCESS_ONCE(rnp->gpnum) != rdp->gpnum) { /* outside lock */ 3281 if (ACCESS_ONCE(rnp->gpnum) != rdp->gpnum ||
3282 unlikely(ACCESS_ONCE(rdp->gpwrap))) { /* outside lock */
3207 rdp->n_rp_gp_started++; 3283 rdp->n_rp_gp_started++;
3208 return 1; 3284 return 1;
3209 } 3285 }
@@ -3386,6 +3462,7 @@ static void _rcu_barrier(struct rcu_state *rsp)
3386 } else { 3462 } else {
3387 _rcu_barrier_trace(rsp, "OnlineNoCB", cpu, 3463 _rcu_barrier_trace(rsp, "OnlineNoCB", cpu,
3388 rsp->n_barrier_done); 3464 rsp->n_barrier_done);
3465 smp_mb__before_atomic();
3389 atomic_inc(&rsp->barrier_cpu_count); 3466 atomic_inc(&rsp->barrier_cpu_count);
3390 __call_rcu(&rdp->barrier_head, 3467 __call_rcu(&rdp->barrier_head,
3391 rcu_barrier_callback, rsp, cpu, 0); 3468 rcu_barrier_callback, rsp, cpu, 0);
@@ -3453,9 +3530,6 @@ rcu_boot_init_percpu_data(int cpu, struct rcu_state *rsp)
3453 /* Set up local state, ensuring consistent view of global state. */ 3530 /* Set up local state, ensuring consistent view of global state. */
3454 raw_spin_lock_irqsave(&rnp->lock, flags); 3531 raw_spin_lock_irqsave(&rnp->lock, flags);
3455 rdp->grpmask = 1UL << (cpu - rdp->mynode->grplo); 3532 rdp->grpmask = 1UL << (cpu - rdp->mynode->grplo);
3456 init_callback_list(rdp);
3457 rdp->qlen_lazy = 0;
3458 ACCESS_ONCE(rdp->qlen) = 0;
3459 rdp->dynticks = &per_cpu(rcu_dynticks, cpu); 3533 rdp->dynticks = &per_cpu(rcu_dynticks, cpu);
3460 WARN_ON_ONCE(rdp->dynticks->dynticks_nesting != DYNTICK_TASK_EXIT_IDLE); 3534 WARN_ON_ONCE(rdp->dynticks->dynticks_nesting != DYNTICK_TASK_EXIT_IDLE);
3461 WARN_ON_ONCE(atomic_read(&rdp->dynticks->dynticks) != 1); 3535 WARN_ON_ONCE(atomic_read(&rdp->dynticks->dynticks) != 1);
@@ -3512,6 +3586,7 @@ rcu_init_percpu_data(int cpu, struct rcu_state *rsp)
3512 rdp->gpnum = rnp->completed; 3586 rdp->gpnum = rnp->completed;
3513 rdp->completed = rnp->completed; 3587 rdp->completed = rnp->completed;
3514 rdp->passed_quiesce = 0; 3588 rdp->passed_quiesce = 0;
3589 rdp->rcu_qs_ctr_snap = __this_cpu_read(rcu_qs_ctr);
3515 rdp->qs_pending = 0; 3590 rdp->qs_pending = 0;
3516 trace_rcu_grace_period(rsp->name, rdp->gpnum, TPS("cpuonl")); 3591 trace_rcu_grace_period(rsp->name, rdp->gpnum, TPS("cpuonl"));
3517 } 3592 }
@@ -3603,17 +3678,35 @@ static int rcu_pm_notify(struct notifier_block *self,
3603static int __init rcu_spawn_gp_kthread(void) 3678static int __init rcu_spawn_gp_kthread(void)
3604{ 3679{
3605 unsigned long flags; 3680 unsigned long flags;
3681 int kthread_prio_in = kthread_prio;
3606 struct rcu_node *rnp; 3682 struct rcu_node *rnp;
3607 struct rcu_state *rsp; 3683 struct rcu_state *rsp;
3684 struct sched_param sp;
3608 struct task_struct *t; 3685 struct task_struct *t;
3609 3686
3687 /* Force priority into range. */
3688 if (IS_ENABLED(CONFIG_RCU_BOOST) && kthread_prio < 1)
3689 kthread_prio = 1;
3690 else if (kthread_prio < 0)
3691 kthread_prio = 0;
3692 else if (kthread_prio > 99)
3693 kthread_prio = 99;
3694 if (kthread_prio != kthread_prio_in)
3695 pr_alert("rcu_spawn_gp_kthread(): Limited prio to %d from %d\n",
3696 kthread_prio, kthread_prio_in);
3697
3610 rcu_scheduler_fully_active = 1; 3698 rcu_scheduler_fully_active = 1;
3611 for_each_rcu_flavor(rsp) { 3699 for_each_rcu_flavor(rsp) {
3612 t = kthread_run(rcu_gp_kthread, rsp, "%s", rsp->name); 3700 t = kthread_create(rcu_gp_kthread, rsp, "%s", rsp->name);
3613 BUG_ON(IS_ERR(t)); 3701 BUG_ON(IS_ERR(t));
3614 rnp = rcu_get_root(rsp); 3702 rnp = rcu_get_root(rsp);
3615 raw_spin_lock_irqsave(&rnp->lock, flags); 3703 raw_spin_lock_irqsave(&rnp->lock, flags);
3616 rsp->gp_kthread = t; 3704 rsp->gp_kthread = t;
3705 if (kthread_prio) {
3706 sp.sched_priority = kthread_prio;
3707 sched_setscheduler_nocheck(t, SCHED_FIFO, &sp);
3708 }
3709 wake_up_process(t);
3617 raw_spin_unlock_irqrestore(&rnp->lock, flags); 3710 raw_spin_unlock_irqrestore(&rnp->lock, flags);
3618 } 3711 }
3619 rcu_spawn_nocb_kthreads(); 3712 rcu_spawn_nocb_kthreads();
diff --git a/kernel/rcu/tree.h b/kernel/rcu/tree.h
index 69eb422445f9..119de399eb2f 100644
--- a/kernel/rcu/tree.h
+++ b/kernel/rcu/tree.h
@@ -27,7 +27,6 @@
27#include <linux/threads.h> 27#include <linux/threads.h>
28#include <linux/cpumask.h> 28#include <linux/cpumask.h>
29#include <linux/seqlock.h> 29#include <linux/seqlock.h>
30#include <linux/irq_work.h>
31 30
32/* 31/*
33 * Define shape of hierarchy based on NR_CPUS, CONFIG_RCU_FANOUT, and 32 * Define shape of hierarchy based on NR_CPUS, CONFIG_RCU_FANOUT, and
@@ -172,11 +171,6 @@ struct rcu_node {
172 /* queued on this rcu_node structure that */ 171 /* queued on this rcu_node structure that */
173 /* are blocking the current grace period, */ 172 /* are blocking the current grace period, */
174 /* there can be no such task. */ 173 /* there can be no such task. */
175 struct completion boost_completion;
176 /* Used to ensure that the rt_mutex used */
177 /* to carry out the boosting is fully */
178 /* released with no future boostee accesses */
179 /* before that rt_mutex is re-initialized. */
180 struct rt_mutex boost_mtx; 174 struct rt_mutex boost_mtx;
181 /* Used only for the priority-boosting */ 175 /* Used only for the priority-boosting */
182 /* side effect, not as a lock. */ 176 /* side effect, not as a lock. */
@@ -257,9 +251,12 @@ struct rcu_data {
257 /* in order to detect GP end. */ 251 /* in order to detect GP end. */
258 unsigned long gpnum; /* Highest gp number that this CPU */ 252 unsigned long gpnum; /* Highest gp number that this CPU */
259 /* is aware of having started. */ 253 /* is aware of having started. */
254 unsigned long rcu_qs_ctr_snap;/* Snapshot of rcu_qs_ctr to check */
255 /* for rcu_all_qs() invocations. */
260 bool passed_quiesce; /* User-mode/idle loop etc. */ 256 bool passed_quiesce; /* User-mode/idle loop etc. */
261 bool qs_pending; /* Core waits for quiesc state. */ 257 bool qs_pending; /* Core waits for quiesc state. */
262 bool beenonline; /* CPU online at least once. */ 258 bool beenonline; /* CPU online at least once. */
259 bool gpwrap; /* Possible gpnum/completed wrap. */
263 struct rcu_node *mynode; /* This CPU's leaf of hierarchy */ 260 struct rcu_node *mynode; /* This CPU's leaf of hierarchy */
264 unsigned long grpmask; /* Mask to apply to leaf qsmask. */ 261 unsigned long grpmask; /* Mask to apply to leaf qsmask. */
265#ifdef CONFIG_RCU_CPU_STALL_INFO 262#ifdef CONFIG_RCU_CPU_STALL_INFO
@@ -340,14 +337,10 @@ struct rcu_data {
340#ifdef CONFIG_RCU_NOCB_CPU 337#ifdef CONFIG_RCU_NOCB_CPU
341 struct rcu_head *nocb_head; /* CBs waiting for kthread. */ 338 struct rcu_head *nocb_head; /* CBs waiting for kthread. */
342 struct rcu_head **nocb_tail; 339 struct rcu_head **nocb_tail;
343 atomic_long_t nocb_q_count; /* # CBs waiting for kthread */ 340 atomic_long_t nocb_q_count; /* # CBs waiting for nocb */
344 atomic_long_t nocb_q_count_lazy; /* (approximate). */ 341 atomic_long_t nocb_q_count_lazy; /* invocation (all stages). */
345 struct rcu_head *nocb_follower_head; /* CBs ready to invoke. */ 342 struct rcu_head *nocb_follower_head; /* CBs ready to invoke. */
346 struct rcu_head **nocb_follower_tail; 343 struct rcu_head **nocb_follower_tail;
347 atomic_long_t nocb_follower_count; /* # CBs ready to invoke. */
348 atomic_long_t nocb_follower_count_lazy; /* (approximate). */
349 int nocb_p_count; /* # CBs being invoked by kthread */
350 int nocb_p_count_lazy; /* (approximate). */
351 wait_queue_head_t nocb_wq; /* For nocb kthreads to sleep on. */ 344 wait_queue_head_t nocb_wq; /* For nocb kthreads to sleep on. */
352 struct task_struct *nocb_kthread; 345 struct task_struct *nocb_kthread;
353 int nocb_defer_wakeup; /* Defer wakeup of nocb_kthread. */ 346 int nocb_defer_wakeup; /* Defer wakeup of nocb_kthread. */
@@ -356,8 +349,6 @@ struct rcu_data {
356 struct rcu_head *nocb_gp_head ____cacheline_internodealigned_in_smp; 349 struct rcu_head *nocb_gp_head ____cacheline_internodealigned_in_smp;
357 /* CBs waiting for GP. */ 350 /* CBs waiting for GP. */
358 struct rcu_head **nocb_gp_tail; 351 struct rcu_head **nocb_gp_tail;
359 long nocb_gp_count;
360 long nocb_gp_count_lazy;
361 bool nocb_leader_sleep; /* Is the nocb leader thread asleep? */ 352 bool nocb_leader_sleep; /* Is the nocb leader thread asleep? */
362 struct rcu_data *nocb_next_follower; 353 struct rcu_data *nocb_next_follower;
363 /* Next follower in wakeup chain. */ 354 /* Next follower in wakeup chain. */
@@ -488,10 +479,14 @@ struct rcu_state {
488 /* due to no GP active. */ 479 /* due to no GP active. */
489 unsigned long gp_start; /* Time at which GP started, */ 480 unsigned long gp_start; /* Time at which GP started, */
490 /* but in jiffies. */ 481 /* but in jiffies. */
482 unsigned long gp_activity; /* Time of last GP kthread */
483 /* activity in jiffies. */
491 unsigned long jiffies_stall; /* Time at which to check */ 484 unsigned long jiffies_stall; /* Time at which to check */
492 /* for CPU stalls. */ 485 /* for CPU stalls. */
493 unsigned long jiffies_resched; /* Time at which to resched */ 486 unsigned long jiffies_resched; /* Time at which to resched */
494 /* a reluctant CPU. */ 487 /* a reluctant CPU. */
488 unsigned long n_force_qs_gpstart; /* Snapshot of n_force_qs at */
489 /* GP start. */
495 unsigned long gp_max; /* Maximum GP duration in */ 490 unsigned long gp_max; /* Maximum GP duration in */
496 /* jiffies. */ 491 /* jiffies. */
497 const char *name; /* Name of structure. */ 492 const char *name; /* Name of structure. */
@@ -514,13 +509,6 @@ extern struct list_head rcu_struct_flavors;
514#define for_each_rcu_flavor(rsp) \ 509#define for_each_rcu_flavor(rsp) \
515 list_for_each_entry((rsp), &rcu_struct_flavors, flavors) 510 list_for_each_entry((rsp), &rcu_struct_flavors, flavors)
516 511
517/* Return values for rcu_preempt_offline_tasks(). */
518
519#define RCU_OFL_TASKS_NORM_GP 0x1 /* Tasks blocking normal */
520 /* GP were moved to root. */
521#define RCU_OFL_TASKS_EXP_GP 0x2 /* Tasks blocking expedited */
522 /* GP were moved to root. */
523
524/* 512/*
525 * RCU implementation internal declarations: 513 * RCU implementation internal declarations:
526 */ 514 */
@@ -549,23 +537,13 @@ static void rcu_bootup_announce(void);
549static void rcu_preempt_note_context_switch(void); 537static void rcu_preempt_note_context_switch(void);
550static int rcu_preempt_blocked_readers_cgp(struct rcu_node *rnp); 538static int rcu_preempt_blocked_readers_cgp(struct rcu_node *rnp);
551#ifdef CONFIG_HOTPLUG_CPU 539#ifdef CONFIG_HOTPLUG_CPU
552static void rcu_report_unblock_qs_rnp(struct rcu_node *rnp, 540static bool rcu_preempt_has_tasks(struct rcu_node *rnp);
553 unsigned long flags);
554#endif /* #ifdef CONFIG_HOTPLUG_CPU */ 541#endif /* #ifdef CONFIG_HOTPLUG_CPU */
555static void rcu_print_detail_task_stall(struct rcu_state *rsp); 542static void rcu_print_detail_task_stall(struct rcu_state *rsp);
556static int rcu_print_task_stall(struct rcu_node *rnp); 543static int rcu_print_task_stall(struct rcu_node *rnp);
557static void rcu_preempt_check_blocked_tasks(struct rcu_node *rnp); 544static void rcu_preempt_check_blocked_tasks(struct rcu_node *rnp);
558#ifdef CONFIG_HOTPLUG_CPU
559static int rcu_preempt_offline_tasks(struct rcu_state *rsp,
560 struct rcu_node *rnp,
561 struct rcu_data *rdp);
562#endif /* #ifdef CONFIG_HOTPLUG_CPU */
563static void rcu_preempt_check_callbacks(void); 545static void rcu_preempt_check_callbacks(void);
564void call_rcu(struct rcu_head *head, void (*func)(struct rcu_head *rcu)); 546void call_rcu(struct rcu_head *head, void (*func)(struct rcu_head *rcu));
565#if defined(CONFIG_HOTPLUG_CPU) || defined(CONFIG_PREEMPT_RCU)
566static void rcu_report_exp_rnp(struct rcu_state *rsp, struct rcu_node *rnp,
567 bool wake);
568#endif /* #if defined(CONFIG_HOTPLUG_CPU) || defined(CONFIG_PREEMPT_RCU) */
569static void __init __rcu_init_preempt(void); 547static void __init __rcu_init_preempt(void);
570static void rcu_initiate_boost(struct rcu_node *rnp, unsigned long flags); 548static void rcu_initiate_boost(struct rcu_node *rnp, unsigned long flags);
571static void rcu_preempt_boost_start_gp(struct rcu_node *rnp); 549static void rcu_preempt_boost_start_gp(struct rcu_node *rnp);
@@ -621,24 +599,15 @@ static void rcu_dynticks_task_exit(void);
621#endif /* #ifndef RCU_TREE_NONCORE */ 599#endif /* #ifndef RCU_TREE_NONCORE */
622 600
623#ifdef CONFIG_RCU_TRACE 601#ifdef CONFIG_RCU_TRACE
624#ifdef CONFIG_RCU_NOCB_CPU 602/* Read out queue lengths for tracing. */
625/* Sum up queue lengths for tracing. */
626static inline void rcu_nocb_q_lengths(struct rcu_data *rdp, long *ql, long *qll) 603static inline void rcu_nocb_q_lengths(struct rcu_data *rdp, long *ql, long *qll)
627{ 604{
628 *ql = atomic_long_read(&rdp->nocb_q_count) + 605#ifdef CONFIG_RCU_NOCB_CPU
629 rdp->nocb_p_count + 606 *ql = atomic_long_read(&rdp->nocb_q_count);
630 atomic_long_read(&rdp->nocb_follower_count) + 607 *qll = atomic_long_read(&rdp->nocb_q_count_lazy);
631 rdp->nocb_p_count + rdp->nocb_gp_count;
632 *qll = atomic_long_read(&rdp->nocb_q_count_lazy) +
633 rdp->nocb_p_count_lazy +
634 atomic_long_read(&rdp->nocb_follower_count_lazy) +
635 rdp->nocb_p_count_lazy + rdp->nocb_gp_count_lazy;
636}
637#else /* #ifdef CONFIG_RCU_NOCB_CPU */ 608#else /* #ifdef CONFIG_RCU_NOCB_CPU */
638static inline void rcu_nocb_q_lengths(struct rcu_data *rdp, long *ql, long *qll)
639{
640 *ql = 0; 609 *ql = 0;
641 *qll = 0; 610 *qll = 0;
642}
643#endif /* #else #ifdef CONFIG_RCU_NOCB_CPU */ 611#endif /* #else #ifdef CONFIG_RCU_NOCB_CPU */
612}
644#endif /* #ifdef CONFIG_RCU_TRACE */ 613#endif /* #ifdef CONFIG_RCU_TRACE */
diff --git a/kernel/rcu/tree_plugin.h b/kernel/rcu/tree_plugin.h
index 07e61a04de1d..2e850a51bb8f 100644
--- a/kernel/rcu/tree_plugin.h
+++ b/kernel/rcu/tree_plugin.h
@@ -34,10 +34,6 @@
34 34
35#include "../locking/rtmutex_common.h" 35#include "../locking/rtmutex_common.h"
36 36
37/* rcuc/rcub kthread realtime priority */
38static int kthread_prio = CONFIG_RCU_KTHREAD_PRIO;
39module_param(kthread_prio, int, 0644);
40
41/* 37/*
42 * Control variables for per-CPU and per-rcu_node kthreads. These 38 * Control variables for per-CPU and per-rcu_node kthreads. These
43 * handle all flavors of RCU. 39 * handle all flavors of RCU.
@@ -103,6 +99,8 @@ RCU_STATE_INITIALIZER(rcu_preempt, 'p', call_rcu);
103static struct rcu_state *rcu_state_p = &rcu_preempt_state; 99static struct rcu_state *rcu_state_p = &rcu_preempt_state;
104 100
105static int rcu_preempted_readers_exp(struct rcu_node *rnp); 101static int rcu_preempted_readers_exp(struct rcu_node *rnp);
102static void rcu_report_exp_rnp(struct rcu_state *rsp, struct rcu_node *rnp,
103 bool wake);
106 104
107/* 105/*
108 * Tell them what RCU they are running. 106 * Tell them what RCU they are running.
@@ -288,15 +286,25 @@ static struct list_head *rcu_next_node_entry(struct task_struct *t,
288} 286}
289 287
290/* 288/*
289 * Return true if the specified rcu_node structure has tasks that were
290 * preempted within an RCU read-side critical section.
291 */
292static bool rcu_preempt_has_tasks(struct rcu_node *rnp)
293{
294 return !list_empty(&rnp->blkd_tasks);
295}
296
297/*
291 * Handle special cases during rcu_read_unlock(), such as needing to 298 * Handle special cases during rcu_read_unlock(), such as needing to
292 * notify RCU core processing or task having blocked during the RCU 299 * notify RCU core processing or task having blocked during the RCU
293 * read-side critical section. 300 * read-side critical section.
294 */ 301 */
295void rcu_read_unlock_special(struct task_struct *t) 302void rcu_read_unlock_special(struct task_struct *t)
296{ 303{
297 int empty; 304 bool empty;
298 int empty_exp; 305 bool empty_exp;
299 int empty_exp_now; 306 bool empty_norm;
307 bool empty_exp_now;
300 unsigned long flags; 308 unsigned long flags;
301 struct list_head *np; 309 struct list_head *np;
302#ifdef CONFIG_RCU_BOOST 310#ifdef CONFIG_RCU_BOOST
@@ -348,7 +356,8 @@ void rcu_read_unlock_special(struct task_struct *t)
348 break; 356 break;
349 raw_spin_unlock(&rnp->lock); /* irqs remain disabled. */ 357 raw_spin_unlock(&rnp->lock); /* irqs remain disabled. */
350 } 358 }
351 empty = !rcu_preempt_blocked_readers_cgp(rnp); 359 empty = !rcu_preempt_has_tasks(rnp);
360 empty_norm = !rcu_preempt_blocked_readers_cgp(rnp);
352 empty_exp = !rcu_preempted_readers_exp(rnp); 361 empty_exp = !rcu_preempted_readers_exp(rnp);
353 smp_mb(); /* ensure expedited fastpath sees end of RCU c-s. */ 362 smp_mb(); /* ensure expedited fastpath sees end of RCU c-s. */
354 np = rcu_next_node_entry(t, rnp); 363 np = rcu_next_node_entry(t, rnp);
@@ -368,13 +377,21 @@ void rcu_read_unlock_special(struct task_struct *t)
368#endif /* #ifdef CONFIG_RCU_BOOST */ 377#endif /* #ifdef CONFIG_RCU_BOOST */
369 378
370 /* 379 /*
380 * If this was the last task on the list, go see if we
381 * need to propagate ->qsmaskinit bit clearing up the
382 * rcu_node tree.
383 */
384 if (!empty && !rcu_preempt_has_tasks(rnp))
385 rcu_cleanup_dead_rnp(rnp);
386
387 /*
371 * If this was the last task on the current list, and if 388 * If this was the last task on the current list, and if
372 * we aren't waiting on any CPUs, report the quiescent state. 389 * we aren't waiting on any CPUs, report the quiescent state.
373 * Note that rcu_report_unblock_qs_rnp() releases rnp->lock, 390 * Note that rcu_report_unblock_qs_rnp() releases rnp->lock,
374 * so we must take a snapshot of the expedited state. 391 * so we must take a snapshot of the expedited state.
375 */ 392 */
376 empty_exp_now = !rcu_preempted_readers_exp(rnp); 393 empty_exp_now = !rcu_preempted_readers_exp(rnp);
377 if (!empty && !rcu_preempt_blocked_readers_cgp(rnp)) { 394 if (!empty_norm && !rcu_preempt_blocked_readers_cgp(rnp)) {
378 trace_rcu_quiescent_state_report(TPS("preempt_rcu"), 395 trace_rcu_quiescent_state_report(TPS("preempt_rcu"),
379 rnp->gpnum, 396 rnp->gpnum,
380 0, rnp->qsmask, 397 0, rnp->qsmask,
@@ -389,10 +406,8 @@ void rcu_read_unlock_special(struct task_struct *t)
389 406
390#ifdef CONFIG_RCU_BOOST 407#ifdef CONFIG_RCU_BOOST
391 /* Unboost if we were boosted. */ 408 /* Unboost if we were boosted. */
392 if (drop_boost_mutex) { 409 if (drop_boost_mutex)
393 rt_mutex_unlock(&rnp->boost_mtx); 410 rt_mutex_unlock(&rnp->boost_mtx);
394 complete(&rnp->boost_completion);
395 }
396#endif /* #ifdef CONFIG_RCU_BOOST */ 411#endif /* #ifdef CONFIG_RCU_BOOST */
397 412
398 /* 413 /*
@@ -500,99 +515,13 @@ static int rcu_print_task_stall(struct rcu_node *rnp)
500static void rcu_preempt_check_blocked_tasks(struct rcu_node *rnp) 515static void rcu_preempt_check_blocked_tasks(struct rcu_node *rnp)
501{ 516{
502 WARN_ON_ONCE(rcu_preempt_blocked_readers_cgp(rnp)); 517 WARN_ON_ONCE(rcu_preempt_blocked_readers_cgp(rnp));
503 if (!list_empty(&rnp->blkd_tasks)) 518 if (rcu_preempt_has_tasks(rnp))
504 rnp->gp_tasks = rnp->blkd_tasks.next; 519 rnp->gp_tasks = rnp->blkd_tasks.next;
505 WARN_ON_ONCE(rnp->qsmask); 520 WARN_ON_ONCE(rnp->qsmask);
506} 521}
507 522
508#ifdef CONFIG_HOTPLUG_CPU 523#ifdef CONFIG_HOTPLUG_CPU
509 524
510/*
511 * Handle tasklist migration for case in which all CPUs covered by the
512 * specified rcu_node have gone offline. Move them up to the root
513 * rcu_node. The reason for not just moving them to the immediate
514 * parent is to remove the need for rcu_read_unlock_special() to
515 * make more than two attempts to acquire the target rcu_node's lock.
516 * Returns true if there were tasks blocking the current RCU grace
517 * period.
518 *
519 * Returns 1 if there was previously a task blocking the current grace
520 * period on the specified rcu_node structure.
521 *
522 * The caller must hold rnp->lock with irqs disabled.
523 */
524static int rcu_preempt_offline_tasks(struct rcu_state *rsp,
525 struct rcu_node *rnp,
526 struct rcu_data *rdp)
527{
528 struct list_head *lp;
529 struct list_head *lp_root;
530 int retval = 0;
531 struct rcu_node *rnp_root = rcu_get_root(rsp);
532 struct task_struct *t;
533
534 if (rnp == rnp_root) {
535 WARN_ONCE(1, "Last CPU thought to be offlined?");
536 return 0; /* Shouldn't happen: at least one CPU online. */
537 }
538
539 /* If we are on an internal node, complain bitterly. */
540 WARN_ON_ONCE(rnp != rdp->mynode);
541
542 /*
543 * Move tasks up to root rcu_node. Don't try to get fancy for
544 * this corner-case operation -- just put this node's tasks
545 * at the head of the root node's list, and update the root node's
546 * ->gp_tasks and ->exp_tasks pointers to those of this node's,
547 * if non-NULL. This might result in waiting for more tasks than
548 * absolutely necessary, but this is a good performance/complexity
549 * tradeoff.
550 */
551 if (rcu_preempt_blocked_readers_cgp(rnp) && rnp->qsmask == 0)
552 retval |= RCU_OFL_TASKS_NORM_GP;
553 if (rcu_preempted_readers_exp(rnp))
554 retval |= RCU_OFL_TASKS_EXP_GP;
555 lp = &rnp->blkd_tasks;
556 lp_root = &rnp_root->blkd_tasks;
557 while (!list_empty(lp)) {
558 t = list_entry(lp->next, typeof(*t), rcu_node_entry);
559 raw_spin_lock(&rnp_root->lock); /* irqs already disabled */
560 smp_mb__after_unlock_lock();
561 list_del(&t->rcu_node_entry);
562 t->rcu_blocked_node = rnp_root;
563 list_add(&t->rcu_node_entry, lp_root);
564 if (&t->rcu_node_entry == rnp->gp_tasks)
565 rnp_root->gp_tasks = rnp->gp_tasks;
566 if (&t->rcu_node_entry == rnp->exp_tasks)
567 rnp_root->exp_tasks = rnp->exp_tasks;
568#ifdef CONFIG_RCU_BOOST
569 if (&t->rcu_node_entry == rnp->boost_tasks)
570 rnp_root->boost_tasks = rnp->boost_tasks;
571#endif /* #ifdef CONFIG_RCU_BOOST */
572 raw_spin_unlock(&rnp_root->lock); /* irqs still disabled */
573 }
574
575 rnp->gp_tasks = NULL;
576 rnp->exp_tasks = NULL;
577#ifdef CONFIG_RCU_BOOST
578 rnp->boost_tasks = NULL;
579 /*
580 * In case root is being boosted and leaf was not. Make sure
581 * that we boost the tasks blocking the current grace period
582 * in this case.
583 */
584 raw_spin_lock(&rnp_root->lock); /* irqs already disabled */
585 smp_mb__after_unlock_lock();
586 if (rnp_root->boost_tasks != NULL &&
587 rnp_root->boost_tasks != rnp_root->gp_tasks &&
588 rnp_root->boost_tasks != rnp_root->exp_tasks)
589 rnp_root->boost_tasks = rnp_root->gp_tasks;
590 raw_spin_unlock(&rnp_root->lock); /* irqs still disabled */
591#endif /* #ifdef CONFIG_RCU_BOOST */
592
593 return retval;
594}
595
596#endif /* #ifdef CONFIG_HOTPLUG_CPU */ 525#endif /* #ifdef CONFIG_HOTPLUG_CPU */
597 526
598/* 527/*
@@ -752,7 +681,7 @@ sync_rcu_preempt_exp_init(struct rcu_state *rsp, struct rcu_node *rnp)
752 681
753 raw_spin_lock_irqsave(&rnp->lock, flags); 682 raw_spin_lock_irqsave(&rnp->lock, flags);
754 smp_mb__after_unlock_lock(); 683 smp_mb__after_unlock_lock();
755 if (list_empty(&rnp->blkd_tasks)) { 684 if (!rcu_preempt_has_tasks(rnp)) {
756 raw_spin_unlock_irqrestore(&rnp->lock, flags); 685 raw_spin_unlock_irqrestore(&rnp->lock, flags);
757 } else { 686 } else {
758 rnp->exp_tasks = rnp->blkd_tasks.next; 687 rnp->exp_tasks = rnp->blkd_tasks.next;
@@ -932,11 +861,12 @@ static int rcu_preempt_blocked_readers_cgp(struct rcu_node *rnp)
932 861
933#ifdef CONFIG_HOTPLUG_CPU 862#ifdef CONFIG_HOTPLUG_CPU
934 863
935/* Because preemptible RCU does not exist, no quieting of tasks. */ 864/*
936static void rcu_report_unblock_qs_rnp(struct rcu_node *rnp, unsigned long flags) 865 * Because there is no preemptible RCU, there can be no readers blocked.
937 __releases(rnp->lock) 866 */
867static bool rcu_preempt_has_tasks(struct rcu_node *rnp)
938{ 868{
939 raw_spin_unlock_irqrestore(&rnp->lock, flags); 869 return false;
940} 870}
941 871
942#endif /* #ifdef CONFIG_HOTPLUG_CPU */ 872#endif /* #ifdef CONFIG_HOTPLUG_CPU */
@@ -968,23 +898,6 @@ static void rcu_preempt_check_blocked_tasks(struct rcu_node *rnp)
968 WARN_ON_ONCE(rnp->qsmask); 898 WARN_ON_ONCE(rnp->qsmask);
969} 899}
970 900
971#ifdef CONFIG_HOTPLUG_CPU
972
973/*
974 * Because preemptible RCU does not exist, it never needs to migrate
975 * tasks that were blocked within RCU read-side critical sections, and
976 * such non-existent tasks cannot possibly have been blocking the current
977 * grace period.
978 */
979static int rcu_preempt_offline_tasks(struct rcu_state *rsp,
980 struct rcu_node *rnp,
981 struct rcu_data *rdp)
982{
983 return 0;
984}
985
986#endif /* #ifdef CONFIG_HOTPLUG_CPU */
987
988/* 901/*
989 * Because preemptible RCU does not exist, it never has any callbacks 902 * Because preemptible RCU does not exist, it never has any callbacks
990 * to check. 903 * to check.
@@ -1003,20 +916,6 @@ void synchronize_rcu_expedited(void)
1003} 916}
1004EXPORT_SYMBOL_GPL(synchronize_rcu_expedited); 917EXPORT_SYMBOL_GPL(synchronize_rcu_expedited);
1005 918
1006#ifdef CONFIG_HOTPLUG_CPU
1007
1008/*
1009 * Because preemptible RCU does not exist, there is never any need to
1010 * report on tasks preempted in RCU read-side critical sections during
1011 * expedited RCU grace periods.
1012 */
1013static void rcu_report_exp_rnp(struct rcu_state *rsp, struct rcu_node *rnp,
1014 bool wake)
1015{
1016}
1017
1018#endif /* #ifdef CONFIG_HOTPLUG_CPU */
1019
1020/* 919/*
1021 * Because preemptible RCU does not exist, rcu_barrier() is just 920 * Because preemptible RCU does not exist, rcu_barrier() is just
1022 * another name for rcu_barrier_sched(). 921 * another name for rcu_barrier_sched().
@@ -1052,7 +951,7 @@ void exit_rcu(void)
1052 951
1053static void rcu_initiate_boost_trace(struct rcu_node *rnp) 952static void rcu_initiate_boost_trace(struct rcu_node *rnp)
1054{ 953{
1055 if (list_empty(&rnp->blkd_tasks)) 954 if (!rcu_preempt_has_tasks(rnp))
1056 rnp->n_balk_blkd_tasks++; 955 rnp->n_balk_blkd_tasks++;
1057 else if (rnp->exp_tasks == NULL && rnp->gp_tasks == NULL) 956 else if (rnp->exp_tasks == NULL && rnp->gp_tasks == NULL)
1058 rnp->n_balk_exp_gp_tasks++; 957 rnp->n_balk_exp_gp_tasks++;
@@ -1099,7 +998,8 @@ static int rcu_boost(struct rcu_node *rnp)
1099 struct task_struct *t; 998 struct task_struct *t;
1100 struct list_head *tb; 999 struct list_head *tb;
1101 1000
1102 if (rnp->exp_tasks == NULL && rnp->boost_tasks == NULL) 1001 if (ACCESS_ONCE(rnp->exp_tasks) == NULL &&
1002 ACCESS_ONCE(rnp->boost_tasks) == NULL)
1103 return 0; /* Nothing left to boost. */ 1003 return 0; /* Nothing left to boost. */
1104 1004
1105 raw_spin_lock_irqsave(&rnp->lock, flags); 1005 raw_spin_lock_irqsave(&rnp->lock, flags);
@@ -1147,15 +1047,11 @@ static int rcu_boost(struct rcu_node *rnp)
1147 */ 1047 */
1148 t = container_of(tb, struct task_struct, rcu_node_entry); 1048 t = container_of(tb, struct task_struct, rcu_node_entry);
1149 rt_mutex_init_proxy_locked(&rnp->boost_mtx, t); 1049 rt_mutex_init_proxy_locked(&rnp->boost_mtx, t);
1150 init_completion(&rnp->boost_completion);
1151 raw_spin_unlock_irqrestore(&rnp->lock, flags); 1050 raw_spin_unlock_irqrestore(&rnp->lock, flags);
1152 /* Lock only for side effect: boosts task t's priority. */ 1051 /* Lock only for side effect: boosts task t's priority. */
1153 rt_mutex_lock(&rnp->boost_mtx); 1052 rt_mutex_lock(&rnp->boost_mtx);
1154 rt_mutex_unlock(&rnp->boost_mtx); /* Then keep lockdep happy. */ 1053 rt_mutex_unlock(&rnp->boost_mtx); /* Then keep lockdep happy. */
1155 1054
1156 /* Wait for boostee to be done w/boost_mtx before reinitializing. */
1157 wait_for_completion(&rnp->boost_completion);
1158
1159 return ACCESS_ONCE(rnp->exp_tasks) != NULL || 1055 return ACCESS_ONCE(rnp->exp_tasks) != NULL ||
1160 ACCESS_ONCE(rnp->boost_tasks) != NULL; 1056 ACCESS_ONCE(rnp->boost_tasks) != NULL;
1161} 1057}
@@ -1388,12 +1284,8 @@ static void rcu_boost_kthread_setaffinity(struct rcu_node *rnp, int outgoingcpu)
1388 for (cpu = rnp->grplo; cpu <= rnp->grphi; cpu++, mask >>= 1) 1284 for (cpu = rnp->grplo; cpu <= rnp->grphi; cpu++, mask >>= 1)
1389 if ((mask & 0x1) && cpu != outgoingcpu) 1285 if ((mask & 0x1) && cpu != outgoingcpu)
1390 cpumask_set_cpu(cpu, cm); 1286 cpumask_set_cpu(cpu, cm);
1391 if (cpumask_weight(cm) == 0) { 1287 if (cpumask_weight(cm) == 0)
1392 cpumask_setall(cm); 1288 cpumask_setall(cm);
1393 for (cpu = rnp->grplo; cpu <= rnp->grphi; cpu++)
1394 cpumask_clear_cpu(cpu, cm);
1395 WARN_ON_ONCE(cpumask_weight(cm) == 0);
1396 }
1397 set_cpus_allowed_ptr(t, cm); 1289 set_cpus_allowed_ptr(t, cm);
1398 free_cpumask_var(cm); 1290 free_cpumask_var(cm);
1399} 1291}
@@ -1418,12 +1310,8 @@ static void __init rcu_spawn_boost_kthreads(void)
1418 for_each_possible_cpu(cpu) 1310 for_each_possible_cpu(cpu)
1419 per_cpu(rcu_cpu_has_work, cpu) = 0; 1311 per_cpu(rcu_cpu_has_work, cpu) = 0;
1420 BUG_ON(smpboot_register_percpu_thread(&rcu_cpu_thread_spec)); 1312 BUG_ON(smpboot_register_percpu_thread(&rcu_cpu_thread_spec));
1421 rnp = rcu_get_root(rcu_state_p); 1313 rcu_for_each_leaf_node(rcu_state_p, rnp)
1422 (void)rcu_spawn_one_boost_kthread(rcu_state_p, rnp); 1314 (void)rcu_spawn_one_boost_kthread(rcu_state_p, rnp);
1423 if (NUM_RCU_NODES > 1) {
1424 rcu_for_each_leaf_node(rcu_state_p, rnp)
1425 (void)rcu_spawn_one_boost_kthread(rcu_state_p, rnp);
1426 }
1427} 1315}
1428 1316
1429static void rcu_prepare_kthreads(int cpu) 1317static void rcu_prepare_kthreads(int cpu)
@@ -1577,7 +1465,8 @@ static bool __maybe_unused rcu_try_advance_all_cbs(void)
1577 * completed since we last checked and there are 1465 * completed since we last checked and there are
1578 * callbacks not yet ready to invoke. 1466 * callbacks not yet ready to invoke.
1579 */ 1467 */
1580 if (rdp->completed != rnp->completed && 1468 if ((rdp->completed != rnp->completed ||
1469 unlikely(ACCESS_ONCE(rdp->gpwrap))) &&
1581 rdp->nxttail[RCU_DONE_TAIL] != rdp->nxttail[RCU_NEXT_TAIL]) 1470 rdp->nxttail[RCU_DONE_TAIL] != rdp->nxttail[RCU_NEXT_TAIL])
1582 note_gp_changes(rsp, rdp); 1471 note_gp_changes(rsp, rdp);
1583 1472
@@ -1870,11 +1759,12 @@ static void print_cpu_stall_info(struct rcu_state *rsp, int cpu)
1870 ticks_value = rsp->gpnum - rdp->gpnum; 1759 ticks_value = rsp->gpnum - rdp->gpnum;
1871 } 1760 }
1872 print_cpu_stall_fast_no_hz(fast_no_hz, cpu); 1761 print_cpu_stall_fast_no_hz(fast_no_hz, cpu);
1873 pr_err("\t%d: (%lu %s) idle=%03x/%llx/%d softirq=%u/%u %s\n", 1762 pr_err("\t%d: (%lu %s) idle=%03x/%llx/%d softirq=%u/%u fqs=%ld %s\n",
1874 cpu, ticks_value, ticks_title, 1763 cpu, ticks_value, ticks_title,
1875 atomic_read(&rdtp->dynticks) & 0xfff, 1764 atomic_read(&rdtp->dynticks) & 0xfff,
1876 rdtp->dynticks_nesting, rdtp->dynticks_nmi_nesting, 1765 rdtp->dynticks_nesting, rdtp->dynticks_nmi_nesting,
1877 rdp->softirq_snap, kstat_softirqs_cpu(RCU_SOFTIRQ, cpu), 1766 rdp->softirq_snap, kstat_softirqs_cpu(RCU_SOFTIRQ, cpu),
1767 ACCESS_ONCE(rsp->n_force_qs) - rsp->n_force_qs_gpstart,
1878 fast_no_hz); 1768 fast_no_hz);
1879} 1769}
1880 1770
@@ -2028,9 +1918,26 @@ static void wake_nocb_leader(struct rcu_data *rdp, bool force)
2028static bool rcu_nocb_cpu_needs_barrier(struct rcu_state *rsp, int cpu) 1918static bool rcu_nocb_cpu_needs_barrier(struct rcu_state *rsp, int cpu)
2029{ 1919{
2030 struct rcu_data *rdp = per_cpu_ptr(rsp->rda, cpu); 1920 struct rcu_data *rdp = per_cpu_ptr(rsp->rda, cpu);
1921 unsigned long ret;
1922#ifdef CONFIG_PROVE_RCU
2031 struct rcu_head *rhp; 1923 struct rcu_head *rhp;
1924#endif /* #ifdef CONFIG_PROVE_RCU */
2032 1925
2033 /* No-CBs CPUs might have callbacks on any of three lists. */ 1926 /*
1927 * Check count of all no-CBs callbacks awaiting invocation.
1928 * There needs to be a barrier before this function is called,
1929 * but associated with a prior determination that no more
1930 * callbacks would be posted. In the worst case, the first
1931 * barrier in _rcu_barrier() suffices (but the caller cannot
1932 * necessarily rely on this, not a substitute for the caller
1933 * getting the concurrency design right!). There must also be
1934 * a barrier between the following load an posting of a callback
1935 * (if a callback is in fact needed). This is associated with an
1936 * atomic_inc() in the caller.
1937 */
1938 ret = atomic_long_read(&rdp->nocb_q_count);
1939
1940#ifdef CONFIG_PROVE_RCU
2034 rhp = ACCESS_ONCE(rdp->nocb_head); 1941 rhp = ACCESS_ONCE(rdp->nocb_head);
2035 if (!rhp) 1942 if (!rhp)
2036 rhp = ACCESS_ONCE(rdp->nocb_gp_head); 1943 rhp = ACCESS_ONCE(rdp->nocb_gp_head);
@@ -2044,8 +1951,9 @@ static bool rcu_nocb_cpu_needs_barrier(struct rcu_state *rsp, int cpu)
2044 cpu, rhp->func); 1951 cpu, rhp->func);
2045 WARN_ON_ONCE(1); 1952 WARN_ON_ONCE(1);
2046 } 1953 }
1954#endif /* #ifdef CONFIG_PROVE_RCU */
2047 1955
2048 return !!rhp; 1956 return !!ret;
2049} 1957}
2050 1958
2051/* 1959/*
@@ -2067,9 +1975,10 @@ static void __call_rcu_nocb_enqueue(struct rcu_data *rdp,
2067 struct task_struct *t; 1975 struct task_struct *t;
2068 1976
2069 /* Enqueue the callback on the nocb list and update counts. */ 1977 /* Enqueue the callback on the nocb list and update counts. */
1978 atomic_long_add(rhcount, &rdp->nocb_q_count);
1979 /* rcu_barrier() relies on ->nocb_q_count add before xchg. */
2070 old_rhpp = xchg(&rdp->nocb_tail, rhtp); 1980 old_rhpp = xchg(&rdp->nocb_tail, rhtp);
2071 ACCESS_ONCE(*old_rhpp) = rhp; 1981 ACCESS_ONCE(*old_rhpp) = rhp;
2072 atomic_long_add(rhcount, &rdp->nocb_q_count);
2073 atomic_long_add(rhcount_lazy, &rdp->nocb_q_count_lazy); 1982 atomic_long_add(rhcount_lazy, &rdp->nocb_q_count_lazy);
2074 smp_mb__after_atomic(); /* Store *old_rhpp before _wake test. */ 1983 smp_mb__after_atomic(); /* Store *old_rhpp before _wake test. */
2075 1984
@@ -2260,9 +2169,6 @@ wait_again:
2260 /* Move callbacks to wait-for-GP list, which is empty. */ 2169 /* Move callbacks to wait-for-GP list, which is empty. */
2261 ACCESS_ONCE(rdp->nocb_head) = NULL; 2170 ACCESS_ONCE(rdp->nocb_head) = NULL;
2262 rdp->nocb_gp_tail = xchg(&rdp->nocb_tail, &rdp->nocb_head); 2171 rdp->nocb_gp_tail = xchg(&rdp->nocb_tail, &rdp->nocb_head);
2263 rdp->nocb_gp_count = atomic_long_xchg(&rdp->nocb_q_count, 0);
2264 rdp->nocb_gp_count_lazy =
2265 atomic_long_xchg(&rdp->nocb_q_count_lazy, 0);
2266 gotcbs = true; 2172 gotcbs = true;
2267 } 2173 }
2268 2174
@@ -2310,9 +2216,6 @@ wait_again:
2310 /* Append callbacks to follower's "done" list. */ 2216 /* Append callbacks to follower's "done" list. */
2311 tail = xchg(&rdp->nocb_follower_tail, rdp->nocb_gp_tail); 2217 tail = xchg(&rdp->nocb_follower_tail, rdp->nocb_gp_tail);
2312 *tail = rdp->nocb_gp_head; 2218 *tail = rdp->nocb_gp_head;
2313 atomic_long_add(rdp->nocb_gp_count, &rdp->nocb_follower_count);
2314 atomic_long_add(rdp->nocb_gp_count_lazy,
2315 &rdp->nocb_follower_count_lazy);
2316 smp_mb__after_atomic(); /* Store *tail before wakeup. */ 2219 smp_mb__after_atomic(); /* Store *tail before wakeup. */
2317 if (rdp != my_rdp && tail == &rdp->nocb_follower_head) { 2220 if (rdp != my_rdp && tail == &rdp->nocb_follower_head) {
2318 /* 2221 /*
@@ -2387,13 +2290,11 @@ static int rcu_nocb_kthread(void *arg)
2387 trace_rcu_nocb_wake(rdp->rsp->name, rdp->cpu, "WokeNonEmpty"); 2290 trace_rcu_nocb_wake(rdp->rsp->name, rdp->cpu, "WokeNonEmpty");
2388 ACCESS_ONCE(rdp->nocb_follower_head) = NULL; 2291 ACCESS_ONCE(rdp->nocb_follower_head) = NULL;
2389 tail = xchg(&rdp->nocb_follower_tail, &rdp->nocb_follower_head); 2292 tail = xchg(&rdp->nocb_follower_tail, &rdp->nocb_follower_head);
2390 c = atomic_long_xchg(&rdp->nocb_follower_count, 0);
2391 cl = atomic_long_xchg(&rdp->nocb_follower_count_lazy, 0);
2392 rdp->nocb_p_count += c;
2393 rdp->nocb_p_count_lazy += cl;
2394 2293
2395 /* Each pass through the following loop invokes a callback. */ 2294 /* Each pass through the following loop invokes a callback. */
2396 trace_rcu_batch_start(rdp->rsp->name, cl, c, -1); 2295 trace_rcu_batch_start(rdp->rsp->name,
2296 atomic_long_read(&rdp->nocb_q_count_lazy),
2297 atomic_long_read(&rdp->nocb_q_count), -1);
2397 c = cl = 0; 2298 c = cl = 0;
2398 while (list) { 2299 while (list) {
2399 next = list->next; 2300 next = list->next;
@@ -2415,9 +2316,9 @@ static int rcu_nocb_kthread(void *arg)
2415 list = next; 2316 list = next;
2416 } 2317 }
2417 trace_rcu_batch_end(rdp->rsp->name, c, !!list, 0, 0, 1); 2318 trace_rcu_batch_end(rdp->rsp->name, c, !!list, 0, 0, 1);
2418 ACCESS_ONCE(rdp->nocb_p_count) = rdp->nocb_p_count - c; 2319 smp_mb__before_atomic(); /* _add after CB invocation. */
2419 ACCESS_ONCE(rdp->nocb_p_count_lazy) = 2320 atomic_long_add(-c, &rdp->nocb_q_count);
2420 rdp->nocb_p_count_lazy - cl; 2321 atomic_long_add(-cl, &rdp->nocb_q_count_lazy);
2421 rdp->n_nocbs_invoked += c; 2322 rdp->n_nocbs_invoked += c;
2422 } 2323 }
2423 return 0; 2324 return 0;
diff --git a/kernel/rcu/tree_trace.c b/kernel/rcu/tree_trace.c
index 5cdc62e1beeb..fbb6240509ea 100644
--- a/kernel/rcu/tree_trace.c
+++ b/kernel/rcu/tree_trace.c
@@ -46,6 +46,8 @@
46#define RCU_TREE_NONCORE 46#define RCU_TREE_NONCORE
47#include "tree.h" 47#include "tree.h"
48 48
49DECLARE_PER_CPU_SHARED_ALIGNED(unsigned long, rcu_qs_ctr);
50
49static int r_open(struct inode *inode, struct file *file, 51static int r_open(struct inode *inode, struct file *file,
50 const struct seq_operations *op) 52 const struct seq_operations *op)
51{ 53{
@@ -115,11 +117,13 @@ static void print_one_rcu_data(struct seq_file *m, struct rcu_data *rdp)
115 117
116 if (!rdp->beenonline) 118 if (!rdp->beenonline)
117 return; 119 return;
118 seq_printf(m, "%3d%cc=%ld g=%ld pq=%d qp=%d", 120 seq_printf(m, "%3d%cc=%ld g=%ld pq=%d/%d qp=%d",
119 rdp->cpu, 121 rdp->cpu,
120 cpu_is_offline(rdp->cpu) ? '!' : ' ', 122 cpu_is_offline(rdp->cpu) ? '!' : ' ',
121 ulong2long(rdp->completed), ulong2long(rdp->gpnum), 123 ulong2long(rdp->completed), ulong2long(rdp->gpnum),
122 rdp->passed_quiesce, rdp->qs_pending); 124 rdp->passed_quiesce,
125 rdp->rcu_qs_ctr_snap == per_cpu(rcu_qs_ctr, rdp->cpu),
126 rdp->qs_pending);
123 seq_printf(m, " dt=%d/%llx/%d df=%lu", 127 seq_printf(m, " dt=%d/%llx/%d df=%lu",
124 atomic_read(&rdp->dynticks->dynticks), 128 atomic_read(&rdp->dynticks->dynticks),
125 rdp->dynticks->dynticks_nesting, 129 rdp->dynticks->dynticks_nesting,
diff --git a/kernel/softirq.c b/kernel/softirq.c
index 501baa9ac1be..8cdb98847c7b 100644
--- a/kernel/softirq.c
+++ b/kernel/softirq.c
@@ -656,9 +656,8 @@ static void run_ksoftirqd(unsigned int cpu)
656 * in the task stack here. 656 * in the task stack here.
657 */ 657 */
658 __do_softirq(); 658 __do_softirq();
659 rcu_note_context_switch();
660 local_irq_enable(); 659 local_irq_enable();
661 cond_resched(); 660 cond_resched_rcu_qs();
662 return; 661 return;
663 } 662 }
664 local_irq_enable(); 663 local_irq_enable();
diff --git a/lib/Kconfig.debug b/lib/Kconfig.debug
index 5f2ce616c046..a2ca213c71ca 100644
--- a/lib/Kconfig.debug
+++ b/lib/Kconfig.debug
@@ -1215,6 +1215,7 @@ config RCU_TORTURE_TEST
1215 tristate "torture tests for RCU" 1215 tristate "torture tests for RCU"
1216 depends on DEBUG_KERNEL 1216 depends on DEBUG_KERNEL
1217 select TORTURE_TEST 1217 select TORTURE_TEST
1218 select SRCU
1218 default n 1219 default n
1219 help 1220 help
1220 This option provides a kernel module that runs torture tests 1221 This option provides a kernel module that runs torture tests
@@ -1257,7 +1258,7 @@ config RCU_CPU_STALL_TIMEOUT
1257config RCU_CPU_STALL_INFO 1258config RCU_CPU_STALL_INFO
1258 bool "Print additional diagnostics on RCU CPU stall" 1259 bool "Print additional diagnostics on RCU CPU stall"
1259 depends on (TREE_RCU || PREEMPT_RCU) && DEBUG_KERNEL 1260 depends on (TREE_RCU || PREEMPT_RCU) && DEBUG_KERNEL
1260 default n 1261 default y
1261 help 1262 help
1262 For each stalled CPU that is aware of the current RCU grace 1263 For each stalled CPU that is aware of the current RCU grace
1263 period, print out additional per-CPU diagnostic information 1264 period, print out additional per-CPU diagnostic information
diff --git a/mm/Kconfig b/mm/Kconfig
index 1d1ae6b078fd..4395b12869c8 100644
--- a/mm/Kconfig
+++ b/mm/Kconfig
@@ -325,6 +325,7 @@ config VIRT_TO_BUS
325 325
326config MMU_NOTIFIER 326config MMU_NOTIFIER
327 bool 327 bool
328 select SRCU
328 329
329config KSM 330config KSM
330 bool "Enable KSM for page merging" 331 bool "Enable KSM for page merging"
diff --git a/security/tomoyo/Kconfig b/security/tomoyo/Kconfig
index 8eb779b9d77f..604e718d68d3 100644
--- a/security/tomoyo/Kconfig
+++ b/security/tomoyo/Kconfig
@@ -5,6 +5,7 @@ config SECURITY_TOMOYO
5 select SECURITYFS 5 select SECURITYFS
6 select SECURITY_PATH 6 select SECURITY_PATH
7 select SECURITY_NETWORK 7 select SECURITY_NETWORK
8 select SRCU
8 default n 9 default n
9 help 10 help
10 This selects TOMOYO Linux, pathname-based access control. 11 This selects TOMOYO Linux, pathname-based access control.
diff --git a/tools/testing/selftests/rcutorture/bin/parse-console.sh b/tools/testing/selftests/rcutorture/bin/parse-console.sh
index f962ba4cf68b..d8f35cf116be 100755
--- a/tools/testing/selftests/rcutorture/bin/parse-console.sh
+++ b/tools/testing/selftests/rcutorture/bin/parse-console.sh
@@ -36,7 +36,7 @@ if grep -Pq '\x00' < $file
36then 36then
37 print_warning Console output contains nul bytes, old qemu still running? 37 print_warning Console output contains nul bytes, old qemu still running?
38fi 38fi
39egrep 'Badness|WARNING:|Warn|BUG|===========|Call Trace:|Oops:' < $file | grep -v 'ODEBUG: ' | grep -v 'Warning: unable to open an initial console' > $T 39egrep 'Badness|WARNING:|Warn|BUG|===========|Call Trace:|Oops:|Stall ended before state dump start' < $file | grep -v 'ODEBUG: ' | grep -v 'Warning: unable to open an initial console' > $T
40if test -s $T 40if test -s $T
41then 41then
42 print_warning Assertion failure in $file $title 42 print_warning Assertion failure in $file $title
generated by cgit v1.2.2 (git 2.25.0) at 2025-12-28 09:31:23 -0500