aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorPaul E. McKenney <paulmck@linux.vnet.ibm.com>2017-05-15 17:57:01 -0400
committerPaul E. McKenney <paulmck@linux.vnet.ibm.com>2017-06-08 21:52:42 -0400
commitbd8cc5a062f41e334596edbe823e2fa0adddd1b7 (patch)
treed484e0604f4bd8634f21bde4ba19edd1f6879873
parent7f0cd6333086ae09962791c31f0d4845a3329df9 (diff)
srcu: Remove Classic SRCU
Classic SRCU was only ever intended to be a fallback in case of issues with Tree/Tiny SRCU, and the latter two are doing quite well in testing. This commit therefore removes Classic SRCU. Signed-off-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
Diffstat
-rw-r--r--include/linux/srcu.h2
-rw-r--r--include/linux/srcuclassic.h100
-rw-r--r--init/Kconfig21
-rw-r--r--kernel/rcu/Makefile1
-rw-r--r--kernel/rcu/rcu.h16
-rw-r--r--kernel/rcu/rcutorture.c17
-rw-r--r--kernel/rcu/srcu.c668
-rw-r--r--tools/testing/selftests/rcutorture/configs/rcu/CFLIST1
-rw-r--r--tools/testing/selftests/rcutorture/configs/rcu/SRCU-C11
-rw-r--r--tools/testing/selftests/rcutorture/configs/rcuperf/SRCUCLASSIC16
10 files changed, 2 insertions, 851 deletions
diff --git a/include/linux/srcu.h b/include/linux/srcu.h
index 5f509018e6b5..39af9bc0f653 100644
--- a/include/linux/srcu.h
+++ b/include/linux/srcu.h
@@ -60,8 +60,6 @@ int init_srcu_struct(struct srcu_struct *sp);
60#include <linux/srcutiny.h> 60#include <linux/srcutiny.h>
61#elif defined(CONFIG_TREE_SRCU) 61#elif defined(CONFIG_TREE_SRCU)
62#include <linux/srcutree.h> 62#include <linux/srcutree.h>
63#elif defined(CONFIG_CLASSIC_SRCU)
64#include <linux/srcuclassic.h>
65#elif defined(CONFIG_SRCU) 63#elif defined(CONFIG_SRCU)
66#error "Unknown SRCU implementation specified to kernel configuration" 64#error "Unknown SRCU implementation specified to kernel configuration"
67#else 65#else
diff --git a/include/linux/srcuclassic.h b/include/linux/srcuclassic.h
deleted file mode 100644
index 67db4a36ef0d..000000000000
--- a/include/linux/srcuclassic.h
+++ /dev/null
@@ -1,100 +0,0 @@
1/*
2 * Sleepable Read-Copy Update mechanism for mutual exclusion,
3 * classic v4.11 variant.
4 *
5 * This program is free software; you can redistribute it and/or modify
6 * it under the terms of the GNU General Public License as published by
7 * the Free Software Foundation; either version 2 of the License, or
8 * (at your option) any later version.
9 *
10 * This program is distributed in the hope that it will be useful,
11 * but WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 * GNU General Public License for more details.
14 *
15 * You should have received a copy of the GNU General Public License
16 * along with this program; if not, you can access it online at
17 * http://www.gnu.org/licenses/gpl-2.0.html.
18 *
19 * Copyright (C) IBM Corporation, 2017
20 *
21 * Author: Paul McKenney <paulmck@us.ibm.com>
22 */
23
24#ifndef _LINUX_SRCU_CLASSIC_H
25#define _LINUX_SRCU_CLASSIC_H
26
27struct srcu_array {
28 unsigned long lock_count[2];
29 unsigned long unlock_count[2];
30};
31
32struct rcu_batch {
33 struct rcu_head *head, **tail;
34};
35
36#define RCU_BATCH_INIT(name) { NULL, &(name.head) }
37
38struct srcu_struct {
39 unsigned long completed;
40 struct srcu_array __percpu *per_cpu_ref;
41 spinlock_t queue_lock; /* protect ->batch_queue, ->running */
42 bool running;
43 /* callbacks just queued */
44 struct rcu_batch batch_queue;
45 /* callbacks try to do the first check_zero */
46 struct rcu_batch batch_check0;
47 /* callbacks done with the first check_zero and the flip */
48 struct rcu_batch batch_check1;
49 struct rcu_batch batch_done;
50 struct delayed_work work;
51#ifdef CONFIG_DEBUG_LOCK_ALLOC
52 struct lockdep_map dep_map;
53#endif /* #ifdef CONFIG_DEBUG_LOCK_ALLOC */
54};
55
56void process_srcu(struct work_struct *work);
57
58#define __SRCU_STRUCT_INIT(name) \
59 { \
60 .completed = -300, \
61 .per_cpu_ref = &name##_srcu_array, \
62 .queue_lock = __SPIN_LOCK_UNLOCKED(name.queue_lock), \
63 .running = false, \
64 .batch_queue = RCU_BATCH_INIT(name.batch_queue), \
65 .batch_check0 = RCU_BATCH_INIT(name.batch_check0), \
66 .batch_check1 = RCU_BATCH_INIT(name.batch_check1), \
67 .batch_done = RCU_BATCH_INIT(name.batch_done), \
68 .work = __DELAYED_WORK_INITIALIZER(name.work, process_srcu, 0),\
69 __SRCU_DEP_MAP_INIT(name) \
70 }
71
72/*
73 * Define and initialize a srcu struct at build time.
74 * Do -not- call init_srcu_struct() nor cleanup_srcu_struct() on it.
75 *
76 * Note that although DEFINE_STATIC_SRCU() hides the name from other
77 * files, the per-CPU variable rules nevertheless require that the
78 * chosen name be globally unique. These rules also prohibit use of
79 * DEFINE_STATIC_SRCU() within a function. If these rules are too
80 * restrictive, declare the srcu_struct manually. For example, in
81 * each file:
82 *
83 * static struct srcu_struct my_srcu;
84 *
85 * Then, before the first use of each my_srcu, manually initialize it:
86 *
87 * init_srcu_struct(&my_srcu);
88 *
89 * See include/linux/percpu-defs.h for the rules on per-CPU variables.
90 */
91#define __DEFINE_SRCU(name, is_static) \
92 static DEFINE_PER_CPU(struct srcu_array, name##_srcu_array);\
93 is_static struct srcu_struct name = __SRCU_STRUCT_INIT(name)
94#define DEFINE_SRCU(name) __DEFINE_SRCU(name, /* not static */)
95#define DEFINE_STATIC_SRCU(name) __DEFINE_SRCU(name, static)
96
97void synchronize_srcu_expedited(struct srcu_struct *sp);
98void srcu_barrier(struct srcu_struct *sp);
99
100#endif
diff --git a/init/Kconfig b/init/Kconfig
index 6f257d51f582..2aa14ff40e88 100644
--- a/init/Kconfig
+++ b/init/Kconfig
@@ -526,32 +526,15 @@ config SRCU
526 permits arbitrary sleeping or blocking within RCU read-side critical 526 permits arbitrary sleeping or blocking within RCU read-side critical
527 sections. 527 sections.
528 528
529config CLASSIC_SRCU
530 bool "Use v4.11 classic SRCU implementation"
531 default n
532 depends on RCU_EXPERT && SRCU
533 help
534 This option selects the traditional well-tested classic SRCU
535 implementation from v4.11, as might be desired for enterprise
536 Linux distributions. Without this option, the shiny new
537 Tiny SRCU and Tree SRCU implementations are used instead.
538 At some point, it is hoped that Tiny SRCU and Tree SRCU
539 will accumulate enough test time and confidence to allow
540 Classic SRCU to be dropped entirely.
541
542 Say Y if you need a rock-solid SRCU.
543
544 Say N if you would like help test Tree SRCU.
545
546config TINY_SRCU 529config TINY_SRCU
547 bool 530 bool
548 default y if SRCU && TINY_RCU && !CLASSIC_SRCU 531 default y if SRCU && TINY_RCU
549 help 532 help
550 This option selects the single-CPU non-preemptible version of SRCU. 533 This option selects the single-CPU non-preemptible version of SRCU.
551 534
552config TREE_SRCU 535config TREE_SRCU
553 bool 536 bool
554 default y if SRCU && !TINY_RCU && !CLASSIC_SRCU 537 default y if SRCU && !TINY_RCU
555 help 538 help
556 This option selects the full-fledged version of SRCU. 539 This option selects the full-fledged version of SRCU.
557 540
diff --git a/kernel/rcu/Makefile b/kernel/rcu/Makefile
index 23803c7d5180..3945337c8ce4 100644
--- a/kernel/rcu/Makefile
+++ b/kernel/rcu/Makefile
@@ -3,7 +3,6 @@
3KCOV_INSTRUMENT := n 3KCOV_INSTRUMENT := n
4 4
5obj-y += update.o sync.o 5obj-y += update.o sync.o
6obj-$(CONFIG_CLASSIC_SRCU) += srcu.o
7obj-$(CONFIG_TREE_SRCU) += srcutree.o 6obj-$(CONFIG_TREE_SRCU) += srcutree.o
8obj-$(CONFIG_TINY_SRCU) += srcutiny.o 7obj-$(CONFIG_TINY_SRCU) += srcutiny.o
9obj-$(CONFIG_RCU_TORTURE_TEST) += rcutorture.o 8obj-$(CONFIG_RCU_TORTURE_TEST) += rcutorture.o
diff --git a/kernel/rcu/rcu.h b/kernel/rcu/rcu.h
index 27f871c88e0a..d06c42deee0b 100644
--- a/kernel/rcu/rcu.h
+++ b/kernel/rcu/rcu.h
@@ -457,22 +457,6 @@ void srcutorture_get_gp_data(enum rcutorture_type test_type,
457 struct srcu_struct *sp, int *flags, 457 struct srcu_struct *sp, int *flags,
458 unsigned long *gpnum, unsigned long *completed); 458 unsigned long *gpnum, unsigned long *completed);
459 459
460#elif defined(CONFIG_CLASSIC_SRCU)
461
462static inline void srcutorture_get_gp_data(enum rcutorture_type test_type,
463 struct srcu_struct *sp, int *flags,
464 unsigned long *gpnum,
465 unsigned long *completed)
466{
467 if (test_type != SRCU_FLAVOR)
468 return;
469 *flags = 0;
470 *completed = sp->completed;
471 *gpnum = *completed;
472 if (sp->batch_queue.head || sp->batch_check0.head || sp->batch_check1.head)
473 (*gpnum)++;
474}
475
476#endif 460#endif
477 461
478#ifdef CONFIG_TINY_RCU 462#ifdef CONFIG_TINY_RCU
diff --git a/kernel/rcu/rcutorture.c b/kernel/rcu/rcutorture.c
index 03cdf79e73d4..b8f7f8ce8575 100644
--- a/kernel/rcu/rcutorture.c
+++ b/kernel/rcu/rcutorture.c
@@ -564,31 +564,19 @@ static void srcu_torture_stats(void)
564 int __maybe_unused cpu; 564 int __maybe_unused cpu;
565 int idx; 565 int idx;
566 566
567#if defined(CONFIG_TREE_SRCU) || defined(CONFIG_CLASSIC_SRCU)
568#ifdef CONFIG_TREE_SRCU 567#ifdef CONFIG_TREE_SRCU
569 idx = srcu_ctlp->srcu_idx & 0x1; 568 idx = srcu_ctlp->srcu_idx & 0x1;
570#else /* #ifdef CONFIG_TREE_SRCU */
571 idx = srcu_ctlp->completed & 0x1;
572#endif /* #else #ifdef CONFIG_TREE_SRCU */
573 pr_alert("%s%s Tree SRCU per-CPU(idx=%d):", 569 pr_alert("%s%s Tree SRCU per-CPU(idx=%d):",
574 torture_type, TORTURE_FLAG, idx); 570 torture_type, TORTURE_FLAG, idx);
575 for_each_possible_cpu(cpu) { 571 for_each_possible_cpu(cpu) {
576 unsigned long l0, l1; 572 unsigned long l0, l1;
577 unsigned long u0, u1; 573 unsigned long u0, u1;
578 long c0, c1; 574 long c0, c1;
579#ifdef CONFIG_TREE_SRCU
580 struct srcu_data *counts; 575 struct srcu_data *counts;
581 576
582 counts = per_cpu_ptr(srcu_ctlp->sda, cpu); 577 counts = per_cpu_ptr(srcu_ctlp->sda, cpu);
583 u0 = counts->srcu_unlock_count[!idx]; 578 u0 = counts->srcu_unlock_count[!idx];
584 u1 = counts->srcu_unlock_count[idx]; 579 u1 = counts->srcu_unlock_count[idx];
585#else /* #ifdef CONFIG_TREE_SRCU */
586 struct srcu_array *counts;
587
588 counts = per_cpu_ptr(srcu_ctlp->per_cpu_ref, cpu);
589 u0 = counts->unlock_count[!idx];
590 u1 = counts->unlock_count[idx];
591#endif /* #else #ifdef CONFIG_TREE_SRCU */
592 580
593 /* 581 /*
594 * Make sure that a lock is always counted if the corresponding 582 * Make sure that a lock is always counted if the corresponding
@@ -596,13 +584,8 @@ static void srcu_torture_stats(void)
596 */ 584 */
597 smp_rmb(); 585 smp_rmb();
598 586
599#ifdef CONFIG_TREE_SRCU
600 l0 = counts->srcu_lock_count[!idx]; 587 l0 = counts->srcu_lock_count[!idx];
601 l1 = counts->srcu_lock_count[idx]; 588 l1 = counts->srcu_lock_count[idx];
602#else /* #ifdef CONFIG_TREE_SRCU */
603 l0 = counts->lock_count[!idx];
604 l1 = counts->lock_count[idx];
605#endif /* #else #ifdef CONFIG_TREE_SRCU */
606 589
607 c0 = l0 - u0; 590 c0 = l0 - u0;
608 c1 = l1 - u1; 591 c1 = l1 - u1;
diff --git a/kernel/rcu/srcu.c b/kernel/rcu/srcu.c
deleted file mode 100644
index 4e3f558409a0..000000000000
--- a/kernel/rcu/srcu.c
+++ /dev/null
@@ -1,668 +0,0 @@
1/*
2 * Sleepable Read-Copy Update mechanism for mutual exclusion.
3 *
4 * This program is free software; you can redistribute it and/or modify
5 * it under the terms of the GNU General Public License as published by
6 * the Free Software Foundation; either version 2 of the License, or
7 * (at your option) any later version.
8 *
9 * This program is distributed in the hope that it will be useful,
10 * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 * GNU General Public License for more details.
13 *
14 * You should have received a copy of the GNU General Public License
15 * along with this program; if not, you can access it online at
16 * http://www.gnu.org/licenses/gpl-2.0.html.
17 *
18 * Copyright (C) IBM Corporation, 2006
19 * Copyright (C) Fujitsu, 2012
20 *
21 * Author: Paul McKenney <paulmck@us.ibm.com>
22 * Lai Jiangshan <laijs@cn.fujitsu.com>
23 *
24 * For detailed explanation of Read-Copy Update mechanism see -
25 * Documentation/RCU/ *.txt
26 *
27 */
28
29#include <linux/export.h>
30#include <linux/mutex.h>
31#include <linux/percpu.h>
32#include <linux/preempt.h>
33#include <linux/rcupdate_wait.h>
34#include <linux/sched.h>
35#include <linux/smp.h>
36#include <linux/delay.h>
37#include <linux/srcu.h>
38
39#include "rcu.h"
40
41/*
42 * Initialize an rcu_batch structure to empty.
43 */
44static inline void rcu_batch_init(struct rcu_batch *b)
45{
46 b->head = NULL;
47 b->tail = &b->head;
48}
49
50/*
51 * Enqueue a callback onto the tail of the specified rcu_batch structure.
52 */
53static inline void rcu_batch_queue(struct rcu_batch *b, struct rcu_head *head)
54{
55 *b->tail = head;
56 b->tail = &head->next;
57}
58
59/*
60 * Is the specified rcu_batch structure empty?
61 */
62static inline bool rcu_batch_empty(struct rcu_batch *b)
63{
64 return b->tail == &b->head;
65}
66
67/*
68 * Remove the callback at the head of the specified rcu_batch structure
69 * and return a pointer to it, or return NULL if the structure is empty.
70 */
71static inline struct rcu_head *rcu_batch_dequeue(struct rcu_batch *b)
72{
73 struct rcu_head *head;
74
75 if (rcu_batch_empty(b))
76 return NULL;
77
78 head = b->head;
79 b->head = head->next;
80 if (b->tail == &head->next)
81 rcu_batch_init(b);
82
83 return head;
84}
85
86/*
87 * Move all callbacks from the rcu_batch structure specified by "from" to
88 * the structure specified by "to".
89 */
90static inline void rcu_batch_move(struct rcu_batch *to, struct rcu_batch *from)
91{
92 if (!rcu_batch_empty(from)) {
93 *to->tail = from->head;
94 to->tail = from->tail;
95 rcu_batch_init(from);
96 }
97}
98
99static int init_srcu_struct_fields(struct srcu_struct *sp)
100{
101 sp->completed = 0;
102 spin_lock_init(&sp->queue_lock);
103 sp->running = false;
104 rcu_batch_init(&sp->batch_queue);
105 rcu_batch_init(&sp->batch_check0);
106 rcu_batch_init(&sp->batch_check1);
107 rcu_batch_init(&sp->batch_done);
108 INIT_DELAYED_WORK(&sp->work, process_srcu);
109 sp->per_cpu_ref = alloc_percpu(struct srcu_array);
110 return sp->per_cpu_ref ? 0 : -ENOMEM;
111}
112
113#ifdef CONFIG_DEBUG_LOCK_ALLOC
114
115int __init_srcu_struct(struct srcu_struct *sp, const char *name,
116 struct lock_class_key *key)
117{
118 /* Don't re-initialize a lock while it is held. */
119 debug_check_no_locks_freed((void *)sp, sizeof(*sp));
120 lockdep_init_map(&sp->dep_map, name, key, 0);
121 return init_srcu_struct_fields(sp);
122}
123EXPORT_SYMBOL_GPL(__init_srcu_struct);
124
125#else /* #ifdef CONFIG_DEBUG_LOCK_ALLOC */
126
127/**
128 * init_srcu_struct - initialize a sleep-RCU structure
129 * @sp: structure to initialize.
130 *
131 * Must invoke this on a given srcu_struct before passing that srcu_struct
132 * to any other function. Each srcu_struct represents a separate domain
133 * of SRCU protection.
134 */
135int init_srcu_struct(struct srcu_struct *sp)
136{
137 return init_srcu_struct_fields(sp);
138}
139EXPORT_SYMBOL_GPL(init_srcu_struct);
140
141#endif /* #else #ifdef CONFIG_DEBUG_LOCK_ALLOC */
142
143/*
144 * Returns approximate total of the readers' ->lock_count[] values for the
145 * rank of per-CPU counters specified by idx.
146 */
147static unsigned long srcu_readers_lock_idx(struct srcu_struct *sp, int idx)
148{
149 int cpu;
150 unsigned long sum = 0;
151
152 for_each_possible_cpu(cpu) {
153 struct srcu_array *cpuc = per_cpu_ptr(sp->per_cpu_ref, cpu);
154
155 sum += READ_ONCE(cpuc->lock_count[idx]);
156 }
157 return sum;
158}
159
160/*
161 * Returns approximate total of the readers' ->unlock_count[] values for the
162 * rank of per-CPU counters specified by idx.
163 */
164static unsigned long srcu_readers_unlock_idx(struct srcu_struct *sp, int idx)
165{
166 int cpu;
167 unsigned long sum = 0;
168
169 for_each_possible_cpu(cpu) {
170 struct srcu_array *cpuc = per_cpu_ptr(sp->per_cpu_ref, cpu);
171
172 sum += READ_ONCE(cpuc->unlock_count[idx]);
173 }
174 return sum;
175}
176
177/*
178 * Return true if the number of pre-existing readers is determined to
179 * be zero.
180 */
181static bool srcu_readers_active_idx_check(struct srcu_struct *sp, int idx)
182{
183 unsigned long unlocks;
184
185 unlocks = srcu_readers_unlock_idx(sp, idx);
186
187 /*
188 * Make sure that a lock is always counted if the corresponding unlock
189 * is counted. Needs to be a smp_mb() as the read side may contain a
190 * read from a variable that is written to before the synchronize_srcu()
191 * in the write side. In this case smp_mb()s A and B act like the store
192 * buffering pattern.
193 *
194 * This smp_mb() also pairs with smp_mb() C to prevent accesses after the
195 * synchronize_srcu() from being executed before the grace period ends.
196 */
197 smp_mb(); /* A */
198
199 /*
200 * If the locks are the same as the unlocks, then there must have
201 * been no readers on this index at some time in between. This does not
202 * mean that there are no more readers, as one could have read the
203 * current index but not have incremented the lock counter yet.
204 *
205 * Possible bug: There is no guarantee that there haven't been ULONG_MAX
206 * increments of ->lock_count[] since the unlocks were counted, meaning
207 * that this could return true even if there are still active readers.
208 * Since there are no memory barriers around srcu_flip(), the CPU is not
209 * required to increment ->completed before running
210 * srcu_readers_unlock_idx(), which means that there could be an
211 * arbitrarily large number of critical sections that execute after
212 * srcu_readers_unlock_idx() but use the old value of ->completed.
213 */
214 return srcu_readers_lock_idx(sp, idx) == unlocks;
215}
216
217/**
218 * srcu_readers_active - returns true if there are readers. and false
219 * otherwise
220 * @sp: which srcu_struct to count active readers (holding srcu_read_lock).
221 *
222 * Note that this is not an atomic primitive, and can therefore suffer
223 * severe errors when invoked on an active srcu_struct. That said, it
224 * can be useful as an error check at cleanup time.
225 */
226static bool srcu_readers_active(struct srcu_struct *sp)
227{
228 int cpu;
229 unsigned long sum = 0;
230
231 for_each_possible_cpu(cpu) {
232 struct srcu_array *cpuc = per_cpu_ptr(sp->per_cpu_ref, cpu);
233
234 sum += READ_ONCE(cpuc->lock_count[0]);
235 sum += READ_ONCE(cpuc->lock_count[1]);
236 sum -= READ_ONCE(cpuc->unlock_count[0]);
237 sum -= READ_ONCE(cpuc->unlock_count[1]);
238 }
239 return sum;
240}
241
242/**
243 * cleanup_srcu_struct - deconstruct a sleep-RCU structure
244 * @sp: structure to clean up.
245 *
246 * Must invoke this only after you are finished using a given srcu_struct
247 * that was initialized via init_srcu_struct(). This code does some
248 * probabalistic checking, spotting late uses of srcu_read_lock(),
249 * synchronize_srcu(), synchronize_srcu_expedited(), and call_srcu().
250 * If any such late uses are detected, the per-CPU memory associated with
251 * the srcu_struct is simply leaked and WARN_ON() is invoked. If the
252 * caller frees the srcu_struct itself, a use-after-free crash will likely
253 * ensue, but at least there will be a warning printed.
254 */
255void cleanup_srcu_struct(struct srcu_struct *sp)
256{
257 if (WARN_ON(srcu_readers_active(sp)))
258 return; /* Leakage unless caller handles error. */
259 free_percpu(sp->per_cpu_ref);
260 sp->per_cpu_ref = NULL;
261}
262EXPORT_SYMBOL_GPL(cleanup_srcu_struct);
263
264/*
265 * Counts the new reader in the appropriate per-CPU element of the
266 * srcu_struct.
267 * Returns an index that must be passed to the matching srcu_read_unlock().
268 */
269int __srcu_read_lock(struct srcu_struct *sp)
270{
271 int idx;
272
273 idx = READ_ONCE(sp->completed) & 0x1;
274 this_cpu_inc(sp->per_cpu_ref->lock_count[idx]);
275 smp_mb(); /* B */ /* Avoid leaking the critical section. */
276 return idx;
277}
278EXPORT_SYMBOL_GPL(__srcu_read_lock);
279
280/*
281 * Removes the count for the old reader from the appropriate per-CPU
282 * element of the srcu_struct. Note that this may well be a different
283 * CPU than that which was incremented by the corresponding srcu_read_lock().
284 */
285void __srcu_read_unlock(struct srcu_struct *sp, int idx)
286{
287 smp_mb(); /* C */ /* Avoid leaking the critical section. */
288 this_cpu_inc(sp->per_cpu_ref->unlock_count[idx]);
289}
290EXPORT_SYMBOL_GPL(__srcu_read_unlock);
291
292/*
293 * We use an adaptive strategy for synchronize_srcu() and especially for
294 * synchronize_srcu_expedited(). We spin for a fixed time period
295 * (defined below) to allow SRCU readers to exit their read-side critical
296 * sections. If there are still some readers after 10 microseconds,
297 * we repeatedly block for 1-millisecond time periods. This approach
298 * has done well in testing, so there is no need for a config parameter.
299 */
300#define SRCU_RETRY_CHECK_DELAY 5
301#define SYNCHRONIZE_SRCU_TRYCOUNT 2
302#define SYNCHRONIZE_SRCU_EXP_TRYCOUNT 12
303
304/*
305 * @@@ Wait until all pre-existing readers complete. Such readers
306 * will have used the index specified by "idx".
307 * the caller should ensures the ->completed is not changed while checking
308 * and idx = (->completed & 1) ^ 1
309 */
310static bool try_check_zero(struct srcu_struct *sp, int idx, int trycount)
311{
312 for (;;) {
313 if (srcu_readers_active_idx_check(sp, idx))
314 return true;
315 if (--trycount <= 0)
316 return false;
317 udelay(SRCU_RETRY_CHECK_DELAY);
318 }
319}
320
321/*
322 * Increment the ->completed counter so that future SRCU readers will
323 * use the other rank of the ->(un)lock_count[] arrays. This allows
324 * us to wait for pre-existing readers in a starvation-free manner.
325 */
326static void srcu_flip(struct srcu_struct *sp)
327{
328 WRITE_ONCE(sp->completed, sp->completed + 1);
329
330 /*
331 * Ensure that if the updater misses an __srcu_read_unlock()
332 * increment, that task's next __srcu_read_lock() will see the
333 * above counter update. Note that both this memory barrier
334 * and the one in srcu_readers_active_idx_check() provide the
335 * guarantee for __srcu_read_lock().
336 */
337 smp_mb(); /* D */ /* Pairs with C. */
338}
339
340/*
341 * Enqueue an SRCU callback on the specified srcu_struct structure,
342 * initiating grace-period processing if it is not already running.
343 *
344 * Note that all CPUs must agree that the grace period extended beyond
345 * all pre-existing SRCU read-side critical section. On systems with
346 * more than one CPU, this means that when "func()" is invoked, each CPU
347 * is guaranteed to have executed a full memory barrier since the end of
348 * its last corresponding SRCU read-side critical section whose beginning
349 * preceded the call to call_rcu(). It also means that each CPU executing
350 * an SRCU read-side critical section that continues beyond the start of
351 * "func()" must have executed a memory barrier after the call_rcu()
352 * but before the beginning of that SRCU read-side critical section.
353 * Note that these guarantees include CPUs that are offline, idle, or
354 * executing in user mode, as well as CPUs that are executing in the kernel.
355 *
356 * Furthermore, if CPU A invoked call_rcu() and CPU B invoked the
357 * resulting SRCU callback function "func()", then both CPU A and CPU
358 * B are guaranteed to execute a full memory barrier during the time
359 * interval between the call to call_rcu() and the invocation of "func()".
360 * This guarantee applies even if CPU A and CPU B are the same CPU (but
361 * again only if the system has more than one CPU).
362 *
363 * Of course, these guarantees apply only for invocations of call_srcu(),
364 * srcu_read_lock(), and srcu_read_unlock() that are all passed the same
365 * srcu_struct structure.
366 */
367void call_srcu(struct srcu_struct *sp, struct rcu_head *head,
368 rcu_callback_t func)
369{
370 unsigned long flags;
371
372 head->next = NULL;
373 head->func = func;
374 spin_lock_irqsave(&sp->queue_lock, flags);
375 smp_mb__after_unlock_lock(); /* Caller's prior accesses before GP. */
376 rcu_batch_queue(&sp->batch_queue, head);
377 if (!sp->running) {
378 sp->running = true;
379 queue_delayed_work(system_power_efficient_wq, &sp->work, 0);
380 }
381 spin_unlock_irqrestore(&sp->queue_lock, flags);
382}
383EXPORT_SYMBOL_GPL(call_srcu);
384
385static void srcu_advance_batches(struct srcu_struct *sp, int trycount);
386static void srcu_reschedule(struct srcu_struct *sp);
387
388/*
389 * Helper function for synchronize_srcu() and synchronize_srcu_expedited().
390 */
391static void __synchronize_srcu(struct srcu_struct *sp, int trycount)
392{
393 struct rcu_synchronize rcu;
394 struct rcu_head *head = &rcu.head;
395 bool done = false;
396
397 RCU_LOCKDEP_WARN(lock_is_held(&sp->dep_map) ||
398 lock_is_held(&rcu_bh_lock_map) ||
399 lock_is_held(&rcu_lock_map) ||
400 lock_is_held(&rcu_sched_lock_map),
401 "Illegal synchronize_srcu() in same-type SRCU (or in RCU) read-side critical section");
402
403 might_sleep();
404 init_completion(&rcu.completion);
405
406 head->next = NULL;
407 head->func = wakeme_after_rcu;
408 spin_lock_irq(&sp->queue_lock);
409 smp_mb__after_unlock_lock(); /* Caller's prior accesses before GP. */
410 if (!sp->running) {
411 /* steal the processing owner */
412 sp->running = true;
413 rcu_batch_queue(&sp->batch_check0, head);
414 spin_unlock_irq(&sp->queue_lock);
415
416 srcu_advance_batches(sp, trycount);
417 if (!rcu_batch_empty(&sp->batch_done)) {
418 BUG_ON(sp->batch_done.head != head);
419 rcu_batch_dequeue(&sp->batch_done);
420 done = true;
421 }
422 /* give the processing owner to work_struct */
423 srcu_reschedule(sp);
424 } else {
425 rcu_batch_queue(&sp->batch_queue, head);
426 spin_unlock_irq(&sp->queue_lock);
427 }
428
429 if (!done) {
430 wait_for_completion(&rcu.completion);
431 smp_mb(); /* Caller's later accesses after GP. */
432 }
433
434}
435
436/**
437 * synchronize_srcu - wait for prior SRCU read-side critical-section completion
438 * @sp: srcu_struct with which to synchronize.
439 *
440 * Wait for the count to drain to zero of both indexes. To avoid the
441 * possible starvation of synchronize_srcu(), it waits for the count of
442 * the index=((->completed & 1) ^ 1) to drain to zero at first,
443 * and then flip the completed and wait for the count of the other index.
444 *
445 * Can block; must be called from process context.
446 *
447 * Note that it is illegal to call synchronize_srcu() from the corresponding
448 * SRCU read-side critical section; doing so will result in deadlock.
449 * However, it is perfectly legal to call synchronize_srcu() on one
450 * srcu_struct from some other srcu_struct's read-side critical section,
451 * as long as the resulting graph of srcu_structs is acyclic.
452 *
453 * There are memory-ordering constraints implied by synchronize_srcu().
454 * On systems with more than one CPU, when synchronize_srcu() returns,
455 * each CPU is guaranteed to have executed a full memory barrier since
456 * the end of its last corresponding SRCU-sched read-side critical section
457 * whose beginning preceded the call to synchronize_srcu(). In addition,
458 * each CPU having an SRCU read-side critical section that extends beyond
459 * the return from synchronize_srcu() is guaranteed to have executed a
460 * full memory barrier after the beginning of synchronize_srcu() and before
461 * the beginning of that SRCU read-side critical section. Note that these
462 * guarantees include CPUs that are offline, idle, or executing in user mode,
463 * as well as CPUs that are executing in the kernel.
464 *
465 * Furthermore, if CPU A invoked synchronize_srcu(), which returned
466 * to its caller on CPU B, then both CPU A and CPU B are guaranteed
467 * to have executed a full memory barrier during the execution of
468 * synchronize_srcu(). This guarantee applies even if CPU A and CPU B
469 * are the same CPU, but again only if the system has more than one CPU.
470 *
471 * Of course, these memory-ordering guarantees apply only when
472 * synchronize_srcu(), srcu_read_lock(), and srcu_read_unlock() are
473 * passed the same srcu_struct structure.
474 */
475void synchronize_srcu(struct srcu_struct *sp)
476{
477 __synchronize_srcu(sp, (rcu_gp_is_expedited() && !rcu_gp_is_normal())
478 ? SYNCHRONIZE_SRCU_EXP_TRYCOUNT
479 : SYNCHRONIZE_SRCU_TRYCOUNT);
480}
481EXPORT_SYMBOL_GPL(synchronize_srcu);
482
483/**
484 * synchronize_srcu_expedited - Brute-force SRCU grace period
485 * @sp: srcu_struct with which to synchronize.
486 *
487 * Wait for an SRCU grace period to elapse, but be more aggressive about
488 * spinning rather than blocking when waiting.
489 *
490 * Note that synchronize_srcu_expedited() has the same deadlock and
491 * memory-ordering properties as does synchronize_srcu().
492 */
493void synchronize_srcu_expedited(struct srcu_struct *sp)
494{
495 __synchronize_srcu(sp, SYNCHRONIZE_SRCU_EXP_TRYCOUNT);
496}
497EXPORT_SYMBOL_GPL(synchronize_srcu_expedited);
498
499/**
500 * srcu_barrier - Wait until all in-flight call_srcu() callbacks complete.
501 * @sp: srcu_struct on which to wait for in-flight callbacks.
502 */
503void srcu_barrier(struct srcu_struct *sp)
504{
505 synchronize_srcu(sp);
506}
507EXPORT_SYMBOL_GPL(srcu_barrier);
508
509/**
510 * srcu_batches_completed - return batches completed.
511 * @sp: srcu_struct on which to report batch completion.
512 *
513 * Report the number of batches, correlated with, but not necessarily
514 * precisely the same as, the number of grace periods that have elapsed.
515 */
516unsigned long srcu_batches_completed(struct srcu_struct *sp)
517{
518 return sp->completed;
519}
520EXPORT_SYMBOL_GPL(srcu_batches_completed);
521
522#define SRCU_CALLBACK_BATCH 10
523#define SRCU_INTERVAL 1
524
525/*
526 * Move any new SRCU callbacks to the first stage of the SRCU grace
527 * period pipeline.
528 */
529static void srcu_collect_new(struct srcu_struct *sp)
530{
531 if (!rcu_batch_empty(&sp->batch_queue)) {
532 spin_lock_irq(&sp->queue_lock);
533 rcu_batch_move(&sp->batch_check0, &sp->batch_queue);
534 spin_unlock_irq(&sp->queue_lock);
535 }
536}
537
538/*
539 * Core SRCU state machine. Advance callbacks from ->batch_check0 to
540 * ->batch_check1 and then to ->batch_done as readers drain.
541 */
542static void srcu_advance_batches(struct srcu_struct *sp, int trycount)
543{
544 int idx = 1 ^ (sp->completed & 1);
545
546 /*
547 * Because readers might be delayed for an extended period after
548 * fetching ->completed for their index, at any point in time there
549 * might well be readers using both idx=0 and idx=1. We therefore
550 * need to wait for readers to clear from both index values before
551 * invoking a callback.
552 */
553
554 if (rcu_batch_empty(&sp->batch_check0) &&
555 rcu_batch_empty(&sp->batch_check1))
556 return; /* no callbacks need to be advanced */
557
558 if (!try_check_zero(sp, idx, trycount))
559 return; /* failed to advance, will try after SRCU_INTERVAL */
560
561 /*
562 * The callbacks in ->batch_check1 have already done with their
563 * first zero check and flip back when they were enqueued on
564 * ->batch_check0 in a previous invocation of srcu_advance_batches().
565 * (Presumably try_check_zero() returned false during that
566 * invocation, leaving the callbacks stranded on ->batch_check1.)
567 * They are therefore ready to invoke, so move them to ->batch_done.
568 */
569 rcu_batch_move(&sp->batch_done, &sp->batch_check1);
570
571 if (rcu_batch_empty(&sp->batch_check0))
572 return; /* no callbacks need to be advanced */
573 srcu_flip(sp);
574
575 /*
576 * The callbacks in ->batch_check0 just finished their
577 * first check zero and flip, so move them to ->batch_check1
578 * for future checking on the other idx.
579 */
580 rcu_batch_move(&sp->batch_check1, &sp->batch_check0);
581
582 /*
583 * SRCU read-side critical sections are normally short, so check
584 * at least twice in quick succession after a flip.
585 */
586 trycount = trycount < 2 ? 2 : trycount;
587 if (!try_check_zero(sp, idx^1, trycount))
588 return; /* failed to advance, will try after SRCU_INTERVAL */
589
590 /*
591 * The callbacks in ->batch_check1 have now waited for all
592 * pre-existing readers using both idx values. They are therefore
593 * ready to invoke, so move them to ->batch_done.
594 */
595 rcu_batch_move(&sp->batch_done, &sp->batch_check1);
596}
597
598/*
599 * Invoke a limited number of SRCU callbacks that have passed through
600 * their grace period. If there are more to do, SRCU will reschedule
601 * the workqueue. Note that needed memory barriers have been executed
602 * in this task's context by srcu_readers_active_idx_check().
603 */
604static void srcu_invoke_callbacks(struct srcu_struct *sp)
605{
606 int i;
607 struct rcu_head *head;
608
609 for (i = 0; i < SRCU_CALLBACK_BATCH; i++) {
610 head = rcu_batch_dequeue(&sp->batch_done);
611 if (!head)
612 break;
613 local_bh_disable();
614 head->func(head);
615 local_bh_enable();
616 }
617}
618
619/*
620 * Finished one round of SRCU grace period. Start another if there are
621 * more SRCU callbacks queued, otherwise put SRCU into not-running state.
622 */
623static void srcu_reschedule(struct srcu_struct *sp)
624{
625 bool pending = true;
626
627 if (rcu_batch_empty(&sp->batch_done) &&
628 rcu_batch_empty(&sp->batch_check1) &&
629 rcu_batch_empty(&sp->batch_check0) &&
630 rcu_batch_empty(&sp->batch_queue)) {
631 spin_lock_irq(&sp->queue_lock);
632 if (rcu_batch_empty(&sp->batch_done) &&
633 rcu_batch_empty(&sp->batch_check1) &&
634 rcu_batch_empty(&sp->batch_check0) &&
635 rcu_batch_empty(&sp->batch_queue)) {
636 sp->running = false;
637 pending = false;
638 }
639 spin_unlock_irq(&sp->queue_lock);
640 }
641
642 if (pending)
643 queue_delayed_work(system_power_efficient_wq,
644 &sp->work, SRCU_INTERVAL);
645}
646
647/*
648 * This is the work-queue function that handles SRCU grace periods.
649 */
650void process_srcu(struct work_struct *work)
651{
652 struct srcu_struct *sp;
653
654 sp = container_of(work, struct srcu_struct, work.work);
655
656 srcu_collect_new(sp);
657 srcu_advance_batches(sp, 1);
658 srcu_invoke_callbacks(sp);
659 srcu_reschedule(sp);
660}
661EXPORT_SYMBOL_GPL(process_srcu);
662
663static int __init srcu_bootup_announce(void)
664{
665 pr_info("Classic SRCU implementation.\n");
666 return 0;
667}
668early_initcall(srcu_bootup_announce);
diff --git a/tools/testing/selftests/rcutorture/configs/rcu/CFLIST b/tools/testing/selftests/rcutorture/configs/rcu/CFLIST
index 0c1da784b8cb..6a0b9f69faad 100644
--- a/tools/testing/selftests/rcutorture/configs/rcu/CFLIST
+++ b/tools/testing/selftests/rcutorture/configs/rcu/CFLIST
@@ -7,7 +7,6 @@ TREE06
7TREE07 7TREE07
8TREE08 8TREE08
9TREE09 9TREE09
10SRCU-C
11SRCU-N 10SRCU-N
12SRCU-P 11SRCU-P
13SRCU-t 12SRCU-t
diff --git a/tools/testing/selftests/rcutorture/configs/rcu/SRCU-C b/tools/testing/selftests/rcutorture/configs/rcu/SRCU-C
deleted file mode 100644
index d4e19c087c21..000000000000
--- a/tools/testing/selftests/rcutorture/configs/rcu/SRCU-C
+++ /dev/null
@@ -1,11 +0,0 @@
1CONFIG_RCU_TRACE=n
2CONFIG_SMP=y
3CONFIG_NR_CPUS=3
4CONFIG_HOTPLUG_CPU=y
5CONFIG_RCU_EXPERT=y
6CONFIG_CLASSIC_SRCU=y
7CONFIG_PREEMPT_NONE=n
8CONFIG_PREEMPT_VOLUNTARY=n
9CONFIG_PREEMPT=y
10CONFIG_DEBUG_LOCK_ALLOC=y
11CONFIG_PROVE_LOCKING=y
diff --git a/tools/testing/selftests/rcutorture/configs/rcuperf/SRCUCLASSIC b/tools/testing/selftests/rcutorture/configs/rcuperf/SRCUCLASSIC
deleted file mode 100644
index a1395af60ef4..000000000000
--- a/tools/testing/selftests/rcutorture/configs/rcuperf/SRCUCLASSIC
+++ /dev/null
@@ -1,16 +0,0 @@
1CONFIG_SMP=y
2CONFIG_HZ_PERIODIC=n
3CONFIG_NO_HZ_IDLE=y
4CONFIG_NO_HZ_FULL=n
5CONFIG_RCU_FAST_NO_HZ=n
6CONFIG_HOTPLUG_CPU=n
7CONFIG_SUSPEND=n
8CONFIG_HIBERNATION=n
9CONFIG_RCU_NOCB_CPU=n
10CONFIG_DEBUG_LOCK_ALLOC=n
11CONFIG_PROVE_LOCKING=n
12CONFIG_RCU_BOOST=n
13CONFIG_DEBUG_OBJECTS_RCU_HEAD=n
14CONFIG_RCU_EXPERT=y
15CONFIG_RCU_TRACE=y
16CONFIG_CLASSIC_SRCU=y
generated by cgit v1.2.2 (git 2.25.0) at 2026-07-08 00:01:57 -0400