aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
-rw-r--r--Documentation/RCU/rcuref.txt74
-rw-r--r--include/linux/rcuref.h220
-rw-r--r--kernel/rcupdate.c14
3 files changed, 308 insertions, 0 deletions
diff --git a/Documentation/RCU/rcuref.txt b/Documentation/RCU/rcuref.txt
new file mode 100644
index 000000000000..a23fee66064d
--- /dev/null
+++ b/Documentation/RCU/rcuref.txt
@@ -0,0 +1,74 @@
1Refcounter framework for elements of lists/arrays protected by
2RCU.
3
4Refcounting on elements of lists which are protected by traditional
5reader/writer spinlocks or semaphores are straight forward as in:
6
71. 2.
8add() search_and_reference()
9{ {
10 alloc_object read_lock(&list_lock);
11 ... search_for_element
12 atomic_set(&el->rc, 1); atomic_inc(&el->rc);
13 write_lock(&list_lock); ...
14 add_element read_unlock(&list_lock);
15 ... ...
16 write_unlock(&list_lock); }
17}
18
193. 4.
20release_referenced() delete()
21{ {
22 ... write_lock(&list_lock);
23 atomic_dec(&el->rc, relfunc) ...
24 ... delete_element
25} write_unlock(&list_lock);
26 ...
27 if (atomic_dec_and_test(&el->rc))
28 kfree(el);
29 ...
30 }
31
32If this list/array is made lock free using rcu as in changing the
33write_lock in add() and delete() to spin_lock and changing read_lock
34in search_and_reference to rcu_read_lock(), the rcuref_get in
35search_and_reference could potentially hold reference to an element which
36has already been deleted from the list/array. rcuref_lf_get_rcu takes
37care of this scenario. search_and_reference should look as;
38
391. 2.
40add() search_and_reference()
41{ {
42 alloc_object rcu_read_lock();
43 ... search_for_element
44 atomic_set(&el->rc, 1); if (rcuref_inc_lf(&el->rc)) {
45 write_lock(&list_lock); rcu_read_unlock();
46 return FAIL;
47 add_element }
48 ... ...
49 write_unlock(&list_lock); rcu_read_unlock();
50} }
513. 4.
52release_referenced() delete()
53{ {
54 ... write_lock(&list_lock);
55 rcuref_dec(&el->rc, relfunc) ...
56 ... delete_element
57} write_unlock(&list_lock);
58 ...
59 if (rcuref_dec_and_test(&el->rc))
60 call_rcu(&el->head, el_free);
61 ...
62 }
63
64Sometimes, reference to the element need to be obtained in the
65update (write) stream. In such cases, rcuref_inc_lf might be an overkill
66since the spinlock serialising list updates are held. rcuref_inc
67is to be used in such cases.
68For arches which do not have cmpxchg rcuref_inc_lf
69api uses a hashed spinlock implementation and the same hashed spinlock
70is acquired in all rcuref_xxx primitives to preserve atomicity.
71Note: Use rcuref_inc api only if you need to use rcuref_inc_lf on the
72refcounter atleast at one place. Mixing rcuref_inc and atomic_xxx api
73might lead to races. rcuref_inc_lf() must be used in lockfree
74RCU critical sections only.
diff --git a/include/linux/rcuref.h b/include/linux/rcuref.h
new file mode 100644
index 000000000000..e1adbba14b67
--- /dev/null
+++ b/include/linux/rcuref.h
@@ -0,0 +1,220 @@
1/*
2 * rcuref.h
3 *
4 * Reference counting for elements of lists/arrays protected by
5 * RCU.
6 *
7 * This program is free software; you can redistribute it and/or modify
8 * it under the terms of the GNU General Public License as published by
9 * the Free Software Foundation; either version 2 of the License, or
10 * (at your option) any later version.
11 *
12 * This program is distributed in the hope that it will be useful,
13 * but WITHOUT ANY WARRANTY; without even the implied warranty of
14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 * GNU General Public License for more details.
16 *
17 * You should have received a copy of the GNU General Public License
18 * along with this program; if not, write to the Free Software
19 * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
20 *
21 * Copyright (C) IBM Corporation, 2005
22 *
23 * Author: Dipankar Sarma <dipankar@in.ibm.com>
24 * Ravikiran Thirumalai <kiran_th@gmail.com>
25 *
26 * See Documentation/RCU/rcuref.txt for detailed user guide.
27 *
28 */
29
30#ifndef _RCUREF_H_
31#define _RCUREF_H_
32
33#ifdef __KERNEL__
34
35#include <linux/types.h>
36#include <linux/interrupt.h>
37#include <linux/spinlock.h>
38#include <asm/atomic.h>
39
40/*
41 * These APIs work on traditional atomic_t counters used in the
42 * kernel for reference counting. Under special circumstances
43 * where a lock-free get() operation races with a put() operation
44 * these APIs can be used. See Documentation/RCU/rcuref.txt.
45 */
46
47#ifdef __HAVE_ARCH_CMPXCHG
48
49/**
50 * rcuref_inc - increment refcount for object.
51 * @rcuref: reference counter in the object in question.
52 *
53 * This should be used only for objects where we use RCU and
54 * use the rcuref_inc_lf() api to acquire a reference
55 * in a lock-free reader-side critical section.
56 */
57static inline void rcuref_inc(atomic_t *rcuref)
58{
59 atomic_inc(rcuref);
60}
61
62/**
63 * rcuref_dec - decrement refcount for object.
64 * @rcuref: reference counter in the object in question.
65 *
66 * This should be used only for objects where we use RCU and
67 * use the rcuref_inc_lf() api to acquire a reference
68 * in a lock-free reader-side critical section.
69 */
70static inline void rcuref_dec(atomic_t *rcuref)
71{
72 atomic_dec(rcuref);
73}
74
75/**
76 * rcuref_dec_and_test - decrement refcount for object and test
77 * @rcuref: reference counter in the object.
78 * @release: pointer to the function that will clean up the object
79 * when the last reference to the object is released.
80 * This pointer is required.
81 *
82 * Decrement the refcount, and if 0, return 1. Else return 0.
83 *
84 * This should be used only for objects where we use RCU and
85 * use the rcuref_inc_lf() api to acquire a reference
86 * in a lock-free reader-side critical section.
87 */
88static inline int rcuref_dec_and_test(atomic_t *rcuref)
89{
90 return atomic_dec_and_test(rcuref);
91}
92
93/*
94 * cmpxchg is needed on UP too, if deletions to the list/array can happen
95 * in interrupt context.
96 */
97
98/**
99 * rcuref_inc_lf - Take reference to an object in a read-side
100 * critical section protected by RCU.
101 * @rcuref: reference counter in the object in question.
102 *
103 * Try and increment the refcount by 1. The increment might fail if
104 * the reference counter has been through a 1 to 0 transition and
105 * is no longer part of the lock-free list.
106 * Returns non-zero on successful increment and zero otherwise.
107 */
108static inline int rcuref_inc_lf(atomic_t *rcuref)
109{
110 int c, old;
111 c = atomic_read(rcuref);
112 while (c && (old = cmpxchg(&rcuref->counter, c, c + 1)) != c)
113 c = old;
114 return c;
115}
116
117#else /* !__HAVE_ARCH_CMPXCHG */
118
119extern spinlock_t __rcuref_hash[];
120
121/*
122 * Use a hash table of locks to protect the reference count
123 * since cmpxchg is not available in this arch.
124 */
125#ifdef CONFIG_SMP
126#define RCUREF_HASH_SIZE 4
127#define RCUREF_HASH(k) \
128 (&__rcuref_hash[(((unsigned long)k)>>8) & (RCUREF_HASH_SIZE-1)])
129#else
130#define RCUREF_HASH_SIZE 1
131#define RCUREF_HASH(k) &__rcuref_hash[0]
132#endif /* CONFIG_SMP */
133
134/**
135 * rcuref_inc - increment refcount for object.
136 * @rcuref: reference counter in the object in question.
137 *
138 * This should be used only for objects where we use RCU and
139 * use the rcuref_inc_lf() api to acquire a reference in a lock-free
140 * reader-side critical section.
141 */
142static inline void rcuref_inc(atomic_t *rcuref)
143{
144 unsigned long flags;
145 spin_lock_irqsave(RCUREF_HASH(rcuref), flags);
146 rcuref->counter += 1;
147 spin_unlock_irqrestore(RCUREF_HASH(rcuref), flags);
148}
149
150/**
151 * rcuref_dec - decrement refcount for object.
152 * @rcuref: reference counter in the object in question.
153 *
154 * This should be used only for objects where we use RCU and
155 * use the rcuref_inc_lf() api to acquire a reference in a lock-free
156 * reader-side critical section.
157 */
158static inline void rcuref_dec(atomic_t *rcuref)
159{
160 unsigned long flags;
161 spin_lock_irqsave(RCUREF_HASH(rcuref), flags);
162 rcuref->counter -= 1;
163 spin_unlock_irqrestore(RCUREF_HASH(rcuref), flags);
164}
165
166/**
167 * rcuref_dec_and_test - decrement refcount for object and test
168 * @rcuref: reference counter in the object.
169 * @release: pointer to the function that will clean up the object
170 * when the last reference to the object is released.
171 * This pointer is required.
172 *
173 * Decrement the refcount, and if 0, return 1. Else return 0.
174 *
175 * This should be used only for objects where we use RCU and
176 * use the rcuref_inc_lf() api to acquire a reference in a lock-free
177 * reader-side critical section.
178 */
179static inline int rcuref_dec_and_test(atomic_t *rcuref)
180{
181 unsigned long flags;
182 spin_lock_irqsave(RCUREF_HASH(rcuref), flags);
183 rcuref->counter--;
184 if (!rcuref->counter) {
185 spin_unlock_irqrestore(RCUREF_HASH(rcuref), flags);
186 return 1;
187 } else {
188 spin_unlock_irqrestore(RCUREF_HASH(rcuref), flags);
189 return 0;
190 }
191}
192
193/**
194 * rcuref_inc_lf - Take reference to an object of a lock-free collection
195 * by traversing a lock-free list/array.
196 * @rcuref: reference counter in the object in question.
197 *
198 * Try and increment the refcount by 1. The increment might fail if
199 * the reference counter has been through a 1 to 0 transition and
200 * object is no longer part of the lock-free list.
201 * Returns non-zero on successful increment and zero otherwise.
202 */
203static inline int rcuref_inc_lf(atomic_t *rcuref)
204{
205 int ret;
206 unsigned long flags;
207 spin_lock_irqsave(RCUREF_HASH(rcuref), flags);
208 if (rcuref->counter)
209 ret = rcuref->counter++;
210 else
211 ret = 0;
212 spin_unlock_irqrestore(RCUREF_HASH(rcuref), flags);
213 return ret;
214}
215
216
217#endif /* !__HAVE_ARCH_CMPXCHG */
218
219#endif /* __KERNEL__ */
220#endif /* _RCUREF_H_ */
diff --git a/kernel/rcupdate.c b/kernel/rcupdate.c
index f436993bd590..bef3b6901b76 100644
--- a/kernel/rcupdate.c
+++ b/kernel/rcupdate.c
@@ -45,6 +45,7 @@
45#include <linux/percpu.h> 45#include <linux/percpu.h>
46#include <linux/notifier.h> 46#include <linux/notifier.h>
47#include <linux/rcupdate.h> 47#include <linux/rcupdate.h>
48#include <linux/rcuref.h>
48#include <linux/cpu.h> 49#include <linux/cpu.h>
49 50
50/* Definition for rcupdate control block. */ 51/* Definition for rcupdate control block. */
@@ -72,6 +73,19 @@ DEFINE_PER_CPU(struct rcu_data, rcu_bh_data) = { 0L };
72static DEFINE_PER_CPU(struct tasklet_struct, rcu_tasklet) = {NULL}; 73static DEFINE_PER_CPU(struct tasklet_struct, rcu_tasklet) = {NULL};
73static int maxbatch = 10; 74static int maxbatch = 10;
74 75
76#ifndef __HAVE_ARCH_CMPXCHG
77/*
78 * We use an array of spinlocks for the rcurefs -- similar to ones in sparc
79 * 32 bit atomic_t implementations, and a hash function similar to that
80 * for our refcounting needs.
81 * Can't help multiprocessors which donot have cmpxchg :(
82 */
83
84spinlock_t __rcuref_hash[RCUREF_HASH_SIZE] = {
85 [0 ... (RCUREF_HASH_SIZE-1)] = SPIN_LOCK_UNLOCKED
86};
87#endif
88
75/** 89/**
76 * call_rcu - Queue an RCU callback for invocation after a grace period. 90 * call_rcu - Queue an RCU callback for invocation after a grace period.
77 * @head: structure to be used for queueing the RCU updates. 91 * @head: structure to be used for queueing the RCU updates.