aboutsummaryrefslogtreecommitdiffstats
path: root/kernel/sched/wait_bit.c
diff options
context:
space:
mode:
Diffstat (limited to 'kernel/sched/wait_bit.c')
-rw-r--r--kernel/sched/wait_bit.c263
1 files changed, 263 insertions, 0 deletions
diff --git a/kernel/sched/wait_bit.c b/kernel/sched/wait_bit.c
new file mode 100644
index 000000000000..463bac84dfd1
--- /dev/null
+++ b/kernel/sched/wait_bit.c
@@ -0,0 +1,263 @@
1/*
2 * The implementation of the wait_bit*() and related waiting APIs:
3 */
4#include <linux/wait_bit.h>
5#include <linux/sched/signal.h>
6#include <linux/sched/debug.h>
7
8int wake_bit_function(struct wait_queue_entry *wq_entry, unsigned mode, int sync, void *arg)
9{
10 struct wait_bit_key *key = arg;
11 struct wait_bit_queue_entry *wait_bit = container_of(wq_entry, struct wait_bit_queue_entry, wq_entry);
12
13 if (wait_bit->key.flags != key->flags ||
14 wait_bit->key.bit_nr != key->bit_nr ||
15 test_bit(key->bit_nr, key->flags))
16 return 0;
17 else
18 return autoremove_wake_function(wq_entry, mode, sync, key);
19}
20EXPORT_SYMBOL(wake_bit_function);
21
22/*
23 * To allow interruptible waiting and asynchronous (i.e. nonblocking)
24 * waiting, the actions of __wait_on_bit() and __wait_on_bit_lock() are
25 * permitted return codes. Nonzero return codes halt waiting and return.
26 */
27int __sched
28__wait_on_bit(struct wait_queue_head *wq_head, struct wait_bit_queue_entry *wbq_entry,
29 wait_bit_action_f *action, unsigned mode)
30{
31 int ret = 0;
32
33 do {
34 prepare_to_wait(wq_head, &wbq_entry->wq_entry, mode);
35 if (test_bit(wbq_entry->key.bit_nr, wbq_entry->key.flags))
36 ret = (*action)(&wbq_entry->key, mode);
37 } while (test_bit(wbq_entry->key.bit_nr, wbq_entry->key.flags) && !ret);
38 finish_wait(wq_head, &wbq_entry->wq_entry);
39 return ret;
40}
41EXPORT_SYMBOL(__wait_on_bit);
42
43int __sched out_of_line_wait_on_bit(void *word, int bit,
44 wait_bit_action_f *action, unsigned mode)
45{
46 struct wait_queue_head *wq_head = bit_waitqueue(word, bit);
47 DEFINE_WAIT_BIT(wq_entry, word, bit);
48
49 return __wait_on_bit(wq_head, &wq_entry, action, mode);
50}
51EXPORT_SYMBOL(out_of_line_wait_on_bit);
52
53int __sched out_of_line_wait_on_bit_timeout(
54 void *word, int bit, wait_bit_action_f *action,
55 unsigned mode, unsigned long timeout)
56{
57 struct wait_queue_head *wq_head = bit_waitqueue(word, bit);
58 DEFINE_WAIT_BIT(wq_entry, word, bit);
59
60 wq_entry.key.timeout = jiffies + timeout;
61 return __wait_on_bit(wq_head, &wq_entry, action, mode);
62}
63EXPORT_SYMBOL_GPL(out_of_line_wait_on_bit_timeout);
64
65int __sched
66__wait_on_bit_lock(struct wait_queue_head *wq_head, struct wait_bit_queue_entry *wbq_entry,
67 wait_bit_action_f *action, unsigned mode)
68{
69 int ret = 0;
70
71 for (;;) {
72 prepare_to_wait_exclusive(wq_head, &wbq_entry->wq_entry, mode);
73 if (test_bit(wbq_entry->key.bit_nr, wbq_entry->key.flags)) {
74 ret = action(&wbq_entry->key, mode);
75 /*
76 * See the comment in prepare_to_wait_event().
77 * finish_wait() does not necessarily takes wwq_head->lock,
78 * but test_and_set_bit() implies mb() which pairs with
79 * smp_mb__after_atomic() before wake_up_page().
80 */
81 if (ret)
82 finish_wait(wq_head, &wbq_entry->wq_entry);
83 }
84 if (!test_and_set_bit(wbq_entry->key.bit_nr, wbq_entry->key.flags)) {
85 if (!ret)
86 finish_wait(wq_head, &wbq_entry->wq_entry);
87 return 0;
88 } else if (ret) {
89 return ret;
90 }
91 }
92}
93EXPORT_SYMBOL(__wait_on_bit_lock);
94
95int __sched out_of_line_wait_on_bit_lock(void *word, int bit,
96 wait_bit_action_f *action, unsigned mode)
97{
98 struct wait_queue_head *wq_head = bit_waitqueue(word, bit);
99 DEFINE_WAIT_BIT(wq_entry, word, bit);
100
101 return __wait_on_bit_lock(wq_head, &wq_entry, action, mode);
102}
103EXPORT_SYMBOL(out_of_line_wait_on_bit_lock);
104
105void __wake_up_bit(struct wait_queue_head *wq_head, void *word, int bit)
106{
107 struct wait_bit_key key = __WAIT_BIT_KEY_INITIALIZER(word, bit);
108 if (waitqueue_active(wq_head))
109 __wake_up(wq_head, TASK_NORMAL, 1, &key);
110}
111EXPORT_SYMBOL(__wake_up_bit);
112
113/**
114 * wake_up_bit - wake up a waiter on a bit
115 * @word: the word being waited on, a kernel virtual address
116 * @bit: the bit of the word being waited on
117 *
118 * There is a standard hashed waitqueue table for generic use. This
119 * is the part of the hashtable's accessor API that wakes up waiters
120 * on a bit. For instance, if one were to have waiters on a bitflag,
121 * one would call wake_up_bit() after clearing the bit.
122 *
123 * In order for this to function properly, as it uses waitqueue_active()
124 * internally, some kind of memory barrier must be done prior to calling
125 * this. Typically, this will be smp_mb__after_atomic(), but in some
126 * cases where bitflags are manipulated non-atomically under a lock, one
127 * may need to use a less regular barrier, such fs/inode.c's smp_mb(),
128 * because spin_unlock() does not guarantee a memory barrier.
129 */
130void wake_up_bit(void *word, int bit)
131{
132 __wake_up_bit(bit_waitqueue(word, bit), word, bit);
133}
134EXPORT_SYMBOL(wake_up_bit);
135
136/*
137 * Manipulate the atomic_t address to produce a better bit waitqueue table hash
138 * index (we're keying off bit -1, but that would produce a horrible hash
139 * value).
140 */
141static inline wait_queue_head_t *atomic_t_waitqueue(atomic_t *p)
142{
143 if (BITS_PER_LONG == 64) {
144 unsigned long q = (unsigned long)p;
145 return bit_waitqueue((void *)(q & ~1), q & 1);
146 }
147 return bit_waitqueue(p, 0);
148}
149
150static int wake_atomic_t_function(struct wait_queue_entry *wq_entry, unsigned mode, int sync,
151 void *arg)
152{
153 struct wait_bit_key *key = arg;
154 struct wait_bit_queue_entry *wait_bit = container_of(wq_entry, struct wait_bit_queue_entry, wq_entry);
155 atomic_t *val = key->flags;
156
157 if (wait_bit->key.flags != key->flags ||
158 wait_bit->key.bit_nr != key->bit_nr ||
159 atomic_read(val) != 0)
160 return 0;
161 return autoremove_wake_function(wq_entry, mode, sync, key);
162}
163
164/*
165 * To allow interruptible waiting and asynchronous (i.e. nonblocking) waiting,
166 * the actions of __wait_on_atomic_t() are permitted return codes. Nonzero
167 * return codes halt waiting and return.
168 */
169static __sched
170int __wait_on_atomic_t(struct wait_queue_head *wq_head, struct wait_bit_queue_entry *wbq_entry,
171 int (*action)(atomic_t *), unsigned mode)
172{
173 atomic_t *val;
174 int ret = 0;
175
176 do {
177 prepare_to_wait(wq_head, &wbq_entry->wq_entry, mode);
178 val = wbq_entry->key.flags;
179 if (atomic_read(val) == 0)
180 break;
181 ret = (*action)(val);
182 } while (!ret && atomic_read(val) != 0);
183 finish_wait(wq_head, &wbq_entry->wq_entry);
184 return ret;
185}
186
187#define DEFINE_WAIT_ATOMIC_T(name, p) \
188 struct wait_bit_queue_entry name = { \
189 .key = __WAIT_ATOMIC_T_KEY_INITIALIZER(p), \
190 .wq_entry = { \
191 .private = current, \
192 .func = wake_atomic_t_function, \
193 .task_list = \
194 LIST_HEAD_INIT((name).wq_entry.task_list), \
195 }, \
196 }
197
198__sched int out_of_line_wait_on_atomic_t(atomic_t *p, int (*action)(atomic_t *),
199 unsigned mode)
200{
201 struct wait_queue_head *wq_head = atomic_t_waitqueue(p);
202 DEFINE_WAIT_ATOMIC_T(wq_entry, p);
203
204 return __wait_on_atomic_t(wq_head, &wq_entry, action, mode);
205}
206EXPORT_SYMBOL(out_of_line_wait_on_atomic_t);
207
208/**
209 * wake_up_atomic_t - Wake up a waiter on a atomic_t
210 * @p: The atomic_t being waited on, a kernel virtual address
211 *
212 * Wake up anyone waiting for the atomic_t to go to zero.
213 *
214 * Abuse the bit-waker function and its waitqueue hash table set (the atomic_t
215 * check is done by the waiter's wake function, not the by the waker itself).
216 */
217void wake_up_atomic_t(atomic_t *p)
218{
219 __wake_up_bit(atomic_t_waitqueue(p), p, WAIT_ATOMIC_T_BIT_NR);
220}
221EXPORT_SYMBOL(wake_up_atomic_t);
222
223__sched int bit_wait(struct wait_bit_key *word, int mode)
224{
225 schedule();
226 if (signal_pending_state(mode, current))
227 return -EINTR;
228 return 0;
229}
230EXPORT_SYMBOL(bit_wait);
231
232__sched int bit_wait_io(struct wait_bit_key *word, int mode)
233{
234 io_schedule();
235 if (signal_pending_state(mode, current))
236 return -EINTR;
237 return 0;
238}
239EXPORT_SYMBOL(bit_wait_io);
240
241__sched int bit_wait_timeout(struct wait_bit_key *word, int mode)
242{
243 unsigned long now = READ_ONCE(jiffies);
244 if (time_after_eq(now, word->timeout))
245 return -EAGAIN;
246 schedule_timeout(word->timeout - now);
247 if (signal_pending_state(mode, current))
248 return -EINTR;
249 return 0;
250}
251EXPORT_SYMBOL_GPL(bit_wait_timeout);
252
253__sched int bit_wait_io_timeout(struct wait_bit_key *word, int mode)
254{
255 unsigned long now = READ_ONCE(jiffies);
256 if (time_after_eq(now, word->timeout))
257 return -EAGAIN;
258 io_schedule_timeout(word->timeout - now);
259 if (signal_pending_state(mode, current))
260 return -EINTR;
261 return 0;
262}
263EXPORT_SYMBOL_GPL(bit_wait_io_timeout);
generated by cgit v1.2.2 (git 2.25.0) at 2025-04-26 05:35:34 -0400