aboutsummaryrefslogtreecommitdiffstats
path: root/kernel/kfifo.c
diff options
context:
space:
mode:
Diffstat (limited to 'kernel/kfifo.c')
-rw-r--r--kernel/kfifo.c750
1 files changed, 453 insertions, 297 deletions
diff --git a/kernel/kfifo.c b/kernel/kfifo.c
index 35edbe22e9a9..4502604ecadf 100644
--- a/kernel/kfifo.c
+++ b/kernel/kfifo.c
@@ -1,8 +1,7 @@
1/* 1/*
2 * A generic kernel FIFO implementation. 2 * A generic kernel FIFO implementation
3 * 3 *
4 * Copyright (C) 2009 Stefani Seibold <stefani@seibold.net> 4 * Copyright (C) 2009/2010 Stefani Seibold <stefani@seibold.net>
5 * Copyright (C) 2004 Stelian Pop <stelian@popies.net>
6 * 5 *
7 * This program is free software; you can redistribute it and/or modify 6 * This program is free software; you can redistribute it and/or modify
8 * it under the terms of the GNU General Public License as published by 7 * it under the terms of the GNU General Public License as published by
@@ -11,7 +10,7 @@
11 * 10 *
12 * This program is distributed in the hope that it will be useful, 11 * This program is distributed in the hope that it will be useful,
13 * but WITHOUT ANY WARRANTY; without even the implied warranty of 12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 * GNU General Public License for more details. 14 * GNU General Public License for more details.
16 * 15 *
17 * You should have received a copy of the GNU General Public License 16 * You should have received a copy of the GNU General Public License
@@ -24,422 +23,579 @@
24#include <linux/module.h> 23#include <linux/module.h>
25#include <linux/slab.h> 24#include <linux/slab.h>
26#include <linux/err.h> 25#include <linux/err.h>
27#include <linux/kfifo.h>
28#include <linux/log2.h> 26#include <linux/log2.h>
29#include <linux/uaccess.h> 27#include <linux/uaccess.h>
28#include <linux/kfifo.h>
30 29
31static void _kfifo_init(struct kfifo *fifo, void *buffer, 30/*
32 unsigned int size) 31 * internal helper to calculate the unused elements in a fifo
33{
34 fifo->buffer = buffer;
35 fifo->size = size;
36
37 kfifo_reset(fifo);
38}
39
40/**
41 * kfifo_init - initialize a FIFO using a preallocated buffer
42 * @fifo: the fifo to assign the buffer
43 * @buffer: the preallocated buffer to be used.
44 * @size: the size of the internal buffer, this has to be a power of 2.
45 *
46 */ 32 */
47void kfifo_init(struct kfifo *fifo, void *buffer, unsigned int size) 33static inline unsigned int kfifo_unused(struct __kfifo *fifo)
48{ 34{
49 /* size must be a power of 2 */ 35 return (fifo->mask + 1) - (fifo->in - fifo->out);
50 BUG_ON(!is_power_of_2(size));
51
52 _kfifo_init(fifo, buffer, size);
53} 36}
54EXPORT_SYMBOL(kfifo_init);
55 37
56/** 38int __kfifo_alloc(struct __kfifo *fifo, unsigned int size,
57 * kfifo_alloc - allocates a new FIFO internal buffer 39 size_t esize, gfp_t gfp_mask)
58 * @fifo: the fifo to assign then new buffer
59 * @size: the size of the buffer to be allocated, this have to be a power of 2.
60 * @gfp_mask: get_free_pages mask, passed to kmalloc()
61 *
62 * This function dynamically allocates a new fifo internal buffer
63 *
64 * The size will be rounded-up to a power of 2.
65 * The buffer will be release with kfifo_free().
66 * Return 0 if no error, otherwise the an error code
67 */
68int kfifo_alloc(struct kfifo *fifo, unsigned int size, gfp_t gfp_mask)
69{ 40{
70 unsigned char *buffer;
71
72 /* 41 /*
73 * round up to the next power of 2, since our 'let the indices 42 * round down to the next power of 2, since our 'let the indices
74 * wrap' technique works only in this case. 43 * wrap' technique works only in this case.
75 */ 44 */
76 if (!is_power_of_2(size)) { 45 if (!is_power_of_2(size))
77 BUG_ON(size > 0x80000000); 46 size = rounddown_pow_of_two(size);
78 size = roundup_pow_of_two(size); 47
48 fifo->in = 0;
49 fifo->out = 0;
50 fifo->esize = esize;
51
52 if (size < 2) {
53 fifo->data = NULL;
54 fifo->mask = 0;
55 return -EINVAL;
79 } 56 }
80 57
81 buffer = kmalloc(size, gfp_mask); 58 fifo->data = kmalloc(size * esize, gfp_mask);
82 if (!buffer) { 59
83 _kfifo_init(fifo, NULL, 0); 60 if (!fifo->data) {
61 fifo->mask = 0;
84 return -ENOMEM; 62 return -ENOMEM;
85 } 63 }
86 64 fifo->mask = size - 1;
87 _kfifo_init(fifo, buffer, size);
88 65
89 return 0; 66 return 0;
90} 67}
91EXPORT_SYMBOL(kfifo_alloc); 68EXPORT_SYMBOL(__kfifo_alloc);
92 69
93/** 70void __kfifo_free(struct __kfifo *fifo)
94 * kfifo_free - frees the FIFO internal buffer
95 * @fifo: the fifo to be freed.
96 */
97void kfifo_free(struct kfifo *fifo)
98{ 71{
99 kfree(fifo->buffer); 72 kfree(fifo->data);
100 _kfifo_init(fifo, NULL, 0); 73 fifo->in = 0;
74 fifo->out = 0;
75 fifo->esize = 0;
76 fifo->data = NULL;
77 fifo->mask = 0;
101} 78}
102EXPORT_SYMBOL(kfifo_free); 79EXPORT_SYMBOL(__kfifo_free);
103 80
104/** 81int __kfifo_init(struct __kfifo *fifo, void *buffer,
105 * kfifo_skip - skip output data 82 unsigned int size, size_t esize)
106 * @fifo: the fifo to be used.
107 * @len: number of bytes to skip
108 */
109void kfifo_skip(struct kfifo *fifo, unsigned int len)
110{ 83{
111 if (len < kfifo_len(fifo)) { 84 size /= esize;
112 __kfifo_add_out(fifo, len); 85
113 return; 86 if (!is_power_of_2(size))
87 size = rounddown_pow_of_two(size);
88
89 fifo->in = 0;
90 fifo->out = 0;
91 fifo->esize = esize;
92 fifo->data = buffer;
93
94 if (size < 2) {
95 fifo->mask = 0;
96 return -EINVAL;
114 } 97 }
115 kfifo_reset_out(fifo); 98 fifo->mask = size - 1;
99
100 return 0;
116} 101}
117EXPORT_SYMBOL(kfifo_skip); 102EXPORT_SYMBOL(__kfifo_init);
118 103
119static inline void __kfifo_in_data(struct kfifo *fifo, 104static void kfifo_copy_in(struct __kfifo *fifo, const void *src,
120 const void *from, unsigned int len, unsigned int off) 105 unsigned int len, unsigned int off)
121{ 106{
107 unsigned int size = fifo->mask + 1;
108 unsigned int esize = fifo->esize;
122 unsigned int l; 109 unsigned int l;
123 110
111 off &= fifo->mask;
112 if (esize != 1) {
113 off *= esize;
114 size *= esize;
115 len *= esize;
116 }
117 l = min(len, size - off);
118
119 memcpy(fifo->data + off, src, l);
120 memcpy(fifo->data, src + l, len - l);
124 /* 121 /*
125 * Ensure that we sample the fifo->out index -before- we 122 * make sure that the data in the fifo is up to date before
126 * start putting bytes into the kfifo. 123 * incrementing the fifo->in index counter
127 */ 124 */
125 smp_wmb();
126}
128 127
129 smp_mb(); 128unsigned int __kfifo_in(struct __kfifo *fifo,
130 129 const void *buf, unsigned int len)
131 off = __kfifo_off(fifo, fifo->in + off); 130{
131 unsigned int l;
132 132
133 /* first put the data starting from fifo->in to buffer end */ 133 l = kfifo_unused(fifo);
134 l = min(len, fifo->size - off); 134 if (len > l)
135 memcpy(fifo->buffer + off, from, l); 135 len = l;
136 136
137 /* then put the rest (if any) at the beginning of the buffer */ 137 kfifo_copy_in(fifo, buf, len, fifo->in);
138 memcpy(fifo->buffer, from + l, len - l); 138 fifo->in += len;
139 return len;
139} 140}
141EXPORT_SYMBOL(__kfifo_in);
140 142
141static inline void __kfifo_out_data(struct kfifo *fifo, 143static void kfifo_copy_out(struct __kfifo *fifo, void *dst,
142 void *to, unsigned int len, unsigned int off) 144 unsigned int len, unsigned int off)
143{ 145{
146 unsigned int size = fifo->mask + 1;
147 unsigned int esize = fifo->esize;
144 unsigned int l; 148 unsigned int l;
145 149
150 off &= fifo->mask;
151 if (esize != 1) {
152 off *= esize;
153 size *= esize;
154 len *= esize;
155 }
156 l = min(len, size - off);
157
158 memcpy(dst, fifo->data + off, l);
159 memcpy(dst + l, fifo->data, len - l);
146 /* 160 /*
147 * Ensure that we sample the fifo->in index -before- we 161 * make sure that the data is copied before
148 * start removing bytes from the kfifo. 162 * incrementing the fifo->out index counter
149 */ 163 */
164 smp_wmb();
165}
150 166
151 smp_rmb(); 167unsigned int __kfifo_out_peek(struct __kfifo *fifo,
168 void *buf, unsigned int len)
169{
170 unsigned int l;
152 171
153 off = __kfifo_off(fifo, fifo->out + off); 172 l = fifo->in - fifo->out;
173 if (len > l)
174 len = l;
154 175
155 /* first get the data from fifo->out until the end of the buffer */ 176 kfifo_copy_out(fifo, buf, len, fifo->out);
156 l = min(len, fifo->size - off); 177 return len;
157 memcpy(to, fifo->buffer + off, l); 178}
179EXPORT_SYMBOL(__kfifo_out_peek);
158 180
159 /* then get the rest (if any) from the beginning of the buffer */ 181unsigned int __kfifo_out(struct __kfifo *fifo,
160 memcpy(to + l, fifo->buffer, len - l); 182 void *buf, unsigned int len)
183{
184 len = __kfifo_out_peek(fifo, buf, len);
185 fifo->out += len;
186 return len;
161} 187}
188EXPORT_SYMBOL(__kfifo_out);
162 189
163static inline int __kfifo_from_user_data(struct kfifo *fifo, 190static unsigned long kfifo_copy_from_user(struct __kfifo *fifo,
164 const void __user *from, unsigned int len, unsigned int off, 191 const void __user *from, unsigned int len, unsigned int off,
165 unsigned *lenout) 192 unsigned int *copied)
166{ 193{
194 unsigned int size = fifo->mask + 1;
195 unsigned int esize = fifo->esize;
167 unsigned int l; 196 unsigned int l;
168 int ret; 197 unsigned long ret;
169 198
199 off &= fifo->mask;
200 if (esize != 1) {
201 off *= esize;
202 size *= esize;
203 len *= esize;
204 }
205 l = min(len, size - off);
206
207 ret = copy_from_user(fifo->data + off, from, l);
208 if (unlikely(ret))
209 ret = DIV_ROUND_UP(ret + len - l, esize);
210 else {
211 ret = copy_from_user(fifo->data, from + l, len - l);
212 if (unlikely(ret))
213 ret = DIV_ROUND_UP(ret, esize);
214 }
170 /* 215 /*
171 * Ensure that we sample the fifo->out index -before- we 216 * make sure that the data in the fifo is up to date before
172 * start putting bytes into the kfifo. 217 * incrementing the fifo->in index counter
173 */ 218 */
219 smp_wmb();
220 *copied = len - ret;
221 /* return the number of elements which are not copied */
222 return ret;
223}
174 224
175 smp_mb(); 225int __kfifo_from_user(struct __kfifo *fifo, const void __user *from,
226 unsigned long len, unsigned int *copied)
227{
228 unsigned int l;
229 unsigned long ret;
230 unsigned int esize = fifo->esize;
231 int err;
176 232
177 off = __kfifo_off(fifo, fifo->in + off); 233 if (esize != 1)
234 len /= esize;
178 235
179 /* first put the data starting from fifo->in to buffer end */ 236 l = kfifo_unused(fifo);
180 l = min(len, fifo->size - off); 237 if (len > l)
181 ret = copy_from_user(fifo->buffer + off, from, l); 238 len = l;
182 if (unlikely(ret)) {
183 *lenout = ret;
184 return -EFAULT;
185 }
186 *lenout = l;
187 239
188 /* then put the rest (if any) at the beginning of the buffer */ 240 ret = kfifo_copy_from_user(fifo, from, len, fifo->in, copied);
189 ret = copy_from_user(fifo->buffer, from + l, len - l); 241 if (unlikely(ret)) {
190 *lenout += ret ? ret : len - l; 242 len -= ret;
191 return ret ? -EFAULT : 0; 243 err = -EFAULT;
244 } else
245 err = 0;
246 fifo->in += len;
247 return err;
192} 248}
249EXPORT_SYMBOL(__kfifo_from_user);
193 250
194static inline int __kfifo_to_user_data(struct kfifo *fifo, 251static unsigned long kfifo_copy_to_user(struct __kfifo *fifo, void __user *to,
195 void __user *to, unsigned int len, unsigned int off, unsigned *lenout) 252 unsigned int len, unsigned int off, unsigned int *copied)
196{ 253{
197 unsigned int l; 254 unsigned int l;
198 int ret; 255 unsigned long ret;
199 256 unsigned int size = fifo->mask + 1;
257 unsigned int esize = fifo->esize;
258
259 off &= fifo->mask;
260 if (esize != 1) {
261 off *= esize;
262 size *= esize;
263 len *= esize;
264 }
265 l = min(len, size - off);
266
267 ret = copy_to_user(to, fifo->data + off, l);
268 if (unlikely(ret))
269 ret = DIV_ROUND_UP(ret + len - l, esize);
270 else {
271 ret = copy_to_user(to + l, fifo->data, len - l);
272 if (unlikely(ret))
273 ret = DIV_ROUND_UP(ret, esize);
274 }
200 /* 275 /*
201 * Ensure that we sample the fifo->in index -before- we 276 * make sure that the data is copied before
202 * start removing bytes from the kfifo. 277 * incrementing the fifo->out index counter
203 */ 278 */
279 smp_wmb();
280 *copied = len - ret;
281 /* return the number of elements which are not copied */
282 return ret;
283}
204 284
205 smp_rmb(); 285int __kfifo_to_user(struct __kfifo *fifo, void __user *to,
286 unsigned long len, unsigned int *copied)
287{
288 unsigned int l;
289 unsigned long ret;
290 unsigned int esize = fifo->esize;
291 int err;
206 292
207 off = __kfifo_off(fifo, fifo->out + off); 293 if (esize != 1)
294 len /= esize;
208 295
209 /* first get the data from fifo->out until the end of the buffer */ 296 l = fifo->in - fifo->out;
210 l = min(len, fifo->size - off); 297 if (len > l)
211 ret = copy_to_user(to, fifo->buffer + off, l); 298 len = l;
212 *lenout = l; 299 ret = kfifo_copy_to_user(fifo, to, len, fifo->out, copied);
213 if (unlikely(ret)) { 300 if (unlikely(ret)) {
214 *lenout -= ret; 301 len -= ret;
215 return -EFAULT; 302 err = -EFAULT;
216 } 303 } else
304 err = 0;
305 fifo->out += len;
306 return err;
307}
308EXPORT_SYMBOL(__kfifo_to_user);
217 309
218 /* then get the rest (if any) from the beginning of the buffer */ 310static int setup_sgl_buf(struct scatterlist *sgl, void *buf,
219 len -= l; 311 int nents, unsigned int len)
220 ret = copy_to_user(to + l, fifo->buffer, len); 312{
221 if (unlikely(ret)) { 313 int n;
222 *lenout += len - ret; 314 unsigned int l;
223 return -EFAULT; 315 unsigned int off;
316 struct page *page;
317
318 if (!nents)
319 return 0;
320
321 if (!len)
322 return 0;
323
324 n = 0;
325 page = virt_to_page(buf);
326 off = offset_in_page(buf);
327 l = 0;
328
329 while (len >= l + PAGE_SIZE - off) {
330 struct page *npage;
331
332 l += PAGE_SIZE;
333 buf += PAGE_SIZE;
334 npage = virt_to_page(buf);
335 if (page_to_phys(page) != page_to_phys(npage) - l) {
336 sg_set_page(sgl, page, l - off, off);
337 sgl = sg_next(sgl);
338 if (++n == nents || sgl == NULL)
339 return n;
340 page = npage;
341 len -= l - off;
342 l = off = 0;
343 }
224 } 344 }
225 *lenout += len; 345 sg_set_page(sgl, page, len, off);
226 return 0; 346 return n + 1;
227} 347}
228 348
229unsigned int __kfifo_in_n(struct kfifo *fifo, 349static unsigned int setup_sgl(struct __kfifo *fifo, struct scatterlist *sgl,
230 const void *from, unsigned int len, unsigned int recsize) 350 int nents, unsigned int len, unsigned int off)
231{ 351{
232 if (kfifo_avail(fifo) < len + recsize) 352 unsigned int size = fifo->mask + 1;
233 return len + 1; 353 unsigned int esize = fifo->esize;
354 unsigned int l;
355 unsigned int n;
234 356
235 __kfifo_in_data(fifo, from, len, recsize); 357 off &= fifo->mask;
236 return 0; 358 if (esize != 1) {
359 off *= esize;
360 size *= esize;
361 len *= esize;
362 }
363 l = min(len, size - off);
364
365 n = setup_sgl_buf(sgl, fifo->data + off, nents, l);
366 n += setup_sgl_buf(sgl + n, fifo->data, nents - n, len - l);
367
368 if (n)
369 sg_mark_end(sgl + n - 1);
370 return n;
237} 371}
238EXPORT_SYMBOL(__kfifo_in_n);
239 372
240/** 373unsigned int __kfifo_dma_in_prepare(struct __kfifo *fifo,
241 * kfifo_in - puts some data into the FIFO 374 struct scatterlist *sgl, int nents, unsigned int len)
242 * @fifo: the fifo to be used.
243 * @from: the data to be added.
244 * @len: the length of the data to be added.
245 *
246 * This function copies at most @len bytes from the @from buffer into
247 * the FIFO depending on the free space, and returns the number of
248 * bytes copied.
249 *
250 * Note that with only one concurrent reader and one concurrent
251 * writer, you don't need extra locking to use these functions.
252 */
253unsigned int kfifo_in(struct kfifo *fifo, const void *from,
254 unsigned int len)
255{ 375{
256 len = min(kfifo_avail(fifo), len); 376 unsigned int l;
257 377
258 __kfifo_in_data(fifo, from, len, 0); 378 l = kfifo_unused(fifo);
259 __kfifo_add_in(fifo, len); 379 if (len > l)
260 return len; 380 len = l;
381
382 return setup_sgl(fifo, sgl, nents, len, fifo->in);
261} 383}
262EXPORT_SYMBOL(kfifo_in); 384EXPORT_SYMBOL(__kfifo_dma_in_prepare);
263 385
264unsigned int __kfifo_in_generic(struct kfifo *fifo, 386unsigned int __kfifo_dma_out_prepare(struct __kfifo *fifo,
265 const void *from, unsigned int len, unsigned int recsize) 387 struct scatterlist *sgl, int nents, unsigned int len)
266{ 388{
267 return __kfifo_in_rec(fifo, from, len, recsize); 389 unsigned int l;
390
391 l = fifo->in - fifo->out;
392 if (len > l)
393 len = l;
394
395 return setup_sgl(fifo, sgl, nents, len, fifo->out);
268} 396}
269EXPORT_SYMBOL(__kfifo_in_generic); 397EXPORT_SYMBOL(__kfifo_dma_out_prepare);
270 398
271unsigned int __kfifo_out_n(struct kfifo *fifo, 399unsigned int __kfifo_max_r(unsigned int len, size_t recsize)
272 void *to, unsigned int len, unsigned int recsize)
273{ 400{
274 if (kfifo_len(fifo) < len + recsize) 401 unsigned int max = (1 << (recsize << 3)) - 1;
275 return len;
276 402
277 __kfifo_out_data(fifo, to, len, recsize); 403 if (len > max)
278 __kfifo_add_out(fifo, len + recsize); 404 return max;
279 return 0; 405 return len;
280} 406}
281EXPORT_SYMBOL(__kfifo_out_n);
282 407
283/** 408#define __KFIFO_PEEK(data, out, mask) \
284 * kfifo_out - gets some data from the FIFO 409 ((data)[(out) & (mask)])
285 * @fifo: the fifo to be used. 410/*
286 * @to: where the data must be copied. 411 * __kfifo_peek_n internal helper function for determinate the length of
287 * @len: the size of the destination buffer. 412 * the next record in the fifo
288 *
289 * This function copies at most @len bytes from the FIFO into the
290 * @to buffer and returns the number of copied bytes.
291 *
292 * Note that with only one concurrent reader and one concurrent
293 * writer, you don't need extra locking to use these functions.
294 */ 413 */
295unsigned int kfifo_out(struct kfifo *fifo, void *to, unsigned int len) 414static unsigned int __kfifo_peek_n(struct __kfifo *fifo, size_t recsize)
296{ 415{
297 len = min(kfifo_len(fifo), len); 416 unsigned int l;
417 unsigned int mask = fifo->mask;
418 unsigned char *data = fifo->data;
298 419
299 __kfifo_out_data(fifo, to, len, 0); 420 l = __KFIFO_PEEK(data, fifo->out, mask);
300 __kfifo_add_out(fifo, len);
301 421
302 return len; 422 if (--recsize)
423 l |= __KFIFO_PEEK(data, fifo->out + 1, mask) << 8;
424
425 return l;
303} 426}
304EXPORT_SYMBOL(kfifo_out); 427
305 428#define __KFIFO_POKE(data, in, mask, val) \
306/** 429 ( \
307 * kfifo_out_peek - copy some data from the FIFO, but do not remove it 430 (data)[(in) & (mask)] = (unsigned char)(val) \
308 * @fifo: the fifo to be used. 431 )
309 * @to: where the data must be copied. 432
310 * @len: the size of the destination buffer. 433/*
311 * @offset: offset into the fifo 434 * __kfifo_poke_n internal helper function for storeing the length of
312 * 435 * the record into the fifo
313 * This function copies at most @len bytes at @offset from the FIFO
314 * into the @to buffer and returns the number of copied bytes.
315 * The data is not removed from the FIFO.
316 */ 436 */
317unsigned int kfifo_out_peek(struct kfifo *fifo, void *to, unsigned int len, 437static void __kfifo_poke_n(struct __kfifo *fifo, unsigned int n, size_t recsize)
318 unsigned offset)
319{ 438{
320 len = min(kfifo_len(fifo), len + offset); 439 unsigned int mask = fifo->mask;
440 unsigned char *data = fifo->data;
321 441
322 __kfifo_out_data(fifo, to, len, offset); 442 __KFIFO_POKE(data, fifo->in, mask, n);
323 return len; 443
444 if (recsize > 1)
445 __KFIFO_POKE(data, fifo->in + 1, mask, n >> 8);
324} 446}
325EXPORT_SYMBOL(kfifo_out_peek);
326 447
327unsigned int __kfifo_out_generic(struct kfifo *fifo, 448unsigned int __kfifo_len_r(struct __kfifo *fifo, size_t recsize)
328 void *to, unsigned int len, unsigned int recsize,
329 unsigned int *total)
330{ 449{
331 return __kfifo_out_rec(fifo, to, len, recsize, total); 450 return __kfifo_peek_n(fifo, recsize);
332} 451}
333EXPORT_SYMBOL(__kfifo_out_generic); 452EXPORT_SYMBOL(__kfifo_len_r);
334 453
335unsigned int __kfifo_from_user_n(struct kfifo *fifo, 454unsigned int __kfifo_in_r(struct __kfifo *fifo, const void *buf,
336 const void __user *from, unsigned int len, unsigned int recsize) 455 unsigned int len, size_t recsize)
337{ 456{
338 unsigned total; 457 if (len + recsize > kfifo_unused(fifo))
458 return 0;
339 459
340 if (kfifo_avail(fifo) < len + recsize) 460 __kfifo_poke_n(fifo, len, recsize);
341 return len + 1;
342 461
343 __kfifo_from_user_data(fifo, from, len, recsize, &total); 462 kfifo_copy_in(fifo, buf, len, fifo->in + recsize);
344 return total; 463 fifo->in += len + recsize;
464 return len;
345} 465}
346EXPORT_SYMBOL(__kfifo_from_user_n); 466EXPORT_SYMBOL(__kfifo_in_r);
347 467
348/** 468static unsigned int kfifo_out_copy_r(struct __kfifo *fifo,
349 * kfifo_from_user - puts some data from user space into the FIFO 469 void *buf, unsigned int len, size_t recsize, unsigned int *n)
350 * @fifo: the fifo to be used.
351 * @from: pointer to the data to be added.
352 * @len: the length of the data to be added.
353 * @total: the actual returned data length.
354 *
355 * This function copies at most @len bytes from the @from into the
356 * FIFO depending and returns -EFAULT/0.
357 *
358 * Note that with only one concurrent reader and one concurrent
359 * writer, you don't need extra locking to use these functions.
360 */
361int kfifo_from_user(struct kfifo *fifo,
362 const void __user *from, unsigned int len, unsigned *total)
363{ 470{
364 int ret; 471 *n = __kfifo_peek_n(fifo, recsize);
365 len = min(kfifo_avail(fifo), len); 472
366 ret = __kfifo_from_user_data(fifo, from, len, 0, total); 473 if (len > *n)
367 if (ret) 474 len = *n;
368 return ret; 475
369 __kfifo_add_in(fifo, len); 476 kfifo_copy_out(fifo, buf, len, fifo->out + recsize);
370 return 0; 477 return len;
478}
479
480unsigned int __kfifo_out_peek_r(struct __kfifo *fifo, void *buf,
481 unsigned int len, size_t recsize)
482{
483 unsigned int n;
484
485 if (fifo->in == fifo->out)
486 return 0;
487
488 return kfifo_out_copy_r(fifo, buf, len, recsize, &n);
371} 489}
372EXPORT_SYMBOL(kfifo_from_user); 490EXPORT_SYMBOL(__kfifo_out_peek_r);
373 491
374unsigned int __kfifo_from_user_generic(struct kfifo *fifo, 492unsigned int __kfifo_out_r(struct __kfifo *fifo, void *buf,
375 const void __user *from, unsigned int len, unsigned int recsize) 493 unsigned int len, size_t recsize)
376{ 494{
377 return __kfifo_from_user_rec(fifo, from, len, recsize); 495 unsigned int n;
496
497 if (fifo->in == fifo->out)
498 return 0;
499
500 len = kfifo_out_copy_r(fifo, buf, len, recsize, &n);
501 fifo->out += n + recsize;
502 return len;
378} 503}
379EXPORT_SYMBOL(__kfifo_from_user_generic); 504EXPORT_SYMBOL(__kfifo_out_r);
380 505
381unsigned int __kfifo_to_user_n(struct kfifo *fifo, 506int __kfifo_from_user_r(struct __kfifo *fifo, const void __user *from,
382 void __user *to, unsigned int len, unsigned int reclen, 507 unsigned long len, unsigned int *copied, size_t recsize)
383 unsigned int recsize)
384{ 508{
385 unsigned int ret, total; 509 unsigned long ret;
386 510
387 if (kfifo_len(fifo) < reclen + recsize) 511 len = __kfifo_max_r(len, recsize);
388 return len;
389 512
390 ret = __kfifo_to_user_data(fifo, to, reclen, recsize, &total); 513 if (len + recsize > kfifo_unused(fifo)) {
514 *copied = 0;
515 return 0;
516 }
391 517
392 if (likely(ret == 0)) 518 __kfifo_poke_n(fifo, len, recsize);
393 __kfifo_add_out(fifo, reclen + recsize);
394 519
395 return total; 520 ret = kfifo_copy_from_user(fifo, from, len, fifo->in + recsize, copied);
521 if (unlikely(ret)) {
522 *copied = 0;
523 return -EFAULT;
524 }
525 fifo->in += len + recsize;
526 return 0;
396} 527}
397EXPORT_SYMBOL(__kfifo_to_user_n); 528EXPORT_SYMBOL(__kfifo_from_user_r);
398 529
399/** 530int __kfifo_to_user_r(struct __kfifo *fifo, void __user *to,
400 * kfifo_to_user - gets data from the FIFO and write it to user space 531 unsigned long len, unsigned int *copied, size_t recsize)
401 * @fifo: the fifo to be used.
402 * @to: where the data must be copied.
403 * @len: the size of the destination buffer.
404 * @lenout: pointer to output variable with copied data
405 *
406 * This function copies at most @len bytes from the FIFO into the
407 * @to buffer and 0 or -EFAULT.
408 *
409 * Note that with only one concurrent reader and one concurrent
410 * writer, you don't need extra locking to use these functions.
411 */
412int kfifo_to_user(struct kfifo *fifo,
413 void __user *to, unsigned int len, unsigned *lenout)
414{ 532{
415 int ret; 533 unsigned long ret;
416 len = min(kfifo_len(fifo), len); 534 unsigned int n;
417 ret = __kfifo_to_user_data(fifo, to, len, 0, lenout); 535
418 __kfifo_add_out(fifo, *lenout); 536 if (fifo->in == fifo->out) {
419 return ret; 537 *copied = 0;
538 return 0;
539 }
540
541 n = __kfifo_peek_n(fifo, recsize);
542 if (len > n)
543 len = n;
544
545 ret = kfifo_copy_to_user(fifo, to, len, fifo->out + recsize, copied);
546 if (unlikely(ret)) {
547 *copied = 0;
548 return -EFAULT;
549 }
550 fifo->out += n + recsize;
551 return 0;
420} 552}
421EXPORT_SYMBOL(kfifo_to_user); 553EXPORT_SYMBOL(__kfifo_to_user_r);
422 554
423unsigned int __kfifo_to_user_generic(struct kfifo *fifo, 555unsigned int __kfifo_dma_in_prepare_r(struct __kfifo *fifo,
424 void __user *to, unsigned int len, unsigned int recsize, 556 struct scatterlist *sgl, int nents, unsigned int len, size_t recsize)
425 unsigned int *total)
426{ 557{
427 return __kfifo_to_user_rec(fifo, to, len, recsize, total); 558 if (!nents)
559 BUG();
560
561 len = __kfifo_max_r(len, recsize);
562
563 if (len + recsize > kfifo_unused(fifo))
564 return 0;
565
566 return setup_sgl(fifo, sgl, nents, len, fifo->in + recsize);
428} 567}
429EXPORT_SYMBOL(__kfifo_to_user_generic); 568EXPORT_SYMBOL(__kfifo_dma_in_prepare_r);
430 569
431unsigned int __kfifo_peek_generic(struct kfifo *fifo, unsigned int recsize) 570void __kfifo_dma_in_finish_r(struct __kfifo *fifo,
571 unsigned int len, size_t recsize)
432{ 572{
433 if (recsize == 0) 573 len = __kfifo_max_r(len, recsize);
434 return kfifo_avail(fifo); 574 __kfifo_poke_n(fifo, len, recsize);
435 575 fifo->in += len + recsize;
436 return __kfifo_peek_n(fifo, recsize);
437} 576}
438EXPORT_SYMBOL(__kfifo_peek_generic); 577EXPORT_SYMBOL(__kfifo_dma_in_finish_r);
439 578
440void __kfifo_skip_generic(struct kfifo *fifo, unsigned int recsize) 579unsigned int __kfifo_dma_out_prepare_r(struct __kfifo *fifo,
580 struct scatterlist *sgl, int nents, unsigned int len, size_t recsize)
441{ 581{
442 __kfifo_skip_rec(fifo, recsize); 582 if (!nents)
583 BUG();
584
585 len = __kfifo_max_r(len, recsize);
586
587 if (len + recsize > fifo->in - fifo->out)
588 return 0;
589
590 return setup_sgl(fifo, sgl, nents, len, fifo->out + recsize);
443} 591}
444EXPORT_SYMBOL(__kfifo_skip_generic); 592EXPORT_SYMBOL(__kfifo_dma_out_prepare_r);
593
594void __kfifo_dma_out_finish_r(struct __kfifo *fifo, size_t recsize)
595{
596 unsigned int len;
445 597
598 len = __kfifo_peek_n(fifo, recsize);
599 fifo->out += len + recsize;
600}
601EXPORT_SYMBOL(__kfifo_dma_out_finish_r);
generated by cgit v1.2.2 (git 2.25.0) at 2025-10-31 20:54:28 -0400