aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
Diffstat
-rw-r--r--Documentation/DocBook/kernel-api.tmpl1
-rw-r--r--include/linux/kfifo-new.h844
-rw-r--r--include/linux/kfifo.h1193
-rw-r--r--kernel/kfifo-new.c602
-rw-r--r--kernel/kfifo.c749
5 files changed, 1164 insertions, 2225 deletions
diff --git a/Documentation/DocBook/kernel-api.tmpl b/Documentation/DocBook/kernel-api.tmpl
index 44b3def961a2..a20c6f6fffc3 100644
--- a/Documentation/DocBook/kernel-api.tmpl
+++ b/Documentation/DocBook/kernel-api.tmpl
@@ -132,7 +132,6 @@ X!Ilib/string.c
132 <title>FIFO Buffer</title> 132 <title>FIFO Buffer</title>
133 <sect1><title>kfifo interface</title> 133 <sect1><title>kfifo interface</title>
134!Iinclude/linux/kfifo.h 134!Iinclude/linux/kfifo.h
135!Ekernel/kfifo.c
136 </sect1> 135 </sect1>
137 </chapter> 136 </chapter>
138 137
diff --git a/include/linux/kfifo-new.h b/include/linux/kfifo-new.h
deleted file mode 100644
index 311f8753d713..000000000000
--- a/include/linux/kfifo-new.h
+++ /dev/null
@@ -1,844 +0,0 @@
1/*
2 * A generic kernel FIFO implementation
3 *
4 * Copyright (C) 2009/2010 Stefani Seibold <stefani@seibold.net>
5 *
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License as published by
8 * the Free Software Foundation; either version 2 of the License, or
9 * (at your option) any later version.
10 *
11 * This program is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 * GNU General Public License for more details.
15 *
16 * You should have received a copy of the GNU General Public License
17 * along with this program; if not, write to the Free Software
18 * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
19 *
20 */
21
22#ifndef _LINUX_KFIFO_H
23#define _LINUX_KFIFO_H
24
25/*
26 * How to porting drivers to the new generic FIFO API:
27 *
28 * - Modify the declaration of the "struct kfifo *" object into a
29 * in-place "struct kfifo" object
30 * - Init the in-place object with kfifo_alloc() or kfifo_init()
31 * Note: The address of the in-place "struct kfifo" object must be
32 * passed as the first argument to this functions
33 * - Replace the use of __kfifo_put into kfifo_in and __kfifo_get
34 * into kfifo_out
35 * - Replace the use of kfifo_put into kfifo_in_spinlocked and kfifo_get
36 * into kfifo_out_spinlocked
37 * Note: the spinlock pointer formerly passed to kfifo_init/kfifo_alloc
38 * must be passed now to the kfifo_in_spinlocked and kfifo_out_spinlocked
39 * as the last parameter
40 * - The formerly __kfifo_* functions are renamed into kfifo_*
41 */
42
43/*
44 * Note about locking : There is no locking required until only * one reader
45 * and one writer is using the fifo and no kfifo_reset() will be * called
46 * kfifo_reset_out() can be safely used, until it will be only called
47 * in the reader thread.
48 * For multiple writer and one reader there is only a need to lock the writer.
49 * And vice versa for only one writer and multiple reader there is only a need
50 * to lock the reader.
51 */
52
53#include <linux/kernel.h>
54#include <linux/spinlock.h>
55#include <linux/stddef.h>
56#include <linux/scatterlist.h>
57
58struct __kfifo {
59 unsigned int in;
60 unsigned int out;
61 unsigned int mask;
62 unsigned int esize;
63 void *data;
64};
65
66#define __STRUCT_KFIFO_COMMON(datatype, recsize, ptrtype) \
67 union { \
68 struct __kfifo kfifo; \
69 datatype *type; \
70 char (*rectype)[recsize]; \
71 ptrtype *ptr; \
72 const ptrtype *ptr_const; \
73 }
74
75#define __STRUCT_KFIFO(type, size, recsize, ptrtype) \
76{ \
77 __STRUCT_KFIFO_COMMON(type, recsize, ptrtype); \
78 type buf[((size < 2) || (size & (size - 1))) ? -1 : size]; \
79}
80
81#define STRUCT_KFIFO(type, size) \
82 struct __STRUCT_KFIFO(type, size, 0, type)
83
84#define __STRUCT_KFIFO_PTR(type, recsize, ptrtype) \
85{ \
86 __STRUCT_KFIFO_COMMON(type, recsize, ptrtype); \
87 type buf[0]; \
88}
89
90#define STRUCT_KFIFO_PTR(type) \
91 struct __STRUCT_KFIFO_PTR(type, 0, type)
92
93/*
94 * define compatibility "struct kfifo" for dynamic allocated fifos
95 */
96struct kfifo __STRUCT_KFIFO_PTR(unsigned char, 0, void);
97
98#define STRUCT_KFIFO_REC_1(size) \
99 struct __STRUCT_KFIFO(unsigned char, size, 1, void)
100
101#define STRUCT_KFIFO_REC_2(size) \
102 struct __STRUCT_KFIFO(unsigned char, size, 2, void)
103
104/*
105 * define kfifo_rec types
106 */
107struct kfifo_rec_ptr_1 __STRUCT_KFIFO_PTR(unsigned char, 1, void);
108struct kfifo_rec_ptr_2 __STRUCT_KFIFO_PTR(unsigned char, 2, void);
109
110/*
111 * helper macro to distinguish between real in place fifo where the fifo
112 * array is a part of the structure and the fifo type where the array is
113 * outside of the fifo structure.
114 */
115#define __is_kfifo_ptr(fifo) (sizeof(*fifo) == sizeof(struct __kfifo))
116
117/**
118 * DECLARE_KFIFO_PTR - macro to declare a fifo pointer object
119 * @fifo: name of the declared fifo
120 * @type: type of the fifo elements
121 */
122#define DECLARE_KFIFO_PTR(fifo, type) STRUCT_KFIFO_PTR(type) fifo
123
124/**
125 * DECLARE_KFIFO - macro to declare a fifo object
126 * @fifo: name of the declared fifo
127 * @type: type of the fifo elements
128 * @size: the number of elements in the fifo, this must be a power of 2
129 */
130#define DECLARE_KFIFO(fifo, type, size) STRUCT_KFIFO(type, size) fifo
131
132/**
133 * INIT_KFIFO - Initialize a fifo declared by DECLARE_KFIFO
134 * @fifo: name of the declared fifo datatype
135 */
136#define INIT_KFIFO(fifo) \
137(void)({ \
138 typeof(&(fifo)) __tmp = &(fifo); \
139 struct __kfifo *__kfifo = &__tmp->kfifo; \
140 __kfifo->in = 0; \
141 __kfifo->out = 0; \
142 __kfifo->mask = __is_kfifo_ptr(__tmp) ? 0 : ARRAY_SIZE(__tmp->buf) - 1;\
143 __kfifo->esize = sizeof(*__tmp->buf); \
144 __kfifo->data = __is_kfifo_ptr(__tmp) ? NULL : __tmp->buf; \
145})
146
147/**
148 * DEFINE_KFIFO - macro to define and initialize a fifo
149 * @fifo: name of the declared fifo datatype
150 * @type: type of the fifo elements
151 * @size: the number of elements in the fifo, this must be a power of 2
152 *
153 * Note: the macro can be used for global and local fifo data type variables.
154 */
155#define DEFINE_KFIFO(fifo, type, size) \
156 DECLARE_KFIFO(fifo, type, size) = \
157 (typeof(fifo)) { \
158 { \
159 { \
160 .in = 0, \
161 .out = 0, \
162 .mask = __is_kfifo_ptr(&(fifo)) ? \
163 0 : \
164 ARRAY_SIZE((fifo).buf) - 1, \
165 .esize = sizeof(*(fifo).buf), \
166 .data = __is_kfifo_ptr(&(fifo)) ? \
167 NULL : \
168 (fifo).buf, \
169 } \
170 } \
171 }
172
173
174static inline unsigned int __must_check
175__kfifo_must_check_helper(unsigned int val)
176{
177 return val;
178}
179
180/**
181 * kfifo_initialized - Check if the fifo is initialized
182 * @fifo: address of the fifo to check
183 *
184 * Return %true if fifo is initialized, otherwise %false.
185 * Assumes the fifo was 0 before.
186 */
187#define kfifo_initialized(fifo) ((fifo)->kfifo.mask)
188
189/**
190 * kfifo_esize - returns the size of the element managed by the fifo
191 * @fifo: address of the fifo to be used
192 */
193#define kfifo_esize(fifo) ((fifo)->kfifo.esize)
194
195/**
196 * kfifo_recsize - returns the size of the record length field
197 * @fifo: address of the fifo to be used
198 */
199#define kfifo_recsize(fifo) (sizeof(*(fifo)->rectype))
200
201/**
202 * kfifo_size - returns the size of the fifo in elements
203 * @fifo: address of the fifo to be used
204 */
205#define kfifo_size(fifo) ((fifo)->kfifo.mask + 1)
206
207/**
208 * kfifo_reset - removes the entire fifo content
209 * @fifo: address of the fifo to be used
210 *
211 * Note: usage of kfifo_reset() is dangerous. It should be only called when the
212 * fifo is exclusived locked or when it is secured that no other thread is
213 * accessing the fifo.
214 */
215#define kfifo_reset(fifo) \
216(void)({ \
217 typeof(fifo + 1) __tmp = (fifo); \
218 __tmp->kfifo.in = __tmp->kfifo.out = 0; \
219})
220
221/**
222 * kfifo_reset_out - skip fifo content
223 * @fifo: address of the fifo to be used
224 *
225 * Note: The usage of kfifo_reset_out() is safe until it will be only called
226 * from the reader thread and there is only one concurrent reader. Otherwise
227 * it is dangerous and must be handled in the same way as kfifo_reset().
228 */
229#define kfifo_reset_out(fifo) \
230(void)({ \
231 typeof(fifo + 1) __tmp = (fifo); \
232 __tmp->kfifo.out = __tmp->kfifo.in; \
233})
234
235/**
236 * kfifo_len - returns the number of used elements in the fifo
237 * @fifo: address of the fifo to be used
238 */
239#define kfifo_len(fifo) \
240({ \
241 typeof(fifo + 1) __tmpl = (fifo); \
242 __tmpl->kfifo.in - __tmpl->kfifo.out; \
243})
244
245/**
246 * kfifo_is_empty - returns true if the fifo is empty
247 * @fifo: address of the fifo to be used
248 */
249#define kfifo_is_empty(fifo) \
250({ \
251 typeof(fifo + 1) __tmpq = (fifo); \
252 __tmpq->kfifo.in == __tmpq->kfifo.out; \
253})
254
255/**
256 * kfifo_is_full - returns true if the fifo is full
257 * @fifo: address of the fifo to be used
258 */
259#define kfifo_is_full(fifo) \
260({ \
261 typeof(fifo + 1) __tmpq = (fifo); \
262 kfifo_len(__tmpq) > __tmpq->kfifo.mask; \
263})
264
265/**
266 * kfifo_avail - returns the number of unused elements in the fifo
267 * @fifo: address of the fifo to be used
268 */
269#define kfifo_avail(fifo) \
270__kfifo_must_check_helper( \
271({ \
272 typeof(fifo + 1) __tmpq = (fifo); \
273 const size_t __recsize = sizeof(*__tmpq->rectype); \
274 unsigned int __avail = kfifo_size(__tmpq) - kfifo_len(__tmpq); \
275 (__recsize) ? ((__avail <= __recsize) ? 0 : \
276 __kfifo_max_r(__avail - __recsize, __recsize)) : \
277 __avail; \
278}) \
279)
280
281/**
282 * kfifo_skip - skip output data
283 * @fifo: address of the fifo to be used
284 */
285#define kfifo_skip(fifo) \
286(void)({ \
287 typeof(fifo + 1) __tmp = (fifo); \
288 const size_t __recsize = sizeof(*__tmp->rectype); \
289 struct __kfifo *__kfifo = &__tmp->kfifo; \
290 if (__recsize) \
291 __kfifo_skip_r(__kfifo, __recsize); \
292 else \
293 __kfifo->out++; \
294})
295
296/**
297 * kfifo_peek_len - gets the size of the next fifo record
298 * @fifo: address of the fifo to be used
299 *
300 * This function returns the size of the next fifo record in number of bytes.
301 */
302#define kfifo_peek_len(fifo) \
303__kfifo_must_check_helper( \
304({ \
305 typeof(fifo + 1) __tmp = (fifo); \
306 const size_t __recsize = sizeof(*__tmp->rectype); \
307 struct __kfifo *__kfifo = &__tmp->kfifo; \
308 (!__recsize) ? kfifo_len(__tmp) * sizeof(*__tmp->type) : \
309 __kfifo_len_r(__kfifo, __recsize); \
310}) \
311)
312
313/**
314 * kfifo_alloc - dynamically allocates a new fifo buffer
315 * @fifo: pointer to the fifo
316 * @size: the number of elements in the fifo, this must be a power of 2
317 * @gfp_mask: get_free_pages mask, passed to kmalloc()
318 *
319 * This macro dynamically allocates a new fifo buffer.
320 *
321 * The numer of elements will be rounded-up to a power of 2.
322 * The fifo will be release with kfifo_free().
323 * Return 0 if no error, otherwise an error code.
324 */
325#define kfifo_alloc(fifo, size, gfp_mask) \
326__kfifo_must_check_helper( \
327({ \
328 typeof(fifo + 1) __tmp = (fifo); \
329 struct __kfifo *__kfifo = &__tmp->kfifo; \
330 __is_kfifo_ptr(__tmp) ? \
331 __kfifo_alloc(__kfifo, size, sizeof(*__tmp->type), gfp_mask) : \
332 -EINVAL; \
333}) \
334)
335
336/**
337 * kfifo_free - frees the fifo
338 * @fifo: the fifo to be freed
339 */
340#define kfifo_free(fifo) \
341({ \
342 typeof(fifo + 1) __tmp = (fifo); \
343 struct __kfifo *__kfifo = &__tmp->kfifo; \
344 if (__is_kfifo_ptr(__tmp)) \
345 __kfifo_free(__kfifo); \
346})
347
348/**
349 * kfifo_init - initialize a fifo using a preallocated buffer
350 * @fifo: the fifo to assign the buffer
351 * @buffer: the preallocated buffer to be used
352 * @size: the size of the internal buffer, this have to be a power of 2
353 *
354 * This macro initialize a fifo using a preallocated buffer.
355 *
356 * The numer of elements will be rounded-up to a power of 2.
357 * Return 0 if no error, otherwise an error code.
358 */
359#define kfifo_init(fifo, buffer, size) \
360({ \
361 typeof(fifo + 1) __tmp = (fifo); \
362 struct __kfifo *__kfifo = &__tmp->kfifo; \
363 __is_kfifo_ptr(__tmp) ? \
364 __kfifo_init(__kfifo, buffer, size, sizeof(*__tmp->type)) : \
365 -EINVAL; \
366})
367
368/**
369 * kfifo_put - put data into the fifo
370 * @fifo: address of the fifo to be used
371 * @val: the data to be added
372 *
373 * This macro copies the given value into the fifo.
374 * It returns 0 if the fifo was full. Otherwise it returns the number
375 * processed elements.
376 *
377 * Note that with only one concurrent reader and one concurrent
378 * writer, you don't need extra locking to use these macro.
379 */
380#define kfifo_put(fifo, val) \
381({ \
382 typeof(fifo + 1) __tmp = (fifo); \
383 typeof(val + 1) __val = (val); \
384 unsigned int __ret; \
385 const size_t __recsize = sizeof(*__tmp->rectype); \
386 struct __kfifo *__kfifo = &__tmp->kfifo; \
387 if (0) { \
388 typeof(__tmp->ptr_const) __dummy __attribute__ ((unused)); \
389 __dummy = (typeof(__val))NULL; \
390 } \
391 if (__recsize) \
392 __ret = __kfifo_in_r(__kfifo, __val, sizeof(*__val), \
393 __recsize); \
394 else { \
395 __ret = !kfifo_is_full(__tmp); \
396 if (__ret) { \
397 (__is_kfifo_ptr(__tmp) ? \
398 ((typeof(__tmp->type))__kfifo->data) : \
399 (__tmp->buf) \
400 )[__kfifo->in & __tmp->kfifo.mask] = \
401 *(typeof(__tmp->type))__val; \
402 smp_wmb(); \
403 __kfifo->in++; \
404 } \
405 } \
406 __ret; \
407})
408
409/**
410 * kfifo_get - get data from the fifo
411 * @fifo: address of the fifo to be used
412 * @val: the var where to store the data to be added
413 *
414 * This macro reads the data from the fifo.
415 * It returns 0 if the fifo was empty. Otherwise it returns the number
416 * processed elements.
417 *
418 * Note that with only one concurrent reader and one concurrent
419 * writer, you don't need extra locking to use these macro.
420 */
421#define kfifo_get(fifo, val) \
422__kfifo_must_check_helper( \
423({ \
424 typeof(fifo + 1) __tmp = (fifo); \
425 typeof(val + 1) __val = (val); \
426 unsigned int __ret; \
427 const size_t __recsize = sizeof(*__tmp->rectype); \
428 struct __kfifo *__kfifo = &__tmp->kfifo; \
429 if (0) \
430 __val = (typeof(__tmp->ptr))0; \
431 if (__recsize) \
432 __ret = __kfifo_out_r(__kfifo, __val, sizeof(*__val), \
433 __recsize); \
434 else { \
435 __ret = !kfifo_is_empty(__tmp); \
436 if (__ret) { \
437 *(typeof(__tmp->type))__val = \
438 (__is_kfifo_ptr(__tmp) ? \
439 ((typeof(__tmp->type))__kfifo->data) : \
440 (__tmp->buf) \
441 )[__kfifo->out & __tmp->kfifo.mask]; \
442 smp_wmb(); \
443 __kfifo->out++; \
444 } \
445 } \
446 __ret; \
447}) \
448)
449
450/**
451 * kfifo_peek - get data from the fifo without removing
452 * @fifo: address of the fifo to be used
453 * @val: the var where to store the data to be added
454 *
455 * This reads the data from the fifo without removing it from the fifo.
456 * It returns 0 if the fifo was empty. Otherwise it returns the number
457 * processed elements.
458 *
459 * Note that with only one concurrent reader and one concurrent
460 * writer, you don't need extra locking to use these macro.
461 */
462#define kfifo_peek(fifo, val) \
463__kfifo_must_check_helper( \
464({ \
465 typeof(fifo + 1) __tmp = (fifo); \
466 typeof(val + 1) __val = (val); \
467 unsigned int __ret; \
468 const size_t __recsize = sizeof(*__tmp->rectype); \
469 struct __kfifo *__kfifo = &__tmp->kfifo; \
470 if (0) \
471 __val = (typeof(__tmp->ptr))NULL; \
472 if (__recsize) \
473 __ret = __kfifo_out_peek_r(__kfifo, __val, sizeof(*__val), \
474 __recsize); \
475 else { \
476 __ret = !kfifo_is_empty(__tmp); \
477 if (__ret) { \
478 *(typeof(__tmp->type))__val = \
479 (__is_kfifo_ptr(__tmp) ? \
480 ((typeof(__tmp->type))__kfifo->data) : \
481 (__tmp->buf) \
482 )[__kfifo->out & __tmp->kfifo.mask]; \
483 smp_wmb(); \
484 } \
485 } \
486 __ret; \
487}) \
488)
489
490/**
491 * kfifo_in - put data into the fifo
492 * @fifo: address of the fifo to be used
493 * @buf: the data to be added
494 * @n: number of elements to be added
495 *
496 * This macro copies the given buffer into the fifo and returns the
497 * number of copied elements.
498 *
499 * Note that with only one concurrent reader and one concurrent
500 * writer, you don't need extra locking to use these macro.
501 */
502#define kfifo_in(fifo, buf, n) \
503({ \
504 typeof(fifo + 1) __tmp = (fifo); \
505 typeof(buf + 1) __buf = (buf); \
506 unsigned long __n = (n); \
507 const size_t __recsize = sizeof(*__tmp->rectype); \
508 struct __kfifo *__kfifo = &__tmp->kfifo; \
509 if (0) { \
510 typeof(__tmp->ptr_const) __dummy __attribute__ ((unused)); \
511 __dummy = (typeof(__buf))NULL; \
512 } \
513 (__recsize) ?\
514 __kfifo_in_r(__kfifo, __buf, __n, __recsize) : \
515 __kfifo_in(__kfifo, __buf, __n); \
516})
517
518/**
519 * kfifo_in_spinlocked - put data into the fifo using a spinlock for locking
520 * @fifo: address of the fifo to be used
521 * @buf: the data to be added
522 * @n: number of elements to be added
523 * @lock: pointer to the spinlock to use for locking
524 *
525 * This macro copies the given values buffer into the fifo and returns the
526 * number of copied elements.
527 */
528#define kfifo_in_spinlocked(fifo, buf, n, lock) \
529({ \
530 unsigned long __flags; \
531 unsigned int __ret; \
532 spin_lock_irqsave(lock, __flags); \
533 __ret = kfifo_in(fifo, buf, n); \
534 spin_unlock_irqrestore(lock, __flags); \
535 __ret; \
536})
537
538/* alias for kfifo_in_spinlocked, will be removed in a future release */
539#define kfifo_in_locked(fifo, buf, n, lock) \
540 kfifo_in_spinlocked(fifo, buf, n, lock)
541
542/**
543 * kfifo_out - get data from the fifo
544 * @fifo: address of the fifo to be used
545 * @buf: pointer to the storage buffer
546 * @n: max. number of elements to get
547 *
548 * This macro get some data from the fifo and return the numbers of elements
549 * copied.
550 *
551 * Note that with only one concurrent reader and one concurrent
552 * writer, you don't need extra locking to use these macro.
553 */
554#define kfifo_out(fifo, buf, n) \
555__kfifo_must_check_helper( \
556({ \
557 typeof(fifo + 1) __tmp = (fifo); \
558 typeof(buf + 1) __buf = (buf); \
559 unsigned long __n = (n); \
560 const size_t __recsize = sizeof(*__tmp->rectype); \
561 struct __kfifo *__kfifo = &__tmp->kfifo; \
562 if (0) { \
563 typeof(__tmp->ptr) __dummy = NULL; \
564 __buf = __dummy; \
565 } \
566 (__recsize) ?\
567 __kfifo_out_r(__kfifo, __buf, __n, __recsize) : \
568 __kfifo_out(__kfifo, __buf, __n); \
569}) \
570)
571
572/**
573 * kfifo_out_spinlocked - get data from the fifo using a spinlock for locking
574 * @fifo: address of the fifo to be used
575 * @buf: pointer to the storage buffer
576 * @n: max. number of elements to get
577 * @lock: pointer to the spinlock to use for locking
578 *
579 * This macro get the data from the fifo and return the numbers of elements
580 * copied.
581 */
582#define kfifo_out_spinlocked(fifo, buf, n, lock) \
583__kfifo_must_check_helper( \
584({ \
585 unsigned long __flags; \
586 unsigned int __ret; \
587 spin_lock_irqsave(lock, __flags); \
588 __ret = kfifo_out(fifo, buf, n); \
589 spin_unlock_irqrestore(lock, __flags); \
590 __ret; \
591}) \
592)
593
594/* alias for kfifo_out_spinlocked, will be removed in a future release */
595#define kfifo_out_locked(fifo, buf, n, lock) \
596 kfifo_out_spinlocked(fifo, buf, n, lock)
597
598/**
599 * kfifo_from_user - puts some data from user space into the fifo
600 * @fifo: address of the fifo to be used
601 * @from: pointer to the data to be added
602 * @len: the length of the data to be added
603 * @copied: pointer to output variable to store the number of copied bytes
604 *
605 * This macro copies at most @len bytes from the @from into the
606 * fifo, depending of the available space and returns -EFAULT/0.
607 *
608 * Note that with only one concurrent reader and one concurrent
609 * writer, you don't need extra locking to use these macro.
610 */
611#define kfifo_from_user(fifo, from, len, copied) \
612__kfifo_must_check_helper( \
613({ \
614 typeof(fifo + 1) __tmp = (fifo); \
615 const void __user *__from = (from); \
616 unsigned int __len = (len); \
617 unsigned int *__copied = (copied); \
618 const size_t __recsize = sizeof(*__tmp->rectype); \
619 struct __kfifo *__kfifo = &__tmp->kfifo; \
620 (__recsize) ? \
621 __kfifo_from_user_r(__kfifo, __from, __len, __copied, __recsize) : \
622 __kfifo_from_user(__kfifo, __from, __len, __copied); \
623}) \
624)
625
626/**
627 * kfifo_to_user - copies data from the fifo into user space
628 * @fifo: address of the fifo to be used
629 * @to: where the data must be copied
630 * @len: the size of the destination buffer
631 * @copied: pointer to output variable to store the number of copied bytes
632 *
633 * This macro copies at most @len bytes from the fifo into the
634 * @to buffer and returns -EFAULT/0.
635 *
636 * Note that with only one concurrent reader and one concurrent
637 * writer, you don't need extra locking to use these macro.
638 */
639#define kfifo_to_user(fifo, to, len, copied) \
640__kfifo_must_check_helper( \
641({ \
642 typeof(fifo + 1) __tmp = (fifo); \
643 void __user *__to = (to); \
644 unsigned int __len = (len); \
645 unsigned int *__copied = (copied); \
646 const size_t __recsize = sizeof(*__tmp->rectype); \
647 struct __kfifo *__kfifo = &__tmp->kfifo; \
648 (__recsize) ? \
649 __kfifo_to_user_r(__kfifo, __to, __len, __copied, __recsize) : \
650 __kfifo_to_user(__kfifo, __to, __len, __copied); \
651}) \
652)
653
654/**
655 * kfifo_dma_in_prepare - setup a scatterlist for DMA input
656 * @fifo: address of the fifo to be used
657 * @sgl: pointer to the scatterlist array
658 * @nents: number of entries in the scatterlist array
659 * @len: number of elements to transfer
660 *
661 * This macro fills a scatterlist for DMA input.
662 * It returns the number entries in the scatterlist array.
663 *
664 * Note that with only one concurrent reader and one concurrent
665 * writer, you don't need extra locking to use these macros.
666 */
667#define kfifo_dma_in_prepare(fifo, sgl, nents, len) \
668({ \
669 typeof(fifo + 1) __tmp = (fifo); \
670 struct scatterlist *__sgl = (sgl); \
671 int __nents = (nents); \
672 unsigned int __len = (len); \
673 const size_t __recsize = sizeof(*__tmp->rectype); \
674 struct __kfifo *__kfifo = &__tmp->kfifo; \
675 (__recsize) ? \
676 __kfifo_dma_in_prepare_r(__kfifo, __sgl, __nents, __len, __recsize) : \
677 __kfifo_dma_in_prepare(__kfifo, __sgl, __nents, __len); \
678})
679
680/**
681 * kfifo_dma_in_finish - finish a DMA IN operation
682 * @fifo: address of the fifo to be used
683 * @len: number of bytes to received
684 *
685 * This macro finish a DMA IN operation. The in counter will be updated by
686 * the len parameter. No error checking will be done.
687 *
688 * Note that with only one concurrent reader and one concurrent
689 * writer, you don't need extra locking to use these macros.
690 */
691#define kfifo_dma_in_finish(fifo, len) \
692(void)({ \
693 typeof(fifo + 1) __tmp = (fifo); \
694 unsigned int __len = (len); \
695 const size_t __recsize = sizeof(*__tmp->rectype); \
696 struct __kfifo *__kfifo = &__tmp->kfifo; \
697 if (__recsize) \
698 __kfifo_dma_in_finish_r(__kfifo, __len, __recsize); \
699 else \
700 __kfifo->in += __len / sizeof(*__tmp->type); \
701})
702
703/**
704 * kfifo_dma_out_prepare - setup a scatterlist for DMA output
705 * @fifo: address of the fifo to be used
706 * @sgl: pointer to the scatterlist array
707 * @nents: number of entries in the scatterlist array
708 * @len: number of elements to transfer
709 *
710 * This macro fills a scatterlist for DMA output which at most @len bytes
711 * to transfer.
712 * It returns the number entries in the scatterlist array.
713 * A zero means there is no space available and the scatterlist is not filled.
714 *
715 * Note that with only one concurrent reader and one concurrent
716 * writer, you don't need extra locking to use these macros.
717 */
718#define kfifo_dma_out_prepare(fifo, sgl, nents, len) \
719({ \
720 typeof(fifo + 1) __tmp = (fifo); \
721 struct scatterlist *__sgl = (sgl); \
722 int __nents = (nents); \
723 unsigned int __len = (len); \
724 const size_t __recsize = sizeof(*__tmp->rectype); \
725 struct __kfifo *__kfifo = &__tmp->kfifo; \
726 (__recsize) ? \
727 __kfifo_dma_out_prepare_r(__kfifo, __sgl, __nents, __len, __recsize) : \
728 __kfifo_dma_out_prepare(__kfifo, __sgl, __nents, __len); \
729})
730
731/**
732 * kfifo_dma_out_finish - finish a DMA OUT operation
733 * @fifo: address of the fifo to be used
734 * @len: number of bytes transferd
735 *
736 * This macro finish a DMA OUT operation. The out counter will be updated by
737 * the len parameter. No error checking will be done.
738 *
739 * Note that with only one concurrent reader and one concurrent
740 * writer, you don't need extra locking to use these macros.
741 */
742#define kfifo_dma_out_finish(fifo, len) \
743(void)({ \
744 typeof(fifo + 1) __tmp = (fifo); \
745 unsigned int __len = (len); \
746 const size_t __recsize = sizeof(*__tmp->rectype); \
747 struct __kfifo *__kfifo = &__tmp->kfifo; \
748 if (__recsize) \
749 __kfifo_dma_out_finish_r(__kfifo, __recsize); \
750 else \
751 __kfifo->out += __len / sizeof(*__tmp->type); \
752})
753
754/**
755 * kfifo_out_peek - gets some data from the fifo
756 * @fifo: address of the fifo to be used
757 * @buf: pointer to the storage buffer
758 * @n: max. number of elements to get
759 *
760 * This macro get the data from the fifo and return the numbers of elements
761 * copied. The data is not removed from the fifo.
762 *
763 * Note that with only one concurrent reader and one concurrent
764 * writer, you don't need extra locking to use these macro.
765 */
766#define kfifo_out_peek(fifo, buf, n) \
767__kfifo_must_check_helper( \
768({ \
769 typeof(fifo + 1) __tmp = (fifo); \
770 typeof(buf + 1) __buf = (buf); \
771 unsigned long __n = (n); \
772 const size_t __recsize = sizeof(*__tmp->rectype); \
773 struct __kfifo *__kfifo = &__tmp->kfifo; \
774 if (0) { \
775 typeof(__tmp->ptr) __dummy __attribute__ ((unused)) = NULL; \
776 __buf = __dummy; \
777 } \
778 (__recsize) ? \
779 __kfifo_out_peek_r(__kfifo, __buf, __n, __recsize) : \
780 __kfifo_out_peek(__kfifo, __buf, __n); \
781}) \
782)
783
784extern int __kfifo_alloc(struct __kfifo *fifo, unsigned int size,
785 size_t esize, gfp_t gfp_mask);
786
787extern void __kfifo_free(struct __kfifo *fifo);
788
789extern int __kfifo_init(struct __kfifo *fifo, void *buffer,
790 unsigned int size, size_t esize);
791
792extern unsigned int __kfifo_in(struct __kfifo *fifo,
793 const void *buf, unsigned int len);
794
795extern unsigned int __kfifo_out(struct __kfifo *fifo,
796 void *buf, unsigned int len);
797
798extern int __kfifo_from_user(struct __kfifo *fifo,
799 const void __user *from, unsigned long len, unsigned int *copied);
800
801extern int __kfifo_to_user(struct __kfifo *fifo,
802 void __user *to, unsigned long len, unsigned int *copied);
803
804extern unsigned int __kfifo_dma_in_prepare(struct __kfifo *fifo,
805 struct scatterlist *sgl, int nents, unsigned int len);
806
807extern unsigned int __kfifo_dma_out_prepare(struct __kfifo *fifo,
808 struct scatterlist *sgl, int nents, unsigned int len);
809
810extern unsigned int __kfifo_out_peek(struct __kfifo *fifo,
811 void *buf, unsigned int len);
812
813extern unsigned int __kfifo_in_r(struct __kfifo *fifo,
814 const void *buf, unsigned int len, size_t recsize);
815
816extern unsigned int __kfifo_out_r(struct __kfifo *fifo,
817 void *buf, unsigned int len, size_t recsize);
818
819extern int __kfifo_from_user_r(struct __kfifo *fifo,
820 const void __user *from, unsigned long len, unsigned int *copied,
821 size_t recsize);
822
823extern int __kfifo_to_user_r(struct __kfifo *fifo, void __user *to,
824 unsigned long len, unsigned int *copied, size_t recsize);
825
826extern unsigned int __kfifo_dma_in_prepare_r(struct __kfifo *fifo,
827 struct scatterlist *sgl, int nents, unsigned int len, size_t recsize);
828
829extern void __kfifo_dma_in_finish_r(struct __kfifo *fifo,
830 unsigned int len, size_t recsize);
831
832extern unsigned int __kfifo_dma_out_prepare_r(struct __kfifo *fifo,
833 struct scatterlist *sgl, int nents, unsigned int len, size_t recsize);
834
835extern void __kfifo_dma_out_finish_r(struct __kfifo *fifo, size_t recsize);
836
837extern unsigned int __kfifo_len_r(struct __kfifo *fifo, size_t recsize);
838
839extern unsigned int __kfifo_out_peek_r(struct __kfifo *fifo,
840 void *buf, unsigned int len, size_t recsize);
841
842extern unsigned int __kfifo_max_r(unsigned int len, size_t recsize);
843
844#endif
diff --git a/include/linux/kfifo.h b/include/linux/kfifo.h
index 57c4eedf4dd6..311f8753d713 100644
--- a/include/linux/kfifo.h
+++ b/include/linux/kfifo.h
@@ -1,8 +1,7 @@
1/* 1/*
2 * A generic kernel FIFO implementation. 2 * A generic kernel FIFO implementation
3 * 3 *
4 * Copyright (C) 2009 Stefani Seibold <stefani@seibold.net> 4 * Copyright (C) 2009/2010 Stefani Seibold <stefani@seibold.net>
5 * Copyright (C) 2004 Stelian Pop <stelian@popies.net>
6 * 5 *
7 * This program is free software; you can redistribute it and/or modify 6 * This program is free software; you can redistribute it and/or modify
8 * it under the terms of the GNU General Public License as published by 7 * it under the terms of the GNU General Public License as published by
@@ -20,8 +19,11 @@
20 * 19 *
21 */ 20 */
22 21
22#ifndef _LINUX_KFIFO_H
23#define _LINUX_KFIFO_H
24
23/* 25/*
24 * Howto porting drivers to the new generic fifo API: 26 * How to porting drivers to the new generic FIFO API:
25 * 27 *
26 * - Modify the declaration of the "struct kfifo *" object into a 28 * - Modify the declaration of the "struct kfifo *" object into a
27 * in-place "struct kfifo" object 29 * in-place "struct kfifo" object
@@ -30,586 +32,813 @@
30 * passed as the first argument to this functions 32 * passed as the first argument to this functions
31 * - Replace the use of __kfifo_put into kfifo_in and __kfifo_get 33 * - Replace the use of __kfifo_put into kfifo_in and __kfifo_get
32 * into kfifo_out 34 * into kfifo_out
33 * - Replace the use of kfifo_put into kfifo_in_locked and kfifo_get 35 * - Replace the use of kfifo_put into kfifo_in_spinlocked and kfifo_get
34 * into kfifo_out_locked 36 * into kfifo_out_spinlocked
35 * Note: the spinlock pointer formerly passed to kfifo_init/kfifo_alloc 37 * Note: the spinlock pointer formerly passed to kfifo_init/kfifo_alloc
36 * must be passed now to the kfifo_in_locked and kfifo_out_locked 38 * must be passed now to the kfifo_in_spinlocked and kfifo_out_spinlocked
37 * as the last parameter. 39 * as the last parameter
38 * - All formerly name __kfifo_* functions has been renamed into kfifo_* 40 * - The formerly __kfifo_* functions are renamed into kfifo_*
39 */ 41 */
40 42
41#ifndef _LINUX_KFIFO_H 43/*
42#define _LINUX_KFIFO_H 44 * Note about locking : There is no locking required until only * one reader
45 * and one writer is using the fifo and no kfifo_reset() will be * called
46 * kfifo_reset_out() can be safely used, until it will be only called
47 * in the reader thread.
48 * For multiple writer and one reader there is only a need to lock the writer.
49 * And vice versa for only one writer and multiple reader there is only a need
50 * to lock the reader.
51 */
43 52
44#include <linux/kernel.h> 53#include <linux/kernel.h>
45#include <linux/spinlock.h> 54#include <linux/spinlock.h>
46 55#include <linux/stddef.h>
47struct kfifo { 56#include <linux/scatterlist.h>
48 unsigned char *buffer; /* the buffer holding the data */ 57
49 unsigned int size; /* the size of the allocated buffer */ 58struct __kfifo {
50 unsigned int in; /* data is added at offset (in % size) */ 59 unsigned int in;
51 unsigned int out; /* data is extracted from off. (out % size) */ 60 unsigned int out;
61 unsigned int mask;
62 unsigned int esize;
63 void *data;
52}; 64};
53 65
54/* 66#define __STRUCT_KFIFO_COMMON(datatype, recsize, ptrtype) \
55 * Macros for declaration and initialization of the kfifo datatype 67 union { \
56 */ 68 struct __kfifo kfifo; \
57 69 datatype *type; \
58/* helper macro */ 70 char (*rectype)[recsize]; \
59#define __kfifo_initializer(s, b) \ 71 ptrtype *ptr; \
60 (struct kfifo) { \ 72 const ptrtype *ptr_const; \
61 .size = s, \
62 .in = 0, \
63 .out = 0, \
64 .buffer = b \
65 } 73 }
66 74
67/** 75#define __STRUCT_KFIFO(type, size, recsize, ptrtype) \
68 * DECLARE_KFIFO - macro to declare a kfifo and the associated buffer 76{ \
69 * @name: name of the declared kfifo datatype 77 __STRUCT_KFIFO_COMMON(type, recsize, ptrtype); \
70 * @size: size of the fifo buffer. Must be a power of two. 78 type buf[((size < 2) || (size & (size - 1))) ? -1 : size]; \
71 *
72 * Note1: the macro can be used inside struct or union declaration
73 * Note2: the macro creates two objects:
74 * A kfifo object with the given name and a buffer for the kfifo
75 * object named name##kfifo_buffer
76 */
77#define DECLARE_KFIFO(name, size) \
78union { \
79 struct kfifo name; \
80 unsigned char name##kfifo_buffer[size + sizeof(struct kfifo)]; \
81} 79}
82 80
83/** 81#define STRUCT_KFIFO(type, size) \
84 * INIT_KFIFO - Initialize a kfifo declared by DECLARE_KFIFO 82 struct __STRUCT_KFIFO(type, size, 0, type)
85 * @name: name of the declared kfifo datatype 83
84#define __STRUCT_KFIFO_PTR(type, recsize, ptrtype) \
85{ \
86 __STRUCT_KFIFO_COMMON(type, recsize, ptrtype); \
87 type buf[0]; \
88}
89
90#define STRUCT_KFIFO_PTR(type) \
91 struct __STRUCT_KFIFO_PTR(type, 0, type)
92
93/*
94 * define compatibility "struct kfifo" for dynamic allocated fifos
86 */ 95 */
87#define INIT_KFIFO(name) \ 96struct kfifo __STRUCT_KFIFO_PTR(unsigned char, 0, void);
88 name = __kfifo_initializer(sizeof(name##kfifo_buffer) - \
89 sizeof(struct kfifo), \
90 name##kfifo_buffer + sizeof(struct kfifo))
91 97
92/** 98#define STRUCT_KFIFO_REC_1(size) \
93 * DEFINE_KFIFO - macro to define and initialize a kfifo 99 struct __STRUCT_KFIFO(unsigned char, size, 1, void)
94 * @name: name of the declared kfifo datatype 100
95 * @size: size of the fifo buffer. Must be a power of two. 101#define STRUCT_KFIFO_REC_2(size) \
96 * 102 struct __STRUCT_KFIFO(unsigned char, size, 2, void)
97 * Note1: the macro can be used for global and local kfifo data type variables 103
98 * Note2: the macro creates two objects: 104/*
99 * A kfifo object with the given name and a buffer for the kfifo 105 * define kfifo_rec types
100 * object named name##kfifo_buffer
101 */ 106 */
102#define DEFINE_KFIFO(name, size) \ 107struct kfifo_rec_ptr_1 __STRUCT_KFIFO_PTR(unsigned char, 1, void);
103 unsigned char name##kfifo_buffer[size]; \ 108struct kfifo_rec_ptr_2 __STRUCT_KFIFO_PTR(unsigned char, 2, void);
104 struct kfifo name = __kfifo_initializer(size, name##kfifo_buffer)
105 109
106extern void kfifo_init(struct kfifo *fifo, void *buffer, 110/*
107 unsigned int size); 111 * helper macro to distinguish between real in place fifo where the fifo
108extern __must_check int kfifo_alloc(struct kfifo *fifo, unsigned int size, 112 * array is a part of the structure and the fifo type where the array is
109 gfp_t gfp_mask); 113 * outside of the fifo structure.
110extern void kfifo_free(struct kfifo *fifo); 114 */
111extern unsigned int kfifo_in(struct kfifo *fifo, 115#define __is_kfifo_ptr(fifo) (sizeof(*fifo) == sizeof(struct __kfifo))
112 const void *from, unsigned int len);
113extern __must_check unsigned int kfifo_out(struct kfifo *fifo,
114 void *to, unsigned int len);
115extern __must_check unsigned int kfifo_out_peek(struct kfifo *fifo,
116 void *to, unsigned int len, unsigned offset);
117 116
118/** 117/**
119 * kfifo_initialized - Check if kfifo is initialized. 118 * DECLARE_KFIFO_PTR - macro to declare a fifo pointer object
120 * @fifo: fifo to check 119 * @fifo: name of the declared fifo
121 * Return %true if FIFO is initialized, otherwise %false. 120 * @type: type of the fifo elements
122 * Assumes the fifo was 0 before.
123 */ 121 */
124static inline bool kfifo_initialized(struct kfifo *fifo) 122#define DECLARE_KFIFO_PTR(fifo, type) STRUCT_KFIFO_PTR(type) fifo
125{
126 return fifo->buffer != NULL;
127}
128 123
129/** 124/**
130 * kfifo_reset - removes the entire FIFO contents 125 * DECLARE_KFIFO - macro to declare a fifo object
131 * @fifo: the fifo to be emptied. 126 * @fifo: name of the declared fifo
127 * @type: type of the fifo elements
128 * @size: the number of elements in the fifo, this must be a power of 2
132 */ 129 */
133static inline void kfifo_reset(struct kfifo *fifo) 130#define DECLARE_KFIFO(fifo, type, size) STRUCT_KFIFO(type, size) fifo
134{
135 fifo->in = fifo->out = 0;
136}
137 131
138/** 132/**
139 * kfifo_reset_out - skip FIFO contents 133 * INIT_KFIFO - Initialize a fifo declared by DECLARE_KFIFO
140 * @fifo: the fifo to be emptied. 134 * @fifo: name of the declared fifo datatype
141 */ 135 */
142static inline void kfifo_reset_out(struct kfifo *fifo) 136#define INIT_KFIFO(fifo) \
143{ 137(void)({ \
144 smp_mb(); 138 typeof(&(fifo)) __tmp = &(fifo); \
145 fifo->out = fifo->in; 139 struct __kfifo *__kfifo = &__tmp->kfifo; \
146} 140 __kfifo->in = 0; \
141 __kfifo->out = 0; \
142 __kfifo->mask = __is_kfifo_ptr(__tmp) ? 0 : ARRAY_SIZE(__tmp->buf) - 1;\
143 __kfifo->esize = sizeof(*__tmp->buf); \
144 __kfifo->data = __is_kfifo_ptr(__tmp) ? NULL : __tmp->buf; \
145})
147 146
148/** 147/**
149 * kfifo_size - returns the size of the fifo in bytes 148 * DEFINE_KFIFO - macro to define and initialize a fifo
150 * @fifo: the fifo to be used. 149 * @fifo: name of the declared fifo datatype
151 */ 150 * @type: type of the fifo elements
152static inline __must_check unsigned int kfifo_size(struct kfifo *fifo) 151 * @size: the number of elements in the fifo, this must be a power of 2
152 *
153 * Note: the macro can be used for global and local fifo data type variables.
154 */
155#define DEFINE_KFIFO(fifo, type, size) \
156 DECLARE_KFIFO(fifo, type, size) = \
157 (typeof(fifo)) { \
158 { \
159 { \
160 .in = 0, \
161 .out = 0, \
162 .mask = __is_kfifo_ptr(&(fifo)) ? \
163 0 : \
164 ARRAY_SIZE((fifo).buf) - 1, \
165 .esize = sizeof(*(fifo).buf), \
166 .data = __is_kfifo_ptr(&(fifo)) ? \
167 NULL : \
168 (fifo).buf, \
169 } \
170 } \
171 }
172
173
174static inline unsigned int __must_check
175__kfifo_must_check_helper(unsigned int val)
153{ 176{
154 return fifo->size; 177 return val;
155} 178}
156 179
157/** 180/**
158 * kfifo_len - returns the number of used bytes in the FIFO 181 * kfifo_initialized - Check if the fifo is initialized
159 * @fifo: the fifo to be used. 182 * @fifo: address of the fifo to check
183 *
184 * Return %true if fifo is initialized, otherwise %false.
185 * Assumes the fifo was 0 before.
160 */ 186 */
161static inline unsigned int kfifo_len(struct kfifo *fifo) 187#define kfifo_initialized(fifo) ((fifo)->kfifo.mask)
162{
163 register unsigned int out;
164
165 out = fifo->out;
166 smp_rmb();
167 return fifo->in - out;
168}
169 188
170/** 189/**
171 * kfifo_is_empty - returns true if the fifo is empty 190 * kfifo_esize - returns the size of the element managed by the fifo
172 * @fifo: the fifo to be used. 191 * @fifo: address of the fifo to be used
173 */ 192 */
174static inline __must_check bool kfifo_is_empty(struct kfifo *fifo) 193#define kfifo_esize(fifo) ((fifo)->kfifo.esize)
175{
176 return fifo->in == fifo->out;
177}
178 194
179/** 195/**
180 * kfifo_is_full - returns true if the fifo is full 196 * kfifo_recsize - returns the size of the record length field
181 * @fifo: the fifo to be used. 197 * @fifo: address of the fifo to be used
182 */ 198 */
183static inline __must_check bool kfifo_is_full(struct kfifo *fifo) 199#define kfifo_recsize(fifo) (sizeof(*(fifo)->rectype))
184{
185 return kfifo_len(fifo) == kfifo_size(fifo);
186}
187 200
188/** 201/**
189 * kfifo_avail - returns the number of bytes available in the FIFO 202 * kfifo_size - returns the size of the fifo in elements
190 * @fifo: the fifo to be used. 203 * @fifo: address of the fifo to be used
191 */ 204 */
192static inline __must_check unsigned int kfifo_avail(struct kfifo *fifo) 205#define kfifo_size(fifo) ((fifo)->kfifo.mask + 1)
193{
194 return kfifo_size(fifo) - kfifo_len(fifo);
195}
196 206
197/** 207/**
198 * kfifo_in_locked - puts some data into the FIFO using a spinlock for locking 208 * kfifo_reset - removes the entire fifo content
199 * @fifo: the fifo to be used. 209 * @fifo: address of the fifo to be used
200 * @from: the data to be added.
201 * @n: the length of the data to be added.
202 * @lock: pointer to the spinlock to use for locking.
203 * 210 *
204 * This function copies at most @n bytes from the @from buffer into 211 * Note: usage of kfifo_reset() is dangerous. It should be only called when the
205 * the FIFO depending on the free space, and returns the number of 212 * fifo is exclusived locked or when it is secured that no other thread is
206 * bytes copied. 213 * accessing the fifo.
207 */ 214 */
208static inline unsigned int kfifo_in_locked(struct kfifo *fifo, 215#define kfifo_reset(fifo) \
209 const void *from, unsigned int n, spinlock_t *lock) 216(void)({ \
210{ 217 typeof(fifo + 1) __tmp = (fifo); \
211 unsigned long flags; 218 __tmp->kfifo.in = __tmp->kfifo.out = 0; \
212 unsigned int ret; 219})
213
214 spin_lock_irqsave(lock, flags);
215
216 ret = kfifo_in(fifo, from, n);
217
218 spin_unlock_irqrestore(lock, flags);
219
220 return ret;
221}
222 220
223/** 221/**
224 * kfifo_out_locked - gets some data from the FIFO using a spinlock for locking 222 * kfifo_reset_out - skip fifo content
225 * @fifo: the fifo to be used. 223 * @fifo: address of the fifo to be used
226 * @to: where the data must be copied.
227 * @n: the size of the destination buffer.
228 * @lock: pointer to the spinlock to use for locking.
229 * 224 *
230 * This function copies at most @n bytes from the FIFO into the 225 * Note: The usage of kfifo_reset_out() is safe until it will be only called
231 * @to buffer and returns the number of copied bytes. 226 * from the reader thread and there is only one concurrent reader. Otherwise
232 */ 227 * it is dangerous and must be handled in the same way as kfifo_reset().
233static inline __must_check unsigned int kfifo_out_locked(struct kfifo *fifo,
234 void *to, unsigned int n, spinlock_t *lock)
235{
236 unsigned long flags;
237 unsigned int ret;
238
239 spin_lock_irqsave(lock, flags);
240
241 ret = kfifo_out(fifo, to, n);
242
243 spin_unlock_irqrestore(lock, flags);
244
245 return ret;
246}
247
248extern void kfifo_skip(struct kfifo *fifo, unsigned int len);
249
250extern __must_check int kfifo_from_user(struct kfifo *fifo,
251 const void __user *from, unsigned int n, unsigned *lenout);
252
253extern __must_check int kfifo_to_user(struct kfifo *fifo,
254 void __user *to, unsigned int n, unsigned *lenout);
255
256/*
257 * __kfifo_add_out internal helper function for updating the out offset
258 */ 228 */
259static inline void __kfifo_add_out(struct kfifo *fifo, 229#define kfifo_reset_out(fifo) \
260 unsigned int off) 230(void)({ \
261{ 231 typeof(fifo + 1) __tmp = (fifo); \
262 smp_mb(); 232 __tmp->kfifo.out = __tmp->kfifo.in; \
263 fifo->out += off; 233})
264}
265 234
266/* 235/**
267 * __kfifo_add_in internal helper function for updating the in offset 236 * kfifo_len - returns the number of used elements in the fifo
237 * @fifo: address of the fifo to be used
268 */ 238 */
269static inline void __kfifo_add_in(struct kfifo *fifo, 239#define kfifo_len(fifo) \
270 unsigned int off) 240({ \
271{ 241 typeof(fifo + 1) __tmpl = (fifo); \
272 smp_wmb(); 242 __tmpl->kfifo.in - __tmpl->kfifo.out; \
273 fifo->in += off; 243})
274}
275 244
276/* 245/**
277 * __kfifo_off internal helper function for calculating the index of a 246 * kfifo_is_empty - returns true if the fifo is empty
278 * given offeset 247 * @fifo: address of the fifo to be used
279 */ 248 */
280static inline unsigned int __kfifo_off(struct kfifo *fifo, unsigned int off) 249#define kfifo_is_empty(fifo) \
281{ 250({ \
282 return off & (fifo->size - 1); 251 typeof(fifo + 1) __tmpq = (fifo); \
283} 252 __tmpq->kfifo.in == __tmpq->kfifo.out; \
253})
284 254
285/* 255/**
286 * __kfifo_peek_n internal helper function for determinate the length of 256 * kfifo_is_full - returns true if the fifo is full
287 * the next record in the fifo 257 * @fifo: address of the fifo to be used
288 */ 258 */
289static inline unsigned int __kfifo_peek_n(struct kfifo *fifo, 259#define kfifo_is_full(fifo) \
290 unsigned int recsize) 260({ \
291{ 261 typeof(fifo + 1) __tmpq = (fifo); \
292#define __KFIFO_GET(fifo, off, shift) \ 262 kfifo_len(__tmpq) > __tmpq->kfifo.mask; \
293 ((fifo)->buffer[__kfifo_off((fifo), (fifo)->out+(off))] << (shift)) 263})
294 264
295 unsigned int l; 265/**
266 * kfifo_avail - returns the number of unused elements in the fifo
267 * @fifo: address of the fifo to be used
268 */
269#define kfifo_avail(fifo) \
270__kfifo_must_check_helper( \
271({ \
272 typeof(fifo + 1) __tmpq = (fifo); \
273 const size_t __recsize = sizeof(*__tmpq->rectype); \
274 unsigned int __avail = kfifo_size(__tmpq) - kfifo_len(__tmpq); \
275 (__recsize) ? ((__avail <= __recsize) ? 0 : \
276 __kfifo_max_r(__avail - __recsize, __recsize)) : \
277 __avail; \
278}) \
279)
296 280
297 l = __KFIFO_GET(fifo, 0, 0); 281/**
282 * kfifo_skip - skip output data
283 * @fifo: address of the fifo to be used
284 */
285#define kfifo_skip(fifo) \
286(void)({ \
287 typeof(fifo + 1) __tmp = (fifo); \
288 const size_t __recsize = sizeof(*__tmp->rectype); \
289 struct __kfifo *__kfifo = &__tmp->kfifo; \
290 if (__recsize) \
291 __kfifo_skip_r(__kfifo, __recsize); \
292 else \
293 __kfifo->out++; \
294})
298 295
299 if (--recsize) 296/**
300 l |= __KFIFO_GET(fifo, 1, 8); 297 * kfifo_peek_len - gets the size of the next fifo record
298 * @fifo: address of the fifo to be used
299 *
300 * This function returns the size of the next fifo record in number of bytes.
301 */
302#define kfifo_peek_len(fifo) \
303__kfifo_must_check_helper( \
304({ \
305 typeof(fifo + 1) __tmp = (fifo); \
306 const size_t __recsize = sizeof(*__tmp->rectype); \
307 struct __kfifo *__kfifo = &__tmp->kfifo; \
308 (!__recsize) ? kfifo_len(__tmp) * sizeof(*__tmp->type) : \
309 __kfifo_len_r(__kfifo, __recsize); \
310}) \
311)
301 312
302 return l; 313/**
303#undef __KFIFO_GET 314 * kfifo_alloc - dynamically allocates a new fifo buffer
304} 315 * @fifo: pointer to the fifo
316 * @size: the number of elements in the fifo, this must be a power of 2
317 * @gfp_mask: get_free_pages mask, passed to kmalloc()
318 *
319 * This macro dynamically allocates a new fifo buffer.
320 *
321 * The numer of elements will be rounded-up to a power of 2.
322 * The fifo will be release with kfifo_free().
323 * Return 0 if no error, otherwise an error code.
324 */
325#define kfifo_alloc(fifo, size, gfp_mask) \
326__kfifo_must_check_helper( \
327({ \
328 typeof(fifo + 1) __tmp = (fifo); \
329 struct __kfifo *__kfifo = &__tmp->kfifo; \
330 __is_kfifo_ptr(__tmp) ? \
331 __kfifo_alloc(__kfifo, size, sizeof(*__tmp->type), gfp_mask) : \
332 -EINVAL; \
333}) \
334)
305 335
306/* 336/**
307 * __kfifo_poke_n internal helper function for storing the length of 337 * kfifo_free - frees the fifo
308 * the next record into the fifo 338 * @fifo: the fifo to be freed
309 */ 339 */
310static inline void __kfifo_poke_n(struct kfifo *fifo, 340#define kfifo_free(fifo) \
311 unsigned int recsize, unsigned int n) 341({ \
312{ 342 typeof(fifo + 1) __tmp = (fifo); \
313#define __KFIFO_PUT(fifo, off, val, shift) \ 343 struct __kfifo *__kfifo = &__tmp->kfifo; \
314 ( \ 344 if (__is_kfifo_ptr(__tmp)) \
315 (fifo)->buffer[__kfifo_off((fifo), (fifo)->in+(off))] = \ 345 __kfifo_free(__kfifo); \
316 (unsigned char)((val) >> (shift)) \ 346})
317 )
318 347
319 __KFIFO_PUT(fifo, 0, n, 0); 348/**
349 * kfifo_init - initialize a fifo using a preallocated buffer
350 * @fifo: the fifo to assign the buffer
351 * @buffer: the preallocated buffer to be used
352 * @size: the size of the internal buffer, this have to be a power of 2
353 *
354 * This macro initialize a fifo using a preallocated buffer.
355 *
356 * The numer of elements will be rounded-up to a power of 2.
357 * Return 0 if no error, otherwise an error code.
358 */
359#define kfifo_init(fifo, buffer, size) \
360({ \
361 typeof(fifo + 1) __tmp = (fifo); \
362 struct __kfifo *__kfifo = &__tmp->kfifo; \
363 __is_kfifo_ptr(__tmp) ? \
364 __kfifo_init(__kfifo, buffer, size, sizeof(*__tmp->type)) : \
365 -EINVAL; \
366})
320 367
321 if (--recsize) 368/**
322 __KFIFO_PUT(fifo, 1, n, 8); 369 * kfifo_put - put data into the fifo
323#undef __KFIFO_PUT 370 * @fifo: address of the fifo to be used
324} 371 * @val: the data to be added
372 *
373 * This macro copies the given value into the fifo.
374 * It returns 0 if the fifo was full. Otherwise it returns the number
375 * processed elements.
376 *
377 * Note that with only one concurrent reader and one concurrent
378 * writer, you don't need extra locking to use these macro.
379 */
380#define kfifo_put(fifo, val) \
381({ \
382 typeof(fifo + 1) __tmp = (fifo); \
383 typeof(val + 1) __val = (val); \
384 unsigned int __ret; \
385 const size_t __recsize = sizeof(*__tmp->rectype); \
386 struct __kfifo *__kfifo = &__tmp->kfifo; \
387 if (0) { \
388 typeof(__tmp->ptr_const) __dummy __attribute__ ((unused)); \
389 __dummy = (typeof(__val))NULL; \
390 } \
391 if (__recsize) \
392 __ret = __kfifo_in_r(__kfifo, __val, sizeof(*__val), \
393 __recsize); \
394 else { \
395 __ret = !kfifo_is_full(__tmp); \
396 if (__ret) { \
397 (__is_kfifo_ptr(__tmp) ? \
398 ((typeof(__tmp->type))__kfifo->data) : \
399 (__tmp->buf) \
400 )[__kfifo->in & __tmp->kfifo.mask] = \
401 *(typeof(__tmp->type))__val; \
402 smp_wmb(); \
403 __kfifo->in++; \
404 } \
405 } \
406 __ret; \
407})
325 408
326/* 409/**
327 * __kfifo_in_... internal functions for put date into the fifo 410 * kfifo_get - get data from the fifo
328 * do not call it directly, use kfifo_in_rec() instead 411 * @fifo: address of the fifo to be used
329 */ 412 * @val: the var where to store the data to be added
330extern unsigned int __kfifo_in_n(struct kfifo *fifo, 413 *
331 const void *from, unsigned int n, unsigned int recsize); 414 * This macro reads the data from the fifo.
415 * It returns 0 if the fifo was empty. Otherwise it returns the number
416 * processed elements.
417 *
418 * Note that with only one concurrent reader and one concurrent
419 * writer, you don't need extra locking to use these macro.
420 */
421#define kfifo_get(fifo, val) \
422__kfifo_must_check_helper( \
423({ \
424 typeof(fifo + 1) __tmp = (fifo); \
425 typeof(val + 1) __val = (val); \
426 unsigned int __ret; \
427 const size_t __recsize = sizeof(*__tmp->rectype); \
428 struct __kfifo *__kfifo = &__tmp->kfifo; \
429 if (0) \
430 __val = (typeof(__tmp->ptr))0; \
431 if (__recsize) \
432 __ret = __kfifo_out_r(__kfifo, __val, sizeof(*__val), \
433 __recsize); \
434 else { \
435 __ret = !kfifo_is_empty(__tmp); \
436 if (__ret) { \
437 *(typeof(__tmp->type))__val = \
438 (__is_kfifo_ptr(__tmp) ? \
439 ((typeof(__tmp->type))__kfifo->data) : \
440 (__tmp->buf) \
441 )[__kfifo->out & __tmp->kfifo.mask]; \
442 smp_wmb(); \
443 __kfifo->out++; \
444 } \
445 } \
446 __ret; \
447}) \
448)
332 449
333extern unsigned int __kfifo_in_generic(struct kfifo *fifo, 450/**
334 const void *from, unsigned int n, unsigned int recsize); 451 * kfifo_peek - get data from the fifo without removing
452 * @fifo: address of the fifo to be used
453 * @val: the var where to store the data to be added
454 *
455 * This reads the data from the fifo without removing it from the fifo.
456 * It returns 0 if the fifo was empty. Otherwise it returns the number
457 * processed elements.
458 *
459 * Note that with only one concurrent reader and one concurrent
460 * writer, you don't need extra locking to use these macro.
461 */
462#define kfifo_peek(fifo, val) \
463__kfifo_must_check_helper( \
464({ \
465 typeof(fifo + 1) __tmp = (fifo); \
466 typeof(val + 1) __val = (val); \
467 unsigned int __ret; \
468 const size_t __recsize = sizeof(*__tmp->rectype); \
469 struct __kfifo *__kfifo = &__tmp->kfifo; \
470 if (0) \
471 __val = (typeof(__tmp->ptr))NULL; \
472 if (__recsize) \
473 __ret = __kfifo_out_peek_r(__kfifo, __val, sizeof(*__val), \
474 __recsize); \
475 else { \
476 __ret = !kfifo_is_empty(__tmp); \
477 if (__ret) { \
478 *(typeof(__tmp->type))__val = \
479 (__is_kfifo_ptr(__tmp) ? \
480 ((typeof(__tmp->type))__kfifo->data) : \
481 (__tmp->buf) \
482 )[__kfifo->out & __tmp->kfifo.mask]; \
483 smp_wmb(); \
484 } \
485 } \
486 __ret; \
487}) \
488)
335 489
336static inline unsigned int __kfifo_in_rec(struct kfifo *fifo, 490/**
337 const void *from, unsigned int n, unsigned int recsize) 491 * kfifo_in - put data into the fifo
338{ 492 * @fifo: address of the fifo to be used
339 unsigned int ret; 493 * @buf: the data to be added
494 * @n: number of elements to be added
495 *
496 * This macro copies the given buffer into the fifo and returns the
497 * number of copied elements.
498 *
499 * Note that with only one concurrent reader and one concurrent
500 * writer, you don't need extra locking to use these macro.
501 */
502#define kfifo_in(fifo, buf, n) \
503({ \
504 typeof(fifo + 1) __tmp = (fifo); \
505 typeof(buf + 1) __buf = (buf); \
506 unsigned long __n = (n); \
507 const size_t __recsize = sizeof(*__tmp->rectype); \
508 struct __kfifo *__kfifo = &__tmp->kfifo; \
509 if (0) { \
510 typeof(__tmp->ptr_const) __dummy __attribute__ ((unused)); \
511 __dummy = (typeof(__buf))NULL; \
512 } \
513 (__recsize) ?\
514 __kfifo_in_r(__kfifo, __buf, __n, __recsize) : \
515 __kfifo_in(__kfifo, __buf, __n); \
516})
340 517
341 ret = __kfifo_in_n(fifo, from, n, recsize); 518/**
519 * kfifo_in_spinlocked - put data into the fifo using a spinlock for locking
520 * @fifo: address of the fifo to be used
521 * @buf: the data to be added
522 * @n: number of elements to be added
523 * @lock: pointer to the spinlock to use for locking
524 *
525 * This macro copies the given values buffer into the fifo and returns the
526 * number of copied elements.
527 */
528#define kfifo_in_spinlocked(fifo, buf, n, lock) \
529({ \
530 unsigned long __flags; \
531 unsigned int __ret; \
532 spin_lock_irqsave(lock, __flags); \
533 __ret = kfifo_in(fifo, buf, n); \
534 spin_unlock_irqrestore(lock, __flags); \
535 __ret; \
536})
537
538/* alias for kfifo_in_spinlocked, will be removed in a future release */
539#define kfifo_in_locked(fifo, buf, n, lock) \
540 kfifo_in_spinlocked(fifo, buf, n, lock)
342 541
343 if (likely(ret == 0)) { 542/**
344 if (recsize) 543 * kfifo_out - get data from the fifo
345 __kfifo_poke_n(fifo, recsize, n); 544 * @fifo: address of the fifo to be used
346 __kfifo_add_in(fifo, n + recsize); 545 * @buf: pointer to the storage buffer
347 } 546 * @n: max. number of elements to get
348 return ret; 547 *
349} 548 * This macro get some data from the fifo and return the numbers of elements
549 * copied.
550 *
551 * Note that with only one concurrent reader and one concurrent
552 * writer, you don't need extra locking to use these macro.
553 */
554#define kfifo_out(fifo, buf, n) \
555__kfifo_must_check_helper( \
556({ \
557 typeof(fifo + 1) __tmp = (fifo); \
558 typeof(buf + 1) __buf = (buf); \
559 unsigned long __n = (n); \
560 const size_t __recsize = sizeof(*__tmp->rectype); \
561 struct __kfifo *__kfifo = &__tmp->kfifo; \
562 if (0) { \
563 typeof(__tmp->ptr) __dummy = NULL; \
564 __buf = __dummy; \
565 } \
566 (__recsize) ?\
567 __kfifo_out_r(__kfifo, __buf, __n, __recsize) : \
568 __kfifo_out(__kfifo, __buf, __n); \
569}) \
570)
571
572/**
573 * kfifo_out_spinlocked - get data from the fifo using a spinlock for locking
574 * @fifo: address of the fifo to be used
575 * @buf: pointer to the storage buffer
576 * @n: max. number of elements to get
577 * @lock: pointer to the spinlock to use for locking
578 *
579 * This macro get the data from the fifo and return the numbers of elements
580 * copied.
581 */
582#define kfifo_out_spinlocked(fifo, buf, n, lock) \
583__kfifo_must_check_helper( \
584({ \
585 unsigned long __flags; \
586 unsigned int __ret; \
587 spin_lock_irqsave(lock, __flags); \
588 __ret = kfifo_out(fifo, buf, n); \
589 spin_unlock_irqrestore(lock, __flags); \
590 __ret; \
591}) \
592)
593
594/* alias for kfifo_out_spinlocked, will be removed in a future release */
595#define kfifo_out_locked(fifo, buf, n, lock) \
596 kfifo_out_spinlocked(fifo, buf, n, lock)
350 597
351/** 598/**
352 * kfifo_in_rec - puts some record data into the FIFO 599 * kfifo_from_user - puts some data from user space into the fifo
353 * @fifo: the fifo to be used. 600 * @fifo: address of the fifo to be used
354 * @from: the data to be added. 601 * @from: pointer to the data to be added
355 * @n: the length of the data to be added. 602 * @len: the length of the data to be added
356 * @recsize: size of record field 603 * @copied: pointer to output variable to store the number of copied bytes
357 * 604 *
358 * This function copies @n bytes from the @from into the FIFO and returns 605 * This macro copies at most @len bytes from the @from into the
359 * the number of bytes which cannot be copied. 606 * fifo, depending of the available space and returns -EFAULT/0.
360 * A returned value greater than the @n value means that the record doesn't
361 * fit into the buffer.
362 * 607 *
363 * Note that with only one concurrent reader and one concurrent 608 * Note that with only one concurrent reader and one concurrent
364 * writer, you don't need extra locking to use these functions. 609 * writer, you don't need extra locking to use these macro.
365 */ 610 */
366static inline __must_check unsigned int kfifo_in_rec(struct kfifo *fifo, 611#define kfifo_from_user(fifo, from, len, copied) \
367 void *from, unsigned int n, unsigned int recsize) 612__kfifo_must_check_helper( \
368{ 613({ \
369 if (!__builtin_constant_p(recsize)) 614 typeof(fifo + 1) __tmp = (fifo); \
370 return __kfifo_in_generic(fifo, from, n, recsize); 615 const void __user *__from = (from); \
371 return __kfifo_in_rec(fifo, from, n, recsize); 616 unsigned int __len = (len); \
372} 617 unsigned int *__copied = (copied); \
618 const size_t __recsize = sizeof(*__tmp->rectype); \
619 struct __kfifo *__kfifo = &__tmp->kfifo; \
620 (__recsize) ? \
621 __kfifo_from_user_r(__kfifo, __from, __len, __copied, __recsize) : \
622 __kfifo_from_user(__kfifo, __from, __len, __copied); \
623}) \
624)
373 625
374/* 626/**
375 * __kfifo_out_... internal functions for get date from the fifo 627 * kfifo_to_user - copies data from the fifo into user space
376 * do not call it directly, use kfifo_out_rec() instead 628 * @fifo: address of the fifo to be used
377 */ 629 * @to: where the data must be copied
378extern unsigned int __kfifo_out_n(struct kfifo *fifo, 630 * @len: the size of the destination buffer
379 void *to, unsigned int reclen, unsigned int recsize); 631 * @copied: pointer to output variable to store the number of copied bytes
632 *
633 * This macro copies at most @len bytes from the fifo into the
634 * @to buffer and returns -EFAULT/0.
635 *
636 * Note that with only one concurrent reader and one concurrent
637 * writer, you don't need extra locking to use these macro.
638 */
639#define kfifo_to_user(fifo, to, len, copied) \
640__kfifo_must_check_helper( \
641({ \
642 typeof(fifo + 1) __tmp = (fifo); \
643 void __user *__to = (to); \
644 unsigned int __len = (len); \
645 unsigned int *__copied = (copied); \
646 const size_t __recsize = sizeof(*__tmp->rectype); \
647 struct __kfifo *__kfifo = &__tmp->kfifo; \
648 (__recsize) ? \
649 __kfifo_to_user_r(__kfifo, __to, __len, __copied, __recsize) : \
650 __kfifo_to_user(__kfifo, __to, __len, __copied); \
651}) \
652)
653
654/**
655 * kfifo_dma_in_prepare - setup a scatterlist for DMA input
656 * @fifo: address of the fifo to be used
657 * @sgl: pointer to the scatterlist array
658 * @nents: number of entries in the scatterlist array
659 * @len: number of elements to transfer
660 *
661 * This macro fills a scatterlist for DMA input.
662 * It returns the number entries in the scatterlist array.
663 *
664 * Note that with only one concurrent reader and one concurrent
665 * writer, you don't need extra locking to use these macros.
666 */
667#define kfifo_dma_in_prepare(fifo, sgl, nents, len) \
668({ \
669 typeof(fifo + 1) __tmp = (fifo); \
670 struct scatterlist *__sgl = (sgl); \
671 int __nents = (nents); \
672 unsigned int __len = (len); \
673 const size_t __recsize = sizeof(*__tmp->rectype); \
674 struct __kfifo *__kfifo = &__tmp->kfifo; \
675 (__recsize) ? \
676 __kfifo_dma_in_prepare_r(__kfifo, __sgl, __nents, __len, __recsize) : \
677 __kfifo_dma_in_prepare(__kfifo, __sgl, __nents, __len); \
678})
380 679
381extern unsigned int __kfifo_out_generic(struct kfifo *fifo, 680/**
382 void *to, unsigned int n, 681 * kfifo_dma_in_finish - finish a DMA IN operation
383 unsigned int recsize, unsigned int *total); 682 * @fifo: address of the fifo to be used
683 * @len: number of bytes to received
684 *
685 * This macro finish a DMA IN operation. The in counter will be updated by
686 * the len parameter. No error checking will be done.
687 *
688 * Note that with only one concurrent reader and one concurrent
689 * writer, you don't need extra locking to use these macros.
690 */
691#define kfifo_dma_in_finish(fifo, len) \
692(void)({ \
693 typeof(fifo + 1) __tmp = (fifo); \
694 unsigned int __len = (len); \
695 const size_t __recsize = sizeof(*__tmp->rectype); \
696 struct __kfifo *__kfifo = &__tmp->kfifo; \
697 if (__recsize) \
698 __kfifo_dma_in_finish_r(__kfifo, __len, __recsize); \
699 else \
700 __kfifo->in += __len / sizeof(*__tmp->type); \
701})
384 702
385static inline unsigned int __kfifo_out_rec(struct kfifo *fifo, 703/**
386 void *to, unsigned int n, unsigned int recsize, 704 * kfifo_dma_out_prepare - setup a scatterlist for DMA output
387 unsigned int *total) 705 * @fifo: address of the fifo to be used
388{ 706 * @sgl: pointer to the scatterlist array
389 unsigned int l; 707 * @nents: number of entries in the scatterlist array
390 708 * @len: number of elements to transfer
391 if (!recsize) { 709 *
392 l = n; 710 * This macro fills a scatterlist for DMA output which at most @len bytes
393 if (total) 711 * to transfer.
394 *total = l; 712 * It returns the number entries in the scatterlist array.
395 } else { 713 * A zero means there is no space available and the scatterlist is not filled.
396 l = __kfifo_peek_n(fifo, recsize); 714 *
397 if (total) 715 * Note that with only one concurrent reader and one concurrent
398 *total = l; 716 * writer, you don't need extra locking to use these macros.
399 if (n < l) 717 */
400 return l; 718#define kfifo_dma_out_prepare(fifo, sgl, nents, len) \
401 } 719({ \
720 typeof(fifo + 1) __tmp = (fifo); \
721 struct scatterlist *__sgl = (sgl); \
722 int __nents = (nents); \
723 unsigned int __len = (len); \
724 const size_t __recsize = sizeof(*__tmp->rectype); \
725 struct __kfifo *__kfifo = &__tmp->kfifo; \
726 (__recsize) ? \
727 __kfifo_dma_out_prepare_r(__kfifo, __sgl, __nents, __len, __recsize) : \
728 __kfifo_dma_out_prepare(__kfifo, __sgl, __nents, __len); \
729})
402 730
403 return __kfifo_out_n(fifo, to, l, recsize); 731/**
404} 732 * kfifo_dma_out_finish - finish a DMA OUT operation
733 * @fifo: address of the fifo to be used
734 * @len: number of bytes transferd
735 *
736 * This macro finish a DMA OUT operation. The out counter will be updated by
737 * the len parameter. No error checking will be done.
738 *
739 * Note that with only one concurrent reader and one concurrent
740 * writer, you don't need extra locking to use these macros.
741 */
742#define kfifo_dma_out_finish(fifo, len) \
743(void)({ \
744 typeof(fifo + 1) __tmp = (fifo); \
745 unsigned int __len = (len); \
746 const size_t __recsize = sizeof(*__tmp->rectype); \
747 struct __kfifo *__kfifo = &__tmp->kfifo; \
748 if (__recsize) \
749 __kfifo_dma_out_finish_r(__kfifo, __recsize); \
750 else \
751 __kfifo->out += __len / sizeof(*__tmp->type); \
752})
405 753
406/** 754/**
407 * kfifo_out_rec - gets some record data from the FIFO 755 * kfifo_out_peek - gets some data from the fifo
408 * @fifo: the fifo to be used. 756 * @fifo: address of the fifo to be used
409 * @to: where the data must be copied. 757 * @buf: pointer to the storage buffer
410 * @n: the size of the destination buffer. 758 * @n: max. number of elements to get
411 * @recsize: size of record field
412 * @total: pointer where the total number of to copied bytes should stored
413 * 759 *
414 * This function copies at most @n bytes from the FIFO to @to and returns the 760 * This macro get the data from the fifo and return the numbers of elements
415 * number of bytes which cannot be copied. 761 * copied. The data is not removed from the fifo.
416 * A returned value greater than the @n value means that the record doesn't
417 * fit into the @to buffer.
418 * 762 *
419 * Note that with only one concurrent reader and one concurrent 763 * Note that with only one concurrent reader and one concurrent
420 * writer, you don't need extra locking to use these functions. 764 * writer, you don't need extra locking to use these macro.
421 */ 765 */
422static inline __must_check unsigned int kfifo_out_rec(struct kfifo *fifo, 766#define kfifo_out_peek(fifo, buf, n) \
423 void *to, unsigned int n, unsigned int recsize, 767__kfifo_must_check_helper( \
424 unsigned int *total) 768({ \
769 typeof(fifo + 1) __tmp = (fifo); \
770 typeof(buf + 1) __buf = (buf); \
771 unsigned long __n = (n); \
772 const size_t __recsize = sizeof(*__tmp->rectype); \
773 struct __kfifo *__kfifo = &__tmp->kfifo; \
774 if (0) { \
775 typeof(__tmp->ptr) __dummy __attribute__ ((unused)) = NULL; \
776 __buf = __dummy; \
777 } \
778 (__recsize) ? \
779 __kfifo_out_peek_r(__kfifo, __buf, __n, __recsize) : \
780 __kfifo_out_peek(__kfifo, __buf, __n); \
781}) \
782)
425 783
426{ 784extern int __kfifo_alloc(struct __kfifo *fifo, unsigned int size,
427 if (!__builtin_constant_p(recsize)) 785 size_t esize, gfp_t gfp_mask);
428 return __kfifo_out_generic(fifo, to, n, recsize, total);
429 return __kfifo_out_rec(fifo, to, n, recsize, total);
430}
431 786
432/* 787extern void __kfifo_free(struct __kfifo *fifo);
433 * __kfifo_from_user_... internal functions for transfer from user space into
434 * the fifo. do not call it directly, use kfifo_from_user_rec() instead
435 */
436extern unsigned int __kfifo_from_user_n(struct kfifo *fifo,
437 const void __user *from, unsigned int n, unsigned int recsize);
438 788
439extern unsigned int __kfifo_from_user_generic(struct kfifo *fifo, 789extern int __kfifo_init(struct __kfifo *fifo, void *buffer,
440 const void __user *from, unsigned int n, unsigned int recsize); 790 unsigned int size, size_t esize);
441 791
442static inline unsigned int __kfifo_from_user_rec(struct kfifo *fifo, 792extern unsigned int __kfifo_in(struct __kfifo *fifo,
443 const void __user *from, unsigned int n, unsigned int recsize) 793 const void *buf, unsigned int len);
444{
445 unsigned int ret;
446 794
447 ret = __kfifo_from_user_n(fifo, from, n, recsize); 795extern unsigned int __kfifo_out(struct __kfifo *fifo,
796 void *buf, unsigned int len);
448 797
449 if (likely(ret == 0)) { 798extern int __kfifo_from_user(struct __kfifo *fifo,
450 if (recsize) 799 const void __user *from, unsigned long len, unsigned int *copied);
451 __kfifo_poke_n(fifo, recsize, n);
452 __kfifo_add_in(fifo, n + recsize);
453 }
454 return ret;
455}
456 800
457/** 801extern int __kfifo_to_user(struct __kfifo *fifo,
458 * kfifo_from_user_rec - puts some data from user space into the FIFO 802 void __user *to, unsigned long len, unsigned int *copied);
459 * @fifo: the fifo to be used.
460 * @from: pointer to the data to be added.
461 * @n: the length of the data to be added.
462 * @recsize: size of record field
463 *
464 * This function copies @n bytes from the @from into the
465 * FIFO and returns the number of bytes which cannot be copied.
466 *
467 * If the returned value is equal or less the @n value, the copy_from_user()
468 * functions has failed. Otherwise the record doesn't fit into the buffer.
469 *
470 * Note that with only one concurrent reader and one concurrent
471 * writer, you don't need extra locking to use these functions.
472 */
473static inline __must_check unsigned int kfifo_from_user_rec(struct kfifo *fifo,
474 const void __user *from, unsigned int n, unsigned int recsize)
475{
476 if (!__builtin_constant_p(recsize))
477 return __kfifo_from_user_generic(fifo, from, n, recsize);
478 return __kfifo_from_user_rec(fifo, from, n, recsize);
479}
480 803
481/* 804extern unsigned int __kfifo_dma_in_prepare(struct __kfifo *fifo,
482 * __kfifo_to_user_... internal functions for transfer fifo data into user space 805 struct scatterlist *sgl, int nents, unsigned int len);
483 * do not call it directly, use kfifo_to_user_rec() instead
484 */
485extern unsigned int __kfifo_to_user_n(struct kfifo *fifo,
486 void __user *to, unsigned int n, unsigned int reclen,
487 unsigned int recsize);
488 806
489extern unsigned int __kfifo_to_user_generic(struct kfifo *fifo, 807extern unsigned int __kfifo_dma_out_prepare(struct __kfifo *fifo,
490 void __user *to, unsigned int n, unsigned int recsize, 808 struct scatterlist *sgl, int nents, unsigned int len);
491 unsigned int *total);
492 809
493static inline unsigned int __kfifo_to_user_rec(struct kfifo *fifo, 810extern unsigned int __kfifo_out_peek(struct __kfifo *fifo,
494 void __user *to, unsigned int n, 811 void *buf, unsigned int len);
495 unsigned int recsize, unsigned int *total)
496{
497 unsigned int l;
498
499 if (!recsize) {
500 l = n;
501 if (total)
502 *total = l;
503 } else {
504 l = __kfifo_peek_n(fifo, recsize);
505 if (total)
506 *total = l;
507 if (n < l)
508 return l;
509 }
510 812
511 return __kfifo_to_user_n(fifo, to, n, l, recsize); 813extern unsigned int __kfifo_in_r(struct __kfifo *fifo,
512} 814 const void *buf, unsigned int len, size_t recsize);
513 815
514/** 816extern unsigned int __kfifo_out_r(struct __kfifo *fifo,
515 * kfifo_to_user_rec - gets data from the FIFO and write it to user space 817 void *buf, unsigned int len, size_t recsize);
516 * @fifo: the fifo to be used.
517 * @to: where the data must be copied.
518 * @n: the size of the destination buffer.
519 * @recsize: size of record field
520 * @total: pointer where the total number of to copied bytes should stored
521 *
522 * This function copies at most @n bytes from the FIFO to the @to.
523 * In case of an error, the function returns the number of bytes which cannot
524 * be copied.
525 * If the returned value is equal or less the @n value, the copy_to_user()
526 * functions has failed. Otherwise the record doesn't fit into the @to buffer.
527 *
528 * Note that with only one concurrent reader and one concurrent
529 * writer, you don't need extra locking to use these functions.
530 */
531static inline __must_check unsigned int kfifo_to_user_rec(struct kfifo *fifo,
532 void __user *to, unsigned int n, unsigned int recsize,
533 unsigned int *total)
534{
535 if (!__builtin_constant_p(recsize))
536 return __kfifo_to_user_generic(fifo, to, n, recsize, total);
537 return __kfifo_to_user_rec(fifo, to, n, recsize, total);
538}
539 818
540/* 819extern int __kfifo_from_user_r(struct __kfifo *fifo,
541 * __kfifo_peek_... internal functions for peek into the next fifo record 820 const void __user *from, unsigned long len, unsigned int *copied,
542 * do not call it directly, use kfifo_peek_rec() instead 821 size_t recsize);
543 */
544extern unsigned int __kfifo_peek_generic(struct kfifo *fifo,
545 unsigned int recsize);
546 822
547/** 823extern int __kfifo_to_user_r(struct __kfifo *fifo, void __user *to,
548 * kfifo_peek_rec - gets the size of the next FIFO record data 824 unsigned long len, unsigned int *copied, size_t recsize);
549 * @fifo: the fifo to be used.
550 * @recsize: size of record field
551 *
552 * This function returns the size of the next FIFO record in number of bytes
553 */
554static inline __must_check unsigned int kfifo_peek_rec(struct kfifo *fifo,
555 unsigned int recsize)
556{
557 if (!__builtin_constant_p(recsize))
558 return __kfifo_peek_generic(fifo, recsize);
559 if (!recsize)
560 return kfifo_len(fifo);
561 return __kfifo_peek_n(fifo, recsize);
562}
563 825
564/* 826extern unsigned int __kfifo_dma_in_prepare_r(struct __kfifo *fifo,
565 * __kfifo_skip_... internal functions for skip the next fifo record 827 struct scatterlist *sgl, int nents, unsigned int len, size_t recsize);
566 * do not call it directly, use kfifo_skip_rec() instead
567 */
568extern void __kfifo_skip_generic(struct kfifo *fifo, unsigned int recsize);
569 828
570static inline void __kfifo_skip_rec(struct kfifo *fifo, 829extern void __kfifo_dma_in_finish_r(struct __kfifo *fifo,
571 unsigned int recsize) 830 unsigned int len, size_t recsize);
572{
573 unsigned int l;
574 831
575 if (recsize) { 832extern unsigned int __kfifo_dma_out_prepare_r(struct __kfifo *fifo,
576 l = __kfifo_peek_n(fifo, recsize); 833 struct scatterlist *sgl, int nents, unsigned int len, size_t recsize);
577 834
578 if (l + recsize <= kfifo_len(fifo)) { 835extern void __kfifo_dma_out_finish_r(struct __kfifo *fifo, size_t recsize);
579 __kfifo_add_out(fifo, l + recsize);
580 return;
581 }
582 }
583 kfifo_reset_out(fifo);
584}
585 836
586/** 837extern unsigned int __kfifo_len_r(struct __kfifo *fifo, size_t recsize);
587 * kfifo_skip_rec - skip the next fifo out record
588 * @fifo: the fifo to be used.
589 * @recsize: size of record field
590 *
591 * This function skips the next FIFO record
592 */
593static inline void kfifo_skip_rec(struct kfifo *fifo,
594 unsigned int recsize)
595{
596 if (!__builtin_constant_p(recsize))
597 __kfifo_skip_generic(fifo, recsize);
598 else
599 __kfifo_skip_rec(fifo, recsize);
600}
601 838
602/** 839extern unsigned int __kfifo_out_peek_r(struct __kfifo *fifo,
603 * kfifo_avail_rec - returns the number of bytes available in a record FIFO 840 void *buf, unsigned int len, size_t recsize);
604 * @fifo: the fifo to be used.
605 * @recsize: size of record field
606 */
607static inline __must_check unsigned int kfifo_avail_rec(struct kfifo *fifo,
608 unsigned int recsize)
609{
610 unsigned int l = kfifo_size(fifo) - kfifo_len(fifo);
611 841
612 return (l > recsize) ? l - recsize : 0; 842extern unsigned int __kfifo_max_r(unsigned int len, size_t recsize);
613}
614 843
615#endif 844#endif
diff --git a/kernel/kfifo-new.c b/kernel/kfifo-new.c
deleted file mode 100644
index 02192dd905cc..000000000000
--- a/kernel/kfifo-new.c
+++ /dev/null
@@ -1,602 +0,0 @@
1/*
2 * A generic kernel FIFO implementation
3 *
4 * Copyright (C) 2009/2010 Stefani Seibold <stefani@seibold.net>
5 *
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License as published by
8 * the Free Software Foundation; either version 2 of the License, or
9 * (at your option) any later version.
10 *
11 * This program is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 * GNU General Public License for more details.
15 *
16 * You should have received a copy of the GNU General Public License
17 * along with this program; if not, write to the Free Software
18 * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
19 *
20 */
21
22#include <linux/kernel.h>
23#include <linux/module.h>
24#include <linux/slab.h>
25#include <linux/err.h>
26#include <linux/log2.h>
27#include <linux/uaccess.h>
28#include <linux/kfifo.h>
29
30/*
31 * internal helper to calculate the unused elements in a fifo
32 */
33static inline unsigned int kfifo_unused(struct __kfifo *fifo)
34{
35 return (fifo->mask + 1) - (fifo->in - fifo->out);
36}
37
38int __kfifo_alloc(struct __kfifo *fifo, unsigned int size,
39 size_t esize, gfp_t gfp_mask)
40{
41 /*
42 * round down to the next power of 2, since our 'let the indices
43 * wrap' technique works only in this case.
44 */
45 if (!is_power_of_2(size))
46 size = rounddown_pow_of_two(size);
47
48 fifo->in = 0;
49 fifo->out = 0;
50 fifo->esize = esize;
51
52 if (size < 2) {
53 fifo->data = NULL;
54 fifo->mask = 0;
55 return -EINVAL;
56 }
57
58 fifo->data = kmalloc(size * esize, gfp_mask);
59
60 if (!fifo->data) {
61 fifo->mask = 0;
62 return -ENOMEM;
63 }
64 fifo->mask = size - 1;
65
66 return 0;
67}
68EXPORT_SYMBOL(__kfifo_alloc);
69
70void __kfifo_free(struct __kfifo *fifo)
71{
72 kfree(fifo->data);
73 fifo->in = 0;
74 fifo->out = 0;
75 fifo->esize = 0;
76 fifo->data = NULL;
77 fifo->mask = 0;
78}
79EXPORT_SYMBOL(__kfifo_free);
80
81int __kfifo_init(struct __kfifo *fifo, void *buffer,
82 unsigned int size, size_t esize)
83{
84 size /= esize;
85
86 if (!is_power_of_2(size))
87 size = rounddown_pow_of_two(size);
88
89 fifo->in = 0;
90 fifo->out = 0;
91 fifo->esize = esize;
92 fifo->data = buffer;
93
94 if (size < 2) {
95 fifo->mask = 0;
96 return -EINVAL;
97 }
98 fifo->mask = size - 1;
99
100 return 0;
101}
102EXPORT_SYMBOL(__kfifo_init);
103
104static void kfifo_copy_in(struct __kfifo *fifo, const void *src,
105 unsigned int len, unsigned int off)
106{
107 unsigned int size = fifo->mask + 1;
108 unsigned int esize = fifo->esize;
109 unsigned int l;
110
111 off &= fifo->mask;
112 if (esize != 1) {
113 off *= esize;
114 size *= esize;
115 len *= esize;
116 }
117 l = min(len, size - off);
118
119 memcpy(fifo->data + off, src, l);
120 memcpy(fifo->data, src + l, len - l);
121 /*
122 * make sure that the data in the fifo is up to date before
123 * incrementing the fifo->in index counter
124 */
125 smp_wmb();
126}
127
128unsigned int __kfifo_in(struct __kfifo *fifo,
129 const void *buf, unsigned int len)
130{
131 unsigned int l;
132
133 l = kfifo_unused(fifo);
134 if (len > l)
135 len = l;
136
137 kfifo_copy_in(fifo, buf, len, fifo->in);
138 fifo->in += len;
139 return len;
140}
141EXPORT_SYMBOL(__kfifo_in);
142
143static void kfifo_copy_out(struct __kfifo *fifo, void *dst,
144 unsigned int len, unsigned int off)
145{
146 unsigned int size = fifo->mask + 1;
147 unsigned int esize = fifo->esize;
148 unsigned int l;
149
150 off &= fifo->mask;
151 if (esize != 1) {
152 off *= esize;
153 size *= esize;
154 len *= esize;
155 }
156 l = min(len, size - off);
157
158 memcpy(dst, fifo->data + off, l);
159 memcpy(dst + l, fifo->data, len - l);
160 /*
161 * make sure that the data is copied before
162 * incrementing the fifo->out index counter
163 */
164 smp_wmb();
165}
166
167unsigned int __kfifo_out_peek(struct __kfifo *fifo,
168 void *buf, unsigned int len)
169{
170 unsigned int l;
171
172 l = fifo->in - fifo->out;
173 if (len > l)
174 len = l;
175
176 kfifo_copy_out(fifo, buf, len, fifo->out);
177 return len;
178}
179EXPORT_SYMBOL(__kfifo_out_peek);
180
181unsigned int __kfifo_out(struct __kfifo *fifo,
182 void *buf, unsigned int len)
183{
184 len = __kfifo_out_peek(fifo, buf, len);
185 fifo->out += len;
186 return len;
187}
188EXPORT_SYMBOL(__kfifo_out);
189
190static unsigned long kfifo_copy_from_user(struct __kfifo *fifo,
191 const void __user *from, unsigned int len, unsigned int off,
192 unsigned int *copied)
193{
194 unsigned int size = fifo->mask + 1;
195 unsigned int esize = fifo->esize;
196 unsigned int l;
197 unsigned long ret;
198
199 off &= fifo->mask;
200 if (esize != 1) {
201 off *= esize;
202 size *= esize;
203 len *= esize;
204 }
205 l = min(len, size - off);
206
207 ret = copy_from_user(fifo->data + off, from, l);
208 if (unlikely(ret))
209 ret = DIV_ROUND_UP(ret + len - l, esize);
210 else {
211 ret = copy_from_user(fifo->data, from + l, len - l);
212 if (unlikely(ret))
213 ret = DIV_ROUND_UP(ret, esize);
214 }
215 /*
216 * make sure that the data in the fifo is up to date before
217 * incrementing the fifo->in index counter
218 */
219 smp_wmb();
220 *copied = len - ret;
221 /* return the number of elements which are not copied */
222 return ret;
223}
224
225int __kfifo_from_user(struct __kfifo *fifo, const void __user *from,
226 unsigned long len, unsigned int *copied)
227{
228 unsigned int l;
229 unsigned long ret;
230 unsigned int esize = fifo->esize;
231 int err;
232
233 if (esize != 1)
234 len /= esize;
235
236 l = kfifo_unused(fifo);
237 if (len > l)
238 len = l;
239
240 ret = kfifo_copy_from_user(fifo, from, len, fifo->in, copied);
241 if (unlikely(ret)) {
242 len -= ret;
243 err = -EFAULT;
244 } else
245 err = 0;
246 fifo->in += len;
247 return err;
248}
249EXPORT_SYMBOL(__kfifo_from_user);
250
251static unsigned long kfifo_copy_to_user(struct __kfifo *fifo, void __user *to,
252 unsigned int len, unsigned int off, unsigned int *copied)
253{
254 unsigned int l;
255 unsigned long ret;
256 unsigned int size = fifo->mask + 1;
257 unsigned int esize = fifo->esize;
258
259 off &= fifo->mask;
260 if (esize != 1) {
261 off *= esize;
262 size *= esize;
263 len *= esize;
264 }
265 l = min(len, size - off);
266
267 ret = copy_to_user(to, fifo->data + off, l);
268 if (unlikely(ret))
269 ret = DIV_ROUND_UP(ret + len - l, esize);
270 else {
271 ret = copy_to_user(to + l, fifo->data, len - l);
272 if (unlikely(ret))
273 ret = DIV_ROUND_UP(ret, esize);
274 }
275 /*
276 * make sure that the data is copied before
277 * incrementing the fifo->out index counter
278 */
279 smp_wmb();
280 *copied = len - ret;
281 /* return the number of elements which are not copied */
282 return ret;
283}
284
285int __kfifo_to_user(struct __kfifo *fifo, void __user *to,
286 unsigned long len, unsigned int *copied)
287{
288 unsigned int l;
289 unsigned long ret;
290 unsigned int esize = fifo->esize;
291 int err;
292
293 if (esize != 1)
294 len /= esize;
295
296 l = fifo->in - fifo->out;
297 if (len > l)
298 len = l;
299 ret = kfifo_copy_to_user(fifo, to, len, fifo->out, copied);
300 if (unlikely(ret)) {
301 len -= ret;
302 err = -EFAULT;
303 } else
304 err = 0;
305 fifo->out += len;
306 return err;
307}
308EXPORT_SYMBOL(__kfifo_to_user);
309
310static int setup_sgl_buf(struct scatterlist *sgl, void *buf,
311 int nents, unsigned int len)
312{
313 int n;
314 unsigned int l;
315 unsigned int off;
316 struct page *page;
317
318 if (!nents)
319 return 0;
320
321 if (!len)
322 return 0;
323
324 n = 0;
325 page = virt_to_page(buf);
326 off = offset_in_page(buf);
327 l = 0;
328
329 while (len >= l + PAGE_SIZE - off) {
330 struct page *npage;
331
332 l += PAGE_SIZE;
333 buf += PAGE_SIZE;
334 npage = virt_to_page(buf);
335 if (page_to_phys(page) != page_to_phys(npage) - l) {
336 sgl->page_link = 0;
337 sg_set_page(sgl++, page, l - off, off);
338 if (++n == nents)
339 return n;
340 page = npage;
341 len -= l - off;
342 l = off = 0;
343 }
344 }
345 sgl->page_link = 0;
346 sg_set_page(sgl++, page, len, off);
347 return n + 1;
348}
349
350static unsigned int setup_sgl(struct __kfifo *fifo, struct scatterlist *sgl,
351 int nents, unsigned int len, unsigned int off)
352{
353 unsigned int size = fifo->mask + 1;
354 unsigned int esize = fifo->esize;
355 unsigned int l;
356 unsigned int n;
357
358 off &= fifo->mask;
359 if (esize != 1) {
360 off *= esize;
361 size *= esize;
362 len *= esize;
363 }
364 l = min(len, size - off);
365
366 n = setup_sgl_buf(sgl, fifo->data + off, nents, l);
367 n += setup_sgl_buf(sgl + n, fifo->data, nents - n, len - l);
368
369 if (n)
370 sg_mark_end(sgl + n - 1);
371 return n;
372}
373
374unsigned int __kfifo_dma_in_prepare(struct __kfifo *fifo,
375 struct scatterlist *sgl, int nents, unsigned int len)
376{
377 unsigned int l;
378
379 l = kfifo_unused(fifo);
380 if (len > l)
381 len = l;
382
383 return setup_sgl(fifo, sgl, nents, len, fifo->in);
384}
385EXPORT_SYMBOL(__kfifo_dma_in_prepare);
386
387unsigned int __kfifo_dma_out_prepare(struct __kfifo *fifo,
388 struct scatterlist *sgl, int nents, unsigned int len)
389{
390 unsigned int l;
391
392 l = fifo->in - fifo->out;
393 if (len > l)
394 len = l;
395
396 return setup_sgl(fifo, sgl, nents, len, fifo->out);
397}
398EXPORT_SYMBOL(__kfifo_dma_out_prepare);
399
400unsigned int __kfifo_max_r(unsigned int len, size_t recsize)
401{
402 unsigned int max = (1 << (recsize << 3)) - 1;
403
404 if (len > max)
405 return max;
406 return len;
407}
408
409#define __KFIFO_PEEK(data, out, mask) \
410 ((data)[(out) & (mask)])
411/*
412 * __kfifo_peek_n internal helper function for determinate the length of
413 * the next record in the fifo
414 */
415static unsigned int __kfifo_peek_n(struct __kfifo *fifo, size_t recsize)
416{
417 unsigned int l;
418 unsigned int mask = fifo->mask;
419 unsigned char *data = fifo->data;
420
421 l = __KFIFO_PEEK(data, fifo->out, mask);
422
423 if (--recsize)
424 l |= __KFIFO_PEEK(data, fifo->out + 1, mask) << 8;
425
426 return l;
427}
428
429#define __KFIFO_POKE(data, in, mask, val) \
430 ( \
431 (data)[(in) & (mask)] = (unsigned char)(val) \
432 )
433
434/*
435 * __kfifo_poke_n internal helper function for storeing the length of
436 * the record into the fifo
437 */
438static void __kfifo_poke_n(struct __kfifo *fifo, unsigned int n, size_t recsize)
439{
440 unsigned int mask = fifo->mask;
441 unsigned char *data = fifo->data;
442
443 __KFIFO_POKE(data, fifo->in, mask, n);
444
445 if (recsize > 1)
446 __KFIFO_POKE(data, fifo->in + 1, mask, n >> 8);
447}
448
449unsigned int __kfifo_len_r(struct __kfifo *fifo, size_t recsize)
450{
451 return __kfifo_peek_n(fifo, recsize);
452}
453EXPORT_SYMBOL(__kfifo_len_r);
454
455unsigned int __kfifo_in_r(struct __kfifo *fifo, const void *buf,
456 unsigned int len, size_t recsize)
457{
458 if (len + recsize > kfifo_unused(fifo))
459 return 0;
460
461 __kfifo_poke_n(fifo, len, recsize);
462
463 kfifo_copy_in(fifo, buf, len, fifo->in + recsize);
464 fifo->in += len + recsize;
465 return len;
466}
467EXPORT_SYMBOL(__kfifo_in_r);
468
469static unsigned int kfifo_out_copy_r(struct __kfifo *fifo,
470 void *buf, unsigned int len, size_t recsize, unsigned int *n)
471{
472 *n = __kfifo_peek_n(fifo, recsize);
473
474 if (len > *n)
475 len = *n;
476
477 kfifo_copy_out(fifo, buf, len, fifo->out + recsize);
478 return len;
479}
480
481unsigned int __kfifo_out_peek_r(struct __kfifo *fifo, void *buf,
482 unsigned int len, size_t recsize)
483{
484 unsigned int n;
485
486 if (fifo->in == fifo->out)
487 return 0;
488
489 return kfifo_out_copy_r(fifo, buf, len, recsize, &n);
490}
491EXPORT_SYMBOL(__kfifo_out_peek_r);
492
493unsigned int __kfifo_out_r(struct __kfifo *fifo, void *buf,
494 unsigned int len, size_t recsize)
495{
496 unsigned int n;
497
498 if (fifo->in == fifo->out)
499 return 0;
500
501 len = kfifo_out_copy_r(fifo, buf, len, recsize, &n);
502 fifo->out += n + recsize;
503 return len;
504}
505EXPORT_SYMBOL(__kfifo_out_r);
506
507int __kfifo_from_user_r(struct __kfifo *fifo, const void __user *from,
508 unsigned long len, unsigned int *copied, size_t recsize)
509{
510 unsigned long ret;
511
512 len = __kfifo_max_r(len, recsize);
513
514 if (len + recsize > kfifo_unused(fifo)) {
515 *copied = 0;
516 return 0;
517 }
518
519 __kfifo_poke_n(fifo, len, recsize);
520
521 ret = kfifo_copy_from_user(fifo, from, len, fifo->in + recsize, copied);
522 if (unlikely(ret)) {
523 *copied = 0;
524 return -EFAULT;
525 }
526 fifo->in += len + recsize;
527 return 0;
528}
529EXPORT_SYMBOL(__kfifo_from_user_r);
530
531int __kfifo_to_user_r(struct __kfifo *fifo, void __user *to,
532 unsigned long len, unsigned int *copied, size_t recsize)
533{
534 unsigned long ret;
535 unsigned int n;
536
537 if (fifo->in == fifo->out) {
538 *copied = 0;
539 return 0;
540 }
541
542 n = __kfifo_peek_n(fifo, recsize);
543 if (len > n)
544 len = n;
545
546 ret = kfifo_copy_to_user(fifo, to, len, fifo->out + recsize, copied);
547 if (unlikely(ret)) {
548 *copied = 0;
549 return -EFAULT;
550 }
551 fifo->out += n + recsize;
552 return 0;
553}
554EXPORT_SYMBOL(__kfifo_to_user_r);
555
556unsigned int __kfifo_dma_in_prepare_r(struct __kfifo *fifo,
557 struct scatterlist *sgl, int nents, unsigned int len, size_t recsize)
558{
559 if (!nents)
560 BUG();
561
562 len = __kfifo_max_r(len, recsize);
563
564 if (len + recsize > kfifo_unused(fifo))
565 return 0;
566
567 return setup_sgl(fifo, sgl, nents, len, fifo->in + recsize);
568}
569EXPORT_SYMBOL(__kfifo_dma_in_prepare_r);
570
571void __kfifo_dma_in_finish_r(struct __kfifo *fifo,
572 unsigned int len, size_t recsize)
573{
574 len = __kfifo_max_r(len, recsize);
575 __kfifo_poke_n(fifo, len, recsize);
576 fifo->in += len + recsize;
577}
578EXPORT_SYMBOL(__kfifo_dma_in_finish_r);
579
580unsigned int __kfifo_dma_out_prepare_r(struct __kfifo *fifo,
581 struct scatterlist *sgl, int nents, unsigned int len, size_t recsize)
582{
583 if (!nents)
584 BUG();
585
586 len = __kfifo_max_r(len, recsize);
587
588 if (len + recsize > fifo->in - fifo->out)
589 return 0;
590
591 return setup_sgl(fifo, sgl, nents, len, fifo->out + recsize);
592}
593EXPORT_SYMBOL(__kfifo_dma_out_prepare_r);
594
595void __kfifo_dma_out_finish_r(struct __kfifo *fifo, size_t recsize)
596{
597 unsigned int len;
598
599 len = __kfifo_peek_n(fifo, recsize);
600 fifo->out += len + recsize;
601}
602EXPORT_SYMBOL(__kfifo_dma_out_finish_r);
diff --git a/kernel/kfifo.c b/kernel/kfifo.c
index 35edbe22e9a9..02192dd905cc 100644
--- a/kernel/kfifo.c
+++ b/kernel/kfifo.c
@@ -1,8 +1,7 @@
1/* 1/*
2 * A generic kernel FIFO implementation. 2 * A generic kernel FIFO implementation
3 * 3 *
4 * Copyright (C) 2009 Stefani Seibold <stefani@seibold.net> 4 * Copyright (C) 2009/2010 Stefani Seibold <stefani@seibold.net>
5 * Copyright (C) 2004 Stelian Pop <stelian@popies.net>
6 * 5 *
7 * This program is free software; you can redistribute it and/or modify 6 * This program is free software; you can redistribute it and/or modify
8 * it under the terms of the GNU General Public License as published by 7 * it under the terms of the GNU General Public License as published by
@@ -24,422 +23,580 @@
24#include <linux/module.h> 23#include <linux/module.h>
25#include <linux/slab.h> 24#include <linux/slab.h>
26#include <linux/err.h> 25#include <linux/err.h>
27#include <linux/kfifo.h>
28#include <linux/log2.h> 26#include <linux/log2.h>
29#include <linux/uaccess.h> 27#include <linux/uaccess.h>
28#include <linux/kfifo.h>
30 29
31static void _kfifo_init(struct kfifo *fifo, void *buffer, 30/*
32 unsigned int size) 31 * internal helper to calculate the unused elements in a fifo
33{
34 fifo->buffer = buffer;
35 fifo->size = size;
36
37 kfifo_reset(fifo);
38}
39
40/**
41 * kfifo_init - initialize a FIFO using a preallocated buffer
42 * @fifo: the fifo to assign the buffer
43 * @buffer: the preallocated buffer to be used.
44 * @size: the size of the internal buffer, this has to be a power of 2.
45 *
46 */ 32 */
47void kfifo_init(struct kfifo *fifo, void *buffer, unsigned int size) 33static inline unsigned int kfifo_unused(struct __kfifo *fifo)
48{ 34{
49 /* size must be a power of 2 */ 35 return (fifo->mask + 1) - (fifo->in - fifo->out);
50 BUG_ON(!is_power_of_2(size));
51
52 _kfifo_init(fifo, buffer, size);
53} 36}
54EXPORT_SYMBOL(kfifo_init);
55 37
56/** 38int __kfifo_alloc(struct __kfifo *fifo, unsigned int size,
57 * kfifo_alloc - allocates a new FIFO internal buffer 39 size_t esize, gfp_t gfp_mask)
58 * @fifo: the fifo to assign then new buffer
59 * @size: the size of the buffer to be allocated, this have to be a power of 2.
60 * @gfp_mask: get_free_pages mask, passed to kmalloc()
61 *
62 * This function dynamically allocates a new fifo internal buffer
63 *
64 * The size will be rounded-up to a power of 2.
65 * The buffer will be release with kfifo_free().
66 * Return 0 if no error, otherwise the an error code
67 */
68int kfifo_alloc(struct kfifo *fifo, unsigned int size, gfp_t gfp_mask)
69{ 40{
70 unsigned char *buffer;
71
72 /* 41 /*
73 * round up to the next power of 2, since our 'let the indices 42 * round down to the next power of 2, since our 'let the indices
74 * wrap' technique works only in this case. 43 * wrap' technique works only in this case.
75 */ 44 */
76 if (!is_power_of_2(size)) { 45 if (!is_power_of_2(size))
77 BUG_ON(size > 0x80000000); 46 size = rounddown_pow_of_two(size);
78 size = roundup_pow_of_two(size); 47
48 fifo->in = 0;
49 fifo->out = 0;
50 fifo->esize = esize;
51
52 if (size < 2) {
53 fifo->data = NULL;
54 fifo->mask = 0;
55 return -EINVAL;
79 } 56 }
80 57
81 buffer = kmalloc(size, gfp_mask); 58 fifo->data = kmalloc(size * esize, gfp_mask);
82 if (!buffer) { 59
83 _kfifo_init(fifo, NULL, 0); 60 if (!fifo->data) {
61 fifo->mask = 0;
84 return -ENOMEM; 62 return -ENOMEM;
85 } 63 }
86 64 fifo->mask = size - 1;
87 _kfifo_init(fifo, buffer, size);
88 65
89 return 0; 66 return 0;
90} 67}
91EXPORT_SYMBOL(kfifo_alloc); 68EXPORT_SYMBOL(__kfifo_alloc);
92 69
93/** 70void __kfifo_free(struct __kfifo *fifo)
94 * kfifo_free - frees the FIFO internal buffer
95 * @fifo: the fifo to be freed.
96 */
97void kfifo_free(struct kfifo *fifo)
98{ 71{
99 kfree(fifo->buffer); 72 kfree(fifo->data);
100 _kfifo_init(fifo, NULL, 0); 73 fifo->in = 0;
74 fifo->out = 0;
75 fifo->esize = 0;
76 fifo->data = NULL;
77 fifo->mask = 0;
101} 78}
102EXPORT_SYMBOL(kfifo_free); 79EXPORT_SYMBOL(__kfifo_free);
103 80
104/** 81int __kfifo_init(struct __kfifo *fifo, void *buffer,
105 * kfifo_skip - skip output data 82 unsigned int size, size_t esize)
106 * @fifo: the fifo to be used.
107 * @len: number of bytes to skip
108 */
109void kfifo_skip(struct kfifo *fifo, unsigned int len)
110{ 83{
111 if (len < kfifo_len(fifo)) { 84 size /= esize;
112 __kfifo_add_out(fifo, len); 85
113 return; 86 if (!is_power_of_2(size))
87 size = rounddown_pow_of_two(size);
88
89 fifo->in = 0;
90 fifo->out = 0;
91 fifo->esize = esize;
92 fifo->data = buffer;
93
94 if (size < 2) {
95 fifo->mask = 0;
96 return -EINVAL;
114 } 97 }
115 kfifo_reset_out(fifo); 98 fifo->mask = size - 1;
99
100 return 0;
116} 101}
117EXPORT_SYMBOL(kfifo_skip); 102EXPORT_SYMBOL(__kfifo_init);
118 103
119static inline void __kfifo_in_data(struct kfifo *fifo, 104static void kfifo_copy_in(struct __kfifo *fifo, const void *src,
120 const void *from, unsigned int len, unsigned int off) 105 unsigned int len, unsigned int off)
121{ 106{
107 unsigned int size = fifo->mask + 1;
108 unsigned int esize = fifo->esize;
122 unsigned int l; 109 unsigned int l;
123 110
111 off &= fifo->mask;
112 if (esize != 1) {
113 off *= esize;
114 size *= esize;
115 len *= esize;
116 }
117 l = min(len, size - off);
118
119 memcpy(fifo->data + off, src, l);
120 memcpy(fifo->data, src + l, len - l);
124 /* 121 /*
125 * Ensure that we sample the fifo->out index -before- we 122 * make sure that the data in the fifo is up to date before
126 * start putting bytes into the kfifo. 123 * incrementing the fifo->in index counter
127 */ 124 */
125 smp_wmb();
126}
128 127
129 smp_mb(); 128unsigned int __kfifo_in(struct __kfifo *fifo,
130 129 const void *buf, unsigned int len)
131 off = __kfifo_off(fifo, fifo->in + off); 130{
131 unsigned int l;
132 132
133 /* first put the data starting from fifo->in to buffer end */ 133 l = kfifo_unused(fifo);
134 l = min(len, fifo->size - off); 134 if (len > l)
135 memcpy(fifo->buffer + off, from, l); 135 len = l;
136 136
137 /* then put the rest (if any) at the beginning of the buffer */ 137 kfifo_copy_in(fifo, buf, len, fifo->in);
138 memcpy(fifo->buffer, from + l, len - l); 138 fifo->in += len;
139 return len;
139} 140}
141EXPORT_SYMBOL(__kfifo_in);
140 142
141static inline void __kfifo_out_data(struct kfifo *fifo, 143static void kfifo_copy_out(struct __kfifo *fifo, void *dst,
142 void *to, unsigned int len, unsigned int off) 144 unsigned int len, unsigned int off)
143{ 145{
146 unsigned int size = fifo->mask + 1;
147 unsigned int esize = fifo->esize;
144 unsigned int l; 148 unsigned int l;
145 149
150 off &= fifo->mask;
151 if (esize != 1) {
152 off *= esize;
153 size *= esize;
154 len *= esize;
155 }
156 l = min(len, size - off);
157
158 memcpy(dst, fifo->data + off, l);
159 memcpy(dst + l, fifo->data, len - l);
146 /* 160 /*
147 * Ensure that we sample the fifo->in index -before- we 161 * make sure that the data is copied before
148 * start removing bytes from the kfifo. 162 * incrementing the fifo->out index counter
149 */ 163 */
164 smp_wmb();
165}
150 166
151 smp_rmb(); 167unsigned int __kfifo_out_peek(struct __kfifo *fifo,
168 void *buf, unsigned int len)
169{
170 unsigned int l;
152 171
153 off = __kfifo_off(fifo, fifo->out + off); 172 l = fifo->in - fifo->out;
173 if (len > l)
174 len = l;
154 175
155 /* first get the data from fifo->out until the end of the buffer */ 176 kfifo_copy_out(fifo, buf, len, fifo->out);
156 l = min(len, fifo->size - off); 177 return len;
157 memcpy(to, fifo->buffer + off, l); 178}
179EXPORT_SYMBOL(__kfifo_out_peek);
158 180
159 /* then get the rest (if any) from the beginning of the buffer */ 181unsigned int __kfifo_out(struct __kfifo *fifo,
160 memcpy(to + l, fifo->buffer, len - l); 182 void *buf, unsigned int len)
183{
184 len = __kfifo_out_peek(fifo, buf, len);
185 fifo->out += len;
186 return len;
161} 187}
188EXPORT_SYMBOL(__kfifo_out);
162 189
163static inline int __kfifo_from_user_data(struct kfifo *fifo, 190static unsigned long kfifo_copy_from_user(struct __kfifo *fifo,
164 const void __user *from, unsigned int len, unsigned int off, 191 const void __user *from, unsigned int len, unsigned int off,
165 unsigned *lenout) 192 unsigned int *copied)
166{ 193{
194 unsigned int size = fifo->mask + 1;
195 unsigned int esize = fifo->esize;
167 unsigned int l; 196 unsigned int l;
168 int ret; 197 unsigned long ret;
169 198
199 off &= fifo->mask;
200 if (esize != 1) {
201 off *= esize;
202 size *= esize;
203 len *= esize;
204 }
205 l = min(len, size - off);
206
207 ret = copy_from_user(fifo->data + off, from, l);
208 if (unlikely(ret))
209 ret = DIV_ROUND_UP(ret + len - l, esize);
210 else {
211 ret = copy_from_user(fifo->data, from + l, len - l);
212 if (unlikely(ret))
213 ret = DIV_ROUND_UP(ret, esize);
214 }
170 /* 215 /*
171 * Ensure that we sample the fifo->out index -before- we 216 * make sure that the data in the fifo is up to date before
172 * start putting bytes into the kfifo. 217 * incrementing the fifo->in index counter
173 */ 218 */
219 smp_wmb();
220 *copied = len - ret;
221 /* return the number of elements which are not copied */
222 return ret;
223}
174 224
175 smp_mb(); 225int __kfifo_from_user(struct __kfifo *fifo, const void __user *from,
226 unsigned long len, unsigned int *copied)
227{
228 unsigned int l;
229 unsigned long ret;
230 unsigned int esize = fifo->esize;
231 int err;
176 232
177 off = __kfifo_off(fifo, fifo->in + off); 233 if (esize != 1)
234 len /= esize;
178 235
179 /* first put the data starting from fifo->in to buffer end */ 236 l = kfifo_unused(fifo);
180 l = min(len, fifo->size - off); 237 if (len > l)
181 ret = copy_from_user(fifo->buffer + off, from, l); 238 len = l;
182 if (unlikely(ret)) {
183 *lenout = ret;
184 return -EFAULT;
185 }
186 *lenout = l;
187 239
188 /* then put the rest (if any) at the beginning of the buffer */ 240 ret = kfifo_copy_from_user(fifo, from, len, fifo->in, copied);
189 ret = copy_from_user(fifo->buffer, from + l, len - l); 241 if (unlikely(ret)) {
190 *lenout += ret ? ret : len - l; 242 len -= ret;
191 return ret ? -EFAULT : 0; 243 err = -EFAULT;
244 } else
245 err = 0;
246 fifo->in += len;
247 return err;
192} 248}
249EXPORT_SYMBOL(__kfifo_from_user);
193 250
194static inline int __kfifo_to_user_data(struct kfifo *fifo, 251static unsigned long kfifo_copy_to_user(struct __kfifo *fifo, void __user *to,
195 void __user *to, unsigned int len, unsigned int off, unsigned *lenout) 252 unsigned int len, unsigned int off, unsigned int *copied)
196{ 253{
197 unsigned int l; 254 unsigned int l;
198 int ret; 255 unsigned long ret;
199 256 unsigned int size = fifo->mask + 1;
257 unsigned int esize = fifo->esize;
258
259 off &= fifo->mask;
260 if (esize != 1) {
261 off *= esize;
262 size *= esize;
263 len *= esize;
264 }
265 l = min(len, size - off);
266
267 ret = copy_to_user(to, fifo->data + off, l);
268 if (unlikely(ret))
269 ret = DIV_ROUND_UP(ret + len - l, esize);
270 else {
271 ret = copy_to_user(to + l, fifo->data, len - l);
272 if (unlikely(ret))
273 ret = DIV_ROUND_UP(ret, esize);
274 }
200 /* 275 /*
201 * Ensure that we sample the fifo->in index -before- we 276 * make sure that the data is copied before
202 * start removing bytes from the kfifo. 277 * incrementing the fifo->out index counter
203 */ 278 */
279 smp_wmb();
280 *copied = len - ret;
281 /* return the number of elements which are not copied */
282 return ret;
283}
204 284
205 smp_rmb(); 285int __kfifo_to_user(struct __kfifo *fifo, void __user *to,
286 unsigned long len, unsigned int *copied)
287{
288 unsigned int l;
289 unsigned long ret;
290 unsigned int esize = fifo->esize;
291 int err;
206 292
207 off = __kfifo_off(fifo, fifo->out + off); 293 if (esize != 1)
294 len /= esize;
208 295
209 /* first get the data from fifo->out until the end of the buffer */ 296 l = fifo->in - fifo->out;
210 l = min(len, fifo->size - off); 297 if (len > l)
211 ret = copy_to_user(to, fifo->buffer + off, l); 298 len = l;
212 *lenout = l; 299 ret = kfifo_copy_to_user(fifo, to, len, fifo->out, copied);
213 if (unlikely(ret)) { 300 if (unlikely(ret)) {
214 *lenout -= ret; 301 len -= ret;
215 return -EFAULT; 302 err = -EFAULT;
216 } 303 } else
304 err = 0;
305 fifo->out += len;
306 return err;
307}
308EXPORT_SYMBOL(__kfifo_to_user);
217 309
218 /* then get the rest (if any) from the beginning of the buffer */ 310static int setup_sgl_buf(struct scatterlist *sgl, void *buf,
219 len -= l; 311 int nents, unsigned int len)
220 ret = copy_to_user(to + l, fifo->buffer, len); 312{
221 if (unlikely(ret)) { 313 int n;
222 *lenout += len - ret; 314 unsigned int l;
223 return -EFAULT; 315 unsigned int off;
316 struct page *page;
317
318 if (!nents)
319 return 0;
320
321 if (!len)
322 return 0;
323
324 n = 0;
325 page = virt_to_page(buf);
326 off = offset_in_page(buf);
327 l = 0;
328
329 while (len >= l + PAGE_SIZE - off) {
330 struct page *npage;
331
332 l += PAGE_SIZE;
333 buf += PAGE_SIZE;
334 npage = virt_to_page(buf);
335 if (page_to_phys(page) != page_to_phys(npage) - l) {
336 sgl->page_link = 0;
337 sg_set_page(sgl++, page, l - off, off);
338 if (++n == nents)
339 return n;
340 page = npage;
341 len -= l - off;
342 l = off = 0;
343 }
224 } 344 }
225 *lenout += len; 345 sgl->page_link = 0;
226 return 0; 346 sg_set_page(sgl++, page, len, off);
347 return n + 1;
227} 348}
228 349
229unsigned int __kfifo_in_n(struct kfifo *fifo, 350static unsigned int setup_sgl(struct __kfifo *fifo, struct scatterlist *sgl,
230 const void *from, unsigned int len, unsigned int recsize) 351 int nents, unsigned int len, unsigned int off)
231{ 352{
232 if (kfifo_avail(fifo) < len + recsize) 353 unsigned int size = fifo->mask + 1;
233 return len + 1; 354 unsigned int esize = fifo->esize;
355 unsigned int l;
356 unsigned int n;
234 357
235 __kfifo_in_data(fifo, from, len, recsize); 358 off &= fifo->mask;
236 return 0; 359 if (esize != 1) {
360 off *= esize;
361 size *= esize;
362 len *= esize;
363 }
364 l = min(len, size - off);
365
366 n = setup_sgl_buf(sgl, fifo->data + off, nents, l);
367 n += setup_sgl_buf(sgl + n, fifo->data, nents - n, len - l);
368
369 if (n)
370 sg_mark_end(sgl + n - 1);
371 return n;
237} 372}
238EXPORT_SYMBOL(__kfifo_in_n);
239 373
240/** 374unsigned int __kfifo_dma_in_prepare(struct __kfifo *fifo,
241 * kfifo_in - puts some data into the FIFO 375 struct scatterlist *sgl, int nents, unsigned int len)
242 * @fifo: the fifo to be used.
243 * @from: the data to be added.
244 * @len: the length of the data to be added.
245 *
246 * This function copies at most @len bytes from the @from buffer into
247 * the FIFO depending on the free space, and returns the number of
248 * bytes copied.
249 *
250 * Note that with only one concurrent reader and one concurrent
251 * writer, you don't need extra locking to use these functions.
252 */
253unsigned int kfifo_in(struct kfifo *fifo, const void *from,
254 unsigned int len)
255{ 376{
256 len = min(kfifo_avail(fifo), len); 377 unsigned int l;
257 378
258 __kfifo_in_data(fifo, from, len, 0); 379 l = kfifo_unused(fifo);
259 __kfifo_add_in(fifo, len); 380 if (len > l)
260 return len; 381 len = l;
382
383 return setup_sgl(fifo, sgl, nents, len, fifo->in);
261} 384}
262EXPORT_SYMBOL(kfifo_in); 385EXPORT_SYMBOL(__kfifo_dma_in_prepare);
263 386
264unsigned int __kfifo_in_generic(struct kfifo *fifo, 387unsigned int __kfifo_dma_out_prepare(struct __kfifo *fifo,
265 const void *from, unsigned int len, unsigned int recsize) 388 struct scatterlist *sgl, int nents, unsigned int len)
266{ 389{
267 return __kfifo_in_rec(fifo, from, len, recsize); 390 unsigned int l;
391
392 l = fifo->in - fifo->out;
393 if (len > l)
394 len = l;
395
396 return setup_sgl(fifo, sgl, nents, len, fifo->out);
268} 397}
269EXPORT_SYMBOL(__kfifo_in_generic); 398EXPORT_SYMBOL(__kfifo_dma_out_prepare);
270 399
271unsigned int __kfifo_out_n(struct kfifo *fifo, 400unsigned int __kfifo_max_r(unsigned int len, size_t recsize)
272 void *to, unsigned int len, unsigned int recsize)
273{ 401{
274 if (kfifo_len(fifo) < len + recsize) 402 unsigned int max = (1 << (recsize << 3)) - 1;
275 return len;
276 403
277 __kfifo_out_data(fifo, to, len, recsize); 404 if (len > max)
278 __kfifo_add_out(fifo, len + recsize); 405 return max;
279 return 0; 406 return len;
280} 407}
281EXPORT_SYMBOL(__kfifo_out_n);
282 408
283/** 409#define __KFIFO_PEEK(data, out, mask) \
284 * kfifo_out - gets some data from the FIFO 410 ((data)[(out) & (mask)])
285 * @fifo: the fifo to be used. 411/*
286 * @to: where the data must be copied. 412 * __kfifo_peek_n internal helper function for determinate the length of
287 * @len: the size of the destination buffer. 413 * the next record in the fifo
288 *
289 * This function copies at most @len bytes from the FIFO into the
290 * @to buffer and returns the number of copied bytes.
291 *
292 * Note that with only one concurrent reader and one concurrent
293 * writer, you don't need extra locking to use these functions.
294 */ 414 */
295unsigned int kfifo_out(struct kfifo *fifo, void *to, unsigned int len) 415static unsigned int __kfifo_peek_n(struct __kfifo *fifo, size_t recsize)
296{ 416{
297 len = min(kfifo_len(fifo), len); 417 unsigned int l;
418 unsigned int mask = fifo->mask;
419 unsigned char *data = fifo->data;
298 420
299 __kfifo_out_data(fifo, to, len, 0); 421 l = __KFIFO_PEEK(data, fifo->out, mask);
300 __kfifo_add_out(fifo, len);
301 422
302 return len; 423 if (--recsize)
424 l |= __KFIFO_PEEK(data, fifo->out + 1, mask) << 8;
425
426 return l;
303} 427}
304EXPORT_SYMBOL(kfifo_out); 428
305 429#define __KFIFO_POKE(data, in, mask, val) \
306/** 430 ( \
307 * kfifo_out_peek - copy some data from the FIFO, but do not remove it 431 (data)[(in) & (mask)] = (unsigned char)(val) \
308 * @fifo: the fifo to be used. 432 )
309 * @to: where the data must be copied. 433
310 * @len: the size of the destination buffer. 434/*
311 * @offset: offset into the fifo 435 * __kfifo_poke_n internal helper function for storeing the length of
312 * 436 * the record into the fifo
313 * This function copies at most @len bytes at @offset from the FIFO
314 * into the @to buffer and returns the number of copied bytes.
315 * The data is not removed from the FIFO.
316 */ 437 */
317unsigned int kfifo_out_peek(struct kfifo *fifo, void *to, unsigned int len, 438static void __kfifo_poke_n(struct __kfifo *fifo, unsigned int n, size_t recsize)
318 unsigned offset)
319{ 439{
320 len = min(kfifo_len(fifo), len + offset); 440 unsigned int mask = fifo->mask;
441 unsigned char *data = fifo->data;
321 442
322 __kfifo_out_data(fifo, to, len, offset); 443 __KFIFO_POKE(data, fifo->in, mask, n);
323 return len; 444
445 if (recsize > 1)
446 __KFIFO_POKE(data, fifo->in + 1, mask, n >> 8);
324} 447}
325EXPORT_SYMBOL(kfifo_out_peek);
326 448
327unsigned int __kfifo_out_generic(struct kfifo *fifo, 449unsigned int __kfifo_len_r(struct __kfifo *fifo, size_t recsize)
328 void *to, unsigned int len, unsigned int recsize,
329 unsigned int *total)
330{ 450{
331 return __kfifo_out_rec(fifo, to, len, recsize, total); 451 return __kfifo_peek_n(fifo, recsize);
332} 452}
333EXPORT_SYMBOL(__kfifo_out_generic); 453EXPORT_SYMBOL(__kfifo_len_r);
334 454
335unsigned int __kfifo_from_user_n(struct kfifo *fifo, 455unsigned int __kfifo_in_r(struct __kfifo *fifo, const void *buf,
336 const void __user *from, unsigned int len, unsigned int recsize) 456 unsigned int len, size_t recsize)
337{ 457{
338 unsigned total; 458 if (len + recsize > kfifo_unused(fifo))
459 return 0;
339 460
340 if (kfifo_avail(fifo) < len + recsize) 461 __kfifo_poke_n(fifo, len, recsize);
341 return len + 1;
342 462
343 __kfifo_from_user_data(fifo, from, len, recsize, &total); 463 kfifo_copy_in(fifo, buf, len, fifo->in + recsize);
344 return total; 464 fifo->in += len + recsize;
465 return len;
345} 466}
346EXPORT_SYMBOL(__kfifo_from_user_n); 467EXPORT_SYMBOL(__kfifo_in_r);
347 468
348/** 469static unsigned int kfifo_out_copy_r(struct __kfifo *fifo,
349 * kfifo_from_user - puts some data from user space into the FIFO 470 void *buf, unsigned int len, size_t recsize, unsigned int *n)
350 * @fifo: the fifo to be used.
351 * @from: pointer to the data to be added.
352 * @len: the length of the data to be added.
353 * @total: the actual returned data length.
354 *
355 * This function copies at most @len bytes from the @from into the
356 * FIFO depending and returns -EFAULT/0.
357 *
358 * Note that with only one concurrent reader and one concurrent
359 * writer, you don't need extra locking to use these functions.
360 */
361int kfifo_from_user(struct kfifo *fifo,
362 const void __user *from, unsigned int len, unsigned *total)
363{ 471{
364 int ret; 472 *n = __kfifo_peek_n(fifo, recsize);
365 len = min(kfifo_avail(fifo), len); 473
366 ret = __kfifo_from_user_data(fifo, from, len, 0, total); 474 if (len > *n)
367 if (ret) 475 len = *n;
368 return ret; 476
369 __kfifo_add_in(fifo, len); 477 kfifo_copy_out(fifo, buf, len, fifo->out + recsize);
370 return 0; 478 return len;
479}
480
481unsigned int __kfifo_out_peek_r(struct __kfifo *fifo, void *buf,
482 unsigned int len, size_t recsize)
483{
484 unsigned int n;
485
486 if (fifo->in == fifo->out)
487 return 0;
488
489 return kfifo_out_copy_r(fifo, buf, len, recsize, &n);
371} 490}
372EXPORT_SYMBOL(kfifo_from_user); 491EXPORT_SYMBOL(__kfifo_out_peek_r);
373 492
374unsigned int __kfifo_from_user_generic(struct kfifo *fifo, 493unsigned int __kfifo_out_r(struct __kfifo *fifo, void *buf,
375 const void __user *from, unsigned int len, unsigned int recsize) 494 unsigned int len, size_t recsize)
376{ 495{
377 return __kfifo_from_user_rec(fifo, from, len, recsize); 496 unsigned int n;
497
498 if (fifo->in == fifo->out)
499 return 0;
500
501 len = kfifo_out_copy_r(fifo, buf, len, recsize, &n);
502 fifo->out += n + recsize;
503 return len;
378} 504}
379EXPORT_SYMBOL(__kfifo_from_user_generic); 505EXPORT_SYMBOL(__kfifo_out_r);
380 506
381unsigned int __kfifo_to_user_n(struct kfifo *fifo, 507int __kfifo_from_user_r(struct __kfifo *fifo, const void __user *from,
382 void __user *to, unsigned int len, unsigned int reclen, 508 unsigned long len, unsigned int *copied, size_t recsize)
383 unsigned int recsize)
384{ 509{
385 unsigned int ret, total; 510 unsigned long ret;
386 511
387 if (kfifo_len(fifo) < reclen + recsize) 512 len = __kfifo_max_r(len, recsize);
388 return len;
389 513
390 ret = __kfifo_to_user_data(fifo, to, reclen, recsize, &total); 514 if (len + recsize > kfifo_unused(fifo)) {
515 *copied = 0;
516 return 0;
517 }
391 518
392 if (likely(ret == 0)) 519 __kfifo_poke_n(fifo, len, recsize);
393 __kfifo_add_out(fifo, reclen + recsize);
394 520
395 return total; 521 ret = kfifo_copy_from_user(fifo, from, len, fifo->in + recsize, copied);
522 if (unlikely(ret)) {
523 *copied = 0;
524 return -EFAULT;
525 }
526 fifo->in += len + recsize;
527 return 0;
396} 528}
397EXPORT_SYMBOL(__kfifo_to_user_n); 529EXPORT_SYMBOL(__kfifo_from_user_r);
398 530
399/** 531int __kfifo_to_user_r(struct __kfifo *fifo, void __user *to,
400 * kfifo_to_user - gets data from the FIFO and write it to user space 532 unsigned long len, unsigned int *copied, size_t recsize)
401 * @fifo: the fifo to be used.
402 * @to: where the data must be copied.
403 * @len: the size of the destination buffer.
404 * @lenout: pointer to output variable with copied data
405 *
406 * This function copies at most @len bytes from the FIFO into the
407 * @to buffer and 0 or -EFAULT.
408 *
409 * Note that with only one concurrent reader and one concurrent
410 * writer, you don't need extra locking to use these functions.
411 */
412int kfifo_to_user(struct kfifo *fifo,
413 void __user *to, unsigned int len, unsigned *lenout)
414{ 533{
415 int ret; 534 unsigned long ret;
416 len = min(kfifo_len(fifo), len); 535 unsigned int n;
417 ret = __kfifo_to_user_data(fifo, to, len, 0, lenout); 536
418 __kfifo_add_out(fifo, *lenout); 537 if (fifo->in == fifo->out) {
419 return ret; 538 *copied = 0;
539 return 0;
540 }
541
542 n = __kfifo_peek_n(fifo, recsize);
543 if (len > n)
544 len = n;
545
546 ret = kfifo_copy_to_user(fifo, to, len, fifo->out + recsize, copied);
547 if (unlikely(ret)) {
548 *copied = 0;
549 return -EFAULT;
550 }
551 fifo->out += n + recsize;
552 return 0;
420} 553}
421EXPORT_SYMBOL(kfifo_to_user); 554EXPORT_SYMBOL(__kfifo_to_user_r);
422 555
423unsigned int __kfifo_to_user_generic(struct kfifo *fifo, 556unsigned int __kfifo_dma_in_prepare_r(struct __kfifo *fifo,
424 void __user *to, unsigned int len, unsigned int recsize, 557 struct scatterlist *sgl, int nents, unsigned int len, size_t recsize)
425 unsigned int *total)
426{ 558{
427 return __kfifo_to_user_rec(fifo, to, len, recsize, total); 559 if (!nents)
560 BUG();
561
562 len = __kfifo_max_r(len, recsize);
563
564 if (len + recsize > kfifo_unused(fifo))
565 return 0;
566
567 return setup_sgl(fifo, sgl, nents, len, fifo->in + recsize);
428} 568}
429EXPORT_SYMBOL(__kfifo_to_user_generic); 569EXPORT_SYMBOL(__kfifo_dma_in_prepare_r);
430 570
431unsigned int __kfifo_peek_generic(struct kfifo *fifo, unsigned int recsize) 571void __kfifo_dma_in_finish_r(struct __kfifo *fifo,
572 unsigned int len, size_t recsize)
432{ 573{
433 if (recsize == 0) 574 len = __kfifo_max_r(len, recsize);
434 return kfifo_avail(fifo); 575 __kfifo_poke_n(fifo, len, recsize);
435 576 fifo->in += len + recsize;
436 return __kfifo_peek_n(fifo, recsize);
437} 577}
438EXPORT_SYMBOL(__kfifo_peek_generic); 578EXPORT_SYMBOL(__kfifo_dma_in_finish_r);
439 579
440void __kfifo_skip_generic(struct kfifo *fifo, unsigned int recsize) 580unsigned int __kfifo_dma_out_prepare_r(struct __kfifo *fifo,
581 struct scatterlist *sgl, int nents, unsigned int len, size_t recsize)
441{ 582{
442 __kfifo_skip_rec(fifo, recsize); 583 if (!nents)
584 BUG();
585
586 len = __kfifo_max_r(len, recsize);
587
588 if (len + recsize > fifo->in - fifo->out)
589 return 0;
590
591 return setup_sgl(fifo, sgl, nents, len, fifo->out + recsize);
443} 592}
444EXPORT_SYMBOL(__kfifo_skip_generic); 593EXPORT_SYMBOL(__kfifo_dma_out_prepare_r);
594
595void __kfifo_dma_out_finish_r(struct __kfifo *fifo, size_t recsize)
596{
597 unsigned int len;
445 598
599 len = __kfifo_peek_n(fifo, recsize);
600 fifo->out += len + recsize;
601}
602EXPORT_SYMBOL(__kfifo_dma_out_finish_r);
generated by cgit v1.2.2 (git 2.25.0) at 2024-12-30 12:03:28 -0500