diff options
Diffstat (limited to 'kernel')
-rw-r--r-- | kernel/Makefile | 1 | ||||
-rw-r--r-- | kernel/relay.c | 919 |
2 files changed, 920 insertions, 0 deletions
diff --git a/kernel/Makefile b/kernel/Makefile index 4ae0fbde815d..aebd7a78984e 100644 --- a/kernel/Makefile +++ b/kernel/Makefile | |||
@@ -34,6 +34,7 @@ obj-$(CONFIG_DETECT_SOFTLOCKUP) += softlockup.o | |||
34 | obj-$(CONFIG_GENERIC_HARDIRQS) += irq/ | 34 | obj-$(CONFIG_GENERIC_HARDIRQS) += irq/ |
35 | obj-$(CONFIG_SECCOMP) += seccomp.o | 35 | obj-$(CONFIG_SECCOMP) += seccomp.o |
36 | obj-$(CONFIG_RCU_TORTURE_TEST) += rcutorture.o | 36 | obj-$(CONFIG_RCU_TORTURE_TEST) += rcutorture.o |
37 | obj-$(CONFIG_RELAY) += relay.o | ||
37 | 38 | ||
38 | ifneq ($(CONFIG_SCHED_NO_NO_OMIT_FRAME_POINTER),y) | 39 | ifneq ($(CONFIG_SCHED_NO_NO_OMIT_FRAME_POINTER),y) |
39 | # According to Alan Modra <alan@linuxcare.com.au>, the -fno-omit-frame-pointer is | 40 | # According to Alan Modra <alan@linuxcare.com.au>, the -fno-omit-frame-pointer is |
diff --git a/kernel/relay.c b/kernel/relay.c new file mode 100644 index 000000000000..9358e8eb8476 --- /dev/null +++ b/kernel/relay.c | |||
@@ -0,0 +1,919 @@ | |||
1 | /* | ||
2 | * Public API and common code for kernel->userspace relay file support. | ||
3 | * | ||
4 | * See Documentation/filesystems/relayfs.txt for an overview of relayfs. | ||
5 | * | ||
6 | * Copyright (C) 2002-2005 - Tom Zanussi (zanussi@us.ibm.com), IBM Corp | ||
7 | * Copyright (C) 1999-2005 - Karim Yaghmour (karim@opersys.com) | ||
8 | * | ||
9 | * Moved to kernel/relay.c by Paul Mundt, 2006. | ||
10 | * | ||
11 | * This file is released under the GPL. | ||
12 | */ | ||
13 | #include <linux/errno.h> | ||
14 | #include <linux/stddef.h> | ||
15 | #include <linux/slab.h> | ||
16 | #include <linux/module.h> | ||
17 | #include <linux/string.h> | ||
18 | #include <linux/relay.h> | ||
19 | #include <linux/vmalloc.h> | ||
20 | #include <linux/mm.h> | ||
21 | |||
22 | /* | ||
23 | * close() vm_op implementation for relay file mapping. | ||
24 | */ | ||
25 | static void relay_file_mmap_close(struct vm_area_struct *vma) | ||
26 | { | ||
27 | struct rchan_buf *buf = vma->vm_private_data; | ||
28 | buf->chan->cb->buf_unmapped(buf, vma->vm_file); | ||
29 | } | ||
30 | |||
31 | /* | ||
32 | * nopage() vm_op implementation for relay file mapping. | ||
33 | */ | ||
34 | static struct page *relay_buf_nopage(struct vm_area_struct *vma, | ||
35 | unsigned long address, | ||
36 | int *type) | ||
37 | { | ||
38 | struct page *page; | ||
39 | struct rchan_buf *buf = vma->vm_private_data; | ||
40 | unsigned long offset = address - vma->vm_start; | ||
41 | |||
42 | if (address > vma->vm_end) | ||
43 | return NOPAGE_SIGBUS; /* Disallow mremap */ | ||
44 | if (!buf) | ||
45 | return NOPAGE_OOM; | ||
46 | |||
47 | page = vmalloc_to_page(buf->start + offset); | ||
48 | if (!page) | ||
49 | return NOPAGE_OOM; | ||
50 | get_page(page); | ||
51 | |||
52 | if (type) | ||
53 | *type = VM_FAULT_MINOR; | ||
54 | |||
55 | return page; | ||
56 | } | ||
57 | |||
58 | /* | ||
59 | * vm_ops for relay file mappings. | ||
60 | */ | ||
61 | static struct vm_operations_struct relay_file_mmap_ops = { | ||
62 | .nopage = relay_buf_nopage, | ||
63 | .close = relay_file_mmap_close, | ||
64 | }; | ||
65 | |||
66 | /** | ||
67 | * relay_mmap_buf: - mmap channel buffer to process address space | ||
68 | * @buf: relay channel buffer | ||
69 | * @vma: vm_area_struct describing memory to be mapped | ||
70 | * | ||
71 | * Returns 0 if ok, negative on error | ||
72 | * | ||
73 | * Caller should already have grabbed mmap_sem. | ||
74 | */ | ||
75 | int relay_mmap_buf(struct rchan_buf *buf, struct vm_area_struct *vma) | ||
76 | { | ||
77 | unsigned long length = vma->vm_end - vma->vm_start; | ||
78 | struct file *filp = vma->vm_file; | ||
79 | |||
80 | if (!buf) | ||
81 | return -EBADF; | ||
82 | |||
83 | if (length != (unsigned long)buf->chan->alloc_size) | ||
84 | return -EINVAL; | ||
85 | |||
86 | vma->vm_ops = &relay_file_mmap_ops; | ||
87 | vma->vm_private_data = buf; | ||
88 | buf->chan->cb->buf_mapped(buf, filp); | ||
89 | |||
90 | return 0; | ||
91 | } | ||
92 | |||
93 | /** | ||
94 | * relay_alloc_buf - allocate a channel buffer | ||
95 | * @buf: the buffer struct | ||
96 | * @size: total size of the buffer | ||
97 | * | ||
98 | * Returns a pointer to the resulting buffer, NULL if unsuccessful | ||
99 | */ | ||
100 | static void *relay_alloc_buf(struct rchan_buf *buf, unsigned long size) | ||
101 | { | ||
102 | void *mem; | ||
103 | unsigned int i, j, n_pages; | ||
104 | |||
105 | size = PAGE_ALIGN(size); | ||
106 | n_pages = size >> PAGE_SHIFT; | ||
107 | |||
108 | buf->page_array = kcalloc(n_pages, sizeof(struct page *), GFP_KERNEL); | ||
109 | if (!buf->page_array) | ||
110 | return NULL; | ||
111 | |||
112 | for (i = 0; i < n_pages; i++) { | ||
113 | buf->page_array[i] = alloc_page(GFP_KERNEL); | ||
114 | if (unlikely(!buf->page_array[i])) | ||
115 | goto depopulate; | ||
116 | } | ||
117 | mem = vmap(buf->page_array, n_pages, VM_MAP, PAGE_KERNEL); | ||
118 | if (!mem) | ||
119 | goto depopulate; | ||
120 | |||
121 | memset(mem, 0, size); | ||
122 | buf->page_count = n_pages; | ||
123 | return mem; | ||
124 | |||
125 | depopulate: | ||
126 | for (j = 0; j < i; j++) | ||
127 | __free_page(buf->page_array[j]); | ||
128 | kfree(buf->page_array); | ||
129 | return NULL; | ||
130 | } | ||
131 | |||
132 | /** | ||
133 | * relay_create_buf - allocate and initialize a channel buffer | ||
134 | * @alloc_size: size of the buffer to allocate | ||
135 | * @n_subbufs: number of sub-buffers in the channel | ||
136 | * | ||
137 | * Returns channel buffer if successful, NULL otherwise | ||
138 | */ | ||
139 | struct rchan_buf *relay_create_buf(struct rchan *chan) | ||
140 | { | ||
141 | struct rchan_buf *buf = kcalloc(1, sizeof(struct rchan_buf), GFP_KERNEL); | ||
142 | if (!buf) | ||
143 | return NULL; | ||
144 | |||
145 | buf->padding = kmalloc(chan->n_subbufs * sizeof(size_t *), GFP_KERNEL); | ||
146 | if (!buf->padding) | ||
147 | goto free_buf; | ||
148 | |||
149 | buf->start = relay_alloc_buf(buf, chan->alloc_size); | ||
150 | if (!buf->start) | ||
151 | goto free_buf; | ||
152 | |||
153 | buf->chan = chan; | ||
154 | kref_get(&buf->chan->kref); | ||
155 | return buf; | ||
156 | |||
157 | free_buf: | ||
158 | kfree(buf->padding); | ||
159 | kfree(buf); | ||
160 | return NULL; | ||
161 | } | ||
162 | |||
163 | /** | ||
164 | * relay_destroy_channel - free the channel struct | ||
165 | * | ||
166 | * Should only be called from kref_put(). | ||
167 | */ | ||
168 | void relay_destroy_channel(struct kref *kref) | ||
169 | { | ||
170 | struct rchan *chan = container_of(kref, struct rchan, kref); | ||
171 | kfree(chan); | ||
172 | } | ||
173 | |||
174 | /** | ||
175 | * relay_destroy_buf - destroy an rchan_buf struct and associated buffer | ||
176 | * @buf: the buffer struct | ||
177 | */ | ||
178 | void relay_destroy_buf(struct rchan_buf *buf) | ||
179 | { | ||
180 | struct rchan *chan = buf->chan; | ||
181 | unsigned int i; | ||
182 | |||
183 | if (likely(buf->start)) { | ||
184 | vunmap(buf->start); | ||
185 | for (i = 0; i < buf->page_count; i++) | ||
186 | __free_page(buf->page_array[i]); | ||
187 | kfree(buf->page_array); | ||
188 | } | ||
189 | kfree(buf->padding); | ||
190 | kfree(buf); | ||
191 | kref_put(&chan->kref, relay_destroy_channel); | ||
192 | } | ||
193 | |||
194 | /** | ||
195 | * relay_remove_buf - remove a channel buffer | ||
196 | * | ||
197 | * Removes the file from the fileystem, which also frees the | ||
198 | * rchan_buf_struct and the channel buffer. Should only be called from | ||
199 | * kref_put(). | ||
200 | */ | ||
201 | void relay_remove_buf(struct kref *kref) | ||
202 | { | ||
203 | struct rchan_buf *buf = container_of(kref, struct rchan_buf, kref); | ||
204 | buf->chan->cb->remove_buf_file(buf->dentry); | ||
205 | relay_destroy_buf(buf); | ||
206 | } | ||
207 | |||
208 | /** | ||
209 | * relay_buf_empty - boolean, is the channel buffer empty? | ||
210 | * @buf: channel buffer | ||
211 | * | ||
212 | * Returns 1 if the buffer is empty, 0 otherwise. | ||
213 | */ | ||
214 | int relay_buf_empty(struct rchan_buf *buf) | ||
215 | { | ||
216 | return (buf->subbufs_produced - buf->subbufs_consumed) ? 0 : 1; | ||
217 | } | ||
218 | EXPORT_SYMBOL_GPL(relay_buf_empty); | ||
219 | |||
220 | /** | ||
221 | * relay_buf_full - boolean, is the channel buffer full? | ||
222 | * @buf: channel buffer | ||
223 | * | ||
224 | * Returns 1 if the buffer is full, 0 otherwise. | ||
225 | */ | ||
226 | int relay_buf_full(struct rchan_buf *buf) | ||
227 | { | ||
228 | size_t ready = buf->subbufs_produced - buf->subbufs_consumed; | ||
229 | return (ready >= buf->chan->n_subbufs) ? 1 : 0; | ||
230 | } | ||
231 | EXPORT_SYMBOL_GPL(relay_buf_full); | ||
232 | |||
233 | /* | ||
234 | * High-level relay kernel API and associated functions. | ||
235 | */ | ||
236 | |||
237 | /* | ||
238 | * rchan_callback implementations defining default channel behavior. Used | ||
239 | * in place of corresponding NULL values in client callback struct. | ||
240 | */ | ||
241 | |||
242 | /* | ||
243 | * subbuf_start() default callback. Does nothing. | ||
244 | */ | ||
245 | static int subbuf_start_default_callback (struct rchan_buf *buf, | ||
246 | void *subbuf, | ||
247 | void *prev_subbuf, | ||
248 | size_t prev_padding) | ||
249 | { | ||
250 | if (relay_buf_full(buf)) | ||
251 | return 0; | ||
252 | |||
253 | return 1; | ||
254 | } | ||
255 | |||
256 | /* | ||
257 | * buf_mapped() default callback. Does nothing. | ||
258 | */ | ||
259 | static void buf_mapped_default_callback(struct rchan_buf *buf, | ||
260 | struct file *filp) | ||
261 | { | ||
262 | } | ||
263 | |||
264 | /* | ||
265 | * buf_unmapped() default callback. Does nothing. | ||
266 | */ | ||
267 | static void buf_unmapped_default_callback(struct rchan_buf *buf, | ||
268 | struct file *filp) | ||
269 | { | ||
270 | } | ||
271 | |||
272 | /* | ||
273 | * create_buf_file_create() default callback. Does nothing. | ||
274 | */ | ||
275 | static struct dentry *create_buf_file_default_callback(const char *filename, | ||
276 | struct dentry *parent, | ||
277 | int mode, | ||
278 | struct rchan_buf *buf, | ||
279 | int *is_global) | ||
280 | { | ||
281 | return NULL; | ||
282 | } | ||
283 | |||
284 | /* | ||
285 | * remove_buf_file() default callback. Does nothing. | ||
286 | */ | ||
287 | static int remove_buf_file_default_callback(struct dentry *dentry) | ||
288 | { | ||
289 | return -EINVAL; | ||
290 | } | ||
291 | |||
292 | /* relay channel default callbacks */ | ||
293 | static struct rchan_callbacks default_channel_callbacks = { | ||
294 | .subbuf_start = subbuf_start_default_callback, | ||
295 | .buf_mapped = buf_mapped_default_callback, | ||
296 | .buf_unmapped = buf_unmapped_default_callback, | ||
297 | .create_buf_file = create_buf_file_default_callback, | ||
298 | .remove_buf_file = remove_buf_file_default_callback, | ||
299 | }; | ||
300 | |||
301 | /** | ||
302 | * wakeup_readers - wake up readers waiting on a channel | ||
303 | * @private: the channel buffer | ||
304 | * | ||
305 | * This is the work function used to defer reader waking. The | ||
306 | * reason waking is deferred is that calling directly from write | ||
307 | * causes problems if you're writing from say the scheduler. | ||
308 | */ | ||
309 | static void wakeup_readers(void *private) | ||
310 | { | ||
311 | struct rchan_buf *buf = private; | ||
312 | wake_up_interruptible(&buf->read_wait); | ||
313 | } | ||
314 | |||
315 | /** | ||
316 | * __relay_reset - reset a channel buffer | ||
317 | * @buf: the channel buffer | ||
318 | * @init: 1 if this is a first-time initialization | ||
319 | * | ||
320 | * See relay_reset for description of effect. | ||
321 | */ | ||
322 | static inline void __relay_reset(struct rchan_buf *buf, unsigned int init) | ||
323 | { | ||
324 | size_t i; | ||
325 | |||
326 | if (init) { | ||
327 | init_waitqueue_head(&buf->read_wait); | ||
328 | kref_init(&buf->kref); | ||
329 | INIT_WORK(&buf->wake_readers, NULL, NULL); | ||
330 | } else { | ||
331 | cancel_delayed_work(&buf->wake_readers); | ||
332 | flush_scheduled_work(); | ||
333 | } | ||
334 | |||
335 | buf->subbufs_produced = 0; | ||
336 | buf->subbufs_consumed = 0; | ||
337 | buf->bytes_consumed = 0; | ||
338 | buf->finalized = 0; | ||
339 | buf->data = buf->start; | ||
340 | buf->offset = 0; | ||
341 | |||
342 | for (i = 0; i < buf->chan->n_subbufs; i++) | ||
343 | buf->padding[i] = 0; | ||
344 | |||
345 | buf->chan->cb->subbuf_start(buf, buf->data, NULL, 0); | ||
346 | } | ||
347 | |||
348 | /** | ||
349 | * relay_reset - reset the channel | ||
350 | * @chan: the channel | ||
351 | * | ||
352 | * This has the effect of erasing all data from all channel buffers | ||
353 | * and restarting the channel in its initial state. The buffers | ||
354 | * are not freed, so any mappings are still in effect. | ||
355 | * | ||
356 | * NOTE: Care should be taken that the channel isn't actually | ||
357 | * being used by anything when this call is made. | ||
358 | */ | ||
359 | void relay_reset(struct rchan *chan) | ||
360 | { | ||
361 | unsigned int i; | ||
362 | struct rchan_buf *prev = NULL; | ||
363 | |||
364 | if (!chan) | ||
365 | return; | ||
366 | |||
367 | for (i = 0; i < NR_CPUS; i++) { | ||
368 | if (!chan->buf[i] || chan->buf[i] == prev) | ||
369 | break; | ||
370 | __relay_reset(chan->buf[i], 0); | ||
371 | prev = chan->buf[i]; | ||
372 | } | ||
373 | } | ||
374 | EXPORT_SYMBOL_GPL(relay_reset); | ||
375 | |||
376 | /** | ||
377 | * relay_open_buf - create a new relay channel buffer | ||
378 | * | ||
379 | * Internal - used by relay_open(). | ||
380 | */ | ||
381 | static struct rchan_buf *relay_open_buf(struct rchan *chan, | ||
382 | const char *filename, | ||
383 | struct dentry *parent, | ||
384 | int *is_global) | ||
385 | { | ||
386 | struct rchan_buf *buf; | ||
387 | struct dentry *dentry; | ||
388 | |||
389 | if (*is_global) | ||
390 | return chan->buf[0]; | ||
391 | |||
392 | buf = relay_create_buf(chan); | ||
393 | if (!buf) | ||
394 | return NULL; | ||
395 | |||
396 | /* Create file in fs */ | ||
397 | dentry = chan->cb->create_buf_file(filename, parent, S_IRUSR, | ||
398 | buf, is_global); | ||
399 | if (!dentry) { | ||
400 | relay_destroy_buf(buf); | ||
401 | return NULL; | ||
402 | } | ||
403 | |||
404 | buf->dentry = dentry; | ||
405 | __relay_reset(buf, 1); | ||
406 | |||
407 | return buf; | ||
408 | } | ||
409 | |||
410 | /** | ||
411 | * relay_close_buf - close a channel buffer | ||
412 | * @buf: channel buffer | ||
413 | * | ||
414 | * Marks the buffer finalized and restores the default callbacks. | ||
415 | * The channel buffer and channel buffer data structure are then freed | ||
416 | * automatically when the last reference is given up. | ||
417 | */ | ||
418 | static inline void relay_close_buf(struct rchan_buf *buf) | ||
419 | { | ||
420 | buf->finalized = 1; | ||
421 | cancel_delayed_work(&buf->wake_readers); | ||
422 | flush_scheduled_work(); | ||
423 | kref_put(&buf->kref, relay_remove_buf); | ||
424 | } | ||
425 | |||
426 | static inline void setup_callbacks(struct rchan *chan, | ||
427 | struct rchan_callbacks *cb) | ||
428 | { | ||
429 | if (!cb) { | ||
430 | chan->cb = &default_channel_callbacks; | ||
431 | return; | ||
432 | } | ||
433 | |||
434 | if (!cb->subbuf_start) | ||
435 | cb->subbuf_start = subbuf_start_default_callback; | ||
436 | if (!cb->buf_mapped) | ||
437 | cb->buf_mapped = buf_mapped_default_callback; | ||
438 | if (!cb->buf_unmapped) | ||
439 | cb->buf_unmapped = buf_unmapped_default_callback; | ||
440 | if (!cb->create_buf_file) | ||
441 | cb->create_buf_file = create_buf_file_default_callback; | ||
442 | if (!cb->remove_buf_file) | ||
443 | cb->remove_buf_file = remove_buf_file_default_callback; | ||
444 | chan->cb = cb; | ||
445 | } | ||
446 | |||
447 | /** | ||
448 | * relay_open - create a new relay channel | ||
449 | * @base_filename: base name of files to create | ||
450 | * @parent: dentry of parent directory, NULL for root directory | ||
451 | * @subbuf_size: size of sub-buffers | ||
452 | * @n_subbufs: number of sub-buffers | ||
453 | * @cb: client callback functions | ||
454 | * | ||
455 | * Returns channel pointer if successful, NULL otherwise. | ||
456 | * | ||
457 | * Creates a channel buffer for each cpu using the sizes and | ||
458 | * attributes specified. The created channel buffer files | ||
459 | * will be named base_filename0...base_filenameN-1. File | ||
460 | * permissions will be S_IRUSR. | ||
461 | */ | ||
462 | struct rchan *relay_open(const char *base_filename, | ||
463 | struct dentry *parent, | ||
464 | size_t subbuf_size, | ||
465 | size_t n_subbufs, | ||
466 | struct rchan_callbacks *cb) | ||
467 | { | ||
468 | unsigned int i; | ||
469 | struct rchan *chan; | ||
470 | char *tmpname; | ||
471 | int is_global = 0; | ||
472 | |||
473 | if (!base_filename) | ||
474 | return NULL; | ||
475 | |||
476 | if (!(subbuf_size && n_subbufs)) | ||
477 | return NULL; | ||
478 | |||
479 | chan = kcalloc(1, sizeof(struct rchan), GFP_KERNEL); | ||
480 | if (!chan) | ||
481 | return NULL; | ||
482 | |||
483 | chan->version = RELAYFS_CHANNEL_VERSION; | ||
484 | chan->n_subbufs = n_subbufs; | ||
485 | chan->subbuf_size = subbuf_size; | ||
486 | chan->alloc_size = FIX_SIZE(subbuf_size * n_subbufs); | ||
487 | setup_callbacks(chan, cb); | ||
488 | kref_init(&chan->kref); | ||
489 | |||
490 | tmpname = kmalloc(NAME_MAX + 1, GFP_KERNEL); | ||
491 | if (!tmpname) | ||
492 | goto free_chan; | ||
493 | |||
494 | for_each_online_cpu(i) { | ||
495 | sprintf(tmpname, "%s%d", base_filename, i); | ||
496 | chan->buf[i] = relay_open_buf(chan, tmpname, parent, | ||
497 | &is_global); | ||
498 | if (!chan->buf[i]) | ||
499 | goto free_bufs; | ||
500 | |||
501 | chan->buf[i]->cpu = i; | ||
502 | } | ||
503 | |||
504 | kfree(tmpname); | ||
505 | return chan; | ||
506 | |||
507 | free_bufs: | ||
508 | for (i = 0; i < NR_CPUS; i++) { | ||
509 | if (!chan->buf[i]) | ||
510 | break; | ||
511 | relay_close_buf(chan->buf[i]); | ||
512 | if (is_global) | ||
513 | break; | ||
514 | } | ||
515 | kfree(tmpname); | ||
516 | |||
517 | free_chan: | ||
518 | kref_put(&chan->kref, relay_destroy_channel); | ||
519 | return NULL; | ||
520 | } | ||
521 | EXPORT_SYMBOL_GPL(relay_open); | ||
522 | |||
523 | /** | ||
524 | * relay_switch_subbuf - switch to a new sub-buffer | ||
525 | * @buf: channel buffer | ||
526 | * @length: size of current event | ||
527 | * | ||
528 | * Returns either the length passed in or 0 if full. | ||
529 | * | ||
530 | * Performs sub-buffer-switch tasks such as invoking callbacks, | ||
531 | * updating padding counts, waking up readers, etc. | ||
532 | */ | ||
533 | size_t relay_switch_subbuf(struct rchan_buf *buf, size_t length) | ||
534 | { | ||
535 | void *old, *new; | ||
536 | size_t old_subbuf, new_subbuf; | ||
537 | |||
538 | if (unlikely(length > buf->chan->subbuf_size)) | ||
539 | goto toobig; | ||
540 | |||
541 | if (buf->offset != buf->chan->subbuf_size + 1) { | ||
542 | buf->prev_padding = buf->chan->subbuf_size - buf->offset; | ||
543 | old_subbuf = buf->subbufs_produced % buf->chan->n_subbufs; | ||
544 | buf->padding[old_subbuf] = buf->prev_padding; | ||
545 | buf->subbufs_produced++; | ||
546 | if (waitqueue_active(&buf->read_wait)) { | ||
547 | PREPARE_WORK(&buf->wake_readers, wakeup_readers, buf); | ||
548 | schedule_delayed_work(&buf->wake_readers, 1); | ||
549 | } | ||
550 | } | ||
551 | |||
552 | old = buf->data; | ||
553 | new_subbuf = buf->subbufs_produced % buf->chan->n_subbufs; | ||
554 | new = buf->start + new_subbuf * buf->chan->subbuf_size; | ||
555 | buf->offset = 0; | ||
556 | if (!buf->chan->cb->subbuf_start(buf, new, old, buf->prev_padding)) { | ||
557 | buf->offset = buf->chan->subbuf_size + 1; | ||
558 | return 0; | ||
559 | } | ||
560 | buf->data = new; | ||
561 | buf->padding[new_subbuf] = 0; | ||
562 | |||
563 | if (unlikely(length + buf->offset > buf->chan->subbuf_size)) | ||
564 | goto toobig; | ||
565 | |||
566 | return length; | ||
567 | |||
568 | toobig: | ||
569 | buf->chan->last_toobig = length; | ||
570 | return 0; | ||
571 | } | ||
572 | EXPORT_SYMBOL_GPL(relay_switch_subbuf); | ||
573 | |||
574 | /** | ||
575 | * relay_subbufs_consumed - update the buffer's sub-buffers-consumed count | ||
576 | * @chan: the channel | ||
577 | * @cpu: the cpu associated with the channel buffer to update | ||
578 | * @subbufs_consumed: number of sub-buffers to add to current buf's count | ||
579 | * | ||
580 | * Adds to the channel buffer's consumed sub-buffer count. | ||
581 | * subbufs_consumed should be the number of sub-buffers newly consumed, | ||
582 | * not the total consumed. | ||
583 | * | ||
584 | * NOTE: kernel clients don't need to call this function if the channel | ||
585 | * mode is 'overwrite'. | ||
586 | */ | ||
587 | void relay_subbufs_consumed(struct rchan *chan, | ||
588 | unsigned int cpu, | ||
589 | size_t subbufs_consumed) | ||
590 | { | ||
591 | struct rchan_buf *buf; | ||
592 | |||
593 | if (!chan) | ||
594 | return; | ||
595 | |||
596 | if (cpu >= NR_CPUS || !chan->buf[cpu]) | ||
597 | return; | ||
598 | |||
599 | buf = chan->buf[cpu]; | ||
600 | buf->subbufs_consumed += subbufs_consumed; | ||
601 | if (buf->subbufs_consumed > buf->subbufs_produced) | ||
602 | buf->subbufs_consumed = buf->subbufs_produced; | ||
603 | } | ||
604 | EXPORT_SYMBOL_GPL(relay_subbufs_consumed); | ||
605 | |||
606 | /** | ||
607 | * relay_close - close the channel | ||
608 | * @chan: the channel | ||
609 | * | ||
610 | * Closes all channel buffers and frees the channel. | ||
611 | */ | ||
612 | void relay_close(struct rchan *chan) | ||
613 | { | ||
614 | unsigned int i; | ||
615 | struct rchan_buf *prev = NULL; | ||
616 | |||
617 | if (!chan) | ||
618 | return; | ||
619 | |||
620 | for (i = 0; i < NR_CPUS; i++) { | ||
621 | if (!chan->buf[i] || chan->buf[i] == prev) | ||
622 | break; | ||
623 | relay_close_buf(chan->buf[i]); | ||
624 | prev = chan->buf[i]; | ||
625 | } | ||
626 | |||
627 | if (chan->last_toobig) | ||
628 | printk(KERN_WARNING "relay: one or more items not logged " | ||
629 | "[item size (%Zd) > sub-buffer size (%Zd)]\n", | ||
630 | chan->last_toobig, chan->subbuf_size); | ||
631 | |||
632 | kref_put(&chan->kref, relay_destroy_channel); | ||
633 | } | ||
634 | EXPORT_SYMBOL_GPL(relay_close); | ||
635 | |||
636 | /** | ||
637 | * relay_flush - close the channel | ||
638 | * @chan: the channel | ||
639 | * | ||
640 | * Flushes all channel buffers i.e. forces buffer switch. | ||
641 | */ | ||
642 | void relay_flush(struct rchan *chan) | ||
643 | { | ||
644 | unsigned int i; | ||
645 | struct rchan_buf *prev = NULL; | ||
646 | |||
647 | if (!chan) | ||
648 | return; | ||
649 | |||
650 | for (i = 0; i < NR_CPUS; i++) { | ||
651 | if (!chan->buf[i] || chan->buf[i] == prev) | ||
652 | break; | ||
653 | relay_switch_subbuf(chan->buf[i], 0); | ||
654 | prev = chan->buf[i]; | ||
655 | } | ||
656 | } | ||
657 | EXPORT_SYMBOL_GPL(relay_flush); | ||
658 | |||
659 | /** | ||
660 | * relay_file_open - open file op for relay files | ||
661 | * @inode: the inode | ||
662 | * @filp: the file | ||
663 | * | ||
664 | * Increments the channel buffer refcount. | ||
665 | */ | ||
666 | static int relay_file_open(struct inode *inode, struct file *filp) | ||
667 | { | ||
668 | struct rchan_buf *buf = inode->u.generic_ip; | ||
669 | kref_get(&buf->kref); | ||
670 | filp->private_data = buf; | ||
671 | |||
672 | return 0; | ||
673 | } | ||
674 | |||
675 | /** | ||
676 | * relay_file_mmap - mmap file op for relay files | ||
677 | * @filp: the file | ||
678 | * @vma: the vma describing what to map | ||
679 | * | ||
680 | * Calls upon relay_mmap_buf to map the file into user space. | ||
681 | */ | ||
682 | static int relay_file_mmap(struct file *filp, struct vm_area_struct *vma) | ||
683 | { | ||
684 | struct rchan_buf *buf = filp->private_data; | ||
685 | return relay_mmap_buf(buf, vma); | ||
686 | } | ||
687 | |||
688 | /** | ||
689 | * relay_file_poll - poll file op for relay files | ||
690 | * @filp: the file | ||
691 | * @wait: poll table | ||
692 | * | ||
693 | * Poll implemention. | ||
694 | */ | ||
695 | static unsigned int relay_file_poll(struct file *filp, poll_table *wait) | ||
696 | { | ||
697 | unsigned int mask = 0; | ||
698 | struct rchan_buf *buf = filp->private_data; | ||
699 | |||
700 | if (buf->finalized) | ||
701 | return POLLERR; | ||
702 | |||
703 | if (filp->f_mode & FMODE_READ) { | ||
704 | poll_wait(filp, &buf->read_wait, wait); | ||
705 | if (!relay_buf_empty(buf)) | ||
706 | mask |= POLLIN | POLLRDNORM; | ||
707 | } | ||
708 | |||
709 | return mask; | ||
710 | } | ||
711 | |||
712 | /** | ||
713 | * relay_file_release - release file op for relay files | ||
714 | * @inode: the inode | ||
715 | * @filp: the file | ||
716 | * | ||
717 | * Decrements the channel refcount, as the filesystem is | ||
718 | * no longer using it. | ||
719 | */ | ||
720 | static int relay_file_release(struct inode *inode, struct file *filp) | ||
721 | { | ||
722 | struct rchan_buf *buf = filp->private_data; | ||
723 | kref_put(&buf->kref, relay_remove_buf); | ||
724 | |||
725 | return 0; | ||
726 | } | ||
727 | |||
728 | /** | ||
729 | * relay_file_read_consume - update the consumed count for the buffer | ||
730 | */ | ||
731 | static void relay_file_read_consume(struct rchan_buf *buf, | ||
732 | size_t read_pos, | ||
733 | size_t bytes_consumed) | ||
734 | { | ||
735 | size_t subbuf_size = buf->chan->subbuf_size; | ||
736 | size_t n_subbufs = buf->chan->n_subbufs; | ||
737 | size_t read_subbuf; | ||
738 | |||
739 | if (buf->bytes_consumed + bytes_consumed > subbuf_size) { | ||
740 | relay_subbufs_consumed(buf->chan, buf->cpu, 1); | ||
741 | buf->bytes_consumed = 0; | ||
742 | } | ||
743 | |||
744 | buf->bytes_consumed += bytes_consumed; | ||
745 | read_subbuf = read_pos / buf->chan->subbuf_size; | ||
746 | if (buf->bytes_consumed + buf->padding[read_subbuf] == subbuf_size) { | ||
747 | if ((read_subbuf == buf->subbufs_produced % n_subbufs) && | ||
748 | (buf->offset == subbuf_size)) | ||
749 | return; | ||
750 | relay_subbufs_consumed(buf->chan, buf->cpu, 1); | ||
751 | buf->bytes_consumed = 0; | ||
752 | } | ||
753 | } | ||
754 | |||
755 | /** | ||
756 | * relay_file_read_avail - boolean, are there unconsumed bytes available? | ||
757 | */ | ||
758 | static int relay_file_read_avail(struct rchan_buf *buf, size_t read_pos) | ||
759 | { | ||
760 | size_t bytes_produced, bytes_consumed, write_offset; | ||
761 | size_t subbuf_size = buf->chan->subbuf_size; | ||
762 | size_t n_subbufs = buf->chan->n_subbufs; | ||
763 | size_t produced = buf->subbufs_produced % n_subbufs; | ||
764 | size_t consumed = buf->subbufs_consumed % n_subbufs; | ||
765 | |||
766 | write_offset = buf->offset > subbuf_size ? subbuf_size : buf->offset; | ||
767 | |||
768 | if (consumed > produced) { | ||
769 | if ((produced > n_subbufs) && | ||
770 | (produced + n_subbufs - consumed <= n_subbufs)) | ||
771 | produced += n_subbufs; | ||
772 | } else if (consumed == produced) { | ||
773 | if (buf->offset > subbuf_size) { | ||
774 | produced += n_subbufs; | ||
775 | if (buf->subbufs_produced == buf->subbufs_consumed) | ||
776 | consumed += n_subbufs; | ||
777 | } | ||
778 | } | ||
779 | |||
780 | if (buf->offset > subbuf_size) | ||
781 | bytes_produced = (produced - 1) * subbuf_size + write_offset; | ||
782 | else | ||
783 | bytes_produced = produced * subbuf_size + write_offset; | ||
784 | bytes_consumed = consumed * subbuf_size + buf->bytes_consumed; | ||
785 | |||
786 | if (bytes_produced == bytes_consumed) | ||
787 | return 0; | ||
788 | |||
789 | relay_file_read_consume(buf, read_pos, 0); | ||
790 | |||
791 | return 1; | ||
792 | } | ||
793 | |||
794 | /** | ||
795 | * relay_file_read_subbuf_avail - return bytes available in sub-buffer | ||
796 | */ | ||
797 | static size_t relay_file_read_subbuf_avail(size_t read_pos, | ||
798 | struct rchan_buf *buf) | ||
799 | { | ||
800 | size_t padding, avail = 0; | ||
801 | size_t read_subbuf, read_offset, write_subbuf, write_offset; | ||
802 | size_t subbuf_size = buf->chan->subbuf_size; | ||
803 | |||
804 | write_subbuf = (buf->data - buf->start) / subbuf_size; | ||
805 | write_offset = buf->offset > subbuf_size ? subbuf_size : buf->offset; | ||
806 | read_subbuf = read_pos / subbuf_size; | ||
807 | read_offset = read_pos % subbuf_size; | ||
808 | padding = buf->padding[read_subbuf]; | ||
809 | |||
810 | if (read_subbuf == write_subbuf) { | ||
811 | if (read_offset + padding < write_offset) | ||
812 | avail = write_offset - (read_offset + padding); | ||
813 | } else | ||
814 | avail = (subbuf_size - padding) - read_offset; | ||
815 | |||
816 | return avail; | ||
817 | } | ||
818 | |||
819 | /** | ||
820 | * relay_file_read_start_pos - find the first available byte to read | ||
821 | * | ||
822 | * If the read_pos is in the middle of padding, return the | ||
823 | * position of the first actually available byte, otherwise | ||
824 | * return the original value. | ||
825 | */ | ||
826 | static size_t relay_file_read_start_pos(size_t read_pos, | ||
827 | struct rchan_buf *buf) | ||
828 | { | ||
829 | size_t read_subbuf, padding, padding_start, padding_end; | ||
830 | size_t subbuf_size = buf->chan->subbuf_size; | ||
831 | size_t n_subbufs = buf->chan->n_subbufs; | ||
832 | |||
833 | read_subbuf = read_pos / subbuf_size; | ||
834 | padding = buf->padding[read_subbuf]; | ||
835 | padding_start = (read_subbuf + 1) * subbuf_size - padding; | ||
836 | padding_end = (read_subbuf + 1) * subbuf_size; | ||
837 | if (read_pos >= padding_start && read_pos < padding_end) { | ||
838 | read_subbuf = (read_subbuf + 1) % n_subbufs; | ||
839 | read_pos = read_subbuf * subbuf_size; | ||
840 | } | ||
841 | |||
842 | return read_pos; | ||
843 | } | ||
844 | |||
845 | /** | ||
846 | * relay_file_read_end_pos - return the new read position | ||
847 | */ | ||
848 | static size_t relay_file_read_end_pos(struct rchan_buf *buf, | ||
849 | size_t read_pos, | ||
850 | size_t count) | ||
851 | { | ||
852 | size_t read_subbuf, padding, end_pos; | ||
853 | size_t subbuf_size = buf->chan->subbuf_size; | ||
854 | size_t n_subbufs = buf->chan->n_subbufs; | ||
855 | |||
856 | read_subbuf = read_pos / subbuf_size; | ||
857 | padding = buf->padding[read_subbuf]; | ||
858 | if (read_pos % subbuf_size + count + padding == subbuf_size) | ||
859 | end_pos = (read_subbuf + 1) * subbuf_size; | ||
860 | else | ||
861 | end_pos = read_pos + count; | ||
862 | if (end_pos >= subbuf_size * n_subbufs) | ||
863 | end_pos = 0; | ||
864 | |||
865 | return end_pos; | ||
866 | } | ||
867 | |||
868 | /** | ||
869 | * relay_file_read - read file op for relay files | ||
870 | * @filp: the file | ||
871 | * @buffer: the userspace buffer | ||
872 | * @count: number of bytes to read | ||
873 | * @ppos: position to read from | ||
874 | * | ||
875 | * Reads count bytes or the number of bytes available in the | ||
876 | * current sub-buffer being read, whichever is smaller. | ||
877 | */ | ||
878 | static ssize_t relay_file_read(struct file *filp, | ||
879 | char __user *buffer, | ||
880 | size_t count, | ||
881 | loff_t *ppos) | ||
882 | { | ||
883 | struct rchan_buf *buf = filp->private_data; | ||
884 | struct inode *inode = filp->f_dentry->d_inode; | ||
885 | size_t read_start, avail; | ||
886 | ssize_t ret = 0; | ||
887 | void *from; | ||
888 | |||
889 | mutex_lock(&inode->i_mutex); | ||
890 | if(!relay_file_read_avail(buf, *ppos)) | ||
891 | goto out; | ||
892 | |||
893 | read_start = relay_file_read_start_pos(*ppos, buf); | ||
894 | avail = relay_file_read_subbuf_avail(read_start, buf); | ||
895 | if (!avail) | ||
896 | goto out; | ||
897 | |||
898 | from = buf->start + read_start; | ||
899 | ret = count = min(count, avail); | ||
900 | if (copy_to_user(buffer, from, count)) { | ||
901 | ret = -EFAULT; | ||
902 | goto out; | ||
903 | } | ||
904 | relay_file_read_consume(buf, read_start, count); | ||
905 | *ppos = relay_file_read_end_pos(buf, read_start, count); | ||
906 | out: | ||
907 | mutex_unlock(&inode->i_mutex); | ||
908 | return ret; | ||
909 | } | ||
910 | |||
911 | struct file_operations relay_file_operations = { | ||
912 | .open = relay_file_open, | ||
913 | .poll = relay_file_poll, | ||
914 | .mmap = relay_file_mmap, | ||
915 | .read = relay_file_read, | ||
916 | .llseek = no_llseek, | ||
917 | .release = relay_file_release, | ||
918 | }; | ||
919 | EXPORT_SYMBOL_GPL(relay_file_operations); | ||