diff options
-rw-r--r-- | arch/ia64/kernel/entry.S | 1 | ||||
-rw-r--r-- | arch/powerpc/kernel/systbl.S | 1 | ||||
-rw-r--r-- | arch/powerpc/platforms/cell/spu_callbacks.c | 1 | ||||
-rw-r--r-- | drivers/char/mem.c | 14 | ||||
-rw-r--r-- | fs/splice.c | 355 | ||||
-rw-r--r-- | include/asm-i386/unistd.h | 3 | ||||
-rw-r--r-- | include/asm-ia64/unistd.h | 3 | ||||
-rw-r--r-- | include/asm-powerpc/unistd.h | 3 | ||||
-rw-r--r-- | include/asm-x86_64/unistd.h | 4 | ||||
-rw-r--r-- | include/linux/pipe_fs_i.h | 17 | ||||
-rw-r--r-- | include/linux/syscalls.h | 3 |
11 files changed, 328 insertions, 77 deletions
diff --git a/arch/ia64/kernel/entry.S b/arch/ia64/kernel/entry.S index e30798811216..bcb80ca5cf40 100644 --- a/arch/ia64/kernel/entry.S +++ b/arch/ia64/kernel/entry.S | |||
@@ -1610,5 +1610,6 @@ sys_call_table: | |||
1610 | data8 sys_get_robust_list | 1610 | data8 sys_get_robust_list |
1611 | data8 sys_sync_file_range // 1300 | 1611 | data8 sys_sync_file_range // 1300 |
1612 | data8 sys_tee | 1612 | data8 sys_tee |
1613 | data8 sys_vmsplice | ||
1613 | 1614 | ||
1614 | .org sys_call_table + 8*NR_syscalls // guard against failures to increase NR_syscalls | 1615 | .org sys_call_table + 8*NR_syscalls // guard against failures to increase NR_syscalls |
diff --git a/arch/powerpc/kernel/systbl.S b/arch/powerpc/kernel/systbl.S index 8d1522690501..0b98eea73c5e 100644 --- a/arch/powerpc/kernel/systbl.S +++ b/arch/powerpc/kernel/systbl.S | |||
@@ -324,6 +324,7 @@ COMPAT_SYS(ppoll) | |||
324 | SYSCALL(unshare) | 324 | SYSCALL(unshare) |
325 | SYSCALL(splice) | 325 | SYSCALL(splice) |
326 | SYSCALL(tee) | 326 | SYSCALL(tee) |
327 | SYSCALL(vmsplice) | ||
327 | 328 | ||
328 | /* | 329 | /* |
329 | * please add new calls to arch/powerpc/platforms/cell/spu_callbacks.c | 330 | * please add new calls to arch/powerpc/platforms/cell/spu_callbacks.c |
diff --git a/arch/powerpc/platforms/cell/spu_callbacks.c b/arch/powerpc/platforms/cell/spu_callbacks.c index deb3afb94484..b283380a2a18 100644 --- a/arch/powerpc/platforms/cell/spu_callbacks.c +++ b/arch/powerpc/platforms/cell/spu_callbacks.c | |||
@@ -318,6 +318,7 @@ void *spu_syscall_table[] = { | |||
318 | [__NR_unshare] sys_unshare, | 318 | [__NR_unshare] sys_unshare, |
319 | [__NR_splice] sys_splice, | 319 | [__NR_splice] sys_splice, |
320 | [__NR_tee] sys_tee, | 320 | [__NR_tee] sys_tee, |
321 | [__NR_vmsplice] sys_vmsplice, | ||
321 | }; | 322 | }; |
322 | 323 | ||
323 | long spu_sys_callback(struct spu_syscall_block *s) | 324 | long spu_sys_callback(struct spu_syscall_block *s) |
diff --git a/drivers/char/mem.c b/drivers/char/mem.c index 66719f9d294c..1fa9fa157c12 100644 --- a/drivers/char/mem.c +++ b/drivers/char/mem.c | |||
@@ -27,6 +27,7 @@ | |||
27 | #include <linux/crash_dump.h> | 27 | #include <linux/crash_dump.h> |
28 | #include <linux/backing-dev.h> | 28 | #include <linux/backing-dev.h> |
29 | #include <linux/bootmem.h> | 29 | #include <linux/bootmem.h> |
30 | #include <linux/pipe_fs_i.h> | ||
30 | 31 | ||
31 | #include <asm/uaccess.h> | 32 | #include <asm/uaccess.h> |
32 | #include <asm/io.h> | 33 | #include <asm/io.h> |
@@ -578,6 +579,18 @@ static ssize_t write_null(struct file * file, const char __user * buf, | |||
578 | return count; | 579 | return count; |
579 | } | 580 | } |
580 | 581 | ||
582 | static int pipe_to_null(struct pipe_inode_info *info, struct pipe_buffer *buf, | ||
583 | struct splice_desc *sd) | ||
584 | { | ||
585 | return sd->len; | ||
586 | } | ||
587 | |||
588 | static ssize_t splice_write_null(struct pipe_inode_info *pipe,struct file *out, | ||
589 | loff_t *ppos, size_t len, unsigned int flags) | ||
590 | { | ||
591 | return splice_from_pipe(pipe, out, ppos, len, flags, pipe_to_null); | ||
592 | } | ||
593 | |||
581 | #ifdef CONFIG_MMU | 594 | #ifdef CONFIG_MMU |
582 | /* | 595 | /* |
583 | * For fun, we are using the MMU for this. | 596 | * For fun, we are using the MMU for this. |
@@ -785,6 +798,7 @@ static struct file_operations null_fops = { | |||
785 | .llseek = null_lseek, | 798 | .llseek = null_lseek, |
786 | .read = read_null, | 799 | .read = read_null, |
787 | .write = write_null, | 800 | .write = write_null, |
801 | .splice_write = splice_write_null, | ||
788 | }; | 802 | }; |
789 | 803 | ||
790 | #if defined(CONFIG_ISA) || !defined(__mc68000__) | 804 | #if defined(CONFIG_ISA) || !defined(__mc68000__) |
diff --git a/fs/splice.c b/fs/splice.c index 0559e7577a04..447ebc0a37f3 100644 --- a/fs/splice.c +++ b/fs/splice.c | |||
@@ -27,15 +27,22 @@ | |||
27 | #include <linux/buffer_head.h> | 27 | #include <linux/buffer_head.h> |
28 | #include <linux/module.h> | 28 | #include <linux/module.h> |
29 | #include <linux/syscalls.h> | 29 | #include <linux/syscalls.h> |
30 | #include <linux/uio.h> | ||
31 | |||
32 | struct partial_page { | ||
33 | unsigned int offset; | ||
34 | unsigned int len; | ||
35 | }; | ||
30 | 36 | ||
31 | /* | 37 | /* |
32 | * Passed to the actors | 38 | * Passed to splice_to_pipe |
33 | */ | 39 | */ |
34 | struct splice_desc { | 40 | struct splice_pipe_desc { |
35 | unsigned int len, total_len; /* current and remaining length */ | 41 | struct page **pages; /* page map */ |
42 | struct partial_page *partial; /* pages[] may not be contig */ | ||
43 | int nr_pages; /* number of pages in map */ | ||
36 | unsigned int flags; /* splice flags */ | 44 | unsigned int flags; /* splice flags */ |
37 | struct file *file; /* file to read/write */ | 45 | struct pipe_buf_operations *ops;/* ops associated with output pipe */ |
38 | loff_t pos; /* file position */ | ||
39 | }; | 46 | }; |
40 | 47 | ||
41 | /* | 48 | /* |
@@ -128,6 +135,19 @@ static void page_cache_pipe_buf_unmap(struct pipe_inode_info *info, | |||
128 | kunmap(buf->page); | 135 | kunmap(buf->page); |
129 | } | 136 | } |
130 | 137 | ||
138 | static void *user_page_pipe_buf_map(struct file *file, | ||
139 | struct pipe_inode_info *pipe, | ||
140 | struct pipe_buffer *buf) | ||
141 | { | ||
142 | return kmap(buf->page); | ||
143 | } | ||
144 | |||
145 | static void user_page_pipe_buf_unmap(struct pipe_inode_info *pipe, | ||
146 | struct pipe_buffer *buf) | ||
147 | { | ||
148 | kunmap(buf->page); | ||
149 | } | ||
150 | |||
131 | static void page_cache_pipe_buf_get(struct pipe_inode_info *info, | 151 | static void page_cache_pipe_buf_get(struct pipe_inode_info *info, |
132 | struct pipe_buffer *buf) | 152 | struct pipe_buffer *buf) |
133 | { | 153 | { |
@@ -143,19 +163,33 @@ static struct pipe_buf_operations page_cache_pipe_buf_ops = { | |||
143 | .get = page_cache_pipe_buf_get, | 163 | .get = page_cache_pipe_buf_get, |
144 | }; | 164 | }; |
145 | 165 | ||
166 | static int user_page_pipe_buf_steal(struct pipe_inode_info *pipe, | ||
167 | struct pipe_buffer *buf) | ||
168 | { | ||
169 | return 1; | ||
170 | } | ||
171 | |||
172 | static struct pipe_buf_operations user_page_pipe_buf_ops = { | ||
173 | .can_merge = 0, | ||
174 | .map = user_page_pipe_buf_map, | ||
175 | .unmap = user_page_pipe_buf_unmap, | ||
176 | .release = page_cache_pipe_buf_release, | ||
177 | .steal = user_page_pipe_buf_steal, | ||
178 | .get = page_cache_pipe_buf_get, | ||
179 | }; | ||
180 | |||
146 | /* | 181 | /* |
147 | * Pipe output worker. This sets up our pipe format with the page cache | 182 | * Pipe output worker. This sets up our pipe format with the page cache |
148 | * pipe buffer operations. Otherwise very similar to the regular pipe_writev(). | 183 | * pipe buffer operations. Otherwise very similar to the regular pipe_writev(). |
149 | */ | 184 | */ |
150 | static ssize_t move_to_pipe(struct pipe_inode_info *pipe, struct page **pages, | 185 | static ssize_t splice_to_pipe(struct pipe_inode_info *pipe, |
151 | int nr_pages, unsigned long len, | 186 | struct splice_pipe_desc *spd) |
152 | unsigned int offset, unsigned int flags) | ||
153 | { | 187 | { |
154 | int ret, do_wakeup, i; | 188 | int ret, do_wakeup, page_nr; |
155 | 189 | ||
156 | ret = 0; | 190 | ret = 0; |
157 | do_wakeup = 0; | 191 | do_wakeup = 0; |
158 | i = 0; | 192 | page_nr = 0; |
159 | 193 | ||
160 | if (pipe->inode) | 194 | if (pipe->inode) |
161 | mutex_lock(&pipe->inode->i_mutex); | 195 | mutex_lock(&pipe->inode->i_mutex); |
@@ -171,27 +205,19 @@ static ssize_t move_to_pipe(struct pipe_inode_info *pipe, struct page **pages, | |||
171 | if (pipe->nrbufs < PIPE_BUFFERS) { | 205 | if (pipe->nrbufs < PIPE_BUFFERS) { |
172 | int newbuf = (pipe->curbuf + pipe->nrbufs) & (PIPE_BUFFERS - 1); | 206 | int newbuf = (pipe->curbuf + pipe->nrbufs) & (PIPE_BUFFERS - 1); |
173 | struct pipe_buffer *buf = pipe->bufs + newbuf; | 207 | struct pipe_buffer *buf = pipe->bufs + newbuf; |
174 | struct page *page = pages[i++]; | ||
175 | unsigned long this_len; | ||
176 | 208 | ||
177 | this_len = PAGE_CACHE_SIZE - offset; | 209 | buf->page = spd->pages[page_nr]; |
178 | if (this_len > len) | 210 | buf->offset = spd->partial[page_nr].offset; |
179 | this_len = len; | 211 | buf->len = spd->partial[page_nr].len; |
180 | 212 | buf->ops = spd->ops; | |
181 | buf->page = page; | ||
182 | buf->offset = offset; | ||
183 | buf->len = this_len; | ||
184 | buf->ops = &page_cache_pipe_buf_ops; | ||
185 | pipe->nrbufs++; | 213 | pipe->nrbufs++; |
214 | page_nr++; | ||
215 | ret += buf->len; | ||
216 | |||
186 | if (pipe->inode) | 217 | if (pipe->inode) |
187 | do_wakeup = 1; | 218 | do_wakeup = 1; |
188 | 219 | ||
189 | ret += this_len; | 220 | if (!--spd->nr_pages) |
190 | len -= this_len; | ||
191 | offset = 0; | ||
192 | if (!--nr_pages) | ||
193 | break; | ||
194 | if (!len) | ||
195 | break; | 221 | break; |
196 | if (pipe->nrbufs < PIPE_BUFFERS) | 222 | if (pipe->nrbufs < PIPE_BUFFERS) |
197 | continue; | 223 | continue; |
@@ -199,7 +225,7 @@ static ssize_t move_to_pipe(struct pipe_inode_info *pipe, struct page **pages, | |||
199 | break; | 225 | break; |
200 | } | 226 | } |
201 | 227 | ||
202 | if (flags & SPLICE_F_NONBLOCK) { | 228 | if (spd->flags & SPLICE_F_NONBLOCK) { |
203 | if (!ret) | 229 | if (!ret) |
204 | ret = -EAGAIN; | 230 | ret = -EAGAIN; |
205 | break; | 231 | break; |
@@ -234,8 +260,8 @@ static ssize_t move_to_pipe(struct pipe_inode_info *pipe, struct page **pages, | |||
234 | kill_fasync(&pipe->fasync_readers, SIGIO, POLL_IN); | 260 | kill_fasync(&pipe->fasync_readers, SIGIO, POLL_IN); |
235 | } | 261 | } |
236 | 262 | ||
237 | while (i < nr_pages) | 263 | while (page_nr < spd->nr_pages) |
238 | page_cache_release(pages[i++]); | 264 | page_cache_release(spd->pages[page_nr++]); |
239 | 265 | ||
240 | return ret; | 266 | return ret; |
241 | } | 267 | } |
@@ -246,17 +272,24 @@ __generic_file_splice_read(struct file *in, loff_t *ppos, | |||
246 | unsigned int flags) | 272 | unsigned int flags) |
247 | { | 273 | { |
248 | struct address_space *mapping = in->f_mapping; | 274 | struct address_space *mapping = in->f_mapping; |
249 | unsigned int loff, offset, nr_pages; | 275 | unsigned int loff, nr_pages; |
250 | struct page *pages[PIPE_BUFFERS]; | 276 | struct page *pages[PIPE_BUFFERS]; |
277 | struct partial_page partial[PIPE_BUFFERS]; | ||
251 | struct page *page; | 278 | struct page *page; |
252 | pgoff_t index, end_index; | 279 | pgoff_t index, end_index; |
253 | loff_t isize; | 280 | loff_t isize; |
254 | size_t bytes; | 281 | size_t total_len; |
255 | int i, error; | 282 | int error; |
283 | struct splice_pipe_desc spd = { | ||
284 | .pages = pages, | ||
285 | .partial = partial, | ||
286 | .flags = flags, | ||
287 | .ops = &page_cache_pipe_buf_ops, | ||
288 | }; | ||
256 | 289 | ||
257 | index = *ppos >> PAGE_CACHE_SHIFT; | 290 | index = *ppos >> PAGE_CACHE_SHIFT; |
258 | loff = offset = *ppos & ~PAGE_CACHE_MASK; | 291 | loff = *ppos & ~PAGE_CACHE_MASK; |
259 | nr_pages = (len + offset + PAGE_CACHE_SIZE - 1) >> PAGE_CACHE_SHIFT; | 292 | nr_pages = (len + loff + PAGE_CACHE_SIZE - 1) >> PAGE_CACHE_SHIFT; |
260 | 293 | ||
261 | if (nr_pages > PIPE_BUFFERS) | 294 | if (nr_pages > PIPE_BUFFERS) |
262 | nr_pages = PIPE_BUFFERS; | 295 | nr_pages = PIPE_BUFFERS; |
@@ -266,15 +299,15 @@ __generic_file_splice_read(struct file *in, loff_t *ppos, | |||
266 | * read-ahead if this is a non-zero offset (we are likely doing small | 299 | * read-ahead if this is a non-zero offset (we are likely doing small |
267 | * chunk splice and the page is already there) for a single page. | 300 | * chunk splice and the page is already there) for a single page. |
268 | */ | 301 | */ |
269 | if (!offset || nr_pages > 1) | 302 | if (!loff || spd.nr_pages > 1) |
270 | do_page_cache_readahead(mapping, in, index, nr_pages); | 303 | do_page_cache_readahead(mapping, in, index, spd.nr_pages); |
271 | 304 | ||
272 | /* | 305 | /* |
273 | * Now fill in the holes: | 306 | * Now fill in the holes: |
274 | */ | 307 | */ |
275 | error = 0; | 308 | error = 0; |
276 | bytes = 0; | 309 | total_len = 0; |
277 | for (i = 0; i < nr_pages; i++, index++) { | 310 | for (spd.nr_pages = 0; spd.nr_pages < nr_pages; spd.nr_pages++, index++) { |
278 | unsigned int this_len; | 311 | unsigned int this_len; |
279 | 312 | ||
280 | if (!len) | 313 | if (!len) |
@@ -283,7 +316,7 @@ __generic_file_splice_read(struct file *in, loff_t *ppos, | |||
283 | /* | 316 | /* |
284 | * this_len is the max we'll use from this page | 317 | * this_len is the max we'll use from this page |
285 | */ | 318 | */ |
286 | this_len = min(len, PAGE_CACHE_SIZE - loff); | 319 | this_len = min_t(unsigned long, len, PAGE_CACHE_SIZE - loff); |
287 | find_page: | 320 | find_page: |
288 | /* | 321 | /* |
289 | * lookup the page for this index | 322 | * lookup the page for this index |
@@ -367,26 +400,29 @@ readpage: | |||
367 | */ | 400 | */ |
368 | if (end_index == index) { | 401 | if (end_index == index) { |
369 | loff = PAGE_CACHE_SIZE - (isize & ~PAGE_CACHE_MASK); | 402 | loff = PAGE_CACHE_SIZE - (isize & ~PAGE_CACHE_MASK); |
370 | if (bytes + loff > isize) { | 403 | if (total_len + loff > isize) { |
371 | page_cache_release(page); | 404 | page_cache_release(page); |
372 | break; | 405 | break; |
373 | } | 406 | } |
374 | /* | 407 | /* |
375 | * force quit after adding this page | 408 | * force quit after adding this page |
376 | */ | 409 | */ |
377 | nr_pages = i; | 410 | nr_pages = spd.nr_pages; |
378 | this_len = min(this_len, loff); | 411 | this_len = min(this_len, loff); |
412 | loff = 0; | ||
379 | } | 413 | } |
380 | } | 414 | } |
381 | fill_it: | 415 | fill_it: |
382 | pages[i] = page; | 416 | pages[spd.nr_pages] = page; |
383 | bytes += this_len; | 417 | partial[spd.nr_pages].offset = loff; |
418 | partial[spd.nr_pages].len = this_len; | ||
384 | len -= this_len; | 419 | len -= this_len; |
420 | total_len += this_len; | ||
385 | loff = 0; | 421 | loff = 0; |
386 | } | 422 | } |
387 | 423 | ||
388 | if (i) | 424 | if (spd.nr_pages) |
389 | return move_to_pipe(pipe, pages, i, bytes, offset, flags); | 425 | return splice_to_pipe(pipe, &spd); |
390 | 426 | ||
391 | return error; | 427 | return error; |
392 | } | 428 | } |
@@ -439,14 +475,13 @@ EXPORT_SYMBOL(generic_file_splice_read); | |||
439 | 475 | ||
440 | /* | 476 | /* |
441 | * Send 'sd->len' bytes to socket from 'sd->file' at position 'sd->pos' | 477 | * Send 'sd->len' bytes to socket from 'sd->file' at position 'sd->pos' |
442 | * using sendpage(). | 478 | * using sendpage(). Return the number of bytes sent. |
443 | */ | 479 | */ |
444 | static int pipe_to_sendpage(struct pipe_inode_info *info, | 480 | static int pipe_to_sendpage(struct pipe_inode_info *info, |
445 | struct pipe_buffer *buf, struct splice_desc *sd) | 481 | struct pipe_buffer *buf, struct splice_desc *sd) |
446 | { | 482 | { |
447 | struct file *file = sd->file; | 483 | struct file *file = sd->file; |
448 | loff_t pos = sd->pos; | 484 | loff_t pos = sd->pos; |
449 | unsigned int offset; | ||
450 | ssize_t ret; | 485 | ssize_t ret; |
451 | void *ptr; | 486 | void *ptr; |
452 | int more; | 487 | int more; |
@@ -461,16 +496,13 @@ static int pipe_to_sendpage(struct pipe_inode_info *info, | |||
461 | if (IS_ERR(ptr)) | 496 | if (IS_ERR(ptr)) |
462 | return PTR_ERR(ptr); | 497 | return PTR_ERR(ptr); |
463 | 498 | ||
464 | offset = pos & ~PAGE_CACHE_MASK; | ||
465 | more = (sd->flags & SPLICE_F_MORE) || sd->len < sd->total_len; | 499 | more = (sd->flags & SPLICE_F_MORE) || sd->len < sd->total_len; |
466 | 500 | ||
467 | ret = file->f_op->sendpage(file, buf->page, offset, sd->len, &pos,more); | 501 | ret = file->f_op->sendpage(file, buf->page, buf->offset, sd->len, |
502 | &pos, more); | ||
468 | 503 | ||
469 | buf->ops->unmap(info, buf); | 504 | buf->ops->unmap(info, buf); |
470 | if (ret == sd->len) | 505 | return ret; |
471 | return 0; | ||
472 | |||
473 | return -EIO; | ||
474 | } | 506 | } |
475 | 507 | ||
476 | /* | 508 | /* |
@@ -499,7 +531,7 @@ static int pipe_to_file(struct pipe_inode_info *info, struct pipe_buffer *buf, | |||
499 | struct file *file = sd->file; | 531 | struct file *file = sd->file; |
500 | struct address_space *mapping = file->f_mapping; | 532 | struct address_space *mapping = file->f_mapping; |
501 | gfp_t gfp_mask = mapping_gfp_mask(mapping); | 533 | gfp_t gfp_mask = mapping_gfp_mask(mapping); |
502 | unsigned int offset; | 534 | unsigned int offset, this_len; |
503 | struct page *page; | 535 | struct page *page; |
504 | pgoff_t index; | 536 | pgoff_t index; |
505 | char *src; | 537 | char *src; |
@@ -515,6 +547,10 @@ static int pipe_to_file(struct pipe_inode_info *info, struct pipe_buffer *buf, | |||
515 | index = sd->pos >> PAGE_CACHE_SHIFT; | 547 | index = sd->pos >> PAGE_CACHE_SHIFT; |
516 | offset = sd->pos & ~PAGE_CACHE_MASK; | 548 | offset = sd->pos & ~PAGE_CACHE_MASK; |
517 | 549 | ||
550 | this_len = sd->len; | ||
551 | if (this_len + offset > PAGE_CACHE_SIZE) | ||
552 | this_len = PAGE_CACHE_SIZE - offset; | ||
553 | |||
518 | /* | 554 | /* |
519 | * Reuse buf page, if SPLICE_F_MOVE is set. | 555 | * Reuse buf page, if SPLICE_F_MOVE is set. |
520 | */ | 556 | */ |
@@ -558,7 +594,7 @@ find_page: | |||
558 | * the full page. | 594 | * the full page. |
559 | */ | 595 | */ |
560 | if (!PageUptodate(page)) { | 596 | if (!PageUptodate(page)) { |
561 | if (sd->len < PAGE_CACHE_SIZE) { | 597 | if (this_len < PAGE_CACHE_SIZE) { |
562 | ret = mapping->a_ops->readpage(file, page); | 598 | ret = mapping->a_ops->readpage(file, page); |
563 | if (unlikely(ret)) | 599 | if (unlikely(ret)) |
564 | goto out; | 600 | goto out; |
@@ -582,7 +618,7 @@ find_page: | |||
582 | } | 618 | } |
583 | } | 619 | } |
584 | 620 | ||
585 | ret = mapping->a_ops->prepare_write(file, page, 0, sd->len); | 621 | ret = mapping->a_ops->prepare_write(file, page, offset, offset+this_len); |
586 | if (ret == AOP_TRUNCATED_PAGE) { | 622 | if (ret == AOP_TRUNCATED_PAGE) { |
587 | page_cache_release(page); | 623 | page_cache_release(page); |
588 | goto find_page; | 624 | goto find_page; |
@@ -592,18 +628,22 @@ find_page: | |||
592 | if (!(buf->flags & PIPE_BUF_FLAG_STOLEN)) { | 628 | if (!(buf->flags & PIPE_BUF_FLAG_STOLEN)) { |
593 | char *dst = kmap_atomic(page, KM_USER0); | 629 | char *dst = kmap_atomic(page, KM_USER0); |
594 | 630 | ||
595 | memcpy(dst + offset, src + buf->offset, sd->len); | 631 | memcpy(dst + offset, src + buf->offset, this_len); |
596 | flush_dcache_page(page); | 632 | flush_dcache_page(page); |
597 | kunmap_atomic(dst, KM_USER0); | 633 | kunmap_atomic(dst, KM_USER0); |
598 | } | 634 | } |
599 | 635 | ||
600 | ret = mapping->a_ops->commit_write(file, page, 0, sd->len); | 636 | ret = mapping->a_ops->commit_write(file, page, offset, offset+this_len); |
601 | if (ret == AOP_TRUNCATED_PAGE) { | 637 | if (ret == AOP_TRUNCATED_PAGE) { |
602 | page_cache_release(page); | 638 | page_cache_release(page); |
603 | goto find_page; | 639 | goto find_page; |
604 | } else if (ret) | 640 | } else if (ret) |
605 | goto out; | 641 | goto out; |
606 | 642 | ||
643 | /* | ||
644 | * Return the number of bytes written. | ||
645 | */ | ||
646 | ret = this_len; | ||
607 | mark_page_accessed(page); | 647 | mark_page_accessed(page); |
608 | balance_dirty_pages_ratelimited(mapping); | 648 | balance_dirty_pages_ratelimited(mapping); |
609 | out: | 649 | out: |
@@ -616,17 +656,14 @@ out_nomem: | |||
616 | return ret; | 656 | return ret; |
617 | } | 657 | } |
618 | 658 | ||
619 | typedef int (splice_actor)(struct pipe_inode_info *, struct pipe_buffer *, | ||
620 | struct splice_desc *); | ||
621 | |||
622 | /* | 659 | /* |
623 | * Pipe input worker. Most of this logic works like a regular pipe, the | 660 | * Pipe input worker. Most of this logic works like a regular pipe, the |
624 | * key here is the 'actor' worker passed in that actually moves the data | 661 | * key here is the 'actor' worker passed in that actually moves the data |
625 | * to the wanted destination. See pipe_to_file/pipe_to_sendpage above. | 662 | * to the wanted destination. See pipe_to_file/pipe_to_sendpage above. |
626 | */ | 663 | */ |
627 | static ssize_t move_from_pipe(struct pipe_inode_info *pipe, struct file *out, | 664 | ssize_t splice_from_pipe(struct pipe_inode_info *pipe, struct file *out, |
628 | loff_t *ppos, size_t len, unsigned int flags, | 665 | loff_t *ppos, size_t len, unsigned int flags, |
629 | splice_actor *actor) | 666 | splice_actor *actor) |
630 | { | 667 | { |
631 | int ret, do_wakeup, err; | 668 | int ret, do_wakeup, err; |
632 | struct splice_desc sd; | 669 | struct splice_desc sd; |
@@ -652,16 +689,22 @@ static ssize_t move_from_pipe(struct pipe_inode_info *pipe, struct file *out, | |||
652 | sd.len = sd.total_len; | 689 | sd.len = sd.total_len; |
653 | 690 | ||
654 | err = actor(pipe, buf, &sd); | 691 | err = actor(pipe, buf, &sd); |
655 | if (err) { | 692 | if (err <= 0) { |
656 | if (!ret && err != -ENODATA) | 693 | if (!ret && err != -ENODATA) |
657 | ret = err; | 694 | ret = err; |
658 | 695 | ||
659 | break; | 696 | break; |
660 | } | 697 | } |
661 | 698 | ||
662 | ret += sd.len; | 699 | ret += err; |
663 | buf->offset += sd.len; | 700 | buf->offset += err; |
664 | buf->len -= sd.len; | 701 | buf->len -= err; |
702 | |||
703 | sd.len -= err; | ||
704 | sd.pos += err; | ||
705 | sd.total_len -= err; | ||
706 | if (sd.len) | ||
707 | continue; | ||
665 | 708 | ||
666 | if (!buf->len) { | 709 | if (!buf->len) { |
667 | buf->ops = NULL; | 710 | buf->ops = NULL; |
@@ -672,8 +715,6 @@ static ssize_t move_from_pipe(struct pipe_inode_info *pipe, struct file *out, | |||
672 | do_wakeup = 1; | 715 | do_wakeup = 1; |
673 | } | 716 | } |
674 | 717 | ||
675 | sd.pos += sd.len; | ||
676 | sd.total_len -= sd.len; | ||
677 | if (!sd.total_len) | 718 | if (!sd.total_len) |
678 | break; | 719 | break; |
679 | } | 720 | } |
@@ -741,7 +782,7 @@ generic_file_splice_write(struct pipe_inode_info *pipe, struct file *out, | |||
741 | struct address_space *mapping = out->f_mapping; | 782 | struct address_space *mapping = out->f_mapping; |
742 | ssize_t ret; | 783 | ssize_t ret; |
743 | 784 | ||
744 | ret = move_from_pipe(pipe, out, ppos, len, flags, pipe_to_file); | 785 | ret = splice_from_pipe(pipe, out, ppos, len, flags, pipe_to_file); |
745 | if (ret > 0) { | 786 | if (ret > 0) { |
746 | struct inode *inode = mapping->host; | 787 | struct inode *inode = mapping->host; |
747 | 788 | ||
@@ -783,7 +824,7 @@ EXPORT_SYMBOL(generic_file_splice_write); | |||
783 | ssize_t generic_splice_sendpage(struct pipe_inode_info *pipe, struct file *out, | 824 | ssize_t generic_splice_sendpage(struct pipe_inode_info *pipe, struct file *out, |
784 | loff_t *ppos, size_t len, unsigned int flags) | 825 | loff_t *ppos, size_t len, unsigned int flags) |
785 | { | 826 | { |
786 | return move_from_pipe(pipe, out, ppos, len, flags, pipe_to_sendpage); | 827 | return splice_from_pipe(pipe, out, ppos, len, flags, pipe_to_sendpage); |
787 | } | 828 | } |
788 | 829 | ||
789 | EXPORT_SYMBOL(generic_splice_sendpage); | 830 | EXPORT_SYMBOL(generic_splice_sendpage); |
@@ -870,7 +911,7 @@ long do_splice_direct(struct file *in, loff_t *ppos, struct file *out, | |||
870 | 911 | ||
871 | /* | 912 | /* |
872 | * We don't have an immediate reader, but we'll read the stuff | 913 | * We don't have an immediate reader, but we'll read the stuff |
873 | * out of the pipe right after the move_to_pipe(). So set | 914 | * out of the pipe right after the splice_to_pipe(). So set |
874 | * PIPE_READERS appropriately. | 915 | * PIPE_READERS appropriately. |
875 | */ | 916 | */ |
876 | pipe->readers = 1; | 917 | pipe->readers = 1; |
@@ -1010,6 +1051,174 @@ static long do_splice(struct file *in, loff_t __user *off_in, | |||
1010 | return -EINVAL; | 1051 | return -EINVAL; |
1011 | } | 1052 | } |
1012 | 1053 | ||
1054 | /* | ||
1055 | * Map an iov into an array of pages and offset/length tupples. With the | ||
1056 | * partial_page structure, we can map several non-contiguous ranges into | ||
1057 | * our ones pages[] map instead of splitting that operation into pieces. | ||
1058 | * Could easily be exported as a generic helper for other users, in which | ||
1059 | * case one would probably want to add a 'max_nr_pages' parameter as well. | ||
1060 | */ | ||
1061 | static int get_iovec_page_array(const struct iovec __user *iov, | ||
1062 | unsigned int nr_vecs, struct page **pages, | ||
1063 | struct partial_page *partial) | ||
1064 | { | ||
1065 | int buffers = 0, error = 0; | ||
1066 | |||
1067 | /* | ||
1068 | * It's ok to take the mmap_sem for reading, even | ||
1069 | * across a "get_user()". | ||
1070 | */ | ||
1071 | down_read(¤t->mm->mmap_sem); | ||
1072 | |||
1073 | while (nr_vecs) { | ||
1074 | unsigned long off, npages; | ||
1075 | void __user *base; | ||
1076 | size_t len; | ||
1077 | int i; | ||
1078 | |||
1079 | /* | ||
1080 | * Get user address base and length for this iovec. | ||
1081 | */ | ||
1082 | error = get_user(base, &iov->iov_base); | ||
1083 | if (unlikely(error)) | ||
1084 | break; | ||
1085 | error = get_user(len, &iov->iov_len); | ||
1086 | if (unlikely(error)) | ||
1087 | break; | ||
1088 | |||
1089 | /* | ||
1090 | * Sanity check this iovec. 0 read succeeds. | ||
1091 | */ | ||
1092 | if (unlikely(!len)) | ||
1093 | break; | ||
1094 | error = -EFAULT; | ||
1095 | if (unlikely(!base)) | ||
1096 | break; | ||
1097 | |||
1098 | /* | ||
1099 | * Get this base offset and number of pages, then map | ||
1100 | * in the user pages. | ||
1101 | */ | ||
1102 | off = (unsigned long) base & ~PAGE_MASK; | ||
1103 | npages = (off + len + PAGE_SIZE - 1) >> PAGE_SHIFT; | ||
1104 | if (npages > PIPE_BUFFERS - buffers) | ||
1105 | npages = PIPE_BUFFERS - buffers; | ||
1106 | |||
1107 | error = get_user_pages(current, current->mm, | ||
1108 | (unsigned long) base, npages, 0, 0, | ||
1109 | &pages[buffers], NULL); | ||
1110 | |||
1111 | if (unlikely(error <= 0)) | ||
1112 | break; | ||
1113 | |||
1114 | /* | ||
1115 | * Fill this contiguous range into the partial page map. | ||
1116 | */ | ||
1117 | for (i = 0; i < error; i++) { | ||
1118 | const int plen = min_t(size_t, len, PAGE_SIZE) - off; | ||
1119 | |||
1120 | partial[buffers].offset = off; | ||
1121 | partial[buffers].len = plen; | ||
1122 | |||
1123 | off = 0; | ||
1124 | len -= plen; | ||
1125 | buffers++; | ||
1126 | } | ||
1127 | |||
1128 | /* | ||
1129 | * We didn't complete this iov, stop here since it probably | ||
1130 | * means we have to move some of this into a pipe to | ||
1131 | * be able to continue. | ||
1132 | */ | ||
1133 | if (len) | ||
1134 | break; | ||
1135 | |||
1136 | /* | ||
1137 | * Don't continue if we mapped fewer pages than we asked for, | ||
1138 | * or if we mapped the max number of pages that we have | ||
1139 | * room for. | ||
1140 | */ | ||
1141 | if (error < npages || buffers == PIPE_BUFFERS) | ||
1142 | break; | ||
1143 | |||
1144 | nr_vecs--; | ||
1145 | iov++; | ||
1146 | } | ||
1147 | |||
1148 | up_read(¤t->mm->mmap_sem); | ||
1149 | |||
1150 | if (buffers) | ||
1151 | return buffers; | ||
1152 | |||
1153 | return error; | ||
1154 | } | ||
1155 | |||
1156 | /* | ||
1157 | * vmsplice splices a user address range into a pipe. It can be thought of | ||
1158 | * as splice-from-memory, where the regular splice is splice-from-file (or | ||
1159 | * to file). In both cases the output is a pipe, naturally. | ||
1160 | * | ||
1161 | * Note that vmsplice only supports splicing _from_ user memory to a pipe, | ||
1162 | * not the other way around. Splicing from user memory is a simple operation | ||
1163 | * that can be supported without any funky alignment restrictions or nasty | ||
1164 | * vm tricks. We simply map in the user memory and fill them into a pipe. | ||
1165 | * The reverse isn't quite as easy, though. There are two possible solutions | ||
1166 | * for that: | ||
1167 | * | ||
1168 | * - memcpy() the data internally, at which point we might as well just | ||
1169 | * do a regular read() on the buffer anyway. | ||
1170 | * - Lots of nasty vm tricks, that are neither fast nor flexible (it | ||
1171 | * has restriction limitations on both ends of the pipe). | ||
1172 | * | ||
1173 | * Alas, it isn't here. | ||
1174 | * | ||
1175 | */ | ||
1176 | static long do_vmsplice(struct file *file, const struct iovec __user *iov, | ||
1177 | unsigned long nr_segs, unsigned int flags) | ||
1178 | { | ||
1179 | struct pipe_inode_info *pipe = file->f_dentry->d_inode->i_pipe; | ||
1180 | struct page *pages[PIPE_BUFFERS]; | ||
1181 | struct partial_page partial[PIPE_BUFFERS]; | ||
1182 | struct splice_pipe_desc spd = { | ||
1183 | .pages = pages, | ||
1184 | .partial = partial, | ||
1185 | .flags = flags, | ||
1186 | .ops = &user_page_pipe_buf_ops, | ||
1187 | }; | ||
1188 | |||
1189 | if (unlikely(!pipe)) | ||
1190 | return -EBADF; | ||
1191 | if (unlikely(nr_segs > UIO_MAXIOV)) | ||
1192 | return -EINVAL; | ||
1193 | else if (unlikely(!nr_segs)) | ||
1194 | return 0; | ||
1195 | |||
1196 | spd.nr_pages = get_iovec_page_array(iov, nr_segs, pages, partial); | ||
1197 | if (spd.nr_pages <= 0) | ||
1198 | return spd.nr_pages; | ||
1199 | |||
1200 | return splice_to_pipe(pipe, &spd); | ||
1201 | } | ||
1202 | |||
1203 | asmlinkage long sys_vmsplice(int fd, const struct iovec __user *iov, | ||
1204 | unsigned long nr_segs, unsigned int flags) | ||
1205 | { | ||
1206 | struct file *file; | ||
1207 | long error; | ||
1208 | int fput; | ||
1209 | |||
1210 | error = -EBADF; | ||
1211 | file = fget_light(fd, &fput); | ||
1212 | if (file) { | ||
1213 | if (file->f_mode & FMODE_WRITE) | ||
1214 | error = do_vmsplice(file, iov, nr_segs, flags); | ||
1215 | |||
1216 | fput_light(file, fput); | ||
1217 | } | ||
1218 | |||
1219 | return error; | ||
1220 | } | ||
1221 | |||
1013 | asmlinkage long sys_splice(int fd_in, loff_t __user *off_in, | 1222 | asmlinkage long sys_splice(int fd_in, loff_t __user *off_in, |
1014 | int fd_out, loff_t __user *off_out, | 1223 | int fd_out, loff_t __user *off_out, |
1015 | size_t len, unsigned int flags) | 1224 | size_t len, unsigned int flags) |
diff --git a/include/asm-i386/unistd.h b/include/asm-i386/unistd.h index d81d6cfc1bb4..eb4b152c82fc 100644 --- a/include/asm-i386/unistd.h +++ b/include/asm-i386/unistd.h | |||
@@ -321,8 +321,9 @@ | |||
321 | #define __NR_splice 313 | 321 | #define __NR_splice 313 |
322 | #define __NR_sync_file_range 314 | 322 | #define __NR_sync_file_range 314 |
323 | #define __NR_tee 315 | 323 | #define __NR_tee 315 |
324 | #define __NR_vmsplice 316 | ||
324 | 325 | ||
325 | #define NR_syscalls 316 | 326 | #define NR_syscalls 317 |
326 | 327 | ||
327 | /* | 328 | /* |
328 | * user-visible error numbers are in the range -1 - -128: see | 329 | * user-visible error numbers are in the range -1 - -128: see |
diff --git a/include/asm-ia64/unistd.h b/include/asm-ia64/unistd.h index a40ebec6aeeb..7107763168bf 100644 --- a/include/asm-ia64/unistd.h +++ b/include/asm-ia64/unistd.h | |||
@@ -290,12 +290,13 @@ | |||
290 | #define __NR_get_robust_list 1299 | 290 | #define __NR_get_robust_list 1299 |
291 | #define __NR_sync_file_range 1300 | 291 | #define __NR_sync_file_range 1300 |
292 | #define __NR_tee 1301 | 292 | #define __NR_tee 1301 |
293 | #define __NR_vmsplice 1302 | ||
293 | 294 | ||
294 | #ifdef __KERNEL__ | 295 | #ifdef __KERNEL__ |
295 | 296 | ||
296 | #include <linux/config.h> | 297 | #include <linux/config.h> |
297 | 298 | ||
298 | #define NR_syscalls 278 /* length of syscall table */ | 299 | #define NR_syscalls 279 /* length of syscall table */ |
299 | 300 | ||
300 | #define __ARCH_WANT_SYS_RT_SIGACTION | 301 | #define __ARCH_WANT_SYS_RT_SIGACTION |
301 | 302 | ||
diff --git a/include/asm-powerpc/unistd.h b/include/asm-powerpc/unistd.h index c612f1a62772..34325e292596 100644 --- a/include/asm-powerpc/unistd.h +++ b/include/asm-powerpc/unistd.h | |||
@@ -303,8 +303,9 @@ | |||
303 | #define __NR_unshare 282 | 303 | #define __NR_unshare 282 |
304 | #define __NR_splice 283 | 304 | #define __NR_splice 283 |
305 | #define __NR_tee 284 | 305 | #define __NR_tee 284 |
306 | #define __NR_vmsplice 285 | ||
306 | 307 | ||
307 | #define __NR_syscalls 285 | 308 | #define __NR_syscalls 286 |
308 | 309 | ||
309 | #ifdef __KERNEL__ | 310 | #ifdef __KERNEL__ |
310 | #define __NR__exit __NR_exit | 311 | #define __NR__exit __NR_exit |
diff --git a/include/asm-x86_64/unistd.h b/include/asm-x86_64/unistd.h index 98c36eae567c..feb77cb8c044 100644 --- a/include/asm-x86_64/unistd.h +++ b/include/asm-x86_64/unistd.h | |||
@@ -615,8 +615,10 @@ __SYSCALL(__NR_splice, sys_splice) | |||
615 | __SYSCALL(__NR_tee, sys_tee) | 615 | __SYSCALL(__NR_tee, sys_tee) |
616 | #define __NR_sync_file_range 277 | 616 | #define __NR_sync_file_range 277 |
617 | __SYSCALL(__NR_sync_file_range, sys_sync_file_range) | 617 | __SYSCALL(__NR_sync_file_range, sys_sync_file_range) |
618 | #define __NR_vmsplice 278 | ||
619 | __SYSCALL(__NR_vmsplice, sys_vmsplice) | ||
618 | 620 | ||
619 | #define __NR_syscall_max __NR_sync_file_range | 621 | #define __NR_syscall_max __NR_vmsplice |
620 | 622 | ||
621 | #ifndef __NO_STUBS | 623 | #ifndef __NO_STUBS |
622 | 624 | ||
diff --git a/include/linux/pipe_fs_i.h b/include/linux/pipe_fs_i.h index ef7f33c0be19..0008d4bd4059 100644 --- a/include/linux/pipe_fs_i.h +++ b/include/linux/pipe_fs_i.h | |||
@@ -61,4 +61,21 @@ void __free_pipe_info(struct pipe_inode_info *); | |||
61 | /* from/to, of course */ | 61 | /* from/to, of course */ |
62 | #define SPLICE_F_MORE (0x04) /* expect more data */ | 62 | #define SPLICE_F_MORE (0x04) /* expect more data */ |
63 | 63 | ||
64 | /* | ||
65 | * Passed to the actors | ||
66 | */ | ||
67 | struct splice_desc { | ||
68 | unsigned int len, total_len; /* current and remaining length */ | ||
69 | unsigned int flags; /* splice flags */ | ||
70 | struct file *file; /* file to read/write */ | ||
71 | loff_t pos; /* file position */ | ||
72 | }; | ||
73 | |||
74 | typedef int (splice_actor)(struct pipe_inode_info *, struct pipe_buffer *, | ||
75 | struct splice_desc *); | ||
76 | |||
77 | extern ssize_t splice_from_pipe(struct pipe_inode_info *, struct file *, | ||
78 | loff_t *, size_t, unsigned int, | ||
79 | splice_actor *); | ||
80 | |||
64 | #endif | 81 | #endif |
diff --git a/include/linux/syscalls.h b/include/linux/syscalls.h index d3ebc0e68b2b..3996960fc565 100644 --- a/include/linux/syscalls.h +++ b/include/linux/syscalls.h | |||
@@ -574,6 +574,9 @@ asmlinkage long sys_splice(int fd_in, loff_t __user *off_in, | |||
574 | int fd_out, loff_t __user *off_out, | 574 | int fd_out, loff_t __user *off_out, |
575 | size_t len, unsigned int flags); | 575 | size_t len, unsigned int flags); |
576 | 576 | ||
577 | asmlinkage long sys_vmsplice(int fd, const struct iovec __user *iov, | ||
578 | unsigned long nr_segs, unsigned int flags); | ||
579 | |||
577 | asmlinkage long sys_tee(int fdin, int fdout, size_t len, unsigned int flags); | 580 | asmlinkage long sys_tee(int fdin, int fdout, size_t len, unsigned int flags); |
578 | 581 | ||
579 | asmlinkage long sys_sync_file_range(int fd, loff_t offset, loff_t nbytes, | 582 | asmlinkage long sys_sync_file_range(int fd, loff_t offset, loff_t nbytes, |