aboutsummaryrefslogtreecommitdiffstats
path: root/fs/splice.c
diff options
context:
space:
mode:
Diffstat (limited to 'fs/splice.c')
-rw-r--r--fs/splice.c963
1 files changed, 745 insertions, 218 deletions
diff --git a/fs/splice.c b/fs/splice.c
index e50a460239dd..a285fd746dc0 100644
--- a/fs/splice.c
+++ b/fs/splice.c
@@ -27,15 +27,22 @@
27#include <linux/buffer_head.h> 27#include <linux/buffer_head.h>
28#include <linux/module.h> 28#include <linux/module.h>
29#include <linux/syscalls.h> 29#include <linux/syscalls.h>
30#include <linux/uio.h>
31
32struct partial_page {
33 unsigned int offset;
34 unsigned int len;
35};
30 36
31/* 37/*
32 * Passed to the actors 38 * Passed to splice_to_pipe
33 */ 39 */
34struct splice_desc { 40struct splice_pipe_desc {
35 unsigned int len, total_len; /* current and remaining length */ 41 struct page **pages; /* page map */
42 struct partial_page *partial; /* pages[] may not be contig */
43 int nr_pages; /* number of pages in map */
36 unsigned int flags; /* splice flags */ 44 unsigned int flags; /* splice flags */
37 struct file *file; /* file to read/write */ 45 struct pipe_buf_operations *ops;/* ops associated with output pipe */
38 loff_t pos; /* file position */
39}; 46};
40 47
41/* 48/*
@@ -44,13 +51,14 @@ struct splice_desc {
44 * addition of remove_mapping(). If success is returned, the caller may 51 * addition of remove_mapping(). If success is returned, the caller may
45 * attempt to reuse this page for another destination. 52 * attempt to reuse this page for another destination.
46 */ 53 */
47static int page_cache_pipe_buf_steal(struct pipe_inode_info *info, 54static int page_cache_pipe_buf_steal(struct pipe_inode_info *pipe,
48 struct pipe_buffer *buf) 55 struct pipe_buffer *buf)
49{ 56{
50 struct page *page = buf->page; 57 struct page *page = buf->page;
51 struct address_space *mapping = page_mapping(page); 58 struct address_space *mapping = page_mapping(page);
52 59
53 WARN_ON(!PageLocked(page)); 60 lock_page(page);
61
54 WARN_ON(!PageUptodate(page)); 62 WARN_ON(!PageUptodate(page));
55 63
56 /* 64 /*
@@ -65,24 +73,24 @@ static int page_cache_pipe_buf_steal(struct pipe_inode_info *info,
65 if (PagePrivate(page)) 73 if (PagePrivate(page))
66 try_to_release_page(page, mapping_gfp_mask(mapping)); 74 try_to_release_page(page, mapping_gfp_mask(mapping));
67 75
68 if (!remove_mapping(mapping, page)) 76 if (!remove_mapping(mapping, page)) {
77 unlock_page(page);
69 return 1; 78 return 1;
79 }
70 80
71 buf->flags |= PIPE_BUF_FLAG_STOLEN | PIPE_BUF_FLAG_LRU; 81 buf->flags |= PIPE_BUF_FLAG_LRU;
72 return 0; 82 return 0;
73} 83}
74 84
75static void page_cache_pipe_buf_release(struct pipe_inode_info *info, 85static void page_cache_pipe_buf_release(struct pipe_inode_info *pipe,
76 struct pipe_buffer *buf) 86 struct pipe_buffer *buf)
77{ 87{
78 page_cache_release(buf->page); 88 page_cache_release(buf->page);
79 buf->page = NULL; 89 buf->flags &= ~PIPE_BUF_FLAG_LRU;
80 buf->flags &= ~(PIPE_BUF_FLAG_STOLEN | PIPE_BUF_FLAG_LRU);
81} 90}
82 91
83static void *page_cache_pipe_buf_map(struct file *file, 92static int page_cache_pipe_buf_pin(struct pipe_inode_info *pipe,
84 struct pipe_inode_info *info, 93 struct pipe_buffer *buf)
85 struct pipe_buffer *buf)
86{ 94{
87 struct page *page = buf->page; 95 struct page *page = buf->page;
88 int err; 96 int err;
@@ -108,44 +116,59 @@ static void *page_cache_pipe_buf_map(struct file *file,
108 } 116 }
109 117
110 /* 118 /*
111 * Page is ok afterall, fall through to mapping. 119 * Page is ok afterall, we are done.
112 */ 120 */
113 unlock_page(page); 121 unlock_page(page);
114 } 122 }
115 123
116 return kmap(page); 124 return 0;
117error: 125error:
118 unlock_page(page); 126 unlock_page(page);
119 return ERR_PTR(err); 127 return err;
120} 128}
121 129
122static void page_cache_pipe_buf_unmap(struct pipe_inode_info *info, 130static struct pipe_buf_operations page_cache_pipe_buf_ops = {
123 struct pipe_buffer *buf) 131 .can_merge = 0,
132 .map = generic_pipe_buf_map,
133 .unmap = generic_pipe_buf_unmap,
134 .pin = page_cache_pipe_buf_pin,
135 .release = page_cache_pipe_buf_release,
136 .steal = page_cache_pipe_buf_steal,
137 .get = generic_pipe_buf_get,
138};
139
140static int user_page_pipe_buf_steal(struct pipe_inode_info *pipe,
141 struct pipe_buffer *buf)
124{ 142{
125 kunmap(buf->page); 143 if (!(buf->flags & PIPE_BUF_FLAG_GIFT))
144 return 1;
145
146 buf->flags |= PIPE_BUF_FLAG_LRU;
147 return generic_pipe_buf_steal(pipe, buf);
126} 148}
127 149
128static struct pipe_buf_operations page_cache_pipe_buf_ops = { 150static struct pipe_buf_operations user_page_pipe_buf_ops = {
129 .can_merge = 0, 151 .can_merge = 0,
130 .map = page_cache_pipe_buf_map, 152 .map = generic_pipe_buf_map,
131 .unmap = page_cache_pipe_buf_unmap, 153 .unmap = generic_pipe_buf_unmap,
154 .pin = generic_pipe_buf_pin,
132 .release = page_cache_pipe_buf_release, 155 .release = page_cache_pipe_buf_release,
133 .steal = page_cache_pipe_buf_steal, 156 .steal = user_page_pipe_buf_steal,
157 .get = generic_pipe_buf_get,
134}; 158};
135 159
136/* 160/*
137 * Pipe output worker. This sets up our pipe format with the page cache 161 * Pipe output worker. This sets up our pipe format with the page cache
138 * pipe buffer operations. Otherwise very similar to the regular pipe_writev(). 162 * pipe buffer operations. Otherwise very similar to the regular pipe_writev().
139 */ 163 */
140static ssize_t move_to_pipe(struct pipe_inode_info *pipe, struct page **pages, 164static ssize_t splice_to_pipe(struct pipe_inode_info *pipe,
141 int nr_pages, unsigned long offset, 165 struct splice_pipe_desc *spd)
142 unsigned long len, unsigned int flags)
143{ 166{
144 int ret, do_wakeup, i; 167 int ret, do_wakeup, page_nr;
145 168
146 ret = 0; 169 ret = 0;
147 do_wakeup = 0; 170 do_wakeup = 0;
148 i = 0; 171 page_nr = 0;
149 172
150 if (pipe->inode) 173 if (pipe->inode)
151 mutex_lock(&pipe->inode->i_mutex); 174 mutex_lock(&pipe->inode->i_mutex);
@@ -161,27 +184,22 @@ static ssize_t move_to_pipe(struct pipe_inode_info *pipe, struct page **pages,
161 if (pipe->nrbufs < PIPE_BUFFERS) { 184 if (pipe->nrbufs < PIPE_BUFFERS) {
162 int newbuf = (pipe->curbuf + pipe->nrbufs) & (PIPE_BUFFERS - 1); 185 int newbuf = (pipe->curbuf + pipe->nrbufs) & (PIPE_BUFFERS - 1);
163 struct pipe_buffer *buf = pipe->bufs + newbuf; 186 struct pipe_buffer *buf = pipe->bufs + newbuf;
164 struct page *page = pages[i++];
165 unsigned long this_len;
166 187
167 this_len = PAGE_CACHE_SIZE - offset; 188 buf->page = spd->pages[page_nr];
168 if (this_len > len) 189 buf->offset = spd->partial[page_nr].offset;
169 this_len = len; 190 buf->len = spd->partial[page_nr].len;
191 buf->ops = spd->ops;
192 if (spd->flags & SPLICE_F_GIFT)
193 buf->flags |= PIPE_BUF_FLAG_GIFT;
170 194
171 buf->page = page;
172 buf->offset = offset;
173 buf->len = this_len;
174 buf->ops = &page_cache_pipe_buf_ops;
175 pipe->nrbufs++; 195 pipe->nrbufs++;
196 page_nr++;
197 ret += buf->len;
198
176 if (pipe->inode) 199 if (pipe->inode)
177 do_wakeup = 1; 200 do_wakeup = 1;
178 201
179 ret += this_len; 202 if (!--spd->nr_pages)
180 len -= this_len;
181 offset = 0;
182 if (!--nr_pages)
183 break;
184 if (!len)
185 break; 203 break;
186 if (pipe->nrbufs < PIPE_BUFFERS) 204 if (pipe->nrbufs < PIPE_BUFFERS)
187 continue; 205 continue;
@@ -189,7 +207,7 @@ static ssize_t move_to_pipe(struct pipe_inode_info *pipe, struct page **pages,
189 break; 207 break;
190 } 208 }
191 209
192 if (flags & SPLICE_F_NONBLOCK) { 210 if (spd->flags & SPLICE_F_NONBLOCK) {
193 if (!ret) 211 if (!ret)
194 ret = -EAGAIN; 212 ret = -EAGAIN;
195 break; 213 break;
@@ -224,26 +242,36 @@ static ssize_t move_to_pipe(struct pipe_inode_info *pipe, struct page **pages,
224 kill_fasync(&pipe->fasync_readers, SIGIO, POLL_IN); 242 kill_fasync(&pipe->fasync_readers, SIGIO, POLL_IN);
225 } 243 }
226 244
227 while (i < nr_pages) 245 while (page_nr < spd->nr_pages)
228 page_cache_release(pages[i++]); 246 page_cache_release(spd->pages[page_nr++]);
229 247
230 return ret; 248 return ret;
231} 249}
232 250
233static int 251static int
234__generic_file_splice_read(struct file *in, struct pipe_inode_info *pipe, 252__generic_file_splice_read(struct file *in, loff_t *ppos,
235 size_t len, unsigned int flags) 253 struct pipe_inode_info *pipe, size_t len,
254 unsigned int flags)
236{ 255{
237 struct address_space *mapping = in->f_mapping; 256 struct address_space *mapping = in->f_mapping;
238 unsigned int offset, nr_pages; 257 unsigned int loff, nr_pages;
239 struct page *pages[PIPE_BUFFERS]; 258 struct page *pages[PIPE_BUFFERS];
259 struct partial_page partial[PIPE_BUFFERS];
240 struct page *page; 260 struct page *page;
241 pgoff_t index; 261 pgoff_t index, end_index;
242 int i, error; 262 loff_t isize;
243 263 size_t total_len;
244 index = in->f_pos >> PAGE_CACHE_SHIFT; 264 int error, page_nr;
245 offset = in->f_pos & ~PAGE_CACHE_MASK; 265 struct splice_pipe_desc spd = {
246 nr_pages = (len + offset + PAGE_CACHE_SIZE - 1) >> PAGE_CACHE_SHIFT; 266 .pages = pages,
267 .partial = partial,
268 .flags = flags,
269 .ops = &page_cache_pipe_buf_ops,
270 };
271
272 index = *ppos >> PAGE_CACHE_SHIFT;
273 loff = *ppos & ~PAGE_CACHE_MASK;
274 nr_pages = (len + loff + PAGE_CACHE_SIZE - 1) >> PAGE_CACHE_SHIFT;
247 275
248 if (nr_pages > PIPE_BUFFERS) 276 if (nr_pages > PIPE_BUFFERS)
249 nr_pages = PIPE_BUFFERS; 277 nr_pages = PIPE_BUFFERS;
@@ -253,49 +281,94 @@ __generic_file_splice_read(struct file *in, struct pipe_inode_info *pipe,
253 * read-ahead if this is a non-zero offset (we are likely doing small 281 * read-ahead if this is a non-zero offset (we are likely doing small
254 * chunk splice and the page is already there) for a single page. 282 * chunk splice and the page is already there) for a single page.
255 */ 283 */
256 if (!offset || nr_pages > 1) 284 if (!loff || nr_pages > 1)
257 do_page_cache_readahead(mapping, in, index, nr_pages); 285 page_cache_readahead(mapping, &in->f_ra, in, index, nr_pages);
258 286
259 /* 287 /*
260 * Now fill in the holes: 288 * Now fill in the holes:
261 */ 289 */
262 error = 0; 290 error = 0;
263 for (i = 0; i < nr_pages; i++, index++) { 291 total_len = 0;
264find_page: 292
293 /*
294 * Lookup the (hopefully) full range of pages we need.
295 */
296 spd.nr_pages = find_get_pages_contig(mapping, index, nr_pages, pages);
297
298 /*
299 * If find_get_pages_contig() returned fewer pages than we needed,
300 * allocate the rest.
301 */
302 index += spd.nr_pages;
303 while (spd.nr_pages < nr_pages) {
265 /* 304 /*
266 * lookup the page for this index 305 * Page could be there, find_get_pages_contig() breaks on
306 * the first hole.
267 */ 307 */
268 page = find_get_page(mapping, index); 308 page = find_get_page(mapping, index);
269 if (!page) { 309 if (!page) {
270 /* 310 /*
271 * If in nonblock mode then dont block on 311 * Make sure the read-ahead engine is notified
272 * readpage (we've kicked readahead so there 312 * about this failure.
273 * will be asynchronous progress):
274 */ 313 */
275 if (flags & SPLICE_F_NONBLOCK) 314 handle_ra_miss(mapping, &in->f_ra, index);
276 break;
277 315
278 /* 316 /*
279 * page didn't exist, allocate one 317 * page didn't exist, allocate one.
280 */ 318 */
281 page = page_cache_alloc_cold(mapping); 319 page = page_cache_alloc_cold(mapping);
282 if (!page) 320 if (!page)
283 break; 321 break;
284 322
285 error = add_to_page_cache_lru(page, mapping, index, 323 error = add_to_page_cache_lru(page, mapping, index,
286 mapping_gfp_mask(mapping)); 324 mapping_gfp_mask(mapping));
287 if (unlikely(error)) { 325 if (unlikely(error)) {
288 page_cache_release(page); 326 page_cache_release(page);
327 if (error == -EEXIST)
328 continue;
289 break; 329 break;
290 } 330 }
291 331 /*
292 goto readpage; 332 * add_to_page_cache() locks the page, unlock it
333 * to avoid convoluting the logic below even more.
334 */
335 unlock_page(page);
293 } 336 }
294 337
338 pages[spd.nr_pages++] = page;
339 index++;
340 }
341
342 /*
343 * Now loop over the map and see if we need to start IO on any
344 * pages, fill in the partial map, etc.
345 */
346 index = *ppos >> PAGE_CACHE_SHIFT;
347 nr_pages = spd.nr_pages;
348 spd.nr_pages = 0;
349 for (page_nr = 0; page_nr < nr_pages; page_nr++) {
350 unsigned int this_len;
351
352 if (!len)
353 break;
354
355 /*
356 * this_len is the max we'll use from this page
357 */
358 this_len = min_t(unsigned long, len, PAGE_CACHE_SIZE - loff);
359 page = pages[page_nr];
360
295 /* 361 /*
296 * If the page isn't uptodate, we may need to start io on it 362 * If the page isn't uptodate, we may need to start io on it
297 */ 363 */
298 if (!PageUptodate(page)) { 364 if (!PageUptodate(page)) {
365 /*
366 * If in nonblock mode then dont block on waiting
367 * for an in-flight io page
368 */
369 if (flags & SPLICE_F_NONBLOCK)
370 break;
371
299 lock_page(page); 372 lock_page(page);
300 373
301 /* 374 /*
@@ -305,7 +378,6 @@ find_page:
305 */ 378 */
306 if (!page->mapping) { 379 if (!page->mapping) {
307 unlock_page(page); 380 unlock_page(page);
308 page_cache_release(page);
309 break; 381 break;
310 } 382 }
311 /* 383 /*
@@ -316,25 +388,66 @@ find_page:
316 goto fill_it; 388 goto fill_it;
317 } 389 }
318 390
319readpage:
320 /* 391 /*
321 * need to read in the page 392 * need to read in the page
322 */ 393 */
323 error = mapping->a_ops->readpage(in, page); 394 error = mapping->a_ops->readpage(in, page);
324
325 if (unlikely(error)) { 395 if (unlikely(error)) {
326 page_cache_release(page); 396 /*
397 * We really should re-lookup the page here,
398 * but it complicates things a lot. Instead
399 * lets just do what we already stored, and
400 * we'll get it the next time we are called.
401 */
327 if (error == AOP_TRUNCATED_PAGE) 402 if (error == AOP_TRUNCATED_PAGE)
328 goto find_page; 403 error = 0;
404
329 break; 405 break;
330 } 406 }
407
408 /*
409 * i_size must be checked after ->readpage().
410 */
411 isize = i_size_read(mapping->host);
412 end_index = (isize - 1) >> PAGE_CACHE_SHIFT;
413 if (unlikely(!isize || index > end_index))
414 break;
415
416 /*
417 * if this is the last page, see if we need to shrink
418 * the length and stop
419 */
420 if (end_index == index) {
421 loff = PAGE_CACHE_SIZE - (isize & ~PAGE_CACHE_MASK);
422 if (total_len + loff > isize)
423 break;
424 /*
425 * force quit after adding this page
426 */
427 len = this_len;
428 this_len = min(this_len, loff);
429 loff = 0;
430 }
331 } 431 }
332fill_it: 432fill_it:
333 pages[i] = page; 433 partial[page_nr].offset = loff;
434 partial[page_nr].len = this_len;
435 len -= this_len;
436 total_len += this_len;
437 loff = 0;
438 spd.nr_pages++;
439 index++;
334 } 440 }
335 441
336 if (i) 442 /*
337 return move_to_pipe(pipe, pages, i, offset, len, flags); 443 * Release any pages at the end, if we quit early. 'i' is how far
444 * we got, 'nr_pages' is how many pages are in the map.
445 */
446 while (page_nr < nr_pages)
447 page_cache_release(pages[page_nr++]);
448
449 if (spd.nr_pages)
450 return splice_to_pipe(pipe, &spd);
338 451
339 return error; 452 return error;
340} 453}
@@ -348,8 +461,9 @@ fill_it:
348 * 461 *
349 * Will read pages from given file and fill them into a pipe. 462 * Will read pages from given file and fill them into a pipe.
350 */ 463 */
351ssize_t generic_file_splice_read(struct file *in, struct pipe_inode_info *pipe, 464ssize_t generic_file_splice_read(struct file *in, loff_t *ppos,
352 size_t len, unsigned int flags) 465 struct pipe_inode_info *pipe, size_t len,
466 unsigned int flags)
353{ 467{
354 ssize_t spliced; 468 ssize_t spliced;
355 int ret; 469 int ret;
@@ -358,19 +472,22 @@ ssize_t generic_file_splice_read(struct file *in, struct pipe_inode_info *pipe,
358 spliced = 0; 472 spliced = 0;
359 473
360 while (len) { 474 while (len) {
361 ret = __generic_file_splice_read(in, pipe, len, flags); 475 ret = __generic_file_splice_read(in, ppos, pipe, len, flags);
362 476
363 if (ret <= 0) 477 if (ret < 0)
364 break; 478 break;
479 else if (!ret) {
480 if (spliced)
481 break;
482 if (flags & SPLICE_F_NONBLOCK) {
483 ret = -EAGAIN;
484 break;
485 }
486 }
365 487
366 in->f_pos += ret; 488 *ppos += ret;
367 len -= ret; 489 len -= ret;
368 spliced += ret; 490 spliced += ret;
369
370 if (!(flags & SPLICE_F_NONBLOCK))
371 continue;
372 ret = -EAGAIN;
373 break;
374 } 491 }
375 492
376 if (spliced) 493 if (spliced)
@@ -383,38 +500,24 @@ EXPORT_SYMBOL(generic_file_splice_read);
383 500
384/* 501/*
385 * Send 'sd->len' bytes to socket from 'sd->file' at position 'sd->pos' 502 * Send 'sd->len' bytes to socket from 'sd->file' at position 'sd->pos'
386 * using sendpage(). 503 * using sendpage(). Return the number of bytes sent.
387 */ 504 */
388static int pipe_to_sendpage(struct pipe_inode_info *info, 505static int pipe_to_sendpage(struct pipe_inode_info *pipe,
389 struct pipe_buffer *buf, struct splice_desc *sd) 506 struct pipe_buffer *buf, struct splice_desc *sd)
390{ 507{
391 struct file *file = sd->file; 508 struct file *file = sd->file;
392 loff_t pos = sd->pos; 509 loff_t pos = sd->pos;
393 unsigned int offset; 510 int ret, more;
394 ssize_t ret;
395 void *ptr;
396 int more;
397
398 /*
399 * Sub-optimal, but we are limited by the pipe ->map. We don't
400 * need a kmap'ed buffer here, we just want to make sure we
401 * have the page pinned if the pipe page originates from the
402 * page cache.
403 */
404 ptr = buf->ops->map(file, info, buf);
405 if (IS_ERR(ptr))
406 return PTR_ERR(ptr);
407 511
408 offset = pos & ~PAGE_CACHE_MASK; 512 ret = buf->ops->pin(pipe, buf);
409 more = (sd->flags & SPLICE_F_MORE) || sd->len < sd->total_len; 513 if (!ret) {
514 more = (sd->flags & SPLICE_F_MORE) || sd->len < sd->total_len;
410 515
411 ret = file->f_op->sendpage(file, buf->page, offset, sd->len, &pos,more); 516 ret = file->f_op->sendpage(file, buf->page, buf->offset,
412 517 sd->len, &pos, more);
413 buf->ops->unmap(info, buf); 518 }
414 if (ret == sd->len)
415 return 0;
416 519
417 return -EIO; 520 return ret;
418} 521}
419 522
420/* 523/*
@@ -437,62 +540,80 @@ static int pipe_to_sendpage(struct pipe_inode_info *info,
437 * SPLICE_F_MOVE isn't set, or we cannot move the page, we simply create 540 * SPLICE_F_MOVE isn't set, or we cannot move the page, we simply create
438 * a new page in the output file page cache and fill/dirty that. 541 * a new page in the output file page cache and fill/dirty that.
439 */ 542 */
440static int pipe_to_file(struct pipe_inode_info *info, struct pipe_buffer *buf, 543static int pipe_to_file(struct pipe_inode_info *pipe, struct pipe_buffer *buf,
441 struct splice_desc *sd) 544 struct splice_desc *sd)
442{ 545{
443 struct file *file = sd->file; 546 struct file *file = sd->file;
444 struct address_space *mapping = file->f_mapping; 547 struct address_space *mapping = file->f_mapping;
445 gfp_t gfp_mask = mapping_gfp_mask(mapping); 548 gfp_t gfp_mask = mapping_gfp_mask(mapping);
446 unsigned int offset; 549 unsigned int offset, this_len;
447 struct page *page; 550 struct page *page;
448 pgoff_t index; 551 pgoff_t index;
449 char *src;
450 int ret; 552 int ret;
451 553
452 /* 554 /*
453 * make sure the data in this buffer is uptodate 555 * make sure the data in this buffer is uptodate
454 */ 556 */
455 src = buf->ops->map(file, info, buf); 557 ret = buf->ops->pin(pipe, buf);
456 if (IS_ERR(src)) 558 if (unlikely(ret))
457 return PTR_ERR(src); 559 return ret;
458 560
459 index = sd->pos >> PAGE_CACHE_SHIFT; 561 index = sd->pos >> PAGE_CACHE_SHIFT;
460 offset = sd->pos & ~PAGE_CACHE_MASK; 562 offset = sd->pos & ~PAGE_CACHE_MASK;
461 563
564 this_len = sd->len;
565 if (this_len + offset > PAGE_CACHE_SIZE)
566 this_len = PAGE_CACHE_SIZE - offset;
567
462 /* 568 /*
463 * Reuse buf page, if SPLICE_F_MOVE is set. 569 * Reuse buf page, if SPLICE_F_MOVE is set and we are doing a full
570 * page.
464 */ 571 */
465 if (sd->flags & SPLICE_F_MOVE) { 572 if ((sd->flags & SPLICE_F_MOVE) && this_len == PAGE_CACHE_SIZE) {
466 /* 573 /*
467 * If steal succeeds, buf->page is now pruned from the vm 574 * If steal succeeds, buf->page is now pruned from the
468 * side (LRU and page cache) and we can reuse it. 575 * pagecache and we can reuse it. The page will also be
576 * locked on successful return.
469 */ 577 */
470 if (buf->ops->steal(info, buf)) 578 if (buf->ops->steal(pipe, buf))
471 goto find_page; 579 goto find_page;
472 580
473 /*
474 * this will also set the page locked
475 */
476 page = buf->page; 581 page = buf->page;
477 if (add_to_page_cache(page, mapping, index, gfp_mask)) 582 if (add_to_page_cache(page, mapping, index, gfp_mask)) {
583 unlock_page(page);
478 goto find_page; 584 goto find_page;
585 }
586
587 page_cache_get(page);
479 588
480 if (!(buf->flags & PIPE_BUF_FLAG_LRU)) 589 if (!(buf->flags & PIPE_BUF_FLAG_LRU))
481 lru_cache_add(page); 590 lru_cache_add(page);
482 } else { 591 } else {
483find_page: 592find_page:
484 ret = -ENOMEM; 593 page = find_lock_page(mapping, index);
485 page = find_or_create_page(mapping, index, gfp_mask); 594 if (!page) {
486 if (!page) 595 ret = -ENOMEM;
487 goto out_nomem; 596 page = page_cache_alloc_cold(mapping);
597 if (unlikely(!page))
598 goto out_nomem;
599
600 /*
601 * This will also lock the page
602 */
603 ret = add_to_page_cache_lru(page, mapping, index,
604 gfp_mask);
605 if (unlikely(ret))
606 goto out;
607 }
488 608
489 /* 609 /*
490 * If the page is uptodate, it is also locked. If it isn't 610 * We get here with the page locked. If the page is also
491 * uptodate, we can mark it uptodate if we are filling the 611 * uptodate, we don't need to do more. If it isn't, we
492 * full page. Otherwise we need to read it in first... 612 * may need to bring it in if we are not going to overwrite
613 * the full page.
493 */ 614 */
494 if (!PageUptodate(page)) { 615 if (!PageUptodate(page)) {
495 if (sd->len < PAGE_CACHE_SIZE) { 616 if (this_len < PAGE_CACHE_SIZE) {
496 ret = mapping->a_ops->readpage(file, page); 617 ret = mapping->a_ops->readpage(file, page);
497 if (unlikely(ret)) 618 if (unlikely(ret))
498 goto out; 619 goto out;
@@ -511,58 +632,72 @@ find_page:
511 ret = -EIO; 632 ret = -EIO;
512 goto out; 633 goto out;
513 } 634 }
514 } else { 635 } else
515 WARN_ON(!PageLocked(page));
516 SetPageUptodate(page); 636 SetPageUptodate(page);
517 }
518 } 637 }
519 } 638 }
520 639
521 ret = mapping->a_ops->prepare_write(file, page, 0, sd->len); 640 ret = mapping->a_ops->prepare_write(file, page, offset, offset+this_len);
522 if (ret == AOP_TRUNCATED_PAGE) { 641 if (unlikely(ret)) {
642 loff_t isize = i_size_read(mapping->host);
643
644 if (ret != AOP_TRUNCATED_PAGE)
645 unlock_page(page);
523 page_cache_release(page); 646 page_cache_release(page);
524 goto find_page; 647 if (ret == AOP_TRUNCATED_PAGE)
525 } else if (ret) 648 goto find_page;
649
650 /*
651 * prepare_write() may have instantiated a few blocks
652 * outside i_size. Trim these off again.
653 */
654 if (sd->pos + this_len > isize)
655 vmtruncate(mapping->host, isize);
656
526 goto out; 657 goto out;
658 }
527 659
528 if (!(buf->flags & PIPE_BUF_FLAG_STOLEN)) { 660 if (buf->page != page) {
529 char *dst = kmap_atomic(page, KM_USER0); 661 /*
662 * Careful, ->map() uses KM_USER0!
663 */
664 char *src = buf->ops->map(pipe, buf, 1);
665 char *dst = kmap_atomic(page, KM_USER1);
530 666
531 memcpy(dst + offset, src + buf->offset, sd->len); 667 memcpy(dst + offset, src + buf->offset, this_len);
532 flush_dcache_page(page); 668 flush_dcache_page(page);
533 kunmap_atomic(dst, KM_USER0); 669 kunmap_atomic(dst, KM_USER1);
670 buf->ops->unmap(pipe, buf, src);
534 } 671 }
535 672
536 ret = mapping->a_ops->commit_write(file, page, 0, sd->len); 673 ret = mapping->a_ops->commit_write(file, page, offset, offset+this_len);
537 if (ret == AOP_TRUNCATED_PAGE) { 674 if (!ret) {
675 /*
676 * Return the number of bytes written and mark page as
677 * accessed, we are now done!
678 */
679 ret = this_len;
680 mark_page_accessed(page);
681 balance_dirty_pages_ratelimited(mapping);
682 } else if (ret == AOP_TRUNCATED_PAGE) {
538 page_cache_release(page); 683 page_cache_release(page);
539 goto find_page; 684 goto find_page;
540 } else if (ret)
541 goto out;
542
543 mark_page_accessed(page);
544 balance_dirty_pages_ratelimited(mapping);
545out:
546 if (!(buf->flags & PIPE_BUF_FLAG_STOLEN)) {
547 page_cache_release(page);
548 unlock_page(page);
549 } 685 }
686out:
687 page_cache_release(page);
688 unlock_page(page);
550out_nomem: 689out_nomem:
551 buf->ops->unmap(info, buf);
552 return ret; 690 return ret;
553} 691}
554 692
555typedef int (splice_actor)(struct pipe_inode_info *, struct pipe_buffer *,
556 struct splice_desc *);
557
558/* 693/*
559 * Pipe input worker. Most of this logic works like a regular pipe, the 694 * Pipe input worker. Most of this logic works like a regular pipe, the
560 * key here is the 'actor' worker passed in that actually moves the data 695 * key here is the 'actor' worker passed in that actually moves the data
561 * to the wanted destination. See pipe_to_file/pipe_to_sendpage above. 696 * to the wanted destination. See pipe_to_file/pipe_to_sendpage above.
562 */ 697 */
563static ssize_t move_from_pipe(struct pipe_inode_info *pipe, struct file *out, 698ssize_t splice_from_pipe(struct pipe_inode_info *pipe, struct file *out,
564 size_t len, unsigned int flags, 699 loff_t *ppos, size_t len, unsigned int flags,
565 splice_actor *actor) 700 splice_actor *actor)
566{ 701{
567 int ret, do_wakeup, err; 702 int ret, do_wakeup, err;
568 struct splice_desc sd; 703 struct splice_desc sd;
@@ -573,7 +708,7 @@ static ssize_t move_from_pipe(struct pipe_inode_info *pipe, struct file *out,
573 sd.total_len = len; 708 sd.total_len = len;
574 sd.flags = flags; 709 sd.flags = flags;
575 sd.file = out; 710 sd.file = out;
576 sd.pos = out->f_pos; 711 sd.pos = *ppos;
577 712
578 if (pipe->inode) 713 if (pipe->inode)
579 mutex_lock(&pipe->inode->i_mutex); 714 mutex_lock(&pipe->inode->i_mutex);
@@ -588,16 +723,22 @@ static ssize_t move_from_pipe(struct pipe_inode_info *pipe, struct file *out,
588 sd.len = sd.total_len; 723 sd.len = sd.total_len;
589 724
590 err = actor(pipe, buf, &sd); 725 err = actor(pipe, buf, &sd);
591 if (err) { 726 if (err <= 0) {
592 if (!ret && err != -ENODATA) 727 if (!ret && err != -ENODATA)
593 ret = err; 728 ret = err;
594 729
595 break; 730 break;
596 } 731 }
597 732
598 ret += sd.len; 733 ret += err;
599 buf->offset += sd.len; 734 buf->offset += err;
600 buf->len -= sd.len; 735 buf->len -= err;
736
737 sd.len -= err;
738 sd.pos += err;
739 sd.total_len -= err;
740 if (sd.len)
741 continue;
601 742
602 if (!buf->len) { 743 if (!buf->len) {
603 buf->ops = NULL; 744 buf->ops = NULL;
@@ -608,8 +749,6 @@ static ssize_t move_from_pipe(struct pipe_inode_info *pipe, struct file *out,
608 do_wakeup = 1; 749 do_wakeup = 1;
609 } 750 }
610 751
611 sd.pos += sd.len;
612 sd.total_len -= sd.len;
613 if (!sd.total_len) 752 if (!sd.total_len)
614 break; 753 break;
615 } 754 }
@@ -656,9 +795,7 @@ static ssize_t move_from_pipe(struct pipe_inode_info *pipe, struct file *out,
656 kill_fasync(&pipe->fasync_writers, SIGIO, POLL_OUT); 795 kill_fasync(&pipe->fasync_writers, SIGIO, POLL_OUT);
657 } 796 }
658 797
659 out->f_pos = sd.pos;
660 return ret; 798 return ret;
661
662} 799}
663 800
664/** 801/**
@@ -674,28 +811,32 @@ static ssize_t move_from_pipe(struct pipe_inode_info *pipe, struct file *out,
674 */ 811 */
675ssize_t 812ssize_t
676generic_file_splice_write(struct pipe_inode_info *pipe, struct file *out, 813generic_file_splice_write(struct pipe_inode_info *pipe, struct file *out,
677 size_t len, unsigned int flags) 814 loff_t *ppos, size_t len, unsigned int flags)
678{ 815{
679 struct address_space *mapping = out->f_mapping; 816 struct address_space *mapping = out->f_mapping;
680 ssize_t ret; 817 ssize_t ret;
681 818
682 ret = move_from_pipe(pipe, out, len, flags, pipe_to_file); 819 ret = splice_from_pipe(pipe, out, ppos, len, flags, pipe_to_file);
683 820 if (ret > 0) {
684 /*
685 * If file or inode is SYNC and we actually wrote some data, sync it.
686 */
687 if (unlikely((out->f_flags & O_SYNC) || IS_SYNC(mapping->host))
688 && ret > 0) {
689 struct inode *inode = mapping->host; 821 struct inode *inode = mapping->host;
690 int err;
691 822
692 mutex_lock(&inode->i_mutex); 823 *ppos += ret;
693 err = generic_osync_inode(mapping->host, mapping,
694 OSYNC_METADATA|OSYNC_DATA);
695 mutex_unlock(&inode->i_mutex);
696 824
697 if (err) 825 /*
698 ret = err; 826 * If file or inode is SYNC and we actually wrote some data,
827 * sync it.
828 */
829 if (unlikely((out->f_flags & O_SYNC) || IS_SYNC(inode))) {
830 int err;
831
832 mutex_lock(&inode->i_mutex);
833 err = generic_osync_inode(inode, mapping,
834 OSYNC_METADATA|OSYNC_DATA);
835 mutex_unlock(&inode->i_mutex);
836
837 if (err)
838 ret = err;
839 }
699 } 840 }
700 841
701 return ret; 842 return ret;
@@ -715,9 +856,9 @@ EXPORT_SYMBOL(generic_file_splice_write);
715 * 856 *
716 */ 857 */
717ssize_t generic_splice_sendpage(struct pipe_inode_info *pipe, struct file *out, 858ssize_t generic_splice_sendpage(struct pipe_inode_info *pipe, struct file *out,
718 size_t len, unsigned int flags) 859 loff_t *ppos, size_t len, unsigned int flags)
719{ 860{
720 return move_from_pipe(pipe, out, len, flags, pipe_to_sendpage); 861 return splice_from_pipe(pipe, out, ppos, len, flags, pipe_to_sendpage);
721} 862}
722 863
723EXPORT_SYMBOL(generic_splice_sendpage); 864EXPORT_SYMBOL(generic_splice_sendpage);
@@ -726,9 +867,8 @@ EXPORT_SYMBOL(generic_splice_sendpage);
726 * Attempt to initiate a splice from pipe to file. 867 * Attempt to initiate a splice from pipe to file.
727 */ 868 */
728static long do_splice_from(struct pipe_inode_info *pipe, struct file *out, 869static long do_splice_from(struct pipe_inode_info *pipe, struct file *out,
729 size_t len, unsigned int flags) 870 loff_t *ppos, size_t len, unsigned int flags)
730{ 871{
731 loff_t pos;
732 int ret; 872 int ret;
733 873
734 if (unlikely(!out->f_op || !out->f_op->splice_write)) 874 if (unlikely(!out->f_op || !out->f_op->splice_write))
@@ -737,22 +877,21 @@ static long do_splice_from(struct pipe_inode_info *pipe, struct file *out,
737 if (unlikely(!(out->f_mode & FMODE_WRITE))) 877 if (unlikely(!(out->f_mode & FMODE_WRITE)))
738 return -EBADF; 878 return -EBADF;
739 879
740 pos = out->f_pos; 880 ret = rw_verify_area(WRITE, out, ppos, len);
741
742 ret = rw_verify_area(WRITE, out, &pos, len);
743 if (unlikely(ret < 0)) 881 if (unlikely(ret < 0))
744 return ret; 882 return ret;
745 883
746 return out->f_op->splice_write(pipe, out, len, flags); 884 return out->f_op->splice_write(pipe, out, ppos, len, flags);
747} 885}
748 886
749/* 887/*
750 * Attempt to initiate a splice from a file to a pipe. 888 * Attempt to initiate a splice from a file to a pipe.
751 */ 889 */
752static long do_splice_to(struct file *in, struct pipe_inode_info *pipe, 890static long do_splice_to(struct file *in, loff_t *ppos,
753 size_t len, unsigned int flags) 891 struct pipe_inode_info *pipe, size_t len,
892 unsigned int flags)
754{ 893{
755 loff_t pos, isize, left; 894 loff_t isize, left;
756 int ret; 895 int ret;
757 896
758 if (unlikely(!in->f_op || !in->f_op->splice_read)) 897 if (unlikely(!in->f_op || !in->f_op->splice_read))
@@ -761,28 +900,27 @@ static long do_splice_to(struct file *in, struct pipe_inode_info *pipe,
761 if (unlikely(!(in->f_mode & FMODE_READ))) 900 if (unlikely(!(in->f_mode & FMODE_READ)))
762 return -EBADF; 901 return -EBADF;
763 902
764 pos = in->f_pos; 903 ret = rw_verify_area(READ, in, ppos, len);
765
766 ret = rw_verify_area(READ, in, &pos, len);
767 if (unlikely(ret < 0)) 904 if (unlikely(ret < 0))
768 return ret; 905 return ret;
769 906
770 isize = i_size_read(in->f_mapping->host); 907 isize = i_size_read(in->f_mapping->host);
771 if (unlikely(in->f_pos >= isize)) 908 if (unlikely(*ppos >= isize))
772 return 0; 909 return 0;
773 910
774 left = isize - in->f_pos; 911 left = isize - *ppos;
775 if (unlikely(left < len)) 912 if (unlikely(left < len))
776 len = left; 913 len = left;
777 914
778 return in->f_op->splice_read(in, pipe, len, flags); 915 return in->f_op->splice_read(in, ppos, pipe, len, flags);
779} 916}
780 917
781long do_splice_direct(struct file *in, struct file *out, size_t len, 918long do_splice_direct(struct file *in, loff_t *ppos, struct file *out,
782 unsigned int flags) 919 size_t len, unsigned int flags)
783{ 920{
784 struct pipe_inode_info *pipe; 921 struct pipe_inode_info *pipe;
785 long ret, bytes; 922 long ret, bytes;
923 loff_t out_off;
786 umode_t i_mode; 924 umode_t i_mode;
787 int i; 925 int i;
788 926
@@ -807,7 +945,7 @@ long do_splice_direct(struct file *in, struct file *out, size_t len,
807 945
808 /* 946 /*
809 * We don't have an immediate reader, but we'll read the stuff 947 * We don't have an immediate reader, but we'll read the stuff
810 * out of the pipe right after the move_to_pipe(). So set 948 * out of the pipe right after the splice_to_pipe(). So set
811 * PIPE_READERS appropriately. 949 * PIPE_READERS appropriately.
812 */ 950 */
813 pipe->readers = 1; 951 pipe->readers = 1;
@@ -820,6 +958,7 @@ long do_splice_direct(struct file *in, struct file *out, size_t len,
820 */ 958 */
821 ret = 0; 959 ret = 0;
822 bytes = 0; 960 bytes = 0;
961 out_off = 0;
823 962
824 while (len) { 963 while (len) {
825 size_t read_len, max_read_len; 964 size_t read_len, max_read_len;
@@ -829,7 +968,7 @@ long do_splice_direct(struct file *in, struct file *out, size_t len,
829 */ 968 */
830 max_read_len = min(len, (size_t)(PIPE_BUFFERS*PAGE_SIZE)); 969 max_read_len = min(len, (size_t)(PIPE_BUFFERS*PAGE_SIZE));
831 970
832 ret = do_splice_to(in, pipe, max_read_len, flags); 971 ret = do_splice_to(in, ppos, pipe, max_read_len, flags);
833 if (unlikely(ret < 0)) 972 if (unlikely(ret < 0))
834 goto out_release; 973 goto out_release;
835 974
@@ -840,7 +979,7 @@ long do_splice_direct(struct file *in, struct file *out, size_t len,
840 * must not do the output in nonblocking mode as then we 979 * must not do the output in nonblocking mode as then we
841 * could get stuck data in the internal pipe: 980 * could get stuck data in the internal pipe:
842 */ 981 */
843 ret = do_splice_from(pipe, out, read_len, 982 ret = do_splice_from(pipe, out, &out_off, read_len,
844 flags & ~SPLICE_F_NONBLOCK); 983 flags & ~SPLICE_F_NONBLOCK);
845 if (unlikely(ret < 0)) 984 if (unlikely(ret < 0))
846 goto out_release; 985 goto out_release;
@@ -898,6 +1037,8 @@ static long do_splice(struct file *in, loff_t __user *off_in,
898 size_t len, unsigned int flags) 1037 size_t len, unsigned int flags)
899{ 1038{
900 struct pipe_inode_info *pipe; 1039 struct pipe_inode_info *pipe;
1040 loff_t offset, *off;
1041 long ret;
901 1042
902 pipe = in->f_dentry->d_inode->i_pipe; 1043 pipe = in->f_dentry->d_inode->i_pipe;
903 if (pipe) { 1044 if (pipe) {
@@ -906,12 +1047,18 @@ static long do_splice(struct file *in, loff_t __user *off_in,
906 if (off_out) { 1047 if (off_out) {
907 if (out->f_op->llseek == no_llseek) 1048 if (out->f_op->llseek == no_llseek)
908 return -EINVAL; 1049 return -EINVAL;
909 if (copy_from_user(&out->f_pos, off_out, 1050 if (copy_from_user(&offset, off_out, sizeof(loff_t)))
910 sizeof(loff_t)))
911 return -EFAULT; 1051 return -EFAULT;
912 } 1052 off = &offset;
1053 } else
1054 off = &out->f_pos;
913 1055
914 return do_splice_from(pipe, out, len, flags); 1056 ret = do_splice_from(pipe, out, off, len, flags);
1057
1058 if (off_out && copy_to_user(off_out, off, sizeof(loff_t)))
1059 ret = -EFAULT;
1060
1061 return ret;
915 } 1062 }
916 1063
917 pipe = out->f_dentry->d_inode->i_pipe; 1064 pipe = out->f_dentry->d_inode->i_pipe;
@@ -921,16 +1068,201 @@ static long do_splice(struct file *in, loff_t __user *off_in,
921 if (off_in) { 1068 if (off_in) {
922 if (in->f_op->llseek == no_llseek) 1069 if (in->f_op->llseek == no_llseek)
923 return -EINVAL; 1070 return -EINVAL;
924 if (copy_from_user(&in->f_pos, off_in, sizeof(loff_t))) 1071 if (copy_from_user(&offset, off_in, sizeof(loff_t)))
925 return -EFAULT; 1072 return -EFAULT;
926 } 1073 off = &offset;
1074 } else
1075 off = &in->f_pos;
927 1076
928 return do_splice_to(in, pipe, len, flags); 1077 ret = do_splice_to(in, off, pipe, len, flags);
1078
1079 if (off_in && copy_to_user(off_in, off, sizeof(loff_t)))
1080 ret = -EFAULT;
1081
1082 return ret;
929 } 1083 }
930 1084
931 return -EINVAL; 1085 return -EINVAL;
932} 1086}
933 1087
1088/*
1089 * Map an iov into an array of pages and offset/length tupples. With the
1090 * partial_page structure, we can map several non-contiguous ranges into
1091 * our ones pages[] map instead of splitting that operation into pieces.
1092 * Could easily be exported as a generic helper for other users, in which
1093 * case one would probably want to add a 'max_nr_pages' parameter as well.
1094 */
1095static int get_iovec_page_array(const struct iovec __user *iov,
1096 unsigned int nr_vecs, struct page **pages,
1097 struct partial_page *partial, int aligned)
1098{
1099 int buffers = 0, error = 0;
1100
1101 /*
1102 * It's ok to take the mmap_sem for reading, even
1103 * across a "get_user()".
1104 */
1105 down_read(&current->mm->mmap_sem);
1106
1107 while (nr_vecs) {
1108 unsigned long off, npages;
1109 void __user *base;
1110 size_t len;
1111 int i;
1112
1113 /*
1114 * Get user address base and length for this iovec.
1115 */
1116 error = get_user(base, &iov->iov_base);
1117 if (unlikely(error))
1118 break;
1119 error = get_user(len, &iov->iov_len);
1120 if (unlikely(error))
1121 break;
1122
1123 /*
1124 * Sanity check this iovec. 0 read succeeds.
1125 */
1126 if (unlikely(!len))
1127 break;
1128 error = -EFAULT;
1129 if (unlikely(!base))
1130 break;
1131
1132 /*
1133 * Get this base offset and number of pages, then map
1134 * in the user pages.
1135 */
1136 off = (unsigned long) base & ~PAGE_MASK;
1137
1138 /*
1139 * If asked for alignment, the offset must be zero and the
1140 * length a multiple of the PAGE_SIZE.
1141 */
1142 error = -EINVAL;
1143 if (aligned && (off || len & ~PAGE_MASK))
1144 break;
1145
1146 npages = (off + len + PAGE_SIZE - 1) >> PAGE_SHIFT;
1147 if (npages > PIPE_BUFFERS - buffers)
1148 npages = PIPE_BUFFERS - buffers;
1149
1150 error = get_user_pages(current, current->mm,
1151 (unsigned long) base, npages, 0, 0,
1152 &pages[buffers], NULL);
1153
1154 if (unlikely(error <= 0))
1155 break;
1156
1157 /*
1158 * Fill this contiguous range into the partial page map.
1159 */
1160 for (i = 0; i < error; i++) {
1161 const int plen = min_t(size_t, len, PAGE_SIZE - off);
1162
1163 partial[buffers].offset = off;
1164 partial[buffers].len = plen;
1165
1166 off = 0;
1167 len -= plen;
1168 buffers++;
1169 }
1170
1171 /*
1172 * We didn't complete this iov, stop here since it probably
1173 * means we have to move some of this into a pipe to
1174 * be able to continue.
1175 */
1176 if (len)
1177 break;
1178
1179 /*
1180 * Don't continue if we mapped fewer pages than we asked for,
1181 * or if we mapped the max number of pages that we have
1182 * room for.
1183 */
1184 if (error < npages || buffers == PIPE_BUFFERS)
1185 break;
1186
1187 nr_vecs--;
1188 iov++;
1189 }
1190
1191 up_read(&current->mm->mmap_sem);
1192
1193 if (buffers)
1194 return buffers;
1195
1196 return error;
1197}
1198
1199/*
1200 * vmsplice splices a user address range into a pipe. It can be thought of
1201 * as splice-from-memory, where the regular splice is splice-from-file (or
1202 * to file). In both cases the output is a pipe, naturally.
1203 *
1204 * Note that vmsplice only supports splicing _from_ user memory to a pipe,
1205 * not the other way around. Splicing from user memory is a simple operation
1206 * that can be supported without any funky alignment restrictions or nasty
1207 * vm tricks. We simply map in the user memory and fill them into a pipe.
1208 * The reverse isn't quite as easy, though. There are two possible solutions
1209 * for that:
1210 *
1211 * - memcpy() the data internally, at which point we might as well just
1212 * do a regular read() on the buffer anyway.
1213 * - Lots of nasty vm tricks, that are neither fast nor flexible (it
1214 * has restriction limitations on both ends of the pipe).
1215 *
1216 * Alas, it isn't here.
1217 *
1218 */
1219static long do_vmsplice(struct file *file, const struct iovec __user *iov,
1220 unsigned long nr_segs, unsigned int flags)
1221{
1222 struct pipe_inode_info *pipe = file->f_dentry->d_inode->i_pipe;
1223 struct page *pages[PIPE_BUFFERS];
1224 struct partial_page partial[PIPE_BUFFERS];
1225 struct splice_pipe_desc spd = {
1226 .pages = pages,
1227 .partial = partial,
1228 .flags = flags,
1229 .ops = &user_page_pipe_buf_ops,
1230 };
1231
1232 if (unlikely(!pipe))
1233 return -EBADF;
1234 if (unlikely(nr_segs > UIO_MAXIOV))
1235 return -EINVAL;
1236 else if (unlikely(!nr_segs))
1237 return 0;
1238
1239 spd.nr_pages = get_iovec_page_array(iov, nr_segs, pages, partial,
1240 flags & SPLICE_F_GIFT);
1241 if (spd.nr_pages <= 0)
1242 return spd.nr_pages;
1243
1244 return splice_to_pipe(pipe, &spd);
1245}
1246
1247asmlinkage long sys_vmsplice(int fd, const struct iovec __user *iov,
1248 unsigned long nr_segs, unsigned int flags)
1249{
1250 struct file *file;
1251 long error;
1252 int fput;
1253
1254 error = -EBADF;
1255 file = fget_light(fd, &fput);
1256 if (file) {
1257 if (file->f_mode & FMODE_WRITE)
1258 error = do_vmsplice(file, iov, nr_segs, flags);
1259
1260 fput_light(file, fput);
1261 }
1262
1263 return error;
1264}
1265
934asmlinkage long sys_splice(int fd_in, loff_t __user *off_in, 1266asmlinkage long sys_splice(int fd_in, loff_t __user *off_in,
935 int fd_out, loff_t __user *off_out, 1267 int fd_out, loff_t __user *off_out,
936 size_t len, unsigned int flags) 1268 size_t len, unsigned int flags)
@@ -961,3 +1293,198 @@ asmlinkage long sys_splice(int fd_in, loff_t __user *off_in,
961 1293
962 return error; 1294 return error;
963} 1295}
1296
1297/*
1298 * Link contents of ipipe to opipe.
1299 */
1300static int link_pipe(struct pipe_inode_info *ipipe,
1301 struct pipe_inode_info *opipe,
1302 size_t len, unsigned int flags)
1303{
1304 struct pipe_buffer *ibuf, *obuf;
1305 int ret, do_wakeup, i, ipipe_first;
1306
1307 ret = do_wakeup = ipipe_first = 0;
1308
1309 /*
1310 * Potential ABBA deadlock, work around it by ordering lock
1311 * grabbing by inode address. Otherwise two different processes
1312 * could deadlock (one doing tee from A -> B, the other from B -> A).
1313 */
1314 if (ipipe->inode < opipe->inode) {
1315 ipipe_first = 1;
1316 mutex_lock(&ipipe->inode->i_mutex);
1317 mutex_lock(&opipe->inode->i_mutex);
1318 } else {
1319 mutex_lock(&opipe->inode->i_mutex);
1320 mutex_lock(&ipipe->inode->i_mutex);
1321 }
1322
1323 for (i = 0;; i++) {
1324 if (!opipe->readers) {
1325 send_sig(SIGPIPE, current, 0);
1326 if (!ret)
1327 ret = -EPIPE;
1328 break;
1329 }
1330 if (ipipe->nrbufs - i) {
1331 ibuf = ipipe->bufs + ((ipipe->curbuf + i) & (PIPE_BUFFERS - 1));
1332
1333 /*
1334 * If we have room, fill this buffer
1335 */
1336 if (opipe->nrbufs < PIPE_BUFFERS) {
1337 int nbuf = (opipe->curbuf + opipe->nrbufs) & (PIPE_BUFFERS - 1);
1338
1339 /*
1340 * Get a reference to this pipe buffer,
1341 * so we can copy the contents over.
1342 */
1343 ibuf->ops->get(ipipe, ibuf);
1344
1345 obuf = opipe->bufs + nbuf;
1346 *obuf = *ibuf;
1347
1348 /*
1349 * Don't inherit the gift flag, we need to
1350 * prevent multiple steals of this page.
1351 */
1352 obuf->flags &= ~PIPE_BUF_FLAG_GIFT;
1353
1354 if (obuf->len > len)
1355 obuf->len = len;
1356
1357 opipe->nrbufs++;
1358 do_wakeup = 1;
1359 ret += obuf->len;
1360 len -= obuf->len;
1361
1362 if (!len)
1363 break;
1364 if (opipe->nrbufs < PIPE_BUFFERS)
1365 continue;
1366 }
1367
1368 /*
1369 * We have input available, but no output room.
1370 * If we already copied data, return that. If we
1371 * need to drop the opipe lock, it must be ordered
1372 * last to avoid deadlocks.
1373 */
1374 if ((flags & SPLICE_F_NONBLOCK) || !ipipe_first) {
1375 if (!ret)
1376 ret = -EAGAIN;
1377 break;
1378 }
1379 if (signal_pending(current)) {
1380 if (!ret)
1381 ret = -ERESTARTSYS;
1382 break;
1383 }
1384 if (do_wakeup) {
1385 smp_mb();
1386 if (waitqueue_active(&opipe->wait))
1387 wake_up_interruptible(&opipe->wait);
1388 kill_fasync(&opipe->fasync_readers, SIGIO, POLL_IN);
1389 do_wakeup = 0;
1390 }
1391
1392 opipe->waiting_writers++;
1393 pipe_wait(opipe);
1394 opipe->waiting_writers--;
1395 continue;
1396 }
1397
1398 /*
1399 * No input buffers, do the usual checks for available
1400 * writers and blocking and wait if necessary
1401 */
1402 if (!ipipe->writers)
1403 break;
1404 if (!ipipe->waiting_writers) {
1405 if (ret)
1406 break;
1407 }
1408 /*
1409 * pipe_wait() drops the ipipe mutex. To avoid deadlocks
1410 * with another process, we can only safely do that if
1411 * the ipipe lock is ordered last.
1412 */
1413 if ((flags & SPLICE_F_NONBLOCK) || ipipe_first) {
1414 if (!ret)
1415 ret = -EAGAIN;
1416 break;
1417 }
1418 if (signal_pending(current)) {
1419 if (!ret)
1420 ret = -ERESTARTSYS;
1421 break;
1422 }
1423
1424 if (waitqueue_active(&ipipe->wait))
1425 wake_up_interruptible_sync(&ipipe->wait);
1426 kill_fasync(&ipipe->fasync_writers, SIGIO, POLL_OUT);
1427
1428 pipe_wait(ipipe);
1429 }
1430
1431 mutex_unlock(&ipipe->inode->i_mutex);
1432 mutex_unlock(&opipe->inode->i_mutex);
1433
1434 if (do_wakeup) {
1435 smp_mb();
1436 if (waitqueue_active(&opipe->wait))
1437 wake_up_interruptible(&opipe->wait);
1438 kill_fasync(&opipe->fasync_readers, SIGIO, POLL_IN);
1439 }
1440
1441 return ret;
1442}
1443
1444/*
1445 * This is a tee(1) implementation that works on pipes. It doesn't copy
1446 * any data, it simply references the 'in' pages on the 'out' pipe.
1447 * The 'flags' used are the SPLICE_F_* variants, currently the only
1448 * applicable one is SPLICE_F_NONBLOCK.
1449 */
1450static long do_tee(struct file *in, struct file *out, size_t len,
1451 unsigned int flags)
1452{
1453 struct pipe_inode_info *ipipe = in->f_dentry->d_inode->i_pipe;
1454 struct pipe_inode_info *opipe = out->f_dentry->d_inode->i_pipe;
1455
1456 /*
1457 * Link ipipe to the two output pipes, consuming as we go along.
1458 */
1459 if (ipipe && opipe)
1460 return link_pipe(ipipe, opipe, len, flags);
1461
1462 return -EINVAL;
1463}
1464
1465asmlinkage long sys_tee(int fdin, int fdout, size_t len, unsigned int flags)
1466{
1467 struct file *in;
1468 int error, fput_in;
1469
1470 if (unlikely(!len))
1471 return 0;
1472
1473 error = -EBADF;
1474 in = fget_light(fdin, &fput_in);
1475 if (in) {
1476 if (in->f_mode & FMODE_READ) {
1477 int fput_out;
1478 struct file *out = fget_light(fdout, &fput_out);
1479
1480 if (out) {
1481 if (out->f_mode & FMODE_WRITE)
1482 error = do_tee(in, out, len, flags);
1483 fput_light(out, fput_out);
1484 }
1485 }
1486 fput_light(in, fput_in);
1487 }
1488
1489 return error;
1490}
generated by cgit v1.2.2 (git 2.25.0) at 2024-11-10 11:18:19 -0500