aboutsummaryrefslogtreecommitdiffstats
path: root/fs/xfs/xfs_buf.c
diff options
context:
space:
mode:
authorChristoph Hellwig <hch@infradead.org>2011-08-12 17:21:35 -0400
committerAlex Elder <aelder@sgi.com>2011-08-12 17:21:35 -0400
commitc59d87c460767bc35dafd490139d3cfe78fb8da4 (patch)
tree2aad8261f86488e501d9645bd35d1398906da46d /fs/xfs/xfs_buf.c
parent06f8e2d6754dc631732415b741b5aa58a0f7133f (diff)
xfs: remove subdirectories
Use the move from Linux 2.6 to Linux 3.x as an excuse to kill the annoying subdirectories in the XFS source code. Besides the large amount of file rename the only changes are to the Makefile, a few files including headers with the subdirectory prefix, and the binary sysctl compat code that includes a header under fs/xfs/ from kernel/. Signed-off-by: Christoph Hellwig <hch@lst.de> Signed-off-by: Alex Elder <aelder@sgi.com>
Diffstat (limited to 'fs/xfs/xfs_buf.c')
-rw-r--r--fs/xfs/xfs_buf.c1876
1 files changed, 1876 insertions, 0 deletions
diff --git a/fs/xfs/xfs_buf.c b/fs/xfs/xfs_buf.c
new file mode 100644
index 000000000000..c57836dc778f
--- /dev/null
+++ b/fs/xfs/xfs_buf.c
@@ -0,0 +1,1876 @@
1/*
2 * Copyright (c) 2000-2006 Silicon Graphics, Inc.
3 * All Rights Reserved.
4 *
5 * This program is free software; you can redistribute it and/or
6 * modify it under the terms of the GNU General Public License as
7 * published by the Free Software Foundation.
8 *
9 * This program is distributed in the hope that it would be useful,
10 * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 * GNU General Public License for more details.
13 *
14 * You should have received a copy of the GNU General Public License
15 * along with this program; if not, write the Free Software Foundation,
16 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
17 */
18#include "xfs.h"
19#include <linux/stddef.h>
20#include <linux/errno.h>
21#include <linux/gfp.h>
22#include <linux/pagemap.h>
23#include <linux/init.h>
24#include <linux/vmalloc.h>
25#include <linux/bio.h>
26#include <linux/sysctl.h>
27#include <linux/proc_fs.h>
28#include <linux/workqueue.h>
29#include <linux/percpu.h>
30#include <linux/blkdev.h>
31#include <linux/hash.h>
32#include <linux/kthread.h>
33#include <linux/migrate.h>
34#include <linux/backing-dev.h>
35#include <linux/freezer.h>
36
37#include "xfs_sb.h"
38#include "xfs_inum.h"
39#include "xfs_log.h"
40#include "xfs_ag.h"
41#include "xfs_mount.h"
42#include "xfs_trace.h"
43
44static kmem_zone_t *xfs_buf_zone;
45STATIC int xfsbufd(void *);
46STATIC void xfs_buf_delwri_queue(xfs_buf_t *, int);
47
48static struct workqueue_struct *xfslogd_workqueue;
49struct workqueue_struct *xfsdatad_workqueue;
50struct workqueue_struct *xfsconvertd_workqueue;
51
52#ifdef XFS_BUF_LOCK_TRACKING
53# define XB_SET_OWNER(bp) ((bp)->b_last_holder = current->pid)
54# define XB_CLEAR_OWNER(bp) ((bp)->b_last_holder = -1)
55# define XB_GET_OWNER(bp) ((bp)->b_last_holder)
56#else
57# define XB_SET_OWNER(bp) do { } while (0)
58# define XB_CLEAR_OWNER(bp) do { } while (0)
59# define XB_GET_OWNER(bp) do { } while (0)
60#endif
61
62#define xb_to_gfp(flags) \
63 ((((flags) & XBF_READ_AHEAD) ? __GFP_NORETRY : \
64 ((flags) & XBF_DONT_BLOCK) ? GFP_NOFS : GFP_KERNEL) | __GFP_NOWARN)
65
66#define xb_to_km(flags) \
67 (((flags) & XBF_DONT_BLOCK) ? KM_NOFS : KM_SLEEP)
68
69#define xfs_buf_allocate(flags) \
70 kmem_zone_alloc(xfs_buf_zone, xb_to_km(flags))
71#define xfs_buf_deallocate(bp) \
72 kmem_zone_free(xfs_buf_zone, (bp));
73
74static inline int
75xfs_buf_is_vmapped(
76 struct xfs_buf *bp)
77{
78 /*
79 * Return true if the buffer is vmapped.
80 *
81 * The XBF_MAPPED flag is set if the buffer should be mapped, but the
82 * code is clever enough to know it doesn't have to map a single page,
83 * so the check has to be both for XBF_MAPPED and bp->b_page_count > 1.
84 */
85 return (bp->b_flags & XBF_MAPPED) && bp->b_page_count > 1;
86}
87
88static inline int
89xfs_buf_vmap_len(
90 struct xfs_buf *bp)
91{
92 return (bp->b_page_count * PAGE_SIZE) - bp->b_offset;
93}
94
95/*
96 * xfs_buf_lru_add - add a buffer to the LRU.
97 *
98 * The LRU takes a new reference to the buffer so that it will only be freed
99 * once the shrinker takes the buffer off the LRU.
100 */
101STATIC void
102xfs_buf_lru_add(
103 struct xfs_buf *bp)
104{
105 struct xfs_buftarg *btp = bp->b_target;
106
107 spin_lock(&btp->bt_lru_lock);
108 if (list_empty(&bp->b_lru)) {
109 atomic_inc(&bp->b_hold);
110 list_add_tail(&bp->b_lru, &btp->bt_lru);
111 btp->bt_lru_nr++;
112 }
113 spin_unlock(&btp->bt_lru_lock);
114}
115
116/*
117 * xfs_buf_lru_del - remove a buffer from the LRU
118 *
119 * The unlocked check is safe here because it only occurs when there are not
120 * b_lru_ref counts left on the inode under the pag->pag_buf_lock. it is there
121 * to optimise the shrinker removing the buffer from the LRU and calling
122 * xfs_buf_free(). i.e. it removes an unnecessary round trip on the
123 * bt_lru_lock.
124 */
125STATIC void
126xfs_buf_lru_del(
127 struct xfs_buf *bp)
128{
129 struct xfs_buftarg *btp = bp->b_target;
130
131 if (list_empty(&bp->b_lru))
132 return;
133
134 spin_lock(&btp->bt_lru_lock);
135 if (!list_empty(&bp->b_lru)) {
136 list_del_init(&bp->b_lru);
137 btp->bt_lru_nr--;
138 }
139 spin_unlock(&btp->bt_lru_lock);
140}
141
142/*
143 * When we mark a buffer stale, we remove the buffer from the LRU and clear the
144 * b_lru_ref count so that the buffer is freed immediately when the buffer
145 * reference count falls to zero. If the buffer is already on the LRU, we need
146 * to remove the reference that LRU holds on the buffer.
147 *
148 * This prevents build-up of stale buffers on the LRU.
149 */
150void
151xfs_buf_stale(
152 struct xfs_buf *bp)
153{
154 bp->b_flags |= XBF_STALE;
155 atomic_set(&(bp)->b_lru_ref, 0);
156 if (!list_empty(&bp->b_lru)) {
157 struct xfs_buftarg *btp = bp->b_target;
158
159 spin_lock(&btp->bt_lru_lock);
160 if (!list_empty(&bp->b_lru)) {
161 list_del_init(&bp->b_lru);
162 btp->bt_lru_nr--;
163 atomic_dec(&bp->b_hold);
164 }
165 spin_unlock(&btp->bt_lru_lock);
166 }
167 ASSERT(atomic_read(&bp->b_hold) >= 1);
168}
169
170STATIC void
171_xfs_buf_initialize(
172 xfs_buf_t *bp,
173 xfs_buftarg_t *target,
174 xfs_off_t range_base,
175 size_t range_length,
176 xfs_buf_flags_t flags)
177{
178 /*
179 * We don't want certain flags to appear in b_flags.
180 */
181 flags &= ~(XBF_LOCK|XBF_MAPPED|XBF_DONT_BLOCK|XBF_READ_AHEAD);
182
183 memset(bp, 0, sizeof(xfs_buf_t));
184 atomic_set(&bp->b_hold, 1);
185 atomic_set(&bp->b_lru_ref, 1);
186 init_completion(&bp->b_iowait);
187 INIT_LIST_HEAD(&bp->b_lru);
188 INIT_LIST_HEAD(&bp->b_list);
189 RB_CLEAR_NODE(&bp->b_rbnode);
190 sema_init(&bp->b_sema, 0); /* held, no waiters */
191 XB_SET_OWNER(bp);
192 bp->b_target = target;
193 bp->b_file_offset = range_base;
194 /*
195 * Set buffer_length and count_desired to the same value initially.
196 * I/O routines should use count_desired, which will be the same in
197 * most cases but may be reset (e.g. XFS recovery).
198 */
199 bp->b_buffer_length = bp->b_count_desired = range_length;
200 bp->b_flags = flags;
201 bp->b_bn = XFS_BUF_DADDR_NULL;
202 atomic_set(&bp->b_pin_count, 0);
203 init_waitqueue_head(&bp->b_waiters);
204
205 XFS_STATS_INC(xb_create);
206
207 trace_xfs_buf_init(bp, _RET_IP_);
208}
209
210/*
211 * Allocate a page array capable of holding a specified number
212 * of pages, and point the page buf at it.
213 */
214STATIC int
215_xfs_buf_get_pages(
216 xfs_buf_t *bp,
217 int page_count,
218 xfs_buf_flags_t flags)
219{
220 /* Make sure that we have a page list */
221 if (bp->b_pages == NULL) {
222 bp->b_offset = xfs_buf_poff(bp->b_file_offset);
223 bp->b_page_count = page_count;
224 if (page_count <= XB_PAGES) {
225 bp->b_pages = bp->b_page_array;
226 } else {
227 bp->b_pages = kmem_alloc(sizeof(struct page *) *
228 page_count, xb_to_km(flags));
229 if (bp->b_pages == NULL)
230 return -ENOMEM;
231 }
232 memset(bp->b_pages, 0, sizeof(struct page *) * page_count);
233 }
234 return 0;
235}
236
237/*
238 * Frees b_pages if it was allocated.
239 */
240STATIC void
241_xfs_buf_free_pages(
242 xfs_buf_t *bp)
243{
244 if (bp->b_pages != bp->b_page_array) {
245 kmem_free(bp->b_pages);
246 bp->b_pages = NULL;
247 }
248}
249
250/*
251 * Releases the specified buffer.
252 *
253 * The modification state of any associated pages is left unchanged.
254 * The buffer most not be on any hash - use xfs_buf_rele instead for
255 * hashed and refcounted buffers
256 */
257void
258xfs_buf_free(
259 xfs_buf_t *bp)
260{
261 trace_xfs_buf_free(bp, _RET_IP_);
262
263 ASSERT(list_empty(&bp->b_lru));
264
265 if (bp->b_flags & _XBF_PAGES) {
266 uint i;
267
268 if (xfs_buf_is_vmapped(bp))
269 vm_unmap_ram(bp->b_addr - bp->b_offset,
270 bp->b_page_count);
271
272 for (i = 0; i < bp->b_page_count; i++) {
273 struct page *page = bp->b_pages[i];
274
275 __free_page(page);
276 }
277 } else if (bp->b_flags & _XBF_KMEM)
278 kmem_free(bp->b_addr);
279 _xfs_buf_free_pages(bp);
280 xfs_buf_deallocate(bp);
281}
282
283/*
284 * Allocates all the pages for buffer in question and builds it's page list.
285 */
286STATIC int
287xfs_buf_allocate_memory(
288 xfs_buf_t *bp,
289 uint flags)
290{
291 size_t size = bp->b_count_desired;
292 size_t nbytes, offset;
293 gfp_t gfp_mask = xb_to_gfp(flags);
294 unsigned short page_count, i;
295 xfs_off_t end;
296 int error;
297
298 /*
299 * for buffers that are contained within a single page, just allocate
300 * the memory from the heap - there's no need for the complexity of
301 * page arrays to keep allocation down to order 0.
302 */
303 if (bp->b_buffer_length < PAGE_SIZE) {
304 bp->b_addr = kmem_alloc(bp->b_buffer_length, xb_to_km(flags));
305 if (!bp->b_addr) {
306 /* low memory - use alloc_page loop instead */
307 goto use_alloc_page;
308 }
309
310 if (((unsigned long)(bp->b_addr + bp->b_buffer_length - 1) &
311 PAGE_MASK) !=
312 ((unsigned long)bp->b_addr & PAGE_MASK)) {
313 /* b_addr spans two pages - use alloc_page instead */
314 kmem_free(bp->b_addr);
315 bp->b_addr = NULL;
316 goto use_alloc_page;
317 }
318 bp->b_offset = offset_in_page(bp->b_addr);
319 bp->b_pages = bp->b_page_array;
320 bp->b_pages[0] = virt_to_page(bp->b_addr);
321 bp->b_page_count = 1;
322 bp->b_flags |= XBF_MAPPED | _XBF_KMEM;
323 return 0;
324 }
325
326use_alloc_page:
327 end = bp->b_file_offset + bp->b_buffer_length;
328 page_count = xfs_buf_btoc(end) - xfs_buf_btoct(bp->b_file_offset);
329 error = _xfs_buf_get_pages(bp, page_count, flags);
330 if (unlikely(error))
331 return error;
332
333 offset = bp->b_offset;
334 bp->b_flags |= _XBF_PAGES;
335
336 for (i = 0; i < bp->b_page_count; i++) {
337 struct page *page;
338 uint retries = 0;
339retry:
340 page = alloc_page(gfp_mask);
341 if (unlikely(page == NULL)) {
342 if (flags & XBF_READ_AHEAD) {
343 bp->b_page_count = i;
344 error = ENOMEM;
345 goto out_free_pages;
346 }
347
348 /*
349 * This could deadlock.
350 *
351 * But until all the XFS lowlevel code is revamped to
352 * handle buffer allocation failures we can't do much.
353 */
354 if (!(++retries % 100))
355 xfs_err(NULL,
356 "possible memory allocation deadlock in %s (mode:0x%x)",
357 __func__, gfp_mask);
358
359 XFS_STATS_INC(xb_page_retries);
360 congestion_wait(BLK_RW_ASYNC, HZ/50);
361 goto retry;
362 }
363
364 XFS_STATS_INC(xb_page_found);
365
366 nbytes = min_t(size_t, size, PAGE_SIZE - offset);
367 size -= nbytes;
368 bp->b_pages[i] = page;
369 offset = 0;
370 }
371 return 0;
372
373out_free_pages:
374 for (i = 0; i < bp->b_page_count; i++)
375 __free_page(bp->b_pages[i]);
376 return error;
377}
378
379/*
380 * Map buffer into kernel address-space if necessary.
381 */
382STATIC int
383_xfs_buf_map_pages(
384 xfs_buf_t *bp,
385 uint flags)
386{
387 ASSERT(bp->b_flags & _XBF_PAGES);
388 if (bp->b_page_count == 1) {
389 /* A single page buffer is always mappable */
390 bp->b_addr = page_address(bp->b_pages[0]) + bp->b_offset;
391 bp->b_flags |= XBF_MAPPED;
392 } else if (flags & XBF_MAPPED) {
393 int retried = 0;
394
395 do {
396 bp->b_addr = vm_map_ram(bp->b_pages, bp->b_page_count,
397 -1, PAGE_KERNEL);
398 if (bp->b_addr)
399 break;
400 vm_unmap_aliases();
401 } while (retried++ <= 1);
402
403 if (!bp->b_addr)
404 return -ENOMEM;
405 bp->b_addr += bp->b_offset;
406 bp->b_flags |= XBF_MAPPED;
407 }
408
409 return 0;
410}
411
412/*
413 * Finding and Reading Buffers
414 */
415
416/*
417 * Look up, and creates if absent, a lockable buffer for
418 * a given range of an inode. The buffer is returned
419 * locked. If other overlapping buffers exist, they are
420 * released before the new buffer is created and locked,
421 * which may imply that this call will block until those buffers
422 * are unlocked. No I/O is implied by this call.
423 */
424xfs_buf_t *
425_xfs_buf_find(
426 xfs_buftarg_t *btp, /* block device target */
427 xfs_off_t ioff, /* starting offset of range */
428 size_t isize, /* length of range */
429 xfs_buf_flags_t flags,
430 xfs_buf_t *new_bp)
431{
432 xfs_off_t range_base;
433 size_t range_length;
434 struct xfs_perag *pag;
435 struct rb_node **rbp;
436 struct rb_node *parent;
437 xfs_buf_t *bp;
438
439 range_base = (ioff << BBSHIFT);
440 range_length = (isize << BBSHIFT);
441
442 /* Check for IOs smaller than the sector size / not sector aligned */
443 ASSERT(!(range_length < (1 << btp->bt_sshift)));
444 ASSERT(!(range_base & (xfs_off_t)btp->bt_smask));
445
446 /* get tree root */
447 pag = xfs_perag_get(btp->bt_mount,
448 xfs_daddr_to_agno(btp->bt_mount, ioff));
449
450 /* walk tree */
451 spin_lock(&pag->pag_buf_lock);
452 rbp = &pag->pag_buf_tree.rb_node;
453 parent = NULL;
454 bp = NULL;
455 while (*rbp) {
456 parent = *rbp;
457 bp = rb_entry(parent, struct xfs_buf, b_rbnode);
458
459 if (range_base < bp->b_file_offset)
460 rbp = &(*rbp)->rb_left;
461 else if (range_base > bp->b_file_offset)
462 rbp = &(*rbp)->rb_right;
463 else {
464 /*
465 * found a block offset match. If the range doesn't
466 * match, the only way this is allowed is if the buffer
467 * in the cache is stale and the transaction that made
468 * it stale has not yet committed. i.e. we are
469 * reallocating a busy extent. Skip this buffer and
470 * continue searching to the right for an exact match.
471 */
472 if (bp->b_buffer_length != range_length) {
473 ASSERT(bp->b_flags & XBF_STALE);
474 rbp = &(*rbp)->rb_right;
475 continue;
476 }
477 atomic_inc(&bp->b_hold);
478 goto found;
479 }
480 }
481
482 /* No match found */
483 if (new_bp) {
484 _xfs_buf_initialize(new_bp, btp, range_base,
485 range_length, flags);
486 rb_link_node(&new_bp->b_rbnode, parent, rbp);
487 rb_insert_color(&new_bp->b_rbnode, &pag->pag_buf_tree);
488 /* the buffer keeps the perag reference until it is freed */
489 new_bp->b_pag = pag;
490 spin_unlock(&pag->pag_buf_lock);
491 } else {
492 XFS_STATS_INC(xb_miss_locked);
493 spin_unlock(&pag->pag_buf_lock);
494 xfs_perag_put(pag);
495 }
496 return new_bp;
497
498found:
499 spin_unlock(&pag->pag_buf_lock);
500 xfs_perag_put(pag);
501
502 if (!xfs_buf_trylock(bp)) {
503 if (flags & XBF_TRYLOCK) {
504 xfs_buf_rele(bp);
505 XFS_STATS_INC(xb_busy_locked);
506 return NULL;
507 }
508 xfs_buf_lock(bp);
509 XFS_STATS_INC(xb_get_locked_waited);
510 }
511
512 /*
513 * if the buffer is stale, clear all the external state associated with
514 * it. We need to keep flags such as how we allocated the buffer memory
515 * intact here.
516 */
517 if (bp->b_flags & XBF_STALE) {
518 ASSERT((bp->b_flags & _XBF_DELWRI_Q) == 0);
519 bp->b_flags &= XBF_MAPPED | _XBF_KMEM | _XBF_PAGES;
520 }
521
522 trace_xfs_buf_find(bp, flags, _RET_IP_);
523 XFS_STATS_INC(xb_get_locked);
524 return bp;
525}
526
527/*
528 * Assembles a buffer covering the specified range.
529 * Storage in memory for all portions of the buffer will be allocated,
530 * although backing storage may not be.
531 */
532xfs_buf_t *
533xfs_buf_get(
534 xfs_buftarg_t *target,/* target for buffer */
535 xfs_off_t ioff, /* starting offset of range */
536 size_t isize, /* length of range */
537 xfs_buf_flags_t flags)
538{
539 xfs_buf_t *bp, *new_bp;
540 int error = 0;
541
542 new_bp = xfs_buf_allocate(flags);
543 if (unlikely(!new_bp))
544 return NULL;
545
546 bp = _xfs_buf_find(target, ioff, isize, flags, new_bp);
547 if (bp == new_bp) {
548 error = xfs_buf_allocate_memory(bp, flags);
549 if (error)
550 goto no_buffer;
551 } else {
552 xfs_buf_deallocate(new_bp);
553 if (unlikely(bp == NULL))
554 return NULL;
555 }
556
557 if (!(bp->b_flags & XBF_MAPPED)) {
558 error = _xfs_buf_map_pages(bp, flags);
559 if (unlikely(error)) {
560 xfs_warn(target->bt_mount,
561 "%s: failed to map pages\n", __func__);
562 goto no_buffer;
563 }
564 }
565
566 XFS_STATS_INC(xb_get);
567
568 /*
569 * Always fill in the block number now, the mapped cases can do
570 * their own overlay of this later.
571 */
572 bp->b_bn = ioff;
573 bp->b_count_desired = bp->b_buffer_length;
574
575 trace_xfs_buf_get(bp, flags, _RET_IP_);
576 return bp;
577
578 no_buffer:
579 if (flags & (XBF_LOCK | XBF_TRYLOCK))
580 xfs_buf_unlock(bp);
581 xfs_buf_rele(bp);
582 return NULL;
583}
584
585STATIC int
586_xfs_buf_read(
587 xfs_buf_t *bp,
588 xfs_buf_flags_t flags)
589{
590 int status;
591
592 ASSERT(!(flags & (XBF_DELWRI|XBF_WRITE)));
593 ASSERT(bp->b_bn != XFS_BUF_DADDR_NULL);
594
595 bp->b_flags &= ~(XBF_WRITE | XBF_ASYNC | XBF_DELWRI | XBF_READ_AHEAD);
596 bp->b_flags |= flags & (XBF_READ | XBF_ASYNC | XBF_READ_AHEAD);
597
598 status = xfs_buf_iorequest(bp);
599 if (status || bp->b_error || (flags & XBF_ASYNC))
600 return status;
601 return xfs_buf_iowait(bp);
602}
603
604xfs_buf_t *
605xfs_buf_read(
606 xfs_buftarg_t *target,
607 xfs_off_t ioff,
608 size_t isize,
609 xfs_buf_flags_t flags)
610{
611 xfs_buf_t *bp;
612
613 flags |= XBF_READ;
614
615 bp = xfs_buf_get(target, ioff, isize, flags);
616 if (bp) {
617 trace_xfs_buf_read(bp, flags, _RET_IP_);
618
619 if (!XFS_BUF_ISDONE(bp)) {
620 XFS_STATS_INC(xb_get_read);
621 _xfs_buf_read(bp, flags);
622 } else if (flags & XBF_ASYNC) {
623 /*
624 * Read ahead call which is already satisfied,
625 * drop the buffer
626 */
627 goto no_buffer;
628 } else {
629 /* We do not want read in the flags */
630 bp->b_flags &= ~XBF_READ;
631 }
632 }
633
634 return bp;
635
636 no_buffer:
637 if (flags & (XBF_LOCK | XBF_TRYLOCK))
638 xfs_buf_unlock(bp);
639 xfs_buf_rele(bp);
640 return NULL;
641}
642
643/*
644 * If we are not low on memory then do the readahead in a deadlock
645 * safe manner.
646 */
647void
648xfs_buf_readahead(
649 xfs_buftarg_t *target,
650 xfs_off_t ioff,
651 size_t isize)
652{
653 if (bdi_read_congested(target->bt_bdi))
654 return;
655
656 xfs_buf_read(target, ioff, isize,
657 XBF_TRYLOCK|XBF_ASYNC|XBF_READ_AHEAD|XBF_DONT_BLOCK);
658}
659
660/*
661 * Read an uncached buffer from disk. Allocates and returns a locked
662 * buffer containing the disk contents or nothing.
663 */
664struct xfs_buf *
665xfs_buf_read_uncached(
666 struct xfs_mount *mp,
667 struct xfs_buftarg *target,
668 xfs_daddr_t daddr,
669 size_t length,
670 int flags)
671{
672 xfs_buf_t *bp;
673 int error;
674
675 bp = xfs_buf_get_uncached(target, length, flags);
676 if (!bp)
677 return NULL;
678
679 /* set up the buffer for a read IO */
680 XFS_BUF_SET_ADDR(bp, daddr);
681 XFS_BUF_READ(bp);
682
683 xfsbdstrat(mp, bp);
684 error = xfs_buf_iowait(bp);
685 if (error || bp->b_error) {
686 xfs_buf_relse(bp);
687 return NULL;
688 }
689 return bp;
690}
691
692xfs_buf_t *
693xfs_buf_get_empty(
694 size_t len,
695 xfs_buftarg_t *target)
696{
697 xfs_buf_t *bp;
698
699 bp = xfs_buf_allocate(0);
700 if (bp)
701 _xfs_buf_initialize(bp, target, 0, len, 0);
702 return bp;
703}
704
705/*
706 * Return a buffer allocated as an empty buffer and associated to external
707 * memory via xfs_buf_associate_memory() back to it's empty state.
708 */
709void
710xfs_buf_set_empty(
711 struct xfs_buf *bp,
712 size_t len)
713{
714 if (bp->b_pages)
715 _xfs_buf_free_pages(bp);
716
717 bp->b_pages = NULL;
718 bp->b_page_count = 0;
719 bp->b_addr = NULL;
720 bp->b_file_offset = 0;
721 bp->b_buffer_length = bp->b_count_desired = len;
722 bp->b_bn = XFS_BUF_DADDR_NULL;
723 bp->b_flags &= ~XBF_MAPPED;
724}
725
726static inline struct page *
727mem_to_page(
728 void *addr)
729{
730 if ((!is_vmalloc_addr(addr))) {
731 return virt_to_page(addr);
732 } else {
733 return vmalloc_to_page(addr);
734 }
735}
736
737int
738xfs_buf_associate_memory(
739 xfs_buf_t *bp,
740 void *mem,
741 size_t len)
742{
743 int rval;
744 int i = 0;
745 unsigned long pageaddr;
746 unsigned long offset;
747 size_t buflen;
748 int page_count;
749
750 pageaddr = (unsigned long)mem & PAGE_MASK;
751 offset = (unsigned long)mem - pageaddr;
752 buflen = PAGE_ALIGN(len + offset);
753 page_count = buflen >> PAGE_SHIFT;
754
755 /* Free any previous set of page pointers */
756 if (bp->b_pages)
757 _xfs_buf_free_pages(bp);
758
759 bp->b_pages = NULL;
760 bp->b_addr = mem;
761
762 rval = _xfs_buf_get_pages(bp, page_count, XBF_DONT_BLOCK);
763 if (rval)
764 return rval;
765
766 bp->b_offset = offset;
767
768 for (i = 0; i < bp->b_page_count; i++) {
769 bp->b_pages[i] = mem_to_page((void *)pageaddr);
770 pageaddr += PAGE_SIZE;
771 }
772
773 bp->b_count_desired = len;
774 bp->b_buffer_length = buflen;
775 bp->b_flags |= XBF_MAPPED;
776
777 return 0;
778}
779
780xfs_buf_t *
781xfs_buf_get_uncached(
782 struct xfs_buftarg *target,
783 size_t len,
784 int flags)
785{
786 unsigned long page_count = PAGE_ALIGN(len) >> PAGE_SHIFT;
787 int error, i;
788 xfs_buf_t *bp;
789
790 bp = xfs_buf_allocate(0);
791 if (unlikely(bp == NULL))
792 goto fail;
793 _xfs_buf_initialize(bp, target, 0, len, 0);
794
795 error = _xfs_buf_get_pages(bp, page_count, 0);
796 if (error)
797 goto fail_free_buf;
798
799 for (i = 0; i < page_count; i++) {
800 bp->b_pages[i] = alloc_page(xb_to_gfp(flags));
801 if (!bp->b_pages[i])
802 goto fail_free_mem;
803 }
804 bp->b_flags |= _XBF_PAGES;
805
806 error = _xfs_buf_map_pages(bp, XBF_MAPPED);
807 if (unlikely(error)) {
808 xfs_warn(target->bt_mount,
809 "%s: failed to map pages\n", __func__);
810 goto fail_free_mem;
811 }
812
813 trace_xfs_buf_get_uncached(bp, _RET_IP_);
814 return bp;
815
816 fail_free_mem:
817 while (--i >= 0)
818 __free_page(bp->b_pages[i]);
819 _xfs_buf_free_pages(bp);
820 fail_free_buf:
821 xfs_buf_deallocate(bp);
822 fail:
823 return NULL;
824}
825
826/*
827 * Increment reference count on buffer, to hold the buffer concurrently
828 * with another thread which may release (free) the buffer asynchronously.
829 * Must hold the buffer already to call this function.
830 */
831void
832xfs_buf_hold(
833 xfs_buf_t *bp)
834{
835 trace_xfs_buf_hold(bp, _RET_IP_);
836 atomic_inc(&bp->b_hold);
837}
838
839/*
840 * Releases a hold on the specified buffer. If the
841 * the hold count is 1, calls xfs_buf_free.
842 */
843void
844xfs_buf_rele(
845 xfs_buf_t *bp)
846{
847 struct xfs_perag *pag = bp->b_pag;
848
849 trace_xfs_buf_rele(bp, _RET_IP_);
850
851 if (!pag) {
852 ASSERT(list_empty(&bp->b_lru));
853 ASSERT(RB_EMPTY_NODE(&bp->b_rbnode));
854 if (atomic_dec_and_test(&bp->b_hold))
855 xfs_buf_free(bp);
856 return;
857 }
858
859 ASSERT(!RB_EMPTY_NODE(&bp->b_rbnode));
860
861 ASSERT(atomic_read(&bp->b_hold) > 0);
862 if (atomic_dec_and_lock(&bp->b_hold, &pag->pag_buf_lock)) {
863 if (!(bp->b_flags & XBF_STALE) &&
864 atomic_read(&bp->b_lru_ref)) {
865 xfs_buf_lru_add(bp);
866 spin_unlock(&pag->pag_buf_lock);
867 } else {
868 xfs_buf_lru_del(bp);
869 ASSERT(!(bp->b_flags & (XBF_DELWRI|_XBF_DELWRI_Q)));
870 rb_erase(&bp->b_rbnode, &pag->pag_buf_tree);
871 spin_unlock(&pag->pag_buf_lock);
872 xfs_perag_put(pag);
873 xfs_buf_free(bp);
874 }
875 }
876}
877
878
879/*
880 * Lock a buffer object, if it is not already locked.
881 *
882 * If we come across a stale, pinned, locked buffer, we know that we are
883 * being asked to lock a buffer that has been reallocated. Because it is
884 * pinned, we know that the log has not been pushed to disk and hence it
885 * will still be locked. Rather than continuing to have trylock attempts
886 * fail until someone else pushes the log, push it ourselves before
887 * returning. This means that the xfsaild will not get stuck trying
888 * to push on stale inode buffers.
889 */
890int
891xfs_buf_trylock(
892 struct xfs_buf *bp)
893{
894 int locked;
895
896 locked = down_trylock(&bp->b_sema) == 0;
897 if (locked)
898 XB_SET_OWNER(bp);
899 else if (atomic_read(&bp->b_pin_count) && (bp->b_flags & XBF_STALE))
900 xfs_log_force(bp->b_target->bt_mount, 0);
901
902 trace_xfs_buf_trylock(bp, _RET_IP_);
903 return locked;
904}
905
906/*
907 * Lock a buffer object.
908 *
909 * If we come across a stale, pinned, locked buffer, we know that we
910 * are being asked to lock a buffer that has been reallocated. Because
911 * it is pinned, we know that the log has not been pushed to disk and
912 * hence it will still be locked. Rather than sleeping until someone
913 * else pushes the log, push it ourselves before trying to get the lock.
914 */
915void
916xfs_buf_lock(
917 struct xfs_buf *bp)
918{
919 trace_xfs_buf_lock(bp, _RET_IP_);
920
921 if (atomic_read(&bp->b_pin_count) && (bp->b_flags & XBF_STALE))
922 xfs_log_force(bp->b_target->bt_mount, 0);
923 down(&bp->b_sema);
924 XB_SET_OWNER(bp);
925
926 trace_xfs_buf_lock_done(bp, _RET_IP_);
927}
928
929/*
930 * Releases the lock on the buffer object.
931 * If the buffer is marked delwri but is not queued, do so before we
932 * unlock the buffer as we need to set flags correctly. We also need to
933 * take a reference for the delwri queue because the unlocker is going to
934 * drop their's and they don't know we just queued it.
935 */
936void
937xfs_buf_unlock(
938 struct xfs_buf *bp)
939{
940 if ((bp->b_flags & (XBF_DELWRI|_XBF_DELWRI_Q)) == XBF_DELWRI) {
941 atomic_inc(&bp->b_hold);
942 bp->b_flags |= XBF_ASYNC;
943 xfs_buf_delwri_queue(bp, 0);
944 }
945
946 XB_CLEAR_OWNER(bp);
947 up(&bp->b_sema);
948
949 trace_xfs_buf_unlock(bp, _RET_IP_);
950}
951
952STATIC void
953xfs_buf_wait_unpin(
954 xfs_buf_t *bp)
955{
956 DECLARE_WAITQUEUE (wait, current);
957
958 if (atomic_read(&bp->b_pin_count) == 0)
959 return;
960
961 add_wait_queue(&bp->b_waiters, &wait);
962 for (;;) {
963 set_current_state(TASK_UNINTERRUPTIBLE);
964 if (atomic_read(&bp->b_pin_count) == 0)
965 break;
966 io_schedule();
967 }
968 remove_wait_queue(&bp->b_waiters, &wait);
969 set_current_state(TASK_RUNNING);
970}
971
972/*
973 * Buffer Utility Routines
974 */
975
976STATIC void
977xfs_buf_iodone_work(
978 struct work_struct *work)
979{
980 xfs_buf_t *bp =
981 container_of(work, xfs_buf_t, b_iodone_work);
982
983 if (bp->b_iodone)
984 (*(bp->b_iodone))(bp);
985 else if (bp->b_flags & XBF_ASYNC)
986 xfs_buf_relse(bp);
987}
988
989void
990xfs_buf_ioend(
991 xfs_buf_t *bp,
992 int schedule)
993{
994 trace_xfs_buf_iodone(bp, _RET_IP_);
995
996 bp->b_flags &= ~(XBF_READ | XBF_WRITE | XBF_READ_AHEAD);
997 if (bp->b_error == 0)
998 bp->b_flags |= XBF_DONE;
999
1000 if ((bp->b_iodone) || (bp->b_flags & XBF_ASYNC)) {
1001 if (schedule) {
1002 INIT_WORK(&bp->b_iodone_work, xfs_buf_iodone_work);
1003 queue_work(xfslogd_workqueue, &bp->b_iodone_work);
1004 } else {
1005 xfs_buf_iodone_work(&bp->b_iodone_work);
1006 }
1007 } else {
1008 complete(&bp->b_iowait);
1009 }
1010}
1011
1012void
1013xfs_buf_ioerror(
1014 xfs_buf_t *bp,
1015 int error)
1016{
1017 ASSERT(error >= 0 && error <= 0xffff);
1018 bp->b_error = (unsigned short)error;
1019 trace_xfs_buf_ioerror(bp, error, _RET_IP_);
1020}
1021
1022int
1023xfs_bwrite(
1024 struct xfs_mount *mp,
1025 struct xfs_buf *bp)
1026{
1027 int error;
1028
1029 bp->b_flags |= XBF_WRITE;
1030 bp->b_flags &= ~(XBF_ASYNC | XBF_READ);
1031
1032 xfs_buf_delwri_dequeue(bp);
1033 xfs_bdstrat_cb(bp);
1034
1035 error = xfs_buf_iowait(bp);
1036 if (error)
1037 xfs_force_shutdown(mp, SHUTDOWN_META_IO_ERROR);
1038 xfs_buf_relse(bp);
1039 return error;
1040}
1041
1042void
1043xfs_bdwrite(
1044 void *mp,
1045 struct xfs_buf *bp)
1046{
1047 trace_xfs_buf_bdwrite(bp, _RET_IP_);
1048
1049 bp->b_flags &= ~XBF_READ;
1050 bp->b_flags |= (XBF_DELWRI | XBF_ASYNC);
1051
1052 xfs_buf_delwri_queue(bp, 1);
1053}
1054
1055/*
1056 * Called when we want to stop a buffer from getting written or read.
1057 * We attach the EIO error, muck with its flags, and call xfs_buf_ioend
1058 * so that the proper iodone callbacks get called.
1059 */
1060STATIC int
1061xfs_bioerror(
1062 xfs_buf_t *bp)
1063{
1064#ifdef XFSERRORDEBUG
1065 ASSERT(XFS_BUF_ISREAD(bp) || bp->b_iodone);
1066#endif
1067
1068 /*
1069 * No need to wait until the buffer is unpinned, we aren't flushing it.
1070 */
1071 xfs_buf_ioerror(bp, EIO);
1072
1073 /*
1074 * We're calling xfs_buf_ioend, so delete XBF_DONE flag.
1075 */
1076 XFS_BUF_UNREAD(bp);
1077 XFS_BUF_UNDELAYWRITE(bp);
1078 XFS_BUF_UNDONE(bp);
1079 XFS_BUF_STALE(bp);
1080
1081 xfs_buf_ioend(bp, 0);
1082
1083 return EIO;
1084}
1085
1086/*
1087 * Same as xfs_bioerror, except that we are releasing the buffer
1088 * here ourselves, and avoiding the xfs_buf_ioend call.
1089 * This is meant for userdata errors; metadata bufs come with
1090 * iodone functions attached, so that we can track down errors.
1091 */
1092STATIC int
1093xfs_bioerror_relse(
1094 struct xfs_buf *bp)
1095{
1096 int64_t fl = bp->b_flags;
1097 /*
1098 * No need to wait until the buffer is unpinned.
1099 * We aren't flushing it.
1100 *
1101 * chunkhold expects B_DONE to be set, whether
1102 * we actually finish the I/O or not. We don't want to
1103 * change that interface.
1104 */
1105 XFS_BUF_UNREAD(bp);
1106 XFS_BUF_UNDELAYWRITE(bp);
1107 XFS_BUF_DONE(bp);
1108 XFS_BUF_STALE(bp);
1109 bp->b_iodone = NULL;
1110 if (!(fl & XBF_ASYNC)) {
1111 /*
1112 * Mark b_error and B_ERROR _both_.
1113 * Lot's of chunkcache code assumes that.
1114 * There's no reason to mark error for
1115 * ASYNC buffers.
1116 */
1117 xfs_buf_ioerror(bp, EIO);
1118 XFS_BUF_FINISH_IOWAIT(bp);
1119 } else {
1120 xfs_buf_relse(bp);
1121 }
1122
1123 return EIO;
1124}
1125
1126
1127/*
1128 * All xfs metadata buffers except log state machine buffers
1129 * get this attached as their b_bdstrat callback function.
1130 * This is so that we can catch a buffer
1131 * after prematurely unpinning it to forcibly shutdown the filesystem.
1132 */
1133int
1134xfs_bdstrat_cb(
1135 struct xfs_buf *bp)
1136{
1137 if (XFS_FORCED_SHUTDOWN(bp->b_target->bt_mount)) {
1138 trace_xfs_bdstrat_shut(bp, _RET_IP_);
1139 /*
1140 * Metadata write that didn't get logged but
1141 * written delayed anyway. These aren't associated
1142 * with a transaction, and can be ignored.
1143 */
1144 if (!bp->b_iodone && !XFS_BUF_ISREAD(bp))
1145 return xfs_bioerror_relse(bp);
1146 else
1147 return xfs_bioerror(bp);
1148 }
1149
1150 xfs_buf_iorequest(bp);
1151 return 0;
1152}
1153
1154/*
1155 * Wrapper around bdstrat so that we can stop data from going to disk in case
1156 * we are shutting down the filesystem. Typically user data goes thru this
1157 * path; one of the exceptions is the superblock.
1158 */
1159void
1160xfsbdstrat(
1161 struct xfs_mount *mp,
1162 struct xfs_buf *bp)
1163{
1164 if (XFS_FORCED_SHUTDOWN(mp)) {
1165 trace_xfs_bdstrat_shut(bp, _RET_IP_);
1166 xfs_bioerror_relse(bp);
1167 return;
1168 }
1169
1170 xfs_buf_iorequest(bp);
1171}
1172
1173STATIC void
1174_xfs_buf_ioend(
1175 xfs_buf_t *bp,
1176 int schedule)
1177{
1178 if (atomic_dec_and_test(&bp->b_io_remaining) == 1)
1179 xfs_buf_ioend(bp, schedule);
1180}
1181
1182STATIC void
1183xfs_buf_bio_end_io(
1184 struct bio *bio,
1185 int error)
1186{
1187 xfs_buf_t *bp = (xfs_buf_t *)bio->bi_private;
1188
1189 xfs_buf_ioerror(bp, -error);
1190
1191 if (!error && xfs_buf_is_vmapped(bp) && (bp->b_flags & XBF_READ))
1192 invalidate_kernel_vmap_range(bp->b_addr, xfs_buf_vmap_len(bp));
1193
1194 _xfs_buf_ioend(bp, 1);
1195 bio_put(bio);
1196}
1197
1198STATIC void
1199_xfs_buf_ioapply(
1200 xfs_buf_t *bp)
1201{
1202 int rw, map_i, total_nr_pages, nr_pages;
1203 struct bio *bio;
1204 int offset = bp->b_offset;
1205 int size = bp->b_count_desired;
1206 sector_t sector = bp->b_bn;
1207
1208 total_nr_pages = bp->b_page_count;
1209 map_i = 0;
1210
1211 if (bp->b_flags & XBF_WRITE) {
1212 if (bp->b_flags & XBF_SYNCIO)
1213 rw = WRITE_SYNC;
1214 else
1215 rw = WRITE;
1216 if (bp->b_flags & XBF_FUA)
1217 rw |= REQ_FUA;
1218 if (bp->b_flags & XBF_FLUSH)
1219 rw |= REQ_FLUSH;
1220 } else if (bp->b_flags & XBF_READ_AHEAD) {
1221 rw = READA;
1222 } else {
1223 rw = READ;
1224 }
1225
1226 /* we only use the buffer cache for meta-data */
1227 rw |= REQ_META;
1228
1229next_chunk:
1230 atomic_inc(&bp->b_io_remaining);
1231 nr_pages = BIO_MAX_SECTORS >> (PAGE_SHIFT - BBSHIFT);
1232 if (nr_pages > total_nr_pages)
1233 nr_pages = total_nr_pages;
1234
1235 bio = bio_alloc(GFP_NOIO, nr_pages);
1236 bio->bi_bdev = bp->b_target->bt_bdev;
1237 bio->bi_sector = sector;
1238 bio->bi_end_io = xfs_buf_bio_end_io;
1239 bio->bi_private = bp;
1240
1241
1242 for (; size && nr_pages; nr_pages--, map_i++) {
1243 int rbytes, nbytes = PAGE_SIZE - offset;
1244
1245 if (nbytes > size)
1246 nbytes = size;
1247
1248 rbytes = bio_add_page(bio, bp->b_pages[map_i], nbytes, offset);
1249 if (rbytes < nbytes)
1250 break;
1251
1252 offset = 0;
1253 sector += nbytes >> BBSHIFT;
1254 size -= nbytes;
1255 total_nr_pages--;
1256 }
1257
1258 if (likely(bio->bi_size)) {
1259 if (xfs_buf_is_vmapped(bp)) {
1260 flush_kernel_vmap_range(bp->b_addr,
1261 xfs_buf_vmap_len(bp));
1262 }
1263 submit_bio(rw, bio);
1264 if (size)
1265 goto next_chunk;
1266 } else {
1267 xfs_buf_ioerror(bp, EIO);
1268 bio_put(bio);
1269 }
1270}
1271
1272int
1273xfs_buf_iorequest(
1274 xfs_buf_t *bp)
1275{
1276 trace_xfs_buf_iorequest(bp, _RET_IP_);
1277
1278 if (bp->b_flags & XBF_DELWRI) {
1279 xfs_buf_delwri_queue(bp, 1);
1280 return 0;
1281 }
1282
1283 if (bp->b_flags & XBF_WRITE) {
1284 xfs_buf_wait_unpin(bp);
1285 }
1286
1287 xfs_buf_hold(bp);
1288
1289 /* Set the count to 1 initially, this will stop an I/O
1290 * completion callout which happens before we have started
1291 * all the I/O from calling xfs_buf_ioend too early.
1292 */
1293 atomic_set(&bp->b_io_remaining, 1);
1294 _xfs_buf_ioapply(bp);
1295 _xfs_buf_ioend(bp, 0);
1296
1297 xfs_buf_rele(bp);
1298 return 0;
1299}
1300
1301/*
1302 * Waits for I/O to complete on the buffer supplied.
1303 * It returns immediately if no I/O is pending.
1304 * It returns the I/O error code, if any, or 0 if there was no error.
1305 */
1306int
1307xfs_buf_iowait(
1308 xfs_buf_t *bp)
1309{
1310 trace_xfs_buf_iowait(bp, _RET_IP_);
1311
1312 wait_for_completion(&bp->b_iowait);
1313
1314 trace_xfs_buf_iowait_done(bp, _RET_IP_);
1315 return bp->b_error;
1316}
1317
1318xfs_caddr_t
1319xfs_buf_offset(
1320 xfs_buf_t *bp,
1321 size_t offset)
1322{
1323 struct page *page;
1324
1325 if (bp->b_flags & XBF_MAPPED)
1326 return bp->b_addr + offset;
1327
1328 offset += bp->b_offset;
1329 page = bp->b_pages[offset >> PAGE_SHIFT];
1330 return (xfs_caddr_t)page_address(page) + (offset & (PAGE_SIZE-1));
1331}
1332
1333/*
1334 * Move data into or out of a buffer.
1335 */
1336void
1337xfs_buf_iomove(
1338 xfs_buf_t *bp, /* buffer to process */
1339 size_t boff, /* starting buffer offset */
1340 size_t bsize, /* length to copy */
1341 void *data, /* data address */
1342 xfs_buf_rw_t mode) /* read/write/zero flag */
1343{
1344 size_t bend, cpoff, csize;
1345 struct page *page;
1346
1347 bend = boff + bsize;
1348 while (boff < bend) {
1349 page = bp->b_pages[xfs_buf_btoct(boff + bp->b_offset)];
1350 cpoff = xfs_buf_poff(boff + bp->b_offset);
1351 csize = min_t(size_t,
1352 PAGE_SIZE-cpoff, bp->b_count_desired-boff);
1353
1354 ASSERT(((csize + cpoff) <= PAGE_SIZE));
1355
1356 switch (mode) {
1357 case XBRW_ZERO:
1358 memset(page_address(page) + cpoff, 0, csize);
1359 break;
1360 case XBRW_READ:
1361 memcpy(data, page_address(page) + cpoff, csize);
1362 break;
1363 case XBRW_WRITE:
1364 memcpy(page_address(page) + cpoff, data, csize);
1365 }
1366
1367 boff += csize;
1368 data += csize;
1369 }
1370}
1371
1372/*
1373 * Handling of buffer targets (buftargs).
1374 */
1375
1376/*
1377 * Wait for any bufs with callbacks that have been submitted but have not yet
1378 * returned. These buffers will have an elevated hold count, so wait on those
1379 * while freeing all the buffers only held by the LRU.
1380 */
1381void
1382xfs_wait_buftarg(
1383 struct xfs_buftarg *btp)
1384{
1385 struct xfs_buf *bp;
1386
1387restart:
1388 spin_lock(&btp->bt_lru_lock);
1389 while (!list_empty(&btp->bt_lru)) {
1390 bp = list_first_entry(&btp->bt_lru, struct xfs_buf, b_lru);
1391 if (atomic_read(&bp->b_hold) > 1) {
1392 spin_unlock(&btp->bt_lru_lock);
1393 delay(100);
1394 goto restart;
1395 }
1396 /*
1397 * clear the LRU reference count so the bufer doesn't get
1398 * ignored in xfs_buf_rele().
1399 */
1400 atomic_set(&bp->b_lru_ref, 0);
1401 spin_unlock(&btp->bt_lru_lock);
1402 xfs_buf_rele(bp);
1403 spin_lock(&btp->bt_lru_lock);
1404 }
1405 spin_unlock(&btp->bt_lru_lock);
1406}
1407
1408int
1409xfs_buftarg_shrink(
1410 struct shrinker *shrink,
1411 struct shrink_control *sc)
1412{
1413 struct xfs_buftarg *btp = container_of(shrink,
1414 struct xfs_buftarg, bt_shrinker);
1415 struct xfs_buf *bp;
1416 int nr_to_scan = sc->nr_to_scan;
1417 LIST_HEAD(dispose);
1418
1419 if (!nr_to_scan)
1420 return btp->bt_lru_nr;
1421
1422 spin_lock(&btp->bt_lru_lock);
1423 while (!list_empty(&btp->bt_lru)) {
1424 if (nr_to_scan-- <= 0)
1425 break;
1426
1427 bp = list_first_entry(&btp->bt_lru, struct xfs_buf, b_lru);
1428
1429 /*
1430 * Decrement the b_lru_ref count unless the value is already
1431 * zero. If the value is already zero, we need to reclaim the
1432 * buffer, otherwise it gets another trip through the LRU.
1433 */
1434 if (!atomic_add_unless(&bp->b_lru_ref, -1, 0)) {
1435 list_move_tail(&bp->b_lru, &btp->bt_lru);
1436 continue;
1437 }
1438
1439 /*
1440 * remove the buffer from the LRU now to avoid needing another
1441 * lock round trip inside xfs_buf_rele().
1442 */
1443 list_move(&bp->b_lru, &dispose);
1444 btp->bt_lru_nr--;
1445 }
1446 spin_unlock(&btp->bt_lru_lock);
1447
1448 while (!list_empty(&dispose)) {
1449 bp = list_first_entry(&dispose, struct xfs_buf, b_lru);
1450 list_del_init(&bp->b_lru);
1451 xfs_buf_rele(bp);
1452 }
1453
1454 return btp->bt_lru_nr;
1455}
1456
1457void
1458xfs_free_buftarg(
1459 struct xfs_mount *mp,
1460 struct xfs_buftarg *btp)
1461{
1462 unregister_shrinker(&btp->bt_shrinker);
1463
1464 xfs_flush_buftarg(btp, 1);
1465 if (mp->m_flags & XFS_MOUNT_BARRIER)
1466 xfs_blkdev_issue_flush(btp);
1467
1468 kthread_stop(btp->bt_task);
1469 kmem_free(btp);
1470}
1471
1472STATIC int
1473xfs_setsize_buftarg_flags(
1474 xfs_buftarg_t *btp,
1475 unsigned int blocksize,
1476 unsigned int sectorsize,
1477 int verbose)
1478{
1479 btp->bt_bsize = blocksize;
1480 btp->bt_sshift = ffs(sectorsize) - 1;
1481 btp->bt_smask = sectorsize - 1;
1482
1483 if (set_blocksize(btp->bt_bdev, sectorsize)) {
1484 xfs_warn(btp->bt_mount,
1485 "Cannot set_blocksize to %u on device %s\n",
1486 sectorsize, xfs_buf_target_name(btp));
1487 return EINVAL;
1488 }
1489
1490 return 0;
1491}
1492
1493/*
1494 * When allocating the initial buffer target we have not yet
1495 * read in the superblock, so don't know what sized sectors
1496 * are being used is at this early stage. Play safe.
1497 */
1498STATIC int
1499xfs_setsize_buftarg_early(
1500 xfs_buftarg_t *btp,
1501 struct block_device *bdev)
1502{
1503 return xfs_setsize_buftarg_flags(btp,
1504 PAGE_SIZE, bdev_logical_block_size(bdev), 0);
1505}
1506
1507int
1508xfs_setsize_buftarg(
1509 xfs_buftarg_t *btp,
1510 unsigned int blocksize,
1511 unsigned int sectorsize)
1512{
1513 return xfs_setsize_buftarg_flags(btp, blocksize, sectorsize, 1);
1514}
1515
1516STATIC int
1517xfs_alloc_delwrite_queue(
1518 xfs_buftarg_t *btp,
1519 const char *fsname)
1520{
1521 INIT_LIST_HEAD(&btp->bt_delwrite_queue);
1522 spin_lock_init(&btp->bt_delwrite_lock);
1523 btp->bt_flags = 0;
1524 btp->bt_task = kthread_run(xfsbufd, btp, "xfsbufd/%s", fsname);
1525 if (IS_ERR(btp->bt_task))
1526 return PTR_ERR(btp->bt_task);
1527 return 0;
1528}
1529
1530xfs_buftarg_t *
1531xfs_alloc_buftarg(
1532 struct xfs_mount *mp,
1533 struct block_device *bdev,
1534 int external,
1535 const char *fsname)
1536{
1537 xfs_buftarg_t *btp;
1538
1539 btp = kmem_zalloc(sizeof(*btp), KM_SLEEP);
1540
1541 btp->bt_mount = mp;
1542 btp->bt_dev = bdev->bd_dev;
1543 btp->bt_bdev = bdev;
1544 btp->bt_bdi = blk_get_backing_dev_info(bdev);
1545 if (!btp->bt_bdi)
1546 goto error;
1547
1548 INIT_LIST_HEAD(&btp->bt_lru);
1549 spin_lock_init(&btp->bt_lru_lock);
1550 if (xfs_setsize_buftarg_early(btp, bdev))
1551 goto error;
1552 if (xfs_alloc_delwrite_queue(btp, fsname))
1553 goto error;
1554 btp->bt_shrinker.shrink = xfs_buftarg_shrink;
1555 btp->bt_shrinker.seeks = DEFAULT_SEEKS;
1556 register_shrinker(&btp->bt_shrinker);
1557 return btp;
1558
1559error:
1560 kmem_free(btp);
1561 return NULL;
1562}
1563
1564
1565/*
1566 * Delayed write buffer handling
1567 */
1568STATIC void
1569xfs_buf_delwri_queue(
1570 xfs_buf_t *bp,
1571 int unlock)
1572{
1573 struct list_head *dwq = &bp->b_target->bt_delwrite_queue;
1574 spinlock_t *dwlk = &bp->b_target->bt_delwrite_lock;
1575
1576 trace_xfs_buf_delwri_queue(bp, _RET_IP_);
1577
1578 ASSERT((bp->b_flags&(XBF_DELWRI|XBF_ASYNC)) == (XBF_DELWRI|XBF_ASYNC));
1579
1580 spin_lock(dwlk);
1581 /* If already in the queue, dequeue and place at tail */
1582 if (!list_empty(&bp->b_list)) {
1583 ASSERT(bp->b_flags & _XBF_DELWRI_Q);
1584 if (unlock)
1585 atomic_dec(&bp->b_hold);
1586 list_del(&bp->b_list);
1587 }
1588
1589 if (list_empty(dwq)) {
1590 /* start xfsbufd as it is about to have something to do */
1591 wake_up_process(bp->b_target->bt_task);
1592 }
1593
1594 bp->b_flags |= _XBF_DELWRI_Q;
1595 list_add_tail(&bp->b_list, dwq);
1596 bp->b_queuetime = jiffies;
1597 spin_unlock(dwlk);
1598
1599 if (unlock)
1600 xfs_buf_unlock(bp);
1601}
1602
1603void
1604xfs_buf_delwri_dequeue(
1605 xfs_buf_t *bp)
1606{
1607 spinlock_t *dwlk = &bp->b_target->bt_delwrite_lock;
1608 int dequeued = 0;
1609
1610 spin_lock(dwlk);
1611 if ((bp->b_flags & XBF_DELWRI) && !list_empty(&bp->b_list)) {
1612 ASSERT(bp->b_flags & _XBF_DELWRI_Q);
1613 list_del_init(&bp->b_list);
1614 dequeued = 1;
1615 }
1616 bp->b_flags &= ~(XBF_DELWRI|_XBF_DELWRI_Q);
1617 spin_unlock(dwlk);
1618
1619 if (dequeued)
1620 xfs_buf_rele(bp);
1621
1622 trace_xfs_buf_delwri_dequeue(bp, _RET_IP_);
1623}
1624
1625/*
1626 * If a delwri buffer needs to be pushed before it has aged out, then promote
1627 * it to the head of the delwri queue so that it will be flushed on the next
1628 * xfsbufd run. We do this by resetting the queuetime of the buffer to be older
1629 * than the age currently needed to flush the buffer. Hence the next time the
1630 * xfsbufd sees it is guaranteed to be considered old enough to flush.
1631 */
1632void
1633xfs_buf_delwri_promote(
1634 struct xfs_buf *bp)
1635{
1636 struct xfs_buftarg *btp = bp->b_target;
1637 long age = xfs_buf_age_centisecs * msecs_to_jiffies(10) + 1;
1638
1639 ASSERT(bp->b_flags & XBF_DELWRI);
1640 ASSERT(bp->b_flags & _XBF_DELWRI_Q);
1641
1642 /*
1643 * Check the buffer age before locking the delayed write queue as we
1644 * don't need to promote buffers that are already past the flush age.
1645 */
1646 if (bp->b_queuetime < jiffies - age)
1647 return;
1648 bp->b_queuetime = jiffies - age;
1649 spin_lock(&btp->bt_delwrite_lock);
1650 list_move(&bp->b_list, &btp->bt_delwrite_queue);
1651 spin_unlock(&btp->bt_delwrite_lock);
1652}
1653
1654STATIC void
1655xfs_buf_runall_queues(
1656 struct workqueue_struct *queue)
1657{
1658 flush_workqueue(queue);
1659}
1660
1661/*
1662 * Move as many buffers as specified to the supplied list
1663 * idicating if we skipped any buffers to prevent deadlocks.
1664 */
1665STATIC int
1666xfs_buf_delwri_split(
1667 xfs_buftarg_t *target,
1668 struct list_head *list,
1669 unsigned long age)
1670{
1671 xfs_buf_t *bp, *n;
1672 struct list_head *dwq = &target->bt_delwrite_queue;
1673 spinlock_t *dwlk = &target->bt_delwrite_lock;
1674 int skipped = 0;
1675 int force;
1676
1677 force = test_and_clear_bit(XBT_FORCE_FLUSH, &target->bt_flags);
1678 INIT_LIST_HEAD(list);
1679 spin_lock(dwlk);
1680 list_for_each_entry_safe(bp, n, dwq, b_list) {
1681 ASSERT(bp->b_flags & XBF_DELWRI);
1682
1683 if (!xfs_buf_ispinned(bp) && xfs_buf_trylock(bp)) {
1684 if (!force &&
1685 time_before(jiffies, bp->b_queuetime + age)) {
1686 xfs_buf_unlock(bp);
1687 break;
1688 }
1689
1690 bp->b_flags &= ~(XBF_DELWRI | _XBF_DELWRI_Q);
1691 bp->b_flags |= XBF_WRITE;
1692 list_move_tail(&bp->b_list, list);
1693 trace_xfs_buf_delwri_split(bp, _RET_IP_);
1694 } else
1695 skipped++;
1696 }
1697 spin_unlock(dwlk);
1698
1699 return skipped;
1700
1701}
1702
1703/*
1704 * Compare function is more complex than it needs to be because
1705 * the return value is only 32 bits and we are doing comparisons
1706 * on 64 bit values
1707 */
1708static int
1709xfs_buf_cmp(
1710 void *priv,
1711 struct list_head *a,
1712 struct list_head *b)
1713{
1714 struct xfs_buf *ap = container_of(a, struct xfs_buf, b_list);
1715 struct xfs_buf *bp = container_of(b, struct xfs_buf, b_list);
1716 xfs_daddr_t diff;
1717
1718 diff = ap->b_bn - bp->b_bn;
1719 if (diff < 0)
1720 return -1;
1721 if (diff > 0)
1722 return 1;
1723 return 0;
1724}
1725
1726STATIC int
1727xfsbufd(
1728 void *data)
1729{
1730 xfs_buftarg_t *target = (xfs_buftarg_t *)data;
1731
1732 current->flags |= PF_MEMALLOC;
1733
1734 set_freezable();
1735
1736 do {
1737 long age = xfs_buf_age_centisecs * msecs_to_jiffies(10);
1738 long tout = xfs_buf_timer_centisecs * msecs_to_jiffies(10);
1739 struct list_head tmp;
1740 struct blk_plug plug;
1741
1742 if (unlikely(freezing(current))) {
1743 set_bit(XBT_FORCE_SLEEP, &target->bt_flags);
1744 refrigerator();
1745 } else {
1746 clear_bit(XBT_FORCE_SLEEP, &target->bt_flags);
1747 }
1748
1749 /* sleep for a long time if there is nothing to do. */
1750 if (list_empty(&target->bt_delwrite_queue))
1751 tout = MAX_SCHEDULE_TIMEOUT;
1752 schedule_timeout_interruptible(tout);
1753
1754 xfs_buf_delwri_split(target, &tmp, age);
1755 list_sort(NULL, &tmp, xfs_buf_cmp);
1756
1757 blk_start_plug(&plug);
1758 while (!list_empty(&tmp)) {
1759 struct xfs_buf *bp;
1760 bp = list_first_entry(&tmp, struct xfs_buf, b_list);
1761 list_del_init(&bp->b_list);
1762 xfs_bdstrat_cb(bp);
1763 }
1764 blk_finish_plug(&plug);
1765 } while (!kthread_should_stop());
1766
1767 return 0;
1768}
1769
1770/*
1771 * Go through all incore buffers, and release buffers if they belong to
1772 * the given device. This is used in filesystem error handling to
1773 * preserve the consistency of its metadata.
1774 */
1775int
1776xfs_flush_buftarg(
1777 xfs_buftarg_t *target,
1778 int wait)
1779{
1780 xfs_buf_t *bp;
1781 int pincount = 0;
1782 LIST_HEAD(tmp_list);
1783 LIST_HEAD(wait_list);
1784 struct blk_plug plug;
1785
1786 xfs_buf_runall_queues(xfsconvertd_workqueue);
1787 xfs_buf_runall_queues(xfsdatad_workqueue);
1788 xfs_buf_runall_queues(xfslogd_workqueue);
1789
1790 set_bit(XBT_FORCE_FLUSH, &target->bt_flags);
1791 pincount = xfs_buf_delwri_split(target, &tmp_list, 0);
1792
1793 /*
1794 * Dropped the delayed write list lock, now walk the temporary list.
1795 * All I/O is issued async and then if we need to wait for completion
1796 * we do that after issuing all the IO.
1797 */
1798 list_sort(NULL, &tmp_list, xfs_buf_cmp);
1799
1800 blk_start_plug(&plug);
1801 while (!list_empty(&tmp_list)) {
1802 bp = list_first_entry(&tmp_list, struct xfs_buf, b_list);
1803 ASSERT(target == bp->b_target);
1804 list_del_init(&bp->b_list);
1805 if (wait) {
1806 bp->b_flags &= ~XBF_ASYNC;
1807 list_add(&bp->b_list, &wait_list);
1808 }
1809 xfs_bdstrat_cb(bp);
1810 }
1811 blk_finish_plug(&plug);
1812
1813 if (wait) {
1814 /* Wait for IO to complete. */
1815 while (!list_empty(&wait_list)) {
1816 bp = list_first_entry(&wait_list, struct xfs_buf, b_list);
1817
1818 list_del_init(&bp->b_list);
1819 xfs_buf_iowait(bp);
1820 xfs_buf_relse(bp);
1821 }
1822 }
1823
1824 return pincount;
1825}
1826
1827int __init
1828xfs_buf_init(void)
1829{
1830 xfs_buf_zone = kmem_zone_init_flags(sizeof(xfs_buf_t), "xfs_buf",
1831 KM_ZONE_HWALIGN, NULL);
1832 if (!xfs_buf_zone)
1833 goto out;
1834
1835 xfslogd_workqueue = alloc_workqueue("xfslogd",
1836 WQ_MEM_RECLAIM | WQ_HIGHPRI, 1);
1837 if (!xfslogd_workqueue)
1838 goto out_free_buf_zone;
1839
1840 xfsdatad_workqueue = alloc_workqueue("xfsdatad", WQ_MEM_RECLAIM, 1);
1841 if (!xfsdatad_workqueue)
1842 goto out_destroy_xfslogd_workqueue;
1843
1844 xfsconvertd_workqueue = alloc_workqueue("xfsconvertd",
1845 WQ_MEM_RECLAIM, 1);
1846 if (!xfsconvertd_workqueue)
1847 goto out_destroy_xfsdatad_workqueue;
1848
1849 return 0;
1850
1851 out_destroy_xfsdatad_workqueue:
1852 destroy_workqueue(xfsdatad_workqueue);
1853 out_destroy_xfslogd_workqueue:
1854 destroy_workqueue(xfslogd_workqueue);
1855 out_free_buf_zone:
1856 kmem_zone_destroy(xfs_buf_zone);
1857 out:
1858 return -ENOMEM;
1859}
1860
1861void
1862xfs_buf_terminate(void)
1863{
1864 destroy_workqueue(xfsconvertd_workqueue);
1865 destroy_workqueue(xfsdatad_workqueue);
1866 destroy_workqueue(xfslogd_workqueue);
1867 kmem_zone_destroy(xfs_buf_zone);
1868}
1869
1870#ifdef CONFIG_KDB_MODULES
1871struct list_head *
1872xfs_get_buftarg_list(void)
1873{
1874 return &xfs_buftarg_list;
1875}
1876#endif
generated by cgit v1.2.2 (git 2.25.0) at 2025-01-09 04:03:41 -0500