/* * linux/fs/ext4/inode.c * * Copyright (C) 1992, 1993, 1994, 1995 * Remy Card (card@masi.ibp.fr) * Laboratoire MASI - Institut Blaise Pascal * Universite Pierre et Marie Curie (Paris VI) * * from * * linux/fs/minix/inode.c * * Copyright (C) 1991, 1992 Linus Torvalds * * Goal-directed block allocation by Stephen Tweedie * (sct@redhat.com), 1993, 1998 * Big-endian to little-endian byte-swapping/bitmaps by * David S. Miller (davem@caip.rutgers.edu), 1995 * 64-bit file support on 64-bit platforms by Jakub Jelinek * (jj@sunsite.ms.mff.cuni.cz) * * Assorted race fixes, rewrite of ext4_get_block() by Al Viro, 2000 */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "ext4_jbd2.h" #include "xattr.h" #include "acl.h" #include "ext4_extents.h" #define MPAGE_DA_EXTENT_TAIL 0x01 static inline int ext4_begin_ordered_truncate(struct inode *inode, loff_t new_size) { return jbd2_journal_begin_ordered_truncate(&EXT4_I(inode)->jinode, new_size); } static void ext4_invalidatepage(struct page *page, unsigned long offset); /* * Test whether an inode is a fast symlink. */ static int ext4_inode_is_fast_symlink(struct inode *inode) { int ea_blocks = EXT4_I(inode)->i_file_acl ? (inode->i_sb->s_blocksize >> 9) : 0; return (S_ISLNK(inode->i_mode) && inode->i_blocks - ea_blocks == 0); } /* * The ext4 forget function must perform a revoke if we are freeing data * which has been journaled. Metadata (eg. indirect blocks) must be * revoked in all cases. * * "bh" may be NULL: a metadata block may have been freed from memory * but there may still be a record of it in the journal, and that record * still needs to be revoked. */ int ext4_forget(handle_t *handle, int is_metadata, struct inode *inode, struct buffer_head *bh, ext4_fsblk_t blocknr) { int err; might_sleep(); BUFFER_TRACE(bh, "enter"); jbd_debug(4, "forgetting bh %p: is_metadata = %d, mode %o, " "data mode %lx\n", bh, is_metadata, inode->i_mode, test_opt(inode->i_sb, DATA_FLAGS)); /* Never use the revoke function if we are doing full data * journaling: there is no need to, and a V1 superblock won't * support it. Otherwise, only skip the revoke on un-journaled * data blocks. */ if (test_opt(inode->i_sb, DATA_FLAGS) == EXT4_MOUNT_JOURNAL_DATA || (!is_metadata && !ext4_should_journal_data(inode))) { if (bh) { BUFFER_TRACE(bh, "call jbd2_journal_forget"); return ext4_journal_forget(handle, bh); } return 0; } /* * data!=journal && (is_metadata || should_journal_data(inode)) */ BUFFER_TRACE(bh, "call ext4_journal_revoke"); err = ext4_journal_revoke(handle, blocknr, bh); if (err) ext4_abort(inode->i_sb, __func__, "error %d when attempting revoke", err); BUFFER_TRACE(bh, "exit"); return err; } /* * Work out how many blocks we need to proceed with the next chunk of a * truncate transaction. */ static unsigned long blocks_for_truncate(struct inode *inode) { ext4_lblk_t needed; needed = inode->i_blocks >> (inode->i_sb->s_blocksize_bits - 9); /* Give ourselves just enough room to cope with inodes in which * i_blocks is corrupt: we've seen disk corruptions in the past * which resulted in random data in an inode which looked enough * like a regular file for ext4 to try to delete it. Things * will go a bit crazy if that happens, but at least we should * try not to panic the whole kernel. */ if (needed < 2) needed = 2; /* But we need to bound the transaction so we don't overflow the * journal. */ if (needed > EXT4_MAX_TRANS_DATA) needed = EXT4_MAX_TRANS_DATA; return EXT4_DATA_TRANS_BLOCKS(inode->i_sb) + needed; } /* * Truncate transactions can be complex and absolutely huge. So we need to * be able to restart the transaction at a conventient checkpoint to make * sure we don't overflow the journal. * * start_transaction gets us a new handle for a truncate transaction, * and extend_transaction tries to extend the existing one a bit. If * extend fails, we need to propagate the failure up and restart the * transaction in the top-level truncate loop. --sct */ static handle_t *start_transaction(struct inode *inode) { handle_t *result; result = ext4_journal_start(inode, blocks_for_truncate(inode)); if (!IS_ERR(result)) return result; ext4_std_error(inode->i_sb, PTR_ERR(result)); return result; } /* * Try to extend this transaction for the purposes of truncation. * * Returns 0 if we managed to create more room. If we can't create more * room, and the transaction must be restarted we return 1. */ static int try_to_extend_transaction(handle_t *handle, struct inode *inode) { if (handle->h_buffer_credits > EXT4_RESERVE_TRANS_BLOCKS) return 0; if (!ext4_journal_extend(handle, blocks_for_truncate(inode))) return 0; return 1; } /* * Restart the transaction associated with *handle. This does a commit, * so before we call here everything must be consistently dirtied against * this transaction. */ static int ext4_journal_test_restart(handle_t *handle, struct inode *inode) { jbd_debug(2, "restarting handle %p\n", handle); return ext4_journal_restart(handle, blocks_for_truncate(inode)); } /* * Called at the last iput() if i_nlink is zero. */ void ext4_delete_inode(struct inode *inode) { handle_t *handle; int err; if (ext4_should_order_data(inode)) ext4_begin_ordered_truncate(inode, 0); truncate_inode_pages(&inode->i_data, 0); if (is_bad_inode(inode)) goto no_delete; handle = ext4_journal_start(inode, blocks_for_truncate(inode)+3); if (IS_ERR(handle)) { ext4_std_error(inode->i_sb, PTR_ERR(handle)); /* * If we're going to skip the normal cleanup, we still need to * make sure that the in-core orphan linked list is properly * cleaned up. */ ext4_orphan_del(NULL, inode); goto no_delete; } if (IS_SYNC(inode)) handle->h_sync = 1; inode->i_size = 0; err = ext4_mark_inode_dirty(handle, inode); if (err) { ext4_warning(inode->i_sb, __func__, "couldn't mark inode dirty (err %d)", err); goto stop_handle; } if (inode->i_blocks) ext4_truncate(inode); /* * ext4_ext_truncate() doesn't reserve any slop when it * restarts journal transactions; therefore there may not be * enough credits left in the handle to remove the inode from * the orphan list and set the dtime field. */ if (handle->h_buffer_credits < 3) { err = ext4_journal_extend(handle, 3); if (err > 0) err = ext4_journal_restart(handle, 3); if (err != 0) { ext4_warning(inode->i_sb, __func__, "couldn't extend journal (err %d)", err); stop_handle: ext4_journal_stop(handle); goto no_delete; } } /* * Kill off the orphan record which ext4_truncate created. * AKPM: I think this can be inside the above `if'. * Note that ext4_orphan_del() has to be able to cope with the * deletion of a non-existent orphan - this is because we don't * know if ext4_truncate() actually created an orphan record. * (Well, we could do this if we need to, but heck - it works) */ ext4_orphan_del(handle, inode); EXT4_I(inode)->i_dtime = get_seconds(); /* * One subtle ordering requirement: if anything has gone wrong * (transaction abort, IO errors, whatever), then we can still * do these next steps (the fs will already have been marked as * having errors), but we can't free the inode if the mark_dirty * fails. */ if (ext4_mark_inode_dirty(handle, inode)) /* If that failed, just do the required in-core inode clear. */ clear_inode(inode); else ext4_free_inode(handle, inode); ext4_journal_stop(handle); return; no_delete: clear_inode(inode); /* We must guarantee clearing of inode... */ } typedef struct { __le32 *p; __le32 key; struct buffer_head *bh; } Indirect; static inline void add_chain(Indirect *p, struct buffer_head *bh, __le32 *v) { p->key = *(p->p = v); p->bh = bh; } /** * ext4_block_to_path - parse the block number into array of offsets * @inode: inode in question (we are only interested in its superblock) * @i_block: block number to be parsed * @offsets: array to store the offsets in * @boundary: set this non-zero if the referred-to block is likely to be * followed (on disk) by an indirect block. * * To store the locations of file's data ext4 uses a data structure common * for UNIX filesystems - tree of pointers anchored in the inode, with * data blocks at leaves and indirect blocks in intermediate nodes. * This function translates the block number into path in that tree - * return value is the path length and @offsets[n] is the offset of * pointer to (n+1)th node in the nth one. If @block is out of range * (negative or too large) warning is printed and zero returned. * * Note: function doesn't find node addresses, so no IO is needed. All * we need to know is the capacity of indirect blocks (taken from the * inode->i_sb). */ /* * Portability note: the last comparison (check that we fit into triple * indirect block) is spelled differently, because otherwise on an * architecture with 32-bit longs and 8Kb pages we might get into trouble * if our filesystem had 8Kb blocks. We might use long long, but that would * kill us on x86. Oh, well, at least the sign propagation does not matter - * i_block would have to be negative in the very beginning, so we would not * get there at all. */ static int ext4_block_to_path(struct inode *inode, ext4_lblk_t i_block, ext4_lblk_t offsets[4], int *boundary) { int ptrs = EXT4_ADDR_PER_BLOCK(inode->i_sb); int ptrs_bits = EXT4_ADDR_PER_BLOCK_BITS(inode->i_sb); const long direct_blocks = EXT4_NDIR_BLOCKS, indirect_blocks = ptrs, double_blocks = (1 << (ptrs_bits * 2)); int n = 0; int final = 0; if (i_block < 0) { ext4_warning(inode->i_sb, "ext4_block_to_path", "block < 0"); } else if (i_block < direct_blocks) { offsets[n++] = i_block; final = direct_blocks; } else if ((i_block -= direct_blocks) < indirect_blocks) { offsets[n++] = EXT4_IND_BLOCK; offsets[n++] = i_block; final = ptrs; } else if ((i_block -= indirect_blocks) < double_blocks) { offsets[n++] = EXT4_DIND_BLOCK; offsets[n++] = i_block >> ptrs_bits; offsets[n++] = i_block & (ptrs - 1); final = ptrs; } else if (((i_block -= double_blocks) >> (ptrs_bits * 2)) < ptrs) { offsets[n++] = EXT4_TIND_BLOCK; offsets[n++] = i_block >> (ptrs_bits * 2); offsets[n++] = (i_block >> ptrs_bits) & (ptrs - 1); offsets[n++] = i_block & (ptrs - 1); final = ptrs; } else { ext4_warning(inode->i_sb, "ext4_block_to_path", "block %lu > max", i_block + direct_blocks + indirect_blocks + double_blocks); } if (boundary) *boundary = final - 1 - (i_block & (ptrs - 1)); return n; } /** * ext4_get_branch - read the chain of indirect blocks leading to data * @inode: inode in question * @depth: depth of the chain (1 - direct pointer, etc.) * @offsets: offsets of pointers in inode/indirect blocks * @chain: place to store the result * @err: here we store the error value * * Function fills the array of triples and returns %NULL * if everything went OK or the pointer to the last filled triple * (incomplete one) otherwise. Upon the return chain[i].key contains * the number of (i+1)-th block in the chain (as it is stored in memory, * i.e. little-endian 32-bit), chain[i].p contains the address of that * number (it points into struct inode for i==0 and into the bh->b_data * for i>0) and chain[i].bh points to the buffer_head of i-th indirect * block for i>0 and NULL for i==0. In other words, it holds the block * numbers of the chain, addresses they were taken from (and where we can * verify that chain did not change) and buffer_heads hosting these * numbers. * * Function stops when it stumbles upon zero pointer (absent block) * (pointer to last triple returned, *@err == 0) * or when it gets an IO error reading an indirect block * (ditto, *@err == -EIO) * or when it reads all @depth-1 indirect blocks successfully and finds * the whole chain, all way to the data (returns %NULL, *err == 0). * * Need to be called with * down_read(&EXT4_I(inode)->i_data_sem) */ static Indirect *ext4_get_branch(struct inode *inode, int depth, ext4_lblk_t *offsets, Indirect chain[4], int *err) { struct super_block *sb = inode->i_sb; Indirect *p = chain; struct buffer_head *bh; *err = 0; /* i_data is not going away, no lock needed */ add_chain(chain, NULL, EXT4_I(inode)->i_data + *offsets); if (!p->key) goto no_block; while (--depth) { bh = sb_bread(sb, le32_to_cpu(p->key)); if (!bh) goto failure; add_chain(++p, bh, (__le32 *)bh->b_data + *++offsets); /* Reader: end */ if (!p->key) goto no_block; } return NULL; failure: *err = -EIO; no_block: return p; } /** * ext4_find_near - find a place for allocation with sufficient locality * @inode: owner * @ind: descriptor of indirect block. * * This function returns the preferred place for block allocation. * It is used when heuristic for sequential allocation fails. * Rules are: * + if there is a block to the left of our position - allocate near it. * + if pointer will live in indirect block - allocate near that block. * + if pointer will live in inode - allocate in the same * cylinder group. * * In the latter case we colour the starting block by the callers PID to * prevent it from clashing with concurrent allocations for a different inode * in the same block group. The PID is used here so that functionally related * files will be close-by on-disk. * * Caller must make sure that @ind is valid and will stay that way. */ static ext4_fsblk_t ext4_find_near(struct inode *inode, Indirect *ind) { struct ext4_inode_info *ei = EXT4_I(inode); __le32 *start = ind->bh ? (__le32 *) ind->bh->b_data : ei->i_data; __le32 *p; ext4_fsblk_t bg_start; ext4_fsblk_t last_block; ext4_grpblk_t colour; /* Try to find previous block */ for (p = ind->p - 1; p >= start; p--) { if (*p) return le32_to_cpu(*p); } /* No such thing, so let's try location of indirect block */ if (ind->bh) return ind->bh->b_blocknr; /* * It is going to be referred to from the inode itself? OK, just put it * into the same cylinder group then. */ bg_start = ext4_group_first_block_no(inode->i_sb, ei->i_block_group); last_block = ext4_blocks_count(EXT4_SB(inode->i_sb)->s_es) - 1; if (bg_start + EXT4_BLOCKS_PER_GROUP(inode->i_sb) <= last_block) colour = (current->pid % 16) * (EXT4_BLOCKS_PER_GROUP(inode->i_sb) / 16); else colour = (current->pid % 16) * ((last_block - bg_start) / 16); return bg_start + colour; } /** * ext4_find_goal - find a preferred place for allocation. * @inode: owner * @block: block we want * @partial: pointer to the last triple within a chain * * Normally this function find the preferred place for block allocation, * returns it. */ static ext4_fsblk_t ext4_find_goal(struct inode *inode, ext4_lblk_t block, Indirect *partial) { /* * XXX need to get goal block from mballoc's data structures */ return ext4_find_near(inode, partial); } /** * ext4_blks_to_allocate: Look up the block map and count the number * of direct blocks need to be allocated for the given branch. * * @branch: chain of indirect blocks * @k: number of blocks need for indirect blocks * @blks: number of data blocks to be mapped. * @blocks_to_boundary: the offset in the indirect block * * return the total number of blocks to be allocate, including the * direct and indirect blocks. */ static int ext4_blks_to_allocate(Indirect *branch, int k, unsigned long blks, int blocks_to_boundary) { unsigned long count = 0; /* * Simple case, [t,d]Indirect block(s) has not allocated yet * then it's clear blocks on that path have not allocated */ if (k > 0) { /* right now we don't handle cross boundary allocation */ if (blks < blocks_to_boundary + 1) count += blks; else count += blocks_to_boundary + 1; return count; } count++; while (count < blks && count <= blocks_to_boundary && le32_to_cpu(*(branch[0].p + count)) == 0) { count++; } return count; } /** * ext4_alloc_blocks: multiple allocate blocks needed for a branch * @indirect_blks: the number of blocks need to allocate for indirect * blocks * * @new_blocks: on return it will store the new block numbers for * the indirect blocks(if needed) and the first direct block, * @blks: on return it will store the total number of allocated * direct blocks */ static int ext4_alloc_blocks(handle_t *handle, struct inode *inode, ext4_lblk_t iblock, ext4_fsblk_t goal, int indirect_blks, int blks, ext4_fsblk_t new_blocks[4], int *err) { int target, i; unsigned long count = 0, blk_allocated = 0; int index = 0; ext4_fsblk_t current_block = 0; int ret = 0; /* * Here we try to allocate the requested multiple blocks at once, * on a best-effort basis. * To build a branch, we should allocate blocks for * the indirect blocks(if not allocated yet), and at least * the first direct block of this branch. That's the * minimum number of blocks need to allocate(required) */ /* first we try to allocate the indirect blocks */ target = indirect_blks; while (target > 0) { count = target; /* allocating blocks for indirect blocks and direct blocks */ current_block = ext4_new_meta_blocks(handle, inode, goal, &count, err); if (*err) goto failed_out; target -= count; /* allocate blocks for indirect blocks */ while (index < indirect_blks && count) { new_blocks[index++] = current_block++; count--; } if (count > 0) { /* * save the new block number * for the first direct block */ new_blocks[index] = current_block; printk(KERN_INFO "%s returned more blocks than " "requested\n", __func__); WARN_ON(1); break; } } target = blks - count ; blk_allocated = count; if (!target) goto allocated; /* Now allocate data blocks */ count = target; /* allocating blocks for data blocks */ current_block = ext4_new_blocks(handle, inode, iblock, goal, &count, err); if (*err && (target == blks)) { /* * if the allocation failed and we didn't allocate * any blocks before */ goto failed_out; } if (!*err) { if (target == blks) { /* * save the new block number * for the first direct block */ new_blocks[index] = current_block; } blk_allocated += count; } allocated: /* total number of blocks allocated for direct blocks */ ret = blk_allocated; *err = 0; return ret; failed_out: for (i = 0; i < index; i++) ext4_free_blocks(handle, inode, new_blocks[i], 1, 0); return ret; } /** * ext4_alloc_branch - allocate and set up a chain of blocks. * @inode: owner * @indirect_blks: number of allocated indirect blocks * @blks: number of allocated direct blocks * @offsets: offsets (in the blocks) to store the pointers to next. * @branch: place to store the chain in. * * This function allocates blocks, zeroes out all but the last one, * links them into chain and (if we are synchronous) writes them to disk. * In other words, it prepares a branch that can be spliced onto the * inode. It stores the information about that chain in the branch[], in * the same format as ext4_get_branch() would do. We are calling it after * we had read the existing part of chain and partial points to the last * triple of that (one with zero ->key). Upon the exit we have the same * picture as after the successful ext4_get_block(), except that in one * place chain is disconnected - *branch->p is still zero (we did not * set the last link), but branch->key contains the number that should * be placed into *branch->p to fill that gap. * * If allocation fails we free all blocks we've allocated (and forget * their buffer_heads) and return the error value the from failed * ext4_alloc_block() (normally -ENOSPC). Otherwise we set the chain * as described above and return 0. */ static int ext4_alloc_branch(handle_t *handle, struct inode *inode, ext4_lblk_t iblock, int indirect_blks, int *blks, ext4_fsblk_t goal, ext4_lblk_t *offsets, Indirect *branch) { int blocksize = inode->i_sb->s_blocksize; int i, n = 0; int err = 0; struct buffer_head *bh; int num; ext4_fsblk_t new_blocks[4]; ext4_fsblk_t current_block; num = ext4_alloc_blocks(handle, inode, iblock, goal, indirect_blks, *blks, new_blocks, &err); if (err) return err; branch[0].key = cpu_to_le32(new_blocks[0]); /* * metadata blocks and data blocks are allocated. */ for (n = 1; n <= indirect_blks; n++) { /* * Get buffer_head for parent block, zero it out * and set the pointer to new one, then send * parent to disk. */ bh = sb_getblk(inode->i_sb, new_blocks[n-1]); branch[n].bh = bh; lock_buffer(bh); BUFFER_TRACE(bh, "call get_create_access"); err = ext4_journal_get_create_access(handle, bh); if (err) { unlock_buffer(bh); brelse(bh); goto failed; } memset(bh->b_data, 0, blocksize); branch[n].p = (__le32 *) bh->b_data + offsets[n]; branch[n].key = cpu_to_le32(new_blocks[n]); *branch[n].p = branch[n].key; if (n == indirect_blks) { current_block = new_blocks[n]; /* * End of chain, update the last new metablock of * the chain to point to the new allocated * data blocks numbers */ for (i=1; i < num; i++) *(branch[n].p + i) = cpu_to_le32(++current_block); } BUFFER_TRACE(bh, "marking uptodate"); set_buffer_uptodate(bh); unlock_buffer(bh); BUFFER_TRACE(bh, "call ext4_journal_dirty_metadata"); err = ext4_journal_dirty_metadata(handle, bh); if (err) goto failed; } *blks = num; return err; failed: /* Allocation failed, free what we already allocated */ for (i = 1; i <= n ; i++) { BUFFER_TRACE(branch[i].bh, "call jbd2_journal_forget"); ext4_journal_forget(handle, branch[i].bh); } for (i = 0; i < indirect_blks; i++) ext4_free_blocks(handle, inode, new_blocks[i], 1, 0); ext4_free_blocks(handle, inode, new_blocks[i], num, 0); return err; } /** * ext4_splice_branch - splice the allocated branch onto inode. * @inode: owner * @block: (logical) number of block we are adding * @chain: chain of indirect blocks (with a missing link - see * ext4_alloc_branch) * @where: location of missing link * @num: number of indirect blocks we are adding * @blks: number of direct blocks we are adding * * This function fills the missing link and does all housekeeping needed in * inode (->i_blocks, etc.). In case of success we end up with the full * chain to new block and return 0. */ static int ext4_splice_branch(handle_t *handle, struct inode *inode, ext4_lblk_t block, Indirect *where, int num, int blks) { int i; int err = 0; ext4_fsblk_t current_block; /* * If we're splicing into a [td]indirect block (as opposed to the * inode) then we need to get write access to the [td]indirect block * before the splice. */ if (where->bh) { BUFFER_TRACE(where->bh, "get_write_access"); err = ext4_journal_get_write_access(handle, where->bh); if (err) goto err_out; } /* That's it */ *where->p = where->key; /* * Update the host buffer_head or inode to point to more just allocated * direct blocks blocks */ if (num == 0 && blks > 1) { current_block = le32_to_cpu(where->key) + 1; for (i = 1; i < blks; i++) *(where->p + i) = cpu_to_le32(current_block++); } /* We are done with atomic stuff, now do the rest of housekeeping */ inode->i_ctime = ext4_current_time(inode); ext4_mark_inode_dirty(handle, inode); /* had we spliced it onto indirect block? */ if (where->bh) { /* * If we spliced it onto an indirect block, we haven't * altered the inode. Note however that if it is being spliced * onto an indirect block at the very end of the file (the * file is growing) then we *will* alter the inode to reflect * the new i_size. But that is not done here - it is done in * generic_commit_write->__mark_inode_dirty->ext4_dirty_inode. */ jbd_debug(5, "splicing indirect only\n"); BUFFER_TRACE(where->bh, "call ext4_journal_dirty_metadata"); err = ext4_journal_dirty_metadata(handle, where->bh); if (err) goto err_out; } else { /* * OK, we spliced it into the inode itself on a direct block. * Inode was dirtied above. */ jbd_debug(5, "splicing direct\n"); } return err; err_out: for (i = 1; i <= num; i++) { BUFFER_TRACE(where[i].bh, "call jbd2_journal_forget"); ext4_journal_forget(handle, where[i].bh); ext4_free_blocks(handle, inode, le32_to_cpu(where[i-1].key), 1, 0); } ext4_free_blocks(handle, inode, le32_to_cpu(where[num].key), blks, 0); return err; } /* * Allocation strategy is simple: if we have to allocate something, we will * have to go the whole way to leaf. So let's do it before attaching anything * to tree, set linkage between the newborn blocks, write them if sync is * required, recheck the path, free and repeat if check fails, otherwise * set the last missing link (that will protect us from any truncate-generated * removals - all blocks on the path are immune now) and possibly force the * write on the parent block. * That has a nice additional property: no special recovery from the failed * allocations is needed - we simply release blocks and do not touch anything * reachable from inode. * * `handle' can be NULL if create == 0. * * return > 0, # of blocks mapped or allocated. * return = 0, if plain lookup failed. * return < 0, error case. * * * Need to be called with * down_read(&EXT4_I(inode)->i_data_sem) if not allocating file system block * (ie, create is zero). Otherwise down_write(&EXT4_I(inode)->i_data_sem) */ int ext4_get_blocks_handle(handle_t *handle, struct inode *inode, ext4_lblk_t iblock, unsigned long maxblocks, struct buffer_head *bh_result, int create, int extend_disksize) { int err = -EIO; ext4_lblk_t offsets[4]; Indirect chain[4]; Indirect *partial; ext4_fsblk_t goal; int indirect_blks; int blocks_to_boundary = 0; int depth; struct ext4_inode_info *ei = EXT4_I(inode); int count = 0; ext4_fsblk_t first_block = 0; loff_t disksize; J_ASSERT(!(EXT4_I(inode)->i_flags & EXT4_EXTENTS_FL)); J_ASSERT(handle != NULL || create == 0); depth = ext4_block_to_path(inode, iblock, offsets, &blocks_to_boundary); if (depth == 0) goto out; partial = ext4_get_branch(inode, depth, offsets, chain, &err); /* Simplest case - block found, no allocation needed */ if (!partial) { first_block = le32_to_cpu(chain[depth - 1].key); clear_buffer_new(bh_result); count++; /*map more blocks*/ while (count < maxblocks && count <= blocks_to_boundary) { ext4_fsblk_t blk; blk = le32_to_cpu(*(chain[depth-1].p + count)); if (blk == first_block + count) count++; else break; } goto got_it; } /* Next simple case - plain lookup or failed read of indirect block */ if (!create || err == -EIO) goto cleanup; /* * Okay, we need to do block allocation. */ goal = ext4_find_goal(inode, iblock, partial); /* the number of blocks need to allocate for [d,t]indirect blocks */ indirect_blks = (chain + depth) - partial - 1; /* * Next look up the indirect map to count the totoal number of * direct blocks to allocate for this branch. */ count = ext4_blks_to_allocate(partial, indirect_blks, maxblocks, blocks_to_boundary); /* * Block out ext4_truncate while we alter the tree */ err = ext4_alloc_branch(handle, inode, iblock, indirect_blks, &count, goal, offsets + (partial - chain), partial); /* * The ext4_splice_branch call will free and forget any buffers * on the new chain if there is a failure, but that risks using * up transaction credits, especially for bitmaps where the * credits cannot be returned. Can we handle this somehow? We * may need to return -EAGAIN upwards in the worst case. --sct */ if (!err) err = ext4_splice_branch(handle, inode, iblock, partial, indirect_blks, count); /* * i_disksize growing is protected by i_data_sem. Don't forget to * protect it if you're about to implement concurrent * ext4_get_block() -bzzz */ if (!err && extend_disksize) { disksize = ((loff_t) iblock + count) << inode->i_blkbits; if (disksize > i_size_read(inode)) disksize = i_size_read(inode); if (disksize > ei->i_disksize) ei->i_disksize = disksize; } if (err) goto cleanup; set_buffer_new(bh_result); got_it: map_bh(bh_result, inode->i_sb, le32_to_cpu(chain[depth-1].key)); if (count > blocks_to_boundary) set_buffer_boundary(bh_result); err = count; /* Clean up and exit */ partial = chain + depth - 1; /* the whole chain */ cleanup: while (partial > chain) { BUFFER_TRACE(partial->bh, "call brelse"); brelse(partial->bh); partial--; } BUFFER_TRACE(bh_result, "returned"); out: return err; } /* * Calculate the number of metadata blocks need to reserve * to allocate @blocks for non extent file based file */ static int ext4_indirect_calc_metadata_amount(struct inode *inode, int blocks) { int icap = EXT4_ADDR_PER_BLOCK(inode->i_sb); int ind_blks, dind_blks, tind_blks; /* number of new indirect blocks needed */ ind_blks = (blocks + icap - 1) / icap; dind_blks = (ind_blks + icap - 1) / icap; tind_blks = 1; return ind_blks + dind_blks + tind_blks; } /* * Calculate the number of metadata blocks need to reserve * to allocate given number of blocks */ static int ext4_calc_metadata_amount(struct inode *inode, int blocks) { if (!blocks) return 0; if (EXT4_I(inode)->i_flags & EXT4_EXTENTS_FL) return ext4_ext_calc_metadata_amount(inode, blocks); return ext4_indirect_calc_metadata_amount(inode, blocks); } static void ext4_da_update_reserve_space(struct inode *inode, int used) { struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb); int total, mdb, mdb_free; spin_lock(&EXT4_I(inode)->i_block_reservation_lock); /* recalculate the number of metablocks still need to be reserved */ total = EXT4_I(inode)->i_reserved_data_blocks - used; mdb = ext4_calc_metadata_amount(inode, total); /* figure out how many metablocks to release */ BUG_ON(mdb > EXT4_I(inode)->i_reserved_meta_blocks); mdb_free = EXT4_I(inode)->i_reserved_meta_blocks - mdb; if (mdb_free) { /* Account for allocated meta_blocks */ mdb_free -= EXT4_I(inode)->i_allocated_meta_blocks; /* update fs dirty blocks counter */ percpu_counter_sub(&sbi->s_dirtyblocks_counter, mdb_free); EXT4_I(inode)->i_allocated_meta_blocks = 0; EXT4_I(inode)->i_reserved_meta_blocks = mdb; } /* update per-inode reservations */ BUG_ON(used > EXT4_I(inode)->i_reserved_data_blocks); EXT4_I(inode)->i_reserved_data_blocks -= used; spin_unlock(&EXT4_I(inode)->i_block_reservation_lock); } /* * The ext4_get_blocks_wrap() function try to look up the requested blocks, * and returns if the blocks are already mapped. * * Otherwise it takes the write lock of the i_data_sem and allocate blocks * and store the allocated blocks in the result buffer head and mark it * mapped. * * If file type is extents based, it will call ext4_ext_get_blocks(), * Otherwise, call with ext4_get_blocks_handle() to handle indirect mapping * based files * * On success, it returns the number of blocks being mapped or allocate. * if create==0 and the blocks are pre-allocated and uninitialized block, * the result buffer head is unmapped. If the create ==1, it will make sure * the buffer head is mapped. * * It returns 0 if plain look up failed (blocks have not been allocated), in * that casem, buffer head is unmapped * * It returns the error in case of allocation failure. */ int ext4_get_blocks_wrap(handle_t *handle, struct inode *inode, sector_t block, unsigned long max_blocks, struct buffer_head *bh, int create, int extend_disksize, int flag) { int retval; clear_buffer_mapped(bh); /* * Try to see if we can get the block without requesting * for new file system block. */ down_read((&EXT4_I(inode)->i_data_sem)); if (EXT4_I(inode)->i_flags & EXT4_EXTENTS_FL) { retval = ext4_ext_get_blocks(handle, inode, block, max_blocks, bh, 0, 0); } else { retval = ext4_get_blocks_handle(handle, inode, block, max_blocks, bh, 0, 0); } up_read((&EXT4_I(inode)->i_data_sem)); /* If it is only a block(s) look up */ if (!create) return retval; /* * Returns if the blocks have already allocated * * Note that if blocks have been preallocated * ext4_ext_get_block() returns th create = 0 * with buffer head unmapped. */ if (retval > 0 && buffer_mapped(bh)) return retval; /* * New blocks allocate and/or writing to uninitialized extent * will possibly result in updating i_data, so we take * the write lock of i_data_sem, and call get_blocks() * with create == 1 flag. */ down_write((&EXT4_I(inode)->i_data_sem)); /* * if the caller is from delayed allocation writeout path * we have already reserved fs blocks for allocation * let the underlying get_block() function know to * avoid double accounting */ if (flag) EXT4_I(inode)->i_delalloc_reserved_flag = 1; /* * We need to check for EXT4 here because migrate * could have changed the inode type in between */ if (EXT4_I(inode)->i_flags & EXT4_EXTENTS_FL) { retval = ext4_ext_get_blocks(handle, inode, block, max_blocks, bh, create, extend_disksize); } else { retval = ext4_get_blocks_handle(handle, inode, block, max_blocks, bh, create, extend_disksize); if (retval > 0 && buffer_new(bh)) { /* * We allocated new blocks which will result in * i_data's format changing. Force the migrate * to fail by clearing migrate flags */ EXT4_I(inode)->i_flags = EXT4_I(inode)->i_flags & ~EXT4_EXT_MIGRATE; } } if (flag) { EXT4_I(inode)->i_delalloc_reserved_flag = 0; /* * Update reserved blocks/metadata blocks * after successful block allocation * which were deferred till now */ if ((retval > 0) && buffer_delay(bh)) ext4_da_update_reserve_space(inode, retval); } up_write((&EXT4_I(inode)->i_data_sem)); return retval; } /* Maximum number of blocks we map for direct IO at once. */ #define DIO_MAX_BLOCKS 4096 int ext4_get_block(struct inode *inode, sector_t iblock, struct buffer_head *bh_result, int create) { handle_t *handle = ext4_journal_current_handle(); int ret = 0, started = 0; unsigned max_blocks = bh_result->b_size >> inode->i_blkbits; int dio_credits; if (create && !handle) { /* Direct IO write... */ if (max_blocks > DIO_MAX_BLOCKS) max_blocks = DIO_MAX_BLOCKS; dio_credits = ext4_chunk_trans_blocks(inode, max_blocks); handle = ext4_journal_start(inode, dio_credits); if (IS_ERR(handle)) { ret = PTR_ERR(handle); goto out; } started = 1; } ret = ext4_get_blocks_wrap(handle, inode, iblock, max_blocks, bh_result, create, 0, 0); if (ret > 0) { bh_result->b_size = (ret << inode->i_blkbits); ret = 0; } if (started) ext4_journal_stop(handle); out: return ret; } /* * `handle' can be NULL if create is zero */ struct buffer_head *ext4_getblk(handle_t *handle, struct inode *inode, ext4_lblk_t block, int create, int *errp) { struct buffer_head dummy; int fatal = 0, err; J_ASSERT(handle != NULL || create == 0); dummy.b_state = 0; dummy.b_blocknr = -1000; buffer_trace_init(&dummy.b_history); err = ext4_get_blocks_wrap(handle, inode, block, 1, &dummy, create, 1, 0); /* * ext4_get_blocks_handle() returns number of blocks * mapped. 0 in case of a HOLE. */ if (err > 0) { if (err > 1) WARN_ON(1); err = 0; } *errp = err; if (!err && buffer_mapped(&dummy)) { struct buffer_head *bh; bh = sb_getblk(inode->i_sb, dummy.b_blocknr); if (!bh) { *errp = -EIO; goto err; } if (buffer_new(&dummy)) { J_ASSERT(create != 0); J_ASSERT(handle != NULL); /* * Now that we do not always journal data, we should * keep in mind whether this should always journal the * new buffer as metadata. For now, regular file * writes use ext4_get_block instead, so it's not a * problem. */ lock_buffer(bh); BUFFER_TRACE(bh, "call get_create_access"); fatal = ext4_journal_get_create_access(handle, bh); if (!fatal && !buffer_uptodate(bh)) { memset(bh->b_data, 0, inode->i_sb->s_blocksize); set_buffer_uptodate(bh); } unlock_buffer(bh); BUFFER_TRACE(bh, "call ext4_journal_dirty_metadata"); err = ext4_journal_dirty_metadata(handle, bh); if (!fatal) fatal = err; } else { BUFFER_TRACE(bh, "not a new buffer"); } if (fatal) { *errp = fatal; brelse(bh); bh = NULL; } return bh; } err: return NULL; } struct buffer_head *ext4_bread(handle_t *handle, struct inode *inode, ext4_lblk_t block, int create, int *err) { struct buffer_head *bh; bh = ext4_getblk(handle, inode, block, create, err); if (!bh) return bh; if (buffer_uptodate(bh)) return bh; ll_rw_block(READ_META, 1, &bh); wait_on_buffer(bh); if (buffer_uptodate(bh)) return bh; put_bh(bh); *err = -EIO; return NULL; } static int walk_page_buffers(handle_t *handle, struct buffer_head *head, unsigned from, unsigned to, int *partial, int (*fn)(handle_t *handle, struct buffer_head *bh)) { struct buffer_head *bh; unsigned block_start, block_end; unsigned blocksize = head->b_size; int err, ret = 0; struct buffer_head *next; for (bh = head, block_start = 0; ret == 0 && (bh != head || !block_start); block_start = block_end, bh = next) { next = bh->b_this_page; block_end = block_start + blocksize; if (block_end <= from || block_start >= to) { if (partial && !buffer_uptodate(bh)) *partial = 1; continue; } err = (*fn)(handle, bh); if (!ret) ret = err; } return ret; } /* * To preserve ordering, it is essential that the hole instantiation and * the data write be encapsulated in a single transaction. We cannot * close off a transaction and start a new one between the ext4_get_block() * and the commit_write(). So doing the jbd2_journal_start at the start of * prepare_write() is the right place. * * Also, this function can nest inside ext4_writepage() -> * block_write_full_page(). In that case, we *know* that ext4_writepage() * has generated enough buffer credits to do the whole page. So we won't * block on the journal in that case, which is good, because the caller may * be PF_MEMALLOC. * * By accident, ext4 can be reentered when a transaction is open via * quota file writes. If we were to commit the transaction while thus * reentered, there can be a deadlock - we would be holding a quota * lock, and the commit would never complete if another thread had a * transaction open and was blocking on the quota lock - a ranking * violation. * * So what we do is to rely on the fact that jbd2_journal_stop/journal_start * will _not_ run commit under these circumstances because handle->h_ref * is elevated. We'll still have enough credits for the tiny quotafile * write. */ static int do_journal_get_write_access(handle_t *handle, struct buffer_head *bh) { if (!buffer_mapped(bh) || buffer_freed(bh)) return 0; return ext4_journal_get_write_access(handle, bh); } static int ext4_write_begin(struct file *file, struct address_space *mapping, loff_t pos, unsigned len, unsigned flags, struct page **pagep, void **fsdata) { struct inode *inode = mapping->host; int ret, needed_blocks = ext4_writepage_trans_blocks(inode); handle_t *handle; int retries = 0; struct page *page; pgoff_t index; unsigned from, to; index = pos >> PAGE_CACHE_SHIFT; from = pos & (PAGE_CACHE_SIZE - 1); to = from + len; retry: handle = ext4_journal_start(inode, needed_blocks); if (IS_ERR(handle)) { ret = PTR_ERR(handle); goto out; } page = __grab_cache_page(mapping, index); if (!page) { ext4_journal_stop(handle); ret = -ENOMEM; goto out; } *pagep = page; ret = block_write_begin(file, mapping, pos, len, flags, pagep, fsdata, ext4_get_block); if (!ret && ext4_should_journal_data(inode)) { ret = walk_page_buffers(handle, page_buffers(page), from, to, NULL, do_journal_get_write_access); } if (ret) { unlock_page(page); ext4_journal_stop(handle); page_cache_release(page); /* * block_write_begin may have instantiated a few blocks * outside i_size. Trim these off again. Don't need * i_size_read because we hold i_mutex. */ if (pos + len > inode->i_size) vmtruncate(inode, inode->i_size); } if (ret == -ENOSPC && ext4_should_retry_alloc(inode->i_sb, &retries)) goto retry; out: return ret; } /* For write_end() in data=journal mode */ static int write_end_fn(handle_t *handle, struct buffer_head *bh) { if (!buffer_mapped(bh) || buffer_freed(bh)) return 0; set_buffer_uptodate(bh); return ext4_journal_dirty_metadata(handle, bh); } /* * We need to pick up the new inode size which generic_commit_write gave us * `file' can be NULL - eg, when called from page_symlink(). * * ext4 never places buffers on inode->i_mapping->private_list. metadata * buffers are managed internally. */ static int ext4_ordered_write_end(struct file *file, struct address_space *mapping, loff_t pos, unsigned len, unsigned copied, struct page *page, void *fsdata) { handle_t *handle = ext4_journal_current_handle(); struct inode *inode = mapping->host; int ret = 0, ret2; ret = ext4_jbd2_file_inode(handle, inode); if (ret == 0) { loff_t new_i_size; new_i_size = pos + copied; if (new_i_size > EXT4_I(inode)->i_disksize) { ext4_update_i_disksize(inode, new_i_size); /* We need to mark inode dirty even if * new_i_size is less that inode->i_size * bu greater than i_disksize.(hint delalloc) */ ext4_mark_inode_dirty(handle, inode); } ret2 = generic_write_end(file, mapping, pos, len, copied, page, fsdata); copied = ret2; if (ret2 < 0) ret = ret2; } ret2 = ext4_journal_stop(handle); if (!ret) ret = ret2; return ret ? ret : copied; } static int ext4_writeback_write_end(struct file *file, struct address_space *mapping, loff_t pos, unsigned len, unsigned copied, struct page *page, void *fsdata) { handle_t *handle = ext4_journal_current_handle(); struct inode *inode = mapping->host; int ret = 0, ret2; loff_t new_i_size; new_i_size = pos + copied; if (new_i_size > EXT4_I(inode)->i_disksize) { ext4_update_i_disksize(inode, new_i_size); /* We need to mark inode dirty even if * new_i_size is less that inode->i_size * bu greater than i_disksize.(hint delalloc) */ ext4_mark_inode_dirty(handle, inode); } ret2 = generic_write_end(file, mapping, pos, len, copied, page, fsdata); copied = ret2; if (ret2 < 0) ret = ret2; ret2 = ext4_journal_stop(handle); if (!ret) ret = ret2; return ret ? ret : copied; } static int ext4_journalled_write_end(struct file *file, struct address_space *mapping, loff_t pos, unsigned len, unsigned copied, struct page *page, void *fsdata) { handle_t *handle = ext4_journal_current_handle(); struct inode *inode = mapping->host; int ret = 0, ret2; int partial = 0; unsigned from, to; loff_t new_i_size; from = pos & (PAGE_CACHE_SIZE - 1); to = from + len; if (copied < len) { if (!PageUptodate(page)) copied = 0; page_zero_new_buffers(page, from+copied, to); } ret = walk_page_buffers(handle, page_buffers(page), from, to, &partial, write_end_fn); if (!partial) SetPageUptodate(page); new_i_size = pos + copied; if (new_i_size > inode->i_size) i_size_write(inode, pos+copied); EXT4_I(inode)->i_state |= EXT4_STATE_JDATA; if (new_i_size > EXT4_I(inode)->i_disksize) { ext4_update_i_disksize(inode, new_i_size); ret2 = ext4_mark_inode_dirty(handle, inode); if (!ret) ret = ret2; } unlock_page(page); ret2 = ext4_journal_stop(handle); if (!ret) ret = ret2; page_cache_release(page); return ret ? ret : copied; } static int ext4_da_reserve_space(struct inode *inode, int nrblocks) { int retries = 0; struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb); unsigned long md_needed, mdblocks, total = 0; /* * recalculate the amount of metadata blocks to reserve * in order to allocate nrblocks * worse case is one extent per block */ repeat: spin_lock(&EXT4_I(inode)->i_block_reservation_lock); total = EXT4_I(inode)->i_reserved_data_blocks + nrblocks; mdblocks = ext4_calc_metadata_amount(inode, total); BUG_ON(mdblocks < EXT4_I(inode)->i_reserved_meta_blocks); md_needed = mdblocks - EXT4_I(inode)->i_reserved_meta_blocks; total = md_needed + nrblocks; if (ext4_claim_free_blocks(sbi, total)) { spin_unlock(&EXT4_I(inode)->i_block_reservation_lock); if (ext4_should_retry_alloc(inode->i_sb, &retries)) { yield(); goto repeat; } return -ENOSPC; } EXT4_I(inode)->i_reserved_data_blocks += nrblocks; EXT4_I(inode)->i_reserved_meta_blocks = mdblocks; spin_unlock(&EXT4_I(inode)->i_block_reservation_lock); return 0; /* success */ } static void ext4_da_release_space(struct inode *inode, int to_free) { struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb); int total, mdb, mdb_free, release; if (!to_free) return; /* Nothing to release, exit */ spin_lock(&EXT4_I(inode)->i_block_reservation_lock); if (!EXT4_I(inode)->i_reserved_data_blocks) { /* * if there is no reserved blocks, but we try to free some * then the counter is messed up somewhere. * but since this function is called from invalidate * page, it's harmless to return without any action */ printk(KERN_INFO "ext4 delalloc try to release %d reserved " "blocks for inode %lu, but there is no reserved " "data blocks\n", to_free, inode->i_ino); spin_unlock(&EXT4_I(inode)->i_block_reservation_lock); return; } /* recalculate the number of metablocks still need to be reserved */ total = EXT4_I(inode)->i_reserved_data_blocks - to_free; mdb = ext4_calc_metadata_amount(inode, total); /* figure out how many metablocks to release */ BUG_ON(mdb > EXT4_I(inode)->i_reserved_meta_blocks); mdb_free = EXT4_I(inode)->i_reserved_meta_blocks - mdb; release = to_free + mdb_free; /* update fs dirty blocks counter for truncate case */ percpu_counter_sub(&sbi->s_dirtyblocks_counter, release); /* update per-inode reservations */ BUG_ON(to_free > EXT4_I(inode)->i_reserved_data_blocks); EXT4_I(inode)->i_reserved_data_blocks -= to_free; BUG_ON(mdb > EXT4_I(inode)->i_reserved_meta_blocks); EXT4_I(inode)->i_reserved_meta_blocks = mdb; spin_unlock(&EXT4_I(inode)->i_block_reservation_lock); } static void ext4_da_page_release_reservation(struct page *page, unsigned long offset) { int to_release = 0; struct buffer_head *head, *bh; unsigned int curr_off = 0; head = page_buffers(page); bh = head; do { unsigned int next_off = curr_off + bh->b_size; if ((offset <= curr_off) && (buffer_delay(bh))) { to_release++; clear_buffer_delay(bh); } curr_off = next_off; } while ((bh = bh->b_this_page) != head); ext4_da_release_space(page->mapping->host, to_release); } /* * Delayed allocation stuff */ struct mpage_da_data { struct inode *inode; struct buffer_head lbh; /* extent of blocks */ unsigned long first_page, next_page; /* extent of pages */ get_block_t *get_block; struct writeback_control *wbc; int io_done; long pages_written; int retval; }; /* * mpage_da_submit_io - walks through extent of pages and try to write * them with writepage() call back * * @mpd->inode: inode * @mpd->first_page: first page of the extent * @mpd->next_page: page after the last page of the extent * @mpd->get_block: the filesystem's block mapper function * * By the time mpage_da_submit_io() is called we expect all blocks * to be allocated. this may be wrong if allocation failed. * * As pages are already locked by write_cache_pages(), we can't use it */ static int mpage_da_submit_io(struct mpage_da_data *mpd) { struct address_space *mapping = mpd->inode->i_mapping; int ret = 0, err, nr_pages, i; unsigned long index, end; struct pagevec pvec; long pages_skipped; BUG_ON(mpd->next_page <= mpd->first_page); pagevec_init(&pvec, 0); index = mpd->first_page; end = mpd->next_page - 1; while (index <= end) { /* * We can use PAGECACHE_TAG_DIRTY lookup here because * even though we have cleared the dirty flag on the page * We still keep the page in the radix tree with tag * PAGECACHE_TAG_DIRTY. See clear_page_dirty_for_io. * The PAGECACHE_TAG_DIRTY is cleared in set_page_writeback * which is called via the below writepage callback. */ nr_pages = pagevec_lookup_tag(&pvec, mapping, &index, PAGECACHE_TAG_DIRTY, min(end - index, (pgoff_t)PAGEVEC_SIZE-1) + 1); if (nr_pages == 0) break; for (i = 0; i < nr_pages; i++) { struct page *page = pvec.pages[i]; pages_skipped = mpd->wbc->pages_skipped; err = mapping->a_ops->writepage(page, mpd->wbc); if (!err && (pages_skipped == mpd->wbc->pages_skipped)) /* * have successfully written the page * without skipping the same */ mpd->pages_written++; /* * In error case, we have to continue because * remaining pages are still locked * XXX: unlock and re-dirty them? */ if (ret == 0) ret = err; } pagevec_release(&pvec); } return ret; } /* * mpage_put_bnr_to_bhs - walk blocks and assign them actual numbers * * @mpd->inode - inode to walk through * @exbh->b_blocknr - first block on a disk * @exbh->b_size - amount of space in bytes * @logical - first logical block to start assignment with * * the function goes through all passed space and put actual disk * block numbers into buffer heads, dropping BH_Delay */ static void mpage_put_bnr_to_bhs(struct mpage_da_data *mpd, sector_t logical, struct buffer_head *exbh) { struct inode *inode = mpd->inode; struct address_space *mapping = inode->i_mapping; int blocks = exbh->b_size >> inode->i_blkbits; sector_t pblock = exbh->b_blocknr, cur_logical; struct buffer_head *head, *bh; pgoff_t index, end; struct pagevec pvec; int nr_pages, i; index = logical >> (PAGE_CACHE_SHIFT - inode->i_blkbits); end = (logical + blocks - 1) >> (PAGE_CACHE_SHIFT - inode->i_blkbits); cur_logical = index << (PAGE_CACHE_SHIFT - inode->i_blkbits); pagevec_init(&pvec, 0); while (index <= end) { /* XXX: optimize tail */ nr_pages = pagevec_lookup(&pvec, mapping, index, PAGEVEC_SIZE); if (nr_pages == 0) break; for (i = 0; i < nr_pages; i++) { struct page *page = pvec.pages[i]; index = page->index; if (index > end) break; index++; BUG_ON(!PageLocked(page)); BUG_ON(PageWriteback(page)); BUG_ON(!page_has_buffers(page)); bh = page_buffers(page); head = bh; /* skip blocks out of the range */ do { if (cur_logical >= logical) break; cur_logical++; } while ((bh = bh->b_this_page) != head); do { if (cur_logical >= logical + blocks) break; if (buffer_delay(bh)) { bh->b_blocknr = pblock; clear_buffer_delay(bh); bh->b_bdev = inode->i_sb->s_bdev; } else if (buffer_unwritten(bh)) { bh->b_blocknr = pblock; clear_buffer_unwritten(bh); set_buffer_mapped(bh); set_buffer_new(bh); bh->b_bdev = inode->i_sb->s_bdev; } else if (buffer_mapped(bh)) BUG_ON(bh->b_blocknr != pblock); cur_logical++; pblock++; } while ((bh = bh->b_this_page) != head); } pagevec_release(&pvec); } } /* * __unmap_underlying_blocks - just a helper function to unmap * set of blocks described by @bh */ static inline void __unmap_underlying_blocks(struct inode *inode, struct buffer_head *bh) { struct block_device *bdev = inode->i_sb->s_bdev; int blocks, i; blocks = bh->b_size >> inode->i_blkbits; for (i = 0; i < blocks; i++) unmap_underlying_metadata(bdev, bh->b_blocknr + i); } static void ext4_da_block_invalidatepages(struct mpage_da_data *mpd, sector_t logical, long blk_cnt) { int nr_pages, i; pgoff_t index, end; struct pagevec pvec; struct inode *inode = mpd->inode; struct address_space *mapping = inode->i_mapping; index = logical >> (PAGE_CACHE_SHIFT - inode->i_blkbits); end = (logical + blk_cnt - 1) >> (PAGE_CACHE_SHIFT - inode->i_blkbits); while (index <= end) { nr_pages = pagevec_lookup(&pvec, mapping, index, PAGEVEC_SIZE); if (nr_pages == 0) break; for (i = 0; i < nr_pages; i++) { struct page *page = pvec.pages[i]; index = page->index; if (index > end) break; index++; BUG_ON(!PageLocked(page)); BUG_ON(PageWriteback(page)); block_invalidatepage(page, 0); ClearPageUptodate(page); unlock_page(page); } } return; } static void ext4_print_free_blocks(struct inode *inode) { struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb); printk(KERN_EMERG "Total free blocks count %lld\n", ext4_count_free_blocks(inode->i_sb)); printk(KERN_EMERG "Free/Dirty block details\n"); printk(KERN_EMERG "free_blocks=%lld\n", percpu_counter_sum(&sbi->s_freeblocks_counter)); printk(KERN_EMERG "dirty_blocks=%lld\n", percpu_counter_sum(&sbi->s_dirtyblocks_counter)); printk(KERN_EMERG "Block reservation details\n"); printk(KERN_EMERG "i_reserved_data_blocks=%lu\n", EXT4_I(inode)->i_reserved_data_blocks); printk(KERN_EMERG "i_reserved_meta_blocks=%lu\n", EXT4_I(inode)->i_reserved_meta_blocks); return; } /* * mpage_da_map_blocks - go through given space * * @mpd->lbh - bh describing space * @mpd->get_block - the filesystem's block mapper function * * The function skips space we know is already mapped to disk blocks. * */ static int mpage_da_map_blocks(struct mpage_da_data *mpd) { int err = 0; struct buffer_head new; struct buffer_head *lbh = &mpd->lbh; sector_t next; /* * We consider only non-mapped and non-allocated blocks */ if (buffer_mapped(lbh) && !buffer_delay(lbh)) return 0; new.b_state = lbh->b_state; new.b_blocknr = 0; new.b_size = lbh->b_size; next = lbh->b_blocknr; /* * If we didn't accumulate anything * to write simply return */ if (!new.b_size) return 0; err = mpd->get_block(mpd->inode, next, &new, 1); if (err) { /* If get block returns with error * we simply return. Later writepage * will redirty the page and writepages * will find the dirty page again */ if (err == -EAGAIN) return 0; if (err == -ENOSPC && ext4_count_free_blocks(mpd->inode->i_sb)) { mpd->retval = err; return 0; } /* * get block failure will cause us * to loop in writepages. Because * a_ops->writepage won't be able to * make progress. The page will be redirtied * by writepage and writepages will again * try to write the same. */ printk(KERN_EMERG "%s block allocation failed for inode %lu " "at logical offset %llu with max blocks " "%zd with error %d\n", __func__, mpd->inode->i_ino, (unsigned long long)next, lbh->b_size >> mpd->inode->i_blkbits, err); printk(KERN_EMERG "This should not happen.!! " "Data will be lost\n"); if (err == -ENOSPC) { ext4_print_free_blocks(mpd->inode); } /* invlaidate all the pages */ ext4_da_block_invalidatepages(mpd, next, lbh->b_size >> mpd->inode->i_blkbits); return err; } BUG_ON(new.b_size == 0); if (buffer_new(&new)) __unmap_underlying_blocks(mpd->inode, &new); /* * If blocks are delayed marked, we need to * put actual blocknr and drop delayed bit */ if (buffer_delay(lbh) || buffer_unwritten(lbh)) mpage_put_bnr_to_bhs(mpd, next, &new); return 0; } #define BH_FLAGS ((1 << BH_Uptodate) | (1 << BH_Mapped) | \ (1 << BH_Delay) | (1 << BH_Unwritten)) /* * mpage_add_bh_to_extent - try to add one more block to extent of blocks * * @mpd->lbh - extent of blocks * @logical - logical number of the block in the file * @bh - bh of the block (used to access block's state) * * the function is used to collect contig. blocks in same state */ static void mpage_add_bh_to_extent(struct mpage_da_data *mpd, sector_t logical, struct buffer_head *bh) { sector_t next; size_t b_size = bh->b_size; struct buffer_head *lbh = &mpd->lbh; int nrblocks = lbh->b_size >> mpd->inode->i_blkbits; /* check if thereserved journal credits might overflow */ if (!(EXT4_I(mpd->inode)->i_flags & EXT4_EXTENTS_FL)) { if (nrblocks >= EXT4_MAX_TRANS_DATA) { /* * With non-extent format we are limited by the journal * credit available. Total credit needed to insert * nrblocks contiguous blocks is dependent on the * nrblocks. So limit nrblocks. */ goto flush_it; } else if ((nrblocks + (b_size >> mpd->inode->i_blkbits)) > EXT4_MAX_TRANS_DATA) { /* * Adding the new buffer_head would make it cross the * allowed limit for which we have journal credit * reserved. So limit the new bh->b_size */ b_size = (EXT4_MAX_TRANS_DATA - nrblocks) << mpd->inode->i_blkbits; /* we will do mpage_da_submit_io in the next loop */ } } /* * First block in the extent */ if (lbh->b_size == 0) { lbh->b_blocknr = logical; lbh->b_size = b_size; lbh->b_state = bh->b_state & BH_FLAGS; return; } next = lbh->b_blocknr + nrblocks; /* * Can we merge the block to our big extent? */ if (logical == next && (bh->b_state & BH_FLAGS) == lbh->b_state) { lbh->b_size += b_size; return; } flush_it: /* * We couldn't merge the block to our extent, so we * need to flush current extent and start new one */ if (mpage_da_map_blocks(mpd) == 0) mpage_da_submit_io(mpd); mpd->io_done = 1; return; } /* * __mpage_da_writepage - finds extent of pages and blocks * * @page: page to consider * @wbc: not used, we just follow rules * @data: context * * The function finds extents of pages and scan them for all blocks. */ static int __mpage_da_writepage(struct page *page, struct writeback_control *wbc, void *data) { struct mpage_da_data *mpd = data; struct inode *inode = mpd->inode; struct buffer_head *bh, *head, fake; sector_t logical; if (mpd->io_done) { /* * Rest of the page in the page_vec * redirty then and skip then. We will * try to to write them again after * starting a new transaction */ redirty_page_for_writepage(wbc, page); unlock_page(page); return MPAGE_DA_EXTENT_TAIL; } /* * Can we merge this page to current extent? */ if (mpd->next_page != page->index) { /* * Nope, we can't. So, we map non-allocated blocks * and start IO on them using writepage() */ if (mpd->next_page != mpd->first_page) { if (mpage_da_map_blocks(mpd) == 0) mpage_da_submit_io(mpd); /* * skip rest of the page in the page_vec */ mpd->io_done = 1; redirty_page_for_writepage(wbc, page); unlock_page(page); return MPAGE_DA_EXTENT_TAIL; } /* * Start next extent of pages ... */ mpd->first_page = page->index; /* * ... and blocks */ mpd->lbh.b_size = 0; mpd->lbh.b_state = 0; mpd->lbh.b_blocknr = 0; } mpd->next_page = page->index + 1; logical = (sector_t) page->index << (PAGE_CACHE_SHIFT - inode->i_blkbits); if (!page_has_buffers(page)) { /* * There is no attached buffer heads yet (mmap?) * we treat the page asfull of dirty blocks */ bh = &fake; bh->b_size = PAGE_CACHE_SIZE; bh->b_state = 0; set_buffer_dirty(bh); set_buffer_uptodate(bh); mpage_add_bh_to_extent(mpd, logical, bh); if (mpd->io_done) return MPAGE_DA_EXTENT_TAIL; } else { /* * Page with regular buffer heads, just add all dirty ones */ head = page_buffers(page); bh = head; do { BUG_ON(buffer_locked(bh)); if (buffer_dirty(bh) && (!buffer_mapped(bh) || buffer_delay(bh))) { mpage_add_bh_to_extent(mpd, logical, bh); if (mpd->io_done) return MPAGE_DA_EXTENT_TAIL; } logical++; } while ((bh = bh->b_this_page) != head); } return 0; } /* * mpage_da_writepages - walk the list of dirty pages of the given * address space, allocates non-allocated blocks, maps newly-allocated * blocks to existing bhs and issue IO them * * @mapping: address space structure to write * @wbc: subtract the number of written pages from *@wbc->nr_to_write * @get_block: the filesystem's block mapper function. * * This is a library function, which implements the writepages() * address_space_operation. */ static int mpage_da_writepages(struct address_space *mapping, struct writeback_control *wbc, struct mpage_da_data *mpd) { int ret; if (!mpd->get_block) return generic_writepages(mapping, wbc); mpd->lbh.b_size = 0; mpd->lbh.b_state = 0; mpd->lbh.b_blocknr = 0; mpd->first_page = 0; mpd->next_page = 0; mpd->io_done = 0; mpd->pages_written = 0; mpd->retval = 0; ret = write_cache_pages(mapping, wbc, __mpage_da_writepage, mpd); /* * Handle last extent of pages */ if (!mpd->io_done && mpd->next_page != mpd->first_page) { if (mpage_da_map_blocks(mpd) == 0) mpage_da_submit_io(mpd); mpd->io_done = 1; ret = MPAGE_DA_EXTENT_TAIL; } wbc->nr_to_write -= mpd->pages_written; return ret; } /* * this is a special callback for ->write_begin() only * it's intention is to return mapped block or reserve space */ static int ext4_da_get_block_prep(struct inode *inode, sector_t iblock, struct buffer_head *bh_result, int create) { int ret = 0; BUG_ON(create == 0); BUG_ON(bh_result->b_size != inode->i_sb->s_blocksize); /* * first, we need to know whether the block is allocated already * preallocated blocks are unmapped but should treated * the same as allocated blocks. */ ret = ext4_get_blocks_wrap(NULL, inode, iblock, 1, bh_result, 0, 0, 0); if ((ret == 0) && !buffer_delay(bh_result)) { /* the block isn't (pre)allocated yet, let's reserve space */ /* * XXX: __block_prepare_write() unmaps passed block, * is it OK? */ ret = ext4_da_reserve_space(inode, 1); if (ret) /* not enough space to reserve */ return ret; map_bh(bh_result, inode->i_sb, 0); set_buffer_new(bh_result); set_buffer_delay(bh_result); } else if (ret > 0) { bh_result->b_size = (ret << inode->i_blkbits); ret = 0; } return ret; } #define EXT4_DELALLOC_RSVED 1 static int ext4_da_get_block_write(struct inode *inode, sector_t iblock, struct buffer_head *bh_result, int create) { int ret; unsigned max_blocks = bh_result->b_size >> inode->i_blkbits; loff_t disksize = EXT4_I(inode)->i_disksize; handle_t *handle = NULL; handle = ext4_journal_current_handle(); BUG_ON(!handle); ret = ext4_get_blocks_wrap(handle, inode, iblock, max_blocks, bh_result, create, 0, EXT4_DELALLOC_RSVED); if (ret > 0) { bh_result->b_size = (ret << inode->i_blkbits); if (ext4_should_order_data(inode)) { int retval; retval = ext4_jbd2_file_inode(handle, inode); if (retval) /* * Failed to add inode for ordered * mode. Don't update file size */ return retval; } /* * Update on-disk size along with block allocation * we don't use 'extend_disksize' as size may change * within already allocated block -bzzz */ disksize = ((loff_t) iblock + ret) << inode->i_blkbits; if (disksize > i_size_read(inode)) disksize = i_size_read(inode); if (disksize > EXT4_I(inode)->i_disksize) { ext4_update_i_disksize(inode, disksize); ret = ext4_mark_inode_dirty(handle, inode); return ret; } ret = 0; } return ret; } static int ext4_bh_unmapped_or_delay(handle_t *handle, struct buffer_head *bh) { /* * unmapped buffer is possible for holes. * delay buffer is possible with delayed allocation */ return ((!buffer_mapped(bh) || buffer_delay(bh)) && buffer_dirty(bh)); } static int ext4_normal_get_block_write(struct inode *inode, sector_t iblock, struct buffer_head *bh_result, int create) { int ret = 0; unsigned max_blocks = bh_result->b_size >> inode->i_blkbits; /* * we don't want to do block allocation in writepage * so call get_block_wrap with create = 0 */ ret = ext4_get_blocks_wrap(NULL, inode, iblock, max_blocks, bh_result, 0, 0, 0); if (ret > 0) { bh_result->b_size = (ret << inode->i_blkbits); ret = 0; } return ret; } /* * get called vi ext4_da_writepages after taking page lock (have journal handle) * get called via journal_submit_inode_data_buffers (no journal handle) * get called via shrink_page_list via pdflush (no journal handle) * or grab_page_cache when doing write_begin (have journal handle) */ static int ext4_da_writepage(struct page *page, struct writeback_control *wbc) { int ret = 0; loff_t size; unsigned long len; struct buffer_head *page_bufs; struct inode *inode = page->mapping->host; size = i_size_read(inode); if (page->index == size >> PAGE_CACHE_SHIFT) l