From 3b663780347ce532b08be1c859b1df14f0eea4c8 Mon Sep 17 00:00:00 2001 From: Sage Weil Date: Wed, 18 May 2011 16:12:12 -0700 Subject: ceph: take reference on mds request r_unsafe_dir We put ourselves on an inode list for the parent directory of metadata operations so that an fsync on the directory will wait for metadata updates to commit to disk. We weren't holding a reference to that directory, however, and under certain workloads (fsstress in this case) the directory can go away. Signed-off-by: Sage Weil --- fs/ceph/mds_client.c | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'fs') diff --git a/fs/ceph/mds_client.c b/fs/ceph/mds_client.c index d0fae4ce9ba..ebce88ab398 100644 --- a/fs/ceph/mds_client.c +++ b/fs/ceph/mds_client.c @@ -578,6 +578,7 @@ static void __register_request(struct ceph_mds_client *mdsc, if (dir) { struct ceph_inode_info *ci = ceph_inode(dir); + ihold(dir); spin_lock(&ci->i_unsafe_lock); req->r_unsafe_dir = dir; list_add_tail(&req->r_unsafe_dir_item, &ci->i_unsafe_dirops); @@ -598,6 +599,9 @@ static void __unregister_request(struct ceph_mds_client *mdsc, spin_lock(&ci->i_unsafe_lock); list_del_init(&req->r_unsafe_dir_item); spin_unlock(&ci->i_unsafe_lock); + + iput(req->r_unsafe_dir); + req->r_unsafe_dir = NULL; } ceph_mdsc_put_request(req); -- cgit v1.2.2 From 1b36698577c1008dc1e63f0bf4b6f3d9deada94a Mon Sep 17 00:00:00 2001 From: Sage Weil Date: Thu, 12 May 2011 14:14:51 -0700 Subject: libceph: remove unused variable Signed-off-by: Sage Weil --- fs/ceph/mds_client.c | 2 -- 1 file changed, 2 deletions(-) (limited to 'fs') diff --git a/fs/ceph/mds_client.c b/fs/ceph/mds_client.c index ebce88ab398..c12d2e9a0ec 100644 --- a/fs/ceph/mds_client.c +++ b/fs/ceph/mds_client.c @@ -2695,7 +2695,6 @@ static void handle_lease(struct ceph_mds_client *mdsc, { struct super_block *sb = mdsc->fsc->sb; struct inode *inode; - struct ceph_inode_info *ci; struct dentry *parent, *dentry; struct ceph_dentry_info *di; int mds = session->s_mds; @@ -2732,7 +2731,6 @@ static void handle_lease(struct ceph_mds_client *mdsc, dout("handle_lease no inode %llx\n", vino.ino); goto release; } - ci = ceph_inode(inode); /* dentry */ parent = d_find_alias(inode); -- cgit v1.2.2 From ae598083015e22d1802617c18d3408971b1bddc0 Mon Sep 17 00:00:00 2001 From: Sage Weil Date: Thu, 12 May 2011 14:28:05 -0700 Subject: ceph: use snprintf for dirstat content We allocate a buffer for rstats if the dirstat option is enabled. Use snprintf. Signed-off-by: Sage Weil --- fs/ceph/dir.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) (limited to 'fs') diff --git a/fs/ceph/dir.c b/fs/ceph/dir.c index 1a867a3601a..53a5eb41785 100644 --- a/fs/ceph/dir.c +++ b/fs/ceph/dir.c @@ -1066,16 +1066,17 @@ static ssize_t ceph_read_dir(struct file *file, char __user *buf, size_t size, struct inode *inode = file->f_dentry->d_inode; struct ceph_inode_info *ci = ceph_inode(inode); int left; + const int bufsize = 1024; if (!ceph_test_mount_opt(ceph_sb_to_client(inode->i_sb), DIRSTAT)) return -EISDIR; if (!cf->dir_info) { - cf->dir_info = kmalloc(1024, GFP_NOFS); + cf->dir_info = kmalloc(bufsize, GFP_NOFS); if (!cf->dir_info) return -ENOMEM; cf->dir_info_len = - sprintf(cf->dir_info, + snprintf(cf->dir_info, bufsize, "entries: %20lld\n" " files: %20lld\n" " subdirs: %20lld\n" -- cgit v1.2.2 From 3540303f87115cbdae6ed2cab44ce6a7676d48d3 Mon Sep 17 00:00:00 2001 From: Sage Weil Date: Thu, 12 May 2011 15:13:23 -0700 Subject: ceph: fix rare potential cap leak If we grab new_cap, retake the lock, and find we already have a cap now for the given mds, release new_cap. Signed-off-by: Sage Weil --- fs/ceph/caps.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'fs') diff --git a/fs/ceph/caps.c b/fs/ceph/caps.c index 2a5404c1c42..591202bc966 100644 --- a/fs/ceph/caps.c +++ b/fs/ceph/caps.c @@ -569,7 +569,8 @@ retry: list_add_tail(&cap->session_caps, &session->s_caps); session->s_nr_caps++; spin_unlock(&session->s_cap_lock); - } + } else if (new_cap) + ceph_put_cap(mdsc, new_cap); if (!ci->i_snap_realm) { /* -- cgit v1.2.2 From da39822c6565095a0151ccf9d6b95e2ae5612885 Mon Sep 17 00:00:00 2001 From: Sage Weil Date: Thu, 12 May 2011 15:28:11 -0700 Subject: ceph: fix broken comparison in readdir loop Both off and fi->offset are unsigned, so the difference is always >= 0. Compare them directly instead of the sign of the difference. Signed-off-by: Sage Weil --- fs/ceph/dir.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'fs') diff --git a/fs/ceph/dir.c b/fs/ceph/dir.c index 53a5eb41785..33729e822bb 100644 --- a/fs/ceph/dir.c +++ b/fs/ceph/dir.c @@ -360,7 +360,7 @@ more: rinfo = &fi->last_readdir->r_reply_info; dout("readdir frag %x num %d off %d chunkoff %d\n", frag, rinfo->dir_nr, off, fi->offset); - while (off - fi->offset >= 0 && off - fi->offset < rinfo->dir_nr) { + while (off >= fi->offset && off - fi->offset < rinfo->dir_nr) { u64 pos = ceph_make_fpos(frag, off); struct ceph_mds_reply_inode *in = rinfo->dir_in[off - fi->offset].in; -- cgit v1.2.2 From 6b4a3b517a767c483d16a200730b2967e0e23b83 Mon Sep 17 00:00:00 2001 From: Sage Weil Date: Thu, 12 May 2011 15:43:48 -0700 Subject: ceph: remove useless check rc is only ever 0 or negative in this method. Signed-off-by: Sage Weil --- fs/ceph/addr.c | 2 -- 1 file changed, 2 deletions(-) (limited to 'fs') diff --git a/fs/ceph/addr.c b/fs/ceph/addr.c index 38b8ab55492..54967268a34 100644 --- a/fs/ceph/addr.c +++ b/fs/ceph/addr.c @@ -880,8 +880,6 @@ release_pvec_pages: out: if (req) ceph_osdc_put_request(req); - if (rc > 0) - rc = 0; /* vfs expects us to return 0 */ ceph_put_snap_context(snapc); dout("writepages done, rc = %d\n", rc); return rc; -- cgit v1.2.2 From 9d6fcb081a4770c3772c51c59c7251c22716d7bb Mon Sep 17 00:00:00 2001 From: Sage Weil Date: Thu, 12 May 2011 15:48:16 -0700 Subject: ceph: check return value for start_request in writepages Since we pass the nofail arg, we should never get an error; BUG if we do. (And fix the function to not return an error if __map_request fails.) Signed-off-by: Sage Weil --- fs/ceph/addr.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'fs') diff --git a/fs/ceph/addr.c b/fs/ceph/addr.c index 54967268a34..33da49dc3cc 100644 --- a/fs/ceph/addr.c +++ b/fs/ceph/addr.c @@ -848,7 +848,8 @@ get_more_pages: op->payload_len = cpu_to_le32(len); req->r_request->hdr.data_len = cpu_to_le32(len); - ceph_osdc_start_request(&fsc->client->osdc, req, true); + rc = ceph_osdc_start_request(&fsc->client->osdc, req, true); + BUG_ON(rc); req = NULL; /* continue? */ -- cgit v1.2.2 From 3c454cf21645bc96668e286f6352ac2c4c895fa2 Mon Sep 17 00:00:00 2001 From: Sage Weil Date: Wed, 6 Apr 2011 09:31:40 -0700 Subject: ceph: use LOOKUPINO to make unconnected nfs fh more reliable If we are unable to locate an inode by ino, ask the MDS using the new LOOKUPINO command. Signed-off-by: Sage Weil --- fs/ceph/export.c | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) (limited to 'fs') diff --git a/fs/ceph/export.c b/fs/ceph/export.c index e41056174bf..f1828af0991 100644 --- a/fs/ceph/export.c +++ b/fs/ceph/export.c @@ -86,6 +86,7 @@ static int ceph_encode_fh(struct dentry *dentry, u32 *rawfh, int *max_len, static struct dentry *__fh_to_dentry(struct super_block *sb, struct ceph_nfs_fh *fh) { + struct ceph_mds_client *mdsc = ceph_sb_to_client(sb)->mdsc; struct inode *inode; struct dentry *dentry; struct ceph_vino vino; @@ -95,8 +96,22 @@ static struct dentry *__fh_to_dentry(struct super_block *sb, vino.ino = fh->ino; vino.snap = CEPH_NOSNAP; inode = ceph_find_inode(sb, vino); - if (!inode) - return ERR_PTR(-ESTALE); + if (!inode) { + struct ceph_mds_request *req; + + req = ceph_mdsc_create_request(mdsc, CEPH_MDS_OP_LOOKUPINO, + USE_ANY_MDS); + if (IS_ERR(req)) + return ERR_CAST(req); + + req->r_ino1 = vino; + req->r_num_caps = 1; + err = ceph_mdsc_do_request(mdsc, NULL, req); + ceph_mdsc_put_request(req); + inode = ceph_find_inode(sb, vino); + if (!inode) + return ERR_PTR(-ESTALE); + } dentry = d_obtain_alias(inode); if (IS_ERR(dentry)) { -- cgit v1.2.2 From 45e3d3eeb6578e523e100622266945ecd71723bb Mon Sep 17 00:00:00 2001 From: Sage Weil Date: Wed, 6 Apr 2011 09:35:00 -0700 Subject: ceph: avoid inode lookup on nfs fh reconnect If we get the inode from the MDS, we have a reference in req; don't do a fresh lookup. Signed-off-by: Sage Weil --- fs/ceph/export.c | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) (limited to 'fs') diff --git a/fs/ceph/export.c b/fs/ceph/export.c index f1828af0991..a610d3d6748 100644 --- a/fs/ceph/export.c +++ b/fs/ceph/export.c @@ -107,8 +107,10 @@ static struct dentry *__fh_to_dentry(struct super_block *sb, req->r_ino1 = vino; req->r_num_caps = 1; err = ceph_mdsc_do_request(mdsc, NULL, req); + inode = req->r_target_inode; + if (inode) + igrab(inode); ceph_mdsc_put_request(req); - inode = ceph_find_inode(sb, vino); if (!inode) return ERR_PTR(-ESTALE); } @@ -163,8 +165,10 @@ static struct dentry *__cfh_to_dentry(struct super_block *sb, snprintf(req->r_path2, 16, "%d", cfh->parent_name_hash); req->r_num_caps = 1; err = ceph_mdsc_do_request(mdsc, NULL, req); + inode = req->r_target_inode; + if (inode) + igrab(inode); ceph_mdsc_put_request(req); - inode = ceph_find_inode(sb, vino); if (!inode) return ERR_PTR(err ? err : -ESTALE); } -- cgit v1.2.2 From db3540522e955c1ebb391f4f5324dff4f20ecd09 Mon Sep 17 00:00:00 2001 From: Sage Weil Date: Tue, 24 May 2011 11:46:31 -0700 Subject: ceph: fix cap flush race reentrancy In e9964c10 we change cap flushing to do a delicate dance because some inodes on the cap_dirty list could be in a migrating state (got EXPORT but not IMPORT) in which we couldn't actually flush and move from dirty->flushing, breaking the while (!empty) { process first } loop structure. It worked for a single sync thread, but was not reentrant and triggered infinite loops when multiple syncers came along. Instead, move inodes with dirty to a separate cap_dirty_migrating list when in the limbo export-but-no-import state, allowing us to go back to the simple loop structure (which was reentrant). This is cleaner and more robust. Audited the cap_dirty users and this looks fine: list_empty(&ci->i_dirty_item) is still a reliable indicator of whether we have dirty caps (which list we're on is irrelevant) and list_del_init() calls still do the right thing. Signed-off-by: Sage Weil --- fs/ceph/caps.c | 58 ++++++++++++++++++++++++++-------------------------- fs/ceph/mds_client.c | 1 + fs/ceph/mds_client.h | 1 + 3 files changed, 31 insertions(+), 29 deletions(-) (limited to 'fs') diff --git a/fs/ceph/caps.c b/fs/ceph/caps.c index 591202bc966..1f72b00447c 100644 --- a/fs/ceph/caps.c +++ b/fs/ceph/caps.c @@ -2635,6 +2635,7 @@ static void handle_cap_export(struct inode *inode, struct ceph_mds_caps *ex, struct ceph_mds_session *session, int *open_target_sessions) { + struct ceph_mds_client *mdsc = ceph_inode_to_client(inode)->mdsc; struct ceph_inode_info *ci = ceph_inode(inode); int mds = session->s_mds; unsigned mseq = le32_to_cpu(ex->migrate_seq); @@ -2671,6 +2672,19 @@ static void handle_cap_export(struct inode *inode, struct ceph_mds_caps *ex, * export targets, so that we get the matching IMPORT */ *open_target_sessions = 1; + + /* + * we can't flush dirty caps that we've seen the + * EXPORT but no IMPORT for + */ + spin_lock(&mdsc->cap_dirty_lock); + if (!list_empty(&ci->i_dirty_item)) { + dout(" moving %p to cap_dirty_migrating\n", + inode); + list_move(&ci->i_dirty_item, + &mdsc->cap_dirty_migrating); + } + spin_unlock(&mdsc->cap_dirty_lock); } __ceph_remove_cap(cap); } @@ -2708,6 +2722,13 @@ static void handle_cap_import(struct ceph_mds_client *mdsc, ci->i_cap_exporting_issued = 0; ci->i_cap_exporting_mseq = 0; ci->i_cap_exporting_mds = -1; + + spin_lock(&mdsc->cap_dirty_lock); + if (!list_empty(&ci->i_dirty_item)) { + dout(" moving %p back to cap_dirty\n", inode); + list_move(&ci->i_dirty_item, &mdsc->cap_dirty); + } + spin_unlock(&mdsc->cap_dirty_lock); } else { dout("handle_cap_import inode %p ci %p mds%d mseq %d\n", inode, ci, mds, mseq); @@ -2911,38 +2932,16 @@ void ceph_check_delayed_caps(struct ceph_mds_client *mdsc) */ void ceph_flush_dirty_caps(struct ceph_mds_client *mdsc) { - struct ceph_inode_info *ci, *nci = NULL; - struct inode *inode, *ninode = NULL; - struct list_head *p, *n; + struct ceph_inode_info *ci; + struct inode *inode; dout("flush_dirty_caps\n"); spin_lock(&mdsc->cap_dirty_lock); - list_for_each_safe(p, n, &mdsc->cap_dirty) { - if (nci) { - ci = nci; - inode = ninode; - ci->i_ceph_flags &= ~CEPH_I_NOFLUSH; - dout("flush_dirty_caps inode %p (was next inode)\n", - inode); - } else { - ci = list_entry(p, struct ceph_inode_info, - i_dirty_item); - inode = igrab(&ci->vfs_inode); - BUG_ON(!inode); - dout("flush_dirty_caps inode %p\n", inode); - } - if (n != &mdsc->cap_dirty) { - nci = list_entry(n, struct ceph_inode_info, - i_dirty_item); - ninode = igrab(&nci->vfs_inode); - BUG_ON(!ninode); - nci->i_ceph_flags |= CEPH_I_NOFLUSH; - dout("flush_dirty_caps next inode %p, noflush\n", - ninode); - } else { - nci = NULL; - ninode = NULL; - } + while (!list_empty(&mdsc->cap_dirty)) { + ci = list_first_entry(&mdsc->cap_dirty, struct ceph_inode_info, + i_dirty_item); + inode = igrab(&ci->vfs_inode); + dout("flush_dirty_caps %p\n", inode); spin_unlock(&mdsc->cap_dirty_lock); if (inode) { ceph_check_caps(ci, CHECK_CAPS_NODELAY|CHECK_CAPS_FLUSH, @@ -2952,6 +2951,7 @@ void ceph_flush_dirty_caps(struct ceph_mds_client *mdsc) spin_lock(&mdsc->cap_dirty_lock); } spin_unlock(&mdsc->cap_dirty_lock); + dout("flush_dirty_caps done\n"); } /* diff --git a/fs/ceph/mds_client.c b/fs/ceph/mds_client.c index c12d2e9a0ec..79743d146be 100644 --- a/fs/ceph/mds_client.c +++ b/fs/ceph/mds_client.c @@ -3004,6 +3004,7 @@ int ceph_mdsc_init(struct ceph_fs_client *fsc) spin_lock_init(&mdsc->snap_flush_lock); mdsc->cap_flush_seq = 0; INIT_LIST_HEAD(&mdsc->cap_dirty); + INIT_LIST_HEAD(&mdsc->cap_dirty_migrating); mdsc->num_cap_flushing = 0; spin_lock_init(&mdsc->cap_dirty_lock); init_waitqueue_head(&mdsc->cap_flushing_wq); diff --git a/fs/ceph/mds_client.h b/fs/ceph/mds_client.h index 4e3a9cc0bba..7d8a0d662d5 100644 --- a/fs/ceph/mds_client.h +++ b/fs/ceph/mds_client.h @@ -278,6 +278,7 @@ struct ceph_mds_client { u64 cap_flush_seq; struct list_head cap_dirty; /* inodes with dirty caps */ + struct list_head cap_dirty_migrating; /* ...that are migration... */ int num_cap_flushing; /* # caps we are flushing */ spinlock_t cap_dirty_lock; /* protects above items */ wait_queue_head_t cap_flushing_wq; -- cgit v1.2.2