fs: cache optimise dentry and inode for rcu-walk

Put dentry and inode fields into top of data structure.  This allows RCU path
traversal to perform an RCU dentry lookup in a path walk by touching only the
first 56 bytes of the dentry.

We also fit in 8 bytes of inline name in the first 64 bytes, so for short
names, only 64 bytes needs to be touched to perform the lookup. We should
get rid of the hash->prev pointer from the first 64 bytes, and fit 16 bytes
of name in there, which will take care of 81% rather than 32% of the kernel
tree.

inode is also rearranged so that RCU lookup will only touch a single cacheline
in the inode, plus one in the i_ops structure.

This is important for directory component lookups in RCU path walking. In the
kernel source, directory names average is around 6 chars, so this works.

When we reach the last element of the lookup, we need to lock it and take its
refcount which requires another cacheline access.

Align dentry and inode operations structs, so members will be at predictable
offsets and we can group common operations into head of structure.

Signed-off-by: Nick Piggin <npiggin@kernel.dk>

1 files changed, 23 insertions, 19 deletions

diff --git a/include/linux/fs.h b/include/linux/fs.h
index ea202fff44f8..a04aa96acb71 100644
--- a/include/linux/fs.h
+++ b/include/linux/fs.h
@@ -733,6 +733,20 @@ struct posix_acl;
 #define ACL_NOT_CACHED ((void *)(-1))
 
 struct inode {
+        /* RCU path lookup touches following: */
+        umode_t                 i_mode;
+        uid_t                   i_uid;
+        gid_t                   i_gid;
+        const struct inode_operations   *i_op;
+        struct super_block      *i_sb;
+
+        spinlock_t              i_lock; /* i_blocks, i_bytes, maybe i_size */
+        unsigned int            i_flags;
+        struct mutex            i_mutex;
+
+        unsigned long           i_state;
+        unsigned long           dirtied_when;   /* jiffies of first dirtying */
+
         struct hlist_node       i_hash;
         struct list_head        i_wb_list;      /* backing dev IO list */
         struct list_head        i_lru;          /* inode LRU list */
@@ -744,8 +758,6 @@ struct inode {
         unsigned long           i_ino;
         atomic_t                i_count;
         unsigned int            i_nlink;
-        uid_t                   i_uid;
-        gid_t                   i_gid;
         dev_t                   i_rdev;
         unsigned int            i_blkbits;
         u64                     i_version;
@@ -758,13 +770,8 @@ struct inode {
         struct timespec         i_ctime;
         blkcnt_t                i_blocks;
         unsigned short          i_bytes;
-        umode_t                 i_mode;
-        spinlock_t              i_lock; /* i_blocks, i_bytes, maybe i_size */
-        struct mutex            i_mutex;
         struct rw_semaphore     i_alloc_sem;
-        const struct inode_operations   *i_op;
         const struct file_operations    *i_fop; /* former ->i_op->default_file_ops */
-        struct super_block      *i_sb;
         struct file_lock        *i_flock;
         struct address_space    *i_mapping;
         struct address_space    i_data;
@@ -785,11 +792,6 @@ struct inode {
         struct hlist_head       i_fsnotify_marks;
 #endif
 
-        unsigned long           i_state;
-        unsigned long           dirtied_when;   /* jiffies of first dirtying */
-
-        unsigned int            i_flags;
-
 #ifdef CONFIG_IMA
         /* protected by i_lock */
         unsigned int            i_readcount; /* struct files open RO */
@@ -1549,8 +1551,15 @@ struct file_operations {
 };
 
 struct inode_operations {
-        int (*create) (struct inode *,struct dentry *,int, struct nameidata *);
         struct dentry * (*lookup) (struct inode *,struct dentry *, struct nameidata *);
+        void * (*follow_link) (struct dentry *, struct nameidata *);
+        int (*permission) (struct inode *, int);
+        int (*check_acl)(struct inode *, int);
+
+        int (*readlink) (struct dentry *, char __user *,int);
+        void (*put_link) (struct dentry *, struct nameidata *, void *);
+
+        int (*create) (struct inode *,struct dentry *,int, struct nameidata *);
         int (*link) (struct dentry *,struct inode *,struct dentry *);
         int (*unlink) (struct inode *,struct dentry *);
         int (*symlink) (struct inode *,struct dentry *,const char *);
@@ -1559,12 +1568,7 @@ struct inode_operations {
         int (*mknod) (struct inode *,struct dentry *,int,dev_t);
         int (*rename) (struct inode *, struct dentry *,
                         struct inode *, struct dentry *);
-        int (*readlink) (struct dentry *, char __user *,int);
-        void * (*follow_link) (struct dentry *, struct nameidata *);
-        void (*put_link) (struct dentry *, struct nameidata *, void *);
         void (*truncate) (struct inode *);
-        int (*permission) (struct inode *, int);
-        int (*check_acl)(struct inode *, int);
         int (*setattr) (struct dentry *, struct iattr *);
         int (*getattr) (struct vfsmount *mnt, struct dentry *, struct kstat *);
         int (*setxattr) (struct dentry *, const char *,const void *,size_t,int);
@@ -1576,7 +1580,7 @@ struct inode_operations {
                           loff_t len);
         int (*fiemap)(struct inode *, struct fiemap_extent_info *, u64 start,
                       u64 len);
-};
+} ____cacheline_aligned;
 
 struct seq_file;