From badf16621c1f9d1ac753be056fce11b43d6e0be5 Mon Sep 17 00:00:00 2001 From: Dipankar Sarma Date: Fri, 9 Sep 2005 13:04:10 -0700 Subject: [PATCH] files: break up files struct In order for the RCU to work, the file table array, sets and their sizes must be updated atomically. Instead of ensuring this through too many memory barriers, we put the arrays and their sizes in a separate structure. This patch takes the first step of putting the file table elements in a separate structure fdtable that is embedded withing files_struct. It also changes all the users to refer to the file table using files_fdtable() macro. Subsequent applciation of RCU becomes easier after this. Signed-off-by: Dipankar Sarma Signed-Off-By: David Howells Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- fs/fcntl.c | 47 +++++++++++++++++++++++++++++------------------ 1 file changed, 29 insertions(+), 18 deletions(-) (limited to 'fs/fcntl.c') diff --git a/fs/fcntl.c b/fs/fcntl.c index 6fbc9d8fcc36..bfecc6238083 100644 --- a/fs/fcntl.c +++ b/fs/fcntl.c @@ -24,20 +24,24 @@ void fastcall set_close_on_exec(unsigned int fd, int flag) { struct files_struct *files = current->files; + struct fdtable *fdt; spin_lock(&files->file_lock); + fdt = files_fdtable(files); if (flag) - FD_SET(fd, files->close_on_exec); + FD_SET(fd, fdt->close_on_exec); else - FD_CLR(fd, files->close_on_exec); + FD_CLR(fd, fdt->close_on_exec); spin_unlock(&files->file_lock); } static inline int get_close_on_exec(unsigned int fd) { struct files_struct *files = current->files; + struct fdtable *fdt; int res; spin_lock(&files->file_lock); - res = FD_ISSET(fd, files->close_on_exec); + fdt = files_fdtable(files); + res = FD_ISSET(fd, fdt->close_on_exec); spin_unlock(&files->file_lock); return res; } @@ -54,24 +58,26 @@ static int locate_fd(struct files_struct *files, unsigned int newfd; unsigned int start; int error; + struct fdtable *fdt; error = -EINVAL; if (orig_start >= current->signal->rlim[RLIMIT_NOFILE].rlim_cur) goto out; + fdt = files_fdtable(files); repeat: /* * Someone might have closed fd's in the range - * orig_start..files->next_fd + * orig_start..fdt->next_fd */ start = orig_start; - if (start < files->next_fd) - start = files->next_fd; + if (start < fdt->next_fd) + start = fdt->next_fd; newfd = start; - if (start < files->max_fdset) { - newfd = find_next_zero_bit(files->open_fds->fds_bits, - files->max_fdset, start); + if (start < fdt->max_fdset) { + newfd = find_next_zero_bit(fdt->open_fds->fds_bits, + fdt->max_fdset, start); } error = -EMFILE; @@ -89,8 +95,8 @@ repeat: if (error) goto repeat; - if (start <= files->next_fd) - files->next_fd = newfd + 1; + if (start <= fdt->next_fd) + fdt->next_fd = newfd + 1; error = newfd; @@ -101,13 +107,16 @@ out: static int dupfd(struct file *file, unsigned int start) { struct files_struct * files = current->files; + struct fdtable *fdt; int fd; spin_lock(&files->file_lock); fd = locate_fd(files, file, start); if (fd >= 0) { - FD_SET(fd, files->open_fds); - FD_CLR(fd, files->close_on_exec); + /* locate_fd() may have expanded fdtable, load the ptr */ + fdt = files_fdtable(files); + FD_SET(fd, fdt->open_fds); + FD_CLR(fd, fdt->close_on_exec); spin_unlock(&files->file_lock); fd_install(fd, file); } else { @@ -123,6 +132,7 @@ asmlinkage long sys_dup2(unsigned int oldfd, unsigned int newfd) int err = -EBADF; struct file * file, *tofree; struct files_struct * files = current->files; + struct fdtable *fdt; spin_lock(&files->file_lock); if (!(file = fcheck(oldfd))) @@ -148,13 +158,14 @@ asmlinkage long sys_dup2(unsigned int oldfd, unsigned int newfd) /* Yes. It's a race. In user space. Nothing sane to do */ err = -EBUSY; - tofree = files->fd[newfd]; - if (!tofree && FD_ISSET(newfd, files->open_fds)) + fdt = files_fdtable(files); + tofree = fdt->fd[newfd]; + if (!tofree && FD_ISSET(newfd, fdt->open_fds)) goto out_fput; - files->fd[newfd] = file; - FD_SET(newfd, files->open_fds); - FD_CLR(newfd, files->close_on_exec); + fdt->fd[newfd] = file; + FD_SET(newfd, fdt->open_fds); + FD_CLR(newfd, fdt->close_on_exec); spin_unlock(&files->file_lock); if (tofree) -- cgit v1.2.2 From ab2af1f5005069321c5d130f09cce577b03f43ef Mon Sep 17 00:00:00 2001 From: Dipankar Sarma Date: Fri, 9 Sep 2005 13:04:13 -0700 Subject: [PATCH] files: files struct with RCU Patch to eliminate struct files_struct.file_lock spinlock on the reader side and use rcu refcounting rcuref_xxx api for the f_count refcounter. The updates to the fdtable are done by allocating a new fdtable structure and setting files->fdt to point to the new structure. The fdtable structure is protected by RCU thereby allowing lock-free lookup. For fd arrays/sets that are vmalloced, we use keventd to free them since RCU callbacks can't sleep. A global list of fdtable to be freed is not scalable, so we use a per-cpu list. If keventd is already handling the current cpu's work, we use a timer to defer queueing of that work. Since the last publication, this patch has been re-written to avoid using explicit memory barriers and use rcu_assign_pointer(), rcu_dereference() premitives instead. This required that the fd information is kept in a separate structure (fdtable) and updated atomically. Signed-off-by: Dipankar Sarma Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- fs/fcntl.c | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) (limited to 'fs/fcntl.c') diff --git a/fs/fcntl.c b/fs/fcntl.c index bfecc6238083..d2f3ed8acd93 100644 --- a/fs/fcntl.c +++ b/fs/fcntl.c @@ -16,6 +16,7 @@ #include #include #include +#include #include #include @@ -64,8 +65,8 @@ static int locate_fd(struct files_struct *files, if (orig_start >= current->signal->rlim[RLIMIT_NOFILE].rlim_cur) goto out; - fdt = files_fdtable(files); repeat: + fdt = files_fdtable(files); /* * Someone might have closed fd's in the range * orig_start..fdt->next_fd @@ -95,9 +96,15 @@ repeat: if (error) goto repeat; + /* + * We reacquired files_lock, so we are safe as long as + * we reacquire the fdtable pointer and use it while holding + * the lock, no one can free it during that time. + */ + fdt = files_fdtable(files); if (start <= fdt->next_fd) fdt->next_fd = newfd + 1; - + error = newfd; out: @@ -163,7 +170,7 @@ asmlinkage long sys_dup2(unsigned int oldfd, unsigned int newfd) if (!tofree && FD_ISSET(newfd, fdt->open_fds)) goto out_fput; - fdt->fd[newfd] = file; + rcu_assign_pointer(fdt->fd[newfd], file); FD_SET(newfd, fdt->open_fds); FD_CLR(newfd, fdt->close_on_exec); spin_unlock(&files->file_lock); -- cgit v1.2.2 From b835996f628eadb55c5fb222ba46fe9395bf73c7 Mon Sep 17 00:00:00 2001 From: Dipankar Sarma Date: Fri, 9 Sep 2005 13:04:14 -0700 Subject: [PATCH] files: lock-free fd look-up With the use of RCU in files structure, the look-up of files using fds can now be lock-free. The lookup is protected by rcu_read_lock()/rcu_read_unlock(). This patch changes the readers to use lock-free lookup. Signed-off-by: Maneesh Soni Signed-off-by: Ravikiran Thirumalai Signed-off-by: Dipankar Sarma Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- fs/fcntl.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'fs/fcntl.c') diff --git a/fs/fcntl.c b/fs/fcntl.c index d2f3ed8acd93..863b46e0d78a 100644 --- a/fs/fcntl.c +++ b/fs/fcntl.c @@ -40,10 +40,10 @@ static inline int get_close_on_exec(unsigned int fd) struct files_struct *files = current->files; struct fdtable *fdt; int res; - spin_lock(&files->file_lock); + rcu_read_lock(); fdt = files_fdtable(files); res = FD_ISSET(fd, fdt->close_on_exec); - spin_unlock(&files->file_lock); + rcu_read_unlock(); return res; } -- cgit v1.2.2