[PATCH] fix RLIM_NOFILE handling

* dup2() should return -EBADF on exceeded sysctl_nr_open
* dup() should *not* return -EINVAL even if you have rlimit set to 0;
  it should get -EMFILE instead.

Check for orig_start exceeding rlimit taken to sys_fcntl().
Failing expand_files() in dup{2,3}() now gets -EMFILE remapped to -EBADF.
Consequently, remaining checks for rlimit are taken to expand_files().

Signed-off-by: Al Viro <viro@zeniv.linux.org.uk>

3 files changed, 15 insertions, 21 deletions

diff --git a/fs/fcntl.c b/fs/fcntl.c
index 3deec9887089..61d625136813 100644
--- a/fs/fcntl.c
+++ b/fs/fcntl.c
@@ -64,11 +64,6 @@ static int locate_fd(unsigned int orig_start, int cloexec)
         struct fdtable *fdt;
 
         spin_lock(&files->file_lock);
-
-        error = -EINVAL;
-        if (orig_start >= current->signal->rlim[RLIMIT_NOFILE].rlim_cur)
-                goto out;
-
 repeat:
         fdt = files_fdtable(files);
         /*
@@ -83,10 +78,6 @@ repeat:
         if (start < fdt->max_fds)
                 newfd = find_next_zero_bit(fdt->open_fds->fds_bits,
                                            fdt->max_fds, start);
-        
-        error = -EMFILE;
-        if (newfd >= current->signal->rlim[RLIMIT_NOFILE].rlim_cur)
-                goto out;
 
         error = expand_files(files, newfd);
         if (error < 0)
@@ -141,13 +132,14 @@ asmlinkage long sys_dup3(unsigned int oldfd, unsigned int newfd, int flags)
         spin_lock(&files->file_lock);
         if (!(file = fcheck(oldfd)))
                 goto out_unlock;
-        if (newfd >= current->signal->rlim[RLIMIT_NOFILE].rlim_cur)
-                goto out_unlock;
         get_file(file);                 /* We are now finished with oldfd */
 
         err = expand_files(files, newfd);
-        if (err < 0)
+        if (unlikely(err < 0)) {
+                if (err == -EMFILE)
+                        err = -EBADF;
                 goto out_fput;
+        }
 
         /* To avoid races with open() and dup(), we will mark the fd as
          * in-use in the open-file bitmap throughout the entire dup2()
@@ -328,6 +320,8 @@ static long do_fcntl(int fd, unsigned int cmd, unsigned long arg,
         switch (cmd) {
         case F_DUPFD:
         case F_DUPFD_CLOEXEC:
+                if (arg >= current->signal->rlim[RLIMIT_NOFILE].rlim_cur)
+                        break;
                 get_file(filp);
                 err = dupfd(filp, arg, cmd == F_DUPFD_CLOEXEC);
                 break;
diff --git a/fs/file.c b/fs/file.c
index 7b3887e054d0..d8773b19fe47 100644
--- a/fs/file.c
+++ b/fs/file.c
@@ -250,9 +250,18 @@ int expand_files(struct files_struct *files, int nr)
         struct fdtable *fdt;
 
         fdt = files_fdtable(files);
+
+        /*
+         * N.B. For clone tasks sharing a files structure, this test
+         * will limit the total number of files that can be opened.
+         */
+        if (nr >= current->signal->rlim[RLIMIT_NOFILE].rlim_cur)
+                return -EMFILE;
+
         /* Do we need to expand? */
         if (nr < fdt->max_fds)
                 return 0;
+
         /* Can we expand? */
         if (nr >= sysctl_nr_open)
                 return -EMFILE;
diff --git a/fs/open.c b/fs/open.c
index 3fe1a6857c75..52647be277a2 100644
--- a/fs/open.c
+++ b/fs/open.c
@@ -972,7 +972,6 @@ int get_unused_fd_flags(int flags)
         int fd, error;
         struct fdtable *fdt;
 
-        error = -EMFILE;
         spin_lock(&files->file_lock);
 
 repeat:
@@ -980,13 +979,6 @@ repeat:
         fd = find_next_zero_bit(fdt->open_fds->fds_bits, fdt->max_fds,
                                 files->next_fd);
 
-        /*
-         * N.B. For clone tasks sharing a files structure, this test
-         * will limit the total number of files that can be opened.
-         */
-        if (fd >= current->signal->rlim[RLIMIT_NOFILE].rlim_cur)
-                goto out;
-
         /* Do we need to expand the fd array or fd set?  */
         error = expand_files(files, fd);
         if (error < 0)
@@ -997,7 +989,6 @@ repeat:
                  * If we needed to expand the fs array we
                  * might have blocked - try again.
                  */
-                error = -EMFILE;
                 goto repeat;
         }