10 files changed, 39 insertions, 39 deletions

diff --git a/arch/mips/kernel/irixelf.c b/arch/mips/kernel/irixelf.c
index 5a3fe43e3019..b997af713eb3 100644
--- a/arch/mips/kernel/irixelf.c
+++ b/arch/mips/kernel/irixelf.c
@@ -44,7 +44,7 @@
 static int load_irix_binary(struct linux_binprm * bprm, struct pt_regs * regs);
 static int load_irix_library(struct file *);
 static int irix_core_dump(long signr, struct pt_regs * regs,
-                          struct file *file);
+                          struct file *file, unsigned long limit);
 
 static struct linux_binfmt irix_format = {
         .module         = THIS_MODULE,
@@ -1091,7 +1091,7 @@ end_coredump:
  * and then they are actually written out.  If we run out of core limit
  * we just truncate.
  */
-static int irix_core_dump(long signr, struct pt_regs * regs, struct file *file)
+static int irix_core_dump(long signr, struct pt_regs *regs, struct file *file, unsigned long limit)
 {
         int has_dumped = 0;
         mm_segment_t fs;
@@ -1101,7 +1101,6 @@ static int irix_core_dump(long signr, struct pt_regs * regs, struct file *file)
         struct vm_area_struct *vma;
         struct elfhdr elf;
         off_t offset = 0, dataoff;
-        int limit = current->signal->rlim[RLIMIT_CORE].rlim_cur;
         int numnote = 3;
         struct memelfnote notes[3];
         struct elf_prstatus prstatus;   /* NT_PRSTATUS */
diff --git a/arch/sparc64/kernel/binfmt_aout32.c b/arch/sparc64/kernel/binfmt_aout32.c
index c8acbeab49b4..92c1b36a2e16 100644
--- a/arch/sparc64/kernel/binfmt_aout32.c
+++ b/arch/sparc64/kernel/binfmt_aout32.c
@@ -35,7 +35,7 @@
 
 static int load_aout32_binary(struct linux_binprm *, struct pt_regs * regs);
 static int load_aout32_library(struct file*);
-static int aout32_core_dump(long signr, struct pt_regs * regs, struct file *file);
+static int aout32_core_dump(long signr, struct pt_regs *regs, struct file *file, unsigned long limit);
 
 static struct linux_binfmt aout32_format = {
         .module         = THIS_MODULE,
@@ -86,7 +86,7 @@ if (file->f_op->llseek) { \
  * dumping of the process results in another error..
  */
 
-static int aout32_core_dump(long signr, struct pt_regs *regs, struct file *file)
+static int aout32_core_dump(long signr, struct pt_regs *regs, struct file *file, unsigned long limit)
 {
         mm_segment_t fs;
         int has_dumped = 0;
@@ -105,13 +105,11 @@ static int aout32_core_dump(long signr, struct pt_regs *regs, struct file *file)
 
 /* If the size of the dump file exceeds the rlimit, then see what would happen
    if we wrote the stack, but not the data area.  */
-        if ((dump.u_dsize+dump.u_ssize) >
-            current->signal->rlim[RLIMIT_CORE].rlim_cur)
+        if (dump.u_dsize + dump.u_ssize > limit)
                 dump.u_dsize = 0;
 
 /* Make sure we have enough room to write the stack and data areas. */
-        if ((dump.u_ssize) >
-            current->signal->rlim[RLIMIT_CORE].rlim_cur)
+        if (dump.u_ssize > limit)
                 dump.u_ssize = 0;
 
 /* make sure we actually have a data and stack area to dump */
diff --git a/arch/x86/ia32/ia32_aout.c b/arch/x86/ia32/ia32_aout.c
index 08781370256d..7cf1c29bf90e 100644
--- a/arch/x86/ia32/ia32_aout.c
+++ b/arch/x86/ia32/ia32_aout.c
@@ -40,7 +40,7 @@ static int load_aout_binary(struct linux_binprm *, struct pt_regs * regs);
 static int load_aout_library(struct file*);
 
 #ifdef CORE_DUMP
-static int aout_core_dump(long signr, struct pt_regs * regs, struct file *file);
+static int aout_core_dump(long signr, struct pt_regs *regs, struct file *file, unsigned long limit);
 
 /*
  * fill in the user structure for a core dump..
@@ -148,7 +148,7 @@ if (file->f_op->llseek) { \
  * dumping of the process results in another error..
  */
 
-static int aout_core_dump(long signr, struct pt_regs * regs, struct file *file)
+static int aout_core_dump(long signr, struct pt_regs *regs, struct file *file, unsigned long limit)
 {
         mm_segment_t fs;
         int has_dumped = 0;
@@ -168,13 +168,11 @@ static int aout_core_dump(long signr, struct pt_regs * regs, struct file *file)
 
 /* If the size of the dump file exceeds the rlimit, then see what would happen
    if we wrote the stack, but not the data area.  */
-        if ((dump.u_dsize+dump.u_ssize+1) * PAGE_SIZE >
-            current->signal->rlim[RLIMIT_CORE].rlim_cur)
+        if ((dump.u_dsize + dump.u_ssize + 1) * PAGE_SIZE > limit)
                 dump.u_dsize = 0;
 
 /* Make sure we have enough room to write the stack and data areas. */
-        if ((dump.u_ssize+1) * PAGE_SIZE >
-            current->signal->rlim[RLIMIT_CORE].rlim_cur)
+        if ((dump.u_ssize + 1) * PAGE_SIZE > limit)
                 dump.u_ssize = 0;
 
 /* make sure we actually have a data and stack area to dump */
diff --git a/fs/binfmt_aout.c b/fs/binfmt_aout.c
index 813a887cd2b3..e176d195e7e5 100644
--- a/fs/binfmt_aout.c
+++ b/fs/binfmt_aout.c
@@ -31,7 +31,7 @@
 
 static int load_aout_binary(struct linux_binprm *, struct pt_regs * regs);
 static int load_aout_library(struct file*);
-static int aout_core_dump(long signr, struct pt_regs * regs, struct file *file);
+static int aout_core_dump(long signr, struct pt_regs *regs, struct file *file, unsigned long limit);
 
 static struct linux_binfmt aout_format = {
         .module         = THIS_MODULE,
@@ -88,7 +88,7 @@ if (file->f_op->llseek) { \
  * dumping of the process results in another error..
  */
 
-static int aout_core_dump(long signr, struct pt_regs * regs, struct file *file)
+static int aout_core_dump(long signr, struct pt_regs *regs, struct file *file, unsigned long limit)
 {
         mm_segment_t fs;
         int has_dumped = 0;
@@ -123,23 +123,19 @@ static int aout_core_dump(long signr, struct pt_regs * regs, struct file *file)
 /* If the size of the dump file exceeds the rlimit, then see what would happen
    if we wrote the stack, but not the data area.  */
 #ifdef __sparc__
-        if ((dump.u_dsize+dump.u_ssize) >
-            current->signal->rlim[RLIMIT_CORE].rlim_cur)
+        if ((dump.u_dsize + dump.u_ssize) > limit)
                 dump.u_dsize = 0;
 #else
-        if ((dump.u_dsize+dump.u_ssize+1) * PAGE_SIZE >
-            current->signal->rlim[RLIMIT_CORE].rlim_cur)
+        if ((dump.u_dsize + dump.u_ssize+1) * PAGE_SIZE > limit)
                 dump.u_dsize = 0;
 #endif
 
 /* Make sure we have enough room to write the stack and data areas. */
 #ifdef __sparc__
-        if ((dump.u_ssize) >
-            current->signal->rlim[RLIMIT_CORE].rlim_cur)
+        if (dump.u_ssize > limit)
                 dump.u_ssize = 0;
 #else
-        if ((dump.u_ssize+1) * PAGE_SIZE >
-            current->signal->rlim[RLIMIT_CORE].rlim_cur)
+        if ((dump.u_ssize + 1) * PAGE_SIZE > limit)
                 dump.u_ssize = 0;
 #endif
 
diff --git a/fs/binfmt_elf.c b/fs/binfmt_elf.c
index 2f31c4c3fd48..3dc6a123fa10 100644
--- a/fs/binfmt_elf.c
+++ b/fs/binfmt_elf.c
@@ -52,7 +52,7 @@ static unsigned long elf_map (struct file *, unsigned long, struct elf_phdr *, i
  * don't even try.
  */
 #if defined(USE_ELF_CORE_DUMP) && defined(CONFIG_ELF_CORE)
-static int elf_core_dump(long signr, struct pt_regs *regs, struct file *file);
+static int elf_core_dump(long signr, struct pt_regs *regs, struct file *file, unsigned long limit);
 #else
 #define elf_core_dump   NULL
 #endif
@@ -1488,7 +1488,7 @@ static struct vm_area_struct *next_vma(struct vm_area_struct *this_vma,
  * and then they are actually written out.  If we run out of core limit
  * we just truncate.
  */
-static int elf_core_dump(long signr, struct pt_regs *regs, struct file *file)
+static int elf_core_dump(long signr, struct pt_regs *regs, struct file *file, unsigned long limit)
 {
 #define NUM_NOTES       6
         int has_dumped = 0;
@@ -1499,7 +1499,6 @@ static int elf_core_dump(long signr, struct pt_regs *regs, struct file *file)
         struct vm_area_struct *vma, *gate_vma;
         struct elfhdr *elf = NULL;
         loff_t offset = 0, dataoff, foffset;
-        unsigned long limit = current->signal->rlim[RLIMIT_CORE].rlim_cur;
         int numnote;
         struct memelfnote *notes = NULL;
         struct elf_prstatus *prstatus = NULL;   /* NT_PRSTATUS */
diff --git a/fs/binfmt_elf_fdpic.c b/fs/binfmt_elf_fdpic.c
index faae02189598..033861c6b8f1 100644
--- a/fs/binfmt_elf_fdpic.c
+++ b/fs/binfmt_elf_fdpic.c
@@ -75,7 +75,7 @@ static int elf_fdpic_map_file_by_direct_mmap(struct elf_fdpic_params *,
                                              struct file *, struct mm_struct *);
 
 #if defined(USE_ELF_CORE_DUMP) && defined(CONFIG_ELF_CORE)
-static int elf_fdpic_core_dump(long, struct pt_regs *, struct file *);
+static int elf_fdpic_core_dump(long, struct pt_regs *, struct file *, unsigned long limit);
 #endif
 
 static struct linux_binfmt elf_fdpic_format = {
@@ -1552,7 +1552,7 @@ static int elf_fdpic_dump_segments(struct file *file, size_t *size,
  * we just truncate.
  */
 static int elf_fdpic_core_dump(long signr, struct pt_regs *regs,
-                               struct file *file)
+                               struct file *file, unsigned long limit)
 {
 #define NUM_NOTES       6
         int has_dumped = 0;
@@ -1563,7 +1563,6 @@ static int elf_fdpic_core_dump(long signr, struct pt_regs *regs,
         struct vm_area_struct *vma;
         struct elfhdr *elf = NULL;
         loff_t offset = 0, dataoff;
-        unsigned long limit = current->signal->rlim[RLIMIT_CORE].rlim_cur;
         int numnote;
         struct memelfnote *notes = NULL;
         struct elf_prstatus *prstatus = NULL;   /* NT_PRSTATUS */
diff --git a/fs/binfmt_flat.c b/fs/binfmt_flat.c
index fcb3405bb14e..265fac868067 100644
--- a/fs/binfmt_flat.c
+++ b/fs/binfmt_flat.c
@@ -75,7 +75,7 @@ static int load_flat_shared_library(int id, struct lib_info *p);
 #endif
 
 static int load_flat_binary(struct linux_binprm *, struct pt_regs * regs);
-static int flat_core_dump(long signr, struct pt_regs * regs, struct file *file);
+static int flat_core_dump(long signr, struct pt_regs *regs, struct file *file, unsigned long limit);
 
 static struct linux_binfmt flat_format = {
         .module         = THIS_MODULE,
@@ -90,7 +90,7 @@ static struct linux_binfmt flat_format = {
  * Currently only a stub-function.
  */
 
-static int flat_core_dump(long signr, struct pt_regs * regs, struct file *file)
+static int flat_core_dump(long signr, struct pt_regs *regs, struct file *file, unsigned long limit)
 {
         printk("Process %s:%d received signr %d and should have core dumped\n",
                         current->comm, current->pid, (int) signr);
diff --git a/fs/binfmt_som.c b/fs/binfmt_som.c
index 5bcdaaf4eae0..9208c41209f9 100644
--- a/fs/binfmt_som.c
+++ b/fs/binfmt_som.c
@@ -44,7 +44,7 @@ static int load_som_library(struct file *);
  * don't even try.
  */
 #if 0
-static int som_core_dump(long signr, struct pt_regs * regs);
+static int som_core_dump(long signr, struct pt_regs *regs, unsigned long limit);
 #else
 #define som_core_dump   NULL
 #endif
diff --git a/fs/exec.c b/fs/exec.c
index 550ae9b22f8d..86c455447bc8 100644
--- a/fs/exec.c
+++ b/fs/exec.c
@@ -1697,6 +1697,7 @@ int do_coredump(long signr, int exit_code, struct pt_regs * regs)
         int fsuid = current->fsuid;
         int flag = 0;
         int ispipe = 0;
+        unsigned long core_limit = current->signal->rlim[RLIMIT_CORE].rlim_cur;
 
         audit_core_dumps(signr);
 
@@ -1730,9 +1731,6 @@ int do_coredump(long signr, int exit_code, struct pt_regs * regs)
          */
         clear_thread_flag(TIF_SIGPENDING);
 
-        if (current->signal->rlim[RLIMIT_CORE].rlim_cur < binfmt->min_coredump)
-                goto fail_unlock;
-
         /*
          * lock_kernel() because format_corename() is controlled by sysctl, which
          * uses lock_kernel()
@@ -1740,7 +1738,20 @@ int do_coredump(long signr, int exit_code, struct pt_regs * regs)
         lock_kernel();
         ispipe = format_corename(corename, core_pattern, signr);
         unlock_kernel();
+        /*
+         * Don't bother to check the RLIMIT_CORE value if core_pattern points
+         * to a pipe.  Since we're not writing directly to the filesystem
+         * RLIMIT_CORE doesn't really apply, as no actual core file will be
+         * created unless the pipe reader choses to write out the core file
+         * at which point file size limits and permissions will be imposed
+         * as it does with any other process
+         */
+        if ((!ispipe) &&
+           (core_limit < binfmt->min_coredump))
+                goto fail_unlock;
+
         if (ispipe) {
+                core_limit = RLIM_INFINITY;
                 /* SIGPIPE can happen, but it's just never processed */
                 if(call_usermodehelper_pipe(corename+1, NULL, NULL, &file)) {
                         printk(KERN_INFO "Core dump to %s pipe failed\n",
@@ -1770,7 +1781,7 @@ int do_coredump(long signr, int exit_code, struct pt_regs * regs)
         if (!ispipe && do_truncate(file->f_path.dentry, 0, 0, file) != 0)
                 goto close_fail;
 
-        retval = binfmt->core_dump(signr, regs, file);
+        retval = binfmt->core_dump(signr, regs, file, core_limit);
 
         if (retval)
                 current->signal->group_exit_code |= 0x80;
diff --git a/include/linux/binfmts.h b/include/linux/binfmts.h
index 63d54366e0e5..3a6512f8ec94 100644
--- a/include/linux/binfmts.h
+++ b/include/linux/binfmts.h
@@ -67,7 +67,7 @@ struct linux_binfmt {
         struct module *module;
         int (*load_binary)(struct linux_binprm *, struct  pt_regs * regs);
         int (*load_shlib)(struct file *);
-        int (*core_dump)(long signr, struct pt_regs * regs, struct file * file);
+        int (*core_dump)(long signr, struct pt_regs *regs, struct file *file, unsigned long limit);
         unsigned long min_coredump;     /* minimal dump size */
         int hasvdso;
 };