oom: prevent livelock when oom_kill_allocating_task is set

When /proc/sys/vm/oom_kill_allocating_task is set for large systems that
want to avoid the lengthy tasklist scan, it's possible to livelock if
current is ineligible for oom kill.  This normally happens when it is set
to OOM_DISABLE, but is also possible if any threads are sharing the same
->mm with a different tgid.

So change __out_of_memory() to fall back to the full task-list scan if it
was unable to kill `current'.

Cc: Nick Piggin <npiggin@suse.de>
Signed-off-by: David Rientjes <rientjes@google.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>

1 files changed, 21 insertions, 23 deletions

diff --git a/mm/oom_kill.c b/mm/oom_kill.c
index 2f3166e308d9..92bcf1db16b2 100644
--- a/mm/oom_kill.c
+++ b/mm/oom_kill.c
@@ -514,34 +514,32 @@ void clear_zonelist_oom(struct zonelist *zonelist, gfp_t gfp_mask)
  */
 static void __out_of_memory(gfp_t gfp_mask, int order)
 {
-        if (sysctl_oom_kill_allocating_task) {
-                oom_kill_process(current, gfp_mask, order, 0, NULL,
-                                "Out of memory (oom_kill_allocating_task)");
-
-        } else {
-                unsigned long points;
-                struct task_struct *p;
-
-retry:
-                /*
-                 * Rambo mode: Shoot down a process and hope it solves whatever
-                 * issues we may have.
-                 */
-                p = select_bad_process(&points, NULL);
+        struct task_struct *p;
+        unsigned long points;
 
-                if (PTR_ERR(p) == -1UL)
+        if (sysctl_oom_kill_allocating_task)
+                if (!oom_kill_process(current, gfp_mask, order, 0, NULL,
+                                "Out of memory (oom_kill_allocating_task)"))
                         return;
+retry:
+        /*
+         * Rambo mode: Shoot down a process and hope it solves whatever
+         * issues we may have.
+         */
+        p = select_bad_process(&points, NULL);
 
-                /* Found nothing?!?! Either we hang forever, or we panic. */
-                if (!p) {
-                        read_unlock(&tasklist_lock);
-                        panic("Out of memory and no killable processes...\n");
-                }
+        if (PTR_ERR(p) == -1UL)
+                return;
 
-                if (oom_kill_process(p, gfp_mask, order, points, NULL,
-                                     "Out of memory"))
-                        goto retry;
+        /* Found nothing?!?! Either we hang forever, or we panic. */
+        if (!p) {
+                read_unlock(&tasklist_lock);
+                panic("Out of memory and no killable processes...\n");
         }
+
+        if (oom_kill_process(p, gfp_mask, order, points, NULL,
+                             "Out of memory"))
+                goto retry;
 }
 
 /*