mm, oom: fix and cleanup oom score calculations

The divide in p->signal->oom_score_adj * totalpages / 1000 within
oom_badness() was causing an overflow of the signed long data type.

This adds both the root bias and p->signal->oom_score_adj before doing the
normalization which fixes the issue and also cleans up the calculation.

Tested-by: Dave Jones <davej@redhat.com>
Signed-off-by: David Rientjes <rientjes@google.com>
Cc: KOSAKI Motohiro <kosaki.motohiro@gmail.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>

1 files changed, 7 insertions, 8 deletions

diff --git a/mm/oom_kill.c b/mm/oom_kill.c
index 416637f0e924..77775138e930 100644
--- a/mm/oom_kill.c
+++ b/mm/oom_kill.c
@@ -184,6 +184,7 @@ unsigned long oom_badness(struct task_struct *p, struct mem_cgroup *memcg,
                           const nodemask_t *nodemask, unsigned long totalpages)
 {
         long points;
+        long adj;
 
         if (oom_unkillable_task(p, memcg, nodemask))
                 return 0;
@@ -192,7 +193,8 @@ unsigned long oom_badness(struct task_struct *p, struct mem_cgroup *memcg,
         if (!p)
                 return 0;
 
-        if (p->signal->oom_score_adj == OOM_SCORE_ADJ_MIN) {
+        adj = p->signal->oom_score_adj;
+        if (adj == OOM_SCORE_ADJ_MIN) {
                 task_unlock(p);
                 return 0;
         }
@@ -210,14 +212,11 @@ unsigned long oom_badness(struct task_struct *p, struct mem_cgroup *memcg,
          * implementation used by LSMs.
          */
         if (has_capability_noaudit(p, CAP_SYS_ADMIN))
-                points -= 30 * totalpages / 1000;
+                adj -= 30;
 
-        /*
-         * /proc/pid/oom_score_adj ranges from -1000 to +1000 such that it may
-         * either completely disable oom killing or always prefer a certain
-         * task.
-         */
-        points += p->signal->oom_score_adj * totalpages / 1000;
+        /* Normalize to oom_score_adj units */
+        adj *= totalpages / 1000;
+        points += adj;
 
         /*
          * Never return 0 for an eligible task regardless of the root bonus and