1 files changed, 112 insertions, 8 deletions

diff --git a/kernel/sched/membarrier.c b/kernel/sched/membarrier.c
index 678577267a9a..d2087d5f9837 100644
--- a/kernel/sched/membarrier.c
+++ b/kernel/sched/membarrier.c
@@ -27,7 +27,9 @@
  * except MEMBARRIER_CMD_QUERY.
  */
 #define MEMBARRIER_CMD_BITMASK  \
-        (MEMBARRIER_CMD_SHARED | MEMBARRIER_CMD_PRIVATE_EXPEDITED       \
+        (MEMBARRIER_CMD_GLOBAL | MEMBARRIER_CMD_GLOBAL_EXPEDITED \
+        | MEMBARRIER_CMD_REGISTER_GLOBAL_EXPEDITED \
+        | MEMBARRIER_CMD_PRIVATE_EXPEDITED      \
         | MEMBARRIER_CMD_REGISTER_PRIVATE_EXPEDITED)
 
 static void ipi_mb(void *info)
@@ -35,6 +37,73 @@ static void ipi_mb(void *info)
         smp_mb();       /* IPIs should be serializing but paranoid. */
 }
 
+static int membarrier_global_expedited(void)
+{
+        int cpu;
+        bool fallback = false;
+        cpumask_var_t tmpmask;
+
+        if (num_online_cpus() == 1)
+                return 0;
+
+        /*
+         * Matches memory barriers around rq->curr modification in
+         * scheduler.
+         */
+        smp_mb();       /* system call entry is not a mb. */
+
+        /*
+         * Expedited membarrier commands guarantee that they won't
+         * block, hence the GFP_NOWAIT allocation flag and fallback
+         * implementation.
+         */
+        if (!zalloc_cpumask_var(&tmpmask, GFP_NOWAIT)) {
+                /* Fallback for OOM. */
+                fallback = true;
+        }
+
+        cpus_read_lock();
+        for_each_online_cpu(cpu) {
+                struct task_struct *p;
+
+                /*
+                 * Skipping the current CPU is OK even through we can be
+                 * migrated at any point. The current CPU, at the point
+                 * where we read raw_smp_processor_id(), is ensured to
+                 * be in program order with respect to the caller
+                 * thread. Therefore, we can skip this CPU from the
+                 * iteration.
+                 */
+                if (cpu == raw_smp_processor_id())
+                        continue;
+                rcu_read_lock();
+                p = task_rcu_dereference(&cpu_rq(cpu)->curr);
+                if (p && p->mm && (atomic_read(&p->mm->membarrier_state) &
+                                   MEMBARRIER_STATE_GLOBAL_EXPEDITED)) {
+                        if (!fallback)
+                                __cpumask_set_cpu(cpu, tmpmask);
+                        else
+                                smp_call_function_single(cpu, ipi_mb, NULL, 1);
+                }
+                rcu_read_unlock();
+        }
+        if (!fallback) {
+                preempt_disable();
+                smp_call_function_many(tmpmask, ipi_mb, NULL, 1);
+                preempt_enable();
+                free_cpumask_var(tmpmask);
+        }
+        cpus_read_unlock();
+
+        /*
+         * Memory barrier on the caller thread _after_ we finished
+         * waiting for the last IPI. Matches memory barriers around
+         * rq->curr modification in scheduler.
+         */
+        smp_mb();       /* exit from system call is not a mb */
+        return 0;
+}
+
 static int membarrier_private_expedited(void)
 {
         int cpu;
@@ -105,7 +174,38 @@ static int membarrier_private_expedited(void)
         return 0;
 }
 
-static void membarrier_register_private_expedited(void)
+static int membarrier_register_global_expedited(void)
+{
+        struct task_struct *p = current;
+        struct mm_struct *mm = p->mm;
+
+        if (atomic_read(&mm->membarrier_state) &
+            MEMBARRIER_STATE_GLOBAL_EXPEDITED_READY)
+                return 0;
+        atomic_or(MEMBARRIER_STATE_GLOBAL_EXPEDITED, &mm->membarrier_state);
+        if (atomic_read(&mm->mm_users) == 1 && get_nr_threads(p) == 1) {
+                /*
+                 * For single mm user, single threaded process, we can
+                 * simply issue a memory barrier after setting
+                 * MEMBARRIER_STATE_GLOBAL_EXPEDITED to guarantee that
+                 * no memory access following registration is reordered
+                 * before registration.
+                 */
+                smp_mb();
+        } else {
+                /*
+                 * For multi-mm user threads, we need to ensure all
+                 * future scheduler executions will observe the new
+                 * thread flag state for this mm.
+                 */
+                synchronize_sched();
+        }
+        atomic_or(MEMBARRIER_STATE_GLOBAL_EXPEDITED_READY,
+                  &mm->membarrier_state);
+        return 0;
+}
+
+static int membarrier_register_private_expedited(void)
 {
         struct task_struct *p = current;
         struct mm_struct *mm = p->mm;
@@ -117,7 +217,7 @@ static void membarrier_register_private_expedited(void)
          */
         if (atomic_read(&mm->membarrier_state)
                         & MEMBARRIER_STATE_PRIVATE_EXPEDITED_READY)
-                return;
+                return 0;
         atomic_or(MEMBARRIER_STATE_PRIVATE_EXPEDITED, &mm->membarrier_state);
         if (!(atomic_read(&mm->mm_users) == 1 && get_nr_threads(p) == 1)) {
                 /*
@@ -128,6 +228,7 @@ static void membarrier_register_private_expedited(void)
         }
         atomic_or(MEMBARRIER_STATE_PRIVATE_EXPEDITED_READY,
                         &mm->membarrier_state);
+        return 0;
 }
 
 /**
@@ -167,21 +268,24 @@ SYSCALL_DEFINE2(membarrier, int, cmd, int, flags)
                 int cmd_mask = MEMBARRIER_CMD_BITMASK;
 
                 if (tick_nohz_full_enabled())
-                        cmd_mask &= ~MEMBARRIER_CMD_SHARED;
+                        cmd_mask &= ~MEMBARRIER_CMD_GLOBAL;
                 return cmd_mask;
         }
-        case MEMBARRIER_CMD_SHARED:
-                /* MEMBARRIER_CMD_SHARED is not compatible with nohz_full. */
+        case MEMBARRIER_CMD_GLOBAL:
+                /* MEMBARRIER_CMD_GLOBAL is not compatible with nohz_full. */
                 if (tick_nohz_full_enabled())
                         return -EINVAL;
                 if (num_online_cpus() > 1)
                         synchronize_sched();
                 return 0;
+        case MEMBARRIER_CMD_GLOBAL_EXPEDITED:
+                return membarrier_global_expedited();
+        case MEMBARRIER_CMD_REGISTER_GLOBAL_EXPEDITED:
+                return membarrier_register_global_expedited();
         case MEMBARRIER_CMD_PRIVATE_EXPEDITED:
                 return membarrier_private_expedited();
         case MEMBARRIER_CMD_REGISTER_PRIVATE_EXPEDITED:
-                membarrier_register_private_expedited();
-                return 0;
+                return membarrier_register_private_expedited();
         default:
                 return -EINVAL;
         }