Merge branch 'mpi-master' into wip-k-fmlpwip-k-fmlp

Conflicts:
	litmus/sched_cedf.c

1 files changed, 124 insertions, 100 deletions

diff --git a/arch/x86/kernel/apic/x2apic_cluster.c b/arch/x86/kernel/apic/x2apic_cluster.c
index cf69c59f4910..500795875827 100644
--- a/arch/x86/kernel/apic/x2apic_cluster.c
+++ b/arch/x86/kernel/apic/x2apic_cluster.c
@@ -5,118 +5,95 @@
 #include <linux/ctype.h>
 #include <linux/init.h>
 #include <linux/dmar.h>
+#include <linux/cpu.h>
 
 #include <asm/smp.h>
-#include <asm/apic.h>
-#include <asm/ipi.h>
+#include <asm/x2apic.h>
 
 static DEFINE_PER_CPU(u32, x86_cpu_to_logical_apicid);
+static DEFINE_PER_CPU(cpumask_var_t, cpus_in_cluster);
+static DEFINE_PER_CPU(cpumask_var_t, ipi_mask);
 
 static int x2apic_acpi_madt_oem_check(char *oem_id, char *oem_table_id)
 {
         return x2apic_enabled();
 }
 
-/*
- * need to use more than cpu 0, because we need more vectors when
- * MSI-X are used.
- */
-static const struct cpumask *x2apic_target_cpus(void)
+static inline u32 x2apic_cluster(int cpu)
 {
-        return cpu_online_mask;
-}
-
-/*
- * for now each logical cpu is in its own vector allocation domain.
- */
-static void x2apic_vector_allocation_domain(int cpu, struct cpumask *retmask)
-{
-        cpumask_clear(retmask);
-        cpumask_set_cpu(cpu, retmask);
+        return per_cpu(x86_cpu_to_logical_apicid, cpu) >> 16;
 }
 
 static void
- __x2apic_send_IPI_dest(unsigned int apicid, int vector, unsigned int dest)
+__x2apic_send_IPI_mask(const struct cpumask *mask, int vector, int apic_dest)
 {
-        unsigned long cfg;
+        struct cpumask *cpus_in_cluster_ptr;
+        struct cpumask *ipi_mask_ptr;
+        unsigned int cpu, this_cpu;
+        unsigned long flags;
+        u32 dest;
+
+        x2apic_wrmsr_fence();
+
+        local_irq_save(flags);
 
-        cfg = __prepare_ICR(0, vector, dest);
+        this_cpu = smp_processor_id();
 
         /*
-         * send the IPI.
+         * We are to modify mask, so we need an own copy
+         * and be sure it's manipulated with irq off.
          */
-        native_x2apic_icr_write(cfg, apicid);
-}
+        ipi_mask_ptr = __raw_get_cpu_var(ipi_mask);
+        cpumask_copy(ipi_mask_ptr, mask);
 
-/*
- * for now, we send the IPI's one by one in the cpumask.
- * TBD: Based on the cpu mask, we can send the IPI's to the cluster group
- * at once. We have 16 cpu's in a cluster. This will minimize IPI register
- * writes.
- */
-static void x2apic_send_IPI_mask(const struct cpumask *mask, int vector)
-{
-        unsigned long query_cpu;
-        unsigned long flags;
+        /*
+         * The idea is to send one IPI per cluster.
+         */
+        for_each_cpu(cpu, ipi_mask_ptr) {
+                unsigned long i;
 
-        x2apic_wrmsr_fence();
+                cpus_in_cluster_ptr = per_cpu(cpus_in_cluster, cpu);
+                dest = 0;
 
-        local_irq_save(flags);
-        for_each_cpu(query_cpu, mask) {
-                __x2apic_send_IPI_dest(
-                        per_cpu(x86_cpu_to_logical_apicid, query_cpu),
-                        vector, apic->dest_logical);
+                /* Collect cpus in cluster. */
+                for_each_cpu_and(i, ipi_mask_ptr, cpus_in_cluster_ptr) {
+                        if (apic_dest == APIC_DEST_ALLINC || i != this_cpu)
+                                dest |= per_cpu(x86_cpu_to_logical_apicid, i);
+                }
+
+                if (!dest)
+                        continue;
+
+                __x2apic_send_IPI_dest(dest, vector, apic->dest_logical);
+                /*
+                 * Cluster sibling cpus should be discared now so
+                 * we would not send IPI them second time.
+                 */
+                cpumask_andnot(ipi_mask_ptr, ipi_mask_ptr, cpus_in_cluster_ptr);
         }
+
         local_irq_restore(flags);
 }
 
+static void x2apic_send_IPI_mask(const struct cpumask *mask, int vector)
+{
+        __x2apic_send_IPI_mask(mask, vector, APIC_DEST_ALLINC);
+}
+
 static void
  x2apic_send_IPI_mask_allbutself(const struct cpumask *mask, int vector)
 {
-        unsigned long this_cpu = smp_processor_id();
-        unsigned long query_cpu;
-        unsigned long flags;
-
-        x2apic_wrmsr_fence();
-
-        local_irq_save(flags);
-        for_each_cpu(query_cpu, mask) {
-                if (query_cpu == this_cpu)
-                        continue;
-                __x2apic_send_IPI_dest(
-                                per_cpu(x86_cpu_to_logical_apicid, query_cpu),
-                                vector, apic->dest_logical);
-        }
-        local_irq_restore(flags);
+        __x2apic_send_IPI_mask(mask, vector, APIC_DEST_ALLBUT);
 }
 
 static void x2apic_send_IPI_allbutself(int vector)
 {
-        unsigned long this_cpu = smp_processor_id();
-        unsigned long query_cpu;
-        unsigned long flags;
-
-        x2apic_wrmsr_fence();
-
-        local_irq_save(flags);
-        for_each_online_cpu(query_cpu) {
-                if (query_cpu == this_cpu)
-                        continue;
-                __x2apic_send_IPI_dest(
-                                per_cpu(x86_cpu_to_logical_apicid, query_cpu),
-                                vector, apic->dest_logical);
-        }
-        local_irq_restore(flags);
+        __x2apic_send_IPI_mask(cpu_online_mask, vector, APIC_DEST_ALLBUT);
 }
 
 static void x2apic_send_IPI_all(int vector)
 {
-        x2apic_send_IPI_mask(cpu_online_mask, vector);
-}
-
-static int x2apic_apic_id_registered(void)
-{
-        return 1;
+        __x2apic_send_IPI_mask(cpu_online_mask, vector, APIC_DEST_ALLINC);
 }
 
 static unsigned int x2apic_cpu_mask_to_apicid(const struct cpumask *cpumask)
@@ -151,43 +128,90 @@ x2apic_cpu_mask_to_apicid_and(const struct cpumask *cpumask,
         return per_cpu(x86_cpu_to_logical_apicid, cpu);
 }
 
-static unsigned int x2apic_cluster_phys_get_apic_id(unsigned long x)
+static void init_x2apic_ldr(void)
 {
-        unsigned int id;
+        unsigned int this_cpu = smp_processor_id();
+        unsigned int cpu;
 
-        id = x;
-        return id;
+        per_cpu(x86_cpu_to_logical_apicid, this_cpu) = apic_read(APIC_LDR);
+
+        __cpu_set(this_cpu, per_cpu(cpus_in_cluster, this_cpu));
+        for_each_online_cpu(cpu) {
+                if (x2apic_cluster(this_cpu) != x2apic_cluster(cpu))
+                        continue;
+                __cpu_set(this_cpu, per_cpu(cpus_in_cluster, cpu));
+                __cpu_set(cpu, per_cpu(cpus_in_cluster, this_cpu));
+        }
 }
 
-static unsigned long set_apic_id(unsigned int id)
+ /*
+  * At CPU state changes, update the x2apic cluster sibling info.
+  */
+static int __cpuinit
+update_clusterinfo(struct notifier_block *nfb, unsigned long action, void *hcpu)
 {
-        unsigned long x;
+        unsigned int this_cpu = (unsigned long)hcpu;
+        unsigned int cpu;
+        int err = 0;
+
+        switch (action) {
+        case CPU_UP_PREPARE:
+                if (!zalloc_cpumask_var(&per_cpu(cpus_in_cluster, this_cpu),
+                                        GFP_KERNEL)) {
+                        err = -ENOMEM;
+                } else if (!zalloc_cpumask_var(&per_cpu(ipi_mask, this_cpu),
+                                               GFP_KERNEL)) {
+                        free_cpumask_var(per_cpu(cpus_in_cluster, this_cpu));
+                        err = -ENOMEM;
+                }
+                break;
+        case CPU_UP_CANCELED:
+        case CPU_UP_CANCELED_FROZEN:
+        case CPU_DEAD:
+                for_each_online_cpu(cpu) {
+                        if (x2apic_cluster(this_cpu) != x2apic_cluster(cpu))
+                                continue;
+                        __cpu_clear(this_cpu, per_cpu(cpus_in_cluster, cpu));
+                        __cpu_clear(cpu, per_cpu(cpus_in_cluster, this_cpu));
+                }
+                free_cpumask_var(per_cpu(cpus_in_cluster, this_cpu));
+                free_cpumask_var(per_cpu(ipi_mask, this_cpu));
+                break;
+        }
 
-        x = id;
-        return x;
+        return notifier_from_errno(err);
 }
 
-static int x2apic_cluster_phys_pkg_id(int initial_apicid, int index_msb)
-{
-        return initial_apicid >> index_msb;
-}
+static struct notifier_block __refdata x2apic_cpu_notifier = {
+        .notifier_call = update_clusterinfo,
+};
 
-static void x2apic_send_IPI_self(int vector)
+static int x2apic_init_cpu_notifier(void)
 {
-        apic_write(APIC_SELF_IPI, vector);
+        int cpu = smp_processor_id();
+
+        zalloc_cpumask_var(&per_cpu(cpus_in_cluster, cpu), GFP_KERNEL);
+        zalloc_cpumask_var(&per_cpu(ipi_mask, cpu), GFP_KERNEL);
+
+        BUG_ON(!per_cpu(cpus_in_cluster, cpu) || !per_cpu(ipi_mask, cpu));
+
+        __cpu_set(cpu, per_cpu(cpus_in_cluster, cpu));
+        register_hotcpu_notifier(&x2apic_cpu_notifier);
+        return 1;
 }
 
-static void init_x2apic_ldr(void)
+static int x2apic_cluster_probe(void)
 {
-        int cpu = smp_processor_id();
-
-        per_cpu(x86_cpu_to_logical_apicid, cpu) = apic_read(APIC_LDR);
+        if (x2apic_mode)
+                return x2apic_init_cpu_notifier();
+        else
+                return 0;
 }
 
-struct apic apic_x2apic_cluster = {
+static struct apic apic_x2apic_cluster = {
 
         .name                           = "cluster x2apic",
-        .probe                          = NULL,
+        .probe                          = x2apic_cluster_probe,
         .acpi_madt_oem_check            = x2apic_acpi_madt_oem_check,
         .apic_id_registered             = x2apic_apic_id_registered,
 
@@ -206,18 +230,16 @@ struct apic apic_x2apic_cluster = {
         .ioapic_phys_id_map             = NULL,
         .setup_apic_routing             = NULL,
         .multi_timer_check              = NULL,
-        .apicid_to_node                 = NULL,
-        .cpu_to_logical_apicid          = NULL,
         .cpu_present_to_apicid          = default_cpu_present_to_apicid,
         .apicid_to_cpu_present          = NULL,
         .setup_portio_remap             = NULL,
         .check_phys_apicid_present      = default_check_phys_apicid_present,
         .enable_apic_mode               = NULL,
-        .phys_pkg_id                    = x2apic_cluster_phys_pkg_id,
+        .phys_pkg_id                    = x2apic_phys_pkg_id,
         .mps_oem_check                  = NULL,
 
-        .get_apic_id                    = x2apic_cluster_phys_get_apic_id,
-        .set_apic_id                    = set_apic_id,
+        .get_apic_id                    = x2apic_get_apic_id,
+        .set_apic_id                    = x2apic_set_apic_id,
         .apic_id_mask                   = 0xFFFFFFFFu,
 
         .cpu_mask_to_apicid             = x2apic_cpu_mask_to_apicid,
@@ -242,3 +264,5 @@ struct apic apic_x2apic_cluster = {
         .wait_icr_idle                  = native_x2apic_wait_icr_idle,
         .safe_wait_icr_idle             = native_safe_x2apic_wait_icr_idle,
 };
+
+apic_driver(apic_x2apic_cluster);