Merge branch 'for-3.18-consistent-ops' of git://git.kernel.org/pub/scm/linux/kernel/git/tj/percpu

Pull percpu consistent-ops changes from Tejun Heo:
 "Way back, before the current percpu allocator was implemented, static
  and dynamic percpu memory areas were allocated and handled separately
  and had their own accessors.  The distinction has been gone for many
  years now; however, the now duplicate two sets of accessors remained
  with the pointer based ones - this_cpu_*() - evolving various other
  operations over time.  During the process, we also accumulated other
  inconsistent operations.

  This pull request contains Christoph's patches to clean up the
  duplicate accessor situation.  __get_cpu_var() uses are replaced with
  with this_cpu_ptr() and __this_cpu_ptr() with raw_cpu_ptr().

  Unfortunately, the former sometimes is tricky thanks to C being a bit
  messy with the distinction between lvalues and pointers, which led to
  a rather ugly solution for cpumask_var_t involving the introduction of
  this_cpu_cpumask_var_ptr().

  This converts most of the uses but not all.  Christoph will follow up
  with the remaining conversions in this merge window and hopefully
  remove the obsolete accessors"

* 'for-3.18-consistent-ops' of git://git.kernel.org/pub/scm/linux/kernel/git/tj/percpu: (38 commits)
  irqchip: Properly fetch the per cpu offset
  percpu: Resolve ambiguities in __get_cpu_var/cpumask_var_t -fix
  ia64: sn_nodepda cannot be assigned to after this_cpu conversion. Use __this_cpu_write.
  percpu: Resolve ambiguities in __get_cpu_var/cpumask_var_t
  Revert "powerpc: Replace __get_cpu_var uses"
  percpu: Remove __this_cpu_ptr
  clocksource: Replace __this_cpu_ptr with raw_cpu_ptr
  sparc: Replace __get_cpu_var uses
  avr32: Replace __get_cpu_var with __this_cpu_write
  blackfin: Replace __get_cpu_var uses
  tile: Use this_cpu_ptr() for hardware counters
  tile: Replace __get_cpu_var uses
  powerpc: Replace __get_cpu_var uses
  alpha: Replace __get_cpu_var
  ia64: Replace __get_cpu_var uses
  s390: cio driver &__get_cpu_var replacements
  s390: Replace __get_cpu_var uses
  mips: Replace __get_cpu_var uses
  MIPS: Replace __get_cpu_var uses in FPU emulator.
  arm: Replace __this_cpu_ptr with raw_cpu_ptr
  ...

10 files changed, 45 insertions, 41 deletions

diff --git a/arch/s390/include/asm/irq.h b/arch/s390/include/asm/irq.h
index e787cc1bff8f..b0d5f0a97a01 100644
--- a/arch/s390/include/asm/irq.h
+++ b/arch/s390/include/asm/irq.h
@@ -82,7 +82,7 @@ DECLARE_PER_CPU_SHARED_ALIGNED(struct irq_stat, irq_stat);
 
 static __always_inline void inc_irq_stat(enum interruption_class irq)
 {
-        __get_cpu_var(irq_stat).irqs[irq]++;
+        __this_cpu_inc(irq_stat.irqs[irq]);
 }
 
 struct ext_code {
diff --git a/arch/s390/include/asm/percpu.h b/arch/s390/include/asm/percpu.h
index fa91e0097458..933355e0d091 100644
--- a/arch/s390/include/asm/percpu.h
+++ b/arch/s390/include/asm/percpu.h
@@ -31,7 +31,7 @@
         pcp_op_T__ old__, new__, prev__;                                \
         pcp_op_T__ *ptr__;                                              \
         preempt_disable();                                              \
-        ptr__ = __this_cpu_ptr(&(pcp));                                 \
+        ptr__ = raw_cpu_ptr(&(pcp));                                    \
         prev__ = *ptr__;                                                \
         do {                                                            \
                 old__ = prev__;                                         \
@@ -70,7 +70,7 @@
         pcp_op_T__ val__ = (val);                                       \
         pcp_op_T__ old__, *ptr__;                                       \
         preempt_disable();                                              \
-        ptr__ = __this_cpu_ptr(&(pcp));                                 \
+        ptr__ = raw_cpu_ptr(&(pcp));                            \
         if (__builtin_constant_p(val__) &&                              \
             ((szcast)val__ > -129) && ((szcast)val__ < 128)) {          \
                 asm volatile(                                           \
@@ -97,7 +97,7 @@
         pcp_op_T__ val__ = (val);                                       \
         pcp_op_T__ old__, *ptr__;                                       \
         preempt_disable();                                              \
-        ptr__ = __this_cpu_ptr(&(pcp));                                 \
+        ptr__ = raw_cpu_ptr(&(pcp));                                    \
         asm volatile(                                                   \
                 op "    %[old__],%[val__],%[ptr__]\n"                   \
                 : [old__] "=d" (old__), [ptr__] "+Q" (*ptr__)           \
@@ -116,7 +116,7 @@
         pcp_op_T__ val__ = (val);                                       \
         pcp_op_T__ old__, *ptr__;                                       \
         preempt_disable();                                              \
-        ptr__ = __this_cpu_ptr(&(pcp));                                 \
+        ptr__ = raw_cpu_ptr(&(pcp));                                    \
         asm volatile(                                                   \
                 op "    %[old__],%[val__],%[ptr__]\n"                   \
                 : [old__] "=d" (old__), [ptr__] "+Q" (*ptr__)           \
@@ -138,7 +138,7 @@
         pcp_op_T__ ret__;                                               \
         pcp_op_T__ *ptr__;                                              \
         preempt_disable();                                              \
-        ptr__ = __this_cpu_ptr(&(pcp));                                 \
+        ptr__ = raw_cpu_ptr(&(pcp));                                    \
         ret__ = cmpxchg(ptr__, oval, nval);                             \
         preempt_enable();                                               \
         ret__;                                                          \
@@ -154,7 +154,7 @@
         typeof(pcp) *ptr__;                                             \
         typeof(pcp) ret__;                                              \
         preempt_disable();                                              \
-        ptr__ = __this_cpu_ptr(&(pcp));                                 \
+        ptr__ = raw_cpu_ptr(&(pcp));                                    \
         ret__ = xchg(ptr__, nval);                                      \
         preempt_enable();                                               \
         ret__;                                                          \
@@ -173,8 +173,8 @@
         typeof(pcp2) *p2__;                                             \
         int ret__;                                                      \
         preempt_disable();                                              \
-        p1__ = __this_cpu_ptr(&(pcp1));                                 \
-        p2__ = __this_cpu_ptr(&(pcp2));                                 \
+        p1__ = raw_cpu_ptr(&(pcp1));                                    \
+        p2__ = raw_cpu_ptr(&(pcp2));                                    \
         ret__ = __cmpxchg_double(p1__, p2__, o1__, o2__, n1__, n2__);   \
         preempt_enable();                                               \
         ret__;                                                          \
diff --git a/arch/s390/kernel/idle.c b/arch/s390/kernel/idle.c
index c846aee7372f..7559f1beab29 100644
--- a/arch/s390/kernel/idle.c
+++ b/arch/s390/kernel/idle.c
@@ -21,7 +21,7 @@ static DEFINE_PER_CPU(struct s390_idle_data, s390_idle);
 
 void __kprobes enabled_wait(void)
 {
-        struct s390_idle_data *idle = &__get_cpu_var(s390_idle);
+        struct s390_idle_data *idle = this_cpu_ptr(&s390_idle);
         unsigned long long idle_time;
         unsigned long psw_mask;
 
diff --git a/arch/s390/kernel/kprobes.c b/arch/s390/kernel/kprobes.c
index 27ae5433fe4d..014d4729b134 100644
--- a/arch/s390/kernel/kprobes.c
+++ b/arch/s390/kernel/kprobes.c
@@ -218,9 +218,9 @@ static void __kprobes disable_singlestep(struct kprobe_ctlblk *kcb,
  */
 static void __kprobes push_kprobe(struct kprobe_ctlblk *kcb, struct kprobe *p)
 {
-        kcb->prev_kprobe.kp = __get_cpu_var(current_kprobe);
+        kcb->prev_kprobe.kp = __this_cpu_read(current_kprobe);
         kcb->prev_kprobe.status = kcb->kprobe_status;
-        __get_cpu_var(current_kprobe) = p;
+        __this_cpu_write(current_kprobe, p);
 }
 
 /*
@@ -230,7 +230,7 @@ static void __kprobes push_kprobe(struct kprobe_ctlblk *kcb, struct kprobe *p)
  */
 static void __kprobes pop_kprobe(struct kprobe_ctlblk *kcb)
 {
-        __get_cpu_var(current_kprobe) = kcb->prev_kprobe.kp;
+        __this_cpu_write(current_kprobe, kcb->prev_kprobe.kp);
         kcb->kprobe_status = kcb->prev_kprobe.status;
 }
 
@@ -311,7 +311,7 @@ static int __kprobes kprobe_handler(struct pt_regs *regs)
                 enable_singlestep(kcb, regs, (unsigned long) p->ainsn.insn);
                 return 1;
         } else if (kprobe_running()) {
-                p = __get_cpu_var(current_kprobe);
+                p = __this_cpu_read(current_kprobe);
                 if (p->break_handler && p->break_handler(p, regs)) {
                         /*
                          * Continuation after the jprobe completed and
diff --git a/arch/s390/kernel/nmi.c b/arch/s390/kernel/nmi.c
index db96b418160a..dd1c24ceda50 100644
--- a/arch/s390/kernel/nmi.c
+++ b/arch/s390/kernel/nmi.c
@@ -54,8 +54,12 @@ void s390_handle_mcck(void)
          */
         local_irq_save(flags);
         local_mcck_disable();
-        mcck = __get_cpu_var(cpu_mcck);
-        memset(&__get_cpu_var(cpu_mcck), 0, sizeof(struct mcck_struct));
+        /*
+         * Ummm... Does this make sense at all? Copying the percpu struct
+         * and then zapping it one statement later?
+         */
+        memcpy(&mcck, this_cpu_ptr(&cpu_mcck), sizeof(mcck));
+        memset(&mcck, 0, sizeof(struct mcck_struct));
         clear_cpu_flag(CIF_MCCK_PENDING);
         local_mcck_enable();
         local_irq_restore(flags);
@@ -269,7 +273,7 @@ void notrace s390_do_machine_check(struct pt_regs *regs)
         nmi_enter();
         inc_irq_stat(NMI_NMI);
         mci = (struct mci *) &S390_lowcore.mcck_interruption_code;
-        mcck = &__get_cpu_var(cpu_mcck);
+        mcck = this_cpu_ptr(&cpu_mcck);
         umode = user_mode(regs);
 
         if (mci->sd) {
diff --git a/arch/s390/kernel/perf_cpum_cf.c b/arch/s390/kernel/perf_cpum_cf.c
index d3194de7ae1e..56fdad479115 100644
--- a/arch/s390/kernel/perf_cpum_cf.c
+++ b/arch/s390/kernel/perf_cpum_cf.c
@@ -173,7 +173,7 @@ static int validate_ctr_auth(const struct hw_perf_event *hwc)
  */
 static void cpumf_pmu_enable(struct pmu *pmu)
 {
-        struct cpu_hw_events *cpuhw = &__get_cpu_var(cpu_hw_events);
+        struct cpu_hw_events *cpuhw = this_cpu_ptr(&cpu_hw_events);
         int err;
 
         if (cpuhw->flags & PMU_F_ENABLED)
@@ -196,7 +196,7 @@ static void cpumf_pmu_enable(struct pmu *pmu)
  */
 static void cpumf_pmu_disable(struct pmu *pmu)
 {
-        struct cpu_hw_events *cpuhw = &__get_cpu_var(cpu_hw_events);
+        struct cpu_hw_events *cpuhw = this_cpu_ptr(&cpu_hw_events);
         int err;
         u64 inactive;
 
@@ -230,7 +230,7 @@ static void cpumf_measurement_alert(struct ext_code ext_code,
                 return;
 
         inc_irq_stat(IRQEXT_CMC);
-        cpuhw = &__get_cpu_var(cpu_hw_events);
+        cpuhw = this_cpu_ptr(&cpu_hw_events);
 
         /* Measurement alerts are shared and might happen when the PMU
          * is not reserved.  Ignore these alerts in this case. */
@@ -250,7 +250,7 @@ static void cpumf_measurement_alert(struct ext_code ext_code,
 #define PMC_RELEASE   1
 static void setup_pmc_cpu(void *flags)
 {
-        struct cpu_hw_events *cpuhw = &__get_cpu_var(cpu_hw_events);
+        struct cpu_hw_events *cpuhw = this_cpu_ptr(&cpu_hw_events);
 
         switch (*((int *) flags)) {
         case PMC_INIT:
@@ -475,7 +475,7 @@ static void cpumf_pmu_read(struct perf_event *event)
 
 static void cpumf_pmu_start(struct perf_event *event, int flags)
 {
-        struct cpu_hw_events *cpuhw = &__get_cpu_var(cpu_hw_events);
+        struct cpu_hw_events *cpuhw = this_cpu_ptr(&cpu_hw_events);
         struct hw_perf_event *hwc = &event->hw;
 
         if (WARN_ON_ONCE(!(hwc->state & PERF_HES_STOPPED)))
@@ -506,7 +506,7 @@ static void cpumf_pmu_start(struct perf_event *event, int flags)
 
 static void cpumf_pmu_stop(struct perf_event *event, int flags)
 {
-        struct cpu_hw_events *cpuhw = &__get_cpu_var(cpu_hw_events);
+        struct cpu_hw_events *cpuhw = this_cpu_ptr(&cpu_hw_events);
         struct hw_perf_event *hwc = &event->hw;
 
         if (!(hwc->state & PERF_HES_STOPPED)) {
@@ -527,7 +527,7 @@ static void cpumf_pmu_stop(struct perf_event *event, int flags)
 
 static int cpumf_pmu_add(struct perf_event *event, int flags)
 {
-        struct cpu_hw_events *cpuhw = &__get_cpu_var(cpu_hw_events);
+        struct cpu_hw_events *cpuhw = this_cpu_ptr(&cpu_hw_events);
 
         /* Check authorization for the counter set to which this
          * counter belongs.
@@ -551,7 +551,7 @@ static int cpumf_pmu_add(struct perf_event *event, int flags)
 
 static void cpumf_pmu_del(struct perf_event *event, int flags)
 {
-        struct cpu_hw_events *cpuhw = &__get_cpu_var(cpu_hw_events);
+        struct cpu_hw_events *cpuhw = this_cpu_ptr(&cpu_hw_events);
 
         cpumf_pmu_stop(event, PERF_EF_UPDATE);
 
@@ -575,7 +575,7 @@ static void cpumf_pmu_del(struct perf_event *event, int flags)
  */
 static void cpumf_pmu_start_txn(struct pmu *pmu)
 {
-        struct cpu_hw_events *cpuhw = &__get_cpu_var(cpu_hw_events);
+        struct cpu_hw_events *cpuhw = this_cpu_ptr(&cpu_hw_events);
 
         perf_pmu_disable(pmu);
         cpuhw->flags |= PERF_EVENT_TXN;
@@ -589,7 +589,7 @@ static void cpumf_pmu_start_txn(struct pmu *pmu)
  */
 static void cpumf_pmu_cancel_txn(struct pmu *pmu)
 {
-        struct cpu_hw_events *cpuhw = &__get_cpu_var(cpu_hw_events);
+        struct cpu_hw_events *cpuhw = this_cpu_ptr(&cpu_hw_events);
 
         WARN_ON(cpuhw->tx_state != cpuhw->state);
 
@@ -604,7 +604,7 @@ static void cpumf_pmu_cancel_txn(struct pmu *pmu)
  */
 static int cpumf_pmu_commit_txn(struct pmu *pmu)
 {
-        struct cpu_hw_events *cpuhw = &__get_cpu_var(cpu_hw_events);
+        struct cpu_hw_events *cpuhw = this_cpu_ptr(&cpu_hw_events);
         u64 state;
 
         /* check if the updated state can be scheduled */
diff --git a/arch/s390/kernel/perf_cpum_sf.c b/arch/s390/kernel/perf_cpum_sf.c
index ea0c7b2ef030..08e761318c17 100644
--- a/arch/s390/kernel/perf_cpum_sf.c
+++ b/arch/s390/kernel/perf_cpum_sf.c
@@ -562,7 +562,7 @@ static DEFINE_MUTEX(pmc_reserve_mutex);
 static void setup_pmc_cpu(void *flags)
 {
         int err;
-        struct cpu_hw_sf *cpusf = &__get_cpu_var(cpu_hw_sf);
+        struct cpu_hw_sf *cpusf = this_cpu_ptr(&cpu_hw_sf);
 
         err = 0;
         switch (*((int *) flags)) {
@@ -849,7 +849,7 @@ static int cpumsf_pmu_event_init(struct perf_event *event)
 
 static void cpumsf_pmu_enable(struct pmu *pmu)
 {
-        struct cpu_hw_sf *cpuhw = &__get_cpu_var(cpu_hw_sf);
+        struct cpu_hw_sf *cpuhw = this_cpu_ptr(&cpu_hw_sf);
         struct hw_perf_event *hwc;
         int err;
 
@@ -898,7 +898,7 @@ static void cpumsf_pmu_enable(struct pmu *pmu)
 
 static void cpumsf_pmu_disable(struct pmu *pmu)
 {
-        struct cpu_hw_sf *cpuhw = &__get_cpu_var(cpu_hw_sf);
+        struct cpu_hw_sf *cpuhw = this_cpu_ptr(&cpu_hw_sf);
         struct hws_lsctl_request_block inactive;
         struct hws_qsi_info_block si;
         int err;
@@ -1306,7 +1306,7 @@ static void cpumsf_pmu_read(struct perf_event *event)
  */
 static void cpumsf_pmu_start(struct perf_event *event, int flags)
 {
-        struct cpu_hw_sf *cpuhw = &__get_cpu_var(cpu_hw_sf);
+        struct cpu_hw_sf *cpuhw = this_cpu_ptr(&cpu_hw_sf);
 
         if (WARN_ON_ONCE(!(event->hw.state & PERF_HES_STOPPED)))
                 return;
@@ -1327,7 +1327,7 @@ static void cpumsf_pmu_start(struct perf_event *event, int flags)
  */
 static void cpumsf_pmu_stop(struct perf_event *event, int flags)
 {
-        struct cpu_hw_sf *cpuhw = &__get_cpu_var(cpu_hw_sf);
+        struct cpu_hw_sf *cpuhw = this_cpu_ptr(&cpu_hw_sf);
 
         if (event->hw.state & PERF_HES_STOPPED)
                 return;
@@ -1346,7 +1346,7 @@ static void cpumsf_pmu_stop(struct perf_event *event, int flags)
 
 static int cpumsf_pmu_add(struct perf_event *event, int flags)
 {
-        struct cpu_hw_sf *cpuhw = &__get_cpu_var(cpu_hw_sf);
+        struct cpu_hw_sf *cpuhw = this_cpu_ptr(&cpu_hw_sf);
         int err;
 
         if (cpuhw->flags & PMU_F_IN_USE)
@@ -1397,7 +1397,7 @@ out:
 
 static void cpumsf_pmu_del(struct perf_event *event, int flags)
 {
-        struct cpu_hw_sf *cpuhw = &__get_cpu_var(cpu_hw_sf);
+        struct cpu_hw_sf *cpuhw = this_cpu_ptr(&cpu_hw_sf);
 
         perf_pmu_disable(event->pmu);
         cpumsf_pmu_stop(event, PERF_EF_UPDATE);
@@ -1470,7 +1470,7 @@ static void cpumf_measurement_alert(struct ext_code ext_code,
         if (!(alert & CPU_MF_INT_SF_MASK))
                 return;
         inc_irq_stat(IRQEXT_CMS);
-        cpuhw = &__get_cpu_var(cpu_hw_sf);
+        cpuhw = this_cpu_ptr(&cpu_hw_sf);
 
         /* Measurement alerts are shared and might happen when the PMU
          * is not reserved.  Ignore these alerts in this case. */
diff --git a/arch/s390/kernel/processor.c b/arch/s390/kernel/processor.c
index edefead3b43a..dbdd33ee0102 100644
--- a/arch/s390/kernel/processor.c
+++ b/arch/s390/kernel/processor.c
@@ -23,7 +23,7 @@ static DEFINE_PER_CPU(struct cpuid, cpu_id);
  */
 void cpu_init(void)
 {
-        struct cpuid *id = &__get_cpu_var(cpu_id);
+        struct cpuid *id = this_cpu_ptr(&cpu_id);
 
         get_cpu_id(id);
         atomic_inc(&init_mm.mm_count);
diff --git a/arch/s390/kernel/time.c b/arch/s390/kernel/time.c
index 69e980de0f62..005d665fe4a5 100644
--- a/arch/s390/kernel/time.c
+++ b/arch/s390/kernel/time.c
@@ -92,7 +92,7 @@ void clock_comparator_work(void)
         struct clock_event_device *cd;
 
         S390_lowcore.clock_comparator = -1ULL;
-        cd = &__get_cpu_var(comparators);
+        cd = this_cpu_ptr(&comparators);
         cd->event_handler(cd);
 }
 
@@ -373,7 +373,7 @@ EXPORT_SYMBOL(get_sync_clock);
  */
 static void disable_sync_clock(void *dummy)
 {
-        atomic_t *sw_ptr = &__get_cpu_var(clock_sync_word);
+        atomic_t *sw_ptr = this_cpu_ptr(&clock_sync_word);
         /*
          * Clear the in-sync bit 2^31. All get_sync_clock calls will
          * fail until the sync bit is turned back on. In addition
@@ -390,7 +390,7 @@ static void disable_sync_clock(void *dummy)
  */
 static void enable_sync_clock(void)
 {
-        atomic_t *sw_ptr = &__get_cpu_var(clock_sync_word);
+        atomic_t *sw_ptr = this_cpu_ptr(&clock_sync_word);
         atomic_set_mask(0x80000000, sw_ptr);
 }
 
diff --git a/arch/s390/oprofile/hwsampler.c b/arch/s390/oprofile/hwsampler.c
index e53c6f268807..ff9b4eb34589 100644
--- a/arch/s390/oprofile/hwsampler.c
+++ b/arch/s390/oprofile/hwsampler.c
@@ -178,7 +178,7 @@ static int smp_ctl_qsi(int cpu)
 static void hws_ext_handler(struct ext_code ext_code,
                             unsigned int param32, unsigned long param64)
 {
-        struct hws_cpu_buffer *cb = &__get_cpu_var(sampler_cpu_buffer);
+        struct hws_cpu_buffer *cb = this_cpu_ptr(&sampler_cpu_buffer);
 
         if (!(param32 & CPU_MF_INT_SF_MASK))
                 return;