Merge branch 'for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tj/percpu

* 'for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tj/percpu:
  percpu: add __percpu sparse annotations to what's left
  percpu: add __percpu sparse annotations to fs
  percpu: add __percpu sparse annotations to core kernel subsystems
  local_t: Remove leftover local.h
  this_cpu: Remove pageset_notifier
  this_cpu: Page allocator conversion
  percpu, x86: Generic inc / dec percpu instructions
  local_t: Move local.h include to ringbuffer.c and ring_buffer_benchmark.c
  module: Use this_cpu_xx to dynamically allocate counters
  local_t: Remove cpu_local_xx macros
  percpu: refactor the code in pcpu_[de]populate_chunk()
  percpu: remove compile warnings caused by __verify_pcpu_ptr()
  percpu: make accessors check for percpu pointer in sparse
  percpu: add __percpu for sparse.
  percpu: make access macros universal
  percpu: remove per_cpu__ prefix.

15 files changed, 82 insertions, 83 deletions

diff --git a/include/linux/blktrace_api.h b/include/linux/blktrace_api.h
index 3b73b9992b2..416bf62d6d4 100644
--- a/include/linux/blktrace_api.h
+++ b/include/linux/blktrace_api.h
@@ -150,8 +150,8 @@ struct blk_user_trace_setup {
 struct blk_trace {
         int trace_state;
         struct rchan *rchan;
-        unsigned long *sequence;
-        unsigned char *msg_data;
+        unsigned long __percpu *sequence;
+        unsigned char __percpu *msg_data;
         u16 act_mask;
         u64 start_lba;
         u64 end_lba;
diff --git a/include/linux/compiler.h b/include/linux/compiler.h
index 188fcae10a9..a5a472b1074 100644
--- a/include/linux/compiler.h
+++ b/include/linux/compiler.h
@@ -5,7 +5,7 @@
 
 #ifdef __CHECKER__
 # define __user         __attribute__((noderef, address_space(1)))
-# define __kernel       /* default address space */
+# define __kernel       __attribute__((address_space(0)))
 # define __safe         __attribute__((safe))
 # define __force        __attribute__((force))
 # define __nocast       __attribute__((nocast))
diff --git a/include/linux/dmaengine.h b/include/linux/dmaengine.h
index 78784982b33..21fd9b7c6a4 100644
--- a/include/linux/dmaengine.h
+++ b/include/linux/dmaengine.h
@@ -162,7 +162,7 @@ struct dma_chan {
         struct dma_chan_dev *dev;
 
         struct list_head device_node;
-        struct dma_chan_percpu *local;
+        struct dma_chan_percpu __percpu *local;
         int client_count;
         int table_count;
         void *private;
diff --git a/include/linux/genhd.h b/include/linux/genhd.h
index 9717081c75a..56b50514ab2 100644
--- a/include/linux/genhd.h
+++ b/include/linux/genhd.h
@@ -101,7 +101,7 @@ struct hd_struct {
         unsigned long stamp;
         int in_flight[2];
 #ifdef  CONFIG_SMP
-        struct disk_stats *dkstats;
+        struct disk_stats __percpu *dkstats;
 #else
         struct disk_stats dkstats;
 #endif
diff --git a/include/linux/kexec.h b/include/linux/kexec.h
index c356b6914ff..03e8e8dbc57 100644
--- a/include/linux/kexec.h
+++ b/include/linux/kexec.h
@@ -199,7 +199,7 @@ extern struct kimage *kexec_crash_image;
  */
 extern struct resource crashk_res;
 typedef u32 note_buf_t[KEXEC_NOTE_BYTES/4];
-extern note_buf_t *crash_notes;
+extern note_buf_t __percpu *crash_notes;
 extern u32 vmcoreinfo_note[VMCOREINFO_NOTE_SIZE/4];
 extern size_t vmcoreinfo_size;
 extern size_t vmcoreinfo_max_size;
diff --git a/include/linux/mm.h b/include/linux/mm.h
index 8b2fa8593c6..2e724c877ec 100644
--- a/include/linux/mm.h
+++ b/include/linux/mm.h
@@ -1081,11 +1081,7 @@ extern void si_meminfo(struct sysinfo * val);
 extern void si_meminfo_node(struct sysinfo *val, int nid);
 extern int after_bootmem;
 
-#ifdef CONFIG_NUMA
 extern void setup_per_cpu_pageset(void);
-#else
-static inline void setup_per_cpu_pageset(void) {}
-#endif
 
 extern void zone_pcp_update(struct zone *zone);
 
diff --git a/include/linux/mmzone.h b/include/linux/mmzone.h
index 30fe668c254..41acd4bf766 100644
--- a/include/linux/mmzone.h
+++ b/include/linux/mmzone.h
@@ -184,13 +184,7 @@ struct per_cpu_pageset {
         s8 stat_threshold;
         s8 vm_stat_diff[NR_VM_ZONE_STAT_ITEMS];
 #endif
-} ____cacheline_aligned_in_smp;
-
-#ifdef CONFIG_NUMA
-#define zone_pcp(__z, __cpu) ((__z)->pageset[(__cpu)])
-#else
-#define zone_pcp(__z, __cpu) (&(__z)->pageset[(__cpu)])
-#endif
+};
 
 #endif /* !__GENERATING_BOUNDS.H */
 
@@ -306,10 +300,8 @@ struct zone {
          */
         unsigned long           min_unmapped_pages;
         unsigned long           min_slab_pages;
-        struct per_cpu_pageset  *pageset[NR_CPUS];
-#else
-        struct per_cpu_pageset  pageset[NR_CPUS];
 #endif
+        struct per_cpu_pageset __percpu *pageset;
         /*
          * free areas of different sizes
          */
diff --git a/include/linux/module.h b/include/linux/module.h
index 6cb1a3cab5d..dd618eb026a 100644
--- a/include/linux/module.h
+++ b/include/linux/module.h
@@ -17,7 +17,7 @@
 #include <linux/moduleparam.h>
 #include <linux/tracepoint.h>
 
-#include <asm/local.h>
+#include <linux/percpu.h>
 #include <asm/module.h>
 
 #include <trace/events/module.h>
@@ -363,11 +363,9 @@ struct module
         /* Destruction function. */
         void (*exit)(void);
 
-#ifdef CONFIG_SMP
-        char *refptr;
-#else
-        local_t ref;
-#endif
+        struct module_ref {
+                int count;
+        } __percpu *refptr;
 #endif
 
 #ifdef CONFIG_CONSTRUCTORS
@@ -454,25 +452,16 @@ void __symbol_put(const char *symbol);
 #define symbol_put(x) __symbol_put(MODULE_SYMBOL_PREFIX #x)
 void symbol_put_addr(void *addr);
 
-static inline local_t *__module_ref_addr(struct module *mod, int cpu)
-{
-#ifdef CONFIG_SMP
-        return (local_t *) (mod->refptr + per_cpu_offset(cpu));
-#else
-        return &mod->ref;
-#endif
-}
-
 /* Sometimes we know we already have a refcount, and it's easier not
    to handle the error case (which only happens with rmmod --wait). */
 static inline void __module_get(struct module *module)
 {
         if (module) {
-                unsigned int cpu = get_cpu();
-                local_inc(__module_ref_addr(module, cpu));
+                preempt_disable();
+                __this_cpu_inc(module->refptr->count);
                 trace_module_get(module, _THIS_IP_,
-                                 local_read(__module_ref_addr(module, cpu)));
-                put_cpu();
+                                 __this_cpu_read(module->refptr->count));
+                preempt_enable();
         }
 }
 
@@ -481,15 +470,17 @@ static inline int try_module_get(struct module *module)
         int ret = 1;
 
         if (module) {
-                unsigned int cpu = get_cpu();
+                preempt_disable();
+
                 if (likely(module_is_live(module))) {
-                        local_inc(__module_ref_addr(module, cpu));
+                        __this_cpu_inc(module->refptr->count);
                         trace_module_get(module, _THIS_IP_,
-                                local_read(__module_ref_addr(module, cpu)));
+                                __this_cpu_read(module->refptr->count));
                 }
                 else
                         ret = 0;
-                put_cpu();
+
+                preempt_enable();
         }
         return ret;
 }
diff --git a/include/linux/mount.h b/include/linux/mount.h
index 5d527536486..b5f43a34ef8 100644
--- a/include/linux/mount.h
+++ b/include/linux/mount.h
@@ -66,7 +66,7 @@ struct vfsmount {
         int mnt_pinned;
         int mnt_ghosts;
 #ifdef CONFIG_SMP
-        int *mnt_writers;
+        int __percpu *mnt_writers;
 #else
         int mnt_writers;
 #endif
diff --git a/include/linux/nfs_fs_sb.h b/include/linux/nfs_fs_sb.h
index 34fc6be5bfc..6a2e44fd75e 100644
--- a/include/linux/nfs_fs_sb.h
+++ b/include/linux/nfs_fs_sb.h
@@ -105,7 +105,7 @@ struct nfs_server {
         struct rpc_clnt *       client;         /* RPC client handle */
         struct rpc_clnt *       client_acl;     /* ACL RPC client handle */
         struct nlm_host         *nlm_host;      /* NLM client handle */
-        struct nfs_iostats *    io_stats;       /* I/O statistics */
+        struct nfs_iostats __percpu *io_stats;  /* I/O statistics */
         struct backing_dev_info backing_dev_info;
         atomic_long_t           writeback;      /* number of writeback pages */
         int                     flags;          /* various flags */
diff --git a/include/linux/percpu-defs.h b/include/linux/percpu-defs.h
index 5a5d6ce4bd5..68567c0b3a5 100644
--- a/include/linux/percpu-defs.h
+++ b/include/linux/percpu-defs.h
@@ -2,12 +2,6 @@
 #define _LINUX_PERCPU_DEFS_H
 
 /*
- * Determine the real variable name from the name visible in the
- * kernel sources.
- */
-#define per_cpu_var(var) per_cpu__##var
-
-/*
  * Base implementations of per-CPU variable declarations and definitions, where
  * the section in which the variable is to be placed is provided by the
  * 'sec' argument.  This may be used to affect the parameters governing the
@@ -18,13 +12,23 @@
  * that section.
  */
 #define __PCPU_ATTRS(sec)                                               \
-        __attribute__((section(PER_CPU_BASE_SECTION sec)))              \
+        __percpu __attribute__((section(PER_CPU_BASE_SECTION sec)))     \
         PER_CPU_ATTRIBUTES
 
 #define __PCPU_DUMMY_ATTRS                                              \
         __attribute__((section(".discard"), unused))
 
 /*
+ * Macro which verifies @ptr is a percpu pointer without evaluating
+ * @ptr.  This is to be used in percpu accessors to verify that the
+ * input parameter is a percpu pointer.
+ */
+#define __verify_pcpu_ptr(ptr)  do {                                    \
+        const void __percpu *__vpp_verify = (typeof(ptr))NULL;          \
+        (void)__vpp_verify;                                             \
+} while (0)
+
+/*
  * s390 and alpha modules require percpu variables to be defined as
  * weak to force the compiler to generate GOT based external
  * references for them.  This is necessary because percpu sections
@@ -56,24 +60,24 @@
  */
 #define DECLARE_PER_CPU_SECTION(type, name, sec)                        \
         extern __PCPU_DUMMY_ATTRS char __pcpu_scope_##name;             \
-        extern __PCPU_ATTRS(sec) __typeof__(type) per_cpu__##name
+        extern __PCPU_ATTRS(sec) __typeof__(type) name
 
 #define DEFINE_PER_CPU_SECTION(type, name, sec)                         \
         __PCPU_DUMMY_ATTRS char __pcpu_scope_##name;                    \
         extern __PCPU_DUMMY_ATTRS char __pcpu_unique_##name;            \
         __PCPU_DUMMY_ATTRS char __pcpu_unique_##name;                   \
         __PCPU_ATTRS(sec) PER_CPU_DEF_ATTRIBUTES __weak                 \
-        __typeof__(type) per_cpu__##name
+        __typeof__(type) name
 #else
 /*
  * Normal declaration and definition macros.
  */
 #define DECLARE_PER_CPU_SECTION(type, name, sec)                        \
-        extern __PCPU_ATTRS(sec) __typeof__(type) per_cpu__##name
+        extern __PCPU_ATTRS(sec) __typeof__(type) name
 
 #define DEFINE_PER_CPU_SECTION(type, name, sec)                         \
         __PCPU_ATTRS(sec) PER_CPU_DEF_ATTRIBUTES                        \
-        __typeof__(type) per_cpu__##name
+        __typeof__(type) name
 #endif
 
 /*
@@ -135,10 +139,16 @@
         __aligned(PAGE_SIZE)
 
 /*
- * Intermodule exports for per-CPU variables.
+ * Intermodule exports for per-CPU variables.  sparse forgets about
+ * address space across EXPORT_SYMBOL(), change EXPORT_SYMBOL() to
+ * noop if __CHECKER__.
  */
-#define EXPORT_PER_CPU_SYMBOL(var) EXPORT_SYMBOL(per_cpu__##var)
-#define EXPORT_PER_CPU_SYMBOL_GPL(var) EXPORT_SYMBOL_GPL(per_cpu__##var)
-
+#ifndef __CHECKER__
+#define EXPORT_PER_CPU_SYMBOL(var) EXPORT_SYMBOL(var)
+#define EXPORT_PER_CPU_SYMBOL_GPL(var) EXPORT_SYMBOL_GPL(var)
+#else
+#define EXPORT_PER_CPU_SYMBOL(var)
+#define EXPORT_PER_CPU_SYMBOL_GPL(var)
+#endif
 
 #endif /* _LINUX_PERCPU_DEFS_H */
diff --git a/include/linux/percpu.h b/include/linux/percpu.h
index cf5efbcf716..a93e5bfdccb 100644
--- a/include/linux/percpu.h
+++ b/include/linux/percpu.h
@@ -27,10 +27,17 @@
  * we force a syntax error here if it isn't.
  */
 #define get_cpu_var(var) (*({                           \
-        extern int simple_identifier_##var(void);       \
         preempt_disable();                              \
         &__get_cpu_var(var); }))
-#define put_cpu_var(var) preempt_enable()
+
+/*
+ * The weird & is necessary because sparse considers (void)(var) to be
+ * a direct dereference of percpu variable (var).
+ */
+#define put_cpu_var(var) do {                           \
+        (void)&(var);                                   \
+        preempt_enable();                               \
+} while (0)
 
 #ifdef CONFIG_SMP
 
@@ -127,9 +134,9 @@ extern int __init pcpu_page_first_chunk(size_t reserved_size,
  */
 #define per_cpu_ptr(ptr, cpu)   SHIFT_PERCPU_PTR((ptr), per_cpu_offset((cpu)))
 
-extern void *__alloc_reserved_percpu(size_t size, size_t align);
-extern void *__alloc_percpu(size_t size, size_t align);
-extern void free_percpu(void *__pdata);
+extern void __percpu *__alloc_reserved_percpu(size_t size, size_t align);
+extern void __percpu *__alloc_percpu(size_t size, size_t align);
+extern void free_percpu(void __percpu *__pdata);
 extern phys_addr_t per_cpu_ptr_to_phys(void *addr);
 
 #ifndef CONFIG_HAVE_SETUP_PER_CPU_AREA
@@ -140,7 +147,7 @@ extern void __init setup_per_cpu_areas(void);
 
 #define per_cpu_ptr(ptr, cpu) ({ (void)(cpu); (ptr); })
 
-static inline void *__alloc_percpu(size_t size, size_t align)
+static inline void __percpu *__alloc_percpu(size_t size, size_t align)
 {
         /*
          * Can't easily make larger alignment work with kmalloc.  WARN
@@ -151,7 +158,7 @@ static inline void *__alloc_percpu(size_t size, size_t align)
         return kzalloc(size, GFP_KERNEL);
 }
 
-static inline void free_percpu(void *p)
+static inline void free_percpu(void __percpu *p)
 {
         kfree(p);
 }
@@ -171,7 +178,7 @@ static inline void *pcpu_lpage_remapped(void *kaddr)
 #endif /* CONFIG_SMP */
 
 #define alloc_percpu(type)      \
-        (typeof(type) *)__alloc_percpu(sizeof(type), __alignof__(type))
+        (typeof(type) __percpu *)__alloc_percpu(sizeof(type), __alignof__(type))
 
 /*
  * Optional methods for optimized non-lvalue per-cpu variable access.
@@ -188,17 +195,19 @@ static inline void *pcpu_lpage_remapped(void *kaddr)
 #ifndef percpu_read
 # define percpu_read(var)                                               \
   ({                                                                    \
-        typeof(per_cpu_var(var)) __tmp_var__;                           \
-        __tmp_var__ = get_cpu_var(var);                                 \
-        put_cpu_var(var);                                               \
-        __tmp_var__;                                                    \
+        typeof(var) *pr_ptr__ = &(var);                                 \
+        typeof(var) pr_ret__;                                           \
+        pr_ret__ = get_cpu_var(*pr_ptr__);                              \
+        put_cpu_var(*pr_ptr__);                                         \
+        pr_ret__;                                                       \
   })
 #endif
 
 #define __percpu_generic_to_op(var, val, op)                            \
 do {                                                                    \
-        get_cpu_var(var) op val;                                        \
-        put_cpu_var(var);                                               \
+        typeof(var) *pgto_ptr__ = &(var);                               \
+        get_cpu_var(*pgto_ptr__) op val;                                \
+        put_cpu_var(*pgto_ptr__);                                       \
 } while (0)
 
 #ifndef percpu_write
@@ -234,6 +243,7 @@ extern void __bad_size_call_parameter(void);
 
 #define __pcpu_size_call_return(stem, variable)                         \
 ({      typeof(variable) pscr_ret__;                                    \
+        __verify_pcpu_ptr(&(variable));                                 \
         switch(sizeof(variable)) {                                      \
         case 1: pscr_ret__ = stem##1(variable);break;                   \
         case 2: pscr_ret__ = stem##2(variable);break;                   \
@@ -247,6 +257,7 @@ extern void __bad_size_call_parameter(void);
 
 #define __pcpu_size_call(stem, variable, ...)                           \
 do {                                                                    \
+        __verify_pcpu_ptr(&(variable));                                 \
         switch(sizeof(variable)) {                                      \
                 case 1: stem##1(variable, __VA_ARGS__);break;           \
                 case 2: stem##2(variable, __VA_ARGS__);break;           \
@@ -259,8 +270,7 @@ do {									\
 
 /*
  * Optimized manipulation for memory allocated through the per cpu
- * allocator or for addresses of per cpu variables (can be determined
- * using per_cpu_var(xx).
+ * allocator or for addresses of per cpu variables.
  *
  * These operation guarantee exclusivity of access for other operations
  * on the *same* processor. The assumption is that per cpu data is only
@@ -311,7 +321,7 @@ do {									\
 #define _this_cpu_generic_to_op(pcp, val, op)                           \
 do {                                                                    \
         preempt_disable();                                              \
-        *__this_cpu_ptr(&pcp) op val;                                   \
+        *__this_cpu_ptr(&(pcp)) op val;                                 \
         preempt_enable();                                               \
 } while (0)
 
diff --git a/include/linux/percpu_counter.h b/include/linux/percpu_counter.h
index 794662b2be5..c88d67b5939 100644
--- a/include/linux/percpu_counter.h
+++ b/include/linux/percpu_counter.h
@@ -21,7 +21,7 @@ struct percpu_counter {
 #ifdef CONFIG_HOTPLUG_CPU
         struct list_head list;  /* All percpu_counters are on a list */
 #endif
-        s32 *counters;
+        s32 __percpu *counters;
 };
 
 extern int percpu_counter_batch;
diff --git a/include/linux/srcu.h b/include/linux/srcu.h
index 3084f80909c..4d5ecb222af 100644
--- a/include/linux/srcu.h
+++ b/include/linux/srcu.h
@@ -33,7 +33,7 @@ struct srcu_struct_array {
 
 struct srcu_struct {
         int completed;
-        struct srcu_struct_array *per_cpu_ref;
+        struct srcu_struct_array __percpu *per_cpu_ref;
         struct mutex mutex;
 #ifdef CONFIG_DEBUG_LOCK_ALLOC
         struct lockdep_map dep_map;
diff --git a/include/linux/vmstat.h b/include/linux/vmstat.h
index ee03bba9c5d..117f0dd8ad0 100644
--- a/include/linux/vmstat.h
+++ b/include/linux/vmstat.h
@@ -78,22 +78,22 @@ DECLARE_PER_CPU(struct vm_event_state, vm_event_states);
 
 static inline void __count_vm_event(enum vm_event_item item)
 {
-        __this_cpu_inc(per_cpu_var(vm_event_states).event[item]);
+        __this_cpu_inc(vm_event_states.event[item]);
 }
 
 static inline void count_vm_event(enum vm_event_item item)
 {
-        this_cpu_inc(per_cpu_var(vm_event_states).event[item]);
+        this_cpu_inc(vm_event_states.event[item]);
 }
 
 static inline void __count_vm_events(enum vm_event_item item, long delta)
 {
-        __this_cpu_add(per_cpu_var(vm_event_states).event[item], delta);
+        __this_cpu_add(vm_event_states.event[item], delta);
 }
 
 static inline void count_vm_events(enum vm_event_item item, long delta)
 {
-        this_cpu_add(per_cpu_var(vm_event_states).event[item], delta);
+        this_cpu_add(vm_event_states.event[item], delta);
 }
 
 extern void all_vm_events(unsigned long *);