aboutsummaryrefslogtreecommitdiffstats
path: root/arch/x86/include
diff options
context:
space:
mode:
Diffstat (limited to 'arch/x86/include')
-rw-r--r--arch/x86/include/asm/amd_nb.h6
-rw-r--r--arch/x86/include/asm/apic.h2
-rw-r--r--arch/x86/include/asm/archrandom.h75
-rw-r--r--arch/x86/include/asm/atomic.h8
-rw-r--r--arch/x86/include/asm/atomic64_64.h6
-rw-r--r--arch/x86/include/asm/cmpxchg.h205
-rw-r--r--arch/x86/include/asm/cmpxchg_32.h114
-rw-r--r--arch/x86/include/asm/cmpxchg_64.h131
-rw-r--r--arch/x86/include/asm/compat.h3
-rw-r--r--arch/x86/include/asm/cpufeature.h1
-rw-r--r--arch/x86/include/asm/device.h2
-rw-r--r--arch/x86/include/asm/dma-mapping.h2
-rw-r--r--arch/x86/include/asm/dwarf2.h2
-rw-r--r--arch/x86/include/asm/elf.h31
-rw-r--r--arch/x86/include/asm/hw_irq.h2
-rw-r--r--arch/x86/include/asm/hyperv.h1
-rw-r--r--arch/x86/include/asm/irq_remapping.h6
-rw-r--r--arch/x86/include/asm/irq_vectors.h12
-rw-r--r--arch/x86/include/asm/nmi.h37
-rw-r--r--arch/x86/include/asm/perf_event.h55
-rw-r--r--arch/x86/include/asm/processor.h4
-rw-r--r--arch/x86/include/asm/reboot.h2
-rw-r--r--arch/x86/include/asm/rwsem.h8
-rw-r--r--arch/x86/include/asm/spinlock.h114
-rw-r--r--arch/x86/include/asm/spinlock_types.h22
-rw-r--r--arch/x86/include/asm/unistd_64.h1
-rw-r--r--arch/x86/include/asm/uv/uv_bau.h7
-rw-r--r--arch/x86/include/asm/uv/uv_hub.h37
-rw-r--r--arch/x86/include/asm/xen/page.h6
29 files changed, 497 insertions, 405 deletions
diff --git a/arch/x86/include/asm/amd_nb.h b/arch/x86/include/asm/amd_nb.h
index 67f87f257611..8e41071704a5 100644
--- a/arch/x86/include/asm/amd_nb.h
+++ b/arch/x86/include/asm/amd_nb.h
@@ -19,9 +19,15 @@ extern int amd_numa_init(void);
19extern int amd_get_subcaches(int); 19extern int amd_get_subcaches(int);
20extern int amd_set_subcaches(int, int); 20extern int amd_set_subcaches(int, int);
21 21
22struct amd_l3_cache {
23 unsigned indices;
24 u8 subcaches[4];
25};
26
22struct amd_northbridge { 27struct amd_northbridge {
23 struct pci_dev *misc; 28 struct pci_dev *misc;
24 struct pci_dev *link; 29 struct pci_dev *link;
30 struct amd_l3_cache l3_cache;
25}; 31};
26 32
27struct amd_northbridge_info { 33struct amd_northbridge_info {
diff --git a/arch/x86/include/asm/apic.h b/arch/x86/include/asm/apic.h
index 7b3ca8324b69..9b7273cb2193 100644
--- a/arch/x86/include/asm/apic.h
+++ b/arch/x86/include/asm/apic.h
@@ -495,7 +495,7 @@ static inline void default_wait_for_init_deassert(atomic_t *deassert)
495 return; 495 return;
496} 496}
497 497
498extern struct apic *generic_bigsmp_probe(void); 498extern void generic_bigsmp_probe(void);
499 499
500 500
501#ifdef CONFIG_X86_LOCAL_APIC 501#ifdef CONFIG_X86_LOCAL_APIC
diff --git a/arch/x86/include/asm/archrandom.h b/arch/x86/include/asm/archrandom.h
new file mode 100644
index 000000000000..0d9ec770f2f8
--- /dev/null
+++ b/arch/x86/include/asm/archrandom.h
@@ -0,0 +1,75 @@
1/*
2 * This file is part of the Linux kernel.
3 *
4 * Copyright (c) 2011, Intel Corporation
5 * Authors: Fenghua Yu <fenghua.yu@intel.com>,
6 * H. Peter Anvin <hpa@linux.intel.com>
7 *
8 * This program is free software; you can redistribute it and/or modify it
9 * under the terms and conditions of the GNU General Public License,
10 * version 2, as published by the Free Software Foundation.
11 *
12 * This program is distributed in the hope it will be useful, but WITHOUT
13 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
14 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
15 * more details.
16 *
17 * You should have received a copy of the GNU General Public License along with
18 * this program; if not, write to the Free Software Foundation, Inc.,
19 * 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
20 *
21 */
22
23#ifndef ASM_X86_ARCHRANDOM_H
24#define ASM_X86_ARCHRANDOM_H
25
26#include <asm/processor.h>
27#include <asm/cpufeature.h>
28#include <asm/alternative.h>
29#include <asm/nops.h>
30
31#define RDRAND_RETRY_LOOPS 10
32
33#define RDRAND_INT ".byte 0x0f,0xc7,0xf0"
34#ifdef CONFIG_X86_64
35# define RDRAND_LONG ".byte 0x48,0x0f,0xc7,0xf0"
36#else
37# define RDRAND_LONG RDRAND_INT
38#endif
39
40#ifdef CONFIG_ARCH_RANDOM
41
42#define GET_RANDOM(name, type, rdrand, nop) \
43static inline int name(type *v) \
44{ \
45 int ok; \
46 alternative_io("movl $0, %0\n\t" \
47 nop, \
48 "\n1: " rdrand "\n\t" \
49 "jc 2f\n\t" \
50 "decl %0\n\t" \
51 "jnz 1b\n\t" \
52 "2:", \
53 X86_FEATURE_RDRAND, \
54 ASM_OUTPUT2("=r" (ok), "=a" (*v)), \
55 "0" (RDRAND_RETRY_LOOPS)); \
56 return ok; \
57}
58
59#ifdef CONFIG_X86_64
60
61GET_RANDOM(arch_get_random_long, unsigned long, RDRAND_LONG, ASM_NOP5);
62GET_RANDOM(arch_get_random_int, unsigned int, RDRAND_INT, ASM_NOP4);
63
64#else
65
66GET_RANDOM(arch_get_random_long, unsigned long, RDRAND_LONG, ASM_NOP3);
67GET_RANDOM(arch_get_random_int, unsigned int, RDRAND_INT, ASM_NOP3);
68
69#endif /* CONFIG_X86_64 */
70
71#endif /* CONFIG_ARCH_RANDOM */
72
73extern void x86_init_rdrand(struct cpuinfo_x86 *c);
74
75#endif /* ASM_X86_ARCHRANDOM_H */
diff --git a/arch/x86/include/asm/atomic.h b/arch/x86/include/asm/atomic.h
index 10572e309ab2..58cb6d4085f7 100644
--- a/arch/x86/include/asm/atomic.h
+++ b/arch/x86/include/asm/atomic.h
@@ -172,18 +172,14 @@ static inline int atomic_add_negative(int i, atomic_t *v)
172 */ 172 */
173static inline int atomic_add_return(int i, atomic_t *v) 173static inline int atomic_add_return(int i, atomic_t *v)
174{ 174{
175 int __i;
176#ifdef CONFIG_M386 175#ifdef CONFIG_M386
176 int __i;
177 unsigned long flags; 177 unsigned long flags;
178 if (unlikely(boot_cpu_data.x86 <= 3)) 178 if (unlikely(boot_cpu_data.x86 <= 3))
179 goto no_xadd; 179 goto no_xadd;
180#endif 180#endif
181 /* Modern 486+ processor */ 181 /* Modern 486+ processor */
182 __i = i; 182 return i + xadd(&v->counter, i);
183 asm volatile(LOCK_PREFIX "xaddl %0, %1"
184 : "+r" (i), "+m" (v->counter)
185 : : "memory");
186 return i + __i;
187 183
188#ifdef CONFIG_M386 184#ifdef CONFIG_M386
189no_xadd: /* Legacy 386 processor */ 185no_xadd: /* Legacy 386 processor */
diff --git a/arch/x86/include/asm/atomic64_64.h b/arch/x86/include/asm/atomic64_64.h
index 017594d403f6..0e1cbfc8ee06 100644
--- a/arch/x86/include/asm/atomic64_64.h
+++ b/arch/x86/include/asm/atomic64_64.h
@@ -170,11 +170,7 @@ static inline int atomic64_add_negative(long i, atomic64_t *v)
170 */ 170 */
171static inline long atomic64_add_return(long i, atomic64_t *v) 171static inline long atomic64_add_return(long i, atomic64_t *v)
172{ 172{
173 long __i = i; 173 return i + xadd(&v->counter, i);
174 asm volatile(LOCK_PREFIX "xaddq %0, %1;"
175 : "+r" (i), "+m" (v->counter)
176 : : "memory");
177 return i + __i;
178} 174}
179 175
180static inline long atomic64_sub_return(long i, atomic64_t *v) 176static inline long atomic64_sub_return(long i, atomic64_t *v)
diff --git a/arch/x86/include/asm/cmpxchg.h b/arch/x86/include/asm/cmpxchg.h
index a460fa088d4c..5d3acdf5a7a6 100644
--- a/arch/x86/include/asm/cmpxchg.h
+++ b/arch/x86/include/asm/cmpxchg.h
@@ -1,5 +1,210 @@
1#ifndef ASM_X86_CMPXCHG_H
2#define ASM_X86_CMPXCHG_H
3
4#include <linux/compiler.h>
5#include <asm/alternative.h> /* Provides LOCK_PREFIX */
6
7/*
8 * Non-existant functions to indicate usage errors at link time
9 * (or compile-time if the compiler implements __compiletime_error().
10 */
11extern void __xchg_wrong_size(void)
12 __compiletime_error("Bad argument size for xchg");
13extern void __cmpxchg_wrong_size(void)
14 __compiletime_error("Bad argument size for cmpxchg");
15extern void __xadd_wrong_size(void)
16 __compiletime_error("Bad argument size for xadd");
17
18/*
19 * Constants for operation sizes. On 32-bit, the 64-bit size it set to
20 * -1 because sizeof will never return -1, thereby making those switch
21 * case statements guaranteeed dead code which the compiler will
22 * eliminate, and allowing the "missing symbol in the default case" to
23 * indicate a usage error.
24 */
25#define __X86_CASE_B 1
26#define __X86_CASE_W 2
27#define __X86_CASE_L 4
28#ifdef CONFIG_64BIT
29#define __X86_CASE_Q 8
30#else
31#define __X86_CASE_Q -1 /* sizeof will never return -1 */
32#endif
33
34/*
35 * Note: no "lock" prefix even on SMP: xchg always implies lock anyway.
36 * Since this is generally used to protect other memory information, we
37 * use "asm volatile" and "memory" clobbers to prevent gcc from moving
38 * information around.
39 */
40#define __xchg(x, ptr, size) \
41({ \
42 __typeof(*(ptr)) __x = (x); \
43 switch (size) { \
44 case __X86_CASE_B: \
45 { \
46 volatile u8 *__ptr = (volatile u8 *)(ptr); \
47 asm volatile("xchgb %0,%1" \
48 : "=q" (__x), "+m" (*__ptr) \
49 : "0" (__x) \
50 : "memory"); \
51 break; \
52 } \
53 case __X86_CASE_W: \
54 { \
55 volatile u16 *__ptr = (volatile u16 *)(ptr); \
56 asm volatile("xchgw %0,%1" \
57 : "=r" (__x), "+m" (*__ptr) \
58 : "0" (__x) \
59 : "memory"); \
60 break; \
61 } \
62 case __X86_CASE_L: \
63 { \
64 volatile u32 *__ptr = (volatile u32 *)(ptr); \
65 asm volatile("xchgl %0,%1" \
66 : "=r" (__x), "+m" (*__ptr) \
67 : "0" (__x) \
68 : "memory"); \
69 break; \
70 } \
71 case __X86_CASE_Q: \
72 { \
73 volatile u64 *__ptr = (volatile u64 *)(ptr); \
74 asm volatile("xchgq %0,%1" \
75 : "=r" (__x), "+m" (*__ptr) \
76 : "0" (__x) \
77 : "memory"); \
78 break; \
79 } \
80 default: \
81 __xchg_wrong_size(); \
82 } \
83 __x; \
84})
85
86#define xchg(ptr, v) \
87 __xchg((v), (ptr), sizeof(*ptr))
88
89/*
90 * Atomic compare and exchange. Compare OLD with MEM, if identical,
91 * store NEW in MEM. Return the initial value in MEM. Success is
92 * indicated by comparing RETURN with OLD.
93 */
94#define __raw_cmpxchg(ptr, old, new, size, lock) \
95({ \
96 __typeof__(*(ptr)) __ret; \
97 __typeof__(*(ptr)) __old = (old); \
98 __typeof__(*(ptr)) __new = (new); \
99 switch (size) { \
100 case __X86_CASE_B: \
101 { \
102 volatile u8 *__ptr = (volatile u8 *)(ptr); \
103 asm volatile(lock "cmpxchgb %2,%1" \
104 : "=a" (__ret), "+m" (*__ptr) \
105 : "q" (__new), "0" (__old) \
106 : "memory"); \
107 break; \
108 } \
109 case __X86_CASE_W: \
110 { \
111 volatile u16 *__ptr = (volatile u16 *)(ptr); \
112 asm volatile(lock "cmpxchgw %2,%1" \
113 : "=a" (__ret), "+m" (*__ptr) \
114 : "r" (__new), "0" (__old) \
115 : "memory"); \
116 break; \
117 } \
118 case __X86_CASE_L: \
119 { \
120 volatile u32 *__ptr = (volatile u32 *)(ptr); \
121 asm volatile(lock "cmpxchgl %2,%1" \
122 : "=a" (__ret), "+m" (*__ptr) \
123 : "r" (__new), "0" (__old) \
124 : "memory"); \
125 break; \
126 } \
127 case __X86_CASE_Q: \
128 { \
129 volatile u64 *__ptr = (volatile u64 *)(ptr); \
130 asm volatile(lock "cmpxchgq %2,%1" \
131 : "=a" (__ret), "+m" (*__ptr) \
132 : "r" (__new), "0" (__old) \
133 : "memory"); \
134 break; \
135 } \
136 default: \
137 __cmpxchg_wrong_size(); \
138 } \
139 __ret; \
140})
141
142#define __cmpxchg(ptr, old, new, size) \
143 __raw_cmpxchg((ptr), (old), (new), (size), LOCK_PREFIX)
144
145#define __sync_cmpxchg(ptr, old, new, size) \
146 __raw_cmpxchg((ptr), (old), (new), (size), "lock; ")
147
148#define __cmpxchg_local(ptr, old, new, size) \
149 __raw_cmpxchg((ptr), (old), (new), (size), "")
150
1#ifdef CONFIG_X86_32 151#ifdef CONFIG_X86_32
2# include "cmpxchg_32.h" 152# include "cmpxchg_32.h"
3#else 153#else
4# include "cmpxchg_64.h" 154# include "cmpxchg_64.h"
5#endif 155#endif
156
157#ifdef __HAVE_ARCH_CMPXCHG
158#define cmpxchg(ptr, old, new) \
159 __cmpxchg((ptr), (old), (new), sizeof(*ptr))
160
161#define sync_cmpxchg(ptr, old, new) \
162 __sync_cmpxchg((ptr), (old), (new), sizeof(*ptr))
163
164#define cmpxchg_local(ptr, old, new) \
165 __cmpxchg_local((ptr), (old), (new), sizeof(*ptr))
166#endif
167
168#define __xadd(ptr, inc, lock) \
169 ({ \
170 __typeof__ (*(ptr)) __ret = (inc); \
171 switch (sizeof(*(ptr))) { \
172 case __X86_CASE_B: \
173 asm volatile (lock "xaddb %b0, %1\n" \
174 : "+r" (__ret), "+m" (*(ptr)) \
175 : : "memory", "cc"); \
176 break; \
177 case __X86_CASE_W: \
178 asm volatile (lock "xaddw %w0, %1\n" \
179 : "+r" (__ret), "+m" (*(ptr)) \
180 : : "memory", "cc"); \
181 break; \
182 case __X86_CASE_L: \
183 asm volatile (lock "xaddl %0, %1\n" \
184 : "+r" (__ret), "+m" (*(ptr)) \
185 : : "memory", "cc"); \
186 break; \
187 case __X86_CASE_Q: \
188 asm volatile (lock "xaddq %q0, %1\n" \
189 : "+r" (__ret), "+m" (*(ptr)) \
190 : : "memory", "cc"); \
191 break; \
192 default: \
193 __xadd_wrong_size(); \
194 } \
195 __ret; \
196 })
197
198/*
199 * xadd() adds "inc" to "*ptr" and atomically returns the previous
200 * value of "*ptr".
201 *
202 * xadd() is locked when multiple CPUs are online
203 * xadd_sync() is always locked
204 * xadd_local() is never locked
205 */
206#define xadd(ptr, inc) __xadd((ptr), (inc), LOCK_PREFIX)
207#define xadd_sync(ptr, inc) __xadd((ptr), (inc), "lock; ")
208#define xadd_local(ptr, inc) __xadd((ptr), (inc), "")
209
210#endif /* ASM_X86_CMPXCHG_H */
diff --git a/arch/x86/include/asm/cmpxchg_32.h b/arch/x86/include/asm/cmpxchg_32.h
index 3deb7250624c..fbebb07dd80b 100644
--- a/arch/x86/include/asm/cmpxchg_32.h
+++ b/arch/x86/include/asm/cmpxchg_32.h
@@ -1,61 +1,11 @@
1#ifndef _ASM_X86_CMPXCHG_32_H 1#ifndef _ASM_X86_CMPXCHG_32_H
2#define _ASM_X86_CMPXCHG_32_H 2#define _ASM_X86_CMPXCHG_32_H
3 3
4#include <linux/bitops.h> /* for LOCK_PREFIX */
5
6/* 4/*
7 * Note: if you use set64_bit(), __cmpxchg64(), or their variants, you 5 * Note: if you use set64_bit(), __cmpxchg64(), or their variants, you
8 * you need to test for the feature in boot_cpu_data. 6 * you need to test for the feature in boot_cpu_data.
9 */ 7 */
10 8
11extern void __xchg_wrong_size(void);
12
13/*
14 * Note: no "lock" prefix even on SMP: xchg always implies lock anyway.
15 * Since this is generally used to protect other memory information, we
16 * use "asm volatile" and "memory" clobbers to prevent gcc from moving
17 * information around.
18 */
19#define __xchg(x, ptr, size) \
20({ \
21 __typeof(*(ptr)) __x = (x); \
22 switch (size) { \
23 case 1: \
24 { \
25 volatile u8 *__ptr = (volatile u8 *)(ptr); \
26 asm volatile("xchgb %0,%1" \
27 : "=q" (__x), "+m" (*__ptr) \
28 : "0" (__x) \
29 : "memory"); \
30 break; \
31 } \
32 case 2: \
33 { \
34 volatile u16 *__ptr = (volatile u16 *)(ptr); \
35 asm volatile("xchgw %0,%1" \
36 : "=r" (__x), "+m" (*__ptr) \
37 : "0" (__x) \
38 : "memory"); \
39 break; \
40 } \
41 case 4: \
42 { \
43 volatile u32 *__ptr = (volatile u32 *)(ptr); \
44 asm volatile("xchgl %0,%1" \
45 : "=r" (__x), "+m" (*__ptr) \
46 : "0" (__x) \
47 : "memory"); \
48 break; \
49 } \
50 default: \
51 __xchg_wrong_size(); \
52 } \
53 __x; \
54})
55
56#define xchg(ptr, v) \
57 __xchg((v), (ptr), sizeof(*ptr))
58
59/* 9/*
60 * CMPXCHG8B only writes to the target if we had the previous 10 * CMPXCHG8B only writes to the target if we had the previous
61 * value in registers, otherwise it acts as a read and gives us the 11 * value in registers, otherwise it acts as a read and gives us the
@@ -84,72 +34,8 @@ static inline void set_64bit(volatile u64 *ptr, u64 value)
84 : "memory"); 34 : "memory");
85} 35}
86 36
87extern void __cmpxchg_wrong_size(void);
88
89/*
90 * Atomic compare and exchange. Compare OLD with MEM, if identical,
91 * store NEW in MEM. Return the initial value in MEM. Success is
92 * indicated by comparing RETURN with OLD.
93 */
94#define __raw_cmpxchg(ptr, old, new, size, lock) \
95({ \
96 __typeof__(*(ptr)) __ret; \
97 __typeof__(*(ptr)) __old = (old); \
98 __typeof__(*(ptr)) __new = (new); \
99 switch (size) { \
100 case 1: \
101 { \
102 volatile u8 *__ptr = (volatile u8 *)(ptr); \
103 asm volatile(lock "cmpxchgb %2,%1" \
104 : "=a" (__ret), "+m" (*__ptr) \
105 : "q" (__new), "0" (__old) \
106 : "memory"); \
107 break; \
108 } \
109 case 2: \
110 { \
111 volatile u16 *__ptr = (volatile u16 *)(ptr); \
112 asm volatile(lock "cmpxchgw %2,%1" \
113 : "=a" (__ret), "+m" (*__ptr) \
114 : "r" (__new), "0" (__old) \
115 : "memory"); \
116 break; \
117 } \
118 case 4: \
119 { \
120 volatile u32 *__ptr = (volatile u32 *)(ptr); \
121 asm volatile(lock "cmpxchgl %2,%1" \
122 : "=a" (__ret), "+m" (*__ptr) \
123 : "r" (__new), "0" (__old) \
124 : "memory"); \
125 break; \
126 } \
127 default: \
128 __cmpxchg_wrong_size(); \
129 } \
130 __ret; \
131})
132
133#define __cmpxchg(ptr, old, new, size) \
134 __raw_cmpxchg((ptr), (old), (new), (size), LOCK_PREFIX)
135
136#define __sync_cmpxchg(ptr, old, new, size) \
137 __raw_cmpxchg((ptr), (old), (new), (size), "lock; ")
138
139#define __cmpxchg_local(ptr, old, new, size) \
140 __raw_cmpxchg((ptr), (old), (new), (size), "")
141
142#ifdef CONFIG_X86_CMPXCHG 37#ifdef CONFIG_X86_CMPXCHG
143#define __HAVE_ARCH_CMPXCHG 1 38#define __HAVE_ARCH_CMPXCHG 1
144
145#define cmpxchg(ptr, old, new) \
146 __cmpxchg((ptr), (old), (new), sizeof(*ptr))
147
148#define sync_cmpxchg(ptr, old, new) \
149 __sync_cmpxchg((ptr), (old), (new), sizeof(*ptr))
150
151#define cmpxchg_local(ptr, old, new) \
152 __cmpxchg_local((ptr), (old), (new), sizeof(*ptr))
153#endif 39#endif
154 40
155#ifdef CONFIG_X86_CMPXCHG64 41#ifdef CONFIG_X86_CMPXCHG64
diff --git a/arch/x86/include/asm/cmpxchg_64.h b/arch/x86/include/asm/cmpxchg_64.h
index 7cf5c0a24434..285da02c38fa 100644
--- a/arch/x86/include/asm/cmpxchg_64.h
+++ b/arch/x86/include/asm/cmpxchg_64.h
@@ -1,144 +1,13 @@
1#ifndef _ASM_X86_CMPXCHG_64_H 1#ifndef _ASM_X86_CMPXCHG_64_H
2#define _ASM_X86_CMPXCHG_64_H 2#define _ASM_X86_CMPXCHG_64_H
3 3
4#include <asm/alternative.h> /* Provides LOCK_PREFIX */
5
6static inline void set_64bit(volatile u64 *ptr, u64 val) 4static inline void set_64bit(volatile u64 *ptr, u64 val)
7{ 5{
8 *ptr = val; 6 *ptr = val;
9} 7}
10 8
11extern void __xchg_wrong_size(void);
12extern void __cmpxchg_wrong_size(void);
13
14/*
15 * Note: no "lock" prefix even on SMP: xchg always implies lock anyway.
16 * Since this is generally used to protect other memory information, we
17 * use "asm volatile" and "memory" clobbers to prevent gcc from moving
18 * information around.
19 */
20#define __xchg(x, ptr, size) \
21({ \
22 __typeof(*(ptr)) __x = (x); \
23 switch (size) { \
24 case 1: \
25 { \
26 volatile u8 *__ptr = (volatile u8 *)(ptr); \
27 asm volatile("xchgb %0,%1" \
28 : "=q" (__x), "+m" (*__ptr) \
29 : "0" (__x) \
30 : "memory"); \
31 break; \
32 } \
33 case 2: \
34 { \
35 volatile u16 *__ptr = (volatile u16 *)(ptr); \
36 asm volatile("xchgw %0,%1" \
37 : "=r" (__x), "+m" (*__ptr) \
38 : "0" (__x) \
39 : "memory"); \
40 break; \
41 } \
42 case 4: \
43 { \
44 volatile u32 *__ptr = (volatile u32 *)(ptr); \
45 asm volatile("xchgl %0,%1" \
46 : "=r" (__x), "+m" (*__ptr) \
47 : "0" (__x) \
48 : "memory"); \
49 break; \
50 } \
51 case 8: \
52 { \
53 volatile u64 *__ptr = (volatile u64 *)(ptr); \
54 asm volatile("xchgq %0,%1" \
55 : "=r" (__x), "+m" (*__ptr) \
56 : "0" (__x) \
57 : "memory"); \
58 break; \
59 } \
60 default: \
61 __xchg_wrong_size(); \
62 } \
63 __x; \
64})
65
66#define xchg(ptr, v) \
67 __xchg((v), (ptr), sizeof(*ptr))
68
69#define __HAVE_ARCH_CMPXCHG 1 9#define __HAVE_ARCH_CMPXCHG 1
70 10
71/*
72 * Atomic compare and exchange. Compare OLD with MEM, if identical,
73 * store NEW in MEM. Return the initial value in MEM. Success is
74 * indicated by comparing RETURN with OLD.
75 */
76#define __raw_cmpxchg(ptr, old, new, size, lock) \
77({ \
78 __typeof__(*(ptr)) __ret; \
79 __typeof__(*(ptr)) __old = (old); \
80 __typeof__(*(ptr)) __new = (new); \
81 switch (size) { \
82 case 1: \
83 { \
84 volatile u8 *__ptr = (volatile u8 *)(ptr); \
85 asm volatile(lock "cmpxchgb %2,%1" \
86 : "=a" (__ret), "+m" (*__ptr) \
87 : "q" (__new), "0" (__old) \
88 : "memory"); \
89 break; \
90 } \
91 case 2: \
92 { \
93 volatile u16 *__ptr = (volatile u16 *)(ptr); \
94 asm volatile(lock "cmpxchgw %2,%1" \
95 : "=a" (__ret), "+m" (*__ptr) \
96 : "r" (__new), "0" (__old) \
97 : "memory"); \
98 break; \
99 } \
100 case 4: \
101 { \
102 volatile u32 *__ptr = (volatile u32 *)(ptr); \
103 asm volatile(lock "cmpxchgl %2,%1" \
104 : "=a" (__ret), "+m" (*__ptr) \
105 : "r" (__new), "0" (__old) \
106 : "memory"); \
107 break; \
108 } \
109 case 8: \
110 { \
111 volatile u64 *__ptr = (volatile u64 *)(ptr); \
112 asm volatile(lock "cmpxchgq %2,%1" \
113 : "=a" (__ret), "+m" (*__ptr) \
114 : "r" (__new), "0" (__old) \
115 : "memory"); \
116 break; \
117 } \
118 default: \
119 __cmpxchg_wrong_size(); \
120 } \
121 __ret; \
122})
123
124#define __cmpxchg(ptr, old, new, size) \
125 __raw_cmpxchg((ptr), (old), (new), (size), LOCK_PREFIX)
126
127#define __sync_cmpxchg(ptr, old, new, size) \
128 __raw_cmpxchg((ptr), (old), (new), (size), "lock; ")
129
130#define __cmpxchg_local(ptr, old, new, size) \
131 __raw_cmpxchg((ptr), (old), (new), (size), "")
132
133#define cmpxchg(ptr, old, new) \
134 __cmpxchg((ptr), (old), (new), sizeof(*ptr))
135
136#define sync_cmpxchg(ptr, old, new) \
137 __sync_cmpxchg((ptr), (old), (new), sizeof(*ptr))
138
139#define cmpxchg_local(ptr, old, new) \
140 __cmpxchg_local((ptr), (old), (new), sizeof(*ptr))
141
142#define cmpxchg64(ptr, o, n) \ 11#define cmpxchg64(ptr, o, n) \
143({ \ 12({ \
144 BUILD_BUG_ON(sizeof(*(ptr)) != 8); \ 13 BUILD_BUG_ON(sizeof(*(ptr)) != 8); \
diff --git a/arch/x86/include/asm/compat.h b/arch/x86/include/asm/compat.h
index 1d9cd27c2920..30d737ef2a42 100644
--- a/arch/x86/include/asm/compat.h
+++ b/arch/x86/include/asm/compat.h
@@ -108,7 +108,8 @@ struct compat_statfs {
108 compat_fsid_t f_fsid; 108 compat_fsid_t f_fsid;
109 int f_namelen; /* SunOS ignores this field. */ 109 int f_namelen; /* SunOS ignores this field. */
110 int f_frsize; 110 int f_frsize;
111 int f_spare[5]; 111 int f_flags;
112 int f_spare[4];
112}; 113};
113 114
114#define COMPAT_RLIM_OLD_INFINITY 0x7fffffff 115#define COMPAT_RLIM_OLD_INFINITY 0x7fffffff
diff --git a/arch/x86/include/asm/cpufeature.h b/arch/x86/include/asm/cpufeature.h
index 88b23a43f340..aa6a488cd075 100644
--- a/arch/x86/include/asm/cpufeature.h
+++ b/arch/x86/include/asm/cpufeature.h
@@ -114,6 +114,7 @@
114#define X86_FEATURE_CX16 (4*32+13) /* CMPXCHG16B */ 114#define X86_FEATURE_CX16 (4*32+13) /* CMPXCHG16B */
115#define X86_FEATURE_XTPR (4*32+14) /* Send Task Priority Messages */ 115#define X86_FEATURE_XTPR (4*32+14) /* Send Task Priority Messages */
116#define X86_FEATURE_PDCM (4*32+15) /* Performance Capabilities */ 116#define X86_FEATURE_PDCM (4*32+15) /* Performance Capabilities */
117#define X86_FEATURE_PCID (4*32+17) /* Process Context Identifiers */
117#define X86_FEATURE_DCA (4*32+18) /* Direct Cache Access */ 118#define X86_FEATURE_DCA (4*32+18) /* Direct Cache Access */
118#define X86_FEATURE_XMM4_1 (4*32+19) /* "sse4_1" SSE-4.1 */ 119#define X86_FEATURE_XMM4_1 (4*32+19) /* "sse4_1" SSE-4.1 */
119#define X86_FEATURE_XMM4_2 (4*32+20) /* "sse4_2" SSE-4.2 */ 120#define X86_FEATURE_XMM4_2 (4*32+20) /* "sse4_2" SSE-4.2 */
diff --git a/arch/x86/include/asm/device.h b/arch/x86/include/asm/device.h
index 029f230ab637..63a2a03d7d51 100644
--- a/arch/x86/include/asm/device.h
+++ b/arch/x86/include/asm/device.h
@@ -8,7 +8,7 @@ struct dev_archdata {
8#ifdef CONFIG_X86_64 8#ifdef CONFIG_X86_64
9struct dma_map_ops *dma_ops; 9struct dma_map_ops *dma_ops;
10#endif 10#endif
11#if defined(CONFIG_DMAR) || defined(CONFIG_AMD_IOMMU) 11#if defined(CONFIG_INTEL_IOMMU) || defined(CONFIG_AMD_IOMMU)
12 void *iommu; /* hook for IOMMU specific extension */ 12 void *iommu; /* hook for IOMMU specific extension */
13#endif 13#endif
14}; 14};
diff --git a/arch/x86/include/asm/dma-mapping.h b/arch/x86/include/asm/dma-mapping.h
index d4c419f883a0..ed3065fd6314 100644
--- a/arch/x86/include/asm/dma-mapping.h
+++ b/arch/x86/include/asm/dma-mapping.h
@@ -2,7 +2,7 @@
2#define _ASM_X86_DMA_MAPPING_H 2#define _ASM_X86_DMA_MAPPING_H
3 3
4/* 4/*
5 * IOMMU interface. See Documentation/PCI/PCI-DMA-mapping.txt and 5 * IOMMU interface. See Documentation/DMA-API-HOWTO.txt and
6 * Documentation/DMA-API.txt for documentation. 6 * Documentation/DMA-API.txt for documentation.
7 */ 7 */
8 8
diff --git a/arch/x86/include/asm/dwarf2.h b/arch/x86/include/asm/dwarf2.h
index 326099199318..f6f15986df6c 100644
--- a/arch/x86/include/asm/dwarf2.h
+++ b/arch/x86/include/asm/dwarf2.h
@@ -27,6 +27,7 @@
27#define CFI_REMEMBER_STATE .cfi_remember_state 27#define CFI_REMEMBER_STATE .cfi_remember_state
28#define CFI_RESTORE_STATE .cfi_restore_state 28#define CFI_RESTORE_STATE .cfi_restore_state
29#define CFI_UNDEFINED .cfi_undefined 29#define CFI_UNDEFINED .cfi_undefined
30#define CFI_ESCAPE .cfi_escape
30 31
31#ifdef CONFIG_AS_CFI_SIGNAL_FRAME 32#ifdef CONFIG_AS_CFI_SIGNAL_FRAME
32#define CFI_SIGNAL_FRAME .cfi_signal_frame 33#define CFI_SIGNAL_FRAME .cfi_signal_frame
@@ -68,6 +69,7 @@
68#define CFI_REMEMBER_STATE cfi_ignore 69#define CFI_REMEMBER_STATE cfi_ignore
69#define CFI_RESTORE_STATE cfi_ignore 70#define CFI_RESTORE_STATE cfi_ignore
70#define CFI_UNDEFINED cfi_ignore 71#define CFI_UNDEFINED cfi_ignore
72#define CFI_ESCAPE cfi_ignore
71#define CFI_SIGNAL_FRAME cfi_ignore 73#define CFI_SIGNAL_FRAME cfi_ignore
72 74
73#endif 75#endif
diff --git a/arch/x86/include/asm/elf.h b/arch/x86/include/asm/elf.h
index f2ad2163109d..5f962df30d0f 100644
--- a/arch/x86/include/asm/elf.h
+++ b/arch/x86/include/asm/elf.h
@@ -4,6 +4,7 @@
4/* 4/*
5 * ELF register definitions.. 5 * ELF register definitions..
6 */ 6 */
7#include <linux/thread_info.h>
7 8
8#include <asm/ptrace.h> 9#include <asm/ptrace.h>
9#include <asm/user.h> 10#include <asm/user.h>
@@ -320,4 +321,34 @@ extern int syscall32_setup_pages(struct linux_binprm *, int exstack);
320extern unsigned long arch_randomize_brk(struct mm_struct *mm); 321extern unsigned long arch_randomize_brk(struct mm_struct *mm);
321#define arch_randomize_brk arch_randomize_brk 322#define arch_randomize_brk arch_randomize_brk
322 323
324/*
325 * True on X86_32 or when emulating IA32 on X86_64
326 */
327static inline int mmap_is_ia32(void)
328{
329#ifdef CONFIG_X86_32
330 return 1;
331#endif
332#ifdef CONFIG_IA32_EMULATION
333 if (test_thread_flag(TIF_IA32))
334 return 1;
335#endif
336 return 0;
337}
338
339/* The first two values are special, do not change. See align_addr() */
340enum align_flags {
341 ALIGN_VA_32 = BIT(0),
342 ALIGN_VA_64 = BIT(1),
343 ALIGN_VDSO = BIT(2),
344 ALIGN_TOPDOWN = BIT(3),
345};
346
347struct va_alignment {
348 int flags;
349 unsigned long mask;
350} ____cacheline_aligned;
351
352extern struct va_alignment va_align;
353extern unsigned long align_addr(unsigned long, struct file *, enum align_flags);
323#endif /* _ASM_X86_ELF_H */ 354#endif /* _ASM_X86_ELF_H */
diff --git a/arch/x86/include/asm/hw_irq.h b/arch/x86/include/asm/hw_irq.h
index 09199052060f..eb92a6ed2be7 100644
--- a/arch/x86/include/asm/hw_irq.h
+++ b/arch/x86/include/asm/hw_irq.h
@@ -119,7 +119,7 @@ struct irq_cfg {
119 cpumask_var_t old_domain; 119 cpumask_var_t old_domain;
120 u8 vector; 120 u8 vector;
121 u8 move_in_progress : 1; 121 u8 move_in_progress : 1;
122#ifdef CONFIG_INTR_REMAP 122#ifdef CONFIG_IRQ_REMAP
123 struct irq_2_iommu irq_2_iommu; 123 struct irq_2_iommu irq_2_iommu;
124#endif 124#endif
125}; 125};
diff --git a/arch/x86/include/asm/hyperv.h b/arch/x86/include/asm/hyperv.h
index 5df477ac3af7..b80420bcd09d 100644
--- a/arch/x86/include/asm/hyperv.h
+++ b/arch/x86/include/asm/hyperv.h
@@ -189,5 +189,6 @@
189#define HV_STATUS_INVALID_HYPERCALL_CODE 2 189#define HV_STATUS_INVALID_HYPERCALL_CODE 2
190#define HV_STATUS_INVALID_HYPERCALL_INPUT 3 190#define HV_STATUS_INVALID_HYPERCALL_INPUT 3
191#define HV_STATUS_INVALID_ALIGNMENT 4 191#define HV_STATUS_INVALID_ALIGNMENT 4
192#define HV_STATUS_INSUFFICIENT_BUFFERS 19
192 193
193#endif 194#endif
diff --git a/arch/x86/include/asm/irq_remapping.h b/arch/x86/include/asm/irq_remapping.h
index 1c23360fb2d8..47d99934580f 100644
--- a/arch/x86/include/asm/irq_remapping.h
+++ b/arch/x86/include/asm/irq_remapping.h
@@ -3,7 +3,8 @@
3 3
4#define IRTE_DEST(dest) ((x2apic_mode) ? dest : dest << 8) 4#define IRTE_DEST(dest) ((x2apic_mode) ? dest : dest << 8)
5 5
6#ifdef CONFIG_INTR_REMAP 6#ifdef CONFIG_IRQ_REMAP
7static void irq_remap_modify_chip_defaults(struct irq_chip *chip);
7static inline void prepare_irte(struct irte *irte, int vector, 8static inline void prepare_irte(struct irte *irte, int vector,
8 unsigned int dest) 9 unsigned int dest)
9{ 10{
@@ -36,6 +37,9 @@ static inline bool irq_remapped(struct irq_cfg *cfg)
36{ 37{
37 return false; 38 return false;
38} 39}
40static inline void irq_remap_modify_chip_defaults(struct irq_chip *chip)
41{
42}
39#endif 43#endif
40 44
41#endif /* _ASM_X86_IRQ_REMAPPING_H */ 45#endif /* _ASM_X86_IRQ_REMAPPING_H */
diff --git a/arch/x86/include/asm/irq_vectors.h b/arch/x86/include/asm/irq_vectors.h
index 7e50f06393aa..4b4448761e88 100644
--- a/arch/x86/include/asm/irq_vectors.h
+++ b/arch/x86/include/asm/irq_vectors.h
@@ -160,19 +160,11 @@ static inline int invalid_vm86_irq(int irq)
160#define IO_APIC_VECTOR_LIMIT ( 32 * MAX_IO_APICS ) 160#define IO_APIC_VECTOR_LIMIT ( 32 * MAX_IO_APICS )
161 161
162#ifdef CONFIG_X86_IO_APIC 162#ifdef CONFIG_X86_IO_APIC
163# ifdef CONFIG_SPARSE_IRQ 163# define CPU_VECTOR_LIMIT (64 * NR_CPUS)
164# define CPU_VECTOR_LIMIT (64 * NR_CPUS) 164# define NR_IRQS \
165# define NR_IRQS \
166 (CPU_VECTOR_LIMIT > IO_APIC_VECTOR_LIMIT ? \ 165 (CPU_VECTOR_LIMIT > IO_APIC_VECTOR_LIMIT ? \
167 (NR_VECTORS + CPU_VECTOR_LIMIT) : \ 166 (NR_VECTORS + CPU_VECTOR_LIMIT) : \
168 (NR_VECTORS + IO_APIC_VECTOR_LIMIT)) 167 (NR_VECTORS + IO_APIC_VECTOR_LIMIT))
169# else
170# define CPU_VECTOR_LIMIT (32 * NR_CPUS)
171# define NR_IRQS \
172 (CPU_VECTOR_LIMIT < IO_APIC_VECTOR_LIMIT ? \
173 (NR_VECTORS + CPU_VECTOR_LIMIT) : \
174 (NR_VECTORS + IO_APIC_VECTOR_LIMIT))
175# endif
176#else /* !CONFIG_X86_IO_APIC: */ 168#else /* !CONFIG_X86_IO_APIC: */
177# define NR_IRQS NR_IRQS_LEGACY 169# define NR_IRQS NR_IRQS_LEGACY
178#endif 170#endif
diff --git a/arch/x86/include/asm/nmi.h b/arch/x86/include/asm/nmi.h
index 4886a68f267e..fd3f9f18cf3f 100644
--- a/arch/x86/include/asm/nmi.h
+++ b/arch/x86/include/asm/nmi.h
@@ -22,27 +22,26 @@ void arch_trigger_all_cpu_backtrace(void);
22#define arch_trigger_all_cpu_backtrace arch_trigger_all_cpu_backtrace 22#define arch_trigger_all_cpu_backtrace arch_trigger_all_cpu_backtrace
23#endif 23#endif
24 24
25/* 25#define NMI_FLAG_FIRST 1
26 * Define some priorities for the nmi notifier call chain. 26
27 * 27enum {
28 * Create a local nmi bit that has a higher priority than 28 NMI_LOCAL=0,
29 * external nmis, because the local ones are more frequent. 29 NMI_UNKNOWN,
30 * 30 NMI_MAX
31 * Also setup some default high/normal/low settings for 31};
32 * subsystems to registers with. Using 4 bits to separate 32
33 * the priorities. This can go a lot higher if needed be. 33#define NMI_DONE 0
34 */ 34#define NMI_HANDLED 1
35 35
36#define NMI_LOCAL_SHIFT 16 /* randomly picked */ 36typedef int (*nmi_handler_t)(unsigned int, struct pt_regs *);
37#define NMI_LOCAL_BIT (1ULL << NMI_LOCAL_SHIFT) 37
38#define NMI_HIGH_PRIOR (1ULL << 8) 38int register_nmi_handler(unsigned int, nmi_handler_t, unsigned long,
39#define NMI_NORMAL_PRIOR (1ULL << 4) 39 const char *);
40#define NMI_LOW_PRIOR (1ULL << 0) 40
41#define NMI_LOCAL_HIGH_PRIOR (NMI_LOCAL_BIT | NMI_HIGH_PRIOR) 41void unregister_nmi_handler(unsigned int, const char *);
42#define NMI_LOCAL_NORMAL_PRIOR (NMI_LOCAL_BIT | NMI_NORMAL_PRIOR)
43#define NMI_LOCAL_LOW_PRIOR (NMI_LOCAL_BIT | NMI_LOW_PRIOR)
44 42
45void stop_nmi(void); 43void stop_nmi(void);
46void restart_nmi(void); 44void restart_nmi(void);
45void local_touch_nmi(void);
47 46
48#endif /* _ASM_X86_NMI_H */ 47#endif /* _ASM_X86_NMI_H */
diff --git a/arch/x86/include/asm/perf_event.h b/arch/x86/include/asm/perf_event.h
index 094fb30817ab..f61c62f7d5d8 100644
--- a/arch/x86/include/asm/perf_event.h
+++ b/arch/x86/include/asm/perf_event.h
@@ -29,6 +29,9 @@
29#define ARCH_PERFMON_EVENTSEL_INV (1ULL << 23) 29#define ARCH_PERFMON_EVENTSEL_INV (1ULL << 23)
30#define ARCH_PERFMON_EVENTSEL_CMASK 0xFF000000ULL 30#define ARCH_PERFMON_EVENTSEL_CMASK 0xFF000000ULL
31 31
32#define AMD_PERFMON_EVENTSEL_GUESTONLY (1ULL << 40)
33#define AMD_PERFMON_EVENTSEL_HOSTONLY (1ULL << 41)
34
32#define AMD64_EVENTSEL_EVENT \ 35#define AMD64_EVENTSEL_EVENT \
33 (ARCH_PERFMON_EVENTSEL_EVENT | (0x0FULL << 32)) 36 (ARCH_PERFMON_EVENTSEL_EVENT | (0x0FULL << 32))
34#define INTEL_ARCH_EVENT_MASK \ 37#define INTEL_ARCH_EVENT_MASK \
@@ -43,14 +46,17 @@
43#define AMD64_RAW_EVENT_MASK \ 46#define AMD64_RAW_EVENT_MASK \
44 (X86_RAW_EVENT_MASK | \ 47 (X86_RAW_EVENT_MASK | \
45 AMD64_EVENTSEL_EVENT) 48 AMD64_EVENTSEL_EVENT)
49#define AMD64_NUM_COUNTERS 4
50#define AMD64_NUM_COUNTERS_F15H 6
51#define AMD64_NUM_COUNTERS_MAX AMD64_NUM_COUNTERS_F15H
46 52
47#define ARCH_PERFMON_UNHALTED_CORE_CYCLES_SEL 0x3c 53#define ARCH_PERFMON_UNHALTED_CORE_CYCLES_SEL 0x3c
48#define ARCH_PERFMON_UNHALTED_CORE_CYCLES_UMASK (0x00 << 8) 54#define ARCH_PERFMON_UNHALTED_CORE_CYCLES_UMASK (0x00 << 8)
49#define ARCH_PERFMON_UNHALTED_CORE_CYCLES_INDEX 0 55#define ARCH_PERFMON_UNHALTED_CORE_CYCLES_INDEX 0
50#define ARCH_PERFMON_UNHALTED_CORE_CYCLES_PRESENT \ 56#define ARCH_PERFMON_UNHALTED_CORE_CYCLES_PRESENT \
51 (1 << (ARCH_PERFMON_UNHALTED_CORE_CYCLES_INDEX)) 57 (1 << (ARCH_PERFMON_UNHALTED_CORE_CYCLES_INDEX))
52 58
53#define ARCH_PERFMON_BRANCH_MISSES_RETIRED 6 59#define ARCH_PERFMON_BRANCH_MISSES_RETIRED 6
54 60
55/* 61/*
56 * Intel "Architectural Performance Monitoring" CPUID 62 * Intel "Architectural Performance Monitoring" CPUID
@@ -110,6 +116,35 @@ union cpuid10_edx {
110 */ 116 */
111#define X86_PMC_IDX_FIXED_BTS (X86_PMC_IDX_FIXED + 16) 117#define X86_PMC_IDX_FIXED_BTS (X86_PMC_IDX_FIXED + 16)
112 118
119/*
120 * IBS cpuid feature detection
121 */
122
123#define IBS_CPUID_FEATURES 0x8000001b
124
125/*
126 * Same bit mask as for IBS cpuid feature flags (Fn8000_001B_EAX), but
127 * bit 0 is used to indicate the existence of IBS.
128 */
129#define IBS_CAPS_AVAIL (1U<<0)
130#define IBS_CAPS_FETCHSAM (1U<<1)
131#define IBS_CAPS_OPSAM (1U<<2)
132#define IBS_CAPS_RDWROPCNT (1U<<3)
133#define IBS_CAPS_OPCNT (1U<<4)
134#define IBS_CAPS_BRNTRGT (1U<<5)
135#define IBS_CAPS_OPCNTEXT (1U<<6)
136
137#define IBS_CAPS_DEFAULT (IBS_CAPS_AVAIL \
138 | IBS_CAPS_FETCHSAM \
139 | IBS_CAPS_OPSAM)
140
141/*
142 * IBS APIC setup
143 */
144#define IBSCTL 0x1cc
145#define IBSCTL_LVT_OFFSET_VALID (1ULL<<8)
146#define IBSCTL_LVT_OFFSET_MASK 0x0F
147
113/* IbsFetchCtl bits/masks */ 148/* IbsFetchCtl bits/masks */
114#define IBS_FETCH_RAND_EN (1ULL<<57) 149#define IBS_FETCH_RAND_EN (1ULL<<57)
115#define IBS_FETCH_VAL (1ULL<<49) 150#define IBS_FETCH_VAL (1ULL<<49)
@@ -124,6 +159,8 @@ union cpuid10_edx {
124#define IBS_OP_MAX_CNT 0x0000FFFFULL 159#define IBS_OP_MAX_CNT 0x0000FFFFULL
125#define IBS_OP_MAX_CNT_EXT 0x007FFFFFULL /* not a register bit mask */ 160#define IBS_OP_MAX_CNT_EXT 0x007FFFFFULL /* not a register bit mask */
126 161
162extern u32 get_ibs_caps(void);
163
127#ifdef CONFIG_PERF_EVENTS 164#ifdef CONFIG_PERF_EVENTS
128extern void perf_events_lapic_init(void); 165extern void perf_events_lapic_init(void);
129 166
@@ -159,7 +196,19 @@ extern unsigned long perf_misc_flags(struct pt_regs *regs);
159 ); \ 196 ); \
160} 197}
161 198
199struct perf_guest_switch_msr {
200 unsigned msr;
201 u64 host, guest;
202};
203
204extern struct perf_guest_switch_msr *perf_guest_get_msrs(int *nr);
162#else 205#else
206static inline perf_guest_switch_msr *perf_guest_get_msrs(int *nr)
207{
208 *nr = 0;
209 return NULL;
210}
211
163static inline void perf_events_lapic_init(void) { } 212static inline void perf_events_lapic_init(void) { }
164#endif 213#endif
165 214
diff --git a/arch/x86/include/asm/processor.h b/arch/x86/include/asm/processor.h
index 0d1171c97729..b650435ffb53 100644
--- a/arch/x86/include/asm/processor.h
+++ b/arch/x86/include/asm/processor.h
@@ -111,6 +111,7 @@ struct cpuinfo_x86 {
111 /* Index into per_cpu list: */ 111 /* Index into per_cpu list: */
112 u16 cpu_index; 112 u16 cpu_index;
113#endif 113#endif
114 u32 microcode;
114} __attribute__((__aligned__(SMP_CACHE_BYTES))); 115} __attribute__((__aligned__(SMP_CACHE_BYTES)));
115 116
116#define X86_VENDOR_INTEL 0 117#define X86_VENDOR_INTEL 0
@@ -179,7 +180,8 @@ static inline void native_cpuid(unsigned int *eax, unsigned int *ebx,
179 "=b" (*ebx), 180 "=b" (*ebx),
180 "=c" (*ecx), 181 "=c" (*ecx),
181 "=d" (*edx) 182 "=d" (*edx)
182 : "0" (*eax), "2" (*ecx)); 183 : "0" (*eax), "2" (*ecx)
184 : "memory");
183} 185}
184 186
185static inline void load_cr3(pgd_t *pgdir) 187static inline void load_cr3(pgd_t *pgdir)
diff --git a/arch/x86/include/asm/reboot.h b/arch/x86/include/asm/reboot.h
index 3250e3d605d9..92f297069e87 100644
--- a/arch/x86/include/asm/reboot.h
+++ b/arch/x86/include/asm/reboot.h
@@ -23,7 +23,7 @@ void machine_real_restart(unsigned int type);
23#define MRR_BIOS 0 23#define MRR_BIOS 0
24#define MRR_APM 1 24#define MRR_APM 1
25 25
26typedef void (*nmi_shootdown_cb)(int, struct die_args*); 26typedef void (*nmi_shootdown_cb)(int, struct pt_regs*);
27void nmi_shootdown_cpus(nmi_shootdown_cb callback); 27void nmi_shootdown_cpus(nmi_shootdown_cb callback);
28 28
29#endif /* _ASM_X86_REBOOT_H */ 29#endif /* _ASM_X86_REBOOT_H */
diff --git a/arch/x86/include/asm/rwsem.h b/arch/x86/include/asm/rwsem.h
index df4cd32b4cc6..2dbe4a721ce5 100644
--- a/arch/x86/include/asm/rwsem.h
+++ b/arch/x86/include/asm/rwsem.h
@@ -204,13 +204,7 @@ static inline void rwsem_atomic_add(long delta, struct rw_semaphore *sem)
204 */ 204 */
205static inline long rwsem_atomic_update(long delta, struct rw_semaphore *sem) 205static inline long rwsem_atomic_update(long delta, struct rw_semaphore *sem)
206{ 206{
207 long tmp = delta; 207 return delta + xadd(&sem->count, delta);
208
209 asm volatile(LOCK_PREFIX "xadd %0,%1"
210 : "+r" (tmp), "+m" (sem->count)
211 : : "memory");
212
213 return tmp + delta;
214} 208}
215 209
216#endif /* __KERNEL__ */ 210#endif /* __KERNEL__ */
diff --git a/arch/x86/include/asm/spinlock.h b/arch/x86/include/asm/spinlock.h
index ee67edf86fdd..972c260919a3 100644
--- a/arch/x86/include/asm/spinlock.h
+++ b/arch/x86/include/asm/spinlock.h
@@ -49,109 +49,49 @@
49 * issues and should be optimal for the uncontended case. Note the tail must be 49 * issues and should be optimal for the uncontended case. Note the tail must be
50 * in the high part, because a wide xadd increment of the low part would carry 50 * in the high part, because a wide xadd increment of the low part would carry
51 * up and contaminate the high part. 51 * up and contaminate the high part.
52 *
53 * With fewer than 2^8 possible CPUs, we can use x86's partial registers to
54 * save some instructions and make the code more elegant. There really isn't
55 * much between them in performance though, especially as locks are out of line.
56 */ 52 */
57#if (NR_CPUS < 256)
58#define TICKET_SHIFT 8
59
60static __always_inline void __ticket_spin_lock(arch_spinlock_t *lock) 53static __always_inline void __ticket_spin_lock(arch_spinlock_t *lock)
61{ 54{
62 short inc = 0x0100; 55 register struct __raw_tickets inc = { .tail = 1 };
63 56
64 asm volatile ( 57 inc = xadd(&lock->tickets, inc);
65 LOCK_PREFIX "xaddw %w0, %1\n" 58
66 "1:\t" 59 for (;;) {
67 "cmpb %h0, %b0\n\t" 60 if (inc.head == inc.tail)
68 "je 2f\n\t" 61 break;
69 "rep ; nop\n\t" 62 cpu_relax();
70 "movb %1, %b0\n\t" 63 inc.head = ACCESS_ONCE(lock->tickets.head);
71 /* don't need lfence here, because loads are in-order */ 64 }
72 "jmp 1b\n" 65 barrier(); /* make sure nothing creeps before the lock is taken */
73 "2:"
74 : "+Q" (inc), "+m" (lock->slock)
75 :
76 : "memory", "cc");
77} 66}
78 67
79static __always_inline int __ticket_spin_trylock(arch_spinlock_t *lock) 68static __always_inline int __ticket_spin_trylock(arch_spinlock_t *lock)
80{ 69{
81 int tmp, new; 70 arch_spinlock_t old, new;
82 71
83 asm volatile("movzwl %2, %0\n\t" 72 old.tickets = ACCESS_ONCE(lock->tickets);
84 "cmpb %h0,%b0\n\t" 73 if (old.tickets.head != old.tickets.tail)
85 "leal 0x100(%" REG_PTR_MODE "0), %1\n\t" 74 return 0;
86 "jne 1f\n\t" 75
87 LOCK_PREFIX "cmpxchgw %w1,%2\n\t" 76 new.head_tail = old.head_tail + (1 << TICKET_SHIFT);
88 "1:"
89 "sete %b1\n\t"
90 "movzbl %b1,%0\n\t"
91 : "=&a" (tmp), "=&q" (new), "+m" (lock->slock)
92 :
93 : "memory", "cc");
94 77
95 return tmp; 78 /* cmpxchg is a full barrier, so nothing can move before it */
79 return cmpxchg(&lock->head_tail, old.head_tail, new.head_tail) == old.head_tail;
96} 80}
97 81
82#if (NR_CPUS < 256)
98static __always_inline void __ticket_spin_unlock(arch_spinlock_t *lock) 83static __always_inline void __ticket_spin_unlock(arch_spinlock_t *lock)
99{ 84{
100 asm volatile(UNLOCK_LOCK_PREFIX "incb %0" 85 asm volatile(UNLOCK_LOCK_PREFIX "incb %0"
101 : "+m" (lock->slock) 86 : "+m" (lock->head_tail)
102 : 87 :
103 : "memory", "cc"); 88 : "memory", "cc");
104} 89}
105#else 90#else
106#define TICKET_SHIFT 16
107
108static __always_inline void __ticket_spin_lock(arch_spinlock_t *lock)
109{
110 int inc = 0x00010000;
111 int tmp;
112
113 asm volatile(LOCK_PREFIX "xaddl %0, %1\n"
114 "movzwl %w0, %2\n\t"
115 "shrl $16, %0\n\t"
116 "1:\t"
117 "cmpl %0, %2\n\t"
118 "je 2f\n\t"
119 "rep ; nop\n\t"
120 "movzwl %1, %2\n\t"
121 /* don't need lfence here, because loads are in-order */
122 "jmp 1b\n"
123 "2:"
124 : "+r" (inc), "+m" (lock->slock), "=&r" (tmp)
125 :
126 : "memory", "cc");
127}
128
129static __always_inline int __ticket_spin_trylock(arch_spinlock_t *lock)
130{
131 int tmp;
132 int new;
133
134 asm volatile("movl %2,%0\n\t"
135 "movl %0,%1\n\t"
136 "roll $16, %0\n\t"
137 "cmpl %0,%1\n\t"
138 "leal 0x00010000(%" REG_PTR_MODE "0), %1\n\t"
139 "jne 1f\n\t"
140 LOCK_PREFIX "cmpxchgl %1,%2\n\t"
141 "1:"
142 "sete %b1\n\t"
143 "movzbl %b1,%0\n\t"
144 : "=&a" (tmp), "=&q" (new), "+m" (lock->slock)
145 :
146 : "memory", "cc");
147
148 return tmp;
149}
150
151static __always_inline void __ticket_spin_unlock(arch_spinlock_t *lock) 91static __always_inline void __ticket_spin_unlock(arch_spinlock_t *lock)
152{ 92{
153 asm volatile(UNLOCK_LOCK_PREFIX "incw %0" 93 asm volatile(UNLOCK_LOCK_PREFIX "incw %0"
154 : "+m" (lock->slock) 94 : "+m" (lock->head_tail)
155 : 95 :
156 : "memory", "cc"); 96 : "memory", "cc");
157} 97}
@@ -159,16 +99,16 @@ static __always_inline void __ticket_spin_unlock(arch_spinlock_t *lock)
159 99
160static inline int __ticket_spin_is_locked(arch_spinlock_t *lock) 100static inline int __ticket_spin_is_locked(arch_spinlock_t *lock)
161{ 101{
162 int tmp = ACCESS_ONCE(lock->slock); 102 struct __raw_tickets tmp = ACCESS_ONCE(lock->tickets);
163 103
164 return !!(((tmp >> TICKET_SHIFT) ^ tmp) & ((1 << TICKET_SHIFT) - 1)); 104 return !!(tmp.tail ^ tmp.head);
165} 105}
166 106
167static inline int __ticket_spin_is_contended(arch_spinlock_t *lock) 107static inline int __ticket_spin_is_contended(arch_spinlock_t *lock)
168{ 108{
169 int tmp = ACCESS_ONCE(lock->slock); 109 struct __raw_tickets tmp = ACCESS_ONCE(lock->tickets);
170 110
171 return (((tmp >> TICKET_SHIFT) - tmp) & ((1 << TICKET_SHIFT) - 1)) > 1; 111 return ((tmp.tail - tmp.head) & TICKET_MASK) > 1;
172} 112}
173 113
174#ifndef CONFIG_PARAVIRT_SPINLOCKS 114#ifndef CONFIG_PARAVIRT_SPINLOCKS
diff --git a/arch/x86/include/asm/spinlock_types.h b/arch/x86/include/asm/spinlock_types.h
index 7c7a486fcb68..8ebd5df7451e 100644
--- a/arch/x86/include/asm/spinlock_types.h
+++ b/arch/x86/include/asm/spinlock_types.h
@@ -5,11 +5,29 @@
5# error "please don't include this file directly" 5# error "please don't include this file directly"
6#endif 6#endif
7 7
8#include <linux/types.h>
9
10#if (CONFIG_NR_CPUS < 256)
11typedef u8 __ticket_t;
12typedef u16 __ticketpair_t;
13#else
14typedef u16 __ticket_t;
15typedef u32 __ticketpair_t;
16#endif
17
18#define TICKET_SHIFT (sizeof(__ticket_t) * 8)
19#define TICKET_MASK ((__ticket_t)((1 << TICKET_SHIFT) - 1))
20
8typedef struct arch_spinlock { 21typedef struct arch_spinlock {
9 unsigned int slock; 22 union {
23 __ticketpair_t head_tail;
24 struct __raw_tickets {
25 __ticket_t head, tail;
26 } tickets;
27 };
10} arch_spinlock_t; 28} arch_spinlock_t;
11 29
12#define __ARCH_SPIN_LOCK_UNLOCKED { 0 } 30#define __ARCH_SPIN_LOCK_UNLOCKED { { 0 } }
13 31
14#include <asm/rwlock.h> 32#include <asm/rwlock.h>
15 33
diff --git a/arch/x86/include/asm/unistd_64.h b/arch/x86/include/asm/unistd_64.h
index 201040573444..0a6ba337a2eb 100644
--- a/arch/x86/include/asm/unistd_64.h
+++ b/arch/x86/include/asm/unistd_64.h
@@ -624,7 +624,6 @@ __SYSCALL(__NR_vmsplice, sys_vmsplice)
624__SYSCALL(__NR_move_pages, sys_move_pages) 624__SYSCALL(__NR_move_pages, sys_move_pages)
625#define __NR_utimensat 280 625#define __NR_utimensat 280
626__SYSCALL(__NR_utimensat, sys_utimensat) 626__SYSCALL(__NR_utimensat, sys_utimensat)
627#define __IGNORE_getcpu /* implemented as a vsyscall */
628#define __NR_epoll_pwait 281 627#define __NR_epoll_pwait 281
629__SYSCALL(__NR_epoll_pwait, sys_epoll_pwait) 628__SYSCALL(__NR_epoll_pwait, sys_epoll_pwait)
630#define __NR_signalfd 282 629#define __NR_signalfd 282
diff --git a/arch/x86/include/asm/uv/uv_bau.h b/arch/x86/include/asm/uv/uv_bau.h
index 37d369859c8e..8e862aaf0d90 100644
--- a/arch/x86/include/asm/uv/uv_bau.h
+++ b/arch/x86/include/asm/uv/uv_bau.h
@@ -55,6 +55,7 @@
55#define UV_BAU_TUNABLES_DIR "sgi_uv" 55#define UV_BAU_TUNABLES_DIR "sgi_uv"
56#define UV_BAU_TUNABLES_FILE "bau_tunables" 56#define UV_BAU_TUNABLES_FILE "bau_tunables"
57#define WHITESPACE " \t\n" 57#define WHITESPACE " \t\n"
58#define uv_mmask ((1UL << uv_hub_info->m_val) - 1)
58#define uv_physnodeaddr(x) ((__pa((unsigned long)(x)) & uv_mmask)) 59#define uv_physnodeaddr(x) ((__pa((unsigned long)(x)) & uv_mmask))
59#define cpubit_isset(cpu, bau_local_cpumask) \ 60#define cpubit_isset(cpu, bau_local_cpumask) \
60 test_bit((cpu), (bau_local_cpumask).bits) 61 test_bit((cpu), (bau_local_cpumask).bits)
@@ -656,11 +657,7 @@ static inline int atomic_read_short(const struct atomic_short *v)
656 */ 657 */
657static inline int atom_asr(short i, struct atomic_short *v) 658static inline int atom_asr(short i, struct atomic_short *v)
658{ 659{
659 short __i = i; 660 return i + xadd(&v->counter, i);
660 asm volatile(LOCK_PREFIX "xaddw %0, %1"
661 : "+r" (i), "+m" (v->counter)
662 : : "memory");
663 return i + __i;
664} 661}
665 662
666/* 663/*
diff --git a/arch/x86/include/asm/uv/uv_hub.h b/arch/x86/include/asm/uv/uv_hub.h
index f26544a15214..54a13aaebc40 100644
--- a/arch/x86/include/asm/uv/uv_hub.h
+++ b/arch/x86/include/asm/uv/uv_hub.h
@@ -46,6 +46,13 @@
46 * PNODE - the low N bits of the GNODE. The PNODE is the most useful variant 46 * PNODE - the low N bits of the GNODE. The PNODE is the most useful variant
47 * of the nasid for socket usage. 47 * of the nasid for socket usage.
48 * 48 *
49 * GPA - (global physical address) a socket physical address converted
50 * so that it can be used by the GRU as a global address. Socket
51 * physical addresses 1) need additional NASID (node) bits added
52 * to the high end of the address, and 2) unaliased if the
53 * partition does not have a physical address 0. In addition, on
54 * UV2 rev 1, GPAs need the gnode left shifted to bits 39 or 40.
55 *
49 * 56 *
50 * NumaLink Global Physical Address Format: 57 * NumaLink Global Physical Address Format:
51 * +--------------------------------+---------------------+ 58 * +--------------------------------+---------------------+
@@ -141,6 +148,8 @@ struct uv_hub_info_s {
141 unsigned int gnode_extra; 148 unsigned int gnode_extra;
142 unsigned char hub_revision; 149 unsigned char hub_revision;
143 unsigned char apic_pnode_shift; 150 unsigned char apic_pnode_shift;
151 unsigned char m_shift;
152 unsigned char n_lshift;
144 unsigned long gnode_upper; 153 unsigned long gnode_upper;
145 unsigned long lowmem_remap_top; 154 unsigned long lowmem_remap_top;
146 unsigned long lowmem_remap_base; 155 unsigned long lowmem_remap_base;
@@ -177,6 +186,16 @@ static inline int is_uv2_hub(void)
177 return uv_hub_info->hub_revision >= UV2_HUB_REVISION_BASE; 186 return uv_hub_info->hub_revision >= UV2_HUB_REVISION_BASE;
178} 187}
179 188
189static inline int is_uv2_1_hub(void)
190{
191 return uv_hub_info->hub_revision == UV2_HUB_REVISION_BASE;
192}
193
194static inline int is_uv2_2_hub(void)
195{
196 return uv_hub_info->hub_revision == UV2_HUB_REVISION_BASE + 1;
197}
198
180union uvh_apicid { 199union uvh_apicid {
181 unsigned long v; 200 unsigned long v;
182 struct uvh_apicid_s { 201 struct uvh_apicid_s {
@@ -276,7 +295,10 @@ static inline unsigned long uv_soc_phys_ram_to_gpa(unsigned long paddr)
276{ 295{
277 if (paddr < uv_hub_info->lowmem_remap_top) 296 if (paddr < uv_hub_info->lowmem_remap_top)
278 paddr |= uv_hub_info->lowmem_remap_base; 297 paddr |= uv_hub_info->lowmem_remap_base;
279 return paddr | uv_hub_info->gnode_upper; 298 paddr |= uv_hub_info->gnode_upper;
299 paddr = ((paddr << uv_hub_info->m_shift) >> uv_hub_info->m_shift) |
300 ((paddr >> uv_hub_info->m_val) << uv_hub_info->n_lshift);
301 return paddr;
280} 302}
281 303
282 304
@@ -300,16 +322,19 @@ static inline unsigned long uv_gpa_to_soc_phys_ram(unsigned long gpa)
300 unsigned long remap_base = uv_hub_info->lowmem_remap_base; 322 unsigned long remap_base = uv_hub_info->lowmem_remap_base;
301 unsigned long remap_top = uv_hub_info->lowmem_remap_top; 323 unsigned long remap_top = uv_hub_info->lowmem_remap_top;
302 324
325 gpa = ((gpa << uv_hub_info->m_shift) >> uv_hub_info->m_shift) |
326 ((gpa >> uv_hub_info->n_lshift) << uv_hub_info->m_val);
327 gpa = gpa & uv_hub_info->gpa_mask;
303 if (paddr >= remap_base && paddr < remap_base + remap_top) 328 if (paddr >= remap_base && paddr < remap_base + remap_top)
304 paddr -= remap_base; 329 paddr -= remap_base;
305 return paddr; 330 return paddr;
306} 331}
307 332
308 333
309/* gnode -> pnode */ 334/* gpa -> pnode */
310static inline unsigned long uv_gpa_to_gnode(unsigned long gpa) 335static inline unsigned long uv_gpa_to_gnode(unsigned long gpa)
311{ 336{
312 return gpa >> uv_hub_info->m_val; 337 return gpa >> uv_hub_info->n_lshift;
313} 338}
314 339
315/* gpa -> pnode */ 340/* gpa -> pnode */
@@ -320,6 +345,12 @@ static inline int uv_gpa_to_pnode(unsigned long gpa)
320 return uv_gpa_to_gnode(gpa) & n_mask; 345 return uv_gpa_to_gnode(gpa) & n_mask;
321} 346}
322 347
348/* gpa -> node offset*/
349static inline unsigned long uv_gpa_to_offset(unsigned long gpa)
350{
351 return (gpa << uv_hub_info->m_shift) >> uv_hub_info->m_shift;
352}
353
323/* pnode, offset --> socket virtual */ 354/* pnode, offset --> socket virtual */
324static inline void *uv_pnode_offset_to_vaddr(int pnode, unsigned long offset) 355static inline void *uv_pnode_offset_to_vaddr(int pnode, unsigned long offset)
325{ 356{
diff --git a/arch/x86/include/asm/xen/page.h b/arch/x86/include/asm/xen/page.h
index 7ff4669580cf..c34f96c2f7a0 100644
--- a/arch/x86/include/asm/xen/page.h
+++ b/arch/x86/include/asm/xen/page.h
@@ -12,6 +12,7 @@
12#include <asm/pgtable.h> 12#include <asm/pgtable.h>
13 13
14#include <xen/interface/xen.h> 14#include <xen/interface/xen.h>
15#include <xen/grant_table.h>
15#include <xen/features.h> 16#include <xen/features.h>
16 17
17/* Xen machine address */ 18/* Xen machine address */
@@ -48,14 +49,11 @@ extern unsigned long set_phys_range_identity(unsigned long pfn_s,
48 unsigned long pfn_e); 49 unsigned long pfn_e);
49 50
50extern int m2p_add_override(unsigned long mfn, struct page *page, 51extern int m2p_add_override(unsigned long mfn, struct page *page,
51 bool clear_pte); 52 struct gnttab_map_grant_ref *kmap_op);
52extern int m2p_remove_override(struct page *page, bool clear_pte); 53extern int m2p_remove_override(struct page *page, bool clear_pte);
53extern struct page *m2p_find_override(unsigned long mfn); 54extern struct page *m2p_find_override(unsigned long mfn);
54extern unsigned long m2p_find_override_pfn(unsigned long mfn, unsigned long pfn); 55extern unsigned long m2p_find_override_pfn(unsigned long mfn, unsigned long pfn);
55 56
56#ifdef CONFIG_XEN_DEBUG_FS
57extern int p2m_dump_show(struct seq_file *m, void *v);
58#endif
59static inline unsigned long pfn_to_mfn(unsigned long pfn) 57static inline unsigned long pfn_to_mfn(unsigned long pfn)
60{ 58{
61 unsigned long mfn; 59 unsigned long mfn;
generated by cgit v1.2.2 (git 2.25.0) at 2024-12-24 19:07:05 -0500