aboutsummaryrefslogtreecommitdiffstats
path: root/include/asm-i386
diff options
context:
space:
mode:
Diffstat (limited to 'include/asm-i386')
-rw-r--r--include/asm-i386/alternative.h129
-rw-r--r--include/asm-i386/arch_hooks.h3
-rw-r--r--include/asm-i386/atomic.h36
-rw-r--r--include/asm-i386/bitops.h7
-rw-r--r--include/asm-i386/cache.h2
-rw-r--r--include/asm-i386/cpufeature.h1
-rw-r--r--include/asm-i386/mach-default/do_timer.h2
-rw-r--r--include/asm-i386/mach-es7000/mach_mpparse.h10
-rw-r--r--include/asm-i386/mach-visws/do_timer.h2
-rw-r--r--include/asm-i386/mach-voyager/do_timer.h2
-rw-r--r--include/asm-i386/mpspec.h1
-rw-r--r--include/asm-i386/mtrr.h1
-rw-r--r--include/asm-i386/mutex.h6
-rw-r--r--include/asm-i386/pgtable-2level.h2
-rw-r--r--include/asm-i386/pgtable-3level.h2
-rw-r--r--include/asm-i386/rwlock.h56
-rw-r--r--include/asm-i386/semaphore.h8
-rw-r--r--include/asm-i386/spinlock.h34
-rw-r--r--include/asm-i386/system.h62
-rw-r--r--include/asm-i386/uaccess.h12
-rw-r--r--include/asm-i386/unistd.h36
21 files changed, 259 insertions, 155 deletions
diff --git a/include/asm-i386/alternative.h b/include/asm-i386/alternative.h
new file mode 100644
index 000000000000..e201decea0c9
--- /dev/null
+++ b/include/asm-i386/alternative.h
@@ -0,0 +1,129 @@
1#ifndef _I386_ALTERNATIVE_H
2#define _I386_ALTERNATIVE_H
3
4#ifdef __KERNEL__
5
6struct alt_instr {
7 u8 *instr; /* original instruction */
8 u8 *replacement;
9 u8 cpuid; /* cpuid bit set for replacement */
10 u8 instrlen; /* length of original instruction */
11 u8 replacementlen; /* length of new instruction, <= instrlen */
12 u8 pad;
13};
14
15extern void apply_alternatives(struct alt_instr *start, struct alt_instr *end);
16
17struct module;
18extern void alternatives_smp_module_add(struct module *mod, char *name,
19 void *locks, void *locks_end,
20 void *text, void *text_end);
21extern void alternatives_smp_module_del(struct module *mod);
22extern void alternatives_smp_switch(int smp);
23
24#endif
25
26/*
27 * Alternative instructions for different CPU types or capabilities.
28 *
29 * This allows to use optimized instructions even on generic binary
30 * kernels.
31 *
32 * length of oldinstr must be longer or equal the length of newinstr
33 * It can be padded with nops as needed.
34 *
35 * For non barrier like inlines please define new variants
36 * without volatile and memory clobber.
37 */
38#define alternative(oldinstr, newinstr, feature) \
39 asm volatile ("661:\n\t" oldinstr "\n662:\n" \
40 ".section .altinstructions,\"a\"\n" \
41 " .align 4\n" \
42 " .long 661b\n" /* label */ \
43 " .long 663f\n" /* new instruction */ \
44 " .byte %c0\n" /* feature bit */ \
45 " .byte 662b-661b\n" /* sourcelen */ \
46 " .byte 664f-663f\n" /* replacementlen */ \
47 ".previous\n" \
48 ".section .altinstr_replacement,\"ax\"\n" \
49 "663:\n\t" newinstr "\n664:\n" /* replacement */\
50 ".previous" :: "i" (feature) : "memory")
51
52/*
53 * Alternative inline assembly with input.
54 *
55 * Pecularities:
56 * No memory clobber here.
57 * Argument numbers start with 1.
58 * Best is to use constraints that are fixed size (like (%1) ... "r")
59 * If you use variable sized constraints like "m" or "g" in the
60 * replacement maake sure to pad to the worst case length.
61 */
62#define alternative_input(oldinstr, newinstr, feature, input...) \
63 asm volatile ("661:\n\t" oldinstr "\n662:\n" \
64 ".section .altinstructions,\"a\"\n" \
65 " .align 4\n" \
66 " .long 661b\n" /* label */ \
67 " .long 663f\n" /* new instruction */ \
68 " .byte %c0\n" /* feature bit */ \
69 " .byte 662b-661b\n" /* sourcelen */ \
70 " .byte 664f-663f\n" /* replacementlen */ \
71 ".previous\n" \
72 ".section .altinstr_replacement,\"ax\"\n" \
73 "663:\n\t" newinstr "\n664:\n" /* replacement */\
74 ".previous" :: "i" (feature), ##input)
75
76/*
77 * Alternative inline assembly for SMP.
78 *
79 * alternative_smp() takes two versions (SMP first, UP second) and is
80 * for more complex stuff such as spinlocks.
81 *
82 * The LOCK_PREFIX macro defined here replaces the LOCK and
83 * LOCK_PREFIX macros used everywhere in the source tree.
84 *
85 * SMP alternatives use the same data structures as the other
86 * alternatives and the X86_FEATURE_UP flag to indicate the case of a
87 * UP system running a SMP kernel. The existing apply_alternatives()
88 * works fine for patching a SMP kernel for UP.
89 *
90 * The SMP alternative tables can be kept after boot and contain both
91 * UP and SMP versions of the instructions to allow switching back to
92 * SMP at runtime, when hotplugging in a new CPU, which is especially
93 * useful in virtualized environments.
94 *
95 * The very common lock prefix is handled as special case in a
96 * separate table which is a pure address list without replacement ptr
97 * and size information. That keeps the table sizes small.
98 */
99
100#ifdef CONFIG_SMP
101#define alternative_smp(smpinstr, upinstr, args...) \
102 asm volatile ("661:\n\t" smpinstr "\n662:\n" \
103 ".section .smp_altinstructions,\"a\"\n" \
104 " .align 4\n" \
105 " .long 661b\n" /* label */ \
106 " .long 663f\n" /* new instruction */ \
107 " .byte 0x68\n" /* X86_FEATURE_UP */ \
108 " .byte 662b-661b\n" /* sourcelen */ \
109 " .byte 664f-663f\n" /* replacementlen */ \
110 ".previous\n" \
111 ".section .smp_altinstr_replacement,\"awx\"\n" \
112 "663:\n\t" upinstr "\n" /* replacement */ \
113 "664:\n\t.fill 662b-661b,1,0x42\n" /* space for original */ \
114 ".previous" : args)
115
116#define LOCK_PREFIX \
117 ".section .smp_locks,\"a\"\n" \
118 " .align 4\n" \
119 " .long 661f\n" /* address */ \
120 ".previous\n" \
121 "661:\n\tlock; "
122
123#else /* ! CONFIG_SMP */
124#define alternative_smp(smpinstr, upinstr, args...) \
125 asm volatile (upinstr : args)
126#define LOCK_PREFIX ""
127#endif
128
129#endif /* _I386_ALTERNATIVE_H */
diff --git a/include/asm-i386/arch_hooks.h b/include/asm-i386/arch_hooks.h
index 28b96a6fb9fa..238cf4275b96 100644
--- a/include/asm-i386/arch_hooks.h
+++ b/include/asm-i386/arch_hooks.h
@@ -24,4 +24,7 @@ extern void trap_init_hook(void);
24extern void time_init_hook(void); 24extern void time_init_hook(void);
25extern void mca_nmi_hook(void); 25extern void mca_nmi_hook(void);
26 26
27extern int setup_early_printk(char *);
28extern void early_printk(const char *fmt, ...) __attribute__((format(printf,1,2)));
29
27#endif 30#endif
diff --git a/include/asm-i386/atomic.h b/include/asm-i386/atomic.h
index de649d3aa2d4..22d80ece95cb 100644
--- a/include/asm-i386/atomic.h
+++ b/include/asm-i386/atomic.h
@@ -10,12 +10,6 @@
10 * resource counting etc.. 10 * resource counting etc..
11 */ 11 */
12 12
13#ifdef CONFIG_SMP
14#define LOCK "lock ; "
15#else
16#define LOCK ""
17#endif
18
19/* 13/*
20 * Make sure gcc doesn't try to be clever and move things around 14 * Make sure gcc doesn't try to be clever and move things around
21 * on us. We need to use _exactly_ the address the user gave us, 15 * on us. We need to use _exactly_ the address the user gave us,
@@ -52,7 +46,7 @@ typedef struct { volatile int counter; } atomic_t;
52static __inline__ void atomic_add(int i, atomic_t *v) 46static __inline__ void atomic_add(int i, atomic_t *v)
53{ 47{
54 __asm__ __volatile__( 48 __asm__ __volatile__(
55 LOCK "addl %1,%0" 49 LOCK_PREFIX "addl %1,%0"
56 :"=m" (v->counter) 50 :"=m" (v->counter)
57 :"ir" (i), "m" (v->counter)); 51 :"ir" (i), "m" (v->counter));
58} 52}
@@ -67,7 +61,7 @@ static __inline__ void atomic_add(int i, atomic_t *v)
67static __inline__ void atomic_sub(int i, atomic_t *v) 61static __inline__ void atomic_sub(int i, atomic_t *v)
68{ 62{
69 __asm__ __volatile__( 63 __asm__ __volatile__(
70 LOCK "subl %1,%0" 64 LOCK_PREFIX "subl %1,%0"
71 :"=m" (v->counter) 65 :"=m" (v->counter)
72 :"ir" (i), "m" (v->counter)); 66 :"ir" (i), "m" (v->counter));
73} 67}
@@ -86,7 +80,7 @@ static __inline__ int atomic_sub_and_test(int i, atomic_t *v)
86 unsigned char c; 80 unsigned char c;
87 81
88 __asm__ __volatile__( 82 __asm__ __volatile__(
89 LOCK "subl %2,%0; sete %1" 83 LOCK_PREFIX "subl %2,%0; sete %1"
90 :"=m" (v->counter), "=qm" (c) 84 :"=m" (v->counter), "=qm" (c)
91 :"ir" (i), "m" (v->counter) : "memory"); 85 :"ir" (i), "m" (v->counter) : "memory");
92 return c; 86 return c;
@@ -101,7 +95,7 @@ static __inline__ int atomic_sub_and_test(int i, atomic_t *v)
101static __inline__ void atomic_inc(atomic_t *v) 95static __inline__ void atomic_inc(atomic_t *v)
102{ 96{
103 __asm__ __volatile__( 97 __asm__ __volatile__(
104 LOCK "incl %0" 98 LOCK_PREFIX "incl %0"
105 :"=m" (v->counter) 99 :"=m" (v->counter)
106 :"m" (v->counter)); 100 :"m" (v->counter));
107} 101}
@@ -115,7 +109,7 @@ static __inline__ void atomic_inc(atomic_t *v)
115static __inline__ void atomic_dec(atomic_t *v) 109static __inline__ void atomic_dec(atomic_t *v)
116{ 110{
117 __asm__ __volatile__( 111 __asm__ __volatile__(
118 LOCK "decl %0" 112 LOCK_PREFIX "decl %0"
119 :"=m" (v->counter) 113 :"=m" (v->counter)
120 :"m" (v->counter)); 114 :"m" (v->counter));
121} 115}
@@ -133,7 +127,7 @@ static __inline__ int atomic_dec_and_test(atomic_t *v)
133 unsigned char c; 127 unsigned char c;
134 128
135 __asm__ __volatile__( 129 __asm__ __volatile__(
136 LOCK "decl %0; sete %1" 130 LOCK_PREFIX "decl %0; sete %1"
137 :"=m" (v->counter), "=qm" (c) 131 :"=m" (v->counter), "=qm" (c)
138 :"m" (v->counter) : "memory"); 132 :"m" (v->counter) : "memory");
139 return c != 0; 133 return c != 0;
@@ -152,7 +146,7 @@ static __inline__ int atomic_inc_and_test(atomic_t *v)
152 unsigned char c; 146 unsigned char c;
153 147
154 __asm__ __volatile__( 148 __asm__ __volatile__(
155 LOCK "incl %0; sete %1" 149 LOCK_PREFIX "incl %0; sete %1"
156 :"=m" (v->counter), "=qm" (c) 150 :"=m" (v->counter), "=qm" (c)
157 :"m" (v->counter) : "memory"); 151 :"m" (v->counter) : "memory");
158 return c != 0; 152 return c != 0;
@@ -172,7 +166,7 @@ static __inline__ int atomic_add_negative(int i, atomic_t *v)
172 unsigned char c; 166 unsigned char c;
173 167
174 __asm__ __volatile__( 168 __asm__ __volatile__(
175 LOCK "addl %2,%0; sets %1" 169 LOCK_PREFIX "addl %2,%0; sets %1"
176 :"=m" (v->counter), "=qm" (c) 170 :"=m" (v->counter), "=qm" (c)
177 :"ir" (i), "m" (v->counter) : "memory"); 171 :"ir" (i), "m" (v->counter) : "memory");
178 return c; 172 return c;
@@ -195,7 +189,7 @@ static __inline__ int atomic_add_return(int i, atomic_t *v)
195 /* Modern 486+ processor */ 189 /* Modern 486+ processor */
196 __i = i; 190 __i = i;
197 __asm__ __volatile__( 191 __asm__ __volatile__(
198 LOCK "xaddl %0, %1;" 192 LOCK_PREFIX "xaddl %0, %1;"
199 :"=r"(i) 193 :"=r"(i)
200 :"m"(v->counter), "0"(i)); 194 :"m"(v->counter), "0"(i));
201 return i + __i; 195 return i + __i;
@@ -231,8 +225,14 @@ static __inline__ int atomic_sub_return(int i, atomic_t *v)
231({ \ 225({ \
232 int c, old; \ 226 int c, old; \
233 c = atomic_read(v); \ 227 c = atomic_read(v); \
234 while (c != (u) && (old = atomic_cmpxchg((v), c, c + (a))) != c) \ 228 for (;;) { \
229 if (unlikely(c == (u))) \
230 break; \
231 old = atomic_cmpxchg((v), c, c + (a)); \
232 if (likely(old == c)) \
233 break; \
235 c = old; \ 234 c = old; \
235 } \
236 c != (u); \ 236 c != (u); \
237}) 237})
238#define atomic_inc_not_zero(v) atomic_add_unless((v), 1, 0) 238#define atomic_inc_not_zero(v) atomic_add_unless((v), 1, 0)
@@ -242,11 +242,11 @@ static __inline__ int atomic_sub_return(int i, atomic_t *v)
242 242
243/* These are x86-specific, used by some header files */ 243/* These are x86-specific, used by some header files */
244#define atomic_clear_mask(mask, addr) \ 244#define atomic_clear_mask(mask, addr) \
245__asm__ __volatile__(LOCK "andl %0,%1" \ 245__asm__ __volatile__(LOCK_PREFIX "andl %0,%1" \
246: : "r" (~(mask)),"m" (*addr) : "memory") 246: : "r" (~(mask)),"m" (*addr) : "memory")
247 247
248#define atomic_set_mask(mask, addr) \ 248#define atomic_set_mask(mask, addr) \
249__asm__ __volatile__(LOCK "orl %0,%1" \ 249__asm__ __volatile__(LOCK_PREFIX "orl %0,%1" \
250: : "r" (mask),"m" (*(addr)) : "memory") 250: : "r" (mask),"m" (*(addr)) : "memory")
251 251
252/* Atomic operations are already serializing on x86 */ 252/* Atomic operations are already serializing on x86 */
diff --git a/include/asm-i386/bitops.h b/include/asm-i386/bitops.h
index 88e6ca248cd7..7d20b95edb3b 100644
--- a/include/asm-i386/bitops.h
+++ b/include/asm-i386/bitops.h
@@ -7,6 +7,7 @@
7 7
8#include <linux/config.h> 8#include <linux/config.h>
9#include <linux/compiler.h> 9#include <linux/compiler.h>
10#include <asm/alternative.h>
10 11
11/* 12/*
12 * These have to be done with inline assembly: that way the bit-setting 13 * These have to be done with inline assembly: that way the bit-setting
@@ -16,12 +17,6 @@
16 * bit 0 is the LSB of addr; bit 32 is the LSB of (addr+1). 17 * bit 0 is the LSB of addr; bit 32 is the LSB of (addr+1).
17 */ 18 */
18 19
19#ifdef CONFIG_SMP
20#define LOCK_PREFIX "lock ; "
21#else
22#define LOCK_PREFIX ""
23#endif
24
25#define ADDR (*(volatile long *) addr) 20#define ADDR (*(volatile long *) addr)
26 21
27/** 22/**
diff --git a/include/asm-i386/cache.h b/include/asm-i386/cache.h
index 615911e5bd24..ca15c9c665cf 100644
--- a/include/asm-i386/cache.h
+++ b/include/asm-i386/cache.h
@@ -10,4 +10,6 @@
10#define L1_CACHE_SHIFT (CONFIG_X86_L1_CACHE_SHIFT) 10#define L1_CACHE_SHIFT (CONFIG_X86_L1_CACHE_SHIFT)
11#define L1_CACHE_BYTES (1 << L1_CACHE_SHIFT) 11#define L1_CACHE_BYTES (1 << L1_CACHE_SHIFT)
12 12
13#define __read_mostly __attribute__((__section__(".data.read_mostly")))
14
13#endif 15#endif
diff --git a/include/asm-i386/cpufeature.h b/include/asm-i386/cpufeature.h
index c4ec2a4d8fdf..5c0b5876b931 100644
--- a/include/asm-i386/cpufeature.h
+++ b/include/asm-i386/cpufeature.h
@@ -70,6 +70,7 @@
70#define X86_FEATURE_P3 (3*32+ 6) /* P3 */ 70#define X86_FEATURE_P3 (3*32+ 6) /* P3 */
71#define X86_FEATURE_P4 (3*32+ 7) /* P4 */ 71#define X86_FEATURE_P4 (3*32+ 7) /* P4 */
72#define X86_FEATURE_CONSTANT_TSC (3*32+ 8) /* TSC ticks at a constant rate */ 72#define X86_FEATURE_CONSTANT_TSC (3*32+ 8) /* TSC ticks at a constant rate */
73#define X86_FEATURE_UP (3*32+ 9) /* smp kernel running on up */
73 74
74/* Intel-defined CPU features, CPUID level 0x00000001 (ecx), word 4 */ 75/* Intel-defined CPU features, CPUID level 0x00000001 (ecx), word 4 */
75#define X86_FEATURE_XMM3 (4*32+ 0) /* Streaming SIMD Extensions-3 */ 76#define X86_FEATURE_XMM3 (4*32+ 0) /* Streaming SIMD Extensions-3 */
diff --git a/include/asm-i386/mach-default/do_timer.h b/include/asm-i386/mach-default/do_timer.h
index 56211414fc95..6312c3e79814 100644
--- a/include/asm-i386/mach-default/do_timer.h
+++ b/include/asm-i386/mach-default/do_timer.h
@@ -18,7 +18,7 @@ static inline void do_timer_interrupt_hook(struct pt_regs *regs)
18{ 18{
19 do_timer(regs); 19 do_timer(regs);
20#ifndef CONFIG_SMP 20#ifndef CONFIG_SMP
21 update_process_times(user_mode(regs)); 21 update_process_times(user_mode_vm(regs));
22#endif 22#endif
23/* 23/*
24 * In the SMP case we use the local APIC timer interrupt to do the 24 * In the SMP case we use the local APIC timer interrupt to do the
diff --git a/include/asm-i386/mach-es7000/mach_mpparse.h b/include/asm-i386/mach-es7000/mach_mpparse.h
index 4a0637a3e208..99f66be240be 100644
--- a/include/asm-i386/mach-es7000/mach_mpparse.h
+++ b/include/asm-i386/mach-es7000/mach_mpparse.h
@@ -30,7 +30,8 @@ static inline int mps_oem_check(struct mp_config_table *mpc, char *oem,
30 return 0; 30 return 0;
31} 31}
32 32
33static inline int es7000_check_dsdt() 33#ifdef CONFIG_ACPI
34static inline int es7000_check_dsdt(void)
34{ 35{
35 struct acpi_table_header *header = NULL; 36 struct acpi_table_header *header = NULL;
36 if(!acpi_get_table_header_early(ACPI_DSDT, &header)) 37 if(!acpi_get_table_header_early(ACPI_DSDT, &header))
@@ -54,6 +55,11 @@ static inline int acpi_madt_oem_check(char *oem_id, char *oem_table_id)
54 } 55 }
55 return 0; 56 return 0;
56} 57}
57 58#else
59static inline int acpi_madt_oem_check(char *oem_id, char *oem_table_id)
60{
61 return 0;
62}
63#endif
58 64
59#endif /* __ASM_MACH_MPPARSE_H */ 65#endif /* __ASM_MACH_MPPARSE_H */
diff --git a/include/asm-i386/mach-visws/do_timer.h b/include/asm-i386/mach-visws/do_timer.h
index 92d638fc8b11..95568e6ca91c 100644
--- a/include/asm-i386/mach-visws/do_timer.h
+++ b/include/asm-i386/mach-visws/do_timer.h
@@ -11,7 +11,7 @@ static inline void do_timer_interrupt_hook(struct pt_regs *regs)
11 11
12 do_timer(regs); 12 do_timer(regs);
13#ifndef CONFIG_SMP 13#ifndef CONFIG_SMP
14 update_process_times(user_mode(regs)); 14 update_process_times(user_mode_vm(regs));
15#endif 15#endif
16/* 16/*
17 * In the SMP case we use the local APIC timer interrupt to do the 17 * In the SMP case we use the local APIC timer interrupt to do the
diff --git a/include/asm-i386/mach-voyager/do_timer.h b/include/asm-i386/mach-voyager/do_timer.h
index ae510e5d0d78..eaf518098981 100644
--- a/include/asm-i386/mach-voyager/do_timer.h
+++ b/include/asm-i386/mach-voyager/do_timer.h
@@ -5,7 +5,7 @@ static inline void do_timer_interrupt_hook(struct pt_regs *regs)
5{ 5{
6 do_timer(regs); 6 do_timer(regs);
7#ifndef CONFIG_SMP 7#ifndef CONFIG_SMP
8 update_process_times(user_mode(regs)); 8 update_process_times(user_mode_vm(regs));
9#endif 9#endif
10 10
11 voyager_timer_interrupt(regs); 11 voyager_timer_interrupt(regs);
diff --git a/include/asm-i386/mpspec.h b/include/asm-i386/mpspec.h
index 64a0b8e6afeb..62113d3bfdc2 100644
--- a/include/asm-i386/mpspec.h
+++ b/include/asm-i386/mpspec.h
@@ -22,7 +22,6 @@ extern int mp_bus_id_to_type [MAX_MP_BUSSES];
22extern int mp_irq_entries; 22extern int mp_irq_entries;
23extern struct mpc_config_intsrc mp_irqs [MAX_IRQ_SOURCES]; 23extern struct mpc_config_intsrc mp_irqs [MAX_IRQ_SOURCES];
24extern int mpc_default_type; 24extern int mpc_default_type;
25extern int mp_bus_id_to_pci_bus [MAX_MP_BUSSES];
26extern unsigned long mp_lapic_addr; 25extern unsigned long mp_lapic_addr;
27extern int pic_mode; 26extern int pic_mode;
28extern int using_apic_timer; 27extern int using_apic_timer;
diff --git a/include/asm-i386/mtrr.h b/include/asm-i386/mtrr.h
index 5b6ceda68c5f..64cf937c7e33 100644
--- a/include/asm-i386/mtrr.h
+++ b/include/asm-i386/mtrr.h
@@ -25,6 +25,7 @@
25 25
26#include <linux/config.h> 26#include <linux/config.h>
27#include <linux/ioctl.h> 27#include <linux/ioctl.h>
28#include <linux/errno.h>
28 29
29#define MTRR_IOCTL_BASE 'M' 30#define MTRR_IOCTL_BASE 'M'
30 31
diff --git a/include/asm-i386/mutex.h b/include/asm-i386/mutex.h
index 9b2199e829f3..05a538531229 100644
--- a/include/asm-i386/mutex.h
+++ b/include/asm-i386/mutex.h
@@ -9,6 +9,8 @@
9#ifndef _ASM_MUTEX_H 9#ifndef _ASM_MUTEX_H
10#define _ASM_MUTEX_H 10#define _ASM_MUTEX_H
11 11
12#include "asm/alternative.h"
13
12/** 14/**
13 * __mutex_fastpath_lock - try to take the lock by moving the count 15 * __mutex_fastpath_lock - try to take the lock by moving the count
14 * from 1 to a 0 value 16 * from 1 to a 0 value
@@ -27,7 +29,7 @@ do { \
27 typecheck_fn(fastcall void (*)(atomic_t *), fail_fn); \ 29 typecheck_fn(fastcall void (*)(atomic_t *), fail_fn); \
28 \ 30 \
29 __asm__ __volatile__( \ 31 __asm__ __volatile__( \
30 LOCK " decl (%%eax) \n" \ 32 LOCK_PREFIX " decl (%%eax) \n" \
31 " js 2f \n" \ 33 " js 2f \n" \
32 "1: \n" \ 34 "1: \n" \
33 \ 35 \
@@ -83,7 +85,7 @@ do { \
83 typecheck_fn(fastcall void (*)(atomic_t *), fail_fn); \ 85 typecheck_fn(fastcall void (*)(atomic_t *), fail_fn); \
84 \ 86 \
85 __asm__ __volatile__( \ 87 __asm__ __volatile__( \
86 LOCK " incl (%%eax) \n" \ 88 LOCK_PREFIX " incl (%%eax) \n" \
87 " jle 2f \n" \ 89 " jle 2f \n" \
88 "1: \n" \ 90 "1: \n" \
89 \ 91 \
diff --git a/include/asm-i386/pgtable-2level.h b/include/asm-i386/pgtable-2level.h
index 74ef721b534d..27bde973abc7 100644
--- a/include/asm-i386/pgtable-2level.h
+++ b/include/asm-i386/pgtable-2level.h
@@ -61,4 +61,6 @@ static inline int pte_exec_kernel(pte_t pte)
61#define __pte_to_swp_entry(pte) ((swp_entry_t) { (pte).pte_low }) 61#define __pte_to_swp_entry(pte) ((swp_entry_t) { (pte).pte_low })
62#define __swp_entry_to_pte(x) ((pte_t) { (x).val }) 62#define __swp_entry_to_pte(x) ((pte_t) { (x).val })
63 63
64void vmalloc_sync_all(void);
65
64#endif /* _I386_PGTABLE_2LEVEL_H */ 66#endif /* _I386_PGTABLE_2LEVEL_H */
diff --git a/include/asm-i386/pgtable-3level.h b/include/asm-i386/pgtable-3level.h
index f1a8b454920a..36a5aa63cbbf 100644
--- a/include/asm-i386/pgtable-3level.h
+++ b/include/asm-i386/pgtable-3level.h
@@ -152,4 +152,6 @@ static inline pmd_t pfn_pmd(unsigned long page_nr, pgprot_t pgprot)
152 152
153#define __pmd_free_tlb(tlb, x) do { } while (0) 153#define __pmd_free_tlb(tlb, x) do { } while (0)
154 154
155#define vmalloc_sync_all() ((void)0)
156
155#endif /* _I386_PGTABLE_3LEVEL_H */ 157#endif /* _I386_PGTABLE_3LEVEL_H */
diff --git a/include/asm-i386/rwlock.h b/include/asm-i386/rwlock.h
index b57cc7afdf7e..94f00195d543 100644
--- a/include/asm-i386/rwlock.h
+++ b/include/asm-i386/rwlock.h
@@ -21,21 +21,23 @@
21#define RW_LOCK_BIAS_STR "0x01000000" 21#define RW_LOCK_BIAS_STR "0x01000000"
22 22
23#define __build_read_lock_ptr(rw, helper) \ 23#define __build_read_lock_ptr(rw, helper) \
24 asm volatile(LOCK "subl $1,(%0)\n\t" \ 24 alternative_smp("lock; subl $1,(%0)\n\t" \
25 "jns 1f\n" \ 25 "jns 1f\n" \
26 "call " helper "\n\t" \ 26 "call " helper "\n\t" \
27 "1:\n" \ 27 "1:\n", \
28 ::"a" (rw) : "memory") 28 "subl $1,(%0)\n\t", \
29 :"a" (rw) : "memory")
29 30
30#define __build_read_lock_const(rw, helper) \ 31#define __build_read_lock_const(rw, helper) \
31 asm volatile(LOCK "subl $1,%0\n\t" \ 32 alternative_smp("lock; subl $1,%0\n\t" \
32 "jns 1f\n" \ 33 "jns 1f\n" \
33 "pushl %%eax\n\t" \ 34 "pushl %%eax\n\t" \
34 "leal %0,%%eax\n\t" \ 35 "leal %0,%%eax\n\t" \
35 "call " helper "\n\t" \ 36 "call " helper "\n\t" \
36 "popl %%eax\n\t" \ 37 "popl %%eax\n\t" \
37 "1:\n" \ 38 "1:\n", \
38 :"=m" (*(volatile int *)rw) : : "memory") 39 "subl $1,%0\n\t", \
40 "=m" (*(volatile int *)rw) : : "memory")
39 41
40#define __build_read_lock(rw, helper) do { \ 42#define __build_read_lock(rw, helper) do { \
41 if (__builtin_constant_p(rw)) \ 43 if (__builtin_constant_p(rw)) \
@@ -45,21 +47,23 @@
45 } while (0) 47 } while (0)
46 48
47#define __build_write_lock_ptr(rw, helper) \ 49#define __build_write_lock_ptr(rw, helper) \
48 asm volatile(LOCK "subl $" RW_LOCK_BIAS_STR ",(%0)\n\t" \ 50 alternative_smp("lock; subl $" RW_LOCK_BIAS_STR ",(%0)\n\t" \
49 "jz 1f\n" \ 51 "jz 1f\n" \
50 "call " helper "\n\t" \ 52 "call " helper "\n\t" \
51 "1:\n" \ 53 "1:\n", \
52 ::"a" (rw) : "memory") 54 "subl $" RW_LOCK_BIAS_STR ",(%0)\n\t", \
55 :"a" (rw) : "memory")
53 56
54#define __build_write_lock_const(rw, helper) \ 57#define __build_write_lock_const(rw, helper) \
55 asm volatile(LOCK "subl $" RW_LOCK_BIAS_STR ",%0\n\t" \ 58 alternative_smp("lock; subl $" RW_LOCK_BIAS_STR ",%0\n\t" \
56 "jz 1f\n" \ 59 "jz 1f\n" \
57 "pushl %%eax\n\t" \ 60 "pushl %%eax\n\t" \
58 "leal %0,%%eax\n\t" \ 61 "leal %0,%%eax\n\t" \
59 "call " helper "\n\t" \ 62 "call " helper "\n\t" \
60 "popl %%eax\n\t" \ 63 "popl %%eax\n\t" \
61 "1:\n" \ 64 "1:\n", \
62 :"=m" (*(volatile int *)rw) : : "memory") 65 "subl $" RW_LOCK_BIAS_STR ",%0\n\t", \
66 "=m" (*(volatile int *)rw) : : "memory")
63 67
64#define __build_write_lock(rw, helper) do { \ 68#define __build_write_lock(rw, helper) do { \
65 if (__builtin_constant_p(rw)) \ 69 if (__builtin_constant_p(rw)) \
diff --git a/include/asm-i386/semaphore.h b/include/asm-i386/semaphore.h
index 6a42b2142fd6..f7a0f310c524 100644
--- a/include/asm-i386/semaphore.h
+++ b/include/asm-i386/semaphore.h
@@ -99,7 +99,7 @@ static inline void down(struct semaphore * sem)
99 might_sleep(); 99 might_sleep();
100 __asm__ __volatile__( 100 __asm__ __volatile__(
101 "# atomic down operation\n\t" 101 "# atomic down operation\n\t"
102 LOCK "decl %0\n\t" /* --sem->count */ 102 LOCK_PREFIX "decl %0\n\t" /* --sem->count */
103 "js 2f\n" 103 "js 2f\n"
104 "1:\n" 104 "1:\n"
105 LOCK_SECTION_START("") 105 LOCK_SECTION_START("")
@@ -123,7 +123,7 @@ static inline int down_interruptible(struct semaphore * sem)
123 might_sleep(); 123 might_sleep();
124 __asm__ __volatile__( 124 __asm__ __volatile__(
125 "# atomic interruptible down operation\n\t" 125 "# atomic interruptible down operation\n\t"
126 LOCK "decl %1\n\t" /* --sem->count */ 126 LOCK_PREFIX "decl %1\n\t" /* --sem->count */
127 "js 2f\n\t" 127 "js 2f\n\t"
128 "xorl %0,%0\n" 128 "xorl %0,%0\n"
129 "1:\n" 129 "1:\n"
@@ -148,7 +148,7 @@ static inline int down_trylock(struct semaphore * sem)
148 148
149 __asm__ __volatile__( 149 __asm__ __volatile__(
150 "# atomic interruptible down operation\n\t" 150 "# atomic interruptible down operation\n\t"
151 LOCK "decl %1\n\t" /* --sem->count */ 151 LOCK_PREFIX "decl %1\n\t" /* --sem->count */
152 "js 2f\n\t" 152 "js 2f\n\t"
153 "xorl %0,%0\n" 153 "xorl %0,%0\n"
154 "1:\n" 154 "1:\n"
@@ -173,7 +173,7 @@ static inline void up(struct semaphore * sem)
173{ 173{
174 __asm__ __volatile__( 174 __asm__ __volatile__(
175 "# atomic up operation\n\t" 175 "# atomic up operation\n\t"
176 LOCK "incl %0\n\t" /* ++sem->count */ 176 LOCK_PREFIX "incl %0\n\t" /* ++sem->count */
177 "jle 2f\n" 177 "jle 2f\n"
178 "1:\n" 178 "1:\n"
179 LOCK_SECTION_START("") 179 LOCK_SECTION_START("")
diff --git a/include/asm-i386/spinlock.h b/include/asm-i386/spinlock.h
index 23604350cdf4..d76b7693cf1d 100644
--- a/include/asm-i386/spinlock.h
+++ b/include/asm-i386/spinlock.h
@@ -35,31 +35,41 @@
35#define __raw_spin_lock_string_flags \ 35#define __raw_spin_lock_string_flags \
36 "\n1:\t" \ 36 "\n1:\t" \
37 "lock ; decb %0\n\t" \ 37 "lock ; decb %0\n\t" \
38 "jns 4f\n\t" \ 38 "jns 5f\n" \
39 "2:\t" \ 39 "2:\t" \
40 "testl $0x200, %1\n\t" \ 40 "testl $0x200, %1\n\t" \
41 "jz 3f\n\t" \ 41 "jz 4f\n\t" \
42 "sti\n\t" \ 42 "sti\n" \
43 "3:\t" \ 43 "3:\t" \
44 "rep;nop\n\t" \ 44 "rep;nop\n\t" \
45 "cmpb $0, %0\n\t" \ 45 "cmpb $0, %0\n\t" \
46 "jle 3b\n\t" \ 46 "jle 3b\n\t" \
47 "cli\n\t" \ 47 "cli\n\t" \
48 "jmp 1b\n" \ 48 "jmp 1b\n" \
49 "4:\n\t" 49 "4:\t" \
50 "rep;nop\n\t" \
51 "cmpb $0, %0\n\t" \
52 "jg 1b\n\t" \
53 "jmp 4b\n" \
54 "5:\n\t"
55
56#define __raw_spin_lock_string_up \
57 "\n\tdecb %0"
50 58
51static inline void __raw_spin_lock(raw_spinlock_t *lock) 59static inline void __raw_spin_lock(raw_spinlock_t *lock)
52{ 60{
53 __asm__ __volatile__( 61 alternative_smp(
54 __raw_spin_lock_string 62 __raw_spin_lock_string,
55 :"=m" (lock->slock) : : "memory"); 63 __raw_spin_lock_string_up,
64 "=m" (lock->slock) : : "memory");
56} 65}
57 66
58static inline void __raw_spin_lock_flags(raw_spinlock_t *lock, unsigned long flags) 67static inline void __raw_spin_lock_flags(raw_spinlock_t *lock, unsigned long flags)
59{ 68{
60 __asm__ __volatile__( 69 alternative_smp(
61 __raw_spin_lock_string_flags 70 __raw_spin_lock_string_flags,
62 :"=m" (lock->slock) : "r" (flags) : "memory"); 71 __raw_spin_lock_string_up,
72 "=m" (lock->slock) : "r" (flags) : "memory");
63} 73}
64 74
65static inline int __raw_spin_trylock(raw_spinlock_t *lock) 75static inline int __raw_spin_trylock(raw_spinlock_t *lock)
@@ -178,12 +188,12 @@ static inline int __raw_write_trylock(raw_rwlock_t *lock)
178 188
179static inline void __raw_read_unlock(raw_rwlock_t *rw) 189static inline void __raw_read_unlock(raw_rwlock_t *rw)
180{ 190{
181 asm volatile("lock ; incl %0" :"=m" (rw->lock) : : "memory"); 191 asm volatile(LOCK_PREFIX "incl %0" :"=m" (rw->lock) : : "memory");
182} 192}
183 193
184static inline void __raw_write_unlock(raw_rwlock_t *rw) 194static inline void __raw_write_unlock(raw_rwlock_t *rw)
185{ 195{
186 asm volatile("lock ; addl $" RW_LOCK_BIAS_STR ", %0" 196 asm volatile(LOCK_PREFIX "addl $" RW_LOCK_BIAS_STR ", %0"
187 : "=m" (rw->lock) : : "memory"); 197 : "=m" (rw->lock) : : "memory");
188} 198}
189 199
diff --git a/include/asm-i386/system.h b/include/asm-i386/system.h
index 399145a247f2..d0d8d7448d88 100644
--- a/include/asm-i386/system.h
+++ b/include/asm-i386/system.h
@@ -352,67 +352,6 @@ static inline unsigned long long __cmpxchg64(volatile void *ptr, unsigned long l
352 352
353#endif 353#endif
354 354
355#ifdef __KERNEL__
356struct alt_instr {
357 __u8 *instr; /* original instruction */
358 __u8 *replacement;
359 __u8 cpuid; /* cpuid bit set for replacement */
360 __u8 instrlen; /* length of original instruction */
361 __u8 replacementlen; /* length of new instruction, <= instrlen */
362 __u8 pad;
363};
364#endif
365
366/*
367 * Alternative instructions for different CPU types or capabilities.
368 *
369 * This allows to use optimized instructions even on generic binary
370 * kernels.
371 *
372 * length of oldinstr must be longer or equal the length of newinstr
373 * It can be padded with nops as needed.
374 *
375 * For non barrier like inlines please define new variants
376 * without volatile and memory clobber.
377 */
378#define alternative(oldinstr, newinstr, feature) \
379 asm volatile ("661:\n\t" oldinstr "\n662:\n" \
380 ".section .altinstructions,\"a\"\n" \
381 " .align 4\n" \
382 " .long 661b\n" /* label */ \
383 " .long 663f\n" /* new instruction */ \
384 " .byte %c0\n" /* feature bit */ \
385 " .byte 662b-661b\n" /* sourcelen */ \
386 " .byte 664f-663f\n" /* replacementlen */ \
387 ".previous\n" \
388 ".section .altinstr_replacement,\"ax\"\n" \
389 "663:\n\t" newinstr "\n664:\n" /* replacement */ \
390 ".previous" :: "i" (feature) : "memory")
391
392/*
393 * Alternative inline assembly with input.
394 *
395 * Pecularities:
396 * No memory clobber here.
397 * Argument numbers start with 1.
398 * Best is to use constraints that are fixed size (like (%1) ... "r")
399 * If you use variable sized constraints like "m" or "g" in the
400 * replacement maake sure to pad to the worst case length.
401 */
402#define alternative_input(oldinstr, newinstr, feature, input...) \
403 asm volatile ("661:\n\t" oldinstr "\n662:\n" \
404 ".section .altinstructions,\"a\"\n" \
405 " .align 4\n" \
406 " .long 661b\n" /* label */ \
407 " .long 663f\n" /* new instruction */ \
408 " .byte %c0\n" /* feature bit */ \
409 " .byte 662b-661b\n" /* sourcelen */ \
410 " .byte 664f-663f\n" /* replacementlen */ \
411 ".previous\n" \
412 ".section .altinstr_replacement,\"ax\"\n" \
413 "663:\n\t" newinstr "\n664:\n" /* replacement */ \
414 ".previous" :: "i" (feature), ##input)
415
416/* 355/*
417 * Force strict CPU ordering. 356 * Force strict CPU ordering.
418 * And yes, this is required on UP too when we're talking 357 * And yes, this is required on UP too when we're talking
@@ -558,5 +497,6 @@ static inline void sched_cacheflush(void)
558} 497}
559 498
560extern unsigned long arch_align_stack(unsigned long sp); 499extern unsigned long arch_align_stack(unsigned long sp);
500extern void free_init_pages(char *what, unsigned long begin, unsigned long end);
561 501
562#endif 502#endif
diff --git a/include/asm-i386/uaccess.h b/include/asm-i386/uaccess.h
index 3f1337c34208..371457b1ceb6 100644
--- a/include/asm-i386/uaccess.h
+++ b/include/asm-i386/uaccess.h
@@ -197,13 +197,15 @@ extern void __put_user_8(void);
197 197
198#define put_user(x,ptr) \ 198#define put_user(x,ptr) \
199({ int __ret_pu; \ 199({ int __ret_pu; \
200 __typeof__(*(ptr)) __pu_val; \
200 __chk_user_ptr(ptr); \ 201 __chk_user_ptr(ptr); \
202 __pu_val = x; \
201 switch(sizeof(*(ptr))) { \ 203 switch(sizeof(*(ptr))) { \
202 case 1: __put_user_1(x, ptr); break; \ 204 case 1: __put_user_1(__pu_val, ptr); break; \
203 case 2: __put_user_2(x, ptr); break; \ 205 case 2: __put_user_2(__pu_val, ptr); break; \
204 case 4: __put_user_4(x, ptr); break; \ 206 case 4: __put_user_4(__pu_val, ptr); break; \
205 case 8: __put_user_8(x, ptr); break; \ 207 case 8: __put_user_8(__pu_val, ptr); break; \
206 default:__put_user_X(x, ptr); break; \ 208 default:__put_user_X(__pu_val, ptr); break; \
207 } \ 209 } \
208 __ret_pu; \ 210 __ret_pu; \
209}) 211})
diff --git a/include/asm-i386/unistd.h b/include/asm-i386/unistd.h
index dc81a55dd94d..d8afd0e3b81a 100644
--- a/include/asm-i386/unistd.h
+++ b/include/asm-i386/unistd.h
@@ -347,9 +347,9 @@ __syscall_return(type,__res); \
347type name(type1 arg1) \ 347type name(type1 arg1) \
348{ \ 348{ \
349long __res; \ 349long __res; \
350__asm__ volatile ("int $0x80" \ 350__asm__ volatile ("push %%ebx ; movl %2,%%ebx ; int $0x80 ; pop %%ebx" \
351 : "=a" (__res) \ 351 : "=a" (__res) \
352 : "0" (__NR_##name),"b" ((long)(arg1)) : "memory"); \ 352 : "0" (__NR_##name),"ri" ((long)(arg1)) : "memory"); \
353__syscall_return(type,__res); \ 353__syscall_return(type,__res); \
354} 354}
355 355
@@ -357,9 +357,10 @@ __syscall_return(type,__res); \
357type name(type1 arg1,type2 arg2) \ 357type name(type1 arg1,type2 arg2) \
358{ \ 358{ \
359long __res; \ 359long __res; \
360__asm__ volatile ("int $0x80" \ 360__asm__ volatile ("push %%ebx ; movl %2,%%ebx ; int $0x80 ; pop %%ebx" \
361 : "=a" (__res) \ 361 : "=a" (__res) \
362 : "0" (__NR_##name),"b" ((long)(arg1)),"c" ((long)(arg2)) : "memory"); \ 362 : "0" (__NR_##name),"ri" ((long)(arg1)),"c" ((long)(arg2)) \
363 : "memory"); \
363__syscall_return(type,__res); \ 364__syscall_return(type,__res); \
364} 365}
365 366
@@ -367,9 +368,9 @@ __syscall_return(type,__res); \
367type name(type1 arg1,type2 arg2,type3 arg3) \ 368type name(type1 arg1,type2 arg2,type3 arg3) \
368{ \ 369{ \
369long __res; \ 370long __res; \
370__asm__ volatile ("int $0x80" \ 371__asm__ volatile ("push %%ebx ; movl %2,%%ebx ; int $0x80 ; pop %%ebx" \
371 : "=a" (__res) \ 372 : "=a" (__res) \
372 : "0" (__NR_##name),"b" ((long)(arg1)),"c" ((long)(arg2)), \ 373 : "0" (__NR_##name),"ri" ((long)(arg1)),"c" ((long)(arg2)), \
373 "d" ((long)(arg3)) : "memory"); \ 374 "d" ((long)(arg3)) : "memory"); \
374__syscall_return(type,__res); \ 375__syscall_return(type,__res); \
375} 376}
@@ -378,9 +379,9 @@ __syscall_return(type,__res); \
378type name (type1 arg1, type2 arg2, type3 arg3, type4 arg4) \ 379type name (type1 arg1, type2 arg2, type3 arg3, type4 arg4) \
379{ \ 380{ \
380long __res; \ 381long __res; \
381__asm__ volatile ("int $0x80" \ 382__asm__ volatile ("push %%ebx ; movl %2,%%ebx ; int $0x80 ; pop %%ebx" \
382 : "=a" (__res) \ 383 : "=a" (__res) \
383 : "0" (__NR_##name),"b" ((long)(arg1)),"c" ((long)(arg2)), \ 384 : "0" (__NR_##name),"ri" ((long)(arg1)),"c" ((long)(arg2)), \
384 "d" ((long)(arg3)),"S" ((long)(arg4)) : "memory"); \ 385 "d" ((long)(arg3)),"S" ((long)(arg4)) : "memory"); \
385__syscall_return(type,__res); \ 386__syscall_return(type,__res); \
386} 387}
@@ -390,10 +391,12 @@ __syscall_return(type,__res); \
390type name (type1 arg1,type2 arg2,type3 arg3,type4 arg4,type5 arg5) \ 391type name (type1 arg1,type2 arg2,type3 arg3,type4 arg4,type5 arg5) \
391{ \ 392{ \
392long __res; \ 393long __res; \
393__asm__ volatile ("int $0x80" \ 394__asm__ volatile ("push %%ebx ; movl %2,%%ebx ; movl %1,%%eax ; " \
395 "int $0x80 ; pop %%ebx" \
394 : "=a" (__res) \ 396 : "=a" (__res) \
395 : "0" (__NR_##name),"b" ((long)(arg1)),"c" ((long)(arg2)), \ 397 : "i" (__NR_##name),"ri" ((long)(arg1)),"c" ((long)(arg2)), \
396 "d" ((long)(arg3)),"S" ((long)(arg4)),"D" ((long)(arg5)) : "memory"); \ 398 "d" ((long)(arg3)),"S" ((long)(arg4)),"D" ((long)(arg5)) \
399 : "memory"); \
397__syscall_return(type,__res); \ 400__syscall_return(type,__res); \
398} 401}
399 402
@@ -402,11 +405,14 @@ __syscall_return(type,__res); \
402type name (type1 arg1,type2 arg2,type3 arg3,type4 arg4,type5 arg5,type6 arg6) \ 405type name (type1 arg1,type2 arg2,type3 arg3,type4 arg4,type5 arg5,type6 arg6) \
403{ \ 406{ \
404long __res; \ 407long __res; \
405__asm__ volatile ("push %%ebp ; movl %%eax,%%ebp ; movl %1,%%eax ; int $0x80 ; pop %%ebp" \ 408 struct { long __a1; long __a6; } __s = { (long)arg1, (long)arg6 }; \
409__asm__ volatile ("push %%ebp ; push %%ebx ; movl 4(%2),%%ebp ; " \
410 "movl 0(%2),%%ebx ; movl %1,%%eax ; int $0x80 ; " \
411 "pop %%ebx ; pop %%ebp" \
406 : "=a" (__res) \ 412 : "=a" (__res) \
407 : "i" (__NR_##name),"b" ((long)(arg1)),"c" ((long)(arg2)), \ 413 : "i" (__NR_##name),"0" ((long)(&__s)),"c" ((long)(arg2)), \
408 "d" ((long)(arg3)),"S" ((long)(arg4)),"D" ((long)(arg5)), \ 414 "d" ((long)(arg3)),"S" ((long)(arg4)),"D" ((long)(arg5)) \
409 "0" ((long)(arg6)) : "memory"); \ 415 : "memory"); \
410__syscall_return(type,__res); \ 416__syscall_return(type,__res); \
411} 417}
412 418
generated by cgit v1.2.2 (git 2.25.0) at 2024-11-16 02:26:05 -0500