diff options
Diffstat (limited to 'arch/x86/xen/xen-asm_32.S')
-rw-r--r-- | arch/x86/xen/xen-asm_32.S | 238 |
1 files changed, 124 insertions, 114 deletions
diff --git a/arch/x86/xen/xen-asm_32.S b/arch/x86/xen/xen-asm_32.S index 082d173caaf3..88e15deb8b82 100644 --- a/arch/x86/xen/xen-asm_32.S +++ b/arch/x86/xen/xen-asm_32.S | |||
@@ -1,17 +1,16 @@ | |||
1 | /* | 1 | /* |
2 | Asm versions of Xen pv-ops, suitable for either direct use or inlining. | 2 | * Asm versions of Xen pv-ops, suitable for either direct use or |
3 | The inline versions are the same as the direct-use versions, with the | 3 | * inlining. The inline versions are the same as the direct-use |
4 | pre- and post-amble chopped off. | 4 | * versions, with the pre- and post-amble chopped off. |
5 | 5 | * | |
6 | This code is encoded for size rather than absolute efficiency, | 6 | * This code is encoded for size rather than absolute efficiency, with |
7 | with a view to being able to inline as much as possible. | 7 | * a view to being able to inline as much as possible. |
8 | 8 | * | |
9 | We only bother with direct forms (ie, vcpu in pda) of the operations | 9 | * We only bother with direct forms (ie, vcpu in pda) of the |
10 | here; the indirect forms are better handled in C, since they're | 10 | * operations here; the indirect forms are better handled in C, since |
11 | generally too large to inline anyway. | 11 | * they're generally too large to inline anyway. |
12 | */ | 12 | */ |
13 | 13 | ||
14 | //#include <asm/asm-offsets.h> | ||
15 | #include <asm/thread_info.h> | 14 | #include <asm/thread_info.h> |
16 | #include <asm/processor-flags.h> | 15 | #include <asm/processor-flags.h> |
17 | #include <asm/segment.h> | 16 | #include <asm/segment.h> |
@@ -21,8 +20,8 @@ | |||
21 | #include "xen-asm.h" | 20 | #include "xen-asm.h" |
22 | 21 | ||
23 | /* | 22 | /* |
24 | Force an event check by making a hypercall, | 23 | * Force an event check by making a hypercall, but preserve regs |
25 | but preserve regs before making the call. | 24 | * before making the call. |
26 | */ | 25 | */ |
27 | check_events: | 26 | check_events: |
28 | push %eax | 27 | push %eax |
@@ -35,10 +34,10 @@ check_events: | |||
35 | ret | 34 | ret |
36 | 35 | ||
37 | /* | 36 | /* |
38 | We can't use sysexit directly, because we're not running in ring0. | 37 | * We can't use sysexit directly, because we're not running in ring0. |
39 | But we can easily fake it up using iret. Assuming xen_sysexit | 38 | * But we can easily fake it up using iret. Assuming xen_sysexit is |
40 | is jumped to with a standard stack frame, we can just strip it | 39 | * jumped to with a standard stack frame, we can just strip it back to |
41 | back to a standard iret frame and use iret. | 40 | * a standard iret frame and use iret. |
42 | */ | 41 | */ |
43 | ENTRY(xen_sysexit) | 42 | ENTRY(xen_sysexit) |
44 | movl PT_EAX(%esp), %eax /* Shouldn't be necessary? */ | 43 | movl PT_EAX(%esp), %eax /* Shouldn't be necessary? */ |
@@ -49,33 +48,31 @@ ENTRY(xen_sysexit) | |||
49 | ENDPROC(xen_sysexit) | 48 | ENDPROC(xen_sysexit) |
50 | 49 | ||
51 | /* | 50 | /* |
52 | This is run where a normal iret would be run, with the same stack setup: | 51 | * This is run where a normal iret would be run, with the same stack setup: |
53 | 8: eflags | 52 | * 8: eflags |
54 | 4: cs | 53 | * 4: cs |
55 | esp-> 0: eip | 54 | * esp-> 0: eip |
56 | 55 | * | |
57 | This attempts to make sure that any pending events are dealt | 56 | * This attempts to make sure that any pending events are dealt with |
58 | with on return to usermode, but there is a small window in | 57 | * on return to usermode, but there is a small window in which an |
59 | which an event can happen just before entering usermode. If | 58 | * event can happen just before entering usermode. If the nested |
60 | the nested interrupt ends up setting one of the TIF_WORK_MASK | 59 | * interrupt ends up setting one of the TIF_WORK_MASK pending work |
61 | pending work flags, they will not be tested again before | 60 | * flags, they will not be tested again before returning to |
62 | returning to usermode. This means that a process can end up | 61 | * usermode. This means that a process can end up with pending work, |
63 | with pending work, which will be unprocessed until the process | 62 | * which will be unprocessed until the process enters and leaves the |
64 | enters and leaves the kernel again, which could be an | 63 | * kernel again, which could be an unbounded amount of time. This |
65 | unbounded amount of time. This means that a pending signal or | 64 | * means that a pending signal or reschedule event could be |
66 | reschedule event could be indefinitely delayed. | 65 | * indefinitely delayed. |
67 | 66 | * | |
68 | The fix is to notice a nested interrupt in the critical | 67 | * The fix is to notice a nested interrupt in the critical window, and |
69 | window, and if one occurs, then fold the nested interrupt into | 68 | * if one occurs, then fold the nested interrupt into the current |
70 | the current interrupt stack frame, and re-process it | 69 | * interrupt stack frame, and re-process it iteratively rather than |
71 | iteratively rather than recursively. This means that it will | 70 | * recursively. This means that it will exit via the normal path, and |
72 | exit via the normal path, and all pending work will be dealt | 71 | * all pending work will be dealt with appropriately. |
73 | with appropriately. | 72 | * |
74 | 73 | * Because the nested interrupt handler needs to deal with the current | |
75 | Because the nested interrupt handler needs to deal with the | 74 | * stack state in whatever form its in, we keep things simple by only |
76 | current stack state in whatever form its in, we keep things | 75 | * using a single register which is pushed/popped on the stack. |
77 | simple by only using a single register which is pushed/popped | ||
78 | on the stack. | ||
79 | */ | 76 | */ |
80 | ENTRY(xen_iret) | 77 | ENTRY(xen_iret) |
81 | /* test eflags for special cases */ | 78 | /* test eflags for special cases */ |
@@ -85,13 +82,15 @@ ENTRY(xen_iret) | |||
85 | push %eax | 82 | push %eax |
86 | ESP_OFFSET=4 # bytes pushed onto stack | 83 | ESP_OFFSET=4 # bytes pushed onto stack |
87 | 84 | ||
88 | /* Store vcpu_info pointer for easy access. Do it this | 85 | /* |
89 | way to avoid having to reload %fs */ | 86 | * Store vcpu_info pointer for easy access. Do it this way to |
87 | * avoid having to reload %fs | ||
88 | */ | ||
90 | #ifdef CONFIG_SMP | 89 | #ifdef CONFIG_SMP |
91 | GET_THREAD_INFO(%eax) | 90 | GET_THREAD_INFO(%eax) |
92 | movl TI_cpu(%eax),%eax | 91 | movl TI_cpu(%eax), %eax |
93 | movl __per_cpu_offset(,%eax,4),%eax | 92 | movl __per_cpu_offset(,%eax,4), %eax |
94 | mov per_cpu__xen_vcpu(%eax),%eax | 93 | mov per_cpu__xen_vcpu(%eax), %eax |
95 | #else | 94 | #else |
96 | movl per_cpu__xen_vcpu, %eax | 95 | movl per_cpu__xen_vcpu, %eax |
97 | #endif | 96 | #endif |
@@ -99,37 +98,46 @@ ENTRY(xen_iret) | |||
99 | /* check IF state we're restoring */ | 98 | /* check IF state we're restoring */ |
100 | testb $X86_EFLAGS_IF>>8, 8+1+ESP_OFFSET(%esp) | 99 | testb $X86_EFLAGS_IF>>8, 8+1+ESP_OFFSET(%esp) |
101 | 100 | ||
102 | /* Maybe enable events. Once this happens we could get a | 101 | /* |
103 | recursive event, so the critical region starts immediately | 102 | * Maybe enable events. Once this happens we could get a |
104 | afterwards. However, if that happens we don't end up | 103 | * recursive event, so the critical region starts immediately |
105 | resuming the code, so we don't have to be worried about | 104 | * afterwards. However, if that happens we don't end up |
106 | being preempted to another CPU. */ | 105 | * resuming the code, so we don't have to be worried about |
106 | * being preempted to another CPU. | ||
107 | */ | ||
107 | setz XEN_vcpu_info_mask(%eax) | 108 | setz XEN_vcpu_info_mask(%eax) |
108 | xen_iret_start_crit: | 109 | xen_iret_start_crit: |
109 | 110 | ||
110 | /* check for unmasked and pending */ | 111 | /* check for unmasked and pending */ |
111 | cmpw $0x0001, XEN_vcpu_info_pending(%eax) | 112 | cmpw $0x0001, XEN_vcpu_info_pending(%eax) |
112 | 113 | ||
113 | /* If there's something pending, mask events again so we | 114 | /* |
114 | can jump back into xen_hypervisor_callback */ | 115 | * If there's something pending, mask events again so we can |
116 | * jump back into xen_hypervisor_callback | ||
117 | */ | ||
115 | sete XEN_vcpu_info_mask(%eax) | 118 | sete XEN_vcpu_info_mask(%eax) |
116 | 119 | ||
117 | popl %eax | 120 | popl %eax |
118 | 121 | ||
119 | /* From this point on the registers are restored and the stack | 122 | /* |
120 | updated, so we don't need to worry about it if we're preempted */ | 123 | * From this point on the registers are restored and the stack |
124 | * updated, so we don't need to worry about it if we're | ||
125 | * preempted | ||
126 | */ | ||
121 | iret_restore_end: | 127 | iret_restore_end: |
122 | 128 | ||
123 | /* Jump to hypervisor_callback after fixing up the stack. | 129 | /* |
124 | Events are masked, so jumping out of the critical | 130 | * Jump to hypervisor_callback after fixing up the stack. |
125 | region is OK. */ | 131 | * Events are masked, so jumping out of the critical region is |
132 | * OK. | ||
133 | */ | ||
126 | je xen_hypervisor_callback | 134 | je xen_hypervisor_callback |
127 | 135 | ||
128 | 1: iret | 136 | 1: iret |
129 | xen_iret_end_crit: | 137 | xen_iret_end_crit: |
130 | .section __ex_table,"a" | 138 | .section __ex_table, "a" |
131 | .align 4 | 139 | .align 4 |
132 | .long 1b,iret_exc | 140 | .long 1b, iret_exc |
133 | .previous | 141 | .previous |
134 | 142 | ||
135 | hyper_iret: | 143 | hyper_iret: |
@@ -139,55 +147,55 @@ hyper_iret: | |||
139 | .globl xen_iret_start_crit, xen_iret_end_crit | 147 | .globl xen_iret_start_crit, xen_iret_end_crit |
140 | 148 | ||
141 | /* | 149 | /* |
142 | This is called by xen_hypervisor_callback in entry.S when it sees | 150 | * This is called by xen_hypervisor_callback in entry.S when it sees |
143 | that the EIP at the time of interrupt was between xen_iret_start_crit | 151 | * that the EIP at the time of interrupt was between |
144 | and xen_iret_end_crit. We're passed the EIP in %eax so we can do | 152 | * xen_iret_start_crit and xen_iret_end_crit. We're passed the EIP in |
145 | a more refined determination of what to do. | 153 | * %eax so we can do a more refined determination of what to do. |
146 | 154 | * | |
147 | The stack format at this point is: | 155 | * The stack format at this point is: |
148 | ---------------- | 156 | * ---------------- |
149 | ss : (ss/esp may be present if we came from usermode) | 157 | * ss : (ss/esp may be present if we came from usermode) |
150 | esp : | 158 | * esp : |
151 | eflags } outer exception info | 159 | * eflags } outer exception info |
152 | cs } | 160 | * cs } |
153 | eip } | 161 | * eip } |
154 | ---------------- <- edi (copy dest) | 162 | * ---------------- <- edi (copy dest) |
155 | eax : outer eax if it hasn't been restored | 163 | * eax : outer eax if it hasn't been restored |
156 | ---------------- | 164 | * ---------------- |
157 | eflags } nested exception info | 165 | * eflags } nested exception info |
158 | cs } (no ss/esp because we're nested | 166 | * cs } (no ss/esp because we're nested |
159 | eip } from the same ring) | 167 | * eip } from the same ring) |
160 | orig_eax }<- esi (copy src) | 168 | * orig_eax }<- esi (copy src) |
161 | - - - - - - - - | 169 | * - - - - - - - - |
162 | fs } | 170 | * fs } |
163 | es } | 171 | * es } |
164 | ds } SAVE_ALL state | 172 | * ds } SAVE_ALL state |
165 | eax } | 173 | * eax } |
166 | : : | 174 | * : : |
167 | ebx }<- esp | 175 | * ebx }<- esp |
168 | ---------------- | 176 | * ---------------- |
169 | 177 | * | |
170 | In order to deliver the nested exception properly, we need to shift | 178 | * In order to deliver the nested exception properly, we need to shift |
171 | everything from the return addr up to the error code so it | 179 | * everything from the return addr up to the error code so it sits |
172 | sits just under the outer exception info. This means that when we | 180 | * just under the outer exception info. This means that when we |
173 | handle the exception, we do it in the context of the outer exception | 181 | * handle the exception, we do it in the context of the outer |
174 | rather than starting a new one. | 182 | * exception rather than starting a new one. |
175 | 183 | * | |
176 | The only caveat is that if the outer eax hasn't been | 184 | * The only caveat is that if the outer eax hasn't been restored yet |
177 | restored yet (ie, it's still on stack), we need to insert | 185 | * (ie, it's still on stack), we need to insert its value into the |
178 | its value into the SAVE_ALL state before going on, since | 186 | * SAVE_ALL state before going on, since it's usermode state which we |
179 | it's usermode state which we eventually need to restore. | 187 | * eventually need to restore. |
180 | */ | 188 | */ |
181 | ENTRY(xen_iret_crit_fixup) | 189 | ENTRY(xen_iret_crit_fixup) |
182 | /* | 190 | /* |
183 | Paranoia: Make sure we're really coming from kernel space. | 191 | * Paranoia: Make sure we're really coming from kernel space. |
184 | One could imagine a case where userspace jumps into the | 192 | * One could imagine a case where userspace jumps into the |
185 | critical range address, but just before the CPU delivers a GP, | 193 | * critical range address, but just before the CPU delivers a |
186 | it decides to deliver an interrupt instead. Unlikely? | 194 | * GP, it decides to deliver an interrupt instead. Unlikely? |
187 | Definitely. Easy to avoid? Yes. The Intel documents | 195 | * Definitely. Easy to avoid? Yes. The Intel documents |
188 | explicitly say that the reported EIP for a bad jump is the | 196 | * explicitly say that the reported EIP for a bad jump is the |
189 | jump instruction itself, not the destination, but some virtual | 197 | * jump instruction itself, not the destination, but some |
190 | environments get this wrong. | 198 | * virtual environments get this wrong. |
191 | */ | 199 | */ |
192 | movl PT_CS(%esp), %ecx | 200 | movl PT_CS(%esp), %ecx |
193 | andl $SEGMENT_RPL_MASK, %ecx | 201 | andl $SEGMENT_RPL_MASK, %ecx |
@@ -197,15 +205,17 @@ ENTRY(xen_iret_crit_fixup) | |||
197 | lea PT_ORIG_EAX(%esp), %esi | 205 | lea PT_ORIG_EAX(%esp), %esi |
198 | lea PT_EFLAGS(%esp), %edi | 206 | lea PT_EFLAGS(%esp), %edi |
199 | 207 | ||
200 | /* If eip is before iret_restore_end then stack | 208 | /* |
201 | hasn't been restored yet. */ | 209 | * If eip is before iret_restore_end then stack |
210 | * hasn't been restored yet. | ||
211 | */ | ||
202 | cmp $iret_restore_end, %eax | 212 | cmp $iret_restore_end, %eax |
203 | jae 1f | 213 | jae 1f |
204 | 214 | ||
205 | movl 0+4(%edi),%eax /* copy EAX (just above top of frame) */ | 215 | movl 0+4(%edi), %eax /* copy EAX (just above top of frame) */ |
206 | movl %eax, PT_EAX(%esp) | 216 | movl %eax, PT_EAX(%esp) |
207 | 217 | ||
208 | lea ESP_OFFSET(%edi),%edi /* move dest up over saved regs */ | 218 | lea ESP_OFFSET(%edi), %edi /* move dest up over saved regs */ |
209 | 219 | ||
210 | /* set up the copy */ | 220 | /* set up the copy */ |
211 | 1: std | 221 | 1: std |
@@ -213,6 +223,6 @@ ENTRY(xen_iret_crit_fixup) | |||
213 | rep movsl | 223 | rep movsl |
214 | cld | 224 | cld |
215 | 225 | ||
216 | lea 4(%edi),%esp /* point esp to new frame */ | 226 | lea 4(%edi), %esp /* point esp to new frame */ |
217 | 2: jmp xen_do_upcall | 227 | 2: jmp xen_do_upcall |
218 | 228 | ||