aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorIsaku Yamahata <yamahata@valinux.co.jp>2009-03-04 07:06:52 -0500
committerTony Luck <tony.luck@intel.com>2009-03-26 14:02:42 -0400
commit03f511dd02f1431ef652fb97a7f2fe7aef47e025 (patch)
tree87bbff26febaa0eb0e85d1c23886a1cc913af30a
parentbf7ab02f620c1020c869fc71a2c855918b6a5375 (diff)
ia64/pv_ops: implement binary patching optimization for native.
implement binary patching optimization for pv_cpu_ops. With this optimization, indirect call for pv_cpu_ops methods can be converted into inline execution or direct call. Signed-off-by: Isaku Yamahata <yamahata@valinux.co.jp> Signed-off-by: Tony Luck <tony.luck@intel.com>
Diffstat
-rw-r--r--arch/ia64/include/asm/intrinsics.h6
-rw-r--r--arch/ia64/include/asm/paravirt.h8
-rw-r--r--arch/ia64/include/asm/paravirt_privop.h341
-rw-r--r--arch/ia64/kernel/Makefile3
-rw-r--r--arch/ia64/kernel/paravirt.c520
-rw-r--r--arch/ia64/kernel/paravirtentry.S43
-rw-r--r--arch/ia64/kernel/setup.c2
7 files changed, 898 insertions, 25 deletions
diff --git a/arch/ia64/include/asm/intrinsics.h b/arch/ia64/include/asm/intrinsics.h
index a3e44a5ed497..fbe2ad9234d0 100644
--- a/arch/ia64/include/asm/intrinsics.h
+++ b/arch/ia64/include/asm/intrinsics.h
@@ -201,7 +201,11 @@ extern long ia64_cmpxchg_called_with_bad_pointer (void);
201 201
202#ifndef __ASSEMBLY__ 202#ifndef __ASSEMBLY__
203#if defined(CONFIG_PARAVIRT) && defined(__KERNEL__) 203#if defined(CONFIG_PARAVIRT) && defined(__KERNEL__)
204#define IA64_INTRINSIC_API(name) pv_cpu_ops.name 204#ifdef ASM_SUPPORTED
205# define IA64_INTRINSIC_API(name) paravirt_ ## name
206#else
207# define IA64_INTRINSIC_API(name) pv_cpu_ops.name
208#endif
205#define IA64_INTRINSIC_MACRO(name) paravirt_ ## name 209#define IA64_INTRINSIC_MACRO(name) paravirt_ ## name
206#else 210#else
207#define IA64_INTRINSIC_API(name) ia64_native_ ## name 211#define IA64_INTRINSIC_API(name) ia64_native_ ## name
diff --git a/arch/ia64/include/asm/paravirt.h b/arch/ia64/include/asm/paravirt.h
index fc433f6c3275..2eb0a981a09a 100644
--- a/arch/ia64/include/asm/paravirt.h
+++ b/arch/ia64/include/asm/paravirt.h
@@ -118,6 +118,14 @@ struct pv_init_ops {
118 int (*arch_setup_nomca)(void); 118 int (*arch_setup_nomca)(void);
119 119
120 void (*post_smp_prepare_boot_cpu)(void); 120 void (*post_smp_prepare_boot_cpu)(void);
121
122#ifdef ASM_SUPPORTED
123 unsigned long (*patch_bundle)(void *sbundle, void *ebundle,
124 unsigned long type);
125 unsigned long (*patch_inst)(unsigned long stag, unsigned long etag,
126 unsigned long type);
127#endif
128 void (*patch_branch)(unsigned long tag, unsigned long type);
121}; 129};
122 130
123extern struct pv_init_ops pv_init_ops; 131extern struct pv_init_ops pv_init_ops;
diff --git a/arch/ia64/include/asm/paravirt_privop.h b/arch/ia64/include/asm/paravirt_privop.h
index 33c8e55f5775..76d6a6943e83 100644
--- a/arch/ia64/include/asm/paravirt_privop.h
+++ b/arch/ia64/include/asm/paravirt_privop.h
@@ -60,12 +60,18 @@ extern unsigned long ia64_native_getreg_func(int regnum);
60/* Instructions paravirtualized for performance */ 60/* Instructions paravirtualized for performance */
61/************************************************/ 61/************************************************/
62 62
63#ifndef ASM_SUPPORTED
64#define paravirt_ssm_i() pv_cpu_ops.ssm_i()
65#define paravirt_rsm_i() pv_cpu_ops.rsm_i()
66#define __paravirt_getreg() pv_cpu_ops.getreg()
67#endif
68
63/* mask for ia64_native_ssm/rsm() must be constant.("i" constraing). 69/* mask for ia64_native_ssm/rsm() must be constant.("i" constraing).
64 * static inline function doesn't satisfy it. */ 70 * static inline function doesn't satisfy it. */
65#define paravirt_ssm(mask) \ 71#define paravirt_ssm(mask) \
66 do { \ 72 do { \
67 if ((mask) == IA64_PSR_I) \ 73 if ((mask) == IA64_PSR_I) \
68 pv_cpu_ops.ssm_i(); \ 74 paravirt_ssm_i(); \
69 else \ 75 else \
70 ia64_native_ssm(mask); \ 76 ia64_native_ssm(mask); \
71 } while (0) 77 } while (0)
@@ -73,7 +79,7 @@ extern unsigned long ia64_native_getreg_func(int regnum);
73#define paravirt_rsm(mask) \ 79#define paravirt_rsm(mask) \
74 do { \ 80 do { \
75 if ((mask) == IA64_PSR_I) \ 81 if ((mask) == IA64_PSR_I) \
76 pv_cpu_ops.rsm_i(); \ 82 paravirt_rsm_i(); \
77 else \ 83 else \
78 ia64_native_rsm(mask); \ 84 ia64_native_rsm(mask); \
79 } while (0) 85 } while (0)
@@ -86,7 +92,7 @@ extern unsigned long ia64_native_getreg_func(int regnum);
86 if ((reg) == _IA64_REG_IP) \ 92 if ((reg) == _IA64_REG_IP) \
87 res = ia64_native_getreg(_IA64_REG_IP); \ 93 res = ia64_native_getreg(_IA64_REG_IP); \
88 else \ 94 else \
89 res = pv_cpu_ops.getreg(reg); \ 95 res = __paravirt_getreg(reg); \
90 res; \ 96 res; \
91 }) 97 })
92 98
@@ -121,4 +127,333 @@ void paravirt_cpu_asm_init(const struct pv_cpu_asm_switch *cpu_asm_switch);
121 IA64_PARAVIRT_ASM_FUNC(work_processed_syscall) 127 IA64_PARAVIRT_ASM_FUNC(work_processed_syscall)
122#define ia64_leave_kernel IA64_PARAVIRT_ASM_FUNC(leave_kernel) 128#define ia64_leave_kernel IA64_PARAVIRT_ASM_FUNC(leave_kernel)
123 129
130
131#if defined(CONFIG_PARAVIRT)
132/******************************************************************************
133 * binary patching infrastructure
134 */
135#define PARAVIRT_PATCH_TYPE_FC 1
136#define PARAVIRT_PATCH_TYPE_THASH 2
137#define PARAVIRT_PATCH_TYPE_GET_CPUID 3
138#define PARAVIRT_PATCH_TYPE_GET_PMD 4
139#define PARAVIRT_PATCH_TYPE_PTCGA 5
140#define PARAVIRT_PATCH_TYPE_GET_RR 6
141#define PARAVIRT_PATCH_TYPE_SET_RR 7
142#define PARAVIRT_PATCH_TYPE_SET_RR0_TO_RR4 8
143#define PARAVIRT_PATCH_TYPE_SSM_I 9
144#define PARAVIRT_PATCH_TYPE_RSM_I 10
145#define PARAVIRT_PATCH_TYPE_GET_PSR_I 11
146#define PARAVIRT_PATCH_TYPE_INTRIN_LOCAL_IRQ_RESTORE 12
147
148/* PARAVIRT_PATY_TYPE_[GS]ETREG + _IA64_REG_xxx */
149#define PARAVIRT_PATCH_TYPE_GETREG 0x10000000
150#define PARAVIRT_PATCH_TYPE_SETREG 0x20000000
151
152/*
153 * struct task_struct* (*ia64_switch_to)(void* next_task);
154 * void *ia64_leave_syscall;
155 * void *ia64_work_processed_syscall
156 * void *ia64_leave_kernel;
157 */
158
159#define PARAVIRT_PATCH_TYPE_BR_START 0x30000000
160#define PARAVIRT_PATCH_TYPE_BR_SWITCH_TO \
161 (PARAVIRT_PATCH_TYPE_BR_START + 0)
162#define PARAVIRT_PATCH_TYPE_BR_LEAVE_SYSCALL \
163 (PARAVIRT_PATCH_TYPE_BR_START + 1)
164#define PARAVIRT_PATCH_TYPE_BR_WORK_PROCESSED_SYSCALL \
165 (PARAVIRT_PATCH_TYPE_BR_START + 2)
166#define PARAVIRT_PATCH_TYPE_BR_LEAVE_KERNEL \
167 (PARAVIRT_PATCH_TYPE_BR_START + 3)
168
169#ifdef ASM_SUPPORTED
170#include <asm/paravirt_patch.h>
171
172/*
173 * pv_cpu_ops calling stub.
174 * normal function call convension can't be written by gcc
175 * inline assembly.
176 *
177 * from the caller's point of view,
178 * the following registers will be clobbered.
179 * r2, r3
180 * r8-r15
181 * r16, r17
182 * b6, b7
183 * p6-p15
184 * ar.ccv
185 *
186 * from the callee's point of view ,
187 * the following registers can be used.
188 * r2, r3: scratch
189 * r8: scratch, input argument0 and return value
190 * r0-r15: scratch, input argument1-5
191 * b6: return pointer
192 * b7: scratch
193 * p6-p15: scratch
194 * ar.ccv: scratch
195 *
196 * other registers must not be changed. especially
197 * b0: rp: preserved. gcc ignores b0 in clobbered register.
198 * r16: saved gp
199 */
200/* 5 bundles */
201#define __PARAVIRT_BR \
202 ";;\n" \
203 "{ .mlx\n" \
204 "nop 0\n" \
205 "movl r2 = %[op_addr]\n"/* get function pointer address */ \
206 ";;\n" \
207 "}\n" \
208 "1:\n" \
209 "{ .mii\n" \
210 "ld8 r2 = [r2]\n" /* load function descriptor address */ \
211 "mov r17 = ip\n" /* get ip to calc return address */ \
212 "mov r16 = gp\n" /* save gp */ \
213 ";;\n" \
214 "}\n" \
215 "{ .mii\n" \
216 "ld8 r3 = [r2], 8\n" /* load entry address */ \
217 "adds r17 = 1f - 1b, r17\n" /* calculate return address */ \
218 ";;\n" \
219 "mov b7 = r3\n" /* set entry address */ \
220 "}\n" \
221 "{ .mib\n" \
222 "ld8 gp = [r2]\n" /* load gp value */ \
223 "mov b6 = r17\n" /* set return address */ \
224 "br.cond.sptk.few b7\n" /* intrinsics are very short isns */ \
225 "}\n" \
226 "1:\n" \
227 "{ .mii\n" \
228 "mov gp = r16\n" /* restore gp value */ \
229 "nop 0\n" \
230 "nop 0\n" \
231 ";;\n" \
232 "}\n"
233
234#define PARAVIRT_OP(op) \
235 [op_addr] "i"(&pv_cpu_ops.op)
236
237#define PARAVIRT_TYPE(type) \
238 PARAVIRT_PATCH_TYPE_ ## type
239
240#define PARAVIRT_REG_CLOBBERS0 \
241 "r2", "r3", /*"r8",*/ "r9", "r10", "r11", "r14", \
242 "r15", "r16", "r17"
243
244#define PARAVIRT_REG_CLOBBERS1 \
245 "r2","r3", /*"r8",*/ "r9", "r10", "r11", "r14", \
246 "r15", "r16", "r17"
247
248#define PARAVIRT_REG_CLOBBERS2 \
249 "r2", "r3", /*"r8", "r9",*/ "r10", "r11", "r14", \
250 "r15", "r16", "r17"
251
252#define PARAVIRT_REG_CLOBBERS5 \
253 "r2", "r3", /*"r8", "r9", "r10", "r11", "r14",*/ \
254 "r15", "r16", "r17"
255
256#define PARAVIRT_BR_CLOBBERS \
257 "b6", "b7"
258
259#define PARAVIRT_PR_CLOBBERS \
260 "p6", "p7", "p8", "p9", "p10", "p11", "p12", "p13", "p14", "p15"
261
262#define PARAVIRT_AR_CLOBBERS \
263 "ar.ccv"
264
265#define PARAVIRT_CLOBBERS0 \
266 PARAVIRT_REG_CLOBBERS0, \
267 PARAVIRT_BR_CLOBBERS, \
268 PARAVIRT_PR_CLOBBERS, \
269 PARAVIRT_AR_CLOBBERS, \
270 "memory"
271
272#define PARAVIRT_CLOBBERS1 \
273 PARAVIRT_REG_CLOBBERS1, \
274 PARAVIRT_BR_CLOBBERS, \
275 PARAVIRT_PR_CLOBBERS, \
276 PARAVIRT_AR_CLOBBERS, \
277 "memory"
278
279#define PARAVIRT_CLOBBERS2 \
280 PARAVIRT_REG_CLOBBERS2, \
281 PARAVIRT_BR_CLOBBERS, \
282 PARAVIRT_PR_CLOBBERS, \
283 PARAVIRT_AR_CLOBBERS, \
284 "memory"
285
286#define PARAVIRT_CLOBBERS5 \
287 PARAVIRT_REG_CLOBBERS5, \
288 PARAVIRT_BR_CLOBBERS, \
289 PARAVIRT_PR_CLOBBERS, \
290 PARAVIRT_AR_CLOBBERS, \
291 "memory"
292
293#define PARAVIRT_BR0(op, type) \
294 register unsigned long ia64_clobber asm ("r8"); \
295 asm volatile (paravirt_alt_bundle(__PARAVIRT_BR, \
296 PARAVIRT_TYPE(type)) \
297 : "=r"(ia64_clobber) \
298 : PARAVIRT_OP(op) \
299 : PARAVIRT_CLOBBERS0)
300
301#define PARAVIRT_BR0_RET(op, type) \
302 register unsigned long ia64_intri_res asm ("r8"); \
303 asm volatile (paravirt_alt_bundle(__PARAVIRT_BR, \
304 PARAVIRT_TYPE(type)) \
305 : "=r"(ia64_intri_res) \
306 : PARAVIRT_OP(op) \
307 : PARAVIRT_CLOBBERS0)
308
309#define PARAVIRT_BR1(op, type, arg1) \
310 register unsigned long __##arg1 asm ("r8") = arg1; \
311 register unsigned long ia64_clobber asm ("r8"); \
312 asm volatile (paravirt_alt_bundle(__PARAVIRT_BR, \
313 PARAVIRT_TYPE(type)) \
314 : "=r"(ia64_clobber) \
315 : PARAVIRT_OP(op), "0"(__##arg1) \
316 : PARAVIRT_CLOBBERS1)
317
318#define PARAVIRT_BR1_RET(op, type, arg1) \
319 register unsigned long ia64_intri_res asm ("r8"); \
320 register unsigned long __##arg1 asm ("r8") = arg1; \
321 asm volatile (paravirt_alt_bundle(__PARAVIRT_BR, \
322 PARAVIRT_TYPE(type)) \
323 : "=r"(ia64_intri_res) \
324 : PARAVIRT_OP(op), "0"(__##arg1) \
325 : PARAVIRT_CLOBBERS1)
326
327#define PARAVIRT_BR2(op, type, arg1, arg2) \
328 register unsigned long __##arg1 asm ("r8") = arg1; \
329 register unsigned long __##arg2 asm ("r9") = arg2; \
330 register unsigned long ia64_clobber1 asm ("r8"); \
331 register unsigned long ia64_clobber2 asm ("r9"); \
332 asm volatile (paravirt_alt_bundle(__PARAVIRT_BR, \
333 PARAVIRT_TYPE(type)) \
334 : "=r"(ia64_clobber1), "=r"(ia64_clobber2) \
335 : PARAVIRT_OP(op), "0"(__##arg1), "1"(__##arg2) \
336 : PARAVIRT_CLOBBERS2)
337
338
339#define PARAVIRT_DEFINE_CPU_OP0(op, type) \
340 static inline void \
341 paravirt_ ## op (void) \
342 { \
343 PARAVIRT_BR0(op, type); \
344 }
345
346#define PARAVIRT_DEFINE_CPU_OP0_RET(op, type) \
347 static inline unsigned long \
348 paravirt_ ## op (void) \
349 { \
350 PARAVIRT_BR0_RET(op, type); \
351 return ia64_intri_res; \
352 }
353
354#define PARAVIRT_DEFINE_CPU_OP1(op, type) \
355 static inline void \
356 paravirt_ ## op (unsigned long arg1) \
357 { \
358 PARAVIRT_BR1(op, type, arg1); \
359 }
360
361#define PARAVIRT_DEFINE_CPU_OP1_RET(op, type) \
362 static inline unsigned long \
363 paravirt_ ## op (unsigned long arg1) \
364 { \
365 PARAVIRT_BR1_RET(op, type, arg1); \
366 return ia64_intri_res; \
367 }
368
369#define PARAVIRT_DEFINE_CPU_OP2(op, type) \
370 static inline void \
371 paravirt_ ## op (unsigned long arg1, \
372 unsigned long arg2) \
373 { \
374 PARAVIRT_BR2(op, type, arg1, arg2); \
375 }
376
377
378PARAVIRT_DEFINE_CPU_OP1(fc, FC);
379PARAVIRT_DEFINE_CPU_OP1_RET(thash, THASH)
380PARAVIRT_DEFINE_CPU_OP1_RET(get_cpuid, GET_CPUID)
381PARAVIRT_DEFINE_CPU_OP1_RET(get_pmd, GET_PMD)
382PARAVIRT_DEFINE_CPU_OP2(ptcga, PTCGA)
383PARAVIRT_DEFINE_CPU_OP1_RET(get_rr, GET_RR)
384PARAVIRT_DEFINE_CPU_OP2(set_rr, SET_RR)
385PARAVIRT_DEFINE_CPU_OP0(ssm_i, SSM_I)
386PARAVIRT_DEFINE_CPU_OP0(rsm_i, RSM_I)
387PARAVIRT_DEFINE_CPU_OP0_RET(get_psr_i, GET_PSR_I)
388PARAVIRT_DEFINE_CPU_OP1(intrin_local_irq_restore, INTRIN_LOCAL_IRQ_RESTORE)
389
390static inline void
391paravirt_set_rr0_to_rr4(unsigned long val0, unsigned long val1,
392 unsigned long val2, unsigned long val3,
393 unsigned long val4)
394{
395 register unsigned long __val0 asm ("r8") = val0;
396 register unsigned long __val1 asm ("r9") = val1;
397 register unsigned long __val2 asm ("r10") = val2;
398 register unsigned long __val3 asm ("r11") = val3;
399 register unsigned long __val4 asm ("r14") = val4;
400
401 register unsigned long ia64_clobber0 asm ("r8");
402 register unsigned long ia64_clobber1 asm ("r9");
403 register unsigned long ia64_clobber2 asm ("r10");
404 register unsigned long ia64_clobber3 asm ("r11");
405 register unsigned long ia64_clobber4 asm ("r14");
406
407 asm volatile (paravirt_alt_bundle(__PARAVIRT_BR,
408 PARAVIRT_TYPE(SET_RR0_TO_RR4))
409 : "=r"(ia64_clobber0),
410 "=r"(ia64_clobber1),
411 "=r"(ia64_clobber2),
412 "=r"(ia64_clobber3),
413 "=r"(ia64_clobber4)
414 : PARAVIRT_OP(set_rr0_to_rr4),
415 "0"(__val0), "1"(__val1), "2"(__val2),
416 "3"(__val3), "4"(__val4)
417 : PARAVIRT_CLOBBERS5);
418}
419
420/* unsigned long paravirt_getreg(int reg) */
421#define __paravirt_getreg(reg) \
422 ({ \
423 register unsigned long ia64_intri_res asm ("r8"); \
424 register unsigned long __reg asm ("r8") = (reg); \
425 \
426 BUILD_BUG_ON(!__builtin_constant_p(reg)); \
427 asm volatile (paravirt_alt_bundle(__PARAVIRT_BR, \
428 PARAVIRT_TYPE(GETREG) \
429 + (reg)) \
430 : "=r"(ia64_intri_res) \
431 : PARAVIRT_OP(getreg), "0"(__reg) \
432 : PARAVIRT_CLOBBERS1); \
433 \
434 ia64_intri_res; \
435 })
436
437/* void paravirt_setreg(int reg, unsigned long val) */
438#define paravirt_setreg(reg, val) \
439 do { \
440 register unsigned long __val asm ("r8") = val; \
441 register unsigned long __reg asm ("r9") = reg; \
442 register unsigned long ia64_clobber1 asm ("r8"); \
443 register unsigned long ia64_clobber2 asm ("r9"); \
444 \
445 BUILD_BUG_ON(!__builtin_constant_p(reg)); \
446 asm volatile (paravirt_alt_bundle(__PARAVIRT_BR, \
447 PARAVIRT_TYPE(SETREG) \
448 + (reg)) \
449 : "=r"(ia64_clobber1), \
450 "=r"(ia64_clobber2) \
451 : PARAVIRT_OP(setreg), \
452 "1"(__reg), "0"(__val) \
453 : PARAVIRT_CLOBBERS2); \
454 } while (0)
455
456#endif /* ASM_SUPPORTED */
457#endif /* CONFIG_PARAVIRT && ASM_SUPPOTED */
458
124#endif /* _ASM_IA64_PARAVIRT_PRIVOP_H */ 459#endif /* _ASM_IA64_PARAVIRT_PRIVOP_H */
diff --git a/arch/ia64/kernel/Makefile b/arch/ia64/kernel/Makefile
index 8dc9df8a87a5..dbc19e4d5ef7 100644
--- a/arch/ia64/kernel/Makefile
+++ b/arch/ia64/kernel/Makefile
@@ -36,7 +36,8 @@ obj-$(CONFIG_PCI_MSI) += msi_ia64.o
36mca_recovery-y += mca_drv.o mca_drv_asm.o 36mca_recovery-y += mca_drv.o mca_drv_asm.o
37obj-$(CONFIG_IA64_MC_ERR_INJECT)+= err_inject.o 37obj-$(CONFIG_IA64_MC_ERR_INJECT)+= err_inject.o
38 38
39obj-$(CONFIG_PARAVIRT) += paravirt.o paravirtentry.o 39obj-$(CONFIG_PARAVIRT) += paravirt.o paravirtentry.o \
40 paravirt_patch.o
40 41
41obj-$(CONFIG_IA64_ESI) += esi.o 42obj-$(CONFIG_IA64_ESI) += esi.o
42ifneq ($(CONFIG_IA64_ESI),) 43ifneq ($(CONFIG_IA64_ESI),)
diff --git a/arch/ia64/kernel/paravirt.c b/arch/ia64/kernel/paravirt.c
index 6bc33a6db755..158d52414e96 100644
--- a/arch/ia64/kernel/paravirt.c
+++ b/arch/ia64/kernel/paravirt.c
@@ -46,13 +46,23 @@ struct pv_info pv_info = {
46 * initialization hooks. 46 * initialization hooks.
47 */ 47 */
48 48
49struct pv_init_ops pv_init_ops; 49static void __init
50ia64_native_patch_branch(unsigned long tag, unsigned long type);
51
52struct pv_init_ops pv_init_ops =
53{
54#ifdef ASM_SUPPORTED
55 .patch_bundle = ia64_native_patch_bundle,
56#endif
57 .patch_branch = ia64_native_patch_branch,
58};
50 59
51/*************************************************************************** 60/***************************************************************************
52 * pv_cpu_ops 61 * pv_cpu_ops
53 * intrinsics hooks. 62 * intrinsics hooks.
54 */ 63 */
55 64
65#ifndef ASM_SUPPORTED
56/* ia64_native_xxx are macros so that we have to make them real functions */ 66/* ia64_native_xxx are macros so that we have to make them real functions */
57 67
58#define DEFINE_VOID_FUNC1(name) \ 68#define DEFINE_VOID_FUNC1(name) \
@@ -274,6 +284,261 @@ ia64_native_setreg_func(int regnum, unsigned long val)
274 break; 284 break;
275 } 285 }
276} 286}
287#else
288
289#define __DEFINE_FUNC(name, code) \
290 extern const char ia64_native_ ## name ## _direct_start[]; \
291 extern const char ia64_native_ ## name ## _direct_end[]; \
292 asm (".align 32\n" \
293 ".proc ia64_native_" #name "_func\n" \
294 "ia64_native_" #name "_func:\n" \
295 "ia64_native_" #name "_direct_start:\n" \
296 code \
297 "ia64_native_" #name "_direct_end:\n" \
298 "br.cond.sptk.many b6\n" \
299 ".endp ia64_native_" #name "_func\n")
300
301#define DEFINE_VOID_FUNC0(name, code) \
302 extern void \
303 ia64_native_ ## name ## _func(void); \
304 __DEFINE_FUNC(name, code)
305
306#define DEFINE_VOID_FUNC1(name, code) \
307 extern void \
308 ia64_native_ ## name ## _func(unsigned long arg); \
309 __DEFINE_FUNC(name, code)
310
311#define DEFINE_VOID_FUNC2(name, code) \
312 extern void \
313 ia64_native_ ## name ## _func(unsigned long arg0, \
314 unsigned long arg1); \
315 __DEFINE_FUNC(name, code)
316
317#define DEFINE_FUNC0(name, code) \
318 extern unsigned long \
319 ia64_native_ ## name ## _func(void); \
320 __DEFINE_FUNC(name, code)
321
322#define DEFINE_FUNC1(name, type, code) \
323 extern unsigned long \
324 ia64_native_ ## name ## _func(type arg); \
325 __DEFINE_FUNC(name, code)
326
327DEFINE_VOID_FUNC1(fc,
328 "fc r8\n");
329DEFINE_VOID_FUNC1(intrin_local_irq_restore,
330 ";;\n"
331 " cmp.ne p6, p7 = r8, r0\n"
332 ";;\n"
333 "(p6) ssm psr.i\n"
334 "(p7) rsm psr.i\n"
335 ";;\n"
336 "(p6) srlz.d\n");
337
338DEFINE_VOID_FUNC2(ptcga,
339 "ptc.ga r8, r9\n");
340DEFINE_VOID_FUNC2(set_rr,
341 "mov rr[r8] = r9\n");
342
343/* ia64_native_getreg(_IA64_REG_PSR) & IA64_PSR_I */
344DEFINE_FUNC0(get_psr_i,
345 "mov r2 = " __stringify(1 << IA64_PSR_I_BIT) "\n"
346 "mov r8 = psr\n"
347 ";;\n"
348 "and r8 = r2, r8\n");
349
350DEFINE_FUNC1(thash, unsigned long,
351 "thash r8 = r8\n");
352DEFINE_FUNC1(get_cpuid, int,
353 "mov r8 = cpuid[r8]\n");
354DEFINE_FUNC1(get_pmd, int,
355 "mov r8 = pmd[r8]\n");
356DEFINE_FUNC1(get_rr, unsigned long,
357 "mov r8 = rr[r8]\n");
358
359DEFINE_VOID_FUNC0(ssm_i,
360 "ssm psr.i\n");
361DEFINE_VOID_FUNC0(rsm_i,
362 "rsm psr.i\n");
363
364extern void
365ia64_native_set_rr0_to_rr4_func(unsigned long val0, unsigned long val1,
366 unsigned long val2, unsigned long val3,
367 unsigned long val4);
368__DEFINE_FUNC(set_rr0_to_rr4,
369 "mov rr[r0] = r8\n"
370 "movl r2 = 0x2000000000000000\n"
371 ";;\n"
372 "mov rr[r2] = r9\n"
373 "shl r3 = r2, 1\n" /* movl r3 = 0x4000000000000000 */
374 ";;\n"
375 "add r2 = r2, r3\n" /* movl r2 = 0x6000000000000000 */
376 "mov rr[r3] = r10\n"
377 ";;\n"
378 "mov rr[r2] = r11\n"
379 "shl r3 = r3, 1\n" /* movl r3 = 0x8000000000000000 */
380 ";;\n"
381 "mov rr[r3] = r14\n");
382
383extern unsigned long ia64_native_getreg_func(int regnum);
384asm(".global ia64_native_getreg_func\n");
385#define __DEFINE_GET_REG(id, reg) \
386 "mov r2 = " __stringify(_IA64_REG_ ## id) "\n" \
387 ";;\n" \
388 "cmp.eq p6, p0 = r2, r8\n" \
389 ";;\n" \
390 "(p6) mov r8 = " #reg "\n" \
391 "(p6) br.cond.sptk.many b6\n" \
392 ";;\n"
393#define __DEFINE_GET_AR(id, reg) __DEFINE_GET_REG(AR_ ## id, ar.reg)
394#define __DEFINE_GET_CR(id, reg) __DEFINE_GET_REG(CR_ ## id, cr.reg)
395
396__DEFINE_FUNC(getreg,
397 __DEFINE_GET_REG(GP, gp)
398 /*__DEFINE_GET_REG(IP, ip)*/ /* returned ip value shouldn't be constant */
399 __DEFINE_GET_REG(PSR, psr)
400 __DEFINE_GET_REG(TP, tp)
401 __DEFINE_GET_REG(SP, sp)
402
403 __DEFINE_GET_REG(AR_KR0, ar0)
404 __DEFINE_GET_REG(AR_KR1, ar1)
405 __DEFINE_GET_REG(AR_KR2, ar2)
406 __DEFINE_GET_REG(AR_KR3, ar3)
407 __DEFINE_GET_REG(AR_KR4, ar4)
408 __DEFINE_GET_REG(AR_KR5, ar5)
409 __DEFINE_GET_REG(AR_KR6, ar6)
410 __DEFINE_GET_REG(AR_KR7, ar7)
411 __DEFINE_GET_AR(RSC, rsc)
412 __DEFINE_GET_AR(BSP, bsp)
413 __DEFINE_GET_AR(BSPSTORE, bspstore)
414 __DEFINE_GET_AR(RNAT, rnat)
415 __DEFINE_GET_AR(FCR, fcr)
416 __DEFINE_GET_AR(EFLAG, eflag)
417 __DEFINE_GET_AR(CSD, csd)
418 __DEFINE_GET_AR(SSD, ssd)
419 __DEFINE_GET_REG(AR_CFLAG, ar27)
420 __DEFINE_GET_AR(FSR, fsr)
421 __DEFINE_GET_AR(FIR, fir)
422 __DEFINE_GET_AR(FDR, fdr)
423 __DEFINE_GET_AR(CCV, ccv)
424 __DEFINE_GET_AR(UNAT, unat)
425 __DEFINE_GET_AR(FPSR, fpsr)
426 __DEFINE_GET_AR(ITC, itc)
427 __DEFINE_GET_AR(PFS, pfs)
428 __DEFINE_GET_AR(LC, lc)
429 __DEFINE_GET_AR(EC, ec)
430
431 __DEFINE_GET_CR(DCR, dcr)
432 __DEFINE_GET_CR(ITM, itm)
433 __DEFINE_GET_CR(IVA, iva)
434 __DEFINE_GET_CR(PTA, pta)
435 __DEFINE_GET_CR(IPSR, ipsr)
436 __DEFINE_GET_CR(ISR, isr)
437 __DEFINE_GET_CR(IIP, iip)
438 __DEFINE_GET_CR(IFA, ifa)
439 __DEFINE_GET_CR(ITIR, itir)
440 __DEFINE_GET_CR(IIPA, iipa)
441 __DEFINE_GET_CR(IFS, ifs)
442 __DEFINE_GET_CR(IIM, iim)
443 __DEFINE_GET_CR(IHA, iha)
444 __DEFINE_GET_CR(LID, lid)
445 __DEFINE_GET_CR(IVR, ivr)
446 __DEFINE_GET_CR(TPR, tpr)
447 __DEFINE_GET_CR(EOI, eoi)
448 __DEFINE_GET_CR(IRR0, irr0)
449 __DEFINE_GET_CR(IRR1, irr1)
450 __DEFINE_GET_CR(IRR2, irr2)
451 __DEFINE_GET_CR(IRR3, irr3)
452 __DEFINE_GET_CR(ITV, itv)
453 __DEFINE_GET_CR(PMV, pmv)
454 __DEFINE_GET_CR(CMCV, cmcv)
455 __DEFINE_GET_CR(LRR0, lrr0)
456 __DEFINE_GET_CR(LRR1, lrr1)
457
458 "mov r8 = -1\n" /* unsupported case */
459 );
460
461extern void ia64_native_setreg_func(int regnum, unsigned long val);
462asm(".global ia64_native_setreg_func\n");
463#define __DEFINE_SET_REG(id, reg) \
464 "mov r2 = " __stringify(_IA64_REG_ ## id) "\n" \
465 ";;\n" \
466 "cmp.eq p6, p0 = r2, r9\n" \
467 ";;\n" \
468 "(p6) mov " #reg " = r8\n" \
469 "(p6) br.cond.sptk.many b6\n" \
470 ";;\n"
471#define __DEFINE_SET_AR(id, reg) __DEFINE_SET_REG(AR_ ## id, ar.reg)
472#define __DEFINE_SET_CR(id, reg) __DEFINE_SET_REG(CR_ ## id, cr.reg)
473__DEFINE_FUNC(setreg,
474 "mov r2 = " __stringify(_IA64_REG_PSR_L) "\n"
475 ";;\n"
476 "cmp.eq p6, p0 = r2, r9\n"
477 ";;\n"
478 "(p6) mov psr.l = r8\n"
479#ifdef HAVE_SERIALIZE_DIRECTIVE
480 ".serialize.data\n"
481#endif
482 "(p6) br.cond.sptk.many b6\n"
483 __DEFINE_SET_REG(GP, gp)
484 __DEFINE_SET_REG(SP, sp)
485
486 __DEFINE_SET_REG(AR_KR0, ar0)
487 __DEFINE_SET_REG(AR_KR1, ar1)
488 __DEFINE_SET_REG(AR_KR2, ar2)
489 __DEFINE_SET_REG(AR_KR3, ar3)
490 __DEFINE_SET_REG(AR_KR4, ar4)
491 __DEFINE_SET_REG(AR_KR5, ar5)
492 __DEFINE_SET_REG(AR_KR6, ar6)
493 __DEFINE_SET_REG(AR_KR7, ar7)
494 __DEFINE_SET_AR(RSC, rsc)
495 __DEFINE_SET_AR(BSP, bsp)
496 __DEFINE_SET_AR(BSPSTORE, bspstore)
497 __DEFINE_SET_AR(RNAT, rnat)
498 __DEFINE_SET_AR(FCR, fcr)
499 __DEFINE_SET_AR(EFLAG, eflag)
500 __DEFINE_SET_AR(CSD, csd)
501 __DEFINE_SET_AR(SSD, ssd)
502 __DEFINE_SET_REG(AR_CFLAG, ar27)
503 __DEFINE_SET_AR(FSR, fsr)
504 __DEFINE_SET_AR(FIR, fir)
505 __DEFINE_SET_AR(FDR, fdr)
506 __DEFINE_SET_AR(CCV, ccv)
507 __DEFINE_SET_AR(UNAT, unat)
508 __DEFINE_SET_AR(FPSR, fpsr)
509 __DEFINE_SET_AR(ITC, itc)
510 __DEFINE_SET_AR(PFS, pfs)
511 __DEFINE_SET_AR(LC, lc)
512 __DEFINE_SET_AR(EC, ec)
513
514 __DEFINE_SET_CR(DCR, dcr)
515 __DEFINE_SET_CR(ITM, itm)
516 __DEFINE_SET_CR(IVA, iva)
517 __DEFINE_SET_CR(PTA, pta)
518 __DEFINE_SET_CR(IPSR, ipsr)
519 __DEFINE_SET_CR(ISR, isr)
520 __DEFINE_SET_CR(IIP, iip)
521 __DEFINE_SET_CR(IFA, ifa)
522 __DEFINE_SET_CR(ITIR, itir)
523 __DEFINE_SET_CR(IIPA, iipa)
524 __DEFINE_SET_CR(IFS, ifs)
525 __DEFINE_SET_CR(IIM, iim)
526 __DEFINE_SET_CR(IHA, iha)
527 __DEFINE_SET_CR(LID, lid)
528 __DEFINE_SET_CR(IVR, ivr)
529 __DEFINE_SET_CR(TPR, tpr)
530 __DEFINE_SET_CR(EOI, eoi)
531 __DEFINE_SET_CR(IRR0, irr0)
532 __DEFINE_SET_CR(IRR1, irr1)
533 __DEFINE_SET_CR(IRR2, irr2)
534 __DEFINE_SET_CR(IRR3, irr3)
535 __DEFINE_SET_CR(ITV, itv)
536 __DEFINE_SET_CR(PMV, pmv)
537 __DEFINE_SET_CR(CMCV, cmcv)
538 __DEFINE_SET_CR(LRR0, lrr0)
539 __DEFINE_SET_CR(LRR1, lrr1)
540 );
541#endif
277 542
278struct pv_cpu_ops pv_cpu_ops = { 543struct pv_cpu_ops pv_cpu_ops = {
279 .fc = ia64_native_fc_func, 544 .fc = ia64_native_fc_func,
@@ -368,3 +633,256 @@ struct pv_time_ops pv_time_ops = {
368 .do_steal_accounting = ia64_native_do_steal_accounting, 633 .do_steal_accounting = ia64_native_do_steal_accounting,
369 .sched_clock = ia64_native_sched_clock, 634 .sched_clock = ia64_native_sched_clock,
370}; 635};
636
637/***************************************************************************
638 * binary pacthing
639 * pv_init_ops.patch_bundle
640 */
641
642#ifdef ASM_SUPPORTED
643#define IA64_NATIVE_PATCH_DEFINE_GET_REG(name, reg) \
644 __DEFINE_FUNC(get_ ## name, \
645 ";;\n" \
646 "mov r8 = " #reg "\n" \
647 ";;\n")
648
649#define IA64_NATIVE_PATCH_DEFINE_SET_REG(name, reg) \
650 __DEFINE_FUNC(set_ ## name, \
651 ";;\n" \
652 "mov " #reg " = r8\n" \
653 ";;\n")
654
655#define IA64_NATIVE_PATCH_DEFINE_REG(name, reg) \
656 IA64_NATIVE_PATCH_DEFINE_GET_REG(name, reg); \
657 IA64_NATIVE_PATCH_DEFINE_SET_REG(name, reg) \
658
659#define IA64_NATIVE_PATCH_DEFINE_AR(name, reg) \
660 IA64_NATIVE_PATCH_DEFINE_REG(ar_ ## name, ar.reg)
661
662#define IA64_NATIVE_PATCH_DEFINE_CR(name, reg) \
663 IA64_NATIVE_PATCH_DEFINE_REG(cr_ ## name, cr.reg)
664
665
666IA64_NATIVE_PATCH_DEFINE_GET_REG(psr, psr);
667IA64_NATIVE_PATCH_DEFINE_GET_REG(tp, tp);
668
669/* IA64_NATIVE_PATCH_DEFINE_SET_REG(psr_l, psr.l); */
670__DEFINE_FUNC(set_psr_l,
671 ";;\n"
672 "mov psr.l = r8\n"
673#ifdef HAVE_SERIALIZE_DIRECTIVE
674 ".serialize.data\n"
675#endif
676 ";;\n");
677
678IA64_NATIVE_PATCH_DEFINE_REG(gp, gp);
679IA64_NATIVE_PATCH_DEFINE_REG(sp, sp);
680
681IA64_NATIVE_PATCH_DEFINE_REG(kr0, ar0);
682IA64_NATIVE_PATCH_DEFINE_REG(kr1, ar1);
683IA64_NATIVE_PATCH_DEFINE_REG(kr2, ar2);
684IA64_NATIVE_PATCH_DEFINE_REG(kr3, ar3);
685IA64_NATIVE_PATCH_DEFINE_REG(kr4, ar4);
686IA64_NATIVE_PATCH_DEFINE_REG(kr5, ar5);
687IA64_NATIVE_PATCH_DEFINE_REG(kr6, ar6);
688IA64_NATIVE_PATCH_DEFINE_REG(kr7, ar7);
689
690IA64_NATIVE_PATCH_DEFINE_AR(rsc, rsc);
691IA64_NATIVE_PATCH_DEFINE_AR(bsp, bsp);
692IA64_NATIVE_PATCH_DEFINE_AR(bspstore, bspstore);
693IA64_NATIVE_PATCH_DEFINE_AR(rnat, rnat);
694IA64_NATIVE_PATCH_DEFINE_AR(fcr, fcr);
695IA64_NATIVE_PATCH_DEFINE_AR(eflag, eflag);
696IA64_NATIVE_PATCH_DEFINE_AR(csd, csd);
697IA64_NATIVE_PATCH_DEFINE_AR(ssd, ssd);
698IA64_NATIVE_PATCH_DEFINE_REG(ar27, ar27);
699IA64_NATIVE_PATCH_DEFINE_AR(fsr, fsr);
700IA64_NATIVE_PATCH_DEFINE_AR(fir, fir);
701IA64_NATIVE_PATCH_DEFINE_AR(fdr, fdr);
702IA64_NATIVE_PATCH_DEFINE_AR(ccv, ccv);
703IA64_NATIVE_PATCH_DEFINE_AR(unat, unat);
704IA64_NATIVE_PATCH_DEFINE_AR(fpsr, fpsr);
705IA64_NATIVE_PATCH_DEFINE_AR(itc, itc);
706IA64_NATIVE_PATCH_DEFINE_AR(pfs, pfs);
707IA64_NATIVE_PATCH_DEFINE_AR(lc, lc);
708IA64_NATIVE_PATCH_DEFINE_AR(ec, ec);
709
710IA64_NATIVE_PATCH_DEFINE_CR(dcr, dcr);
711IA64_NATIVE_PATCH_DEFINE_CR(itm, itm);
712IA64_NATIVE_PATCH_DEFINE_CR(iva, iva);
713IA64_NATIVE_PATCH_DEFINE_CR(pta, pta);
714IA64_NATIVE_PATCH_DEFINE_CR(ipsr, ipsr);
715IA64_NATIVE_PATCH_DEFINE_CR(isr, isr);
716IA64_NATIVE_PATCH_DEFINE_CR(iip, iip);
717IA64_NATIVE_PATCH_DEFINE_CR(ifa, ifa);
718IA64_NATIVE_PATCH_DEFINE_CR(itir, itir);
719IA64_NATIVE_PATCH_DEFINE_CR(iipa, iipa);
720IA64_NATIVE_PATCH_DEFINE_CR(ifs, ifs);
721IA64_NATIVE_PATCH_DEFINE_CR(iim, iim);
722IA64_NATIVE_PATCH_DEFINE_CR(iha, iha);
723IA64_NATIVE_PATCH_DEFINE_CR(lid, lid);
724IA64_NATIVE_PATCH_DEFINE_CR(ivr, ivr);
725IA64_NATIVE_PATCH_DEFINE_CR(tpr, tpr);
726IA64_NATIVE_PATCH_DEFINE_CR(eoi, eoi);
727IA64_NATIVE_PATCH_DEFINE_CR(irr0, irr0);
728IA64_NATIVE_PATCH_DEFINE_CR(irr1, irr1);
729IA64_NATIVE_PATCH_DEFINE_CR(irr2, irr2);
730IA64_NATIVE_PATCH_DEFINE_CR(irr3, irr3);
731IA64_NATIVE_PATCH_DEFINE_CR(itv, itv);
732IA64_NATIVE_PATCH_DEFINE_CR(pmv, pmv);
733IA64_NATIVE_PATCH_DEFINE_CR(cmcv, cmcv);
734IA64_NATIVE_PATCH_DEFINE_CR(lrr0, lrr0);
735IA64_NATIVE_PATCH_DEFINE_CR(lrr1, lrr1);
736
737static const struct paravirt_patch_bundle_elem ia64_native_patch_bundle_elems[]
738__initdata_or_module =
739{
740#define IA64_NATIVE_PATCH_BUNDLE_ELEM(name, type) \
741 { \
742 (void*)ia64_native_ ## name ## _direct_start, \
743 (void*)ia64_native_ ## name ## _direct_end, \
744 PARAVIRT_PATCH_TYPE_ ## type, \
745 }
746
747 IA64_NATIVE_PATCH_BUNDLE_ELEM(fc, FC),
748 IA64_NATIVE_PATCH_BUNDLE_ELEM(thash, THASH),
749 IA64_NATIVE_PATCH_BUNDLE_ELEM(get_cpuid, GET_CPUID),
750 IA64_NATIVE_PATCH_BUNDLE_ELEM(get_pmd, GET_PMD),
751 IA64_NATIVE_PATCH_BUNDLE_ELEM(ptcga, PTCGA),
752 IA64_NATIVE_PATCH_BUNDLE_ELEM(get_rr, GET_RR),
753 IA64_NATIVE_PATCH_BUNDLE_ELEM(set_rr, SET_RR),
754 IA64_NATIVE_PATCH_BUNDLE_ELEM(set_rr0_to_rr4, SET_RR0_TO_RR4),
755 IA64_NATIVE_PATCH_BUNDLE_ELEM(ssm_i, SSM_I),
756 IA64_NATIVE_PATCH_BUNDLE_ELEM(rsm_i, RSM_I),
757 IA64_NATIVE_PATCH_BUNDLE_ELEM(get_psr_i, GET_PSR_I),
758 IA64_NATIVE_PATCH_BUNDLE_ELEM(intrin_local_irq_restore,
759 INTRIN_LOCAL_IRQ_RESTORE),
760
761#define IA64_NATIVE_PATCH_BUNDLE_ELEM_GETREG(name, reg) \
762 { \
763 (void*)ia64_native_get_ ## name ## _direct_start, \
764 (void*)ia64_native_get_ ## name ## _direct_end, \
765 PARAVIRT_PATCH_TYPE_GETREG + _IA64_REG_ ## reg, \
766 }
767
768#define IA64_NATIVE_PATCH_BUNDLE_ELEM_SETREG(name, reg) \
769 { \
770 (void*)ia64_native_set_ ## name ## _direct_start, \
771 (void*)ia64_native_set_ ## name ## _direct_end, \
772 PARAVIRT_PATCH_TYPE_SETREG + _IA64_REG_ ## reg, \
773 }
774
775#define IA64_NATIVE_PATCH_BUNDLE_ELEM_REG(name, reg) \
776 IA64_NATIVE_PATCH_BUNDLE_ELEM_GETREG(name, reg), \
777 IA64_NATIVE_PATCH_BUNDLE_ELEM_SETREG(name, reg) \
778
779#define IA64_NATIVE_PATCH_BUNDLE_ELEM_AR(name, reg) \
780 IA64_NATIVE_PATCH_BUNDLE_ELEM_REG(ar_ ## name, AR_ ## reg)
781
782#define IA64_NATIVE_PATCH_BUNDLE_ELEM_CR(name, reg) \
783 IA64_NATIVE_PATCH_BUNDLE_ELEM_REG(cr_ ## name, CR_ ## reg)
784
785 IA64_NATIVE_PATCH_BUNDLE_ELEM_GETREG(psr, PSR),
786 IA64_NATIVE_PATCH_BUNDLE_ELEM_GETREG(tp, TP),
787
788 IA64_NATIVE_PATCH_BUNDLE_ELEM_SETREG(psr_l, PSR_L),
789
790 IA64_NATIVE_PATCH_BUNDLE_ELEM_REG(gp, GP),
791 IA64_NATIVE_PATCH_BUNDLE_ELEM_REG(sp, SP),
792
793 IA64_NATIVE_PATCH_BUNDLE_ELEM_REG(kr0, AR_KR0),
794 IA64_NATIVE_PATCH_BUNDLE_ELEM_REG(kr1, AR_KR1),
795 IA64_NATIVE_PATCH_BUNDLE_ELEM_REG(kr2, AR_KR2),
796 IA64_NATIVE_PATCH_BUNDLE_ELEM_REG(kr3, AR_KR3),
797 IA64_NATIVE_PATCH_BUNDLE_ELEM_REG(kr4, AR_KR4),
798 IA64_NATIVE_PATCH_BUNDLE_ELEM_REG(kr5, AR_KR5),
799 IA64_NATIVE_PATCH_BUNDLE_ELEM_REG(kr6, AR_KR6),
800 IA64_NATIVE_PATCH_BUNDLE_ELEM_REG(kr7, AR_KR7),
801
802 IA64_NATIVE_PATCH_BUNDLE_ELEM_AR(rsc, RSC),
803 IA64_NATIVE_PATCH_BUNDLE_ELEM_AR(bsp, BSP),
804 IA64_NATIVE_PATCH_BUNDLE_ELEM_AR(bspstore, BSPSTORE),
805 IA64_NATIVE_PATCH_BUNDLE_ELEM_AR(rnat, RNAT),
806 IA64_NATIVE_PATCH_BUNDLE_ELEM_AR(fcr, FCR),
807 IA64_NATIVE_PATCH_BUNDLE_ELEM_AR(eflag, EFLAG),
808 IA64_NATIVE_PATCH_BUNDLE_ELEM_AR(csd, CSD),
809 IA64_NATIVE_PATCH_BUNDLE_ELEM_AR(ssd, SSD),
810 IA64_NATIVE_PATCH_BUNDLE_ELEM_REG(ar27, AR_CFLAG),
811 IA64_NATIVE_PATCH_BUNDLE_ELEM_AR(fsr, FSR),
812 IA64_NATIVE_PATCH_BUNDLE_ELEM_AR(fir, FIR),
813 IA64_NATIVE_PATCH_BUNDLE_ELEM_AR(fdr, FDR),
814 IA64_NATIVE_PATCH_BUNDLE_ELEM_AR(ccv, CCV),
815 IA64_NATIVE_PATCH_BUNDLE_ELEM_AR(unat, UNAT),
816 IA64_NATIVE_PATCH_BUNDLE_ELEM_AR(fpsr, FPSR),
817 IA64_NATIVE_PATCH_BUNDLE_ELEM_AR(itc, ITC),
818 IA64_NATIVE_PATCH_BUNDLE_ELEM_AR(pfs, PFS),
819 IA64_NATIVE_PATCH_BUNDLE_ELEM_AR(lc, LC),
820 IA64_NATIVE_PATCH_BUNDLE_ELEM_AR(ec, EC),
821
822 IA64_NATIVE_PATCH_BUNDLE_ELEM_CR(dcr, DCR),
823 IA64_NATIVE_PATCH_BUNDLE_ELEM_CR(itm, ITM),
824 IA64_NATIVE_PATCH_BUNDLE_ELEM_CR(iva, IVA),
825 IA64_NATIVE_PATCH_BUNDLE_ELEM_CR(pta, PTA),
826 IA64_NATIVE_PATCH_BUNDLE_ELEM_CR(ipsr, IPSR),
827 IA64_NATIVE_PATCH_BUNDLE_ELEM_CR(isr, ISR),
828 IA64_NATIVE_PATCH_BUNDLE_ELEM_CR(iip, IIP),
829 IA64_NATIVE_PATCH_BUNDLE_ELEM_CR(ifa, IFA),
830 IA64_NATIVE_PATCH_BUNDLE_ELEM_CR(itir, ITIR),
831 IA64_NATIVE_PATCH_BUNDLE_ELEM_CR(iipa, IIPA),
832 IA64_NATIVE_PATCH_BUNDLE_ELEM_CR(ifs, IFS),
833 IA64_NATIVE_PATCH_BUNDLE_ELEM_CR(iim, IIM),
834 IA64_NATIVE_PATCH_BUNDLE_ELEM_CR(iha, IHA),
835 IA64_NATIVE_PATCH_BUNDLE_ELEM_CR(lid, LID),
836 IA64_NATIVE_PATCH_BUNDLE_ELEM_CR(ivr, IVR),
837 IA64_NATIVE_PATCH_BUNDLE_ELEM_CR(tpr, TPR),
838 IA64_NATIVE_PATCH_BUNDLE_ELEM_CR(eoi, EOI),
839 IA64_NATIVE_PATCH_BUNDLE_ELEM_CR(irr0, IRR0),
840 IA64_NATIVE_PATCH_BUNDLE_ELEM_CR(irr1, IRR1),
841 IA64_NATIVE_PATCH_BUNDLE_ELEM_CR(irr2, IRR2),
842 IA64_NATIVE_PATCH_BUNDLE_ELEM_CR(irr3, IRR3),
843 IA64_NATIVE_PATCH_BUNDLE_ELEM_CR(itv, ITV),
844 IA64_NATIVE_PATCH_BUNDLE_ELEM_CR(pmv, PMV),
845 IA64_NATIVE_PATCH_BUNDLE_ELEM_CR(cmcv, CMCV),
846 IA64_NATIVE_PATCH_BUNDLE_ELEM_CR(lrr0, LRR0),
847 IA64_NATIVE_PATCH_BUNDLE_ELEM_CR(lrr1, LRR1),
848};
849
850unsigned long __init_or_module
851ia64_native_patch_bundle(void *sbundle, void *ebundle, unsigned long type)
852{
853 const unsigned long nelems = sizeof(ia64_native_patch_bundle_elems) /
854 sizeof(ia64_native_patch_bundle_elems[0]);
855
856 return __paravirt_patch_apply_bundle(sbundle, ebundle, type,
857 ia64_native_patch_bundle_elems,
858 nelems, NULL);
859}
860#endif /* ASM_SUPPOTED */
861
862extern const char ia64_native_switch_to[];
863extern const char ia64_native_leave_syscall[];
864extern const char ia64_native_work_processed_syscall[];
865extern const char ia64_native_leave_kernel[];
866
867const struct paravirt_patch_branch_target ia64_native_branch_target[]
868__initconst = {
869#define PARAVIRT_BR_TARGET(name, type) \
870 { \
871 ia64_native_ ## name, \
872 PARAVIRT_PATCH_TYPE_BR_ ## type, \
873 }
874 PARAVIRT_BR_TARGET(switch_to, SWITCH_TO),
875 PARAVIRT_BR_TARGET(leave_syscall, LEAVE_SYSCALL),
876 PARAVIRT_BR_TARGET(work_processed_syscall, WORK_PROCESSED_SYSCALL),
877 PARAVIRT_BR_TARGET(leave_kernel, LEAVE_KERNEL),
878};
879
880static void __init
881ia64_native_patch_branch(unsigned long tag, unsigned long type)
882{
883 const unsigned long nelem =
884 sizeof(ia64_native_branch_target) /
885 sizeof(ia64_native_branch_target[0]);
886 __paravirt_patch_apply_branch(tag, type,
887 ia64_native_branch_target, nelem);
888}
diff --git a/arch/ia64/kernel/paravirtentry.S b/arch/ia64/kernel/paravirtentry.S
index 80c0d365cbc0..6158560d7f17 100644
--- a/arch/ia64/kernel/paravirtentry.S
+++ b/arch/ia64/kernel/paravirtentry.S
@@ -20,8 +20,11 @@
20 * 20 *
21 */ 21 */
22 22
23#include <linux/init.h>
23#include <asm/asmmacro.h> 24#include <asm/asmmacro.h>
24#include <asm/asm-offsets.h> 25#include <asm/asm-offsets.h>
26#include <asm/paravirt_privop.h>
27#include <asm/paravirt_patch.h>
25#include "entry.h" 28#include "entry.h"
26 29
27#define DATA8(sym, init_value) \ 30#define DATA8(sym, init_value) \
@@ -32,32 +35,34 @@
32 data8 init_value ; \ 35 data8 init_value ; \
33 .popsection 36 .popsection
34 37
35#define BRANCH(targ, reg, breg) \ 38#define BRANCH(targ, reg, breg, type) \
36 movl reg=targ ; \ 39 PARAVIRT_PATCH_SITE_BR(PARAVIRT_PATCH_TYPE_BR_ ## type) ; \
37 ;; \ 40 ;; \
38 ld8 reg=[reg] ; \ 41 movl reg=targ ; \
39 ;; \ 42 ;; \
40 mov breg=reg ; \ 43 ld8 reg=[reg] ; \
44 ;; \
45 mov breg=reg ; \
41 br.cond.sptk.many breg 46 br.cond.sptk.many breg
42 47
43#define BRANCH_PROC(sym, reg, breg) \ 48#define BRANCH_PROC(sym, reg, breg, type) \
44 DATA8(paravirt_ ## sym ## _targ, ia64_native_ ## sym) ; \ 49 DATA8(paravirt_ ## sym ## _targ, ia64_native_ ## sym) ; \
45 GLOBAL_ENTRY(paravirt_ ## sym) ; \ 50 GLOBAL_ENTRY(paravirt_ ## sym) ; \
46 BRANCH(paravirt_ ## sym ## _targ, reg, breg) ; \ 51 BRANCH(paravirt_ ## sym ## _targ, reg, breg, type) ; \
47 END(paravirt_ ## sym) 52 END(paravirt_ ## sym)
48 53
49#define BRANCH_PROC_UNWINFO(sym, reg, breg) \ 54#define BRANCH_PROC_UNWINFO(sym, reg, breg, type) \
50 DATA8(paravirt_ ## sym ## _targ, ia64_native_ ## sym) ; \ 55 DATA8(paravirt_ ## sym ## _targ, ia64_native_ ## sym) ; \
51 GLOBAL_ENTRY(paravirt_ ## sym) ; \ 56 GLOBAL_ENTRY(paravirt_ ## sym) ; \
52 PT_REGS_UNWIND_INFO(0) ; \ 57 PT_REGS_UNWIND_INFO(0) ; \
53 BRANCH(paravirt_ ## sym ## _targ, reg, breg) ; \ 58 BRANCH(paravirt_ ## sym ## _targ, reg, breg, type) ; \
54 END(paravirt_ ## sym) 59 END(paravirt_ ## sym)
55 60
56 61
57BRANCH_PROC(switch_to, r22, b7) 62BRANCH_PROC(switch_to, r22, b7, SWITCH_TO)
58BRANCH_PROC_UNWINFO(leave_syscall, r22, b7) 63BRANCH_PROC_UNWINFO(leave_syscall, r22, b7, LEAVE_SYSCALL)
59BRANCH_PROC(work_processed_syscall, r2, b7) 64BRANCH_PROC(work_processed_syscall, r2, b7, WORK_PROCESSED_SYSCALL)
60BRANCH_PROC_UNWINFO(leave_kernel, r22, b7) 65BRANCH_PROC_UNWINFO(leave_kernel, r22, b7, LEAVE_KERNEL)
61 66
62 67
63#ifdef CONFIG_MODULES 68#ifdef CONFIG_MODULES
diff --git a/arch/ia64/kernel/setup.c b/arch/ia64/kernel/setup.c
index 865af27c7737..4ed3e1c117e7 100644
--- a/arch/ia64/kernel/setup.c
+++ b/arch/ia64/kernel/setup.c
@@ -52,6 +52,7 @@
52#include <asm/meminit.h> 52#include <asm/meminit.h>
53#include <asm/page.h> 53#include <asm/page.h>
54#include <asm/paravirt.h> 54#include <asm/paravirt.h>
55#include <asm/paravirt_patch.h>
55#include <asm/patch.h> 56#include <asm/patch.h>
56#include <asm/pgtable.h> 57#include <asm/pgtable.h>
57#include <asm/processor.h> 58#include <asm/processor.h>
@@ -537,6 +538,7 @@ setup_arch (char **cmdline_p)
537 paravirt_arch_setup_early(); 538 paravirt_arch_setup_early();
538 539
539 ia64_patch_vtop((u64) __start___vtop_patchlist, (u64) __end___vtop_patchlist); 540 ia64_patch_vtop((u64) __start___vtop_patchlist, (u64) __end___vtop_patchlist);
541 paravirt_patch_apply();
540 542
541 *cmdline_p = __va(ia64_boot_param->command_line); 543 *cmdline_p = __va(ia64_boot_param->command_line);
542 strlcpy(boot_command_line, *cmdline_p, COMMAND_LINE_SIZE); 544 strlcpy(boot_command_line, *cmdline_p, COMMAND_LINE_SIZE);
generated by cgit v1.2.2 (git 2.25.0) at 2024-12-20 23:48:17 -0500