aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorJeremy Fitzhardinge <jeremy@goop.org>2009-01-28 17:35:07 -0500
committerH. Peter Anvin <hpa@linux.intel.com>2009-01-30 17:51:45 -0500
commitda5de7c22eb705be709a57e486e7475a6969b994 (patch)
tree29b3655e38fea6bd6ef11437d2fea14b397c8b03
parent791bad9d28d405d9397ea0c370ffb7c7bdd2aa6e (diff)
x86/paravirt: use callee-saved convention for pte_val/make_pte/etc
Impact: Optimization In the native case, pte_val, make_pte, etc are all just identity functions, so there's no need to clobber a lot of registers over them. (This changes the 32-bit callee-save calling convention to return both EAX and EDX so functions can return 64-bit values.) Signed-off-by: Jeremy Fitzhardinge <jeremy.fitzhardinge@citrix.com> Signed-off-by: H. Peter Anvin <hpa@zytor.com>
-rw-r--r--arch/x86/include/asm/paravirt.h78
-rw-r--r--arch/x86/kernel/paravirt.c53
-rw-r--r--arch/x86/xen/mmu.c24
3 files changed, 67 insertions, 88 deletions
diff --git a/arch/x86/include/asm/paravirt.h b/arch/x86/include/asm/paravirt.h
index beb10ecdbe67..2d098b78bc10 100644
--- a/arch/x86/include/asm/paravirt.h
+++ b/arch/x86/include/asm/paravirt.h
@@ -19,7 +19,7 @@
19#define CLBR_ANY ((1 << 4) - 1) 19#define CLBR_ANY ((1 << 4) - 1)
20 20
21#define CLBR_ARG_REGS (CLBR_EAX | CLBR_EDX | CLBR_ECX) 21#define CLBR_ARG_REGS (CLBR_EAX | CLBR_EDX | CLBR_ECX)
22#define CLBR_RET_REG (CLBR_EAX) 22#define CLBR_RET_REG (CLBR_EAX | CLBR_EDX)
23#define CLBR_SCRATCH (0) 23#define CLBR_SCRATCH (0)
24#else 24#else
25#define CLBR_RAX CLBR_EAX 25#define CLBR_RAX CLBR_EAX
@@ -308,11 +308,11 @@ struct pv_mmu_ops {
308 void (*ptep_modify_prot_commit)(struct mm_struct *mm, unsigned long addr, 308 void (*ptep_modify_prot_commit)(struct mm_struct *mm, unsigned long addr,
309 pte_t *ptep, pte_t pte); 309 pte_t *ptep, pte_t pte);
310 310
311 pteval_t (*pte_val)(pte_t); 311 struct paravirt_callee_save pte_val;
312 pte_t (*make_pte)(pteval_t pte); 312 struct paravirt_callee_save make_pte;
313 313
314 pgdval_t (*pgd_val)(pgd_t); 314 struct paravirt_callee_save pgd_val;
315 pgd_t (*make_pgd)(pgdval_t pgd); 315 struct paravirt_callee_save make_pgd;
316 316
317#if PAGETABLE_LEVELS >= 3 317#if PAGETABLE_LEVELS >= 3
318#ifdef CONFIG_X86_PAE 318#ifdef CONFIG_X86_PAE
@@ -327,12 +327,12 @@ struct pv_mmu_ops {
327 327
328 void (*set_pud)(pud_t *pudp, pud_t pudval); 328 void (*set_pud)(pud_t *pudp, pud_t pudval);
329 329
330 pmdval_t (*pmd_val)(pmd_t); 330 struct paravirt_callee_save pmd_val;
331 pmd_t (*make_pmd)(pmdval_t pmd); 331 struct paravirt_callee_save make_pmd;
332 332
333#if PAGETABLE_LEVELS == 4 333#if PAGETABLE_LEVELS == 4
334 pudval_t (*pud_val)(pud_t); 334 struct paravirt_callee_save pud_val;
335 pud_t (*make_pud)(pudval_t pud); 335 struct paravirt_callee_save make_pud;
336 336
337 void (*set_pgd)(pgd_t *pudp, pgd_t pgdval); 337 void (*set_pgd)(pgd_t *pudp, pgd_t pgdval);
338#endif /* PAGETABLE_LEVELS == 4 */ 338#endif /* PAGETABLE_LEVELS == 4 */
@@ -1155,13 +1155,13 @@ static inline pte_t __pte(pteval_t val)
1155 pteval_t ret; 1155 pteval_t ret;
1156 1156
1157 if (sizeof(pteval_t) > sizeof(long)) 1157 if (sizeof(pteval_t) > sizeof(long))
1158 ret = PVOP_CALL2(pteval_t, 1158 ret = PVOP_CALLEE2(pteval_t,
1159 pv_mmu_ops.make_pte, 1159 pv_mmu_ops.make_pte,
1160 val, (u64)val >> 32); 1160 val, (u64)val >> 32);
1161 else 1161 else
1162 ret = PVOP_CALL1(pteval_t, 1162 ret = PVOP_CALLEE1(pteval_t,
1163 pv_mmu_ops.make_pte, 1163 pv_mmu_ops.make_pte,
1164 val); 1164 val);
1165 1165
1166 return (pte_t) { .pte = ret }; 1166 return (pte_t) { .pte = ret };
1167} 1167}
@@ -1171,11 +1171,11 @@ static inline pteval_t pte_val(pte_t pte)
1171 pteval_t ret; 1171 pteval_t ret;
1172 1172
1173 if (sizeof(pteval_t) > sizeof(long)) 1173 if (sizeof(pteval_t) > sizeof(long))
1174 ret = PVOP_CALL2(pteval_t, pv_mmu_ops.pte_val, 1174 ret = PVOP_CALLEE2(pteval_t, pv_mmu_ops.pte_val,
1175 pte.pte, (u64)pte.pte >> 32); 1175 pte.pte, (u64)pte.pte >> 32);
1176 else 1176 else
1177 ret = PVOP_CALL1(pteval_t, pv_mmu_ops.pte_val, 1177 ret = PVOP_CALLEE1(pteval_t, pv_mmu_ops.pte_val,
1178 pte.pte); 1178 pte.pte);
1179 1179
1180 return ret; 1180 return ret;
1181} 1181}
@@ -1185,11 +1185,11 @@ static inline pgd_t __pgd(pgdval_t val)
1185 pgdval_t ret; 1185 pgdval_t ret;
1186 1186
1187 if (sizeof(pgdval_t) > sizeof(long)) 1187 if (sizeof(pgdval_t) > sizeof(long))
1188 ret = PVOP_CALL2(pgdval_t, pv_mmu_ops.make_pgd, 1188 ret = PVOP_CALLEE2(pgdval_t, pv_mmu_ops.make_pgd,
1189 val, (u64)val >> 32); 1189 val, (u64)val >> 32);
1190 else 1190 else
1191 ret = PVOP_CALL1(pgdval_t, pv_mmu_ops.make_pgd, 1191 ret = PVOP_CALLEE1(pgdval_t, pv_mmu_ops.make_pgd,
1192 val); 1192 val);
1193 1193
1194 return (pgd_t) { ret }; 1194 return (pgd_t) { ret };
1195} 1195}
@@ -1199,11 +1199,11 @@ static inline pgdval_t pgd_val(pgd_t pgd)
1199 pgdval_t ret; 1199 pgdval_t ret;
1200 1200
1201 if (sizeof(pgdval_t) > sizeof(long)) 1201 if (sizeof(pgdval_t) > sizeof(long))
1202 ret = PVOP_CALL2(pgdval_t, pv_mmu_ops.pgd_val, 1202 ret = PVOP_CALLEE2(pgdval_t, pv_mmu_ops.pgd_val,
1203 pgd.pgd, (u64)pgd.pgd >> 32); 1203 pgd.pgd, (u64)pgd.pgd >> 32);
1204 else 1204 else
1205 ret = PVOP_CALL1(pgdval_t, pv_mmu_ops.pgd_val, 1205 ret = PVOP_CALLEE1(pgdval_t, pv_mmu_ops.pgd_val,
1206 pgd.pgd); 1206 pgd.pgd);
1207 1207
1208 return ret; 1208 return ret;
1209} 1209}
@@ -1267,11 +1267,11 @@ static inline pmd_t __pmd(pmdval_t val)
1267 pmdval_t ret; 1267 pmdval_t ret;
1268 1268
1269 if (sizeof(pmdval_t) > sizeof(long)) 1269 if (sizeof(pmdval_t) > sizeof(long))
1270 ret = PVOP_CALL2(pmdval_t, pv_mmu_ops.make_pmd, 1270 ret = PVOP_CALLEE2(pmdval_t, pv_mmu_ops.make_pmd,
1271 val, (u64)val >> 32); 1271 val, (u64)val >> 32);
1272 else 1272 else
1273 ret = PVOP_CALL1(pmdval_t, pv_mmu_ops.make_pmd, 1273 ret = PVOP_CALLEE1(pmdval_t, pv_mmu_ops.make_pmd,
1274 val); 1274 val);
1275 1275
1276 return (pmd_t) { ret }; 1276 return (pmd_t) { ret };
1277} 1277}
@@ -1281,11 +1281,11 @@ static inline pmdval_t pmd_val(pmd_t pmd)
1281 pmdval_t ret; 1281 pmdval_t ret;
1282 1282
1283 if (sizeof(pmdval_t) > sizeof(long)) 1283 if (sizeof(pmdval_t) > sizeof(long))
1284 ret = PVOP_CALL2(pmdval_t, pv_mmu_ops.pmd_val, 1284 ret = PVOP_CALLEE2(pmdval_t, pv_mmu_ops.pmd_val,
1285 pmd.pmd, (u64)pmd.pmd >> 32); 1285 pmd.pmd, (u64)pmd.pmd >> 32);
1286 else 1286 else
1287 ret = PVOP_CALL1(pmdval_t, pv_mmu_ops.pmd_val, 1287 ret = PVOP_CALLEE1(pmdval_t, pv_mmu_ops.pmd_val,
1288 pmd.pmd); 1288 pmd.pmd);
1289 1289
1290 return ret; 1290 return ret;
1291} 1291}
@@ -1307,11 +1307,11 @@ static inline pud_t __pud(pudval_t val)
1307 pudval_t ret; 1307 pudval_t ret;
1308 1308
1309 if (sizeof(pudval_t) > sizeof(long)) 1309 if (sizeof(pudval_t) > sizeof(long))
1310 ret = PVOP_CALL2(pudval_t, pv_mmu_ops.make_pud, 1310 ret = PVOP_CALLEE2(pudval_t, pv_mmu_ops.make_pud,
1311 val, (u64)val >> 32); 1311 val, (u64)val >> 32);
1312 else 1312 else
1313 ret = PVOP_CALL1(pudval_t, pv_mmu_ops.make_pud, 1313 ret = PVOP_CALLEE1(pudval_t, pv_mmu_ops.make_pud,
1314 val); 1314 val);
1315 1315
1316 return (pud_t) { ret }; 1316 return (pud_t) { ret };
1317} 1317}
diff --git a/arch/x86/kernel/paravirt.c b/arch/x86/kernel/paravirt.c
index 8adb6b5aa421..cea11c8e3049 100644
--- a/arch/x86/kernel/paravirt.c
+++ b/arch/x86/kernel/paravirt.c
@@ -391,43 +391,12 @@ struct pv_apic_ops pv_apic_ops = {
391#endif 391#endif
392}; 392};
393 393
394typedef pte_t make_pte_t(pteval_t);
395typedef pmd_t make_pmd_t(pmdval_t);
396typedef pud_t make_pud_t(pudval_t);
397typedef pgd_t make_pgd_t(pgdval_t);
398
399typedef pteval_t pte_val_t(pte_t);
400typedef pmdval_t pmd_val_t(pmd_t);
401typedef pudval_t pud_val_t(pud_t);
402typedef pgdval_t pgd_val_t(pgd_t);
403
404
405#if defined(CONFIG_X86_32) && !defined(CONFIG_X86_PAE) 394#if defined(CONFIG_X86_32) && !defined(CONFIG_X86_PAE)
406/* 32-bit pagetable entries */ 395/* 32-bit pagetable entries */
407#define paravirt_native_make_pte (make_pte_t *)_paravirt_ident_32 396#define PTE_IDENT __PV_IS_CALLEE_SAVE(_paravirt_ident_32)
408#define paravirt_native_pte_val (pte_val_t *)_paravirt_ident_32
409
410#define paravirt_native_make_pmd (make_pmd_t *)_paravirt_ident_32
411#define paravirt_native_pmd_val (pmd_val_t *)_paravirt_ident_32
412
413#define paravirt_native_make_pud (make_pud_t *)_paravirt_ident_32
414#define paravirt_native_pud_val (pud_val_t *)_paravirt_ident_32
415
416#define paravirt_native_make_pgd (make_pgd_t *)_paravirt_ident_32
417#define paravirt_native_pgd_val (pgd_val_t *)_paravirt_ident_32
418#else 397#else
419/* 64-bit pagetable entries */ 398/* 64-bit pagetable entries */
420#define paravirt_native_make_pte (make_pte_t *)_paravirt_ident_64 399#define PTE_IDENT __PV_IS_CALLEE_SAVE(_paravirt_ident_64)
421#define paravirt_native_pte_val (pte_val_t *)_paravirt_ident_64
422
423#define paravirt_native_make_pmd (make_pmd_t *)_paravirt_ident_64
424#define paravirt_native_pmd_val (pmd_val_t *)_paravirt_ident_64
425
426#define paravirt_native_make_pud (make_pud_t *)_paravirt_ident_64
427#define paravirt_native_pud_val (pud_val_t *)_paravirt_ident_64
428
429#define paravirt_native_make_pgd (make_pgd_t *)_paravirt_ident_64
430#define paravirt_native_pgd_val (pgd_val_t *)_paravirt_ident_64
431#endif 400#endif
432 401
433struct pv_mmu_ops pv_mmu_ops = { 402struct pv_mmu_ops pv_mmu_ops = {
@@ -481,21 +450,23 @@ struct pv_mmu_ops pv_mmu_ops = {
481 .pmd_clear = native_pmd_clear, 450 .pmd_clear = native_pmd_clear,
482#endif 451#endif
483 .set_pud = native_set_pud, 452 .set_pud = native_set_pud,
484 .pmd_val = paravirt_native_pmd_val, 453
485 .make_pmd = paravirt_native_make_pmd, 454 .pmd_val = PTE_IDENT,
455 .make_pmd = PTE_IDENT,
486 456
487#if PAGETABLE_LEVELS == 4 457#if PAGETABLE_LEVELS == 4
488 .pud_val = paravirt_native_pud_val, 458 .pud_val = PTE_IDENT,
489 .make_pud = paravirt_native_make_pud, 459 .make_pud = PTE_IDENT,
460
490 .set_pgd = native_set_pgd, 461 .set_pgd = native_set_pgd,
491#endif 462#endif
492#endif /* PAGETABLE_LEVELS >= 3 */ 463#endif /* PAGETABLE_LEVELS >= 3 */
493 464
494 .pte_val = paravirt_native_pte_val, 465 .pte_val = PTE_IDENT,
495 .pgd_val = paravirt_native_pgd_val, 466 .pgd_val = PTE_IDENT,
496 467
497 .make_pte = paravirt_native_make_pte, 468 .make_pte = PTE_IDENT,
498 .make_pgd = paravirt_native_make_pgd, 469 .make_pgd = PTE_IDENT,
499 470
500 .dup_mmap = paravirt_nop, 471 .dup_mmap = paravirt_nop,
501 .exit_mmap = paravirt_nop, 472 .exit_mmap = paravirt_nop,
diff --git a/arch/x86/xen/mmu.c b/arch/x86/xen/mmu.c
index 94e452c0b00c..5e41f7fc6cf1 100644
--- a/arch/x86/xen/mmu.c
+++ b/arch/x86/xen/mmu.c
@@ -492,28 +492,33 @@ pteval_t xen_pte_val(pte_t pte)
492{ 492{
493 return pte_mfn_to_pfn(pte.pte); 493 return pte_mfn_to_pfn(pte.pte);
494} 494}
495PV_CALLEE_SAVE_REGS_THUNK(xen_pte_val);
495 496
496pgdval_t xen_pgd_val(pgd_t pgd) 497pgdval_t xen_pgd_val(pgd_t pgd)
497{ 498{
498 return pte_mfn_to_pfn(pgd.pgd); 499 return pte_mfn_to_pfn(pgd.pgd);
499} 500}
501PV_CALLEE_SAVE_REGS_THUNK(xen_pgd_val);
500 502
501pte_t xen_make_pte(pteval_t pte) 503pte_t xen_make_pte(pteval_t pte)
502{ 504{
503 pte = pte_pfn_to_mfn(pte); 505 pte = pte_pfn_to_mfn(pte);
504 return native_make_pte(pte); 506 return native_make_pte(pte);
505} 507}
508PV_CALLEE_SAVE_REGS_THUNK(xen_make_pte);
506 509
507pgd_t xen_make_pgd(pgdval_t pgd) 510pgd_t xen_make_pgd(pgdval_t pgd)
508{ 511{
509 pgd = pte_pfn_to_mfn(pgd); 512 pgd = pte_pfn_to_mfn(pgd);
510 return native_make_pgd(pgd); 513 return native_make_pgd(pgd);
511} 514}
515PV_CALLEE_SAVE_REGS_THUNK(xen_make_pgd);
512 516
513pmdval_t xen_pmd_val(pmd_t pmd) 517pmdval_t xen_pmd_val(pmd_t pmd)
514{ 518{
515 return pte_mfn_to_pfn(pmd.pmd); 519 return pte_mfn_to_pfn(pmd.pmd);
516} 520}
521PV_CALLEE_SAVE_REGS_THUNK(xen_pmd_val);
517 522
518void xen_set_pud_hyper(pud_t *ptr, pud_t val) 523void xen_set_pud_hyper(pud_t *ptr, pud_t val)
519{ 524{
@@ -590,12 +595,14 @@ pmd_t xen_make_pmd(pmdval_t pmd)
590 pmd = pte_pfn_to_mfn(pmd); 595 pmd = pte_pfn_to_mfn(pmd);
591 return native_make_pmd(pmd); 596 return native_make_pmd(pmd);
592} 597}
598PV_CALLEE_SAVE_REGS_THUNK(xen_make_pmd);
593 599
594#if PAGETABLE_LEVELS == 4 600#if PAGETABLE_LEVELS == 4
595pudval_t xen_pud_val(pud_t pud) 601pudval_t xen_pud_val(pud_t pud)
596{ 602{
597 return pte_mfn_to_pfn(pud.pud); 603 return pte_mfn_to_pfn(pud.pud);
598} 604}
605PV_CALLEE_SAVE_REGS_THUNK(xen_pud_val);
599 606
600pud_t xen_make_pud(pudval_t pud) 607pud_t xen_make_pud(pudval_t pud)
601{ 608{
@@ -603,6 +610,7 @@ pud_t xen_make_pud(pudval_t pud)
603 610
604 return native_make_pud(pud); 611 return native_make_pud(pud);
605} 612}
613PV_CALLEE_SAVE_REGS_THUNK(xen_make_pud);
606 614
607pgd_t *xen_get_user_pgd(pgd_t *pgd) 615pgd_t *xen_get_user_pgd(pgd_t *pgd)
608{ 616{
@@ -1813,11 +1821,11 @@ const struct pv_mmu_ops xen_mmu_ops __initdata = {
1813 .ptep_modify_prot_start = __ptep_modify_prot_start, 1821 .ptep_modify_prot_start = __ptep_modify_prot_start,
1814 .ptep_modify_prot_commit = __ptep_modify_prot_commit, 1822 .ptep_modify_prot_commit = __ptep_modify_prot_commit,
1815 1823
1816 .pte_val = xen_pte_val, 1824 .pte_val = PV_CALLEE_SAVE(xen_pte_val),
1817 .pgd_val = xen_pgd_val, 1825 .pgd_val = PV_CALLEE_SAVE(xen_pgd_val),
1818 1826
1819 .make_pte = xen_make_pte, 1827 .make_pte = PV_CALLEE_SAVE(xen_make_pte),
1820 .make_pgd = xen_make_pgd, 1828 .make_pgd = PV_CALLEE_SAVE(xen_make_pgd),
1821 1829
1822#ifdef CONFIG_X86_PAE 1830#ifdef CONFIG_X86_PAE
1823 .set_pte_atomic = xen_set_pte_atomic, 1831 .set_pte_atomic = xen_set_pte_atomic,
@@ -1827,12 +1835,12 @@ const struct pv_mmu_ops xen_mmu_ops __initdata = {
1827#endif /* CONFIG_X86_PAE */ 1835#endif /* CONFIG_X86_PAE */
1828 .set_pud = xen_set_pud_hyper, 1836 .set_pud = xen_set_pud_hyper,
1829 1837
1830 .make_pmd = xen_make_pmd, 1838 .make_pmd = PV_CALLEE_SAVE(xen_make_pmd),
1831 .pmd_val = xen_pmd_val, 1839 .pmd_val = PV_CALLEE_SAVE(xen_pmd_val),
1832 1840
1833#if PAGETABLE_LEVELS == 4 1841#if PAGETABLE_LEVELS == 4
1834 .pud_val = xen_pud_val, 1842 .pud_val = PV_CALLEE_SAVE(xen_pud_val),
1835 .make_pud = xen_make_pud, 1843 .make_pud = PV_CALLEE_SAVE(xen_make_pud),
1836 .set_pgd = xen_set_pgd_hyper, 1844 .set_pgd = xen_set_pgd_hyper,
1837 1845
1838 .alloc_pud = xen_alloc_pte_init, 1846 .alloc_pud = xen_alloc_pte_init,