aboutsummaryrefslogtreecommitdiffstats
path: root/arch
diff options
context:
space:
mode:
authorDavid Daney <ddaney@caviumnetworks.com>2010-12-20 18:54:50 -0500
committerRalf Baechle <ralf@linux-mips.org>2011-01-18 13:30:23 -0500
commitcc33ae437975416a1b78f99e2715e91ab643526a (patch)
tree958e4165ddcd45bf6ba9c498fa8f736dcbcbe770 /arch
parentafc7c9864a2d1b0c398425aac84b8a095c8dfa7c (diff)
MIPS: Use BBIT instructions in TLB handlers
If the CPU supports BBIT0 and BBIT1, use them in TLB handlers as they are more efficient than an AND followed by an branch and then restoring the clobbered register. Signed-off-by: David Daney <ddaney@caviumnetworks.com> To: linux-mips@linux-mips.org Patchwork: https://patchwork.linux-mips.org/patch/1873/ Signed-off-by: Ralf Baechle <ralf@linux-mips.org>
Diffstat (limited to 'arch')
-rw-r--r--arch/mips/mm/tlbex.c119
1 files changed, 90 insertions, 29 deletions
diff --git a/arch/mips/mm/tlbex.c b/arch/mips/mm/tlbex.c
index 0bb4c3b89c78..883cf76fb2bd 100644
--- a/arch/mips/mm/tlbex.c
+++ b/arch/mips/mm/tlbex.c
@@ -65,6 +65,18 @@ static inline int __maybe_unused r10000_llsc_war(void)
65 return R10000_LLSC_WAR; 65 return R10000_LLSC_WAR;
66} 66}
67 67
68static int use_bbit_insns(void)
69{
70 switch (current_cpu_type()) {
71 case CPU_CAVIUM_OCTEON:
72 case CPU_CAVIUM_OCTEON_PLUS:
73 case CPU_CAVIUM_OCTEON2:
74 return 1;
75 default:
76 return 0;
77 }
78}
79
68/* 80/*
69 * Found by experiment: At least some revisions of the 4kc throw under 81 * Found by experiment: At least some revisions of the 4kc throw under
70 * some circumstances a machine check exception, triggered by invalid 82 * some circumstances a machine check exception, triggered by invalid
@@ -511,8 +523,12 @@ build_is_huge_pte(u32 **p, struct uasm_reloc **r, unsigned int tmp,
511 unsigned int pmd, int lid) 523 unsigned int pmd, int lid)
512{ 524{
513 UASM_i_LW(p, tmp, 0, pmd); 525 UASM_i_LW(p, tmp, 0, pmd);
514 uasm_i_andi(p, tmp, tmp, _PAGE_HUGE); 526 if (use_bbit_insns()) {
515 uasm_il_bnez(p, r, tmp, lid); 527 uasm_il_bbit1(p, r, tmp, ilog2(_PAGE_HUGE), lid);
528 } else {
529 uasm_i_andi(p, tmp, tmp, _PAGE_HUGE);
530 uasm_il_bnez(p, r, tmp, lid);
531 }
516} 532}
517 533
518static __cpuinit void build_huge_update_entries(u32 **p, 534static __cpuinit void build_huge_update_entries(u32 **p,
@@ -1187,14 +1203,20 @@ build_pte_present(u32 **p, struct uasm_reloc **r,
1187 unsigned int pte, unsigned int ptr, enum label_id lid) 1203 unsigned int pte, unsigned int ptr, enum label_id lid)
1188{ 1204{
1189 if (kernel_uses_smartmips_rixi) { 1205 if (kernel_uses_smartmips_rixi) {
1190 uasm_i_andi(p, pte, pte, _PAGE_PRESENT); 1206 if (use_bbit_insns()) {
1191 uasm_il_beqz(p, r, pte, lid); 1207 uasm_il_bbit0(p, r, pte, ilog2(_PAGE_PRESENT), lid);
1208 uasm_i_nop(p);
1209 } else {
1210 uasm_i_andi(p, pte, pte, _PAGE_PRESENT);
1211 uasm_il_beqz(p, r, pte, lid);
1212 iPTE_LW(p, pte, ptr);
1213 }
1192 } else { 1214 } else {
1193 uasm_i_andi(p, pte, pte, _PAGE_PRESENT | _PAGE_READ); 1215 uasm_i_andi(p, pte, pte, _PAGE_PRESENT | _PAGE_READ);
1194 uasm_i_xori(p, pte, pte, _PAGE_PRESENT | _PAGE_READ); 1216 uasm_i_xori(p, pte, pte, _PAGE_PRESENT | _PAGE_READ);
1195 uasm_il_bnez(p, r, pte, lid); 1217 uasm_il_bnez(p, r, pte, lid);
1218 iPTE_LW(p, pte, ptr);
1196 } 1219 }
1197 iPTE_LW(p, pte, ptr);
1198} 1220}
1199 1221
1200/* Make PTE valid, store result in PTR. */ 1222/* Make PTE valid, store result in PTR. */
@@ -1215,10 +1237,17 @@ static void __cpuinit
1215build_pte_writable(u32 **p, struct uasm_reloc **r, 1237build_pte_writable(u32 **p, struct uasm_reloc **r,
1216 unsigned int pte, unsigned int ptr, enum label_id lid) 1238 unsigned int pte, unsigned int ptr, enum label_id lid)
1217{ 1239{
1218 uasm_i_andi(p, pte, pte, _PAGE_PRESENT | _PAGE_WRITE); 1240 if (use_bbit_insns()) {
1219 uasm_i_xori(p, pte, pte, _PAGE_PRESENT | _PAGE_WRITE); 1241 uasm_il_bbit0(p, r, pte, ilog2(_PAGE_PRESENT), lid);
1220 uasm_il_bnez(p, r, pte, lid); 1242 uasm_i_nop(p);
1221 iPTE_LW(p, pte, ptr); 1243 uasm_il_bbit0(p, r, pte, ilog2(_PAGE_WRITE), lid);
1244 uasm_i_nop(p);
1245 } else {
1246 uasm_i_andi(p, pte, pte, _PAGE_PRESENT | _PAGE_WRITE);
1247 uasm_i_xori(p, pte, pte, _PAGE_PRESENT | _PAGE_WRITE);
1248 uasm_il_bnez(p, r, pte, lid);
1249 iPTE_LW(p, pte, ptr);
1250 }
1222} 1251}
1223 1252
1224/* Make PTE writable, update software status bits as well, then store 1253/* Make PTE writable, update software status bits as well, then store
@@ -1242,9 +1271,14 @@ static void __cpuinit
1242build_pte_modifiable(u32 **p, struct uasm_reloc **r, 1271build_pte_modifiable(u32 **p, struct uasm_reloc **r,
1243 unsigned int pte, unsigned int ptr, enum label_id lid) 1272 unsigned int pte, unsigned int ptr, enum label_id lid)
1244{ 1273{
1245 uasm_i_andi(p, pte, pte, _PAGE_WRITE); 1274 if (use_bbit_insns()) {
1246 uasm_il_beqz(p, r, pte, lid); 1275 uasm_il_bbit0(p, r, pte, ilog2(_PAGE_WRITE), lid);
1247 iPTE_LW(p, pte, ptr); 1276 uasm_i_nop(p);
1277 } else {
1278 uasm_i_andi(p, pte, pte, _PAGE_WRITE);
1279 uasm_il_beqz(p, r, pte, lid);
1280 iPTE_LW(p, pte, ptr);
1281 }
1248} 1282}
1249 1283
1250#ifndef CONFIG_MIPS_PGD_C0_CONTEXT 1284#ifndef CONFIG_MIPS_PGD_C0_CONTEXT
@@ -1491,14 +1525,23 @@ static void __cpuinit build_r4000_tlb_load_handler(void)
1491 * If the page is not _PAGE_VALID, RI or XI could not 1525 * If the page is not _PAGE_VALID, RI or XI could not
1492 * have triggered it. Skip the expensive test.. 1526 * have triggered it. Skip the expensive test..
1493 */ 1527 */
1494 uasm_i_andi(&p, K0, K0, _PAGE_VALID); 1528 if (use_bbit_insns()) {
1495 uasm_il_beqz(&p, &r, K0, label_tlbl_goaround1); 1529 uasm_il_bbit0(&p, &r, K0, ilog2(_PAGE_VALID),
1530 label_tlbl_goaround1);
1531 } else {
1532 uasm_i_andi(&p, K0, K0, _PAGE_VALID);
1533 uasm_il_beqz(&p, &r, K0, label_tlbl_goaround1);
1534 }
1496 uasm_i_nop(&p); 1535 uasm_i_nop(&p);
1497 1536
1498 uasm_i_tlbr(&p); 1537 uasm_i_tlbr(&p);
1499 /* Examine entrylo 0 or 1 based on ptr. */ 1538 /* Examine entrylo 0 or 1 based on ptr. */
1500 uasm_i_andi(&p, K0, K1, sizeof(pte_t)); 1539 if (use_bbit_insns()) {
1501 uasm_i_beqz(&p, K0, 8); 1540 uasm_i_bbit0(&p, K1, ilog2(sizeof(pte_t)), 8);
1541 } else {
1542 uasm_i_andi(&p, K0, K1, sizeof(pte_t));
1543 uasm_i_beqz(&p, K0, 8);
1544 }
1502 1545
1503 UASM_i_MFC0(&p, K0, C0_ENTRYLO0); /* load it in the delay slot*/ 1546 UASM_i_MFC0(&p, K0, C0_ENTRYLO0); /* load it in the delay slot*/
1504 UASM_i_MFC0(&p, K0, C0_ENTRYLO1); /* load it if ptr is odd */ 1547 UASM_i_MFC0(&p, K0, C0_ENTRYLO1); /* load it if ptr is odd */
@@ -1506,12 +1549,18 @@ static void __cpuinit build_r4000_tlb_load_handler(void)
1506 * If the entryLo (now in K0) is valid (bit 1), RI or 1549 * If the entryLo (now in K0) is valid (bit 1), RI or
1507 * XI must have triggered it. 1550 * XI must have triggered it.
1508 */ 1551 */
1509 uasm_i_andi(&p, K0, K0, 2); 1552 if (use_bbit_insns()) {
1510 uasm_il_bnez(&p, &r, K0, label_nopage_tlbl); 1553 uasm_il_bbit1(&p, &r, K0, 1, label_nopage_tlbl);
1511 1554 /* Reload the PTE value */
1512 uasm_l_tlbl_goaround1(&l, p); 1555 iPTE_LW(&p, K0, K1);
1513 /* Reload the PTE value */ 1556 uasm_l_tlbl_goaround1(&l, p);
1514 iPTE_LW(&p, K0, K1); 1557 } else {
1558 uasm_i_andi(&p, K0, K0, 2);
1559 uasm_il_bnez(&p, &r, K0, label_nopage_tlbl);
1560 uasm_l_tlbl_goaround1(&l, p);
1561 /* Reload the PTE value */
1562 iPTE_LW(&p, K0, K1);
1563 }
1515 } 1564 }
1516 build_make_valid(&p, &r, K0, K1); 1565 build_make_valid(&p, &r, K0, K1);
1517 build_r4000_tlbchange_handler_tail(&p, &l, &r, K0, K1); 1566 build_r4000_tlbchange_handler_tail(&p, &l, &r, K0, K1);
@@ -1531,23 +1580,35 @@ static void __cpuinit build_r4000_tlb_load_handler(void)
1531 * If the page is not _PAGE_VALID, RI or XI could not 1580 * If the page is not _PAGE_VALID, RI or XI could not
1532 * have triggered it. Skip the expensive test.. 1581 * have triggered it. Skip the expensive test..
1533 */ 1582 */
1534 uasm_i_andi(&p, K0, K0, _PAGE_VALID); 1583 if (use_bbit_insns()) {
1535 uasm_il_beqz(&p, &r, K0, label_tlbl_goaround2); 1584 uasm_il_bbit0(&p, &r, K0, ilog2(_PAGE_VALID),
1585 label_tlbl_goaround2);
1586 } else {
1587 uasm_i_andi(&p, K0, K0, _PAGE_VALID);
1588 uasm_il_beqz(&p, &r, K0, label_tlbl_goaround2);
1589 }
1536 uasm_i_nop(&p); 1590 uasm_i_nop(&p);
1537 1591
1538 uasm_i_tlbr(&p); 1592 uasm_i_tlbr(&p);
1539 /* Examine entrylo 0 or 1 based on ptr. */ 1593 /* Examine entrylo 0 or 1 based on ptr. */
1540 uasm_i_andi(&p, K0, K1, sizeof(pte_t)); 1594 if (use_bbit_insns()) {
1541 uasm_i_beqz(&p, K0, 8); 1595 uasm_i_bbit0(&p, K1, ilog2(sizeof(pte_t)), 8);
1542 1596 } else {
1597 uasm_i_andi(&p, K0, K1, sizeof(pte_t));
1598 uasm_i_beqz(&p, K0, 8);
1599 }
1543 UASM_i_MFC0(&p, K0, C0_ENTRYLO0); /* load it in the delay slot*/ 1600 UASM_i_MFC0(&p, K0, C0_ENTRYLO0); /* load it in the delay slot*/
1544 UASM_i_MFC0(&p, K0, C0_ENTRYLO1); /* load it if ptr is odd */ 1601 UASM_i_MFC0(&p, K0, C0_ENTRYLO1); /* load it if ptr is odd */
1545 /* 1602 /*
1546 * If the entryLo (now in K0) is valid (bit 1), RI or 1603 * If the entryLo (now in K0) is valid (bit 1), RI or
1547 * XI must have triggered it. 1604 * XI must have triggered it.
1548 */ 1605 */
1549 uasm_i_andi(&p, K0, K0, 2); 1606 if (use_bbit_insns()) {
1550 uasm_il_beqz(&p, &r, K0, label_tlbl_goaround2); 1607 uasm_il_bbit0(&p, &r, K0, 1, label_tlbl_goaround2);
1608 } else {
1609 uasm_i_andi(&p, K0, K0, 2);
1610 uasm_il_beqz(&p, &r, K0, label_tlbl_goaround2);
1611 }
1551 /* Reload the PTE value */ 1612 /* Reload the PTE value */
1552 iPTE_LW(&p, K0, K1); 1613 iPTE_LW(&p, K0, K1);
1553 1614