aboutsummaryrefslogtreecommitdiffstats
path: root/include
diff options
context:
space:
mode:
authorDavid S. Miller <davem@sunset.davemloft.net>2006-02-13 00:10:07 -0500
committerDavid S. Miller <davem@sunset.davemloft.net>2006-03-20 04:12:32 -0500
commitcf627156c450cd5a0741b31f55181db3400d4887 (patch)
treee8f44d2509f5544ee5b5d583da3e10ac99ca3629 /include
parentff02e0d26f139ad95ec3a7e94f88faccaa180dff (diff)
[SPARC64]: Use inline patching for critical PTE operations.
This handles the SUN4U vs SUN4V PTE layout differences with near zero performance cost. Signed-off-by: David S. Miller <davem@davemloft.net>
Diffstat (limited to 'include')
-rw-r--r--include/asm-sparc64/pgtable.h488
1 files changed, 485 insertions, 3 deletions
diff --git a/include/asm-sparc64/pgtable.h b/include/asm-sparc64/pgtable.h
index 3c02d5d9a533..00eecbb52f95 100644
--- a/include/asm-sparc64/pgtable.h
+++ b/include/asm-sparc64/pgtable.h
@@ -227,11 +227,493 @@ extern struct page *mem_map_zero;
227 * the first physical page in the machine is at some huge physical address, 227 * the first physical page in the machine is at some huge physical address,
228 * such as 4GB. This is common on a partitioned E10000, for example. 228 * such as 4GB. This is common on a partitioned E10000, for example.
229 */ 229 */
230extern pte_t pfn_pte(unsigned long, pgprot_t); 230static inline pte_t pfn_pte(unsigned long pfn, pgprot_t prot)
231{
232 unsigned long paddr = pfn << PAGE_SHIFT;
233 unsigned long sz_bits;
234
235 BUILD_BUG_ON(!__builtin_constant_p(_PAGE_SZBITS_4U) ||
236 !__builtin_constant_p(_PAGE_SZBITS_4V));
237
238 sz_bits = 0UL;
239 if (_PAGE_SZBITS_4U != 0UL || _PAGE_SZBITS_4V != 0UL) {
240 BUILD_BUG_ON((_PAGE_SZBITS_4U & ~(0xfffffc0000000000UL)) ||
241 (_PAGE_SZBITS_4V & ~(0x0000000000000fffUL)));
242 __asm__ __volatile__(
243 "\n661: sethi %uhi(%1), %0\n"
244 " sllx %0, 32, %0\n"
245 " .section .sun4v_2insn_patch, \"ax\"\n"
246 " .word 661b\n"
247 " mov %2, %0\n"
248 " nop\n"
249 " .previous\n"
250 : "=r" (sz_bits)
251 : "i" (_PAGE_SZBITS_4U), "i" (_PAGE_SZBITS_4V));
252 }
253 return __pte(paddr | sz_bits | pgprot_val(prot));
254}
231#define mk_pte(page, pgprot) pfn_pte(page_to_pfn(page), (pgprot)) 255#define mk_pte(page, pgprot) pfn_pte(page_to_pfn(page), (pgprot))
232extern unsigned long pte_pfn(pte_t); 256
257/* This one can be done with two shifts. */
258static inline unsigned long pte_pfn(pte_t pte)
259{
260 const unsigned long pte_paddr_shl_sun4u = 21;
261 const unsigned long pte_paddr_shr_sun4u = 21 + PAGE_SHIFT;
262 const unsigned long pte_paddr_shl_sun4v = 8;
263 const unsigned long pte_paddr_shr_sun4v = 8 + PAGE_SHIFT;
264 unsigned long ret;
265
266 __asm__ __volatile__(
267 "\n661: sllx %1, %2, %0\n"
268 " srlx %0, %3, %0\n"
269 " .section .sun4v_2insn_patch, \"ax\"\n"
270 " .word 661b\n"
271 " sllx %1, %4, %0\n"
272 " srlx %0, %5, %0\n"
273 " .previous\n"
274 : "=r" (ret)
275 : "r" (pte_val(pte)),
276 "i" (pte_paddr_shl_sun4u), "i" (pte_paddr_shr_sun4u),
277 "i" (pte_paddr_shl_sun4v), "i" (pte_paddr_shr_sun4v));
278
279 return ret;
280}
233#define pte_page(x) pfn_to_page(pte_pfn(x)) 281#define pte_page(x) pfn_to_page(pte_pfn(x))
234extern pte_t pte_modify(pte_t, pgprot_t); 282
283static inline pte_t pte_modify(pte_t pte, pgprot_t prot)
284{
285 const unsigned long preserve_mask_sun4u = (_PAGE_PADDR_4U |
286 _PAGE_MODIFIED_4U |
287 _PAGE_ACCESSED_4U |
288 _PAGE_CP_4U |
289 _PAGE_CV_4U |
290 _PAGE_E_4U |
291 _PAGE_PRESENT_4U |
292 _PAGE_SZBITS_4U);
293 const unsigned long preserve_mask_sun4v = (_PAGE_PADDR_4V |
294 _PAGE_MODIFIED_4V |
295 _PAGE_ACCESSED_4V |
296 _PAGE_CP_4V |
297 _PAGE_CV_4V |
298 _PAGE_E_4V |
299 _PAGE_PRESENT_4V |
300 _PAGE_SZBITS_4V);
301 unsigned long mask, tmp;
302
303 /* SUN4U: 0x600307ffffffecb8 (negated == 0x9ffcf80000001347)
304 * SUN4V: 0x30ffffffffffee17 (negated == 0xcf000000000011e8)
305 *
306 * Even if we use negation tricks the result is still a 6
307 * instruction sequence, so don't try to play fancy and just
308 * do the most straightforward implementation.
309 *
310 * Note: We encode this into 3 sun4v 2-insn patch sequences.
311 */
312
313 __asm__ __volatile__(
314 "\n661: sethi %%uhi(%2), %1\n"
315 " sethi %%hi(%2), %0\n"
316 "\n662: or %1, %%ulo(%2), %1\n"
317 " or %0, %%lo(%2), %0\n"
318 "\n663: sllx %1, 32, %1\n"
319 " or %0, %1, %0\n"
320 " .section .sun4v_2insn_patch, \"ax\"\n"
321 " .word 661b\n"
322 " sethi %%uhi(%3), %1\n"
323 " sethi %%hi(%3), %0\n"
324 " .word 662b\n"
325 " or %1, %%ulo(%3), %1\n"
326 " or %0, %%lo(%3), %0\n"
327 " .word 663b\n"
328 " sllx %1, 32, %1\n"
329 " or %0, %1, %0\n"
330 " .previous\n"
331 : "=r" (mask), "=r" (tmp)
332 : "i" (preserve_mask_sun4u), "i" (preserve_mask_sun4v));
333
334 return __pte((pte_val(pte) & mask) | (pgprot_val(prot) & ~mask));
335}
336
337static inline pte_t pgoff_to_pte(unsigned long off)
338{
339 off <<= PAGE_SHIFT;
340
341 BUILD_BUG_ON((_PAGE_FILE_4U & ~0xfffUL) ||
342 (_PAGE_FILE_4V & ~0xfffUL));
343
344 __asm__ __volatile__(
345 "\n661: or %0, %2, %0\n"
346 " .section .sun4v_1insn_patch, \"ax\"\n"
347 " .word 661b\n"
348 " or %0, %3, %0\n"
349 " .previous\n"
350 : "=r" (off)
351 : "0" (off), "i" (_PAGE_FILE_4U), "i" (_PAGE_FILE_4V));
352
353 return __pte(off);
354}
355
356static inline pgprot_t pgprot_noncached(pgprot_t prot)
357{
358 unsigned long val = pgprot_val(prot);
359
360 BUILD_BUG_ON(((_PAGE_CP_4U | _PAGE_CP_4U | _PAGE_E_4U) & ~(0xfffUL)) ||
361 ((_PAGE_CP_4V | _PAGE_CP_4V | _PAGE_E_4V) & ~(0xfffUL)));
362
363 __asm__ __volatile__(
364 "\n661: andn %0, %2, %0\n"
365 " or %0, %3, %0\n"
366 " .section .sun4v_2insn_patch, \"ax\"\n"
367 " .word 661b\n"
368 " andn %0, %4, %0\n"
369 " or %0, %3, %0\n"
370 " .previous\n"
371 : "=r" (val)
372 : "0" (val), "i" (_PAGE_CP_4U | _PAGE_CV_4U), "i" (_PAGE_E_4U),
373 "i" (_PAGE_CP_4V | _PAGE_CV_4V), "i" (_PAGE_E_4V));
374
375 return __pgprot(val);
376}
377/* Various pieces of code check for platform support by ifdef testing
378 * on "pgprot_noncached". That's broken and should be fixed, but for
379 * now...
380 */
381#define pgprot_noncached pgprot_noncached
382
383static inline pte_t pte_mkhuge(pte_t pte)
384{
385 const unsigned long mask_4u = _PAGE_SZHUGE_4U;
386 const unsigned long mask_4v = _PAGE_SZHUGE_4V;
387 unsigned long mask;
388
389 BUILD_BUG_ON((mask_4u & ~(0xfffffc0000000000UL)) ||
390 (mask_4v & ~(0xfffUL)));
391
392 __asm__ __volatile__(
393 "\n661: sethi %%uhi(%1), %0\n"
394 " sllx %0, 32, %0\n"
395 " .section .sun4v_2insn_patch, \"ax\"\n"
396 " .word 661b\n"
397 " mov %2, %0\n"
398 " nop\n"
399 " .previous\n"
400 : "=r" (mask)
401 : "i" (mask_4u), "i" (mask_4v));
402
403 return __pte(pte_val(pte) | mask);
404}
405
406static inline pte_t pte_mkdirty(pte_t pte)
407{
408 const unsigned long mask_4u = _PAGE_MODIFIED_4U | _PAGE_W_4U;
409 const unsigned long mask_4v = _PAGE_MODIFIED_4V | _PAGE_W_4V;
410 unsigned long val = pte_val(pte), tmp;
411
412 BUILD_BUG_ON((mask_4u & ~(0x0000000000000fffUL)) ||
413 (mask_4v & ~(0xfffffc0000000fffUL)));
414
415 __asm__ __volatile__(
416 "\n661: or %0, %3, %0\n"
417 " nop\n"
418 "\n662: nop\n"
419 " nop\n"
420 " .section .sun4v_2insn_patch, \"ax\"\n"
421 " .word 661b\n"
422 " sethi %%uhi(%4), %1\n"
423 " sllx %1, 32, %1\n"
424 " .word 662b\n"
425 " or %1, %%lo(%4), %1\n"
426 " or %0, %1, %0\n"
427 " .previous\n"
428 : "=r" (val), "=r" (tmp)
429 : "0" (val), "i" (mask_4u), "i" (mask_4v));
430
431 return __pte(val);
432}
433
434static inline pte_t pte_mkclean(pte_t pte)
435{
436 const unsigned long mask_4u = _PAGE_MODIFIED_4U | _PAGE_W_4U;
437 const unsigned long mask_4v = _PAGE_MODIFIED_4V | _PAGE_W_4V;
438 unsigned long val = pte_val(pte), tmp;
439
440 BUILD_BUG_ON((mask_4u & ~(0x0000000000000fffUL)) ||
441 (mask_4v & ~(0xfffffc0000000fffUL)));
442
443 __asm__ __volatile__(
444 "\n661: andn %0, %3, %0\n"
445 " nop\n"
446 "\n662: nop\n"
447 " nop\n"
448 " .section .sun4v_2insn_patch, \"ax\"\n"
449 " .word 661b\n"
450 " sethi %%uhi(%4), %1\n"
451 " sllx %1, 32, %1\n"
452 " .word 662b\n"
453 " or %1, %%lo(%4), %1\n"
454 " andn %0, %1, %0\n"
455 " .previous\n"
456 : "=r" (val), "=r" (tmp)
457 : "0" (val), "i" (mask_4u), "i" (mask_4v));
458
459 return __pte(val);
460}
461
462static inline pte_t pte_mkwrite(pte_t pte)
463{
464 const unsigned long mask_4u = _PAGE_WRITE_4U;
465 const unsigned long mask_4v = _PAGE_WRITE_4V;
466 unsigned long val = pte_val(pte), mask;
467
468 BUILD_BUG_ON((mask_4u & ~(0x0000000000000fffUL)) ||
469 (mask_4v & ~(0xfffffc0000000000UL)));
470
471 __asm__ __volatile__(
472 "\n661: mov %1, %0\n"
473 " nop\n"
474 " .section .sun4v_2insn_patch, \"ax\"\n"
475 " .word 661b\n"
476 " sethi %%uhi(%2), %0\n"
477 " sllx %0, 32, %0\n"
478 " .previous\n"
479 : "=r" (mask)
480 : "i" (mask_4u), "i" (mask_4v));
481
482 return __pte(val | mask);
483}
484
485static inline pte_t pte_wrprotect(pte_t pte)
486{
487 const unsigned long mask_4u = _PAGE_WRITE_4U | _PAGE_W_4U;
488 const unsigned long mask_4v = _PAGE_WRITE_4V | _PAGE_W_4V;
489 unsigned long val = pte_val(pte), tmp;
490
491 BUILD_BUG_ON((mask_4u & ~(0x0000000000000fffUL)) ||
492 (mask_4v & ~(0xfffffc0000000fffUL)));
493
494 __asm__ __volatile__(
495 "\n661: andn %0, %3, %0\n"
496 " nop\n"
497 "\n662: nop\n"
498 " nop\n"
499 " .section .sun4v_2insn_patch, \"ax\"\n"
500 " .word 661b\n"
501 " sethi %%uhi(%4), %1\n"
502 " sllx %1, 32, %1\n"
503 " .word 662b\n"
504 " or %1, %%lo(%4), %1\n"
505 " andn %0, %1, %0\n"
506 " .previous\n"
507 : "=r" (val), "=r" (tmp)
508 : "0" (val), "i" (mask_4u), "i" (mask_4v));
509
510 return __pte(val);
511}
512
513static inline pte_t pte_mkold(pte_t pte)
514{
515 const unsigned long mask_4u = _PAGE_ACCESSED_4U;
516 const unsigned long mask_4v = _PAGE_ACCESSED_4V;
517 unsigned long mask;
518
519 BUILD_BUG_ON((mask_4u & ~(0x0000000000000fffUL)) ||
520 (mask_4v & ~(0xfffffc0000000000UL)));
521
522 __asm__ __volatile__(
523 "\n661: mov %1, %0\n"
524 " nop\n"
525 " .section .sun4v_2insn_patch, \"ax\"\n"
526 " .word 661b\n"
527 " sethi %%uhi(%2), %0\n"
528 " sllx %0, 32, %0\n"
529 " .previous\n"
530 : "=r" (mask)
531 : "i" (mask_4u), "i" (mask_4v));
532
533 mask |= _PAGE_R;
534
535 return __pte(pte_val(pte) & ~mask);
536}
537
538static inline pte_t pte_mkyoung(pte_t pte)
539{
540 const unsigned long mask_4u = _PAGE_ACCESSED_4U;
541 const unsigned long mask_4v = _PAGE_ACCESSED_4V;
542 unsigned long mask;
543
544 BUILD_BUG_ON((mask_4u & ~(0x0000000000000fffUL)) ||
545 (mask_4v & ~(0xfffffc0000000000UL)));
546
547 __asm__ __volatile__(
548 "\n661: mov %1, %0\n"
549 " nop\n"
550 " .section .sun4v_2insn_patch, \"ax\"\n"
551 " .word 661b\n"
552 " sethi %%uhi(%2), %0\n"
553 " sllx %0, 32, %0\n"
554 " .previous\n"
555 : "=r" (mask)
556 : "i" (mask_4u), "i" (mask_4v));
557
558 mask |= _PAGE_R;
559
560 return __pte(pte_val(pte) | mask);
561}
562
563static inline unsigned long pte_young(pte_t pte)
564{
565 const unsigned long mask_4u = _PAGE_ACCESSED_4U;
566 const unsigned long mask_4v = _PAGE_ACCESSED_4V;
567 unsigned long mask;
568
569 BUILD_BUG_ON((mask_4u & ~(0x0000000000000fffUL)) ||
570 (mask_4v & ~(0xfffffc0000000000UL)));
571
572 __asm__ __volatile__(
573 "\n661: mov %1, %0\n"
574 " nop\n"
575 " .section .sun4v_2insn_patch, \"ax\"\n"
576 " .word 661b\n"
577 " sethi %%uhi(%2), %0\n"
578 " sllx %0, 32, %0\n"
579 " .previous\n"
580 : "=r" (mask)
581 : "i" (mask_4u), "i" (mask_4v));
582
583 return (pte_val(pte) & mask);
584}
585
586static inline unsigned long pte_dirty(pte_t pte)
587{
588 const unsigned long mask_4u = _PAGE_MODIFIED_4U;
589 const unsigned long mask_4v = _PAGE_MODIFIED_4V;
590 unsigned long mask;
591
592 BUILD_BUG_ON((mask_4u & ~(0x0000000000000fffUL)) ||
593 (mask_4v & ~(0xfffffc0000000000UL)));
594
595 __asm__ __volatile__(
596 "\n661: mov %1, %0\n"
597 " nop\n"
598 " .section .sun4v_2insn_patch, \"ax\"\n"
599 " .word 661b\n"
600 " sethi %%uhi(%2), %0\n"
601 " sllx %0, 32, %0\n"
602 " .previous\n"
603 : "=r" (mask)
604 : "i" (mask_4u), "i" (mask_4v));
605
606 return (pte_val(pte) & mask);
607}
608
609static inline unsigned long pte_write(pte_t pte)
610{
611 const unsigned long mask_4u = _PAGE_WRITE_4U;
612 const unsigned long mask_4v = _PAGE_WRITE_4V;
613 unsigned long mask;
614
615 BUILD_BUG_ON((mask_4u & ~(0x0000000000000fffUL)) ||
616 (mask_4v & ~(0xfffffc0000000000UL)));
617
618 __asm__ __volatile__(
619 "\n661: mov %1, %0\n"
620 " nop\n"
621 " .section .sun4v_2insn_patch, \"ax\"\n"
622 " .word 661b\n"
623 " sethi %%uhi(%2), %0\n"
624 " sllx %0, 32, %0\n"
625 " .previous\n"
626 : "=r" (mask)
627 : "i" (mask_4u), "i" (mask_4v));
628
629 return (pte_val(pte) & mask);
630}
631
632static inline unsigned long pte_exec(pte_t pte)
633{
634 const unsigned long mask_4u = _PAGE_EXEC_4U;
635 const unsigned long mask_4v = _PAGE_EXEC_4V;
636 unsigned long mask;
637
638 BUILD_BUG_ON((mask_4u & ~(0x00000000fffffc00UL)) ||
639 (mask_4v & ~(0x0000000000000fffUL)));
640
641 __asm__ __volatile__(
642 "\n661: sethi %%hi(%1), %0\n"
643 " .section .sun4v_1insn_patch, \"ax\"\n"
644 " .word 661b\n"
645 " mov %2, %0\n"
646 " .previous\n"
647 : "=r" (mask)
648 : "i" (mask_4u), "i" (mask_4v));
649
650 return (pte_val(pte) & mask);
651}
652
653static inline unsigned long pte_read(pte_t pte)
654{
655 const unsigned long mask_4u = _PAGE_READ_4U;
656 const unsigned long mask_4v = _PAGE_READ_4V;
657 unsigned long mask;
658
659 BUILD_BUG_ON((mask_4u & ~(0x0000000000000fffUL)) ||
660 (mask_4v & ~(0xfffffc0000000000UL)));
661
662 __asm__ __volatile__(
663 "\n661: mov %1, %0\n"
664 " nop\n"
665 " .section .sun4v_2insn_patch, \"ax\"\n"
666 " .word 661b\n"
667 " sethi %%uhi(%2), %0\n"
668 " sllx %0, 32, %0\n"
669 " .previous\n"
670 : "=r" (mask)
671 : "i" (mask_4u), "i" (mask_4v));
672
673 return (pte_val(pte) & mask);
674}
675
676static inline unsigned long pte_file(pte_t pte)
677{
678 const unsigned long mask_4u = _PAGE_FILE_4U;
679 const unsigned long mask_4v = _PAGE_FILE_4V;
680 unsigned long val = pte_val(pte);
681
682 BUILD_BUG_ON((mask_4u & ~(0x0000000000000fffUL)) ||
683 (mask_4v & ~(0x0000000000000fffUL)));
684
685 __asm__ __volatile__(
686 "\n661: and %0, %2, %0\n"
687 " .section .sun4v_1insn_patch, \"ax\"\n"
688 " .word 661b\n"
689 " and %0, %3, %0\n"
690 " .previous\n"
691 : "=r" (val)
692 : "0" (val), "i" (mask_4u), "i" (mask_4v));
693
694 return val;
695}
696
697static inline unsigned long pte_present(pte_t pte)
698{
699 const unsigned long mask_4u = _PAGE_PRESENT_4U;
700 const unsigned long mask_4v = _PAGE_PRESENT_4V;
701 unsigned long val = pte_val(pte);
702
703 BUILD_BUG_ON((mask_4u & ~(0x0000000000000fffUL)) ||
704 (mask_4v & ~(0x0000000000000fffUL)));
705
706 __asm__ __volatile__(
707 "\n661: and %0, %2, %0\n"
708 " .section .sun4v_1insn_patch, \"ax\"\n"
709 " .word 661b\n"
710 " and %0, %3, %0\n"
711 " .previous\n"
712 : "=r" (val)
713 : "0" (val), "i" (mask_4u), "i" (mask_4v));
714
715 return val;
716}
235 717
236#define pmd_set(pmdp, ptep) \ 718#define pmd_set(pmdp, ptep) \
237 (pmd_val(*(pmdp)) = (__pa((unsigned long) (ptep)) >> 11UL)) 719 (pmd_val(*(pmdp)) = (__pa((unsigned long) (ptep)) >> 11UL))
generated by cgit v1.2.2 (git 2.25.0) at 2024-11-16 08:23:28 -0500