diff options
Diffstat (limited to 'include/linux/mm.h')
-rw-r--r-- | include/linux/mm.h | 118 |
1 files changed, 92 insertions, 26 deletions
diff --git a/include/linux/mm.h b/include/linux/mm.h index 8b6e55ee8855..81443d557a2e 100644 --- a/include/linux/mm.h +++ b/include/linux/mm.h | |||
@@ -581,11 +581,11 @@ static inline pte_t maybe_mkwrite(pte_t pte, struct vm_area_struct *vma) | |||
581 | * sets it, so none of the operations on it need to be atomic. | 581 | * sets it, so none of the operations on it need to be atomic. |
582 | */ | 582 | */ |
583 | 583 | ||
584 | /* Page flags: | [SECTION] | [NODE] | ZONE | [LAST_NID] | ... | FLAGS | */ | 584 | /* Page flags: | [SECTION] | [NODE] | ZONE | [LAST_CPUPID] | ... | FLAGS | */ |
585 | #define SECTIONS_PGOFF ((sizeof(unsigned long)*8) - SECTIONS_WIDTH) | 585 | #define SECTIONS_PGOFF ((sizeof(unsigned long)*8) - SECTIONS_WIDTH) |
586 | #define NODES_PGOFF (SECTIONS_PGOFF - NODES_WIDTH) | 586 | #define NODES_PGOFF (SECTIONS_PGOFF - NODES_WIDTH) |
587 | #define ZONES_PGOFF (NODES_PGOFF - ZONES_WIDTH) | 587 | #define ZONES_PGOFF (NODES_PGOFF - ZONES_WIDTH) |
588 | #define LAST_NID_PGOFF (ZONES_PGOFF - LAST_NID_WIDTH) | 588 | #define LAST_CPUPID_PGOFF (ZONES_PGOFF - LAST_CPUPID_WIDTH) |
589 | 589 | ||
590 | /* | 590 | /* |
591 | * Define the bit shifts to access each section. For non-existent | 591 | * Define the bit shifts to access each section. For non-existent |
@@ -595,7 +595,7 @@ static inline pte_t maybe_mkwrite(pte_t pte, struct vm_area_struct *vma) | |||
595 | #define SECTIONS_PGSHIFT (SECTIONS_PGOFF * (SECTIONS_WIDTH != 0)) | 595 | #define SECTIONS_PGSHIFT (SECTIONS_PGOFF * (SECTIONS_WIDTH != 0)) |
596 | #define NODES_PGSHIFT (NODES_PGOFF * (NODES_WIDTH != 0)) | 596 | #define NODES_PGSHIFT (NODES_PGOFF * (NODES_WIDTH != 0)) |
597 | #define ZONES_PGSHIFT (ZONES_PGOFF * (ZONES_WIDTH != 0)) | 597 | #define ZONES_PGSHIFT (ZONES_PGOFF * (ZONES_WIDTH != 0)) |
598 | #define LAST_NID_PGSHIFT (LAST_NID_PGOFF * (LAST_NID_WIDTH != 0)) | 598 | #define LAST_CPUPID_PGSHIFT (LAST_CPUPID_PGOFF * (LAST_CPUPID_WIDTH != 0)) |
599 | 599 | ||
600 | /* NODE:ZONE or SECTION:ZONE is used to ID a zone for the buddy allocator */ | 600 | /* NODE:ZONE or SECTION:ZONE is used to ID a zone for the buddy allocator */ |
601 | #ifdef NODE_NOT_IN_PAGE_FLAGS | 601 | #ifdef NODE_NOT_IN_PAGE_FLAGS |
@@ -617,7 +617,7 @@ static inline pte_t maybe_mkwrite(pte_t pte, struct vm_area_struct *vma) | |||
617 | #define ZONES_MASK ((1UL << ZONES_WIDTH) - 1) | 617 | #define ZONES_MASK ((1UL << ZONES_WIDTH) - 1) |
618 | #define NODES_MASK ((1UL << NODES_WIDTH) - 1) | 618 | #define NODES_MASK ((1UL << NODES_WIDTH) - 1) |
619 | #define SECTIONS_MASK ((1UL << SECTIONS_WIDTH) - 1) | 619 | #define SECTIONS_MASK ((1UL << SECTIONS_WIDTH) - 1) |
620 | #define LAST_NID_MASK ((1UL << LAST_NID_WIDTH) - 1) | 620 | #define LAST_CPUPID_MASK ((1UL << LAST_CPUPID_WIDTH) - 1) |
621 | #define ZONEID_MASK ((1UL << ZONEID_SHIFT) - 1) | 621 | #define ZONEID_MASK ((1UL << ZONEID_SHIFT) - 1) |
622 | 622 | ||
623 | static inline enum zone_type page_zonenum(const struct page *page) | 623 | static inline enum zone_type page_zonenum(const struct page *page) |
@@ -661,51 +661,117 @@ static inline int page_to_nid(const struct page *page) | |||
661 | #endif | 661 | #endif |
662 | 662 | ||
663 | #ifdef CONFIG_NUMA_BALANCING | 663 | #ifdef CONFIG_NUMA_BALANCING |
664 | #ifdef LAST_NID_NOT_IN_PAGE_FLAGS | 664 | static inline int cpu_pid_to_cpupid(int cpu, int pid) |
665 | static inline int page_nid_xchg_last(struct page *page, int nid) | ||
666 | { | 665 | { |
667 | return xchg(&page->_last_nid, nid); | 666 | return ((cpu & LAST__CPU_MASK) << LAST__PID_SHIFT) | (pid & LAST__PID_MASK); |
668 | } | 667 | } |
669 | 668 | ||
670 | static inline int page_nid_last(struct page *page) | 669 | static inline int cpupid_to_pid(int cpupid) |
671 | { | 670 | { |
672 | return page->_last_nid; | 671 | return cpupid & LAST__PID_MASK; |
673 | } | 672 | } |
674 | static inline void page_nid_reset_last(struct page *page) | 673 | |
674 | static inline int cpupid_to_cpu(int cpupid) | ||
675 | { | 675 | { |
676 | page->_last_nid = -1; | 676 | return (cpupid >> LAST__PID_SHIFT) & LAST__CPU_MASK; |
677 | } | 677 | } |
678 | #else | 678 | |
679 | static inline int page_nid_last(struct page *page) | 679 | static inline int cpupid_to_nid(int cpupid) |
680 | { | 680 | { |
681 | return (page->flags >> LAST_NID_PGSHIFT) & LAST_NID_MASK; | 681 | return cpu_to_node(cpupid_to_cpu(cpupid)); |
682 | } | 682 | } |
683 | 683 | ||
684 | extern int page_nid_xchg_last(struct page *page, int nid); | 684 | static inline bool cpupid_pid_unset(int cpupid) |
685 | { | ||
686 | return cpupid_to_pid(cpupid) == (-1 & LAST__PID_MASK); | ||
687 | } | ||
685 | 688 | ||
686 | static inline void page_nid_reset_last(struct page *page) | 689 | static inline bool cpupid_cpu_unset(int cpupid) |
687 | { | 690 | { |
688 | int nid = (1 << LAST_NID_SHIFT) - 1; | 691 | return cpupid_to_cpu(cpupid) == (-1 & LAST__CPU_MASK); |
692 | } | ||
689 | 693 | ||
690 | page->flags &= ~(LAST_NID_MASK << LAST_NID_PGSHIFT); | 694 | static inline bool __cpupid_match_pid(pid_t task_pid, int cpupid) |
691 | page->flags |= (nid & LAST_NID_MASK) << LAST_NID_PGSHIFT; | 695 | { |
696 | return (task_pid & LAST__PID_MASK) == cpupid_to_pid(cpupid); | ||
697 | } | ||
698 | |||
699 | #define cpupid_match_pid(task, cpupid) __cpupid_match_pid(task->pid, cpupid) | ||
700 | #ifdef LAST_CPUPID_NOT_IN_PAGE_FLAGS | ||
701 | static inline int page_cpupid_xchg_last(struct page *page, int cpupid) | ||
702 | { | ||
703 | return xchg(&page->_last_cpupid, cpupid); | ||
704 | } | ||
705 | |||
706 | static inline int page_cpupid_last(struct page *page) | ||
707 | { | ||
708 | return page->_last_cpupid; | ||
709 | } | ||
710 | static inline void page_cpupid_reset_last(struct page *page) | ||
711 | { | ||
712 | page->_last_cpupid = -1; | ||
692 | } | 713 | } |
693 | #endif /* LAST_NID_NOT_IN_PAGE_FLAGS */ | ||
694 | #else | 714 | #else |
695 | static inline int page_nid_xchg_last(struct page *page, int nid) | 715 | static inline int page_cpupid_last(struct page *page) |
696 | { | 716 | { |
697 | return page_to_nid(page); | 717 | return (page->flags >> LAST_CPUPID_PGSHIFT) & LAST_CPUPID_MASK; |
698 | } | 718 | } |
699 | 719 | ||
700 | static inline int page_nid_last(struct page *page) | 720 | extern int page_cpupid_xchg_last(struct page *page, int cpupid); |
721 | |||
722 | static inline void page_cpupid_reset_last(struct page *page) | ||
701 | { | 723 | { |
702 | return page_to_nid(page); | 724 | int cpupid = (1 << LAST_CPUPID_SHIFT) - 1; |
725 | |||
726 | page->flags &= ~(LAST_CPUPID_MASK << LAST_CPUPID_PGSHIFT); | ||
727 | page->flags |= (cpupid & LAST_CPUPID_MASK) << LAST_CPUPID_PGSHIFT; | ||
728 | } | ||
729 | #endif /* LAST_CPUPID_NOT_IN_PAGE_FLAGS */ | ||
730 | #else /* !CONFIG_NUMA_BALANCING */ | ||
731 | static inline int page_cpupid_xchg_last(struct page *page, int cpupid) | ||
732 | { | ||
733 | return page_to_nid(page); /* XXX */ | ||
703 | } | 734 | } |
704 | 735 | ||
705 | static inline void page_nid_reset_last(struct page *page) | 736 | static inline int page_cpupid_last(struct page *page) |
706 | { | 737 | { |
738 | return page_to_nid(page); /* XXX */ | ||
707 | } | 739 | } |
708 | #endif | 740 | |
741 | static inline int cpupid_to_nid(int cpupid) | ||
742 | { | ||
743 | return -1; | ||
744 | } | ||
745 | |||
746 | static inline int cpupid_to_pid(int cpupid) | ||
747 | { | ||
748 | return -1; | ||
749 | } | ||
750 | |||
751 | static inline int cpupid_to_cpu(int cpupid) | ||
752 | { | ||
753 | return -1; | ||
754 | } | ||
755 | |||
756 | static inline int cpu_pid_to_cpupid(int nid, int pid) | ||
757 | { | ||
758 | return -1; | ||
759 | } | ||
760 | |||
761 | static inline bool cpupid_pid_unset(int cpupid) | ||
762 | { | ||
763 | return 1; | ||
764 | } | ||
765 | |||
766 | static inline void page_cpupid_reset_last(struct page *page) | ||
767 | { | ||
768 | } | ||
769 | |||
770 | static inline bool cpupid_match_pid(struct task_struct *task, int cpupid) | ||
771 | { | ||
772 | return false; | ||
773 | } | ||
774 | #endif /* CONFIG_NUMA_BALANCING */ | ||
709 | 775 | ||
710 | static inline struct zone *page_zone(const struct page *page) | 776 | static inline struct zone *page_zone(const struct page *page) |
711 | { | 777 | { |