aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
-rw-r--r--include/linux/mempolicy.h2
-rw-r--r--mm/huge_memory.c42
-rw-r--r--mm/mempolicy.c2
3 files changed, 17 insertions, 29 deletions
diff --git a/include/linux/mempolicy.h b/include/linux/mempolicy.h
index bac395f1d00a..5228c62af416 100644
--- a/include/linux/mempolicy.h
+++ b/include/linux/mempolicy.h
@@ -139,8 +139,6 @@ struct mempolicy *mpol_shared_policy_lookup(struct shared_policy *sp,
139struct mempolicy *get_task_policy(struct task_struct *p); 139struct mempolicy *get_task_policy(struct task_struct *p);
140struct mempolicy *__get_vma_policy(struct vm_area_struct *vma, 140struct mempolicy *__get_vma_policy(struct vm_area_struct *vma,
141 unsigned long addr); 141 unsigned long addr);
142struct mempolicy *get_vma_policy(struct vm_area_struct *vma,
143 unsigned long addr);
144bool vma_policy_mof(struct vm_area_struct *vma); 142bool vma_policy_mof(struct vm_area_struct *vma);
145 143
146extern void numa_default_policy(void); 144extern void numa_default_policy(void);
diff --git a/mm/huge_memory.c b/mm/huge_memory.c
index 622cced74fd9..f2d19e4fe854 100644
--- a/mm/huge_memory.c
+++ b/mm/huge_memory.c
@@ -632,37 +632,27 @@ release:
632static inline gfp_t alloc_hugepage_direct_gfpmask(struct vm_area_struct *vma, unsigned long addr) 632static inline gfp_t alloc_hugepage_direct_gfpmask(struct vm_area_struct *vma, unsigned long addr)
633{ 633{
634 const bool vma_madvised = !!(vma->vm_flags & VM_HUGEPAGE); 634 const bool vma_madvised = !!(vma->vm_flags & VM_HUGEPAGE);
635 gfp_t this_node = 0; 635 const gfp_t gfp_mask = GFP_TRANSHUGE_LIGHT | __GFP_THISNODE;
636
637#ifdef CONFIG_NUMA
638 struct mempolicy *pol;
639 /*
640 * __GFP_THISNODE is used only when __GFP_DIRECT_RECLAIM is not
641 * specified, to express a general desire to stay on the current
642 * node for optimistic allocation attempts. If the defrag mode
643 * and/or madvise hint requires the direct reclaim then we prefer
644 * to fallback to other node rather than node reclaim because that
645 * can lead to excessive reclaim even though there is free memory
646 * on other nodes. We expect that NUMA preferences are specified
647 * by memory policies.
648 */
649 pol = get_vma_policy(vma, addr);
650 if (pol->mode != MPOL_BIND)
651 this_node = __GFP_THISNODE;
652 mpol_cond_put(pol);
653#endif
654 636
637 /* Always do synchronous compaction */
655 if (test_bit(TRANSPARENT_HUGEPAGE_DEFRAG_DIRECT_FLAG, &transparent_hugepage_flags)) 638 if (test_bit(TRANSPARENT_HUGEPAGE_DEFRAG_DIRECT_FLAG, &transparent_hugepage_flags))
656 return GFP_TRANSHUGE | (vma_madvised ? 0 : __GFP_NORETRY); 639 return GFP_TRANSHUGE | __GFP_THISNODE |
640 (vma_madvised ? 0 : __GFP_NORETRY);
641
642 /* Kick kcompactd and fail quickly */
657 if (test_bit(TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_FLAG, &transparent_hugepage_flags)) 643 if (test_bit(TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_FLAG, &transparent_hugepage_flags))
658 return GFP_TRANSHUGE_LIGHT | __GFP_KSWAPD_RECLAIM | this_node; 644 return gfp_mask | __GFP_KSWAPD_RECLAIM;
645
646 /* Synchronous compaction if madvised, otherwise kick kcompactd */
659 if (test_bit(TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_OR_MADV_FLAG, &transparent_hugepage_flags)) 647 if (test_bit(TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_OR_MADV_FLAG, &transparent_hugepage_flags))
660 return GFP_TRANSHUGE_LIGHT | (vma_madvised ? __GFP_DIRECT_RECLAIM : 648 return gfp_mask | (vma_madvised ? __GFP_DIRECT_RECLAIM :
661 __GFP_KSWAPD_RECLAIM | this_node); 649 __GFP_KSWAPD_RECLAIM);
650
651 /* Only do synchronous compaction if madvised */
662 if (test_bit(TRANSPARENT_HUGEPAGE_DEFRAG_REQ_MADV_FLAG, &transparent_hugepage_flags)) 652 if (test_bit(TRANSPARENT_HUGEPAGE_DEFRAG_REQ_MADV_FLAG, &transparent_hugepage_flags))
663 return GFP_TRANSHUGE_LIGHT | (vma_madvised ? __GFP_DIRECT_RECLAIM : 653 return gfp_mask | (vma_madvised ? __GFP_DIRECT_RECLAIM : 0);
664 this_node); 654
665 return GFP_TRANSHUGE_LIGHT | this_node; 655 return gfp_mask;
666} 656}
667 657
668/* Caller must hold page table lock. */ 658/* Caller must hold page table lock. */
diff --git a/mm/mempolicy.c b/mm/mempolicy.c
index 5837a067124d..69e278b469ef 100644
--- a/mm/mempolicy.c
+++ b/mm/mempolicy.c
@@ -1662,7 +1662,7 @@ struct mempolicy *__get_vma_policy(struct vm_area_struct *vma,
1662 * freeing by another task. It is the caller's responsibility to free the 1662 * freeing by another task. It is the caller's responsibility to free the
1663 * extra reference for shared policies. 1663 * extra reference for shared policies.
1664 */ 1664 */
1665struct mempolicy *get_vma_policy(struct vm_area_struct *vma, 1665static struct mempolicy *get_vma_policy(struct vm_area_struct *vma,
1666 unsigned long addr) 1666 unsigned long addr)
1667{ 1667{
1668 struct mempolicy *pol = __get_vma_policy(vma, addr); 1668 struct mempolicy *pol = __get_vma_policy(vma, addr);