aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
-rw-r--r--include/linux/mmzone.h6
-rw-r--r--include/trace/events/migrate.h27
-rw-r--r--mm/migrate.c65
-rw-r--r--mm/page_alloc.c2
4 files changed, 0 insertions, 100 deletions
diff --git a/include/linux/mmzone.h b/include/linux/mmzone.h
index 1e22d96734e0..3f4c0b167333 100644
--- a/include/linux/mmzone.h
+++ b/include/linux/mmzone.h
@@ -671,12 +671,6 @@ typedef struct pglist_data {
671#ifdef CONFIG_NUMA_BALANCING 671#ifdef CONFIG_NUMA_BALANCING
672 /* Lock serializing the migrate rate limiting window */ 672 /* Lock serializing the migrate rate limiting window */
673 spinlock_t numabalancing_migrate_lock; 673 spinlock_t numabalancing_migrate_lock;
674
675 /* Rate limiting time interval */
676 unsigned long numabalancing_migrate_next_window;
677
678 /* Number of pages migrated during the rate limiting time interval */
679 unsigned long numabalancing_migrate_nr_pages;
680#endif 674#endif
681 /* 675 /*
682 * This is a per-node reserve of pages that are not available 676 * This is a per-node reserve of pages that are not available
diff --git a/include/trace/events/migrate.h b/include/trace/events/migrate.h
index 711372845945..705b33d1e395 100644
--- a/include/trace/events/migrate.h
+++ b/include/trace/events/migrate.h
@@ -70,33 +70,6 @@ TRACE_EVENT(mm_migrate_pages,
70 __print_symbolic(__entry->mode, MIGRATE_MODE), 70 __print_symbolic(__entry->mode, MIGRATE_MODE),
71 __print_symbolic(__entry->reason, MIGRATE_REASON)) 71 __print_symbolic(__entry->reason, MIGRATE_REASON))
72); 72);
73
74TRACE_EVENT(mm_numa_migrate_ratelimit,
75
76 TP_PROTO(struct task_struct *p, int dst_nid, unsigned long nr_pages),
77
78 TP_ARGS(p, dst_nid, nr_pages),
79
80 TP_STRUCT__entry(
81 __array( char, comm, TASK_COMM_LEN)
82 __field( pid_t, pid)
83 __field( int, dst_nid)
84 __field( unsigned long, nr_pages)
85 ),
86
87 TP_fast_assign(
88 memcpy(__entry->comm, p->comm, TASK_COMM_LEN);
89 __entry->pid = p->pid;
90 __entry->dst_nid = dst_nid;
91 __entry->nr_pages = nr_pages;
92 ),
93
94 TP_printk("comm=%s pid=%d dst_nid=%d nr_pages=%lu",
95 __entry->comm,
96 __entry->pid,
97 __entry->dst_nid,
98 __entry->nr_pages)
99);
100#endif /* _TRACE_MIGRATE_H */ 73#endif /* _TRACE_MIGRATE_H */
101 74
102/* This part must be outside protection */ 75/* This part must be outside protection */
diff --git a/mm/migrate.c b/mm/migrate.c
index 4f1d894835b5..5e285c1249a0 100644
--- a/mm/migrate.c
+++ b/mm/migrate.c
@@ -1855,54 +1855,6 @@ static struct page *alloc_misplaced_dst_page(struct page *page,
1855 return newpage; 1855 return newpage;
1856} 1856}
1857 1857
1858/*
1859 * page migration rate limiting control.
1860 * Do not migrate more than @pages_to_migrate in a @migrate_interval_millisecs
1861 * window of time. Default here says do not migrate more than 1280M per second.
1862 */
1863static unsigned int migrate_interval_millisecs __read_mostly = 100;
1864static unsigned int ratelimit_pages __read_mostly = 128 << (20 - PAGE_SHIFT);
1865
1866/* Returns true if the node is migrate rate-limited after the update */
1867static bool numamigrate_update_ratelimit(pg_data_t *pgdat,
1868 unsigned long nr_pages)
1869{
1870 unsigned long next_window, interval;
1871
1872 next_window = READ_ONCE(pgdat->numabalancing_migrate_next_window);
1873 interval = msecs_to_jiffies(migrate_interval_millisecs);
1874
1875 /*
1876 * Rate-limit the amount of data that is being migrated to a node.
1877 * Optimal placement is no good if the memory bus is saturated and
1878 * all the time is being spent migrating!
1879 */
1880 if (time_after(jiffies, next_window) &&
1881 spin_trylock(&pgdat->numabalancing_migrate_lock)) {
1882 pgdat->numabalancing_migrate_nr_pages = 0;
1883 do {
1884 next_window += interval;
1885 } while (unlikely(time_after(jiffies, next_window)));
1886
1887 WRITE_ONCE(pgdat->numabalancing_migrate_next_window, next_window);
1888 spin_unlock(&pgdat->numabalancing_migrate_lock);
1889 }
1890 if (pgdat->numabalancing_migrate_nr_pages > ratelimit_pages) {
1891 trace_mm_numa_migrate_ratelimit(current, pgdat->node_id,
1892 nr_pages);
1893 return true;
1894 }
1895
1896 /*
1897 * This is an unlocked non-atomic update so errors are possible.
1898 * The consequences are failing to migrate when we potentiall should
1899 * have which is not severe enough to warrant locking. If it is ever
1900 * a problem, it can be converted to a per-cpu counter.
1901 */
1902 pgdat->numabalancing_migrate_nr_pages += nr_pages;
1903 return false;
1904}
1905
1906static int numamigrate_isolate_page(pg_data_t *pgdat, struct page *page) 1858static int numamigrate_isolate_page(pg_data_t *pgdat, struct page *page)
1907{ 1859{
1908 int page_lru; 1860 int page_lru;
@@ -1975,14 +1927,6 @@ int migrate_misplaced_page(struct page *page, struct vm_area_struct *vma,
1975 if (page_is_file_cache(page) && PageDirty(page)) 1927 if (page_is_file_cache(page) && PageDirty(page))
1976 goto out; 1928 goto out;
1977 1929
1978 /*
1979 * Rate-limit the amount of data that is being migrated to a node.
1980 * Optimal placement is no good if the memory bus is saturated and
1981 * all the time is being spent migrating!
1982 */
1983 if (numamigrate_update_ratelimit(pgdat, 1))
1984 goto out;
1985
1986 isolated = numamigrate_isolate_page(pgdat, page); 1930 isolated = numamigrate_isolate_page(pgdat, page);
1987 if (!isolated) 1931 if (!isolated)
1988 goto out; 1932 goto out;
@@ -2029,14 +1973,6 @@ int migrate_misplaced_transhuge_page(struct mm_struct *mm,
2029 unsigned long mmun_start = address & HPAGE_PMD_MASK; 1973 unsigned long mmun_start = address & HPAGE_PMD_MASK;
2030 unsigned long mmun_end = mmun_start + HPAGE_PMD_SIZE; 1974 unsigned long mmun_end = mmun_start + HPAGE_PMD_SIZE;
2031 1975
2032 /*
2033 * Rate-limit the amount of data that is being migrated to a node.
2034 * Optimal placement is no good if the memory bus is saturated and
2035 * all the time is being spent migrating!
2036 */
2037 if (numamigrate_update_ratelimit(pgdat, HPAGE_PMD_NR))
2038 goto out_dropref;
2039
2040 new_page = alloc_pages_node(node, 1976 new_page = alloc_pages_node(node,
2041 (GFP_TRANSHUGE_LIGHT | __GFP_THISNODE), 1977 (GFP_TRANSHUGE_LIGHT | __GFP_THISNODE),
2042 HPAGE_PMD_ORDER); 1978 HPAGE_PMD_ORDER);
@@ -2133,7 +2069,6 @@ int migrate_misplaced_transhuge_page(struct mm_struct *mm,
2133 2069
2134out_fail: 2070out_fail:
2135 count_vm_events(PGMIGRATE_FAIL, HPAGE_PMD_NR); 2071 count_vm_events(PGMIGRATE_FAIL, HPAGE_PMD_NR);
2136out_dropref:
2137 ptl = pmd_lock(mm, pmd); 2072 ptl = pmd_lock(mm, pmd);
2138 if (pmd_same(*pmd, entry)) { 2073 if (pmd_same(*pmd, entry)) {
2139 entry = pmd_modify(entry, vma->vm_page_prot); 2074 entry = pmd_modify(entry, vma->vm_page_prot);
diff --git a/mm/page_alloc.c b/mm/page_alloc.c
index 89d2a2ab3fe6..706a738c0aee 100644
--- a/mm/page_alloc.c
+++ b/mm/page_alloc.c
@@ -6197,8 +6197,6 @@ static unsigned long __init calc_memmap_size(unsigned long spanned_pages,
6197static void pgdat_init_numabalancing(struct pglist_data *pgdat) 6197static void pgdat_init_numabalancing(struct pglist_data *pgdat)
6198{ 6198{
6199 spin_lock_init(&pgdat->numabalancing_migrate_lock); 6199 spin_lock_init(&pgdat->numabalancing_migrate_lock);
6200 pgdat->numabalancing_migrate_nr_pages = 0;
6201 pgdat->numabalancing_migrate_next_window = jiffies;
6202} 6200}
6203#else 6201#else
6204static void pgdat_init_numabalancing(struct pglist_data *pgdat) {} 6202static void pgdat_init_numabalancing(struct pglist_data *pgdat) {}