diff options
| -rw-r--r-- | include/linux/mmzone.h | 6 | ||||
| -rw-r--r-- | include/trace/events/migrate.h | 27 | ||||
| -rw-r--r-- | mm/migrate.c | 65 | ||||
| -rw-r--r-- | mm/page_alloc.c | 2 |
4 files changed, 0 insertions, 100 deletions
diff --git a/include/linux/mmzone.h b/include/linux/mmzone.h index 1e22d96734e0..3f4c0b167333 100644 --- a/include/linux/mmzone.h +++ b/include/linux/mmzone.h | |||
| @@ -671,12 +671,6 @@ typedef struct pglist_data { | |||
| 671 | #ifdef CONFIG_NUMA_BALANCING | 671 | #ifdef CONFIG_NUMA_BALANCING |
| 672 | /* Lock serializing the migrate rate limiting window */ | 672 | /* Lock serializing the migrate rate limiting window */ |
| 673 | spinlock_t numabalancing_migrate_lock; | 673 | spinlock_t numabalancing_migrate_lock; |
| 674 | |||
| 675 | /* Rate limiting time interval */ | ||
| 676 | unsigned long numabalancing_migrate_next_window; | ||
| 677 | |||
| 678 | /* Number of pages migrated during the rate limiting time interval */ | ||
| 679 | unsigned long numabalancing_migrate_nr_pages; | ||
| 680 | #endif | 674 | #endif |
| 681 | /* | 675 | /* |
| 682 | * This is a per-node reserve of pages that are not available | 676 | * This is a per-node reserve of pages that are not available |
diff --git a/include/trace/events/migrate.h b/include/trace/events/migrate.h index 711372845945..705b33d1e395 100644 --- a/include/trace/events/migrate.h +++ b/include/trace/events/migrate.h | |||
| @@ -70,33 +70,6 @@ TRACE_EVENT(mm_migrate_pages, | |||
| 70 | __print_symbolic(__entry->mode, MIGRATE_MODE), | 70 | __print_symbolic(__entry->mode, MIGRATE_MODE), |
| 71 | __print_symbolic(__entry->reason, MIGRATE_REASON)) | 71 | __print_symbolic(__entry->reason, MIGRATE_REASON)) |
| 72 | ); | 72 | ); |
| 73 | |||
| 74 | TRACE_EVENT(mm_numa_migrate_ratelimit, | ||
| 75 | |||
| 76 | TP_PROTO(struct task_struct *p, int dst_nid, unsigned long nr_pages), | ||
| 77 | |||
| 78 | TP_ARGS(p, dst_nid, nr_pages), | ||
| 79 | |||
| 80 | TP_STRUCT__entry( | ||
| 81 | __array( char, comm, TASK_COMM_LEN) | ||
| 82 | __field( pid_t, pid) | ||
| 83 | __field( int, dst_nid) | ||
| 84 | __field( unsigned long, nr_pages) | ||
| 85 | ), | ||
| 86 | |||
| 87 | TP_fast_assign( | ||
| 88 | memcpy(__entry->comm, p->comm, TASK_COMM_LEN); | ||
| 89 | __entry->pid = p->pid; | ||
| 90 | __entry->dst_nid = dst_nid; | ||
| 91 | __entry->nr_pages = nr_pages; | ||
| 92 | ), | ||
| 93 | |||
| 94 | TP_printk("comm=%s pid=%d dst_nid=%d nr_pages=%lu", | ||
| 95 | __entry->comm, | ||
| 96 | __entry->pid, | ||
| 97 | __entry->dst_nid, | ||
| 98 | __entry->nr_pages) | ||
| 99 | ); | ||
| 100 | #endif /* _TRACE_MIGRATE_H */ | 73 | #endif /* _TRACE_MIGRATE_H */ |
| 101 | 74 | ||
| 102 | /* This part must be outside protection */ | 75 | /* This part must be outside protection */ |
diff --git a/mm/migrate.c b/mm/migrate.c index 4f1d894835b5..5e285c1249a0 100644 --- a/mm/migrate.c +++ b/mm/migrate.c | |||
| @@ -1855,54 +1855,6 @@ static struct page *alloc_misplaced_dst_page(struct page *page, | |||
| 1855 | return newpage; | 1855 | return newpage; |
| 1856 | } | 1856 | } |
| 1857 | 1857 | ||
| 1858 | /* | ||
| 1859 | * page migration rate limiting control. | ||
| 1860 | * Do not migrate more than @pages_to_migrate in a @migrate_interval_millisecs | ||
| 1861 | * window of time. Default here says do not migrate more than 1280M per second. | ||
| 1862 | */ | ||
| 1863 | static unsigned int migrate_interval_millisecs __read_mostly = 100; | ||
| 1864 | static unsigned int ratelimit_pages __read_mostly = 128 << (20 - PAGE_SHIFT); | ||
| 1865 | |||
| 1866 | /* Returns true if the node is migrate rate-limited after the update */ | ||
| 1867 | static bool numamigrate_update_ratelimit(pg_data_t *pgdat, | ||
| 1868 | unsigned long nr_pages) | ||
| 1869 | { | ||
| 1870 | unsigned long next_window, interval; | ||
| 1871 | |||
| 1872 | next_window = READ_ONCE(pgdat->numabalancing_migrate_next_window); | ||
| 1873 | interval = msecs_to_jiffies(migrate_interval_millisecs); | ||
| 1874 | |||
| 1875 | /* | ||
| 1876 | * Rate-limit the amount of data that is being migrated to a node. | ||
| 1877 | * Optimal placement is no good if the memory bus is saturated and | ||
| 1878 | * all the time is being spent migrating! | ||
| 1879 | */ | ||
| 1880 | if (time_after(jiffies, next_window) && | ||
| 1881 | spin_trylock(&pgdat->numabalancing_migrate_lock)) { | ||
| 1882 | pgdat->numabalancing_migrate_nr_pages = 0; | ||
| 1883 | do { | ||
| 1884 | next_window += interval; | ||
| 1885 | } while (unlikely(time_after(jiffies, next_window))); | ||
| 1886 | |||
| 1887 | WRITE_ONCE(pgdat->numabalancing_migrate_next_window, next_window); | ||
| 1888 | spin_unlock(&pgdat->numabalancing_migrate_lock); | ||
| 1889 | } | ||
| 1890 | if (pgdat->numabalancing_migrate_nr_pages > ratelimit_pages) { | ||
| 1891 | trace_mm_numa_migrate_ratelimit(current, pgdat->node_id, | ||
| 1892 | nr_pages); | ||
| 1893 | return true; | ||
| 1894 | } | ||
| 1895 | |||
| 1896 | /* | ||
| 1897 | * This is an unlocked non-atomic update so errors are possible. | ||
| 1898 | * The consequences are failing to migrate when we potentiall should | ||
| 1899 | * have which is not severe enough to warrant locking. If it is ever | ||
| 1900 | * a problem, it can be converted to a per-cpu counter. | ||
| 1901 | */ | ||
| 1902 | pgdat->numabalancing_migrate_nr_pages += nr_pages; | ||
| 1903 | return false; | ||
| 1904 | } | ||
| 1905 | |||
| 1906 | static int numamigrate_isolate_page(pg_data_t *pgdat, struct page *page) | 1858 | static int numamigrate_isolate_page(pg_data_t *pgdat, struct page *page) |
| 1907 | { | 1859 | { |
| 1908 | int page_lru; | 1860 | int page_lru; |
| @@ -1975,14 +1927,6 @@ int migrate_misplaced_page(struct page *page, struct vm_area_struct *vma, | |||
| 1975 | if (page_is_file_cache(page) && PageDirty(page)) | 1927 | if (page_is_file_cache(page) && PageDirty(page)) |
| 1976 | goto out; | 1928 | goto out; |
| 1977 | 1929 | ||
| 1978 | /* | ||
| 1979 | * Rate-limit the amount of data that is being migrated to a node. | ||
| 1980 | * Optimal placement is no good if the memory bus is saturated and | ||
| 1981 | * all the time is being spent migrating! | ||
| 1982 | */ | ||
| 1983 | if (numamigrate_update_ratelimit(pgdat, 1)) | ||
| 1984 | goto out; | ||
| 1985 | |||
| 1986 | isolated = numamigrate_isolate_page(pgdat, page); | 1930 | isolated = numamigrate_isolate_page(pgdat, page); |
| 1987 | if (!isolated) | 1931 | if (!isolated) |
| 1988 | goto out; | 1932 | goto out; |
| @@ -2029,14 +1973,6 @@ int migrate_misplaced_transhuge_page(struct mm_struct *mm, | |||
| 2029 | unsigned long mmun_start = address & HPAGE_PMD_MASK; | 1973 | unsigned long mmun_start = address & HPAGE_PMD_MASK; |
| 2030 | unsigned long mmun_end = mmun_start + HPAGE_PMD_SIZE; | 1974 | unsigned long mmun_end = mmun_start + HPAGE_PMD_SIZE; |
| 2031 | 1975 | ||
| 2032 | /* | ||
| 2033 | * Rate-limit the amount of data that is being migrated to a node. | ||
| 2034 | * Optimal placement is no good if the memory bus is saturated and | ||
| 2035 | * all the time is being spent migrating! | ||
| 2036 | */ | ||
| 2037 | if (numamigrate_update_ratelimit(pgdat, HPAGE_PMD_NR)) | ||
| 2038 | goto out_dropref; | ||
| 2039 | |||
| 2040 | new_page = alloc_pages_node(node, | 1976 | new_page = alloc_pages_node(node, |
| 2041 | (GFP_TRANSHUGE_LIGHT | __GFP_THISNODE), | 1977 | (GFP_TRANSHUGE_LIGHT | __GFP_THISNODE), |
| 2042 | HPAGE_PMD_ORDER); | 1978 | HPAGE_PMD_ORDER); |
| @@ -2133,7 +2069,6 @@ int migrate_misplaced_transhuge_page(struct mm_struct *mm, | |||
| 2133 | 2069 | ||
| 2134 | out_fail: | 2070 | out_fail: |
| 2135 | count_vm_events(PGMIGRATE_FAIL, HPAGE_PMD_NR); | 2071 | count_vm_events(PGMIGRATE_FAIL, HPAGE_PMD_NR); |
| 2136 | out_dropref: | ||
| 2137 | ptl = pmd_lock(mm, pmd); | 2072 | ptl = pmd_lock(mm, pmd); |
| 2138 | if (pmd_same(*pmd, entry)) { | 2073 | if (pmd_same(*pmd, entry)) { |
| 2139 | entry = pmd_modify(entry, vma->vm_page_prot); | 2074 | entry = pmd_modify(entry, vma->vm_page_prot); |
diff --git a/mm/page_alloc.c b/mm/page_alloc.c index 89d2a2ab3fe6..706a738c0aee 100644 --- a/mm/page_alloc.c +++ b/mm/page_alloc.c | |||
| @@ -6197,8 +6197,6 @@ static unsigned long __init calc_memmap_size(unsigned long spanned_pages, | |||
| 6197 | static void pgdat_init_numabalancing(struct pglist_data *pgdat) | 6197 | static void pgdat_init_numabalancing(struct pglist_data *pgdat) |
| 6198 | { | 6198 | { |
| 6199 | spin_lock_init(&pgdat->numabalancing_migrate_lock); | 6199 | spin_lock_init(&pgdat->numabalancing_migrate_lock); |
| 6200 | pgdat->numabalancing_migrate_nr_pages = 0; | ||
| 6201 | pgdat->numabalancing_migrate_next_window = jiffies; | ||
| 6202 | } | 6200 | } |
| 6203 | #else | 6201 | #else |
| 6204 | static void pgdat_init_numabalancing(struct pglist_data *pgdat) {} | 6202 | static void pgdat_init_numabalancing(struct pglist_data *pgdat) {} |
