summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
-rw-r--r--mm/percpu-internal.h1
-rw-r--r--mm/percpu-stats.c1
-rw-r--r--mm/percpu.c11
3 files changed, 10 insertions, 3 deletions
diff --git a/mm/percpu-internal.h b/mm/percpu-internal.h
index 34cb9799e324..c4c8fc49780b 100644
--- a/mm/percpu-internal.h
+++ b/mm/percpu-internal.h
@@ -32,6 +32,7 @@ struct pcpu_chunk {
32 32
33 int nr_pages; /* # of pages served by this chunk */ 33 int nr_pages; /* # of pages served by this chunk */
34 int nr_populated; /* # of populated pages */ 34 int nr_populated; /* # of populated pages */
35 int nr_empty_pop_pages; /* # of empty populated pages */
35 unsigned long populated[]; /* populated bitmap */ 36 unsigned long populated[]; /* populated bitmap */
36}; 37};
37 38
diff --git a/mm/percpu-stats.c b/mm/percpu-stats.c
index ffbdb96cdbeb..e146b585fd18 100644
--- a/mm/percpu-stats.c
+++ b/mm/percpu-stats.c
@@ -100,6 +100,7 @@ static void chunk_map_stats(struct seq_file *m, struct pcpu_chunk *chunk,
100 100
101 P("nr_alloc", chunk->nr_alloc); 101 P("nr_alloc", chunk->nr_alloc);
102 P("max_alloc_size", chunk->max_alloc_size); 102 P("max_alloc_size", chunk->max_alloc_size);
103 P("empty_pop_pages", chunk->nr_empty_pop_pages);
103 P("free_size", chunk->free_size); 104 P("free_size", chunk->free_size);
104 P("contig_hint", chunk->contig_hint); 105 P("contig_hint", chunk->contig_hint);
105 P("sum_frag", sum_frag); 106 P("sum_frag", sum_frag);
diff --git a/mm/percpu.c b/mm/percpu.c
index 773dafea181e..657ab0821cf0 100644
--- a/mm/percpu.c
+++ b/mm/percpu.c
@@ -757,11 +757,14 @@ static struct pcpu_chunk * __init pcpu_alloc_first_chunk(unsigned long tmp_addr,
757 chunk->immutable = true; 757 chunk->immutable = true;
758 bitmap_fill(chunk->populated, chunk->nr_pages); 758 bitmap_fill(chunk->populated, chunk->nr_pages);
759 chunk->nr_populated = chunk->nr_pages; 759 chunk->nr_populated = chunk->nr_pages;
760 chunk->nr_empty_pop_pages = chunk->nr_pages;
760 761
761 chunk->contig_hint = chunk->free_size = map_size; 762 chunk->contig_hint = chunk->free_size = map_size;
762 763
763 if (chunk->start_offset) { 764 if (chunk->start_offset) {
764 /* hide the beginning of the bitmap */ 765 /* hide the beginning of the bitmap */
766 chunk->nr_empty_pop_pages--;
767
765 chunk->map[0] = 1; 768 chunk->map[0] = 1;
766 chunk->map[1] = chunk->start_offset; 769 chunk->map[1] = chunk->start_offset;
767 chunk->map_used = 1; 770 chunk->map_used = 1;
@@ -773,6 +776,8 @@ static struct pcpu_chunk * __init pcpu_alloc_first_chunk(unsigned long tmp_addr,
773 776
774 if (chunk->end_offset) { 777 if (chunk->end_offset) {
775 /* hide the end of the bitmap */ 778 /* hide the end of the bitmap */
779 chunk->nr_empty_pop_pages--;
780
776 chunk->map[++chunk->map_used] = region_size | 1; 781 chunk->map[++chunk->map_used] = region_size | 1;
777 } 782 }
778 783
@@ -836,6 +841,7 @@ static void pcpu_chunk_populated(struct pcpu_chunk *chunk,
836 841
837 bitmap_set(chunk->populated, page_start, nr); 842 bitmap_set(chunk->populated, page_start, nr);
838 chunk->nr_populated += nr; 843 chunk->nr_populated += nr;
844 chunk->nr_empty_pop_pages += nr;
839 pcpu_nr_empty_pop_pages += nr; 845 pcpu_nr_empty_pop_pages += nr;
840} 846}
841 847
@@ -858,6 +864,7 @@ static void pcpu_chunk_depopulated(struct pcpu_chunk *chunk,
858 864
859 bitmap_clear(chunk->populated, page_start, nr); 865 bitmap_clear(chunk->populated, page_start, nr);
860 chunk->nr_populated -= nr; 866 chunk->nr_populated -= nr;
867 chunk->nr_empty_pop_pages -= nr;
861 pcpu_nr_empty_pop_pages -= nr; 868 pcpu_nr_empty_pop_pages -= nr;
862} 869}
863 870
@@ -1782,9 +1789,7 @@ int __init pcpu_setup_first_chunk(const struct pcpu_alloc_info *ai,
1782 1789
1783 /* link the first chunk in */ 1790 /* link the first chunk in */
1784 pcpu_first_chunk = chunk; 1791 pcpu_first_chunk = chunk;
1785 i = (pcpu_first_chunk->start_offset) ? 1 : 0; 1792 pcpu_nr_empty_pop_pages = pcpu_first_chunk->nr_empty_pop_pages;
1786 pcpu_nr_empty_pop_pages +=
1787 pcpu_count_occupied_pages(pcpu_first_chunk, i);
1788 pcpu_chunk_relocate(pcpu_first_chunk, -1); 1793 pcpu_chunk_relocate(pcpu_first_chunk, -1);
1789 1794
1790 pcpu_stats_chunk_alloc(); 1795 pcpu_stats_chunk_alloc();