diff options
Diffstat (limited to 'mm/percpu.c')
-rw-r--r-- | mm/percpu.c | 90 |
1 files changed, 45 insertions, 45 deletions
diff --git a/mm/percpu.c b/mm/percpu.c index 21d938a10662..639fce4d2caf 100644 --- a/mm/percpu.c +++ b/mm/percpu.c | |||
@@ -181,12 +181,6 @@ static int pcpu_page_idx(unsigned int cpu, int page_idx) | |||
181 | return cpu * pcpu_unit_pages + page_idx; | 181 | return cpu * pcpu_unit_pages + page_idx; |
182 | } | 182 | } |
183 | 183 | ||
184 | static struct page **pcpu_chunk_pagep(struct pcpu_chunk *chunk, | ||
185 | unsigned int cpu, int page_idx) | ||
186 | { | ||
187 | return &chunk->page[pcpu_page_idx(cpu, page_idx)]; | ||
188 | } | ||
189 | |||
190 | static unsigned long pcpu_chunk_addr(struct pcpu_chunk *chunk, | 184 | static unsigned long pcpu_chunk_addr(struct pcpu_chunk *chunk, |
191 | unsigned int cpu, int page_idx) | 185 | unsigned int cpu, int page_idx) |
192 | { | 186 | { |
@@ -194,6 +188,12 @@ static unsigned long pcpu_chunk_addr(struct pcpu_chunk *chunk, | |||
194 | (pcpu_page_idx(cpu, page_idx) << PAGE_SHIFT); | 188 | (pcpu_page_idx(cpu, page_idx) << PAGE_SHIFT); |
195 | } | 189 | } |
196 | 190 | ||
191 | static struct page **pcpu_chunk_pagep(struct pcpu_chunk *chunk, | ||
192 | unsigned int cpu, int page_idx) | ||
193 | { | ||
194 | return &chunk->page[pcpu_page_idx(cpu, page_idx)]; | ||
195 | } | ||
196 | |||
197 | static bool pcpu_chunk_page_occupied(struct pcpu_chunk *chunk, | 197 | static bool pcpu_chunk_page_occupied(struct pcpu_chunk *chunk, |
198 | int page_idx) | 198 | int page_idx) |
199 | { | 199 | { |
@@ -583,6 +583,45 @@ static void pcpu_unmap(struct pcpu_chunk *chunk, int page_start, int page_end, | |||
583 | pcpu_chunk_addr(chunk, last, page_end)); | 583 | pcpu_chunk_addr(chunk, last, page_end)); |
584 | } | 584 | } |
585 | 585 | ||
586 | static int __pcpu_map_pages(unsigned long addr, struct page **pages, | ||
587 | int nr_pages) | ||
588 | { | ||
589 | return map_kernel_range_noflush(addr, nr_pages << PAGE_SHIFT, | ||
590 | PAGE_KERNEL, pages); | ||
591 | } | ||
592 | |||
593 | /** | ||
594 | * pcpu_map - map pages into a pcpu_chunk | ||
595 | * @chunk: chunk of interest | ||
596 | * @page_start: page index of the first page to map | ||
597 | * @page_end: page index of the last page to map + 1 | ||
598 | * | ||
599 | * For each cpu, map pages [@page_start,@page_end) into @chunk. | ||
600 | * vcache is flushed afterwards. | ||
601 | */ | ||
602 | static int pcpu_map(struct pcpu_chunk *chunk, int page_start, int page_end) | ||
603 | { | ||
604 | unsigned int last = num_possible_cpus() - 1; | ||
605 | unsigned int cpu; | ||
606 | int err; | ||
607 | |||
608 | /* map must not be done on immutable chunk */ | ||
609 | WARN_ON(chunk->immutable); | ||
610 | |||
611 | for_each_possible_cpu(cpu) { | ||
612 | err = __pcpu_map_pages(pcpu_chunk_addr(chunk, cpu, page_start), | ||
613 | pcpu_chunk_pagep(chunk, cpu, page_start), | ||
614 | page_end - page_start); | ||
615 | if (err < 0) | ||
616 | return err; | ||
617 | } | ||
618 | |||
619 | /* flush at once, please read comments in pcpu_unmap() */ | ||
620 | flush_cache_vmap(pcpu_chunk_addr(chunk, 0, page_start), | ||
621 | pcpu_chunk_addr(chunk, last, page_end)); | ||
622 | return 0; | ||
623 | } | ||
624 | |||
586 | /** | 625 | /** |
587 | * pcpu_depopulate_chunk - depopulate and unmap an area of a pcpu_chunk | 626 | * pcpu_depopulate_chunk - depopulate and unmap an area of a pcpu_chunk |
588 | * @chunk: chunk to depopulate | 627 | * @chunk: chunk to depopulate |
@@ -632,45 +671,6 @@ static void pcpu_depopulate_chunk(struct pcpu_chunk *chunk, int off, int size, | |||
632 | pcpu_unmap(chunk, unmap_start, unmap_end, flush); | 671 | pcpu_unmap(chunk, unmap_start, unmap_end, flush); |
633 | } | 672 | } |
634 | 673 | ||
635 | static int __pcpu_map_pages(unsigned long addr, struct page **pages, | ||
636 | int nr_pages) | ||
637 | { | ||
638 | return map_kernel_range_noflush(addr, nr_pages << PAGE_SHIFT, | ||
639 | PAGE_KERNEL, pages); | ||
640 | } | ||
641 | |||
642 | /** | ||
643 | * pcpu_map - map pages into a pcpu_chunk | ||
644 | * @chunk: chunk of interest | ||
645 | * @page_start: page index of the first page to map | ||
646 | * @page_end: page index of the last page to map + 1 | ||
647 | * | ||
648 | * For each cpu, map pages [@page_start,@page_end) into @chunk. | ||
649 | * vcache is flushed afterwards. | ||
650 | */ | ||
651 | static int pcpu_map(struct pcpu_chunk *chunk, int page_start, int page_end) | ||
652 | { | ||
653 | unsigned int last = num_possible_cpus() - 1; | ||
654 | unsigned int cpu; | ||
655 | int err; | ||
656 | |||
657 | /* map must not be done on immutable chunk */ | ||
658 | WARN_ON(chunk->immutable); | ||
659 | |||
660 | for_each_possible_cpu(cpu) { | ||
661 | err = __pcpu_map_pages(pcpu_chunk_addr(chunk, cpu, page_start), | ||
662 | pcpu_chunk_pagep(chunk, cpu, page_start), | ||
663 | page_end - page_start); | ||
664 | if (err < 0) | ||
665 | return err; | ||
666 | } | ||
667 | |||
668 | /* flush at once, please read comments in pcpu_unmap() */ | ||
669 | flush_cache_vmap(pcpu_chunk_addr(chunk, 0, page_start), | ||
670 | pcpu_chunk_addr(chunk, last, page_end)); | ||
671 | return 0; | ||
672 | } | ||
673 | |||
674 | /** | 674 | /** |
675 | * pcpu_populate_chunk - populate and map an area of a pcpu_chunk | 675 | * pcpu_populate_chunk - populate and map an area of a pcpu_chunk |
676 | * @chunk: chunk of interest | 676 | * @chunk: chunk of interest |