diff options
author | Linus Torvalds <torvalds@linux-foundation.org> | 2014-08-07 00:14:42 -0400 |
---|---|---|
committer | Linus Torvalds <torvalds@linux-foundation.org> | 2014-08-07 00:14:42 -0400 |
commit | 33caee39925b887a99a2400dc5c980097c3573f9 (patch) | |
tree | 8e68ad97e1fee88c4a3f31453041f8d139f2027e | |
parent | 6456a0438b984186a0c9c8ecc9fe3d97b7ac3613 (diff) | |
parent | f84223087402c45179be5e7060c5736c17a7b271 (diff) |
Merge branch 'akpm' (patchbomb from Andrew Morton)
Merge incoming from Andrew Morton:
- Various misc things.
- arch/sh updates.
- Part of ocfs2. Review is slow.
- Slab updates.
- Most of -mm.
- printk updates.
- lib/ updates.
- checkpatch updates.
* emailed patches from Andrew Morton <akpm@linux-foundation.org>: (226 commits)
checkpatch: update $declaration_macros, add uninitialized_var
checkpatch: warn on missing spaces in broken up quoted
checkpatch: fix false positives for --strict "space after cast" test
checkpatch: fix false positive MISSING_BREAK warnings with --file
checkpatch: add test for native c90 types in unusual order
checkpatch: add signed generic types
checkpatch: add short int to c variable types
checkpatch: add for_each tests to indentation and brace tests
checkpatch: fix brace style misuses of else and while
checkpatch: add --fix option for a couple OPEN_BRACE misuses
checkpatch: use the correct indentation for which()
checkpatch: add fix_insert_line and fix_delete_line helpers
checkpatch: add ability to insert and delete lines to patch/file
checkpatch: add an index variable for fixed lines
checkpatch: warn on break after goto or return with same tab indentation
checkpatch: emit a warning on file add/move/delete
checkpatch: add test for commit id formatting style in commit log
checkpatch: emit fewer kmalloc_array/kcalloc conversion warnings
checkpatch: improve "no space after cast" test
checkpatch: allow multiple const * types
...
156 files changed, 3930 insertions, 2919 deletions
diff --git a/Documentation/RCU/whatisRCU.txt b/Documentation/RCU/whatisRCU.txt index 49b8551a3b68..e48c57f1943b 100644 --- a/Documentation/RCU/whatisRCU.txt +++ b/Documentation/RCU/whatisRCU.txt | |||
@@ -818,7 +818,7 @@ RCU pointer/list update: | |||
818 | list_add_tail_rcu | 818 | list_add_tail_rcu |
819 | list_del_rcu | 819 | list_del_rcu |
820 | list_replace_rcu | 820 | list_replace_rcu |
821 | hlist_add_after_rcu | 821 | hlist_add_behind_rcu |
822 | hlist_add_before_rcu | 822 | hlist_add_before_rcu |
823 | hlist_add_head_rcu | 823 | hlist_add_head_rcu |
824 | hlist_del_rcu | 824 | hlist_del_rcu |
diff --git a/Documentation/kernel-parameters.txt b/Documentation/kernel-parameters.txt index 883901b9ac4f..9344d833b7ea 100644 --- a/Documentation/kernel-parameters.txt +++ b/Documentation/kernel-parameters.txt | |||
@@ -1716,8 +1716,12 @@ bytes respectively. Such letter suffixes can also be entirely omitted. | |||
1716 | 7 (KERN_DEBUG) debug-level messages | 1716 | 7 (KERN_DEBUG) debug-level messages |
1717 | 1717 | ||
1718 | log_buf_len=n[KMG] Sets the size of the printk ring buffer, | 1718 | log_buf_len=n[KMG] Sets the size of the printk ring buffer, |
1719 | in bytes. n must be a power of two. The default | 1719 | in bytes. n must be a power of two and greater |
1720 | size is set in the kernel config file. | 1720 | than the minimal size. The minimal size is defined |
1721 | by LOG_BUF_SHIFT kernel config parameter. There is | ||
1722 | also CONFIG_LOG_CPU_MAX_BUF_SHIFT config parameter | ||
1723 | that allows to increase the default size depending on | ||
1724 | the number of CPUs. See init/Kconfig for more details. | ||
1721 | 1725 | ||
1722 | logo.nologo [FB] Disables display of the built-in Linux logo. | 1726 | logo.nologo [FB] Disables display of the built-in Linux logo. |
1723 | This may be used to provide more screen space for | 1727 | This may be used to provide more screen space for |
diff --git a/Documentation/trace/postprocess/trace-vmscan-postprocess.pl b/Documentation/trace/postprocess/trace-vmscan-postprocess.pl index 78c9a7b2b58f..8f961ef2b457 100644 --- a/Documentation/trace/postprocess/trace-vmscan-postprocess.pl +++ b/Documentation/trace/postprocess/trace-vmscan-postprocess.pl | |||
@@ -47,6 +47,10 @@ use constant HIGH_KSWAPD_REWAKEUP => 21; | |||
47 | use constant HIGH_NR_SCANNED => 22; | 47 | use constant HIGH_NR_SCANNED => 22; |
48 | use constant HIGH_NR_TAKEN => 23; | 48 | use constant HIGH_NR_TAKEN => 23; |
49 | use constant HIGH_NR_RECLAIMED => 24; | 49 | use constant HIGH_NR_RECLAIMED => 24; |
50 | use constant HIGH_NR_FILE_SCANNED => 25; | ||
51 | use constant HIGH_NR_ANON_SCANNED => 26; | ||
52 | use constant HIGH_NR_FILE_RECLAIMED => 27; | ||
53 | use constant HIGH_NR_ANON_RECLAIMED => 28; | ||
50 | 54 | ||
51 | my %perprocesspid; | 55 | my %perprocesspid; |
52 | my %perprocess; | 56 | my %perprocess; |
@@ -56,14 +60,18 @@ my $opt_read_procstat; | |||
56 | 60 | ||
57 | my $total_wakeup_kswapd; | 61 | my $total_wakeup_kswapd; |
58 | my ($total_direct_reclaim, $total_direct_nr_scanned); | 62 | my ($total_direct_reclaim, $total_direct_nr_scanned); |
63 | my ($total_direct_nr_file_scanned, $total_direct_nr_anon_scanned); | ||
59 | my ($total_direct_latency, $total_kswapd_latency); | 64 | my ($total_direct_latency, $total_kswapd_latency); |
60 | my ($total_direct_nr_reclaimed); | 65 | my ($total_direct_nr_reclaimed); |
66 | my ($total_direct_nr_file_reclaimed, $total_direct_nr_anon_reclaimed); | ||
61 | my ($total_direct_writepage_file_sync, $total_direct_writepage_file_async); | 67 | my ($total_direct_writepage_file_sync, $total_direct_writepage_file_async); |
62 | my ($total_direct_writepage_anon_sync, $total_direct_writepage_anon_async); | 68 | my ($total_direct_writepage_anon_sync, $total_direct_writepage_anon_async); |
63 | my ($total_kswapd_nr_scanned, $total_kswapd_wake); | 69 | my ($total_kswapd_nr_scanned, $total_kswapd_wake); |
70 | my ($total_kswapd_nr_file_scanned, $total_kswapd_nr_anon_scanned); | ||
64 | my ($total_kswapd_writepage_file_sync, $total_kswapd_writepage_file_async); | 71 | my ($total_kswapd_writepage_file_sync, $total_kswapd_writepage_file_async); |
65 | my ($total_kswapd_writepage_anon_sync, $total_kswapd_writepage_anon_async); | 72 | my ($total_kswapd_writepage_anon_sync, $total_kswapd_writepage_anon_async); |
66 | my ($total_kswapd_nr_reclaimed); | 73 | my ($total_kswapd_nr_reclaimed); |
74 | my ($total_kswapd_nr_file_reclaimed, $total_kswapd_nr_anon_reclaimed); | ||
67 | 75 | ||
68 | # Catch sigint and exit on request | 76 | # Catch sigint and exit on request |
69 | my $sigint_report = 0; | 77 | my $sigint_report = 0; |
@@ -374,6 +382,7 @@ EVENT_PROCESS: | |||
374 | } | 382 | } |
375 | my $isolate_mode = $1; | 383 | my $isolate_mode = $1; |
376 | my $nr_scanned = $4; | 384 | my $nr_scanned = $4; |
385 | my $file = $6; | ||
377 | 386 | ||
378 | # To closer match vmstat scanning statistics, only count isolate_both | 387 | # To closer match vmstat scanning statistics, only count isolate_both |
379 | # and isolate_inactive as scanning. isolate_active is rotation | 388 | # and isolate_inactive as scanning. isolate_active is rotation |
@@ -382,6 +391,11 @@ EVENT_PROCESS: | |||
382 | # isolate_both == 3 | 391 | # isolate_both == 3 |
383 | if ($isolate_mode != 2) { | 392 | if ($isolate_mode != 2) { |
384 | $perprocesspid{$process_pid}->{HIGH_NR_SCANNED} += $nr_scanned; | 393 | $perprocesspid{$process_pid}->{HIGH_NR_SCANNED} += $nr_scanned; |
394 | if ($file == 1) { | ||
395 | $perprocesspid{$process_pid}->{HIGH_NR_FILE_SCANNED} += $nr_scanned; | ||
396 | } else { | ||
397 | $perprocesspid{$process_pid}->{HIGH_NR_ANON_SCANNED} += $nr_scanned; | ||
398 | } | ||
385 | } | 399 | } |
386 | } elsif ($tracepoint eq "mm_vmscan_lru_shrink_inactive") { | 400 | } elsif ($tracepoint eq "mm_vmscan_lru_shrink_inactive") { |
387 | $details = $6; | 401 | $details = $6; |
@@ -391,8 +405,19 @@ EVENT_PROCESS: | |||
391 | print " $regex_lru_shrink_inactive/o\n"; | 405 | print " $regex_lru_shrink_inactive/o\n"; |
392 | next; | 406 | next; |
393 | } | 407 | } |
408 | |||
394 | my $nr_reclaimed = $4; | 409 | my $nr_reclaimed = $4; |
410 | my $flags = $6; | ||
411 | my $file = 0; | ||
412 | if ($flags =~ /RECLAIM_WB_FILE/) { | ||
413 | $file = 1; | ||
414 | } | ||
395 | $perprocesspid{$process_pid}->{HIGH_NR_RECLAIMED} += $nr_reclaimed; | 415 | $perprocesspid{$process_pid}->{HIGH_NR_RECLAIMED} += $nr_reclaimed; |
416 | if ($file) { | ||
417 | $perprocesspid{$process_pid}->{HIGH_NR_FILE_RECLAIMED} += $nr_reclaimed; | ||
418 | } else { | ||
419 | $perprocesspid{$process_pid}->{HIGH_NR_ANON_RECLAIMED} += $nr_reclaimed; | ||
420 | } | ||
396 | } elsif ($tracepoint eq "mm_vmscan_writepage") { | 421 | } elsif ($tracepoint eq "mm_vmscan_writepage") { |
397 | $details = $6; | 422 | $details = $6; |
398 | if ($details !~ /$regex_writepage/o) { | 423 | if ($details !~ /$regex_writepage/o) { |
@@ -493,7 +518,11 @@ sub dump_stats { | |||
493 | $total_direct_reclaim += $stats{$process_pid}->{MM_VMSCAN_DIRECT_RECLAIM_BEGIN}; | 518 | $total_direct_reclaim += $stats{$process_pid}->{MM_VMSCAN_DIRECT_RECLAIM_BEGIN}; |
494 | $total_wakeup_kswapd += $stats{$process_pid}->{MM_VMSCAN_WAKEUP_KSWAPD}; | 519 | $total_wakeup_kswapd += $stats{$process_pid}->{MM_VMSCAN_WAKEUP_KSWAPD}; |
495 | $total_direct_nr_scanned += $stats{$process_pid}->{HIGH_NR_SCANNED}; | 520 | $total_direct_nr_scanned += $stats{$process_pid}->{HIGH_NR_SCANNED}; |
521 | $total_direct_nr_file_scanned += $stats{$process_pid}->{HIGH_NR_FILE_SCANNED}; | ||
522 | $total_direct_nr_anon_scanned += $stats{$process_pid}->{HIGH_NR_ANON_SCANNED}; | ||
496 | $total_direct_nr_reclaimed += $stats{$process_pid}->{HIGH_NR_RECLAIMED}; | 523 | $total_direct_nr_reclaimed += $stats{$process_pid}->{HIGH_NR_RECLAIMED}; |
524 | $total_direct_nr_file_reclaimed += $stats{$process_pid}->{HIGH_NR_FILE_RECLAIMED}; | ||
525 | $total_direct_nr_anon_reclaimed += $stats{$process_pid}->{HIGH_NR_ANON_RECLAIMED}; | ||
497 | $total_direct_writepage_file_sync += $stats{$process_pid}->{MM_VMSCAN_WRITEPAGE_FILE_SYNC}; | 526 | $total_direct_writepage_file_sync += $stats{$process_pid}->{MM_VMSCAN_WRITEPAGE_FILE_SYNC}; |
498 | $total_direct_writepage_anon_sync += $stats{$process_pid}->{MM_VMSCAN_WRITEPAGE_ANON_SYNC}; | 527 | $total_direct_writepage_anon_sync += $stats{$process_pid}->{MM_VMSCAN_WRITEPAGE_ANON_SYNC}; |
499 | $total_direct_writepage_file_async += $stats{$process_pid}->{MM_VMSCAN_WRITEPAGE_FILE_ASYNC}; | 528 | $total_direct_writepage_file_async += $stats{$process_pid}->{MM_VMSCAN_WRITEPAGE_FILE_ASYNC}; |
@@ -513,7 +542,11 @@ sub dump_stats { | |||
513 | $stats{$process_pid}->{MM_VMSCAN_DIRECT_RECLAIM_BEGIN}, | 542 | $stats{$process_pid}->{MM_VMSCAN_DIRECT_RECLAIM_BEGIN}, |
514 | $stats{$process_pid}->{MM_VMSCAN_WAKEUP_KSWAPD}, | 543 | $stats{$process_pid}->{MM_VMSCAN_WAKEUP_KSWAPD}, |
515 | $stats{$process_pid}->{HIGH_NR_SCANNED}, | 544 | $stats{$process_pid}->{HIGH_NR_SCANNED}, |
545 | $stats{$process_pid}->{HIGH_NR_FILE_SCANNED}, | ||
546 | $stats{$process_pid}->{HIGH_NR_ANON_SCANNED}, | ||
516 | $stats{$process_pid}->{HIGH_NR_RECLAIMED}, | 547 | $stats{$process_pid}->{HIGH_NR_RECLAIMED}, |
548 | $stats{$process_pid}->{HIGH_NR_FILE_RECLAIMED}, | ||
549 | $stats{$process_pid}->{HIGH_NR_ANON_RECLAIMED}, | ||
517 | $stats{$process_pid}->{MM_VMSCAN_WRITEPAGE_FILE_SYNC} + $stats{$process_pid}->{MM_VMSCAN_WRITEPAGE_ANON_SYNC}, | 550 | $stats{$process_pid}->{MM_VMSCAN_WRITEPAGE_FILE_SYNC} + $stats{$process_pid}->{MM_VMSCAN_WRITEPAGE_ANON_SYNC}, |
518 | $stats{$process_pid}->{MM_VMSCAN_WRITEPAGE_FILE_ASYNC} + $stats{$process_pid}->{MM_VMSCAN_WRITEPAGE_ANON_ASYNC}, | 551 | $stats{$process_pid}->{MM_VMSCAN_WRITEPAGE_FILE_ASYNC} + $stats{$process_pid}->{MM_VMSCAN_WRITEPAGE_ANON_ASYNC}, |
519 | $this_reclaim_delay / 1000); | 552 | $this_reclaim_delay / 1000); |
@@ -552,7 +585,11 @@ sub dump_stats { | |||
552 | 585 | ||
553 | $total_kswapd_wake += $stats{$process_pid}->{MM_VMSCAN_KSWAPD_WAKE}; | 586 | $total_kswapd_wake += $stats{$process_pid}->{MM_VMSCAN_KSWAPD_WAKE}; |
554 | $total_kswapd_nr_scanned += $stats{$process_pid}->{HIGH_NR_SCANNED}; | 587 | $total_kswapd_nr_scanned += $stats{$process_pid}->{HIGH_NR_SCANNED}; |
588 | $total_kswapd_nr_file_scanned += $stats{$process_pid}->{HIGH_NR_FILE_SCANNED}; | ||
589 | $total_kswapd_nr_anon_scanned += $stats{$process_pid}->{HIGH_NR_ANON_SCANNED}; | ||
555 | $total_kswapd_nr_reclaimed += $stats{$process_pid}->{HIGH_NR_RECLAIMED}; | 590 | $total_kswapd_nr_reclaimed += $stats{$process_pid}->{HIGH_NR_RECLAIMED}; |
591 | $total_kswapd_nr_file_reclaimed += $stats{$process_pid}->{HIGH_NR_FILE_RECLAIMED}; | ||
592 | $total_kswapd_nr_anon_reclaimed += $stats{$process_pid}->{HIGH_NR_ANON_RECLAIMED}; | ||
556 | $total_kswapd_writepage_file_sync += $stats{$process_pid}->{MM_VMSCAN_WRITEPAGE_FILE_SYNC}; | 593 | $total_kswapd_writepage_file_sync += $stats{$process_pid}->{MM_VMSCAN_WRITEPAGE_FILE_SYNC}; |
557 | $total_kswapd_writepage_anon_sync += $stats{$process_pid}->{MM_VMSCAN_WRITEPAGE_ANON_SYNC}; | 594 | $total_kswapd_writepage_anon_sync += $stats{$process_pid}->{MM_VMSCAN_WRITEPAGE_ANON_SYNC}; |
558 | $total_kswapd_writepage_file_async += $stats{$process_pid}->{MM_VMSCAN_WRITEPAGE_FILE_ASYNC}; | 595 | $total_kswapd_writepage_file_async += $stats{$process_pid}->{MM_VMSCAN_WRITEPAGE_FILE_ASYNC}; |
@@ -563,7 +600,11 @@ sub dump_stats { | |||
563 | $stats{$process_pid}->{MM_VMSCAN_KSWAPD_WAKE}, | 600 | $stats{$process_pid}->{MM_VMSCAN_KSWAPD_WAKE}, |
564 | $stats{$process_pid}->{HIGH_KSWAPD_REWAKEUP}, | 601 | $stats{$process_pid}->{HIGH_KSWAPD_REWAKEUP}, |
565 | $stats{$process_pid}->{HIGH_NR_SCANNED}, | 602 | $stats{$process_pid}->{HIGH_NR_SCANNED}, |
603 | $stats{$process_pid}->{HIGH_NR_FILE_SCANNED}, | ||
604 | $stats{$process_pid}->{HIGH_NR_ANON_SCANNED}, | ||
566 | $stats{$process_pid}->{HIGH_NR_RECLAIMED}, | 605 | $stats{$process_pid}->{HIGH_NR_RECLAIMED}, |
606 | $stats{$process_pid}->{HIGH_NR_FILE_RECLAIMED}, | ||
607 | $stats{$process_pid}->{HIGH_NR_ANON_RECLAIMED}, | ||
567 | $stats{$process_pid}->{MM_VMSCAN_WRITEPAGE_FILE_SYNC} + $stats{$process_pid}->{MM_VMSCAN_WRITEPAGE_ANON_SYNC}, | 608 | $stats{$process_pid}->{MM_VMSCAN_WRITEPAGE_FILE_SYNC} + $stats{$process_pid}->{MM_VMSCAN_WRITEPAGE_ANON_SYNC}, |
568 | $stats{$process_pid}->{MM_VMSCAN_WRITEPAGE_FILE_ASYNC} + $stats{$process_pid}->{MM_VMSCAN_WRITEPAGE_ANON_ASYNC}); | 609 | $stats{$process_pid}->{MM_VMSCAN_WRITEPAGE_FILE_ASYNC} + $stats{$process_pid}->{MM_VMSCAN_WRITEPAGE_ANON_ASYNC}); |
569 | 610 | ||
@@ -594,7 +635,11 @@ sub dump_stats { | |||
594 | print "\nSummary\n"; | 635 | print "\nSummary\n"; |
595 | print "Direct reclaims: $total_direct_reclaim\n"; | 636 | print "Direct reclaims: $total_direct_reclaim\n"; |
596 | print "Direct reclaim pages scanned: $total_direct_nr_scanned\n"; | 637 | print "Direct reclaim pages scanned: $total_direct_nr_scanned\n"; |
638 | print "Direct reclaim file pages scanned: $total_direct_nr_file_scanned\n"; | ||
639 | print "Direct reclaim anon pages scanned: $total_direct_nr_anon_scanned\n"; | ||
597 | print "Direct reclaim pages reclaimed: $total_direct_nr_reclaimed\n"; | 640 | print "Direct reclaim pages reclaimed: $total_direct_nr_reclaimed\n"; |
641 | print "Direct reclaim file pages reclaimed: $total_direct_nr_file_reclaimed\n"; | ||
642 | print "Direct reclaim anon pages reclaimed: $total_direct_nr_anon_reclaimed\n"; | ||
598 | print "Direct reclaim write file sync I/O: $total_direct_writepage_file_sync\n"; | 643 | print "Direct reclaim write file sync I/O: $total_direct_writepage_file_sync\n"; |
599 | print "Direct reclaim write anon sync I/O: $total_direct_writepage_anon_sync\n"; | 644 | print "Direct reclaim write anon sync I/O: $total_direct_writepage_anon_sync\n"; |
600 | print "Direct reclaim write file async I/O: $total_direct_writepage_file_async\n"; | 645 | print "Direct reclaim write file async I/O: $total_direct_writepage_file_async\n"; |
@@ -604,7 +649,11 @@ sub dump_stats { | |||
604 | print "\n"; | 649 | print "\n"; |
605 | print "Kswapd wakeups: $total_kswapd_wake\n"; | 650 | print "Kswapd wakeups: $total_kswapd_wake\n"; |
606 | print "Kswapd pages scanned: $total_kswapd_nr_scanned\n"; | 651 | print "Kswapd pages scanned: $total_kswapd_nr_scanned\n"; |
652 | print "Kswapd file pages scanned: $total_kswapd_nr_file_scanned\n"; | ||
653 | print "Kswapd anon pages scanned: $total_kswapd_nr_anon_scanned\n"; | ||
607 | print "Kswapd pages reclaimed: $total_kswapd_nr_reclaimed\n"; | 654 | print "Kswapd pages reclaimed: $total_kswapd_nr_reclaimed\n"; |
655 | print "Kswapd file pages reclaimed: $total_kswapd_nr_file_reclaimed\n"; | ||
656 | print "Kswapd anon pages reclaimed: $total_kswapd_nr_anon_reclaimed\n"; | ||
608 | print "Kswapd reclaim write file sync I/O: $total_kswapd_writepage_file_sync\n"; | 657 | print "Kswapd reclaim write file sync I/O: $total_kswapd_writepage_file_sync\n"; |
609 | print "Kswapd reclaim write anon sync I/O: $total_kswapd_writepage_anon_sync\n"; | 658 | print "Kswapd reclaim write anon sync I/O: $total_kswapd_writepage_anon_sync\n"; |
610 | print "Kswapd reclaim write file async I/O: $total_kswapd_writepage_file_async\n"; | 659 | print "Kswapd reclaim write file async I/O: $total_kswapd_writepage_file_async\n"; |
@@ -629,7 +678,11 @@ sub aggregate_perprocesspid() { | |||
629 | $perprocess{$process}->{MM_VMSCAN_WAKEUP_KSWAPD} += $perprocesspid{$process_pid}->{MM_VMSCAN_WAKEUP_KSWAPD}; | 678 | $perprocess{$process}->{MM_VMSCAN_WAKEUP_KSWAPD} += $perprocesspid{$process_pid}->{MM_VMSCAN_WAKEUP_KSWAPD}; |
630 | $perprocess{$process}->{HIGH_KSWAPD_REWAKEUP} += $perprocesspid{$process_pid}->{HIGH_KSWAPD_REWAKEUP}; | 679 | $perprocess{$process}->{HIGH_KSWAPD_REWAKEUP} += $perprocesspid{$process_pid}->{HIGH_KSWAPD_REWAKEUP}; |
631 | $perprocess{$process}->{HIGH_NR_SCANNED} += $perprocesspid{$process_pid}->{HIGH_NR_SCANNED}; | 680 | $perprocess{$process}->{HIGH_NR_SCANNED} += $perprocesspid{$process_pid}->{HIGH_NR_SCANNED}; |
681 | $perprocess{$process}->{HIGH_NR_FILE_SCANNED} += $perprocesspid{$process_pid}->{HIGH_NR_FILE_SCANNED}; | ||
682 | $perprocess{$process}->{HIGH_NR_ANON_SCANNED} += $perprocesspid{$process_pid}->{HIGH_NR_ANON_SCANNED}; | ||
632 | $perprocess{$process}->{HIGH_NR_RECLAIMED} += $perprocesspid{$process_pid}->{HIGH_NR_RECLAIMED}; | 683 | $perprocess{$process}->{HIGH_NR_RECLAIMED} += $perprocesspid{$process_pid}->{HIGH_NR_RECLAIMED}; |
684 | $perprocess{$process}->{HIGH_NR_FILE_RECLAIMED} += $perprocesspid{$process_pid}->{HIGH_NR_FILE_RECLAIMED}; | ||
685 | $perprocess{$process}->{HIGH_NR_ANON_RECLAIMED} += $perprocesspid{$process_pid}->{HIGH_NR_ANON_RECLAIMED}; | ||
633 | $perprocess{$process}->{MM_VMSCAN_WRITEPAGE_FILE_SYNC} += $perprocesspid{$process_pid}->{MM_VMSCAN_WRITEPAGE_FILE_SYNC}; | 686 | $perprocess{$process}->{MM_VMSCAN_WRITEPAGE_FILE_SYNC} += $perprocesspid{$process_pid}->{MM_VMSCAN_WRITEPAGE_FILE_SYNC}; |
634 | $perprocess{$process}->{MM_VMSCAN_WRITEPAGE_ANON_SYNC} += $perprocesspid{$process_pid}->{MM_VMSCAN_WRITEPAGE_ANON_SYNC}; | 687 | $perprocess{$process}->{MM_VMSCAN_WRITEPAGE_ANON_SYNC} += $perprocesspid{$process_pid}->{MM_VMSCAN_WRITEPAGE_ANON_SYNC}; |
635 | $perprocess{$process}->{MM_VMSCAN_WRITEPAGE_FILE_ASYNC} += $perprocesspid{$process_pid}->{MM_VMSCAN_WRITEPAGE_FILE_ASYNC}; | 688 | $perprocess{$process}->{MM_VMSCAN_WRITEPAGE_FILE_ASYNC} += $perprocesspid{$process_pid}->{MM_VMSCAN_WRITEPAGE_FILE_ASYNC}; |
@@ -621,6 +621,9 @@ else | |||
621 | KBUILD_CFLAGS += -O2 | 621 | KBUILD_CFLAGS += -O2 |
622 | endif | 622 | endif |
623 | 623 | ||
624 | # Tell gcc to never replace conditional load with a non-conditional one | ||
625 | KBUILD_CFLAGS += $(call cc-option,--param=allow-store-data-races=0) | ||
626 | |||
624 | ifdef CONFIG_READABLE_ASM | 627 | ifdef CONFIG_READABLE_ASM |
625 | # Disable optimizations that make assembler listings hard to read. | 628 | # Disable optimizations that make assembler listings hard to read. |
626 | # reorder blocks reorders the control in the function | 629 | # reorder blocks reorders the control in the function |
@@ -636,6 +639,22 @@ KBUILD_CFLAGS += $(call cc-option,-Wframe-larger-than=${CONFIG_FRAME_WARN}) | |||
636 | endif | 639 | endif |
637 | 640 | ||
638 | # Handle stack protector mode. | 641 | # Handle stack protector mode. |
642 | # | ||
643 | # Since kbuild can potentially perform two passes (first with the old | ||
644 | # .config values and then with updated .config values), we cannot error out | ||
645 | # if a desired compiler option is unsupported. If we were to error, kbuild | ||
646 | # could never get to the second pass and actually notice that we changed | ||
647 | # the option to something that was supported. | ||
648 | # | ||
649 | # Additionally, we don't want to fallback and/or silently change which compiler | ||
650 | # flags will be used, since that leads to producing kernels with different | ||
651 | # security feature characteristics depending on the compiler used. ("But I | ||
652 | # selected CC_STACKPROTECTOR_STRONG! Why did it build with _REGULAR?!") | ||
653 | # | ||
654 | # The middle ground is to warn here so that the failed option is obvious, but | ||
655 | # to let the build fail with bad compiler flags so that we can't produce a | ||
656 | # kernel when there is a CONFIG and compiler mismatch. | ||
657 | # | ||
639 | ifdef CONFIG_CC_STACKPROTECTOR_REGULAR | 658 | ifdef CONFIG_CC_STACKPROTECTOR_REGULAR |
640 | stackp-flag := -fstack-protector | 659 | stackp-flag := -fstack-protector |
641 | ifeq ($(call cc-option, $(stackp-flag)),) | 660 | ifeq ($(call cc-option, $(stackp-flag)),) |
diff --git a/arch/arm/mm/dma-mapping.c b/arch/arm/mm/dma-mapping.c index 1f88db06b133..7a996aaa061e 100644 --- a/arch/arm/mm/dma-mapping.c +++ b/arch/arm/mm/dma-mapping.c | |||
@@ -26,6 +26,7 @@ | |||
26 | #include <linux/io.h> | 26 | #include <linux/io.h> |
27 | #include <linux/vmalloc.h> | 27 | #include <linux/vmalloc.h> |
28 | #include <linux/sizes.h> | 28 | #include <linux/sizes.h> |
29 | #include <linux/cma.h> | ||
29 | 30 | ||
30 | #include <asm/memory.h> | 31 | #include <asm/memory.h> |
31 | #include <asm/highmem.h> | 32 | #include <asm/highmem.h> |
diff --git a/arch/ia64/mm/init.c b/arch/ia64/mm/init.c index 25c350264a41..892d43e32f3b 100644 --- a/arch/ia64/mm/init.c +++ b/arch/ia64/mm/init.c | |||
@@ -631,7 +631,8 @@ int arch_add_memory(int nid, u64 start, u64 size) | |||
631 | 631 | ||
632 | pgdat = NODE_DATA(nid); | 632 | pgdat = NODE_DATA(nid); |
633 | 633 | ||
634 | zone = pgdat->node_zones + ZONE_NORMAL; | 634 | zone = pgdat->node_zones + |
635 | zone_for_memory(nid, start, size, ZONE_NORMAL); | ||
635 | ret = __add_pages(nid, zone, start_pfn, nr_pages); | 636 | ret = __add_pages(nid, zone, start_pfn, nr_pages); |
636 | 637 | ||
637 | if (ret) | 638 | if (ret) |
diff --git a/arch/powerpc/kvm/Makefile b/arch/powerpc/kvm/Makefile index ce569b6bf4d8..72905c30082e 100644 --- a/arch/powerpc/kvm/Makefile +++ b/arch/powerpc/kvm/Makefile | |||
@@ -90,7 +90,6 @@ kvm-book3s_64-builtin-objs-$(CONFIG_KVM_BOOK3S_64_HANDLER) += \ | |||
90 | book3s_hv_rm_mmu.o \ | 90 | book3s_hv_rm_mmu.o \ |
91 | book3s_hv_ras.o \ | 91 | book3s_hv_ras.o \ |
92 | book3s_hv_builtin.o \ | 92 | book3s_hv_builtin.o \ |
93 | book3s_hv_cma.o \ | ||
94 | $(kvm-book3s_64-builtin-xics-objs-y) | 93 | $(kvm-book3s_64-builtin-xics-objs-y) |
95 | endif | 94 | endif |
96 | 95 | ||
diff --git a/arch/powerpc/kvm/book3s_64_mmu_hv.c b/arch/powerpc/kvm/book3s_64_mmu_hv.c index 68468d695f12..a01744fc3483 100644 --- a/arch/powerpc/kvm/book3s_64_mmu_hv.c +++ b/arch/powerpc/kvm/book3s_64_mmu_hv.c | |||
@@ -37,8 +37,6 @@ | |||
37 | #include <asm/ppc-opcode.h> | 37 | #include <asm/ppc-opcode.h> |
38 | #include <asm/cputable.h> | 38 | #include <asm/cputable.h> |
39 | 39 | ||
40 | #include "book3s_hv_cma.h" | ||
41 | |||
42 | /* POWER7 has 10-bit LPIDs, PPC970 has 6-bit LPIDs */ | 40 | /* POWER7 has 10-bit LPIDs, PPC970 has 6-bit LPIDs */ |
43 | #define MAX_LPID_970 63 | 41 | #define MAX_LPID_970 63 |
44 | 42 | ||
@@ -64,10 +62,10 @@ long kvmppc_alloc_hpt(struct kvm *kvm, u32 *htab_orderp) | |||
64 | } | 62 | } |
65 | 63 | ||
66 | kvm->arch.hpt_cma_alloc = 0; | 64 | kvm->arch.hpt_cma_alloc = 0; |
67 | VM_BUG_ON(order < KVM_CMA_CHUNK_ORDER); | ||
68 | page = kvm_alloc_hpt(1 << (order - PAGE_SHIFT)); | 65 | page = kvm_alloc_hpt(1 << (order - PAGE_SHIFT)); |
69 | if (page) { | 66 | if (page) { |
70 | hpt = (unsigned long)pfn_to_kaddr(page_to_pfn(page)); | 67 | hpt = (unsigned long)pfn_to_kaddr(page_to_pfn(page)); |
68 | memset((void *)hpt, 0, (1 << order)); | ||
71 | kvm->arch.hpt_cma_alloc = 1; | 69 | kvm->arch.hpt_cma_alloc = 1; |
72 | } | 70 | } |
73 | 71 | ||
diff --git a/arch/powerpc/kvm/book3s_hv_builtin.c b/arch/powerpc/kvm/book3s_hv_builtin.c index 7cde8a665205..6cf498a9bc98 100644 --- a/arch/powerpc/kvm/book3s_hv_builtin.c +++ b/arch/powerpc/kvm/book3s_hv_builtin.c | |||
@@ -16,12 +16,14 @@ | |||
16 | #include <linux/init.h> | 16 | #include <linux/init.h> |
17 | #include <linux/memblock.h> | 17 | #include <linux/memblock.h> |
18 | #include <linux/sizes.h> | 18 | #include <linux/sizes.h> |
19 | #include <linux/cma.h> | ||
19 | 20 | ||
20 | #include <asm/cputable.h> | 21 | #include <asm/cputable.h> |
21 | #include <asm/kvm_ppc.h> | 22 | #include <asm/kvm_ppc.h> |
22 | #include <asm/kvm_book3s.h> | 23 | #include <asm/kvm_book3s.h> |
23 | 24 | ||
24 | #include "book3s_hv_cma.h" | 25 | #define KVM_CMA_CHUNK_ORDER 18 |
26 | |||
25 | /* | 27 | /* |
26 | * Hash page table alignment on newer cpus(CPU_FTR_ARCH_206) | 28 | * Hash page table alignment on newer cpus(CPU_FTR_ARCH_206) |
27 | * should be power of 2. | 29 | * should be power of 2. |
@@ -43,6 +45,8 @@ static unsigned long kvm_cma_resv_ratio = 5; | |||
43 | unsigned long kvm_rma_pages = (1 << 27) >> PAGE_SHIFT; /* 128MB */ | 45 | unsigned long kvm_rma_pages = (1 << 27) >> PAGE_SHIFT; /* 128MB */ |
44 | EXPORT_SYMBOL_GPL(kvm_rma_pages); | 46 | EXPORT_SYMBOL_GPL(kvm_rma_pages); |
45 | 47 | ||
48 | static struct cma *kvm_cma; | ||
49 | |||
46 | /* Work out RMLS (real mode limit selector) field value for a given RMA size. | 50 | /* Work out RMLS (real mode limit selector) field value for a given RMA size. |
47 | Assumes POWER7 or PPC970. */ | 51 | Assumes POWER7 or PPC970. */ |
48 | static inline int lpcr_rmls(unsigned long rma_size) | 52 | static inline int lpcr_rmls(unsigned long rma_size) |
@@ -97,7 +101,7 @@ struct kvm_rma_info *kvm_alloc_rma() | |||
97 | ri = kmalloc(sizeof(struct kvm_rma_info), GFP_KERNEL); | 101 | ri = kmalloc(sizeof(struct kvm_rma_info), GFP_KERNEL); |
98 | if (!ri) | 102 | if (!ri) |
99 | return NULL; | 103 | return NULL; |
100 | page = kvm_alloc_cma(kvm_rma_pages, kvm_rma_pages); | 104 | page = cma_alloc(kvm_cma, kvm_rma_pages, get_order(kvm_rma_pages)); |
101 | if (!page) | 105 | if (!page) |
102 | goto err_out; | 106 | goto err_out; |
103 | atomic_set(&ri->use_count, 1); | 107 | atomic_set(&ri->use_count, 1); |
@@ -112,7 +116,7 @@ EXPORT_SYMBOL_GPL(kvm_alloc_rma); | |||
112 | void kvm_release_rma(struct kvm_rma_info *ri) | 116 | void kvm_release_rma(struct kvm_rma_info *ri) |
113 | { | 117 | { |
114 | if (atomic_dec_and_test(&ri->use_count)) { | 118 | if (atomic_dec_and_test(&ri->use_count)) { |
115 | kvm_release_cma(pfn_to_page(ri->base_pfn), kvm_rma_pages); | 119 | cma_release(kvm_cma, pfn_to_page(ri->base_pfn), kvm_rma_pages); |
116 | kfree(ri); | 120 | kfree(ri); |
117 | } | 121 | } |
118 | } | 122 | } |
@@ -131,16 +135,18 @@ struct page *kvm_alloc_hpt(unsigned long nr_pages) | |||
131 | { | 135 | { |
132 | unsigned long align_pages = HPT_ALIGN_PAGES; | 136 | unsigned long align_pages = HPT_ALIGN_PAGES; |
133 | 137 | ||
138 | VM_BUG_ON(get_order(nr_pages) < KVM_CMA_CHUNK_ORDER - PAGE_SHIFT); | ||
139 | |||
134 | /* Old CPUs require HPT aligned on a multiple of its size */ | 140 | /* Old CPUs require HPT aligned on a multiple of its size */ |
135 | if (!cpu_has_feature(CPU_FTR_ARCH_206)) | 141 | if (!cpu_has_feature(CPU_FTR_ARCH_206)) |
136 | align_pages = nr_pages; | 142 | align_pages = nr_pages; |
137 | return kvm_alloc_cma(nr_pages, align_pages); | 143 | return cma_alloc(kvm_cma, nr_pages, get_order(align_pages)); |
138 | } | 144 | } |
139 | EXPORT_SYMBOL_GPL(kvm_alloc_hpt); | 145 | EXPORT_SYMBOL_GPL(kvm_alloc_hpt); |
140 | 146 | ||
141 | void kvm_release_hpt(struct page *page, unsigned long nr_pages) | 147 | void kvm_release_hpt(struct page *page, unsigned long nr_pages) |
142 | { | 148 | { |
143 | kvm_release_cma(page, nr_pages); | 149 | cma_release(kvm_cma, page, nr_pages); |
144 | } | 150 | } |
145 | EXPORT_SYMBOL_GPL(kvm_release_hpt); | 151 | EXPORT_SYMBOL_GPL(kvm_release_hpt); |
146 | 152 | ||
@@ -179,7 +185,8 @@ void __init kvm_cma_reserve(void) | |||
179 | align_size = HPT_ALIGN_PAGES << PAGE_SHIFT; | 185 | align_size = HPT_ALIGN_PAGES << PAGE_SHIFT; |
180 | 186 | ||
181 | align_size = max(kvm_rma_pages << PAGE_SHIFT, align_size); | 187 | align_size = max(kvm_rma_pages << PAGE_SHIFT, align_size); |
182 | kvm_cma_declare_contiguous(selected_size, align_size); | 188 | cma_declare_contiguous(0, selected_size, 0, align_size, |
189 | KVM_CMA_CHUNK_ORDER - PAGE_SHIFT, false, &kvm_cma); | ||
183 | } | 190 | } |
184 | } | 191 | } |
185 | 192 | ||
diff --git a/arch/powerpc/kvm/book3s_hv_cma.c b/arch/powerpc/kvm/book3s_hv_cma.c deleted file mode 100644 index d9d3d8553d51..000000000000 --- a/arch/powerpc/kvm/book3s_hv_cma.c +++ /dev/null | |||
@@ -1,240 +0,0 @@ | |||
1 | /* | ||
2 | * Contiguous Memory Allocator for ppc KVM hash pagetable based on CMA | ||
3 | * for DMA mapping framework | ||
4 | * | ||
5 | * Copyright IBM Corporation, 2013 | ||
6 | * Author Aneesh Kumar K.V <aneesh.kumar@linux.vnet.ibm.com> | ||
7 | * | ||
8 | * This program is free software; you can redistribute it and/or | ||
9 | * modify it under the terms of the GNU General Public License as | ||
10 | * published by the Free Software Foundation; either version 2 of the | ||
11 | * License or (at your optional) any later version of the license. | ||
12 | * | ||
13 | */ | ||
14 | #define pr_fmt(fmt) "kvm_cma: " fmt | ||
15 | |||
16 | #ifdef CONFIG_CMA_DEBUG | ||
17 | #ifndef DEBUG | ||
18 | # define DEBUG | ||
19 | #endif | ||
20 | #endif | ||
21 | |||
22 | #include <linux/memblock.h> | ||
23 | #include <linux/mutex.h> | ||
24 | #include <linux/sizes.h> | ||
25 | #include <linux/slab.h> | ||
26 | |||
27 | #include "book3s_hv_cma.h" | ||
28 | |||
29 | struct kvm_cma { | ||
30 | unsigned long base_pfn; | ||
31 | unsigned long count; | ||
32 | unsigned long *bitmap; | ||
33 | }; | ||
34 | |||
35 | static DEFINE_MUTEX(kvm_cma_mutex); | ||
36 | static struct kvm_cma kvm_cma_area; | ||
37 | |||
38 | /** | ||
39 | * kvm_cma_declare_contiguous() - reserve area for contiguous memory handling | ||
40 | * for kvm hash pagetable | ||
41 | * @size: Size of the reserved memory. | ||
42 | * @alignment: Alignment for the contiguous memory area | ||
43 | * | ||
44 | * This function reserves memory for kvm cma area. It should be | ||
45 | * called by arch code when early allocator (memblock or bootmem) | ||
46 | * is still activate. | ||
47 | */ | ||
48 | long __init kvm_cma_declare_contiguous(phys_addr_t size, phys_addr_t alignment) | ||
49 | { | ||
50 | long base_pfn; | ||
51 | phys_addr_t addr; | ||
52 | struct kvm_cma *cma = &kvm_cma_area; | ||
53 | |||
54 | pr_debug("%s(size %lx)\n", __func__, (unsigned long)size); | ||
55 | |||
56 | if (!size) | ||
57 | return -EINVAL; | ||
58 | /* | ||
59 | * Sanitise input arguments. | ||
60 | * We should be pageblock aligned for CMA. | ||
61 | */ | ||
62 | alignment = max(alignment, (phys_addr_t)(PAGE_SIZE << pageblock_order)); | ||
63 | size = ALIGN(size, alignment); | ||
64 | /* | ||
65 | * Reserve memory | ||
66 | * Use __memblock_alloc_base() since | ||
67 | * memblock_alloc_base() panic()s. | ||
68 | */ | ||
69 | addr = __memblock_alloc_base(size, alignment, 0); | ||
70 | if (!addr) { | ||
71 | base_pfn = -ENOMEM; | ||
72 | goto err; | ||
73 | } else | ||
74 | base_pfn = PFN_DOWN(addr); | ||
75 | |||
76 | /* | ||
77 | * Each reserved area must be initialised later, when more kernel | ||
78 | * subsystems (like slab allocator) are available. | ||
79 | */ | ||
80 | cma->base_pfn = base_pfn; | ||
81 | cma->count = size >> PAGE_SHIFT; | ||
82 | pr_info("CMA: reserved %ld MiB\n", (unsigned long)size / SZ_1M); | ||
83 | return 0; | ||
84 | err: | ||
85 | pr_err("CMA: failed to reserve %ld MiB\n", (unsigned long)size / SZ_1M); | ||
86 | return base_pfn; | ||
87 | } | ||
88 | |||
89 | /** | ||
90 | * kvm_alloc_cma() - allocate pages from contiguous area | ||
91 | * @nr_pages: Requested number of pages. | ||
92 | * @align_pages: Requested alignment in number of pages | ||
93 | * | ||
94 | * This function allocates memory buffer for hash pagetable. | ||
95 | */ | ||
96 | struct page *kvm_alloc_cma(unsigned long nr_pages, unsigned long align_pages) | ||
97 | { | ||
98 | int ret; | ||
99 | struct page *page = NULL; | ||
100 | struct kvm_cma *cma = &kvm_cma_area; | ||
101 | unsigned long chunk_count, nr_chunk; | ||
102 | unsigned long mask, pfn, pageno, start = 0; | ||
103 | |||
104 | |||
105 | if (!cma || !cma->count) | ||
106 | return NULL; | ||
107 | |||
108 | pr_debug("%s(cma %p, count %lu, align pages %lu)\n", __func__, | ||
109 | (void *)cma, nr_pages, align_pages); | ||
110 | |||
111 | if (!nr_pages) | ||
112 | return NULL; | ||
113 | /* | ||
114 | * align mask with chunk size. The bit tracks pages in chunk size | ||
115 | */ | ||
116 | VM_BUG_ON(!is_power_of_2(align_pages)); | ||
117 | mask = (align_pages >> (KVM_CMA_CHUNK_ORDER - PAGE_SHIFT)) - 1; | ||
118 | BUILD_BUG_ON(PAGE_SHIFT > KVM_CMA_CHUNK_ORDER); | ||
119 | |||
120 | chunk_count = cma->count >> (KVM_CMA_CHUNK_ORDER - PAGE_SHIFT); | ||
121 | nr_chunk = nr_pages >> (KVM_CMA_CHUNK_ORDER - PAGE_SHIFT); | ||
122 | |||
123 | mutex_lock(&kvm_cma_mutex); | ||
124 | for (;;) { | ||
125 | pageno = bitmap_find_next_zero_area(cma->bitmap, chunk_count, | ||
126 | start, nr_chunk, mask); | ||
127 | if (pageno >= chunk_count) | ||
128 | break; | ||
129 | |||
130 | pfn = cma->base_pfn + (pageno << (KVM_CMA_CHUNK_ORDER - PAGE_SHIFT)); | ||
131 | ret = alloc_contig_range(pfn, pfn + nr_pages, MIGRATE_CMA); | ||
132 | if (ret == 0) { | ||
133 | bitmap_set(cma->bitmap, pageno, nr_chunk); | ||
134 | page = pfn_to_page(pfn); | ||
135 | memset(pfn_to_kaddr(pfn), 0, nr_pages << PAGE_SHIFT); | ||
136 | break; | ||
137 | } else if (ret != -EBUSY) { | ||
138 | break; | ||
139 | } | ||
140 | pr_debug("%s(): memory range at %p is busy, retrying\n", | ||
141 | __func__, pfn_to_page(pfn)); | ||
142 | /* try again with a bit different memory target */ | ||
143 | start = pageno + mask + 1; | ||
144 | } | ||
145 | mutex_unlock(&kvm_cma_mutex); | ||
146 | pr_debug("%s(): returned %p\n", __func__, page); | ||
147 | return page; | ||
148 | } | ||
149 | |||
150 | /** | ||
151 | * kvm_release_cma() - release allocated pages for hash pagetable | ||
152 | * @pages: Allocated pages. | ||
153 | * @nr_pages: Number of allocated pages. | ||
154 | * | ||
155 | * This function releases memory allocated by kvm_alloc_cma(). | ||
156 | * It returns false when provided pages do not belong to contiguous area and | ||
157 | * true otherwise. | ||
158 | */ | ||
159 | bool kvm_release_cma(struct page *pages, unsigned long nr_pages) | ||
160 | { | ||
161 | unsigned long pfn; | ||
162 | unsigned long nr_chunk; | ||
163 | struct kvm_cma *cma = &kvm_cma_area; | ||
164 | |||
165 | if (!cma || !pages) | ||
166 | return false; | ||
167 | |||
168 | pr_debug("%s(page %p count %lu)\n", __func__, (void *)pages, nr_pages); | ||
169 | |||
170 | pfn = page_to_pfn(pages); | ||
171 | |||
172 | if (pfn < cma->base_pfn || pfn >= cma->base_pfn + cma->count) | ||
173 | return false; | ||
174 | |||
175 | VM_BUG_ON(pfn + nr_pages > cma->base_pfn + cma->count); | ||
176 | nr_chunk = nr_pages >> (KVM_CMA_CHUNK_ORDER - PAGE_SHIFT); | ||
177 | |||
178 | mutex_lock(&kvm_cma_mutex); | ||
179 | bitmap_clear(cma->bitmap, | ||
180 | (pfn - cma->base_pfn) >> (KVM_CMA_CHUNK_ORDER - PAGE_SHIFT), | ||
181 | nr_chunk); | ||
182 | free_contig_range(pfn, nr_pages); | ||
183 | mutex_unlock(&kvm_cma_mutex); | ||
184 | |||
185 | return true; | ||
186 | } | ||
187 | |||
188 | static int __init kvm_cma_activate_area(unsigned long base_pfn, | ||
189 | unsigned long count) | ||
190 | { | ||
191 | unsigned long pfn = base_pfn; | ||
192 | unsigned i = count >> pageblock_order; | ||
193 | struct zone *zone; | ||
194 | |||
195 | WARN_ON_ONCE(!pfn_valid(pfn)); | ||
196 | zone = page_zone(pfn_to_page(pfn)); | ||
197 | do { | ||
198 | unsigned j; | ||
199 | base_pfn = pfn; | ||
200 | for (j = pageblock_nr_pages; j; --j, pfn++) { | ||
201 | WARN_ON_ONCE(!pfn_valid(pfn)); | ||
202 | /* | ||
203 | * alloc_contig_range requires the pfn range | ||
204 | * specified to be in the same zone. Make this | ||
205 | * simple by forcing the entire CMA resv range | ||
206 | * to be in the same zone. | ||
207 | */ | ||
208 | if (page_zone(pfn_to_page(pfn)) != zone) | ||
209 | return -EINVAL; | ||
210 | } | ||
211 | init_cma_reserved_pageblock(pfn_to_page(base_pfn)); | ||
212 | } while (--i); | ||
213 | return 0; | ||
214 | } | ||
215 | |||
216 | static int __init kvm_cma_init_reserved_areas(void) | ||
217 | { | ||
218 | int bitmap_size, ret; | ||
219 | unsigned long chunk_count; | ||
220 | struct kvm_cma *cma = &kvm_cma_area; | ||
221 | |||
222 | pr_debug("%s()\n", __func__); | ||
223 | if (!cma->count) | ||
224 | return 0; | ||
225 | chunk_count = cma->count >> (KVM_CMA_CHUNK_ORDER - PAGE_SHIFT); | ||
226 | bitmap_size = BITS_TO_LONGS(chunk_count) * sizeof(long); | ||
227 | cma->bitmap = kzalloc(bitmap_size, GFP_KERNEL); | ||
228 | if (!cma->bitmap) | ||
229 | return -ENOMEM; | ||
230 | |||
231 | ret = kvm_cma_activate_area(cma->base_pfn, cma->count); | ||
232 | if (ret) | ||
233 | goto error; | ||
234 | return 0; | ||
235 | |||
236 | error: | ||
237 | kfree(cma->bitmap); | ||
238 | return ret; | ||
239 | } | ||
240 | core_initcall(kvm_cma_init_reserved_areas); | ||
diff --git a/arch/powerpc/kvm/book3s_hv_cma.h b/arch/powerpc/kvm/book3s_hv_cma.h deleted file mode 100644 index 655144f75fa5..000000000000 --- a/arch/powerpc/kvm/book3s_hv_cma.h +++ /dev/null | |||
@@ -1,27 +0,0 @@ | |||
1 | /* | ||
2 | * Contiguous Memory Allocator for ppc KVM hash pagetable based on CMA | ||
3 | * for DMA mapping framework | ||
4 | * | ||
5 | * Copyright IBM Corporation, 2013 | ||
6 | * Author Aneesh Kumar K.V <aneesh.kumar@linux.vnet.ibm.com> | ||
7 | * | ||
8 | * This program is free software; you can redistribute it and/or | ||
9 | * modify it under the terms of the GNU General Public License as | ||
10 | * published by the Free Software Foundation; either version 2 of the | ||
11 | * License or (at your optional) any later version of the license. | ||
12 | * | ||
13 | */ | ||
14 | |||
15 | #ifndef __POWERPC_KVM_CMA_ALLOC_H__ | ||
16 | #define __POWERPC_KVM_CMA_ALLOC_H__ | ||
17 | /* | ||
18 | * Both RMA and Hash page allocation will be multiple of 256K. | ||
19 | */ | ||
20 | #define KVM_CMA_CHUNK_ORDER 18 | ||
21 | |||
22 | extern struct page *kvm_alloc_cma(unsigned long nr_pages, | ||
23 | unsigned long align_pages); | ||
24 | extern bool kvm_release_cma(struct page *pages, unsigned long nr_pages); | ||
25 | extern long kvm_cma_declare_contiguous(phys_addr_t size, | ||
26 | phys_addr_t alignment) __init; | ||
27 | #endif | ||
diff --git a/arch/powerpc/mm/mem.c b/arch/powerpc/mm/mem.c index 2c8e90f5789e..e0f7a189c48e 100644 --- a/arch/powerpc/mm/mem.c +++ b/arch/powerpc/mm/mem.c | |||
@@ -128,7 +128,8 @@ int arch_add_memory(int nid, u64 start, u64 size) | |||
128 | return -EINVAL; | 128 | return -EINVAL; |
129 | 129 | ||
130 | /* this should work for most non-highmem platforms */ | 130 | /* this should work for most non-highmem platforms */ |
131 | zone = pgdata->node_zones; | 131 | zone = pgdata->node_zones + |
132 | zone_for_memory(nid, start, size, 0); | ||
132 | 133 | ||
133 | return __add_pages(nid, zone, start_pfn, nr_pages); | 134 | return __add_pages(nid, zone, start_pfn, nr_pages); |
134 | } | 135 | } |
diff --git a/arch/score/include/uapi/asm/ptrace.h b/arch/score/include/uapi/asm/ptrace.h index f59771a3f127..5c5e794058be 100644 --- a/arch/score/include/uapi/asm/ptrace.h +++ b/arch/score/include/uapi/asm/ptrace.h | |||
@@ -4,17 +4,6 @@ | |||
4 | #define PTRACE_GETREGS 12 | 4 | #define PTRACE_GETREGS 12 |
5 | #define PTRACE_SETREGS 13 | 5 | #define PTRACE_SETREGS 13 |
6 | 6 | ||
7 | #define PC 32 | ||
8 | #define CONDITION 33 | ||
9 | #define ECR 34 | ||
10 | #define EMA 35 | ||
11 | #define CEH 36 | ||
12 | #define CEL 37 | ||
13 | #define COUNTER 38 | ||
14 | #define LDCR 39 | ||
15 | #define STCR 40 | ||
16 | #define PSR 41 | ||
17 | |||
18 | #define SINGLESTEP16_INSN 0x7006 | 7 | #define SINGLESTEP16_INSN 0x7006 |
19 | #define SINGLESTEP32_INSN 0x840C8000 | 8 | #define SINGLESTEP32_INSN 0x840C8000 |
20 | #define BREAKPOINT16_INSN 0x7002 /* work on SPG300 */ | 9 | #define BREAKPOINT16_INSN 0x7002 /* work on SPG300 */ |
diff --git a/arch/sh/drivers/dma/Kconfig b/arch/sh/drivers/dma/Kconfig index cfd5b90a8628..78bc97b1d027 100644 --- a/arch/sh/drivers/dma/Kconfig +++ b/arch/sh/drivers/dma/Kconfig | |||
@@ -12,9 +12,8 @@ config SH_DMA_IRQ_MULTI | |||
12 | default y if CPU_SUBTYPE_SH7750 || CPU_SUBTYPE_SH7751 || \ | 12 | default y if CPU_SUBTYPE_SH7750 || CPU_SUBTYPE_SH7751 || \ |
13 | CPU_SUBTYPE_SH7750S || CPU_SUBTYPE_SH7750R || \ | 13 | CPU_SUBTYPE_SH7750S || CPU_SUBTYPE_SH7750R || \ |
14 | CPU_SUBTYPE_SH7751R || CPU_SUBTYPE_SH7091 || \ | 14 | CPU_SUBTYPE_SH7751R || CPU_SUBTYPE_SH7091 || \ |
15 | CPU_SUBTYPE_SH7763 || CPU_SUBTYPE_SH7764 || \ | 15 | CPU_SUBTYPE_SH7763 || CPU_SUBTYPE_SH7780 || \ |
16 | CPU_SUBTYPE_SH7780 || CPU_SUBTYPE_SH7785 || \ | 16 | CPU_SUBTYPE_SH7785 || CPU_SUBTYPE_SH7760 |
17 | CPU_SUBTYPE_SH7760 | ||
18 | 17 | ||
19 | config SH_DMA_API | 18 | config SH_DMA_API |
20 | depends on SH_DMA | 19 | depends on SH_DMA |
diff --git a/arch/sh/include/asm/io_noioport.h b/arch/sh/include/asm/io_noioport.h index 4d48f1436a63..c727e6ddf69e 100644 --- a/arch/sh/include/asm/io_noioport.h +++ b/arch/sh/include/asm/io_noioport.h | |||
@@ -34,6 +34,17 @@ static inline void outl(unsigned int x, unsigned long port) | |||
34 | BUG(); | 34 | BUG(); |
35 | } | 35 | } |
36 | 36 | ||
37 | static inline void __iomem *ioport_map(unsigned long port, unsigned int size) | ||
38 | { | ||
39 | BUG(); | ||
40 | return NULL; | ||
41 | } | ||
42 | |||
43 | static inline void ioport_unmap(void __iomem *addr) | ||
44 | { | ||
45 | BUG(); | ||
46 | } | ||
47 | |||
37 | #define inb_p(addr) inb(addr) | 48 | #define inb_p(addr) inb(addr) |
38 | #define inw_p(addr) inw(addr) | 49 | #define inw_p(addr) inw(addr) |
39 | #define inl_p(addr) inl(addr) | 50 | #define inl_p(addr) inl(addr) |
diff --git a/arch/sh/include/cpu-sh4/cpu/dma-register.h b/arch/sh/include/cpu-sh4/cpu/dma-register.h index 02788b6a03b7..9cd81e54056a 100644 --- a/arch/sh/include/cpu-sh4/cpu/dma-register.h +++ b/arch/sh/include/cpu-sh4/cpu/dma-register.h | |||
@@ -32,7 +32,6 @@ | |||
32 | #define CHCR_TS_HIGH_SHIFT (20 - 2) /* 2 bits for shifted low TS */ | 32 | #define CHCR_TS_HIGH_SHIFT (20 - 2) /* 2 bits for shifted low TS */ |
33 | #elif defined(CONFIG_CPU_SUBTYPE_SH7757) || \ | 33 | #elif defined(CONFIG_CPU_SUBTYPE_SH7757) || \ |
34 | defined(CONFIG_CPU_SUBTYPE_SH7763) || \ | 34 | defined(CONFIG_CPU_SUBTYPE_SH7763) || \ |
35 | defined(CONFIG_CPU_SUBTYPE_SH7764) || \ | ||
36 | defined(CONFIG_CPU_SUBTYPE_SH7780) || \ | 35 | defined(CONFIG_CPU_SUBTYPE_SH7780) || \ |
37 | defined(CONFIG_CPU_SUBTYPE_SH7785) | 36 | defined(CONFIG_CPU_SUBTYPE_SH7785) |
38 | #define CHCR_TS_LOW_MASK 0x00000018 | 37 | #define CHCR_TS_LOW_MASK 0x00000018 |
diff --git a/arch/sh/include/cpu-sh4a/cpu/dma.h b/arch/sh/include/cpu-sh4a/cpu/dma.h index 89afb650ce25..8ceccceae844 100644 --- a/arch/sh/include/cpu-sh4a/cpu/dma.h +++ b/arch/sh/include/cpu-sh4a/cpu/dma.h | |||
@@ -14,8 +14,7 @@ | |||
14 | #define DMTE4_IRQ evt2irq(0xb80) | 14 | #define DMTE4_IRQ evt2irq(0xb80) |
15 | #define DMAE0_IRQ evt2irq(0xbc0) /* DMA Error IRQ*/ | 15 | #define DMAE0_IRQ evt2irq(0xbc0) /* DMA Error IRQ*/ |
16 | #define SH_DMAC_BASE0 0xFE008020 | 16 | #define SH_DMAC_BASE0 0xFE008020 |
17 | #elif defined(CONFIG_CPU_SUBTYPE_SH7763) || \ | 17 | #elif defined(CONFIG_CPU_SUBTYPE_SH7763) |
18 | defined(CONFIG_CPU_SUBTYPE_SH7764) | ||
19 | #define DMTE0_IRQ evt2irq(0x640) | 18 | #define DMTE0_IRQ evt2irq(0x640) |
20 | #define DMTE4_IRQ evt2irq(0x780) | 19 | #define DMTE4_IRQ evt2irq(0x780) |
21 | #define DMAE0_IRQ evt2irq(0x6c0) | 20 | #define DMAE0_IRQ evt2irq(0x6c0) |
diff --git a/arch/sh/kernel/cpu/sh4a/clock-sh7724.c b/arch/sh/kernel/cpu/sh4a/clock-sh7724.c index f579dd528198..c187b9579c21 100644 --- a/arch/sh/kernel/cpu/sh4a/clock-sh7724.c +++ b/arch/sh/kernel/cpu/sh4a/clock-sh7724.c | |||
@@ -307,7 +307,7 @@ static struct clk_lookup lookups[] = { | |||
307 | CLKDEV_ICK_ID("fck", "sh-tmu.0", &mstp_clks[HWBLK_TMU0]), | 307 | CLKDEV_ICK_ID("fck", "sh-tmu.0", &mstp_clks[HWBLK_TMU0]), |
308 | CLKDEV_ICK_ID("fck", "sh-tmu.1", &mstp_clks[HWBLK_TMU1]), | 308 | CLKDEV_ICK_ID("fck", "sh-tmu.1", &mstp_clks[HWBLK_TMU1]), |
309 | 309 | ||
310 | CLKDEV_ICK_ID("fck", "sh-cmt-16.0", &mstp_clks[HWBLK_CMT]), | 310 | CLKDEV_ICK_ID("fck", "sh-cmt-32.0", &mstp_clks[HWBLK_CMT]), |
311 | CLKDEV_DEV_ID("sh-wdt.0", &mstp_clks[HWBLK_RWDT]), | 311 | CLKDEV_DEV_ID("sh-wdt.0", &mstp_clks[HWBLK_RWDT]), |
312 | CLKDEV_DEV_ID("sh-dma-engine.1", &mstp_clks[HWBLK_DMAC1]), | 312 | CLKDEV_DEV_ID("sh-dma-engine.1", &mstp_clks[HWBLK_DMAC1]), |
313 | 313 | ||
@@ -332,6 +332,8 @@ static struct clk_lookup lookups[] = { | |||
332 | CLKDEV_CON_ID("tsif0", &mstp_clks[HWBLK_TSIF]), | 332 | CLKDEV_CON_ID("tsif0", &mstp_clks[HWBLK_TSIF]), |
333 | CLKDEV_DEV_ID("renesas_usbhs.1", &mstp_clks[HWBLK_USB1]), | 333 | CLKDEV_DEV_ID("renesas_usbhs.1", &mstp_clks[HWBLK_USB1]), |
334 | CLKDEV_DEV_ID("renesas_usbhs.0", &mstp_clks[HWBLK_USB0]), | 334 | CLKDEV_DEV_ID("renesas_usbhs.0", &mstp_clks[HWBLK_USB0]), |
335 | CLKDEV_CON_ID("usb1", &mstp_clks[HWBLK_USB1]), | ||
336 | CLKDEV_CON_ID("usb0", &mstp_clks[HWBLK_USB0]), | ||
335 | CLKDEV_CON_ID("2dg0", &mstp_clks[HWBLK_2DG]), | 337 | CLKDEV_CON_ID("2dg0", &mstp_clks[HWBLK_2DG]), |
336 | CLKDEV_DEV_ID("sh_mobile_sdhi.0", &mstp_clks[HWBLK_SDHI0]), | 338 | CLKDEV_DEV_ID("sh_mobile_sdhi.0", &mstp_clks[HWBLK_SDHI0]), |
337 | CLKDEV_DEV_ID("sh_mobile_sdhi.1", &mstp_clks[HWBLK_SDHI1]), | 339 | CLKDEV_DEV_ID("sh_mobile_sdhi.1", &mstp_clks[HWBLK_SDHI1]), |
diff --git a/arch/sh/kernel/time.c b/arch/sh/kernel/time.c index 552c8fcf9416..d6d0a986c6e9 100644 --- a/arch/sh/kernel/time.c +++ b/arch/sh/kernel/time.c | |||
@@ -80,10 +80,8 @@ static int __init rtc_generic_init(void) | |||
80 | return -ENODEV; | 80 | return -ENODEV; |
81 | 81 | ||
82 | pdev = platform_device_register_simple("rtc-generic", -1, NULL, 0); | 82 | pdev = platform_device_register_simple("rtc-generic", -1, NULL, 0); |
83 | if (IS_ERR(pdev)) | ||
84 | return PTR_ERR(pdev); | ||
85 | 83 | ||
86 | return 0; | 84 | return PTR_ERR_OR_ZERO(pdev); |
87 | } | 85 | } |
88 | module_init(rtc_generic_init); | 86 | module_init(rtc_generic_init); |
89 | 87 | ||
diff --git a/arch/sh/mm/asids-debugfs.c b/arch/sh/mm/asids-debugfs.c index 74c03ecc4871..ecfc6b0c1da1 100644 --- a/arch/sh/mm/asids-debugfs.c +++ b/arch/sh/mm/asids-debugfs.c | |||
@@ -67,10 +67,8 @@ static int __init asids_debugfs_init(void) | |||
67 | NULL, &asids_debugfs_fops); | 67 | NULL, &asids_debugfs_fops); |
68 | if (!asids_dentry) | 68 | if (!asids_dentry) |
69 | return -ENOMEM; | 69 | return -ENOMEM; |
70 | if (IS_ERR(asids_dentry)) | ||
71 | return PTR_ERR(asids_dentry); | ||
72 | 70 | ||
73 | return 0; | 71 | return PTR_ERR_OR_ZERO(asids_dentry); |
74 | } | 72 | } |
75 | module_init(asids_debugfs_init); | 73 | module_init(asids_debugfs_init); |
76 | 74 | ||
diff --git a/arch/sh/mm/init.c b/arch/sh/mm/init.c index 2d089fe2cba9..2790b6a64157 100644 --- a/arch/sh/mm/init.c +++ b/arch/sh/mm/init.c | |||
@@ -495,8 +495,9 @@ int arch_add_memory(int nid, u64 start, u64 size) | |||
495 | pgdat = NODE_DATA(nid); | 495 | pgdat = NODE_DATA(nid); |
496 | 496 | ||
497 | /* We only have ZONE_NORMAL, so this is easy.. */ | 497 | /* We only have ZONE_NORMAL, so this is easy.. */ |
498 | ret = __add_pages(nid, pgdat->node_zones + ZONE_NORMAL, | 498 | ret = __add_pages(nid, pgdat->node_zones + |
499 | start_pfn, nr_pages); | 499 | zone_for_memory(nid, start, size, ZONE_NORMAL), |
500 | start_pfn, nr_pages); | ||
500 | if (unlikely(ret)) | 501 | if (unlikely(ret)) |
501 | printk("%s: Failed, __add_pages() == %d\n", __func__, ret); | 502 | printk("%s: Failed, __add_pages() == %d\n", __func__, ret); |
502 | 503 | ||
diff --git a/arch/tile/kernel/module.c b/arch/tile/kernel/module.c index 4918d91bc3a6..d19b13e3a59f 100644 --- a/arch/tile/kernel/module.c +++ b/arch/tile/kernel/module.c | |||
@@ -58,7 +58,7 @@ void *module_alloc(unsigned long size) | |||
58 | area->nr_pages = npages; | 58 | area->nr_pages = npages; |
59 | area->pages = pages; | 59 | area->pages = pages; |
60 | 60 | ||
61 | if (map_vm_area(area, prot_rwx, &pages)) { | 61 | if (map_vm_area(area, prot_rwx, pages)) { |
62 | vunmap(area->addr); | 62 | vunmap(area->addr); |
63 | goto error; | 63 | goto error; |
64 | } | 64 | } |
diff --git a/arch/x86/mm/fault.c b/arch/x86/mm/fault.c index 1dbade870f90..a24194681513 100644 --- a/arch/x86/mm/fault.c +++ b/arch/x86/mm/fault.c | |||
@@ -1218,7 +1218,8 @@ good_area: | |||
1218 | /* | 1218 | /* |
1219 | * If for any reason at all we couldn't handle the fault, | 1219 | * If for any reason at all we couldn't handle the fault, |
1220 | * make sure we exit gracefully rather than endlessly redo | 1220 | * make sure we exit gracefully rather than endlessly redo |
1221 | * the fault: | 1221 | * the fault. Since we never set FAULT_FLAG_RETRY_NOWAIT, if |
1222 | * we get VM_FAULT_RETRY back, the mmap_sem has been unlocked. | ||
1222 | */ | 1223 | */ |
1223 | fault = handle_mm_fault(mm, vma, address, flags); | 1224 | fault = handle_mm_fault(mm, vma, address, flags); |
1224 | 1225 | ||
diff --git a/arch/x86/mm/init_32.c b/arch/x86/mm/init_32.c index e39504878aec..7d05565ba781 100644 --- a/arch/x86/mm/init_32.c +++ b/arch/x86/mm/init_32.c | |||
@@ -825,7 +825,8 @@ void __init mem_init(void) | |||
825 | int arch_add_memory(int nid, u64 start, u64 size) | 825 | int arch_add_memory(int nid, u64 start, u64 size) |
826 | { | 826 | { |
827 | struct pglist_data *pgdata = NODE_DATA(nid); | 827 | struct pglist_data *pgdata = NODE_DATA(nid); |
828 | struct zone *zone = pgdata->node_zones + ZONE_HIGHMEM; | 828 | struct zone *zone = pgdata->node_zones + |
829 | zone_for_memory(nid, start, size, ZONE_HIGHMEM); | ||
829 | unsigned long start_pfn = start >> PAGE_SHIFT; | 830 | unsigned long start_pfn = start >> PAGE_SHIFT; |
830 | unsigned long nr_pages = size >> PAGE_SHIFT; | 831 | unsigned long nr_pages = size >> PAGE_SHIFT; |
831 | 832 | ||
diff --git a/arch/x86/mm/init_64.c b/arch/x86/mm/init_64.c index df1a9927ad29..5621c47d7a1a 100644 --- a/arch/x86/mm/init_64.c +++ b/arch/x86/mm/init_64.c | |||
@@ -691,7 +691,8 @@ static void update_end_of_memory_vars(u64 start, u64 size) | |||
691 | int arch_add_memory(int nid, u64 start, u64 size) | 691 | int arch_add_memory(int nid, u64 start, u64 size) |
692 | { | 692 | { |
693 | struct pglist_data *pgdat = NODE_DATA(nid); | 693 | struct pglist_data *pgdat = NODE_DATA(nid); |
694 | struct zone *zone = pgdat->node_zones + ZONE_NORMAL; | 694 | struct zone *zone = pgdat->node_zones + |
695 | zone_for_memory(nid, start, size, ZONE_NORMAL); | ||
695 | unsigned long start_pfn = start >> PAGE_SHIFT; | 696 | unsigned long start_pfn = start >> PAGE_SHIFT; |
696 | unsigned long nr_pages = size >> PAGE_SHIFT; | 697 | unsigned long nr_pages = size >> PAGE_SHIFT; |
697 | int ret; | 698 | int ret; |
diff --git a/drivers/ata/Kconfig b/drivers/ata/Kconfig index e65d400efd44..e1b92788c225 100644 --- a/drivers/ata/Kconfig +++ b/drivers/ata/Kconfig | |||
@@ -16,6 +16,7 @@ menuconfig ATA | |||
16 | depends on BLOCK | 16 | depends on BLOCK |
17 | depends on !(M32R || M68K || S390) || BROKEN | 17 | depends on !(M32R || M68K || S390) || BROKEN |
18 | select SCSI | 18 | select SCSI |
19 | select GLOB | ||
19 | ---help--- | 20 | ---help--- |
20 | If you want to use an ATA hard disk, ATA tape drive, ATA CD-ROM or | 21 | If you want to use an ATA hard disk, ATA tape drive, ATA CD-ROM or |
21 | any other ATA device under Linux, say Y and make sure that you know | 22 | any other ATA device under Linux, say Y and make sure that you know |
diff --git a/drivers/ata/libata-core.c b/drivers/ata/libata-core.c index 677c0c1b03bd..dbdc5d32343f 100644 --- a/drivers/ata/libata-core.c +++ b/drivers/ata/libata-core.c | |||
@@ -59,6 +59,7 @@ | |||
59 | #include <linux/async.h> | 59 | #include <linux/async.h> |
60 | #include <linux/log2.h> | 60 | #include <linux/log2.h> |
61 | #include <linux/slab.h> | 61 | #include <linux/slab.h> |
62 | #include <linux/glob.h> | ||
62 | #include <scsi/scsi.h> | 63 | #include <scsi/scsi.h> |
63 | #include <scsi/scsi_cmnd.h> | 64 | #include <scsi/scsi_cmnd.h> |
64 | #include <scsi/scsi_host.h> | 65 | #include <scsi/scsi_host.h> |
@@ -4250,73 +4251,6 @@ static const struct ata_blacklist_entry ata_device_blacklist [] = { | |||
4250 | { } | 4251 | { } |
4251 | }; | 4252 | }; |
4252 | 4253 | ||
4253 | /** | ||
4254 | * glob_match - match a text string against a glob-style pattern | ||
4255 | * @text: the string to be examined | ||
4256 | * @pattern: the glob-style pattern to be matched against | ||
4257 | * | ||
4258 | * Either/both of text and pattern can be empty strings. | ||
4259 | * | ||
4260 | * Match text against a glob-style pattern, with wildcards and simple sets: | ||
4261 | * | ||
4262 | * ? matches any single character. | ||
4263 | * * matches any run of characters. | ||
4264 | * [xyz] matches a single character from the set: x, y, or z. | ||
4265 | * [a-d] matches a single character from the range: a, b, c, or d. | ||
4266 | * [a-d0-9] matches a single character from either range. | ||
4267 | * | ||
4268 | * The special characters ?, [, -, or *, can be matched using a set, eg. [*] | ||
4269 | * Behaviour with malformed patterns is undefined, though generally reasonable. | ||
4270 | * | ||
4271 | * Sample patterns: "SD1?", "SD1[0-5]", "*R0", "SD*1?[012]*xx" | ||
4272 | * | ||
4273 | * This function uses one level of recursion per '*' in pattern. | ||
4274 | * Since it calls _nothing_ else, and has _no_ explicit local variables, | ||
4275 | * this will not cause stack problems for any reasonable use here. | ||
4276 | * | ||
4277 | * RETURNS: | ||
4278 | * 0 on match, 1 otherwise. | ||
4279 | */ | ||
4280 | static int glob_match (const char *text, const char *pattern) | ||
4281 | { | ||
4282 | do { | ||
4283 | /* Match single character or a '?' wildcard */ | ||
4284 | if (*text == *pattern || *pattern == '?') { | ||
4285 | if (!*pattern++) | ||
4286 | return 0; /* End of both strings: match */ | ||
4287 | } else { | ||
4288 | /* Match single char against a '[' bracketed ']' pattern set */ | ||
4289 | if (!*text || *pattern != '[') | ||
4290 | break; /* Not a pattern set */ | ||
4291 | while (*++pattern && *pattern != ']' && *text != *pattern) { | ||
4292 | if (*pattern == '-' && *(pattern - 1) != '[') | ||
4293 | if (*text > *(pattern - 1) && *text < *(pattern + 1)) { | ||
4294 | ++pattern; | ||
4295 | break; | ||
4296 | } | ||
4297 | } | ||
4298 | if (!*pattern || *pattern == ']') | ||
4299 | return 1; /* No match */ | ||
4300 | while (*pattern && *pattern++ != ']'); | ||
4301 | } | ||
4302 | } while (*++text && *pattern); | ||
4303 | |||
4304 | /* Match any run of chars against a '*' wildcard */ | ||
4305 | if (*pattern == '*') { | ||
4306 | if (!*++pattern) | ||
4307 | return 0; /* Match: avoid recursion at end of pattern */ | ||
4308 | /* Loop to handle additional pattern chars after the wildcard */ | ||
4309 | while (*text) { | ||
4310 | if (glob_match(text, pattern) == 0) | ||
4311 | return 0; /* Remainder matched */ | ||
4312 | ++text; /* Absorb (match) this char and try again */ | ||
4313 | } | ||
4314 | } | ||
4315 | if (!*text && !*pattern) | ||
4316 | return 0; /* End of both strings: match */ | ||
4317 | return 1; /* No match */ | ||
4318 | } | ||
4319 | |||
4320 | static unsigned long ata_dev_blacklisted(const struct ata_device *dev) | 4254 | static unsigned long ata_dev_blacklisted(const struct ata_device *dev) |
4321 | { | 4255 | { |
4322 | unsigned char model_num[ATA_ID_PROD_LEN + 1]; | 4256 | unsigned char model_num[ATA_ID_PROD_LEN + 1]; |
@@ -4327,10 +4261,10 @@ static unsigned long ata_dev_blacklisted(const struct ata_device *dev) | |||
4327 | ata_id_c_string(dev->id, model_rev, ATA_ID_FW_REV, sizeof(model_rev)); | 4261 | ata_id_c_string(dev->id, model_rev, ATA_ID_FW_REV, sizeof(model_rev)); |
4328 | 4262 | ||
4329 | while (ad->model_num) { | 4263 | while (ad->model_num) { |
4330 | if (!glob_match(model_num, ad->model_num)) { | 4264 | if (glob_match(model_num, ad->model_num)) { |
4331 | if (ad->model_rev == NULL) | 4265 | if (ad->model_rev == NULL) |
4332 | return ad->horkage; | 4266 | return ad->horkage; |
4333 | if (!glob_match(model_rev, ad->model_rev)) | 4267 | if (glob_match(model_rev, ad->model_rev)) |
4334 | return ad->horkage; | 4268 | return ad->horkage; |
4335 | } | 4269 | } |
4336 | ad++; | 4270 | ad++; |
diff --git a/drivers/base/Kconfig b/drivers/base/Kconfig index 88500fed3c7a..4e7f0ff83ae7 100644 --- a/drivers/base/Kconfig +++ b/drivers/base/Kconfig | |||
@@ -289,16 +289,6 @@ config CMA_ALIGNMENT | |||
289 | 289 | ||
290 | If unsure, leave the default value "8". | 290 | If unsure, leave the default value "8". |
291 | 291 | ||
292 | config CMA_AREAS | ||
293 | int "Maximum count of the CMA device-private areas" | ||
294 | default 7 | ||
295 | help | ||
296 | CMA allows to create CMA areas for particular devices. This parameter | ||
297 | sets the maximum number of such device private CMA areas in the | ||
298 | system. | ||
299 | |||
300 | If unsure, leave the default value "7". | ||
301 | |||
302 | endif | 292 | endif |
303 | 293 | ||
304 | endmenu | 294 | endmenu |
diff --git a/drivers/base/dma-contiguous.c b/drivers/base/dma-contiguous.c index 6467c919c509..6606abdf880c 100644 --- a/drivers/base/dma-contiguous.c +++ b/drivers/base/dma-contiguous.c | |||
@@ -24,23 +24,9 @@ | |||
24 | 24 | ||
25 | #include <linux/memblock.h> | 25 | #include <linux/memblock.h> |
26 | #include <linux/err.h> | 26 | #include <linux/err.h> |
27 | #include <linux/mm.h> | ||
28 | #include <linux/mutex.h> | ||
29 | #include <linux/page-isolation.h> | ||
30 | #include <linux/sizes.h> | 27 | #include <linux/sizes.h> |
31 | #include <linux/slab.h> | ||
32 | #include <linux/swap.h> | ||
33 | #include <linux/mm_types.h> | ||
34 | #include <linux/dma-contiguous.h> | 28 | #include <linux/dma-contiguous.h> |
35 | 29 | #include <linux/cma.h> | |
36 | struct cma { | ||
37 | unsigned long base_pfn; | ||
38 | unsigned long count; | ||
39 | unsigned long *bitmap; | ||
40 | struct mutex lock; | ||
41 | }; | ||
42 | |||
43 | struct cma *dma_contiguous_default_area; | ||
44 | 30 | ||
45 | #ifdef CONFIG_CMA_SIZE_MBYTES | 31 | #ifdef CONFIG_CMA_SIZE_MBYTES |
46 | #define CMA_SIZE_MBYTES CONFIG_CMA_SIZE_MBYTES | 32 | #define CMA_SIZE_MBYTES CONFIG_CMA_SIZE_MBYTES |
@@ -48,6 +34,8 @@ struct cma *dma_contiguous_default_area; | |||
48 | #define CMA_SIZE_MBYTES 0 | 34 | #define CMA_SIZE_MBYTES 0 |
49 | #endif | 35 | #endif |
50 | 36 | ||
37 | struct cma *dma_contiguous_default_area; | ||
38 | |||
51 | /* | 39 | /* |
52 | * Default global CMA area size can be defined in kernel's .config. | 40 | * Default global CMA area size can be defined in kernel's .config. |
53 | * This is useful mainly for distro maintainers to create a kernel | 41 | * This is useful mainly for distro maintainers to create a kernel |
@@ -154,65 +142,6 @@ void __init dma_contiguous_reserve(phys_addr_t limit) | |||
154 | } | 142 | } |
155 | } | 143 | } |
156 | 144 | ||
157 | static DEFINE_MUTEX(cma_mutex); | ||
158 | |||
159 | static int __init cma_activate_area(struct cma *cma) | ||
160 | { | ||
161 | int bitmap_size = BITS_TO_LONGS(cma->count) * sizeof(long); | ||
162 | unsigned long base_pfn = cma->base_pfn, pfn = base_pfn; | ||
163 | unsigned i = cma->count >> pageblock_order; | ||
164 | struct zone *zone; | ||
165 | |||
166 | cma->bitmap = kzalloc(bitmap_size, GFP_KERNEL); | ||
167 | |||
168 | if (!cma->bitmap) | ||
169 | return -ENOMEM; | ||
170 | |||
171 | WARN_ON_ONCE(!pfn_valid(pfn)); | ||
172 | zone = page_zone(pfn_to_page(pfn)); | ||
173 | |||
174 | do { | ||
175 | unsigned j; | ||
176 | base_pfn = pfn; | ||
177 | for (j = pageblock_nr_pages; j; --j, pfn++) { | ||
178 | WARN_ON_ONCE(!pfn_valid(pfn)); | ||
179 | /* | ||
180 | * alloc_contig_range requires the pfn range | ||
181 | * specified to be in the same zone. Make this | ||
182 | * simple by forcing the entire CMA resv range | ||
183 | * to be in the same zone. | ||
184 | */ | ||
185 | if (page_zone(pfn_to_page(pfn)) != zone) | ||
186 | goto err; | ||
187 | } | ||
188 | init_cma_reserved_pageblock(pfn_to_page(base_pfn)); | ||
189 | } while (--i); | ||
190 | |||
191 | mutex_init(&cma->lock); | ||
192 | return 0; | ||
193 | |||
194 | err: | ||
195 | kfree(cma->bitmap); | ||
196 | return -EINVAL; | ||
197 | } | ||
198 | |||
199 | static struct cma cma_areas[MAX_CMA_AREAS]; | ||
200 | static unsigned cma_area_count; | ||
201 | |||
202 | static int __init cma_init_reserved_areas(void) | ||
203 | { | ||
204 | int i; | ||
205 | |||
206 | for (i = 0; i < cma_area_count; i++) { | ||
207 | int ret = cma_activate_area(&cma_areas[i]); | ||
208 | if (ret) | ||
209 | return ret; | ||
210 | } | ||
211 | |||
212 | return 0; | ||
213 | } | ||
214 | core_initcall(cma_init_reserved_areas); | ||
215 | |||
216 | /** | 145 | /** |
217 | * dma_contiguous_reserve_area() - reserve custom contiguous area | 146 | * dma_contiguous_reserve_area() - reserve custom contiguous area |
218 | * @size: Size of the reserved area (in bytes), | 147 | * @size: Size of the reserved area (in bytes), |
@@ -234,72 +163,17 @@ int __init dma_contiguous_reserve_area(phys_addr_t size, phys_addr_t base, | |||
234 | phys_addr_t limit, struct cma **res_cma, | 163 | phys_addr_t limit, struct cma **res_cma, |
235 | bool fixed) | 164 | bool fixed) |
236 | { | 165 | { |
237 | struct cma *cma = &cma_areas[cma_area_count]; | 166 | int ret; |
238 | phys_addr_t alignment; | ||
239 | int ret = 0; | ||
240 | |||
241 | pr_debug("%s(size %lx, base %08lx, limit %08lx)\n", __func__, | ||
242 | (unsigned long)size, (unsigned long)base, | ||
243 | (unsigned long)limit); | ||
244 | |||
245 | /* Sanity checks */ | ||
246 | if (cma_area_count == ARRAY_SIZE(cma_areas)) { | ||
247 | pr_err("Not enough slots for CMA reserved regions!\n"); | ||
248 | return -ENOSPC; | ||
249 | } | ||
250 | |||
251 | if (!size) | ||
252 | return -EINVAL; | ||
253 | |||
254 | /* Sanitise input arguments */ | ||
255 | alignment = PAGE_SIZE << max(MAX_ORDER - 1, pageblock_order); | ||
256 | base = ALIGN(base, alignment); | ||
257 | size = ALIGN(size, alignment); | ||
258 | limit &= ~(alignment - 1); | ||
259 | |||
260 | /* Reserve memory */ | ||
261 | if (base && fixed) { | ||
262 | if (memblock_is_region_reserved(base, size) || | ||
263 | memblock_reserve(base, size) < 0) { | ||
264 | ret = -EBUSY; | ||
265 | goto err; | ||
266 | } | ||
267 | } else { | ||
268 | phys_addr_t addr = memblock_alloc_range(size, alignment, base, | ||
269 | limit); | ||
270 | if (!addr) { | ||
271 | ret = -ENOMEM; | ||
272 | goto err; | ||
273 | } else { | ||
274 | base = addr; | ||
275 | } | ||
276 | } | ||
277 | |||
278 | /* | ||
279 | * Each reserved area must be initialised later, when more kernel | ||
280 | * subsystems (like slab allocator) are available. | ||
281 | */ | ||
282 | cma->base_pfn = PFN_DOWN(base); | ||
283 | cma->count = size >> PAGE_SHIFT; | ||
284 | *res_cma = cma; | ||
285 | cma_area_count++; | ||
286 | 167 | ||
287 | pr_info("CMA: reserved %ld MiB at %08lx\n", (unsigned long)size / SZ_1M, | 168 | ret = cma_declare_contiguous(base, size, limit, 0, 0, fixed, res_cma); |
288 | (unsigned long)base); | 169 | if (ret) |
170 | return ret; | ||
289 | 171 | ||
290 | /* Architecture specific contiguous memory fixup. */ | 172 | /* Architecture specific contiguous memory fixup. */ |
291 | dma_contiguous_early_fixup(base, size); | 173 | dma_contiguous_early_fixup(cma_get_base(*res_cma), |
292 | return 0; | 174 | cma_get_size(*res_cma)); |
293 | err: | ||
294 | pr_err("CMA: failed to reserve %ld MiB\n", (unsigned long)size / SZ_1M); | ||
295 | return ret; | ||
296 | } | ||
297 | 175 | ||
298 | static void clear_cma_bitmap(struct cma *cma, unsigned long pfn, int count) | 176 | return 0; |
299 | { | ||
300 | mutex_lock(&cma->lock); | ||
301 | bitmap_clear(cma->bitmap, pfn - cma->base_pfn, count); | ||
302 | mutex_unlock(&cma->lock); | ||
303 | } | 177 | } |
304 | 178 | ||
305 | /** | 179 | /** |
@@ -316,62 +190,10 @@ static void clear_cma_bitmap(struct cma *cma, unsigned long pfn, int count) | |||
316 | struct page *dma_alloc_from_contiguous(struct device *dev, int count, | 190 | struct page *dma_alloc_from_contiguous(struct device *dev, int count, |
317 | unsigned int align) | 191 | unsigned int align) |
318 | { | 192 | { |
319 | unsigned long mask, pfn, pageno, start = 0; | ||
320 | struct cma *cma = dev_get_cma_area(dev); | ||
321 | struct page *page = NULL; | ||
322 | int ret; | ||
323 | |||
324 | if (!cma || !cma->count) | ||
325 | return NULL; | ||
326 | |||
327 | if (align > CONFIG_CMA_ALIGNMENT) | 193 | if (align > CONFIG_CMA_ALIGNMENT) |
328 | align = CONFIG_CMA_ALIGNMENT; | 194 | align = CONFIG_CMA_ALIGNMENT; |
329 | 195 | ||
330 | pr_debug("%s(cma %p, count %d, align %d)\n", __func__, (void *)cma, | 196 | return cma_alloc(dev_get_cma_area(dev), count, align); |
331 | count, align); | ||
332 | |||
333 | if (!count) | ||
334 | return NULL; | ||
335 | |||
336 | mask = (1 << align) - 1; | ||
337 | |||
338 | |||
339 | for (;;) { | ||
340 | mutex_lock(&cma->lock); | ||
341 | pageno = bitmap_find_next_zero_area(cma->bitmap, cma->count, | ||
342 | start, count, mask); | ||
343 | if (pageno >= cma->count) { | ||
344 | mutex_unlock(&cma->lock); | ||
345 | break; | ||
346 | } | ||
347 | bitmap_set(cma->bitmap, pageno, count); | ||
348 | /* | ||
349 | * It's safe to drop the lock here. We've marked this region for | ||
350 | * our exclusive use. If the migration fails we will take the | ||
351 | * lock again and unmark it. | ||
352 | */ | ||
353 | mutex_unlock(&cma->lock); | ||
354 | |||
355 | pfn = cma->base_pfn + pageno; | ||
356 | mutex_lock(&cma_mutex); | ||
357 | ret = alloc_contig_range(pfn, pfn + count, MIGRATE_CMA); | ||
358 | mutex_unlock(&cma_mutex); | ||
359 | if (ret == 0) { | ||
360 | page = pfn_to_page(pfn); | ||
361 | break; | ||
362 | } else if (ret != -EBUSY) { | ||
363 | clear_cma_bitmap(cma, pfn, count); | ||
364 | break; | ||
365 | } | ||
366 | clear_cma_bitmap(cma, pfn, count); | ||
367 | pr_debug("%s(): memory range at %p is busy, retrying\n", | ||
368 | __func__, pfn_to_page(pfn)); | ||
369 | /* try again with a bit different memory target */ | ||
370 | start = pageno + mask + 1; | ||
371 | } | ||
372 | |||
373 | pr_debug("%s(): returned %p\n", __func__, page); | ||
374 | return page; | ||
375 | } | 197 | } |
376 | 198 | ||
377 | /** | 199 | /** |
@@ -387,23 +209,5 @@ struct page *dma_alloc_from_contiguous(struct device *dev, int count, | |||
387 | bool dma_release_from_contiguous(struct device *dev, struct page *pages, | 209 | bool dma_release_from_contiguous(struct device *dev, struct page *pages, |
388 | int count) | 210 | int count) |
389 | { | 211 | { |
390 | struct cma *cma = dev_get_cma_area(dev); | 212 | return cma_release(dev_get_cma_area(dev), pages, count); |
391 | unsigned long pfn; | ||
392 | |||
393 | if (!cma || !pages) | ||
394 | return false; | ||
395 | |||
396 | pr_debug("%s(page %p)\n", __func__, (void *)pages); | ||
397 | |||
398 | pfn = page_to_pfn(pages); | ||
399 | |||
400 | if (pfn < cma->base_pfn || pfn >= cma->base_pfn + cma->count) | ||
401 | return false; | ||
402 | |||
403 | VM_BUG_ON(pfn + count > cma->base_pfn + cma->count); | ||
404 | |||
405 | free_contig_range(pfn, count); | ||
406 | clear_cma_bitmap(cma, pfn, count); | ||
407 | |||
408 | return true; | ||
409 | } | 213 | } |
diff --git a/drivers/base/memory.c b/drivers/base/memory.c index 89f752dd8465..a2e13e250bba 100644 --- a/drivers/base/memory.c +++ b/drivers/base/memory.c | |||
@@ -284,7 +284,7 @@ static int memory_subsys_online(struct device *dev) | |||
284 | * attribute and need to set the online_type. | 284 | * attribute and need to set the online_type. |
285 | */ | 285 | */ |
286 | if (mem->online_type < 0) | 286 | if (mem->online_type < 0) |
287 | mem->online_type = ONLINE_KEEP; | 287 | mem->online_type = MMOP_ONLINE_KEEP; |
288 | 288 | ||
289 | ret = memory_block_change_state(mem, MEM_ONLINE, MEM_OFFLINE); | 289 | ret = memory_block_change_state(mem, MEM_ONLINE, MEM_OFFLINE); |
290 | 290 | ||
@@ -315,23 +315,23 @@ store_mem_state(struct device *dev, | |||
315 | if (ret) | 315 | if (ret) |
316 | return ret; | 316 | return ret; |
317 | 317 | ||
318 | if (!strncmp(buf, "online_kernel", min_t(int, count, 13))) | 318 | if (sysfs_streq(buf, "online_kernel")) |
319 | online_type = ONLINE_KERNEL; | 319 | online_type = MMOP_ONLINE_KERNEL; |
320 | else if (!strncmp(buf, "online_movable", min_t(int, count, 14))) | 320 | else if (sysfs_streq(buf, "online_movable")) |
321 | online_type = ONLINE_MOVABLE; | 321 | online_type = MMOP_ONLINE_MOVABLE; |
322 | else if (!strncmp(buf, "online", min_t(int, count, 6))) | 322 | else if (sysfs_streq(buf, "online")) |
323 | online_type = ONLINE_KEEP; | 323 | online_type = MMOP_ONLINE_KEEP; |
324 | else if (!strncmp(buf, "offline", min_t(int, count, 7))) | 324 | else if (sysfs_streq(buf, "offline")) |
325 | online_type = -1; | 325 | online_type = MMOP_OFFLINE; |
326 | else { | 326 | else { |
327 | ret = -EINVAL; | 327 | ret = -EINVAL; |
328 | goto err; | 328 | goto err; |
329 | } | 329 | } |
330 | 330 | ||
331 | switch (online_type) { | 331 | switch (online_type) { |
332 | case ONLINE_KERNEL: | 332 | case MMOP_ONLINE_KERNEL: |
333 | case ONLINE_MOVABLE: | 333 | case MMOP_ONLINE_MOVABLE: |
334 | case ONLINE_KEEP: | 334 | case MMOP_ONLINE_KEEP: |
335 | /* | 335 | /* |
336 | * mem->online_type is not protected so there can be a | 336 | * mem->online_type is not protected so there can be a |
337 | * race here. However, when racing online, the first | 337 | * race here. However, when racing online, the first |
@@ -342,7 +342,7 @@ store_mem_state(struct device *dev, | |||
342 | mem->online_type = online_type; | 342 | mem->online_type = online_type; |
343 | ret = device_online(&mem->dev); | 343 | ret = device_online(&mem->dev); |
344 | break; | 344 | break; |
345 | case -1: | 345 | case MMOP_OFFLINE: |
346 | ret = device_offline(&mem->dev); | 346 | ret = device_offline(&mem->dev); |
347 | break; | 347 | break; |
348 | default: | 348 | default: |
@@ -406,7 +406,9 @@ memory_probe_store(struct device *dev, struct device_attribute *attr, | |||
406 | int i, ret; | 406 | int i, ret; |
407 | unsigned long pages_per_block = PAGES_PER_SECTION * sections_per_block; | 407 | unsigned long pages_per_block = PAGES_PER_SECTION * sections_per_block; |
408 | 408 | ||
409 | phys_addr = simple_strtoull(buf, NULL, 0); | 409 | ret = kstrtoull(buf, 0, &phys_addr); |
410 | if (ret) | ||
411 | return ret; | ||
410 | 412 | ||
411 | if (phys_addr & ((pages_per_block << PAGE_SHIFT) - 1)) | 413 | if (phys_addr & ((pages_per_block << PAGE_SHIFT) - 1)) |
412 | return -EINVAL; | 414 | return -EINVAL; |
diff --git a/drivers/base/node.c b/drivers/base/node.c index 8f7ed9933a7c..c6d3ae05f1ca 100644 --- a/drivers/base/node.c +++ b/drivers/base/node.c | |||
@@ -126,7 +126,7 @@ static ssize_t node_read_meminfo(struct device *dev, | |||
126 | nid, K(node_page_state(nid, NR_FILE_PAGES)), | 126 | nid, K(node_page_state(nid, NR_FILE_PAGES)), |
127 | nid, K(node_page_state(nid, NR_FILE_MAPPED)), | 127 | nid, K(node_page_state(nid, NR_FILE_MAPPED)), |
128 | nid, K(node_page_state(nid, NR_ANON_PAGES)), | 128 | nid, K(node_page_state(nid, NR_ANON_PAGES)), |
129 | nid, K(node_page_state(nid, NR_SHMEM)), | 129 | nid, K(i.sharedram), |
130 | nid, node_page_state(nid, NR_KERNEL_STACK) * | 130 | nid, node_page_state(nid, NR_KERNEL_STACK) * |
131 | THREAD_SIZE / 1024, | 131 | THREAD_SIZE / 1024, |
132 | nid, K(node_page_state(nid, NR_PAGETABLE)), | 132 | nid, K(node_page_state(nid, NR_PAGETABLE)), |
diff --git a/drivers/block/zram/zram_drv.c b/drivers/block/zram/zram_drv.c index 36e54be402df..dfa4024c448a 100644 --- a/drivers/block/zram/zram_drv.c +++ b/drivers/block/zram/zram_drv.c | |||
@@ -183,19 +183,32 @@ static ssize_t comp_algorithm_store(struct device *dev, | |||
183 | static int zram_test_flag(struct zram_meta *meta, u32 index, | 183 | static int zram_test_flag(struct zram_meta *meta, u32 index, |
184 | enum zram_pageflags flag) | 184 | enum zram_pageflags flag) |
185 | { | 185 | { |
186 | return meta->table[index].flags & BIT(flag); | 186 | return meta->table[index].value & BIT(flag); |
187 | } | 187 | } |
188 | 188 | ||
189 | static void zram_set_flag(struct zram_meta *meta, u32 index, | 189 | static void zram_set_flag(struct zram_meta *meta, u32 index, |
190 | enum zram_pageflags flag) | 190 | enum zram_pageflags flag) |
191 | { | 191 | { |
192 | meta->table[index].flags |= BIT(flag); | 192 | meta->table[index].value |= BIT(flag); |
193 | } | 193 | } |
194 | 194 | ||
195 | static void zram_clear_flag(struct zram_meta *meta, u32 index, | 195 | static void zram_clear_flag(struct zram_meta *meta, u32 index, |
196 | enum zram_pageflags flag) | 196 | enum zram_pageflags flag) |
197 | { | 197 | { |
198 | meta->table[index].flags &= ~BIT(flag); | 198 | meta->table[index].value &= ~BIT(flag); |
199 | } | ||
200 | |||
201 | static size_t zram_get_obj_size(struct zram_meta *meta, u32 index) | ||
202 | { | ||
203 | return meta->table[index].value & (BIT(ZRAM_FLAG_SHIFT) - 1); | ||
204 | } | ||
205 | |||
206 | static void zram_set_obj_size(struct zram_meta *meta, | ||
207 | u32 index, size_t size) | ||
208 | { | ||
209 | unsigned long flags = meta->table[index].value >> ZRAM_FLAG_SHIFT; | ||
210 | |||
211 | meta->table[index].value = (flags << ZRAM_FLAG_SHIFT) | size; | ||
199 | } | 212 | } |
200 | 213 | ||
201 | static inline int is_partial_io(struct bio_vec *bvec) | 214 | static inline int is_partial_io(struct bio_vec *bvec) |
@@ -255,7 +268,6 @@ static struct zram_meta *zram_meta_alloc(u64 disksize) | |||
255 | goto free_table; | 268 | goto free_table; |
256 | } | 269 | } |
257 | 270 | ||
258 | rwlock_init(&meta->tb_lock); | ||
259 | return meta; | 271 | return meta; |
260 | 272 | ||
261 | free_table: | 273 | free_table: |
@@ -304,7 +316,12 @@ static void handle_zero_page(struct bio_vec *bvec) | |||
304 | flush_dcache_page(page); | 316 | flush_dcache_page(page); |
305 | } | 317 | } |
306 | 318 | ||
307 | /* NOTE: caller should hold meta->tb_lock with write-side */ | 319 | |
320 | /* | ||
321 | * To protect concurrent access to the same index entry, | ||
322 | * caller should hold this table index entry's bit_spinlock to | ||
323 | * indicate this index entry is accessing. | ||
324 | */ | ||
308 | static void zram_free_page(struct zram *zram, size_t index) | 325 | static void zram_free_page(struct zram *zram, size_t index) |
309 | { | 326 | { |
310 | struct zram_meta *meta = zram->meta; | 327 | struct zram_meta *meta = zram->meta; |
@@ -324,11 +341,12 @@ static void zram_free_page(struct zram *zram, size_t index) | |||
324 | 341 | ||
325 | zs_free(meta->mem_pool, handle); | 342 | zs_free(meta->mem_pool, handle); |
326 | 343 | ||
327 | atomic64_sub(meta->table[index].size, &zram->stats.compr_data_size); | 344 | atomic64_sub(zram_get_obj_size(meta, index), |
345 | &zram->stats.compr_data_size); | ||
328 | atomic64_dec(&zram->stats.pages_stored); | 346 | atomic64_dec(&zram->stats.pages_stored); |
329 | 347 | ||
330 | meta->table[index].handle = 0; | 348 | meta->table[index].handle = 0; |
331 | meta->table[index].size = 0; | 349 | zram_set_obj_size(meta, index, 0); |
332 | } | 350 | } |
333 | 351 | ||
334 | static int zram_decompress_page(struct zram *zram, char *mem, u32 index) | 352 | static int zram_decompress_page(struct zram *zram, char *mem, u32 index) |
@@ -337,14 +355,14 @@ static int zram_decompress_page(struct zram *zram, char *mem, u32 index) | |||
337 | unsigned char *cmem; | 355 | unsigned char *cmem; |
338 | struct zram_meta *meta = zram->meta; | 356 | struct zram_meta *meta = zram->meta; |
339 | unsigned long handle; | 357 | unsigned long handle; |
340 | u16 size; | 358 | size_t size; |
341 | 359 | ||
342 | read_lock(&meta->tb_lock); | 360 | bit_spin_lock(ZRAM_ACCESS, &meta->table[index].value); |
343 | handle = meta->table[index].handle; | 361 | handle = meta->table[index].handle; |
344 | size = meta->table[index].size; | 362 | size = zram_get_obj_size(meta, index); |
345 | 363 | ||
346 | if (!handle || zram_test_flag(meta, index, ZRAM_ZERO)) { | 364 | if (!handle || zram_test_flag(meta, index, ZRAM_ZERO)) { |
347 | read_unlock(&meta->tb_lock); | 365 | bit_spin_unlock(ZRAM_ACCESS, &meta->table[index].value); |
348 | clear_page(mem); | 366 | clear_page(mem); |
349 | return 0; | 367 | return 0; |
350 | } | 368 | } |
@@ -355,7 +373,7 @@ static int zram_decompress_page(struct zram *zram, char *mem, u32 index) | |||
355 | else | 373 | else |
356 | ret = zcomp_decompress(zram->comp, cmem, size, mem); | 374 | ret = zcomp_decompress(zram->comp, cmem, size, mem); |
357 | zs_unmap_object(meta->mem_pool, handle); | 375 | zs_unmap_object(meta->mem_pool, handle); |
358 | read_unlock(&meta->tb_lock); | 376 | bit_spin_unlock(ZRAM_ACCESS, &meta->table[index].value); |
359 | 377 | ||
360 | /* Should NEVER happen. Return bio error if it does. */ | 378 | /* Should NEVER happen. Return bio error if it does. */ |
361 | if (unlikely(ret)) { | 379 | if (unlikely(ret)) { |
@@ -376,14 +394,14 @@ static int zram_bvec_read(struct zram *zram, struct bio_vec *bvec, | |||
376 | struct zram_meta *meta = zram->meta; | 394 | struct zram_meta *meta = zram->meta; |
377 | page = bvec->bv_page; | 395 | page = bvec->bv_page; |
378 | 396 | ||
379 | read_lock(&meta->tb_lock); | 397 | bit_spin_lock(ZRAM_ACCESS, &meta->table[index].value); |
380 | if (unlikely(!meta->table[index].handle) || | 398 | if (unlikely(!meta->table[index].handle) || |
381 | zram_test_flag(meta, index, ZRAM_ZERO)) { | 399 | zram_test_flag(meta, index, ZRAM_ZERO)) { |
382 | read_unlock(&meta->tb_lock); | 400 | bit_spin_unlock(ZRAM_ACCESS, &meta->table[index].value); |
383 | handle_zero_page(bvec); | 401 | handle_zero_page(bvec); |
384 | return 0; | 402 | return 0; |
385 | } | 403 | } |
386 | read_unlock(&meta->tb_lock); | 404 | bit_spin_unlock(ZRAM_ACCESS, &meta->table[index].value); |
387 | 405 | ||
388 | if (is_partial_io(bvec)) | 406 | if (is_partial_io(bvec)) |
389 | /* Use a temporary buffer to decompress the page */ | 407 | /* Use a temporary buffer to decompress the page */ |
@@ -461,10 +479,10 @@ static int zram_bvec_write(struct zram *zram, struct bio_vec *bvec, u32 index, | |||
461 | if (page_zero_filled(uncmem)) { | 479 | if (page_zero_filled(uncmem)) { |
462 | kunmap_atomic(user_mem); | 480 | kunmap_atomic(user_mem); |
463 | /* Free memory associated with this sector now. */ | 481 | /* Free memory associated with this sector now. */ |
464 | write_lock(&zram->meta->tb_lock); | 482 | bit_spin_lock(ZRAM_ACCESS, &meta->table[index].value); |
465 | zram_free_page(zram, index); | 483 | zram_free_page(zram, index); |
466 | zram_set_flag(meta, index, ZRAM_ZERO); | 484 | zram_set_flag(meta, index, ZRAM_ZERO); |
467 | write_unlock(&zram->meta->tb_lock); | 485 | bit_spin_unlock(ZRAM_ACCESS, &meta->table[index].value); |
468 | 486 | ||
469 | atomic64_inc(&zram->stats.zero_pages); | 487 | atomic64_inc(&zram->stats.zero_pages); |
470 | ret = 0; | 488 | ret = 0; |
@@ -514,12 +532,12 @@ static int zram_bvec_write(struct zram *zram, struct bio_vec *bvec, u32 index, | |||
514 | * Free memory associated with this sector | 532 | * Free memory associated with this sector |
515 | * before overwriting unused sectors. | 533 | * before overwriting unused sectors. |
516 | */ | 534 | */ |
517 | write_lock(&zram->meta->tb_lock); | 535 | bit_spin_lock(ZRAM_ACCESS, &meta->table[index].value); |
518 | zram_free_page(zram, index); | 536 | zram_free_page(zram, index); |
519 | 537 | ||
520 | meta->table[index].handle = handle; | 538 | meta->table[index].handle = handle; |
521 | meta->table[index].size = clen; | 539 | zram_set_obj_size(meta, index, clen); |
522 | write_unlock(&zram->meta->tb_lock); | 540 | bit_spin_unlock(ZRAM_ACCESS, &meta->table[index].value); |
523 | 541 | ||
524 | /* Update stats */ | 542 | /* Update stats */ |
525 | atomic64_add(clen, &zram->stats.compr_data_size); | 543 | atomic64_add(clen, &zram->stats.compr_data_size); |
@@ -560,6 +578,7 @@ static void zram_bio_discard(struct zram *zram, u32 index, | |||
560 | int offset, struct bio *bio) | 578 | int offset, struct bio *bio) |
561 | { | 579 | { |
562 | size_t n = bio->bi_iter.bi_size; | 580 | size_t n = bio->bi_iter.bi_size; |
581 | struct zram_meta *meta = zram->meta; | ||
563 | 582 | ||
564 | /* | 583 | /* |
565 | * zram manages data in physical block size units. Because logical block | 584 | * zram manages data in physical block size units. Because logical block |
@@ -580,13 +599,9 @@ static void zram_bio_discard(struct zram *zram, u32 index, | |||
580 | } | 599 | } |
581 | 600 | ||
582 | while (n >= PAGE_SIZE) { | 601 | while (n >= PAGE_SIZE) { |
583 | /* | 602 | bit_spin_lock(ZRAM_ACCESS, &meta->table[index].value); |
584 | * Discard request can be large so the lock hold times could be | ||
585 | * lengthy. So take the lock once per page. | ||
586 | */ | ||
587 | write_lock(&zram->meta->tb_lock); | ||
588 | zram_free_page(zram, index); | 603 | zram_free_page(zram, index); |
589 | write_unlock(&zram->meta->tb_lock); | 604 | bit_spin_unlock(ZRAM_ACCESS, &meta->table[index].value); |
590 | index++; | 605 | index++; |
591 | n -= PAGE_SIZE; | 606 | n -= PAGE_SIZE; |
592 | } | 607 | } |
@@ -821,9 +836,9 @@ static void zram_slot_free_notify(struct block_device *bdev, | |||
821 | zram = bdev->bd_disk->private_data; | 836 | zram = bdev->bd_disk->private_data; |
822 | meta = zram->meta; | 837 | meta = zram->meta; |
823 | 838 | ||
824 | write_lock(&meta->tb_lock); | 839 | bit_spin_lock(ZRAM_ACCESS, &meta->table[index].value); |
825 | zram_free_page(zram, index); | 840 | zram_free_page(zram, index); |
826 | write_unlock(&meta->tb_lock); | 841 | bit_spin_unlock(ZRAM_ACCESS, &meta->table[index].value); |
827 | atomic64_inc(&zram->stats.notify_free); | 842 | atomic64_inc(&zram->stats.notify_free); |
828 | } | 843 | } |
829 | 844 | ||
diff --git a/drivers/block/zram/zram_drv.h b/drivers/block/zram/zram_drv.h index 7f21c145e317..5b0afde729cd 100644 --- a/drivers/block/zram/zram_drv.h +++ b/drivers/block/zram/zram_drv.h | |||
@@ -43,7 +43,6 @@ static const size_t max_zpage_size = PAGE_SIZE / 4 * 3; | |||
43 | /*-- End of configurable params */ | 43 | /*-- End of configurable params */ |
44 | 44 | ||
45 | #define SECTOR_SHIFT 9 | 45 | #define SECTOR_SHIFT 9 |
46 | #define SECTOR_SIZE (1 << SECTOR_SHIFT) | ||
47 | #define SECTORS_PER_PAGE_SHIFT (PAGE_SHIFT - SECTOR_SHIFT) | 46 | #define SECTORS_PER_PAGE_SHIFT (PAGE_SHIFT - SECTOR_SHIFT) |
48 | #define SECTORS_PER_PAGE (1 << SECTORS_PER_PAGE_SHIFT) | 47 | #define SECTORS_PER_PAGE (1 << SECTORS_PER_PAGE_SHIFT) |
49 | #define ZRAM_LOGICAL_BLOCK_SHIFT 12 | 48 | #define ZRAM_LOGICAL_BLOCK_SHIFT 12 |
@@ -51,10 +50,24 @@ static const size_t max_zpage_size = PAGE_SIZE / 4 * 3; | |||
51 | #define ZRAM_SECTOR_PER_LOGICAL_BLOCK \ | 50 | #define ZRAM_SECTOR_PER_LOGICAL_BLOCK \ |
52 | (1 << (ZRAM_LOGICAL_BLOCK_SHIFT - SECTOR_SHIFT)) | 51 | (1 << (ZRAM_LOGICAL_BLOCK_SHIFT - SECTOR_SHIFT)) |
53 | 52 | ||
54 | /* Flags for zram pages (table[page_no].flags) */ | 53 | |
54 | /* | ||
55 | * The lower ZRAM_FLAG_SHIFT bits of table.value is for | ||
56 | * object size (excluding header), the higher bits is for | ||
57 | * zram_pageflags. | ||
58 | * | ||
59 | * zram is mainly used for memory efficiency so we want to keep memory | ||
60 | * footprint small so we can squeeze size and flags into a field. | ||
61 | * The lower ZRAM_FLAG_SHIFT bits is for object size (excluding header), | ||
62 | * the higher bits is for zram_pageflags. | ||
63 | */ | ||
64 | #define ZRAM_FLAG_SHIFT 24 | ||
65 | |||
66 | /* Flags for zram pages (table[page_no].value) */ | ||
55 | enum zram_pageflags { | 67 | enum zram_pageflags { |
56 | /* Page consists entirely of zeros */ | 68 | /* Page consists entirely of zeros */ |
57 | ZRAM_ZERO, | 69 | ZRAM_ZERO = ZRAM_FLAG_SHIFT + 1, |
70 | ZRAM_ACCESS, /* page in now accessed */ | ||
58 | 71 | ||
59 | __NR_ZRAM_PAGEFLAGS, | 72 | __NR_ZRAM_PAGEFLAGS, |
60 | }; | 73 | }; |
@@ -62,11 +75,10 @@ enum zram_pageflags { | |||
62 | /*-- Data structures */ | 75 | /*-- Data structures */ |
63 | 76 | ||
64 | /* Allocated for each disk page */ | 77 | /* Allocated for each disk page */ |
65 | struct table { | 78 | struct zram_table_entry { |
66 | unsigned long handle; | 79 | unsigned long handle; |
67 | u16 size; /* object size (excluding header) */ | 80 | unsigned long value; |
68 | u8 flags; | 81 | }; |
69 | } __aligned(4); | ||
70 | 82 | ||
71 | struct zram_stats { | 83 | struct zram_stats { |
72 | atomic64_t compr_data_size; /* compressed size of pages stored */ | 84 | atomic64_t compr_data_size; /* compressed size of pages stored */ |
@@ -81,8 +93,7 @@ struct zram_stats { | |||
81 | }; | 93 | }; |
82 | 94 | ||
83 | struct zram_meta { | 95 | struct zram_meta { |
84 | rwlock_t tb_lock; /* protect table */ | 96 | struct zram_table_entry *table; |
85 | struct table *table; | ||
86 | struct zs_pool *mem_pool; | 97 | struct zs_pool *mem_pool; |
87 | }; | 98 | }; |
88 | 99 | ||
diff --git a/drivers/firmware/memmap.c b/drivers/firmware/memmap.c index 17cf96c45f2b..79f18e6d9c4f 100644 --- a/drivers/firmware/memmap.c +++ b/drivers/firmware/memmap.c | |||
@@ -286,7 +286,11 @@ int __meminit firmware_map_add_hotplug(u64 start, u64 end, const char *type) | |||
286 | { | 286 | { |
287 | struct firmware_map_entry *entry; | 287 | struct firmware_map_entry *entry; |
288 | 288 | ||
289 | entry = firmware_map_find_entry_bootmem(start, end, type); | 289 | entry = firmware_map_find_entry(start, end - 1, type); |
290 | if (entry) | ||
291 | return 0; | ||
292 | |||
293 | entry = firmware_map_find_entry_bootmem(start, end - 1, type); | ||
290 | if (!entry) { | 294 | if (!entry) { |
291 | entry = kzalloc(sizeof(struct firmware_map_entry), GFP_ATOMIC); | 295 | entry = kzalloc(sizeof(struct firmware_map_entry), GFP_ATOMIC); |
292 | if (!entry) | 296 | if (!entry) |
diff --git a/drivers/gpu/drm/drm_hashtab.c b/drivers/gpu/drm/drm_hashtab.c index 7e4bae760e27..c3b80fd65d62 100644 --- a/drivers/gpu/drm/drm_hashtab.c +++ b/drivers/gpu/drm/drm_hashtab.c | |||
@@ -125,7 +125,7 @@ int drm_ht_insert_item(struct drm_open_hash *ht, struct drm_hash_item *item) | |||
125 | parent = &entry->head; | 125 | parent = &entry->head; |
126 | } | 126 | } |
127 | if (parent) { | 127 | if (parent) { |
128 | hlist_add_after_rcu(parent, &item->head); | 128 | hlist_add_behind_rcu(&item->head, parent); |
129 | } else { | 129 | } else { |
130 | hlist_add_head_rcu(&item->head, h_list); | 130 | hlist_add_head_rcu(&item->head, h_list); |
131 | } | 131 | } |
diff --git a/drivers/hwmon/asus_atk0110.c b/drivers/hwmon/asus_atk0110.c index ae208f612198..cccef87963e0 100644 --- a/drivers/hwmon/asus_atk0110.c +++ b/drivers/hwmon/asus_atk0110.c | |||
@@ -688,7 +688,7 @@ static int atk_debugfs_gitm_get(void *p, u64 *val) | |||
688 | DEFINE_SIMPLE_ATTRIBUTE(atk_debugfs_gitm, | 688 | DEFINE_SIMPLE_ATTRIBUTE(atk_debugfs_gitm, |
689 | atk_debugfs_gitm_get, | 689 | atk_debugfs_gitm_get, |
690 | NULL, | 690 | NULL, |
691 | "0x%08llx\n") | 691 | "0x%08llx\n"); |
692 | 692 | ||
693 | static int atk_acpi_print(char *buf, size_t sz, union acpi_object *obj) | 693 | static int atk_acpi_print(char *buf, size_t sz, union acpi_object *obj) |
694 | { | 694 | { |
diff --git a/drivers/lguest/core.c b/drivers/lguest/core.c index 0bf1e4edf04d..6590558d1d31 100644 --- a/drivers/lguest/core.c +++ b/drivers/lguest/core.c | |||
@@ -42,7 +42,6 @@ DEFINE_MUTEX(lguest_lock); | |||
42 | static __init int map_switcher(void) | 42 | static __init int map_switcher(void) |
43 | { | 43 | { |
44 | int i, err; | 44 | int i, err; |
45 | struct page **pagep; | ||
46 | 45 | ||
47 | /* | 46 | /* |
48 | * Map the Switcher in to high memory. | 47 | * Map the Switcher in to high memory. |
@@ -110,11 +109,9 @@ static __init int map_switcher(void) | |||
110 | * This code actually sets up the pages we've allocated to appear at | 109 | * This code actually sets up the pages we've allocated to appear at |
111 | * switcher_addr. map_vm_area() takes the vma we allocated above, the | 110 | * switcher_addr. map_vm_area() takes the vma we allocated above, the |
112 | * kind of pages we're mapping (kernel pages), and a pointer to our | 111 | * kind of pages we're mapping (kernel pages), and a pointer to our |
113 | * array of struct pages. It increments that pointer, but we don't | 112 | * array of struct pages. |
114 | * care. | ||
115 | */ | 113 | */ |
116 | pagep = lg_switcher_pages; | 114 | err = map_vm_area(switcher_vma, PAGE_KERNEL_EXEC, lg_switcher_pages); |
117 | err = map_vm_area(switcher_vma, PAGE_KERNEL_EXEC, &pagep); | ||
118 | if (err) { | 115 | if (err) { |
119 | printk("lguest: map_vm_area failed: %i\n", err); | 116 | printk("lguest: map_vm_area failed: %i\n", err); |
120 | goto free_vma; | 117 | goto free_vma; |
diff --git a/drivers/net/ethernet/intel/i40e/i40e_ethtool.c b/drivers/net/ethernet/intel/i40e/i40e_ethtool.c index 681a9e81ff51..e8ba7470700a 100644 --- a/drivers/net/ethernet/intel/i40e/i40e_ethtool.c +++ b/drivers/net/ethernet/intel/i40e/i40e_ethtool.c | |||
@@ -1948,7 +1948,7 @@ static int i40e_update_ethtool_fdir_entry(struct i40e_vsi *vsi, | |||
1948 | 1948 | ||
1949 | /* add filter to the list */ | 1949 | /* add filter to the list */ |
1950 | if (parent) | 1950 | if (parent) |
1951 | hlist_add_after(&parent->fdir_node, &input->fdir_node); | 1951 | hlist_add_behind(&input->fdir_node, &parent->fdir_node); |
1952 | else | 1952 | else |
1953 | hlist_add_head(&input->fdir_node, | 1953 | hlist_add_head(&input->fdir_node, |
1954 | &pf->fdir_filter_list); | 1954 | &pf->fdir_filter_list); |
diff --git a/drivers/net/ethernet/intel/ixgbe/ixgbe_ethtool.c b/drivers/net/ethernet/intel/ixgbe/ixgbe_ethtool.c index 94a1c07efeb0..e4100b5737b6 100644 --- a/drivers/net/ethernet/intel/ixgbe/ixgbe_ethtool.c +++ b/drivers/net/ethernet/intel/ixgbe/ixgbe_ethtool.c | |||
@@ -2517,7 +2517,7 @@ static int ixgbe_update_ethtool_fdir_entry(struct ixgbe_adapter *adapter, | |||
2517 | 2517 | ||
2518 | /* add filter to the list */ | 2518 | /* add filter to the list */ |
2519 | if (parent) | 2519 | if (parent) |
2520 | hlist_add_after(&parent->fdir_node, &input->fdir_node); | 2520 | hlist_add_behind(&input->fdir_node, &parent->fdir_node); |
2521 | else | 2521 | else |
2522 | hlist_add_head(&input->fdir_node, | 2522 | hlist_add_head(&input->fdir_node, |
2523 | &adapter->fdir_filter_list); | 2523 | &adapter->fdir_filter_list); |
diff --git a/drivers/staging/android/binder.c b/drivers/staging/android/binder.c index 02b0379ae550..4f34dc0095b5 100644 --- a/drivers/staging/android/binder.c +++ b/drivers/staging/android/binder.c | |||
@@ -585,7 +585,6 @@ static int binder_update_page_range(struct binder_proc *proc, int allocate, | |||
585 | 585 | ||
586 | for (page_addr = start; page_addr < end; page_addr += PAGE_SIZE) { | 586 | for (page_addr = start; page_addr < end; page_addr += PAGE_SIZE) { |
587 | int ret; | 587 | int ret; |
588 | struct page **page_array_ptr; | ||
589 | 588 | ||
590 | page = &proc->pages[(page_addr - proc->buffer) / PAGE_SIZE]; | 589 | page = &proc->pages[(page_addr - proc->buffer) / PAGE_SIZE]; |
591 | 590 | ||
@@ -598,8 +597,7 @@ static int binder_update_page_range(struct binder_proc *proc, int allocate, | |||
598 | } | 597 | } |
599 | tmp_area.addr = page_addr; | 598 | tmp_area.addr = page_addr; |
600 | tmp_area.size = PAGE_SIZE + PAGE_SIZE /* guard page? */; | 599 | tmp_area.size = PAGE_SIZE + PAGE_SIZE /* guard page? */; |
601 | page_array_ptr = page; | 600 | ret = map_vm_area(&tmp_area, PAGE_KERNEL, page); |
602 | ret = map_vm_area(&tmp_area, PAGE_KERNEL, &page_array_ptr); | ||
603 | if (ret) { | 601 | if (ret) { |
604 | pr_err("%d: binder_alloc_buf failed to map page at %p in kernel\n", | 602 | pr_err("%d: binder_alloc_buf failed to map page at %p in kernel\n", |
605 | proc->pid, page_addr); | 603 | proc->pid, page_addr); |
diff --git a/drivers/staging/lustre/lustre/libcfs/hash.c b/drivers/staging/lustre/lustre/libcfs/hash.c index 5dde79418297..8ef1deb59d4a 100644 --- a/drivers/staging/lustre/lustre/libcfs/hash.c +++ b/drivers/staging/lustre/lustre/libcfs/hash.c | |||
@@ -351,7 +351,7 @@ cfs_hash_dh_hnode_add(struct cfs_hash *hs, struct cfs_hash_bd *bd, | |||
351 | cfs_hash_dhead_t, dh_head); | 351 | cfs_hash_dhead_t, dh_head); |
352 | 352 | ||
353 | if (dh->dh_tail != NULL) /* not empty */ | 353 | if (dh->dh_tail != NULL) /* not empty */ |
354 | hlist_add_after(dh->dh_tail, hnode); | 354 | hlist_add_behind(hnode, dh->dh_tail); |
355 | else /* empty list */ | 355 | else /* empty list */ |
356 | hlist_add_head(hnode, &dh->dh_head); | 356 | hlist_add_head(hnode, &dh->dh_head); |
357 | dh->dh_tail = hnode; | 357 | dh->dh_tail = hnode; |
@@ -406,7 +406,7 @@ cfs_hash_dd_hnode_add(struct cfs_hash *hs, struct cfs_hash_bd *bd, | |||
406 | cfs_hash_dhead_dep_t, dd_head); | 406 | cfs_hash_dhead_dep_t, dd_head); |
407 | 407 | ||
408 | if (dh->dd_tail != NULL) /* not empty */ | 408 | if (dh->dd_tail != NULL) /* not empty */ |
409 | hlist_add_after(dh->dd_tail, hnode); | 409 | hlist_add_behind(hnode, dh->dd_tail); |
410 | else /* empty list */ | 410 | else /* empty list */ |
411 | hlist_add_head(hnode, &dh->dd_head); | 411 | hlist_add_head(hnode, &dh->dd_head); |
412 | dh->dd_tail = hnode; | 412 | dh->dd_tail = hnode; |
diff --git a/drivers/tty/sysrq.c b/drivers/tty/sysrq.c index 454b65898e2c..42bad18c66c9 100644 --- a/drivers/tty/sysrq.c +++ b/drivers/tty/sysrq.c | |||
@@ -355,7 +355,7 @@ static struct sysrq_key_op sysrq_term_op = { | |||
355 | 355 | ||
356 | static void moom_callback(struct work_struct *ignored) | 356 | static void moom_callback(struct work_struct *ignored) |
357 | { | 357 | { |
358 | out_of_memory(node_zonelist(first_online_node, GFP_KERNEL), GFP_KERNEL, | 358 | out_of_memory(node_zonelist(first_memory_node, GFP_KERNEL), GFP_KERNEL, |
359 | 0, NULL, true); | 359 | 0, NULL, true); |
360 | } | 360 | } |
361 | 361 | ||
diff --git a/fs/fscache/main.c b/fs/fscache/main.c index a31b83c5cbd9..b39d487ccfb0 100644 --- a/fs/fscache/main.c +++ b/fs/fscache/main.c | |||
@@ -67,7 +67,7 @@ static int fscache_max_active_sysctl(struct ctl_table *table, int write, | |||
67 | return ret; | 67 | return ret; |
68 | } | 68 | } |
69 | 69 | ||
70 | struct ctl_table fscache_sysctls[] = { | 70 | static struct ctl_table fscache_sysctls[] = { |
71 | { | 71 | { |
72 | .procname = "object_max_active", | 72 | .procname = "object_max_active", |
73 | .data = &fscache_object_max_active, | 73 | .data = &fscache_object_max_active, |
@@ -87,7 +87,7 @@ struct ctl_table fscache_sysctls[] = { | |||
87 | {} | 87 | {} |
88 | }; | 88 | }; |
89 | 89 | ||
90 | struct ctl_table fscache_sysctls_root[] = { | 90 | static struct ctl_table fscache_sysctls_root[] = { |
91 | { | 91 | { |
92 | .procname = "fscache", | 92 | .procname = "fscache", |
93 | .mode = 0555, | 93 | .mode = 0555, |
diff --git a/fs/logfs/readwrite.c b/fs/logfs/readwrite.c index 48140315f627..380d86e1ab45 100644 --- a/fs/logfs/readwrite.c +++ b/fs/logfs/readwrite.c | |||
@@ -1019,11 +1019,11 @@ static int __logfs_is_valid_block(struct inode *inode, u64 bix, u64 ofs) | |||
1019 | /** | 1019 | /** |
1020 | * logfs_is_valid_block - check whether this block is still valid | 1020 | * logfs_is_valid_block - check whether this block is still valid |
1021 | * | 1021 | * |
1022 | * @sb - superblock | 1022 | * @sb: superblock |
1023 | * @ofs - block physical offset | 1023 | * @ofs: block physical offset |
1024 | * @ino - block inode number | 1024 | * @ino: block inode number |
1025 | * @bix - block index | 1025 | * @bix: block index |
1026 | * @level - block level | 1026 | * @gc_level: block level |
1027 | * | 1027 | * |
1028 | * Returns 0 if the block is invalid, 1 if it is valid and 2 if it will | 1028 | * Returns 0 if the block is invalid, 1 if it is valid and 2 if it will |
1029 | * become invalid once the journal is written. | 1029 | * become invalid once the journal is written. |
@@ -2226,10 +2226,9 @@ void btree_write_block(struct logfs_block *block) | |||
2226 | * | 2226 | * |
2227 | * @inode: parent inode (ifile or directory) | 2227 | * @inode: parent inode (ifile or directory) |
2228 | * @buf: object to write (inode or dentry) | 2228 | * @buf: object to write (inode or dentry) |
2229 | * @n: object size | 2229 | * @count: object size |
2230 | * @_pos: object number (file position in blocks/objects) | 2230 | * @bix: block index |
2231 | * @flags: write flags | 2231 | * @flags: write flags |
2232 | * @lock: 0 if write lock is already taken, 1 otherwise | ||
2233 | * @shadow_tree: shadow below this inode | 2232 | * @shadow_tree: shadow below this inode |
2234 | * | 2233 | * |
2235 | * FIXME: All caller of this put a 200-300 byte variable on the stack, | 2234 | * FIXME: All caller of this put a 200-300 byte variable on the stack, |
diff --git a/fs/namespace.c b/fs/namespace.c index 182bc41cd887..2a1447c946e7 100644 --- a/fs/namespace.c +++ b/fs/namespace.c | |||
@@ -798,7 +798,7 @@ static void commit_tree(struct mount *mnt, struct mount *shadows) | |||
798 | list_splice(&head, n->list.prev); | 798 | list_splice(&head, n->list.prev); |
799 | 799 | ||
800 | if (shadows) | 800 | if (shadows) |
801 | hlist_add_after_rcu(&shadows->mnt_hash, &mnt->mnt_hash); | 801 | hlist_add_behind_rcu(&mnt->mnt_hash, &shadows->mnt_hash); |
802 | else | 802 | else |
803 | hlist_add_head_rcu(&mnt->mnt_hash, | 803 | hlist_add_head_rcu(&mnt->mnt_hash, |
804 | m_hash(&parent->mnt, mnt->mnt_mountpoint)); | 804 | m_hash(&parent->mnt, mnt->mnt_mountpoint)); |
diff --git a/fs/notify/fanotify/fanotify.c b/fs/notify/fanotify/fanotify.c index ee9cb3795c2b..30d3addfad75 100644 --- a/fs/notify/fanotify/fanotify.c +++ b/fs/notify/fanotify/fanotify.c | |||
@@ -70,8 +70,15 @@ static int fanotify_get_response(struct fsnotify_group *group, | |||
70 | wait_event(group->fanotify_data.access_waitq, event->response || | 70 | wait_event(group->fanotify_data.access_waitq, event->response || |
71 | atomic_read(&group->fanotify_data.bypass_perm)); | 71 | atomic_read(&group->fanotify_data.bypass_perm)); |
72 | 72 | ||
73 | if (!event->response) /* bypass_perm set */ | 73 | if (!event->response) { /* bypass_perm set */ |
74 | /* | ||
75 | * Event was canceled because group is being destroyed. Remove | ||
76 | * it from group's event list because we are responsible for | ||
77 | * freeing the permission event. | ||
78 | */ | ||
79 | fsnotify_remove_event(group, &event->fae.fse); | ||
74 | return 0; | 80 | return 0; |
81 | } | ||
75 | 82 | ||
76 | /* userspace responded, convert to something usable */ | 83 | /* userspace responded, convert to something usable */ |
77 | switch (event->response) { | 84 | switch (event->response) { |
@@ -210,7 +217,7 @@ static int fanotify_handle_event(struct fsnotify_group *group, | |||
210 | return -ENOMEM; | 217 | return -ENOMEM; |
211 | 218 | ||
212 | fsn_event = &event->fse; | 219 | fsn_event = &event->fse; |
213 | ret = fsnotify_add_notify_event(group, fsn_event, fanotify_merge); | 220 | ret = fsnotify_add_event(group, fsn_event, fanotify_merge); |
214 | if (ret) { | 221 | if (ret) { |
215 | /* Permission events shouldn't be merged */ | 222 | /* Permission events shouldn't be merged */ |
216 | BUG_ON(ret == 1 && mask & FAN_ALL_PERM_EVENTS); | 223 | BUG_ON(ret == 1 && mask & FAN_ALL_PERM_EVENTS); |
diff --git a/fs/notify/fanotify/fanotify_user.c b/fs/notify/fanotify/fanotify_user.c index 3fdc8a3e1134..b13992a41bd9 100644 --- a/fs/notify/fanotify/fanotify_user.c +++ b/fs/notify/fanotify/fanotify_user.c | |||
@@ -66,7 +66,7 @@ static struct fsnotify_event *get_one_event(struct fsnotify_group *group, | |||
66 | 66 | ||
67 | /* held the notification_mutex the whole time, so this is the | 67 | /* held the notification_mutex the whole time, so this is the |
68 | * same event we peeked above */ | 68 | * same event we peeked above */ |
69 | return fsnotify_remove_notify_event(group); | 69 | return fsnotify_remove_first_event(group); |
70 | } | 70 | } |
71 | 71 | ||
72 | static int create_fd(struct fsnotify_group *group, | 72 | static int create_fd(struct fsnotify_group *group, |
@@ -359,6 +359,11 @@ static int fanotify_release(struct inode *ignored, struct file *file) | |||
359 | #ifdef CONFIG_FANOTIFY_ACCESS_PERMISSIONS | 359 | #ifdef CONFIG_FANOTIFY_ACCESS_PERMISSIONS |
360 | struct fanotify_perm_event_info *event, *next; | 360 | struct fanotify_perm_event_info *event, *next; |
361 | 361 | ||
362 | /* | ||
363 | * There may be still new events arriving in the notification queue | ||
364 | * but since userspace cannot use fanotify fd anymore, no event can | ||
365 | * enter or leave access_list by now. | ||
366 | */ | ||
362 | spin_lock(&group->fanotify_data.access_lock); | 367 | spin_lock(&group->fanotify_data.access_lock); |
363 | 368 | ||
364 | atomic_inc(&group->fanotify_data.bypass_perm); | 369 | atomic_inc(&group->fanotify_data.bypass_perm); |
@@ -373,6 +378,13 @@ static int fanotify_release(struct inode *ignored, struct file *file) | |||
373 | } | 378 | } |
374 | spin_unlock(&group->fanotify_data.access_lock); | 379 | spin_unlock(&group->fanotify_data.access_lock); |
375 | 380 | ||
381 | /* | ||
382 | * Since bypass_perm is set, newly queued events will not wait for | ||
383 | * access response. Wake up the already sleeping ones now. | ||
384 | * synchronize_srcu() in fsnotify_destroy_group() will wait for all | ||
385 | * processes sleeping in fanotify_handle_event() waiting for access | ||
386 | * response and thus also for all permission events to be freed. | ||
387 | */ | ||
376 | wake_up(&group->fanotify_data.access_waitq); | 388 | wake_up(&group->fanotify_data.access_waitq); |
377 | #endif | 389 | #endif |
378 | 390 | ||
diff --git a/fs/notify/inode_mark.c b/fs/notify/inode_mark.c index 74825be65b7b..9ce062218de9 100644 --- a/fs/notify/inode_mark.c +++ b/fs/notify/inode_mark.c | |||
@@ -232,7 +232,7 @@ int fsnotify_add_inode_mark(struct fsnotify_mark *mark, | |||
232 | 232 | ||
233 | BUG_ON(last == NULL); | 233 | BUG_ON(last == NULL); |
234 | /* mark should be the last entry. last is the current last entry */ | 234 | /* mark should be the last entry. last is the current last entry */ |
235 | hlist_add_after_rcu(&last->i.i_list, &mark->i.i_list); | 235 | hlist_add_behind_rcu(&mark->i.i_list, &last->i.i_list); |
236 | out: | 236 | out: |
237 | fsnotify_recalc_inode_mask_locked(inode); | 237 | fsnotify_recalc_inode_mask_locked(inode); |
238 | spin_unlock(&inode->i_lock); | 238 | spin_unlock(&inode->i_lock); |
diff --git a/fs/notify/inotify/inotify_fsnotify.c b/fs/notify/inotify/inotify_fsnotify.c index 43ab1e1a07a2..0f88bc0b4e6c 100644 --- a/fs/notify/inotify/inotify_fsnotify.c +++ b/fs/notify/inotify/inotify_fsnotify.c | |||
@@ -108,7 +108,7 @@ int inotify_handle_event(struct fsnotify_group *group, | |||
108 | if (len) | 108 | if (len) |
109 | strcpy(event->name, file_name); | 109 | strcpy(event->name, file_name); |
110 | 110 | ||
111 | ret = fsnotify_add_notify_event(group, fsn_event, inotify_merge); | 111 | ret = fsnotify_add_event(group, fsn_event, inotify_merge); |
112 | if (ret) { | 112 | if (ret) { |
113 | /* Our event wasn't used in the end. Free it. */ | 113 | /* Our event wasn't used in the end. Free it. */ |
114 | fsnotify_destroy_event(group, fsn_event); | 114 | fsnotify_destroy_event(group, fsn_event); |
diff --git a/fs/notify/inotify/inotify_user.c b/fs/notify/inotify/inotify_user.c index cc423a30a0c8..daf76652fe58 100644 --- a/fs/notify/inotify/inotify_user.c +++ b/fs/notify/inotify/inotify_user.c | |||
@@ -149,7 +149,7 @@ static struct fsnotify_event *get_one_event(struct fsnotify_group *group, | |||
149 | if (fsnotify_notify_queue_is_empty(group)) | 149 | if (fsnotify_notify_queue_is_empty(group)) |
150 | return NULL; | 150 | return NULL; |
151 | 151 | ||
152 | event = fsnotify_peek_notify_event(group); | 152 | event = fsnotify_peek_first_event(group); |
153 | 153 | ||
154 | pr_debug("%s: group=%p event=%p\n", __func__, group, event); | 154 | pr_debug("%s: group=%p event=%p\n", __func__, group, event); |
155 | 155 | ||
@@ -159,7 +159,7 @@ static struct fsnotify_event *get_one_event(struct fsnotify_group *group, | |||
159 | 159 | ||
160 | /* held the notification_mutex the whole time, so this is the | 160 | /* held the notification_mutex the whole time, so this is the |
161 | * same event we peeked above */ | 161 | * same event we peeked above */ |
162 | fsnotify_remove_notify_event(group); | 162 | fsnotify_remove_first_event(group); |
163 | 163 | ||
164 | return event; | 164 | return event; |
165 | } | 165 | } |
diff --git a/fs/notify/notification.c b/fs/notify/notification.c index 1e58402171a5..a95d8e037aeb 100644 --- a/fs/notify/notification.c +++ b/fs/notify/notification.c | |||
@@ -73,7 +73,8 @@ void fsnotify_destroy_event(struct fsnotify_group *group, | |||
73 | /* Overflow events are per-group and we don't want to free them */ | 73 | /* Overflow events are per-group and we don't want to free them */ |
74 | if (!event || event->mask == FS_Q_OVERFLOW) | 74 | if (!event || event->mask == FS_Q_OVERFLOW) |
75 | return; | 75 | return; |
76 | 76 | /* If the event is still queued, we have a problem... */ | |
77 | WARN_ON(!list_empty(&event->list)); | ||
77 | group->ops->free_event(event); | 78 | group->ops->free_event(event); |
78 | } | 79 | } |
79 | 80 | ||
@@ -83,10 +84,10 @@ void fsnotify_destroy_event(struct fsnotify_group *group, | |||
83 | * added to the queue, 1 if the event was merged with some other queued event, | 84 | * added to the queue, 1 if the event was merged with some other queued event, |
84 | * 2 if the queue of events has overflown. | 85 | * 2 if the queue of events has overflown. |
85 | */ | 86 | */ |
86 | int fsnotify_add_notify_event(struct fsnotify_group *group, | 87 | int fsnotify_add_event(struct fsnotify_group *group, |
87 | struct fsnotify_event *event, | 88 | struct fsnotify_event *event, |
88 | int (*merge)(struct list_head *, | 89 | int (*merge)(struct list_head *, |
89 | struct fsnotify_event *)) | 90 | struct fsnotify_event *)) |
90 | { | 91 | { |
91 | int ret = 0; | 92 | int ret = 0; |
92 | struct list_head *list = &group->notification_list; | 93 | struct list_head *list = &group->notification_list; |
@@ -125,10 +126,25 @@ queue: | |||
125 | } | 126 | } |
126 | 127 | ||
127 | /* | 128 | /* |
129 | * Remove @event from group's notification queue. It is the responsibility of | ||
130 | * the caller to destroy the event. | ||
131 | */ | ||
132 | void fsnotify_remove_event(struct fsnotify_group *group, | ||
133 | struct fsnotify_event *event) | ||
134 | { | ||
135 | mutex_lock(&group->notification_mutex); | ||
136 | if (!list_empty(&event->list)) { | ||
137 | list_del_init(&event->list); | ||
138 | group->q_len--; | ||
139 | } | ||
140 | mutex_unlock(&group->notification_mutex); | ||
141 | } | ||
142 | |||
143 | /* | ||
128 | * Remove and return the first event from the notification list. It is the | 144 | * Remove and return the first event from the notification list. It is the |
129 | * responsibility of the caller to destroy the obtained event | 145 | * responsibility of the caller to destroy the obtained event |
130 | */ | 146 | */ |
131 | struct fsnotify_event *fsnotify_remove_notify_event(struct fsnotify_group *group) | 147 | struct fsnotify_event *fsnotify_remove_first_event(struct fsnotify_group *group) |
132 | { | 148 | { |
133 | struct fsnotify_event *event; | 149 | struct fsnotify_event *event; |
134 | 150 | ||
@@ -140,7 +156,7 @@ struct fsnotify_event *fsnotify_remove_notify_event(struct fsnotify_group *group | |||
140 | struct fsnotify_event, list); | 156 | struct fsnotify_event, list); |
141 | /* | 157 | /* |
142 | * We need to init list head for the case of overflow event so that | 158 | * We need to init list head for the case of overflow event so that |
143 | * check in fsnotify_add_notify_events() works | 159 | * check in fsnotify_add_event() works |
144 | */ | 160 | */ |
145 | list_del_init(&event->list); | 161 | list_del_init(&event->list); |
146 | group->q_len--; | 162 | group->q_len--; |
@@ -149,9 +165,10 @@ struct fsnotify_event *fsnotify_remove_notify_event(struct fsnotify_group *group | |||
149 | } | 165 | } |
150 | 166 | ||
151 | /* | 167 | /* |
152 | * This will not remove the event, that must be done with fsnotify_remove_notify_event() | 168 | * This will not remove the event, that must be done with |
169 | * fsnotify_remove_first_event() | ||
153 | */ | 170 | */ |
154 | struct fsnotify_event *fsnotify_peek_notify_event(struct fsnotify_group *group) | 171 | struct fsnotify_event *fsnotify_peek_first_event(struct fsnotify_group *group) |
155 | { | 172 | { |
156 | BUG_ON(!mutex_is_locked(&group->notification_mutex)); | 173 | BUG_ON(!mutex_is_locked(&group->notification_mutex)); |
157 | 174 | ||
@@ -169,7 +186,7 @@ void fsnotify_flush_notify(struct fsnotify_group *group) | |||
169 | 186 | ||
170 | mutex_lock(&group->notification_mutex); | 187 | mutex_lock(&group->notification_mutex); |
171 | while (!fsnotify_notify_queue_is_empty(group)) { | 188 | while (!fsnotify_notify_queue_is_empty(group)) { |
172 | event = fsnotify_remove_notify_event(group); | 189 | event = fsnotify_remove_first_event(group); |
173 | fsnotify_destroy_event(group, event); | 190 | fsnotify_destroy_event(group, event); |
174 | } | 191 | } |
175 | mutex_unlock(&group->notification_mutex); | 192 | mutex_unlock(&group->notification_mutex); |
diff --git a/fs/notify/vfsmount_mark.c b/fs/notify/vfsmount_mark.c index 68ca5a8704b5..ac851e8376b1 100644 --- a/fs/notify/vfsmount_mark.c +++ b/fs/notify/vfsmount_mark.c | |||
@@ -191,7 +191,7 @@ int fsnotify_add_vfsmount_mark(struct fsnotify_mark *mark, | |||
191 | 191 | ||
192 | BUG_ON(last == NULL); | 192 | BUG_ON(last == NULL); |
193 | /* mark should be the last entry. last is the current last entry */ | 193 | /* mark should be the last entry. last is the current last entry */ |
194 | hlist_add_after_rcu(&last->m.m_list, &mark->m.m_list); | 194 | hlist_add_behind_rcu(&mark->m.m_list, &last->m.m_list); |
195 | out: | 195 | out: |
196 | fsnotify_recalc_vfsmount_mask_locked(mnt); | 196 | fsnotify_recalc_vfsmount_mask_locked(mnt); |
197 | spin_unlock(&mnt->mnt_root->d_lock); | 197 | spin_unlock(&mnt->mnt_root->d_lock); |
diff --git a/fs/ntfs/file.c b/fs/ntfs/file.c index 5c9e2c81cb11..f5ec1ce7a532 100644 --- a/fs/ntfs/file.c +++ b/fs/ntfs/file.c | |||
@@ -74,8 +74,6 @@ static int ntfs_file_open(struct inode *vi, struct file *filp) | |||
74 | * ntfs_attr_extend_initialized - extend the initialized size of an attribute | 74 | * ntfs_attr_extend_initialized - extend the initialized size of an attribute |
75 | * @ni: ntfs inode of the attribute to extend | 75 | * @ni: ntfs inode of the attribute to extend |
76 | * @new_init_size: requested new initialized size in bytes | 76 | * @new_init_size: requested new initialized size in bytes |
77 | * @cached_page: store any allocated but unused page here | ||
78 | * @lru_pvec: lru-buffering pagevec of the caller | ||
79 | * | 77 | * |
80 | * Extend the initialized size of an attribute described by the ntfs inode @ni | 78 | * Extend the initialized size of an attribute described by the ntfs inode @ni |
81 | * to @new_init_size bytes. This involves zeroing any non-sparse space between | 79 | * to @new_init_size bytes. This involves zeroing any non-sparse space between |
@@ -395,7 +393,6 @@ static inline void ntfs_fault_in_pages_readable_iovec(const struct iovec *iov, | |||
395 | * @nr_pages: number of page cache pages to obtain | 393 | * @nr_pages: number of page cache pages to obtain |
396 | * @pages: array of pages in which to return the obtained page cache pages | 394 | * @pages: array of pages in which to return the obtained page cache pages |
397 | * @cached_page: allocated but as yet unused page | 395 | * @cached_page: allocated but as yet unused page |
398 | * @lru_pvec: lru-buffering pagevec of caller | ||
399 | * | 396 | * |
400 | * Obtain @nr_pages locked page cache pages from the mapping @mapping and | 397 | * Obtain @nr_pages locked page cache pages from the mapping @mapping and |
401 | * starting at index @index. | 398 | * starting at index @index. |
diff --git a/fs/ocfs2/alloc.c b/fs/ocfs2/alloc.c index 9d8fcf2f3b94..a93bf9892256 100644 --- a/fs/ocfs2/alloc.c +++ b/fs/ocfs2/alloc.c | |||
@@ -4961,6 +4961,15 @@ leftright: | |||
4961 | 4961 | ||
4962 | el = path_leaf_el(path); | 4962 | el = path_leaf_el(path); |
4963 | split_index = ocfs2_search_extent_list(el, cpos); | 4963 | split_index = ocfs2_search_extent_list(el, cpos); |
4964 | if (split_index == -1) { | ||
4965 | ocfs2_error(ocfs2_metadata_cache_get_super(et->et_ci), | ||
4966 | "Owner %llu has an extent at cpos %u " | ||
4967 | "which can no longer be found.\n", | ||
4968 | (unsigned long long)ocfs2_metadata_cache_owner(et->et_ci), | ||
4969 | cpos); | ||
4970 | ret = -EROFS; | ||
4971 | goto out; | ||
4972 | } | ||
4964 | goto leftright; | 4973 | goto leftright; |
4965 | } | 4974 | } |
4966 | out: | 4975 | out: |
@@ -5135,7 +5144,7 @@ int ocfs2_change_extent_flag(handle_t *handle, | |||
5135 | el = path_leaf_el(left_path); | 5144 | el = path_leaf_el(left_path); |
5136 | 5145 | ||
5137 | index = ocfs2_search_extent_list(el, cpos); | 5146 | index = ocfs2_search_extent_list(el, cpos); |
5138 | if (index == -1 || index >= le16_to_cpu(el->l_next_free_rec)) { | 5147 | if (index == -1) { |
5139 | ocfs2_error(sb, | 5148 | ocfs2_error(sb, |
5140 | "Owner %llu has an extent at cpos %u which can no " | 5149 | "Owner %llu has an extent at cpos %u which can no " |
5141 | "longer be found.\n", | 5150 | "longer be found.\n", |
@@ -5491,7 +5500,7 @@ int ocfs2_remove_extent(handle_t *handle, | |||
5491 | 5500 | ||
5492 | el = path_leaf_el(path); | 5501 | el = path_leaf_el(path); |
5493 | index = ocfs2_search_extent_list(el, cpos); | 5502 | index = ocfs2_search_extent_list(el, cpos); |
5494 | if (index == -1 || index >= le16_to_cpu(el->l_next_free_rec)) { | 5503 | if (index == -1) { |
5495 | ocfs2_error(ocfs2_metadata_cache_get_super(et->et_ci), | 5504 | ocfs2_error(ocfs2_metadata_cache_get_super(et->et_ci), |
5496 | "Owner %llu has an extent at cpos %u which can no " | 5505 | "Owner %llu has an extent at cpos %u which can no " |
5497 | "longer be found.\n", | 5506 | "longer be found.\n", |
@@ -5557,7 +5566,7 @@ int ocfs2_remove_extent(handle_t *handle, | |||
5557 | 5566 | ||
5558 | el = path_leaf_el(path); | 5567 | el = path_leaf_el(path); |
5559 | index = ocfs2_search_extent_list(el, cpos); | 5568 | index = ocfs2_search_extent_list(el, cpos); |
5560 | if (index == -1 || index >= le16_to_cpu(el->l_next_free_rec)) { | 5569 | if (index == -1) { |
5561 | ocfs2_error(ocfs2_metadata_cache_get_super(et->et_ci), | 5570 | ocfs2_error(ocfs2_metadata_cache_get_super(et->et_ci), |
5562 | "Owner %llu: split at cpos %u lost record.", | 5571 | "Owner %llu: split at cpos %u lost record.", |
5563 | (unsigned long long)ocfs2_metadata_cache_owner(et->et_ci), | 5572 | (unsigned long long)ocfs2_metadata_cache_owner(et->et_ci), |
diff --git a/fs/ocfs2/dlm/dlmdomain.c b/fs/ocfs2/dlm/dlmdomain.c index 39efc5057a36..3fcf205ee900 100644 --- a/fs/ocfs2/dlm/dlmdomain.c +++ b/fs/ocfs2/dlm/dlmdomain.c | |||
@@ -1923,12 +1923,11 @@ static int dlm_join_domain(struct dlm_ctxt *dlm) | |||
1923 | goto bail; | 1923 | goto bail; |
1924 | } | 1924 | } |
1925 | 1925 | ||
1926 | if (total_backoff > | 1926 | if (total_backoff > DLM_JOIN_TIMEOUT_MSECS) { |
1927 | msecs_to_jiffies(DLM_JOIN_TIMEOUT_MSECS)) { | ||
1928 | status = -ERESTARTSYS; | 1927 | status = -ERESTARTSYS; |
1929 | mlog(ML_NOTICE, "Timed out joining dlm domain " | 1928 | mlog(ML_NOTICE, "Timed out joining dlm domain " |
1930 | "%s after %u msecs\n", dlm->name, | 1929 | "%s after %u msecs\n", dlm->name, |
1931 | jiffies_to_msecs(total_backoff)); | 1930 | total_backoff); |
1932 | goto bail; | 1931 | goto bail; |
1933 | } | 1932 | } |
1934 | 1933 | ||
diff --git a/fs/ocfs2/dlm/dlmmaster.c b/fs/ocfs2/dlm/dlmmaster.c index 82abf0cc9a12..3ec906ef5d9a 100644 --- a/fs/ocfs2/dlm/dlmmaster.c +++ b/fs/ocfs2/dlm/dlmmaster.c | |||
@@ -2405,6 +2405,10 @@ static int dlm_is_lockres_migrateable(struct dlm_ctxt *dlm, | |||
2405 | if (res->state & DLM_LOCK_RES_MIGRATING) | 2405 | if (res->state & DLM_LOCK_RES_MIGRATING) |
2406 | return 0; | 2406 | return 0; |
2407 | 2407 | ||
2408 | /* delay migration when the lockres is in RECOCERING state */ | ||
2409 | if (res->state & DLM_LOCK_RES_RECOVERING) | ||
2410 | return 0; | ||
2411 | |||
2408 | if (res->owner != dlm->node_num) | 2412 | if (res->owner != dlm->node_num) |
2409 | return 0; | 2413 | return 0; |
2410 | 2414 | ||
diff --git a/fs/ocfs2/move_extents.c b/fs/ocfs2/move_extents.c index 599eb4c4c8be..6219aaadeb08 100644 --- a/fs/ocfs2/move_extents.c +++ b/fs/ocfs2/move_extents.c | |||
@@ -98,7 +98,7 @@ static int __ocfs2_move_extent(handle_t *handle, | |||
98 | el = path_leaf_el(path); | 98 | el = path_leaf_el(path); |
99 | 99 | ||
100 | index = ocfs2_search_extent_list(el, cpos); | 100 | index = ocfs2_search_extent_list(el, cpos); |
101 | if (index == -1 || index >= le16_to_cpu(el->l_next_free_rec)) { | 101 | if (index == -1) { |
102 | ocfs2_error(inode->i_sb, | 102 | ocfs2_error(inode->i_sb, |
103 | "Inode %llu has an extent at cpos %u which can no " | 103 | "Inode %llu has an extent at cpos %u which can no " |
104 | "longer be found.\n", | 104 | "longer be found.\n", |
diff --git a/fs/ocfs2/refcounttree.c b/fs/ocfs2/refcounttree.c index 636aab69ead5..d81f6e2a97f5 100644 --- a/fs/ocfs2/refcounttree.c +++ b/fs/ocfs2/refcounttree.c | |||
@@ -3109,7 +3109,7 @@ static int ocfs2_clear_ext_refcount(handle_t *handle, | |||
3109 | el = path_leaf_el(path); | 3109 | el = path_leaf_el(path); |
3110 | 3110 | ||
3111 | index = ocfs2_search_extent_list(el, cpos); | 3111 | index = ocfs2_search_extent_list(el, cpos); |
3112 | if (index == -1 || index >= le16_to_cpu(el->l_next_free_rec)) { | 3112 | if (index == -1) { |
3113 | ocfs2_error(sb, | 3113 | ocfs2_error(sb, |
3114 | "Inode %llu has an extent at cpos %u which can no " | 3114 | "Inode %llu has an extent at cpos %u which can no " |
3115 | "longer be found.\n", | 3115 | "longer be found.\n", |
diff --git a/fs/ocfs2/slot_map.c b/fs/ocfs2/slot_map.c index 1424c151cccc..a88b2a4fcc85 100644 --- a/fs/ocfs2/slot_map.c +++ b/fs/ocfs2/slot_map.c | |||
@@ -382,7 +382,7 @@ static int ocfs2_map_slot_buffers(struct ocfs2_super *osb, | |||
382 | 382 | ||
383 | trace_ocfs2_map_slot_buffers(bytes, si->si_blocks); | 383 | trace_ocfs2_map_slot_buffers(bytes, si->si_blocks); |
384 | 384 | ||
385 | si->si_bh = kzalloc(sizeof(struct buffer_head *) * si->si_blocks, | 385 | si->si_bh = kcalloc(si->si_blocks, sizeof(struct buffer_head *), |
386 | GFP_KERNEL); | 386 | GFP_KERNEL); |
387 | if (!si->si_bh) { | 387 | if (!si->si_bh) { |
388 | status = -ENOMEM; | 388 | status = -ENOMEM; |
diff --git a/fs/proc/meminfo.c b/fs/proc/meminfo.c index 7445af0b1aa3..aa1eee06420f 100644 --- a/fs/proc/meminfo.c +++ b/fs/proc/meminfo.c | |||
@@ -168,7 +168,7 @@ static int meminfo_proc_show(struct seq_file *m, void *v) | |||
168 | K(global_page_state(NR_WRITEBACK)), | 168 | K(global_page_state(NR_WRITEBACK)), |
169 | K(global_page_state(NR_ANON_PAGES)), | 169 | K(global_page_state(NR_ANON_PAGES)), |
170 | K(global_page_state(NR_FILE_MAPPED)), | 170 | K(global_page_state(NR_FILE_MAPPED)), |
171 | K(global_page_state(NR_SHMEM)), | 171 | K(i.sharedram), |
172 | K(global_page_state(NR_SLAB_RECLAIMABLE) + | 172 | K(global_page_state(NR_SLAB_RECLAIMABLE) + |
173 | global_page_state(NR_SLAB_UNRECLAIMABLE)), | 173 | global_page_state(NR_SLAB_UNRECLAIMABLE)), |
174 | K(global_page_state(NR_SLAB_RECLAIMABLE)), | 174 | K(global_page_state(NR_SLAB_RECLAIMABLE)), |
diff --git a/fs/proc/task_mmu.c b/fs/proc/task_mmu.c index cfa63ee92c96..dfc791c42d64 100644 --- a/fs/proc/task_mmu.c +++ b/fs/proc/task_mmu.c | |||
@@ -925,15 +925,30 @@ static int pagemap_pte_hole(unsigned long start, unsigned long end, | |||
925 | struct mm_walk *walk) | 925 | struct mm_walk *walk) |
926 | { | 926 | { |
927 | struct pagemapread *pm = walk->private; | 927 | struct pagemapread *pm = walk->private; |
928 | unsigned long addr; | 928 | unsigned long addr = start; |
929 | int err = 0; | 929 | int err = 0; |
930 | pagemap_entry_t pme = make_pme(PM_NOT_PRESENT(pm->v2)); | ||
931 | 930 | ||
932 | for (addr = start; addr < end; addr += PAGE_SIZE) { | 931 | while (addr < end) { |
933 | err = add_to_pagemap(addr, &pme, pm); | 932 | struct vm_area_struct *vma = find_vma(walk->mm, addr); |
934 | if (err) | 933 | pagemap_entry_t pme = make_pme(PM_NOT_PRESENT(pm->v2)); |
935 | break; | 934 | unsigned long vm_end; |
935 | |||
936 | if (!vma) { | ||
937 | vm_end = end; | ||
938 | } else { | ||
939 | vm_end = min(end, vma->vm_end); | ||
940 | if (vma->vm_flags & VM_SOFTDIRTY) | ||
941 | pme.pme |= PM_STATUS2(pm->v2, __PM_SOFT_DIRTY); | ||
942 | } | ||
943 | |||
944 | for (; addr < vm_end; addr += PAGE_SIZE) { | ||
945 | err = add_to_pagemap(addr, &pme, pm); | ||
946 | if (err) | ||
947 | goto out; | ||
948 | } | ||
936 | } | 949 | } |
950 | |||
951 | out: | ||
937 | return err; | 952 | return err; |
938 | } | 953 | } |
939 | 954 | ||
diff --git a/fs/squashfs/file_direct.c b/fs/squashfs/file_direct.c index 62a0de6632e1..43e7a7eddac0 100644 --- a/fs/squashfs/file_direct.c +++ b/fs/squashfs/file_direct.c | |||
@@ -44,7 +44,7 @@ int squashfs_readpage_block(struct page *target_page, u64 block, int bsize) | |||
44 | 44 | ||
45 | pages = end_index - start_index + 1; | 45 | pages = end_index - start_index + 1; |
46 | 46 | ||
47 | page = kmalloc(sizeof(void *) * pages, GFP_KERNEL); | 47 | page = kmalloc_array(pages, sizeof(void *), GFP_KERNEL); |
48 | if (page == NULL) | 48 | if (page == NULL) |
49 | return res; | 49 | return res; |
50 | 50 | ||
diff --git a/fs/squashfs/super.c b/fs/squashfs/super.c index 031c8d67fd51..5056babe00df 100644 --- a/fs/squashfs/super.c +++ b/fs/squashfs/super.c | |||
@@ -27,6 +27,8 @@ | |||
27 | * the filesystem. | 27 | * the filesystem. |
28 | */ | 28 | */ |
29 | 29 | ||
30 | #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt | ||
31 | |||
30 | #include <linux/fs.h> | 32 | #include <linux/fs.h> |
31 | #include <linux/vfs.h> | 33 | #include <linux/vfs.h> |
32 | #include <linux/slab.h> | 34 | #include <linux/slab.h> |
@@ -448,8 +450,7 @@ static int __init init_squashfs_fs(void) | |||
448 | return err; | 450 | return err; |
449 | } | 451 | } |
450 | 452 | ||
451 | printk(KERN_INFO "squashfs: version 4.0 (2009/01/31) " | 453 | pr_info("version 4.0 (2009/01/31) Phillip Lougher\n"); |
452 | "Phillip Lougher\n"); | ||
453 | 454 | ||
454 | return 0; | 455 | return 0; |
455 | } | 456 | } |
diff --git a/include/linux/bitmap.h b/include/linux/bitmap.h index 7ad634501e48..e1c8d080c427 100644 --- a/include/linux/bitmap.h +++ b/include/linux/bitmap.h | |||
@@ -88,32 +88,32 @@ | |||
88 | * lib/bitmap.c provides these functions: | 88 | * lib/bitmap.c provides these functions: |
89 | */ | 89 | */ |
90 | 90 | ||
91 | extern int __bitmap_empty(const unsigned long *bitmap, int bits); | 91 | extern int __bitmap_empty(const unsigned long *bitmap, unsigned int nbits); |
92 | extern int __bitmap_full(const unsigned long *bitmap, int bits); | 92 | extern int __bitmap_full(const unsigned long *bitmap, unsigned int nbits); |
93 | extern int __bitmap_equal(const unsigned long *bitmap1, | 93 | extern int __bitmap_equal(const unsigned long *bitmap1, |
94 | const unsigned long *bitmap2, int bits); | 94 | const unsigned long *bitmap2, unsigned int nbits); |
95 | extern void __bitmap_complement(unsigned long *dst, const unsigned long *src, | 95 | extern void __bitmap_complement(unsigned long *dst, const unsigned long *src, |
96 | int bits); | 96 | unsigned int nbits); |
97 | extern void __bitmap_shift_right(unsigned long *dst, | 97 | extern void __bitmap_shift_right(unsigned long *dst, |
98 | const unsigned long *src, int shift, int bits); | 98 | const unsigned long *src, int shift, int bits); |
99 | extern void __bitmap_shift_left(unsigned long *dst, | 99 | extern void __bitmap_shift_left(unsigned long *dst, |
100 | const unsigned long *src, int shift, int bits); | 100 | const unsigned long *src, int shift, int bits); |
101 | extern int __bitmap_and(unsigned long *dst, const unsigned long *bitmap1, | 101 | extern int __bitmap_and(unsigned long *dst, const unsigned long *bitmap1, |
102 | const unsigned long *bitmap2, int bits); | 102 | const unsigned long *bitmap2, unsigned int nbits); |
103 | extern void __bitmap_or(unsigned long *dst, const unsigned long *bitmap1, | 103 | extern void __bitmap_or(unsigned long *dst, const unsigned long *bitmap1, |
104 | const unsigned long *bitmap2, int bits); | 104 | const unsigned long *bitmap2, unsigned int nbits); |
105 | extern void __bitmap_xor(unsigned long *dst, const unsigned long *bitmap1, | 105 | extern void __bitmap_xor(unsigned long *dst, const unsigned long *bitmap1, |
106 | const unsigned long *bitmap2, int bits); | 106 | const unsigned long *bitmap2, unsigned int nbits); |
107 | extern int __bitmap_andnot(unsigned long *dst, const unsigned long *bitmap1, | 107 | extern int __bitmap_andnot(unsigned long *dst, const unsigned long *bitmap1, |
108 | const unsigned long *bitmap2, int bits); | 108 | const unsigned long *bitmap2, unsigned int nbits); |
109 | extern int __bitmap_intersects(const unsigned long *bitmap1, | 109 | extern int __bitmap_intersects(const unsigned long *bitmap1, |
110 | const unsigned long *bitmap2, int bits); | 110 | const unsigned long *bitmap2, unsigned int nbits); |
111 | extern int __bitmap_subset(const unsigned long *bitmap1, | 111 | extern int __bitmap_subset(const unsigned long *bitmap1, |
112 | const unsigned long *bitmap2, int bits); | 112 | const unsigned long *bitmap2, unsigned int nbits); |
113 | extern int __bitmap_weight(const unsigned long *bitmap, int bits); | 113 | extern int __bitmap_weight(const unsigned long *bitmap, unsigned int nbits); |
114 | 114 | ||
115 | extern void bitmap_set(unsigned long *map, int i, int len); | 115 | extern void bitmap_set(unsigned long *map, unsigned int start, int len); |
116 | extern void bitmap_clear(unsigned long *map, int start, int nr); | 116 | extern void bitmap_clear(unsigned long *map, unsigned int start, int len); |
117 | extern unsigned long bitmap_find_next_zero_area(unsigned long *map, | 117 | extern unsigned long bitmap_find_next_zero_area(unsigned long *map, |
118 | unsigned long size, | 118 | unsigned long size, |
119 | unsigned long start, | 119 | unsigned long start, |
@@ -140,9 +140,9 @@ extern void bitmap_onto(unsigned long *dst, const unsigned long *orig, | |||
140 | const unsigned long *relmap, int bits); | 140 | const unsigned long *relmap, int bits); |
141 | extern void bitmap_fold(unsigned long *dst, const unsigned long *orig, | 141 | extern void bitmap_fold(unsigned long *dst, const unsigned long *orig, |
142 | int sz, int bits); | 142 | int sz, int bits); |
143 | extern int bitmap_find_free_region(unsigned long *bitmap, int bits, int order); | 143 | extern int bitmap_find_free_region(unsigned long *bitmap, unsigned int bits, int order); |
144 | extern void bitmap_release_region(unsigned long *bitmap, int pos, int order); | 144 | extern void bitmap_release_region(unsigned long *bitmap, unsigned int pos, int order); |
145 | extern int bitmap_allocate_region(unsigned long *bitmap, int pos, int order); | 145 | extern int bitmap_allocate_region(unsigned long *bitmap, unsigned int pos, int order); |
146 | extern void bitmap_copy_le(void *dst, const unsigned long *src, int nbits); | 146 | extern void bitmap_copy_le(void *dst, const unsigned long *src, int nbits); |
147 | extern int bitmap_ord_to_pos(const unsigned long *bitmap, int n, int bits); | 147 | extern int bitmap_ord_to_pos(const unsigned long *bitmap, int n, int bits); |
148 | 148 | ||
@@ -188,15 +188,15 @@ static inline void bitmap_copy(unsigned long *dst, const unsigned long *src, | |||
188 | } | 188 | } |
189 | 189 | ||
190 | static inline int bitmap_and(unsigned long *dst, const unsigned long *src1, | 190 | static inline int bitmap_and(unsigned long *dst, const unsigned long *src1, |
191 | const unsigned long *src2, int nbits) | 191 | const unsigned long *src2, unsigned int nbits) |
192 | { | 192 | { |
193 | if (small_const_nbits(nbits)) | 193 | if (small_const_nbits(nbits)) |
194 | return (*dst = *src1 & *src2) != 0; | 194 | return (*dst = *src1 & *src2 & BITMAP_LAST_WORD_MASK(nbits)) != 0; |
195 | return __bitmap_and(dst, src1, src2, nbits); | 195 | return __bitmap_and(dst, src1, src2, nbits); |
196 | } | 196 | } |
197 | 197 | ||
198 | static inline void bitmap_or(unsigned long *dst, const unsigned long *src1, | 198 | static inline void bitmap_or(unsigned long *dst, const unsigned long *src1, |
199 | const unsigned long *src2, int nbits) | 199 | const unsigned long *src2, unsigned int nbits) |
200 | { | 200 | { |
201 | if (small_const_nbits(nbits)) | 201 | if (small_const_nbits(nbits)) |
202 | *dst = *src1 | *src2; | 202 | *dst = *src1 | *src2; |
@@ -205,7 +205,7 @@ static inline void bitmap_or(unsigned long *dst, const unsigned long *src1, | |||
205 | } | 205 | } |
206 | 206 | ||
207 | static inline void bitmap_xor(unsigned long *dst, const unsigned long *src1, | 207 | static inline void bitmap_xor(unsigned long *dst, const unsigned long *src1, |
208 | const unsigned long *src2, int nbits) | 208 | const unsigned long *src2, unsigned int nbits) |
209 | { | 209 | { |
210 | if (small_const_nbits(nbits)) | 210 | if (small_const_nbits(nbits)) |
211 | *dst = *src1 ^ *src2; | 211 | *dst = *src1 ^ *src2; |
@@ -214,24 +214,24 @@ static inline void bitmap_xor(unsigned long *dst, const unsigned long *src1, | |||
214 | } | 214 | } |
215 | 215 | ||
216 | static inline int bitmap_andnot(unsigned long *dst, const unsigned long *src1, | 216 | static inline int bitmap_andnot(unsigned long *dst, const unsigned long *src1, |
217 | const unsigned long *src2, int nbits) | 217 | const unsigned long *src2, unsigned int nbits) |
218 | { | 218 | { |
219 | if (small_const_nbits(nbits)) | 219 | if (small_const_nbits(nbits)) |
220 | return (*dst = *src1 & ~(*src2)) != 0; | 220 | return (*dst = *src1 & ~(*src2) & BITMAP_LAST_WORD_MASK(nbits)) != 0; |
221 | return __bitmap_andnot(dst, src1, src2, nbits); | 221 | return __bitmap_andnot(dst, src1, src2, nbits); |
222 | } | 222 | } |
223 | 223 | ||
224 | static inline void bitmap_complement(unsigned long *dst, const unsigned long *src, | 224 | static inline void bitmap_complement(unsigned long *dst, const unsigned long *src, |
225 | int nbits) | 225 | unsigned int nbits) |
226 | { | 226 | { |
227 | if (small_const_nbits(nbits)) | 227 | if (small_const_nbits(nbits)) |
228 | *dst = ~(*src) & BITMAP_LAST_WORD_MASK(nbits); | 228 | *dst = ~(*src); |
229 | else | 229 | else |
230 | __bitmap_complement(dst, src, nbits); | 230 | __bitmap_complement(dst, src, nbits); |
231 | } | 231 | } |
232 | 232 | ||
233 | static inline int bitmap_equal(const unsigned long *src1, | 233 | static inline int bitmap_equal(const unsigned long *src1, |
234 | const unsigned long *src2, int nbits) | 234 | const unsigned long *src2, unsigned int nbits) |
235 | { | 235 | { |
236 | if (small_const_nbits(nbits)) | 236 | if (small_const_nbits(nbits)) |
237 | return ! ((*src1 ^ *src2) & BITMAP_LAST_WORD_MASK(nbits)); | 237 | return ! ((*src1 ^ *src2) & BITMAP_LAST_WORD_MASK(nbits)); |
@@ -240,7 +240,7 @@ static inline int bitmap_equal(const unsigned long *src1, | |||
240 | } | 240 | } |
241 | 241 | ||
242 | static inline int bitmap_intersects(const unsigned long *src1, | 242 | static inline int bitmap_intersects(const unsigned long *src1, |
243 | const unsigned long *src2, int nbits) | 243 | const unsigned long *src2, unsigned int nbits) |
244 | { | 244 | { |
245 | if (small_const_nbits(nbits)) | 245 | if (small_const_nbits(nbits)) |
246 | return ((*src1 & *src2) & BITMAP_LAST_WORD_MASK(nbits)) != 0; | 246 | return ((*src1 & *src2) & BITMAP_LAST_WORD_MASK(nbits)) != 0; |
@@ -249,7 +249,7 @@ static inline int bitmap_intersects(const unsigned long *src1, | |||
249 | } | 249 | } |
250 | 250 | ||
251 | static inline int bitmap_subset(const unsigned long *src1, | 251 | static inline int bitmap_subset(const unsigned long *src1, |
252 | const unsigned long *src2, int nbits) | 252 | const unsigned long *src2, unsigned int nbits) |
253 | { | 253 | { |
254 | if (small_const_nbits(nbits)) | 254 | if (small_const_nbits(nbits)) |
255 | return ! ((*src1 & ~(*src2)) & BITMAP_LAST_WORD_MASK(nbits)); | 255 | return ! ((*src1 & ~(*src2)) & BITMAP_LAST_WORD_MASK(nbits)); |
@@ -257,7 +257,7 @@ static inline int bitmap_subset(const unsigned long *src1, | |||
257 | return __bitmap_subset(src1, src2, nbits); | 257 | return __bitmap_subset(src1, src2, nbits); |
258 | } | 258 | } |
259 | 259 | ||
260 | static inline int bitmap_empty(const unsigned long *src, int nbits) | 260 | static inline int bitmap_empty(const unsigned long *src, unsigned nbits) |
261 | { | 261 | { |
262 | if (small_const_nbits(nbits)) | 262 | if (small_const_nbits(nbits)) |
263 | return ! (*src & BITMAP_LAST_WORD_MASK(nbits)); | 263 | return ! (*src & BITMAP_LAST_WORD_MASK(nbits)); |
@@ -265,7 +265,7 @@ static inline int bitmap_empty(const unsigned long *src, int nbits) | |||
265 | return __bitmap_empty(src, nbits); | 265 | return __bitmap_empty(src, nbits); |
266 | } | 266 | } |
267 | 267 | ||
268 | static inline int bitmap_full(const unsigned long *src, int nbits) | 268 | static inline int bitmap_full(const unsigned long *src, unsigned int nbits) |
269 | { | 269 | { |
270 | if (small_const_nbits(nbits)) | 270 | if (small_const_nbits(nbits)) |
271 | return ! (~(*src) & BITMAP_LAST_WORD_MASK(nbits)); | 271 | return ! (~(*src) & BITMAP_LAST_WORD_MASK(nbits)); |
@@ -273,7 +273,7 @@ static inline int bitmap_full(const unsigned long *src, int nbits) | |||
273 | return __bitmap_full(src, nbits); | 273 | return __bitmap_full(src, nbits); |
274 | } | 274 | } |
275 | 275 | ||
276 | static inline int bitmap_weight(const unsigned long *src, int nbits) | 276 | static inline int bitmap_weight(const unsigned long *src, unsigned int nbits) |
277 | { | 277 | { |
278 | if (small_const_nbits(nbits)) | 278 | if (small_const_nbits(nbits)) |
279 | return hweight_long(*src & BITMAP_LAST_WORD_MASK(nbits)); | 279 | return hweight_long(*src & BITMAP_LAST_WORD_MASK(nbits)); |
@@ -284,7 +284,7 @@ static inline void bitmap_shift_right(unsigned long *dst, | |||
284 | const unsigned long *src, int n, int nbits) | 284 | const unsigned long *src, int n, int nbits) |
285 | { | 285 | { |
286 | if (small_const_nbits(nbits)) | 286 | if (small_const_nbits(nbits)) |
287 | *dst = *src >> n; | 287 | *dst = (*src & BITMAP_LAST_WORD_MASK(nbits)) >> n; |
288 | else | 288 | else |
289 | __bitmap_shift_right(dst, src, n, nbits); | 289 | __bitmap_shift_right(dst, src, n, nbits); |
290 | } | 290 | } |
diff --git a/include/linux/byteorder/generic.h b/include/linux/byteorder/generic.h index 0846e6b931ce..89f67c1c3160 100644 --- a/include/linux/byteorder/generic.h +++ b/include/linux/byteorder/generic.h | |||
@@ -2,7 +2,7 @@ | |||
2 | #define _LINUX_BYTEORDER_GENERIC_H | 2 | #define _LINUX_BYTEORDER_GENERIC_H |
3 | 3 | ||
4 | /* | 4 | /* |
5 | * linux/byteorder_generic.h | 5 | * linux/byteorder/generic.h |
6 | * Generic Byte-reordering support | 6 | * Generic Byte-reordering support |
7 | * | 7 | * |
8 | * The "... p" macros, like le64_to_cpup, can be used with pointers | 8 | * The "... p" macros, like le64_to_cpup, can be used with pointers |
diff --git a/include/linux/cma.h b/include/linux/cma.h new file mode 100644 index 000000000000..371b93042520 --- /dev/null +++ b/include/linux/cma.h | |||
@@ -0,0 +1,27 @@ | |||
1 | #ifndef __CMA_H__ | ||
2 | #define __CMA_H__ | ||
3 | |||
4 | /* | ||
5 | * There is always at least global CMA area and a few optional | ||
6 | * areas configured in kernel .config. | ||
7 | */ | ||
8 | #ifdef CONFIG_CMA_AREAS | ||
9 | #define MAX_CMA_AREAS (1 + CONFIG_CMA_AREAS) | ||
10 | |||
11 | #else | ||
12 | #define MAX_CMA_AREAS (0) | ||
13 | |||
14 | #endif | ||
15 | |||
16 | struct cma; | ||
17 | |||
18 | extern phys_addr_t cma_get_base(struct cma *cma); | ||
19 | extern unsigned long cma_get_size(struct cma *cma); | ||
20 | |||
21 | extern int __init cma_declare_contiguous(phys_addr_t size, | ||
22 | phys_addr_t base, phys_addr_t limit, | ||
23 | phys_addr_t alignment, unsigned int order_per_bit, | ||
24 | bool fixed, struct cma **res_cma); | ||
25 | extern struct page *cma_alloc(struct cma *cma, int count, unsigned int align); | ||
26 | extern bool cma_release(struct cma *cma, struct page *pages, int count); | ||
27 | #endif | ||
diff --git a/include/linux/dma-contiguous.h b/include/linux/dma-contiguous.h index 772eab5d524a..569bbd039896 100644 --- a/include/linux/dma-contiguous.h +++ b/include/linux/dma-contiguous.h | |||
@@ -53,18 +53,13 @@ | |||
53 | 53 | ||
54 | #ifdef __KERNEL__ | 54 | #ifdef __KERNEL__ |
55 | 55 | ||
56 | #include <linux/device.h> | ||
57 | |||
56 | struct cma; | 58 | struct cma; |
57 | struct page; | 59 | struct page; |
58 | struct device; | ||
59 | 60 | ||
60 | #ifdef CONFIG_DMA_CMA | 61 | #ifdef CONFIG_DMA_CMA |
61 | 62 | ||
62 | /* | ||
63 | * There is always at least global CMA area and a few optional device | ||
64 | * private areas configured in kernel .config. | ||
65 | */ | ||
66 | #define MAX_CMA_AREAS (1 + CONFIG_CMA_AREAS) | ||
67 | |||
68 | extern struct cma *dma_contiguous_default_area; | 63 | extern struct cma *dma_contiguous_default_area; |
69 | 64 | ||
70 | static inline struct cma *dev_get_cma_area(struct device *dev) | 65 | static inline struct cma *dev_get_cma_area(struct device *dev) |
@@ -123,8 +118,6 @@ bool dma_release_from_contiguous(struct device *dev, struct page *pages, | |||
123 | 118 | ||
124 | #else | 119 | #else |
125 | 120 | ||
126 | #define MAX_CMA_AREAS (0) | ||
127 | |||
128 | static inline struct cma *dev_get_cma_area(struct device *dev) | 121 | static inline struct cma *dev_get_cma_area(struct device *dev) |
129 | { | 122 | { |
130 | return NULL; | 123 | return NULL; |
diff --git a/include/linux/fs.h b/include/linux/fs.h index 2daccaf4b547..1ab6c6913040 100644 --- a/include/linux/fs.h +++ b/include/linux/fs.h | |||
@@ -2688,7 +2688,7 @@ static const struct file_operations __fops = { \ | |||
2688 | .read = simple_attr_read, \ | 2688 | .read = simple_attr_read, \ |
2689 | .write = simple_attr_write, \ | 2689 | .write = simple_attr_write, \ |
2690 | .llseek = generic_file_llseek, \ | 2690 | .llseek = generic_file_llseek, \ |
2691 | }; | 2691 | } |
2692 | 2692 | ||
2693 | static inline __printf(1, 2) | 2693 | static inline __printf(1, 2) |
2694 | void __simple_attr_check_format(const char *fmt, ...) | 2694 | void __simple_attr_check_format(const char *fmt, ...) |
diff --git a/include/linux/fsnotify_backend.h b/include/linux/fsnotify_backend.h index fc7718c6bd3e..ca060d7c4fa6 100644 --- a/include/linux/fsnotify_backend.h +++ b/include/linux/fsnotify_backend.h | |||
@@ -322,16 +322,18 @@ extern int fsnotify_fasync(int fd, struct file *file, int on); | |||
322 | extern void fsnotify_destroy_event(struct fsnotify_group *group, | 322 | extern void fsnotify_destroy_event(struct fsnotify_group *group, |
323 | struct fsnotify_event *event); | 323 | struct fsnotify_event *event); |
324 | /* attach the event to the group notification queue */ | 324 | /* attach the event to the group notification queue */ |
325 | extern int fsnotify_add_notify_event(struct fsnotify_group *group, | 325 | extern int fsnotify_add_event(struct fsnotify_group *group, |
326 | struct fsnotify_event *event, | 326 | struct fsnotify_event *event, |
327 | int (*merge)(struct list_head *, | 327 | int (*merge)(struct list_head *, |
328 | struct fsnotify_event *)); | 328 | struct fsnotify_event *)); |
329 | /* Remove passed event from groups notification queue */ | ||
330 | extern void fsnotify_remove_event(struct fsnotify_group *group, struct fsnotify_event *event); | ||
329 | /* true if the group notification queue is empty */ | 331 | /* true if the group notification queue is empty */ |
330 | extern bool fsnotify_notify_queue_is_empty(struct fsnotify_group *group); | 332 | extern bool fsnotify_notify_queue_is_empty(struct fsnotify_group *group); |
331 | /* return, but do not dequeue the first event on the notification queue */ | 333 | /* return, but do not dequeue the first event on the notification queue */ |
332 | extern struct fsnotify_event *fsnotify_peek_notify_event(struct fsnotify_group *group); | 334 | extern struct fsnotify_event *fsnotify_peek_first_event(struct fsnotify_group *group); |
333 | /* return AND dequeue the first event on the notification queue */ | 335 | /* return AND dequeue the first event on the notification queue */ |
334 | extern struct fsnotify_event *fsnotify_remove_notify_event(struct fsnotify_group *group); | 336 | extern struct fsnotify_event *fsnotify_remove_first_event(struct fsnotify_group *group); |
335 | 337 | ||
336 | /* functions used to manipulate the marks attached to inodes */ | 338 | /* functions used to manipulate the marks attached to inodes */ |
337 | 339 | ||
diff --git a/include/linux/gfp.h b/include/linux/gfp.h index 6eb1fb37de9a..5e7219dc0fae 100644 --- a/include/linux/gfp.h +++ b/include/linux/gfp.h | |||
@@ -360,7 +360,7 @@ extern unsigned long get_zeroed_page(gfp_t gfp_mask); | |||
360 | void *alloc_pages_exact(size_t size, gfp_t gfp_mask); | 360 | void *alloc_pages_exact(size_t size, gfp_t gfp_mask); |
361 | void free_pages_exact(void *virt, size_t size); | 361 | void free_pages_exact(void *virt, size_t size); |
362 | /* This is different from alloc_pages_exact_node !!! */ | 362 | /* This is different from alloc_pages_exact_node !!! */ |
363 | void *alloc_pages_exact_nid(int nid, size_t size, gfp_t gfp_mask); | 363 | void * __meminit alloc_pages_exact_nid(int nid, size_t size, gfp_t gfp_mask); |
364 | 364 | ||
365 | #define __get_free_page(gfp_mask) \ | 365 | #define __get_free_page(gfp_mask) \ |
366 | __get_free_pages((gfp_mask), 0) | 366 | __get_free_pages((gfp_mask), 0) |
diff --git a/include/linux/glob.h b/include/linux/glob.h new file mode 100644 index 000000000000..861d8347d08e --- /dev/null +++ b/include/linux/glob.h | |||
@@ -0,0 +1,9 @@ | |||
1 | #ifndef _LINUX_GLOB_H | ||
2 | #define _LINUX_GLOB_H | ||
3 | |||
4 | #include <linux/types.h> /* For bool */ | ||
5 | #include <linux/compiler.h> /* For __pure */ | ||
6 | |||
7 | bool __pure glob_match(char const *pat, char const *str); | ||
8 | |||
9 | #endif /* _LINUX_GLOB_H */ | ||
diff --git a/include/linux/highmem.h b/include/linux/highmem.h index 7fb31da45d03..9286a46b7d69 100644 --- a/include/linux/highmem.h +++ b/include/linux/highmem.h | |||
@@ -93,7 +93,7 @@ static inline int kmap_atomic_idx_push(void) | |||
93 | 93 | ||
94 | #ifdef CONFIG_DEBUG_HIGHMEM | 94 | #ifdef CONFIG_DEBUG_HIGHMEM |
95 | WARN_ON_ONCE(in_irq() && !irqs_disabled()); | 95 | WARN_ON_ONCE(in_irq() && !irqs_disabled()); |
96 | BUG_ON(idx > KM_TYPE_NR); | 96 | BUG_ON(idx >= KM_TYPE_NR); |
97 | #endif | 97 | #endif |
98 | return idx; | 98 | return idx; |
99 | } | 99 | } |
diff --git a/include/linux/huge_mm.h b/include/linux/huge_mm.h index b826239bdce0..63579cb8d3dc 100644 --- a/include/linux/huge_mm.h +++ b/include/linux/huge_mm.h | |||
@@ -93,10 +93,6 @@ extern bool is_vma_temporary_stack(struct vm_area_struct *vma); | |||
93 | #endif /* CONFIG_DEBUG_VM */ | 93 | #endif /* CONFIG_DEBUG_VM */ |
94 | 94 | ||
95 | extern unsigned long transparent_hugepage_flags; | 95 | extern unsigned long transparent_hugepage_flags; |
96 | extern int copy_pte_range(struct mm_struct *dst_mm, struct mm_struct *src_mm, | ||
97 | pmd_t *dst_pmd, pmd_t *src_pmd, | ||
98 | struct vm_area_struct *vma, | ||
99 | unsigned long addr, unsigned long end); | ||
100 | extern int split_huge_page_to_list(struct page *page, struct list_head *list); | 96 | extern int split_huge_page_to_list(struct page *page, struct list_head *list); |
101 | static inline int split_huge_page(struct page *page) | 97 | static inline int split_huge_page(struct page *page) |
102 | { | 98 | { |
diff --git a/include/linux/hugetlb.h b/include/linux/hugetlb.h index a23c096b3080..6e6d338641fe 100644 --- a/include/linux/hugetlb.h +++ b/include/linux/hugetlb.h | |||
@@ -87,7 +87,6 @@ pte_t *huge_pmd_share(struct mm_struct *mm, unsigned long addr, pud_t *pud); | |||
87 | #endif | 87 | #endif |
88 | 88 | ||
89 | extern unsigned long hugepages_treat_as_movable; | 89 | extern unsigned long hugepages_treat_as_movable; |
90 | extern const unsigned long hugetlb_zero, hugetlb_infinity; | ||
91 | extern int sysctl_hugetlb_shm_group; | 90 | extern int sysctl_hugetlb_shm_group; |
92 | extern struct list_head huge_boot_pages; | 91 | extern struct list_head huge_boot_pages; |
93 | 92 | ||
diff --git a/include/linux/kernel.h b/include/linux/kernel.h index a9e2268ecccb..3dc22abbc68a 100644 --- a/include/linux/kernel.h +++ b/include/linux/kernel.h | |||
@@ -493,11 +493,6 @@ static inline char *hex_byte_pack_upper(char *buf, u8 byte) | |||
493 | return buf; | 493 | return buf; |
494 | } | 494 | } |
495 | 495 | ||
496 | static inline char * __deprecated pack_hex_byte(char *buf, u8 byte) | ||
497 | { | ||
498 | return hex_byte_pack(buf, byte); | ||
499 | } | ||
500 | |||
501 | extern int hex_to_bin(char ch); | 496 | extern int hex_to_bin(char ch); |
502 | extern int __must_check hex2bin(u8 *dst, const char *src, size_t count); | 497 | extern int __must_check hex2bin(u8 *dst, const char *src, size_t count); |
503 | 498 | ||
diff --git a/include/linux/klist.h b/include/linux/klist.h index a370ce57cf1d..61e5b723ae73 100644 --- a/include/linux/klist.h +++ b/include/linux/klist.h | |||
@@ -44,7 +44,7 @@ struct klist_node { | |||
44 | 44 | ||
45 | extern void klist_add_tail(struct klist_node *n, struct klist *k); | 45 | extern void klist_add_tail(struct klist_node *n, struct klist *k); |
46 | extern void klist_add_head(struct klist_node *n, struct klist *k); | 46 | extern void klist_add_head(struct klist_node *n, struct klist *k); |
47 | extern void klist_add_after(struct klist_node *n, struct klist_node *pos); | 47 | extern void klist_add_behind(struct klist_node *n, struct klist_node *pos); |
48 | extern void klist_add_before(struct klist_node *n, struct klist_node *pos); | 48 | extern void klist_add_before(struct klist_node *n, struct klist_node *pos); |
49 | 49 | ||
50 | extern void klist_del(struct klist_node *n); | 50 | extern void klist_del(struct klist_node *n); |
diff --git a/include/linux/list.h b/include/linux/list.h index ef9594171062..cbbb96fcead9 100644 --- a/include/linux/list.h +++ b/include/linux/list.h | |||
@@ -654,15 +654,15 @@ static inline void hlist_add_before(struct hlist_node *n, | |||
654 | *(n->pprev) = n; | 654 | *(n->pprev) = n; |
655 | } | 655 | } |
656 | 656 | ||
657 | static inline void hlist_add_after(struct hlist_node *n, | 657 | static inline void hlist_add_behind(struct hlist_node *n, |
658 | struct hlist_node *next) | 658 | struct hlist_node *prev) |
659 | { | 659 | { |
660 | next->next = n->next; | 660 | n->next = prev->next; |
661 | n->next = next; | 661 | prev->next = n; |
662 | next->pprev = &n->next; | 662 | n->pprev = &prev->next; |
663 | 663 | ||
664 | if(next->next) | 664 | if (n->next) |
665 | next->next->pprev = &next->next; | 665 | n->next->pprev = &n->next; |
666 | } | 666 | } |
667 | 667 | ||
668 | /* after that we'll appear to be on some hlist and hlist_del will work */ | 668 | /* after that we'll appear to be on some hlist and hlist_del will work */ |
diff --git a/include/linux/memblock.h b/include/linux/memblock.h index b660e05b63d4..e8cc45307f8f 100644 --- a/include/linux/memblock.h +++ b/include/linux/memblock.h | |||
@@ -249,7 +249,7 @@ phys_addr_t memblock_alloc(phys_addr_t size, phys_addr_t align); | |||
249 | /* | 249 | /* |
250 | * Set the allocation direction to bottom-up or top-down. | 250 | * Set the allocation direction to bottom-up or top-down. |
251 | */ | 251 | */ |
252 | static inline void memblock_set_bottom_up(bool enable) | 252 | static inline void __init memblock_set_bottom_up(bool enable) |
253 | { | 253 | { |
254 | memblock.bottom_up = enable; | 254 | memblock.bottom_up = enable; |
255 | } | 255 | } |
@@ -264,7 +264,7 @@ static inline bool memblock_bottom_up(void) | |||
264 | return memblock.bottom_up; | 264 | return memblock.bottom_up; |
265 | } | 265 | } |
266 | #else | 266 | #else |
267 | static inline void memblock_set_bottom_up(bool enable) {} | 267 | static inline void __init memblock_set_bottom_up(bool enable) {} |
268 | static inline bool memblock_bottom_up(void) { return false; } | 268 | static inline bool memblock_bottom_up(void) { return false; } |
269 | #endif | 269 | #endif |
270 | 270 | ||
diff --git a/include/linux/memory_hotplug.h b/include/linux/memory_hotplug.h index 010d125bffbf..d9524c49d767 100644 --- a/include/linux/memory_hotplug.h +++ b/include/linux/memory_hotplug.h | |||
@@ -26,11 +26,12 @@ enum { | |||
26 | MEMORY_HOTPLUG_MAX_BOOTMEM_TYPE = NODE_INFO, | 26 | MEMORY_HOTPLUG_MAX_BOOTMEM_TYPE = NODE_INFO, |
27 | }; | 27 | }; |
28 | 28 | ||
29 | /* Types for control the zone type of onlined memory */ | 29 | /* Types for control the zone type of onlined and offlined memory */ |
30 | enum { | 30 | enum { |
31 | ONLINE_KEEP, | 31 | MMOP_OFFLINE = -1, |
32 | ONLINE_KERNEL, | 32 | MMOP_ONLINE_KEEP, |
33 | ONLINE_MOVABLE, | 33 | MMOP_ONLINE_KERNEL, |
34 | MMOP_ONLINE_MOVABLE, | ||
34 | }; | 35 | }; |
35 | 36 | ||
36 | /* | 37 | /* |
@@ -258,6 +259,7 @@ static inline void remove_memory(int nid, u64 start, u64 size) {} | |||
258 | extern int walk_memory_range(unsigned long start_pfn, unsigned long end_pfn, | 259 | extern int walk_memory_range(unsigned long start_pfn, unsigned long end_pfn, |
259 | void *arg, int (*func)(struct memory_block *, void *)); | 260 | void *arg, int (*func)(struct memory_block *, void *)); |
260 | extern int add_memory(int nid, u64 start, u64 size); | 261 | extern int add_memory(int nid, u64 start, u64 size); |
262 | extern int zone_for_memory(int nid, u64 start, u64 size, int zone_default); | ||
261 | extern int arch_add_memory(int nid, u64 start, u64 size); | 263 | extern int arch_add_memory(int nid, u64 start, u64 size); |
262 | extern int offline_pages(unsigned long start_pfn, unsigned long nr_pages); | 264 | extern int offline_pages(unsigned long start_pfn, unsigned long nr_pages); |
263 | extern bool is_memblock_offlined(struct memory_block *mem); | 265 | extern bool is_memblock_offlined(struct memory_block *mem); |
diff --git a/include/linux/mmdebug.h b/include/linux/mmdebug.h index edd82a105220..2f348d02f640 100644 --- a/include/linux/mmdebug.h +++ b/include/linux/mmdebug.h | |||
@@ -20,11 +20,13 @@ extern void dump_page_badflags(struct page *page, const char *reason, | |||
20 | } while (0) | 20 | } while (0) |
21 | #define VM_WARN_ON(cond) WARN_ON(cond) | 21 | #define VM_WARN_ON(cond) WARN_ON(cond) |
22 | #define VM_WARN_ON_ONCE(cond) WARN_ON_ONCE(cond) | 22 | #define VM_WARN_ON_ONCE(cond) WARN_ON_ONCE(cond) |
23 | #define VM_WARN_ONCE(cond, format...) WARN_ONCE(cond, format) | ||
23 | #else | 24 | #else |
24 | #define VM_BUG_ON(cond) BUILD_BUG_ON_INVALID(cond) | 25 | #define VM_BUG_ON(cond) BUILD_BUG_ON_INVALID(cond) |
25 | #define VM_BUG_ON_PAGE(cond, page) VM_BUG_ON(cond) | 26 | #define VM_BUG_ON_PAGE(cond, page) VM_BUG_ON(cond) |
26 | #define VM_WARN_ON(cond) BUILD_BUG_ON_INVALID(cond) | 27 | #define VM_WARN_ON(cond) BUILD_BUG_ON_INVALID(cond) |
27 | #define VM_WARN_ON_ONCE(cond) BUILD_BUG_ON_INVALID(cond) | 28 | #define VM_WARN_ON_ONCE(cond) BUILD_BUG_ON_INVALID(cond) |
29 | #define VM_WARN_ONCE(cond, format...) BUILD_BUG_ON_INVALID(cond) | ||
28 | #endif | 30 | #endif |
29 | 31 | ||
30 | #ifdef CONFIG_DEBUG_VIRTUAL | 32 | #ifdef CONFIG_DEBUG_VIRTUAL |
diff --git a/include/linux/mmu_notifier.h b/include/linux/mmu_notifier.h index deca87452528..27288692241e 100644 --- a/include/linux/mmu_notifier.h +++ b/include/linux/mmu_notifier.h | |||
@@ -170,6 +170,8 @@ extern int __mmu_notifier_register(struct mmu_notifier *mn, | |||
170 | struct mm_struct *mm); | 170 | struct mm_struct *mm); |
171 | extern void mmu_notifier_unregister(struct mmu_notifier *mn, | 171 | extern void mmu_notifier_unregister(struct mmu_notifier *mn, |
172 | struct mm_struct *mm); | 172 | struct mm_struct *mm); |
173 | extern void mmu_notifier_unregister_no_release(struct mmu_notifier *mn, | ||
174 | struct mm_struct *mm); | ||
173 | extern void __mmu_notifier_mm_destroy(struct mm_struct *mm); | 175 | extern void __mmu_notifier_mm_destroy(struct mm_struct *mm); |
174 | extern void __mmu_notifier_release(struct mm_struct *mm); | 176 | extern void __mmu_notifier_release(struct mm_struct *mm); |
175 | extern int __mmu_notifier_clear_flush_young(struct mm_struct *mm, | 177 | extern int __mmu_notifier_clear_flush_young(struct mm_struct *mm, |
@@ -288,6 +290,10 @@ static inline void mmu_notifier_mm_destroy(struct mm_struct *mm) | |||
288 | set_pte_at(___mm, ___address, __ptep, ___pte); \ | 290 | set_pte_at(___mm, ___address, __ptep, ___pte); \ |
289 | }) | 291 | }) |
290 | 292 | ||
293 | extern void mmu_notifier_call_srcu(struct rcu_head *rcu, | ||
294 | void (*func)(struct rcu_head *rcu)); | ||
295 | extern void mmu_notifier_synchronize(void); | ||
296 | |||
291 | #else /* CONFIG_MMU_NOTIFIER */ | 297 | #else /* CONFIG_MMU_NOTIFIER */ |
292 | 298 | ||
293 | static inline void mmu_notifier_release(struct mm_struct *mm) | 299 | static inline void mmu_notifier_release(struct mm_struct *mm) |
diff --git a/include/linux/mmzone.h b/include/linux/mmzone.h index 6cbd1b6c3d20..318df7051850 100644 --- a/include/linux/mmzone.h +++ b/include/linux/mmzone.h | |||
@@ -143,6 +143,7 @@ enum zone_stat_item { | |||
143 | NR_SHMEM, /* shmem pages (included tmpfs/GEM pages) */ | 143 | NR_SHMEM, /* shmem pages (included tmpfs/GEM pages) */ |
144 | NR_DIRTIED, /* page dirtyings since bootup */ | 144 | NR_DIRTIED, /* page dirtyings since bootup */ |
145 | NR_WRITTEN, /* page writings since bootup */ | 145 | NR_WRITTEN, /* page writings since bootup */ |
146 | NR_PAGES_SCANNED, /* pages scanned since last reclaim */ | ||
146 | #ifdef CONFIG_NUMA | 147 | #ifdef CONFIG_NUMA |
147 | NUMA_HIT, /* allocated in intended node */ | 148 | NUMA_HIT, /* allocated in intended node */ |
148 | NUMA_MISS, /* allocated in non intended node */ | 149 | NUMA_MISS, /* allocated in non intended node */ |
@@ -324,19 +325,12 @@ enum zone_type { | |||
324 | #ifndef __GENERATING_BOUNDS_H | 325 | #ifndef __GENERATING_BOUNDS_H |
325 | 326 | ||
326 | struct zone { | 327 | struct zone { |
327 | /* Fields commonly accessed by the page allocator */ | 328 | /* Read-mostly fields */ |
328 | 329 | ||
329 | /* zone watermarks, access with *_wmark_pages(zone) macros */ | 330 | /* zone watermarks, access with *_wmark_pages(zone) macros */ |
330 | unsigned long watermark[NR_WMARK]; | 331 | unsigned long watermark[NR_WMARK]; |
331 | 332 | ||
332 | /* | 333 | /* |
333 | * When free pages are below this point, additional steps are taken | ||
334 | * when reading the number of free pages to avoid per-cpu counter | ||
335 | * drift allowing watermarks to be breached | ||
336 | */ | ||
337 | unsigned long percpu_drift_mark; | ||
338 | |||
339 | /* | ||
340 | * We don't know if the memory that we're going to allocate will be freeable | 334 | * We don't know if the memory that we're going to allocate will be freeable |
341 | * or/and it will be released eventually, so to avoid totally wasting several | 335 | * or/and it will be released eventually, so to avoid totally wasting several |
342 | * GB of ram we must reserve some of the lower zone memory (otherwise we risk | 336 | * GB of ram we must reserve some of the lower zone memory (otherwise we risk |
@@ -344,41 +338,26 @@ struct zone { | |||
344 | * on the higher zones). This array is recalculated at runtime if the | 338 | * on the higher zones). This array is recalculated at runtime if the |
345 | * sysctl_lowmem_reserve_ratio sysctl changes. | 339 | * sysctl_lowmem_reserve_ratio sysctl changes. |
346 | */ | 340 | */ |
347 | unsigned long lowmem_reserve[MAX_NR_ZONES]; | 341 | long lowmem_reserve[MAX_NR_ZONES]; |
348 | |||
349 | /* | ||
350 | * This is a per-zone reserve of pages that should not be | ||
351 | * considered dirtyable memory. | ||
352 | */ | ||
353 | unsigned long dirty_balance_reserve; | ||
354 | 342 | ||
355 | #ifdef CONFIG_NUMA | 343 | #ifdef CONFIG_NUMA |
356 | int node; | 344 | int node; |
345 | #endif | ||
346 | |||
357 | /* | 347 | /* |
358 | * zone reclaim becomes active if more unmapped pages exist. | 348 | * The target ratio of ACTIVE_ANON to INACTIVE_ANON pages on |
349 | * this zone's LRU. Maintained by the pageout code. | ||
359 | */ | 350 | */ |
360 | unsigned long min_unmapped_pages; | 351 | unsigned int inactive_ratio; |
361 | unsigned long min_slab_pages; | 352 | |
362 | #endif | 353 | struct pglist_data *zone_pgdat; |
363 | struct per_cpu_pageset __percpu *pageset; | 354 | struct per_cpu_pageset __percpu *pageset; |
355 | |||
364 | /* | 356 | /* |
365 | * free areas of different sizes | 357 | * This is a per-zone reserve of pages that should not be |
358 | * considered dirtyable memory. | ||
366 | */ | 359 | */ |
367 | spinlock_t lock; | 360 | unsigned long dirty_balance_reserve; |
368 | #if defined CONFIG_COMPACTION || defined CONFIG_CMA | ||
369 | /* Set to true when the PG_migrate_skip bits should be cleared */ | ||
370 | bool compact_blockskip_flush; | ||
371 | |||
372 | /* pfn where compaction free scanner should start */ | ||
373 | unsigned long compact_cached_free_pfn; | ||
374 | /* pfn where async and sync compaction migration scanner should start */ | ||
375 | unsigned long compact_cached_migrate_pfn[2]; | ||
376 | #endif | ||
377 | #ifdef CONFIG_MEMORY_HOTPLUG | ||
378 | /* see spanned/present_pages for more description */ | ||
379 | seqlock_t span_seqlock; | ||
380 | #endif | ||
381 | struct free_area free_area[MAX_ORDER]; | ||
382 | 361 | ||
383 | #ifndef CONFIG_SPARSEMEM | 362 | #ifndef CONFIG_SPARSEMEM |
384 | /* | 363 | /* |
@@ -388,74 +367,14 @@ struct zone { | |||
388 | unsigned long *pageblock_flags; | 367 | unsigned long *pageblock_flags; |
389 | #endif /* CONFIG_SPARSEMEM */ | 368 | #endif /* CONFIG_SPARSEMEM */ |
390 | 369 | ||
391 | #ifdef CONFIG_COMPACTION | 370 | #ifdef CONFIG_NUMA |
392 | /* | ||
393 | * On compaction failure, 1<<compact_defer_shift compactions | ||
394 | * are skipped before trying again. The number attempted since | ||
395 | * last failure is tracked with compact_considered. | ||
396 | */ | ||
397 | unsigned int compact_considered; | ||
398 | unsigned int compact_defer_shift; | ||
399 | int compact_order_failed; | ||
400 | #endif | ||
401 | |||
402 | ZONE_PADDING(_pad1_) | ||
403 | |||
404 | /* Fields commonly accessed by the page reclaim scanner */ | ||
405 | spinlock_t lru_lock; | ||
406 | struct lruvec lruvec; | ||
407 | |||
408 | /* Evictions & activations on the inactive file list */ | ||
409 | atomic_long_t inactive_age; | ||
410 | |||
411 | unsigned long pages_scanned; /* since last reclaim */ | ||
412 | unsigned long flags; /* zone flags, see below */ | ||
413 | |||
414 | /* Zone statistics */ | ||
415 | atomic_long_t vm_stat[NR_VM_ZONE_STAT_ITEMS]; | ||
416 | |||
417 | /* | ||
418 | * The target ratio of ACTIVE_ANON to INACTIVE_ANON pages on | ||
419 | * this zone's LRU. Maintained by the pageout code. | ||
420 | */ | ||
421 | unsigned int inactive_ratio; | ||
422 | |||
423 | |||
424 | ZONE_PADDING(_pad2_) | ||
425 | /* Rarely used or read-mostly fields */ | ||
426 | |||
427 | /* | 371 | /* |
428 | * wait_table -- the array holding the hash table | 372 | * zone reclaim becomes active if more unmapped pages exist. |
429 | * wait_table_hash_nr_entries -- the size of the hash table array | ||
430 | * wait_table_bits -- wait_table_size == (1 << wait_table_bits) | ||
431 | * | ||
432 | * The purpose of all these is to keep track of the people | ||
433 | * waiting for a page to become available and make them | ||
434 | * runnable again when possible. The trouble is that this | ||
435 | * consumes a lot of space, especially when so few things | ||
436 | * wait on pages at a given time. So instead of using | ||
437 | * per-page waitqueues, we use a waitqueue hash table. | ||
438 | * | ||
439 | * The bucket discipline is to sleep on the same queue when | ||
440 | * colliding and wake all in that wait queue when removing. | ||
441 | * When something wakes, it must check to be sure its page is | ||
442 | * truly available, a la thundering herd. The cost of a | ||
443 | * collision is great, but given the expected load of the | ||
444 | * table, they should be so rare as to be outweighed by the | ||
445 | * benefits from the saved space. | ||
446 | * | ||
447 | * __wait_on_page_locked() and unlock_page() in mm/filemap.c, are the | ||
448 | * primary users of these fields, and in mm/page_alloc.c | ||
449 | * free_area_init_core() performs the initialization of them. | ||
450 | */ | 373 | */ |
451 | wait_queue_head_t * wait_table; | 374 | unsigned long min_unmapped_pages; |
452 | unsigned long wait_table_hash_nr_entries; | 375 | unsigned long min_slab_pages; |
453 | unsigned long wait_table_bits; | 376 | #endif /* CONFIG_NUMA */ |
454 | 377 | ||
455 | /* | ||
456 | * Discontig memory support fields. | ||
457 | */ | ||
458 | struct pglist_data *zone_pgdat; | ||
459 | /* zone_start_pfn == zone_start_paddr >> PAGE_SHIFT */ | 378 | /* zone_start_pfn == zone_start_paddr >> PAGE_SHIFT */ |
460 | unsigned long zone_start_pfn; | 379 | unsigned long zone_start_pfn; |
461 | 380 | ||
@@ -500,9 +419,11 @@ struct zone { | |||
500 | * adjust_managed_page_count() should be used instead of directly | 419 | * adjust_managed_page_count() should be used instead of directly |
501 | * touching zone->managed_pages and totalram_pages. | 420 | * touching zone->managed_pages and totalram_pages. |
502 | */ | 421 | */ |
422 | unsigned long managed_pages; | ||
503 | unsigned long spanned_pages; | 423 | unsigned long spanned_pages; |
504 | unsigned long present_pages; | 424 | unsigned long present_pages; |
505 | unsigned long managed_pages; | 425 | |
426 | const char *name; | ||
506 | 427 | ||
507 | /* | 428 | /* |
508 | * Number of MIGRATE_RESEVE page block. To maintain for just | 429 | * Number of MIGRATE_RESEVE page block. To maintain for just |
@@ -510,10 +431,94 @@ struct zone { | |||
510 | */ | 431 | */ |
511 | int nr_migrate_reserve_block; | 432 | int nr_migrate_reserve_block; |
512 | 433 | ||
434 | #ifdef CONFIG_MEMORY_HOTPLUG | ||
435 | /* see spanned/present_pages for more description */ | ||
436 | seqlock_t span_seqlock; | ||
437 | #endif | ||
438 | |||
513 | /* | 439 | /* |
514 | * rarely used fields: | 440 | * wait_table -- the array holding the hash table |
441 | * wait_table_hash_nr_entries -- the size of the hash table array | ||
442 | * wait_table_bits -- wait_table_size == (1 << wait_table_bits) | ||
443 | * | ||
444 | * The purpose of all these is to keep track of the people | ||
445 | * waiting for a page to become available and make them | ||
446 | * runnable again when possible. The trouble is that this | ||
447 | * consumes a lot of space, especially when so few things | ||
448 | * wait on pages at a given time. So instead of using | ||
449 | * per-page waitqueues, we use a waitqueue hash table. | ||
450 | * | ||
451 | * The bucket discipline is to sleep on the same queue when | ||
452 | * colliding and wake all in that wait queue when removing. | ||
453 | * When something wakes, it must check to be sure its page is | ||
454 | * truly available, a la thundering herd. The cost of a | ||
455 | * collision is great, but given the expected load of the | ||
456 | * table, they should be so rare as to be outweighed by the | ||
457 | * benefits from the saved space. | ||
458 | * | ||
459 | * __wait_on_page_locked() and unlock_page() in mm/filemap.c, are the | ||
460 | * primary users of these fields, and in mm/page_alloc.c | ||
461 | * free_area_init_core() performs the initialization of them. | ||
515 | */ | 462 | */ |
516 | const char *name; | 463 | wait_queue_head_t *wait_table; |
464 | unsigned long wait_table_hash_nr_entries; | ||
465 | unsigned long wait_table_bits; | ||
466 | |||
467 | ZONE_PADDING(_pad1_) | ||
468 | |||
469 | /* Write-intensive fields used from the page allocator */ | ||
470 | spinlock_t lock; | ||
471 | |||
472 | /* free areas of different sizes */ | ||
473 | struct free_area free_area[MAX_ORDER]; | ||
474 | |||
475 | /* zone flags, see below */ | ||
476 | unsigned long flags; | ||
477 | |||
478 | ZONE_PADDING(_pad2_) | ||
479 | |||
480 | /* Write-intensive fields used by page reclaim */ | ||
481 | |||
482 | /* Fields commonly accessed by the page reclaim scanner */ | ||
483 | spinlock_t lru_lock; | ||
484 | struct lruvec lruvec; | ||
485 | |||
486 | /* Evictions & activations on the inactive file list */ | ||
487 | atomic_long_t inactive_age; | ||
488 | |||
489 | /* | ||
490 | * When free pages are below this point, additional steps are taken | ||
491 | * when reading the number of free pages to avoid per-cpu counter | ||
492 | * drift allowing watermarks to be breached | ||
493 | */ | ||
494 | unsigned long percpu_drift_mark; | ||
495 | |||
496 | #if defined CONFIG_COMPACTION || defined CONFIG_CMA | ||
497 | /* pfn where compaction free scanner should start */ | ||
498 | unsigned long compact_cached_free_pfn; | ||
499 | /* pfn where async and sync compaction migration scanner should start */ | ||
500 | unsigned long compact_cached_migrate_pfn[2]; | ||
501 | #endif | ||
502 | |||
503 | #ifdef CONFIG_COMPACTION | ||
504 | /* | ||
505 | * On compaction failure, 1<<compact_defer_shift compactions | ||
506 | * are skipped before trying again. The number attempted since | ||
507 | * last failure is tracked with compact_considered. | ||
508 | */ | ||
509 | unsigned int compact_considered; | ||
510 | unsigned int compact_defer_shift; | ||
511 | int compact_order_failed; | ||
512 | #endif | ||
513 | |||
514 | #if defined CONFIG_COMPACTION || defined CONFIG_CMA | ||
515 | /* Set to true when the PG_migrate_skip bits should be cleared */ | ||
516 | bool compact_blockskip_flush; | ||
517 | #endif | ||
518 | |||
519 | ZONE_PADDING(_pad3_) | ||
520 | /* Zone statistics */ | ||
521 | atomic_long_t vm_stat[NR_VM_ZONE_STAT_ITEMS]; | ||
517 | } ____cacheline_internodealigned_in_smp; | 522 | } ____cacheline_internodealigned_in_smp; |
518 | 523 | ||
519 | typedef enum { | 524 | typedef enum { |
@@ -529,6 +534,7 @@ typedef enum { | |||
529 | ZONE_WRITEBACK, /* reclaim scanning has recently found | 534 | ZONE_WRITEBACK, /* reclaim scanning has recently found |
530 | * many pages under writeback | 535 | * many pages under writeback |
531 | */ | 536 | */ |
537 | ZONE_FAIR_DEPLETED, /* fair zone policy batch depleted */ | ||
532 | } zone_flags_t; | 538 | } zone_flags_t; |
533 | 539 | ||
534 | static inline void zone_set_flag(struct zone *zone, zone_flags_t flag) | 540 | static inline void zone_set_flag(struct zone *zone, zone_flags_t flag) |
@@ -566,6 +572,11 @@ static inline int zone_is_reclaim_locked(const struct zone *zone) | |||
566 | return test_bit(ZONE_RECLAIM_LOCKED, &zone->flags); | 572 | return test_bit(ZONE_RECLAIM_LOCKED, &zone->flags); |
567 | } | 573 | } |
568 | 574 | ||
575 | static inline int zone_is_fair_depleted(const struct zone *zone) | ||
576 | { | ||
577 | return test_bit(ZONE_FAIR_DEPLETED, &zone->flags); | ||
578 | } | ||
579 | |||
569 | static inline int zone_is_oom_locked(const struct zone *zone) | 580 | static inline int zone_is_oom_locked(const struct zone *zone) |
570 | { | 581 | { |
571 | return test_bit(ZONE_OOM_LOCKED, &zone->flags); | 582 | return test_bit(ZONE_OOM_LOCKED, &zone->flags); |
@@ -872,6 +883,8 @@ static inline int zone_movable_is_highmem(void) | |||
872 | { | 883 | { |
873 | #if defined(CONFIG_HIGHMEM) && defined(CONFIG_HAVE_MEMBLOCK_NODE_MAP) | 884 | #if defined(CONFIG_HIGHMEM) && defined(CONFIG_HAVE_MEMBLOCK_NODE_MAP) |
874 | return movable_zone == ZONE_HIGHMEM; | 885 | return movable_zone == ZONE_HIGHMEM; |
886 | #elif defined(CONFIG_HIGHMEM) | ||
887 | return (ZONE_MOVABLE - 1) == ZONE_HIGHMEM; | ||
875 | #else | 888 | #else |
876 | return 0; | 889 | return 0; |
877 | #endif | 890 | #endif |
diff --git a/include/linux/nodemask.h b/include/linux/nodemask.h index 58b9a02c38d2..83a6aeda899d 100644 --- a/include/linux/nodemask.h +++ b/include/linux/nodemask.h | |||
@@ -430,7 +430,15 @@ static inline int num_node_state(enum node_states state) | |||
430 | for_each_node_mask((__node), node_states[__state]) | 430 | for_each_node_mask((__node), node_states[__state]) |
431 | 431 | ||
432 | #define first_online_node first_node(node_states[N_ONLINE]) | 432 | #define first_online_node first_node(node_states[N_ONLINE]) |
433 | #define next_online_node(nid) next_node((nid), node_states[N_ONLINE]) | 433 | #define first_memory_node first_node(node_states[N_MEMORY]) |
434 | static inline int next_online_node(int nid) | ||
435 | { | ||
436 | return next_node(nid, node_states[N_ONLINE]); | ||
437 | } | ||
438 | static inline int next_memory_node(int nid) | ||
439 | { | ||
440 | return next_node(nid, node_states[N_MEMORY]); | ||
441 | } | ||
434 | 442 | ||
435 | extern int nr_node_ids; | 443 | extern int nr_node_ids; |
436 | extern int nr_online_nodes; | 444 | extern int nr_online_nodes; |
@@ -471,6 +479,7 @@ static inline int num_node_state(enum node_states state) | |||
471 | for ( (node) = 0; (node) == 0; (node) = 1) | 479 | for ( (node) = 0; (node) == 0; (node) = 1) |
472 | 480 | ||
473 | #define first_online_node 0 | 481 | #define first_online_node 0 |
482 | #define first_memory_node 0 | ||
474 | #define next_online_node(nid) (MAX_NUMNODES) | 483 | #define next_online_node(nid) (MAX_NUMNODES) |
475 | #define nr_node_ids 1 | 484 | #define nr_node_ids 1 |
476 | #define nr_online_nodes 1 | 485 | #define nr_online_nodes 1 |
diff --git a/include/linux/oom.h b/include/linux/oom.h index 4cd62677feb9..647395a1a550 100644 --- a/include/linux/oom.h +++ b/include/linux/oom.h | |||
@@ -55,8 +55,8 @@ extern void oom_kill_process(struct task_struct *p, gfp_t gfp_mask, int order, | |||
55 | struct mem_cgroup *memcg, nodemask_t *nodemask, | 55 | struct mem_cgroup *memcg, nodemask_t *nodemask, |
56 | const char *message); | 56 | const char *message); |
57 | 57 | ||
58 | extern int try_set_zonelist_oom(struct zonelist *zonelist, gfp_t gfp_flags); | 58 | extern bool oom_zonelist_trylock(struct zonelist *zonelist, gfp_t gfp_flags); |
59 | extern void clear_zonelist_oom(struct zonelist *zonelist, gfp_t gfp_flags); | 59 | extern void oom_zonelist_unlock(struct zonelist *zonelist, gfp_t gfp_flags); |
60 | 60 | ||
61 | extern void check_panic_on_oom(enum oom_constraint constraint, gfp_t gfp_mask, | 61 | extern void check_panic_on_oom(enum oom_constraint constraint, gfp_t gfp_mask, |
62 | int order, const nodemask_t *nodemask); | 62 | int order, const nodemask_t *nodemask); |
diff --git a/include/linux/page-flags.h b/include/linux/page-flags.h index 8304959ad336..e1f5fcd79792 100644 --- a/include/linux/page-flags.h +++ b/include/linux/page-flags.h | |||
@@ -171,13 +171,12 @@ static inline int __TestClearPage##uname(struct page *page) \ | |||
171 | #define __PAGEFLAG(uname, lname) TESTPAGEFLAG(uname, lname) \ | 171 | #define __PAGEFLAG(uname, lname) TESTPAGEFLAG(uname, lname) \ |
172 | __SETPAGEFLAG(uname, lname) __CLEARPAGEFLAG(uname, lname) | 172 | __SETPAGEFLAG(uname, lname) __CLEARPAGEFLAG(uname, lname) |
173 | 173 | ||
174 | #define PAGEFLAG_FALSE(uname) \ | ||
175 | static inline int Page##uname(const struct page *page) \ | ||
176 | { return 0; } | ||
177 | |||
178 | #define TESTSCFLAG(uname, lname) \ | 174 | #define TESTSCFLAG(uname, lname) \ |
179 | TESTSETFLAG(uname, lname) TESTCLEARFLAG(uname, lname) | 175 | TESTSETFLAG(uname, lname) TESTCLEARFLAG(uname, lname) |
180 | 176 | ||
177 | #define TESTPAGEFLAG_FALSE(uname) \ | ||
178 | static inline int Page##uname(const struct page *page) { return 0; } | ||
179 | |||
181 | #define SETPAGEFLAG_NOOP(uname) \ | 180 | #define SETPAGEFLAG_NOOP(uname) \ |
182 | static inline void SetPage##uname(struct page *page) { } | 181 | static inline void SetPage##uname(struct page *page) { } |
183 | 182 | ||
@@ -187,12 +186,21 @@ static inline void ClearPage##uname(struct page *page) { } | |||
187 | #define __CLEARPAGEFLAG_NOOP(uname) \ | 186 | #define __CLEARPAGEFLAG_NOOP(uname) \ |
188 | static inline void __ClearPage##uname(struct page *page) { } | 187 | static inline void __ClearPage##uname(struct page *page) { } |
189 | 188 | ||
189 | #define TESTSETFLAG_FALSE(uname) \ | ||
190 | static inline int TestSetPage##uname(struct page *page) { return 0; } | ||
191 | |||
190 | #define TESTCLEARFLAG_FALSE(uname) \ | 192 | #define TESTCLEARFLAG_FALSE(uname) \ |
191 | static inline int TestClearPage##uname(struct page *page) { return 0; } | 193 | static inline int TestClearPage##uname(struct page *page) { return 0; } |
192 | 194 | ||
193 | #define __TESTCLEARFLAG_FALSE(uname) \ | 195 | #define __TESTCLEARFLAG_FALSE(uname) \ |
194 | static inline int __TestClearPage##uname(struct page *page) { return 0; } | 196 | static inline int __TestClearPage##uname(struct page *page) { return 0; } |
195 | 197 | ||
198 | #define PAGEFLAG_FALSE(uname) TESTPAGEFLAG_FALSE(uname) \ | ||
199 | SETPAGEFLAG_NOOP(uname) CLEARPAGEFLAG_NOOP(uname) | ||
200 | |||
201 | #define TESTSCFLAG_FALSE(uname) \ | ||
202 | TESTSETFLAG_FALSE(uname) TESTCLEARFLAG_FALSE(uname) | ||
203 | |||
196 | struct page; /* forward declaration */ | 204 | struct page; /* forward declaration */ |
197 | 205 | ||
198 | TESTPAGEFLAG(Locked, locked) | 206 | TESTPAGEFLAG(Locked, locked) |
@@ -248,7 +256,6 @@ PAGEFLAG_FALSE(HighMem) | |||
248 | PAGEFLAG(SwapCache, swapcache) | 256 | PAGEFLAG(SwapCache, swapcache) |
249 | #else | 257 | #else |
250 | PAGEFLAG_FALSE(SwapCache) | 258 | PAGEFLAG_FALSE(SwapCache) |
251 | SETPAGEFLAG_NOOP(SwapCache) CLEARPAGEFLAG_NOOP(SwapCache) | ||
252 | #endif | 259 | #endif |
253 | 260 | ||
254 | PAGEFLAG(Unevictable, unevictable) __CLEARPAGEFLAG(Unevictable, unevictable) | 261 | PAGEFLAG(Unevictable, unevictable) __CLEARPAGEFLAG(Unevictable, unevictable) |
@@ -258,8 +265,8 @@ PAGEFLAG(Unevictable, unevictable) __CLEARPAGEFLAG(Unevictable, unevictable) | |||
258 | PAGEFLAG(Mlocked, mlocked) __CLEARPAGEFLAG(Mlocked, mlocked) | 265 | PAGEFLAG(Mlocked, mlocked) __CLEARPAGEFLAG(Mlocked, mlocked) |
259 | TESTSCFLAG(Mlocked, mlocked) __TESTCLEARFLAG(Mlocked, mlocked) | 266 | TESTSCFLAG(Mlocked, mlocked) __TESTCLEARFLAG(Mlocked, mlocked) |
260 | #else | 267 | #else |
261 | PAGEFLAG_FALSE(Mlocked) SETPAGEFLAG_NOOP(Mlocked) | 268 | PAGEFLAG_FALSE(Mlocked) __CLEARPAGEFLAG_NOOP(Mlocked) |
262 | TESTCLEARFLAG_FALSE(Mlocked) __TESTCLEARFLAG_FALSE(Mlocked) | 269 | TESTSCFLAG_FALSE(Mlocked) __TESTCLEARFLAG_FALSE(Mlocked) |
263 | #endif | 270 | #endif |
264 | 271 | ||
265 | #ifdef CONFIG_ARCH_USES_PG_UNCACHED | 272 | #ifdef CONFIG_ARCH_USES_PG_UNCACHED |
diff --git a/include/linux/pagemap.h b/include/linux/pagemap.h index e1474ae18c88..3df8c7db7a4e 100644 --- a/include/linux/pagemap.h +++ b/include/linux/pagemap.h | |||
@@ -484,6 +484,9 @@ static inline int lock_page_killable(struct page *page) | |||
484 | /* | 484 | /* |
485 | * lock_page_or_retry - Lock the page, unless this would block and the | 485 | * lock_page_or_retry - Lock the page, unless this would block and the |
486 | * caller indicated that it can handle a retry. | 486 | * caller indicated that it can handle a retry. |
487 | * | ||
488 | * Return value and mmap_sem implications depend on flags; see | ||
489 | * __lock_page_or_retry(). | ||
487 | */ | 490 | */ |
488 | static inline int lock_page_or_retry(struct page *page, struct mm_struct *mm, | 491 | static inline int lock_page_or_retry(struct page *page, struct mm_struct *mm, |
489 | unsigned int flags) | 492 | unsigned int flags) |
diff --git a/include/linux/printk.h b/include/linux/printk.h index 319ff7e53efb..0990997a5304 100644 --- a/include/linux/printk.h +++ b/include/linux/printk.h | |||
@@ -31,7 +31,7 @@ static inline const char *printk_skip_level(const char *buffer) | |||
31 | } | 31 | } |
32 | 32 | ||
33 | /* printk's without a loglevel use this.. */ | 33 | /* printk's without a loglevel use this.. */ |
34 | #define DEFAULT_MESSAGE_LOGLEVEL CONFIG_DEFAULT_MESSAGE_LOGLEVEL | 34 | #define MESSAGE_LOGLEVEL_DEFAULT CONFIG_MESSAGE_LOGLEVEL_DEFAULT |
35 | 35 | ||
36 | /* We show everything that is MORE important than this.. */ | 36 | /* We show everything that is MORE important than this.. */ |
37 | #define CONSOLE_LOGLEVEL_SILENT 0 /* Mum's the word */ | 37 | #define CONSOLE_LOGLEVEL_SILENT 0 /* Mum's the word */ |
diff --git a/include/linux/rculist.h b/include/linux/rculist.h index 8183b46fbaa2..372ad5e0dcb8 100644 --- a/include/linux/rculist.h +++ b/include/linux/rculist.h | |||
@@ -432,9 +432,9 @@ static inline void hlist_add_before_rcu(struct hlist_node *n, | |||
432 | } | 432 | } |
433 | 433 | ||
434 | /** | 434 | /** |
435 | * hlist_add_after_rcu | 435 | * hlist_add_behind_rcu |
436 | * @prev: the existing element to add the new element after. | ||
437 | * @n: the new element to add to the hash list. | 436 | * @n: the new element to add to the hash list. |
437 | * @prev: the existing element to add the new element after. | ||
438 | * | 438 | * |
439 | * Description: | 439 | * Description: |
440 | * Adds the specified element to the specified hlist | 440 | * Adds the specified element to the specified hlist |
@@ -449,8 +449,8 @@ static inline void hlist_add_before_rcu(struct hlist_node *n, | |||
449 | * hlist_for_each_entry_rcu(), used to prevent memory-consistency | 449 | * hlist_for_each_entry_rcu(), used to prevent memory-consistency |
450 | * problems on Alpha CPUs. | 450 | * problems on Alpha CPUs. |
451 | */ | 451 | */ |
452 | static inline void hlist_add_after_rcu(struct hlist_node *prev, | 452 | static inline void hlist_add_behind_rcu(struct hlist_node *n, |
453 | struct hlist_node *n) | 453 | struct hlist_node *prev) |
454 | { | 454 | { |
455 | n->next = prev->next; | 455 | n->next = prev->next; |
456 | n->pprev = &prev->next; | 456 | n->pprev = &prev->next; |
diff --git a/include/linux/swap.h b/include/linux/swap.h index 4bdbee80eede..1eb64043c076 100644 --- a/include/linux/swap.h +++ b/include/linux/swap.h | |||
@@ -311,7 +311,6 @@ extern void lru_add_page_tail(struct page *page, struct page *page_tail, | |||
311 | struct lruvec *lruvec, struct list_head *head); | 311 | struct lruvec *lruvec, struct list_head *head); |
312 | extern void activate_page(struct page *); | 312 | extern void activate_page(struct page *); |
313 | extern void mark_page_accessed(struct page *); | 313 | extern void mark_page_accessed(struct page *); |
314 | extern void init_page_accessed(struct page *page); | ||
315 | extern void lru_add_drain(void); | 314 | extern void lru_add_drain(void); |
316 | extern void lru_add_drain_cpu(int cpu); | 315 | extern void lru_add_drain_cpu(int cpu); |
317 | extern void lru_add_drain_all(void); | 316 | extern void lru_add_drain_all(void); |
diff --git a/include/linux/vmalloc.h b/include/linux/vmalloc.h index 4b8a89189a29..b87696fdf06a 100644 --- a/include/linux/vmalloc.h +++ b/include/linux/vmalloc.h | |||
@@ -113,7 +113,7 @@ extern struct vm_struct *remove_vm_area(const void *addr); | |||
113 | extern struct vm_struct *find_vm_area(const void *addr); | 113 | extern struct vm_struct *find_vm_area(const void *addr); |
114 | 114 | ||
115 | extern int map_vm_area(struct vm_struct *area, pgprot_t prot, | 115 | extern int map_vm_area(struct vm_struct *area, pgprot_t prot, |
116 | struct page ***pages); | 116 | struct page **pages); |
117 | #ifdef CONFIG_MMU | 117 | #ifdef CONFIG_MMU |
118 | extern int map_kernel_range_noflush(unsigned long start, unsigned long size, | 118 | extern int map_kernel_range_noflush(unsigned long start, unsigned long size, |
119 | pgprot_t prot, struct page **pages); | 119 | pgprot_t prot, struct page **pages); |
diff --git a/include/linux/zbud.h b/include/linux/zbud.h index 13af0d450bf6..f9d41a6e361f 100644 --- a/include/linux/zbud.h +++ b/include/linux/zbud.h | |||
@@ -11,7 +11,7 @@ struct zbud_ops { | |||
11 | 11 | ||
12 | struct zbud_pool *zbud_create_pool(gfp_t gfp, struct zbud_ops *ops); | 12 | struct zbud_pool *zbud_create_pool(gfp_t gfp, struct zbud_ops *ops); |
13 | void zbud_destroy_pool(struct zbud_pool *pool); | 13 | void zbud_destroy_pool(struct zbud_pool *pool); |
14 | int zbud_alloc(struct zbud_pool *pool, unsigned int size, gfp_t gfp, | 14 | int zbud_alloc(struct zbud_pool *pool, size_t size, gfp_t gfp, |
15 | unsigned long *handle); | 15 | unsigned long *handle); |
16 | void zbud_free(struct zbud_pool *pool, unsigned long handle); | 16 | void zbud_free(struct zbud_pool *pool, unsigned long handle); |
17 | int zbud_reclaim_page(struct zbud_pool *pool, unsigned int retries); | 17 | int zbud_reclaim_page(struct zbud_pool *pool, unsigned int retries); |
diff --git a/include/linux/zlib.h b/include/linux/zlib.h index 9c5a6b4de0a3..197abb2a54c5 100644 --- a/include/linux/zlib.h +++ b/include/linux/zlib.h | |||
@@ -493,64 +493,6 @@ extern int deflateInit2 (z_streamp strm, | |||
493 | method). msg is set to null if there is no error message. deflateInit2 does | 493 | method). msg is set to null if there is no error message. deflateInit2 does |
494 | not perform any compression: this will be done by deflate(). | 494 | not perform any compression: this will be done by deflate(). |
495 | */ | 495 | */ |
496 | |||
497 | #if 0 | ||
498 | extern int zlib_deflateSetDictionary (z_streamp strm, | ||
499 | const Byte *dictionary, | ||
500 | uInt dictLength); | ||
501 | #endif | ||
502 | /* | ||
503 | Initializes the compression dictionary from the given byte sequence | ||
504 | without producing any compressed output. This function must be called | ||
505 | immediately after deflateInit, deflateInit2 or deflateReset, before any | ||
506 | call of deflate. The compressor and decompressor must use exactly the same | ||
507 | dictionary (see inflateSetDictionary). | ||
508 | |||
509 | The dictionary should consist of strings (byte sequences) that are likely | ||
510 | to be encountered later in the data to be compressed, with the most commonly | ||
511 | used strings preferably put towards the end of the dictionary. Using a | ||
512 | dictionary is most useful when the data to be compressed is short and can be | ||
513 | predicted with good accuracy; the data can then be compressed better than | ||
514 | with the default empty dictionary. | ||
515 | |||
516 | Depending on the size of the compression data structures selected by | ||
517 | deflateInit or deflateInit2, a part of the dictionary may in effect be | ||
518 | discarded, for example if the dictionary is larger than the window size in | ||
519 | deflate or deflate2. Thus the strings most likely to be useful should be | ||
520 | put at the end of the dictionary, not at the front. | ||
521 | |||
522 | Upon return of this function, strm->adler is set to the Adler32 value | ||
523 | of the dictionary; the decompressor may later use this value to determine | ||
524 | which dictionary has been used by the compressor. (The Adler32 value | ||
525 | applies to the whole dictionary even if only a subset of the dictionary is | ||
526 | actually used by the compressor.) | ||
527 | |||
528 | deflateSetDictionary returns Z_OK if success, or Z_STREAM_ERROR if a | ||
529 | parameter is invalid (such as NULL dictionary) or the stream state is | ||
530 | inconsistent (for example if deflate has already been called for this stream | ||
531 | or if the compression method is bsort). deflateSetDictionary does not | ||
532 | perform any compression: this will be done by deflate(). | ||
533 | */ | ||
534 | |||
535 | #if 0 | ||
536 | extern int zlib_deflateCopy (z_streamp dest, z_streamp source); | ||
537 | #endif | ||
538 | |||
539 | /* | ||
540 | Sets the destination stream as a complete copy of the source stream. | ||
541 | |||
542 | This function can be useful when several compression strategies will be | ||
543 | tried, for example when there are several ways of pre-processing the input | ||
544 | data with a filter. The streams that will be discarded should then be freed | ||
545 | by calling deflateEnd. Note that deflateCopy duplicates the internal | ||
546 | compression state which can be quite large, so this strategy is slow and | ||
547 | can consume lots of memory. | ||
548 | |||
549 | deflateCopy returns Z_OK if success, Z_MEM_ERROR if there was not | ||
550 | enough memory, Z_STREAM_ERROR if the source stream state was inconsistent | ||
551 | (such as zalloc being NULL). msg is left unchanged in both source and | ||
552 | destination. | ||
553 | */ | ||
554 | 496 | ||
555 | extern int zlib_deflateReset (z_streamp strm); | 497 | extern int zlib_deflateReset (z_streamp strm); |
556 | /* | 498 | /* |
@@ -568,27 +510,6 @@ static inline unsigned long deflateBound(unsigned long s) | |||
568 | return s + ((s + 7) >> 3) + ((s + 63) >> 6) + 11; | 510 | return s + ((s + 7) >> 3) + ((s + 63) >> 6) + 11; |
569 | } | 511 | } |
570 | 512 | ||
571 | #if 0 | ||
572 | extern int zlib_deflateParams (z_streamp strm, int level, int strategy); | ||
573 | #endif | ||
574 | /* | ||
575 | Dynamically update the compression level and compression strategy. The | ||
576 | interpretation of level and strategy is as in deflateInit2. This can be | ||
577 | used to switch between compression and straight copy of the input data, or | ||
578 | to switch to a different kind of input data requiring a different | ||
579 | strategy. If the compression level is changed, the input available so far | ||
580 | is compressed with the old level (and may be flushed); the new level will | ||
581 | take effect only at the next call of deflate(). | ||
582 | |||
583 | Before the call of deflateParams, the stream state must be set as for | ||
584 | a call of deflate(), since the currently available input may have to | ||
585 | be compressed and flushed. In particular, strm->avail_out must be non-zero. | ||
586 | |||
587 | deflateParams returns Z_OK if success, Z_STREAM_ERROR if the source | ||
588 | stream state was inconsistent or if a parameter was invalid, Z_BUF_ERROR | ||
589 | if strm->avail_out was zero. | ||
590 | */ | ||
591 | |||
592 | /* | 513 | /* |
593 | extern int inflateInit2 (z_streamp strm, int windowBits); | 514 | extern int inflateInit2 (z_streamp strm, int windowBits); |
594 | 515 | ||
@@ -631,45 +552,6 @@ extern int inflateInit2 (z_streamp strm, int windowBits); | |||
631 | and avail_out are unchanged.) | 552 | and avail_out are unchanged.) |
632 | */ | 553 | */ |
633 | 554 | ||
634 | extern int zlib_inflateSetDictionary (z_streamp strm, | ||
635 | const Byte *dictionary, | ||
636 | uInt dictLength); | ||
637 | /* | ||
638 | Initializes the decompression dictionary from the given uncompressed byte | ||
639 | sequence. This function must be called immediately after a call of inflate, | ||
640 | if that call returned Z_NEED_DICT. The dictionary chosen by the compressor | ||
641 | can be determined from the adler32 value returned by that call of inflate. | ||
642 | The compressor and decompressor must use exactly the same dictionary (see | ||
643 | deflateSetDictionary). For raw inflate, this function can be called | ||
644 | immediately after inflateInit2() or inflateReset() and before any call of | ||
645 | inflate() to set the dictionary. The application must insure that the | ||
646 | dictionary that was used for compression is provided. | ||
647 | |||
648 | inflateSetDictionary returns Z_OK if success, Z_STREAM_ERROR if a | ||
649 | parameter is invalid (such as NULL dictionary) or the stream state is | ||
650 | inconsistent, Z_DATA_ERROR if the given dictionary doesn't match the | ||
651 | expected one (incorrect adler32 value). inflateSetDictionary does not | ||
652 | perform any decompression: this will be done by subsequent calls of | ||
653 | inflate(). | ||
654 | */ | ||
655 | |||
656 | #if 0 | ||
657 | extern int zlib_inflateSync (z_streamp strm); | ||
658 | #endif | ||
659 | /* | ||
660 | Skips invalid compressed data until a full flush point (see above the | ||
661 | description of deflate with Z_FULL_FLUSH) can be found, or until all | ||
662 | available input is skipped. No output is provided. | ||
663 | |||
664 | inflateSync returns Z_OK if a full flush point has been found, Z_BUF_ERROR | ||
665 | if no more input was provided, Z_DATA_ERROR if no flush point has been found, | ||
666 | or Z_STREAM_ERROR if the stream structure was inconsistent. In the success | ||
667 | case, the application may save the current current value of total_in which | ||
668 | indicates where valid compressed data was found. In the error case, the | ||
669 | application may repeatedly call inflateSync, providing more input each time, | ||
670 | until success or end of the input data. | ||
671 | */ | ||
672 | |||
673 | extern int zlib_inflateReset (z_streamp strm); | 555 | extern int zlib_inflateReset (z_streamp strm); |
674 | /* | 556 | /* |
675 | This function is equivalent to inflateEnd followed by inflateInit, | 557 | This function is equivalent to inflateEnd followed by inflateInit, |
diff --git a/include/linux/zpool.h b/include/linux/zpool.h new file mode 100644 index 000000000000..f14bd75f08b3 --- /dev/null +++ b/include/linux/zpool.h | |||
@@ -0,0 +1,106 @@ | |||
1 | /* | ||
2 | * zpool memory storage api | ||
3 | * | ||
4 | * Copyright (C) 2014 Dan Streetman | ||
5 | * | ||
6 | * This is a common frontend for the zbud and zsmalloc memory | ||
7 | * storage pool implementations. Typically, this is used to | ||
8 | * store compressed memory. | ||
9 | */ | ||
10 | |||
11 | #ifndef _ZPOOL_H_ | ||
12 | #define _ZPOOL_H_ | ||
13 | |||
14 | struct zpool; | ||
15 | |||
16 | struct zpool_ops { | ||
17 | int (*evict)(struct zpool *pool, unsigned long handle); | ||
18 | }; | ||
19 | |||
20 | /* | ||
21 | * Control how a handle is mapped. It will be ignored if the | ||
22 | * implementation does not support it. Its use is optional. | ||
23 | * Note that this does not refer to memory protection, it | ||
24 | * refers to how the memory will be copied in/out if copying | ||
25 | * is necessary during mapping; read-write is the safest as | ||
26 | * it copies the existing memory in on map, and copies the | ||
27 | * changed memory back out on unmap. Write-only does not copy | ||
28 | * in the memory and should only be used for initialization. | ||
29 | * If in doubt, use ZPOOL_MM_DEFAULT which is read-write. | ||
30 | */ | ||
31 | enum zpool_mapmode { | ||
32 | ZPOOL_MM_RW, /* normal read-write mapping */ | ||
33 | ZPOOL_MM_RO, /* read-only (no copy-out at unmap time) */ | ||
34 | ZPOOL_MM_WO, /* write-only (no copy-in at map time) */ | ||
35 | |||
36 | ZPOOL_MM_DEFAULT = ZPOOL_MM_RW | ||
37 | }; | ||
38 | |||
39 | struct zpool *zpool_create_pool(char *type, gfp_t gfp, struct zpool_ops *ops); | ||
40 | |||
41 | char *zpool_get_type(struct zpool *pool); | ||
42 | |||
43 | void zpool_destroy_pool(struct zpool *pool); | ||
44 | |||
45 | int zpool_malloc(struct zpool *pool, size_t size, gfp_t gfp, | ||
46 | unsigned long *handle); | ||
47 | |||
48 | void zpool_free(struct zpool *pool, unsigned long handle); | ||
49 | |||
50 | int zpool_shrink(struct zpool *pool, unsigned int pages, | ||
51 | unsigned int *reclaimed); | ||
52 | |||
53 | void *zpool_map_handle(struct zpool *pool, unsigned long handle, | ||
54 | enum zpool_mapmode mm); | ||
55 | |||
56 | void zpool_unmap_handle(struct zpool *pool, unsigned long handle); | ||
57 | |||
58 | u64 zpool_get_total_size(struct zpool *pool); | ||
59 | |||
60 | |||
61 | /** | ||
62 | * struct zpool_driver - driver implementation for zpool | ||
63 | * @type: name of the driver. | ||
64 | * @list: entry in the list of zpool drivers. | ||
65 | * @create: create a new pool. | ||
66 | * @destroy: destroy a pool. | ||
67 | * @malloc: allocate mem from a pool. | ||
68 | * @free: free mem from a pool. | ||
69 | * @shrink: shrink the pool. | ||
70 | * @map: map a handle. | ||
71 | * @unmap: unmap a handle. | ||
72 | * @total_size: get total size of a pool. | ||
73 | * | ||
74 | * This is created by a zpool implementation and registered | ||
75 | * with zpool. | ||
76 | */ | ||
77 | struct zpool_driver { | ||
78 | char *type; | ||
79 | struct module *owner; | ||
80 | atomic_t refcount; | ||
81 | struct list_head list; | ||
82 | |||
83 | void *(*create)(gfp_t gfp, struct zpool_ops *ops); | ||
84 | void (*destroy)(void *pool); | ||
85 | |||
86 | int (*malloc)(void *pool, size_t size, gfp_t gfp, | ||
87 | unsigned long *handle); | ||
88 | void (*free)(void *pool, unsigned long handle); | ||
89 | |||
90 | int (*shrink)(void *pool, unsigned int pages, | ||
91 | unsigned int *reclaimed); | ||
92 | |||
93 | void *(*map)(void *pool, unsigned long handle, | ||
94 | enum zpool_mapmode mm); | ||
95 | void (*unmap)(void *pool, unsigned long handle); | ||
96 | |||
97 | u64 (*total_size)(void *pool); | ||
98 | }; | ||
99 | |||
100 | void zpool_register_driver(struct zpool_driver *driver); | ||
101 | |||
102 | int zpool_unregister_driver(struct zpool_driver *driver); | ||
103 | |||
104 | int zpool_evict(void *pool, unsigned long handle); | ||
105 | |||
106 | #endif | ||
diff --git a/include/trace/events/migrate.h b/include/trace/events/migrate.h index 4e4f2f8b1ac2..dd2b5467d905 100644 --- a/include/trace/events/migrate.h +++ b/include/trace/events/migrate.h | |||
@@ -17,6 +17,7 @@ | |||
17 | {MR_MEMORY_HOTPLUG, "memory_hotplug"}, \ | 17 | {MR_MEMORY_HOTPLUG, "memory_hotplug"}, \ |
18 | {MR_SYSCALL, "syscall_or_cpuset"}, \ | 18 | {MR_SYSCALL, "syscall_or_cpuset"}, \ |
19 | {MR_MEMPOLICY_MBIND, "mempolicy_mbind"}, \ | 19 | {MR_MEMPOLICY_MBIND, "mempolicy_mbind"}, \ |
20 | {MR_NUMA_MISPLACED, "numa_misplaced"}, \ | ||
20 | {MR_CMA, "cma"} | 21 | {MR_CMA, "cma"} |
21 | 22 | ||
22 | TRACE_EVENT(mm_migrate_pages, | 23 | TRACE_EVENT(mm_migrate_pages, |
diff --git a/include/trace/events/pagemap.h b/include/trace/events/pagemap.h index 1c9fabde69e4..ce0803b8d05f 100644 --- a/include/trace/events/pagemap.h +++ b/include/trace/events/pagemap.h | |||
@@ -28,12 +28,10 @@ TRACE_EVENT(mm_lru_insertion, | |||
28 | 28 | ||
29 | TP_PROTO( | 29 | TP_PROTO( |
30 | struct page *page, | 30 | struct page *page, |
31 | unsigned long pfn, | 31 | int lru |
32 | int lru, | ||
33 | unsigned long flags | ||
34 | ), | 32 | ), |
35 | 33 | ||
36 | TP_ARGS(page, pfn, lru, flags), | 34 | TP_ARGS(page, lru), |
37 | 35 | ||
38 | TP_STRUCT__entry( | 36 | TP_STRUCT__entry( |
39 | __field(struct page *, page ) | 37 | __field(struct page *, page ) |
@@ -44,9 +42,9 @@ TRACE_EVENT(mm_lru_insertion, | |||
44 | 42 | ||
45 | TP_fast_assign( | 43 | TP_fast_assign( |
46 | __entry->page = page; | 44 | __entry->page = page; |
47 | __entry->pfn = pfn; | 45 | __entry->pfn = page_to_pfn(page); |
48 | __entry->lru = lru; | 46 | __entry->lru = lru; |
49 | __entry->flags = flags; | 47 | __entry->flags = trace_pagemap_flags(page); |
50 | ), | 48 | ), |
51 | 49 | ||
52 | /* Flag format is based on page-types.c formatting for pagemap */ | 50 | /* Flag format is based on page-types.c formatting for pagemap */ |
@@ -64,9 +62,9 @@ TRACE_EVENT(mm_lru_insertion, | |||
64 | 62 | ||
65 | TRACE_EVENT(mm_lru_activate, | 63 | TRACE_EVENT(mm_lru_activate, |
66 | 64 | ||
67 | TP_PROTO(struct page *page, unsigned long pfn), | 65 | TP_PROTO(struct page *page), |
68 | 66 | ||
69 | TP_ARGS(page, pfn), | 67 | TP_ARGS(page), |
70 | 68 | ||
71 | TP_STRUCT__entry( | 69 | TP_STRUCT__entry( |
72 | __field(struct page *, page ) | 70 | __field(struct page *, page ) |
@@ -75,7 +73,7 @@ TRACE_EVENT(mm_lru_activate, | |||
75 | 73 | ||
76 | TP_fast_assign( | 74 | TP_fast_assign( |
77 | __entry->page = page; | 75 | __entry->page = page; |
78 | __entry->pfn = pfn; | 76 | __entry->pfn = page_to_pfn(page); |
79 | ), | 77 | ), |
80 | 78 | ||
81 | /* Flag format is based on page-types.c formatting for pagemap */ | 79 | /* Flag format is based on page-types.c formatting for pagemap */ |
diff --git a/init/Kconfig b/init/Kconfig index 41066e49e880..a291b7ef4738 100644 --- a/init/Kconfig +++ b/init/Kconfig | |||
@@ -807,15 +807,53 @@ config LOG_BUF_SHIFT | |||
807 | range 12 21 | 807 | range 12 21 |
808 | default 17 | 808 | default 17 |
809 | help | 809 | help |
810 | Select kernel log buffer size as a power of 2. | 810 | Select the minimal kernel log buffer size as a power of 2. |
811 | The final size is affected by LOG_CPU_MAX_BUF_SHIFT config | ||
812 | parameter, see below. Any higher size also might be forced | ||
813 | by "log_buf_len" boot parameter. | ||
814 | |||
811 | Examples: | 815 | Examples: |
812 | 17 => 128 KB | 816 | 17 => 128 KB |
813 | 16 => 64 KB | 817 | 16 => 64 KB |
814 | 15 => 32 KB | 818 | 15 => 32 KB |
815 | 14 => 16 KB | 819 | 14 => 16 KB |
816 | 13 => 8 KB | 820 | 13 => 8 KB |
817 | 12 => 4 KB | 821 | 12 => 4 KB |
818 | 822 | ||
823 | config LOG_CPU_MAX_BUF_SHIFT | ||
824 | int "CPU kernel log buffer size contribution (13 => 8 KB, 17 => 128KB)" | ||
825 | range 0 21 | ||
826 | default 12 if !BASE_SMALL | ||
827 | default 0 if BASE_SMALL | ||
828 | help | ||
829 | This option allows to increase the default ring buffer size | ||
830 | according to the number of CPUs. The value defines the contribution | ||
831 | of each CPU as a power of 2. The used space is typically only few | ||
832 | lines however it might be much more when problems are reported, | ||
833 | e.g. backtraces. | ||
834 | |||
835 | The increased size means that a new buffer has to be allocated and | ||
836 | the original static one is unused. It makes sense only on systems | ||
837 | with more CPUs. Therefore this value is used only when the sum of | ||
838 | contributions is greater than the half of the default kernel ring | ||
839 | buffer as defined by LOG_BUF_SHIFT. The default values are set | ||
840 | so that more than 64 CPUs are needed to trigger the allocation. | ||
841 | |||
842 | Also this option is ignored when "log_buf_len" kernel parameter is | ||
843 | used as it forces an exact (power of two) size of the ring buffer. | ||
844 | |||
845 | The number of possible CPUs is used for this computation ignoring | ||
846 | hotplugging making the compuation optimal for the the worst case | ||
847 | scenerio while allowing a simple algorithm to be used from bootup. | ||
848 | |||
849 | Examples shift values and their meaning: | ||
850 | 17 => 128 KB for each CPU | ||
851 | 16 => 64 KB for each CPU | ||
852 | 15 => 32 KB for each CPU | ||
853 | 14 => 16 KB for each CPU | ||
854 | 13 => 8 KB for each CPU | ||
855 | 12 => 4 KB for each CPU | ||
856 | |||
819 | # | 857 | # |
820 | # Architectures with an unreliable sched_clock() should select this: | 858 | # Architectures with an unreliable sched_clock() should select this: |
821 | # | 859 | # |
diff --git a/kernel/auditfilter.c b/kernel/auditfilter.c index 8e9bc9c3dbb7..c447cd9848d1 100644 --- a/kernel/auditfilter.c +++ b/kernel/auditfilter.c | |||
@@ -106,7 +106,7 @@ static inline struct audit_entry *audit_init_entry(u32 field_count) | |||
106 | if (unlikely(!entry)) | 106 | if (unlikely(!entry)) |
107 | return NULL; | 107 | return NULL; |
108 | 108 | ||
109 | fields = kzalloc(sizeof(*fields) * field_count, GFP_KERNEL); | 109 | fields = kcalloc(field_count, sizeof(*fields), GFP_KERNEL); |
110 | if (unlikely(!fields)) { | 110 | if (unlikely(!fields)) { |
111 | kfree(entry); | 111 | kfree(entry); |
112 | return NULL; | 112 | return NULL; |
@@ -160,7 +160,7 @@ static __u32 *classes[AUDIT_SYSCALL_CLASSES]; | |||
160 | 160 | ||
161 | int __init audit_register_class(int class, unsigned *list) | 161 | int __init audit_register_class(int class, unsigned *list) |
162 | { | 162 | { |
163 | __u32 *p = kzalloc(AUDIT_BITMASK_SIZE * sizeof(__u32), GFP_KERNEL); | 163 | __u32 *p = kcalloc(AUDIT_BITMASK_SIZE, sizeof(__u32), GFP_KERNEL); |
164 | if (!p) | 164 | if (!p) |
165 | return -ENOMEM; | 165 | return -ENOMEM; |
166 | while (*list != ~0U) { | 166 | while (*list != ~0U) { |
diff --git a/kernel/exit.c b/kernel/exit.c index e5c4668f1799..88c6b3e42583 100644 --- a/kernel/exit.c +++ b/kernel/exit.c | |||
@@ -455,6 +455,7 @@ static void exit_mm(struct task_struct * tsk) | |||
455 | task_unlock(tsk); | 455 | task_unlock(tsk); |
456 | mm_update_next_owner(mm); | 456 | mm_update_next_owner(mm); |
457 | mmput(mm); | 457 | mmput(mm); |
458 | clear_thread_flag(TIF_MEMDIE); | ||
458 | } | 459 | } |
459 | 460 | ||
460 | /* | 461 | /* |
diff --git a/kernel/printk/printk.c b/kernel/printk/printk.c index 13e839dbca07..de1a6bb6861d 100644 --- a/kernel/printk/printk.c +++ b/kernel/printk/printk.c | |||
@@ -45,6 +45,7 @@ | |||
45 | #include <linux/poll.h> | 45 | #include <linux/poll.h> |
46 | #include <linux/irq_work.h> | 46 | #include <linux/irq_work.h> |
47 | #include <linux/utsname.h> | 47 | #include <linux/utsname.h> |
48 | #include <linux/ctype.h> | ||
48 | 49 | ||
49 | #include <asm/uaccess.h> | 50 | #include <asm/uaccess.h> |
50 | 51 | ||
@@ -56,7 +57,7 @@ | |||
56 | 57 | ||
57 | int console_printk[4] = { | 58 | int console_printk[4] = { |
58 | CONSOLE_LOGLEVEL_DEFAULT, /* console_loglevel */ | 59 | CONSOLE_LOGLEVEL_DEFAULT, /* console_loglevel */ |
59 | DEFAULT_MESSAGE_LOGLEVEL, /* default_message_loglevel */ | 60 | MESSAGE_LOGLEVEL_DEFAULT, /* default_message_loglevel */ |
60 | CONSOLE_LOGLEVEL_MIN, /* minimum_console_loglevel */ | 61 | CONSOLE_LOGLEVEL_MIN, /* minimum_console_loglevel */ |
61 | CONSOLE_LOGLEVEL_DEFAULT, /* default_console_loglevel */ | 62 | CONSOLE_LOGLEVEL_DEFAULT, /* default_console_loglevel */ |
62 | }; | 63 | }; |
@@ -113,9 +114,9 @@ static int __down_trylock_console_sem(unsigned long ip) | |||
113 | * This is used for debugging the mess that is the VT code by | 114 | * This is used for debugging the mess that is the VT code by |
114 | * keeping track if we have the console semaphore held. It's | 115 | * keeping track if we have the console semaphore held. It's |
115 | * definitely not the perfect debug tool (we don't know if _WE_ | 116 | * definitely not the perfect debug tool (we don't know if _WE_ |
116 | * hold it are racing, but it helps tracking those weird code | 117 | * hold it and are racing, but it helps tracking those weird code |
117 | * path in the console code where we end up in places I want | 118 | * paths in the console code where we end up in places I want |
118 | * locked without the console sempahore held | 119 | * locked without the console sempahore held). |
119 | */ | 120 | */ |
120 | static int console_locked, console_suspended; | 121 | static int console_locked, console_suspended; |
121 | 122 | ||
@@ -146,8 +147,8 @@ static int console_may_schedule; | |||
146 | * the overall length of the record. | 147 | * the overall length of the record. |
147 | * | 148 | * |
148 | * The heads to the first and last entry in the buffer, as well as the | 149 | * The heads to the first and last entry in the buffer, as well as the |
149 | * sequence numbers of these both entries are maintained when messages | 150 | * sequence numbers of these entries are maintained when messages are |
150 | * are stored.. | 151 | * stored. |
151 | * | 152 | * |
152 | * If the heads indicate available messages, the length in the header | 153 | * If the heads indicate available messages, the length in the header |
153 | * tells the start next message. A length == 0 for the next message | 154 | * tells the start next message. A length == 0 for the next message |
@@ -257,7 +258,7 @@ static u64 clear_seq; | |||
257 | static u32 clear_idx; | 258 | static u32 clear_idx; |
258 | 259 | ||
259 | #define PREFIX_MAX 32 | 260 | #define PREFIX_MAX 32 |
260 | #define LOG_LINE_MAX 1024 - PREFIX_MAX | 261 | #define LOG_LINE_MAX (1024 - PREFIX_MAX) |
261 | 262 | ||
262 | /* record buffer */ | 263 | /* record buffer */ |
263 | #if defined(CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS) | 264 | #if defined(CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS) |
@@ -266,6 +267,7 @@ static u32 clear_idx; | |||
266 | #define LOG_ALIGN __alignof__(struct printk_log) | 267 | #define LOG_ALIGN __alignof__(struct printk_log) |
267 | #endif | 268 | #endif |
268 | #define __LOG_BUF_LEN (1 << CONFIG_LOG_BUF_SHIFT) | 269 | #define __LOG_BUF_LEN (1 << CONFIG_LOG_BUF_SHIFT) |
270 | #define __LOG_CPU_MAX_BUF_LEN (1 << CONFIG_LOG_CPU_MAX_BUF_SHIFT) | ||
269 | static char __log_buf[__LOG_BUF_LEN] __aligned(LOG_ALIGN); | 271 | static char __log_buf[__LOG_BUF_LEN] __aligned(LOG_ALIGN); |
270 | static char *log_buf = __log_buf; | 272 | static char *log_buf = __log_buf; |
271 | static u32 log_buf_len = __LOG_BUF_LEN; | 273 | static u32 log_buf_len = __LOG_BUF_LEN; |
@@ -344,7 +346,7 @@ static int log_make_free_space(u32 msg_size) | |||
344 | while (log_first_seq < log_next_seq) { | 346 | while (log_first_seq < log_next_seq) { |
345 | if (logbuf_has_space(msg_size, false)) | 347 | if (logbuf_has_space(msg_size, false)) |
346 | return 0; | 348 | return 0; |
347 | /* drop old messages until we have enough continuous space */ | 349 | /* drop old messages until we have enough contiguous space */ |
348 | log_first_idx = log_next(log_first_idx); | 350 | log_first_idx = log_next(log_first_idx); |
349 | log_first_seq++; | 351 | log_first_seq++; |
350 | } | 352 | } |
@@ -453,11 +455,7 @@ static int log_store(int facility, int level, | |||
453 | return msg->text_len; | 455 | return msg->text_len; |
454 | } | 456 | } |
455 | 457 | ||
456 | #ifdef CONFIG_SECURITY_DMESG_RESTRICT | 458 | int dmesg_restrict = IS_ENABLED(CONFIG_SECURITY_DMESG_RESTRICT); |
457 | int dmesg_restrict = 1; | ||
458 | #else | ||
459 | int dmesg_restrict; | ||
460 | #endif | ||
461 | 459 | ||
462 | static int syslog_action_restricted(int type) | 460 | static int syslog_action_restricted(int type) |
463 | { | 461 | { |
@@ -828,34 +826,74 @@ void log_buf_kexec_setup(void) | |||
828 | /* requested log_buf_len from kernel cmdline */ | 826 | /* requested log_buf_len from kernel cmdline */ |
829 | static unsigned long __initdata new_log_buf_len; | 827 | static unsigned long __initdata new_log_buf_len; |
830 | 828 | ||
831 | /* save requested log_buf_len since it's too early to process it */ | 829 | /* we practice scaling the ring buffer by powers of 2 */ |
832 | static int __init log_buf_len_setup(char *str) | 830 | static void __init log_buf_len_update(unsigned size) |
833 | { | 831 | { |
834 | unsigned size = memparse(str, &str); | ||
835 | |||
836 | if (size) | 832 | if (size) |
837 | size = roundup_pow_of_two(size); | 833 | size = roundup_pow_of_two(size); |
838 | if (size > log_buf_len) | 834 | if (size > log_buf_len) |
839 | new_log_buf_len = size; | 835 | new_log_buf_len = size; |
836 | } | ||
837 | |||
838 | /* save requested log_buf_len since it's too early to process it */ | ||
839 | static int __init log_buf_len_setup(char *str) | ||
840 | { | ||
841 | unsigned size = memparse(str, &str); | ||
842 | |||
843 | log_buf_len_update(size); | ||
840 | 844 | ||
841 | return 0; | 845 | return 0; |
842 | } | 846 | } |
843 | early_param("log_buf_len", log_buf_len_setup); | 847 | early_param("log_buf_len", log_buf_len_setup); |
844 | 848 | ||
849 | static void __init log_buf_add_cpu(void) | ||
850 | { | ||
851 | unsigned int cpu_extra; | ||
852 | |||
853 | /* | ||
854 | * archs should set up cpu_possible_bits properly with | ||
855 | * set_cpu_possible() after setup_arch() but just in | ||
856 | * case lets ensure this is valid. | ||
857 | */ | ||
858 | if (num_possible_cpus() == 1) | ||
859 | return; | ||
860 | |||
861 | cpu_extra = (num_possible_cpus() - 1) * __LOG_CPU_MAX_BUF_LEN; | ||
862 | |||
863 | /* by default this will only continue through for large > 64 CPUs */ | ||
864 | if (cpu_extra <= __LOG_BUF_LEN / 2) | ||
865 | return; | ||
866 | |||
867 | pr_info("log_buf_len individual max cpu contribution: %d bytes\n", | ||
868 | __LOG_CPU_MAX_BUF_LEN); | ||
869 | pr_info("log_buf_len total cpu_extra contributions: %d bytes\n", | ||
870 | cpu_extra); | ||
871 | pr_info("log_buf_len min size: %d bytes\n", __LOG_BUF_LEN); | ||
872 | |||
873 | log_buf_len_update(cpu_extra + __LOG_BUF_LEN); | ||
874 | } | ||
875 | |||
845 | void __init setup_log_buf(int early) | 876 | void __init setup_log_buf(int early) |
846 | { | 877 | { |
847 | unsigned long flags; | 878 | unsigned long flags; |
848 | char *new_log_buf; | 879 | char *new_log_buf; |
849 | int free; | 880 | int free; |
850 | 881 | ||
882 | if (log_buf != __log_buf) | ||
883 | return; | ||
884 | |||
885 | if (!early && !new_log_buf_len) | ||
886 | log_buf_add_cpu(); | ||
887 | |||
851 | if (!new_log_buf_len) | 888 | if (!new_log_buf_len) |
852 | return; | 889 | return; |
853 | 890 | ||
854 | if (early) { | 891 | if (early) { |
855 | new_log_buf = | 892 | new_log_buf = |
856 | memblock_virt_alloc(new_log_buf_len, PAGE_SIZE); | 893 | memblock_virt_alloc(new_log_buf_len, LOG_ALIGN); |
857 | } else { | 894 | } else { |
858 | new_log_buf = memblock_virt_alloc_nopanic(new_log_buf_len, 0); | 895 | new_log_buf = memblock_virt_alloc_nopanic(new_log_buf_len, |
896 | LOG_ALIGN); | ||
859 | } | 897 | } |
860 | 898 | ||
861 | if (unlikely(!new_log_buf)) { | 899 | if (unlikely(!new_log_buf)) { |
@@ -872,7 +910,7 @@ void __init setup_log_buf(int early) | |||
872 | memcpy(log_buf, __log_buf, __LOG_BUF_LEN); | 910 | memcpy(log_buf, __log_buf, __LOG_BUF_LEN); |
873 | raw_spin_unlock_irqrestore(&logbuf_lock, flags); | 911 | raw_spin_unlock_irqrestore(&logbuf_lock, flags); |
874 | 912 | ||
875 | pr_info("log_buf_len: %d\n", log_buf_len); | 913 | pr_info("log_buf_len: %d bytes\n", log_buf_len); |
876 | pr_info("early log buf free: %d(%d%%)\n", | 914 | pr_info("early log buf free: %d(%d%%)\n", |
877 | free, (free * 100) / __LOG_BUF_LEN); | 915 | free, (free * 100) / __LOG_BUF_LEN); |
878 | } | 916 | } |
@@ -881,7 +919,7 @@ static bool __read_mostly ignore_loglevel; | |||
881 | 919 | ||
882 | static int __init ignore_loglevel_setup(char *str) | 920 | static int __init ignore_loglevel_setup(char *str) |
883 | { | 921 | { |
884 | ignore_loglevel = 1; | 922 | ignore_loglevel = true; |
885 | pr_info("debug: ignoring loglevel setting.\n"); | 923 | pr_info("debug: ignoring loglevel setting.\n"); |
886 | 924 | ||
887 | return 0; | 925 | return 0; |
@@ -947,11 +985,7 @@ static inline void boot_delay_msec(int level) | |||
947 | } | 985 | } |
948 | #endif | 986 | #endif |
949 | 987 | ||
950 | #if defined(CONFIG_PRINTK_TIME) | 988 | static bool printk_time = IS_ENABLED(CONFIG_PRINTK_TIME); |
951 | static bool printk_time = 1; | ||
952 | #else | ||
953 | static bool printk_time; | ||
954 | #endif | ||
955 | module_param_named(time, printk_time, bool, S_IRUGO | S_IWUSR); | 989 | module_param_named(time, printk_time, bool, S_IRUGO | S_IWUSR); |
956 | 990 | ||
957 | static size_t print_time(u64 ts, char *buf) | 991 | static size_t print_time(u64 ts, char *buf) |
@@ -1310,7 +1344,7 @@ int do_syslog(int type, char __user *buf, int len, bool from_file) | |||
1310 | * for pending data, not the size; return the count of | 1344 | * for pending data, not the size; return the count of |
1311 | * records, not the length. | 1345 | * records, not the length. |
1312 | */ | 1346 | */ |
1313 | error = log_next_idx - syslog_idx; | 1347 | error = log_next_seq - syslog_seq; |
1314 | } else { | 1348 | } else { |
1315 | u64 seq = syslog_seq; | 1349 | u64 seq = syslog_seq; |
1316 | u32 idx = syslog_idx; | 1350 | u32 idx = syslog_idx; |
@@ -1416,10 +1450,9 @@ static int have_callable_console(void) | |||
1416 | /* | 1450 | /* |
1417 | * Can we actually use the console at this time on this cpu? | 1451 | * Can we actually use the console at this time on this cpu? |
1418 | * | 1452 | * |
1419 | * Console drivers may assume that per-cpu resources have | 1453 | * Console drivers may assume that per-cpu resources have been allocated. So |
1420 | * been allocated. So unless they're explicitly marked as | 1454 | * unless they're explicitly marked as being able to cope (CON_ANYTIME) don't |
1421 | * being able to cope (CON_ANYTIME) don't call them until | 1455 | * call them until this CPU is officially up. |
1422 | * this CPU is officially up. | ||
1423 | */ | 1456 | */ |
1424 | static inline int can_use_console(unsigned int cpu) | 1457 | static inline int can_use_console(unsigned int cpu) |
1425 | { | 1458 | { |
@@ -1432,8 +1465,10 @@ static inline int can_use_console(unsigned int cpu) | |||
1432 | * console_lock held, and 'console_locked' set) if it | 1465 | * console_lock held, and 'console_locked' set) if it |
1433 | * is successful, false otherwise. | 1466 | * is successful, false otherwise. |
1434 | */ | 1467 | */ |
1435 | static int console_trylock_for_printk(unsigned int cpu) | 1468 | static int console_trylock_for_printk(void) |
1436 | { | 1469 | { |
1470 | unsigned int cpu = smp_processor_id(); | ||
1471 | |||
1437 | if (!console_trylock()) | 1472 | if (!console_trylock()) |
1438 | return 0; | 1473 | return 0; |
1439 | /* | 1474 | /* |
@@ -1476,7 +1511,7 @@ static struct cont { | |||
1476 | struct task_struct *owner; /* task of first print*/ | 1511 | struct task_struct *owner; /* task of first print*/ |
1477 | u64 ts_nsec; /* time of first print */ | 1512 | u64 ts_nsec; /* time of first print */ |
1478 | u8 level; /* log level of first message */ | 1513 | u8 level; /* log level of first message */ |
1479 | u8 facility; /* log level of first message */ | 1514 | u8 facility; /* log facility of first message */ |
1480 | enum log_flags flags; /* prefix, newline flags */ | 1515 | enum log_flags flags; /* prefix, newline flags */ |
1481 | bool flushed:1; /* buffer sealed and committed */ | 1516 | bool flushed:1; /* buffer sealed and committed */ |
1482 | } cont; | 1517 | } cont; |
@@ -1608,7 +1643,8 @@ asmlinkage int vprintk_emit(int facility, int level, | |||
1608 | */ | 1643 | */ |
1609 | if (!oops_in_progress && !lockdep_recursing(current)) { | 1644 | if (!oops_in_progress && !lockdep_recursing(current)) { |
1610 | recursion_bug = 1; | 1645 | recursion_bug = 1; |
1611 | goto out_restore_irqs; | 1646 | local_irq_restore(flags); |
1647 | return 0; | ||
1612 | } | 1648 | } |
1613 | zap_locks(); | 1649 | zap_locks(); |
1614 | } | 1650 | } |
@@ -1716,21 +1752,30 @@ asmlinkage int vprintk_emit(int facility, int level, | |||
1716 | 1752 | ||
1717 | logbuf_cpu = UINT_MAX; | 1753 | logbuf_cpu = UINT_MAX; |
1718 | raw_spin_unlock(&logbuf_lock); | 1754 | raw_spin_unlock(&logbuf_lock); |
1755 | lockdep_on(); | ||
1756 | local_irq_restore(flags); | ||
1719 | 1757 | ||
1720 | /* If called from the scheduler, we can not call up(). */ | 1758 | /* If called from the scheduler, we can not call up(). */ |
1721 | if (!in_sched) { | 1759 | if (!in_sched) { |
1760 | lockdep_off(); | ||
1761 | /* | ||
1762 | * Disable preemption to avoid being preempted while holding | ||
1763 | * console_sem which would prevent anyone from printing to | ||
1764 | * console | ||
1765 | */ | ||
1766 | preempt_disable(); | ||
1767 | |||
1722 | /* | 1768 | /* |
1723 | * Try to acquire and then immediately release the console | 1769 | * Try to acquire and then immediately release the console |
1724 | * semaphore. The release will print out buffers and wake up | 1770 | * semaphore. The release will print out buffers and wake up |
1725 | * /dev/kmsg and syslog() users. | 1771 | * /dev/kmsg and syslog() users. |
1726 | */ | 1772 | */ |
1727 | if (console_trylock_for_printk(this_cpu)) | 1773 | if (console_trylock_for_printk()) |
1728 | console_unlock(); | 1774 | console_unlock(); |
1775 | preempt_enable(); | ||
1776 | lockdep_on(); | ||
1729 | } | 1777 | } |
1730 | 1778 | ||
1731 | lockdep_on(); | ||
1732 | out_restore_irqs: | ||
1733 | local_irq_restore(flags); | ||
1734 | return printed_len; | 1779 | return printed_len; |
1735 | } | 1780 | } |
1736 | EXPORT_SYMBOL(vprintk_emit); | 1781 | EXPORT_SYMBOL(vprintk_emit); |
@@ -1802,7 +1847,7 @@ EXPORT_SYMBOL(printk); | |||
1802 | 1847 | ||
1803 | #define LOG_LINE_MAX 0 | 1848 | #define LOG_LINE_MAX 0 |
1804 | #define PREFIX_MAX 0 | 1849 | #define PREFIX_MAX 0 |
1805 | #define LOG_LINE_MAX 0 | 1850 | |
1806 | static u64 syslog_seq; | 1851 | static u64 syslog_seq; |
1807 | static u32 syslog_idx; | 1852 | static u32 syslog_idx; |
1808 | static u64 console_seq; | 1853 | static u64 console_seq; |
@@ -1881,11 +1926,12 @@ static int __add_preferred_console(char *name, int idx, char *options, | |||
1881 | return 0; | 1926 | return 0; |
1882 | } | 1927 | } |
1883 | /* | 1928 | /* |
1884 | * Set up a list of consoles. Called from init/main.c | 1929 | * Set up a console. Called via do_early_param() in init/main.c |
1930 | * for each "console=" parameter in the boot command line. | ||
1885 | */ | 1931 | */ |
1886 | static int __init console_setup(char *str) | 1932 | static int __init console_setup(char *str) |
1887 | { | 1933 | { |
1888 | char buf[sizeof(console_cmdline[0].name) + 4]; /* 4 for index */ | 1934 | char buf[sizeof(console_cmdline[0].name) + 4]; /* 4 for "ttyS" */ |
1889 | char *s, *options, *brl_options = NULL; | 1935 | char *s, *options, *brl_options = NULL; |
1890 | int idx; | 1936 | int idx; |
1891 | 1937 | ||
@@ -1902,7 +1948,8 @@ static int __init console_setup(char *str) | |||
1902 | strncpy(buf, str, sizeof(buf) - 1); | 1948 | strncpy(buf, str, sizeof(buf) - 1); |
1903 | } | 1949 | } |
1904 | buf[sizeof(buf) - 1] = 0; | 1950 | buf[sizeof(buf) - 1] = 0; |
1905 | if ((options = strchr(str, ',')) != NULL) | 1951 | options = strchr(str, ','); |
1952 | if (options) | ||
1906 | *(options++) = 0; | 1953 | *(options++) = 0; |
1907 | #ifdef __sparc__ | 1954 | #ifdef __sparc__ |
1908 | if (!strcmp(str, "ttya")) | 1955 | if (!strcmp(str, "ttya")) |
@@ -1911,7 +1958,7 @@ static int __init console_setup(char *str) | |||
1911 | strcpy(buf, "ttyS1"); | 1958 | strcpy(buf, "ttyS1"); |
1912 | #endif | 1959 | #endif |
1913 | for (s = buf; *s; s++) | 1960 | for (s = buf; *s; s++) |
1914 | if ((*s >= '0' && *s <= '9') || *s == ',') | 1961 | if (isdigit(*s) || *s == ',') |
1915 | break; | 1962 | break; |
1916 | idx = simple_strtoul(s, NULL, 10); | 1963 | idx = simple_strtoul(s, NULL, 10); |
1917 | *s = 0; | 1964 | *s = 0; |
@@ -1950,7 +1997,6 @@ int update_console_cmdline(char *name, int idx, char *name_new, int idx_new, cha | |||
1950 | i++, c++) | 1997 | i++, c++) |
1951 | if (strcmp(c->name, name) == 0 && c->index == idx) { | 1998 | if (strcmp(c->name, name) == 0 && c->index == idx) { |
1952 | strlcpy(c->name, name_new, sizeof(c->name)); | 1999 | strlcpy(c->name, name_new, sizeof(c->name)); |
1953 | c->name[sizeof(c->name) - 1] = 0; | ||
1954 | c->options = options; | 2000 | c->options = options; |
1955 | c->index = idx_new; | 2001 | c->index = idx_new; |
1956 | return i; | 2002 | return i; |
@@ -1959,12 +2005,12 @@ int update_console_cmdline(char *name, int idx, char *name_new, int idx_new, cha | |||
1959 | return -1; | 2005 | return -1; |
1960 | } | 2006 | } |
1961 | 2007 | ||
1962 | bool console_suspend_enabled = 1; | 2008 | bool console_suspend_enabled = true; |
1963 | EXPORT_SYMBOL(console_suspend_enabled); | 2009 | EXPORT_SYMBOL(console_suspend_enabled); |
1964 | 2010 | ||
1965 | static int __init console_suspend_disable(char *str) | 2011 | static int __init console_suspend_disable(char *str) |
1966 | { | 2012 | { |
1967 | console_suspend_enabled = 0; | 2013 | console_suspend_enabled = false; |
1968 | return 1; | 2014 | return 1; |
1969 | } | 2015 | } |
1970 | __setup("no_console_suspend", console_suspend_disable); | 2016 | __setup("no_console_suspend", console_suspend_disable); |
@@ -2045,8 +2091,8 @@ EXPORT_SYMBOL(console_lock); | |||
2045 | /** | 2091 | /** |
2046 | * console_trylock - try to lock the console system for exclusive use. | 2092 | * console_trylock - try to lock the console system for exclusive use. |
2047 | * | 2093 | * |
2048 | * Tried to acquire a lock which guarantees that the caller has | 2094 | * Try to acquire a lock which guarantees that the caller has exclusive |
2049 | * exclusive access to the console system and the console_drivers list. | 2095 | * access to the console system and the console_drivers list. |
2050 | * | 2096 | * |
2051 | * returns 1 on success, and 0 on failure to acquire the lock. | 2097 | * returns 1 on success, and 0 on failure to acquire the lock. |
2052 | */ | 2098 | */ |
@@ -2618,14 +2664,13 @@ EXPORT_SYMBOL(__printk_ratelimit); | |||
2618 | bool printk_timed_ratelimit(unsigned long *caller_jiffies, | 2664 | bool printk_timed_ratelimit(unsigned long *caller_jiffies, |
2619 | unsigned int interval_msecs) | 2665 | unsigned int interval_msecs) |
2620 | { | 2666 | { |
2621 | if (*caller_jiffies == 0 | 2667 | unsigned long elapsed = jiffies - *caller_jiffies; |
2622 | || !time_in_range(jiffies, *caller_jiffies, | 2668 | |
2623 | *caller_jiffies | 2669 | if (*caller_jiffies && elapsed <= msecs_to_jiffies(interval_msecs)) |
2624 | + msecs_to_jiffies(interval_msecs))) { | 2670 | return false; |
2625 | *caller_jiffies = jiffies; | 2671 | |
2626 | return true; | 2672 | *caller_jiffies = jiffies; |
2627 | } | 2673 | return true; |
2628 | return false; | ||
2629 | } | 2674 | } |
2630 | EXPORT_SYMBOL(printk_timed_ratelimit); | 2675 | EXPORT_SYMBOL(printk_timed_ratelimit); |
2631 | 2676 | ||
diff --git a/kernel/smp.c b/kernel/smp.c index 487653b5844f..aff8aa14f547 100644 --- a/kernel/smp.c +++ b/kernel/smp.c | |||
@@ -670,7 +670,7 @@ void on_each_cpu_cond(bool (*cond_func)(int cpu, void *info), | |||
670 | if (cond_func(cpu, info)) { | 670 | if (cond_func(cpu, info)) { |
671 | ret = smp_call_function_single(cpu, func, | 671 | ret = smp_call_function_single(cpu, func, |
672 | info, wait); | 672 | info, wait); |
673 | WARN_ON_ONCE(!ret); | 673 | WARN_ON_ONCE(ret); |
674 | } | 674 | } |
675 | preempt_enable(); | 675 | preempt_enable(); |
676 | } | 676 | } |
diff --git a/kernel/sysctl.c b/kernel/sysctl.c index 75b22e22a72c..75875a741b5e 100644 --- a/kernel/sysctl.c +++ b/kernel/sysctl.c | |||
@@ -1240,8 +1240,7 @@ static struct ctl_table vm_table[] = { | |||
1240 | .maxlen = sizeof(unsigned long), | 1240 | .maxlen = sizeof(unsigned long), |
1241 | .mode = 0644, | 1241 | .mode = 0644, |
1242 | .proc_handler = hugetlb_sysctl_handler, | 1242 | .proc_handler = hugetlb_sysctl_handler, |
1243 | .extra1 = (void *)&hugetlb_zero, | 1243 | .extra1 = &zero, |
1244 | .extra2 = (void *)&hugetlb_infinity, | ||
1245 | }, | 1244 | }, |
1246 | #ifdef CONFIG_NUMA | 1245 | #ifdef CONFIG_NUMA |
1247 | { | 1246 | { |
@@ -1250,8 +1249,7 @@ static struct ctl_table vm_table[] = { | |||
1250 | .maxlen = sizeof(unsigned long), | 1249 | .maxlen = sizeof(unsigned long), |
1251 | .mode = 0644, | 1250 | .mode = 0644, |
1252 | .proc_handler = &hugetlb_mempolicy_sysctl_handler, | 1251 | .proc_handler = &hugetlb_mempolicy_sysctl_handler, |
1253 | .extra1 = (void *)&hugetlb_zero, | 1252 | .extra1 = &zero, |
1254 | .extra2 = (void *)&hugetlb_infinity, | ||
1255 | }, | 1253 | }, |
1256 | #endif | 1254 | #endif |
1257 | { | 1255 | { |
@@ -1274,8 +1272,7 @@ static struct ctl_table vm_table[] = { | |||
1274 | .maxlen = sizeof(unsigned long), | 1272 | .maxlen = sizeof(unsigned long), |
1275 | .mode = 0644, | 1273 | .mode = 0644, |
1276 | .proc_handler = hugetlb_overcommit_handler, | 1274 | .proc_handler = hugetlb_overcommit_handler, |
1277 | .extra1 = (void *)&hugetlb_zero, | 1275 | .extra1 = &zero, |
1278 | .extra2 = (void *)&hugetlb_infinity, | ||
1279 | }, | 1276 | }, |
1280 | #endif | 1277 | #endif |
1281 | { | 1278 | { |
diff --git a/kernel/watchdog.c b/kernel/watchdog.c index c3319bd1b040..51b29e9d2ba6 100644 --- a/kernel/watchdog.c +++ b/kernel/watchdog.c | |||
@@ -260,9 +260,11 @@ static void watchdog_overflow_callback(struct perf_event *event, | |||
260 | return; | 260 | return; |
261 | 261 | ||
262 | if (hardlockup_panic) | 262 | if (hardlockup_panic) |
263 | panic("Watchdog detected hard LOCKUP on cpu %d", this_cpu); | 263 | panic("Watchdog detected hard LOCKUP on cpu %d", |
264 | this_cpu); | ||
264 | else | 265 | else |
265 | WARN(1, "Watchdog detected hard LOCKUP on cpu %d", this_cpu); | 266 | WARN(1, "Watchdog detected hard LOCKUP on cpu %d", |
267 | this_cpu); | ||
266 | 268 | ||
267 | __this_cpu_write(hard_watchdog_warn, true); | 269 | __this_cpu_write(hard_watchdog_warn, true); |
268 | return; | 270 | return; |
@@ -345,7 +347,7 @@ static enum hrtimer_restart watchdog_timer_fn(struct hrtimer *hrtimer) | |||
345 | } | 347 | } |
346 | } | 348 | } |
347 | 349 | ||
348 | printk(KERN_EMERG "BUG: soft lockup - CPU#%d stuck for %us! [%s:%d]\n", | 350 | pr_emerg("BUG: soft lockup - CPU#%d stuck for %us! [%s:%d]\n", |
349 | smp_processor_id(), duration, | 351 | smp_processor_id(), duration, |
350 | current->comm, task_pid_nr(current)); | 352 | current->comm, task_pid_nr(current)); |
351 | print_modules(); | 353 | print_modules(); |
@@ -484,7 +486,7 @@ static int watchdog_nmi_enable(unsigned int cpu) | |||
484 | if (PTR_ERR(event) == -EOPNOTSUPP) | 486 | if (PTR_ERR(event) == -EOPNOTSUPP) |
485 | pr_info("disabled (cpu%i): not supported (no LAPIC?)\n", cpu); | 487 | pr_info("disabled (cpu%i): not supported (no LAPIC?)\n", cpu); |
486 | else if (PTR_ERR(event) == -ENOENT) | 488 | else if (PTR_ERR(event) == -ENOENT) |
487 | pr_warning("disabled (cpu%i): hardware events not enabled\n", | 489 | pr_warn("disabled (cpu%i): hardware events not enabled\n", |
488 | cpu); | 490 | cpu); |
489 | else | 491 | else |
490 | pr_err("disabled (cpu%i): unable to create perf event: %ld\n", | 492 | pr_err("disabled (cpu%i): unable to create perf event: %ld\n", |
diff --git a/lib/Kconfig b/lib/Kconfig index a8a775730c09..df872659ddd3 100644 --- a/lib/Kconfig +++ b/lib/Kconfig | |||
@@ -396,6 +396,39 @@ config CPU_RMAP | |||
396 | config DQL | 396 | config DQL |
397 | bool | 397 | bool |
398 | 398 | ||
399 | config GLOB | ||
400 | bool | ||
401 | # This actually supports modular compilation, but the module overhead | ||
402 | # is ridiculous for the amount of code involved. Until an out-of-tree | ||
403 | # driver asks for it, we'll just link it directly it into the kernel | ||
404 | # when required. Since we're ignoring out-of-tree users, there's also | ||
405 | # no need bother prompting for a manual decision: | ||
406 | # prompt "glob_match() function" | ||
407 | help | ||
408 | This option provides a glob_match function for performing | ||
409 | simple text pattern matching. It originated in the ATA code | ||
410 | to blacklist particular drive models, but other device drivers | ||
411 | may need similar functionality. | ||
412 | |||
413 | All drivers in the Linux kernel tree that require this function | ||
414 | should automatically select this option. Say N unless you | ||
415 | are compiling an out-of tree driver which tells you that it | ||
416 | depends on this. | ||
417 | |||
418 | config GLOB_SELFTEST | ||
419 | bool "glob self-test on init" | ||
420 | default n | ||
421 | depends on GLOB | ||
422 | help | ||
423 | This option enables a simple self-test of the glob_match | ||
424 | function on startup. It is primarily useful for people | ||
425 | working on the code to ensure they haven't introduced any | ||
426 | regressions. | ||
427 | |||
428 | It only adds a little bit of code and slows kernel boot (or | ||
429 | module load) by a small amount, so you're welcome to play with | ||
430 | it, but you probably don't need it. | ||
431 | |||
399 | # | 432 | # |
400 | # Netlink attribute parsing support is select'ed if needed | 433 | # Netlink attribute parsing support is select'ed if needed |
401 | # | 434 | # |
diff --git a/lib/Kconfig.debug b/lib/Kconfig.debug index cfe7df8f62cc..cb45f59685e6 100644 --- a/lib/Kconfig.debug +++ b/lib/Kconfig.debug | |||
@@ -15,7 +15,7 @@ config PRINTK_TIME | |||
15 | The behavior is also controlled by the kernel command line | 15 | The behavior is also controlled by the kernel command line |
16 | parameter printk.time=1. See Documentation/kernel-parameters.txt | 16 | parameter printk.time=1. See Documentation/kernel-parameters.txt |
17 | 17 | ||
18 | config DEFAULT_MESSAGE_LOGLEVEL | 18 | config MESSAGE_LOGLEVEL_DEFAULT |
19 | int "Default message log level (1-7)" | 19 | int "Default message log level (1-7)" |
20 | range 1 7 | 20 | range 1 7 |
21 | default "4" | 21 | default "4" |
diff --git a/lib/Makefile b/lib/Makefile index 8427df95dade..d6b4bc496408 100644 --- a/lib/Makefile +++ b/lib/Makefile | |||
@@ -137,6 +137,8 @@ obj-$(CONFIG_CORDIC) += cordic.o | |||
137 | 137 | ||
138 | obj-$(CONFIG_DQL) += dynamic_queue_limits.o | 138 | obj-$(CONFIG_DQL) += dynamic_queue_limits.o |
139 | 139 | ||
140 | obj-$(CONFIG_GLOB) += glob.o | ||
141 | |||
140 | obj-$(CONFIG_MPILIB) += mpi/ | 142 | obj-$(CONFIG_MPILIB) += mpi/ |
141 | obj-$(CONFIG_SIGNATURE) += digsig.o | 143 | obj-$(CONFIG_SIGNATURE) += digsig.o |
142 | 144 | ||
diff --git a/lib/bitmap.c b/lib/bitmap.c index 06f7e4fe8d2d..1e031f2c9aba 100644 --- a/lib/bitmap.c +++ b/lib/bitmap.c | |||
@@ -40,9 +40,9 @@ | |||
40 | * for the best explanations of this ordering. | 40 | * for the best explanations of this ordering. |
41 | */ | 41 | */ |
42 | 42 | ||
43 | int __bitmap_empty(const unsigned long *bitmap, int bits) | 43 | int __bitmap_empty(const unsigned long *bitmap, unsigned int bits) |
44 | { | 44 | { |
45 | int k, lim = bits/BITS_PER_LONG; | 45 | unsigned int k, lim = bits/BITS_PER_LONG; |
46 | for (k = 0; k < lim; ++k) | 46 | for (k = 0; k < lim; ++k) |
47 | if (bitmap[k]) | 47 | if (bitmap[k]) |
48 | return 0; | 48 | return 0; |
@@ -55,9 +55,9 @@ int __bitmap_empty(const unsigned long *bitmap, int bits) | |||
55 | } | 55 | } |
56 | EXPORT_SYMBOL(__bitmap_empty); | 56 | EXPORT_SYMBOL(__bitmap_empty); |
57 | 57 | ||
58 | int __bitmap_full(const unsigned long *bitmap, int bits) | 58 | int __bitmap_full(const unsigned long *bitmap, unsigned int bits) |
59 | { | 59 | { |
60 | int k, lim = bits/BITS_PER_LONG; | 60 | unsigned int k, lim = bits/BITS_PER_LONG; |
61 | for (k = 0; k < lim; ++k) | 61 | for (k = 0; k < lim; ++k) |
62 | if (~bitmap[k]) | 62 | if (~bitmap[k]) |
63 | return 0; | 63 | return 0; |
@@ -71,9 +71,9 @@ int __bitmap_full(const unsigned long *bitmap, int bits) | |||
71 | EXPORT_SYMBOL(__bitmap_full); | 71 | EXPORT_SYMBOL(__bitmap_full); |
72 | 72 | ||
73 | int __bitmap_equal(const unsigned long *bitmap1, | 73 | int __bitmap_equal(const unsigned long *bitmap1, |
74 | const unsigned long *bitmap2, int bits) | 74 | const unsigned long *bitmap2, unsigned int bits) |
75 | { | 75 | { |
76 | int k, lim = bits/BITS_PER_LONG; | 76 | unsigned int k, lim = bits/BITS_PER_LONG; |
77 | for (k = 0; k < lim; ++k) | 77 | for (k = 0; k < lim; ++k) |
78 | if (bitmap1[k] != bitmap2[k]) | 78 | if (bitmap1[k] != bitmap2[k]) |
79 | return 0; | 79 | return 0; |
@@ -86,14 +86,14 @@ int __bitmap_equal(const unsigned long *bitmap1, | |||
86 | } | 86 | } |
87 | EXPORT_SYMBOL(__bitmap_equal); | 87 | EXPORT_SYMBOL(__bitmap_equal); |
88 | 88 | ||
89 | void __bitmap_complement(unsigned long *dst, const unsigned long *src, int bits) | 89 | void __bitmap_complement(unsigned long *dst, const unsigned long *src, unsigned int bits) |
90 | { | 90 | { |
91 | int k, lim = bits/BITS_PER_LONG; | 91 | unsigned int k, lim = bits/BITS_PER_LONG; |
92 | for (k = 0; k < lim; ++k) | 92 | for (k = 0; k < lim; ++k) |
93 | dst[k] = ~src[k]; | 93 | dst[k] = ~src[k]; |
94 | 94 | ||
95 | if (bits % BITS_PER_LONG) | 95 | if (bits % BITS_PER_LONG) |
96 | dst[k] = ~src[k] & BITMAP_LAST_WORD_MASK(bits); | 96 | dst[k] = ~src[k]; |
97 | } | 97 | } |
98 | EXPORT_SYMBOL(__bitmap_complement); | 98 | EXPORT_SYMBOL(__bitmap_complement); |
99 | 99 | ||
@@ -182,23 +182,26 @@ void __bitmap_shift_left(unsigned long *dst, | |||
182 | EXPORT_SYMBOL(__bitmap_shift_left); | 182 | EXPORT_SYMBOL(__bitmap_shift_left); |
183 | 183 | ||
184 | int __bitmap_and(unsigned long *dst, const unsigned long *bitmap1, | 184 | int __bitmap_and(unsigned long *dst, const unsigned long *bitmap1, |
185 | const unsigned long *bitmap2, int bits) | 185 | const unsigned long *bitmap2, unsigned int bits) |
186 | { | 186 | { |
187 | int k; | 187 | unsigned int k; |
188 | int nr = BITS_TO_LONGS(bits); | 188 | unsigned int lim = bits/BITS_PER_LONG; |
189 | unsigned long result = 0; | 189 | unsigned long result = 0; |
190 | 190 | ||
191 | for (k = 0; k < nr; k++) | 191 | for (k = 0; k < lim; k++) |
192 | result |= (dst[k] = bitmap1[k] & bitmap2[k]); | 192 | result |= (dst[k] = bitmap1[k] & bitmap2[k]); |
193 | if (bits % BITS_PER_LONG) | ||
194 | result |= (dst[k] = bitmap1[k] & bitmap2[k] & | ||
195 | BITMAP_LAST_WORD_MASK(bits)); | ||
193 | return result != 0; | 196 | return result != 0; |
194 | } | 197 | } |
195 | EXPORT_SYMBOL(__bitmap_and); | 198 | EXPORT_SYMBOL(__bitmap_and); |
196 | 199 | ||
197 | void __bitmap_or(unsigned long *dst, const unsigned long *bitmap1, | 200 | void __bitmap_or(unsigned long *dst, const unsigned long *bitmap1, |
198 | const unsigned long *bitmap2, int bits) | 201 | const unsigned long *bitmap2, unsigned int bits) |
199 | { | 202 | { |
200 | int k; | 203 | unsigned int k; |
201 | int nr = BITS_TO_LONGS(bits); | 204 | unsigned int nr = BITS_TO_LONGS(bits); |
202 | 205 | ||
203 | for (k = 0; k < nr; k++) | 206 | for (k = 0; k < nr; k++) |
204 | dst[k] = bitmap1[k] | bitmap2[k]; | 207 | dst[k] = bitmap1[k] | bitmap2[k]; |
@@ -206,10 +209,10 @@ void __bitmap_or(unsigned long *dst, const unsigned long *bitmap1, | |||
206 | EXPORT_SYMBOL(__bitmap_or); | 209 | EXPORT_SYMBOL(__bitmap_or); |
207 | 210 | ||
208 | void __bitmap_xor(unsigned long *dst, const unsigned long *bitmap1, | 211 | void __bitmap_xor(unsigned long *dst, const unsigned long *bitmap1, |
209 | const unsigned long *bitmap2, int bits) | 212 | const unsigned long *bitmap2, unsigned int bits) |
210 | { | 213 | { |
211 | int k; | 214 | unsigned int k; |
212 | int nr = BITS_TO_LONGS(bits); | 215 | unsigned int nr = BITS_TO_LONGS(bits); |
213 | 216 | ||
214 | for (k = 0; k < nr; k++) | 217 | for (k = 0; k < nr; k++) |
215 | dst[k] = bitmap1[k] ^ bitmap2[k]; | 218 | dst[k] = bitmap1[k] ^ bitmap2[k]; |
@@ -217,22 +220,25 @@ void __bitmap_xor(unsigned long *dst, const unsigned long *bitmap1, | |||
217 | EXPORT_SYMBOL(__bitmap_xor); | 220 | EXPORT_SYMBOL(__bitmap_xor); |
218 | 221 | ||
219 | int __bitmap_andnot(unsigned long *dst, const unsigned long *bitmap1, | 222 | int __bitmap_andnot(unsigned long *dst, const unsigned long *bitmap1, |
220 | const unsigned long *bitmap2, int bits) | 223 | const unsigned long *bitmap2, unsigned int bits) |
221 | { | 224 | { |
222 | int k; | 225 | unsigned int k; |
223 | int nr = BITS_TO_LONGS(bits); | 226 | unsigned int lim = bits/BITS_PER_LONG; |
224 | unsigned long result = 0; | 227 | unsigned long result = 0; |
225 | 228 | ||
226 | for (k = 0; k < nr; k++) | 229 | for (k = 0; k < lim; k++) |
227 | result |= (dst[k] = bitmap1[k] & ~bitmap2[k]); | 230 | result |= (dst[k] = bitmap1[k] & ~bitmap2[k]); |
231 | if (bits % BITS_PER_LONG) | ||
232 | result |= (dst[k] = bitmap1[k] & ~bitmap2[k] & | ||
233 | BITMAP_LAST_WORD_MASK(bits)); | ||
228 | return result != 0; | 234 | return result != 0; |
229 | } | 235 | } |
230 | EXPORT_SYMBOL(__bitmap_andnot); | 236 | EXPORT_SYMBOL(__bitmap_andnot); |
231 | 237 | ||
232 | int __bitmap_intersects(const unsigned long *bitmap1, | 238 | int __bitmap_intersects(const unsigned long *bitmap1, |
233 | const unsigned long *bitmap2, int bits) | 239 | const unsigned long *bitmap2, unsigned int bits) |
234 | { | 240 | { |
235 | int k, lim = bits/BITS_PER_LONG; | 241 | unsigned int k, lim = bits/BITS_PER_LONG; |
236 | for (k = 0; k < lim; ++k) | 242 | for (k = 0; k < lim; ++k) |
237 | if (bitmap1[k] & bitmap2[k]) | 243 | if (bitmap1[k] & bitmap2[k]) |
238 | return 1; | 244 | return 1; |
@@ -245,9 +251,9 @@ int __bitmap_intersects(const unsigned long *bitmap1, | |||
245 | EXPORT_SYMBOL(__bitmap_intersects); | 251 | EXPORT_SYMBOL(__bitmap_intersects); |
246 | 252 | ||
247 | int __bitmap_subset(const unsigned long *bitmap1, | 253 | int __bitmap_subset(const unsigned long *bitmap1, |
248 | const unsigned long *bitmap2, int bits) | 254 | const unsigned long *bitmap2, unsigned int bits) |
249 | { | 255 | { |
250 | int k, lim = bits/BITS_PER_LONG; | 256 | unsigned int k, lim = bits/BITS_PER_LONG; |
251 | for (k = 0; k < lim; ++k) | 257 | for (k = 0; k < lim; ++k) |
252 | if (bitmap1[k] & ~bitmap2[k]) | 258 | if (bitmap1[k] & ~bitmap2[k]) |
253 | return 0; | 259 | return 0; |
@@ -259,9 +265,10 @@ int __bitmap_subset(const unsigned long *bitmap1, | |||
259 | } | 265 | } |
260 | EXPORT_SYMBOL(__bitmap_subset); | 266 | EXPORT_SYMBOL(__bitmap_subset); |
261 | 267 | ||
262 | int __bitmap_weight(const unsigned long *bitmap, int bits) | 268 | int __bitmap_weight(const unsigned long *bitmap, unsigned int bits) |
263 | { | 269 | { |
264 | int k, w = 0, lim = bits/BITS_PER_LONG; | 270 | unsigned int k, lim = bits/BITS_PER_LONG; |
271 | int w = 0; | ||
265 | 272 | ||
266 | for (k = 0; k < lim; k++) | 273 | for (k = 0; k < lim; k++) |
267 | w += hweight_long(bitmap[k]); | 274 | w += hweight_long(bitmap[k]); |
@@ -273,42 +280,42 @@ int __bitmap_weight(const unsigned long *bitmap, int bits) | |||
273 | } | 280 | } |
274 | EXPORT_SYMBOL(__bitmap_weight); | 281 | EXPORT_SYMBOL(__bitmap_weight); |
275 | 282 | ||
276 | void bitmap_set(unsigned long *map, int start, int nr) | 283 | void bitmap_set(unsigned long *map, unsigned int start, int len) |
277 | { | 284 | { |
278 | unsigned long *p = map + BIT_WORD(start); | 285 | unsigned long *p = map + BIT_WORD(start); |
279 | const int size = start + nr; | 286 | const unsigned int size = start + len; |
280 | int bits_to_set = BITS_PER_LONG - (start % BITS_PER_LONG); | 287 | int bits_to_set = BITS_PER_LONG - (start % BITS_PER_LONG); |
281 | unsigned long mask_to_set = BITMAP_FIRST_WORD_MASK(start); | 288 | unsigned long mask_to_set = BITMAP_FIRST_WORD_MASK(start); |
282 | 289 | ||
283 | while (nr - bits_to_set >= 0) { | 290 | while (len - bits_to_set >= 0) { |
284 | *p |= mask_to_set; | 291 | *p |= mask_to_set; |
285 | nr -= bits_to_set; | 292 | len -= bits_to_set; |
286 | bits_to_set = BITS_PER_LONG; | 293 | bits_to_set = BITS_PER_LONG; |
287 | mask_to_set = ~0UL; | 294 | mask_to_set = ~0UL; |
288 | p++; | 295 | p++; |
289 | } | 296 | } |
290 | if (nr) { | 297 | if (len) { |
291 | mask_to_set &= BITMAP_LAST_WORD_MASK(size); | 298 | mask_to_set &= BITMAP_LAST_WORD_MASK(size); |
292 | *p |= mask_to_set; | 299 | *p |= mask_to_set; |
293 | } | 300 | } |
294 | } | 301 | } |
295 | EXPORT_SYMBOL(bitmap_set); | 302 | EXPORT_SYMBOL(bitmap_set); |
296 | 303 | ||
297 | void bitmap_clear(unsigned long *map, int start, int nr) | 304 | void bitmap_clear(unsigned long *map, unsigned int start, int len) |
298 | { | 305 | { |
299 | unsigned long *p = map + BIT_WORD(start); | 306 | unsigned long *p = map + BIT_WORD(start); |
300 | const int size = start + nr; | 307 | const unsigned int size = start + len; |
301 | int bits_to_clear = BITS_PER_LONG - (start % BITS_PER_LONG); | 308 | int bits_to_clear = BITS_PER_LONG - (start % BITS_PER_LONG); |
302 | unsigned long mask_to_clear = BITMAP_FIRST_WORD_MASK(start); | 309 | unsigned long mask_to_clear = BITMAP_FIRST_WORD_MASK(start); |
303 | 310 | ||
304 | while (nr - bits_to_clear >= 0) { | 311 | while (len - bits_to_clear >= 0) { |
305 | *p &= ~mask_to_clear; | 312 | *p &= ~mask_to_clear; |
306 | nr -= bits_to_clear; | 313 | len -= bits_to_clear; |
307 | bits_to_clear = BITS_PER_LONG; | 314 | bits_to_clear = BITS_PER_LONG; |
308 | mask_to_clear = ~0UL; | 315 | mask_to_clear = ~0UL; |
309 | p++; | 316 | p++; |
310 | } | 317 | } |
311 | if (nr) { | 318 | if (len) { |
312 | mask_to_clear &= BITMAP_LAST_WORD_MASK(size); | 319 | mask_to_clear &= BITMAP_LAST_WORD_MASK(size); |
313 | *p &= ~mask_to_clear; | 320 | *p &= ~mask_to_clear; |
314 | } | 321 | } |
@@ -664,13 +671,8 @@ static int __bitmap_parselist(const char *buf, unsigned int buflen, | |||
664 | 671 | ||
665 | int bitmap_parselist(const char *bp, unsigned long *maskp, int nmaskbits) | 672 | int bitmap_parselist(const char *bp, unsigned long *maskp, int nmaskbits) |
666 | { | 673 | { |
667 | char *nl = strchr(bp, '\n'); | 674 | char *nl = strchrnul(bp, '\n'); |
668 | int len; | 675 | int len = nl - bp; |
669 | |||
670 | if (nl) | ||
671 | len = nl - bp; | ||
672 | else | ||
673 | len = strlen(bp); | ||
674 | 676 | ||
675 | return __bitmap_parselist(bp, len, 0, maskp, nmaskbits); | 677 | return __bitmap_parselist(bp, len, 0, maskp, nmaskbits); |
676 | } | 678 | } |
@@ -716,7 +718,7 @@ EXPORT_SYMBOL(bitmap_parselist_user); | |||
716 | * | 718 | * |
717 | * If for example, just bits 4 through 7 are set in @buf, then @pos | 719 | * If for example, just bits 4 through 7 are set in @buf, then @pos |
718 | * values 4 through 7 will get mapped to 0 through 3, respectively, | 720 | * values 4 through 7 will get mapped to 0 through 3, respectively, |
719 | * and other @pos values will get mapped to 0. When @pos value 7 | 721 | * and other @pos values will get mapped to -1. When @pos value 7 |
720 | * gets mapped to (returns) @ord value 3 in this example, that means | 722 | * gets mapped to (returns) @ord value 3 in this example, that means |
721 | * that bit 7 is the 3rd (starting with 0th) set bit in @buf. | 723 | * that bit 7 is the 3rd (starting with 0th) set bit in @buf. |
722 | * | 724 | * |
@@ -1046,7 +1048,7 @@ enum { | |||
1046 | REG_OP_RELEASE, /* clear all bits in region */ | 1048 | REG_OP_RELEASE, /* clear all bits in region */ |
1047 | }; | 1049 | }; |
1048 | 1050 | ||
1049 | static int __reg_op(unsigned long *bitmap, int pos, int order, int reg_op) | 1051 | static int __reg_op(unsigned long *bitmap, unsigned int pos, int order, int reg_op) |
1050 | { | 1052 | { |
1051 | int nbits_reg; /* number of bits in region */ | 1053 | int nbits_reg; /* number of bits in region */ |
1052 | int index; /* index first long of region in bitmap */ | 1054 | int index; /* index first long of region in bitmap */ |
@@ -1112,11 +1114,11 @@ done: | |||
1112 | * Return the bit offset in bitmap of the allocated region, | 1114 | * Return the bit offset in bitmap of the allocated region, |
1113 | * or -errno on failure. | 1115 | * or -errno on failure. |
1114 | */ | 1116 | */ |
1115 | int bitmap_find_free_region(unsigned long *bitmap, int bits, int order) | 1117 | int bitmap_find_free_region(unsigned long *bitmap, unsigned int bits, int order) |
1116 | { | 1118 | { |
1117 | int pos, end; /* scans bitmap by regions of size order */ | 1119 | unsigned int pos, end; /* scans bitmap by regions of size order */ |
1118 | 1120 | ||
1119 | for (pos = 0 ; (end = pos + (1 << order)) <= bits; pos = end) { | 1121 | for (pos = 0 ; (end = pos + (1U << order)) <= bits; pos = end) { |
1120 | if (!__reg_op(bitmap, pos, order, REG_OP_ISFREE)) | 1122 | if (!__reg_op(bitmap, pos, order, REG_OP_ISFREE)) |
1121 | continue; | 1123 | continue; |
1122 | __reg_op(bitmap, pos, order, REG_OP_ALLOC); | 1124 | __reg_op(bitmap, pos, order, REG_OP_ALLOC); |
@@ -1137,7 +1139,7 @@ EXPORT_SYMBOL(bitmap_find_free_region); | |||
1137 | * | 1139 | * |
1138 | * No return value. | 1140 | * No return value. |
1139 | */ | 1141 | */ |
1140 | void bitmap_release_region(unsigned long *bitmap, int pos, int order) | 1142 | void bitmap_release_region(unsigned long *bitmap, unsigned int pos, int order) |
1141 | { | 1143 | { |
1142 | __reg_op(bitmap, pos, order, REG_OP_RELEASE); | 1144 | __reg_op(bitmap, pos, order, REG_OP_RELEASE); |
1143 | } | 1145 | } |
@@ -1154,12 +1156,11 @@ EXPORT_SYMBOL(bitmap_release_region); | |||
1154 | * Return 0 on success, or %-EBUSY if specified region wasn't | 1156 | * Return 0 on success, or %-EBUSY if specified region wasn't |
1155 | * free (not all bits were zero). | 1157 | * free (not all bits were zero). |
1156 | */ | 1158 | */ |
1157 | int bitmap_allocate_region(unsigned long *bitmap, int pos, int order) | 1159 | int bitmap_allocate_region(unsigned long *bitmap, unsigned int pos, int order) |
1158 | { | 1160 | { |
1159 | if (!__reg_op(bitmap, pos, order, REG_OP_ISFREE)) | 1161 | if (!__reg_op(bitmap, pos, order, REG_OP_ISFREE)) |
1160 | return -EBUSY; | 1162 | return -EBUSY; |
1161 | __reg_op(bitmap, pos, order, REG_OP_ALLOC); | 1163 | return __reg_op(bitmap, pos, order, REG_OP_ALLOC); |
1162 | return 0; | ||
1163 | } | 1164 | } |
1164 | EXPORT_SYMBOL(bitmap_allocate_region); | 1165 | EXPORT_SYMBOL(bitmap_allocate_region); |
1165 | 1166 | ||
diff --git a/lib/cmdline.c b/lib/cmdline.c index d4932f745e92..76a712e6e20e 100644 --- a/lib/cmdline.c +++ b/lib/cmdline.c | |||
@@ -121,11 +121,7 @@ EXPORT_SYMBOL(get_options); | |||
121 | * @retptr: (output) Optional pointer to next char after parse completes | 121 | * @retptr: (output) Optional pointer to next char after parse completes |
122 | * | 122 | * |
123 | * Parses a string into a number. The number stored at @ptr is | 123 | * Parses a string into a number. The number stored at @ptr is |
124 | * potentially suffixed with %K (for kilobytes, or 1024 bytes), | 124 | * potentially suffixed with K, M, G, T, P, E. |
125 | * %M (for megabytes, or 1048576 bytes), or %G (for gigabytes, or | ||
126 | * 1073741824). If the number is suffixed with K, M, or G, then | ||
127 | * the return value is the number multiplied by one kilobyte, one | ||
128 | * megabyte, or one gigabyte, respectively. | ||
129 | */ | 125 | */ |
130 | 126 | ||
131 | unsigned long long memparse(const char *ptr, char **retptr) | 127 | unsigned long long memparse(const char *ptr, char **retptr) |
@@ -135,6 +131,15 @@ unsigned long long memparse(const char *ptr, char **retptr) | |||
135 | unsigned long long ret = simple_strtoull(ptr, &endptr, 0); | 131 | unsigned long long ret = simple_strtoull(ptr, &endptr, 0); |
136 | 132 | ||
137 | switch (*endptr) { | 133 | switch (*endptr) { |
134 | case 'E': | ||
135 | case 'e': | ||
136 | ret <<= 10; | ||
137 | case 'P': | ||
138 | case 'p': | ||
139 | ret <<= 10; | ||
140 | case 'T': | ||
141 | case 't': | ||
142 | ret <<= 10; | ||
138 | case 'G': | 143 | case 'G': |
139 | case 'g': | 144 | case 'g': |
140 | ret <<= 10; | 145 | ret <<= 10; |
diff --git a/lib/glob.c b/lib/glob.c new file mode 100644 index 000000000000..500fc80d23e1 --- /dev/null +++ b/lib/glob.c | |||
@@ -0,0 +1,287 @@ | |||
1 | #include <linux/module.h> | ||
2 | #include <linux/glob.h> | ||
3 | |||
4 | /* | ||
5 | * The only reason this code can be compiled as a module is because the | ||
6 | * ATA code that depends on it can be as well. In practice, they're | ||
7 | * both usually compiled in and the module overhead goes away. | ||
8 | */ | ||
9 | MODULE_DESCRIPTION("glob(7) matching"); | ||
10 | MODULE_LICENSE("Dual MIT/GPL"); | ||
11 | |||
12 | /** | ||
13 | * glob_match - Shell-style pattern matching, like !fnmatch(pat, str, 0) | ||
14 | * @pat: Shell-style pattern to match, e.g. "*.[ch]". | ||
15 | * @str: String to match. The pattern must match the entire string. | ||
16 | * | ||
17 | * Perform shell-style glob matching, returning true (1) if the match | ||
18 | * succeeds, or false (0) if it fails. Equivalent to !fnmatch(@pat, @str, 0). | ||
19 | * | ||
20 | * Pattern metacharacters are ?, *, [ and \. | ||
21 | * (And, inside character classes, !, - and ].) | ||
22 | * | ||
23 | * This is small and simple implementation intended for device blacklists | ||
24 | * where a string is matched against a number of patterns. Thus, it | ||
25 | * does not preprocess the patterns. It is non-recursive, and run-time | ||
26 | * is at most quadratic: strlen(@str)*strlen(@pat). | ||
27 | * | ||
28 | * An example of the worst case is glob_match("*aaaaa", "aaaaaaaaaa"); | ||
29 | * it takes 6 passes over the pattern before matching the string. | ||
30 | * | ||
31 | * Like !fnmatch(@pat, @str, 0) and unlike the shell, this does NOT | ||
32 | * treat / or leading . specially; it isn't actually used for pathnames. | ||
33 | * | ||
34 | * Note that according to glob(7) (and unlike bash), character classes | ||
35 | * are complemented by a leading !; this does not support the regex-style | ||
36 | * [^a-z] syntax. | ||
37 | * | ||
38 | * An opening bracket without a matching close is matched literally. | ||
39 | */ | ||
40 | bool __pure glob_match(char const *pat, char const *str) | ||
41 | { | ||
42 | /* | ||
43 | * Backtrack to previous * on mismatch and retry starting one | ||
44 | * character later in the string. Because * matches all characters | ||
45 | * (no exception for /), it can be easily proved that there's | ||
46 | * never a need to backtrack multiple levels. | ||
47 | */ | ||
48 | char const *back_pat = NULL, *back_str = back_str; | ||
49 | |||
50 | /* | ||
51 | * Loop over each token (character or class) in pat, matching | ||
52 | * it against the remaining unmatched tail of str. Return false | ||
53 | * on mismatch, or true after matching the trailing nul bytes. | ||
54 | */ | ||
55 | for (;;) { | ||
56 | unsigned char c = *str++; | ||
57 | unsigned char d = *pat++; | ||
58 | |||
59 | switch (d) { | ||
60 | case '?': /* Wildcard: anything but nul */ | ||
61 | if (c == '\0') | ||
62 | return false; | ||
63 | break; | ||
64 | case '*': /* Any-length wildcard */ | ||
65 | if (*pat == '\0') /* Optimize trailing * case */ | ||
66 | return true; | ||
67 | back_pat = pat; | ||
68 | back_str = --str; /* Allow zero-length match */ | ||
69 | break; | ||
70 | case '[': { /* Character class */ | ||
71 | bool match = false, inverted = (*pat == '!'); | ||
72 | char const *class = pat + inverted; | ||
73 | unsigned char a = *class++; | ||
74 | |||
75 | /* | ||
76 | * Iterate over each span in the character class. | ||
77 | * A span is either a single character a, or a | ||
78 | * range a-b. The first span may begin with ']'. | ||
79 | */ | ||
80 | do { | ||
81 | unsigned char b = a; | ||
82 | |||
83 | if (a == '\0') /* Malformed */ | ||
84 | goto literal; | ||
85 | |||
86 | if (class[0] == '-' && class[1] != ']') { | ||
87 | b = class[1]; | ||
88 | |||
89 | if (b == '\0') | ||
90 | goto literal; | ||
91 | |||
92 | class += 2; | ||
93 | /* Any special action if a > b? */ | ||
94 | } | ||
95 | match |= (a <= c && c <= b); | ||
96 | } while ((a = *class++) != ']'); | ||
97 | |||
98 | if (match == inverted) | ||
99 | goto backtrack; | ||
100 | pat = class; | ||
101 | } | ||
102 | break; | ||
103 | case '\\': | ||
104 | d = *pat++; | ||
105 | /*FALLTHROUGH*/ | ||
106 | default: /* Literal character */ | ||
107 | literal: | ||
108 | if (c == d) { | ||
109 | if (d == '\0') | ||
110 | return true; | ||
111 | break; | ||
112 | } | ||
113 | backtrack: | ||
114 | if (c == '\0' || !back_pat) | ||
115 | return false; /* No point continuing */ | ||
116 | /* Try again from last *, one character later in str. */ | ||
117 | pat = back_pat; | ||
118 | str = ++back_str; | ||
119 | break; | ||
120 | } | ||
121 | } | ||
122 | } | ||
123 | EXPORT_SYMBOL(glob_match); | ||
124 | |||
125 | |||
126 | #ifdef CONFIG_GLOB_SELFTEST | ||
127 | |||
128 | #include <linux/printk.h> | ||
129 | #include <linux/moduleparam.h> | ||
130 | |||
131 | /* Boot with "glob.verbose=1" to show successful tests, too */ | ||
132 | static bool verbose = false; | ||
133 | module_param(verbose, bool, 0); | ||
134 | |||
135 | struct glob_test { | ||
136 | char const *pat, *str; | ||
137 | bool expected; | ||
138 | }; | ||
139 | |||
140 | static bool __pure __init test(char const *pat, char const *str, bool expected) | ||
141 | { | ||
142 | bool match = glob_match(pat, str); | ||
143 | bool success = match == expected; | ||
144 | |||
145 | /* Can't get string literals into a particular section, so... */ | ||
146 | static char const msg_error[] __initconst = | ||
147 | KERN_ERR "glob: \"%s\" vs. \"%s\": %s *** ERROR ***\n"; | ||
148 | static char const msg_ok[] __initconst = | ||
149 | KERN_DEBUG "glob: \"%s\" vs. \"%s\": %s OK\n"; | ||
150 | static char const mismatch[] __initconst = "mismatch"; | ||
151 | char const *message; | ||
152 | |||
153 | if (!success) | ||
154 | message = msg_error; | ||
155 | else if (verbose) | ||
156 | message = msg_ok; | ||
157 | else | ||
158 | return success; | ||
159 | |||
160 | printk(message, pat, str, mismatch + 3*match); | ||
161 | return success; | ||
162 | } | ||
163 | |||
164 | /* | ||
165 | * The tests are all jammed together in one array to make it simpler | ||
166 | * to place that array in the .init.rodata section. The obvious | ||
167 | * "array of structures containing char *" has no way to force the | ||
168 | * pointed-to strings to be in a particular section. | ||
169 | * | ||
170 | * Anyway, a test consists of: | ||
171 | * 1. Expected glob_match result: '1' or '0'. | ||
172 | * 2. Pattern to match: null-terminated string | ||
173 | * 3. String to match against: null-terminated string | ||
174 | * | ||
175 | * The list of tests is terminated with a final '\0' instead of | ||
176 | * a glob_match result character. | ||
177 | */ | ||
178 | static char const glob_tests[] __initconst = | ||
179 | /* Some basic tests */ | ||
180 | "1" "a\0" "a\0" | ||
181 | "0" "a\0" "b\0" | ||
182 | "0" "a\0" "aa\0" | ||
183 | "0" "a\0" "\0" | ||
184 | "1" "\0" "\0" | ||
185 | "0" "\0" "a\0" | ||
186 | /* Simple character class tests */ | ||
187 | "1" "[a]\0" "a\0" | ||
188 | "0" "[a]\0" "b\0" | ||
189 | "0" "[!a]\0" "a\0" | ||
190 | "1" "[!a]\0" "b\0" | ||
191 | "1" "[ab]\0" "a\0" | ||
192 | "1" "[ab]\0" "b\0" | ||
193 | "0" "[ab]\0" "c\0" | ||
194 | "1" "[!ab]\0" "c\0" | ||
195 | "1" "[a-c]\0" "b\0" | ||
196 | "0" "[a-c]\0" "d\0" | ||
197 | /* Corner cases in character class parsing */ | ||
198 | "1" "[a-c-e-g]\0" "-\0" | ||
199 | "0" "[a-c-e-g]\0" "d\0" | ||
200 | "1" "[a-c-e-g]\0" "f\0" | ||
201 | "1" "[]a-ceg-ik[]\0" "a\0" | ||
202 | "1" "[]a-ceg-ik[]\0" "]\0" | ||
203 | "1" "[]a-ceg-ik[]\0" "[\0" | ||
204 | "1" "[]a-ceg-ik[]\0" "h\0" | ||
205 | "0" "[]a-ceg-ik[]\0" "f\0" | ||
206 | "0" "[!]a-ceg-ik[]\0" "h\0" | ||
207 | "0" "[!]a-ceg-ik[]\0" "]\0" | ||
208 | "1" "[!]a-ceg-ik[]\0" "f\0" | ||
209 | /* Simple wild cards */ | ||
210 | "1" "?\0" "a\0" | ||
211 | "0" "?\0" "aa\0" | ||
212 | "0" "??\0" "a\0" | ||
213 | "1" "?x?\0" "axb\0" | ||
214 | "0" "?x?\0" "abx\0" | ||
215 | "0" "?x?\0" "xab\0" | ||
216 | /* Asterisk wild cards (backtracking) */ | ||
217 | "0" "*??\0" "a\0" | ||
218 | "1" "*??\0" "ab\0" | ||
219 | "1" "*??\0" "abc\0" | ||
220 | "1" "*??\0" "abcd\0" | ||
221 | "0" "??*\0" "a\0" | ||
222 | "1" "??*\0" "ab\0" | ||
223 | "1" "??*\0" "abc\0" | ||
224 | "1" "??*\0" "abcd\0" | ||
225 | "0" "?*?\0" "a\0" | ||
226 | "1" "?*?\0" "ab\0" | ||
227 | "1" "?*?\0" "abc\0" | ||
228 | "1" "?*?\0" "abcd\0" | ||
229 | "1" "*b\0" "b\0" | ||
230 | "1" "*b\0" "ab\0" | ||
231 | "0" "*b\0" "ba\0" | ||
232 | "1" "*b\0" "bb\0" | ||
233 | "1" "*b\0" "abb\0" | ||
234 | "1" "*b\0" "bab\0" | ||
235 | "1" "*bc\0" "abbc\0" | ||
236 | "1" "*bc\0" "bc\0" | ||
237 | "1" "*bc\0" "bbc\0" | ||
238 | "1" "*bc\0" "bcbc\0" | ||
239 | /* Multiple asterisks (complex backtracking) */ | ||
240 | "1" "*ac*\0" "abacadaeafag\0" | ||
241 | "1" "*ac*ae*ag*\0" "abacadaeafag\0" | ||
242 | "1" "*a*b*[bc]*[ef]*g*\0" "abacadaeafag\0" | ||
243 | "0" "*a*b*[ef]*[cd]*g*\0" "abacadaeafag\0" | ||
244 | "1" "*abcd*\0" "abcabcabcabcdefg\0" | ||
245 | "1" "*ab*cd*\0" "abcabcabcabcdefg\0" | ||
246 | "1" "*abcd*abcdef*\0" "abcabcdabcdeabcdefg\0" | ||
247 | "0" "*abcd*\0" "abcabcabcabcefg\0" | ||
248 | "0" "*ab*cd*\0" "abcabcabcabcefg\0"; | ||
249 | |||
250 | static int __init glob_init(void) | ||
251 | { | ||
252 | unsigned successes = 0; | ||
253 | unsigned n = 0; | ||
254 | char const *p = glob_tests; | ||
255 | static char const message[] __initconst = | ||
256 | KERN_INFO "glob: %u self-tests passed, %u failed\n"; | ||
257 | |||
258 | /* | ||
259 | * Tests are jammed together in a string. The first byte is '1' | ||
260 | * or '0' to indicate the expected outcome, or '\0' to indicate the | ||
261 | * end of the tests. Then come two null-terminated strings: the | ||
262 | * pattern and the string to match it against. | ||
263 | */ | ||
264 | while (*p) { | ||
265 | bool expected = *p++ & 1; | ||
266 | char const *pat = p; | ||
267 | |||
268 | p += strlen(p) + 1; | ||
269 | successes += test(pat, p, expected); | ||
270 | p += strlen(p) + 1; | ||
271 | n++; | ||
272 | } | ||
273 | |||
274 | n -= successes; | ||
275 | printk(message, successes, n); | ||
276 | |||
277 | /* What's the errno for "kernel bug detected"? Guess... */ | ||
278 | return n ? -ECANCELED : 0; | ||
279 | } | ||
280 | |||
281 | /* We need a dummy exit function to allow unload */ | ||
282 | static void __exit glob_fini(void) { } | ||
283 | |||
284 | module_init(glob_init); | ||
285 | module_exit(glob_fini); | ||
286 | |||
287 | #endif /* CONFIG_GLOB_SELFTEST */ | ||
diff --git a/lib/klist.c b/lib/klist.c index 358a368a2947..89b485a2a58d 100644 --- a/lib/klist.c +++ b/lib/klist.c | |||
@@ -140,11 +140,11 @@ void klist_add_tail(struct klist_node *n, struct klist *k) | |||
140 | EXPORT_SYMBOL_GPL(klist_add_tail); | 140 | EXPORT_SYMBOL_GPL(klist_add_tail); |
141 | 141 | ||
142 | /** | 142 | /** |
143 | * klist_add_after - Init a klist_node and add it after an existing node | 143 | * klist_add_behind - Init a klist_node and add it after an existing node |
144 | * @n: node we're adding. | 144 | * @n: node we're adding. |
145 | * @pos: node to put @n after | 145 | * @pos: node to put @n after |
146 | */ | 146 | */ |
147 | void klist_add_after(struct klist_node *n, struct klist_node *pos) | 147 | void klist_add_behind(struct klist_node *n, struct klist_node *pos) |
148 | { | 148 | { |
149 | struct klist *k = knode_klist(pos); | 149 | struct klist *k = knode_klist(pos); |
150 | 150 | ||
@@ -153,7 +153,7 @@ void klist_add_after(struct klist_node *n, struct klist_node *pos) | |||
153 | list_add(&n->n_node, &pos->n_node); | 153 | list_add(&n->n_node, &pos->n_node); |
154 | spin_unlock(&k->k_lock); | 154 | spin_unlock(&k->k_lock); |
155 | } | 155 | } |
156 | EXPORT_SYMBOL_GPL(klist_add_after); | 156 | EXPORT_SYMBOL_GPL(klist_add_behind); |
157 | 157 | ||
158 | /** | 158 | /** |
159 | * klist_add_before - Init a klist_node and add it before an existing node | 159 | * klist_add_before - Init a klist_node and add it before an existing node |
diff --git a/lib/list_sort.c b/lib/list_sort.c index 1183fa70a44d..12bcba1c8612 100644 --- a/lib/list_sort.c +++ b/lib/list_sort.c | |||
@@ -1,3 +1,6 @@ | |||
1 | |||
2 | #define pr_fmt(fmt) "list_sort_test: " fmt | ||
3 | |||
1 | #include <linux/kernel.h> | 4 | #include <linux/kernel.h> |
2 | #include <linux/module.h> | 5 | #include <linux/module.h> |
3 | #include <linux/list_sort.h> | 6 | #include <linux/list_sort.h> |
@@ -47,6 +50,7 @@ static void merge_and_restore_back_links(void *priv, | |||
47 | struct list_head *a, struct list_head *b) | 50 | struct list_head *a, struct list_head *b) |
48 | { | 51 | { |
49 | struct list_head *tail = head; | 52 | struct list_head *tail = head; |
53 | u8 count = 0; | ||
50 | 54 | ||
51 | while (a && b) { | 55 | while (a && b) { |
52 | /* if equal, take 'a' -- important for sort stability */ | 56 | /* if equal, take 'a' -- important for sort stability */ |
@@ -70,7 +74,8 @@ static void merge_and_restore_back_links(void *priv, | |||
70 | * element comparison is needed, so the client's cmp() | 74 | * element comparison is needed, so the client's cmp() |
71 | * routine can invoke cond_resched() periodically. | 75 | * routine can invoke cond_resched() periodically. |
72 | */ | 76 | */ |
73 | (*cmp)(priv, tail->next, tail->next); | 77 | if (unlikely(!(++count))) |
78 | (*cmp)(priv, tail->next, tail->next); | ||
74 | 79 | ||
75 | tail->next->prev = tail; | 80 | tail->next->prev = tail; |
76 | tail = tail->next; | 81 | tail = tail->next; |
@@ -123,9 +128,7 @@ void list_sort(void *priv, struct list_head *head, | |||
123 | } | 128 | } |
124 | if (lev > max_lev) { | 129 | if (lev > max_lev) { |
125 | if (unlikely(lev >= ARRAY_SIZE(part)-1)) { | 130 | if (unlikely(lev >= ARRAY_SIZE(part)-1)) { |
126 | printk_once(KERN_DEBUG "list passed to" | 131 | printk_once(KERN_DEBUG "list too long for efficiency\n"); |
127 | " list_sort() too long for" | ||
128 | " efficiency\n"); | ||
129 | lev--; | 132 | lev--; |
130 | } | 133 | } |
131 | max_lev = lev; | 134 | max_lev = lev; |
@@ -168,27 +171,25 @@ static struct debug_el **elts __initdata; | |||
168 | static int __init check(struct debug_el *ela, struct debug_el *elb) | 171 | static int __init check(struct debug_el *ela, struct debug_el *elb) |
169 | { | 172 | { |
170 | if (ela->serial >= TEST_LIST_LEN) { | 173 | if (ela->serial >= TEST_LIST_LEN) { |
171 | printk(KERN_ERR "list_sort_test: error: incorrect serial %d\n", | 174 | pr_err("error: incorrect serial %d\n", ela->serial); |
172 | ela->serial); | ||
173 | return -EINVAL; | 175 | return -EINVAL; |
174 | } | 176 | } |
175 | if (elb->serial >= TEST_LIST_LEN) { | 177 | if (elb->serial >= TEST_LIST_LEN) { |
176 | printk(KERN_ERR "list_sort_test: error: incorrect serial %d\n", | 178 | pr_err("error: incorrect serial %d\n", elb->serial); |
177 | elb->serial); | ||
178 | return -EINVAL; | 179 | return -EINVAL; |
179 | } | 180 | } |
180 | if (elts[ela->serial] != ela || elts[elb->serial] != elb) { | 181 | if (elts[ela->serial] != ela || elts[elb->serial] != elb) { |
181 | printk(KERN_ERR "list_sort_test: error: phantom element\n"); | 182 | pr_err("error: phantom element\n"); |
182 | return -EINVAL; | 183 | return -EINVAL; |
183 | } | 184 | } |
184 | if (ela->poison1 != TEST_POISON1 || ela->poison2 != TEST_POISON2) { | 185 | if (ela->poison1 != TEST_POISON1 || ela->poison2 != TEST_POISON2) { |
185 | printk(KERN_ERR "list_sort_test: error: bad poison: %#x/%#x\n", | 186 | pr_err("error: bad poison: %#x/%#x\n", |
186 | ela->poison1, ela->poison2); | 187 | ela->poison1, ela->poison2); |
187 | return -EINVAL; | 188 | return -EINVAL; |
188 | } | 189 | } |
189 | if (elb->poison1 != TEST_POISON1 || elb->poison2 != TEST_POISON2) { | 190 | if (elb->poison1 != TEST_POISON1 || elb->poison2 != TEST_POISON2) { |
190 | printk(KERN_ERR "list_sort_test: error: bad poison: %#x/%#x\n", | 191 | pr_err("error: bad poison: %#x/%#x\n", |
191 | elb->poison1, elb->poison2); | 192 | elb->poison1, elb->poison2); |
192 | return -EINVAL; | 193 | return -EINVAL; |
193 | } | 194 | } |
194 | return 0; | 195 | return 0; |
@@ -207,25 +208,23 @@ static int __init cmp(void *priv, struct list_head *a, struct list_head *b) | |||
207 | 208 | ||
208 | static int __init list_sort_test(void) | 209 | static int __init list_sort_test(void) |
209 | { | 210 | { |
210 | int i, count = 1, err = -EINVAL; | 211 | int i, count = 1, err = -ENOMEM; |
211 | struct debug_el *el; | 212 | struct debug_el *el; |
212 | struct list_head *cur, *tmp; | 213 | struct list_head *cur; |
213 | LIST_HEAD(head); | 214 | LIST_HEAD(head); |
214 | 215 | ||
215 | printk(KERN_DEBUG "list_sort_test: start testing list_sort()\n"); | 216 | pr_debug("start testing list_sort()\n"); |
216 | 217 | ||
217 | elts = kmalloc(sizeof(void *) * TEST_LIST_LEN, GFP_KERNEL); | 218 | elts = kcalloc(TEST_LIST_LEN, sizeof(*elts), GFP_KERNEL); |
218 | if (!elts) { | 219 | if (!elts) { |
219 | printk(KERN_ERR "list_sort_test: error: cannot allocate " | 220 | pr_err("error: cannot allocate memory\n"); |
220 | "memory\n"); | 221 | return err; |
221 | goto exit; | ||
222 | } | 222 | } |
223 | 223 | ||
224 | for (i = 0; i < TEST_LIST_LEN; i++) { | 224 | for (i = 0; i < TEST_LIST_LEN; i++) { |
225 | el = kmalloc(sizeof(*el), GFP_KERNEL); | 225 | el = kmalloc(sizeof(*el), GFP_KERNEL); |
226 | if (!el) { | 226 | if (!el) { |
227 | printk(KERN_ERR "list_sort_test: error: cannot " | 227 | pr_err("error: cannot allocate memory\n"); |
228 | "allocate memory\n"); | ||
229 | goto exit; | 228 | goto exit; |
230 | } | 229 | } |
231 | /* force some equivalencies */ | 230 | /* force some equivalencies */ |
@@ -239,52 +238,52 @@ static int __init list_sort_test(void) | |||
239 | 238 | ||
240 | list_sort(NULL, &head, cmp); | 239 | list_sort(NULL, &head, cmp); |
241 | 240 | ||
241 | err = -EINVAL; | ||
242 | for (cur = head.next; cur->next != &head; cur = cur->next) { | 242 | for (cur = head.next; cur->next != &head; cur = cur->next) { |
243 | struct debug_el *el1; | 243 | struct debug_el *el1; |
244 | int cmp_result; | 244 | int cmp_result; |
245 | 245 | ||
246 | if (cur->next->prev != cur) { | 246 | if (cur->next->prev != cur) { |
247 | printk(KERN_ERR "list_sort_test: error: list is " | 247 | pr_err("error: list is corrupted\n"); |
248 | "corrupted\n"); | ||
249 | goto exit; | 248 | goto exit; |
250 | } | 249 | } |
251 | 250 | ||
252 | cmp_result = cmp(NULL, cur, cur->next); | 251 | cmp_result = cmp(NULL, cur, cur->next); |
253 | if (cmp_result > 0) { | 252 | if (cmp_result > 0) { |
254 | printk(KERN_ERR "list_sort_test: error: list is not " | 253 | pr_err("error: list is not sorted\n"); |
255 | "sorted\n"); | ||
256 | goto exit; | 254 | goto exit; |
257 | } | 255 | } |
258 | 256 | ||
259 | el = container_of(cur, struct debug_el, list); | 257 | el = container_of(cur, struct debug_el, list); |
260 | el1 = container_of(cur->next, struct debug_el, list); | 258 | el1 = container_of(cur->next, struct debug_el, list); |
261 | if (cmp_result == 0 && el->serial >= el1->serial) { | 259 | if (cmp_result == 0 && el->serial >= el1->serial) { |
262 | printk(KERN_ERR "list_sort_test: error: order of " | 260 | pr_err("error: order of equivalent elements not " |
263 | "equivalent elements not preserved\n"); | 261 | "preserved\n"); |
264 | goto exit; | 262 | goto exit; |
265 | } | 263 | } |
266 | 264 | ||
267 | if (check(el, el1)) { | 265 | if (check(el, el1)) { |
268 | printk(KERN_ERR "list_sort_test: error: element check " | 266 | pr_err("error: element check failed\n"); |
269 | "failed\n"); | ||
270 | goto exit; | 267 | goto exit; |
271 | } | 268 | } |
272 | count++; | 269 | count++; |
273 | } | 270 | } |
271 | if (head.prev != cur) { | ||
272 | pr_err("error: list is corrupted\n"); | ||
273 | goto exit; | ||
274 | } | ||
275 | |||
274 | 276 | ||
275 | if (count != TEST_LIST_LEN) { | 277 | if (count != TEST_LIST_LEN) { |
276 | printk(KERN_ERR "list_sort_test: error: bad list length %d", | 278 | pr_err("error: bad list length %d", count); |
277 | count); | ||
278 | goto exit; | 279 | goto exit; |
279 | } | 280 | } |
280 | 281 | ||
281 | err = 0; | 282 | err = 0; |
282 | exit: | 283 | exit: |
284 | for (i = 0; i < TEST_LIST_LEN; i++) | ||
285 | kfree(elts[i]); | ||
283 | kfree(elts); | 286 | kfree(elts); |
284 | list_for_each_safe(cur, tmp, &head) { | ||
285 | list_del(cur); | ||
286 | kfree(container_of(cur, struct debug_el, list)); | ||
287 | } | ||
288 | return err; | 287 | return err; |
289 | } | 288 | } |
290 | module_init(list_sort_test); | 289 | module_init(list_sort_test); |
diff --git a/lib/string_helpers.c b/lib/string_helpers.c index ed5c1454dd62..29033f319aea 100644 --- a/lib/string_helpers.c +++ b/lib/string_helpers.c | |||
@@ -25,12 +25,15 @@ | |||
25 | int string_get_size(u64 size, const enum string_size_units units, | 25 | int string_get_size(u64 size, const enum string_size_units units, |
26 | char *buf, int len) | 26 | char *buf, int len) |
27 | { | 27 | { |
28 | static const char *units_10[] = { "B", "kB", "MB", "GB", "TB", "PB", | 28 | static const char *const units_10[] = { |
29 | "EB", "ZB", "YB", NULL}; | 29 | "B", "kB", "MB", "GB", "TB", "PB", "EB", "ZB", "YB", NULL |
30 | static const char *units_2[] = {"B", "KiB", "MiB", "GiB", "TiB", "PiB", | 30 | }; |
31 | "EiB", "ZiB", "YiB", NULL }; | 31 | static const char *const units_2[] = { |
32 | static const char **units_str[] = { | 32 | "B", "KiB", "MiB", "GiB", "TiB", "PiB", "EiB", "ZiB", "YiB", |
33 | [STRING_UNITS_10] = units_10, | 33 | NULL |
34 | }; | ||
35 | static const char *const *const units_str[] = { | ||
36 | [STRING_UNITS_10] = units_10, | ||
34 | [STRING_UNITS_2] = units_2, | 37 | [STRING_UNITS_2] = units_2, |
35 | }; | 38 | }; |
36 | static const unsigned int divisor[] = { | 39 | static const unsigned int divisor[] = { |
diff --git a/lib/test-kstrtox.c b/lib/test-kstrtox.c index bea3f3fa3f02..4137bca5f8e8 100644 --- a/lib/test-kstrtox.c +++ b/lib/test-kstrtox.c | |||
@@ -3,7 +3,7 @@ | |||
3 | #include <linux/module.h> | 3 | #include <linux/module.h> |
4 | 4 | ||
5 | #define for_each_test(i, test) \ | 5 | #define for_each_test(i, test) \ |
6 | for (i = 0; i < sizeof(test) / sizeof(test[0]); i++) | 6 | for (i = 0; i < ARRAY_SIZE(test); i++) |
7 | 7 | ||
8 | struct test_fail { | 8 | struct test_fail { |
9 | const char *str; | 9 | const char *str; |
diff --git a/lib/zlib_deflate/deflate.c b/lib/zlib_deflate/deflate.c index d63381e8e333..d20ef458f137 100644 --- a/lib/zlib_deflate/deflate.c +++ b/lib/zlib_deflate/deflate.c | |||
@@ -250,52 +250,6 @@ int zlib_deflateInit2( | |||
250 | } | 250 | } |
251 | 251 | ||
252 | /* ========================================================================= */ | 252 | /* ========================================================================= */ |
253 | #if 0 | ||
254 | int zlib_deflateSetDictionary( | ||
255 | z_streamp strm, | ||
256 | const Byte *dictionary, | ||
257 | uInt dictLength | ||
258 | ) | ||
259 | { | ||
260 | deflate_state *s; | ||
261 | uInt length = dictLength; | ||
262 | uInt n; | ||
263 | IPos hash_head = 0; | ||
264 | |||
265 | if (strm == NULL || strm->state == NULL || dictionary == NULL) | ||
266 | return Z_STREAM_ERROR; | ||
267 | |||
268 | s = (deflate_state *) strm->state; | ||
269 | if (s->status != INIT_STATE) return Z_STREAM_ERROR; | ||
270 | |||
271 | strm->adler = zlib_adler32(strm->adler, dictionary, dictLength); | ||
272 | |||
273 | if (length < MIN_MATCH) return Z_OK; | ||
274 | if (length > MAX_DIST(s)) { | ||
275 | length = MAX_DIST(s); | ||
276 | #ifndef USE_DICT_HEAD | ||
277 | dictionary += dictLength - length; /* use the tail of the dictionary */ | ||
278 | #endif | ||
279 | } | ||
280 | memcpy((char *)s->window, dictionary, length); | ||
281 | s->strstart = length; | ||
282 | s->block_start = (long)length; | ||
283 | |||
284 | /* Insert all strings in the hash table (except for the last two bytes). | ||
285 | * s->lookahead stays null, so s->ins_h will be recomputed at the next | ||
286 | * call of fill_window. | ||
287 | */ | ||
288 | s->ins_h = s->window[0]; | ||
289 | UPDATE_HASH(s, s->ins_h, s->window[1]); | ||
290 | for (n = 0; n <= length - MIN_MATCH; n++) { | ||
291 | INSERT_STRING(s, n, hash_head); | ||
292 | } | ||
293 | if (hash_head) hash_head = 0; /* to make compiler happy */ | ||
294 | return Z_OK; | ||
295 | } | ||
296 | #endif /* 0 */ | ||
297 | |||
298 | /* ========================================================================= */ | ||
299 | int zlib_deflateReset( | 253 | int zlib_deflateReset( |
300 | z_streamp strm | 254 | z_streamp strm |
301 | ) | 255 | ) |
@@ -326,45 +280,6 @@ int zlib_deflateReset( | |||
326 | return Z_OK; | 280 | return Z_OK; |
327 | } | 281 | } |
328 | 282 | ||
329 | /* ========================================================================= */ | ||
330 | #if 0 | ||
331 | int zlib_deflateParams( | ||
332 | z_streamp strm, | ||
333 | int level, | ||
334 | int strategy | ||
335 | ) | ||
336 | { | ||
337 | deflate_state *s; | ||
338 | compress_func func; | ||
339 | int err = Z_OK; | ||
340 | |||
341 | if (strm == NULL || strm->state == NULL) return Z_STREAM_ERROR; | ||
342 | s = (deflate_state *) strm->state; | ||
343 | |||
344 | if (level == Z_DEFAULT_COMPRESSION) { | ||
345 | level = 6; | ||
346 | } | ||
347 | if (level < 0 || level > 9 || strategy < 0 || strategy > Z_HUFFMAN_ONLY) { | ||
348 | return Z_STREAM_ERROR; | ||
349 | } | ||
350 | func = configuration_table[s->level].func; | ||
351 | |||
352 | if (func != configuration_table[level].func && strm->total_in != 0) { | ||
353 | /* Flush the last buffer: */ | ||
354 | err = zlib_deflate(strm, Z_PARTIAL_FLUSH); | ||
355 | } | ||
356 | if (s->level != level) { | ||
357 | s->level = level; | ||
358 | s->max_lazy_match = configuration_table[level].max_lazy; | ||
359 | s->good_match = configuration_table[level].good_length; | ||
360 | s->nice_match = configuration_table[level].nice_length; | ||
361 | s->max_chain_length = configuration_table[level].max_chain; | ||
362 | } | ||
363 | s->strategy = strategy; | ||
364 | return err; | ||
365 | } | ||
366 | #endif /* 0 */ | ||
367 | |||
368 | /* ========================================================================= | 283 | /* ========================================================================= |
369 | * Put a short in the pending buffer. The 16-bit value is put in MSB order. | 284 | * Put a short in the pending buffer. The 16-bit value is put in MSB order. |
370 | * IN assertion: the stream state is correct and there is enough room in | 285 | * IN assertion: the stream state is correct and there is enough room in |
@@ -568,64 +483,6 @@ int zlib_deflateEnd( | |||
568 | return status == BUSY_STATE ? Z_DATA_ERROR : Z_OK; | 483 | return status == BUSY_STATE ? Z_DATA_ERROR : Z_OK; |
569 | } | 484 | } |
570 | 485 | ||
571 | /* ========================================================================= | ||
572 | * Copy the source state to the destination state. | ||
573 | */ | ||
574 | #if 0 | ||
575 | int zlib_deflateCopy ( | ||
576 | z_streamp dest, | ||
577 | z_streamp source | ||
578 | ) | ||
579 | { | ||
580 | #ifdef MAXSEG_64K | ||
581 | return Z_STREAM_ERROR; | ||
582 | #else | ||
583 | deflate_state *ds; | ||
584 | deflate_state *ss; | ||
585 | ush *overlay; | ||
586 | deflate_workspace *mem; | ||
587 | |||
588 | |||
589 | if (source == NULL || dest == NULL || source->state == NULL) { | ||
590 | return Z_STREAM_ERROR; | ||
591 | } | ||
592 | |||
593 | ss = (deflate_state *) source->state; | ||
594 | |||
595 | *dest = *source; | ||
596 | |||
597 | mem = (deflate_workspace *) dest->workspace; | ||
598 | |||
599 | ds = &(mem->deflate_memory); | ||
600 | |||
601 | dest->state = (struct internal_state *) ds; | ||
602 | *ds = *ss; | ||
603 | ds->strm = dest; | ||
604 | |||
605 | ds->window = (Byte *) mem->window_memory; | ||
606 | ds->prev = (Pos *) mem->prev_memory; | ||
607 | ds->head = (Pos *) mem->head_memory; | ||
608 | overlay = (ush *) mem->overlay_memory; | ||
609 | ds->pending_buf = (uch *) overlay; | ||
610 | |||
611 | memcpy(ds->window, ss->window, ds->w_size * 2 * sizeof(Byte)); | ||
612 | memcpy(ds->prev, ss->prev, ds->w_size * sizeof(Pos)); | ||
613 | memcpy(ds->head, ss->head, ds->hash_size * sizeof(Pos)); | ||
614 | memcpy(ds->pending_buf, ss->pending_buf, (uInt)ds->pending_buf_size); | ||
615 | |||
616 | ds->pending_out = ds->pending_buf + (ss->pending_out - ss->pending_buf); | ||
617 | ds->d_buf = overlay + ds->lit_bufsize/sizeof(ush); | ||
618 | ds->l_buf = ds->pending_buf + (1+sizeof(ush))*ds->lit_bufsize; | ||
619 | |||
620 | ds->l_desc.dyn_tree = ds->dyn_ltree; | ||
621 | ds->d_desc.dyn_tree = ds->dyn_dtree; | ||
622 | ds->bl_desc.dyn_tree = ds->bl_tree; | ||
623 | |||
624 | return Z_OK; | ||
625 | #endif | ||
626 | } | ||
627 | #endif /* 0 */ | ||
628 | |||
629 | /* =========================================================================== | 486 | /* =========================================================================== |
630 | * Read a new buffer from the current input stream, update the adler32 | 487 | * Read a new buffer from the current input stream, update the adler32 |
631 | * and total number of bytes read. All deflate() input goes through | 488 | * and total number of bytes read. All deflate() input goes through |
diff --git a/lib/zlib_inflate/inflate.c b/lib/zlib_inflate/inflate.c index f5ce87b0800e..58a733b10387 100644 --- a/lib/zlib_inflate/inflate.c +++ b/lib/zlib_inflate/inflate.c | |||
@@ -45,21 +45,6 @@ int zlib_inflateReset(z_streamp strm) | |||
45 | return Z_OK; | 45 | return Z_OK; |
46 | } | 46 | } |
47 | 47 | ||
48 | #if 0 | ||
49 | int zlib_inflatePrime(z_streamp strm, int bits, int value) | ||
50 | { | ||
51 | struct inflate_state *state; | ||
52 | |||
53 | if (strm == NULL || strm->state == NULL) return Z_STREAM_ERROR; | ||
54 | state = (struct inflate_state *)strm->state; | ||
55 | if (bits > 16 || state->bits + bits > 32) return Z_STREAM_ERROR; | ||
56 | value &= (1L << bits) - 1; | ||
57 | state->hold += value << state->bits; | ||
58 | state->bits += bits; | ||
59 | return Z_OK; | ||
60 | } | ||
61 | #endif | ||
62 | |||
63 | int zlib_inflateInit2(z_streamp strm, int windowBits) | 48 | int zlib_inflateInit2(z_streamp strm, int windowBits) |
64 | { | 49 | { |
65 | struct inflate_state *state; | 50 | struct inflate_state *state; |
@@ -761,123 +746,6 @@ int zlib_inflateEnd(z_streamp strm) | |||
761 | return Z_OK; | 746 | return Z_OK; |
762 | } | 747 | } |
763 | 748 | ||
764 | #if 0 | ||
765 | int zlib_inflateSetDictionary(z_streamp strm, const Byte *dictionary, | ||
766 | uInt dictLength) | ||
767 | { | ||
768 | struct inflate_state *state; | ||
769 | unsigned long id; | ||
770 | |||
771 | /* check state */ | ||
772 | if (strm == NULL || strm->state == NULL) return Z_STREAM_ERROR; | ||
773 | state = (struct inflate_state *)strm->state; | ||
774 | if (state->wrap != 0 && state->mode != DICT) | ||
775 | return Z_STREAM_ERROR; | ||
776 | |||
777 | /* check for correct dictionary id */ | ||
778 | if (state->mode == DICT) { | ||
779 | id = zlib_adler32(0L, NULL, 0); | ||
780 | id = zlib_adler32(id, dictionary, dictLength); | ||
781 | if (id != state->check) | ||
782 | return Z_DATA_ERROR; | ||
783 | } | ||
784 | |||
785 | /* copy dictionary to window */ | ||
786 | zlib_updatewindow(strm, strm->avail_out); | ||
787 | |||
788 | if (dictLength > state->wsize) { | ||
789 | memcpy(state->window, dictionary + dictLength - state->wsize, | ||
790 | state->wsize); | ||
791 | state->whave = state->wsize; | ||
792 | } | ||
793 | else { | ||
794 | memcpy(state->window + state->wsize - dictLength, dictionary, | ||
795 | dictLength); | ||
796 | state->whave = dictLength; | ||
797 | } | ||
798 | state->havedict = 1; | ||
799 | return Z_OK; | ||
800 | } | ||
801 | #endif | ||
802 | |||
803 | #if 0 | ||
804 | /* | ||
805 | Search buf[0..len-1] for the pattern: 0, 0, 0xff, 0xff. Return when found | ||
806 | or when out of input. When called, *have is the number of pattern bytes | ||
807 | found in order so far, in 0..3. On return *have is updated to the new | ||
808 | state. If on return *have equals four, then the pattern was found and the | ||
809 | return value is how many bytes were read including the last byte of the | ||
810 | pattern. If *have is less than four, then the pattern has not been found | ||
811 | yet and the return value is len. In the latter case, zlib_syncsearch() can be | ||
812 | called again with more data and the *have state. *have is initialized to | ||
813 | zero for the first call. | ||
814 | */ | ||
815 | static unsigned zlib_syncsearch(unsigned *have, unsigned char *buf, | ||
816 | unsigned len) | ||
817 | { | ||
818 | unsigned got; | ||
819 | unsigned next; | ||
820 | |||
821 | got = *have; | ||
822 | next = 0; | ||
823 | while (next < len && got < 4) { | ||
824 | if ((int)(buf[next]) == (got < 2 ? 0 : 0xff)) | ||
825 | got++; | ||
826 | else if (buf[next]) | ||
827 | got = 0; | ||
828 | else | ||
829 | got = 4 - got; | ||
830 | next++; | ||
831 | } | ||
832 | *have = got; | ||
833 | return next; | ||
834 | } | ||
835 | #endif | ||
836 | |||
837 | #if 0 | ||
838 | int zlib_inflateSync(z_streamp strm) | ||
839 | { | ||
840 | unsigned len; /* number of bytes to look at or looked at */ | ||
841 | unsigned long in, out; /* temporary to save total_in and total_out */ | ||
842 | unsigned char buf[4]; /* to restore bit buffer to byte string */ | ||
843 | struct inflate_state *state; | ||
844 | |||
845 | /* check parameters */ | ||
846 | if (strm == NULL || strm->state == NULL) return Z_STREAM_ERROR; | ||
847 | state = (struct inflate_state *)strm->state; | ||
848 | if (strm->avail_in == 0 && state->bits < 8) return Z_BUF_ERROR; | ||
849 | |||
850 | /* if first time, start search in bit buffer */ | ||
851 | if (state->mode != SYNC) { | ||
852 | state->mode = SYNC; | ||
853 | state->hold <<= state->bits & 7; | ||
854 | state->bits -= state->bits & 7; | ||
855 | len = 0; | ||
856 | while (state->bits >= 8) { | ||
857 | buf[len++] = (unsigned char)(state->hold); | ||
858 | state->hold >>= 8; | ||
859 | state->bits -= 8; | ||
860 | } | ||
861 | state->have = 0; | ||
862 | zlib_syncsearch(&(state->have), buf, len); | ||
863 | } | ||
864 | |||
865 | /* search available input */ | ||
866 | len = zlib_syncsearch(&(state->have), strm->next_in, strm->avail_in); | ||
867 | strm->avail_in -= len; | ||
868 | strm->next_in += len; | ||
869 | strm->total_in += len; | ||
870 | |||
871 | /* return no joy or set up to restart inflate() on a new block */ | ||
872 | if (state->have != 4) return Z_DATA_ERROR; | ||
873 | in = strm->total_in; out = strm->total_out; | ||
874 | zlib_inflateReset(strm); | ||
875 | strm->total_in = in; strm->total_out = out; | ||
876 | state->mode = TYPE; | ||
877 | return Z_OK; | ||
878 | } | ||
879 | #endif | ||
880 | |||
881 | /* | 749 | /* |
882 | * This subroutine adds the data at next_in/avail_in to the output history | 750 | * This subroutine adds the data at next_in/avail_in to the output history |
883 | * without performing any output. The output buffer must be "caught up"; | 751 | * without performing any output. The output buffer must be "caught up"; |
diff --git a/mm/Kconfig b/mm/Kconfig index 3e9977a9d657..886db2158538 100644 --- a/mm/Kconfig +++ b/mm/Kconfig | |||
@@ -508,21 +508,34 @@ config CMA_DEBUG | |||
508 | processing calls such as dma_alloc_from_contiguous(). | 508 | processing calls such as dma_alloc_from_contiguous(). |
509 | This option does not affect warning and error messages. | 509 | This option does not affect warning and error messages. |
510 | 510 | ||
511 | config ZBUD | 511 | config CMA_AREAS |
512 | tristate | 512 | int "Maximum count of the CMA areas" |
513 | default n | 513 | depends on CMA |
514 | default 7 | ||
514 | help | 515 | help |
515 | A special purpose allocator for storing compressed pages. | 516 | CMA allows to create CMA areas for particular purpose, mainly, |
516 | It is designed to store up to two compressed pages per physical | 517 | used as device private area. This parameter sets the maximum |
517 | page. While this design limits storage density, it has simple and | 518 | number of CMA area in the system. |
518 | deterministic reclaim properties that make it preferable to a higher | 519 | |
519 | density approach when reclaim will be used. | 520 | If unsure, leave the default value "7". |
521 | |||
522 | config MEM_SOFT_DIRTY | ||
523 | bool "Track memory changes" | ||
524 | depends on CHECKPOINT_RESTORE && HAVE_ARCH_SOFT_DIRTY && PROC_FS | ||
525 | select PROC_PAGE_MONITOR | ||
526 | help | ||
527 | This option enables memory changes tracking by introducing a | ||
528 | soft-dirty bit on pte-s. This bit it set when someone writes | ||
529 | into a page just as regular dirty bit, but unlike the latter | ||
530 | it can be cleared by hands. | ||
531 | |||
532 | See Documentation/vm/soft-dirty.txt for more details. | ||
520 | 533 | ||
521 | config ZSWAP | 534 | config ZSWAP |
522 | bool "Compressed cache for swap pages (EXPERIMENTAL)" | 535 | bool "Compressed cache for swap pages (EXPERIMENTAL)" |
523 | depends on FRONTSWAP && CRYPTO=y | 536 | depends on FRONTSWAP && CRYPTO=y |
524 | select CRYPTO_LZO | 537 | select CRYPTO_LZO |
525 | select ZBUD | 538 | select ZPOOL |
526 | default n | 539 | default n |
527 | help | 540 | help |
528 | A lightweight compressed cache for swap pages. It takes | 541 | A lightweight compressed cache for swap pages. It takes |
@@ -538,17 +551,22 @@ config ZSWAP | |||
538 | they have not be fully explored on the large set of potential | 551 | they have not be fully explored on the large set of potential |
539 | configurations and workloads that exist. | 552 | configurations and workloads that exist. |
540 | 553 | ||
541 | config MEM_SOFT_DIRTY | 554 | config ZPOOL |
542 | bool "Track memory changes" | 555 | tristate "Common API for compressed memory storage" |
543 | depends on CHECKPOINT_RESTORE && HAVE_ARCH_SOFT_DIRTY && PROC_FS | 556 | default n |
544 | select PROC_PAGE_MONITOR | ||
545 | help | 557 | help |
546 | This option enables memory changes tracking by introducing a | 558 | Compressed memory storage API. This allows using either zbud or |
547 | soft-dirty bit on pte-s. This bit it set when someone writes | 559 | zsmalloc. |
548 | into a page just as regular dirty bit, but unlike the latter | ||
549 | it can be cleared by hands. | ||
550 | 560 | ||
551 | See Documentation/vm/soft-dirty.txt for more details. | 561 | config ZBUD |
562 | tristate "Low density storage for compressed pages" | ||
563 | default n | ||
564 | help | ||
565 | A special purpose allocator for storing compressed pages. | ||
566 | It is designed to store up to two compressed pages per physical | ||
567 | page. While this design limits storage density, it has simple and | ||
568 | deterministic reclaim properties that make it preferable to a higher | ||
569 | density approach when reclaim will be used. | ||
552 | 570 | ||
553 | config ZSMALLOC | 571 | config ZSMALLOC |
554 | tristate "Memory allocator for compressed pages" | 572 | tristate "Memory allocator for compressed pages" |
diff --git a/mm/Makefile b/mm/Makefile index 4064f3ec145e..632ae77e6070 100644 --- a/mm/Makefile +++ b/mm/Makefile | |||
@@ -59,6 +59,8 @@ obj-$(CONFIG_DEBUG_KMEMLEAK) += kmemleak.o | |||
59 | obj-$(CONFIG_DEBUG_KMEMLEAK_TEST) += kmemleak-test.o | 59 | obj-$(CONFIG_DEBUG_KMEMLEAK_TEST) += kmemleak-test.o |
60 | obj-$(CONFIG_CLEANCACHE) += cleancache.o | 60 | obj-$(CONFIG_CLEANCACHE) += cleancache.o |
61 | obj-$(CONFIG_MEMORY_ISOLATION) += page_isolation.o | 61 | obj-$(CONFIG_MEMORY_ISOLATION) += page_isolation.o |
62 | obj-$(CONFIG_ZPOOL) += zpool.o | ||
62 | obj-$(CONFIG_ZBUD) += zbud.o | 63 | obj-$(CONFIG_ZBUD) += zbud.o |
63 | obj-$(CONFIG_ZSMALLOC) += zsmalloc.o | 64 | obj-$(CONFIG_ZSMALLOC) += zsmalloc.o |
64 | obj-$(CONFIG_GENERIC_EARLY_IOREMAP) += early_ioremap.o | 65 | obj-$(CONFIG_GENERIC_EARLY_IOREMAP) += early_ioremap.o |
66 | obj-$(CONFIG_CMA) += cma.o | ||
diff --git a/mm/cma.c b/mm/cma.c new file mode 100644 index 000000000000..c17751c0dcaf --- /dev/null +++ b/mm/cma.c | |||
@@ -0,0 +1,335 @@ | |||
1 | /* | ||
2 | * Contiguous Memory Allocator | ||
3 | * | ||
4 | * Copyright (c) 2010-2011 by Samsung Electronics. | ||
5 | * Copyright IBM Corporation, 2013 | ||
6 | * Copyright LG Electronics Inc., 2014 | ||
7 | * Written by: | ||
8 | * Marek Szyprowski <m.szyprowski@samsung.com> | ||
9 | * Michal Nazarewicz <mina86@mina86.com> | ||
10 | * Aneesh Kumar K.V <aneesh.kumar@linux.vnet.ibm.com> | ||
11 | * Joonsoo Kim <iamjoonsoo.kim@lge.com> | ||
12 | * | ||
13 | * This program is free software; you can redistribute it and/or | ||
14 | * modify it under the terms of the GNU General Public License as | ||
15 | * published by the Free Software Foundation; either version 2 of the | ||
16 | * License or (at your optional) any later version of the license. | ||
17 | */ | ||
18 | |||
19 | #define pr_fmt(fmt) "cma: " fmt | ||
20 | |||
21 | #ifdef CONFIG_CMA_DEBUG | ||
22 | #ifndef DEBUG | ||
23 | # define DEBUG | ||
24 | #endif | ||
25 | #endif | ||
26 | |||
27 | #include <linux/memblock.h> | ||
28 | #include <linux/err.h> | ||
29 | #include <linux/mm.h> | ||
30 | #include <linux/mutex.h> | ||
31 | #include <linux/sizes.h> | ||
32 | #include <linux/slab.h> | ||
33 | #include <linux/log2.h> | ||
34 | #include <linux/cma.h> | ||
35 | |||
36 | struct cma { | ||
37 | unsigned long base_pfn; | ||
38 | unsigned long count; | ||
39 | unsigned long *bitmap; | ||
40 | unsigned int order_per_bit; /* Order of pages represented by one bit */ | ||
41 | struct mutex lock; | ||
42 | }; | ||
43 | |||
44 | static struct cma cma_areas[MAX_CMA_AREAS]; | ||
45 | static unsigned cma_area_count; | ||
46 | static DEFINE_MUTEX(cma_mutex); | ||
47 | |||
48 | phys_addr_t cma_get_base(struct cma *cma) | ||
49 | { | ||
50 | return PFN_PHYS(cma->base_pfn); | ||
51 | } | ||
52 | |||
53 | unsigned long cma_get_size(struct cma *cma) | ||
54 | { | ||
55 | return cma->count << PAGE_SHIFT; | ||
56 | } | ||
57 | |||
58 | static unsigned long cma_bitmap_aligned_mask(struct cma *cma, int align_order) | ||
59 | { | ||
60 | return (1UL << (align_order >> cma->order_per_bit)) - 1; | ||
61 | } | ||
62 | |||
63 | static unsigned long cma_bitmap_maxno(struct cma *cma) | ||
64 | { | ||
65 | return cma->count >> cma->order_per_bit; | ||
66 | } | ||
67 | |||
68 | static unsigned long cma_bitmap_pages_to_bits(struct cma *cma, | ||
69 | unsigned long pages) | ||
70 | { | ||
71 | return ALIGN(pages, 1UL << cma->order_per_bit) >> cma->order_per_bit; | ||
72 | } | ||
73 | |||
74 | static void cma_clear_bitmap(struct cma *cma, unsigned long pfn, int count) | ||
75 | { | ||
76 | unsigned long bitmap_no, bitmap_count; | ||
77 | |||
78 | bitmap_no = (pfn - cma->base_pfn) >> cma->order_per_bit; | ||
79 | bitmap_count = cma_bitmap_pages_to_bits(cma, count); | ||
80 | |||
81 | mutex_lock(&cma->lock); | ||
82 | bitmap_clear(cma->bitmap, bitmap_no, bitmap_count); | ||
83 | mutex_unlock(&cma->lock); | ||
84 | } | ||
85 | |||
86 | static int __init cma_activate_area(struct cma *cma) | ||
87 | { | ||
88 | int bitmap_size = BITS_TO_LONGS(cma_bitmap_maxno(cma)) * sizeof(long); | ||
89 | unsigned long base_pfn = cma->base_pfn, pfn = base_pfn; | ||
90 | unsigned i = cma->count >> pageblock_order; | ||
91 | struct zone *zone; | ||
92 | |||
93 | cma->bitmap = kzalloc(bitmap_size, GFP_KERNEL); | ||
94 | |||
95 | if (!cma->bitmap) | ||
96 | return -ENOMEM; | ||
97 | |||
98 | WARN_ON_ONCE(!pfn_valid(pfn)); | ||
99 | zone = page_zone(pfn_to_page(pfn)); | ||
100 | |||
101 | do { | ||
102 | unsigned j; | ||
103 | |||
104 | base_pfn = pfn; | ||
105 | for (j = pageblock_nr_pages; j; --j, pfn++) { | ||
106 | WARN_ON_ONCE(!pfn_valid(pfn)); | ||
107 | /* | ||
108 | * alloc_contig_range requires the pfn range | ||
109 | * specified to be in the same zone. Make this | ||
110 | * simple by forcing the entire CMA resv range | ||
111 | * to be in the same zone. | ||
112 | */ | ||
113 | if (page_zone(pfn_to_page(pfn)) != zone) | ||
114 | goto err; | ||
115 | } | ||
116 | init_cma_reserved_pageblock(pfn_to_page(base_pfn)); | ||
117 | } while (--i); | ||
118 | |||
119 | mutex_init(&cma->lock); | ||
120 | return 0; | ||
121 | |||
122 | err: | ||
123 | kfree(cma->bitmap); | ||
124 | return -EINVAL; | ||
125 | } | ||
126 | |||
127 | static int __init cma_init_reserved_areas(void) | ||
128 | { | ||
129 | int i; | ||
130 | |||
131 | for (i = 0; i < cma_area_count; i++) { | ||
132 | int ret = cma_activate_area(&cma_areas[i]); | ||
133 | |||
134 | if (ret) | ||
135 | return ret; | ||
136 | } | ||
137 | |||
138 | return 0; | ||
139 | } | ||
140 | core_initcall(cma_init_reserved_areas); | ||
141 | |||
142 | /** | ||
143 | * cma_declare_contiguous() - reserve custom contiguous area | ||
144 | * @base: Base address of the reserved area optional, use 0 for any | ||
145 | * @size: Size of the reserved area (in bytes), | ||
146 | * @limit: End address of the reserved memory (optional, 0 for any). | ||
147 | * @alignment: Alignment for the CMA area, should be power of 2 or zero | ||
148 | * @order_per_bit: Order of pages represented by one bit on bitmap. | ||
149 | * @fixed: hint about where to place the reserved area | ||
150 | * @res_cma: Pointer to store the created cma region. | ||
151 | * | ||
152 | * This function reserves memory from early allocator. It should be | ||
153 | * called by arch specific code once the early allocator (memblock or bootmem) | ||
154 | * has been activated and all other subsystems have already allocated/reserved | ||
155 | * memory. This function allows to create custom reserved areas. | ||
156 | * | ||
157 | * If @fixed is true, reserve contiguous area at exactly @base. If false, | ||
158 | * reserve in range from @base to @limit. | ||
159 | */ | ||
160 | int __init cma_declare_contiguous(phys_addr_t base, | ||
161 | phys_addr_t size, phys_addr_t limit, | ||
162 | phys_addr_t alignment, unsigned int order_per_bit, | ||
163 | bool fixed, struct cma **res_cma) | ||
164 | { | ||
165 | struct cma *cma; | ||
166 | int ret = 0; | ||
167 | |||
168 | pr_debug("%s(size %lx, base %08lx, limit %08lx alignment %08lx)\n", | ||
169 | __func__, (unsigned long)size, (unsigned long)base, | ||
170 | (unsigned long)limit, (unsigned long)alignment); | ||
171 | |||
172 | if (cma_area_count == ARRAY_SIZE(cma_areas)) { | ||
173 | pr_err("Not enough slots for CMA reserved regions!\n"); | ||
174 | return -ENOSPC; | ||
175 | } | ||
176 | |||
177 | if (!size) | ||
178 | return -EINVAL; | ||
179 | |||
180 | if (alignment && !is_power_of_2(alignment)) | ||
181 | return -EINVAL; | ||
182 | |||
183 | /* | ||
184 | * Sanitise input arguments. | ||
185 | * Pages both ends in CMA area could be merged into adjacent unmovable | ||
186 | * migratetype page by page allocator's buddy algorithm. In the case, | ||
187 | * you couldn't get a contiguous memory, which is not what we want. | ||
188 | */ | ||
189 | alignment = max(alignment, | ||
190 | (phys_addr_t)PAGE_SIZE << max(MAX_ORDER - 1, pageblock_order)); | ||
191 | base = ALIGN(base, alignment); | ||
192 | size = ALIGN(size, alignment); | ||
193 | limit &= ~(alignment - 1); | ||
194 | |||
195 | /* size should be aligned with order_per_bit */ | ||
196 | if (!IS_ALIGNED(size >> PAGE_SHIFT, 1 << order_per_bit)) | ||
197 | return -EINVAL; | ||
198 | |||
199 | /* Reserve memory */ | ||
200 | if (base && fixed) { | ||
201 | if (memblock_is_region_reserved(base, size) || | ||
202 | memblock_reserve(base, size) < 0) { | ||
203 | ret = -EBUSY; | ||
204 | goto err; | ||
205 | } | ||
206 | } else { | ||
207 | phys_addr_t addr = memblock_alloc_range(size, alignment, base, | ||
208 | limit); | ||
209 | if (!addr) { | ||
210 | ret = -ENOMEM; | ||
211 | goto err; | ||
212 | } else { | ||
213 | base = addr; | ||
214 | } | ||
215 | } | ||
216 | |||
217 | /* | ||
218 | * Each reserved area must be initialised later, when more kernel | ||
219 | * subsystems (like slab allocator) are available. | ||
220 | */ | ||
221 | cma = &cma_areas[cma_area_count]; | ||
222 | cma->base_pfn = PFN_DOWN(base); | ||
223 | cma->count = size >> PAGE_SHIFT; | ||
224 | cma->order_per_bit = order_per_bit; | ||
225 | *res_cma = cma; | ||
226 | cma_area_count++; | ||
227 | |||
228 | pr_info("Reserved %ld MiB at %08lx\n", (unsigned long)size / SZ_1M, | ||
229 | (unsigned long)base); | ||
230 | return 0; | ||
231 | |||
232 | err: | ||
233 | pr_err("Failed to reserve %ld MiB\n", (unsigned long)size / SZ_1M); | ||
234 | return ret; | ||
235 | } | ||
236 | |||
237 | /** | ||
238 | * cma_alloc() - allocate pages from contiguous area | ||
239 | * @cma: Contiguous memory region for which the allocation is performed. | ||
240 | * @count: Requested number of pages. | ||
241 | * @align: Requested alignment of pages (in PAGE_SIZE order). | ||
242 | * | ||
243 | * This function allocates part of contiguous memory on specific | ||
244 | * contiguous memory area. | ||
245 | */ | ||
246 | struct page *cma_alloc(struct cma *cma, int count, unsigned int align) | ||
247 | { | ||
248 | unsigned long mask, pfn, start = 0; | ||
249 | unsigned long bitmap_maxno, bitmap_no, bitmap_count; | ||
250 | struct page *page = NULL; | ||
251 | int ret; | ||
252 | |||
253 | if (!cma || !cma->count) | ||
254 | return NULL; | ||
255 | |||
256 | pr_debug("%s(cma %p, count %d, align %d)\n", __func__, (void *)cma, | ||
257 | count, align); | ||
258 | |||
259 | if (!count) | ||
260 | return NULL; | ||
261 | |||
262 | mask = cma_bitmap_aligned_mask(cma, align); | ||
263 | bitmap_maxno = cma_bitmap_maxno(cma); | ||
264 | bitmap_count = cma_bitmap_pages_to_bits(cma, count); | ||
265 | |||
266 | for (;;) { | ||
267 | mutex_lock(&cma->lock); | ||
268 | bitmap_no = bitmap_find_next_zero_area(cma->bitmap, | ||
269 | bitmap_maxno, start, bitmap_count, mask); | ||
270 | if (bitmap_no >= bitmap_maxno) { | ||
271 | mutex_unlock(&cma->lock); | ||
272 | break; | ||
273 | } | ||
274 | bitmap_set(cma->bitmap, bitmap_no, bitmap_count); | ||
275 | /* | ||
276 | * It's safe to drop the lock here. We've marked this region for | ||
277 | * our exclusive use. If the migration fails we will take the | ||
278 | * lock again and unmark it. | ||
279 | */ | ||
280 | mutex_unlock(&cma->lock); | ||
281 | |||
282 | pfn = cma->base_pfn + (bitmap_no << cma->order_per_bit); | ||
283 | mutex_lock(&cma_mutex); | ||
284 | ret = alloc_contig_range(pfn, pfn + count, MIGRATE_CMA); | ||
285 | mutex_unlock(&cma_mutex); | ||
286 | if (ret == 0) { | ||
287 | page = pfn_to_page(pfn); | ||
288 | break; | ||
289 | } | ||
290 | |||
291 | cma_clear_bitmap(cma, pfn, count); | ||
292 | if (ret != -EBUSY) | ||
293 | break; | ||
294 | |||
295 | pr_debug("%s(): memory range at %p is busy, retrying\n", | ||
296 | __func__, pfn_to_page(pfn)); | ||
297 | /* try again with a bit different memory target */ | ||
298 | start = bitmap_no + mask + 1; | ||
299 | } | ||
300 | |||
301 | pr_debug("%s(): returned %p\n", __func__, page); | ||
302 | return page; | ||
303 | } | ||
304 | |||
305 | /** | ||
306 | * cma_release() - release allocated pages | ||
307 | * @cma: Contiguous memory region for which the allocation is performed. | ||
308 | * @pages: Allocated pages. | ||
309 | * @count: Number of allocated pages. | ||
310 | * | ||
311 | * This function releases memory allocated by alloc_cma(). | ||
312 | * It returns false when provided pages do not belong to contiguous area and | ||
313 | * true otherwise. | ||
314 | */ | ||
315 | bool cma_release(struct cma *cma, struct page *pages, int count) | ||
316 | { | ||
317 | unsigned long pfn; | ||
318 | |||
319 | if (!cma || !pages) | ||
320 | return false; | ||
321 | |||
322 | pr_debug("%s(page %p)\n", __func__, (void *)pages); | ||
323 | |||
324 | pfn = page_to_pfn(pages); | ||
325 | |||
326 | if (pfn < cma->base_pfn || pfn >= cma->base_pfn + cma->count) | ||
327 | return false; | ||
328 | |||
329 | VM_BUG_ON(pfn + count > cma->base_pfn + cma->count); | ||
330 | |||
331 | free_contig_range(pfn, count); | ||
332 | cma_clear_bitmap(cma, pfn, count); | ||
333 | |||
334 | return true; | ||
335 | } | ||
diff --git a/mm/filemap.c b/mm/filemap.c index 65d44fd88c78..af19a6b079f5 100644 --- a/mm/filemap.c +++ b/mm/filemap.c | |||
@@ -808,6 +808,17 @@ int __lock_page_killable(struct page *page) | |||
808 | } | 808 | } |
809 | EXPORT_SYMBOL_GPL(__lock_page_killable); | 809 | EXPORT_SYMBOL_GPL(__lock_page_killable); |
810 | 810 | ||
811 | /* | ||
812 | * Return values: | ||
813 | * 1 - page is locked; mmap_sem is still held. | ||
814 | * 0 - page is not locked. | ||
815 | * mmap_sem has been released (up_read()), unless flags had both | ||
816 | * FAULT_FLAG_ALLOW_RETRY and FAULT_FLAG_RETRY_NOWAIT set, in | ||
817 | * which case mmap_sem is still held. | ||
818 | * | ||
819 | * If neither ALLOW_RETRY nor KILLABLE are set, will always return 1 | ||
820 | * with the page locked and the mmap_sem unperturbed. | ||
821 | */ | ||
811 | int __lock_page_or_retry(struct page *page, struct mm_struct *mm, | 822 | int __lock_page_or_retry(struct page *page, struct mm_struct *mm, |
812 | unsigned int flags) | 823 | unsigned int flags) |
813 | { | 824 | { |
@@ -1091,9 +1102,9 @@ no_page: | |||
1091 | if (WARN_ON_ONCE(!(fgp_flags & FGP_LOCK))) | 1102 | if (WARN_ON_ONCE(!(fgp_flags & FGP_LOCK))) |
1092 | fgp_flags |= FGP_LOCK; | 1103 | fgp_flags |= FGP_LOCK; |
1093 | 1104 | ||
1094 | /* Init accessed so avoit atomic mark_page_accessed later */ | 1105 | /* Init accessed so avoid atomic mark_page_accessed later */ |
1095 | if (fgp_flags & FGP_ACCESSED) | 1106 | if (fgp_flags & FGP_ACCESSED) |
1096 | init_page_accessed(page); | 1107 | __SetPageReferenced(page); |
1097 | 1108 | ||
1098 | err = add_to_page_cache_lru(page, mapping, offset, radix_gfp_mask); | 1109 | err = add_to_page_cache_lru(page, mapping, offset, radix_gfp_mask); |
1099 | if (unlikely(err)) { | 1110 | if (unlikely(err)) { |
@@ -1827,6 +1838,18 @@ static void do_async_mmap_readahead(struct vm_area_struct *vma, | |||
1827 | * The goto's are kind of ugly, but this streamlines the normal case of having | 1838 | * The goto's are kind of ugly, but this streamlines the normal case of having |
1828 | * it in the page cache, and handles the special cases reasonably without | 1839 | * it in the page cache, and handles the special cases reasonably without |
1829 | * having a lot of duplicated code. | 1840 | * having a lot of duplicated code. |
1841 | * | ||
1842 | * vma->vm_mm->mmap_sem must be held on entry. | ||
1843 | * | ||
1844 | * If our return value has VM_FAULT_RETRY set, it's because | ||
1845 | * lock_page_or_retry() returned 0. | ||
1846 | * The mmap_sem has usually been released in this case. | ||
1847 | * See __lock_page_or_retry() for the exception. | ||
1848 | * | ||
1849 | * If our return value does not have VM_FAULT_RETRY set, the mmap_sem | ||
1850 | * has not been released. | ||
1851 | * | ||
1852 | * We never return with VM_FAULT_RETRY and a bit from VM_FAULT_ERROR set. | ||
1830 | */ | 1853 | */ |
1831 | int filemap_fault(struct vm_area_struct *vma, struct vm_fault *vmf) | 1854 | int filemap_fault(struct vm_area_struct *vma, struct vm_fault *vmf) |
1832 | { | 1855 | { |
@@ -258,6 +258,11 @@ unmap: | |||
258 | return ret; | 258 | return ret; |
259 | } | 259 | } |
260 | 260 | ||
261 | /* | ||
262 | * mmap_sem must be held on entry. If @nonblocking != NULL and | ||
263 | * *@flags does not include FOLL_NOWAIT, the mmap_sem may be released. | ||
264 | * If it is, *@nonblocking will be set to 0 and -EBUSY returned. | ||
265 | */ | ||
261 | static int faultin_page(struct task_struct *tsk, struct vm_area_struct *vma, | 266 | static int faultin_page(struct task_struct *tsk, struct vm_area_struct *vma, |
262 | unsigned long address, unsigned int *flags, int *nonblocking) | 267 | unsigned long address, unsigned int *flags, int *nonblocking) |
263 | { | 268 | { |
@@ -373,7 +378,7 @@ static int check_vma_flags(struct vm_area_struct *vma, unsigned long gup_flags) | |||
373 | * with a put_page() call when it is finished with. vmas will only | 378 | * with a put_page() call when it is finished with. vmas will only |
374 | * remain valid while mmap_sem is held. | 379 | * remain valid while mmap_sem is held. |
375 | * | 380 | * |
376 | * Must be called with mmap_sem held for read or write. | 381 | * Must be called with mmap_sem held. It may be released. See below. |
377 | * | 382 | * |
378 | * __get_user_pages walks a process's page tables and takes a reference to | 383 | * __get_user_pages walks a process's page tables and takes a reference to |
379 | * each struct page that each user address corresponds to at a given | 384 | * each struct page that each user address corresponds to at a given |
@@ -396,7 +401,14 @@ static int check_vma_flags(struct vm_area_struct *vma, unsigned long gup_flags) | |||
396 | * | 401 | * |
397 | * If @nonblocking != NULL, __get_user_pages will not wait for disk IO | 402 | * If @nonblocking != NULL, __get_user_pages will not wait for disk IO |
398 | * or mmap_sem contention, and if waiting is needed to pin all pages, | 403 | * or mmap_sem contention, and if waiting is needed to pin all pages, |
399 | * *@nonblocking will be set to 0. | 404 | * *@nonblocking will be set to 0. Further, if @gup_flags does not |
405 | * include FOLL_NOWAIT, the mmap_sem will be released via up_read() in | ||
406 | * this case. | ||
407 | * | ||
408 | * A caller using such a combination of @nonblocking and @gup_flags | ||
409 | * must therefore hold the mmap_sem for reading only, and recognize | ||
410 | * when it's been released. Otherwise, it must be held for either | ||
411 | * reading or writing and will not be released. | ||
400 | * | 412 | * |
401 | * In most cases, get_user_pages or get_user_pages_fast should be used | 413 | * In most cases, get_user_pages or get_user_pages_fast should be used |
402 | * instead of __get_user_pages. __get_user_pages should be used only if | 414 | * instead of __get_user_pages. __get_user_pages should be used only if |
@@ -528,7 +540,7 @@ EXPORT_SYMBOL(__get_user_pages); | |||
528 | * such architectures, gup() will not be enough to make a subsequent access | 540 | * such architectures, gup() will not be enough to make a subsequent access |
529 | * succeed. | 541 | * succeed. |
530 | * | 542 | * |
531 | * This should be called with the mm_sem held for read. | 543 | * This has the same semantics wrt the @mm->mmap_sem as does filemap_fault(). |
532 | */ | 544 | */ |
533 | int fixup_user_fault(struct task_struct *tsk, struct mm_struct *mm, | 545 | int fixup_user_fault(struct task_struct *tsk, struct mm_struct *mm, |
534 | unsigned long address, unsigned int fault_flags) | 546 | unsigned long address, unsigned int fault_flags) |
diff --git a/mm/highmem.c b/mm/highmem.c index b32b70cdaed6..123bcd3ed4f2 100644 --- a/mm/highmem.c +++ b/mm/highmem.c | |||
@@ -44,6 +44,66 @@ DEFINE_PER_CPU(int, __kmap_atomic_idx); | |||
44 | */ | 44 | */ |
45 | #ifdef CONFIG_HIGHMEM | 45 | #ifdef CONFIG_HIGHMEM |
46 | 46 | ||
47 | /* | ||
48 | * Architecture with aliasing data cache may define the following family of | ||
49 | * helper functions in its asm/highmem.h to control cache color of virtual | ||
50 | * addresses where physical memory pages are mapped by kmap. | ||
51 | */ | ||
52 | #ifndef get_pkmap_color | ||
53 | |||
54 | /* | ||
55 | * Determine color of virtual address where the page should be mapped. | ||
56 | */ | ||
57 | static inline unsigned int get_pkmap_color(struct page *page) | ||
58 | { | ||
59 | return 0; | ||
60 | } | ||
61 | #define get_pkmap_color get_pkmap_color | ||
62 | |||
63 | /* | ||
64 | * Get next index for mapping inside PKMAP region for page with given color. | ||
65 | */ | ||
66 | static inline unsigned int get_next_pkmap_nr(unsigned int color) | ||
67 | { | ||
68 | static unsigned int last_pkmap_nr; | ||
69 | |||
70 | last_pkmap_nr = (last_pkmap_nr + 1) & LAST_PKMAP_MASK; | ||
71 | return last_pkmap_nr; | ||
72 | } | ||
73 | |||
74 | /* | ||
75 | * Determine if page index inside PKMAP region (pkmap_nr) of given color | ||
76 | * has wrapped around PKMAP region end. When this happens an attempt to | ||
77 | * flush all unused PKMAP slots is made. | ||
78 | */ | ||
79 | static inline int no_more_pkmaps(unsigned int pkmap_nr, unsigned int color) | ||
80 | { | ||
81 | return pkmap_nr == 0; | ||
82 | } | ||
83 | |||
84 | /* | ||
85 | * Get the number of PKMAP entries of the given color. If no free slot is | ||
86 | * found after checking that many entries, kmap will sleep waiting for | ||
87 | * someone to call kunmap and free PKMAP slot. | ||
88 | */ | ||
89 | static inline int get_pkmap_entries_count(unsigned int color) | ||
90 | { | ||
91 | return LAST_PKMAP; | ||
92 | } | ||
93 | |||
94 | /* | ||
95 | * Get head of a wait queue for PKMAP entries of the given color. | ||
96 | * Wait queues for different mapping colors should be independent to avoid | ||
97 | * unnecessary wakeups caused by freeing of slots of other colors. | ||
98 | */ | ||
99 | static inline wait_queue_head_t *get_pkmap_wait_queue_head(unsigned int color) | ||
100 | { | ||
101 | static DECLARE_WAIT_QUEUE_HEAD(pkmap_map_wait); | ||
102 | |||
103 | return &pkmap_map_wait; | ||
104 | } | ||
105 | #endif | ||
106 | |||
47 | unsigned long totalhigh_pages __read_mostly; | 107 | unsigned long totalhigh_pages __read_mostly; |
48 | EXPORT_SYMBOL(totalhigh_pages); | 108 | EXPORT_SYMBOL(totalhigh_pages); |
49 | 109 | ||
@@ -68,13 +128,10 @@ unsigned int nr_free_highpages (void) | |||
68 | } | 128 | } |
69 | 129 | ||
70 | static int pkmap_count[LAST_PKMAP]; | 130 | static int pkmap_count[LAST_PKMAP]; |
71 | static unsigned int last_pkmap_nr; | ||
72 | static __cacheline_aligned_in_smp DEFINE_SPINLOCK(kmap_lock); | 131 | static __cacheline_aligned_in_smp DEFINE_SPINLOCK(kmap_lock); |
73 | 132 | ||
74 | pte_t * pkmap_page_table; | 133 | pte_t * pkmap_page_table; |
75 | 134 | ||
76 | static DECLARE_WAIT_QUEUE_HEAD(pkmap_map_wait); | ||
77 | |||
78 | /* | 135 | /* |
79 | * Most architectures have no use for kmap_high_get(), so let's abstract | 136 | * Most architectures have no use for kmap_high_get(), so let's abstract |
80 | * the disabling of IRQ out of the locking in that case to save on a | 137 | * the disabling of IRQ out of the locking in that case to save on a |
@@ -161,15 +218,17 @@ static inline unsigned long map_new_virtual(struct page *page) | |||
161 | { | 218 | { |
162 | unsigned long vaddr; | 219 | unsigned long vaddr; |
163 | int count; | 220 | int count; |
221 | unsigned int last_pkmap_nr; | ||
222 | unsigned int color = get_pkmap_color(page); | ||
164 | 223 | ||
165 | start: | 224 | start: |
166 | count = LAST_PKMAP; | 225 | count = get_pkmap_entries_count(color); |
167 | /* Find an empty entry */ | 226 | /* Find an empty entry */ |
168 | for (;;) { | 227 | for (;;) { |
169 | last_pkmap_nr = (last_pkmap_nr + 1) & LAST_PKMAP_MASK; | 228 | last_pkmap_nr = get_next_pkmap_nr(color); |
170 | if (!last_pkmap_nr) { | 229 | if (no_more_pkmaps(last_pkmap_nr, color)) { |
171 | flush_all_zero_pkmaps(); | 230 | flush_all_zero_pkmaps(); |
172 | count = LAST_PKMAP; | 231 | count = get_pkmap_entries_count(color); |
173 | } | 232 | } |
174 | if (!pkmap_count[last_pkmap_nr]) | 233 | if (!pkmap_count[last_pkmap_nr]) |
175 | break; /* Found a usable entry */ | 234 | break; /* Found a usable entry */ |
@@ -181,12 +240,14 @@ start: | |||
181 | */ | 240 | */ |
182 | { | 241 | { |
183 | DECLARE_WAITQUEUE(wait, current); | 242 | DECLARE_WAITQUEUE(wait, current); |
243 | wait_queue_head_t *pkmap_map_wait = | ||
244 | get_pkmap_wait_queue_head(color); | ||
184 | 245 | ||
185 | __set_current_state(TASK_UNINTERRUPTIBLE); | 246 | __set_current_state(TASK_UNINTERRUPTIBLE); |
186 | add_wait_queue(&pkmap_map_wait, &wait); | 247 | add_wait_queue(pkmap_map_wait, &wait); |
187 | unlock_kmap(); | 248 | unlock_kmap(); |
188 | schedule(); | 249 | schedule(); |
189 | remove_wait_queue(&pkmap_map_wait, &wait); | 250 | remove_wait_queue(pkmap_map_wait, &wait); |
190 | lock_kmap(); | 251 | lock_kmap(); |
191 | 252 | ||
192 | /* Somebody else might have mapped it while we slept */ | 253 | /* Somebody else might have mapped it while we slept */ |
@@ -274,6 +335,8 @@ void kunmap_high(struct page *page) | |||
274 | unsigned long nr; | 335 | unsigned long nr; |
275 | unsigned long flags; | 336 | unsigned long flags; |
276 | int need_wakeup; | 337 | int need_wakeup; |
338 | unsigned int color = get_pkmap_color(page); | ||
339 | wait_queue_head_t *pkmap_map_wait; | ||
277 | 340 | ||
278 | lock_kmap_any(flags); | 341 | lock_kmap_any(flags); |
279 | vaddr = (unsigned long)page_address(page); | 342 | vaddr = (unsigned long)page_address(page); |
@@ -299,13 +362,14 @@ void kunmap_high(struct page *page) | |||
299 | * no need for the wait-queue-head's lock. Simply | 362 | * no need for the wait-queue-head's lock. Simply |
300 | * test if the queue is empty. | 363 | * test if the queue is empty. |
301 | */ | 364 | */ |
302 | need_wakeup = waitqueue_active(&pkmap_map_wait); | 365 | pkmap_map_wait = get_pkmap_wait_queue_head(color); |
366 | need_wakeup = waitqueue_active(pkmap_map_wait); | ||
303 | } | 367 | } |
304 | unlock_kmap_any(flags); | 368 | unlock_kmap_any(flags); |
305 | 369 | ||
306 | /* do wake-up, if needed, race-free outside of the spin lock */ | 370 | /* do wake-up, if needed, race-free outside of the spin lock */ |
307 | if (need_wakeup) | 371 | if (need_wakeup) |
308 | wake_up(&pkmap_map_wait); | 372 | wake_up(pkmap_map_wait); |
309 | } | 373 | } |
310 | 374 | ||
311 | EXPORT_SYMBOL(kunmap_high); | 375 | EXPORT_SYMBOL(kunmap_high); |
diff --git a/mm/huge_memory.c b/mm/huge_memory.c index 33514d88fef9..3630d577e987 100644 --- a/mm/huge_memory.c +++ b/mm/huge_memory.c | |||
@@ -827,7 +827,7 @@ int do_huge_pmd_anonymous_page(struct mm_struct *mm, struct vm_area_struct *vma, | |||
827 | count_vm_event(THP_FAULT_FALLBACK); | 827 | count_vm_event(THP_FAULT_FALLBACK); |
828 | return VM_FAULT_FALLBACK; | 828 | return VM_FAULT_FALLBACK; |
829 | } | 829 | } |
830 | if (unlikely(mem_cgroup_charge_anon(page, mm, GFP_KERNEL))) { | 830 | if (unlikely(mem_cgroup_charge_anon(page, mm, GFP_TRANSHUGE))) { |
831 | put_page(page); | 831 | put_page(page); |
832 | count_vm_event(THP_FAULT_FALLBACK); | 832 | count_vm_event(THP_FAULT_FALLBACK); |
833 | return VM_FAULT_FALLBACK; | 833 | return VM_FAULT_FALLBACK; |
@@ -1132,7 +1132,7 @@ alloc: | |||
1132 | goto out; | 1132 | goto out; |
1133 | } | 1133 | } |
1134 | 1134 | ||
1135 | if (unlikely(mem_cgroup_charge_anon(new_page, mm, GFP_KERNEL))) { | 1135 | if (unlikely(mem_cgroup_charge_anon(new_page, mm, GFP_TRANSHUGE))) { |
1136 | put_page(new_page); | 1136 | put_page(new_page); |
1137 | if (page) { | 1137 | if (page) { |
1138 | split_huge_page(page); | 1138 | split_huge_page(page); |
@@ -1681,7 +1681,7 @@ static void __split_huge_page_refcount(struct page *page, | |||
1681 | &page_tail->_count); | 1681 | &page_tail->_count); |
1682 | 1682 | ||
1683 | /* after clearing PageTail the gup refcount can be released */ | 1683 | /* after clearing PageTail the gup refcount can be released */ |
1684 | smp_mb(); | 1684 | smp_mb__after_atomic(); |
1685 | 1685 | ||
1686 | /* | 1686 | /* |
1687 | * retain hwpoison flag of the poisoned tail page: | 1687 | * retain hwpoison flag of the poisoned tail page: |
@@ -1775,6 +1775,8 @@ static int __split_huge_page_map(struct page *page, | |||
1775 | if (pmd) { | 1775 | if (pmd) { |
1776 | pgtable = pgtable_trans_huge_withdraw(mm, pmd); | 1776 | pgtable = pgtable_trans_huge_withdraw(mm, pmd); |
1777 | pmd_populate(mm, &_pmd, pgtable); | 1777 | pmd_populate(mm, &_pmd, pgtable); |
1778 | if (pmd_write(*pmd)) | ||
1779 | BUG_ON(page_mapcount(page) != 1); | ||
1778 | 1780 | ||
1779 | haddr = address; | 1781 | haddr = address; |
1780 | for (i = 0; i < HPAGE_PMD_NR; i++, haddr += PAGE_SIZE) { | 1782 | for (i = 0; i < HPAGE_PMD_NR; i++, haddr += PAGE_SIZE) { |
@@ -1784,8 +1786,6 @@ static int __split_huge_page_map(struct page *page, | |||
1784 | entry = maybe_mkwrite(pte_mkdirty(entry), vma); | 1786 | entry = maybe_mkwrite(pte_mkdirty(entry), vma); |
1785 | if (!pmd_write(*pmd)) | 1787 | if (!pmd_write(*pmd)) |
1786 | entry = pte_wrprotect(entry); | 1788 | entry = pte_wrprotect(entry); |
1787 | else | ||
1788 | BUG_ON(page_mapcount(page) != 1); | ||
1789 | if (!pmd_young(*pmd)) | 1789 | if (!pmd_young(*pmd)) |
1790 | entry = pte_mkold(entry); | 1790 | entry = pte_mkold(entry); |
1791 | if (pmd_numa(*pmd)) | 1791 | if (pmd_numa(*pmd)) |
@@ -2233,6 +2233,30 @@ static void khugepaged_alloc_sleep(void) | |||
2233 | 2233 | ||
2234 | static int khugepaged_node_load[MAX_NUMNODES]; | 2234 | static int khugepaged_node_load[MAX_NUMNODES]; |
2235 | 2235 | ||
2236 | static bool khugepaged_scan_abort(int nid) | ||
2237 | { | ||
2238 | int i; | ||
2239 | |||
2240 | /* | ||
2241 | * If zone_reclaim_mode is disabled, then no extra effort is made to | ||
2242 | * allocate memory locally. | ||
2243 | */ | ||
2244 | if (!zone_reclaim_mode) | ||
2245 | return false; | ||
2246 | |||
2247 | /* If there is a count for this node already, it must be acceptable */ | ||
2248 | if (khugepaged_node_load[nid]) | ||
2249 | return false; | ||
2250 | |||
2251 | for (i = 0; i < MAX_NUMNODES; i++) { | ||
2252 | if (!khugepaged_node_load[i]) | ||
2253 | continue; | ||
2254 | if (node_distance(nid, i) > RECLAIM_DISTANCE) | ||
2255 | return true; | ||
2256 | } | ||
2257 | return false; | ||
2258 | } | ||
2259 | |||
2236 | #ifdef CONFIG_NUMA | 2260 | #ifdef CONFIG_NUMA |
2237 | static int khugepaged_find_target_node(void) | 2261 | static int khugepaged_find_target_node(void) |
2238 | { | 2262 | { |
@@ -2399,7 +2423,7 @@ static void collapse_huge_page(struct mm_struct *mm, | |||
2399 | if (!new_page) | 2423 | if (!new_page) |
2400 | return; | 2424 | return; |
2401 | 2425 | ||
2402 | if (unlikely(mem_cgroup_charge_anon(new_page, mm, GFP_KERNEL))) | 2426 | if (unlikely(mem_cgroup_charge_anon(new_page, mm, GFP_TRANSHUGE))) |
2403 | return; | 2427 | return; |
2404 | 2428 | ||
2405 | /* | 2429 | /* |
@@ -2545,6 +2569,8 @@ static int khugepaged_scan_pmd(struct mm_struct *mm, | |||
2545 | * hit record. | 2569 | * hit record. |
2546 | */ | 2570 | */ |
2547 | node = page_to_nid(page); | 2571 | node = page_to_nid(page); |
2572 | if (khugepaged_scan_abort(node)) | ||
2573 | goto out_unmap; | ||
2548 | khugepaged_node_load[node]++; | 2574 | khugepaged_node_load[node]++; |
2549 | VM_BUG_ON_PAGE(PageCompound(page), page); | 2575 | VM_BUG_ON_PAGE(PageCompound(page), page); |
2550 | if (!PageLRU(page) || PageLocked(page) || !PageAnon(page)) | 2576 | if (!PageLRU(page) || PageLocked(page) || !PageAnon(page)) |
diff --git a/mm/hugetlb.c b/mm/hugetlb.c index 7a0a73d2fcff..eeceeeb09019 100644 --- a/mm/hugetlb.c +++ b/mm/hugetlb.c | |||
@@ -35,7 +35,6 @@ | |||
35 | #include <linux/node.h> | 35 | #include <linux/node.h> |
36 | #include "internal.h" | 36 | #include "internal.h" |
37 | 37 | ||
38 | const unsigned long hugetlb_zero = 0, hugetlb_infinity = ~0UL; | ||
39 | unsigned long hugepages_treat_as_movable; | 38 | unsigned long hugepages_treat_as_movable; |
40 | 39 | ||
41 | int hugetlb_max_hstate __read_mostly; | 40 | int hugetlb_max_hstate __read_mostly; |
@@ -1089,6 +1088,9 @@ void dissolve_free_huge_pages(unsigned long start_pfn, unsigned long end_pfn) | |||
1089 | unsigned long pfn; | 1088 | unsigned long pfn; |
1090 | struct hstate *h; | 1089 | struct hstate *h; |
1091 | 1090 | ||
1091 | if (!hugepages_supported()) | ||
1092 | return; | ||
1093 | |||
1092 | /* Set scan step to minimum hugepage size */ | 1094 | /* Set scan step to minimum hugepage size */ |
1093 | for_each_hstate(h) | 1095 | for_each_hstate(h) |
1094 | if (order > huge_page_order(h)) | 1096 | if (order > huge_page_order(h)) |
@@ -1734,21 +1736,13 @@ static ssize_t nr_hugepages_show_common(struct kobject *kobj, | |||
1734 | return sprintf(buf, "%lu\n", nr_huge_pages); | 1736 | return sprintf(buf, "%lu\n", nr_huge_pages); |
1735 | } | 1737 | } |
1736 | 1738 | ||
1737 | static ssize_t nr_hugepages_store_common(bool obey_mempolicy, | 1739 | static ssize_t __nr_hugepages_store_common(bool obey_mempolicy, |
1738 | struct kobject *kobj, struct kobj_attribute *attr, | 1740 | struct hstate *h, int nid, |
1739 | const char *buf, size_t len) | 1741 | unsigned long count, size_t len) |
1740 | { | 1742 | { |
1741 | int err; | 1743 | int err; |
1742 | int nid; | ||
1743 | unsigned long count; | ||
1744 | struct hstate *h; | ||
1745 | NODEMASK_ALLOC(nodemask_t, nodes_allowed, GFP_KERNEL | __GFP_NORETRY); | 1744 | NODEMASK_ALLOC(nodemask_t, nodes_allowed, GFP_KERNEL | __GFP_NORETRY); |
1746 | 1745 | ||
1747 | err = kstrtoul(buf, 10, &count); | ||
1748 | if (err) | ||
1749 | goto out; | ||
1750 | |||
1751 | h = kobj_to_hstate(kobj, &nid); | ||
1752 | if (hstate_is_gigantic(h) && !gigantic_page_supported()) { | 1746 | if (hstate_is_gigantic(h) && !gigantic_page_supported()) { |
1753 | err = -EINVAL; | 1747 | err = -EINVAL; |
1754 | goto out; | 1748 | goto out; |
@@ -1784,6 +1778,23 @@ out: | |||
1784 | return err; | 1778 | return err; |
1785 | } | 1779 | } |
1786 | 1780 | ||
1781 | static ssize_t nr_hugepages_store_common(bool obey_mempolicy, | ||
1782 | struct kobject *kobj, const char *buf, | ||
1783 | size_t len) | ||
1784 | { | ||
1785 | struct hstate *h; | ||
1786 | unsigned long count; | ||
1787 | int nid; | ||
1788 | int err; | ||
1789 | |||
1790 | err = kstrtoul(buf, 10, &count); | ||
1791 | if (err) | ||
1792 | return err; | ||
1793 | |||
1794 | h = kobj_to_hstate(kobj, &nid); | ||
1795 | return __nr_hugepages_store_common(obey_mempolicy, h, nid, count, len); | ||
1796 | } | ||
1797 | |||
1787 | static ssize_t nr_hugepages_show(struct kobject *kobj, | 1798 | static ssize_t nr_hugepages_show(struct kobject *kobj, |
1788 | struct kobj_attribute *attr, char *buf) | 1799 | struct kobj_attribute *attr, char *buf) |
1789 | { | 1800 | { |
@@ -1793,7 +1804,7 @@ static ssize_t nr_hugepages_show(struct kobject *kobj, | |||
1793 | static ssize_t nr_hugepages_store(struct kobject *kobj, | 1804 | static ssize_t nr_hugepages_store(struct kobject *kobj, |
1794 | struct kobj_attribute *attr, const char *buf, size_t len) | 1805 | struct kobj_attribute *attr, const char *buf, size_t len) |
1795 | { | 1806 | { |
1796 | return nr_hugepages_store_common(false, kobj, attr, buf, len); | 1807 | return nr_hugepages_store_common(false, kobj, buf, len); |
1797 | } | 1808 | } |
1798 | HSTATE_ATTR(nr_hugepages); | 1809 | HSTATE_ATTR(nr_hugepages); |
1799 | 1810 | ||
@@ -1812,7 +1823,7 @@ static ssize_t nr_hugepages_mempolicy_show(struct kobject *kobj, | |||
1812 | static ssize_t nr_hugepages_mempolicy_store(struct kobject *kobj, | 1823 | static ssize_t nr_hugepages_mempolicy_store(struct kobject *kobj, |
1813 | struct kobj_attribute *attr, const char *buf, size_t len) | 1824 | struct kobj_attribute *attr, const char *buf, size_t len) |
1814 | { | 1825 | { |
1815 | return nr_hugepages_store_common(true, kobj, attr, buf, len); | 1826 | return nr_hugepages_store_common(true, kobj, buf, len); |
1816 | } | 1827 | } |
1817 | HSTATE_ATTR(nr_hugepages_mempolicy); | 1828 | HSTATE_ATTR(nr_hugepages_mempolicy); |
1818 | #endif | 1829 | #endif |
@@ -2248,36 +2259,21 @@ static int hugetlb_sysctl_handler_common(bool obey_mempolicy, | |||
2248 | void __user *buffer, size_t *length, loff_t *ppos) | 2259 | void __user *buffer, size_t *length, loff_t *ppos) |
2249 | { | 2260 | { |
2250 | struct hstate *h = &default_hstate; | 2261 | struct hstate *h = &default_hstate; |
2251 | unsigned long tmp; | 2262 | unsigned long tmp = h->max_huge_pages; |
2252 | int ret; | 2263 | int ret; |
2253 | 2264 | ||
2254 | if (!hugepages_supported()) | 2265 | if (!hugepages_supported()) |
2255 | return -ENOTSUPP; | 2266 | return -ENOTSUPP; |
2256 | 2267 | ||
2257 | tmp = h->max_huge_pages; | ||
2258 | |||
2259 | if (write && hstate_is_gigantic(h) && !gigantic_page_supported()) | ||
2260 | return -EINVAL; | ||
2261 | |||
2262 | table->data = &tmp; | 2268 | table->data = &tmp; |
2263 | table->maxlen = sizeof(unsigned long); | 2269 | table->maxlen = sizeof(unsigned long); |
2264 | ret = proc_doulongvec_minmax(table, write, buffer, length, ppos); | 2270 | ret = proc_doulongvec_minmax(table, write, buffer, length, ppos); |
2265 | if (ret) | 2271 | if (ret) |
2266 | goto out; | 2272 | goto out; |
2267 | 2273 | ||
2268 | if (write) { | 2274 | if (write) |
2269 | NODEMASK_ALLOC(nodemask_t, nodes_allowed, | 2275 | ret = __nr_hugepages_store_common(obey_mempolicy, h, |
2270 | GFP_KERNEL | __GFP_NORETRY); | 2276 | NUMA_NO_NODE, tmp, *length); |
2271 | if (!(obey_mempolicy && | ||
2272 | init_nodemask_of_mempolicy(nodes_allowed))) { | ||
2273 | NODEMASK_FREE(nodes_allowed); | ||
2274 | nodes_allowed = &node_states[N_MEMORY]; | ||
2275 | } | ||
2276 | h->max_huge_pages = set_max_huge_pages(h, tmp, nodes_allowed); | ||
2277 | |||
2278 | if (nodes_allowed != &node_states[N_MEMORY]) | ||
2279 | NODEMASK_FREE(nodes_allowed); | ||
2280 | } | ||
2281 | out: | 2277 | out: |
2282 | return ret; | 2278 | return ret; |
2283 | } | 2279 | } |
@@ -2754,8 +2750,8 @@ void unmap_hugepage_range(struct vm_area_struct *vma, unsigned long start, | |||
2754 | * from other VMAs and let the children be SIGKILLed if they are faulting the | 2750 | * from other VMAs and let the children be SIGKILLed if they are faulting the |
2755 | * same region. | 2751 | * same region. |
2756 | */ | 2752 | */ |
2757 | static int unmap_ref_private(struct mm_struct *mm, struct vm_area_struct *vma, | 2753 | static void unmap_ref_private(struct mm_struct *mm, struct vm_area_struct *vma, |
2758 | struct page *page, unsigned long address) | 2754 | struct page *page, unsigned long address) |
2759 | { | 2755 | { |
2760 | struct hstate *h = hstate_vma(vma); | 2756 | struct hstate *h = hstate_vma(vma); |
2761 | struct vm_area_struct *iter_vma; | 2757 | struct vm_area_struct *iter_vma; |
@@ -2794,8 +2790,6 @@ static int unmap_ref_private(struct mm_struct *mm, struct vm_area_struct *vma, | |||
2794 | address + huge_page_size(h), page); | 2790 | address + huge_page_size(h), page); |
2795 | } | 2791 | } |
2796 | mutex_unlock(&mapping->i_mmap_mutex); | 2792 | mutex_unlock(&mapping->i_mmap_mutex); |
2797 | |||
2798 | return 1; | ||
2799 | } | 2793 | } |
2800 | 2794 | ||
2801 | /* | 2795 | /* |
@@ -2810,7 +2804,7 @@ static int hugetlb_cow(struct mm_struct *mm, struct vm_area_struct *vma, | |||
2810 | { | 2804 | { |
2811 | struct hstate *h = hstate_vma(vma); | 2805 | struct hstate *h = hstate_vma(vma); |
2812 | struct page *old_page, *new_page; | 2806 | struct page *old_page, *new_page; |
2813 | int outside_reserve = 0; | 2807 | int ret = 0, outside_reserve = 0; |
2814 | unsigned long mmun_start; /* For mmu_notifiers */ | 2808 | unsigned long mmun_start; /* For mmu_notifiers */ |
2815 | unsigned long mmun_end; /* For mmu_notifiers */ | 2809 | unsigned long mmun_end; /* For mmu_notifiers */ |
2816 | 2810 | ||
@@ -2840,14 +2834,14 @@ retry_avoidcopy: | |||
2840 | 2834 | ||
2841 | page_cache_get(old_page); | 2835 | page_cache_get(old_page); |
2842 | 2836 | ||
2843 | /* Drop page table lock as buddy allocator may be called */ | 2837 | /* |
2838 | * Drop page table lock as buddy allocator may be called. It will | ||
2839 | * be acquired again before returning to the caller, as expected. | ||
2840 | */ | ||
2844 | spin_unlock(ptl); | 2841 | spin_unlock(ptl); |
2845 | new_page = alloc_huge_page(vma, address, outside_reserve); | 2842 | new_page = alloc_huge_page(vma, address, outside_reserve); |
2846 | 2843 | ||
2847 | if (IS_ERR(new_page)) { | 2844 | if (IS_ERR(new_page)) { |
2848 | long err = PTR_ERR(new_page); | ||
2849 | page_cache_release(old_page); | ||
2850 | |||
2851 | /* | 2845 | /* |
2852 | * If a process owning a MAP_PRIVATE mapping fails to COW, | 2846 | * If a process owning a MAP_PRIVATE mapping fails to COW, |
2853 | * it is due to references held by a child and an insufficient | 2847 | * it is due to references held by a child and an insufficient |
@@ -2856,29 +2850,25 @@ retry_avoidcopy: | |||
2856 | * may get SIGKILLed if it later faults. | 2850 | * may get SIGKILLed if it later faults. |
2857 | */ | 2851 | */ |
2858 | if (outside_reserve) { | 2852 | if (outside_reserve) { |
2853 | page_cache_release(old_page); | ||
2859 | BUG_ON(huge_pte_none(pte)); | 2854 | BUG_ON(huge_pte_none(pte)); |
2860 | if (unmap_ref_private(mm, vma, old_page, address)) { | 2855 | unmap_ref_private(mm, vma, old_page, address); |
2861 | BUG_ON(huge_pte_none(pte)); | 2856 | BUG_ON(huge_pte_none(pte)); |
2862 | spin_lock(ptl); | 2857 | spin_lock(ptl); |
2863 | ptep = huge_pte_offset(mm, address & huge_page_mask(h)); | 2858 | ptep = huge_pte_offset(mm, address & huge_page_mask(h)); |
2864 | if (likely(ptep && | 2859 | if (likely(ptep && |
2865 | pte_same(huge_ptep_get(ptep), pte))) | 2860 | pte_same(huge_ptep_get(ptep), pte))) |
2866 | goto retry_avoidcopy; | 2861 | goto retry_avoidcopy; |
2867 | /* | 2862 | /* |
2868 | * race occurs while re-acquiring page table | 2863 | * race occurs while re-acquiring page table |
2869 | * lock, and our job is done. | 2864 | * lock, and our job is done. |
2870 | */ | 2865 | */ |
2871 | return 0; | 2866 | return 0; |
2872 | } | ||
2873 | WARN_ON_ONCE(1); | ||
2874 | } | 2867 | } |
2875 | 2868 | ||
2876 | /* Caller expects lock to be held */ | 2869 | ret = (PTR_ERR(new_page) == -ENOMEM) ? |
2877 | spin_lock(ptl); | 2870 | VM_FAULT_OOM : VM_FAULT_SIGBUS; |
2878 | if (err == -ENOMEM) | 2871 | goto out_release_old; |
2879 | return VM_FAULT_OOM; | ||
2880 | else | ||
2881 | return VM_FAULT_SIGBUS; | ||
2882 | } | 2872 | } |
2883 | 2873 | ||
2884 | /* | 2874 | /* |
@@ -2886,11 +2876,8 @@ retry_avoidcopy: | |||
2886 | * anon_vma prepared. | 2876 | * anon_vma prepared. |
2887 | */ | 2877 | */ |
2888 | if (unlikely(anon_vma_prepare(vma))) { | 2878 | if (unlikely(anon_vma_prepare(vma))) { |
2889 | page_cache_release(new_page); | 2879 | ret = VM_FAULT_OOM; |
2890 | page_cache_release(old_page); | 2880 | goto out_release_all; |
2891 | /* Caller expects lock to be held */ | ||
2892 | spin_lock(ptl); | ||
2893 | return VM_FAULT_OOM; | ||
2894 | } | 2881 | } |
2895 | 2882 | ||
2896 | copy_user_huge_page(new_page, old_page, address, vma, | 2883 | copy_user_huge_page(new_page, old_page, address, vma, |
@@ -2900,6 +2887,7 @@ retry_avoidcopy: | |||
2900 | mmun_start = address & huge_page_mask(h); | 2887 | mmun_start = address & huge_page_mask(h); |
2901 | mmun_end = mmun_start + huge_page_size(h); | 2888 | mmun_end = mmun_start + huge_page_size(h); |
2902 | mmu_notifier_invalidate_range_start(mm, mmun_start, mmun_end); | 2889 | mmu_notifier_invalidate_range_start(mm, mmun_start, mmun_end); |
2890 | |||
2903 | /* | 2891 | /* |
2904 | * Retake the page table lock to check for racing updates | 2892 | * Retake the page table lock to check for racing updates |
2905 | * before the page tables are altered | 2893 | * before the page tables are altered |
@@ -2920,12 +2908,13 @@ retry_avoidcopy: | |||
2920 | } | 2908 | } |
2921 | spin_unlock(ptl); | 2909 | spin_unlock(ptl); |
2922 | mmu_notifier_invalidate_range_end(mm, mmun_start, mmun_end); | 2910 | mmu_notifier_invalidate_range_end(mm, mmun_start, mmun_end); |
2911 | out_release_all: | ||
2923 | page_cache_release(new_page); | 2912 | page_cache_release(new_page); |
2913 | out_release_old: | ||
2924 | page_cache_release(old_page); | 2914 | page_cache_release(old_page); |
2925 | 2915 | ||
2926 | /* Caller expects lock to be held */ | 2916 | spin_lock(ptl); /* Caller expects lock to be held */ |
2927 | spin_lock(ptl); | 2917 | return ret; |
2928 | return 0; | ||
2929 | } | 2918 | } |
2930 | 2919 | ||
2931 | /* Return the pagecache page at a given address within a VMA */ | 2920 | /* Return the pagecache page at a given address within a VMA */ |
diff --git a/mm/hwpoison-inject.c b/mm/hwpoison-inject.c index 95487c71cad5..329caf56df22 100644 --- a/mm/hwpoison-inject.c +++ b/mm/hwpoison-inject.c | |||
@@ -72,8 +72,7 @@ DEFINE_SIMPLE_ATTRIBUTE(unpoison_fops, NULL, hwpoison_unpoison, "%lli\n"); | |||
72 | 72 | ||
73 | static void pfn_inject_exit(void) | 73 | static void pfn_inject_exit(void) |
74 | { | 74 | { |
75 | if (hwpoison_dir) | 75 | debugfs_remove_recursive(hwpoison_dir); |
76 | debugfs_remove_recursive(hwpoison_dir); | ||
77 | } | 76 | } |
78 | 77 | ||
79 | static int pfn_inject_init(void) | 78 | static int pfn_inject_init(void) |
diff --git a/mm/internal.h b/mm/internal.h index 7f22a11fcc66..a1b651b11c5f 100644 --- a/mm/internal.h +++ b/mm/internal.h | |||
@@ -247,7 +247,7 @@ static inline void mlock_migrate_page(struct page *new, struct page *old) { } | |||
247 | static inline struct page *mem_map_offset(struct page *base, int offset) | 247 | static inline struct page *mem_map_offset(struct page *base, int offset) |
248 | { | 248 | { |
249 | if (unlikely(offset >= MAX_ORDER_NR_PAGES)) | 249 | if (unlikely(offset >= MAX_ORDER_NR_PAGES)) |
250 | return pfn_to_page(page_to_pfn(base) + offset); | 250 | return nth_page(base, offset); |
251 | return base + offset; | 251 | return base + offset; |
252 | } | 252 | } |
253 | 253 | ||
diff --git a/mm/madvise.c b/mm/madvise.c index a402f8fdc68e..0938b30da4ab 100644 --- a/mm/madvise.c +++ b/mm/madvise.c | |||
@@ -292,9 +292,6 @@ static long madvise_dontneed(struct vm_area_struct *vma, | |||
292 | /* | 292 | /* |
293 | * Application wants to free up the pages and associated backing store. | 293 | * Application wants to free up the pages and associated backing store. |
294 | * This is effectively punching a hole into the middle of a file. | 294 | * This is effectively punching a hole into the middle of a file. |
295 | * | ||
296 | * NOTE: Currently, only shmfs/tmpfs is supported for this operation. | ||
297 | * Other filesystems return -ENOSYS. | ||
298 | */ | 295 | */ |
299 | static long madvise_remove(struct vm_area_struct *vma, | 296 | static long madvise_remove(struct vm_area_struct *vma, |
300 | struct vm_area_struct **prev, | 297 | struct vm_area_struct **prev, |
diff --git a/mm/memcontrol.c b/mm/memcontrol.c index f009a14918d2..90dc501eaf3f 100644 --- a/mm/memcontrol.c +++ b/mm/memcontrol.c | |||
@@ -2551,55 +2551,72 @@ static int memcg_cpu_hotplug_callback(struct notifier_block *nb, | |||
2551 | return NOTIFY_OK; | 2551 | return NOTIFY_OK; |
2552 | } | 2552 | } |
2553 | 2553 | ||
2554 | 2554 | /** | |
2555 | /* See mem_cgroup_try_charge() for details */ | 2555 | * mem_cgroup_try_charge - try charging a memcg |
2556 | enum { | 2556 | * @memcg: memcg to charge |
2557 | CHARGE_OK, /* success */ | 2557 | * @nr_pages: number of pages to charge |
2558 | CHARGE_RETRY, /* need to retry but retry is not bad */ | 2558 | * |
2559 | CHARGE_NOMEM, /* we can't do more. return -ENOMEM */ | 2559 | * Returns 0 if @memcg was charged successfully, -EINTR if the charge |
2560 | CHARGE_WOULDBLOCK, /* GFP_WAIT wasn't set and no enough res. */ | 2560 | * was bypassed to root_mem_cgroup, and -ENOMEM if the charge failed. |
2561 | }; | 2561 | */ |
2562 | 2562 | static int mem_cgroup_try_charge(struct mem_cgroup *memcg, | |
2563 | static int mem_cgroup_do_charge(struct mem_cgroup *memcg, gfp_t gfp_mask, | 2563 | gfp_t gfp_mask, |
2564 | unsigned int nr_pages, unsigned int min_pages, | 2564 | unsigned int nr_pages) |
2565 | bool invoke_oom) | ||
2566 | { | 2565 | { |
2567 | unsigned long csize = nr_pages * PAGE_SIZE; | 2566 | unsigned int batch = max(CHARGE_BATCH, nr_pages); |
2567 | int nr_retries = MEM_CGROUP_RECLAIM_RETRIES; | ||
2568 | struct mem_cgroup *mem_over_limit; | 2568 | struct mem_cgroup *mem_over_limit; |
2569 | struct res_counter *fail_res; | 2569 | struct res_counter *fail_res; |
2570 | unsigned long nr_reclaimed; | ||
2570 | unsigned long flags = 0; | 2571 | unsigned long flags = 0; |
2571 | int ret; | 2572 | unsigned long long size; |
2573 | int ret = 0; | ||
2572 | 2574 | ||
2573 | ret = res_counter_charge(&memcg->res, csize, &fail_res); | 2575 | retry: |
2576 | if (consume_stock(memcg, nr_pages)) | ||
2577 | goto done; | ||
2574 | 2578 | ||
2575 | if (likely(!ret)) { | 2579 | size = batch * PAGE_SIZE; |
2580 | if (!res_counter_charge(&memcg->res, size, &fail_res)) { | ||
2576 | if (!do_swap_account) | 2581 | if (!do_swap_account) |
2577 | return CHARGE_OK; | 2582 | goto done_restock; |
2578 | ret = res_counter_charge(&memcg->memsw, csize, &fail_res); | 2583 | if (!res_counter_charge(&memcg->memsw, size, &fail_res)) |
2579 | if (likely(!ret)) | 2584 | goto done_restock; |
2580 | return CHARGE_OK; | 2585 | res_counter_uncharge(&memcg->res, size); |
2581 | |||
2582 | res_counter_uncharge(&memcg->res, csize); | ||
2583 | mem_over_limit = mem_cgroup_from_res_counter(fail_res, memsw); | 2586 | mem_over_limit = mem_cgroup_from_res_counter(fail_res, memsw); |
2584 | flags |= MEM_CGROUP_RECLAIM_NOSWAP; | 2587 | flags |= MEM_CGROUP_RECLAIM_NOSWAP; |
2585 | } else | 2588 | } else |
2586 | mem_over_limit = mem_cgroup_from_res_counter(fail_res, res); | 2589 | mem_over_limit = mem_cgroup_from_res_counter(fail_res, res); |
2590 | |||
2591 | if (batch > nr_pages) { | ||
2592 | batch = nr_pages; | ||
2593 | goto retry; | ||
2594 | } | ||
2595 | |||
2587 | /* | 2596 | /* |
2588 | * Never reclaim on behalf of optional batching, retry with a | 2597 | * Unlike in global OOM situations, memcg is not in a physical |
2589 | * single page instead. | 2598 | * memory shortage. Allow dying and OOM-killed tasks to |
2599 | * bypass the last charges so that they can exit quickly and | ||
2600 | * free their memory. | ||
2590 | */ | 2601 | */ |
2591 | if (nr_pages > min_pages) | 2602 | if (unlikely(test_thread_flag(TIF_MEMDIE) || |
2592 | return CHARGE_RETRY; | 2603 | fatal_signal_pending(current) || |
2604 | current->flags & PF_EXITING)) | ||
2605 | goto bypass; | ||
2606 | |||
2607 | if (unlikely(task_in_memcg_oom(current))) | ||
2608 | goto nomem; | ||
2593 | 2609 | ||
2594 | if (!(gfp_mask & __GFP_WAIT)) | 2610 | if (!(gfp_mask & __GFP_WAIT)) |
2595 | return CHARGE_WOULDBLOCK; | 2611 | goto nomem; |
2596 | 2612 | ||
2597 | if (gfp_mask & __GFP_NORETRY) | 2613 | nr_reclaimed = mem_cgroup_reclaim(mem_over_limit, gfp_mask, flags); |
2598 | return CHARGE_NOMEM; | ||
2599 | 2614 | ||
2600 | ret = mem_cgroup_reclaim(mem_over_limit, gfp_mask, flags); | ||
2601 | if (mem_cgroup_margin(mem_over_limit) >= nr_pages) | 2615 | if (mem_cgroup_margin(mem_over_limit) >= nr_pages) |
2602 | return CHARGE_RETRY; | 2616 | goto retry; |
2617 | |||
2618 | if (gfp_mask & __GFP_NORETRY) | ||
2619 | goto nomem; | ||
2603 | /* | 2620 | /* |
2604 | * Even though the limit is exceeded at this point, reclaim | 2621 | * Even though the limit is exceeded at this point, reclaim |
2605 | * may have been able to free some pages. Retry the charge | 2622 | * may have been able to free some pages. Retry the charge |
@@ -2609,96 +2626,38 @@ static int mem_cgroup_do_charge(struct mem_cgroup *memcg, gfp_t gfp_mask, | |||
2609 | * unlikely to succeed so close to the limit, and we fall back | 2626 | * unlikely to succeed so close to the limit, and we fall back |
2610 | * to regular pages anyway in case of failure. | 2627 | * to regular pages anyway in case of failure. |
2611 | */ | 2628 | */ |
2612 | if (nr_pages <= (1 << PAGE_ALLOC_COSTLY_ORDER) && ret) | 2629 | if (nr_reclaimed && nr_pages <= (1 << PAGE_ALLOC_COSTLY_ORDER)) |
2613 | return CHARGE_RETRY; | 2630 | goto retry; |
2614 | |||
2615 | /* | 2631 | /* |
2616 | * At task move, charge accounts can be doubly counted. So, it's | 2632 | * At task move, charge accounts can be doubly counted. So, it's |
2617 | * better to wait until the end of task_move if something is going on. | 2633 | * better to wait until the end of task_move if something is going on. |
2618 | */ | 2634 | */ |
2619 | if (mem_cgroup_wait_acct_move(mem_over_limit)) | 2635 | if (mem_cgroup_wait_acct_move(mem_over_limit)) |
2620 | return CHARGE_RETRY; | 2636 | goto retry; |
2621 | |||
2622 | if (invoke_oom) | ||
2623 | mem_cgroup_oom(mem_over_limit, gfp_mask, get_order(csize)); | ||
2624 | |||
2625 | return CHARGE_NOMEM; | ||
2626 | } | ||
2627 | |||
2628 | /** | ||
2629 | * mem_cgroup_try_charge - try charging a memcg | ||
2630 | * @memcg: memcg to charge | ||
2631 | * @nr_pages: number of pages to charge | ||
2632 | * @oom: trigger OOM if reclaim fails | ||
2633 | * | ||
2634 | * Returns 0 if @memcg was charged successfully, -EINTR if the charge | ||
2635 | * was bypassed to root_mem_cgroup, and -ENOMEM if the charge failed. | ||
2636 | */ | ||
2637 | static int mem_cgroup_try_charge(struct mem_cgroup *memcg, | ||
2638 | gfp_t gfp_mask, | ||
2639 | unsigned int nr_pages, | ||
2640 | bool oom) | ||
2641 | { | ||
2642 | unsigned int batch = max(CHARGE_BATCH, nr_pages); | ||
2643 | int nr_oom_retries = MEM_CGROUP_RECLAIM_RETRIES; | ||
2644 | int ret; | ||
2645 | |||
2646 | if (mem_cgroup_is_root(memcg)) | ||
2647 | goto done; | ||
2648 | /* | ||
2649 | * Unlike in global OOM situations, memcg is not in a physical | ||
2650 | * memory shortage. Allow dying and OOM-killed tasks to | ||
2651 | * bypass the last charges so that they can exit quickly and | ||
2652 | * free their memory. | ||
2653 | */ | ||
2654 | if (unlikely(test_thread_flag(TIF_MEMDIE) || | ||
2655 | fatal_signal_pending(current) || | ||
2656 | current->flags & PF_EXITING)) | ||
2657 | goto bypass; | ||
2658 | 2637 | ||
2659 | if (unlikely(task_in_memcg_oom(current))) | 2638 | if (nr_retries--) |
2660 | goto nomem; | 2639 | goto retry; |
2661 | 2640 | ||
2662 | if (gfp_mask & __GFP_NOFAIL) | 2641 | if (gfp_mask & __GFP_NOFAIL) |
2663 | oom = false; | 2642 | goto bypass; |
2664 | again: | ||
2665 | if (consume_stock(memcg, nr_pages)) | ||
2666 | goto done; | ||
2667 | |||
2668 | do { | ||
2669 | bool invoke_oom = oom && !nr_oom_retries; | ||
2670 | |||
2671 | /* If killed, bypass charge */ | ||
2672 | if (fatal_signal_pending(current)) | ||
2673 | goto bypass; | ||
2674 | 2643 | ||
2675 | ret = mem_cgroup_do_charge(memcg, gfp_mask, batch, | 2644 | if (fatal_signal_pending(current)) |
2676 | nr_pages, invoke_oom); | 2645 | goto bypass; |
2677 | switch (ret) { | ||
2678 | case CHARGE_OK: | ||
2679 | break; | ||
2680 | case CHARGE_RETRY: /* not in OOM situation but retry */ | ||
2681 | batch = nr_pages; | ||
2682 | goto again; | ||
2683 | case CHARGE_WOULDBLOCK: /* !__GFP_WAIT */ | ||
2684 | goto nomem; | ||
2685 | case CHARGE_NOMEM: /* OOM routine works */ | ||
2686 | if (!oom || invoke_oom) | ||
2687 | goto nomem; | ||
2688 | nr_oom_retries--; | ||
2689 | break; | ||
2690 | } | ||
2691 | } while (ret != CHARGE_OK); | ||
2692 | 2646 | ||
2693 | if (batch > nr_pages) | 2647 | mem_cgroup_oom(mem_over_limit, gfp_mask, get_order(nr_pages)); |
2694 | refill_stock(memcg, batch - nr_pages); | ||
2695 | done: | ||
2696 | return 0; | ||
2697 | nomem: | 2648 | nomem: |
2698 | if (!(gfp_mask & __GFP_NOFAIL)) | 2649 | if (!(gfp_mask & __GFP_NOFAIL)) |
2699 | return -ENOMEM; | 2650 | return -ENOMEM; |
2700 | bypass: | 2651 | bypass: |
2701 | return -EINTR; | 2652 | memcg = root_mem_cgroup; |
2653 | ret = -EINTR; | ||
2654 | goto retry; | ||
2655 | |||
2656 | done_restock: | ||
2657 | if (batch > nr_pages) | ||
2658 | refill_stock(memcg, batch - nr_pages); | ||
2659 | done: | ||
2660 | return ret; | ||
2702 | } | 2661 | } |
2703 | 2662 | ||
2704 | /** | 2663 | /** |
@@ -2712,15 +2671,14 @@ bypass: | |||
2712 | */ | 2671 | */ |
2713 | static struct mem_cgroup *mem_cgroup_try_charge_mm(struct mm_struct *mm, | 2672 | static struct mem_cgroup *mem_cgroup_try_charge_mm(struct mm_struct *mm, |
2714 | gfp_t gfp_mask, | 2673 | gfp_t gfp_mask, |
2715 | unsigned int nr_pages, | 2674 | unsigned int nr_pages) |
2716 | bool oom) | ||
2717 | 2675 | ||
2718 | { | 2676 | { |
2719 | struct mem_cgroup *memcg; | 2677 | struct mem_cgroup *memcg; |
2720 | int ret; | 2678 | int ret; |
2721 | 2679 | ||
2722 | memcg = get_mem_cgroup_from_mm(mm); | 2680 | memcg = get_mem_cgroup_from_mm(mm); |
2723 | ret = mem_cgroup_try_charge(memcg, gfp_mask, nr_pages, oom); | 2681 | ret = mem_cgroup_try_charge(memcg, gfp_mask, nr_pages); |
2724 | css_put(&memcg->css); | 2682 | css_put(&memcg->css); |
2725 | if (ret == -EINTR) | 2683 | if (ret == -EINTR) |
2726 | memcg = root_mem_cgroup; | 2684 | memcg = root_mem_cgroup; |
@@ -2738,13 +2696,11 @@ static struct mem_cgroup *mem_cgroup_try_charge_mm(struct mm_struct *mm, | |||
2738 | static void __mem_cgroup_cancel_charge(struct mem_cgroup *memcg, | 2696 | static void __mem_cgroup_cancel_charge(struct mem_cgroup *memcg, |
2739 | unsigned int nr_pages) | 2697 | unsigned int nr_pages) |
2740 | { | 2698 | { |
2741 | if (!mem_cgroup_is_root(memcg)) { | 2699 | unsigned long bytes = nr_pages * PAGE_SIZE; |
2742 | unsigned long bytes = nr_pages * PAGE_SIZE; | ||
2743 | 2700 | ||
2744 | res_counter_uncharge(&memcg->res, bytes); | 2701 | res_counter_uncharge(&memcg->res, bytes); |
2745 | if (do_swap_account) | 2702 | if (do_swap_account) |
2746 | res_counter_uncharge(&memcg->memsw, bytes); | 2703 | res_counter_uncharge(&memcg->memsw, bytes); |
2747 | } | ||
2748 | } | 2704 | } |
2749 | 2705 | ||
2750 | /* | 2706 | /* |
@@ -2756,9 +2712,6 @@ static void __mem_cgroup_cancel_local_charge(struct mem_cgroup *memcg, | |||
2756 | { | 2712 | { |
2757 | unsigned long bytes = nr_pages * PAGE_SIZE; | 2713 | unsigned long bytes = nr_pages * PAGE_SIZE; |
2758 | 2714 | ||
2759 | if (mem_cgroup_is_root(memcg)) | ||
2760 | return; | ||
2761 | |||
2762 | res_counter_uncharge_until(&memcg->res, memcg->res.parent, bytes); | 2715 | res_counter_uncharge_until(&memcg->res, memcg->res.parent, bytes); |
2763 | if (do_swap_account) | 2716 | if (do_swap_account) |
2764 | res_counter_uncharge_until(&memcg->memsw, | 2717 | res_counter_uncharge_until(&memcg->memsw, |
@@ -2842,14 +2795,6 @@ static void __mem_cgroup_commit_charge(struct mem_cgroup *memcg, | |||
2842 | } | 2795 | } |
2843 | 2796 | ||
2844 | pc->mem_cgroup = memcg; | 2797 | pc->mem_cgroup = memcg; |
2845 | /* | ||
2846 | * We access a page_cgroup asynchronously without lock_page_cgroup(). | ||
2847 | * Especially when a page_cgroup is taken from a page, pc->mem_cgroup | ||
2848 | * is accessed after testing USED bit. To make pc->mem_cgroup visible | ||
2849 | * before USED bit, we need memory barrier here. | ||
2850 | * See mem_cgroup_add_lru_list(), etc. | ||
2851 | */ | ||
2852 | smp_wmb(); | ||
2853 | SetPageCgroupUsed(pc); | 2798 | SetPageCgroupUsed(pc); |
2854 | 2799 | ||
2855 | if (lrucare) { | 2800 | if (lrucare) { |
@@ -2937,8 +2882,7 @@ static int memcg_charge_kmem(struct mem_cgroup *memcg, gfp_t gfp, u64 size) | |||
2937 | if (ret) | 2882 | if (ret) |
2938 | return ret; | 2883 | return ret; |
2939 | 2884 | ||
2940 | ret = mem_cgroup_try_charge(memcg, gfp, size >> PAGE_SHIFT, | 2885 | ret = mem_cgroup_try_charge(memcg, gfp, size >> PAGE_SHIFT); |
2941 | oom_gfp_allowed(gfp)); | ||
2942 | if (ret == -EINTR) { | 2886 | if (ret == -EINTR) { |
2943 | /* | 2887 | /* |
2944 | * mem_cgroup_try_charge() chosed to bypass to root due to | 2888 | * mem_cgroup_try_charge() chosed to bypass to root due to |
@@ -3463,12 +3407,13 @@ void __memcg_kmem_commit_charge(struct page *page, struct mem_cgroup *memcg, | |||
3463 | memcg_uncharge_kmem(memcg, PAGE_SIZE << order); | 3407 | memcg_uncharge_kmem(memcg, PAGE_SIZE << order); |
3464 | return; | 3408 | return; |
3465 | } | 3409 | } |
3466 | 3410 | /* | |
3411 | * The page is freshly allocated and not visible to any | ||
3412 | * outside callers yet. Set up pc non-atomically. | ||
3413 | */ | ||
3467 | pc = lookup_page_cgroup(page); | 3414 | pc = lookup_page_cgroup(page); |
3468 | lock_page_cgroup(pc); | ||
3469 | pc->mem_cgroup = memcg; | 3415 | pc->mem_cgroup = memcg; |
3470 | SetPageCgroupUsed(pc); | 3416 | pc->flags = PCG_USED; |
3471 | unlock_page_cgroup(pc); | ||
3472 | } | 3417 | } |
3473 | 3418 | ||
3474 | void __memcg_kmem_uncharge_pages(struct page *page, int order) | 3419 | void __memcg_kmem_uncharge_pages(struct page *page, int order) |
@@ -3478,19 +3423,11 @@ void __memcg_kmem_uncharge_pages(struct page *page, int order) | |||
3478 | 3423 | ||
3479 | 3424 | ||
3480 | pc = lookup_page_cgroup(page); | 3425 | pc = lookup_page_cgroup(page); |
3481 | /* | ||
3482 | * Fast unlocked return. Theoretically might have changed, have to | ||
3483 | * check again after locking. | ||
3484 | */ | ||
3485 | if (!PageCgroupUsed(pc)) | 3426 | if (!PageCgroupUsed(pc)) |
3486 | return; | 3427 | return; |
3487 | 3428 | ||
3488 | lock_page_cgroup(pc); | 3429 | memcg = pc->mem_cgroup; |
3489 | if (PageCgroupUsed(pc)) { | 3430 | pc->flags = 0; |
3490 | memcg = pc->mem_cgroup; | ||
3491 | ClearPageCgroupUsed(pc); | ||
3492 | } | ||
3493 | unlock_page_cgroup(pc); | ||
3494 | 3431 | ||
3495 | /* | 3432 | /* |
3496 | * We trust that only if there is a memcg associated with the page, it | 3433 | * We trust that only if there is a memcg associated with the page, it |
@@ -3531,7 +3468,6 @@ void mem_cgroup_split_huge_fixup(struct page *head) | |||
3531 | for (i = 1; i < HPAGE_PMD_NR; i++) { | 3468 | for (i = 1; i < HPAGE_PMD_NR; i++) { |
3532 | pc = head_pc + i; | 3469 | pc = head_pc + i; |
3533 | pc->mem_cgroup = memcg; | 3470 | pc->mem_cgroup = memcg; |
3534 | smp_wmb();/* see __commit_charge() */ | ||
3535 | pc->flags = head_pc->flags & ~PCGF_NOCOPY_AT_SPLIT; | 3471 | pc->flags = head_pc->flags & ~PCGF_NOCOPY_AT_SPLIT; |
3536 | } | 3472 | } |
3537 | __this_cpu_sub(memcg->stat->count[MEM_CGROUP_STAT_RSS_HUGE], | 3473 | __this_cpu_sub(memcg->stat->count[MEM_CGROUP_STAT_RSS_HUGE], |
@@ -3687,7 +3623,6 @@ int mem_cgroup_charge_anon(struct page *page, | |||
3687 | { | 3623 | { |
3688 | unsigned int nr_pages = 1; | 3624 | unsigned int nr_pages = 1; |
3689 | struct mem_cgroup *memcg; | 3625 | struct mem_cgroup *memcg; |
3690 | bool oom = true; | ||
3691 | 3626 | ||
3692 | if (mem_cgroup_disabled()) | 3627 | if (mem_cgroup_disabled()) |
3693 | return 0; | 3628 | return 0; |
@@ -3699,14 +3634,9 @@ int mem_cgroup_charge_anon(struct page *page, | |||
3699 | if (PageTransHuge(page)) { | 3634 | if (PageTransHuge(page)) { |
3700 | nr_pages <<= compound_order(page); | 3635 | nr_pages <<= compound_order(page); |
3701 | VM_BUG_ON_PAGE(!PageTransHuge(page), page); | 3636 | VM_BUG_ON_PAGE(!PageTransHuge(page), page); |
3702 | /* | ||
3703 | * Never OOM-kill a process for a huge page. The | ||
3704 | * fault handler will fall back to regular pages. | ||
3705 | */ | ||
3706 | oom = false; | ||
3707 | } | 3637 | } |
3708 | 3638 | ||
3709 | memcg = mem_cgroup_try_charge_mm(mm, gfp_mask, nr_pages, oom); | 3639 | memcg = mem_cgroup_try_charge_mm(mm, gfp_mask, nr_pages); |
3710 | if (!memcg) | 3640 | if (!memcg) |
3711 | return -ENOMEM; | 3641 | return -ENOMEM; |
3712 | __mem_cgroup_commit_charge(memcg, page, nr_pages, | 3642 | __mem_cgroup_commit_charge(memcg, page, nr_pages, |
@@ -3743,7 +3673,7 @@ static int __mem_cgroup_try_charge_swapin(struct mm_struct *mm, | |||
3743 | memcg = try_get_mem_cgroup_from_page(page); | 3673 | memcg = try_get_mem_cgroup_from_page(page); |
3744 | if (!memcg) | 3674 | if (!memcg) |
3745 | memcg = get_mem_cgroup_from_mm(mm); | 3675 | memcg = get_mem_cgroup_from_mm(mm); |
3746 | ret = mem_cgroup_try_charge(memcg, mask, 1, true); | 3676 | ret = mem_cgroup_try_charge(memcg, mask, 1); |
3747 | css_put(&memcg->css); | 3677 | css_put(&memcg->css); |
3748 | if (ret == -EINTR) | 3678 | if (ret == -EINTR) |
3749 | memcg = root_mem_cgroup; | 3679 | memcg = root_mem_cgroup; |
@@ -3770,7 +3700,7 @@ int mem_cgroup_try_charge_swapin(struct mm_struct *mm, struct page *page, | |||
3770 | if (!PageSwapCache(page)) { | 3700 | if (!PageSwapCache(page)) { |
3771 | struct mem_cgroup *memcg; | 3701 | struct mem_cgroup *memcg; |
3772 | 3702 | ||
3773 | memcg = mem_cgroup_try_charge_mm(mm, gfp_mask, 1, true); | 3703 | memcg = mem_cgroup_try_charge_mm(mm, gfp_mask, 1); |
3774 | if (!memcg) | 3704 | if (!memcg) |
3775 | return -ENOMEM; | 3705 | return -ENOMEM; |
3776 | *memcgp = memcg; | 3706 | *memcgp = memcg; |
@@ -3839,7 +3769,7 @@ int mem_cgroup_charge_file(struct page *page, struct mm_struct *mm, | |||
3839 | return 0; | 3769 | return 0; |
3840 | } | 3770 | } |
3841 | 3771 | ||
3842 | memcg = mem_cgroup_try_charge_mm(mm, gfp_mask, 1, true); | 3772 | memcg = mem_cgroup_try_charge_mm(mm, gfp_mask, 1); |
3843 | if (!memcg) | 3773 | if (!memcg) |
3844 | return -ENOMEM; | 3774 | return -ENOMEM; |
3845 | __mem_cgroup_commit_charge(memcg, page, 1, type, false); | 3775 | __mem_cgroup_commit_charge(memcg, page, 1, type, false); |
@@ -3993,7 +3923,7 @@ __mem_cgroup_uncharge_common(struct page *page, enum charge_type ctype, | |||
3993 | * replacement page, so leave it alone when phasing out the | 3923 | * replacement page, so leave it alone when phasing out the |
3994 | * page that is unused after the migration. | 3924 | * page that is unused after the migration. |
3995 | */ | 3925 | */ |
3996 | if (!end_migration && !mem_cgroup_is_root(memcg)) | 3926 | if (!end_migration) |
3997 | mem_cgroup_do_uncharge(memcg, nr_pages, ctype); | 3927 | mem_cgroup_do_uncharge(memcg, nr_pages, ctype); |
3998 | 3928 | ||
3999 | return memcg; | 3929 | return memcg; |
@@ -4126,8 +4056,7 @@ void mem_cgroup_uncharge_swap(swp_entry_t ent) | |||
4126 | * We uncharge this because swap is freed. This memcg can | 4056 | * We uncharge this because swap is freed. This memcg can |
4127 | * be obsolete one. We avoid calling css_tryget_online(). | 4057 | * be obsolete one. We avoid calling css_tryget_online(). |
4128 | */ | 4058 | */ |
4129 | if (!mem_cgroup_is_root(memcg)) | 4059 | res_counter_uncharge(&memcg->memsw, PAGE_SIZE); |
4130 | res_counter_uncharge(&memcg->memsw, PAGE_SIZE); | ||
4131 | mem_cgroup_swap_statistics(memcg, false); | 4060 | mem_cgroup_swap_statistics(memcg, false); |
4132 | css_put(&memcg->css); | 4061 | css_put(&memcg->css); |
4133 | } | 4062 | } |
@@ -4817,78 +4746,24 @@ out: | |||
4817 | return retval; | 4746 | return retval; |
4818 | } | 4747 | } |
4819 | 4748 | ||
4820 | |||
4821 | static unsigned long mem_cgroup_recursive_stat(struct mem_cgroup *memcg, | ||
4822 | enum mem_cgroup_stat_index idx) | ||
4823 | { | ||
4824 | struct mem_cgroup *iter; | ||
4825 | long val = 0; | ||
4826 | |||
4827 | /* Per-cpu values can be negative, use a signed accumulator */ | ||
4828 | for_each_mem_cgroup_tree(iter, memcg) | ||
4829 | val += mem_cgroup_read_stat(iter, idx); | ||
4830 | |||
4831 | if (val < 0) /* race ? */ | ||
4832 | val = 0; | ||
4833 | return val; | ||
4834 | } | ||
4835 | |||
4836 | static inline u64 mem_cgroup_usage(struct mem_cgroup *memcg, bool swap) | ||
4837 | { | ||
4838 | u64 val; | ||
4839 | |||
4840 | if (!mem_cgroup_is_root(memcg)) { | ||
4841 | if (!swap) | ||
4842 | return res_counter_read_u64(&memcg->res, RES_USAGE); | ||
4843 | else | ||
4844 | return res_counter_read_u64(&memcg->memsw, RES_USAGE); | ||
4845 | } | ||
4846 | |||
4847 | /* | ||
4848 | * Transparent hugepages are still accounted for in MEM_CGROUP_STAT_RSS | ||
4849 | * as well as in MEM_CGROUP_STAT_RSS_HUGE. | ||
4850 | */ | ||
4851 | val = mem_cgroup_recursive_stat(memcg, MEM_CGROUP_STAT_CACHE); | ||
4852 | val += mem_cgroup_recursive_stat(memcg, MEM_CGROUP_STAT_RSS); | ||
4853 | |||
4854 | if (swap) | ||
4855 | val += mem_cgroup_recursive_stat(memcg, MEM_CGROUP_STAT_SWAP); | ||
4856 | |||
4857 | return val << PAGE_SHIFT; | ||
4858 | } | ||
4859 | |||
4860 | static u64 mem_cgroup_read_u64(struct cgroup_subsys_state *css, | 4749 | static u64 mem_cgroup_read_u64(struct cgroup_subsys_state *css, |
4861 | struct cftype *cft) | 4750 | struct cftype *cft) |
4862 | { | 4751 | { |
4863 | struct mem_cgroup *memcg = mem_cgroup_from_css(css); | 4752 | struct mem_cgroup *memcg = mem_cgroup_from_css(css); |
4864 | u64 val; | 4753 | enum res_type type = MEMFILE_TYPE(cft->private); |
4865 | int name; | 4754 | int name = MEMFILE_ATTR(cft->private); |
4866 | enum res_type type; | ||
4867 | |||
4868 | type = MEMFILE_TYPE(cft->private); | ||
4869 | name = MEMFILE_ATTR(cft->private); | ||
4870 | 4755 | ||
4871 | switch (type) { | 4756 | switch (type) { |
4872 | case _MEM: | 4757 | case _MEM: |
4873 | if (name == RES_USAGE) | 4758 | return res_counter_read_u64(&memcg->res, name); |
4874 | val = mem_cgroup_usage(memcg, false); | ||
4875 | else | ||
4876 | val = res_counter_read_u64(&memcg->res, name); | ||
4877 | break; | ||
4878 | case _MEMSWAP: | 4759 | case _MEMSWAP: |
4879 | if (name == RES_USAGE) | 4760 | return res_counter_read_u64(&memcg->memsw, name); |
4880 | val = mem_cgroup_usage(memcg, true); | ||
4881 | else | ||
4882 | val = res_counter_read_u64(&memcg->memsw, name); | ||
4883 | break; | ||
4884 | case _KMEM: | 4761 | case _KMEM: |
4885 | val = res_counter_read_u64(&memcg->kmem, name); | 4762 | return res_counter_read_u64(&memcg->kmem, name); |
4886 | break; | 4763 | break; |
4887 | default: | 4764 | default: |
4888 | BUG(); | 4765 | BUG(); |
4889 | } | 4766 | } |
4890 | |||
4891 | return val; | ||
4892 | } | 4767 | } |
4893 | 4768 | ||
4894 | #ifdef CONFIG_MEMCG_KMEM | 4769 | #ifdef CONFIG_MEMCG_KMEM |
@@ -5350,7 +5225,10 @@ static void __mem_cgroup_threshold(struct mem_cgroup *memcg, bool swap) | |||
5350 | if (!t) | 5225 | if (!t) |
5351 | goto unlock; | 5226 | goto unlock; |
5352 | 5227 | ||
5353 | usage = mem_cgroup_usage(memcg, swap); | 5228 | if (!swap) |
5229 | usage = res_counter_read_u64(&memcg->res, RES_USAGE); | ||
5230 | else | ||
5231 | usage = res_counter_read_u64(&memcg->memsw, RES_USAGE); | ||
5354 | 5232 | ||
5355 | /* | 5233 | /* |
5356 | * current_threshold points to threshold just below or equal to usage. | 5234 | * current_threshold points to threshold just below or equal to usage. |
@@ -5446,15 +5324,15 @@ static int __mem_cgroup_usage_register_event(struct mem_cgroup *memcg, | |||
5446 | 5324 | ||
5447 | mutex_lock(&memcg->thresholds_lock); | 5325 | mutex_lock(&memcg->thresholds_lock); |
5448 | 5326 | ||
5449 | if (type == _MEM) | 5327 | if (type == _MEM) { |
5450 | thresholds = &memcg->thresholds; | 5328 | thresholds = &memcg->thresholds; |
5451 | else if (type == _MEMSWAP) | 5329 | usage = res_counter_read_u64(&memcg->res, RES_USAGE); |
5330 | } else if (type == _MEMSWAP) { | ||
5452 | thresholds = &memcg->memsw_thresholds; | 5331 | thresholds = &memcg->memsw_thresholds; |
5453 | else | 5332 | usage = res_counter_read_u64(&memcg->memsw, RES_USAGE); |
5333 | } else | ||
5454 | BUG(); | 5334 | BUG(); |
5455 | 5335 | ||
5456 | usage = mem_cgroup_usage(memcg, type == _MEMSWAP); | ||
5457 | |||
5458 | /* Check if a threshold crossed before adding a new one */ | 5336 | /* Check if a threshold crossed before adding a new one */ |
5459 | if (thresholds->primary) | 5337 | if (thresholds->primary) |
5460 | __mem_cgroup_threshold(memcg, type == _MEMSWAP); | 5338 | __mem_cgroup_threshold(memcg, type == _MEMSWAP); |
@@ -5534,18 +5412,19 @@ static void __mem_cgroup_usage_unregister_event(struct mem_cgroup *memcg, | |||
5534 | int i, j, size; | 5412 | int i, j, size; |
5535 | 5413 | ||
5536 | mutex_lock(&memcg->thresholds_lock); | 5414 | mutex_lock(&memcg->thresholds_lock); |
5537 | if (type == _MEM) | 5415 | |
5416 | if (type == _MEM) { | ||
5538 | thresholds = &memcg->thresholds; | 5417 | thresholds = &memcg->thresholds; |
5539 | else if (type == _MEMSWAP) | 5418 | usage = res_counter_read_u64(&memcg->res, RES_USAGE); |
5419 | } else if (type == _MEMSWAP) { | ||
5540 | thresholds = &memcg->memsw_thresholds; | 5420 | thresholds = &memcg->memsw_thresholds; |
5541 | else | 5421 | usage = res_counter_read_u64(&memcg->memsw, RES_USAGE); |
5422 | } else | ||
5542 | BUG(); | 5423 | BUG(); |
5543 | 5424 | ||
5544 | if (!thresholds->primary) | 5425 | if (!thresholds->primary) |
5545 | goto unlock; | 5426 | goto unlock; |
5546 | 5427 | ||
5547 | usage = mem_cgroup_usage(memcg, type == _MEMSWAP); | ||
5548 | |||
5549 | /* Check if a threshold crossed before removing */ | 5428 | /* Check if a threshold crossed before removing */ |
5550 | __mem_cgroup_threshold(memcg, type == _MEMSWAP); | 5429 | __mem_cgroup_threshold(memcg, type == _MEMSWAP); |
5551 | 5430 | ||
@@ -6299,9 +6178,9 @@ mem_cgroup_css_online(struct cgroup_subsys_state *css) | |||
6299 | * core guarantees its existence. | 6178 | * core guarantees its existence. |
6300 | */ | 6179 | */ |
6301 | } else { | 6180 | } else { |
6302 | res_counter_init(&memcg->res, NULL); | 6181 | res_counter_init(&memcg->res, &root_mem_cgroup->res); |
6303 | res_counter_init(&memcg->memsw, NULL); | 6182 | res_counter_init(&memcg->memsw, &root_mem_cgroup->memsw); |
6304 | res_counter_init(&memcg->kmem, NULL); | 6183 | res_counter_init(&memcg->kmem, &root_mem_cgroup->kmem); |
6305 | /* | 6184 | /* |
6306 | * Deeper hierachy with use_hierarchy == false doesn't make | 6185 | * Deeper hierachy with use_hierarchy == false doesn't make |
6307 | * much sense so let cgroup subsystem know about this | 6186 | * much sense so let cgroup subsystem know about this |
@@ -6435,55 +6314,39 @@ static void mem_cgroup_css_reset(struct cgroup_subsys_state *css) | |||
6435 | 6314 | ||
6436 | #ifdef CONFIG_MMU | 6315 | #ifdef CONFIG_MMU |
6437 | /* Handlers for move charge at task migration. */ | 6316 | /* Handlers for move charge at task migration. */ |
6438 | #define PRECHARGE_COUNT_AT_ONCE 256 | ||
6439 | static int mem_cgroup_do_precharge(unsigned long count) | 6317 | static int mem_cgroup_do_precharge(unsigned long count) |
6440 | { | 6318 | { |
6441 | int ret = 0; | 6319 | int ret; |
6442 | int batch_count = PRECHARGE_COUNT_AT_ONCE; | ||
6443 | struct mem_cgroup *memcg = mc.to; | ||
6444 | 6320 | ||
6445 | if (mem_cgroup_is_root(memcg)) { | 6321 | /* Try a single bulk charge without reclaim first */ |
6322 | ret = mem_cgroup_try_charge(mc.to, GFP_KERNEL & ~__GFP_WAIT, count); | ||
6323 | if (!ret) { | ||
6446 | mc.precharge += count; | 6324 | mc.precharge += count; |
6447 | /* we don't need css_get for root */ | ||
6448 | return ret; | 6325 | return ret; |
6449 | } | 6326 | } |
6450 | /* try to charge at once */ | 6327 | if (ret == -EINTR) { |
6451 | if (count > 1) { | 6328 | __mem_cgroup_cancel_charge(root_mem_cgroup, count); |
6452 | struct res_counter *dummy; | ||
6453 | /* | ||
6454 | * "memcg" cannot be under rmdir() because we've already checked | ||
6455 | * by cgroup_lock_live_cgroup() that it is not removed and we | ||
6456 | * are still under the same cgroup_mutex. So we can postpone | ||
6457 | * css_get(). | ||
6458 | */ | ||
6459 | if (res_counter_charge(&memcg->res, PAGE_SIZE * count, &dummy)) | ||
6460 | goto one_by_one; | ||
6461 | if (do_swap_account && res_counter_charge(&memcg->memsw, | ||
6462 | PAGE_SIZE * count, &dummy)) { | ||
6463 | res_counter_uncharge(&memcg->res, PAGE_SIZE * count); | ||
6464 | goto one_by_one; | ||
6465 | } | ||
6466 | mc.precharge += count; | ||
6467 | return ret; | 6329 | return ret; |
6468 | } | 6330 | } |
6469 | one_by_one: | 6331 | |
6470 | /* fall back to one by one charge */ | 6332 | /* Try charges one by one with reclaim */ |
6471 | while (count--) { | 6333 | while (count--) { |
6472 | if (signal_pending(current)) { | 6334 | ret = mem_cgroup_try_charge(mc.to, |
6473 | ret = -EINTR; | 6335 | GFP_KERNEL & ~__GFP_NORETRY, 1); |
6474 | break; | 6336 | /* |
6475 | } | 6337 | * In case of failure, any residual charges against |
6476 | if (!batch_count--) { | 6338 | * mc.to will be dropped by mem_cgroup_clear_mc() |
6477 | batch_count = PRECHARGE_COUNT_AT_ONCE; | 6339 | * later on. However, cancel any charges that are |
6478 | cond_resched(); | 6340 | * bypassed to root right away or they'll be lost. |
6479 | } | 6341 | */ |
6480 | ret = mem_cgroup_try_charge(memcg, GFP_KERNEL, 1, false); | 6342 | if (ret == -EINTR) |
6343 | __mem_cgroup_cancel_charge(root_mem_cgroup, 1); | ||
6481 | if (ret) | 6344 | if (ret) |
6482 | /* mem_cgroup_clear_mc() will do uncharge later */ | ||
6483 | return ret; | 6345 | return ret; |
6484 | mc.precharge++; | 6346 | mc.precharge++; |
6347 | cond_resched(); | ||
6485 | } | 6348 | } |
6486 | return ret; | 6349 | return 0; |
6487 | } | 6350 | } |
6488 | 6351 | ||
6489 | /** | 6352 | /** |
@@ -6760,21 +6623,18 @@ static void __mem_cgroup_clear_mc(void) | |||
6760 | /* we must fixup refcnts and charges */ | 6623 | /* we must fixup refcnts and charges */ |
6761 | if (mc.moved_swap) { | 6624 | if (mc.moved_swap) { |
6762 | /* uncharge swap account from the old cgroup */ | 6625 | /* uncharge swap account from the old cgroup */ |
6763 | if (!mem_cgroup_is_root(mc.from)) | 6626 | res_counter_uncharge(&mc.from->memsw, |
6764 | res_counter_uncharge(&mc.from->memsw, | 6627 | PAGE_SIZE * mc.moved_swap); |
6765 | PAGE_SIZE * mc.moved_swap); | ||
6766 | 6628 | ||
6767 | for (i = 0; i < mc.moved_swap; i++) | 6629 | for (i = 0; i < mc.moved_swap; i++) |
6768 | css_put(&mc.from->css); | 6630 | css_put(&mc.from->css); |
6769 | 6631 | ||
6770 | if (!mem_cgroup_is_root(mc.to)) { | 6632 | /* |
6771 | /* | 6633 | * we charged both to->res and to->memsw, so we should |
6772 | * we charged both to->res and to->memsw, so we should | 6634 | * uncharge to->res. |
6773 | * uncharge to->res. | 6635 | */ |
6774 | */ | 6636 | res_counter_uncharge(&mc.to->res, |
6775 | res_counter_uncharge(&mc.to->res, | 6637 | PAGE_SIZE * mc.moved_swap); |
6776 | PAGE_SIZE * mc.moved_swap); | ||
6777 | } | ||
6778 | /* we've already done css_get(mc.to) */ | 6638 | /* we've already done css_get(mc.to) */ |
6779 | mc.moved_swap = 0; | 6639 | mc.moved_swap = 0; |
6780 | } | 6640 | } |
diff --git a/mm/memory-failure.c b/mm/memory-failure.c index a013bc94ebbe..44c6bd201d3a 100644 --- a/mm/memory-failure.c +++ b/mm/memory-failure.c | |||
@@ -1173,6 +1173,16 @@ int memory_failure(unsigned long pfn, int trapno, int flags) | |||
1173 | lock_page(hpage); | 1173 | lock_page(hpage); |
1174 | 1174 | ||
1175 | /* | 1175 | /* |
1176 | * The page could have changed compound pages during the locking. | ||
1177 | * If this happens just bail out. | ||
1178 | */ | ||
1179 | if (compound_head(p) != hpage) { | ||
1180 | action_result(pfn, "different compound page after locking", IGNORED); | ||
1181 | res = -EBUSY; | ||
1182 | goto out; | ||
1183 | } | ||
1184 | |||
1185 | /* | ||
1176 | * We use page flags to determine what action should be taken, but | 1186 | * We use page flags to determine what action should be taken, but |
1177 | * the flags can be modified by the error containment action. One | 1187 | * the flags can be modified by the error containment action. One |
1178 | * example is an mlocked page, where PG_mlocked is cleared by | 1188 | * example is an mlocked page, where PG_mlocked is cleared by |
diff --git a/mm/memory.c b/mm/memory.c index 8b44f765b645..5c55270729f7 100644 --- a/mm/memory.c +++ b/mm/memory.c | |||
@@ -884,7 +884,7 @@ out_set_pte: | |||
884 | return 0; | 884 | return 0; |
885 | } | 885 | } |
886 | 886 | ||
887 | int copy_pte_range(struct mm_struct *dst_mm, struct mm_struct *src_mm, | 887 | static int copy_pte_range(struct mm_struct *dst_mm, struct mm_struct *src_mm, |
888 | pmd_t *dst_pmd, pmd_t *src_pmd, struct vm_area_struct *vma, | 888 | pmd_t *dst_pmd, pmd_t *src_pmd, struct vm_area_struct *vma, |
889 | unsigned long addr, unsigned long end) | 889 | unsigned long addr, unsigned long end) |
890 | { | 890 | { |
@@ -2399,7 +2399,10 @@ EXPORT_SYMBOL(unmap_mapping_range); | |||
2399 | /* | 2399 | /* |
2400 | * We enter with non-exclusive mmap_sem (to exclude vma changes, | 2400 | * We enter with non-exclusive mmap_sem (to exclude vma changes, |
2401 | * but allow concurrent faults), and pte mapped but not yet locked. | 2401 | * but allow concurrent faults), and pte mapped but not yet locked. |
2402 | * We return with mmap_sem still held, but pte unmapped and unlocked. | 2402 | * We return with pte unmapped and unlocked. |
2403 | * | ||
2404 | * We return with the mmap_sem locked or unlocked in the same cases | ||
2405 | * as does filemap_fault(). | ||
2403 | */ | 2406 | */ |
2404 | static int do_swap_page(struct mm_struct *mm, struct vm_area_struct *vma, | 2407 | static int do_swap_page(struct mm_struct *mm, struct vm_area_struct *vma, |
2405 | unsigned long address, pte_t *page_table, pmd_t *pmd, | 2408 | unsigned long address, pte_t *page_table, pmd_t *pmd, |
@@ -2688,6 +2691,11 @@ oom: | |||
2688 | return VM_FAULT_OOM; | 2691 | return VM_FAULT_OOM; |
2689 | } | 2692 | } |
2690 | 2693 | ||
2694 | /* | ||
2695 | * The mmap_sem must have been held on entry, and may have been | ||
2696 | * released depending on flags and vma->vm_ops->fault() return value. | ||
2697 | * See filemap_fault() and __lock_page_retry(). | ||
2698 | */ | ||
2691 | static int __do_fault(struct vm_area_struct *vma, unsigned long address, | 2699 | static int __do_fault(struct vm_area_struct *vma, unsigned long address, |
2692 | pgoff_t pgoff, unsigned int flags, struct page **page) | 2700 | pgoff_t pgoff, unsigned int flags, struct page **page) |
2693 | { | 2701 | { |
@@ -2744,7 +2752,7 @@ void do_set_pte(struct vm_area_struct *vma, unsigned long address, | |||
2744 | if (write) | 2752 | if (write) |
2745 | entry = maybe_mkwrite(pte_mkdirty(entry), vma); | 2753 | entry = maybe_mkwrite(pte_mkdirty(entry), vma); |
2746 | else if (pte_file(*pte) && pte_file_soft_dirty(*pte)) | 2754 | else if (pte_file(*pte) && pte_file_soft_dirty(*pte)) |
2747 | pte_mksoft_dirty(entry); | 2755 | entry = pte_mksoft_dirty(entry); |
2748 | if (anon) { | 2756 | if (anon) { |
2749 | inc_mm_counter_fast(vma->vm_mm, MM_ANONPAGES); | 2757 | inc_mm_counter_fast(vma->vm_mm, MM_ANONPAGES); |
2750 | page_add_new_anon_rmap(page, vma, address); | 2758 | page_add_new_anon_rmap(page, vma, address); |
@@ -2758,17 +2766,8 @@ void do_set_pte(struct vm_area_struct *vma, unsigned long address, | |||
2758 | update_mmu_cache(vma, address, pte); | 2766 | update_mmu_cache(vma, address, pte); |
2759 | } | 2767 | } |
2760 | 2768 | ||
2761 | static unsigned long fault_around_bytes = rounddown_pow_of_two(65536); | 2769 | static unsigned long fault_around_bytes __read_mostly = |
2762 | 2770 | rounddown_pow_of_two(65536); | |
2763 | static inline unsigned long fault_around_pages(void) | ||
2764 | { | ||
2765 | return fault_around_bytes >> PAGE_SHIFT; | ||
2766 | } | ||
2767 | |||
2768 | static inline unsigned long fault_around_mask(void) | ||
2769 | { | ||
2770 | return ~(fault_around_bytes - 1) & PAGE_MASK; | ||
2771 | } | ||
2772 | 2771 | ||
2773 | #ifdef CONFIG_DEBUG_FS | 2772 | #ifdef CONFIG_DEBUG_FS |
2774 | static int fault_around_bytes_get(void *data, u64 *val) | 2773 | static int fault_around_bytes_get(void *data, u64 *val) |
@@ -2834,12 +2833,15 @@ late_initcall(fault_around_debugfs); | |||
2834 | static void do_fault_around(struct vm_area_struct *vma, unsigned long address, | 2833 | static void do_fault_around(struct vm_area_struct *vma, unsigned long address, |
2835 | pte_t *pte, pgoff_t pgoff, unsigned int flags) | 2834 | pte_t *pte, pgoff_t pgoff, unsigned int flags) |
2836 | { | 2835 | { |
2837 | unsigned long start_addr; | 2836 | unsigned long start_addr, nr_pages, mask; |
2838 | pgoff_t max_pgoff; | 2837 | pgoff_t max_pgoff; |
2839 | struct vm_fault vmf; | 2838 | struct vm_fault vmf; |
2840 | int off; | 2839 | int off; |
2841 | 2840 | ||
2842 | start_addr = max(address & fault_around_mask(), vma->vm_start); | 2841 | nr_pages = ACCESS_ONCE(fault_around_bytes) >> PAGE_SHIFT; |
2842 | mask = ~(nr_pages * PAGE_SIZE - 1) & PAGE_MASK; | ||
2843 | |||
2844 | start_addr = max(address & mask, vma->vm_start); | ||
2843 | off = ((address - start_addr) >> PAGE_SHIFT) & (PTRS_PER_PTE - 1); | 2845 | off = ((address - start_addr) >> PAGE_SHIFT) & (PTRS_PER_PTE - 1); |
2844 | pte -= off; | 2846 | pte -= off; |
2845 | pgoff -= off; | 2847 | pgoff -= off; |
@@ -2851,7 +2853,7 @@ static void do_fault_around(struct vm_area_struct *vma, unsigned long address, | |||
2851 | max_pgoff = pgoff - ((start_addr >> PAGE_SHIFT) & (PTRS_PER_PTE - 1)) + | 2853 | max_pgoff = pgoff - ((start_addr >> PAGE_SHIFT) & (PTRS_PER_PTE - 1)) + |
2852 | PTRS_PER_PTE - 1; | 2854 | PTRS_PER_PTE - 1; |
2853 | max_pgoff = min3(max_pgoff, vma_pages(vma) + vma->vm_pgoff - 1, | 2855 | max_pgoff = min3(max_pgoff, vma_pages(vma) + vma->vm_pgoff - 1, |
2854 | pgoff + fault_around_pages() - 1); | 2856 | pgoff + nr_pages - 1); |
2855 | 2857 | ||
2856 | /* Check if it makes any sense to call ->map_pages */ | 2858 | /* Check if it makes any sense to call ->map_pages */ |
2857 | while (!pte_none(*pte)) { | 2859 | while (!pte_none(*pte)) { |
@@ -2886,7 +2888,7 @@ static int do_read_fault(struct mm_struct *mm, struct vm_area_struct *vma, | |||
2886 | * something). | 2888 | * something). |
2887 | */ | 2889 | */ |
2888 | if (vma->vm_ops->map_pages && !(flags & FAULT_FLAG_NONLINEAR) && | 2890 | if (vma->vm_ops->map_pages && !(flags & FAULT_FLAG_NONLINEAR) && |
2889 | fault_around_pages() > 1) { | 2891 | fault_around_bytes >> PAGE_SHIFT > 1) { |
2890 | pte = pte_offset_map_lock(mm, pmd, address, &ptl); | 2892 | pte = pte_offset_map_lock(mm, pmd, address, &ptl); |
2891 | do_fault_around(vma, address, pte, pgoff, flags); | 2893 | do_fault_around(vma, address, pte, pgoff, flags); |
2892 | if (!pte_same(*pte, orig_pte)) | 2894 | if (!pte_same(*pte, orig_pte)) |
@@ -3016,6 +3018,12 @@ static int do_shared_fault(struct mm_struct *mm, struct vm_area_struct *vma, | |||
3016 | return ret; | 3018 | return ret; |
3017 | } | 3019 | } |
3018 | 3020 | ||
3021 | /* | ||
3022 | * We enter with non-exclusive mmap_sem (to exclude vma changes, | ||
3023 | * but allow concurrent faults). | ||
3024 | * The mmap_sem may have been released depending on flags and our | ||
3025 | * return value. See filemap_fault() and __lock_page_or_retry(). | ||
3026 | */ | ||
3019 | static int do_linear_fault(struct mm_struct *mm, struct vm_area_struct *vma, | 3027 | static int do_linear_fault(struct mm_struct *mm, struct vm_area_struct *vma, |
3020 | unsigned long address, pte_t *page_table, pmd_t *pmd, | 3028 | unsigned long address, pte_t *page_table, pmd_t *pmd, |
3021 | unsigned int flags, pte_t orig_pte) | 3029 | unsigned int flags, pte_t orig_pte) |
@@ -3040,7 +3048,9 @@ static int do_linear_fault(struct mm_struct *mm, struct vm_area_struct *vma, | |||
3040 | * | 3048 | * |
3041 | * We enter with non-exclusive mmap_sem (to exclude vma changes, | 3049 | * We enter with non-exclusive mmap_sem (to exclude vma changes, |
3042 | * but allow concurrent faults), and pte mapped but not yet locked. | 3050 | * but allow concurrent faults), and pte mapped but not yet locked. |
3043 | * We return with mmap_sem still held, but pte unmapped and unlocked. | 3051 | * We return with pte unmapped and unlocked. |
3052 | * The mmap_sem may have been released depending on flags and our | ||
3053 | * return value. See filemap_fault() and __lock_page_or_retry(). | ||
3044 | */ | 3054 | */ |
3045 | static int do_nonlinear_fault(struct mm_struct *mm, struct vm_area_struct *vma, | 3055 | static int do_nonlinear_fault(struct mm_struct *mm, struct vm_area_struct *vma, |
3046 | unsigned long address, pte_t *page_table, pmd_t *pmd, | 3056 | unsigned long address, pte_t *page_table, pmd_t *pmd, |
@@ -3172,7 +3182,10 @@ out: | |||
3172 | * | 3182 | * |
3173 | * We enter with non-exclusive mmap_sem (to exclude vma changes, | 3183 | * We enter with non-exclusive mmap_sem (to exclude vma changes, |
3174 | * but allow concurrent faults), and pte mapped but not yet locked. | 3184 | * but allow concurrent faults), and pte mapped but not yet locked. |
3175 | * We return with mmap_sem still held, but pte unmapped and unlocked. | 3185 | * We return with pte unmapped and unlocked. |
3186 | * | ||
3187 | * The mmap_sem may have been released depending on flags and our | ||
3188 | * return value. See filemap_fault() and __lock_page_or_retry(). | ||
3176 | */ | 3189 | */ |
3177 | static int handle_pte_fault(struct mm_struct *mm, | 3190 | static int handle_pte_fault(struct mm_struct *mm, |
3178 | struct vm_area_struct *vma, unsigned long address, | 3191 | struct vm_area_struct *vma, unsigned long address, |
@@ -3181,7 +3194,7 @@ static int handle_pte_fault(struct mm_struct *mm, | |||
3181 | pte_t entry; | 3194 | pte_t entry; |
3182 | spinlock_t *ptl; | 3195 | spinlock_t *ptl; |
3183 | 3196 | ||
3184 | entry = *pte; | 3197 | entry = ACCESS_ONCE(*pte); |
3185 | if (!pte_present(entry)) { | 3198 | if (!pte_present(entry)) { |
3186 | if (pte_none(entry)) { | 3199 | if (pte_none(entry)) { |
3187 | if (vma->vm_ops) { | 3200 | if (vma->vm_ops) { |
@@ -3232,6 +3245,9 @@ unlock: | |||
3232 | 3245 | ||
3233 | /* | 3246 | /* |
3234 | * By the time we get here, we already hold the mm semaphore | 3247 | * By the time we get here, we already hold the mm semaphore |
3248 | * | ||
3249 | * The mmap_sem may have been released depending on flags and our | ||
3250 | * return value. See filemap_fault() and __lock_page_or_retry(). | ||
3235 | */ | 3251 | */ |
3236 | static int __handle_mm_fault(struct mm_struct *mm, struct vm_area_struct *vma, | 3252 | static int __handle_mm_fault(struct mm_struct *mm, struct vm_area_struct *vma, |
3237 | unsigned long address, unsigned int flags) | 3253 | unsigned long address, unsigned int flags) |
@@ -3313,6 +3329,12 @@ static int __handle_mm_fault(struct mm_struct *mm, struct vm_area_struct *vma, | |||
3313 | return handle_pte_fault(mm, vma, address, pte, pmd, flags); | 3329 | return handle_pte_fault(mm, vma, address, pte, pmd, flags); |
3314 | } | 3330 | } |
3315 | 3331 | ||
3332 | /* | ||
3333 | * By the time we get here, we already hold the mm semaphore | ||
3334 | * | ||
3335 | * The mmap_sem may have been released depending on flags and our | ||
3336 | * return value. See filemap_fault() and __lock_page_or_retry(). | ||
3337 | */ | ||
3316 | int handle_mm_fault(struct mm_struct *mm, struct vm_area_struct *vma, | 3338 | int handle_mm_fault(struct mm_struct *mm, struct vm_area_struct *vma, |
3317 | unsigned long address, unsigned int flags) | 3339 | unsigned long address, unsigned int flags) |
3318 | { | 3340 | { |
@@ -3591,11 +3613,13 @@ static int __access_remote_vm(struct task_struct *tsk, struct mm_struct *mm, | |||
3591 | ret = get_user_pages(tsk, mm, addr, 1, | 3613 | ret = get_user_pages(tsk, mm, addr, 1, |
3592 | write, 1, &page, &vma); | 3614 | write, 1, &page, &vma); |
3593 | if (ret <= 0) { | 3615 | if (ret <= 0) { |
3616 | #ifndef CONFIG_HAVE_IOREMAP_PROT | ||
3617 | break; | ||
3618 | #else | ||
3594 | /* | 3619 | /* |
3595 | * Check if this is a VM_IO | VM_PFNMAP VMA, which | 3620 | * Check if this is a VM_IO | VM_PFNMAP VMA, which |
3596 | * we can access using slightly different code. | 3621 | * we can access using slightly different code. |
3597 | */ | 3622 | */ |
3598 | #ifdef CONFIG_HAVE_IOREMAP_PROT | ||
3599 | vma = find_vma(mm, addr); | 3623 | vma = find_vma(mm, addr); |
3600 | if (!vma || vma->vm_start > addr) | 3624 | if (!vma || vma->vm_start > addr) |
3601 | break; | 3625 | break; |
@@ -3603,9 +3627,9 @@ static int __access_remote_vm(struct task_struct *tsk, struct mm_struct *mm, | |||
3603 | ret = vma->vm_ops->access(vma, addr, buf, | 3627 | ret = vma->vm_ops->access(vma, addr, buf, |
3604 | len, write); | 3628 | len, write); |
3605 | if (ret <= 0) | 3629 | if (ret <= 0) |
3606 | #endif | ||
3607 | break; | 3630 | break; |
3608 | bytes = ret; | 3631 | bytes = ret; |
3632 | #endif | ||
3609 | } else { | 3633 | } else { |
3610 | bytes = len; | 3634 | bytes = len; |
3611 | offset = addr & (PAGE_SIZE-1); | 3635 | offset = addr & (PAGE_SIZE-1); |
diff --git a/mm/memory_hotplug.c b/mm/memory_hotplug.c index 469bbf505f85..2ff8c2325e96 100644 --- a/mm/memory_hotplug.c +++ b/mm/memory_hotplug.c | |||
@@ -284,8 +284,8 @@ void register_page_bootmem_info_node(struct pglist_data *pgdat) | |||
284 | } | 284 | } |
285 | #endif /* CONFIG_HAVE_BOOTMEM_INFO_NODE */ | 285 | #endif /* CONFIG_HAVE_BOOTMEM_INFO_NODE */ |
286 | 286 | ||
287 | static void grow_zone_span(struct zone *zone, unsigned long start_pfn, | 287 | static void __meminit grow_zone_span(struct zone *zone, unsigned long start_pfn, |
288 | unsigned long end_pfn) | 288 | unsigned long end_pfn) |
289 | { | 289 | { |
290 | unsigned long old_zone_end_pfn; | 290 | unsigned long old_zone_end_pfn; |
291 | 291 | ||
@@ -427,8 +427,8 @@ out_fail: | |||
427 | return -1; | 427 | return -1; |
428 | } | 428 | } |
429 | 429 | ||
430 | static void grow_pgdat_span(struct pglist_data *pgdat, unsigned long start_pfn, | 430 | static void __meminit grow_pgdat_span(struct pglist_data *pgdat, unsigned long start_pfn, |
431 | unsigned long end_pfn) | 431 | unsigned long end_pfn) |
432 | { | 432 | { |
433 | unsigned long old_pgdat_end_pfn = pgdat_end_pfn(pgdat); | 433 | unsigned long old_pgdat_end_pfn = pgdat_end_pfn(pgdat); |
434 | 434 | ||
@@ -977,15 +977,18 @@ int __ref online_pages(unsigned long pfn, unsigned long nr_pages, int online_typ | |||
977 | zone = page_zone(pfn_to_page(pfn)); | 977 | zone = page_zone(pfn_to_page(pfn)); |
978 | 978 | ||
979 | ret = -EINVAL; | 979 | ret = -EINVAL; |
980 | if ((zone_idx(zone) > ZONE_NORMAL || online_type == ONLINE_MOVABLE) && | 980 | if ((zone_idx(zone) > ZONE_NORMAL || |
981 | online_type == MMOP_ONLINE_MOVABLE) && | ||
981 | !can_online_high_movable(zone)) | 982 | !can_online_high_movable(zone)) |
982 | goto out; | 983 | goto out; |
983 | 984 | ||
984 | if (online_type == ONLINE_KERNEL && zone_idx(zone) == ZONE_MOVABLE) { | 985 | if (online_type == MMOP_ONLINE_KERNEL && |
986 | zone_idx(zone) == ZONE_MOVABLE) { | ||
985 | if (move_pfn_range_left(zone - 1, zone, pfn, pfn + nr_pages)) | 987 | if (move_pfn_range_left(zone - 1, zone, pfn, pfn + nr_pages)) |
986 | goto out; | 988 | goto out; |
987 | } | 989 | } |
988 | if (online_type == ONLINE_MOVABLE && zone_idx(zone) == ZONE_MOVABLE - 1) { | 990 | if (online_type == MMOP_ONLINE_MOVABLE && |
991 | zone_idx(zone) == ZONE_MOVABLE - 1) { | ||
989 | if (move_pfn_range_right(zone, zone + 1, pfn, pfn + nr_pages)) | 992 | if (move_pfn_range_right(zone, zone + 1, pfn, pfn + nr_pages)) |
990 | goto out; | 993 | goto out; |
991 | } | 994 | } |
@@ -1156,6 +1159,34 @@ static int check_hotplug_memory_range(u64 start, u64 size) | |||
1156 | return 0; | 1159 | return 0; |
1157 | } | 1160 | } |
1158 | 1161 | ||
1162 | /* | ||
1163 | * If movable zone has already been setup, newly added memory should be check. | ||
1164 | * If its address is higher than movable zone, it should be added as movable. | ||
1165 | * Without this check, movable zone may overlap with other zone. | ||
1166 | */ | ||
1167 | static int should_add_memory_movable(int nid, u64 start, u64 size) | ||
1168 | { | ||
1169 | unsigned long start_pfn = start >> PAGE_SHIFT; | ||
1170 | pg_data_t *pgdat = NODE_DATA(nid); | ||
1171 | struct zone *movable_zone = pgdat->node_zones + ZONE_MOVABLE; | ||
1172 | |||
1173 | if (zone_is_empty(movable_zone)) | ||
1174 | return 0; | ||
1175 | |||
1176 | if (movable_zone->zone_start_pfn <= start_pfn) | ||
1177 | return 1; | ||
1178 | |||
1179 | return 0; | ||
1180 | } | ||
1181 | |||
1182 | int zone_for_memory(int nid, u64 start, u64 size, int zone_default) | ||
1183 | { | ||
1184 | if (should_add_memory_movable(nid, start, size)) | ||
1185 | return ZONE_MOVABLE; | ||
1186 | |||
1187 | return zone_default; | ||
1188 | } | ||
1189 | |||
1159 | /* we are OK calling __meminit stuff here - we have CONFIG_MEMORY_HOTPLUG */ | 1190 | /* we are OK calling __meminit stuff here - we have CONFIG_MEMORY_HOTPLUG */ |
1160 | int __ref add_memory(int nid, u64 start, u64 size) | 1191 | int __ref add_memory(int nid, u64 start, u64 size) |
1161 | { | 1192 | { |
diff --git a/mm/mlock.c b/mm/mlock.c index b1eb53634005..ce84cb0b83ef 100644 --- a/mm/mlock.c +++ b/mm/mlock.c | |||
@@ -210,12 +210,19 @@ out: | |||
210 | * @vma: target vma | 210 | * @vma: target vma |
211 | * @start: start address | 211 | * @start: start address |
212 | * @end: end address | 212 | * @end: end address |
213 | * @nonblocking: | ||
213 | * | 214 | * |
214 | * This takes care of making the pages present too. | 215 | * This takes care of making the pages present too. |
215 | * | 216 | * |
216 | * return 0 on success, negative error code on error. | 217 | * return 0 on success, negative error code on error. |
217 | * | 218 | * |
218 | * vma->vm_mm->mmap_sem must be held for at least read. | 219 | * vma->vm_mm->mmap_sem must be held. |
220 | * | ||
221 | * If @nonblocking is NULL, it may be held for read or write and will | ||
222 | * be unperturbed. | ||
223 | * | ||
224 | * If @nonblocking is non-NULL, it must held for read only and may be | ||
225 | * released. If it's released, *@nonblocking will be set to 0. | ||
219 | */ | 226 | */ |
220 | long __mlock_vma_pages_range(struct vm_area_struct *vma, | 227 | long __mlock_vma_pages_range(struct vm_area_struct *vma, |
221 | unsigned long start, unsigned long end, int *nonblocking) | 228 | unsigned long start, unsigned long end, int *nonblocking) |
@@ -31,6 +31,7 @@ | |||
31 | #include <linux/mempolicy.h> | 31 | #include <linux/mempolicy.h> |
32 | #include <linux/rmap.h> | 32 | #include <linux/rmap.h> |
33 | #include <linux/mmu_notifier.h> | 33 | #include <linux/mmu_notifier.h> |
34 | #include <linux/mmdebug.h> | ||
34 | #include <linux/perf_event.h> | 35 | #include <linux/perf_event.h> |
35 | #include <linux/audit.h> | 36 | #include <linux/audit.h> |
36 | #include <linux/khugepaged.h> | 37 | #include <linux/khugepaged.h> |
@@ -134,6 +135,10 @@ int __vm_enough_memory(struct mm_struct *mm, long pages, int cap_sys_admin) | |||
134 | { | 135 | { |
135 | unsigned long free, allowed, reserve; | 136 | unsigned long free, allowed, reserve; |
136 | 137 | ||
138 | VM_WARN_ONCE(percpu_counter_read(&vm_committed_as) < | ||
139 | -(s64)vm_committed_as_batch * num_online_cpus(), | ||
140 | "memory commitment underflow"); | ||
141 | |||
137 | vm_acct_memory(pages); | 142 | vm_acct_memory(pages); |
138 | 143 | ||
139 | /* | 144 | /* |
diff --git a/mm/mmu_notifier.c b/mm/mmu_notifier.c index 41cefdf0aadd..950813b1eb36 100644 --- a/mm/mmu_notifier.c +++ b/mm/mmu_notifier.c | |||
@@ -23,6 +23,25 @@ | |||
23 | static struct srcu_struct srcu; | 23 | static struct srcu_struct srcu; |
24 | 24 | ||
25 | /* | 25 | /* |
26 | * This function allows mmu_notifier::release callback to delay a call to | ||
27 | * a function that will free appropriate resources. The function must be | ||
28 | * quick and must not block. | ||
29 | */ | ||
30 | void mmu_notifier_call_srcu(struct rcu_head *rcu, | ||
31 | void (*func)(struct rcu_head *rcu)) | ||
32 | { | ||
33 | call_srcu(&srcu, rcu, func); | ||
34 | } | ||
35 | EXPORT_SYMBOL_GPL(mmu_notifier_call_srcu); | ||
36 | |||
37 | void mmu_notifier_synchronize(void) | ||
38 | { | ||
39 | /* Wait for any running method to finish. */ | ||
40 | srcu_barrier(&srcu); | ||
41 | } | ||
42 | EXPORT_SYMBOL_GPL(mmu_notifier_synchronize); | ||
43 | |||
44 | /* | ||
26 | * This function can't run concurrently against mmu_notifier_register | 45 | * This function can't run concurrently against mmu_notifier_register |
27 | * because mm->mm_users > 0 during mmu_notifier_register and exit_mmap | 46 | * because mm->mm_users > 0 during mmu_notifier_register and exit_mmap |
28 | * runs with mm_users == 0. Other tasks may still invoke mmu notifiers | 47 | * runs with mm_users == 0. Other tasks may still invoke mmu notifiers |
@@ -53,7 +72,6 @@ void __mmu_notifier_release(struct mm_struct *mm) | |||
53 | */ | 72 | */ |
54 | if (mn->ops->release) | 73 | if (mn->ops->release) |
55 | mn->ops->release(mn, mm); | 74 | mn->ops->release(mn, mm); |
56 | srcu_read_unlock(&srcu, id); | ||
57 | 75 | ||
58 | spin_lock(&mm->mmu_notifier_mm->lock); | 76 | spin_lock(&mm->mmu_notifier_mm->lock); |
59 | while (unlikely(!hlist_empty(&mm->mmu_notifier_mm->list))) { | 77 | while (unlikely(!hlist_empty(&mm->mmu_notifier_mm->list))) { |
@@ -69,6 +87,7 @@ void __mmu_notifier_release(struct mm_struct *mm) | |||
69 | hlist_del_init_rcu(&mn->hlist); | 87 | hlist_del_init_rcu(&mn->hlist); |
70 | } | 88 | } |
71 | spin_unlock(&mm->mmu_notifier_mm->lock); | 89 | spin_unlock(&mm->mmu_notifier_mm->lock); |
90 | srcu_read_unlock(&srcu, id); | ||
72 | 91 | ||
73 | /* | 92 | /* |
74 | * synchronize_srcu here prevents mmu_notifier_release from returning to | 93 | * synchronize_srcu here prevents mmu_notifier_release from returning to |
@@ -325,6 +344,25 @@ void mmu_notifier_unregister(struct mmu_notifier *mn, struct mm_struct *mm) | |||
325 | } | 344 | } |
326 | EXPORT_SYMBOL_GPL(mmu_notifier_unregister); | 345 | EXPORT_SYMBOL_GPL(mmu_notifier_unregister); |
327 | 346 | ||
347 | /* | ||
348 | * Same as mmu_notifier_unregister but no callback and no srcu synchronization. | ||
349 | */ | ||
350 | void mmu_notifier_unregister_no_release(struct mmu_notifier *mn, | ||
351 | struct mm_struct *mm) | ||
352 | { | ||
353 | spin_lock(&mm->mmu_notifier_mm->lock); | ||
354 | /* | ||
355 | * Can not use list_del_rcu() since __mmu_notifier_release | ||
356 | * can delete it before we hold the lock. | ||
357 | */ | ||
358 | hlist_del_init_rcu(&mn->hlist); | ||
359 | spin_unlock(&mm->mmu_notifier_mm->lock); | ||
360 | |||
361 | BUG_ON(atomic_read(&mm->mm_count) <= 0); | ||
362 | mmdrop(mm); | ||
363 | } | ||
364 | EXPORT_SYMBOL_GPL(mmu_notifier_unregister_no_release); | ||
365 | |||
328 | static int __init mmu_notifier_init(void) | 366 | static int __init mmu_notifier_init(void) |
329 | { | 367 | { |
330 | return init_srcu_struct(&srcu); | 368 | return init_srcu_struct(&srcu); |
diff --git a/mm/oom_kill.c b/mm/oom_kill.c index 3291e82d4352..1e11df8fa7ec 100644 --- a/mm/oom_kill.c +++ b/mm/oom_kill.c | |||
@@ -258,8 +258,6 @@ enum oom_scan_t oom_scan_process_thread(struct task_struct *task, | |||
258 | unsigned long totalpages, const nodemask_t *nodemask, | 258 | unsigned long totalpages, const nodemask_t *nodemask, |
259 | bool force_kill) | 259 | bool force_kill) |
260 | { | 260 | { |
261 | if (task->exit_state) | ||
262 | return OOM_SCAN_CONTINUE; | ||
263 | if (oom_unkillable_task(task, NULL, nodemask)) | 261 | if (oom_unkillable_task(task, NULL, nodemask)) |
264 | return OOM_SCAN_CONTINUE; | 262 | return OOM_SCAN_CONTINUE; |
265 | 263 | ||
@@ -559,28 +557,25 @@ EXPORT_SYMBOL_GPL(unregister_oom_notifier); | |||
559 | * if a parallel OOM killing is already taking place that includes a zone in | 557 | * if a parallel OOM killing is already taking place that includes a zone in |
560 | * the zonelist. Otherwise, locks all zones in the zonelist and returns 1. | 558 | * the zonelist. Otherwise, locks all zones in the zonelist and returns 1. |
561 | */ | 559 | */ |
562 | int try_set_zonelist_oom(struct zonelist *zonelist, gfp_t gfp_mask) | 560 | bool oom_zonelist_trylock(struct zonelist *zonelist, gfp_t gfp_mask) |
563 | { | 561 | { |
564 | struct zoneref *z; | 562 | struct zoneref *z; |
565 | struct zone *zone; | 563 | struct zone *zone; |
566 | int ret = 1; | 564 | bool ret = true; |
567 | 565 | ||
568 | spin_lock(&zone_scan_lock); | 566 | spin_lock(&zone_scan_lock); |
569 | for_each_zone_zonelist(zone, z, zonelist, gfp_zone(gfp_mask)) { | 567 | for_each_zone_zonelist(zone, z, zonelist, gfp_zone(gfp_mask)) |
570 | if (zone_is_oom_locked(zone)) { | 568 | if (zone_is_oom_locked(zone)) { |
571 | ret = 0; | 569 | ret = false; |
572 | goto out; | 570 | goto out; |
573 | } | 571 | } |
574 | } | ||
575 | 572 | ||
576 | for_each_zone_zonelist(zone, z, zonelist, gfp_zone(gfp_mask)) { | 573 | /* |
577 | /* | 574 | * Lock each zone in the zonelist under zone_scan_lock so a parallel |
578 | * Lock each zone in the zonelist under zone_scan_lock so a | 575 | * call to oom_zonelist_trylock() doesn't succeed when it shouldn't. |
579 | * parallel invocation of try_set_zonelist_oom() doesn't succeed | 576 | */ |
580 | * when it shouldn't. | 577 | for_each_zone_zonelist(zone, z, zonelist, gfp_zone(gfp_mask)) |
581 | */ | ||
582 | zone_set_flag(zone, ZONE_OOM_LOCKED); | 578 | zone_set_flag(zone, ZONE_OOM_LOCKED); |
583 | } | ||
584 | 579 | ||
585 | out: | 580 | out: |
586 | spin_unlock(&zone_scan_lock); | 581 | spin_unlock(&zone_scan_lock); |
@@ -592,15 +587,14 @@ out: | |||
592 | * allocation attempts with zonelists containing them may now recall the OOM | 587 | * allocation attempts with zonelists containing them may now recall the OOM |
593 | * killer, if necessary. | 588 | * killer, if necessary. |
594 | */ | 589 | */ |
595 | void clear_zonelist_oom(struct zonelist *zonelist, gfp_t gfp_mask) | 590 | void oom_zonelist_unlock(struct zonelist *zonelist, gfp_t gfp_mask) |
596 | { | 591 | { |
597 | struct zoneref *z; | 592 | struct zoneref *z; |
598 | struct zone *zone; | 593 | struct zone *zone; |
599 | 594 | ||
600 | spin_lock(&zone_scan_lock); | 595 | spin_lock(&zone_scan_lock); |
601 | for_each_zone_zonelist(zone, z, zonelist, gfp_zone(gfp_mask)) { | 596 | for_each_zone_zonelist(zone, z, zonelist, gfp_zone(gfp_mask)) |
602 | zone_clear_flag(zone, ZONE_OOM_LOCKED); | 597 | zone_clear_flag(zone, ZONE_OOM_LOCKED); |
603 | } | ||
604 | spin_unlock(&zone_scan_lock); | 598 | spin_unlock(&zone_scan_lock); |
605 | } | 599 | } |
606 | 600 | ||
@@ -694,9 +688,9 @@ void pagefault_out_of_memory(void) | |||
694 | if (mem_cgroup_oom_synchronize(true)) | 688 | if (mem_cgroup_oom_synchronize(true)) |
695 | return; | 689 | return; |
696 | 690 | ||
697 | zonelist = node_zonelist(first_online_node, GFP_KERNEL); | 691 | zonelist = node_zonelist(first_memory_node, GFP_KERNEL); |
698 | if (try_set_zonelist_oom(zonelist, GFP_KERNEL)) { | 692 | if (oom_zonelist_trylock(zonelist, GFP_KERNEL)) { |
699 | out_of_memory(NULL, 0, 0, NULL, false); | 693 | out_of_memory(NULL, 0, 0, NULL, false); |
700 | clear_zonelist_oom(zonelist, GFP_KERNEL); | 694 | oom_zonelist_unlock(zonelist, GFP_KERNEL); |
701 | } | 695 | } |
702 | } | 696 | } |
diff --git a/mm/page-writeback.c b/mm/page-writeback.c index e0c943014eb7..91d73ef1744d 100644 --- a/mm/page-writeback.c +++ b/mm/page-writeback.c | |||
@@ -261,14 +261,11 @@ static unsigned long global_dirtyable_memory(void) | |||
261 | */ | 261 | */ |
262 | void global_dirty_limits(unsigned long *pbackground, unsigned long *pdirty) | 262 | void global_dirty_limits(unsigned long *pbackground, unsigned long *pdirty) |
263 | { | 263 | { |
264 | const unsigned long available_memory = global_dirtyable_memory(); | ||
264 | unsigned long background; | 265 | unsigned long background; |
265 | unsigned long dirty; | 266 | unsigned long dirty; |
266 | unsigned long uninitialized_var(available_memory); | ||
267 | struct task_struct *tsk; | 267 | struct task_struct *tsk; |
268 | 268 | ||
269 | if (!vm_dirty_bytes || !dirty_background_bytes) | ||
270 | available_memory = global_dirtyable_memory(); | ||
271 | |||
272 | if (vm_dirty_bytes) | 269 | if (vm_dirty_bytes) |
273 | dirty = DIV_ROUND_UP(vm_dirty_bytes, PAGE_SIZE); | 270 | dirty = DIV_ROUND_UP(vm_dirty_bytes, PAGE_SIZE); |
274 | else | 271 | else |
diff --git a/mm/page_alloc.c b/mm/page_alloc.c index ef44ad736ca1..18cee0d4c8a2 100644 --- a/mm/page_alloc.c +++ b/mm/page_alloc.c | |||
@@ -680,9 +680,12 @@ static void free_pcppages_bulk(struct zone *zone, int count, | |||
680 | int migratetype = 0; | 680 | int migratetype = 0; |
681 | int batch_free = 0; | 681 | int batch_free = 0; |
682 | int to_free = count; | 682 | int to_free = count; |
683 | unsigned long nr_scanned; | ||
683 | 684 | ||
684 | spin_lock(&zone->lock); | 685 | spin_lock(&zone->lock); |
685 | zone->pages_scanned = 0; | 686 | nr_scanned = zone_page_state(zone, NR_PAGES_SCANNED); |
687 | if (nr_scanned) | ||
688 | __mod_zone_page_state(zone, NR_PAGES_SCANNED, -nr_scanned); | ||
686 | 689 | ||
687 | while (to_free) { | 690 | while (to_free) { |
688 | struct page *page; | 691 | struct page *page; |
@@ -731,8 +734,11 @@ static void free_one_page(struct zone *zone, | |||
731 | unsigned int order, | 734 | unsigned int order, |
732 | int migratetype) | 735 | int migratetype) |
733 | { | 736 | { |
737 | unsigned long nr_scanned; | ||
734 | spin_lock(&zone->lock); | 738 | spin_lock(&zone->lock); |
735 | zone->pages_scanned = 0; | 739 | nr_scanned = zone_page_state(zone, NR_PAGES_SCANNED); |
740 | if (nr_scanned) | ||
741 | __mod_zone_page_state(zone, NR_PAGES_SCANNED, -nr_scanned); | ||
736 | 742 | ||
737 | __free_one_page(page, pfn, zone, order, migratetype); | 743 | __free_one_page(page, pfn, zone, order, migratetype); |
738 | if (unlikely(!is_migrate_isolate(migratetype))) | 744 | if (unlikely(!is_migrate_isolate(migratetype))) |
@@ -1257,15 +1263,11 @@ static int rmqueue_bulk(struct zone *zone, unsigned int order, | |||
1257 | void drain_zone_pages(struct zone *zone, struct per_cpu_pages *pcp) | 1263 | void drain_zone_pages(struct zone *zone, struct per_cpu_pages *pcp) |
1258 | { | 1264 | { |
1259 | unsigned long flags; | 1265 | unsigned long flags; |
1260 | int to_drain; | 1266 | int to_drain, batch; |
1261 | unsigned long batch; | ||
1262 | 1267 | ||
1263 | local_irq_save(flags); | 1268 | local_irq_save(flags); |
1264 | batch = ACCESS_ONCE(pcp->batch); | 1269 | batch = ACCESS_ONCE(pcp->batch); |
1265 | if (pcp->count >= batch) | 1270 | to_drain = min(pcp->count, batch); |
1266 | to_drain = batch; | ||
1267 | else | ||
1268 | to_drain = pcp->count; | ||
1269 | if (to_drain > 0) { | 1271 | if (to_drain > 0) { |
1270 | free_pcppages_bulk(zone, to_drain, pcp); | 1272 | free_pcppages_bulk(zone, to_drain, pcp); |
1271 | pcp->count -= to_drain; | 1273 | pcp->count -= to_drain; |
@@ -1610,6 +1612,9 @@ again: | |||
1610 | } | 1612 | } |
1611 | 1613 | ||
1612 | __mod_zone_page_state(zone, NR_ALLOC_BATCH, -(1 << order)); | 1614 | __mod_zone_page_state(zone, NR_ALLOC_BATCH, -(1 << order)); |
1615 | if (zone_page_state(zone, NR_ALLOC_BATCH) == 0 && | ||
1616 | !zone_is_fair_depleted(zone)) | ||
1617 | zone_set_flag(zone, ZONE_FAIR_DEPLETED); | ||
1613 | 1618 | ||
1614 | __count_zone_vm_events(PGALLOC, zone, 1 << order); | 1619 | __count_zone_vm_events(PGALLOC, zone, 1 << order); |
1615 | zone_statistics(preferred_zone, zone, gfp_flags); | 1620 | zone_statistics(preferred_zone, zone, gfp_flags); |
@@ -1712,7 +1717,6 @@ static bool __zone_watermark_ok(struct zone *z, unsigned int order, | |||
1712 | { | 1717 | { |
1713 | /* free_pages my go negative - that's OK */ | 1718 | /* free_pages my go negative - that's OK */ |
1714 | long min = mark; | 1719 | long min = mark; |
1715 | long lowmem_reserve = z->lowmem_reserve[classzone_idx]; | ||
1716 | int o; | 1720 | int o; |
1717 | long free_cma = 0; | 1721 | long free_cma = 0; |
1718 | 1722 | ||
@@ -1727,7 +1731,7 @@ static bool __zone_watermark_ok(struct zone *z, unsigned int order, | |||
1727 | free_cma = zone_page_state(z, NR_FREE_CMA_PAGES); | 1731 | free_cma = zone_page_state(z, NR_FREE_CMA_PAGES); |
1728 | #endif | 1732 | #endif |
1729 | 1733 | ||
1730 | if (free_pages - free_cma <= min + lowmem_reserve) | 1734 | if (free_pages - free_cma <= min + z->lowmem_reserve[classzone_idx]) |
1731 | return false; | 1735 | return false; |
1732 | for (o = 0; o < order; o++) { | 1736 | for (o = 0; o < order; o++) { |
1733 | /* At the next order, this order's pages become unavailable */ | 1737 | /* At the next order, this order's pages become unavailable */ |
@@ -1922,6 +1926,18 @@ static bool zone_allows_reclaim(struct zone *local_zone, struct zone *zone) | |||
1922 | 1926 | ||
1923 | #endif /* CONFIG_NUMA */ | 1927 | #endif /* CONFIG_NUMA */ |
1924 | 1928 | ||
1929 | static void reset_alloc_batches(struct zone *preferred_zone) | ||
1930 | { | ||
1931 | struct zone *zone = preferred_zone->zone_pgdat->node_zones; | ||
1932 | |||
1933 | do { | ||
1934 | mod_zone_page_state(zone, NR_ALLOC_BATCH, | ||
1935 | high_wmark_pages(zone) - low_wmark_pages(zone) - | ||
1936 | atomic_long_read(&zone->vm_stat[NR_ALLOC_BATCH])); | ||
1937 | zone_clear_flag(zone, ZONE_FAIR_DEPLETED); | ||
1938 | } while (zone++ != preferred_zone); | ||
1939 | } | ||
1940 | |||
1925 | /* | 1941 | /* |
1926 | * get_page_from_freelist goes through the zonelist trying to allocate | 1942 | * get_page_from_freelist goes through the zonelist trying to allocate |
1927 | * a page. | 1943 | * a page. |
@@ -1939,8 +1955,12 @@ get_page_from_freelist(gfp_t gfp_mask, nodemask_t *nodemask, unsigned int order, | |||
1939 | int did_zlc_setup = 0; /* just call zlc_setup() one time */ | 1955 | int did_zlc_setup = 0; /* just call zlc_setup() one time */ |
1940 | bool consider_zone_dirty = (alloc_flags & ALLOC_WMARK_LOW) && | 1956 | bool consider_zone_dirty = (alloc_flags & ALLOC_WMARK_LOW) && |
1941 | (gfp_mask & __GFP_WRITE); | 1957 | (gfp_mask & __GFP_WRITE); |
1958 | int nr_fair_skipped = 0; | ||
1959 | bool zonelist_rescan; | ||
1942 | 1960 | ||
1943 | zonelist_scan: | 1961 | zonelist_scan: |
1962 | zonelist_rescan = false; | ||
1963 | |||
1944 | /* | 1964 | /* |
1945 | * Scan zonelist, looking for a zone with enough free. | 1965 | * Scan zonelist, looking for a zone with enough free. |
1946 | * See also __cpuset_node_allowed_softwall() comment in kernel/cpuset.c. | 1966 | * See also __cpuset_node_allowed_softwall() comment in kernel/cpuset.c. |
@@ -1964,9 +1984,11 @@ zonelist_scan: | |||
1964 | */ | 1984 | */ |
1965 | if (alloc_flags & ALLOC_FAIR) { | 1985 | if (alloc_flags & ALLOC_FAIR) { |
1966 | if (!zone_local(preferred_zone, zone)) | 1986 | if (!zone_local(preferred_zone, zone)) |
1987 | break; | ||
1988 | if (zone_is_fair_depleted(zone)) { | ||
1989 | nr_fair_skipped++; | ||
1967 | continue; | 1990 | continue; |
1968 | if (zone_page_state(zone, NR_ALLOC_BATCH) <= 0) | 1991 | } |
1969 | continue; | ||
1970 | } | 1992 | } |
1971 | /* | 1993 | /* |
1972 | * When allocating a page cache page for writing, we | 1994 | * When allocating a page cache page for writing, we |
@@ -2072,13 +2094,7 @@ this_zone_full: | |||
2072 | zlc_mark_zone_full(zonelist, z); | 2094 | zlc_mark_zone_full(zonelist, z); |
2073 | } | 2095 | } |
2074 | 2096 | ||
2075 | if (unlikely(IS_ENABLED(CONFIG_NUMA) && page == NULL && zlc_active)) { | 2097 | if (page) { |
2076 | /* Disable zlc cache for second zonelist scan */ | ||
2077 | zlc_active = 0; | ||
2078 | goto zonelist_scan; | ||
2079 | } | ||
2080 | |||
2081 | if (page) | ||
2082 | /* | 2098 | /* |
2083 | * page->pfmemalloc is set when ALLOC_NO_WATERMARKS was | 2099 | * page->pfmemalloc is set when ALLOC_NO_WATERMARKS was |
2084 | * necessary to allocate the page. The expectation is | 2100 | * necessary to allocate the page. The expectation is |
@@ -2087,8 +2103,37 @@ this_zone_full: | |||
2087 | * for !PFMEMALLOC purposes. | 2103 | * for !PFMEMALLOC purposes. |
2088 | */ | 2104 | */ |
2089 | page->pfmemalloc = !!(alloc_flags & ALLOC_NO_WATERMARKS); | 2105 | page->pfmemalloc = !!(alloc_flags & ALLOC_NO_WATERMARKS); |
2106 | return page; | ||
2107 | } | ||
2090 | 2108 | ||
2091 | return page; | 2109 | /* |
2110 | * The first pass makes sure allocations are spread fairly within the | ||
2111 | * local node. However, the local node might have free pages left | ||
2112 | * after the fairness batches are exhausted, and remote zones haven't | ||
2113 | * even been considered yet. Try once more without fairness, and | ||
2114 | * include remote zones now, before entering the slowpath and waking | ||
2115 | * kswapd: prefer spilling to a remote zone over swapping locally. | ||
2116 | */ | ||
2117 | if (alloc_flags & ALLOC_FAIR) { | ||
2118 | alloc_flags &= ~ALLOC_FAIR; | ||
2119 | if (nr_fair_skipped) { | ||
2120 | zonelist_rescan = true; | ||
2121 | reset_alloc_batches(preferred_zone); | ||
2122 | } | ||
2123 | if (nr_online_nodes > 1) | ||
2124 | zonelist_rescan = true; | ||
2125 | } | ||
2126 | |||
2127 | if (unlikely(IS_ENABLED(CONFIG_NUMA) && zlc_active)) { | ||
2128 | /* Disable zlc cache for second zonelist scan */ | ||
2129 | zlc_active = 0; | ||
2130 | zonelist_rescan = true; | ||
2131 | } | ||
2132 | |||
2133 | if (zonelist_rescan) | ||
2134 | goto zonelist_scan; | ||
2135 | |||
2136 | return NULL; | ||
2092 | } | 2137 | } |
2093 | 2138 | ||
2094 | /* | 2139 | /* |
@@ -2201,8 +2246,8 @@ __alloc_pages_may_oom(gfp_t gfp_mask, unsigned int order, | |||
2201 | { | 2246 | { |
2202 | struct page *page; | 2247 | struct page *page; |
2203 | 2248 | ||
2204 | /* Acquire the OOM killer lock for the zones in zonelist */ | 2249 | /* Acquire the per-zone oom lock for each zone */ |
2205 | if (!try_set_zonelist_oom(zonelist, gfp_mask)) { | 2250 | if (!oom_zonelist_trylock(zonelist, gfp_mask)) { |
2206 | schedule_timeout_uninterruptible(1); | 2251 | schedule_timeout_uninterruptible(1); |
2207 | return NULL; | 2252 | return NULL; |
2208 | } | 2253 | } |
@@ -2240,7 +2285,7 @@ __alloc_pages_may_oom(gfp_t gfp_mask, unsigned int order, | |||
2240 | out_of_memory(zonelist, gfp_mask, order, nodemask, false); | 2285 | out_of_memory(zonelist, gfp_mask, order, nodemask, false); |
2241 | 2286 | ||
2242 | out: | 2287 | out: |
2243 | clear_zonelist_oom(zonelist, gfp_mask); | 2288 | oom_zonelist_unlock(zonelist, gfp_mask); |
2244 | return page; | 2289 | return page; |
2245 | } | 2290 | } |
2246 | 2291 | ||
@@ -2409,28 +2454,6 @@ __alloc_pages_high_priority(gfp_t gfp_mask, unsigned int order, | |||
2409 | return page; | 2454 | return page; |
2410 | } | 2455 | } |
2411 | 2456 | ||
2412 | static void reset_alloc_batches(struct zonelist *zonelist, | ||
2413 | enum zone_type high_zoneidx, | ||
2414 | struct zone *preferred_zone) | ||
2415 | { | ||
2416 | struct zoneref *z; | ||
2417 | struct zone *zone; | ||
2418 | |||
2419 | for_each_zone_zonelist(zone, z, zonelist, high_zoneidx) { | ||
2420 | /* | ||
2421 | * Only reset the batches of zones that were actually | ||
2422 | * considered in the fairness pass, we don't want to | ||
2423 | * trash fairness information for zones that are not | ||
2424 | * actually part of this zonelist's round-robin cycle. | ||
2425 | */ | ||
2426 | if (!zone_local(preferred_zone, zone)) | ||
2427 | continue; | ||
2428 | mod_zone_page_state(zone, NR_ALLOC_BATCH, | ||
2429 | high_wmark_pages(zone) - low_wmark_pages(zone) - | ||
2430 | atomic_long_read(&zone->vm_stat[NR_ALLOC_BATCH])); | ||
2431 | } | ||
2432 | } | ||
2433 | |||
2434 | static void wake_all_kswapds(unsigned int order, | 2457 | static void wake_all_kswapds(unsigned int order, |
2435 | struct zonelist *zonelist, | 2458 | struct zonelist *zonelist, |
2436 | enum zone_type high_zoneidx, | 2459 | enum zone_type high_zoneidx, |
@@ -2616,14 +2639,6 @@ rebalance: | |||
2616 | goto got_pg; | 2639 | goto got_pg; |
2617 | 2640 | ||
2618 | /* | 2641 | /* |
2619 | * It can become very expensive to allocate transparent hugepages at | ||
2620 | * fault, so use asynchronous memory compaction for THP unless it is | ||
2621 | * khugepaged trying to collapse. | ||
2622 | */ | ||
2623 | if (!(gfp_mask & __GFP_NO_KSWAPD) || (current->flags & PF_KTHREAD)) | ||
2624 | migration_mode = MIGRATE_SYNC_LIGHT; | ||
2625 | |||
2626 | /* | ||
2627 | * If compaction is deferred for high-order allocations, it is because | 2642 | * If compaction is deferred for high-order allocations, it is because |
2628 | * sync compaction recently failed. In this is the case and the caller | 2643 | * sync compaction recently failed. In this is the case and the caller |
2629 | * requested a movable allocation that does not heavily disrupt the | 2644 | * requested a movable allocation that does not heavily disrupt the |
@@ -2633,6 +2648,15 @@ rebalance: | |||
2633 | (gfp_mask & __GFP_NO_KSWAPD)) | 2648 | (gfp_mask & __GFP_NO_KSWAPD)) |
2634 | goto nopage; | 2649 | goto nopage; |
2635 | 2650 | ||
2651 | /* | ||
2652 | * It can become very expensive to allocate transparent hugepages at | ||
2653 | * fault, so use asynchronous memory compaction for THP unless it is | ||
2654 | * khugepaged trying to collapse. | ||
2655 | */ | ||
2656 | if ((gfp_mask & GFP_TRANSHUGE) != GFP_TRANSHUGE || | ||
2657 | (current->flags & PF_KTHREAD)) | ||
2658 | migration_mode = MIGRATE_SYNC_LIGHT; | ||
2659 | |||
2636 | /* Try direct reclaim and then allocating */ | 2660 | /* Try direct reclaim and then allocating */ |
2637 | page = __alloc_pages_direct_reclaim(gfp_mask, order, | 2661 | page = __alloc_pages_direct_reclaim(gfp_mask, order, |
2638 | zonelist, high_zoneidx, | 2662 | zonelist, high_zoneidx, |
@@ -2766,29 +2790,12 @@ retry_cpuset: | |||
2766 | if (allocflags_to_migratetype(gfp_mask) == MIGRATE_MOVABLE) | 2790 | if (allocflags_to_migratetype(gfp_mask) == MIGRATE_MOVABLE) |
2767 | alloc_flags |= ALLOC_CMA; | 2791 | alloc_flags |= ALLOC_CMA; |
2768 | #endif | 2792 | #endif |
2769 | retry: | ||
2770 | /* First allocation attempt */ | 2793 | /* First allocation attempt */ |
2771 | page = get_page_from_freelist(gfp_mask|__GFP_HARDWALL, nodemask, order, | 2794 | page = get_page_from_freelist(gfp_mask|__GFP_HARDWALL, nodemask, order, |
2772 | zonelist, high_zoneidx, alloc_flags, | 2795 | zonelist, high_zoneidx, alloc_flags, |
2773 | preferred_zone, classzone_idx, migratetype); | 2796 | preferred_zone, classzone_idx, migratetype); |
2774 | if (unlikely(!page)) { | 2797 | if (unlikely(!page)) { |
2775 | /* | 2798 | /* |
2776 | * The first pass makes sure allocations are spread | ||
2777 | * fairly within the local node. However, the local | ||
2778 | * node might have free pages left after the fairness | ||
2779 | * batches are exhausted, and remote zones haven't | ||
2780 | * even been considered yet. Try once more without | ||
2781 | * fairness, and include remote zones now, before | ||
2782 | * entering the slowpath and waking kswapd: prefer | ||
2783 | * spilling to a remote zone over swapping locally. | ||
2784 | */ | ||
2785 | if (alloc_flags & ALLOC_FAIR) { | ||
2786 | reset_alloc_batches(zonelist, high_zoneidx, | ||
2787 | preferred_zone); | ||
2788 | alloc_flags &= ~ALLOC_FAIR; | ||
2789 | goto retry; | ||
2790 | } | ||
2791 | /* | ||
2792 | * Runtime PM, block IO and its error handling path | 2799 | * Runtime PM, block IO and its error handling path |
2793 | * can deadlock because I/O on the device might not | 2800 | * can deadlock because I/O on the device might not |
2794 | * complete. | 2801 | * complete. |
@@ -2962,7 +2969,7 @@ EXPORT_SYMBOL(alloc_pages_exact); | |||
2962 | * Note this is not alloc_pages_exact_node() which allocates on a specific node, | 2969 | * Note this is not alloc_pages_exact_node() which allocates on a specific node, |
2963 | * but is not exact. | 2970 | * but is not exact. |
2964 | */ | 2971 | */ |
2965 | void *alloc_pages_exact_nid(int nid, size_t size, gfp_t gfp_mask) | 2972 | void * __meminit alloc_pages_exact_nid(int nid, size_t size, gfp_t gfp_mask) |
2966 | { | 2973 | { |
2967 | unsigned order = get_order(size); | 2974 | unsigned order = get_order(size); |
2968 | struct page *p = alloc_pages_node(nid, gfp_mask, order); | 2975 | struct page *p = alloc_pages_node(nid, gfp_mask, order); |
@@ -2970,7 +2977,6 @@ void *alloc_pages_exact_nid(int nid, size_t size, gfp_t gfp_mask) | |||
2970 | return NULL; | 2977 | return NULL; |
2971 | return make_alloc_exact((unsigned long)page_address(p), order, size); | 2978 | return make_alloc_exact((unsigned long)page_address(p), order, size); |
2972 | } | 2979 | } |
2973 | EXPORT_SYMBOL(alloc_pages_exact_nid); | ||
2974 | 2980 | ||
2975 | /** | 2981 | /** |
2976 | * free_pages_exact - release memory allocated via alloc_pages_exact() | 2982 | * free_pages_exact - release memory allocated via alloc_pages_exact() |
@@ -3052,7 +3058,7 @@ static inline void show_node(struct zone *zone) | |||
3052 | void si_meminfo(struct sysinfo *val) | 3058 | void si_meminfo(struct sysinfo *val) |
3053 | { | 3059 | { |
3054 | val->totalram = totalram_pages; | 3060 | val->totalram = totalram_pages; |
3055 | val->sharedram = 0; | 3061 | val->sharedram = global_page_state(NR_SHMEM); |
3056 | val->freeram = global_page_state(NR_FREE_PAGES); | 3062 | val->freeram = global_page_state(NR_FREE_PAGES); |
3057 | val->bufferram = nr_blockdev_pages(); | 3063 | val->bufferram = nr_blockdev_pages(); |
3058 | val->totalhigh = totalhigh_pages; | 3064 | val->totalhigh = totalhigh_pages; |
@@ -3072,6 +3078,7 @@ void si_meminfo_node(struct sysinfo *val, int nid) | |||
3072 | for (zone_type = 0; zone_type < MAX_NR_ZONES; zone_type++) | 3078 | for (zone_type = 0; zone_type < MAX_NR_ZONES; zone_type++) |
3073 | managed_pages += pgdat->node_zones[zone_type].managed_pages; | 3079 | managed_pages += pgdat->node_zones[zone_type].managed_pages; |
3074 | val->totalram = managed_pages; | 3080 | val->totalram = managed_pages; |
3081 | val->sharedram = node_page_state(nid, NR_SHMEM); | ||
3075 | val->freeram = node_page_state(nid, NR_FREE_PAGES); | 3082 | val->freeram = node_page_state(nid, NR_FREE_PAGES); |
3076 | #ifdef CONFIG_HIGHMEM | 3083 | #ifdef CONFIG_HIGHMEM |
3077 | val->totalhigh = pgdat->node_zones[ZONE_HIGHMEM].managed_pages; | 3084 | val->totalhigh = pgdat->node_zones[ZONE_HIGHMEM].managed_pages; |
@@ -3253,12 +3260,12 @@ void show_free_areas(unsigned int filter) | |||
3253 | K(zone_page_state(zone, NR_BOUNCE)), | 3260 | K(zone_page_state(zone, NR_BOUNCE)), |
3254 | K(zone_page_state(zone, NR_FREE_CMA_PAGES)), | 3261 | K(zone_page_state(zone, NR_FREE_CMA_PAGES)), |
3255 | K(zone_page_state(zone, NR_WRITEBACK_TEMP)), | 3262 | K(zone_page_state(zone, NR_WRITEBACK_TEMP)), |
3256 | zone->pages_scanned, | 3263 | K(zone_page_state(zone, NR_PAGES_SCANNED)), |
3257 | (!zone_reclaimable(zone) ? "yes" : "no") | 3264 | (!zone_reclaimable(zone) ? "yes" : "no") |
3258 | ); | 3265 | ); |
3259 | printk("lowmem_reserve[]:"); | 3266 | printk("lowmem_reserve[]:"); |
3260 | for (i = 0; i < MAX_NR_ZONES; i++) | 3267 | for (i = 0; i < MAX_NR_ZONES; i++) |
3261 | printk(" %lu", zone->lowmem_reserve[i]); | 3268 | printk(" %ld", zone->lowmem_reserve[i]); |
3262 | printk("\n"); | 3269 | printk("\n"); |
3263 | } | 3270 | } |
3264 | 3271 | ||
@@ -5579,7 +5586,7 @@ static void calculate_totalreserve_pages(void) | |||
5579 | for_each_online_pgdat(pgdat) { | 5586 | for_each_online_pgdat(pgdat) { |
5580 | for (i = 0; i < MAX_NR_ZONES; i++) { | 5587 | for (i = 0; i < MAX_NR_ZONES; i++) { |
5581 | struct zone *zone = pgdat->node_zones + i; | 5588 | struct zone *zone = pgdat->node_zones + i; |
5582 | unsigned long max = 0; | 5589 | long max = 0; |
5583 | 5590 | ||
5584 | /* Find valid and maximum lowmem_reserve in the zone */ | 5591 | /* Find valid and maximum lowmem_reserve in the zone */ |
5585 | for (j = i; j < MAX_NR_ZONES; j++) { | 5592 | for (j = i; j < MAX_NR_ZONES; j++) { |
diff --git a/mm/readahead.c b/mm/readahead.c index 0ca36a7770b1..17b9172ec37f 100644 --- a/mm/readahead.c +++ b/mm/readahead.c | |||
@@ -326,7 +326,6 @@ static unsigned long get_next_ra_size(struct file_ra_state *ra, | |||
326 | * - thrashing threshold in memory tight systems | 326 | * - thrashing threshold in memory tight systems |
327 | */ | 327 | */ |
328 | static pgoff_t count_history_pages(struct address_space *mapping, | 328 | static pgoff_t count_history_pages(struct address_space *mapping, |
329 | struct file_ra_state *ra, | ||
330 | pgoff_t offset, unsigned long max) | 329 | pgoff_t offset, unsigned long max) |
331 | { | 330 | { |
332 | pgoff_t head; | 331 | pgoff_t head; |
@@ -349,7 +348,7 @@ static int try_context_readahead(struct address_space *mapping, | |||
349 | { | 348 | { |
350 | pgoff_t size; | 349 | pgoff_t size; |
351 | 350 | ||
352 | size = count_history_pages(mapping, ra, offset, max); | 351 | size = count_history_pages(mapping, offset, max); |
353 | 352 | ||
354 | /* | 353 | /* |
355 | * not enough history pages: | 354 | * not enough history pages: |
diff --git a/mm/shmem.c b/mm/shmem.c index af68b15a8fc1..302d1cf7ad07 100644 --- a/mm/shmem.c +++ b/mm/shmem.c | |||
@@ -149,6 +149,19 @@ static inline void shmem_unacct_size(unsigned long flags, loff_t size) | |||
149 | vm_unacct_memory(VM_ACCT(size)); | 149 | vm_unacct_memory(VM_ACCT(size)); |
150 | } | 150 | } |
151 | 151 | ||
152 | static inline int shmem_reacct_size(unsigned long flags, | ||
153 | loff_t oldsize, loff_t newsize) | ||
154 | { | ||
155 | if (!(flags & VM_NORESERVE)) { | ||
156 | if (VM_ACCT(newsize) > VM_ACCT(oldsize)) | ||
157 | return security_vm_enough_memory_mm(current->mm, | ||
158 | VM_ACCT(newsize) - VM_ACCT(oldsize)); | ||
159 | else if (VM_ACCT(newsize) < VM_ACCT(oldsize)) | ||
160 | vm_unacct_memory(VM_ACCT(oldsize) - VM_ACCT(newsize)); | ||
161 | } | ||
162 | return 0; | ||
163 | } | ||
164 | |||
152 | /* | 165 | /* |
153 | * ... whereas tmpfs objects are accounted incrementally as | 166 | * ... whereas tmpfs objects are accounted incrementally as |
154 | * pages are allocated, in order to allow huge sparse files. | 167 | * pages are allocated, in order to allow huge sparse files. |
@@ -280,7 +293,7 @@ static bool shmem_confirm_swap(struct address_space *mapping, | |||
280 | */ | 293 | */ |
281 | static int shmem_add_to_page_cache(struct page *page, | 294 | static int shmem_add_to_page_cache(struct page *page, |
282 | struct address_space *mapping, | 295 | struct address_space *mapping, |
283 | pgoff_t index, gfp_t gfp, void *expected) | 296 | pgoff_t index, void *expected) |
284 | { | 297 | { |
285 | int error; | 298 | int error; |
286 | 299 | ||
@@ -549,6 +562,10 @@ static int shmem_setattr(struct dentry *dentry, struct iattr *attr) | |||
549 | loff_t newsize = attr->ia_size; | 562 | loff_t newsize = attr->ia_size; |
550 | 563 | ||
551 | if (newsize != oldsize) { | 564 | if (newsize != oldsize) { |
565 | error = shmem_reacct_size(SHMEM_I(inode)->flags, | ||
566 | oldsize, newsize); | ||
567 | if (error) | ||
568 | return error; | ||
552 | i_size_write(inode, newsize); | 569 | i_size_write(inode, newsize); |
553 | inode->i_ctime = inode->i_mtime = CURRENT_TIME; | 570 | inode->i_ctime = inode->i_mtime = CURRENT_TIME; |
554 | } | 571 | } |
@@ -649,7 +666,7 @@ static int shmem_unuse_inode(struct shmem_inode_info *info, | |||
649 | */ | 666 | */ |
650 | if (!error) | 667 | if (!error) |
651 | error = shmem_add_to_page_cache(*pagep, mapping, index, | 668 | error = shmem_add_to_page_cache(*pagep, mapping, index, |
652 | GFP_NOWAIT, radswap); | 669 | radswap); |
653 | if (error != -ENOMEM) { | 670 | if (error != -ENOMEM) { |
654 | /* | 671 | /* |
655 | * Truncation and eviction use free_swap_and_cache(), which | 672 | * Truncation and eviction use free_swap_and_cache(), which |
@@ -1095,7 +1112,7 @@ repeat: | |||
1095 | gfp & GFP_RECLAIM_MASK); | 1112 | gfp & GFP_RECLAIM_MASK); |
1096 | if (!error) { | 1113 | if (!error) { |
1097 | error = shmem_add_to_page_cache(page, mapping, index, | 1114 | error = shmem_add_to_page_cache(page, mapping, index, |
1098 | gfp, swp_to_radix_entry(swap)); | 1115 | swp_to_radix_entry(swap)); |
1099 | /* | 1116 | /* |
1100 | * We already confirmed swap under page lock, and make | 1117 | * We already confirmed swap under page lock, and make |
1101 | * no memory allocation here, so usually no possibility | 1118 | * no memory allocation here, so usually no possibility |
@@ -1149,7 +1166,7 @@ repeat: | |||
1149 | __SetPageSwapBacked(page); | 1166 | __SetPageSwapBacked(page); |
1150 | __set_page_locked(page); | 1167 | __set_page_locked(page); |
1151 | if (sgp == SGP_WRITE) | 1168 | if (sgp == SGP_WRITE) |
1152 | init_page_accessed(page); | 1169 | __SetPageReferenced(page); |
1153 | 1170 | ||
1154 | error = mem_cgroup_charge_file(page, current->mm, | 1171 | error = mem_cgroup_charge_file(page, current->mm, |
1155 | gfp & GFP_RECLAIM_MASK); | 1172 | gfp & GFP_RECLAIM_MASK); |
@@ -1158,7 +1175,7 @@ repeat: | |||
1158 | error = radix_tree_maybe_preload(gfp & GFP_RECLAIM_MASK); | 1175 | error = radix_tree_maybe_preload(gfp & GFP_RECLAIM_MASK); |
1159 | if (!error) { | 1176 | if (!error) { |
1160 | error = shmem_add_to_page_cache(page, mapping, index, | 1177 | error = shmem_add_to_page_cache(page, mapping, index, |
1161 | gfp, NULL); | 1178 | NULL); |
1162 | radix_tree_preload_end(); | 1179 | radix_tree_preload_end(); |
1163 | } | 1180 | } |
1164 | if (error) { | 1181 | if (error) { |
@@ -2932,16 +2949,16 @@ static struct file *__shmem_file_setup(const char *name, loff_t size, | |||
2932 | this.len = strlen(name); | 2949 | this.len = strlen(name); |
2933 | this.hash = 0; /* will go */ | 2950 | this.hash = 0; /* will go */ |
2934 | sb = shm_mnt->mnt_sb; | 2951 | sb = shm_mnt->mnt_sb; |
2952 | path.mnt = mntget(shm_mnt); | ||
2935 | path.dentry = d_alloc_pseudo(sb, &this); | 2953 | path.dentry = d_alloc_pseudo(sb, &this); |
2936 | if (!path.dentry) | 2954 | if (!path.dentry) |
2937 | goto put_memory; | 2955 | goto put_memory; |
2938 | d_set_d_op(path.dentry, &anon_ops); | 2956 | d_set_d_op(path.dentry, &anon_ops); |
2939 | path.mnt = mntget(shm_mnt); | ||
2940 | 2957 | ||
2941 | res = ERR_PTR(-ENOSPC); | 2958 | res = ERR_PTR(-ENOSPC); |
2942 | inode = shmem_get_inode(sb, NULL, S_IFREG | S_IRWXUGO, 0, flags); | 2959 | inode = shmem_get_inode(sb, NULL, S_IFREG | S_IRWXUGO, 0, flags); |
2943 | if (!inode) | 2960 | if (!inode) |
2944 | goto put_dentry; | 2961 | goto put_memory; |
2945 | 2962 | ||
2946 | inode->i_flags |= i_flags; | 2963 | inode->i_flags |= i_flags; |
2947 | d_instantiate(path.dentry, inode); | 2964 | d_instantiate(path.dentry, inode); |
@@ -2949,19 +2966,19 @@ static struct file *__shmem_file_setup(const char *name, loff_t size, | |||
2949 | clear_nlink(inode); /* It is unlinked */ | 2966 | clear_nlink(inode); /* It is unlinked */ |
2950 | res = ERR_PTR(ramfs_nommu_expand_for_mapping(inode, size)); | 2967 | res = ERR_PTR(ramfs_nommu_expand_for_mapping(inode, size)); |
2951 | if (IS_ERR(res)) | 2968 | if (IS_ERR(res)) |
2952 | goto put_dentry; | 2969 | goto put_path; |
2953 | 2970 | ||
2954 | res = alloc_file(&path, FMODE_WRITE | FMODE_READ, | 2971 | res = alloc_file(&path, FMODE_WRITE | FMODE_READ, |
2955 | &shmem_file_operations); | 2972 | &shmem_file_operations); |
2956 | if (IS_ERR(res)) | 2973 | if (IS_ERR(res)) |
2957 | goto put_dentry; | 2974 | goto put_path; |
2958 | 2975 | ||
2959 | return res; | 2976 | return res; |
2960 | 2977 | ||
2961 | put_dentry: | ||
2962 | path_put(&path); | ||
2963 | put_memory: | 2978 | put_memory: |
2964 | shmem_unacct_size(flags, size); | 2979 | shmem_unacct_size(flags, size); |
2980 | put_path: | ||
2981 | path_put(&path); | ||
2965 | return res; | 2982 | return res; |
2966 | } | 2983 | } |
2967 | 2984 | ||
@@ -191,7 +191,6 @@ struct array_cache { | |||
191 | unsigned int limit; | 191 | unsigned int limit; |
192 | unsigned int batchcount; | 192 | unsigned int batchcount; |
193 | unsigned int touched; | 193 | unsigned int touched; |
194 | spinlock_t lock; | ||
195 | void *entry[]; /* | 194 | void *entry[]; /* |
196 | * Must have this definition in here for the proper | 195 | * Must have this definition in here for the proper |
197 | * alignment of array_cache. Also simplifies accessing | 196 | * alignment of array_cache. Also simplifies accessing |
@@ -203,6 +202,11 @@ struct array_cache { | |||
203 | */ | 202 | */ |
204 | }; | 203 | }; |
205 | 204 | ||
205 | struct alien_cache { | ||
206 | spinlock_t lock; | ||
207 | struct array_cache ac; | ||
208 | }; | ||
209 | |||
206 | #define SLAB_OBJ_PFMEMALLOC 1 | 210 | #define SLAB_OBJ_PFMEMALLOC 1 |
207 | static inline bool is_obj_pfmemalloc(void *objp) | 211 | static inline bool is_obj_pfmemalloc(void *objp) |
208 | { | 212 | { |
@@ -242,7 +246,8 @@ static struct kmem_cache_node __initdata init_kmem_cache_node[NUM_INIT_LISTS]; | |||
242 | static int drain_freelist(struct kmem_cache *cache, | 246 | static int drain_freelist(struct kmem_cache *cache, |
243 | struct kmem_cache_node *n, int tofree); | 247 | struct kmem_cache_node *n, int tofree); |
244 | static void free_block(struct kmem_cache *cachep, void **objpp, int len, | 248 | static void free_block(struct kmem_cache *cachep, void **objpp, int len, |
245 | int node); | 249 | int node, struct list_head *list); |
250 | static void slabs_destroy(struct kmem_cache *cachep, struct list_head *list); | ||
246 | static int enable_cpucache(struct kmem_cache *cachep, gfp_t gfp); | 251 | static int enable_cpucache(struct kmem_cache *cachep, gfp_t gfp); |
247 | static void cache_reap(struct work_struct *unused); | 252 | static void cache_reap(struct work_struct *unused); |
248 | 253 | ||
@@ -267,7 +272,7 @@ static void kmem_cache_node_init(struct kmem_cache_node *parent) | |||
267 | #define MAKE_LIST(cachep, listp, slab, nodeid) \ | 272 | #define MAKE_LIST(cachep, listp, slab, nodeid) \ |
268 | do { \ | 273 | do { \ |
269 | INIT_LIST_HEAD(listp); \ | 274 | INIT_LIST_HEAD(listp); \ |
270 | list_splice(&(cachep->node[nodeid]->slab), listp); \ | 275 | list_splice(&get_node(cachep, nodeid)->slab, listp); \ |
271 | } while (0) | 276 | } while (0) |
272 | 277 | ||
273 | #define MAKE_ALL_LISTS(cachep, ptr, nodeid) \ | 278 | #define MAKE_ALL_LISTS(cachep, ptr, nodeid) \ |
@@ -465,143 +470,6 @@ static struct kmem_cache kmem_cache_boot = { | |||
465 | .name = "kmem_cache", | 470 | .name = "kmem_cache", |
466 | }; | 471 | }; |
467 | 472 | ||
468 | #define BAD_ALIEN_MAGIC 0x01020304ul | ||
469 | |||
470 | #ifdef CONFIG_LOCKDEP | ||
471 | |||
472 | /* | ||
473 | * Slab sometimes uses the kmalloc slabs to store the slab headers | ||
474 | * for other slabs "off slab". | ||
475 | * The locking for this is tricky in that it nests within the locks | ||
476 | * of all other slabs in a few places; to deal with this special | ||
477 | * locking we put on-slab caches into a separate lock-class. | ||
478 | * | ||
479 | * We set lock class for alien array caches which are up during init. | ||
480 | * The lock annotation will be lost if all cpus of a node goes down and | ||
481 | * then comes back up during hotplug | ||
482 | */ | ||
483 | static struct lock_class_key on_slab_l3_key; | ||
484 | static struct lock_class_key on_slab_alc_key; | ||
485 | |||
486 | static struct lock_class_key debugobj_l3_key; | ||
487 | static struct lock_class_key debugobj_alc_key; | ||
488 | |||
489 | static void slab_set_lock_classes(struct kmem_cache *cachep, | ||
490 | struct lock_class_key *l3_key, struct lock_class_key *alc_key, | ||
491 | int q) | ||
492 | { | ||
493 | struct array_cache **alc; | ||
494 | struct kmem_cache_node *n; | ||
495 | int r; | ||
496 | |||
497 | n = cachep->node[q]; | ||
498 | if (!n) | ||
499 | return; | ||
500 | |||
501 | lockdep_set_class(&n->list_lock, l3_key); | ||
502 | alc = n->alien; | ||
503 | /* | ||
504 | * FIXME: This check for BAD_ALIEN_MAGIC | ||
505 | * should go away when common slab code is taught to | ||
506 | * work even without alien caches. | ||
507 | * Currently, non NUMA code returns BAD_ALIEN_MAGIC | ||
508 | * for alloc_alien_cache, | ||
509 | */ | ||
510 | if (!alc || (unsigned long)alc == BAD_ALIEN_MAGIC) | ||
511 | return; | ||
512 | for_each_node(r) { | ||
513 | if (alc[r]) | ||
514 | lockdep_set_class(&alc[r]->lock, alc_key); | ||
515 | } | ||
516 | } | ||
517 | |||
518 | static void slab_set_debugobj_lock_classes_node(struct kmem_cache *cachep, int node) | ||
519 | { | ||
520 | slab_set_lock_classes(cachep, &debugobj_l3_key, &debugobj_alc_key, node); | ||
521 | } | ||
522 | |||
523 | static void slab_set_debugobj_lock_classes(struct kmem_cache *cachep) | ||
524 | { | ||
525 | int node; | ||
526 | |||
527 | for_each_online_node(node) | ||
528 | slab_set_debugobj_lock_classes_node(cachep, node); | ||
529 | } | ||
530 | |||
531 | static void init_node_lock_keys(int q) | ||
532 | { | ||
533 | int i; | ||
534 | |||
535 | if (slab_state < UP) | ||
536 | return; | ||
537 | |||
538 | for (i = 1; i <= KMALLOC_SHIFT_HIGH; i++) { | ||
539 | struct kmem_cache_node *n; | ||
540 | struct kmem_cache *cache = kmalloc_caches[i]; | ||
541 | |||
542 | if (!cache) | ||
543 | continue; | ||
544 | |||
545 | n = cache->node[q]; | ||
546 | if (!n || OFF_SLAB(cache)) | ||
547 | continue; | ||
548 | |||
549 | slab_set_lock_classes(cache, &on_slab_l3_key, | ||
550 | &on_slab_alc_key, q); | ||
551 | } | ||
552 | } | ||
553 | |||
554 | static void on_slab_lock_classes_node(struct kmem_cache *cachep, int q) | ||
555 | { | ||
556 | if (!cachep->node[q]) | ||
557 | return; | ||
558 | |||
559 | slab_set_lock_classes(cachep, &on_slab_l3_key, | ||
560 | &on_slab_alc_key, q); | ||
561 | } | ||
562 | |||
563 | static inline void on_slab_lock_classes(struct kmem_cache *cachep) | ||
564 | { | ||
565 | int node; | ||
566 | |||
567 | VM_BUG_ON(OFF_SLAB(cachep)); | ||
568 | for_each_node(node) | ||
569 | on_slab_lock_classes_node(cachep, node); | ||
570 | } | ||
571 | |||
572 | static inline void init_lock_keys(void) | ||
573 | { | ||
574 | int node; | ||
575 | |||
576 | for_each_node(node) | ||
577 | init_node_lock_keys(node); | ||
578 | } | ||
579 | #else | ||
580 | static void init_node_lock_keys(int q) | ||
581 | { | ||
582 | } | ||
583 | |||
584 | static inline void init_lock_keys(void) | ||
585 | { | ||
586 | } | ||
587 | |||
588 | static inline void on_slab_lock_classes(struct kmem_cache *cachep) | ||
589 | { | ||
590 | } | ||
591 | |||
592 | static inline void on_slab_lock_classes_node(struct kmem_cache *cachep, int node) | ||
593 | { | ||
594 | } | ||
595 | |||
596 | static void slab_set_debugobj_lock_classes_node(struct kmem_cache *cachep, int node) | ||
597 | { | ||
598 | } | ||
599 | |||
600 | static void slab_set_debugobj_lock_classes(struct kmem_cache *cachep) | ||
601 | { | ||
602 | } | ||
603 | #endif | ||
604 | |||
605 | static DEFINE_PER_CPU(struct delayed_work, slab_reap_work); | 473 | static DEFINE_PER_CPU(struct delayed_work, slab_reap_work); |
606 | 474 | ||
607 | static inline struct array_cache *cpu_cache_get(struct kmem_cache *cachep) | 475 | static inline struct array_cache *cpu_cache_get(struct kmem_cache *cachep) |
@@ -792,13 +660,8 @@ static void start_cpu_timer(int cpu) | |||
792 | } | 660 | } |
793 | } | 661 | } |
794 | 662 | ||
795 | static struct array_cache *alloc_arraycache(int node, int entries, | 663 | static void init_arraycache(struct array_cache *ac, int limit, int batch) |
796 | int batchcount, gfp_t gfp) | ||
797 | { | 664 | { |
798 | int memsize = sizeof(void *) * entries + sizeof(struct array_cache); | ||
799 | struct array_cache *nc = NULL; | ||
800 | |||
801 | nc = kmalloc_node(memsize, gfp, node); | ||
802 | /* | 665 | /* |
803 | * The array_cache structures contain pointers to free object. | 666 | * The array_cache structures contain pointers to free object. |
804 | * However, when such objects are allocated or transferred to another | 667 | * However, when such objects are allocated or transferred to another |
@@ -806,15 +669,24 @@ static struct array_cache *alloc_arraycache(int node, int entries, | |||
806 | * valid references during a kmemleak scan. Therefore, kmemleak must | 669 | * valid references during a kmemleak scan. Therefore, kmemleak must |
807 | * not scan such objects. | 670 | * not scan such objects. |
808 | */ | 671 | */ |
809 | kmemleak_no_scan(nc); | 672 | kmemleak_no_scan(ac); |
810 | if (nc) { | 673 | if (ac) { |
811 | nc->avail = 0; | 674 | ac->avail = 0; |
812 | nc->limit = entries; | 675 | ac->limit = limit; |
813 | nc->batchcount = batchcount; | 676 | ac->batchcount = batch; |
814 | nc->touched = 0; | 677 | ac->touched = 0; |
815 | spin_lock_init(&nc->lock); | ||
816 | } | 678 | } |
817 | return nc; | 679 | } |
680 | |||
681 | static struct array_cache *alloc_arraycache(int node, int entries, | ||
682 | int batchcount, gfp_t gfp) | ||
683 | { | ||
684 | size_t memsize = sizeof(void *) * entries + sizeof(struct array_cache); | ||
685 | struct array_cache *ac = NULL; | ||
686 | |||
687 | ac = kmalloc_node(memsize, gfp, node); | ||
688 | init_arraycache(ac, entries, batchcount); | ||
689 | return ac; | ||
818 | } | 690 | } |
819 | 691 | ||
820 | static inline bool is_slab_pfmemalloc(struct page *page) | 692 | static inline bool is_slab_pfmemalloc(struct page *page) |
@@ -826,7 +698,7 @@ static inline bool is_slab_pfmemalloc(struct page *page) | |||
826 | static void recheck_pfmemalloc_active(struct kmem_cache *cachep, | 698 | static void recheck_pfmemalloc_active(struct kmem_cache *cachep, |
827 | struct array_cache *ac) | 699 | struct array_cache *ac) |
828 | { | 700 | { |
829 | struct kmem_cache_node *n = cachep->node[numa_mem_id()]; | 701 | struct kmem_cache_node *n = get_node(cachep, numa_mem_id()); |
830 | struct page *page; | 702 | struct page *page; |
831 | unsigned long flags; | 703 | unsigned long flags; |
832 | 704 | ||
@@ -881,7 +753,7 @@ static void *__ac_get_obj(struct kmem_cache *cachep, struct array_cache *ac, | |||
881 | * If there are empty slabs on the slabs_free list and we are | 753 | * If there are empty slabs on the slabs_free list and we are |
882 | * being forced to refill the cache, mark this one !pfmemalloc. | 754 | * being forced to refill the cache, mark this one !pfmemalloc. |
883 | */ | 755 | */ |
884 | n = cachep->node[numa_mem_id()]; | 756 | n = get_node(cachep, numa_mem_id()); |
885 | if (!list_empty(&n->slabs_free) && force_refill) { | 757 | if (!list_empty(&n->slabs_free) && force_refill) { |
886 | struct page *page = virt_to_head_page(objp); | 758 | struct page *page = virt_to_head_page(objp); |
887 | ClearPageSlabPfmemalloc(page); | 759 | ClearPageSlabPfmemalloc(page); |
@@ -961,12 +833,13 @@ static int transfer_objects(struct array_cache *to, | |||
961 | #define drain_alien_cache(cachep, alien) do { } while (0) | 833 | #define drain_alien_cache(cachep, alien) do { } while (0) |
962 | #define reap_alien(cachep, n) do { } while (0) | 834 | #define reap_alien(cachep, n) do { } while (0) |
963 | 835 | ||
964 | static inline struct array_cache **alloc_alien_cache(int node, int limit, gfp_t gfp) | 836 | static inline struct alien_cache **alloc_alien_cache(int node, |
837 | int limit, gfp_t gfp) | ||
965 | { | 838 | { |
966 | return (struct array_cache **)BAD_ALIEN_MAGIC; | 839 | return NULL; |
967 | } | 840 | } |
968 | 841 | ||
969 | static inline void free_alien_cache(struct array_cache **ac_ptr) | 842 | static inline void free_alien_cache(struct alien_cache **ac_ptr) |
970 | { | 843 | { |
971 | } | 844 | } |
972 | 845 | ||
@@ -992,46 +865,60 @@ static inline void *____cache_alloc_node(struct kmem_cache *cachep, | |||
992 | static void *____cache_alloc_node(struct kmem_cache *, gfp_t, int); | 865 | static void *____cache_alloc_node(struct kmem_cache *, gfp_t, int); |
993 | static void *alternate_node_alloc(struct kmem_cache *, gfp_t); | 866 | static void *alternate_node_alloc(struct kmem_cache *, gfp_t); |
994 | 867 | ||
995 | static struct array_cache **alloc_alien_cache(int node, int limit, gfp_t gfp) | 868 | static struct alien_cache *__alloc_alien_cache(int node, int entries, |
869 | int batch, gfp_t gfp) | ||
870 | { | ||
871 | size_t memsize = sizeof(void *) * entries + sizeof(struct alien_cache); | ||
872 | struct alien_cache *alc = NULL; | ||
873 | |||
874 | alc = kmalloc_node(memsize, gfp, node); | ||
875 | init_arraycache(&alc->ac, entries, batch); | ||
876 | spin_lock_init(&alc->lock); | ||
877 | return alc; | ||
878 | } | ||
879 | |||
880 | static struct alien_cache **alloc_alien_cache(int node, int limit, gfp_t gfp) | ||
996 | { | 881 | { |
997 | struct array_cache **ac_ptr; | 882 | struct alien_cache **alc_ptr; |
998 | int memsize = sizeof(void *) * nr_node_ids; | 883 | size_t memsize = sizeof(void *) * nr_node_ids; |
999 | int i; | 884 | int i; |
1000 | 885 | ||
1001 | if (limit > 1) | 886 | if (limit > 1) |
1002 | limit = 12; | 887 | limit = 12; |
1003 | ac_ptr = kzalloc_node(memsize, gfp, node); | 888 | alc_ptr = kzalloc_node(memsize, gfp, node); |
1004 | if (ac_ptr) { | 889 | if (!alc_ptr) |
1005 | for_each_node(i) { | 890 | return NULL; |
1006 | if (i == node || !node_online(i)) | 891 | |
1007 | continue; | 892 | for_each_node(i) { |
1008 | ac_ptr[i] = alloc_arraycache(node, limit, 0xbaadf00d, gfp); | 893 | if (i == node || !node_online(i)) |
1009 | if (!ac_ptr[i]) { | 894 | continue; |
1010 | for (i--; i >= 0; i--) | 895 | alc_ptr[i] = __alloc_alien_cache(node, limit, 0xbaadf00d, gfp); |
1011 | kfree(ac_ptr[i]); | 896 | if (!alc_ptr[i]) { |
1012 | kfree(ac_ptr); | 897 | for (i--; i >= 0; i--) |
1013 | return NULL; | 898 | kfree(alc_ptr[i]); |
1014 | } | 899 | kfree(alc_ptr); |
900 | return NULL; | ||
1015 | } | 901 | } |
1016 | } | 902 | } |
1017 | return ac_ptr; | 903 | return alc_ptr; |
1018 | } | 904 | } |
1019 | 905 | ||
1020 | static void free_alien_cache(struct array_cache **ac_ptr) | 906 | static void free_alien_cache(struct alien_cache **alc_ptr) |
1021 | { | 907 | { |
1022 | int i; | 908 | int i; |
1023 | 909 | ||
1024 | if (!ac_ptr) | 910 | if (!alc_ptr) |
1025 | return; | 911 | return; |
1026 | for_each_node(i) | 912 | for_each_node(i) |
1027 | kfree(ac_ptr[i]); | 913 | kfree(alc_ptr[i]); |
1028 | kfree(ac_ptr); | 914 | kfree(alc_ptr); |
1029 | } | 915 | } |
1030 | 916 | ||
1031 | static void __drain_alien_cache(struct kmem_cache *cachep, | 917 | static void __drain_alien_cache(struct kmem_cache *cachep, |
1032 | struct array_cache *ac, int node) | 918 | struct array_cache *ac, int node, |
919 | struct list_head *list) | ||
1033 | { | 920 | { |
1034 | struct kmem_cache_node *n = cachep->node[node]; | 921 | struct kmem_cache_node *n = get_node(cachep, node); |
1035 | 922 | ||
1036 | if (ac->avail) { | 923 | if (ac->avail) { |
1037 | spin_lock(&n->list_lock); | 924 | spin_lock(&n->list_lock); |
@@ -1043,7 +930,7 @@ static void __drain_alien_cache(struct kmem_cache *cachep, | |||
1043 | if (n->shared) | 930 | if (n->shared) |
1044 | transfer_objects(n->shared, ac, ac->limit); | 931 | transfer_objects(n->shared, ac, ac->limit); |
1045 | 932 | ||
1046 | free_block(cachep, ac->entry, ac->avail, node); | 933 | free_block(cachep, ac->entry, ac->avail, node, list); |
1047 | ac->avail = 0; | 934 | ac->avail = 0; |
1048 | spin_unlock(&n->list_lock); | 935 | spin_unlock(&n->list_lock); |
1049 | } | 936 | } |
@@ -1057,28 +944,40 @@ static void reap_alien(struct kmem_cache *cachep, struct kmem_cache_node *n) | |||
1057 | int node = __this_cpu_read(slab_reap_node); | 944 | int node = __this_cpu_read(slab_reap_node); |
1058 | 945 | ||
1059 | if (n->alien) { | 946 | if (n->alien) { |
1060 | struct array_cache *ac = n->alien[node]; | 947 | struct alien_cache *alc = n->alien[node]; |
948 | struct array_cache *ac; | ||
949 | |||
950 | if (alc) { | ||
951 | ac = &alc->ac; | ||
952 | if (ac->avail && spin_trylock_irq(&alc->lock)) { | ||
953 | LIST_HEAD(list); | ||
1061 | 954 | ||
1062 | if (ac && ac->avail && spin_trylock_irq(&ac->lock)) { | 955 | __drain_alien_cache(cachep, ac, node, &list); |
1063 | __drain_alien_cache(cachep, ac, node); | 956 | spin_unlock_irq(&alc->lock); |
1064 | spin_unlock_irq(&ac->lock); | 957 | slabs_destroy(cachep, &list); |
958 | } | ||
1065 | } | 959 | } |
1066 | } | 960 | } |
1067 | } | 961 | } |
1068 | 962 | ||
1069 | static void drain_alien_cache(struct kmem_cache *cachep, | 963 | static void drain_alien_cache(struct kmem_cache *cachep, |
1070 | struct array_cache **alien) | 964 | struct alien_cache **alien) |
1071 | { | 965 | { |
1072 | int i = 0; | 966 | int i = 0; |
967 | struct alien_cache *alc; | ||
1073 | struct array_cache *ac; | 968 | struct array_cache *ac; |
1074 | unsigned long flags; | 969 | unsigned long flags; |
1075 | 970 | ||
1076 | for_each_online_node(i) { | 971 | for_each_online_node(i) { |
1077 | ac = alien[i]; | 972 | alc = alien[i]; |
1078 | if (ac) { | 973 | if (alc) { |
1079 | spin_lock_irqsave(&ac->lock, flags); | 974 | LIST_HEAD(list); |
1080 | __drain_alien_cache(cachep, ac, i); | 975 | |
1081 | spin_unlock_irqrestore(&ac->lock, flags); | 976 | ac = &alc->ac; |
977 | spin_lock_irqsave(&alc->lock, flags); | ||
978 | __drain_alien_cache(cachep, ac, i, &list); | ||
979 | spin_unlock_irqrestore(&alc->lock, flags); | ||
980 | slabs_destroy(cachep, &list); | ||
1082 | } | 981 | } |
1083 | } | 982 | } |
1084 | } | 983 | } |
@@ -1087,8 +986,10 @@ static inline int cache_free_alien(struct kmem_cache *cachep, void *objp) | |||
1087 | { | 986 | { |
1088 | int nodeid = page_to_nid(virt_to_page(objp)); | 987 | int nodeid = page_to_nid(virt_to_page(objp)); |
1089 | struct kmem_cache_node *n; | 988 | struct kmem_cache_node *n; |
1090 | struct array_cache *alien = NULL; | 989 | struct alien_cache *alien = NULL; |
990 | struct array_cache *ac; | ||
1091 | int node; | 991 | int node; |
992 | LIST_HEAD(list); | ||
1092 | 993 | ||
1093 | node = numa_mem_id(); | 994 | node = numa_mem_id(); |
1094 | 995 | ||
@@ -1099,21 +1000,25 @@ static inline int cache_free_alien(struct kmem_cache *cachep, void *objp) | |||
1099 | if (likely(nodeid == node)) | 1000 | if (likely(nodeid == node)) |
1100 | return 0; | 1001 | return 0; |
1101 | 1002 | ||
1102 | n = cachep->node[node]; | 1003 | n = get_node(cachep, node); |
1103 | STATS_INC_NODEFREES(cachep); | 1004 | STATS_INC_NODEFREES(cachep); |
1104 | if (n->alien && n->alien[nodeid]) { | 1005 | if (n->alien && n->alien[nodeid]) { |
1105 | alien = n->alien[nodeid]; | 1006 | alien = n->alien[nodeid]; |
1007 | ac = &alien->ac; | ||
1106 | spin_lock(&alien->lock); | 1008 | spin_lock(&alien->lock); |
1107 | if (unlikely(alien->avail == alien->limit)) { | 1009 | if (unlikely(ac->avail == ac->limit)) { |
1108 | STATS_INC_ACOVERFLOW(cachep); | 1010 | STATS_INC_ACOVERFLOW(cachep); |
1109 | __drain_alien_cache(cachep, alien, nodeid); | 1011 | __drain_alien_cache(cachep, ac, nodeid, &list); |
1110 | } | 1012 | } |
1111 | ac_put_obj(cachep, alien, objp); | 1013 | ac_put_obj(cachep, ac, objp); |
1112 | spin_unlock(&alien->lock); | 1014 | spin_unlock(&alien->lock); |
1015 | slabs_destroy(cachep, &list); | ||
1113 | } else { | 1016 | } else { |
1114 | spin_lock(&(cachep->node[nodeid])->list_lock); | 1017 | n = get_node(cachep, nodeid); |
1115 | free_block(cachep, &objp, 1, nodeid); | 1018 | spin_lock(&n->list_lock); |
1116 | spin_unlock(&(cachep->node[nodeid])->list_lock); | 1019 | free_block(cachep, &objp, 1, nodeid, &list); |
1020 | spin_unlock(&n->list_lock); | ||
1021 | slabs_destroy(cachep, &list); | ||
1117 | } | 1022 | } |
1118 | return 1; | 1023 | return 1; |
1119 | } | 1024 | } |
@@ -1132,7 +1037,7 @@ static int init_cache_node_node(int node) | |||
1132 | { | 1037 | { |
1133 | struct kmem_cache *cachep; | 1038 | struct kmem_cache *cachep; |
1134 | struct kmem_cache_node *n; | 1039 | struct kmem_cache_node *n; |
1135 | const int memsize = sizeof(struct kmem_cache_node); | 1040 | const size_t memsize = sizeof(struct kmem_cache_node); |
1136 | 1041 | ||
1137 | list_for_each_entry(cachep, &slab_caches, list) { | 1042 | list_for_each_entry(cachep, &slab_caches, list) { |
1138 | /* | 1043 | /* |
@@ -1140,7 +1045,8 @@ static int init_cache_node_node(int node) | |||
1140 | * begin anything. Make sure some other cpu on this | 1045 | * begin anything. Make sure some other cpu on this |
1141 | * node has not already allocated this | 1046 | * node has not already allocated this |
1142 | */ | 1047 | */ |
1143 | if (!cachep->node[node]) { | 1048 | n = get_node(cachep, node); |
1049 | if (!n) { | ||
1144 | n = kmalloc_node(memsize, GFP_KERNEL, node); | 1050 | n = kmalloc_node(memsize, GFP_KERNEL, node); |
1145 | if (!n) | 1051 | if (!n) |
1146 | return -ENOMEM; | 1052 | return -ENOMEM; |
@@ -1156,11 +1062,11 @@ static int init_cache_node_node(int node) | |||
1156 | cachep->node[node] = n; | 1062 | cachep->node[node] = n; |
1157 | } | 1063 | } |
1158 | 1064 | ||
1159 | spin_lock_irq(&cachep->node[node]->list_lock); | 1065 | spin_lock_irq(&n->list_lock); |
1160 | cachep->node[node]->free_limit = | 1066 | n->free_limit = |
1161 | (1 + nr_cpus_node(node)) * | 1067 | (1 + nr_cpus_node(node)) * |
1162 | cachep->batchcount + cachep->num; | 1068 | cachep->batchcount + cachep->num; |
1163 | spin_unlock_irq(&cachep->node[node]->list_lock); | 1069 | spin_unlock_irq(&n->list_lock); |
1164 | } | 1070 | } |
1165 | return 0; | 1071 | return 0; |
1166 | } | 1072 | } |
@@ -1181,12 +1087,13 @@ static void cpuup_canceled(long cpu) | |||
1181 | list_for_each_entry(cachep, &slab_caches, list) { | 1087 | list_for_each_entry(cachep, &slab_caches, list) { |
1182 | struct array_cache *nc; | 1088 | struct array_cache *nc; |
1183 | struct array_cache *shared; | 1089 | struct array_cache *shared; |
1184 | struct array_cache **alien; | 1090 | struct alien_cache **alien; |
1091 | LIST_HEAD(list); | ||
1185 | 1092 | ||
1186 | /* cpu is dead; no one can alloc from it. */ | 1093 | /* cpu is dead; no one can alloc from it. */ |
1187 | nc = cachep->array[cpu]; | 1094 | nc = cachep->array[cpu]; |
1188 | cachep->array[cpu] = NULL; | 1095 | cachep->array[cpu] = NULL; |
1189 | n = cachep->node[node]; | 1096 | n = get_node(cachep, node); |
1190 | 1097 | ||
1191 | if (!n) | 1098 | if (!n) |
1192 | goto free_array_cache; | 1099 | goto free_array_cache; |
@@ -1196,7 +1103,7 @@ static void cpuup_canceled(long cpu) | |||
1196 | /* Free limit for this kmem_cache_node */ | 1103 | /* Free limit for this kmem_cache_node */ |
1197 | n->free_limit -= cachep->batchcount; | 1104 | n->free_limit -= cachep->batchcount; |
1198 | if (nc) | 1105 | if (nc) |
1199 | free_block(cachep, nc->entry, nc->avail, node); | 1106 | free_block(cachep, nc->entry, nc->avail, node, &list); |
1200 | 1107 | ||
1201 | if (!cpumask_empty(mask)) { | 1108 | if (!cpumask_empty(mask)) { |
1202 | spin_unlock_irq(&n->list_lock); | 1109 | spin_unlock_irq(&n->list_lock); |
@@ -1206,7 +1113,7 @@ static void cpuup_canceled(long cpu) | |||
1206 | shared = n->shared; | 1113 | shared = n->shared; |
1207 | if (shared) { | 1114 | if (shared) { |
1208 | free_block(cachep, shared->entry, | 1115 | free_block(cachep, shared->entry, |
1209 | shared->avail, node); | 1116 | shared->avail, node, &list); |
1210 | n->shared = NULL; | 1117 | n->shared = NULL; |
1211 | } | 1118 | } |
1212 | 1119 | ||
@@ -1221,6 +1128,7 @@ static void cpuup_canceled(long cpu) | |||
1221 | free_alien_cache(alien); | 1128 | free_alien_cache(alien); |
1222 | } | 1129 | } |
1223 | free_array_cache: | 1130 | free_array_cache: |
1131 | slabs_destroy(cachep, &list); | ||
1224 | kfree(nc); | 1132 | kfree(nc); |
1225 | } | 1133 | } |
1226 | /* | 1134 | /* |
@@ -1229,7 +1137,7 @@ free_array_cache: | |||
1229 | * shrink each nodelist to its limit. | 1137 | * shrink each nodelist to its limit. |
1230 | */ | 1138 | */ |
1231 | list_for_each_entry(cachep, &slab_caches, list) { | 1139 | list_for_each_entry(cachep, &slab_caches, list) { |
1232 | n = cachep->node[node]; | 1140 | n = get_node(cachep, node); |
1233 | if (!n) | 1141 | if (!n) |
1234 | continue; | 1142 | continue; |
1235 | drain_freelist(cachep, n, slabs_tofree(cachep, n)); | 1143 | drain_freelist(cachep, n, slabs_tofree(cachep, n)); |
@@ -1260,7 +1168,7 @@ static int cpuup_prepare(long cpu) | |||
1260 | list_for_each_entry(cachep, &slab_caches, list) { | 1168 | list_for_each_entry(cachep, &slab_caches, list) { |
1261 | struct array_cache *nc; | 1169 | struct array_cache *nc; |
1262 | struct array_cache *shared = NULL; | 1170 | struct array_cache *shared = NULL; |
1263 | struct array_cache **alien = NULL; | 1171 | struct alien_cache **alien = NULL; |
1264 | 1172 | ||
1265 | nc = alloc_arraycache(node, cachep->limit, | 1173 | nc = alloc_arraycache(node, cachep->limit, |
1266 | cachep->batchcount, GFP_KERNEL); | 1174 | cachep->batchcount, GFP_KERNEL); |
@@ -1284,7 +1192,7 @@ static int cpuup_prepare(long cpu) | |||
1284 | } | 1192 | } |
1285 | } | 1193 | } |
1286 | cachep->array[cpu] = nc; | 1194 | cachep->array[cpu] = nc; |
1287 | n = cachep->node[node]; | 1195 | n = get_node(cachep, node); |
1288 | BUG_ON(!n); | 1196 | BUG_ON(!n); |
1289 | 1197 | ||
1290 | spin_lock_irq(&n->list_lock); | 1198 | spin_lock_irq(&n->list_lock); |
@@ -1305,13 +1213,7 @@ static int cpuup_prepare(long cpu) | |||
1305 | spin_unlock_irq(&n->list_lock); | 1213 | spin_unlock_irq(&n->list_lock); |
1306 | kfree(shared); | 1214 | kfree(shared); |
1307 | free_alien_cache(alien); | 1215 | free_alien_cache(alien); |
1308 | if (cachep->flags & SLAB_DEBUG_OBJECTS) | ||
1309 | slab_set_debugobj_lock_classes_node(cachep, node); | ||
1310 | else if (!OFF_SLAB(cachep) && | ||
1311 | !(cachep->flags & SLAB_DESTROY_BY_RCU)) | ||
1312 | on_slab_lock_classes_node(cachep, node); | ||
1313 | } | 1216 | } |
1314 | init_node_lock_keys(node); | ||
1315 | 1217 | ||
1316 | return 0; | 1218 | return 0; |
1317 | bad: | 1219 | bad: |
@@ -1395,7 +1297,7 @@ static int __meminit drain_cache_node_node(int node) | |||
1395 | list_for_each_entry(cachep, &slab_caches, list) { | 1297 | list_for_each_entry(cachep, &slab_caches, list) { |
1396 | struct kmem_cache_node *n; | 1298 | struct kmem_cache_node *n; |
1397 | 1299 | ||
1398 | n = cachep->node[node]; | 1300 | n = get_node(cachep, node); |
1399 | if (!n) | 1301 | if (!n) |
1400 | continue; | 1302 | continue; |
1401 | 1303 | ||
@@ -1575,10 +1477,6 @@ void __init kmem_cache_init(void) | |||
1575 | 1477 | ||
1576 | memcpy(ptr, cpu_cache_get(kmem_cache), | 1478 | memcpy(ptr, cpu_cache_get(kmem_cache), |
1577 | sizeof(struct arraycache_init)); | 1479 | sizeof(struct arraycache_init)); |
1578 | /* | ||
1579 | * Do not assume that spinlocks can be initialized via memcpy: | ||
1580 | */ | ||
1581 | spin_lock_init(&ptr->lock); | ||
1582 | 1480 | ||
1583 | kmem_cache->array[smp_processor_id()] = ptr; | 1481 | kmem_cache->array[smp_processor_id()] = ptr; |
1584 | 1482 | ||
@@ -1588,10 +1486,6 @@ void __init kmem_cache_init(void) | |||
1588 | != &initarray_generic.cache); | 1486 | != &initarray_generic.cache); |
1589 | memcpy(ptr, cpu_cache_get(kmalloc_caches[INDEX_AC]), | 1487 | memcpy(ptr, cpu_cache_get(kmalloc_caches[INDEX_AC]), |
1590 | sizeof(struct arraycache_init)); | 1488 | sizeof(struct arraycache_init)); |
1591 | /* | ||
1592 | * Do not assume that spinlocks can be initialized via memcpy: | ||
1593 | */ | ||
1594 | spin_lock_init(&ptr->lock); | ||
1595 | 1489 | ||
1596 | kmalloc_caches[INDEX_AC]->array[smp_processor_id()] = ptr; | 1490 | kmalloc_caches[INDEX_AC]->array[smp_processor_id()] = ptr; |
1597 | } | 1491 | } |
@@ -1628,9 +1522,6 @@ void __init kmem_cache_init_late(void) | |||
1628 | BUG(); | 1522 | BUG(); |
1629 | mutex_unlock(&slab_mutex); | 1523 | mutex_unlock(&slab_mutex); |
1630 | 1524 | ||
1631 | /* Annotate slab for lockdep -- annotate the malloc caches */ | ||
1632 | init_lock_keys(); | ||
1633 | |||
1634 | /* Done! */ | 1525 | /* Done! */ |
1635 | slab_state = FULL; | 1526 | slab_state = FULL; |
1636 | 1527 | ||
@@ -1690,14 +1581,10 @@ slab_out_of_memory(struct kmem_cache *cachep, gfp_t gfpflags, int nodeid) | |||
1690 | printk(KERN_WARNING " cache: %s, object size: %d, order: %d\n", | 1581 | printk(KERN_WARNING " cache: %s, object size: %d, order: %d\n", |
1691 | cachep->name, cachep->size, cachep->gfporder); | 1582 | cachep->name, cachep->size, cachep->gfporder); |
1692 | 1583 | ||
1693 | for_each_online_node(node) { | 1584 | for_each_kmem_cache_node(cachep, node, n) { |
1694 | unsigned long active_objs = 0, num_objs = 0, free_objects = 0; | 1585 | unsigned long active_objs = 0, num_objs = 0, free_objects = 0; |
1695 | unsigned long active_slabs = 0, num_slabs = 0; | 1586 | unsigned long active_slabs = 0, num_slabs = 0; |
1696 | 1587 | ||
1697 | n = cachep->node[node]; | ||
1698 | if (!n) | ||
1699 | continue; | ||
1700 | |||
1701 | spin_lock_irqsave(&n->list_lock, flags); | 1588 | spin_lock_irqsave(&n->list_lock, flags); |
1702 | list_for_each_entry(page, &n->slabs_full, lru) { | 1589 | list_for_each_entry(page, &n->slabs_full, lru) { |
1703 | active_objs += cachep->num; | 1590 | active_objs += cachep->num; |
@@ -1724,7 +1611,8 @@ slab_out_of_memory(struct kmem_cache *cachep, gfp_t gfpflags, int nodeid) | |||
1724 | } | 1611 | } |
1725 | 1612 | ||
1726 | /* | 1613 | /* |
1727 | * Interface to system's page allocator. No need to hold the cache-lock. | 1614 | * Interface to system's page allocator. No need to hold the |
1615 | * kmem_cache_node ->list_lock. | ||
1728 | * | 1616 | * |
1729 | * If we requested dmaable memory, we will get it. Even if we | 1617 | * If we requested dmaable memory, we will get it. Even if we |
1730 | * did not request dmaable memory, we might get it, but that | 1618 | * did not request dmaable memory, we might get it, but that |
@@ -2026,9 +1914,9 @@ static void slab_destroy_debugcheck(struct kmem_cache *cachep, | |||
2026 | * @cachep: cache pointer being destroyed | 1914 | * @cachep: cache pointer being destroyed |
2027 | * @page: page pointer being destroyed | 1915 | * @page: page pointer being destroyed |
2028 | * | 1916 | * |
2029 | * Destroy all the objs in a slab, and release the mem back to the system. | 1917 | * Destroy all the objs in a slab page, and release the mem back to the system. |
2030 | * Before calling the slab must have been unlinked from the cache. The | 1918 | * Before calling the slab page must have been unlinked from the cache. The |
2031 | * cache-lock is not held/needed. | 1919 | * kmem_cache_node ->list_lock is not held/needed. |
2032 | */ | 1920 | */ |
2033 | static void slab_destroy(struct kmem_cache *cachep, struct page *page) | 1921 | static void slab_destroy(struct kmem_cache *cachep, struct page *page) |
2034 | { | 1922 | { |
@@ -2060,6 +1948,16 @@ static void slab_destroy(struct kmem_cache *cachep, struct page *page) | |||
2060 | kmem_cache_free(cachep->freelist_cache, freelist); | 1948 | kmem_cache_free(cachep->freelist_cache, freelist); |
2061 | } | 1949 | } |
2062 | 1950 | ||
1951 | static void slabs_destroy(struct kmem_cache *cachep, struct list_head *list) | ||
1952 | { | ||
1953 | struct page *page, *n; | ||
1954 | |||
1955 | list_for_each_entry_safe(page, n, list, lru) { | ||
1956 | list_del(&page->lru); | ||
1957 | slab_destroy(cachep, page); | ||
1958 | } | ||
1959 | } | ||
1960 | |||
2063 | /** | 1961 | /** |
2064 | * calculate_slab_order - calculate size (page order) of slabs | 1962 | * calculate_slab_order - calculate size (page order) of slabs |
2065 | * @cachep: pointer to the cache that is being created | 1963 | * @cachep: pointer to the cache that is being created |
@@ -2405,17 +2303,6 @@ __kmem_cache_create (struct kmem_cache *cachep, unsigned long flags) | |||
2405 | return err; | 2303 | return err; |
2406 | } | 2304 | } |
2407 | 2305 | ||
2408 | if (flags & SLAB_DEBUG_OBJECTS) { | ||
2409 | /* | ||
2410 | * Would deadlock through slab_destroy()->call_rcu()-> | ||
2411 | * debug_object_activate()->kmem_cache_alloc(). | ||
2412 | */ | ||
2413 | WARN_ON_ONCE(flags & SLAB_DESTROY_BY_RCU); | ||
2414 | |||
2415 | slab_set_debugobj_lock_classes(cachep); | ||
2416 | } else if (!OFF_SLAB(cachep) && !(flags & SLAB_DESTROY_BY_RCU)) | ||
2417 | on_slab_lock_classes(cachep); | ||
2418 | |||
2419 | return 0; | 2306 | return 0; |
2420 | } | 2307 | } |
2421 | 2308 | ||
@@ -2434,7 +2321,7 @@ static void check_spinlock_acquired(struct kmem_cache *cachep) | |||
2434 | { | 2321 | { |
2435 | #ifdef CONFIG_SMP | 2322 | #ifdef CONFIG_SMP |
2436 | check_irq_off(); | 2323 | check_irq_off(); |
2437 | assert_spin_locked(&cachep->node[numa_mem_id()]->list_lock); | 2324 | assert_spin_locked(&get_node(cachep, numa_mem_id())->list_lock); |
2438 | #endif | 2325 | #endif |
2439 | } | 2326 | } |
2440 | 2327 | ||
@@ -2442,7 +2329,7 @@ static void check_spinlock_acquired_node(struct kmem_cache *cachep, int node) | |||
2442 | { | 2329 | { |
2443 | #ifdef CONFIG_SMP | 2330 | #ifdef CONFIG_SMP |
2444 | check_irq_off(); | 2331 | check_irq_off(); |
2445 | assert_spin_locked(&cachep->node[node]->list_lock); | 2332 | assert_spin_locked(&get_node(cachep, node)->list_lock); |
2446 | #endif | 2333 | #endif |
2447 | } | 2334 | } |
2448 | 2335 | ||
@@ -2462,12 +2349,16 @@ static void do_drain(void *arg) | |||
2462 | struct kmem_cache *cachep = arg; | 2349 | struct kmem_cache *cachep = arg; |
2463 | struct array_cache *ac; | 2350 | struct array_cache *ac; |
2464 | int node = numa_mem_id(); | 2351 | int node = numa_mem_id(); |
2352 | struct kmem_cache_node *n; | ||
2353 | LIST_HEAD(list); | ||
2465 | 2354 | ||
2466 | check_irq_off(); | 2355 | check_irq_off(); |
2467 | ac = cpu_cache_get(cachep); | 2356 | ac = cpu_cache_get(cachep); |
2468 | spin_lock(&cachep->node[node]->list_lock); | 2357 | n = get_node(cachep, node); |
2469 | free_block(cachep, ac->entry, ac->avail, node); | 2358 | spin_lock(&n->list_lock); |
2470 | spin_unlock(&cachep->node[node]->list_lock); | 2359 | free_block(cachep, ac->entry, ac->avail, node, &list); |
2360 | spin_unlock(&n->list_lock); | ||
2361 | slabs_destroy(cachep, &list); | ||
2471 | ac->avail = 0; | 2362 | ac->avail = 0; |
2472 | } | 2363 | } |
2473 | 2364 | ||
@@ -2478,17 +2369,12 @@ static void drain_cpu_caches(struct kmem_cache *cachep) | |||
2478 | 2369 | ||
2479 | on_each_cpu(do_drain, cachep, 1); | 2370 | on_each_cpu(do_drain, cachep, 1); |
2480 | check_irq_on(); | 2371 | check_irq_on(); |
2481 | for_each_online_node(node) { | 2372 | for_each_kmem_cache_node(cachep, node, n) |
2482 | n = cachep->node[node]; | 2373 | if (n->alien) |
2483 | if (n && n->alien) | ||
2484 | drain_alien_cache(cachep, n->alien); | 2374 | drain_alien_cache(cachep, n->alien); |
2485 | } | ||
2486 | 2375 | ||
2487 | for_each_online_node(node) { | 2376 | for_each_kmem_cache_node(cachep, node, n) |
2488 | n = cachep->node[node]; | 2377 | drain_array(cachep, n, n->shared, 1, node); |
2489 | if (n) | ||
2490 | drain_array(cachep, n, n->shared, 1, node); | ||
2491 | } | ||
2492 | } | 2378 | } |
2493 | 2379 | ||
2494 | /* | 2380 | /* |
@@ -2534,17 +2420,14 @@ out: | |||
2534 | 2420 | ||
2535 | int __kmem_cache_shrink(struct kmem_cache *cachep) | 2421 | int __kmem_cache_shrink(struct kmem_cache *cachep) |
2536 | { | 2422 | { |
2537 | int ret = 0, i = 0; | 2423 | int ret = 0; |
2424 | int node; | ||
2538 | struct kmem_cache_node *n; | 2425 | struct kmem_cache_node *n; |
2539 | 2426 | ||
2540 | drain_cpu_caches(cachep); | 2427 | drain_cpu_caches(cachep); |
2541 | 2428 | ||
2542 | check_irq_on(); | 2429 | check_irq_on(); |
2543 | for_each_online_node(i) { | 2430 | for_each_kmem_cache_node(cachep, node, n) { |
2544 | n = cachep->node[i]; | ||
2545 | if (!n) | ||
2546 | continue; | ||
2547 | |||
2548 | drain_freelist(cachep, n, slabs_tofree(cachep, n)); | 2431 | drain_freelist(cachep, n, slabs_tofree(cachep, n)); |
2549 | 2432 | ||
2550 | ret += !list_empty(&n->slabs_full) || | 2433 | ret += !list_empty(&n->slabs_full) || |
@@ -2566,13 +2449,11 @@ int __kmem_cache_shutdown(struct kmem_cache *cachep) | |||
2566 | kfree(cachep->array[i]); | 2449 | kfree(cachep->array[i]); |
2567 | 2450 | ||
2568 | /* NUMA: free the node structures */ | 2451 | /* NUMA: free the node structures */ |
2569 | for_each_online_node(i) { | 2452 | for_each_kmem_cache_node(cachep, i, n) { |
2570 | n = cachep->node[i]; | 2453 | kfree(n->shared); |
2571 | if (n) { | 2454 | free_alien_cache(n->alien); |
2572 | kfree(n->shared); | 2455 | kfree(n); |
2573 | free_alien_cache(n->alien); | 2456 | cachep->node[i] = NULL; |
2574 | kfree(n); | ||
2575 | } | ||
2576 | } | 2457 | } |
2577 | return 0; | 2458 | return 0; |
2578 | } | 2459 | } |
@@ -2751,7 +2632,7 @@ static int cache_grow(struct kmem_cache *cachep, | |||
2751 | 2632 | ||
2752 | /* Take the node list lock to change the colour_next on this node */ | 2633 | /* Take the node list lock to change the colour_next on this node */ |
2753 | check_irq_off(); | 2634 | check_irq_off(); |
2754 | n = cachep->node[nodeid]; | 2635 | n = get_node(cachep, nodeid); |
2755 | spin_lock(&n->list_lock); | 2636 | spin_lock(&n->list_lock); |
2756 | 2637 | ||
2757 | /* Get colour for the slab, and cal the next value. */ | 2638 | /* Get colour for the slab, and cal the next value. */ |
@@ -2920,7 +2801,7 @@ retry: | |||
2920 | */ | 2801 | */ |
2921 | batchcount = BATCHREFILL_LIMIT; | 2802 | batchcount = BATCHREFILL_LIMIT; |
2922 | } | 2803 | } |
2923 | n = cachep->node[node]; | 2804 | n = get_node(cachep, node); |
2924 | 2805 | ||
2925 | BUG_ON(ac->avail > 0 || !n); | 2806 | BUG_ON(ac->avail > 0 || !n); |
2926 | spin_lock(&n->list_lock); | 2807 | spin_lock(&n->list_lock); |
@@ -3060,7 +2941,7 @@ static void *cache_alloc_debugcheck_after(struct kmem_cache *cachep, | |||
3060 | 2941 | ||
3061 | static bool slab_should_failslab(struct kmem_cache *cachep, gfp_t flags) | 2942 | static bool slab_should_failslab(struct kmem_cache *cachep, gfp_t flags) |
3062 | { | 2943 | { |
3063 | if (cachep == kmem_cache) | 2944 | if (unlikely(cachep == kmem_cache)) |
3064 | return false; | 2945 | return false; |
3065 | 2946 | ||
3066 | return should_failslab(cachep->object_size, flags, cachep->flags); | 2947 | return should_failslab(cachep->object_size, flags, cachep->flags); |
@@ -3169,8 +3050,8 @@ retry: | |||
3169 | nid = zone_to_nid(zone); | 3050 | nid = zone_to_nid(zone); |
3170 | 3051 | ||
3171 | if (cpuset_zone_allowed_hardwall(zone, flags) && | 3052 | if (cpuset_zone_allowed_hardwall(zone, flags) && |
3172 | cache->node[nid] && | 3053 | get_node(cache, nid) && |
3173 | cache->node[nid]->free_objects) { | 3054 | get_node(cache, nid)->free_objects) { |
3174 | obj = ____cache_alloc_node(cache, | 3055 | obj = ____cache_alloc_node(cache, |
3175 | flags | GFP_THISNODE, nid); | 3056 | flags | GFP_THISNODE, nid); |
3176 | if (obj) | 3057 | if (obj) |
@@ -3233,7 +3114,7 @@ static void *____cache_alloc_node(struct kmem_cache *cachep, gfp_t flags, | |||
3233 | int x; | 3114 | int x; |
3234 | 3115 | ||
3235 | VM_BUG_ON(nodeid > num_online_nodes()); | 3116 | VM_BUG_ON(nodeid > num_online_nodes()); |
3236 | n = cachep->node[nodeid]; | 3117 | n = get_node(cachep, nodeid); |
3237 | BUG_ON(!n); | 3118 | BUG_ON(!n); |
3238 | 3119 | ||
3239 | retry: | 3120 | retry: |
@@ -3304,7 +3185,7 @@ slab_alloc_node(struct kmem_cache *cachep, gfp_t flags, int nodeid, | |||
3304 | if (nodeid == NUMA_NO_NODE) | 3185 | if (nodeid == NUMA_NO_NODE) |
3305 | nodeid = slab_node; | 3186 | nodeid = slab_node; |
3306 | 3187 | ||
3307 | if (unlikely(!cachep->node[nodeid])) { | 3188 | if (unlikely(!get_node(cachep, nodeid))) { |
3308 | /* Node not bootstrapped yet */ | 3189 | /* Node not bootstrapped yet */ |
3309 | ptr = fallback_alloc(cachep, flags); | 3190 | ptr = fallback_alloc(cachep, flags); |
3310 | goto out; | 3191 | goto out; |
@@ -3405,12 +3286,13 @@ slab_alloc(struct kmem_cache *cachep, gfp_t flags, unsigned long caller) | |||
3405 | 3286 | ||
3406 | /* | 3287 | /* |
3407 | * Caller needs to acquire correct kmem_cache_node's list_lock | 3288 | * Caller needs to acquire correct kmem_cache_node's list_lock |
3289 | * @list: List of detached free slabs should be freed by caller | ||
3408 | */ | 3290 | */ |
3409 | static void free_block(struct kmem_cache *cachep, void **objpp, int nr_objects, | 3291 | static void free_block(struct kmem_cache *cachep, void **objpp, |
3410 | int node) | 3292 | int nr_objects, int node, struct list_head *list) |
3411 | { | 3293 | { |
3412 | int i; | 3294 | int i; |
3413 | struct kmem_cache_node *n; | 3295 | struct kmem_cache_node *n = get_node(cachep, node); |
3414 | 3296 | ||
3415 | for (i = 0; i < nr_objects; i++) { | 3297 | for (i = 0; i < nr_objects; i++) { |
3416 | void *objp; | 3298 | void *objp; |
@@ -3420,7 +3302,6 @@ static void free_block(struct kmem_cache *cachep, void **objpp, int nr_objects, | |||
3420 | objp = objpp[i]; | 3302 | objp = objpp[i]; |
3421 | 3303 | ||
3422 | page = virt_to_head_page(objp); | 3304 | page = virt_to_head_page(objp); |
3423 | n = cachep->node[node]; | ||
3424 | list_del(&page->lru); | 3305 | list_del(&page->lru); |
3425 | check_spinlock_acquired_node(cachep, node); | 3306 | check_spinlock_acquired_node(cachep, node); |
3426 | slab_put_obj(cachep, page, objp, node); | 3307 | slab_put_obj(cachep, page, objp, node); |
@@ -3431,13 +3312,7 @@ static void free_block(struct kmem_cache *cachep, void **objpp, int nr_objects, | |||
3431 | if (page->active == 0) { | 3312 | if (page->active == 0) { |
3432 | if (n->free_objects > n->free_limit) { | 3313 | if (n->free_objects > n->free_limit) { |
3433 | n->free_objects -= cachep->num; | 3314 | n->free_objects -= cachep->num; |
3434 | /* No need to drop any previously held | 3315 | list_add_tail(&page->lru, list); |
3435 | * lock here, even if we have a off-slab slab | ||
3436 | * descriptor it is guaranteed to come from | ||
3437 | * a different cache, refer to comments before | ||
3438 | * alloc_slabmgmt. | ||
3439 | */ | ||
3440 | slab_destroy(cachep, page); | ||
3441 | } else { | 3316 | } else { |
3442 | list_add(&page->lru, &n->slabs_free); | 3317 | list_add(&page->lru, &n->slabs_free); |
3443 | } | 3318 | } |
@@ -3456,13 +3331,14 @@ static void cache_flusharray(struct kmem_cache *cachep, struct array_cache *ac) | |||
3456 | int batchcount; | 3331 | int batchcount; |
3457 | struct kmem_cache_node *n; | 3332 | struct kmem_cache_node *n; |
3458 | int node = numa_mem_id(); | 3333 | int node = numa_mem_id(); |
3334 | LIST_HEAD(list); | ||
3459 | 3335 | ||
3460 | batchcount = ac->batchcount; | 3336 | batchcount = ac->batchcount; |
3461 | #if DEBUG | 3337 | #if DEBUG |
3462 | BUG_ON(!batchcount || batchcount > ac->avail); | 3338 | BUG_ON(!batchcount || batchcount > ac->avail); |
3463 | #endif | 3339 | #endif |
3464 | check_irq_off(); | 3340 | check_irq_off(); |
3465 | n = cachep->node[node]; | 3341 | n = get_node(cachep, node); |
3466 | spin_lock(&n->list_lock); | 3342 | spin_lock(&n->list_lock); |
3467 | if (n->shared) { | 3343 | if (n->shared) { |
3468 | struct array_cache *shared_array = n->shared; | 3344 | struct array_cache *shared_array = n->shared; |
@@ -3477,7 +3353,7 @@ static void cache_flusharray(struct kmem_cache *cachep, struct array_cache *ac) | |||
3477 | } | 3353 | } |
3478 | } | 3354 | } |
3479 | 3355 | ||
3480 | free_block(cachep, ac->entry, batchcount, node); | 3356 | free_block(cachep, ac->entry, batchcount, node, &list); |
3481 | free_done: | 3357 | free_done: |
3482 | #if STATS | 3358 | #if STATS |
3483 | { | 3359 | { |
@@ -3498,6 +3374,7 @@ free_done: | |||
3498 | } | 3374 | } |
3499 | #endif | 3375 | #endif |
3500 | spin_unlock(&n->list_lock); | 3376 | spin_unlock(&n->list_lock); |
3377 | slabs_destroy(cachep, &list); | ||
3501 | ac->avail -= batchcount; | 3378 | ac->avail -= batchcount; |
3502 | memmove(ac->entry, &(ac->entry[batchcount]), sizeof(void *)*ac->avail); | 3379 | memmove(ac->entry, &(ac->entry[batchcount]), sizeof(void *)*ac->avail); |
3503 | } | 3380 | } |
@@ -3754,7 +3631,7 @@ static int alloc_kmem_cache_node(struct kmem_cache *cachep, gfp_t gfp) | |||
3754 | int node; | 3631 | int node; |
3755 | struct kmem_cache_node *n; | 3632 | struct kmem_cache_node *n; |
3756 | struct array_cache *new_shared; | 3633 | struct array_cache *new_shared; |
3757 | struct array_cache **new_alien = NULL; | 3634 | struct alien_cache **new_alien = NULL; |
3758 | 3635 | ||
3759 | for_each_online_node(node) { | 3636 | for_each_online_node(node) { |
3760 | 3637 | ||
@@ -3775,15 +3652,16 @@ static int alloc_kmem_cache_node(struct kmem_cache *cachep, gfp_t gfp) | |||
3775 | } | 3652 | } |
3776 | } | 3653 | } |
3777 | 3654 | ||
3778 | n = cachep->node[node]; | 3655 | n = get_node(cachep, node); |
3779 | if (n) { | 3656 | if (n) { |
3780 | struct array_cache *shared = n->shared; | 3657 | struct array_cache *shared = n->shared; |
3658 | LIST_HEAD(list); | ||
3781 | 3659 | ||
3782 | spin_lock_irq(&n->list_lock); | 3660 | spin_lock_irq(&n->list_lock); |
3783 | 3661 | ||
3784 | if (shared) | 3662 | if (shared) |
3785 | free_block(cachep, shared->entry, | 3663 | free_block(cachep, shared->entry, |
3786 | shared->avail, node); | 3664 | shared->avail, node, &list); |
3787 | 3665 | ||
3788 | n->shared = new_shared; | 3666 | n->shared = new_shared; |
3789 | if (!n->alien) { | 3667 | if (!n->alien) { |
@@ -3793,6 +3671,7 @@ static int alloc_kmem_cache_node(struct kmem_cache *cachep, gfp_t gfp) | |||
3793 | n->free_limit = (1 + nr_cpus_node(node)) * | 3671 | n->free_limit = (1 + nr_cpus_node(node)) * |
3794 | cachep->batchcount + cachep->num; | 3672 | cachep->batchcount + cachep->num; |
3795 | spin_unlock_irq(&n->list_lock); | 3673 | spin_unlock_irq(&n->list_lock); |
3674 | slabs_destroy(cachep, &list); | ||
3796 | kfree(shared); | 3675 | kfree(shared); |
3797 | free_alien_cache(new_alien); | 3676 | free_alien_cache(new_alien); |
3798 | continue; | 3677 | continue; |
@@ -3820,9 +3699,8 @@ fail: | |||
3820 | /* Cache is not active yet. Roll back what we did */ | 3699 | /* Cache is not active yet. Roll back what we did */ |
3821 | node--; | 3700 | node--; |
3822 | while (node >= 0) { | 3701 | while (node >= 0) { |
3823 | if (cachep->node[node]) { | 3702 | n = get_node(cachep, node); |
3824 | n = cachep->node[node]; | 3703 | if (n) { |
3825 | |||
3826 | kfree(n->shared); | 3704 | kfree(n->shared); |
3827 | free_alien_cache(n->alien); | 3705 | free_alien_cache(n->alien); |
3828 | kfree(n); | 3706 | kfree(n); |
@@ -3883,12 +3761,20 @@ static int __do_tune_cpucache(struct kmem_cache *cachep, int limit, | |||
3883 | cachep->shared = shared; | 3761 | cachep->shared = shared; |
3884 | 3762 | ||
3885 | for_each_online_cpu(i) { | 3763 | for_each_online_cpu(i) { |
3764 | LIST_HEAD(list); | ||
3886 | struct array_cache *ccold = new->new[i]; | 3765 | struct array_cache *ccold = new->new[i]; |
3766 | int node; | ||
3767 | struct kmem_cache_node *n; | ||
3768 | |||
3887 | if (!ccold) | 3769 | if (!ccold) |
3888 | continue; | 3770 | continue; |
3889 | spin_lock_irq(&cachep->node[cpu_to_mem(i)]->list_lock); | 3771 | |
3890 | free_block(cachep, ccold->entry, ccold->avail, cpu_to_mem(i)); | 3772 | node = cpu_to_mem(i); |
3891 | spin_unlock_irq(&cachep->node[cpu_to_mem(i)]->list_lock); | 3773 | n = get_node(cachep, node); |
3774 | spin_lock_irq(&n->list_lock); | ||
3775 | free_block(cachep, ccold->entry, ccold->avail, node, &list); | ||
3776 | spin_unlock_irq(&n->list_lock); | ||
3777 | slabs_destroy(cachep, &list); | ||
3892 | kfree(ccold); | 3778 | kfree(ccold); |
3893 | } | 3779 | } |
3894 | kfree(new); | 3780 | kfree(new); |
@@ -3996,6 +3882,7 @@ skip_setup: | |||
3996 | static void drain_array(struct kmem_cache *cachep, struct kmem_cache_node *n, | 3882 | static void drain_array(struct kmem_cache *cachep, struct kmem_cache_node *n, |
3997 | struct array_cache *ac, int force, int node) | 3883 | struct array_cache *ac, int force, int node) |
3998 | { | 3884 | { |
3885 | LIST_HEAD(list); | ||
3999 | int tofree; | 3886 | int tofree; |
4000 | 3887 | ||
4001 | if (!ac || !ac->avail) | 3888 | if (!ac || !ac->avail) |
@@ -4008,12 +3895,13 @@ static void drain_array(struct kmem_cache *cachep, struct kmem_cache_node *n, | |||
4008 | tofree = force ? ac->avail : (ac->limit + 4) / 5; | 3895 | tofree = force ? ac->avail : (ac->limit + 4) / 5; |
4009 | if (tofree > ac->avail) | 3896 | if (tofree > ac->avail) |
4010 | tofree = (ac->avail + 1) / 2; | 3897 | tofree = (ac->avail + 1) / 2; |
4011 | free_block(cachep, ac->entry, tofree, node); | 3898 | free_block(cachep, ac->entry, tofree, node, &list); |
4012 | ac->avail -= tofree; | 3899 | ac->avail -= tofree; |
4013 | memmove(ac->entry, &(ac->entry[tofree]), | 3900 | memmove(ac->entry, &(ac->entry[tofree]), |
4014 | sizeof(void *) * ac->avail); | 3901 | sizeof(void *) * ac->avail); |
4015 | } | 3902 | } |
4016 | spin_unlock_irq(&n->list_lock); | 3903 | spin_unlock_irq(&n->list_lock); |
3904 | slabs_destroy(cachep, &list); | ||
4017 | } | 3905 | } |
4018 | } | 3906 | } |
4019 | 3907 | ||
@@ -4048,7 +3936,7 @@ static void cache_reap(struct work_struct *w) | |||
4048 | * have established with reasonable certainty that | 3936 | * have established with reasonable certainty that |
4049 | * we can do some work if the lock was obtained. | 3937 | * we can do some work if the lock was obtained. |
4050 | */ | 3938 | */ |
4051 | n = searchp->node[node]; | 3939 | n = get_node(searchp, node); |
4052 | 3940 | ||
4053 | reap_alien(searchp, n); | 3941 | reap_alien(searchp, n); |
4054 | 3942 | ||
@@ -4100,10 +3988,7 @@ void get_slabinfo(struct kmem_cache *cachep, struct slabinfo *sinfo) | |||
4100 | 3988 | ||
4101 | active_objs = 0; | 3989 | active_objs = 0; |
4102 | num_slabs = 0; | 3990 | num_slabs = 0; |
4103 | for_each_online_node(node) { | 3991 | for_each_kmem_cache_node(cachep, node, n) { |
4104 | n = cachep->node[node]; | ||
4105 | if (!n) | ||
4106 | continue; | ||
4107 | 3992 | ||
4108 | check_irq_on(); | 3993 | check_irq_on(); |
4109 | spin_lock_irq(&n->list_lock); | 3994 | spin_lock_irq(&n->list_lock); |
@@ -4328,10 +4213,7 @@ static int leaks_show(struct seq_file *m, void *p) | |||
4328 | 4213 | ||
4329 | x[1] = 0; | 4214 | x[1] = 0; |
4330 | 4215 | ||
4331 | for_each_online_node(node) { | 4216 | for_each_kmem_cache_node(cachep, node, n) { |
4332 | n = cachep->node[node]; | ||
4333 | if (!n) | ||
4334 | continue; | ||
4335 | 4217 | ||
4336 | check_irq_on(); | 4218 | check_irq_on(); |
4337 | spin_lock_irq(&n->list_lock); | 4219 | spin_lock_irq(&n->list_lock); |
@@ -256,13 +256,12 @@ static inline struct kmem_cache *cache_from_obj(struct kmem_cache *s, void *x) | |||
256 | return cachep; | 256 | return cachep; |
257 | 257 | ||
258 | pr_err("%s: Wrong slab cache. %s but object is from %s\n", | 258 | pr_err("%s: Wrong slab cache. %s but object is from %s\n", |
259 | __FUNCTION__, cachep->name, s->name); | 259 | __func__, cachep->name, s->name); |
260 | WARN_ON_ONCE(1); | 260 | WARN_ON_ONCE(1); |
261 | return s; | 261 | return s; |
262 | } | 262 | } |
263 | #endif | ||
264 | |||
265 | 263 | ||
264 | #ifndef CONFIG_SLOB | ||
266 | /* | 265 | /* |
267 | * The slab lists for all objects. | 266 | * The slab lists for all objects. |
268 | */ | 267 | */ |
@@ -277,7 +276,7 @@ struct kmem_cache_node { | |||
277 | unsigned int free_limit; | 276 | unsigned int free_limit; |
278 | unsigned int colour_next; /* Per-node cache coloring */ | 277 | unsigned int colour_next; /* Per-node cache coloring */ |
279 | struct array_cache *shared; /* shared per node */ | 278 | struct array_cache *shared; /* shared per node */ |
280 | struct array_cache **alien; /* on other nodes */ | 279 | struct alien_cache **alien; /* on other nodes */ |
281 | unsigned long next_reap; /* updated without locking */ | 280 | unsigned long next_reap; /* updated without locking */ |
282 | int free_touched; /* updated without locking */ | 281 | int free_touched; /* updated without locking */ |
283 | #endif | 282 | #endif |
@@ -294,5 +293,22 @@ struct kmem_cache_node { | |||
294 | 293 | ||
295 | }; | 294 | }; |
296 | 295 | ||
296 | static inline struct kmem_cache_node *get_node(struct kmem_cache *s, int node) | ||
297 | { | ||
298 | return s->node[node]; | ||
299 | } | ||
300 | |||
301 | /* | ||
302 | * Iterator over all nodes. The body will be executed for each node that has | ||
303 | * a kmem_cache_node structure allocated (which is true for all online nodes) | ||
304 | */ | ||
305 | #define for_each_kmem_cache_node(__s, __node, __n) \ | ||
306 | for (__node = 0; __n = get_node(__s, __node), __node < nr_node_ids; __node++) \ | ||
307 | if (__n) | ||
308 | |||
309 | #endif | ||
310 | |||
297 | void *slab_next(struct seq_file *m, void *p, loff_t *pos); | 311 | void *slab_next(struct seq_file *m, void *p, loff_t *pos); |
298 | void slab_stop(struct seq_file *m, void *p); | 312 | void slab_stop(struct seq_file *m, void *p); |
313 | |||
314 | #endif /* MM_SLAB_H */ | ||
diff --git a/mm/slab_common.c b/mm/slab_common.c index d31c4bacc6a2..d319502b2403 100644 --- a/mm/slab_common.c +++ b/mm/slab_common.c | |||
@@ -19,6 +19,8 @@ | |||
19 | #include <asm/tlbflush.h> | 19 | #include <asm/tlbflush.h> |
20 | #include <asm/page.h> | 20 | #include <asm/page.h> |
21 | #include <linux/memcontrol.h> | 21 | #include <linux/memcontrol.h> |
22 | |||
23 | #define CREATE_TRACE_POINTS | ||
22 | #include <trace/events/kmem.h> | 24 | #include <trace/events/kmem.h> |
23 | 25 | ||
24 | #include "slab.h" | 26 | #include "slab.h" |
@@ -787,3 +789,102 @@ static int __init slab_proc_init(void) | |||
787 | } | 789 | } |
788 | module_init(slab_proc_init); | 790 | module_init(slab_proc_init); |
789 | #endif /* CONFIG_SLABINFO */ | 791 | #endif /* CONFIG_SLABINFO */ |
792 | |||
793 | static __always_inline void *__do_krealloc(const void *p, size_t new_size, | ||
794 | gfp_t flags) | ||
795 | { | ||
796 | void *ret; | ||
797 | size_t ks = 0; | ||
798 | |||
799 | if (p) | ||
800 | ks = ksize(p); | ||
801 | |||
802 | if (ks >= new_size) | ||
803 | return (void *)p; | ||
804 | |||
805 | ret = kmalloc_track_caller(new_size, flags); | ||
806 | if (ret && p) | ||
807 | memcpy(ret, p, ks); | ||
808 | |||
809 | return ret; | ||
810 | } | ||
811 | |||
812 | /** | ||
813 | * __krealloc - like krealloc() but don't free @p. | ||
814 | * @p: object to reallocate memory for. | ||
815 | * @new_size: how many bytes of memory are required. | ||
816 | * @flags: the type of memory to allocate. | ||
817 | * | ||
818 | * This function is like krealloc() except it never frees the originally | ||
819 | * allocated buffer. Use this if you don't want to free the buffer immediately | ||
820 | * like, for example, with RCU. | ||
821 | */ | ||
822 | void *__krealloc(const void *p, size_t new_size, gfp_t flags) | ||
823 | { | ||
824 | if (unlikely(!new_size)) | ||
825 | return ZERO_SIZE_PTR; | ||
826 | |||
827 | return __do_krealloc(p, new_size, flags); | ||
828 | |||
829 | } | ||
830 | EXPORT_SYMBOL(__krealloc); | ||
831 | |||
832 | /** | ||
833 | * krealloc - reallocate memory. The contents will remain unchanged. | ||
834 | * @p: object to reallocate memory for. | ||
835 | * @new_size: how many bytes of memory are required. | ||
836 | * @flags: the type of memory to allocate. | ||
837 | * | ||
838 | * The contents of the object pointed to are preserved up to the | ||
839 | * lesser of the new and old sizes. If @p is %NULL, krealloc() | ||
840 | * behaves exactly like kmalloc(). If @new_size is 0 and @p is not a | ||
841 | * %NULL pointer, the object pointed to is freed. | ||
842 | */ | ||
843 | void *krealloc(const void *p, size_t new_size, gfp_t flags) | ||
844 | { | ||
845 | void *ret; | ||
846 | |||
847 | if (unlikely(!new_size)) { | ||
848 | kfree(p); | ||
849 | return ZERO_SIZE_PTR; | ||
850 | } | ||
851 | |||
852 | ret = __do_krealloc(p, new_size, flags); | ||
853 | if (ret && p != ret) | ||
854 | kfree(p); | ||
855 | |||
856 | return ret; | ||
857 | } | ||
858 | EXPORT_SYMBOL(krealloc); | ||
859 | |||
860 | /** | ||
861 | * kzfree - like kfree but zero memory | ||
862 | * @p: object to free memory of | ||
863 | * | ||
864 | * The memory of the object @p points to is zeroed before freed. | ||
865 | * If @p is %NULL, kzfree() does nothing. | ||
866 | * | ||
867 | * Note: this function zeroes the whole allocated buffer which can be a good | ||
868 | * deal bigger than the requested buffer size passed to kmalloc(). So be | ||
869 | * careful when using this function in performance sensitive code. | ||
870 | */ | ||
871 | void kzfree(const void *p) | ||
872 | { | ||
873 | size_t ks; | ||
874 | void *mem = (void *)p; | ||
875 | |||
876 | if (unlikely(ZERO_OR_NULL_PTR(mem))) | ||
877 | return; | ||
878 | ks = ksize(mem); | ||
879 | memset(mem, 0, ks); | ||
880 | kfree(mem); | ||
881 | } | ||
882 | EXPORT_SYMBOL(kzfree); | ||
883 | |||
884 | /* Tracepoints definitions. */ | ||
885 | EXPORT_TRACEPOINT_SYMBOL(kmalloc); | ||
886 | EXPORT_TRACEPOINT_SYMBOL(kmem_cache_alloc); | ||
887 | EXPORT_TRACEPOINT_SYMBOL(kmalloc_node); | ||
888 | EXPORT_TRACEPOINT_SYMBOL(kmem_cache_alloc_node); | ||
889 | EXPORT_TRACEPOINT_SYMBOL(kfree); | ||
890 | EXPORT_TRACEPOINT_SYMBOL(kmem_cache_free); | ||
@@ -233,11 +233,6 @@ static inline void stat(const struct kmem_cache *s, enum stat_item si) | |||
233 | * Core slab cache functions | 233 | * Core slab cache functions |
234 | *******************************************************************/ | 234 | *******************************************************************/ |
235 | 235 | ||
236 | static inline struct kmem_cache_node *get_node(struct kmem_cache *s, int node) | ||
237 | { | ||
238 | return s->node[node]; | ||
239 | } | ||
240 | |||
241 | /* Verify that a pointer has an address that is valid within a slab page */ | 236 | /* Verify that a pointer has an address that is valid within a slab page */ |
242 | static inline int check_valid_pointer(struct kmem_cache *s, | 237 | static inline int check_valid_pointer(struct kmem_cache *s, |
243 | struct page *page, const void *object) | 238 | struct page *page, const void *object) |
@@ -288,6 +283,10 @@ static inline void set_freepointer(struct kmem_cache *s, void *object, void *fp) | |||
288 | for (__p = (__addr); __p < (__addr) + (__objects) * (__s)->size;\ | 283 | for (__p = (__addr); __p < (__addr) + (__objects) * (__s)->size;\ |
289 | __p += (__s)->size) | 284 | __p += (__s)->size) |
290 | 285 | ||
286 | #define for_each_object_idx(__p, __idx, __s, __addr, __objects) \ | ||
287 | for (__p = (__addr), __idx = 1; __idx <= __objects;\ | ||
288 | __p += (__s)->size, __idx++) | ||
289 | |||
291 | /* Determine object index from a given position */ | 290 | /* Determine object index from a given position */ |
292 | static inline int slab_index(void *p, struct kmem_cache *s, void *addr) | 291 | static inline int slab_index(void *p, struct kmem_cache *s, void *addr) |
293 | { | 292 | { |
@@ -382,9 +381,9 @@ static inline bool __cmpxchg_double_slab(struct kmem_cache *s, struct page *page | |||
382 | defined(CONFIG_HAVE_ALIGNED_STRUCT_PAGE) | 381 | defined(CONFIG_HAVE_ALIGNED_STRUCT_PAGE) |
383 | if (s->flags & __CMPXCHG_DOUBLE) { | 382 | if (s->flags & __CMPXCHG_DOUBLE) { |
384 | if (cmpxchg_double(&page->freelist, &page->counters, | 383 | if (cmpxchg_double(&page->freelist, &page->counters, |
385 | freelist_old, counters_old, | 384 | freelist_old, counters_old, |
386 | freelist_new, counters_new)) | 385 | freelist_new, counters_new)) |
387 | return 1; | 386 | return 1; |
388 | } else | 387 | } else |
389 | #endif | 388 | #endif |
390 | { | 389 | { |
@@ -418,9 +417,9 @@ static inline bool cmpxchg_double_slab(struct kmem_cache *s, struct page *page, | |||
418 | defined(CONFIG_HAVE_ALIGNED_STRUCT_PAGE) | 417 | defined(CONFIG_HAVE_ALIGNED_STRUCT_PAGE) |
419 | if (s->flags & __CMPXCHG_DOUBLE) { | 418 | if (s->flags & __CMPXCHG_DOUBLE) { |
420 | if (cmpxchg_double(&page->freelist, &page->counters, | 419 | if (cmpxchg_double(&page->freelist, &page->counters, |
421 | freelist_old, counters_old, | 420 | freelist_old, counters_old, |
422 | freelist_new, counters_new)) | 421 | freelist_new, counters_new)) |
423 | return 1; | 422 | return 1; |
424 | } else | 423 | } else |
425 | #endif | 424 | #endif |
426 | { | 425 | { |
@@ -945,60 +944,6 @@ static void trace(struct kmem_cache *s, struct page *page, void *object, | |||
945 | } | 944 | } |
946 | 945 | ||
947 | /* | 946 | /* |
948 | * Hooks for other subsystems that check memory allocations. In a typical | ||
949 | * production configuration these hooks all should produce no code at all. | ||
950 | */ | ||
951 | static inline void kmalloc_large_node_hook(void *ptr, size_t size, gfp_t flags) | ||
952 | { | ||
953 | kmemleak_alloc(ptr, size, 1, flags); | ||
954 | } | ||
955 | |||
956 | static inline void kfree_hook(const void *x) | ||
957 | { | ||
958 | kmemleak_free(x); | ||
959 | } | ||
960 | |||
961 | static inline int slab_pre_alloc_hook(struct kmem_cache *s, gfp_t flags) | ||
962 | { | ||
963 | flags &= gfp_allowed_mask; | ||
964 | lockdep_trace_alloc(flags); | ||
965 | might_sleep_if(flags & __GFP_WAIT); | ||
966 | |||
967 | return should_failslab(s->object_size, flags, s->flags); | ||
968 | } | ||
969 | |||
970 | static inline void slab_post_alloc_hook(struct kmem_cache *s, | ||
971 | gfp_t flags, void *object) | ||
972 | { | ||
973 | flags &= gfp_allowed_mask; | ||
974 | kmemcheck_slab_alloc(s, flags, object, slab_ksize(s)); | ||
975 | kmemleak_alloc_recursive(object, s->object_size, 1, s->flags, flags); | ||
976 | } | ||
977 | |||
978 | static inline void slab_free_hook(struct kmem_cache *s, void *x) | ||
979 | { | ||
980 | kmemleak_free_recursive(x, s->flags); | ||
981 | |||
982 | /* | ||
983 | * Trouble is that we may no longer disable interrupts in the fast path | ||
984 | * So in order to make the debug calls that expect irqs to be | ||
985 | * disabled we need to disable interrupts temporarily. | ||
986 | */ | ||
987 | #if defined(CONFIG_KMEMCHECK) || defined(CONFIG_LOCKDEP) | ||
988 | { | ||
989 | unsigned long flags; | ||
990 | |||
991 | local_irq_save(flags); | ||
992 | kmemcheck_slab_free(s, x, s->object_size); | ||
993 | debug_check_no_locks_freed(x, s->object_size); | ||
994 | local_irq_restore(flags); | ||
995 | } | ||
996 | #endif | ||
997 | if (!(s->flags & SLAB_DEBUG_OBJECTS)) | ||
998 | debug_check_no_obj_freed(x, s->object_size); | ||
999 | } | ||
1000 | |||
1001 | /* | ||
1002 | * Tracking of fully allocated slabs for debugging purposes. | 947 | * Tracking of fully allocated slabs for debugging purposes. |
1003 | */ | 948 | */ |
1004 | static void add_full(struct kmem_cache *s, | 949 | static void add_full(struct kmem_cache *s, |
@@ -1282,6 +1227,12 @@ static inline void inc_slabs_node(struct kmem_cache *s, int node, | |||
1282 | static inline void dec_slabs_node(struct kmem_cache *s, int node, | 1227 | static inline void dec_slabs_node(struct kmem_cache *s, int node, |
1283 | int objects) {} | 1228 | int objects) {} |
1284 | 1229 | ||
1230 | #endif /* CONFIG_SLUB_DEBUG */ | ||
1231 | |||
1232 | /* | ||
1233 | * Hooks for other subsystems that check memory allocations. In a typical | ||
1234 | * production configuration these hooks all should produce no code at all. | ||
1235 | */ | ||
1285 | static inline void kmalloc_large_node_hook(void *ptr, size_t size, gfp_t flags) | 1236 | static inline void kmalloc_large_node_hook(void *ptr, size_t size, gfp_t flags) |
1286 | { | 1237 | { |
1287 | kmemleak_alloc(ptr, size, 1, flags); | 1238 | kmemleak_alloc(ptr, size, 1, flags); |
@@ -1293,21 +1244,44 @@ static inline void kfree_hook(const void *x) | |||
1293 | } | 1244 | } |
1294 | 1245 | ||
1295 | static inline int slab_pre_alloc_hook(struct kmem_cache *s, gfp_t flags) | 1246 | static inline int slab_pre_alloc_hook(struct kmem_cache *s, gfp_t flags) |
1296 | { return 0; } | 1247 | { |
1248 | flags &= gfp_allowed_mask; | ||
1249 | lockdep_trace_alloc(flags); | ||
1250 | might_sleep_if(flags & __GFP_WAIT); | ||
1251 | |||
1252 | return should_failslab(s->object_size, flags, s->flags); | ||
1253 | } | ||
1297 | 1254 | ||
1298 | static inline void slab_post_alloc_hook(struct kmem_cache *s, gfp_t flags, | 1255 | static inline void slab_post_alloc_hook(struct kmem_cache *s, |
1299 | void *object) | 1256 | gfp_t flags, void *object) |
1300 | { | 1257 | { |
1301 | kmemleak_alloc_recursive(object, s->object_size, 1, s->flags, | 1258 | flags &= gfp_allowed_mask; |
1302 | flags & gfp_allowed_mask); | 1259 | kmemcheck_slab_alloc(s, flags, object, slab_ksize(s)); |
1260 | kmemleak_alloc_recursive(object, s->object_size, 1, s->flags, flags); | ||
1303 | } | 1261 | } |
1304 | 1262 | ||
1305 | static inline void slab_free_hook(struct kmem_cache *s, void *x) | 1263 | static inline void slab_free_hook(struct kmem_cache *s, void *x) |
1306 | { | 1264 | { |
1307 | kmemleak_free_recursive(x, s->flags); | 1265 | kmemleak_free_recursive(x, s->flags); |
1308 | } | ||
1309 | 1266 | ||
1310 | #endif /* CONFIG_SLUB_DEBUG */ | 1267 | /* |
1268 | * Trouble is that we may no longer disable interrupts in the fast path | ||
1269 | * So in order to make the debug calls that expect irqs to be | ||
1270 | * disabled we need to disable interrupts temporarily. | ||
1271 | */ | ||
1272 | #if defined(CONFIG_KMEMCHECK) || defined(CONFIG_LOCKDEP) | ||
1273 | { | ||
1274 | unsigned long flags; | ||
1275 | |||
1276 | local_irq_save(flags); | ||
1277 | kmemcheck_slab_free(s, x, s->object_size); | ||
1278 | debug_check_no_locks_freed(x, s->object_size); | ||
1279 | local_irq_restore(flags); | ||
1280 | } | ||
1281 | #endif | ||
1282 | if (!(s->flags & SLAB_DEBUG_OBJECTS)) | ||
1283 | debug_check_no_obj_freed(x, s->object_size); | ||
1284 | } | ||
1311 | 1285 | ||
1312 | /* | 1286 | /* |
1313 | * Slab allocation and freeing | 1287 | * Slab allocation and freeing |
@@ -1409,9 +1383,9 @@ static struct page *new_slab(struct kmem_cache *s, gfp_t flags, int node) | |||
1409 | { | 1383 | { |
1410 | struct page *page; | 1384 | struct page *page; |
1411 | void *start; | 1385 | void *start; |
1412 | void *last; | ||
1413 | void *p; | 1386 | void *p; |
1414 | int order; | 1387 | int order; |
1388 | int idx; | ||
1415 | 1389 | ||
1416 | BUG_ON(flags & GFP_SLAB_BUG_MASK); | 1390 | BUG_ON(flags & GFP_SLAB_BUG_MASK); |
1417 | 1391 | ||
@@ -1432,14 +1406,13 @@ static struct page *new_slab(struct kmem_cache *s, gfp_t flags, int node) | |||
1432 | if (unlikely(s->flags & SLAB_POISON)) | 1406 | if (unlikely(s->flags & SLAB_POISON)) |
1433 | memset(start, POISON_INUSE, PAGE_SIZE << order); | 1407 | memset(start, POISON_INUSE, PAGE_SIZE << order); |
1434 | 1408 | ||
1435 | last = start; | 1409 | for_each_object_idx(p, idx, s, start, page->objects) { |
1436 | for_each_object(p, s, start, page->objects) { | 1410 | setup_object(s, page, p); |
1437 | setup_object(s, page, last); | 1411 | if (likely(idx < page->objects)) |
1438 | set_freepointer(s, last, p); | 1412 | set_freepointer(s, p, p + s->size); |
1439 | last = p; | 1413 | else |
1414 | set_freepointer(s, p, NULL); | ||
1440 | } | 1415 | } |
1441 | setup_object(s, page, last); | ||
1442 | set_freepointer(s, last, NULL); | ||
1443 | 1416 | ||
1444 | page->freelist = start; | 1417 | page->freelist = start; |
1445 | page->inuse = page->objects; | 1418 | page->inuse = page->objects; |
@@ -2162,6 +2135,7 @@ slab_out_of_memory(struct kmem_cache *s, gfp_t gfpflags, int nid) | |||
2162 | static DEFINE_RATELIMIT_STATE(slub_oom_rs, DEFAULT_RATELIMIT_INTERVAL, | 2135 | static DEFINE_RATELIMIT_STATE(slub_oom_rs, DEFAULT_RATELIMIT_INTERVAL, |
2163 | DEFAULT_RATELIMIT_BURST); | 2136 | DEFAULT_RATELIMIT_BURST); |
2164 | int node; | 2137 | int node; |
2138 | struct kmem_cache_node *n; | ||
2165 | 2139 | ||
2166 | if ((gfpflags & __GFP_NOWARN) || !__ratelimit(&slub_oom_rs)) | 2140 | if ((gfpflags & __GFP_NOWARN) || !__ratelimit(&slub_oom_rs)) |
2167 | return; | 2141 | return; |
@@ -2176,15 +2150,11 @@ slab_out_of_memory(struct kmem_cache *s, gfp_t gfpflags, int nid) | |||
2176 | pr_warn(" %s debugging increased min order, use slub_debug=O to disable.\n", | 2150 | pr_warn(" %s debugging increased min order, use slub_debug=O to disable.\n", |
2177 | s->name); | 2151 | s->name); |
2178 | 2152 | ||
2179 | for_each_online_node(node) { | 2153 | for_each_kmem_cache_node(s, node, n) { |
2180 | struct kmem_cache_node *n = get_node(s, node); | ||
2181 | unsigned long nr_slabs; | 2154 | unsigned long nr_slabs; |
2182 | unsigned long nr_objs; | 2155 | unsigned long nr_objs; |
2183 | unsigned long nr_free; | 2156 | unsigned long nr_free; |
2184 | 2157 | ||
2185 | if (!n) | ||
2186 | continue; | ||
2187 | |||
2188 | nr_free = count_partial(n, count_free); | 2158 | nr_free = count_partial(n, count_free); |
2189 | nr_slabs = node_nr_slabs(n); | 2159 | nr_slabs = node_nr_slabs(n); |
2190 | nr_objs = node_nr_objs(n); | 2160 | nr_objs = node_nr_objs(n); |
@@ -2928,13 +2898,10 @@ static void early_kmem_cache_node_alloc(int node) | |||
2928 | static void free_kmem_cache_nodes(struct kmem_cache *s) | 2898 | static void free_kmem_cache_nodes(struct kmem_cache *s) |
2929 | { | 2899 | { |
2930 | int node; | 2900 | int node; |
2901 | struct kmem_cache_node *n; | ||
2931 | 2902 | ||
2932 | for_each_node_state(node, N_NORMAL_MEMORY) { | 2903 | for_each_kmem_cache_node(s, node, n) { |
2933 | struct kmem_cache_node *n = s->node[node]; | 2904 | kmem_cache_free(kmem_cache_node, n); |
2934 | |||
2935 | if (n) | ||
2936 | kmem_cache_free(kmem_cache_node, n); | ||
2937 | |||
2938 | s->node[node] = NULL; | 2905 | s->node[node] = NULL; |
2939 | } | 2906 | } |
2940 | } | 2907 | } |
@@ -3222,12 +3189,11 @@ static void free_partial(struct kmem_cache *s, struct kmem_cache_node *n) | |||
3222 | static inline int kmem_cache_close(struct kmem_cache *s) | 3189 | static inline int kmem_cache_close(struct kmem_cache *s) |
3223 | { | 3190 | { |
3224 | int node; | 3191 | int node; |
3192 | struct kmem_cache_node *n; | ||
3225 | 3193 | ||
3226 | flush_all(s); | 3194 | flush_all(s); |
3227 | /* Attempt to free all objects */ | 3195 | /* Attempt to free all objects */ |
3228 | for_each_node_state(node, N_NORMAL_MEMORY) { | 3196 | for_each_kmem_cache_node(s, node, n) { |
3229 | struct kmem_cache_node *n = get_node(s, node); | ||
3230 | |||
3231 | free_partial(s, n); | 3197 | free_partial(s, n); |
3232 | if (n->nr_partial || slabs_node(s, node)) | 3198 | if (n->nr_partial || slabs_node(s, node)) |
3233 | return 1; | 3199 | return 1; |
@@ -3412,9 +3378,7 @@ int __kmem_cache_shrink(struct kmem_cache *s) | |||
3412 | return -ENOMEM; | 3378 | return -ENOMEM; |
3413 | 3379 | ||
3414 | flush_all(s); | 3380 | flush_all(s); |
3415 | for_each_node_state(node, N_NORMAL_MEMORY) { | 3381 | for_each_kmem_cache_node(s, node, n) { |
3416 | n = get_node(s, node); | ||
3417 | |||
3418 | if (!n->nr_partial) | 3382 | if (!n->nr_partial) |
3419 | continue; | 3383 | continue; |
3420 | 3384 | ||
@@ -3586,6 +3550,7 @@ static struct kmem_cache * __init bootstrap(struct kmem_cache *static_cache) | |||
3586 | { | 3550 | { |
3587 | int node; | 3551 | int node; |
3588 | struct kmem_cache *s = kmem_cache_zalloc(kmem_cache, GFP_NOWAIT); | 3552 | struct kmem_cache *s = kmem_cache_zalloc(kmem_cache, GFP_NOWAIT); |
3553 | struct kmem_cache_node *n; | ||
3589 | 3554 | ||
3590 | memcpy(s, static_cache, kmem_cache->object_size); | 3555 | memcpy(s, static_cache, kmem_cache->object_size); |
3591 | 3556 | ||
@@ -3595,19 +3560,16 @@ static struct kmem_cache * __init bootstrap(struct kmem_cache *static_cache) | |||
3595 | * IPIs around. | 3560 | * IPIs around. |
3596 | */ | 3561 | */ |
3597 | __flush_cpu_slab(s, smp_processor_id()); | 3562 | __flush_cpu_slab(s, smp_processor_id()); |
3598 | for_each_node_state(node, N_NORMAL_MEMORY) { | 3563 | for_each_kmem_cache_node(s, node, n) { |
3599 | struct kmem_cache_node *n = get_node(s, node); | ||
3600 | struct page *p; | 3564 | struct page *p; |
3601 | 3565 | ||
3602 | if (n) { | 3566 | list_for_each_entry(p, &n->partial, lru) |
3603 | list_for_each_entry(p, &n->partial, lru) | 3567 | p->slab_cache = s; |
3604 | p->slab_cache = s; | ||
3605 | 3568 | ||
3606 | #ifdef CONFIG_SLUB_DEBUG | 3569 | #ifdef CONFIG_SLUB_DEBUG |
3607 | list_for_each_entry(p, &n->full, lru) | 3570 | list_for_each_entry(p, &n->full, lru) |
3608 | p->slab_cache = s; | 3571 | p->slab_cache = s; |
3609 | #endif | 3572 | #endif |
3610 | } | ||
3611 | } | 3573 | } |
3612 | list_add(&s->list, &slab_caches); | 3574 | list_add(&s->list, &slab_caches); |
3613 | return s; | 3575 | return s; |
@@ -3960,16 +3922,14 @@ static long validate_slab_cache(struct kmem_cache *s) | |||
3960 | unsigned long count = 0; | 3922 | unsigned long count = 0; |
3961 | unsigned long *map = kmalloc(BITS_TO_LONGS(oo_objects(s->max)) * | 3923 | unsigned long *map = kmalloc(BITS_TO_LONGS(oo_objects(s->max)) * |
3962 | sizeof(unsigned long), GFP_KERNEL); | 3924 | sizeof(unsigned long), GFP_KERNEL); |
3925 | struct kmem_cache_node *n; | ||
3963 | 3926 | ||
3964 | if (!map) | 3927 | if (!map) |
3965 | return -ENOMEM; | 3928 | return -ENOMEM; |
3966 | 3929 | ||
3967 | flush_all(s); | 3930 | flush_all(s); |
3968 | for_each_node_state(node, N_NORMAL_MEMORY) { | 3931 | for_each_kmem_cache_node(s, node, n) |
3969 | struct kmem_cache_node *n = get_node(s, node); | ||
3970 | |||
3971 | count += validate_slab_node(s, n, map); | 3932 | count += validate_slab_node(s, n, map); |
3972 | } | ||
3973 | kfree(map); | 3933 | kfree(map); |
3974 | return count; | 3934 | return count; |
3975 | } | 3935 | } |
@@ -4123,6 +4083,7 @@ static int list_locations(struct kmem_cache *s, char *buf, | |||
4123 | int node; | 4083 | int node; |
4124 | unsigned long *map = kmalloc(BITS_TO_LONGS(oo_objects(s->max)) * | 4084 | unsigned long *map = kmalloc(BITS_TO_LONGS(oo_objects(s->max)) * |
4125 | sizeof(unsigned long), GFP_KERNEL); | 4085 | sizeof(unsigned long), GFP_KERNEL); |
4086 | struct kmem_cache_node *n; | ||
4126 | 4087 | ||
4127 | if (!map || !alloc_loc_track(&t, PAGE_SIZE / sizeof(struct location), | 4088 | if (!map || !alloc_loc_track(&t, PAGE_SIZE / sizeof(struct location), |
4128 | GFP_TEMPORARY)) { | 4089 | GFP_TEMPORARY)) { |
@@ -4132,8 +4093,7 @@ static int list_locations(struct kmem_cache *s, char *buf, | |||
4132 | /* Push back cpu slabs */ | 4093 | /* Push back cpu slabs */ |
4133 | flush_all(s); | 4094 | flush_all(s); |
4134 | 4095 | ||
4135 | for_each_node_state(node, N_NORMAL_MEMORY) { | 4096 | for_each_kmem_cache_node(s, node, n) { |
4136 | struct kmem_cache_node *n = get_node(s, node); | ||
4137 | unsigned long flags; | 4097 | unsigned long flags; |
4138 | struct page *page; | 4098 | struct page *page; |
4139 | 4099 | ||
@@ -4205,7 +4165,7 @@ static int list_locations(struct kmem_cache *s, char *buf, | |||
4205 | #endif | 4165 | #endif |
4206 | 4166 | ||
4207 | #ifdef SLUB_RESILIENCY_TEST | 4167 | #ifdef SLUB_RESILIENCY_TEST |
4208 | static void resiliency_test(void) | 4168 | static void __init resiliency_test(void) |
4209 | { | 4169 | { |
4210 | u8 *p; | 4170 | u8 *p; |
4211 | 4171 | ||
@@ -4332,8 +4292,9 @@ static ssize_t show_slab_objects(struct kmem_cache *s, | |||
4332 | get_online_mems(); | 4292 | get_online_mems(); |
4333 | #ifdef CONFIG_SLUB_DEBUG | 4293 | #ifdef CONFIG_SLUB_DEBUG |
4334 | if (flags & SO_ALL) { | 4294 | if (flags & SO_ALL) { |
4335 | for_each_node_state(node, N_NORMAL_MEMORY) { | 4295 | struct kmem_cache_node *n; |
4336 | struct kmem_cache_node *n = get_node(s, node); | 4296 | |
4297 | for_each_kmem_cache_node(s, node, n) { | ||
4337 | 4298 | ||
4338 | if (flags & SO_TOTAL) | 4299 | if (flags & SO_TOTAL) |
4339 | x = atomic_long_read(&n->total_objects); | 4300 | x = atomic_long_read(&n->total_objects); |
@@ -4349,9 +4310,9 @@ static ssize_t show_slab_objects(struct kmem_cache *s, | |||
4349 | } else | 4310 | } else |
4350 | #endif | 4311 | #endif |
4351 | if (flags & SO_PARTIAL) { | 4312 | if (flags & SO_PARTIAL) { |
4352 | for_each_node_state(node, N_NORMAL_MEMORY) { | 4313 | struct kmem_cache_node *n; |
4353 | struct kmem_cache_node *n = get_node(s, node); | ||
4354 | 4314 | ||
4315 | for_each_kmem_cache_node(s, node, n) { | ||
4355 | if (flags & SO_TOTAL) | 4316 | if (flags & SO_TOTAL) |
4356 | x = count_partial(n, count_total); | 4317 | x = count_partial(n, count_total); |
4357 | else if (flags & SO_OBJECTS) | 4318 | else if (flags & SO_OBJECTS) |
@@ -4364,7 +4325,7 @@ static ssize_t show_slab_objects(struct kmem_cache *s, | |||
4364 | } | 4325 | } |
4365 | x = sprintf(buf, "%lu", total); | 4326 | x = sprintf(buf, "%lu", total); |
4366 | #ifdef CONFIG_NUMA | 4327 | #ifdef CONFIG_NUMA |
4367 | for_each_node_state(node, N_NORMAL_MEMORY) | 4328 | for (node = 0; node < nr_node_ids; node++) |
4368 | if (nodes[node]) | 4329 | if (nodes[node]) |
4369 | x += sprintf(buf + x, " N%d=%lu", | 4330 | x += sprintf(buf + x, " N%d=%lu", |
4370 | node, nodes[node]); | 4331 | node, nodes[node]); |
@@ -4378,16 +4339,12 @@ static ssize_t show_slab_objects(struct kmem_cache *s, | |||
4378 | static int any_slab_objects(struct kmem_cache *s) | 4339 | static int any_slab_objects(struct kmem_cache *s) |
4379 | { | 4340 | { |
4380 | int node; | 4341 | int node; |
4342 | struct kmem_cache_node *n; | ||
4381 | 4343 | ||
4382 | for_each_online_node(node) { | 4344 | for_each_kmem_cache_node(s, node, n) |
4383 | struct kmem_cache_node *n = get_node(s, node); | ||
4384 | |||
4385 | if (!n) | ||
4386 | continue; | ||
4387 | |||
4388 | if (atomic_long_read(&n->total_objects)) | 4345 | if (atomic_long_read(&n->total_objects)) |
4389 | return 1; | 4346 | return 1; |
4390 | } | 4347 | |
4391 | return 0; | 4348 | return 0; |
4392 | } | 4349 | } |
4393 | #endif | 4350 | #endif |
@@ -4509,7 +4466,7 @@ SLAB_ATTR_RO(ctor); | |||
4509 | 4466 | ||
4510 | static ssize_t aliases_show(struct kmem_cache *s, char *buf) | 4467 | static ssize_t aliases_show(struct kmem_cache *s, char *buf) |
4511 | { | 4468 | { |
4512 | return sprintf(buf, "%d\n", s->refcount - 1); | 4469 | return sprintf(buf, "%d\n", s->refcount < 0 ? 0 : s->refcount - 1); |
4513 | } | 4470 | } |
4514 | SLAB_ATTR_RO(aliases); | 4471 | SLAB_ATTR_RO(aliases); |
4515 | 4472 | ||
@@ -5171,12 +5128,6 @@ static char *create_unique_id(struct kmem_cache *s) | |||
5171 | *p++ = '-'; | 5128 | *p++ = '-'; |
5172 | p += sprintf(p, "%07d", s->size); | 5129 | p += sprintf(p, "%07d", s->size); |
5173 | 5130 | ||
5174 | #ifdef CONFIG_MEMCG_KMEM | ||
5175 | if (!is_root_cache(s)) | ||
5176 | p += sprintf(p, "-%08d", | ||
5177 | memcg_cache_id(s->memcg_params->memcg)); | ||
5178 | #endif | ||
5179 | |||
5180 | BUG_ON(p > name + ID_STR_LENGTH - 1); | 5131 | BUG_ON(p > name + ID_STR_LENGTH - 1); |
5181 | return name; | 5132 | return name; |
5182 | } | 5133 | } |
@@ -5342,13 +5293,9 @@ void get_slabinfo(struct kmem_cache *s, struct slabinfo *sinfo) | |||
5342 | unsigned long nr_objs = 0; | 5293 | unsigned long nr_objs = 0; |
5343 | unsigned long nr_free = 0; | 5294 | unsigned long nr_free = 0; |
5344 | int node; | 5295 | int node; |
5296 | struct kmem_cache_node *n; | ||
5345 | 5297 | ||
5346 | for_each_online_node(node) { | 5298 | for_each_kmem_cache_node(s, node, n) { |
5347 | struct kmem_cache_node *n = get_node(s, node); | ||
5348 | |||
5349 | if (!n) | ||
5350 | continue; | ||
5351 | |||
5352 | nr_slabs += node_nr_slabs(n); | 5299 | nr_slabs += node_nr_slabs(n); |
5353 | nr_objs += node_nr_objs(n); | 5300 | nr_objs += node_nr_objs(n); |
5354 | nr_free += count_partial(n, count_free); | 5301 | nr_free += count_partial(n, count_free); |
@@ -501,7 +501,7 @@ static void __activate_page(struct page *page, struct lruvec *lruvec, | |||
501 | SetPageActive(page); | 501 | SetPageActive(page); |
502 | lru += LRU_ACTIVE; | 502 | lru += LRU_ACTIVE; |
503 | add_page_to_lru_list(page, lruvec, lru); | 503 | add_page_to_lru_list(page, lruvec, lru); |
504 | trace_mm_lru_activate(page, page_to_pfn(page)); | 504 | trace_mm_lru_activate(page); |
505 | 505 | ||
506 | __count_vm_event(PGACTIVATE); | 506 | __count_vm_event(PGACTIVATE); |
507 | update_page_reclaim_stat(lruvec, file, 1); | 507 | update_page_reclaim_stat(lruvec, file, 1); |
@@ -589,6 +589,9 @@ static void __lru_cache_activate_page(struct page *page) | |||
589 | * inactive,unreferenced -> inactive,referenced | 589 | * inactive,unreferenced -> inactive,referenced |
590 | * inactive,referenced -> active,unreferenced | 590 | * inactive,referenced -> active,unreferenced |
591 | * active,unreferenced -> active,referenced | 591 | * active,unreferenced -> active,referenced |
592 | * | ||
593 | * When a newly allocated page is not yet visible, so safe for non-atomic ops, | ||
594 | * __SetPageReferenced(page) may be substituted for mark_page_accessed(page). | ||
592 | */ | 595 | */ |
593 | void mark_page_accessed(struct page *page) | 596 | void mark_page_accessed(struct page *page) |
594 | { | 597 | { |
@@ -614,17 +617,6 @@ void mark_page_accessed(struct page *page) | |||
614 | } | 617 | } |
615 | EXPORT_SYMBOL(mark_page_accessed); | 618 | EXPORT_SYMBOL(mark_page_accessed); |
616 | 619 | ||
617 | /* | ||
618 | * Used to mark_page_accessed(page) that is not visible yet and when it is | ||
619 | * still safe to use non-atomic ops | ||
620 | */ | ||
621 | void init_page_accessed(struct page *page) | ||
622 | { | ||
623 | if (!PageReferenced(page)) | ||
624 | __SetPageReferenced(page); | ||
625 | } | ||
626 | EXPORT_SYMBOL(init_page_accessed); | ||
627 | |||
628 | static void __lru_cache_add(struct page *page) | 620 | static void __lru_cache_add(struct page *page) |
629 | { | 621 | { |
630 | struct pagevec *pvec = &get_cpu_var(lru_add_pvec); | 622 | struct pagevec *pvec = &get_cpu_var(lru_add_pvec); |
@@ -996,7 +988,7 @@ static void __pagevec_lru_add_fn(struct page *page, struct lruvec *lruvec, | |||
996 | SetPageLRU(page); | 988 | SetPageLRU(page); |
997 | add_page_to_lru_list(page, lruvec, lru); | 989 | add_page_to_lru_list(page, lruvec, lru); |
998 | update_page_reclaim_stat(lruvec, file, active); | 990 | update_page_reclaim_stat(lruvec, file, active); |
999 | trace_mm_lru_insertion(page, page_to_pfn(page), lru, trace_pagemap_flags(page)); | 991 | trace_mm_lru_insertion(page, lru); |
1000 | } | 992 | } |
1001 | 993 | ||
1002 | /* | 994 | /* |
@@ -16,9 +16,6 @@ | |||
16 | 16 | ||
17 | #include "internal.h" | 17 | #include "internal.h" |
18 | 18 | ||
19 | #define CREATE_TRACE_POINTS | ||
20 | #include <trace/events/kmem.h> | ||
21 | |||
22 | /** | 19 | /** |
23 | * kstrdup - allocate space for and copy an existing string | 20 | * kstrdup - allocate space for and copy an existing string |
24 | * @s: the string to duplicate | 21 | * @s: the string to duplicate |
@@ -112,97 +109,6 @@ void *memdup_user(const void __user *src, size_t len) | |||
112 | } | 109 | } |
113 | EXPORT_SYMBOL(memdup_user); | 110 | EXPORT_SYMBOL(memdup_user); |
114 | 111 | ||
115 | static __always_inline void *__do_krealloc(const void *p, size_t new_size, | ||
116 | gfp_t flags) | ||
117 | { | ||
118 | void *ret; | ||
119 | size_t ks = 0; | ||
120 | |||
121 | if (p) | ||
122 | ks = ksize(p); | ||
123 | |||
124 | if (ks >= new_size) | ||
125 | return (void *)p; | ||
126 | |||
127 | ret = kmalloc_track_caller(new_size, flags); | ||
128 | if (ret && p) | ||
129 | memcpy(ret, p, ks); | ||
130 | |||
131 | return ret; | ||
132 | } | ||
133 | |||
134 | /** | ||
135 | * __krealloc - like krealloc() but don't free @p. | ||
136 | * @p: object to reallocate memory for. | ||
137 | * @new_size: how many bytes of memory are required. | ||
138 | * @flags: the type of memory to allocate. | ||
139 | * | ||
140 | * This function is like krealloc() except it never frees the originally | ||
141 | * allocated buffer. Use this if you don't want to free the buffer immediately | ||
142 | * like, for example, with RCU. | ||
143 | */ | ||
144 | void *__krealloc(const void *p, size_t new_size, gfp_t flags) | ||
145 | { | ||
146 | if (unlikely(!new_size)) | ||
147 | return ZERO_SIZE_PTR; | ||
148 | |||
149 | return __do_krealloc(p, new_size, flags); | ||
150 | |||
151 | } | ||
152 | EXPORT_SYMBOL(__krealloc); | ||
153 | |||
154 | /** | ||
155 | * krealloc - reallocate memory. The contents will remain unchanged. | ||
156 | * @p: object to reallocate memory for. | ||
157 | * @new_size: how many bytes of memory are required. | ||
158 | * @flags: the type of memory to allocate. | ||
159 | * | ||
160 | * The contents of the object pointed to are preserved up to the | ||
161 | * lesser of the new and old sizes. If @p is %NULL, krealloc() | ||
162 | * behaves exactly like kmalloc(). If @new_size is 0 and @p is not a | ||
163 | * %NULL pointer, the object pointed to is freed. | ||
164 | */ | ||
165 | void *krealloc(const void *p, size_t new_size, gfp_t flags) | ||
166 | { | ||
167 | void *ret; | ||
168 | |||
169 | if (unlikely(!new_size)) { | ||
170 | kfree(p); | ||
171 | return ZERO_SIZE_PTR; | ||
172 | } | ||
173 | |||
174 | ret = __do_krealloc(p, new_size, flags); | ||
175 | if (ret && p != ret) | ||
176 | kfree(p); | ||
177 | |||
178 | return ret; | ||
179 | } | ||
180 | EXPORT_SYMBOL(krealloc); | ||
181 | |||
182 | /** | ||
183 | * kzfree - like kfree but zero memory | ||
184 | * @p: object to free memory of | ||
185 | * | ||
186 | * The memory of the object @p points to is zeroed before freed. | ||
187 | * If @p is %NULL, kzfree() does nothing. | ||
188 | * | ||
189 | * Note: this function zeroes the whole allocated buffer which can be a good | ||
190 | * deal bigger than the requested buffer size passed to kmalloc(). So be | ||
191 | * careful when using this function in performance sensitive code. | ||
192 | */ | ||
193 | void kzfree(const void *p) | ||
194 | { | ||
195 | size_t ks; | ||
196 | void *mem = (void *)p; | ||
197 | |||
198 | if (unlikely(ZERO_OR_NULL_PTR(mem))) | ||
199 | return; | ||
200 | ks = ksize(mem); | ||
201 | memset(mem, 0, ks); | ||
202 | kfree(mem); | ||
203 | } | ||
204 | EXPORT_SYMBOL(kzfree); | ||
205 | |||
206 | /* | 112 | /* |
207 | * strndup_user - duplicate an existing string from user space | 113 | * strndup_user - duplicate an existing string from user space |
208 | * @s: The string to duplicate | 114 | * @s: The string to duplicate |
@@ -504,11 +410,3 @@ out_mm: | |||
504 | out: | 410 | out: |
505 | return res; | 411 | return res; |
506 | } | 412 | } |
507 | |||
508 | /* Tracepoints definitions. */ | ||
509 | EXPORT_TRACEPOINT_SYMBOL(kmalloc); | ||
510 | EXPORT_TRACEPOINT_SYMBOL(kmem_cache_alloc); | ||
511 | EXPORT_TRACEPOINT_SYMBOL(kmalloc_node); | ||
512 | EXPORT_TRACEPOINT_SYMBOL(kmem_cache_alloc_node); | ||
513 | EXPORT_TRACEPOINT_SYMBOL(kfree); | ||
514 | EXPORT_TRACEPOINT_SYMBOL(kmem_cache_free); | ||
diff --git a/mm/vmalloc.c b/mm/vmalloc.c index f64632b67196..2b0aa5486092 100644 --- a/mm/vmalloc.c +++ b/mm/vmalloc.c | |||
@@ -1270,19 +1270,15 @@ void unmap_kernel_range(unsigned long addr, unsigned long size) | |||
1270 | } | 1270 | } |
1271 | EXPORT_SYMBOL_GPL(unmap_kernel_range); | 1271 | EXPORT_SYMBOL_GPL(unmap_kernel_range); |
1272 | 1272 | ||
1273 | int map_vm_area(struct vm_struct *area, pgprot_t prot, struct page ***pages) | 1273 | int map_vm_area(struct vm_struct *area, pgprot_t prot, struct page **pages) |
1274 | { | 1274 | { |
1275 | unsigned long addr = (unsigned long)area->addr; | 1275 | unsigned long addr = (unsigned long)area->addr; |
1276 | unsigned long end = addr + get_vm_area_size(area); | 1276 | unsigned long end = addr + get_vm_area_size(area); |
1277 | int err; | 1277 | int err; |
1278 | 1278 | ||
1279 | err = vmap_page_range(addr, end, prot, *pages); | 1279 | err = vmap_page_range(addr, end, prot, pages); |
1280 | if (err > 0) { | ||
1281 | *pages += err; | ||
1282 | err = 0; | ||
1283 | } | ||
1284 | 1280 | ||
1285 | return err; | 1281 | return err > 0 ? 0 : err; |
1286 | } | 1282 | } |
1287 | EXPORT_SYMBOL_GPL(map_vm_area); | 1283 | EXPORT_SYMBOL_GPL(map_vm_area); |
1288 | 1284 | ||
@@ -1548,7 +1544,7 @@ void *vmap(struct page **pages, unsigned int count, | |||
1548 | if (!area) | 1544 | if (!area) |
1549 | return NULL; | 1545 | return NULL; |
1550 | 1546 | ||
1551 | if (map_vm_area(area, prot, &pages)) { | 1547 | if (map_vm_area(area, prot, pages)) { |
1552 | vunmap(area->addr); | 1548 | vunmap(area->addr); |
1553 | return NULL; | 1549 | return NULL; |
1554 | } | 1550 | } |
@@ -1566,7 +1562,8 @@ static void *__vmalloc_area_node(struct vm_struct *area, gfp_t gfp_mask, | |||
1566 | const int order = 0; | 1562 | const int order = 0; |
1567 | struct page **pages; | 1563 | struct page **pages; |
1568 | unsigned int nr_pages, array_size, i; | 1564 | unsigned int nr_pages, array_size, i; |
1569 | gfp_t nested_gfp = (gfp_mask & GFP_RECLAIM_MASK) | __GFP_ZERO; | 1565 | const gfp_t nested_gfp = (gfp_mask & GFP_RECLAIM_MASK) | __GFP_ZERO; |
1566 | const gfp_t alloc_mask = gfp_mask | __GFP_NOWARN; | ||
1570 | 1567 | ||
1571 | nr_pages = get_vm_area_size(area) >> PAGE_SHIFT; | 1568 | nr_pages = get_vm_area_size(area) >> PAGE_SHIFT; |
1572 | array_size = (nr_pages * sizeof(struct page *)); | 1569 | array_size = (nr_pages * sizeof(struct page *)); |
@@ -1589,12 +1586,11 @@ static void *__vmalloc_area_node(struct vm_struct *area, gfp_t gfp_mask, | |||
1589 | 1586 | ||
1590 | for (i = 0; i < area->nr_pages; i++) { | 1587 | for (i = 0; i < area->nr_pages; i++) { |
1591 | struct page *page; | 1588 | struct page *page; |
1592 | gfp_t tmp_mask = gfp_mask | __GFP_NOWARN; | ||
1593 | 1589 | ||
1594 | if (node == NUMA_NO_NODE) | 1590 | if (node == NUMA_NO_NODE) |
1595 | page = alloc_page(tmp_mask); | 1591 | page = alloc_page(alloc_mask); |
1596 | else | 1592 | else |
1597 | page = alloc_pages_node(node, tmp_mask, order); | 1593 | page = alloc_pages_node(node, alloc_mask, order); |
1598 | 1594 | ||
1599 | if (unlikely(!page)) { | 1595 | if (unlikely(!page)) { |
1600 | /* Successfully allocated i pages, free them in __vunmap() */ | 1596 | /* Successfully allocated i pages, free them in __vunmap() */ |
@@ -1602,9 +1598,11 @@ static void *__vmalloc_area_node(struct vm_struct *area, gfp_t gfp_mask, | |||
1602 | goto fail; | 1598 | goto fail; |
1603 | } | 1599 | } |
1604 | area->pages[i] = page; | 1600 | area->pages[i] = page; |
1601 | if (gfp_mask & __GFP_WAIT) | ||
1602 | cond_resched(); | ||
1605 | } | 1603 | } |
1606 | 1604 | ||
1607 | if (map_vm_area(area, prot, &pages)) | 1605 | if (map_vm_area(area, prot, pages)) |
1608 | goto fail; | 1606 | goto fail; |
1609 | return area->addr; | 1607 | return area->addr; |
1610 | 1608 | ||
@@ -2690,14 +2688,14 @@ void get_vmalloc_info(struct vmalloc_info *vmi) | |||
2690 | 2688 | ||
2691 | prev_end = VMALLOC_START; | 2689 | prev_end = VMALLOC_START; |
2692 | 2690 | ||
2693 | spin_lock(&vmap_area_lock); | 2691 | rcu_read_lock(); |
2694 | 2692 | ||
2695 | if (list_empty(&vmap_area_list)) { | 2693 | if (list_empty(&vmap_area_list)) { |
2696 | vmi->largest_chunk = VMALLOC_TOTAL; | 2694 | vmi->largest_chunk = VMALLOC_TOTAL; |
2697 | goto out; | 2695 | goto out; |
2698 | } | 2696 | } |
2699 | 2697 | ||
2700 | list_for_each_entry(va, &vmap_area_list, list) { | 2698 | list_for_each_entry_rcu(va, &vmap_area_list, list) { |
2701 | unsigned long addr = va->va_start; | 2699 | unsigned long addr = va->va_start; |
2702 | 2700 | ||
2703 | /* | 2701 | /* |
@@ -2724,7 +2722,7 @@ void get_vmalloc_info(struct vmalloc_info *vmi) | |||
2724 | vmi->largest_chunk = VMALLOC_END - prev_end; | 2722 | vmi->largest_chunk = VMALLOC_END - prev_end; |
2725 | 2723 | ||
2726 | out: | 2724 | out: |
2727 | spin_unlock(&vmap_area_lock); | 2725 | rcu_read_unlock(); |
2728 | } | 2726 | } |
2729 | #endif | 2727 | #endif |
2730 | 2728 | ||
diff --git a/mm/vmscan.c b/mm/vmscan.c index 0f16ffe8eb67..d2f65c856350 100644 --- a/mm/vmscan.c +++ b/mm/vmscan.c | |||
@@ -59,35 +59,20 @@ | |||
59 | #include <trace/events/vmscan.h> | 59 | #include <trace/events/vmscan.h> |
60 | 60 | ||
61 | struct scan_control { | 61 | struct scan_control { |
62 | /* Incremented by the number of inactive pages that were scanned */ | ||
63 | unsigned long nr_scanned; | ||
64 | |||
65 | /* Number of pages freed so far during a call to shrink_zones() */ | ||
66 | unsigned long nr_reclaimed; | ||
67 | |||
68 | /* How many pages shrink_list() should reclaim */ | 62 | /* How many pages shrink_list() should reclaim */ |
69 | unsigned long nr_to_reclaim; | 63 | unsigned long nr_to_reclaim; |
70 | 64 | ||
71 | unsigned long hibernation_mode; | ||
72 | |||
73 | /* This context's GFP mask */ | 65 | /* This context's GFP mask */ |
74 | gfp_t gfp_mask; | 66 | gfp_t gfp_mask; |
75 | 67 | ||
76 | int may_writepage; | 68 | /* Allocation order */ |
77 | |||
78 | /* Can mapped pages be reclaimed? */ | ||
79 | int may_unmap; | ||
80 | |||
81 | /* Can pages be swapped as part of reclaim? */ | ||
82 | int may_swap; | ||
83 | |||
84 | int order; | 69 | int order; |
85 | 70 | ||
86 | /* Scan (total_size >> priority) pages at once */ | 71 | /* |
87 | int priority; | 72 | * Nodemask of nodes allowed by the caller. If NULL, all nodes |
88 | 73 | * are scanned. | |
89 | /* anon vs. file LRUs scanning "ratio" */ | 74 | */ |
90 | int swappiness; | 75 | nodemask_t *nodemask; |
91 | 76 | ||
92 | /* | 77 | /* |
93 | * The memory cgroup that hit its limit and as a result is the | 78 | * The memory cgroup that hit its limit and as a result is the |
@@ -95,11 +80,27 @@ struct scan_control { | |||
95 | */ | 80 | */ |
96 | struct mem_cgroup *target_mem_cgroup; | 81 | struct mem_cgroup *target_mem_cgroup; |
97 | 82 | ||
98 | /* | 83 | /* Scan (total_size >> priority) pages at once */ |
99 | * Nodemask of nodes allowed by the caller. If NULL, all nodes | 84 | int priority; |
100 | * are scanned. | 85 | |
101 | */ | 86 | unsigned int may_writepage:1; |
102 | nodemask_t *nodemask; | 87 | |
88 | /* Can mapped pages be reclaimed? */ | ||
89 | unsigned int may_unmap:1; | ||
90 | |||
91 | /* Can pages be swapped as part of reclaim? */ | ||
92 | unsigned int may_swap:1; | ||
93 | |||
94 | unsigned int hibernation_mode:1; | ||
95 | |||
96 | /* One of the zones is ready for compaction */ | ||
97 | unsigned int compaction_ready:1; | ||
98 | |||
99 | /* Incremented by the number of inactive pages that were scanned */ | ||
100 | unsigned long nr_scanned; | ||
101 | |||
102 | /* Number of pages freed so far during a call to shrink_zones() */ | ||
103 | unsigned long nr_reclaimed; | ||
103 | }; | 104 | }; |
104 | 105 | ||
105 | #define lru_to_page(_head) (list_entry((_head)->prev, struct page, lru)) | 106 | #define lru_to_page(_head) (list_entry((_head)->prev, struct page, lru)) |
@@ -136,7 +137,11 @@ struct scan_control { | |||
136 | * From 0 .. 100. Higher means more swappy. | 137 | * From 0 .. 100. Higher means more swappy. |
137 | */ | 138 | */ |
138 | int vm_swappiness = 60; | 139 | int vm_swappiness = 60; |
139 | unsigned long vm_total_pages; /* The total number of pages which the VM controls */ | 140 | /* |
141 | * The total number of pages which are beyond the high watermark within all | ||
142 | * zones. | ||
143 | */ | ||
144 | unsigned long vm_total_pages; | ||
140 | 145 | ||
141 | static LIST_HEAD(shrinker_list); | 146 | static LIST_HEAD(shrinker_list); |
142 | static DECLARE_RWSEM(shrinker_rwsem); | 147 | static DECLARE_RWSEM(shrinker_rwsem); |
@@ -169,7 +174,8 @@ static unsigned long zone_reclaimable_pages(struct zone *zone) | |||
169 | 174 | ||
170 | bool zone_reclaimable(struct zone *zone) | 175 | bool zone_reclaimable(struct zone *zone) |
171 | { | 176 | { |
172 | return zone->pages_scanned < zone_reclaimable_pages(zone) * 6; | 177 | return zone_page_state(zone, NR_PAGES_SCANNED) < |
178 | zone_reclaimable_pages(zone) * 6; | ||
173 | } | 179 | } |
174 | 180 | ||
175 | static unsigned long get_lru_size(struct lruvec *lruvec, enum lru_list lru) | 181 | static unsigned long get_lru_size(struct lruvec *lruvec, enum lru_list lru) |
@@ -1503,7 +1509,7 @@ shrink_inactive_list(unsigned long nr_to_scan, struct lruvec *lruvec, | |||
1503 | __mod_zone_page_state(zone, NR_ISOLATED_ANON + file, nr_taken); | 1509 | __mod_zone_page_state(zone, NR_ISOLATED_ANON + file, nr_taken); |
1504 | 1510 | ||
1505 | if (global_reclaim(sc)) { | 1511 | if (global_reclaim(sc)) { |
1506 | zone->pages_scanned += nr_scanned; | 1512 | __mod_zone_page_state(zone, NR_PAGES_SCANNED, nr_scanned); |
1507 | if (current_is_kswapd()) | 1513 | if (current_is_kswapd()) |
1508 | __count_zone_vm_events(PGSCAN_KSWAPD, zone, nr_scanned); | 1514 | __count_zone_vm_events(PGSCAN_KSWAPD, zone, nr_scanned); |
1509 | else | 1515 | else |
@@ -1693,7 +1699,7 @@ static void shrink_active_list(unsigned long nr_to_scan, | |||
1693 | nr_taken = isolate_lru_pages(nr_to_scan, lruvec, &l_hold, | 1699 | nr_taken = isolate_lru_pages(nr_to_scan, lruvec, &l_hold, |
1694 | &nr_scanned, sc, isolate_mode, lru); | 1700 | &nr_scanned, sc, isolate_mode, lru); |
1695 | if (global_reclaim(sc)) | 1701 | if (global_reclaim(sc)) |
1696 | zone->pages_scanned += nr_scanned; | 1702 | __mod_zone_page_state(zone, NR_PAGES_SCANNED, nr_scanned); |
1697 | 1703 | ||
1698 | reclaim_stat->recent_scanned[file] += nr_taken; | 1704 | reclaim_stat->recent_scanned[file] += nr_taken; |
1699 | 1705 | ||
@@ -1750,7 +1756,7 @@ static void shrink_active_list(unsigned long nr_to_scan, | |||
1750 | * Count referenced pages from currently used mappings as rotated, | 1756 | * Count referenced pages from currently used mappings as rotated, |
1751 | * even though only some of them are actually re-activated. This | 1757 | * even though only some of them are actually re-activated. This |
1752 | * helps balance scan pressure between file and anonymous pages in | 1758 | * helps balance scan pressure between file and anonymous pages in |
1753 | * get_scan_ratio. | 1759 | * get_scan_count. |
1754 | */ | 1760 | */ |
1755 | reclaim_stat->recent_rotated[file] += nr_rotated; | 1761 | reclaim_stat->recent_rotated[file] += nr_rotated; |
1756 | 1762 | ||
@@ -1865,8 +1871,8 @@ enum scan_balance { | |||
1865 | * nr[0] = anon inactive pages to scan; nr[1] = anon active pages to scan | 1871 | * nr[0] = anon inactive pages to scan; nr[1] = anon active pages to scan |
1866 | * nr[2] = file inactive pages to scan; nr[3] = file active pages to scan | 1872 | * nr[2] = file inactive pages to scan; nr[3] = file active pages to scan |
1867 | */ | 1873 | */ |
1868 | static void get_scan_count(struct lruvec *lruvec, struct scan_control *sc, | 1874 | static void get_scan_count(struct lruvec *lruvec, int swappiness, |
1869 | unsigned long *nr) | 1875 | struct scan_control *sc, unsigned long *nr) |
1870 | { | 1876 | { |
1871 | struct zone_reclaim_stat *reclaim_stat = &lruvec->reclaim_stat; | 1877 | struct zone_reclaim_stat *reclaim_stat = &lruvec->reclaim_stat; |
1872 | u64 fraction[2]; | 1878 | u64 fraction[2]; |
@@ -1909,7 +1915,7 @@ static void get_scan_count(struct lruvec *lruvec, struct scan_control *sc, | |||
1909 | * using the memory controller's swap limit feature would be | 1915 | * using the memory controller's swap limit feature would be |
1910 | * too expensive. | 1916 | * too expensive. |
1911 | */ | 1917 | */ |
1912 | if (!global_reclaim(sc) && !sc->swappiness) { | 1918 | if (!global_reclaim(sc) && !swappiness) { |
1913 | scan_balance = SCAN_FILE; | 1919 | scan_balance = SCAN_FILE; |
1914 | goto out; | 1920 | goto out; |
1915 | } | 1921 | } |
@@ -1919,16 +1925,11 @@ static void get_scan_count(struct lruvec *lruvec, struct scan_control *sc, | |||
1919 | * system is close to OOM, scan both anon and file equally | 1925 | * system is close to OOM, scan both anon and file equally |
1920 | * (unless the swappiness setting disagrees with swapping). | 1926 | * (unless the swappiness setting disagrees with swapping). |
1921 | */ | 1927 | */ |
1922 | if (!sc->priority && sc->swappiness) { | 1928 | if (!sc->priority && swappiness) { |
1923 | scan_balance = SCAN_EQUAL; | 1929 | scan_balance = SCAN_EQUAL; |
1924 | goto out; | 1930 | goto out; |
1925 | } | 1931 | } |
1926 | 1932 | ||
1927 | anon = get_lru_size(lruvec, LRU_ACTIVE_ANON) + | ||
1928 | get_lru_size(lruvec, LRU_INACTIVE_ANON); | ||
1929 | file = get_lru_size(lruvec, LRU_ACTIVE_FILE) + | ||
1930 | get_lru_size(lruvec, LRU_INACTIVE_FILE); | ||
1931 | |||
1932 | /* | 1933 | /* |
1933 | * Prevent the reclaimer from falling into the cache trap: as | 1934 | * Prevent the reclaimer from falling into the cache trap: as |
1934 | * cache pages start out inactive, every cache fault will tip | 1935 | * cache pages start out inactive, every cache fault will tip |
@@ -1939,9 +1940,14 @@ static void get_scan_count(struct lruvec *lruvec, struct scan_control *sc, | |||
1939 | * anon pages. Try to detect this based on file LRU size. | 1940 | * anon pages. Try to detect this based on file LRU size. |
1940 | */ | 1941 | */ |
1941 | if (global_reclaim(sc)) { | 1942 | if (global_reclaim(sc)) { |
1942 | unsigned long free = zone_page_state(zone, NR_FREE_PAGES); | 1943 | unsigned long zonefile; |
1944 | unsigned long zonefree; | ||
1943 | 1945 | ||
1944 | if (unlikely(file + free <= high_wmark_pages(zone))) { | 1946 | zonefree = zone_page_state(zone, NR_FREE_PAGES); |
1947 | zonefile = zone_page_state(zone, NR_ACTIVE_FILE) + | ||
1948 | zone_page_state(zone, NR_INACTIVE_FILE); | ||
1949 | |||
1950 | if (unlikely(zonefile + zonefree <= high_wmark_pages(zone))) { | ||
1945 | scan_balance = SCAN_ANON; | 1951 | scan_balance = SCAN_ANON; |
1946 | goto out; | 1952 | goto out; |
1947 | } | 1953 | } |
@@ -1962,7 +1968,7 @@ static void get_scan_count(struct lruvec *lruvec, struct scan_control *sc, | |||
1962 | * With swappiness at 100, anonymous and file have the same priority. | 1968 | * With swappiness at 100, anonymous and file have the same priority. |
1963 | * This scanning priority is essentially the inverse of IO cost. | 1969 | * This scanning priority is essentially the inverse of IO cost. |
1964 | */ | 1970 | */ |
1965 | anon_prio = sc->swappiness; | 1971 | anon_prio = swappiness; |
1966 | file_prio = 200 - anon_prio; | 1972 | file_prio = 200 - anon_prio; |
1967 | 1973 | ||
1968 | /* | 1974 | /* |
@@ -1976,6 +1982,12 @@ static void get_scan_count(struct lruvec *lruvec, struct scan_control *sc, | |||
1976 | * | 1982 | * |
1977 | * anon in [0], file in [1] | 1983 | * anon in [0], file in [1] |
1978 | */ | 1984 | */ |
1985 | |||
1986 | anon = get_lru_size(lruvec, LRU_ACTIVE_ANON) + | ||
1987 | get_lru_size(lruvec, LRU_INACTIVE_ANON); | ||
1988 | file = get_lru_size(lruvec, LRU_ACTIVE_FILE) + | ||
1989 | get_lru_size(lruvec, LRU_INACTIVE_FILE); | ||
1990 | |||
1979 | spin_lock_irq(&zone->lru_lock); | 1991 | spin_lock_irq(&zone->lru_lock); |
1980 | if (unlikely(reclaim_stat->recent_scanned[0] > anon / 4)) { | 1992 | if (unlikely(reclaim_stat->recent_scanned[0] > anon / 4)) { |
1981 | reclaim_stat->recent_scanned[0] /= 2; | 1993 | reclaim_stat->recent_scanned[0] /= 2; |
@@ -2052,7 +2064,8 @@ out: | |||
2052 | /* | 2064 | /* |
2053 | * This is a basic per-zone page freer. Used by both kswapd and direct reclaim. | 2065 | * This is a basic per-zone page freer. Used by both kswapd and direct reclaim. |
2054 | */ | 2066 | */ |
2055 | static void shrink_lruvec(struct lruvec *lruvec, struct scan_control *sc) | 2067 | static void shrink_lruvec(struct lruvec *lruvec, int swappiness, |
2068 | struct scan_control *sc) | ||
2056 | { | 2069 | { |
2057 | unsigned long nr[NR_LRU_LISTS]; | 2070 | unsigned long nr[NR_LRU_LISTS]; |
2058 | unsigned long targets[NR_LRU_LISTS]; | 2071 | unsigned long targets[NR_LRU_LISTS]; |
@@ -2063,7 +2076,7 @@ static void shrink_lruvec(struct lruvec *lruvec, struct scan_control *sc) | |||
2063 | struct blk_plug plug; | 2076 | struct blk_plug plug; |
2064 | bool scan_adjusted; | 2077 | bool scan_adjusted; |
2065 | 2078 | ||
2066 | get_scan_count(lruvec, sc, nr); | 2079 | get_scan_count(lruvec, swappiness, sc, nr); |
2067 | 2080 | ||
2068 | /* Record the original scan target for proportional adjustments later */ | 2081 | /* Record the original scan target for proportional adjustments later */ |
2069 | memcpy(targets, nr, sizeof(nr)); | 2082 | memcpy(targets, nr, sizeof(nr)); |
@@ -2241,9 +2254,10 @@ static inline bool should_continue_reclaim(struct zone *zone, | |||
2241 | } | 2254 | } |
2242 | } | 2255 | } |
2243 | 2256 | ||
2244 | static void shrink_zone(struct zone *zone, struct scan_control *sc) | 2257 | static bool shrink_zone(struct zone *zone, struct scan_control *sc) |
2245 | { | 2258 | { |
2246 | unsigned long nr_reclaimed, nr_scanned; | 2259 | unsigned long nr_reclaimed, nr_scanned; |
2260 | bool reclaimable = false; | ||
2247 | 2261 | ||
2248 | do { | 2262 | do { |
2249 | struct mem_cgroup *root = sc->target_mem_cgroup; | 2263 | struct mem_cgroup *root = sc->target_mem_cgroup; |
@@ -2259,11 +2273,12 @@ static void shrink_zone(struct zone *zone, struct scan_control *sc) | |||
2259 | memcg = mem_cgroup_iter(root, NULL, &reclaim); | 2273 | memcg = mem_cgroup_iter(root, NULL, &reclaim); |
2260 | do { | 2274 | do { |
2261 | struct lruvec *lruvec; | 2275 | struct lruvec *lruvec; |
2276 | int swappiness; | ||
2262 | 2277 | ||
2263 | lruvec = mem_cgroup_zone_lruvec(zone, memcg); | 2278 | lruvec = mem_cgroup_zone_lruvec(zone, memcg); |
2279 | swappiness = mem_cgroup_swappiness(memcg); | ||
2264 | 2280 | ||
2265 | sc->swappiness = mem_cgroup_swappiness(memcg); | 2281 | shrink_lruvec(lruvec, swappiness, sc); |
2266 | shrink_lruvec(lruvec, sc); | ||
2267 | 2282 | ||
2268 | /* | 2283 | /* |
2269 | * Direct reclaim and kswapd have to scan all memory | 2284 | * Direct reclaim and kswapd have to scan all memory |
@@ -2287,20 +2302,21 @@ static void shrink_zone(struct zone *zone, struct scan_control *sc) | |||
2287 | sc->nr_scanned - nr_scanned, | 2302 | sc->nr_scanned - nr_scanned, |
2288 | sc->nr_reclaimed - nr_reclaimed); | 2303 | sc->nr_reclaimed - nr_reclaimed); |
2289 | 2304 | ||
2305 | if (sc->nr_reclaimed - nr_reclaimed) | ||
2306 | reclaimable = true; | ||
2307 | |||
2290 | } while (should_continue_reclaim(zone, sc->nr_reclaimed - nr_reclaimed, | 2308 | } while (should_continue_reclaim(zone, sc->nr_reclaimed - nr_reclaimed, |
2291 | sc->nr_scanned - nr_scanned, sc)); | 2309 | sc->nr_scanned - nr_scanned, sc)); |
2310 | |||
2311 | return reclaimable; | ||
2292 | } | 2312 | } |
2293 | 2313 | ||
2294 | /* Returns true if compaction should go ahead for a high-order request */ | 2314 | /* Returns true if compaction should go ahead for a high-order request */ |
2295 | static inline bool compaction_ready(struct zone *zone, struct scan_control *sc) | 2315 | static inline bool compaction_ready(struct zone *zone, int order) |
2296 | { | 2316 | { |
2297 | unsigned long balance_gap, watermark; | 2317 | unsigned long balance_gap, watermark; |
2298 | bool watermark_ok; | 2318 | bool watermark_ok; |
2299 | 2319 | ||
2300 | /* Do not consider compaction for orders reclaim is meant to satisfy */ | ||
2301 | if (sc->order <= PAGE_ALLOC_COSTLY_ORDER) | ||
2302 | return false; | ||
2303 | |||
2304 | /* | 2320 | /* |
2305 | * Compaction takes time to run and there are potentially other | 2321 | * Compaction takes time to run and there are potentially other |
2306 | * callers using the pages just freed. Continue reclaiming until | 2322 | * callers using the pages just freed. Continue reclaiming until |
@@ -2309,18 +2325,18 @@ static inline bool compaction_ready(struct zone *zone, struct scan_control *sc) | |||
2309 | */ | 2325 | */ |
2310 | balance_gap = min(low_wmark_pages(zone), DIV_ROUND_UP( | 2326 | balance_gap = min(low_wmark_pages(zone), DIV_ROUND_UP( |
2311 | zone->managed_pages, KSWAPD_ZONE_BALANCE_GAP_RATIO)); | 2327 | zone->managed_pages, KSWAPD_ZONE_BALANCE_GAP_RATIO)); |
2312 | watermark = high_wmark_pages(zone) + balance_gap + (2UL << sc->order); | 2328 | watermark = high_wmark_pages(zone) + balance_gap + (2UL << order); |
2313 | watermark_ok = zone_watermark_ok_safe(zone, 0, watermark, 0, 0); | 2329 | watermark_ok = zone_watermark_ok_safe(zone, 0, watermark, 0, 0); |
2314 | 2330 | ||
2315 | /* | 2331 | /* |
2316 | * If compaction is deferred, reclaim up to a point where | 2332 | * If compaction is deferred, reclaim up to a point where |
2317 | * compaction will have a chance of success when re-enabled | 2333 | * compaction will have a chance of success when re-enabled |
2318 | */ | 2334 | */ |
2319 | if (compaction_deferred(zone, sc->order)) | 2335 | if (compaction_deferred(zone, order)) |
2320 | return watermark_ok; | 2336 | return watermark_ok; |
2321 | 2337 | ||
2322 | /* If compaction is not ready to start, keep reclaiming */ | 2338 | /* If compaction is not ready to start, keep reclaiming */ |
2323 | if (!compaction_suitable(zone, sc->order)) | 2339 | if (!compaction_suitable(zone, order)) |
2324 | return false; | 2340 | return false; |
2325 | 2341 | ||
2326 | return watermark_ok; | 2342 | return watermark_ok; |
@@ -2342,10 +2358,7 @@ static inline bool compaction_ready(struct zone *zone, struct scan_control *sc) | |||
2342 | * If a zone is deemed to be full of pinned pages then just give it a light | 2358 | * If a zone is deemed to be full of pinned pages then just give it a light |
2343 | * scan then give up on it. | 2359 | * scan then give up on it. |
2344 | * | 2360 | * |
2345 | * This function returns true if a zone is being reclaimed for a costly | 2361 | * Returns true if a zone was reclaimable. |
2346 | * high-order allocation and compaction is ready to begin. This indicates to | ||
2347 | * the caller that it should consider retrying the allocation instead of | ||
2348 | * further reclaim. | ||
2349 | */ | 2362 | */ |
2350 | static bool shrink_zones(struct zonelist *zonelist, struct scan_control *sc) | 2363 | static bool shrink_zones(struct zonelist *zonelist, struct scan_control *sc) |
2351 | { | 2364 | { |
@@ -2354,13 +2367,13 @@ static bool shrink_zones(struct zonelist *zonelist, struct scan_control *sc) | |||
2354 | unsigned long nr_soft_reclaimed; | 2367 | unsigned long nr_soft_reclaimed; |
2355 | unsigned long nr_soft_scanned; | 2368 | unsigned long nr_soft_scanned; |
2356 | unsigned long lru_pages = 0; | 2369 | unsigned long lru_pages = 0; |
2357 | bool aborted_reclaim = false; | ||
2358 | struct reclaim_state *reclaim_state = current->reclaim_state; | 2370 | struct reclaim_state *reclaim_state = current->reclaim_state; |
2359 | gfp_t orig_mask; | 2371 | gfp_t orig_mask; |
2360 | struct shrink_control shrink = { | 2372 | struct shrink_control shrink = { |
2361 | .gfp_mask = sc->gfp_mask, | 2373 | .gfp_mask = sc->gfp_mask, |
2362 | }; | 2374 | }; |
2363 | enum zone_type requested_highidx = gfp_zone(sc->gfp_mask); | 2375 | enum zone_type requested_highidx = gfp_zone(sc->gfp_mask); |
2376 | bool reclaimable = false; | ||
2364 | 2377 | ||
2365 | /* | 2378 | /* |
2366 | * If the number of buffer_heads in the machine exceeds the maximum | 2379 | * If the number of buffer_heads in the machine exceeds the maximum |
@@ -2391,22 +2404,24 @@ static bool shrink_zones(struct zonelist *zonelist, struct scan_control *sc) | |||
2391 | if (sc->priority != DEF_PRIORITY && | 2404 | if (sc->priority != DEF_PRIORITY && |
2392 | !zone_reclaimable(zone)) | 2405 | !zone_reclaimable(zone)) |
2393 | continue; /* Let kswapd poll it */ | 2406 | continue; /* Let kswapd poll it */ |
2394 | if (IS_ENABLED(CONFIG_COMPACTION)) { | 2407 | |
2395 | /* | 2408 | /* |
2396 | * If we already have plenty of memory free for | 2409 | * If we already have plenty of memory free for |
2397 | * compaction in this zone, don't free any more. | 2410 | * compaction in this zone, don't free any more. |
2398 | * Even though compaction is invoked for any | 2411 | * Even though compaction is invoked for any |
2399 | * non-zero order, only frequent costly order | 2412 | * non-zero order, only frequent costly order |
2400 | * reclamation is disruptive enough to become a | 2413 | * reclamation is disruptive enough to become a |
2401 | * noticeable problem, like transparent huge | 2414 | * noticeable problem, like transparent huge |
2402 | * page allocations. | 2415 | * page allocations. |
2403 | */ | 2416 | */ |
2404 | if ((zonelist_zone_idx(z) <= requested_highidx) | 2417 | if (IS_ENABLED(CONFIG_COMPACTION) && |
2405 | && compaction_ready(zone, sc)) { | 2418 | sc->order > PAGE_ALLOC_COSTLY_ORDER && |
2406 | aborted_reclaim = true; | 2419 | zonelist_zone_idx(z) <= requested_highidx && |
2407 | continue; | 2420 | compaction_ready(zone, sc->order)) { |
2408 | } | 2421 | sc->compaction_ready = true; |
2422 | continue; | ||
2409 | } | 2423 | } |
2424 | |||
2410 | /* | 2425 | /* |
2411 | * This steals pages from memory cgroups over softlimit | 2426 | * This steals pages from memory cgroups over softlimit |
2412 | * and returns the number of reclaimed pages and | 2427 | * and returns the number of reclaimed pages and |
@@ -2419,10 +2434,17 @@ static bool shrink_zones(struct zonelist *zonelist, struct scan_control *sc) | |||
2419 | &nr_soft_scanned); | 2434 | &nr_soft_scanned); |
2420 | sc->nr_reclaimed += nr_soft_reclaimed; | 2435 | sc->nr_reclaimed += nr_soft_reclaimed; |
2421 | sc->nr_scanned += nr_soft_scanned; | 2436 | sc->nr_scanned += nr_soft_scanned; |
2437 | if (nr_soft_reclaimed) | ||
2438 | reclaimable = true; | ||
2422 | /* need some check for avoid more shrink_zone() */ | 2439 | /* need some check for avoid more shrink_zone() */ |
2423 | } | 2440 | } |
2424 | 2441 | ||
2425 | shrink_zone(zone, sc); | 2442 | if (shrink_zone(zone, sc)) |
2443 | reclaimable = true; | ||
2444 | |||
2445 | if (global_reclaim(sc) && | ||
2446 | !reclaimable && zone_reclaimable(zone)) | ||
2447 | reclaimable = true; | ||
2426 | } | 2448 | } |
2427 | 2449 | ||
2428 | /* | 2450 | /* |
@@ -2445,27 +2467,7 @@ static bool shrink_zones(struct zonelist *zonelist, struct scan_control *sc) | |||
2445 | */ | 2467 | */ |
2446 | sc->gfp_mask = orig_mask; | 2468 | sc->gfp_mask = orig_mask; |
2447 | 2469 | ||
2448 | return aborted_reclaim; | 2470 | return reclaimable; |
2449 | } | ||
2450 | |||
2451 | /* All zones in zonelist are unreclaimable? */ | ||
2452 | static bool all_unreclaimable(struct zonelist *zonelist, | ||
2453 | struct scan_control *sc) | ||
2454 | { | ||
2455 | struct zoneref *z; | ||
2456 | struct zone *zone; | ||
2457 | |||
2458 | for_each_zone_zonelist_nodemask(zone, z, zonelist, | ||
2459 | gfp_zone(sc->gfp_mask), sc->nodemask) { | ||
2460 | if (!populated_zone(zone)) | ||
2461 | continue; | ||
2462 | if (!cpuset_zone_allowed_hardwall(zone, GFP_KERNEL)) | ||
2463 | continue; | ||
2464 | if (zone_reclaimable(zone)) | ||
2465 | return false; | ||
2466 | } | ||
2467 | |||
2468 | return true; | ||
2469 | } | 2471 | } |
2470 | 2472 | ||
2471 | /* | 2473 | /* |
@@ -2489,7 +2491,7 @@ static unsigned long do_try_to_free_pages(struct zonelist *zonelist, | |||
2489 | { | 2491 | { |
2490 | unsigned long total_scanned = 0; | 2492 | unsigned long total_scanned = 0; |
2491 | unsigned long writeback_threshold; | 2493 | unsigned long writeback_threshold; |
2492 | bool aborted_reclaim; | 2494 | bool zones_reclaimable; |
2493 | 2495 | ||
2494 | delayacct_freepages_start(); | 2496 | delayacct_freepages_start(); |
2495 | 2497 | ||
@@ -2500,11 +2502,14 @@ static unsigned long do_try_to_free_pages(struct zonelist *zonelist, | |||
2500 | vmpressure_prio(sc->gfp_mask, sc->target_mem_cgroup, | 2502 | vmpressure_prio(sc->gfp_mask, sc->target_mem_cgroup, |
2501 | sc->priority); | 2503 | sc->priority); |
2502 | sc->nr_scanned = 0; | 2504 | sc->nr_scanned = 0; |
2503 | aborted_reclaim = shrink_zones(zonelist, sc); | 2505 | zones_reclaimable = shrink_zones(zonelist, sc); |
2504 | 2506 | ||
2505 | total_scanned += sc->nr_scanned; | 2507 | total_scanned += sc->nr_scanned; |
2506 | if (sc->nr_reclaimed >= sc->nr_to_reclaim) | 2508 | if (sc->nr_reclaimed >= sc->nr_to_reclaim) |
2507 | goto out; | 2509 | break; |
2510 | |||
2511 | if (sc->compaction_ready) | ||
2512 | break; | ||
2508 | 2513 | ||
2509 | /* | 2514 | /* |
2510 | * If we're getting trouble reclaiming, start doing | 2515 | * If we're getting trouble reclaiming, start doing |
@@ -2526,28 +2531,19 @@ static unsigned long do_try_to_free_pages(struct zonelist *zonelist, | |||
2526 | WB_REASON_TRY_TO_FREE_PAGES); | 2531 | WB_REASON_TRY_TO_FREE_PAGES); |
2527 | sc->may_writepage = 1; | 2532 | sc->may_writepage = 1; |
2528 | } | 2533 | } |
2529 | } while (--sc->priority >= 0 && !aborted_reclaim); | 2534 | } while (--sc->priority >= 0); |
2530 | 2535 | ||
2531 | out: | ||
2532 | delayacct_freepages_end(); | 2536 | delayacct_freepages_end(); |
2533 | 2537 | ||
2534 | if (sc->nr_reclaimed) | 2538 | if (sc->nr_reclaimed) |
2535 | return sc->nr_reclaimed; | 2539 | return sc->nr_reclaimed; |
2536 | 2540 | ||
2537 | /* | ||
2538 | * As hibernation is going on, kswapd is freezed so that it can't mark | ||
2539 | * the zone into all_unreclaimable. Thus bypassing all_unreclaimable | ||
2540 | * check. | ||
2541 | */ | ||
2542 | if (oom_killer_disabled) | ||
2543 | return 0; | ||
2544 | |||
2545 | /* Aborted reclaim to try compaction? don't OOM, then */ | 2541 | /* Aborted reclaim to try compaction? don't OOM, then */ |
2546 | if (aborted_reclaim) | 2542 | if (sc->compaction_ready) |
2547 | return 1; | 2543 | return 1; |
2548 | 2544 | ||
2549 | /* top priority shrink_zones still had more to do? don't OOM, then */ | 2545 | /* Any of the zones still reclaimable? Don't OOM. */ |
2550 | if (global_reclaim(sc) && !all_unreclaimable(zonelist, sc)) | 2546 | if (zones_reclaimable) |
2551 | return 1; | 2547 | return 1; |
2552 | 2548 | ||
2553 | return 0; | 2549 | return 0; |
@@ -2684,15 +2680,14 @@ unsigned long try_to_free_pages(struct zonelist *zonelist, int order, | |||
2684 | { | 2680 | { |
2685 | unsigned long nr_reclaimed; | 2681 | unsigned long nr_reclaimed; |
2686 | struct scan_control sc = { | 2682 | struct scan_control sc = { |
2683 | .nr_to_reclaim = SWAP_CLUSTER_MAX, | ||
2687 | .gfp_mask = (gfp_mask = memalloc_noio_flags(gfp_mask)), | 2684 | .gfp_mask = (gfp_mask = memalloc_noio_flags(gfp_mask)), |
2685 | .order = order, | ||
2686 | .nodemask = nodemask, | ||
2687 | .priority = DEF_PRIORITY, | ||
2688 | .may_writepage = !laptop_mode, | 2688 | .may_writepage = !laptop_mode, |
2689 | .nr_to_reclaim = SWAP_CLUSTER_MAX, | ||
2690 | .may_unmap = 1, | 2689 | .may_unmap = 1, |
2691 | .may_swap = 1, | 2690 | .may_swap = 1, |
2692 | .order = order, | ||
2693 | .priority = DEF_PRIORITY, | ||
2694 | .target_mem_cgroup = NULL, | ||
2695 | .nodemask = nodemask, | ||
2696 | }; | 2691 | }; |
2697 | 2692 | ||
2698 | /* | 2693 | /* |
@@ -2722,17 +2717,14 @@ unsigned long mem_cgroup_shrink_node_zone(struct mem_cgroup *memcg, | |||
2722 | unsigned long *nr_scanned) | 2717 | unsigned long *nr_scanned) |
2723 | { | 2718 | { |
2724 | struct scan_control sc = { | 2719 | struct scan_control sc = { |
2725 | .nr_scanned = 0, | ||
2726 | .nr_to_reclaim = SWAP_CLUSTER_MAX, | 2720 | .nr_to_reclaim = SWAP_CLUSTER_MAX, |
2721 | .target_mem_cgroup = memcg, | ||
2727 | .may_writepage = !laptop_mode, | 2722 | .may_writepage = !laptop_mode, |
2728 | .may_unmap = 1, | 2723 | .may_unmap = 1, |
2729 | .may_swap = !noswap, | 2724 | .may_swap = !noswap, |
2730 | .order = 0, | ||
2731 | .priority = 0, | ||
2732 | .swappiness = mem_cgroup_swappiness(memcg), | ||
2733 | .target_mem_cgroup = memcg, | ||
2734 | }; | 2725 | }; |
2735 | struct lruvec *lruvec = mem_cgroup_zone_lruvec(zone, memcg); | 2726 | struct lruvec *lruvec = mem_cgroup_zone_lruvec(zone, memcg); |
2727 | int swappiness = mem_cgroup_swappiness(memcg); | ||
2736 | 2728 | ||
2737 | sc.gfp_mask = (gfp_mask & GFP_RECLAIM_MASK) | | 2729 | sc.gfp_mask = (gfp_mask & GFP_RECLAIM_MASK) | |
2738 | (GFP_HIGHUSER_MOVABLE & ~GFP_RECLAIM_MASK); | 2730 | (GFP_HIGHUSER_MOVABLE & ~GFP_RECLAIM_MASK); |
@@ -2748,7 +2740,7 @@ unsigned long mem_cgroup_shrink_node_zone(struct mem_cgroup *memcg, | |||
2748 | * will pick up pages from other mem cgroup's as well. We hack | 2740 | * will pick up pages from other mem cgroup's as well. We hack |
2749 | * the priority and make it zero. | 2741 | * the priority and make it zero. |
2750 | */ | 2742 | */ |
2751 | shrink_lruvec(lruvec, &sc); | 2743 | shrink_lruvec(lruvec, swappiness, &sc); |
2752 | 2744 | ||
2753 | trace_mm_vmscan_memcg_softlimit_reclaim_end(sc.nr_reclaimed); | 2745 | trace_mm_vmscan_memcg_softlimit_reclaim_end(sc.nr_reclaimed); |
2754 | 2746 | ||
@@ -2764,16 +2756,14 @@ unsigned long try_to_free_mem_cgroup_pages(struct mem_cgroup *memcg, | |||
2764 | unsigned long nr_reclaimed; | 2756 | unsigned long nr_reclaimed; |
2765 | int nid; | 2757 | int nid; |
2766 | struct scan_control sc = { | 2758 | struct scan_control sc = { |
2767 | .may_writepage = !laptop_mode, | ||
2768 | .may_unmap = 1, | ||
2769 | .may_swap = !noswap, | ||
2770 | .nr_to_reclaim = SWAP_CLUSTER_MAX, | 2759 | .nr_to_reclaim = SWAP_CLUSTER_MAX, |
2771 | .order = 0, | ||
2772 | .priority = DEF_PRIORITY, | ||
2773 | .target_mem_cgroup = memcg, | ||
2774 | .nodemask = NULL, /* we don't care the placement */ | ||
2775 | .gfp_mask = (gfp_mask & GFP_RECLAIM_MASK) | | 2760 | .gfp_mask = (gfp_mask & GFP_RECLAIM_MASK) | |
2776 | (GFP_HIGHUSER_MOVABLE & ~GFP_RECLAIM_MASK), | 2761 | (GFP_HIGHUSER_MOVABLE & ~GFP_RECLAIM_MASK), |
2762 | .target_mem_cgroup = memcg, | ||
2763 | .priority = DEF_PRIORITY, | ||
2764 | .may_writepage = !laptop_mode, | ||
2765 | .may_unmap = 1, | ||
2766 | .may_swap = !noswap, | ||
2777 | }; | 2767 | }; |
2778 | 2768 | ||
2779 | /* | 2769 | /* |
@@ -3031,12 +3021,11 @@ static unsigned long balance_pgdat(pg_data_t *pgdat, int order, | |||
3031 | unsigned long nr_soft_scanned; | 3021 | unsigned long nr_soft_scanned; |
3032 | struct scan_control sc = { | 3022 | struct scan_control sc = { |
3033 | .gfp_mask = GFP_KERNEL, | 3023 | .gfp_mask = GFP_KERNEL, |
3024 | .order = order, | ||
3034 | .priority = DEF_PRIORITY, | 3025 | .priority = DEF_PRIORITY, |
3026 | .may_writepage = !laptop_mode, | ||
3035 | .may_unmap = 1, | 3027 | .may_unmap = 1, |
3036 | .may_swap = 1, | 3028 | .may_swap = 1, |
3037 | .may_writepage = !laptop_mode, | ||
3038 | .order = order, | ||
3039 | .target_mem_cgroup = NULL, | ||
3040 | }; | 3029 | }; |
3041 | count_vm_event(PAGEOUTRUN); | 3030 | count_vm_event(PAGEOUTRUN); |
3042 | 3031 | ||
@@ -3417,14 +3406,13 @@ unsigned long shrink_all_memory(unsigned long nr_to_reclaim) | |||
3417 | { | 3406 | { |
3418 | struct reclaim_state reclaim_state; | 3407 | struct reclaim_state reclaim_state; |
3419 | struct scan_control sc = { | 3408 | struct scan_control sc = { |
3409 | .nr_to_reclaim = nr_to_reclaim, | ||
3420 | .gfp_mask = GFP_HIGHUSER_MOVABLE, | 3410 | .gfp_mask = GFP_HIGHUSER_MOVABLE, |
3421 | .may_swap = 1, | 3411 | .priority = DEF_PRIORITY, |
3422 | .may_unmap = 1, | ||
3423 | .may_writepage = 1, | 3412 | .may_writepage = 1, |
3424 | .nr_to_reclaim = nr_to_reclaim, | 3413 | .may_unmap = 1, |
3414 | .may_swap = 1, | ||
3425 | .hibernation_mode = 1, | 3415 | .hibernation_mode = 1, |
3426 | .order = 0, | ||
3427 | .priority = DEF_PRIORITY, | ||
3428 | }; | 3416 | }; |
3429 | struct zonelist *zonelist = node_zonelist(numa_node_id(), sc.gfp_mask); | 3417 | struct zonelist *zonelist = node_zonelist(numa_node_id(), sc.gfp_mask); |
3430 | struct task_struct *p = current; | 3418 | struct task_struct *p = current; |
@@ -3604,13 +3592,13 @@ static int __zone_reclaim(struct zone *zone, gfp_t gfp_mask, unsigned int order) | |||
3604 | struct task_struct *p = current; | 3592 | struct task_struct *p = current; |
3605 | struct reclaim_state reclaim_state; | 3593 | struct reclaim_state reclaim_state; |
3606 | struct scan_control sc = { | 3594 | struct scan_control sc = { |
3607 | .may_writepage = !!(zone_reclaim_mode & RECLAIM_WRITE), | ||
3608 | .may_unmap = !!(zone_reclaim_mode & RECLAIM_SWAP), | ||
3609 | .may_swap = 1, | ||
3610 | .nr_to_reclaim = max(nr_pages, SWAP_CLUSTER_MAX), | 3595 | .nr_to_reclaim = max(nr_pages, SWAP_CLUSTER_MAX), |
3611 | .gfp_mask = (gfp_mask = memalloc_noio_flags(gfp_mask)), | 3596 | .gfp_mask = (gfp_mask = memalloc_noio_flags(gfp_mask)), |
3612 | .order = order, | 3597 | .order = order, |
3613 | .priority = ZONE_RECLAIM_PRIORITY, | 3598 | .priority = ZONE_RECLAIM_PRIORITY, |
3599 | .may_writepage = !!(zone_reclaim_mode & RECLAIM_WRITE), | ||
3600 | .may_unmap = !!(zone_reclaim_mode & RECLAIM_SWAP), | ||
3601 | .may_swap = 1, | ||
3614 | }; | 3602 | }; |
3615 | struct shrink_control shrink = { | 3603 | struct shrink_control shrink = { |
3616 | .gfp_mask = sc.gfp_mask, | 3604 | .gfp_mask = sc.gfp_mask, |
diff --git a/mm/vmstat.c b/mm/vmstat.c index b37bd49bfd55..e9ab104b956f 100644 --- a/mm/vmstat.c +++ b/mm/vmstat.c | |||
@@ -200,7 +200,7 @@ void set_pgdat_percpu_threshold(pg_data_t *pgdat, | |||
200 | continue; | 200 | continue; |
201 | 201 | ||
202 | threshold = (*calculate_pressure)(zone); | 202 | threshold = (*calculate_pressure)(zone); |
203 | for_each_possible_cpu(cpu) | 203 | for_each_online_cpu(cpu) |
204 | per_cpu_ptr(zone->pageset, cpu)->stat_threshold | 204 | per_cpu_ptr(zone->pageset, cpu)->stat_threshold |
205 | = threshold; | 205 | = threshold; |
206 | } | 206 | } |
@@ -763,6 +763,7 @@ const char * const vmstat_text[] = { | |||
763 | "nr_shmem", | 763 | "nr_shmem", |
764 | "nr_dirtied", | 764 | "nr_dirtied", |
765 | "nr_written", | 765 | "nr_written", |
766 | "nr_pages_scanned", | ||
766 | 767 | ||
767 | #ifdef CONFIG_NUMA | 768 | #ifdef CONFIG_NUMA |
768 | "numa_hit", | 769 | "numa_hit", |
@@ -1067,7 +1068,7 @@ static void zoneinfo_show_print(struct seq_file *m, pg_data_t *pgdat, | |||
1067 | min_wmark_pages(zone), | 1068 | min_wmark_pages(zone), |
1068 | low_wmark_pages(zone), | 1069 | low_wmark_pages(zone), |
1069 | high_wmark_pages(zone), | 1070 | high_wmark_pages(zone), |
1070 | zone->pages_scanned, | 1071 | zone_page_state(zone, NR_PAGES_SCANNED), |
1071 | zone->spanned_pages, | 1072 | zone->spanned_pages, |
1072 | zone->present_pages, | 1073 | zone->present_pages, |
1073 | zone->managed_pages); | 1074 | zone->managed_pages); |
@@ -1077,10 +1078,10 @@ static void zoneinfo_show_print(struct seq_file *m, pg_data_t *pgdat, | |||
1077 | zone_page_state(zone, i)); | 1078 | zone_page_state(zone, i)); |
1078 | 1079 | ||
1079 | seq_printf(m, | 1080 | seq_printf(m, |
1080 | "\n protection: (%lu", | 1081 | "\n protection: (%ld", |
1081 | zone->lowmem_reserve[0]); | 1082 | zone->lowmem_reserve[0]); |
1082 | for (i = 1; i < ARRAY_SIZE(zone->lowmem_reserve); i++) | 1083 | for (i = 1; i < ARRAY_SIZE(zone->lowmem_reserve); i++) |
1083 | seq_printf(m, ", %lu", zone->lowmem_reserve[i]); | 1084 | seq_printf(m, ", %ld", zone->lowmem_reserve[i]); |
1084 | seq_printf(m, | 1085 | seq_printf(m, |
1085 | ")" | 1086 | ")" |
1086 | "\n pagesets"); | 1087 | "\n pagesets"); |
@@ -51,6 +51,7 @@ | |||
51 | #include <linux/slab.h> | 51 | #include <linux/slab.h> |
52 | #include <linux/spinlock.h> | 52 | #include <linux/spinlock.h> |
53 | #include <linux/zbud.h> | 53 | #include <linux/zbud.h> |
54 | #include <linux/zpool.h> | ||
54 | 55 | ||
55 | /***************** | 56 | /***************** |
56 | * Structures | 57 | * Structures |
@@ -113,6 +114,90 @@ struct zbud_header { | |||
113 | }; | 114 | }; |
114 | 115 | ||
115 | /***************** | 116 | /***************** |
117 | * zpool | ||
118 | ****************/ | ||
119 | |||
120 | #ifdef CONFIG_ZPOOL | ||
121 | |||
122 | static int zbud_zpool_evict(struct zbud_pool *pool, unsigned long handle) | ||
123 | { | ||
124 | return zpool_evict(pool, handle); | ||
125 | } | ||
126 | |||
127 | static struct zbud_ops zbud_zpool_ops = { | ||
128 | .evict = zbud_zpool_evict | ||
129 | }; | ||
130 | |||
131 | static void *zbud_zpool_create(gfp_t gfp, struct zpool_ops *zpool_ops) | ||
132 | { | ||
133 | return zbud_create_pool(gfp, &zbud_zpool_ops); | ||
134 | } | ||
135 | |||
136 | static void zbud_zpool_destroy(void *pool) | ||
137 | { | ||
138 | zbud_destroy_pool(pool); | ||
139 | } | ||
140 | |||
141 | static int zbud_zpool_malloc(void *pool, size_t size, gfp_t gfp, | ||
142 | unsigned long *handle) | ||
143 | { | ||
144 | return zbud_alloc(pool, size, gfp, handle); | ||
145 | } | ||
146 | static void zbud_zpool_free(void *pool, unsigned long handle) | ||
147 | { | ||
148 | zbud_free(pool, handle); | ||
149 | } | ||
150 | |||
151 | static int zbud_zpool_shrink(void *pool, unsigned int pages, | ||
152 | unsigned int *reclaimed) | ||
153 | { | ||
154 | unsigned int total = 0; | ||
155 | int ret = -EINVAL; | ||
156 | |||
157 | while (total < pages) { | ||
158 | ret = zbud_reclaim_page(pool, 8); | ||
159 | if (ret < 0) | ||
160 | break; | ||
161 | total++; | ||
162 | } | ||
163 | |||
164 | if (reclaimed) | ||
165 | *reclaimed = total; | ||
166 | |||
167 | return ret; | ||
168 | } | ||
169 | |||
170 | static void *zbud_zpool_map(void *pool, unsigned long handle, | ||
171 | enum zpool_mapmode mm) | ||
172 | { | ||
173 | return zbud_map(pool, handle); | ||
174 | } | ||
175 | static void zbud_zpool_unmap(void *pool, unsigned long handle) | ||
176 | { | ||
177 | zbud_unmap(pool, handle); | ||
178 | } | ||
179 | |||
180 | static u64 zbud_zpool_total_size(void *pool) | ||
181 | { | ||
182 | return zbud_get_pool_size(pool) * PAGE_SIZE; | ||
183 | } | ||
184 | |||
185 | static struct zpool_driver zbud_zpool_driver = { | ||
186 | .type = "zbud", | ||
187 | .owner = THIS_MODULE, | ||
188 | .create = zbud_zpool_create, | ||
189 | .destroy = zbud_zpool_destroy, | ||
190 | .malloc = zbud_zpool_malloc, | ||
191 | .free = zbud_zpool_free, | ||
192 | .shrink = zbud_zpool_shrink, | ||
193 | .map = zbud_zpool_map, | ||
194 | .unmap = zbud_zpool_unmap, | ||
195 | .total_size = zbud_zpool_total_size, | ||
196 | }; | ||
197 | |||
198 | #endif /* CONFIG_ZPOOL */ | ||
199 | |||
200 | /***************** | ||
116 | * Helpers | 201 | * Helpers |
117 | *****************/ | 202 | *****************/ |
118 | /* Just to make the code easier to read */ | 203 | /* Just to make the code easier to read */ |
@@ -122,7 +207,7 @@ enum buddy { | |||
122 | }; | 207 | }; |
123 | 208 | ||
124 | /* Converts an allocation size in bytes to size in zbud chunks */ | 209 | /* Converts an allocation size in bytes to size in zbud chunks */ |
125 | static int size_to_chunks(int size) | 210 | static int size_to_chunks(size_t size) |
126 | { | 211 | { |
127 | return (size + CHUNK_SIZE - 1) >> CHUNK_SHIFT; | 212 | return (size + CHUNK_SIZE - 1) >> CHUNK_SHIFT; |
128 | } | 213 | } |
@@ -247,7 +332,7 @@ void zbud_destroy_pool(struct zbud_pool *pool) | |||
247 | * gfp arguments are invalid or -ENOMEM if the pool was unable to allocate | 332 | * gfp arguments are invalid or -ENOMEM if the pool was unable to allocate |
248 | * a new page. | 333 | * a new page. |
249 | */ | 334 | */ |
250 | int zbud_alloc(struct zbud_pool *pool, unsigned int size, gfp_t gfp, | 335 | int zbud_alloc(struct zbud_pool *pool, size_t size, gfp_t gfp, |
251 | unsigned long *handle) | 336 | unsigned long *handle) |
252 | { | 337 | { |
253 | int chunks, i, freechunks; | 338 | int chunks, i, freechunks; |
@@ -511,11 +596,20 @@ static int __init init_zbud(void) | |||
511 | /* Make sure the zbud header will fit in one chunk */ | 596 | /* Make sure the zbud header will fit in one chunk */ |
512 | BUILD_BUG_ON(sizeof(struct zbud_header) > ZHDR_SIZE_ALIGNED); | 597 | BUILD_BUG_ON(sizeof(struct zbud_header) > ZHDR_SIZE_ALIGNED); |
513 | pr_info("loaded\n"); | 598 | pr_info("loaded\n"); |
599 | |||
600 | #ifdef CONFIG_ZPOOL | ||
601 | zpool_register_driver(&zbud_zpool_driver); | ||
602 | #endif | ||
603 | |||
514 | return 0; | 604 | return 0; |
515 | } | 605 | } |
516 | 606 | ||
517 | static void __exit exit_zbud(void) | 607 | static void __exit exit_zbud(void) |
518 | { | 608 | { |
609 | #ifdef CONFIG_ZPOOL | ||
610 | zpool_unregister_driver(&zbud_zpool_driver); | ||
611 | #endif | ||
612 | |||
519 | pr_info("unloaded\n"); | 613 | pr_info("unloaded\n"); |
520 | } | 614 | } |
521 | 615 | ||
diff --git a/mm/zpool.c b/mm/zpool.c new file mode 100644 index 000000000000..e40612a1df00 --- /dev/null +++ b/mm/zpool.c | |||
@@ -0,0 +1,364 @@ | |||
1 | /* | ||
2 | * zpool memory storage api | ||
3 | * | ||
4 | * Copyright (C) 2014 Dan Streetman | ||
5 | * | ||
6 | * This is a common frontend for memory storage pool implementations. | ||
7 | * Typically, this is used to store compressed memory. | ||
8 | */ | ||
9 | |||
10 | #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt | ||
11 | |||
12 | #include <linux/list.h> | ||
13 | #include <linux/types.h> | ||
14 | #include <linux/mm.h> | ||
15 | #include <linux/slab.h> | ||
16 | #include <linux/spinlock.h> | ||
17 | #include <linux/module.h> | ||
18 | #include <linux/zpool.h> | ||
19 | |||
20 | struct zpool { | ||
21 | char *type; | ||
22 | |||
23 | struct zpool_driver *driver; | ||
24 | void *pool; | ||
25 | struct zpool_ops *ops; | ||
26 | |||
27 | struct list_head list; | ||
28 | }; | ||
29 | |||
30 | static LIST_HEAD(drivers_head); | ||
31 | static DEFINE_SPINLOCK(drivers_lock); | ||
32 | |||
33 | static LIST_HEAD(pools_head); | ||
34 | static DEFINE_SPINLOCK(pools_lock); | ||
35 | |||
36 | /** | ||
37 | * zpool_register_driver() - register a zpool implementation. | ||
38 | * @driver: driver to register | ||
39 | */ | ||
40 | void zpool_register_driver(struct zpool_driver *driver) | ||
41 | { | ||
42 | spin_lock(&drivers_lock); | ||
43 | atomic_set(&driver->refcount, 0); | ||
44 | list_add(&driver->list, &drivers_head); | ||
45 | spin_unlock(&drivers_lock); | ||
46 | } | ||
47 | EXPORT_SYMBOL(zpool_register_driver); | ||
48 | |||
49 | /** | ||
50 | * zpool_unregister_driver() - unregister a zpool implementation. | ||
51 | * @driver: driver to unregister. | ||
52 | * | ||
53 | * Module usage counting is used to prevent using a driver | ||
54 | * while/after unloading, so if this is called from module | ||
55 | * exit function, this should never fail; if called from | ||
56 | * other than the module exit function, and this returns | ||
57 | * failure, the driver is in use and must remain available. | ||
58 | */ | ||
59 | int zpool_unregister_driver(struct zpool_driver *driver) | ||
60 | { | ||
61 | int ret = 0, refcount; | ||
62 | |||
63 | spin_lock(&drivers_lock); | ||
64 | refcount = atomic_read(&driver->refcount); | ||
65 | WARN_ON(refcount < 0); | ||
66 | if (refcount > 0) | ||
67 | ret = -EBUSY; | ||
68 | else | ||
69 | list_del(&driver->list); | ||
70 | spin_unlock(&drivers_lock); | ||
71 | |||
72 | return ret; | ||
73 | } | ||
74 | EXPORT_SYMBOL(zpool_unregister_driver); | ||
75 | |||
76 | /** | ||
77 | * zpool_evict() - evict callback from a zpool implementation. | ||
78 | * @pool: pool to evict from. | ||
79 | * @handle: handle to evict. | ||
80 | * | ||
81 | * This can be used by zpool implementations to call the | ||
82 | * user's evict zpool_ops struct evict callback. | ||
83 | */ | ||
84 | int zpool_evict(void *pool, unsigned long handle) | ||
85 | { | ||
86 | struct zpool *zpool; | ||
87 | |||
88 | spin_lock(&pools_lock); | ||
89 | list_for_each_entry(zpool, &pools_head, list) { | ||
90 | if (zpool->pool == pool) { | ||
91 | spin_unlock(&pools_lock); | ||
92 | if (!zpool->ops || !zpool->ops->evict) | ||
93 | return -EINVAL; | ||
94 | return zpool->ops->evict(zpool, handle); | ||
95 | } | ||
96 | } | ||
97 | spin_unlock(&pools_lock); | ||
98 | |||
99 | return -ENOENT; | ||
100 | } | ||
101 | EXPORT_SYMBOL(zpool_evict); | ||
102 | |||
103 | static struct zpool_driver *zpool_get_driver(char *type) | ||
104 | { | ||
105 | struct zpool_driver *driver; | ||
106 | |||
107 | spin_lock(&drivers_lock); | ||
108 | list_for_each_entry(driver, &drivers_head, list) { | ||
109 | if (!strcmp(driver->type, type)) { | ||
110 | bool got = try_module_get(driver->owner); | ||
111 | |||
112 | if (got) | ||
113 | atomic_inc(&driver->refcount); | ||
114 | spin_unlock(&drivers_lock); | ||
115 | return got ? driver : NULL; | ||
116 | } | ||
117 | } | ||
118 | |||
119 | spin_unlock(&drivers_lock); | ||
120 | return NULL; | ||
121 | } | ||
122 | |||
123 | static void zpool_put_driver(struct zpool_driver *driver) | ||
124 | { | ||
125 | atomic_dec(&driver->refcount); | ||
126 | module_put(driver->owner); | ||
127 | } | ||
128 | |||
129 | /** | ||
130 | * zpool_create_pool() - Create a new zpool | ||
131 | * @type The type of the zpool to create (e.g. zbud, zsmalloc) | ||
132 | * @gfp The GFP flags to use when allocating the pool. | ||
133 | * @ops The optional ops callback. | ||
134 | * | ||
135 | * This creates a new zpool of the specified type. The gfp flags will be | ||
136 | * used when allocating memory, if the implementation supports it. If the | ||
137 | * ops param is NULL, then the created zpool will not be shrinkable. | ||
138 | * | ||
139 | * Implementations must guarantee this to be thread-safe. | ||
140 | * | ||
141 | * Returns: New zpool on success, NULL on failure. | ||
142 | */ | ||
143 | struct zpool *zpool_create_pool(char *type, gfp_t gfp, struct zpool_ops *ops) | ||
144 | { | ||
145 | struct zpool_driver *driver; | ||
146 | struct zpool *zpool; | ||
147 | |||
148 | pr_info("creating pool type %s\n", type); | ||
149 | |||
150 | driver = zpool_get_driver(type); | ||
151 | |||
152 | if (!driver) { | ||
153 | request_module(type); | ||
154 | driver = zpool_get_driver(type); | ||
155 | } | ||
156 | |||
157 | if (!driver) { | ||
158 | pr_err("no driver for type %s\n", type); | ||
159 | return NULL; | ||
160 | } | ||
161 | |||
162 | zpool = kmalloc(sizeof(*zpool), gfp); | ||
163 | if (!zpool) { | ||
164 | pr_err("couldn't create zpool - out of memory\n"); | ||
165 | zpool_put_driver(driver); | ||
166 | return NULL; | ||
167 | } | ||
168 | |||
169 | zpool->type = driver->type; | ||
170 | zpool->driver = driver; | ||
171 | zpool->pool = driver->create(gfp, ops); | ||
172 | zpool->ops = ops; | ||
173 | |||
174 | if (!zpool->pool) { | ||
175 | pr_err("couldn't create %s pool\n", type); | ||
176 | zpool_put_driver(driver); | ||
177 | kfree(zpool); | ||
178 | return NULL; | ||
179 | } | ||
180 | |||
181 | pr_info("created %s pool\n", type); | ||
182 | |||
183 | spin_lock(&pools_lock); | ||
184 | list_add(&zpool->list, &pools_head); | ||
185 | spin_unlock(&pools_lock); | ||
186 | |||
187 | return zpool; | ||
188 | } | ||
189 | |||
190 | /** | ||
191 | * zpool_destroy_pool() - Destroy a zpool | ||
192 | * @pool The zpool to destroy. | ||
193 | * | ||
194 | * Implementations must guarantee this to be thread-safe, | ||
195 | * however only when destroying different pools. The same | ||
196 | * pool should only be destroyed once, and should not be used | ||
197 | * after it is destroyed. | ||
198 | * | ||
199 | * This destroys an existing zpool. The zpool should not be in use. | ||
200 | */ | ||
201 | void zpool_destroy_pool(struct zpool *zpool) | ||
202 | { | ||
203 | pr_info("destroying pool type %s\n", zpool->type); | ||
204 | |||
205 | spin_lock(&pools_lock); | ||
206 | list_del(&zpool->list); | ||
207 | spin_unlock(&pools_lock); | ||
208 | zpool->driver->destroy(zpool->pool); | ||
209 | zpool_put_driver(zpool->driver); | ||
210 | kfree(zpool); | ||
211 | } | ||
212 | |||
213 | /** | ||
214 | * zpool_get_type() - Get the type of the zpool | ||
215 | * @pool The zpool to check | ||
216 | * | ||
217 | * This returns the type of the pool. | ||
218 | * | ||
219 | * Implementations must guarantee this to be thread-safe. | ||
220 | * | ||
221 | * Returns: The type of zpool. | ||
222 | */ | ||
223 | char *zpool_get_type(struct zpool *zpool) | ||
224 | { | ||
225 | return zpool->type; | ||
226 | } | ||
227 | |||
228 | /** | ||
229 | * zpool_malloc() - Allocate memory | ||
230 | * @pool The zpool to allocate from. | ||
231 | * @size The amount of memory to allocate. | ||
232 | * @gfp The GFP flags to use when allocating memory. | ||
233 | * @handle Pointer to the handle to set | ||
234 | * | ||
235 | * This allocates the requested amount of memory from the pool. | ||
236 | * The gfp flags will be used when allocating memory, if the | ||
237 | * implementation supports it. The provided @handle will be | ||
238 | * set to the allocated object handle. | ||
239 | * | ||
240 | * Implementations must guarantee this to be thread-safe. | ||
241 | * | ||
242 | * Returns: 0 on success, negative value on error. | ||
243 | */ | ||
244 | int zpool_malloc(struct zpool *zpool, size_t size, gfp_t gfp, | ||
245 | unsigned long *handle) | ||
246 | { | ||
247 | return zpool->driver->malloc(zpool->pool, size, gfp, handle); | ||
248 | } | ||
249 | |||
250 | /** | ||
251 | * zpool_free() - Free previously allocated memory | ||
252 | * @pool The zpool that allocated the memory. | ||
253 | * @handle The handle to the memory to free. | ||
254 | * | ||
255 | * This frees previously allocated memory. This does not guarantee | ||
256 | * that the pool will actually free memory, only that the memory | ||
257 | * in the pool will become available for use by the pool. | ||
258 | * | ||
259 | * Implementations must guarantee this to be thread-safe, | ||
260 | * however only when freeing different handles. The same | ||
261 | * handle should only be freed once, and should not be used | ||
262 | * after freeing. | ||
263 | */ | ||
264 | void zpool_free(struct zpool *zpool, unsigned long handle) | ||
265 | { | ||
266 | zpool->driver->free(zpool->pool, handle); | ||
267 | } | ||
268 | |||
269 | /** | ||
270 | * zpool_shrink() - Shrink the pool size | ||
271 | * @pool The zpool to shrink. | ||
272 | * @pages The number of pages to shrink the pool. | ||
273 | * @reclaimed The number of pages successfully evicted. | ||
274 | * | ||
275 | * This attempts to shrink the actual memory size of the pool | ||
276 | * by evicting currently used handle(s). If the pool was | ||
277 | * created with no zpool_ops, or the evict call fails for any | ||
278 | * of the handles, this will fail. If non-NULL, the @reclaimed | ||
279 | * parameter will be set to the number of pages reclaimed, | ||
280 | * which may be more than the number of pages requested. | ||
281 | * | ||
282 | * Implementations must guarantee this to be thread-safe. | ||
283 | * | ||
284 | * Returns: 0 on success, negative value on error/failure. | ||
285 | */ | ||
286 | int zpool_shrink(struct zpool *zpool, unsigned int pages, | ||
287 | unsigned int *reclaimed) | ||
288 | { | ||
289 | return zpool->driver->shrink(zpool->pool, pages, reclaimed); | ||
290 | } | ||
291 | |||
292 | /** | ||
293 | * zpool_map_handle() - Map a previously allocated handle into memory | ||
294 | * @pool The zpool that the handle was allocated from | ||
295 | * @handle The handle to map | ||
296 | * @mm How the memory should be mapped | ||
297 | * | ||
298 | * This maps a previously allocated handle into memory. The @mm | ||
299 | * param indicates to the implementation how the memory will be | ||
300 | * used, i.e. read-only, write-only, read-write. If the | ||
301 | * implementation does not support it, the memory will be treated | ||
302 | * as read-write. | ||
303 | * | ||
304 | * This may hold locks, disable interrupts, and/or preemption, | ||
305 | * and the zpool_unmap_handle() must be called to undo those | ||
306 | * actions. The code that uses the mapped handle should complete | ||
307 | * its operatons on the mapped handle memory quickly and unmap | ||
308 | * as soon as possible. As the implementation may use per-cpu | ||
309 | * data, multiple handles should not be mapped concurrently on | ||
310 | * any cpu. | ||
311 | * | ||
312 | * Returns: A pointer to the handle's mapped memory area. | ||
313 | */ | ||
314 | void *zpool_map_handle(struct zpool *zpool, unsigned long handle, | ||
315 | enum zpool_mapmode mapmode) | ||
316 | { | ||
317 | return zpool->driver->map(zpool->pool, handle, mapmode); | ||
318 | } | ||
319 | |||
320 | /** | ||
321 | * zpool_unmap_handle() - Unmap a previously mapped handle | ||
322 | * @pool The zpool that the handle was allocated from | ||
323 | * @handle The handle to unmap | ||
324 | * | ||
325 | * This unmaps a previously mapped handle. Any locks or other | ||
326 | * actions that the implementation took in zpool_map_handle() | ||
327 | * will be undone here. The memory area returned from | ||
328 | * zpool_map_handle() should no longer be used after this. | ||
329 | */ | ||
330 | void zpool_unmap_handle(struct zpool *zpool, unsigned long handle) | ||
331 | { | ||
332 | zpool->driver->unmap(zpool->pool, handle); | ||
333 | } | ||
334 | |||
335 | /** | ||
336 | * zpool_get_total_size() - The total size of the pool | ||
337 | * @pool The zpool to check | ||
338 | * | ||
339 | * This returns the total size in bytes of the pool. | ||
340 | * | ||
341 | * Returns: Total size of the zpool in bytes. | ||
342 | */ | ||
343 | u64 zpool_get_total_size(struct zpool *zpool) | ||
344 | { | ||
345 | return zpool->driver->total_size(zpool->pool); | ||
346 | } | ||
347 | |||
348 | static int __init init_zpool(void) | ||
349 | { | ||
350 | pr_info("loaded\n"); | ||
351 | return 0; | ||
352 | } | ||
353 | |||
354 | static void __exit exit_zpool(void) | ||
355 | { | ||
356 | pr_info("unloaded\n"); | ||
357 | } | ||
358 | |||
359 | module_init(init_zpool); | ||
360 | module_exit(exit_zpool); | ||
361 | |||
362 | MODULE_LICENSE("GPL"); | ||
363 | MODULE_AUTHOR("Dan Streetman <ddstreet@ieee.org>"); | ||
364 | MODULE_DESCRIPTION("Common API for compressed memory storage"); | ||
diff --git a/mm/zsmalloc.c b/mm/zsmalloc.c index fe78189624cf..4e2fc83cb394 100644 --- a/mm/zsmalloc.c +++ b/mm/zsmalloc.c | |||
@@ -92,6 +92,7 @@ | |||
92 | #include <linux/spinlock.h> | 92 | #include <linux/spinlock.h> |
93 | #include <linux/types.h> | 93 | #include <linux/types.h> |
94 | #include <linux/zsmalloc.h> | 94 | #include <linux/zsmalloc.h> |
95 | #include <linux/zpool.h> | ||
95 | 96 | ||
96 | /* | 97 | /* |
97 | * This must be power of 2 and greater than of equal to sizeof(link_free). | 98 | * This must be power of 2 and greater than of equal to sizeof(link_free). |
@@ -240,6 +241,81 @@ struct mapping_area { | |||
240 | enum zs_mapmode vm_mm; /* mapping mode */ | 241 | enum zs_mapmode vm_mm; /* mapping mode */ |
241 | }; | 242 | }; |
242 | 243 | ||
244 | /* zpool driver */ | ||
245 | |||
246 | #ifdef CONFIG_ZPOOL | ||
247 | |||
248 | static void *zs_zpool_create(gfp_t gfp, struct zpool_ops *zpool_ops) | ||
249 | { | ||
250 | return zs_create_pool(gfp); | ||
251 | } | ||
252 | |||
253 | static void zs_zpool_destroy(void *pool) | ||
254 | { | ||
255 | zs_destroy_pool(pool); | ||
256 | } | ||
257 | |||
258 | static int zs_zpool_malloc(void *pool, size_t size, gfp_t gfp, | ||
259 | unsigned long *handle) | ||
260 | { | ||
261 | *handle = zs_malloc(pool, size); | ||
262 | return *handle ? 0 : -1; | ||
263 | } | ||
264 | static void zs_zpool_free(void *pool, unsigned long handle) | ||
265 | { | ||
266 | zs_free(pool, handle); | ||
267 | } | ||
268 | |||
269 | static int zs_zpool_shrink(void *pool, unsigned int pages, | ||
270 | unsigned int *reclaimed) | ||
271 | { | ||
272 | return -EINVAL; | ||
273 | } | ||
274 | |||
275 | static void *zs_zpool_map(void *pool, unsigned long handle, | ||
276 | enum zpool_mapmode mm) | ||
277 | { | ||
278 | enum zs_mapmode zs_mm; | ||
279 | |||
280 | switch (mm) { | ||
281 | case ZPOOL_MM_RO: | ||
282 | zs_mm = ZS_MM_RO; | ||
283 | break; | ||
284 | case ZPOOL_MM_WO: | ||
285 | zs_mm = ZS_MM_WO; | ||
286 | break; | ||
287 | case ZPOOL_MM_RW: /* fallthru */ | ||
288 | default: | ||
289 | zs_mm = ZS_MM_RW; | ||
290 | break; | ||
291 | } | ||
292 | |||
293 | return zs_map_object(pool, handle, zs_mm); | ||
294 | } | ||
295 | static void zs_zpool_unmap(void *pool, unsigned long handle) | ||
296 | { | ||
297 | zs_unmap_object(pool, handle); | ||
298 | } | ||
299 | |||
300 | static u64 zs_zpool_total_size(void *pool) | ||
301 | { | ||
302 | return zs_get_total_size_bytes(pool); | ||
303 | } | ||
304 | |||
305 | static struct zpool_driver zs_zpool_driver = { | ||
306 | .type = "zsmalloc", | ||
307 | .owner = THIS_MODULE, | ||
308 | .create = zs_zpool_create, | ||
309 | .destroy = zs_zpool_destroy, | ||
310 | .malloc = zs_zpool_malloc, | ||
311 | .free = zs_zpool_free, | ||
312 | .shrink = zs_zpool_shrink, | ||
313 | .map = zs_zpool_map, | ||
314 | .unmap = zs_zpool_unmap, | ||
315 | .total_size = zs_zpool_total_size, | ||
316 | }; | ||
317 | |||
318 | #endif /* CONFIG_ZPOOL */ | ||
243 | 319 | ||
244 | /* per-cpu VM mapping areas for zspage accesses that cross page boundaries */ | 320 | /* per-cpu VM mapping areas for zspage accesses that cross page boundaries */ |
245 | static DEFINE_PER_CPU(struct mapping_area, zs_map_area); | 321 | static DEFINE_PER_CPU(struct mapping_area, zs_map_area); |
@@ -690,7 +766,7 @@ static inline void __zs_cpu_down(struct mapping_area *area) | |||
690 | static inline void *__zs_map_object(struct mapping_area *area, | 766 | static inline void *__zs_map_object(struct mapping_area *area, |
691 | struct page *pages[2], int off, int size) | 767 | struct page *pages[2], int off, int size) |
692 | { | 768 | { |
693 | BUG_ON(map_vm_area(area->vm, PAGE_KERNEL, &pages)); | 769 | BUG_ON(map_vm_area(area->vm, PAGE_KERNEL, pages)); |
694 | area->vm_addr = area->vm->addr; | 770 | area->vm_addr = area->vm->addr; |
695 | return area->vm_addr + off; | 771 | return area->vm_addr + off; |
696 | } | 772 | } |
@@ -814,6 +890,10 @@ static void zs_exit(void) | |||
814 | { | 890 | { |
815 | int cpu; | 891 | int cpu; |
816 | 892 | ||
893 | #ifdef CONFIG_ZPOOL | ||
894 | zpool_unregister_driver(&zs_zpool_driver); | ||
895 | #endif | ||
896 | |||
817 | cpu_notifier_register_begin(); | 897 | cpu_notifier_register_begin(); |
818 | 898 | ||
819 | for_each_online_cpu(cpu) | 899 | for_each_online_cpu(cpu) |
@@ -840,6 +920,10 @@ static int zs_init(void) | |||
840 | 920 | ||
841 | cpu_notifier_register_done(); | 921 | cpu_notifier_register_done(); |
842 | 922 | ||
923 | #ifdef CONFIG_ZPOOL | ||
924 | zpool_register_driver(&zs_zpool_driver); | ||
925 | #endif | ||
926 | |||
843 | return 0; | 927 | return 0; |
844 | fail: | 928 | fail: |
845 | zs_exit(); | 929 | zs_exit(); |
diff --git a/mm/zswap.c b/mm/zswap.c index 008388fe7b0f..032c21eeab2b 100644 --- a/mm/zswap.c +++ b/mm/zswap.c | |||
@@ -34,7 +34,7 @@ | |||
34 | #include <linux/swap.h> | 34 | #include <linux/swap.h> |
35 | #include <linux/crypto.h> | 35 | #include <linux/crypto.h> |
36 | #include <linux/mempool.h> | 36 | #include <linux/mempool.h> |
37 | #include <linux/zbud.h> | 37 | #include <linux/zpool.h> |
38 | 38 | ||
39 | #include <linux/mm_types.h> | 39 | #include <linux/mm_types.h> |
40 | #include <linux/page-flags.h> | 40 | #include <linux/page-flags.h> |
@@ -45,8 +45,8 @@ | |||
45 | /********************************* | 45 | /********************************* |
46 | * statistics | 46 | * statistics |
47 | **********************************/ | 47 | **********************************/ |
48 | /* Number of memory pages used by the compressed pool */ | 48 | /* Total bytes used by the compressed storage */ |
49 | static u64 zswap_pool_pages; | 49 | static u64 zswap_pool_total_size; |
50 | /* The number of compressed pages currently stored in zswap */ | 50 | /* The number of compressed pages currently stored in zswap */ |
51 | static atomic_t zswap_stored_pages = ATOMIC_INIT(0); | 51 | static atomic_t zswap_stored_pages = ATOMIC_INIT(0); |
52 | 52 | ||
@@ -89,8 +89,13 @@ static unsigned int zswap_max_pool_percent = 20; | |||
89 | module_param_named(max_pool_percent, | 89 | module_param_named(max_pool_percent, |
90 | zswap_max_pool_percent, uint, 0644); | 90 | zswap_max_pool_percent, uint, 0644); |
91 | 91 | ||
92 | /* zbud_pool is shared by all of zswap backend */ | 92 | /* Compressed storage to use */ |
93 | static struct zbud_pool *zswap_pool; | 93 | #define ZSWAP_ZPOOL_DEFAULT "zbud" |
94 | static char *zswap_zpool_type = ZSWAP_ZPOOL_DEFAULT; | ||
95 | module_param_named(zpool, zswap_zpool_type, charp, 0444); | ||
96 | |||
97 | /* zpool is shared by all of zswap backend */ | ||
98 | static struct zpool *zswap_pool; | ||
94 | 99 | ||
95 | /********************************* | 100 | /********************************* |
96 | * compression functions | 101 | * compression functions |
@@ -168,7 +173,7 @@ static void zswap_comp_exit(void) | |||
168 | * be held while changing the refcount. Since the lock must | 173 | * be held while changing the refcount. Since the lock must |
169 | * be held, there is no reason to also make refcount atomic. | 174 | * be held, there is no reason to also make refcount atomic. |
170 | * offset - the swap offset for the entry. Index into the red-black tree. | 175 | * offset - the swap offset for the entry. Index into the red-black tree. |
171 | * handle - zbud allocation handle that stores the compressed page data | 176 | * handle - zpool allocation handle that stores the compressed page data |
172 | * length - the length in bytes of the compressed page data. Needed during | 177 | * length - the length in bytes of the compressed page data. Needed during |
173 | * decompression | 178 | * decompression |
174 | */ | 179 | */ |
@@ -284,15 +289,15 @@ static void zswap_rb_erase(struct rb_root *root, struct zswap_entry *entry) | |||
284 | } | 289 | } |
285 | 290 | ||
286 | /* | 291 | /* |
287 | * Carries out the common pattern of freeing and entry's zbud allocation, | 292 | * Carries out the common pattern of freeing and entry's zpool allocation, |
288 | * freeing the entry itself, and decrementing the number of stored pages. | 293 | * freeing the entry itself, and decrementing the number of stored pages. |
289 | */ | 294 | */ |
290 | static void zswap_free_entry(struct zswap_entry *entry) | 295 | static void zswap_free_entry(struct zswap_entry *entry) |
291 | { | 296 | { |
292 | zbud_free(zswap_pool, entry->handle); | 297 | zpool_free(zswap_pool, entry->handle); |
293 | zswap_entry_cache_free(entry); | 298 | zswap_entry_cache_free(entry); |
294 | atomic_dec(&zswap_stored_pages); | 299 | atomic_dec(&zswap_stored_pages); |
295 | zswap_pool_pages = zbud_get_pool_size(zswap_pool); | 300 | zswap_pool_total_size = zpool_get_total_size(zswap_pool); |
296 | } | 301 | } |
297 | 302 | ||
298 | /* caller must hold the tree lock */ | 303 | /* caller must hold the tree lock */ |
@@ -409,7 +414,7 @@ cleanup: | |||
409 | static bool zswap_is_full(void) | 414 | static bool zswap_is_full(void) |
410 | { | 415 | { |
411 | return totalram_pages * zswap_max_pool_percent / 100 < | 416 | return totalram_pages * zswap_max_pool_percent / 100 < |
412 | zswap_pool_pages; | 417 | DIV_ROUND_UP(zswap_pool_total_size, PAGE_SIZE); |
413 | } | 418 | } |
414 | 419 | ||
415 | /********************************* | 420 | /********************************* |
@@ -525,7 +530,7 @@ static int zswap_get_swap_cache_page(swp_entry_t entry, | |||
525 | * the swap cache, the compressed version stored by zswap can be | 530 | * the swap cache, the compressed version stored by zswap can be |
526 | * freed. | 531 | * freed. |
527 | */ | 532 | */ |
528 | static int zswap_writeback_entry(struct zbud_pool *pool, unsigned long handle) | 533 | static int zswap_writeback_entry(struct zpool *pool, unsigned long handle) |
529 | { | 534 | { |
530 | struct zswap_header *zhdr; | 535 | struct zswap_header *zhdr; |
531 | swp_entry_t swpentry; | 536 | swp_entry_t swpentry; |
@@ -541,9 +546,9 @@ static int zswap_writeback_entry(struct zbud_pool *pool, unsigned long handle) | |||
541 | }; | 546 | }; |
542 | 547 | ||
543 | /* extract swpentry from data */ | 548 | /* extract swpentry from data */ |
544 | zhdr = zbud_map(pool, handle); | 549 | zhdr = zpool_map_handle(pool, handle, ZPOOL_MM_RO); |
545 | swpentry = zhdr->swpentry; /* here */ | 550 | swpentry = zhdr->swpentry; /* here */ |
546 | zbud_unmap(pool, handle); | 551 | zpool_unmap_handle(pool, handle); |
547 | tree = zswap_trees[swp_type(swpentry)]; | 552 | tree = zswap_trees[swp_type(swpentry)]; |
548 | offset = swp_offset(swpentry); | 553 | offset = swp_offset(swpentry); |
549 | 554 | ||
@@ -573,13 +578,13 @@ static int zswap_writeback_entry(struct zbud_pool *pool, unsigned long handle) | |||
573 | case ZSWAP_SWAPCACHE_NEW: /* page is locked */ | 578 | case ZSWAP_SWAPCACHE_NEW: /* page is locked */ |
574 | /* decompress */ | 579 | /* decompress */ |
575 | dlen = PAGE_SIZE; | 580 | dlen = PAGE_SIZE; |
576 | src = (u8 *)zbud_map(zswap_pool, entry->handle) + | 581 | src = (u8 *)zpool_map_handle(zswap_pool, entry->handle, |
577 | sizeof(struct zswap_header); | 582 | ZPOOL_MM_RO) + sizeof(struct zswap_header); |
578 | dst = kmap_atomic(page); | 583 | dst = kmap_atomic(page); |
579 | ret = zswap_comp_op(ZSWAP_COMPOP_DECOMPRESS, src, | 584 | ret = zswap_comp_op(ZSWAP_COMPOP_DECOMPRESS, src, |
580 | entry->length, dst, &dlen); | 585 | entry->length, dst, &dlen); |
581 | kunmap_atomic(dst); | 586 | kunmap_atomic(dst); |
582 | zbud_unmap(zswap_pool, entry->handle); | 587 | zpool_unmap_handle(zswap_pool, entry->handle); |
583 | BUG_ON(ret); | 588 | BUG_ON(ret); |
584 | BUG_ON(dlen != PAGE_SIZE); | 589 | BUG_ON(dlen != PAGE_SIZE); |
585 | 590 | ||
@@ -652,7 +657,7 @@ static int zswap_frontswap_store(unsigned type, pgoff_t offset, | |||
652 | /* reclaim space if needed */ | 657 | /* reclaim space if needed */ |
653 | if (zswap_is_full()) { | 658 | if (zswap_is_full()) { |
654 | zswap_pool_limit_hit++; | 659 | zswap_pool_limit_hit++; |
655 | if (zbud_reclaim_page(zswap_pool, 8)) { | 660 | if (zpool_shrink(zswap_pool, 1, NULL)) { |
656 | zswap_reject_reclaim_fail++; | 661 | zswap_reject_reclaim_fail++; |
657 | ret = -ENOMEM; | 662 | ret = -ENOMEM; |
658 | goto reject; | 663 | goto reject; |
@@ -679,7 +684,7 @@ static int zswap_frontswap_store(unsigned type, pgoff_t offset, | |||
679 | 684 | ||
680 | /* store */ | 685 | /* store */ |
681 | len = dlen + sizeof(struct zswap_header); | 686 | len = dlen + sizeof(struct zswap_header); |
682 | ret = zbud_alloc(zswap_pool, len, __GFP_NORETRY | __GFP_NOWARN, | 687 | ret = zpool_malloc(zswap_pool, len, __GFP_NORETRY | __GFP_NOWARN, |
683 | &handle); | 688 | &handle); |
684 | if (ret == -ENOSPC) { | 689 | if (ret == -ENOSPC) { |
685 | zswap_reject_compress_poor++; | 690 | zswap_reject_compress_poor++; |
@@ -689,11 +694,11 @@ static int zswap_frontswap_store(unsigned type, pgoff_t offset, | |||
689 | zswap_reject_alloc_fail++; | 694 | zswap_reject_alloc_fail++; |
690 | goto freepage; | 695 | goto freepage; |
691 | } | 696 | } |
692 | zhdr = zbud_map(zswap_pool, handle); | 697 | zhdr = zpool_map_handle(zswap_pool, handle, ZPOOL_MM_RW); |
693 | zhdr->swpentry = swp_entry(type, offset); | 698 | zhdr->swpentry = swp_entry(type, offset); |
694 | buf = (u8 *)(zhdr + 1); | 699 | buf = (u8 *)(zhdr + 1); |
695 | memcpy(buf, dst, dlen); | 700 | memcpy(buf, dst, dlen); |
696 | zbud_unmap(zswap_pool, handle); | 701 | zpool_unmap_handle(zswap_pool, handle); |
697 | put_cpu_var(zswap_dstmem); | 702 | put_cpu_var(zswap_dstmem); |
698 | 703 | ||
699 | /* populate entry */ | 704 | /* populate entry */ |
@@ -716,7 +721,7 @@ static int zswap_frontswap_store(unsigned type, pgoff_t offset, | |||
716 | 721 | ||
717 | /* update stats */ | 722 | /* update stats */ |
718 | atomic_inc(&zswap_stored_pages); | 723 | atomic_inc(&zswap_stored_pages); |
719 | zswap_pool_pages = zbud_get_pool_size(zswap_pool); | 724 | zswap_pool_total_size = zpool_get_total_size(zswap_pool); |
720 | 725 | ||
721 | return 0; | 726 | return 0; |
722 | 727 | ||
@@ -752,13 +757,13 @@ static int zswap_frontswap_load(unsigned type, pgoff_t offset, | |||
752 | 757 | ||
753 | /* decompress */ | 758 | /* decompress */ |
754 | dlen = PAGE_SIZE; | 759 | dlen = PAGE_SIZE; |
755 | src = (u8 *)zbud_map(zswap_pool, entry->handle) + | 760 | src = (u8 *)zpool_map_handle(zswap_pool, entry->handle, |
756 | sizeof(struct zswap_header); | 761 | ZPOOL_MM_RO) + sizeof(struct zswap_header); |
757 | dst = kmap_atomic(page); | 762 | dst = kmap_atomic(page); |
758 | ret = zswap_comp_op(ZSWAP_COMPOP_DECOMPRESS, src, entry->length, | 763 | ret = zswap_comp_op(ZSWAP_COMPOP_DECOMPRESS, src, entry->length, |
759 | dst, &dlen); | 764 | dst, &dlen); |
760 | kunmap_atomic(dst); | 765 | kunmap_atomic(dst); |
761 | zbud_unmap(zswap_pool, entry->handle); | 766 | zpool_unmap_handle(zswap_pool, entry->handle); |
762 | BUG_ON(ret); | 767 | BUG_ON(ret); |
763 | 768 | ||
764 | spin_lock(&tree->lock); | 769 | spin_lock(&tree->lock); |
@@ -811,7 +816,7 @@ static void zswap_frontswap_invalidate_area(unsigned type) | |||
811 | zswap_trees[type] = NULL; | 816 | zswap_trees[type] = NULL; |
812 | } | 817 | } |
813 | 818 | ||
814 | static struct zbud_ops zswap_zbud_ops = { | 819 | static struct zpool_ops zswap_zpool_ops = { |
815 | .evict = zswap_writeback_entry | 820 | .evict = zswap_writeback_entry |
816 | }; | 821 | }; |
817 | 822 | ||
@@ -869,8 +874,8 @@ static int __init zswap_debugfs_init(void) | |||
869 | zswap_debugfs_root, &zswap_written_back_pages); | 874 | zswap_debugfs_root, &zswap_written_back_pages); |
870 | debugfs_create_u64("duplicate_entry", S_IRUGO, | 875 | debugfs_create_u64("duplicate_entry", S_IRUGO, |
871 | zswap_debugfs_root, &zswap_duplicate_entry); | 876 | zswap_debugfs_root, &zswap_duplicate_entry); |
872 | debugfs_create_u64("pool_pages", S_IRUGO, | 877 | debugfs_create_u64("pool_total_size", S_IRUGO, |
873 | zswap_debugfs_root, &zswap_pool_pages); | 878 | zswap_debugfs_root, &zswap_pool_total_size); |
874 | debugfs_create_atomic_t("stored_pages", S_IRUGO, | 879 | debugfs_create_atomic_t("stored_pages", S_IRUGO, |
875 | zswap_debugfs_root, &zswap_stored_pages); | 880 | zswap_debugfs_root, &zswap_stored_pages); |
876 | 881 | ||
@@ -895,16 +900,26 @@ static void __exit zswap_debugfs_exit(void) { } | |||
895 | **********************************/ | 900 | **********************************/ |
896 | static int __init init_zswap(void) | 901 | static int __init init_zswap(void) |
897 | { | 902 | { |
903 | gfp_t gfp = __GFP_NORETRY | __GFP_NOWARN; | ||
904 | |||
898 | if (!zswap_enabled) | 905 | if (!zswap_enabled) |
899 | return 0; | 906 | return 0; |
900 | 907 | ||
901 | pr_info("loading zswap\n"); | 908 | pr_info("loading zswap\n"); |
902 | 909 | ||
903 | zswap_pool = zbud_create_pool(GFP_KERNEL, &zswap_zbud_ops); | 910 | zswap_pool = zpool_create_pool(zswap_zpool_type, gfp, &zswap_zpool_ops); |
911 | if (!zswap_pool && strcmp(zswap_zpool_type, ZSWAP_ZPOOL_DEFAULT)) { | ||
912 | pr_info("%s zpool not available\n", zswap_zpool_type); | ||
913 | zswap_zpool_type = ZSWAP_ZPOOL_DEFAULT; | ||
914 | zswap_pool = zpool_create_pool(zswap_zpool_type, gfp, | ||
915 | &zswap_zpool_ops); | ||
916 | } | ||
904 | if (!zswap_pool) { | 917 | if (!zswap_pool) { |
905 | pr_err("zbud pool creation failed\n"); | 918 | pr_err("%s zpool not available\n", zswap_zpool_type); |
919 | pr_err("zpool creation failed\n"); | ||
906 | goto error; | 920 | goto error; |
907 | } | 921 | } |
922 | pr_info("using %s pool\n", zswap_zpool_type); | ||
908 | 923 | ||
909 | if (zswap_entry_cache_create()) { | 924 | if (zswap_entry_cache_create()) { |
910 | pr_err("entry cache creation failed\n"); | 925 | pr_err("entry cache creation failed\n"); |
@@ -928,7 +943,7 @@ pcpufail: | |||
928 | compfail: | 943 | compfail: |
929 | zswap_entry_cache_destory(); | 944 | zswap_entry_cache_destory(); |
930 | cachefail: | 945 | cachefail: |
931 | zbud_destroy_pool(zswap_pool); | 946 | zpool_destroy_pool(zswap_pool); |
932 | error: | 947 | error: |
933 | return -ENOMEM; | 948 | return -ENOMEM; |
934 | } | 949 | } |
diff --git a/net/batman-adv/fragmentation.c b/net/batman-adv/fragmentation.c index 022d18ab27a6..52c43f904220 100644 --- a/net/batman-adv/fragmentation.c +++ b/net/batman-adv/fragmentation.c | |||
@@ -188,7 +188,7 @@ static bool batadv_frag_insert_packet(struct batadv_orig_node *orig_node, | |||
188 | 188 | ||
189 | /* Reached the end of the list, so insert after 'frag_entry_last'. */ | 189 | /* Reached the end of the list, so insert after 'frag_entry_last'. */ |
190 | if (likely(frag_entry_last)) { | 190 | if (likely(frag_entry_last)) { |
191 | hlist_add_after(&frag_entry_last->list, &frag_entry_new->list); | 191 | hlist_add_behind(&frag_entry_last->list, &frag_entry_new->list); |
192 | chain->size += skb->len - hdr_size; | 192 | chain->size += skb->len - hdr_size; |
193 | chain->timestamp = jiffies; | 193 | chain->timestamp = jiffies; |
194 | ret = true; | 194 | ret = true; |
diff --git a/net/bridge/br_multicast.c b/net/bridge/br_multicast.c index b4845f4b2bb4..7751c92c8c57 100644 --- a/net/bridge/br_multicast.c +++ b/net/bridge/br_multicast.c | |||
@@ -1174,7 +1174,7 @@ static void br_multicast_add_router(struct net_bridge *br, | |||
1174 | } | 1174 | } |
1175 | 1175 | ||
1176 | if (slot) | 1176 | if (slot) |
1177 | hlist_add_after_rcu(slot, &port->rlist); | 1177 | hlist_add_behind_rcu(&port->rlist, slot); |
1178 | else | 1178 | else |
1179 | hlist_add_head_rcu(&port->rlist, &br->router_list); | 1179 | hlist_add_head_rcu(&port->rlist, &br->router_list); |
1180 | } | 1180 | } |
diff --git a/net/ipv4/fib_trie.c b/net/ipv4/fib_trie.c index 5afeb5aa4c7c..e9cb2588e416 100644 --- a/net/ipv4/fib_trie.c +++ b/net/ipv4/fib_trie.c | |||
@@ -940,7 +940,7 @@ static void insert_leaf_info(struct hlist_head *head, struct leaf_info *new) | |||
940 | last = li; | 940 | last = li; |
941 | } | 941 | } |
942 | if (last) | 942 | if (last) |
943 | hlist_add_after_rcu(&last->hlist, &new->hlist); | 943 | hlist_add_behind_rcu(&new->hlist, &last->hlist); |
944 | else | 944 | else |
945 | hlist_add_before_rcu(&new->hlist, &li->hlist); | 945 | hlist_add_before_rcu(&new->hlist, &li->hlist); |
946 | } | 946 | } |
diff --git a/net/ipv6/addrlabel.c b/net/ipv6/addrlabel.c index 731e1e1722d9..fd0dc47f471d 100644 --- a/net/ipv6/addrlabel.c +++ b/net/ipv6/addrlabel.c | |||
@@ -277,7 +277,7 @@ static int __ip6addrlbl_add(struct ip6addrlbl_entry *newp, int replace) | |||
277 | last = p; | 277 | last = p; |
278 | } | 278 | } |
279 | if (last) | 279 | if (last) |
280 | hlist_add_after_rcu(&last->list, &newp->list); | 280 | hlist_add_behind_rcu(&newp->list, &last->list); |
281 | else | 281 | else |
282 | hlist_add_head_rcu(&newp->list, &ip6addrlbl_table.head); | 282 | hlist_add_head_rcu(&newp->list, &ip6addrlbl_table.head); |
283 | out: | 283 | out: |
diff --git a/net/xfrm/xfrm_policy.c b/net/xfrm/xfrm_policy.c index 0525d78ba328..beeed602aeb3 100644 --- a/net/xfrm/xfrm_policy.c +++ b/net/xfrm/xfrm_policy.c | |||
@@ -389,7 +389,7 @@ redo: | |||
389 | if (h != h0) | 389 | if (h != h0) |
390 | continue; | 390 | continue; |
391 | hlist_del(&pol->bydst); | 391 | hlist_del(&pol->bydst); |
392 | hlist_add_after(entry0, &pol->bydst); | 392 | hlist_add_behind(&pol->bydst, entry0); |
393 | } | 393 | } |
394 | entry0 = &pol->bydst; | 394 | entry0 = &pol->bydst; |
395 | } | 395 | } |
@@ -654,7 +654,7 @@ int xfrm_policy_insert(int dir, struct xfrm_policy *policy, int excl) | |||
654 | break; | 654 | break; |
655 | } | 655 | } |
656 | if (newpos) | 656 | if (newpos) |
657 | hlist_add_after(newpos, &policy->bydst); | 657 | hlist_add_behind(&policy->bydst, newpos); |
658 | else | 658 | else |
659 | hlist_add_head(&policy->bydst, chain); | 659 | hlist_add_head(&policy->bydst, chain); |
660 | xfrm_pol_hold(policy); | 660 | xfrm_pol_hold(policy); |
diff --git a/scripts/checkpatch.pl b/scripts/checkpatch.pl index 182be0f12407..31a731e06f50 100755 --- a/scripts/checkpatch.pl +++ b/scripts/checkpatch.pl | |||
@@ -309,9 +309,12 @@ our $Operators = qr{ | |||
309 | our $c90_Keywords = qr{do|for|while|if|else|return|goto|continue|switch|default|case|break}x; | 309 | our $c90_Keywords = qr{do|for|while|if|else|return|goto|continue|switch|default|case|break}x; |
310 | 310 | ||
311 | our $NonptrType; | 311 | our $NonptrType; |
312 | our $NonptrTypeMisordered; | ||
312 | our $NonptrTypeWithAttr; | 313 | our $NonptrTypeWithAttr; |
313 | our $Type; | 314 | our $Type; |
315 | our $TypeMisordered; | ||
314 | our $Declare; | 316 | our $Declare; |
317 | our $DeclareMisordered; | ||
315 | 318 | ||
316 | our $NON_ASCII_UTF8 = qr{ | 319 | our $NON_ASCII_UTF8 = qr{ |
317 | [\xC2-\xDF][\x80-\xBF] # non-overlong 2-byte | 320 | [\xC2-\xDF][\x80-\xBF] # non-overlong 2-byte |
@@ -353,16 +356,36 @@ our $signature_tags = qr{(?xi: | |||
353 | Cc: | 356 | Cc: |
354 | )}; | 357 | )}; |
355 | 358 | ||
359 | our @typeListMisordered = ( | ||
360 | qr{char\s+(?:un)?signed}, | ||
361 | qr{int\s+(?:(?:un)?signed\s+)?short\s}, | ||
362 | qr{int\s+short(?:\s+(?:un)?signed)}, | ||
363 | qr{short\s+int(?:\s+(?:un)?signed)}, | ||
364 | qr{(?:un)?signed\s+int\s+short}, | ||
365 | qr{short\s+(?:un)?signed}, | ||
366 | qr{long\s+int\s+(?:un)?signed}, | ||
367 | qr{int\s+long\s+(?:un)?signed}, | ||
368 | qr{long\s+(?:un)?signed\s+int}, | ||
369 | qr{int\s+(?:un)?signed\s+long}, | ||
370 | qr{int\s+(?:un)?signed}, | ||
371 | qr{int\s+long\s+long\s+(?:un)?signed}, | ||
372 | qr{long\s+long\s+int\s+(?:un)?signed}, | ||
373 | qr{long\s+long\s+(?:un)?signed\s+int}, | ||
374 | qr{long\s+long\s+(?:un)?signed}, | ||
375 | qr{long\s+(?:un)?signed}, | ||
376 | ); | ||
377 | |||
356 | our @typeList = ( | 378 | our @typeList = ( |
357 | qr{void}, | 379 | qr{void}, |
358 | qr{(?:unsigned\s+)?char}, | 380 | qr{(?:(?:un)?signed\s+)?char}, |
359 | qr{(?:unsigned\s+)?short}, | 381 | qr{(?:(?:un)?signed\s+)?short\s+int}, |
360 | qr{(?:unsigned\s+)?int}, | 382 | qr{(?:(?:un)?signed\s+)?short}, |
361 | qr{(?:unsigned\s+)?long}, | 383 | qr{(?:(?:un)?signed\s+)?int}, |
362 | qr{(?:unsigned\s+)?long\s+int}, | 384 | qr{(?:(?:un)?signed\s+)?long\s+int}, |
363 | qr{(?:unsigned\s+)?long\s+long}, | 385 | qr{(?:(?:un)?signed\s+)?long\s+long\s+int}, |
364 | qr{(?:unsigned\s+)?long\s+long\s+int}, | 386 | qr{(?:(?:un)?signed\s+)?long\s+long}, |
365 | qr{unsigned}, | 387 | qr{(?:(?:un)?signed\s+)?long}, |
388 | qr{(?:un)?signed}, | ||
366 | qr{float}, | 389 | qr{float}, |
367 | qr{double}, | 390 | qr{double}, |
368 | qr{bool}, | 391 | qr{bool}, |
@@ -372,6 +395,7 @@ our @typeList = ( | |||
372 | qr{${Ident}_t}, | 395 | qr{${Ident}_t}, |
373 | qr{${Ident}_handler}, | 396 | qr{${Ident}_handler}, |
374 | qr{${Ident}_handler_fn}, | 397 | qr{${Ident}_handler_fn}, |
398 | @typeListMisordered, | ||
375 | ); | 399 | ); |
376 | our @typeListWithAttr = ( | 400 | our @typeListWithAttr = ( |
377 | @typeList, | 401 | @typeList, |
@@ -399,11 +423,6 @@ foreach my $entry (@mode_permission_funcs) { | |||
399 | $mode_perms_search .= $entry->[0]; | 423 | $mode_perms_search .= $entry->[0]; |
400 | } | 424 | } |
401 | 425 | ||
402 | our $declaration_macros = qr{(?x: | ||
403 | (?:$Storage\s+)?(?:DECLARE|DEFINE)_[A-Z]+\s*\(| | ||
404 | (?:$Storage\s+)?LIST_HEAD\s*\( | ||
405 | )}; | ||
406 | |||
407 | our $allowed_asm_includes = qr{(?x: | 426 | our $allowed_asm_includes = qr{(?x: |
408 | irq| | 427 | irq| |
409 | memory | 428 | memory |
@@ -413,6 +432,7 @@ our $allowed_asm_includes = qr{(?x: | |||
413 | sub build_types { | 432 | sub build_types { |
414 | my $mods = "(?x: \n" . join("|\n ", @modifierList) . "\n)"; | 433 | my $mods = "(?x: \n" . join("|\n ", @modifierList) . "\n)"; |
415 | my $all = "(?x: \n" . join("|\n ", @typeList) . "\n)"; | 434 | my $all = "(?x: \n" . join("|\n ", @typeList) . "\n)"; |
435 | my $Misordered = "(?x: \n" . join("|\n ", @typeListMisordered) . "\n)"; | ||
416 | my $allWithAttr = "(?x: \n" . join("|\n ", @typeListWithAttr) . "\n)"; | 436 | my $allWithAttr = "(?x: \n" . join("|\n ", @typeListWithAttr) . "\n)"; |
417 | $Modifier = qr{(?:$Attribute|$Sparse|$mods)}; | 437 | $Modifier = qr{(?:$Attribute|$Sparse|$mods)}; |
418 | $NonptrType = qr{ | 438 | $NonptrType = qr{ |
@@ -424,6 +444,13 @@ sub build_types { | |||
424 | ) | 444 | ) |
425 | (?:\s+$Modifier|\s+const)* | 445 | (?:\s+$Modifier|\s+const)* |
426 | }x; | 446 | }x; |
447 | $NonptrTypeMisordered = qr{ | ||
448 | (?:$Modifier\s+|const\s+)* | ||
449 | (?: | ||
450 | (?:${Misordered}\b) | ||
451 | ) | ||
452 | (?:\s+$Modifier|\s+const)* | ||
453 | }x; | ||
427 | $NonptrTypeWithAttr = qr{ | 454 | $NonptrTypeWithAttr = qr{ |
428 | (?:$Modifier\s+|const\s+)* | 455 | (?:$Modifier\s+|const\s+)* |
429 | (?: | 456 | (?: |
@@ -435,10 +462,16 @@ sub build_types { | |||
435 | }x; | 462 | }x; |
436 | $Type = qr{ | 463 | $Type = qr{ |
437 | $NonptrType | 464 | $NonptrType |
438 | (?:(?:\s|\*|\[\])+\s*const|(?:\s|\*|\[\])+|(?:\s*\[\s*\])+)? | 465 | (?:(?:\s|\*|\[\])+\s*const|(?:\s|\*\s*(?:const\s*)?|\[\])+|(?:\s*\[\s*\])+)? |
466 | (?:\s+$Inline|\s+$Modifier)* | ||
467 | }x; | ||
468 | $TypeMisordered = qr{ | ||
469 | $NonptrTypeMisordered | ||
470 | (?:(?:\s|\*|\[\])+\s*const|(?:\s|\*\s*(?:const\s*)?|\[\])+|(?:\s*\[\s*\])+)? | ||
439 | (?:\s+$Inline|\s+$Modifier)* | 471 | (?:\s+$Inline|\s+$Modifier)* |
440 | }x; | 472 | }x; |
441 | $Declare = qr{(?:$Storage\s+(?:$Inline\s+)?)?$Type}; | 473 | $Declare = qr{(?:$Storage\s+(?:$Inline\s+)?)?$Type}; |
474 | $DeclareMisordered = qr{(?:$Storage\s+(?:$Inline\s+)?)?$TypeMisordered}; | ||
442 | } | 475 | } |
443 | build_types(); | 476 | build_types(); |
444 | 477 | ||
@@ -452,6 +485,12 @@ our $balanced_parens = qr/(\((?:[^\(\)]++|(?-1))*\))/; | |||
452 | our $LvalOrFunc = qr{((?:[\&\*]\s*)?$Lval)\s*($balanced_parens{0,1})\s*}; | 485 | our $LvalOrFunc = qr{((?:[\&\*]\s*)?$Lval)\s*($balanced_parens{0,1})\s*}; |
453 | our $FuncArg = qr{$Typecast{0,1}($LvalOrFunc|$Constant)}; | 486 | our $FuncArg = qr{$Typecast{0,1}($LvalOrFunc|$Constant)}; |
454 | 487 | ||
488 | our $declaration_macros = qr{(?x: | ||
489 | (?:$Storage\s+)?(?:[A-Z_][A-Z0-9]*_){0,2}(?:DEFINE|DECLARE)(?:_[A-Z0-9]+){1,2}\s*\(| | ||
490 | (?:$Storage\s+)?LIST_HEAD\s*\(| | ||
491 | (?:$Storage\s+)?${Type}\s+uninitialized_var\s*\( | ||
492 | )}; | ||
493 | |||
455 | sub deparenthesize { | 494 | sub deparenthesize { |
456 | my ($string) = @_; | 495 | my ($string) = @_; |
457 | return "" if (!defined($string)); | 496 | return "" if (!defined($string)); |
@@ -550,11 +589,43 @@ sub seed_camelcase_includes { | |||
550 | } | 589 | } |
551 | } | 590 | } |
552 | 591 | ||
592 | sub git_commit_info { | ||
593 | my ($commit, $id, $desc) = @_; | ||
594 | |||
595 | return ($id, $desc) if ((which("git") eq "") || !(-e ".git")); | ||
596 | |||
597 | my $output = `git log --no-color --format='%H %s' -1 $commit 2>&1`; | ||
598 | $output =~ s/^\s*//gm; | ||
599 | my @lines = split("\n", $output); | ||
600 | |||
601 | if ($lines[0] =~ /^error: short SHA1 $commit is ambiguous\./) { | ||
602 | # Maybe one day convert this block of bash into something that returns | ||
603 | # all matching commit ids, but it's very slow... | ||
604 | # | ||
605 | # echo "checking commits $1..." | ||
606 | # git rev-list --remotes | grep -i "^$1" | | ||
607 | # while read line ; do | ||
608 | # git log --format='%H %s' -1 $line | | ||
609 | # echo "commit $(cut -c 1-12,41-)" | ||
610 | # done | ||
611 | } elsif ($lines[0] =~ /^fatal: ambiguous argument '$commit': unknown revision or path not in the working tree\./) { | ||
612 | } else { | ||
613 | $id = substr($lines[0], 0, 12); | ||
614 | $desc = substr($lines[0], 41); | ||
615 | } | ||
616 | |||
617 | return ($id, $desc); | ||
618 | } | ||
619 | |||
553 | $chk_signoff = 0 if ($file); | 620 | $chk_signoff = 0 if ($file); |
554 | 621 | ||
555 | my @rawlines = (); | 622 | my @rawlines = (); |
556 | my @lines = (); | 623 | my @lines = (); |
557 | my @fixed = (); | 624 | my @fixed = (); |
625 | my @fixed_inserted = (); | ||
626 | my @fixed_deleted = (); | ||
627 | my $fixlinenr = -1; | ||
628 | |||
558 | my $vname; | 629 | my $vname; |
559 | for my $filename (@ARGV) { | 630 | for my $filename (@ARGV) { |
560 | my $FILE; | 631 | my $FILE; |
@@ -583,6 +654,9 @@ for my $filename (@ARGV) { | |||
583 | @rawlines = (); | 654 | @rawlines = (); |
584 | @lines = (); | 655 | @lines = (); |
585 | @fixed = (); | 656 | @fixed = (); |
657 | @fixed_inserted = (); | ||
658 | @fixed_deleted = (); | ||
659 | $fixlinenr = -1; | ||
586 | } | 660 | } |
587 | 661 | ||
588 | exit($exit); | 662 | exit($exit); |
@@ -674,6 +748,18 @@ sub format_email { | |||
674 | return $formatted_email; | 748 | return $formatted_email; |
675 | } | 749 | } |
676 | 750 | ||
751 | sub which { | ||
752 | my ($bin) = @_; | ||
753 | |||
754 | foreach my $path (split(/:/, $ENV{PATH})) { | ||
755 | if (-e "$path/$bin") { | ||
756 | return "$path/$bin"; | ||
757 | } | ||
758 | } | ||
759 | |||
760 | return ""; | ||
761 | } | ||
762 | |||
677 | sub which_conf { | 763 | sub which_conf { |
678 | my ($conf) = @_; | 764 | my ($conf) = @_; |
679 | 765 | ||
@@ -1483,6 +1569,90 @@ sub report_dump { | |||
1483 | our @report; | 1569 | our @report; |
1484 | } | 1570 | } |
1485 | 1571 | ||
1572 | sub fixup_current_range { | ||
1573 | my ($lineRef, $offset, $length) = @_; | ||
1574 | |||
1575 | if ($$lineRef =~ /^\@\@ -\d+,\d+ \+(\d+),(\d+) \@\@/) { | ||
1576 | my $o = $1; | ||
1577 | my $l = $2; | ||
1578 | my $no = $o + $offset; | ||
1579 | my $nl = $l + $length; | ||
1580 | $$lineRef =~ s/\+$o,$l \@\@/\+$no,$nl \@\@/; | ||
1581 | } | ||
1582 | } | ||
1583 | |||
1584 | sub fix_inserted_deleted_lines { | ||
1585 | my ($linesRef, $insertedRef, $deletedRef) = @_; | ||
1586 | |||
1587 | my $range_last_linenr = 0; | ||
1588 | my $delta_offset = 0; | ||
1589 | |||
1590 | my $old_linenr = 0; | ||
1591 | my $new_linenr = 0; | ||
1592 | |||
1593 | my $next_insert = 0; | ||
1594 | my $next_delete = 0; | ||
1595 | |||
1596 | my @lines = (); | ||
1597 | |||
1598 | my $inserted = @{$insertedRef}[$next_insert++]; | ||
1599 | my $deleted = @{$deletedRef}[$next_delete++]; | ||
1600 | |||
1601 | foreach my $old_line (@{$linesRef}) { | ||
1602 | my $save_line = 1; | ||
1603 | my $line = $old_line; #don't modify the array | ||
1604 | if ($line =~ /^(?:\+\+\+\|\-\-\-)\s+\S+/) { #new filename | ||
1605 | $delta_offset = 0; | ||
1606 | } elsif ($line =~ /^\@\@ -\d+,\d+ \+\d+,\d+ \@\@/) { #new hunk | ||
1607 | $range_last_linenr = $new_linenr; | ||
1608 | fixup_current_range(\$line, $delta_offset, 0); | ||
1609 | } | ||
1610 | |||
1611 | while (defined($deleted) && ${$deleted}{'LINENR'} == $old_linenr) { | ||
1612 | $deleted = @{$deletedRef}[$next_delete++]; | ||
1613 | $save_line = 0; | ||
1614 | fixup_current_range(\$lines[$range_last_linenr], $delta_offset--, -1); | ||
1615 | } | ||
1616 | |||
1617 | while (defined($inserted) && ${$inserted}{'LINENR'} == $old_linenr) { | ||
1618 | push(@lines, ${$inserted}{'LINE'}); | ||
1619 | $inserted = @{$insertedRef}[$next_insert++]; | ||
1620 | $new_linenr++; | ||
1621 | fixup_current_range(\$lines[$range_last_linenr], $delta_offset++, 1); | ||
1622 | } | ||
1623 | |||
1624 | if ($save_line) { | ||
1625 | push(@lines, $line); | ||
1626 | $new_linenr++; | ||
1627 | } | ||
1628 | |||
1629 | $old_linenr++; | ||
1630 | } | ||
1631 | |||
1632 | return @lines; | ||
1633 | } | ||
1634 | |||
1635 | sub fix_insert_line { | ||
1636 | my ($linenr, $line) = @_; | ||
1637 | |||
1638 | my $inserted = { | ||
1639 | LINENR => $linenr, | ||
1640 | LINE => $line, | ||
1641 | }; | ||
1642 | push(@fixed_inserted, $inserted); | ||
1643 | } | ||
1644 | |||
1645 | sub fix_delete_line { | ||
1646 | my ($linenr, $line) = @_; | ||
1647 | |||
1648 | my $deleted = { | ||
1649 | LINENR => $linenr, | ||
1650 | LINE => $line, | ||
1651 | }; | ||
1652 | |||
1653 | push(@fixed_deleted, $deleted); | ||
1654 | } | ||
1655 | |||
1486 | sub ERROR { | 1656 | sub ERROR { |
1487 | my ($type, $msg) = @_; | 1657 | my ($type, $msg) = @_; |
1488 | 1658 | ||
@@ -1637,11 +1807,13 @@ sub process { | |||
1637 | my $signoff = 0; | 1807 | my $signoff = 0; |
1638 | my $is_patch = 0; | 1808 | my $is_patch = 0; |
1639 | 1809 | ||
1640 | my $in_header_lines = 1; | 1810 | my $in_header_lines = $file ? 0 : 1; |
1641 | my $in_commit_log = 0; #Scanning lines before patch | 1811 | my $in_commit_log = 0; #Scanning lines before patch |
1642 | 1812 | my $reported_maintainer_file = 0; | |
1643 | my $non_utf8_charset = 0; | 1813 | my $non_utf8_charset = 0; |
1644 | 1814 | ||
1815 | my $last_blank_line = 0; | ||
1816 | |||
1645 | our @report = (); | 1817 | our @report = (); |
1646 | our $cnt_lines = 0; | 1818 | our $cnt_lines = 0; |
1647 | our $cnt_error = 0; | 1819 | our $cnt_error = 0; |
@@ -1759,8 +1931,10 @@ sub process { | |||
1759 | 1931 | ||
1760 | $realcnt = 0; | 1932 | $realcnt = 0; |
1761 | $linenr = 0; | 1933 | $linenr = 0; |
1934 | $fixlinenr = -1; | ||
1762 | foreach my $line (@lines) { | 1935 | foreach my $line (@lines) { |
1763 | $linenr++; | 1936 | $linenr++; |
1937 | $fixlinenr++; | ||
1764 | my $sline = $line; #copy of $line | 1938 | my $sline = $line; #copy of $line |
1765 | $sline =~ s/$;/ /g; #with comments as spaces | 1939 | $sline =~ s/$;/ /g; #with comments as spaces |
1766 | 1940 | ||
@@ -1891,7 +2065,7 @@ sub process { | |||
1891 | if (WARN("BAD_SIGN_OFF", | 2065 | if (WARN("BAD_SIGN_OFF", |
1892 | "Do not use whitespace before $ucfirst_sign_off\n" . $herecurr) && | 2066 | "Do not use whitespace before $ucfirst_sign_off\n" . $herecurr) && |
1893 | $fix) { | 2067 | $fix) { |
1894 | $fixed[$linenr - 1] = | 2068 | $fixed[$fixlinenr] = |
1895 | "$ucfirst_sign_off $email"; | 2069 | "$ucfirst_sign_off $email"; |
1896 | } | 2070 | } |
1897 | } | 2071 | } |
@@ -1899,7 +2073,7 @@ sub process { | |||
1899 | if (WARN("BAD_SIGN_OFF", | 2073 | if (WARN("BAD_SIGN_OFF", |
1900 | "'$ucfirst_sign_off' is the preferred signature form\n" . $herecurr) && | 2074 | "'$ucfirst_sign_off' is the preferred signature form\n" . $herecurr) && |
1901 | $fix) { | 2075 | $fix) { |
1902 | $fixed[$linenr - 1] = | 2076 | $fixed[$fixlinenr] = |
1903 | "$ucfirst_sign_off $email"; | 2077 | "$ucfirst_sign_off $email"; |
1904 | } | 2078 | } |
1905 | 2079 | ||
@@ -1908,7 +2082,7 @@ sub process { | |||
1908 | if (WARN("BAD_SIGN_OFF", | 2082 | if (WARN("BAD_SIGN_OFF", |
1909 | "Use a single space after $ucfirst_sign_off\n" . $herecurr) && | 2083 | "Use a single space after $ucfirst_sign_off\n" . $herecurr) && |
1910 | $fix) { | 2084 | $fix) { |
1911 | $fixed[$linenr - 1] = | 2085 | $fixed[$fixlinenr] = |
1912 | "$ucfirst_sign_off $email"; | 2086 | "$ucfirst_sign_off $email"; |
1913 | } | 2087 | } |
1914 | } | 2088 | } |
@@ -1956,6 +2130,31 @@ sub process { | |||
1956 | "Remove Gerrit Change-Id's before submitting upstream.\n" . $herecurr); | 2130 | "Remove Gerrit Change-Id's before submitting upstream.\n" . $herecurr); |
1957 | } | 2131 | } |
1958 | 2132 | ||
2133 | # Check for improperly formed commit descriptions | ||
2134 | if ($in_commit_log && | ||
2135 | $line =~ /\bcommit\s+[0-9a-f]{5,}/i && | ||
2136 | $line !~ /\b[Cc]ommit [0-9a-f]{12,16} \("/) { | ||
2137 | $line =~ /\b(c)ommit\s+([0-9a-f]{5,})/i; | ||
2138 | my $init_char = $1; | ||
2139 | my $orig_commit = lc($2); | ||
2140 | my $id = '01234567890ab'; | ||
2141 | my $desc = 'commit description'; | ||
2142 | ($id, $desc) = git_commit_info($orig_commit, $id, $desc); | ||
2143 | ERROR("GIT_COMMIT_ID", | ||
2144 | "Please use 12 to 16 chars for the git commit ID like: '${init_char}ommit $id (\"$desc\")'\n" . $herecurr); | ||
2145 | } | ||
2146 | |||
2147 | # Check for added, moved or deleted files | ||
2148 | if (!$reported_maintainer_file && !$in_commit_log && | ||
2149 | ($line =~ /^(?:new|deleted) file mode\s*\d+\s*$/ || | ||
2150 | $line =~ /^rename (?:from|to) [\w\/\.\-]+\s*$/ || | ||
2151 | ($line =~ /\{\s*([\w\/\.\-]*)\s*\=\>\s*([\w\/\.\-]*)\s*\}/ && | ||
2152 | (defined($1) || defined($2))))) { | ||
2153 | $reported_maintainer_file = 1; | ||
2154 | WARN("FILE_PATH_CHANGES", | ||
2155 | "added, moved or deleted file(s), does MAINTAINERS need updating?\n" . $herecurr); | ||
2156 | } | ||
2157 | |||
1959 | # Check for wrappage within a valid hunk of the file | 2158 | # Check for wrappage within a valid hunk of the file |
1960 | if ($realcnt != 0 && $line !~ m{^(?:\+|-| |\\ No newline|$)}) { | 2159 | if ($realcnt != 0 && $line !~ m{^(?:\+|-| |\\ No newline|$)}) { |
1961 | ERROR("CORRUPTED_PATCH", | 2160 | ERROR("CORRUPTED_PATCH", |
@@ -1993,7 +2192,8 @@ sub process { | |||
1993 | # Check if it's the start of a commit log | 2192 | # Check if it's the start of a commit log |
1994 | # (not a header line and we haven't seen the patch filename) | 2193 | # (not a header line and we haven't seen the patch filename) |
1995 | if ($in_header_lines && $realfile =~ /^$/ && | 2194 | if ($in_header_lines && $realfile =~ /^$/ && |
1996 | $rawline !~ /^(commit\b|from\b|[\w-]+:).+$/i) { | 2195 | !($rawline =~ /^\s+\S/ || |
2196 | $rawline =~ /^(commit\b|from\b|[\w-]+:).*$/i)) { | ||
1997 | $in_header_lines = 0; | 2197 | $in_header_lines = 0; |
1998 | $in_commit_log = 1; | 2198 | $in_commit_log = 1; |
1999 | } | 2199 | } |
@@ -2021,14 +2221,14 @@ sub process { | |||
2021 | if (ERROR("DOS_LINE_ENDINGS", | 2221 | if (ERROR("DOS_LINE_ENDINGS", |
2022 | "DOS line endings\n" . $herevet) && | 2222 | "DOS line endings\n" . $herevet) && |
2023 | $fix) { | 2223 | $fix) { |
2024 | $fixed[$linenr - 1] =~ s/[\s\015]+$//; | 2224 | $fixed[$fixlinenr] =~ s/[\s\015]+$//; |
2025 | } | 2225 | } |
2026 | } elsif ($rawline =~ /^\+.*\S\s+$/ || $rawline =~ /^\+\s+$/) { | 2226 | } elsif ($rawline =~ /^\+.*\S\s+$/ || $rawline =~ /^\+\s+$/) { |
2027 | my $herevet = "$here\n" . cat_vet($rawline) . "\n"; | 2227 | my $herevet = "$here\n" . cat_vet($rawline) . "\n"; |
2028 | if (ERROR("TRAILING_WHITESPACE", | 2228 | if (ERROR("TRAILING_WHITESPACE", |
2029 | "trailing whitespace\n" . $herevet) && | 2229 | "trailing whitespace\n" . $herevet) && |
2030 | $fix) { | 2230 | $fix) { |
2031 | $fixed[$linenr - 1] =~ s/\s+$//; | 2231 | $fixed[$fixlinenr] =~ s/\s+$//; |
2032 | } | 2232 | } |
2033 | 2233 | ||
2034 | $rpt_cleaners = 1; | 2234 | $rpt_cleaners = 1; |
@@ -2049,7 +2249,7 @@ sub process { | |||
2049 | # Only applies when adding the entry originally, after that we do not have | 2249 | # Only applies when adding the entry originally, after that we do not have |
2050 | # sufficient context to determine whether it is indeed long enough. | 2250 | # sufficient context to determine whether it is indeed long enough. |
2051 | if ($realfile =~ /Kconfig/ && | 2251 | if ($realfile =~ /Kconfig/ && |
2052 | $line =~ /.\s*config\s+/) { | 2252 | $line =~ /^\+\s*config\s+/) { |
2053 | my $length = 0; | 2253 | my $length = 0; |
2054 | my $cnt = $realcnt; | 2254 | my $cnt = $realcnt; |
2055 | my $ln = $linenr + 1; | 2255 | my $ln = $linenr + 1; |
@@ -2062,10 +2262,11 @@ sub process { | |||
2062 | $is_end = $lines[$ln - 1] =~ /^\+/; | 2262 | $is_end = $lines[$ln - 1] =~ /^\+/; |
2063 | 2263 | ||
2064 | next if ($f =~ /^-/); | 2264 | next if ($f =~ /^-/); |
2265 | last if (!$file && $f =~ /^\@\@/); | ||
2065 | 2266 | ||
2066 | if ($lines[$ln - 1] =~ /.\s*(?:bool|tristate)\s*\"/) { | 2267 | if ($lines[$ln - 1] =~ /^\+\s*(?:bool|tristate)\s*\"/) { |
2067 | $is_start = 1; | 2268 | $is_start = 1; |
2068 | } elsif ($lines[$ln - 1] =~ /.\s*(?:---)?help(?:---)?$/) { | 2269 | } elsif ($lines[$ln - 1] =~ /^\+\s*(?:---)?help(?:---)?$/) { |
2069 | $length = -1; | 2270 | $length = -1; |
2070 | } | 2271 | } |
2071 | 2272 | ||
@@ -2161,12 +2362,18 @@ sub process { | |||
2161 | "quoted string split across lines\n" . $hereprev); | 2362 | "quoted string split across lines\n" . $hereprev); |
2162 | } | 2363 | } |
2163 | 2364 | ||
2365 | # check for missing a space in a string concatination | ||
2366 | if ($prevrawline =~ /[^\\]\w"$/ && $rawline =~ /^\+[\t ]+"\w/) { | ||
2367 | WARN('MISSING_SPACE', | ||
2368 | "break quoted strings at a space character\n" . $hereprev); | ||
2369 | } | ||
2370 | |||
2164 | # check for spaces before a quoted newline | 2371 | # check for spaces before a quoted newline |
2165 | if ($rawline =~ /^.*\".*\s\\n/) { | 2372 | if ($rawline =~ /^.*\".*\s\\n/) { |
2166 | if (WARN("QUOTED_WHITESPACE_BEFORE_NEWLINE", | 2373 | if (WARN("QUOTED_WHITESPACE_BEFORE_NEWLINE", |
2167 | "unnecessary whitespace before a quoted newline\n" . $herecurr) && | 2374 | "unnecessary whitespace before a quoted newline\n" . $herecurr) && |
2168 | $fix) { | 2375 | $fix) { |
2169 | $fixed[$linenr - 1] =~ s/^(\+.*\".*)\s+\\n/$1\\n/; | 2376 | $fixed[$fixlinenr] =~ s/^(\+.*\".*)\s+\\n/$1\\n/; |
2170 | } | 2377 | } |
2171 | 2378 | ||
2172 | } | 2379 | } |
@@ -2203,7 +2410,7 @@ sub process { | |||
2203 | if (ERROR("CODE_INDENT", | 2410 | if (ERROR("CODE_INDENT", |
2204 | "code indent should use tabs where possible\n" . $herevet) && | 2411 | "code indent should use tabs where possible\n" . $herevet) && |
2205 | $fix) { | 2412 | $fix) { |
2206 | $fixed[$linenr - 1] =~ s/^\+([ \t]+)/"\+" . tabify($1)/e; | 2413 | $fixed[$fixlinenr] =~ s/^\+([ \t]+)/"\+" . tabify($1)/e; |
2207 | } | 2414 | } |
2208 | } | 2415 | } |
2209 | 2416 | ||
@@ -2213,9 +2420,9 @@ sub process { | |||
2213 | if (WARN("SPACE_BEFORE_TAB", | 2420 | if (WARN("SPACE_BEFORE_TAB", |
2214 | "please, no space before tabs\n" . $herevet) && | 2421 | "please, no space before tabs\n" . $herevet) && |
2215 | $fix) { | 2422 | $fix) { |
2216 | while ($fixed[$linenr - 1] =~ | 2423 | while ($fixed[$fixlinenr] =~ |
2217 | s/(^\+.*) {8,8}+\t/$1\t\t/) {} | 2424 | s/(^\+.*) {8,8}+\t/$1\t\t/) {} |
2218 | while ($fixed[$linenr - 1] =~ | 2425 | while ($fixed[$fixlinenr] =~ |
2219 | s/(^\+.*) +\t/$1\t/) {} | 2426 | s/(^\+.*) +\t/$1\t/) {} |
2220 | } | 2427 | } |
2221 | } | 2428 | } |
@@ -2249,19 +2456,19 @@ sub process { | |||
2249 | if (CHK("PARENTHESIS_ALIGNMENT", | 2456 | if (CHK("PARENTHESIS_ALIGNMENT", |
2250 | "Alignment should match open parenthesis\n" . $hereprev) && | 2457 | "Alignment should match open parenthesis\n" . $hereprev) && |
2251 | $fix && $line =~ /^\+/) { | 2458 | $fix && $line =~ /^\+/) { |
2252 | $fixed[$linenr - 1] =~ | 2459 | $fixed[$fixlinenr] =~ |
2253 | s/^\+[ \t]*/\+$goodtabindent/; | 2460 | s/^\+[ \t]*/\+$goodtabindent/; |
2254 | } | 2461 | } |
2255 | } | 2462 | } |
2256 | } | 2463 | } |
2257 | } | 2464 | } |
2258 | 2465 | ||
2259 | if ($line =~ /^\+.*\*[ \t]*\)[ \t]+(?!$Assignment|$Arithmetic)/) { | 2466 | if ($line =~ /^\+.*\(\s*$Type\s*\)[ \t]+(?!$Assignment|$Arithmetic|{)/) { |
2260 | if (CHK("SPACING", | 2467 | if (CHK("SPACING", |
2261 | "No space is necessary after a cast\n" . $hereprev) && | 2468 | "No space is necessary after a cast\n" . $herecurr) && |
2262 | $fix) { | 2469 | $fix) { |
2263 | $fixed[$linenr - 1] =~ | 2470 | $fixed[$fixlinenr] =~ |
2264 | s/^(\+.*\*[ \t]*\))[ \t]+/$1/; | 2471 | s/(\(\s*$Type\s*\))[ \t]+/$1/; |
2265 | } | 2472 | } |
2266 | } | 2473 | } |
2267 | 2474 | ||
@@ -2291,10 +2498,44 @@ sub process { | |||
2291 | "networking block comments put the trailing */ on a separate line\n" . $herecurr); | 2498 | "networking block comments put the trailing */ on a separate line\n" . $herecurr); |
2292 | } | 2499 | } |
2293 | 2500 | ||
2501 | # check for missing blank lines after struct/union declarations | ||
2502 | # with exceptions for various attributes and macros | ||
2503 | if ($prevline =~ /^[\+ ]};?\s*$/ && | ||
2504 | $line =~ /^\+/ && | ||
2505 | !($line =~ /^\+\s*$/ || | ||
2506 | $line =~ /^\+\s*EXPORT_SYMBOL/ || | ||
2507 | $line =~ /^\+\s*MODULE_/i || | ||
2508 | $line =~ /^\+\s*\#\s*(?:end|elif|else)/ || | ||
2509 | $line =~ /^\+[a-z_]*init/ || | ||
2510 | $line =~ /^\+\s*(?:static\s+)?[A-Z_]*ATTR/ || | ||
2511 | $line =~ /^\+\s*DECLARE/ || | ||
2512 | $line =~ /^\+\s*__setup/)) { | ||
2513 | if (CHK("LINE_SPACING", | ||
2514 | "Please use a blank line after function/struct/union/enum declarations\n" . $hereprev) && | ||
2515 | $fix) { | ||
2516 | fix_insert_line($fixlinenr, "\+"); | ||
2517 | } | ||
2518 | } | ||
2519 | |||
2520 | # check for multiple consecutive blank lines | ||
2521 | if ($prevline =~ /^[\+ ]\s*$/ && | ||
2522 | $line =~ /^\+\s*$/ && | ||
2523 | $last_blank_line != ($linenr - 1)) { | ||
2524 | if (CHK("LINE_SPACING", | ||
2525 | "Please don't use multiple blank lines\n" . $hereprev) && | ||
2526 | $fix) { | ||
2527 | fix_delete_line($fixlinenr, $rawline); | ||
2528 | } | ||
2529 | |||
2530 | $last_blank_line = $linenr; | ||
2531 | } | ||
2532 | |||
2294 | # check for missing blank lines after declarations | 2533 | # check for missing blank lines after declarations |
2295 | if ($sline =~ /^\+\s+\S/ && #Not at char 1 | 2534 | if ($sline =~ /^\+\s+\S/ && #Not at char 1 |
2296 | # actual declarations | 2535 | # actual declarations |
2297 | ($prevline =~ /^\+\s+$Declare\s*$Ident\s*[=,;:\[]/ || | 2536 | ($prevline =~ /^\+\s+$Declare\s*$Ident\s*[=,;:\[]/ || |
2537 | # function pointer declarations | ||
2538 | $prevline =~ /^\+\s+$Declare\s*\(\s*\*\s*$Ident\s*\)\s*[=,;:\[\(]/ || | ||
2298 | # foo bar; where foo is some local typedef or #define | 2539 | # foo bar; where foo is some local typedef or #define |
2299 | $prevline =~ /^\+\s+$Ident(?:\s+|\s*\*\s*)$Ident\s*[=,;\[]/ || | 2540 | $prevline =~ /^\+\s+$Ident(?:\s+|\s*\*\s*)$Ident\s*[=,;\[]/ || |
2300 | # known declaration macros | 2541 | # known declaration macros |
@@ -2307,6 +2548,8 @@ sub process { | |||
2307 | $prevline =~ /(?:\{\s*|\\)$/) && | 2548 | $prevline =~ /(?:\{\s*|\\)$/) && |
2308 | # looks like a declaration | 2549 | # looks like a declaration |
2309 | !($sline =~ /^\+\s+$Declare\s*$Ident\s*[=,;:\[]/ || | 2550 | !($sline =~ /^\+\s+$Declare\s*$Ident\s*[=,;:\[]/ || |
2551 | # function pointer declarations | ||
2552 | $sline =~ /^\+\s+$Declare\s*\(\s*\*\s*$Ident\s*\)\s*[=,;:\[\(]/ || | ||
2310 | # foo bar; where foo is some local typedef or #define | 2553 | # foo bar; where foo is some local typedef or #define |
2311 | $sline =~ /^\+\s+$Ident(?:\s+|\s*\*\s*)$Ident\s*[=,;\[]/ || | 2554 | $sline =~ /^\+\s+$Ident(?:\s+|\s*\*\s*)$Ident\s*[=,;\[]/ || |
2312 | # known declaration macros | 2555 | # known declaration macros |
@@ -2321,8 +2564,11 @@ sub process { | |||
2321 | $sline =~ /^\+\s+\(?\s*(?:$Compare|$Assignment|$Operators)/) && | 2564 | $sline =~ /^\+\s+\(?\s*(?:$Compare|$Assignment|$Operators)/) && |
2322 | # indentation of previous and current line are the same | 2565 | # indentation of previous and current line are the same |
2323 | (($prevline =~ /\+(\s+)\S/) && $sline =~ /^\+$1\S/)) { | 2566 | (($prevline =~ /\+(\s+)\S/) && $sline =~ /^\+$1\S/)) { |
2324 | WARN("SPACING", | 2567 | if (WARN("LINE_SPACING", |
2325 | "Missing a blank line after declarations\n" . $hereprev); | 2568 | "Missing a blank line after declarations\n" . $hereprev) && |
2569 | $fix) { | ||
2570 | fix_insert_line($fixlinenr, "\+"); | ||
2571 | } | ||
2326 | } | 2572 | } |
2327 | 2573 | ||
2328 | # check for spaces at the beginning of a line. | 2574 | # check for spaces at the beginning of a line. |
@@ -2335,13 +2581,33 @@ sub process { | |||
2335 | if (WARN("LEADING_SPACE", | 2581 | if (WARN("LEADING_SPACE", |
2336 | "please, no spaces at the start of a line\n" . $herevet) && | 2582 | "please, no spaces at the start of a line\n" . $herevet) && |
2337 | $fix) { | 2583 | $fix) { |
2338 | $fixed[$linenr - 1] =~ s/^\+([ \t]+)/"\+" . tabify($1)/e; | 2584 | $fixed[$fixlinenr] =~ s/^\+([ \t]+)/"\+" . tabify($1)/e; |
2339 | } | 2585 | } |
2340 | } | 2586 | } |
2341 | 2587 | ||
2342 | # check we are in a valid C source file if not then ignore this hunk | 2588 | # check we are in a valid C source file if not then ignore this hunk |
2343 | next if ($realfile !~ /\.(h|c)$/); | 2589 | next if ($realfile !~ /\.(h|c)$/); |
2344 | 2590 | ||
2591 | # check indentation of any line with a bare else | ||
2592 | # if the previous line is a break or return and is indented 1 tab more... | ||
2593 | if ($sline =~ /^\+([\t]+)(?:}[ \t]*)?else(?:[ \t]*{)?\s*$/) { | ||
2594 | my $tabs = length($1) + 1; | ||
2595 | if ($prevline =~ /^\+\t{$tabs,$tabs}(?:break|return)\b/) { | ||
2596 | WARN("UNNECESSARY_ELSE", | ||
2597 | "else is not generally useful after a break or return\n" . $hereprev); | ||
2598 | } | ||
2599 | } | ||
2600 | |||
2601 | # check indentation of a line with a break; | ||
2602 | # if the previous line is a goto or return and is indented the same # of tabs | ||
2603 | if ($sline =~ /^\+([\t]+)break\s*;\s*$/) { | ||
2604 | my $tabs = $1; | ||
2605 | if ($prevline =~ /^\+$tabs(?:goto|return)\b/) { | ||
2606 | WARN("UNNECESSARY_BREAK", | ||
2607 | "break is not useful after a goto or return\n" . $hereprev); | ||
2608 | } | ||
2609 | } | ||
2610 | |||
2345 | # discourage the addition of CONFIG_EXPERIMENTAL in #if(def). | 2611 | # discourage the addition of CONFIG_EXPERIMENTAL in #if(def). |
2346 | if ($line =~ /^\+\s*\#\s*if.*\bCONFIG_EXPERIMENTAL\b/) { | 2612 | if ($line =~ /^\+\s*\#\s*if.*\bCONFIG_EXPERIMENTAL\b/) { |
2347 | WARN("CONFIG_EXPERIMENTAL", | 2613 | WARN("CONFIG_EXPERIMENTAL", |
@@ -2477,7 +2743,7 @@ sub process { | |||
2477 | 2743 | ||
2478 | # if/while/etc brace do not go on next line, unless defining a do while loop, | 2744 | # if/while/etc brace do not go on next line, unless defining a do while loop, |
2479 | # or if that brace on the next line is for something else | 2745 | # or if that brace on the next line is for something else |
2480 | if ($line =~ /(.*)\b((?:if|while|for|switch)\s*\(|do\b|else\b)/ && $line !~ /^.\s*\#/) { | 2746 | if ($line =~ /(.*)\b((?:if|while|for|switch|(?:[a-z_]+|)for_each[a-z_]+)\s*\(|do\b|else\b)/ && $line !~ /^.\s*\#/) { |
2481 | my $pre_ctx = "$1$2"; | 2747 | my $pre_ctx = "$1$2"; |
2482 | 2748 | ||
2483 | my ($level, @ctx) = ctx_statement_level($linenr, $realcnt, 0); | 2749 | my ($level, @ctx) = ctx_statement_level($linenr, $realcnt, 0); |
@@ -2504,7 +2770,7 @@ sub process { | |||
2504 | #print "realcnt<$realcnt> ctx_cnt<$ctx_cnt>\n"; | 2770 | #print "realcnt<$realcnt> ctx_cnt<$ctx_cnt>\n"; |
2505 | #print "pre<$pre_ctx>\nline<$line>\nctx<$ctx>\nnext<$lines[$ctx_ln - 1]>\n"; | 2771 | #print "pre<$pre_ctx>\nline<$line>\nctx<$ctx>\nnext<$lines[$ctx_ln - 1]>\n"; |
2506 | 2772 | ||
2507 | if ($ctx !~ /{\s*/ && defined($lines[$ctx_ln -1]) && $lines[$ctx_ln - 1] =~ /^\+\s*{/) { | 2773 | if ($ctx !~ /{\s*/ && defined($lines[$ctx_ln - 1]) && $lines[$ctx_ln - 1] =~ /^\+\s*{/) { |
2508 | ERROR("OPEN_BRACE", | 2774 | ERROR("OPEN_BRACE", |
2509 | "that open brace { should be on the previous line\n" . | 2775 | "that open brace { should be on the previous line\n" . |
2510 | "$here\n$ctx\n$rawlines[$ctx_ln - 1]\n"); | 2776 | "$here\n$ctx\n$rawlines[$ctx_ln - 1]\n"); |
@@ -2523,7 +2789,7 @@ sub process { | |||
2523 | } | 2789 | } |
2524 | 2790 | ||
2525 | # Check relative indent for conditionals and blocks. | 2791 | # Check relative indent for conditionals and blocks. |
2526 | if ($line =~ /\b(?:(?:if|while|for)\s*\(|do\b)/ && $line !~ /^.\s*#/ && $line !~ /\}\s*while\s*/) { | 2792 | if ($line =~ /\b(?:(?:if|while|for|(?:[a-z_]+|)for_each[a-z_]+)\s*\(|do\b)/ && $line !~ /^.\s*#/ && $line !~ /\}\s*while\s*/) { |
2527 | ($stat, $cond, $line_nr_next, $remain_next, $off_next) = | 2793 | ($stat, $cond, $line_nr_next, $remain_next, $off_next) = |
2528 | ctx_statement_block($linenr, $realcnt, 0) | 2794 | ctx_statement_block($linenr, $realcnt, 0) |
2529 | if (!defined $stat); | 2795 | if (!defined $stat); |
@@ -2654,8 +2920,18 @@ sub process { | |||
2654 | # check for initialisation to aggregates open brace on the next line | 2920 | # check for initialisation to aggregates open brace on the next line |
2655 | if ($line =~ /^.\s*{/ && | 2921 | if ($line =~ /^.\s*{/ && |
2656 | $prevline =~ /(?:^|[^=])=\s*$/) { | 2922 | $prevline =~ /(?:^|[^=])=\s*$/) { |
2657 | ERROR("OPEN_BRACE", | 2923 | if (ERROR("OPEN_BRACE", |
2658 | "that open brace { should be on the previous line\n" . $hereprev); | 2924 | "that open brace { should be on the previous line\n" . $hereprev) && |
2925 | $fix && $prevline =~ /^\+/ && $line =~ /^\+/) { | ||
2926 | fix_delete_line($fixlinenr - 1, $prevrawline); | ||
2927 | fix_delete_line($fixlinenr, $rawline); | ||
2928 | my $fixedline = $prevrawline; | ||
2929 | $fixedline =~ s/\s*=\s*$/ = {/; | ||
2930 | fix_insert_line($fixlinenr, $fixedline); | ||
2931 | $fixedline = $line; | ||
2932 | $fixedline =~ s/^(.\s*){\s*/$1/; | ||
2933 | fix_insert_line($fixlinenr, $fixedline); | ||
2934 | } | ||
2659 | } | 2935 | } |
2660 | 2936 | ||
2661 | # | 2937 | # |
@@ -2680,10 +2956,10 @@ sub process { | |||
2680 | if (ERROR("C99_COMMENTS", | 2956 | if (ERROR("C99_COMMENTS", |
2681 | "do not use C99 // comments\n" . $herecurr) && | 2957 | "do not use C99 // comments\n" . $herecurr) && |
2682 | $fix) { | 2958 | $fix) { |
2683 | my $line = $fixed[$linenr - 1]; | 2959 | my $line = $fixed[$fixlinenr]; |
2684 | if ($line =~ /\/\/(.*)$/) { | 2960 | if ($line =~ /\/\/(.*)$/) { |
2685 | my $comment = trim($1); | 2961 | my $comment = trim($1); |
2686 | $fixed[$linenr - 1] =~ s@\/\/(.*)$@/\* $comment \*/@; | 2962 | $fixed[$fixlinenr] =~ s@\/\/(.*)$@/\* $comment \*/@; |
2687 | } | 2963 | } |
2688 | } | 2964 | } |
2689 | } | 2965 | } |
@@ -2742,7 +3018,7 @@ sub process { | |||
2742 | "do not initialise globals to 0 or NULL\n" . | 3018 | "do not initialise globals to 0 or NULL\n" . |
2743 | $herecurr) && | 3019 | $herecurr) && |
2744 | $fix) { | 3020 | $fix) { |
2745 | $fixed[$linenr - 1] =~ s/($Type\s*$Ident\s*(?:\s+$Modifier))*\s*=\s*(0|NULL|false)\s*;/$1;/; | 3021 | $fixed[$fixlinenr] =~ s/($Type\s*$Ident\s*(?:\s+$Modifier))*\s*=\s*(0|NULL|false)\s*;/$1;/; |
2746 | } | 3022 | } |
2747 | } | 3023 | } |
2748 | # check for static initialisers. | 3024 | # check for static initialisers. |
@@ -2751,10 +3027,17 @@ sub process { | |||
2751 | "do not initialise statics to 0 or NULL\n" . | 3027 | "do not initialise statics to 0 or NULL\n" . |
2752 | $herecurr) && | 3028 | $herecurr) && |
2753 | $fix) { | 3029 | $fix) { |
2754 | $fixed[$linenr - 1] =~ s/(\bstatic\s.*?)\s*=\s*(0|NULL|false)\s*;/$1;/; | 3030 | $fixed[$fixlinenr] =~ s/(\bstatic\s.*?)\s*=\s*(0|NULL|false)\s*;/$1;/; |
2755 | } | 3031 | } |
2756 | } | 3032 | } |
2757 | 3033 | ||
3034 | # check for misordered declarations of char/short/int/long with signed/unsigned | ||
3035 | while ($sline =~ m{(\b$TypeMisordered\b)}g) { | ||
3036 | my $tmp = trim($1); | ||
3037 | WARN("MISORDERED_TYPE", | ||
3038 | "type '$tmp' should be specified in [[un]signed] [short|int|long|long long] order\n" . $herecurr); | ||
3039 | } | ||
3040 | |||
2758 | # check for static const char * arrays. | 3041 | # check for static const char * arrays. |
2759 | if ($line =~ /\bstatic\s+const\s+char\s*\*\s*(\w+)\s*\[\s*\]\s*=\s*/) { | 3042 | if ($line =~ /\bstatic\s+const\s+char\s*\*\s*(\w+)\s*\[\s*\]\s*=\s*/) { |
2760 | WARN("STATIC_CONST_CHAR_ARRAY", | 3043 | WARN("STATIC_CONST_CHAR_ARRAY", |
@@ -2781,7 +3064,7 @@ sub process { | |||
2781 | if (ERROR("FUNCTION_WITHOUT_ARGS", | 3064 | if (ERROR("FUNCTION_WITHOUT_ARGS", |
2782 | "Bad function definition - $1() should probably be $1(void)\n" . $herecurr) && | 3065 | "Bad function definition - $1() should probably be $1(void)\n" . $herecurr) && |
2783 | $fix) { | 3066 | $fix) { |
2784 | $fixed[$linenr - 1] =~ s/(\b($Type)\s+($Ident))\s*\(\s*\)/$2 $3(void)/; | 3067 | $fixed[$fixlinenr] =~ s/(\b($Type)\s+($Ident))\s*\(\s*\)/$2 $3(void)/; |
2785 | } | 3068 | } |
2786 | } | 3069 | } |
2787 | 3070 | ||
@@ -2790,7 +3073,7 @@ sub process { | |||
2790 | if (WARN("DEFINE_PCI_DEVICE_TABLE", | 3073 | if (WARN("DEFINE_PCI_DEVICE_TABLE", |
2791 | "Prefer struct pci_device_id over deprecated DEFINE_PCI_DEVICE_TABLE\n" . $herecurr) && | 3074 | "Prefer struct pci_device_id over deprecated DEFINE_PCI_DEVICE_TABLE\n" . $herecurr) && |
2792 | $fix) { | 3075 | $fix) { |
2793 | $fixed[$linenr - 1] =~ s/\b(?:static\s+|)DEFINE_PCI_DEVICE_TABLE\s*\(\s*(\w+)\s*\)\s*=\s*/static const struct pci_device_id $1\[\] = /; | 3076 | $fixed[$fixlinenr] =~ s/\b(?:static\s+|)DEFINE_PCI_DEVICE_TABLE\s*\(\s*(\w+)\s*\)\s*=\s*/static const struct pci_device_id $1\[\] = /; |
2794 | } | 3077 | } |
2795 | } | 3078 | } |
2796 | 3079 | ||
@@ -2827,7 +3110,7 @@ sub process { | |||
2827 | my $sub_from = $ident; | 3110 | my $sub_from = $ident; |
2828 | my $sub_to = $ident; | 3111 | my $sub_to = $ident; |
2829 | $sub_to =~ s/\Q$from\E/$to/; | 3112 | $sub_to =~ s/\Q$from\E/$to/; |
2830 | $fixed[$linenr - 1] =~ | 3113 | $fixed[$fixlinenr] =~ |
2831 | s@\Q$sub_from\E@$sub_to@; | 3114 | s@\Q$sub_from\E@$sub_to@; |
2832 | } | 3115 | } |
2833 | } | 3116 | } |
@@ -2855,7 +3138,7 @@ sub process { | |||
2855 | my $sub_from = $match; | 3138 | my $sub_from = $match; |
2856 | my $sub_to = $match; | 3139 | my $sub_to = $match; |
2857 | $sub_to =~ s/\Q$from\E/$to/; | 3140 | $sub_to =~ s/\Q$from\E/$to/; |
2858 | $fixed[$linenr - 1] =~ | 3141 | $fixed[$fixlinenr] =~ |
2859 | s@\Q$sub_from\E@$sub_to@; | 3142 | s@\Q$sub_from\E@$sub_to@; |
2860 | } | 3143 | } |
2861 | } | 3144 | } |
@@ -2917,7 +3200,7 @@ sub process { | |||
2917 | if (WARN("PREFER_PR_LEVEL", | 3200 | if (WARN("PREFER_PR_LEVEL", |
2918 | "Prefer pr_warn(... to pr_warning(...\n" . $herecurr) && | 3201 | "Prefer pr_warn(... to pr_warning(...\n" . $herecurr) && |
2919 | $fix) { | 3202 | $fix) { |
2920 | $fixed[$linenr - 1] =~ | 3203 | $fixed[$fixlinenr] =~ |
2921 | s/\bpr_warning\b/pr_warn/; | 3204 | s/\bpr_warning\b/pr_warn/; |
2922 | } | 3205 | } |
2923 | } | 3206 | } |
@@ -2933,17 +3216,40 @@ sub process { | |||
2933 | 3216 | ||
2934 | # function brace can't be on same line, except for #defines of do while, | 3217 | # function brace can't be on same line, except for #defines of do while, |
2935 | # or if closed on same line | 3218 | # or if closed on same line |
2936 | if (($line=~/$Type\s*$Ident\(.*\).*\s{/) and | 3219 | if (($line=~/$Type\s*$Ident\(.*\).*\s*{/) and |
2937 | !($line=~/\#\s*define.*do\s{/) and !($line=~/}/)) { | 3220 | !($line=~/\#\s*define.*do\s{/) and !($line=~/}/)) { |
2938 | ERROR("OPEN_BRACE", | 3221 | if (ERROR("OPEN_BRACE", |
2939 | "open brace '{' following function declarations go on the next line\n" . $herecurr); | 3222 | "open brace '{' following function declarations go on the next line\n" . $herecurr) && |
3223 | $fix) { | ||
3224 | fix_delete_line($fixlinenr, $rawline); | ||
3225 | my $fixed_line = $rawline; | ||
3226 | $fixed_line =~ /(^..*$Type\s*$Ident\(.*\)\s*){(.*)$/; | ||
3227 | my $line1 = $1; | ||
3228 | my $line2 = $2; | ||
3229 | fix_insert_line($fixlinenr, ltrim($line1)); | ||
3230 | fix_insert_line($fixlinenr, "\+{"); | ||
3231 | if ($line2 !~ /^\s*$/) { | ||
3232 | fix_insert_line($fixlinenr, "\+\t" . trim($line2)); | ||
3233 | } | ||
3234 | } | ||
2940 | } | 3235 | } |
2941 | 3236 | ||
2942 | # open braces for enum, union and struct go on the same line. | 3237 | # open braces for enum, union and struct go on the same line. |
2943 | if ($line =~ /^.\s*{/ && | 3238 | if ($line =~ /^.\s*{/ && |
2944 | $prevline =~ /^.\s*(?:typedef\s+)?(enum|union|struct)(?:\s+$Ident)?\s*$/) { | 3239 | $prevline =~ /^.\s*(?:typedef\s+)?(enum|union|struct)(?:\s+$Ident)?\s*$/) { |
2945 | ERROR("OPEN_BRACE", | 3240 | if (ERROR("OPEN_BRACE", |
2946 | "open brace '{' following $1 go on the same line\n" . $hereprev); | 3241 | "open brace '{' following $1 go on the same line\n" . $hereprev) && |
3242 | $fix && $prevline =~ /^\+/ && $line =~ /^\+/) { | ||
3243 | fix_delete_line($fixlinenr - 1, $prevrawline); | ||
3244 | fix_delete_line($fixlinenr, $rawline); | ||
3245 | my $fixedline = rtrim($prevrawline) . " {"; | ||
3246 | fix_insert_line($fixlinenr, $fixedline); | ||
3247 | $fixedline = $rawline; | ||
3248 | $fixedline =~ s/^(.\s*){\s*/$1\t/; | ||
3249 | if ($fixedline !~ /^\+\s*$/) { | ||
3250 | fix_insert_line($fixlinenr, $fixedline); | ||
3251 | } | ||
3252 | } | ||
2947 | } | 3253 | } |
2948 | 3254 | ||
2949 | # missing space after union, struct or enum definition | 3255 | # missing space after union, struct or enum definition |
@@ -2951,7 +3257,7 @@ sub process { | |||
2951 | if (WARN("SPACING", | 3257 | if (WARN("SPACING", |
2952 | "missing space after $1 definition\n" . $herecurr) && | 3258 | "missing space after $1 definition\n" . $herecurr) && |
2953 | $fix) { | 3259 | $fix) { |
2954 | $fixed[$linenr - 1] =~ | 3260 | $fixed[$fixlinenr] =~ |
2955 | s/^(.\s*(?:typedef\s+)?(?:enum|union|struct)(?:\s+$Ident){1,2})([=\{])/$1 $2/; | 3261 | s/^(.\s*(?:typedef\s+)?(?:enum|union|struct)(?:\s+$Ident){1,2})([=\{])/$1 $2/; |
2956 | } | 3262 | } |
2957 | } | 3263 | } |
@@ -3021,7 +3327,7 @@ sub process { | |||
3021 | } | 3327 | } |
3022 | 3328 | ||
3023 | if (show_type("SPACING") && $fix) { | 3329 | if (show_type("SPACING") && $fix) { |
3024 | $fixed[$linenr - 1] =~ | 3330 | $fixed[$fixlinenr] =~ |
3025 | s/^(.\s*)$Declare\s*\(\s*\*\s*$Ident\s*\)\s*\(/$1 . $declare . $post_declare_space . '(*' . $funcname . ')('/ex; | 3331 | s/^(.\s*)$Declare\s*\(\s*\*\s*$Ident\s*\)\s*\(/$1 . $declare . $post_declare_space . '(*' . $funcname . ')('/ex; |
3026 | } | 3332 | } |
3027 | } | 3333 | } |
@@ -3038,7 +3344,7 @@ sub process { | |||
3038 | if (ERROR("BRACKET_SPACE", | 3344 | if (ERROR("BRACKET_SPACE", |
3039 | "space prohibited before open square bracket '['\n" . $herecurr) && | 3345 | "space prohibited before open square bracket '['\n" . $herecurr) && |
3040 | $fix) { | 3346 | $fix) { |
3041 | $fixed[$linenr - 1] =~ | 3347 | $fixed[$fixlinenr] =~ |
3042 | s/^(\+.*?)\s+\[/$1\[/; | 3348 | s/^(\+.*?)\s+\[/$1\[/; |
3043 | } | 3349 | } |
3044 | } | 3350 | } |
@@ -3073,7 +3379,7 @@ sub process { | |||
3073 | if (WARN("SPACING", | 3379 | if (WARN("SPACING", |
3074 | "space prohibited between function name and open parenthesis '('\n" . $herecurr) && | 3380 | "space prohibited between function name and open parenthesis '('\n" . $herecurr) && |
3075 | $fix) { | 3381 | $fix) { |
3076 | $fixed[$linenr - 1] =~ | 3382 | $fixed[$fixlinenr] =~ |
3077 | s/\b$name\s+\(/$name\(/; | 3383 | s/\b$name\s+\(/$name\(/; |
3078 | } | 3384 | } |
3079 | } | 3385 | } |
@@ -3341,8 +3647,8 @@ sub process { | |||
3341 | $fixed_line = $fixed_line . $fix_elements[$#elements]; | 3647 | $fixed_line = $fixed_line . $fix_elements[$#elements]; |
3342 | } | 3648 | } |
3343 | 3649 | ||
3344 | if ($fix && $line_fixed && $fixed_line ne $fixed[$linenr - 1]) { | 3650 | if ($fix && $line_fixed && $fixed_line ne $fixed[$fixlinenr]) { |
3345 | $fixed[$linenr - 1] = $fixed_line; | 3651 | $fixed[$fixlinenr] = $fixed_line; |
3346 | } | 3652 | } |
3347 | 3653 | ||
3348 | 3654 | ||
@@ -3353,7 +3659,7 @@ sub process { | |||
3353 | if (WARN("SPACING", | 3659 | if (WARN("SPACING", |
3354 | "space prohibited before semicolon\n" . $herecurr) && | 3660 | "space prohibited before semicolon\n" . $herecurr) && |
3355 | $fix) { | 3661 | $fix) { |
3356 | 1 while $fixed[$linenr - 1] =~ | 3662 | 1 while $fixed[$fixlinenr] =~ |
3357 | s/^(\+.*\S)\s+;/$1;/; | 3663 | s/^(\+.*\S)\s+;/$1;/; |
3358 | } | 3664 | } |
3359 | } | 3665 | } |
@@ -3386,7 +3692,7 @@ sub process { | |||
3386 | if (ERROR("SPACING", | 3692 | if (ERROR("SPACING", |
3387 | "space required before the open brace '{'\n" . $herecurr) && | 3693 | "space required before the open brace '{'\n" . $herecurr) && |
3388 | $fix) { | 3694 | $fix) { |
3389 | $fixed[$linenr - 1] =~ s/^(\+.*(?:do|\))){/$1 {/; | 3695 | $fixed[$fixlinenr] =~ s/^(\+.*(?:do|\))){/$1 {/; |
3390 | } | 3696 | } |
3391 | } | 3697 | } |
3392 | 3698 | ||
@@ -3404,7 +3710,7 @@ sub process { | |||
3404 | if (ERROR("SPACING", | 3710 | if (ERROR("SPACING", |
3405 | "space required after that close brace '}'\n" . $herecurr) && | 3711 | "space required after that close brace '}'\n" . $herecurr) && |
3406 | $fix) { | 3712 | $fix) { |
3407 | $fixed[$linenr - 1] =~ | 3713 | $fixed[$fixlinenr] =~ |
3408 | s/}((?!(?:,|;|\)))\S)/} $1/; | 3714 | s/}((?!(?:,|;|\)))\S)/} $1/; |
3409 | } | 3715 | } |
3410 | } | 3716 | } |
@@ -3414,7 +3720,7 @@ sub process { | |||
3414 | if (ERROR("SPACING", | 3720 | if (ERROR("SPACING", |
3415 | "space prohibited after that open square bracket '['\n" . $herecurr) && | 3721 | "space prohibited after that open square bracket '['\n" . $herecurr) && |
3416 | $fix) { | 3722 | $fix) { |
3417 | $fixed[$linenr - 1] =~ | 3723 | $fixed[$fixlinenr] =~ |
3418 | s/\[\s+/\[/; | 3724 | s/\[\s+/\[/; |
3419 | } | 3725 | } |
3420 | } | 3726 | } |
@@ -3422,7 +3728,7 @@ sub process { | |||
3422 | if (ERROR("SPACING", | 3728 | if (ERROR("SPACING", |
3423 | "space prohibited before that close square bracket ']'\n" . $herecurr) && | 3729 | "space prohibited before that close square bracket ']'\n" . $herecurr) && |
3424 | $fix) { | 3730 | $fix) { |
3425 | $fixed[$linenr - 1] =~ | 3731 | $fixed[$fixlinenr] =~ |
3426 | s/\s+\]/\]/; | 3732 | s/\s+\]/\]/; |
3427 | } | 3733 | } |
3428 | } | 3734 | } |
@@ -3433,7 +3739,7 @@ sub process { | |||
3433 | if (ERROR("SPACING", | 3739 | if (ERROR("SPACING", |
3434 | "space prohibited after that open parenthesis '('\n" . $herecurr) && | 3740 | "space prohibited after that open parenthesis '('\n" . $herecurr) && |
3435 | $fix) { | 3741 | $fix) { |
3436 | $fixed[$linenr - 1] =~ | 3742 | $fixed[$fixlinenr] =~ |
3437 | s/\(\s+/\(/; | 3743 | s/\(\s+/\(/; |
3438 | } | 3744 | } |
3439 | } | 3745 | } |
@@ -3443,18 +3749,27 @@ sub process { | |||
3443 | if (ERROR("SPACING", | 3749 | if (ERROR("SPACING", |
3444 | "space prohibited before that close parenthesis ')'\n" . $herecurr) && | 3750 | "space prohibited before that close parenthesis ')'\n" . $herecurr) && |
3445 | $fix) { | 3751 | $fix) { |
3446 | $fixed[$linenr - 1] =~ | 3752 | print("fixlinenr: <$fixlinenr> fixed[fixlinenr]: <$fixed[$fixlinenr]>\n"); |
3753 | $fixed[$fixlinenr] =~ | ||
3447 | s/\s+\)/\)/; | 3754 | s/\s+\)/\)/; |
3448 | } | 3755 | } |
3449 | } | 3756 | } |
3450 | 3757 | ||
3758 | # check unnecessary parentheses around addressof/dereference single $Lvals | ||
3759 | # ie: &(foo->bar) should be &foo->bar and *(foo->bar) should be *foo->bar | ||
3760 | |||
3761 | while ($line =~ /(?:[^&]&\s*|\*)\(\s*($Ident\s*(?:$Member\s*)+)\s*\)/g) { | ||
3762 | CHK("UNNECESSARY_PARENTHESES", | ||
3763 | "Unnecessary parentheses around $1\n" . $herecurr); | ||
3764 | } | ||
3765 | |||
3451 | #goto labels aren't indented, allow a single space however | 3766 | #goto labels aren't indented, allow a single space however |
3452 | if ($line=~/^.\s+[A-Za-z\d_]+:(?![0-9]+)/ and | 3767 | if ($line=~/^.\s+[A-Za-z\d_]+:(?![0-9]+)/ and |
3453 | !($line=~/^. [A-Za-z\d_]+:/) and !($line=~/^.\s+default:/)) { | 3768 | !($line=~/^. [A-Za-z\d_]+:/) and !($line=~/^.\s+default:/)) { |
3454 | if (WARN("INDENTED_LABEL", | 3769 | if (WARN("INDENTED_LABEL", |
3455 | "labels should not be indented\n" . $herecurr) && | 3770 | "labels should not be indented\n" . $herecurr) && |
3456 | $fix) { | 3771 | $fix) { |
3457 | $fixed[$linenr - 1] =~ | 3772 | $fixed[$fixlinenr] =~ |
3458 | s/^(.)\s+/$1/; | 3773 | s/^(.)\s+/$1/; |
3459 | } | 3774 | } |
3460 | } | 3775 | } |
@@ -3516,7 +3831,7 @@ sub process { | |||
3516 | if (ERROR("SPACING", | 3831 | if (ERROR("SPACING", |
3517 | "space required before the open parenthesis '('\n" . $herecurr) && | 3832 | "space required before the open parenthesis '('\n" . $herecurr) && |
3518 | $fix) { | 3833 | $fix) { |
3519 | $fixed[$linenr - 1] =~ | 3834 | $fixed[$fixlinenr] =~ |
3520 | s/\b(if|while|for|switch)\(/$1 \(/; | 3835 | s/\b(if|while|for|switch)\(/$1 \(/; |
3521 | } | 3836 | } |
3522 | } | 3837 | } |
@@ -3606,7 +3921,7 @@ sub process { | |||
3606 | # if should not continue a brace | 3921 | # if should not continue a brace |
3607 | if ($line =~ /}\s*if\b/) { | 3922 | if ($line =~ /}\s*if\b/) { |
3608 | ERROR("TRAILING_STATEMENTS", | 3923 | ERROR("TRAILING_STATEMENTS", |
3609 | "trailing statements should be on next line\n" . | 3924 | "trailing statements should be on next line (or did you mean 'else if'?)\n" . |
3610 | $herecurr); | 3925 | $herecurr); |
3611 | } | 3926 | } |
3612 | # case and default should not have general statements after them | 3927 | # case and default should not have general statements after them |
@@ -3622,14 +3937,26 @@ sub process { | |||
3622 | 3937 | ||
3623 | # Check for }<nl>else {, these must be at the same | 3938 | # Check for }<nl>else {, these must be at the same |
3624 | # indent level to be relevant to each other. | 3939 | # indent level to be relevant to each other. |
3625 | if ($prevline=~/}\s*$/ and $line=~/^.\s*else\s*/ and | 3940 | if ($prevline=~/}\s*$/ and $line=~/^.\s*else\s*/ && |
3626 | $previndent == $indent) { | 3941 | $previndent == $indent) { |
3627 | ERROR("ELSE_AFTER_BRACE", | 3942 | if (ERROR("ELSE_AFTER_BRACE", |
3628 | "else should follow close brace '}'\n" . $hereprev); | 3943 | "else should follow close brace '}'\n" . $hereprev) && |
3944 | $fix && $prevline =~ /^\+/ && $line =~ /^\+/) { | ||
3945 | fix_delete_line($fixlinenr - 1, $prevrawline); | ||
3946 | fix_delete_line($fixlinenr, $rawline); | ||
3947 | my $fixedline = $prevrawline; | ||
3948 | $fixedline =~ s/}\s*$//; | ||
3949 | if ($fixedline !~ /^\+\s*$/) { | ||
3950 | fix_insert_line($fixlinenr, $fixedline); | ||
3951 | } | ||
3952 | $fixedline = $rawline; | ||
3953 | $fixedline =~ s/^(.\s*)else/$1} else/; | ||
3954 | fix_insert_line($fixlinenr, $fixedline); | ||
3955 | } | ||
3629 | } | 3956 | } |
3630 | 3957 | ||
3631 | if ($prevline=~/}\s*$/ and $line=~/^.\s*while\s*/ and | 3958 | if ($prevline=~/}\s*$/ and $line=~/^.\s*while\s*/ && |
3632 | $previndent == $indent) { | 3959 | $previndent == $indent) { |
3633 | my ($s, $c) = ctx_statement_block($linenr, $realcnt, 0); | 3960 | my ($s, $c) = ctx_statement_block($linenr, $realcnt, 0); |
3634 | 3961 | ||
3635 | # Find out what is on the end of the line after the | 3962 | # Find out what is on the end of the line after the |
@@ -3638,8 +3965,18 @@ sub process { | |||
3638 | $s =~ s/\n.*//g; | 3965 | $s =~ s/\n.*//g; |
3639 | 3966 | ||
3640 | if ($s =~ /^\s*;/) { | 3967 | if ($s =~ /^\s*;/) { |
3641 | ERROR("WHILE_AFTER_BRACE", | 3968 | if (ERROR("WHILE_AFTER_BRACE", |
3642 | "while should follow close brace '}'\n" . $hereprev); | 3969 | "while should follow close brace '}'\n" . $hereprev) && |
3970 | $fix && $prevline =~ /^\+/ && $line =~ /^\+/) { | ||
3971 | fix_delete_line($fixlinenr - 1, $prevrawline); | ||
3972 | fix_delete_line($fixlinenr, $rawline); | ||
3973 | my $fixedline = $prevrawline; | ||
3974 | my $trailing = $rawline; | ||
3975 | $trailing =~ s/^\+//; | ||
3976 | $trailing = trim($trailing); | ||
3977 | $fixedline =~ s/}\s*$/} $trailing/; | ||
3978 | fix_insert_line($fixlinenr, $fixedline); | ||
3979 | } | ||
3643 | } | 3980 | } |
3644 | } | 3981 | } |
3645 | 3982 | ||
@@ -3653,7 +3990,7 @@ sub process { | |||
3653 | "Avoid gcc v4.3+ binary constant extension: <$var>\n" . $herecurr) && | 3990 | "Avoid gcc v4.3+ binary constant extension: <$var>\n" . $herecurr) && |
3654 | $fix) { | 3991 | $fix) { |
3655 | my $hexval = sprintf("0x%x", oct($var)); | 3992 | my $hexval = sprintf("0x%x", oct($var)); |
3656 | $fixed[$linenr - 1] =~ | 3993 | $fixed[$fixlinenr] =~ |
3657 | s/\b$var\b/$hexval/; | 3994 | s/\b$var\b/$hexval/; |
3658 | } | 3995 | } |
3659 | } | 3996 | } |
@@ -3689,7 +4026,7 @@ sub process { | |||
3689 | if (WARN("WHITESPACE_AFTER_LINE_CONTINUATION", | 4026 | if (WARN("WHITESPACE_AFTER_LINE_CONTINUATION", |
3690 | "Whitespace after \\ makes next lines useless\n" . $herecurr) && | 4027 | "Whitespace after \\ makes next lines useless\n" . $herecurr) && |
3691 | $fix) { | 4028 | $fix) { |
3692 | $fixed[$linenr - 1] =~ s/\s+$//; | 4029 | $fixed[$fixlinenr] =~ s/\s+$//; |
3693 | } | 4030 | } |
3694 | } | 4031 | } |
3695 | 4032 | ||
@@ -3762,7 +4099,7 @@ sub process { | |||
3762 | $dstat !~ /^(?:$Ident|-?$Constant),$/ && # 10, // foo(), | 4099 | $dstat !~ /^(?:$Ident|-?$Constant),$/ && # 10, // foo(), |
3763 | $dstat !~ /^(?:$Ident|-?$Constant);$/ && # foo(); | 4100 | $dstat !~ /^(?:$Ident|-?$Constant);$/ && # foo(); |
3764 | $dstat !~ /^[!~-]?(?:$Lval|$Constant)$/ && # 10 // foo() // !foo // ~foo // -foo // foo->bar // foo.bar->baz | 4101 | $dstat !~ /^[!~-]?(?:$Lval|$Constant)$/ && # 10 // foo() // !foo // ~foo // -foo // foo->bar // foo.bar->baz |
3765 | $dstat !~ /^'X'$/ && # character constants | 4102 | $dstat !~ /^'X'$/ && $dstat !~ /^'XX'$/ && # character constants |
3766 | $dstat !~ /$exceptions/ && | 4103 | $dstat !~ /$exceptions/ && |
3767 | $dstat !~ /^\.$Ident\s*=/ && # .foo = | 4104 | $dstat !~ /^\.$Ident\s*=/ && # .foo = |
3768 | $dstat !~ /^(?:\#\s*$Ident|\#\s*$Constant)\s*$/ && # stringification #foo | 4105 | $dstat !~ /^(?:\#\s*$Ident|\#\s*$Constant)\s*$/ && # stringification #foo |
@@ -4014,6 +4351,23 @@ sub process { | |||
4014 | } | 4351 | } |
4015 | } | 4352 | } |
4016 | 4353 | ||
4354 | # check for unnecessary "Out of Memory" messages | ||
4355 | if ($line =~ /^\+.*\b$logFunctions\s*\(/ && | ||
4356 | $prevline =~ /^[ \+]\s*if\s*\(\s*(\!\s*|NULL\s*==\s*)?($Lval)(\s*==\s*NULL\s*)?\s*\)/ && | ||
4357 | (defined $1 || defined $3) && | ||
4358 | $linenr > 3) { | ||
4359 | my $testval = $2; | ||
4360 | my $testline = $lines[$linenr - 3]; | ||
4361 | |||
4362 | my ($s, $c) = ctx_statement_block($linenr - 3, $realcnt, 0); | ||
4363 | # print("line: <$line>\nprevline: <$prevline>\ns: <$s>\nc: <$c>\n\n\n"); | ||
4364 | |||
4365 | if ($c =~ /(?:^|\n)[ \+]\s*(?:$Type\s*)?\Q$testval\E\s*=\s*(?:\([^\)]*\)\s*)?\s*(?:devm_)?(?:[kv][czm]alloc(?:_node|_array)?\b|kstrdup|(?:dev_)?alloc_skb)/) { | ||
4366 | WARN("OOM_MESSAGE", | ||
4367 | "Possible unnecessary 'out of memory' message\n" . $hereprev); | ||
4368 | } | ||
4369 | } | ||
4370 | |||
4017 | # check for bad placement of section $InitAttribute (e.g.: __initdata) | 4371 | # check for bad placement of section $InitAttribute (e.g.: __initdata) |
4018 | if ($line =~ /(\b$InitAttribute\b)/) { | 4372 | if ($line =~ /(\b$InitAttribute\b)/) { |
4019 | my $attr = $1; | 4373 | my $attr = $1; |
@@ -4027,7 +4381,7 @@ sub process { | |||
4027 | WARN("MISPLACED_INIT", | 4381 | WARN("MISPLACED_INIT", |
4028 | "$attr should be placed after $var\n" . $herecurr))) && | 4382 | "$attr should be placed after $var\n" . $herecurr))) && |
4029 | $fix) { | 4383 | $fix) { |
4030 | $fixed[$linenr - 1] =~ s/(\bstatic\s+(?:const\s+)?)(?:$attr\s+)?($NonptrTypeWithAttr)\s+(?:$attr\s+)?($Ident(?:\[[^]]*\])?)\s*([=;])\s*/"$1" . trim(string_find_replace($2, "\\s*$attr\\s*", " ")) . " " . trim(string_find_replace($3, "\\s*$attr\\s*", "")) . " $attr" . ("$4" eq ";" ? ";" : " = ")/e; | 4384 | $fixed[$fixlinenr] =~ s/(\bstatic\s+(?:const\s+)?)(?:$attr\s+)?($NonptrTypeWithAttr)\s+(?:$attr\s+)?($Ident(?:\[[^]]*\])?)\s*([=;])\s*/"$1" . trim(string_find_replace($2, "\\s*$attr\\s*", " ")) . " " . trim(string_find_replace($3, "\\s*$attr\\s*", "")) . " $attr" . ("$4" eq ";" ? ";" : " = ")/e; |
4031 | } | 4385 | } |
4032 | } | 4386 | } |
4033 | } | 4387 | } |
@@ -4041,7 +4395,7 @@ sub process { | |||
4041 | if (ERROR("INIT_ATTRIBUTE", | 4395 | if (ERROR("INIT_ATTRIBUTE", |
4042 | "Use of const init definition must use ${attr_prefix}initconst\n" . $herecurr) && | 4396 | "Use of const init definition must use ${attr_prefix}initconst\n" . $herecurr) && |
4043 | $fix) { | 4397 | $fix) { |
4044 | $fixed[$linenr - 1] =~ | 4398 | $fixed[$fixlinenr] =~ |
4045 | s/$InitAttributeData/${attr_prefix}initconst/; | 4399 | s/$InitAttributeData/${attr_prefix}initconst/; |
4046 | } | 4400 | } |
4047 | } | 4401 | } |
@@ -4052,12 +4406,12 @@ sub process { | |||
4052 | if (ERROR("INIT_ATTRIBUTE", | 4406 | if (ERROR("INIT_ATTRIBUTE", |
4053 | "Use of $attr requires a separate use of const\n" . $herecurr) && | 4407 | "Use of $attr requires a separate use of const\n" . $herecurr) && |
4054 | $fix) { | 4408 | $fix) { |
4055 | my $lead = $fixed[$linenr - 1] =~ | 4409 | my $lead = $fixed[$fixlinenr] =~ |
4056 | /(^\+\s*(?:static\s+))/; | 4410 | /(^\+\s*(?:static\s+))/; |
4057 | $lead = rtrim($1); | 4411 | $lead = rtrim($1); |
4058 | $lead = "$lead " if ($lead !~ /^\+$/); | 4412 | $lead = "$lead " if ($lead !~ /^\+$/); |
4059 | $lead = "${lead}const "; | 4413 | $lead = "${lead}const "; |
4060 | $fixed[$linenr - 1] =~ s/(^\+\s*(?:static\s+))/$lead/; | 4414 | $fixed[$fixlinenr] =~ s/(^\+\s*(?:static\s+))/$lead/; |
4061 | } | 4415 | } |
4062 | } | 4416 | } |
4063 | 4417 | ||
@@ -4070,7 +4424,7 @@ sub process { | |||
4070 | if (WARN("CONSTANT_CONVERSION", | 4424 | if (WARN("CONSTANT_CONVERSION", |
4071 | "$constant_func should be $func\n" . $herecurr) && | 4425 | "$constant_func should be $func\n" . $herecurr) && |
4072 | $fix) { | 4426 | $fix) { |
4073 | $fixed[$linenr - 1] =~ s/\b$constant_func\b/$func/g; | 4427 | $fixed[$fixlinenr] =~ s/\b$constant_func\b/$func/g; |
4074 | } | 4428 | } |
4075 | } | 4429 | } |
4076 | 4430 | ||
@@ -4120,7 +4474,7 @@ sub process { | |||
4120 | if (ERROR("SPACING", | 4474 | if (ERROR("SPACING", |
4121 | "exactly one space required after that #$1\n" . $herecurr) && | 4475 | "exactly one space required after that #$1\n" . $herecurr) && |
4122 | $fix) { | 4476 | $fix) { |
4123 | $fixed[$linenr - 1] =~ | 4477 | $fixed[$fixlinenr] =~ |
4124 | s/^(.\s*\#\s*(ifdef|ifndef|elif))\s{2,}/$1 /; | 4478 | s/^(.\s*\#\s*(ifdef|ifndef|elif))\s{2,}/$1 /; |
4125 | } | 4479 | } |
4126 | 4480 | ||
@@ -4168,7 +4522,7 @@ sub process { | |||
4168 | if (WARN("INLINE", | 4522 | if (WARN("INLINE", |
4169 | "plain inline is preferred over $1\n" . $herecurr) && | 4523 | "plain inline is preferred over $1\n" . $herecurr) && |
4170 | $fix) { | 4524 | $fix) { |
4171 | $fixed[$linenr - 1] =~ s/\b(__inline__|__inline)\b/inline/; | 4525 | $fixed[$fixlinenr] =~ s/\b(__inline__|__inline)\b/inline/; |
4172 | 4526 | ||
4173 | } | 4527 | } |
4174 | } | 4528 | } |
@@ -4193,7 +4547,7 @@ sub process { | |||
4193 | if (WARN("PREFER_PRINTF", | 4547 | if (WARN("PREFER_PRINTF", |
4194 | "__printf(string-index, first-to-check) is preferred over __attribute__((format(printf, string-index, first-to-check)))\n" . $herecurr) && | 4548 | "__printf(string-index, first-to-check) is preferred over __attribute__((format(printf, string-index, first-to-check)))\n" . $herecurr) && |
4195 | $fix) { | 4549 | $fix) { |
4196 | $fixed[$linenr - 1] =~ s/\b__attribute__\s*\(\s*\(\s*format\s*\(\s*printf\s*,\s*(.*)\)\s*\)\s*\)/"__printf(" . trim($1) . ")"/ex; | 4550 | $fixed[$fixlinenr] =~ s/\b__attribute__\s*\(\s*\(\s*format\s*\(\s*printf\s*,\s*(.*)\)\s*\)\s*\)/"__printf(" . trim($1) . ")"/ex; |
4197 | 4551 | ||
4198 | } | 4552 | } |
4199 | } | 4553 | } |
@@ -4204,7 +4558,7 @@ sub process { | |||
4204 | if (WARN("PREFER_SCANF", | 4558 | if (WARN("PREFER_SCANF", |
4205 | "__scanf(string-index, first-to-check) is preferred over __attribute__((format(scanf, string-index, first-to-check)))\n" . $herecurr) && | 4559 | "__scanf(string-index, first-to-check) is preferred over __attribute__((format(scanf, string-index, first-to-check)))\n" . $herecurr) && |
4206 | $fix) { | 4560 | $fix) { |
4207 | $fixed[$linenr - 1] =~ s/\b__attribute__\s*\(\s*\(\s*format\s*\(\s*scanf\s*,\s*(.*)\)\s*\)\s*\)/"__scanf(" . trim($1) . ")"/ex; | 4561 | $fixed[$fixlinenr] =~ s/\b__attribute__\s*\(\s*\(\s*format\s*\(\s*scanf\s*,\s*(.*)\)\s*\)\s*\)/"__scanf(" . trim($1) . ")"/ex; |
4208 | } | 4562 | } |
4209 | } | 4563 | } |
4210 | 4564 | ||
@@ -4219,7 +4573,7 @@ sub process { | |||
4219 | if (WARN("SIZEOF_PARENTHESIS", | 4573 | if (WARN("SIZEOF_PARENTHESIS", |
4220 | "sizeof $1 should be sizeof($1)\n" . $herecurr) && | 4574 | "sizeof $1 should be sizeof($1)\n" . $herecurr) && |
4221 | $fix) { | 4575 | $fix) { |
4222 | $fixed[$linenr - 1] =~ s/\bsizeof\s+((?:\*\s*|)$Lval|$Type(?:\s+$Lval|))/"sizeof(" . trim($1) . ")"/ex; | 4576 | $fixed[$fixlinenr] =~ s/\bsizeof\s+((?:\*\s*|)$Lval|$Type(?:\s+$Lval|))/"sizeof(" . trim($1) . ")"/ex; |
4223 | } | 4577 | } |
4224 | } | 4578 | } |
4225 | 4579 | ||
@@ -4242,7 +4596,7 @@ sub process { | |||
4242 | if (WARN("PREFER_SEQ_PUTS", | 4596 | if (WARN("PREFER_SEQ_PUTS", |
4243 | "Prefer seq_puts to seq_printf\n" . $herecurr) && | 4597 | "Prefer seq_puts to seq_printf\n" . $herecurr) && |
4244 | $fix) { | 4598 | $fix) { |
4245 | $fixed[$linenr - 1] =~ s/\bseq_printf\b/seq_puts/; | 4599 | $fixed[$fixlinenr] =~ s/\bseq_printf\b/seq_puts/; |
4246 | } | 4600 | } |
4247 | } | 4601 | } |
4248 | } | 4602 | } |
@@ -4271,7 +4625,7 @@ sub process { | |||
4271 | if (WARN("PREFER_ETHER_ADDR_COPY", | 4625 | if (WARN("PREFER_ETHER_ADDR_COPY", |
4272 | "Prefer ether_addr_copy() over memcpy() if the Ethernet addresses are __aligned(2)\n" . $herecurr) && | 4626 | "Prefer ether_addr_copy() over memcpy() if the Ethernet addresses are __aligned(2)\n" . $herecurr) && |
4273 | $fix) { | 4627 | $fix) { |
4274 | $fixed[$linenr - 1] =~ s/\bmemcpy\s*\(\s*$FuncArg\s*,\s*$FuncArg\s*\,\s*ETH_ALEN\s*\)/ether_addr_copy($2, $7)/; | 4628 | $fixed[$fixlinenr] =~ s/\bmemcpy\s*\(\s*$FuncArg\s*,\s*$FuncArg\s*\,\s*ETH_ALEN\s*\)/ether_addr_copy($2, $7)/; |
4275 | } | 4629 | } |
4276 | } | 4630 | } |
4277 | 4631 | ||
@@ -4359,7 +4713,7 @@ sub process { | |||
4359 | if (CHK("AVOID_EXTERNS", | 4713 | if (CHK("AVOID_EXTERNS", |
4360 | "extern prototypes should be avoided in .h files\n" . $herecurr) && | 4714 | "extern prototypes should be avoided in .h files\n" . $herecurr) && |
4361 | $fix) { | 4715 | $fix) { |
4362 | $fixed[$linenr - 1] =~ s/(.*)\bextern\b\s*(.*)/$1$2/; | 4716 | $fixed[$fixlinenr] =~ s/(.*)\bextern\b\s*(.*)/$1$2/; |
4363 | } | 4717 | } |
4364 | } | 4718 | } |
4365 | 4719 | ||
@@ -4419,23 +4773,24 @@ sub process { | |||
4419 | 4773 | ||
4420 | # check for k[mz]alloc with multiplies that could be kmalloc_array/kcalloc | 4774 | # check for k[mz]alloc with multiplies that could be kmalloc_array/kcalloc |
4421 | if ($^V && $^V ge 5.10.0 && | 4775 | if ($^V && $^V ge 5.10.0 && |
4422 | $line =~ /\b($Lval)\s*\=\s*(?:$balanced_parens)?\s*(k[mz]alloc)\s*\(\s*($FuncArg)\s*\*\s*($FuncArg)/) { | 4776 | $line =~ /\b($Lval)\s*\=\s*(?:$balanced_parens)?\s*(k[mz]alloc)\s*\(\s*($FuncArg)\s*\*\s*($FuncArg)\s*,/) { |
4423 | my $oldfunc = $3; | 4777 | my $oldfunc = $3; |
4424 | my $a1 = $4; | 4778 | my $a1 = $4; |
4425 | my $a2 = $10; | 4779 | my $a2 = $10; |
4426 | my $newfunc = "kmalloc_array"; | 4780 | my $newfunc = "kmalloc_array"; |
4427 | $newfunc = "kcalloc" if ($oldfunc eq "kzalloc"); | 4781 | $newfunc = "kcalloc" if ($oldfunc eq "kzalloc"); |
4428 | if ($a1 =~ /^sizeof\s*\S/ || $a2 =~ /^sizeof\s*\S/) { | 4782 | my $r1 = $a1; |
4783 | my $r2 = $a2; | ||
4784 | if ($a1 =~ /^sizeof\s*\S/) { | ||
4785 | $r1 = $a2; | ||
4786 | $r2 = $a1; | ||
4787 | } | ||
4788 | if ($r1 !~ /^sizeof\b/ && $r2 =~ /^sizeof\s*\S/ && | ||
4789 | !($r1 =~ /^$Constant$/ || $r1 =~ /^[A-Z_][A-Z0-9_]*$/)) { | ||
4429 | if (WARN("ALLOC_WITH_MULTIPLY", | 4790 | if (WARN("ALLOC_WITH_MULTIPLY", |
4430 | "Prefer $newfunc over $oldfunc with multiply\n" . $herecurr) && | 4791 | "Prefer $newfunc over $oldfunc with multiply\n" . $herecurr) && |
4431 | $fix) { | 4792 | $fix) { |
4432 | my $r1 = $a1; | 4793 | $fixed[$fixlinenr] =~ s/\b($Lval)\s*\=\s*(?:$balanced_parens)?\s*(k[mz]alloc)\s*\(\s*($FuncArg)\s*\*\s*($FuncArg)/$1 . ' = ' . "$newfunc(" . trim($r1) . ', ' . trim($r2)/e; |
4433 | my $r2 = $a2; | ||
4434 | if ($a1 =~ /^sizeof\s*\S/) { | ||
4435 | $r1 = $a2; | ||
4436 | $r2 = $a1; | ||
4437 | } | ||
4438 | $fixed[$linenr - 1] =~ s/\b($Lval)\s*\=\s*(?:$balanced_parens)?\s*(k[mz]alloc)\s*\(\s*($FuncArg)\s*\*\s*($FuncArg)/$1 . ' = ' . "$newfunc(" . trim($r1) . ', ' . trim($r2)/e; | ||
4439 | 4794 | ||
4440 | } | 4795 | } |
4441 | } | 4796 | } |
@@ -4459,17 +4814,17 @@ sub process { | |||
4459 | if (WARN("ONE_SEMICOLON", | 4814 | if (WARN("ONE_SEMICOLON", |
4460 | "Statements terminations use 1 semicolon\n" . $herecurr) && | 4815 | "Statements terminations use 1 semicolon\n" . $herecurr) && |
4461 | $fix) { | 4816 | $fix) { |
4462 | $fixed[$linenr - 1] =~ s/(\s*;\s*){2,}$/;/g; | 4817 | $fixed[$fixlinenr] =~ s/(\s*;\s*){2,}$/;/g; |
4463 | } | 4818 | } |
4464 | } | 4819 | } |
4465 | 4820 | ||
4466 | # check for case / default statements not preceeded by break/fallthrough/switch | 4821 | # check for case / default statements not preceded by break/fallthrough/switch |
4467 | if ($line =~ /^.\s*(?:case\s+(?:$Ident|$Constant)\s*|default):/) { | 4822 | if ($line =~ /^.\s*(?:case\s+(?:$Ident|$Constant)\s*|default):/) { |
4468 | my $has_break = 0; | 4823 | my $has_break = 0; |
4469 | my $has_statement = 0; | 4824 | my $has_statement = 0; |
4470 | my $count = 0; | 4825 | my $count = 0; |
4471 | my $prevline = $linenr; | 4826 | my $prevline = $linenr; |
4472 | while ($prevline > 1 && $count < 3 && !$has_break) { | 4827 | while ($prevline > 1 && ($file || $count < 3) && !$has_break) { |
4473 | $prevline--; | 4828 | $prevline--; |
4474 | my $rline = $rawlines[$prevline - 1]; | 4829 | my $rline = $rawlines[$prevline - 1]; |
4475 | my $fline = $lines[$prevline - 1]; | 4830 | my $fline = $lines[$prevline - 1]; |
@@ -4507,7 +4862,7 @@ sub process { | |||
4507 | if (WARN("USE_FUNC", | 4862 | if (WARN("USE_FUNC", |
4508 | "__func__ should be used instead of gcc specific __FUNCTION__\n" . $herecurr) && | 4863 | "__func__ should be used instead of gcc specific __FUNCTION__\n" . $herecurr) && |
4509 | $fix) { | 4864 | $fix) { |
4510 | $fixed[$linenr - 1] =~ s/\b__FUNCTION__\b/__func__/g; | 4865 | $fixed[$fixlinenr] =~ s/\b__FUNCTION__\b/__func__/g; |
4511 | } | 4866 | } |
4512 | } | 4867 | } |
4513 | 4868 | ||
@@ -4750,12 +5105,16 @@ sub process { | |||
4750 | hash_show_words(\%use_type, "Used"); | 5105 | hash_show_words(\%use_type, "Used"); |
4751 | hash_show_words(\%ignore_type, "Ignored"); | 5106 | hash_show_words(\%ignore_type, "Ignored"); |
4752 | 5107 | ||
4753 | if ($clean == 0 && $fix && "@rawlines" ne "@fixed") { | 5108 | if ($clean == 0 && $fix && |
5109 | ("@rawlines" ne "@fixed" || | ||
5110 | $#fixed_inserted >= 0 || $#fixed_deleted >= 0)) { | ||
4754 | my $newfile = $filename; | 5111 | my $newfile = $filename; |
4755 | $newfile .= ".EXPERIMENTAL-checkpatch-fixes" if (!$fix_inplace); | 5112 | $newfile .= ".EXPERIMENTAL-checkpatch-fixes" if (!$fix_inplace); |
4756 | my $linecount = 0; | 5113 | my $linecount = 0; |
4757 | my $f; | 5114 | my $f; |
4758 | 5115 | ||
5116 | @fixed = fix_inserted_deleted_lines(\@fixed, \@fixed_inserted, \@fixed_deleted); | ||
5117 | |||
4759 | open($f, '>', $newfile) | 5118 | open($f, '>', $newfile) |
4760 | or die "$P: Can't open $newfile for write\n"; | 5119 | or die "$P: Can't open $newfile for write\n"; |
4761 | foreach my $fixed_line (@fixed) { | 5120 | foreach my $fixed_line (@fixed) { |
@@ -4763,7 +5122,7 @@ sub process { | |||
4763 | if ($file) { | 5122 | if ($file) { |
4764 | if ($linecount > 3) { | 5123 | if ($linecount > 3) { |
4765 | $fixed_line =~ s/^\+//; | 5124 | $fixed_line =~ s/^\+//; |
4766 | print $f $fixed_line. "\n"; | 5125 | print $f $fixed_line . "\n"; |
4767 | } | 5126 | } |
4768 | } else { | 5127 | } else { |
4769 | print $f $fixed_line . "\n"; | 5128 | print $f $fixed_line . "\n"; |